1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 static zend_op_array dummy_op_array;
20 static zend_jit_trace_info *zend_jit_traces = NULL;
21 static const void **zend_jit_exit_groups = NULL;
22
23 #define ZEND_JIT_COUNTER_NUM zend_jit_traces[0].root
24 #define ZEND_JIT_TRACE_NUM zend_jit_traces[0].id
25 #define ZEND_JIT_EXIT_NUM zend_jit_traces[0].exit_count
26 #define ZEND_JIT_EXIT_COUNTERS zend_jit_traces[0].exit_counters
27
28 #define ZEND_JIT_TRACE_STOP_DESCRIPTION(name, description) \
29 description,
30
31 static const char * zend_jit_trace_stop_description[] = {
32 ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_DESCRIPTION)
33 };
34
zend_jit_trace_star_desc(uint8_t trace_flags)35 static zend_always_inline const char *zend_jit_trace_star_desc(uint8_t trace_flags)
36 {
37 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
38 return "loop";
39 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
40 return "enter";
41 } else if (trace_flags & ZEND_JIT_TRACE_START_RETURN) {
42 return "return";
43 } else {
44 ZEND_UNREACHABLE();
45 return "???";
46 }
47 }
48
zend_jit_trace_startup(bool reattached)49 static void zend_jit_trace_startup(bool reattached)
50 {
51 if (!reattached) {
52 zend_jit_traces = (zend_jit_trace_info*)zend_shared_alloc(sizeof(zend_jit_trace_info) * JIT_G(max_root_traces));
53 if (!zend_jit_traces) {
54 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not allocate JIT root traces buffer!");
55 }
56 zend_jit_exit_groups = (const void**)zend_shared_alloc(sizeof(void*) * (ZEND_JIT_TRACE_MAX_EXITS/ZEND_JIT_EXIT_POINTS_PER_GROUP));
57 if (!zend_jit_exit_groups) {
58 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not allocate JIT exit groups buffer!");
59 }
60 ZEND_JIT_TRACE_NUM = 1;
61 ZEND_JIT_COUNTER_NUM = 0;
62 ZEND_JIT_EXIT_NUM = 0;
63 ZEND_JIT_EXIT_COUNTERS = 0;
64 ZCSG(jit_traces) = zend_jit_traces;
65 ZCSG(jit_exit_groups) = zend_jit_exit_groups;
66 ZCSG(jit_counters_stopped) = false;
67 } else {
68 zend_jit_traces = ZCSG(jit_traces);
69 if (!zend_jit_traces) {
70 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not obtain JIT traces buffer!");
71 }
72 zend_jit_exit_groups = ZCSG(jit_exit_groups);
73 if (!zend_jit_exit_groups) {
74 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not obtain JIT exit groups buffer!");
75 }
76 }
77
78 memset(&dummy_op_array, 0, sizeof(dummy_op_array));
79 dummy_op_array.fn_flags = ZEND_ACC_DONE_PASS_TWO;
80
81 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
82 if (JIT_G(exit_counters) == NULL) {
83 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not allocate JIT exit counters buffer!");
84 }
85 }
86
zend_jit_trace_allocate_exit_point(uint32_t n)87 static const void *zend_jit_trace_allocate_exit_point(uint32_t n)
88 {
89 const void *group = NULL;
90
91 if (UNEXPECTED(n >= ZEND_JIT_TRACE_MAX_EXITS)) {
92 return NULL;
93 }
94 do {
95 group = zend_jit_trace_allocate_exit_group(ZEND_JIT_EXIT_NUM);
96 if (!group) {
97 return NULL;
98 }
99 zend_jit_exit_groups[ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP] =
100 group;
101 ZEND_JIT_EXIT_NUM += ZEND_JIT_EXIT_POINTS_PER_GROUP;
102 } while (n >= ZEND_JIT_EXIT_NUM);
103 return (const void*)
104 ((const char*)group +
105 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
106 }
107
zend_jit_trace_get_exit_addr(uint32_t n)108 static const void *zend_jit_trace_get_exit_addr(uint32_t n)
109 {
110 if (UNEXPECTED(n >= ZEND_JIT_EXIT_NUM)) {
111 return zend_jit_trace_allocate_exit_point(n);
112 }
113 return (const void*)
114 ((const char*)zend_jit_exit_groups[n / ZEND_JIT_EXIT_POINTS_PER_GROUP] +
115 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
116 }
117
zend_jit_exit_point_by_addr(const void * addr)118 static uint32_t zend_jit_exit_point_by_addr(const void *addr)
119 {
120 uint32_t n = (ZEND_JIT_EXIT_NUM + (ZEND_JIT_EXIT_POINTS_PER_GROUP - 1)) / ZEND_JIT_EXIT_POINTS_PER_GROUP;
121 uint32_t i;
122
123 for (i = 0; i < n; i++) {
124 if ((char*)addr >= (char*)zend_jit_exit_groups[i]
125 && (char*)addr <= (char*)zend_jit_exit_groups[i] + ((ZEND_JIT_EXIT_POINTS_PER_GROUP - 1) * ZEND_JIT_EXIT_POINTS_SPACING)) {
126 return (i * ZEND_JIT_EXIT_POINTS_PER_GROUP) +
127 (((char*)addr - (char*)zend_jit_exit_groups[i]) / ZEND_JIT_EXIT_POINTS_SPACING);
128 }
129 }
130 return (uint32_t)-1;
131 }
132
zend_jit_trace_get_exit_point(const zend_op * to_opline,uint32_t flags)133 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags)
134 {
135 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
136 uint32_t exit_point;
137 const zend_op_array *op_array;
138 uint32_t stack_offset = (uint32_t)-1;
139 uint32_t stack_size;
140 zend_jit_trace_stack *stack = NULL;
141
142 if (delayed_call_chain) {
143 assert(to_opline != NULL); /* CALL and IP share the same register */
144 flags |= ZEND_JIT_EXIT_RESTORE_CALL;
145 }
146 if (JIT_G(current_frame)) {
147 op_array = &JIT_G(current_frame)->func->op_array;
148 stack_size = op_array->last_var + op_array->T;
149 if (stack_size) {
150 stack = JIT_G(current_frame)->stack;
151 do {
152 if (STACK_TYPE(stack, stack_size-1) != IS_UNKNOWN
153 || STACK_MEM_TYPE(stack, stack_size-1) != IS_UNKNOWN
154 || STACK_REF(stack, stack_size-1) != IR_UNUSED
155 ) {
156 break;
157 }
158 stack_size--;
159 } while (stack_size);
160 }
161 } else {
162 op_array = NULL;
163 stack_size = 0;
164 }
165
166 /* Try to reuse exit points */
167 if (to_opline != NULL
168 && !(flags & ZEND_JIT_EXIT_METHOD_CALL)
169 && t->exit_count > 0) {
170 uint32_t i = t->exit_count;
171
172 do {
173 i--;
174 if (stack_size == 0
175 || (t->exit_info[i].stack_size >= stack_size
176 && memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) {
177 if (t->exit_info[i].opline == to_opline
178 && t->exit_info[i].flags == flags
179 && t->exit_info[i].stack_size == stack_size) {
180 return i;
181 }
182 }
183 } while (i > 0);
184 }
185
186 exit_point = t->exit_count;
187 if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) {
188 if (stack_size != 0 && stack_offset == (uint32_t)-1) {
189 stack_offset = t->stack_map_size;
190 t->stack_map_size += stack_size;
191 // TODO: reduce number of reallocations ???
192 t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
193 memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack));
194 }
195 t->exit_count++;
196 t->exit_info[exit_point].opline = to_opline;
197 t->exit_info[exit_point].op_array = op_array;
198 t->exit_info[exit_point].flags = flags;
199 t->exit_info[exit_point].stack_size = stack_size;
200 t->exit_info[exit_point].stack_offset = stack_offset;
201 t->exit_info[exit_point].poly_func_ref = 0;
202 t->exit_info[exit_point].poly_this_ref = 0;
203 t->exit_info[exit_point].poly_func_reg = ZREG_NONE;
204 t->exit_info[exit_point].poly_this_reg = ZREG_NONE;
205 }
206
207 return exit_point;
208 }
209
zend_jit_trace_add_code(const void * start,uint32_t size)210 static void zend_jit_trace_add_code(const void *start, uint32_t size)
211 {
212 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
213
214 t->code_start = start;
215 t->code_size = size;
216 }
217
218 /**
219 * Locate a trace in the #zend_jit_traces array with the specified
220 * #code_start address.
221 *
222 * @return the #zend_jit_traces index or 0 if no such #code_start
223 * address was found
224 */
zend_jit_find_trace(const void * addr)225 static uint32_t zend_jit_find_trace(const void *addr)
226 {
227 uint32_t i;
228
229 for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) {
230 if (zend_jit_traces[i].code_start == addr) {
231 return i;
232 }
233 }
234 return 0;
235 }
236
zend_jit_trace_name(const zend_op_array * op_array,uint32_t lineno)237 static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t lineno)
238 {
239 smart_str buf = {0};
240
241 smart_str_appends(&buf, TRACE_PREFIX);
242 smart_str_append_long(&buf, (zend_long)ZEND_JIT_TRACE_NUM);
243 smart_str_appendc(&buf, '$');
244 if (op_array->function_name) {
245 if (op_array->scope) {
246 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
247 smart_str_appends(&buf, "::");
248 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
249 } else {
250 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
251 }
252 } else if (op_array->filename) {
253 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
254 }
255 smart_str_appendc(&buf, '$');
256 smart_str_append_long(&buf, (zend_long)lineno);
257 smart_str_0(&buf);
258 return buf.s;
259 }
260
zend_jit_trace_may_exit(const zend_op_array * op_array,const zend_op * opline)261 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline)
262 {
263 switch (opline->opcode) {
264 case ZEND_IS_IDENTICAL:
265 case ZEND_IS_NOT_IDENTICAL:
266 case ZEND_IS_EQUAL:
267 case ZEND_IS_NOT_EQUAL:
268 case ZEND_IS_SMALLER:
269 case ZEND_IS_SMALLER_OR_EQUAL:
270 case ZEND_CASE:
271 case ZEND_CASE_STRICT:
272 case ZEND_ISSET_ISEMPTY_CV:
273 case ZEND_ISSET_ISEMPTY_VAR:
274 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
275 case ZEND_ISSET_ISEMPTY_PROP_OBJ:
276 case ZEND_ISSET_ISEMPTY_STATIC_PROP:
277 case ZEND_INSTANCEOF:
278 case ZEND_TYPE_CHECK:
279 case ZEND_DEFINED:
280 case ZEND_IN_ARRAY:
281 case ZEND_ARRAY_KEY_EXISTS:
282 if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) {
283 /* smart branch */
284 return 1;
285 }
286 break;
287 case ZEND_JMPZ:
288 case ZEND_JMPNZ:
289 case ZEND_JMPZ_EX:
290 case ZEND_JMPNZ_EX:
291 case ZEND_JMP_SET:
292 case ZEND_COALESCE:
293 case ZEND_JMP_NULL:
294 case ZEND_FE_RESET_R:
295 case ZEND_FE_RESET_RW:
296 case ZEND_ASSERT_CHECK:
297 case ZEND_FE_FETCH_R:
298 case ZEND_FE_FETCH_RW:
299 case ZEND_SWITCH_LONG:
300 case ZEND_SWITCH_STRING:
301 case ZEND_MATCH:
302 case ZEND_BIND_INIT_STATIC_OR_JMP:
303 case ZEND_JMP_FRAMELESS:
304 /* branch opcodes */
305 return 1;
306 case ZEND_NEW:
307 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
308 /* NEW may skip constructor without arguments */
309 return 1;
310 }
311 break;
312 case ZEND_CATCH:
313 case ZEND_FAST_CALL:
314 case ZEND_FAST_RET:
315 case ZEND_GENERATOR_CREATE:
316 case ZEND_GENERATOR_RETURN:
317 case ZEND_YIELD:
318 case ZEND_YIELD_FROM:
319 case ZEND_INCLUDE_OR_EVAL:
320 case ZEND_MATCH_ERROR:
321 /* unsupported */
322 return 1;
323 case ZEND_DO_FCALL:
324 /* potentially polymorphic call */
325 return 1;
326 #if 0
327 case ZEND_DO_UCALL:
328 case ZEND_DO_FCALL_BY_NAME:
329 /* monomorphic call */
330 // TODO: recompilation may change target ???
331 return 0;
332 #endif
333 case ZEND_RETURN_BY_REF:
334 case ZEND_RETURN:
335 /* return */
336 return !JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame));
337 default:
338 break;
339 }
340 return 0;
341 }
342
zend_jit_trace_type_to_info_ex(uint8_t type,uint32_t info)343 static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(uint8_t type, uint32_t info)
344 {
345 if (type == IS_UNKNOWN) {
346 return info;
347 }
348 ZEND_ASSERT(info & (1 << type));
349 if (type < IS_STRING) {
350 return (1 << type);
351 } else if (type != IS_ARRAY) {
352 return (1 << type) | (info & (MAY_BE_RC1|MAY_BE_RCN));
353 } else {
354 return MAY_BE_ARRAY | (info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
355 }
356 }
357
zend_jit_trace_type_to_info(uint8_t type)358 static zend_always_inline uint32_t zend_jit_trace_type_to_info(uint8_t type)
359 {
360 return zend_jit_trace_type_to_info_ex(type, -1);
361 }
362
zend_jit_var_may_alias(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t var)363 static zend_always_inline zend_ssa_alias_kind zend_jit_var_may_alias(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t var)
364 {
365 if (var >= op_array->last_var) {
366 return NO_ALIAS;
367 } else if ((!op_array->function_name || (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS))) {
368 return SYMTABLE_ALIAS;
369 } else if (ssa->vars) {
370 return ssa->vars[var].alias;
371 } else if (zend_string_equals_literal(op_array->vars[var], "http_response_header")) {
372 return HTTP_RESPONSE_HEADER_ALIAS;
373 }
374 return NO_ALIAS;
375 }
376
zend_jit_trace_add_op_guard(zend_ssa * tssa,int ssa_var,uint8_t op_type)377 static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa *tssa,
378 int ssa_var,
379 uint8_t op_type)
380 {
381 zend_ssa_var_info *info = &tssa->var_info[ssa_var];
382
383 if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op_type)) {
384 if (UNEXPECTED(tssa->vars[ssa_var].alias != NO_ALIAS)) {
385 info->type |= MAY_BE_GUARD;
386 } else {
387 info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op_type, info->type);
388 }
389 }
390 }
391
392 #define ADD_OP_GUARD(_ssa_var, _op_type) do { \
393 if (_ssa_var >= 0 && _op_type != IS_UNKNOWN) { \
394 zend_jit_trace_add_op_guard(tssa, _ssa_var, _op_type); \
395 } \
396 } while (0)
397
398 #define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \
399 if (op_type != IS_UNKNOWN) { \
400 if ((op_info & MAY_BE_GUARD) != 0) { \
401 if (!zend_jit_type_guard(&ctx, opline, _var, op_type)) { \
402 goto jit_failure; \
403 } \
404 if (ssa->vars[_ssa_var].alias != NO_ALIAS) { \
405 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), IS_UNKNOWN, 1); \
406 op_info = zend_jit_trace_type_to_info(op_type); \
407 } else { \
408 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), op_type, 1); \
409 op_info &= ~MAY_BE_GUARD; \
410 ssa->var_info[_ssa_var].type &= op_info; \
411 } \
412 } \
413 } \
414 } while (0)
415
416 #define ADD_OP1_TRACE_GUARD() \
417 ADD_OP_GUARD(tssa->ops[idx].op1_use, op1_type)
418 #define ADD_OP2_TRACE_GUARD() \
419 ADD_OP_GUARD(tssa->ops[idx].op2_use, op2_type)
420 #define ADD_OP1_DATA_TRACE_GUARD() \
421 ADD_OP_GUARD(tssa->ops[idx+1].op1_use, op3_type)
422
423 #define CHECK_OP1_TRACE_TYPE() \
424 CHECK_OP_TRACE_TYPE(opline->op1.var, ssa_op->op1_use, op1_info, op1_type)
425 #define CHECK_OP2_TRACE_TYPE() \
426 CHECK_OP_TRACE_TYPE(opline->op2.var, ssa_op->op2_use, op2_info, op2_type)
427 #define CHECK_OP1_DATA_TRACE_TYPE() \
428 CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type)
429
zend_jit_trace_frame_size(const zend_op_array * op_array)430 static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array)
431 {
432 if (op_array && op_array->type == ZEND_USER_FUNCTION) {
433 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack)));
434 } else if (op_array) {
435 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack)));
436 } else {
437 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack));
438 }
439 }
440
zend_jit_trace_call_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)441 static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
442 {
443 return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array));
444 }
445
zend_jit_trace_ret_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)446 static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
447 {
448 return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array));
449 }
450
zend_jit_trace_send_type(const zend_op * opline,zend_jit_trace_stack_frame * call,uint8_t type)451 static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, uint8_t type)
452 {
453 zend_jit_trace_stack *stack = call->stack;
454 const zend_op_array *op_array = &call->func->op_array;
455 uint32_t arg_num = opline->op2.num;
456
457 if (arg_num > op_array->num_args) {
458 return;
459 }
460 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
461 zend_arg_info *arg_info;
462
463 ZEND_ASSERT(arg_num <= op_array->num_args);
464 arg_info = &op_array->arg_info[arg_num-1];
465
466 if (ZEND_TYPE_IS_SET(arg_info->type)) {
467 if (!(ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
468 return;
469 }
470 }
471 }
472 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
473 }
474
zend_jit_needs_arg_dtor(const zend_function * func,uint32_t arg_num,zend_call_info * call_info)475 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info)
476 {
477 if (func
478 && func->type == ZEND_INTERNAL_FUNCTION
479 && (func->internal_function.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0
480 && arg_num < func->internal_function.num_args) {
481 const zend_internal_arg_info *arg_info = &func->internal_function.arg_info[arg_num];
482
483 if (ZEND_ARG_SEND_MODE(arg_info) == ZEND_SEND_BY_VAL
484 && ZEND_TYPE_IS_SET(arg_info->type)
485 && (ZEND_TYPE_FULL_MASK(arg_info->type) & MAY_BE_ANY) != MAY_BE_ANY) {
486 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
487 && JIT_G(current_frame)
488 && JIT_G(current_frame)->call
489 && JIT_G(current_frame)->call->func) {
490 uint32_t type = STACK_TYPE(JIT_G(current_frame)->call->stack, arg_num);
491
492 if (type != IS_UNKNOWN
493 && type < IS_STRING
494 && ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type)) {
495 return 0;
496 }
497 }
498 if (call_info && arg_num < call_info->num_args && call_info->arg_info[arg_num].opline) {
499 const zend_op *opline = call_info->arg_info[arg_num].opline;
500
501 if (opline->opcode == ZEND_SEND_VAL && opline->op1_type == IS_CONST) {
502 zval *zv = RT_CONSTANT(opline, opline->op1);
503
504 if (!Z_REFCOUNTED_P(zv)) {
505 uint32_t type = Z_TYPE_P(zv);
506
507 // TODO: few functions (e.g. pcntl_exec) modify arrays in-place ???
508 if (type != IS_ARRAY
509 && (ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
510 return 0;
511 }
512 }
513 }
514 }
515 }
516 }
517
518 return 1;
519 }
520
zend_jit_trace_build_ssa(const zend_op_array * op_array,zend_script * script)521 static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script)
522 {
523 zend_jit_op_array_trace_extension *jit_extension;
524 zend_ssa *ssa;
525
526 jit_extension =
527 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
528 jit_extension->func_info.num = 0;
529 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
530 | ZEND_FUNC_JIT_ON_PROF_REQUEST
531 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
532 | ZEND_FUNC_JIT_ON_HOT_TRACE;
533 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
534 ssa = &jit_extension->func_info.ssa;
535
536 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
537 do {
538 if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) {
539 break;
540 }
541
542 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
543 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info);
544 jit_extension->func_info.call_map = zend_build_call_map(&CG(arena), &jit_extension->func_info, op_array);
545 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
546 zend_init_func_return_info(op_array, script, &jit_extension->func_info.return_info);
547 }
548 }
549
550 if (zend_jit_op_array_analyze2(op_array, script, ssa, 0) != SUCCESS) {
551 break;
552 }
553
554 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
555 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa);
556 }
557 return ssa;
558 } while (0);
559 }
560
561 memset(ssa, 0, sizeof(zend_ssa));
562 ssa->cfg.blocks_count = 1;
563
564 if (JIT_G(opt_level) == ZEND_JIT_LEVEL_INLINE) {
565 zend_cfg cfg;
566 void *checkpoint = zend_arena_checkpoint(CG(arena));
567
568 if (zend_jit_build_cfg(op_array, &cfg) == SUCCESS) {
569 ssa->cfg.flags = cfg.flags;
570 } else{
571 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
572 }
573
574 /* TODO: move this to zend_cfg.c ? */
575 if (!op_array->function_name) {
576 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
577 }
578
579 zend_arena_release(&CG(arena), checkpoint);
580 }
581
582 return ssa;
583 }
584
585 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa);
586 static void zend_jit_dump_exit_info(zend_jit_trace_info *t);
587
zend_jit_trace_op_len(const zend_op * opline)588 static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline)
589 {
590 int len;
591
592 switch (opline->opcode) {
593 case ZEND_ASSIGN_DIM:
594 case ZEND_ASSIGN_OBJ:
595 case ZEND_ASSIGN_STATIC_PROP:
596 case ZEND_ASSIGN_DIM_OP:
597 case ZEND_ASSIGN_OBJ_OP:
598 case ZEND_ASSIGN_STATIC_PROP_OP:
599 case ZEND_ASSIGN_OBJ_REF:
600 case ZEND_ASSIGN_STATIC_PROP_REF:
601 case ZEND_FRAMELESS_ICALL_3:
602 return 2; /* OP_DATA */
603 case ZEND_RECV_INIT:
604 len = 1;
605 opline++;
606 while (opline->opcode == ZEND_RECV_INIT) {
607 len++;
608 opline++;
609 }
610 return len;
611 case ZEND_BIND_GLOBAL:
612 len = 1;
613 opline++;
614 while (opline->opcode == ZEND_BIND_GLOBAL) {
615 len++;
616 opline++;
617 }
618 return len;
619 // case ZEND_IS_IDENTICAL:
620 // case ZEND_IS_NOT_IDENTICAL:
621 // case ZEND_IS_EQUAL:
622 // case ZEND_IS_NOT_EQUAL:
623 // case ZEND_IS_SMALLER:
624 // case ZEND_IS_SMALLER_OR_EQUAL:
625 // case ZEND_CASE:
626 // case ZEND_ISSET_ISEMPTY_CV:
627 // case ZEND_ISSET_ISEMPTY_VAR:
628 // case ZEND_ISSET_ISEMPTY_DIM_OBJ:
629 // case ZEND_ISSET_ISEMPTY_PROP_OBJ:
630 // case ZEND_ISSET_ISEMPTY_STATIC_PROP:
631 // case ZEND_INSTANCEOF:
632 // case ZEND_TYPE_CHECK:
633 // case ZEND_DEFINED:
634 // case ZEND_IN_ARRAY:
635 // case ZEND_ARRAY_KEY_EXISTS:
636 default:
637 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
638 return 2; /* JMPZ/JMPNZ */
639 }
640 return 1;
641 }
642 }
643
zend_jit_trace_add_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)644 static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
645 {
646 const zend_op_array *op_array;
647 zend_jit_trace_rec *p;
648 int k, vars_count;
649 zend_bitset use, def;
650 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
651 uint32_t set_size;
652 zend_ssa_phi *prev = NULL;
653 int level = 0;
654 ALLOCA_FLAG(use_heap);
655
656 op_array = trace_buffer->op_array;
657 set_size = zend_bitset_len(op_array->last_var + op_array->T);
658 use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap);
659 memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE);
660 def = use + set_size;
661 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
662 for (;;p++) {
663 if (p->op == ZEND_JIT_TRACE_VM && level == 0) {
664 const zend_op *opline = p->opline;
665 int len;
666
667 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
668 len = zend_jit_trace_op_len(opline);
669 while (len > 1) {
670 opline++;
671 if (opline->opcode != ZEND_OP_DATA) {
672 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
673 }
674 len--;
675 }
676 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
677 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
678 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
679 level++;
680 } else if (p->op == ZEND_JIT_TRACE_BACK) {
681 if (level == 0) {
682 // Phi for recursive calls and returns are not supported yet ???
683 assert(0);
684 } else {
685 level--;
686 }
687 } else if (p->op == ZEND_JIT_TRACE_END) {
688 break;
689 }
690 }
691
692 zend_bitset_intersection(use, def, set_size);
693
694 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
695 vars_count = op_array->last_var;
696 } else {
697 vars_count = op_array->last_var + op_array->T;
698 }
699 for (k = 0; k < vars_count; k++) {
700 if (zend_bitset_in(use, k)) {
701 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
702 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
703 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
704 sizeof(void*) * 2);
705 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
706 phi->sources[0] = STACK_VAR(stack, k);
707 phi->sources[1] = -1;
708 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
709 phi->pi = -1;
710 phi->var = k;
711 phi->ssa_var = ssa_vars_count;
712 SET_STACK_VAR(stack, k, ssa_vars_count);
713 ssa_vars_count++;
714 phi->block = 1;
715 if (prev) {
716 prev->next = phi;
717 } else {
718 tssa->blocks[1].phis = phi;
719 }
720 prev = phi;
721 }
722 }
723
724 free_alloca(use, use_heap);
725
726 return ssa_vars_count;
727 }
728
zend_jit_trace_add_call_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)729 static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
730 {
731 zend_ssa_phi *prev = NULL;
732 const zend_op_array *op_array = trace_buffer->op_array;
733 const zend_op *opline = trace_buffer[1].opline;
734 int count = opline - op_array->opcodes;
735 int i;
736
737 for(i = 0; i < count; i++) {
738 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
739 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
740 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
741 sizeof(void*) * 2);
742 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
743 phi->sources[0] = STACK_VAR(stack, i);
744 phi->sources[1] = -1;
745 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
746 phi->pi = -1;
747 phi->var = i;
748 phi->ssa_var = ssa_vars_count;
749 SET_STACK_VAR(stack, i, ssa_vars_count);
750 ssa_vars_count++;
751 phi->block = 1;
752 if (prev) {
753 prev->next = phi;
754 } else {
755 tssa->blocks[1].phis = phi;
756 }
757 prev = phi;
758 }
759 return ssa_vars_count;
760 }
761
zend_jit_trace_add_ret_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)762 static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
763 {
764 const zend_op *opline = trace_buffer[1].opline - 1;
765 int i;
766
767 if (RETURN_VALUE_USED(opline)) {
768 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
769 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
770 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
771 sizeof(void*) * 2);
772
773 i = EX_VAR_TO_NUM(opline->result.var);
774 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
775 phi->sources[0] = STACK_VAR(stack, i);
776 phi->sources[1] = -1;
777 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
778 phi->pi = -1;
779 phi->var = i;
780 phi->ssa_var = ssa_vars_count;
781 SET_STACK_VAR(stack, i, ssa_vars_count);
782 ssa_vars_count++;
783 phi->block = 1;
784 tssa->blocks[1].phis = phi;
785 }
786 return ssa_vars_count;
787 }
788
zend_jit_trace_is_false_loop(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa)789 static bool zend_jit_trace_is_false_loop(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa)
790 {
791 const zend_op *opline;
792 uint32_t b;
793 zend_basic_block *bb;
794
795 ZEND_ASSERT(tssa->cfg.blocks_count == 2);
796 ZEND_ASSERT(tssa->cfg.blocks[1].len > 0);
797
798 b = ssa->cfg.map[tssa_opcodes[0] - op_array->opcodes];
799 opline = tssa_opcodes[tssa->cfg.blocks[1].len - 1];
800 if (opline >= op_array->opcodes && opline < op_array->opcodes + op_array->last) {
801 bb = ssa->cfg.blocks + ssa->cfg.map[opline - op_array->opcodes];
802 return bb->loop_header != b;
803 } else {
804 return 0;
805 }
806 }
807
zend_jit_trace_copy_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var,const zend_op * opline)808 static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array,
809 const zend_ssa *ssa,
810 const zend_op **tssa_opcodes,
811 zend_ssa *tssa,
812 int ssa_var,
813 const zend_op *opline)
814 {
815 int var, use, def, src;
816 zend_ssa_op *op;
817
818 if (tssa->vars[ssa_var].definition_phi) {
819 uint32_t b = ssa->cfg.map[tssa_opcodes[0] - op_array->opcodes];
820 zend_basic_block *bb = ssa->cfg.blocks + b;
821
822 if ((bb->flags & ZEND_BB_LOOP_HEADER)
823 && !zend_jit_trace_is_false_loop(op_array, ssa, tssa_opcodes, tssa)) {
824 zend_ssa_phi *phi = ssa->blocks[b].phis;
825 zend_ssa_phi *pi = NULL;
826
827 var = tssa->vars[ssa_var].var;
828 while (phi) {
829 if (ssa->vars[phi->ssa_var].var == var) {
830 if (phi->pi >= 0) {
831 pi = phi;
832 } else {
833 src = phi->ssa_var;
834 goto copy_info;
835 }
836 }
837 phi = phi->next;
838 }
839 if (pi) {
840 src = pi->ssa_var;
841 goto copy_info;
842 }
843 #if 0
844 while (bb->idom >= 0) {
845 uint32_t n;
846
847 b = bb->idom;
848 bb = ssa->cfg.blocks + b;
849
850 for (n = bb->len, op = ssa->ops + bb->start + n; n > 0; n--) {
851 op--;
852 if (op->result_def >= 0 && ssa->vars[op->result_def].var == var) {
853 src = op->result_def;
854 goto copy_info;
855 } else if (op->op2_def >= 0 && ssa->vars[op->op2_def].var == var) {
856 src = op->op2_def;
857 goto copy_info;
858 } else if (op->op1_def >= 0 && ssa->vars[op->op1_def].var == var) {
859 src = op->op1_def;
860 goto copy_info;
861 }
862 }
863
864 phi = ssa->blocks[b].phis;
865 zend_ssa_phi *pi = NULL;
866 while (phi) {
867 if (ssa->vars[phi->ssa_var].var == var) {
868 if (phi->pi >= 0) {
869 pi = phi;
870 } else {
871 src = phi->ssa_var;
872 goto copy_info;
873 }
874 }
875 phi = phi->next;
876 }
877 if (pi) {
878 src = pi->ssa_var;
879 goto copy_info;
880 }
881 }
882 #endif
883 }
884 } else if (tssa->vars[ssa_var].definition >= 0) {
885 def = tssa->vars[ssa_var].definition;
886 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
887 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
888 if (tssa->ops[def].op1_def == ssa_var) {
889 src = op->op1_def;
890 } else if (tssa->ops[def].op2_def == ssa_var) {
891 src = op->op2_def;
892 } else if (tssa->ops[def].result_def == ssa_var) {
893 src = op->result_def;
894 } else {
895 assert(0);
896 return 0;
897 }
898 goto copy_info;
899 }
900
901 if (tssa->vars[ssa_var].phi_use_chain) {
902 // TODO: this may be incorrect ???
903 var = tssa->vars[ssa_var].phi_use_chain->ssa_var;
904 } else {
905 var = ssa_var;
906 }
907 use = tssa->vars[var].use_chain;
908 if (use >= 0) {
909 ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last);
910 op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes);
911 if (tssa->ops[use].op1_use == var) {
912 src = op->op1_use;
913 } else if (tssa->ops[use].op2_use == var) {
914 src = op->op2_use;
915 } else if (tssa->ops[use].result_use == var) {
916 src = op->result_use;
917 } else {
918 assert(0);
919 return 0;
920 }
921 if (opline) {
922 /* Try to find a difinition in SSA dominators tree */
923 var = tssa->vars[ssa_var].var;
924 uint32_t op_num = opline - op_array->opcodes;
925 uint32_t b = ssa->cfg.map[op_num];
926 zend_basic_block *bb = ssa->cfg.blocks + b;
927 zend_ssa_phi *pi, *phi;
928
929 while (1) {
930 while (op_num > bb->start) {
931 op_num--;
932 op = ssa->ops + op_num;
933 if (op->result_def >= 0 && ssa->vars[op->result_def].var == var) {
934 src = op->result_def;
935 goto copy_info;
936 } else if (op->op2_def >= 0 && ssa->vars[op->op2_def].var == var) {
937 src = op->op2_def;
938 goto copy_info;
939 } else if (op->op1_def >= 0 && ssa->vars[op->op1_def].var == var) {
940 src = op->op1_def;
941 goto copy_info;
942 }
943 }
944 phi = ssa->blocks[b].phis;
945 pi = NULL;
946 while (phi) {
947 if (ssa->vars[phi->ssa_var].var == var) {
948 if (phi->pi >= 0) {
949 pi = phi;
950 } else {
951 src = phi->ssa_var;
952 goto copy_info;
953 }
954 }
955 phi = phi->next;
956 }
957 if (pi) {
958 src = pi->ssa_var;
959 goto copy_info;
960 }
961 if (bb->idom < 0) {
962 break;
963 }
964 b = bb->idom;
965 bb = ssa->cfg.blocks + b;
966 op_num = bb->start + bb->len;
967 }
968 }
969 goto copy_info;
970 }
971 return 0;
972
973 copy_info:
974 tssa->vars[ssa_var].no_val = ssa->vars[src].no_val;
975 tssa->vars[ssa_var].alias = ssa->vars[src].alias;
976 memcpy(&tssa->var_info[ssa_var], &ssa->var_info[src], sizeof(zend_ssa_var_info));
977 return 1;
978 }
979
zend_jit_trace_propagate_range(const zend_op_array * op_array,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)980 static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
981 {
982 zend_ssa_range tmp;
983 int def = tssa->vars[ssa_var].definition;
984
985 if (tssa->vars[ssa_var].alias == NO_ALIAS
986 && zend_inference_propagate_range(op_array, tssa, tssa_opcodes[def], &tssa->ops[def], ssa_var, &tmp)) {
987 tssa->var_info[ssa_var].range.min = tmp.min;
988 tssa->var_info[ssa_var].range.max = tmp.max;
989 tssa->var_info[ssa_var].range.underflow = tmp.underflow;
990 tssa->var_info[ssa_var].range.overflow = tmp.overflow;
991 tssa->var_info[ssa_var].has_range = 1;
992 }
993 }
994
zend_jit_trace_copy_ssa_var_range(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)995 static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
996 {
997 int def;
998 zend_ssa_op *op;
999 zend_ssa_var_info *info;
1000 unsigned int no_val;
1001 zend_ssa_alias_kind alias;
1002
1003 def = tssa->vars[ssa_var].definition;
1004 if (def >= 0) {
1005 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
1006 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
1007 if (tssa->ops[def].op1_def == ssa_var) {
1008 no_val = ssa->vars[op->op1_def].no_val;
1009 alias = ssa->vars[op->op1_def].alias;
1010 info = ssa->var_info + op->op1_def;
1011 } else if (tssa->ops[def].op2_def == ssa_var) {
1012 no_val = ssa->vars[op->op2_def].no_val;
1013 alias = ssa->vars[op->op2_def].alias;
1014 info = ssa->var_info + op->op2_def;
1015 } else if (tssa->ops[def].result_def == ssa_var) {
1016 no_val = ssa->vars[op->result_def].no_val;
1017 alias = ssa->vars[op->result_def].alias;
1018 info = ssa->var_info + op->result_def;
1019 } else {
1020 assert(0);
1021 return;
1022 }
1023
1024 tssa->vars[ssa_var].no_val = no_val;
1025 tssa->vars[ssa_var].alias = alias;
1026
1027 if (!(info->type & MAY_BE_REF)) {
1028 zend_jit_trace_propagate_range(op_array, tssa_opcodes, tssa, ssa_var);
1029 }
1030
1031 if (info->has_range) {
1032 if (tssa->var_info[ssa_var].has_range) {
1033 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
1034 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
1035 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
1036 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
1037 } else {
1038 tssa->var_info[ssa_var].has_range = 1;
1039 tssa->var_info[ssa_var].range = info->range;
1040 }
1041 }
1042 }
1043 }
1044
zend_jit_trace_restrict_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)1045 static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
1046 {
1047 int def;
1048 zend_ssa_op *op;
1049 zend_ssa_var_info *info;
1050
1051 def = tssa->vars[ssa_var].definition;
1052 if (def >= 0) {
1053 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
1054 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
1055 if (tssa->ops[def].op1_def == ssa_var) {
1056 info = ssa->var_info + op->op1_def;
1057 } else if (tssa->ops[def].op2_def == ssa_var) {
1058 info = ssa->var_info + op->op2_def;
1059 } else if (tssa->ops[def].result_def == ssa_var) {
1060 info = ssa->var_info + op->result_def;
1061 } else {
1062 assert(0);
1063 return 0;
1064 }
1065 tssa->var_info[ssa_var].type &= info->type;
1066 if (info->ce) {
1067 if (tssa->var_info[ssa_var].ce) {
1068 if (tssa->var_info[ssa_var].ce != info->ce) {
1069 if (instanceof_function(tssa->var_info[ssa_var].ce, info->ce)) {
1070 /* everything fine */
1071 } else if (instanceof_function(info->ce, tssa->var_info[ssa_var].ce)) {
1072 // TODO: TSSA may miss Pi() functions and corresponding instanceof() constraints ???
1073 } else {
1074 // TODO: classes may implement the same interface ???
1075 //ZEND_UNREACHABLE();
1076 }
1077 }
1078 tssa->var_info[ssa_var].is_instanceof =
1079 tssa->var_info[ssa_var].is_instanceof && info->is_instanceof;
1080 } else {
1081 tssa->var_info[ssa_var].ce = info->ce;
1082 tssa->var_info[ssa_var].is_instanceof = info->is_instanceof;
1083 }
1084 }
1085 if (info->has_range) {
1086 if (tssa->var_info[ssa_var].has_range) {
1087 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
1088 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
1089 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
1090 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
1091 } else {
1092 tssa->var_info[ssa_var].has_range = 1;
1093 tssa->var_info[ssa_var].range = info->range;
1094 }
1095 }
1096 return 1;
1097 }
1098 return 0;
1099 }
1100
find_return_ssa_var(zend_jit_trace_rec * p,zend_ssa_op * ssa_op)1101 static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op)
1102 {
1103 while (1) {
1104 if (p->op == ZEND_JIT_TRACE_VM) {
1105 if (p->opline->opcode == ZEND_DO_UCALL
1106 || p->opline->opcode == ZEND_DO_FCALL_BY_NAME
1107 || p->opline->opcode == ZEND_DO_FCALL) {
1108 if (p->opline->result_type != IS_UNUSED) {
1109 return ssa_op->result_def;
1110 }
1111 }
1112 return -1;
1113 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
1114 /*skip */
1115 } else {
1116 return -1;
1117 }
1118 p--;
1119 }
1120 }
1121
zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec * p,const zend_op_array * op_array)1122 static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, const zend_op_array *op_array)
1123 {
1124 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1125 p--;
1126 while (1) {
1127 if (p->op == ZEND_JIT_TRACE_VM) {
1128 if (p->opline->opcode == ZEND_INIT_FCALL
1129 || p->opline->opcode == ZEND_INIT_FCALL_BY_NAME
1130 || p->opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME
1131 || p->opline->opcode == ZEND_INIT_DYNAMIC_CALL
1132 || p->opline->opcode == ZEND_INIT_USER_CALL
1133 || p->opline->opcode == ZEND_NEW
1134 || p->opline->opcode == ZEND_INIT_METHOD_CALL
1135 || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL
1136 || p->opline->opcode == ZEND_INIT_PARENT_PROPERTY_HOOK_CALL) {
1137 return p->opline;
1138 }
1139 return NULL;
1140 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
1141 /*skip */
1142 } else {
1143 return NULL;
1144 }
1145 p--;
1146 }
1147 } else {
1148 const zend_op *opline = NULL;
1149 int call_level = 0;
1150
1151 p++;
1152 while (1) {
1153 if (p->op == ZEND_JIT_TRACE_VM) {
1154 opline = p->opline;
1155 break;
1156 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1157 call_level++;
1158 /*skip */
1159 } else {
1160 return NULL;
1161 }
1162 p--;
1163 }
1164 if (opline) {
1165 while (opline > op_array->opcodes) {
1166 opline--;
1167 switch (opline->opcode) {
1168 case ZEND_INIT_FCALL:
1169 case ZEND_INIT_FCALL_BY_NAME:
1170 case ZEND_INIT_NS_FCALL_BY_NAME:
1171 case ZEND_INIT_METHOD_CALL:
1172 case ZEND_INIT_DYNAMIC_CALL:
1173 case ZEND_INIT_STATIC_METHOD_CALL:
1174 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
1175 case ZEND_INIT_USER_CALL:
1176 case ZEND_NEW:
1177 if (call_level == 0) {
1178 return opline;
1179 }
1180 call_level--;
1181 break;
1182 case ZEND_DO_FCALL:
1183 case ZEND_DO_ICALL:
1184 case ZEND_DO_UCALL:
1185 case ZEND_DO_FCALL_BY_NAME:
1186 case ZEND_CALLABLE_CONVERT:
1187 call_level++;
1188 break;
1189 }
1190 }
1191 }
1192 }
1193 return NULL;
1194 }
1195
is_checked_guard(const zend_ssa * tssa,const zend_op ** ssa_opcodes,uint32_t var,uint32_t phi_var)1196 static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var)
1197 {
1198 if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG
1199 && !(tssa->var_info[var].type & MAY_BE_REF)) {
1200 int idx = tssa->vars[var].definition;
1201
1202 if (idx >= 0) {
1203 if (tssa->ops[idx].op1_def == var) {
1204 const zend_op *opline = ssa_opcodes[idx];
1205 if (opline->opcode == ZEND_PRE_DEC
1206 || opline->opcode == ZEND_PRE_INC
1207 || opline->opcode == ZEND_POST_DEC
1208 || opline->opcode == ZEND_POST_INC) {
1209 if (tssa->ops[idx].op1_use >= 0
1210 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_STRING)) {
1211 return 0;
1212 }
1213 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1214 return 0;
1215 }
1216 return 1;
1217 } else if (opline->opcode == ZEND_ASSIGN_OP
1218 && (opline->extended_value == ZEND_ADD
1219 || opline->extended_value == ZEND_SUB
1220 || opline->extended_value == ZEND_MUL)) {
1221 if ((opline->op2_type & (IS_VAR|IS_CV))
1222 && tssa->ops[idx].op2_use >= 0
1223 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1224 return 0;
1225 }
1226 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1227 return 0;
1228 }
1229 if (opline->op2_type == IS_CONST) {
1230 zval *zv = RT_CONSTANT(opline, opline->op2);
1231 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1232 return 0;
1233 }
1234 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1235 return 0;
1236 }
1237 return 1;
1238 }
1239 }
1240 if (tssa->ops[idx].result_def == var) {
1241 const zend_op *opline = ssa_opcodes[idx];
1242 if (opline->opcode == ZEND_ADD
1243 || opline->opcode == ZEND_SUB
1244 || opline->opcode == ZEND_MUL) {
1245 if ((opline->op1_type & (IS_VAR|IS_CV))
1246 && tssa->ops[idx].op1_use >= 0
1247 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1248 return 0;
1249 }
1250 if ((opline->op2_type & (IS_VAR|IS_CV))
1251 && tssa->ops[idx].op2_use >= 0
1252 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1253 return 0;
1254 }
1255 if (opline->op1_type == IS_CONST) {
1256 zval *zv = RT_CONSTANT(opline, opline->op1);
1257 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1258 return 0;
1259 }
1260 } else if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1261 return 0;
1262 }
1263 if (opline->op2_type == IS_CONST) {
1264 zval *zv = RT_CONSTANT(opline, opline->op2);
1265 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1266 return 0;
1267 }
1268 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1269 return 0;
1270 }
1271 return 1;
1272 } else if (opline->opcode == ZEND_PRE_DEC
1273 || opline->opcode == ZEND_PRE_INC
1274 || opline->opcode == ZEND_POST_DEC
1275 || opline->opcode == ZEND_POST_INC) {
1276 if ((opline->op1_type & (IS_VAR|IS_CV))
1277 && tssa->ops[idx].op1_use >= 0
1278 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1279 return 0;
1280 }
1281 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1282 return 0;
1283 }
1284 return 1;
1285 }
1286 }
1287 }
1288 }
1289 return 0;
1290 }
1291
1292 typedef struct _zend_tssa {
1293 zend_ssa ssa;
1294 const zend_op **tssa_opcodes;
1295 int used_stack;
1296 } zend_tssa;
1297
1298 static const zend_op _nop_opcode = {0};
1299
zend_jit_trace_build_tssa(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num,zend_script * script,const zend_op_array ** op_arrays,int * num_op_arrays_ptr)1300 static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr)
1301 {
1302 zend_ssa *tssa;
1303 zend_ssa_op *ssa_ops, *op;
1304 zend_ssa_var *ssa_vars;
1305 zend_ssa_var_info *ssa_var_info;
1306 const zend_op_array *op_array;
1307 const zend_op *opline;
1308 const zend_op **ssa_opcodes;
1309 zend_jit_trace_rec *p;
1310 int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count;
1311 zend_jit_trace_stack *stack;
1312 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
1313 uint32_t optimization_level = 0;
1314 int call_level, level, num_op_arrays, used_stack, max_used_stack;
1315 size_t frame_size, stack_top, stack_size, stack_bottom;
1316 zend_jit_op_array_trace_extension *jit_extension;
1317 zend_ssa *ssa;
1318 zend_jit_trace_stack_frame *frame, *top, *call;
1319 zend_ssa_var_info return_value_info;
1320
1321 /* 1. Count number of TSSA opcodes;
1322 * Count number of activation frames;
1323 * Calculate size of abstract stack;
1324 * Construct regular SSA for involved op_array */
1325 op_array = trace_buffer->op_array;
1326 stack_top = stack_size = zend_jit_trace_frame_size(op_array);
1327 stack_bottom = 0;
1328 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1329 ssa_ops_count = 0;
1330 call_level = 0;
1331 level = 0;
1332 num_op_arrays = 0;
1333 /* Remember op_array to cleanup */
1334 op_arrays[num_op_arrays++] = op_array;
1335 /* Build SSA */
1336 ssa = zend_jit_trace_build_ssa(op_array, script);
1337 for (;;p++) {
1338 if (p->op == ZEND_JIT_TRACE_VM) {
1339 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1340 const zend_op *opline = p->opline;
1341
1342 switch (opline->opcode) {
1343 case ZEND_INCLUDE_OR_EVAL:
1344 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1345 break;
1346 case ZEND_FETCH_R:
1347 case ZEND_FETCH_W:
1348 case ZEND_FETCH_RW:
1349 case ZEND_FETCH_FUNC_ARG:
1350 case ZEND_FETCH_IS:
1351 case ZEND_FETCH_UNSET:
1352 case ZEND_UNSET_VAR:
1353 case ZEND_ISSET_ISEMPTY_VAR:
1354 if (opline->extended_value & ZEND_FETCH_LOCAL) {
1355 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1356 } else if ((opline->extended_value & (ZEND_FETCH_GLOBAL | ZEND_FETCH_GLOBAL_LOCK)) &&
1357 !op_array->function_name) {
1358 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1359 }
1360 break;
1361 }
1362 }
1363 ssa_ops_count += zend_jit_trace_op_len(p->opline);
1364 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1365 call_level++;
1366 stack_top += zend_jit_trace_frame_size(p->op_array);
1367 if (stack_top > stack_size) {
1368 stack_size = stack_top;
1369 }
1370 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
1371 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1372 if (p->func
1373 && p->func != (zend_function*)&zend_pass_function
1374 && (zend_string_equals_literal(p->func->common.function_name, "extract")
1375 || zend_string_equals_literal(p->func->common.function_name, "compact")
1376 || zend_string_equals_literal(p->func->common.function_name, "get_defined_vars"))) {
1377 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1378 }
1379 }
1380 frame_size = zend_jit_trace_frame_size(p->op_array);
1381 if (call_level == 0) {
1382 if (stack_top + frame_size > stack_size) {
1383 stack_size = stack_top + frame_size;
1384 }
1385 } else {
1386 call_level--;
1387 stack_top -= frame_size;
1388 }
1389 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1390 op_array = p->op_array;
1391 if (call_level == 0) {
1392 stack_top += zend_jit_trace_frame_size(op_array);
1393 if (stack_top > stack_size) {
1394 stack_size = stack_top;
1395 }
1396 } else {
1397 call_level--;
1398 }
1399 level++;
1400 jit_extension =
1401 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1402 ssa = &jit_extension->func_info.ssa;
1403 if (ssa->cfg.blocks_count) {
1404 /* pass */
1405 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1406 /* Too many functions in single trace */
1407 *num_op_arrays_ptr = num_op_arrays;
1408 return NULL;
1409 } else {
1410 /* Remember op_array to cleanup */
1411 op_arrays[num_op_arrays++] = op_array;
1412 /* Build SSA */
1413 ssa = zend_jit_trace_build_ssa(op_array, script);
1414 }
1415 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1416 if (level == 0) {
1417 stack_bottom += zend_jit_trace_frame_size(p->op_array);
1418 jit_extension =
1419 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1420 ssa = &jit_extension->func_info.ssa;
1421 if (ssa->cfg.blocks_count) {
1422 /* pass */
1423 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1424 /* Too many functions in single trace */
1425 *num_op_arrays_ptr = num_op_arrays;
1426 return NULL;
1427 } else {
1428 /* Remember op_array to cleanup */
1429 op_arrays[num_op_arrays++] = op_array;
1430 /* Build SSA */
1431 ssa = zend_jit_trace_build_ssa(op_array, script);
1432 }
1433 } else {
1434 stack_top -= zend_jit_trace_frame_size(op_array);
1435 level--;
1436 }
1437 op_array = p->op_array;
1438 } else if (p->op == ZEND_JIT_TRACE_END) {
1439 break;
1440 }
1441 }
1442 *num_op_arrays_ptr = num_op_arrays;
1443
1444 /* Allocate space for abstract stack */
1445 JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom);
1446
1447 /* 2. Construct TSSA */
1448 tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa));
1449 tssa->cfg.flags = ZEND_SSA_TSSA;
1450 tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block));
1451 tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block));
1452 tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int));
1453
1454 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1455 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1456 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1457 tssa->cfg.blocks_count = 2;
1458 tssa->cfg.edges_count = 2;
1459
1460 tssa->cfg.predecessors[0] = 0;
1461 tssa->cfg.predecessors[1] = 1;
1462
1463 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_REACHABLE;
1464 tssa->cfg.blocks[0].successors_count = 1;
1465 tssa->cfg.blocks[0].predecessors_count = 0;
1466 tssa->cfg.blocks[0].successors = tssa->cfg.blocks[0].successors_storage;
1467 tssa->cfg.blocks[0].successors[0] = 1;
1468
1469 tssa->cfg.blocks[1].flags = ZEND_BB_FOLLOW|ZEND_BB_TARGET|ZEND_BB_LOOP_HEADER|ZEND_BB_REACHABLE;
1470 tssa->cfg.blocks[1].len = ssa_ops_count;
1471 tssa->cfg.blocks[1].successors_count = 1;
1472 tssa->cfg.blocks[1].predecessors_count = 2;
1473 tssa->cfg.blocks[1].successors = tssa->cfg.blocks[1].successors_storage;
1474 tssa->cfg.blocks[1].successors[1] = 1;
1475 } else {
1476 tssa->cfg.blocks_count = 1;
1477 tssa->cfg.edges_count = 0;
1478
1479 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_EXIT|ZEND_BB_REACHABLE;
1480 tssa->cfg.blocks[0].len = ssa_ops_count;
1481 tssa->cfg.blocks[0].successors_count = 0;
1482 tssa->cfg.blocks[0].predecessors_count = 0;
1483 }
1484 ((zend_tssa*)tssa)->used_stack = -1;
1485
1486 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1487 return tssa;
1488 }
1489
1490 tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op));
1491 memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op));
1492 ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*));
1493 ((zend_tssa*)tssa)->tssa_opcodes = ssa_opcodes;
1494 ssa_opcodes[ssa_ops_count] = &_nop_opcode;
1495
1496 op_array = trace_buffer->op_array;
1497 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1498 ssa_vars_count = op_array->last_var;
1499 } else {
1500 ssa_vars_count = op_array->last_var + op_array->T;
1501 }
1502 stack = frame->stack;
1503 for (i = 0; i < ssa_vars_count; i++) {
1504 SET_STACK_VAR(stack, i, i);
1505 }
1506
1507 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1508 // TODO: For tracing, it's possible, to create pseudo Phi functions
1509 // at the end of loop, without this additional pass (like LuaJIT) ???
1510 ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, stack);
1511 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
1512 ssa_vars_count = zend_jit_trace_add_call_phis(trace_buffer, ssa_vars_count, tssa, stack);
1513 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1514 ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack);
1515 }
1516
1517 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1518 idx = 0;
1519 level = 0;
1520 for (;;p++) {
1521 if (p->op == ZEND_JIT_TRACE_VM) {
1522 opline = p->opline;
1523 ssa_opcodes[idx] = opline;
1524 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1525 idx++;
1526 len = zend_jit_trace_op_len(p->opline);
1527 while (len > 1) {
1528 opline++;
1529 ssa_opcodes[idx] = opline;
1530 if (opline->opcode != ZEND_OP_DATA) {
1531 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1532 }
1533 idx++;
1534 len--;
1535 }
1536 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1537 frame = zend_jit_trace_call_frame(frame, op_array);
1538 stack = frame->stack;
1539 op_array = p->op_array;
1540 level++;
1541 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1542 return NULL;
1543 }
1544 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1545 for (i = 0; i < op_array->last_var; i++) {
1546 SET_STACK_VAR(stack, i, ssa_vars_count++);
1547 }
1548 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1549 op_array = p->op_array;
1550 frame = zend_jit_trace_ret_frame(frame, op_array);
1551 stack = frame->stack;
1552 if (level == 0) {
1553 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1554 return NULL;
1555 }
1556 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1557 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1558 SET_STACK_VAR(stack, i, ssa_vars_count++);
1559 }
1560 } else {
1561 level--;
1562 }
1563 } else if (p->op == ZEND_JIT_TRACE_END) {
1564 break;
1565 }
1566 }
1567
1568 op_array = trace_buffer->op_array;
1569 tssa->vars_count = ssa_vars_count;
1570 tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var));
1571 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1572 vars_count = op_array->last_var;
1573 } else {
1574 vars_count = op_array->last_var + op_array->T;
1575 }
1576 i = 0;
1577 while (i < vars_count) {
1578 ssa_vars[i].var = i;
1579 ssa_vars[i].scc = -1;
1580 ssa_vars[i].definition = -1;
1581 ssa_vars[i].use_chain = -1;
1582 i++;
1583 }
1584 while (i < tssa->vars_count) {
1585 ssa_vars[i].var = -1;
1586 ssa_vars[i].scc = -1;
1587 ssa_vars[i].definition = -1;
1588 ssa_vars[i].use_chain = -1;
1589 i++;
1590 }
1591
1592 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1593 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1594 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1595 /* Update Phi sources */
1596 zend_ssa_phi *phi = tssa->blocks[1].phis;
1597
1598 while (phi) {
1599 phi->sources[1] = STACK_VAR(stack, phi->var);
1600 ssa_vars[phi->ssa_var].var = phi->var;
1601 ssa_vars[phi->ssa_var].definition_phi = phi;
1602 ssa_vars[phi->sources[0]].phi_use_chain = phi;
1603 ssa_vars[phi->sources[1]].phi_use_chain = phi;
1604 phi = phi->next;
1605 }
1606 }
1607
1608 /* 3. Compute use-def chains */
1609 idx = (ssa_ops_count - 1);
1610 op = ssa_ops + idx;
1611 while (idx >= 0) {
1612 opline = ssa_opcodes[idx];
1613 if (op->op1_use >= 0) {
1614 op->op1_use_chain = ssa_vars[op->op1_use].use_chain;
1615 ssa_vars[op->op1_use].use_chain = idx;
1616 }
1617 if (op->op2_use >= 0 && op->op2_use != op->op1_use) {
1618 op->op2_use_chain = ssa_vars[op->op2_use].use_chain;
1619 ssa_vars[op->op2_use].use_chain = idx;
1620 }
1621 if (op->result_use >= 0 && op->result_use != op->op1_use && op->result_use != op->op2_use) {
1622 op->res_use_chain = ssa_vars[op->result_use].use_chain;
1623 ssa_vars[op->result_use].use_chain = idx;
1624 }
1625 if (op->op1_def >= 0) {
1626 ssa_vars[op->op1_def].var = EX_VAR_TO_NUM(opline->op1.var);
1627 ssa_vars[op->op1_def].definition = idx;
1628 }
1629 if (op->op2_def >= 0) {
1630 ssa_vars[op->op2_def].var = EX_VAR_TO_NUM(opline->op2.var);
1631 ssa_vars[op->op2_def].definition = idx;
1632 }
1633 if (op->result_def >= 0) {
1634 ssa_vars[op->result_def].var = EX_VAR_TO_NUM(opline->result.var);
1635 ssa_vars[op->result_def].definition = idx;
1636 }
1637 op--;
1638 idx--;
1639 }
1640
1641 /* 4. Type inference */
1642 op_array = trace_buffer->op_array;
1643 opline = trace_buffer[1].opline;
1644 jit_extension =
1645 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1646 ssa = &jit_extension->func_info.ssa;
1647
1648 tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info));
1649
1650 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1651 i = 0;
1652 while (i < op_array->last_var) {
1653 if (i < op_array->num_args) {
1654 if (ssa->var_info
1655 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i, NULL)) {
1656 /* pass */
1657 } else {
1658 if (ssa->vars) {
1659 ssa_vars[i].no_val = ssa->vars[i].no_val;
1660 ssa_vars[i].alias = ssa->vars[i].alias;
1661 } else {
1662 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1663 }
1664 if (op_array->arg_info && i < trace_buffer[1].opline - op_array->opcodes) {
1665 zend_arg_info *arg_info = &op_array->arg_info[i];
1666 zend_class_entry *ce;
1667 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
1668
1669 if (ZEND_ARG_SEND_MODE(arg_info)) {
1670 tmp |= MAY_BE_REF;
1671 }
1672 ssa_var_info[i].type = tmp;
1673 ssa_var_info[i].ce = ce;
1674 ssa_var_info[i].is_instanceof = 1;
1675 } else {
1676 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1677 }
1678 }
1679 } else {
1680 if (ssa->vars) {
1681 ssa_vars[i].no_val = ssa->vars[i].no_val;
1682 ssa_vars[i].alias = ssa->vars[i].alias;
1683 } else {
1684 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1685 }
1686 if (ssa_vars[i].alias == NO_ALIAS) {
1687 ssa_var_info[i].type = MAY_BE_UNDEF;
1688 } else {
1689 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1690 }
1691 }
1692 i++;
1693 }
1694 } else {
1695 int parent_vars_count = 0;
1696 zend_jit_trace_stack *parent_stack = NULL;
1697
1698 i = 0;
1699 if (parent_trace) {
1700 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
1701 op_array->last_var + op_array->T);
1702 if (parent_vars_count) {
1703 parent_stack =
1704 zend_jit_traces[parent_trace].stack_map +
1705 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
1706 }
1707 }
1708 while (i < op_array->last_var + op_array->T) {
1709 if (!ssa->var_info
1710 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i, opline)) {
1711 if (ssa->vars && i < ssa->vars_count) {
1712 ssa_vars[i].alias = ssa->vars[i].alias;
1713 } else {
1714 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1715 }
1716 if (i < op_array->last_var) {
1717 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1718 } else {
1719 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1720 }
1721 }
1722 if (i < parent_vars_count) {
1723 /* Initialize TSSA variable from parent trace */
1724 uint8_t op_type = STACK_TYPE(parent_stack, i);
1725
1726 if (op_type != IS_UNKNOWN) {
1727 ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type);
1728 if (!ssa_var_info[i].type
1729 && op_type == IS_UNDEF
1730 && i >= op_array->last_var) {
1731 // TODO: It's better to use NULL instead of UNDEF for temporary variables
1732 ssa_var_info[i].type |= MAY_BE_UNDEF;
1733 }
1734 }
1735 }
1736 i++;
1737 }
1738 }
1739
1740 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1741 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1742 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1743 /* Propagate initial value through Phi functions */
1744 zend_ssa_phi *phi = tssa->blocks[1].phis;
1745
1746 while (phi) {
1747 if (!ssa->var_info
1748 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var, NULL)) {
1749 ssa_vars[phi->ssa_var].alias = ssa_vars[phi->sources[0]].alias;
1750 ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type;
1751 }
1752 phi = phi->next;
1753 }
1754 }
1755
1756 frame = JIT_G(current_frame);
1757 top = zend_jit_trace_call_frame(frame, op_array);
1758 TRACE_FRAME_INIT(frame, op_array, 0, 0);
1759 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
1760 frame->used_stack = 0;
1761 memset(&return_value_info, 0, sizeof(return_value_info));
1762
1763 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1764 max_used_stack = used_stack = 0;
1765 } else {
1766 max_used_stack = used_stack = -1;
1767 }
1768
1769 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1770 idx = 0;
1771 level = 0;
1772 opline = NULL;
1773 for (;;p++) {
1774 if (p->op == ZEND_JIT_TRACE_VM) {
1775 uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type;
1776 uint8_t val_type = IS_UNKNOWN;
1777 // zend_class_entry *op1_ce = NULL;
1778 zend_class_entry *op2_ce = NULL;
1779
1780 opline = p->opline;
1781
1782 op1_type = orig_op1_type = p->op1_type;
1783 op2_type = orig_op2_type = p->op2_type;
1784 op3_type = p->op3_type;
1785 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1786 op1_type = IS_UNKNOWN;
1787 }
1788 if (op1_type != IS_UNKNOWN) {
1789 op1_type &= ~IS_TRACE_PACKED;
1790 }
1791 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1792 op2_type = IS_UNKNOWN;
1793 }
1794 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1795 op3_type = IS_UNKNOWN;
1796 }
1797
1798 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
1799 // op1_ce = (zend_class_entry*)(p+1)->ce;
1800 p++;
1801 }
1802 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
1803 op2_ce = (zend_class_entry*)(p+1)->ce;
1804 p++;
1805 }
1806 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
1807 val_type = (p+1)->op1_type;
1808 p++;
1809 }
1810
1811 switch (opline->opcode) {
1812 case ZEND_ASSIGN_OP:
1813 if (opline->extended_value == ZEND_POW
1814 || opline->extended_value == ZEND_DIV) {
1815 // TODO: check for division by zero ???
1816 break;
1817 }
1818 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1819 break;
1820 }
1821 ADD_OP1_TRACE_GUARD();
1822 ADD_OP2_TRACE_GUARD();
1823 break;
1824 case ZEND_ASSIGN_DIM_OP:
1825 if (opline->extended_value == ZEND_POW
1826 || opline->extended_value == ZEND_DIV) {
1827 // TODO: check for division by zero ???
1828 break;
1829 }
1830 if (opline->result_type != IS_UNUSED) {
1831 break;
1832 }
1833 if (op3_type != IS_UNKNOWN
1834 && !zend_jit_supported_binary_op(
1835 opline->extended_value, MAY_BE_ANY, (1<<op3_type))) {
1836 break;
1837 }
1838 ZEND_FALLTHROUGH;
1839 case ZEND_ASSIGN_DIM:
1840 if (opline->op1_type == IS_CV) {
1841 if ((opline+1)->op1_type == IS_CV
1842 && (opline+1)->op1.var == opline->op1.var) {
1843 /* skip $a[x] = $a; */
1844 break;
1845 }
1846 ADD_OP1_DATA_TRACE_GUARD();
1847 ADD_OP2_TRACE_GUARD();
1848 ADD_OP1_TRACE_GUARD();
1849 } else if (orig_op1_type != IS_UNKNOWN
1850 && (orig_op1_type & IS_TRACE_INDIRECT)
1851 && opline->result_type == IS_UNUSED) {
1852 if (opline->opcode == ZEND_ASSIGN_DIM_OP) {
1853 ADD_OP1_DATA_TRACE_GUARD();
1854 }
1855 ADD_OP2_TRACE_GUARD();
1856 }
1857 if (op1_type == IS_ARRAY
1858 && ((opline->op2_type == IS_CONST
1859 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1860 || (opline->op2_type != IS_CONST
1861 && op2_type == IS_LONG))) {
1862
1863 if (!(orig_op1_type & IS_TRACE_PACKED)) {
1864 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1865
1866 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1867 info->type |= MAY_BE_PACKED_GUARD;
1868 info->type &= ~MAY_BE_ARRAY_PACKED;
1869 }
1870 } else if (opline->opcode == ZEND_ASSIGN_DIM_OP
1871 && val_type != IS_UNKNOWN
1872 && val_type != IS_UNDEF) {
1873 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1874
1875 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1876 info->type |= MAY_BE_PACKED_GUARD;
1877 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1878 }
1879 }
1880 }
1881 break;
1882 case ZEND_ASSIGN_OBJ_OP:
1883 if (opline->extended_value == ZEND_POW
1884 || opline->extended_value == ZEND_DIV) {
1885 // TODO: check for division by zero ???
1886 break;
1887 }
1888 if (opline->result_type != IS_UNUSED) {
1889 break;
1890 }
1891 ZEND_FALLTHROUGH;
1892 case ZEND_ASSIGN_OBJ:
1893 case ZEND_PRE_INC_OBJ:
1894 case ZEND_PRE_DEC_OBJ:
1895 case ZEND_POST_INC_OBJ:
1896 case ZEND_POST_DEC_OBJ:
1897 if (opline->op2_type != IS_CONST
1898 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1899 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1900 break;
1901 }
1902 if (opline->opcode == ZEND_ASSIGN_OBJ
1903 || opline->opcode == ZEND_ASSIGN_OBJ_OP) {
1904 if (opline->op1_type == IS_CV
1905 && (opline+1)->op1_type == IS_CV
1906 && (opline+1)->op1.var == opline->op1.var) {
1907 /* skip $a->prop += $a; */
1908 break;
1909 }
1910 ADD_OP1_DATA_TRACE_GUARD();
1911 }
1912 ADD_OP1_TRACE_GUARD();
1913 break;
1914 case ZEND_CONCAT:
1915 case ZEND_FAST_CONCAT:
1916 if ((opline->op1_type == IS_CONST || orig_op1_type == IS_STRING)
1917 && (opline->op2_type == IS_CONST || orig_op2_type == IS_STRING)) {
1918 ADD_OP2_TRACE_GUARD();
1919 ADD_OP1_TRACE_GUARD();
1920 }
1921 break;
1922 case ZEND_ADD:
1923 case ZEND_SUB:
1924 case ZEND_MUL:
1925 // case ZEND_DIV: // TODO: check for division by zero ???
1926 if (orig_op1_type == IS_UNDEF || orig_op2_type == IS_UNDEF) {
1927 break;
1928 }
1929 ZEND_FALLTHROUGH;
1930 case ZEND_IS_EQUAL:
1931 case ZEND_IS_NOT_EQUAL:
1932 case ZEND_IS_SMALLER:
1933 case ZEND_IS_SMALLER_OR_EQUAL:
1934 case ZEND_CASE:
1935 case ZEND_IS_IDENTICAL:
1936 case ZEND_IS_NOT_IDENTICAL:
1937 case ZEND_CASE_STRICT:
1938 case ZEND_BW_OR:
1939 case ZEND_BW_AND:
1940 case ZEND_BW_XOR:
1941 case ZEND_SL:
1942 case ZEND_SR:
1943 case ZEND_MOD:
1944 ADD_OP2_TRACE_GUARD();
1945 ZEND_FALLTHROUGH;
1946 case ZEND_ECHO:
1947 case ZEND_STRLEN:
1948 case ZEND_COUNT:
1949 case ZEND_QM_ASSIGN:
1950 case ZEND_FE_RESET_R:
1951 ADD_OP1_TRACE_GUARD();
1952 break;
1953 case ZEND_FE_FETCH_R:
1954 ADD_OP1_TRACE_GUARD();
1955 if (op1_type == IS_ARRAY && (orig_op1_type & ~IS_TRACE_PACKED) == IS_ARRAY) {
1956
1957 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1958
1959 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1960 info->type |= MAY_BE_PACKED_GUARD;
1961 if (orig_op1_type & IS_TRACE_PACKED) {
1962 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1963 } else {
1964 info->type &= ~MAY_BE_ARRAY_PACKED;
1965 }
1966 }
1967 }
1968 break;
1969 case ZEND_VERIFY_RETURN_TYPE:
1970 if (opline->op1_type == IS_UNUSED) {
1971 /* Always throws */
1972 break;
1973 }
1974 if (opline->op1_type == IS_CONST) {
1975 /* TODO Different instruction format, has return value */
1976 break;
1977 }
1978 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
1979 /* Not worth bothering with */
1980 break;
1981 }
1982 ADD_OP1_TRACE_GUARD();
1983 break;
1984 case ZEND_FETCH_DIM_FUNC_ARG:
1985 if (!frame
1986 || !frame->call
1987 || !frame->call->func
1988 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
1989 break;
1990 }
1991 ADD_OP2_TRACE_GUARD();
1992 ADD_OP1_TRACE_GUARD();
1993 break;
1994 case ZEND_PRE_INC:
1995 case ZEND_PRE_DEC:
1996 case ZEND_POST_INC:
1997 case ZEND_POST_DEC:
1998 if (opline->op1_type != IS_CV) {
1999 break;
2000 }
2001 ADD_OP1_TRACE_GUARD();
2002 break;
2003 case ZEND_ASSIGN:
2004 if (opline->op1_type != IS_CV) {
2005 break;
2006 }
2007 ADD_OP2_TRACE_GUARD();
2008 if (op1_type != IS_UNKNOWN
2009 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
2010 ADD_OP1_TRACE_GUARD();
2011 }
2012 break;
2013 case ZEND_CAST:
2014 if (opline->extended_value != op1_type) {
2015 break;
2016 }
2017 ADD_OP1_TRACE_GUARD();
2018 break;
2019 case ZEND_JMPZ:
2020 case ZEND_JMPNZ:
2021 case ZEND_JMPZ_EX:
2022 case ZEND_JMPNZ_EX:
2023 case ZEND_BOOL:
2024 case ZEND_BOOL_NOT:
2025 ADD_OP1_TRACE_GUARD();
2026 break;
2027 case ZEND_ISSET_ISEMPTY_CV:
2028 if ((opline->extended_value & ZEND_ISEMPTY)) {
2029 // TODO: support for empty() ???
2030 break;
2031 }
2032 ADD_OP1_TRACE_GUARD();
2033 break;
2034 case ZEND_IN_ARRAY:
2035 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
2036 break;
2037 }
2038 ADD_OP1_TRACE_GUARD();
2039 break;
2040 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
2041 if ((opline->extended_value & ZEND_ISEMPTY)) {
2042 // TODO: support for empty() ???
2043 break;
2044 }
2045 ZEND_FALLTHROUGH;
2046 case ZEND_FETCH_DIM_R:
2047 case ZEND_FETCH_DIM_IS:
2048 case ZEND_FETCH_LIST_R:
2049 ADD_OP1_TRACE_GUARD();
2050 ADD_OP2_TRACE_GUARD();
2051
2052 if (op1_type == IS_ARRAY
2053 && opline->op1_type != IS_CONST
2054 && ((opline->op2_type == IS_CONST
2055 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2056 || (opline->op2_type != IS_CONST
2057 && op2_type == IS_LONG))) {
2058
2059 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
2060
2061 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
2062 info->type |= MAY_BE_PACKED_GUARD;
2063 if (orig_op1_type & IS_TRACE_PACKED) {
2064 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
2065 } else {
2066 info->type &= ~MAY_BE_ARRAY_PACKED;
2067 }
2068 }
2069 }
2070 break;
2071 case ZEND_FETCH_DIM_W:
2072 case ZEND_FETCH_DIM_RW:
2073 // case ZEND_FETCH_DIM_UNSET:
2074 case ZEND_FETCH_LIST_W:
2075 if (opline->op1_type != IS_CV
2076 && (orig_op1_type == IS_UNKNOWN
2077 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
2078 break;
2079 }
2080 ADD_OP1_TRACE_GUARD();
2081 ADD_OP2_TRACE_GUARD();
2082 if (op1_type == IS_ARRAY
2083 && !(orig_op1_type & IS_TRACE_PACKED)
2084 && ((opline->op2_type == IS_CONST
2085 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2086 || (opline->op2_type != IS_CONST
2087 && op2_type == IS_LONG))) {
2088
2089 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
2090
2091 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
2092 info->type |= MAY_BE_PACKED_GUARD;
2093 info->type &= ~MAY_BE_ARRAY_PACKED;
2094 }
2095 }
2096 break;
2097 case ZEND_SEND_VAL_EX:
2098 case ZEND_SEND_VAR_EX:
2099 case ZEND_SEND_VAR_NO_REF_EX:
2100 if (opline->op2_type == IS_CONST) {
2101 /* Named parameters not supported in JIT */
2102 break;
2103 }
2104 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
2105 goto propagate_arg;
2106 }
2107 ZEND_FALLTHROUGH;
2108 case ZEND_SEND_VAL:
2109 case ZEND_SEND_VAR:
2110 case ZEND_SEND_VAR_NO_REF:
2111 case ZEND_SEND_FUNC_ARG:
2112 if (opline->op2_type == IS_CONST) {
2113 /* Named parameters not supported in JIT */
2114 break;
2115 }
2116 ADD_OP1_TRACE_GUARD();
2117 propagate_arg:
2118 /* Propagate argument type */
2119 if (frame->call
2120 && frame->call->func
2121 && frame->call->func->type == ZEND_USER_FUNCTION
2122 && opline->op2.num <= frame->call->func->op_array.num_args) {
2123 uint32_t info;
2124
2125 if (opline->op1_type == IS_CONST) {
2126 info = _const_op_type(RT_CONSTANT(opline, opline->op1));
2127 } else {
2128 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2129 info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD;
2130 }
2131 if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
2132 zend_arg_info *arg_info;
2133
2134 ZEND_ASSERT(frame->call->func->op_array.arg_info);
2135 arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1];
2136 if (ZEND_TYPE_IS_SET(arg_info->type)) {
2137 zend_class_entry *ce;
2138 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2139 info &= tmp;
2140 if (!info) {
2141 break;
2142 }
2143 }
2144 }
2145 if (opline->op1_type == IS_CV && (info & MAY_BE_RC1)) {
2146 info |= MAY_BE_RCN;
2147 }
2148 if (info & MAY_BE_UNDEF) {
2149 info |= MAY_BE_NULL;
2150 info &= ~MAY_BE_UNDEF;
2151 }
2152 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2153 info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY;
2154 }
2155 SET_STACK_INFO(frame->call->stack, opline->op2.num - 1, info);
2156 }
2157 break;
2158 case ZEND_RETURN:
2159 ADD_OP1_TRACE_GUARD();
2160 /* Propagate return value types */
2161 if (opline->op1_type == IS_UNUSED) {
2162 return_value_info.type = MAY_BE_NULL;
2163 } else if (opline->op1_type == IS_CONST) {
2164 return_value_info.type = _const_op_type(RT_CONSTANT(opline, opline->op1));
2165 } else {
2166 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2167 return_value_info = ssa_var_info[ssa_ops[idx].op1_use];
2168 if (return_value_info.type & MAY_BE_UNDEF) {
2169 return_value_info.type &= ~MAY_BE_UNDEF;
2170 return_value_info.type |= MAY_BE_NULL;
2171 }
2172 if (return_value_info.type & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
2173 /* CVs are going to be destructed and the reference-counter
2174 of return value may be decremented to 1 */
2175 return_value_info.type |= MAY_BE_RC1;
2176 }
2177 return_value_info.type &= ~MAY_BE_GUARD;
2178 }
2179 break;
2180 case ZEND_CHECK_FUNC_ARG:
2181 if (!frame
2182 || !frame->call
2183 || !frame->call->func) {
2184 break;
2185 }
2186 if (opline->op2_type == IS_CONST
2187 || opline->op2.num > MAX_ARG_FLAG_NUM) {
2188 /* Named parameters not supported in JIT */
2189 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame->call);
2190 break;
2191 }
2192 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2193 TRACE_FRAME_SET_LAST_SEND_BY_REF(frame->call);
2194 } else {
2195 TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame->call);
2196 }
2197 break;
2198 case ZEND_FETCH_OBJ_FUNC_ARG:
2199 if (!frame
2200 || !frame->call
2201 || !frame->call->func
2202 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
2203 break;
2204 }
2205 ZEND_FALLTHROUGH;
2206 case ZEND_FETCH_OBJ_R:
2207 case ZEND_FETCH_OBJ_IS:
2208 case ZEND_FETCH_OBJ_W:
2209 if (opline->op2_type != IS_CONST
2210 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2211 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2212 break;
2213 }
2214 if (opline->op1_type != IS_UNUSED && op1_type == IS_OBJECT) {
2215 ADD_OP1_TRACE_GUARD();
2216 }
2217 break;
2218 case ZEND_INIT_METHOD_CALL:
2219 if (opline->op2_type != IS_CONST
2220 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2221 break;
2222 }
2223 ADD_OP1_TRACE_GUARD();
2224 break;
2225 case ZEND_INIT_DYNAMIC_CALL:
2226 if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) {
2227 ADD_OP2_TRACE_GUARD();
2228 }
2229 break;
2230 case ZEND_SEND_ARRAY:
2231 case ZEND_SEND_UNPACK:
2232 case ZEND_CHECK_UNDEF_ARGS:
2233 case ZEND_INCLUDE_OR_EVAL:
2234 max_used_stack = used_stack = -1;
2235 break;
2236 case ZEND_TYPE_CHECK:
2237 if (opline->extended_value == MAY_BE_RESOURCE) {
2238 // TODO: support for is_resource() ???
2239 break;
2240 }
2241 if (op1_type != IS_UNKNOWN
2242 && (opline->extended_value == (1 << op1_type)
2243 || opline->extended_value == MAY_BE_ANY - (1 << op1_type))) {
2244 /* add guards only for exact checks, to avoid code duplication */
2245 ADD_OP1_TRACE_GUARD();
2246 }
2247 break;
2248 case ZEND_ROPE_INIT:
2249 case ZEND_ROPE_ADD:
2250 case ZEND_ROPE_END:
2251 if (op2_type == IS_STRING) {
2252 ADD_OP2_TRACE_GUARD();
2253 }
2254 break;
2255 default:
2256 break;
2257 }
2258 len = zend_jit_trace_op_len(opline);
2259 if (ssa->var_info) {
2260 /* Add statically inferred ranges */
2261 if (ssa_ops[idx].op1_def >= 0) {
2262 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2263 }
2264 if (ssa_ops[idx].op2_def >= 0) {
2265 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2266 }
2267 if (ssa_ops[idx].result_def >= 0) {
2268 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2269 }
2270 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2271 if (ssa_ops[idx+1].op1_def >= 0) {
2272 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2273 }
2274 if (ssa_ops[idx+1].op2_def >= 0) {
2275 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2276 }
2277 if (ssa_ops[idx+1].result_def >= 0) {
2278 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2279 }
2280 }
2281 } else {
2282 if (ssa_ops[idx].op1_def >= 0) {
2283 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2284 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2285 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2286 }
2287 }
2288 if (ssa_ops[idx].op2_def >= 0) {
2289 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2290 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2291 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2292 }
2293 }
2294 if (ssa_ops[idx].result_def >= 0) {
2295 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2296 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2297 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2298 }
2299 }
2300 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2301 if (ssa_ops[idx+1].op1_def >= 0) {
2302 ssa_vars[ssa_ops[idx+1].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op1.var));
2303 if (ssa_ops[idx+1].op1_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op1_use].type & MAY_BE_REF)) {
2304 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2305 }
2306 }
2307 if (ssa_ops[idx+1].op2_def >= 0) {
2308 ssa_vars[ssa_ops[idx+1].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op2.var));
2309 if (ssa_ops[idx+1].op2_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op2_use].type & MAY_BE_REF)) {
2310 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2311 }
2312 }
2313 if (ssa_ops[idx+1].result_def >= 0) {
2314 ssa_vars[ssa_ops[idx+1].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->result.var));
2315 if (ssa_ops[idx+1].result_use < 0 || !(ssa_var_info[ssa_ops[idx+1].result_use].type & MAY_BE_REF)) {
2316 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2317 }
2318 }
2319 }
2320 }
2321 if (opline->opcode == ZEND_RECV_INIT
2322 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2323 /* RECV_INIT always copy the constant */
2324 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2325 } else if ((opline->opcode == ZEND_FE_FETCH_R || opline->opcode == ZEND_FE_FETCH_RW)
2326 && ssa_opcodes[idx + 1] == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2327 if (ssa_ops[idx].op2_use >= 0 && ssa_ops[idx].op2_def >= 0) {
2328 ssa_var_info[ssa_ops[idx].op2_def] = ssa_var_info[ssa_ops[idx].op2_use];
2329 }
2330 } else {
2331 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2332 // TODO:
2333 assert(0);
2334 }
2335 if (opline->opcode == ZEND_ASSIGN_DIM_OP
2336 && ssa_ops[idx].op1_def >= 0
2337 && op1_type == IS_ARRAY
2338 && (orig_op1_type & IS_TRACE_PACKED)
2339 && val_type != IS_UNKNOWN
2340 && val_type != IS_UNDEF
2341 && ((opline->op2_type == IS_CONST
2342 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2343 || (opline->op2_type != IS_CONST
2344 && op2_type == IS_LONG))) {
2345 zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_def];
2346
2347 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
2348 }
2349 }
2350 if (ssa->var_info) {
2351 /* Add statically inferred restrictions */
2352 if (ssa_ops[idx].op1_def >= 0) {
2353 if (opline->opcode == ZEND_SEND_VAR_EX
2354 && frame
2355 && frame->call
2356 && frame->call->func
2357 && !ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2358 ssa_var_info[ssa_ops[idx].op1_def] = ssa_var_info[ssa_ops[idx].op1_use];
2359 ssa_var_info[ssa_ops[idx].op1_def].type &= ~MAY_BE_GUARD;
2360 if (ssa_var_info[ssa_ops[idx].op1_def].type & MAY_BE_RC1) {
2361 ssa_var_info[ssa_ops[idx].op1_def].type |= MAY_BE_RCN;
2362 }
2363 } else {
2364 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2365 }
2366 }
2367 if (ssa_ops[idx].op2_def >= 0) {
2368 if ((opline->opcode != ZEND_FE_FETCH_R && opline->opcode != ZEND_FE_FETCH_RW)
2369 || ssa_opcodes[idx + 1] != ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2370 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2371 }
2372 }
2373 if (ssa_ops[idx].result_def >= 0) {
2374 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2375 }
2376 }
2377 idx++;
2378 while (len > 1) {
2379 opline++;
2380 if (opline->opcode != ZEND_OP_DATA) {
2381 if (ssa->var_info) {
2382 /* Add statically inferred ranges */
2383 if (ssa_ops[idx].op1_def >= 0) {
2384 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2385 }
2386 if (ssa_ops[idx].op2_def >= 0) {
2387 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2388 }
2389 if (ssa_ops[idx].result_def >= 0) {
2390 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2391 }
2392 } else {
2393 if (ssa_ops[idx].op1_def >= 0) {
2394 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2395 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2396 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2397 }
2398 }
2399 if (ssa_ops[idx].op2_def >= 0) {
2400 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2401 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2402 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2403 }
2404 }
2405 if (ssa_ops[idx].result_def >= 0) {
2406 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2407 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2408 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2409 }
2410 }
2411 }
2412 if (opline->opcode == ZEND_RECV_INIT
2413 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2414 /* RECV_INIT always copy the constant */
2415 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2416 } else {
2417 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2418 // TODO:
2419 assert(0);
2420 }
2421 }
2422 }
2423 if (ssa->var_info) {
2424 /* Add statically inferred restrictions */
2425 if (ssa_ops[idx].op1_def >= 0) {
2426 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2427 }
2428 if (ssa_ops[idx].op2_def >= 0) {
2429 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2430 }
2431 if (ssa_ops[idx].result_def >= 0) {
2432 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2433 }
2434 }
2435 idx++;
2436 len--;
2437 }
2438
2439 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
2440 op_array = p->op_array;
2441 jit_extension =
2442 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2443 ssa = &jit_extension->func_info.ssa;
2444
2445 call = frame->call;
2446 if (!call) {
2447 /* Trace missed INIT_FCALL opcode */
2448 call = top;
2449 TRACE_FRAME_INIT(call, op_array, 0, 0);
2450 call->used_stack = 0;
2451 top = zend_jit_trace_call_frame(top, op_array);
2452 } else {
2453 ZEND_ASSERT(&call->func->op_array == op_array);
2454 }
2455 frame->call = call->prev;
2456 call->prev = frame;
2457 TRACE_FRAME_SET_RETURN_SSA_VAR(call, find_return_ssa_var(p - 1, ssa_ops + (idx - 1)));
2458 frame = call;
2459
2460 level++;
2461 i = 0;
2462 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2463 while (i < op_array->last_var) {
2464 ssa_vars[v].var = i;
2465 if (i < op_array->num_args) {
2466 if (ssa->var_info
2467 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v, NULL)) {
2468 /* pass */
2469 } else {
2470 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2471 if (op_array->arg_info) {
2472 zend_arg_info *arg_info = &op_array->arg_info[i];
2473 zend_class_entry *ce;
2474 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2475
2476 if (ZEND_ARG_SEND_MODE(arg_info)) {
2477 tmp |= MAY_BE_REF;
2478 }
2479 ssa_var_info[v].type = tmp;
2480 ssa_var_info[v].ce = ce;
2481 ssa_var_info[v].is_instanceof = 1;
2482 } else {
2483 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2484 }
2485 }
2486 } else {
2487 if (ssa->vars) {
2488 ssa_vars[v].no_val = ssa->vars[i].no_val;
2489 ssa_vars[v].alias = ssa->vars[i].alias;
2490 } else {
2491 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2492 }
2493 if (ssa_vars[v].alias == NO_ALIAS) {
2494 ssa_var_info[v].type = MAY_BE_UNDEF;
2495 } else {
2496 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2497 }
2498 }
2499 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)
2500 && i < op_array->num_args) {
2501 /* Propagate argument type */
2502 ssa_var_info[v].type &= STACK_INFO(frame->stack, i);
2503 }
2504 i++;
2505 v++;
2506 }
2507 } else if (p->op == ZEND_JIT_TRACE_BACK) {
2508 op_array = p->op_array;
2509 jit_extension =
2510 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2511 ssa = &jit_extension->func_info.ssa;
2512 if (level == 0) {
2513 i = 0;
2514 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2515 while (i < op_array->last_var) {
2516 ssa_vars[v].var = i;
2517 if (!ssa->var_info
2518 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v, NULL)) {
2519 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2520 }
2521 i++;
2522 v++;
2523 }
2524 while (i < op_array->last_var + op_array->T) {
2525 ssa_vars[v].var = i;
2526 if (!ssa->var_info
2527 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v, NULL)) {
2528 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2529 }
2530 i++;
2531 v++;
2532 }
2533 if (return_value_info.type != 0) {
2534 zend_jit_trace_rec *q = p + 1;
2535 while (q->op == ZEND_JIT_TRACE_INIT_CALL) {
2536 q++;
2537 }
2538 if (q->op == ZEND_JIT_TRACE_VM
2539 || (q->op == ZEND_JIT_TRACE_END
2540 && q->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)) {
2541 const zend_op *opline = q->opline - 1;
2542 if (opline->result_type != IS_UNUSED) {
2543 ssa_var_info[
2544 ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info) +
2545 EX_VAR_TO_NUM(opline->result.var)] = return_value_info;
2546 }
2547 }
2548 memset(&return_value_info, 0, sizeof(return_value_info));
2549 }
2550 } else {
2551 level--;
2552 if (return_value_info.type != 0) {
2553 if ((p+1)->op == ZEND_JIT_TRACE_VM) {
2554 const zend_op *opline = (p+1)->opline - 1;
2555 if (opline->result_type != IS_UNUSED) {
2556 if (TRACE_FRAME_RETURN_SSA_VAR(frame) >= 0) {
2557 ssa_var_info[TRACE_FRAME_RETURN_SSA_VAR(frame)] = return_value_info;
2558 }
2559 }
2560 }
2561 memset(&return_value_info, 0, sizeof(return_value_info));
2562 }
2563 }
2564
2565 top = frame;
2566 if (frame->prev) {
2567 if (used_stack > 0) {
2568 used_stack -= frame->used_stack;
2569 }
2570 frame = frame->prev;
2571 ZEND_ASSERT(&frame->func->op_array == op_array);
2572 } else {
2573 max_used_stack = used_stack = -1;
2574 frame = zend_jit_trace_ret_frame(frame, op_array);
2575 TRACE_FRAME_INIT(frame, op_array, 0, 0);
2576 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
2577 frame->used_stack = 0;
2578 }
2579
2580 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
2581 call = top;
2582 TRACE_FRAME_INIT(call, p->func, 0, 0);
2583 call->prev = frame->call;
2584 call->used_stack = 0;
2585 frame->call = call;
2586 top = zend_jit_trace_call_frame(top, p->op_array);
2587 if (p->func && p->func->type == ZEND_USER_FUNCTION) {
2588 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
2589 SET_STACK_INFO(call->stack, i, -1);
2590 }
2591 }
2592 if (used_stack >= 0
2593 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
2594 if (p->func == NULL || (p-1)->op != ZEND_JIT_TRACE_VM) {
2595 max_used_stack = used_stack = -1;
2596 } else {
2597 const zend_op *opline = (p-1)->opline;
2598
2599 switch (opline->opcode) {
2600 case ZEND_INIT_FCALL:
2601 case ZEND_INIT_FCALL_BY_NAME:
2602 case ZEND_INIT_NS_FCALL_BY_NAME:
2603 case ZEND_INIT_METHOD_CALL:
2604 case ZEND_INIT_DYNAMIC_CALL:
2605 //case ZEND_INIT_STATIC_METHOD_CALL:
2606 //case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
2607 //case ZEND_INIT_USER_CALL:
2608 //case ZEND_NEW:
2609 frame->used_stack = zend_vm_calc_used_stack(opline->extended_value, (zend_function*)p->func);
2610 used_stack += frame->used_stack;
2611 if (used_stack > max_used_stack) {
2612 max_used_stack = used_stack;
2613 }
2614 break;
2615 default:
2616 max_used_stack = used_stack = -1;
2617 }
2618 }
2619 }
2620 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
2621 call = frame->call;
2622 if (call) {
2623 top = call;
2624 frame->call = call->prev;
2625 }
2626
2627 if (idx > 0
2628 && ssa_ops[idx-1].result_def >= 0
2629 && (p->func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)
2630 && !(p->func->common.fn_flags & ZEND_ACC_RETURN_REFERENCE)) {
2631 ZEND_ASSERT(ssa_opcodes[idx-1] == opline);
2632 ZEND_ASSERT(opline->opcode == ZEND_DO_ICALL ||
2633 opline->opcode == ZEND_DO_FCALL ||
2634 opline->opcode == ZEND_DO_FCALL_BY_NAME);
2635
2636 if (opline->result_type != IS_UNDEF) {
2637 zend_class_entry *ce;
2638 const zend_function *func = p->func;
2639 zend_arg_info *ret_info = func->common.arg_info - 1;
2640 uint32_t ret_type = zend_fetch_arg_info_type(NULL, ret_info, &ce);
2641
2642 ssa_var_info[ssa_ops[idx-1].result_def].type &= ret_type;
2643 }
2644 }
2645 } else if (p->op == ZEND_JIT_TRACE_END) {
2646 break;
2647 }
2648 }
2649
2650 ((zend_tssa*)tssa)->used_stack = max_used_stack;
2651
2652 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2653 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2654 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2655 /* Propagate guards through Phi sources */
2656 zend_ssa_phi *phi = tssa->blocks[1].phis;
2657
2658 op_array = trace_buffer->op_array;
2659 jit_extension =
2660 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2661 ssa = &jit_extension->func_info.ssa;
2662
2663 while (phi) {
2664 uint32_t t = ssa_var_info[phi->ssa_var].type;
2665
2666 if ((t & MAY_BE_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2667 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2668 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2669
2670 if (((t0 | t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2671 if (!((t0 | t1) & MAY_BE_GUARD)) {
2672 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2673 }
2674 } else if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2675 if (!(t1 & MAY_BE_GUARD)
2676 || is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2677 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2678 t0 = (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2679 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2680 MAY_BE_GUARD;
2681 if (!(t0 & MAY_BE_ARRAY)) {
2682 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2683 }
2684 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2685 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2686 }
2687 ssa_var_info[phi->sources[0]].type = t0;
2688 ssa_var_info[phi->sources[0]].type = t0;
2689 }
2690 } else {
2691 if ((t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2692 t0 = (t & t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2693 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2694 MAY_BE_GUARD;
2695 if (!(t0 & MAY_BE_ARRAY)) {
2696 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2697 }
2698 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2699 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2700 }
2701 ssa_var_info[phi->sources[0]].type = t0;
2702 }
2703 if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2704 if (((t & t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != 0
2705 && is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2706 t1 = (t & t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2707 (t1 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2708 MAY_BE_GUARD;
2709 if (!(t1 & MAY_BE_ARRAY)) {
2710 t1 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2711 }
2712 if (!(t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2713 t1 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2714 }
2715 ssa_var_info[phi->sources[1]].type = t1;
2716 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2717 }
2718 }
2719 }
2720 t = ssa_var_info[phi->ssa_var].type;
2721 }
2722
2723 if ((t & MAY_BE_PACKED_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2724 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2725 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2726
2727 if (((t0 | t1) & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2728 if (!((t0 | t1) & MAY_BE_PACKED_GUARD)) {
2729 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2730 }
2731 } else if ((t1 & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2732 if (!(t1 & MAY_BE_PACKED_GUARD)) {
2733 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2734 ssa_var_info[phi->sources[0]].type =
2735 (t0 & ~MAY_BE_ARRAY_KEY_ANY) | (t & MAY_BE_ARRAY_KEY_ANY) | MAY_BE_PACKED_GUARD;
2736 }
2737 }
2738 }
2739 phi = phi->next;
2740 }
2741 }
2742
2743 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_TSSA)) {
2744 if (parent_trace) {
2745 fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s%s%s() %s:%d\n",
2746 ZEND_JIT_TRACE_NUM,
2747 parent_trace,
2748 exit_num,
2749 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2750 trace_buffer->op_array->scope ? "::" : "",
2751 trace_buffer->op_array->function_name ?
2752 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2753 ZSTR_VAL(trace_buffer->op_array->filename),
2754 trace_buffer[1].opline->lineno);
2755 } else {
2756 fprintf(stderr, "---- TRACE %d TSSA start (%s) %s%s%s() %s:%d\n",
2757 ZEND_JIT_TRACE_NUM,
2758 zend_jit_trace_star_desc(trace_buffer->start),
2759 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2760 trace_buffer->op_array->scope ? "::" : "",
2761 trace_buffer->op_array->function_name ?
2762 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2763 ZSTR_VAL(trace_buffer->op_array->filename),
2764 trace_buffer[1].opline->lineno);
2765 }
2766 zend_jit_dump_trace(trace_buffer, tssa);
2767 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LINK) {
2768 uint32_t idx = trace_buffer[1].last;
2769 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
2770 fprintf(stderr, "---- TRACE %d TSSA stop (link to %d)\n",
2771 ZEND_JIT_TRACE_NUM,
2772 link_to);
2773 } else {
2774 fprintf(stderr, "---- TRACE %d TSSA stop (%s)\n",
2775 ZEND_JIT_TRACE_NUM,
2776 zend_jit_trace_stop_description[trace_buffer->stop]);
2777 }
2778 }
2779
2780 return tssa;
2781 }
2782
2783 #define RA_HAS_IVAL(var) (ra[var].ref != 0)
2784 #define RA_IVAL_FLAGS(var) ra[var].flags
2785 #define RA_IVAL_START(var, line) do {ra[var].ref = IR_NULL;} while (0)
2786 #define RA_IVAL_END(var, line)
2787 #define RA_IVAL_CLOSE(var, line)
2788 #define RA_IVAL_DEL(var) do {ra[var].ref = IR_UNUSED;} while (0)
2789 #define RA_HAS_REG(var) (ra[var].ref != 0)
2790 #define RA_REG_FLAGS(var) ra[var].flags
2791 #define RA_REG_START(var, line) do {ra[var].ref = IR_NULL;} while (0)
2792 #define RA_REG_DEL(var) do {ra[var].ref = IR_UNUSED;} while (0)
2793
zend_jit_trace_use_var(int line,int var,int def,int use_chain,zend_jit_reg_var * ra,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array,const zend_ssa * op_array_ssa)2794 static void zend_jit_trace_use_var(int line, int var, int def, int use_chain, zend_jit_reg_var *ra, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array, const zend_ssa *op_array_ssa)
2795 {
2796 ZEND_ASSERT(RA_HAS_IVAL(var));
2797 ZEND_ASSERT(!(RA_IVAL_FLAGS(var) & ZREG_LAST_USE));
2798 RA_IVAL_END(var, line);
2799 if (def >= 0) {
2800 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2801 } else if (use_chain < 0 && (RA_IVAL_FLAGS(var) & (ZREG_LOAD|ZREG_STORE))) {
2802 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2803 } else if (use_chain >= 0 && !zend_ssa_is_no_val_use(ssa_opcodes[use_chain], ssa->ops + use_chain, var)) {
2804 /* pass */
2805 } else if (op_array_ssa->vars) {
2806 uint32_t use = ssa_opcodes[line] - op_array->opcodes;
2807
2808 if (ssa->ops[line].op1_use == var) {
2809 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op1_use, use)) {
2810 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2811 }
2812 } else if (ssa->ops[line].op2_use == var) {
2813 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op2_use, use)) {
2814 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2815 }
2816 } else if (ssa->ops[line].result_use == var) {
2817 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].result_use, use)) {
2818 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2819 }
2820 }
2821 }
2822 }
2823
zend_jit_trace_allocate_registers(zend_jit_trace_rec * trace_buffer,zend_ssa * ssa,uint32_t parent_trace,uint32_t exit_num)2824 static zend_jit_reg_var* zend_jit_trace_allocate_registers(zend_jit_trace_rec *trace_buffer, zend_ssa *ssa, uint32_t parent_trace, uint32_t exit_num)
2825 {
2826 const zend_op **ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
2827 zend_jit_trace_rec *p;
2828 const zend_op_array *op_array;
2829 zend_jit_op_array_trace_extension *jit_extension;
2830 const zend_ssa *op_array_ssa;
2831 const zend_ssa_op *ssa_op;
2832 int i, j, idx, count, level;
2833 zend_jit_reg_var *ra;
2834 const zend_op_array **vars_op_array;
2835 void *checkpoint;
2836 zend_jit_trace_stack_frame *frame;
2837 zend_jit_trace_stack *stack;
2838 uint32_t parent_vars_count = parent_trace ?
2839 zend_jit_traces[parent_trace].exit_info[exit_num].stack_size : 0;
2840 zend_jit_trace_stack *parent_stack = parent_vars_count ?
2841 zend_jit_traces[parent_trace].stack_map +
2842 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset : NULL;
2843 checkpoint = zend_arena_checkpoint(CG(arena));
2844 ra = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_jit_reg_var));
2845 vars_op_array = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_op_array*));
2846 memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count);
2847
2848 op_array = trace_buffer->op_array;
2849 jit_extension =
2850 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2851 op_array_ssa = &jit_extension->func_info.ssa;
2852 frame = JIT_G(current_frame);
2853 frame->prev = NULL;
2854 frame->func = (const zend_function*)op_array;
2855 stack = frame->stack;
2856
2857 count = 0;
2858
2859 i = 0;
2860 j = op_array->last_var;
2861 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
2862 j += op_array->T;
2863 }
2864 while (i < j) {
2865 SET_STACK_VAR(stack, i, i);
2866 vars_op_array[i] = op_array;
2867 /* We don't start intervals for variables used in Phi */
2868 if ((ssa->vars[i].use_chain >= 0 /*|| ssa->vars[i].phi_use_chain*/)
2869 && !zend_ssa_is_no_val_use(ssa_opcodes[ssa->vars[i].use_chain], ssa->ops + ssa->vars[i].use_chain, i)
2870 && ssa->vars[i].alias == NO_ALIAS
2871 && zend_jit_var_supports_reg(ssa, i)) {
2872 RA_IVAL_START(i, 0);
2873 if (i < parent_vars_count
2874 && STACK_REG(parent_stack, i) != ZREG_NONE
2875 && STACK_FLAGS(parent_stack, i) != ZREG_ZVAL_COPY
2876 ) {
2877 /* We will try to reuse register from parent trace */
2878 RA_IVAL_FLAGS(i) = STACK_FLAGS(parent_stack, i);
2879 count += 2;
2880 } else {
2881 RA_IVAL_FLAGS(i) = ZREG_LOAD;
2882 count++;
2883 }
2884 }
2885 i++;
2886 }
2887
2888 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
2889 j = op_array->last_var + op_array->T;
2890 while (i < j) {
2891 SET_STACK_VAR(stack, i, -1);
2892 i++;
2893 }
2894 }
2895
2896 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2897 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2898 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2899 zend_ssa_phi *phi = ssa->blocks[1].phis;
2900
2901 while (phi) {
2902 SET_STACK_VAR(stack, phi->var, phi->ssa_var);
2903 vars_op_array[phi->ssa_var] = op_array;
2904 if (ssa->vars[phi->ssa_var].use_chain >= 0
2905 && ssa->vars[phi->ssa_var].alias == NO_ALIAS
2906 && zend_jit_var_supports_reg(ssa, phi->ssa_var)) {
2907 RA_IVAL_START(phi->ssa_var, 0);
2908 count++;
2909 }
2910 phi = phi->next;
2911 }
2912 }
2913
2914 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
2915 level = 0;
2916 ssa_op = ssa->ops;
2917 idx = 0;
2918 for (;;p++) {
2919 if (p->op == ZEND_JIT_TRACE_VM) {
2920 const zend_op *opline = p->opline;
2921 int len;
2922 bool support_opline;
2923
2924 support_opline =
2925 zend_jit_opline_supports_reg(op_array, ssa, opline, ssa_op, p);
2926
2927 if (support_opline
2928 && opline->opcode == ZEND_ASSIGN
2929 && opline->op1_type == IS_CV
2930 && ssa_op->op1_def >= 0
2931 && ssa->vars[ssa_op->op1_def].alias != NO_ALIAS) {
2932 /* avoid register allocation in case of possibility of indirect modification*/
2933 support_opline = 0;
2934 }
2935
2936 if (ssa_op->op1_use >= 0
2937 && RA_HAS_IVAL(ssa_op->op1_use)) {
2938 if (!support_opline) {
2939 RA_IVAL_DEL(ssa_op->op1_use);
2940 count--;
2941 } else if (!zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2942 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain,
2943 ra,
2944 ssa, ssa_opcodes, op_array, op_array_ssa);
2945 if (opline->op1_type != IS_CV) {
2946 if (opline->opcode == ZEND_CASE
2947 || opline->opcode == ZEND_CASE_STRICT
2948 || opline->opcode == ZEND_SWITCH_LONG
2949 || opline->opcode == ZEND_MATCH
2950 || opline->opcode == ZEND_FETCH_LIST_R
2951 || opline->opcode == ZEND_COPY_TMP
2952 || opline->opcode == ZEND_SWITCH_STRING
2953 || opline->opcode == ZEND_FE_FETCH_R
2954 || opline->opcode == ZEND_FE_FETCH_RW
2955 || opline->opcode == ZEND_FETCH_LIST_W
2956 || opline->opcode == ZEND_VERIFY_RETURN_TYPE
2957 || opline->opcode == ZEND_BIND_LEXICAL
2958 || opline->opcode == ZEND_ROPE_ADD) {
2959 /* The value is kept alive and may be used outside of the trace */
2960 RA_IVAL_FLAGS(ssa_op->op1_use) |= ZREG_STORE;
2961 } else {
2962 RA_IVAL_FLAGS(ssa_op->op1_use) |= ZREG_LAST_USE;
2963 }
2964 }
2965 }
2966 }
2967 if (ssa_op->op2_use >= 0
2968 && ssa_op->op2_use != ssa_op->op1_use
2969 && RA_HAS_IVAL(ssa_op->op2_use)) {
2970 if (!support_opline) {
2971 RA_IVAL_DEL(ssa_op->op2_use);
2972 count--;
2973 } else if (!zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op2_use)) {
2974 zend_jit_trace_use_var(idx, ssa_op->op2_use, ssa_op->op2_def, ssa_op->op2_use_chain,
2975 ra,
2976 ssa, ssa_opcodes, op_array, op_array_ssa);
2977 if (opline->op2_type != IS_CV) {
2978 RA_IVAL_FLAGS(ssa_op->op2_use) |= ZREG_LAST_USE;
2979 }
2980 }
2981 }
2982 if (ssa_op->result_use >= 0
2983 && ssa_op->result_use != ssa_op->op1_use
2984 && ssa_op->result_use != ssa_op->op2_use
2985 && RA_HAS_IVAL(ssa_op->result_use)) {
2986 if (!support_opline) {
2987 RA_IVAL_DEL(ssa_op->result_use);
2988 count--;
2989 } else if (!zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->result_use)) {
2990 zend_jit_trace_use_var(idx, ssa_op->result_use, ssa_op->result_def, ssa_op->res_use_chain,
2991 ra,
2992 ssa, ssa_opcodes, op_array, op_array_ssa);
2993 }
2994 }
2995
2996 if (ssa_op->op1_def >= 0) {
2997 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op1.var), idx);
2998 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2999 }
3000 if (ssa_op->op2_def >= 0) {
3001 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op2.var), idx);
3002 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op2.var), ssa_op->op2_def);
3003 }
3004 if (ssa_op->result_def >= 0) {
3005 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->result.var), idx);
3006 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
3007 }
3008
3009 if (support_opline) {
3010 if (ssa_op->result_def >= 0
3011 && (ssa->vars[ssa_op->result_def].use_chain >= 0
3012 || ssa->vars[ssa_op->result_def].phi_use_chain)
3013 && ssa->vars[ssa_op->result_def].alias == NO_ALIAS
3014 && zend_jit_var_supports_reg(ssa, ssa_op->result_def)) {
3015 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
3016 || opline->opcode == ZEND_PRE_INC
3017 || opline->opcode == ZEND_PRE_DEC
3018 || opline->opcode == ZEND_POST_INC
3019 || opline->opcode == ZEND_POST_DEC
3020 || opline->opcode == ZEND_ADD
3021 || opline->opcode == ZEND_SUB
3022 || opline->opcode == ZEND_MUL
3023 || opline->opcode == ZEND_FETCH_DIM_R
3024 || opline->opcode == ZEND_FETCH_OBJ_R
3025 || opline->opcode == ZEND_FETCH_CONSTANT) {
3026 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_DOUBLE)
3027 || (opline->opcode != ZEND_PRE_INC && opline->opcode != ZEND_PRE_DEC)) {
3028 vars_op_array[ssa_op->result_def] = op_array;
3029 RA_IVAL_START(ssa_op->result_def, idx);
3030 count++;
3031 }
3032 }
3033 }
3034 if (ssa_op->op1_def >= 0
3035 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
3036 || ssa->vars[ssa_op->op1_def].phi_use_chain)
3037 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
3038 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)
3039 && (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
3040 || opline->opcode == ZEND_PRE_INC
3041 || opline->opcode == ZEND_PRE_DEC
3042 || opline->opcode == ZEND_POST_INC
3043 || opline->opcode == ZEND_POST_DEC)) {
3044 vars_op_array[ssa_op->op1_def] = op_array;
3045 RA_IVAL_START(ssa_op->op1_def, idx);
3046 count++;
3047 }
3048 if (ssa_op->op2_def >= 0
3049 && (ssa->vars[ssa_op->op2_def].use_chain >= 0
3050 || ssa->vars[ssa_op->op2_def].phi_use_chain)
3051 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS
3052 && zend_jit_var_supports_reg(ssa, ssa_op->op2_def)
3053 && !(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)) {
3054 vars_op_array[ssa_op->op2_def] = op_array;
3055 RA_IVAL_START(ssa_op->op2_def, idx);
3056 count++;
3057 }
3058 }
3059
3060 len = zend_jit_trace_op_len(opline);
3061 switch (opline->opcode) {
3062 case ZEND_ASSIGN_DIM:
3063 case ZEND_ASSIGN_OBJ:
3064 case ZEND_ASSIGN_STATIC_PROP:
3065 case ZEND_ASSIGN_DIM_OP:
3066 case ZEND_ASSIGN_OBJ_OP:
3067 case ZEND_ASSIGN_STATIC_PROP_OP:
3068 case ZEND_ASSIGN_OBJ_REF:
3069 case ZEND_ASSIGN_STATIC_PROP_REF:
3070 case ZEND_FRAMELESS_ICALL_3:
3071 /* OP_DATA */
3072 ssa_op++;
3073 opline++;
3074 if (ssa_op->op1_use >= 0
3075 && RA_HAS_IVAL(ssa_op->op1_use)
3076 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
3077 if (support_opline) {
3078 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain,
3079 ra,
3080 ssa, ssa_opcodes, op_array, op_array_ssa);
3081 if (opline->op1_type != IS_CV) {
3082 RA_IVAL_FLAGS(ssa_op->op1_use) |= ZREG_LAST_USE;
3083 }
3084 } else {
3085 RA_IVAL_DEL(ssa_op->op1_use);
3086 count--;
3087 }
3088 }
3089 if (ssa_op->op1_def >= 0) {
3090 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op1.var), idx);
3091 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3092 if (support_opline
3093 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
3094 || ssa->vars[ssa_op->op1_def].phi_use_chain)
3095 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
3096 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)
3097 && !(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)) {
3098 vars_op_array[ssa_op->op1_def] = op_array;
3099 RA_IVAL_START(ssa_op->op1_def, idx);
3100 count++;
3101 }
3102 }
3103 ssa_op++;
3104 opline++;
3105 idx+=2;
3106 break;
3107 case ZEND_RECV_INIT:
3108 ssa_op++;
3109 opline++;
3110 idx++;
3111 while (opline->opcode == ZEND_RECV_INIT) {
3112 /* RECV_INIT doesn't support registers */
3113 if (ssa_op->result_use >= 0 && RA_HAS_IVAL(ssa_op->result_use)) {
3114 RA_IVAL_DEL(ssa_op->result_use);
3115 count--;
3116 }
3117 if (ssa_op->result_def >= 0) {
3118 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->result.var), idx);
3119 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
3120 }
3121 ssa_op++;
3122 opline++;
3123 idx++;
3124 }
3125 break;
3126 case ZEND_BIND_GLOBAL:
3127 ssa_op++;
3128 opline++;
3129 idx++;
3130 while (opline->opcode == ZEND_BIND_GLOBAL) {
3131 /* BIND_GLOBAL doesn't support registers */
3132 if (ssa_op->op1_def >= 0) {
3133 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op1.var), idx);
3134 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3135 }
3136 ssa_op++;
3137 opline++;
3138 idx++;
3139 }
3140 break;
3141 default:
3142 ssa_op += len;
3143 idx += len;
3144 break;
3145 }
3146 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
3147 /* New call frames */
3148 zend_jit_trace_stack_frame *prev_frame = frame;
3149
3150 /* Clear allocated registers */
3151 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3152 j = STACK_VAR(stack, i);
3153 if (j >= 0 && RA_HAS_IVAL(j) && !(RA_IVAL_FLAGS(j) & ZREG_LAST_USE)) {
3154 RA_IVAL_DEL(j);
3155 count--;
3156 }
3157 }
3158
3159 frame = zend_jit_trace_call_frame(frame, op_array);
3160 frame->prev = prev_frame;
3161 frame->func = (const zend_function*)p->op_array;
3162 stack = frame->stack;
3163 op_array = p->op_array;
3164 jit_extension =
3165 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3166 op_array_ssa = &jit_extension->func_info.ssa;
3167 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3168 for (i = 0; i < op_array->last_var; i++) {
3169 SET_STACK_VAR(stack, i, j);
3170 vars_op_array[j] = op_array;
3171 if (ssa->vars[j].use_chain >= 0
3172 && ssa->vars[j].alias == NO_ALIAS
3173 && zend_jit_var_supports_reg(ssa, j)) {
3174 RA_IVAL_START(j, idx);
3175 RA_IVAL_FLAGS(j) = ZREG_LOAD;
3176 count++;
3177 }
3178 j++;
3179 }
3180 for (i = op_array->last_var; i < op_array->last_var + op_array->T; i++) {
3181 SET_STACK_VAR(stack, i, -1);
3182 }
3183 level++;
3184 } else if (p->op == ZEND_JIT_TRACE_BACK) {
3185 /* Close exiting call frames */
3186 for (i = 0; i < op_array->last_var; i++) {
3187 RA_IVAL_CLOSE(i, idx-1);
3188 }
3189 op_array = p->op_array;
3190 jit_extension =
3191 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3192 op_array_ssa = &jit_extension->func_info.ssa;
3193 frame = zend_jit_trace_ret_frame(frame, op_array);
3194 stack = frame->stack;
3195 if (level == 0) {
3196 /* New return frames */
3197 frame->prev = NULL;
3198 frame->func = (const zend_function*)op_array;
3199 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3200 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3201 SET_STACK_VAR(stack, i, j);
3202 vars_op_array[j] = op_array;
3203 if (ssa->vars[j].use_chain >= 0
3204 && ssa->vars[j].alias == NO_ALIAS
3205 && zend_jit_var_supports_reg(ssa, j)
3206 && !(ssa->var_info[j].type & MAY_BE_GUARD)) {
3207 RA_IVAL_START(j, idx);
3208 RA_IVAL_FLAGS(j) = ZREG_LOAD;
3209 count++;
3210 }
3211 j++;
3212 }
3213 } else {
3214 level--;
3215 }
3216 } else if (p->op == ZEND_JIT_TRACE_END) {
3217 break;
3218 }
3219 }
3220
3221 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3222 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3223 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3224 zend_ssa_phi *phi = ssa->blocks[1].phis;
3225
3226 while (phi) {
3227 i = phi->sources[1];
3228 if (RA_HAS_IVAL(i) && !ssa->vars[phi->ssa_var].no_val) {
3229 RA_IVAL_END(i, idx);
3230 RA_IVAL_FLAGS(i) &= ~ZREG_LAST_USE;
3231 }
3232 phi = phi->next;
3233 }
3234
3235 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
3236 for (i = 0; i < op_array->last_var; i++) {
3237 if (RA_HAS_IVAL(i) && !ssa->vars[i].phi_use_chain) {
3238 RA_IVAL_END(i, idx);
3239 RA_IVAL_FLAGS(i) &= ~ZREG_LAST_USE;
3240 } else {
3241 RA_IVAL_CLOSE(i, idx);
3242 }
3243 }
3244 }
3245 }
3246
3247 if (count) {
3248 for (i = 0; i < ssa->vars_count; i++) {
3249 if (RA_HAS_REG(i)) {
3250 if ((RA_REG_FLAGS(i) & ZREG_LOAD) &&
3251 (RA_REG_FLAGS(i) & ZREG_LAST_USE) &&
3252 (i >= parent_vars_count || STACK_REG(parent_stack, i) == ZREG_NONE) &&
3253 zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0) {
3254 /* skip life range with single use */
3255 RA_REG_DEL(i);
3256 count--;
3257 }
3258 }
3259 }
3260 }
3261
3262 if (count) {
3263 /* SSA resolution */
3264 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3265 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3266 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3267 zend_ssa_phi *phi = ssa->blocks[1].phis;
3268
3269 while (phi) {
3270 int def = phi->ssa_var;
3271 int use = phi->sources[1];
3272
3273 if (RA_HAS_REG(def)) {
3274 if (!RA_HAS_REG(use)) {
3275 RA_REG_FLAGS(def) |= ZREG_LOAD;
3276 if ((RA_REG_FLAGS(def) & ZREG_LAST_USE)
3277 && ssa->vars[def].use_chain >= 0
3278 && !ssa->vars[def].phi_use_chain
3279 && zend_ssa_next_use(ssa->ops, def, ssa->vars[def].use_chain) < 0
3280 ) {
3281 /* remove interval used once */
3282 RA_REG_DEL(def);
3283 count--;
3284 }
3285 } else if ((ssa->var_info[def].type & MAY_BE_ANY) != (ssa->var_info[use].type & MAY_BE_ANY)) {
3286 RA_REG_FLAGS(def) |= ZREG_LOAD;
3287 RA_REG_FLAGS(use) |= ZREG_STORE;
3288 } else {
3289 use = phi->sources[0];
3290 if (zend_jit_var_supports_reg(ssa, use)) {
3291 ZEND_ASSERT(!RA_HAS_REG(use));
3292 RA_REG_START(use, 0);
3293 RA_REG_FLAGS(use) = ZREG_LOAD;
3294 count++;
3295 } else {
3296 RA_REG_FLAGS(def) |= ZREG_LOAD;
3297 }
3298 }
3299 } else if (RA_HAS_REG(use)) {
3300 if (ssa->vars[use].use_chain >= 0) {
3301 RA_REG_FLAGS(use) |= ZREG_STORE; // TODO: ext/opcache/tests/jit/reg_alloc_00[67].phpt ???
3302 } else {
3303 RA_REG_DEL(use);
3304 count--;
3305 }
3306 }
3307 phi = phi->next;
3308 }
3309 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
3310 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
3311 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3312 int var = STACK_VAR(stack, i);
3313 if (var >= 0 && RA_HAS_REG(var)
3314 && !(RA_REG_FLAGS(var) & (ZREG_LOAD|ZREG_STORE|ZREG_LAST_USE))) {
3315 RA_REG_FLAGS(var) |= ZREG_STORE;
3316 }
3317 }
3318 }
3319
3320 if (!count) {
3321 zend_arena_release(&CG(arena), checkpoint);
3322 return NULL;
3323 }
3324
3325 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3326 fprintf(stderr, "---- TRACE %d Live Ranges \"%s\"\n", ZEND_JIT_TRACE_NUM, op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
3327 for (i = 0; i < ssa->vars_count; i++) {
3328 if (RA_HAS_REG(i)) {
3329 fprintf(stderr, "#%d.", i);
3330 uint32_t var_num = ssa->vars[i].var;
3331 zend_dump_var(vars_op_array[i], (var_num < vars_op_array[i]->last_var ? IS_CV : 0), var_num);
3332 if (RA_REG_FLAGS(i) & ZREG_LAST_USE) {
3333 fprintf(stderr, " last_use");
3334 }
3335 if (RA_REG_FLAGS(i) & ZREG_LOAD) {
3336 fprintf(stderr, " load");
3337 }
3338 if (RA_REG_FLAGS(i) & ZREG_STORE) {
3339 fprintf(stderr, " store");
3340 }
3341 fprintf(stderr, "\n");
3342 }
3343 }
3344 fprintf(stderr, "\n");
3345 }
3346
3347 return ra;
3348 }
3349
3350 zend_arena_release(&CG(arena), checkpoint);
3351 return NULL;
3352 }
3353
zend_jit_trace_cleanup_stack(zend_jit_ctx * jit,zend_jit_trace_stack * stack,const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes)3354 static void zend_jit_trace_cleanup_stack(zend_jit_ctx *jit, zend_jit_trace_stack *stack, const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes)
3355 {
3356 if (ssa_op->op1_use >= 0
3357 && jit->ra[ssa_op->op1_use].ref
3358 && (jit->ra[ssa_op->op1_use].flags & ZREG_LAST_USE)
3359 && (ssa_op->op1_use_chain == -1
3360 || zend_ssa_is_no_val_use(ssa_opcodes[ssa_op->op1_use_chain], ssa->ops + ssa_op->op1_use_chain, ssa_op->op1_use))) {
3361 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var));
3362 }
3363 if (ssa_op->op2_use >= 0
3364 && ssa_op->op2_use != ssa_op->op1_use
3365 && jit->ra[ssa_op->op2_use].ref
3366 && (jit->ra[ssa_op->op2_use].flags & ZREG_LAST_USE)
3367 && (ssa_op->op2_use_chain == -1
3368 || zend_ssa_is_no_val_use(ssa_opcodes[ssa_op->op2_use_chain], ssa->ops + ssa_op->op2_use_chain, ssa_op->op2_use))) {
3369 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->op2.var));
3370 }
3371 if (ssa_op->result_use >= 0
3372 && ssa_op->result_use != ssa_op->op1_use
3373 && ssa_op->result_use != ssa_op->op2_use
3374 && jit->ra[ssa_op->result_use].ref
3375 && (jit->ra[ssa_op->result_use].flags & ZREG_LAST_USE)
3376 && (ssa_op->res_use_chain == -1
3377 || zend_ssa_is_no_val_use(ssa_opcodes[ssa_op->res_use_chain], ssa->ops + ssa_op->res_use_chain, ssa_op->result_use))) {
3378 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
3379 }
3380 }
3381
zend_jit_trace_setup_ret_counter(const zend_op * opline,size_t offset)3382 static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offset)
3383 {
3384 zend_op *next_opline = (zend_op*)(opline + 1);
3385
3386 if (JIT_G(hot_return) && !ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags) {
3387 ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL);
3388 if (!ZEND_OP_TRACE_INFO(next_opline, offset)->counter) {
3389 ZEND_OP_TRACE_INFO(next_opline, offset)->counter =
3390 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
3391 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
3392 }
3393 ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN;
3394 next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler;
3395 }
3396 }
3397
zend_jit_may_delay_fetch_this(const zend_op_array * op_array,zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_ssa_op * ssa_op)3398 static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_ssa_op *ssa_op)
3399 {
3400 int var = ssa_op->result_def;
3401 int i;
3402 int use = ssa->vars[var].use_chain;
3403 const zend_op *opline;
3404
3405 if (use < 0
3406 || ssa->vars[var].phi_use_chain
3407 || ssa->ops[use].op1_use != var
3408 || ssa->ops[use].op1_use_chain != -1) {
3409 return 0;
3410 }
3411
3412 opline = ssa_opcodes[use];
3413 if (opline->opcode == ZEND_INIT_METHOD_CALL) {
3414 return (opline->op2_type == IS_CONST &&
3415 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING);
3416 } else if (opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG) {
3417 if (!JIT_G(current_frame)
3418 || !JIT_G(current_frame)->call
3419 || !JIT_G(current_frame)->call->func
3420 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
3421 return 0;
3422 }
3423 } else if (opline->opcode != ZEND_FETCH_OBJ_R
3424 && opline->opcode != ZEND_FETCH_OBJ_IS
3425 && opline->opcode != ZEND_FETCH_OBJ_W
3426 && opline->opcode != ZEND_ASSIGN_OBJ
3427 && opline->opcode != ZEND_ASSIGN_OBJ_OP
3428 && opline->opcode != ZEND_PRE_INC_OBJ
3429 && opline->opcode != ZEND_PRE_DEC_OBJ
3430 && opline->opcode != ZEND_POST_INC_OBJ
3431 && opline->opcode != ZEND_POST_DEC_OBJ) {
3432 return 0;
3433 }
3434
3435 if (opline->op2_type != IS_CONST
3436 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3437 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3438 return 0;
3439 }
3440
3441 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
3442 if (opline->op1_type == IS_CV
3443 && (opline+1)->op1_type == IS_CV
3444 && (opline+1)->op1.var == opline->op1.var) {
3445 /* skip $a->prop += $a; */
3446 return 0;
3447 }
3448 if (!zend_jit_supported_binary_op(
3449 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3450 return 0;
3451 }
3452 }
3453
3454 for (i = ssa->vars[var].definition; i < use; i++) {
3455 if (ssa_opcodes[i]->opcode == ZEND_DO_UCALL
3456 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME
3457 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL
3458 || ssa_opcodes[i]->opcode == ZEND_INCLUDE_OR_EVAL) {
3459 return 0;
3460 }
3461 }
3462
3463 return 1;
3464 }
3465
zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack * stack,uint32_t stack_size)3466 static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size)
3467 {
3468 uint32_t i;
3469
3470 for (i = 0; i < stack_size; i++) {
3471 if (STACK_FLAGS(stack, i) & ~(ZREG_LOAD|ZREG_STORE|ZREG_LAST_USE)) {
3472 return 1;
3473 } else if (STACK_REG(stack, i) != ZREG_NONE) {
3474 return 1;
3475 }
3476 }
3477 return 0;
3478 }
3479
zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num,uint32_t exit_num)3480 static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num)
3481 {
3482 const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
3483 uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags;
3484 uint32_t stack_size;
3485 zend_jit_trace_stack *stack;
3486
3487 if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) {
3488 return 1;
3489 }
3490
3491 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
3492 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
3493 return zend_jit_trace_stack_needs_deoptimization(stack, stack_size);
3494 }
3495
zend_jit_trace_deoptimization(zend_jit_ctx * jit,uint32_t flags,const zend_op * opline,zend_jit_trace_stack * parent_stack,int parent_vars_count,zend_ssa * ssa,zend_jit_trace_stack * stack,zend_jit_exit_const * constants,int8_t func_reg,bool polymorphic_side_trace)3496 static int zend_jit_trace_deoptimization(
3497 zend_jit_ctx *jit,
3498 uint32_t flags,
3499 const zend_op *opline,
3500 zend_jit_trace_stack *parent_stack,
3501 int parent_vars_count,
3502 zend_ssa *ssa,
3503 zend_jit_trace_stack *stack,
3504 zend_jit_exit_const *constants,
3505 int8_t func_reg,
3506 bool polymorphic_side_trace)
3507 {
3508 int i;
3509 int check2 = -1;
3510
3511 // TODO: Merge this loop with the following register LOAD loop to implement parallel move ???
3512 for (i = 0; i < parent_vars_count; i++) {
3513 int8_t reg = STACK_REG(parent_stack, i);
3514
3515 if (STACK_FLAGS(parent_stack, i) == ZREG_CONST) {
3516 uint8_t type = STACK_TYPE(parent_stack, i);
3517
3518 if (type == IS_LONG) {
3519 if (!zend_jit_store_const_long(jit, i,
3520 (zend_long)constants[STACK_REF(parent_stack, i)].i)) {
3521 return 0;
3522 }
3523 } else if (type == IS_DOUBLE) {
3524 if (!zend_jit_store_const_double(jit, i,
3525 constants[STACK_REF(parent_stack, i)].d)) {
3526 return 0;
3527 }
3528 } else {
3529 ZEND_UNREACHABLE();
3530 }
3531 if (stack) {
3532 SET_STACK_TYPE(stack, i, type, 1);
3533 if (jit->ra && jit->ra[i].ref) {
3534 SET_STACK_REF(stack, i, jit->ra[i].ref);
3535 }
3536 }
3537 } else if (STACK_FLAGS(parent_stack, i) == ZREG_TYPE_ONLY) {
3538 uint8_t type = STACK_TYPE(parent_stack, i);
3539
3540 if (!zend_jit_store_type(jit, i, type)) {
3541 return 0;
3542 }
3543 if (stack) {
3544 SET_STACK_TYPE(stack, i, type, 1);
3545 }
3546 } else if (STACK_FLAGS(parent_stack, i) == ZREG_THIS) {
3547 if (polymorphic_side_trace) {
3548 ssa->var_info[i].delayed_fetch_this = 1;
3549 if (stack) {
3550 SET_STACK_REG_EX(stack, i, ZREG_NONE, ZREG_THIS);
3551 }
3552 } else if (!zend_jit_load_this(jit, EX_NUM_TO_VAR(i))) {
3553 return 0;
3554 }
3555 } else if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_ADDREF) {
3556 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(i));
3557 zend_jit_zval_try_addref(jit, dst);
3558 } else if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_COPY) {
3559 ZEND_ASSERT(reg != ZREG_NONE);
3560 ZEND_ASSERT(check2 == -1);
3561 check2 = i;
3562 } else if (STACK_FLAGS(parent_stack, i) & ZREG_SPILL_SLOT) {
3563 if (ssa && ssa->vars[i].no_val) {
3564 /* pass */
3565 } else {
3566 uint8_t type = STACK_TYPE(parent_stack, i);
3567
3568 if (!zend_jit_store_spill_slot(jit, 1 << type, i, reg, STACK_REF(parent_stack, i),
3569 STACK_MEM_TYPE(parent_stack, i) != type)) {
3570 return 0;
3571 }
3572 if (stack) {
3573 if (jit->ra && jit->ra[i].ref) {
3574 SET_STACK_TYPE(stack, i, type, 0);
3575 if ((STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) != 0) {
3576 SET_STACK_REF_EX(stack, i, jit->ra[i].ref, ZREG_LOAD);
3577 } else {
3578 SET_STACK_REF(stack, i, jit->ra[i].ref);
3579 }
3580 } else {
3581 SET_STACK_TYPE(stack, i, type, 1);
3582 }
3583 }
3584 }
3585 } else if (reg != ZREG_NONE) {
3586 if (ssa && ssa->vars[i].no_val) {
3587 /* pass */
3588 } else {
3589 uint8_t type = STACK_TYPE(parent_stack, i);
3590
3591 if (!zend_jit_store_reg(jit, 1 << type, i, reg,
3592 (STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) != 0,
3593 STACK_MEM_TYPE(parent_stack, i) != type)) {
3594 return 0;
3595 }
3596 if (stack) {
3597 if (jit->ra && jit->ra[i].ref) {
3598 SET_STACK_TYPE(stack, i, type, 0);
3599 if ((STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) != 0) {
3600 SET_STACK_REF_EX(stack, i, jit->ra[i].ref, ZREG_LOAD);
3601 } else {
3602 SET_STACK_REF(stack, i, jit->ra[i].ref);
3603 }
3604 } else {
3605 SET_STACK_TYPE(stack, i, type, 1);
3606 }
3607 }
3608 }
3609 }
3610 }
3611
3612 if (check2 != -1) {
3613 int8_t reg = STACK_REG(parent_stack, check2);
3614
3615 ZEND_ASSERT(STACK_FLAGS(parent_stack, check2) == ZREG_ZVAL_COPY);
3616 ZEND_ASSERT(reg != ZREG_NONE);
3617 if (!zend_jit_escape_if_undef(jit, check2, flags, opline, reg)) {
3618 return 0;
3619 }
3620 if (!zend_jit_restore_zval(jit, EX_NUM_TO_VAR(check2), reg)) {
3621 return 0;
3622 }
3623 }
3624
3625 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
3626 if (!zend_jit_save_call_chain(jit, -1)) {
3627 return 0;
3628 }
3629 }
3630
3631 if (flags & ZEND_JIT_EXIT_FREE_OP2) {
3632 const zend_op *op = opline - 1;
3633
3634 if (!zend_jit_free_op(jit, op, -1, op->op2.var)) {
3635 return 0;
3636 }
3637 }
3638
3639 if (flags & ZEND_JIT_EXIT_FREE_OP1) {
3640 const zend_op *op = opline - 1;
3641
3642 if (!zend_jit_free_op(jit, op, -1, op->op1.var)) {
3643 return 0;
3644 }
3645 }
3646
3647 if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
3648 zend_jit_check_exception(jit);
3649 }
3650
3651 if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) {
3652 if (!zend_jit_free_trampoline(jit, func_reg)) {
3653 return 0;
3654 }
3655 }
3656
3657 return 1;
3658 }
3659
zend_jit_trace_set_var_range(zend_ssa_var_info * info,zend_long min,zend_long max)3660 static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, zend_long max)
3661 {
3662 info->has_range = 1;
3663 info->range.min = min;
3664 info->range.max = max;
3665 info->range.underflow = 0;
3666 info->range.overflow = 0;
3667 }
3668
zend_jit_trace_update_condition_ranges(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa,bool exit_if_true)3669 static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, bool exit_if_true)
3670 {
3671 zend_long op1_min, op1_max, op2_min, op2_max;
3672
3673 if ((OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG
3674 || (OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG) {
3675 return;
3676 }
3677
3678 op1_min = OP1_MIN_RANGE();
3679 op1_max = OP1_MAX_RANGE();
3680 op2_min = OP2_MIN_RANGE();
3681 op2_max = OP2_MAX_RANGE();
3682
3683 switch (opline->opcode) {
3684 case ZEND_IS_EQUAL:
3685 case ZEND_CASE:
3686 case ZEND_IS_IDENTICAL:
3687 case ZEND_CASE_STRICT:
3688 case ZEND_IS_NOT_IDENTICAL:
3689 if (!exit_if_true) {
3690 /* op1 == op2 */
3691 if (ssa_op->op1_use >= 0) {
3692 zend_jit_trace_set_var_range(
3693 &ssa->var_info[ssa_op->op1_use],
3694 MAX(op1_min, op2_min),
3695 MIN(op1_max, op2_max));
3696 }
3697 if (ssa_op->op2_use >= 0) {
3698 zend_jit_trace_set_var_range(
3699 &ssa->var_info[ssa_op->op2_use],
3700 MAX(op2_min, op1_min),
3701 MIN(op2_max, op1_max));
3702 }
3703 }
3704 break;
3705 case ZEND_IS_NOT_EQUAL:
3706 if (exit_if_true) {
3707 /* op1 == op2 */
3708 if (ssa_op->op1_use >= 0) {
3709 zend_jit_trace_set_var_range(
3710 &ssa->var_info[ssa_op->op1_use],
3711 MAX(op1_min, op2_min),
3712 MIN(op1_max, op2_max));
3713 }
3714 if (ssa_op->op2_use >= 0) {
3715 zend_jit_trace_set_var_range(
3716 &ssa->var_info[ssa_op->op2_use],
3717 MAX(op2_min, op1_min),
3718 MIN(op2_max, op1_max));
3719 }
3720 }
3721 break;
3722 case ZEND_IS_SMALLER_OR_EQUAL:
3723 if (!exit_if_true) {
3724 /* op1 <= op2 */
3725 if (ssa_op->op1_use >= 0) {
3726 zend_jit_trace_set_var_range(
3727 &ssa->var_info[ssa_op->op1_use],
3728 op1_min,
3729 MIN(op1_max, op2_max));
3730 }
3731 if (ssa_op->op2_use >= 0) {
3732 zend_jit_trace_set_var_range(
3733 &ssa->var_info[ssa_op->op2_use],
3734 MAX(op2_min, op1_min),
3735 op2_max);
3736 }
3737 } else {
3738 /* op1 > op2 */
3739 if (ssa_op->op1_use >= 0) {
3740 zend_jit_trace_set_var_range(
3741 &ssa->var_info[ssa_op->op1_use],
3742 op2_min != ZEND_LONG_MAX ? MAX(op1_min, op2_min + 1) : op1_min,
3743 op1_max);
3744 }
3745 if (ssa_op->op2_use >= 0) {
3746 zend_jit_trace_set_var_range(
3747 &ssa->var_info[ssa_op->op2_use],
3748 op2_min,
3749 op2_max != ZEND_LONG_MIN ?MIN(op2_max, op1_max - 1) : op1_max);
3750 }
3751 }
3752 break;
3753 case ZEND_IS_SMALLER:
3754 if (!exit_if_true) {
3755 /* op1 < op2 */
3756 if (ssa_op->op1_use >= 0) {
3757 zend_jit_trace_set_var_range(
3758 &ssa->var_info[ssa_op->op1_use],
3759 op1_min,
3760 op2_max != ZEND_LONG_MIN ? MIN(op1_max, op2_max - 1) : op1_max);
3761 }
3762 if (ssa_op->op2_use >= 0) {
3763 zend_jit_trace_set_var_range(
3764 &ssa->var_info[ssa_op->op2_use],
3765 op1_min != ZEND_LONG_MAX ? MAX(op2_min, op1_min + 1) : op2_min,
3766 op2_max);
3767 }
3768 } else {
3769 /* op1 >= op2 */
3770 if (ssa_op->op1_use >= 0) {
3771 zend_jit_trace_set_var_range(
3772 &ssa->var_info[ssa_op->op1_use],
3773 MAX(op1_min, op2_min),
3774 op1_max);
3775 }
3776 if (ssa_op->op2_use >= 0) {
3777 zend_jit_trace_set_var_range(
3778 &ssa->var_info[ssa_op->op2_use],
3779 op2_min,
3780 MIN(op2_max, op1_max));
3781 }
3782 }
3783 break;
3784 }
3785 }
3786
zend_jit_may_skip_comparison(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array)3787 static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array)
3788 {
3789 uint8_t prev_opcode;
3790
3791 if (opline->op1_type == IS_CONST
3792 && Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) == IS_LONG
3793 && Z_LVAL_P(RT_CONSTANT(opline, opline->op1)) == 0) {
3794 if (ssa_op->op2_use >= 0) {
3795 if ((ssa_op-1)->op1_def == ssa_op->op2_use) {
3796 ssa_op--;
3797 opline = ssa_opcodes[ssa_op - ssa->ops];
3798 prev_opcode = opline->opcode;
3799 if (prev_opcode == ZEND_PRE_INC
3800 || prev_opcode == ZEND_PRE_DEC
3801 || prev_opcode == ZEND_POST_INC
3802 || prev_opcode == ZEND_POST_DEC) {
3803 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3804 }
3805 } else if ((ssa_op-1)->result_def == ssa_op->op2_use) {
3806 ssa_op--;
3807 opline = ssa_opcodes[ssa_op - ssa->ops];
3808 prev_opcode = opline->opcode;
3809 if (prev_opcode == ZEND_ADD
3810 || prev_opcode == ZEND_SUB) {
3811 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3812 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3813 }
3814 }
3815 }
3816 } else if (opline->op2_type == IS_CONST
3817 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG
3818 && Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) == 0) {
3819 if (ssa_op->op1_use >= 0) {
3820 if ((ssa_op-1)->op1_def == ssa_op->op1_use) {
3821 ssa_op--;
3822 opline = ssa_opcodes[ssa_op - ssa->ops];
3823 prev_opcode = opline->opcode;
3824 if (prev_opcode == ZEND_PRE_INC
3825 || prev_opcode == ZEND_PRE_DEC
3826 || prev_opcode == ZEND_POST_INC
3827 || prev_opcode == ZEND_POST_DEC) {
3828 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3829 }
3830 } else if ((ssa_op-1)->result_def == ssa_op->op1_use) {
3831 ssa_op--;
3832 opline = ssa_opcodes[ssa_op - ssa->ops];
3833 prev_opcode = opline->opcode;
3834 if (prev_opcode == ZEND_ADD
3835 || prev_opcode == ZEND_SUB) {
3836 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3837 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3838 }
3839 }
3840 }
3841 } else {
3842 const zend_ssa_op *prev_ssa_op = ssa_op - 1;
3843 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3844
3845 if ((prev_opcode == ZEND_JMPZ || prev_opcode == ZEND_JMPNZ)
3846 && prev_ssa_op != ssa->ops
3847 && prev_ssa_op->op1_use >= 0
3848 && prev_ssa_op->op1_use == (prev_ssa_op-1)->result_def) {
3849 prev_ssa_op--;
3850 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3851 }
3852
3853 if (ssa_op->op1_use == prev_ssa_op->op1_use
3854 && ssa_op->op2_use == prev_ssa_op->op2_use) {
3855 if (prev_opcode == ZEND_IS_EQUAL
3856 || prev_opcode == ZEND_IS_NOT_EQUAL
3857 || prev_opcode == ZEND_IS_SMALLER
3858 || prev_opcode == ZEND_IS_SMALLER_OR_EQUAL
3859 || prev_opcode == ZEND_CASE
3860 || prev_opcode == ZEND_IS_IDENTICAL
3861 || prev_opcode == ZEND_IS_NOT_IDENTICAL
3862 || prev_opcode == ZEND_CASE_STRICT) {
3863 if (ssa_op->op1_use < 0) {
3864 if (RT_CONSTANT(opline, opline->op1) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op1)) {
3865 return 0;
3866 }
3867 }
3868 if (ssa_op->op2_use < 0) {
3869 if (RT_CONSTANT(opline, opline->op2) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op2)) {
3870 return 0;
3871 }
3872 }
3873 return 1;
3874 }
3875 }
3876 }
3877 return 0;
3878 }
3879
zend_jit_trace_next_is_send_result(const zend_op * opline,zend_jit_trace_rec * p,zend_jit_trace_stack_frame * frame)3880 static bool zend_jit_trace_next_is_send_result(const zend_op *opline,
3881 zend_jit_trace_rec *p,
3882 zend_jit_trace_stack_frame *frame)
3883 {
3884 if (opline->result_type == IS_TMP_VAR
3885 && (p+1)->op == ZEND_JIT_TRACE_VM
3886 && (p+1)->opline == opline + 1
3887 && ((opline+1)->opcode == ZEND_SEND_VAL
3888 || ((opline+1)->opcode == ZEND_SEND_VAL_EX
3889 && frame
3890 && frame->call
3891 && frame->call->func
3892 && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num)))
3893 && (opline+1)->op1_type == IS_TMP_VAR
3894 && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */
3895 && (opline+1)->op1.var == opline->result.var) {
3896
3897 if (frame->call && frame->call->func) {
3898 uint8_t res_type = (p+1)->op1_type;
3899
3900 if (res_type != IS_UNKNOWN && !(res_type & IS_TRACE_REFERENCE) ) {
3901 zend_jit_trace_send_type(opline+1, frame->call, res_type);
3902 }
3903 }
3904 return 1;
3905 }
3906 return 0;
3907 }
3908
zend_jit_find_ssa_var(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t opline_num,uint32_t var_num)3909 static int zend_jit_find_ssa_var(const zend_op_array *op_array,
3910 const zend_ssa *ssa,
3911 uint32_t opline_num,
3912 uint32_t var_num)
3913 {
3914 int ssa_var, j, b = ssa->cfg.map[opline_num];
3915 const zend_basic_block *bb = ssa->cfg.blocks + b;
3916 const zend_ssa_phi *phi;
3917 const zend_ssa_op *ssa_op;
3918 zend_worklist worklist;
3919 ALLOCA_FLAG(use_heap)
3920
3921 while (1) {
3922 ssa_op = ssa->ops + opline_num;
3923 ssa_var = ssa_op->result_def;
3924 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3925 return ssa_var;
3926 }
3927 ssa_var = ssa_op->op2_def;
3928 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3929 return ssa_var;
3930 }
3931 ssa_var = ssa_op->op1_def;
3932 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3933 return ssa_var;
3934 }
3935 if (opline_num == bb->start) {
3936 break;
3937 }
3938 opline_num--;
3939 }
3940 phi = ssa->blocks[b].phis;
3941 ssa_var = -1;
3942 while (phi) {
3943 if (phi->var == var_num) {
3944 ssa_var = phi->ssa_var;
3945 }
3946 phi = phi->next;
3947 }
3948 if (ssa_var >= 0) {
3949 return ssa_var;
3950 }
3951
3952 if (!bb->predecessors_count) {
3953 return -1;
3954 }
3955
3956 ZEND_WORKLIST_ALLOCA(&worklist, ssa->cfg.blocks_count, use_heap);
3957
3958 for (j = 0; j < bb->predecessors_count; j++) {
3959 b = ssa->cfg.predecessors[bb->predecessor_offset + j];
3960 zend_worklist_push(&worklist, b);
3961 }
3962
3963 while (zend_worklist_len(&worklist) != 0) {
3964 b = zend_worklist_pop(&worklist);
3965 bb = &ssa->cfg.blocks[b];
3966 if (bb->len) {
3967 opline_num = bb->start + bb->len - 1;
3968 while (1) {
3969 ssa_op = ssa->ops + opline_num;
3970 ssa_var = ssa_op->result_def;
3971 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3972 goto found;
3973 }
3974 ssa_var = ssa_op->op2_def;
3975 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3976 goto found;
3977 }
3978 ssa_var = ssa_op->op1_def;
3979 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3980 goto found;
3981 }
3982 if (opline_num == bb->start) {
3983 break;
3984 }
3985 opline_num--;
3986 }
3987 }
3988 phi = ssa->blocks[b].phis;
3989 ssa_var = -1;
3990 while (phi) {
3991 if (phi->var == var_num) {
3992 ssa_var = phi->ssa_var;
3993 }
3994 phi = phi->next;
3995 }
3996 if (ssa_var >= 0) {
3997 goto found;
3998 }
3999 for (j = 0; j < bb->predecessors_count; j++) {
4000 b = ssa->cfg.predecessors[bb->predecessor_offset + j];
4001 zend_worklist_push(&worklist, b);
4002 }
4003 }
4004 ssa_var = -1;
4005
4006 found:
4007 ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
4008 return ssa_var;
4009 }
4010
zend_jit_trace_must_store_type(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t opline_num,uint32_t var_num,uint8_t type)4011 static bool zend_jit_trace_must_store_type(const zend_op_array *op_array,
4012 const zend_ssa *ssa,
4013 uint32_t opline_num,
4014 uint32_t var_num,
4015 uint8_t type)
4016 {
4017 if (ssa->var_info) {
4018 int ssa_var = zend_jit_find_ssa_var(op_array, ssa, opline_num, var_num);
4019
4020 if (ssa_var >= 0) {
4021 if ((ssa->var_info[ssa_var].type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1U << type)) {
4022 return 0;
4023 }
4024 }
4025 }
4026 return 1;
4027 }
4028
zend_jit_trace_may_throw(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,const zend_ssa * ssa,uint32_t t1,uint32_t t2,uint32_t t3,uint32_t val_type)4029 static bool zend_jit_trace_may_throw(const zend_op *opline,
4030 const zend_ssa_op *ssa_op,
4031 const zend_op_array *op_array,
4032 const zend_ssa *ssa,
4033 uint32_t t1,
4034 uint32_t t2,
4035 uint32_t t3,
4036 uint32_t val_type)
4037 {
4038 switch (opline->opcode) {
4039 case ZEND_ASSIGN_DIM_OP:
4040 if (opline->extended_value != ZEND_CONCAT
4041 && val_type == IS_LONG
4042 && (t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_ARRAY
4043 && MAY_BE_PACKED_ONLY(t1)
4044 && !(t1 & MAY_BE_ARRAY_OF_REF)
4045 && (t2 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_LONG
4046 && (t3 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_LONG) {
4047 return 0;
4048 }
4049 break;
4050 default:
4051 break;
4052 }
4053 return zend_may_throw_ex(opline, ssa_op, op_array, ssa, t1, t2);
4054 }
4055
zend_jit_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num)4056 static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num)
4057 {
4058 const void *handler = NULL;
4059 zend_jit_ctx ctx;
4060 zend_jit_ctx *jit = &ctx;
4061 zend_jit_reg_var *ra = NULL;
4062 zend_script *script = NULL;
4063 zend_string *name = NULL;
4064 void *checkpoint;
4065 const zend_op_array *op_array;
4066 zend_ssa *ssa, *op_array_ssa;
4067 const zend_op **ssa_opcodes;
4068 zend_jit_trace_rec *p;
4069 zend_jit_op_array_trace_extension *jit_extension;
4070 int num_op_arrays = 0;
4071 zend_jit_trace_info *t;
4072 const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS];
4073 uint8_t smart_branch_opcode;
4074 const void *exit_addr;
4075 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info, op1_mem_info;
4076 bool send_result = 0;
4077 bool skip_comparison;
4078 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
4079 zend_class_entry *ce;
4080 bool ce_is_instanceof;
4081 bool on_this = 0;
4082 bool delayed_fetch_this = 0;
4083 bool avoid_refcounting = 0;
4084 bool polymorphic_side_trace =
4085 parent_trace &&
4086 (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL);
4087 uint32_t i;
4088 zend_jit_trace_stack_frame *frame, *top, *call;
4089 zend_jit_trace_stack *stack;
4090 uint8_t res_type = IS_UNKNOWN;
4091 const zend_op *opline, *orig_opline;
4092 const zend_ssa_op *ssa_op, *orig_ssa_op;
4093 int checked_stack;
4094 int peek_checked_stack;
4095 uint32_t frame_flags = 0;
4096
4097 JIT_G(current_trace) = trace_buffer;
4098
4099 checkpoint = zend_arena_checkpoint(CG(arena));
4100
4101 ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays);
4102
4103 if (!ssa) {
4104 goto jit_cleanup;
4105 }
4106
4107 ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
4108
4109 op_array = trace_buffer->op_array;
4110 opline = trace_buffer[1].opline;
4111 name = zend_jit_trace_name(op_array, opline->lineno);
4112 zend_jit_trace_start(&ctx, op_array, ssa, name, ZEND_JIT_TRACE_NUM,
4113 parent_trace ? &zend_jit_traces[parent_trace] : NULL, exit_num);
4114 ctx.trace = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
4115
4116 /* Register allocation */
4117 if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL))
4118 && JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4119 ctx.ra = ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num);
4120 }
4121
4122 p = trace_buffer;
4123 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
4124 op_array = p->op_array;
4125 frame = JIT_G(current_frame);
4126 top = zend_jit_trace_call_frame(frame, op_array);
4127 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
4128 frame->used_stack = checked_stack = peek_checked_stack = 0;
4129 stack = frame->stack;
4130 for (i = 0; i < op_array->last_var + op_array->T; i++) {
4131 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4132 }
4133
4134 opline = p[1].opline;
4135 p += ZEND_JIT_TRACE_START_REC_SIZE;
4136
4137 jit_extension =
4138 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
4139 op_array_ssa = &jit_extension->func_info.ssa;
4140
4141 if (!parent_trace) {
4142 zend_jit_set_last_valid_opline(&ctx, opline);
4143 zend_jit_track_last_valid_opline(&ctx);
4144 } else {
4145 if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
4146 zend_jit_trace_opline_guard(&ctx, opline);
4147 } else {
4148 zend_jit_reset_last_valid_opline(&ctx);
4149 }
4150 }
4151
4152 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4153 int last_var;
4154 int parent_vars_count = 0;
4155 zend_jit_trace_stack *parent_stack = NULL;
4156 int used_stack = ((zend_tssa*)ssa)->used_stack;
4157
4158 if (used_stack > 0) {
4159 peek_checked_stack = used_stack;
4160 if (!zend_jit_stack_check(&ctx, opline, used_stack)) {
4161 goto jit_failure;
4162 }
4163 }
4164
4165 if (parent_trace) {
4166 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
4167 op_array->last_var + op_array->T);
4168 if (parent_vars_count) {
4169 parent_stack =
4170 zend_jit_traces[parent_trace].stack_map +
4171 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
4172 }
4173 }
4174
4175 last_var = op_array->last_var;
4176 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4177 last_var += op_array->T;
4178 }
4179
4180 for (i = 0; i < last_var; i++) {
4181 uint32_t info = ssa->var_info[i].type;
4182
4183 if (!(info & MAY_BE_GUARD) && has_concrete_type(info)) {
4184 uint8_t type, mem_type;
4185
4186 type = concrete_type(info);
4187 if (i < parent_vars_count
4188 && STACK_TYPE(parent_stack, i) == type) {
4189 mem_type = STACK_MEM_TYPE(parent_stack, i);
4190 if (mem_type != IS_UNKNOWN) {
4191 SET_STACK_TYPE(stack, i, mem_type, 1);
4192 }
4193 SET_STACK_TYPE(stack, i, type, 0);
4194 } else {
4195 SET_STACK_TYPE(stack, i, type, 1);
4196 }
4197 } else if (ssa->vars[i].alias != NO_ALIAS) {
4198 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4199 } else if (i < parent_vars_count
4200 && STACK_TYPE(parent_stack, i) != IS_UNKNOWN) {
4201 /* This must be already handled by trace type inference */
4202 ZEND_ASSERT(ssa->vars[i].use_chain < 0 && !ssa->vars[i].phi_use_chain);
4203 SET_STACK_TYPE(stack, i, STACK_TYPE(parent_stack, i), 1);
4204 } else if ((info & MAY_BE_GUARD) != 0
4205 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4206 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4207 || (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET
4208 && EX_VAR_TO_NUM((opline-1)->result.var) == i))
4209 && (ssa->vars[i].use_chain != -1
4210 || (ssa->vars[i].phi_use_chain
4211 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD)))) {
4212 /* Check loop-invariant variable type */
4213 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(i), concrete_type(info))) {
4214 goto jit_failure;
4215 }
4216 info &= ~MAY_BE_GUARD;
4217 ssa->var_info[i].type = info;
4218 SET_STACK_TYPE(stack, i, concrete_type(info), 1);
4219 } else if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER
4220 && op_array->function_name
4221 && i >= op_array->num_args) {
4222 /* This must be already handled by trace type inference */
4223 ZEND_UNREACHABLE();
4224 // SET_STACK_TYPE(stack, i, IS_UNDEF, 1);
4225 }
4226
4227 if ((info & MAY_BE_PACKED_GUARD) != 0
4228 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4229 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4230 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)
4231 && (ssa->vars[i].use_chain != -1
4232 || (ssa->vars[i].phi_use_chain
4233 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_PACKED_GUARD)))) {
4234 if (!zend_jit_packed_guard(&ctx, opline, EX_NUM_TO_VAR(i), info)) {
4235 goto jit_failure;
4236 }
4237 info &= ~MAY_BE_PACKED_GUARD;
4238 ssa->var_info[i].type = info;
4239 }
4240 }
4241
4242 if (parent_trace) {
4243 /* Deoptimization */
4244 if (!zend_jit_trace_deoptimization(&ctx,
4245 zend_jit_traces[parent_trace].exit_info[exit_num].flags,
4246 zend_jit_traces[parent_trace].exit_info[exit_num].opline,
4247 parent_stack, parent_vars_count, ssa, stack,
4248 zend_jit_traces[parent_trace].constants,
4249 zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg,
4250 polymorphic_side_trace)) {
4251 goto jit_failure;
4252 }
4253 }
4254
4255 if (ra
4256 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4257 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4258 for (i = 0; i < last_var; i++) {
4259 if (RA_HAS_REG(i)
4260 && (RA_REG_FLAGS(i) & ZREG_LOAD) != 0
4261 && ra[i].ref != STACK_REF(stack, i)
4262 ) {
4263
4264 if ((ssa->var_info[i].type & MAY_BE_GUARD) != 0) {
4265 uint8_t op_type;
4266
4267 ssa->var_info[i].type &= ~MAY_BE_GUARD;
4268 op_type = concrete_type(ssa->var_info[i].type);
4269 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(i), op_type)) {
4270 goto jit_failure;
4271 }
4272 SET_STACK_TYPE(stack, i, op_type, 1);
4273 }
4274
4275 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
4276 if (!zend_jit_load_var(&ctx, ssa->var_info[i].type, i, i)) {
4277 goto jit_failure;
4278 }
4279 SET_STACK_REF_EX(stack, i, ra[i].ref, ZREG_LOAD);
4280 } else {
4281 SET_STACK_REF_EX(stack, i, IR_NULL, ZREG_LOAD);
4282 }
4283 }
4284 }
4285 }
4286 }
4287
4288 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4289 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4290 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4291
4292 jit->trace_loop_ref = zend_jit_trace_begin_loop(&ctx); /* start of of trace loop */
4293
4294 if (ra) {
4295 zend_ssa_phi *phi = ssa->blocks[1].phis;
4296
4297 /* First try to insert IR Phi */
4298 while (phi) {
4299 if (RA_HAS_REG(phi->ssa_var)
4300 && !(RA_REG_FLAGS(phi->ssa_var) & ZREG_LOAD)) {
4301 zend_jit_trace_gen_phi(&ctx, phi);
4302 SET_STACK_REF(stack, phi->var, ra[phi->ssa_var].ref);
4303 }
4304 phi = phi->next;
4305 }
4306
4307 phi = ssa->blocks[1].phis;
4308 while (phi) {
4309 if (RA_HAS_REG(phi->ssa_var)) {
4310 if (RA_REG_FLAGS(phi->ssa_var) & ZREG_LOAD) {
4311 uint32_t info = ssa->var_info[phi->ssa_var].type;
4312
4313 if (info & MAY_BE_GUARD) {
4314 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(phi->var), concrete_type(info))) {
4315 goto jit_failure;
4316 }
4317 info &= ~MAY_BE_GUARD;
4318 ssa->var_info[phi->ssa_var].type = info;
4319 SET_STACK_TYPE(stack, phi->var, concrete_type(info), 1);
4320 }
4321 if (!zend_jit_load_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var)) {
4322 goto jit_failure;
4323 }
4324 SET_STACK_REF_EX(stack, phi->var, ra[phi->ssa_var].ref, ZREG_LOAD);
4325 } else if (RA_REG_FLAGS(phi->ssa_var) & ZREG_STORE) {
4326
4327 if (!zend_jit_store_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var,
4328 STACK_MEM_TYPE(stack, phi->var) != ssa->var_info[phi->ssa_var].type)) {
4329 goto jit_failure;
4330 }
4331 SET_STACK_REF_EX(stack, phi->var, ra[phi->ssa_var].ref, ZREG_STORE);
4332 } else {
4333 /* Register has to be written back on side exit */
4334 SET_STACK_REF(stack, phi->var, ra[phi->ssa_var].ref);
4335 }
4336 }
4337 phi = phi->next;
4338 }
4339 }
4340
4341 // if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4342 // if (ra && dzend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
4343 // uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
4344 //
4345 // timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4346 // if (!timeout_exit_addr) {
4347 // goto jit_failure;
4348 // }
4349 // }
4350 // }
4351
4352 }
4353
4354 ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL;
4355 for (;;p++) {
4356 if (p->op == ZEND_JIT_TRACE_VM) {
4357 uint8_t op1_type = p->op1_type;
4358 uint8_t op2_type = p->op2_type;
4359 uint8_t op3_type = p->op3_type;
4360 uint8_t orig_op1_type = op1_type;
4361 uint8_t orig_op2_type = op2_type;
4362 uint8_t val_type = IS_UNKNOWN;
4363 bool op1_indirect;
4364 zend_class_entry *op1_ce = NULL;
4365 zend_class_entry *op2_ce = NULL;
4366 bool gen_handler = false;
4367
4368 opline = p->opline;
4369 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4370 op1_type = IS_UNKNOWN;
4371 }
4372 if (op1_type != IS_UNKNOWN) {
4373 op1_type &= ~IS_TRACE_PACKED;
4374 }
4375 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4376 op2_type = IS_UNKNOWN;
4377 }
4378 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4379 op3_type = IS_UNKNOWN;
4380 }
4381
4382 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
4383 op1_ce = (zend_class_entry*)(p+1)->ce;
4384 p++;
4385 }
4386 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4387 op2_ce = (zend_class_entry*)(p+1)->ce;
4388 p++;
4389 }
4390 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
4391 val_type = (p+1)->op1_type;
4392 p++;
4393 }
4394
4395 frame_flags = 0;
4396
4397 switch (opline->opcode) {
4398 case ZEND_INIT_FCALL:
4399 case ZEND_INIT_FCALL_BY_NAME:
4400 case ZEND_INIT_NS_FCALL_BY_NAME:
4401 case ZEND_INIT_METHOD_CALL:
4402 case ZEND_INIT_DYNAMIC_CALL:
4403 case ZEND_INIT_STATIC_METHOD_CALL:
4404 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
4405 case ZEND_INIT_USER_CALL:
4406 case ZEND_NEW:
4407 frame->call_level++;
4408 }
4409
4410 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4411 switch (opline->opcode) {
4412 case ZEND_PRE_INC:
4413 case ZEND_PRE_DEC:
4414 case ZEND_POST_INC:
4415 case ZEND_POST_DEC:
4416 if (opline->op1_type != IS_CV) {
4417 break;
4418 }
4419 op1_info = OP1_INFO();
4420 CHECK_OP1_TRACE_TYPE();
4421 if (!(op1_info & MAY_BE_LONG)) {
4422 break;
4423 }
4424 if (opline->result_type != IS_UNUSED) {
4425 res_use_info = zend_jit_trace_type_to_info(
4426 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
4427 if (opline->result_type == IS_CV) {
4428 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4429 }
4430 res_info = RES_INFO();
4431 res_addr = RES_REG_ADDR();
4432 } else {
4433 res_use_info = -1;
4434 res_info = -1;
4435 res_addr = 0;
4436 }
4437 op1_def_info = OP1_DEF_INFO();
4438 if (op1_def_info & MAY_BE_GUARD
4439 && !has_concrete_type(op1_def_info)) {
4440 op1_def_info &= ~MAY_BE_GUARD;
4441 }
4442 if (!zend_jit_inc_dec(&ctx, opline,
4443 op1_info, OP1_REG_ADDR(),
4444 op1_def_info, OP1_DEF_REG_ADDR(),
4445 res_use_info, res_info,
4446 res_addr,
4447 (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4448 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4449 goto jit_failure;
4450 }
4451 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4452 && !(op1_info & MAY_BE_STRING)) {
4453 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4454 if (opline->result_type != IS_UNUSED) {
4455 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4456 }
4457 } else if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD)
4458 && !(op1_info & MAY_BE_STRING)) {
4459 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4460 if (opline->result_type != IS_UNUSED) {
4461 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4462 }
4463 }
4464 if (opline->result_type != IS_UNUSED
4465 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4466 && !(op1_info & MAY_BE_STRING)) {
4467 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4468 } else if (opline->result_type != IS_UNUSED
4469 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD)
4470 && !(res_info & MAY_BE_STRING)) {
4471 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4472 }
4473 goto done;
4474 case ZEND_BW_OR:
4475 case ZEND_BW_AND:
4476 case ZEND_BW_XOR:
4477 case ZEND_SL:
4478 case ZEND_SR:
4479 case ZEND_MOD:
4480 op1_info = OP1_INFO();
4481 CHECK_OP1_TRACE_TYPE();
4482 op2_info = OP2_INFO();
4483 CHECK_OP2_TRACE_TYPE();
4484 if (!(op1_info & MAY_BE_LONG)
4485 || !(op2_info & MAY_BE_LONG)) {
4486 break;
4487 }
4488 res_addr = RES_REG_ADDR();
4489 if (Z_MODE(res_addr) != IS_REG
4490 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4491 send_result = 1;
4492 res_use_info = -1;
4493 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4494 if (!zend_jit_reuse_ip(&ctx)) {
4495 goto jit_failure;
4496 }
4497 } else {
4498 res_use_info = zend_jit_trace_type_to_info(
4499 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
4500 if (opline->result_type == IS_CV) {
4501 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4502 }
4503 }
4504 res_info = RES_INFO();
4505 if (!zend_jit_long_math(&ctx, opline,
4506 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
4507 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
4508 res_use_info, res_info, res_addr,
4509 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4510 goto jit_failure;
4511 }
4512 goto done;
4513 case ZEND_ADD:
4514 case ZEND_SUB:
4515 case ZEND_MUL:
4516 // case ZEND_DIV: // TODO: check for division by zero ???
4517 op1_info = OP1_INFO();
4518 op1_addr = OP1_REG_ADDR();
4519 op2_info = OP2_INFO();
4520 op2_addr = OP2_REG_ADDR();
4521 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4522 break;
4523 }
4524 if (opline->opcode == ZEND_ADD &&
4525 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4526 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4527 /* pass */
4528 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
4529 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4530 break;
4531 }
4532 if (orig_op1_type != IS_UNKNOWN
4533 && (orig_op1_type & IS_TRACE_REFERENCE)
4534 && opline->op1_type == IS_CV
4535 && (orig_op2_type == IS_UNKNOWN || !(orig_op2_type & IS_TRACE_REFERENCE))) {
4536 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4537 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4538 goto jit_failure;
4539 }
4540 if (ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
4541 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
4542 }
4543 } else {
4544 CHECK_OP1_TRACE_TYPE();
4545 }
4546 if (orig_op2_type != IS_UNKNOWN
4547 && (orig_op2_type & IS_TRACE_REFERENCE)
4548 && opline->op2_type == IS_CV
4549 && (orig_op1_type == IS_UNKNOWN || !(orig_op1_type & IS_TRACE_REFERENCE))) {
4550 if (!zend_jit_fetch_reference(&ctx, opline, orig_op2_type, &op2_info, &op2_addr,
4551 !ssa->var_info[ssa_op->op2_use].guarded_reference, 1)) {
4552 goto jit_failure;
4553 }
4554 if (ssa->vars[ssa_op->op2_use].alias == NO_ALIAS) {
4555 ssa->var_info[ssa_op->op2_use].guarded_reference = 1;
4556 }
4557 } else {
4558 CHECK_OP2_TRACE_TYPE();
4559 }
4560 res_addr = RES_REG_ADDR();
4561 if (Z_MODE(res_addr) != IS_REG
4562 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4563 send_result = 1;
4564 res_use_info = -1;
4565 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4566 if (!zend_jit_reuse_ip(&ctx)) {
4567 goto jit_failure;
4568 }
4569 } else {
4570 res_use_info = zend_jit_trace_type_to_info(
4571 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
4572 if (opline->result_type == IS_CV) {
4573 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4574 }
4575 }
4576 res_info = RES_INFO();
4577 if (opline->opcode == ZEND_ADD &&
4578 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4579 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4580 if (!zend_jit_add_arrays(&ctx, opline, op1_info, op1_addr, op2_info, op2_addr, res_addr)) {
4581 goto jit_failure;
4582 }
4583 } else {
4584 bool may_overflow = (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa);
4585
4586 if (ra
4587 && may_overflow
4588 && ((res_info & MAY_BE_GUARD)
4589 && (res_info & MAY_BE_ANY) == MAY_BE_LONG)
4590 && ((opline->opcode == ZEND_ADD
4591 && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)
4592 || (opline->opcode == ZEND_SUB
4593 && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1))) {
4594 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
4595 }
4596 if (!zend_jit_math(&ctx, opline,
4597 op1_info, op1_addr,
4598 op2_info, op2_addr,
4599 res_use_info, res_info, res_addr,
4600 may_overflow,
4601 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4602 goto jit_failure;
4603 }
4604 if (((res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4605 || (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
4606 && has_concrete_type(op1_info)
4607 && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
4608 && has_concrete_type(op2_info)
4609 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4610 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4611 }
4612 }
4613 goto done;
4614 case ZEND_CONCAT:
4615 case ZEND_FAST_CONCAT:
4616 op1_info = OP1_INFO();
4617 CHECK_OP1_TRACE_TYPE();
4618 op2_info = OP2_INFO();
4619 CHECK_OP2_TRACE_TYPE();
4620 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4621 break;
4622 }
4623 if (!(op1_info & MAY_BE_STRING) ||
4624 !(op2_info & MAY_BE_STRING)) {
4625 break;
4626 }
4627 res_addr = RES_REG_ADDR();
4628 if (zend_jit_trace_next_is_send_result(opline, p, frame)) {
4629 send_result = 1;
4630 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4631 if (!zend_jit_reuse_ip(&ctx)) {
4632 goto jit_failure;
4633 }
4634 }
4635 if (!zend_jit_concat(&ctx, opline,
4636 op1_info, op2_info, res_addr,
4637 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4638 goto jit_failure;
4639 }
4640 goto done;
4641 case ZEND_ASSIGN_OP:
4642 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
4643 break;
4644 }
4645 op1_info = OP1_INFO();
4646 CHECK_OP1_TRACE_TYPE();
4647 op2_info = OP2_INFO();
4648 CHECK_OP2_TRACE_TYPE();
4649 if (!zend_jit_supported_binary_op(
4650 opline->extended_value, op1_info, op2_info)) {
4651 break;
4652 }
4653 op1_addr = OP1_REG_ADDR();
4654 if (Z_MODE(op1_addr) != IS_REG
4655 || Z_LOAD(op1_addr)
4656 || Z_STORE(op1_addr)) {
4657 op1_mem_info = op1_info;
4658 } else {
4659 op1_mem_info = zend_jit_trace_type_to_info(
4660 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)));
4661 }
4662 op1_def_info = OP1_DEF_INFO();
4663 if (op1_def_info & MAY_BE_GUARD
4664 && !has_concrete_type(op1_def_info)) {
4665 op1_def_info &= ~MAY_BE_GUARD;
4666 }
4667 if (!zend_jit_assign_op(&ctx, opline,
4668 op1_info, op1_addr, OP1_RANGE(),
4669 op1_def_info, OP1_DEF_REG_ADDR(), op1_mem_info,
4670 op2_info, OP2_REG_ADDR(), OP2_RANGE(),
4671 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4672 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4673 goto jit_failure;
4674 }
4675 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4676 && has_concrete_type(op1_info)
4677 && has_concrete_type(op2_info)) {
4678 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4679 if (opline->result_type != IS_UNUSED) {
4680 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4681 }
4682 }
4683 goto done;
4684 case ZEND_ASSIGN_DIM_OP:
4685 if (opline->result_type != IS_UNUSED) {
4686 break;
4687 }
4688 if (!zend_jit_supported_binary_op(
4689 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4690 break;
4691 }
4692 if (opline->op1_type == IS_CV
4693 && (opline+1)->op1_type == IS_CV
4694 && (opline+1)->op1.var == opline->op1.var) {
4695 /* skip $a[x] += $a; */
4696 break;
4697 }
4698 op1_info = OP1_INFO();
4699 op1_addr = OP1_REG_ADDR();
4700 if (opline->op1_type == IS_VAR) {
4701 if (orig_op1_type != IS_UNKNOWN
4702 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4703 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4704 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4705 goto jit_failure;
4706 }
4707 } else {
4708 break;
4709 }
4710 }
4711 if (orig_op1_type != IS_UNKNOWN
4712 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4713 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4714 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4715 goto jit_failure;
4716 }
4717 if (opline->op1_type == IS_CV
4718 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4719 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4720 }
4721 } else {
4722 CHECK_OP1_TRACE_TYPE();
4723 }
4724 op2_info = OP2_INFO();
4725 CHECK_OP2_TRACE_TYPE();
4726 op1_data_info = OP1_DATA_INFO();
4727 CHECK_OP1_DATA_TRACE_TYPE();
4728 op1_def_info = OP1_DEF_INFO();
4729 if (!zend_jit_assign_dim_op(&ctx, opline,
4730 op1_info, op1_def_info, op1_addr,
4731 op2_info, (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
4732 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
4733 op1_data_info, OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(), val_type,
4734 zend_jit_trace_may_throw(opline, ssa_op, op_array, ssa,
4735 op1_info, op2_info, op1_data_info, val_type))) {
4736 goto jit_failure;
4737 }
4738 goto done;
4739 case ZEND_PRE_INC_OBJ:
4740 case ZEND_PRE_DEC_OBJ:
4741 case ZEND_POST_INC_OBJ:
4742 case ZEND_POST_DEC_OBJ:
4743 if (opline->op2_type != IS_CONST
4744 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4745 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4746 break;
4747 }
4748 ce = NULL;
4749 ce_is_instanceof = 0;
4750 on_this = delayed_fetch_this = 0;
4751 op1_indirect = 0;
4752 if (opline->op1_type == IS_UNUSED) {
4753 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4754 ce = op_array->scope;
4755 /* scope is NULL for closures. */
4756 if (ce) {
4757 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4758 }
4759 op1_addr = 0;
4760 on_this = 1;
4761 } else {
4762 if (ssa_op->op1_use >= 0) {
4763 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4764 }
4765 op1_info = OP1_INFO();
4766 if (!(op1_info & MAY_BE_OBJECT)) {
4767 break;
4768 }
4769 op1_addr = OP1_REG_ADDR();
4770 if (opline->op1_type == IS_VAR) {
4771 if (orig_op1_type != IS_UNKNOWN
4772 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4773 op1_indirect = 1;
4774 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4775 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4776 goto jit_failure;
4777 }
4778 }
4779 }
4780 if (orig_op1_type != IS_UNKNOWN
4781 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4782 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4783 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4784 goto jit_failure;
4785 }
4786 if (opline->op1_type == IS_CV
4787 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4788 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4789 }
4790 } else {
4791 CHECK_OP1_TRACE_TYPE();
4792 }
4793 if (!(op1_info & MAY_BE_OBJECT)) {
4794 break;
4795 }
4796 if (ssa->var_info && ssa->ops) {
4797 if (ssa_op->op1_use >= 0) {
4798 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4799 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4800 ce = op1_ssa->ce;
4801 ce_is_instanceof = op1_ssa->is_instanceof;
4802 }
4803 }
4804 }
4805 if (delayed_fetch_this) {
4806 on_this = 1;
4807 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4808 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4809 } else if (op_array_ssa->ops
4810 && op_array_ssa->vars
4811 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4812 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4813 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4814 }
4815 }
4816 if (!zend_jit_incdec_obj(&ctx, opline, op_array, ssa, ssa_op,
4817 op1_info, op1_addr,
4818 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4819 val_type)) {
4820 goto jit_failure;
4821 }
4822 goto done;
4823 case ZEND_ASSIGN_OBJ_OP:
4824 if (opline->result_type != IS_UNUSED) {
4825 break;
4826 }
4827 if (opline->op2_type != IS_CONST
4828 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4829 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4830 break;
4831 }
4832 if (opline->op1_type == IS_CV
4833 && (opline+1)->op1_type == IS_CV
4834 && (opline+1)->op1.var == opline->op1.var) {
4835 /* skip $a->prop += $a; */
4836 break;
4837 }
4838 if (!zend_jit_supported_binary_op(
4839 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4840 break;
4841 }
4842 ce = NULL;
4843 ce_is_instanceof = 0;
4844 on_this = delayed_fetch_this = 0;
4845 op1_indirect = 0;
4846 if (opline->op1_type == IS_UNUSED) {
4847 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4848 ce = op_array->scope;
4849 /* scope is NULL for closures. */
4850 if (ce) {
4851 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4852 }
4853 op1_addr = 0;
4854 on_this = 1;
4855 } else {
4856 if (ssa_op->op1_use >= 0) {
4857 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4858 }
4859 op1_info = OP1_INFO();
4860 if (!(op1_info & MAY_BE_OBJECT)) {
4861 break;
4862 }
4863 op1_addr = OP1_REG_ADDR();
4864 if (opline->op1_type == IS_VAR) {
4865 if (orig_op1_type != IS_UNKNOWN
4866 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4867 op1_indirect = 1;
4868 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4869 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4870 goto jit_failure;
4871 }
4872 }
4873 }
4874 if (orig_op1_type != IS_UNKNOWN
4875 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4876 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4877 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4878 goto jit_failure;
4879 }
4880 if (opline->op1_type == IS_CV
4881 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4882 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4883 }
4884 } else {
4885 CHECK_OP1_TRACE_TYPE();
4886 }
4887 if (!(op1_info & MAY_BE_OBJECT)) {
4888 break;
4889 }
4890 if (ssa->var_info && ssa->ops) {
4891 if (ssa_op->op1_use >= 0) {
4892 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4893 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4894 ce = op1_ssa->ce;
4895 ce_is_instanceof = op1_ssa->is_instanceof;
4896 }
4897 }
4898 }
4899 if (delayed_fetch_this) {
4900 on_this = 1;
4901 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4902 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4903 } else if (op_array_ssa->ops
4904 && op_array_ssa->vars
4905 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4906 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4907 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4908 }
4909 }
4910 op1_data_info = OP1_DATA_INFO();
4911 CHECK_OP1_DATA_TRACE_TYPE();
4912 if (!zend_jit_assign_obj_op(&ctx, opline, op_array, ssa, ssa_op,
4913 op1_info, op1_addr, op1_data_info, OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(),
4914 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4915 val_type)) {
4916 goto jit_failure;
4917 }
4918 goto done;
4919 case ZEND_ASSIGN_OBJ:
4920 if (opline->op2_type != IS_CONST
4921 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4922 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4923 break;
4924 }
4925 ce = NULL;
4926 ce_is_instanceof = 0;
4927 on_this = delayed_fetch_this = 0;
4928 op1_indirect = 0;
4929 if (opline->op1_type == IS_UNUSED) {
4930 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4931 ce = op_array->scope;
4932 /* scope is NULL for closures. */
4933 if (ce) {
4934 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4935 }
4936 op1_addr = 0;
4937 on_this = 1;
4938 } else {
4939 if (ssa_op->op1_use >= 0) {
4940 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4941 }
4942 op1_info = OP1_INFO();
4943 if (!(op1_info & MAY_BE_OBJECT)) {
4944 break;
4945 }
4946 op1_addr = OP1_REG_ADDR();
4947 if (opline->op1_type == IS_VAR) {
4948 if (orig_op1_type != IS_UNKNOWN
4949 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4950 op1_indirect = 1;
4951 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4952 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4953 goto jit_failure;
4954 }
4955 }
4956 }
4957 if (orig_op1_type != IS_UNKNOWN
4958 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4959 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4960 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4961 goto jit_failure;
4962 }
4963 if (opline->op1_type == IS_CV
4964 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4965 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4966 }
4967 } else {
4968 CHECK_OP1_TRACE_TYPE();
4969 }
4970 if (!(op1_info & MAY_BE_OBJECT)) {
4971 break;
4972 }
4973 if (ssa->var_info && ssa->ops) {
4974 if (ssa_op->op1_use >= 0) {
4975 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4976 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4977 ce = op1_ssa->ce;
4978 ce_is_instanceof = op1_ssa->is_instanceof;
4979 }
4980 }
4981 }
4982 if (delayed_fetch_this) {
4983 on_this = 1;
4984 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4985 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4986 } else if (op_array_ssa->ops
4987 && op_array_ssa->vars
4988 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4989 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4990 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4991 }
4992 }
4993 op1_data_info = OP1_DATA_INFO();
4994 CHECK_OP1_DATA_TRACE_TYPE();
4995 if (!zend_jit_assign_obj(&ctx, opline, op_array, ssa, ssa_op,
4996 op1_info, op1_addr, op1_data_info, OP1_DATA_REG_ADDR(), OP1_DATA_DEF_REG_ADDR(),
4997 (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
4998 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4999 val_type,
5000 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5001 goto jit_failure;
5002 }
5003 if ((opline+1)->op1_type == IS_CV
5004 && (ssa_op+1)->op1_def >= 0
5005 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
5006 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
5007 }
5008 goto done;
5009 case ZEND_ASSIGN_DIM:
5010 op1_info = OP1_INFO();
5011 op1_addr = OP1_REG_ADDR();
5012 if (opline->op1_type == IS_CV
5013 && (opline+1)->op1_type == IS_CV
5014 && (opline+1)->op1.var == opline->op1.var) {
5015 /* skip $a[x] = $a; */
5016 break;
5017 }
5018 if (opline->op1_type == IS_VAR) {
5019 if (orig_op1_type != IS_UNKNOWN
5020 && (orig_op1_type & IS_TRACE_INDIRECT)
5021 && opline->result_type == IS_UNUSED) {
5022 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
5023 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5024 goto jit_failure;
5025 }
5026 } else {
5027 break;
5028 }
5029 }
5030 if (orig_op1_type != IS_UNKNOWN
5031 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5032 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5033 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5034 goto jit_failure;
5035 }
5036 if (opline->op1_type == IS_CV
5037 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5038 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5039 }
5040 } else {
5041 CHECK_OP1_TRACE_TYPE();
5042 }
5043 op2_info = OP2_INFO();
5044 CHECK_OP2_TRACE_TYPE();
5045 op1_data_info = OP1_DATA_INFO();
5046 CHECK_OP1_DATA_TRACE_TYPE();
5047 if (!zend_jit_assign_dim(&ctx, opline,
5048 op1_info, op1_addr,
5049 op2_info, (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
5050 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
5051 op1_data_info, OP1_DATA_REG_ADDR(),
5052 (ctx.ra && (ssa_op+1)->op1_def >= 0) ? OP1_DATA_DEF_REG_ADDR() : 0,
5053 (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
5054 val_type,
5055 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
5056 goto jit_failure;
5057 }
5058 if ((opline+1)->op1_type == IS_CV
5059 && (ssa_op+1)->op1_def >= 0
5060 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
5061 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
5062 }
5063 goto done;
5064 case ZEND_ASSIGN:
5065 if (opline->op1_type != IS_CV) {
5066 break;
5067 }
5068 op2_addr = OP2_REG_ADDR();
5069 op2_info = OP2_INFO();
5070 zend_jit_addr ref_addr = 0;
5071
5072 if (ssa_op->op2_def < 0 || (Z_MODE(op2_addr) == IS_REG && ssa->vars[ssa_op->op2_def].no_val)) {
5073 op2_def_addr = op2_addr;
5074 } else {
5075 op2_def_addr = OP2_DEF_REG_ADDR();
5076 }
5077 CHECK_OP2_TRACE_TYPE();
5078 op1_info = OP1_INFO();
5079 op1_def_info = OP1_DEF_INFO();
5080 if (op1_type != IS_UNKNOWN && (op1_info & MAY_BE_GUARD)) {
5081 if (op1_type < IS_STRING
5082 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (op1_def_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
5083 if (!zend_jit_scalar_type_guard(&ctx, opline, opline->op1.var)) {
5084 goto jit_failure;
5085 }
5086 op1_info &= ~(MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD);
5087 } else {
5088 CHECK_OP1_TRACE_TYPE();
5089 }
5090 }
5091 op1_addr = OP1_REG_ADDR();
5092 op1_def_addr = OP1_DEF_REG_ADDR();
5093 if (Z_MODE(op1_def_addr) != IS_REG &&
5094 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) !=
5095 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var))) {
5096 /* type may be not set */
5097 op1_info |= MAY_BE_NULL;
5098 }
5099 if (orig_op1_type != IS_UNKNOWN) {
5100 if (orig_op1_type & IS_TRACE_REFERENCE) {
5101 if (!zend_jit_guard_reference(&ctx, opline, &op1_addr, &ref_addr,
5102 !ssa->var_info[ssa_op->op1_use].guarded_reference)) {
5103 goto jit_failure;
5104 }
5105 op1_info &= ~MAY_BE_REF;
5106 if (opline->op1_type == IS_CV
5107 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5108 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5109 }
5110 if (opline->result_type == IS_UNUSED) {
5111 res_addr = 0;
5112 } else {
5113 res_addr = RES_REG_ADDR();
5114 if (Z_MODE(res_addr) != IS_REG
5115 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
5116 send_result = 1;
5117 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5118 if (!zend_jit_reuse_ip(&ctx)) {
5119 goto jit_failure;
5120 }
5121 }
5122 }
5123 op1_def_addr = op1_addr;
5124 op1_def_info &= ~MAY_BE_REF;
5125 } else if (op1_info & MAY_BE_REF) {
5126 if (!zend_jit_noref_guard(&ctx, opline, op1_addr)) {
5127 goto jit_failure;
5128 }
5129 op1_info &= ~MAY_BE_REF;
5130 op1_def_info &= ~MAY_BE_REF;
5131 }
5132 }
5133 if (opline->result_type == IS_UNUSED) {
5134 res_addr = 0;
5135 res_info = -1;
5136 } else {
5137 res_addr = RES_REG_ADDR();
5138 res_info = RES_INFO();
5139 if (Z_MODE(res_addr) != IS_REG
5140 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
5141 send_result = 1;
5142 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5143 if (!zend_jit_reuse_ip(&ctx)) {
5144 goto jit_failure;
5145 }
5146 }
5147 }
5148 if (!zend_jit_assign(&ctx, opline,
5149 op1_info, op1_addr,
5150 op1_def_info, op1_def_addr,
5151 op2_info, op2_addr, op2_def_addr,
5152 res_info, res_addr,
5153 ref_addr,
5154 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
5155 goto jit_failure;
5156 }
5157 if (ssa_op->op2_def >= 0
5158 && Z_MODE(op2_addr) == IS_REG
5159 && ssa->vars[ssa_op->op2_def].no_val) {
5160 uint8_t type = (op2_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5161 uint32_t var_num = EX_VAR_TO_NUM(opline->op2.var);
5162
5163 if (STACK_MEM_TYPE(stack, var_num) != type
5164 && ssa->vars[ssa_op->op2_def].use_chain < 0
5165 && !ssa->vars[ssa_op->op2_def].phi_use_chain) {
5166 if (!zend_jit_store_type(&ctx, var_num, type)) {
5167 return 0;
5168 }
5169 SET_STACK_TYPE(stack, var_num, type, 1);
5170 }
5171 }
5172 if (opline->op2_type == IS_CV
5173 && ssa_op->op2_def >= 0
5174 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS) {
5175 ssa->var_info[ssa_op->op2_def].guarded_reference = ssa->var_info[ssa_op->op2_use].guarded_reference;
5176 }
5177 goto done;
5178 case ZEND_CAST:
5179 if (opline->extended_value != op1_type) {
5180 break;
5181 }
5182 ZEND_FALLTHROUGH;
5183 case ZEND_QM_ASSIGN:
5184 op1_addr = OP1_REG_ADDR();
5185 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5186 op1_def_addr = op1_addr;
5187 } else {
5188 op1_def_addr = OP1_DEF_REG_ADDR();
5189 }
5190 op1_info = OP1_INFO();
5191 CHECK_OP1_TRACE_TYPE();
5192 res_info = RES_INFO();
5193 res_use_info = zend_jit_trace_type_to_info(
5194 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
5195 if (opline->result_type == IS_CV) {
5196 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
5197 }
5198 res_addr = RES_REG_ADDR();
5199 if (Z_MODE(res_addr) != IS_REG &&
5200 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) !=
5201 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) {
5202 /* type may be not set */
5203 res_use_info |= MAY_BE_NULL;
5204 }
5205 if (!zend_jit_qm_assign(&ctx, opline,
5206 op1_info, op1_addr, op1_def_addr,
5207 res_use_info, res_info, res_addr)) {
5208 goto jit_failure;
5209 }
5210 if (ssa_op->op1_def >= 0
5211 && Z_MODE(op1_addr) == IS_REG
5212 && ssa->vars[ssa_op->op1_def].no_val) {
5213 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5214 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5215
5216 if (STACK_MEM_TYPE(stack, var_num) != type
5217 && ssa->vars[ssa_op->op1_def].use_chain < 0
5218 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5219 if (!zend_jit_store_type(&ctx, var_num, type)) {
5220 return 0;
5221 }
5222 SET_STACK_TYPE(stack, var_num, type, 1);
5223 }
5224 }
5225 if (opline->op1_type == IS_CV
5226 && ssa_op->op1_def >= 0
5227 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5228 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5229 }
5230 goto done;
5231 case ZEND_INIT_FCALL:
5232 case ZEND_INIT_FCALL_BY_NAME:
5233 case ZEND_INIT_NS_FCALL_BY_NAME:
5234 frame_flags = TRACE_FRAME_MASK_NESTED;
5235 if (!zend_jit_init_fcall(&ctx, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
5236 goto jit_failure;
5237 }
5238 goto done;
5239 case ZEND_SEND_VAL:
5240 case ZEND_SEND_VAL_EX:
5241 if (opline->op2_type == IS_CONST) {
5242 /* Named parameters not supported in JIT */
5243 break;
5244 }
5245 if (opline->opcode == ZEND_SEND_VAL_EX
5246 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5247 break;
5248 }
5249 op1_info = OP1_INFO();
5250 CHECK_OP1_TRACE_TYPE();
5251 if (!zend_jit_send_val(&ctx, opline,
5252 op1_info, OP1_REG_ADDR())) {
5253 goto jit_failure;
5254 }
5255 if (frame->call && frame->call->func) {
5256 if (opline->op1_type == IS_CONST) {
5257 zend_jit_trace_send_type(opline, frame->call, Z_TYPE_P(RT_CONSTANT(opline, opline->op1)));
5258 } else if (op1_type != IS_UNKNOWN) {
5259 if (op1_type == IS_UNDEF) {
5260 op1_type = IS_NULL;
5261 }
5262 zend_jit_trace_send_type(opline, frame->call, op1_type);
5263 }
5264 }
5265 goto done;
5266 case ZEND_SEND_REF:
5267 if (opline->op2_type == IS_CONST) {
5268 /* Named parameters not supported in JIT */
5269 break;
5270 }
5271 op1_info = OP1_INFO();
5272 if (!zend_jit_send_ref(&ctx, opline, op_array,
5273 op1_info, 0)) {
5274 goto jit_failure;
5275 }
5276 if (opline->op1_type == IS_CV
5277 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5278 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5279 }
5280 goto done;
5281 case ZEND_SEND_VAR:
5282 case ZEND_SEND_VAR_EX:
5283 case ZEND_SEND_VAR_NO_REF:
5284 case ZEND_SEND_VAR_NO_REF_EX:
5285 case ZEND_SEND_FUNC_ARG:
5286 if (opline->op2_type == IS_CONST) {
5287 /* Named parameters not supported in JIT */
5288 break;
5289 }
5290 if ((opline->opcode == ZEND_SEND_VAR_EX
5291 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
5292 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5293 break;
5294 }
5295 op1_addr = OP1_REG_ADDR();
5296 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5297 op1_def_addr = op1_addr;
5298 } else {
5299 op1_def_addr = OP1_DEF_REG_ADDR();
5300 }
5301 op1_info = OP1_INFO();
5302 CHECK_OP1_TRACE_TYPE();
5303 if (!zend_jit_send_var(&ctx, opline, op_array,
5304 op1_info, op1_addr, op1_def_addr)) {
5305 goto jit_failure;
5306 }
5307 if (ssa_op->op1_def >= 0
5308 && Z_MODE(op1_addr) == IS_REG
5309 && ssa->vars[ssa_op->op1_def].no_val) {
5310 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5311 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5312
5313 if (STACK_MEM_TYPE(stack, var_num) != type
5314 && ssa->vars[ssa_op->op1_def].use_chain < 0
5315 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5316 if (!zend_jit_store_type(&ctx, var_num, type)) {
5317 return 0;
5318 }
5319 SET_STACK_TYPE(stack, var_num, type, 1);
5320 }
5321 }
5322 if (opline->op1_type == IS_CV
5323 && ssa_op->op1_def >= 0
5324 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5325 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5326 }
5327 if (frame->call && frame->call->func) {
5328 if ((opline->opcode == ZEND_SEND_VAR_EX
5329 || opline->opcode == ZEND_SEND_FUNC_ARG)
5330 && ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
5331 goto done;
5332 }
5333 if (op1_type != IS_UNKNOWN) {
5334 if (op1_type == IS_UNDEF) {
5335 op1_type = IS_NULL;
5336 }
5337 zend_jit_trace_send_type(opline, frame->call, op1_type);
5338 }
5339 }
5340 goto done;
5341 case ZEND_CHECK_FUNC_ARG:
5342 if (!JIT_G(current_frame)
5343 || !JIT_G(current_frame)->call
5344 || !JIT_G(current_frame)->call->func) {
5345 break;
5346 }
5347 if (opline->op2_type == IS_CONST
5348 || opline->op2.num > MAX_ARG_FLAG_NUM) {
5349 /* Named parameters not supported in JIT */
5350 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(JIT_G(current_frame)->call);
5351 break;
5352 }
5353 if (!zend_jit_check_func_arg(&ctx, opline)) {
5354 goto jit_failure;
5355 }
5356 goto done;
5357 case ZEND_CHECK_UNDEF_ARGS:
5358 if (JIT_G(current_frame)
5359 && JIT_G(current_frame)->call) {
5360 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
5361 }
5362 if (!zend_jit_check_undef_args(&ctx, opline)) {
5363 goto jit_failure;
5364 }
5365 goto done;
5366 case ZEND_DO_UCALL:
5367 case ZEND_DO_ICALL:
5368 case ZEND_DO_FCALL_BY_NAME:
5369 case ZEND_DO_FCALL:
5370 if (!zend_jit_do_fcall(&ctx, opline, op_array, op_array_ssa, frame->call_level, -1, p + 1)) {
5371 goto jit_failure;
5372 }
5373 goto done;
5374 case ZEND_IS_EQUAL:
5375 case ZEND_IS_NOT_EQUAL:
5376 case ZEND_IS_SMALLER:
5377 case ZEND_IS_SMALLER_OR_EQUAL:
5378 case ZEND_CASE:
5379 op1_info = OP1_INFO();
5380 op2_info = OP2_INFO();
5381 skip_comparison =
5382 ssa_op != ssa->ops &&
5383 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5384 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5385 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5386 CHECK_OP1_TRACE_TYPE();
5387 CHECK_OP2_TRACE_TYPE();
5388 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5389 bool exit_if_true = 0;
5390 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5391 uint32_t exit_point;
5392
5393 if (ra) {
5394 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5395 }
5396 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5397 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5398 if (!exit_addr) {
5399 goto jit_failure;
5400 }
5401 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5402 if (!zend_jit_cmp(&ctx, opline,
5403 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5404 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5405 RES_REG_ADDR(),
5406 zend_may_throw(opline, ssa_op, op_array, ssa),
5407 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5408 goto jit_failure;
5409 }
5410 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5411 } else {
5412 smart_branch_opcode = 0;
5413 exit_addr = NULL;
5414 if (!zend_jit_cmp(&ctx, opline,
5415 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5416 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5417 RES_REG_ADDR(),
5418 zend_may_throw(opline, ssa_op, op_array, ssa),
5419 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5420 goto jit_failure;
5421 }
5422 }
5423 goto done;
5424 case ZEND_IS_IDENTICAL:
5425 case ZEND_IS_NOT_IDENTICAL:
5426 case ZEND_CASE_STRICT:
5427 op1_info = OP1_INFO();
5428 op2_info = OP2_INFO();
5429 skip_comparison =
5430 ssa_op != ssa->ops &&
5431 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5432 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5433 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5434 CHECK_OP1_TRACE_TYPE();
5435 CHECK_OP2_TRACE_TYPE();
5436 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5437 bool exit_if_true = 0;
5438 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5439 uint32_t exit_point;
5440
5441 if (ra) {
5442 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5443 }
5444 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5445 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5446 if (!exit_addr) {
5447 goto jit_failure;
5448 }
5449 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
5450 exit_if_true = !exit_if_true;
5451 }
5452 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5453 if (!zend_jit_identical(&ctx, opline,
5454 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5455 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5456 RES_REG_ADDR(),
5457 zend_may_throw(opline, ssa_op, op_array, ssa),
5458 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5459 goto jit_failure;
5460 }
5461 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5462 } else {
5463 smart_branch_opcode = 0;
5464 exit_addr = NULL;
5465 if (!zend_jit_identical(&ctx, opline,
5466 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5467 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5468 RES_REG_ADDR(),
5469 zend_may_throw(opline, ssa_op, op_array, ssa),
5470 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5471 goto jit_failure;
5472 }
5473 }
5474 goto done;
5475 case ZEND_DEFINED:
5476 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5477 bool exit_if_true = 0;
5478 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5479 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5480
5481 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5482 if (!exit_addr) {
5483 goto jit_failure;
5484 }
5485 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5486 } else {
5487 smart_branch_opcode = 0;
5488 exit_addr = NULL;
5489 }
5490 if (!zend_jit_defined(&ctx, opline, smart_branch_opcode, -1, -1, exit_addr)) {
5491 goto jit_failure;
5492 }
5493 goto done;
5494 case ZEND_TYPE_CHECK:
5495 if (opline->extended_value == MAY_BE_RESOURCE) {
5496 // TODO: support for is_resource() ???
5497 break;
5498 }
5499 op1_info = OP1_INFO();
5500 CHECK_OP1_TRACE_TYPE();
5501 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5502 bool exit_if_true = 0;
5503 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5504 uint32_t exit_point;
5505
5506 if (ra) {
5507 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5508 }
5509 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5510 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5511 if (!exit_addr) {
5512 goto jit_failure;
5513 }
5514 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5515 } else {
5516 smart_branch_opcode = 0;
5517 exit_addr = NULL;
5518 }
5519 if (!zend_jit_type_check(&ctx, opline, op1_info, smart_branch_opcode, -1, -1, exit_addr)) {
5520 goto jit_failure;
5521 }
5522 goto done;
5523 case ZEND_RETURN:
5524 op1_info = OP1_INFO();
5525 CHECK_OP1_TRACE_TYPE();
5526 if (opline->op1_type == IS_CONST) {
5527 res_type = Z_TYPE_P(RT_CONSTANT(opline, opline->op1));
5528 } else if (op1_type != IS_UNKNOWN) {
5529 res_type = op1_type;
5530 }
5531 if (op_array->type == ZEND_EVAL_CODE
5532 // TODO: support for top-level code
5533 || !op_array->function_name
5534 // TODO: support for IS_UNDEF ???
5535 || (op1_info & MAY_BE_UNDEF)) {
5536 if (!zend_jit_trace_handler(&ctx, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
5537 goto jit_failure;
5538 }
5539 } else {
5540 int j;
5541 int may_throw = 0;
5542 bool left_frame = 0;
5543
5544 if (!zend_jit_return(&ctx, opline, op_array,
5545 op1_info, OP1_REG_ADDR())) {
5546 goto jit_failure;
5547 }
5548 if (op_array->last_var > 100) {
5549 /* To many CVs to unroll */
5550 if (!zend_jit_free_cvs(&ctx)) {
5551 goto jit_failure;
5552 }
5553 left_frame = 1;
5554 }
5555 if (!left_frame) {
5556 for (j = 0 ; j < op_array->last_var; j++) {
5557 uint32_t info;
5558 uint8_t type;
5559
5560 info = zend_ssa_cv_info(op_array, op_array_ssa, j);
5561 type = STACK_TYPE(stack, j);
5562 info = zend_jit_trace_type_to_info_ex(type, info);
5563 if (opline->op1_type == IS_CV
5564 && EX_VAR_TO_NUM(opline->op1.var) == j
5565 && !(op1_info & (MAY_BE_REF|MAY_BE_OBJECT))) {
5566 if (JIT_G(current_frame)
5567 && TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
5568 continue;
5569 } else {
5570 info |= MAY_BE_NULL;
5571 }
5572 }
5573 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
5574 if (!left_frame) {
5575 left_frame = 1;
5576 if (!zend_jit_leave_frame(&ctx)) {
5577 goto jit_failure;
5578 }
5579 }
5580 if (!zend_jit_free_cv(&ctx, info, j)) {
5581 goto jit_failure;
5582 }
5583 if (info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_RESOURCE)) {
5584 if (info & MAY_BE_RC1) {
5585 may_throw = 1;
5586 }
5587 }
5588 }
5589 }
5590 }
5591 if (!zend_jit_leave_func(&ctx, op_array, opline, op1_info, left_frame,
5592 p + 1, &zend_jit_traces[ZEND_JIT_TRACE_NUM],
5593 (op_array_ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, may_throw)) {
5594 goto jit_failure;
5595 }
5596 }
5597 goto done;
5598 case ZEND_BOOL:
5599 case ZEND_BOOL_NOT:
5600 op1_info = OP1_INFO();
5601 CHECK_OP1_TRACE_TYPE();
5602 if (!zend_jit_bool_jmpznz(&ctx, opline,
5603 op1_info, OP1_REG_ADDR(), RES_REG_ADDR(),
5604 -1, -1,
5605 zend_may_throw(opline, ssa_op, op_array, ssa),
5606 opline->opcode, NULL)) {
5607 goto jit_failure;
5608 }
5609 goto done;
5610 case ZEND_JMPZ:
5611 case ZEND_JMPNZ:
5612 case ZEND_JMPZ_EX:
5613 case ZEND_JMPNZ_EX:
5614 op1_info = OP1_INFO();
5615 CHECK_OP1_TRACE_TYPE();
5616 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
5617 const zend_op *exit_opline = NULL;
5618 uint32_t exit_point;
5619
5620 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5621 /* taken branch */
5622 if (opline->opcode == ZEND_JMPNZ_EX) {
5623 smart_branch_opcode = ZEND_JMPZ_EX;
5624 } else if (opline->opcode == ZEND_JMPZ_EX) {
5625 smart_branch_opcode = ZEND_JMPNZ_EX;
5626 } else if (opline->opcode == ZEND_JMPNZ) {
5627 smart_branch_opcode = ZEND_JMPZ;
5628 } else {
5629 smart_branch_opcode = ZEND_JMPNZ;
5630 }
5631 exit_opline = opline + 1;
5632 } else if ((p+1)->opline == opline + 1) {
5633 /* not taken branch */
5634 smart_branch_opcode = opline->opcode;
5635 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5636 } else {
5637 ZEND_UNREACHABLE();
5638 }
5639 if (ra) {
5640 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5641 }
5642 if (!(op1_info & MAY_BE_GUARD)
5643 && has_concrete_type(op1_info)
5644 && concrete_type(op1_info) <= IS_TRUE) {
5645 /* unconditional branch */
5646 exit_addr = NULL;
5647 } else if (opline->result_type == IS_TMP_VAR) {
5648 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
5649
5650 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
5651 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5652 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
5653 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5654 if (!exit_addr) {
5655 goto jit_failure;
5656 }
5657 } else {
5658 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5659 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5660 if (!exit_addr) {
5661 goto jit_failure;
5662 }
5663 }
5664 } else {
5665 ZEND_UNREACHABLE();
5666 }
5667 if (opline->result_type == IS_UNDEF) {
5668 res_addr = 0;
5669 } else {
5670 res_addr = RES_REG_ADDR();
5671 }
5672 if (!zend_jit_bool_jmpznz(&ctx, opline,
5673 op1_info, OP1_REG_ADDR(), res_addr,
5674 -1, -1,
5675 zend_may_throw(opline, ssa_op, op_array, ssa),
5676 smart_branch_opcode, exit_addr)) {
5677 goto jit_failure;
5678 }
5679 goto done;
5680 case ZEND_JMP_FRAMELESS:
5681 op1_info = OP1_INFO();
5682 ZEND_ASSERT((p+1)->op == ZEND_JIT_TRACE_VM);
5683 const zend_op *exit_opline = NULL;
5684 uint32_t exit_point;
5685 zend_jmp_fl_result guard;
5686
5687 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5688 /* taken branch */
5689 guard = ZEND_JMP_FL_HIT;
5690 exit_opline = opline + 1;
5691 } else if ((p+1)->opline == opline + 1) {
5692 /* not taken branch */
5693 guard = ZEND_JMP_FL_MISS;
5694 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5695 } else {
5696 ZEND_UNREACHABLE();
5697 }
5698 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5699 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5700 if (!exit_addr) {
5701 goto jit_failure;
5702 }
5703 if (!zend_jit_jmp_frameless(&ctx, opline, exit_addr, guard)) {
5704 goto jit_failure;
5705 }
5706 goto done;
5707 case ZEND_ISSET_ISEMPTY_CV:
5708 if ((opline->extended_value & ZEND_ISEMPTY)) {
5709 // TODO: support for empty() ???
5710 break;
5711 }
5712 op1_info = OP1_INFO();
5713 op1_addr = OP1_REG_ADDR();
5714 if (orig_op1_type != IS_UNKNOWN
5715 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5716 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5717 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5718 goto jit_failure;
5719 }
5720 if (opline->op1_type == IS_CV
5721 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5722 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5723 }
5724 } else {
5725 CHECK_OP1_TRACE_TYPE();
5726 }
5727 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5728 bool exit_if_true = 0;
5729 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5730 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5731
5732 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5733 if (!exit_addr) {
5734 goto jit_failure;
5735 }
5736 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5737 } else {
5738 smart_branch_opcode = 0;
5739 exit_addr = NULL;
5740 }
5741 if (!zend_jit_isset_isempty_cv(&ctx, opline,
5742 op1_info, op1_addr,
5743 smart_branch_opcode, -1, -1, exit_addr)) {
5744 goto jit_failure;
5745 }
5746 goto done;
5747 case ZEND_IN_ARRAY:
5748 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
5749 break;
5750 }
5751 op1_info = OP1_INFO();
5752 op1_addr = OP1_REG_ADDR();
5753 CHECK_OP1_TRACE_TYPE();
5754 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
5755 break;
5756 }
5757 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5758 bool exit_if_true = 0;
5759 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5760 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5761
5762 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5763 if (!exit_addr) {
5764 goto jit_failure;
5765 }
5766 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5767 } else {
5768 smart_branch_opcode = 0;
5769 exit_addr = NULL;
5770 }
5771 if (!zend_jit_in_array(&ctx, opline,
5772 op1_info, op1_addr,
5773 smart_branch_opcode, -1, -1, exit_addr)) {
5774 goto jit_failure;
5775 }
5776 goto done;
5777 case ZEND_FETCH_DIM_FUNC_ARG:
5778 if (!JIT_G(current_frame)
5779 || !JIT_G(current_frame)->call
5780 || !JIT_G(current_frame)->call->func
5781 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5782 break;
5783 }
5784 ZEND_FALLTHROUGH;
5785 case ZEND_FETCH_DIM_R:
5786 case ZEND_FETCH_DIM_IS:
5787 case ZEND_FETCH_LIST_R:
5788 op1_info = OP1_INFO();
5789 op1_addr = OP1_REG_ADDR();
5790 if (orig_op1_type != IS_UNKNOWN
5791 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5792 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5793 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5794 goto jit_failure;
5795 }
5796 if (opline->op1_type == IS_CV
5797 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5798 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5799 if (ssa_op->op1_def >= 0) {
5800 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5801 }
5802 }
5803 } else {
5804 CHECK_OP1_TRACE_TYPE();
5805 }
5806 op2_info = OP2_INFO();
5807 CHECK_OP2_TRACE_TYPE();
5808 res_info = RES_INFO();
5809 avoid_refcounting =
5810 ssa_op->op1_use >= 0 &&
5811 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5812 if (op1_info & MAY_BE_PACKED_GUARD) {
5813 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5814 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5815 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5816 && MAY_BE_PACKED(op1_info)
5817 && MAY_BE_HASH(op1_info)
5818 && orig_op1_type != IS_UNKNOWN) {
5819 op1_info |= MAY_BE_PACKED_GUARD;
5820 if (orig_op1_type & IS_TRACE_PACKED) {
5821 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5822 if (op1_type != IS_UNKNOWN) {
5823 ssa->var_info[ssa_op->op1_use].type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5824 }
5825 } else {
5826 op1_info &= ~MAY_BE_ARRAY_PACKED;
5827 if (op1_type != IS_UNKNOWN) {
5828 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_ARRAY_PACKED;
5829 }
5830 }
5831 }
5832 if (!zend_jit_fetch_dim_read(&ctx, opline, ssa, ssa_op,
5833 op1_info, op1_addr, avoid_refcounting,
5834 op2_info, OP2_REG_ADDR(), OP2_RANGE(),
5835 res_info, RES_REG_ADDR(), val_type)) {
5836 goto jit_failure;
5837 }
5838 if (ssa_op->op1_def >= 0 && op1_type != IS_UNKNOWN) {
5839 ssa->var_info[ssa_op->op1_def].type = ssa->var_info[ssa_op->op1_use].type;
5840 }
5841 goto done;
5842 case ZEND_FETCH_DIM_W:
5843 case ZEND_FETCH_DIM_RW:
5844 // case ZEND_FETCH_DIM_UNSET:
5845 case ZEND_FETCH_LIST_W:
5846 if (opline->op1_type != IS_CV
5847 && (orig_op1_type == IS_UNKNOWN
5848 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
5849 break;
5850 }
5851 op1_info = OP1_INFO();
5852 op1_addr = OP1_REG_ADDR();
5853 if (opline->op1_type == IS_VAR) {
5854 if (orig_op1_type != IS_UNKNOWN
5855 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5856 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
5857 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5858 goto jit_failure;
5859 }
5860 } else {
5861 break;
5862 }
5863 }
5864 if (orig_op1_type != IS_UNKNOWN
5865 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5866 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5867 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5868 goto jit_failure;
5869 }
5870 if (opline->op1_type == IS_CV
5871 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5872 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5873 }
5874 } else {
5875 CHECK_OP1_TRACE_TYPE();
5876 }
5877 op2_info = OP2_INFO();
5878 CHECK_OP2_TRACE_TYPE();
5879 op1_def_info = OP1_DEF_INFO();
5880 if (!zend_jit_fetch_dim(&ctx, opline,
5881 op1_info, op1_addr,
5882 op2_info, (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
5883 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
5884 RES_REG_ADDR(), val_type)) {
5885 goto jit_failure;
5886 }
5887 if (ssa_op->result_def >= 0
5888 && (opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_LIST_W)
5889 && !(op1_info & (MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))
5890 && !(op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
5891 ssa->var_info[ssa_op->result_def].indirect_reference = 1;
5892 }
5893 goto done;
5894 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
5895 if ((opline->extended_value & ZEND_ISEMPTY)) {
5896 // TODO: support for empty() ???
5897 break;
5898 }
5899 op1_info = OP1_INFO();
5900 op1_addr = OP1_REG_ADDR();
5901 if (orig_op1_type != IS_UNKNOWN
5902 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5903 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5904 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5905 goto jit_failure;
5906 }
5907 if (opline->op1_type == IS_CV
5908 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5909 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5910 }
5911 } else {
5912 CHECK_OP1_TRACE_TYPE();
5913 }
5914 op2_info = OP2_INFO();
5915 CHECK_OP2_TRACE_TYPE();
5916 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5917 bool exit_if_true = 0;
5918 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5919 uint32_t exit_point;
5920 int32_t old_ref = 0;
5921 uint8_t old_flags = 0;
5922
5923 if (ra) {
5924 if (opline->op2_type != IS_CONST) {
5925 old_ref = STACK_REF(stack, EX_VAR_TO_NUM(opline->op2.var));
5926 old_flags = STACK_FLAGS(stack, EX_VAR_TO_NUM(opline->op2.var));
5927 }
5928 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5929 }
5930 if (ssa_op->op1_use >= 0
5931 && ssa->var_info[ssa_op->op1_use].avoid_refcounting) {
5932 /* Temporary reset ZREG_ZVAL_TRY_ADDREF */
5933 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
5934
5935 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
5936 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5937 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
5938 } else {
5939 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5940 }
5941 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5942 if (!exit_addr) {
5943 goto jit_failure;
5944 }
5945 if (old_ref) {
5946 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op2.var), old_ref, old_flags);
5947 }
5948 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5949 } else {
5950 smart_branch_opcode = 0;
5951 exit_addr = NULL;
5952 }
5953 avoid_refcounting =
5954 ssa_op->op1_use >= 0 &&
5955 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5956 if (op1_info & MAY_BE_PACKED_GUARD) {
5957 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5958 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5959 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5960 && MAY_BE_PACKED(op1_info)
5961 && MAY_BE_HASH(op1_info)
5962 && orig_op1_type != IS_UNKNOWN) {
5963 op1_info |= MAY_BE_PACKED_GUARD;
5964 if (orig_op1_type & IS_TRACE_PACKED) {
5965 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5966 } else {
5967 op1_info &= ~MAY_BE_ARRAY_PACKED;
5968 }
5969 }
5970 if (!zend_jit_isset_isempty_dim(&ctx, opline,
5971 op1_info, op1_addr, avoid_refcounting,
5972 op2_info, OP2_REG_ADDR(), OP2_RANGE(), val_type,
5973 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info),
5974 smart_branch_opcode, -1, -1,
5975 exit_addr)) {
5976 goto jit_failure;
5977 }
5978 goto done;
5979 case ZEND_FETCH_OBJ_FUNC_ARG:
5980 if (!JIT_G(current_frame)
5981 || !JIT_G(current_frame)->call
5982 || !JIT_G(current_frame)->call->func
5983 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5984 break;
5985 }
5986 ZEND_FALLTHROUGH;
5987 case ZEND_FETCH_OBJ_R:
5988 case ZEND_FETCH_OBJ_IS:
5989 case ZEND_FETCH_OBJ_W:
5990 on_this = delayed_fetch_this = 0;
5991 avoid_refcounting = 0;
5992 if (opline->op2_type != IS_CONST
5993 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
5994 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
5995 break;
5996 }
5997 ce = NULL;
5998 ce_is_instanceof = 0;
5999 op1_indirect = 0;
6000 if (opline->op1_type == IS_UNUSED) {
6001 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
6002 ce = op_array->scope;
6003 /* scope is NULL for closures. */
6004 if (ce) {
6005 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
6006 }
6007 op1_addr = 0;
6008 on_this = 1;
6009 } else {
6010 op1_info = OP1_INFO();
6011 if (!(op1_info & MAY_BE_OBJECT)) {
6012 break;
6013 }
6014 op1_addr = OP1_REG_ADDR();
6015 if (opline->op1_type == IS_VAR
6016 && opline->opcode == ZEND_FETCH_OBJ_W) {
6017 if (orig_op1_type != IS_UNKNOWN
6018 && (orig_op1_type & IS_TRACE_INDIRECT)) {
6019 op1_indirect = 1;
6020 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
6021 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
6022 goto jit_failure;
6023 }
6024 }
6025 }
6026 if (orig_op1_type != IS_UNKNOWN
6027 && (orig_op1_type & IS_TRACE_REFERENCE)) {
6028 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6029 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6030 goto jit_failure;
6031 }
6032 if (opline->op1_type == IS_CV
6033 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6034 ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = 1;
6035 }
6036 } else {
6037 CHECK_OP1_TRACE_TYPE();
6038 }
6039 if (!(op1_info & MAY_BE_OBJECT)) {
6040 break;
6041 }
6042 if (ssa->var_info && ssa->ops) {
6043 if (ssa_op->op1_use >= 0) {
6044 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
6045 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
6046 ce = op1_ssa->ce;
6047 ce_is_instanceof = op1_ssa->is_instanceof;
6048 }
6049 }
6050 }
6051 if (ssa_op->op1_use >= 0) {
6052 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
6053 avoid_refcounting = ssa->var_info[ssa_op->op1_use].avoid_refcounting;
6054 }
6055 if (delayed_fetch_this) {
6056 on_this = 1;
6057 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
6058 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
6059 } else if (op_array_ssa->ops
6060 && op_array_ssa->vars
6061 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
6062 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
6063 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
6064 }
6065 }
6066 if (!zend_jit_fetch_obj(&ctx, opline, op_array, ssa, ssa_op,
6067 op1_info, op1_addr, op1_indirect, ce, ce_is_instanceof,
6068 on_this, delayed_fetch_this, avoid_refcounting, op1_ce,
6069 RES_REG_ADDR(), val_type,
6070 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, MAY_BE_STRING))) {
6071 goto jit_failure;
6072 }
6073 goto done;
6074 case ZEND_BIND_GLOBAL:
6075 orig_opline = opline;
6076 orig_ssa_op = ssa_op;
6077 while (1) {
6078 if (!ssa->ops || !ssa->var_info) {
6079 op1_info = MAY_BE_ANY|MAY_BE_REF;
6080 } else {
6081 op1_info = OP1_INFO();
6082 }
6083 if (ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
6084 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
6085 }
6086 if (!zend_jit_bind_global(&ctx, opline, op1_info)) {
6087 goto jit_failure;
6088 }
6089 if ((opline+1)->opcode == ZEND_BIND_GLOBAL) {
6090 opline++;
6091 ssa_op++;
6092 } else {
6093 break;
6094 }
6095 }
6096 opline = orig_opline;
6097 ssa_op = orig_ssa_op;
6098 goto done;
6099 case ZEND_RECV:
6100 if (!zend_jit_recv(&ctx, opline, op_array)) {
6101 goto jit_failure;
6102 }
6103 goto done;
6104 case ZEND_RECV_INIT:
6105 orig_opline = opline;
6106 orig_ssa_op = ssa_op;
6107 while (1) {
6108 if (!zend_jit_recv_init(&ctx, opline, op_array,
6109 (opline + 1)->opcode != ZEND_RECV_INIT,
6110 zend_may_throw(opline, ssa_op, op_array, ssa))) {
6111 goto jit_failure;
6112 }
6113 if ((opline+1)->opcode == ZEND_RECV_INIT) {
6114 opline++;
6115 ssa_op++;
6116 } else {
6117 break;
6118 }
6119 }
6120 opline = orig_opline;
6121 ssa_op = orig_ssa_op;
6122 goto done;
6123 case ZEND_FREE:
6124 case ZEND_FE_FREE:
6125 op1_info = OP1_INFO();
6126 if (!zend_jit_free(&ctx, opline, op1_info,
6127 zend_may_throw(opline, ssa_op, op_array, ssa))) {
6128 goto jit_failure;
6129 }
6130 goto done;
6131 case ZEND_ECHO:
6132 op1_info = OP1_INFO();
6133 CHECK_OP1_TRACE_TYPE();
6134 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6135 break;
6136 }
6137 if (!zend_jit_echo(&ctx, opline, op1_info)) {
6138 goto jit_failure;
6139 }
6140 goto done;
6141 case ZEND_STRLEN:
6142 op1_info = OP1_INFO();
6143 op1_addr = OP1_REG_ADDR();
6144 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_STRING)) {
6145 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6146 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6147 goto jit_failure;
6148 }
6149 if (opline->op1_type == IS_CV
6150 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6151 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6152 }
6153 } else {
6154 CHECK_OP1_TRACE_TYPE();
6155 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6156 break;
6157 }
6158 }
6159 if (!zend_jit_strlen(&ctx, opline, op1_info, op1_addr, RES_REG_ADDR())) {
6160 goto jit_failure;
6161 }
6162 goto done;
6163 case ZEND_COUNT:
6164 op1_info = OP1_INFO();
6165 op1_addr = OP1_REG_ADDR();
6166 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_ARRAY)) {
6167 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6168 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6169 goto jit_failure;
6170 }
6171 if (opline->op1_type == IS_CV
6172 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6173 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6174 }
6175 } else {
6176 CHECK_OP1_TRACE_TYPE();
6177 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
6178 break;
6179 }
6180 }
6181 if (!zend_jit_count(&ctx, opline, op1_info, op1_addr, RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
6182 goto jit_failure;
6183 }
6184 goto done;
6185 case ZEND_FETCH_THIS:
6186 delayed_fetch_this = 0;
6187 if (ssa_op->result_def >= 0 && opline->result_type != IS_CV) {
6188 if (zend_jit_may_delay_fetch_this(op_array, ssa, ssa_opcodes, ssa_op)) {
6189 ssa->var_info[ssa_op->result_def].delayed_fetch_this = 1;
6190 delayed_fetch_this = 1;
6191 }
6192 }
6193 if (!zend_jit_fetch_this(&ctx, opline, op_array, delayed_fetch_this)) {
6194 goto jit_failure;
6195 }
6196 goto done;
6197 case ZEND_SWITCH_LONG:
6198 case ZEND_SWITCH_STRING:
6199 case ZEND_MATCH:
6200 if (!zend_jit_switch(&ctx, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) {
6201 goto jit_failure;
6202 }
6203 goto done;
6204 case ZEND_VERIFY_RETURN_TYPE:
6205 if (opline->op1_type == IS_UNUSED) {
6206 /* Always throws */
6207 break;
6208 }
6209 if (opline->op1_type == IS_CONST) {
6210 /* TODO Different instruction format, has return value */
6211 break;
6212 }
6213 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
6214 /* Not worth bothering with */
6215 break;
6216 }
6217 op1_info = OP1_INFO();
6218 CHECK_OP1_TRACE_TYPE();
6219 if (op1_info & MAY_BE_REF) {
6220 /* TODO May need reference unwrapping. */
6221 break;
6222 }
6223 if (!zend_jit_verify_return_type(&ctx, opline, op_array, op1_info)) {
6224 goto jit_failure;
6225 }
6226 goto done;
6227 case ZEND_FE_RESET_R:
6228 op1_info = OP1_INFO();
6229 CHECK_OP1_TRACE_TYPE();
6230 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
6231 break;
6232 }
6233 if (!zend_jit_fe_reset(&ctx, opline, op1_info)) {
6234 goto jit_failure;
6235 }
6236 goto done;
6237 case ZEND_FE_FETCH_R:
6238 op1_info = OP1_INFO();
6239 CHECK_OP1_TRACE_TYPE();
6240 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
6241 break;
6242 }
6243 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
6244 const zend_op *exit_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
6245 uint32_t exit_point;
6246
6247 if ((p+1)->opline == exit_opline) {
6248 /* taken branch (exit from loop) */
6249 exit_opline = opline;
6250 smart_branch_opcode = ZEND_NOP;
6251 } else if ((p+1)->opline == opline + 1) {
6252 /* not taken branch (loop) */
6253 smart_branch_opcode = ZEND_JMP;
6254 } else {
6255 ZEND_UNREACHABLE();
6256 }
6257 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
6258 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6259 if (!exit_addr) {
6260 goto jit_failure;
6261 }
6262 } else {
6263 ZEND_UNREACHABLE();
6264 }
6265 if (!zend_jit_fe_fetch(&ctx, opline, op1_info, OP2_INFO(),
6266 -1, smart_branch_opcode, exit_addr)) {
6267 goto jit_failure;
6268 }
6269 goto done;
6270 case ZEND_FETCH_CONSTANT:
6271 if (!zend_jit_fetch_constant(&ctx, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
6272 goto jit_failure;
6273 }
6274 goto done;
6275 case ZEND_INIT_METHOD_CALL:
6276 if (opline->op2_type != IS_CONST
6277 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
6278 break;
6279 }
6280 on_this = delayed_fetch_this = 0;
6281 ce = NULL;
6282 ce_is_instanceof = 0;
6283 if (opline->op1_type == IS_UNUSED) {
6284 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
6285 ce = op_array->scope;
6286 /* scope is NULL for closures. */
6287 if (ce) {
6288 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
6289 }
6290 op1_addr = 0;
6291 on_this = 1;
6292 } else {
6293 op1_info = OP1_INFO();
6294 op1_addr = OP1_REG_ADDR();
6295 if (polymorphic_side_trace) {
6296 op1_info = MAY_BE_OBJECT;
6297 } else if (orig_op1_type != IS_UNKNOWN
6298 && (orig_op1_type & IS_TRACE_REFERENCE)) {
6299 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6300 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6301 goto jit_failure;
6302 }
6303 if (opline->op1_type == IS_CV
6304 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6305 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6306 }
6307 } else {
6308 CHECK_OP1_TRACE_TYPE();
6309 }
6310 if (ssa->var_info && ssa->ops) {
6311 if (ssa_op->op1_use >= 0) {
6312 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
6313 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
6314 ce = op1_ssa->ce;
6315 ce_is_instanceof = op1_ssa->is_instanceof;
6316 }
6317 }
6318 }
6319 if (ssa_op->op1_use >= 0) {
6320 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
6321 }
6322 if (delayed_fetch_this) {
6323 on_this = 1;
6324 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
6325 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
6326 } else if (op_array_ssa->ops
6327 && op_array_ssa->vars
6328 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
6329 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
6330 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
6331 }
6332 }
6333 frame_flags = TRACE_FRAME_MASK_NESTED;
6334 if (!zend_jit_init_method_call(&ctx, opline,
6335 op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1,
6336 op_array, ssa, ssa_op, frame->call_level,
6337 op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
6338 p + 1, peek_checked_stack - checked_stack,
6339 polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg : -1,
6340 polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_this_reg : -1,
6341 polymorphic_side_trace)) {
6342 goto jit_failure;
6343 }
6344 goto done;
6345 case ZEND_INIT_DYNAMIC_CALL:
6346 if (orig_op2_type != IS_OBJECT || op2_ce != zend_ce_closure) {
6347 break;
6348 }
6349 op2_info = OP2_INFO();
6350 CHECK_OP2_TRACE_TYPE();
6351 frame_flags = TRACE_FRAME_MASK_NESTED;
6352 if (!zend_jit_init_closure_call(&ctx, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
6353 goto jit_failure;
6354 }
6355 goto done;
6356 case ZEND_SEND_ARRAY:
6357 case ZEND_SEND_UNPACK:
6358 if (JIT_G(current_frame)
6359 && JIT_G(current_frame)->call) {
6360 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
6361 }
6362 break;
6363 case ZEND_ROPE_INIT:
6364 case ZEND_ROPE_ADD:
6365 case ZEND_ROPE_END:
6366 op2_info = OP2_INFO();
6367 CHECK_OP2_TRACE_TYPE();
6368 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6369 break;
6370 }
6371 if (!zend_jit_rope(&ctx, opline, op2_info)) {
6372 goto jit_failure;
6373 }
6374 goto done;
6375 case ZEND_FRAMELESS_ICALL_0:
6376 jit_frameless_icall0(jit, opline);
6377 goto done;
6378 case ZEND_FRAMELESS_ICALL_1:
6379 op1_info = OP1_INFO();
6380 jit_frameless_icall1(jit, opline, op1_info);
6381 goto done;
6382 case ZEND_FRAMELESS_ICALL_2:
6383 op1_info = OP1_INFO();
6384 op2_info = OP2_INFO();
6385 jit_frameless_icall2(jit, opline, op1_info, op2_info);
6386 goto done;
6387 case ZEND_FRAMELESS_ICALL_3:
6388 op1_info = OP1_INFO();
6389 op2_info = OP2_INFO();
6390 jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO());
6391 goto done;
6392 default:
6393 break;
6394 }
6395 }
6396
6397 if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) {
6398 gen_handler = 1;
6399 op1_info = OP1_INFO();
6400 op2_info = OP2_INFO();
6401 if (op1_info & MAY_BE_GUARD) {
6402 op1_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6403 }
6404 if (op2_info & MAY_BE_GUARD) {
6405 op2_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6406 }
6407 if (!zend_jit_trace_handler(&ctx, op_array, opline,
6408 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info), p + 1)) {
6409 goto jit_failure;
6410 }
6411 if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func) {
6412 if (opline->opcode == ZEND_NEW && opline->result_type != IS_UNUSED) {
6413 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_OBJECT, 1);
6414 }
6415 if (zend_jit_may_be_polymorphic_call(opline) ||
6416 zend_jit_may_be_modified((p+1)->func, op_array)) {
6417 if (!zend_jit_init_fcall_guard(&ctx, 0, (p+1)->func, opline+1)) {
6418 goto jit_failure;
6419 }
6420 }
6421 }
6422 }
6423
6424 done:
6425 polymorphic_side_trace = 0;
6426 switch (opline->opcode) {
6427 case ZEND_DO_FCALL:
6428 case ZEND_DO_ICALL:
6429 case ZEND_DO_UCALL:
6430 case ZEND_DO_FCALL_BY_NAME:
6431 case ZEND_CALLABLE_CONVERT:
6432 frame->call_level--;
6433 }
6434
6435 if (ra) {
6436 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
6437 }
6438
6439 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
6440 && STACK_FLAGS(stack, EX_VAR_TO_NUM(opline->op1.var)) & (ZREG_ZVAL_ADDREF|ZREG_THIS)) {
6441 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
6442 }
6443
6444 if (opline->opcode == ZEND_ROPE_INIT) {
6445 /* clear stack slots used by rope */
6446 uint32_t var = EX_VAR_TO_NUM(opline->result.var);
6447 uint32_t count =
6448 ((opline->extended_value * sizeof(void*)) + (sizeof(zval)-1)) / sizeof(zval);
6449
6450 do {
6451 SET_STACK_TYPE(stack, var, IS_UNKNOWN, 1);
6452 var++;
6453 count--;
6454 } while (count);
6455 }
6456
6457 if (ssa_op) {
6458 zend_ssa_range tmp;
6459
6460 /* Keep information about known types on abstract stack */
6461 if (ssa_op->result_def >= 0) {
6462 uint8_t type = IS_UNKNOWN;
6463
6464 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0
6465 || send_result) {
6466 /* we didn't set result variable */
6467 type = IS_UNKNOWN;
6468 } else if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6469 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6470 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6471 } else if (opline->opcode == ZEND_QM_ASSIGN) {
6472 if (opline->op1_type != IS_CONST) {
6473 /* copy */
6474 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6475 }
6476 } else if (opline->opcode == ZEND_ASSIGN) {
6477 if (opline->op2_type != IS_CONST
6478 && ssa_op->op1_use >= 0
6479 /* assignment to typed reference may cause conversion */
6480 && (ssa->var_info[ssa_op->op1_use].type & MAY_BE_REF) == 0) {
6481 /* copy */
6482 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6483 }
6484 } else if (opline->opcode == ZEND_POST_INC
6485 || opline->opcode == ZEND_POST_DEC) {
6486 /* copy */
6487 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6488 }
6489 if (opline->opcode == ZEND_JMP_SET
6490 || opline->opcode == ZEND_COALESCE
6491 || opline->opcode == ZEND_JMP_NULL) {
6492 if ((p+1)->op != ZEND_JIT_TRACE_VM) {
6493 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
6494 } else if ((p+1)->opline != (opline + 1)) {
6495 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
6496 }
6497 } else {
6498 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6499 (gen_handler || type == IS_UNKNOWN || !ra || !RA_HAS_REG(ssa_op->result_def)));
6500
6501 if (op_array->last_live_range
6502 && opline->result.var > op_array->last_var
6503 && STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) != type) {
6504 if (!gen_handler && type != IS_UNKNOWN && ra && RA_HAS_REG(ssa_op->result_def)) {
6505 uint32_t var_num = opline->result.var;
6506 uint32_t op_num = opline - op_array->opcodes;
6507 const zend_live_range *range = op_array->live_range;
6508 int j;
6509
6510 op_num += zend_jit_trace_op_len(opline);
6511 for (j = 0; j < op_array->last_live_range; range++, j++) {
6512 if (range->start > op_num) {
6513 /* further blocks will not be relevant... */
6514 break;
6515 } else if (op_num < range->end && var_num == (range->var & ~ZEND_LIVE_MASK)) {
6516 /* check if opcodes in range may throw */
6517 bool store_type = 0;
6518 const zend_ssa_op *next_ssa_op = ssa_op + zend_jit_trace_op_len(opline);
6519 const zend_jit_trace_rec *q = p + 1;
6520
6521 while (1) {
6522 if (q->op != ZEND_JIT_TRACE_VM) {
6523 store_type = 1;
6524 break;
6525 }
6526 op_num = q->opline - op_array->opcodes;
6527 if (op_num >= range->end || op_num < range->start) {
6528 break;
6529 }
6530 if (zend_may_throw(q->opline, next_ssa_op, op_array, ssa)) {
6531 store_type = 1;
6532 break;
6533 }
6534 next_ssa_op += zend_jit_trace_op_len(q->opline);
6535 q++;
6536 }
6537 if (store_type) {
6538 var_num = EX_VAR_TO_NUM(var_num);
6539
6540 if (!zend_jit_store_type(&ctx, var_num, type)) {
6541 return 0;
6542 }
6543 SET_STACK_TYPE(stack, var_num, type, 1);
6544 }
6545 break;
6546 }
6547 }
6548 }
6549 }
6550 if (ssa->var_info[ssa_op->result_def].type & MAY_BE_INDIRECT) {
6551 RESET_STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
6552 }
6553 if (type != IS_UNKNOWN) {
6554 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
6555 if (opline->opcode == ZEND_FETCH_THIS
6556 && delayed_fetch_this) {
6557 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_THIS);
6558 } else if (ssa->var_info[ssa_op->result_def].avoid_refcounting) {
6559 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_ZVAL_ADDREF);
6560 } else if (ra && RA_HAS_REG(ssa_op->result_def)) {
6561 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def].ref,
6562 RA_REG_FLAGS(ssa_op->result_def) & ZREG_STORE);
6563 }
6564 }
6565 }
6566
6567 if (type == IS_LONG
6568 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->result_def, &tmp)) {
6569 ssa->var_info[ssa_op->result_def].range.min = tmp.min;
6570 ssa->var_info[ssa_op->result_def].range.max = tmp.max;
6571 ssa->var_info[ssa_op->result_def].range.underflow = 0;
6572 ssa->var_info[ssa_op->result_def].range.overflow = 0;
6573 ssa->var_info[ssa_op->result_def].has_range = 1;
6574 }
6575 }
6576 if (ssa_op->op1_def >= 0
6577 && ((opline->opcode != ZEND_QM_ASSIGN && opline->opcode != ZEND_CAST)
6578 || opline->result_type != IS_CV
6579 || opline->result.var != opline->op1.var)) {
6580 uint8_t type = IS_UNKNOWN;
6581
6582 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6583 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6584 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6585 } else if (opline->opcode == ZEND_ASSIGN) {
6586 if (!(OP1_INFO() & MAY_BE_REF)
6587 || STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) != IS_UNKNOWN) {
6588 if (opline->op2_type != IS_CONST) {
6589 /* copy */
6590 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6591 }
6592 }
6593 } else if (opline->opcode == ZEND_SEND_VAR
6594 || opline->opcode == ZEND_CAST
6595 || opline->opcode == ZEND_QM_ASSIGN
6596 || opline->opcode == ZEND_JMP_SET
6597 || opline->opcode == ZEND_COALESCE
6598 || opline->opcode == ZEND_JMP_NULL
6599 || opline->opcode == ZEND_FE_RESET_R) {
6600 /* keep old value */
6601 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6602 }
6603 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6604 (gen_handler || type == IS_UNKNOWN || !ra ||
6605 (!RA_HAS_REG(ssa_op->op1_def) &&
6606 !(ssa->vars[ssa_op->op1_def].no_val &&
6607 Z_MODE(OP1_REG_ADDR()) == IS_REG &&
6608 (opline->opcode == ZEND_QM_ASSIGN ||
6609 opline->opcode == ZEND_SEND_VAR ||
6610 opline->opcode == ZEND_SEND_VAR_EX ||
6611 opline->opcode == ZEND_SEND_VAR_NO_REF ||
6612 opline->opcode == ZEND_SEND_VAR_NO_REF_EX ||
6613 opline->opcode == ZEND_SEND_FUNC_ARG)))));
6614 if (type != IS_UNKNOWN) {
6615 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6616 if (ra && RA_HAS_REG(ssa_op->op1_def)) {
6617 uint8_t flags = RA_REG_FLAGS(ssa_op->op1_def) & ZREG_STORE;
6618
6619 if (ssa_op->op1_use >= 0) {
6620 if (opline->opcode == ZEND_SEND_VAR
6621 || opline->opcode == ZEND_CAST
6622 || opline->opcode == ZEND_QM_ASSIGN
6623 || opline->opcode == ZEND_JMP_SET
6624 || opline->opcode == ZEND_COALESCE
6625 || opline->opcode == ZEND_JMP_NULL
6626 || opline->opcode == ZEND_FE_RESET_R) {
6627 if (!RA_HAS_REG(ssa_op->op1_use)) {
6628 flags |= ZREG_LOAD;
6629 }
6630 }
6631 }
6632 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def].ref, flags);
6633 }
6634 }
6635 if (type == IS_LONG
6636 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6637 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6638 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6639 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6640 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6641 ssa->var_info[ssa_op->op1_def].has_range = 1;
6642 }
6643 }
6644 if (ssa_op->op2_def >= 0
6645 && (opline->opcode != ZEND_ASSIGN
6646 || opline->op1_type != IS_CV
6647 || opline->op1.var != opline->op2.var)) {
6648 uint8_t type = IS_UNKNOWN;
6649
6650 if (!(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)
6651 && has_concrete_type(ssa->var_info[ssa_op->op2_def].type)) {
6652 type = concrete_type(ssa->var_info[ssa_op->op2_def].type);
6653 } else if (opline->opcode == ZEND_ASSIGN) {
6654 /* keep old value */
6655 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6656 }
6657 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), type,
6658 (gen_handler || type == IS_UNKNOWN || !ra ||
6659 (!RA_HAS_REG(ssa_op->op2_def) &&
6660 !(ssa->vars[ssa_op->op2_def].no_val &&
6661 Z_MODE(OP2_REG_ADDR()) == IS_REG &&
6662 opline->opcode == ZEND_ASSIGN))));
6663 if (type != IS_UNKNOWN) {
6664 ssa->var_info[ssa_op->op2_def].type &= ~MAY_BE_GUARD;
6665 if (ra && RA_HAS_REG(ssa_op->op2_def)) {
6666 uint8_t flags = RA_REG_FLAGS(ssa_op->op2_def) & ZREG_STORE;
6667
6668 if (ssa_op->op2_use >= 0) {
6669 if (opline->opcode == ZEND_ASSIGN) {
6670 if (!RA_HAS_REG(ssa_op->op2_use)
6671 ) {
6672 flags |= ZREG_LOAD;
6673 }
6674 }
6675 }
6676 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op2.var), ra[ssa_op->op2_def].ref, flags);
6677 }
6678 }
6679 if (type == IS_LONG
6680 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op2_def, &tmp)) {
6681 ssa->var_info[ssa_op->op2_def].range.min = tmp.min;
6682 ssa->var_info[ssa_op->op2_def].range.max = tmp.max;
6683 ssa->var_info[ssa_op->op2_def].range.underflow = 0;
6684 ssa->var_info[ssa_op->op2_def].range.overflow = 0;
6685 ssa->var_info[ssa_op->op2_def].has_range = 1;
6686 }
6687 }
6688
6689 switch (opline->opcode) {
6690 case ZEND_ASSIGN_DIM:
6691 case ZEND_ASSIGN_OBJ:
6692 case ZEND_ASSIGN_STATIC_PROP:
6693 case ZEND_ASSIGN_DIM_OP:
6694 case ZEND_ASSIGN_OBJ_OP:
6695 case ZEND_ASSIGN_STATIC_PROP_OP:
6696 case ZEND_ASSIGN_OBJ_REF:
6697 case ZEND_ASSIGN_STATIC_PROP_REF:
6698 /* OP_DATA */
6699 ssa_op++;
6700 opline++;
6701 if (ssa_op->op1_def >= 0) {
6702 uint8_t type = IS_UNKNOWN;
6703
6704 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6705 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6706 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6707 } else if ((opline-1)->opcode == ZEND_ASSIGN_DIM
6708 || (opline-1)->opcode == ZEND_ASSIGN_OBJ
6709 || (opline-1)->opcode == ZEND_ASSIGN_STATIC_PROP) {
6710 /* keep old value */
6711 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6712 }
6713 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6714 (gen_handler || type == IS_UNKNOWN || !ra || !RA_HAS_REG(ssa_op->op1_def)));
6715 if (type != IS_UNKNOWN) {
6716 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6717 if (ra && RA_HAS_REG(ssa_op->op1_def)) {
6718 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def].ref,
6719 RA_REG_FLAGS(ssa_op->op1_def) & ZREG_STORE);
6720 }
6721 }
6722 if (type == IS_LONG
6723 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6724 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6725 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6726 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6727 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6728 ssa->var_info[ssa_op->op1_def].has_range = 1;
6729 }
6730 }
6731 ssa_op++;
6732 break;
6733 case ZEND_RECV_INIT:
6734 ssa_op++;
6735 opline++;
6736 while (opline->opcode == ZEND_RECV_INIT) {
6737 if (ssa_op->result_def >= 0) {
6738 uint8_t type = IS_UNKNOWN;
6739
6740 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6741 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6742 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6743 }
6744 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6745 (gen_handler || !ra || !RA_HAS_REG(ssa_op->result_def)));
6746 if (ra && RA_HAS_REG(ssa_op->result_def)) {
6747 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def].ref,
6748 RA_REG_FLAGS(ssa_op->result_def) & ZREG_STORE);
6749 }
6750 }
6751 ssa_op++;
6752 opline++;
6753 }
6754 break;
6755 case ZEND_BIND_GLOBAL:
6756 ssa_op++;
6757 opline++;
6758 while (opline->opcode == ZEND_BIND_GLOBAL) {
6759 if (ssa_op->op1_def >= 0) {
6760 uint8_t type = IS_UNKNOWN;
6761
6762 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6763 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6764 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6765 }
6766 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6767 (gen_handler || !ra || !RA_HAS_REG(ssa_op->op1_def)));
6768 if (ra && RA_HAS_REG(ssa_op->op1_def)) {
6769 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def].ref,
6770 RA_REG_FLAGS(ssa_op->op1_def) & ZREG_STORE);
6771 }
6772 }
6773 ssa_op++;
6774 opline++;
6775 }
6776 break;
6777 default:
6778 ssa_op += zend_jit_trace_op_len(opline);
6779 break;
6780 }
6781
6782 if (send_result) {
6783 ssa_op++;
6784 p++;
6785 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
6786 p++;
6787 }
6788 send_result = 0;
6789 }
6790 }
6791 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
6792 call = frame->call;
6793 assert(call && &call->func->op_array == p->op_array);
6794
6795 if (opline->opcode == ZEND_DO_UCALL
6796 || opline->opcode == ZEND_DO_FCALL_BY_NAME
6797 || opline->opcode == ZEND_DO_FCALL) {
6798
6799 frame->call_opline = opline;
6800
6801 /* Check if SEND_UNPACK/SEND_ARRAY may cause enter at different opline */
6802 if (opline > op_array->opcodes) {
6803 const zend_op *prev_opline = opline - 1;
6804
6805 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
6806 prev_opline--;
6807 }
6808 JIT_G(current_frame) = call;
6809 if ((prev_opline->opcode == ZEND_SEND_ARRAY
6810 || prev_opline->opcode == ZEND_SEND_UNPACK
6811 || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS)
6812 && p->op_array->num_args
6813 && (p->op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0
6814 && ((p+1)->op == ZEND_JIT_TRACE_VM
6815 || (p+1)->op == ZEND_JIT_TRACE_END)
6816 && (TRACE_FRAME_NUM_ARGS(call) < 0
6817 || TRACE_FRAME_NUM_ARGS(call) < p->op_array->num_args)
6818 && !zend_jit_trace_opline_guard(&ctx, (p+1)->opline)) {
6819 goto jit_failure;
6820 }
6821 JIT_G(current_frame) = frame;
6822 }
6823 }
6824
6825 if ((p+1)->op == ZEND_JIT_TRACE_END) {
6826 p++;
6827 break;
6828 }
6829 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
6830 if (TRACE_FRAME_IS_THIS_CHECKED(frame)) {
6831 TRACE_FRAME_SET_THIS_CHECKED(call);
6832 }
6833 } else if (op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) {
6834 TRACE_FRAME_SET_THIS_CHECKED(call);
6835 }
6836 op_array = (zend_op_array*)p->op_array;
6837 ctx.current_op_array = op_array;
6838 jit_extension =
6839 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6840 op_array_ssa = &jit_extension->func_info.ssa;
6841 frame->call = call->prev;
6842 call->prev = frame;
6843 if (p->info & ZEND_JIT_TRACE_RETURN_VALUE_USED) {
6844 TRACE_FRAME_SET_RETURN_VALUE_USED(call);
6845 } else {
6846 TRACE_FRAME_SET_RETURN_VALUE_UNUSED(call);
6847 }
6848 JIT_G(current_frame) = frame = call;
6849 stack = frame->stack;
6850 if (ra) {
6851 int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6852
6853 for (i = 0; i < op_array->last_var; i++, j++) {
6854 if (RA_HAS_REG(j) && (RA_REG_FLAGS(j) & ZREG_LOAD) != 0) {
6855 if ((ssa->var_info[j].type & MAY_BE_GUARD) != 0) {
6856 uint8_t op_type;
6857
6858 ssa->var_info[j].type &= ~MAY_BE_GUARD;
6859 op_type = concrete_type(ssa->var_info[j].type);
6860 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(i), op_type)) {
6861 goto jit_failure;
6862 }
6863 SET_STACK_TYPE(stack, i, op_type, 1);
6864 }
6865 if (!zend_jit_load_var(&ctx, ssa->var_info[j].type, i, j)) {
6866 goto jit_failure;
6867 }
6868 SET_STACK_REF_EX(stack, i, ra[j].ref, ZREG_LOAD);
6869 }
6870 }
6871 }
6872 } else if (p->op == ZEND_JIT_TRACE_BACK) {
6873 op_array = (zend_op_array*)p->op_array;
6874 ctx.current_op_array = op_array;
6875 jit_extension =
6876 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6877 op_array_ssa = &jit_extension->func_info.ssa;
6878 top = frame;
6879 if (frame->prev) {
6880 checked_stack = frame->old_checked_stack;
6881 peek_checked_stack = frame->old_peek_checked_stack;
6882 frame = frame->prev;
6883 stack = frame->stack;
6884 ZEND_ASSERT(&frame->func->op_array == op_array);
6885 } else {
6886 frame = zend_jit_trace_ret_frame(frame, op_array);
6887 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
6888 frame->used_stack = checked_stack = peek_checked_stack = 0;
6889 stack = frame->stack;
6890 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6891 uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6892
6893 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6894 /* Initialize abstract stack using SSA */
6895 if (!(ssa->var_info[j].type & MAY_BE_GUARD)
6896 && has_concrete_type(ssa->var_info[j].type)) {
6897 SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[j].type), 1);
6898 } else {
6899 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6900 }
6901 }
6902 if (ra) {
6903 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6904 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6905 if (RA_HAS_REG(j) && (RA_REG_FLAGS(j) & ZREG_LOAD) != 0) {
6906 if (!zend_jit_load_var(&ctx, ssa->var_info[j].type, i, j)) {
6907 goto jit_failure;
6908 }
6909 SET_STACK_REF_EX(stack, i, ra[j].ref, ZREG_LOAD);
6910 }
6911 }
6912 }
6913 } else {
6914 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6915 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6916 }
6917 }
6918 opline = NULL;
6919 }
6920 JIT_G(current_frame) = frame;
6921 if (res_type != IS_UNKNOWN
6922 && (p+1)->op == ZEND_JIT_TRACE_VM) {
6923 const zend_op *opline = (p+1)->opline - 1;
6924 if (opline->result_type != IS_UNUSED) {
6925 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), res_type, 1);
6926 }
6927 }
6928 res_type = IS_UNKNOWN;
6929 } else if (p->op == ZEND_JIT_TRACE_END) {
6930 break;
6931 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
6932 const zend_op *init_opline = zend_jit_trace_find_init_fcall_op(p, op_array);
6933 int num_args = -1;
6934
6935 if (init_opline
6936 && init_opline->extended_value <= TRACE_FRAME_MAX_NUM_ARGS) {
6937 num_args = init_opline->extended_value;
6938 }
6939
6940 call = top;
6941 TRACE_FRAME_INIT(call, p->func, frame_flags, num_args);
6942 call->prev = frame->call;
6943 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
6944 TRACE_FRAME_SET_LAST_SEND_BY_VAL(call);
6945 if (init_opline && init_opline->opcode == ZEND_INIT_DYNAMIC_CALL) {
6946 TRACE_FRAME_SET_CLOSURE_CALL(call);
6947 }
6948 }
6949 if (init_opline) {
6950 if (init_opline->opcode != ZEND_NEW
6951 && (init_opline->opcode != ZEND_INIT_METHOD_CALL
6952 || init_opline->op1_type == IS_UNDEF
6953 || (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6954 && ssa_op
6955 && (ssa_op-1)->op1_use >=0
6956 && ssa->var_info[(ssa_op-1)->op1_use].delayed_fetch_this))
6957 && (init_opline->opcode != ZEND_INIT_USER_CALL
6958 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6959 && (init_opline->opcode != ZEND_INIT_DYNAMIC_CALL
6960 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6961 ) {
6962 TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(call);
6963 } else if (init_opline->opcode == ZEND_NEW
6964 || (init_opline->opcode == ZEND_INIT_METHOD_CALL
6965 && init_opline->op1_type != IS_UNDEF
6966 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6967 && p->func && p->func->common.scope && !(p->func->common.fn_flags & ZEND_ACC_STATIC))) {
6968 TRACE_FRAME_SET_ALWAYS_RELEASE_THIS(call);
6969 }
6970 }
6971 frame->call = call;
6972 top = zend_jit_trace_call_frame(top, p->op_array);
6973 if (p->func) {
6974 if (p->func->type == ZEND_USER_FUNCTION) {
6975 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6976 zend_jit_op_array_trace_extension *jit_extension =
6977 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array);
6978
6979 i = 0;
6980 while (i < p->op_array->num_args) {
6981 /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */
6982 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6983 i++;
6984 }
6985 while (i < p->op_array->last_var) {
6986 if (jit_extension
6987 && zend_jit_var_may_alias(p->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) {
6988 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6989 } else {
6990 SET_STACK_TYPE(call->stack, i, IS_UNDEF, 1);
6991 }
6992 i++;
6993 }
6994 while (i < p->op_array->last_var + p->op_array->T) {
6995 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6996 i++;
6997 }
6998 } else {
6999 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
7000 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
7001 }
7002 }
7003 } else {
7004 ZEND_ASSERT(p->func->type == ZEND_INTERNAL_FUNCTION);
7005 for (i = 0; i < p->op_array->num_args; i++) {
7006 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
7007 }
7008 }
7009 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
7010 int skip_guard = 0;
7011
7012 if (init_opline) {
7013 zend_call_info *call_info = jit_extension->func_info.callee_info;
7014
7015 while (call_info) {
7016 if (call_info->caller_init_opline == init_opline
7017 && !call_info->is_prototype) {
7018 if (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) {
7019 if (init_opline->opcode == ZEND_INIT_STATIC_METHOD_CALL
7020 && init_opline->op1_type != IS_CONST) {
7021 break;
7022 } else if (init_opline->opcode == ZEND_INIT_METHOD_CALL) {
7023 break;
7024 }
7025 }
7026 skip_guard = 1;
7027 break;
7028 }
7029 call_info = call_info->next_callee;
7030 }
7031 if (!skip_guard
7032 && !zend_jit_may_be_polymorphic_call(init_opline)
7033 && !zend_jit_may_be_modified(p->func, op_array)) {
7034 skip_guard = 1;
7035 }
7036 }
7037
7038 if (!skip_guard) {
7039 if (!opline) {
7040 zend_jit_trace_rec *q = p + 1;
7041 while (q->op != ZEND_JIT_TRACE_VM && q->op != ZEND_JIT_TRACE_END) {
7042 q++;
7043 }
7044 opline = q->opline;
7045 ZEND_ASSERT(opline != NULL);
7046 }
7047 if (!zend_jit_init_fcall_guard(&ctx,
7048 ZEND_JIT_TRACE_FAKE_LEVEL(p->info), p->func, opline)) {
7049 goto jit_failure;
7050 }
7051 }
7052 }
7053 }
7054 call->old_checked_stack = checked_stack;
7055 call->old_peek_checked_stack = peek_checked_stack;
7056 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
7057 frame->call_level++;
7058 call->used_stack = checked_stack = peek_checked_stack = 0;
7059 } else {
7060 if (p->func) {
7061 call->used_stack = zend_vm_calc_used_stack(init_opline->extended_value, (zend_function*)p->func);
7062 } else {
7063 call->used_stack = (ZEND_CALL_FRAME_SLOT + init_opline->extended_value) * sizeof(zval);
7064 }
7065 switch (init_opline->opcode) {
7066 case ZEND_INIT_FCALL:
7067 case ZEND_INIT_FCALL_BY_NAME:
7068 case ZEND_INIT_NS_FCALL_BY_NAME:
7069 case ZEND_INIT_METHOD_CALL:
7070 case ZEND_INIT_DYNAMIC_CALL:
7071 //case ZEND_INIT_STATIC_METHOD_CALL:
7072 //case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
7073 //case ZEND_INIT_USER_CALL:
7074 //case ZEND_NEW:
7075 checked_stack += call->used_stack;
7076 if (checked_stack > peek_checked_stack) {
7077 peek_checked_stack = checked_stack;
7078 }
7079 break;
7080 default:
7081 checked_stack = peek_checked_stack = 0;
7082 }
7083 }
7084 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
7085 call = frame->call;
7086 if (call) {
7087 checked_stack = call->old_checked_stack;
7088 peek_checked_stack = call->old_peek_checked_stack;
7089 top = call;
7090 frame->call = call->prev;
7091 }
7092 } else {
7093 ZEND_UNREACHABLE();
7094 }
7095 }
7096
7097 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END);
7098
7099 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7100
7101 if (!parent_trace && zend_jit_trace_uses_initial_ip(&ctx)) {
7102 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
7103 }
7104
7105 if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
7106 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
7107 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7108 if (ra) {
7109 zend_ssa_phi *phi = ssa->blocks[1].phis;
7110
7111 while (phi) {
7112 if (RA_HAS_REG(phi->sources[1])
7113 && STACK_MEM_TYPE(stack, phi->var) != STACK_TYPE(stack, phi->var)
7114 && (RA_REG_FLAGS(phi->sources[1]) & (ZREG_LOAD|ZREG_STORE)) == 0) {
7115
7116 if (!RA_HAS_REG(phi->ssa_var)
7117 || (RA_REG_FLAGS(phi->ssa_var) & (ZREG_LOAD|ZREG_STORE)) == 0) {
7118 /* Store actual type to memory to avoid deoptimization mistakes */
7119 zend_jit_store_var_type(&ctx, phi->var, STACK_TYPE(stack, phi->var));
7120 }
7121 }
7122 phi = phi->next;
7123 }
7124 }
7125 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7126 if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
7127 && !zend_jit_set_ip(&ctx, p->opline)) {
7128 goto jit_failure;
7129 }
7130 }
7131 t->link = ZEND_JIT_TRACE_NUM;
7132 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7133 t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT;
7134 }
7135 if (!(t->flags & ZEND_JIT_TRACE_LOOP)) {
7136 const void *timeout_exit_addr = NULL;
7137
7138 t->flags |= ZEND_JIT_TRACE_LOOP;
7139
7140 if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7141 if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
7142 || (ra
7143 && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) {
7144 /* Deoptimize to the first instruction of the loop */
7145 uint32_t exit_point = zend_jit_trace_get_exit_point(trace_buffer[1].opline, ZEND_JIT_EXIT_TO_VM);
7146
7147 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
7148 if (!timeout_exit_addr) {
7149 goto jit_failure;
7150 }
7151 } else {
7152 timeout_exit_addr = zend_jit_stub_handlers[jit_stub_interrupt_handler];
7153 }
7154 }
7155
7156 zend_jit_trace_end_loop(&ctx, jit->trace_loop_ref, timeout_exit_addr); /* jump back to start of the trace loop */
7157 }
7158 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
7159 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
7160 if (ra
7161 && (p-1)->op != ZEND_JIT_TRACE_ENTER
7162 && (p-1)->op != ZEND_JIT_TRACE_BACK
7163 && opline->opcode != ZEND_DO_UCALL
7164 && opline->opcode != ZEND_DO_FCALL
7165 && opline->opcode != ZEND_DO_FCALL_BY_NAME
7166 && opline->opcode != ZEND_INCLUDE_OR_EVAL) {
7167 for (i = 0; i < op_array->last_var + op_array->T; i++) {
7168 int32_t ref = STACK_REF(stack, i);
7169 uint8_t type = STACK_TYPE(stack, i);
7170
7171 if (ref && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) {
7172 if (!zend_jit_store_ref(jit, 1 << type, i, ref, STACK_MEM_TYPE(stack, i) != type)) {
7173 goto jit_failure;
7174 }
7175 SET_STACK_TYPE(stack, i, type, 1);
7176 } else if (i < op_array->last_var
7177 && type != IS_UNKNOWN
7178 && type != STACK_MEM_TYPE(stack, i)
7179 && zend_jit_trace_must_store_type(op_array, op_array_ssa, opline - op_array->opcodes, i, type)) {
7180 if (!zend_jit_store_type(jit, i, type)) {
7181 return 0;
7182 }
7183 SET_STACK_TYPE(stack, i, type, 1);
7184 }
7185 CLEAR_STACK_REF(stack, i);
7186 }
7187 }
7188 if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
7189 const void *timeout_exit_addr = NULL;
7190
7191 t->link = zend_jit_find_trace(p->opline->handler);
7192 if (t->link == 0) {
7193 /* this can happen if ZEND_JIT_EXIT_INVALIDATE was handled
7194 * by zend_jit_trace_exit() in another thread after this
7195 * thread set ZEND_JIT_TRACE_STOP_LINK in zend_jit_trace_execute();
7196 * ZEND_JIT_EXIT_INVALIDATE resets the opline handler to one of
7197 * the "_counter_handler" functions, and these are not registered
7198 * tracer functions */
7199 goto jit_failure;
7200 }
7201 if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
7202 && !zend_jit_set_ip(&ctx, p->opline)) {
7203 goto jit_failure;
7204 }
7205 if (!parent_trace && zend_jit_trace_uses_initial_ip(&ctx)) {
7206 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
7207 }
7208 if (parent_trace
7209 && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT)
7210 && zend_jit_traces[parent_trace].root == t->link) {
7211 if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) {
7212 uint32_t exit_point;
7213
7214 for (i = 0; i < op_array->last_var + op_array->T; i++) {
7215 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
7216 }
7217 exit_point = zend_jit_trace_get_exit_point(zend_jit_traces[t->link].opline, ZEND_JIT_EXIT_TO_VM);
7218 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
7219 if (!timeout_exit_addr) {
7220 goto jit_failure;
7221 }
7222 } else {
7223 timeout_exit_addr = zend_jit_stub_handlers[jit_stub_interrupt_handler];
7224 }
7225 }
7226 zend_jit_trace_link_to_root(&ctx, &zend_jit_traces[t->link], timeout_exit_addr);
7227 } else {
7228 zend_jit_trace_return(&ctx, 0, NULL);
7229 }
7230 } else if (p->stop == ZEND_JIT_TRACE_STOP_RETURN) {
7231 zend_jit_trace_return(&ctx, 0, NULL);
7232 } else {
7233 // TODO: not implemented ???
7234 ZEND_ASSERT(0 && p->stop);
7235 }
7236
7237 if (ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7238 goto jit_failure;
7239 }
7240
7241 handler = zend_jit_finish(&ctx);
7242
7243 if (handler) {
7244 if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
7245 const zend_op_array *rec_op_array;
7246
7247 rec_op_array = op_array = trace_buffer->op_array;
7248 jit_extension =
7249 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7250 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
7251 for (;;p++) {
7252 if (p->op == ZEND_JIT_TRACE_VM) {
7253 opline = p->opline;
7254 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7255 if (p->op_array == rec_op_array) {
7256 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7257 }
7258 op_array = p->op_array;
7259 jit_extension =
7260 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7261 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7262 op_array = p->op_array;
7263 jit_extension =
7264 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7265 } else if (p->op == ZEND_JIT_TRACE_END) {
7266 break;
7267 }
7268 }
7269 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
7270 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
7271 if (opline
7272 && (opline->opcode == ZEND_DO_UCALL
7273 || opline->opcode == ZEND_DO_FCALL
7274 || opline->opcode == ZEND_DO_FCALL_BY_NAME
7275 || opline->opcode == ZEND_YIELD
7276 || opline->opcode == ZEND_YIELD_FROM
7277 || opline->opcode == ZEND_INCLUDE_OR_EVAL)) {
7278 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7279 }
7280 if (JIT_G(current_frame)
7281 && JIT_G(current_frame)->prev) {
7282 frame = JIT_G(current_frame)->prev;
7283 do {
7284 if (frame->call_opline) {
7285 op_array = &frame->func->op_array;
7286 jit_extension =
7287 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7288 zend_jit_trace_setup_ret_counter(frame->call_opline, jit_extension->offset);
7289 }
7290 frame = frame->prev;
7291 } while (frame);
7292 }
7293 }
7294 }
7295
7296 jit_failure:
7297 zend_jit_free_ctx(&ctx);
7298
7299 if (name) {
7300 zend_string_release(name);
7301 }
7302
7303 jit_cleanup:
7304 /* Clean up used op_arrays */
7305 while (num_op_arrays > 0) {
7306 op_array = op_arrays[--num_op_arrays];
7307 jit_extension =
7308 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7309
7310 jit_extension->func_info.num = 0;
7311 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
7312 | ZEND_FUNC_JIT_ON_PROF_REQUEST
7313 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
7314 | ZEND_FUNC_JIT_ON_HOT_TRACE;
7315 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
7316 }
7317
7318 zend_arena_release(&CG(arena), checkpoint);
7319
7320 JIT_G(current_frame) = NULL;
7321 JIT_G(current_trace) = NULL;
7322
7323 return handler;
7324 }
7325
zend_jit_trace_escape_name(uint32_t trace_num,uint32_t exit_num)7326 static zend_string *zend_jit_trace_escape_name(uint32_t trace_num, uint32_t exit_num)
7327 {
7328 smart_str buf = {0};
7329
7330 smart_str_appends(&buf," ESCAPE-");
7331 smart_str_append_long(&buf, (zend_long)trace_num);
7332 smart_str_appendc(&buf, '-');
7333 smart_str_append_long(&buf, (zend_long)exit_num);
7334 smart_str_0(&buf);
7335 return buf.s;
7336 }
7337
zend_jit_trace_exit_to_vm(uint32_t trace_num,uint32_t exit_num)7338 static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num)
7339 {
7340 const void *handler = NULL;
7341 zend_jit_ctx ctx;
7342 zend_string *name;
7343 void *checkpoint;
7344 const zend_op *opline;
7345 uint32_t stack_size;
7346 zend_jit_trace_stack *stack;
7347 bool original_handler = 0;
7348
7349 if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) {
7350 return zend_jit_stub_handlers[jit_stub_trace_escape];
7351 }
7352
7353 name = zend_jit_trace_escape_name(trace_num, exit_num);
7354
7355 if (!zend_jit_deoptimizer_start(&ctx, name, trace_num, exit_num)) {
7356 zend_string_release(name);
7357 return NULL;
7358 }
7359
7360 checkpoint = zend_arena_checkpoint(CG(arena));;
7361
7362 /* Deoptimization */
7363 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
7364 stack = zend_jit_traces[trace_num].exit_info[exit_num].stack_size ?
7365 zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset :
7366 NULL;
7367
7368 if (!zend_jit_trace_deoptimization(&ctx,
7369 zend_jit_traces[trace_num].exit_info[exit_num].flags,
7370 zend_jit_traces[trace_num].exit_info[exit_num].opline,
7371 stack, stack_size, NULL, NULL,
7372 zend_jit_traces[trace_num].constants,
7373 zend_jit_traces[trace_num].exit_info[exit_num].poly_func_reg,
7374 0)) {
7375 goto jit_failure;
7376 }
7377
7378 opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
7379 if (opline) {
7380 if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) {
7381 zend_jit_op_array_trace_extension *jit_extension =
7382 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(zend_jit_traces[zend_jit_traces[trace_num].root].op_array);
7383
7384 if (ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->orig_handler != opline->handler) {
7385 /* prevent endless loop */
7386 original_handler = 1;
7387 }
7388 }
7389 zend_jit_set_ip_ex(&ctx, opline, original_handler);
7390 }
7391
7392 zend_jit_trace_return(&ctx, original_handler, opline);
7393
7394 handler = zend_jit_finish(&ctx);
7395
7396 jit_failure:
7397 zend_jit_free_ctx(&ctx);
7398 zend_string_release(name);
7399 zend_arena_release(&CG(arena), checkpoint);
7400 return handler;
7401 }
7402
zend_jit_compile_root_trace(zend_jit_trace_rec * trace_buffer,const zend_op * opline,size_t offset)7403 static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset)
7404 {
7405 zend_jit_trace_stop ret;
7406 const void *handler;
7407 uint8_t orig_trigger;
7408 zend_jit_trace_info *t = NULL;
7409 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7410 bool do_bailout = 0;
7411
7412 zend_shared_alloc_lock();
7413
7414 /* Checks under lock */
7415 if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) {
7416 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7417 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7418 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7419 } else {
7420 zend_try {
7421 SHM_UNPROTECT();
7422 zend_jit_unprotect();
7423
7424 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7425
7426 t->id = ZEND_JIT_TRACE_NUM;
7427 t->root = ZEND_JIT_TRACE_NUM;
7428 t->parent = 0;
7429 t->link = 0;
7430 t->exit_count = 0;
7431 t->child_count = 0;
7432 t->stack_map_size = 0;
7433 t->flags = 0;
7434 t->polymorphism = 0;
7435 t->jmp_table_size = 0;
7436 t->op_array = trace_buffer[0].op_array;
7437 t->opline = trace_buffer[1].opline;
7438 t->exit_info = exit_info;
7439 t->stack_map = NULL;
7440 t->consts_count = 0;
7441 t->constants = NULL;
7442
7443 orig_trigger = JIT_G(trigger);
7444 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7445
7446 handler = zend_jit_trace(trace_buffer, 0, 0);
7447
7448 JIT_G(trigger) = orig_trigger;
7449
7450 if (handler) {
7451 zend_jit_trace_exit_info *shared_exit_info = NULL;
7452
7453 t->exit_info = NULL;
7454 if (t->exit_count) {
7455 /* reallocate exit_info into shared memory */
7456 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7457 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7458
7459 if (!shared_exit_info) {
7460 if (t->stack_map) {
7461 efree(t->stack_map);
7462 t->stack_map = NULL;
7463 }
7464 if (t->constants) {
7465 efree(t->constants);
7466 t->constants = NULL;
7467 }
7468 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7469 goto exit;
7470 }
7471 memcpy(shared_exit_info, exit_info,
7472 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7473 t->exit_info = shared_exit_info;
7474 }
7475
7476 if (t->stack_map_size) {
7477 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7478 if (!shared_stack_map) {
7479 efree(t->stack_map);
7480 t->stack_map = NULL;
7481 if (t->constants) {
7482 efree(t->constants);
7483 t->constants = NULL;
7484 }
7485 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7486 goto exit;
7487 }
7488 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7489 efree(t->stack_map);
7490 t->stack_map = shared_stack_map;
7491 }
7492
7493 if (t->consts_count) {
7494 zend_jit_exit_const *constants = (zend_jit_exit_const*)zend_shared_alloc(t->consts_count * sizeof(zend_jit_exit_const));
7495 if (!constants) {
7496 efree(t->constants);
7497 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7498 goto exit;
7499 }
7500 memcpy(constants, t->constants, t->consts_count * sizeof(zend_jit_exit_const));
7501 efree(t->constants);
7502 t->constants = constants;
7503 }
7504
7505 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7506 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7507
7508 ((zend_op*)opline)->handler = handler;
7509
7510 ZEND_JIT_TRACE_NUM++;
7511 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED;
7512
7513 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7514 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7515 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7516 if (t->stack_map) {
7517 efree(t->stack_map);
7518 t->stack_map = NULL;
7519 }
7520 if (t->constants) {
7521 efree(t->constants);
7522 t->constants = NULL;
7523 }
7524 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7525 } else {
7526 if (t->stack_map) {
7527 efree(t->stack_map);
7528 t->stack_map = NULL;
7529 }
7530 if (t->constants) {
7531 efree(t->constants);
7532 t->constants = NULL;
7533 }
7534 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7535 }
7536
7537 exit:;
7538 } zend_catch {
7539 do_bailout = 1;
7540 } zend_end_try();
7541
7542 zend_jit_protect();
7543 SHM_PROTECT();
7544 }
7545
7546 zend_shared_alloc_unlock();
7547
7548 if (do_bailout) {
7549 zend_bailout();
7550 }
7551
7552 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
7553 && ret == ZEND_JIT_TRACE_STOP_COMPILED
7554 && t->exit_count > 0) {
7555 zend_jit_dump_exit_info(t);
7556 }
7557
7558 return ret;
7559 }
7560
7561 /* Set counting handler back to original VM handler. */
zend_jit_stop_hot_trace_counters(zend_op_array * op_array)7562 static void zend_jit_stop_hot_trace_counters(zend_op_array *op_array)
7563 {
7564 zend_jit_op_array_trace_extension *jit_extension;
7565 uint32_t i;
7566
7567 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7568 for (i = 0; i < op_array->last; i++) {
7569 /* Opline with Jit-ed code handler is skipped. */
7570 if (jit_extension->trace_info[i].trace_flags &
7571 (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7572 continue;
7573 }
7574 if (jit_extension->trace_info[i].trace_flags &
7575 (ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_START_RETURN)) {
7576 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
7577 }
7578 }
7579 }
7580
7581 /* Get the tracing op_array. */
zend_jit_stop_persistent_op_array(zend_op_array * op_array)7582 static void zend_jit_stop_persistent_op_array(zend_op_array *op_array)
7583 {
7584 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
7585 if (!func_info) {
7586 return;
7587 }
7588 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
7589 zend_jit_stop_hot_trace_counters(op_array);
7590 }
7591 }
7592
7593 /* Get all op_arrays with counter handler. */
zend_jit_stop_persistent_script(zend_persistent_script * script)7594 static void zend_jit_stop_persistent_script(zend_persistent_script *script)
7595 {
7596 zend_class_entry *ce;
7597 zend_op_array *op_array;
7598
7599 zend_jit_stop_persistent_op_array(&script->script.main_op_array);
7600
7601 ZEND_HASH_FOREACH_PTR(&script->script.function_table, op_array) {
7602 zend_jit_stop_persistent_op_array(op_array);
7603 } ZEND_HASH_FOREACH_END();
7604
7605 ZEND_HASH_FOREACH_PTR(&script->script.class_table, ce) {
7606 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
7607 if (op_array->type == ZEND_USER_FUNCTION) {
7608 zend_jit_stop_persistent_op_array(op_array);
7609 }
7610 } ZEND_HASH_FOREACH_END();
7611 } ZEND_HASH_FOREACH_END();
7612 }
7613
7614 /* Get all scripts which are accelerated by JIT */
zend_jit_stop_counter_handlers(void)7615 static void zend_jit_stop_counter_handlers(void)
7616 {
7617 if (ZCSG(jit_counters_stopped)) {
7618 return;
7619 }
7620
7621 zend_shared_alloc_lock();
7622 /* mprotect has an extreme overhead, avoid calls to it for every function. */
7623 SHM_UNPROTECT();
7624 if (!ZCSG(jit_counters_stopped)) {
7625 ZCSG(jit_counters_stopped) = true;
7626 for (uint32_t i = 0; i < ZCSG(hash).max_num_entries; i++) {
7627 zend_accel_hash_entry *cache_entry;
7628 for (cache_entry = ZCSG(hash).hash_table[i]; cache_entry; cache_entry = cache_entry->next) {
7629 zend_persistent_script *script;
7630 if (cache_entry->indirect) continue;
7631 script = (zend_persistent_script *)cache_entry->data;
7632 zend_jit_stop_persistent_script(script);
7633 }
7634 }
7635 }
7636 SHM_PROTECT();
7637 zend_shared_alloc_unlock();
7638 }
7639
zend_jit_blacklist_root_trace(const zend_op * opline,size_t offset)7640 static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset)
7641 {
7642 zend_shared_alloc_lock();
7643
7644 if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
7645 SHM_UNPROTECT();
7646 zend_jit_unprotect();
7647
7648 ((zend_op*)opline)->handler =
7649 ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
7650
7651 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
7652
7653 zend_jit_protect();
7654 SHM_PROTECT();
7655 }
7656
7657 zend_shared_alloc_unlock();
7658 }
7659
zend_jit_blacklist_function(zend_op_array * op_array)7660 ZEND_EXT_API void zend_jit_blacklist_function(zend_op_array *op_array) {
7661 zend_jit_op_array_trace_extension *jit_extension = (zend_jit_op_array_trace_extension *)ZEND_FUNC_INFO(op_array);
7662 if (!jit_extension || !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)) {
7663 return;
7664 }
7665
7666 zend_shared_alloc_lock();
7667 SHM_UNPROTECT();
7668 zend_jit_unprotect();
7669
7670 zend_jit_stop_persistent_op_array(op_array);
7671 jit_extension->func_info.flags &= ~ZEND_FUNC_JIT_ON_HOT_TRACE;
7672
7673 zend_jit_protect();
7674 SHM_PROTECT();
7675 zend_shared_alloc_unlock();
7676 }
7677
zend_jit_trace_is_bad_root(const zend_op * opline,zend_jit_trace_stop stop,size_t offset)7678 static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_stop stop, size_t offset)
7679 {
7680 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
7681 uint8_t *cache_count = JIT_G(bad_root_cache_count);
7682 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
7683 uint32_t cache_slot = JIT_G(bad_root_slot);
7684 uint32_t i;
7685
7686 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
7687 if (cache_opline[i] == opline) {
7688 if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) {
7689 cache_opline[i] = NULL;
7690 return 1;
7691 } else {
7692 #if 0
7693 if (ZEND_OP_TRACE_INFO(opline, offset)->counter) {
7694 *ZEND_OP_TRACE_INFO(opline, offset)->counter =
7695 random() % ZEND_JIT_TRACE_COUNTER_MAX;
7696 }
7697 #endif
7698 cache_count[i]++;
7699 cache_stop[i] = stop;
7700 return 0;
7701 }
7702 }
7703 }
7704 i = cache_slot;
7705 cache_opline[i] = opline;
7706 cache_count[i] = 1;
7707 cache_stop[i] = stop;
7708 cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS;
7709 JIT_G(bad_root_slot) = cache_slot;
7710 return 0;
7711 }
7712
zend_jit_dump_trace(zend_jit_trace_rec * trace_buffer,zend_ssa * tssa)7713 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa)
7714 {
7715 zend_jit_trace_rec *p = trace_buffer;
7716 const zend_op_array *op_array;
7717 const zend_op *opline;
7718 uint32_t level = 1 + trace_buffer[0].level;
7719 int idx, len, i, v, vars_count, call_level;
7720
7721 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
7722 op_array = p->op_array;
7723 p += ZEND_JIT_TRACE_START_REC_SIZE;
7724 idx = 0;
7725 call_level = 0;
7726
7727 if (tssa && tssa->var_info) {
7728 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
7729 vars_count = op_array->last_var;
7730 } else {
7731 vars_count = op_array->last_var + op_array->T;
7732 }
7733 for (i = 0; i < vars_count; i++) {
7734 if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) {
7735 fprintf(stderr, " %*c;", level, ' ');
7736 zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE);
7737 fprintf(stderr, "\n");
7738 }
7739 }
7740 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
7741 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
7742 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7743 zend_ssa_phi *p = tssa->blocks[1].phis;
7744
7745 fprintf(stderr, "LOOP:\n");
7746
7747 while (p) {
7748 fprintf(stderr, " ;");
7749 zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE);
7750 fprintf(stderr, " = Phi(");
7751 zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7752 fprintf(stderr, ", ");
7753 zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7754 fprintf(stderr, ")\n");
7755 p = p->next;
7756 }
7757 }
7758 }
7759
7760 while (1) {
7761 if (p->op == ZEND_JIT_TRACE_VM) {
7762 uint8_t op1_type, op2_type, op3_type;
7763
7764 opline = p->opline;
7765 fprintf(stderr, "%04d%*c",
7766 (int)(opline - op_array->opcodes),
7767 level, ' ');
7768 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7769
7770 op1_type = p->op1_type;
7771 op2_type = p->op2_type;
7772 op3_type = p->op3_type;
7773 if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) {
7774 fprintf(stderr, " ;");
7775 if (op1_type != IS_UNKNOWN) {
7776 const char *ref = (op1_type & IS_TRACE_INDIRECT) ?
7777 ((op1_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7778 ((op1_type & IS_TRACE_REFERENCE) ? "&" : "");
7779 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
7780 p++;
7781 fprintf(stderr, " op1(%sobject of class %s)", ref,
7782 ZSTR_VAL(p->ce->name));
7783 } else {
7784 const char *type = ((op1_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
7785 fprintf(stderr, " op1(%s%s%s)", ref, (op1_type & IS_TRACE_PACKED) ? "packed " : "", type);
7786 }
7787 }
7788 if (op2_type != IS_UNKNOWN) {
7789 const char *ref = (op2_type & IS_TRACE_INDIRECT) ?
7790 ((op2_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7791 ((op2_type & IS_TRACE_REFERENCE) ? "&" : "");
7792 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
7793 p++;
7794 fprintf(stderr, " op2(%sobject of class %s)", ref,
7795 ZSTR_VAL(p->ce->name));
7796 } else {
7797 const char *type = ((op2_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7798 fprintf(stderr, " op2(%s%s)", ref, type);
7799 }
7800 }
7801 if (op3_type != IS_UNKNOWN) {
7802 const char *ref = (op3_type & IS_TRACE_INDIRECT) ?
7803 ((op3_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7804 ((op3_type & IS_TRACE_REFERENCE) ? "&" : "");
7805 const char *type = ((op3_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op3_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7806 fprintf(stderr, " op3(%s%s)", ref, type);
7807 }
7808 }
7809 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
7810 uint8_t val_type;
7811 const char *type;
7812
7813 if (op1_type == IS_UNKNOWN && op2_type == IS_UNKNOWN && op3_type == IS_UNKNOWN) {
7814 fprintf(stderr, " ;");
7815 }
7816 p++;
7817 val_type = p->op1_type;
7818
7819 if (val_type == IS_UNDEF) {
7820 type = "undef";
7821 } else if (val_type == IS_REFERENCE) {
7822 type = "ref";
7823 } else {
7824 type = zend_get_type_by_const(val_type);
7825 }
7826 fprintf(stderr, " val(%s)", type);
7827 }
7828 fprintf(stderr, "\n");
7829 idx++;
7830
7831 len = zend_jit_trace_op_len(opline);
7832 while (len > 1) {
7833 opline++;
7834 fprintf(stderr, "%04d%*c;",
7835 (int)(opline - op_array->opcodes),
7836 level, ' ');
7837 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7838 idx++;
7839 len--;
7840 fprintf(stderr, "\n");
7841 }
7842 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7843 op_array = p->op_array;
7844 fprintf(stderr, " %*c>enter %s%s%s\n",
7845 level, ' ',
7846 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7847 op_array->scope ? "::" : "",
7848 op_array->function_name ?
7849 ZSTR_VAL(op_array->function_name) :
7850 ZSTR_VAL(op_array->filename));
7851 level++;
7852 if (tssa && tssa->var_info) {
7853 call_level++;
7854 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7855 vars_count = op_array->last_var;
7856 for (i = 0; i < vars_count; i++, v++) {
7857 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7858 fprintf(stderr, " %*c;", level, ' ');
7859 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7860 fprintf(stderr, "\n");
7861 }
7862 }
7863 }
7864 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7865 op_array = p->op_array;
7866 level--;
7867 fprintf(stderr, " %*c<back %s%s%s\n",
7868 level, ' ',
7869 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7870 op_array->scope ? "::" : "",
7871 op_array->function_name ?
7872 ZSTR_VAL(op_array->function_name) :
7873 ZSTR_VAL(op_array->filename));
7874 if (tssa && tssa->var_info) {
7875 if (call_level == 0) {
7876 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7877 vars_count = op_array->last_var + op_array->T;
7878 for (i = 0; i < vars_count; i++, v++) {
7879 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7880 fprintf(stderr, " %*c;", level, ' ');
7881 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7882 fprintf(stderr, "\n");
7883 }
7884 }
7885 } else {
7886 call_level--;
7887 }
7888 }
7889 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
7890 if (p->func != (zend_function*)&zend_pass_function) {
7891 fprintf(stderr, (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n",
7892 level, ' ',
7893 (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "",
7894 (p->func && p->func->common.scope) ? "::" : "",
7895 p->func ? ZSTR_VAL(p->func->common.function_name) : "???");
7896 } else {
7897 fprintf(stderr, " %*c>skip\n",
7898 level, ' ');
7899 }
7900 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
7901 if (p->func != (zend_function*)&zend_pass_function) {
7902 fprintf(stderr, " %*c>call %s%s%s\n",
7903 level, ' ',
7904 p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "",
7905 p->func->common.scope ? "::" : "",
7906 ZSTR_VAL(p->func->common.function_name));
7907 } else {
7908 fprintf(stderr, " %*c>skip\n",
7909 level, ' ');
7910 }
7911 } else if (p->op == ZEND_JIT_TRACE_END) {
7912 break;
7913 }
7914 p++;
7915 }
7916 }
7917
zend_jit_dump_exit_info(zend_jit_trace_info * t)7918 static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
7919 {
7920 int i, j;
7921
7922 fprintf(stderr, "---- TRACE %d exit info\n", t->id);
7923 for (i = 0; i < t->exit_count; i++) {
7924 const zend_op_array *op_array = t->exit_info[i].op_array;
7925 uint32_t stack_size = t->exit_info[i].stack_size;
7926 zend_jit_trace_stack *stack = t->exit_info[i].stack_size ? t->stack_map + t->exit_info[i].stack_offset : NULL;
7927
7928 fprintf(stderr, " exit_%d:", i);
7929 if (t->exit_info[i].opline) {
7930 fprintf(stderr, " %04d/", (int)(t->exit_info[i].opline - op_array->opcodes));
7931 } else {
7932 fprintf(stderr, " ----/");
7933 }
7934 if (t->exit_info[i].stack_size) {
7935 fprintf(stderr, "%04d/%d", t->exit_info[i].stack_offset, t->exit_info[i].stack_size);
7936 } else {
7937 fprintf(stderr, "----/0");
7938 }
7939 if (t->exit_info[i].flags & ZEND_JIT_EXIT_TO_VM) {
7940 fprintf(stderr, "/VM");
7941 }
7942 if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
7943 fprintf(stderr, "/CALL");
7944 }
7945 if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) {
7946 fprintf(stderr, "/POLY");
7947 if (t->exit_info[i].flags & ZEND_JIT_EXIT_METHOD_CALL) {
7948 fprintf(stderr, "(%s, %s)",
7949 t->exit_info[i].poly_func_reg != ZREG_NONE ? zend_reg_name(t->exit_info[i].poly_func_reg) : "?",
7950 t->exit_info[i].poly_this_reg != ZREG_NONE ? zend_reg_name(t->exit_info[i].poly_this_reg) : "?");
7951 }
7952 }
7953 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
7954 fprintf(stderr, "/FREE_OP1");
7955 }
7956 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP2) {
7957 fprintf(stderr, "/FREE_OP2");
7958 }
7959 for (j = 0; j < stack_size; j++) {
7960 uint8_t type = STACK_TYPE(stack, j);
7961 if (type != IS_UNKNOWN) {
7962 fprintf(stderr, " ");
7963 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7964 fprintf(stderr, ":");
7965 if (type == IS_UNDEF) {
7966 fprintf(stderr, "undef");
7967 } else {
7968 fprintf(stderr, "%s", zend_get_type_by_const(type));
7969 }
7970 if (STACK_FLAGS(stack, j) == ZREG_CONST) {
7971 if (type == IS_LONG) {
7972 fprintf(stderr, "(" ZEND_LONG_FMT ")", (zend_long)t->constants[STACK_REF(stack, j)].i);
7973 } else if (type == IS_DOUBLE) {
7974 fprintf(stderr, "(%g)", t->constants[STACK_REF(stack, j)].d);
7975 } else {
7976 ZEND_UNREACHABLE();
7977 }
7978 } else if (STACK_FLAGS(stack, j) == ZREG_TYPE_ONLY) {
7979 fprintf(stderr, "(type_only)");
7980 } else if (STACK_FLAGS(stack, j) == ZREG_THIS) {
7981 fprintf(stderr, "(this)");
7982 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_ADDREF) {
7983 fprintf(stderr, "(zval_try_addref)");
7984 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_COPY) {
7985 fprintf(stderr, "zval_copy(%s)", zend_reg_name(STACK_REG(stack, j)));
7986 } else if (STACK_FLAGS(stack, j) & ZREG_SPILL_SLOT) {
7987 if (STACK_REG(stack, j) == ZREG_NONE) {
7988 fprintf(stderr, "(spill=0x%x", STACK_REF(stack, j));
7989 } else {
7990 fprintf(stderr, "(spill=0x%x(%s)", STACK_REF(stack, j), zend_reg_name(STACK_REG(stack, j)));
7991 }
7992 if (STACK_FLAGS(stack, j) != 0) {
7993 fprintf(stderr, ":%x", STACK_FLAGS(stack, j));
7994 }
7995 fprintf(stderr, ")");
7996 } else if (STACK_REG(stack, j) != ZREG_NONE) {
7997 fprintf(stderr, "(%s", zend_reg_name(STACK_REG(stack, j)));
7998 if (STACK_FLAGS(stack, j) != 0) {
7999 fprintf(stderr, ":%x", STACK_FLAGS(stack, j));
8000 }
8001 fprintf(stderr, ")");
8002 }
8003 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_ADDREF) {
8004 fprintf(stderr, ":unknown(zval_try_addref)");
8005 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_COPY) {
8006 fprintf(stderr, " ");
8007 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
8008 fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name(STACK_REG(stack, j)));
8009 }
8010 }
8011 fprintf(stderr, "\n");
8012 }
8013 }
8014
zend_jit_trace_hot_root(zend_execute_data * execute_data,const zend_op * opline)8015 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline)
8016 {
8017 const zend_op *orig_opline;
8018 zend_jit_trace_stop stop;
8019 int ret = 0;
8020 zend_op_array *op_array;
8021 zend_jit_op_array_trace_extension *jit_extension;
8022 size_t offset;
8023 uint32_t trace_num;
8024 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
8025
8026 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
8027 ZEND_ASSERT(opline >= EX(func)->op_array.opcodes &&
8028 opline < EX(func)->op_array.opcodes + EX(func)->op_array.last);
8029
8030 repeat:
8031 trace_num = ZEND_JIT_TRACE_NUM;
8032 orig_opline = opline;
8033 op_array = &EX(func)->op_array;
8034 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8035 offset = jit_extension->offset;
8036
8037 EX(opline) = opline;
8038
8039 /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */
8040 if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
8041 return 0;
8042 }
8043
8044 if (JIT_G(tracing)) {
8045 ++(*ZEND_OP_TRACE_INFO(opline, offset)->counter);
8046 return 0;
8047 }
8048
8049 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8050 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8051 trace_num,
8052 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8053 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8054 EX(func)->op_array.scope ? "::" : "",
8055 EX(func)->op_array.function_name ?
8056 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8057 ZSTR_VAL(EX(func)->op_array.filename),
8058 opline->lineno);
8059 }
8060
8061 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8062 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8063 zend_jit_stop_counter_handlers();
8064 goto abort;
8065 }
8066
8067 JIT_G(tracing) = 1;
8068 stop = zend_jit_trace_execute(execute_data, opline, trace_buffer,
8069 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0);
8070 JIT_G(tracing) = 0;
8071
8072 if (stop & ZEND_JIT_TRACE_HALT) {
8073 ret = -1;
8074 }
8075 stop &= ~ZEND_JIT_TRACE_HALT;
8076
8077 if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) {
8078 orig_opline = trace_buffer[1].opline;
8079 op_array = (zend_op_array*)trace_buffer[0].op_array;
8080 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8081 offset = jit_extension->offset;
8082 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8083 const zend_op_array *op_array = trace_buffer[0].op_array;
8084 const zend_op *opline = trace_buffer[1].opline;
8085 zend_jit_op_array_trace_extension *jit_extension =
8086 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8087 size_t offset = jit_extension->offset;
8088
8089 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8090 trace_num,
8091 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8092 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
8093 op_array->scope ? "::" : "",
8094 op_array->function_name ?
8095 ZSTR_VAL(op_array->function_name) : "$main",
8096 ZSTR_VAL(op_array->filename),
8097 opline->lineno);
8098 }
8099 }
8100
8101 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
8102 zend_jit_dump_trace(trace_buffer, NULL);
8103 }
8104
8105 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
8106 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
8107 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
8108 uint32_t idx = trace_buffer[1].last;
8109 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
8110 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
8111 trace_num,
8112 link_to);
8113 } else {
8114 fprintf(stderr, "---- TRACE %d stop (%s)\n",
8115 trace_num,
8116 zend_jit_trace_stop_description[stop]);
8117 }
8118 }
8119 stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset);
8120 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
8121 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
8122 fprintf(stderr, "---- TRACE %d %s\n",
8123 trace_num,
8124 zend_jit_trace_stop_description[stop]);
8125 }
8126 } else {
8127 goto abort;
8128 }
8129 } else {
8130 abort:
8131 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
8132 fprintf(stderr, "---- TRACE %d abort (%s)\n",
8133 trace_num,
8134 zend_jit_trace_stop_description[stop]);
8135 }
8136 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
8137 || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) {
8138 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8139 fprintf(stderr, "---- TRACE %d blacklisted\n",
8140 trace_num);
8141 }
8142 zend_jit_blacklist_root_trace(orig_opline, offset);
8143 }
8144 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
8145 execute_data = EG(current_execute_data);
8146 opline = EX(opline);
8147 goto repeat;
8148 }
8149 }
8150
8151 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
8152 fprintf(stderr, "\n");
8153 }
8154
8155 return ret;
8156 }
8157
zend_jit_blacklist_trace_exit(uint32_t trace_num,uint32_t exit_num)8158 static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
8159 {
8160 const void *handler;
8161 bool do_bailout = 0;
8162
8163 zend_shared_alloc_lock();
8164
8165 if (!(zend_jit_traces[trace_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
8166 SHM_UNPROTECT();
8167 zend_jit_unprotect();
8168
8169 zend_try {
8170 handler = zend_jit_trace_exit_to_vm(trace_num, exit_num);
8171
8172 if (handler) {
8173 zend_jit_link_side_trace(
8174 zend_jit_traces[trace_num].code_start,
8175 zend_jit_traces[trace_num].code_size,
8176 zend_jit_traces[trace_num].jmp_table_size,
8177 exit_num,
8178 handler);
8179 }
8180 zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED;
8181 } zend_catch {
8182 do_bailout = 1;
8183 } zend_end_try();
8184
8185 zend_jit_protect();
8186 SHM_PROTECT();
8187 }
8188
8189 zend_shared_alloc_unlock();
8190
8191 if (do_bailout) {
8192 zend_bailout();
8193 }
8194 }
8195
zend_jit_trace_exit_is_bad(uint32_t trace_num,uint32_t exit_num)8196 static bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num)
8197 {
8198 uint8_t *counter = JIT_G(exit_counters) +
8199 zend_jit_traces[trace_num].exit_counters + exit_num;
8200
8201 if (*counter + 1 >= JIT_G(hot_side_exit) + JIT_G(blacklist_side_trace)) {
8202 return 1;
8203 }
8204 (*counter)++;
8205 return 0;
8206 }
8207
zend_jit_trace_exit_is_hot(uint32_t trace_num,uint32_t exit_num)8208 static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num)
8209 {
8210 uint8_t *counter = JIT_G(exit_counters) +
8211 zend_jit_traces[trace_num].exit_counters + exit_num;
8212
8213 if (*counter + 1 >= JIT_G(hot_side_exit)) {
8214 return 1;
8215 }
8216 (*counter)++;
8217 return 0;
8218 }
8219
zend_jit_compile_side_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_num,uint32_t exit_num,uint32_t polymorphism)8220 static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism)
8221 {
8222 zend_jit_trace_stop ret;
8223 const void *handler;
8224 uint8_t orig_trigger;
8225 zend_jit_trace_info *t;
8226 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
8227 bool do_bailout = 0;
8228
8229 zend_shared_alloc_lock();
8230
8231 /* Checks under lock */
8232 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8233 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
8234 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8235 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8236 } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
8237 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
8238 } else {
8239 SHM_UNPROTECT();
8240 zend_jit_unprotect();
8241
8242 zend_try {
8243 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
8244
8245 t->id = ZEND_JIT_TRACE_NUM;
8246 t->root = zend_jit_traces[parent_num].root;
8247 t->parent = parent_num;
8248 t->link = 0;
8249 t->exit_count = 0;
8250 t->child_count = 0;
8251 t->stack_map_size = 0;
8252 t->flags = 0;
8253 t->polymorphism = polymorphism;
8254 t->jmp_table_size = 0;
8255 t->opline = NULL;
8256 t->exit_info = exit_info;
8257 t->stack_map = NULL;
8258 t->consts_count = 0;
8259 t->constants = NULL;
8260
8261 orig_trigger = JIT_G(trigger);
8262 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
8263
8264 handler = zend_jit_trace(trace_buffer, parent_num, exit_num);
8265
8266 JIT_G(trigger) = orig_trigger;
8267
8268 if (handler) {
8269 zend_jit_trace_exit_info *shared_exit_info = NULL;
8270
8271 t->exit_info = NULL;
8272 if (t->exit_count) {
8273 /* reallocate exit_info into shared memory */
8274 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
8275 sizeof(zend_jit_trace_exit_info) * t->exit_count);
8276
8277 if (!shared_exit_info) {
8278 if (t->stack_map) {
8279 efree(t->stack_map);
8280 t->stack_map = NULL;
8281 }
8282 if (t->constants) {
8283 efree(t->constants);
8284 t->constants = NULL;
8285 }
8286 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
8287 goto exit;
8288 }
8289 memcpy(shared_exit_info, exit_info,
8290 sizeof(zend_jit_trace_exit_info) * t->exit_count);
8291 t->exit_info = shared_exit_info;
8292 }
8293
8294 if (t->stack_map_size) {
8295 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
8296 if (!shared_stack_map) {
8297 efree(t->stack_map);
8298 t->stack_map = NULL;
8299 if (t->constants) {
8300 efree(t->constants);
8301 t->constants = NULL;
8302 }
8303 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
8304 goto exit;
8305 }
8306 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
8307 efree(t->stack_map);
8308 t->stack_map = shared_stack_map;
8309 }
8310
8311 if (t->consts_count) {
8312 zend_jit_exit_const *constants = (zend_jit_exit_const*)zend_shared_alloc(t->consts_count * sizeof(zend_jit_exit_const));
8313 if (!constants) {
8314 efree(t->constants);
8315 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
8316 goto exit;
8317 }
8318 memcpy(constants, t->constants, t->consts_count * sizeof(zend_jit_exit_const));
8319 efree(t->constants);
8320 t->constants = constants;
8321 }
8322
8323 zend_jit_link_side_trace(
8324 zend_jit_traces[parent_num].code_start,
8325 zend_jit_traces[parent_num].code_size,
8326 zend_jit_traces[parent_num].jmp_table_size,
8327 exit_num,
8328 handler);
8329
8330 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
8331 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
8332
8333 zend_jit_traces[zend_jit_traces[parent_num].root].child_count++;
8334 ZEND_JIT_TRACE_NUM++;
8335 zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED;
8336
8337 ret = ZEND_JIT_TRACE_STOP_COMPILED;
8338 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
8339 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
8340 if (t->stack_map) {
8341 efree(t->stack_map);
8342 t->stack_map = NULL;
8343 }
8344 if (t->constants) {
8345 efree(t->constants);
8346 t->constants = NULL;
8347 }
8348 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
8349 } else {
8350 if (t->stack_map) {
8351 efree(t->stack_map);
8352 t->stack_map = NULL;
8353 }
8354 if (t->constants) {
8355 efree(t->constants);
8356 t->constants = NULL;
8357 }
8358 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
8359 }
8360
8361 exit:;
8362 } zend_catch {
8363 do_bailout = 1;
8364 } zend_end_try();
8365
8366 zend_jit_protect();
8367 SHM_PROTECT();
8368 }
8369
8370 zend_shared_alloc_unlock();
8371
8372 if (do_bailout) {
8373 zend_bailout();
8374 }
8375
8376 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
8377 && ret == ZEND_JIT_TRACE_STOP_COMPILED
8378 && t->exit_count > 0) {
8379 zend_jit_dump_exit_info(t);
8380 }
8381
8382 return ret;
8383 }
8384
zend_jit_trace_hot_side(zend_execute_data * execute_data,uint32_t parent_num,uint32_t exit_num)8385 int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num)
8386 {
8387 zend_jit_trace_stop stop;
8388 int ret = 0;
8389 uint32_t trace_num;
8390 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
8391 uint32_t is_megamorphic = 0;
8392 uint32_t polymorphism = 0;
8393
8394 trace_num = ZEND_JIT_TRACE_NUM;
8395
8396 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8397 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8398 return 0;
8399 }
8400
8401 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8402 fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s%s%s() %s:%d\n",
8403 trace_num, parent_num, exit_num,
8404 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8405 EX(func)->op_array.scope ? "::" : "",
8406 EX(func)->op_array.function_name ?
8407 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8408 ZSTR_VAL(EX(func)->op_array.filename),
8409 EX(opline)->lineno);
8410 }
8411
8412 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8413 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8414 goto abort;
8415 }
8416
8417 if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
8418 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
8419 goto abort;
8420 }
8421
8422 if (JIT_G(max_polymorphic_calls) > 0) {
8423 if ((zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL))
8424 || ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)
8425 && EX(call))) {
8426 if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
8427 is_megamorphic = zend_jit_traces[parent_num].exit_info[exit_num].flags &
8428 (ZEND_JIT_EXIT_METHOD_CALL | ZEND_JIT_EXIT_CLOSURE_CALL | ZEND_JIT_EXIT_POLYMORPHISM);
8429 } else if (!zend_jit_traces[parent_num].polymorphism) {
8430 polymorphism = 1;
8431 } else if (exit_num == 0) {
8432 polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
8433 }
8434 }
8435 }
8436
8437 JIT_G(tracing) = 1;
8438 stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic);
8439 JIT_G(tracing) = 0;
8440
8441 if (stop & ZEND_JIT_TRACE_HALT) {
8442 ret = -1;
8443 }
8444 stop &= ~ZEND_JIT_TRACE_HALT;
8445
8446 if (UNEXPECTED(trace_buffer->start != ZEND_JIT_TRACE_START_SIDE)) {
8447 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8448 const zend_op_array *op_array = trace_buffer[0].op_array;
8449 const zend_op *opline = trace_buffer[1].opline;
8450 zend_jit_op_array_trace_extension *jit_extension =
8451 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8452 size_t offset = jit_extension->offset;
8453
8454 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8455 trace_num,
8456 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8457 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
8458 op_array->scope ? "::" : "",
8459 op_array->function_name ?
8460 ZSTR_VAL(op_array->function_name) : "$main",
8461 ZSTR_VAL(op_array->filename),
8462 opline->lineno);
8463 }
8464 }
8465
8466 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
8467 zend_jit_dump_trace(trace_buffer, NULL);
8468 }
8469
8470 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
8471 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
8472 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
8473 uint32_t idx = trace_buffer[1].last;
8474 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);;
8475 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
8476 trace_num,
8477 link_to);
8478 } else {
8479 fprintf(stderr, "---- TRACE %d stop (%s)\n",
8480 trace_num,
8481 zend_jit_trace_stop_description[stop]);
8482 }
8483 }
8484 if (EXPECTED(trace_buffer->start == ZEND_JIT_TRACE_START_SIDE)) {
8485 stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism);
8486 } else {
8487 const zend_op_array *op_array = trace_buffer[0].op_array;
8488 zend_jit_op_array_trace_extension *jit_extension =
8489 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8490 const zend_op *opline = trace_buffer[1].opline;
8491
8492 stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset);
8493 }
8494 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
8495 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
8496 fprintf(stderr, "---- TRACE %d %s\n",
8497 trace_num,
8498 zend_jit_trace_stop_description[stop]);
8499 }
8500 } else {
8501 goto abort;
8502 }
8503 } else {
8504 abort:
8505 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
8506 fprintf(stderr, "---- TRACE %d abort (%s)\n",
8507 trace_num,
8508 zend_jit_trace_stop_description[stop]);
8509 }
8510 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
8511 || zend_jit_trace_exit_is_bad(parent_num, exit_num)) {
8512 zend_jit_blacklist_trace_exit(parent_num, exit_num);
8513 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8514 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8515 parent_num, exit_num);
8516 }
8517 }
8518 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
8519 execute_data = EG(current_execute_data);
8520 return zend_jit_trace_hot_root(execute_data, EX(opline));
8521 }
8522 }
8523
8524 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
8525 fprintf(stderr, "\n");
8526 }
8527
8528 return ret;
8529 }
8530
zend_jit_trace_exit(uint32_t exit_num,zend_jit_registers_buf * regs)8531 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs)
8532 {
8533 uint32_t trace_num = EG(jit_trace_num);
8534 zend_execute_data *execute_data = EG(current_execute_data);
8535 const zend_op *orig_opline = EX(opline);
8536 const zend_op *opline;
8537 zend_jit_trace_info *t = &zend_jit_traces[trace_num];
8538 int repeat_last_opline = 0;
8539
8540 /* Deoptimization of VM stack state */
8541 uint32_t i;
8542 uint32_t stack_size = t->exit_info[exit_num].stack_size;
8543 zend_jit_trace_stack *stack = stack_size ? t->stack_map + t->exit_info[exit_num].stack_offset : NULL;
8544
8545 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
8546 zend_execute_data *call = (zend_execute_data *)regs->gpr[ZREG_RX];
8547 call->prev_execute_data = EX(call);
8548 EX(call) = call;
8549 }
8550
8551 for (i = 0; i < stack_size; i++) {
8552 if (STACK_FLAGS(stack, i) == ZREG_CONST) {
8553 if (STACK_TYPE(stack, i) == IS_LONG) {
8554 ZVAL_LONG(EX_VAR_NUM(i), (zend_long)t->constants[STACK_REF(stack, i)].i);
8555 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8556 ZVAL_DOUBLE(EX_VAR_NUM(i), t->constants[STACK_REF(stack, i)].d);
8557 } else {
8558 ZEND_UNREACHABLE();
8559 }
8560 } else if (STACK_FLAGS(stack, i) == ZREG_TYPE_ONLY) {
8561 uint32_t type = STACK_TYPE(stack, i);
8562 if (type <= IS_DOUBLE) {
8563 Z_TYPE_INFO_P(EX_VAR_NUM(i)) = type;
8564 } else {
8565 ZEND_UNREACHABLE();
8566 }
8567 } else if (STACK_FLAGS(stack, i) == ZREG_THIS) {
8568 zend_object *obj = Z_OBJ(EX(This));
8569
8570 GC_ADDREF(obj);
8571 ZVAL_OBJ(EX_VAR_NUM(i), obj);
8572 } else if (STACK_FLAGS(stack, i) == ZREG_ZVAL_ADDREF) {
8573 Z_TRY_ADDREF_P(EX_VAR_NUM(i));
8574 } else if (STACK_FLAGS(stack, i) == ZREG_ZVAL_COPY) {
8575 zval *val = (zval*)regs->gpr[STACK_REG(stack, i)];
8576
8577 if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) {
8578 /* Undefined array index or property */
8579 const zend_op *op = t->exit_info[exit_num].opline;
8580 ZEND_ASSERT(op);
8581 op--;
8582 if (op->opcode == ZEND_FETCH_DIM_IS || op->opcode == ZEND_FETCH_OBJ_IS) {
8583 ZVAL_NULL(EX_VAR_NUM(i));
8584 } else {
8585 assert(op->opcode == ZEND_FETCH_DIM_R || op->opcode == ZEND_FETCH_LIST_R || op->opcode == ZEND_FETCH_OBJ_R);
8586 repeat_last_opline = 1;
8587 }
8588 } else {
8589 ZVAL_COPY(EX_VAR_NUM(i), val);
8590 }
8591 } else if (STACK_FLAGS(stack, i) & ZREG_SPILL_SLOT) {
8592 ZEND_ASSERT(STACK_REG(stack, i) != ZREG_NONE);
8593 uintptr_t ptr = (uintptr_t)regs->gpr[STACK_REG(stack, i)] + STACK_REF(stack, i);
8594
8595 if (STACK_TYPE(stack, i) == IS_LONG) {
8596 ZVAL_LONG(EX_VAR_NUM(i), *(zend_long*)ptr);
8597 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8598 ZVAL_DOUBLE(EX_VAR_NUM(i), *(double*)ptr);
8599 } else {
8600 ZEND_UNREACHABLE();
8601 }
8602 } else if (STACK_REG(stack, i) != ZREG_NONE) {
8603 if (STACK_TYPE(stack, i) == IS_LONG) {
8604 zend_long val = regs->gpr[STACK_REG(stack, i)];
8605 ZVAL_LONG(EX_VAR_NUM(i), val);
8606 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8607 double val = regs->fpr[STACK_REG(stack, i) - ZREG_FIRST_FPR];
8608 ZVAL_DOUBLE(EX_VAR_NUM(i), val);
8609 } else {
8610 ZEND_UNREACHABLE();
8611 }
8612 }
8613 }
8614
8615 if (repeat_last_opline) {
8616 EX(opline) = t->exit_info[exit_num].opline - 1;
8617 if ((EX(opline)->op1_type & (IS_VAR|IS_TMP_VAR))
8618 && !(t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1)
8619 && EX(opline)->opcode != ZEND_FETCH_LIST_R) {
8620 Z_TRY_ADDREF_P(EX_VAR(EX(opline)->op1.var));
8621 }
8622 return 1;
8623 }
8624
8625 opline = t->exit_info[exit_num].opline;
8626
8627 if (opline) {
8628 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) {
8629 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8630 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8631 || (opline-1)->opcode == ZEND_FETCH_LIST_R
8632 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG);
8633 EX(opline) = opline-1;
8634 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op2.var));
8635 }
8636 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) {
8637 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8638 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8639 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG
8640 || (opline-1)->opcode == ZEND_FETCH_OBJ_R
8641 || (opline-1)->opcode == ZEND_FETCH_OBJ_IS
8642 || (opline-1)->opcode == ZEND_FETCH_OBJ_FUNC_ARG);
8643 EX(opline) = opline-1;
8644 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op1.var));
8645 }
8646 if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
8647 if (EG(exception)) {
8648 return 1;
8649 }
8650 }
8651 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8652 ZEND_ASSERT(t->exit_info[exit_num].poly_func_reg >= 0);
8653 zend_function *func = (zend_function*)regs->gpr[t->exit_info[exit_num].poly_func_reg];
8654
8655 if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) {
8656 zend_string_release_ex(func->common.function_name, 0);
8657 zend_free_trampoline(func);
8658 EX(opline) = opline;
8659 return 1;
8660 }
8661 }
8662
8663 /* Set VM opline to continue interpretation */
8664 EX(opline) = opline;
8665 }
8666
8667 if (zend_atomic_bool_load_ex(&EG(vm_interrupt)) || JIT_G(tracing)) {
8668 return 1;
8669 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8670 } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8671 return 0;
8672 }
8673
8674 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
8675 ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes &&
8676 EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last);
8677
8678 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT) {
8679 fprintf(stderr, " TRACE %d exit %d %s%s%s() %s:%d\n",
8680 trace_num,
8681 exit_num,
8682 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8683 EX(func)->op_array.scope ? "::" : "",
8684 EX(func)->op_array.function_name ?
8685 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8686 ZSTR_VAL(EX(func)->op_array.filename),
8687 EX(opline)->lineno);
8688 }
8689
8690 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_INVALIDATE) {
8691 zend_jit_op_array_trace_extension *jit_extension;
8692 uint32_t num = trace_num;
8693
8694 while (t->root != num) {
8695 num = t->root;
8696 t = &zend_jit_traces[num];
8697 }
8698
8699 zend_shared_alloc_lock();
8700
8701 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(t->op_array);
8702
8703 /* Checks under lock, just in case something has changed while we were waiting for the lock */
8704 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) {
8705 /* skip: not JIT-ed nor blacklisted */
8706 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8707 /* too many root traces, blacklist the root trace */
8708 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
8709 SHM_UNPROTECT();
8710 zend_jit_unprotect();
8711
8712 ((zend_op*)opline)->handler =
8713 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->orig_handler;
8714
8715 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &= ~ZEND_JIT_TRACE_JITED;
8716 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
8717
8718 zend_jit_protect();
8719 SHM_PROTECT();
8720 }
8721 } else {
8722 SHM_UNPROTECT();
8723 zend_jit_unprotect();
8724
8725 if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_LOOP) {
8726 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_loop_trace_counter_handler;
8727 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_ENTER) {
8728 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_func_trace_counter_handler;
8729 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_RETURN) {
8730 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_ret_trace_counter_handler;
8731 }
8732 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &=
8733 ZEND_JIT_TRACE_START_LOOP|ZEND_JIT_TRACE_START_ENTER|ZEND_JIT_TRACE_START_RETURN;
8734
8735 zend_jit_protect();
8736 SHM_PROTECT();
8737 }
8738
8739 zend_shared_alloc_unlock();
8740
8741 return 0;
8742 }
8743
8744 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) {
8745 if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) {
8746 zend_jit_blacklist_trace_exit(trace_num, exit_num);
8747 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8748 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8749 trace_num, exit_num);
8750 }
8751 return 0;
8752 }
8753 } else if (JIT_G(hot_side_exit) && zend_jit_trace_exit_is_hot(trace_num, exit_num)) {
8754 return zend_jit_trace_hot_side(execute_data, trace_num, exit_num);
8755 }
8756
8757 /* Return 1 to call original handler instead of the same JIT-ed trace */
8758 return (orig_opline == t->opline && EX(opline) == orig_opline);
8759 }
8760
zend_jit_trace_supported(const zend_op * opline)8761 static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline)
8762 {
8763 switch (opline->opcode) {
8764 case ZEND_CATCH:
8765 case ZEND_FAST_CALL:
8766 case ZEND_FAST_RET:
8767 return ZEND_JIT_TRACE_UNSUPPORTED;
8768 default:
8769 return ZEND_JIT_TRACE_SUPPORTED;
8770 }
8771 }
8772
zend_jit_restart_hot_trace_counters(zend_op_array * op_array)8773 static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array)
8774 {
8775 zend_jit_op_array_trace_extension *jit_extension;
8776 uint32_t i;
8777
8778 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8779 for (i = 0; i < op_array->last; i++) {
8780 jit_extension->trace_info[i].trace_flags &=
8781 ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED;
8782 if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) {
8783 op_array->opcodes[i].handler = (const void*)zend_jit_loop_trace_counter_handler;
8784 } else if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_ENTER) {
8785 op_array->opcodes[i].handler = (const void*)zend_jit_func_trace_counter_handler;
8786 } else {
8787 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
8788 }
8789 }
8790 return SUCCESS;
8791 }
8792
zend_jit_setup_hot_trace_counters(zend_op_array * op_array)8793 static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
8794 {
8795 zend_op *opline;
8796 zend_jit_op_array_trace_extension *jit_extension;
8797 uint32_t i;
8798
8799 ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op));
8800
8801 jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info));
8802 if (!jit_extension) {
8803 return FAILURE;
8804 }
8805 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
8806 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_TRACE;
8807 jit_extension->op_array = op_array;
8808 jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes;
8809 for (i = 0; i < op_array->last; i++) {
8810 jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler;
8811 jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]);
8812 jit_extension->trace_info[i].counter = NULL;
8813 jit_extension->trace_info[i].trace_flags =
8814 zend_jit_trace_supported(&op_array->opcodes[i]);
8815 }
8816 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
8817
8818 if (JIT_G(hot_loop)) {
8819 zend_cfg cfg;
8820
8821 ZEND_ASSERT(zend_jit_loop_trace_counter_handler != NULL);
8822
8823 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
8824 return FAILURE;
8825 }
8826
8827 for (i = 0; i < cfg.blocks_count; i++) {
8828 if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) {
8829 if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) {
8830 /* loop header */
8831 opline = op_array->opcodes + cfg.blocks[i].start;
8832 if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
8833 opline->handler = (const void*)zend_jit_loop_trace_counter_handler;
8834 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) {
8835 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8836 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8837 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8838 }
8839 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8840 ZEND_JIT_TRACE_START_LOOP;
8841 }
8842 }
8843 }
8844 }
8845 }
8846
8847 if (JIT_G(hot_func)) {
8848 ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL);
8849 opline = op_array->opcodes;
8850 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
8851 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
8852 opline++;
8853 }
8854 }
8855
8856 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) {
8857 /* function entry */
8858 opline->handler = (const void*)zend_jit_func_trace_counter_handler;
8859 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8860 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8861 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8862 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8863 ZEND_JIT_TRACE_START_ENTER;
8864 }
8865 }
8866
8867 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
8868
8869 return SUCCESS;
8870 }
8871
zend_jit_trace_init_caches(void)8872 static void zend_jit_trace_init_caches(void)
8873 {
8874 memset(ZEND_VOIDP(JIT_G(bad_root_cache_opline)), 0, sizeof(JIT_G(bad_root_cache_opline)));
8875 memset(JIT_G(bad_root_cache_count), 0, sizeof(JIT_G(bad_root_cache_count)));
8876 memset(JIT_G(bad_root_cache_stop), 0, sizeof(JIT_G(bad_root_cache_count)));
8877 JIT_G(bad_root_slot) = 0;
8878
8879 if (JIT_G(exit_counters)) {
8880 memset(JIT_G(exit_counters), 0, JIT_G(max_exit_counters));
8881 }
8882 }
8883
zend_jit_trace_reset_caches(void)8884 static void zend_jit_trace_reset_caches(void)
8885 {
8886 JIT_G(tracing) = 0;
8887 #ifdef ZTS
8888 if (!JIT_G(exit_counters)) {
8889 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
8890 }
8891 #endif
8892 }
8893
zend_jit_trace_free_caches(zend_jit_globals * jit_globals)8894 static void zend_jit_trace_free_caches(zend_jit_globals *jit_globals)
8895 {
8896 if (jit_globals->exit_counters) {
8897 free(jit_globals->exit_counters);
8898 }
8899 }
8900
zend_jit_trace_restart(void)8901 static void zend_jit_trace_restart(void)
8902 {
8903 ZEND_JIT_TRACE_NUM = 1;
8904 ZEND_JIT_COUNTER_NUM = 0;
8905 ZEND_JIT_EXIT_NUM = 0;
8906 ZEND_JIT_EXIT_COUNTERS = 0;
8907 ZCSG(jit_counters_stopped) = false;
8908
8909 zend_jit_trace_init_caches();
8910 }
8911
