1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 static zend_op_array dummy_op_array;
20 static zend_jit_trace_info *zend_jit_traces = NULL;
21 static const void **zend_jit_exit_groups = NULL;
22
23 #define ZEND_JIT_COUNTER_NUM zend_jit_traces[0].root
24 #define ZEND_JIT_TRACE_NUM zend_jit_traces[0].id
25 #define ZEND_JIT_EXIT_NUM zend_jit_traces[0].exit_count
26 #define ZEND_JIT_EXIT_COUNTERS zend_jit_traces[0].exit_counters
27
28 #define ZEND_JIT_TRACE_STOP_DESCRIPTION(name, description) \
29 description,
30
31 static const char * zend_jit_trace_stop_description[] = {
32 ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_DESCRIPTION)
33 };
34
zend_jit_trace_star_desc(uint8_t trace_flags)35 static zend_always_inline const char *zend_jit_trace_star_desc(uint8_t trace_flags)
36 {
37 if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
38 return "loop";
39 } else if (trace_flags & ZEND_JIT_TRACE_START_ENTER) {
40 return "enter";
41 } else if (trace_flags & ZEND_JIT_TRACE_START_RETURN) {
42 return "return";
43 } else {
44 ZEND_UNREACHABLE();
45 return "???";
46 }
47 }
48
zend_jit_trace_startup(bool reattached)49 static void zend_jit_trace_startup(bool reattached)
50 {
51 if (!reattached) {
52 zend_jit_traces = (zend_jit_trace_info*)zend_shared_alloc(sizeof(zend_jit_trace_info) * JIT_G(max_root_traces));
53 if (!zend_jit_traces) {
54 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not allocate JIT root traces buffer!");
55 }
56 zend_jit_exit_groups = (const void**)zend_shared_alloc(sizeof(void*) * (ZEND_JIT_TRACE_MAX_EXITS/ZEND_JIT_EXIT_POINTS_PER_GROUP));
57 if (!zend_jit_exit_groups) {
58 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not allocate JIT exit groups buffer!");
59 }
60 ZEND_JIT_TRACE_NUM = 1;
61 ZEND_JIT_COUNTER_NUM = 0;
62 ZEND_JIT_EXIT_NUM = 0;
63 ZEND_JIT_EXIT_COUNTERS = 0;
64 ZCSG(jit_traces) = zend_jit_traces;
65 ZCSG(jit_exit_groups) = zend_jit_exit_groups;
66 ZCSG(jit_counters_stopped) = false;
67 } else {
68 zend_jit_traces = ZCSG(jit_traces);
69 if (!zend_jit_traces) {
70 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not obtain JIT traces buffer!");
71 }
72 zend_jit_exit_groups = ZCSG(jit_exit_groups);
73 if (!zend_jit_exit_groups) {
74 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not obtain JIT exit groups buffer!");
75 }
76 }
77
78 memset(&dummy_op_array, 0, sizeof(dummy_op_array));
79 dummy_op_array.fn_flags = ZEND_ACC_DONE_PASS_TWO;
80
81 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
82 if (JIT_G(exit_counters) == NULL) {
83 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not allocate JIT exit counters buffer!");
84 }
85 }
86
zend_jit_trace_allocate_exit_point(uint32_t n)87 static const void *zend_jit_trace_allocate_exit_point(uint32_t n)
88 {
89 const void *group = NULL;
90
91 if (UNEXPECTED(n >= ZEND_JIT_TRACE_MAX_EXITS)) {
92 return NULL;
93 }
94 do {
95 group = zend_jit_trace_allocate_exit_group(ZEND_JIT_EXIT_NUM);
96 if (!group) {
97 return NULL;
98 }
99 zend_jit_exit_groups[ZEND_JIT_EXIT_NUM / ZEND_JIT_EXIT_POINTS_PER_GROUP] =
100 group;
101 ZEND_JIT_EXIT_NUM += ZEND_JIT_EXIT_POINTS_PER_GROUP;
102 } while (n >= ZEND_JIT_EXIT_NUM);
103 return (const void*)
104 ((const char*)group +
105 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
106 }
107
zend_jit_trace_get_exit_addr(uint32_t n)108 static const void *zend_jit_trace_get_exit_addr(uint32_t n)
109 {
110 if (UNEXPECTED(n >= ZEND_JIT_EXIT_NUM)) {
111 return zend_jit_trace_allocate_exit_point(n);
112 }
113 return (const void*)
114 ((const char*)zend_jit_exit_groups[n / ZEND_JIT_EXIT_POINTS_PER_GROUP] +
115 ((n % ZEND_JIT_EXIT_POINTS_PER_GROUP) * ZEND_JIT_EXIT_POINTS_SPACING));
116 }
117
zend_jit_exit_point_by_addr(const void * addr)118 static uint32_t zend_jit_exit_point_by_addr(const void *addr)
119 {
120 uint32_t n = (ZEND_JIT_EXIT_NUM + (ZEND_JIT_EXIT_POINTS_PER_GROUP - 1)) / ZEND_JIT_EXIT_POINTS_PER_GROUP;
121 uint32_t i;
122
123 for (i = 0; i < n; i++) {
124 if ((char*)addr >= (char*)zend_jit_exit_groups[i]
125 && (char*)addr <= (char*)zend_jit_exit_groups[i] + ((ZEND_JIT_EXIT_POINTS_PER_GROUP - 1) * ZEND_JIT_EXIT_POINTS_SPACING)) {
126 return (i * ZEND_JIT_EXIT_POINTS_PER_GROUP) +
127 (((char*)addr - (char*)zend_jit_exit_groups[i]) / ZEND_JIT_EXIT_POINTS_SPACING);
128 }
129 }
130 return (uint32_t)-1;
131 }
132
zend_jit_trace_get_exit_point(const zend_op * to_opline,uint32_t flags)133 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags)
134 {
135 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
136 uint32_t exit_point;
137 const zend_op_array *op_array;
138 uint32_t stack_offset = (uint32_t)-1;
139 uint32_t stack_size;
140 zend_jit_trace_stack *stack = NULL;
141
142 if (delayed_call_chain) {
143 assert(to_opline != NULL); /* CALL and IP share the same register */
144 flags |= ZEND_JIT_EXIT_RESTORE_CALL;
145 }
146 if (JIT_G(current_frame)) {
147 op_array = &JIT_G(current_frame)->func->op_array;
148 stack_size = op_array->last_var + op_array->T;
149 if (stack_size) {
150 stack = JIT_G(current_frame)->stack;
151 do {
152 if (STACK_TYPE(stack, stack_size-1) != IS_UNKNOWN
153 || STACK_MEM_TYPE(stack, stack_size-1) != IS_UNKNOWN
154 || STACK_REF(stack, stack_size-1) != IR_UNUSED
155 ) {
156 break;
157 }
158 stack_size--;
159 } while (stack_size);
160 }
161 } else {
162 op_array = NULL;
163 stack_size = 0;
164 }
165
166 /* Try to reuse exit points */
167 if (to_opline != NULL
168 && !(flags & ZEND_JIT_EXIT_METHOD_CALL)
169 && t->exit_count > 0) {
170 uint32_t i = t->exit_count;
171
172 do {
173 i--;
174 if (stack_size == 0
175 || (t->exit_info[i].stack_size >= stack_size
176 && memcmp(t->stack_map + t->exit_info[i].stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack)) == 0)) {
177 if (t->exit_info[i].opline == to_opline
178 && t->exit_info[i].flags == flags
179 && t->exit_info[i].stack_size == stack_size) {
180 return i;
181 }
182 }
183 } while (i > 0);
184 }
185
186 exit_point = t->exit_count;
187 if (exit_point < ZEND_JIT_TRACE_MAX_EXITS) {
188 if (stack_size != 0 && stack_offset == (uint32_t)-1) {
189 stack_offset = t->stack_map_size;
190 t->stack_map_size += stack_size;
191 // TODO: reduce number of reallocations ???
192 t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
193 memcpy(t->stack_map + stack_offset, stack, stack_size * sizeof(zend_jit_trace_stack));
194 }
195 t->exit_count++;
196 t->exit_info[exit_point].opline = to_opline;
197 t->exit_info[exit_point].op_array = op_array;
198 t->exit_info[exit_point].flags = flags;
199 t->exit_info[exit_point].stack_size = stack_size;
200 t->exit_info[exit_point].stack_offset = stack_offset;
201 t->exit_info[exit_point].poly_func_ref = 0;
202 t->exit_info[exit_point].poly_this_ref = 0;
203 t->exit_info[exit_point].poly_func_reg = ZREG_NONE;
204 t->exit_info[exit_point].poly_this_reg = ZREG_NONE;
205 }
206
207 return exit_point;
208 }
209
zend_jit_trace_add_code(const void * start,uint32_t size)210 static void zend_jit_trace_add_code(const void *start, uint32_t size)
211 {
212 zend_jit_trace_info *t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
213
214 t->code_start = start;
215 t->code_size = size;
216 }
217
218 /**
219 * Locate a trace in the #zend_jit_traces array with the specified
220 * #code_start address.
221 *
222 * @return the #zend_jit_traces index or 0 if no such #code_start
223 * address was found
224 */
zend_jit_find_trace(const void * addr)225 static uint32_t zend_jit_find_trace(const void *addr)
226 {
227 uint32_t i;
228
229 for (i = 1; i < ZEND_JIT_TRACE_NUM; i++) {
230 if (zend_jit_traces[i].code_start == addr) {
231 return i;
232 }
233 }
234 return 0;
235 }
236
zend_jit_trace_name(const zend_op_array * op_array,uint32_t lineno)237 static zend_string *zend_jit_trace_name(const zend_op_array *op_array, uint32_t lineno)
238 {
239 smart_str buf = {0};
240
241 smart_str_appends(&buf, TRACE_PREFIX);
242 smart_str_append_long(&buf, (zend_long)ZEND_JIT_TRACE_NUM);
243 smart_str_appendc(&buf, '$');
244 if (op_array->function_name) {
245 if (op_array->scope) {
246 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
247 smart_str_appends(&buf, "::");
248 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
249 } else {
250 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
251 }
252 } else if (op_array->filename) {
253 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
254 }
255 smart_str_appendc(&buf, '$');
256 smart_str_append_long(&buf, (zend_long)lineno);
257 smart_str_0(&buf);
258 return buf.s;
259 }
260
zend_jit_trace_may_exit(const zend_op_array * op_array,const zend_op * opline)261 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline)
262 {
263 switch (opline->opcode) {
264 case ZEND_IS_IDENTICAL:
265 case ZEND_IS_NOT_IDENTICAL:
266 case ZEND_IS_EQUAL:
267 case ZEND_IS_NOT_EQUAL:
268 case ZEND_IS_SMALLER:
269 case ZEND_IS_SMALLER_OR_EQUAL:
270 case ZEND_CASE:
271 case ZEND_CASE_STRICT:
272 case ZEND_ISSET_ISEMPTY_CV:
273 case ZEND_ISSET_ISEMPTY_VAR:
274 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
275 case ZEND_ISSET_ISEMPTY_PROP_OBJ:
276 case ZEND_ISSET_ISEMPTY_STATIC_PROP:
277 case ZEND_INSTANCEOF:
278 case ZEND_TYPE_CHECK:
279 case ZEND_DEFINED:
280 case ZEND_IN_ARRAY:
281 case ZEND_ARRAY_KEY_EXISTS:
282 if (opline->result_type & (IS_SMART_BRANCH_JMPNZ | IS_SMART_BRANCH_JMPZ)) {
283 /* smart branch */
284 return 1;
285 }
286 break;
287 case ZEND_JMPZ:
288 case ZEND_JMPNZ:
289 case ZEND_JMPZ_EX:
290 case ZEND_JMPNZ_EX:
291 case ZEND_JMP_SET:
292 case ZEND_COALESCE:
293 case ZEND_JMP_NULL:
294 case ZEND_FE_RESET_R:
295 case ZEND_FE_RESET_RW:
296 case ZEND_ASSERT_CHECK:
297 case ZEND_FE_FETCH_R:
298 case ZEND_FE_FETCH_RW:
299 case ZEND_SWITCH_LONG:
300 case ZEND_SWITCH_STRING:
301 case ZEND_MATCH:
302 case ZEND_BIND_INIT_STATIC_OR_JMP:
303 case ZEND_JMP_FRAMELESS:
304 /* branch opcodes */
305 return 1;
306 case ZEND_NEW:
307 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
308 /* NEW may skip constructor without arguments */
309 return 1;
310 }
311 break;
312 case ZEND_CATCH:
313 case ZEND_FAST_CALL:
314 case ZEND_FAST_RET:
315 case ZEND_GENERATOR_CREATE:
316 case ZEND_GENERATOR_RETURN:
317 case ZEND_EXIT:
318 case ZEND_YIELD:
319 case ZEND_YIELD_FROM:
320 case ZEND_INCLUDE_OR_EVAL:
321 case ZEND_MATCH_ERROR:
322 /* unsupported */
323 return 1;
324 case ZEND_DO_FCALL:
325 /* potentially polymorphic call */
326 return 1;
327 #if 0
328 case ZEND_DO_UCALL:
329 case ZEND_DO_FCALL_BY_NAME:
330 /* monomorphic call */
331 // TODO: recompilation may change target ???
332 return 0;
333 #endif
334 case ZEND_RETURN_BY_REF:
335 case ZEND_RETURN:
336 /* return */
337 return !JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame));
338 default:
339 break;
340 }
341 return 0;
342 }
343
zend_jit_trace_type_to_info_ex(uint8_t type,uint32_t info)344 static zend_always_inline uint32_t zend_jit_trace_type_to_info_ex(uint8_t type, uint32_t info)
345 {
346 if (type == IS_UNKNOWN) {
347 return info;
348 }
349 ZEND_ASSERT(info & (1 << type));
350 if (type < IS_STRING) {
351 return (1 << type);
352 } else if (type != IS_ARRAY) {
353 return (1 << type) | (info & (MAY_BE_RC1|MAY_BE_RCN));
354 } else {
355 return MAY_BE_ARRAY | (info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
356 }
357 }
358
zend_jit_trace_type_to_info(uint8_t type)359 static zend_always_inline uint32_t zend_jit_trace_type_to_info(uint8_t type)
360 {
361 return zend_jit_trace_type_to_info_ex(type, -1);
362 }
363
zend_jit_var_may_alias(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t var)364 static zend_always_inline zend_ssa_alias_kind zend_jit_var_may_alias(const zend_op_array *op_array, const zend_ssa *ssa, uint32_t var)
365 {
366 if (var >= op_array->last_var) {
367 return NO_ALIAS;
368 } else if ((!op_array->function_name || (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS))) {
369 return SYMTABLE_ALIAS;
370 } else if (ssa->vars) {
371 return ssa->vars[var].alias;
372 } else if (zend_string_equals_literal(op_array->vars[var], "http_response_header")) {
373 return HTTP_RESPONSE_HEADER_ALIAS;
374 }
375 return NO_ALIAS;
376 }
377
zend_jit_trace_add_op_guard(zend_ssa * tssa,int ssa_var,uint8_t op_type)378 static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa *tssa,
379 int ssa_var,
380 uint8_t op_type)
381 {
382 zend_ssa_var_info *info = &tssa->var_info[ssa_var];
383
384 if ((info->type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << op_type)) {
385 if (UNEXPECTED(tssa->vars[ssa_var].alias != NO_ALIAS)) {
386 info->type |= MAY_BE_GUARD;
387 } else {
388 info->type = MAY_BE_GUARD | zend_jit_trace_type_to_info_ex(op_type, info->type);
389 }
390 }
391 }
392
393 #define ADD_OP_GUARD(_ssa_var, _op_type) do { \
394 if (_ssa_var >= 0 && _op_type != IS_UNKNOWN) { \
395 zend_jit_trace_add_op_guard(tssa, _ssa_var, _op_type); \
396 } \
397 } while (0)
398
399 #define CHECK_OP_TRACE_TYPE(_var, _ssa_var, op_info, op_type) do { \
400 if (op_type != IS_UNKNOWN) { \
401 if ((op_info & MAY_BE_GUARD) != 0) { \
402 if (!zend_jit_type_guard(&ctx, opline, _var, op_type)) { \
403 goto jit_failure; \
404 } \
405 if (ssa->vars[_ssa_var].alias != NO_ALIAS) { \
406 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), IS_UNKNOWN, 1); \
407 op_info = zend_jit_trace_type_to_info(op_type); \
408 } else { \
409 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(_var), op_type, 1); \
410 op_info &= ~MAY_BE_GUARD; \
411 ssa->var_info[_ssa_var].type &= op_info; \
412 } \
413 } \
414 } \
415 } while (0)
416
417 #define ADD_OP1_TRACE_GUARD() \
418 ADD_OP_GUARD(tssa->ops[idx].op1_use, op1_type)
419 #define ADD_OP2_TRACE_GUARD() \
420 ADD_OP_GUARD(tssa->ops[idx].op2_use, op2_type)
421 #define ADD_OP1_DATA_TRACE_GUARD() \
422 ADD_OP_GUARD(tssa->ops[idx+1].op1_use, op3_type)
423
424 #define CHECK_OP1_TRACE_TYPE() \
425 CHECK_OP_TRACE_TYPE(opline->op1.var, ssa_op->op1_use, op1_info, op1_type)
426 #define CHECK_OP2_TRACE_TYPE() \
427 CHECK_OP_TRACE_TYPE(opline->op2.var, ssa_op->op2_use, op2_info, op2_type)
428 #define CHECK_OP1_DATA_TRACE_TYPE() \
429 CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type)
430
zend_jit_trace_frame_size(const zend_op_array * op_array)431 static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array)
432 {
433 if (op_array && op_array->type == ZEND_USER_FUNCTION) {
434 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack)));
435 } else if (op_array) {
436 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack)));
437 } else {
438 return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack));
439 }
440 }
441
zend_jit_trace_call_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)442 static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
443 {
444 return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array));
445 }
446
zend_jit_trace_ret_frame(zend_jit_trace_stack_frame * frame,const zend_op_array * op_array)447 static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array)
448 {
449 return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array));
450 }
451
zend_jit_trace_send_type(const zend_op * opline,zend_jit_trace_stack_frame * call,uint8_t type)452 static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, uint8_t type)
453 {
454 zend_jit_trace_stack *stack = call->stack;
455 const zend_op_array *op_array = &call->func->op_array;
456 uint32_t arg_num = opline->op2.num;
457
458 if (arg_num > op_array->num_args) {
459 return;
460 }
461 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
462 zend_arg_info *arg_info;
463
464 ZEND_ASSERT(arg_num <= op_array->num_args);
465 arg_info = &op_array->arg_info[arg_num-1];
466
467 if (ZEND_TYPE_IS_SET(arg_info->type)) {
468 if (!(ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
469 return;
470 }
471 }
472 }
473 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
474 }
475
zend_jit_needs_arg_dtor(const zend_function * func,uint32_t arg_num,zend_call_info * call_info)476 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info)
477 {
478 if (func
479 && func->type == ZEND_INTERNAL_FUNCTION
480 && (func->internal_function.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0
481 && arg_num < func->internal_function.num_args) {
482 const zend_internal_arg_info *arg_info = &func->internal_function.arg_info[arg_num];
483
484 if (ZEND_ARG_SEND_MODE(arg_info) == ZEND_SEND_BY_VAL
485 && ZEND_TYPE_IS_SET(arg_info->type)
486 && (ZEND_TYPE_FULL_MASK(arg_info->type) & MAY_BE_ANY) != MAY_BE_ANY) {
487 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
488 && JIT_G(current_frame)
489 && JIT_G(current_frame)->call
490 && JIT_G(current_frame)->call->func) {
491 uint32_t type = STACK_TYPE(JIT_G(current_frame)->call->stack, arg_num);
492
493 if (type != IS_UNKNOWN
494 && type < IS_STRING
495 && ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type)) {
496 return 0;
497 }
498 }
499 if (call_info && arg_num < call_info->num_args && call_info->arg_info[arg_num].opline) {
500 const zend_op *opline = call_info->arg_info[arg_num].opline;
501
502 if (opline->opcode == ZEND_SEND_VAL && opline->op1_type == IS_CONST) {
503 zval *zv = RT_CONSTANT(opline, opline->op1);
504
505 if (!Z_REFCOUNTED_P(zv)) {
506 uint32_t type = Z_TYPE_P(zv);
507
508 // TODO: few functions (e.g. pcntl_exec) modify arrays in-place ???
509 if (type != IS_ARRAY
510 && (ZEND_TYPE_FULL_MASK(arg_info->type) & (1u << type))) {
511 return 0;
512 }
513 }
514 }
515 }
516 }
517 }
518
519 return 1;
520 }
521
zend_jit_trace_build_ssa(const zend_op_array * op_array,zend_script * script)522 static zend_ssa *zend_jit_trace_build_ssa(const zend_op_array *op_array, zend_script *script)
523 {
524 zend_jit_op_array_trace_extension *jit_extension;
525 zend_ssa *ssa;
526
527 jit_extension =
528 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
529 jit_extension->func_info.num = 0;
530 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
531 | ZEND_FUNC_JIT_ON_PROF_REQUEST
532 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
533 | ZEND_FUNC_JIT_ON_HOT_TRACE;
534 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
535 ssa = &jit_extension->func_info.ssa;
536
537 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
538 do {
539 if (zend_jit_op_array_analyze1(op_array, script, ssa) != SUCCESS) {
540 break;
541 }
542
543 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
544 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, (zend_op_array*)op_array, &jit_extension->func_info);
545 jit_extension->func_info.call_map = zend_build_call_map(&CG(arena), &jit_extension->func_info, op_array);
546 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
547 zend_init_func_return_info(op_array, script, &jit_extension->func_info.return_info);
548 }
549 }
550
551 if (zend_jit_op_array_analyze2(op_array, script, ssa, 0) != SUCCESS) {
552 break;
553 }
554
555 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
556 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", ssa);
557 }
558 return ssa;
559 } while (0);
560 }
561
562 memset(ssa, 0, sizeof(zend_ssa));
563 ssa->cfg.blocks_count = 1;
564
565 if (JIT_G(opt_level) == ZEND_JIT_LEVEL_INLINE) {
566 zend_cfg cfg;
567 void *checkpoint = zend_arena_checkpoint(CG(arena));
568
569 if (zend_jit_build_cfg(op_array, &cfg) == SUCCESS) {
570 ssa->cfg.flags = cfg.flags;
571 } else{
572 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
573 }
574
575 /* TODO: move this to zend_cfg.c ? */
576 if (!op_array->function_name) {
577 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
578 }
579
580 zend_arena_release(&CG(arena), checkpoint);
581 }
582
583 return ssa;
584 }
585
586 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa);
587 static void zend_jit_dump_exit_info(zend_jit_trace_info *t);
588
zend_jit_trace_op_len(const zend_op * opline)589 static zend_always_inline int zend_jit_trace_op_len(const zend_op *opline)
590 {
591 int len;
592
593 switch (opline->opcode) {
594 case ZEND_ASSIGN_DIM:
595 case ZEND_ASSIGN_OBJ:
596 case ZEND_ASSIGN_STATIC_PROP:
597 case ZEND_ASSIGN_DIM_OP:
598 case ZEND_ASSIGN_OBJ_OP:
599 case ZEND_ASSIGN_STATIC_PROP_OP:
600 case ZEND_ASSIGN_OBJ_REF:
601 case ZEND_ASSIGN_STATIC_PROP_REF:
602 case ZEND_FRAMELESS_ICALL_3:
603 return 2; /* OP_DATA */
604 case ZEND_RECV_INIT:
605 len = 1;
606 opline++;
607 while (opline->opcode == ZEND_RECV_INIT) {
608 len++;
609 opline++;
610 }
611 return len;
612 case ZEND_BIND_GLOBAL:
613 len = 1;
614 opline++;
615 while (opline->opcode == ZEND_BIND_GLOBAL) {
616 len++;
617 opline++;
618 }
619 return len;
620 // case ZEND_IS_IDENTICAL:
621 // case ZEND_IS_NOT_IDENTICAL:
622 // case ZEND_IS_EQUAL:
623 // case ZEND_IS_NOT_EQUAL:
624 // case ZEND_IS_SMALLER:
625 // case ZEND_IS_SMALLER_OR_EQUAL:
626 // case ZEND_CASE:
627 // case ZEND_ISSET_ISEMPTY_CV:
628 // case ZEND_ISSET_ISEMPTY_VAR:
629 // case ZEND_ISSET_ISEMPTY_DIM_OBJ:
630 // case ZEND_ISSET_ISEMPTY_PROP_OBJ:
631 // case ZEND_ISSET_ISEMPTY_STATIC_PROP:
632 // case ZEND_INSTANCEOF:
633 // case ZEND_TYPE_CHECK:
634 // case ZEND_DEFINED:
635 // case ZEND_IN_ARRAY:
636 // case ZEND_ARRAY_KEY_EXISTS:
637 default:
638 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
639 return 2; /* JMPZ/JMPNZ */
640 }
641 return 1;
642 }
643 }
644
zend_jit_trace_add_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)645 static int zend_jit_trace_add_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
646 {
647 const zend_op_array *op_array;
648 zend_jit_trace_rec *p;
649 int k, vars_count;
650 zend_bitset use, def;
651 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
652 uint32_t set_size;
653 zend_ssa_phi *prev = NULL;
654 int level = 0;
655 ALLOCA_FLAG(use_heap);
656
657 op_array = trace_buffer->op_array;
658 set_size = zend_bitset_len(op_array->last_var + op_array->T);
659 use = ZEND_BITSET_ALLOCA(set_size * 2, use_heap);
660 memset(use, 0, set_size * 2 * ZEND_BITSET_ELM_SIZE);
661 def = use + set_size;
662 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
663 for (;;p++) {
664 if (p->op == ZEND_JIT_TRACE_VM && level == 0) {
665 const zend_op *opline = p->opline;
666 int len;
667
668 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
669 len = zend_jit_trace_op_len(opline);
670 while (len > 1) {
671 opline++;
672 if (opline->opcode != ZEND_OP_DATA) {
673 zend_dfg_add_use_def_op(op_array, opline, build_flags, use, def);
674 }
675 len--;
676 }
677 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
678 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
679 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
680 level++;
681 } else if (p->op == ZEND_JIT_TRACE_BACK) {
682 if (level == 0) {
683 // Phi for recursive calls and returns are not supported yet ???
684 assert(0);
685 } else {
686 level--;
687 }
688 } else if (p->op == ZEND_JIT_TRACE_END) {
689 break;
690 }
691 }
692
693 zend_bitset_intersection(use, def, set_size);
694
695 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
696 vars_count = op_array->last_var;
697 } else {
698 vars_count = op_array->last_var + op_array->T;
699 }
700 for (k = 0; k < vars_count; k++) {
701 if (zend_bitset_in(use, k)) {
702 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
703 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
704 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
705 sizeof(void*) * 2);
706 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
707 phi->sources[0] = STACK_VAR(stack, k);
708 phi->sources[1] = -1;
709 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
710 phi->pi = -1;
711 phi->var = k;
712 phi->ssa_var = ssa_vars_count;
713 SET_STACK_VAR(stack, k, ssa_vars_count);
714 ssa_vars_count++;
715 phi->block = 1;
716 if (prev) {
717 prev->next = phi;
718 } else {
719 tssa->blocks[1].phis = phi;
720 }
721 prev = phi;
722 }
723 }
724
725 free_alloca(use, use_heap);
726
727 return ssa_vars_count;
728 }
729
zend_jit_trace_add_call_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)730 static int zend_jit_trace_add_call_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
731 {
732 zend_ssa_phi *prev = NULL;
733 const zend_op_array *op_array = trace_buffer->op_array;
734 const zend_op *opline = trace_buffer[1].opline;
735 int count = opline - op_array->opcodes;
736 int i;
737
738 for(i = 0; i < count; i++) {
739 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
740 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
741 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
742 sizeof(void*) * 2);
743 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
744 phi->sources[0] = STACK_VAR(stack, i);
745 phi->sources[1] = -1;
746 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
747 phi->pi = -1;
748 phi->var = i;
749 phi->ssa_var = ssa_vars_count;
750 SET_STACK_VAR(stack, i, ssa_vars_count);
751 ssa_vars_count++;
752 phi->block = 1;
753 if (prev) {
754 prev->next = phi;
755 } else {
756 tssa->blocks[1].phis = phi;
757 }
758 prev = phi;
759 }
760 return ssa_vars_count;
761 }
762
zend_jit_trace_add_ret_phis(zend_jit_trace_rec * trace_buffer,uint32_t ssa_vars_count,zend_ssa * tssa,zend_jit_trace_stack * stack)763 static int zend_jit_trace_add_ret_phis(zend_jit_trace_rec *trace_buffer, uint32_t ssa_vars_count, zend_ssa *tssa, zend_jit_trace_stack *stack)
764 {
765 const zend_op *opline = trace_buffer[1].opline - 1;
766 int i;
767
768 if (RETURN_VALUE_USED(opline)) {
769 zend_ssa_phi *phi = zend_arena_calloc(&CG(arena), 1,
770 ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)) +
771 ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2) +
772 sizeof(void*) * 2);
773
774 i = EX_VAR_TO_NUM(opline->result.var);
775 phi->sources = (int*)(((char*)phi) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_ssa_phi)));
776 phi->sources[0] = STACK_VAR(stack, i);
777 phi->sources[1] = -1;
778 phi->use_chains = (zend_ssa_phi**)(((char*)phi->sources) + ZEND_MM_ALIGNED_SIZE(sizeof(int) * 2));
779 phi->pi = -1;
780 phi->var = i;
781 phi->ssa_var = ssa_vars_count;
782 SET_STACK_VAR(stack, i, ssa_vars_count);
783 ssa_vars_count++;
784 phi->block = 1;
785 tssa->blocks[1].phis = phi;
786 }
787 return ssa_vars_count;
788 }
789
zend_jit_trace_is_false_loop(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa)790 static bool zend_jit_trace_is_false_loop(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa)
791 {
792 const zend_op *opline;
793 uint32_t b;
794 zend_basic_block *bb;
795
796 ZEND_ASSERT(tssa->cfg.blocks_count == 2);
797 ZEND_ASSERT(tssa->cfg.blocks[1].len > 0);
798
799 b = ssa->cfg.map[tssa_opcodes[0] - op_array->opcodes];
800 opline = tssa_opcodes[tssa->cfg.blocks[1].len - 1];
801 if (opline >= op_array->opcodes && opline < op_array->opcodes + op_array->last) {
802 bb = ssa->cfg.blocks + ssa->cfg.map[opline - op_array->opcodes];
803 return bb->loop_header != b;
804 } else {
805 return 0;
806 }
807 }
808
zend_jit_trace_copy_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)809 static int zend_jit_trace_copy_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
810 {
811 int var, use, def, src;
812 zend_ssa_op *op;
813
814 if (tssa->vars[ssa_var].definition_phi) {
815 uint32_t b = ssa->cfg.map[tssa_opcodes[0] - op_array->opcodes];
816 zend_basic_block *bb = ssa->cfg.blocks + b;
817
818 if ((bb->flags & ZEND_BB_LOOP_HEADER)
819 && !zend_jit_trace_is_false_loop(op_array, ssa, tssa_opcodes, tssa)) {
820 zend_ssa_phi *phi = ssa->blocks[b].phis;
821 zend_ssa_phi *pi = NULL;
822
823 var = tssa->vars[ssa_var].var;
824 while (phi) {
825 if (ssa->vars[phi->ssa_var].var == var) {
826 if (phi->pi >= 0) {
827 pi = phi;
828 } else {
829 src = phi->ssa_var;
830 goto copy_info;
831 }
832 }
833 phi = phi->next;
834 }
835 if (pi) {
836 src = pi->ssa_var;
837 goto copy_info;
838 }
839 #if 0
840 while (bb->idom >= 0) {
841 uint32_t n;
842
843 b = bb->idom;
844 bb = ssa->cfg.blocks + b;
845
846 for (n = bb->len, op = ssa->ops + bb->start + n; n > 0; n--) {
847 op--;
848 if (op->result_def >= 0 && ssa->vars[op->result_def].var == var) {
849 src = op->result_def;
850 goto copy_info;
851 } else if (op->op2_def >= 0 && ssa->vars[op->op2_def].var == var) {
852 src = op->op2_def;
853 goto copy_info;
854 } else if (op->op1_def >= 0 && ssa->vars[op->op1_def].var == var) {
855 src = op->op1_def;
856 goto copy_info;
857 }
858 }
859
860 phi = ssa->blocks[b].phis;
861 zend_ssa_phi *pi = NULL;
862 while (phi) {
863 if (ssa->vars[phi->ssa_var].var == var) {
864 if (phi->pi >= 0) {
865 pi = phi;
866 } else {
867 src = phi->ssa_var;
868 goto copy_info;
869 }
870 }
871 phi = phi->next;
872 }
873 if (pi) {
874 src = pi->ssa_var;
875 goto copy_info;
876 }
877 }
878 #endif
879 }
880 } else if (tssa->vars[ssa_var].definition >= 0) {
881 def = tssa->vars[ssa_var].definition;
882 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
883 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
884 if (tssa->ops[def].op1_def == ssa_var) {
885 src = op->op1_def;
886 } else if (tssa->ops[def].op2_def == ssa_var) {
887 src = op->op2_def;
888 } else if (tssa->ops[def].result_def == ssa_var) {
889 src = op->result_def;
890 } else {
891 assert(0);
892 return 0;
893 }
894 goto copy_info;
895 }
896
897 if (tssa->vars[ssa_var].phi_use_chain) {
898 // TODO: this may be incorrect ???
899 var = tssa->vars[ssa_var].phi_use_chain->ssa_var;
900 } else {
901 var = ssa_var;
902 }
903 use = tssa->vars[var].use_chain;
904 if (use >= 0) {
905 ZEND_ASSERT((tssa_opcodes[use] - op_array->opcodes) < op_array->last);
906 op = ssa->ops + (tssa_opcodes[use] - op_array->opcodes);
907 if (tssa->ops[use].op1_use == var) {
908 src = op->op1_use;
909 } else if (tssa->ops[use].op2_use == var) {
910 src = op->op2_use;
911 } else if (tssa->ops[use].result_use == var) {
912 src = op->result_use;
913 } else {
914 assert(0);
915 return 0;
916 }
917 goto copy_info;
918 }
919 return 0;
920
921 copy_info:
922 tssa->vars[ssa_var].no_val = ssa->vars[src].no_val;
923 tssa->vars[ssa_var].alias = ssa->vars[src].alias;
924 memcpy(&tssa->var_info[ssa_var], &ssa->var_info[src], sizeof(zend_ssa_var_info));
925 return 1;
926 }
927
zend_jit_trace_propagate_range(const zend_op_array * op_array,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)928 static void zend_jit_trace_propagate_range(const zend_op_array *op_array, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
929 {
930 zend_ssa_range tmp;
931 int def = tssa->vars[ssa_var].definition;
932
933 if (tssa->vars[ssa_var].alias == NO_ALIAS
934 && zend_inference_propagate_range(op_array, tssa, tssa_opcodes[def], &tssa->ops[def], ssa_var, &tmp)) {
935 tssa->var_info[ssa_var].range.min = tmp.min;
936 tssa->var_info[ssa_var].range.max = tmp.max;
937 tssa->var_info[ssa_var].range.underflow = tmp.underflow;
938 tssa->var_info[ssa_var].range.overflow = tmp.overflow;
939 tssa->var_info[ssa_var].has_range = 1;
940 }
941 }
942
zend_jit_trace_copy_ssa_var_range(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)943 static void zend_jit_trace_copy_ssa_var_range(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
944 {
945 int def;
946 zend_ssa_op *op;
947 zend_ssa_var_info *info;
948 unsigned int no_val;
949 zend_ssa_alias_kind alias;
950
951 def = tssa->vars[ssa_var].definition;
952 if (def >= 0) {
953 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
954 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
955 if (tssa->ops[def].op1_def == ssa_var) {
956 no_val = ssa->vars[op->op1_def].no_val;
957 alias = ssa->vars[op->op1_def].alias;
958 info = ssa->var_info + op->op1_def;
959 } else if (tssa->ops[def].op2_def == ssa_var) {
960 no_val = ssa->vars[op->op2_def].no_val;
961 alias = ssa->vars[op->op2_def].alias;
962 info = ssa->var_info + op->op2_def;
963 } else if (tssa->ops[def].result_def == ssa_var) {
964 no_val = ssa->vars[op->result_def].no_val;
965 alias = ssa->vars[op->result_def].alias;
966 info = ssa->var_info + op->result_def;
967 } else {
968 assert(0);
969 return;
970 }
971
972 tssa->vars[ssa_var].no_val = no_val;
973 tssa->vars[ssa_var].alias = alias;
974
975 if (!(info->type & MAY_BE_REF)) {
976 zend_jit_trace_propagate_range(op_array, tssa_opcodes, tssa, ssa_var);
977 }
978
979 if (info->has_range) {
980 if (tssa->var_info[ssa_var].has_range) {
981 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
982 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
983 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
984 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
985 } else {
986 tssa->var_info[ssa_var].has_range = 1;
987 tssa->var_info[ssa_var].range = info->range;
988 }
989 }
990 }
991 }
992
zend_jit_trace_restrict_ssa_var_info(const zend_op_array * op_array,const zend_ssa * ssa,const zend_op ** tssa_opcodes,zend_ssa * tssa,int ssa_var)993 static int zend_jit_trace_restrict_ssa_var_info(const zend_op_array *op_array, const zend_ssa *ssa, const zend_op **tssa_opcodes, zend_ssa *tssa, int ssa_var)
994 {
995 int def;
996 zend_ssa_op *op;
997 zend_ssa_var_info *info;
998
999 def = tssa->vars[ssa_var].definition;
1000 if (def >= 0) {
1001 ZEND_ASSERT((tssa_opcodes[def] - op_array->opcodes) < op_array->last);
1002 op = ssa->ops + (tssa_opcodes[def] - op_array->opcodes);
1003 if (tssa->ops[def].op1_def == ssa_var) {
1004 info = ssa->var_info + op->op1_def;
1005 } else if (tssa->ops[def].op2_def == ssa_var) {
1006 info = ssa->var_info + op->op2_def;
1007 } else if (tssa->ops[def].result_def == ssa_var) {
1008 info = ssa->var_info + op->result_def;
1009 } else {
1010 assert(0);
1011 return 0;
1012 }
1013 tssa->var_info[ssa_var].type &= info->type;
1014 if (info->ce) {
1015 if (tssa->var_info[ssa_var].ce) {
1016 if (tssa->var_info[ssa_var].ce != info->ce) {
1017 if (instanceof_function(tssa->var_info[ssa_var].ce, info->ce)) {
1018 /* everything fine */
1019 } else if (instanceof_function(info->ce, tssa->var_info[ssa_var].ce)) {
1020 // TODO: TSSA may miss Pi() functions and corresponding instanceof() constraints ???
1021 } else {
1022 // TODO: classes may implement the same interface ???
1023 //ZEND_UNREACHABLE();
1024 }
1025 }
1026 tssa->var_info[ssa_var].is_instanceof =
1027 tssa->var_info[ssa_var].is_instanceof && info->is_instanceof;
1028 } else {
1029 tssa->var_info[ssa_var].ce = info->ce;
1030 tssa->var_info[ssa_var].is_instanceof = info->is_instanceof;
1031 }
1032 }
1033 if (info->has_range) {
1034 if (tssa->var_info[ssa_var].has_range) {
1035 tssa->var_info[ssa_var].range.min = MAX(tssa->var_info[ssa_var].range.min, info->range.min);
1036 tssa->var_info[ssa_var].range.max = MIN(tssa->var_info[ssa_var].range.max, info->range.max);
1037 tssa->var_info[ssa_var].range.underflow = tssa->var_info[ssa_var].range.underflow && info->range.underflow;
1038 tssa->var_info[ssa_var].range.overflow = tssa->var_info[ssa_var].range.overflow && info->range.overflow;
1039 } else {
1040 tssa->var_info[ssa_var].has_range = 1;
1041 tssa->var_info[ssa_var].range = info->range;
1042 }
1043 }
1044 return 1;
1045 }
1046 return 0;
1047 }
1048
find_return_ssa_var(zend_jit_trace_rec * p,zend_ssa_op * ssa_op)1049 static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op)
1050 {
1051 while (1) {
1052 if (p->op == ZEND_JIT_TRACE_VM) {
1053 if (p->opline->opcode == ZEND_DO_UCALL
1054 || p->opline->opcode == ZEND_DO_FCALL_BY_NAME
1055 || p->opline->opcode == ZEND_DO_FCALL) {
1056 if (p->opline->result_type != IS_UNUSED) {
1057 return ssa_op->result_def;
1058 }
1059 }
1060 return -1;
1061 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
1062 /*skip */
1063 } else {
1064 return -1;
1065 }
1066 p--;
1067 }
1068 }
1069
zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec * p,const zend_op_array * op_array)1070 static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, const zend_op_array *op_array)
1071 {
1072 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
1073 p--;
1074 while (1) {
1075 if (p->op == ZEND_JIT_TRACE_VM) {
1076 if (p->opline->opcode == ZEND_INIT_FCALL
1077 || p->opline->opcode == ZEND_INIT_FCALL_BY_NAME
1078 || p->opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME
1079 || p->opline->opcode == ZEND_INIT_DYNAMIC_CALL
1080 || p->opline->opcode == ZEND_INIT_USER_CALL
1081 || p->opline->opcode == ZEND_NEW
1082 || p->opline->opcode == ZEND_INIT_METHOD_CALL
1083 || p->opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
1084 return p->opline;
1085 }
1086 return NULL;
1087 } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) {
1088 /*skip */
1089 } else {
1090 return NULL;
1091 }
1092 p--;
1093 }
1094 } else {
1095 const zend_op *opline = NULL;
1096 int call_level = 0;
1097
1098 p++;
1099 while (1) {
1100 if (p->op == ZEND_JIT_TRACE_VM) {
1101 opline = p->opline;
1102 break;
1103 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1104 call_level++;
1105 /*skip */
1106 } else {
1107 return NULL;
1108 }
1109 p--;
1110 }
1111 if (opline) {
1112 while (opline > op_array->opcodes) {
1113 opline--;
1114 switch (opline->opcode) {
1115 case ZEND_INIT_FCALL:
1116 case ZEND_INIT_FCALL_BY_NAME:
1117 case ZEND_INIT_NS_FCALL_BY_NAME:
1118 case ZEND_INIT_METHOD_CALL:
1119 case ZEND_INIT_DYNAMIC_CALL:
1120 case ZEND_INIT_STATIC_METHOD_CALL:
1121 case ZEND_INIT_USER_CALL:
1122 case ZEND_NEW:
1123 if (call_level == 0) {
1124 return opline;
1125 }
1126 call_level--;
1127 break;
1128 case ZEND_DO_FCALL:
1129 case ZEND_DO_ICALL:
1130 case ZEND_DO_UCALL:
1131 case ZEND_DO_FCALL_BY_NAME:
1132 case ZEND_CALLABLE_CONVERT:
1133 call_level++;
1134 break;
1135 }
1136 }
1137 }
1138 }
1139 return NULL;
1140 }
1141
is_checked_guard(const zend_ssa * tssa,const zend_op ** ssa_opcodes,uint32_t var,uint32_t phi_var)1142 static int is_checked_guard(const zend_ssa *tssa, const zend_op **ssa_opcodes, uint32_t var, uint32_t phi_var)
1143 {
1144 if ((tssa->var_info[phi_var].type & MAY_BE_ANY) == MAY_BE_LONG
1145 && !(tssa->var_info[var].type & MAY_BE_REF)) {
1146 int idx = tssa->vars[var].definition;
1147
1148 if (idx >= 0) {
1149 if (tssa->ops[idx].op1_def == var) {
1150 const zend_op *opline = ssa_opcodes[idx];
1151 if (opline->opcode == ZEND_PRE_DEC
1152 || opline->opcode == ZEND_PRE_INC
1153 || opline->opcode == ZEND_POST_DEC
1154 || opline->opcode == ZEND_POST_INC) {
1155 if (tssa->ops[idx].op1_use >= 0
1156 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_STRING)) {
1157 return 0;
1158 }
1159 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1160 return 0;
1161 }
1162 return 1;
1163 } else if (opline->opcode == ZEND_ASSIGN_OP
1164 && (opline->extended_value == ZEND_ADD
1165 || opline->extended_value == ZEND_SUB
1166 || opline->extended_value == ZEND_MUL)) {
1167 if ((opline->op2_type & (IS_VAR|IS_CV))
1168 && tssa->ops[idx].op2_use >= 0
1169 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1170 return 0;
1171 }
1172 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1173 return 0;
1174 }
1175 if (opline->op2_type == IS_CONST) {
1176 zval *zv = RT_CONSTANT(opline, opline->op2);
1177 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1178 return 0;
1179 }
1180 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1181 return 0;
1182 }
1183 return 1;
1184 }
1185 }
1186 if (tssa->ops[idx].result_def == var) {
1187 const zend_op *opline = ssa_opcodes[idx];
1188 if (opline->opcode == ZEND_ADD
1189 || opline->opcode == ZEND_SUB
1190 || opline->opcode == ZEND_MUL) {
1191 if ((opline->op1_type & (IS_VAR|IS_CV))
1192 && tssa->ops[idx].op1_use >= 0
1193 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1194 return 0;
1195 }
1196 if ((opline->op2_type & (IS_VAR|IS_CV))
1197 && tssa->ops[idx].op2_use >= 0
1198 && (tssa->var_info[tssa->ops[idx].op2_use].type & MAY_BE_REF)) {
1199 return 0;
1200 }
1201 if (opline->op1_type == IS_CONST) {
1202 zval *zv = RT_CONSTANT(opline, opline->op1);
1203 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1204 return 0;
1205 }
1206 } else if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1207 return 0;
1208 }
1209 if (opline->op2_type == IS_CONST) {
1210 zval *zv = RT_CONSTANT(opline, opline->op2);
1211 if (Z_TYPE_P(zv) != IS_LONG && Z_TYPE_P(zv) != IS_DOUBLE) {
1212 return 0;
1213 }
1214 } else if (!(tssa->var_info[tssa->ops[idx].op2_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1215 return 0;
1216 }
1217 return 1;
1218 } else if (opline->opcode == ZEND_PRE_DEC
1219 || opline->opcode == ZEND_PRE_INC
1220 || opline->opcode == ZEND_POST_DEC
1221 || opline->opcode == ZEND_POST_INC) {
1222 if ((opline->op1_type & (IS_VAR|IS_CV))
1223 && tssa->ops[idx].op1_use >= 0
1224 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1225 return 0;
1226 }
1227 if (!(tssa->var_info[tssa->ops[idx].op1_use].type & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1228 return 0;
1229 }
1230 return 1;
1231 }
1232 }
1233 }
1234 }
1235 return 0;
1236 }
1237
1238 typedef struct _zend_tssa {
1239 zend_ssa ssa;
1240 const zend_op **tssa_opcodes;
1241 int used_stack;
1242 } zend_tssa;
1243
1244 static const zend_op _nop_opcode = {0};
1245
zend_jit_trace_build_tssa(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num,zend_script * script,const zend_op_array ** op_arrays,int * num_op_arrays_ptr)1246 static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr)
1247 {
1248 zend_ssa *tssa;
1249 zend_ssa_op *ssa_ops, *op;
1250 zend_ssa_var *ssa_vars;
1251 zend_ssa_var_info *ssa_var_info;
1252 const zend_op_array *op_array;
1253 const zend_op *opline;
1254 const zend_op **ssa_opcodes;
1255 zend_jit_trace_rec *p;
1256 int i, v, idx, len, ssa_ops_count, vars_count, ssa_vars_count;
1257 zend_jit_trace_stack *stack;
1258 uint32_t build_flags = ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS;
1259 uint32_t optimization_level = 0;
1260 int call_level, level, num_op_arrays, used_stack, max_used_stack;
1261 size_t frame_size, stack_top, stack_size, stack_bottom;
1262 zend_jit_op_array_trace_extension *jit_extension;
1263 zend_ssa *ssa;
1264 zend_jit_trace_stack_frame *frame, *top, *call;
1265 zend_ssa_var_info return_value_info;
1266
1267 /* 1. Count number of TSSA opcodes;
1268 * Count number of activation frames;
1269 * Calculate size of abstract stack;
1270 * Construct regular SSA for involved op_array */
1271 op_array = trace_buffer->op_array;
1272 stack_top = stack_size = zend_jit_trace_frame_size(op_array);
1273 stack_bottom = 0;
1274 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1275 ssa_ops_count = 0;
1276 call_level = 0;
1277 level = 0;
1278 num_op_arrays = 0;
1279 /* Remember op_array to cleanup */
1280 op_arrays[num_op_arrays++] = op_array;
1281 /* Build SSA */
1282 ssa = zend_jit_trace_build_ssa(op_array, script);
1283 for (;;p++) {
1284 if (p->op == ZEND_JIT_TRACE_VM) {
1285 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1286 const zend_op *opline = p->opline;
1287
1288 switch (opline->opcode) {
1289 case ZEND_INCLUDE_OR_EVAL:
1290 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1291 break;
1292 case ZEND_FETCH_R:
1293 case ZEND_FETCH_W:
1294 case ZEND_FETCH_RW:
1295 case ZEND_FETCH_FUNC_ARG:
1296 case ZEND_FETCH_IS:
1297 case ZEND_FETCH_UNSET:
1298 case ZEND_UNSET_VAR:
1299 case ZEND_ISSET_ISEMPTY_VAR:
1300 if (opline->extended_value & ZEND_FETCH_LOCAL) {
1301 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1302 } else if ((opline->extended_value & (ZEND_FETCH_GLOBAL | ZEND_FETCH_GLOBAL_LOCK)) &&
1303 !op_array->function_name) {
1304 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1305 }
1306 break;
1307 }
1308 }
1309 ssa_ops_count += zend_jit_trace_op_len(p->opline);
1310 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
1311 call_level++;
1312 stack_top += zend_jit_trace_frame_size(p->op_array);
1313 if (stack_top > stack_size) {
1314 stack_size = stack_top;
1315 }
1316 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
1317 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) {
1318 if (p->func != (zend_function*)&zend_pass_function
1319 && (zend_string_equals_literal(p->func->common.function_name, "extract")
1320 || zend_string_equals_literal(p->func->common.function_name, "compact")
1321 || zend_string_equals_literal(p->func->common.function_name, "get_defined_vars"))) {
1322 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1323 }
1324 }
1325 frame_size = zend_jit_trace_frame_size(p->op_array);
1326 if (call_level == 0) {
1327 if (stack_top + frame_size > stack_size) {
1328 stack_size = stack_top + frame_size;
1329 }
1330 } else {
1331 call_level--;
1332 stack_top -= frame_size;
1333 }
1334 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1335 op_array = p->op_array;
1336 if (call_level == 0) {
1337 stack_top += zend_jit_trace_frame_size(op_array);
1338 if (stack_top > stack_size) {
1339 stack_size = stack_top;
1340 }
1341 } else {
1342 call_level--;
1343 }
1344 level++;
1345 jit_extension =
1346 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1347 ssa = &jit_extension->func_info.ssa;
1348 if (ssa->cfg.blocks_count) {
1349 /* pass */
1350 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1351 /* Too many functions in single trace */
1352 *num_op_arrays_ptr = num_op_arrays;
1353 return NULL;
1354 } else {
1355 /* Remember op_array to cleanup */
1356 op_arrays[num_op_arrays++] = op_array;
1357 /* Build SSA */
1358 ssa = zend_jit_trace_build_ssa(op_array, script);
1359 }
1360 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1361 if (level == 0) {
1362 stack_bottom += zend_jit_trace_frame_size(p->op_array);
1363 jit_extension =
1364 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1365 ssa = &jit_extension->func_info.ssa;
1366 if (ssa->cfg.blocks_count) {
1367 /* pass */
1368 } else if (num_op_arrays == ZEND_JIT_TRACE_MAX_FUNCS) {
1369 /* Too many functions in single trace */
1370 *num_op_arrays_ptr = num_op_arrays;
1371 return NULL;
1372 } else {
1373 /* Remember op_array to cleanup */
1374 op_arrays[num_op_arrays++] = op_array;
1375 /* Build SSA */
1376 ssa = zend_jit_trace_build_ssa(op_array, script);
1377 }
1378 } else {
1379 stack_top -= zend_jit_trace_frame_size(op_array);
1380 level--;
1381 }
1382 op_array = p->op_array;
1383 } else if (p->op == ZEND_JIT_TRACE_END) {
1384 break;
1385 }
1386 }
1387 *num_op_arrays_ptr = num_op_arrays;
1388
1389 /* Allocate space for abstract stack */
1390 JIT_G(current_frame) = frame = (zend_jit_trace_stack_frame*)((char*)zend_arena_alloc(&CG(arena), stack_bottom + stack_size) + stack_bottom);
1391
1392 /* 2. Construct TSSA */
1393 tssa = zend_arena_calloc(&CG(arena), 1, sizeof(zend_tssa));
1394 tssa->cfg.flags = ZEND_SSA_TSSA;
1395 tssa->cfg.blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_basic_block));
1396 tssa->blocks = zend_arena_calloc(&CG(arena), 2, sizeof(zend_ssa_block));
1397 tssa->cfg.predecessors = zend_arena_calloc(&CG(arena), 2, sizeof(int));
1398
1399 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1400 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1401 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1402 tssa->cfg.blocks_count = 2;
1403 tssa->cfg.edges_count = 2;
1404
1405 tssa->cfg.predecessors[0] = 0;
1406 tssa->cfg.predecessors[1] = 1;
1407
1408 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_REACHABLE;
1409 tssa->cfg.blocks[0].successors_count = 1;
1410 tssa->cfg.blocks[0].predecessors_count = 0;
1411 tssa->cfg.blocks[0].successors = tssa->cfg.blocks[0].successors_storage;
1412 tssa->cfg.blocks[0].successors[0] = 1;
1413
1414 tssa->cfg.blocks[1].flags = ZEND_BB_FOLLOW|ZEND_BB_TARGET|ZEND_BB_LOOP_HEADER|ZEND_BB_REACHABLE;
1415 tssa->cfg.blocks[1].len = ssa_ops_count;
1416 tssa->cfg.blocks[1].successors_count = 1;
1417 tssa->cfg.blocks[1].predecessors_count = 2;
1418 tssa->cfg.blocks[1].successors = tssa->cfg.blocks[1].successors_storage;
1419 tssa->cfg.blocks[1].successors[1] = 1;
1420 } else {
1421 tssa->cfg.blocks_count = 1;
1422 tssa->cfg.edges_count = 0;
1423
1424 tssa->cfg.blocks[0].flags = ZEND_BB_START|ZEND_BB_EXIT|ZEND_BB_REACHABLE;
1425 tssa->cfg.blocks[0].len = ssa_ops_count;
1426 tssa->cfg.blocks[0].successors_count = 0;
1427 tssa->cfg.blocks[0].predecessors_count = 0;
1428 }
1429 ((zend_tssa*)tssa)->used_stack = -1;
1430
1431 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1432 return tssa;
1433 }
1434
1435 tssa->ops = ssa_ops = zend_arena_alloc(&CG(arena), ssa_ops_count * sizeof(zend_ssa_op));
1436 memset(ssa_ops, -1, ssa_ops_count * sizeof(zend_ssa_op));
1437 ssa_opcodes = zend_arena_calloc(&CG(arena), ssa_ops_count + 1, sizeof(zend_op*));
1438 ((zend_tssa*)tssa)->tssa_opcodes = ssa_opcodes;
1439 ssa_opcodes[ssa_ops_count] = &_nop_opcode;
1440
1441 op_array = trace_buffer->op_array;
1442 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1443 ssa_vars_count = op_array->last_var;
1444 } else {
1445 ssa_vars_count = op_array->last_var + op_array->T;
1446 }
1447 stack = frame->stack;
1448 for (i = 0; i < ssa_vars_count; i++) {
1449 SET_STACK_VAR(stack, i, i);
1450 }
1451
1452 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1453 // TODO: For tracing, it's possible, to create pseudo Phi functions
1454 // at the end of loop, without this additional pass (like LuaJIT) ???
1455 ssa_vars_count = zend_jit_trace_add_phis(trace_buffer, ssa_vars_count, tssa, stack);
1456 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
1457 ssa_vars_count = zend_jit_trace_add_call_phis(trace_buffer, ssa_vars_count, tssa, stack);
1458 } else if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1459 ssa_vars_count = zend_jit_trace_add_ret_phis(trace_buffer, ssa_vars_count, tssa, stack);
1460 }
1461
1462 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1463 idx = 0;
1464 level = 0;
1465 for (;;p++) {
1466 if (p->op == ZEND_JIT_TRACE_VM) {
1467 opline = p->opline;
1468 ssa_opcodes[idx] = opline;
1469 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1470 idx++;
1471 len = zend_jit_trace_op_len(p->opline);
1472 while (len > 1) {
1473 opline++;
1474 ssa_opcodes[idx] = opline;
1475 if (opline->opcode != ZEND_OP_DATA) {
1476 ssa_vars_count = zend_ssa_rename_op(op_array, opline, idx, build_flags, ssa_vars_count, ssa_ops, (int*)stack);
1477 }
1478 idx++;
1479 len--;
1480 }
1481 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
1482 frame = zend_jit_trace_call_frame(frame, op_array);
1483 stack = frame->stack;
1484 op_array = p->op_array;
1485 level++;
1486 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1487 return NULL;
1488 }
1489 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1490 for (i = 0; i < op_array->last_var; i++) {
1491 SET_STACK_VAR(stack, i, ssa_vars_count++);
1492 }
1493 } else if (p->op == ZEND_JIT_TRACE_BACK) {
1494 op_array = p->op_array;
1495 frame = zend_jit_trace_ret_frame(frame, op_array);
1496 stack = frame->stack;
1497 if (level == 0) {
1498 if (ssa_vars_count >= ZEND_JIT_TRACE_MAX_SSA_VAR) {
1499 return NULL;
1500 }
1501 ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(p->info, ssa_vars_count);
1502 for (i = 0; i < op_array->last_var + op_array->T; i++) {
1503 SET_STACK_VAR(stack, i, ssa_vars_count++);
1504 }
1505 } else {
1506 level--;
1507 }
1508 } else if (p->op == ZEND_JIT_TRACE_END) {
1509 break;
1510 }
1511 }
1512
1513 op_array = trace_buffer->op_array;
1514 tssa->vars_count = ssa_vars_count;
1515 tssa->vars = ssa_vars = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var));
1516 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1517 vars_count = op_array->last_var;
1518 } else {
1519 vars_count = op_array->last_var + op_array->T;
1520 }
1521 i = 0;
1522 while (i < vars_count) {
1523 ssa_vars[i].var = i;
1524 ssa_vars[i].scc = -1;
1525 ssa_vars[i].definition = -1;
1526 ssa_vars[i].use_chain = -1;
1527 i++;
1528 }
1529 while (i < tssa->vars_count) {
1530 ssa_vars[i].var = -1;
1531 ssa_vars[i].scc = -1;
1532 ssa_vars[i].definition = -1;
1533 ssa_vars[i].use_chain = -1;
1534 i++;
1535 }
1536
1537 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1538 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1539 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1540 /* Update Phi sources */
1541 zend_ssa_phi *phi = tssa->blocks[1].phis;
1542
1543 while (phi) {
1544 phi->sources[1] = STACK_VAR(stack, phi->var);
1545 ssa_vars[phi->ssa_var].var = phi->var;
1546 ssa_vars[phi->ssa_var].definition_phi = phi;
1547 ssa_vars[phi->sources[0]].phi_use_chain = phi;
1548 ssa_vars[phi->sources[1]].phi_use_chain = phi;
1549 phi = phi->next;
1550 }
1551 }
1552
1553 /* 3. Compute use-def chains */
1554 idx = (ssa_ops_count - 1);
1555 op = ssa_ops + idx;
1556 while (idx >= 0) {
1557 opline = ssa_opcodes[idx];
1558 if (op->op1_use >= 0) {
1559 op->op1_use_chain = ssa_vars[op->op1_use].use_chain;
1560 ssa_vars[op->op1_use].use_chain = idx;
1561 }
1562 if (op->op2_use >= 0 && op->op2_use != op->op1_use) {
1563 op->op2_use_chain = ssa_vars[op->op2_use].use_chain;
1564 ssa_vars[op->op2_use].use_chain = idx;
1565 }
1566 if (op->result_use >= 0 && op->result_use != op->op1_use && op->result_use != op->op2_use) {
1567 op->res_use_chain = ssa_vars[op->result_use].use_chain;
1568 ssa_vars[op->result_use].use_chain = idx;
1569 }
1570 if (op->op1_def >= 0) {
1571 ssa_vars[op->op1_def].var = EX_VAR_TO_NUM(opline->op1.var);
1572 ssa_vars[op->op1_def].definition = idx;
1573 }
1574 if (op->op2_def >= 0) {
1575 ssa_vars[op->op2_def].var = EX_VAR_TO_NUM(opline->op2.var);
1576 ssa_vars[op->op2_def].definition = idx;
1577 }
1578 if (op->result_def >= 0) {
1579 ssa_vars[op->result_def].var = EX_VAR_TO_NUM(opline->result.var);
1580 ssa_vars[op->result_def].definition = idx;
1581 }
1582 op--;
1583 idx--;
1584 }
1585
1586 /* 4. Type inference */
1587 op_array = trace_buffer->op_array;
1588 jit_extension =
1589 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
1590 ssa = &jit_extension->func_info.ssa;
1591
1592 tssa->var_info = ssa_var_info = zend_arena_calloc(&CG(arena), tssa->vars_count, sizeof(zend_ssa_var_info));
1593
1594 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
1595 i = 0;
1596 while (i < op_array->last_var) {
1597 if (i < op_array->num_args) {
1598 if (ssa->var_info
1599 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1600 /* pass */
1601 } else {
1602 if (ssa->vars) {
1603 ssa_vars[i].no_val = ssa->vars[i].no_val;
1604 ssa_vars[i].alias = ssa->vars[i].alias;
1605 } else {
1606 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1607 }
1608 if (op_array->arg_info && i < trace_buffer[1].opline - op_array->opcodes) {
1609 zend_arg_info *arg_info = &op_array->arg_info[i];
1610 zend_class_entry *ce;
1611 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
1612
1613 if (ZEND_ARG_SEND_MODE(arg_info)) {
1614 tmp |= MAY_BE_REF;
1615 }
1616 ssa_var_info[i].type = tmp;
1617 ssa_var_info[i].ce = ce;
1618 ssa_var_info[i].is_instanceof = 1;
1619 } else {
1620 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1621 }
1622 }
1623 } else {
1624 if (ssa->vars) {
1625 ssa_vars[i].no_val = ssa->vars[i].no_val;
1626 ssa_vars[i].alias = ssa->vars[i].alias;
1627 } else {
1628 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1629 }
1630 if (ssa_vars[i].alias == NO_ALIAS) {
1631 ssa_var_info[i].type = MAY_BE_UNDEF;
1632 } else {
1633 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1634 }
1635 }
1636 i++;
1637 }
1638 } else {
1639 int parent_vars_count = 0;
1640 zend_jit_trace_stack *parent_stack = NULL;
1641
1642 i = 0;
1643 if (parent_trace) {
1644 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
1645 op_array->last_var + op_array->T);
1646 if (parent_vars_count) {
1647 parent_stack =
1648 zend_jit_traces[parent_trace].stack_map +
1649 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
1650 }
1651 }
1652 while (i < op_array->last_var + op_array->T) {
1653 if (!ssa->var_info
1654 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, i)) {
1655 if (ssa->vars && i < ssa->vars_count) {
1656 ssa_vars[i].alias = ssa->vars[i].alias;
1657 } else {
1658 ssa_vars[i].alias = zend_jit_var_may_alias(op_array, ssa, i);
1659 }
1660 if (i < op_array->last_var) {
1661 ssa_var_info[i].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1662 } else {
1663 ssa_var_info[i].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
1664 }
1665 }
1666 if (i < parent_vars_count) {
1667 /* Initialize TSSA variable from parent trace */
1668 uint8_t op_type = STACK_TYPE(parent_stack, i);
1669
1670 if (op_type != IS_UNKNOWN) {
1671 ssa_var_info[i].type &= zend_jit_trace_type_to_info(op_type);
1672 if (!ssa_var_info[i].type
1673 && op_type == IS_UNDEF
1674 && i >= op_array->last_var) {
1675 // TODO: It's better to use NULL instead of UNDEF for temporary variables
1676 ssa_var_info[i].type |= MAY_BE_UNDEF;
1677 }
1678 }
1679 }
1680 i++;
1681 }
1682 }
1683
1684 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
1685 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
1686 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
1687 /* Propagate initial value through Phi functions */
1688 zend_ssa_phi *phi = tssa->blocks[1].phis;
1689
1690 while (phi) {
1691 if (!ssa->var_info
1692 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, phi->ssa_var)) {
1693 ssa_vars[phi->ssa_var].alias = ssa_vars[phi->sources[0]].alias;
1694 ssa_var_info[phi->ssa_var].type = ssa_var_info[phi->sources[0]].type;
1695 }
1696 phi = phi->next;
1697 }
1698 }
1699
1700 frame = JIT_G(current_frame);
1701 top = zend_jit_trace_call_frame(frame, op_array);
1702 TRACE_FRAME_INIT(frame, op_array, 0, 0);
1703 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
1704 frame->used_stack = 0;
1705 memset(&return_value_info, 0, sizeof(return_value_info));
1706
1707 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
1708 max_used_stack = used_stack = 0;
1709 } else {
1710 max_used_stack = used_stack = -1;
1711 }
1712
1713 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
1714 idx = 0;
1715 level = 0;
1716 opline = NULL;
1717 for (;;p++) {
1718 if (p->op == ZEND_JIT_TRACE_VM) {
1719 uint8_t orig_op1_type, orig_op2_type, op1_type, op2_type, op3_type;
1720 uint8_t val_type = IS_UNKNOWN;
1721 // zend_class_entry *op1_ce = NULL;
1722 zend_class_entry *op2_ce = NULL;
1723
1724 opline = p->opline;
1725
1726 op1_type = orig_op1_type = p->op1_type;
1727 op2_type = orig_op2_type = p->op2_type;
1728 op3_type = p->op3_type;
1729 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1730 op1_type = IS_UNKNOWN;
1731 }
1732 if (op1_type != IS_UNKNOWN) {
1733 op1_type &= ~IS_TRACE_PACKED;
1734 }
1735 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1736 op2_type = IS_UNKNOWN;
1737 }
1738 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
1739 op3_type = IS_UNKNOWN;
1740 }
1741
1742 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
1743 // op1_ce = (zend_class_entry*)(p+1)->ce;
1744 p++;
1745 }
1746 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
1747 op2_ce = (zend_class_entry*)(p+1)->ce;
1748 p++;
1749 }
1750 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
1751 val_type = (p+1)->op1_type;
1752 p++;
1753 }
1754
1755 switch (opline->opcode) {
1756 case ZEND_ASSIGN_OP:
1757 if (opline->extended_value == ZEND_POW
1758 || opline->extended_value == ZEND_DIV) {
1759 // TODO: check for division by zero ???
1760 break;
1761 }
1762 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1763 break;
1764 }
1765 ADD_OP1_TRACE_GUARD();
1766 ADD_OP2_TRACE_GUARD();
1767 break;
1768 case ZEND_ASSIGN_DIM_OP:
1769 if (opline->extended_value == ZEND_POW
1770 || opline->extended_value == ZEND_DIV) {
1771 // TODO: check for division by zero ???
1772 break;
1773 }
1774 if (opline->result_type != IS_UNUSED) {
1775 break;
1776 }
1777 if (op3_type != IS_UNKNOWN
1778 && !zend_jit_supported_binary_op(
1779 opline->extended_value, MAY_BE_ANY, (1<<op3_type))) {
1780 break;
1781 }
1782 ZEND_FALLTHROUGH;
1783 case ZEND_ASSIGN_DIM:
1784 if (opline->op1_type == IS_CV) {
1785 if ((opline+1)->op1_type == IS_CV
1786 && (opline+1)->op1.var == opline->op1.var) {
1787 /* skip $a[x] = $a; */
1788 break;
1789 }
1790 ADD_OP1_DATA_TRACE_GUARD();
1791 ADD_OP2_TRACE_GUARD();
1792 ADD_OP1_TRACE_GUARD();
1793 } else if (orig_op1_type != IS_UNKNOWN
1794 && (orig_op1_type & IS_TRACE_INDIRECT)
1795 && opline->result_type == IS_UNUSED) {
1796 if (opline->opcode == ZEND_ASSIGN_DIM_OP) {
1797 ADD_OP1_DATA_TRACE_GUARD();
1798 }
1799 ADD_OP2_TRACE_GUARD();
1800 }
1801 if (op1_type == IS_ARRAY
1802 && ((opline->op2_type == IS_CONST
1803 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1804 || (opline->op2_type != IS_CONST
1805 && op2_type == IS_LONG))) {
1806
1807 if (!(orig_op1_type & IS_TRACE_PACKED)) {
1808 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1809
1810 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1811 info->type |= MAY_BE_PACKED_GUARD;
1812 info->type &= ~MAY_BE_ARRAY_PACKED;
1813 }
1814 } else if (opline->opcode == ZEND_ASSIGN_DIM_OP
1815 && val_type != IS_UNKNOWN
1816 && val_type != IS_UNDEF) {
1817 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1818
1819 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1820 info->type |= MAY_BE_PACKED_GUARD;
1821 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1822 }
1823 }
1824 }
1825 break;
1826 case ZEND_ASSIGN_OBJ_OP:
1827 if (opline->extended_value == ZEND_POW
1828 || opline->extended_value == ZEND_DIV) {
1829 // TODO: check for division by zero ???
1830 break;
1831 }
1832 if (opline->result_type != IS_UNUSED) {
1833 break;
1834 }
1835 ZEND_FALLTHROUGH;
1836 case ZEND_ASSIGN_OBJ:
1837 case ZEND_PRE_INC_OBJ:
1838 case ZEND_PRE_DEC_OBJ:
1839 case ZEND_POST_INC_OBJ:
1840 case ZEND_POST_DEC_OBJ:
1841 if (opline->op2_type != IS_CONST
1842 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1843 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1844 break;
1845 }
1846 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
1847 if (opline->op1_type == IS_CV
1848 && (opline+1)->op1_type == IS_CV
1849 && (opline+1)->op1.var == opline->op1.var) {
1850 /* skip $a->prop += $a; */
1851 break;
1852 }
1853 ADD_OP1_DATA_TRACE_GUARD();
1854 }
1855 ADD_OP1_TRACE_GUARD();
1856 break;
1857 case ZEND_CONCAT:
1858 case ZEND_FAST_CONCAT:
1859 if ((opline->op1_type == IS_CONST || orig_op1_type == IS_STRING)
1860 && (opline->op2_type == IS_CONST || orig_op2_type == IS_STRING)) {
1861 ADD_OP2_TRACE_GUARD();
1862 ADD_OP1_TRACE_GUARD();
1863 }
1864 break;
1865 case ZEND_ADD:
1866 case ZEND_SUB:
1867 case ZEND_MUL:
1868 // case ZEND_DIV: // TODO: check for division by zero ???
1869 if (orig_op1_type == IS_UNDEF || orig_op2_type == IS_UNDEF) {
1870 break;
1871 }
1872 ZEND_FALLTHROUGH;
1873 case ZEND_IS_EQUAL:
1874 case ZEND_IS_NOT_EQUAL:
1875 case ZEND_IS_SMALLER:
1876 case ZEND_IS_SMALLER_OR_EQUAL:
1877 case ZEND_CASE:
1878 case ZEND_IS_IDENTICAL:
1879 case ZEND_IS_NOT_IDENTICAL:
1880 case ZEND_CASE_STRICT:
1881 case ZEND_BW_OR:
1882 case ZEND_BW_AND:
1883 case ZEND_BW_XOR:
1884 case ZEND_SL:
1885 case ZEND_SR:
1886 case ZEND_MOD:
1887 ADD_OP2_TRACE_GUARD();
1888 ZEND_FALLTHROUGH;
1889 case ZEND_ECHO:
1890 case ZEND_STRLEN:
1891 case ZEND_COUNT:
1892 case ZEND_QM_ASSIGN:
1893 case ZEND_FE_RESET_R:
1894 ADD_OP1_TRACE_GUARD();
1895 break;
1896 case ZEND_FE_FETCH_R:
1897 ADD_OP1_TRACE_GUARD();
1898 if (op1_type == IS_ARRAY && (orig_op1_type & ~IS_TRACE_PACKED) == IS_ARRAY) {
1899
1900 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
1901
1902 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
1903 info->type |= MAY_BE_PACKED_GUARD;
1904 if (orig_op1_type & IS_TRACE_PACKED) {
1905 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
1906 } else {
1907 info->type &= ~MAY_BE_ARRAY_PACKED;
1908 }
1909 }
1910 }
1911 break;
1912 case ZEND_VERIFY_RETURN_TYPE:
1913 if (opline->op1_type == IS_UNUSED) {
1914 /* Always throws */
1915 break;
1916 }
1917 if (opline->op1_type == IS_CONST) {
1918 /* TODO Different instruction format, has return value */
1919 break;
1920 }
1921 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
1922 /* Not worth bothering with */
1923 break;
1924 }
1925 ADD_OP1_TRACE_GUARD();
1926 break;
1927 case ZEND_FETCH_DIM_FUNC_ARG:
1928 if (!frame
1929 || !frame->call
1930 || !frame->call->func
1931 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
1932 break;
1933 }
1934 ADD_OP2_TRACE_GUARD();
1935 ADD_OP1_TRACE_GUARD();
1936 break;
1937 case ZEND_PRE_INC:
1938 case ZEND_PRE_DEC:
1939 case ZEND_POST_INC:
1940 case ZEND_POST_DEC:
1941 if (opline->op1_type != IS_CV) {
1942 break;
1943 }
1944 ADD_OP1_TRACE_GUARD();
1945 break;
1946 case ZEND_ASSIGN:
1947 if (opline->op1_type != IS_CV) {
1948 break;
1949 }
1950 ADD_OP2_TRACE_GUARD();
1951 if (op1_type != IS_UNKNOWN
1952 && (tssa->var_info[tssa->ops[idx].op1_use].type & MAY_BE_REF)) {
1953 ADD_OP1_TRACE_GUARD();
1954 }
1955 break;
1956 case ZEND_CAST:
1957 if (opline->extended_value != op1_type) {
1958 break;
1959 }
1960 ADD_OP1_TRACE_GUARD();
1961 break;
1962 case ZEND_JMPZ:
1963 case ZEND_JMPNZ:
1964 case ZEND_JMPZ_EX:
1965 case ZEND_JMPNZ_EX:
1966 case ZEND_BOOL:
1967 case ZEND_BOOL_NOT:
1968 ADD_OP1_TRACE_GUARD();
1969 break;
1970 case ZEND_ISSET_ISEMPTY_CV:
1971 if ((opline->extended_value & ZEND_ISEMPTY)) {
1972 // TODO: support for empty() ???
1973 break;
1974 }
1975 ADD_OP1_TRACE_GUARD();
1976 break;
1977 case ZEND_IN_ARRAY:
1978 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
1979 break;
1980 }
1981 ADD_OP1_TRACE_GUARD();
1982 break;
1983 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
1984 if ((opline->extended_value & ZEND_ISEMPTY)) {
1985 // TODO: support for empty() ???
1986 break;
1987 }
1988 ZEND_FALLTHROUGH;
1989 case ZEND_FETCH_DIM_R:
1990 case ZEND_FETCH_DIM_IS:
1991 case ZEND_FETCH_LIST_R:
1992 ADD_OP1_TRACE_GUARD();
1993 ADD_OP2_TRACE_GUARD();
1994
1995 if (op1_type == IS_ARRAY
1996 && opline->op1_type != IS_CONST
1997 && ((opline->op2_type == IS_CONST
1998 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
1999 || (opline->op2_type != IS_CONST
2000 && op2_type == IS_LONG))) {
2001
2002 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
2003
2004 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
2005 info->type |= MAY_BE_PACKED_GUARD;
2006 if (orig_op1_type & IS_TRACE_PACKED) {
2007 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
2008 } else {
2009 info->type &= ~MAY_BE_ARRAY_PACKED;
2010 }
2011 }
2012 }
2013 break;
2014 case ZEND_FETCH_DIM_W:
2015 case ZEND_FETCH_DIM_RW:
2016 // case ZEND_FETCH_DIM_UNSET:
2017 case ZEND_FETCH_LIST_W:
2018 if (opline->op1_type != IS_CV
2019 && (orig_op1_type == IS_UNKNOWN
2020 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
2021 break;
2022 }
2023 ADD_OP1_TRACE_GUARD();
2024 ADD_OP2_TRACE_GUARD();
2025 if (op1_type == IS_ARRAY
2026 && !(orig_op1_type & IS_TRACE_PACKED)
2027 && ((opline->op2_type == IS_CONST
2028 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2029 || (opline->op2_type != IS_CONST
2030 && op2_type == IS_LONG))) {
2031
2032 zend_ssa_var_info *info = &tssa->var_info[tssa->ops[idx].op1_use];
2033
2034 if (MAY_BE_PACKED(info->type) && MAY_BE_HASH(info->type)) {
2035 info->type |= MAY_BE_PACKED_GUARD;
2036 info->type &= ~MAY_BE_ARRAY_PACKED;
2037 }
2038 }
2039 break;
2040 case ZEND_SEND_VAL_EX:
2041 case ZEND_SEND_VAR_EX:
2042 case ZEND_SEND_VAR_NO_REF_EX:
2043 if (opline->op2_type == IS_CONST) {
2044 /* Named parameters not supported in JIT */
2045 break;
2046 }
2047 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
2048 goto propagate_arg;
2049 }
2050 ZEND_FALLTHROUGH;
2051 case ZEND_SEND_VAL:
2052 case ZEND_SEND_VAR:
2053 case ZEND_SEND_VAR_NO_REF:
2054 case ZEND_SEND_FUNC_ARG:
2055 if (opline->op2_type == IS_CONST) {
2056 /* Named parameters not supported in JIT */
2057 break;
2058 }
2059 ADD_OP1_TRACE_GUARD();
2060 propagate_arg:
2061 /* Propagate argument type */
2062 if (frame->call
2063 && frame->call->func
2064 && frame->call->func->type == ZEND_USER_FUNCTION
2065 && opline->op2.num <= frame->call->func->op_array.num_args) {
2066 uint32_t info;
2067
2068 if (opline->op1_type == IS_CONST) {
2069 info = _const_op_type(RT_CONSTANT(opline, opline->op1));
2070 } else {
2071 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2072 info = ssa_var_info[ssa_ops[idx].op1_use].type & ~MAY_BE_GUARD;
2073 }
2074 if (frame->call->func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
2075 zend_arg_info *arg_info;
2076
2077 ZEND_ASSERT(frame->call->func->op_array.arg_info);
2078 arg_info = &frame->call->func->op_array.arg_info[opline->op2.num - 1];
2079 if (ZEND_TYPE_IS_SET(arg_info->type)) {
2080 zend_class_entry *ce;
2081 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2082 info &= tmp;
2083 if (!info) {
2084 break;
2085 }
2086 }
2087 }
2088 if (opline->op1_type == IS_CV && (info & MAY_BE_RC1)) {
2089 info |= MAY_BE_RCN;
2090 }
2091 if (info & MAY_BE_UNDEF) {
2092 info |= MAY_BE_NULL;
2093 info &= ~MAY_BE_UNDEF;
2094 }
2095 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2096 info |= MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_ANY|MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_KEY_ANY;
2097 }
2098 SET_STACK_INFO(frame->call->stack, opline->op2.num - 1, info);
2099 }
2100 break;
2101 case ZEND_RETURN:
2102 ADD_OP1_TRACE_GUARD();
2103 /* Propagate return value types */
2104 if (opline->op1_type == IS_UNUSED) {
2105 return_value_info.type = MAY_BE_NULL;
2106 } else if (opline->op1_type == IS_CONST) {
2107 return_value_info.type = _const_op_type(RT_CONSTANT(opline, opline->op1));
2108 } else {
2109 ZEND_ASSERT(ssa_ops[idx].op1_use >= 0);
2110 return_value_info = ssa_var_info[ssa_ops[idx].op1_use];
2111 if (return_value_info.type & MAY_BE_UNDEF) {
2112 return_value_info.type &= ~MAY_BE_UNDEF;
2113 return_value_info.type |= MAY_BE_NULL;
2114 }
2115 if (return_value_info.type & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
2116 /* CVs are going to be destructed and the reference-counter
2117 of return value may be decremented to 1 */
2118 return_value_info.type |= MAY_BE_RC1;
2119 }
2120 return_value_info.type &= ~MAY_BE_GUARD;
2121 }
2122 break;
2123 case ZEND_CHECK_FUNC_ARG:
2124 if (!frame
2125 || !frame->call
2126 || !frame->call->func) {
2127 break;
2128 }
2129 if (opline->op2_type == IS_CONST
2130 || opline->op2.num > MAX_ARG_FLAG_NUM) {
2131 /* Named parameters not supported in JIT */
2132 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame->call);
2133 break;
2134 }
2135 if (ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2136 TRACE_FRAME_SET_LAST_SEND_BY_REF(frame->call);
2137 } else {
2138 TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame->call);
2139 }
2140 break;
2141 case ZEND_FETCH_OBJ_FUNC_ARG:
2142 if (!frame
2143 || !frame->call
2144 || !frame->call->func
2145 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame->call)) {
2146 break;
2147 }
2148 ZEND_FALLTHROUGH;
2149 case ZEND_FETCH_OBJ_R:
2150 case ZEND_FETCH_OBJ_IS:
2151 case ZEND_FETCH_OBJ_W:
2152 if (opline->op2_type != IS_CONST
2153 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2154 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2155 break;
2156 }
2157 if (opline->op1_type != IS_UNUSED && op1_type == IS_OBJECT) {
2158 ADD_OP1_TRACE_GUARD();
2159 }
2160 break;
2161 case ZEND_INIT_METHOD_CALL:
2162 if (opline->op2_type != IS_CONST
2163 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2164 break;
2165 }
2166 ADD_OP1_TRACE_GUARD();
2167 break;
2168 case ZEND_INIT_DYNAMIC_CALL:
2169 if (orig_op2_type == IS_OBJECT && op2_ce == zend_ce_closure) {
2170 ADD_OP2_TRACE_GUARD();
2171 }
2172 break;
2173 case ZEND_SEND_ARRAY:
2174 case ZEND_SEND_UNPACK:
2175 case ZEND_CHECK_UNDEF_ARGS:
2176 case ZEND_INCLUDE_OR_EVAL:
2177 max_used_stack = used_stack = -1;
2178 break;
2179 case ZEND_TYPE_CHECK:
2180 if (opline->extended_value == MAY_BE_RESOURCE) {
2181 // TODO: support for is_resource() ???
2182 break;
2183 }
2184 if (op1_type != IS_UNKNOWN
2185 && (opline->extended_value == (1 << op1_type)
2186 || opline->extended_value == MAY_BE_ANY - (1 << op1_type))) {
2187 /* add guards only for exact checks, to avoid code duplication */
2188 ADD_OP1_TRACE_GUARD();
2189 }
2190 break;
2191 case ZEND_ROPE_INIT:
2192 case ZEND_ROPE_ADD:
2193 case ZEND_ROPE_END:
2194 if (op2_type == IS_STRING) {
2195 ADD_OP2_TRACE_GUARD();
2196 }
2197 break;
2198 default:
2199 break;
2200 }
2201 len = zend_jit_trace_op_len(opline);
2202 if (ssa->var_info) {
2203 /* Add statically inferred ranges */
2204 if (ssa_ops[idx].op1_def >= 0) {
2205 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2206 }
2207 if (ssa_ops[idx].op2_def >= 0) {
2208 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2209 }
2210 if (ssa_ops[idx].result_def >= 0) {
2211 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2212 }
2213 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2214 if (ssa_ops[idx+1].op1_def >= 0) {
2215 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2216 }
2217 if (ssa_ops[idx+1].op2_def >= 0) {
2218 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2219 }
2220 if (ssa_ops[idx+1].result_def >= 0) {
2221 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2222 }
2223 }
2224 } else {
2225 if (ssa_ops[idx].op1_def >= 0) {
2226 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2227 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2228 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2229 }
2230 }
2231 if (ssa_ops[idx].op2_def >= 0) {
2232 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2233 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2234 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2235 }
2236 }
2237 if (ssa_ops[idx].result_def >= 0) {
2238 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2239 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2240 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2241 }
2242 }
2243 if (len == 2 && (opline+1)->opcode == ZEND_OP_DATA) {
2244 if (ssa_ops[idx+1].op1_def >= 0) {
2245 ssa_vars[ssa_ops[idx+1].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op1.var));
2246 if (ssa_ops[idx+1].op1_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op1_use].type & MAY_BE_REF)) {
2247 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op1_def);
2248 }
2249 }
2250 if (ssa_ops[idx+1].op2_def >= 0) {
2251 ssa_vars[ssa_ops[idx+1].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->op2.var));
2252 if (ssa_ops[idx+1].op2_use < 0 || !(ssa_var_info[ssa_ops[idx+1].op2_use].type & MAY_BE_REF)) {
2253 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].op2_def);
2254 }
2255 }
2256 if (ssa_ops[idx+1].result_def >= 0) {
2257 ssa_vars[ssa_ops[idx+1].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM((opline+1)->result.var));
2258 if (ssa_ops[idx+1].result_use < 0 || !(ssa_var_info[ssa_ops[idx+1].result_use].type & MAY_BE_REF)) {
2259 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx+1].result_def);
2260 }
2261 }
2262 }
2263 }
2264 if (opline->opcode == ZEND_RECV_INIT
2265 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2266 /* RECV_INIT always copy the constant */
2267 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2268 } else if ((opline->opcode == ZEND_FE_FETCH_R || opline->opcode == ZEND_FE_FETCH_RW)
2269 && ssa_opcodes[idx + 1] == ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2270 if (ssa_ops[idx].op2_use >= 0 && ssa_ops[idx].op2_def >= 0) {
2271 ssa_var_info[ssa_ops[idx].op2_def] = ssa_var_info[ssa_ops[idx].op2_use];
2272 }
2273 } else {
2274 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2275 // TODO:
2276 assert(0);
2277 }
2278 if (opline->opcode == ZEND_ASSIGN_DIM_OP
2279 && ssa_ops[idx].op1_def > 0
2280 && op1_type == IS_ARRAY
2281 && (orig_op1_type & IS_TRACE_PACKED)
2282 && val_type != IS_UNKNOWN
2283 && val_type != IS_UNDEF
2284 && ((opline->op2_type == IS_CONST
2285 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG)
2286 || (opline->op2_type != IS_CONST
2287 && op2_type == IS_LONG))) {
2288 zend_ssa_var_info *info = &ssa_var_info[ssa_ops[idx].op1_def];
2289
2290 info->type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
2291 }
2292 }
2293 if (ssa->var_info) {
2294 /* Add statically inferred restrictions */
2295 if (ssa_ops[idx].op1_def >= 0) {
2296 if (opline->opcode == ZEND_SEND_VAR_EX
2297 && frame
2298 && frame->call
2299 && frame->call->func
2300 && !ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
2301 ssa_var_info[ssa_ops[idx].op1_def] = ssa_var_info[ssa_ops[idx].op1_use];
2302 ssa_var_info[ssa_ops[idx].op1_def].type &= ~MAY_BE_GUARD;
2303 if (ssa_var_info[ssa_ops[idx].op1_def].type & MAY_BE_RC1) {
2304 ssa_var_info[ssa_ops[idx].op1_def].type |= MAY_BE_RCN;
2305 }
2306 } else {
2307 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2308 }
2309 }
2310 if (ssa_ops[idx].op2_def >= 0) {
2311 if ((opline->opcode != ZEND_FE_FETCH_R && opline->opcode != ZEND_FE_FETCH_RW)
2312 || ssa_opcodes[idx + 1] != ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value)) {
2313 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2314 }
2315 }
2316 if (ssa_ops[idx].result_def >= 0) {
2317 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2318 }
2319 }
2320 idx++;
2321 while (len > 1) {
2322 opline++;
2323 if (opline->opcode != ZEND_OP_DATA) {
2324 if (ssa->var_info) {
2325 /* Add statically inferred ranges */
2326 if (ssa_ops[idx].op1_def >= 0) {
2327 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2328 }
2329 if (ssa_ops[idx].op2_def >= 0) {
2330 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2331 }
2332 if (ssa_ops[idx].result_def >= 0) {
2333 zend_jit_trace_copy_ssa_var_range(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2334 }
2335 } else {
2336 if (ssa_ops[idx].op1_def >= 0) {
2337 ssa_vars[ssa_ops[idx].op1_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op1.var));
2338 if (ssa_ops[idx].op1_use < 0 || !(ssa_var_info[ssa_ops[idx].op1_use].type & MAY_BE_REF)) {
2339 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2340 }
2341 }
2342 if (ssa_ops[idx].op2_def >= 0) {
2343 ssa_vars[ssa_ops[idx].op2_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->op2.var));
2344 if (ssa_ops[idx].op2_use < 0 || !(ssa_var_info[ssa_ops[idx].op2_use].type & MAY_BE_REF)) {
2345 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2346 }
2347 }
2348 if (ssa_ops[idx].result_def >= 0) {
2349 ssa_vars[ssa_ops[idx].result_def].alias = zend_jit_var_may_alias(op_array, ssa, EX_VAR_TO_NUM(opline->result.var));
2350 if (ssa_ops[idx].result_use < 0 || !(ssa_var_info[ssa_ops[idx].result_use].type & MAY_BE_REF)) {
2351 zend_jit_trace_propagate_range(op_array, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2352 }
2353 }
2354 }
2355 if (opline->opcode == ZEND_RECV_INIT
2356 && !(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2357 /* RECV_INIT always copy the constant */
2358 ssa_var_info[ssa_ops[idx].result_def].type = _const_op_type(RT_CONSTANT(opline, opline->op2));
2359 } else {
2360 if (zend_update_type_info(op_array, tssa, script, (zend_op*)opline, ssa_ops + idx, ssa_opcodes, optimization_level) == FAILURE) {
2361 // TODO:
2362 assert(0);
2363 }
2364 }
2365 }
2366 if (ssa->var_info) {
2367 /* Add statically inferred restrictions */
2368 if (ssa_ops[idx].op1_def >= 0) {
2369 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op1_def);
2370 }
2371 if (ssa_ops[idx].op2_def >= 0) {
2372 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].op2_def);
2373 }
2374 if (ssa_ops[idx].result_def >= 0) {
2375 zend_jit_trace_restrict_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, ssa_ops[idx].result_def);
2376 }
2377 }
2378 idx++;
2379 len--;
2380 }
2381
2382 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
2383 op_array = p->op_array;
2384 jit_extension =
2385 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2386 ssa = &jit_extension->func_info.ssa;
2387
2388 call = frame->call;
2389 if (!call) {
2390 /* Trace missed INIT_FCALL opcode */
2391 call = top;
2392 TRACE_FRAME_INIT(call, op_array, 0, 0);
2393 call->used_stack = 0;
2394 top = zend_jit_trace_call_frame(top, op_array);
2395 } else {
2396 ZEND_ASSERT(&call->func->op_array == op_array);
2397 }
2398 frame->call = call->prev;
2399 call->prev = frame;
2400 TRACE_FRAME_SET_RETURN_SSA_VAR(call, find_return_ssa_var(p - 1, ssa_ops + (idx - 1)));
2401 frame = call;
2402
2403 level++;
2404 i = 0;
2405 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2406 while (i < op_array->last_var) {
2407 ssa_vars[v].var = i;
2408 if (i < op_array->num_args) {
2409 if (ssa->var_info
2410 && zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2411 /* pass */
2412 } else {
2413 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2414 if (op_array->arg_info) {
2415 zend_arg_info *arg_info = &op_array->arg_info[i];
2416 zend_class_entry *ce;
2417 uint32_t tmp = zend_fetch_arg_info_type(script, arg_info, &ce);
2418
2419 if (ZEND_ARG_SEND_MODE(arg_info)) {
2420 tmp |= MAY_BE_REF;
2421 }
2422 ssa_var_info[v].type = tmp;
2423 ssa_var_info[v].ce = ce;
2424 ssa_var_info[v].is_instanceof = 1;
2425 } else {
2426 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2427 }
2428 }
2429 } else {
2430 if (ssa->vars) {
2431 ssa_vars[v].no_val = ssa->vars[i].no_val;
2432 ssa_vars[v].alias = ssa->vars[i].alias;
2433 } else {
2434 ssa_vars[v].alias = zend_jit_var_may_alias(op_array, ssa, i);
2435 }
2436 if (ssa_vars[v].alias == NO_ALIAS) {
2437 ssa_var_info[v].type = MAY_BE_UNDEF;
2438 } else {
2439 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2440 }
2441 }
2442 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)
2443 && i < op_array->num_args) {
2444 /* Propagate argument type */
2445 ssa_var_info[v].type &= STACK_INFO(frame->stack, i);
2446 }
2447 i++;
2448 v++;
2449 }
2450 } else if (p->op == ZEND_JIT_TRACE_BACK) {
2451 op_array = p->op_array;
2452 jit_extension =
2453 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2454 ssa = &jit_extension->func_info.ssa;
2455 if (level == 0) {
2456 i = 0;
2457 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
2458 while (i < op_array->last_var) {
2459 ssa_vars[v].var = i;
2460 if (!ssa->var_info
2461 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2462 ssa_var_info[v].type = MAY_BE_UNDEF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2463 }
2464 i++;
2465 v++;
2466 }
2467 while (i < op_array->last_var + op_array->T) {
2468 ssa_vars[v].var = i;
2469 if (!ssa->var_info
2470 || !zend_jit_trace_copy_ssa_var_info(op_array, ssa, ssa_opcodes, tssa, v)) {
2471 ssa_var_info[v].type = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
2472 }
2473 i++;
2474 v++;
2475 }
2476 if (return_value_info.type != 0) {
2477 zend_jit_trace_rec *q = p + 1;
2478 while (q->op == ZEND_JIT_TRACE_INIT_CALL) {
2479 q++;
2480 }
2481 if (q->op == ZEND_JIT_TRACE_VM
2482 || (q->op == ZEND_JIT_TRACE_END
2483 && q->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)) {
2484 const zend_op *opline = q->opline - 1;
2485 if (opline->result_type != IS_UNUSED) {
2486 ssa_var_info[
2487 ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info) +
2488 EX_VAR_TO_NUM(opline->result.var)] = return_value_info;
2489 }
2490 }
2491 memset(&return_value_info, 0, sizeof(return_value_info));
2492 }
2493 } else {
2494 level--;
2495 if (return_value_info.type != 0) {
2496 if ((p+1)->op == ZEND_JIT_TRACE_VM) {
2497 const zend_op *opline = (p+1)->opline - 1;
2498 if (opline->result_type != IS_UNUSED) {
2499 if (TRACE_FRAME_RETURN_SSA_VAR(frame) >= 0) {
2500 ssa_var_info[TRACE_FRAME_RETURN_SSA_VAR(frame)] = return_value_info;
2501 }
2502 }
2503 }
2504 memset(&return_value_info, 0, sizeof(return_value_info));
2505 }
2506 }
2507
2508 top = frame;
2509 if (frame->prev) {
2510 if (used_stack > 0) {
2511 used_stack -= frame->used_stack;
2512 }
2513 frame = frame->prev;
2514 ZEND_ASSERT(&frame->func->op_array == op_array);
2515 } else {
2516 max_used_stack = used_stack = -1;
2517 frame = zend_jit_trace_ret_frame(frame, op_array);
2518 TRACE_FRAME_INIT(frame, op_array, 0, 0);
2519 TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1);
2520 frame->used_stack = 0;
2521 }
2522
2523 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
2524 call = top;
2525 TRACE_FRAME_INIT(call, p->func, 0, 0);
2526 call->prev = frame->call;
2527 call->used_stack = 0;
2528 frame->call = call;
2529 top = zend_jit_trace_call_frame(top, p->op_array);
2530 if (p->func && p->func->type == ZEND_USER_FUNCTION) {
2531 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
2532 SET_STACK_INFO(call->stack, i, -1);
2533 }
2534 }
2535 if (used_stack >= 0
2536 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
2537 if (p->func == NULL || (p-1)->op != ZEND_JIT_TRACE_VM) {
2538 max_used_stack = used_stack = -1;
2539 } else {
2540 const zend_op *opline = (p-1)->opline;
2541
2542 switch (opline->opcode) {
2543 case ZEND_INIT_FCALL:
2544 case ZEND_INIT_FCALL_BY_NAME:
2545 case ZEND_INIT_NS_FCALL_BY_NAME:
2546 case ZEND_INIT_METHOD_CALL:
2547 case ZEND_INIT_DYNAMIC_CALL:
2548 //case ZEND_INIT_STATIC_METHOD_CALL:
2549 //case ZEND_INIT_USER_CALL:
2550 //case ZEND_NEW:
2551 frame->used_stack = zend_vm_calc_used_stack(opline->extended_value, (zend_function*)p->func);
2552 used_stack += frame->used_stack;
2553 if (used_stack > max_used_stack) {
2554 max_used_stack = used_stack;
2555 }
2556 break;
2557 default:
2558 max_used_stack = used_stack = -1;
2559 }
2560 }
2561 }
2562 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
2563 call = frame->call;
2564 if (call) {
2565 top = call;
2566 frame->call = call->prev;
2567 }
2568
2569 if (idx > 0
2570 && ssa_ops[idx-1].result_def >= 0
2571 && (p->func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)
2572 && !(p->func->common.fn_flags & ZEND_ACC_RETURN_REFERENCE)) {
2573 ZEND_ASSERT(ssa_opcodes[idx-1] == opline);
2574 ZEND_ASSERT(opline->opcode == ZEND_DO_ICALL ||
2575 opline->opcode == ZEND_DO_FCALL ||
2576 opline->opcode == ZEND_DO_FCALL_BY_NAME);
2577
2578 if (opline->result_type != IS_UNDEF) {
2579 zend_class_entry *ce;
2580 const zend_function *func = p->func;
2581 zend_arg_info *ret_info = func->common.arg_info - 1;
2582 uint32_t ret_type = zend_fetch_arg_info_type(NULL, ret_info, &ce);
2583
2584 ssa_var_info[ssa_ops[idx-1].result_def].type &= ret_type;
2585 }
2586 }
2587 } else if (p->op == ZEND_JIT_TRACE_END) {
2588 break;
2589 }
2590 }
2591
2592 ((zend_tssa*)tssa)->used_stack = max_used_stack;
2593
2594 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2595 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2596 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2597 /* Propagate guards through Phi sources */
2598 zend_ssa_phi *phi = tssa->blocks[1].phis;
2599
2600 op_array = trace_buffer->op_array;
2601 jit_extension =
2602 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2603 ssa = &jit_extension->func_info.ssa;
2604
2605 while (phi) {
2606 uint32_t t = ssa_var_info[phi->ssa_var].type;
2607
2608 if ((t & MAY_BE_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2609 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2610 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2611
2612 if (((t0 | t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2613 if (!((t0 | t1) & MAY_BE_GUARD)) {
2614 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2615 }
2616 } else if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2617 if (!(t1 & MAY_BE_GUARD)
2618 || is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2619 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2620 t0 = (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2621 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2622 MAY_BE_GUARD;
2623 if (!(t0 & MAY_BE_ARRAY)) {
2624 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2625 }
2626 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2627 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2628 }
2629 ssa_var_info[phi->sources[0]].type = t0;
2630 ssa_var_info[phi->sources[0]].type = t0;
2631 }
2632 } else {
2633 if ((t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2634 t0 = (t & t0 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2635 (t0 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2636 MAY_BE_GUARD;
2637 if (!(t0 & MAY_BE_ARRAY)) {
2638 t0 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2639 }
2640 if (!(t0 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2641 t0 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2642 }
2643 ssa_var_info[phi->sources[0]].type = t0;
2644 }
2645 if ((t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != (t & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF))) {
2646 if (((t & t1) & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != 0
2647 && is_checked_guard(tssa, ssa_opcodes, phi->sources[1], phi->ssa_var)) {
2648 t1 = (t & t1 & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2649 (t1 & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) |
2650 MAY_BE_GUARD;
2651 if (!(t1 & MAY_BE_ARRAY)) {
2652 t1 &= ~(MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY);
2653 }
2654 if (!(t1 & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
2655 t1 &= ~(MAY_BE_RC1|MAY_BE_RCN);
2656 }
2657 ssa_var_info[phi->sources[1]].type = t1;
2658 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_GUARD;
2659 }
2660 }
2661 }
2662 t = ssa_var_info[phi->ssa_var].type;
2663 }
2664
2665 if ((t & MAY_BE_PACKED_GUARD) && tssa->vars[phi->ssa_var].alias == NO_ALIAS) {
2666 uint32_t t0 = ssa_var_info[phi->sources[0]].type;
2667 uint32_t t1 = ssa_var_info[phi->sources[1]].type;
2668
2669 if (((t0 | t1) & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2670 if (!((t0 | t1) & MAY_BE_PACKED_GUARD)) {
2671 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2672 }
2673 } else if ((t1 & MAY_BE_ARRAY_KEY_ANY) == (t & MAY_BE_ARRAY_KEY_ANY)) {
2674 if (!(t1 & MAY_BE_PACKED_GUARD)) {
2675 ssa_var_info[phi->ssa_var].type = t & ~MAY_BE_PACKED_GUARD;
2676 ssa_var_info[phi->sources[0]].type =
2677 (t0 & ~MAY_BE_ARRAY_KEY_ANY) | (t & MAY_BE_ARRAY_KEY_ANY) | MAY_BE_PACKED_GUARD;
2678 }
2679 }
2680 }
2681 phi = phi->next;
2682 }
2683 }
2684
2685 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_TSSA)) {
2686 if (parent_trace) {
2687 fprintf(stderr, "---- TRACE %d TSSA start (side trace %d/%d) %s%s%s() %s:%d\n",
2688 ZEND_JIT_TRACE_NUM,
2689 parent_trace,
2690 exit_num,
2691 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2692 trace_buffer->op_array->scope ? "::" : "",
2693 trace_buffer->op_array->function_name ?
2694 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2695 ZSTR_VAL(trace_buffer->op_array->filename),
2696 trace_buffer[1].opline->lineno);
2697 } else {
2698 fprintf(stderr, "---- TRACE %d TSSA start (%s) %s%s%s() %s:%d\n",
2699 ZEND_JIT_TRACE_NUM,
2700 zend_jit_trace_star_desc(trace_buffer->start),
2701 trace_buffer->op_array->scope ? ZSTR_VAL(trace_buffer->op_array->scope->name) : "",
2702 trace_buffer->op_array->scope ? "::" : "",
2703 trace_buffer->op_array->function_name ?
2704 ZSTR_VAL(trace_buffer->op_array->function_name) : "$main",
2705 ZSTR_VAL(trace_buffer->op_array->filename),
2706 trace_buffer[1].opline->lineno);
2707 }
2708 zend_jit_dump_trace(trace_buffer, tssa);
2709 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LINK) {
2710 uint32_t idx = trace_buffer[1].last;
2711 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
2712 fprintf(stderr, "---- TRACE %d TSSA stop (link to %d)\n",
2713 ZEND_JIT_TRACE_NUM,
2714 link_to);
2715 } else {
2716 fprintf(stderr, "---- TRACE %d TSSA stop (%s)\n",
2717 ZEND_JIT_TRACE_NUM,
2718 zend_jit_trace_stop_description[trace_buffer->stop]);
2719 }
2720 }
2721
2722 return tssa;
2723 }
2724
2725 #define RA_HAS_IVAL(var) (ra[var].ref != 0)
2726 #define RA_IVAL_FLAGS(var) ra[var].flags
2727 #define RA_IVAL_START(var, line) do {ra[var].ref = IR_NULL;} while (0)
2728 #define RA_IVAL_END(var, line)
2729 #define RA_IVAL_CLOSE(var, line)
2730 #define RA_IVAL_DEL(var) do {ra[var].ref = IR_UNUSED;} while (0)
2731 #define RA_HAS_REG(var) (ra[var].ref != 0)
2732 #define RA_REG_FLAGS(var) ra[var].flags
2733 #define RA_REG_START(var, line) do {ra[var].ref = IR_NULL;} while (0)
2734 #define RA_REG_DEL(var) do {ra[var].ref = IR_UNUSED;} while (0)
2735
zend_jit_trace_use_var(int line,int var,int def,int use_chain,zend_jit_reg_var * ra,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array,const zend_ssa * op_array_ssa)2736 static void zend_jit_trace_use_var(int line, int var, int def, int use_chain, zend_jit_reg_var *ra, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array, const zend_ssa *op_array_ssa)
2737 {
2738 ZEND_ASSERT(RA_HAS_IVAL(var));
2739 ZEND_ASSERT(!(RA_IVAL_FLAGS(var) & ZREG_LAST_USE));
2740 RA_IVAL_END(var, line);
2741 if (def >= 0) {
2742 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2743 } else if (use_chain < 0 && (RA_IVAL_FLAGS(var) & (ZREG_LOAD|ZREG_STORE))) {
2744 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2745 } else if (use_chain >= 0 && !zend_ssa_is_no_val_use(ssa_opcodes[use_chain], ssa->ops + use_chain, var)) {
2746 /* pass */
2747 } else if (op_array_ssa->vars) {
2748 uint32_t use = ssa_opcodes[line] - op_array->opcodes;
2749
2750 if (ssa->ops[line].op1_use == var) {
2751 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op1_use, use)) {
2752 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2753 }
2754 } else if (ssa->ops[line].op2_use == var) {
2755 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].op2_use, use)) {
2756 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2757 }
2758 } else if (ssa->ops[line].result_use == var) {
2759 if (zend_ssa_is_last_use(op_array, op_array_ssa, op_array_ssa->ops[use].result_use, use)) {
2760 RA_IVAL_FLAGS(var) |= ZREG_LAST_USE;
2761 }
2762 }
2763 }
2764 }
2765
zend_jit_trace_allocate_registers(zend_jit_trace_rec * trace_buffer,zend_ssa * ssa,uint32_t parent_trace,uint32_t exit_num)2766 static zend_jit_reg_var* zend_jit_trace_allocate_registers(zend_jit_trace_rec *trace_buffer, zend_ssa *ssa, uint32_t parent_trace, uint32_t exit_num)
2767 {
2768 const zend_op **ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
2769 zend_jit_trace_rec *p;
2770 const zend_op_array *op_array;
2771 zend_jit_op_array_trace_extension *jit_extension;
2772 const zend_ssa *op_array_ssa;
2773 const zend_ssa_op *ssa_op;
2774 int i, j, idx, count, level;
2775 zend_jit_reg_var *ra;
2776 const zend_op_array **vars_op_array;
2777 void *checkpoint;
2778 zend_jit_trace_stack_frame *frame;
2779 zend_jit_trace_stack *stack;
2780 uint32_t parent_vars_count = parent_trace ?
2781 zend_jit_traces[parent_trace].exit_info[exit_num].stack_size : 0;
2782 zend_jit_trace_stack *parent_stack = parent_vars_count ?
2783 zend_jit_traces[parent_trace].stack_map +
2784 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset : NULL;
2785 checkpoint = zend_arena_checkpoint(CG(arena));
2786 ra = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_jit_reg_var));
2787 vars_op_array = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_op_array*));
2788 memset(ZEND_VOIDP(vars_op_array), 0, sizeof(zend_op_array*) * ssa->vars_count);
2789
2790 op_array = trace_buffer->op_array;
2791 jit_extension =
2792 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
2793 op_array_ssa = &jit_extension->func_info.ssa;
2794 frame = JIT_G(current_frame);
2795 frame->prev = NULL;
2796 frame->func = (const zend_function*)op_array;
2797 stack = frame->stack;
2798
2799 count = 0;
2800
2801 i = 0;
2802 j = op_array->last_var;
2803 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
2804 j += op_array->T;
2805 }
2806 while (i < j) {
2807 SET_STACK_VAR(stack, i, i);
2808 vars_op_array[i] = op_array;
2809 /* We don't start intervals for variables used in Phi */
2810 if ((ssa->vars[i].use_chain >= 0 /*|| ssa->vars[i].phi_use_chain*/)
2811 && !zend_ssa_is_no_val_use(ssa_opcodes[ssa->vars[i].use_chain], ssa->ops + ssa->vars[i].use_chain, i)
2812 && ssa->vars[i].alias == NO_ALIAS
2813 && zend_jit_var_supports_reg(ssa, i)) {
2814 RA_IVAL_START(i, 0);
2815 if (i < parent_vars_count
2816 && STACK_REG(parent_stack, i) != ZREG_NONE
2817 && STACK_FLAGS(parent_stack, i) != ZREG_ZVAL_COPY
2818 ) {
2819 /* We will try to reuse register from parent trace */
2820 RA_IVAL_FLAGS(i) = STACK_FLAGS(parent_stack, i);
2821 count += 2;
2822 } else {
2823 RA_IVAL_FLAGS(i) = ZREG_LOAD;
2824 count++;
2825 }
2826 }
2827 i++;
2828 }
2829
2830 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
2831 j = op_array->last_var + op_array->T;
2832 while (i < j) {
2833 SET_STACK_VAR(stack, i, -1);
2834 i++;
2835 }
2836 }
2837
2838 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
2839 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
2840 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
2841 zend_ssa_phi *phi = ssa->blocks[1].phis;
2842
2843 while (phi) {
2844 SET_STACK_VAR(stack, phi->var, phi->ssa_var);
2845 vars_op_array[phi->ssa_var] = op_array;
2846 if (ssa->vars[phi->ssa_var].use_chain >= 0
2847 && ssa->vars[phi->ssa_var].alias == NO_ALIAS
2848 && zend_jit_var_supports_reg(ssa, phi->ssa_var)) {
2849 RA_IVAL_START(phi->ssa_var, 0);
2850 count++;
2851 }
2852 phi = phi->next;
2853 }
2854 }
2855
2856 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
2857 level = 0;
2858 ssa_op = ssa->ops;
2859 idx = 0;
2860 for (;;p++) {
2861 if (p->op == ZEND_JIT_TRACE_VM) {
2862 const zend_op *opline = p->opline;
2863 int len;
2864 bool support_opline;
2865
2866 support_opline =
2867 zend_jit_opline_supports_reg(op_array, ssa, opline, ssa_op, p);
2868
2869 if (support_opline
2870 && opline->opcode == ZEND_ASSIGN
2871 && opline->op1_type == IS_CV
2872 && ssa_op->op1_def >= 0
2873 && ssa->vars[ssa_op->op1_def].alias != NO_ALIAS) {
2874 /* avoid register allocation in case of possibility of indirect modification*/
2875 support_opline = 0;
2876 }
2877
2878 if (ssa_op->op1_use >= 0
2879 && RA_HAS_IVAL(ssa_op->op1_use)) {
2880 if (!support_opline) {
2881 RA_IVAL_DEL(ssa_op->op1_use);
2882 count--;
2883 } else if (!zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
2884 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain,
2885 ra,
2886 ssa, ssa_opcodes, op_array, op_array_ssa);
2887 if (opline->op1_type != IS_CV) {
2888 if (opline->opcode == ZEND_CASE
2889 || opline->opcode == ZEND_CASE_STRICT
2890 || opline->opcode == ZEND_SWITCH_LONG
2891 || opline->opcode == ZEND_MATCH
2892 || opline->opcode == ZEND_FETCH_LIST_R
2893 || opline->opcode == ZEND_COPY_TMP
2894 || opline->opcode == ZEND_SWITCH_STRING
2895 || opline->opcode == ZEND_FE_FETCH_R
2896 || opline->opcode == ZEND_FE_FETCH_RW
2897 || opline->opcode == ZEND_FETCH_LIST_W
2898 || opline->opcode == ZEND_VERIFY_RETURN_TYPE
2899 || opline->opcode == ZEND_BIND_LEXICAL
2900 || opline->opcode == ZEND_ROPE_ADD) {
2901 /* The value is kept alive and may be used outside of the trace */
2902 RA_IVAL_FLAGS(ssa_op->op1_use) |= ZREG_STORE;
2903 } else {
2904 RA_IVAL_FLAGS(ssa_op->op1_use) |= ZREG_LAST_USE;
2905 }
2906 }
2907 }
2908 }
2909 if (ssa_op->op2_use >= 0
2910 && ssa_op->op2_use != ssa_op->op1_use
2911 && RA_HAS_IVAL(ssa_op->op2_use)) {
2912 /* Quick workaround to disable register allocation for unsupported operand */
2913 // TODO: Find a general solution ???
2914 if (!support_opline || opline->opcode == ZEND_FETCH_DIM_R) {
2915 RA_IVAL_DEL(ssa_op->op2_use);
2916 count--;
2917 } else if (!zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op2_use)) {
2918 zend_jit_trace_use_var(idx, ssa_op->op2_use, ssa_op->op2_def, ssa_op->op2_use_chain,
2919 ra,
2920 ssa, ssa_opcodes, op_array, op_array_ssa);
2921 if (opline->op2_type != IS_CV) {
2922 RA_IVAL_FLAGS(ssa_op->op2_use) |= ZREG_LAST_USE;
2923 }
2924 }
2925 }
2926 if (ssa_op->result_use >= 0
2927 && ssa_op->result_use != ssa_op->op1_use
2928 && ssa_op->result_use != ssa_op->op2_use
2929 && RA_HAS_IVAL(ssa_op->result_use)) {
2930 if (!support_opline) {
2931 RA_IVAL_DEL(ssa_op->result_use);
2932 count--;
2933 } else if (!zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->result_use)) {
2934 zend_jit_trace_use_var(idx, ssa_op->result_use, ssa_op->result_def, ssa_op->res_use_chain,
2935 ra,
2936 ssa, ssa_opcodes, op_array, op_array_ssa);
2937 }
2938 }
2939
2940 if (ssa_op->op1_def >= 0) {
2941 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op1.var), idx);
2942 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
2943 }
2944 if (ssa_op->op2_def >= 0) {
2945 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op2.var), idx);
2946 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op2.var), ssa_op->op2_def);
2947 }
2948 if (ssa_op->result_def >= 0) {
2949 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->result.var), idx);
2950 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
2951 }
2952
2953 if (support_opline) {
2954 if (ssa_op->result_def >= 0
2955 && (ssa->vars[ssa_op->result_def].use_chain >= 0
2956 || ssa->vars[ssa_op->result_def].phi_use_chain)
2957 && ssa->vars[ssa_op->result_def].alias == NO_ALIAS
2958 && zend_jit_var_supports_reg(ssa, ssa_op->result_def)) {
2959 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
2960 || opline->opcode == ZEND_PRE_INC
2961 || opline->opcode == ZEND_PRE_DEC
2962 || opline->opcode == ZEND_POST_INC
2963 || opline->opcode == ZEND_POST_DEC
2964 || opline->opcode == ZEND_ADD
2965 || opline->opcode == ZEND_SUB
2966 || opline->opcode == ZEND_MUL
2967 || opline->opcode == ZEND_FETCH_DIM_R
2968 || opline->opcode == ZEND_FETCH_CONSTANT) {
2969 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_DOUBLE)
2970 || (opline->opcode != ZEND_PRE_INC && opline->opcode != ZEND_PRE_DEC)) {
2971 vars_op_array[ssa_op->result_def] = op_array;
2972 RA_IVAL_START(ssa_op->result_def, idx);
2973 count++;
2974 }
2975 }
2976 }
2977 if (ssa_op->op1_def >= 0
2978 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
2979 || ssa->vars[ssa_op->op1_def].phi_use_chain)
2980 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
2981 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)
2982 && (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
2983 || opline->opcode == ZEND_PRE_INC
2984 || opline->opcode == ZEND_PRE_DEC
2985 || opline->opcode == ZEND_POST_INC
2986 || opline->opcode == ZEND_POST_DEC)) {
2987 vars_op_array[ssa_op->op1_def] = op_array;
2988 RA_IVAL_START(ssa_op->op1_def, idx);
2989 count++;
2990 }
2991 if (ssa_op->op2_def >= 0
2992 && (ssa->vars[ssa_op->op2_def].use_chain >= 0
2993 || ssa->vars[ssa_op->op2_def].phi_use_chain)
2994 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS
2995 && zend_jit_var_supports_reg(ssa, ssa_op->op2_def)
2996 && !(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)) {
2997 vars_op_array[ssa_op->op2_def] = op_array;
2998 RA_IVAL_START(ssa_op->op2_def, idx);
2999 count++;
3000 }
3001 }
3002
3003 len = zend_jit_trace_op_len(opline);
3004 switch (opline->opcode) {
3005 case ZEND_ASSIGN_DIM:
3006 case ZEND_ASSIGN_OBJ:
3007 case ZEND_ASSIGN_STATIC_PROP:
3008 case ZEND_ASSIGN_DIM_OP:
3009 case ZEND_ASSIGN_OBJ_OP:
3010 case ZEND_ASSIGN_STATIC_PROP_OP:
3011 case ZEND_ASSIGN_OBJ_REF:
3012 case ZEND_ASSIGN_STATIC_PROP_REF:
3013 case ZEND_FRAMELESS_ICALL_3:
3014 /* OP_DATA */
3015 ssa_op++;
3016 opline++;
3017 if (ssa_op->op1_use >= 0
3018 && RA_HAS_IVAL(ssa_op->op1_use)
3019 && !zend_ssa_is_no_val_use(opline, ssa_op, ssa_op->op1_use)) {
3020 if (support_opline) {
3021 zend_jit_trace_use_var(idx, ssa_op->op1_use, ssa_op->op1_def, ssa_op->op1_use_chain,
3022 ra,
3023 ssa, ssa_opcodes, op_array, op_array_ssa);
3024 if (opline->op1_type != IS_CV) {
3025 RA_IVAL_FLAGS(ssa_op->op1_use) |= ZREG_LAST_USE;
3026 }
3027 } else {
3028 RA_IVAL_DEL(ssa_op->op1_use);
3029 count--;
3030 }
3031 }
3032 if (ssa_op->op1_def >= 0) {
3033 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op1.var), idx);
3034 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3035 if (support_opline
3036 && (ssa->vars[ssa_op->op1_def].use_chain >= 0
3037 || ssa->vars[ssa_op->op1_def].phi_use_chain)
3038 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS
3039 && zend_jit_var_supports_reg(ssa, ssa_op->op1_def)) {
3040 vars_op_array[ssa_op->op1_def] = op_array;
3041 RA_IVAL_START(ssa_op->op1_def, idx);
3042 count++;
3043 }
3044 }
3045 ssa_op++;
3046 opline++;
3047 idx+=2;
3048 break;
3049 case ZEND_RECV_INIT:
3050 ssa_op++;
3051 opline++;
3052 idx++;
3053 while (opline->opcode == ZEND_RECV_INIT) {
3054 /* RECV_INIT doesn't support registers */
3055 if (ssa_op->result_use >= 0 && RA_HAS_IVAL(ssa_op->result_use)) {
3056 RA_IVAL_DEL(ssa_op->result_use);
3057 count--;
3058 }
3059 if (ssa_op->result_def >= 0) {
3060 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->result.var), idx);
3061 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->result.var), ssa_op->result_def);
3062 }
3063 ssa_op++;
3064 opline++;
3065 idx++;
3066 }
3067 break;
3068 case ZEND_BIND_GLOBAL:
3069 ssa_op++;
3070 opline++;
3071 idx++;
3072 while (opline->opcode == ZEND_BIND_GLOBAL) {
3073 /* BIND_GLOBAL doesn't support registers */
3074 if (ssa_op->op1_def >= 0) {
3075 RA_IVAL_CLOSE(EX_VAR_TO_NUM(opline->op1.var), idx);
3076 SET_STACK_VAR(stack, EX_VAR_TO_NUM(opline->op1.var), ssa_op->op1_def);
3077 }
3078 ssa_op++;
3079 opline++;
3080 idx++;
3081 }
3082 break;
3083 default:
3084 ssa_op += len;
3085 idx += len;
3086 break;
3087 }
3088 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
3089 /* New call frames */
3090 zend_jit_trace_stack_frame *prev_frame = frame;
3091
3092 /* Clear allocated registers */
3093 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3094 j = STACK_VAR(stack, i);
3095 if (j >= 0 && RA_HAS_IVAL(j) && !(RA_IVAL_FLAGS(j) & ZREG_LAST_USE)) {
3096 RA_IVAL_DEL(j);
3097 count--;
3098 }
3099 }
3100
3101 frame = zend_jit_trace_call_frame(frame, op_array);
3102 frame->prev = prev_frame;
3103 frame->func = (const zend_function*)p->op_array;
3104 stack = frame->stack;
3105 op_array = p->op_array;
3106 jit_extension =
3107 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3108 op_array_ssa = &jit_extension->func_info.ssa;
3109 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3110 for (i = 0; i < op_array->last_var; i++) {
3111 SET_STACK_VAR(stack, i, j);
3112 vars_op_array[j] = op_array;
3113 if (ssa->vars[j].use_chain >= 0
3114 && ssa->vars[j].alias == NO_ALIAS
3115 && zend_jit_var_supports_reg(ssa, j)) {
3116 RA_IVAL_START(j, idx);
3117 RA_IVAL_FLAGS(j) = ZREG_LOAD;
3118 count++;
3119 }
3120 j++;
3121 }
3122 for (i = op_array->last_var; i < op_array->last_var + op_array->T; i++) {
3123 SET_STACK_VAR(stack, i, -1);
3124 }
3125 level++;
3126 } else if (p->op == ZEND_JIT_TRACE_BACK) {
3127 /* Close exiting call frames */
3128 for (i = 0; i < op_array->last_var; i++) {
3129 RA_IVAL_CLOSE(i, idx-1);
3130 }
3131 op_array = p->op_array;
3132 jit_extension =
3133 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
3134 op_array_ssa = &jit_extension->func_info.ssa;
3135 frame = zend_jit_trace_ret_frame(frame, op_array);
3136 stack = frame->stack;
3137 if (level == 0) {
3138 /* New return frames */
3139 frame->prev = NULL;
3140 frame->func = (const zend_function*)op_array;
3141 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
3142 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3143 SET_STACK_VAR(stack, i, j);
3144 vars_op_array[j] = op_array;
3145 if (ssa->vars[j].use_chain >= 0
3146 && ssa->vars[j].alias == NO_ALIAS
3147 && zend_jit_var_supports_reg(ssa, j)
3148 && !(ssa->var_info[j].type & MAY_BE_GUARD)) {
3149 RA_IVAL_START(j, idx);
3150 RA_IVAL_FLAGS(j) = ZREG_LOAD;
3151 count++;
3152 }
3153 j++;
3154 }
3155 } else {
3156 level--;
3157 }
3158 } else if (p->op == ZEND_JIT_TRACE_END) {
3159 break;
3160 }
3161 }
3162
3163 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3164 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3165 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3166 zend_ssa_phi *phi = ssa->blocks[1].phis;
3167
3168 while (phi) {
3169 i = phi->sources[1];
3170 if (RA_HAS_IVAL(i) && !ssa->vars[phi->ssa_var].no_val) {
3171 RA_IVAL_END(i, idx);
3172 RA_IVAL_FLAGS(i) &= ~ZREG_LAST_USE;
3173 }
3174 phi = phi->next;
3175 }
3176
3177 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
3178 for (i = 0; i < op_array->last_var; i++) {
3179 if (RA_HAS_IVAL(i) && !ssa->vars[i].phi_use_chain) {
3180 RA_IVAL_END(i, idx);
3181 RA_IVAL_FLAGS(i) &= ~ZREG_LAST_USE;
3182 } else {
3183 RA_IVAL_CLOSE(i, idx);
3184 }
3185 }
3186 }
3187 }
3188
3189 if (count) {
3190 for (i = 0; i < ssa->vars_count; i++) {
3191 if (RA_HAS_REG(i)) {
3192 if ((RA_REG_FLAGS(i) & ZREG_LOAD) &&
3193 (RA_REG_FLAGS(i) & ZREG_LAST_USE) &&
3194 (i >= parent_vars_count || STACK_REG(parent_stack, i) == ZREG_NONE) &&
3195 zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0) {
3196 /* skip life range with single use */
3197 RA_REG_DEL(i);
3198 count--;
3199 }
3200 }
3201 }
3202 }
3203
3204 if (count) {
3205 /* SSA resolution */
3206 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
3207 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
3208 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
3209 zend_ssa_phi *phi = ssa->blocks[1].phis;
3210
3211 while (phi) {
3212 int def = phi->ssa_var;
3213 int use = phi->sources[1];
3214
3215 if (RA_HAS_REG(def)) {
3216 if (!RA_HAS_REG(use)) {
3217 RA_REG_FLAGS(def) |= ZREG_LOAD;
3218 if ((RA_REG_FLAGS(def) & ZREG_LAST_USE)
3219 && ssa->vars[def].use_chain >= 0
3220 && !ssa->vars[def].phi_use_chain
3221 && zend_ssa_next_use(ssa->ops, def, ssa->vars[def].use_chain) < 0
3222 ) {
3223 /* remove interval used once */
3224 RA_REG_DEL(def);
3225 count--;
3226 }
3227 } else if ((ssa->var_info[def].type & MAY_BE_ANY) != (ssa->var_info[use].type & MAY_BE_ANY)) {
3228 RA_REG_FLAGS(def) |= ZREG_LOAD;
3229 RA_REG_FLAGS(use) |= ZREG_STORE;
3230 } else {
3231 use = phi->sources[0];
3232 if (zend_jit_var_supports_reg(ssa, use)) {
3233 ZEND_ASSERT(!RA_HAS_REG(use));
3234 RA_REG_START(use, 0);
3235 RA_REG_FLAGS(use) = ZREG_LOAD;
3236 count++;
3237 } else {
3238 RA_REG_FLAGS(def) |= ZREG_LOAD;
3239 }
3240 }
3241 } else if (RA_HAS_REG(use)
3242 && (!ssa->vars[def].no_val
3243 )) {
3244 if (ssa->vars[use].use_chain >= 0) {
3245 RA_REG_FLAGS(use) |= ZREG_STORE; // TODO: ext/opcache/tests/jit/reg_alloc_00[67].phpt ???
3246 } else {
3247 RA_REG_DEL(use);
3248 count--;
3249 }
3250 }
3251 phi = phi->next;
3252 }
3253 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
3254 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
3255 for (i = 0; i < op_array->last_var + op_array->T; i++) {
3256 int var = STACK_VAR(stack, i);
3257 if (var >= 0 && RA_HAS_REG(var)
3258 && !(RA_REG_FLAGS(var) & (ZREG_LOAD|ZREG_STORE|ZREG_LAST_USE))) {
3259 RA_REG_FLAGS(var) |= ZREG_STORE;
3260 }
3261 }
3262 }
3263
3264 if (!count) {
3265 zend_arena_release(&CG(arena), checkpoint);
3266 return NULL;
3267 }
3268
3269 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
3270 fprintf(stderr, "---- TRACE %d Live Ranges \"%s\"\n", ZEND_JIT_TRACE_NUM, op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
3271 for (i = 0; i < ssa->vars_count; i++) {
3272 if (RA_HAS_REG(i)) {
3273 fprintf(stderr, "#%d.", i);
3274 uint32_t var_num = ssa->vars[i].var;
3275 zend_dump_var(vars_op_array[i], (var_num < vars_op_array[i]->last_var ? IS_CV : 0), var_num);
3276 if (RA_REG_FLAGS(i) & ZREG_LAST_USE) {
3277 fprintf(stderr, " last_use");
3278 }
3279 if (RA_REG_FLAGS(i) & ZREG_LOAD) {
3280 fprintf(stderr, " load");
3281 }
3282 if (RA_REG_FLAGS(i) & ZREG_STORE) {
3283 fprintf(stderr, " store");
3284 }
3285 fprintf(stderr, "\n");
3286 }
3287 }
3288 fprintf(stderr, "\n");
3289 }
3290
3291 return ra;
3292 }
3293
3294 zend_arena_release(&CG(arena), checkpoint);
3295 return NULL;
3296 }
3297
zend_jit_trace_cleanup_stack(zend_jit_ctx * jit,zend_jit_trace_stack * stack,const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes)3298 static void zend_jit_trace_cleanup_stack(zend_jit_ctx *jit, zend_jit_trace_stack *stack, const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes)
3299 {
3300 if (ssa_op->op1_use >= 0
3301 && jit->ra[ssa_op->op1_use].ref
3302 && (jit->ra[ssa_op->op1_use].flags & ZREG_LAST_USE)
3303 && (ssa_op->op1_use_chain == -1
3304 || zend_ssa_is_no_val_use(ssa_opcodes[ssa_op->op1_use_chain], ssa->ops + ssa_op->op1_use_chain, ssa_op->op1_use))) {
3305 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var));
3306 }
3307 if (ssa_op->op2_use >= 0
3308 && jit->ra[ssa_op->op2_use].ref
3309 && (jit->ra[ssa_op->op2_use].flags & ZREG_LAST_USE)
3310 && (ssa_op->op2_use_chain == -1
3311 || zend_ssa_is_no_val_use(ssa_opcodes[ssa_op->op2_use_chain], ssa->ops + ssa_op->op2_use_chain, ssa_op->op2_use))) {
3312 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->op2.var));
3313 }
3314 if (ssa_op->result_use >= 0
3315 && jit->ra[ssa_op->result_use].ref
3316 && (jit->ra[ssa_op->result_use].flags & ZREG_LAST_USE)
3317 && (ssa_op->res_use_chain == -1
3318 || zend_ssa_is_no_val_use(ssa_opcodes[ssa_op->res_use_chain], ssa->ops + ssa_op->res_use_chain, ssa_op->result_use))) {
3319 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
3320 }
3321 }
3322
zend_jit_trace_setup_ret_counter(const zend_op * opline,size_t offset)3323 static void zend_jit_trace_setup_ret_counter(const zend_op *opline, size_t offset)
3324 {
3325 zend_op *next_opline = (zend_op*)(opline + 1);
3326
3327 if (JIT_G(hot_return) && !ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags) {
3328 ZEND_ASSERT(zend_jit_ret_trace_counter_handler != NULL);
3329 if (!ZEND_OP_TRACE_INFO(next_opline, offset)->counter) {
3330 ZEND_OP_TRACE_INFO(next_opline, offset)->counter =
3331 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
3332 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
3333 }
3334 ZEND_OP_TRACE_INFO(next_opline, offset)->trace_flags = ZEND_JIT_TRACE_START_RETURN;
3335 next_opline->handler = (const void*)zend_jit_ret_trace_counter_handler;
3336 }
3337 }
3338
zend_jit_may_delay_fetch_this(const zend_op_array * op_array,zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_ssa_op * ssa_op)3339 static bool zend_jit_may_delay_fetch_this(const zend_op_array *op_array, zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_ssa_op *ssa_op)
3340 {
3341 int var = ssa_op->result_def;
3342 int i;
3343 int use = ssa->vars[var].use_chain;
3344 const zend_op *opline;
3345
3346 if (use < 0
3347 || ssa->vars[var].phi_use_chain
3348 || ssa->ops[use].op1_use != var
3349 || ssa->ops[use].op1_use_chain != -1) {
3350 return 0;
3351 }
3352
3353 opline = ssa_opcodes[use];
3354 if (opline->opcode == ZEND_INIT_METHOD_CALL) {
3355 return (opline->op2_type == IS_CONST &&
3356 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING);
3357 } else if (opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG) {
3358 if (!JIT_G(current_frame)
3359 || !JIT_G(current_frame)->call
3360 || !JIT_G(current_frame)->call->func
3361 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
3362 return 0;
3363 }
3364 } else if (opline->opcode != ZEND_FETCH_OBJ_R
3365 && opline->opcode != ZEND_FETCH_OBJ_IS
3366 && opline->opcode != ZEND_FETCH_OBJ_W
3367 && opline->opcode != ZEND_ASSIGN_OBJ
3368 && opline->opcode != ZEND_ASSIGN_OBJ_OP
3369 && opline->opcode != ZEND_PRE_INC_OBJ
3370 && opline->opcode != ZEND_PRE_DEC_OBJ
3371 && opline->opcode != ZEND_POST_INC_OBJ
3372 && opline->opcode != ZEND_POST_DEC_OBJ) {
3373 return 0;
3374 }
3375
3376 if (opline->op2_type != IS_CONST
3377 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3378 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3379 return 0;
3380 }
3381
3382 if (opline->opcode == ZEND_ASSIGN_OBJ_OP) {
3383 if (opline->op1_type == IS_CV
3384 && (opline+1)->op1_type == IS_CV
3385 && (opline+1)->op1.var == opline->op1.var) {
3386 /* skip $a->prop += $a; */
3387 return 0;
3388 }
3389 if (!zend_jit_supported_binary_op(
3390 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3391 return 0;
3392 }
3393 }
3394
3395 for (i = ssa->vars[var].definition; i < use; i++) {
3396 if (ssa_opcodes[i]->opcode == ZEND_DO_UCALL
3397 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL_BY_NAME
3398 || ssa_opcodes[i]->opcode == ZEND_DO_FCALL
3399 || ssa_opcodes[i]->opcode == ZEND_INCLUDE_OR_EVAL) {
3400 return 0;
3401 }
3402 }
3403
3404 return 1;
3405 }
3406
zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack * stack,uint32_t stack_size)3407 static int zend_jit_trace_stack_needs_deoptimization(zend_jit_trace_stack *stack, uint32_t stack_size)
3408 {
3409 uint32_t i;
3410
3411 for (i = 0; i < stack_size; i++) {
3412 if (STACK_FLAGS(stack, i) & ~(ZREG_LOAD|ZREG_STORE|ZREG_LAST_USE)) {
3413 return 1;
3414 } else if (STACK_REG(stack, i) != ZREG_NONE) {
3415 return 1;
3416 }
3417 }
3418 return 0;
3419 }
3420
zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num,uint32_t exit_num)3421 static int zend_jit_trace_exit_needs_deoptimization(uint32_t trace_num, uint32_t exit_num)
3422 {
3423 const zend_op *opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
3424 uint32_t flags = zend_jit_traces[trace_num].exit_info[exit_num].flags;
3425 uint32_t stack_size;
3426 zend_jit_trace_stack *stack;
3427
3428 if (opline || (flags & (ZEND_JIT_EXIT_RESTORE_CALL|ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2))) {
3429 return 1;
3430 }
3431
3432 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
3433 stack = zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset;
3434 return zend_jit_trace_stack_needs_deoptimization(stack, stack_size);
3435 }
3436
zend_jit_trace_deoptimization(zend_jit_ctx * jit,uint32_t flags,const zend_op * opline,zend_jit_trace_stack * parent_stack,int parent_vars_count,zend_ssa * ssa,zend_jit_trace_stack * stack,zend_jit_exit_const * constants,int8_t func_reg,bool polymorphic_side_trace)3437 static int zend_jit_trace_deoptimization(
3438 zend_jit_ctx *jit,
3439 uint32_t flags,
3440 const zend_op *opline,
3441 zend_jit_trace_stack *parent_stack,
3442 int parent_vars_count,
3443 zend_ssa *ssa,
3444 zend_jit_trace_stack *stack,
3445 zend_jit_exit_const *constants,
3446 int8_t func_reg,
3447 bool polymorphic_side_trace)
3448 {
3449 int i;
3450 int check2 = -1;
3451
3452 // TODO: Merge this loop with the following register LOAD loop to implement parallel move ???
3453 for (i = 0; i < parent_vars_count; i++) {
3454 int8_t reg = STACK_REG(parent_stack, i);
3455
3456 if (STACK_FLAGS(parent_stack, i) == ZREG_CONST) {
3457 uint8_t type = STACK_TYPE(parent_stack, i);
3458
3459 if (type == IS_LONG) {
3460 if (!zend_jit_store_const_long(jit, i,
3461 (zend_long)constants[STACK_REF(parent_stack, i)].i)) {
3462 return 0;
3463 }
3464 } else if (type == IS_DOUBLE) {
3465 if (!zend_jit_store_const_double(jit, i,
3466 constants[STACK_REF(parent_stack, i)].d)) {
3467 return 0;
3468 }
3469 } else {
3470 ZEND_UNREACHABLE();
3471 }
3472 if (stack) {
3473 SET_STACK_TYPE(stack, i, type, 1);
3474 if (jit->ra && jit->ra[i].ref) {
3475 SET_STACK_REF(stack, i, jit->ra[i].ref);
3476 }
3477 }
3478 } else if (STACK_FLAGS(parent_stack, i) == ZREG_TYPE_ONLY) {
3479 uint8_t type = STACK_TYPE(parent_stack, i);
3480
3481 if (!zend_jit_store_type(jit, i, type)) {
3482 return 0;
3483 }
3484 if (stack) {
3485 SET_STACK_TYPE(stack, i, type, 1);
3486 }
3487 } else if (STACK_FLAGS(parent_stack, i) == ZREG_THIS) {
3488 if (polymorphic_side_trace) {
3489 ssa->var_info[i].delayed_fetch_this = 1;
3490 if (stack) {
3491 SET_STACK_REG_EX(stack, i, ZREG_NONE, ZREG_THIS);
3492 }
3493 } else if (!zend_jit_load_this(jit, EX_NUM_TO_VAR(i))) {
3494 return 0;
3495 }
3496 } else if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_ADDREF) {
3497 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(i));
3498 zend_jit_zval_try_addref(jit, dst);
3499 } else if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_COPY) {
3500 ZEND_ASSERT(reg != ZREG_NONE);
3501 ZEND_ASSERT(check2 == -1);
3502 check2 = i;
3503 } else if (STACK_FLAGS(parent_stack, i) & ZREG_SPILL_SLOT) {
3504 if (ssa && ssa->vars[i].no_val) {
3505 /* pass */
3506 } else {
3507 uint8_t type = STACK_TYPE(parent_stack, i);
3508
3509 if (!zend_jit_store_spill_slot(jit, 1 << type, i, reg, STACK_REF(parent_stack, i),
3510 STACK_MEM_TYPE(parent_stack, i) != type)) {
3511 return 0;
3512 }
3513 if (stack) {
3514 if (jit->ra && jit->ra[i].ref) {
3515 SET_STACK_TYPE(stack, i, type, 0);
3516 if ((STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) != 0) {
3517 SET_STACK_REF_EX(stack, i, jit->ra[i].ref, ZREG_LOAD);
3518 } else {
3519 SET_STACK_REF(stack, i, jit->ra[i].ref);
3520 }
3521 } else {
3522 SET_STACK_TYPE(stack, i, type, 1);
3523 }
3524 }
3525 }
3526 } else if (reg != ZREG_NONE) {
3527 if (ssa && ssa->vars[i].no_val) {
3528 /* pass */
3529 } else {
3530 uint8_t type = STACK_TYPE(parent_stack, i);
3531
3532 if (!zend_jit_store_reg(jit, 1 << type, i, reg,
3533 (STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) != 0,
3534 STACK_MEM_TYPE(parent_stack, i) != type)) {
3535 return 0;
3536 }
3537 if (stack) {
3538 if (jit->ra && jit->ra[i].ref) {
3539 SET_STACK_TYPE(stack, i, type, 0);
3540 if ((STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) != 0) {
3541 SET_STACK_REF_EX(stack, i, jit->ra[i].ref, ZREG_LOAD);
3542 } else {
3543 SET_STACK_REF(stack, i, jit->ra[i].ref);
3544 }
3545 } else {
3546 SET_STACK_TYPE(stack, i, type, 1);
3547 }
3548 }
3549 }
3550 }
3551 }
3552
3553 if (check2 != -1) {
3554 int8_t reg = STACK_REG(parent_stack, check2);
3555
3556 ZEND_ASSERT(STACK_FLAGS(parent_stack, check2) == ZREG_ZVAL_COPY);
3557 ZEND_ASSERT(reg != ZREG_NONE);
3558 if (!zend_jit_escape_if_undef(jit, check2, flags, opline, reg)) {
3559 return 0;
3560 }
3561 if (!zend_jit_restore_zval(jit, EX_NUM_TO_VAR(check2), reg)) {
3562 return 0;
3563 }
3564 }
3565
3566 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
3567 if (!zend_jit_save_call_chain(jit, -1)) {
3568 return 0;
3569 }
3570 }
3571
3572 if (flags & ZEND_JIT_EXIT_FREE_OP2) {
3573 const zend_op *op = opline - 1;
3574
3575 if (!zend_jit_free_op(jit, op, -1, op->op2.var)) {
3576 return 0;
3577 }
3578 }
3579
3580 if (flags & ZEND_JIT_EXIT_FREE_OP1) {
3581 const zend_op *op = opline - 1;
3582
3583 if (!zend_jit_free_op(jit, op, -1, op->op1.var)) {
3584 return 0;
3585 }
3586 }
3587
3588 if (flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
3589 zend_jit_check_exception(jit);
3590 }
3591
3592 if ((flags & ZEND_JIT_EXIT_METHOD_CALL) && !polymorphic_side_trace) {
3593 if (!zend_jit_free_trampoline(jit, func_reg)) {
3594 return 0;
3595 }
3596 }
3597
3598 return 1;
3599 }
3600
zend_jit_trace_set_var_range(zend_ssa_var_info * info,zend_long min,zend_long max)3601 static void zend_jit_trace_set_var_range(zend_ssa_var_info *info, zend_long min, zend_long max)
3602 {
3603 info->has_range = 1;
3604 info->range.min = min;
3605 info->range.max = max;
3606 info->range.underflow = 0;
3607 info->range.overflow = 0;
3608 }
3609
zend_jit_trace_update_condition_ranges(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa,bool exit_if_true)3610 static void zend_jit_trace_update_condition_ranges(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, bool exit_if_true)
3611 {
3612 zend_long op1_min, op1_max, op2_min, op2_max;
3613
3614 if ((OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG
3615 || (OP1_INFO() & MAY_BE_ANY) != MAY_BE_LONG) {
3616 return;
3617 }
3618
3619 op1_min = OP1_MIN_RANGE();
3620 op1_max = OP1_MAX_RANGE();
3621 op2_min = OP2_MIN_RANGE();
3622 op2_max = OP2_MAX_RANGE();
3623
3624 switch (opline->opcode) {
3625 case ZEND_IS_EQUAL:
3626 case ZEND_CASE:
3627 case ZEND_IS_IDENTICAL:
3628 case ZEND_CASE_STRICT:
3629 case ZEND_IS_NOT_IDENTICAL:
3630 if (!exit_if_true) {
3631 /* op1 == op2 */
3632 if (ssa_op->op1_use >= 0) {
3633 zend_jit_trace_set_var_range(
3634 &ssa->var_info[ssa_op->op1_use],
3635 MAX(op1_min, op2_min),
3636 MIN(op1_max, op2_max));
3637 }
3638 if (ssa_op->op2_use >= 0) {
3639 zend_jit_trace_set_var_range(
3640 &ssa->var_info[ssa_op->op2_use],
3641 MAX(op2_min, op1_min),
3642 MIN(op2_max, op1_max));
3643 }
3644 }
3645 break;
3646 case ZEND_IS_NOT_EQUAL:
3647 if (exit_if_true) {
3648 /* op1 == op2 */
3649 if (ssa_op->op1_use >= 0) {
3650 zend_jit_trace_set_var_range(
3651 &ssa->var_info[ssa_op->op1_use],
3652 MAX(op1_min, op2_min),
3653 MIN(op1_max, op2_max));
3654 }
3655 if (ssa_op->op2_use >= 0) {
3656 zend_jit_trace_set_var_range(
3657 &ssa->var_info[ssa_op->op2_use],
3658 MAX(op2_min, op1_min),
3659 MIN(op2_max, op1_max));
3660 }
3661 }
3662 break;
3663 case ZEND_IS_SMALLER_OR_EQUAL:
3664 if (!exit_if_true) {
3665 /* op1 <= op2 */
3666 if (ssa_op->op1_use >= 0) {
3667 zend_jit_trace_set_var_range(
3668 &ssa->var_info[ssa_op->op1_use],
3669 op1_min,
3670 MIN(op1_max, op2_max));
3671 }
3672 if (ssa_op->op2_use >= 0) {
3673 zend_jit_trace_set_var_range(
3674 &ssa->var_info[ssa_op->op2_use],
3675 MAX(op2_min, op1_min),
3676 op2_max);
3677 }
3678 } else {
3679 /* op1 > op2 */
3680 if (ssa_op->op1_use >= 0) {
3681 zend_jit_trace_set_var_range(
3682 &ssa->var_info[ssa_op->op1_use],
3683 op2_min != ZEND_LONG_MAX ? MAX(op1_min, op2_min + 1) : op1_min,
3684 op1_max);
3685 }
3686 if (ssa_op->op2_use >= 0) {
3687 zend_jit_trace_set_var_range(
3688 &ssa->var_info[ssa_op->op2_use],
3689 op2_min,
3690 op2_max != ZEND_LONG_MIN ?MIN(op2_max, op1_max - 1) : op1_max);
3691 }
3692 }
3693 break;
3694 case ZEND_IS_SMALLER:
3695 if (!exit_if_true) {
3696 /* op1 < op2 */
3697 if (ssa_op->op1_use >= 0) {
3698 zend_jit_trace_set_var_range(
3699 &ssa->var_info[ssa_op->op1_use],
3700 op1_min,
3701 op2_max != ZEND_LONG_MIN ? MIN(op1_max, op2_max - 1) : op1_max);
3702 }
3703 if (ssa_op->op2_use >= 0) {
3704 zend_jit_trace_set_var_range(
3705 &ssa->var_info[ssa_op->op2_use],
3706 op1_min != ZEND_LONG_MAX ? MAX(op2_min, op1_min + 1) : op2_min,
3707 op2_max);
3708 }
3709 } else {
3710 /* op1 >= op2 */
3711 if (ssa_op->op1_use >= 0) {
3712 zend_jit_trace_set_var_range(
3713 &ssa->var_info[ssa_op->op1_use],
3714 MAX(op1_min, op2_min),
3715 op1_max);
3716 }
3717 if (ssa_op->op2_use >= 0) {
3718 zend_jit_trace_set_var_range(
3719 &ssa->var_info[ssa_op->op2_use],
3720 op2_min,
3721 MIN(op2_max, op1_max));
3722 }
3723 }
3724 break;
3725 }
3726 }
3727
zend_jit_may_skip_comparison(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_ssa * ssa,const zend_op ** ssa_opcodes,const zend_op_array * op_array)3728 static bool zend_jit_may_skip_comparison(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_ssa *ssa, const zend_op **ssa_opcodes, const zend_op_array *op_array)
3729 {
3730 uint8_t prev_opcode;
3731
3732 if (opline->op1_type == IS_CONST
3733 && Z_TYPE_P(RT_CONSTANT(opline, opline->op1)) == IS_LONG
3734 && Z_LVAL_P(RT_CONSTANT(opline, opline->op1)) == 0) {
3735 if (ssa_op->op2_use >= 0) {
3736 if ((ssa_op-1)->op1_def == ssa_op->op2_use) {
3737 ssa_op--;
3738 opline = ssa_opcodes[ssa_op - ssa->ops];
3739 prev_opcode = opline->opcode;
3740 if (prev_opcode == ZEND_PRE_INC
3741 || prev_opcode == ZEND_PRE_DEC
3742 || prev_opcode == ZEND_POST_INC
3743 || prev_opcode == ZEND_POST_DEC) {
3744 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3745 }
3746 } else if ((ssa_op-1)->result_def == ssa_op->op2_use) {
3747 ssa_op--;
3748 opline = ssa_opcodes[ssa_op - ssa->ops];
3749 prev_opcode = opline->opcode;
3750 if (prev_opcode == ZEND_ADD
3751 || prev_opcode == ZEND_SUB) {
3752 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3753 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3754 }
3755 }
3756 }
3757 } else if (opline->op2_type == IS_CONST
3758 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG
3759 && Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) == 0) {
3760 if (ssa_op->op1_use >= 0) {
3761 if ((ssa_op-1)->op1_def == ssa_op->op1_use) {
3762 ssa_op--;
3763 opline = ssa_opcodes[ssa_op - ssa->ops];
3764 prev_opcode = opline->opcode;
3765 if (prev_opcode == ZEND_PRE_INC
3766 || prev_opcode == ZEND_PRE_DEC
3767 || prev_opcode == ZEND_POST_INC
3768 || prev_opcode == ZEND_POST_DEC) {
3769 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3770 }
3771 } else if ((ssa_op-1)->result_def == ssa_op->op1_use) {
3772 ssa_op--;
3773 opline = ssa_opcodes[ssa_op - ssa->ops];
3774 prev_opcode = opline->opcode;
3775 if (prev_opcode == ZEND_ADD
3776 || prev_opcode == ZEND_SUB) {
3777 return (OP1_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0 &&
3778 (OP2_INFO() & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-MAY_BE_LONG)) == 0;
3779 }
3780 }
3781 }
3782 } else {
3783 const zend_ssa_op *prev_ssa_op = ssa_op - 1;
3784 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3785
3786 if ((prev_opcode == ZEND_JMPZ || prev_opcode == ZEND_JMPNZ)
3787 && prev_ssa_op != ssa->ops
3788 && prev_ssa_op->op1_use >= 0
3789 && prev_ssa_op->op1_use == (prev_ssa_op-1)->result_def) {
3790 prev_ssa_op--;
3791 prev_opcode = ssa_opcodes[prev_ssa_op - ssa->ops]->opcode;
3792 }
3793
3794 if (ssa_op->op1_use == prev_ssa_op->op1_use
3795 && ssa_op->op2_use == prev_ssa_op->op2_use) {
3796 if (prev_opcode == ZEND_IS_EQUAL
3797 || prev_opcode == ZEND_IS_NOT_EQUAL
3798 || prev_opcode == ZEND_IS_SMALLER
3799 || prev_opcode == ZEND_IS_SMALLER_OR_EQUAL
3800 || prev_opcode == ZEND_CASE
3801 || prev_opcode == ZEND_IS_IDENTICAL
3802 || prev_opcode == ZEND_IS_NOT_IDENTICAL
3803 || prev_opcode == ZEND_CASE_STRICT) {
3804 if (ssa_op->op1_use < 0) {
3805 if (RT_CONSTANT(opline, opline->op1) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op1)) {
3806 return 0;
3807 }
3808 }
3809 if (ssa_op->op2_use < 0) {
3810 if (RT_CONSTANT(opline, opline->op2) != RT_CONSTANT(&ssa_opcodes[prev_ssa_op - ssa->ops], ssa_opcodes[prev_ssa_op - ssa->ops]->op2)) {
3811 return 0;
3812 }
3813 }
3814 return 1;
3815 }
3816 }
3817 }
3818 return 0;
3819 }
3820
zend_jit_trace_next_is_send_result(const zend_op * opline,zend_jit_trace_rec * p,zend_jit_trace_stack_frame * frame)3821 static bool zend_jit_trace_next_is_send_result(const zend_op *opline,
3822 zend_jit_trace_rec *p,
3823 zend_jit_trace_stack_frame *frame)
3824 {
3825 if (opline->result_type == IS_TMP_VAR
3826 && (p+1)->op == ZEND_JIT_TRACE_VM
3827 && (p+1)->opline == opline + 1
3828 && ((opline+1)->opcode == ZEND_SEND_VAL
3829 || ((opline+1)->opcode == ZEND_SEND_VAL_EX
3830 && frame
3831 && frame->call
3832 && frame->call->func
3833 && !ARG_MUST_BE_SENT_BY_REF(frame->call->func, (opline+1)->op2.num)))
3834 && (opline+1)->op1_type == IS_TMP_VAR
3835 && (opline+1)->op2_type != IS_CONST /* Named parameters not supported in JIT */
3836 && (opline+1)->op1.var == opline->result.var) {
3837
3838 if (frame->call && frame->call->func) {
3839 uint8_t res_type = (p+1)->op1_type;
3840
3841 if (res_type != IS_UNKNOWN && !(res_type & IS_TRACE_REFERENCE) ) {
3842 zend_jit_trace_send_type(opline+1, frame->call, res_type);
3843 }
3844 }
3845 return 1;
3846 }
3847 return 0;
3848 }
3849
zend_jit_find_ssa_var(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t opline_num,uint32_t var_num)3850 static int zend_jit_find_ssa_var(const zend_op_array *op_array,
3851 const zend_ssa *ssa,
3852 uint32_t opline_num,
3853 uint32_t var_num)
3854 {
3855 int ssa_var, j, b = ssa->cfg.map[opline_num];
3856 const zend_basic_block *bb = ssa->cfg.blocks + b;
3857 const zend_ssa_phi *phi;
3858 const zend_ssa_op *ssa_op;
3859 zend_worklist worklist;
3860 ALLOCA_FLAG(use_heap)
3861
3862 while (1) {
3863 ssa_op = ssa->ops + opline_num;
3864 ssa_var = ssa_op->result_def;
3865 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3866 return ssa_var;
3867 }
3868 ssa_var = ssa_op->op2_def;
3869 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3870 return ssa_var;
3871 }
3872 ssa_var = ssa_op->op1_def;
3873 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3874 return ssa_var;
3875 }
3876 if (opline_num == bb->start) {
3877 break;
3878 }
3879 opline_num--;
3880 }
3881 phi = ssa->blocks[b].phis;
3882 ssa_var = -1;
3883 while (phi) {
3884 if (phi->var == var_num) {
3885 ssa_var = phi->ssa_var;
3886 }
3887 phi = phi->next;
3888 }
3889 if (ssa_var >= 0) {
3890 return ssa_var;
3891 }
3892
3893 if (!bb->predecessors_count) {
3894 return -1;
3895 }
3896
3897 ZEND_WORKLIST_ALLOCA(&worklist, ssa->cfg.blocks_count, use_heap);
3898
3899 for (j = 0; j < bb->predecessors_count; j++) {
3900 b = ssa->cfg.predecessors[bb->predecessor_offset + j];
3901 zend_worklist_push(&worklist, b);
3902 }
3903
3904 while (zend_worklist_len(&worklist) != 0) {
3905 b = zend_worklist_pop(&worklist);
3906 bb = &ssa->cfg.blocks[b];
3907 if (bb->len) {
3908 opline_num = bb->start + bb->len - 1;
3909 while (1) {
3910 ssa_op = ssa->ops + opline_num;
3911 ssa_var = ssa_op->result_def;
3912 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3913 goto found;
3914 }
3915 ssa_var = ssa_op->op2_def;
3916 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3917 goto found;
3918 }
3919 ssa_var = ssa_op->op1_def;
3920 if (ssa_var >= 0 && ssa->vars[ssa_var].var == var_num) {
3921 goto found;
3922 }
3923 if (opline_num == bb->start) {
3924 break;
3925 }
3926 opline_num--;
3927 }
3928 }
3929 phi = ssa->blocks[b].phis;
3930 ssa_var = -1;
3931 while (phi) {
3932 if (phi->var == var_num) {
3933 ssa_var = phi->ssa_var;
3934 }
3935 phi = phi->next;
3936 }
3937 if (ssa_var >= 0) {
3938 goto found;
3939 }
3940 for (j = 0; j < bb->predecessors_count; j++) {
3941 b = ssa->cfg.predecessors[bb->predecessor_offset + j];
3942 zend_worklist_push(&worklist, b);
3943 }
3944 }
3945 ssa_var = -1;
3946
3947 found:
3948 ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
3949 return ssa_var;
3950 }
3951
zend_jit_trace_must_store_type(const zend_op_array * op_array,const zend_ssa * ssa,uint32_t opline_num,uint32_t var_num,uint8_t type)3952 static bool zend_jit_trace_must_store_type(const zend_op_array *op_array,
3953 const zend_ssa *ssa,
3954 uint32_t opline_num,
3955 uint32_t var_num,
3956 uint8_t type)
3957 {
3958 if (ssa->var_info) {
3959 int ssa_var = zend_jit_find_ssa_var(op_array, ssa, opline_num, var_num);
3960
3961 if (ssa_var >= 0) {
3962 if ((ssa->var_info[ssa_var].type & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1U << type)) {
3963 return 0;
3964 }
3965 }
3966 }
3967 return 1;
3968 }
3969
zend_jit_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_trace,uint32_t exit_num)3970 static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num)
3971 {
3972 const void *handler = NULL;
3973 zend_jit_ctx ctx;
3974 zend_jit_ctx *jit = &ctx;
3975 zend_jit_reg_var *ra = NULL;
3976 zend_script *script = NULL;
3977 zend_string *name = NULL;
3978 void *checkpoint;
3979 const zend_op_array *op_array;
3980 zend_ssa *ssa, *op_array_ssa;
3981 const zend_op **ssa_opcodes;
3982 zend_jit_trace_rec *p;
3983 zend_jit_op_array_trace_extension *jit_extension;
3984 int num_op_arrays = 0;
3985 zend_jit_trace_info *t;
3986 const zend_op_array *op_arrays[ZEND_JIT_TRACE_MAX_FUNCS];
3987 uint8_t smart_branch_opcode;
3988 const void *exit_addr;
3989 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_data_info;
3990 bool send_result = 0;
3991 bool skip_comparison;
3992 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
3993 zend_class_entry *ce;
3994 bool ce_is_instanceof;
3995 bool on_this = 0;
3996 bool delayed_fetch_this = 0;
3997 bool avoid_refcounting = 0;
3998 bool polymorphic_side_trace =
3999 parent_trace &&
4000 (zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL);
4001 uint32_t i;
4002 zend_jit_trace_stack_frame *frame, *top, *call;
4003 zend_jit_trace_stack *stack;
4004 uint8_t res_type = IS_UNKNOWN;
4005 const zend_op *opline, *orig_opline;
4006 const zend_ssa_op *ssa_op, *orig_ssa_op;
4007 int checked_stack;
4008 int peek_checked_stack;
4009 uint32_t frame_flags = 0;
4010
4011 JIT_G(current_trace) = trace_buffer;
4012
4013 checkpoint = zend_arena_checkpoint(CG(arena));
4014
4015 ssa = zend_jit_trace_build_tssa(trace_buffer, parent_trace, exit_num, script, op_arrays, &num_op_arrays);
4016
4017 if (!ssa) {
4018 goto jit_cleanup;
4019 }
4020
4021 ssa_opcodes = ((zend_tssa*)ssa)->tssa_opcodes;
4022
4023 op_array = trace_buffer->op_array;
4024 opline = trace_buffer[1].opline;
4025 name = zend_jit_trace_name(op_array, opline->lineno);
4026 zend_jit_trace_start(&ctx, op_array, ssa, name, ZEND_JIT_TRACE_NUM,
4027 parent_trace ? &zend_jit_traces[parent_trace] : NULL, exit_num);
4028 ctx.trace = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
4029
4030 /* Register allocation */
4031 if ((JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL))
4032 && JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4033 ctx.ra = ra = zend_jit_trace_allocate_registers(trace_buffer, ssa, parent_trace, exit_num);
4034 }
4035
4036 p = trace_buffer;
4037 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
4038 op_array = p->op_array;
4039 frame = JIT_G(current_frame);
4040 top = zend_jit_trace_call_frame(frame, op_array);
4041 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
4042 frame->used_stack = checked_stack = peek_checked_stack = 0;
4043 stack = frame->stack;
4044 for (i = 0; i < op_array->last_var + op_array->T; i++) {
4045 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4046 }
4047
4048 opline = p[1].opline;
4049 p += ZEND_JIT_TRACE_START_REC_SIZE;
4050
4051 jit_extension =
4052 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
4053 op_array_ssa = &jit_extension->func_info.ssa;
4054
4055 if (!parent_trace) {
4056 zend_jit_set_last_valid_opline(&ctx, opline);
4057 zend_jit_track_last_valid_opline(&ctx);
4058 } else {
4059 if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
4060 zend_jit_trace_opline_guard(&ctx, opline);
4061 } else {
4062 zend_jit_reset_last_valid_opline(&ctx);
4063 }
4064 }
4065
4066 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4067 int last_var;
4068 int parent_vars_count = 0;
4069 zend_jit_trace_stack *parent_stack = NULL;
4070 int used_stack = ((zend_tssa*)ssa)->used_stack;
4071
4072 if (used_stack > 0) {
4073 peek_checked_stack = used_stack;
4074 if (!zend_jit_stack_check(&ctx, opline, used_stack)) {
4075 goto jit_failure;
4076 }
4077 }
4078
4079 if (parent_trace) {
4080 parent_vars_count = MIN(zend_jit_traces[parent_trace].exit_info[exit_num].stack_size,
4081 op_array->last_var + op_array->T);
4082 if (parent_vars_count) {
4083 parent_stack =
4084 zend_jit_traces[parent_trace].stack_map +
4085 zend_jit_traces[parent_trace].exit_info[exit_num].stack_offset;
4086 }
4087 }
4088
4089 last_var = op_array->last_var;
4090 if (trace_buffer->start != ZEND_JIT_TRACE_START_ENTER) {
4091 last_var += op_array->T;
4092 }
4093
4094 for (i = 0; i < last_var; i++) {
4095 uint32_t info = ssa->var_info[i].type;
4096
4097 if (!(info & MAY_BE_GUARD) && has_concrete_type(info)) {
4098 uint8_t type, mem_type;
4099
4100 type = concrete_type(info);
4101 if (i < parent_vars_count
4102 && STACK_TYPE(parent_stack, i) == type) {
4103 mem_type = STACK_MEM_TYPE(parent_stack, i);
4104 if (mem_type != IS_UNKNOWN) {
4105 SET_STACK_TYPE(stack, i, mem_type, 1);
4106 }
4107 SET_STACK_TYPE(stack, i, type, 0);
4108 } else {
4109 SET_STACK_TYPE(stack, i, type, 1);
4110 }
4111 } else if (ssa->vars[i].alias != NO_ALIAS) {
4112 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
4113 } else if (i < parent_vars_count
4114 && STACK_TYPE(parent_stack, i) != IS_UNKNOWN) {
4115 /* This must be already handled by trace type inference */
4116 ZEND_UNREACHABLE();
4117 // SET_STACK_TYPE(stack, i, STACK_TYPE(parent_stack, i));
4118 } else if ((info & MAY_BE_GUARD) != 0
4119 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4120 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4121 || (trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET
4122 && (opline-1)->result_type == IS_VAR
4123 && EX_VAR_TO_NUM((opline-1)->result.var) == i))
4124 && (ssa->vars[i].use_chain != -1
4125 || (ssa->vars[i].phi_use_chain
4126 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_GUARD)))) {
4127 /* Check loop-invariant variable type */
4128 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(i), concrete_type(info))) {
4129 goto jit_failure;
4130 }
4131 info &= ~MAY_BE_GUARD;
4132 ssa->var_info[i].type = info;
4133 SET_STACK_TYPE(stack, i, concrete_type(info), 1);
4134 } else if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER
4135 && op_array->function_name
4136 && i >= op_array->num_args) {
4137 /* This must be already handled by trace type inference */
4138 ZEND_UNREACHABLE();
4139 // SET_STACK_TYPE(stack, i, IS_UNDEF, 1);
4140 }
4141
4142 if ((info & MAY_BE_PACKED_GUARD) != 0
4143 && (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4144 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4145 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET)
4146 && (ssa->vars[i].use_chain != -1
4147 || (ssa->vars[i].phi_use_chain
4148 && !(ssa->var_info[ssa->vars[i].phi_use_chain->ssa_var].type & MAY_BE_PACKED_GUARD)))) {
4149 if (!zend_jit_packed_guard(&ctx, opline, EX_NUM_TO_VAR(i), info)) {
4150 goto jit_failure;
4151 }
4152 info &= ~MAY_BE_PACKED_GUARD;
4153 ssa->var_info[i].type = info;
4154 }
4155 }
4156
4157 if (parent_trace) {
4158 /* Deoptimization */
4159 if (!zend_jit_trace_deoptimization(&ctx,
4160 zend_jit_traces[parent_trace].exit_info[exit_num].flags,
4161 zend_jit_traces[parent_trace].exit_info[exit_num].opline,
4162 parent_stack, parent_vars_count, ssa, stack,
4163 zend_jit_traces[parent_trace].constants,
4164 zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg,
4165 polymorphic_side_trace)) {
4166 goto jit_failure;
4167 }
4168 }
4169
4170 if (ra
4171 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4172 && trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4173 for (i = 0; i < last_var; i++) {
4174 if (RA_HAS_REG(i)
4175 && (RA_REG_FLAGS(i) & ZREG_LOAD) != 0
4176 && ra[i].ref != STACK_REF(stack, i)
4177 ) {
4178
4179 if ((ssa->var_info[i].type & MAY_BE_GUARD) != 0) {
4180 uint8_t op_type;
4181
4182 ssa->var_info[i].type &= ~MAY_BE_GUARD;
4183 op_type = concrete_type(ssa->var_info[i].type);
4184 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(i), op_type)) {
4185 goto jit_failure;
4186 }
4187 SET_STACK_TYPE(stack, i, op_type, 1);
4188 }
4189
4190 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP) {
4191 if (!zend_jit_load_var(&ctx, ssa->var_info[i].type, i, i)) {
4192 goto jit_failure;
4193 }
4194 SET_STACK_REF_EX(stack, i, ra[i].ref, ZREG_LOAD);
4195 } else {
4196 SET_STACK_REF_EX(stack, i, IR_NULL, ZREG_LOAD);
4197 }
4198 }
4199 }
4200 }
4201 }
4202
4203 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
4204 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
4205 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4206
4207 jit->trace_loop_ref = zend_jit_trace_begin_loop(&ctx); /* start of of trace loop */
4208
4209 if (ra) {
4210 zend_ssa_phi *phi = ssa->blocks[1].phis;
4211
4212 /* First try to insert IR Phi */
4213 while (phi) {
4214 if (RA_HAS_REG(phi->ssa_var)
4215 && !(RA_REG_FLAGS(phi->ssa_var) & ZREG_LOAD)) {
4216 zend_jit_trace_gen_phi(&ctx, phi);
4217 SET_STACK_REF(stack, phi->var, ra[phi->ssa_var].ref);
4218 }
4219 phi = phi->next;
4220 }
4221
4222 phi = ssa->blocks[1].phis;
4223 while (phi) {
4224 if (RA_HAS_REG(phi->ssa_var)) {
4225 if (RA_REG_FLAGS(phi->ssa_var) & ZREG_LOAD) {
4226 uint32_t info = ssa->var_info[phi->ssa_var].type;
4227
4228 if (info & MAY_BE_GUARD) {
4229 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(phi->var), concrete_type(info))) {
4230 goto jit_failure;
4231 }
4232 info &= ~MAY_BE_GUARD;
4233 ssa->var_info[phi->ssa_var].type = info;
4234 SET_STACK_TYPE(stack, phi->var, concrete_type(info), 1);
4235 }
4236 if (!zend_jit_load_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var)) {
4237 goto jit_failure;
4238 }
4239 SET_STACK_REF_EX(stack, phi->var, ra[phi->ssa_var].ref, ZREG_LOAD);
4240 } else if (RA_REG_FLAGS(phi->ssa_var) & ZREG_STORE) {
4241
4242 if (!zend_jit_store_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var,
4243 STACK_MEM_TYPE(stack, phi->var) != ssa->var_info[phi->ssa_var].type)) {
4244 goto jit_failure;
4245 }
4246 SET_STACK_REF_EX(stack, phi->var, ra[phi->ssa_var].ref, ZREG_STORE);
4247 } else {
4248 /* Register has to be written back on side exit */
4249 SET_STACK_REF(stack, phi->var, ra[phi->ssa_var].ref);
4250 }
4251 }
4252 phi = phi->next;
4253 }
4254 }
4255
4256 // if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
4257 // if (ra && dzend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
4258 // uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
4259 //
4260 // timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4261 // if (!timeout_exit_addr) {
4262 // goto jit_failure;
4263 // }
4264 // }
4265 // }
4266
4267 }
4268
4269 ssa_op = (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) ? ssa->ops : NULL;
4270 for (;;p++) {
4271 if (p->op == ZEND_JIT_TRACE_VM) {
4272 uint8_t op1_type = p->op1_type;
4273 uint8_t op2_type = p->op2_type;
4274 uint8_t op3_type = p->op3_type;
4275 uint8_t orig_op1_type = op1_type;
4276 uint8_t orig_op2_type = op2_type;
4277 uint8_t val_type = IS_UNKNOWN;
4278 bool op1_indirect;
4279 zend_class_entry *op1_ce = NULL;
4280 zend_class_entry *op2_ce = NULL;
4281 bool gen_handler = false;
4282
4283 opline = p->opline;
4284 if (op1_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4285 op1_type = IS_UNKNOWN;
4286 }
4287 if (op1_type != IS_UNKNOWN) {
4288 op1_type &= ~IS_TRACE_PACKED;
4289 }
4290 if (op2_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4291 op2_type = IS_UNKNOWN;
4292 }
4293 if (op3_type & (IS_TRACE_REFERENCE|IS_TRACE_INDIRECT)) {
4294 op3_type = IS_UNKNOWN;
4295 }
4296
4297 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
4298 op1_ce = (zend_class_entry*)(p+1)->ce;
4299 p++;
4300 }
4301 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
4302 op2_ce = (zend_class_entry*)(p+1)->ce;
4303 p++;
4304 }
4305 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
4306 val_type = (p+1)->op1_type;
4307 p++;
4308 }
4309
4310 frame_flags = 0;
4311
4312 switch (opline->opcode) {
4313 case ZEND_INIT_FCALL:
4314 case ZEND_INIT_FCALL_BY_NAME:
4315 case ZEND_INIT_NS_FCALL_BY_NAME:
4316 case ZEND_INIT_METHOD_CALL:
4317 case ZEND_INIT_DYNAMIC_CALL:
4318 case ZEND_INIT_STATIC_METHOD_CALL:
4319 case ZEND_INIT_USER_CALL:
4320 case ZEND_NEW:
4321 frame->call_level++;
4322 }
4323
4324 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
4325 switch (opline->opcode) {
4326 case ZEND_PRE_INC:
4327 case ZEND_PRE_DEC:
4328 case ZEND_POST_INC:
4329 case ZEND_POST_DEC:
4330 if (opline->op1_type != IS_CV) {
4331 break;
4332 }
4333 op1_info = OP1_INFO();
4334 CHECK_OP1_TRACE_TYPE();
4335 if (!(op1_info & MAY_BE_LONG)) {
4336 break;
4337 }
4338 if (opline->result_type != IS_UNUSED) {
4339 res_use_info = zend_jit_trace_type_to_info(
4340 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
4341 if (opline->result_type == IS_CV) {
4342 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4343 }
4344 res_info = RES_INFO();
4345 res_addr = RES_REG_ADDR();
4346 } else {
4347 res_use_info = -1;
4348 res_info = -1;
4349 res_addr = 0;
4350 }
4351 op1_def_info = OP1_DEF_INFO();
4352 if (op1_def_info & MAY_BE_GUARD
4353 && !has_concrete_type(op1_def_info)) {
4354 op1_def_info &= ~MAY_BE_GUARD;
4355 }
4356 if (!zend_jit_inc_dec(&ctx, opline,
4357 op1_info, OP1_REG_ADDR(),
4358 op1_def_info, OP1_DEF_REG_ADDR(),
4359 res_use_info, res_info,
4360 res_addr,
4361 (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4362 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4363 goto jit_failure;
4364 }
4365 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4366 && !(op1_info & MAY_BE_STRING)) {
4367 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4368 if (opline->result_type != IS_UNUSED) {
4369 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4370 }
4371 } else if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD)
4372 && !(op1_info & MAY_BE_STRING)) {
4373 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4374 if (opline->result_type != IS_UNUSED) {
4375 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4376 }
4377 }
4378 if (opline->result_type != IS_UNUSED
4379 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4380 && !(op1_info & MAY_BE_STRING)) {
4381 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4382 } else if (opline->result_type != IS_UNUSED
4383 && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD)
4384 && !(res_info & MAY_BE_STRING)) {
4385 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4386 }
4387 goto done;
4388 case ZEND_BW_OR:
4389 case ZEND_BW_AND:
4390 case ZEND_BW_XOR:
4391 case ZEND_SL:
4392 case ZEND_SR:
4393 case ZEND_MOD:
4394 op1_info = OP1_INFO();
4395 CHECK_OP1_TRACE_TYPE();
4396 op2_info = OP2_INFO();
4397 CHECK_OP2_TRACE_TYPE();
4398 if (!(op1_info & MAY_BE_LONG)
4399 || !(op2_info & MAY_BE_LONG)) {
4400 break;
4401 }
4402 res_addr = RES_REG_ADDR();
4403 if (Z_MODE(res_addr) != IS_REG
4404 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4405 send_result = 1;
4406 res_use_info = -1;
4407 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4408 if (!zend_jit_reuse_ip(&ctx)) {
4409 goto jit_failure;
4410 }
4411 } else {
4412 res_use_info = zend_jit_trace_type_to_info(
4413 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
4414 if (opline->result_type == IS_CV) {
4415 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4416 }
4417 }
4418 res_info = RES_INFO();
4419 if (!zend_jit_long_math(&ctx, opline,
4420 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
4421 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
4422 res_use_info, res_info, res_addr,
4423 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4424 goto jit_failure;
4425 }
4426 goto done;
4427 case ZEND_ADD:
4428 case ZEND_SUB:
4429 case ZEND_MUL:
4430 // case ZEND_DIV: // TODO: check for division by zero ???
4431 op1_info = OP1_INFO();
4432 op1_addr = OP1_REG_ADDR();
4433 op2_info = OP2_INFO();
4434 op2_addr = OP2_REG_ADDR();
4435 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4436 break;
4437 }
4438 if (opline->opcode == ZEND_ADD &&
4439 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4440 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4441 /* pass */
4442 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
4443 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4444 break;
4445 }
4446 if (orig_op1_type != IS_UNKNOWN
4447 && (orig_op1_type & IS_TRACE_REFERENCE)
4448 && opline->op1_type == IS_CV
4449 && (orig_op2_type == IS_UNKNOWN || !(orig_op2_type & IS_TRACE_REFERENCE))) {
4450 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4451 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4452 goto jit_failure;
4453 }
4454 if (ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
4455 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
4456 }
4457 } else {
4458 CHECK_OP1_TRACE_TYPE();
4459 }
4460 if (orig_op2_type != IS_UNKNOWN
4461 && (orig_op2_type & IS_TRACE_REFERENCE)
4462 && opline->op2_type == IS_CV
4463 && (orig_op1_type == IS_UNKNOWN || !(orig_op1_type & IS_TRACE_REFERENCE))) {
4464 if (!zend_jit_fetch_reference(&ctx, opline, orig_op2_type, &op2_info, &op2_addr,
4465 !ssa->var_info[ssa_op->op2_use].guarded_reference, 1)) {
4466 goto jit_failure;
4467 }
4468 if (ssa->vars[ssa_op->op2_use].alias == NO_ALIAS) {
4469 ssa->var_info[ssa_op->op2_use].guarded_reference = 1;
4470 }
4471 } else {
4472 CHECK_OP2_TRACE_TYPE();
4473 }
4474 res_addr = RES_REG_ADDR();
4475 if (Z_MODE(res_addr) != IS_REG
4476 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
4477 send_result = 1;
4478 res_use_info = -1;
4479 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4480 if (!zend_jit_reuse_ip(&ctx)) {
4481 goto jit_failure;
4482 }
4483 } else {
4484 res_use_info = zend_jit_trace_type_to_info(
4485 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
4486 if (opline->result_type == IS_CV) {
4487 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
4488 }
4489 }
4490 res_info = RES_INFO();
4491 if (opline->opcode == ZEND_ADD &&
4492 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
4493 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
4494 if (!zend_jit_add_arrays(&ctx, opline, op1_info, op1_addr, op2_info, op2_addr, res_addr)) {
4495 goto jit_failure;
4496 }
4497 } else {
4498 bool may_overflow = (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa);
4499
4500 if (ra
4501 && may_overflow
4502 && ((res_info & MAY_BE_GUARD)
4503 && (res_info & MAY_BE_ANY) == MAY_BE_LONG)
4504 && ((opline->opcode == ZEND_ADD
4505 && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)
4506 || (opline->opcode == ZEND_SUB
4507 && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1))) {
4508 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
4509 }
4510 if (!zend_jit_math(&ctx, opline,
4511 op1_info, op1_addr,
4512 op2_info, op2_addr,
4513 res_use_info, res_info, res_addr,
4514 may_overflow,
4515 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4516 goto jit_failure;
4517 }
4518 if (((res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4519 || (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
4520 && has_concrete_type(op1_info)
4521 && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
4522 && has_concrete_type(op2_info)
4523 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
4524 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4525 }
4526 }
4527 goto done;
4528 case ZEND_CONCAT:
4529 case ZEND_FAST_CONCAT:
4530 op1_info = OP1_INFO();
4531 CHECK_OP1_TRACE_TYPE();
4532 op2_info = OP2_INFO();
4533 CHECK_OP2_TRACE_TYPE();
4534 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
4535 break;
4536 }
4537 if (!(op1_info & MAY_BE_STRING) ||
4538 !(op2_info & MAY_BE_STRING)) {
4539 break;
4540 }
4541 res_addr = RES_REG_ADDR();
4542 if (zend_jit_trace_next_is_send_result(opline, p, frame)) {
4543 send_result = 1;
4544 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
4545 if (!zend_jit_reuse_ip(&ctx)) {
4546 goto jit_failure;
4547 }
4548 }
4549 if (!zend_jit_concat(&ctx, opline,
4550 op1_info, op2_info, res_addr,
4551 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4552 goto jit_failure;
4553 }
4554 goto done;
4555 case ZEND_ASSIGN_OP:
4556 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
4557 break;
4558 }
4559 op1_info = OP1_INFO();
4560 CHECK_OP1_TRACE_TYPE();
4561 op2_info = OP2_INFO();
4562 CHECK_OP2_TRACE_TYPE();
4563 if (!zend_jit_supported_binary_op(
4564 opline->extended_value, op1_info, op2_info)) {
4565 break;
4566 }
4567 op1_def_info = OP1_DEF_INFO();
4568 if (op1_def_info & MAY_BE_GUARD
4569 && !has_concrete_type(op1_def_info)) {
4570 op1_def_info &= ~MAY_BE_GUARD;
4571 }
4572 if (!zend_jit_assign_op(&ctx, opline,
4573 op1_info, op1_def_info, OP1_RANGE(),
4574 op2_info, OP2_RANGE(),
4575 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) && zend_may_overflow(opline, ssa_op, op_array, ssa),
4576 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4577 goto jit_failure;
4578 }
4579 if ((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)
4580 && has_concrete_type(op1_info)
4581 && has_concrete_type(op2_info)) {
4582 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
4583 if (opline->result_type != IS_UNUSED) {
4584 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
4585 }
4586 }
4587 goto done;
4588 case ZEND_ASSIGN_DIM_OP:
4589 if (opline->result_type != IS_UNUSED) {
4590 break;
4591 }
4592 if (!zend_jit_supported_binary_op(
4593 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4594 break;
4595 }
4596 if (opline->op1_type == IS_CV
4597 && (opline+1)->op1_type == IS_CV
4598 && (opline+1)->op1.var == opline->op1.var) {
4599 /* skip $a[x] += $a; */
4600 break;
4601 }
4602 op1_info = OP1_INFO();
4603 op1_addr = OP1_REG_ADDR();
4604 if (opline->op1_type == IS_VAR) {
4605 if (orig_op1_type != IS_UNKNOWN
4606 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4607 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4608 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4609 goto jit_failure;
4610 }
4611 } else {
4612 break;
4613 }
4614 }
4615 if (orig_op1_type != IS_UNKNOWN
4616 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4617 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4618 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4619 goto jit_failure;
4620 }
4621 if (opline->op1_type == IS_CV
4622 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4623 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4624 }
4625 } else {
4626 CHECK_OP1_TRACE_TYPE();
4627 }
4628 op2_info = OP2_INFO();
4629 CHECK_OP2_TRACE_TYPE();
4630 op1_data_info = OP1_DATA_INFO();
4631 CHECK_OP1_DATA_TRACE_TYPE();
4632 op1_def_info = OP1_DEF_INFO();
4633 if (!zend_jit_assign_dim_op(&ctx, opline,
4634 op1_info, op1_def_info, op1_addr, op2_info,
4635 op1_data_info, OP1_DATA_RANGE(), val_type,
4636 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4637 goto jit_failure;
4638 }
4639 goto done;
4640 case ZEND_PRE_INC_OBJ:
4641 case ZEND_PRE_DEC_OBJ:
4642 case ZEND_POST_INC_OBJ:
4643 case ZEND_POST_DEC_OBJ:
4644 if (opline->op2_type != IS_CONST
4645 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4646 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4647 break;
4648 }
4649 ce = NULL;
4650 ce_is_instanceof = 0;
4651 on_this = delayed_fetch_this = 0;
4652 op1_indirect = 0;
4653 if (opline->op1_type == IS_UNUSED) {
4654 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4655 ce = op_array->scope;
4656 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4657 op1_addr = 0;
4658 on_this = 1;
4659 } else {
4660 if (ssa_op->op1_use >= 0) {
4661 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4662 }
4663 op1_info = OP1_INFO();
4664 if (!(op1_info & MAY_BE_OBJECT)) {
4665 break;
4666 }
4667 op1_addr = OP1_REG_ADDR();
4668 if (opline->op1_type == IS_VAR) {
4669 if (orig_op1_type != IS_UNKNOWN
4670 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4671 op1_indirect = 1;
4672 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4673 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4674 goto jit_failure;
4675 }
4676 }
4677 }
4678 if (orig_op1_type != IS_UNKNOWN
4679 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4680 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4681 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4682 goto jit_failure;
4683 }
4684 if (opline->op1_type == IS_CV
4685 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4686 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4687 }
4688 } else {
4689 CHECK_OP1_TRACE_TYPE();
4690 }
4691 if (!(op1_info & MAY_BE_OBJECT)) {
4692 break;
4693 }
4694 if (ssa->var_info && ssa->ops) {
4695 if (ssa_op->op1_use >= 0) {
4696 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4697 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4698 ce = op1_ssa->ce;
4699 ce_is_instanceof = op1_ssa->is_instanceof;
4700 }
4701 }
4702 }
4703 if (delayed_fetch_this) {
4704 on_this = 1;
4705 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4706 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4707 } else if (op_array_ssa->ops
4708 && op_array_ssa->vars
4709 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4710 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4711 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4712 }
4713 }
4714 if (!zend_jit_incdec_obj(&ctx, opline, op_array, ssa, ssa_op,
4715 op1_info, op1_addr,
4716 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4717 val_type)) {
4718 goto jit_failure;
4719 }
4720 goto done;
4721 case ZEND_ASSIGN_OBJ_OP:
4722 if (opline->result_type != IS_UNUSED) {
4723 break;
4724 }
4725 if (opline->op2_type != IS_CONST
4726 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4727 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4728 break;
4729 }
4730 if (opline->op1_type == IS_CV
4731 && (opline+1)->op1_type == IS_CV
4732 && (opline+1)->op1.var == opline->op1.var) {
4733 /* skip $a->prop += $a; */
4734 break;
4735 }
4736 if (!zend_jit_supported_binary_op(
4737 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
4738 break;
4739 }
4740 ce = NULL;
4741 ce_is_instanceof = 0;
4742 on_this = delayed_fetch_this = 0;
4743 op1_indirect = 0;
4744 if (opline->op1_type == IS_UNUSED) {
4745 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4746 ce = op_array->scope;
4747 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4748 op1_addr = 0;
4749 on_this = 1;
4750 } else {
4751 if (ssa_op->op1_use >= 0) {
4752 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4753 }
4754 op1_info = OP1_INFO();
4755 if (!(op1_info & MAY_BE_OBJECT)) {
4756 break;
4757 }
4758 op1_addr = OP1_REG_ADDR();
4759 if (opline->op1_type == IS_VAR) {
4760 if (orig_op1_type != IS_UNKNOWN
4761 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4762 op1_indirect = 1;
4763 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4764 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4765 goto jit_failure;
4766 }
4767 }
4768 }
4769 if (orig_op1_type != IS_UNKNOWN
4770 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4771 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4772 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4773 goto jit_failure;
4774 }
4775 if (opline->op1_type == IS_CV
4776 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4777 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4778 }
4779 } else {
4780 CHECK_OP1_TRACE_TYPE();
4781 }
4782 if (!(op1_info & MAY_BE_OBJECT)) {
4783 break;
4784 }
4785 if (ssa->var_info && ssa->ops) {
4786 if (ssa_op->op1_use >= 0) {
4787 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4788 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4789 ce = op1_ssa->ce;
4790 ce_is_instanceof = op1_ssa->is_instanceof;
4791 }
4792 }
4793 }
4794 if (delayed_fetch_this) {
4795 on_this = 1;
4796 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4797 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4798 } else if (op_array_ssa->ops
4799 && op_array_ssa->vars
4800 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4801 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4802 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4803 }
4804 }
4805 op1_data_info = OP1_DATA_INFO();
4806 CHECK_OP1_DATA_TRACE_TYPE();
4807 if (!zend_jit_assign_obj_op(&ctx, opline, op_array, ssa, ssa_op,
4808 op1_info, op1_addr, op1_data_info, OP1_DATA_RANGE(),
4809 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4810 val_type)) {
4811 goto jit_failure;
4812 }
4813 goto done;
4814 case ZEND_ASSIGN_OBJ:
4815 if (opline->op2_type != IS_CONST
4816 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
4817 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
4818 break;
4819 }
4820 ce = NULL;
4821 ce_is_instanceof = 0;
4822 on_this = delayed_fetch_this = 0;
4823 op1_indirect = 0;
4824 if (opline->op1_type == IS_UNUSED) {
4825 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
4826 ce = op_array->scope;
4827 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
4828 op1_addr = 0;
4829 on_this = 1;
4830 } else {
4831 if (ssa_op->op1_use >= 0) {
4832 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
4833 }
4834 op1_info = OP1_INFO();
4835 if (!(op1_info & MAY_BE_OBJECT)) {
4836 break;
4837 }
4838 op1_addr = OP1_REG_ADDR();
4839 if (opline->op1_type == IS_VAR) {
4840 if (orig_op1_type != IS_UNKNOWN
4841 && (orig_op1_type & IS_TRACE_INDIRECT)) {
4842 op1_indirect = 1;
4843 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4844 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4845 goto jit_failure;
4846 }
4847 }
4848 }
4849 if (orig_op1_type != IS_UNKNOWN
4850 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4851 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4852 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4853 goto jit_failure;
4854 }
4855 if (opline->op1_type == IS_CV
4856 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4857 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4858 }
4859 } else {
4860 CHECK_OP1_TRACE_TYPE();
4861 }
4862 if (!(op1_info & MAY_BE_OBJECT)) {
4863 break;
4864 }
4865 if (ssa->var_info && ssa->ops) {
4866 if (ssa_op->op1_use >= 0) {
4867 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
4868 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
4869 ce = op1_ssa->ce;
4870 ce_is_instanceof = op1_ssa->is_instanceof;
4871 }
4872 }
4873 }
4874 if (delayed_fetch_this) {
4875 on_this = 1;
4876 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
4877 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
4878 } else if (op_array_ssa->ops
4879 && op_array_ssa->vars
4880 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
4881 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
4882 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
4883 }
4884 }
4885 op1_data_info = OP1_DATA_INFO();
4886 CHECK_OP1_DATA_TRACE_TYPE();
4887 if (!zend_jit_assign_obj(&ctx, opline, op_array, ssa, ssa_op,
4888 op1_info, op1_addr, op1_data_info,
4889 op1_indirect, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
4890 val_type,
4891 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4892 goto jit_failure;
4893 }
4894 if ((opline+1)->op1_type == IS_CV
4895 && (ssa_op+1)->op1_def >= 0
4896 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4897 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4898 }
4899 goto done;
4900 case ZEND_ASSIGN_DIM:
4901 op1_info = OP1_INFO();
4902 op1_addr = OP1_REG_ADDR();
4903 if (opline->op1_type == IS_CV
4904 && (opline+1)->op1_type == IS_CV
4905 && (opline+1)->op1.var == opline->op1.var) {
4906 /* skip $a[x] = $a; */
4907 break;
4908 }
4909 if (opline->op1_type == IS_VAR) {
4910 if (orig_op1_type != IS_UNKNOWN
4911 && (orig_op1_type & IS_TRACE_INDIRECT)
4912 && opline->result_type == IS_UNUSED) {
4913 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
4914 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
4915 goto jit_failure;
4916 }
4917 } else {
4918 break;
4919 }
4920 }
4921 if (orig_op1_type != IS_UNKNOWN
4922 && (orig_op1_type & IS_TRACE_REFERENCE)) {
4923 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
4924 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
4925 goto jit_failure;
4926 }
4927 if (opline->op1_type == IS_CV
4928 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4929 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4930 }
4931 } else {
4932 CHECK_OP1_TRACE_TYPE();
4933 }
4934 op2_info = OP2_INFO();
4935 CHECK_OP2_TRACE_TYPE();
4936 op1_data_info = OP1_DATA_INFO();
4937 CHECK_OP1_DATA_TRACE_TYPE();
4938 if (!zend_jit_assign_dim(&ctx, opline,
4939 op1_info, op1_addr, op2_info, op1_data_info, val_type,
4940 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
4941 goto jit_failure;
4942 }
4943 if ((opline+1)->op1_type == IS_CV
4944 && (ssa_op+1)->op1_def >= 0
4945 && ssa->vars[(ssa_op+1)->op1_def].alias == NO_ALIAS) {
4946 ssa->var_info[(ssa_op+1)->op1_def].guarded_reference = ssa->var_info[(ssa_op+1)->op1_use].guarded_reference;
4947 }
4948 goto done;
4949 case ZEND_ASSIGN:
4950 if (opline->op1_type != IS_CV) {
4951 break;
4952 }
4953 op2_addr = OP2_REG_ADDR();
4954 op2_info = OP2_INFO();
4955 zend_jit_addr ref_addr = 0;
4956
4957 if (ssa_op->op2_def < 0 || (Z_MODE(op2_addr) == IS_REG && ssa->vars[ssa_op->op2_def].no_val)) {
4958 op2_def_addr = op2_addr;
4959 } else {
4960 op2_def_addr = OP2_DEF_REG_ADDR();
4961 }
4962 CHECK_OP2_TRACE_TYPE();
4963 op1_info = OP1_INFO();
4964 op1_def_info = OP1_DEF_INFO();
4965 if (op1_type != IS_UNKNOWN && (op1_info & MAY_BE_GUARD)) {
4966 if (op1_type < IS_STRING
4967 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (op1_def_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
4968 if (!zend_jit_scalar_type_guard(&ctx, opline, opline->op1.var)) {
4969 goto jit_failure;
4970 }
4971 op1_info &= ~(MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD);
4972 } else {
4973 CHECK_OP1_TRACE_TYPE();
4974 }
4975 }
4976 op1_addr = OP1_REG_ADDR();
4977 op1_def_addr = OP1_DEF_REG_ADDR();
4978 if (Z_MODE(op1_def_addr) != IS_REG &&
4979 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) !=
4980 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var))) {
4981 /* type may be not set */
4982 op1_info |= MAY_BE_NULL;
4983 }
4984 if (orig_op1_type != IS_UNKNOWN) {
4985 if (orig_op1_type & IS_TRACE_REFERENCE) {
4986 if (!zend_jit_guard_reference(&ctx, opline, &op1_addr, &ref_addr,
4987 !ssa->var_info[ssa_op->op1_use].guarded_reference)) {
4988 goto jit_failure;
4989 }
4990 op1_info &= ~MAY_BE_REF;
4991 if (opline->op1_type == IS_CV
4992 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
4993 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
4994 }
4995 if (opline->result_type == IS_UNUSED) {
4996 res_addr = 0;
4997 } else {
4998 res_addr = RES_REG_ADDR();
4999 if (Z_MODE(res_addr) != IS_REG
5000 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
5001 send_result = 1;
5002 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5003 if (!zend_jit_reuse_ip(&ctx)) {
5004 goto jit_failure;
5005 }
5006 }
5007 }
5008 op1_def_addr = op1_addr;
5009 op1_def_info &= ~MAY_BE_REF;
5010 } else if (op1_info & MAY_BE_REF) {
5011 if (!zend_jit_noref_guard(&ctx, opline, op1_addr)) {
5012 goto jit_failure;
5013 }
5014 op1_info &= ~MAY_BE_REF;
5015 op1_def_info &= ~MAY_BE_REF;
5016 }
5017 }
5018 if (opline->result_type == IS_UNUSED) {
5019 res_addr = 0;
5020 res_info = -1;
5021 } else {
5022 res_addr = RES_REG_ADDR();
5023 res_info = RES_INFO();
5024 if (Z_MODE(res_addr) != IS_REG
5025 && zend_jit_trace_next_is_send_result(opline, p, frame)) {
5026 send_result = 1;
5027 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
5028 if (!zend_jit_reuse_ip(&ctx)) {
5029 goto jit_failure;
5030 }
5031 }
5032 }
5033 if (!zend_jit_assign(&ctx, opline,
5034 op1_info, op1_addr,
5035 op1_def_info, op1_def_addr,
5036 op2_info, op2_addr, op2_def_addr,
5037 res_info, res_addr,
5038 ref_addr,
5039 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) {
5040 goto jit_failure;
5041 }
5042 if (ssa_op->op2_def > 0
5043 && Z_MODE(op2_addr) == IS_REG
5044 && ssa->vars[ssa_op->op2_def].no_val) {
5045 uint8_t type = (op2_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5046 uint32_t var_num = EX_VAR_TO_NUM(opline->op2.var);
5047
5048 if (STACK_MEM_TYPE(stack, var_num) != type
5049 && ssa->vars[ssa_op->op2_def].use_chain < 0
5050 && !ssa->vars[ssa_op->op2_def].phi_use_chain) {
5051 if (!zend_jit_store_type(&ctx, var_num, type)) {
5052 return 0;
5053 }
5054 SET_STACK_TYPE(stack, var_num, type, 1);
5055 }
5056 }
5057 if (opline->op2_type == IS_CV
5058 && ssa_op->op2_def >= 0
5059 && ssa->vars[ssa_op->op2_def].alias == NO_ALIAS) {
5060 ssa->var_info[ssa_op->op2_def].guarded_reference = ssa->var_info[ssa_op->op2_use].guarded_reference;
5061 }
5062 goto done;
5063 case ZEND_CAST:
5064 if (opline->extended_value != op1_type) {
5065 break;
5066 }
5067 ZEND_FALLTHROUGH;
5068 case ZEND_QM_ASSIGN:
5069 op1_addr = OP1_REG_ADDR();
5070 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5071 op1_def_addr = op1_addr;
5072 } else {
5073 op1_def_addr = OP1_DEF_REG_ADDR();
5074 }
5075 op1_info = OP1_INFO();
5076 CHECK_OP1_TRACE_TYPE();
5077 res_info = RES_INFO();
5078 res_use_info = zend_jit_trace_type_to_info(
5079 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)));
5080 if (opline->result_type == IS_CV) {
5081 res_use_info &= (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE);
5082 }
5083 res_addr = RES_REG_ADDR();
5084 if (Z_MODE(res_addr) != IS_REG &&
5085 STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) !=
5086 STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var))) {
5087 /* type may be not set */
5088 res_use_info |= MAY_BE_NULL;
5089 }
5090 if (!zend_jit_qm_assign(&ctx, opline,
5091 op1_info, op1_addr, op1_def_addr,
5092 res_use_info, res_info, res_addr)) {
5093 goto jit_failure;
5094 }
5095 if (ssa_op->op1_def > 0
5096 && Z_MODE(op1_addr) == IS_REG
5097 && ssa->vars[ssa_op->op1_def].no_val) {
5098 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5099 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5100
5101 if (STACK_MEM_TYPE(stack, var_num) != type
5102 && ssa->vars[ssa_op->op1_def].use_chain < 0
5103 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5104 if (!zend_jit_store_type(&ctx, var_num, type)) {
5105 return 0;
5106 }
5107 SET_STACK_TYPE(stack, var_num, type, 1);
5108 }
5109 }
5110 if (opline->op1_type == IS_CV
5111 && ssa_op->op1_def >= 0
5112 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5113 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5114 }
5115 goto done;
5116 case ZEND_INIT_FCALL:
5117 case ZEND_INIT_FCALL_BY_NAME:
5118 case ZEND_INIT_NS_FCALL_BY_NAME:
5119 frame_flags = TRACE_FRAME_MASK_NESTED;
5120 if (!zend_jit_init_fcall(&ctx, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
5121 goto jit_failure;
5122 }
5123 goto done;
5124 case ZEND_SEND_VAL:
5125 case ZEND_SEND_VAL_EX:
5126 if (opline->op2_type == IS_CONST) {
5127 /* Named parameters not supported in JIT */
5128 break;
5129 }
5130 if (opline->opcode == ZEND_SEND_VAL_EX
5131 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5132 break;
5133 }
5134 op1_info = OP1_INFO();
5135 CHECK_OP1_TRACE_TYPE();
5136 if (!zend_jit_send_val(&ctx, opline,
5137 op1_info, OP1_REG_ADDR())) {
5138 goto jit_failure;
5139 }
5140 if (frame->call && frame->call->func) {
5141 if (opline->op1_type == IS_CONST) {
5142 zend_jit_trace_send_type(opline, frame->call, Z_TYPE_P(RT_CONSTANT(opline, opline->op1)));
5143 } else if (op1_type != IS_UNKNOWN) {
5144 if (op1_type == IS_UNDEF) {
5145 op1_type = IS_NULL;
5146 }
5147 zend_jit_trace_send_type(opline, frame->call, op1_type);
5148 }
5149 }
5150 goto done;
5151 case ZEND_SEND_REF:
5152 if (opline->op2_type == IS_CONST) {
5153 /* Named parameters not supported in JIT */
5154 break;
5155 }
5156 op1_info = OP1_INFO();
5157 if (!zend_jit_send_ref(&ctx, opline, op_array,
5158 op1_info, 0)) {
5159 goto jit_failure;
5160 }
5161 if (opline->op1_type == IS_CV
5162 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5163 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5164 }
5165 goto done;
5166 case ZEND_SEND_VAR:
5167 case ZEND_SEND_VAR_EX:
5168 case ZEND_SEND_VAR_NO_REF:
5169 case ZEND_SEND_VAR_NO_REF_EX:
5170 case ZEND_SEND_FUNC_ARG:
5171 if (opline->op2_type == IS_CONST) {
5172 /* Named parameters not supported in JIT */
5173 break;
5174 }
5175 if ((opline->opcode == ZEND_SEND_VAR_EX
5176 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
5177 && opline->op2.num > MAX_ARG_FLAG_NUM) {
5178 break;
5179 }
5180 op1_addr = OP1_REG_ADDR();
5181 if (ssa_op->op1_def < 0 || (Z_MODE(op1_addr) == IS_REG && ssa->vars[ssa_op->op1_def].no_val)) {
5182 op1_def_addr = op1_addr;
5183 } else {
5184 op1_def_addr = OP1_DEF_REG_ADDR();
5185 }
5186 op1_info = OP1_INFO();
5187 CHECK_OP1_TRACE_TYPE();
5188 if (!zend_jit_send_var(&ctx, opline, op_array,
5189 op1_info, op1_addr, op1_def_addr)) {
5190 goto jit_failure;
5191 }
5192 if (ssa_op->op1_def > 0
5193 && Z_MODE(op1_addr) == IS_REG
5194 && ssa->vars[ssa_op->op1_def].no_val) {
5195 uint8_t type = (op1_info & MAY_BE_LONG) ? IS_LONG : IS_DOUBLE;
5196 uint32_t var_num = EX_VAR_TO_NUM(opline->op1.var);
5197
5198 if (STACK_MEM_TYPE(stack, var_num) != type
5199 && ssa->vars[ssa_op->op1_def].use_chain < 0
5200 && !ssa->vars[ssa_op->op1_def].phi_use_chain) {
5201 if (!zend_jit_store_type(&ctx, var_num, type)) {
5202 return 0;
5203 }
5204 SET_STACK_TYPE(stack, var_num, type, 1);
5205 }
5206 }
5207 if (opline->op1_type == IS_CV
5208 && ssa_op->op1_def >= 0
5209 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5210 ssa->var_info[ssa_op->op1_def].guarded_reference = ssa->var_info[ssa_op->op1_use].guarded_reference;
5211 }
5212 if (frame->call && frame->call->func) {
5213 if ((opline->opcode == ZEND_SEND_VAR_EX
5214 || opline->opcode == ZEND_SEND_FUNC_ARG)
5215 && ARG_SHOULD_BE_SENT_BY_REF(frame->call->func, opline->op2.num)) {
5216 goto done;
5217 }
5218 if (op1_type != IS_UNKNOWN) {
5219 if (op1_type == IS_UNDEF) {
5220 op1_type = IS_NULL;
5221 }
5222 zend_jit_trace_send_type(opline, frame->call, op1_type);
5223 }
5224 }
5225 goto done;
5226 case ZEND_CHECK_FUNC_ARG:
5227 if (!JIT_G(current_frame)
5228 || !JIT_G(current_frame)->call
5229 || !JIT_G(current_frame)->call->func) {
5230 break;
5231 }
5232 if (opline->op2_type == IS_CONST
5233 || opline->op2.num > MAX_ARG_FLAG_NUM) {
5234 /* Named parameters not supported in JIT */
5235 TRACE_FRAME_SET_LAST_SEND_UNKNOWN(JIT_G(current_frame)->call);
5236 break;
5237 }
5238 if (!zend_jit_check_func_arg(&ctx, opline)) {
5239 goto jit_failure;
5240 }
5241 goto done;
5242 case ZEND_CHECK_UNDEF_ARGS:
5243 if (JIT_G(current_frame)
5244 && JIT_G(current_frame)->call) {
5245 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
5246 }
5247 if (!zend_jit_check_undef_args(&ctx, opline)) {
5248 goto jit_failure;
5249 }
5250 goto done;
5251 case ZEND_DO_UCALL:
5252 case ZEND_DO_ICALL:
5253 case ZEND_DO_FCALL_BY_NAME:
5254 case ZEND_DO_FCALL:
5255 if (!zend_jit_do_fcall(&ctx, opline, op_array, op_array_ssa, frame->call_level, -1, p + 1)) {
5256 goto jit_failure;
5257 }
5258 goto done;
5259 case ZEND_IS_EQUAL:
5260 case ZEND_IS_NOT_EQUAL:
5261 case ZEND_IS_SMALLER:
5262 case ZEND_IS_SMALLER_OR_EQUAL:
5263 case ZEND_CASE:
5264 op1_info = OP1_INFO();
5265 op2_info = OP2_INFO();
5266 skip_comparison =
5267 ssa_op != ssa->ops &&
5268 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5269 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5270 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5271 CHECK_OP1_TRACE_TYPE();
5272 CHECK_OP2_TRACE_TYPE();
5273 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5274 bool exit_if_true = 0;
5275 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5276 uint32_t exit_point;
5277
5278 if (ra) {
5279 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5280 }
5281 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5282 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5283 if (!exit_addr) {
5284 goto jit_failure;
5285 }
5286 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5287 if (!zend_jit_cmp(&ctx, opline,
5288 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5289 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5290 RES_REG_ADDR(),
5291 zend_may_throw(opline, ssa_op, op_array, ssa),
5292 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5293 goto jit_failure;
5294 }
5295 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5296 } else {
5297 smart_branch_opcode = 0;
5298 exit_addr = NULL;
5299 if (!zend_jit_cmp(&ctx, opline,
5300 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5301 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5302 RES_REG_ADDR(),
5303 zend_may_throw(opline, ssa_op, op_array, ssa),
5304 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5305 goto jit_failure;
5306 }
5307 }
5308 goto done;
5309 case ZEND_IS_IDENTICAL:
5310 case ZEND_IS_NOT_IDENTICAL:
5311 case ZEND_CASE_STRICT:
5312 op1_info = OP1_INFO();
5313 op2_info = OP2_INFO();
5314 skip_comparison =
5315 ssa_op != ssa->ops &&
5316 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5317 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG &&
5318 zend_jit_may_skip_comparison(opline, ssa_op, ssa, ssa_opcodes, op_array);
5319 CHECK_OP1_TRACE_TYPE();
5320 CHECK_OP2_TRACE_TYPE();
5321 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5322 bool exit_if_true = 0;
5323 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5324 uint32_t exit_point;
5325
5326 if (ra) {
5327 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5328 }
5329 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5330 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5331 if (!exit_addr) {
5332 goto jit_failure;
5333 }
5334 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
5335 exit_if_true = !exit_if_true;
5336 }
5337 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5338 if (!zend_jit_identical(&ctx, opline,
5339 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5340 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5341 RES_REG_ADDR(),
5342 zend_may_throw(opline, ssa_op, op_array, ssa),
5343 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5344 goto jit_failure;
5345 }
5346 zend_jit_trace_update_condition_ranges(opline, ssa_op, op_array, ssa, exit_if_true);
5347 } else {
5348 smart_branch_opcode = 0;
5349 exit_addr = NULL;
5350 if (!zend_jit_identical(&ctx, opline,
5351 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
5352 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
5353 RES_REG_ADDR(),
5354 zend_may_throw(opline, ssa_op, op_array, ssa),
5355 smart_branch_opcode, -1, -1, exit_addr, skip_comparison)) {
5356 goto jit_failure;
5357 }
5358 }
5359 goto done;
5360 case ZEND_DEFINED:
5361 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5362 bool exit_if_true = 0;
5363 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5364 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5365
5366 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5367 if (!exit_addr) {
5368 goto jit_failure;
5369 }
5370 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5371 } else {
5372 smart_branch_opcode = 0;
5373 exit_addr = NULL;
5374 }
5375 if (!zend_jit_defined(&ctx, opline, smart_branch_opcode, -1, -1, exit_addr)) {
5376 goto jit_failure;
5377 }
5378 goto done;
5379 case ZEND_TYPE_CHECK:
5380 if (opline->extended_value == MAY_BE_RESOURCE) {
5381 // TODO: support for is_resource() ???
5382 break;
5383 }
5384 op1_info = OP1_INFO();
5385 CHECK_OP1_TRACE_TYPE();
5386 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5387 bool exit_if_true = 0;
5388 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5389 uint32_t exit_point;
5390
5391 if (ra) {
5392 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5393 }
5394 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5395 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5396 if (!exit_addr) {
5397 goto jit_failure;
5398 }
5399 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5400 } else {
5401 smart_branch_opcode = 0;
5402 exit_addr = NULL;
5403 }
5404 if (!zend_jit_type_check(&ctx, opline, op1_info, smart_branch_opcode, -1, -1, exit_addr)) {
5405 goto jit_failure;
5406 }
5407 goto done;
5408 case ZEND_RETURN:
5409 op1_info = OP1_INFO();
5410 CHECK_OP1_TRACE_TYPE();
5411 if (opline->op1_type == IS_CONST) {
5412 res_type = Z_TYPE_P(RT_CONSTANT(opline, opline->op1));
5413 } else if (op1_type != IS_UNKNOWN) {
5414 res_type = op1_type;
5415 }
5416 if (op_array->type == ZEND_EVAL_CODE
5417 // TODO: support for top-level code
5418 || !op_array->function_name
5419 // TODO: support for IS_UNDEF ???
5420 || (op1_info & MAY_BE_UNDEF)) {
5421 if (!zend_jit_trace_handler(&ctx, op_array, opline, zend_may_throw(opline, ssa_op, op_array, ssa), p + 1)) {
5422 goto jit_failure;
5423 }
5424 } else {
5425 int j;
5426 int may_throw = 0;
5427 bool left_frame = 0;
5428
5429 if (!zend_jit_return(&ctx, opline, op_array,
5430 op1_info, OP1_REG_ADDR())) {
5431 goto jit_failure;
5432 }
5433 if (op_array->last_var > 100) {
5434 /* To many CVs to unroll */
5435 if (!zend_jit_free_cvs(&ctx)) {
5436 goto jit_failure;
5437 }
5438 left_frame = 1;
5439 }
5440 if (!left_frame) {
5441 for (j = 0 ; j < op_array->last_var; j++) {
5442 uint32_t info;
5443 uint8_t type;
5444
5445 info = zend_ssa_cv_info(op_array, op_array_ssa, j);
5446 type = STACK_TYPE(stack, j);
5447 info = zend_jit_trace_type_to_info_ex(type, info);
5448 if (opline->op1_type == IS_CV
5449 && EX_VAR_TO_NUM(opline->op1.var) == j
5450 && !(op1_info & (MAY_BE_REF|MAY_BE_OBJECT))) {
5451 if (JIT_G(current_frame)
5452 && TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
5453 continue;
5454 } else {
5455 info |= MAY_BE_NULL;
5456 }
5457 }
5458 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
5459 if (!left_frame) {
5460 left_frame = 1;
5461 if (!zend_jit_leave_frame(&ctx)) {
5462 goto jit_failure;
5463 }
5464 }
5465 if (!zend_jit_free_cv(&ctx, info, j)) {
5466 goto jit_failure;
5467 }
5468 if (info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_RESOURCE)) {
5469 if (info & MAY_BE_RC1) {
5470 may_throw = 1;
5471 }
5472 }
5473 }
5474 }
5475 }
5476 if (!zend_jit_leave_func(&ctx, op_array, opline, op1_info, left_frame,
5477 p + 1, &zend_jit_traces[ZEND_JIT_TRACE_NUM],
5478 (op_array_ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, may_throw)) {
5479 goto jit_failure;
5480 }
5481 }
5482 goto done;
5483 case ZEND_BOOL:
5484 case ZEND_BOOL_NOT:
5485 op1_info = OP1_INFO();
5486 CHECK_OP1_TRACE_TYPE();
5487 if (!zend_jit_bool_jmpznz(&ctx, opline,
5488 op1_info, OP1_REG_ADDR(), RES_REG_ADDR(),
5489 -1, -1,
5490 zend_may_throw(opline, ssa_op, op_array, ssa),
5491 opline->opcode, NULL)) {
5492 goto jit_failure;
5493 }
5494 goto done;
5495 case ZEND_JMPZ:
5496 case ZEND_JMPNZ:
5497 case ZEND_JMPZ_EX:
5498 case ZEND_JMPNZ_EX:
5499 op1_info = OP1_INFO();
5500 CHECK_OP1_TRACE_TYPE();
5501 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
5502 const zend_op *exit_opline = NULL;
5503 uint32_t exit_point;
5504
5505 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5506 /* taken branch */
5507 if (opline->opcode == ZEND_JMPNZ_EX) {
5508 smart_branch_opcode = ZEND_JMPZ_EX;
5509 } else if (opline->opcode == ZEND_JMPZ_EX) {
5510 smart_branch_opcode = ZEND_JMPNZ_EX;
5511 } else if (opline->opcode == ZEND_JMPNZ) {
5512 smart_branch_opcode = ZEND_JMPZ;
5513 } else {
5514 smart_branch_opcode = ZEND_JMPNZ;
5515 }
5516 exit_opline = opline + 1;
5517 } else if ((p+1)->opline == opline + 1) {
5518 /* not taken branch */
5519 smart_branch_opcode = opline->opcode;
5520 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5521 } else {
5522 ZEND_UNREACHABLE();
5523 }
5524 if (ra) {
5525 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5526 }
5527 if (!(op1_info & MAY_BE_GUARD)
5528 && has_concrete_type(op1_info)
5529 && concrete_type(op1_info) <= IS_TRUE) {
5530 /* unconditional branch */
5531 exit_addr = NULL;
5532 } else if (opline->result_type == IS_TMP_VAR) {
5533 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5534 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
5535
5536 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
5537 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5538 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
5539 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5540 if (!exit_addr) {
5541 goto jit_failure;
5542 }
5543 } else {
5544 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5545 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5546 if (!exit_addr) {
5547 goto jit_failure;
5548 }
5549 }
5550 } else {
5551 ZEND_UNREACHABLE();
5552 }
5553 if (opline->result_type == IS_UNDEF) {
5554 res_addr = 0;
5555 } else {
5556 res_addr = RES_REG_ADDR();
5557 }
5558 if (!zend_jit_bool_jmpznz(&ctx, opline,
5559 op1_info, OP1_REG_ADDR(), res_addr,
5560 -1, -1,
5561 zend_may_throw(opline, ssa_op, op_array, ssa),
5562 smart_branch_opcode, exit_addr)) {
5563 goto jit_failure;
5564 }
5565 goto done;
5566 case ZEND_JMP_FRAMELESS:
5567 op1_info = OP1_INFO();
5568 ZEND_ASSERT((p+1)->op == ZEND_JIT_TRACE_VM);
5569 const zend_op *exit_opline = NULL;
5570 uint32_t exit_point;
5571 zend_jmp_fl_result guard;
5572
5573 if ((p+1)->opline == OP_JMP_ADDR(opline, opline->op2)) {
5574 /* taken branch */
5575 guard = ZEND_JMP_FL_HIT;
5576 exit_opline = opline + 1;
5577 } else if ((p+1)->opline == opline + 1) {
5578 /* not taken branch */
5579 guard = ZEND_JMP_FL_MISS;
5580 exit_opline = OP_JMP_ADDR(opline, opline->op2);
5581 } else {
5582 ZEND_UNREACHABLE();
5583 }
5584 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5585 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5586 if (!exit_addr) {
5587 goto jit_failure;
5588 }
5589 if (!zend_jit_jmp_frameless(&ctx, opline, exit_addr, guard)) {
5590 goto jit_failure;
5591 }
5592 goto done;
5593 case ZEND_ISSET_ISEMPTY_CV:
5594 if ((opline->extended_value & ZEND_ISEMPTY)) {
5595 // TODO: support for empty() ???
5596 break;
5597 }
5598 op1_info = OP1_INFO();
5599 op1_addr = OP1_REG_ADDR();
5600 if (orig_op1_type != IS_UNKNOWN
5601 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5602 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5603 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5604 goto jit_failure;
5605 }
5606 if (opline->op1_type == IS_CV
5607 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5608 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5609 }
5610 } else {
5611 CHECK_OP1_TRACE_TYPE();
5612 }
5613 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5614 bool exit_if_true = 0;
5615 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5616 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5617
5618 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5619 if (!exit_addr) {
5620 goto jit_failure;
5621 }
5622 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5623 } else {
5624 smart_branch_opcode = 0;
5625 exit_addr = NULL;
5626 }
5627 if (!zend_jit_isset_isempty_cv(&ctx, opline,
5628 op1_info, op1_addr,
5629 smart_branch_opcode, -1, -1, exit_addr)) {
5630 goto jit_failure;
5631 }
5632 goto done;
5633 case ZEND_IN_ARRAY:
5634 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
5635 break;
5636 }
5637 op1_info = OP1_INFO();
5638 op1_addr = OP1_REG_ADDR();
5639 CHECK_OP1_TRACE_TYPE();
5640 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
5641 break;
5642 }
5643 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5644 bool exit_if_true = 0;
5645 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5646 uint32_t exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5647
5648 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5649 if (!exit_addr) {
5650 goto jit_failure;
5651 }
5652 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5653 } else {
5654 smart_branch_opcode = 0;
5655 exit_addr = NULL;
5656 }
5657 if (!zend_jit_in_array(&ctx, opline,
5658 op1_info, op1_addr,
5659 smart_branch_opcode, -1, -1, exit_addr)) {
5660 goto jit_failure;
5661 }
5662 goto done;
5663 case ZEND_FETCH_DIM_FUNC_ARG:
5664 if (!JIT_G(current_frame)
5665 || !JIT_G(current_frame)->call
5666 || !JIT_G(current_frame)->call->func
5667 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5668 break;
5669 }
5670 ZEND_FALLTHROUGH;
5671 case ZEND_FETCH_DIM_R:
5672 case ZEND_FETCH_DIM_IS:
5673 case ZEND_FETCH_LIST_R:
5674 op1_info = OP1_INFO();
5675 op1_addr = OP1_REG_ADDR();
5676 if (orig_op1_type != IS_UNKNOWN
5677 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5678 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5679 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5680 goto jit_failure;
5681 }
5682 if (opline->op1_type == IS_CV
5683 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5684 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5685 if (ssa_op->op1_def >= 0) {
5686 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5687 }
5688 }
5689 } else {
5690 CHECK_OP1_TRACE_TYPE();
5691 }
5692 op2_info = OP2_INFO();
5693 CHECK_OP2_TRACE_TYPE();
5694 res_info = RES_INFO();
5695 avoid_refcounting =
5696 ssa_op->op1_use >= 0 &&
5697 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5698 if (op1_info & MAY_BE_PACKED_GUARD) {
5699 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5700 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5701 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5702 && MAY_BE_PACKED(op1_info)
5703 && MAY_BE_HASH(op1_info)
5704 && orig_op1_type != IS_UNKNOWN) {
5705 op1_info |= MAY_BE_PACKED_GUARD;
5706 if (orig_op1_type & IS_TRACE_PACKED) {
5707 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5708 if (op1_type != IS_UNKNOWN) {
5709 ssa->var_info[ssa_op->op1_use].type &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5710 }
5711 } else {
5712 op1_info &= ~MAY_BE_ARRAY_PACKED;
5713 if (op1_type != IS_UNKNOWN) {
5714 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_ARRAY_PACKED;
5715 }
5716 }
5717 }
5718 if (!zend_jit_fetch_dim_read(&ctx, opline, ssa, ssa_op,
5719 op1_info, op1_addr, avoid_refcounting,
5720 op2_info, res_info, RES_REG_ADDR(), val_type)) {
5721 goto jit_failure;
5722 }
5723 if (ssa_op->op1_def >= 0 && op1_type != IS_UNKNOWN) {
5724 ssa->var_info[ssa_op->op1_def].type = ssa->var_info[ssa_op->op1_use].type;
5725 }
5726 goto done;
5727 case ZEND_FETCH_DIM_W:
5728 case ZEND_FETCH_DIM_RW:
5729 // case ZEND_FETCH_DIM_UNSET:
5730 case ZEND_FETCH_LIST_W:
5731 if (opline->op1_type != IS_CV
5732 && (orig_op1_type == IS_UNKNOWN
5733 || !(orig_op1_type & IS_TRACE_INDIRECT))) {
5734 break;
5735 }
5736 op1_info = OP1_INFO();
5737 op1_addr = OP1_REG_ADDR();
5738 if (opline->op1_type == IS_VAR) {
5739 if (orig_op1_type != IS_UNKNOWN
5740 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5741 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
5742 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5743 goto jit_failure;
5744 }
5745 } else {
5746 break;
5747 }
5748 }
5749 if (orig_op1_type != IS_UNKNOWN
5750 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5751 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5752 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5753 goto jit_failure;
5754 }
5755 if (opline->op1_type == IS_CV
5756 && ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5757 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5758 }
5759 } else {
5760 CHECK_OP1_TRACE_TYPE();
5761 }
5762 op2_info = OP2_INFO();
5763 CHECK_OP2_TRACE_TYPE();
5764 op1_def_info = OP1_DEF_INFO();
5765 if (!zend_jit_fetch_dim(&ctx, opline,
5766 op1_info, op1_addr, op2_info, RES_REG_ADDR(), val_type)) {
5767 goto jit_failure;
5768 }
5769 if (ssa_op->result_def > 0
5770 && (opline->opcode == ZEND_FETCH_DIM_W || opline->opcode == ZEND_FETCH_LIST_W)
5771 && !(op1_info & (MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE|MAY_BE_STRING|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))
5772 && !(op2_info & (MAY_BE_UNDEF|MAY_BE_RESOURCE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
5773 ssa->var_info[ssa_op->result_def].indirect_reference = 1;
5774 }
5775 goto done;
5776 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
5777 if ((opline->extended_value & ZEND_ISEMPTY)) {
5778 // TODO: support for empty() ???
5779 break;
5780 }
5781 op1_info = OP1_INFO();
5782 op1_addr = OP1_REG_ADDR();
5783 if (orig_op1_type != IS_UNKNOWN
5784 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5785 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5786 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5787 goto jit_failure;
5788 }
5789 if (opline->op1_type == IS_CV
5790 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5791 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
5792 }
5793 } else {
5794 CHECK_OP1_TRACE_TYPE();
5795 }
5796 op2_info = OP2_INFO();
5797 CHECK_OP2_TRACE_TYPE();
5798 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
5799 bool exit_if_true = 0;
5800 const zend_op *exit_opline = zend_jit_trace_get_exit_opline(p + 1, opline + 1, &exit_if_true);
5801 uint32_t exit_point;
5802
5803 if (ra) {
5804 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
5805 }
5806 if (ssa_op->op1_use >= 0
5807 && ssa->var_info[ssa_op->op1_use].avoid_refcounting) {
5808 /* Temporary reset ZREG_ZVAL_TRY_ADDREF */
5809 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
5810 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
5811
5812 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
5813 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5814 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
5815 } else {
5816 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
5817 }
5818 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
5819 if (!exit_addr) {
5820 goto jit_failure;
5821 }
5822 smart_branch_opcode = exit_if_true ? ZEND_JMPNZ : ZEND_JMPZ;
5823 } else {
5824 smart_branch_opcode = 0;
5825 exit_addr = NULL;
5826 }
5827 avoid_refcounting =
5828 ssa_op->op1_use >= 0 &&
5829 ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5830 if (op1_info & MAY_BE_PACKED_GUARD) {
5831 ssa->var_info[ssa_op->op1_use].type &= ~MAY_BE_PACKED_GUARD;
5832 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG
5833 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
5834 && MAY_BE_PACKED(op1_info)
5835 && MAY_BE_HASH(op1_info)
5836 && orig_op1_type != IS_UNKNOWN) {
5837 op1_info |= MAY_BE_PACKED_GUARD;
5838 if (orig_op1_type & IS_TRACE_PACKED) {
5839 op1_info &= ~(MAY_BE_ARRAY_NUMERIC_HASH|MAY_BE_ARRAY_STRING_HASH);
5840 } else {
5841 op1_info &= ~MAY_BE_ARRAY_PACKED;
5842 }
5843 }
5844 if (!zend_jit_isset_isempty_dim(&ctx, opline,
5845 op1_info, op1_addr, avoid_refcounting,
5846 op2_info, val_type,
5847 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info),
5848 smart_branch_opcode, -1, -1,
5849 exit_addr)) {
5850 goto jit_failure;
5851 }
5852 goto done;
5853 case ZEND_FETCH_OBJ_FUNC_ARG:
5854 if (!JIT_G(current_frame)
5855 || !JIT_G(current_frame)->call
5856 || !JIT_G(current_frame)->call->func
5857 || !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
5858 break;
5859 }
5860 ZEND_FALLTHROUGH;
5861 case ZEND_FETCH_OBJ_R:
5862 case ZEND_FETCH_OBJ_IS:
5863 case ZEND_FETCH_OBJ_W:
5864 on_this = delayed_fetch_this = 0;
5865 avoid_refcounting = 0;
5866 if (opline->op2_type != IS_CONST
5867 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
5868 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
5869 break;
5870 }
5871 ce = NULL;
5872 ce_is_instanceof = 0;
5873 op1_indirect = 0;
5874 if (opline->op1_type == IS_UNUSED) {
5875 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
5876 ce = op_array->scope;
5877 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
5878 op1_addr = 0;
5879 on_this = 1;
5880 } else {
5881 op1_info = OP1_INFO();
5882 if (!(op1_info & MAY_BE_OBJECT)) {
5883 break;
5884 }
5885 op1_addr = OP1_REG_ADDR();
5886 if (opline->op1_type == IS_VAR
5887 && opline->opcode == ZEND_FETCH_OBJ_W) {
5888 if (orig_op1_type != IS_UNKNOWN
5889 && (orig_op1_type & IS_TRACE_INDIRECT)) {
5890 op1_indirect = 1;
5891 if (!zend_jit_fetch_indirect_var(&ctx, opline, orig_op1_type,
5892 &op1_info, &op1_addr, !ssa->var_info[ssa_op->op1_use].indirect_reference)) {
5893 goto jit_failure;
5894 }
5895 }
5896 }
5897 if (orig_op1_type != IS_UNKNOWN
5898 && (orig_op1_type & IS_TRACE_REFERENCE)) {
5899 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
5900 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
5901 goto jit_failure;
5902 }
5903 if (opline->op1_type == IS_CV
5904 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
5905 ssa->var_info[ssa_op->op1_def >= 0 ? ssa_op->op1_def : ssa_op->op1_use].guarded_reference = 1;
5906 }
5907 } else {
5908 CHECK_OP1_TRACE_TYPE();
5909 }
5910 if (!(op1_info & MAY_BE_OBJECT)) {
5911 break;
5912 }
5913 if (ssa->var_info && ssa->ops) {
5914 if (ssa_op->op1_use >= 0) {
5915 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
5916 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
5917 ce = op1_ssa->ce;
5918 ce_is_instanceof = op1_ssa->is_instanceof;
5919 }
5920 }
5921 }
5922 if (ssa_op->op1_use >= 0) {
5923 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
5924 avoid_refcounting = ssa->var_info[ssa_op->op1_use].avoid_refcounting;
5925 }
5926 if (delayed_fetch_this) {
5927 on_this = 1;
5928 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
5929 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
5930 } else if (op_array_ssa->ops
5931 && op_array_ssa->vars
5932 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
5933 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
5934 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
5935 }
5936 }
5937 if (!zend_jit_fetch_obj(&ctx, opline, op_array, ssa, ssa_op,
5938 op1_info, op1_addr, op1_indirect, ce, ce_is_instanceof,
5939 on_this, delayed_fetch_this, avoid_refcounting, op1_ce, val_type,
5940 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, MAY_BE_STRING))) {
5941 goto jit_failure;
5942 }
5943 goto done;
5944 case ZEND_BIND_GLOBAL:
5945 orig_opline = opline;
5946 orig_ssa_op = ssa_op;
5947 while (1) {
5948 if (!ssa->ops || !ssa->var_info) {
5949 op1_info = MAY_BE_ANY|MAY_BE_REF;
5950 } else {
5951 op1_info = OP1_INFO();
5952 }
5953 if (ssa->vars[ssa_op->op1_def].alias == NO_ALIAS) {
5954 ssa->var_info[ssa_op->op1_def].guarded_reference = 1;
5955 }
5956 if (!zend_jit_bind_global(&ctx, opline, op1_info)) {
5957 goto jit_failure;
5958 }
5959 if ((opline+1)->opcode == ZEND_BIND_GLOBAL) {
5960 opline++;
5961 ssa_op++;
5962 } else {
5963 break;
5964 }
5965 }
5966 opline = orig_opline;
5967 ssa_op = orig_ssa_op;
5968 goto done;
5969 case ZEND_RECV:
5970 if (!zend_jit_recv(&ctx, opline, op_array)) {
5971 goto jit_failure;
5972 }
5973 goto done;
5974 case ZEND_RECV_INIT:
5975 orig_opline = opline;
5976 orig_ssa_op = ssa_op;
5977 while (1) {
5978 if (!zend_jit_recv_init(&ctx, opline, op_array,
5979 (opline + 1)->opcode != ZEND_RECV_INIT,
5980 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5981 goto jit_failure;
5982 }
5983 if ((opline+1)->opcode == ZEND_RECV_INIT) {
5984 opline++;
5985 ssa_op++;
5986 } else {
5987 break;
5988 }
5989 }
5990 opline = orig_opline;
5991 ssa_op = orig_ssa_op;
5992 goto done;
5993 case ZEND_FREE:
5994 case ZEND_FE_FREE:
5995 op1_info = OP1_INFO();
5996 if (!zend_jit_free(&ctx, opline, op1_info,
5997 zend_may_throw(opline, ssa_op, op_array, ssa))) {
5998 goto jit_failure;
5999 }
6000 goto done;
6001 case ZEND_ECHO:
6002 op1_info = OP1_INFO();
6003 CHECK_OP1_TRACE_TYPE();
6004 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6005 break;
6006 }
6007 if (!zend_jit_echo(&ctx, opline, op1_info)) {
6008 goto jit_failure;
6009 }
6010 goto done;
6011 case ZEND_STRLEN:
6012 op1_info = OP1_INFO();
6013 op1_addr = OP1_REG_ADDR();
6014 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_STRING)) {
6015 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6016 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6017 goto jit_failure;
6018 }
6019 if (opline->op1_type == IS_CV
6020 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6021 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6022 }
6023 } else {
6024 CHECK_OP1_TRACE_TYPE();
6025 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6026 break;
6027 }
6028 }
6029 if (!zend_jit_strlen(&ctx, opline, op1_info, op1_addr, RES_REG_ADDR())) {
6030 goto jit_failure;
6031 }
6032 goto done;
6033 case ZEND_COUNT:
6034 op1_info = OP1_INFO();
6035 op1_addr = OP1_REG_ADDR();
6036 if (orig_op1_type == (IS_TRACE_REFERENCE|IS_ARRAY)) {
6037 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6038 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6039 goto jit_failure;
6040 }
6041 if (opline->op1_type == IS_CV
6042 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6043 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6044 }
6045 } else {
6046 CHECK_OP1_TRACE_TYPE();
6047 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
6048 break;
6049 }
6050 }
6051 if (!zend_jit_count(&ctx, opline, op1_info, op1_addr, RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
6052 goto jit_failure;
6053 }
6054 goto done;
6055 case ZEND_FETCH_THIS:
6056 delayed_fetch_this = 0;
6057 if (ssa_op->result_def >= 0 && opline->result_type != IS_CV) {
6058 if (zend_jit_may_delay_fetch_this(op_array, ssa, ssa_opcodes, ssa_op)) {
6059 ssa->var_info[ssa_op->result_def].delayed_fetch_this = 1;
6060 delayed_fetch_this = 1;
6061 }
6062 }
6063 if (!zend_jit_fetch_this(&ctx, opline, op_array, delayed_fetch_this)) {
6064 goto jit_failure;
6065 }
6066 goto done;
6067 case ZEND_SWITCH_LONG:
6068 case ZEND_SWITCH_STRING:
6069 case ZEND_MATCH:
6070 if (!zend_jit_switch(&ctx, opline, op_array, op_array_ssa, p+1, &zend_jit_traces[ZEND_JIT_TRACE_NUM])) {
6071 goto jit_failure;
6072 }
6073 goto done;
6074 case ZEND_VERIFY_RETURN_TYPE:
6075 if (opline->op1_type == IS_UNUSED) {
6076 /* Always throws */
6077 break;
6078 }
6079 if (opline->op1_type == IS_CONST) {
6080 /* TODO Different instruction format, has return value */
6081 break;
6082 }
6083 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
6084 /* Not worth bothering with */
6085 break;
6086 }
6087 op1_info = OP1_INFO();
6088 CHECK_OP1_TRACE_TYPE();
6089 if (op1_info & MAY_BE_REF) {
6090 /* TODO May need reference unwrapping. */
6091 break;
6092 }
6093 if (!zend_jit_verify_return_type(&ctx, opline, op_array, op1_info)) {
6094 goto jit_failure;
6095 }
6096 goto done;
6097 case ZEND_FE_RESET_R:
6098 op1_info = OP1_INFO();
6099 CHECK_OP1_TRACE_TYPE();
6100 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
6101 break;
6102 }
6103 if (!zend_jit_fe_reset(&ctx, opline, op1_info)) {
6104 goto jit_failure;
6105 }
6106 goto done;
6107 case ZEND_FE_FETCH_R:
6108 op1_info = OP1_INFO();
6109 CHECK_OP1_TRACE_TYPE();
6110 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
6111 break;
6112 }
6113 if ((p+1)->op == ZEND_JIT_TRACE_VM || (p+1)->op == ZEND_JIT_TRACE_END) {
6114 const zend_op *exit_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
6115 uint32_t exit_point;
6116
6117 if ((p+1)->opline == exit_opline) {
6118 /* taken branch (exit from loop) */
6119 exit_opline = opline;
6120 smart_branch_opcode = ZEND_NOP;
6121 } else if ((p+1)->opline == opline + 1) {
6122 /* not taken branch (loop) */
6123 smart_branch_opcode = ZEND_JMP;
6124 } else {
6125 ZEND_UNREACHABLE();
6126 }
6127 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
6128 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6129 if (!exit_addr) {
6130 goto jit_failure;
6131 }
6132 } else {
6133 ZEND_UNREACHABLE();
6134 }
6135 if (!zend_jit_fe_fetch(&ctx, opline, op1_info, OP2_INFO(),
6136 -1, smart_branch_opcode, exit_addr)) {
6137 goto jit_failure;
6138 }
6139 goto done;
6140 case ZEND_FETCH_CONSTANT:
6141 if (!zend_jit_fetch_constant(&ctx, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
6142 goto jit_failure;
6143 }
6144 goto done;
6145 case ZEND_INIT_METHOD_CALL:
6146 if (opline->op2_type != IS_CONST
6147 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
6148 break;
6149 }
6150 on_this = delayed_fetch_this = 0;
6151 ce = NULL;
6152 ce_is_instanceof = 0;
6153 if (opline->op1_type == IS_UNUSED) {
6154 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
6155 ce = op_array->scope;
6156 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
6157 op1_addr = 0;
6158 on_this = 1;
6159 } else {
6160 op1_info = OP1_INFO();
6161 op1_addr = OP1_REG_ADDR();
6162 if (polymorphic_side_trace) {
6163 op1_info = MAY_BE_OBJECT;
6164 } else if (orig_op1_type != IS_UNKNOWN
6165 && (orig_op1_type & IS_TRACE_REFERENCE)) {
6166 if (!zend_jit_fetch_reference(&ctx, opline, orig_op1_type, &op1_info, &op1_addr,
6167 !ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
6168 goto jit_failure;
6169 }
6170 if (opline->op1_type == IS_CV
6171 && ssa->vars[ssa_op->op1_use].alias == NO_ALIAS) {
6172 ssa->var_info[ssa_op->op1_use].guarded_reference = 1;
6173 }
6174 } else {
6175 CHECK_OP1_TRACE_TYPE();
6176 }
6177 if (ssa->var_info && ssa->ops) {
6178 if (ssa_op->op1_use >= 0) {
6179 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
6180 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
6181 ce = op1_ssa->ce;
6182 ce_is_instanceof = op1_ssa->is_instanceof;
6183 }
6184 }
6185 }
6186 if (ssa_op->op1_use >= 0) {
6187 delayed_fetch_this = ssa->var_info[ssa_op->op1_use].delayed_fetch_this;
6188 }
6189 if (delayed_fetch_this) {
6190 on_this = 1;
6191 } else if (ssa_op->op1_use >= 0 && ssa->vars[ssa_op->op1_use].definition >= 0) {
6192 on_this = ssa_opcodes[ssa->vars[ssa_op->op1_use].definition]->opcode == ZEND_FETCH_THIS;
6193 } else if (op_array_ssa->ops
6194 && op_array_ssa->vars
6195 && op_array_ssa->ops[opline-op_array->opcodes].op1_use >= 0
6196 && op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition >= 0) {
6197 on_this = op_array->opcodes[op_array_ssa->vars[op_array_ssa->ops[opline-op_array->opcodes].op1_use].definition].opcode == ZEND_FETCH_THIS;
6198 }
6199 }
6200 frame_flags = TRACE_FRAME_MASK_NESTED;
6201 if (!zend_jit_init_method_call(&ctx, opline,
6202 op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1,
6203 op_array, ssa, ssa_op, frame->call_level,
6204 op1_info, op1_addr, ce, ce_is_instanceof, on_this, delayed_fetch_this, op1_ce,
6205 p + 1, peek_checked_stack - checked_stack,
6206 polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_func_reg : -1,
6207 polymorphic_side_trace ? zend_jit_traces[parent_trace].exit_info[exit_num].poly_this_reg : -1,
6208 polymorphic_side_trace)) {
6209 goto jit_failure;
6210 }
6211 goto done;
6212 case ZEND_INIT_DYNAMIC_CALL:
6213 if (orig_op2_type != IS_OBJECT || op2_ce != zend_ce_closure) {
6214 break;
6215 }
6216 op2_info = OP2_INFO();
6217 CHECK_OP2_TRACE_TYPE();
6218 frame_flags = TRACE_FRAME_MASK_NESTED;
6219 if (!zend_jit_init_closure_call(&ctx, opline, op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1, op_array, ssa, ssa_op, frame->call_level, p + 1, peek_checked_stack - checked_stack)) {
6220 goto jit_failure;
6221 }
6222 goto done;
6223 case ZEND_SEND_ARRAY:
6224 case ZEND_SEND_UNPACK:
6225 if (JIT_G(current_frame)
6226 && JIT_G(current_frame)->call) {
6227 TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(JIT_G(current_frame)->call);
6228 }
6229 break;
6230 case ZEND_ROPE_INIT:
6231 case ZEND_ROPE_ADD:
6232 case ZEND_ROPE_END:
6233 op2_info = OP2_INFO();
6234 CHECK_OP2_TRACE_TYPE();
6235 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
6236 break;
6237 }
6238 if (!zend_jit_rope(&ctx, opline, op2_info)) {
6239 goto jit_failure;
6240 }
6241 goto done;
6242 case ZEND_FRAMELESS_ICALL_0:
6243 jit_frameless_icall0(jit, opline);
6244 goto done;
6245 case ZEND_FRAMELESS_ICALL_1:
6246 op1_info = OP1_INFO();
6247 jit_frameless_icall1(jit, opline, op1_info);
6248 goto done;
6249 case ZEND_FRAMELESS_ICALL_2:
6250 op1_info = OP1_INFO();
6251 op2_info = OP2_INFO();
6252 jit_frameless_icall2(jit, opline, op1_info, op2_info);
6253 goto done;
6254 case ZEND_FRAMELESS_ICALL_3:
6255 op1_info = OP1_INFO();
6256 op2_info = OP2_INFO();
6257 jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO());
6258 goto done;
6259 default:
6260 break;
6261 }
6262 }
6263
6264 if (opline->opcode != ZEND_NOP && opline->opcode != ZEND_JMP) {
6265 gen_handler = 1;
6266 op1_info = OP1_INFO();
6267 op2_info = OP2_INFO();
6268 if (op1_info & MAY_BE_GUARD) {
6269 op1_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6270 }
6271 if (op2_info & MAY_BE_GUARD) {
6272 op2_info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
6273 }
6274 if (!zend_jit_trace_handler(&ctx, op_array, opline,
6275 zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info), p + 1)) {
6276 goto jit_failure;
6277 }
6278 if ((p+1)->op == ZEND_JIT_TRACE_INIT_CALL && (p+1)->func) {
6279 if (opline->opcode == ZEND_NEW && opline->result_type != IS_UNUSED) {
6280 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_OBJECT, 1);
6281 }
6282 if (zend_jit_may_be_polymorphic_call(opline) ||
6283 zend_jit_may_be_modified((p+1)->func, op_array)) {
6284 if (!zend_jit_init_fcall_guard(&ctx, 0, (p+1)->func, opline+1)) {
6285 goto jit_failure;
6286 }
6287 }
6288 }
6289 }
6290
6291 done:
6292 polymorphic_side_trace = 0;
6293 switch (opline->opcode) {
6294 case ZEND_DO_FCALL:
6295 case ZEND_DO_ICALL:
6296 case ZEND_DO_UCALL:
6297 case ZEND_DO_FCALL_BY_NAME:
6298 case ZEND_CALLABLE_CONVERT:
6299 frame->call_level--;
6300 }
6301
6302 if (ra) {
6303 zend_jit_trace_cleanup_stack(&ctx, stack, opline, ssa_op, ssa, ssa_opcodes);
6304 }
6305
6306 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
6307 && STACK_FLAGS(stack, EX_VAR_TO_NUM(opline->op1.var)) & (ZREG_ZVAL_ADDREF|ZREG_THIS)) {
6308 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
6309 }
6310
6311 if (opline->opcode == ZEND_ROPE_INIT) {
6312 /* clear stack slots used by rope */
6313 uint32_t var = EX_VAR_TO_NUM(opline->result.var);
6314 uint32_t count =
6315 ((opline->extended_value * sizeof(void*)) + (sizeof(zval)-1)) / sizeof(zval);
6316
6317 do {
6318 SET_STACK_TYPE(stack, var, IS_UNKNOWN, 1);
6319 var++;
6320 count--;
6321 } while (count);
6322 }
6323
6324 if (ssa_op) {
6325 zend_ssa_range tmp;
6326
6327 /* Keep information about known types on abstract stack */
6328 if (ssa_op->result_def >= 0) {
6329 uint8_t type = IS_UNKNOWN;
6330
6331 if ((opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0
6332 || send_result) {
6333 /* we didn't set result variable */
6334 type = IS_UNKNOWN;
6335 } else if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6336 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6337 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6338 } else if (opline->opcode == ZEND_QM_ASSIGN) {
6339 if (opline->op1_type != IS_CONST) {
6340 /* copy */
6341 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6342 }
6343 } else if (opline->opcode == ZEND_ASSIGN) {
6344 if (opline->op2_type != IS_CONST
6345 && ssa_op->op1_use >= 0
6346 /* assignment to typed reference may cause conversion */
6347 && (ssa->var_info[ssa_op->op1_use].type & MAY_BE_REF) == 0) {
6348 /* copy */
6349 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6350 }
6351 } else if (opline->opcode == ZEND_POST_INC
6352 || opline->opcode == ZEND_POST_DEC) {
6353 /* copy */
6354 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6355 }
6356 if (opline->opcode == ZEND_JMP_SET
6357 || opline->opcode == ZEND_COALESCE
6358 || opline->opcode == ZEND_JMP_NULL) {
6359 if ((p+1)->op != ZEND_JIT_TRACE_VM) {
6360 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
6361 } else if ((p+1)->opline != (opline + 1)) {
6362 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type, 1);
6363 }
6364 } else {
6365 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6366 (gen_handler || type == IS_UNKNOWN || !ra || !RA_HAS_REG(ssa_op->result_def)));
6367
6368 if (op_array->last_live_range
6369 && opline->result.var > op_array->last_var
6370 && STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)) != type) {
6371 if (!gen_handler && type != IS_UNKNOWN && ra && RA_HAS_REG(ssa_op->result_def)) {
6372 uint32_t var_num = opline->result.var;
6373 uint32_t op_num = opline - op_array->opcodes;
6374 const zend_live_range *range = op_array->live_range;
6375 int j;
6376
6377 op_num += zend_jit_trace_op_len(opline);
6378 for (j = 0; j < op_array->last_live_range; range++, j++) {
6379 if (range->start > op_num) {
6380 /* further blocks will not be relevant... */
6381 break;
6382 } else if (op_num < range->end && var_num == (range->var & ~ZEND_LIVE_MASK)) {
6383 /* check if opcodes in range may throw */
6384 bool store_type = 0;
6385 const zend_ssa_op *next_ssa_op = ssa_op + zend_jit_trace_op_len(opline);
6386 const zend_jit_trace_rec *q = p + 1;
6387
6388 while (1) {
6389 if (q->op != ZEND_JIT_TRACE_VM) {
6390 store_type = 1;
6391 break;
6392 }
6393 op_num = q->opline - op_array->opcodes;
6394 if (op_num >= range->end || op_num < range->start) {
6395 break;
6396 }
6397 if (zend_may_throw(q->opline, next_ssa_op, op_array, ssa)) {
6398 store_type = 1;
6399 break;
6400 }
6401 next_ssa_op += zend_jit_trace_op_len(q->opline);
6402 q++;
6403 }
6404 if (store_type) {
6405 var_num = EX_VAR_TO_NUM(var_num);
6406
6407 if (!zend_jit_store_type(&ctx, var_num, type)) {
6408 return 0;
6409 }
6410 SET_STACK_TYPE(stack, var_num, type, 1);
6411 }
6412 break;
6413 }
6414 }
6415 }
6416 }
6417 if (ssa->var_info[ssa_op->result_def].type & MAY_BE_INDIRECT) {
6418 RESET_STACK_MEM_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
6419 }
6420 if (type != IS_UNKNOWN) {
6421 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
6422 if (opline->opcode == ZEND_FETCH_THIS
6423 && delayed_fetch_this) {
6424 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_THIS);
6425 } else if (ssa->var_info[ssa_op->result_def].avoid_refcounting) {
6426 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_ZVAL_ADDREF);
6427 } else if (ra && RA_HAS_REG(ssa_op->result_def)) {
6428 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def].ref,
6429 RA_REG_FLAGS(ssa_op->result_def) & ZREG_STORE);
6430 }
6431 }
6432 }
6433
6434 if (type == IS_LONG
6435 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->result_def, &tmp)) {
6436 ssa->var_info[ssa_op->result_def].range.min = tmp.min;
6437 ssa->var_info[ssa_op->result_def].range.max = tmp.max;
6438 ssa->var_info[ssa_op->result_def].range.underflow = 0;
6439 ssa->var_info[ssa_op->result_def].range.overflow = 0;
6440 ssa->var_info[ssa_op->result_def].has_range = 1;
6441 }
6442 }
6443 if (ssa_op->op1_def >= 0
6444 && ((opline->opcode != ZEND_QM_ASSIGN && opline->opcode != ZEND_CAST)
6445 || opline->result_type != IS_CV
6446 || opline->result.var != opline->op1.var)) {
6447 uint8_t type = IS_UNKNOWN;
6448
6449 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6450 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6451 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6452 } else if (opline->opcode == ZEND_ASSIGN) {
6453 if (!(OP1_INFO() & MAY_BE_REF)
6454 || STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var)) != IS_UNKNOWN) {
6455 if (opline->op2_type != IS_CONST) {
6456 /* copy */
6457 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6458 }
6459 }
6460 } else if (opline->opcode == ZEND_SEND_VAR
6461 || opline->opcode == ZEND_CAST
6462 || opline->opcode == ZEND_QM_ASSIGN
6463 || opline->opcode == ZEND_JMP_SET
6464 || opline->opcode == ZEND_COALESCE
6465 || opline->opcode == ZEND_JMP_NULL
6466 || opline->opcode == ZEND_FE_RESET_R) {
6467 /* keep old value */
6468 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6469 }
6470 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6471 (gen_handler || type == IS_UNKNOWN || !ra ||
6472 (!RA_HAS_REG(ssa_op->op1_def) &&
6473 !(ssa->vars[ssa_op->op1_def].no_val &&
6474 Z_MODE(OP1_REG_ADDR()) == IS_REG &&
6475 (opline->opcode == ZEND_QM_ASSIGN ||
6476 opline->opcode == ZEND_SEND_VAR ||
6477 opline->opcode == ZEND_SEND_VAR_EX ||
6478 opline->opcode == ZEND_SEND_VAR_NO_REF ||
6479 opline->opcode == ZEND_SEND_VAR_NO_REF_EX ||
6480 opline->opcode == ZEND_SEND_FUNC_ARG)))));
6481 if (type != IS_UNKNOWN) {
6482 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6483 if (ra && RA_HAS_REG(ssa_op->op1_def)) {
6484 uint8_t flags = RA_REG_FLAGS(ssa_op->op1_def) & ZREG_STORE;
6485
6486 if (ssa_op->op1_use >= 0) {
6487 if (opline->opcode == ZEND_SEND_VAR
6488 || opline->opcode == ZEND_CAST
6489 || opline->opcode == ZEND_QM_ASSIGN
6490 || opline->opcode == ZEND_JMP_SET
6491 || opline->opcode == ZEND_COALESCE
6492 || opline->opcode == ZEND_JMP_NULL
6493 || opline->opcode == ZEND_FE_RESET_R) {
6494 if (!RA_HAS_REG(ssa_op->op1_use)) {
6495 flags |= ZREG_LOAD;
6496 }
6497 }
6498 }
6499 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def].ref, flags);
6500 }
6501 }
6502 if (type == IS_LONG
6503 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6504 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6505 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6506 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6507 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6508 ssa->var_info[ssa_op->op1_def].has_range = 1;
6509 }
6510 }
6511 if (ssa_op->op2_def >= 0
6512 && (opline->opcode != ZEND_ASSIGN
6513 || opline->op1_type != IS_CV
6514 || opline->op1.var != opline->op2.var)) {
6515 uint8_t type = IS_UNKNOWN;
6516
6517 if (!(ssa->var_info[ssa_op->op2_def].type & MAY_BE_GUARD)
6518 && has_concrete_type(ssa->var_info[ssa_op->op2_def].type)) {
6519 type = concrete_type(ssa->var_info[ssa_op->op2_def].type);
6520 } else if (opline->opcode == ZEND_ASSIGN) {
6521 /* keep old value */
6522 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var));
6523 }
6524 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), type,
6525 (gen_handler || type == IS_UNKNOWN || !ra ||
6526 (!RA_HAS_REG(ssa_op->op2_def) &&
6527 !(ssa->vars[ssa_op->op2_def].no_val &&
6528 Z_MODE(OP2_REG_ADDR()) == IS_REG &&
6529 opline->opcode == ZEND_ASSIGN))));
6530 if (type != IS_UNKNOWN) {
6531 ssa->var_info[ssa_op->op2_def].type &= ~MAY_BE_GUARD;
6532 if (ra && RA_HAS_REG(ssa_op->op2_def)) {
6533 uint8_t flags = RA_REG_FLAGS(ssa_op->op2_def) & ZREG_STORE;
6534
6535 if (ssa_op->op2_use >= 0) {
6536 if (opline->opcode == ZEND_ASSIGN) {
6537 if (!RA_HAS_REG(ssa_op->op2_use)
6538 ) {
6539 flags |= ZREG_LOAD;
6540 }
6541 }
6542 }
6543 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op2.var), ra[ssa_op->op2_def].ref, flags);
6544 }
6545 }
6546 if (type == IS_LONG
6547 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op2_def, &tmp)) {
6548 ssa->var_info[ssa_op->op2_def].range.min = tmp.min;
6549 ssa->var_info[ssa_op->op2_def].range.max = tmp.max;
6550 ssa->var_info[ssa_op->op2_def].range.underflow = 0;
6551 ssa->var_info[ssa_op->op2_def].range.overflow = 0;
6552 ssa->var_info[ssa_op->op2_def].has_range = 1;
6553 }
6554 }
6555
6556 switch (opline->opcode) {
6557 case ZEND_ASSIGN_DIM:
6558 case ZEND_ASSIGN_OBJ:
6559 case ZEND_ASSIGN_STATIC_PROP:
6560 case ZEND_ASSIGN_DIM_OP:
6561 case ZEND_ASSIGN_OBJ_OP:
6562 case ZEND_ASSIGN_STATIC_PROP_OP:
6563 case ZEND_ASSIGN_OBJ_REF:
6564 case ZEND_ASSIGN_STATIC_PROP_REF:
6565 /* OP_DATA */
6566 ssa_op++;
6567 opline++;
6568 if (ssa_op->op1_def >= 0) {
6569 uint8_t type = IS_UNKNOWN;
6570
6571 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6572 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6573 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6574 } else if ((opline-1)->opcode == ZEND_ASSIGN_DIM
6575 || (opline-1)->opcode == ZEND_ASSIGN_OBJ
6576 || (opline-1)->opcode == ZEND_ASSIGN_STATIC_PROP) {
6577 /* keep old value */
6578 type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var));
6579 }
6580 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6581 (gen_handler || type == IS_UNKNOWN || !ra || !RA_HAS_REG(ssa_op->op1_def)));
6582 if (type != IS_UNKNOWN) {
6583 ssa->var_info[ssa_op->op1_def].type &= ~MAY_BE_GUARD;
6584 if (ra && RA_HAS_REG(ssa_op->op1_def)) {
6585 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def].ref,
6586 RA_REG_FLAGS(ssa_op->op1_def) & ZREG_STORE);
6587 }
6588 }
6589 if (type == IS_LONG
6590 && zend_inference_propagate_range(op_array, ssa, opline, ssa_op, ssa_op->op1_def, &tmp)) {
6591 ssa->var_info[ssa_op->op1_def].range.min = tmp.min;
6592 ssa->var_info[ssa_op->op1_def].range.max = tmp.max;
6593 ssa->var_info[ssa_op->op1_def].range.underflow = 0;
6594 ssa->var_info[ssa_op->op1_def].range.overflow = 0;
6595 ssa->var_info[ssa_op->op1_def].has_range = 1;
6596 }
6597 }
6598 ssa_op++;
6599 break;
6600 case ZEND_RECV_INIT:
6601 ssa_op++;
6602 opline++;
6603 while (opline->opcode == ZEND_RECV_INIT) {
6604 if (ssa_op->result_def >= 0) {
6605 uint8_t type = IS_UNKNOWN;
6606
6607 if (!(ssa->var_info[ssa_op->result_def].type & MAY_BE_GUARD)
6608 && has_concrete_type(ssa->var_info[ssa_op->result_def].type)) {
6609 type = concrete_type(ssa->var_info[ssa_op->result_def].type);
6610 }
6611 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), type,
6612 (gen_handler || !ra || !RA_HAS_REG(ssa_op->result_def)));
6613 if (ra && RA_HAS_REG(ssa_op->result_def)) {
6614 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ra[ssa_op->result_def].ref,
6615 RA_REG_FLAGS(ssa_op->result_def) & ZREG_STORE);
6616 }
6617 }
6618 ssa_op++;
6619 opline++;
6620 }
6621 break;
6622 case ZEND_BIND_GLOBAL:
6623 ssa_op++;
6624 opline++;
6625 while (opline->opcode == ZEND_BIND_GLOBAL) {
6626 if (ssa_op->op1_def >= 0) {
6627 uint8_t type = IS_UNKNOWN;
6628
6629 if (!(ssa->var_info[ssa_op->op1_def].type & MAY_BE_GUARD)
6630 && has_concrete_type(ssa->var_info[ssa_op->op1_def].type)) {
6631 type = concrete_type(ssa->var_info[ssa_op->op1_def].type);
6632 }
6633 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), type,
6634 (gen_handler || !ra || !RA_HAS_REG(ssa_op->op1_def)));
6635 if (ra && RA_HAS_REG(ssa_op->op1_def)) {
6636 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->op1.var), ra[ssa_op->op1_def].ref,
6637 RA_REG_FLAGS(ssa_op->op1_def) & ZREG_STORE);
6638 }
6639 }
6640 ssa_op++;
6641 opline++;
6642 }
6643 break;
6644 default:
6645 ssa_op += zend_jit_trace_op_len(opline);
6646 break;
6647 }
6648
6649 if (send_result) {
6650 ssa_op++;
6651 p++;
6652 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
6653 p++;
6654 }
6655 send_result = 0;
6656 }
6657 }
6658 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
6659 call = frame->call;
6660 assert(call && &call->func->op_array == p->op_array);
6661
6662 if (opline->opcode == ZEND_DO_UCALL
6663 || opline->opcode == ZEND_DO_FCALL_BY_NAME
6664 || opline->opcode == ZEND_DO_FCALL) {
6665
6666 frame->call_opline = opline;
6667
6668 /* Check if SEND_UNPACK/SEND_ARRAY may cause enter at different opline */
6669 if (opline > op_array->opcodes) {
6670 const zend_op *prev_opline = opline - 1;
6671
6672 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
6673 prev_opline--;
6674 }
6675 JIT_G(current_frame) = call;
6676 if ((prev_opline->opcode == ZEND_SEND_ARRAY
6677 || prev_opline->opcode == ZEND_SEND_UNPACK
6678 || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS)
6679 && p->op_array->num_args
6680 && (p->op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0
6681 && ((p+1)->op == ZEND_JIT_TRACE_VM
6682 || (p+1)->op == ZEND_JIT_TRACE_END)
6683 && (TRACE_FRAME_NUM_ARGS(call) < 0
6684 || TRACE_FRAME_NUM_ARGS(call) < p->op_array->num_args)
6685 && !zend_jit_trace_opline_guard(&ctx, (p+1)->opline)) {
6686 goto jit_failure;
6687 }
6688 JIT_G(current_frame) = frame;
6689 }
6690 }
6691
6692 if ((p+1)->op == ZEND_JIT_TRACE_END) {
6693 p++;
6694 break;
6695 }
6696 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
6697 if (TRACE_FRAME_IS_THIS_CHECKED(frame)) {
6698 TRACE_FRAME_SET_THIS_CHECKED(call);
6699 }
6700 } else if (op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC)) {
6701 TRACE_FRAME_SET_THIS_CHECKED(call);
6702 }
6703 op_array = (zend_op_array*)p->op_array;
6704 ctx.current_op_array = op_array;
6705 jit_extension =
6706 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6707 op_array_ssa = &jit_extension->func_info.ssa;
6708 frame->call = call->prev;
6709 call->prev = frame;
6710 if (p->info & ZEND_JIT_TRACE_RETURN_VALUE_USED) {
6711 TRACE_FRAME_SET_RETURN_VALUE_USED(call);
6712 } else {
6713 TRACE_FRAME_SET_RETURN_VALUE_UNUSED(call);
6714 }
6715 JIT_G(current_frame) = frame = call;
6716 stack = frame->stack;
6717 if (ra) {
6718 int j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6719
6720 for (i = 0; i < op_array->last_var; i++, j++) {
6721 if (RA_HAS_REG(j) && (RA_REG_FLAGS(j) & ZREG_LOAD) != 0) {
6722 if ((ssa->var_info[j].type & MAY_BE_GUARD) != 0) {
6723 uint8_t op_type;
6724
6725 ssa->var_info[j].type &= ~MAY_BE_GUARD;
6726 op_type = concrete_type(ssa->var_info[j].type);
6727 if (!zend_jit_type_guard(&ctx, opline, EX_NUM_TO_VAR(i), op_type)) {
6728 goto jit_failure;
6729 }
6730 SET_STACK_TYPE(stack, i, op_type, 1);
6731 }
6732 if (!zend_jit_load_var(&ctx, ssa->var_info[j].type, i, j)) {
6733 goto jit_failure;
6734 }
6735 SET_STACK_REF_EX(stack, i, ra[j].ref, ZREG_LOAD);
6736 }
6737 }
6738 }
6739 } else if (p->op == ZEND_JIT_TRACE_BACK) {
6740 op_array = (zend_op_array*)p->op_array;
6741 ctx.current_op_array = op_array;
6742 jit_extension =
6743 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
6744 op_array_ssa = &jit_extension->func_info.ssa;
6745 top = frame;
6746 if (frame->prev) {
6747 checked_stack = frame->old_checked_stack;
6748 peek_checked_stack = frame->old_peek_checked_stack;
6749 frame = frame->prev;
6750 stack = frame->stack;
6751 ZEND_ASSERT(&frame->func->op_array == op_array);
6752 } else {
6753 frame = zend_jit_trace_ret_frame(frame, op_array);
6754 TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1);
6755 frame->used_stack = checked_stack = peek_checked_stack = 0;
6756 stack = frame->stack;
6757 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6758 uint32_t j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6759
6760 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6761 /* Initialize abstract stack using SSA */
6762 if (!(ssa->var_info[j].type & MAY_BE_GUARD)
6763 && has_concrete_type(ssa->var_info[j].type)) {
6764 SET_STACK_TYPE(stack, i, concrete_type(ssa->var_info[j].type), 1);
6765 } else {
6766 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6767 }
6768 }
6769 if (ra) {
6770 j = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
6771 for (i = 0; i < op_array->last_var + op_array->T; i++, j++) {
6772 if (RA_HAS_REG(j) && (RA_REG_FLAGS(j) & ZREG_LOAD) != 0) {
6773 if (!zend_jit_load_var(&ctx, ssa->var_info[j].type, i, j)) {
6774 goto jit_failure;
6775 }
6776 SET_STACK_REF_EX(stack, i, ra[j].ref, ZREG_LOAD);
6777 }
6778 }
6779 }
6780 } else {
6781 for (i = 0; i < op_array->last_var + op_array->T; i++) {
6782 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
6783 }
6784 }
6785 opline = NULL;
6786 }
6787 JIT_G(current_frame) = frame;
6788 if (res_type != IS_UNKNOWN
6789 && (p+1)->op == ZEND_JIT_TRACE_VM) {
6790 const zend_op *opline = (p+1)->opline - 1;
6791 if (opline->result_type != IS_UNUSED) {
6792 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), res_type, 1);
6793 }
6794 }
6795 res_type = IS_UNKNOWN;
6796 } else if (p->op == ZEND_JIT_TRACE_END) {
6797 break;
6798 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
6799 const zend_op *init_opline = zend_jit_trace_find_init_fcall_op(p, op_array);
6800 int num_args = -1;
6801
6802 if (init_opline
6803 && init_opline->extended_value <= TRACE_FRAME_MAX_NUM_ARGS) {
6804 num_args = init_opline->extended_value;
6805 }
6806
6807 call = top;
6808 TRACE_FRAME_INIT(call, p->func, frame_flags, num_args);
6809 call->prev = frame->call;
6810 if (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
6811 TRACE_FRAME_SET_LAST_SEND_BY_VAL(call);
6812 if (init_opline && init_opline->opcode == ZEND_INIT_DYNAMIC_CALL) {
6813 TRACE_FRAME_SET_CLOSURE_CALL(call);
6814 }
6815 }
6816 if (init_opline) {
6817 if (init_opline->opcode != ZEND_NEW
6818 && (init_opline->opcode != ZEND_INIT_METHOD_CALL
6819 || init_opline->op1_type == IS_UNDEF
6820 || (!(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6821 && ssa_op
6822 && (ssa_op-1)->op1_use >=0
6823 && ssa->var_info[(ssa_op-1)->op1_use].delayed_fetch_this))
6824 && (init_opline->opcode != ZEND_INIT_USER_CALL
6825 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6826 && (init_opline->opcode != ZEND_INIT_DYNAMIC_CALL
6827 || (p->func && (!p->func->common.scope || (p->func->common.fn_flags & ZEND_ACC_STATIC))))
6828 ) {
6829 TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(call);
6830 } else if (init_opline->opcode == ZEND_NEW
6831 || (init_opline->opcode == ZEND_INIT_METHOD_CALL
6832 && init_opline->op1_type != IS_UNDEF
6833 && !(p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL)
6834 && p->func && p->func->common.scope && !(p->func->common.fn_flags & ZEND_ACC_STATIC))) {
6835 TRACE_FRAME_SET_ALWAYS_RELEASE_THIS(call);
6836 }
6837 }
6838 frame->call = call;
6839 top = zend_jit_trace_call_frame(top, p->op_array);
6840 if (p->func) {
6841 if (p->func->type == ZEND_USER_FUNCTION) {
6842 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
6843 zend_jit_op_array_trace_extension *jit_extension =
6844 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(p->op_array);
6845
6846 i = 0;
6847 while (i < p->op_array->num_args) {
6848 /* Types of arguments are going to be stored in abstract stack when processing SEV instruction */
6849 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6850 i++;
6851 }
6852 while (i < p->op_array->last_var) {
6853 if (jit_extension
6854 && zend_jit_var_may_alias(p->op_array, &jit_extension->func_info.ssa, i) != NO_ALIAS) {
6855 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6856 } else {
6857 SET_STACK_TYPE(call->stack, i, IS_UNDEF, 1);
6858 }
6859 i++;
6860 }
6861 while (i < p->op_array->last_var + p->op_array->T) {
6862 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6863 i++;
6864 }
6865 } else {
6866 for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) {
6867 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6868 }
6869 }
6870 } else {
6871 ZEND_ASSERT(p->func->type == ZEND_INTERNAL_FUNCTION);
6872 for (i = 0; i < p->op_array->num_args; i++) {
6873 SET_STACK_TYPE(call->stack, i, IS_UNKNOWN, 1);
6874 }
6875 }
6876 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6877 int skip_guard = 0;
6878
6879 if (init_opline) {
6880 zend_call_info *call_info = jit_extension->func_info.callee_info;
6881
6882 while (call_info) {
6883 if (call_info->caller_init_opline == init_opline
6884 && !call_info->is_prototype) {
6885 if (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) {
6886 if (init_opline->opcode == ZEND_INIT_STATIC_METHOD_CALL
6887 && init_opline->op1_type != IS_CONST) {
6888 break;
6889 } else if (init_opline->opcode == ZEND_INIT_METHOD_CALL) {
6890 break;
6891 }
6892 }
6893 skip_guard = 1;
6894 break;
6895 }
6896 call_info = call_info->next_callee;
6897 }
6898 if (!skip_guard
6899 && !zend_jit_may_be_polymorphic_call(init_opline)
6900 && !zend_jit_may_be_modified(p->func, op_array)) {
6901 skip_guard = 1;
6902 }
6903 }
6904
6905 if (!skip_guard) {
6906 if (!opline) {
6907 zend_jit_trace_rec *q = p + 1;
6908 while (q->op != ZEND_JIT_TRACE_VM && q->op != ZEND_JIT_TRACE_END) {
6909 q++;
6910 }
6911 opline = q->opline;
6912 ZEND_ASSERT(opline != NULL);
6913 }
6914 if (!zend_jit_init_fcall_guard(&ctx,
6915 ZEND_JIT_TRACE_FAKE_LEVEL(p->info), p->func, opline)) {
6916 goto jit_failure;
6917 }
6918 }
6919 }
6920 }
6921 call->old_checked_stack = checked_stack;
6922 call->old_peek_checked_stack = peek_checked_stack;
6923 if (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) {
6924 frame->call_level++;
6925 call->used_stack = checked_stack = peek_checked_stack = 0;
6926 } else {
6927 if (p->func) {
6928 call->used_stack = zend_vm_calc_used_stack(init_opline->extended_value, (zend_function*)p->func);
6929 } else {
6930 call->used_stack = (ZEND_CALL_FRAME_SLOT + init_opline->extended_value) * sizeof(zval);
6931 }
6932 switch (init_opline->opcode) {
6933 case ZEND_INIT_FCALL:
6934 case ZEND_INIT_FCALL_BY_NAME:
6935 case ZEND_INIT_NS_FCALL_BY_NAME:
6936 case ZEND_INIT_METHOD_CALL:
6937 case ZEND_INIT_DYNAMIC_CALL:
6938 //case ZEND_INIT_STATIC_METHOD_CALL:
6939 //case ZEND_INIT_USER_CALL:
6940 //case ZEND_NEW:
6941 checked_stack += call->used_stack;
6942 if (checked_stack > peek_checked_stack) {
6943 peek_checked_stack = checked_stack;
6944 }
6945 break;
6946 default:
6947 checked_stack = peek_checked_stack = 0;
6948 }
6949 }
6950 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
6951 call = frame->call;
6952 if (call) {
6953 checked_stack = call->old_checked_stack;
6954 peek_checked_stack = call->old_peek_checked_stack;
6955 top = call;
6956 frame->call = call->prev;
6957 }
6958 } else {
6959 ZEND_UNREACHABLE();
6960 }
6961 }
6962
6963 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_END);
6964
6965 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
6966
6967 if (!parent_trace && zend_jit_trace_uses_initial_ip(&ctx)) {
6968 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
6969 }
6970
6971 if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
6972 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
6973 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6974 if (ra) {
6975 zend_ssa_phi *phi = ssa->blocks[1].phis;
6976
6977 while (phi) {
6978 if (RA_HAS_REG(phi->sources[1])
6979 && STACK_MEM_TYPE(stack, phi->var) != STACK_TYPE(stack, phi->var)
6980 && (RA_REG_FLAGS(phi->sources[1]) & (ZREG_LOAD|ZREG_STORE)) == 0) {
6981
6982 if (!RA_HAS_REG(phi->ssa_var)
6983 || (RA_REG_FLAGS(phi->ssa_var) & (ZREG_LOAD|ZREG_STORE)) == 0) {
6984 /* Store actual type to memory to avoid deoptimization mistakes */
6985 zend_jit_store_var_type(&ctx, phi->var, STACK_TYPE(stack, phi->var));
6986 }
6987 }
6988 phi = phi->next;
6989 }
6990 }
6991 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6992 if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
6993 && !zend_jit_set_ip(&ctx, p->opline)) {
6994 goto jit_failure;
6995 }
6996 }
6997 t->link = ZEND_JIT_TRACE_NUM;
6998 if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
6999 t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT;
7000 }
7001 if (!(t->flags & ZEND_JIT_TRACE_LOOP)) {
7002 const void *timeout_exit_addr = NULL;
7003
7004 t->flags |= ZEND_JIT_TRACE_LOOP;
7005
7006 if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7007 if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
7008 || (ra
7009 && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) {
7010 /* Deoptimize to the first instruction of the loop */
7011 uint32_t exit_point = zend_jit_trace_get_exit_point(trace_buffer[1].opline, ZEND_JIT_EXIT_TO_VM);
7012
7013 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
7014 if (!timeout_exit_addr) {
7015 goto jit_failure;
7016 }
7017 } else {
7018 timeout_exit_addr = zend_jit_stub_handlers[jit_stub_interrupt_handler];
7019 }
7020 }
7021
7022 zend_jit_trace_end_loop(&ctx, jit->trace_loop_ref, timeout_exit_addr); /* jump back to start of the trace loop */
7023 }
7024 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
7025 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
7026 if (ra
7027 && (p-1)->op != ZEND_JIT_TRACE_ENTER
7028 && (p-1)->op != ZEND_JIT_TRACE_BACK
7029 && opline->opcode != ZEND_DO_UCALL
7030 && opline->opcode != ZEND_DO_FCALL
7031 && opline->opcode != ZEND_DO_FCALL_BY_NAME
7032 && opline->opcode != ZEND_INCLUDE_OR_EVAL) {
7033 for (i = 0; i < op_array->last_var + op_array->T; i++) {
7034 int32_t ref = STACK_REF(stack, i);
7035 uint8_t type = STACK_TYPE(stack, i);
7036
7037 if (ref && !(STACK_FLAGS(stack, i) & (ZREG_LOAD|ZREG_STORE))) {
7038 if (!zend_jit_store_ref(jit, 1 << type, i, ref, STACK_MEM_TYPE(stack, i) != type)) {
7039 goto jit_failure;
7040 }
7041 SET_STACK_TYPE(stack, i, type, 1);
7042 } else if (i < op_array->last_var
7043 && type != IS_UNKNOWN
7044 && type != STACK_MEM_TYPE(stack, i)
7045 && zend_jit_trace_must_store_type(op_array, op_array_ssa, opline - op_array->opcodes, i, type)) {
7046 if (!zend_jit_store_type(jit, i, type)) {
7047 return 0;
7048 }
7049 SET_STACK_TYPE(stack, i, type, 1);
7050 }
7051 CLEAR_STACK_REF(stack, i);
7052 }
7053 }
7054 if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
7055 const void *timeout_exit_addr = NULL;
7056
7057 t->link = zend_jit_find_trace(p->opline->handler);
7058 if (t->link == 0) {
7059 /* this can happen if ZEND_JIT_EXIT_INVALIDATE was handled
7060 * by zend_jit_trace_exit() in another thread after this
7061 * thread set ZEND_JIT_TRACE_STOP_LINK in zend_jit_trace_execute();
7062 * ZEND_JIT_EXIT_INVALIDATE resets the opline handler to one of
7063 * the "_counter_handler" functions, and these are not registered
7064 * tracer functions */
7065 goto jit_failure;
7066 }
7067 if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
7068 && !zend_jit_set_ip(&ctx, p->opline)) {
7069 goto jit_failure;
7070 }
7071 if (!parent_trace && zend_jit_trace_uses_initial_ip(&ctx)) {
7072 t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
7073 }
7074 if (parent_trace
7075 && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT)
7076 && zend_jit_traces[parent_trace].root == t->link) {
7077 if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) {
7078 uint32_t exit_point;
7079
7080 for (i = 0; i < op_array->last_var + op_array->T; i++) {
7081 SET_STACK_TYPE(stack, i, IS_UNKNOWN, 1);
7082 }
7083 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
7084 timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
7085 if (!timeout_exit_addr) {
7086 goto jit_failure;
7087 }
7088 } else {
7089 timeout_exit_addr = zend_jit_stub_handlers[jit_stub_interrupt_handler];
7090 }
7091 }
7092 zend_jit_trace_link_to_root(&ctx, &zend_jit_traces[t->link], timeout_exit_addr);
7093 } else {
7094 zend_jit_trace_return(&ctx, 0, NULL);
7095 }
7096 } else if (p->stop == ZEND_JIT_TRACE_STOP_RETURN) {
7097 zend_jit_trace_return(&ctx, 0, NULL);
7098 } else {
7099 // TODO: not implemented ???
7100 ZEND_ASSERT(0 && p->stop);
7101 }
7102
7103 if (ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7104 goto jit_failure;
7105 }
7106
7107 handler = zend_jit_finish(&ctx);
7108
7109 if (handler) {
7110 if (p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL) {
7111 const zend_op_array *rec_op_array;
7112
7113 rec_op_array = op_array = trace_buffer->op_array;
7114 jit_extension =
7115 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7116 p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE;
7117 for (;;p++) {
7118 if (p->op == ZEND_JIT_TRACE_VM) {
7119 opline = p->opline;
7120 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7121 if (p->op_array == rec_op_array) {
7122 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7123 }
7124 op_array = p->op_array;
7125 jit_extension =
7126 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7127 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7128 op_array = p->op_array;
7129 jit_extension =
7130 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7131 } else if (p->op == ZEND_JIT_TRACE_END) {
7132 break;
7133 }
7134 }
7135 } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
7136 || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
7137 if (opline
7138 && (opline->opcode == ZEND_DO_UCALL
7139 || opline->opcode == ZEND_DO_FCALL
7140 || opline->opcode == ZEND_DO_FCALL_BY_NAME
7141 || opline->opcode == ZEND_YIELD
7142 || opline->opcode == ZEND_YIELD_FROM
7143 || opline->opcode == ZEND_INCLUDE_OR_EVAL)) {
7144 zend_jit_trace_setup_ret_counter(opline, jit_extension->offset);
7145 }
7146 if (JIT_G(current_frame)
7147 && JIT_G(current_frame)->prev) {
7148 frame = JIT_G(current_frame)->prev;
7149 do {
7150 if (frame->call_opline) {
7151 op_array = &frame->func->op_array;
7152 jit_extension =
7153 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7154 zend_jit_trace_setup_ret_counter(frame->call_opline, jit_extension->offset);
7155 }
7156 frame = frame->prev;
7157 } while (frame);
7158 }
7159 }
7160 }
7161
7162 jit_failure:
7163 zend_jit_free_ctx(&ctx);
7164
7165 if (name) {
7166 zend_string_release(name);
7167 }
7168
7169 jit_cleanup:
7170 /* Clean up used op_arrays */
7171 while (num_op_arrays > 0) {
7172 op_array = op_arrays[--num_op_arrays];
7173 jit_extension =
7174 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7175
7176 jit_extension->func_info.num = 0;
7177 jit_extension->func_info.flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
7178 | ZEND_FUNC_JIT_ON_PROF_REQUEST
7179 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
7180 | ZEND_FUNC_JIT_ON_HOT_TRACE;
7181 memset(&jit_extension->func_info.ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
7182 }
7183
7184 zend_arena_release(&CG(arena), checkpoint);
7185
7186 JIT_G(current_frame) = NULL;
7187 JIT_G(current_trace) = NULL;
7188
7189 return handler;
7190 }
7191
zend_jit_trace_escape_name(uint32_t trace_num,uint32_t exit_num)7192 static zend_string *zend_jit_trace_escape_name(uint32_t trace_num, uint32_t exit_num)
7193 {
7194 smart_str buf = {0};
7195
7196 smart_str_appends(&buf," ESCAPE-");
7197 smart_str_append_long(&buf, (zend_long)trace_num);
7198 smart_str_appendc(&buf, '-');
7199 smart_str_append_long(&buf, (zend_long)exit_num);
7200 smart_str_0(&buf);
7201 return buf.s;
7202 }
7203
zend_jit_trace_exit_to_vm(uint32_t trace_num,uint32_t exit_num)7204 static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_num)
7205 {
7206 const void *handler = NULL;
7207 zend_jit_ctx ctx;
7208 zend_string *name;
7209 void *checkpoint;
7210 const zend_op *opline;
7211 uint32_t stack_size;
7212 zend_jit_trace_stack *stack;
7213 bool original_handler = 0;
7214
7215 if (!zend_jit_trace_exit_needs_deoptimization(trace_num, exit_num)) {
7216 return zend_jit_stub_handlers[jit_stub_trace_escape];
7217 }
7218
7219 name = zend_jit_trace_escape_name(trace_num, exit_num);
7220
7221 if (!zend_jit_deoptimizer_start(&ctx, name, trace_num, exit_num)) {
7222 zend_string_release(name);
7223 return NULL;
7224 }
7225
7226 checkpoint = zend_arena_checkpoint(CG(arena));;
7227
7228 /* Deoptimization */
7229 stack_size = zend_jit_traces[trace_num].exit_info[exit_num].stack_size;
7230 stack = zend_jit_traces[trace_num].exit_info[exit_num].stack_size ?
7231 zend_jit_traces[trace_num].stack_map + zend_jit_traces[trace_num].exit_info[exit_num].stack_offset :
7232 NULL;
7233
7234 if (!zend_jit_trace_deoptimization(&ctx,
7235 zend_jit_traces[trace_num].exit_info[exit_num].flags,
7236 zend_jit_traces[trace_num].exit_info[exit_num].opline,
7237 stack, stack_size, NULL, NULL,
7238 zend_jit_traces[trace_num].constants,
7239 zend_jit_traces[trace_num].exit_info[exit_num].poly_func_reg,
7240 0)) {
7241 goto jit_failure;
7242 }
7243
7244 opline = zend_jit_traces[trace_num].exit_info[exit_num].opline;
7245 if (opline) {
7246 if (opline == zend_jit_traces[zend_jit_traces[trace_num].root].opline) {
7247 zend_jit_op_array_trace_extension *jit_extension =
7248 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(zend_jit_traces[zend_jit_traces[trace_num].root].op_array);
7249
7250 if (ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->orig_handler != opline->handler) {
7251 /* prevent endless loop */
7252 original_handler = 1;
7253 }
7254 }
7255 zend_jit_set_ip_ex(&ctx, opline, original_handler);
7256 }
7257
7258 zend_jit_trace_return(&ctx, original_handler, opline);
7259
7260 handler = zend_jit_finish(&ctx);
7261
7262 jit_failure:
7263 zend_jit_free_ctx(&ctx);
7264 zend_string_release(name);
7265 zend_arena_release(&CG(arena), checkpoint);
7266 return handler;
7267 }
7268
zend_jit_compile_root_trace(zend_jit_trace_rec * trace_buffer,const zend_op * opline,size_t offset)7269 static zend_jit_trace_stop zend_jit_compile_root_trace(zend_jit_trace_rec *trace_buffer, const zend_op *opline, size_t offset)
7270 {
7271 zend_jit_trace_stop ret;
7272 const void *handler;
7273 uint8_t orig_trigger;
7274 zend_jit_trace_info *t = NULL;
7275 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
7276 bool do_bailout = 0;
7277
7278 zend_shared_alloc_lock();
7279
7280 /* Checks under lock */
7281 if ((ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_JITED)) {
7282 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
7283 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7284 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7285 } else {
7286 zend_try {
7287 SHM_UNPROTECT();
7288 zend_jit_unprotect();
7289
7290 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
7291
7292 t->id = ZEND_JIT_TRACE_NUM;
7293 t->root = ZEND_JIT_TRACE_NUM;
7294 t->parent = 0;
7295 t->link = 0;
7296 t->exit_count = 0;
7297 t->child_count = 0;
7298 t->stack_map_size = 0;
7299 t->flags = 0;
7300 t->polymorphism = 0;
7301 t->jmp_table_size = 0;
7302 t->op_array = trace_buffer[0].op_array;
7303 t->opline = trace_buffer[1].opline;
7304 t->exit_info = exit_info;
7305 t->stack_map = NULL;
7306 t->consts_count = 0;
7307 t->constants = NULL;
7308
7309 orig_trigger = JIT_G(trigger);
7310 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
7311
7312 handler = zend_jit_trace(trace_buffer, 0, 0);
7313
7314 JIT_G(trigger) = orig_trigger;
7315
7316 if (handler) {
7317 zend_jit_trace_exit_info *shared_exit_info = NULL;
7318
7319 t->exit_info = NULL;
7320 if (t->exit_count) {
7321 /* reallocate exit_info into shared memory */
7322 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
7323 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7324
7325 if (!shared_exit_info) {
7326 if (t->stack_map) {
7327 efree(t->stack_map);
7328 t->stack_map = NULL;
7329 }
7330 if (t->constants) {
7331 efree(t->constants);
7332 t->constants = NULL;
7333 }
7334 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7335 goto exit;
7336 }
7337 memcpy(shared_exit_info, exit_info,
7338 sizeof(zend_jit_trace_exit_info) * t->exit_count);
7339 t->exit_info = shared_exit_info;
7340 }
7341
7342 if (t->stack_map_size) {
7343 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
7344 if (!shared_stack_map) {
7345 efree(t->stack_map);
7346 t->stack_map = NULL;
7347 if (t->constants) {
7348 efree(t->constants);
7349 t->constants = NULL;
7350 }
7351 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7352 goto exit;
7353 }
7354 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
7355 efree(t->stack_map);
7356 t->stack_map = shared_stack_map;
7357 }
7358
7359 if (t->consts_count) {
7360 zend_jit_exit_const *constants = (zend_jit_exit_const*)zend_shared_alloc(t->consts_count * sizeof(zend_jit_exit_const));
7361 if (!constants) {
7362 efree(t->constants);
7363 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
7364 goto exit;
7365 }
7366 memcpy(constants, t->constants, t->consts_count * sizeof(zend_jit_exit_const));
7367 efree(t->constants);
7368 t->constants = constants;
7369 }
7370
7371 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
7372 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
7373
7374 ((zend_op*)opline)->handler = handler;
7375
7376 ZEND_JIT_TRACE_NUM++;
7377 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_JITED;
7378
7379 ret = ZEND_JIT_TRACE_STOP_COMPILED;
7380 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
7381 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
7382 if (t->stack_map) {
7383 efree(t->stack_map);
7384 t->stack_map = NULL;
7385 }
7386 if (t->constants) {
7387 efree(t->constants);
7388 t->constants = NULL;
7389 }
7390 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
7391 } else {
7392 if (t->stack_map) {
7393 efree(t->stack_map);
7394 t->stack_map = NULL;
7395 }
7396 if (t->constants) {
7397 efree(t->constants);
7398 t->constants = NULL;
7399 }
7400 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
7401 }
7402
7403 exit:;
7404 } zend_catch {
7405 do_bailout = 1;
7406 } zend_end_try();
7407
7408 zend_jit_protect();
7409 SHM_PROTECT();
7410 }
7411
7412 zend_shared_alloc_unlock();
7413
7414 if (do_bailout) {
7415 zend_bailout();
7416 }
7417
7418 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
7419 && ret == ZEND_JIT_TRACE_STOP_COMPILED
7420 && t->exit_count > 0) {
7421 zend_jit_dump_exit_info(t);
7422 }
7423
7424 return ret;
7425 }
7426
7427 /* Set counting handler back to original VM handler. */
zend_jit_stop_hot_trace_counters(zend_op_array * op_array)7428 static void zend_jit_stop_hot_trace_counters(zend_op_array *op_array)
7429 {
7430 zend_jit_op_array_trace_extension *jit_extension;
7431 uint32_t i;
7432
7433 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7434 for (i = 0; i < op_array->last; i++) {
7435 /* Opline with Jit-ed code handler is skipped. */
7436 if (jit_extension->trace_info[i].trace_flags &
7437 (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7438 continue;
7439 }
7440 if (jit_extension->trace_info[i].trace_flags &
7441 (ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_START_RETURN)) {
7442 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
7443 }
7444 }
7445 }
7446
7447 /* Get the tracing op_array. */
zend_jit_stop_persistent_op_array(zend_op_array * op_array)7448 static void zend_jit_stop_persistent_op_array(zend_op_array *op_array)
7449 {
7450 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
7451 if (!func_info) {
7452 return;
7453 }
7454 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
7455 zend_jit_stop_hot_trace_counters(op_array);
7456 }
7457 }
7458
7459 /* Get all op_arrays with counter handler. */
zend_jit_stop_persistent_script(zend_persistent_script * script)7460 static void zend_jit_stop_persistent_script(zend_persistent_script *script)
7461 {
7462 zend_class_entry *ce;
7463 zend_op_array *op_array;
7464
7465 zend_jit_stop_persistent_op_array(&script->script.main_op_array);
7466
7467 ZEND_HASH_FOREACH_PTR(&script->script.function_table, op_array) {
7468 zend_jit_stop_persistent_op_array(op_array);
7469 } ZEND_HASH_FOREACH_END();
7470
7471 ZEND_HASH_FOREACH_PTR(&script->script.class_table, ce) {
7472 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
7473 if (op_array->type == ZEND_USER_FUNCTION) {
7474 zend_jit_stop_persistent_op_array(op_array);
7475 }
7476 } ZEND_HASH_FOREACH_END();
7477 } ZEND_HASH_FOREACH_END();
7478 }
7479
7480 /* Get all scripts which are accelerated by JIT */
zend_jit_stop_counter_handlers(void)7481 static void zend_jit_stop_counter_handlers(void)
7482 {
7483 if (ZCSG(jit_counters_stopped)) {
7484 return;
7485 }
7486
7487 zend_shared_alloc_lock();
7488 /* mprotect has an extreme overhead, avoid calls to it for every function. */
7489 SHM_UNPROTECT();
7490 if (!ZCSG(jit_counters_stopped)) {
7491 ZCSG(jit_counters_stopped) = true;
7492 for (uint32_t i = 0; i < ZCSG(hash).max_num_entries; i++) {
7493 zend_accel_hash_entry *cache_entry;
7494 for (cache_entry = ZCSG(hash).hash_table[i]; cache_entry; cache_entry = cache_entry->next) {
7495 zend_persistent_script *script;
7496 if (cache_entry->indirect) continue;
7497 script = (zend_persistent_script *)cache_entry->data;
7498 zend_jit_stop_persistent_script(script);
7499 }
7500 }
7501 }
7502 SHM_PROTECT();
7503 zend_shared_alloc_unlock();
7504 }
7505
zend_jit_blacklist_root_trace(const zend_op * opline,size_t offset)7506 static void zend_jit_blacklist_root_trace(const zend_op *opline, size_t offset)
7507 {
7508 zend_shared_alloc_lock();
7509
7510 if (!(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_BLACKLISTED)) {
7511 SHM_UNPROTECT();
7512 zend_jit_unprotect();
7513
7514 ((zend_op*)opline)->handler =
7515 ZEND_OP_TRACE_INFO(opline, offset)->orig_handler;
7516
7517 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags |= ZEND_JIT_TRACE_BLACKLISTED;
7518
7519 zend_jit_protect();
7520 SHM_PROTECT();
7521 }
7522
7523 zend_shared_alloc_unlock();
7524 }
7525
zend_jit_trace_is_bad_root(const zend_op * opline,zend_jit_trace_stop stop,size_t offset)7526 static bool zend_jit_trace_is_bad_root(const zend_op *opline, zend_jit_trace_stop stop, size_t offset)
7527 {
7528 const zend_op **cache_opline = JIT_G(bad_root_cache_opline);
7529 uint8_t *cache_count = JIT_G(bad_root_cache_count);
7530 uint8_t *cache_stop = JIT_G(bad_root_cache_stop);
7531 uint32_t cache_slot = JIT_G(bad_root_slot);
7532 uint32_t i;
7533
7534 for (i = 0; i < ZEND_JIT_TRACE_BAD_ROOT_SLOTS; i++) {
7535 if (cache_opline[i] == opline) {
7536 if (cache_count[i] >= JIT_G(blacklist_root_trace) - 1) {
7537 cache_opline[i] = NULL;
7538 return 1;
7539 } else {
7540 #if 0
7541 if (ZEND_OP_TRACE_INFO(opline, offset)->counter) {
7542 *ZEND_OP_TRACE_INFO(opline, offset)->counter =
7543 random() % ZEND_JIT_TRACE_COUNTER_MAX;
7544 }
7545 #endif
7546 cache_count[i]++;
7547 cache_stop[i] = stop;
7548 return 0;
7549 }
7550 }
7551 }
7552 i = cache_slot;
7553 cache_opline[i] = opline;
7554 cache_count[i] = 1;
7555 cache_stop[i] = stop;
7556 cache_slot = (i + 1) % ZEND_JIT_TRACE_BAD_ROOT_SLOTS;
7557 JIT_G(bad_root_slot) = cache_slot;
7558 return 0;
7559 }
7560
zend_jit_dump_trace(zend_jit_trace_rec * trace_buffer,zend_ssa * tssa)7561 static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa)
7562 {
7563 zend_jit_trace_rec *p = trace_buffer;
7564 const zend_op_array *op_array;
7565 const zend_op *opline;
7566 uint32_t level = 1 + trace_buffer[0].level;
7567 int idx, len, i, v, vars_count, call_level;
7568
7569 ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START);
7570 op_array = p->op_array;
7571 p += ZEND_JIT_TRACE_START_REC_SIZE;
7572 idx = 0;
7573 call_level = 0;
7574
7575 if (tssa && tssa->var_info) {
7576 if (trace_buffer->start == ZEND_JIT_TRACE_START_ENTER) {
7577 vars_count = op_array->last_var;
7578 } else {
7579 vars_count = op_array->last_var + op_array->T;
7580 }
7581 for (i = 0; i < vars_count; i++) {
7582 if (tssa->vars[i].use_chain >= 0 || tssa->vars[i].phi_use_chain) {
7583 fprintf(stderr, " %*c;", level, ' ');
7584 zend_dump_ssa_var(op_array, tssa, i, 0, i, ZEND_DUMP_RC_INFERENCE);
7585 fprintf(stderr, "\n");
7586 }
7587 }
7588 if (trace_buffer->stop == ZEND_JIT_TRACE_STOP_LOOP
7589 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
7590 || trace_buffer->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
7591 zend_ssa_phi *p = tssa->blocks[1].phis;
7592
7593 fprintf(stderr, "LOOP:\n");
7594
7595 while (p) {
7596 fprintf(stderr, " ;");
7597 zend_dump_ssa_var(op_array, tssa, p->ssa_var, 0, p->var, ZEND_DUMP_RC_INFERENCE);
7598 fprintf(stderr, " = Phi(");
7599 zend_dump_ssa_var(op_array, tssa, p->sources[0], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7600 fprintf(stderr, ", ");
7601 zend_dump_ssa_var(op_array, tssa, p->sources[1], 0, p->var, ZEND_DUMP_RC_INFERENCE);
7602 fprintf(stderr, ")\n");
7603 p = p->next;
7604 }
7605 }
7606 }
7607
7608 while (1) {
7609 if (p->op == ZEND_JIT_TRACE_VM) {
7610 uint8_t op1_type, op2_type, op3_type;
7611
7612 opline = p->opline;
7613 fprintf(stderr, "%04d%*c",
7614 (int)(opline - op_array->opcodes),
7615 level, ' ');
7616 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7617
7618 op1_type = p->op1_type;
7619 op2_type = p->op2_type;
7620 op3_type = p->op3_type;
7621 if (op1_type != IS_UNKNOWN || op2_type != IS_UNKNOWN || op3_type != IS_UNKNOWN) {
7622 fprintf(stderr, " ;");
7623 if (op1_type != IS_UNKNOWN) {
7624 const char *ref = (op1_type & IS_TRACE_INDIRECT) ?
7625 ((op1_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7626 ((op1_type & IS_TRACE_REFERENCE) ? "&" : "");
7627 if ((p+1)->op == ZEND_JIT_TRACE_OP1_TYPE) {
7628 p++;
7629 fprintf(stderr, " op1(%sobject of class %s)", ref,
7630 ZSTR_VAL(p->ce->name));
7631 } else {
7632 const char *type = ((op1_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
7633 fprintf(stderr, " op1(%s%s%s)", ref, (op1_type & IS_TRACE_PACKED) ? "packed " : "", type);
7634 }
7635 }
7636 if (op2_type != IS_UNKNOWN) {
7637 const char *ref = (op2_type & IS_TRACE_INDIRECT) ?
7638 ((op2_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7639 ((op2_type & IS_TRACE_REFERENCE) ? "&" : "");
7640 if ((p+1)->op == ZEND_JIT_TRACE_OP2_TYPE) {
7641 p++;
7642 fprintf(stderr, " op2(%sobject of class %s)", ref,
7643 ZSTR_VAL(p->ce->name));
7644 } else {
7645 const char *type = ((op2_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7646 fprintf(stderr, " op2(%s%s)", ref, type);
7647 }
7648 }
7649 if (op3_type != IS_UNKNOWN) {
7650 const char *ref = (op3_type & IS_TRACE_INDIRECT) ?
7651 ((op3_type & IS_TRACE_REFERENCE) ? "*&" : "*") :
7652 ((op3_type & IS_TRACE_REFERENCE) ? "&" : "");
7653 const char *type = ((op3_type & ~IS_TRACE_INDIRECT) == 0) ? "undef" : zend_get_type_by_const(op3_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT));
7654 fprintf(stderr, " op3(%s%s)", ref, type);
7655 }
7656 }
7657 if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) {
7658 uint8_t val_type;
7659 const char *type;
7660
7661 if (op1_type == IS_UNKNOWN && op2_type == IS_UNKNOWN && op3_type == IS_UNKNOWN) {
7662 fprintf(stderr, " ;");
7663 }
7664 p++;
7665 val_type = p->op1_type;
7666
7667 if (val_type == IS_UNDEF) {
7668 type = "undef";
7669 } else if (val_type == IS_REFERENCE) {
7670 type = "ref";
7671 } else {
7672 type = zend_get_type_by_const(val_type);
7673 }
7674 fprintf(stderr, " val(%s)", type);
7675 }
7676 fprintf(stderr, "\n");
7677 idx++;
7678
7679 len = zend_jit_trace_op_len(opline);
7680 while (len > 1) {
7681 opline++;
7682 fprintf(stderr, "%04d%*c;",
7683 (int)(opline - op_array->opcodes),
7684 level, ' ');
7685 zend_dump_op(op_array, NULL, opline, ZEND_DUMP_RC_INFERENCE, tssa, (tssa && tssa->ops) ? tssa->ops + idx : NULL);
7686 idx++;
7687 len--;
7688 fprintf(stderr, "\n");
7689 }
7690 } else if (p->op == ZEND_JIT_TRACE_ENTER) {
7691 op_array = p->op_array;
7692 fprintf(stderr, " %*c>enter %s%s%s\n",
7693 level, ' ',
7694 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7695 op_array->scope ? "::" : "",
7696 op_array->function_name ?
7697 ZSTR_VAL(op_array->function_name) :
7698 ZSTR_VAL(op_array->filename));
7699 level++;
7700 if (tssa && tssa->var_info) {
7701 call_level++;
7702 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7703 vars_count = op_array->last_var;
7704 for (i = 0; i < vars_count; i++, v++) {
7705 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7706 fprintf(stderr, " %*c;", level, ' ');
7707 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7708 fprintf(stderr, "\n");
7709 }
7710 }
7711 }
7712 } else if (p->op == ZEND_JIT_TRACE_BACK) {
7713 op_array = p->op_array;
7714 level--;
7715 fprintf(stderr, " %*c<back %s%s%s\n",
7716 level, ' ',
7717 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7718 op_array->scope ? "::" : "",
7719 op_array->function_name ?
7720 ZSTR_VAL(op_array->function_name) :
7721 ZSTR_VAL(op_array->filename));
7722 if (tssa && tssa->var_info) {
7723 if (call_level == 0) {
7724 v = ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(p->info);
7725 vars_count = op_array->last_var + op_array->T;
7726 for (i = 0; i < vars_count; i++, v++) {
7727 if (tssa->vars[v].use_chain >= 0 || tssa->vars[v].phi_use_chain) {
7728 fprintf(stderr, " %*c;", level, ' ');
7729 zend_dump_ssa_var(op_array, tssa, v, 0, i, ZEND_DUMP_RC_INFERENCE);
7730 fprintf(stderr, "\n");
7731 }
7732 }
7733 } else {
7734 call_level--;
7735 }
7736 }
7737 } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) {
7738 if (p->func != (zend_function*)&zend_pass_function) {
7739 fprintf(stderr, (p->info & ZEND_JIT_TRACE_FAKE_INIT_CALL) ? " %*c>fake_init %s%s%s\n" : " %*c>init %s%s%s\n",
7740 level, ' ',
7741 (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "",
7742 (p->func && p->func->common.scope) ? "::" : "",
7743 p->func ? ZSTR_VAL(p->func->common.function_name) : "???");
7744 } else {
7745 fprintf(stderr, " %*c>skip\n",
7746 level, ' ');
7747 }
7748 } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) {
7749 if (p->func != (zend_function*)&zend_pass_function) {
7750 fprintf(stderr, " %*c>call %s%s%s\n",
7751 level, ' ',
7752 p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "",
7753 p->func->common.scope ? "::" : "",
7754 ZSTR_VAL(p->func->common.function_name));
7755 } else {
7756 fprintf(stderr, " %*c>skip\n",
7757 level, ' ');
7758 }
7759 } else if (p->op == ZEND_JIT_TRACE_END) {
7760 break;
7761 }
7762 p++;
7763 }
7764 }
7765
zend_jit_dump_exit_info(zend_jit_trace_info * t)7766 static void zend_jit_dump_exit_info(zend_jit_trace_info *t)
7767 {
7768 int i, j;
7769
7770 fprintf(stderr, "---- TRACE %d exit info\n", t->id);
7771 for (i = 0; i < t->exit_count; i++) {
7772 const zend_op_array *op_array = t->exit_info[i].op_array;
7773 uint32_t stack_size = t->exit_info[i].stack_size;
7774 zend_jit_trace_stack *stack = t->exit_info[i].stack_size ? t->stack_map + t->exit_info[i].stack_offset : NULL;
7775
7776 fprintf(stderr, " exit_%d:", i);
7777 if (t->exit_info[i].opline) {
7778 fprintf(stderr, " %04d/", (int)(t->exit_info[i].opline - op_array->opcodes));
7779 } else {
7780 fprintf(stderr, " ----/");
7781 }
7782 if (t->exit_info[i].stack_size) {
7783 fprintf(stderr, "%04d/%d", t->exit_info[i].stack_offset, t->exit_info[i].stack_size);
7784 } else {
7785 fprintf(stderr, "----/0");
7786 }
7787 if (t->exit_info[i].flags & ZEND_JIT_EXIT_TO_VM) {
7788 fprintf(stderr, "/VM");
7789 }
7790 if (t->exit_info[i].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
7791 fprintf(stderr, "/CALL");
7792 }
7793 if (t->exit_info[i].flags & (ZEND_JIT_EXIT_POLYMORPHISM|ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL)) {
7794 fprintf(stderr, "/POLY");
7795 if (t->exit_info[i].flags & ZEND_JIT_EXIT_METHOD_CALL) {
7796 fprintf(stderr, "(%s, %s)",
7797 t->exit_info[i].poly_func_reg != ZREG_NONE ? zend_reg_name(t->exit_info[i].poly_func_reg) : "?",
7798 t->exit_info[i].poly_this_reg != ZREG_NONE ? zend_reg_name(t->exit_info[i].poly_this_reg) : "?");
7799 }
7800 }
7801 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP1) {
7802 fprintf(stderr, "/FREE_OP1");
7803 }
7804 if (t->exit_info[i].flags & ZEND_JIT_EXIT_FREE_OP2) {
7805 fprintf(stderr, "/FREE_OP2");
7806 }
7807 for (j = 0; j < stack_size; j++) {
7808 uint8_t type = STACK_TYPE(stack, j);
7809 if (type != IS_UNKNOWN) {
7810 fprintf(stderr, " ");
7811 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7812 fprintf(stderr, ":");
7813 if (type == IS_UNDEF) {
7814 fprintf(stderr, "undef");
7815 } else {
7816 fprintf(stderr, "%s", zend_get_type_by_const(type));
7817 }
7818 if (STACK_FLAGS(stack, j) == ZREG_CONST) {
7819 if (type == IS_LONG) {
7820 fprintf(stderr, "(" ZEND_LONG_FMT ")", (zend_long)t->constants[STACK_REF(stack, j)].i);
7821 } else if (type == IS_DOUBLE) {
7822 fprintf(stderr, "(%g)", t->constants[STACK_REF(stack, j)].d);
7823 } else {
7824 ZEND_UNREACHABLE();
7825 }
7826 } else if (STACK_FLAGS(stack, j) == ZREG_TYPE_ONLY) {
7827 fprintf(stderr, "(type_only)");
7828 } else if (STACK_FLAGS(stack, j) == ZREG_THIS) {
7829 fprintf(stderr, "(this)");
7830 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_ADDREF) {
7831 fprintf(stderr, "(zval_try_addref)");
7832 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_COPY) {
7833 fprintf(stderr, "zval_copy(%s)", zend_reg_name(STACK_REG(stack, j)));
7834 } else if (STACK_FLAGS(stack, j) & ZREG_SPILL_SLOT) {
7835 if (STACK_REG(stack, j) == ZREG_NONE) {
7836 fprintf(stderr, "(spill=0x%x", STACK_REF(stack, j));
7837 } else {
7838 fprintf(stderr, "(spill=0x%x(%s)", STACK_REF(stack, j), zend_reg_name(STACK_REG(stack, j)));
7839 }
7840 if (STACK_FLAGS(stack, j) != 0) {
7841 fprintf(stderr, ":%x", STACK_FLAGS(stack, j));
7842 }
7843 fprintf(stderr, ")");
7844 } else if (STACK_REG(stack, j) != ZREG_NONE) {
7845 fprintf(stderr, "(%s", zend_reg_name(STACK_REG(stack, j)));
7846 if (STACK_FLAGS(stack, j) != 0) {
7847 fprintf(stderr, ":%x", STACK_FLAGS(stack, j));
7848 }
7849 fprintf(stderr, ")");
7850 }
7851 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_ADDREF) {
7852 fprintf(stderr, ":unknown(zval_try_addref)");
7853 } else if (STACK_FLAGS(stack, j) == ZREG_ZVAL_COPY) {
7854 fprintf(stderr, " ");
7855 zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j);
7856 fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name(STACK_REG(stack, j)));
7857 }
7858 }
7859 fprintf(stderr, "\n");
7860 }
7861 }
7862
zend_jit_trace_hot_root(zend_execute_data * execute_data,const zend_op * opline)7863 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline)
7864 {
7865 const zend_op *orig_opline;
7866 zend_jit_trace_stop stop;
7867 int ret = 0;
7868 zend_op_array *op_array;
7869 zend_jit_op_array_trace_extension *jit_extension;
7870 size_t offset;
7871 uint32_t trace_num;
7872 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
7873
7874 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
7875 ZEND_ASSERT(opline >= EX(func)->op_array.opcodes &&
7876 opline < EX(func)->op_array.opcodes + EX(func)->op_array.last);
7877
7878 repeat:
7879 trace_num = ZEND_JIT_TRACE_NUM;
7880 orig_opline = opline;
7881 op_array = &EX(func)->op_array;
7882 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7883 offset = jit_extension->offset;
7884
7885 EX(opline) = opline;
7886
7887 /* Lock-free check if the root trace was already JIT-ed or blacklist-ed in another process */
7888 if (ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED)) {
7889 return 0;
7890 }
7891
7892 if (JIT_G(tracing)) {
7893 ++(*ZEND_OP_TRACE_INFO(opline, offset)->counter);
7894 return 0;
7895 }
7896
7897 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7898 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7899 trace_num,
7900 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7901 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
7902 EX(func)->op_array.scope ? "::" : "",
7903 EX(func)->op_array.function_name ?
7904 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
7905 ZSTR_VAL(EX(func)->op_array.filename),
7906 opline->lineno);
7907 }
7908
7909 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
7910 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
7911 zend_jit_stop_counter_handlers();
7912 goto abort;
7913 }
7914
7915 JIT_G(tracing) = 1;
7916 stop = zend_jit_trace_execute(execute_data, opline, trace_buffer,
7917 ZEND_OP_TRACE_INFO(opline, offset)->trace_flags & ZEND_JIT_TRACE_START_MASK, 0);
7918 JIT_G(tracing) = 0;
7919
7920 if (stop & ZEND_JIT_TRACE_HALT) {
7921 ret = -1;
7922 }
7923 stop &= ~ZEND_JIT_TRACE_HALT;
7924
7925 if (UNEXPECTED(trace_buffer[1].opline != orig_opline)) {
7926 orig_opline = trace_buffer[1].opline;
7927 op_array = (zend_op_array*)trace_buffer[0].op_array;
7928 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7929 offset = jit_extension->offset;
7930 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
7931 const zend_op_array *op_array = trace_buffer[0].op_array;
7932 const zend_op *opline = trace_buffer[1].opline;
7933 zend_jit_op_array_trace_extension *jit_extension =
7934 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
7935 size_t offset = jit_extension->offset;
7936
7937 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
7938 trace_num,
7939 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
7940 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
7941 op_array->scope ? "::" : "",
7942 op_array->function_name ?
7943 ZSTR_VAL(op_array->function_name) : "$main",
7944 ZSTR_VAL(op_array->filename),
7945 opline->lineno);
7946 }
7947 }
7948
7949 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
7950 zend_jit_dump_trace(trace_buffer, NULL);
7951 }
7952
7953 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
7954 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
7955 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
7956 uint32_t idx = trace_buffer[1].last;
7957 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);
7958 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
7959 trace_num,
7960 link_to);
7961 } else {
7962 fprintf(stderr, "---- TRACE %d stop (%s)\n",
7963 trace_num,
7964 zend_jit_trace_stop_description[stop]);
7965 }
7966 }
7967 stop = zend_jit_compile_root_trace(trace_buffer, orig_opline, offset);
7968 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
7969 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
7970 fprintf(stderr, "---- TRACE %d %s\n",
7971 trace_num,
7972 zend_jit_trace_stop_description[stop]);
7973 }
7974 } else {
7975 goto abort;
7976 }
7977 } else {
7978 abort:
7979 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
7980 fprintf(stderr, "---- TRACE %d abort (%s)\n",
7981 trace_num,
7982 zend_jit_trace_stop_description[stop]);
7983 }
7984 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
7985 || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) {
7986 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
7987 fprintf(stderr, "---- TRACE %d blacklisted\n",
7988 trace_num);
7989 }
7990 zend_jit_blacklist_root_trace(orig_opline, offset);
7991 }
7992 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
7993 execute_data = EG(current_execute_data);
7994 opline = EX(opline);
7995 goto repeat;
7996 }
7997 }
7998
7999 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
8000 fprintf(stderr, "\n");
8001 }
8002
8003 return ret;
8004 }
8005
zend_jit_blacklist_trace_exit(uint32_t trace_num,uint32_t exit_num)8006 static void zend_jit_blacklist_trace_exit(uint32_t trace_num, uint32_t exit_num)
8007 {
8008 const void *handler;
8009 bool do_bailout = 0;
8010
8011 zend_shared_alloc_lock();
8012
8013 if (!(zend_jit_traces[trace_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED))) {
8014 SHM_UNPROTECT();
8015 zend_jit_unprotect();
8016
8017 zend_try {
8018 handler = zend_jit_trace_exit_to_vm(trace_num, exit_num);
8019
8020 if (handler) {
8021 zend_jit_link_side_trace(
8022 zend_jit_traces[trace_num].code_start,
8023 zend_jit_traces[trace_num].code_size,
8024 zend_jit_traces[trace_num].jmp_table_size,
8025 exit_num,
8026 handler);
8027 }
8028 zend_jit_traces[trace_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_BLACKLISTED;
8029 } zend_catch {
8030 do_bailout = 1;
8031 } zend_end_try();
8032
8033 zend_jit_protect();
8034 SHM_PROTECT();
8035 }
8036
8037 zend_shared_alloc_unlock();
8038
8039 if (do_bailout) {
8040 zend_bailout();
8041 }
8042 }
8043
zend_jit_trace_exit_is_bad(uint32_t trace_num,uint32_t exit_num)8044 static bool zend_jit_trace_exit_is_bad(uint32_t trace_num, uint32_t exit_num)
8045 {
8046 uint8_t *counter = JIT_G(exit_counters) +
8047 zend_jit_traces[trace_num].exit_counters + exit_num;
8048
8049 if (*counter + 1 >= JIT_G(hot_side_exit) + JIT_G(blacklist_side_trace)) {
8050 return 1;
8051 }
8052 (*counter)++;
8053 return 0;
8054 }
8055
zend_jit_trace_exit_is_hot(uint32_t trace_num,uint32_t exit_num)8056 static bool zend_jit_trace_exit_is_hot(uint32_t trace_num, uint32_t exit_num)
8057 {
8058 uint8_t *counter = JIT_G(exit_counters) +
8059 zend_jit_traces[trace_num].exit_counters + exit_num;
8060
8061 if (*counter + 1 >= JIT_G(hot_side_exit)) {
8062 return 1;
8063 }
8064 (*counter)++;
8065 return 0;
8066 }
8067
zend_jit_compile_side_trace(zend_jit_trace_rec * trace_buffer,uint32_t parent_num,uint32_t exit_num,uint32_t polymorphism)8068 static zend_jit_trace_stop zend_jit_compile_side_trace(zend_jit_trace_rec *trace_buffer, uint32_t parent_num, uint32_t exit_num, uint32_t polymorphism)
8069 {
8070 zend_jit_trace_stop ret;
8071 const void *handler;
8072 uint8_t orig_trigger;
8073 zend_jit_trace_info *t;
8074 zend_jit_trace_exit_info exit_info[ZEND_JIT_TRACE_MAX_EXITS];
8075 bool do_bailout = 0;
8076
8077 zend_shared_alloc_lock();
8078
8079 /* Checks under lock */
8080 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8081 ret = ZEND_JIT_TRACE_STOP_ALREADY_DONE;
8082 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8083 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8084 } else if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
8085 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
8086 } else {
8087 SHM_UNPROTECT();
8088 zend_jit_unprotect();
8089
8090 zend_try {
8091 t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
8092
8093 t->id = ZEND_JIT_TRACE_NUM;
8094 t->root = zend_jit_traces[parent_num].root;
8095 t->parent = parent_num;
8096 t->link = 0;
8097 t->exit_count = 0;
8098 t->child_count = 0;
8099 t->stack_map_size = 0;
8100 t->flags = 0;
8101 t->polymorphism = polymorphism;
8102 t->jmp_table_size = 0;
8103 t->opline = NULL;
8104 t->exit_info = exit_info;
8105 t->stack_map = NULL;
8106 t->consts_count = 0;
8107 t->constants = NULL;
8108
8109 orig_trigger = JIT_G(trigger);
8110 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
8111
8112 handler = zend_jit_trace(trace_buffer, parent_num, exit_num);
8113
8114 JIT_G(trigger) = orig_trigger;
8115
8116 if (handler) {
8117 zend_jit_trace_exit_info *shared_exit_info = NULL;
8118
8119 t->exit_info = NULL;
8120 if (t->exit_count) {
8121 /* reallocate exit_info into shared memory */
8122 shared_exit_info = (zend_jit_trace_exit_info*)zend_shared_alloc(
8123 sizeof(zend_jit_trace_exit_info) * t->exit_count);
8124
8125 if (!shared_exit_info) {
8126 if (t->stack_map) {
8127 efree(t->stack_map);
8128 t->stack_map = NULL;
8129 }
8130 if (t->constants) {
8131 efree(t->constants);
8132 t->constants = NULL;
8133 }
8134 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
8135 goto exit;
8136 }
8137 memcpy(shared_exit_info, exit_info,
8138 sizeof(zend_jit_trace_exit_info) * t->exit_count);
8139 t->exit_info = shared_exit_info;
8140 }
8141
8142 if (t->stack_map_size) {
8143 zend_jit_trace_stack *shared_stack_map = (zend_jit_trace_stack*)zend_shared_alloc(t->stack_map_size * sizeof(zend_jit_trace_stack));
8144 if (!shared_stack_map) {
8145 efree(t->stack_map);
8146 t->stack_map = NULL;
8147 if (t->constants) {
8148 efree(t->constants);
8149 t->constants = NULL;
8150 }
8151 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
8152 goto exit;
8153 }
8154 memcpy(shared_stack_map, t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
8155 efree(t->stack_map);
8156 t->stack_map = shared_stack_map;
8157 }
8158
8159 if (t->consts_count) {
8160 zend_jit_exit_const *constants = (zend_jit_exit_const*)zend_shared_alloc(t->consts_count * sizeof(zend_jit_exit_const));
8161 if (!constants) {
8162 efree(t->constants);
8163 ret = ZEND_JIT_TRACE_STOP_NO_SHM;
8164 goto exit;
8165 }
8166 memcpy(constants, t->constants, t->consts_count * sizeof(zend_jit_exit_const));
8167 efree(t->constants);
8168 t->constants = constants;
8169 }
8170
8171 zend_jit_link_side_trace(
8172 zend_jit_traces[parent_num].code_start,
8173 zend_jit_traces[parent_num].code_size,
8174 zend_jit_traces[parent_num].jmp_table_size,
8175 exit_num,
8176 handler);
8177
8178 t->exit_counters = ZEND_JIT_EXIT_COUNTERS;
8179 ZEND_JIT_EXIT_COUNTERS += t->exit_count;
8180
8181 zend_jit_traces[zend_jit_traces[parent_num].root].child_count++;
8182 ZEND_JIT_TRACE_NUM++;
8183 zend_jit_traces[parent_num].exit_info[exit_num].flags |= ZEND_JIT_EXIT_JITED;
8184
8185 ret = ZEND_JIT_TRACE_STOP_COMPILED;
8186 } else if (t->exit_count >= ZEND_JIT_TRACE_MAX_EXITS ||
8187 ZEND_JIT_EXIT_COUNTERS + t->exit_count >= JIT_G(max_exit_counters)) {
8188 if (t->stack_map) {
8189 efree(t->stack_map);
8190 t->stack_map = NULL;
8191 }
8192 if (t->constants) {
8193 efree(t->constants);
8194 t->constants = NULL;
8195 }
8196 ret = ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
8197 } else {
8198 if (t->stack_map) {
8199 efree(t->stack_map);
8200 t->stack_map = NULL;
8201 }
8202 if (t->constants) {
8203 efree(t->constants);
8204 t->constants = NULL;
8205 }
8206 ret = ZEND_JIT_TRACE_STOP_COMPILER_ERROR;
8207 }
8208
8209 exit:;
8210 } zend_catch {
8211 do_bailout = 1;
8212 } zend_end_try();
8213
8214 zend_jit_protect();
8215 SHM_PROTECT();
8216 }
8217
8218 zend_shared_alloc_unlock();
8219
8220 if (do_bailout) {
8221 zend_bailout();
8222 }
8223
8224 if ((JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT_INFO) != 0
8225 && ret == ZEND_JIT_TRACE_STOP_COMPILED
8226 && t->exit_count > 0) {
8227 zend_jit_dump_exit_info(t);
8228 }
8229
8230 return ret;
8231 }
8232
zend_jit_trace_hot_side(zend_execute_data * execute_data,uint32_t parent_num,uint32_t exit_num)8233 int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint32_t parent_num, uint32_t exit_num)
8234 {
8235 zend_jit_trace_stop stop;
8236 int ret = 0;
8237 uint32_t trace_num;
8238 zend_jit_trace_rec trace_buffer[ZEND_JIT_TRACE_MAX_LENGTH];
8239 uint32_t is_megamorphic = 0;
8240 uint32_t polymorphism = 0;
8241
8242 trace_num = ZEND_JIT_TRACE_NUM;
8243
8244 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8245 if (zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8246 return 0;
8247 }
8248
8249 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8250 fprintf(stderr, "---- TRACE %d start (side trace %d/%d) %s%s%s() %s:%d\n",
8251 trace_num, parent_num, exit_num,
8252 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8253 EX(func)->op_array.scope ? "::" : "",
8254 EX(func)->op_array.function_name ?
8255 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8256 ZSTR_VAL(EX(func)->op_array.filename),
8257 EX(opline)->lineno);
8258 }
8259
8260 if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8261 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_TRACES;
8262 goto abort;
8263 }
8264
8265 if (zend_jit_traces[zend_jit_traces[parent_num].root].child_count >= JIT_G(max_side_traces)) {
8266 stop = ZEND_JIT_TRACE_STOP_TOO_MANY_CHILDREN;
8267 goto abort;
8268 }
8269
8270 if (JIT_G(max_polymorphic_calls) > 0) {
8271 if ((zend_jit_traces[parent_num].exit_info[exit_num].flags & (ZEND_JIT_EXIT_METHOD_CALL|ZEND_JIT_EXIT_CLOSURE_CALL))
8272 || ((zend_jit_traces[parent_num].exit_info[exit_num].flags & ZEND_JIT_EXIT_POLYMORPHISM)
8273 && EX(call))) {
8274 if (zend_jit_traces[parent_num].polymorphism >= JIT_G(max_polymorphic_calls) - 1) {
8275 is_megamorphic = zend_jit_traces[parent_num].exit_info[exit_num].flags &
8276 (ZEND_JIT_EXIT_METHOD_CALL | ZEND_JIT_EXIT_CLOSURE_CALL | ZEND_JIT_EXIT_POLYMORPHISM);
8277 } else if (!zend_jit_traces[parent_num].polymorphism) {
8278 polymorphism = 1;
8279 } else if (exit_num == 0) {
8280 polymorphism = zend_jit_traces[parent_num].polymorphism + 1;
8281 }
8282 }
8283 }
8284
8285 JIT_G(tracing) = 1;
8286 stop = zend_jit_trace_execute(execute_data, EX(opline), trace_buffer, ZEND_JIT_TRACE_START_SIDE, is_megamorphic);
8287 JIT_G(tracing) = 0;
8288
8289 if (stop & ZEND_JIT_TRACE_HALT) {
8290 ret = -1;
8291 }
8292 stop &= ~ZEND_JIT_TRACE_HALT;
8293
8294 if (UNEXPECTED(trace_buffer->start != ZEND_JIT_TRACE_START_SIDE)) {
8295 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_START) {
8296 const zend_op_array *op_array = trace_buffer[0].op_array;
8297 const zend_op *opline = trace_buffer[1].opline;
8298 zend_jit_op_array_trace_extension *jit_extension =
8299 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8300 size_t offset = jit_extension->offset;
8301
8302 fprintf(stderr, "---- TRACE %d start (%s) %s%s%s() %s:%d\n",
8303 trace_num,
8304 zend_jit_trace_star_desc(ZEND_OP_TRACE_INFO(opline, offset)->trace_flags),
8305 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
8306 op_array->scope ? "::" : "",
8307 op_array->function_name ?
8308 ZSTR_VAL(op_array->function_name) : "$main",
8309 ZSTR_VAL(op_array->filename),
8310 opline->lineno);
8311 }
8312 }
8313
8314 if (UNEXPECTED(JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BYTECODE)) {
8315 zend_jit_dump_trace(trace_buffer, NULL);
8316 }
8317
8318 if (ZEND_JIT_TRACE_STOP_OK(stop)) {
8319 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_STOP) {
8320 if (stop == ZEND_JIT_TRACE_STOP_LINK) {
8321 uint32_t idx = trace_buffer[1].last;
8322 uint32_t link_to = zend_jit_find_trace(trace_buffer[idx].opline->handler);;
8323 fprintf(stderr, "---- TRACE %d stop (link to %d)\n",
8324 trace_num,
8325 link_to);
8326 } else {
8327 fprintf(stderr, "---- TRACE %d stop (%s)\n",
8328 trace_num,
8329 zend_jit_trace_stop_description[stop]);
8330 }
8331 }
8332 if (EXPECTED(trace_buffer->start == ZEND_JIT_TRACE_START_SIDE)) {
8333 stop = zend_jit_compile_side_trace(trace_buffer, parent_num, exit_num, polymorphism);
8334 } else {
8335 const zend_op_array *op_array = trace_buffer[0].op_array;
8336 zend_jit_op_array_trace_extension *jit_extension =
8337 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8338 const zend_op *opline = trace_buffer[1].opline;
8339
8340 stop = zend_jit_compile_root_trace(trace_buffer, opline, jit_extension->offset);
8341 }
8342 if (EXPECTED(ZEND_JIT_TRACE_STOP_DONE(stop))) {
8343 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_COMPILED) {
8344 fprintf(stderr, "---- TRACE %d %s\n",
8345 trace_num,
8346 zend_jit_trace_stop_description[stop]);
8347 }
8348 } else {
8349 goto abort;
8350 }
8351 } else {
8352 abort:
8353 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_ABORT) {
8354 fprintf(stderr, "---- TRACE %d abort (%s)\n",
8355 trace_num,
8356 zend_jit_trace_stop_description[stop]);
8357 }
8358 if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
8359 || zend_jit_trace_exit_is_bad(parent_num, exit_num)) {
8360 zend_jit_blacklist_trace_exit(parent_num, exit_num);
8361 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8362 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8363 parent_num, exit_num);
8364 }
8365 }
8366 if (ZEND_JIT_TRACE_STOP_REPEAT(stop)) {
8367 execute_data = EG(current_execute_data);
8368 return zend_jit_trace_hot_root(execute_data, EX(opline));
8369 }
8370 }
8371
8372 if (JIT_G(debug) & (ZEND_JIT_DEBUG_TRACE_STOP|ZEND_JIT_DEBUG_TRACE_ABORT|ZEND_JIT_DEBUG_TRACE_COMPILED|ZEND_JIT_DEBUG_TRACE_BLACKLIST)) {
8373 fprintf(stderr, "\n");
8374 }
8375
8376 return ret;
8377 }
8378
zend_jit_trace_exit(uint32_t exit_num,zend_jit_registers_buf * regs)8379 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs)
8380 {
8381 uint32_t trace_num = EG(jit_trace_num);
8382 zend_execute_data *execute_data = EG(current_execute_data);
8383 const zend_op *orig_opline = EX(opline);
8384 const zend_op *opline;
8385 zend_jit_trace_info *t = &zend_jit_traces[trace_num];
8386 int repeat_last_opline = 0;
8387
8388 /* Deoptimization of VM stack state */
8389 uint32_t i;
8390 uint32_t stack_size = t->exit_info[exit_num].stack_size;
8391 zend_jit_trace_stack *stack = stack_size ? t->stack_map + t->exit_info[exit_num].stack_offset : NULL;
8392
8393 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_RESTORE_CALL) {
8394 zend_execute_data *call = (zend_execute_data *)regs->gpr[ZREG_RX];
8395 call->prev_execute_data = EX(call);
8396 EX(call) = call;
8397 }
8398
8399 for (i = 0; i < stack_size; i++) {
8400 if (STACK_FLAGS(stack, i) == ZREG_CONST) {
8401 if (STACK_TYPE(stack, i) == IS_LONG) {
8402 ZVAL_LONG(EX_VAR_NUM(i), (zend_long)t->constants[STACK_REF(stack, i)].i);
8403 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8404 ZVAL_DOUBLE(EX_VAR_NUM(i), t->constants[STACK_REF(stack, i)].d);
8405 } else {
8406 ZEND_UNREACHABLE();
8407 }
8408 } else if (STACK_FLAGS(stack, i) == ZREG_TYPE_ONLY) {
8409 uint32_t type = STACK_TYPE(stack, i);
8410 if (type <= IS_DOUBLE) {
8411 Z_TYPE_INFO_P(EX_VAR_NUM(i)) = type;
8412 } else {
8413 ZEND_UNREACHABLE();
8414 }
8415 } else if (STACK_FLAGS(stack, i) == ZREG_THIS) {
8416 zend_object *obj = Z_OBJ(EX(This));
8417
8418 GC_ADDREF(obj);
8419 ZVAL_OBJ(EX_VAR_NUM(i), obj);
8420 } else if (STACK_FLAGS(stack, i) == ZREG_ZVAL_ADDREF) {
8421 Z_TRY_ADDREF_P(EX_VAR_NUM(i));
8422 } else if (STACK_FLAGS(stack, i) == ZREG_ZVAL_COPY) {
8423 zval *val = (zval*)regs->gpr[STACK_REG(stack, i)];
8424
8425 if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) {
8426 /* Undefined array index or property */
8427 repeat_last_opline = 1;
8428 } else {
8429 ZVAL_COPY(EX_VAR_NUM(i), val);
8430 }
8431 } else if (STACK_FLAGS(stack, i) & ZREG_SPILL_SLOT) {
8432 ZEND_ASSERT(STACK_REG(stack, i) != ZREG_NONE);
8433 uintptr_t ptr = (uintptr_t)regs->gpr[STACK_REG(stack, i)] + STACK_REF(stack, i);
8434
8435 if (STACK_TYPE(stack, i) == IS_LONG) {
8436 ZVAL_LONG(EX_VAR_NUM(i), *(zend_long*)ptr);
8437 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8438 ZVAL_DOUBLE(EX_VAR_NUM(i), *(double*)ptr);
8439 } else {
8440 ZEND_UNREACHABLE();
8441 }
8442 } else if (STACK_REG(stack, i) != ZREG_NONE) {
8443 if (STACK_TYPE(stack, i) == IS_LONG) {
8444 zend_long val = regs->gpr[STACK_REG(stack, i)];
8445 ZVAL_LONG(EX_VAR_NUM(i), val);
8446 } else if (STACK_TYPE(stack, i) == IS_DOUBLE) {
8447 double val = regs->fpr[STACK_REG(stack, i) - ZREG_FIRST_FPR];
8448 ZVAL_DOUBLE(EX_VAR_NUM(i), val);
8449 } else {
8450 ZEND_UNREACHABLE();
8451 }
8452 }
8453 }
8454
8455 if (repeat_last_opline) {
8456 EX(opline) = t->exit_info[exit_num].opline - 1;
8457 if ((EX(opline)->op1_type & (IS_VAR|IS_TMP_VAR))
8458 && !(t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1)
8459 && EX(opline)->opcode != ZEND_FETCH_LIST_R) {
8460 Z_TRY_ADDREF_P(EX_VAR(EX(opline)->op1.var));
8461 }
8462 return 1;
8463 }
8464
8465 opline = t->exit_info[exit_num].opline;
8466
8467 if (opline) {
8468 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP2) {
8469 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8470 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8471 || (opline-1)->opcode == ZEND_FETCH_LIST_R
8472 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG);
8473 EX(opline) = opline-1;
8474 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op2.var));
8475 }
8476 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_FREE_OP1) {
8477 ZEND_ASSERT((opline-1)->opcode == ZEND_FETCH_DIM_R
8478 || (opline-1)->opcode == ZEND_FETCH_DIM_IS
8479 || (opline-1)->opcode == ZEND_FETCH_DIM_FUNC_ARG
8480 || (opline-1)->opcode == ZEND_FETCH_OBJ_R
8481 || (opline-1)->opcode == ZEND_FETCH_OBJ_IS
8482 || (opline-1)->opcode == ZEND_FETCH_OBJ_FUNC_ARG);
8483 EX(opline) = opline-1;
8484 zval_ptr_dtor_nogc(EX_VAR((opline-1)->op1.var));
8485 }
8486 if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_FREE_OP1|ZEND_JIT_EXIT_FREE_OP2)) {
8487 if (EG(exception)) {
8488 return 1;
8489 }
8490 }
8491 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
8492 ZEND_ASSERT(t->exit_info[exit_num].poly_func_reg >= 0);
8493 zend_function *func = (zend_function*)regs->gpr[t->exit_info[exit_num].poly_func_reg];
8494
8495 if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) {
8496 zend_string_release_ex(func->common.function_name, 0);
8497 zend_free_trampoline(func);
8498 EX(opline) = opline;
8499 return 1;
8500 }
8501 }
8502
8503 /* Set VM opline to continue interpretation */
8504 EX(opline) = opline;
8505 }
8506
8507 if (zend_atomic_bool_load_ex(&EG(vm_interrupt)) || JIT_G(tracing)) {
8508 return 1;
8509 /* Lock-free check if the side trace was already JIT-ed or blacklist-ed in another process */
8510 } else if (t->exit_info[exit_num].flags & (ZEND_JIT_EXIT_JITED|ZEND_JIT_EXIT_BLACKLISTED)) {
8511 return 0;
8512 }
8513
8514 ZEND_ASSERT(EX(func)->type == ZEND_USER_FUNCTION);
8515 ZEND_ASSERT(EX(opline) >= EX(func)->op_array.opcodes &&
8516 EX(opline) < EX(func)->op_array.opcodes + EX(func)->op_array.last);
8517
8518 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_EXIT) {
8519 fprintf(stderr, " TRACE %d exit %d %s%s%s() %s:%d\n",
8520 trace_num,
8521 exit_num,
8522 EX(func)->op_array.scope ? ZSTR_VAL(EX(func)->op_array.scope->name) : "",
8523 EX(func)->op_array.scope ? "::" : "",
8524 EX(func)->op_array.function_name ?
8525 ZSTR_VAL(EX(func)->op_array.function_name) : "$main",
8526 ZSTR_VAL(EX(func)->op_array.filename),
8527 EX(opline)->lineno);
8528 }
8529
8530 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_INVALIDATE) {
8531 zend_jit_op_array_trace_extension *jit_extension;
8532 uint32_t num = trace_num;
8533
8534 while (t->root != num) {
8535 num = t->root;
8536 t = &zend_jit_traces[num];
8537 }
8538
8539 zend_shared_alloc_lock();
8540
8541 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(t->op_array);
8542
8543 /* Checks under lock, just in case something has changed while we were waiting for the lock */
8544 if (!(ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & (ZEND_JIT_TRACE_JITED|ZEND_JIT_TRACE_BLACKLISTED))) {
8545 /* skip: not JIT-ed nor blacklisted */
8546 } else if (ZEND_JIT_TRACE_NUM >= JIT_G(max_root_traces)) {
8547 /* skip: too many root traces */
8548 } else {
8549 SHM_UNPROTECT();
8550 zend_jit_unprotect();
8551
8552 if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_LOOP) {
8553 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_loop_trace_counter_handler;
8554 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_ENTER) {
8555 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_func_trace_counter_handler;
8556 } else if (ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_START_RETURN) {
8557 ((zend_op*)(t->opline))->handler = (const void*)zend_jit_ret_trace_counter_handler;
8558 }
8559 ZEND_OP_TRACE_INFO(t->opline, jit_extension->offset)->trace_flags &=
8560 ZEND_JIT_TRACE_START_LOOP|ZEND_JIT_TRACE_START_ENTER|ZEND_JIT_TRACE_START_RETURN;
8561
8562 zend_jit_protect();
8563 SHM_PROTECT();
8564 }
8565
8566 zend_shared_alloc_unlock();
8567
8568 return 0;
8569 }
8570
8571 if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_TO_VM) {
8572 if (zend_jit_trace_exit_is_bad(trace_num, exit_num)) {
8573 zend_jit_blacklist_trace_exit(trace_num, exit_num);
8574 if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
8575 fprintf(stderr, "---- EXIT %d/%d blacklisted\n",
8576 trace_num, exit_num);
8577 }
8578 return 0;
8579 }
8580 } else if (JIT_G(hot_side_exit) && zend_jit_trace_exit_is_hot(trace_num, exit_num)) {
8581 return zend_jit_trace_hot_side(execute_data, trace_num, exit_num);
8582 }
8583
8584 /* Return 1 to call original handler instead of the same JIT-ed trace */
8585 return (orig_opline == t->opline && EX(opline) == orig_opline);
8586 }
8587
zend_jit_trace_supported(const zend_op * opline)8588 static zend_always_inline uint8_t zend_jit_trace_supported(const zend_op *opline)
8589 {
8590 switch (opline->opcode) {
8591 case ZEND_CATCH:
8592 case ZEND_FAST_CALL:
8593 case ZEND_FAST_RET:
8594 return ZEND_JIT_TRACE_UNSUPPORTED;
8595 default:
8596 return ZEND_JIT_TRACE_SUPPORTED;
8597 }
8598 }
8599
zend_jit_restart_hot_trace_counters(zend_op_array * op_array)8600 static int zend_jit_restart_hot_trace_counters(zend_op_array *op_array)
8601 {
8602 zend_jit_op_array_trace_extension *jit_extension;
8603 uint32_t i;
8604
8605 jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
8606 for (i = 0; i < op_array->last; i++) {
8607 jit_extension->trace_info[i].trace_flags &=
8608 ZEND_JIT_TRACE_START_LOOP | ZEND_JIT_TRACE_START_ENTER | ZEND_JIT_TRACE_UNSUPPORTED;
8609 if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_LOOP) {
8610 op_array->opcodes[i].handler = (const void*)zend_jit_loop_trace_counter_handler;
8611 } else if (jit_extension->trace_info[i].trace_flags == ZEND_JIT_TRACE_START_ENTER) {
8612 op_array->opcodes[i].handler = (const void*)zend_jit_func_trace_counter_handler;
8613 } else {
8614 op_array->opcodes[i].handler = jit_extension->trace_info[i].orig_handler;
8615 }
8616 }
8617 return SUCCESS;
8618 }
8619
zend_jit_setup_hot_trace_counters(zend_op_array * op_array)8620 static int zend_jit_setup_hot_trace_counters(zend_op_array *op_array)
8621 {
8622 zend_op *opline;
8623 zend_jit_op_array_trace_extension *jit_extension;
8624 uint32_t i;
8625
8626 ZEND_ASSERT(sizeof(zend_op_trace_info) == sizeof(zend_op));
8627
8628 jit_extension = (zend_jit_op_array_trace_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_trace_extension) + (op_array->last - 1) * sizeof(zend_op_trace_info));
8629 if (!jit_extension) {
8630 return FAILURE;
8631 }
8632 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
8633 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_TRACE;
8634 jit_extension->op_array = op_array;
8635 jit_extension->offset = (char*)jit_extension->trace_info - (char*)op_array->opcodes;
8636 for (i = 0; i < op_array->last; i++) {
8637 jit_extension->trace_info[i].orig_handler = op_array->opcodes[i].handler;
8638 jit_extension->trace_info[i].call_handler = zend_get_opcode_handler_func(&op_array->opcodes[i]);
8639 jit_extension->trace_info[i].counter = NULL;
8640 jit_extension->trace_info[i].trace_flags =
8641 zend_jit_trace_supported(&op_array->opcodes[i]);
8642 }
8643 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
8644
8645 if (JIT_G(hot_loop)) {
8646 zend_cfg cfg;
8647
8648 ZEND_ASSERT(zend_jit_loop_trace_counter_handler != NULL);
8649
8650 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
8651 return FAILURE;
8652 }
8653
8654 for (i = 0; i < cfg.blocks_count; i++) {
8655 if (cfg.blocks[i].flags & ZEND_BB_REACHABLE) {
8656 if (cfg.blocks[i].flags & ZEND_BB_LOOP_HEADER) {
8657 /* loop header */
8658 opline = op_array->opcodes + cfg.blocks[i].start;
8659 if (!(ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags & ZEND_JIT_TRACE_UNSUPPORTED)) {
8660 opline->handler = (const void*)zend_jit_loop_trace_counter_handler;
8661 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter) {
8662 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8663 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8664 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8665 }
8666 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8667 ZEND_JIT_TRACE_START_LOOP;
8668 }
8669 }
8670 }
8671 }
8672 }
8673
8674 if (JIT_G(hot_func)) {
8675 ZEND_ASSERT(zend_jit_func_trace_counter_handler != NULL);
8676 opline = op_array->opcodes;
8677 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
8678 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
8679 opline++;
8680 }
8681 }
8682
8683 if (!ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags) {
8684 /* function entry */
8685 opline->handler = (const void*)zend_jit_func_trace_counter_handler;
8686 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->counter =
8687 &zend_jit_hot_counters[ZEND_JIT_COUNTER_NUM];
8688 ZEND_JIT_COUNTER_NUM = (ZEND_JIT_COUNTER_NUM + 1) % ZEND_HOT_COUNTERS_COUNT;
8689 ZEND_OP_TRACE_INFO(opline, jit_extension->offset)->trace_flags |=
8690 ZEND_JIT_TRACE_START_ENTER;
8691 }
8692 }
8693
8694 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
8695
8696 return SUCCESS;
8697 }
8698
zend_jit_trace_init_caches(void)8699 static void zend_jit_trace_init_caches(void)
8700 {
8701 memset(ZEND_VOIDP(JIT_G(bad_root_cache_opline)), 0, sizeof(JIT_G(bad_root_cache_opline)));
8702 memset(JIT_G(bad_root_cache_count), 0, sizeof(JIT_G(bad_root_cache_count)));
8703 memset(JIT_G(bad_root_cache_stop), 0, sizeof(JIT_G(bad_root_cache_count)));
8704 JIT_G(bad_root_slot) = 0;
8705
8706 if (JIT_G(exit_counters)) {
8707 memset(JIT_G(exit_counters), 0, JIT_G(max_exit_counters));
8708 }
8709 }
8710
zend_jit_trace_reset_caches(void)8711 static void zend_jit_trace_reset_caches(void)
8712 {
8713 JIT_G(tracing) = 0;
8714 #ifdef ZTS
8715 if (!JIT_G(exit_counters)) {
8716 JIT_G(exit_counters) = calloc(JIT_G(max_exit_counters), 1);
8717 }
8718 #endif
8719 }
8720
zend_jit_trace_free_caches(zend_jit_globals * jit_globals)8721 static void zend_jit_trace_free_caches(zend_jit_globals *jit_globals)
8722 {
8723 if (jit_globals->exit_counters) {
8724 free(jit_globals->exit_counters);
8725 }
8726 }
8727
zend_jit_trace_restart(void)8728 static void zend_jit_trace_restart(void)
8729 {
8730 ZEND_JIT_TRACE_NUM = 1;
8731 ZEND_JIT_COUNTER_NUM = 0;
8732 ZEND_JIT_EXIT_NUM = 0;
8733 ZEND_JIT_EXIT_COUNTERS = 0;
8734 ZCSG(jit_counters_stopped) = false;
8735
8736 zend_jit_trace_init_caches();
8737 }
8738
