1 /*
2 * +----------------------------------------------------------------------+
3 * | Zend JIT |
4 * +----------------------------------------------------------------------+
5 * | Copyright (c) The PHP Group |
6 * +----------------------------------------------------------------------+
7 * | This source file is subject to version 3.01 of the PHP license, |
8 * | that is bundled with this package in the file LICENSE, and is |
9 * | available through the world-wide-web at the following url: |
10 * | https://www.php.net/license/3_01.txt |
11 * | If you did not receive a copy of the PHP license and are unable to |
12 * | obtain it through the world-wide-web, please send a note to |
13 * | license@php.net so we can mail you a copy immediately. |
14 * +----------------------------------------------------------------------+
15 * | Authors: Dmitry Stogov <dmitry@php.net> |
16 * +----------------------------------------------------------------------+
17 */
18
19 #include "jit/ir/ir.h"
20 #include "jit/ir/ir_builder.h"
21
22 #if defined(IR_TARGET_X86)
23 # define IR_REG_SP 4 /* IR_REG_RSP */
24 # define IR_REG_FP 5 /* IR_REG_RBP */
25 # define ZREG_FP 6 /* IR_REG_RSI */
26 # define ZREG_IP 7 /* IR_REG_RDI */
27 # define ZREG_FIRST_FPR 8
28 # define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7)) /* all preserved registers */
29 #elif defined(IR_TARGET_X64)
30 # define IR_REG_SP 4 /* IR_REG_RSP */
31 # define IR_REG_FP 5 /* IR_REG_RBP */
32 # define ZREG_FP 14 /* IR_REG_R14 */
33 # define ZREG_IP 15 /* IR_REG_R15 */
34 # define ZREG_FIRST_FPR 16
35 # if defined(_WIN64)
36 # define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15))
37 /*
38 # define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15) | \
39 (1<<(16+6)) | (1<<(16+7)) | (1<<(16+8)) | (1<<(16+9)) | (1<<(16+10)) | \
40 (1<<(16+11)) | (1<<(16+12)) | (1<<(16+13)) | (1<<(16+14)) | (1<<(16+15)))
41 */
42 # else
43 # define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<12) | (1<<13) | (1<<14) | (1<<15)) /* all preserved registers */
44 # endif
45 #elif defined(IR_TARGET_AARCH64)
46 # define IR_REG_SP 31 /* IR_REG_RSP */
47 # define IR_REG_FP 29 /* IR_REG_X29 */
48 # define ZREG_FP 27 /* IR_REG_X27 */
49 # define ZREG_IP 28 /* IR_REG_X28 */
50 # define ZREG_FIRST_FPR 32
51 # define IR_REGSET_PRESERVED ((1<<19) | (1<<20) | (1<<21) | (1<<22) | (1<<23) | \
52 (1<<24) | (1<<25) | (1<<26) | (1<<27) | (1<<28)) /* all preserved registers */
53 #else
54 # error "Unknown IR target"
55 #endif
56
57 #define ZREG_RX ZREG_IP
58
59 #define OPTIMIZE_FOR_SIZE 0
60
61 /* IR builder defines */
62 #undef _ir_CTX
63 #define _ir_CTX (&jit->ctx)
64
65 #undef ir_CONST_ADDR
66 #define ir_CONST_ADDR(_addr) jit_CONST_ADDR(jit, (uintptr_t)(_addr))
67 #define ir_CONST_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), 0)
68 #define ir_CONST_FC_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), IR_FASTCALL_FUNC)
69 #define ir_CAST_FC_FUNC(_addr) ir_fold2(_ir_CTX, IR_OPT(IR_PROTO, IR_ADDR), (_addr), \
70 ir_proto_0(_ir_CTX, IR_FASTCALL_FUNC, IR_I32))
71
72 #define ir_CONST_FUNC_PROTO(_addr, _proto) \
73 jit_CONST_FUNC_PROTO(jit, (uintptr_t)(_addr), (_proto))
74
75 #undef ir_ADD_OFFSET
76 #define ir_ADD_OFFSET(_addr, _offset) \
77 jit_ADD_OFFSET(jit, _addr, _offset)
78
79 #ifdef ZEND_ENABLE_ZVAL_LONG64
80 # define IR_LONG IR_I64
81 # define ir_CONST_LONG ir_CONST_I64
82 # define ir_UNARY_OP_L ir_UNARY_OP_I64
83 # define ir_BINARY_OP_L ir_BINARY_OP_I64
84 # define ir_ADD_L ir_ADD_I64
85 # define ir_SUB_L ir_SUB_I64
86 # define ir_MUL_L ir_MUL_I64
87 # define ir_DIV_L ir_DIV_I64
88 # define ir_MOD_L ir_MOD_I64
89 # define ir_NEG_L ir_NEG_I64
90 # define ir_ABS_L ir_ABS_I64
91 # define ir_SEXT_L ir_SEXT_I64
92 # define ir_ZEXT_L ir_ZEXT_I64
93 # define ir_TRUNC_L ir_TRUNC_I64
94 # define ir_BITCAST_L ir_BITCAST_I64
95 # define ir_FP2L ir_FP2I64
96 # define ir_ADD_OV_L ir_ADD_OV_I64
97 # define ir_SUB_OV_L ir_SUB_OV_I64
98 # define ir_MUL_OV_L ir_MUL_OV_I64
99 # define ir_NOT_L ir_NOT_I64
100 # define ir_OR_L ir_OR_I64
101 # define ir_AND_L ir_AND_I64
102 # define ir_XOR_L ir_XOR_I64
103 # define ir_SHL_L ir_SHL_I64
104 # define ir_SHR_L ir_SHR_I64
105 # define ir_SAR_L ir_SAR_I64
106 # define ir_ROL_L ir_ROL_I64
107 # define ir_ROR_L ir_ROR_I64
108 # define ir_MIN_L ir_MIN_I64
109 # define ir_MAX_L ir_MAX_I64
110 # define ir_LOAD_L ir_LOAD_I64
111 #else
112 # define IR_LONG IR_I32
113 # define ir_CONST_LONG ir_CONST_I32
114 # define ir_UNARY_OP_L ir_UNARY_OP_I32
115 # define ir_BINARY_OP_L ir_BINARY_OP_I32
116 # define ir_ADD_L ir_ADD_I32
117 # define ir_SUB_L ir_SUB_I32
118 # define ir_MUL_L ir_MUL_I32
119 # define ir_DIV_L ir_DIV_I32
120 # define ir_MOD_L ir_MOD_I32
121 # define ir_NEG_L ir_NEG_I32
122 # define ir_ABS_L ir_ABS_I32
123 # define ir_SEXT_L ir_SEXT_I32
124 # define ir_ZEXT_L ir_ZEXT_I32
125 # define ir_TRUNC_L ir_TRUNC_I32
126 # define ir_BITCAST_L ir_BITCAST_I32
127 # define ir_FP2L ir_FP2I32
128 # define ir_ADD_OV_L ir_ADD_OV_I32
129 # define ir_SUB_OV_L ir_SUB_OV_I32
130 # define ir_MUL_OV_L ir_MUL_OV_I32
131 # define ir_NOT_L ir_NOT_I32
132 # define ir_OR_L ir_OR_I32
133 # define ir_AND_L ir_AND_I32
134 # define ir_XOR_L ir_XOR_I32
135 # define ir_SHL_L ir_SHL_I32
136 # define ir_SHR_L ir_SHR_I32
137 # define ir_SAR_L ir_SAR_I32
138 # define ir_ROL_L ir_ROL_I32
139 # define ir_ROR_L ir_ROR_I32
140 # define ir_MIN_L ir_MIN_I32
141 # define ir_MAX_L ir_MAX_I32
142 # define ir_LOAD_L ir_LOAD_I32
143 #endif
144
145 /* A helper structure to collect IT rers for the following use in (MERGE/PHI)_N */
146 typedef struct _ir_refs {
147 uint32_t count;
148 uint32_t limit;
149 ir_ref refs[] ZEND_ELEMENT_COUNT(count);
150 } ir_refs;
151
152 #define ir_refs_size(_n) (offsetof(ir_refs, refs) + sizeof(ir_ref) * (_n))
153 #define ir_refs_init(_name, _n) _name = alloca(ir_refs_size(_n)); \
154 do {_name->count = 0; _name->limit = (_n);} while (0)
155
ir_refs_add(ir_refs * refs,ir_ref ref)156 static void ir_refs_add(ir_refs *refs, ir_ref ref)
157 {
158 ir_ref *ptr;
159
160 ZEND_ASSERT(refs->count < refs->limit);
161 ptr = refs->refs;
162 ptr[refs->count++] = ref;
163 }
164
165 static size_t zend_jit_trace_prologue_size = (size_t)-1;
166 #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
167 static uint32_t allowed_opt_flags = 0;
168 static uint32_t default_mflags = 0;
169 #endif
170 static bool delayed_call_chain = 0; // TODO: remove this var (use jit->delayed_call_level) ???
171
172 #ifdef ZTS
173 # ifdef _WIN32
174 extern uint32_t _tls_index;
175 extern char *_tls_start;
176 extern char *_tls_end;
177 # endif
178
179 static size_t tsrm_ls_cache_tcb_offset = 0;
180 static size_t tsrm_tls_index = 0;
181 static size_t tsrm_tls_offset = 0;
182
183 # define EG_TLS_OFFSET(field) \
184 (executor_globals_offset + offsetof(zend_executor_globals, field))
185
186 # define CG_TLS_OFFSET(field) \
187 (compiler_globals_offset + offsetof(zend_compiler_globals, field))
188
189 # define jit_EG(_field) \
190 ir_ADD_OFFSET(jit_TLS(jit), EG_TLS_OFFSET(_field))
191
192 # define jit_CG(_field) \
193 ir_ADD_OFFSET(jit_TLS(jit), CG_TLS_OFFSET(_field))
194
195 #else
196
197 # define jit_EG(_field) \
198 ir_CONST_ADDR(&EG(_field))
199
200 # define jit_CG(_field) \
201 ir_CONST_ADDR(&CG(_field))
202
203 #endif
204
205 #define jit_CALL(_call, _field) \
206 ir_ADD_OFFSET(_call, offsetof(zend_execute_data, _field))
207
208 #define jit_EX(_field) \
209 jit_CALL(jit_FP(jit), _field)
210
211 #define jit_RX(_field) \
212 jit_CALL(jit_IP(jit), _field)
213
214 #define JIT_STUBS(_) \
215 _(exception_handler, IR_SKIP_PROLOGUE) \
216 _(exception_handler_undef, IR_SKIP_PROLOGUE) \
217 _(exception_handler_free_op2, IR_SKIP_PROLOGUE) \
218 _(exception_handler_free_op1_op2, IR_SKIP_PROLOGUE) \
219 _(interrupt_handler, IR_SKIP_PROLOGUE) \
220 _(leave_function_handler, IR_SKIP_PROLOGUE) \
221 _(negative_shift, IR_SKIP_PROLOGUE) \
222 _(mod_by_zero, IR_SKIP_PROLOGUE) \
223 _(invalid_this, IR_SKIP_PROLOGUE) \
224 _(undefined_function, IR_SKIP_PROLOGUE) \
225 _(throw_cannot_pass_by_ref, IR_SKIP_PROLOGUE) \
226 _(icall_throw, IR_SKIP_PROLOGUE) \
227 _(leave_throw, IR_SKIP_PROLOGUE) \
228 _(hybrid_runtime_jit, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
229 _(hybrid_profile_jit, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
230 _(hybrid_func_hot_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
231 _(hybrid_loop_hot_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
232 _(hybrid_func_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
233 _(hybrid_ret_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
234 _(hybrid_loop_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
235 _(trace_halt, IR_SKIP_PROLOGUE) \
236 _(trace_escape, IR_SKIP_PROLOGUE) \
237 _(trace_exit, IR_SKIP_PROLOGUE) \
238 _(undefined_offset, IR_FUNCTION | IR_FASTCALL_FUNC) \
239 _(undefined_key, IR_FUNCTION | IR_FASTCALL_FUNC) \
240 _(cannot_add_element, IR_FUNCTION | IR_FASTCALL_FUNC) \
241 _(assign_const, IR_FUNCTION | IR_FASTCALL_FUNC) \
242 _(assign_tmp, IR_FUNCTION | IR_FASTCALL_FUNC) \
243 _(assign_var, IR_FUNCTION | IR_FASTCALL_FUNC) \
244 _(assign_cv_noref, IR_FUNCTION | IR_FASTCALL_FUNC) \
245 _(assign_cv, IR_FUNCTION | IR_FASTCALL_FUNC) \
246 _(new_array, IR_FUNCTION | IR_FASTCALL_FUNC) \
247
248 #define JIT_STUB_ID(name, flags) \
249 jit_stub_ ## name,
250
251 #define JIT_STUB_FORWARD(name, flags) \
252 static int zend_jit_ ## name ## _stub(zend_jit_ctx *jit);
253
254 #define JIT_STUB(name, flags) \
255 {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub, flags},
256
257 typedef enum _jit_stub_id {
258 JIT_STUBS(JIT_STUB_ID)
259 jit_last_stub
260 } jit_stub_id;
261
262 typedef struct _zend_jit_reg_var {
263 ir_ref ref;
264 uint32_t flags;
265 } zend_jit_reg_var;
266
267 typedef struct _zend_jit_ctx {
268 ir_ctx ctx;
269 const zend_op *last_valid_opline;
270 bool use_last_valid_opline;
271 bool track_last_valid_opline;
272 bool reuse_ip;
273 uint32_t delayed_call_level;
274 int b; /* current basic block number or -1 */
275 #ifdef ZTS
276 ir_ref tls;
277 #endif
278 ir_ref fp;
279 ir_ref trace_loop_ref;
280 ir_ref return_inputs;
281 const zend_op_array *op_array;
282 const zend_op_array *current_op_array;
283 zend_ssa *ssa;
284 zend_string *name;
285 ir_ref *bb_start_ref; /* PHP BB -> IR ref mapping */
286 ir_ref *bb_predecessors; /* PHP BB -> index in bb_edges -> IR refs of predessors */
287 ir_ref *bb_edges;
288 zend_jit_trace_info *trace;
289 zend_jit_reg_var *ra;
290 int delay_var;
291 ir_refs *delay_refs;
292 ir_ref eg_exception_addr;
293 HashTable addr_hash;
294 ir_ref stub_addr[jit_last_stub];
295 } zend_jit_ctx;
296
297 typedef int8_t zend_reg;
298
299 typedef struct _zend_jit_registers_buf {
300 #if defined(IR_TARGET_X64)
301 uint64_t gpr[16]; /* general purpose integer register */
302 double fpr[16]; /* floating point registers */
303 #elif defined(IR_TARGET_X86)
304 uint32_t gpr[8]; /* general purpose integer register */
305 double fpr[8]; /* floating point registers */
306 #elif defined (IR_TARGET_AARCH64)
307 uint64_t gpr[32]; /* general purpose integer register */
308 double fpr[32]; /* floating point registers */
309 #else
310 # error "Unknown IR target"
311 #endif
312 } zend_jit_registers_buf;
313
314 /* Keep 32 exit points in a single code block */
315 #define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes
316 #define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points
317
318 static uint32_t zend_jit_exit_point_by_addr(const void *addr);
319 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs);
320
321 static int zend_jit_assign_to_variable(zend_jit_ctx *jit,
322 const zend_op *opline,
323 zend_jit_addr var_use_addr,
324 zend_jit_addr var_addr,
325 uint32_t var_info,
326 uint32_t var_def_info,
327 uint8_t val_type,
328 zend_jit_addr val_addr,
329 uint32_t val_info,
330 zend_jit_addr res_addr,
331 zend_jit_addr ref_addr,
332 bool check_exception);
333
334 typedef struct _zend_jit_stub {
335 const char *name;
336 int (*stub)(zend_jit_ctx *jit);
337 uint32_t flags;
338 } zend_jit_stub;
339
340 JIT_STUBS(JIT_STUB_FORWARD)
341
342 static const zend_jit_stub zend_jit_stubs[] = {
343 JIT_STUBS(JIT_STUB)
344 };
345
346 #if defined(_WIN32) || defined(IR_TARGET_AARCH64)
347 /* We keep addresses in SHM to share them between sepaeate processes (on Windows) or to support veneers (on AArch64) */
348 static void** zend_jit_stub_handlers = NULL;
349 #else
350 static void* zend_jit_stub_handlers[sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])];
351 #endif
352
353 #if defined(IR_TARGET_AARCH64)
354
355 # ifdef __FreeBSD__
356 /* https://github.com/freebsd/freebsd-src/blob/c52ca7dd09066648b1cc40f758289404d68ab886/libexec/rtld-elf/aarch64/reloc.c#L180-L184 */
357 typedef struct TLSDescriptor {
358 void* thunk;
359 int index;
360 size_t offset;
361 } TLSDescriptor;
362 # endif
363
364 #define IR_HAS_VENEERS (1U<<31) /* IR_RESERVED_FLAG_1 */
365
zend_jit_get_veneer(ir_ctx * ctx,const void * addr)366 static const void *zend_jit_get_veneer(ir_ctx *ctx, const void *addr)
367 {
368 int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
369
370 for (i = 0; i < count; i++) {
371 if (zend_jit_stub_handlers[i] == addr) {
372 return zend_jit_stub_handlers[count + i];
373 }
374 }
375
376 if (((zend_jit_ctx*)ctx)->trace
377 && (void*)addr >= dasm_buf && (void*)addr < dasm_end) {
378 uint32_t exit_point = zend_jit_exit_point_by_addr(addr);
379
380 if (exit_point != (uint32_t)-1) {
381 zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;
382
383 ZEND_ASSERT(exit_point < t->exit_count);
384 return (const void*)((char*)ctx->deoptimization_exits_base + (exit_point * 4));
385 }
386 }
387
388 return NULL;
389 }
390
zend_jit_set_veneer(ir_ctx * ctx,const void * addr,const void * veneer)391 static bool zend_jit_set_veneer(ir_ctx *ctx, const void *addr, const void *veneer)
392 {
393 int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
394 uint32_t exit_point = zend_jit_exit_point_by_addr(addr);
395
396 if (exit_point != (uint32_t)-1) {
397 return 1;
398 }
399 for (i = 0; i < count; i++) {
400 if (zend_jit_stub_handlers[i] == addr) {
401 const void **ptr = (const void**)&zend_jit_stub_handlers[count + i];
402 *ptr = veneer;
403 ctx->flags2 |= IR_HAS_VENEERS;
404 #ifdef HAVE_CAPSTONE
405 int64_t offset;
406 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
407 const char *name = ir_disasm_find_symbol((uint64_t)(uintptr_t)addr, &offset);
408
409 if (name && !offset) {
410 if (strstr(name, "@veneer") == NULL) {
411 char *new_name;
412
413 zend_spprintf(&new_name, 0, "%s@veneer", name);
414 ir_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4);
415 efree(new_name);
416 } else {
417 ir_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4);
418 }
419 }
420 }
421 #endif
422 return 1;
423 }
424 }
425
426 return 0;
427 }
428
zend_jit_commit_veneers(void)429 static void zend_jit_commit_veneers(void)
430 {
431 int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
432
433 for (i = 0; i < count; i++) {
434 if (zend_jit_stub_handlers[count + i]) {
435 zend_jit_stub_handlers[i] = zend_jit_stub_handlers[count + i];
436 zend_jit_stub_handlers[count + i] = NULL;
437 }
438 }
439 }
440 #endif
441
zend_jit_prefer_const_addr_load(zend_jit_ctx * jit,uintptr_t addr)442 static bool zend_jit_prefer_const_addr_load(zend_jit_ctx *jit, uintptr_t addr)
443 {
444 #if defined(IR_TARGET_X86)
445 return 0; /* always use immediate value */
446 #elif defined(IR_TARGET_X64)
447 return addr > 0xffffffff; /* prefer loading long constant from memery */
448 #elif defined(IR_TARGET_AARCH64)
449 return addr > 0xffff;
450 #else
451 # error "Unknown IR target"
452 #endif
453 }
454
zend_reg_name(int8_t reg)455 static const char* zend_reg_name(int8_t reg)
456 {
457 return ir_reg_name(reg, ir_reg_is_int(reg) ? IR_LONG : IR_DOUBLE);
458 }
459
460 /* IR helpers */
461
462 #ifdef ZTS
jit_TLS(zend_jit_ctx * jit)463 static ir_ref jit_TLS(zend_jit_ctx *jit)
464 {
465 ZEND_ASSERT(jit->ctx.control);
466 if (jit->tls) {
467 /* Emit "TLS" once for basic block */
468 ir_insn *insn;
469 ir_ref ref = jit->ctx.control;
470
471 while (1) {
472 if (ref == jit->tls) {
473 return jit->tls;
474 }
475 insn = &jit->ctx.ir_base[ref];
476 if (insn->op >= IR_START || insn->op == IR_CALL) {
477 break;
478 }
479 ref = insn->op1;
480 }
481 }
482 jit->tls = ir_TLS(
483 tsrm_ls_cache_tcb_offset ? tsrm_ls_cache_tcb_offset : tsrm_tls_index,
484 tsrm_ls_cache_tcb_offset ? IR_NULL : tsrm_tls_offset);
485 return jit->tls;
486 }
487 #endif
488
jit_CONST_ADDR(zend_jit_ctx * jit,uintptr_t addr)489 static ir_ref jit_CONST_ADDR(zend_jit_ctx *jit, uintptr_t addr)
490 {
491 ir_ref ref;
492 zval *zv;
493
494 if (addr == 0) {
495 return IR_NULL;
496 }
497 zv = zend_hash_index_lookup(&jit->addr_hash, addr);
498 if (Z_TYPE_P(zv) == IS_LONG) {
499 ref = Z_LVAL_P(zv);
500 ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_ADDR, IR_ADDR));
501 } else {
502 ref = ir_unique_const_addr(&jit->ctx, addr);
503 ZVAL_LONG(zv, ref);
504 }
505 return ref;
506 }
507
jit_CONST_FUNC_PROTO(zend_jit_ctx * jit,uintptr_t addr,ir_ref proto)508 static ir_ref jit_CONST_FUNC_PROTO(zend_jit_ctx *jit, uintptr_t addr, ir_ref proto)
509 {
510 ir_ref ref;
511 ir_insn *insn;
512 zval *zv;
513
514 ZEND_ASSERT(addr != 0);
515 zv = zend_hash_index_lookup(&jit->addr_hash, addr);
516 if (Z_TYPE_P(zv) == IS_LONG) {
517 ref = Z_LVAL_P(zv);
518 ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_FUNC_ADDR, IR_ADDR) && jit->ctx.ir_base[ref].proto == proto);
519 } else {
520 ref = ir_unique_const_addr(&jit->ctx, addr);
521 insn = &jit->ctx.ir_base[ref];
522 insn->optx = IR_OPT(IR_FUNC_ADDR, IR_ADDR);
523 insn->proto = proto;
524 ZVAL_LONG(zv, ref);
525 }
526 return ref;
527 }
528
jit_CONST_FUNC(zend_jit_ctx * jit,uintptr_t addr,uint16_t flags)529 static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags)
530 {
531 #if defined(IR_TARGET_X86)
532 /* TODO: dummy prototype (only flags matter) ??? */
533 ir_ref proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0;
534 #else
535 ir_ref proto = 0;
536 #endif
537
538 return jit_CONST_FUNC_PROTO(jit, addr, proto);
539 }
540
jit_ADD_OFFSET(zend_jit_ctx * jit,ir_ref addr,uintptr_t offset)541 static ir_ref jit_ADD_OFFSET(zend_jit_ctx *jit, ir_ref addr, uintptr_t offset)
542 {
543 if (offset) {
544 addr = ir_ADD_A(addr, ir_CONST_ADDR(offset));
545 }
546 return addr;
547 }
548
jit_EG_exception(zend_jit_ctx * jit)549 static ir_ref jit_EG_exception(zend_jit_ctx *jit)
550 {
551 #ifdef ZTS
552 return jit_EG(exception);
553 #else
554 ir_ref ref = jit->eg_exception_addr;
555
556 if (UNEXPECTED(!ref)) {
557 ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)&EG(exception));
558 jit->eg_exception_addr = ref;
559 }
560 return ref;
561 #endif
562 }
563
jit_STUB_ADDR(zend_jit_ctx * jit,jit_stub_id id)564 static ir_ref jit_STUB_ADDR(zend_jit_ctx *jit, jit_stub_id id)
565 {
566 ir_ref ref = jit->stub_addr[id];
567
568 if (UNEXPECTED(!ref)) {
569 ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]);
570 jit->stub_addr[id] = ref;
571 }
572 return ref;
573 }
574
jit_STUB_FUNC_ADDR(zend_jit_ctx * jit,jit_stub_id id,uint16_t flags)575 static ir_ref jit_STUB_FUNC_ADDR(zend_jit_ctx *jit, jit_stub_id id, uint16_t flags)
576 {
577 ir_ref ref = jit->stub_addr[id];
578 ir_insn *insn;
579
580 if (UNEXPECTED(!ref)) {
581 ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]);
582 insn = &jit->ctx.ir_base[ref];
583 insn->optx = IR_OPT(IR_FUNC_ADDR, IR_ADDR);
584 #if defined(IR_TARGET_X86)
585 /* TODO: dummy prototype (only flags matter) ??? */
586 insn->proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0;
587 #else
588 insn->proto = 0;
589 #endif
590 jit->stub_addr[id] = ref;
591 }
592 return ref;
593 }
594
jit_SNAPSHOT(zend_jit_ctx * jit,ir_ref addr)595 static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr)
596 {
597 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) {
598 const void *ptr = (const void*)jit->ctx.ir_base[addr].val.addr;
599 const zend_op_array *op_array = &JIT_G(current_frame)->func->op_array;
600 uint32_t stack_size = op_array->last_var + op_array->T;
601
602 if (ptr == zend_jit_stub_handlers[jit_stub_exception_handler]
603 || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_undef]
604 || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op1_op2]
605 || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op2]
606 || ptr == zend_jit_stub_handlers[jit_stub_interrupt_handler]
607 || ptr == zend_jit_stub_handlers[jit_stub_leave_function_handler]
608 || ptr == zend_jit_stub_handlers[jit_stub_negative_shift]
609 || ptr == zend_jit_stub_handlers[jit_stub_mod_by_zero]
610 || ptr == zend_jit_stub_handlers[jit_stub_invalid_this]
611 || ptr == zend_jit_stub_handlers[jit_stub_undefined_function]
612 || ptr == zend_jit_stub_handlers[jit_stub_throw_cannot_pass_by_ref]
613 || ptr == zend_jit_stub_handlers[jit_stub_icall_throw]
614 || ptr == zend_jit_stub_handlers[jit_stub_leave_throw]
615 || ptr == zend_jit_stub_handlers[jit_stub_trace_halt]
616 || ptr == zend_jit_stub_handlers[jit_stub_trace_escape]) {
617 /* This is a GUARD that trigger exit through a stub code (without deoptimization) */
618 return;
619 }
620
621 /* Check if we need snapshot entries for polymorphic method call */
622 zend_jit_trace_info *t = jit->trace;
623 uint32_t exit_point = 0, n = 0;
624
625 if (addr < 0) {
626 if (t->exit_count > 0
627 && jit->ctx.ir_base[addr].val.u64 == (uintptr_t)zend_jit_trace_get_exit_addr(t->exit_count - 1)) {
628 exit_point = t->exit_count - 1;
629 if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) {
630 n = 2;
631 }
632 }
633 }
634
635 if (stack_size || n) {
636 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
637 uint32_t snapshot_size, i;
638
639 snapshot_size = stack_size;
640 while (snapshot_size > 0) {
641 ir_ref ref = STACK_REF(stack, snapshot_size - 1);
642
643 if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, snapshot_size - 1) & (/*ZREG_LOAD|*/ZREG_STORE))) {
644 snapshot_size--;
645 } else {
646 break;
647 }
648 }
649 if (snapshot_size || n) {
650 ir_ref snapshot;
651
652 snapshot = ir_SNAPSHOT(snapshot_size + n);
653 for (i = 0; i < snapshot_size; i++) {
654 ir_ref ref = STACK_REF(stack, i);
655
656 if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, i) & (/*ZREG_LOAD|*/ZREG_STORE))) {
657 ref = IR_UNUSED;
658 }
659 ir_SNAPSHOT_SET_OP(snapshot, i + 1, ref);
660 }
661 if (n) {
662 ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func_ref);
663 ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this_ref);
664 }
665 }
666 }
667 }
668 }
669
_add_trace_const(zend_jit_trace_info * t,int64_t val)670 static int32_t _add_trace_const(zend_jit_trace_info *t, int64_t val)
671 {
672 int32_t i;
673
674 for (i = 0; i < t->consts_count; i++) {
675 if (t->constants[i].i == val) {
676 return i;
677 }
678 }
679 ZEND_ASSERT(i < 0x7fffffff);
680 t->consts_count = i + 1;
681 t->constants = erealloc(t->constants, (i + 1) * sizeof(zend_jit_exit_const));
682 t->constants[i].i = val;
683 return i;
684 }
685
zend_jit_duplicate_exit_point(ir_ctx * ctx,zend_jit_trace_info * t,uint32_t exit_point,ir_ref snapshot_ref)686 uint32_t zend_jit_duplicate_exit_point(ir_ctx *ctx, zend_jit_trace_info *t, uint32_t exit_point, ir_ref snapshot_ref)
687 {
688 uint32_t stack_size, stack_offset;
689 uint32_t new_exit_point = t->exit_count;
690
691 if (new_exit_point >= ZEND_JIT_TRACE_MAX_EXITS) {
692 ctx->status = -ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
693 return exit_point;
694 }
695
696 t->exit_count++;
697 memcpy(&t->exit_info[new_exit_point], &t->exit_info[exit_point], sizeof(zend_jit_trace_exit_info));
698 stack_size = t->exit_info[new_exit_point].stack_size;
699 if (stack_size != 0) {
700 stack_offset = t->stack_map_size;
701 t->stack_map_size += stack_size;
702 // TODO: reduce number of reallocations ???
703 t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
704 memcpy(t->stack_map + stack_offset, t->stack_map + t->exit_info[new_exit_point].stack_offset, stack_size * sizeof(zend_jit_trace_stack));
705 t->exit_info[new_exit_point].stack_offset = stack_offset;
706 }
707 t->exit_info[new_exit_point].flags &= ~ZEND_JIT_EXIT_FIXED;
708
709 return new_exit_point;
710 }
711
zend_jit_snapshot_handler(ir_ctx * ctx,ir_ref snapshot_ref,ir_insn * snapshot,void * addr)712 void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, void *addr)
713 {
714 zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;
715 uint32_t exit_point, exit_flags;
716 ir_ref n = snapshot->inputs_count;
717 ir_ref i;
718
719 exit_point = zend_jit_exit_point_by_addr(addr);
720 ZEND_ASSERT(exit_point < t->exit_count);
721 exit_flags = t->exit_info[exit_point].flags;
722
723 if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) {
724 int8_t *reg_ops = ctx->regs[snapshot_ref];
725
726 ZEND_ASSERT(reg_ops[n - 1] != -1 && reg_ops[n] != -1);
727 if ((exit_flags & ZEND_JIT_EXIT_FIXED)
728 && (t->exit_info[exit_point].poly_func_reg != reg_ops[n - 1]
729 || t->exit_info[exit_point].poly_this_reg != reg_ops[n])) {
730 exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
731 addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
732 exit_flags &= ~ZEND_JIT_EXIT_FIXED;
733 }
734 t->exit_info[exit_point].poly_func_reg = reg_ops[n - 1];
735 t->exit_info[exit_point].poly_this_reg = reg_ops[n];
736 n -= 2;
737 }
738
739 for (i = 2; i <= n; i++) {
740 ir_ref ref = ir_insn_op(snapshot, i);
741
742 if (ref) {
743 int8_t *reg_ops = ctx->regs[snapshot_ref];
744 int8_t reg = reg_ops[i];
745 ir_ref var = i - 2;
746
747 ZEND_ASSERT(var < t->exit_info[exit_point].stack_size);
748 if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags == ZREG_ZVAL_COPY) {
749 ZEND_ASSERT(reg != ZREG_NONE);
750 if ((exit_flags & ZEND_JIT_EXIT_FIXED)
751 && t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)) {
752 exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
753 addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
754 exit_flags &= ~ZEND_JIT_EXIT_FIXED;
755 }
756 t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);
757 } else if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_CONST) {
758 ZEND_ASSERT(t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_LONG ||
759 t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_DOUBLE);
760
761 if (ref > 0) {
762 if (reg != ZREG_NONE) {
763 if (reg & IR_REG_SPILL_LOAD) {
764 ZEND_ASSERT(!(reg & IR_REG_SPILL_SPECIAL));
765 /* spill slot on a CPU stack */
766 if ((exit_flags & ZEND_JIT_EXIT_FIXED)
767 && (t->stack_map[t->exit_info[exit_point].stack_offset + var].ref != ref
768 || t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
769 || !(t->stack_map[t->exit_info[exit_point].stack_offset + var].flags & ZREG_SPILL_SLOT))) {
770 exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
771 addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
772 exit_flags &= ~ZEND_JIT_EXIT_FIXED;
773 }
774 t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = ref;
775 t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;
776 t->stack_map[t->exit_info[exit_point].stack_offset + var].flags |= ZREG_SPILL_SLOT;
777 } else if (reg & IR_REG_SPILL_SPECIAL) {
778 /* spill slot on a VM stack */
779 if ((exit_flags & ZEND_JIT_EXIT_FIXED)
780 && (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
781 || t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) {
782 exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
783 addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
784 exit_flags &= ~ZEND_JIT_EXIT_FIXED;
785 }
786 t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;
787 t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;
788 } else {
789 if ((exit_flags & ZEND_JIT_EXIT_FIXED)
790 && t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)) {
791 exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
792 addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
793 exit_flags &= ~ZEND_JIT_EXIT_FIXED;
794 }
795 t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);
796 }
797 } else {
798 if ((exit_flags & ZEND_JIT_EXIT_FIXED)
799 && (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
800 || t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) {
801 exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
802 addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
803 exit_flags &= ~ZEND_JIT_EXIT_FIXED;
804 }
805 t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;
806 }
807 } else if (!(exit_flags & ZEND_JIT_EXIT_FIXED)) {
808 int32_t idx = _add_trace_const(t, ctx->ir_base[ref].val.i64);
809 t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_CONST;
810 t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = idx;
811 }
812 }
813 }
814 }
815 t->exit_info[exit_point].flags |= ZEND_JIT_EXIT_FIXED;
816 return addr;
817 }
818
jit_SIDE_EXIT(zend_jit_ctx * jit,ir_ref addr)819 static void jit_SIDE_EXIT(zend_jit_ctx *jit, ir_ref addr)
820 {
821 jit_SNAPSHOT(jit, addr);
822 ir_IJMP(addr);
823 }
824
825 /* PHP JIT helpers */
826
jit_EMALLOC(zend_jit_ctx * jit,size_t size,const zend_op_array * op_array,const zend_op * opline)827 static ir_ref jit_EMALLOC(zend_jit_ctx *jit, size_t size, const zend_op_array *op_array, const zend_op *opline)
828 {
829 #if ZEND_DEBUG
830 return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_emalloc),
831 ir_CONST_ADDR(size),
832 op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL,
833 ir_CONST_U32(opline ? opline->lineno : 0),
834 IR_NULL,
835 ir_CONST_U32(0));
836 #elif defined(HAVE_BUILTIN_CONSTANT_P)
837 if (size > 24 && size <= 32) {
838 return ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(_emalloc_32));
839 } else {
840 return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size));
841 }
842 #else
843 return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size));
844 #endif
845 }
846
jit_EFREE(zend_jit_ctx * jit,ir_ref ptr,size_t size,const zend_op_array * op_array,const zend_op * opline)847 static ir_ref jit_EFREE(zend_jit_ctx *jit, ir_ref ptr, size_t size, const zend_op_array *op_array, const zend_op *opline)
848 {
849 #if ZEND_DEBUG
850 return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_efree),
851 ptr,
852 op_array && op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL,
853 ir_CONST_U32(opline ? opline->lineno : 0),
854 IR_NULL,
855 ir_CONST_U32(0));
856 #elif defined(HAVE_BUILTIN_CONSTANT_P)
857 if (size > 24 && size <= 32) {
858 return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree_32), ptr);
859 } else {
860 return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr);
861 }
862 #else
863 return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr);
864 #endif
865 }
866
jit_FP(zend_jit_ctx * jit)867 static ir_ref jit_FP(zend_jit_ctx *jit)
868 {
869 ZEND_ASSERT(jit->ctx.control);
870 if (jit->fp == IR_UNUSED) {
871 /* Emit "RLOAD FP" once for basic block */
872 jit->fp = ir_RLOAD_A(ZREG_FP);
873 } else {
874 ir_insn *insn;
875 ir_ref ref = jit->ctx.control;
876
877 while (1) {
878 if (ref == jit->fp) {
879 break;
880 }
881 insn = &jit->ctx.ir_base[ref];
882 if (insn->op >= IR_START || insn->op == IR_CALL) {
883 jit->fp = ir_RLOAD_A(ZREG_FP);
884 break;
885 }
886 ref = insn->op1;
887 }
888 }
889 return jit->fp;
890 }
891
jit_STORE_FP(zend_jit_ctx * jit,ir_ref ref)892 static void jit_STORE_FP(zend_jit_ctx *jit, ir_ref ref)
893 {
894 ir_RSTORE(ZREG_FP, ref);
895 jit->fp = IR_UNUSED;
896 }
897
jit_IP(zend_jit_ctx * jit)898 static ir_ref jit_IP(zend_jit_ctx *jit)
899 {
900 return ir_RLOAD_A(ZREG_IP);
901 }
902
jit_STORE_IP(zend_jit_ctx * jit,ir_ref ref)903 static void jit_STORE_IP(zend_jit_ctx *jit, ir_ref ref)
904 {
905 ir_RSTORE(ZREG_IP, ref);
906 }
907
jit_IP32(zend_jit_ctx * jit)908 static ir_ref jit_IP32(zend_jit_ctx *jit)
909 {
910 return ir_RLOAD_U32(ZREG_IP);
911 }
912
jit_LOAD_IP(zend_jit_ctx * jit,ir_ref ref)913 static void jit_LOAD_IP(zend_jit_ctx *jit, ir_ref ref)
914 {
915 if (GCC_GLOBAL_REGS) {
916 jit_STORE_IP(jit, ref);
917 } else {
918 ir_STORE(jit_EX(opline), ref);
919 }
920 }
921
jit_LOAD_IP_ADDR(zend_jit_ctx * jit,const zend_op * target)922 static void jit_LOAD_IP_ADDR(zend_jit_ctx *jit, const zend_op *target)
923 {
924 jit_LOAD_IP(jit, ir_CONST_ADDR(target));
925 }
926
zend_jit_track_last_valid_opline(zend_jit_ctx * jit)927 static void zend_jit_track_last_valid_opline(zend_jit_ctx *jit)
928 {
929 jit->use_last_valid_opline = 0;
930 jit->track_last_valid_opline = 1;
931 }
932
zend_jit_use_last_valid_opline(zend_jit_ctx * jit)933 static void zend_jit_use_last_valid_opline(zend_jit_ctx *jit)
934 {
935 if (jit->track_last_valid_opline) {
936 jit->use_last_valid_opline = 1;
937 jit->track_last_valid_opline = 0;
938 }
939 }
940
zend_jit_trace_uses_initial_ip(zend_jit_ctx * jit)941 static bool zend_jit_trace_uses_initial_ip(zend_jit_ctx *jit)
942 {
943 return jit->use_last_valid_opline;
944 }
945
zend_jit_set_last_valid_opline(zend_jit_ctx * jit,const zend_op * opline)946 static void zend_jit_set_last_valid_opline(zend_jit_ctx *jit, const zend_op *opline)
947 {
948 if (!jit->reuse_ip) {
949 jit->track_last_valid_opline = 1;
950 jit->last_valid_opline = opline;
951 }
952 }
953
zend_jit_reset_last_valid_opline(zend_jit_ctx * jit)954 static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit)
955 {
956 jit->track_last_valid_opline = 0;
957 jit->last_valid_opline = NULL;
958 }
959
zend_jit_start_reuse_ip(zend_jit_ctx * jit)960 static void zend_jit_start_reuse_ip(zend_jit_ctx *jit)
961 {
962 zend_jit_reset_last_valid_opline(jit);
963 jit->reuse_ip = 1;
964 }
965
zend_jit_reuse_ip(zend_jit_ctx * jit)966 static int zend_jit_reuse_ip(zend_jit_ctx *jit)
967 {
968 if (!jit->reuse_ip) {
969 zend_jit_start_reuse_ip(jit);
970 // RX = EX(call);
971 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call)));
972 }
973 return 1;
974 }
975
zend_jit_stop_reuse_ip(zend_jit_ctx * jit)976 static void zend_jit_stop_reuse_ip(zend_jit_ctx *jit)
977 {
978 jit->reuse_ip = 0;
979 }
980
zend_jit_save_call_chain(zend_jit_ctx * jit,uint32_t call_level)981 static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level)
982 {
983 ir_ref rx, call;
984
985 if (call_level == 1) {
986 // JIT: call = NULL;
987 call = IR_NULL;
988 } else {
989 // JIT: call = EX(call);
990 call = ir_LOAD_A(jit_EX(call));
991 }
992
993 rx = jit_IP(jit);
994
995 // JIT: call->prev_execute_data = call;
996 ir_STORE(jit_CALL(rx, prev_execute_data), call);
997
998 // JIT: EX(call) = call;
999 ir_STORE(jit_EX(call), rx);
1000
1001 jit->delayed_call_level = 0;
1002 delayed_call_chain = 0;
1003
1004 return 1;
1005 }
1006
zend_jit_set_ip(zend_jit_ctx * jit,const zend_op * target)1007 static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target)
1008 {
1009 ir_ref ref;
1010 ir_ref addr = IR_UNUSED;
1011
1012 if (jit->delayed_call_level) {
1013 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
1014 return 0;
1015 }
1016 }
1017
1018 if (jit->last_valid_opline) {
1019 zend_jit_use_last_valid_opline(jit);
1020 if (jit->last_valid_opline != target) {
1021 if (GCC_GLOBAL_REGS) {
1022 ref = jit_IP(jit);
1023 } else {
1024 addr = jit_EX(opline);
1025 ref = ir_LOAD_A(addr);
1026 }
1027 if (target > jit->last_valid_opline) {
1028 ref = ir_ADD_OFFSET(ref, (uintptr_t)target - (uintptr_t)jit->last_valid_opline);
1029 } else {
1030 ref = ir_SUB_A(ref, ir_CONST_ADDR((uintptr_t)jit->last_valid_opline - (uintptr_t)target));
1031 }
1032 if (GCC_GLOBAL_REGS) {
1033 jit_STORE_IP(jit, ref);
1034 } else {
1035 ir_STORE(addr, ref);
1036 }
1037 }
1038 } else {
1039 if (GCC_GLOBAL_REGS) {
1040 jit_STORE_IP(jit, ir_CONST_ADDR(target));
1041 } else {
1042 ir_STORE(jit_EX(opline), ir_CONST_ADDR(target));
1043 }
1044 }
1045 jit->reuse_ip = 0;
1046 zend_jit_set_last_valid_opline(jit, target);
1047 return 1;
1048 }
1049
zend_jit_set_ip_ex(zend_jit_ctx * jit,const zend_op * target,bool set_ip_reg)1050 static int zend_jit_set_ip_ex(zend_jit_ctx *jit, const zend_op *target, bool set_ip_reg)
1051 {
1052 if (!GCC_GLOBAL_REGS && set_ip_reg && !jit->last_valid_opline) {
1053 /* Optimization to avoid duplicate constant load */
1054 ir_STORE(jit_EX(opline), ir_HARD_COPY_A(ir_CONST_ADDR(target)));
1055 return 1;
1056 }
1057 return zend_jit_set_ip(jit, target);
1058 }
1059
jit_SET_EX_OPLINE(zend_jit_ctx * jit,const zend_op * target)1060 static void jit_SET_EX_OPLINE(zend_jit_ctx *jit, const zend_op *target)
1061 {
1062 if (jit->last_valid_opline == target) {
1063 zend_jit_use_last_valid_opline(jit);
1064 if (GCC_GLOBAL_REGS) {
1065 // EX(opline) = opline
1066 ir_STORE(jit_EX(opline), jit_IP(jit));
1067 }
1068 } else {
1069 ir_STORE(jit_EX(opline), ir_CONST_ADDR(target));
1070 if (!GCC_GLOBAL_REGS) {
1071 zend_jit_reset_last_valid_opline(jit);
1072 }
1073 }
1074 }
1075
jit_ZVAL_ADDR(zend_jit_ctx * jit,zend_jit_addr addr)1076 static ir_ref jit_ZVAL_ADDR(zend_jit_ctx *jit, zend_jit_addr addr)
1077 {
1078 if (Z_MODE(addr) == IS_MEM_ZVAL) {
1079 ir_ref reg;
1080
1081 if (Z_REG(addr) == ZREG_FP) {
1082 reg = jit_FP(jit);
1083 } else if (Z_REG(addr) == ZREG_RX) {
1084 reg = jit_IP(jit);
1085 } else {
1086 ZEND_UNREACHABLE();
1087 }
1088 return ir_ADD_OFFSET(reg, Z_OFFSET(addr));
1089 } else if (Z_MODE(addr) == IS_REF_ZVAL) {
1090 return Z_IR_REF(addr);
1091 } else {
1092 ZEND_ASSERT(Z_MODE(addr) == IS_CONST_ZVAL);
1093 return ir_CONST_ADDR(Z_ZV(addr));
1094 }
1095 }
1096
jit_Z_TYPE_ref(zend_jit_ctx * jit,ir_ref ref)1097 static ir_ref jit_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref)
1098 {
1099 return ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zval, u1.v.type)));
1100 }
1101
jit_Z_TYPE(zend_jit_ctx * jit,zend_jit_addr addr)1102 static ir_ref jit_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr)
1103 {
1104 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1105 return ir_CONST_U8(Z_TYPE_P(Z_ZV(addr)));
1106 } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
1107 ir_ref reg;
1108
1109 ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
1110 if (Z_REG(addr) == ZREG_FP) {
1111 reg = jit_FP(jit);
1112 } else if (Z_REG(addr) == ZREG_RX) {
1113 reg = jit_IP(jit);
1114 } else {
1115 ZEND_UNREACHABLE();
1116 }
1117 return ir_LOAD_U8(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.v.type)));
1118 } else {
1119 return jit_Z_TYPE_ref(jit, jit_ZVAL_ADDR(jit, addr));
1120 }
1121 }
1122
jit_Z_TYPE_FLAGS_ref(zend_jit_ctx * jit,ir_ref ref)1123 static ir_ref jit_Z_TYPE_FLAGS_ref(zend_jit_ctx *jit, ir_ref ref)
1124 {
1125 return ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zval, u1.v.type_flags)));
1126 }
1127
jit_Z_TYPE_FLAGS(zend_jit_ctx * jit,zend_jit_addr addr)1128 static ir_ref jit_Z_TYPE_FLAGS(zend_jit_ctx *jit, zend_jit_addr addr)
1129 {
1130 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1131 return ir_CONST_U8(Z_TYPE_FLAGS_P(Z_ZV(addr)));
1132 } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
1133 ir_ref reg;
1134
1135 ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
1136 if (Z_REG(addr) == ZREG_FP) {
1137 reg = jit_FP(jit);
1138 } else if (Z_REG(addr) == ZREG_RX) {
1139 reg = jit_IP(jit);
1140 } else {
1141 ZEND_UNREACHABLE();
1142 }
1143 return ir_LOAD_U8(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.v.type_flags)));
1144 } else {
1145 return jit_Z_TYPE_FLAGS_ref(jit, jit_ZVAL_ADDR(jit, addr));
1146 }
1147 }
1148
jit_Z_TYPE_INFO_ref(zend_jit_ctx * jit,ir_ref ref)1149 static ir_ref jit_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref)
1150 {
1151 return ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zval, u1.type_info)));
1152 }
1153
jit_Z_TYPE_INFO(zend_jit_ctx * jit,zend_jit_addr addr)1154 static ir_ref jit_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr)
1155 {
1156 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1157 return ir_CONST_U32(Z_TYPE_INFO_P(Z_ZV(addr)));
1158 } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
1159 ir_ref reg;
1160
1161 ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
1162 if (Z_REG(addr) == ZREG_FP) {
1163 reg = jit_FP(jit);
1164 } else if (Z_REG(addr) == ZREG_RX) {
1165 reg = jit_IP(jit);
1166 } else {
1167 ZEND_UNREACHABLE();
1168 }
1169 return ir_LOAD_U32(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.type_info)));
1170 } else {
1171 return jit_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr));
1172 }
1173 }
1174
jit_set_Z_TYPE_INFO_ref(zend_jit_ctx * jit,ir_ref ref,ir_ref type_info)1175 static void jit_set_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type_info)
1176 {
1177 ir_STORE(ir_ADD_OFFSET(ref, offsetof(zval, u1.type_info)), type_info);
1178 }
1179
jit_set_Z_TYPE_INFO_ex(zend_jit_ctx * jit,zend_jit_addr addr,ir_ref type_info)1180 static void jit_set_Z_TYPE_INFO_ex(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref type_info)
1181 {
1182 if (Z_MODE(addr) == IS_MEM_ZVAL) {
1183 ir_ref reg;
1184
1185 ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
1186 if (Z_REG(addr) == ZREG_FP) {
1187 reg = jit_FP(jit);
1188 } else if (Z_REG(addr) == ZREG_RX) {
1189 reg = jit_IP(jit);
1190 } else {
1191 ZEND_UNREACHABLE();
1192 }
1193 ir_STORE(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.type_info)), type_info);
1194 } else {
1195 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr), type_info);
1196 }
1197 }
1198
jit_set_Z_TYPE_INFO(zend_jit_ctx * jit,zend_jit_addr addr,uint32_t type_info)1199 static void jit_set_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr, uint32_t type_info)
1200 {
1201 if (type_info < IS_STRING
1202 && Z_MODE(addr) == IS_MEM_ZVAL
1203 && Z_REG(addr) == ZREG_FP
1204 && JIT_G(current_frame)
1205 && STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(addr))) == type_info) {
1206 /* type is already set */
1207 return;
1208 }
1209 jit_set_Z_TYPE_INFO_ex(jit, addr, ir_CONST_U32(type_info));
1210 }
1211
jit_if_Z_TYPE_ref(zend_jit_ctx * jit,ir_ref ref,ir_ref type)1212 static ir_ref jit_if_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type)
1213 {
1214 return ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), type));
1215 }
1216
jit_if_Z_TYPE(zend_jit_ctx * jit,zend_jit_addr addr,uint8_t type)1217 static ir_ref jit_if_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type)
1218 {
1219 ZEND_ASSERT(type != IS_UNDEF);
1220 return ir_IF(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type)));
1221 }
1222
jit_if_not_Z_TYPE(zend_jit_ctx * jit,zend_jit_addr addr,uint8_t type)1223 static ir_ref jit_if_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type)
1224 {
1225 ir_ref ref = jit_Z_TYPE(jit, addr);
1226
1227 if (type != IS_UNDEF) {
1228 ref = ir_NE(ref, ir_CONST_U8(type));
1229 }
1230 return ir_IF(ref);
1231 }
1232
jit_guard_Z_TYPE(zend_jit_ctx * jit,zend_jit_addr addr,uint8_t type,const void * exit_addr)1233 static void jit_guard_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_addr)
1234 {
1235 ir_ref ref = jit_Z_TYPE(jit, addr);
1236
1237 if (type != IS_UNDEF) {
1238 ir_GUARD(ir_EQ(ref, ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr));
1239 } else {
1240 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
1241 }
1242 }
1243
jit_guard_not_Z_TYPE(zend_jit_ctx * jit,zend_jit_addr addr,uint8_t type,const void * exit_addr)1244 static void jit_guard_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_addr)
1245 {
1246 ir_ref ref = jit_Z_TYPE(jit, addr);
1247
1248 if (type != IS_UNDEF) {
1249 ref = ir_NE(ref, ir_CONST_U8(type));
1250 }
1251 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
1252 }
1253
jit_if_REFCOUNTED(zend_jit_ctx * jit,zend_jit_addr addr)1254 static ir_ref jit_if_REFCOUNTED(zend_jit_ctx *jit, zend_jit_addr addr)
1255 {
1256 return ir_IF(jit_Z_TYPE_FLAGS(jit, addr));
1257 }
1258
jit_if_COLLECTABLE_ref(zend_jit_ctx * jit,ir_ref addr_ref)1259 static ir_ref jit_if_COLLECTABLE_ref(zend_jit_ctx *jit, ir_ref addr_ref)
1260 {
1261 return ir_IF(ir_AND_U8(jit_Z_TYPE_FLAGS_ref(jit, addr_ref), ir_CONST_U8(IS_TYPE_COLLECTABLE)));
1262 }
1263
jit_Z_LVAL_ref(zend_jit_ctx * jit,ir_ref ref)1264 static ir_ref jit_Z_LVAL_ref(zend_jit_ctx *jit, ir_ref ref)
1265 {
1266 return ir_LOAD_L(ref);
1267 }
1268
jit_Z_DVAL_ref(zend_jit_ctx * jit,ir_ref ref)1269 static ir_ref jit_Z_DVAL_ref(zend_jit_ctx *jit, ir_ref ref)
1270 {
1271 return ir_LOAD_D(ref);
1272 }
1273
zend_jit_spilling_may_cause_conflict(zend_jit_ctx * jit,int var,ir_ref val)1274 static bool zend_jit_spilling_may_cause_conflict(zend_jit_ctx *jit, int var, ir_ref val)
1275 {
1276 if (jit->ctx.ir_base[val].op == IR_RLOAD) {
1277 /* Deoptimization */
1278 return 0;
1279 }
1280 // if (jit->ctx.ir_base[val].op == IR_LOAD
1281 // && jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op == IR_ADD
1282 // && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op == IR_RLOAD
1283 // && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op2 == ZREG_FP
1284 // && IR_IS_CONST_REF(jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2)
1285 // && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr == (uintptr_t)EX_NUM_TO_VAR(jit->ssa->vars[var].var)) {
1286 // /* LOAD from the same location (the LOAD is pinned) */
1287 // // TODO: should be anti-dependent with the following stores ???
1288 // return 0;
1289 // }
1290 if (jit->ssa->vars[var].var < jit->current_op_array->last_var) {
1291 /* IS_CV */
1292 return 0;
1293 }
1294 return 1;
1295 }
1296
zend_jit_def_reg(zend_jit_ctx * jit,zend_jit_addr addr,ir_ref val)1297 static void zend_jit_def_reg(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref val)
1298 {
1299 int var;
1300
1301 ZEND_ASSERT(Z_MODE(addr) == IS_REG);
1302 var = Z_SSA_VAR(addr);
1303 if (var == jit->delay_var) {
1304 ir_refs_add(jit->delay_refs, val);
1305 return;
1306 }
1307 ZEND_ASSERT(jit->ra && jit->ra[var].ref == IR_NULL);
1308
1309 /* Negative "var" has special meaning for IR */
1310 if (val > 0 && !zend_jit_spilling_may_cause_conflict(jit, var, val)) {
1311 val = ir_bind(&jit->ctx, -EX_NUM_TO_VAR(jit->ssa->vars[var].var), val);
1312 }
1313 jit->ra[var].ref = val;
1314
1315 if (jit->ra[var].flags & ZREG_FORWARD) {
1316 zend_ssa_phi *phi = jit->ssa->vars[var].phi_use_chain;
1317 zend_basic_block *bb;
1318 int n, j, *p;
1319 ir_ref *q;
1320
1321 jit->ra[var].flags &= ~ZREG_FORWARD;
1322 while (phi != NULL) {
1323 zend_ssa_phi *dst_phi = phi;
1324 int src_var = var;
1325
1326 if (dst_phi->pi >= 0) {
1327 jit->ra[src_var].ref = val;
1328 src_var = dst_phi->ssa_var;
1329 if (!(jit->ra[src_var].flags & ZREG_FORWARD)) {
1330 phi = zend_ssa_next_use_phi(jit->ssa, var, phi);
1331 continue;
1332 }
1333 dst_phi = jit->ssa->vars[src_var].phi_use_chain;
1334 ZEND_ASSERT(dst_phi != NULL && "reg forwarding");
1335 ZEND_ASSERT(!zend_ssa_next_use_phi(jit->ssa, src_var, dst_phi) && "reg forwarding");
1336 jit->ra[src_var].flags &= ~ZREG_FORWARD;
1337 }
1338
1339 if (jit->ra[dst_phi->ssa_var].ref > 0) {
1340 ir_insn *phi_insn = &jit->ctx.ir_base[jit->ra[dst_phi->ssa_var].ref];
1341 if (phi_insn->op == IR_PHI) {
1342 // ZEND_ASSERT(ir_operands_count(ctx, phi_insn) == n + 1);
1343 bb = &jit->ssa->cfg.blocks[dst_phi->block];
1344 n = bb->predecessors_count;
1345 for (j = 0, p = &dst_phi->sources[0], q = phi_insn->ops + 2; j < n; j++, p++, q++) {
1346 if (*p == src_var) {
1347 *q = val;
1348 }
1349 }
1350 }
1351 }
1352
1353 phi = zend_ssa_next_use_phi(jit->ssa, var, phi);
1354 }
1355 }
1356 }
1357
zend_jit_use_reg(zend_jit_ctx * jit,zend_jit_addr addr)1358 static ir_ref zend_jit_use_reg(zend_jit_ctx *jit, zend_jit_addr addr)
1359 {
1360 int var = Z_SSA_VAR(addr);
1361
1362 ZEND_ASSERT(Z_MODE(addr) == IS_REG);
1363 ZEND_ASSERT(jit->ra && jit->ra[var].ref);
1364 if (jit->ra[var].ref == IR_NULL) {
1365 zend_jit_addr mem_addr;
1366 ir_ref ref;
1367
1368 ZEND_ASSERT(jit->ra[var].flags & ZREG_LOAD);
1369 mem_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(jit->ssa->vars[var].var));
1370 if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_LONG) {
1371 ref = jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr));
1372 } else if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) {
1373 ref = jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr));
1374 } else {
1375 ZEND_UNREACHABLE();
1376 }
1377 zend_jit_def_reg(jit, addr, ref);
1378 return ref;
1379 }
1380 return jit->ra[Z_SSA_VAR(addr)].ref;
1381 }
1382
zend_jit_gen_pi(zend_jit_ctx * jit,zend_ssa_phi * phi)1383 static void zend_jit_gen_pi(zend_jit_ctx *jit, zend_ssa_phi *phi)
1384 {
1385 int src_var = phi->sources[0];
1386 int dst_var = phi->ssa_var;
1387
1388 ZEND_ASSERT(phi->pi >= 0);
1389 ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
1390 ZEND_ASSERT(jit->ra[src_var].ref);
1391
1392 if (jit->ra[src_var].ref == IR_NULL) {
1393 /* Not defined yet */
1394 if (jit->ssa->vars[dst_var].use_chain < 0
1395 && jit->ssa->vars[dst_var].phi_use_chain) {
1396 zend_ssa_phi *phi = jit->ssa->vars[dst_var].phi_use_chain;
1397 if (!zend_ssa_next_use_phi(jit->ssa, dst_var, phi)) {
1398 /* This is a Pi forwarded to Phi */
1399 jit->ra[src_var].flags |= ZREG_FORWARD;
1400 return;
1401 }
1402 }
1403 ZEND_ASSERT(0 && "Not defined Pi source");
1404 }
1405 /* Reuse register */
1406 zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var),
1407 zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)));
1408 }
1409
zend_jit_gen_phi(zend_jit_ctx * jit,zend_ssa_phi * phi)1410 static void zend_jit_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi)
1411 {
1412 int dst_var = phi->ssa_var;
1413 zend_basic_block *bb = &jit->ssa->cfg.blocks[phi->block];
1414 int n = bb->predecessors_count;
1415 int i;
1416 ir_type type = (jit->ssa->var_info[phi->ssa_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE;
1417 ir_ref merge = jit->bb_start_ref[phi->block];
1418 ir_ref ref;
1419 ir_ref old_insns_count = jit->ctx.insns_count;
1420 ir_ref same_src_ref = IR_UNUSED;
1421 bool phi_inputs_are_the_same = 1;
1422
1423 ZEND_ASSERT(phi->pi < 0);
1424 ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
1425 ZEND_ASSERT(merge);
1426 ZEND_ASSERT(jit->ctx.ir_base[merge].op == IR_MERGE || jit->ctx.ir_base[merge].op == IR_LOOP_BEGIN);
1427 ZEND_ASSERT(n == jit->ctx.ir_base[merge].inputs_count);
1428
1429 ref = ir_emit_N(&jit->ctx, IR_OPT(IR_PHI, type), n + 1);
1430 ir_set_op(&jit->ctx, ref, 1, merge);
1431
1432 for (i = 0; i < n; i++) {
1433 int src_var = phi->sources[i];
1434
1435 ZEND_ASSERT(jit->ra[src_var].ref);
1436 if (jit->ra[src_var].ref == IR_NULL) {
1437 jit->ra[src_var].flags |= ZREG_FORWARD;
1438 phi_inputs_are_the_same = 0;
1439 } else {
1440 ir_ref src_ref = zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var));
1441 if (i == 0) {
1442 same_src_ref = src_ref;
1443 } else if (same_src_ref != src_ref) {
1444 phi_inputs_are_the_same = 0;
1445 }
1446 ir_set_op(&jit->ctx, ref, i + 2, src_ref);
1447 }
1448 }
1449 if (phi_inputs_are_the_same) {
1450 ref = same_src_ref;
1451 jit->ctx.insns_count = old_insns_count;
1452 }
1453
1454 zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref);
1455 }
1456
jit_Z_LVAL(zend_jit_ctx * jit,zend_jit_addr addr)1457 static ir_ref jit_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr)
1458 {
1459 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1460 return ir_CONST_LONG(Z_LVAL_P(Z_ZV(addr)));
1461 } else if (Z_MODE(addr) == IS_REG) {
1462 return zend_jit_use_reg(jit, addr);
1463 } else {
1464 return jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, addr));
1465 }
1466 }
1467
jit_set_Z_LVAL(zend_jit_ctx * jit,zend_jit_addr addr,ir_ref lval)1468 static void jit_set_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval)
1469 {
1470 if (Z_MODE(addr) == IS_REG) {
1471 zend_jit_def_reg(jit, addr, lval);
1472 } else {
1473 ir_STORE(jit_ZVAL_ADDR(jit, addr), lval);
1474 }
1475 }
1476
1477 #if SIZEOF_ZEND_LONG == 4
jit_Z_W2(zend_jit_ctx * jit,zend_jit_addr addr)1478 static ir_ref jit_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr)
1479 {
1480 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1481 return ir_CONST_U32((Z_ZV(addr))->value.ww.w2);
1482 } else {
1483 return ir_LOAD_L(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2)));
1484 }
1485 }
1486
jit_set_Z_W2(zend_jit_ctx * jit,zend_jit_addr addr,ir_ref lval)1487 static void jit_set_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval)
1488 {
1489 ir_STORE(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2)), lval);
1490 }
1491 #endif
1492
jit_Z_DVAL(zend_jit_ctx * jit,zend_jit_addr addr)1493 static ir_ref jit_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr)
1494 {
1495 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1496 return ir_CONST_DOUBLE(Z_DVAL_P(Z_ZV(addr)));
1497 } else if (Z_MODE(addr) == IS_REG) {
1498 return zend_jit_use_reg(jit, addr);
1499 } else {
1500 return jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, addr));
1501 }
1502 }
1503
jit_set_Z_DVAL(zend_jit_ctx * jit,zend_jit_addr addr,ir_ref dval)1504 static void jit_set_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref dval)
1505 {
1506 if (Z_MODE(addr) == IS_REG) {
1507 zend_jit_def_reg(jit, addr, dval);
1508 } else {
1509 ir_STORE(jit_ZVAL_ADDR(jit, addr), dval);
1510 }
1511 }
1512
jit_Z_PTR_ref(zend_jit_ctx * jit,ir_ref ref)1513 static ir_ref jit_Z_PTR_ref(zend_jit_ctx *jit, ir_ref ref)
1514 {
1515 return ir_LOAD_A(ref);
1516 }
1517
jit_Z_PTR(zend_jit_ctx * jit,zend_jit_addr addr)1518 static ir_ref jit_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr)
1519 {
1520 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1521 return ir_CONST_ADDR(Z_PTR_P(Z_ZV(addr)));
1522 } else {
1523 return jit_Z_PTR_ref(jit, jit_ZVAL_ADDR(jit, addr));
1524 }
1525 }
1526
jit_set_Z_PTR(zend_jit_ctx * jit,zend_jit_addr addr,ir_ref ptr)1527 static void jit_set_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref ptr)
1528 {
1529 ir_STORE(jit_ZVAL_ADDR(jit, addr), ptr);
1530 }
1531
jit_GC_REFCOUNT(zend_jit_ctx * jit,ir_ref ref)1532 static ir_ref jit_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref)
1533 {
1534 return ir_LOAD_U32(ref);
1535 }
1536
jit_set_GC_REFCOUNT(zend_jit_ctx * jit,ir_ref ref,uint32_t refcount)1537 static void jit_set_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref, uint32_t refcount)
1538 {
1539 ir_STORE(ref, ir_CONST_U32(refcount));
1540 }
1541
jit_GC_ADDREF(zend_jit_ctx * jit,ir_ref ref)1542 static void jit_GC_ADDREF(zend_jit_ctx *jit, ir_ref ref)
1543 {
1544 ir_STORE(ref, ir_ADD_U32(ir_LOAD_U32(ref), ir_CONST_U32(1)));
1545 }
1546
jit_GC_ADDREF2(zend_jit_ctx * jit,ir_ref ref)1547 static void jit_GC_ADDREF2(zend_jit_ctx *jit, ir_ref ref)
1548 {
1549 ir_ref counter = ir_LOAD_U32(ref);
1550 ir_STORE(ref, ir_ADD_U32(counter, ir_CONST_U32(2)));
1551 }
1552
jit_GC_DELREF(zend_jit_ctx * jit,ir_ref ref)1553 static ir_ref jit_GC_DELREF(zend_jit_ctx *jit, ir_ref ref)
1554 {
1555 ir_ref counter = ir_LOAD_U32(ref);
1556 counter = ir_SUB_U32(counter, ir_CONST_U32(1));
1557 ir_STORE(ref, counter);
1558 return counter;
1559 }
1560
jit_if_GC_MAY_NOT_LEAK(zend_jit_ctx * jit,ir_ref ref)1561 static ir_ref jit_if_GC_MAY_NOT_LEAK(zend_jit_ctx *jit, ir_ref ref)
1562 {
1563 return ir_IF(
1564 ir_AND_U32(
1565 ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_refcounted, gc.u.type_info))),
1566 ir_CONST_U32(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT))));
1567 }
1568
jit_ZVAL_COPY_CONST(zend_jit_ctx * jit,zend_jit_addr dst,uint32_t dst_info,uint32_t dst_def_info,zval * zv,bool addref)1569 static void jit_ZVAL_COPY_CONST(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, uint32_t dst_def_info, zval *zv, bool addref)
1570 {
1571 ir_ref ref = IR_UNUSED;
1572
1573 if (Z_TYPE_P(zv) > IS_TRUE) {
1574 if (Z_TYPE_P(zv) == IS_DOUBLE) {
1575 jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE(Z_DVAL_P(zv)));
1576 } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) {
1577 jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE((double)Z_LVAL_P(zv)));
1578 } else if (Z_TYPE_P(zv) == IS_LONG) {
1579 jit_set_Z_LVAL(jit, dst, ir_CONST_LONG(Z_LVAL_P(zv)));
1580 } else {
1581 ref = ir_CONST_ADDR(Z_PTR_P(zv));
1582 jit_set_Z_PTR(jit, dst, ref);
1583 if (addref && Z_REFCOUNTED_P(zv)) {
1584 jit_GC_ADDREF(jit, ref);
1585 }
1586 }
1587 }
1588 if (Z_MODE(dst) != IS_REG) {
1589 if (dst_def_info == MAY_BE_DOUBLE) {
1590 if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
1591 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
1592 }
1593 } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
1594 jit_set_Z_TYPE_INFO(jit, dst, Z_TYPE_INFO_P(zv));
1595 }
1596 }
1597 }
1598
jit_if_TYPED_REF(zend_jit_ctx * jit,ir_ref ref)1599 static ir_ref jit_if_TYPED_REF(zend_jit_ctx *jit, ir_ref ref)
1600 {
1601 return ir_IF(ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_reference, sources.ptr))));
1602 }
1603
jit_ZVAL_COPY(zend_jit_ctx * jit,zend_jit_addr dst,uint32_t dst_info,zend_jit_addr src,uint32_t src_info,bool addref)1604 static void jit_ZVAL_COPY(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, zend_jit_addr src, uint32_t src_info, bool addref)
1605 {
1606 ir_ref ref = IR_UNUSED;
1607
1608 if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
1609 if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
1610 jit_set_Z_LVAL(jit, dst, jit_Z_LVAL(jit, src));
1611 } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
1612 jit_set_Z_DVAL(jit, dst, jit_Z_DVAL(jit, src));
1613 } else {
1614 #if SIZEOF_ZEND_LONG == 4
1615 if (src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) {
1616 jit_set_Z_W2(jit, dst, jit_Z_W2(jit, src));
1617 }
1618 #endif
1619 ref = jit_Z_PTR(jit, src);
1620 jit_set_Z_PTR(jit, dst, ref);
1621 }
1622 }
1623 if (has_concrete_type(src_info & MAY_BE_ANY)
1624 && (src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))
1625 && !(src_info & MAY_BE_GUARD)) {
1626 if (Z_MODE(dst) != IS_REG
1627 && (dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
1628 uint8_t type = concrete_type(src_info);
1629 jit_set_Z_TYPE_INFO(jit, dst, type);
1630 }
1631 } else {
1632 ir_ref type = jit_Z_TYPE_INFO(jit, src);
1633 jit_set_Z_TYPE_INFO_ex(jit, dst, type);
1634 if (addref) {
1635 if (src_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
1636 ir_ref if_refcounted = IR_UNUSED;
1637
1638 if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
1639 if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(0xff00)));
1640 ir_IF_TRUE(if_refcounted);
1641 }
1642
1643 jit_GC_ADDREF(jit, ref);
1644
1645 if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
1646 ir_MERGE_WITH_EMPTY_FALSE(if_refcounted);
1647 }
1648 }
1649 }
1650 }
1651 }
1652
jit_ZVAL_COPY_2(zend_jit_ctx * jit,zend_jit_addr dst2,zend_jit_addr dst,uint32_t dst_info,zend_jit_addr src,uint32_t src_info,int addref)1653 static void jit_ZVAL_COPY_2(zend_jit_ctx *jit, zend_jit_addr dst2, zend_jit_addr dst, uint32_t dst_info, zend_jit_addr src, uint32_t src_info, int addref)
1654 {
1655 ir_ref ref = IR_UNUSED;
1656
1657 if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
1658 if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
1659 ref = jit_Z_LVAL(jit, src);
1660 jit_set_Z_LVAL(jit, dst, ref);
1661 jit_set_Z_LVAL(jit, dst2, ref);
1662 } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
1663 ref = jit_Z_DVAL(jit, src);
1664 jit_set_Z_DVAL(jit, dst, ref);
1665 jit_set_Z_DVAL(jit, dst2, ref);
1666 } else {
1667 #if SIZEOF_ZEND_LONG == 4
1668 if (src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) {
1669 ref = jit_Z_W2(jit, src);
1670 jit_set_Z_W2(jit, dst, ref);
1671 jit_set_Z_W2(jit, dst2, ref);
1672 }
1673 #endif
1674 ref = jit_Z_PTR(jit, src);
1675 jit_set_Z_PTR(jit, dst, ref);
1676 jit_set_Z_PTR(jit, dst2, ref);
1677 }
1678 }
1679 if (has_concrete_type(src_info & MAY_BE_ANY)
1680 && (src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))
1681 && !(src_info & MAY_BE_GUARD)) {
1682 uint8_t type = concrete_type(src_info);
1683 ir_ref type_ref = ir_CONST_U32(type);
1684
1685 if (Z_MODE(dst) != IS_REG
1686 && (dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
1687 jit_set_Z_TYPE_INFO_ex(jit, dst, type_ref);
1688 }
1689 if (Z_MODE(dst2) != IS_REG) {
1690 jit_set_Z_TYPE_INFO_ex(jit, dst2, type_ref);
1691 }
1692 } else {
1693 ir_ref type = jit_Z_TYPE_INFO(jit, src);
1694 jit_set_Z_TYPE_INFO_ex(jit, dst, type);
1695 jit_set_Z_TYPE_INFO_ex(jit, dst2, type);
1696 if (addref) {
1697 if (src_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
1698 ir_ref if_refcounted = IR_UNUSED;
1699
1700 if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
1701 if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(0xff00)));
1702 ir_IF_TRUE(if_refcounted);
1703 }
1704
1705 if (addref == 2) {
1706 jit_GC_ADDREF2(jit, ref);
1707 } else {
1708 jit_GC_ADDREF(jit, ref);
1709 }
1710
1711 if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
1712 ir_MERGE_WITH_EMPTY_FALSE(if_refcounted);
1713 }
1714 }
1715 }
1716 }
1717 }
1718
jit_ZVAL_DTOR(zend_jit_ctx * jit,ir_ref ref,uint32_t op_info,const zend_op * opline)1719 static void jit_ZVAL_DTOR(zend_jit_ctx *jit, ir_ref ref, uint32_t op_info, const zend_op *opline)
1720 {
1721 if (!((op_info) & MAY_BE_GUARD)
1722 && has_concrete_type((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
1723 uint8_t type = concrete_type((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE));
1724 if (type == IS_STRING && !ZEND_DEBUG) {
1725 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(_efree), ref);
1726 return;
1727 } else if (type == IS_ARRAY) {
1728 if ((op_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) {
1729 if (opline && ((op_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) {
1730 jit_SET_EX_OPLINE(jit, opline);
1731 }
1732 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_array_destroy), ref);
1733 } else {
1734 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_array_free), ref);
1735 }
1736 return;
1737 } else if (type == IS_OBJECT) {
1738 if (opline) {
1739 jit_SET_EX_OPLINE(jit, opline);
1740 }
1741 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_objects_store_del), ref);
1742 return;
1743 }
1744 }
1745 if (opline) {
1746 jit_SET_EX_OPLINE(jit, opline);
1747 }
1748 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(rc_dtor_func), ref);
1749 }
1750
jit_ZVAL_PTR_DTOR(zend_jit_ctx * jit,zend_jit_addr addr,uint32_t op_info,bool gc,const zend_op * opline)1751 static void jit_ZVAL_PTR_DTOR(zend_jit_ctx *jit,
1752 zend_jit_addr addr,
1753 uint32_t op_info,
1754 bool gc,
1755 const zend_op *opline)
1756 {
1757 ir_ref ref, ref2;
1758 ir_ref if_refcounted = IR_UNUSED;
1759 ir_ref if_not_zero = IR_UNUSED;
1760 ir_ref end_inputs = IR_UNUSED;
1761
1762 if (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) {
1763 if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
1764 if_refcounted = jit_if_REFCOUNTED(jit, addr);
1765 ir_IF_FALSE(if_refcounted);
1766 ir_END_list(end_inputs);
1767 ir_IF_TRUE(if_refcounted);
1768 }
1769 ref = jit_Z_PTR(jit, addr);
1770 ref2 = jit_GC_DELREF(jit, ref);
1771
1772 if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) {
1773 if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) {
1774 if_not_zero = ir_IF(ref2);
1775 ir_IF_FALSE(if_not_zero);
1776 }
1777 // zval_dtor_func(r);
1778 jit_ZVAL_DTOR(jit, ref, op_info, opline);
1779 if (if_not_zero) {
1780 ir_END_list(end_inputs);
1781 ir_IF_TRUE(if_not_zero);
1782 }
1783 }
1784 if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
1785 ir_ref if_may_not_leak;
1786
1787 if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) {
1788 ir_ref if_ref, if_collectable;
1789
1790 if_ref = jit_if_Z_TYPE(jit, addr, IS_REFERENCE);
1791 ir_IF_TRUE(if_ref);
1792
1793 ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
1794
1795 if_collectable = jit_if_COLLECTABLE_ref(jit, ref2);
1796 ir_IF_FALSE(if_collectable);
1797 ir_END_list(end_inputs);
1798 ir_IF_TRUE(if_collectable);
1799
1800 ref2 = jit_Z_PTR_ref(jit, ref2);
1801
1802 ir_MERGE_WITH_EMPTY_FALSE(if_ref);
1803 ref = ir_PHI_2(IR_ADDR, ref2, ref);
1804 }
1805
1806 if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
1807 ir_IF_TRUE(if_may_not_leak);
1808 ir_END_list(end_inputs);
1809 ir_IF_FALSE(if_may_not_leak);
1810
1811 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
1812 }
1813
1814 if (end_inputs) {
1815 ir_END_list(end_inputs);
1816 ir_MERGE_list(end_inputs);
1817 }
1818 }
1819 }
1820
jit_FREE_OP(zend_jit_ctx * jit,uint8_t op_type,znode_op op,uint32_t op_info,const zend_op * opline)1821 static void jit_FREE_OP(zend_jit_ctx *jit,
1822 uint8_t op_type,
1823 znode_op op,
1824 uint32_t op_info,
1825 const zend_op *opline)
1826 {
1827 if (op_type & (IS_VAR|IS_TMP_VAR)) {
1828 jit_ZVAL_PTR_DTOR(jit,
1829 ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var),
1830 op_info, 0, opline);
1831 }
1832 }
1833
jit_OBJ_RELEASE(zend_jit_ctx * jit,ir_ref ref)1834 static void jit_OBJ_RELEASE(zend_jit_ctx *jit, ir_ref ref)
1835 {
1836 ir_ref end_inputs = IR_UNUSED;
1837 ir_ref if_not_zero, if_may_not_leak;
1838
1839 // JIT: if (GC_DELREF(obj) == 0) {
1840 if_not_zero = ir_IF(jit_GC_DELREF(jit, ref));
1841 ir_IF_FALSE(if_not_zero);
1842
1843 // JIT: zend_objects_store_del(obj)
1844 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_objects_store_del), ref);
1845 ir_END_list(end_inputs);
1846
1847 ir_IF_TRUE(if_not_zero);
1848 if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
1849
1850 ir_IF_TRUE(if_may_not_leak);
1851 ir_END_list(end_inputs);
1852
1853 ir_IF_FALSE(if_may_not_leak);
1854 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
1855 ir_END_list(end_inputs);
1856
1857 ir_MERGE_list(end_inputs);
1858 }
1859
zend_jit_check_timeout(zend_jit_ctx * jit,const zend_op * opline,const void * exit_addr)1860 static int zend_jit_check_timeout(zend_jit_ctx *jit, const zend_op *opline, const void *exit_addr)
1861 {
1862 ir_ref ref = ir_LOAD_U8(jit_EG(vm_interrupt));
1863
1864 if (exit_addr) {
1865 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
1866 } else if (!opline || jit->last_valid_opline == opline) {
1867 ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_interrupt_handler));
1868 } else {
1869 ir_ref if_timeout = ir_IF(ref);
1870
1871 ir_IF_TRUE_cold(if_timeout);
1872 jit_LOAD_IP_ADDR(jit, opline);
1873 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_interrupt_handler));
1874 ir_IF_FALSE(if_timeout);
1875 }
1876 return 1;
1877 }
1878
1879 /* stubs */
1880
zend_jit_exception_handler_stub(zend_jit_ctx * jit)1881 static int zend_jit_exception_handler_stub(zend_jit_ctx *jit)
1882 {
1883 const void *handler;
1884
1885 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
1886 handler = zend_get_opcode_handler_func(EG(exception_op));
1887
1888 ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
1889 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
1890 } else {
1891 handler = EG(exception_op)->handler;
1892
1893 if (GCC_GLOBAL_REGS) {
1894 ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
1895 } else {
1896 ir_ref ref, if_negative;
1897
1898 ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), jit_FP(jit));
1899 if_negative = ir_IF(ir_LT(ref, ir_CONST_U32(0)));
1900 ir_IF_TRUE(if_negative);
1901 ir_MERGE_WITH_EMPTY_FALSE(if_negative);
1902 ref = ir_PHI_2(IR_I32, ref, ir_CONST_I32(1));
1903 ir_RETURN(ref);
1904 }
1905 }
1906 return 1;
1907 }
1908
zend_jit_exception_handler_undef_stub(zend_jit_ctx * jit)1909 static int zend_jit_exception_handler_undef_stub(zend_jit_ctx *jit)
1910 {
1911 ir_ref ref, result_type, if_result_used;
1912
1913 ref = jit_EG(opline_before_exception);
1914 result_type = ir_LOAD_U8(ir_ADD_OFFSET(ir_LOAD_A(ref), offsetof(zend_op, result_type)));
1915
1916 if_result_used = ir_IF(ir_AND_U8(result_type, ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
1917 ir_IF_TRUE(if_result_used);
1918
1919 ref = ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(ref), offsetof(zend_op, result.var)));
1920 if (sizeof(void*) == 8) {
1921 ref = ir_ZEXT_A(ref);
1922 }
1923 ir_STORE(ir_ADD_OFFSET(ir_ADD_A(jit_FP(jit), ref), offsetof(zval, u1.type_info)), ir_CONST_U32(IS_UNDEF));
1924 ir_MERGE_WITH_EMPTY_FALSE(if_result_used);
1925
1926 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
1927
1928 return 1;
1929 }
1930
zend_jit_exception_handler_free_op1_op2_stub(zend_jit_ctx * jit)1931 static int zend_jit_exception_handler_free_op1_op2_stub(zend_jit_ctx *jit)
1932 {
1933 ir_ref ref, if_dtor;
1934 zend_jit_addr var_addr;
1935
1936 ref = ir_LOAD_A(jit_EG(opline_before_exception));
1937 if_dtor = ir_IF(ir_AND_U8(ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zend_op, op1_type))),
1938 ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
1939 ir_IF_TRUE(if_dtor);
1940 ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op1.var)));
1941 if (sizeof(void*) == 8) {
1942 ref = ir_ZEXT_A(ref);
1943 }
1944 ref = ir_ADD_A(jit_FP(jit), ref);
1945 var_addr = ZEND_ADDR_REF_ZVAL(ref);
1946 jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
1947 ir_MERGE_WITH_EMPTY_FALSE(if_dtor);
1948
1949 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
1950
1951 return 1;
1952 }
1953
zend_jit_exception_handler_free_op2_stub(zend_jit_ctx * jit)1954 static int zend_jit_exception_handler_free_op2_stub(zend_jit_ctx *jit)
1955 {
1956 ir_ref ref, if_dtor;
1957 zend_jit_addr var_addr;
1958
1959 ref = ir_LOAD_A(jit_EG(opline_before_exception));
1960 if_dtor = ir_IF(ir_AND_U8(ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zend_op, op2_type))),
1961 ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
1962 ir_IF_TRUE(if_dtor);
1963 ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op2.var)));
1964 if (sizeof(void*) == 8) {
1965 ref = ir_ZEXT_A(ref);
1966 }
1967 ref = ir_ADD_A(jit_FP(jit), ref);
1968 var_addr = ZEND_ADDR_REF_ZVAL(ref);
1969 jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
1970 ir_MERGE_WITH_EMPTY_FALSE(if_dtor);
1971
1972 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
1973
1974 return 1;
1975 }
1976
zend_jit_interrupt_handler_stub(zend_jit_ctx * jit)1977 static int zend_jit_interrupt_handler_stub(zend_jit_ctx *jit)
1978 {
1979 ir_ref if_timeout, if_exception;
1980
1981 if (GCC_GLOBAL_REGS) {
1982 // EX(opline) = opline
1983 ir_STORE(jit_EX(opline), jit_IP(jit));
1984 }
1985
1986 ir_STORE(jit_EG(vm_interrupt), ir_CONST_U8(0));
1987 if_timeout = ir_IF(ir_EQ(ir_LOAD_U8(jit_EG(timed_out)), ir_CONST_U8(0)));
1988 ir_IF_FALSE(if_timeout);
1989 ir_CALL(IR_VOID, ir_CONST_FUNC(zend_timeout));
1990 ir_MERGE_WITH_EMPTY_TRUE(if_timeout);
1991
1992 if (zend_interrupt_function) {
1993 ir_CALL_1(IR_VOID, ir_CONST_FUNC(zend_interrupt_function), jit_FP(jit));
1994 if_exception = ir_IF(ir_LOAD_A(jit_EG(exception)));
1995 ir_IF_TRUE(if_exception);
1996 ir_CALL(IR_VOID, ir_CONST_FUNC(zend_jit_exception_in_interrupt_handler_helper));
1997 ir_MERGE_WITH_EMPTY_FALSE(if_exception);
1998
1999 jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data)));
2000 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
2001 }
2002
2003 if (GCC_GLOBAL_REGS) {
2004 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
2005 } else {
2006 ir_RETURN(ir_CONST_I32(1));
2007 }
2008 return 1;
2009 }
2010
zend_jit_leave_function_handler_stub(zend_jit_ctx * jit)2011 static int zend_jit_leave_function_handler_stub(zend_jit_ctx *jit)
2012 {
2013 ir_ref call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
2014 ir_ref if_top = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_TOP)));
2015
2016 ir_IF_FALSE(if_top);
2017
2018 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
2019 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info);
2020 jit_STORE_IP(jit,
2021 ir_LOAD_A(jit_EX(opline)));
2022 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
2023 } else if (GCC_GLOBAL_REGS) {
2024 ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info);
2025 } else {
2026 ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info, jit_FP(jit));
2027 }
2028
2029 ir_IF_TRUE(if_top);
2030
2031 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
2032 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info);
2033 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
2034 } else if (GCC_GLOBAL_REGS) {
2035 ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info);
2036 } else {
2037 ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info, jit_FP(jit));
2038 }
2039
2040 return 1;
2041 }
2042
zend_jit_negative_shift_stub(zend_jit_ctx * jit)2043 static int zend_jit_negative_shift_stub(zend_jit_ctx *jit)
2044 {
2045 ir_CALL_2(IR_VOID,
2046 ir_CONST_FUNC_PROTO(zend_throw_error,
2047 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
2048 ir_CONST_ADDR(zend_ce_arithmetic_error),
2049 ir_CONST_ADDR("Bit shift by negative number"));
2050 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2));
2051 return 1;
2052 }
2053
zend_jit_mod_by_zero_stub(zend_jit_ctx * jit)2054 static int zend_jit_mod_by_zero_stub(zend_jit_ctx *jit)
2055 {
2056 ir_CALL_2(IR_VOID,
2057 ir_CONST_FUNC_PROTO(zend_throw_error,
2058 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
2059 ir_CONST_ADDR(zend_ce_division_by_zero_error),
2060 ir_CONST_ADDR("Modulo by zero"));
2061 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2));
2062 return 1;
2063 }
2064
zend_jit_invalid_this_stub(zend_jit_ctx * jit)2065 static int zend_jit_invalid_this_stub(zend_jit_ctx *jit)
2066 {
2067 ir_CALL_2(IR_VOID,
2068 ir_CONST_FUNC_PROTO(zend_throw_error,
2069 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
2070 IR_NULL,
2071 ir_CONST_ADDR("Using $this when not in object context"));
2072 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
2073 return 1;
2074 }
2075
zend_jit_undefined_function_stub(zend_jit_ctx * jit)2076 static int zend_jit_undefined_function_stub(zend_jit_ctx *jit)
2077 {
2078 // JIT: load EX(opline)
2079 ir_ref ref = ir_LOAD_A(jit_FP(jit));
2080 ir_ref arg3 = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op2.constant)));
2081
2082 if (sizeof(void*) == 8) {
2083 arg3 = ir_LOAD_A(ir_ADD_A(ref, ir_SEXT_A(arg3)));
2084 } else {
2085 arg3 = ir_LOAD_A(arg3);
2086 }
2087 arg3 = ir_ADD_OFFSET(arg3, offsetof(zend_string, val));
2088
2089 ir_CALL_3(IR_VOID,
2090 ir_CONST_FUNC_PROTO(zend_throw_error,
2091 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
2092 IR_NULL,
2093 ir_CONST_ADDR("Call to undefined function %s()"),
2094 arg3);
2095
2096 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
2097
2098 return 1;
2099 }
2100
zend_jit_throw_cannot_pass_by_ref_stub(zend_jit_ctx * jit)2101 static int zend_jit_throw_cannot_pass_by_ref_stub(zend_jit_ctx *jit)
2102 {
2103 ir_ref opline, ref, rx, if_eq, if_tmp;
2104
2105 // JIT: opline = EX(opline)
2106 opline = ir_LOAD_A(jit_FP(jit));
2107
2108 // JIT: ZVAL_UNDEF(ZEND_CALL_VAR(RX, opline->result.var))
2109 ref = ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, result.var)));
2110 if (sizeof(void*) == 8) {
2111 ref = ir_ZEXT_A(ref);
2112 }
2113 rx = jit_IP(jit);
2114 jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(rx, ref), ir_CONST_U32(IS_UNDEF));
2115
2116 // last EX(call) frame may be delayed
2117 // JIT: if (EX(call) == RX)
2118 ref = ir_LOAD_A(jit_EX(call));
2119 if_eq = ir_IF(ir_EQ(rx, ref));
2120 ir_IF_FALSE(if_eq);
2121
2122 // JIT: RX->prev_execute_data == EX(call)
2123 ir_STORE(jit_CALL(rx, prev_execute_data), ref);
2124
2125 // JIT: EX(call) = RX
2126 ir_STORE(jit_EX(call), rx);
2127 ir_MERGE_WITH_EMPTY_TRUE(if_eq);
2128
2129 // JIT: IP = opline
2130 jit_STORE_IP(jit, opline);
2131
2132 // JIT: zend_cannot_pass_by_reference(opline->op2.num)
2133 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_cannot_pass_by_reference),
2134 ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, op2.num))));
2135
2136 // JIT: if (IP->op1_type == IS_TMP_VAR)
2137 ref = ir_LOAD_U8(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1_type)));
2138 if_tmp = ir_IF(ir_EQ(ref, ir_CONST_U8(IS_TMP_VAR)));
2139 ir_IF_TRUE(if_tmp);
2140
2141 // JIT: zval_ptr_dtor(EX_VAR(IP->op1.var))
2142 ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1.var)));
2143 if (sizeof(void*) == 8) {
2144 ref = ir_ZEXT_A(ref);
2145 }
2146 ref = ir_ADD_A(jit_FP(jit), ref);
2147 jit_ZVAL_PTR_DTOR(jit,
2148 ZEND_ADDR_REF_ZVAL(ref),
2149 MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
2150 ir_MERGE_WITH_EMPTY_FALSE(if_tmp);
2151
2152 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
2153
2154 return 1;
2155 }
2156
zend_jit_icall_throw_stub(zend_jit_ctx * jit)2157 static int zend_jit_icall_throw_stub(zend_jit_ctx *jit)
2158 {
2159 ir_ref ip, if_set;
2160
2161 // JIT: zend_rethrow_exception(zend_execute_data *execute_data)
2162 // JIT: if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) {
2163 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
2164 ip = jit_IP(jit);
2165 if_set = ir_IF(ir_EQ(ir_LOAD_U8(ir_ADD_OFFSET(ip, offsetof(zend_op, opcode))),
2166 ir_CONST_U8(ZEND_HANDLE_EXCEPTION)));
2167 ir_IF_FALSE(if_set);
2168
2169 // JIT: EG(opline_before_exception) = opline;
2170 ir_STORE(jit_EG(opline_before_exception), ip);
2171 ir_MERGE_WITH_EMPTY_TRUE(if_set);
2172
2173 // JIT: opline = EG(exception_op);
2174 jit_STORE_IP(jit, jit_EG(exception_op));
2175
2176 if (GCC_GLOBAL_REGS) {
2177 ir_STORE(jit_EX(opline), jit_IP(jit));
2178 }
2179
2180 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
2181
2182 return 1;
2183 }
2184
zend_jit_leave_throw_stub(zend_jit_ctx * jit)2185 static int zend_jit_leave_throw_stub(zend_jit_ctx *jit)
2186 {
2187 ir_ref ip, if_set;
2188
2189 // JIT: if (opline->opcode != ZEND_HANDLE_EXCEPTION) {
2190 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
2191 ip = jit_IP(jit);
2192 if_set = ir_IF(ir_EQ(ir_LOAD_U8(ir_ADD_OFFSET(ip, offsetof(zend_op, opcode))),
2193 ir_CONST_U8(ZEND_HANDLE_EXCEPTION)));
2194 ir_IF_FALSE(if_set);
2195
2196 // JIT: EG(opline_before_exception) = opline;
2197 ir_STORE(jit_EG(opline_before_exception), ip);
2198 ir_MERGE_WITH_EMPTY_TRUE(if_set);
2199
2200 // JIT: opline = EG(exception_op);
2201 jit_LOAD_IP(jit, jit_EG(exception_op));
2202
2203 if (GCC_GLOBAL_REGS) {
2204 ir_STORE(jit_EX(opline), jit_IP(jit));
2205
2206 // JIT: HANDLE_EXCEPTION()
2207 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
2208 } else {
2209 ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
2210 }
2211
2212 return 1;
2213 }
2214
zend_jit_hybrid_runtime_jit_stub(zend_jit_ctx * jit)2215 static int zend_jit_hybrid_runtime_jit_stub(zend_jit_ctx *jit)
2216 {
2217 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
2218 return 0;
2219 }
2220
2221 ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_runtime_jit));
2222 ir_IJMP(ir_LOAD_A(jit_IP(jit)));
2223 return 1;
2224 }
2225
zend_jit_hybrid_profile_jit_stub(zend_jit_ctx * jit)2226 static int zend_jit_hybrid_profile_jit_stub(zend_jit_ctx *jit)
2227 {
2228 ir_ref addr, func, run_time_cache, jit_extension;
2229
2230 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
2231 return 0;
2232 }
2233
2234 addr = ir_CONST_ADDR(&zend_jit_profile_counter),
2235 ir_STORE(addr, ir_ADD_L(ir_LOAD_L(addr), ir_CONST_LONG(1)));
2236
2237 func = ir_LOAD_A(jit_EX(func));
2238 run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
2239 jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
2240
2241 if (zend_jit_profile_counter_rid) {
2242 addr = ir_ADD_OFFSET(run_time_cache, zend_jit_profile_counter_rid * sizeof(void*));
2243 } else {
2244 addr = run_time_cache;
2245 }
2246 ir_STORE(addr, ir_ADD_L(ir_LOAD_L(addr), ir_CONST_LONG(1)));
2247
2248 addr = ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_extension, orig_handler));
2249 ir_IJMP(ir_LOAD_A(addr));
2250
2251 return 1;
2252 }
2253
_zend_jit_hybrid_hot_counter_stub(zend_jit_ctx * jit,uint32_t cost)2254 static int _zend_jit_hybrid_hot_counter_stub(zend_jit_ctx *jit, uint32_t cost)
2255 {
2256 ir_ref func, jit_extension, addr, ref, if_overflow;
2257
2258 func = ir_LOAD_A(jit_EX(func));
2259 jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
2260 addr = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_hot_extension, counter)));
2261 ref = ir_SUB_I16(ir_LOAD_I16(addr), ir_CONST_I16(cost));
2262 ir_STORE(addr, ref);
2263 if_overflow = ir_IF(ir_LE(ref, ir_CONST_I16(0)));
2264
2265 ir_IF_TRUE_cold(if_overflow);
2266 ir_STORE(addr, ir_CONST_I16(ZEND_JIT_COUNTER_INIT));
2267 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_hot_func),
2268 jit_FP(jit),
2269 jit_IP(jit));
2270 ir_IJMP(ir_LOAD_A(jit_IP(jit)));
2271
2272 ir_IF_FALSE(if_overflow);
2273 ref = ir_SUB_A(jit_IP(jit),
2274 ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, opcodes))));
2275 ref = ir_DIV_A(ref, ir_CONST_ADDR(sizeof(zend_op) / sizeof(void*)));
2276
2277 addr = ir_ADD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_hot_extension, orig_handlers)),
2278 ref);
2279 ir_IJMP(ir_LOAD_A(addr));
2280
2281 return 1;
2282 }
2283
zend_jit_hybrid_func_hot_counter_stub(zend_jit_ctx * jit)2284 static int zend_jit_hybrid_func_hot_counter_stub(zend_jit_ctx *jit)
2285 {
2286 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
2287 return 0;
2288 }
2289
2290 return _zend_jit_hybrid_hot_counter_stub(jit,
2291 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
2292 }
2293
zend_jit_hybrid_loop_hot_counter_stub(zend_jit_ctx * jit)2294 static int zend_jit_hybrid_loop_hot_counter_stub(zend_jit_ctx *jit)
2295 {
2296 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
2297 return 0;
2298 }
2299
2300 return _zend_jit_hybrid_hot_counter_stub(jit,
2301 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
2302 }
2303
_zend_jit_orig_opline_handler(zend_jit_ctx * jit,ir_ref offset)2304 static ir_ref _zend_jit_orig_opline_handler(zend_jit_ctx *jit, ir_ref offset)
2305 {
2306 ir_ref addr;
2307
2308 if (GCC_GLOBAL_REGS) {
2309 addr = ir_ADD_A(offset, jit_IP(jit));
2310 } else {
2311 addr = ir_ADD_A(offset, ir_LOAD_A(jit_EX(opline)));
2312 }
2313
2314 return ir_LOAD_A(addr);
2315 }
2316
zend_jit_orig_opline_handler(zend_jit_ctx * jit)2317 static ir_ref zend_jit_orig_opline_handler(zend_jit_ctx *jit)
2318 {
2319 ir_ref func, jit_extension, offset;
2320
2321 func = ir_LOAD_A(jit_EX(func));
2322 jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
2323 offset = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_trace_extension, offset)));
2324 return _zend_jit_orig_opline_handler(jit, offset);
2325 }
2326
_zend_jit_hybrid_trace_counter_stub(zend_jit_ctx * jit,uint32_t cost)2327 static int _zend_jit_hybrid_trace_counter_stub(zend_jit_ctx *jit, uint32_t cost)
2328 {
2329 ir_ref func, jit_extension, offset, addr, ref, if_overflow, ret, if_halt;
2330
2331 func = ir_LOAD_A(jit_EX(func));
2332 jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
2333 offset = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_trace_extension, offset)));
2334 addr = ir_LOAD_A(ir_ADD_OFFSET(ir_ADD_A(offset, jit_IP(jit)), offsetof(zend_op_trace_info, counter)));
2335 ref = ir_SUB_I16(ir_LOAD_I16(addr), ir_CONST_I16(cost));
2336 ir_STORE(addr, ref);
2337 if_overflow = ir_IF(ir_LE(ref, ir_CONST_I16(0)));
2338
2339 ir_IF_TRUE_cold(if_overflow);
2340 ir_STORE(addr, ir_CONST_I16(ZEND_JIT_COUNTER_INIT));
2341 ret = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_trace_hot_root),
2342 jit_FP(jit),
2343 jit_IP(jit));
2344 if_halt = ir_IF(ir_LT(ret, ir_CONST_I32(0)));
2345 ir_IF_FALSE(if_halt);
2346
2347 ref = jit_EG(current_execute_data);
2348 jit_STORE_FP(jit, ir_LOAD_A(ref));
2349 ref = ir_LOAD_A(jit_EX(opline));
2350 jit_STORE_IP(jit, ref);
2351 ir_IJMP(ir_LOAD_A(jit_IP(jit)));
2352
2353 ir_IF_FALSE(if_overflow);
2354 ir_IJMP(_zend_jit_orig_opline_handler(jit, offset));
2355
2356 ir_IF_TRUE(if_halt);
2357 ir_IJMP(ir_CONST_FC_FUNC(zend_jit_halt_op->handler));
2358
2359 return 1;
2360 }
2361
zend_jit_hybrid_func_trace_counter_stub(zend_jit_ctx * jit)2362 static int zend_jit_hybrid_func_trace_counter_stub(zend_jit_ctx *jit)
2363 {
2364 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
2365 return 0;
2366 }
2367
2368 return _zend_jit_hybrid_trace_counter_stub(jit,
2369 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
2370 }
2371
zend_jit_hybrid_ret_trace_counter_stub(zend_jit_ctx * jit)2372 static int zend_jit_hybrid_ret_trace_counter_stub(zend_jit_ctx *jit)
2373 {
2374 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) {
2375 return 0;
2376 }
2377
2378 return _zend_jit_hybrid_trace_counter_stub(jit,
2379 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
2380 }
2381
zend_jit_hybrid_loop_trace_counter_stub(zend_jit_ctx * jit)2382 static int zend_jit_hybrid_loop_trace_counter_stub(zend_jit_ctx *jit)
2383 {
2384 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
2385 return 0;
2386 }
2387
2388 return _zend_jit_hybrid_trace_counter_stub(jit,
2389 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
2390 }
2391
zend_jit_trace_halt_stub(zend_jit_ctx * jit)2392 static int zend_jit_trace_halt_stub(zend_jit_ctx *jit)
2393 {
2394 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
2395 ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_halt_op->handler));
2396 } else if (GCC_GLOBAL_REGS) {
2397 jit_STORE_IP(jit, IR_NULL);
2398 ir_RETURN(IR_VOID);
2399 } else {
2400 ir_RETURN(ir_CONST_I32(-1)); // ZEND_VM_RETURN
2401 }
2402 return 1;
2403 }
2404
zend_jit_trace_escape_stub(zend_jit_ctx * jit)2405 static int zend_jit_trace_escape_stub(zend_jit_ctx *jit)
2406 {
2407 if (GCC_GLOBAL_REGS) {
2408 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
2409 } else {
2410 ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
2411 }
2412
2413 return 1;
2414 }
2415
zend_jit_trace_exit_stub(zend_jit_ctx * jit)2416 static int zend_jit_trace_exit_stub(zend_jit_ctx *jit)
2417 {
2418 ir_ref ref, ret, if_zero, addr;
2419
2420 if (GCC_GLOBAL_REGS) {
2421 // EX(opline) = opline
2422 ir_STORE(jit_EX(opline), jit_IP(jit));
2423 }
2424
2425 ret = ir_EXITCALL(ir_CONST_FC_FUNC(zend_jit_trace_exit));
2426
2427 if_zero = ir_IF(ir_EQ(ret, ir_CONST_I32(0)));
2428
2429 ir_IF_TRUE(if_zero);
2430
2431 if (GCC_GLOBAL_REGS) {
2432 ref = jit_EG(current_execute_data);
2433 jit_STORE_FP(jit, ir_LOAD_A(ref));
2434 ref = ir_LOAD_A(jit_EX(opline));
2435 jit_STORE_IP(jit, ref);
2436 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
2437 } else {
2438 ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
2439 }
2440
2441 ir_IF_FALSE(if_zero);
2442
2443 ir_GUARD(ir_GE(ret, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
2444
2445 ref = jit_EG(current_execute_data);
2446 jit_STORE_FP(jit, ir_LOAD_A(ref));
2447
2448 if (GCC_GLOBAL_REGS) {
2449 ref = ir_LOAD_A(jit_EX(opline));
2450 jit_STORE_IP(jit, ref);
2451 }
2452
2453 // check for interrupt (try to avoid this ???)
2454 if (!zend_jit_check_timeout(jit, NULL, NULL)) {
2455 return 0;
2456 }
2457
2458 addr = zend_jit_orig_opline_handler(jit);
2459 if (GCC_GLOBAL_REGS) {
2460 ir_TAILCALL(IR_VOID, addr);
2461 } else {
2462 #if defined(IR_TARGET_X86)
2463 addr = ir_CAST_FC_FUNC(addr);
2464 #endif
2465 ref = ir_CALL_1(IR_I32, addr, jit_FP(jit));
2466 ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
2467 ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
2468 }
2469
2470 return 1;
2471 }
2472
zend_jit_undefined_offset_stub(zend_jit_ctx * jit)2473 static int zend_jit_undefined_offset_stub(zend_jit_ctx *jit)
2474 {
2475 if (GCC_GLOBAL_REGS) {
2476 ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key));
2477 } else {
2478 ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key), jit_FP(jit));
2479 }
2480
2481 return 1;
2482 }
2483
zend_jit_undefined_key_stub(zend_jit_ctx * jit)2484 static int zend_jit_undefined_key_stub(zend_jit_ctx *jit)
2485 {
2486 if (GCC_GLOBAL_REGS) {
2487 ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key));
2488 } else {
2489 ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key), jit_FP(jit));
2490 }
2491
2492 return 1;
2493 }
2494
zend_jit_cannot_add_element_stub(zend_jit_ctx * jit)2495 static int zend_jit_cannot_add_element_stub(zend_jit_ctx *jit)
2496 {
2497 ir_ref opline = ir_LOAD_A(jit_EX(opline));
2498 ir_ref ref, if_result_used;
2499
2500 if_result_used = ir_IF(ir_AND_U8(
2501 ir_LOAD_U8(ir_ADD_OFFSET(opline, offsetof(zend_op, result_type))),
2502 ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
2503 ir_IF_TRUE(if_result_used);
2504
2505 ref = ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, result.var)));
2506 if (sizeof(void*) == 8) {
2507 ref = ir_ZEXT_A(ref);
2508 }
2509 jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(jit_FP(jit), ref), ir_CONST_U32(IS_UNDEF));
2510 ir_MERGE_WITH_EMPTY_FALSE(if_result_used);
2511
2512 ir_CALL_2(IR_VOID,
2513 ir_CONST_FUNC_PROTO(zend_throw_error,
2514 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
2515 IR_NULL,
2516 ir_CONST_ADDR("Cannot add element to the array as the next element is already occupied"));
2517 ir_RETURN(IR_VOID);
2518
2519 return 1;
2520 }
2521
zend_jit_assign_const_stub(zend_jit_ctx * jit)2522 static int zend_jit_assign_const_stub(zend_jit_ctx *jit)
2523 {
2524 ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
2525 ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
2526
2527 zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
2528 zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
2529 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
2530
2531 if (!zend_jit_assign_to_variable(
2532 jit, NULL,
2533 var_addr, var_addr, -1, -1,
2534 IS_CONST, val_addr, val_info,
2535 0, 0, 0)) {
2536 return 0;
2537 }
2538 ir_RETURN(IR_VOID);
2539 return 1;
2540 }
2541
zend_jit_assign_tmp_stub(zend_jit_ctx * jit)2542 static int zend_jit_assign_tmp_stub(zend_jit_ctx *jit)
2543 {
2544 ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
2545 ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
2546
2547 zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
2548 zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
2549 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
2550
2551 if (!zend_jit_assign_to_variable(
2552 jit, NULL,
2553 var_addr, var_addr, -1, -1,
2554 IS_TMP_VAR, val_addr, val_info,
2555 0, 0, 0)) {
2556 return 0;
2557 }
2558 ir_RETURN(IR_VOID);
2559 return 1;
2560 }
2561
zend_jit_assign_var_stub(zend_jit_ctx * jit)2562 static int zend_jit_assign_var_stub(zend_jit_ctx *jit)
2563 {
2564 ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
2565 ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
2566
2567 zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
2568 zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
2569 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF;
2570
2571 if (!zend_jit_assign_to_variable(
2572 jit, NULL,
2573 var_addr, var_addr, -1, -1,
2574 IS_VAR, val_addr, val_info,
2575 0, 0, 0)) {
2576 return 0;
2577 }
2578 ir_RETURN(IR_VOID);
2579 return 1;
2580 }
2581
zend_jit_assign_cv_noref_stub(zend_jit_ctx * jit)2582 static int zend_jit_assign_cv_noref_stub(zend_jit_ctx *jit)
2583 {
2584 ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
2585 ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
2586
2587 zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
2588 zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
2589 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/;
2590
2591 if (!zend_jit_assign_to_variable(
2592 jit, NULL,
2593 var_addr, var_addr, -1, -1,
2594 IS_CV, val_addr, val_info,
2595 0, 0, 0)) {
2596 return 0;
2597 }
2598 ir_RETURN(IR_VOID);
2599 return 1;
2600 }
2601
zend_jit_new_array_stub(zend_jit_ctx * jit)2602 static int zend_jit_new_array_stub(zend_jit_ctx *jit)
2603 {
2604 ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
2605 zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
2606 ir_ref ref = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(_zend_new_array_0));
2607
2608 jit_set_Z_PTR(jit, var_addr, ref);
2609 jit_set_Z_TYPE_INFO(jit, var_addr, IS_ARRAY_EX);
2610 ir_RETURN(ref);
2611 return 1;
2612 }
2613
zend_jit_assign_cv_stub(zend_jit_ctx * jit)2614 static int zend_jit_assign_cv_stub(zend_jit_ctx *jit)
2615 {
2616 ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
2617 ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
2618
2619 zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
2620 zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
2621 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/;
2622
2623 if (!zend_jit_assign_to_variable(
2624 jit, NULL,
2625 var_addr, var_addr, -1, -1,
2626 IS_CV, val_addr, val_info,
2627 0, 0, 0)) {
2628 return 0;
2629 }
2630 ir_RETURN(IR_VOID);
2631 return 1;
2632 }
2633
zend_jit_init_ctx(zend_jit_ctx * jit,uint32_t flags)2634 static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags)
2635 {
2636 #if defined (__CET__) && (__CET__ & 1) != 0
2637 flags |= IR_GEN_ENDBR;
2638 #endif
2639 flags |= IR_OPT_FOLDING | IR_OPT_CFG | IR_OPT_CODEGEN;
2640
2641 ir_init(&jit->ctx, flags, 256, 1024);
2642 jit->ctx.ret_type = -1;
2643
2644 #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
2645 jit->ctx.mflags |= default_mflags;
2646 if (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) {
2647 jit->ctx.mflags |= IR_X86_AVX;
2648 }
2649 #elif defined(IR_TARGET_AARCH64)
2650 jit->ctx.get_veneer = zend_jit_get_veneer;
2651 jit->ctx.set_veneer = zend_jit_set_veneer;
2652 #endif
2653
2654 jit->ctx.fixed_regset = (1<<ZREG_FP) | (1<<ZREG_IP);
2655 if (!(flags & IR_FUNCTION)) {
2656 jit->ctx.flags |= IR_NO_STACK_COMBINE;
2657 if (zend_jit_vm_kind == ZEND_VM_KIND_CALL) {
2658 jit->ctx.flags |= IR_FUNCTION;
2659 /* Stack must be 16 byte aligned */
2660 /* TODO: select stack size ??? */
2661 #if defined(IR_TARGET_AARCH64)
2662 jit->ctx.flags |= IR_USE_FRAME_POINTER;
2663 jit->ctx.fixed_stack_frame_size = sizeof(void*) * 16; /* 10 saved registers and 6 spill slots (8 bytes) */
2664 #elif defined(_WIN64)
2665 jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 8 saved registers and 3 spill slots (8 bytes) */
2666 #elif defined(IR_TARGET_X86_64)
2667 jit->ctx.fixed_stack_frame_size = sizeof(void*) * 9; /* 6 saved registers and 3 spill slots (8 bytes) */
2668 #else /* IR_TARGET_x86 */
2669 jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 4 saved registers and 7 spill slots (4 bytes) */
2670 #endif
2671 if (GCC_GLOBAL_REGS) {
2672 jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED & ~((1<<ZREG_FP) | (1<<ZREG_IP));
2673 } else {
2674 jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED;
2675 //#ifdef _WIN64
2676 // jit->ctx.fixed_save_regset &= 0xffff; // TODO: don't save FP registers ???
2677 //#endif
2678 }
2679 jit->ctx.fixed_call_stack_size = 16;
2680 } else {
2681 #ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
2682 jit->ctx.fixed_stack_red_zone = ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE;
2683 if (jit->ctx.fixed_stack_red_zone > 16) {
2684 jit->ctx.fixed_stack_frame_size = jit->ctx.fixed_stack_red_zone - 16;
2685 jit->ctx.fixed_call_stack_size = 16;
2686 }
2687 jit->ctx.flags |= IR_MERGE_EMPTY_ENTRIES;
2688 #else
2689 jit->ctx.fixed_stack_red_zone = 0;
2690 jit->ctx.fixed_stack_frame_size = 32; /* 4 spill slots (8 bytes) or 8 spill slots (4 bytes) */
2691 jit->ctx.fixed_call_stack_size = 16;
2692 #endif
2693 #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
2694 jit->ctx.fixed_regset |= (1<<IR_REG_FP); /* prevent %rbp (%r5) usage */
2695 #endif
2696 }
2697 }
2698
2699 jit->ctx.snapshot_create = (ir_snapshot_create_t)jit_SNAPSHOT;
2700
2701 jit->op_array = NULL;
2702 jit->current_op_array = NULL;
2703 jit->ssa = NULL;
2704 jit->name = NULL;
2705 jit->last_valid_opline = NULL;
2706 jit->use_last_valid_opline = 0;
2707 jit->track_last_valid_opline = 0;
2708 jit->reuse_ip = 0;
2709 jit->delayed_call_level = 0;
2710 delayed_call_chain = 0;
2711 jit->b = -1;
2712 #ifdef ZTS
2713 jit->tls = IR_UNUSED;
2714 #endif
2715 jit->fp = IR_UNUSED;
2716 jit->trace_loop_ref = IR_UNUSED;
2717 jit->return_inputs = IR_UNUSED;
2718 jit->bb_start_ref = NULL;
2719 jit->bb_predecessors = NULL;
2720 jit->bb_edges = NULL;
2721 jit->trace = NULL;
2722 jit->ra = NULL;
2723 jit->delay_var = -1;
2724 jit->delay_refs = NULL;
2725 jit->eg_exception_addr = 0;
2726 zend_hash_init(&jit->addr_hash, 64, NULL, NULL, 0);
2727 memset(jit->stub_addr, 0, sizeof(jit->stub_addr));
2728
2729 ir_START();
2730 }
2731
zend_jit_free_ctx(zend_jit_ctx * jit)2732 static int zend_jit_free_ctx(zend_jit_ctx *jit)
2733 {
2734 if (jit->name) {
2735 zend_string_release(jit->name);
2736 }
2737 zend_hash_destroy(&jit->addr_hash);
2738 ir_free(&jit->ctx);
2739 return 1;
2740 }
2741
zend_jit_ir_compile(ir_ctx * ctx,size_t * size,const char * name)2742 static void *zend_jit_ir_compile(ir_ctx *ctx, size_t *size, const char *name)
2743 {
2744 void *entry;
2745 ir_code_buffer code_buffer;
2746
2747 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_SRC) {
2748 if (name) fprintf(stderr, "%s: ; after folding\n", name);
2749 ir_save(ctx, 0, stderr);
2750 }
2751
2752 #if ZEND_DEBUG
2753 ir_check(ctx);
2754 #endif
2755
2756 ir_build_def_use_lists(ctx);
2757
2758 #if ZEND_DEBUG
2759 ir_check(ctx);
2760 #endif
2761
2762 #if 1
2763 ir_sccp(ctx);
2764 #endif
2765
2766 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_SCCP) {
2767 if (name) fprintf(stderr, "%s: ; after SCCP\n", name);
2768 ir_save(ctx, 0, stderr);
2769 }
2770
2771 ir_build_cfg(ctx);
2772 ir_build_dominators_tree(ctx);
2773 ir_find_loops(ctx);
2774
2775 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_CFG) {
2776 if (name) fprintf(stderr, "%s: ; after CFG\n", name);
2777 ir_save(ctx, IR_SAVE_CFG, stderr);
2778 }
2779
2780 ir_gcm(ctx);
2781
2782 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_GCM) {
2783 if (name) fprintf(stderr, "%s: ; after GCM\n", name);
2784 ir_save(ctx, IR_SAVE_CFG|IR_SAVE_CFG_MAP, stderr);
2785 }
2786
2787 ir_schedule(ctx);
2788
2789 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE) {
2790 if (name) fprintf(stderr, "%s: ; after schedule\n", name);
2791 ir_save(ctx, IR_SAVE_CFG, stderr);
2792 }
2793
2794 ir_match(ctx);
2795 #if !defined(IR_TARGET_AARCH64)
2796 ctx->flags &= ~IR_USE_FRAME_POINTER; /* don't use FRAME_POINTER even with ALLOCA, TODO: cleanup this ??? */
2797 #endif
2798 ir_assign_virtual_registers(ctx);
2799 ir_compute_live_ranges(ctx);
2800 ir_coalesce(ctx);
2801 ir_reg_alloc(ctx);
2802
2803 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_REGS) {
2804 if (name) fprintf(stderr, "%s: ; after register allocation\n", name);
2805 ir_save(ctx, IR_SAVE_CFG|IR_SAVE_RULES|IR_SAVE_REGS, stderr);
2806 ir_dump_live_ranges(ctx, stderr);
2807 }
2808
2809 ir_schedule_blocks(ctx);
2810
2811 if (JIT_G(debug) & (ZEND_JIT_DEBUG_IR_FINAL|ZEND_JIT_DEBUG_IR_CODEGEN)) {
2812 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_CODEGEN) {
2813 if (name) fprintf(stderr, "%s: ; codegen\n", name);
2814 ir_dump_codegen(ctx, stderr);
2815 } else {
2816 if (name) fprintf(stderr, "%s: ; final\n", name);
2817 ir_save(ctx, IR_SAVE_CFG|IR_SAVE_RULES|IR_SAVE_REGS, stderr);
2818 }
2819 }
2820
2821 #if ZEND_DEBUG
2822 ir_check(ctx);
2823 #endif
2824
2825 code_buffer.start = dasm_buf;
2826 code_buffer.end = dasm_end;
2827 code_buffer.pos = *dasm_ptr;
2828 ctx->code_buffer = &code_buffer;
2829
2830 entry = ir_emit_code(ctx, size);
2831
2832 *dasm_ptr = code_buffer.pos;
2833
2834 #if defined(IR_TARGET_AARCH64)
2835 if (ctx->flags2 & IR_HAS_VENEERS) {
2836 zend_jit_commit_veneers();
2837 }
2838 #endif
2839
2840 return entry;
2841 }
2842
zend_jit_setup_stubs(void)2843 static void zend_jit_setup_stubs(void)
2844 {
2845 zend_jit_ctx jit;
2846 void *entry;
2847 size_t size;
2848 uint32_t i;
2849
2850 for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) {
2851 zend_jit_init_ctx(&jit, zend_jit_stubs[i].flags);
2852
2853 if (!zend_jit_stubs[i].stub(&jit)) {
2854 zend_jit_free_ctx(&jit);
2855 zend_jit_stub_handlers[i] = NULL;
2856 continue;
2857 }
2858
2859 entry = zend_jit_ir_compile(&jit.ctx, &size, zend_jit_stubs[i].name);
2860 if (!entry) {
2861 zend_jit_free_ctx(&jit);
2862 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: could not compile stub");
2863 }
2864
2865 zend_jit_stub_handlers[i] = entry;
2866
2867 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
2868 #ifdef HAVE_CAPSTONE
2869 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
2870 ir_disasm_add_symbol(zend_jit_stubs[i].name, (uintptr_t)entry, size);
2871 }
2872 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_STUBS) {
2873 ir_disasm(zend_jit_stubs[i].name,
2874 entry, size, (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0, &jit.ctx, stderr);
2875 }
2876 #endif
2877 #ifndef _WIN32
2878 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
2879 // ir_mem_unprotect(entry, size);
2880 ir_gdb_register(zend_jit_stubs[i].name, entry, size, 0, 0);
2881 // ir_mem_protect(entry, size);
2882 }
2883
2884 if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
2885 ir_perf_map_register(zend_jit_stubs[i].name, entry, size);
2886 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
2887 ir_perf_jitdump_register(zend_jit_stubs[i].name, entry, size);
2888 }
2889 }
2890 #endif
2891 }
2892 zend_jit_free_ctx(&jit);
2893 }
2894 }
2895
2896 #define REGISTER_HELPER(n) \
2897 ir_disasm_add_symbol(#n, (uint64_t)(uintptr_t)n, sizeof(void*));
2898 #define REGISTER_DATA(n) \
2899 ir_disasm_add_symbol(#n, (uint64_t)(uintptr_t)&n, sizeof(n));
2900
zend_jit_setup_disasm(void)2901 static void zend_jit_setup_disasm(void)
2902 {
2903 #ifdef HAVE_CAPSTONE
2904 ir_disasm_init();
2905
2906 if (zend_vm_kind() == ZEND_VM_KIND_HYBRID) {
2907 zend_op opline;
2908
2909 memset(&opline, 0, sizeof(opline));
2910
2911 opline.opcode = ZEND_DO_UCALL;
2912 opline.result_type = IS_UNUSED;
2913 zend_vm_set_opcode_handler(&opline);
2914 ir_disasm_add_symbol("ZEND_DO_UCALL_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2915
2916 opline.opcode = ZEND_DO_UCALL;
2917 opline.result_type = IS_VAR;
2918 zend_vm_set_opcode_handler(&opline);
2919 ir_disasm_add_symbol("ZEND_DO_UCALL_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2920
2921 opline.opcode = ZEND_DO_FCALL_BY_NAME;
2922 opline.result_type = IS_UNUSED;
2923 zend_vm_set_opcode_handler(&opline);
2924 ir_disasm_add_symbol("ZEND_DO_FCALL_BY_NAME_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2925
2926 opline.opcode = ZEND_DO_FCALL_BY_NAME;
2927 opline.result_type = IS_VAR;
2928 zend_vm_set_opcode_handler(&opline);
2929 ir_disasm_add_symbol("ZEND_DO_FCALL_BY_NAME_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2930
2931 opline.opcode = ZEND_DO_FCALL;
2932 opline.result_type = IS_UNUSED;
2933 zend_vm_set_opcode_handler(&opline);
2934 ir_disasm_add_symbol("ZEND_DO_FCALL_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2935
2936 opline.opcode = ZEND_DO_FCALL;
2937 opline.result_type = IS_VAR;
2938 zend_vm_set_opcode_handler(&opline);
2939 ir_disasm_add_symbol("ZEND_DO_FCALL_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2940
2941 opline.opcode = ZEND_RETURN;
2942 opline.op1_type = IS_CONST;
2943 zend_vm_set_opcode_handler(&opline);
2944 ir_disasm_add_symbol("ZEND_RETURN_SPEC_CONST_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2945
2946 opline.opcode = ZEND_RETURN;
2947 opline.op1_type = IS_TMP_VAR;
2948 zend_vm_set_opcode_handler(&opline);
2949 ir_disasm_add_symbol("ZEND_RETURN_SPEC_TMP_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2950
2951 opline.opcode = ZEND_RETURN;
2952 opline.op1_type = IS_VAR;
2953 zend_vm_set_opcode_handler(&opline);
2954 ir_disasm_add_symbol("ZEND_RETURN_SPEC_VAR_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2955
2956 opline.opcode = ZEND_RETURN;
2957 opline.op1_type = IS_CV;
2958 zend_vm_set_opcode_handler(&opline);
2959 ir_disasm_add_symbol("ZEND_RETURN_SPEC_CV_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2960
2961 ir_disasm_add_symbol("ZEND_HYBRID_HALT_LABEL", (uint64_t)(uintptr_t)zend_jit_halt_op->handler, sizeof(void*));
2962 }
2963
2964 REGISTER_DATA(zend_jit_profile_counter);
2965
2966 REGISTER_HELPER(zend_runtime_jit);
2967 REGISTER_HELPER(zend_jit_hot_func);
2968 REGISTER_HELPER(zend_jit_trace_hot_root);
2969 REGISTER_HELPER(zend_jit_trace_exit);
2970
2971 REGISTER_HELPER(zend_jit_array_free);
2972 REGISTER_HELPER(zend_jit_undefined_op_helper);
2973 REGISTER_HELPER(zend_jit_pre_inc_typed_ref);
2974 REGISTER_HELPER(zend_jit_pre_dec_typed_ref);
2975 REGISTER_HELPER(zend_jit_post_inc_typed_ref);
2976 REGISTER_HELPER(zend_jit_post_dec_typed_ref);
2977 REGISTER_HELPER(zend_jit_pre_inc);
2978 REGISTER_HELPER(zend_jit_pre_dec);
2979 REGISTER_HELPER(zend_jit_add_arrays_helper);
2980 REGISTER_HELPER(zend_jit_fast_assign_concat_helper);
2981 REGISTER_HELPER(zend_jit_fast_concat_helper);
2982 REGISTER_HELPER(zend_jit_fast_concat_tmp_helper);
2983 REGISTER_HELPER(zend_jit_assign_op_to_typed_ref_tmp);
2984 REGISTER_HELPER(zend_jit_assign_op_to_typed_ref);
2985 REGISTER_HELPER(zend_jit_assign_const_to_typed_ref);
2986 REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref);
2987 REGISTER_HELPER(zend_jit_assign_var_to_typed_ref);
2988 REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref);
2989 REGISTER_HELPER(zend_jit_assign_const_to_typed_ref2);
2990 REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref2);
2991 REGISTER_HELPER(zend_jit_assign_var_to_typed_ref2);
2992 REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref2);
2993 REGISTER_HELPER(zend_jit_check_constant);
2994 REGISTER_HELPER(zend_jit_get_constant);
2995 REGISTER_HELPER(zend_jit_int_extend_stack_helper);
2996 REGISTER_HELPER(zend_jit_extend_stack_helper);
2997 REGISTER_HELPER(zend_jit_init_func_run_time_cache_helper);
2998 REGISTER_HELPER(zend_jit_find_func_helper);
2999 REGISTER_HELPER(zend_jit_find_ns_func_helper);
3000 REGISTER_HELPER(zend_jit_jmp_frameless_helper);
3001 REGISTER_HELPER(zend_jit_unref_helper);
3002 REGISTER_HELPER(zend_jit_invalid_method_call);
3003 REGISTER_HELPER(zend_jit_invalid_method_call_tmp);
3004 REGISTER_HELPER(zend_jit_find_method_helper);
3005 REGISTER_HELPER(zend_jit_find_method_tmp_helper);
3006 REGISTER_HELPER(zend_jit_push_static_metod_call_frame);
3007 REGISTER_HELPER(zend_jit_push_static_metod_call_frame_tmp);
3008 REGISTER_HELPER(zend_jit_free_trampoline_helper);
3009 REGISTER_HELPER(zend_jit_verify_return_slow);
3010 REGISTER_HELPER(zend_jit_deprecated_helper);
3011 REGISTER_HELPER(zend_jit_undefined_long_key);
3012 REGISTER_HELPER(zend_jit_undefined_string_key);
3013 REGISTER_HELPER(zend_jit_copy_extra_args_helper);
3014 REGISTER_HELPER(zend_jit_vm_stack_free_args_helper);
3015 REGISTER_HELPER(zend_free_extra_named_params);
3016 REGISTER_HELPER(zend_jit_free_call_frame);
3017 REGISTER_HELPER(zend_jit_exception_in_interrupt_handler_helper);
3018 REGISTER_HELPER(zend_jit_verify_arg_slow);
3019 REGISTER_HELPER(zend_missing_arg_error);
3020 REGISTER_HELPER(zend_jit_only_vars_by_reference);
3021 REGISTER_HELPER(zend_jit_leave_func_helper);
3022 REGISTER_HELPER(zend_jit_leave_nested_func_helper);
3023 REGISTER_HELPER(zend_jit_leave_top_func_helper);
3024 REGISTER_HELPER(zend_jit_fetch_global_helper);
3025 REGISTER_HELPER(zend_jit_hash_index_lookup_rw_no_packed);
3026 REGISTER_HELPER(zend_jit_hash_index_lookup_rw);
3027 REGISTER_HELPER(zend_jit_hash_lookup_rw);
3028 REGISTER_HELPER(zend_jit_symtable_find);
3029 REGISTER_HELPER(zend_jit_symtable_lookup_w);
3030 REGISTER_HELPER(zend_jit_symtable_lookup_rw);
3031 REGISTER_HELPER(zend_jit_fetch_dim_r_helper);
3032 REGISTER_HELPER(zend_jit_fetch_dim_is_helper);
3033 REGISTER_HELPER(zend_jit_fetch_dim_isset_helper);
3034 REGISTER_HELPER(zend_jit_fetch_dim_rw_helper);
3035 REGISTER_HELPER(zend_jit_fetch_dim_w_helper);
3036 REGISTER_HELPER(zend_jit_fetch_dim_str_offset_r_helper);
3037 REGISTER_HELPER(zend_jit_fetch_dim_str_r_helper);
3038 REGISTER_HELPER(zend_jit_fetch_dim_str_is_helper);
3039 REGISTER_HELPER(zend_jit_fetch_dim_obj_r_helper);
3040 REGISTER_HELPER(zend_jit_fetch_dim_obj_is_helper);
3041 REGISTER_HELPER(zend_jit_invalid_array_access);
3042 REGISTER_HELPER(zend_jit_zval_array_dup);
3043 REGISTER_HELPER(zend_jit_prepare_assign_dim_ref);
3044 REGISTER_HELPER(zend_jit_fetch_dim_obj_w_helper);
3045 REGISTER_HELPER(zend_jit_fetch_dim_obj_rw_helper);
3046 REGISTER_HELPER(zend_jit_isset_dim_helper);
3047 REGISTER_HELPER(zend_jit_assign_dim_helper);
3048 REGISTER_HELPER(zend_jit_assign_dim_op_helper);
3049 REGISTER_HELPER(zend_jit_fetch_obj_w_slow);
3050 REGISTER_HELPER(zend_jit_fetch_obj_r_slow);
3051 REGISTER_HELPER(zend_jit_fetch_obj_r_slow_ex);
3052 REGISTER_HELPER(zend_jit_fetch_obj_is_slow);
3053 REGISTER_HELPER(zend_jit_fetch_obj_is_slow_ex);
3054 REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic);
3055 REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic_ex);
3056 REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic);
3057 REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic_ex);
3058 REGISTER_HELPER(zend_jit_check_array_promotion);
3059 REGISTER_HELPER(zend_jit_create_typed_ref);
3060 REGISTER_HELPER(zend_jit_invalid_property_write);
3061 REGISTER_HELPER(zend_jit_invalid_property_read);
3062 REGISTER_HELPER(zend_jit_extract_helper);
3063 REGISTER_HELPER(zend_jit_invalid_property_assign);
3064 REGISTER_HELPER(zend_jit_assign_to_typed_prop);
3065 REGISTER_HELPER(zend_jit_assign_obj_helper);
3066 REGISTER_HELPER(zend_jit_invalid_property_assign_op);
3067 REGISTER_HELPER(zend_jit_assign_op_to_typed_prop);
3068 REGISTER_HELPER(zend_jit_assign_obj_op_helper);
3069 REGISTER_HELPER(zend_jit_invalid_property_incdec);
3070 REGISTER_HELPER(zend_jit_inc_typed_prop);
3071 REGISTER_HELPER(zend_jit_dec_typed_prop);
3072 REGISTER_HELPER(zend_jit_pre_inc_typed_prop);
3073 REGISTER_HELPER(zend_jit_post_inc_typed_prop);
3074 REGISTER_HELPER(zend_jit_pre_dec_typed_prop);
3075 REGISTER_HELPER(zend_jit_post_dec_typed_prop);
3076 REGISTER_HELPER(zend_jit_pre_inc_obj_helper);
3077 REGISTER_HELPER(zend_jit_post_inc_obj_helper);
3078 REGISTER_HELPER(zend_jit_pre_dec_obj_helper);
3079 REGISTER_HELPER(zend_jit_post_dec_obj_helper);
3080 REGISTER_HELPER(zend_jit_rope_end);
3081
3082 #ifndef ZTS
3083 REGISTER_DATA(EG(current_execute_data));
3084 REGISTER_DATA(EG(exception));
3085 REGISTER_DATA(EG(opline_before_exception));
3086 REGISTER_DATA(EG(vm_interrupt));
3087 REGISTER_DATA(EG(timed_out));
3088 REGISTER_DATA(EG(uninitialized_zval));
3089 REGISTER_DATA(EG(zend_constants));
3090 REGISTER_DATA(EG(jit_trace_num));
3091 REGISTER_DATA(EG(vm_stack_top));
3092 REGISTER_DATA(EG(vm_stack_end));
3093 REGISTER_DATA(EG(exception_op));
3094 REGISTER_DATA(EG(symbol_table));
3095
3096 REGISTER_DATA(CG(map_ptr_base));
3097 #endif
3098 #endif
3099 }
3100
zend_jit_calc_trace_prologue_size(void)3101 static void zend_jit_calc_trace_prologue_size(void)
3102 {
3103 zend_jit_ctx jit_ctx;
3104 zend_jit_ctx *jit = &jit_ctx;
3105 void *entry;
3106 size_t size;
3107
3108 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
3109
3110 if (!GCC_GLOBAL_REGS) {
3111 ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
3112 jit_STORE_FP(jit, ref);
3113 jit->ctx.flags |= IR_FASTCALL_FUNC;
3114 }
3115
3116 ir_UNREACHABLE();
3117
3118 entry = zend_jit_ir_compile(&jit->ctx, &size, "JIT$trace_prologue");
3119 zend_jit_free_ctx(jit);
3120
3121 if (!entry) {
3122 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: could not compile prologue");
3123 }
3124
3125 zend_jit_trace_prologue_size = size;
3126 }
3127
3128 #if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64)
3129 static uintptr_t zend_jit_hybrid_vm_sp_adj = 0;
3130
3131 typedef struct _Unwind_Context _Unwind_Context;
3132 typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *);
3133 extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *);
3134 extern uintptr_t _Unwind_GetCFA(_Unwind_Context *);
3135
3136 typedef struct _zend_jit_unwind_arg {
3137 int cnt;
3138 uintptr_t cfa[3];
3139 } zend_jit_unwind_arg;
3140
zend_jit_unwind_cb(_Unwind_Context * ctx,void * a)3141 static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a)
3142 {
3143 zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a;
3144 arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx);
3145 arg->cnt++;
3146 if (arg->cnt == 3) {
3147 return 5; // _URC_END_OF_STACK
3148 }
3149 return 0; // _URC_NO_REASON;
3150 }
3151
zend_jit_touch_vm_stack_data(void * vm_stack_data)3152 static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data)
3153 {
3154 zend_jit_unwind_arg arg;
3155
3156 memset(&arg, 0, sizeof(arg));
3157 _Unwind_Backtrace(zend_jit_unwind_cb, &arg);
3158 if (arg.cnt == 3) {
3159 zend_jit_hybrid_vm_sp_adj = arg.cfa[2] - arg.cfa[1];
3160 }
3161 }
3162
3163 extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data);
3164
zend_jit_set_sp_adj_vm(void)3165 static zend_never_inline void zend_jit_set_sp_adj_vm(void)
3166 {
3167 void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *);
3168
3169 orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data;
3170 zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data;
3171 execute_ex(NULL); // set sp_adj[SP_ADJ_VM]
3172 zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data;
3173 }
3174 #endif
3175
zend_jit_setup(void)3176 static void zend_jit_setup(void)
3177 {
3178 #if defined(IR_TARGET_X86)
3179 if (!zend_cpu_supports_sse2()) {
3180 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: CPU doesn't support SSE2");
3181 }
3182 #endif
3183 #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
3184 allowed_opt_flags = 0;
3185 if (zend_cpu_supports_avx()) {
3186 allowed_opt_flags |= ZEND_JIT_CPU_AVX;
3187 }
3188 # if PHP_HAVE_BUILTIN_CPU_SUPPORTS && defined(__GNUC__) && (ZEND_GCC_VERSION >= 11000)
3189 if (zend_cpu_supports_cldemote()) {
3190 default_mflags |= IR_X86_CLDEMOTE;
3191 }
3192 # endif
3193 #endif
3194 #ifdef ZTS
3195 #if defined(IR_TARGET_AARCH64)
3196 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
3197
3198 # ifdef __FreeBSD__
3199 if (tsrm_ls_cache_tcb_offset == 0) {
3200 TLSDescriptor **where;
3201
3202 __asm__(
3203 "adrp %0, :tlsdesc:_tsrm_ls_cache\n"
3204 "add %0, %0, :tlsdesc_lo12:_tsrm_ls_cache\n"
3205 : "=r" (where));
3206 /* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst
3207 * section "Relocations for thread-local storage".
3208 * The first entry holds a pointer to the variable's TLS descriptor resolver function and the second entry holds
3209 * a platform-specific offset or pointer. */
3210 TLSDescriptor *tlsdesc = where[1];
3211
3212 tsrm_tls_offset = tlsdesc->offset;
3213 /* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/22ca6db50f4e6bd75a141f57cf953d8de6531a06/lib/libc/gen/tls.c#L88) */
3214 tsrm_tls_index = (tlsdesc->index + 1) * 8;
3215 }
3216 # else
3217 ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0);
3218 # endif
3219 # elif defined(_WIN64)
3220 tsrm_tls_index = _tls_index * sizeof(void*);
3221
3222 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
3223 /* Probably, it might be better solution */
3224 do {
3225 void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index];
3226 void *val = _tsrm_ls_cache;
3227 size_t offset = 0;
3228 size_t size = (char*)&_tls_end - (char*)&_tls_start;
3229
3230 while (offset < size) {
3231 if (*tls_mem == val) {
3232 tsrm_tls_offset = offset;
3233 break;
3234 }
3235 tls_mem++;
3236 offset += sizeof(void*);
3237 }
3238 if (offset >= size) {
3239 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size");
3240 }
3241 } while(0);
3242 # elif defined(ZEND_WIN32)
3243 tsrm_tls_index = _tls_index * sizeof(void*);
3244
3245 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
3246 /* Probably, it might be better solution */
3247 do {
3248 void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index];
3249 void *val = _tsrm_ls_cache;
3250 size_t offset = 0;
3251 size_t size = (char*)&_tls_end - (char*)&_tls_start;
3252
3253 while (offset < size) {
3254 if (*tls_mem == val) {
3255 tsrm_tls_offset = offset;
3256 break;
3257 }
3258 tls_mem++;
3259 offset += sizeof(void*);
3260 }
3261 if (offset >= size) {
3262 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size");
3263 }
3264 } while(0);
3265 # elif defined(__APPLE__) && defined(__x86_64__)
3266 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
3267 if (tsrm_ls_cache_tcb_offset == 0) {
3268 size_t *ti;
3269 __asm__(
3270 "leaq __tsrm_ls_cache(%%rip),%0"
3271 : "=r" (ti));
3272 tsrm_tls_offset = ti[2];
3273 tsrm_tls_index = ti[1] * 8;
3274 }
3275 # elif defined(__GNUC__) && defined(__x86_64__)
3276 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
3277 if (tsrm_ls_cache_tcb_offset == 0) {
3278 #if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && \
3279 !defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
3280 size_t ret;
3281
3282 asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0"
3283 : "=r" (ret));
3284 tsrm_ls_cache_tcb_offset = ret;
3285 #elif defined(__MUSL__)
3286 size_t *ti;
3287
3288 __asm__(
3289 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
3290 : "=a" (ti));
3291 tsrm_tls_offset = ti[1];
3292 tsrm_tls_index = ti[0] * 8;
3293 #elif defined(__FreeBSD__)
3294 size_t *ti;
3295
3296 __asm__(
3297 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
3298 : "=a" (ti));
3299 tsrm_tls_offset = ti[1];
3300 /* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/bf56e8b9c8639ac4447d223b83cdc128107cc3cd/libexec/rtld-elf/rtld.c#L5260) */
3301 tsrm_tls_index = (ti[0] + 1) * 8;
3302 #else
3303 size_t *ti;
3304
3305 __asm__(
3306 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
3307 : "=a" (ti));
3308 tsrm_tls_offset = ti[1];
3309 tsrm_tls_index = ti[0] * 16;
3310 #endif
3311 }
3312 # elif defined(__GNUC__) && defined(__i386__)
3313 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
3314 if (tsrm_ls_cache_tcb_offset == 0) {
3315 #if !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
3316 size_t ret;
3317
3318 asm ("leal _tsrm_ls_cache@ntpoff,%0\n"
3319 : "=a" (ret));
3320 tsrm_ls_cache_tcb_offset = ret;
3321 #else
3322 size_t *ti, _ebx, _ecx, _edx;
3323
3324 __asm__(
3325 "call 1f\n"
3326 ".subsection 1\n"
3327 "1:\tmovl (%%esp), %%ebx\n\t"
3328 "ret\n"
3329 ".previous\n\t"
3330 "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t"
3331 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t"
3332 "call ___tls_get_addr@plt\n\t"
3333 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n"
3334 : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx));
3335 tsrm_tls_offset = ti[1];
3336 tsrm_tls_index = ti[0] * 8;
3337 #endif
3338 }
3339 # endif
3340 #endif
3341
3342 #if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64)
3343 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
3344 zend_jit_set_sp_adj_vm(); // set zend_jit_hybrid_vm_sp_adj
3345 }
3346 #endif
3347
3348 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
3349 zend_jit_setup_disasm();
3350 }
3351
3352 #ifndef _WIN32
3353 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
3354 ir_perf_jitdump_open();
3355 }
3356
3357 #endif
3358 zend_long debug = JIT_G(debug);
3359 if (!(debug & ZEND_JIT_DEBUG_ASM_STUBS)) {
3360 JIT_G(debug) &= ~(ZEND_JIT_DEBUG_IR_SRC|ZEND_JIT_DEBUG_IR_FINAL|
3361 ZEND_JIT_DEBUG_IR_CODEGEN|
3362 ZEND_JIT_DEBUG_IR_AFTER_SCCP|ZEND_JIT_DEBUG_IR_AFTER_CFG|ZEND_JIT_DEBUG_IR_AFTER_GCM|
3363 ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE|ZEND_JIT_DEBUG_IR_AFTER_REGS);
3364 }
3365
3366 zend_jit_calc_trace_prologue_size();
3367 zend_jit_setup_stubs();
3368 JIT_G(debug) = debug;
3369 }
3370
zend_jit_shutdown_ir(void)3371 static void zend_jit_shutdown_ir(void)
3372 {
3373 #ifndef _WIN32
3374 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
3375 ir_perf_jitdump_close();
3376 }
3377 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
3378 ir_gdb_unregister_all();
3379 }
3380 #endif
3381 #ifdef HAVE_CAPSTONE
3382 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
3383 ir_disasm_free();
3384 }
3385 #endif
3386 }
3387
3388 /* PHP control flow reconstruction helpers */
jit_IF_ex(zend_jit_ctx * jit,ir_ref condition,ir_ref true_block)3389 static ir_ref jit_IF_ex(zend_jit_ctx *jit, ir_ref condition, ir_ref true_block)
3390 {
3391 ir_ref ref = ir_IF(condition);
3392 /* op3 is used as a temporary storage for PHP BB number to reconstruct PHP control flow.
3393 *
3394 * It's used in jit_IF_TRUE_FALSE_ex() to select IF_TRUE or IF_FALSE instructions
3395 * to start target block
3396 */
3397 ir_set_op(&jit->ctx, ref, 3, true_block);
3398 return ref;
3399 }
3400
jit_IF_TRUE_FALSE_ex(zend_jit_ctx * jit,ir_ref if_ref,ir_ref true_block)3401 static void jit_IF_TRUE_FALSE_ex(zend_jit_ctx *jit, ir_ref if_ref, ir_ref true_block)
3402 {
3403 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
3404 ZEND_ASSERT(if_ref);
3405 ZEND_ASSERT(jit->ctx.ir_base[if_ref].op == IR_IF);
3406 ZEND_ASSERT(jit->ctx.ir_base[if_ref].op3);
3407 if (jit->ctx.ir_base[if_ref].op3 == true_block) {
3408 ir_IF_TRUE(if_ref);
3409 } else {
3410 ir_IF_FALSE(if_ref);
3411 }
3412 }
3413
_zend_jit_add_predecessor_ref(zend_jit_ctx * jit,int b,int pred,ir_ref ref)3414 static void _zend_jit_add_predecessor_ref(zend_jit_ctx *jit, int b, int pred, ir_ref ref)
3415 {
3416 int i, *p;
3417 zend_basic_block *bb;
3418 ir_ref *r, header;
3419
3420 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
3421 bb = &jit->ssa->cfg.blocks[b];
3422 p = &jit->ssa->cfg.predecessors[bb->predecessor_offset];
3423 r = &jit->bb_edges[jit->bb_predecessors[b]];
3424 for (i = 0; i < bb->predecessors_count; i++, p++, r++) {
3425 if (*p == pred) {
3426 ZEND_ASSERT(*r == IR_UNUSED || *r == ref);
3427 header = jit->bb_start_ref[b];
3428 if (header) {
3429 /* this is back edge */
3430 ZEND_ASSERT(jit->ctx.ir_base[header].op == IR_LOOP_BEGIN);
3431 if (jit->ctx.ir_base[ref].op == IR_END) {
3432 jit->ctx.ir_base[ref].op = IR_LOOP_END;
3433 } else if (jit->ctx.ir_base[ref].op == IR_IF) {
3434 jit_IF_TRUE_FALSE_ex(jit, ref, b);
3435 ref = ir_LOOP_END();
3436 } else if (jit->ctx.ir_base[ref].op == IR_UNREACHABLE) {
3437 ir_BEGIN(ref);
3438 ref = ir_LOOP_END();
3439 } else {
3440 ZEND_UNREACHABLE();
3441 }
3442 ir_MERGE_SET_OP(header, i + 1, ref);
3443 }
3444 *r = ref;
3445 return;
3446 }
3447 }
3448 ZEND_UNREACHABLE();
3449 }
3450
_zend_jit_merge_smart_branch_inputs(zend_jit_ctx * jit,uint32_t true_label,uint32_t false_label,ir_ref true_inputs,ir_ref false_inputs)3451 static void _zend_jit_merge_smart_branch_inputs(zend_jit_ctx *jit,
3452 uint32_t true_label,
3453 uint32_t false_label,
3454 ir_ref true_inputs,
3455 ir_ref false_inputs)
3456 {
3457 ir_ref true_path = IR_UNUSED, false_path = IR_UNUSED;
3458
3459 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
3460 if (true_inputs) {
3461 ZEND_ASSERT(jit->ctx.ir_base[true_inputs].op == IR_END);
3462 if (!jit->ctx.ir_base[true_inputs].op2) {
3463 true_path = true_inputs;
3464 } else {
3465 ir_MERGE_list(true_inputs);
3466 true_path = ir_END();
3467 }
3468 }
3469 if (false_inputs) {
3470 ZEND_ASSERT(jit->ctx.ir_base[false_inputs].op == IR_END);
3471 if (!jit->ctx.ir_base[false_inputs].op2) {
3472 false_path = false_inputs;
3473 } else {
3474 ir_MERGE_list(false_inputs);
3475 false_path = ir_END();
3476 }
3477 }
3478
3479 if (true_label == false_label && true_path && false_path) {
3480 ir_MERGE_2(true_path, false_path);
3481 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END());
3482 } else if (!true_path && !false_path) {
3483 /* dead code */
3484 true_path = ir_END();
3485 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path);
3486 } else {
3487 if (true_path) {
3488 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path);
3489 }
3490 if (false_path) {
3491 _zend_jit_add_predecessor_ref(jit, false_label, jit->b, false_path);
3492 }
3493 }
3494
3495 jit->b = -1;
3496 }
3497
_zend_jit_fix_merges(zend_jit_ctx * jit)3498 static void _zend_jit_fix_merges(zend_jit_ctx *jit)
3499 {
3500 int i, count;
3501 ir_ref j, k, n, *p, *q, *r;
3502 ir_ref ref;
3503 ir_insn *insn, *phi;
3504
3505 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
3506 count = jit->ssa->cfg.blocks_count;
3507 for (i = 0, p = jit->bb_start_ref; i < count; i++, p++) {
3508 ref = *p;
3509 if (ref) {
3510 insn = &jit->ctx.ir_base[ref];
3511 if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN) {
3512 n = insn->inputs_count;
3513 /* Remove IS_UNUSED inputs */
3514 for (j = k = 0, q = r = insn->ops + 1; j < n; j++, q++) {
3515 if (*q) {
3516 if (q != r) {
3517 *r = *q;
3518 phi = insn + 1 + (n >> 2);
3519 while (phi->op == IR_PI) {
3520 phi++;
3521 }
3522 while (phi->op == IR_PHI) {
3523 ir_insn_set_op(phi, k + 2, ir_insn_op(phi, j + 2));
3524 phi += 1 + ((n + 1) >> 2);
3525 }
3526 }
3527 k++;
3528 r++;
3529 }
3530 }
3531 if (k != n) {
3532 ir_ref n2, k2;
3533
3534 if (k <= 1) {
3535 insn->op = IR_BEGIN;
3536 insn->inputs_count = 0;
3537 } else {
3538 insn->inputs_count = k;
3539 }
3540 n2 = 1 + (n >> 2);
3541 k2 = 1 + (k >> 2);
3542 while (k2 != n2) {
3543 (insn+k2)->optx = IR_NOP;
3544 k2++;
3545 }
3546 phi = insn + 1 + (n >> 2);
3547 while (phi->op == IR_PI) {
3548 phi++;
3549 }
3550 while (phi->op == IR_PHI) {
3551 if (k <= 1) {
3552 phi->op = IR_COPY;
3553 phi->op1 = phi->op2;
3554 phi->op2 = 1;
3555 phi->inputs_count = 0;
3556 } else {
3557 phi->inputs_count = k + 1;
3558 }
3559 n2 = 1 + ((n + 1) >> 2);
3560 k2 = 1 + ((k + 1) >> 2);
3561 while (k2 != n2) {
3562 (phi+k2)->optx = IR_NOP;
3563 k2++;
3564 }
3565 phi += 1 + ((n + 1) >> 2);
3566 }
3567 }
3568 }
3569 }
3570 }
3571 }
3572
zend_jit_case_start(zend_jit_ctx * jit,int switch_b,int case_b,ir_ref switch_ref)3573 static void zend_jit_case_start(zend_jit_ctx *jit, int switch_b, int case_b, ir_ref switch_ref)
3574 {
3575 zend_basic_block *bb = &jit->ssa->cfg.blocks[switch_b];
3576 const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1];
3577
3578 if (opline->opcode == ZEND_SWITCH_LONG
3579 || opline->opcode == ZEND_SWITCH_STRING
3580 || opline->opcode == ZEND_MATCH) {
3581 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
3582 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
3583 int default_b = jit->ssa->cfg.map[default_opline - jit->op_array->opcodes];
3584 zval *zv;
3585 ir_ref list = IR_UNUSED, idx;
3586 bool first = 1;
3587
3588 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
3589 const zend_op *target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
3590 int b = jit->ssa->cfg.map[target - jit->op_array->opcodes];
3591
3592 if (b == case_b) {
3593 if (!first) {
3594 ir_END_list(list);
3595 }
3596 if (HT_IS_PACKED(jumptable)) {
3597 idx = ir_CONST_LONG(zv - jumptable->arPacked);
3598 } else {
3599 idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
3600 }
3601 ir_CASE_VAL(switch_ref, idx);
3602 first = 0;
3603 }
3604 } ZEND_HASH_FOREACH_END();
3605 if (default_b == case_b) {
3606 if (!first) {
3607 ir_END_list(list);
3608 }
3609 if (jit->ctx.ir_base[switch_ref].op3) {
3610 /* op3 may contain a list of additional "default" path inputs for MATCH */
3611 ir_ref ref = jit->ctx.ir_base[switch_ref].op3;
3612 jit->ctx.ir_base[switch_ref].op3 = IS_UNDEF;
3613 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END);
3614 ir_ref end = ref;
3615 while (jit->ctx.ir_base[end].op2) {
3616 ZEND_ASSERT(jit->ctx.ir_base[end].op == IR_END);
3617 end = jit->ctx.ir_base[end].op2;
3618 }
3619 jit->ctx.ir_base[end].op2 = list;
3620 list = ref;
3621 }
3622 ir_CASE_DEFAULT(switch_ref);
3623 }
3624 if (list) {
3625 ir_END_list(list);
3626 ir_MERGE_list(list);
3627 }
3628 } else {
3629 ZEND_UNREACHABLE();
3630 }
3631 }
3632
zend_jit_bb_start(zend_jit_ctx * jit,int b)3633 static int zend_jit_bb_start(zend_jit_ctx *jit, int b)
3634 {
3635 zend_basic_block *bb;
3636 int i, n, *p, pred;
3637 ir_ref ref, bb_start;
3638
3639 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
3640 ZEND_ASSERT(b < jit->ssa->cfg.blocks_count);
3641 bb = &jit->ssa->cfg.blocks[b];
3642 ZEND_ASSERT((bb->flags & ZEND_BB_REACHABLE) != 0);
3643 n = bb->predecessors_count;
3644
3645 if (n == 0) {
3646 /* pass */
3647 ZEND_ASSERT(jit->ctx.control);
3648 #if ZEND_DEBUG
3649 ref = jit->ctx.control;
3650 ir_insn *insn = &jit->ctx.ir_base[ref];
3651 while (insn->op >= IR_CALL && insn->op <= IR_TRAP) {
3652 ref = insn->op1;
3653 insn = &jit->ctx.ir_base[ref];
3654 }
3655 ZEND_ASSERT(insn->op == IR_START);
3656 ZEND_ASSERT(ref == 1);
3657 #endif
3658 bb_start = 1;
3659 if (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY) {
3660 /* prvent END/BEGIN merging */
3661 jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ir_END());
3662 bb_start = jit->ctx.control;
3663 }
3664 } else if (n == 1) {
3665 ZEND_ASSERT(!jit->ctx.control);
3666 pred = jit->ssa->cfg.predecessors[bb->predecessor_offset];
3667 ref = jit->bb_edges[jit->bb_predecessors[b]];
3668 if (ref == IR_UNUSED) {
3669 if (!jit->ctx.control) {
3670 ir_BEGIN(IR_UNUSED); /* unreachable block */
3671 }
3672 } else {
3673 ir_op op = jit->ctx.ir_base[ref].op;
3674
3675 if (op == IR_IF) {
3676 if (!jit->ctx.control) {
3677 jit_IF_TRUE_FALSE_ex(jit, ref, b);
3678 } else {
3679 ir_ref entry_path = ir_END();
3680 jit_IF_TRUE_FALSE_ex(jit, ref, b);
3681 ir_MERGE_WITH(entry_path);
3682 }
3683 } else if (op == IR_SWITCH) {
3684 zend_jit_case_start(jit, pred, b, ref);
3685 } else {
3686 if (!jit->ctx.control) {
3687 ZEND_ASSERT(op == IR_END || op == IR_UNREACHABLE || op == IR_RETURN);
3688 if ((jit->ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY)
3689 && (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)) {
3690 /* prvent END/BEGIN merging */
3691 jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ref);
3692 } else {
3693 ir_BEGIN(ref);
3694 }
3695 } else {
3696 ir_MERGE_WITH(ref);
3697 }
3698 }
3699 }
3700 bb_start = jit->ctx.control;
3701 } else {
3702 int forward_edges_count = 0;
3703 int back_edges_count = 0;
3704 ir_ref *pred_refs;
3705 ir_ref entry_path = IR_UNUSED;
3706 ALLOCA_FLAG(use_heap);
3707
3708 ZEND_ASSERT(!jit->ctx.control);
3709 if (jit->ctx.control) {
3710 entry_path = ir_END();
3711 }
3712 pred_refs = (ir_ref *)do_alloca(sizeof(ir_ref) * n, use_heap);
3713 for (i = 0, p = jit->ssa->cfg.predecessors + bb->predecessor_offset; i < n; p++, i++) {
3714 pred = *p;
3715 if (jit->bb_start_ref[pred]) {
3716 /* forward edge */
3717 forward_edges_count++;
3718 ref = jit->bb_edges[jit->bb_predecessors[b] + i];
3719 if (ref == IR_UNUSED) {
3720 /* dead edge */
3721 pred_refs[i] = IR_UNUSED;
3722 } else {
3723 ir_op op = jit->ctx.ir_base[ref].op;
3724
3725 if (op == IR_IF) {
3726 jit_IF_TRUE_FALSE_ex(jit, ref, b);
3727 pred_refs[i] = ir_END();
3728 } else if (op == IR_SWITCH) {
3729 zend_jit_case_start(jit, pred, b, ref);
3730 pred_refs[i] = ir_END();
3731 } else {
3732 ZEND_ASSERT(op == IR_END || op == IR_UNREACHABLE || op == IR_RETURN);
3733 pred_refs[i] = ref;
3734 }
3735 }
3736 } else {
3737 /* backward edge */
3738 back_edges_count++;
3739 pred_refs[i] = IR_UNUSED;
3740 }
3741 }
3742
3743 if (bb->flags & ZEND_BB_LOOP_HEADER) {
3744 ZEND_ASSERT(back_edges_count != 0);
3745 ZEND_ASSERT(forward_edges_count != 0);
3746 ir_MERGE_N(n, pred_refs);
3747 jit->ctx.ir_base[jit->ctx.control].op = IR_LOOP_BEGIN;
3748 bb_start = jit->ctx.control;
3749 if (entry_path) {
3750 ir_MERGE_WITH(entry_path);
3751 }
3752 } else {
3753 // ZEND_ASSERT(back_edges_count != 0);
3754 /* edges from exceptional blocks may be counted as back edges */
3755 ir_MERGE_N(n, pred_refs);
3756 bb_start = jit->ctx.control;
3757 if (entry_path) {
3758 ir_MERGE_WITH(entry_path);
3759 }
3760 }
3761 free_alloca(pred_refs, use_heap);
3762 }
3763 jit->b = b;
3764 jit->bb_start_ref[b] = bb_start;
3765
3766 if ((bb->flags & ZEND_BB_ENTRY) || (bb->idom >= 0 && jit->bb_start_ref[bb->idom] < jit->ctx.fold_cse_limit)) {
3767 jit->ctx.fold_cse_limit = bb_start;
3768 }
3769
3770 return 1;
3771 }
3772
zend_jit_bb_end(zend_jit_ctx * jit,int b)3773 static int zend_jit_bb_end(zend_jit_ctx *jit, int b)
3774 {
3775 int succ;
3776 zend_basic_block *bb;
3777
3778 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
3779 if (jit->b != b) {
3780 return 1;
3781 }
3782
3783 bb = &jit->ssa->cfg.blocks[b];
3784 ZEND_ASSERT(bb->successors_count != 0);
3785 if (bb->successors_count == 1) {
3786 succ = bb->successors[0];
3787 } else {
3788 const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1];
3789
3790 /* Use only the following successor of SWITCH and FE_RESET_R */
3791 ZEND_ASSERT(opline->opcode == ZEND_SWITCH_LONG
3792 || opline->opcode == ZEND_SWITCH_STRING
3793 || opline->opcode == ZEND_MATCH
3794 || opline->opcode == ZEND_FE_RESET_R);
3795 succ = b + 1;
3796 }
3797 _zend_jit_add_predecessor_ref(jit, succ, b, ir_END());
3798 jit->b = -1;
3799 return 1;
3800 }
3801
jit_CMP_IP(zend_jit_ctx * jit,ir_op op,const zend_op * next_opline)3802 static int jit_CMP_IP(zend_jit_ctx *jit, ir_op op, const zend_op *next_opline)
3803 {
3804 ir_ref ref;
3805
3806 #if 1
3807 if (GCC_GLOBAL_REGS) {
3808 ref = jit_IP32(jit);
3809 } else {
3810 ref = ir_LOAD_U32(jit_EX(opline));
3811 }
3812 ref = ir_CMP_OP(op, ref, ir_CONST_U32((uint32_t)(uintptr_t)next_opline));
3813 #else
3814 if (GCC_GLOBAL_REGS) {
3815 ref = jit_IP(jit);
3816 } else {
3817 ref = ir_LOAD_A(jit_EX(opline));
3818 }
3819 ref = ir_CMP_OP(op, ref, ir_CONST_ADDR(next_opline));
3820 #endif
3821 return ref;
3822 }
3823
zend_jit_jmp_frameless(zend_jit_ctx * jit,const zend_op * opline,const void * exit_addr,zend_jmp_fl_result guard)3824 static int zend_jit_jmp_frameless(
3825 zend_jit_ctx *jit,
3826 const zend_op *opline,
3827 const void *exit_addr,
3828 zend_jmp_fl_result guard
3829 ) {
3830 ir_ref ref, if_ref, cache_result, function_result, phi_result, cache_slot_ref;
3831 zend_basic_block *bb;
3832
3833 // JIT: CACHED_PTR(opline->extended_value)
3834 cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value);
3835 cache_result = ir_LOAD_L(cache_slot_ref);
3836
3837 // JIT: if (UNEXPECTED(!result))
3838 if_ref = ir_IF(cache_result);
3839 ir_IF_FALSE_cold(if_ref);
3840 zval *func_name_zv = RT_CONSTANT(opline, opline->op1);
3841 function_result = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_jit_jmp_frameless_helper),
3842 ir_CONST_ADDR(func_name_zv),
3843 cache_slot_ref);
3844 ir_MERGE_WITH_EMPTY_TRUE(if_ref);
3845
3846 phi_result = ir_PHI_2(IR_LONG, function_result, cache_result);
3847
3848 if (exit_addr) {
3849 ir_GUARD(ir_EQ(phi_result, ir_CONST_LONG(guard)), ir_CONST_ADDR(exit_addr));
3850 } else {
3851 ZEND_ASSERT(jit->b >= 0);
3852 bb = &jit->ssa->cfg.blocks[jit->b];
3853 // JIT: if (result == ZEND_JMP_FL_HIT)
3854 ref = jit_IF_ex(jit, ir_EQ(phi_result, ir_CONST_LONG(ZEND_JMP_FL_HIT)), bb->successors[0]);
3855 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
3856 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
3857 jit->b = -1;
3858 }
3859
3860 return 1;
3861 }
3862
zend_jit_cond_jmp(zend_jit_ctx * jit,const zend_op * next_opline,int target_block)3863 static int zend_jit_cond_jmp(zend_jit_ctx *jit, const zend_op *next_opline, int target_block)
3864 {
3865 ir_ref ref;
3866 zend_basic_block *bb;
3867
3868 ZEND_ASSERT(jit->b >= 0);
3869 bb = &jit->ssa->cfg.blocks[jit->b];
3870
3871 ZEND_ASSERT(bb->successors_count == 2);
3872 if (bb->successors[0] == bb->successors[1]) {
3873 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
3874 jit->b = -1;
3875 zend_jit_set_last_valid_opline(jit, next_opline);
3876 return 1;
3877 }
3878
3879 ref = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, next_opline), target_block);
3880
3881 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
3882 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
3883
3884 jit->b = -1;
3885 zend_jit_set_last_valid_opline(jit, next_opline);
3886
3887 return 1;
3888 }
3889
zend_jit_set_cond(zend_jit_ctx * jit,const zend_op * next_opline,uint32_t var)3890 static int zend_jit_set_cond(zend_jit_ctx *jit, const zend_op *next_opline, uint32_t var)
3891 {
3892 ir_ref ref;
3893
3894 ref = ir_ADD_U32(ir_ZEXT_U32(jit_CMP_IP(jit, IR_EQ, next_opline)), ir_CONST_U32(IS_FALSE));
3895
3896 // EX_VAR(var) = ...
3897 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), var + offsetof(zval, u1.type_info)), ref);
3898
3899 zend_jit_reset_last_valid_opline(jit);
3900 return zend_jit_set_ip(jit, next_opline - 1);
3901 }
3902
3903 /* PHP JIT handlers */
zend_jit_check_exception(zend_jit_ctx * jit)3904 static void zend_jit_check_exception(zend_jit_ctx *jit)
3905 {
3906 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
3907 jit_STUB_ADDR(jit, jit_stub_exception_handler));
3908 }
3909
zend_jit_check_exception_undef_result(zend_jit_ctx * jit,const zend_op * opline)3910 static void zend_jit_check_exception_undef_result(zend_jit_ctx *jit, const zend_op *opline)
3911 {
3912 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
3913 jit_STUB_ADDR(jit,
3914 (opline->result_type & (IS_TMP_VAR|IS_VAR)) ? jit_stub_exception_handler_undef : jit_stub_exception_handler));
3915 }
3916
zend_jit_type_check_undef(zend_jit_ctx * jit,ir_ref type,uint32_t var,const zend_op * opline,bool check_exception,bool in_cold_path,bool undef_result)3917 static void zend_jit_type_check_undef(zend_jit_ctx *jit,
3918 ir_ref type,
3919 uint32_t var,
3920 const zend_op *opline,
3921 bool check_exception,
3922 bool in_cold_path,
3923 bool undef_result)
3924 {
3925 ir_ref if_def = ir_IF(type);
3926
3927 if (!in_cold_path) {
3928 ir_IF_FALSE_cold(if_def);
3929 } else {
3930 ir_IF_FALSE(if_def);
3931 }
3932 if (opline) {
3933 jit_SET_EX_OPLINE(jit, opline);
3934 }
3935 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(var));
3936 if (check_exception) {
3937 if (undef_result) {
3938 zend_jit_check_exception_undef_result(jit, opline);
3939 } else {
3940 zend_jit_check_exception(jit);
3941 }
3942 }
3943 ir_MERGE_WITH_EMPTY_TRUE(if_def);
3944 }
3945
zend_jit_zval_check_undef(zend_jit_ctx * jit,ir_ref ref,uint32_t var,const zend_op * opline,bool check_exception)3946 static ir_ref zend_jit_zval_check_undef(zend_jit_ctx *jit,
3947 ir_ref ref,
3948 uint32_t var,
3949 const zend_op *opline,
3950 bool check_exception)
3951 {
3952 ir_ref if_def, ref2;
3953
3954 if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
3955 ir_IF_FALSE_cold(if_def);
3956
3957 if (opline) {
3958 jit_SET_EX_OPLINE(jit, opline);
3959 }
3960
3961 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(var));
3962
3963 if (check_exception) {
3964 zend_jit_check_exception(jit);
3965 }
3966
3967 ref2 = jit_EG(uninitialized_zval);
3968
3969 ir_MERGE_WITH_EMPTY_TRUE(if_def);
3970
3971 return ir_PHI_2(IR_ADDR, ref2, ref);
3972 }
3973
zend_jit_recv_entry(zend_jit_ctx * jit,int b)3974 static void zend_jit_recv_entry(zend_jit_ctx *jit, int b)
3975 {
3976 zend_basic_block *bb = &jit->ssa->cfg.blocks[b];
3977 int pred;
3978 ir_ref ref;
3979
3980 ZEND_ASSERT(bb->predecessors_count > 0);
3981
3982 pred = jit->bb_predecessors[b];
3983 ref = jit->bb_edges[pred];
3984
3985 ZEND_ASSERT(ref);
3986 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END);
3987
3988 /* Insert a MERGE block with additional ENTRY input between predecessor and this one */
3989 ir_ENTRY(ref, bb->start);
3990 if (!GCC_GLOBAL_REGS) {
3991 /* 2 is hardcoded reference to IR_PARAM */
3992 ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
3993 ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
3994 jit_STORE_FP(jit, 2);
3995 }
3996
3997 ir_MERGE_WITH(ref);
3998 jit->bb_edges[pred] = ir_END();
3999 }
4000
zend_jit_osr_entry(zend_jit_ctx * jit,int b)4001 static void zend_jit_osr_entry(zend_jit_ctx *jit, int b)
4002 {
4003 zend_basic_block *bb = &jit->ssa->cfg.blocks[b];
4004 ir_ref ref = ir_END();
4005
4006 /* Insert a MERGE block with additional ENTRY input between predecessor and this one */
4007 ir_ENTRY(ref, bb->start);
4008 if (!GCC_GLOBAL_REGS) {
4009 /* 2 is hardcoded reference to IR_PARAM */
4010 ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
4011 ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
4012 jit_STORE_FP(jit, 2);
4013 }
4014
4015 ir_MERGE_WITH(ref);
4016 }
4017
zend_jit_continue_entry(zend_jit_ctx * jit,ir_ref src,unsigned int label)4018 static ir_ref zend_jit_continue_entry(zend_jit_ctx *jit, ir_ref src, unsigned int label)
4019 {
4020 ir_ENTRY(src, label);
4021 if (!GCC_GLOBAL_REGS) {
4022 /* 2 is hardcoded reference to IR_PARAM */
4023 ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
4024 ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
4025 jit_STORE_FP(jit, 2);
4026 }
4027 return ir_END();
4028 }
4029
zend_jit_handler(zend_jit_ctx * jit,const zend_op * opline,int may_throw)4030 static int zend_jit_handler(zend_jit_ctx *jit, const zend_op *opline, int may_throw)
4031 {
4032 ir_ref ref;
4033 const void *handler;
4034
4035 zend_jit_set_ip(jit, opline);
4036 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
4037 handler = zend_get_opcode_handler_func(opline);
4038 } else {
4039 handler = opline->handler;
4040 }
4041 if (GCC_GLOBAL_REGS) {
4042 ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
4043 } else {
4044 ref = jit_FP(jit);
4045 ref = ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(handler), ref);
4046 }
4047 if (may_throw) {
4048 zend_jit_check_exception(jit);
4049 }
4050 /* Skip the following OP_DATA */
4051 switch (opline->opcode) {
4052 case ZEND_ASSIGN_DIM:
4053 case ZEND_ASSIGN_OBJ:
4054 case ZEND_ASSIGN_STATIC_PROP:
4055 case ZEND_ASSIGN_DIM_OP:
4056 case ZEND_ASSIGN_OBJ_OP:
4057 case ZEND_ASSIGN_STATIC_PROP_OP:
4058 case ZEND_ASSIGN_STATIC_PROP_REF:
4059 case ZEND_ASSIGN_OBJ_REF:
4060 zend_jit_set_last_valid_opline(jit, opline + 2);
4061 break;
4062 default:
4063 zend_jit_set_last_valid_opline(jit, opline + 1);
4064 break;
4065 }
4066 return 1;
4067 }
4068
zend_jit_tail_handler(zend_jit_ctx * jit,const zend_op * opline)4069 static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline)
4070 {
4071 const void *handler;
4072 ir_ref ref;
4073 zend_basic_block *bb;
4074
4075 zend_jit_set_ip(jit, opline);
4076 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
4077 if (opline->opcode == ZEND_DO_UCALL ||
4078 opline->opcode == ZEND_DO_FCALL_BY_NAME ||
4079 opline->opcode == ZEND_DO_FCALL ||
4080 opline->opcode == ZEND_RETURN) {
4081
4082 /* Use inlined HYBRID VM handler */
4083 handler = opline->handler;
4084 ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
4085 } else {
4086 handler = zend_get_opcode_handler_func(opline);
4087 ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
4088 ref = ir_LOAD_A(jit_IP(jit));
4089 ir_TAILCALL(IR_VOID, ref);
4090 }
4091 } else {
4092 handler = opline->handler;
4093 if (GCC_GLOBAL_REGS) {
4094 ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
4095 } else if ((jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)
4096 && (opline->opcode == ZEND_CATCH
4097 || opline->opcode == ZEND_FAST_CALL
4098 || opline->opcode == ZEND_FAST_RET
4099 || opline->opcode == ZEND_MATCH_ERROR
4100 || opline->opcode == ZEND_THROW
4101 || opline->opcode == ZEND_VERIFY_NEVER_TYPE)) {
4102 ref = jit_FP(jit);
4103 ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
4104 ir_RETURN(ir_CONST_I32(1));
4105 } else {
4106 ref = jit_FP(jit);
4107 ir_TAILCALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
4108 }
4109 }
4110 if (jit->b >= 0) {
4111 bb = &jit->ssa->cfg.blocks[jit->b];
4112 if (bb->successors_count > 0
4113 && (opline->opcode == ZEND_DO_FCALL
4114 || opline->opcode == ZEND_DO_UCALL
4115 || opline->opcode == ZEND_DO_FCALL_BY_NAME
4116 || opline->opcode == ZEND_INCLUDE_OR_EVAL
4117 || opline->opcode == ZEND_GENERATOR_CREATE
4118 || opline->opcode == ZEND_YIELD
4119 || opline->opcode == ZEND_YIELD_FROM
4120 || opline->opcode == ZEND_FAST_CALL)) {
4121 /* Add a fake control edge from UNREACHABLE to the following ENTRY */
4122 int succ;
4123
4124 if (bb->successors_count == 1) {
4125 succ = bb->successors[0];
4126 ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY);
4127 } else {
4128 /* Use only the following successor of FAST_CALL */
4129 ZEND_ASSERT(opline->opcode == ZEND_FAST_CALL);
4130 succ = jit->b + 1;
4131 /* we need an entry */
4132 jit->ssa->cfg.blocks[succ].flags |= ZEND_BB_ENTRY;
4133 }
4134 ref = jit->ctx.insns_count - 1;
4135 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE || jit->ctx.ir_base[ref].op == IR_RETURN);
4136 ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start);
4137 _zend_jit_add_predecessor_ref(jit, succ, jit->b, ref);
4138 }
4139 jit->b = -1;
4140 zend_jit_reset_last_valid_opline(jit);
4141 }
4142 return 1;
4143 }
4144
zend_jit_call(zend_jit_ctx * jit,const zend_op * opline,unsigned int next_block)4145 static int zend_jit_call(zend_jit_ctx *jit, const zend_op *opline, unsigned int next_block)
4146 {
4147 return zend_jit_tail_handler(jit, opline);
4148 }
4149
zend_jit_spill_store(zend_jit_ctx * jit,zend_jit_addr src,zend_jit_addr dst,uint32_t info,bool set_type)4150 static int zend_jit_spill_store(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type)
4151 {
4152 ZEND_ASSERT(Z_MODE(src) == IS_REG);
4153 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
4154
4155 if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
4156 jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src));
4157 if (set_type &&
4158 (Z_REG(dst) != ZREG_FP ||
4159 !JIT_G(current_frame) ||
4160 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
4161 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4162 }
4163 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4164 jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src));
4165 if (set_type &&
4166 (Z_REG(dst) != ZREG_FP ||
4167 !JIT_G(current_frame) ||
4168 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
4169 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4170 }
4171 } else {
4172 ZEND_UNREACHABLE();
4173 }
4174 return 1;
4175 }
4176
zend_jit_spill_store_inv(zend_jit_ctx * jit,zend_jit_addr src,zend_jit_addr dst,uint32_t info)4177 static int zend_jit_spill_store_inv(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
4178 {
4179 ZEND_ASSERT(Z_MODE(src) == IS_REG);
4180 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
4181
4182 if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
4183 jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src));
4184 if (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame)) {
4185 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4186 } else if (STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG) {
4187 /* invalidate memory type */
4188 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) = IS_UNKNOWN;
4189 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4190 }
4191 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4192 jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src));
4193 if (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame)) {
4194 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4195 } else if (STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE) {
4196 /* invalidate memory type */
4197 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) = IS_UNKNOWN;
4198 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4199 }
4200 } else {
4201 ZEND_UNREACHABLE();
4202 }
4203 return 1;
4204 }
4205
zend_jit_load_reg(zend_jit_ctx * jit,zend_jit_addr src,zend_jit_addr dst,uint32_t info)4206 static int zend_jit_load_reg(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
4207 {
4208 ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL);
4209 ZEND_ASSERT(Z_MODE(dst) == IS_REG);
4210
4211 if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
4212 zend_jit_def_reg(jit, dst, jit_Z_LVAL(jit, src));
4213 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4214 zend_jit_def_reg(jit, dst, jit_Z_DVAL(jit, src));
4215 } else {
4216 ZEND_UNREACHABLE();
4217 }
4218 return 1;
4219 }
4220
zend_jit_store_var(zend_jit_ctx * jit,uint32_t info,int var,int ssa_var,bool set_type)4221 static int zend_jit_store_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var, bool set_type)
4222 {
4223 zend_jit_addr src = ZEND_ADDR_REG(ssa_var);
4224 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4225
4226 return zend_jit_spill_store(jit, src, dst, info, set_type);
4227 }
4228
zend_jit_store_ref(zend_jit_ctx * jit,uint32_t info,int var,int32_t src,bool set_type)4229 static int zend_jit_store_ref(zend_jit_ctx *jit, uint32_t info, int var, int32_t src, bool set_type)
4230 {
4231 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4232
4233 if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
4234 jit_set_Z_LVAL(jit, dst, src);
4235 if (set_type &&
4236 (Z_REG(dst) != ZREG_FP ||
4237 !JIT_G(current_frame) ||
4238 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
4239 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4240 }
4241 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4242 jit_set_Z_DVAL(jit, dst, src);
4243 if (set_type &&
4244 (Z_REG(dst) != ZREG_FP ||
4245 !JIT_G(current_frame) ||
4246 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
4247 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4248 }
4249 } else {
4250 ZEND_UNREACHABLE();
4251 }
4252 return 1;
4253 }
4254
zend_jit_deopt_rload(zend_jit_ctx * jit,ir_type type,int32_t reg)4255 static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t reg)
4256 {
4257 ir_ref ref = jit->ctx.control;
4258 ir_insn *insn;
4259
4260 while (1) {
4261 insn = &jit->ctx.ir_base[ref];
4262 if (insn->op == IR_RLOAD && insn->op2 == reg) {
4263 ZEND_ASSERT(insn->type == type);
4264 return ref;
4265 } else if (insn->op == IR_START) {
4266 break;
4267 }
4268 ref = insn->op1;
4269 }
4270 return ir_RLOAD(type, reg);
4271 }
4272
zend_jit_store_const_long(zend_jit_ctx * jit,int var,zend_long val)4273 static int zend_jit_store_const_long(zend_jit_ctx *jit, int var, zend_long val)
4274 {
4275 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4276 ir_ref src = ir_CONST_LONG(val);
4277
4278 if (jit->ra && jit->ra[var].ref == IR_NULL) {
4279 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4280 }
4281 jit_set_Z_LVAL(jit, dst, src);
4282 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4283 return 1;
4284 }
4285
zend_jit_store_const_double(zend_jit_ctx * jit,int var,double val)4286 static int zend_jit_store_const_double(zend_jit_ctx *jit, int var, double val)
4287 {
4288 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4289 ir_ref src = ir_CONST_DOUBLE(val);
4290
4291 if (jit->ra && jit->ra[var].ref == IR_NULL) {
4292 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4293 }
4294 jit_set_Z_DVAL(jit, dst, src);
4295 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4296 return 1;
4297 }
4298
zend_jit_store_type(zend_jit_ctx * jit,int var,uint8_t type)4299 static int zend_jit_store_type(zend_jit_ctx *jit, int var, uint8_t type)
4300 {
4301 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4302
4303 ZEND_ASSERT(type <= IS_DOUBLE);
4304 jit_set_Z_TYPE_INFO(jit, dst, type);
4305 return 1;
4306 }
4307
zend_jit_store_reg(zend_jit_ctx * jit,uint32_t info,int var,int8_t reg,bool in_mem,bool set_type)4308 static int zend_jit_store_reg(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, bool in_mem, bool set_type)
4309 {
4310 zend_jit_addr src;
4311 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4312 ir_type type;
4313
4314 if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
4315 type = IR_LONG;
4316 src = zend_jit_deopt_rload(jit, type, reg);
4317 if (jit->ra && jit->ra[var].ref == IR_NULL) {
4318 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4319 } else if (!in_mem) {
4320 jit_set_Z_LVAL(jit, dst, src);
4321 if (set_type &&
4322 (Z_REG(dst) != ZREG_FP ||
4323 !JIT_G(current_frame) ||
4324 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
4325 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4326 }
4327 }
4328 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4329 type = IR_DOUBLE;
4330 src = zend_jit_deopt_rload(jit, type, reg);
4331 if (jit->ra && jit->ra[var].ref == IR_NULL) {
4332 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4333 } else if (!in_mem) {
4334 jit_set_Z_DVAL(jit, dst, src);
4335 if (set_type &&
4336 (Z_REG(dst) != ZREG_FP ||
4337 !JIT_G(current_frame) ||
4338 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
4339 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4340 }
4341 }
4342 } else {
4343 ZEND_UNREACHABLE();
4344 }
4345 return 1;
4346 }
4347
zend_jit_store_spill_slot(zend_jit_ctx * jit,uint32_t info,int var,int8_t reg,int32_t offset,bool set_type)4348 static int zend_jit_store_spill_slot(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, int32_t offset, bool set_type)
4349 {
4350 zend_jit_addr src;
4351 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4352
4353 if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
4354 src = ir_LOAD_L(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset));
4355 if (jit->ra && jit->ra[var].ref == IR_NULL) {
4356 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4357 } else {
4358 jit_set_Z_LVAL(jit, dst, src);
4359 if (set_type &&
4360 (Z_REG(dst) != ZREG_FP ||
4361 !JIT_G(current_frame) ||
4362 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
4363 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4364 }
4365 }
4366 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4367 src = ir_LOAD_D(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset));
4368 if (jit->ra && jit->ra[var].ref == IR_NULL) {
4369 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4370 } else {
4371 jit_set_Z_DVAL(jit, dst, src);
4372 if (set_type &&
4373 (Z_REG(dst) != ZREG_FP ||
4374 !JIT_G(current_frame) ||
4375 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
4376 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4377 }
4378 }
4379 } else {
4380 ZEND_UNREACHABLE();
4381 }
4382 return 1;
4383 }
4384
zend_jit_store_var_type(zend_jit_ctx * jit,int var,uint32_t type)4385 static int zend_jit_store_var_type(zend_jit_ctx *jit, int var, uint32_t type)
4386 {
4387 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4388
4389 jit_set_Z_TYPE_INFO(jit, dst, type);
4390 return 1;
4391 }
4392
zend_jit_zval_try_addref(zend_jit_ctx * jit,zend_jit_addr var_addr)4393 static int zend_jit_zval_try_addref(zend_jit_ctx *jit, zend_jit_addr var_addr)
4394 {
4395 ir_ref if_refcounted, end1;
4396
4397 if_refcounted = jit_if_REFCOUNTED(jit, var_addr);
4398 ir_IF_FALSE(if_refcounted);
4399 end1 = ir_END();
4400 ir_IF_TRUE(if_refcounted);
4401 jit_GC_ADDREF(jit, jit_Z_PTR(jit, var_addr));
4402 ir_MERGE_WITH(end1);
4403 return 1;
4404 }
4405
zend_jit_store_var_if_necessary(zend_jit_ctx * jit,int var,zend_jit_addr src,uint32_t info)4406 static int zend_jit_store_var_if_necessary(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t info)
4407 {
4408 if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
4409 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
4410 return zend_jit_spill_store(jit, src, dst, info, 1);
4411 }
4412 return 1;
4413 }
4414
zend_jit_store_var_if_necessary_ex(zend_jit_ctx * jit,int var,zend_jit_addr src,uint32_t info,zend_jit_addr old,uint32_t old_info)4415 static int zend_jit_store_var_if_necessary_ex(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info)
4416 {
4417 if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
4418 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
4419 bool set_type = 1;
4420
4421 if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) ==
4422 (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) {
4423 if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) {
4424 if (JIT_G(current_frame)) {
4425 uint32_t mem_type = STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var));
4426
4427 if (mem_type != IS_UNKNOWN
4428 && (info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == (1 << mem_type)) {
4429 set_type = 0;
4430 }
4431 } else {
4432 set_type = 0;
4433 }
4434 }
4435 }
4436 return zend_jit_spill_store(jit, src, dst, info, set_type);
4437 }
4438 return 1;
4439 }
4440
zend_jit_load_var(zend_jit_ctx * jit,uint32_t info,int var,int ssa_var)4441 static int zend_jit_load_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var)
4442 {
4443 zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4444 zend_jit_addr dst = ZEND_ADDR_REG(ssa_var);
4445
4446 return zend_jit_load_reg(jit, src, dst, info);
4447 }
4448
zend_jit_invalidate_var_if_necessary(zend_jit_ctx * jit,uint8_t op_type,zend_jit_addr addr,znode_op op)4449 static int zend_jit_invalidate_var_if_necessary(zend_jit_ctx *jit, uint8_t op_type, zend_jit_addr addr, znode_op op)
4450 {
4451 if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) {
4452 /* Invalidate operand type to prevent incorrect destuction by exception_handler_free_op1_op2() */
4453 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var);
4454 jit_set_Z_TYPE_INFO(jit, dst, IS_UNDEF);
4455 }
4456 return 1;
4457 }
4458
zend_jit_update_regs(zend_jit_ctx * jit,uint32_t var,zend_jit_addr src,zend_jit_addr dst,uint32_t info)4459 static int zend_jit_update_regs(zend_jit_ctx *jit, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
4460 {
4461 if (!zend_jit_same_addr(src, dst)) {
4462 if (Z_MODE(src) == IS_REG) {
4463 if (Z_MODE(dst) == IS_REG) {
4464 zend_jit_def_reg(jit, dst, zend_jit_use_reg(jit, src));
4465 if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) {
4466 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
4467
4468 if (!zend_jit_spill_store(jit, dst, var_addr, info,
4469 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
4470 JIT_G(current_frame) == NULL ||
4471 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
4472 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
4473 )) {
4474 return 0;
4475 }
4476 }
4477 } else if (Z_MODE(dst) == IS_MEM_ZVAL) {
4478 if (!Z_LOAD(src) && !Z_STORE(src)) {
4479 if (!zend_jit_spill_store(jit, src, dst, info,
4480 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
4481 JIT_G(current_frame) == NULL ||
4482 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
4483 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
4484 )) {
4485 return 0;
4486 }
4487 }
4488 } else {
4489 ZEND_UNREACHABLE();
4490 }
4491 } else if (Z_MODE(src) == IS_MEM_ZVAL) {
4492 if (Z_MODE(dst) == IS_REG) {
4493 if (!zend_jit_load_reg(jit, src, dst, info)) {
4494 return 0;
4495 }
4496 } else {
4497 ZEND_UNREACHABLE();
4498 }
4499 } else {
4500 ZEND_UNREACHABLE();
4501 }
4502 } else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) {
4503 dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
4504 if (!zend_jit_spill_store(jit, src, dst, info,
4505 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
4506 JIT_G(current_frame) == NULL ||
4507 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
4508 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
4509 )) {
4510 return 0;
4511 }
4512 }
4513 return 1;
4514 }
4515
zend_jit_inc_dec(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_addr,uint32_t op1_def_info,zend_jit_addr op1_def_addr,uint32_t res_use_info,uint32_t res_info,zend_jit_addr res_addr,int may_overflow,int may_throw)4516 static int zend_jit_inc_dec(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
4517 {
4518 ir_ref if_long = IR_UNUSED;
4519 ir_ref op1_lval_ref = IR_UNUSED;
4520 ir_ref ref;
4521 ir_op op;
4522
4523 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) {
4524 if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
4525 ir_IF_TRUE(if_long);
4526 }
4527 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
4528 op1_lval_ref = jit_Z_LVAL(jit, op1_addr);
4529 jit_set_Z_LVAL(jit, res_addr, op1_lval_ref);
4530 if (Z_MODE(res_addr) != IS_REG) {
4531 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
4532 }
4533 }
4534 if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL
4535 && Z_MODE(op1_addr) == IS_REG
4536 && !Z_LOAD(op1_addr)
4537 && !Z_STORE(op1_addr)) {
4538 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_LONG);
4539 }
4540 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
4541 op = may_overflow ? IR_ADD_OV : IR_ADD;
4542 } else {
4543 op = may_overflow ? IR_SUB_OV : IR_SUB;
4544 }
4545 if (!op1_lval_ref) {
4546 op1_lval_ref = jit_Z_LVAL(jit, op1_addr);
4547 }
4548 ref = ir_BINARY_OP_L(op, op1_lval_ref, ir_CONST_LONG(1));
4549 if (op1_def_info & MAY_BE_LONG) {
4550 jit_set_Z_LVAL(jit, op1_def_addr, ref);
4551 }
4552 if (may_overflow &&
4553 (((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)) ||
4554 ((opline->result_type != IS_UNUSED && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD))))) {
4555 int32_t exit_point;
4556 const void *exit_addr;
4557 zend_jit_trace_stack *stack;
4558 uint32_t old_op1_info, old_res_info = 0;
4559
4560 stack = JIT_G(current_frame)->stack;
4561 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
4562 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0);
4563 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
4564 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
4565 } else {
4566 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
4567 }
4568 if (opline->result_type != IS_UNUSED) {
4569 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
4570 if (opline->opcode == ZEND_PRE_INC) {
4571 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
4572 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
4573 } else if (opline->opcode == ZEND_PRE_DEC) {
4574 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
4575 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
4576 } else if (opline->opcode == ZEND_POST_INC) {
4577 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
4578 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_LONG(ZEND_LONG_MAX));
4579 } else if (opline->opcode == ZEND_POST_DEC) {
4580 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
4581 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_LONG(ZEND_LONG_MIN));
4582 }
4583 }
4584
4585 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
4586 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4587 ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
4588
4589 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
4590 opline->result_type != IS_UNUSED) {
4591 jit_set_Z_LVAL(jit, res_addr, ref);
4592 if (Z_MODE(res_addr) != IS_REG) {
4593 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
4594 }
4595 }
4596
4597 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
4598 if (opline->result_type != IS_UNUSED) {
4599 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
4600 }
4601 } else if (may_overflow) {
4602 ir_ref if_overflow;
4603 ir_ref merge_inputs = IR_UNUSED;
4604
4605 if (((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
4606 || (opline->result_type != IS_UNUSED && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))) {
4607 int32_t exit_point;
4608 const void *exit_addr;
4609 zend_jit_trace_stack *stack;
4610 uint32_t old_res_info = 0;
4611
4612 stack = JIT_G(current_frame)->stack;
4613 if (opline->result_type != IS_UNUSED) {
4614 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
4615 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) {
4616 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
4617 }
4618 }
4619 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
4620 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4621 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
4622 opline->result_type != IS_UNUSED) {
4623 if_overflow = ir_IF(ir_OVERFLOW(ref));
4624 ir_IF_FALSE_cold(if_overflow);
4625 jit_set_Z_LVAL(jit, res_addr, ref);
4626 if (Z_MODE(res_addr) != IS_REG) {
4627 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
4628 }
4629 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
4630 ir_IF_TRUE(if_overflow);
4631 } else {
4632 ir_GUARD(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
4633 }
4634 if (opline->result_type != IS_UNUSED) {
4635 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
4636 }
4637 } else {
4638 if_overflow = ir_IF(ir_OVERFLOW(ref));
4639 ir_IF_FALSE(if_overflow);
4640 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
4641 opline->result_type != IS_UNUSED) {
4642 jit_set_Z_LVAL(jit, res_addr, ref);
4643 if (Z_MODE(res_addr) != IS_REG) {
4644 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
4645 }
4646 }
4647 ir_END_list(merge_inputs);
4648
4649 /* overflow => cold path */
4650 ir_IF_TRUE_cold(if_overflow);
4651 }
4652
4653 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
4654 if (Z_MODE(op1_def_addr) == IS_REG) {
4655 jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
4656 } else {
4657 #if SIZEOF_ZEND_LONG == 4
4658 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0));
4659 jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0x41e00000));
4660 #else
4661 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x43e0000000000000));
4662 #endif
4663 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE);
4664 }
4665 } else {
4666 if (Z_MODE(op1_def_addr) == IS_REG) {
4667 jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
4668 } else {
4669 #if SIZEOF_ZEND_LONG == 4
4670 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x00200000));
4671 jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0xc1e00000));
4672 #else
4673 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0xc3e0000000000000));
4674 #endif
4675 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE);
4676 }
4677 }
4678 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
4679 opline->result_type != IS_UNUSED) {
4680 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
4681 if (Z_MODE(res_addr) == IS_REG) {
4682 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
4683 } else {
4684 #if SIZEOF_ZEND_LONG == 4
4685 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
4686 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
4687 #else
4688 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
4689 #endif
4690 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4691 }
4692 } else {
4693 if (Z_MODE(res_addr) == IS_REG) {
4694 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
4695 } else {
4696 #if SIZEOF_ZEND_LONG == 4
4697 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
4698 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
4699 #else
4700 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
4701 #endif
4702 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4703 }
4704 }
4705 }
4706
4707 if (merge_inputs) {
4708 ir_END_list(merge_inputs);
4709 ir_MERGE_list(merge_inputs);
4710 }
4711 } else {
4712 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
4713 opline->result_type != IS_UNUSED) {
4714 jit_set_Z_LVAL(jit, res_addr, ref);
4715 if (Z_MODE(res_addr) != IS_REG) {
4716 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
4717 }
4718 }
4719 }
4720 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
4721 ir_ref merge_inputs = ir_END();
4722
4723 /* !is_long => cold path */
4724 ir_IF_FALSE_cold(if_long);
4725 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
4726 jit_SET_EX_OPLINE(jit, opline);
4727 if (op1_info & MAY_BE_UNDEF) {
4728 ir_ref if_def;
4729
4730 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
4731 ir_IF_FALSE_cold(if_def);
4732
4733 // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
4734 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
4735
4736 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_NULL);
4737 ir_MERGE_WITH_EMPTY_TRUE(if_def);
4738
4739 op1_info |= MAY_BE_NULL;
4740 }
4741
4742 ref = jit_ZVAL_ADDR(jit, op1_addr);
4743
4744 if (op1_info & MAY_BE_REF) {
4745 ir_ref if_ref, if_typed, func, ref2, arg2;
4746
4747 if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
4748 ir_IF_TRUE(if_ref);
4749 ref2 = jit_Z_PTR_ref(jit, ref);
4750
4751 if_typed = jit_if_TYPED_REF(jit, ref2);
4752 ir_IF_TRUE(if_typed);
4753
4754 if (RETURN_VALUE_USED(opline)) {
4755 ZEND_ASSERT(Z_MODE(res_addr) != IS_REG);
4756 arg2 = jit_ZVAL_ADDR(jit, res_addr);
4757 } else {
4758 arg2 = IR_NULL;
4759 }
4760 if (opline->opcode == ZEND_PRE_INC) {
4761 func = ir_CONST_FC_FUNC(zend_jit_pre_inc_typed_ref);
4762 } else if (opline->opcode == ZEND_PRE_DEC) {
4763 func = ir_CONST_FC_FUNC(zend_jit_pre_dec_typed_ref);
4764 } else if (opline->opcode == ZEND_POST_INC) {
4765 func = ir_CONST_FC_FUNC(zend_jit_post_inc_typed_ref);
4766 } else if (opline->opcode == ZEND_POST_DEC) {
4767 func = ir_CONST_FC_FUNC(zend_jit_post_dec_typed_ref);
4768 } else {
4769 ZEND_UNREACHABLE();
4770 }
4771
4772 ir_CALL_2(IR_VOID, func, ref2, arg2);
4773 zend_jit_check_exception(jit);
4774 ir_END_list(merge_inputs);
4775
4776 ir_IF_FALSE(if_typed);
4777 ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
4778 ir_MERGE_WITH_EMPTY_FALSE(if_ref);
4779 ref = ir_PHI_2(IR_ADDR, ref2, ref);
4780 }
4781
4782 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
4783 jit_ZVAL_COPY(jit,
4784 res_addr,
4785 res_use_info,
4786 ZEND_ADDR_REF_ZVAL(ref), op1_info, 1);
4787 }
4788 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
4789 if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) {
4790 ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr);
4791 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_inc), ref, arg2);
4792 } else {
4793 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(increment_function), ref);
4794 }
4795 } else {
4796 if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) {
4797 ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr);
4798 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_dec), ref, arg2);
4799 } else {
4800 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(decrement_function), ref);
4801 }
4802 }
4803 if (may_throw) {
4804 zend_jit_check_exception(jit);
4805 }
4806 } else {
4807 ref = jit_Z_DVAL(jit, op1_addr);
4808 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
4809 jit_set_Z_DVAL(jit, res_addr, ref);
4810 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4811 }
4812 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
4813 op = IR_ADD;
4814 } else {
4815 op = IR_SUB;
4816 }
4817 ref = ir_BINARY_OP_D(op, ref, ir_CONST_DOUBLE(1.0));
4818 jit_set_Z_DVAL(jit, op1_def_addr, ref);
4819 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
4820 opline->result_type != IS_UNUSED) {
4821 jit_set_Z_DVAL(jit, res_addr, ref);
4822 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4823 }
4824 }
4825 ir_END_list(merge_inputs);
4826 ir_MERGE_list(merge_inputs);
4827 }
4828 if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) {
4829 return 0;
4830 }
4831 if (opline->result_type != IS_UNUSED) {
4832 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
4833 return 0;
4834 }
4835 }
4836 return 1;
4837 }
4838
zend_jit_math_long_long(zend_jit_ctx * jit,const zend_op * opline,uint8_t opcode,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint32_t res_info,uint32_t res_use_info,int may_overflow)4839 static int zend_jit_math_long_long(zend_jit_ctx *jit,
4840 const zend_op *opline,
4841 uint8_t opcode,
4842 zend_jit_addr op1_addr,
4843 zend_jit_addr op2_addr,
4844 zend_jit_addr res_addr,
4845 uint32_t res_info,
4846 uint32_t res_use_info,
4847 int may_overflow)
4848 {
4849 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
4850 ir_op op;
4851 ir_ref op1, op2, ref, if_overflow = IR_UNUSED;
4852
4853 if (opcode == ZEND_ADD) {
4854 op = may_overflow ? IR_ADD_OV : IR_ADD;
4855 } else if (opcode == ZEND_SUB) {
4856 op = may_overflow ? IR_SUB_OV : IR_SUB;
4857 } else if (opcode == ZEND_MUL) {
4858 op = may_overflow ? IR_MUL_OV : IR_MUL;
4859 } else {
4860 ZEND_UNREACHABLE();
4861 }
4862 op1 = jit_Z_LVAL(jit, op1_addr);
4863 op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
4864 ref = ir_BINARY_OP_L(op, op1, op2);
4865
4866 if (may_overflow) {
4867 if (res_info & MAY_BE_GUARD) {
4868 if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) {
4869 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
4870 uint32_t old_res_info;
4871 int32_t exit_point;
4872 const void *exit_addr;
4873
4874 if (opline->opcode == ZEND_ADD
4875 && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
4876 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
4877 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
4878 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
4879 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
4880 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
4881 } else if (opline->opcode == ZEND_SUB
4882 && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
4883 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
4884 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
4885 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
4886 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
4887 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
4888 } else {
4889 exit_point = zend_jit_trace_get_exit_point(opline, 0);
4890 }
4891
4892 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4893 if (!exit_addr) {
4894 return 0;
4895 }
4896 ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
4897 may_overflow = 0;
4898 } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4899 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
4900 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4901
4902 if (!exit_addr) {
4903 return 0;
4904 }
4905 ir_GUARD(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
4906 } else {
4907 ZEND_UNREACHABLE();
4908 }
4909 } else {
4910 if_overflow = ir_IF(ir_OVERFLOW(ref));
4911 ir_IF_FALSE(if_overflow);
4912 }
4913 }
4914
4915 if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
4916 jit_set_Z_LVAL(jit, res_addr, ref);
4917
4918 if (Z_MODE(res_addr) != IS_REG) {
4919 if (!zend_jit_same_addr(op1_addr, res_addr)) {
4920 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
4921 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
4922 }
4923 }
4924 }
4925 }
4926
4927 if (may_overflow) {
4928 ir_ref fast_path = IR_UNUSED;
4929
4930 if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
4931 fast_path = ir_END();
4932 ir_IF_TRUE_cold(if_overflow);
4933 }
4934 if (opcode == ZEND_ADD) {
4935 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
4936 if (Z_MODE(res_addr) == IS_REG) {
4937 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
4938 } else {
4939 #if SIZEOF_ZEND_LONG == 4
4940 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
4941 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
4942 #else
4943 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
4944 #endif
4945 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4946 }
4947 if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
4948 ir_MERGE_WITH(fast_path);
4949 }
4950 return 1;
4951 }
4952 op = IR_ADD;
4953 } else if (opcode == ZEND_SUB) {
4954 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
4955 if (Z_MODE(res_addr) == IS_REG) {
4956 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
4957 } else {
4958 #if SIZEOF_ZEND_LONG == 4
4959 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
4960 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
4961 #else
4962 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
4963 #endif
4964 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4965 }
4966 if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
4967 ir_MERGE_WITH(fast_path);
4968 }
4969 return 1;
4970 }
4971 op = IR_SUB;
4972 } else if (opcode == ZEND_MUL) {
4973 op = IR_MUL;
4974 } else {
4975 ZEND_UNREACHABLE();
4976 }
4977 #if 1
4978 /* reload */
4979 op1 = jit_Z_LVAL(jit, op1_addr);
4980 op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
4981 #endif
4982 #if 1
4983 /* disable CSE */
4984 ir_ref old_cse_limit = jit->ctx.fold_cse_limit;
4985 jit->ctx.fold_cse_limit = 0x7fffffff;
4986 #endif
4987 op1 = ir_INT2D(op1);
4988 op2 = ir_INT2D(op2);
4989 #if 1
4990 jit->ctx.fold_cse_limit = old_cse_limit;
4991 #endif
4992 ref = ir_BINARY_OP_D(op, op1, op2);
4993 jit_set_Z_DVAL(jit, res_addr, ref);
4994 if (Z_MODE(res_addr) != IS_REG) {
4995 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4996 }
4997 if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
4998 ir_MERGE_WITH(fast_path);
4999 }
5000 }
5001
5002 return 1;
5003 }
5004
zend_jit_math_long_double(zend_jit_ctx * jit,uint8_t opcode,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint32_t res_use_info)5005 static int zend_jit_math_long_double(zend_jit_ctx *jit,
5006 uint8_t opcode,
5007 zend_jit_addr op1_addr,
5008 zend_jit_addr op2_addr,
5009 zend_jit_addr res_addr,
5010 uint32_t res_use_info)
5011 {
5012 ir_op op;
5013 ir_ref op1, op2, ref;
5014
5015 if (opcode == ZEND_ADD) {
5016 op = IR_ADD;
5017 } else if (opcode == ZEND_SUB) {
5018 op = IR_SUB;
5019 } else if (opcode == ZEND_MUL) {
5020 op = IR_MUL;
5021 } else if (opcode == ZEND_DIV) {
5022 op = IR_DIV;
5023 } else {
5024 ZEND_UNREACHABLE();
5025 }
5026 op1 = jit_Z_LVAL(jit, op1_addr);
5027 op2 = jit_Z_DVAL(jit, op2_addr);
5028 ref = ir_BINARY_OP_D(op, ir_INT2D(op1), op2);
5029 jit_set_Z_DVAL(jit, res_addr, ref);
5030
5031 if (Z_MODE(res_addr) != IS_REG) {
5032 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
5033 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
5034 }
5035 }
5036 return 1;
5037 }
5038
zend_jit_math_double_long(zend_jit_ctx * jit,uint8_t opcode,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint32_t res_use_info)5039 static int zend_jit_math_double_long(zend_jit_ctx *jit,
5040 uint8_t opcode,
5041 zend_jit_addr op1_addr,
5042 zend_jit_addr op2_addr,
5043 zend_jit_addr res_addr,
5044 uint32_t res_use_info)
5045 {
5046 ir_op op;
5047 ir_ref op1, op2, ref;
5048
5049 if (opcode == ZEND_ADD) {
5050 op = IR_ADD;
5051 } else if (opcode == ZEND_SUB) {
5052 op = IR_SUB;
5053 } else if (opcode == ZEND_MUL) {
5054 op = IR_MUL;
5055 } else if (opcode == ZEND_DIV) {
5056 op = IR_DIV;
5057 } else {
5058 ZEND_UNREACHABLE();
5059 }
5060 op1 = jit_Z_DVAL(jit, op1_addr);
5061 op2 = jit_Z_LVAL(jit, op2_addr);
5062 ref = ir_BINARY_OP_D(op, op1, ir_INT2D(op2));
5063 jit_set_Z_DVAL(jit, res_addr, ref);
5064
5065 if (Z_MODE(res_addr) != IS_REG) {
5066 if (!zend_jit_same_addr(op1_addr, res_addr)) {
5067 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
5068 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
5069 }
5070 }
5071 }
5072 return 1;
5073 }
5074
zend_jit_math_double_double(zend_jit_ctx * jit,uint8_t opcode,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint32_t res_use_info)5075 static int zend_jit_math_double_double(zend_jit_ctx *jit,
5076 uint8_t opcode,
5077 zend_jit_addr op1_addr,
5078 zend_jit_addr op2_addr,
5079 zend_jit_addr res_addr,
5080 uint32_t res_use_info)
5081 {
5082 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
5083 ir_op op;
5084 ir_ref op1, op2, ref;
5085
5086 if (opcode == ZEND_ADD) {
5087 op = IR_ADD;
5088 } else if (opcode == ZEND_SUB) {
5089 op = IR_SUB;
5090 } else if (opcode == ZEND_MUL) {
5091 op = IR_MUL;
5092 } else if (opcode == ZEND_DIV) {
5093 op = IR_DIV;
5094 } else {
5095 ZEND_UNREACHABLE();
5096 }
5097 op1 = jit_Z_DVAL(jit, op1_addr);
5098 op2 = (same_ops) ? op1 : jit_Z_DVAL(jit, op2_addr);
5099 ref = ir_BINARY_OP_D(op, op1, op2);
5100 jit_set_Z_DVAL(jit, res_addr, ref);
5101
5102 if (Z_MODE(res_addr) != IS_REG) {
5103 if (!zend_jit_same_addr(op1_addr, res_addr)) {
5104 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
5105 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
5106 }
5107 }
5108 }
5109 return 1;
5110 }
5111
zend_jit_math_helper(zend_jit_ctx * jit,const zend_op * opline,uint8_t opcode,uint8_t op1_type,znode_op op1,zend_jit_addr op1_addr,uint32_t op1_info,uint8_t op2_type,znode_op op2,zend_jit_addr op2_addr,uint32_t op2_info,uint32_t res_var,zend_jit_addr res_addr,uint32_t res_info,uint32_t res_use_info,int may_overflow,int may_throw)5112 static int zend_jit_math_helper(zend_jit_ctx *jit,
5113 const zend_op *opline,
5114 uint8_t opcode,
5115 uint8_t op1_type,
5116 znode_op op1,
5117 zend_jit_addr op1_addr,
5118 uint32_t op1_info,
5119 uint8_t op2_type,
5120 znode_op op2,
5121 zend_jit_addr op2_addr,
5122 uint32_t op2_info,
5123 uint32_t res_var,
5124 zend_jit_addr res_addr,
5125 uint32_t res_info,
5126 uint32_t res_use_info,
5127 int may_overflow,
5128 int may_throw)
5129 {
5130 ir_ref if_op1_long = IR_UNUSED;
5131 ir_ref if_op1_double = IR_UNUSED;
5132 ir_ref if_op2_double = IR_UNUSED;
5133 ir_ref if_op1_long_op2_long = IR_UNUSED;
5134 ir_ref if_op1_long_op2_double = IR_UNUSED;
5135 ir_ref if_op1_double_op2_double = IR_UNUSED;
5136 ir_ref if_op1_double_op2_long = IR_UNUSED;
5137 ir_ref slow_inputs = IR_UNUSED;
5138 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
5139 ir_refs *end_inputs;
5140 ir_refs *res_inputs;
5141
5142 ir_refs_init(end_inputs, 6);
5143 ir_refs_init(res_inputs, 6);
5144
5145 if (Z_MODE(op1_addr) == IS_REG) {
5146 if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) {
5147 /* Force load */
5148 zend_jit_use_reg(jit, op1_addr);
5149 }
5150 } else if (Z_MODE(op2_addr) == IS_REG) {
5151 if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) {
5152 /* Force load */
5153 zend_jit_use_reg(jit, op2_addr);
5154 }
5155 }
5156
5157 if (Z_MODE(res_addr) == IS_REG) {
5158 jit->delay_var = Z_SSA_VAR(res_addr);
5159 jit->delay_refs = res_inputs;
5160 }
5161
5162 if ((res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG) && (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
5163 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
5164 if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
5165 ir_IF_TRUE(if_op1_long);
5166 }
5167 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
5168 if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
5169 ir_IF_TRUE(if_op1_long_op2_long);
5170 }
5171 if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
5172 return 0;
5173 }
5174 ir_refs_add(end_inputs, ir_END());
5175 if (if_op1_long) {
5176 ir_IF_FALSE_cold(if_op1_long);
5177 ir_END_list(slow_inputs);
5178 }
5179 if (if_op1_long_op2_long) {
5180 ir_IF_FALSE_cold(if_op1_long_op2_long);
5181 ir_END_list(slow_inputs);
5182 }
5183 } else if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
5184 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
5185 if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
5186 ir_IF_TRUE(if_op1_long);
5187 }
5188 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
5189 if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
5190 ir_IF_FALSE_cold(if_op1_long_op2_long);
5191 if (op2_info & MAY_BE_DOUBLE) {
5192 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
5193 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
5194 ir_IF_FALSE_cold(if_op1_long_op2_double);
5195 ir_END_list(slow_inputs);
5196 ir_IF_TRUE(if_op1_long_op2_double);
5197 }
5198 if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5199 return 0;
5200 }
5201 ir_refs_add(end_inputs, ir_END());
5202 } else {
5203 ir_END_list(slow_inputs);
5204 }
5205 ir_IF_TRUE(if_op1_long_op2_long);
5206 }
5207 if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
5208 return 0;
5209 }
5210 ir_refs_add(end_inputs, ir_END());
5211
5212 if (if_op1_long) {
5213 ir_IF_FALSE_cold(if_op1_long);
5214 }
5215
5216 if (op1_info & MAY_BE_DOUBLE) {
5217 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
5218 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
5219 ir_IF_FALSE_cold(if_op1_double);
5220 ir_END_list(slow_inputs);
5221 ir_IF_TRUE(if_op1_double);
5222 }
5223 if (op2_info & MAY_BE_DOUBLE) {
5224 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
5225 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
5226 ir_IF_TRUE(if_op1_double_op2_double);
5227 }
5228 if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5229 return 0;
5230 }
5231 ir_refs_add(end_inputs, ir_END());
5232 if (if_op1_double_op2_double) {
5233 ir_IF_FALSE_cold(if_op1_double_op2_double);
5234 }
5235 }
5236 if (!same_ops) {
5237 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
5238 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
5239 ir_IF_FALSE_cold(if_op1_double_op2_long);
5240 ir_END_list(slow_inputs);
5241 ir_IF_TRUE(if_op1_double_op2_long);
5242 }
5243 if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5244 return 0;
5245 }
5246 ir_refs_add(end_inputs, ir_END());
5247 } else if (if_op1_double_op2_double) {
5248 ir_END_list(slow_inputs);
5249 }
5250 } else if (if_op1_long) {
5251 ir_END_list(slow_inputs);
5252 }
5253 } else if ((op1_info & MAY_BE_DOUBLE) &&
5254 !(op1_info & MAY_BE_LONG) &&
5255 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
5256 (res_info & MAY_BE_DOUBLE)) {
5257 if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
5258 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
5259 ir_IF_FALSE_cold(if_op1_double);
5260 ir_END_list(slow_inputs);
5261 ir_IF_TRUE(if_op1_double);
5262 }
5263 if (op2_info & MAY_BE_DOUBLE) {
5264 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
5265 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
5266 ir_IF_TRUE(if_op1_double_op2_double);
5267 }
5268 if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5269 return 0;
5270 }
5271 ir_refs_add(end_inputs, ir_END());
5272 if (if_op1_double_op2_double) {
5273 ir_IF_FALSE_cold(if_op1_double_op2_double);
5274 }
5275 }
5276 if (!same_ops && (op2_info & MAY_BE_LONG)) {
5277 if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
5278 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
5279 ir_IF_FALSE_cold(if_op1_double_op2_long);
5280 ir_END_list(slow_inputs);
5281 ir_IF_TRUE(if_op1_double_op2_long);
5282 }
5283 if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5284 return 0;
5285 }
5286 ir_refs_add(end_inputs, ir_END());
5287 } else if (if_op1_double_op2_double) {
5288 ir_END_list(slow_inputs);
5289 }
5290 } else if ((op2_info & MAY_BE_DOUBLE) &&
5291 !(op2_info & MAY_BE_LONG) &&
5292 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
5293 (res_info & MAY_BE_DOUBLE)) {
5294 if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
5295 if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
5296 ir_IF_FALSE_cold(if_op2_double);
5297 ir_END_list(slow_inputs);
5298 ir_IF_TRUE(if_op2_double);
5299 }
5300 if (op1_info & MAY_BE_DOUBLE) {
5301 if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
5302 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
5303 ir_IF_TRUE(if_op1_double_op2_double);
5304 }
5305 if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5306 return 0;
5307 }
5308 ir_refs_add(end_inputs, ir_END());
5309 if (if_op1_double_op2_double) {
5310 ir_IF_FALSE_cold(if_op1_double_op2_double);
5311 }
5312 }
5313 if (!same_ops && (op1_info & MAY_BE_LONG)) {
5314 if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
5315 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
5316 ir_IF_FALSE_cold(if_op1_long_op2_double);
5317 ir_END_list(slow_inputs);
5318 ir_IF_TRUE(if_op1_long_op2_double);
5319 }
5320 if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5321 return 0;
5322 }
5323 ir_refs_add(end_inputs, ir_END());
5324 } else if (if_op1_double_op2_double) {
5325 ir_END_list(slow_inputs);
5326 }
5327 }
5328
5329 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
5330 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
5331 ir_ref func, arg1, arg2, arg3;
5332
5333 if (slow_inputs) {
5334 ir_MERGE_list(slow_inputs);
5335 }
5336
5337 if (Z_MODE(op1_addr) == IS_REG) {
5338 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
5339 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
5340 return 0;
5341 }
5342 op1_addr = real_addr;
5343 }
5344 if (Z_MODE(op2_addr) == IS_REG) {
5345 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
5346 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
5347 return 0;
5348 }
5349 op2_addr = real_addr;
5350 }
5351 if (Z_MODE(res_addr) == IS_REG) {
5352 arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var));
5353 } else {
5354 arg1 = jit_ZVAL_ADDR(jit, res_addr);
5355 }
5356 arg2 = jit_ZVAL_ADDR(jit, op1_addr);
5357 arg3 = jit_ZVAL_ADDR(jit, op2_addr);
5358 jit_SET_EX_OPLINE(jit, opline);
5359 if (opcode == ZEND_ADD) {
5360 func = ir_CONST_FC_FUNC(add_function);
5361 } else if (opcode == ZEND_SUB) {
5362 func = ir_CONST_FC_FUNC(sub_function);
5363 } else if (opcode == ZEND_MUL) {
5364 func = ir_CONST_FC_FUNC(mul_function);
5365 } else if (opcode == ZEND_DIV) {
5366 func = ir_CONST_FC_FUNC(div_function);
5367 } else {
5368 ZEND_UNREACHABLE();
5369 }
5370 ir_CALL_3(IR_VOID, func, arg1, arg2, arg3);
5371
5372 jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
5373 jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
5374
5375 if (may_throw) {
5376 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
5377 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
5378 jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
5379 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
5380 zend_jit_check_exception_undef_result(jit, opline);
5381 } else {
5382 zend_jit_check_exception(jit);
5383 }
5384 }
5385 if (Z_MODE(res_addr) == IS_REG) {
5386 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
5387 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
5388 return 0;
5389 }
5390 }
5391 ir_refs_add(end_inputs, ir_END());
5392 }
5393
5394 if (end_inputs->count) {
5395 ir_MERGE_N(end_inputs->count, end_inputs->refs);
5396 }
5397
5398 if (Z_MODE(res_addr) == IS_REG) {
5399 ZEND_ASSERT(jit->delay_refs == res_inputs);
5400 ZEND_ASSERT(end_inputs->count == res_inputs->count);
5401 jit->delay_var = -1;
5402 jit->delay_refs = NULL;
5403 if (res_inputs->count == 1) {
5404 zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]);
5405 } else {
5406 ir_ref phi = ir_PHI_N((res_info & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE, res_inputs->count, res_inputs->refs);
5407 zend_jit_def_reg(jit, res_addr, phi);
5408 }
5409 }
5410
5411 return 1;
5412 }
5413
zend_jit_math(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_addr,uint32_t op2_info,zend_jit_addr op2_addr,uint32_t res_use_info,uint32_t res_info,zend_jit_addr res_addr,int may_overflow,int may_throw)5414 static int zend_jit_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
5415 {
5416 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
5417
5418 if (!zend_jit_math_helper(jit, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) {
5419 return 0;
5420 }
5421 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
5422 return 0;
5423 }
5424 return 1;
5425 }
5426
zend_jit_add_arrays(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_addr,uint32_t op2_info,zend_jit_addr op2_addr,zend_jit_addr res_addr)5427 static int zend_jit_add_arrays(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr)
5428 {
5429 ir_ref ref;
5430 ir_ref arg1 = jit_Z_PTR(jit, op1_addr);
5431 ir_ref arg2 = jit_Z_PTR(jit, op2_addr);
5432
5433 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_add_arrays_helper), arg1, arg2);
5434 jit_set_Z_PTR(jit, res_addr, ref);
5435 jit_set_Z_TYPE_INFO(jit, res_addr, IS_ARRAY_EX);
5436 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
5437 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
5438 return 1;
5439 }
5440
zend_jit_long_math_helper(zend_jit_ctx * jit,const zend_op * opline,uint8_t opcode,uint8_t op1_type,znode_op op1,zend_jit_addr op1_addr,uint32_t op1_info,zend_ssa_range * op1_range,uint8_t op2_type,znode_op op2,zend_jit_addr op2_addr,uint32_t op2_info,zend_ssa_range * op2_range,uint32_t res_var,zend_jit_addr res_addr,uint32_t res_info,uint32_t res_use_info,int may_throw)5441 static int zend_jit_long_math_helper(zend_jit_ctx *jit,
5442 const zend_op *opline,
5443 uint8_t opcode,
5444 uint8_t op1_type,
5445 znode_op op1,
5446 zend_jit_addr op1_addr,
5447 uint32_t op1_info,
5448 zend_ssa_range *op1_range,
5449 uint8_t op2_type,
5450 znode_op op2,
5451 zend_jit_addr op2_addr,
5452 uint32_t op2_info,
5453 zend_ssa_range *op2_range,
5454 uint32_t res_var,
5455 zend_jit_addr res_addr,
5456 uint32_t res_info,
5457 uint32_t res_use_info,
5458 int may_throw)
5459 {
5460 ir_ref ref = IR_UNUSED;
5461 ir_ref if_long1 = IR_UNUSED;
5462 ir_ref if_long2 = IR_UNUSED;
5463 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
5464 ir_refs *res_inputs;
5465
5466 ir_refs_init(res_inputs, 2);
5467
5468 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
5469 if_long1 = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
5470 ir_IF_TRUE(if_long1);
5471 }
5472 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
5473 if_long2 = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
5474 ir_IF_TRUE(if_long2);
5475 }
5476
5477 if (opcode == ZEND_SL) {
5478 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
5479 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
5480
5481 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
5482 if (EXPECTED(op2_lval > 0)) {
5483 ref = ir_CONST_LONG(0);
5484 } else {
5485 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
5486 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
5487 jit_SET_EX_OPLINE(jit, opline);
5488 ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift));
5489 if (Z_MODE(res_addr) == IS_REG) {
5490 ref = ir_CONST_LONG(0); // dead code
5491 }
5492 }
5493 } else {
5494 ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
5495 }
5496 } else {
5497 ref = jit_Z_LVAL(jit, op2_addr);
5498 if (!op2_range ||
5499 op2_range->min < 0 ||
5500 op2_range->max >= SIZEOF_ZEND_LONG * 8) {
5501
5502 ir_ref if_wrong, cold_path, ref2, if_ok;
5503 ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr);
5504
5505 if_wrong = ir_IF(ir_UGT(ref, ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1)));
5506 ir_IF_TRUE_cold(if_wrong);
5507 if_ok = ir_IF(ir_GE(ref, ir_CONST_LONG(0)));
5508 ir_IF_FALSE(if_ok);
5509 jit_SET_EX_OPLINE(jit, opline);
5510 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
5511 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
5512 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift));
5513 ir_IF_TRUE(if_ok);
5514 ref2 = ir_CONST_LONG(0);
5515 cold_path = ir_END();
5516 ir_IF_FALSE(if_wrong);
5517 ref = ir_SHL_L(op1_ref, ref);
5518 ir_MERGE_WITH(cold_path);
5519 ref = ir_PHI_2(IR_LONG, ref, ref2);
5520 } else {
5521 ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ref);
5522 }
5523 }
5524 } else if (opcode == ZEND_SR) {
5525 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
5526 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
5527
5528 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
5529 if (EXPECTED(op2_lval > 0)) {
5530 ref = ir_SAR_L(
5531 jit_Z_LVAL(jit, op1_addr),
5532 ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1));
5533 } else {
5534 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
5535 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
5536 jit_SET_EX_OPLINE(jit, opline);
5537 ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift));
5538 if (Z_MODE(res_addr) == IS_REG) {
5539 ref = ir_CONST_LONG(0); // dead code
5540 }
5541 }
5542 } else {
5543 ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
5544 }
5545 } else {
5546 ref = jit_Z_LVAL(jit, op2_addr);
5547 if (!op2_range ||
5548 op2_range->min < 0 ||
5549 op2_range->max >= SIZEOF_ZEND_LONG * 8) {
5550
5551 ir_ref if_wrong, cold_path, ref2, if_ok;
5552
5553 if_wrong = ir_IF(ir_UGT(ref, ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1)));
5554 ir_IF_TRUE_cold(if_wrong);
5555 if_ok = ir_IF(ir_GE(ref, ir_CONST_LONG(0)));
5556 ir_IF_FALSE(if_ok);
5557 jit_SET_EX_OPLINE(jit, opline);
5558 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
5559 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
5560 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift));
5561 ir_IF_TRUE(if_ok);
5562 ref2 = ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1);
5563 cold_path = ir_END();
5564 ir_IF_FALSE(if_wrong);
5565 ir_MERGE_WITH(cold_path);
5566 ref = ir_PHI_2(IR_LONG, ref, ref2);
5567 }
5568 ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ref);
5569 }
5570 } else if (opcode == ZEND_MOD) {
5571 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
5572 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
5573
5574 if (op2_lval == 0) {
5575 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
5576 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
5577 jit_SET_EX_OPLINE(jit, opline);
5578 ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_mod_by_zero));
5579 if (Z_MODE(res_addr) == IS_REG) {
5580 ref = ir_CONST_LONG(0); // dead code
5581 }
5582 } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) {
5583 ref = ir_AND_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval - 1));
5584 } else {
5585 ref = ir_MOD_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
5586 }
5587 } else {
5588 ir_ref zero_path = 0;
5589 ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr);
5590
5591 ref = jit_Z_LVAL(jit, op2_addr);
5592 if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) {
5593 ir_ref if_ok = ir_IF(ref);
5594 ir_IF_FALSE(if_ok);
5595 jit_SET_EX_OPLINE(jit, opline);
5596 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
5597 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
5598 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_mod_by_zero));
5599 ir_IF_TRUE(if_ok);
5600 }
5601
5602 /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */
5603 if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) {
5604 ir_ref if_minus_one = ir_IF(ir_EQ(ref, ir_CONST_LONG(-1)));
5605 ir_IF_TRUE_cold(if_minus_one);
5606 zero_path = ir_END();
5607 ir_IF_FALSE(if_minus_one);
5608 }
5609 ref = ir_MOD_L(op1_ref, ref);
5610
5611 if (zero_path) {
5612 ir_MERGE_WITH(zero_path);
5613 ref = ir_PHI_2(IR_LONG, ref, ir_CONST_LONG(0));
5614 }
5615 }
5616 } else {
5617 ir_op op;
5618 ir_ref op1, op2;
5619
5620 if (opcode == ZEND_BW_OR) {
5621 op = IR_OR;
5622 } else if (opcode == ZEND_BW_AND) {
5623 op = IR_AND;
5624 } else if (opcode == ZEND_BW_XOR) {
5625 op = IR_XOR;
5626 } else {
5627 ZEND_UNREACHABLE();
5628 }
5629 op1 = jit_Z_LVAL(jit, op1_addr);
5630 op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
5631 ref = ir_BINARY_OP_L(op, op1, op2);
5632 }
5633
5634 if (ref) {
5635 if (Z_MODE(res_addr) == IS_REG
5636 && ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))
5637 || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
5638 jit->delay_var = Z_SSA_VAR(res_addr);
5639 jit->delay_refs = res_inputs;
5640 }
5641 jit_set_Z_LVAL(jit, res_addr, ref);
5642 if (Z_MODE(res_addr) != IS_REG) {
5643 if (!zend_jit_same_addr(op1_addr, res_addr)) {
5644 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
5645 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
5646 }
5647 }
5648 }
5649 }
5650
5651 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) ||
5652 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
5653 ir_ref fast_path = ir_END();
5654 ir_ref func, arg1, arg2, arg3;
5655
5656 if (if_long2 && if_long1) {
5657 ir_ref ref;
5658 ir_IF_FALSE_cold(if_long2);
5659 ref = ir_END();
5660 ir_IF_FALSE_cold(if_long1);
5661 ir_MERGE_2(ref, ir_END());
5662 } else if (if_long1) {
5663 ir_IF_FALSE_cold(if_long1);
5664 } else if (if_long2) {
5665 ir_IF_FALSE_cold(if_long2);
5666 }
5667
5668 if (op1_info & MAY_BE_UNDEF) {
5669 ir_ref if_def, ref, ref2;
5670
5671 ref = jit_ZVAL_ADDR(jit, op1_addr);
5672 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
5673 ir_IF_FALSE_cold(if_def);
5674
5675 // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
5676 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
5677
5678 ref2 = jit_EG(uninitialized_zval);
5679 ir_MERGE_WITH_EMPTY_TRUE(if_def);
5680 ref = ir_PHI_2(IR_ADDR, ref2, ref);
5681 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
5682 }
5683
5684 if (op2_info & MAY_BE_UNDEF) {
5685 ir_ref if_def, ref, ref2;
5686
5687 ref = jit_ZVAL_ADDR(jit, op2_addr);
5688 if_def = jit_if_not_Z_TYPE(jit, op2_addr, IS_UNDEF);
5689 ir_IF_FALSE_cold(if_def);
5690
5691 // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op2.var)));
5692 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op2.var));
5693
5694 ref2 = jit_EG(uninitialized_zval);
5695 ir_MERGE_WITH_EMPTY_TRUE(if_def);
5696 ref = ir_PHI_2(IR_ADDR, ref2, ref);
5697 op2_addr = ZEND_ADDR_REF_ZVAL(ref);
5698 }
5699
5700 if (Z_MODE(op1_addr) == IS_REG) {
5701 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
5702 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
5703 return 0;
5704 }
5705 op1_addr = real_addr;
5706 }
5707 if (Z_MODE(op2_addr) == IS_REG) {
5708 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
5709 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
5710 return 0;
5711 }
5712 op2_addr = real_addr;
5713 }
5714 if (Z_MODE(res_addr) == IS_REG) {
5715 arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var));
5716 } else {
5717 arg1 = jit_ZVAL_ADDR(jit, res_addr);
5718 }
5719 arg2 = jit_ZVAL_ADDR(jit, op1_addr);
5720 arg3 = jit_ZVAL_ADDR(jit, op2_addr);
5721 jit_SET_EX_OPLINE(jit, opline);
5722 if (opcode == ZEND_BW_OR) {
5723 func = ir_CONST_FC_FUNC(bitwise_or_function);
5724 } else if (opcode == ZEND_BW_AND) {
5725 func = ir_CONST_FC_FUNC(bitwise_and_function);
5726 } else if (opcode == ZEND_BW_XOR) {
5727 func = ir_CONST_FC_FUNC(bitwise_xor_function);
5728 } else if (opcode == ZEND_SL) {
5729 func = ir_CONST_FC_FUNC(shift_left_function);
5730 } else if (opcode == ZEND_SR) {
5731 func = ir_CONST_FC_FUNC(shift_right_function);
5732 } else if (opcode == ZEND_MOD) {
5733 func = ir_CONST_FC_FUNC(mod_function);
5734 } else {
5735 ZEND_UNREACHABLE();
5736 }
5737 ir_CALL_3(IR_VOID, func, arg1, arg2, arg3);
5738
5739 if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) {
5740 /* compound assignment may decrement "op2" refcount */
5741 op2_info |= MAY_BE_RC1;
5742 }
5743
5744 jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
5745 jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
5746
5747 if (may_throw) {
5748 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
5749 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
5750 jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
5751 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
5752 zend_jit_check_exception_undef_result(jit, opline);
5753 } else {
5754 zend_jit_check_exception(jit);
5755 }
5756 }
5757
5758 if (Z_MODE(res_addr) == IS_REG) {
5759 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
5760 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
5761 return 0;
5762 }
5763 }
5764
5765 ir_MERGE_2(fast_path, ir_END());
5766
5767 if (Z_MODE(res_addr) == IS_REG) {
5768 ZEND_ASSERT(jit->delay_refs == res_inputs);
5769 ZEND_ASSERT(res_inputs->count == 2);
5770 jit->delay_var = -1;
5771 jit->delay_refs = NULL;
5772 if (res_inputs->count == 1) {
5773 zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]);
5774 } else {
5775 ir_ref phi = ir_PHI_N(IR_LONG, res_inputs->count, res_inputs->refs);
5776 zend_jit_def_reg(jit, res_addr, phi);
5777 }
5778 }
5779 }
5780
5781 return 1;
5782 }
5783
zend_jit_long_math(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_ssa_range * op1_range,zend_jit_addr op1_addr,uint32_t op2_info,zend_ssa_range * op2_range,zend_jit_addr op2_addr,uint32_t res_use_info,uint32_t res_info,zend_jit_addr res_addr,int may_throw)5784 static int zend_jit_long_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw)
5785 {
5786 ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG));
5787
5788 if (!zend_jit_long_math_helper(jit, opline, opline->opcode,
5789 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
5790 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
5791 opline->result.var, res_addr, res_info, res_use_info, may_throw)) {
5792 return 0;
5793 }
5794 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
5795 return 0;
5796 }
5797 return 1;
5798 }
5799
zend_jit_concat_helper(zend_jit_ctx * jit,const zend_op * opline,uint8_t op1_type,znode_op op1,zend_jit_addr op1_addr,uint32_t op1_info,uint8_t op2_type,znode_op op2,zend_jit_addr op2_addr,uint32_t op2_info,zend_jit_addr res_addr,int may_throw)5800 static int zend_jit_concat_helper(zend_jit_ctx *jit,
5801 const zend_op *opline,
5802 uint8_t op1_type,
5803 znode_op op1,
5804 zend_jit_addr op1_addr,
5805 uint32_t op1_info,
5806 uint8_t op2_type,
5807 znode_op op2,
5808 zend_jit_addr op2_addr,
5809 uint32_t op2_info,
5810 zend_jit_addr res_addr,
5811 int may_throw)
5812 {
5813 ir_ref if_op1_string = IR_UNUSED;
5814 ir_ref if_op2_string = IR_UNUSED;
5815 ir_ref fast_path = IR_UNUSED;
5816
5817 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
5818 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
5819 if_op1_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
5820 ir_IF_TRUE(if_op1_string);
5821 }
5822 if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
5823 if_op2_string = jit_if_Z_TYPE(jit, op2_addr, IS_STRING);
5824 ir_IF_TRUE(if_op2_string);
5825 }
5826 if (zend_jit_same_addr(op1_addr, res_addr)) {
5827 ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
5828 ir_ref arg2 = jit_ZVAL_ADDR(jit, op2_addr);
5829
5830 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_assign_concat_helper), arg1, arg2);
5831 /* concatenation with itself may reduce refcount */
5832 op2_info |= MAY_BE_RC1;
5833 } else {
5834 ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
5835 ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr);
5836 ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr);
5837
5838 if (op1_type == IS_CV || op1_type == IS_CONST) {
5839 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_concat_helper), arg1, arg2, arg3);
5840 } else {
5841 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_concat_tmp_helper), arg1, arg2, arg3);
5842 }
5843 }
5844 /* concatenation with empty string may increase refcount */
5845 op2_info |= MAY_BE_RCN;
5846 jit_FREE_OP(jit, op2_type, op2, op2_info, opline);
5847 if (if_op1_string || if_op2_string) {
5848 fast_path = ir_END();
5849 }
5850 }
5851 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) ||
5852 (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) {
5853 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
5854 if (if_op1_string && if_op2_string) {
5855 ir_IF_FALSE(if_op1_string);
5856 ir_MERGE_WITH_EMPTY_FALSE(if_op2_string);
5857 } else if (if_op1_string) {
5858 ir_IF_FALSE_cold(if_op1_string);
5859 } else if (if_op2_string) {
5860 ir_IF_FALSE_cold(if_op2_string);
5861 }
5862 }
5863 ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
5864 ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr);
5865 ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr);
5866
5867 jit_SET_EX_OPLINE(jit, opline);
5868 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(concat_function), arg1, arg2, arg3);
5869 /* concatenation with empty string may increase refcount */
5870 op1_info |= MAY_BE_RCN;
5871 op2_info |= MAY_BE_RCN;
5872 jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
5873 jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
5874 if (may_throw) {
5875 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
5876 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
5877 jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
5878 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
5879 zend_jit_check_exception_undef_result(jit, opline);
5880 } else {
5881 zend_jit_check_exception(jit);
5882 }
5883 }
5884 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
5885 ir_MERGE_WITH(fast_path);
5886 }
5887 }
5888 return 1;
5889 }
5890
zend_jit_concat(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,uint32_t op2_info,zend_jit_addr res_addr,int may_throw)5891 static int zend_jit_concat(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw)
5892 {
5893 zend_jit_addr op1_addr, op2_addr;
5894
5895 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
5896 ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING));
5897
5898 op1_addr = OP1_ADDR();
5899 op2_addr = OP2_ADDR();
5900
5901 return zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw);
5902 }
5903
zend_jit_assign_op(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_addr,zend_ssa_range * op1_range,uint32_t op1_def_info,zend_jit_addr op1_def_addr,uint32_t op1_mem_info,uint32_t op2_info,zend_jit_addr op2_addr,zend_ssa_range * op2_range,int may_overflow,int may_throw)5904 static int zend_jit_assign_op(zend_jit_ctx *jit,
5905 const zend_op *opline,
5906 uint32_t op1_info,
5907 zend_jit_addr op1_addr,
5908 zend_ssa_range *op1_range,
5909 uint32_t op1_def_info,
5910 zend_jit_addr op1_def_addr,
5911 uint32_t op1_mem_info,
5912 uint32_t op2_info,
5913 zend_jit_addr op2_addr,
5914 zend_ssa_range *op2_range,
5915 int may_overflow,
5916 int may_throw)
5917 {
5918 int result = 1;
5919 ir_ref slow_path = IR_UNUSED;
5920
5921 ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED);
5922 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
5923
5924 if (op1_info & MAY_BE_REF) {
5925 ir_ref ref, ref2, arg2, op1_noref_path;
5926 ir_ref if_op1_ref = IR_UNUSED;
5927 ir_ref if_op1_typed = IR_UNUSED;
5928 binary_op_type binary_op = get_binary_op(opline->extended_value);
5929
5930 ref = jit_ZVAL_ADDR(jit, op1_addr);
5931 if_op1_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
5932 ir_IF_FALSE(if_op1_ref);
5933 op1_noref_path = ir_END();
5934 ir_IF_TRUE(if_op1_ref);
5935 ref2 = jit_Z_PTR_ref(jit, ref);
5936
5937 if_op1_typed = jit_if_TYPED_REF(jit, ref2);
5938 ir_IF_TRUE_cold(if_op1_typed);
5939
5940 if (Z_MODE(op2_addr) == IS_REG) {
5941 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
5942 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
5943 return 0;
5944 }
5945 arg2 = jit_ZVAL_ADDR(jit, real_addr);
5946 } else {
5947 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
5948 }
5949 jit_SET_EX_OPLINE(jit, opline);
5950 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
5951 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
5952 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref_tmp),
5953 ref2, arg2, ir_CONST_FC_FUNC(binary_op));
5954 } else {
5955 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
5956 ref2, arg2, ir_CONST_FC_FUNC(binary_op));
5957 }
5958 zend_jit_check_exception(jit);
5959 slow_path = ir_END();
5960
5961 ir_IF_FALSE(if_op1_typed);
5962 ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
5963
5964 ir_MERGE_WITH(op1_noref_path);
5965 ref = ir_PHI_2(IR_ADDR, ref2, ref);
5966 ZEND_ASSERT(op1_addr == op1_def_addr);
5967 op1_def_addr = op1_addr = ZEND_ADDR_REF_ZVAL(ref);
5968 }
5969
5970 switch (opline->extended_value) {
5971 case ZEND_ADD:
5972 case ZEND_SUB:
5973 case ZEND_MUL:
5974 case ZEND_DIV:
5975 result = zend_jit_math_helper(jit, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_def_addr, op1_def_info, op1_mem_info, may_overflow, may_throw);
5976 break;
5977 case ZEND_BW_OR:
5978 case ZEND_BW_AND:
5979 case ZEND_BW_XOR:
5980 case ZEND_SL:
5981 case ZEND_SR:
5982 case ZEND_MOD:
5983 result = zend_jit_long_math_helper(jit, opline, opline->extended_value,
5984 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
5985 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
5986 opline->op1.var, op1_def_addr, op1_def_info, op1_mem_info, may_throw);
5987 break;
5988 case ZEND_CONCAT:
5989 result = zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_def_addr, may_throw);
5990 break;
5991 default:
5992 ZEND_UNREACHABLE();
5993 }
5994
5995 if (op1_info & MAY_BE_REF) {
5996 ir_MERGE_WITH(slow_path);
5997 }
5998
5999 return result;
6000 }
6001
jit_ZVAL_DEREF_ref(zend_jit_ctx * jit,ir_ref ref)6002 static ir_ref jit_ZVAL_DEREF_ref(zend_jit_ctx *jit, ir_ref ref)
6003 {
6004 ir_ref if_ref, ref2;
6005
6006 if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_REFERENCE)));
6007 ir_IF_TRUE(if_ref);
6008 ref2 = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val));
6009 ir_MERGE_WITH_EMPTY_FALSE(if_ref);
6010 return ir_PHI_2(IR_ADDR, ref2, ref);
6011 }
6012
jit_ZVAL_DEREF(zend_jit_ctx * jit,zend_jit_addr addr)6013 static zend_jit_addr jit_ZVAL_DEREF(zend_jit_ctx *jit, zend_jit_addr addr)
6014 {
6015 ir_ref ref = jit_ZVAL_ADDR(jit, addr);
6016 ref = jit_ZVAL_DEREF_ref(jit, ref);
6017 return ZEND_ADDR_REF_ZVAL(ref);
6018 }
6019
jit_ZVAL_INDIRECT_DEREF_ref(zend_jit_ctx * jit,ir_ref ref)6020 static ir_ref jit_ZVAL_INDIRECT_DEREF_ref(zend_jit_ctx *jit, ir_ref ref)
6021 {
6022 ir_ref if_ref, ref2;
6023
6024 if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_INDIRECT)));
6025 ir_IF_TRUE(if_ref);
6026 ref2 = jit_Z_PTR_ref(jit, ref);
6027 ir_MERGE_WITH_EMPTY_FALSE(if_ref);
6028 return ir_PHI_2(IR_ADDR, ref2, ref);
6029 }
6030
jit_ZVAL_INDIRECT_DEREF(zend_jit_ctx * jit,zend_jit_addr addr)6031 static zend_jit_addr jit_ZVAL_INDIRECT_DEREF(zend_jit_ctx *jit, zend_jit_addr addr)
6032 {
6033 ir_ref ref = jit_ZVAL_ADDR(jit, addr);
6034 ref = jit_ZVAL_INDIRECT_DEREF_ref(jit, ref);
6035 return ZEND_ADDR_REF_ZVAL(ref);
6036 }
6037
zend_jit_simple_assign(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr var_addr,uint32_t var_info,uint32_t var_def_info,uint8_t val_type,zend_jit_addr val_addr,uint32_t val_info,zend_jit_addr res_addr,bool check_exception)6038 static int zend_jit_simple_assign(zend_jit_ctx *jit,
6039 const zend_op *opline,
6040 zend_jit_addr var_addr,
6041 uint32_t var_info,
6042 uint32_t var_def_info,
6043 uint8_t val_type,
6044 zend_jit_addr val_addr,
6045 uint32_t val_info,
6046 zend_jit_addr res_addr,
6047 bool check_exception)
6048 {
6049 ir_ref end_inputs = IR_UNUSED;
6050
6051 if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
6052 zval *zv = Z_ZV(val_addr);
6053
6054 if (!res_addr) {
6055 jit_ZVAL_COPY_CONST(jit,
6056 var_addr,
6057 var_info, var_def_info,
6058 zv, 1);
6059 } else {
6060 jit_ZVAL_COPY_CONST(jit,
6061 var_addr,
6062 var_info, var_def_info,
6063 zv, 1);
6064 jit_ZVAL_COPY_CONST(jit,
6065 res_addr,
6066 -1, var_def_info,
6067 zv, 1);
6068 }
6069 } else {
6070 if (val_info & MAY_BE_UNDEF) {
6071 ir_ref if_def, ret;
6072
6073 if_def = jit_if_not_Z_TYPE(jit, val_addr, IS_UNDEF);
6074 ir_IF_FALSE_cold(if_def);
6075
6076 jit_set_Z_TYPE_INFO(jit, var_addr, IS_NULL);
6077 if (res_addr) {
6078 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
6079 }
6080 jit_SET_EX_OPLINE(jit, opline);
6081
6082 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL);
6083 // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
6084 ret = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(Z_OFFSET(val_addr)));
6085
6086 if (check_exception) {
6087 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
6088 }
6089
6090 ir_END_list(end_inputs);
6091 ir_IF_TRUE(if_def);
6092 }
6093 if (val_info & MAY_BE_REF) {
6094 if (val_type == IS_CV) {
6095 ir_ref ref = jit_ZVAL_ADDR(jit, val_addr);
6096 ref = jit_ZVAL_DEREF_ref(jit, ref);
6097 val_addr = ZEND_ADDR_REF_ZVAL(ref);
6098 } else {
6099 ir_ref ref, type, if_ref, ref2, refcount, if_not_zero;
6100
6101 ref = jit_ZVAL_ADDR(jit, val_addr);
6102 type = jit_Z_TYPE_ref(jit, ref);
6103 if_ref = ir_IF(ir_EQ(type, ir_CONST_U8(IS_REFERENCE)));
6104
6105 ir_IF_TRUE_cold(if_ref);
6106 ref = jit_Z_PTR_ref(jit, ref);
6107 ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
6108 if (!res_addr) {
6109 jit_ZVAL_COPY(jit,
6110 var_addr,
6111 var_info,
6112 ZEND_ADDR_REF_ZVAL(ref2), val_info, 1);
6113 } else {
6114 jit_ZVAL_COPY_2(jit,
6115 res_addr,
6116 var_addr,
6117 var_info,
6118 ZEND_ADDR_REF_ZVAL(ref2), val_info, 2);
6119 }
6120
6121 refcount = jit_GC_DELREF(jit, ref);
6122 if_not_zero = ir_IF(refcount);
6123 ir_IF_FALSE(if_not_zero);
6124 // TODO: instead of dtor() call and ADDREF above, we may call efree() and move addref at "true" path ???
6125 // This is related to GH-10168 (keep this before GH-10168 is completely closed)
6126 // jit_EFREE(jit, ref, sizeof(zend_reference), NULL, NULL);
6127 jit_ZVAL_DTOR(jit, ref, val_info, opline);
6128 ir_END_list(end_inputs);
6129 ir_IF_TRUE(if_not_zero);
6130 ir_END_list(end_inputs);
6131
6132 ir_IF_FALSE(if_ref);
6133 }
6134 }
6135
6136 if (!res_addr) {
6137 jit_ZVAL_COPY(jit,
6138 var_addr,
6139 var_info,
6140 val_addr, val_info, val_type == IS_CV);
6141 } else {
6142 jit_ZVAL_COPY_2(jit,
6143 res_addr,
6144 var_addr,
6145 var_info,
6146 val_addr, val_info, val_type == IS_CV ? 2 : 1);
6147 }
6148 }
6149
6150 if (end_inputs) {
6151 ir_END_list(end_inputs);
6152 ir_MERGE_list(end_inputs);
6153 }
6154
6155 return 1;
6156 }
6157
zend_jit_assign_to_variable_call(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr __var_use_addr,zend_jit_addr var_addr,uint32_t __var_info,uint32_t __var_def_info,uint8_t val_type,zend_jit_addr val_addr,uint32_t val_info,zend_jit_addr __res_addr,bool __check_exception)6158 static int zend_jit_assign_to_variable_call(zend_jit_ctx *jit,
6159 const zend_op *opline,
6160 zend_jit_addr __var_use_addr,
6161 zend_jit_addr var_addr,
6162 uint32_t __var_info,
6163 uint32_t __var_def_info,
6164 uint8_t val_type,
6165 zend_jit_addr val_addr,
6166 uint32_t val_info,
6167 zend_jit_addr __res_addr,
6168 bool __check_exception)
6169 {
6170 jit_stub_id func;
6171 ir_ref undef_path = IR_UNUSED;
6172
6173 if (val_info & MAY_BE_UNDEF) {
6174 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
6175 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
6176 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6177
6178 if (!exit_addr) {
6179 return 0;
6180 }
6181
6182 jit_guard_not_Z_TYPE(jit, val_addr, IS_UNDEF, exit_addr);
6183 } else {
6184 ir_ref if_def;
6185
6186 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP);
6187 if_def = ir_IF(jit_Z_TYPE(jit, val_addr));
6188 ir_IF_FALSE_cold(if_def);
6189 jit_SET_EX_OPLINE(jit, opline);
6190 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(Z_OFFSET(val_addr)));
6191
6192 ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_assign_const, IR_FASTCALL_FUNC),
6193 jit_ZVAL_ADDR(jit, var_addr),
6194 jit_EG(uninitialized_zval));
6195
6196 undef_path = ir_END();
6197 ir_IF_TRUE(if_def);
6198 }
6199 }
6200
6201 if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
6202 func = jit_stub_assign_tmp;
6203 } else if (val_type == IS_CONST) {
6204 func = jit_stub_assign_const;
6205 } else if (val_type == IS_TMP_VAR) {
6206 func = jit_stub_assign_tmp;
6207 } else if (val_type == IS_VAR) {
6208 if (!(val_info & MAY_BE_REF)) {
6209 func = jit_stub_assign_tmp;
6210 } else {
6211 func = jit_stub_assign_var;
6212 }
6213 } else if (val_type == IS_CV) {
6214 if (!(val_info & MAY_BE_REF)) {
6215 func = jit_stub_assign_cv_noref;
6216 } else {
6217 func = jit_stub_assign_cv;
6218 }
6219 } else {
6220 ZEND_UNREACHABLE();
6221 }
6222
6223 if (opline) {
6224 jit_SET_EX_OPLINE(jit, opline);
6225 }
6226
6227 ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, func, IR_FASTCALL_FUNC),
6228 jit_ZVAL_ADDR(jit, var_addr),
6229 jit_ZVAL_ADDR(jit, val_addr));
6230
6231 if (undef_path) {
6232 ir_MERGE_WITH(undef_path);
6233 }
6234
6235 return 1;
6236 }
6237
zend_jit_assign_to_variable(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr var_use_addr,zend_jit_addr var_addr,uint32_t var_info,uint32_t var_def_info,uint8_t val_type,zend_jit_addr val_addr,uint32_t val_info,zend_jit_addr res_addr,zend_jit_addr ref_addr,bool check_exception)6238 static int zend_jit_assign_to_variable(zend_jit_ctx *jit,
6239 const zend_op *opline,
6240 zend_jit_addr var_use_addr,
6241 zend_jit_addr var_addr,
6242 uint32_t var_info,
6243 uint32_t var_def_info,
6244 uint8_t val_type,
6245 zend_jit_addr val_addr,
6246 uint32_t val_info,
6247 zend_jit_addr res_addr,
6248 zend_jit_addr ref_addr,
6249 bool check_exception)
6250 {
6251 ir_ref if_refcounted = IR_UNUSED;
6252 ir_ref simple_inputs = IR_UNUSED;
6253 bool done = 0;
6254 zend_jit_addr real_res_addr = 0;
6255 ir_refs *end_inputs;
6256 ir_refs *res_inputs;
6257
6258 ir_refs_init(end_inputs, 6);
6259 ir_refs_init(res_inputs, 6);
6260
6261 if (Z_MODE(val_addr) == IS_REG && jit->ra[Z_SSA_VAR(val_addr)].ref == IR_NULL) {
6262 /* Force load */
6263 zend_jit_use_reg(jit, val_addr);
6264 }
6265
6266 if (Z_MODE(var_addr) == IS_REG) {
6267 jit->delay_var = Z_SSA_VAR(var_addr);
6268 jit->delay_refs = res_inputs;
6269 if (Z_MODE(res_addr) == IS_REG) {
6270 real_res_addr = res_addr;
6271 res_addr = 0;
6272 }
6273 } else if (Z_MODE(res_addr) == IS_REG) {
6274 jit->delay_var = Z_SSA_VAR(res_addr);
6275 jit->delay_refs = res_inputs;
6276 }
6277
6278 if ((var_info & MAY_BE_REF) || ref_addr) {
6279 ir_ref ref = 0, if_ref = 0, ref2, arg2, if_typed, non_ref_path;
6280 uintptr_t func;
6281
6282 if (!ref_addr) {
6283 ref = jit_ZVAL_ADDR(jit, var_use_addr);
6284 if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
6285 ir_IF_TRUE(if_ref);
6286 ref2 = jit_Z_PTR_ref(jit, ref);
6287 } else {
6288 ref2 = jit_ZVAL_ADDR(jit, ref_addr);
6289 }
6290 if_typed = jit_if_TYPED_REF(jit, ref2);
6291 ir_IF_TRUE_cold(if_typed);
6292 jit_SET_EX_OPLINE(jit, opline);
6293 if (Z_MODE(val_addr) == IS_REG) {
6294 zend_jit_addr real_addr;
6295
6296 if (opline->opcode == ZEND_ASSIGN_DIM || opline->opcode == ZEND_ASSIGN_OBJ) {
6297 real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
6298 } else {
6299 ZEND_ASSERT(opline->opcode == ZEND_ASSIGN);
6300 real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
6301 }
6302 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
6303 return 0;
6304 }
6305 arg2 = jit_ZVAL_ADDR(jit, real_addr);
6306 } else {
6307 arg2 = jit_ZVAL_ADDR(jit, val_addr);
6308 }
6309 if (!res_addr) {
6310 if (val_type == IS_CONST) {
6311 func = (uintptr_t)zend_jit_assign_const_to_typed_ref;
6312 } else if (val_type == IS_TMP_VAR) {
6313 func = (uintptr_t)zend_jit_assign_tmp_to_typed_ref;
6314 } else if (val_type == IS_VAR) {
6315 func = (uintptr_t)zend_jit_assign_var_to_typed_ref;
6316 } else if (val_type == IS_CV) {
6317 func = (uintptr_t)zend_jit_assign_cv_to_typed_ref;
6318 } else {
6319 ZEND_UNREACHABLE();
6320 }
6321 ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2);
6322 } else {
6323 if (val_type == IS_CONST) {
6324 func = (uintptr_t)zend_jit_assign_const_to_typed_ref2;
6325 } else if (val_type == IS_TMP_VAR) {
6326 func = (uintptr_t)zend_jit_assign_tmp_to_typed_ref2;
6327 } else if (val_type == IS_VAR) {
6328 func = (uintptr_t)zend_jit_assign_var_to_typed_ref2;
6329 } else if (val_type == IS_CV) {
6330 func = (uintptr_t)zend_jit_assign_cv_to_typed_ref2;
6331 } else {
6332 ZEND_UNREACHABLE();
6333 }
6334 ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2, jit_ZVAL_ADDR(jit, res_addr));
6335 }
6336 if (check_exception) {
6337 zend_jit_check_exception(jit);
6338 }
6339 ir_refs_add(end_inputs, ir_END());
6340
6341 if (!ref_addr) {
6342 ir_IF_FALSE(if_ref);
6343 non_ref_path = ir_END();
6344 ir_IF_FALSE(if_typed);
6345 ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
6346 ir_MERGE_WITH(non_ref_path);
6347 ref = ir_PHI_2(IR_ADDR, ref2, ref);
6348 var_addr = var_use_addr = ZEND_ADDR_REF_ZVAL(ref);
6349 } else {
6350 ir_IF_FALSE(if_typed);
6351 }
6352 }
6353
6354 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
6355 ir_ref ref, counter, if_not_zero;
6356
6357 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
6358 if_refcounted = jit_if_REFCOUNTED(jit, var_use_addr);
6359 ir_IF_FALSE(if_refcounted);
6360 ir_END_list(simple_inputs);
6361 ir_IF_TRUE_cold(if_refcounted);
6362 } else if (RC_MAY_BE_1(var_info)) {
6363 done = 1;
6364 }
6365 ref = jit_Z_PTR(jit, var_use_addr);
6366 if (RC_MAY_BE_1(var_info)) {
6367 if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0)) {
6368 return 0;
6369 }
6370 counter = jit_GC_DELREF(jit, ref);
6371
6372 if_not_zero = ir_IF(counter);
6373 ir_IF_FALSE(if_not_zero);
6374 jit_ZVAL_DTOR(jit, ref, var_info, opline);
6375 if (check_exception) {
6376 zend_jit_check_exception(jit);
6377 }
6378 ir_refs_add(end_inputs, ir_END());
6379 ir_IF_TRUE(if_not_zero);
6380 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
6381 ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
6382 ir_IF_FALSE(if_may_leak);
6383 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
6384
6385 if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
6386 ZEND_ASSERT(jit->delay_refs == res_inputs);
6387 ZEND_ASSERT(res_inputs->count > 0);
6388 ir_refs_add(res_inputs, res_inputs->refs[res_inputs->count - 1]);
6389 }
6390 if (check_exception && (val_info & MAY_BE_UNDEF)) {
6391 zend_jit_check_exception(jit);
6392 }
6393 ir_refs_add(end_inputs, ir_END());
6394 ir_IF_TRUE(if_may_leak);
6395 }
6396 if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
6397 ZEND_ASSERT(jit->delay_refs == res_inputs);
6398 ZEND_ASSERT(res_inputs->count > 0);
6399 ir_refs_add(res_inputs, res_inputs->refs[res_inputs->count - 1]);
6400 }
6401 if (check_exception && (val_info & MAY_BE_UNDEF)) {
6402 zend_jit_check_exception(jit);
6403 }
6404 ir_refs_add(end_inputs, ir_END());
6405 } else /* if (RC_MAY_BE_N(var_info)) */ {
6406 jit_GC_DELREF(jit, ref);
6407 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) {
6408 ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
6409 ir_IF_FALSE(if_may_leak);
6410 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
6411 ir_END_list(simple_inputs);
6412 ir_IF_TRUE(if_may_leak);
6413 }
6414 ir_END_list(simple_inputs);
6415 }
6416 }
6417
6418 if (simple_inputs) {
6419 ir_MERGE_list(simple_inputs);
6420 }
6421
6422 if (!done) {
6423 if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, check_exception)) {
6424 return 0;
6425 }
6426 if (end_inputs->count) {
6427 ir_refs_add(end_inputs, ir_END());
6428 }
6429 }
6430
6431 if (end_inputs->count) {
6432 ir_MERGE_N(end_inputs->count, end_inputs->refs);
6433 }
6434
6435 if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
6436 ir_ref phi;
6437
6438 ZEND_ASSERT(jit->delay_refs == res_inputs);
6439 ZEND_ASSERT(end_inputs->count == res_inputs->count || (end_inputs->count == 0 && res_inputs->count == 1));
6440 jit->delay_var = -1;
6441 jit->delay_refs = NULL;
6442 if (res_inputs->count == 1) {
6443 phi = res_inputs->refs[0];
6444 } else {
6445 phi = ir_PHI_N((var_def_info & MAY_BE_LONG & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE,
6446 res_inputs->count, res_inputs->refs);
6447 }
6448 if (Z_MODE(var_addr) == IS_REG) {
6449 if ((var_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || ref_addr) {
6450 phi = ir_emit2(&jit->ctx, IR_OPT(IR_COPY, jit->ctx.ir_base[phi].type), phi, 1);
6451 }
6452 zend_jit_def_reg(jit, var_addr, phi);
6453 if (real_res_addr) {
6454 if (var_def_info & MAY_BE_LONG) {
6455 jit_set_Z_LVAL(jit, real_res_addr, jit_Z_LVAL(jit, var_addr));
6456 } else {
6457 jit_set_Z_DVAL(jit, real_res_addr, jit_Z_DVAL(jit, var_addr));
6458 }
6459 }
6460 } else {
6461 zend_jit_def_reg(jit, res_addr, phi);
6462 }
6463 }
6464
6465 return 1;
6466 }
6467
zend_jit_qm_assign(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_addr,zend_jit_addr op1_def_addr,uint32_t res_use_info,uint32_t res_info,zend_jit_addr res_addr)6468 static int zend_jit_qm_assign(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr)
6469 {
6470 if (op1_addr != op1_def_addr) {
6471 if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
6472 return 0;
6473 }
6474 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
6475 op1_addr = op1_def_addr;
6476 }
6477 }
6478
6479 if (!zend_jit_simple_assign(jit, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 1)) {
6480 return 0;
6481 }
6482 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
6483 return 0;
6484 }
6485 return 1;
6486 }
6487
zend_jit_assign(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_use_addr,uint32_t op1_def_info,zend_jit_addr op1_addr,uint32_t op2_info,zend_jit_addr op2_addr,zend_jit_addr op2_def_addr,uint32_t res_info,zend_jit_addr res_addr,zend_jit_addr ref_addr,int may_throw)6488 static int zend_jit_assign(zend_jit_ctx *jit,
6489 const zend_op *opline,
6490 uint32_t op1_info,
6491 zend_jit_addr op1_use_addr,
6492 uint32_t op1_def_info,
6493 zend_jit_addr op1_addr,
6494 uint32_t op2_info,
6495 zend_jit_addr op2_addr,
6496 zend_jit_addr op2_def_addr,
6497 uint32_t res_info,
6498 zend_jit_addr res_addr,
6499 zend_jit_addr ref_addr,
6500 int may_throw)
6501 {
6502 ZEND_ASSERT(opline->op1_type == IS_CV);
6503
6504 if (op2_addr != op2_def_addr) {
6505 if (!zend_jit_update_regs(jit, opline->op2.var, op2_addr, op2_def_addr, op2_info)) {
6506 return 0;
6507 }
6508 if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) {
6509 op2_addr = op2_def_addr;
6510 }
6511 }
6512
6513 if (Z_MODE(op1_addr) != IS_REG
6514 && Z_MODE(op1_use_addr) == IS_REG
6515 && !Z_LOAD(op1_use_addr)
6516 && !Z_STORE(op1_use_addr)) {
6517 /* Force type update */
6518 op1_info |= MAY_BE_UNDEF;
6519 }
6520 if (!zend_jit_assign_to_variable(jit, opline, op1_use_addr, op1_addr, op1_info, op1_def_info,
6521 opline->op2_type, op2_addr, op2_info, res_addr, ref_addr, may_throw)) {
6522 return 0;
6523 }
6524 if (Z_MODE(op1_addr) == IS_REG) {
6525 if (Z_STORE(op1_addr)) {
6526 if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) {
6527 return 0;
6528 }
6529 } else if ((op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
6530 && Z_MODE(op1_use_addr) == IS_MEM_ZVAL
6531 && Z_REG(op1_use_addr) == ZREG_FP
6532 && EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)) < jit->current_op_array->last_var) {
6533 /* We have to update type of CV because it may be captured by exception backtrace or released on RETURN */
6534 if ((op1_def_info & MAY_BE_ANY) == MAY_BE_LONG) {
6535 jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_LONG);
6536 if (JIT_G(current_frame)) {
6537 SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)), IS_LONG, 1);
6538 }
6539 } else if ((op1_def_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
6540 jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_DOUBLE);
6541 if (JIT_G(current_frame)) {
6542 SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)), IS_DOUBLE, 1);
6543 }
6544 } else {
6545 ZEND_UNREACHABLE();
6546 }
6547 }
6548 }
6549 if (opline->result_type != IS_UNUSED) {
6550 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
6551 return 0;
6552 }
6553 }
6554
6555 return 1;
6556 }
6557
zend_jit_cmp_op(const zend_op * opline)6558 static ir_op zend_jit_cmp_op(const zend_op *opline)
6559 {
6560 ir_op op;
6561
6562 switch (opline->opcode) {
6563 case ZEND_IS_EQUAL:
6564 case ZEND_IS_IDENTICAL:
6565 case ZEND_CASE:
6566 case ZEND_CASE_STRICT:
6567 op = IR_EQ;
6568 break;
6569 case ZEND_IS_NOT_EQUAL:
6570 case ZEND_IS_NOT_IDENTICAL:
6571 op = IR_NE;
6572 break;
6573 case ZEND_IS_SMALLER:
6574 op = IR_LT;
6575 break;
6576 case ZEND_IS_SMALLER_OR_EQUAL:
6577 op = IR_LE;
6578 break;
6579 default:
6580 ZEND_UNREACHABLE();
6581 }
6582 return op;
6583 }
6584
zend_jit_cmp_long_long(zend_jit_ctx * jit,const zend_op * opline,zend_ssa_range * op1_range,zend_jit_addr op1_addr,zend_ssa_range * op2_range,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr,bool skip_comparison)6585 static ir_ref zend_jit_cmp_long_long(zend_jit_ctx *jit,
6586 const zend_op *opline,
6587 zend_ssa_range *op1_range,
6588 zend_jit_addr op1_addr,
6589 zend_ssa_range *op2_range,
6590 zend_jit_addr op2_addr,
6591 zend_jit_addr res_addr,
6592 uint8_t smart_branch_opcode,
6593 uint32_t target_label,
6594 uint32_t target_label2,
6595 const void *exit_addr,
6596 bool skip_comparison)
6597 {
6598 ir_ref ref;
6599 bool result;
6600
6601 if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) {
6602 if (!smart_branch_opcode ||
6603 smart_branch_opcode == ZEND_JMPZ_EX ||
6604 smart_branch_opcode == ZEND_JMPNZ_EX) {
6605 jit_set_Z_TYPE_INFO(jit, res_addr, result ? IS_TRUE : IS_FALSE);
6606 }
6607 if (smart_branch_opcode && !exit_addr) {
6608 if (smart_branch_opcode == ZEND_JMPZ ||
6609 smart_branch_opcode == ZEND_JMPZ_EX) {
6610 return jit_IF_ex(jit, IR_FALSE, result ? target_label : target_label2);
6611 } else if (smart_branch_opcode == ZEND_JMPNZ ||
6612 smart_branch_opcode == ZEND_JMPNZ_EX) {
6613 return jit_IF_ex(jit, IR_TRUE, result ? target_label : target_label2);
6614 } else {
6615 ZEND_UNREACHABLE();
6616 }
6617 }
6618 if (opline->opcode != ZEND_IS_IDENTICAL
6619 && opline->opcode != ZEND_IS_NOT_IDENTICAL
6620 && opline->opcode != ZEND_CASE_STRICT) {
6621 return ir_END();
6622 } else {
6623 return IR_NULL; /* success */
6624 }
6625 }
6626
6627 ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_LVAL(jit, op1_addr), jit_Z_LVAL(jit, op2_addr));
6628
6629 if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
6630 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
6631 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
6632 }
6633 if (exit_addr) {
6634 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
6635 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
6636 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6637 } else {
6638 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6639 }
6640 } else {
6641 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
6642 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6643 } else {
6644 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6645 }
6646 }
6647 } else if (smart_branch_opcode) {
6648 return jit_IF_ex(jit, ref,
6649 (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
6650 }
6651
6652 if (opline->opcode != ZEND_IS_IDENTICAL
6653 && opline->opcode != ZEND_IS_NOT_IDENTICAL
6654 && opline->opcode != ZEND_CASE_STRICT) {
6655 return ir_END();
6656 } else {
6657 return IR_NULL; /* success */
6658 }
6659 }
6660
zend_jit_cmp_long_double(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)6661 static ir_ref zend_jit_cmp_long_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
6662 {
6663 ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), ir_INT2D(jit_Z_LVAL(jit, op1_addr)), jit_Z_DVAL(jit, op2_addr));
6664
6665 if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
6666 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
6667 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
6668 }
6669 if (exit_addr) {
6670 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
6671 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6672 } else {
6673 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6674 }
6675 } else if (smart_branch_opcode) {
6676 return jit_IF_ex(jit, ref,
6677 (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
6678 }
6679 return ir_END();
6680 }
6681
zend_jit_cmp_double_long(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)6682 static ir_ref zend_jit_cmp_double_long(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
6683 {
6684 ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), ir_INT2D(jit_Z_LVAL(jit, op2_addr)));
6685
6686 if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
6687 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
6688 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
6689 }
6690 if (exit_addr) {
6691 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
6692 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6693 } else {
6694 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6695 }
6696 } else if (smart_branch_opcode) {
6697 return jit_IF_ex(jit, ref,
6698 (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
6699 }
6700 return ir_END();
6701 }
6702
zend_jit_cmp_double_double(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)6703 static ir_ref zend_jit_cmp_double_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
6704 {
6705 ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), jit_Z_DVAL(jit, op2_addr));
6706
6707 if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
6708 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
6709 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
6710 }
6711 if (exit_addr) {
6712 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
6713 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
6714 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6715 } else {
6716 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6717 }
6718 } else {
6719 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
6720 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6721 } else {
6722 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6723 }
6724 }
6725 } else if (smart_branch_opcode) {
6726 return jit_IF_ex(jit, ref,
6727 (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
6728 }
6729 if (opline->opcode != ZEND_IS_IDENTICAL
6730 && opline->opcode != ZEND_IS_NOT_IDENTICAL
6731 && opline->opcode != ZEND_CASE_STRICT) {
6732 return ir_END();
6733 } else {
6734 return IR_NULL; /* success */
6735 }
6736 }
6737
zend_jit_cmp_slow(zend_jit_ctx * jit,ir_ref ref,const zend_op * opline,zend_jit_addr res_addr,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)6738 static ir_ref zend_jit_cmp_slow(zend_jit_ctx *jit, ir_ref ref, const zend_op *opline, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
6739 {
6740 ref = ir_CMP_OP(zend_jit_cmp_op(opline), ref, ir_CONST_I32(0));
6741
6742 if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
6743 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
6744 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
6745 }
6746 if (exit_addr) {
6747 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
6748 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6749 } else {
6750 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6751 }
6752 } else if (smart_branch_opcode) {
6753 return jit_IF_ex(jit, ref,
6754 (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
6755 }
6756
6757 return ir_END();
6758 }
6759
zend_jit_cmp(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_ssa_range * op1_range,zend_jit_addr op1_addr,uint32_t op2_info,zend_ssa_range * op2_range,zend_jit_addr op2_addr,zend_jit_addr res_addr,int may_throw,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr,bool skip_comparison)6760 static int zend_jit_cmp(zend_jit_ctx *jit,
6761 const zend_op *opline,
6762 uint32_t op1_info,
6763 zend_ssa_range *op1_range,
6764 zend_jit_addr op1_addr,
6765 uint32_t op2_info,
6766 zend_ssa_range *op2_range,
6767 zend_jit_addr op2_addr,
6768 zend_jit_addr res_addr,
6769 int may_throw,
6770 uint8_t smart_branch_opcode,
6771 uint32_t target_label,
6772 uint32_t target_label2,
6773 const void *exit_addr,
6774 bool skip_comparison)
6775 {
6776 ir_ref ref = IR_UNUSED;
6777 ir_ref if_op1_long = IR_UNUSED;
6778 ir_ref if_op1_double = IR_UNUSED;
6779 ir_ref if_op2_double = IR_UNUSED;
6780 ir_ref if_op1_long_op2_long = IR_UNUSED;
6781 ir_ref if_op1_long_op2_double = IR_UNUSED;
6782 ir_ref if_op1_double_op2_double = IR_UNUSED;
6783 ir_ref if_op1_double_op2_long = IR_UNUSED;
6784 ir_ref slow_inputs = IR_UNUSED;
6785 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
6786 bool has_slow =
6787 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
6788 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
6789 ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
6790 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))));
6791 ir_refs *end_inputs;
6792
6793 ir_refs_init(end_inputs, 8);
6794
6795 if (Z_MODE(op1_addr) == IS_REG) {
6796 if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) {
6797 /* Force load */
6798 zend_jit_use_reg(jit, op1_addr);
6799 }
6800 } else if (Z_MODE(op2_addr) == IS_REG) {
6801 if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) {
6802 /* Force load */
6803 zend_jit_use_reg(jit, op2_addr);
6804 }
6805 }
6806
6807 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
6808 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
6809 if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
6810 ir_IF_TRUE(if_op1_long);
6811 }
6812 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
6813 if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
6814 ir_IF_FALSE_cold(if_op1_long_op2_long);
6815 if (op2_info & MAY_BE_DOUBLE) {
6816 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
6817 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
6818 ir_IF_FALSE_cold(if_op1_long_op2_double);
6819 ir_END_list(slow_inputs);
6820 ir_IF_TRUE(if_op1_long_op2_double);
6821 }
6822 ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6823 if (!ref) {
6824 return 0;
6825 }
6826 ir_refs_add(end_inputs, ref);
6827 } else {
6828 ir_END_list(slow_inputs);
6829 }
6830 ir_IF_TRUE(if_op1_long_op2_long);
6831 }
6832 ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison);
6833 if (!ref) {
6834 return 0;
6835 }
6836 ir_refs_add(end_inputs, ref);
6837
6838 if (if_op1_long) {
6839 ir_IF_FALSE_cold(if_op1_long);
6840 }
6841 if (op1_info & MAY_BE_DOUBLE) {
6842 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
6843 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
6844 ir_IF_FALSE_cold(if_op1_double);
6845 ir_END_list(slow_inputs);
6846 ir_IF_TRUE(if_op1_double);
6847 }
6848 if (op2_info & MAY_BE_DOUBLE) {
6849 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
6850 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
6851 ir_IF_TRUE(if_op1_double_op2_double);
6852 }
6853 ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6854 if (!ref) {
6855 return 0;
6856 }
6857 ir_refs_add(end_inputs, ref);
6858 if (if_op1_double_op2_double) {
6859 ir_IF_FALSE_cold(if_op1_double_op2_double);
6860 }
6861 }
6862 if (!same_ops) {
6863 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
6864 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
6865 ir_IF_FALSE_cold(if_op1_double_op2_long);
6866 ir_END_list(slow_inputs);
6867 ir_IF_TRUE(if_op1_double_op2_long);
6868 }
6869 ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6870 if (!ref) {
6871 return 0;
6872 }
6873 ir_refs_add(end_inputs, ref);
6874 } else if (if_op1_double_op2_double) {
6875 ir_END_list(slow_inputs);
6876 }
6877 } else if (if_op1_long) {
6878 ir_END_list(slow_inputs);
6879 }
6880 } else if ((op1_info & MAY_BE_DOUBLE) &&
6881 !(op1_info & MAY_BE_LONG) &&
6882 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
6883 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
6884 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
6885 ir_IF_FALSE_cold(if_op1_double);
6886 ir_END_list(slow_inputs);
6887 ir_IF_TRUE(if_op1_double);
6888 }
6889 if (op2_info & MAY_BE_DOUBLE) {
6890 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
6891 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
6892 ir_IF_TRUE(if_op1_double_op2_double);
6893 }
6894 ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6895 if (!ref) {
6896 return 0;
6897 }
6898 ir_refs_add(end_inputs, ref);
6899 if (if_op1_double_op2_double) {
6900 ir_IF_FALSE_cold(if_op1_double_op2_double);
6901 }
6902 }
6903 if (!same_ops && (op2_info & MAY_BE_LONG)) {
6904 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
6905 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
6906 ir_IF_FALSE_cold(if_op1_double_op2_long);
6907 ir_END_list(slow_inputs);
6908 ir_IF_TRUE(if_op1_double_op2_long);
6909 }
6910 ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6911 if (!ref) {
6912 return 0;
6913 }
6914 ir_refs_add(end_inputs, ref);
6915 } else if (if_op1_double_op2_double) {
6916 ir_END_list(slow_inputs);
6917 }
6918 } else if ((op2_info & MAY_BE_DOUBLE) &&
6919 !(op2_info & MAY_BE_LONG) &&
6920 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
6921 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
6922 if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
6923 ir_IF_FALSE_cold(if_op2_double);
6924 ir_END_list(slow_inputs);
6925 ir_IF_TRUE(if_op2_double);
6926 }
6927 if (op1_info & MAY_BE_DOUBLE) {
6928 if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
6929 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
6930 ir_IF_TRUE(if_op1_double_op2_double);
6931 }
6932 ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6933 if (!ref) {
6934 return 0;
6935 }
6936 ir_refs_add(end_inputs, ref);
6937 if (if_op1_double_op2_double) {
6938 ir_IF_FALSE_cold(if_op1_double_op2_double);
6939 }
6940 }
6941 if (!same_ops && (op1_info & MAY_BE_LONG)) {
6942 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
6943 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
6944 ir_IF_FALSE_cold(if_op1_long_op2_double);
6945 ir_END_list(slow_inputs);
6946 ir_IF_TRUE(if_op1_long_op2_double);
6947 }
6948 ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6949 if (!ref) {
6950 return 0;
6951 }
6952 ir_refs_add(end_inputs, ref);
6953 } else if (if_op1_double_op2_double) {
6954 ir_END_list(slow_inputs);
6955 }
6956 }
6957
6958 if (has_slow ||
6959 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
6960 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
6961 ir_ref op1, op2, ref;
6962
6963 if (slow_inputs) {
6964 ir_MERGE_list(slow_inputs);
6965 }
6966 jit_SET_EX_OPLINE(jit, opline);
6967
6968 if (Z_MODE(op1_addr) == IS_REG) {
6969 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
6970 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
6971 return 0;
6972 }
6973 op1_addr = real_addr;
6974 }
6975 if (Z_MODE(op2_addr) == IS_REG) {
6976 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
6977 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
6978 return 0;
6979 }
6980 op2_addr = real_addr;
6981 }
6982
6983 op1 = jit_ZVAL_ADDR(jit, op1_addr);
6984 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
6985 op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, NULL, 0);
6986 }
6987 op2 = jit_ZVAL_ADDR(jit, op2_addr);
6988 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
6989 op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, NULL, 0);
6990 }
6991 ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_compare), op1, op2);
6992 if (opline->opcode != ZEND_CASE) {
6993 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
6994 }
6995 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
6996 if (may_throw) {
6997 zend_jit_check_exception_undef_result(jit, opline);
6998 }
6999
7000 ref = zend_jit_cmp_slow(jit, ref, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
7001 if (!ref) {
7002 return 0;
7003 }
7004 ir_refs_add(end_inputs, ref);
7005 }
7006
7007 if (end_inputs->count) {
7008 uint32_t n = end_inputs->count;
7009
7010 if (smart_branch_opcode && !exit_addr) {
7011 zend_basic_block *bb;
7012 ir_ref ref;
7013 uint32_t label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
7014 target_label2 : target_label;
7015 uint32_t label2 = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
7016 target_label : target_label2;
7017
7018 ZEND_ASSERT(jit->b >= 0);
7019 bb = &jit->ssa->cfg.blocks[jit->b];
7020 ZEND_ASSERT(bb->successors_count == 2);
7021
7022 if (UNEXPECTED(bb->successors[0] == bb->successors[1])) {
7023 ir_ref merge_inputs = IR_UNUSED;
7024
7025 while (n) {
7026 n--;
7027 ir_IF_TRUE(end_inputs->refs[n]);
7028 ir_END_list(merge_inputs);
7029 ir_IF_FALSE(end_inputs->refs[n]);
7030 ir_END_list(merge_inputs);
7031 }
7032 ir_MERGE_list(merge_inputs);
7033 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
7034 } else if (n == 1) {
7035 ref = end_inputs->refs[0];
7036 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
7037 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
7038 } else {
7039 ir_ref true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
7040
7041 while (n) {
7042 n--;
7043 ir_IF_TRUE(end_inputs->refs[n]);
7044 ir_END_list(true_inputs);
7045 ir_IF_FALSE(end_inputs->refs[n]);
7046 ir_END_list(false_inputs);
7047 }
7048 ir_MERGE_list(true_inputs);
7049 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
7050 ir_MERGE_list(false_inputs);
7051 _zend_jit_add_predecessor_ref(jit, label2, jit->b, ir_END());
7052 }
7053 jit->b = -1;
7054 } else {
7055 ir_MERGE_N(n, end_inputs->refs);
7056 }
7057 } else if (smart_branch_opcode && !exit_addr) {
7058 /* dead code */
7059 _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
7060 jit->b = -1;
7061 }
7062
7063 return 1;
7064 }
7065
zend_jit_identical(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_ssa_range * op1_range,zend_jit_addr op1_addr,uint32_t op2_info,zend_ssa_range * op2_range,zend_jit_addr op2_addr,zend_jit_addr res_addr,int may_throw,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr,bool skip_comparison)7066 static int zend_jit_identical(zend_jit_ctx *jit,
7067 const zend_op *opline,
7068 uint32_t op1_info,
7069 zend_ssa_range *op1_range,
7070 zend_jit_addr op1_addr,
7071 uint32_t op2_info,
7072 zend_ssa_range *op2_range,
7073 zend_jit_addr op2_addr,
7074 zend_jit_addr res_addr,
7075 int may_throw,
7076 uint8_t smart_branch_opcode,
7077 uint32_t target_label,
7078 uint32_t target_label2,
7079 const void *exit_addr,
7080 bool skip_comparison)
7081 {
7082 bool always_false = 0, always_true = 0;
7083 ir_ref ref = IR_UNUSED;
7084
7085 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
7086 ir_ref op1 = jit_ZVAL_ADDR(jit, op1_addr);
7087 op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, opline, 0);
7088 op1_info |= MAY_BE_NULL;
7089 op1_addr = ZEND_ADDR_REF_ZVAL(op1);
7090 }
7091 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
7092 ir_ref op2 = jit_ZVAL_ADDR(jit, op2_addr);
7093 op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, opline, 0);
7094 op2_info |= MAY_BE_NULL;
7095 op2_addr = ZEND_ADDR_REF_ZVAL(op2);
7096 }
7097
7098 if ((op1_info & op2_info & MAY_BE_ANY) == 0) {
7099 always_false = 1;
7100 } else if (has_concrete_type(op1_info)
7101 && has_concrete_type(op2_info)
7102 && concrete_type(op1_info) == concrete_type(op2_info)
7103 && concrete_type(op1_info) <= IS_TRUE) {
7104 always_true = 1;
7105 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) {
7106 if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) {
7107 always_true = 1;
7108 } else {
7109 always_false = 1;
7110 }
7111 }
7112
7113 if (always_true) {
7114 if (opline->opcode != ZEND_CASE_STRICT) {
7115 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
7116 }
7117 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
7118 if (!smart_branch_opcode
7119 || smart_branch_opcode == ZEND_JMPZ_EX
7120 || smart_branch_opcode == ZEND_JMPNZ_EX) {
7121 jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE);
7122 }
7123 if (may_throw) {
7124 zend_jit_check_exception(jit);
7125 }
7126 if (exit_addr) {
7127 if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) {
7128 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7129 }
7130 } else if (smart_branch_opcode) {
7131 uint32_t label;
7132
7133 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
7134 label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
7135 target_label : target_label2;
7136 } else {
7137 label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
7138 target_label2 : target_label;
7139 }
7140 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
7141 jit->b = -1;
7142 }
7143 return 1;
7144 } else if (always_false) {
7145 if (opline->opcode != ZEND_CASE_STRICT) {
7146 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
7147 }
7148 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
7149 if (!smart_branch_opcode
7150 || smart_branch_opcode == ZEND_JMPZ_EX
7151 || smart_branch_opcode == ZEND_JMPNZ_EX) {
7152 jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE);
7153 }
7154 if (may_throw) {
7155 zend_jit_check_exception(jit);
7156 }
7157 if (exit_addr) {
7158 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
7159 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7160 }
7161 } else if (smart_branch_opcode) {
7162 uint32_t label;
7163
7164 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
7165 label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
7166 target_label2 : target_label;
7167 } else {
7168 label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
7169 target_label : target_label2;
7170 }
7171 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
7172 jit->b = -1;
7173 }
7174 return 1;
7175 }
7176
7177 if ((opline->op1_type & (IS_CV|IS_VAR)) && (op1_info & MAY_BE_REF)) {
7178 ref = jit_ZVAL_ADDR(jit, op1_addr);
7179 ref = jit_ZVAL_DEREF_ref(jit, ref);
7180 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
7181 }
7182 if ((opline->op2_type & (IS_CV|IS_VAR)) && (op2_info & MAY_BE_REF)) {
7183 ref = jit_ZVAL_ADDR(jit, op2_addr);
7184 ref = jit_ZVAL_DEREF_ref(jit, ref);
7185 op2_addr = ZEND_ADDR_REF_ZVAL(ref);
7186 }
7187
7188 if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG &&
7189 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) {
7190 ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison);
7191 if (!ref) {
7192 return 0;
7193 }
7194 } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE &&
7195 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) {
7196 ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
7197 if (!ref) {
7198 return 0;
7199 }
7200 } else {
7201 if (opline->op1_type != IS_CONST) {
7202 if (Z_MODE(op1_addr) == IS_REG) {
7203 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
7204 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
7205 return 0;
7206 }
7207 op1_addr = real_addr;
7208 }
7209 }
7210 if (opline->op2_type != IS_CONST) {
7211 if (Z_MODE(op2_addr) == IS_REG) {
7212 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
7213 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
7214 return 0;
7215 }
7216 }
7217 }
7218
7219 if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) {
7220 zval *val = Z_ZV(op1_addr);
7221
7222 ref = ir_EQ(jit_Z_TYPE(jit, op2_addr), ir_CONST_U8(Z_TYPE_P(val)));
7223 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) {
7224 zval *val = Z_ZV(op2_addr);
7225
7226 ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(Z_TYPE_P(val)));
7227 } else {
7228 if (Z_MODE(op1_addr) == IS_REG) {
7229 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
7230 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
7231 return 0;
7232 }
7233 op1_addr = real_addr;
7234 }
7235 if (Z_MODE(op2_addr) == IS_REG) {
7236 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
7237 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
7238 return 0;
7239 }
7240 op2_addr = real_addr;
7241 }
7242
7243 ref = ir_CALL_2(IR_BOOL, ir_CONST_FC_FUNC(zend_is_identical),
7244 jit_ZVAL_ADDR(jit, op1_addr),
7245 jit_ZVAL_ADDR(jit, op2_addr));
7246 }
7247
7248 if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
7249 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
7250 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7251 ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
7252 } else {
7253 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7254 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
7255 }
7256 }
7257 if (opline->opcode != ZEND_CASE_STRICT) {
7258 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
7259 }
7260 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
7261 if (may_throw) {
7262 zend_jit_check_exception_undef_result(jit, opline);
7263 }
7264 if (exit_addr) {
7265 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
7266 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
7267 } else {
7268 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
7269 }
7270 } else if (smart_branch_opcode) {
7271 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
7272 /* swap labels */
7273 uint32_t tmp = target_label;
7274 target_label = target_label2;
7275 target_label2 = tmp;
7276 }
7277 ref = jit_IF_ex(jit, ref,
7278 (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
7279 }
7280 }
7281
7282 if (smart_branch_opcode && !exit_addr) {
7283 zend_basic_block *bb;
7284
7285 ZEND_ASSERT(jit->b >= 0);
7286 bb = &jit->ssa->cfg.blocks[jit->b];
7287 ZEND_ASSERT(bb->successors_count == 2);
7288
7289 if (bb->successors_count == 2 && bb->successors[0] == bb->successors[1]) {
7290 ir_IF_TRUE(ref);
7291 ir_MERGE_WITH_EMPTY_FALSE(ref);
7292 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
7293 } else {
7294 ZEND_ASSERT(bb->successors_count == 2);
7295 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
7296 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
7297 }
7298 jit->b = -1;
7299 }
7300
7301 return 1;
7302 }
7303
zend_jit_bool_jmpznz(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_addr,zend_jit_addr res_addr,uint32_t target_label,uint32_t target_label2,int may_throw,uint8_t branch_opcode,const void * exit_addr)7304 static int zend_jit_bool_jmpznz(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, uint8_t branch_opcode, const void *exit_addr)
7305 {
7306 uint32_t true_label = -1;
7307 uint32_t false_label = -1;
7308 bool set_bool = 0;
7309 bool set_bool_not = 0;
7310 bool always_true = 0, always_false = 0;
7311 ir_ref ref, end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
7312 ir_type type = IR_UNUSED;
7313
7314 if (branch_opcode == ZEND_BOOL) {
7315 set_bool = 1;
7316 } else if (branch_opcode == ZEND_BOOL_NOT) {
7317 set_bool = 1;
7318 set_bool_not = 1;
7319 } else if (branch_opcode == ZEND_JMPZ) {
7320 true_label = target_label2;
7321 false_label = target_label;
7322 } else if (branch_opcode == ZEND_JMPNZ) {
7323 true_label = target_label;
7324 false_label = target_label2;
7325 } else if (branch_opcode == ZEND_JMPZ_EX) {
7326 set_bool = 1;
7327 true_label = target_label2;
7328 false_label = target_label;
7329 } else if (branch_opcode == ZEND_JMPNZ_EX) {
7330 set_bool = 1;
7331 true_label = target_label;
7332 false_label = target_label2;
7333 } else {
7334 ZEND_UNREACHABLE();
7335 }
7336
7337 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) {
7338 ref = jit_ZVAL_ADDR(jit, op1_addr);
7339 ref = jit_ZVAL_DEREF_ref(jit, ref);
7340 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
7341 }
7342
7343 if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
7344 if (zend_is_true(Z_ZV(op1_addr))) {
7345 always_true = 1;
7346 } else {
7347 always_false = 1;
7348 }
7349 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
7350 if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) {
7351 always_true = 1;
7352 } else if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) {
7353 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
7354 ref = jit_ZVAL_ADDR(jit, op1_addr);
7355 zend_jit_zval_check_undef(jit, ref, opline->op1.var, opline, 0);
7356 }
7357 always_false = 1;
7358 }
7359 }
7360
7361 if (always_true) {
7362 if (set_bool) {
7363 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
7364 }
7365 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
7366 if (may_throw) {
7367 zend_jit_check_exception(jit);
7368 }
7369 if (true_label != (uint32_t)-1) {
7370 ZEND_ASSERT(exit_addr == NULL);
7371 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END());
7372 jit->b = -1;
7373 }
7374 return 1;
7375 } else if (always_false) {
7376 if (set_bool) {
7377 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
7378 }
7379 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
7380 if (may_throw) {
7381 zend_jit_check_exception(jit);
7382 }
7383 if (false_label != (uint32_t)-1) {
7384 ZEND_ASSERT(exit_addr == NULL);
7385 _zend_jit_add_predecessor_ref(jit, false_label, jit->b, ir_END());
7386 jit->b = -1;
7387 }
7388 return 1;
7389 }
7390
7391 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
7392 type = jit_Z_TYPE(jit, op1_addr);
7393 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) {
7394 ir_ref if_type = ir_IF(ir_LT(type, ir_CONST_U8(IS_TRUE)));
7395
7396 ir_IF_TRUE_cold(if_type);
7397
7398 if (op1_info & MAY_BE_UNDEF) {
7399 zend_jit_type_check_undef(jit,
7400 type,
7401 opline->op1.var,
7402 opline, 1, 0, 1);
7403 }
7404 if (set_bool) {
7405 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
7406 }
7407 if (exit_addr) {
7408 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
7409 ir_END_list(end_inputs);
7410 } else {
7411 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7412 }
7413 } else if (false_label != (uint32_t)-1) {
7414 ir_END_list(false_inputs);
7415 } else {
7416 ir_END_list(end_inputs);
7417 }
7418 ir_IF_FALSE(if_type);
7419 }
7420
7421 if (op1_info & MAY_BE_TRUE) {
7422 ir_ref if_type = IR_UNUSED;
7423
7424 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
7425 if_type = ir_IF(ir_EQ(type, ir_CONST_U8(IS_TRUE)));
7426
7427 ir_IF_TRUE(if_type);
7428 }
7429 if (set_bool) {
7430 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
7431 }
7432 if (exit_addr) {
7433 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
7434 ir_END_list(end_inputs);
7435 } else {
7436 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7437 }
7438 } else if (true_label != (uint32_t)-1) {
7439 ir_END_list(true_inputs);
7440 } else {
7441 ir_END_list(end_inputs);
7442 }
7443 if (if_type) {
7444 ir_IF_FALSE(if_type);
7445 }
7446 }
7447 }
7448
7449 if (op1_info & MAY_BE_LONG) {
7450 ir_ref if_long = IR_UNUSED;
7451 ir_ref ref;
7452
7453 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
7454 if (!type) {
7455 type = jit_Z_TYPE(jit, op1_addr);
7456 }
7457 if_long = ir_IF(ir_EQ(type, ir_CONST_U8(IS_LONG)));
7458 ir_IF_TRUE(if_long);
7459 }
7460 ref = jit_Z_LVAL(jit, op1_addr);
7461 if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
7462 ref = ir_NE(ref, ir_CONST_LONG(0));
7463 if (set_bool_not) {
7464 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7465 ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
7466 } else {
7467 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7468 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
7469 }
7470 ir_END_list(end_inputs);
7471 } else if (exit_addr) {
7472 if (set_bool) {
7473 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7474 ir_ADD_U32(ir_ZEXT_U32(ir_NE(ref, ir_CONST_LONG(0))), ir_CONST_U32(IS_FALSE)));
7475 }
7476 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
7477 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
7478 } else {
7479 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
7480 }
7481 ir_END_list(end_inputs);
7482 } else {
7483 ir_ref if_val = ir_IF(ref);
7484 ir_IF_TRUE(if_val);
7485 if (set_bool) {
7486 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
7487 }
7488 ir_END_list(true_inputs);
7489 ir_IF_FALSE(if_val);
7490 if (set_bool) {
7491 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
7492 }
7493 ir_END_list(false_inputs);
7494 }
7495 if (if_long) {
7496 ir_IF_FALSE(if_long);
7497 }
7498 }
7499
7500 if (op1_info & MAY_BE_DOUBLE) {
7501 ir_ref if_double = IR_UNUSED;
7502 ir_ref ref;
7503
7504 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))) {
7505 if (!type) {
7506 type = jit_Z_TYPE(jit, op1_addr);
7507 }
7508 if_double = ir_IF(ir_EQ(type, ir_CONST_U8(IS_DOUBLE)));
7509 ir_IF_TRUE(if_double);
7510 }
7511 ref = ir_NE(jit_Z_DVAL(jit, op1_addr), ir_CONST_DOUBLE(0.0));
7512 if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
7513 if (set_bool_not) {
7514 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7515 ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
7516 } else {
7517 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7518 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
7519 }
7520 ir_END_list(end_inputs);
7521 } else if (exit_addr) {
7522 if (set_bool) {
7523 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7524 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
7525 }
7526 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
7527 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
7528 } else {
7529 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
7530 }
7531 ir_END_list(end_inputs);
7532 } else {
7533 ir_ref if_val = ir_IF(ref);
7534 ir_IF_TRUE(if_val);
7535 if (set_bool) {
7536 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
7537 }
7538 ir_END_list(true_inputs);
7539 ir_IF_FALSE(if_val);
7540 if (set_bool) {
7541 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
7542 }
7543 ir_END_list(false_inputs);
7544 }
7545 if (if_double) {
7546 ir_IF_FALSE(if_double);
7547 }
7548 }
7549
7550 if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))) {
7551 jit_SET_EX_OPLINE(jit, opline);
7552 ref = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_is_true), jit_ZVAL_ADDR(jit, op1_addr));
7553 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
7554 if (may_throw) {
7555 zend_jit_check_exception_undef_result(jit, opline);
7556 }
7557 if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
7558 if (set_bool_not) {
7559 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7560 ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
7561 } else {
7562 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7563 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
7564 }
7565 if (end_inputs) {
7566 ir_END_list(end_inputs);
7567 }
7568 } else if (exit_addr) {
7569 if (set_bool) {
7570 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7571 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
7572 }
7573 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
7574 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
7575 } else {
7576 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
7577 }
7578 if (end_inputs) {
7579 ir_END_list(end_inputs);
7580 }
7581 } else {
7582 ir_ref if_val = ir_IF(ref);
7583 ir_IF_TRUE(if_val);
7584 if (set_bool) {
7585 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
7586 }
7587 ir_END_list(true_inputs);
7588 ir_IF_FALSE(if_val);
7589 if (set_bool) {
7590 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
7591 }
7592 ir_END_list(false_inputs);
7593 }
7594 }
7595
7596 if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT || exit_addr) {
7597 if (end_inputs) {
7598 ir_MERGE_list(end_inputs);
7599 }
7600 } else {
7601 _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
7602 }
7603
7604 return 1;
7605 }
7606
zend_jit_defined(zend_jit_ctx * jit,const zend_op * opline,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)7607 static int zend_jit_defined(zend_jit_ctx *jit, const zend_op *opline, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
7608 {
7609 uint32_t defined_label = (uint32_t)-1;
7610 uint32_t undefined_label = (uint32_t)-1;
7611 zval *zv = RT_CONSTANT(opline, opline->op1);
7612 zend_jit_addr res_addr = 0;
7613 ir_ref ref, ref2, if_set, if_zero, if_set2;
7614 ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
7615
7616 if (smart_branch_opcode && !exit_addr) {
7617 if (smart_branch_opcode == ZEND_JMPZ) {
7618 defined_label = target_label2;
7619 undefined_label = target_label;
7620 } else if (smart_branch_opcode == ZEND_JMPNZ) {
7621 defined_label = target_label;
7622 undefined_label = target_label2;
7623 } else {
7624 ZEND_UNREACHABLE();
7625 }
7626 } else {
7627 res_addr = RES_ADDR();
7628 }
7629
7630 // if (CACHED_PTR(opline->extended_value)) {
7631 ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value));
7632
7633 if_set = ir_IF(ref);
7634
7635 ir_IF_FALSE_cold(if_set);
7636 if_zero = ir_END();
7637
7638 ir_IF_TRUE(if_set);
7639 if_set2 = ir_IF(ir_AND_A(ref, ir_CONST_ADDR(CACHE_SPECIAL)));
7640 ir_IF_FALSE(if_set2);
7641
7642 if (exit_addr) {
7643 if (smart_branch_opcode == ZEND_JMPNZ) {
7644 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7645 } else {
7646 ir_END_list(end_inputs);
7647 }
7648 } else if (smart_branch_opcode) {
7649 ir_END_list(true_inputs);
7650 } else {
7651 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
7652 ir_END_list(end_inputs);
7653 }
7654
7655 ir_IF_TRUE_cold(if_set2);
7656
7657 ref2 = jit_EG(zend_constants);
7658 ref = ir_SHR_A(ref, ir_CONST_ADDR(1));
7659 if (sizeof(void*) == 8) {
7660 ref = ir_TRUNC_U32(ref);
7661 }
7662 ref2 = ir_EQ(ref, ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(ref2), offsetof(HashTable, nNumOfElements))));
7663 ref2 = ir_IF(ref2);
7664 ir_IF_TRUE(ref2);
7665
7666 if (exit_addr) {
7667 if (smart_branch_opcode == ZEND_JMPZ) {
7668 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7669 } else {
7670 ir_END_list(end_inputs);
7671 }
7672 } else if (smart_branch_opcode) {
7673 ir_END_list(false_inputs);
7674 } else {
7675 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
7676 ir_END_list(end_inputs);
7677 }
7678
7679 ir_IF_FALSE(ref2);
7680 ir_MERGE_2(if_zero, ir_END());
7681
7682 jit_SET_EX_OPLINE(jit, opline);
7683 ref2 = ir_NE(ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_check_constant), ir_CONST_ADDR(zv)), IR_NULL);
7684 if (exit_addr) {
7685 if (smart_branch_opcode == ZEND_JMPZ) {
7686 ir_GUARD(ref2, ir_CONST_ADDR(exit_addr));
7687 } else {
7688 ir_GUARD_NOT(ref2, ir_CONST_ADDR(exit_addr));
7689 }
7690 ir_END_list(end_inputs);
7691 } else if (smart_branch_opcode) {
7692 ref2 = ir_IF(ref2);
7693 ir_IF_TRUE(ref2);
7694 ir_END_list(true_inputs);
7695 ir_IF_FALSE(ref2);
7696 ir_END_list(false_inputs);
7697 } else {
7698 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7699 ir_ADD_U32(ir_ZEXT_U32(ref2), ir_CONST_U32(IS_FALSE)));
7700 ir_END_list(end_inputs);
7701 }
7702
7703 if (!smart_branch_opcode || exit_addr) {
7704 if (end_inputs) {
7705 ir_MERGE_list(end_inputs);
7706 }
7707 } else {
7708 _zend_jit_merge_smart_branch_inputs(jit, defined_label, undefined_label, true_inputs, false_inputs);
7709 }
7710
7711 return 1;
7712 }
7713
zend_jit_escape_if_undef(zend_jit_ctx * jit,int var,uint32_t flags,const zend_op * opline,int8_t reg)7714 static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opline, int8_t reg)
7715 {
7716 zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg));
7717 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, reg_addr));
7718
7719 ir_IF_FALSE_cold(if_def);
7720
7721 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
7722 if (!zend_jit_save_call_chain(jit, -1)) {
7723 return 0;
7724 }
7725 }
7726
7727 if ((opline-1)->opcode != ZEND_FETCH_CONSTANT
7728 && (opline-1)->opcode != ZEND_FETCH_LIST_R
7729 && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR))
7730 && !(flags & ZEND_JIT_EXIT_FREE_OP1)) {
7731 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var);
7732
7733 zend_jit_zval_try_addref(jit, val_addr);
7734 }
7735
7736 jit_LOAD_IP_ADDR(jit, opline - 1);
7737 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape));
7738
7739 ir_IF_TRUE(if_def);
7740
7741 return 1;
7742 }
7743
zend_jit_restore_zval(zend_jit_ctx * jit,int var,int8_t reg)7744 static int zend_jit_restore_zval(zend_jit_ctx *jit, int var, int8_t reg)
7745 {
7746 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
7747 zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg));
7748
7749 // JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
7750 jit_ZVAL_COPY(jit, var_addr, MAY_BE_ANY, reg_addr, MAY_BE_ANY, 1);
7751 return 1;
7752 }
7753
zend_jit_guard_fetch_result_type(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr val_addr,uint8_t type,bool deref,uint32_t flags,bool op1_avoid_refcounting)7754 static zend_jit_addr zend_jit_guard_fetch_result_type(zend_jit_ctx *jit,
7755 const zend_op *opline,
7756 zend_jit_addr val_addr,
7757 uint8_t type,
7758 bool deref,
7759 uint32_t flags,
7760 bool op1_avoid_refcounting)
7761 {
7762 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
7763 int32_t exit_point;
7764 const void *res_exit_addr = NULL;
7765 ir_ref end1 = IR_UNUSED, ref1 = IR_UNUSED;
7766 ir_ref ref = jit_ZVAL_ADDR(jit, val_addr);
7767 uint32_t old_op1_info = 0;
7768 uint32_t old_info;
7769 ir_ref old_ref;
7770
7771
7772 if (opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) {
7773 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
7774 if (op1_avoid_refcounting
7775 || ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
7776 && STACK_FLAGS(stack, EX_VAR_TO_NUM(opline->op1.var)) & (ZREG_ZVAL_ADDREF|ZREG_THIS))) {
7777 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
7778 }
7779 }
7780 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
7781 old_ref = STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
7782 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
7783 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
7784
7785 if (deref) {
7786 ir_ref if_type;
7787
7788 if (type == IS_NULL && (opline->opcode == ZEND_FETCH_DIM_IS || opline->opcode == ZEND_FETCH_OBJ_IS)) {
7789 if_type = ir_IF(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type)));
7790 } else {
7791 if_type = jit_if_Z_TYPE(jit, val_addr, type);
7792 }
7793 ir_IF_TRUE(if_type);
7794 end1 = ir_END();
7795 ref1 = ref;
7796 ir_IF_FALSE_cold(if_type);
7797
7798 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY);
7799 exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
7800 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
7801 if (!res_exit_addr) {
7802 return 0;
7803 }
7804
7805 jit_guard_Z_TYPE(jit, val_addr, IS_REFERENCE, res_exit_addr);
7806 ref = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val));
7807 val_addr = ZEND_ADDR_REF_ZVAL(ref);
7808 }
7809
7810 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY);
7811 exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
7812 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
7813 if (!res_exit_addr) {
7814 return 0;
7815 }
7816
7817 if (!deref && type == IS_NULL && (opline->opcode == ZEND_FETCH_DIM_IS || opline->opcode == ZEND_FETCH_OBJ_IS)) {
7818 ir_GUARD(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type)), ir_CONST_ADDR(res_exit_addr));
7819 } else {
7820 jit_guard_Z_TYPE(jit, val_addr, type, res_exit_addr);
7821 }
7822
7823 if (deref) {
7824 ir_MERGE_WITH(end1);
7825 ref = ir_PHI_2(IR_ADDR, ref, ref1);
7826 }
7827
7828 val_addr = ZEND_ADDR_REF_ZVAL(ref);
7829
7830 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), old_ref);
7831 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
7832 if (opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) {
7833 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
7834 }
7835
7836 return val_addr;
7837 }
7838
zend_jit_fetch_constant(zend_jit_ctx * jit,const zend_op * opline,const zend_op_array * op_array,zend_ssa * ssa,const zend_ssa_op * ssa_op,zend_jit_addr res_addr)7839 static int zend_jit_fetch_constant(zend_jit_ctx *jit,
7840 const zend_op *opline,
7841 const zend_op_array *op_array,
7842 zend_ssa *ssa,
7843 const zend_ssa_op *ssa_op,
7844 zend_jit_addr res_addr)
7845 {
7846 zval *zv = RT_CONSTANT(opline, opline->op2) + 1;
7847 uint32_t res_info = RES_INFO();
7848 ir_ref ref, ref2, if_set, if_special, not_set_path, special_path, fast_path;
7849
7850 // JIT: c = CACHED_PTR(opline->extended_value);
7851 ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value));
7852
7853 // JIT: if (c != NULL)
7854 if_set = ir_IF(ref);
7855
7856 if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) {
7857 // JIT: if (!IS_SPECIAL_CACHE_VAL(c))
7858 ir_IF_FALSE_cold(if_set);
7859 not_set_path = ir_END();
7860 ir_IF_TRUE(if_set);
7861 if_special = ir_IF(ir_AND_A(ref, ir_CONST_ADDR(CACHE_SPECIAL)));
7862 ir_IF_TRUE_cold(if_special);
7863 special_path = ir_END();
7864 ir_IF_FALSE(if_special);
7865 fast_path = ir_END();
7866 ir_MERGE_2(not_set_path, special_path);
7867 } else {
7868 ir_IF_TRUE(if_set);
7869 fast_path = ir_END();
7870 ir_IF_FALSE_cold(if_set);
7871 }
7872
7873 // JIT: zend_jit_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num);
7874 jit_SET_EX_OPLINE(jit, opline);
7875 ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_constant),
7876 ir_CONST_ADDR(zv),
7877 ir_CONST_U32(opline->op1.num));
7878 ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
7879
7880 ir_MERGE_WITH(fast_path);
7881 ref = ir_PHI_2(IR_ADDR, ref2, ref);
7882
7883 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) {
7884 uint8_t type = concrete_type(res_info);
7885 zend_jit_addr const_addr = ZEND_ADDR_REF_ZVAL(ref);
7886
7887 const_addr = zend_jit_guard_fetch_result_type(jit, opline, const_addr, type, 0, 0, 0);
7888 if (!const_addr) {
7889 return 0;
7890 }
7891
7892 res_info &= ~MAY_BE_GUARD;
7893 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
7894
7895 // JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
7896 jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, res_info, 1);
7897 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
7898 return 0;
7899 }
7900 } else {
7901 ir_ref const_addr = ZEND_ADDR_REF_ZVAL(ref);
7902
7903 // JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
7904 jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, 1);
7905 }
7906
7907
7908 return 1;
7909 }
7910
zend_jit_type_check(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)7911 static int zend_jit_type_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
7912 {
7913 uint32_t mask;
7914 zend_jit_addr op1_addr = OP1_ADDR();
7915 zend_jit_addr res_addr = 0;
7916 uint32_t true_label = -1, false_label = -1;
7917 ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
7918
7919 // TODO: support for is_resource() ???
7920 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);
7921
7922 if (smart_branch_opcode && !exit_addr) {
7923 if (smart_branch_opcode == ZEND_JMPZ) {
7924 true_label = target_label2;
7925 false_label = target_label;
7926 } else if (smart_branch_opcode == ZEND_JMPNZ) {
7927 true_label = target_label;
7928 false_label = target_label2;
7929 } else {
7930 ZEND_UNREACHABLE();
7931 }
7932 } else {
7933 res_addr = RES_ADDR();
7934 }
7935
7936 if (op1_info & MAY_BE_UNDEF) {
7937 ir_ref if_def = IR_UNUSED;
7938
7939 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
7940 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
7941 ir_IF_FALSE_cold(if_def);
7942 }
7943
7944 jit_SET_EX_OPLINE(jit, opline);
7945 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
7946 zend_jit_check_exception_undef_result(jit, opline);
7947 if (opline->extended_value & MAY_BE_NULL) {
7948 if (exit_addr) {
7949 if (smart_branch_opcode == ZEND_JMPNZ) {
7950 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7951 } else {
7952 ir_END_list(end_inputs);
7953 }
7954 } else if (smart_branch_opcode) {
7955 ir_END_list(true_inputs);
7956 } else {
7957 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
7958 ir_END_list(end_inputs);
7959 }
7960 } else {
7961 if (exit_addr) {
7962 if (smart_branch_opcode == ZEND_JMPZ) {
7963 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7964 } else {
7965 ir_END_list(end_inputs);
7966 }
7967 } else if (smart_branch_opcode) {
7968 ir_END_list(false_inputs);
7969 } else {
7970 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
7971 if (if_def) {
7972 ir_END_list(end_inputs);
7973 }
7974 }
7975 }
7976
7977 if (if_def) {
7978 ir_IF_TRUE(if_def);
7979 op1_info |= MAY_BE_NULL;
7980 }
7981 }
7982
7983 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
7984 mask = opline->extended_value;
7985 if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) {
7986 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
7987 if (exit_addr) {
7988 if (smart_branch_opcode == ZEND_JMPNZ) {
7989 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7990 } else if (end_inputs) {
7991 ir_END_list(end_inputs);
7992 }
7993 } else if (smart_branch_opcode) {
7994 ir_END_list(true_inputs);
7995 } else {
7996 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
7997 ir_END_list(end_inputs);
7998 }
7999 } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) {
8000 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
8001 if (exit_addr) {
8002 if (smart_branch_opcode == ZEND_JMPZ) {
8003 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
8004 } else if (end_inputs) {
8005 ir_END_list(end_inputs);
8006 }
8007 } else if (smart_branch_opcode) {
8008 ir_END_list(false_inputs);
8009 } else {
8010 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
8011 ir_END_list(end_inputs);
8012 }
8013 } else {
8014 ir_ref ref;
8015 bool invert = 0;
8016 uint8_t type;
8017
8018 switch (mask) {
8019 case MAY_BE_NULL: type = IS_NULL; break;
8020 case MAY_BE_FALSE: type = IS_FALSE; break;
8021 case MAY_BE_TRUE: type = IS_TRUE; break;
8022 case MAY_BE_LONG: type = IS_LONG; break;
8023 case MAY_BE_DOUBLE: type = IS_DOUBLE; break;
8024 case MAY_BE_STRING: type = IS_STRING; break;
8025 case MAY_BE_ARRAY: type = IS_ARRAY; break;
8026 case MAY_BE_OBJECT: type = IS_OBJECT; break;
8027 case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break;
8028 case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break;
8029 case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break;
8030 case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break;
8031 case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break;
8032 case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break;
8033 case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break;
8034 case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break;
8035 case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break;
8036 default:
8037 type = 0;
8038 }
8039
8040 if (op1_info & MAY_BE_REF) {
8041 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
8042 ref = jit_ZVAL_DEREF_ref(jit, ref);
8043 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
8044 }
8045 if (type == 0) {
8046 ref = ir_AND_U32(ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)), ir_CONST_U32(mask));
8047 if (!smart_branch_opcode) {
8048 ref = ir_NE(ref, ir_CONST_U32(0));
8049 }
8050 } else if (invert) {
8051 ref = ir_NE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type));
8052 } else {
8053 ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type));
8054 }
8055
8056 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
8057
8058 if (exit_addr) {
8059 if (smart_branch_opcode == ZEND_JMPZ) {
8060 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
8061 } else {
8062 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
8063 }
8064 if (end_inputs) {
8065 ir_END_list(end_inputs);
8066 }
8067 } else if (smart_branch_opcode) {
8068 ir_ref if_val = ir_IF(ref);
8069 ir_IF_TRUE(if_val);
8070 ir_END_list(true_inputs);
8071 ir_IF_FALSE(if_val);
8072 ir_END_list(false_inputs);
8073 } else {
8074 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
8075 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
8076 ir_END_list(end_inputs);
8077 }
8078 }
8079 }
8080
8081 if (!smart_branch_opcode || exit_addr) {
8082 if (end_inputs) {
8083 ir_MERGE_list(end_inputs);
8084 }
8085 } else {
8086 _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
8087 }
8088
8089 return 1;
8090 }
8091
zend_jit_isset_isempty_cv(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_addr,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)8092 static int zend_jit_isset_isempty_cv(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
8093 {
8094 zend_jit_addr res_addr = RES_ADDR();
8095 uint32_t true_label = -1, false_label = -1;
8096 ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
8097
8098 // TODO: support for empty() ???
8099 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);
8100
8101 if (smart_branch_opcode && !exit_addr) {
8102 if (smart_branch_opcode == ZEND_JMPZ) {
8103 true_label = target_label2;
8104 false_label = target_label;
8105 } else if (smart_branch_opcode == ZEND_JMPNZ) {
8106 true_label = target_label;
8107 false_label = target_label2;
8108 } else {
8109 ZEND_UNREACHABLE();
8110 }
8111 } else {
8112 res_addr = RES_ADDR();
8113 }
8114
8115 if (op1_info & MAY_BE_REF) {
8116 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
8117 ref = jit_ZVAL_DEREF_ref(jit, ref);
8118 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
8119 }
8120
8121 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
8122 if (exit_addr) {
8123 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ);
8124 } else if (smart_branch_opcode) {
8125 ir_END_list(true_inputs);
8126 } else {
8127 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
8128 ir_END_list(end_inputs);
8129 }
8130 } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) {
8131 if (exit_addr) {
8132 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ);
8133 } else if (smart_branch_opcode) {
8134 ir_END_list(false_inputs);
8135 } else {
8136 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
8137 ir_END_list(end_inputs);
8138 }
8139 } else {
8140 ir_ref ref = ir_GT(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL));
8141 if (exit_addr) {
8142 if (smart_branch_opcode == ZEND_JMPNZ) {
8143 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
8144 } else {
8145 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
8146 }
8147 } else if (smart_branch_opcode) {
8148 ir_ref if_val = ir_IF(ref);
8149 ir_IF_TRUE(if_val);
8150 ir_END_list(true_inputs);
8151 ir_IF_FALSE(if_val);
8152 ir_END_list(false_inputs);
8153 } else {
8154 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
8155 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
8156 ir_END_list(end_inputs);
8157 }
8158 }
8159
8160 if (!smart_branch_opcode || exit_addr) {
8161 if (end_inputs) {
8162 ir_MERGE_list(end_inputs);
8163 }
8164 } else {
8165 _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
8166 }
8167
8168 return 1;
8169 }
8170
8171 /* copy of hidden zend_closure */
8172 typedef struct _zend_closure {
8173 zend_object std;
8174 zend_function func;
8175 zval this_ptr;
8176 zend_class_entry *called_scope;
8177 zif_handler orig_internal_handler;
8178 } zend_closure;
8179
zend_jit_stack_check(zend_jit_ctx * jit,const zend_op * opline,uint32_t used_stack)8180 static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t used_stack)
8181 {
8182 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
8183 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8184
8185 if (!exit_addr) {
8186 return 0;
8187 }
8188
8189 // JIT: if (EG(vm_stack_end) - EG(vm_stack_top) < used_stack)
8190 ir_GUARD(
8191 ir_UGE(
8192 ir_SUB_A(ir_LOAD_A(jit_EG(vm_stack_end)), ir_LOAD_A(jit_EG(vm_stack_top))),
8193 ir_CONST_ADDR(used_stack)),
8194 ir_CONST_ADDR(exit_addr));
8195
8196 return 1;
8197 }
8198
zend_jit_free_trampoline(zend_jit_ctx * jit,int8_t func_reg)8199 static int zend_jit_free_trampoline(zend_jit_ctx *jit, int8_t func_reg)
8200 {
8201 // JIT: if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
8202 ir_ref func = ir_RLOAD_A(func_reg);
8203 ir_ref if_trampoline = ir_IF(ir_AND_U32(
8204 ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, common.fn_flags))),
8205 ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE)));
8206
8207 ir_IF_TRUE(if_trampoline);
8208 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_free_trampoline_helper), func);
8209 ir_MERGE_WITH_EMPTY_FALSE(if_trampoline);
8210
8211 return 1;
8212 }
8213
zend_jit_push_call_frame(zend_jit_ctx * jit,const zend_op * opline,const zend_op_array * op_array,zend_function * func,bool is_closure,bool delayed_fetch_this,int checked_stack,ir_ref func_ref,ir_ref this_ref)8214 static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack, ir_ref func_ref, ir_ref this_ref)
8215 {
8216 uint32_t used_stack;
8217 ir_ref used_stack_ref = IR_UNUSED;
8218 bool stack_check = 1;
8219 ir_ref rx, ref, top, if_enough_stack, cold_path = IR_UNUSED;
8220
8221 ZEND_ASSERT(func_ref != IR_NULL);
8222 if (func) {
8223 used_stack = zend_vm_calc_used_stack(opline->extended_value, func);
8224 if ((int)used_stack <= checked_stack) {
8225 stack_check = 0;
8226 }
8227 used_stack_ref = ir_CONST_ADDR(used_stack);
8228 } else {
8229 ir_ref num_args_ref;
8230 ir_ref if_internal_func = IR_UNUSED;
8231
8232 used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value + ZEND_OBSERVER_ENABLED) * sizeof(zval);
8233 used_stack_ref = ir_CONST_ADDR(used_stack);
8234
8235 if (!is_closure) {
8236 used_stack_ref = ir_HARD_COPY_A(used_stack_ref); /* load constant once */
8237
8238 // JIT: if (EXPECTED(ZEND_USER_CODE(func->type))) {
8239 ir_ref tmp = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type)));
8240 if_internal_func = ir_IF(ir_AND_U8(tmp, ir_CONST_U8(1)));
8241 ir_IF_FALSE(if_internal_func);
8242 }
8243
8244 // JIT: used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval);
8245 num_args_ref = ir_CONST_U32(opline->extended_value);
8246 if (!is_closure) {
8247 ref = ir_SUB_U32(
8248 ir_SUB_U32(
8249 ir_MIN_U32(
8250 num_args_ref,
8251 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.num_args)))),
8252 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.last_var)))),
8253 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.T))));
8254 } else {
8255 ref = ir_SUB_U32(
8256 ir_SUB_U32(
8257 ir_MIN_U32(
8258 num_args_ref,
8259 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.num_args)))),
8260 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.last_var)))),
8261 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.T))));
8262 }
8263 ref = ir_MUL_U32(ref, ir_CONST_U32(sizeof(zval)));
8264 if (sizeof(void*) == 8) {
8265 ref = ir_SEXT_A(ref);
8266 }
8267 ref = ir_SUB_A(used_stack_ref, ref);
8268
8269 if (is_closure) {
8270 used_stack_ref = ref;
8271 } else {
8272 ir_MERGE_WITH_EMPTY_TRUE(if_internal_func);
8273 used_stack_ref = ir_PHI_2(IR_ADDR, ref, used_stack_ref);
8274 }
8275 }
8276
8277 zend_jit_start_reuse_ip(jit);
8278
8279 // JIT: if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) {
8280 jit_STORE_IP(jit, ir_LOAD_A(jit_EG(vm_stack_top)));
8281
8282 if (stack_check) {
8283 // JIT: Check Stack Overflow
8284 ref = ir_UGE(
8285 ir_SUB_A(
8286 ir_LOAD_A(jit_EG(vm_stack_end)),
8287 jit_IP(jit)),
8288 used_stack_ref);
8289
8290 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
8291 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
8292 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8293
8294 if (!exit_addr) {
8295 return 0;
8296 }
8297
8298 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
8299 } else {
8300 if_enough_stack = ir_IF(ref);
8301 ir_IF_FALSE_cold(if_enough_stack);
8302
8303 #ifdef _WIN32
8304 if (0) {
8305 #else
8306 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
8307 #endif
8308 jit_SET_EX_OPLINE(jit, opline);
8309 ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_int_extend_stack_helper), used_stack_ref);
8310 } else {
8311 if (!is_closure) {
8312 ref = func_ref;
8313 } else {
8314 ref = ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func));
8315 }
8316 jit_SET_EX_OPLINE(jit, opline);
8317 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_extend_stack_helper),
8318 used_stack_ref, ref);
8319 }
8320 jit_STORE_IP(jit, ref);
8321
8322 cold_path = ir_END();
8323 ir_IF_TRUE(if_enough_stack);
8324 }
8325 }
8326
8327 ref = jit_EG(vm_stack_top);
8328 rx = jit_IP(jit);
8329 #if !OPTIMIZE_FOR_SIZE
8330 /* JIT: EG(vm_stack_top) = (zval*)((char*)call + used_stack);
8331 * This vesions is longer but faster
8332 * mov EG(vm_stack_top), %CALL
8333 * lea size(%call), %tmp
8334 * mov %tmp, EG(vm_stack_top)
8335 */
8336 top = rx;
8337 #else
8338 /* JIT: EG(vm_stack_top) += used_stack;
8339 * Use ir_emit() because ir_LOAD() makes load forwarding and doesn't allow load/store fusion
8340 * mov EG(vm_stack_top), %CALL
8341 * add $size, EG(vm_stack_top)
8342 */
8343 top = jit->ctx.control = ir_emit2(&jit->ctx, IR_OPT(IR_LOAD, IR_ADDR), jit->ctx.control, ref);
8344 #endif
8345 ir_STORE(ref, ir_ADD_A(top, used_stack_ref));
8346
8347 // JIT: zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object);
8348 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) {
8349 // JIT: ZEND_SET_CALL_INFO(call, 0, call_info);
8350 ir_STORE(jit_CALL(rx, This.u1.type_info), ir_CONST_U32(IS_UNDEF | ZEND_CALL_NESTED_FUNCTION));
8351 }
8352 #ifdef _WIN32
8353 if (0) {
8354 #else
8355 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
8356 #endif
8357 if (cold_path) {
8358 ir_MERGE_WITH(cold_path);
8359 rx = jit_IP(jit);
8360 }
8361
8362 // JIT: call->func = func;
8363 ir_STORE(jit_CALL(rx, func), func_ref);
8364 } else {
8365 if (!is_closure) {
8366 // JIT: call->func = func;
8367 ir_STORE(jit_CALL(rx, func), func_ref);
8368 } else {
8369 // JIT: call->func = &closure->func;
8370 ir_STORE(jit_CALL(rx, func), ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func)));
8371 }
8372 if (cold_path) {
8373 ir_MERGE_WITH(cold_path);
8374 rx = jit_IP(jit);
8375 }
8376 }
8377 if (opline->opcode == ZEND_INIT_METHOD_CALL) {
8378 // JIT: Z_PTR(call->This) = obj;
8379 ZEND_ASSERT(this_ref != IR_NULL);
8380 ir_STORE(jit_CALL(rx, This.value.ptr), this_ref);
8381 if (opline->op1_type == IS_UNUSED || delayed_fetch_this) {
8382 // JIT: call->call_info |= ZEND_CALL_HAS_THIS;
8383 ref = jit_CALL(rx, This.u1.type_info);
8384 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
8385 ir_STORE(ref, ir_CONST_U32( ZEND_CALL_HAS_THIS));
8386 } else {
8387 ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_HAS_THIS)));
8388 }
8389 } else {
8390 if (opline->op1_type == IS_CV) {
8391 // JIT: GC_ADDREF(obj);
8392 jit_GC_ADDREF(jit, this_ref);
8393 }
8394
8395 // JIT: call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS;
8396 ref = jit_CALL(rx, This.u1.type_info);
8397 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
8398 ir_STORE(ref, ir_CONST_U32( ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS));
8399 } else {
8400 ir_STORE(ref,
8401 ir_OR_U32(ir_LOAD_U32(ref),
8402 ir_CONST_U32(ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS)));
8403 }
8404 }
8405 } else if (!is_closure) {
8406 // JIT: Z_CE(call->This) = called_scope;
8407 ir_STORE(jit_CALL(rx, This), IR_NULL);
8408 } else {
8409 ir_ref object_or_called_scope, call_info, call_info2, object, if_cond;
8410
8411 if (opline->op2_type == IS_CV) {
8412 // JIT: GC_ADDREF(closure);
8413 jit_GC_ADDREF(jit, func_ref);
8414 }
8415
8416 // JIT: RX(object_or_called_scope) = closure->called_scope;
8417 object_or_called_scope = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, called_scope)));
8418
8419 // JIT: call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE |
8420 // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE);
8421 call_info = ir_OR_U32(
8422 ir_AND_U32(
8423 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.common.fn_flags))),
8424 ir_CONST_U32(ZEND_ACC_FAKE_CLOSURE)),
8425 ir_CONST_U32(ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE));
8426 // JIT: if (Z_TYPE(closure->this_ptr) != IS_UNDEF) {
8427 if_cond = ir_IF(ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, this_ptr.u1.v.type))));
8428 ir_IF_TRUE(if_cond);
8429
8430 // JIT: call_info |= ZEND_CALL_HAS_THIS;
8431 call_info2 = ir_OR_U32(call_info, ir_CONST_U32(ZEND_CALL_HAS_THIS));
8432
8433 // JIT: object_or_called_scope = Z_OBJ(closure->this_ptr);
8434 object = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, this_ptr.value.ptr)));
8435
8436 ir_MERGE_WITH_EMPTY_FALSE(if_cond);
8437 call_info = ir_PHI_2(IR_U32, call_info2, call_info);
8438 object_or_called_scope = ir_PHI_2(IR_ADDR, object, object_or_called_scope);
8439
8440 // JIT: ZEND_SET_CALL_INFO(call, 0, call_info);
8441 ref = jit_CALL(rx, This.u1.type_info);
8442 ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), call_info));
8443
8444 // JIT: Z_PTR(call->This) = object_or_called_scope;
8445 ir_STORE(jit_CALL(rx, This.value.ptr), object_or_called_scope);
8446
8447 // JIT: if (closure->func.op_array.run_time_cache__ptr)
8448 if_cond = ir_IF(ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.run_time_cache__ptr))));
8449 ir_IF_FALSE(if_cond);
8450
8451 // JIT: zend_jit_init_func_run_time_cache_helper(closure->func);
8452 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_init_func_run_time_cache_helper),
8453 ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func)));
8454
8455 ir_MERGE_WITH_EMPTY_TRUE(if_cond);
8456 }
8457
8458 // JIT: ZEND_CALL_NUM_ARGS(call) = num_args;
8459 ir_STORE(jit_CALL(rx, This.u2.num_args), ir_CONST_U32(opline->extended_value));
8460
8461 return 1;
8462 }
8463
8464 static int zend_jit_init_fcall_guard(zend_jit_ctx *jit, uint32_t level, const zend_function *func, const zend_op *to_opline)
8465 {
8466 int32_t exit_point;
8467 const void *exit_addr;
8468 ir_ref call;
8469
8470 if (func->type == ZEND_INTERNAL_FUNCTION) {
8471 #ifdef ZEND_WIN32
8472 // TODO: ASLR may cause different addresses in different workers ???
8473 return 0;
8474 #endif
8475 } else if (func->type == ZEND_USER_FUNCTION) {
8476 if (!zend_accel_in_shm(func->op_array.opcodes)) {
8477 /* op_array and op_array->opcodes are not persistent. We can't link. */
8478 return 0;
8479 }
8480 } else {
8481 ZEND_UNREACHABLE();
8482 return 0;
8483 }
8484
8485 exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM);
8486 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8487 if (!exit_addr) {
8488 return 0;
8489 }
8490
8491 // call = EX(call);
8492 call = ir_LOAD_A(jit_EX(call));
8493 while (level > 0) {
8494 // call = call->prev_execute_data
8495 call = ir_LOAD_A(jit_CALL(call, prev_execute_data));
8496 level--;
8497 }
8498
8499 if (func->type == ZEND_USER_FUNCTION &&
8500 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
8501 (func->common.fn_flags & ZEND_ACC_CLOSURE) ||
8502 !func->common.function_name)) {
8503 const zend_op *opcodes = func->op_array.opcodes;
8504
8505 // JIT: if (call->func.op_array.opcodes != opcodes) goto exit_addr;
8506 ir_GUARD(
8507 ir_EQ(
8508 ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(call, func)), offsetof(zend_op_array, opcodes))),
8509 ir_CONST_ADDR(opcodes)),
8510 ir_CONST_ADDR(exit_addr));
8511 } else {
8512 // JIT: if (call->func != func) goto exit_addr;
8513 ir_GUARD(ir_EQ(ir_LOAD_A(jit_CALL(call, func)), ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
8514 }
8515
8516 return 1;
8517 }
8518
8519 static int zend_jit_init_fcall(zend_jit_ctx *jit, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack)
8520 {
8521 zend_func_info *info = ZEND_FUNC_INFO(op_array);
8522 zend_call_info *call_info = NULL;
8523 zend_function *func = NULL;
8524 ir_ref func_ref = IR_UNUSED;
8525
8526 if (jit->delayed_call_level) {
8527 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
8528 return 0;
8529 }
8530 }
8531
8532 if (info) {
8533 call_info = info->callee_info;
8534 while (call_info && call_info->caller_init_opline != opline) {
8535 call_info = call_info->next_callee;
8536 }
8537 if (call_info && call_info->callee_func && !call_info->is_prototype) {
8538 func = call_info->callee_func;
8539 }
8540 }
8541
8542 if (!func
8543 && trace
8544 && trace->op == ZEND_JIT_TRACE_INIT_CALL) {
8545 #ifdef _WIN32
8546 /* ASLR */
8547 if (trace->func->type != ZEND_INTERNAL_FUNCTION) {
8548 func = (zend_function*)trace->func;
8549 }
8550 #else
8551 func = (zend_function*)trace->func;
8552 #endif
8553 }
8554
8555 #ifdef _WIN32
8556 if (0) {
8557 #else
8558 if (opline->opcode == ZEND_INIT_FCALL
8559 && func
8560 && func->type == ZEND_INTERNAL_FUNCTION) {
8561 #endif
8562 /* load constant address later */
8563 func_ref = ir_CONST_ADDR(func);
8564 } else if (func && op_array == &func->op_array) {
8565 /* recursive call */
8566 if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)
8567 || zend_jit_prefer_const_addr_load(jit, (uintptr_t)func)) {
8568 func_ref = ir_LOAD_A(jit_EX(func));
8569 } else {
8570 func_ref = ir_CONST_ADDR(func);
8571 }
8572 } else {
8573 ir_ref if_func, cache_slot_ref, ref;
8574
8575 // JIT: if (CACHED_PTR(opline->result.num))
8576 cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num);
8577 func_ref = ir_LOAD_A(cache_slot_ref);
8578 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
8579 && func
8580 && (func->common.fn_flags & ZEND_ACC_IMMUTABLE)
8581 && opline->opcode != ZEND_INIT_FCALL) {
8582 /* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */
8583 if_func = ir_IF(ir_EQ(func_ref, ir_CONST_ADDR(func)));
8584 } else {
8585 if_func = ir_IF(func_ref);
8586 }
8587 ir_IF_FALSE_cold(if_func);
8588 if (opline->opcode == ZEND_INIT_FCALL
8589 && func
8590 && func->type == ZEND_USER_FUNCTION
8591 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) {
8592 ref = ir_HARD_COPY_A(ir_CONST_ADDR(func)); /* load constant once */
8593 ir_STORE(cache_slot_ref, ref);
8594 ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_init_func_run_time_cache_helper), ref);
8595 } else {
8596 zval *zv = RT_CONSTANT(opline, opline->op2);
8597
8598 if (opline->opcode == ZEND_INIT_FCALL) {
8599 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_func_helper),
8600 ir_CONST_ADDR(Z_STR_P(zv)),
8601 cache_slot_ref);
8602 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) {
8603 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_func_helper),
8604 ir_CONST_ADDR(Z_STR_P(zv + 1)),
8605 cache_slot_ref);
8606 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) {
8607 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_ns_func_helper),
8608 ir_CONST_ADDR(zv),
8609 cache_slot_ref);
8610 } else {
8611 ZEND_UNREACHABLE();
8612 }
8613 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
8614 int32_t exit_point = zend_jit_trace_get_exit_point(opline,
8615 func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0);
8616 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8617
8618 if (!exit_addr) {
8619 return 0;
8620 }
8621 if (!func || opline->opcode == ZEND_INIT_FCALL) {
8622 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
8623 } else if (func->type == ZEND_USER_FUNCTION
8624 && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) {
8625 const zend_op *opcodes = func->op_array.opcodes;
8626
8627 ir_GUARD(
8628 ir_EQ(
8629 ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_op_array, opcodes))),
8630 ir_CONST_ADDR(opcodes)),
8631 ir_CONST_ADDR(exit_addr));
8632 } else {
8633 ir_GUARD(ir_EQ(ref, ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
8634 }
8635 } else {
8636 jit_SET_EX_OPLINE(jit, opline);
8637 ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_undefined_function));
8638 }
8639 }
8640 ir_MERGE_WITH_EMPTY_TRUE(if_func);
8641 func_ref = ir_PHI_2(IR_ADDR, ref, func_ref);
8642 }
8643
8644 if (!zend_jit_push_call_frame(jit, opline, op_array, func, 0, 0, checked_stack, func_ref, IR_UNUSED)) {
8645 return 0;
8646 }
8647
8648 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
8649 if (!zend_jit_save_call_chain(jit, call_level)) {
8650 return 0;
8651 }
8652 } else {
8653 ZEND_ASSERT(call_level > 0);
8654 jit->delayed_call_level = call_level;
8655 delayed_call_chain = 1;
8656 }
8657
8658 return 1;
8659 }
8660
8661 static int zend_jit_init_method_call(zend_jit_ctx *jit,
8662 const zend_op *opline,
8663 uint32_t b,
8664 const zend_op_array *op_array,
8665 zend_ssa *ssa,
8666 const zend_ssa_op *ssa_op,
8667 int call_level,
8668 uint32_t op1_info,
8669 zend_jit_addr op1_addr,
8670 zend_class_entry *ce,
8671 bool ce_is_instanceof,
8672 bool on_this,
8673 bool delayed_fetch_this,
8674 zend_class_entry *trace_ce,
8675 zend_jit_trace_rec *trace,
8676 int checked_stack,
8677 int8_t func_reg,
8678 int8_t this_reg,
8679 bool polymorphic_side_trace)
8680 {
8681 zend_func_info *info = ZEND_FUNC_INFO(op_array);
8682 zend_call_info *call_info = NULL;
8683 zend_function *func = NULL;
8684 zval *function_name;
8685 ir_ref if_static = IR_UNUSED, cold_path, this_ref = IR_NULL, func_ref = IR_NULL;
8686
8687 ZEND_ASSERT(opline->op2_type == IS_CONST);
8688 ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
8689
8690 function_name = RT_CONSTANT(opline, opline->op2);
8691
8692 if (info) {
8693 call_info = info->callee_info;
8694 while (call_info && call_info->caller_init_opline != opline) {
8695 call_info = call_info->next_callee;
8696 }
8697 if (call_info && call_info->callee_func && !call_info->is_prototype) {
8698 func = call_info->callee_func;
8699 }
8700 }
8701
8702 if (polymorphic_side_trace) {
8703 /* function is passed in r0 from parent_trace */
8704 ZEND_ASSERT(func_reg >= 0 && this_reg >= 0);
8705 func_ref = zend_jit_deopt_rload(jit, IR_ADDR, func_reg);
8706 this_ref = zend_jit_deopt_rload(jit, IR_ADDR, this_reg);
8707 } else {
8708 ir_ref ref, ref2, if_found, fast_path, run_time_cache, this_ref2;
8709
8710 if (on_this) {
8711 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
8712 this_ref = jit_Z_PTR(jit, this_addr);
8713 } else {
8714 if (op1_info & MAY_BE_REF) {
8715 if (opline->op1_type == IS_CV) {
8716 // JIT: ZVAL_DEREF(op1)
8717 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
8718 ref = jit_ZVAL_DEREF_ref(jit, ref);
8719 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
8720 } else {
8721 ir_ref if_ref;
8722
8723 /* Hack: Convert reference to regular value to simplify JIT code */
8724 ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP);
8725
8726 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
8727 ir_IF_TRUE(if_ref);
8728 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper), jit_ZVAL_ADDR(jit, op1_addr));
8729
8730 ir_MERGE_WITH_EMPTY_FALSE(if_ref);
8731 }
8732 }
8733 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
8734 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
8735 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
8736 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8737
8738 if (!exit_addr) {
8739 return 0;
8740 }
8741 ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_OBJECT)),
8742 ir_CONST_ADDR(exit_addr));
8743 } else {
8744 ir_ref if_object = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
8745
8746 ir_IF_FALSE_cold(if_object);
8747
8748 jit_SET_EX_OPLINE(jit, opline);
8749 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
8750 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_method_call_tmp),
8751 jit_ZVAL_ADDR(jit, op1_addr));
8752 } else {
8753 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_method_call),
8754 jit_ZVAL_ADDR(jit, op1_addr));
8755 }
8756 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
8757 ir_IF_TRUE(if_object);
8758 }
8759 }
8760
8761 this_ref = jit_Z_PTR(jit, op1_addr);
8762 }
8763
8764 if (jit->delayed_call_level) {
8765 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
8766 return 0;
8767 }
8768 }
8769
8770 if (func) {
8771 // JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*));
8772 ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num + sizeof(void*)));
8773
8774 if_found = ir_IF(ref);
8775 ir_IF_TRUE(if_found);
8776 fast_path = ir_END();
8777 } else {
8778 // JIT: if (CACHED_PTR(opline->result.num) == obj->ce)) {
8779 run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
8780 ref = ir_EQ(
8781 ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->result.num)),
8782 ir_LOAD_A(ir_ADD_OFFSET(this_ref, offsetof(zend_object, ce))));
8783 if_found = ir_IF(ref);
8784 ir_IF_TRUE(if_found);
8785
8786 // JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*));
8787 ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->result.num + sizeof(void*)));
8788 fast_path = ir_END();
8789
8790 }
8791
8792 ir_IF_FALSE_cold(if_found);
8793 jit_SET_EX_OPLINE(jit, opline);
8794
8795 if (!jit->ctx.fixed_call_stack_size) {
8796 // JIT: alloca(sizeof(void*));
8797 this_ref2 = ir_ALLOCA(ir_CONST_ADDR(0x10));
8798 } else {
8799 this_ref2 = ir_HARD_COPY_A(ir_RLOAD_A(IR_REG_SP));
8800 }
8801 ir_STORE(this_ref2, this_ref);
8802
8803 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
8804 ref2 = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_method_tmp_helper),
8805 this_ref,
8806 ir_CONST_ADDR(function_name),
8807 this_ref2);
8808 } else {
8809 ref2 = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_method_helper),
8810 this_ref,
8811 ir_CONST_ADDR(function_name),
8812 this_ref2);
8813 }
8814
8815 this_ref2 = ir_LOAD_A(ir_RLOAD_A(IR_REG_SP));
8816 if (!jit->ctx.fixed_call_stack_size) {
8817 // JIT: revert alloca
8818 ir_AFREE(ir_CONST_ADDR(0x10));
8819 }
8820
8821 ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
8822
8823 ir_MERGE_WITH(fast_path);
8824 func_ref = ir_PHI_2(IR_ADDR, ref2, ref);
8825 this_ref = ir_PHI_2(IR_ADDR, this_ref2, this_ref);
8826 }
8827
8828 if ((!func || zend_jit_may_be_modified(func, op_array))
8829 && trace
8830 && trace->op == ZEND_JIT_TRACE_INIT_CALL
8831 && trace->func
8832 #ifdef _WIN32
8833 && trace->func->type != ZEND_INTERNAL_FUNCTION
8834 #endif
8835 ) {
8836 int32_t exit_point;
8837 const void *exit_addr;
8838
8839 exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL);
8840 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8841 if (!exit_addr) {
8842 return 0;
8843 }
8844
8845 jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
8846 jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
8847
8848 func = (zend_function*)trace->func;
8849
8850 if (func->type == ZEND_USER_FUNCTION &&
8851 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
8852 (func->common.fn_flags & ZEND_ACC_CLOSURE) ||
8853 !func->common.function_name)) {
8854 const zend_op *opcodes = func->op_array.opcodes;
8855
8856 ir_GUARD(
8857 ir_EQ(
8858 ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes))),
8859 ir_CONST_ADDR(opcodes)),
8860 ir_CONST_ADDR(exit_addr));
8861 } else {
8862 ir_GUARD(ir_EQ(func_ref, ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
8863 }
8864 }
8865
8866 if (!func) {
8867 // JIT: if (fbc->common.fn_flags & ZEND_ACC_STATIC) {
8868 if_static = ir_IF(ir_AND_U32(
8869 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, common.fn_flags))),
8870 ir_CONST_U32(ZEND_ACC_STATIC)));
8871 ir_IF_TRUE_cold(if_static);
8872 }
8873
8874 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) {
8875 ir_ref ret;
8876
8877 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
8878 ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_metod_call_frame_tmp),
8879 this_ref,
8880 func_ref,
8881 ir_CONST_U32(opline->extended_value));
8882 } else {
8883 ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_metod_call_frame),
8884 this_ref,
8885 func_ref,
8886 ir_CONST_U32(opline->extended_value));
8887 }
8888
8889 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) {
8890 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
8891 }
8892 jit_STORE_IP(jit, ret);
8893 }
8894
8895 if (!func) {
8896 cold_path = ir_END();
8897 ir_IF_FALSE(if_static);
8898 }
8899
8900 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) {
8901 if (!zend_jit_push_call_frame(jit, opline, NULL, func, 0, delayed_fetch_this, checked_stack, func_ref, this_ref)) {
8902 return 0;
8903 }
8904 }
8905
8906 if (!func) {
8907 ir_MERGE_WITH(cold_path);
8908 }
8909 zend_jit_start_reuse_ip(jit);
8910
8911 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
8912 if (!zend_jit_save_call_chain(jit, call_level)) {
8913 return 0;
8914 }
8915 } else {
8916 ZEND_ASSERT(call_level > 0);
8917 delayed_call_chain = 1;
8918 jit->delayed_call_level = call_level;
8919 }
8920
8921 return 1;
8922 }
8923
8924 static int zend_jit_init_closure_call(zend_jit_ctx *jit,
8925 const zend_op *opline,
8926 uint32_t b,
8927 const zend_op_array *op_array,
8928 zend_ssa *ssa,
8929 const zend_ssa_op *ssa_op,
8930 int call_level,
8931 zend_jit_trace_rec *trace,
8932 int checked_stack)
8933 {
8934 zend_function *func = NULL;
8935 zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
8936 ir_ref ref;
8937
8938 ref = jit_Z_PTR(jit, op2_addr);
8939
8940 if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure
8941 && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) {
8942 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
8943 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8944
8945 if (!exit_addr) {
8946 return 0;
8947 }
8948
8949 ir_GUARD(
8950 ir_EQ(
8951 ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_object, ce))),
8952 ir_CONST_ADDR(zend_ce_closure)),
8953 ir_CONST_ADDR(exit_addr));
8954
8955 if (ssa->var_info && ssa_op->op2_use >= 0) {
8956 ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD;
8957 ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure;
8958 ssa->var_info[ssa_op->op2_use].is_instanceof = 0;
8959 }
8960 }
8961
8962 if (trace
8963 && trace->op == ZEND_JIT_TRACE_INIT_CALL
8964 && trace->func
8965 && trace->func->type == ZEND_USER_FUNCTION) {
8966 const zend_op *opcodes;
8967 int32_t exit_point;
8968 const void *exit_addr;
8969
8970 func = (zend_function*)trace->func;
8971 opcodes = func->op_array.opcodes;
8972 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL);
8973 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8974 if (!exit_addr) {
8975 return 0;
8976 }
8977
8978 ir_GUARD(
8979 ir_EQ(
8980 ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_closure, func.op_array.opcodes))),
8981 ir_CONST_ADDR(opcodes)),
8982 ir_CONST_ADDR(exit_addr));
8983 }
8984
8985 if (jit->delayed_call_level) {
8986 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
8987 return 0;
8988 }
8989 }
8990
8991 if (!zend_jit_push_call_frame(jit, opline, NULL, func, 1, 0, checked_stack, ref, IR_UNUSED)) {
8992 return 0;
8993 }
8994
8995 if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
8996 if (!zend_jit_save_call_chain(jit, call_level)) {
8997 return 0;
8998 }
8999 } else {
9000 ZEND_ASSERT(call_level > 0);
9001 delayed_call_chain = 1;
9002 jit->delayed_call_level = call_level;
9003 }
9004
9005 if (trace
9006 && trace->op == ZEND_JIT_TRACE_END
9007 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
9008 if (!zend_jit_set_ip(jit, opline + 1)) {
9009 return 0;
9010 }
9011 }
9012
9013 return 1;
9014 }
9015
9016 static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr)
9017 {
9018 uint32_t arg_num = opline->op2.num;
9019 zend_jit_addr arg_addr;
9020
9021 ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM);
9022
9023 if (!zend_jit_reuse_ip(jit)) {
9024 return 0;
9025 }
9026
9027 if (opline->opcode == ZEND_SEND_VAL_EX) {
9028 uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2);
9029
9030 ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM);
9031
9032 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
9033 && JIT_G(current_frame)
9034 && JIT_G(current_frame)->call
9035 && JIT_G(current_frame)->call->func) {
9036 if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
9037 /* Don't generate code that always throws exception */
9038 return 0;
9039 }
9040 } else {
9041 ir_ref cond = ir_AND_U32(
9042 ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))),
9043 ir_CONST_U32(mask));
9044
9045 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
9046 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
9047 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
9048 if (!exit_addr) {
9049 return 0;
9050 }
9051 ir_GUARD_NOT(cond, ir_CONST_ADDR(exit_addr));
9052 } else {
9053 ir_ref if_pass_by_ref;
9054
9055 if_pass_by_ref = ir_IF(cond);
9056
9057 ir_IF_TRUE_cold(if_pass_by_ref);
9058 if (Z_MODE(op1_addr) == IS_REG) {
9059 /* set type to avoid zval_ptr_dtor() on uninitialized value */
9060 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
9061 jit_set_Z_TYPE_INFO(jit, addr, IS_UNDEF);
9062 }
9063 jit_SET_EX_OPLINE(jit, opline);
9064 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_throw_cannot_pass_by_ref));
9065
9066 ir_IF_FALSE(if_pass_by_ref);
9067 }
9068 }
9069 }
9070
9071 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
9072
9073 if (opline->op1_type == IS_CONST) {
9074 zval *zv = RT_CONSTANT(opline, opline->op1);
9075
9076 jit_ZVAL_COPY_CONST(jit,
9077 arg_addr,
9078 MAY_BE_ANY, MAY_BE_ANY,
9079 zv, 1);
9080 } else {
9081 jit_ZVAL_COPY(jit,
9082 arg_addr,
9083 MAY_BE_ANY,
9084 op1_addr, op1_info, 0);
9085 }
9086
9087 return 1;
9088 }
9089
9090 static int zend_jit_send_ref(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold)
9091 {
9092 zend_jit_addr op1_addr, arg_addr, ref_addr;
9093 ir_ref ref_path = IR_UNUSED;
9094
9095 op1_addr = OP1_ADDR();
9096 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
9097
9098 if (!zend_jit_reuse_ip(jit)) {
9099 return 0;
9100 }
9101
9102 if (opline->op1_type == IS_VAR) {
9103 if (op1_info & MAY_BE_INDIRECT) {
9104 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
9105 }
9106 } else if (opline->op1_type == IS_CV) {
9107 if (op1_info & MAY_BE_UNDEF) {
9108 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
9109 // JIT: if (Z_TYPE_P(op1) == IS_UNDEF)
9110 ir_ref if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
9111 ir_IF_FALSE(if_def);
9112 // JIT: ZVAL_NULL(op1)
9113 jit_set_Z_TYPE_INFO(jit,op1_addr, IS_NULL);
9114 ir_MERGE_WITH_EMPTY_TRUE(if_def);
9115 }
9116 op1_info &= ~MAY_BE_UNDEF;
9117 op1_info |= MAY_BE_NULL;
9118 }
9119 } else {
9120 ZEND_UNREACHABLE();
9121 }
9122
9123 if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) {
9124 ir_ref ref, ref2;
9125
9126 if (op1_info & MAY_BE_REF) {
9127 ir_ref if_ref;
9128
9129 // JIT: if (Z_TYPE_P(op1) == IS_UNDEF)
9130 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
9131 ir_IF_TRUE(if_ref);
9132 // JIT: ref = Z_PTR_P(op1)
9133 ref = jit_Z_PTR(jit, op1_addr);
9134 // JIT: GC_ADDREF(ref)
9135 jit_GC_ADDREF(jit, ref);
9136 // JIT: ZVAL_REFERENCE(arg, ref)
9137 jit_set_Z_PTR(jit, arg_addr, ref);
9138 jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
9139 ref_path = ir_END();
9140 ir_IF_FALSE(if_ref);
9141 }
9142
9143 // JIT: ZVAL_NEW_REF(arg, varptr);
9144 // JIT: ref = emalloc(sizeof(zend_reference));
9145 ref = jit_EMALLOC(jit, sizeof(zend_reference), op_array, opline);
9146 // JIT: GC_REFCOUNT(ref) = 2
9147 jit_set_GC_REFCOUNT(jit, ref, 2);
9148 // JIT: GC_TYPE(ref) = GC_REFERENCE
9149 ir_STORE(ir_ADD_OFFSET(ref, offsetof(zend_reference, gc.u.type_info)), ir_CONST_U32(GC_REFERENCE));
9150 ir_STORE(ir_ADD_OFFSET(ref, offsetof(zend_reference, sources.ptr)), IR_NULL);
9151 ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
9152 ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
9153
9154 // JIT: ZVAL_COPY_VALUE(&ref->val, op1)
9155 jit_ZVAL_COPY(jit,
9156 ref_addr,
9157 MAY_BE_ANY,
9158 op1_addr, op1_info, 0);
9159
9160 // JIT: ZVAL_REFERENCE(arg, ref)
9161 jit_set_Z_PTR(jit, op1_addr, ref);
9162 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
9163
9164 // JIT: ZVAL_REFERENCE(arg, ref)
9165 jit_set_Z_PTR(jit, arg_addr, ref);
9166 jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
9167 }
9168
9169 if (ref_path) {
9170 ir_MERGE_WITH(ref_path);
9171 }
9172
9173 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
9174
9175 return 1;
9176 }
9177
9178 static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr)
9179 {
9180 uint32_t arg_num = opline->op2.num;
9181 zend_jit_addr arg_addr;
9182 ir_ref end_inputs = IR_UNUSED;
9183
9184 ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX &&
9185 opline->opcode != ZEND_SEND_VAR_NO_REF_EX) ||
9186 arg_num <= MAX_ARG_FLAG_NUM);
9187
9188 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
9189
9190 if (!zend_jit_reuse_ip(jit)) {
9191 return 0;
9192 }
9193
9194 if (opline->opcode == ZEND_SEND_VAR_EX) {
9195 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
9196 && JIT_G(current_frame)
9197 && JIT_G(current_frame)->call
9198 && JIT_G(current_frame)->call->func) {
9199 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
9200 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
9201 return 0;
9202 }
9203 return 1;
9204 }
9205 } else {
9206 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
9207
9208 // JIT: if (RX->func->quick_arg_flags & mask)
9209 ir_ref if_send_by_ref = ir_IF(ir_AND_U32(
9210 ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))),
9211 ir_CONST_U32(mask)));
9212 ir_IF_TRUE_cold(if_send_by_ref);
9213
9214 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
9215 return 0;
9216 }
9217
9218 ir_END_list(end_inputs);
9219 ir_IF_FALSE(if_send_by_ref);
9220 }
9221 } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
9222 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
9223 && JIT_G(current_frame)
9224 && JIT_G(current_frame)->call
9225 && JIT_G(current_frame)->call->func) {
9226 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
9227
9228 // JIT: ZVAL_COPY_VALUE(arg, op1)
9229 jit_ZVAL_COPY(jit,
9230 arg_addr,
9231 MAY_BE_ANY,
9232 op1_addr, op1_info, 0);
9233
9234 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
9235 if (!(op1_info & MAY_BE_REF)) {
9236 /* Don't generate code that always throws exception */
9237 return 0;
9238 } else {
9239 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
9240 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
9241 if (!exit_addr) {
9242 return 0;
9243 }
9244
9245 // JIT: if (Z_TYPE_P(op1) != IS_REFERENCE)
9246 ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U32(IS_REFERENCE)),
9247 ir_CONST_ADDR(exit_addr));
9248 }
9249 }
9250 return 1;
9251 }
9252 } else {
9253 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
9254 ir_ref func, if_send_by_ref, if_prefer_ref;
9255
9256 // JIT: if (RX->func->quick_arg_flags & mask)
9257 func = ir_LOAD_A(jit_RX(func));
9258 if_send_by_ref = ir_IF(ir_AND_U32(
9259 ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))),
9260 ir_CONST_U32(mask)));
9261 ir_IF_TRUE_cold(if_send_by_ref);
9262
9263 mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2);
9264
9265 // JIT: ZVAL_COPY_VALUE(arg, op1)
9266 jit_ZVAL_COPY(jit,
9267 arg_addr,
9268 MAY_BE_ANY,
9269 op1_addr, op1_info, 0);
9270
9271 if (op1_info & MAY_BE_REF) {
9272 ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
9273 ir_IF_TRUE(if_ref);
9274 ir_END_list(end_inputs);
9275 ir_IF_FALSE(if_ref);
9276 }
9277
9278 // JIT: if (RX->func->quick_arg_flags & mask)
9279 if_prefer_ref = ir_IF(ir_AND_U32(
9280 ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))),
9281 ir_CONST_U32(mask)));
9282 ir_IF_TRUE(if_prefer_ref);
9283 ir_END_list(end_inputs);
9284 ir_IF_FALSE(if_prefer_ref);
9285
9286 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
9287 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
9288 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
9289 if (!exit_addr) {
9290 return 0;
9291 }
9292 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
9293 } else {
9294 jit_SET_EX_OPLINE(jit, opline);
9295 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_only_vars_by_reference),
9296 jit_ZVAL_ADDR(jit, arg_addr));
9297 zend_jit_check_exception(jit);
9298 ir_END_list(end_inputs);
9299 }
9300
9301 ir_IF_FALSE(if_send_by_ref);
9302 }
9303 } else if (opline->opcode == ZEND_SEND_FUNC_ARG) {
9304 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
9305 && JIT_G(current_frame)
9306 && JIT_G(current_frame)->call
9307 && JIT_G(current_frame)->call->func) {
9308 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
9309 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
9310 return 0;
9311 }
9312 return 1;
9313 }
9314 } else {
9315 // JIT: if (RX->This.u1.type_info & ZEND_CALL_SEND_ARG_BY_REF)
9316 ir_ref if_send_by_ref = ir_IF(ir_AND_U32(
9317 ir_LOAD_U32(jit_RX(This.u1.type_info)),
9318 ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
9319 ir_IF_TRUE_cold(if_send_by_ref);
9320
9321 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
9322 return 0;
9323 }
9324
9325 ir_END_list(end_inputs);
9326 ir_IF_FALSE(if_send_by_ref);
9327 }
9328 }
9329
9330 if (op1_info & MAY_BE_UNDEF) {
9331 ir_ref ref, if_def = IR_UNUSED;
9332
9333 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
9334 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
9335 ir_IF_FALSE_cold(if_def);
9336 }
9337
9338 // JIT: zend_jit_undefined_op_helper(opline->op1.var)
9339 jit_SET_EX_OPLINE(jit, opline);
9340 ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
9341 ir_CONST_U32(opline->op1.var));
9342
9343 // JIT: ZVAL_NULL(arg)
9344 jit_set_Z_TYPE_INFO(jit, arg_addr, IS_NULL);
9345
9346 // JIT: check_exception
9347 ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9348
9349 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
9350 ir_END_list(end_inputs);
9351 ir_IF_TRUE(if_def);
9352 } else {
9353 if (end_inputs) {
9354 ir_END_list(end_inputs);
9355 ir_MERGE_list(end_inputs);
9356 }
9357 return 1;
9358 }
9359 }
9360
9361 if (opline->opcode == ZEND_SEND_VAR_NO_REF) {
9362 // JIT: ZVAL_COPY_VALUE(arg, op1)
9363 jit_ZVAL_COPY(jit,
9364 arg_addr,
9365 MAY_BE_ANY,
9366 op1_addr, op1_info, 0);
9367 if (op1_info & MAY_BE_REF) {
9368 // JIT: if (Z_TYPE_P(arg) == IS_REFERENCE)
9369 ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
9370 ir_IF_TRUE(if_ref);
9371 ir_END_list(end_inputs);
9372 ir_IF_FALSE(if_ref);
9373 }
9374 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
9375 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
9376 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
9377 if (!exit_addr) {
9378 return 0;
9379 }
9380 ir_GUARD(IR_FALSE, ir_CONST_ADDR(exit_addr));
9381 } else {
9382 jit_SET_EX_OPLINE(jit, opline);
9383 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_only_vars_by_reference),
9384 jit_ZVAL_ADDR(jit, arg_addr));
9385 zend_jit_check_exception(jit);
9386 }
9387 } else {
9388 if (op1_info & MAY_BE_REF) {
9389 if (opline->op1_type == IS_CV) {
9390 ir_ref ref;
9391
9392 // JIT: ZVAL_DEREF(op1)
9393 ref = jit_ZVAL_ADDR(jit, op1_addr);
9394 ref = jit_ZVAL_DEREF_ref(jit, ref);
9395 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
9396
9397 // JIT: ZVAL_COPY(arg, op1)
9398 jit_ZVAL_COPY(jit,
9399 arg_addr,
9400 MAY_BE_ANY,
9401 op1_addr, op1_info, 1);
9402 } else {
9403 ir_ref if_ref, ref, ref2, refcount, if_not_zero, if_refcounted;
9404 zend_jit_addr ref_addr;
9405
9406 // JIT: if (Z_TYPE_P(op1) == IS_REFERENCE)
9407 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
9408 ir_IF_TRUE_cold(if_ref);
9409
9410 // JIT: ref = Z_COUNTED_P(op1);
9411 ref = jit_Z_PTR(jit, op1_addr);
9412 ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
9413 ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
9414
9415 // JIT: ZVAL_COPY_VALUE(arg, op1);
9416 jit_ZVAL_COPY(jit,
9417 arg_addr,
9418 MAY_BE_ANY,
9419 ref_addr, op1_info, 0);
9420
9421 // JIT: if (GC_DELREF(ref) != 0)
9422 refcount = jit_GC_DELREF(jit, ref);
9423 if_not_zero = ir_IF(refcount);
9424 ir_IF_TRUE(if_not_zero);
9425
9426 // JIT: if (Z_REFCOUNTED_P(arg)
9427 if_refcounted = jit_if_REFCOUNTED(jit, arg_addr);
9428 ir_IF_TRUE(if_refcounted);
9429 // JIT: Z_ADDREF_P(arg)
9430 jit_GC_ADDREF(jit, jit_Z_PTR(jit, arg_addr));
9431 ir_END_list(end_inputs);
9432 ir_IF_FALSE(if_refcounted);
9433 ir_END_list(end_inputs);
9434
9435 ir_IF_FALSE(if_not_zero);
9436
9437 // JIT: efree(ref)
9438 jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
9439 ir_END_list(end_inputs);
9440
9441 ir_IF_FALSE(if_ref);
9442
9443 // JIT: ZVAL_COPY_VALUE(arg, op1);
9444 jit_ZVAL_COPY(jit,
9445 arg_addr,
9446 MAY_BE_ANY,
9447 op1_addr, op1_info, 0);
9448 }
9449 } else {
9450 if (op1_addr != op1_def_addr) {
9451 if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
9452 return 0;
9453 }
9454 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
9455 op1_addr = op1_def_addr;
9456 }
9457 }
9458
9459 // JIT: ZVAL_COPY_VALUE(arg, op1)
9460 jit_ZVAL_COPY(jit,
9461 arg_addr,
9462 MAY_BE_ANY,
9463 op1_addr, op1_info, opline->op1_type == IS_CV);
9464 }
9465 }
9466
9467 if (end_inputs) {
9468 ir_END_list(end_inputs);
9469 ir_MERGE_list(end_inputs);
9470 }
9471
9472 return 1;
9473 }
9474
9475 static int zend_jit_check_func_arg(zend_jit_ctx *jit, const zend_op *opline)
9476 {
9477 uint32_t arg_num = opline->op2.num;
9478 ir_ref ref;
9479
9480 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
9481 && JIT_G(current_frame)
9482 && JIT_G(current_frame)->call
9483 && JIT_G(current_frame)->call->func) {
9484 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
9485 if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) {
9486 TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call);
9487 // JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
9488 if (jit->reuse_ip) {
9489 ref = jit_IP(jit);
9490 } else {
9491 ref = ir_LOAD_A(jit_EX(call));
9492 }
9493 ref = jit_CALL(ref, This.u1.type_info);
9494 ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
9495 }
9496 } else {
9497 if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
9498 TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call);
9499 // JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
9500 if (jit->reuse_ip) {
9501 ref = jit_IP(jit);
9502 } else {
9503 ref = ir_LOAD_A(jit_EX(call));
9504 }
9505 ref = jit_CALL(ref, This.u1.type_info);
9506 ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF)));
9507 }
9508 }
9509 } else {
9510 // JIT: if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) {
9511 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
9512 ir_ref rx, if_ref, cold_path;
9513
9514 if (!zend_jit_reuse_ip(jit)) {
9515 return 0;
9516 }
9517
9518 rx = jit_IP(jit);
9519
9520 ref = ir_AND_U32(
9521 ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(rx, func)), offsetof(zend_function, quick_arg_flags))),
9522 ir_CONST_U32(mask));
9523 if_ref = ir_IF(ref);
9524 ir_IF_TRUE_cold(if_ref);
9525
9526 // JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
9527 ref = jit_CALL(rx, This.u1.type_info);
9528 ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
9529
9530 cold_path = ir_END();
9531 ir_IF_FALSE(if_ref);
9532
9533 // JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
9534 ref = jit_CALL(rx, This.u1.type_info);
9535 ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF)));
9536
9537 ir_MERGE_WITH(cold_path);
9538 }
9539
9540 return 1;
9541 }
9542
9543 static int zend_jit_check_undef_args(zend_jit_ctx *jit, const zend_op *opline)
9544 {
9545 ir_ref call, if_may_have_undef, ret;
9546
9547 if (jit->reuse_ip) {
9548 call = jit_IP(jit);
9549 } else {
9550 call = ir_LOAD_A(jit_EX(call));
9551 }
9552
9553 if_may_have_undef = ir_IF(ir_AND_U8(
9554 ir_LOAD_U8(ir_ADD_OFFSET(call, offsetof(zend_execute_data, This.u1.type_info) + 3)),
9555 ir_CONST_U8(ZEND_CALL_MAY_HAVE_UNDEF >> 24)));
9556
9557 ir_IF_TRUE_cold(if_may_have_undef);
9558 jit_SET_EX_OPLINE(jit, opline);
9559 ret = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_handle_undef_args), call);
9560 ir_GUARD_NOT(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9561 ir_MERGE_WITH_EMPTY_FALSE(if_may_have_undef);
9562
9563 return 1;
9564 }
9565
9566 static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace)
9567 {
9568 zend_func_info *info = ZEND_FUNC_INFO(op_array);
9569 zend_call_info *call_info = NULL;
9570 const zend_function *func = NULL;
9571 uint32_t i;
9572 uint32_t call_num_args = 0;
9573 bool unknown_num_args = 0;
9574 const void *exit_addr = NULL;
9575 const zend_op *prev_opline;
9576 ir_ref rx, func_ref = IR_UNUSED, if_user = IR_UNUSED, user_path = IR_UNUSED;
9577
9578 prev_opline = opline - 1;
9579 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
9580 prev_opline--;
9581 }
9582 if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY ||
9583 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
9584 unknown_num_args = 1;
9585 }
9586
9587 if (info) {
9588 call_info = info->callee_info;
9589 while (call_info && call_info->caller_call_opline != opline) {
9590 call_info = call_info->next_callee;
9591 }
9592 if (call_info && call_info->callee_func && !call_info->is_prototype) {
9593 func = call_info->callee_func;
9594 }
9595 if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)
9596 && JIT_G(current_frame)
9597 && JIT_G(current_frame)->call
9598 && !JIT_G(current_frame)->call->func) {
9599 call_info = NULL; func = NULL; /* megamorphic call from trait */
9600 }
9601 }
9602 if (!func) {
9603 /* resolve function at run time */
9604 } else if (func->type == ZEND_USER_FUNCTION) {
9605 ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL);
9606 call_num_args = call_info->num_args;
9607 } else if (func->type == ZEND_INTERNAL_FUNCTION) {
9608 ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL);
9609 call_num_args = call_info->num_args;
9610 } else {
9611 ZEND_UNREACHABLE();
9612 }
9613
9614 if (trace && !func) {
9615 if (trace->op == ZEND_JIT_TRACE_DO_ICALL) {
9616 ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION);
9617 #ifndef ZEND_WIN32
9618 // TODO: ASLR may cause different addresses in different workers ???
9619 func = trace->func;
9620 if (JIT_G(current_frame) &&
9621 JIT_G(current_frame)->call &&
9622 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
9623 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
9624 } else {
9625 unknown_num_args = 1;
9626 }
9627 #endif
9628 } else if (trace->op == ZEND_JIT_TRACE_ENTER) {
9629 ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION);
9630 if (zend_accel_in_shm(trace->func->op_array.opcodes)) {
9631 func = trace->func;
9632 if (JIT_G(current_frame) &&
9633 JIT_G(current_frame)->call &&
9634 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
9635 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
9636 } else {
9637 unknown_num_args = 1;
9638 }
9639 }
9640 }
9641 }
9642
9643 bool may_have_extra_named_params =
9644 opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS &&
9645 (!func || func->common.fn_flags & ZEND_ACC_VARIADIC);
9646
9647 if (!jit->reuse_ip) {
9648 zend_jit_start_reuse_ip(jit);
9649 // JIT: call = EX(call);
9650 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call)));
9651 }
9652 rx = jit_IP(jit);
9653 zend_jit_stop_reuse_ip(jit);
9654
9655 jit_SET_EX_OPLINE(jit, opline);
9656
9657 if (opline->opcode == ZEND_DO_FCALL) {
9658 if (!func) {
9659 if (trace) {
9660 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
9661
9662 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
9663 if (!exit_addr) {
9664 return 0;
9665 }
9666
9667 func_ref = ir_LOAD_A(jit_CALL(rx, func));
9668 ir_GUARD_NOT(
9669 ir_AND_U32(
9670 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
9671 ir_CONST_U32(ZEND_ACC_DEPRECATED)),
9672 ir_CONST_ADDR(exit_addr));
9673 }
9674 }
9675 }
9676
9677 if (!jit->delayed_call_level) {
9678 // JIT: EX(call) = call->prev_execute_data;
9679 ir_STORE(jit_EX(call),
9680 (call_level == 1) ? IR_NULL : ir_LOAD_A(jit_CALL(rx, prev_execute_data)));
9681 }
9682 delayed_call_chain = 0;
9683 jit->delayed_call_level = 0;
9684
9685 // JIT: call->prev_execute_data = execute_data;
9686 ir_STORE(jit_CALL(rx, prev_execute_data), jit_FP(jit));
9687
9688 if (!func) {
9689 if (!func_ref) {
9690 func_ref = ir_LOAD_A(jit_CALL(rx, func));
9691 }
9692 }
9693
9694 if (opline->opcode == ZEND_DO_FCALL) {
9695 if (!func) {
9696 if (!trace) {
9697 ir_ref if_deprecated, ret;
9698
9699 if_deprecated = ir_IF(ir_AND_U32(
9700 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
9701 ir_CONST_U32(ZEND_ACC_DEPRECATED)));
9702 ir_IF_TRUE_cold(if_deprecated);
9703
9704 if (GCC_GLOBAL_REGS) {
9705 ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
9706 } else {
9707 ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
9708 }
9709 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9710 ir_MERGE_WITH_EMPTY_FALSE(if_deprecated);
9711 }
9712 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
9713 ir_ref ret;
9714
9715 if (GCC_GLOBAL_REGS) {
9716 ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
9717 } else {
9718 ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
9719 }
9720 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9721 }
9722 }
9723
9724 if (!func
9725 && opline->opcode != ZEND_DO_UCALL
9726 && opline->opcode != ZEND_DO_ICALL) {
9727 ir_ref type_ref = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type)));
9728 if_user = ir_IF(ir_EQ(type_ref, ir_CONST_U8(ZEND_USER_FUNCTION)));
9729 ir_IF_TRUE(if_user);
9730 }
9731
9732 if ((!func || func->type == ZEND_USER_FUNCTION)
9733 && opline->opcode != ZEND_DO_ICALL) {
9734 bool recursive_call_through_jmp = 0;
9735
9736 // JIT: EX(call) = NULL;
9737 ir_STORE(jit_CALL(rx, call), IR_NULL);
9738
9739 // JIT: EX(return_value) = RETURN_VALUE_USED(opline) ? EX_VAR(opline->result.var) : 0;
9740 ir_STORE(jit_CALL(rx, return_value),
9741 RETURN_VALUE_USED(opline) ?
9742 jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var)) :
9743 IR_NULL);
9744
9745 // JIT: EX_LOAD_RUN_TIME_CACHE(op_array);
9746 if (!func || func->op_array.cache_size) {
9747 ir_ref run_time_cache;
9748
9749 if (func && op_array == &func->op_array) {
9750 /* recursive call */
9751 run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
9752 } else if (func
9753 && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)
9754 && ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) {
9755 run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CG(map_ptr_base)),
9756 (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache)));
9757 } else if ((func && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) ||
9758 (JIT_G(current_frame) &&
9759 JIT_G(current_frame)->call &&
9760 TRACE_FRAME_IS_CLOSURE_CALL(JIT_G(current_frame)->call))) {
9761 /* Closures always use direct pointers */
9762 ir_ref local_func_ref = func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func));
9763
9764 run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(local_func_ref, offsetof(zend_op_array, run_time_cache__ptr)));
9765 } else {
9766 ir_ref if_odd, run_time_cache2;
9767 ir_ref local_func_ref = func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func));
9768
9769 run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(local_func_ref, offsetof(zend_op_array, run_time_cache__ptr)));
9770 if_odd = ir_IF(ir_AND_A(run_time_cache, ir_CONST_ADDR(1)));
9771 ir_IF_TRUE(if_odd);
9772
9773 run_time_cache2 = ir_LOAD_A(ir_ADD_A(run_time_cache, ir_LOAD_A(jit_CG(map_ptr_base))));
9774
9775 ir_MERGE_WITH_EMPTY_FALSE(if_odd);
9776 run_time_cache = ir_PHI_2(IR_ADDR, run_time_cache2, run_time_cache);
9777 }
9778
9779 ir_STORE(jit_CALL(rx, run_time_cache), run_time_cache);
9780 }
9781
9782 // JIT: EG(current_execute_data) = execute_data = call;
9783 ir_STORE(jit_EG(current_execute_data), rx);
9784 jit_STORE_FP(jit, rx);
9785
9786 // JIT: opline = op_array->opcodes;
9787 if (func && !unknown_num_args) {
9788
9789 for (i = call_num_args; i < func->op_array.last_var; i++) {
9790 uint32_t n = EX_NUM_TO_VAR(i);
9791 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, n);
9792
9793 jit_set_Z_TYPE_INFO_ex(jit, var_addr, ir_CONST_U32(IS_UNDEF));
9794 }
9795
9796 if (call_num_args <= func->op_array.num_args) {
9797 if (!trace || (trace->op == ZEND_JIT_TRACE_END
9798 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
9799 uint32_t num_args;
9800
9801 if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) {
9802 if (trace) {
9803 num_args = 0;
9804 } else if (call_info) {
9805 num_args = skip_valid_arguments(op_array, ssa, call_info);
9806 } else {
9807 num_args = call_num_args;
9808 }
9809 } else {
9810 num_args = call_num_args;
9811 }
9812 if (zend_accel_in_shm(func->op_array.opcodes)) {
9813 jit_LOAD_IP_ADDR(jit, func->op_array.opcodes + num_args);
9814 } else {
9815 if (!func_ref) {
9816 func_ref = ir_LOAD_A(jit_CALL(rx, func));
9817 }
9818 ir_ref ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
9819 if (num_args) {
9820 ip = ir_ADD_OFFSET(ip, num_args * sizeof(zend_op));
9821 }
9822 jit_LOAD_IP(jit, ip);
9823 }
9824
9825 if (!trace && op_array == &func->op_array && call_num_args >= op_array->required_num_args) {
9826 /* recursive call */
9827 recursive_call_through_jmp = 1;
9828 }
9829 }
9830 } else {
9831 if (!trace || (trace->op == ZEND_JIT_TRACE_END
9832 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
9833 ir_ref ip;
9834
9835 if (zend_accel_in_shm(func->op_array.opcodes)) {
9836 ip = ir_CONST_ADDR(func->op_array.opcodes);
9837 } else {
9838 if (!func_ref) {
9839 func_ref = ir_LOAD_A(jit_CALL(rx, func));
9840 }
9841 ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
9842 }
9843 jit_LOAD_IP(jit, ip);
9844 }
9845 if (GCC_GLOBAL_REGS) {
9846 ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper));
9847 } else {
9848 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper), jit_FP(jit));
9849 }
9850 }
9851 } else {
9852 ir_ref ip;
9853 ir_ref merge_inputs = IR_UNUSED;
9854
9855 // JIT: opline = op_array->opcodes
9856 if (func && zend_accel_in_shm(func->op_array.opcodes)) {
9857 ip = ir_CONST_ADDR(func->op_array.opcodes);
9858 } else {
9859 if (!func_ref) {
9860 func_ref = ir_LOAD_A(jit_CALL(rx, func));
9861 }
9862 ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
9863 }
9864 jit_LOAD_IP(jit, ip);
9865
9866 // JIT: num_args = EX_NUM_ARGS();
9867 ir_ref num_args, first_extra_arg;
9868
9869 num_args = ir_LOAD_U32(jit_EX(This.u2.num_args));
9870 if (func) {
9871 first_extra_arg = ir_CONST_U32(func->op_array.num_args);
9872 } else {
9873 // JIT: first_extra_arg = op_array->num_args;
9874 ZEND_ASSERT(func_ref);
9875 first_extra_arg = ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, num_args)));
9876 }
9877
9878 // JIT: if (UNEXPECTED(num_args > first_extra_arg))
9879 ir_ref if_extra_args = ir_IF(ir_GT(num_args, first_extra_arg));
9880 ir_IF_TRUE_cold(if_extra_args);
9881 if (GCC_GLOBAL_REGS) {
9882 ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper));
9883 } else {
9884 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper), jit_FP(jit));
9885 }
9886 ir_END_list(merge_inputs);
9887 ir_IF_FALSE(if_extra_args);
9888 if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) {
9889 if (!func) {
9890 // JIT: if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0))
9891 ir_ref if_has_type_hints = ir_IF(ir_AND_U32(
9892 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
9893 ir_CONST_U32(ZEND_ACC_HAS_TYPE_HINTS)));
9894 ir_IF_TRUE(if_has_type_hints);
9895 ir_END_list(merge_inputs);
9896 ir_IF_FALSE(if_has_type_hints);
9897 }
9898 // JIT: opline += num_args;
9899
9900 ir_ref ref = ir_MUL_U32(num_args, ir_CONST_U32(sizeof(zend_op)));
9901
9902 if (sizeof(void*) == 8) {
9903 ref = ir_ZEXT_A(ref);
9904 }
9905
9906 if (GCC_GLOBAL_REGS) {
9907 jit_STORE_IP(jit, ir_ADD_A(jit_IP(jit), ref));
9908 } else {
9909 ir_ref addr = jit_EX(opline);
9910
9911 ir_STORE(addr, ir_ADD_A(ir_LOAD_A(addr), ref));
9912 }
9913 }
9914
9915 ir_END_list(merge_inputs);
9916 ir_MERGE_list(merge_inputs);
9917
9918 // JIT: if (EXPECTED((int)num_args < op_array->last_var)) {
9919 ir_ref last_var;
9920
9921 if (func) {
9922 last_var = ir_CONST_U32(func->op_array.last_var);
9923 } else {
9924 ZEND_ASSERT(func_ref);
9925 last_var = ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, last_var)));
9926 }
9927
9928 ir_ref idx = ir_SUB_U32(last_var, num_args);
9929 ir_ref if_need = ir_IF(ir_GT(idx, ir_CONST_U32(0)));
9930 ir_IF_TRUE(if_need);
9931
9932 // JIT: zval *var = EX_VAR_NUM(num_args);
9933 if (sizeof(void*) == 8) {
9934 num_args = ir_ZEXT_A(num_args);
9935 }
9936 ir_ref var_ref = ir_ADD_OFFSET(
9937 ir_ADD_A(jit_FP(jit), ir_MUL_A(num_args, ir_CONST_ADDR(sizeof(zval)))),
9938 (ZEND_CALL_FRAME_SLOT * sizeof(zval)) + offsetof(zval, u1.type_info));
9939
9940 ir_ref loop = ir_LOOP_BEGIN(ir_END());
9941 var_ref = ir_PHI_2(IR_ADDR, var_ref, IR_UNUSED);
9942 idx = ir_PHI_2(IR_U32, idx, IR_UNUSED);
9943 ir_STORE(var_ref, ir_CONST_I32(IS_UNDEF));
9944 ir_PHI_SET_OP(var_ref, 2, ir_ADD_OFFSET(var_ref, sizeof(zval)));
9945 ir_ref idx2 = ir_SUB_U32(idx, ir_CONST_U32(1));
9946 ir_PHI_SET_OP(idx, 2, idx2);
9947 ir_ref if_not_zero = ir_IF(idx2);
9948 ir_IF_TRUE(if_not_zero);
9949 ir_MERGE_SET_OP(loop, 2, ir_LOOP_END());
9950 ir_IF_FALSE(if_not_zero);
9951 ir_MERGE_WITH_EMPTY_FALSE(if_need);
9952 }
9953
9954 if (ZEND_OBSERVER_ENABLED) {
9955 if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
9956 ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END);
9957 jit_SET_EX_OPLINE(jit, trace[1].opline);
9958 } else if (GCC_GLOBAL_REGS) {
9959 // EX(opline) = opline
9960 ir_STORE(jit_EX(opline), jit_IP(jit));
9961 }
9962 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_begin), jit_FP(jit));
9963 }
9964
9965 if (trace) {
9966 if (!func && (opline->opcode != ZEND_DO_UCALL)) {
9967 user_path = ir_END();
9968 }
9969 } else {
9970 zend_basic_block *bb;
9971
9972 do {
9973 if (recursive_call_through_jmp) {
9974 ir_ref begin, end;
9975 ir_insn *insn;
9976
9977 /* attempt to convert direct recursive call into loop */
9978 begin = jit->bb_start_ref[call_num_args];
9979 ZEND_ASSERT(begin != IR_UNUSED);
9980 insn = &jit->ctx.ir_base[begin];
9981 if (insn->op == IR_BEGIN) {
9982 end = ir_LOOP_END();
9983 insn = &jit->ctx.ir_base[begin];
9984 insn->op = IR_LOOP_BEGIN;
9985 insn->inputs_count = 2;
9986 insn->op2 = end;
9987 break;
9988 } else if ((insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN)
9989 && insn->inputs_count == 2) {
9990 end = ir_LOOP_END();
9991 insn = &jit->ctx.ir_base[begin];
9992 insn->op = IR_LOOP_BEGIN;
9993 insn->inputs_count = 3;
9994 insn->op3 = end;
9995 break;
9996 } else if (insn->op == IR_LOOP_BEGIN && insn->inputs_count == 3) {
9997 ZEND_ASSERT(jit->ctx.ir_base[insn->op3].op == IR_LOOP_END);
9998 jit->ctx.ir_base[insn->op3].op = IR_END;
9999 ir_MERGE_2(insn->op3, ir_END());
10000 end = ir_LOOP_END();
10001 insn = &jit->ctx.ir_base[begin];
10002 insn->op3 = end;
10003 break;
10004 }
10005 }
10006 /* fallback to indirect JMP or RETURN */
10007 if (GCC_GLOBAL_REGS) {
10008 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
10009 } else {
10010 ir_RETURN(ir_CONST_I32(1));
10011 }
10012 } while (0);
10013
10014 bb = &jit->ssa->cfg.blocks[jit->b];
10015 if (bb->successors_count > 0) {
10016 int succ;
10017 ir_ref ref;
10018
10019 ZEND_ASSERT(bb->successors_count == 1);
10020 succ = bb->successors[0];
10021 /* Add a fake control edge from UNREACHABLE/RETURN to the following ENTRY */
10022 ref = jit->ctx.insns_count - 1;
10023 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE
10024 || jit->ctx.ir_base[ref].op == IR_RETURN
10025 || jit->ctx.ir_base[ref].op == IR_LOOP_END);
10026 ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY);
10027 ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start);
10028 if (func || (opline->opcode == ZEND_DO_UCALL)) {
10029 _zend_jit_add_predecessor_ref(jit, succ, jit->b, ref);
10030 jit->b = -1;
10031 } else {
10032 user_path = ref;
10033 }
10034 }
10035 }
10036 }
10037
10038 if ((!func || func->type == ZEND_INTERNAL_FUNCTION)
10039 && (opline->opcode != ZEND_DO_UCALL)) {
10040 if (!func && (opline->opcode != ZEND_DO_ICALL)) {
10041 ir_IF_FALSE(if_user);
10042 }
10043 if (opline->opcode == ZEND_DO_FCALL_BY_NAME) {
10044 if (!func) {
10045 if (trace) {
10046 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
10047
10048 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
10049 if (!exit_addr) {
10050 return 0;
10051 }
10052 ZEND_ASSERT(func_ref);
10053 ir_GUARD_NOT(
10054 ir_AND_U32(
10055 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
10056 ir_CONST_U32(ZEND_ACC_DEPRECATED)),
10057 ir_CONST_ADDR(exit_addr));
10058 } else {
10059 ir_ref if_deprecated, ret;
10060
10061 if_deprecated = ir_IF(ir_AND_U32(
10062 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
10063 ir_CONST_U32(ZEND_ACC_DEPRECATED)));
10064 ir_IF_TRUE_cold(if_deprecated);
10065
10066 if (GCC_GLOBAL_REGS) {
10067 ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
10068 } else {
10069 ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
10070 }
10071 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
10072 ir_MERGE_WITH_EMPTY_FALSE(if_deprecated);
10073 }
10074 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
10075 ir_ref ret;
10076
10077 if (GCC_GLOBAL_REGS) {
10078 ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
10079 } else {
10080 ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
10081 }
10082 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
10083 }
10084 }
10085
10086 // JIT: EG(current_execute_data) = execute_data;
10087 ir_STORE(jit_EG(current_execute_data), rx);
10088
10089 if (ZEND_OBSERVER_ENABLED) {
10090 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_begin), rx);
10091 }
10092
10093 // JIT: ZVAL_NULL(EX_VAR(opline->result.var));
10094 ir_ref res_addr = IR_UNUSED, func_ptr;
10095
10096 if (RETURN_VALUE_USED(opline)) {
10097 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
10098 } else {
10099 /* CPU stack allocated temporary zval */
10100 ir_ref ptr;
10101
10102 if (!jit->ctx.fixed_call_stack_size) {
10103 // JIT: alloca(sizeof(void*));
10104 ptr = ir_ALLOCA(ir_CONST_ADDR(sizeof(zval)));
10105 } else {
10106 ptr = ir_HARD_COPY_A(ir_RLOAD_A(IR_REG_SP));
10107 }
10108 res_addr = ZEND_ADDR_REF_ZVAL(ptr);
10109 }
10110
10111 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
10112
10113 zend_jit_reset_last_valid_opline(jit);
10114
10115 // JIT: (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret);
10116 if (zend_execute_internal) {
10117 ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_execute_internal), rx, jit_ZVAL_ADDR(jit, res_addr));
10118 } else {
10119 if (func) {
10120 func_ptr = ir_CONST_FC_FUNC(func->internal_function.handler);
10121 } else {
10122 func_ptr = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_internal_function, handler)));
10123 #if defined(IR_TARGET_X86)
10124 func_ptr = ir_CAST_FC_FUNC(func_ptr);
10125 #endif
10126 }
10127 ir_CALL_2(IR_VOID, func_ptr, rx, jit_ZVAL_ADDR(jit, res_addr));
10128 }
10129
10130 if (ZEND_OBSERVER_ENABLED) {
10131 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_end),
10132 rx, jit_ZVAL_ADDR(jit, res_addr));
10133 }
10134
10135 // JIT: EG(current_execute_data) = execute_data;
10136 ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
10137
10138 // JIT: zend_vm_stack_free_args(call);
10139 if (func && !unknown_num_args) {
10140 for (i = 0; i < call_num_args; i++ ) {
10141 if (zend_jit_needs_arg_dtor(func, i, call_info)) {
10142 uint32_t offset = EX_NUM_TO_VAR(i);
10143 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset);
10144
10145 jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, opline);
10146 }
10147 }
10148 } else {
10149 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_vm_stack_free_args_helper), rx);
10150 }
10151
10152 if (may_have_extra_named_params) {
10153 // JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS))
10154 ir_ref if_has_named = ir_IF(ir_AND_U8(
10155 ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 3)),
10156 ir_CONST_U8(ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24)));
10157 ir_IF_TRUE_cold(if_has_named);
10158
10159 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_extra_named_params),
10160 ir_LOAD_A(jit_CALL(rx, extra_named_params)));
10161
10162 ir_MERGE_WITH_EMPTY_FALSE(if_has_named);
10163 }
10164
10165 if (opline->opcode == ZEND_DO_FCALL) {
10166 // TODO: optimize ???
10167 // JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS))
10168 ir_ref if_release_this = ir_IF(ir_AND_U8(
10169 ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 2)),
10170 ir_CONST_U8(ZEND_CALL_RELEASE_THIS >> 16)));
10171 ir_IF_TRUE_cold(if_release_this);
10172
10173 // JIT: OBJ_RELEASE(Z_OBJ(RX->This));
10174 jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_CALL(rx, This.value.obj)));
10175
10176 ir_MERGE_WITH_EMPTY_FALSE(if_release_this);
10177 }
10178
10179
10180 ir_ref allocated_path = IR_UNUSED;
10181
10182 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
10183 !JIT_G(current_frame) ||
10184 !JIT_G(current_frame)->call ||
10185 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) ||
10186 prev_opline->opcode == ZEND_SEND_UNPACK ||
10187 prev_opline->opcode == ZEND_SEND_ARRAY ||
10188 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
10189
10190 // JIT: zend_vm_stack_free_call_frame(call);
10191 // JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_ALLOCATED))
10192 ir_ref if_allocated = ir_IF(ir_AND_U8(
10193 ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 2)),
10194 ir_CONST_U8(ZEND_CALL_ALLOCATED >> 16)));
10195 ir_IF_TRUE_cold(if_allocated);
10196
10197 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_free_call_frame), rx);
10198
10199 allocated_path = ir_END();
10200 ir_IF_FALSE(if_allocated);
10201 }
10202
10203 ir_STORE(jit_EG(vm_stack_top), rx);
10204
10205 if (allocated_path) {
10206 ir_MERGE_WITH(allocated_path);
10207 }
10208
10209 if (!RETURN_VALUE_USED(opline)) {
10210 zend_class_entry *ce;
10211 bool ce_is_instanceof;
10212 uint32_t func_info = call_info ?
10213 zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) :
10214 (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN);
10215
10216 /* If an exception is thrown, the return_value may stay at the
10217 * original value of null. */
10218 func_info |= MAY_BE_NULL;
10219
10220 if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
10221 ir_ref sp = ir_RLOAD_A(IR_REG_SP);
10222 res_addr = ZEND_ADDR_REF_ZVAL(sp);
10223 jit_ZVAL_PTR_DTOR(jit, res_addr, func_info, 1, opline);
10224 }
10225 if (!jit->ctx.fixed_call_stack_size) {
10226 // JIT: revert alloca
10227 ir_AFREE(ir_CONST_ADDR(sizeof(zval)));
10228 }
10229 }
10230
10231 // JIT: if (UNEXPECTED(EG(exception) != NULL)) {
10232 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
10233 jit_STUB_ADDR(jit, jit_stub_icall_throw));
10234
10235 // TODO: Can we avoid checking for interrupts after each call ???
10236 if (trace && jit->last_valid_opline != opline) {
10237 int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM);
10238
10239 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
10240 if (!exit_addr) {
10241 return 0;
10242 }
10243 } else {
10244 exit_addr = NULL;
10245 }
10246
10247 if (!zend_jit_check_timeout(jit, opline + 1, exit_addr)) {
10248 return 0;
10249 }
10250
10251 if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) {
10252 jit_LOAD_IP_ADDR(jit, opline + 1);
10253 } else if (trace
10254 && trace->op == ZEND_JIT_TRACE_END
10255 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
10256 jit_LOAD_IP_ADDR(jit, opline + 1);
10257 }
10258 }
10259
10260 if (user_path) {
10261 ir_MERGE_WITH(user_path);
10262 }
10263
10264 return 1;
10265 }
10266
10267 static int zend_jit_constructor(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, int next_block)
10268 {
10269 ir_ref if_skip_constructor = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, opline), next_block);
10270
10271 ir_IF_FALSE(if_skip_constructor);
10272
10273 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
10274 if (!zend_jit_tail_handler(jit, opline)) {
10275 return 0;
10276 }
10277 } else {
10278 if (!zend_jit_do_fcall(jit, opline, op_array, ssa, call_level, next_block, NULL)) {
10279 return 0;
10280 }
10281 }
10282
10283 /* override predecessors of the next block */
10284 ZEND_ASSERT(jit->ssa->cfg.blocks[next_block].predecessors_count == 1);
10285 if (!jit->ctx.control) {
10286 ZEND_ASSERT(jit->bb_edges[jit->bb_predecessors[next_block]]);
10287 ir_IF_TRUE(if_skip_constructor);
10288 ir_MERGE_2(jit->bb_edges[jit->bb_predecessors[next_block]], ir_END());
10289 jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
10290 } else {
10291 ZEND_ASSERT(!jit->bb_edges[jit->bb_predecessors[next_block]]);
10292 /* merge current control path with the true branch of constructor skip condition */
10293 ir_MERGE_WITH_EMPTY_TRUE(if_skip_constructor);
10294 jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
10295
10296 jit->b = -1;
10297 }
10298
10299 return 1;
10300 }
10301
10302 static int zend_jit_verify_arg_type(zend_jit_ctx *jit, const zend_op *opline, zend_arg_info *arg_info, bool check_exception)
10303 {
10304 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
10305 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
10306 ir_ref ref, fast_path = IR_UNUSED;
10307
10308 ref = jit_ZVAL_ADDR(jit, res_addr);
10309 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
10310 && JIT_G(current_frame)
10311 && JIT_G(current_frame)->prev) {
10312 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
10313 uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
10314
10315 if (type != IS_UNKNOWN && (type_mask & (1u << type))) {
10316 return 1;
10317 }
10318 }
10319
10320 if (ZEND_ARG_SEND_MODE(arg_info)) {
10321 if (opline->opcode == ZEND_RECV_INIT) {
10322 ref = jit_ZVAL_DEREF_ref(jit, ref);
10323 } else {
10324 ref = jit_Z_PTR_ref(jit, ref);
10325 ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
10326 }
10327 }
10328
10329 if (type_mask != 0) {
10330 if (is_power_of_two(type_mask)) {
10331 uint32_t type_code = concrete_type(type_mask);
10332 ir_ref if_ok = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(type_code));
10333 ir_IF_TRUE(if_ok);
10334 fast_path = ir_END();
10335 ir_IF_FALSE_cold(if_ok);
10336 } else {
10337 ir_ref if_ok = ir_IF(ir_AND_U32(
10338 ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE_ref(jit, ref)),
10339 ir_CONST_U32(type_mask)));
10340 ir_IF_TRUE(if_ok);
10341 fast_path = ir_END();
10342 ir_IF_FALSE_cold(if_ok);
10343 }
10344 }
10345
10346 jit_SET_EX_OPLINE(jit, opline);
10347 ref = ir_CALL_2(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_verify_arg_slow),
10348 ref, ir_CONST_ADDR(arg_info));
10349
10350 if (check_exception) {
10351 ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
10352 }
10353
10354 if (fast_path) {
10355 ir_MERGE_WITH(fast_path);
10356 }
10357
10358 return 1;
10359 }
10360
10361 static int zend_jit_recv(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array)
10362 {
10363 uint32_t arg_num = opline->op1.num;
10364 zend_arg_info *arg_info = NULL;
10365
10366 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
10367 if (EXPECTED(arg_num <= op_array->num_args)) {
10368 arg_info = &op_array->arg_info[arg_num-1];
10369 } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) {
10370 arg_info = &op_array->arg_info[op_array->num_args];
10371 }
10372 if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) {
10373 arg_info = NULL;
10374 }
10375 }
10376
10377 if (arg_info || (opline+1)->opcode != ZEND_RECV) {
10378 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
10379 if (!JIT_G(current_frame) ||
10380 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 ||
10381 arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
10382 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
10383 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
10384
10385 if (!exit_addr) {
10386 return 0;
10387 }
10388 ir_GUARD(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)),
10389 ir_CONST_ADDR(exit_addr));
10390 }
10391 } else {
10392 ir_ref if_ok =ir_IF(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)));
10393 ir_IF_FALSE_cold(if_ok);
10394
10395 jit_SET_EX_OPLINE(jit, opline);
10396 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_missing_arg_error), jit_FP(jit));
10397 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
10398 ir_IF_TRUE(if_ok);
10399 }
10400 }
10401
10402 if (arg_info) {
10403 if (!zend_jit_verify_arg_type(jit, opline, arg_info, 1)) {
10404 return 0;
10405 }
10406 }
10407
10408 return 1;
10409 }
10410
10411 static int zend_jit_recv_init(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw)
10412 {
10413 uint32_t arg_num = opline->op1.num;
10414 zval *zv = RT_CONSTANT(opline, opline->op2);
10415 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
10416 ir_ref ref, if_fail, skip_path = IR_UNUSED;
10417
10418 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
10419 && JIT_G(current_frame)
10420 && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) {
10421 if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
10422 jit_ZVAL_COPY_CONST(jit,
10423 res_addr,
10424 -1, -1,
10425 zv, 1);
10426 }
10427 } else {
10428 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
10429 (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
10430 ir_ref if_skip = ir_IF(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)));
10431 ir_IF_TRUE(if_skip);
10432 skip_path = ir_END();
10433 ir_IF_FALSE(if_skip);
10434 }
10435 jit_ZVAL_COPY_CONST(jit,
10436 res_addr,
10437 -1, -1,
10438 zv, 1);
10439 }
10440
10441 if (Z_CONSTANT_P(zv)) {
10442 jit_SET_EX_OPLINE(jit, opline);
10443 ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zval_update_constant_ex),
10444 jit_ZVAL_ADDR(jit, res_addr),
10445 ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), offsetof(zend_op_array, scope))));
10446
10447 if_fail = ir_IF(ref);
10448 ir_IF_TRUE_cold(if_fail);
10449 jit_ZVAL_PTR_DTOR(jit, res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, opline);
10450 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
10451 ir_IF_FALSE(if_fail);
10452 }
10453
10454 if (skip_path) {
10455 ir_MERGE_WITH(skip_path);
10456 }
10457
10458 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
10459 do {
10460 zend_arg_info *arg_info;
10461
10462 if (arg_num <= op_array->num_args) {
10463 arg_info = &op_array->arg_info[arg_num-1];
10464 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
10465 arg_info = &op_array->arg_info[op_array->num_args];
10466 } else {
10467 break;
10468 }
10469 if (!ZEND_TYPE_IS_SET(arg_info->type)) {
10470 break;
10471 }
10472 if (!zend_jit_verify_arg_type(jit, opline, arg_info, may_throw)) {
10473 return 0;
10474 }
10475 } while (0);
10476 }
10477
10478 return 1;
10479 }
10480
10481 static bool zend_jit_verify_return_type(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info)
10482 {
10483 zend_arg_info *arg_info = &op_array->arg_info[-1];
10484 ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type));
10485 zend_jit_addr op1_addr = OP1_ADDR();
10486 bool needs_slow_check = 1;
10487 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
10488 ir_ref fast_path = IR_UNUSED;
10489
10490 if (type_mask != 0) {
10491 if (((op1_info & MAY_BE_ANY) & type_mask) == 0) {
10492 /* pass */
10493 } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) {
10494 needs_slow_check = 0;
10495 } else if (is_power_of_two(type_mask)) {
10496 uint32_t type_code = concrete_type(type_mask);
10497 ir_ref if_ok = jit_if_Z_TYPE(jit, op1_addr, type_code);
10498
10499 ir_IF_TRUE(if_ok);
10500 fast_path = ir_END();
10501 ir_IF_FALSE_cold(if_ok);
10502 } else {
10503 ir_ref if_ok = ir_IF(ir_AND_U32(
10504 ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)),
10505 ir_CONST_U32(type_mask)));
10506
10507 ir_IF_TRUE(if_ok);
10508 fast_path = ir_END();
10509 ir_IF_FALSE_cold(if_ok);
10510 }
10511 }
10512 if (needs_slow_check) {
10513 ir_ref ref;
10514
10515 jit_SET_EX_OPLINE(jit, opline);
10516 ref = jit_ZVAL_ADDR(jit, op1_addr);
10517 if (op1_info & MAY_BE_UNDEF) {
10518 ref = zend_jit_zval_check_undef(jit, ref, opline->op1.var, NULL, 1);
10519 }
10520
10521 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_verify_return_slow),
10522 ref,
10523 ir_LOAD_A(jit_EX(func)),
10524 ir_CONST_ADDR(arg_info),
10525 ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->op2.num));
10526
10527 zend_jit_check_exception(jit);
10528
10529 if (fast_path) {
10530 ir_MERGE_WITH(fast_path);
10531 }
10532 }
10533
10534 return 1;
10535 }
10536
10537 static int zend_jit_leave_frame(zend_jit_ctx *jit)
10538 {
10539 // JIT: EG(current_execute_data) = EX(prev_execute_data);
10540 ir_STORE(jit_EG(current_execute_data), ir_LOAD_A(jit_EX(prev_execute_data)));
10541 return 1;
10542 }
10543
10544 static int zend_jit_free_cvs(zend_jit_ctx *jit)
10545 {
10546 // JIT: EG(current_execute_data) = EX(prev_execute_data);
10547 ir_STORE(jit_EG(current_execute_data), ir_LOAD_A(jit_EX(prev_execute_data)));
10548
10549 // JIT: zend_free_compiled_variables(execute_data);
10550 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_compiled_variables), jit_FP(jit));
10551 return 1;
10552 }
10553
10554 static int zend_jit_free_cv(zend_jit_ctx *jit, uint32_t info, uint32_t var)
10555 {
10556 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
10557 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
10558
10559 jit_ZVAL_PTR_DTOR(jit, var_addr, info, 1, NULL);
10560 }
10561 return 1;
10562 }
10563
10564 static int zend_jit_free_op(zend_jit_ctx *jit, const zend_op *opline, uint32_t info, uint32_t var_offset)
10565 {
10566 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
10567 jit_ZVAL_PTR_DTOR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, opline);
10568 }
10569 return 1;
10570 }
10571
10572 static int zend_jit_leave_func(zend_jit_ctx *jit,
10573 const zend_op_array *op_array,
10574 const zend_op *opline,
10575 uint32_t op1_info,
10576 bool left_frame,
10577 zend_jit_trace_rec *trace,
10578 zend_jit_trace_info *trace_info,
10579 int indirect_var_access,
10580 int may_throw)
10581 {
10582 bool may_be_top_frame =
10583 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
10584 !JIT_G(current_frame) ||
10585 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame));
10586 bool may_need_call_helper =
10587 indirect_var_access || /* may have symbol table */
10588 !op_array->function_name || /* may have symbol table */
10589 may_be_top_frame ||
10590 (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */
10591 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
10592 !JIT_G(current_frame) ||
10593 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */
10594 (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */
10595 bool may_need_release_this =
10596 !(op_array->fn_flags & ZEND_ACC_CLOSURE) &&
10597 op_array->scope &&
10598 !(op_array->fn_flags & ZEND_ACC_STATIC) &&
10599 (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
10600 !JIT_G(current_frame) ||
10601 !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame)));
10602 ir_ref call_info = IR_UNUSED, ref, cold_path = IR_UNUSED;
10603
10604 if (may_need_call_helper) {
10605 if (!left_frame) {
10606 left_frame = 1;
10607 if (!zend_jit_leave_frame(jit)) {
10608 return 0;
10609 }
10610 }
10611 /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */
10612 call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
10613 ref = ir_AND_U32(call_info,
10614 ir_CONST_U32(ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE));
10615 if (trace && trace->op != ZEND_JIT_TRACE_END) {
10616 ir_ref if_slow = ir_IF(ref);
10617
10618 ir_IF_TRUE_cold(if_slow);
10619 if (!GCC_GLOBAL_REGS) {
10620 ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_func_helper), jit_FP(jit));
10621 } else {
10622 ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_func_helper));
10623 }
10624
10625 if (may_be_top_frame) {
10626 // TODO: try to avoid this check ???
10627 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
10628 #if 0
10629 /* this check should be handled by the following OPLINE guard */
10630 | cmp IP, zend_jit_halt_op
10631 | je ->trace_halt
10632 #endif
10633 } else if (GCC_GLOBAL_REGS) {
10634 ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
10635 } else {
10636 ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
10637 }
10638 }
10639
10640 if (!GCC_GLOBAL_REGS) {
10641 // execute_data = EG(current_execute_data)
10642 jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data)));
10643 }
10644 cold_path = ir_END();
10645 ir_IF_FALSE(if_slow);
10646 } else {
10647 ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_leave_function_handler));
10648 }
10649 }
10650
10651 if ((op_array->fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) == ZEND_ACC_CLOSURE) {
10652 if (!left_frame) {
10653 left_frame = 1;
10654 if (!zend_jit_leave_frame(jit)) {
10655 return 0;
10656 }
10657 }
10658 // JIT: OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
10659 jit_OBJ_RELEASE(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), -sizeof(zend_object)));
10660 } else if (may_need_release_this) {
10661 ir_ref if_release, fast_path = IR_UNUSED;
10662
10663 if (!left_frame) {
10664 left_frame = 1;
10665 if (!zend_jit_leave_frame(jit)) {
10666 return 0;
10667 }
10668 }
10669 if (!JIT_G(current_frame) || !TRACE_FRAME_ALWAYS_RELEASE_THIS(JIT_G(current_frame))) {
10670 // JIT: if (call_info & ZEND_CALL_RELEASE_THIS)
10671 if (!call_info) {
10672 call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
10673 }
10674 if_release = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_RELEASE_THIS)));
10675 ir_IF_FALSE(if_release);
10676 fast_path = ir_END();
10677 ir_IF_TRUE(if_release);
10678 }
10679 // JIT: OBJ_RELEASE(execute_data->This))
10680 jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_EX(This.value.obj)));
10681 if (fast_path) {
10682 ir_MERGE_WITH(fast_path);
10683 }
10684 // TODO: avoid EG(excption) check for $this->foo() calls
10685 may_throw = 1;
10686 }
10687
10688 // JIT: EG(vm_stack_top) = (zval*)execute_data
10689 ir_STORE(jit_EG(vm_stack_top), jit_FP(jit));
10690
10691 // JITL execute_data = EX(prev_execute_data)
10692 jit_STORE_FP(jit, ir_LOAD_A(jit_EX(prev_execute_data)));
10693
10694 if (!left_frame) {
10695 // JIT: EG(current_execute_data) = execute_data
10696 ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
10697 }
10698
10699 if (trace) {
10700 if (trace->op != ZEND_JIT_TRACE_END
10701 && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
10702 zend_jit_reset_last_valid_opline(jit);
10703 } else {
10704 if (GCC_GLOBAL_REGS) {
10705 /* We add extra RLOAD and RSTORE to make fusion for persistent register
10706 * mov (%FP), %IP
10707 * add $0x1c, %IP
10708 * The naive (commented) code leads to extra register allocation and move.
10709 * mov (%FP), %tmp
10710 * add $0x1c, %tmp
10711 * mov %tmp, %FP
10712 */
10713 #if 0
10714 jit_STORE_IP(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(opline)), sizeof(zend_op)));
10715 #else
10716 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
10717 jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
10718 #endif
10719 } else {
10720 ir_ref ref = jit_EX(opline);
10721
10722 ir_STORE(ref, ir_ADD_OFFSET(ir_LOAD_A(ref), sizeof(zend_op)));
10723 }
10724 }
10725
10726 if (cold_path) {
10727 ir_MERGE_WITH(cold_path);
10728 }
10729
10730 if (trace->op == ZEND_JIT_TRACE_BACK
10731 && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
10732 const zend_op *next_opline = trace->opline;
10733
10734 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
10735 && (op1_info & MAY_BE_RC1)
10736 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) {
10737 /* exception might be thrown during destruction of unused return value */
10738 // JIT: if (EG(exception))
10739 ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
10740 }
10741 do {
10742 trace++;
10743 } while (trace->op == ZEND_JIT_TRACE_INIT_CALL);
10744 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
10745 next_opline = trace->opline;
10746 ZEND_ASSERT(next_opline != NULL);
10747
10748 if (trace->op == ZEND_JIT_TRACE_END
10749 && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
10750 trace_info->flags |= ZEND_JIT_TRACE_LOOP;
10751
10752 ir_ref if_eq = ir_IF(jit_CMP_IP(jit, IR_EQ, next_opline));
10753
10754 ir_IF_TRUE(if_eq);
10755 ZEND_ASSERT(jit->trace_loop_ref);
10756 ZEND_ASSERT(jit->ctx.ir_base[jit->trace_loop_ref].op2 == IR_UNUSED);
10757 ir_MERGE_SET_OP(jit->trace_loop_ref, 2, ir_END());
10758 ir_IF_FALSE(if_eq);
10759
10760 #ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
10761 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
10762 #else
10763 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape));
10764 #endif
10765 } else {
10766 ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), jit_STUB_ADDR(jit, jit_stub_trace_escape));
10767 }
10768
10769 zend_jit_set_last_valid_opline(jit, trace->opline);
10770
10771 return 1;
10772 } else if (may_throw ||
10773 (((opline->op1_type & (IS_VAR|IS_TMP_VAR))
10774 && (op1_info & MAY_BE_RC1)
10775 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)))
10776 && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) {
10777 // JIT: if (EG(exception))
10778 ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
10779 }
10780
10781 return 1;
10782 } else {
10783 // JIT: if (EG(exception))
10784 ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
10785 // JIT: opline = EX(opline) + 1
10786 if (GCC_GLOBAL_REGS) {
10787 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
10788 jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
10789 } else {
10790 ir_ref ref = jit_EX(opline);
10791
10792 ir_STORE(ref, ir_ADD_OFFSET(ir_LOAD_A(ref), sizeof(zend_op)));
10793 }
10794 }
10795
10796 if (GCC_GLOBAL_REGS) {
10797 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
10798 } else {
10799 ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
10800 }
10801
10802 jit->b = -1;
10803
10804 return 1;
10805 }
10806
10807 static void zend_jit_common_return(zend_jit_ctx *jit)
10808 {
10809 ZEND_ASSERT(jit->return_inputs);
10810 ir_MERGE_list(jit->return_inputs);
10811 }
10812
10813 static int zend_jit_return(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr)
10814 {
10815 zend_jit_addr ret_addr;
10816 int8_t return_value_used = -1;
10817 ir_ref return_value = IR_UNUSED, ref, refcount, if_return_value_used = IR_UNUSED;
10818
10819 ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name);
10820 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF));
10821
10822 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
10823 jit->return_inputs = IR_UNUSED;
10824 if (JIT_G(current_frame)) {
10825 if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
10826 return_value_used = 1;
10827 } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) {
10828 return_value_used = 0;
10829 } else {
10830 return_value_used = -1;
10831 }
10832 }
10833 }
10834
10835 if (ZEND_OBSERVER_ENABLED) {
10836 if (Z_MODE(op1_addr) == IS_REG) {
10837 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
10838
10839 if (!zend_jit_spill_store_inv(jit, op1_addr, dst, op1_info)) {
10840 return 0;
10841 }
10842 op1_addr = dst;
10843 }
10844 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_end),
10845 jit_FP(jit),
10846 jit_ZVAL_ADDR(jit, op1_addr));
10847 }
10848
10849 // JIT: if (!EX(return_value))
10850 return_value = ir_LOAD_A(jit_EX(return_value));
10851 ret_addr = ZEND_ADDR_REF_ZVAL(return_value);
10852 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
10853 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
10854 if (return_value_used == -1) {
10855 if_return_value_used = ir_IF(return_value);
10856 ir_IF_FALSE_cold(if_return_value_used);
10857 }
10858 if (return_value_used != 1) {
10859 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
10860 ir_ref if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
10861 ir_IF_FALSE(if_refcounted);
10862 ir_END_list(jit->return_inputs);
10863 ir_IF_TRUE(if_refcounted);
10864 }
10865 ref = jit_Z_PTR(jit, op1_addr);
10866 refcount = jit_GC_DELREF(jit, ref);
10867
10868 if (RC_MAY_BE_1(op1_info)) {
10869 if (RC_MAY_BE_N(op1_info)) {
10870 ir_ref if_non_zero = ir_IF(refcount);
10871 ir_IF_TRUE(if_non_zero);
10872 ir_END_list(jit->return_inputs);
10873 ir_IF_FALSE(if_non_zero);
10874 }
10875 jit_ZVAL_DTOR(jit, ref, op1_info, opline);
10876 }
10877 if (return_value_used == -1) {
10878 ir_END_list(jit->return_inputs);
10879 }
10880 }
10881 } else if (return_value_used == -1) {
10882 if_return_value_used = ir_IF(return_value);
10883 ir_IF_FALSE_cold(if_return_value_used);
10884 ir_END_list(jit->return_inputs);
10885 }
10886
10887 if (if_return_value_used) {
10888 ir_IF_TRUE(if_return_value_used);
10889 }
10890
10891 if (return_value_used == 0) {
10892 if (jit->return_inputs) {
10893 ZEND_ASSERT(JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE);
10894 ir_END_list(jit->return_inputs);
10895 ir_MERGE_list(jit->return_inputs);
10896 jit->return_inputs = IR_UNUSED;
10897 }
10898 return 1;
10899 }
10900
10901 if (opline->op1_type == IS_CONST) {
10902 zval *zv = RT_CONSTANT(opline, opline->op1);
10903
10904 jit_ZVAL_COPY_CONST(jit, ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
10905 } else if (opline->op1_type == IS_TMP_VAR) {
10906 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
10907 } else if (opline->op1_type == IS_CV) {
10908 if (op1_info & MAY_BE_REF) {
10909 ref = jit_ZVAL_ADDR(jit, op1_addr);
10910 ref = jit_ZVAL_DEREF_ref(jit, ref);
10911 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
10912 }
10913
10914 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
10915 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
10916 (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) ||
10917 !op_array->function_name) {
10918 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 1);
10919 } else if (return_value_used != 1) {
10920 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
10921 // JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr);
10922 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_NULL);
10923 } else {
10924 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
10925 }
10926 } else {
10927 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
10928 }
10929 } else {
10930 if (op1_info & MAY_BE_REF) {
10931 ir_ref if_ref, ref2, if_non_zero;
10932 zend_jit_addr ref_addr;
10933
10934 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
10935 ir_IF_TRUE_cold(if_ref);
10936
10937 // JIT: zend_refcounted *ref = Z_COUNTED_P(retval_ptr)
10938 ref = jit_Z_PTR(jit, op1_addr);
10939
10940 // JIT: ZVAL_COPY_VALUE(return_value, &ref->value)
10941 ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
10942 ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
10943 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, ref_addr, op1_info, 0);
10944 ref2 = jit_GC_DELREF(jit, ref);
10945 if_non_zero = ir_IF(ref2);
10946 ir_IF_TRUE(if_non_zero);
10947
10948 // JIT: if (IS_REFCOUNTED())
10949 ir_ref if_refcounted = jit_if_REFCOUNTED(jit, ret_addr);
10950 ir_IF_FALSE(if_refcounted);
10951 ir_END_list(jit->return_inputs);
10952 ir_IF_TRUE(if_refcounted);
10953
10954 // JIT: ADDREF
10955 ref2 = jit_Z_PTR(jit, ret_addr);
10956 jit_GC_ADDREF(jit, ref2);
10957 ir_END_list(jit->return_inputs);
10958
10959 ir_IF_FALSE(if_non_zero);
10960
10961 jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
10962 ir_END_list(jit->return_inputs);
10963
10964 ir_IF_FALSE(if_ref);
10965 }
10966 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
10967 }
10968
10969 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
10970 if (jit->return_inputs) {
10971 ir_END_list(jit->return_inputs);
10972 ir_MERGE_list(jit->return_inputs);
10973 jit->return_inputs = IR_UNUSED;
10974 }
10975 } else {
10976 ir_END_list(jit->return_inputs);
10977 jit->b = -1;
10978 }
10979
10980 return 1;
10981 }
10982
10983 static int zend_jit_bind_global(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
10984 {
10985 zend_jit_addr op1_addr = OP1_ADDR();
10986 zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2));
10987 ir_ref cache_slot_ref, idx_ref, num_used_ref, bucket_ref, ref, ref2;
10988 ir_ref if_fit, if_reference, if_same_key, fast_path;
10989 ir_ref slow_inputs = IR_UNUSED, end_inputs = IR_UNUSED;
10990
10991 // JIT: idx = (uintptr_t)CACHED_PTR(opline->extended_value) - 1;
10992 cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value);
10993 idx_ref = ir_SUB_A(ir_LOAD_A(cache_slot_ref), ir_CONST_ADDR(1));
10994
10995 // JIT: if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket)))
10996 num_used_ref = ir_MUL_U32(ir_LOAD_U32(jit_EG(symbol_table.nNumUsed)),
10997 ir_CONST_U32(sizeof(Bucket)));
10998 if (sizeof(void*) == 8) {
10999 num_used_ref = ir_ZEXT_A(num_used_ref);
11000 }
11001 if_fit = ir_IF(ir_ULT(idx_ref, num_used_ref));
11002 ir_IF_FALSE_cold(if_fit);
11003 ir_END_list(slow_inputs);
11004 ir_IF_TRUE(if_fit);
11005
11006 // JIT: Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx);
11007 bucket_ref = ir_ADD_A(ir_LOAD_A(jit_EG(symbol_table.arData)), idx_ref);
11008 if_reference = jit_if_Z_TYPE_ref(jit, bucket_ref, ir_CONST_U8(IS_REFERENCE));
11009 ir_IF_FALSE_cold(if_reference);
11010 ir_END_list(slow_inputs);
11011 ir_IF_TRUE(if_reference);
11012
11013 // JIT: (EXPECTED(p->key == varname))
11014 if_same_key = ir_IF(ir_EQ(ir_LOAD_A(ir_ADD_OFFSET(bucket_ref, offsetof(Bucket, key))), ir_CONST_ADDR(varname)));
11015 ir_IF_FALSE_cold(if_same_key);
11016 ir_END_list(slow_inputs);
11017 ir_IF_TRUE(if_same_key);
11018
11019 // JIT: GC_ADDREF(Z_PTR(p->val))
11020 ref = jit_Z_PTR_ref(jit, bucket_ref);
11021 jit_GC_ADDREF(jit, ref);
11022
11023 fast_path = ir_END();
11024 ir_MERGE_list(slow_inputs);
11025
11026 ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_global_helper),
11027 ir_CONST_ADDR(varname),
11028 cache_slot_ref);
11029
11030 ir_MERGE_WITH(fast_path);
11031 ref = ir_PHI_2(IR_ADDR, ref2, ref);
11032
11033 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
11034 ir_ref if_refcounted = IR_UNUSED, refcount, if_non_zero, if_may_not_leak;
11035
11036 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
11037 // JIT: if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr)))
11038 if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
11039 ir_IF_TRUE_cold(if_refcounted);
11040 }
11041
11042 // JIT:zend_refcounted *garbage = Z_COUNTED_P(variable_ptr);
11043 ref2 = jit_Z_PTR(jit, op1_addr);
11044
11045 // JIT: ZVAL_REF(variable_ptr, ref)
11046 jit_set_Z_PTR(jit, op1_addr, ref);
11047 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
11048
11049 // JIT: if (GC_DELREF(garbage) == 0)
11050 refcount = jit_GC_DELREF(jit, ref2);
11051 if_non_zero = ir_IF(refcount);
11052 if (!(op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
11053 ir_IF_TRUE(if_non_zero);
11054 ir_END_list(end_inputs);
11055 }
11056 ir_IF_FALSE(if_non_zero);
11057
11058 jit_ZVAL_DTOR(jit, ref2, op1_info, opline);
11059 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) {
11060 ir_END_list(end_inputs);
11061 ir_IF_TRUE(if_non_zero);
11062
11063 // JIT: GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr)
11064 if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref2);
11065 ir_IF_TRUE(if_may_not_leak);
11066 ir_END_list(end_inputs);
11067 ir_IF_FALSE(if_may_not_leak);
11068 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref2);
11069 }
11070 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
11071 ir_END_list(end_inputs);
11072 ir_IF_FALSE(if_refcounted);
11073 }
11074 }
11075
11076 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
11077 // JIT: ZVAL_REF(variable_ptr, ref)
11078 jit_set_Z_PTR(jit, op1_addr, ref);
11079 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
11080 }
11081
11082 if (end_inputs) {
11083 ir_END_list(end_inputs);
11084 ir_MERGE_list(end_inputs);
11085 }
11086
11087 return 1;
11088 }
11089
11090 static int zend_jit_free(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, int may_throw)
11091 {
11092 zend_jit_addr op1_addr = OP1_ADDR();
11093
11094 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
11095 if (may_throw) {
11096 jit_SET_EX_OPLINE(jit, opline);
11097 }
11098 if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) {
11099 ir_ref ref, if_array, if_exists, end_inputs = IR_UNUSED;
11100
11101 if (op1_info & MAY_BE_ARRAY) {
11102 if_array = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
11103 ir_IF_TRUE(if_array);
11104 ir_END_list(end_inputs);
11105 ir_IF_FALSE(if_array);
11106 }
11107 ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_iter_idx)));
11108 if_exists = ir_IF(ir_EQ(ref, ir_CONST_U32(-1)));
11109 ir_IF_TRUE(if_exists);
11110 ir_END_list(end_inputs);
11111 ir_IF_FALSE(if_exists);
11112
11113 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_hash_iterator_del), ref);
11114
11115 ir_END_list(end_inputs);
11116 ir_MERGE_list(end_inputs);
11117 }
11118
11119 jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
11120
11121 if (may_throw) {
11122 zend_jit_check_exception(jit);
11123 }
11124 }
11125
11126 return 1;
11127 }
11128
11129 static int zend_jit_echo(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
11130 {
11131 if (opline->op1_type == IS_CONST) {
11132 zval *zv;
11133 size_t len;
11134
11135 zv = RT_CONSTANT(opline, opline->op1);
11136 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
11137 len = Z_STRLEN_P(zv);
11138
11139 if (len > 0) {
11140 const char *str = Z_STRVAL_P(zv);
11141
11142 jit_SET_EX_OPLINE(jit, opline);
11143 ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_write),
11144 ir_CONST_ADDR(str), ir_CONST_ADDR(len));
11145
11146 zend_jit_check_exception(jit);
11147 }
11148 } else {
11149 zend_jit_addr op1_addr = OP1_ADDR();
11150 ir_ref ref;
11151
11152 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
11153
11154 jit_SET_EX_OPLINE(jit, opline);
11155
11156 ref = jit_Z_PTR(jit, op1_addr);
11157 ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_write),
11158 ir_ADD_OFFSET(ref, offsetof(zend_string, val)),
11159 ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_string, len))));
11160
11161 if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) {
11162 jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
11163 }
11164
11165 zend_jit_check_exception(jit);
11166 }
11167 return 1;
11168 }
11169
11170 static int zend_jit_strlen(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr)
11171 {
11172 if (opline->op1_type == IS_CONST) {
11173 zval *zv;
11174 size_t len;
11175
11176 zv = RT_CONSTANT(opline, opline->op1);
11177 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
11178 len = Z_STRLEN_P(zv);
11179
11180 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(len));
11181 if (Z_MODE(res_addr) != IS_REG) {
11182 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11183 } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11184 return 0;
11185 }
11186 } else {
11187 ir_ref ref;
11188
11189 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
11190
11191 ref = jit_Z_PTR(jit, op1_addr);
11192 ref = ir_LOAD_L(ir_ADD_OFFSET(ref, offsetof(zend_string, len)));
11193 jit_set_Z_LVAL(jit, res_addr, ref);
11194
11195 if (Z_MODE(res_addr) == IS_REG) {
11196 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11197 return 0;
11198 }
11199 } else {
11200 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11201 }
11202 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
11203 }
11204 return 1;
11205 }
11206
11207 static int zend_jit_count(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw)
11208 {
11209 if (opline->op1_type == IS_CONST) {
11210 zval *zv;
11211 zend_long count;
11212
11213 zv = RT_CONSTANT(opline, opline->op1);
11214 ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY);
11215 count = zend_hash_num_elements(Z_ARRVAL_P(zv));
11216
11217 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(count));
11218 if (Z_MODE(res_addr) != IS_REG) {
11219 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11220 } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11221 return 0;
11222 }
11223 } else {
11224 ir_ref ref;
11225
11226 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY);
11227 // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+.
11228
11229 ref = jit_Z_PTR(jit, op1_addr);
11230 if (sizeof(void*) == 8) {
11231 ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(HashTable, nNumOfElements)));
11232 ref = ir_ZEXT_L(ref);
11233 } else {
11234 ref = ir_LOAD_L(ir_ADD_OFFSET(ref, offsetof(HashTable, nNumOfElements)));
11235 }
11236 jit_set_Z_LVAL(jit, res_addr, ref);
11237
11238 if (Z_MODE(res_addr) == IS_REG) {
11239 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11240 return 0;
11241 }
11242 } else {
11243 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11244 }
11245 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
11246 }
11247
11248 if (may_throw) {
11249 zend_jit_check_exception(jit);
11250 }
11251 return 1;
11252 }
11253
11254 static int zend_jit_in_array(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
11255 {
11256 HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
11257 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
11258 ir_ref ref;
11259
11260 ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR);
11261 ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING);
11262
11263 // JIT: result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST);
11264 if (opline->op1_type != IS_CONST) {
11265 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find),
11266 ir_CONST_ADDR(ht),
11267 jit_Z_PTR(jit, op1_addr));
11268 } else {
11269 zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1));
11270
11271 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash),
11272 ir_CONST_ADDR(ht), ir_CONST_ADDR(str));
11273 }
11274
11275 if (exit_addr) {
11276 if (smart_branch_opcode == ZEND_JMPZ) {
11277 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
11278 } else {
11279 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
11280 }
11281 } else if (smart_branch_opcode) {
11282 zend_basic_block *bb;
11283
11284 ZEND_ASSERT(jit->b >= 0);
11285 bb = &jit->ssa->cfg.blocks[jit->b];
11286 ZEND_ASSERT(bb->successors_count == 2);
11287 ref = jit_IF_ex(jit, ref,
11288 (smart_branch_opcode == ZEND_JMPZ) ? target_label2 : target_label);
11289 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
11290 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
11291 jit->b = -1;
11292 } else {
11293 jit_set_Z_TYPE_INFO_ex(jit, res_addr,
11294 ir_ADD_U32(ir_ZEXT_U32(ir_NE(ref, IR_NULL)), ir_CONST_U32(IS_FALSE)));
11295 }
11296
11297 return 1;
11298 }
11299
11300 static int zend_jit_rope(zend_jit_ctx *jit, const zend_op *opline, uint32_t op2_info)
11301 {
11302 uint32_t offset;
11303
11304 offset = (opline->opcode == ZEND_ROPE_INIT) ?
11305 opline->result.var :
11306 opline->op1.var + opline->extended_value * sizeof(zend_string*);
11307
11308 if (opline->op2_type == IS_CONST) {
11309 zval *zv = RT_CONSTANT(opline, opline->op2);
11310 zend_string *str;
11311
11312 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
11313 str = Z_STR_P(zv);
11314
11315 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ir_CONST_ADDR(str));
11316 } else {
11317 zend_jit_addr op2_addr = OP2_ADDR();
11318 ir_ref ref;
11319
11320 ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
11321
11322 ref = jit_Z_PTR(jit, op2_addr);
11323 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ref);
11324 if (opline->op2_type == IS_CV) {
11325 ir_ref if_refcounted, long_path;
11326
11327 if_refcounted = jit_if_REFCOUNTED(jit, op2_addr);
11328 ir_IF_TRUE(if_refcounted);
11329 jit_GC_ADDREF(jit, ref);
11330 long_path = ir_END();
11331
11332 ir_IF_FALSE(if_refcounted);
11333 ir_MERGE_WITH(long_path);
11334 }
11335 }
11336
11337 if (opline->opcode == ZEND_ROPE_END) {
11338 zend_jit_addr res_addr = RES_ADDR();
11339 ir_ref ref;
11340
11341 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_rope_end),
11342 ir_ADD_OFFSET(jit_FP(jit), opline->op1.var),
11343 ir_CONST_U32(opline->extended_value));
11344
11345 jit_set_Z_PTR(jit, res_addr, ref);
11346 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
11347 }
11348
11349 return 1;
11350 }
11351
11352 static int zend_jit_zval_copy_deref(zend_jit_ctx *jit, zend_jit_addr res_addr, zend_jit_addr val_addr, ir_ref type)
11353 {
11354 ir_ref if_refcounted, if_reference, if_refcounted2, ptr, val2, ptr2, type2;
11355 ir_refs *merge_inputs, *types, *ptrs;
11356 #if SIZEOF_ZEND_LONG == 4
11357 ir_ref val = jit_ZVAL_ADDR(jit, val_addr);
11358 ir_refs *values; /* we need this only for zval.w2 copy */
11359 #endif
11360
11361 ir_refs_init(merge_inputs, 4);
11362 ir_refs_init(types, 4);
11363 ir_refs_init(ptrs, 4);
11364 #if SIZEOF_ZEND_LONG == 4
11365 ir_refs_init(values, 4);
11366 #endif
11367
11368 // JIT: ptr = Z_PTR_P(val);
11369 ptr = jit_Z_PTR(jit, val_addr);
11370
11371 // JIT: if (Z_OPT_REFCOUNTED_P(val)) {
11372 if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(Z_TYPE_FLAGS_MASK)));
11373 ir_IF_FALSE_cold(if_refcounted);
11374 ir_refs_add(merge_inputs, ir_END());
11375 ir_refs_add(types, type);
11376 ir_refs_add(ptrs, ptr);
11377 #if SIZEOF_ZEND_LONG == 4
11378 ir_refs_add(values, val);
11379 #endif
11380
11381 ir_IF_TRUE(if_refcounted);
11382
11383 // JIT: if (UNEXPECTED(Z_OPT_ISREF_P(val))) {
11384 if_reference = ir_IF(ir_EQ(type, ir_CONST_U32(IS_REFERENCE_EX)));
11385 // if_reference = ir_IF(ir_EQ(ir_TRUNC_U8(type), ir_CONST_U8(IS_REFERENCE))); // TODO: fix IR to avoid need for extra register ???
11386 ir_IF_TRUE(if_reference);
11387
11388 // JIT: val = Z_REFVAL_P(val);
11389 val2 = ir_ADD_OFFSET(ptr, offsetof(zend_reference, val));
11390 type2 = jit_Z_TYPE_INFO_ref(jit, val2);
11391 ptr2 = jit_Z_PTR_ref(jit, val2);
11392
11393 // JIT: if (Z_OPT_REFCOUNTED_P(val)) {
11394 if_refcounted2 = ir_IF(ir_AND_U32(type2, ir_CONST_U32(Z_TYPE_FLAGS_MASK)));
11395 ir_IF_FALSE_cold(if_refcounted2);
11396 ir_refs_add(merge_inputs, ir_END());
11397 ir_refs_add(types, type2);
11398 ir_refs_add(ptrs, ptr2);
11399 #if SIZEOF_ZEND_LONG == 4
11400 ir_refs_add(values, val2);
11401 #endif
11402
11403 ir_IF_TRUE(if_refcounted2);
11404 ir_MERGE_WITH_EMPTY_FALSE(if_reference);
11405 type = ir_PHI_2(IR_U32, type2, type);
11406 ptr = ir_PHI_2(IR_ADDR, ptr2, ptr);
11407 #if SIZEOF_ZEND_LONG == 4
11408 val = ir_PHI_2(IR_ADDR, val2, val);
11409 #endif
11410
11411 // JIT: Z_ADDREF_P(val);
11412 jit_GC_ADDREF(jit, ptr);
11413 ir_refs_add(merge_inputs, ir_END());
11414 ir_refs_add(types, type);
11415 ir_refs_add(ptrs, ptr);
11416 #if SIZEOF_ZEND_LONG == 4
11417 ir_refs_add(values, val);
11418 #endif
11419
11420 ir_MERGE_N(merge_inputs->count, merge_inputs->refs);
11421 type = ir_PHI_N(IR_U32, types->count, types->refs);
11422 ptr = ir_PHI_N(IR_ADDR, ptrs->count, ptrs->refs);
11423 #if SIZEOF_ZEND_LONG == 4
11424 val = ir_PHI_N(IR_ADDR, values->count, values->refs);
11425 val_addr = ZEND_ADDR_REF_ZVAL(val);
11426 #endif
11427
11428 // JIT: Z_PTR_P(res) = ptr;
11429 jit_set_Z_PTR(jit, res_addr, ptr);
11430 #if SIZEOF_ZEND_LONG == 4
11431 jit_set_Z_W2(jit, res_addr, jit_Z_W2(jit, val_addr));
11432 #endif
11433 jit_set_Z_TYPE_INFO_ex(jit, res_addr, type);
11434
11435 return 1;
11436 }
11437
11438 static int zend_jit_fetch_dimension_address_inner(zend_jit_ctx *jit,
11439 const zend_op *opline,
11440 uint32_t type,
11441 uint32_t op1_info,
11442 uint32_t op2_info,
11443 zend_jit_addr op2_addr,
11444 zend_ssa_range *op2_range,
11445 uint8_t dim_type,
11446 const void *found_exit_addr,
11447 const void *not_found_exit_addr,
11448 const void *exit_addr,
11449 bool result_type_guard,
11450 ir_ref ht_ref,
11451 ir_refs *found_inputs,
11452 ir_refs *found_vals,
11453 ir_ref *end_inputs,
11454 ir_ref *not_found_inputs)
11455 {
11456 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
11457 ir_ref ref = IR_UNUSED, cond, if_found;
11458 ir_ref if_type = IS_UNUSED;
11459 ir_refs *test_zval_inputs, *test_zval_values;
11460
11461 ir_refs_init(test_zval_inputs, 4);
11462 ir_refs_init(test_zval_values, 4);
11463
11464 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
11465 && type == BP_VAR_R
11466 && !exit_addr) {
11467 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
11468 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
11469 if (!exit_addr) {
11470 return 0;
11471 }
11472 }
11473
11474 if (op2_info & MAY_BE_LONG) {
11475 bool op2_loaded = 0;
11476 bool packed_loaded = 0;
11477 bool bad_packed_key = 0;
11478 ir_ref if_packed = IS_UNDEF;
11479 ir_ref h = IR_UNUSED;
11480 ir_ref idx_not_found_inputs = IR_UNUSED;
11481
11482 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) {
11483 // JIT: if (EXPECTED(Z_TYPE_P(dim) == IS_LONG))
11484 if_type = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
11485 ir_IF_TRUE(if_type);
11486 }
11487 if (op1_info & MAY_BE_PACKED_GUARD) {
11488 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
11489 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
11490
11491 if (!exit_addr) {
11492 return 0;
11493 }
11494 cond = ir_AND_U32(
11495 ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
11496 ir_CONST_U32(HASH_FLAG_PACKED));
11497 if (op1_info & MAY_BE_ARRAY_PACKED) {
11498 ir_GUARD(cond, ir_CONST_ADDR(exit_addr));
11499 } else {
11500 ir_GUARD_NOT(cond, ir_CONST_ADDR(exit_addr));
11501 }
11502 }
11503 if (type == BP_VAR_W) {
11504 // JIT: hval = Z_LVAL_P(dim);
11505 h = jit_Z_LVAL(jit, op2_addr);
11506 op2_loaded = 1;
11507 }
11508 if (op1_info & MAY_BE_ARRAY_PACKED) {
11509 zend_long val = -1;
11510
11511 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
11512 val = Z_LVAL_P(Z_ZV(op2_addr));
11513 if (val >= 0 && val < HT_MAX_SIZE) {
11514 packed_loaded = 1;
11515 } else {
11516 bad_packed_key = 1;
11517 }
11518 h = ir_CONST_LONG(val);
11519 } else {
11520 if (!op2_loaded) {
11521 // JIT: hval = Z_LVAL_P(dim);
11522 h = jit_Z_LVAL(jit, op2_addr);
11523 op2_loaded = 1;
11524 }
11525 packed_loaded = 1;
11526 }
11527
11528 if (dim_type == IS_UNDEF && type == BP_VAR_W && packed_loaded) {
11529 /* don't generate "fast" code for packed array */
11530 packed_loaded = 0;
11531 }
11532
11533 if (packed_loaded) {
11534 // JIT: ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef);
11535 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
11536 if_packed = ir_IF(
11537 ir_AND_U32(
11538 ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
11539 ir_CONST_U32(HASH_FLAG_PACKED)));
11540 ir_IF_TRUE(if_packed);
11541 }
11542 // JIT: if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed))
11543 ref = ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed)));
11544 #if SIZEOF_ZEND_LONG == 8
11545 if ((Z_MODE(op2_addr) == IS_CONST_ZVAL && val >= 0 && val <= UINT32_MAX)
11546 || (op2_range && op2_range->min >= 0 && op2_range->max <= UINT32_MAX)) {
11547 /* comapre only the lower 32-bits to allow load fusion on x86_64 */
11548 cond = ir_ULT(ir_TRUNC_U32(h), ref);
11549 } else {
11550 cond = ir_ULT(h, ir_ZEXT_L(ref));
11551 }
11552 #else
11553 cond = ir_ULT(h, ref);
11554 #endif
11555 if (type == BP_JIT_IS) {
11556 if (not_found_exit_addr) {
11557 ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
11558 } else {
11559 ir_ref if_fit = ir_IF(cond);
11560 ir_IF_FALSE(if_fit);
11561 ir_END_list(*end_inputs);
11562 ir_IF_TRUE(if_fit);
11563 }
11564 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
11565 ir_GUARD(cond, ir_CONST_ADDR(exit_addr));
11566 } else if (type == BP_VAR_IS && not_found_exit_addr) {
11567 ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
11568 } else if (type == BP_VAR_RW && not_found_exit_addr) {
11569 ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
11570 } else if (type == BP_VAR_IS && result_type_guard) {
11571 ir_ref if_fit = ir_IF(cond);
11572 ir_IF_FALSE(if_fit);
11573 ir_END_list(*not_found_inputs);
11574 ir_IF_TRUE(if_fit);
11575 } else {
11576 ir_ref if_fit = ir_IF(cond);
11577 ir_IF_FALSE(if_fit);
11578 ir_END_list(idx_not_found_inputs);
11579 ir_IF_TRUE(if_fit);
11580 }
11581 // JIT: _ret = &_ht->arPacked[h];
11582 ref = ir_MUL_L(h, ir_CONST_LONG(sizeof(zval)));
11583 ref = ir_BITCAST_A(ref);
11584 ref = ir_ADD_A(ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arPacked))), ref);
11585 if (type == BP_JIT_IS) {
11586 ir_refs_add(test_zval_values, ref);
11587 ir_refs_add(test_zval_inputs, ir_END());
11588 }
11589 }
11590 }
11591 switch (type) {
11592 case BP_JIT_IS:
11593 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
11594 if (if_packed) {
11595 ir_IF_FALSE(if_packed);
11596 if_packed = IR_UNUSED;
11597 }
11598 if (!op2_loaded) {
11599 // JIT: hval = Z_LVAL_P(dim);
11600 h = jit_Z_LVAL(jit, op2_addr);
11601 }
11602 if (packed_loaded) {
11603 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(_zend_hash_index_find), ht_ref, h);
11604 } else {
11605 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_find), ht_ref, h);
11606 }
11607 if (not_found_exit_addr) {
11608 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11609 } else {
11610 if_found = ir_IF(ref);
11611 ir_IF_FALSE(if_found);
11612 ir_END_list(*end_inputs);
11613 ir_IF_TRUE(if_found);
11614 }
11615 ir_refs_add(test_zval_values, ref);
11616 ir_refs_add(test_zval_inputs, ir_END());
11617 } else if (!not_found_exit_addr && !packed_loaded) {
11618 ir_END_list(*end_inputs);
11619 }
11620 break;
11621 case BP_VAR_R:
11622 case BP_VAR_IS:
11623 case BP_VAR_UNSET:
11624 if (packed_loaded) {
11625 ir_ref type_ref = jit_Z_TYPE_ref(jit, ref);
11626
11627 if (result_type_guard) {
11628 /* perform IS_UNDEF check only after result type guard (during deoptimization) */
11629 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
11630 ir_GUARD(type_ref, ir_CONST_ADDR(exit_addr));
11631 } else if (type == BP_VAR_IS && not_found_exit_addr) {
11632 ir_GUARD(type_ref, ir_CONST_ADDR(not_found_exit_addr));
11633 } else {
11634 ir_ref if_def = ir_IF(type_ref);
11635 ir_IF_FALSE(if_def);
11636 ir_END_list(idx_not_found_inputs);
11637 ir_IF_TRUE(if_def);
11638 }
11639 ir_refs_add(found_inputs, ir_END());
11640 ir_refs_add(found_vals, ref);
11641 }
11642 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
11643 if (if_packed) {
11644 ir_IF_FALSE(if_packed);
11645 if_packed = IR_UNUSED;
11646 }
11647 if (!op2_loaded) {
11648 // JIT: hval = Z_LVAL_P(dim);
11649 h = jit_Z_LVAL(jit, op2_addr);
11650 }
11651 if (packed_loaded) {
11652 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(_zend_hash_index_find), ht_ref, h);
11653 } else {
11654 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_find), ht_ref, h);
11655 }
11656 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
11657 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
11658 } else if (type == BP_VAR_IS && not_found_exit_addr) {
11659 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11660 } else if (type == BP_VAR_IS && result_type_guard) {
11661 if_found = ir_IF(ref);
11662 ir_IF_FALSE(if_found);
11663 ir_END_list(*not_found_inputs);
11664 ir_IF_TRUE(if_found);
11665 } else {
11666 if_found = ir_IF(ref);
11667 ir_IF_FALSE(if_found);
11668 ir_END_list(idx_not_found_inputs);
11669 ir_IF_TRUE(if_found);
11670 }
11671 ir_refs_add(found_inputs, ir_END());
11672 ir_refs_add(found_vals, ref);
11673 } else if (!packed_loaded) {
11674 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
11675 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
11676 } else if (type == BP_VAR_IS && not_found_exit_addr) {
11677 jit_SIDE_EXIT(jit, ir_CONST_ADDR(not_found_exit_addr));
11678 } else if (type == BP_VAR_IS && result_type_guard) {
11679 ir_END_list(*not_found_inputs);
11680 } else {
11681 ir_END_list(idx_not_found_inputs);
11682 }
11683 }
11684
11685 if (idx_not_found_inputs) {
11686 ir_MERGE_list(idx_not_found_inputs);
11687 switch (type) {
11688 case BP_VAR_R:
11689 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
11690 // JIT: zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval);
11691 // JIT: retval = &EG(uninitialized_zval);
11692 jit_SET_EX_OPLINE(jit, opline);
11693 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_offset, IR_FASTCALL_FUNC));
11694 ir_END_list(*end_inputs);
11695 break;
11696 case BP_VAR_IS:
11697 case BP_VAR_UNSET:
11698 if (!not_found_exit_addr) {
11699 // JIT: retval = &EG(uninitialized_zval);
11700 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
11701 ir_END_list(*end_inputs);
11702 }
11703 break;
11704 default:
11705 ZEND_UNREACHABLE();
11706 }
11707 }
11708 break;
11709 case BP_VAR_RW:
11710 if (packed_loaded) {
11711 if (not_found_exit_addr) {
11712 ir_refs_add(found_inputs, ir_END());
11713 ir_refs_add(found_vals, ref);
11714 } else {
11715 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
11716 ir_IF_TRUE(if_def);
11717 ir_refs_add(found_inputs, ir_END());
11718 ir_refs_add(found_vals, ref);
11719 ir_IF_FALSE_cold(if_def);
11720 ir_END_list(idx_not_found_inputs);
11721 }
11722 }
11723 if (!packed_loaded ||
11724 !not_found_exit_addr ||
11725 (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
11726 if (if_packed) {
11727 ir_IF_FALSE(if_packed);
11728 if_packed = IR_UNUSED;
11729 ir_END_list(idx_not_found_inputs);
11730 } else if (!packed_loaded) {
11731 ir_END_list(idx_not_found_inputs);
11732 }
11733
11734 ir_MERGE_list(idx_not_found_inputs);
11735 if (!op2_loaded) {
11736 // JIT: hval = Z_LVAL_P(dim);
11737 h = jit_Z_LVAL(jit, op2_addr);
11738 }
11739 if (packed_loaded) {
11740 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_index_lookup_rw_no_packed),
11741 ht_ref, h);
11742 } else {
11743 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_index_lookup_rw), ht_ref, h);
11744 }
11745 if (not_found_exit_addr) {
11746 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11747 } else {
11748 if_found = ir_IF(ref);
11749 ir_IF_FALSE(if_found);
11750 ir_END_list(*end_inputs);
11751 ir_IF_TRUE(if_found);
11752 }
11753 ir_refs_add(found_inputs, ir_END());
11754 ir_refs_add(found_vals, ref);
11755 }
11756 break;
11757 case BP_VAR_W:
11758 if (packed_loaded) {
11759 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
11760 ir_IF_TRUE_cold(if_def);
11761 ir_refs_add(found_inputs, ir_END());
11762 ir_refs_add(found_vals, ref);
11763 ir_IF_FALSE(if_def);
11764 ir_END_list(idx_not_found_inputs);
11765 }
11766 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) {
11767 if (if_packed) {
11768 ir_IF_FALSE(if_packed);
11769 if_packed = IR_UNUSED;
11770 ir_END_list(idx_not_found_inputs);
11771 } else if (!packed_loaded) {
11772 ir_END_list(idx_not_found_inputs);
11773 }
11774 ir_MERGE_list(idx_not_found_inputs);
11775 if (!op2_loaded) {
11776 // JIT: hval = Z_LVAL_P(dim);
11777 h = jit_Z_LVAL(jit, op2_addr);
11778 }
11779 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_lookup), ht_ref, h);
11780 ir_refs_add(found_inputs, ir_END());
11781 ir_refs_add(found_vals, ref);
11782 }
11783 break;
11784 default:
11785 ZEND_UNREACHABLE();
11786 }
11787 }
11788
11789 if (op2_info & MAY_BE_STRING) {
11790 ir_ref key;
11791
11792 if (if_type) {
11793 ir_IF_FALSE(if_type);
11794 if_type = IS_UNUSED;
11795 }
11796
11797 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
11798 // JIT: if (EXPECTED(Z_TYPE_P(dim) == IS_STRING))
11799 if_type = jit_if_Z_TYPE(jit, op2_addr, IS_STRING);
11800 ir_IF_TRUE(if_type);
11801 }
11802
11803 // JIT: offset_key = Z_STR_P(dim);
11804 key = jit_Z_PTR(jit, op2_addr);
11805
11806 // JIT: retval = zend_hash_find(ht, offset_key);
11807 switch (type) {
11808 case BP_JIT_IS:
11809 if (opline->op2_type != IS_CONST) {
11810 ir_ref if_num, end1, ref2;
11811
11812 if_num = ir_IF(
11813 ir_ULE(
11814 ir_LOAD_C(ir_ADD_OFFSET(key, offsetof(zend_string, val))),
11815 ir_CONST_CHAR('9')));
11816 ir_IF_TRUE_cold(if_num);
11817 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_find), ht_ref, key);
11818 end1 = ir_END();
11819 ir_IF_FALSE(if_num);
11820 ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find), ht_ref, key);
11821 ir_MERGE_WITH(end1);
11822 ref = ir_PHI_2(IR_ADDR, ref2, ref);
11823 } else {
11824 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash), ht_ref, key);
11825 }
11826 if (not_found_exit_addr) {
11827 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11828 } else {
11829 if_found = ir_IF(ref);
11830 ir_IF_FALSE(if_found);
11831 ir_END_list(*end_inputs);
11832 ir_IF_TRUE(if_found);
11833 }
11834 ir_refs_add(test_zval_values, ref);
11835 ir_refs_add(test_zval_inputs, ir_END());
11836 break;
11837 case BP_VAR_R:
11838 case BP_VAR_IS:
11839 case BP_VAR_UNSET:
11840 if (opline->op2_type != IS_CONST) {
11841 ir_ref if_num, end1, ref2;
11842
11843 if_num = ir_IF(
11844 ir_ULE(
11845 ir_LOAD_C(ir_ADD_OFFSET(key, offsetof(zend_string, val))),
11846 ir_CONST_CHAR('9')));
11847 ir_IF_TRUE_cold(if_num);
11848 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_find), ht_ref, key);
11849 end1 = ir_END();
11850 ir_IF_FALSE(if_num);
11851 ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find), ht_ref, key);
11852 ir_MERGE_WITH(end1);
11853 ref = ir_PHI_2(IR_ADDR, ref2, ref);
11854 } else {
11855 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash), ht_ref, key);
11856 }
11857 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
11858 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
11859 } else if (type == BP_VAR_IS && not_found_exit_addr) {
11860 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11861 } else if (type == BP_VAR_IS && result_type_guard) {
11862 if_found = ir_IF(ref);
11863 ir_IF_FALSE(if_found);
11864 ir_END_list(*not_found_inputs);
11865 ir_IF_TRUE(if_found);
11866 } else {
11867 if_found = ir_IF(ref);
11868 switch (type) {
11869 case BP_VAR_R:
11870 ir_IF_FALSE_cold(if_found);
11871 // JIT: zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key));
11872 jit_SET_EX_OPLINE(jit, opline);
11873 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_key, IR_FASTCALL_FUNC));
11874 ir_END_list(*end_inputs);
11875 break;
11876 case BP_VAR_IS:
11877 case BP_VAR_UNSET:
11878 ir_IF_FALSE(if_found);
11879 // JIT: retval = &EG(uninitialized_zval);
11880 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
11881 ir_END_list(*end_inputs);
11882 break;
11883 default:
11884 ZEND_UNREACHABLE();
11885 }
11886 ir_IF_TRUE(if_found);
11887 }
11888 ir_refs_add(found_inputs, ir_END());
11889 ir_refs_add(found_vals, ref);
11890 break;
11891 case BP_VAR_RW:
11892 if (opline->op2_type != IS_CONST) {
11893 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_lookup_rw), ht_ref, key);
11894 } else {
11895 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_lookup_rw), ht_ref, key);
11896 }
11897 if (not_found_exit_addr) {
11898 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11899 } else {
11900 if_found = ir_IF(ref);
11901 ir_IF_FALSE(if_found);
11902 ir_END_list(*end_inputs);
11903 ir_IF_TRUE(if_found);
11904 }
11905 ir_refs_add(found_inputs, ir_END());
11906 ir_refs_add(found_vals, ref);
11907 break;
11908 case BP_VAR_W:
11909 if (opline->op2_type != IS_CONST) {
11910 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_lookup_w), ht_ref, key);
11911 } else {
11912 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_lookup), ht_ref, key);
11913 }
11914 ir_refs_add(found_inputs, ir_END());
11915 ir_refs_add(found_vals, ref);
11916 break;
11917 default:
11918 ZEND_UNREACHABLE();
11919 }
11920 }
11921
11922 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
11923 if (if_type) {
11924 ir_IF_FALSE_cold(if_type);
11925 if_type = IS_UNDEF;
11926 }
11927 if (type != BP_VAR_RW) {
11928 jit_SET_EX_OPLINE(jit, opline);
11929 }
11930 ref = jit_ZVAL_ADDR(jit, op2_addr);
11931 switch (type) {
11932 case BP_VAR_R:
11933 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_r_helper),
11934 ht_ref,
11935 ref,
11936 jit_ZVAL_ADDR(jit, res_addr));
11937 ir_END_list(*end_inputs);
11938 break;
11939 case BP_JIT_IS:
11940 ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_fetch_dim_isset_helper), ht_ref, ref);
11941 if (not_found_exit_addr) {
11942 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11943 ir_refs_add(found_inputs, ir_END());
11944 } else if (found_exit_addr) {
11945 ir_GUARD_NOT(ref, ir_CONST_ADDR(found_exit_addr));
11946 ir_END_list(*end_inputs);
11947 } else {
11948 if_found = ir_IF(ref);
11949 ir_IF_TRUE(if_found);
11950 ir_refs_add(found_inputs, ir_END());
11951 ir_IF_FALSE(if_found);
11952 ir_END_list(*end_inputs);
11953 }
11954 break;
11955 case BP_VAR_IS:
11956 case BP_VAR_UNSET:
11957 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_is_helper),
11958 ht_ref,
11959 ref,
11960 jit_ZVAL_ADDR(jit, res_addr));
11961 ir_END_list(*end_inputs);
11962 break;
11963 case BP_VAR_RW:
11964 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_rw_helper), ht_ref, ref);
11965 if_found = ir_IF(ref);
11966 ir_IF_TRUE(if_found);
11967 ir_refs_add(found_inputs, ir_END());
11968 ir_refs_add(found_vals, ref);
11969 ir_IF_FALSE(if_found);
11970 ir_END_list(*end_inputs);
11971 break;
11972 case BP_VAR_W:
11973 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_w_helper), ht_ref, ref);
11974 if_found = ir_IF(ref);
11975 ir_IF_TRUE(if_found);
11976 ir_refs_add(found_inputs, ir_END());
11977 ir_refs_add(found_vals, ref);
11978 ir_IF_FALSE(if_found);
11979 ir_END_list(*end_inputs);
11980 break;
11981 default:
11982 ZEND_UNREACHABLE();
11983 }
11984 }
11985
11986 if (type == BP_JIT_IS
11987 && !(op2_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
11988 /* dead code */
11989 ir_END_list(*end_inputs);
11990 } else if (type == BP_JIT_IS
11991 && (op1_info & MAY_BE_ARRAY)
11992 && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))
11993 && test_zval_inputs->count) {
11994
11995 ir_MERGE_N(test_zval_inputs->count, test_zval_inputs->refs);
11996 ref = ir_PHI_N(IR_ADDR, test_zval_values->count, test_zval_values->refs);
11997
11998 if (op1_info & MAY_BE_ARRAY_OF_REF) {
11999 ref = jit_ZVAL_DEREF_ref(jit, ref);
12000 }
12001 cond = ir_GT(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_NULL));
12002 if (not_found_exit_addr) {
12003 ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
12004 ir_refs_add(found_inputs, ir_END());
12005 } else if (found_exit_addr) {
12006 ir_GUARD_NOT(cond, ir_CONST_ADDR(found_exit_addr));
12007 ir_END_list(*end_inputs);
12008 } else {
12009 ir_ref if_set = ir_IF(cond);
12010 ir_IF_FALSE(if_set);
12011 ir_END_list(*end_inputs);
12012 ir_IF_TRUE(if_set);
12013 ir_refs_add(found_inputs, ir_END());
12014 }
12015 }
12016
12017 return 1;
12018 }
12019
12020 static int zend_jit_fetch_dim_read(zend_jit_ctx *jit,
12021 const zend_op *opline,
12022 zend_ssa *ssa,
12023 const zend_ssa_op *ssa_op,
12024 uint32_t op1_info,
12025 zend_jit_addr op1_addr,
12026 bool op1_avoid_refcounting,
12027 uint32_t op2_info,
12028 zend_jit_addr op2_addr,
12029 zend_ssa_range *op2_range,
12030 uint32_t res_info,
12031 zend_jit_addr res_addr,
12032 uint8_t dim_type)
12033 {
12034 zend_jit_addr orig_op1_addr;
12035 const void *exit_addr = NULL;
12036 const void *not_found_exit_addr = NULL;
12037 bool result_type_guard = 0;
12038 bool result_avoid_refcounting = 0;
12039 uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0;
12040 int may_throw = 0;
12041 ir_ref if_type = IR_UNUSED;
12042 ir_ref end_inputs = IR_UNUSED;
12043 ir_ref not_found_inputs = IR_UNUSED;
12044
12045 orig_op1_addr = OP1_ADDR();
12046
12047 if (opline->opcode != ZEND_FETCH_DIM_IS
12048 && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
12049 && !has_concrete_type(op1_info)) {
12050 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
12051 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
12052 if (!exit_addr) {
12053 return 0;
12054 }
12055 }
12056
12057 if ((res_info & MAY_BE_GUARD)
12058 && JIT_G(current_frame)
12059 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
12060
12061 if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) {
12062 result_type_guard = 1;
12063 res_info &= ~MAY_BE_GUARD;
12064 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
12065 }
12066
12067 if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
12068 && (opline->opcode == ZEND_FETCH_LIST_R
12069 || !(opline->op1_type & (IS_VAR|IS_TMP_VAR))
12070 || op1_avoid_refcounting)
12071 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
12072 && (ssa_op+1)->op1_use == ssa_op->result_def
12073 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))
12074 && zend_jit_may_avoid_refcounting(opline+1, res_info)) {
12075 result_avoid_refcounting = 1;
12076 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
12077 }
12078
12079 if (opline->opcode == ZEND_FETCH_DIM_IS
12080 && !(res_info & MAY_BE_NULL)) {
12081 uint32_t flags = 0;
12082 uint32_t old_op1_info = 0;
12083 uint32_t old_info;
12084 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
12085 int32_t exit_point;
12086
12087 if (opline->opcode != ZEND_FETCH_LIST_R
12088 && (opline->op1_type & (IS_VAR|IS_TMP_VAR))
12089 && !op1_avoid_refcounting) {
12090 flags |= ZEND_JIT_EXIT_FREE_OP1;
12091 }
12092 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
12093 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
12094 flags |= ZEND_JIT_EXIT_FREE_OP2;
12095 }
12096
12097 if (op1_avoid_refcounting) {
12098 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
12099 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
12100 }
12101
12102 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
12103 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0);
12104 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_TYPE_ONLY);
12105 exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
12106 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
12107 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
12108 if (!not_found_exit_addr) {
12109 return 0;
12110 }
12111
12112 if (op1_avoid_refcounting) {
12113 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
12114 }
12115 }
12116 }
12117
12118 if (op1_info & MAY_BE_REF) {
12119 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
12120 ref = jit_ZVAL_DEREF_ref(jit, ref);
12121 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
12122 }
12123
12124 if (op1_info & MAY_BE_ARRAY) {
12125 ir_ref ht_ref, ref;
12126 zend_jit_addr val_addr;
12127 ir_refs *found_inputs, *found_vals;
12128
12129 ir_refs_init(found_inputs, 10);
12130 ir_refs_init(found_vals, 10);
12131
12132 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
12133 if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) {
12134 jit_guard_Z_TYPE(jit, op1_addr, IS_ARRAY, exit_addr);
12135 } else {
12136 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
12137 ir_IF_TRUE(if_type);
12138 }
12139 }
12140
12141 ht_ref = jit_Z_PTR(jit, op1_addr);
12142
12143 if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) ||
12144 (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) {
12145 may_throw = 1;
12146 }
12147
12148 if (!zend_jit_fetch_dimension_address_inner(jit, opline,
12149 (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS,
12150 op1_info, op2_info, op2_addr, op2_range, dim_type, NULL, not_found_exit_addr, exit_addr,
12151 result_type_guard, ht_ref, found_inputs, found_vals,
12152 &end_inputs, ¬_found_inputs)) {
12153 return 0;
12154 }
12155
12156 if (found_inputs->count) {
12157 ir_MERGE_N(found_inputs->count, found_inputs->refs);
12158 ref = ir_PHI_N(IR_ADDR, found_vals->count, found_vals->refs);
12159 val_addr = ZEND_ADDR_REF_ZVAL(ref);
12160
12161 if (result_type_guard) {
12162 uint8_t type = concrete_type(res_info);
12163 uint32_t flags = 0;
12164
12165 if (opline->opcode != ZEND_FETCH_LIST_R
12166 && (opline->op1_type & (IS_VAR|IS_TMP_VAR))
12167 && !op1_avoid_refcounting) {
12168 flags |= ZEND_JIT_EXIT_FREE_OP1;
12169 }
12170 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
12171 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
12172 flags |= ZEND_JIT_EXIT_FREE_OP2;
12173 }
12174
12175 val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
12176 (op1_info & MAY_BE_ARRAY_OF_REF) != 0, flags, op1_avoid_refcounting);
12177 if (!val_addr) {
12178 return 0;
12179 }
12180
12181 if (not_found_inputs) {
12182 ir_END_list(not_found_inputs);
12183 ir_MERGE_list(not_found_inputs);
12184 }
12185
12186 // ZVAL_COPY
12187 jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
12188 if (Z_MODE(res_addr) != IS_REG) {
12189 } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
12190 return 0;
12191 }
12192 } else if (op1_info & MAY_BE_ARRAY_OF_REF) {
12193 // ZVAL_COPY_DEREF
12194 ir_ref type_info = jit_Z_TYPE_INFO(jit, val_addr);
12195 if (!zend_jit_zval_copy_deref(jit, res_addr, val_addr, type_info)) {
12196 return 0;
12197 }
12198 } else {
12199 // ZVAL_COPY
12200 jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, 1);
12201 }
12202
12203 ir_END_list(end_inputs);
12204 } else if (not_found_inputs) {
12205 ir_MERGE_list(not_found_inputs);
12206 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12207 ir_END_list(end_inputs);
12208 } else if (!end_inputs && jit->ctx.control) {
12209 ir_END_list(end_inputs); /* dead code */
12210 }
12211 }
12212
12213 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
12214 if (if_type) {
12215 ir_IF_FALSE_cold(if_type);
12216 if_type = IS_UNDEF;
12217 }
12218
12219 if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) {
12220 ir_ref str_ref;
12221
12222 may_throw = 1;
12223 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) {
12224 if (exit_addr && !(op1_info & MAY_BE_OBJECT)) {
12225 jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, exit_addr);
12226 } else {
12227 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
12228 ir_IF_TRUE(if_type);
12229 }
12230 }
12231 jit_SET_EX_OPLINE(jit, opline);
12232 str_ref = jit_Z_PTR(jit, op1_addr);
12233 if (opline->opcode != ZEND_FETCH_DIM_IS) {
12234 ir_ref ref;
12235
12236 if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) {
12237 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_offset_r_helper),
12238 str_ref, jit_Z_LVAL(jit, op2_addr));
12239 } else {
12240 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_r_helper),
12241 str_ref, jit_ZVAL_ADDR(jit, op2_addr));
12242 }
12243 jit_set_Z_PTR(jit, res_addr, ref);
12244 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
12245 } else {
12246 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_is_helper),
12247 str_ref,
12248 jit_ZVAL_ADDR(jit, op2_addr),
12249 jit_ZVAL_ADDR(jit, res_addr));
12250 }
12251 ir_END_list(end_inputs);
12252 }
12253
12254 if (op1_info & MAY_BE_OBJECT) {
12255 ir_ref arg2;
12256
12257 if (if_type) {
12258 ir_IF_FALSE_cold(if_type);
12259 if_type = IS_UNDEF;
12260 }
12261
12262 may_throw = 1;
12263 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) {
12264 if (exit_addr) {
12265 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
12266 } else {
12267 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
12268 ir_IF_TRUE(if_type);
12269 }
12270 }
12271
12272 jit_SET_EX_OPLINE(jit, opline);
12273 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
12274 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
12275 arg2 = ir_CONST_ADDR(Z_ZV(op2_addr)+1);
12276 } else {
12277 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
12278 }
12279
12280 if (opline->opcode != ZEND_FETCH_DIM_IS) {
12281 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_r_helper),
12282 jit_ZVAL_ADDR(jit, op1_addr),
12283 arg2,
12284 jit_ZVAL_ADDR(jit, res_addr));
12285 } else {
12286 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_is_helper),
12287 jit_ZVAL_ADDR(jit, op1_addr),
12288 arg2,
12289 jit_ZVAL_ADDR(jit, res_addr));
12290 }
12291
12292 ir_END_list(end_inputs);
12293 }
12294
12295 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))
12296 && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) {
12297
12298 if (if_type) {
12299 ir_IF_FALSE_cold(if_type);
12300 if_type = IS_UNDEF;
12301 }
12302
12303 if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) {
12304 jit_SET_EX_OPLINE(jit, opline);
12305 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) {
12306 may_throw = 1;
12307 zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op1_addr), opline->op1.var, NULL, 0, 1, 0);
12308 }
12309
12310 if (op2_info & MAY_BE_UNDEF) {
12311 may_throw = 1;
12312 zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op2_addr), opline->op2.var, NULL, 0, 1, 0);
12313 }
12314 }
12315
12316 if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) {
12317 ir_ref ref;
12318
12319 may_throw = 1;
12320 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
12321 ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
12322 } else {
12323 jit_SET_EX_OPLINE(jit, opline);
12324 ref = jit_ZVAL_ADDR(jit, op1_addr);
12325 }
12326 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_array_access), ref);
12327 }
12328
12329 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12330 ir_END_list(end_inputs);
12331 }
12332 }
12333
12334 if (end_inputs) {
12335 ir_MERGE_list(end_inputs);
12336
12337 #ifdef ZEND_JIT_USE_RC_INFERENCE
12338 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
12339 /* Magic offsetGet() may increase refcount of the key */
12340 op2_info |= MAY_BE_RCN;
12341 }
12342 #endif
12343
12344 if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) {
12345 if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) {
12346 may_throw = 1;
12347 }
12348 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
12349 }
12350 if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) {
12351 if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) {
12352 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
12353 may_throw = 1;
12354 }
12355 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
12356 }
12357 }
12358
12359 if (may_throw) {
12360 zend_jit_check_exception(jit);
12361 }
12362 } else if (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) {
12363 ir_BEGIN(IR_UNUSED); /* unreachable tail */
12364 }
12365
12366 return 1;
12367 }
12368
12369 static zend_jit_addr zend_jit_prepare_array_update(zend_jit_ctx *jit,
12370 const zend_op *opline,
12371 uint32_t op1_info,
12372 zend_jit_addr op1_addr,
12373 ir_ref *if_type,
12374 ir_ref *ht_ref,
12375 int *may_throw)
12376 {
12377 ir_ref ref = IR_UNUSED;
12378 ir_ref array_reference_end = IR_UNUSED, array_reference_ref = IR_UNUSED;
12379 ir_refs *array_inputs, *array_values;
12380
12381 ir_refs_init(array_inputs, 4);
12382 ir_refs_init(array_values, 4);
12383
12384 ref = jit_ZVAL_ADDR(jit, op1_addr);
12385 if (op1_info & MAY_BE_REF) {
12386 ir_ref if_reference, if_array, end1, ref2;
12387
12388 *may_throw = 1;
12389 if_reference = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
12390 ir_IF_FALSE(if_reference);
12391 end1 = ir_END();
12392 ir_IF_TRUE_cold(if_reference);
12393 array_reference_ref = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val));
12394 if_array = jit_if_Z_TYPE_ref(jit, array_reference_ref, ir_CONST_U8(IS_ARRAY));
12395 ir_IF_TRUE(if_array);
12396 array_reference_end = ir_END();
12397 ir_IF_FALSE_cold(if_array);
12398 if (opline->opcode != ZEND_FETCH_DIM_RW && opline->opcode != ZEND_ASSIGN_DIM_OP) {
12399 jit_SET_EX_OPLINE(jit, opline);
12400 }
12401 ref2 = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_prepare_assign_dim_ref), ref);
12402 ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
12403
12404 ir_MERGE_WITH(end1);
12405 ref = ir_PHI_2(IR_ADDR, ref2, ref);
12406 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
12407 }
12408
12409 if (op1_info & MAY_BE_ARRAY) {
12410 ir_ref op1_ref = ref;
12411
12412 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
12413 *if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
12414 ir_IF_TRUE(*if_type);
12415 }
12416 if (array_reference_end) {
12417 ir_MERGE_WITH(array_reference_end);
12418 op1_ref = ir_PHI_2(IR_ADDR, ref, array_reference_ref);
12419 }
12420 // JIT: SEPARATE_ARRAY()
12421 ref = jit_Z_PTR_ref(jit, op1_ref);
12422 if (RC_MAY_BE_N(op1_info)) {
12423 if (RC_MAY_BE_1(op1_info)) {
12424 ir_ref if_refcount_1 = ir_IF(ir_EQ(jit_GC_REFCOUNT(jit, ref), ir_CONST_U32(1)));
12425 ir_IF_TRUE(if_refcount_1);
12426 ir_refs_add(array_inputs, ir_END());
12427 ir_refs_add(array_values, ref);
12428 ir_IF_FALSE(if_refcount_1);
12429 }
12430 ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_zval_array_dup), op1_ref);
12431 }
12432 if (array_inputs->count || (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
12433 ir_refs_add(array_inputs, ir_END());
12434 ir_refs_add(array_values, ref);
12435 }
12436 }
12437
12438 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
12439 if (*if_type) {
12440 ir_IF_FALSE_cold(*if_type);
12441 *if_type = IR_UNUSED;
12442 }
12443 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
12444 *if_type = ir_IF(ir_LE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL)));
12445 ir_IF_TRUE(*if_type);
12446 }
12447 if ((op1_info & MAY_BE_UNDEF)
12448 && (opline->opcode == ZEND_FETCH_DIM_RW || opline->opcode == ZEND_ASSIGN_DIM_OP)) {
12449 ir_ref end1 = IR_UNUSED;
12450
12451 *may_throw = 1;
12452 if (op1_info & MAY_BE_NULL) {
12453 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
12454 ir_IF_TRUE(if_def);
12455 end1 = ir_END();
12456 ir_IF_FALSE(if_def);
12457 }
12458 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
12459 if (end1) {
12460 ir_MERGE_WITH(end1);
12461 }
12462 }
12463 // JIT: ZVAL_ARR(container, zend_new_array(8));
12464 ref = ir_CALL_1(IR_ADDR,
12465 jit_STUB_FUNC_ADDR(jit, jit_stub_new_array, IR_FASTCALL_FUNC),
12466 jit_ZVAL_ADDR(jit, op1_addr));
12467 if (array_inputs->count) {
12468 ir_refs_add(array_inputs, ir_END());
12469 ir_refs_add(array_values, ref);
12470 }
12471 }
12472
12473 if (array_inputs->count) {
12474 ir_MERGE_N(array_inputs->count, array_inputs->refs);
12475 ref = ir_PHI_N(IR_ADDR, array_values->count, array_values->refs);
12476 }
12477
12478 *ht_ref = ref;
12479 return op1_addr;
12480 }
12481
12482 static int zend_jit_fetch_dim(zend_jit_ctx *jit,
12483 const zend_op *opline,
12484 uint32_t op1_info,
12485 zend_jit_addr op1_addr,
12486 uint32_t op2_info,
12487 zend_jit_addr op2_addr,
12488 zend_ssa_range *op2_range,
12489 zend_jit_addr res_addr,
12490 uint8_t dim_type)
12491 {
12492 int may_throw = 0;
12493 ir_ref end_inputs = IR_UNUSED;
12494 ir_ref ref, if_type = IR_UNUSED, ht_ref;
12495
12496 if (opline->opcode == ZEND_FETCH_DIM_RW) {
12497 jit_SET_EX_OPLINE(jit, opline);
12498 }
12499
12500 op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
12501
12502 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
12503 ir_refs *found_inputs, *found_vals;
12504
12505 ir_refs_init(found_inputs, 8);
12506 ir_refs_init(found_vals, 8);
12507
12508 if (opline->op2_type == IS_UNUSED) {
12509 ir_ref if_ok;
12510
12511 may_throw = 1;
12512 // JIT:var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
12513 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
12514 ht_ref, jit_EG(uninitialized_zval));
12515
12516 // JIT: if (UNEXPECTED(!var_ptr)) {
12517 if_ok = ir_IF(ref);
12518 ir_IF_FALSE_cold(if_ok);
12519 if (opline->opcode != ZEND_FETCH_DIM_RW) {
12520 jit_SET_EX_OPLINE(jit, opline);
12521 }
12522 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
12523 ir_END_list(end_inputs);
12524
12525 ir_IF_TRUE(if_ok);
12526 jit_set_Z_PTR(jit, res_addr, ref);
12527 jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
12528
12529 ir_END_list(end_inputs);
12530 } else {
12531 uint32_t type;
12532
12533 switch (opline->opcode) {
12534 case ZEND_FETCH_DIM_W:
12535 case ZEND_FETCH_LIST_W:
12536 type = BP_VAR_W;
12537 break;
12538 case ZEND_FETCH_DIM_RW:
12539 may_throw = 1;
12540 type = BP_VAR_RW;
12541 break;
12542 case ZEND_FETCH_DIM_UNSET:
12543 type = BP_VAR_UNSET;
12544 break;
12545 default:
12546 ZEND_UNREACHABLE();
12547 }
12548
12549 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
12550 may_throw = 1;
12551 }
12552 if (!zend_jit_fetch_dimension_address_inner(jit, opline, type, op1_info,
12553 op2_info, op2_addr, op2_range, dim_type, NULL, NULL, NULL,
12554 0, ht_ref, found_inputs, found_vals, &end_inputs, NULL)) {
12555 return 0;
12556 }
12557
12558 if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
12559 if (end_inputs) {
12560 ir_MERGE_list(end_inputs);
12561 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12562 end_inputs = ir_END();
12563 }
12564 } else if (!(op2_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
12565 /* impossible dead path */
12566 end_inputs = ir_END();
12567 } else {
12568 ZEND_ASSERT(end_inputs == IR_UNUSED);
12569 }
12570
12571 if (found_inputs->count) {
12572 ir_MERGE_N(found_inputs->count, found_inputs->refs);
12573 ref = ir_PHI_N(IR_ADDR, found_vals->count, found_vals->refs);
12574 jit_set_Z_PTR(jit, res_addr, ref);
12575 jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
12576 ir_END_list(end_inputs);
12577 }
12578
12579 }
12580 }
12581
12582 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
12583 ir_ref arg2;
12584
12585 may_throw = 1;
12586
12587 if (if_type) {
12588 ir_IF_FALSE(if_type);
12589 if_type = IR_UNUSED;
12590 }
12591
12592 if (opline->opcode != ZEND_FETCH_DIM_RW) {
12593 jit_SET_EX_OPLINE(jit, opline);
12594 }
12595
12596 if (opline->op2_type == IS_UNUSED) {
12597 arg2 = IR_NULL;
12598 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
12599 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
12600 arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
12601 } else {
12602 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
12603 }
12604
12605 switch (opline->opcode) {
12606 case ZEND_FETCH_DIM_W:
12607 case ZEND_FETCH_LIST_W:
12608 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_w_helper),
12609 jit_ZVAL_ADDR(jit, op1_addr),
12610 arg2,
12611 jit_ZVAL_ADDR(jit, res_addr));
12612 break;
12613 case ZEND_FETCH_DIM_RW:
12614 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_rw_helper),
12615 jit_ZVAL_ADDR(jit, op1_addr),
12616 arg2,
12617 jit_ZVAL_ADDR(jit, res_addr));
12618 break;
12619 // case ZEND_FETCH_DIM_UNSET:
12620 // | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0
12621 // break;
12622 default:
12623 ZEND_UNREACHABLE();
12624 }
12625
12626 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
12627 ir_END_list(end_inputs);
12628 }
12629 }
12630
12631 #ifdef ZEND_JIT_USE_RC_INFERENCE
12632 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
12633 /* ASSIGN_DIM may increase refcount of the key */
12634 op2_info |= MAY_BE_RCN;
12635 }
12636 #endif
12637
12638 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
12639 && (op2_info & MAY_HAVE_DTOR)
12640 && (op2_info & MAY_BE_RC1)) {
12641 may_throw = 1;
12642 }
12643
12644 if (end_inputs) {
12645 ir_MERGE_list(end_inputs);
12646 }
12647
12648 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
12649
12650 if (may_throw) {
12651 zend_jit_check_exception(jit);
12652 }
12653
12654 return 1;
12655 }
12656
12657 static int zend_jit_isset_isempty_dim(zend_jit_ctx *jit,
12658 const zend_op *opline,
12659 uint32_t op1_info,
12660 zend_jit_addr op1_addr,
12661 bool op1_avoid_refcounting,
12662 uint32_t op2_info,
12663 zend_jit_addr op2_addr,
12664 zend_ssa_range *op2_range,
12665 uint8_t dim_type,
12666 int may_throw,
12667 uint8_t smart_branch_opcode,
12668 uint32_t target_label,
12669 uint32_t target_label2,
12670 const void *exit_addr)
12671 {
12672 zend_jit_addr res_addr;
12673 ir_ref if_type = IR_UNUSED;
12674 ir_ref false_inputs = IR_UNUSED, end_inputs = IR_UNUSED;
12675 ir_refs *true_inputs;
12676
12677 ir_refs_init(true_inputs, 8);
12678
12679 // TODO: support for empty() ???
12680 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY));
12681
12682 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
12683
12684 if (op1_info & MAY_BE_REF) {
12685 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
12686 ref = jit_ZVAL_DEREF_ref(jit, ref);
12687 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
12688 }
12689
12690 if (op1_info & MAY_BE_ARRAY) {
12691 const void *found_exit_addr = NULL;
12692 const void *not_found_exit_addr = NULL;
12693 ir_ref ht_ref;
12694
12695 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
12696 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
12697 ir_IF_TRUE(if_type);
12698 }
12699
12700 ht_ref = jit_Z_PTR(jit, op1_addr);
12701
12702 if (exit_addr
12703 && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY))
12704 && !may_throw
12705 && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting)
12706 && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
12707 if (smart_branch_opcode == ZEND_JMPNZ) {
12708 found_exit_addr = exit_addr;
12709 } else {
12710 not_found_exit_addr = exit_addr;
12711 }
12712 }
12713 if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_JIT_IS, op1_info,
12714 op2_info, op2_addr, op2_range, dim_type, found_exit_addr, not_found_exit_addr, NULL,
12715 0, ht_ref, true_inputs, NULL, &false_inputs, NULL)) {
12716 return 0;
12717 }
12718
12719 if (found_exit_addr) {
12720 ir_MERGE_list(false_inputs);
12721 return 1;
12722 } else if (not_found_exit_addr) {
12723 ir_MERGE_N(true_inputs->count, true_inputs->refs);
12724 return 1;
12725 }
12726 }
12727
12728 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
12729 if (if_type) {
12730 ir_IF_FALSE(if_type);
12731 if_type = IR_UNUSED;
12732 }
12733
12734 if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) {
12735 ir_ref ref, arg1, arg2, if_true;
12736
12737 jit_SET_EX_OPLINE(jit, opline);
12738 arg1 = jit_ZVAL_ADDR(jit, op1_addr);
12739 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
12740 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
12741 arg2 = ir_CONST_ADDR(Z_ZV(op2_addr)+1);
12742 } else {
12743 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
12744 }
12745 ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_isset_dim_helper), arg1, arg2);
12746 if_true = ir_IF(ref);
12747 ir_IF_TRUE(if_true);
12748 ir_refs_add(true_inputs, ir_END());
12749 ir_IF_FALSE(if_true);
12750 ir_END_list(false_inputs);
12751 } else {
12752 if (op2_info & MAY_BE_UNDEF) {
12753 ir_ref end1 = IR_UNUSED;
12754
12755 if (op2_info & MAY_BE_ANY) {
12756 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op2_addr));
12757 ir_IF_TRUE(if_def);
12758 end1 = ir_END();
12759 ir_IF_FALSE(if_def);
12760 }
12761 jit_SET_EX_OPLINE(jit, opline);
12762 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op2.var));
12763 if (end1) {
12764 ir_MERGE_WITH(end1);
12765 }
12766 }
12767 ir_END_list(false_inputs);
12768 }
12769 }
12770
12771 #ifdef ZEND_JIT_USE_RC_INFERENCE
12772 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
12773 /* Magic offsetExists() may increase refcount of the key */
12774 op2_info |= MAY_BE_RCN;
12775 }
12776 #endif
12777
12778 if (true_inputs->count) {
12779 ir_MERGE_N(true_inputs->count, true_inputs->refs);
12780
12781 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
12782 if (!op1_avoid_refcounting) {
12783 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
12784 }
12785 if (may_throw) {
12786 zend_jit_check_exception_undef_result(jit, opline);
12787 }
12788 if (!(opline->extended_value & ZEND_ISEMPTY)) {
12789 if (exit_addr) {
12790 if (smart_branch_opcode == ZEND_JMPNZ) {
12791 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
12792 } else {
12793 ir_END_list(end_inputs);
12794 }
12795 } else if (smart_branch_opcode) {
12796 if (smart_branch_opcode == ZEND_JMPZ) {
12797 _zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
12798 } else if (smart_branch_opcode == ZEND_JMPNZ) {
12799 _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
12800 } else {
12801 ZEND_UNREACHABLE();
12802 }
12803 } else {
12804 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
12805 ir_END_list(end_inputs);
12806 }
12807 } else {
12808 ZEND_UNREACHABLE(); // TODO: support for empty()
12809 }
12810 }
12811
12812 ir_MERGE_list(false_inputs);
12813 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
12814 if (!op1_avoid_refcounting) {
12815 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
12816 }
12817 if (may_throw) {
12818 zend_jit_check_exception_undef_result(jit, opline);
12819 }
12820 if (!(opline->extended_value & ZEND_ISEMPTY)) {
12821 if (exit_addr) {
12822 if (smart_branch_opcode == ZEND_JMPZ) {
12823 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
12824 } else {
12825 ir_END_list(end_inputs);
12826 }
12827 } else if (smart_branch_opcode) {
12828 if (smart_branch_opcode == ZEND_JMPZ) {
12829 _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
12830 } else if (smart_branch_opcode == ZEND_JMPNZ) {
12831 _zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
12832 } else {
12833 ZEND_UNREACHABLE();
12834 }
12835 } else {
12836 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
12837 ir_END_list(end_inputs);
12838 }
12839 } else {
12840 ZEND_UNREACHABLE(); // TODO: support for empty()
12841 }
12842
12843 if (!exit_addr && smart_branch_opcode) {
12844 jit->b = -1;
12845 } else {
12846 ir_MERGE_list(end_inputs);
12847 }
12848
12849 return 1;
12850 }
12851
12852 static int zend_jit_assign_dim(zend_jit_ctx *jit,
12853 const zend_op *opline,
12854 uint32_t op1_info,
12855 zend_jit_addr op1_addr,
12856 uint32_t op2_info,
12857 zend_jit_addr op2_addr,
12858 zend_ssa_range *op2_range,
12859 uint32_t val_info,
12860 zend_jit_addr op3_addr,
12861 zend_jit_addr op3_def_addr,
12862 zend_jit_addr res_addr,
12863 uint8_t dim_type,
12864 int may_throw)
12865 {
12866 ir_ref if_type = IR_UNUSED;
12867 ir_ref end_inputs = IR_UNUSED, ht_ref;
12868
12869 if (op3_addr != op3_def_addr && op3_def_addr) {
12870 if (!zend_jit_update_regs(jit, (opline+1)->op1.var, op3_addr, op3_def_addr, val_info)) {
12871 return 0;
12872 }
12873 if (Z_MODE(op3_def_addr) == IS_REG && Z_MODE(op3_addr) != IS_REG) {
12874 op3_addr = op3_def_addr;
12875 }
12876 }
12877
12878 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) {
12879 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
12880 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
12881
12882 if (!exit_addr) {
12883 return 0;
12884 }
12885
12886 jit_guard_not_Z_TYPE(jit, op3_addr, IS_UNDEF, exit_addr);
12887
12888 val_info &= ~MAY_BE_UNDEF;
12889 }
12890
12891 op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
12892
12893 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
12894 if (opline->op2_type == IS_UNUSED) {
12895 uint32_t var_info = MAY_BE_NULL;
12896 ir_ref if_ok, ref;
12897 zend_jit_addr var_addr;
12898
12899 // JIT: var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
12900 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
12901 ht_ref, jit_EG(uninitialized_zval));
12902
12903 // JIT: if (UNEXPECTED(!var_ptr)) {
12904 if_ok = ir_IF(ref);
12905 ir_IF_FALSE_cold(if_ok);
12906
12907 // JIT: zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
12908 jit_SET_EX_OPLINE(jit, opline);
12909 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
12910
12911 ir_END_list(end_inputs);
12912
12913 ir_IF_TRUE(if_ok);
12914 var_addr = ZEND_ADDR_REF_ZVAL(ref);
12915 if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
12916 return 0;
12917 }
12918 } else {
12919 uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
12920 zend_jit_addr var_addr;
12921 ir_ref ref;
12922 ir_refs *found_inputs, *found_values;
12923
12924 ir_refs_init(found_inputs, 8);
12925 ir_refs_init(found_values, 8);
12926
12927 if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_W, op1_info,
12928 op2_info, op2_addr, op2_range, dim_type, NULL, NULL, NULL,
12929 0, ht_ref, found_inputs, found_values, &end_inputs, NULL)) {
12930 return 0;
12931 }
12932
12933 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
12934 var_info |= MAY_BE_REF;
12935 }
12936 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
12937 var_info |= MAY_BE_RC1;
12938 }
12939
12940 if (found_inputs->count) {
12941 ir_MERGE_N(found_inputs->count, found_inputs->refs);
12942 ref = ir_PHI_N(IR_ADDR, found_values->count, found_values->refs);
12943 var_addr = ZEND_ADDR_REF_ZVAL(ref);
12944
12945 // JIT: value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE);
12946 if (opline->op1_type == IS_VAR
12947 && Z_MODE(op3_addr) != IS_REG
12948 && (res_addr == 0 || Z_MODE(res_addr) != IS_REG)) {
12949 ZEND_ASSERT(opline->result_type == IS_UNUSED);
12950 if (!zend_jit_assign_to_variable_call(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
12951 return 0;
12952 }
12953 } else {
12954 if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) {
12955 return 0;
12956 }
12957 }
12958 }
12959 }
12960
12961 ir_END_list(end_inputs);
12962 }
12963
12964 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
12965 ir_ref arg2, arg4;
12966
12967 if (if_type) {
12968 ir_IF_FALSE_cold(if_type);
12969 if_type = IR_UNUSED;
12970 }
12971
12972 jit_SET_EX_OPLINE(jit, opline);
12973
12974 if (opline->op2_type == IS_UNUSED) {
12975 arg2 = IR_NULL;
12976 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
12977 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
12978 arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
12979 } else {
12980 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
12981 }
12982
12983 if (opline->result_type == IS_UNUSED) {
12984 arg4 = IR_NULL;
12985 } else {
12986 arg4 = jit_ZVAL_ADDR(jit, res_addr);
12987 }
12988 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_dim_helper),
12989 jit_ZVAL_ADDR(jit, op1_addr),
12990 arg2,
12991 jit_ZVAL_ADDR(jit, op3_addr),
12992 arg4);
12993
12994 #ifdef ZEND_JIT_USE_RC_INFERENCE
12995 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) {
12996 /* ASSIGN_DIM may increase refcount of the value */
12997 val_info |= MAY_BE_RCN;
12998 }
12999 #endif
13000
13001 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, NULL);
13002
13003 ir_END_list(end_inputs);
13004 }
13005
13006 #ifdef ZEND_JIT_USE_RC_INFERENCE
13007 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
13008 /* ASSIGN_DIM may increase refcount of the key */
13009 op2_info |= MAY_BE_RCN;
13010 }
13011 #endif
13012
13013 ir_MERGE_list(end_inputs);
13014 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
13015
13016 if (may_throw) {
13017 zend_jit_check_exception(jit);
13018 }
13019
13020 return 1;
13021 }
13022
13023 static int zend_jit_assign_dim_op(zend_jit_ctx *jit,
13024 const zend_op *opline,
13025 uint32_t op1_info,
13026 uint32_t op1_def_info,
13027 zend_jit_addr op1_addr,
13028 uint32_t op2_info,
13029 zend_jit_addr op2_addr,
13030 zend_ssa_range *op2_range,
13031 uint32_t op1_data_info,
13032 zend_jit_addr op3_addr,
13033 zend_ssa_range *op1_data_range,
13034 uint8_t dim_type,
13035 int may_throw)
13036 {
13037 zend_jit_addr var_addr = IS_UNUSED;
13038 const void *not_found_exit_addr = NULL;
13039 uint32_t var_info = MAY_BE_NULL;
13040 ir_ref if_type = IS_UNUSED;
13041 ir_ref end_inputs = IR_UNUSED, ht_ref;
13042 bool emit_fast_path = 1;
13043
13044 ZEND_ASSERT(opline->result_type == IS_UNUSED);
13045
13046 if (may_throw) {
13047 jit_SET_EX_OPLINE(jit, opline);
13048 }
13049
13050 op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
13051
13052 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
13053 uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0);
13054
13055 if (opline->op2_type == IS_UNUSED) {
13056 var_info = MAY_BE_NULL;
13057 ir_ref if_ok, ref;
13058
13059 // JIT: var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
13060 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
13061 ht_ref, jit_EG(uninitialized_zval));
13062
13063 // JIT: if (UNEXPECTED(!var_ptr)) {
13064 if_ok = ir_IF(ref);
13065 ir_IF_FALSE_cold(if_ok);
13066
13067 // JIT: zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
13068 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
13069
13070 ir_END_list(end_inputs);
13071
13072 ir_IF_TRUE(if_ok);
13073 var_addr = ZEND_ADDR_REF_ZVAL(ref);
13074 } else {
13075 ir_ref ref;
13076 ir_refs *found_inputs, *found_values;
13077
13078 ir_refs_init(found_inputs, 8);
13079 ir_refs_init(found_values, 8);
13080
13081 var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
13082 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
13083 var_info |= MAY_BE_REF;
13084 }
13085 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
13086 var_info |= MAY_BE_RC1;
13087 }
13088
13089 if (dim_type != IS_UNKNOWN
13090 && dim_type != IS_UNDEF
13091 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
13092 && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))
13093 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
13094 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
13095 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
13096 if (!not_found_exit_addr) {
13097 return 0;
13098 }
13099 }
13100
13101 if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_RW, op1_info,
13102 op2_info, op2_addr, op2_range, dim_type, NULL, not_found_exit_addr, NULL,
13103 0, ht_ref, found_inputs, found_values, &end_inputs, NULL)) {
13104 return 0;
13105 }
13106
13107 if (found_inputs->count) {
13108 ir_MERGE_N(found_inputs->count, found_inputs->refs);
13109 ref = ir_PHI_N(IR_ADDR, found_values->count, found_values->refs);
13110 var_addr = ZEND_ADDR_REF_ZVAL(ref);
13111
13112 if (not_found_exit_addr && dim_type != IS_REFERENCE) {
13113 jit_guard_Z_TYPE(jit, var_addr, dim_type, not_found_exit_addr);
13114 var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
13115 }
13116 if (var_info & MAY_BE_REF) {
13117 binary_op_type binary_op = get_binary_op(opline->extended_value);
13118 ir_ref if_ref, if_typed, noref_path, ref_path, ref, reference, ref2, arg2;
13119
13120 ref = jit_ZVAL_ADDR(jit, var_addr);
13121 if_ref = jit_if_Z_TYPE(jit, var_addr, IS_REFERENCE);
13122 ir_IF_FALSE(if_ref);
13123 noref_path = ir_END();
13124 ir_IF_TRUE(if_ref);
13125
13126 reference = jit_Z_PTR_ref(jit, ref);
13127 ref2 = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
13128 if_typed = jit_if_TYPED_REF(jit, reference);
13129 ir_IF_FALSE(if_typed);
13130 ref_path = ir_END();
13131 ir_IF_TRUE_cold(if_typed);
13132
13133 arg2 = jit_ZVAL_ADDR(jit, op3_addr);
13134 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
13135 reference, arg2, ir_CONST_FC_FUNC(binary_op));
13136
13137 ir_END_list(end_inputs);
13138
13139 ir_MERGE_2(noref_path, ref_path);
13140 ref = ir_PHI_2(IR_ADDR, ref, ref2);
13141 var_addr = ZEND_ADDR_REF_ZVAL(ref);
13142 }
13143 } else {
13144 emit_fast_path = 0;
13145 }
13146 }
13147
13148 if (emit_fast_path) {
13149 uint8_t val_op_type = (opline+1)->op1_type;
13150
13151 if (val_op_type & (IS_TMP_VAR|IS_VAR)) {
13152 /* prevent FREE_OP in the helpers */
13153 val_op_type = IS_CV;
13154 }
13155
13156 switch (opline->extended_value) {
13157 case ZEND_ADD:
13158 case ZEND_SUB:
13159 case ZEND_MUL:
13160 case ZEND_DIV:
13161 if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info,
13162 1 /* may overflow */, may_throw)) {
13163 return 0;
13164 }
13165 break;
13166 case ZEND_BW_OR:
13167 case ZEND_BW_AND:
13168 case ZEND_BW_XOR:
13169 case ZEND_SL:
13170 case ZEND_SR:
13171 case ZEND_MOD:
13172 if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
13173 IS_CV, opline->op1, var_addr, var_info, NULL,
13174 val_op_type, (opline+1)->op1, op3_addr, op1_data_info,
13175 op1_data_range,
13176 0, var_addr, var_def_info, var_info, may_throw)) {
13177 return 0;
13178 }
13179 break;
13180 case ZEND_CONCAT:
13181 if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr,
13182 may_throw)) {
13183 return 0;
13184 }
13185 break;
13186 default:
13187 ZEND_UNREACHABLE();
13188 }
13189
13190 ir_END_list(end_inputs);
13191 }
13192 }
13193
13194 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
13195 binary_op_type binary_op;
13196 ir_ref arg2;
13197
13198 if (if_type) {
13199 ir_IF_FALSE_cold(if_type);
13200 if_type = IS_UNUSED;
13201 }
13202
13203 if (opline->op2_type == IS_UNUSED) {
13204 arg2 = IR_NULL;
13205 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
13206 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
13207 arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
13208 } else {
13209 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
13210 }
13211 binary_op = get_binary_op(opline->extended_value);
13212 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_dim_op_helper),
13213 jit_ZVAL_ADDR(jit, op1_addr),
13214 arg2,
13215 jit_ZVAL_ADDR(jit, op3_addr),
13216 ir_CONST_FC_FUNC(binary_op));
13217 ir_END_list(end_inputs);
13218 }
13219
13220 if (end_inputs) {
13221 ir_MERGE_list(end_inputs);
13222 }
13223
13224 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
13225 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
13226 if (may_throw) {
13227 zend_jit_check_exception(jit);
13228 }
13229
13230 return 1;
13231 }
13232
13233 static int zend_jit_fe_reset(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
13234 {
13235 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
13236
13237 // JIT: ZVAL_COPY(res, value);
13238 if (opline->op1_type == IS_CONST) {
13239 zval *zv = RT_CONSTANT(opline, opline->op1);
13240
13241 jit_ZVAL_COPY_CONST(jit, res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
13242 } else {
13243 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
13244
13245 jit_ZVAL_COPY(jit, res_addr, -1, op1_addr, op1_info, opline->op1_type == IS_CV);
13246 }
13247
13248 // JIT: Z_FE_POS_P(res) = 0;
13249 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->result.var + offsetof(zval, u2.fe_pos)), ir_CONST_U32(0));
13250
13251 return 1;
13252 }
13253
13254 static int zend_jit_packed_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint32_t op_info)
13255 {
13256 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
13257 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
13258 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
13259 ir_ref ref;
13260
13261 if (!exit_addr) {
13262 return 0;
13263 }
13264
13265 ref = ir_AND_U32(
13266 ir_LOAD_U32(ir_ADD_OFFSET(jit_Z_PTR(jit, addr), offsetof(zend_array, u.flags))),
13267 ir_CONST_U32(HASH_FLAG_PACKED));
13268 if (op_info & MAY_BE_ARRAY_PACKED) {
13269 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
13270 } else {
13271 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
13272 }
13273
13274 return 1;
13275 }
13276
13277 static int zend_jit_fe_fetch(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, uint8_t exit_opcode, const void *exit_addr)
13278 {
13279 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
13280 ir_ref ref, ht_ref, hash_pos_ref, packed_pos_ref, hash_p_ref = IR_UNUSED, packed_p_ref = IR_UNUSED, if_packed = IR_UNUSED;
13281 ir_ref if_def_hash = IR_UNUSED, if_def_packed = IR_UNUSED;
13282 ir_ref exit_inputs = IR_UNUSED;
13283
13284 if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) {
13285 /* empty array */
13286 if (exit_addr) {
13287 if (exit_opcode == ZEND_JMP) {
13288 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
13289 }
13290 } else {
13291 zend_basic_block *bb;
13292
13293 ZEND_ASSERT(jit->b >= 0);
13294 bb = &jit->ssa->cfg.blocks[jit->b];
13295 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
13296 jit->b = -1;
13297 }
13298 return 1;
13299 }
13300
13301 // JIT: array = EX_VAR(opline->op1.var);
13302 // JIT: fe_ht = Z_ARRVAL_P(array);
13303 ht_ref = jit_Z_PTR(jit, op1_addr);
13304
13305 if (op1_info & MAY_BE_PACKED_GUARD) {
13306 if (!zend_jit_packed_guard(jit, opline, opline->op1.var, op1_info)) {
13307 return 0;
13308 }
13309 }
13310
13311 // JIT: pos = Z_FE_POS_P(array);
13312 hash_pos_ref = packed_pos_ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)));
13313
13314 if (MAY_BE_HASH(op1_info)) {
13315 ir_ref loop_ref, pos2_ref, p2_ref;
13316
13317 if (MAY_BE_PACKED(op1_info)) {
13318 ref = ir_AND_U32(
13319 ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
13320 ir_CONST_U32(HASH_FLAG_PACKED));
13321 if_packed = ir_IF(ref);
13322 ir_IF_FALSE(if_packed);
13323 }
13324
13325 // JIT: p = fe_ht->arData + pos;
13326 if (sizeof(void*) == 8) {
13327 ref = ir_ZEXT_A(hash_pos_ref);
13328 } else {
13329 ref = ir_BITCAST_A(hash_pos_ref);
13330 }
13331 hash_p_ref = ir_ADD_A(
13332 ir_MUL_A(ref, ir_CONST_ADDR(sizeof(Bucket))),
13333 ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arData))));
13334
13335 loop_ref = ir_LOOP_BEGIN(ir_END());
13336 hash_pos_ref = ir_PHI_2(IR_U32, hash_pos_ref, IR_UNUSED);
13337 hash_p_ref = ir_PHI_2(IR_ADDR, hash_p_ref, IR_UNUSED);
13338
13339 // JIT: if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
13340 ref = ir_ULT(hash_pos_ref,
13341 ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed))));
13342
13343 // JIT: ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
13344 // JIT: ZEND_VM_CONTINUE();
13345
13346 if (exit_addr) {
13347 if (exit_opcode == ZEND_JMP) {
13348 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
13349 } else {
13350 ir_ref if_fit = ir_IF(ref);
13351 ir_IF_FALSE(if_fit);
13352 ir_END_list(exit_inputs);
13353 ir_IF_TRUE(if_fit);
13354 }
13355 } else {
13356 ir_ref if_fit = ir_IF(ref);
13357 ir_IF_FALSE(if_fit);
13358 ir_END_list(exit_inputs);
13359 ir_IF_TRUE(if_fit);
13360 }
13361
13362 // JIT: pos++;
13363 pos2_ref = ir_ADD_U32(hash_pos_ref, ir_CONST_U32(1));
13364
13365 // JIT: value_type = Z_TYPE_INFO_P(value);
13366 // JIT: if (EXPECTED(value_type != IS_UNDEF)) {
13367 if (!exit_addr || exit_opcode == ZEND_JMP) {
13368 if_def_hash = ir_IF(jit_Z_TYPE_ref(jit, hash_p_ref));
13369 ir_IF_FALSE(if_def_hash);
13370 } else {
13371 ir_GUARD_NOT(jit_Z_TYPE_ref(jit, hash_p_ref), ir_CONST_ADDR(exit_addr));
13372 }
13373
13374 // JIT: p++;
13375 p2_ref = ir_ADD_OFFSET(hash_p_ref, sizeof(Bucket));
13376
13377 ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
13378 ir_PHI_SET_OP(hash_pos_ref, 2, pos2_ref);
13379 ir_PHI_SET_OP(hash_p_ref, 2, p2_ref);
13380
13381 if (MAY_BE_PACKED(op1_info)) {
13382 ir_IF_TRUE(if_packed);
13383 }
13384 }
13385 if (MAY_BE_PACKED(op1_info)) {
13386 ir_ref loop_ref, pos2_ref, p2_ref;
13387
13388 // JIT: p = fe_ht->arPacked + pos;
13389 if (sizeof(void*) == 8) {
13390 ref = ir_ZEXT_A(packed_pos_ref);
13391 } else {
13392 ref = ir_BITCAST_A(packed_pos_ref);
13393 }
13394 packed_p_ref = ir_ADD_A(
13395 ir_MUL_A(ref, ir_CONST_ADDR(sizeof(zval))),
13396 ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arPacked))));
13397
13398 loop_ref = ir_LOOP_BEGIN(ir_END());
13399 packed_pos_ref = ir_PHI_2(IR_U32, packed_pos_ref, IR_UNUSED);
13400 packed_p_ref = ir_PHI_2(IR_ADDR, packed_p_ref, IR_UNUSED);
13401
13402 // JIT: if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
13403 ref = ir_ULT(packed_pos_ref,
13404 ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed))));
13405
13406 // JIT: ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
13407 // JIT: ZEND_VM_CONTINUE();
13408 if (exit_addr) {
13409 if (exit_opcode == ZEND_JMP) {
13410 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
13411 } else {
13412 ir_ref if_fit = ir_IF(ref);
13413 ir_IF_FALSE(if_fit);
13414 ir_END_list(exit_inputs);
13415 ir_IF_TRUE(if_fit);
13416 }
13417 } else {
13418 ir_ref if_fit = ir_IF(ref);
13419 ir_IF_FALSE(if_fit);
13420 ir_END_list(exit_inputs);
13421 ir_IF_TRUE(if_fit);
13422 }
13423
13424 // JIT: pos++;
13425 pos2_ref = ir_ADD_U32(packed_pos_ref, ir_CONST_U32(1));
13426
13427 // JIT: value_type = Z_TYPE_INFO_P(value);
13428 // JIT: if (EXPECTED(value_type != IS_UNDEF)) {
13429 if (!exit_addr || exit_opcode == ZEND_JMP) {
13430 if_def_packed = ir_IF(jit_Z_TYPE_ref(jit, packed_p_ref));
13431 ir_IF_FALSE(if_def_packed);
13432 } else {
13433 ir_GUARD_NOT(jit_Z_TYPE_ref(jit, packed_p_ref), ir_CONST_ADDR(exit_addr));
13434 }
13435
13436 // JIT: p++;
13437 p2_ref = ir_ADD_OFFSET(packed_p_ref, sizeof(zval));
13438
13439 ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
13440 ir_PHI_SET_OP(packed_pos_ref, 2, pos2_ref);
13441 ir_PHI_SET_OP(packed_p_ref, 2, p2_ref);
13442 }
13443
13444 if (!exit_addr || exit_opcode == ZEND_JMP) {
13445 zend_jit_addr val_addr;
13446 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
13447 uint32_t val_info;
13448 ir_ref p_ref = IR_UNUSED, hash_path = IR_UNUSED;
13449
13450 if (RETURN_VALUE_USED(opline)) {
13451 zend_jit_addr res_addr = RES_ADDR();
13452
13453 if (MAY_BE_HASH(op1_info)) {
13454 ir_ref key_ref = IR_UNUSED, if_key = IR_UNUSED, key_path = IR_UNUSED;
13455
13456 ZEND_ASSERT(if_def_hash);
13457 ir_IF_TRUE(if_def_hash);
13458
13459 // JIT: Z_FE_POS_P(array) = pos + 1;
13460 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
13461 ir_ADD_U32(hash_pos_ref, ir_CONST_U32(1)));
13462
13463 if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
13464 key_ref = ir_LOAD_A(ir_ADD_OFFSET(hash_p_ref, offsetof(Bucket, key)));
13465 }
13466 if ((op1_info & MAY_BE_ARRAY_KEY_LONG)
13467 && (op1_info & MAY_BE_ARRAY_KEY_STRING)) {
13468 // JIT: if (!p->key) {
13469 if_key = ir_IF(key_ref);
13470 ir_IF_TRUE(if_key);
13471 }
13472 if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
13473 ir_ref if_interned, interned_path;
13474
13475 // JIT: ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key);
13476 jit_set_Z_PTR(jit, res_addr, key_ref);
13477 ref = ir_AND_U32(
13478 ir_LOAD_U32(ir_ADD_OFFSET(key_ref, offsetof(zend_refcounted, gc.u.type_info))),
13479 ir_CONST_U32(IS_STR_INTERNED));
13480 if_interned = ir_IF(ref);
13481 ir_IF_TRUE(if_interned);
13482
13483 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
13484
13485 interned_path = ir_END();
13486 ir_IF_FALSE(if_interned);
13487
13488 jit_GC_ADDREF(jit, key_ref);
13489 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
13490
13491 ir_MERGE_WITH(interned_path);
13492
13493 if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
13494 key_path = ir_END();
13495 }
13496 }
13497 if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
13498 if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
13499 ir_IF_FALSE(if_key);
13500 }
13501 // JIT: ZVAL_LONG(EX_VAR(opline->result.var), p->h);
13502 ref = ir_LOAD_L(ir_ADD_OFFSET(hash_p_ref, offsetof(Bucket, h)));
13503 jit_set_Z_LVAL(jit, res_addr, ref);
13504 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
13505
13506 if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
13507 ir_MERGE_WITH(key_path);
13508 }
13509 }
13510 if (MAY_BE_PACKED(op1_info)) {
13511 hash_path = ir_END();
13512 } else {
13513 p_ref = hash_p_ref;
13514 }
13515 }
13516 if (MAY_BE_PACKED(op1_info)) {
13517 ZEND_ASSERT(if_def_packed);
13518 ir_IF_TRUE(if_def_packed);
13519
13520 // JIT: Z_FE_POS_P(array) = pos + 1;
13521 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
13522 ir_ADD_U32(packed_pos_ref, ir_CONST_U32(1)));
13523
13524 // JIT: ZVAL_LONG(EX_VAR(opline->result.var), pos);
13525 if (sizeof(zend_long) == 8) {
13526 packed_pos_ref = ir_ZEXT_L(packed_pos_ref);
13527 } else {
13528 packed_pos_ref = ir_BITCAST_L(packed_pos_ref);
13529 }
13530 jit_set_Z_LVAL(jit, res_addr, packed_pos_ref);
13531 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
13532
13533 if (MAY_BE_HASH(op1_info)) {
13534 ir_MERGE_WITH(hash_path);
13535 p_ref = ir_PHI_2(IR_ADDR, packed_p_ref, hash_p_ref);
13536 } else {
13537 p_ref = packed_p_ref;
13538 }
13539 }
13540 } else {
13541 ir_ref pos_ref = IR_UNUSED;
13542
13543 if (if_def_hash && if_def_packed) {
13544 ir_IF_TRUE(if_def_hash);
13545 ir_MERGE_WITH_EMPTY_TRUE(if_def_packed);
13546 pos_ref = ir_PHI_2(IR_U32, hash_pos_ref, packed_pos_ref);
13547 p_ref = ir_PHI_2(IR_ADDR, hash_p_ref, packed_p_ref);
13548 } else if (if_def_hash) {
13549 ir_IF_TRUE(if_def_hash);
13550 pos_ref = hash_pos_ref;
13551 p_ref = hash_p_ref;
13552 } else if (if_def_packed) {
13553 ir_IF_TRUE(if_def_packed);
13554 pos_ref = packed_pos_ref;
13555 p_ref = packed_p_ref;
13556 } else {
13557 ZEND_UNREACHABLE();
13558 }
13559
13560 // JIT: Z_FE_POS_P(array) = pos + 1;
13561 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
13562 ir_ADD_U32(pos_ref, ir_CONST_U32(1)));
13563 }
13564
13565 val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT);
13566 if (val_info & MAY_BE_ARRAY) {
13567 val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
13568 }
13569 if (op1_info & MAY_BE_ARRAY_OF_REF) {
13570 val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY |
13571 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
13572 } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
13573 val_info |= MAY_BE_RC1 | MAY_BE_RCN;
13574 }
13575
13576 val_addr = ZEND_ADDR_REF_ZVAL(p_ref);
13577 if (opline->op2_type == IS_CV) {
13578 // JIT: zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES());
13579 if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 0, 1)) {
13580 return 0;
13581 }
13582 } else {
13583 // JIT: ZVAL_COPY(res, value);
13584 jit_ZVAL_COPY(jit, var_addr, -1, val_addr, val_info, 1);
13585 }
13586
13587 if (!exit_addr) {
13588 zend_basic_block *bb;
13589
13590 ZEND_ASSERT(jit->b >= 0);
13591 bb = &jit->ssa->cfg.blocks[jit->b];
13592 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ir_END());
13593 ZEND_ASSERT(exit_inputs);
13594 if (!jit->ctx.ir_base[exit_inputs].op2) {
13595 ref = exit_inputs;
13596 } else {
13597 ir_MERGE_list(exit_inputs);
13598 ref = ir_END();
13599 }
13600 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
13601 jit->b = -1;
13602 }
13603 } else {
13604 ZEND_ASSERT(exit_inputs);
13605 ir_MERGE_list(exit_inputs);
13606 }
13607
13608 return 1;
13609 }
13610
13611 static int zend_jit_load_this(zend_jit_ctx *jit, uint32_t var)
13612 {
13613 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
13614 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
13615 ir_ref ref = jit_Z_PTR(jit, this_addr);
13616
13617 jit_set_Z_PTR(jit, var_addr, ref);
13618 jit_set_Z_TYPE_INFO(jit, var_addr, IS_OBJECT_EX);
13619 jit_GC_ADDREF(jit, ref);
13620
13621 return 1;
13622 }
13623
13624 static int zend_jit_fetch_this(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, bool check_only)
13625 {
13626 if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) {
13627 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
13628 if (!JIT_G(current_frame) ||
13629 !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) {
13630
13631 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
13632 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
13633 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
13634
13635 if (!exit_addr) {
13636 return 0;
13637 }
13638
13639 jit_guard_Z_TYPE(jit, this_addr, IS_OBJECT, exit_addr);
13640
13641 if (JIT_G(current_frame)) {
13642 TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame));
13643 }
13644 }
13645 } else {
13646 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
13647 ir_ref if_object = jit_if_Z_TYPE(jit, this_addr, IS_OBJECT);
13648
13649 ir_IF_FALSE_cold(if_object);
13650 jit_SET_EX_OPLINE(jit, opline);
13651 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_invalid_this));
13652
13653 ir_IF_TRUE(if_object);
13654 }
13655 }
13656
13657 if (!check_only) {
13658 if (!zend_jit_load_this(jit, opline->result.var)) {
13659 return 0;
13660 }
13661 }
13662
13663 return 1;
13664 }
13665
13666 static int zend_jit_class_guard(zend_jit_ctx *jit, const zend_op *opline, ir_ref obj_ref, zend_class_entry *ce)
13667 {
13668 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
13669 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
13670
13671 if (!exit_addr) {
13672 return 0;
13673 }
13674
13675 ir_GUARD(ir_EQ(ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))), ir_CONST_ADDR(ce)),
13676 ir_CONST_ADDR(exit_addr));
13677
13678 return 1;
13679 }
13680
13681 static int zend_jit_fetch_obj(zend_jit_ctx *jit,
13682 const zend_op *opline,
13683 const zend_op_array *op_array,
13684 zend_ssa *ssa,
13685 const zend_ssa_op *ssa_op,
13686 uint32_t op1_info,
13687 zend_jit_addr op1_addr,
13688 bool op1_indirect,
13689 zend_class_entry *ce,
13690 bool ce_is_instanceof,
13691 bool on_this,
13692 bool delayed_fetch_this,
13693 bool op1_avoid_refcounting,
13694 zend_class_entry *trace_ce,
13695 zend_jit_addr res_addr,
13696 uint8_t prop_type,
13697 int may_throw)
13698 {
13699 zval *member;
13700 zend_property_info *prop_info;
13701 bool may_be_dynamic = 1;
13702 zend_jit_addr prop_addr;
13703 uint32_t res_info = RES_INFO();
13704 ir_ref prop_type_ref = IR_UNUSED;
13705 ir_ref obj_ref = IR_UNUSED;
13706 ir_ref prop_ref = IR_UNUSED;
13707 ir_ref end_inputs = IR_UNUSED;
13708 ir_ref slow_inputs = IR_UNUSED;
13709 ir_ref end_values = IR_UNUSED;
13710
13711 ZEND_ASSERT(opline->op2_type == IS_CONST);
13712 ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
13713
13714 member = RT_CONSTANT(opline, opline->op2);
13715 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
13716 prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename);
13717
13718 if (on_this) {
13719 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
13720 obj_ref = jit_Z_PTR(jit, this_addr);
13721 } else {
13722 if (opline->op1_type == IS_VAR
13723 && opline->opcode == ZEND_FETCH_OBJ_W
13724 && (op1_info & MAY_BE_INDIRECT)
13725 && Z_REG(op1_addr) == ZREG_FP) {
13726 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
13727 }
13728 if (op1_info & MAY_BE_REF) {
13729 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
13730 }
13731 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
13732 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
13733 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
13734 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
13735
13736 if (!exit_addr) {
13737 return 0;
13738 }
13739 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
13740 } else {
13741 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
13742
13743 ir_IF_FALSE_cold(if_obj);
13744 if (opline->opcode != ZEND_FETCH_OBJ_IS) {
13745 ir_ref op1_ref = IR_UNUSED;
13746
13747 jit_SET_EX_OPLINE(jit, opline);
13748 if (opline->opcode != ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_UNDEF)) {
13749 zend_jit_addr orig_op1_addr = OP1_ADDR();
13750 ir_ref fast_path = IR_UNUSED;
13751
13752 if (op1_info & MAY_BE_ANY) {
13753 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
13754 ir_IF_TRUE(if_def);
13755 fast_path = ir_END();
13756 ir_IF_FALSE_cold(if_def);
13757 }
13758 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
13759 ir_CONST_U32(opline->op1.var));
13760 if (fast_path) {
13761 ir_MERGE_WITH(fast_path);
13762 }
13763 op1_ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
13764 } else {
13765 op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
13766 }
13767 if (opline->opcode == ZEND_FETCH_OBJ_W) {
13768 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_write),
13769 op1_ref, ir_CONST_ADDR(Z_STRVAL_P(member)));
13770 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
13771 } else {
13772 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_read),
13773 op1_ref, ir_CONST_ADDR(Z_STRVAL_P(member)));
13774 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
13775 }
13776 } else {
13777 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
13778 }
13779 ir_END_list(end_inputs);
13780
13781 ir_IF_TRUE(if_obj);
13782 }
13783 }
13784 obj_ref = jit_Z_PTR(jit, op1_addr);
13785 }
13786
13787 ZEND_ASSERT(obj_ref);
13788 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
13789 prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename);
13790 if (prop_info) {
13791 ce = trace_ce;
13792 ce_is_instanceof = 0;
13793 if (!(op1_info & MAY_BE_CLASS_GUARD)) {
13794 if (on_this && JIT_G(current_frame)
13795 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
13796 ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
13797 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
13798 if (on_this && JIT_G(current_frame)) {
13799 JIT_G(current_frame)->ce = ce;
13800 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
13801 }
13802 } else {
13803 return 0;
13804 }
13805 if (ssa->var_info && ssa_op->op1_use >= 0) {
13806 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
13807 ssa->var_info[ssa_op->op1_use].ce = ce;
13808 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
13809 }
13810 }
13811 }
13812 }
13813
13814 if (!prop_info) {
13815 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
13816 ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
13817 ir_ref if_same = ir_IF(ir_EQ(ref,
13818 ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
13819
13820 ir_IF_FALSE_cold(if_same);
13821 ir_END_list(slow_inputs);
13822
13823 ir_IF_TRUE(if_same);
13824 ir_ref offset_ref = ir_LOAD_A(
13825 ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
13826
13827 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename);
13828 if (may_be_dynamic) {
13829 ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, IR_NULL));
13830 if (opline->opcode == ZEND_FETCH_OBJ_W) {
13831 ir_IF_TRUE_cold(if_dynamic);
13832 ir_END_list(slow_inputs);
13833 } else {
13834 ir_IF_TRUE_cold(if_dynamic);
13835 jit_SET_EX_OPLINE(jit, opline);
13836
13837 if (opline->opcode != ZEND_FETCH_OBJ_IS) {
13838 if (((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info)
13839 || Z_MODE(res_addr) == IS_REG) {
13840 ir_ref val_addr = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_dynamic_ex),
13841 obj_ref, offset_ref);
13842 ir_END_PHI_list(end_values, val_addr);
13843 } else {
13844 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_dynamic),
13845 obj_ref, offset_ref);
13846 ir_END_list(end_inputs);
13847 }
13848 } else {
13849 if (((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info)
13850 || Z_MODE(res_addr) == IS_REG) {
13851 ir_ref val_addr = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_dynamic_ex),
13852 obj_ref, offset_ref);
13853 ir_END_PHI_list(end_values, val_addr);
13854 } else {
13855 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_dynamic),
13856 obj_ref, offset_ref);
13857 ir_END_list(end_inputs);
13858 }
13859 }
13860 }
13861 ir_IF_FALSE(if_dynamic);
13862 }
13863 prop_ref = ir_ADD_A(obj_ref, offset_ref);
13864 prop_type_ref = jit_Z_TYPE_ref(jit, prop_ref);
13865 ir_ref if_def = ir_IF(prop_type_ref);
13866 ir_IF_FALSE_cold(if_def);
13867 ir_END_list(slow_inputs);
13868 ir_IF_TRUE(if_def);
13869 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
13870 if (opline->opcode == ZEND_FETCH_OBJ_W
13871 && (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) {
13872 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
13873
13874 ir_ref prop_info_ref = ir_LOAD_A(
13875 ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
13876 ir_ref if_has_prop_info = ir_IF(prop_info_ref);
13877
13878 ir_IF_TRUE_cold(if_has_prop_info);
13879
13880 ir_ref if_readonly = ir_IF(
13881 ir_AND_U32(ir_LOAD_U32(ir_ADD_OFFSET(prop_info_ref, offsetof(zend_property_info, flags))),
13882 ir_CONST_U32(ZEND_ACC_READONLY)));
13883 ir_IF_TRUE(if_readonly);
13884
13885 ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
13886 ir_IF_TRUE(if_prop_obj);
13887 ref = jit_Z_PTR(jit, prop_addr);
13888 jit_GC_ADDREF(jit, ref);
13889 jit_set_Z_PTR(jit, res_addr, ref);
13890 jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
13891 ir_END_list(end_inputs);
13892
13893 ir_IF_FALSE_cold(if_prop_obj);
13894
13895 ir_ref extra_addr = ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, prop_addr), offsetof(zval, u2.extra));
13896 ir_ref extra = ir_LOAD_U32(extra_addr);
13897 ir_ref if_reinitable = ir_IF(ir_AND_U32(extra, ir_CONST_U32(IS_PROP_REINITABLE)));
13898 ir_IF_TRUE(if_reinitable);
13899 ir_STORE(extra_addr, ir_AND_U32(extra, ir_CONST_U32(~IS_PROP_REINITABLE)));
13900 ir_ref reinit_path = ir_END();
13901
13902 ir_IF_FALSE(if_reinitable);
13903
13904 jit_SET_EX_OPLINE(jit, opline);
13905 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_readonly_property_modification_error), prop_info_ref);
13906 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
13907 ir_END_list(end_inputs);
13908
13909 if (flags == ZEND_FETCH_DIM_WRITE) {
13910 ir_IF_FALSE_cold(if_readonly);
13911 ir_MERGE_WITH(reinit_path);
13912 jit_SET_EX_OPLINE(jit, opline);
13913 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_check_array_promotion),
13914 prop_ref, prop_info_ref);
13915 ir_END_list(end_inputs);
13916 ir_IF_FALSE(if_has_prop_info);
13917 } else if (flags == ZEND_FETCH_REF) {
13918 ir_IF_FALSE_cold(if_readonly);
13919 ir_MERGE_WITH(reinit_path);
13920 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_create_typed_ref),
13921 prop_ref,
13922 prop_info_ref,
13923 jit_ZVAL_ADDR(jit, res_addr));
13924 ir_END_list(end_inputs);
13925 ir_IF_FALSE(if_has_prop_info);
13926 } else {
13927 ir_ref list = reinit_path;
13928
13929 ZEND_ASSERT(flags == 0);
13930 ir_IF_FALSE(if_has_prop_info);
13931 ir_END_list(list);
13932 ir_IF_FALSE(if_readonly);
13933 ir_END_list(list);
13934 ir_MERGE_list(list);
13935 }
13936 }
13937 } else {
13938 prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
13939 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
13940 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
13941 if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) {
13942 /* perform IS_UNDEF check only after result type guard (during deoptimization) */
13943 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
13944 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
13945
13946 if (!exit_addr) {
13947 return 0;
13948 }
13949 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
13950 ir_GUARD(prop_type_ref, ir_CONST_ADDR(exit_addr));
13951 }
13952 } else {
13953 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
13954 ir_ref if_def = ir_IF(prop_type_ref);
13955 ir_IF_FALSE_cold(if_def);
13956 ir_END_list(slow_inputs);
13957 ir_IF_TRUE(if_def);
13958 }
13959 if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) {
13960 if (!prop_type_ref) {
13961 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
13962 }
13963 ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
13964 ir_IF_TRUE(if_prop_obj);
13965 ir_ref ref = jit_Z_PTR(jit, prop_addr);
13966 jit_GC_ADDREF(jit, ref);
13967 jit_set_Z_PTR(jit, res_addr, ref);
13968 jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
13969 ir_END_list(end_inputs);
13970
13971 ir_IF_FALSE_cold(if_prop_obj);
13972
13973 ir_ref extra_addr = ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, prop_addr), offsetof(zval, u2.extra));
13974 ir_ref extra = ir_LOAD_U32(extra_addr);
13975 ir_ref if_reinitable = ir_IF(ir_AND_U32(extra, ir_CONST_U32(IS_PROP_REINITABLE)));
13976
13977 ir_IF_FALSE(if_reinitable);
13978 jit_SET_EX_OPLINE(jit, opline);
13979 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_readonly_property_modification_error), ir_CONST_ADDR(prop_info));
13980 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
13981 ir_END_list(end_inputs);
13982
13983 ir_IF_TRUE(if_reinitable);
13984 ir_STORE(extra_addr, ir_AND_U32(extra, ir_CONST_U32(~IS_PROP_REINITABLE)));
13985 }
13986
13987 if (opline->opcode == ZEND_FETCH_OBJ_W
13988 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS)
13989 && ZEND_TYPE_IS_SET(prop_info->type)) {
13990 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
13991
13992 if (flags == ZEND_FETCH_DIM_WRITE) {
13993 if ((ZEND_TYPE_FULL_MASK(prop_info->type) & MAY_BE_ARRAY) == 0) {
13994 if (!prop_type_ref) {
13995 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
13996 }
13997 ir_ref if_null_or_flase = ir_IF(ir_LE(prop_type_ref, ir_CONST_U32(IR_FALSE)));
13998 ir_IF_TRUE_cold(if_null_or_flase);
13999 jit_SET_EX_OPLINE(jit, opline);
14000 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_check_array_promotion),
14001 prop_ref, ir_CONST_ADDR(prop_info));
14002 ir_END_list(end_inputs);
14003 ir_IF_FALSE(if_null_or_flase);
14004 }
14005 } else if (flags == ZEND_FETCH_REF) {
14006 ir_ref ref;
14007
14008 if (!prop_type_ref) {
14009 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
14010 }
14011
14012 ir_ref if_reference = ir_IF(ir_EQ(prop_type_ref, ir_CONST_U32(IS_REFERENCE_EX)));
14013 ir_IF_FALSE(if_reference);
14014 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
14015 ref = ir_CONST_ADDR(prop_info);
14016 } else {
14017 int prop_info_offset =
14018 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
14019
14020 ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
14021 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
14022 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
14023 }
14024 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_create_typed_ref),
14025 prop_ref,
14026 ref,
14027 jit_ZVAL_ADDR(jit, res_addr));
14028 ir_END_list(end_inputs);
14029 ir_IF_TRUE(if_reference);
14030 } else {
14031 ZEND_UNREACHABLE();
14032 }
14033 }
14034 }
14035
14036 if (opline->opcode == ZEND_FETCH_OBJ_W) {
14037 ZEND_ASSERT(prop_ref);
14038 jit_set_Z_PTR(jit, res_addr, prop_ref);
14039 jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
14040 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) {
14041 ssa->var_info[ssa_op->result_def].indirect_reference = 1;
14042 }
14043 ir_END_list(end_inputs);
14044 } else {
14045 if (((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info)
14046 || Z_MODE(res_addr) == IS_REG) {
14047 ir_END_PHI_list(end_values, jit_ZVAL_ADDR(jit, prop_addr));
14048 } else {
14049 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
14050
14051 if (!zend_jit_zval_copy_deref(jit, res_addr, prop_addr, prop_type_ref)) {
14052 return 0;
14053 }
14054 ir_END_list(end_inputs);
14055 }
14056 }
14057
14058 if (op1_avoid_refcounting) {
14059 SET_STACK_REG(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
14060 }
14061
14062 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) {
14063 ir_MERGE_list(slow_inputs);
14064 jit_SET_EX_OPLINE(jit, opline);
14065
14066 if (opline->opcode == ZEND_FETCH_OBJ_W) {
14067 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_w_slow), obj_ref);
14068 ir_END_list(end_inputs);
14069 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) {
14070 if (Z_MODE(res_addr) == IS_REG) {
14071 ir_ref val_ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_slow_ex), obj_ref);
14072 ir_END_PHI_list(end_values, val_ref);
14073 } else {
14074 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_slow), obj_ref);
14075 ir_END_list(end_inputs);
14076 }
14077 } else {
14078 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_slow), obj_ref);
14079 ir_END_list(end_inputs);
14080 }
14081 }
14082
14083 if (end_values) {
14084 ir_ref val_ref = ir_PHI_list(end_values);
14085 zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val_ref);
14086 bool result_avoid_refcounting = 0;
14087
14088 ZEND_ASSERT(opline->opcode == ZEND_FETCH_OBJ_R
14089 || opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG
14090 || opline->opcode == ZEND_FETCH_OBJ_IS);
14091 ZEND_ASSERT(end_inputs == IR_UNUSED);
14092 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) {
14093 uint8_t type = concrete_type(res_info);
14094 uint32_t flags = 0;
14095
14096 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
14097 && !delayed_fetch_this
14098 && !op1_avoid_refcounting) {
14099 flags = ZEND_JIT_EXIT_FREE_OP1;
14100 }
14101
14102 if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
14103 && !(flags & ZEND_JIT_EXIT_FREE_OP1)
14104 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
14105 && (ssa_op+1)->op1_use == ssa_op->result_def
14106 && zend_jit_may_avoid_refcounting(opline+1, res_info)) {
14107 result_avoid_refcounting = 1;
14108 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
14109 }
14110
14111 val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
14112 1, flags, op1_avoid_refcounting);
14113 if (!val_addr) {
14114 return 0;
14115 }
14116
14117 res_info &= ~MAY_BE_GUARD;
14118 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
14119 }
14120
14121 // ZVAL_COPY
14122 jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
14123
14124 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
14125 return 0;
14126 }
14127 } else {
14128 ir_MERGE_list(end_inputs);
14129 }
14130
14131 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
14132 if (opline->op1_type == IS_VAR
14133 && opline->opcode == ZEND_FETCH_OBJ_W
14134 && (op1_info & MAY_BE_RC1)) {
14135 zend_jit_addr orig_op1_addr = OP1_ADDR();
14136 ir_ref if_refcounted, ptr, refcount, if_non_zero;
14137 ir_ref merge_inputs = IR_UNUSED;
14138
14139 if_refcounted = jit_if_REFCOUNTED(jit, orig_op1_addr);
14140 ir_IF_FALSE( if_refcounted);
14141 ir_END_list(merge_inputs);
14142 ir_IF_TRUE( if_refcounted);
14143 ptr = jit_Z_PTR(jit, orig_op1_addr);
14144 refcount = jit_GC_DELREF(jit, ptr);
14145 if_non_zero = ir_IF(refcount);
14146 ir_IF_TRUE( if_non_zero);
14147 ir_END_list(merge_inputs);
14148 ir_IF_FALSE( if_non_zero);
14149 jit_SET_EX_OPLINE(jit, opline);
14150 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_extract_helper), ptr);
14151 ir_END_list(merge_inputs);
14152 ir_MERGE_list(merge_inputs);
14153 } else if (!op1_avoid_refcounting) {
14154 if (on_this) {
14155 op1_info &= ~MAY_BE_RC1;
14156 }
14157 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
14158 }
14159 }
14160
14161 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
14162 && prop_info
14163 && (opline->opcode != ZEND_FETCH_OBJ_W ||
14164 !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) ||
14165 !ZEND_TYPE_IS_SET(prop_info->type))
14166 && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) {
14167 may_throw = 0;
14168 }
14169
14170 if (may_throw) {
14171 zend_jit_check_exception(jit);
14172 }
14173
14174 return 1;
14175 }
14176
14177 static int zend_jit_assign_obj(zend_jit_ctx *jit,
14178 const zend_op *opline,
14179 const zend_op_array *op_array,
14180 zend_ssa *ssa,
14181 const zend_ssa_op *ssa_op,
14182 uint32_t op1_info,
14183 zend_jit_addr op1_addr,
14184 uint32_t val_info,
14185 zend_jit_addr val_addr,
14186 zend_jit_addr val_def_addr,
14187 zend_jit_addr res_addr,
14188 bool op1_indirect,
14189 zend_class_entry *ce,
14190 bool ce_is_instanceof,
14191 bool on_this,
14192 bool delayed_fetch_this,
14193 zend_class_entry *trace_ce,
14194 uint8_t prop_type,
14195 int may_throw)
14196 {
14197 zval *member;
14198 zend_string *name;
14199 zend_property_info *prop_info;
14200 zend_jit_addr prop_addr;
14201 ir_ref obj_ref = IR_UNUSED;
14202 ir_ref prop_ref = IR_UNUSED;
14203 ir_ref delayed_end_input = IR_UNUSED;
14204 ir_ref end_inputs = IR_UNUSED;
14205 ir_ref slow_inputs = IR_UNUSED;
14206 uint32_t res_info = RES_INFO();
14207
14208 if (val_addr != val_def_addr && val_def_addr) {
14209 if (!zend_jit_update_regs(jit, (opline+1)->op1.var, val_addr, val_def_addr, val_info)) {
14210 return 0;
14211 }
14212 if (Z_MODE(val_def_addr) == IS_REG && Z_MODE(val_addr) != IS_REG) {
14213 val_addr = val_def_addr;
14214 }
14215 }
14216
14217 ZEND_ASSERT(opline->op2_type == IS_CONST);
14218 ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
14219
14220 member = RT_CONSTANT(opline, opline->op2);
14221 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
14222 name = Z_STR_P(member);
14223 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
14224
14225 if (on_this) {
14226 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
14227 obj_ref = jit_Z_PTR(jit, this_addr);
14228 } else {
14229 if (opline->op1_type == IS_VAR
14230 && (op1_info & MAY_BE_INDIRECT)
14231 && Z_REG(op1_addr) == ZREG_FP) {
14232 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
14233 }
14234 if (op1_info & MAY_BE_REF) {
14235 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
14236 }
14237 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
14238 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
14239 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
14240 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
14241
14242 if (!exit_addr) {
14243 return 0;
14244 }
14245 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
14246 } else {
14247 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
14248 ir_IF_FALSE_cold(if_obj);
14249
14250 jit_SET_EX_OPLINE(jit, opline);
14251 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_assign),
14252 jit_ZVAL_ADDR(jit, op1_addr),
14253 ir_CONST_ADDR(ZSTR_VAL(name)));
14254
14255 if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) != IS_REG) {
14256 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
14257 }
14258
14259 ir_END_list(end_inputs);
14260
14261 ir_IF_TRUE(if_obj);
14262 }
14263 }
14264 obj_ref = jit_Z_PTR(jit, op1_addr);
14265 }
14266
14267 ZEND_ASSERT(obj_ref);
14268 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
14269 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
14270 if (prop_info) {
14271 ce = trace_ce;
14272 ce_is_instanceof = 0;
14273 if (!(op1_info & MAY_BE_CLASS_GUARD)) {
14274 if (on_this && JIT_G(current_frame)
14275 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
14276 ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
14277 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
14278 if (on_this && JIT_G(current_frame)) {
14279 JIT_G(current_frame)->ce = ce;
14280 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
14281 }
14282 } else {
14283 return 0;
14284 }
14285 if (ssa->var_info && ssa_op->op1_use >= 0) {
14286 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
14287 ssa->var_info[ssa_op->op1_use].ce = ce;
14288 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
14289 }
14290 if (ssa->var_info && ssa_op->op1_def >= 0) {
14291 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
14292 ssa->var_info[ssa_op->op1_def].ce = ce;
14293 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
14294 }
14295 }
14296 }
14297 }
14298
14299 if (!prop_info) {
14300 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
14301 ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
14302 ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
14303
14304 ir_IF_FALSE_cold(if_same);
14305 ir_END_list(slow_inputs);
14306
14307 ir_IF_TRUE(if_same);
14308 ir_ref offset_ref = ir_LOAD_A(
14309 ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
14310
14311 ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, IR_NULL));
14312 ir_IF_TRUE_cold(if_dynamic);
14313 ir_END_list(slow_inputs);
14314
14315 ir_IF_FALSE(if_dynamic);
14316 prop_ref = ir_ADD_A(obj_ref, offset_ref);
14317 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
14318 ir_IF_FALSE_cold(if_def);
14319 ir_END_list(slow_inputs);
14320
14321 ir_IF_TRUE(if_def);
14322 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14323
14324 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
14325 ir_ref arg3, arg4;
14326 ir_ref prop_info_ref = ir_LOAD_A(
14327 ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
14328 ir_ref if_has_prop_info = ir_IF(prop_info_ref);
14329 ir_IF_TRUE_cold(if_has_prop_info);
14330
14331 if (Z_MODE(val_addr) == IS_REG) {
14332 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
14333 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
14334 return 0;
14335 }
14336 arg3 = jit_ZVAL_ADDR(jit, real_addr);
14337 } else {
14338 arg3 = jit_ZVAL_ADDR(jit, val_addr);
14339 }
14340
14341 if (!RETURN_VALUE_USED(opline)) {
14342 arg4 = IR_NULL;
14343 } else if (Z_MODE(res_addr) == IS_REG) {
14344 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
14345 arg4 = jit_ZVAL_ADDR(jit, real_addr);
14346 } else {
14347 arg4 = jit_ZVAL_ADDR(jit, res_addr);
14348 }
14349 // JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
14350 jit_SET_EX_OPLINE(jit, opline);
14351 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_to_typed_prop),
14352 prop_ref,
14353 prop_info_ref,
14354 arg3,
14355 arg4);
14356
14357 if ((opline+1)->op1_type == IS_CONST) {
14358 // TODO: ???
14359 // if (Z_TYPE_P(value) == orig_type) {
14360 // CACHE_PTR_EX(cache_slot + 2, NULL);
14361 }
14362
14363 if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) == IS_REG) {
14364 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
14365 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
14366 return 0;
14367 }
14368 }
14369
14370 ir_END_list(end_inputs);
14371 ir_IF_FALSE(if_has_prop_info);
14372 }
14373 } else {
14374 prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
14375 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14376 if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) {
14377 // Undefined property with magic __get()/__set()
14378 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
14379 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
14380 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
14381
14382 if (!exit_addr) {
14383 return 0;
14384 }
14385 ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
14386 } else {
14387 ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
14388 ir_IF_FALSE_cold(if_def);
14389 ir_END_list(slow_inputs);
14390 ir_IF_TRUE(if_def);
14391 }
14392 }
14393 if (ZEND_TYPE_IS_SET(prop_info->type)) {
14394 ir_ref ref, arg3, arg4;
14395
14396 // JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
14397 jit_SET_EX_OPLINE(jit, opline);
14398 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
14399 ref = ir_CONST_ADDR(prop_info);
14400 } else {
14401 int prop_info_offset =
14402 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
14403
14404 ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
14405 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
14406 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
14407 }
14408 if (Z_MODE(val_addr) == IS_REG) {
14409 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
14410 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
14411 return 0;
14412 }
14413 arg3 = jit_ZVAL_ADDR(jit, real_addr);
14414 } else {
14415 arg3 = jit_ZVAL_ADDR(jit, val_addr);
14416 }
14417 if (!RETURN_VALUE_USED(opline)) {
14418 arg4 = IR_NULL;
14419 } else if (Z_MODE(res_addr) == IS_REG) {
14420 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
14421 arg4 = jit_ZVAL_ADDR(jit, real_addr);
14422 } else {
14423 arg4 = jit_ZVAL_ADDR(jit, res_addr);
14424 }
14425 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_to_typed_prop),
14426 prop_ref,
14427 ref,
14428 arg3,
14429 arg4);
14430
14431 if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) == IS_REG) {
14432 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
14433 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
14434 return 0;
14435 }
14436 }
14437 ir_END_list(end_inputs);
14438 }
14439 }
14440
14441 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
14442 if (Z_MODE(val_addr) != IS_REG
14443 && (res_addr == 0 || Z_MODE(res_addr) != IS_REG)
14444 && opline->result_type == IS_UNUSED) {
14445 if (!zend_jit_assign_to_variable_call(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) {
14446 return 0;
14447 }
14448 } else {
14449 if (!zend_jit_assign_to_variable(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0, 0)) {
14450 return 0;
14451 }
14452 }
14453 if (end_inputs || slow_inputs) {
14454 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
14455 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
14456 /* skip FREE_OP_DATA() */
14457 delayed_end_input = ir_END();
14458 } else {
14459 ir_END_list(end_inputs);
14460 }
14461 }
14462 }
14463
14464 if (slow_inputs) {
14465 ir_ref arg3, arg5;
14466
14467 ir_MERGE_list(slow_inputs);
14468 jit_SET_EX_OPLINE(jit, opline);
14469
14470 if (Z_MODE(val_addr) == IS_REG) {
14471 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
14472 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
14473 return 0;
14474 }
14475 arg3 = jit_ZVAL_ADDR(jit, real_addr);
14476 } else {
14477 arg3 = jit_ZVAL_ADDR(jit, val_addr);
14478 }
14479 if (!RETURN_VALUE_USED(opline)) {
14480 arg5 = IR_NULL;
14481 } else if (Z_MODE(res_addr) == IS_REG) {
14482 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
14483 arg5 = jit_ZVAL_ADDR(jit, real_addr);
14484 } else {
14485 arg5 = jit_ZVAL_ADDR(jit, res_addr);
14486 }
14487
14488 // JIT: value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
14489 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
14490 ir_CALL_5(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_obj_helper),
14491 obj_ref,
14492 ir_CONST_ADDR(name),
14493 arg3,
14494 ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
14495 arg5);
14496
14497 if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) == IS_REG) {
14498 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
14499 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
14500 return 0;
14501 }
14502 }
14503 ir_END_list(end_inputs);
14504 }
14505
14506 if (end_inputs) {
14507 ir_MERGE_list(end_inputs);
14508
14509 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
14510 val_info |= MAY_BE_RC1|MAY_BE_RCN;
14511 }
14512 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
14513
14514 if (delayed_end_input) {
14515 ir_MERGE_WITH(delayed_end_input);
14516 }
14517 }
14518
14519 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
14520 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
14521 }
14522
14523 if (may_throw) {
14524 zend_jit_check_exception(jit);
14525 }
14526
14527 return 1;
14528 }
14529
14530 static int zend_jit_assign_obj_op(zend_jit_ctx *jit,
14531 const zend_op *opline,
14532 const zend_op_array *op_array,
14533 zend_ssa *ssa,
14534 const zend_ssa_op *ssa_op,
14535 uint32_t op1_info,
14536 zend_jit_addr op1_addr,
14537 uint32_t val_info,
14538 zend_jit_addr val_addr,
14539 zend_ssa_range *val_range,
14540 bool op1_indirect,
14541 zend_class_entry *ce,
14542 bool ce_is_instanceof,
14543 bool on_this,
14544 bool delayed_fetch_this,
14545 zend_class_entry *trace_ce,
14546 uint8_t prop_type)
14547 {
14548 zval *member;
14549 zend_string *name;
14550 zend_property_info *prop_info;
14551 zend_jit_addr prop_addr;
14552 bool use_prop_guard = 0;
14553 bool may_throw = 0;
14554 binary_op_type binary_op = get_binary_op(opline->extended_value);
14555 ir_ref obj_ref = IR_UNUSED;
14556 ir_ref prop_ref = IR_UNUSED;
14557 ir_ref end_inputs = IR_UNUSED;
14558 ir_ref slow_inputs = IR_UNUSED;
14559
14560 ZEND_ASSERT(opline->op2_type == IS_CONST);
14561 ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
14562 ZEND_ASSERT(opline->result_type == IS_UNUSED);
14563
14564 member = RT_CONSTANT(opline, opline->op2);
14565 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
14566 name = Z_STR_P(member);
14567 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
14568
14569 if (on_this) {
14570 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
14571 obj_ref = jit_Z_PTR(jit, this_addr);
14572 } else {
14573 if (opline->op1_type == IS_VAR
14574 && (op1_info & MAY_BE_INDIRECT)
14575 && Z_REG(op1_addr) == ZREG_FP) {
14576 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
14577 }
14578 if (op1_info & MAY_BE_REF) {
14579 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
14580 }
14581 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
14582 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
14583 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
14584 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
14585
14586 if (!exit_addr) {
14587 return 0;
14588 }
14589 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
14590 } else {
14591 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
14592 ir_IF_FALSE_cold(if_obj);
14593
14594 jit_SET_EX_OPLINE(jit, opline);
14595 ir_CALL_2(IR_VOID,
14596 (op1_info & MAY_BE_UNDEF) ?
14597 ir_CONST_FC_FUNC(zend_jit_invalid_property_assign_op) :
14598 ir_CONST_FC_FUNC(zend_jit_invalid_property_assign),
14599 jit_ZVAL_ADDR(jit, op1_addr),
14600 ir_CONST_ADDR(ZSTR_VAL(name)));
14601
14602 may_throw = 1;
14603
14604 ir_END_list(end_inputs);
14605 ir_IF_TRUE(if_obj);
14606 }
14607 }
14608 obj_ref = jit_Z_PTR(jit, op1_addr);
14609 }
14610
14611 ZEND_ASSERT(obj_ref);
14612 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
14613 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
14614 if (prop_info) {
14615 ce = trace_ce;
14616 ce_is_instanceof = 0;
14617 if (!(op1_info & MAY_BE_CLASS_GUARD)) {
14618 if (on_this && JIT_G(current_frame)
14619 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
14620 ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
14621 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
14622 if (on_this && JIT_G(current_frame)) {
14623 JIT_G(current_frame)->ce = ce;
14624 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
14625 }
14626 } else {
14627 return 0;
14628 }
14629 if (ssa->var_info && ssa_op->op1_use >= 0) {
14630 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
14631 ssa->var_info[ssa_op->op1_use].ce = ce;
14632 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
14633 }
14634 if (ssa->var_info && ssa_op->op1_def >= 0) {
14635 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
14636 ssa->var_info[ssa_op->op1_def].ce = ce;
14637 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
14638 }
14639 }
14640 }
14641 }
14642
14643 use_prop_guard = (prop_type != IS_UNKNOWN
14644 && prop_type != IS_UNDEF
14645 && prop_type != IS_REFERENCE
14646 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
14647
14648 if (!prop_info) {
14649 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
14650 ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, (opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
14651 ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
14652
14653 ir_IF_FALSE_cold(if_same);
14654 ir_END_list(slow_inputs);
14655
14656 ir_IF_TRUE(if_same);
14657 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
14658 ir_ref prop_info_ref = ir_LOAD_A(
14659 ir_ADD_OFFSET(run_time_cache, ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
14660 ir_ref if_has_prop_info = ir_IF(prop_info_ref);
14661 ir_IF_TRUE_cold(if_has_prop_info);
14662 ir_END_list(slow_inputs);
14663
14664 ir_IF_FALSE(if_has_prop_info);
14665 }
14666 ir_ref offset_ref = ir_LOAD_A(
14667 ir_ADD_OFFSET(run_time_cache, ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
14668
14669 ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, IR_NULL));
14670 ir_IF_TRUE_cold(if_dynamic);
14671 ir_END_list(slow_inputs);
14672
14673 ir_IF_FALSE(if_dynamic);
14674
14675 prop_ref = ir_ADD_A(obj_ref, offset_ref);
14676 if (!use_prop_guard) {
14677 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
14678 ir_IF_FALSE_cold(if_def);
14679 ir_END_list(slow_inputs);
14680
14681 ir_IF_TRUE(if_def);
14682 }
14683 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14684 } else {
14685 prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
14686 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14687
14688 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
14689 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
14690 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
14691 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
14692
14693 if (!exit_addr) {
14694 return 0;
14695 }
14696 ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
14697 } else {
14698 ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
14699 ir_IF_FALSE_cold(if_def);
14700 ir_END_list(slow_inputs);
14701 ir_IF_TRUE(if_def);
14702 }
14703 }
14704 if (ZEND_TYPE_IS_SET(prop_info->type)) {
14705 ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref, arg2;
14706
14707 may_throw = 1;
14708
14709 jit_SET_EX_OPLINE(jit, opline);
14710
14711 if (Z_MODE(val_addr) == IS_REG) {
14712 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
14713 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
14714 return 0;
14715 }
14716 arg2 = jit_ZVAL_ADDR(jit, real_addr);
14717 } else {
14718 arg2 = jit_ZVAL_ADDR(jit, val_addr);
14719 }
14720
14721 if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
14722 ir_IF_FALSE(if_ref);
14723 noref_path = ir_END();
14724 ir_IF_TRUE(if_ref);
14725
14726 reference = jit_Z_PTR(jit, prop_addr);
14727 ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
14728 if_typed = jit_if_TYPED_REF(jit, reference);
14729 ir_IF_FALSE(if_typed);
14730 ref_path = ir_END();
14731 ir_IF_TRUE_cold(if_typed);
14732
14733 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
14734 reference,
14735 arg2,
14736 ir_CONST_FC_FUNC(binary_op));
14737
14738 ir_END_list(end_inputs);
14739
14740 ir_MERGE_2(noref_path, ref_path);
14741 prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
14742 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14743
14744 // JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
14745 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
14746 ref = ir_CONST_ADDR(prop_info);
14747 } else {
14748 int prop_info_offset =
14749 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
14750
14751 ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
14752 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
14753 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
14754 }
14755
14756 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_prop),
14757 prop_ref,
14758 ref,
14759 arg2,
14760 ir_CONST_FC_FUNC(binary_op));
14761
14762 ir_END_list(end_inputs);
14763 }
14764 }
14765
14766 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
14767 zend_jit_addr var_addr = prop_addr;
14768 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
14769 uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
14770
14771 if (use_prop_guard) {
14772 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
14773 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
14774 if (!exit_addr) {
14775 return 0;
14776 }
14777
14778 jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
14779 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
14780 }
14781
14782 if (var_info & MAY_BE_REF) {
14783 ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref, arg2;
14784
14785 may_throw = 1;
14786
14787 if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
14788 ir_IF_FALSE(if_ref);
14789 noref_path = ir_END();
14790 ir_IF_TRUE(if_ref);
14791
14792 reference = jit_Z_PTR(jit, var_addr);
14793 ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
14794 if_typed = jit_if_TYPED_REF(jit, reference);
14795 ir_IF_FALSE(if_typed);
14796 ref_path = ir_END();
14797 ir_IF_TRUE_cold(if_typed);
14798
14799 jit_SET_EX_OPLINE(jit, opline);
14800
14801 if (Z_MODE(val_addr) == IS_REG) {
14802 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
14803 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
14804 return 0;
14805 }
14806 arg2 = jit_ZVAL_ADDR(jit, real_addr);
14807 } else {
14808 arg2 = jit_ZVAL_ADDR(jit, val_addr);
14809 }
14810 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
14811 reference,
14812 arg2,
14813 ir_CONST_FC_FUNC(binary_op));
14814
14815 ir_END_list(end_inputs);
14816
14817 ir_MERGE_2(noref_path, ref_path);
14818 prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
14819 var_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14820
14821 var_info &= ~MAY_BE_REF;
14822 }
14823
14824 uint8_t val_op_type = (opline+1)->op1_type;
14825 if (val_op_type & (IS_TMP_VAR|IS_VAR)) {
14826 /* prevent FREE_OP in the helpers */
14827 val_op_type = IS_CV;
14828 }
14829
14830 switch (opline->extended_value) {
14831 case ZEND_ADD:
14832 case ZEND_SUB:
14833 case ZEND_MUL:
14834 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
14835 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
14836 if (opline->extended_value != ZEND_ADD ||
14837 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
14838 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) {
14839 may_throw = 1;
14840 }
14841 }
14842 if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info,
14843 1 /* may overflow */, 0)) {
14844 return 0;
14845 }
14846 break;
14847 case ZEND_BW_OR:
14848 case ZEND_BW_AND:
14849 case ZEND_BW_XOR:
14850 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
14851 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
14852 if ((var_info & MAY_BE_ANY) != MAY_BE_STRING ||
14853 (val_info & MAY_BE_ANY) != MAY_BE_STRING) {
14854 may_throw = 1;
14855 }
14856 }
14857 goto long_math;
14858 case ZEND_SL:
14859 case ZEND_SR:
14860 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
14861 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
14862 may_throw = 1;
14863 }
14864 if (val_op_type != IS_CONST ||
14865 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
14866 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) {
14867 may_throw = 1;
14868 }
14869 goto long_math;
14870 case ZEND_MOD:
14871 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
14872 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
14873 may_throw = 1;
14874 }
14875 if (val_op_type != IS_CONST ||
14876 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
14877 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) {
14878 may_throw = 1;
14879 }
14880 long_math:
14881 if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
14882 IS_CV, opline->op1, var_addr, var_info, NULL,
14883 val_op_type, (opline+1)->op1, val_addr, val_info,
14884 val_range,
14885 0, var_addr, var_def_info, var_info, /* may throw */ 1)) {
14886 return 0;
14887 }
14888 break;
14889 case ZEND_CONCAT:
14890 may_throw = 1;
14891 if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, val_addr, val_info, var_addr,
14892 0)) {
14893 return 0;
14894 }
14895 break;
14896 default:
14897 ZEND_UNREACHABLE();
14898 }
14899 if (end_inputs || slow_inputs) {
14900 ir_END_list(end_inputs);
14901 }
14902 }
14903
14904 if (slow_inputs) {
14905 ir_ref arg3;
14906
14907 ir_MERGE_list(slow_inputs);
14908
14909 may_throw = 1;
14910
14911 if (Z_MODE(val_addr) == IS_REG) {
14912 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
14913 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
14914 return 0;
14915 }
14916 arg3 = jit_ZVAL_ADDR(jit, real_addr);
14917 } else {
14918 arg3 = jit_ZVAL_ADDR(jit, val_addr);
14919 }
14920 jit_SET_EX_OPLINE(jit, opline);
14921 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
14922 ir_CALL_5(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_obj_op_helper),
14923 obj_ref,
14924 ir_CONST_ADDR(name),
14925 arg3,
14926 ir_ADD_OFFSET(run_time_cache, (opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
14927 ir_CONST_FC_FUNC(binary_op));
14928
14929 ir_END_list(end_inputs);
14930 }
14931
14932 if (end_inputs) {
14933 ir_MERGE_list(end_inputs);
14934 }
14935
14936 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
14937 val_info |= MAY_BE_RC1|MAY_BE_RCN;
14938 }
14939
14940 // JIT: FREE_OP_DATA();
14941 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
14942
14943 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
14944 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
14945 may_throw = 1;
14946 }
14947 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
14948 }
14949
14950 if (may_throw) {
14951 zend_jit_check_exception(jit);
14952 }
14953
14954 return 1;
14955 }
14956
14957 static int zend_jit_incdec_obj(zend_jit_ctx *jit,
14958 const zend_op *opline,
14959 const zend_op_array *op_array,
14960 zend_ssa *ssa,
14961 const zend_ssa_op *ssa_op,
14962 uint32_t op1_info,
14963 zend_jit_addr op1_addr,
14964 bool op1_indirect,
14965 zend_class_entry *ce,
14966 bool ce_is_instanceof,
14967 bool on_this,
14968 bool delayed_fetch_this,
14969 zend_class_entry *trace_ce,
14970 uint8_t prop_type)
14971 {
14972 zval *member;
14973 zend_string *name;
14974 zend_property_info *prop_info;
14975 zend_jit_addr res_addr = 0;
14976 zend_jit_addr prop_addr;
14977 bool use_prop_guard = 0;
14978 bool may_throw = 0;
14979 uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0;
14980 ir_ref obj_ref = IR_UNUSED;
14981 ir_ref prop_ref = IR_UNUSED;
14982 ir_ref end_inputs = IR_UNUSED;
14983 ir_ref slow_inputs = IR_UNUSED;
14984
14985 ZEND_ASSERT(opline->op2_type == IS_CONST);
14986 ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
14987
14988 if (opline->result_type != IS_UNUSED) {
14989 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
14990 }
14991
14992 member = RT_CONSTANT(opline, opline->op2);
14993 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
14994 name = Z_STR_P(member);
14995 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
14996
14997 if (on_this) {
14998 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
14999 obj_ref = jit_Z_PTR(jit, this_addr);
15000 } else {
15001 if (opline->op1_type == IS_VAR
15002 && (op1_info & MAY_BE_INDIRECT)
15003 && Z_REG(op1_addr) == ZREG_FP) {
15004 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
15005 }
15006 if (op1_info & MAY_BE_REF) {
15007 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
15008 }
15009 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
15010 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
15011 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
15012 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15013
15014 if (!exit_addr) {
15015 return 0;
15016 }
15017 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
15018 } else {
15019 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
15020 ir_IF_FALSE_cold(if_obj);
15021
15022 jit_SET_EX_OPLINE(jit, opline);
15023 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_incdec),
15024 jit_ZVAL_ADDR(jit, op1_addr),
15025 ir_CONST_ADDR(ZSTR_VAL(name)));
15026
15027 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
15028 ir_IF_TRUE(if_obj);
15029 }
15030 }
15031 obj_ref = jit_Z_PTR(jit, op1_addr);
15032 }
15033
15034 ZEND_ASSERT(obj_ref);
15035 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
15036 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
15037 if (prop_info) {
15038 ce = trace_ce;
15039 ce_is_instanceof = 0;
15040 if (!(op1_info & MAY_BE_CLASS_GUARD)) {
15041 if (on_this && JIT_G(current_frame)
15042 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
15043 ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
15044 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
15045 if (on_this && JIT_G(current_frame)) {
15046 JIT_G(current_frame)->ce = ce;
15047 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
15048 }
15049 } else {
15050 return 0;
15051 }
15052 if (ssa->var_info && ssa_op->op1_use >= 0) {
15053 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
15054 ssa->var_info[ssa_op->op1_use].ce = ce;
15055 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
15056 }
15057 if (ssa->var_info && ssa_op->op1_def >= 0) {
15058 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
15059 ssa->var_info[ssa_op->op1_def].ce = ce;
15060 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
15061 }
15062 }
15063 }
15064 }
15065
15066 use_prop_guard = (prop_type != IS_UNKNOWN
15067 && prop_type != IS_UNDEF
15068 && prop_type != IS_REFERENCE
15069 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
15070
15071 if (!prop_info) {
15072 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
15073 ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
15074 ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
15075
15076 ir_IF_FALSE_cold(if_same);
15077 ir_END_list(slow_inputs);
15078
15079 ir_IF_TRUE(if_same);
15080 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
15081 ir_ref prop_info_ref = ir_LOAD_A(
15082 ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
15083 ir_ref if_has_prop_info = ir_IF(prop_info_ref);
15084 ir_IF_TRUE_cold(if_has_prop_info);
15085 ir_END_list(slow_inputs);
15086
15087 ir_IF_FALSE(if_has_prop_info);
15088 }
15089 ir_ref offset_ref = ir_LOAD_A(
15090 ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
15091
15092 ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, IR_NULL));
15093 ir_IF_TRUE_cold(if_dynamic);
15094 ir_END_list(slow_inputs);
15095
15096 ir_IF_FALSE(if_dynamic);
15097
15098 prop_ref = ir_ADD_A(obj_ref, offset_ref);
15099 if (!use_prop_guard) {
15100 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
15101 ir_IF_FALSE_cold(if_def);
15102 ir_END_list(slow_inputs);
15103
15104 ir_IF_TRUE(if_def);
15105 }
15106 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
15107 } else {
15108 prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
15109 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
15110
15111 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
15112 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
15113 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
15114 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15115
15116 if (!exit_addr) {
15117 return 0;
15118 }
15119 ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
15120 } else {
15121 ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
15122 ir_IF_FALSE_cold(if_def);
15123 ir_END_list(slow_inputs);
15124 ir_IF_TRUE(if_def);
15125 }
15126 }
15127
15128 if (ZEND_TYPE_IS_SET(prop_info->type)) {
15129 const void *func;
15130 ir_ref ref;
15131
15132 may_throw = 1;
15133 jit_SET_EX_OPLINE(jit, opline);
15134
15135 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
15136 ref = ir_CONST_ADDR(prop_info);
15137 } else {
15138 int prop_info_offset =
15139 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
15140
15141 ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
15142 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
15143 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
15144 }
15145
15146 if (opline->result_type == IS_UNUSED) {
15147 switch (opline->opcode) {
15148 case ZEND_PRE_INC_OBJ:
15149 case ZEND_POST_INC_OBJ:
15150 func = zend_jit_inc_typed_prop;
15151 break;
15152 case ZEND_PRE_DEC_OBJ:
15153 case ZEND_POST_DEC_OBJ:
15154 func = zend_jit_dec_typed_prop;
15155 break;
15156 default:
15157 ZEND_UNREACHABLE();
15158 }
15159
15160 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(func), prop_ref, ref);
15161 } else {
15162 switch (opline->opcode) {
15163 case ZEND_PRE_INC_OBJ:
15164 func = zend_jit_pre_inc_typed_prop;
15165 break;
15166 case ZEND_PRE_DEC_OBJ:
15167 func = zend_jit_pre_dec_typed_prop;
15168 break;
15169 case ZEND_POST_INC_OBJ:
15170 func = zend_jit_post_inc_typed_prop;
15171 break;
15172 case ZEND_POST_DEC_OBJ:
15173 func = zend_jit_post_dec_typed_prop;
15174 break;
15175 default:
15176 ZEND_UNREACHABLE();
15177 }
15178 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(func),
15179 prop_ref,
15180 ref,
15181 jit_ZVAL_ADDR(jit, res_addr));
15182 }
15183 ir_END_list(end_inputs);
15184 }
15185 }
15186
15187 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
15188 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
15189 zend_jit_addr var_addr = prop_addr;
15190 ir_ref if_long = IR_UNUSED;
15191 ir_ref if_overflow = IR_UNUSED;
15192
15193 if (use_prop_guard) {
15194 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
15195 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15196 if (!exit_addr) {
15197 return 0;
15198 }
15199
15200 jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
15201 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
15202 }
15203
15204 if (var_info & MAY_BE_REF) {
15205 const void *func;
15206 ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref;
15207
15208 if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
15209 ir_IF_FALSE(if_ref);
15210 noref_path = ir_END();
15211 ir_IF_TRUE(if_ref);
15212
15213 reference = jit_Z_PTR(jit, var_addr);
15214 ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
15215 if_typed = jit_if_TYPED_REF(jit, reference);
15216 ir_IF_FALSE(if_typed);
15217 ref_path = ir_END();
15218 ir_IF_TRUE_cold(if_typed);
15219
15220 switch (opline->opcode) {
15221 case ZEND_PRE_INC_OBJ:
15222 func = zend_jit_pre_inc_typed_ref;
15223 break;
15224 case ZEND_PRE_DEC_OBJ:
15225 func = zend_jit_pre_dec_typed_ref;
15226 break;
15227 case ZEND_POST_INC_OBJ:
15228 func = zend_jit_post_inc_typed_ref;
15229 break;
15230 case ZEND_POST_DEC_OBJ:
15231 func = zend_jit_post_dec_typed_ref;
15232 break;
15233 default:
15234 ZEND_UNREACHABLE();
15235 }
15236
15237 may_throw = 1;
15238 jit_SET_EX_OPLINE(jit, opline);
15239 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(func),
15240 reference,
15241 (opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
15242
15243 ir_END_list(end_inputs);
15244
15245 ir_MERGE_2(noref_path, ref_path);
15246 prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
15247 var_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
15248
15249 var_info &= ~MAY_BE_REF;
15250 }
15251
15252 if (var_info & MAY_BE_LONG) {
15253 ir_ref addr, ref;
15254
15255 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
15256 if_long = jit_if_Z_TYPE(jit, var_addr, IS_LONG);
15257 ir_IF_TRUE(if_long);
15258 }
15259
15260 addr = jit_ZVAL_ADDR(jit, var_addr);
15261 ref = ir_LOAD_L(addr);
15262 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
15263 if (opline->result_type != IS_UNUSED) {
15264 jit_set_Z_LVAL(jit, res_addr, ref);
15265 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
15266 }
15267 }
15268 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
15269 ref = ir_ADD_OV_L(ref, ir_CONST_LONG(1));
15270 } else {
15271 ref = ir_SUB_OV_L(ref, ir_CONST_LONG(1));
15272 }
15273
15274 ir_STORE(addr, ref);
15275 if_overflow = ir_IF(ir_OVERFLOW(ref));
15276 ir_IF_FALSE(if_overflow);
15277
15278 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) {
15279 if (opline->result_type != IS_UNUSED) {
15280 jit_set_Z_LVAL(jit, res_addr, ref);
15281 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
15282 }
15283 }
15284 ir_END_list(end_inputs);
15285 }
15286
15287 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
15288 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
15289 may_throw = 1;
15290 }
15291 if (if_long) {
15292 ir_IF_FALSE_cold(if_long);
15293 }
15294 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
15295 jit_ZVAL_COPY(jit, res_addr, -1, var_addr, var_info, 1);
15296 }
15297 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
15298 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
15299 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_inc),
15300 jit_ZVAL_ADDR(jit, var_addr),
15301 jit_ZVAL_ADDR(jit, res_addr));
15302 } else {
15303 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(increment_function),
15304 jit_ZVAL_ADDR(jit, var_addr));
15305 }
15306 } else {
15307 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
15308 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_dec),
15309 jit_ZVAL_ADDR(jit, var_addr),
15310 jit_ZVAL_ADDR(jit, res_addr));
15311 } else {
15312 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(decrement_function),
15313 jit_ZVAL_ADDR(jit, var_addr));
15314 }
15315 }
15316
15317 ir_END_list(end_inputs);
15318 }
15319 if (var_info & MAY_BE_LONG) {
15320 ir_IF_TRUE_cold(if_overflow);
15321 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
15322 #if SIZEOF_ZEND_LONG == 4
15323 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0));
15324 jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0x41e00000));
15325 #else
15326 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x43e0000000000000));
15327 #endif
15328 jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
15329 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
15330 #if SIZEOF_ZEND_LONG == 4
15331 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
15332 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
15333 #else
15334 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
15335 #endif
15336 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
15337 }
15338 } else {
15339 #if SIZEOF_ZEND_LONG == 4
15340 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x00200000));
15341 jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0xc1e00000));
15342 #else
15343 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0xc3e0000000000000));
15344 #endif
15345 jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
15346 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
15347 #if SIZEOF_ZEND_LONG == 4
15348 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
15349 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
15350 #else
15351 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
15352 #endif
15353 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
15354 }
15355 }
15356 if (opline->result_type != IS_UNUSED
15357 && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ)
15358 && prop_info
15359 && !ZEND_TYPE_IS_SET(prop_info->type)
15360 && (res_info & MAY_BE_GUARD)
15361 && (res_info & MAY_BE_LONG)) {
15362 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
15363 uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
15364 int32_t exit_point;
15365 const void *exit_addr;
15366
15367 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
15368 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
15369 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15370 if (!exit_addr) {
15371 return 0;
15372 }
15373 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
15374 ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD;
15375 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
15376 } else {
15377 ir_END_list(end_inputs);
15378 }
15379 }
15380 }
15381
15382 if (slow_inputs) {
15383 const void *func;
15384
15385 ir_MERGE_list(slow_inputs);
15386
15387 // JIT: zend_jit_pre_inc_obj_helper(zobj, name, CACHE_ADDR(opline->extended_value), result);
15388 switch (opline->opcode) {
15389 case ZEND_PRE_INC_OBJ:
15390 func = zend_jit_pre_inc_obj_helper;
15391 break;
15392 case ZEND_PRE_DEC_OBJ:
15393 func = zend_jit_pre_dec_obj_helper;
15394 break;
15395 case ZEND_POST_INC_OBJ:
15396 func = zend_jit_post_inc_obj_helper;
15397 break;
15398 case ZEND_POST_DEC_OBJ:
15399 func = zend_jit_post_dec_obj_helper;
15400 break;
15401 default:
15402 ZEND_UNREACHABLE();
15403 }
15404
15405 may_throw = 1;
15406 jit_SET_EX_OPLINE(jit, opline);
15407 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
15408 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(func),
15409 obj_ref,
15410 ir_CONST_ADDR(name),
15411 ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
15412 (opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
15413
15414 ir_END_list(end_inputs);
15415 }
15416
15417 if (end_inputs) {
15418 ir_MERGE_list(end_inputs);
15419 }
15420
15421 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this && !op1_indirect) {
15422 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
15423 may_throw = 1;
15424 }
15425 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
15426 }
15427
15428 if (may_throw) {
15429 zend_jit_check_exception(jit);
15430 }
15431
15432 return 1;
15433 }
15434
15435 static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info)
15436 {
15437 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
15438 const zend_op *next_opline = NULL;
15439 ir_refs *slow_inputs;
15440
15441 ir_refs_init(slow_inputs, 8);
15442
15443 if (trace) {
15444 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
15445 ZEND_ASSERT(trace->opline != NULL);
15446 next_opline = trace->opline;
15447 }
15448
15449 if (opline->op1_type == IS_CONST) {
15450 zval *zv = RT_CONSTANT(opline, opline->op1);
15451 zval *jump_zv = NULL;
15452 int b;
15453
15454 if (opline->opcode == ZEND_SWITCH_LONG) {
15455 if (Z_TYPE_P(zv) == IS_LONG) {
15456 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
15457 }
15458 } else if (opline->opcode == ZEND_SWITCH_STRING) {
15459 if (Z_TYPE_P(zv) == IS_STRING) {
15460 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
15461 }
15462 } else if (opline->opcode == ZEND_MATCH) {
15463 if (Z_TYPE_P(zv) == IS_LONG) {
15464 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
15465 } else if (Z_TYPE_P(zv) == IS_STRING) {
15466 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
15467 }
15468 } else {
15469 ZEND_UNREACHABLE();
15470 }
15471 if (next_opline) {
15472 const zend_op *target;
15473
15474 if (jump_zv != NULL) {
15475 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv));
15476 } else {
15477 target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
15478 }
15479 ZEND_ASSERT(target == next_opline);
15480 } else {
15481 if (jump_zv != NULL) {
15482 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
15483 } else {
15484 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
15485 }
15486 _zend_jit_add_predecessor_ref(jit, b, jit->b, ir_END());
15487 jit->b = -1;
15488 }
15489 } else {
15490 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[opline - op_array->opcodes] : NULL;
15491 uint32_t op1_info = OP1_INFO();
15492 zend_jit_addr op1_addr = OP1_ADDR();
15493 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
15494 const zend_op *target;
15495 int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes];
15496 int b;
15497 int32_t exit_point;
15498 const void *exit_addr;
15499 const void *default_label = NULL;
15500 zval *zv;
15501
15502 if (next_opline) {
15503 if (next_opline != default_opline) {
15504 exit_point = zend_jit_trace_get_exit_point(default_opline, 0);
15505 default_label = zend_jit_trace_get_exit_addr(exit_point);
15506 if (!default_label) {
15507 return 0;
15508 }
15509 }
15510 }
15511
15512 if (opline->opcode == ZEND_SWITCH_LONG) {
15513 if (op1_info & MAY_BE_LONG) {
15514 const void *fallback_label = NULL;
15515
15516 if (next_opline) {
15517 if (next_opline != opline + 1) {
15518 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
15519 fallback_label = zend_jit_trace_get_exit_addr(exit_point);
15520 if (!fallback_label) {
15521 return 0;
15522 }
15523 }
15524 }
15525 if (op1_info & MAY_BE_REF) {
15526 ir_ref ref, if_long, fast_path, ref2;
15527
15528 ref = jit_ZVAL_ADDR(jit, op1_addr);
15529 if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15530 ir_IF_TRUE(if_long);
15531 fast_path = ir_END();
15532 ir_IF_FALSE_cold(if_long);
15533
15534 // JIT: ZVAL_DEREF(op)
15535 if (fallback_label) {
15536 jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
15537 } else {
15538 ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
15539 ir_IF_FALSE_cold(if_ref);
15540 ir_refs_add(slow_inputs, ir_END());
15541 ir_IF_TRUE(if_ref);
15542 }
15543
15544 ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
15545 op1_addr = ZEND_ADDR_REF_ZVAL(ref2);
15546
15547 if (fallback_label) {
15548 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
15549 } else {
15550 if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15551 ir_IF_FALSE_cold(if_long);
15552 ir_refs_add(slow_inputs, ir_END());
15553 ir_IF_TRUE(if_long);
15554 }
15555
15556 ir_MERGE_2(fast_path, ir_END());
15557 ref = ir_PHI_2(IR_ADDR, ref, ref2);
15558 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
15559 } else if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
15560 if (fallback_label) {
15561 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
15562 } else {
15563 ir_ref if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15564 ir_IF_FALSE_cold(if_long);
15565 ir_refs_add(slow_inputs, ir_END());
15566 ir_IF_TRUE(if_long);
15567 }
15568 }
15569 ir_ref ref = jit_Z_LVAL(jit, op1_addr);
15570
15571 if (!HT_IS_PACKED(jumptable)) {
15572 ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_index_find),
15573 ir_CONST_ADDR(jumptable), ref);
15574 ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
15575 /* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
15576 if (sizeof(Bucket) == 32) {
15577 ref = ir_SHR_L(ref, ir_CONST_LONG(5));
15578 } else {
15579 ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
15580 }
15581 }
15582 ref = ir_SWITCH(ref);
15583
15584 if (next_opline) {
15585 ir_ref continue_list = IR_UNUSED;
15586
15587 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
15588 ir_ref idx;
15589 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
15590
15591 if (HT_IS_PACKED(jumptable)) {
15592 idx = ir_CONST_LONG(zv - jumptable->arPacked);
15593 } else {
15594 idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
15595 }
15596 ir_CASE_VAL(ref, idx);
15597 if (target == next_opline) {
15598 ir_END_list(continue_list);
15599 } else {
15600 exit_point = zend_jit_trace_get_exit_point(target, 0);
15601 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15602 if (!exit_addr) {
15603 return 0;
15604 }
15605 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
15606 }
15607 } ZEND_HASH_FOREACH_END();
15608
15609 ir_CASE_DEFAULT(ref);
15610 if (next_opline == default_opline) {
15611 ir_END_list(continue_list);
15612 } else {
15613 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
15614 }
15615 if (continue_list) {
15616 ir_MERGE_list(continue_list);
15617 } else {
15618 ZEND_ASSERT(slow_inputs->count);
15619 ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
15620 }
15621 } else {
15622 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
15623 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
15624 b = ssa->cfg.map[target - op_array->opcodes];
15625 _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
15626 } ZEND_HASH_FOREACH_END();
15627
15628 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
15629 if (slow_inputs->count) {
15630 ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
15631 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
15632 }
15633 jit->b = -1;
15634 }
15635 } else if (!next_opline) {
15636 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
15637 jit->b = -1;
15638 }
15639 } else if (opline->opcode == ZEND_SWITCH_STRING) {
15640 if (op1_info & MAY_BE_STRING) {
15641 const void *fallback_label = NULL;
15642
15643 if (next_opline) {
15644 if (next_opline != opline + 1) {
15645 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
15646 fallback_label = zend_jit_trace_get_exit_addr(exit_point);
15647 if (!fallback_label) {
15648 return 0;
15649 }
15650 }
15651 }
15652 if (op1_info & MAY_BE_REF) {
15653 ir_ref ref, if_string, fast_path, ref2;
15654
15655 ref = jit_ZVAL_ADDR(jit, op1_addr);
15656 if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15657 ir_IF_TRUE(if_string);
15658 fast_path = ir_END();
15659 ir_IF_FALSE_cold(if_string);
15660
15661 // JIT: ZVAL_DEREF(op)
15662 if (fallback_label) {
15663 jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
15664 } else {
15665 ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15666 ir_IF_FALSE_cold(if_ref);
15667 ir_refs_add(slow_inputs, ir_END());
15668 ir_IF_TRUE(if_ref);
15669 }
15670
15671 ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
15672 op1_addr = ZEND_ADDR_REF_ZVAL(ref2);
15673
15674 if (fallback_label) {
15675 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
15676 } else {
15677 if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15678 ir_IF_FALSE_cold(if_string);
15679 ir_refs_add(slow_inputs, ir_END());
15680 ir_IF_TRUE(if_string);
15681 }
15682
15683 ir_MERGE_2(fast_path, ir_END());
15684 ref = ir_PHI_2(IR_ADDR, ref, ref2);
15685 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
15686 } else if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) {
15687 if (fallback_label) {
15688 jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, fallback_label);
15689 } else {
15690 ir_ref if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15691 ir_IF_FALSE_cold(if_string);
15692 ir_refs_add(slow_inputs, ir_END());
15693 ir_IF_TRUE(if_string);
15694 }
15695 }
15696
15697 ir_ref ref = jit_Z_PTR(jit, op1_addr);
15698 ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_find),
15699 ir_CONST_ADDR(jumptable), ref);
15700 ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
15701 /* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
15702 if (sizeof(Bucket) == 32) {
15703 ref = ir_SHR_L(ref, ir_CONST_LONG(5));
15704 } else {
15705 ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
15706 }
15707 ref = ir_SWITCH(ref);
15708
15709 if (next_opline) {
15710 ir_ref continue_list = IR_UNUSED;
15711
15712 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
15713 ir_ref idx;
15714 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
15715
15716 if (HT_IS_PACKED(jumptable)) {
15717 idx = ir_CONST_LONG(zv - jumptable->arPacked);
15718 } else {
15719 idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
15720 }
15721 ir_CASE_VAL(ref, idx);
15722 if (target == next_opline) {
15723 ir_END_list(continue_list);
15724 } else {
15725 exit_point = zend_jit_trace_get_exit_point(target, 0);
15726 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15727 if (!exit_addr) {
15728 return 0;
15729 }
15730 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
15731 }
15732 } ZEND_HASH_FOREACH_END();
15733
15734 ir_CASE_DEFAULT(ref);
15735 if (next_opline == default_opline) {
15736 ir_END_list(continue_list);
15737 } else {
15738 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
15739 }
15740 if (continue_list) {
15741 ir_MERGE_list(continue_list);
15742 } else {
15743 ZEND_ASSERT(slow_inputs->count);
15744 ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
15745 }
15746 } else {
15747 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
15748 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
15749 b = ssa->cfg.map[target - op_array->opcodes];
15750 _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
15751 } ZEND_HASH_FOREACH_END();
15752 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
15753 if (slow_inputs->count) {
15754 ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
15755 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
15756 }
15757 jit->b = -1;
15758 }
15759 } else if (!next_opline) {
15760 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
15761 jit->b = -1;
15762 }
15763 } else if (opline->opcode == ZEND_MATCH) {
15764 ir_ref if_type = IR_UNUSED, default_input_list = IR_UNUSED, ref = IR_UNUSED;
15765 ir_ref continue_list = IR_UNUSED;
15766
15767 if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) {
15768 ir_ref long_path = IR_UNUSED;
15769
15770 if (op1_info & MAY_BE_REF) {
15771 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
15772 }
15773 if (op1_info & MAY_BE_LONG) {
15774 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
15775 if (op1_info & (MAY_BE_STRING|MAY_BE_UNDEF)) {
15776 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15777 ir_IF_TRUE(if_type);
15778 } else if (default_label) {
15779 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, default_label);
15780 } else if (next_opline) {
15781 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15782 ir_IF_FALSE(if_type);
15783 ir_END_list(continue_list);
15784 ir_IF_TRUE(if_type);
15785 } else {
15786 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15787 ir_IF_FALSE(if_type);
15788 ir_END_list(default_input_list);
15789 ir_IF_TRUE(if_type);
15790 }
15791 }
15792 ref = jit_Z_LVAL(jit, op1_addr);
15793 ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_index_find),
15794 ir_CONST_ADDR(jumptable), ref);
15795 if (op1_info & MAY_BE_STRING) {
15796 long_path = ir_END();
15797 }
15798 }
15799 if (op1_info & MAY_BE_STRING) {
15800 if (if_type) {
15801 ir_IF_FALSE(if_type);
15802 if_type = IS_UNUSED;
15803 }
15804 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) {
15805 if (op1_info & MAY_BE_UNDEF) {
15806 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15807 ir_IF_TRUE(if_type);
15808 } else if (default_label) {
15809 jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, default_label);
15810 } else if (next_opline) {
15811 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15812 ir_IF_FALSE(if_type);
15813 ir_END_list(continue_list);
15814 ir_IF_TRUE(if_type);
15815 } else {
15816 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15817 ir_IF_FALSE(if_type);
15818 ir_END_list(default_input_list);
15819 ir_IF_TRUE(if_type);
15820 }
15821 }
15822 ir_ref ref2 = jit_Z_PTR(jit, op1_addr);
15823 ref2 = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_find),
15824 ir_CONST_ADDR(jumptable), ref2);
15825 if (op1_info & MAY_BE_LONG) {
15826 ir_MERGE_WITH(long_path);
15827 ref = ir_PHI_2(IR_LONG, ref2, ref);
15828 } else {
15829 ref = ref2;
15830 }
15831 }
15832
15833 ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
15834 /* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
15835 if (HT_IS_PACKED(jumptable)) {
15836 ZEND_ASSERT(sizeof(zval) == 16);
15837 ref = ir_SHR_L(ref, ir_CONST_LONG(4));
15838 } else {
15839 if (sizeof(Bucket) == 32) {
15840 ref = ir_SHR_L(ref, ir_CONST_LONG(5));
15841 } else {
15842 ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
15843 }
15844 }
15845 ref = ir_SWITCH(ref);
15846
15847 if (next_opline) {
15848 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
15849 ir_ref idx;
15850 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
15851
15852 if (HT_IS_PACKED(jumptable)) {
15853 idx = ir_CONST_LONG(zv - jumptable->arPacked);
15854 } else {
15855 idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
15856 }
15857 ir_CASE_VAL(ref, idx);
15858 if (target == next_opline) {
15859 ir_END_list(continue_list);
15860 } else {
15861 exit_point = zend_jit_trace_get_exit_point(target, 0);
15862 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15863 if (!exit_addr) {
15864 return 0;
15865 }
15866 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
15867 }
15868 } ZEND_HASH_FOREACH_END();
15869
15870 ir_CASE_DEFAULT(ref);
15871 if (next_opline == default_opline) {
15872 ir_END_list(continue_list);
15873 } else {
15874 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
15875 }
15876 } else {
15877 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
15878 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
15879 b = ssa->cfg.map[target - op_array->opcodes];
15880 _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
15881 } ZEND_HASH_FOREACH_END();
15882 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
15883 }
15884 } else if (!(op1_info & MAY_BE_UNDEF)) {
15885 if (next_opline) {
15886 if (next_opline == default_opline) {
15887 ir_END_list(continue_list);
15888 } else {
15889 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
15890 }
15891 } else {
15892 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
15893 }
15894 }
15895
15896 if (op1_info & MAY_BE_UNDEF) {
15897 if (if_type) {
15898 ir_IF_FALSE(if_type);
15899 if_type = IS_UNUSED;
15900 }
15901 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) {
15902 if (default_label) {
15903 jit_guard_Z_TYPE(jit, op1_addr, IS_UNDEF, default_label);
15904 } else if (next_opline) {
15905 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
15906 ir_IF_TRUE(if_def);
15907 ir_END_list(continue_list);
15908 ir_IF_FALSE_cold(if_def);
15909 } else {
15910 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
15911 ir_IF_TRUE(if_def);
15912 ir_END_list(default_input_list);
15913 ir_IF_FALSE_cold(if_def);
15914 }
15915 }
15916
15917 jit_SET_EX_OPLINE(jit, opline);
15918 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
15919 ir_CONST_U32(opline->op1.var));
15920 zend_jit_check_exception_undef_result(jit, opline);
15921 if (default_label) {
15922 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
15923 } else if (next_opline) {
15924 ir_END_list(continue_list);
15925 } else {
15926 ir_END_list(default_input_list);
15927 }
15928 }
15929 if (next_opline) {
15930 ZEND_ASSERT(continue_list);
15931 ir_MERGE_list(continue_list);
15932 } else {
15933 if (default_input_list) {
15934 if (jit->ctx.ir_base[ref].op == IR_SWITCH) {
15935 ZEND_ASSERT(jit->ctx.ir_base[ref].op3 == IR_UNUSED);
15936 jit->ctx.ir_base[ref].op3 = default_input_list;
15937 } else {
15938 ir_MERGE_list(default_input_list);
15939 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
15940 }
15941 }
15942 jit->b = -1;
15943 }
15944 } else {
15945 ZEND_UNREACHABLE();
15946 }
15947 }
15948 return 1;
15949 }
15950
15951 static int zend_jit_start(zend_jit_ctx *jit, const zend_op_array *op_array, zend_ssa *ssa)
15952 {
15953 int i, count;
15954 zend_basic_block *bb;
15955
15956 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : (IR_START_BR_TARGET|IR_ENTRY_BR_TARGET));
15957
15958 jit->ctx.spill_base = ZREG_FP;
15959
15960 jit->op_array = jit->current_op_array = op_array;
15961 jit->ssa = ssa;
15962 jit->bb_start_ref = zend_arena_calloc(&CG(arena), ssa->cfg.blocks_count * 2, sizeof(ir_ref));
15963 jit->bb_predecessors = jit->bb_start_ref + ssa->cfg.blocks_count;
15964
15965 count = 0;
15966 for (i = 0, bb = ssa->cfg.blocks; i < ssa->cfg.blocks_count; i++, bb++) {
15967 jit->bb_predecessors[i] = count;
15968 count += bb->predecessors_count;
15969 }
15970 jit->bb_edges = zend_arena_calloc(&CG(arena), count, sizeof(ir_ref));
15971
15972 if (!GCC_GLOBAL_REGS) {
15973 ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
15974 jit_STORE_FP(jit, ref);
15975 jit->ctx.flags |= IR_FASTCALL_FUNC;
15976 }
15977
15978 return 1;
15979 }
15980
15981 static void *zend_jit_finish(zend_jit_ctx *jit)
15982 {
15983 void *entry;
15984 size_t size;
15985 zend_string *str = NULL;
15986
15987 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP|
15988 ZEND_JIT_DEBUG_IR_SRC|ZEND_JIT_DEBUG_IR_AFTER_SCCP|ZEND_JIT_DEBUG_IR_AFTER_SCCP|
15989 ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE|ZEND_JIT_DEBUG_IR_AFTER_REGS|ZEND_JIT_DEBUG_IR_FINAL|ZEND_JIT_DEBUG_IR_CODEGEN)) {
15990 if (jit->name) {
15991 str = zend_string_copy(jit->name);
15992 } else {
15993 str = zend_jit_func_name(jit->op_array);
15994 }
15995 }
15996
15997 if (jit->op_array) {
15998 /* Only for function JIT */
15999 _zend_jit_fix_merges(jit);
16000 #if defined(IR_TARGET_AARCH64)
16001 } else if (jit->trace) {
16002 jit->ctx.deoptimization_exits = jit->trace->exit_count;
16003 jit->ctx.get_exit_addr = zend_jit_trace_get_exit_addr;
16004 #endif
16005 } else {
16006 #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
16007 jit->ctx.flags |= IR_GEN_CACHE_DEMOTE;
16008 #endif
16009 }
16010
16011 entry = zend_jit_ir_compile(&jit->ctx, &size, str ? ZSTR_VAL(str) : NULL);
16012 if (entry) {
16013 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
16014 #ifdef HAVE_CAPSTONE
16015 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
16016 if (str) {
16017 ir_disasm_add_symbol(ZSTR_VAL(str), (uintptr_t)entry, size);
16018 }
16019 ir_disasm(str ? ZSTR_VAL(str) : "unknown",
16020 entry, size,
16021 (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0,
16022 &jit->ctx, stderr);
16023 }
16024 #endif
16025 #ifndef _WIN32
16026 if (str) {
16027 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
16028 uintptr_t sp_offset = 0;
16029
16030 // ir_mem_unprotect(entry, size);
16031 if (!(jit->ctx.flags & IR_FUNCTION)
16032 && zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
16033 #if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64)
16034 sp_offset = zend_jit_hybrid_vm_sp_adj;
16035 #else
16036 sp_offset = sizeof(void*);
16037 #endif
16038 } else {
16039 sp_offset = sizeof(void*);
16040 }
16041 ir_gdb_register(ZSTR_VAL(str), entry, size, sp_offset, 0);
16042 // ir_mem_protect(entry, size);
16043 }
16044
16045 if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
16046 ir_perf_map_register(ZSTR_VAL(str), entry, size);
16047 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
16048 ir_perf_jitdump_register(ZSTR_VAL(str), entry, size);
16049 }
16050 }
16051 }
16052 #endif
16053 }
16054
16055 if (jit->op_array) {
16056 /* Only for function JIT */
16057 const zend_op_array *op_array = jit->op_array;
16058 zend_op *opline = (zend_op*)op_array->opcodes;
16059
16060 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
16061 while (opline->opcode == ZEND_RECV) {
16062 opline++;
16063 }
16064 }
16065 opline->handler = entry;
16066
16067 if (jit->ctx.entries_count) {
16068 /* For all entries */
16069 int i = jit->ctx.entries_count;
16070 do {
16071 ir_insn *insn = &jit->ctx.ir_base[jit->ctx.entries[--i]];
16072 op_array->opcodes[insn->op2].handler = (char*)entry + insn->op3;
16073 } while (i != 0);
16074 }
16075 } else {
16076 /* Only for tracing JIT */
16077 zend_jit_trace_info *t = jit->trace;
16078 zend_jit_trace_stack *stack;
16079 uint32_t i;
16080
16081 if (t) {
16082 for (i = 0; i < t->stack_map_size; i++) {
16083 stack = t->stack_map + i;
16084 if (stack->flags & ZREG_SPILL_SLOT) {
16085 stack->reg = (jit->ctx.flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP;
16086 stack->ref = ir_get_spill_slot_offset(&jit->ctx, stack->ref);
16087 }
16088 }
16089 }
16090
16091 zend_jit_trace_add_code(entry, size);
16092 }
16093 }
16094
16095 if (str) {
16096 zend_string_release(str);
16097 }
16098
16099 return entry;
16100 }
16101
16102 static const void *zend_jit_trace_allocate_exit_group(uint32_t n)
16103 {
16104 const void *entry;
16105 size_t size;
16106 ir_code_buffer code_buffer;
16107
16108 code_buffer.start = dasm_buf;
16109 code_buffer.end = dasm_end;
16110 code_buffer.pos = *dasm_ptr;
16111
16112 entry = ir_emit_exitgroup(n, ZEND_JIT_EXIT_POINTS_PER_GROUP, zend_jit_stub_handlers[jit_stub_trace_exit],
16113 &code_buffer, &size);
16114
16115 *dasm_ptr = code_buffer.pos;
16116
16117 if (entry) {
16118 #ifdef HAVE_CAPSTONE
16119 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
16120 uint32_t i;
16121 char name[32];
16122
16123 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
16124 sprintf(name, "jit$$trace_exit_%d", n + i);
16125 ir_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING);
16126 }
16127 }
16128 #endif
16129 }
16130
16131 return entry;
16132 }
16133
16134 static int zend_jit_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint8_t type)
16135 {
16136 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
16137 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
16138 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
16139
16140 if (!exit_addr) {
16141 return 0;
16142 }
16143 ir_GUARD(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr));
16144
16145 return 1;
16146 }
16147
16148 static int zend_jit_scalar_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var)
16149 {
16150 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
16151 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
16152 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
16153
16154 if (!exit_addr) {
16155 return 0;
16156 }
16157 ir_GUARD(ir_LT(jit_Z_TYPE(jit, addr), ir_CONST_U8(IS_STRING)), ir_CONST_ADDR(exit_addr));
16158
16159 return 1;
16160 }
16161
16162 static bool zend_jit_noref_guard(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_addr)
16163 {
16164 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
16165 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
16166
16167 if (!exit_addr) {
16168 return 0;
16169 }
16170 ir_GUARD(ir_NE(jit_Z_TYPE(jit, var_addr), ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
16171
16172 return 1;
16173 }
16174
16175 static int zend_jit_trace_opline_guard(zend_jit_ctx *jit, const zend_op *opline)
16176 {
16177 uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0);
16178 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
16179
16180 if (!exit_addr) {
16181 return 0;
16182 }
16183
16184 ir_GUARD(jit_CMP_IP(jit, IR_EQ, opline), ir_CONST_ADDR(exit_addr));
16185 zend_jit_set_last_valid_opline(jit, opline);
16186
16187 return 1;
16188 }
16189
16190 static bool zend_jit_guard_reference(zend_jit_ctx *jit,
16191 const zend_op *opline,
16192 zend_jit_addr *var_addr_ptr,
16193 zend_jit_addr *ref_addr_ptr,
16194 bool add_ref_guard)
16195 {
16196 zend_jit_addr var_addr = *var_addr_ptr;
16197 const void *exit_addr = NULL;
16198 ir_ref ref;
16199
16200 if (add_ref_guard) {
16201 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
16202
16203 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
16204 if (!exit_addr) {
16205 return 0;
16206 }
16207
16208 ref = jit_Z_TYPE(jit, var_addr);
16209 ir_GUARD(ir_EQ(ref, ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
16210 }
16211
16212 ref = jit_Z_PTR(jit, var_addr);
16213 *ref_addr_ptr = ZEND_ADDR_REF_ZVAL(ref);
16214 ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
16215 var_addr = ZEND_ADDR_REF_ZVAL(ref);
16216 *var_addr_ptr = var_addr;
16217
16218 return 1;
16219 }
16220
16221 static bool zend_jit_fetch_reference(zend_jit_ctx *jit,
16222 const zend_op *opline,
16223 uint8_t var_type,
16224 uint32_t *var_info_ptr,
16225 zend_jit_addr *var_addr_ptr,
16226 bool add_ref_guard,
16227 bool add_type_guard)
16228 {
16229 zend_jit_addr var_addr = *var_addr_ptr;
16230 uint32_t var_info = *var_info_ptr;
16231 const void *exit_addr = NULL;
16232 ir_ref ref;
16233
16234 if (add_ref_guard || add_type_guard) {
16235 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
16236
16237 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
16238 if (!exit_addr) {
16239 return 0;
16240 }
16241 }
16242
16243 if (add_ref_guard) {
16244 ref = jit_Z_TYPE(jit, var_addr);
16245 ir_GUARD(ir_EQ(ref, ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
16246 }
16247 if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) {
16248 /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */
16249 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper),
16250 jit_ZVAL_ADDR(jit, var_addr));
16251 *var_addr_ptr = var_addr;
16252 } else {
16253 ref = jit_Z_PTR(jit, var_addr);
16254 ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
16255 var_addr = ZEND_ADDR_REF_ZVAL(ref);
16256 *var_addr_ptr = var_addr;
16257 }
16258
16259 if (var_type != IS_UNKNOWN) {
16260 var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED);
16261 }
16262 if (add_type_guard
16263 && var_type != IS_UNKNOWN
16264 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
16265 ref = jit_Z_TYPE(jit, var_addr);
16266 ir_GUARD(ir_EQ(ref, ir_CONST_U8(var_type)), ir_CONST_ADDR(exit_addr));
16267
16268 ZEND_ASSERT(var_info & (1 << var_type));
16269 if (var_type < IS_STRING) {
16270 var_info = (1 << var_type);
16271 } else if (var_type != IS_ARRAY) {
16272 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
16273 } else {
16274 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
16275 }
16276
16277 *var_info_ptr = var_info;
16278 } else {
16279 var_info &= ~MAY_BE_REF;
16280 *var_info_ptr = var_info;
16281 }
16282 *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */
16283
16284 return 1;
16285 }
16286
16287 static bool zend_jit_fetch_indirect_var(zend_jit_ctx *jit, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard)
16288 {
16289 zend_jit_addr var_addr = *var_addr_ptr;
16290 uint32_t var_info = *var_info_ptr;
16291 int32_t exit_point;
16292 const void *exit_addr;
16293 ir_ref ref = IR_UNUSED;
16294
16295 if (add_indirect_guard) {
16296 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
16297 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
16298
16299 if (!exit_addr) {
16300 return 0;
16301 }
16302 jit_guard_Z_TYPE(jit, var_addr, IS_INDIRECT, exit_addr);
16303 ref = jit_Z_PTR(jit, var_addr);
16304 } else {
16305 /* This LOAD of INDIRECT VAR, stored by the previous FETCH_(DIM/OBJ)_W,
16306 * is eliminated by store forwarding (S2L) */
16307 ref = jit_Z_PTR(jit, var_addr);
16308 }
16309 *var_info_ptr &= ~MAY_BE_INDIRECT;
16310 var_addr = ZEND_ADDR_REF_ZVAL(ref);
16311 *var_addr_ptr = var_addr;
16312
16313 if (var_type != IS_UNKNOWN) {
16314 var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED);
16315 }
16316 if (!(var_type & IS_TRACE_REFERENCE)
16317 && var_type != IS_UNKNOWN
16318 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
16319 exit_point = zend_jit_trace_get_exit_point(opline, 0);
16320 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
16321
16322 if (!exit_addr) {
16323 return 0;
16324 }
16325
16326 jit_guard_Z_TYPE(jit, var_addr, var_type, exit_addr);
16327
16328 //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info);
16329 ZEND_ASSERT(var_info & (1 << var_type));
16330 if (var_type < IS_STRING) {
16331 var_info = (1 << var_type);
16332 } else if (var_type != IS_ARRAY) {
16333 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
16334 } else {
16335 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
16336 }
16337
16338 *var_info_ptr = var_info;
16339 }
16340
16341 return 1;
16342 }
16343
16344 static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace)
16345 {
16346 zend_jit_op_array_trace_extension *jit_extension =
16347 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
16348 size_t offset = jit_extension->offset;
16349 const void *handler =
16350 (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
16351 ir_ref ref;
16352
16353 zend_jit_set_ip(jit, opline);
16354 if (GCC_GLOBAL_REGS) {
16355 ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
16356 } else {
16357 ref = jit_FP(jit);
16358 ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
16359 }
16360 if (may_throw
16361 && opline->opcode != ZEND_RETURN
16362 && opline->opcode != ZEND_RETURN_BY_REF) {
16363 zend_jit_check_exception(jit);
16364 }
16365
16366 while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) {
16367 trace++;
16368 }
16369
16370 if (!GCC_GLOBAL_REGS
16371 && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) {
16372 if (opline->opcode == ZEND_RETURN ||
16373 opline->opcode == ZEND_RETURN_BY_REF ||
16374 opline->opcode == ZEND_DO_UCALL ||
16375 opline->opcode == ZEND_DO_FCALL_BY_NAME ||
16376 opline->opcode == ZEND_DO_FCALL ||
16377 opline->opcode == ZEND_GENERATOR_CREATE) {
16378
16379 ir_ref addr = jit_EG(current_execute_data);
16380
16381 jit_STORE_FP(jit, ir_LOAD_A(addr));
16382 }
16383 }
16384
16385 if (zend_jit_trace_may_exit(op_array, opline)) {
16386 if (opline->opcode == ZEND_RETURN ||
16387 opline->opcode == ZEND_RETURN_BY_REF ||
16388 opline->opcode == ZEND_GENERATOR_CREATE) {
16389
16390 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
16391 if (trace->op != ZEND_JIT_TRACE_END ||
16392 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
16393 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
16394 /* this check may be handled by the following OPLINE guard or jmp [IP] */
16395 ir_GUARD(ir_NE(jit_IP(jit), ir_CONST_ADDR(zend_jit_halt_op)),
16396 jit_STUB_ADDR(jit, jit_stub_trace_halt));
16397 }
16398 } else if (GCC_GLOBAL_REGS) {
16399 ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
16400 } else {
16401 ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
16402 }
16403 } else if (opline->opcode == ZEND_EXIT ||
16404 opline->opcode == ZEND_GENERATOR_RETURN ||
16405 opline->opcode == ZEND_YIELD ||
16406 opline->opcode == ZEND_YIELD_FROM) {
16407 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_halt));
16408 ir_BEGIN(IR_UNUSED); /* unreachable block */
16409 }
16410 if (trace->op != ZEND_JIT_TRACE_END ||
16411 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
16412 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
16413
16414 const zend_op *next_opline = trace->opline;
16415 const zend_op *exit_opline = NULL;
16416 uint32_t exit_point;
16417 const void *exit_addr;
16418 uint32_t old_info = 0;
16419 uint32_t old_res_info = 0;
16420 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
16421
16422 if (zend_is_smart_branch(opline)) {
16423 bool exit_if_true = 0;
16424 exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true);
16425 } else {
16426 switch (opline->opcode) {
16427 case ZEND_JMPZ:
16428 case ZEND_JMPNZ:
16429 case ZEND_JMPZ_EX:
16430 case ZEND_JMPNZ_EX:
16431 case ZEND_JMP_SET:
16432 case ZEND_COALESCE:
16433 case ZEND_JMP_NULL:
16434 case ZEND_FE_RESET_R:
16435 case ZEND_FE_RESET_RW:
16436 exit_opline = (trace->opline == opline + 1) ?
16437 OP_JMP_ADDR(opline, opline->op2) :
16438 opline + 1;
16439 break;
16440 case ZEND_FE_FETCH_R:
16441 case ZEND_FE_FETCH_RW:
16442 exit_opline = (trace->opline == opline + 1) ?
16443 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
16444 opline + 1;
16445 break;
16446
16447 }
16448 }
16449
16450 switch (opline->opcode) {
16451 case ZEND_FE_FETCH_R:
16452 case ZEND_FE_FETCH_RW:
16453 if (opline->op2_type != IS_UNUSED) {
16454 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var));
16455 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1);
16456 }
16457 break;
16458 case ZEND_BIND_INIT_STATIC_OR_JMP:
16459 if (opline->op1_type == IS_CV) {
16460 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
16461 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1);
16462 }
16463 break;
16464 }
16465 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
16466 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
16467 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
16468 }
16469 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
16470 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
16471
16472 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
16473 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
16474 }
16475 switch (opline->opcode) {
16476 case ZEND_FE_FETCH_R:
16477 case ZEND_FE_FETCH_RW:
16478 if (opline->op2_type != IS_UNUSED) {
16479 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info);
16480 }
16481 break;
16482 case ZEND_BIND_INIT_STATIC_OR_JMP:
16483 if (opline->op1_type == IS_CV) {
16484 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
16485 }
16486 break;
16487 }
16488
16489 if (!exit_addr) {
16490 return 0;
16491 }
16492 ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), ir_CONST_ADDR(exit_addr));
16493 }
16494 }
16495
16496 zend_jit_set_last_valid_opline(jit, trace->opline);
16497
16498 return 1;
16499 }
16500
16501 static int zend_jit_deoptimizer_start(zend_jit_ctx *jit,
16502 zend_string *name,
16503 uint32_t trace_num,
16504 uint32_t exit_num)
16505 {
16506 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
16507
16508 jit->ctx.spill_base = ZREG_FP;
16509
16510 jit->op_array = NULL;
16511 jit->ssa = NULL;
16512 jit->name = zend_string_copy(name);
16513
16514 jit->ctx.flags |= IR_SKIP_PROLOGUE;
16515
16516 return 1;
16517 }
16518
16519 static int zend_jit_trace_start(zend_jit_ctx *jit,
16520 const zend_op_array *op_array,
16521 zend_ssa *ssa,
16522 zend_string *name,
16523 uint32_t trace_num,
16524 zend_jit_trace_info *parent,
16525 uint32_t exit_num)
16526 {
16527 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
16528
16529 jit->ctx.spill_base = ZREG_FP;
16530
16531 jit->op_array = NULL;
16532 jit->current_op_array = op_array;
16533 jit->ssa = ssa;
16534 jit->name = zend_string_copy(name);
16535
16536 if (!GCC_GLOBAL_REGS) {
16537 if (!parent) {
16538 ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
16539 jit_STORE_FP(jit, ref);
16540 jit->ctx.flags |= IR_FASTCALL_FUNC;
16541 }
16542 }
16543
16544 if (parent) {
16545 jit->ctx.flags |= IR_SKIP_PROLOGUE;
16546 }
16547
16548 if (parent) {
16549 int i;
16550 int parent_vars_count = parent->exit_info[exit_num].stack_size;
16551 zend_jit_trace_stack *parent_stack = parent_vars_count == 0 ? NULL :
16552 parent->stack_map +
16553 parent->exit_info[exit_num].stack_offset;
16554
16555 /* prevent clobbering of registers used for deoptimization */
16556 for (i = 0; i < parent_vars_count; i++) {
16557 if (STACK_FLAGS(parent_stack, i) != ZREG_CONST
16558 && STACK_REG(parent_stack, i) != ZREG_NONE) {
16559 int32_t reg = STACK_REG(parent_stack, i);
16560 ir_type type;
16561
16562 if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_COPY) {
16563 type = IR_ADDR;
16564 } else if (STACK_TYPE(parent_stack, i) == IS_LONG) {
16565 type = IR_LONG;
16566 } else if (STACK_TYPE(parent_stack, i) == IS_DOUBLE) {
16567 type = IR_DOUBLE;
16568 } else {
16569 ZEND_UNREACHABLE();
16570 }
16571 if (ssa && ssa->vars[i].no_val) {
16572 /* pass */
16573 } else {
16574 ir_ref ref = ir_RLOAD(type, reg);
16575
16576 if (STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) {
16577 /* op3 is used as a flag that the value is already stored in memory.
16578 * In case the IR framework decides to spill the result of IR_LOAD,
16579 * it doesn't have to store the value once again.
16580 *
16581 * See: insn->op3 check in ir_emit_rload()
16582 */
16583 ir_set_op(&jit->ctx, ref, 3, EX_NUM_TO_VAR(i));
16584 }
16585 }
16586 }
16587 }
16588 }
16589
16590 if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
16591 ZEND_ASSERT(parent->exit_info[exit_num].poly_func_reg >= 0 && parent->exit_info[exit_num].poly_this_reg >= 0);
16592 ir_RLOAD_A(parent->exit_info[exit_num].poly_func_reg);
16593 ir_RLOAD_A(parent->exit_info[exit_num].poly_this_reg);
16594 }
16595
16596 ir_STORE(jit_EG(jit_trace_num), ir_CONST_U32(trace_num));
16597
16598 return 1;
16599 }
16600
16601 static int zend_jit_trace_begin_loop(zend_jit_ctx *jit)
16602 {
16603 return ir_LOOP_BEGIN(ir_END());
16604 }
16605
16606 static void zend_jit_trace_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi)
16607 {
16608 int dst_var = phi->ssa_var;
16609 int src_var = phi->sources[0];
16610 ir_ref ref;
16611
16612 ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
16613 ZEND_ASSERT(jit->ra[src_var].ref != IR_UNUSED && jit->ra[src_var].ref != IR_NULL);
16614
16615 ref = ir_PHI_2(
16616 (jit->ssa->var_info[src_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE,
16617 zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)), IR_UNUSED);
16618
16619 src_var = phi->sources[1];
16620 ZEND_ASSERT(jit->ra[src_var].ref == IR_NULL);
16621 jit->ra[src_var].flags |= ZREG_FORWARD;
16622
16623 zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref);
16624 }
16625
16626 static int zend_jit_trace_end_loop(zend_jit_ctx *jit, int loop_ref, const void *timeout_exit_addr)
16627 {
16628 if (timeout_exit_addr) {
16629 zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
16630 }
16631 ZEND_ASSERT(jit->ctx.ir_base[loop_ref].op2 == IR_UNUSED);
16632 ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
16633 return 1;
16634 }
16635
16636 static int zend_jit_trace_return(zend_jit_ctx *jit, bool original_handler, const zend_op *opline)
16637 {
16638 if (GCC_GLOBAL_REGS) {
16639 if (!original_handler) {
16640 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
16641 } else {
16642 ir_TAILCALL(IR_VOID, zend_jit_orig_opline_handler(jit));
16643 }
16644 } else {
16645 if (original_handler) {
16646 ir_ref ref;
16647 ir_ref addr = zend_jit_orig_opline_handler(jit);
16648
16649 #if defined(IR_TARGET_X86)
16650 addr = ir_CAST_FC_FUNC(addr);
16651 #endif
16652 ref = ir_CALL_1(IR_I32, addr, jit_FP(jit));
16653 if (opline &&
16654 (opline->opcode == ZEND_RETURN
16655 || opline->opcode == ZEND_RETURN_BY_REF
16656 || opline->opcode == ZEND_GENERATOR_RETURN
16657 || opline->opcode == ZEND_GENERATOR_CREATE
16658 || opline->opcode == ZEND_YIELD
16659 || opline->opcode == ZEND_YIELD_FROM)) {
16660 ir_RETURN(ref);
16661 return 1;
16662 }
16663 }
16664 ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
16665 }
16666 return 1;
16667 }
16668
16669 static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr)
16670 {
16671 return ir_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
16672 }
16673
16674 static int zend_jit_trace_link_to_root(zend_jit_ctx *jit, zend_jit_trace_info *t, const void *timeout_exit_addr)
16675 {
16676 const void *link_addr;
16677
16678 /* Skip prologue. */
16679 ZEND_ASSERT(zend_jit_trace_prologue_size != (size_t)-1);
16680 link_addr = (const void*)((const char*)t->code_start + zend_jit_trace_prologue_size);
16681
16682 if (timeout_exit_addr) {
16683 zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
16684 }
16685 ir_IJMP(ir_CONST_ADDR(link_addr));
16686
16687 return 1;
16688 }
16689
16690 static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace)
16691 {
16692 uint32_t op1_info, op2_info;
16693
16694 switch (opline->opcode) {
16695 case ZEND_SEND_VAR:
16696 case ZEND_SEND_VAL:
16697 case ZEND_SEND_VAL_EX:
16698 return (opline->op2_type != IS_CONST) && (opline->opcode != ZEND_SEND_VAL_EX || opline->op2.num <= MAX_ARG_FLAG_NUM);
16699 case ZEND_QM_ASSIGN:
16700 case ZEND_IS_SMALLER:
16701 case ZEND_IS_SMALLER_OR_EQUAL:
16702 case ZEND_IS_EQUAL:
16703 case ZEND_IS_NOT_EQUAL:
16704 case ZEND_IS_IDENTICAL:
16705 case ZEND_IS_NOT_IDENTICAL:
16706 case ZEND_CASE:
16707 return 1;
16708 case ZEND_RETURN:
16709 return (op_array->type != ZEND_EVAL_CODE && op_array->function_name);
16710 case ZEND_ASSIGN:
16711 return (opline->op1_type == IS_CV);
16712 case ZEND_ASSIGN_OP:
16713 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
16714 return 0;
16715 }
16716 op1_info = OP1_INFO();
16717 op2_info = OP2_INFO();
16718 return zend_jit_supported_binary_op(opline->extended_value, op1_info, op2_info);
16719 case ZEND_ADD:
16720 case ZEND_SUB:
16721 case ZEND_MUL:
16722 op1_info = OP1_INFO();
16723 op2_info = OP2_INFO();
16724 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
16725 return 0;
16726 }
16727 if (trace && trace->op1_type != IS_UNKNOWN) {
16728 op1_info &= 1U << (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
16729 }
16730 if (trace && trace->op2_type != IS_UNKNOWN) {
16731 op2_info &= 1U << (trace->op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
16732 }
16733 return !(op1_info & MAY_BE_UNDEF)
16734 && !(op2_info & MAY_BE_UNDEF)
16735 && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
16736 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
16737 case ZEND_BW_OR:
16738 case ZEND_BW_AND:
16739 case ZEND_BW_XOR:
16740 case ZEND_SL:
16741 case ZEND_SR:
16742 case ZEND_MOD:
16743 op1_info = OP1_INFO();
16744 op2_info = OP2_INFO();
16745 if (trace && trace->op1_type != IS_UNKNOWN) {
16746 op1_info &= 1U << (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
16747 }
16748 if (trace && trace->op2_type != IS_UNKNOWN) {
16749 op2_info &= 1U << (trace->op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
16750 }
16751 return (op1_info & MAY_BE_LONG)
16752 && (op2_info & MAY_BE_LONG);
16753 case ZEND_PRE_INC:
16754 case ZEND_PRE_DEC:
16755 case ZEND_POST_INC:
16756 case ZEND_POST_DEC:
16757 op1_info = OP1_INFO();
16758 return opline->op1_type == IS_CV
16759 && (op1_info & MAY_BE_LONG)
16760 && !(op1_info & MAY_BE_REF);
16761 case ZEND_STRLEN:
16762 op1_info = OP1_INFO();
16763 return (opline->op1_type & (IS_CV|IS_CONST))
16764 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING;
16765 case ZEND_COUNT:
16766 op1_info = OP1_INFO();
16767 return (opline->op1_type & (IS_CV|IS_CONST))
16768 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY;
16769 case ZEND_JMPZ:
16770 case ZEND_JMPNZ:
16771 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
16772 if (!ssa->cfg.map) {
16773 return 0;
16774 }
16775 if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start &&
16776 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
16777 return 0;
16778 }
16779 }
16780 ZEND_FALLTHROUGH;
16781 case ZEND_BOOL:
16782 case ZEND_BOOL_NOT:
16783 case ZEND_JMPZ_EX:
16784 case ZEND_JMPNZ_EX:
16785 return 1;
16786 case ZEND_FETCH_CONSTANT:
16787 return 1;
16788 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
16789 if ((opline->extended_value & ZEND_ISEMPTY)) {
16790 return 0;
16791 }
16792 ZEND_FALLTHROUGH;
16793 case ZEND_FETCH_DIM_R:
16794 case ZEND_FETCH_DIM_IS:
16795 case ZEND_FETCH_LIST_R:
16796 op1_info = OP1_INFO();
16797 op2_info = OP2_INFO();
16798 if (trace
16799 && trace->op1_type != IS_UNKNOWN
16800 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) {
16801 op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY);
16802 }
16803 return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) &&
16804 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) ||
16805 ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING));
16806 case ZEND_ASSIGN_DIM_OP:
16807 if (opline->result_type != IS_UNUSED) {
16808 return 0;
16809 }
16810 if (!zend_jit_supported_binary_op(opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
16811 return 0;
16812 }
16813 ZEND_FALLTHROUGH;
16814 case ZEND_ASSIGN_DIM:
16815 case ZEND_FETCH_DIM_W:
16816 case ZEND_FETCH_DIM_RW:
16817 case ZEND_FETCH_LIST_W:
16818 op1_info = OP1_INFO();
16819 op2_info = OP2_INFO();
16820 if (trace) {
16821 if (opline->op1_type == IS_CV) {
16822 if ((opline->opcode == ZEND_ASSIGN_DIM
16823 || opline->opcode == ZEND_ASSIGN_DIM_OP)
16824 && (opline+1)->op1_type == IS_CV
16825 && (opline+1)->op1.var == opline->op1.var) {
16826 /* skip $a[x] = $a; */
16827 return 0;
16828 }
16829 } else if (opline->op1_type == IS_VAR) {
16830 if (trace->op1_type == IS_UNKNOWN
16831 || !(trace->op1_type & IS_TRACE_INDIRECT)
16832 || opline->result_type != IS_UNUSED) {
16833 return 0;
16834 }
16835 }
16836 if (trace->op1_type != IS_UNKNOWN
16837 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) {
16838 op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY);
16839 }
16840 } else {
16841 if (opline->op1_type != IS_CV) {
16842 return 0;
16843 }
16844 }
16845 return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) &&
16846 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) ||
16847 ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING));
16848 case ZEND_ASSIGN_OBJ_OP:
16849 if (opline->result_type != IS_UNUSED) {
16850 return 0;
16851 }
16852 if (!zend_jit_supported_binary_op(opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
16853 return 0;
16854 }
16855 ZEND_FALLTHROUGH;
16856 case ZEND_FETCH_OBJ_R:
16857 case ZEND_ASSIGN_OBJ:
16858 if (opline->op2_type != IS_CONST
16859 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
16860 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
16861 return 0;
16862 }
16863 op1_info = OP1_INFO();
16864 return opline->op1_type == IS_UNUSED || (op1_info & MAY_BE_OBJECT);
16865 }
16866 return 0;
16867 }
16868
16869 static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var)
16870 {
16871 if (ssa->vars[var].no_val) {
16872 /* we don't need the value */
16873 return 0;
16874 }
16875
16876 if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) {
16877 /* Disable global register allocation,
16878 * register allocation for SSA variables connected through Phi functions
16879 */
16880 if (ssa->vars[var].definition_phi) {
16881 return 0;
16882 }
16883 if (ssa->vars[var].phi_use_chain) {
16884 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
16885 do {
16886 if (!ssa->vars[phi->ssa_var].no_val) {
16887 return 0;
16888 }
16889 phi = zend_ssa_next_use_phi(ssa, var, phi);
16890 } while (phi);
16891 }
16892 }
16893
16894 if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) &&
16895 ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) {
16896 /* bad type */
16897 return 0;
16898 }
16899
16900 return 1;
16901 }
16902
16903 static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var)
16904 {
16905 if (!zend_jit_var_supports_reg(ssa, var)) {
16906 return 0;
16907 }
16908
16909 if (ssa->vars[var].definition >= 0) {
16910 uint32_t def = ssa->vars[var].definition;
16911 if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) {
16912 return 0;
16913 }
16914 }
16915
16916 if (ssa->vars[var].use_chain >= 0) {
16917 int use = ssa->vars[var].use_chain;
16918
16919 do {
16920 if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) &&
16921 !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) {
16922 return 0;
16923 }
16924 use = zend_ssa_next_use(ssa->ops, var, use);
16925 } while (use >= 0);
16926 }
16927
16928 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
16929 int def_block, use_block, b, use, j;
16930 zend_basic_block *bb;
16931 zend_ssa_phi *p;
16932 bool ret = 1;
16933 zend_worklist worklist;
16934 ALLOCA_FLAG(use_heap)
16935
16936 /* Check if live range is split by ENTRY block */
16937 if (ssa->vars[var].definition >= 0) {
16938 def_block =ssa->cfg.map[ssa->vars[var].definition];
16939 } else {
16940 ZEND_ASSERT(ssa->vars[var].definition_phi);
16941 def_block = ssa->vars[var].definition_phi->block;
16942 }
16943
16944 ZEND_WORKLIST_ALLOCA(&worklist, ssa->cfg.blocks_count, use_heap);
16945
16946 if (ssa->vars[var].use_chain >= 0) {
16947 use = ssa->vars[var].use_chain;
16948 do {
16949 use_block = ssa->cfg.map[use];
16950 if (use_block != def_block) {
16951 zend_worklist_push(&worklist, use_block);
16952 }
16953 use = zend_ssa_next_use(ssa->ops, var, use);
16954 } while (use >= 0);
16955 }
16956
16957 p = ssa->vars[var].phi_use_chain;
16958 while (p) {
16959 use_block = p->block;
16960 if (use_block != def_block) {
16961 bb = &ssa->cfg.blocks[use_block];
16962 for (j = 0; j < bb->predecessors_count; j++) {
16963 if (p->sources[j] == var) {
16964 use_block = ssa->cfg.predecessors[bb->predecessor_offset + j];
16965 if (use_block != def_block) {
16966 zend_worklist_push(&worklist, use_block);
16967 }
16968 }
16969 }
16970 }
16971 p = zend_ssa_next_use_phi(ssa, var, p);
16972 }
16973
16974 while (zend_worklist_len(&worklist) != 0) {
16975 b = zend_worklist_pop(&worklist);
16976 bb = &ssa->cfg.blocks[b];
16977 if (bb->flags & (ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) {
16978 ret = 0;
16979 break;
16980 }
16981 for (j = 0; j < bb->predecessors_count; j++) {
16982 b = ssa->cfg.predecessors[bb->predecessor_offset + j];
16983 if (b != def_block) {
16984 zend_worklist_push(&worklist, b);
16985 }
16986 }
16987 }
16988
16989 ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
16990
16991 return ret;
16992 }
16993
16994 return 1;
16995 }
16996
16997 static void jit_frameless_icall0(zend_jit_ctx *jit, const zend_op *opline)
16998 {
16999 jit_SET_EX_OPLINE(jit, opline);
17000
17001 void *function = ZEND_FLF_HANDLER(opline);
17002 zend_jit_addr res_addr = RES_ADDR();
17003 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
17004 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
17005 ir_CALL_1(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref);
17006 zend_jit_check_exception(jit);
17007 }
17008
17009 static void jit_frameless_icall1(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
17010 {
17011 jit_SET_EX_OPLINE(jit, opline);
17012
17013 /* Avoid dropping RC check in case op escapes. */
17014 if (op1_info & MAY_BE_RC1) {
17015 op1_info |= MAY_BE_RCN;
17016 }
17017
17018 void *function = ZEND_FLF_HANDLER(opline);
17019 zend_jit_addr res_addr = RES_ADDR();
17020 zend_jit_addr op1_addr = OP1_ADDR();
17021 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
17022 ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
17023 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
17024 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
17025 zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
17026 }
17027 if (op1_info & MAY_BE_REF) {
17028 op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
17029 }
17030 ir_CALL_2(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref);
17031 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
17032 zend_jit_check_exception(jit);
17033 }
17034
17035 static void jit_frameless_icall2(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info)
17036 {
17037 jit_SET_EX_OPLINE(jit, opline);
17038
17039 /* Avoid dropping RC check in case op escapes. */
17040 if (op1_info & MAY_BE_RC1) {
17041 op1_info |= MAY_BE_RCN;
17042 }
17043 if (op2_info & MAY_BE_RC1) {
17044 op2_info |= MAY_BE_RCN;
17045 }
17046
17047 void *function = ZEND_FLF_HANDLER(opline);
17048 zend_jit_addr res_addr = RES_ADDR();
17049 zend_jit_addr op1_addr = OP1_ADDR();
17050 zend_jit_addr op2_addr = OP2_ADDR();
17051 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
17052 ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
17053 ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
17054 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
17055 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
17056 zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
17057 }
17058 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
17059 zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
17060 }
17061 if (op1_info & MAY_BE_REF) {
17062 op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
17063 }
17064 if (op2_info & MAY_BE_REF) {
17065 op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
17066 }
17067 ir_CALL_3(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref, op2_ref);
17068 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
17069 /* Set OP1 to UNDEF in case FREE_OP2() throws. */
17070 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) != 0 && (opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0) {
17071 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
17072 }
17073 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
17074 zend_jit_check_exception(jit);
17075 }
17076
17077 static void jit_frameless_icall3(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, uint32_t op1_data_info)
17078 {
17079 jit_SET_EX_OPLINE(jit, opline);
17080
17081 /* Avoid dropping RC check in case op escapes. */
17082 if (op1_info & MAY_BE_RC1) {
17083 op1_info |= MAY_BE_RCN;
17084 }
17085 if (op2_info & MAY_BE_RC1) {
17086 op2_info |= MAY_BE_RCN;
17087 }
17088 if (op1_data_info & MAY_BE_RC1) {
17089 op1_data_info |= MAY_BE_RCN;
17090 }
17091
17092 void *function = ZEND_FLF_HANDLER(opline);
17093 uint8_t op_data_type = (opline + 1)->op1_type;
17094 zend_jit_addr res_addr = RES_ADDR();
17095 zend_jit_addr op1_addr = OP1_ADDR();
17096 zend_jit_addr op2_addr = OP2_ADDR();
17097 zend_jit_addr op3_addr = OP1_DATA_ADDR();
17098 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
17099 ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
17100 ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
17101 ir_ref op3_ref = jit_ZVAL_ADDR(jit, op3_addr);
17102 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
17103 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
17104 zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
17105 }
17106 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
17107 zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
17108 }
17109 if ((opline+1)->op1_type == IS_CV && (op1_data_info & MAY_BE_UNDEF)) {
17110 zend_jit_zval_check_undef(jit, op3_ref, (opline+1)->op1.var, opline, 1);
17111 }
17112 if (op1_info & MAY_BE_REF) {
17113 op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
17114 }
17115 if (op2_info & MAY_BE_REF) {
17116 op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
17117 }
17118 if (op1_data_info & MAY_BE_REF) {
17119 op3_ref = jit_ZVAL_DEREF_ref(jit, op3_ref);
17120 }
17121 ir_CALL_4(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref, op2_ref, op3_ref);
17122 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
17123 /* Set OP1 to UNDEF in case FREE_OP2() throws. */
17124 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
17125 && ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
17126 || (op_data_type & (IS_VAR|IS_TMP_VAR)))) {
17127 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
17128 }
17129 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
17130 /* If OP1 is a TMP|VAR, we don't need to set OP2 to UNDEF on free because
17131 * zend_fetch_debug_backtrace aborts when it encounters the first UNDEF TMP|VAR. */
17132 if (!(opline->op1_type & (IS_VAR|IS_TMP_VAR))
17133 && (opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0
17134 && (op_data_type & (IS_VAR|IS_TMP_VAR)) != 0) {
17135 jit_set_Z_TYPE_INFO(jit, op2_addr, IS_UNDEF);
17136 }
17137 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
17138 zend_jit_check_exception(jit);
17139 }
17140
17141 /*
17142 * Local variables:
17143 * tab-width: 4
17144 * c-basic-offset: 4
17145 * indent-tabs-mode: t
17146 * End:
17147 */
17148