1 /*
2 * +----------------------------------------------------------------------+
3 * | Zend JIT |
4 * +----------------------------------------------------------------------+
5 * | Copyright (c) The PHP Group |
6 * +----------------------------------------------------------------------+
7 * | This source file is subject to version 3.01 of the PHP license, |
8 * | that is bundled with this package in the file LICENSE, and is |
9 * | available through the world-wide-web at the following url: |
10 * | https://www.php.net/license/3_01.txt |
11 * | If you did not receive a copy of the PHP license and are unable to |
12 * | obtain it through the world-wide-web, please send a note to |
13 * | license@php.net so we can mail you a copy immediately. |
14 * +----------------------------------------------------------------------+
15 * | Authors: Dmitry Stogov <dmitry@php.net> |
16 * +----------------------------------------------------------------------+
17 */
18
19 #include "jit/ir/ir.h"
20 #include "jit/ir/ir_builder.h"
21
22 #if defined(IR_TARGET_X86)
23 # define IR_REG_SP 4 /* IR_REG_RSP */
24 # define IR_REG_FP 5 /* IR_REG_RBP */
25 # define ZREG_FP 6 /* IR_REG_RSI */
26 # define ZREG_IP 7 /* IR_REG_RDI */
27 # define ZREG_FIRST_FPR 8
28 # define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7)) /* all preserved registers */
29 #elif defined(IR_TARGET_X64)
30 # define IR_REG_SP 4 /* IR_REG_RSP */
31 # define IR_REG_FP 5 /* IR_REG_RBP */
32 # define ZREG_FP 14 /* IR_REG_R14 */
33 # define ZREG_IP 15 /* IR_REG_R15 */
34 # define ZREG_FIRST_FPR 16
35 # if defined(_WIN64)
36 # define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15))
37 /*
38 # define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15) | \
39 (1<<(16+6)) | (1<<(16+7)) | (1<<(16+8)) | (1<<(16+9)) | (1<<(16+10)) | \
40 (1<<(16+11)) | (1<<(16+12)) | (1<<(16+13)) | (1<<(16+14)) | (1<<(16+15)))
41 */
42 # else
43 # define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<12) | (1<<13) | (1<<14) | (1<<15)) /* all preserved registers */
44 # endif
45 #elif defined(IR_TARGET_AARCH64)
46 # define IR_REG_SP 31 /* IR_REG_RSP */
47 # define IR_REG_FP 29 /* IR_REG_X29 */
48 # define ZREG_FP 27 /* IR_REG_X27 */
49 # define ZREG_IP 28 /* IR_REG_X28 */
50 # define ZREG_FIRST_FPR 32
51 # define IR_REGSET_PRESERVED ((1<<19) | (1<<20) | (1<<21) | (1<<22) | (1<<23) | \
52 (1<<24) | (1<<25) | (1<<26) | (1<<27) | (1<<28)) /* all preserved registers */
53 #else
54 # error "Unknown IR target"
55 #endif
56
57 #define ZREG_RX ZREG_IP
58
59 #define OPTIMIZE_FOR_SIZE 0
60
61 /* IR builder defines */
62 #undef _ir_CTX
63 #define _ir_CTX (&jit->ctx)
64
65 #undef ir_CONST_ADDR
66 #define ir_CONST_ADDR(_addr) jit_CONST_ADDR(jit, (uintptr_t)(_addr))
67 #define ir_CONST_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), 0)
68 #define ir_CONST_FC_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), IR_FASTCALL_FUNC)
69 #define ir_CAST_FC_FUNC(_addr) ir_fold2(_ir_CTX, IR_OPT(IR_PROTO, IR_ADDR), (_addr), \
70 ir_proto_0(_ir_CTX, IR_FASTCALL_FUNC, IR_I32))
71
72 #define ir_CONST_FUNC_PROTO(_addr, _proto) \
73 jit_CONST_FUNC_PROTO(jit, (uintptr_t)(_addr), (_proto))
74
75 #undef ir_ADD_OFFSET
76 #define ir_ADD_OFFSET(_addr, _offset) \
77 jit_ADD_OFFSET(jit, _addr, _offset)
78
79 #ifdef ZEND_ENABLE_ZVAL_LONG64
80 # define IR_LONG IR_I64
81 # define ir_CONST_LONG ir_CONST_I64
82 # define ir_UNARY_OP_L ir_UNARY_OP_I64
83 # define ir_BINARY_OP_L ir_BINARY_OP_I64
84 # define ir_ADD_L ir_ADD_I64
85 # define ir_SUB_L ir_SUB_I64
86 # define ir_MUL_L ir_MUL_I64
87 # define ir_DIV_L ir_DIV_I64
88 # define ir_MOD_L ir_MOD_I64
89 # define ir_NEG_L ir_NEG_I64
90 # define ir_ABS_L ir_ABS_I64
91 # define ir_SEXT_L ir_SEXT_I64
92 # define ir_ZEXT_L ir_ZEXT_I64
93 # define ir_TRUNC_L ir_TRUNC_I64
94 # define ir_BITCAST_L ir_BITCAST_I64
95 # define ir_FP2L ir_FP2I64
96 # define ir_ADD_OV_L ir_ADD_OV_I64
97 # define ir_SUB_OV_L ir_SUB_OV_I64
98 # define ir_MUL_OV_L ir_MUL_OV_I64
99 # define ir_NOT_L ir_NOT_I64
100 # define ir_OR_L ir_OR_I64
101 # define ir_AND_L ir_AND_I64
102 # define ir_XOR_L ir_XOR_I64
103 # define ir_SHL_L ir_SHL_I64
104 # define ir_SHR_L ir_SHR_I64
105 # define ir_SAR_L ir_SAR_I64
106 # define ir_ROL_L ir_ROL_I64
107 # define ir_ROR_L ir_ROR_I64
108 # define ir_MIN_L ir_MIN_I64
109 # define ir_MAX_L ir_MAX_I64
110 # define ir_LOAD_L ir_LOAD_I64
111 #else
112 # define IR_LONG IR_I32
113 # define ir_CONST_LONG ir_CONST_I32
114 # define ir_UNARY_OP_L ir_UNARY_OP_I32
115 # define ir_BINARY_OP_L ir_BINARY_OP_I32
116 # define ir_ADD_L ir_ADD_I32
117 # define ir_SUB_L ir_SUB_I32
118 # define ir_MUL_L ir_MUL_I32
119 # define ir_DIV_L ir_DIV_I32
120 # define ir_MOD_L ir_MOD_I32
121 # define ir_NEG_L ir_NEG_I32
122 # define ir_ABS_L ir_ABS_I32
123 # define ir_SEXT_L ir_SEXT_I32
124 # define ir_ZEXT_L ir_ZEXT_I32
125 # define ir_TRUNC_L ir_TRUNC_I32
126 # define ir_BITCAST_L ir_BITCAST_I32
127 # define ir_FP2L ir_FP2I32
128 # define ir_ADD_OV_L ir_ADD_OV_I32
129 # define ir_SUB_OV_L ir_SUB_OV_I32
130 # define ir_MUL_OV_L ir_MUL_OV_I32
131 # define ir_NOT_L ir_NOT_I32
132 # define ir_OR_L ir_OR_I32
133 # define ir_AND_L ir_AND_I32
134 # define ir_XOR_L ir_XOR_I32
135 # define ir_SHL_L ir_SHL_I32
136 # define ir_SHR_L ir_SHR_I32
137 # define ir_SAR_L ir_SAR_I32
138 # define ir_ROL_L ir_ROL_I32
139 # define ir_ROR_L ir_ROR_I32
140 # define ir_MIN_L ir_MIN_I32
141 # define ir_MAX_L ir_MAX_I32
142 # define ir_LOAD_L ir_LOAD_I32
143 #endif
144
145 /* A helper structure to collect IT rers for the following use in (MERGE/PHI)_N */
146 typedef struct _ir_refs {
147 uint32_t count;
148 uint32_t limit;
149 ir_ref refs[0] ZEND_ELEMENT_COUNT(count);
150 } ir_refs;
151
152 #define ir_refs_size(_n) (offsetof(ir_refs, refs) + sizeof(ir_ref) * (_n))
153 #define ir_refs_init(_name, _n) _name = alloca(ir_refs_size(_n)); \
154 do {_name->count = 0; _name->limit = (_n);} while (0)
155
ir_refs_add(ir_refs * refs,ir_ref ref)156 static void ir_refs_add(ir_refs *refs, ir_ref ref)
157 {
158 ir_ref *ptr;
159
160 ZEND_ASSERT(refs->count < refs->limit);
161 ptr = refs->refs;
162 ptr[refs->count++] = ref;
163 }
164
165 static size_t zend_jit_trace_prologue_size = (size_t)-1;
166 #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
167 static uint32_t allowed_opt_flags = 0;
168 static uint32_t default_mflags = 0;
169 #endif
170 static bool delayed_call_chain = 0; // TODO: remove this var (use jit->delayed_call_level) ???
171
172 #ifdef ZTS
173 # ifdef _WIN32
174 extern uint32_t _tls_index;
175 extern char *_tls_start;
176 extern char *_tls_end;
177 # endif
178
179 static size_t tsrm_ls_cache_tcb_offset = 0;
180 static size_t tsrm_tls_index = 0;
181 static size_t tsrm_tls_offset = 0;
182
183 # define EG_TLS_OFFSET(field) \
184 (executor_globals_offset + offsetof(zend_executor_globals, field))
185
186 # define CG_TLS_OFFSET(field) \
187 (compiler_globals_offset + offsetof(zend_compiler_globals, field))
188
189 # define jit_EG(_field) \
190 ir_ADD_OFFSET(jit_TLS(jit), EG_TLS_OFFSET(_field))
191
192 # define jit_CG(_field) \
193 ir_ADD_OFFSET(jit_TLS(jit), CG_TLS_OFFSET(_field))
194
195 #else
196
197 # define jit_EG(_field) \
198 ir_CONST_ADDR(&EG(_field))
199
200 # define jit_CG(_field) \
201 ir_CONST_ADDR(&CG(_field))
202
203 #endif
204
205 #define jit_CALL(_call, _field) \
206 ir_ADD_OFFSET(_call, offsetof(zend_execute_data, _field))
207
208 #define jit_EX(_field) \
209 jit_CALL(jit_FP(jit), _field)
210
211 #define jit_RX(_field) \
212 jit_CALL(jit_IP(jit), _field)
213
214 #define JIT_STUBS(_) \
215 _(exception_handler, IR_SKIP_PROLOGUE) \
216 _(exception_handler_undef, IR_SKIP_PROLOGUE) \
217 _(exception_handler_free_op2, IR_SKIP_PROLOGUE) \
218 _(exception_handler_free_op1_op2, IR_SKIP_PROLOGUE) \
219 _(interrupt_handler, IR_SKIP_PROLOGUE) \
220 _(leave_function_handler, IR_SKIP_PROLOGUE) \
221 _(negative_shift, IR_SKIP_PROLOGUE) \
222 _(mod_by_zero, IR_SKIP_PROLOGUE) \
223 _(invalid_this, IR_SKIP_PROLOGUE) \
224 _(undefined_function, IR_SKIP_PROLOGUE) \
225 _(throw_cannot_pass_by_ref, IR_SKIP_PROLOGUE) \
226 _(icall_throw, IR_SKIP_PROLOGUE) \
227 _(leave_throw, IR_SKIP_PROLOGUE) \
228 _(hybrid_runtime_jit, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
229 _(hybrid_profile_jit, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
230 _(hybrid_func_hot_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
231 _(hybrid_loop_hot_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
232 _(hybrid_func_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
233 _(hybrid_ret_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
234 _(hybrid_loop_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
235 _(trace_halt, IR_SKIP_PROLOGUE) \
236 _(trace_escape, IR_SKIP_PROLOGUE) \
237 _(trace_exit, IR_SKIP_PROLOGUE) \
238 _(undefined_offset, IR_FUNCTION | IR_FASTCALL_FUNC) \
239 _(undefined_key, IR_FUNCTION | IR_FASTCALL_FUNC) \
240 _(cannot_add_element, IR_FUNCTION | IR_FASTCALL_FUNC) \
241 _(assign_const, IR_FUNCTION | IR_FASTCALL_FUNC) \
242 _(assign_tmp, IR_FUNCTION | IR_FASTCALL_FUNC) \
243 _(assign_var, IR_FUNCTION | IR_FASTCALL_FUNC) \
244 _(assign_cv_noref, IR_FUNCTION | IR_FASTCALL_FUNC) \
245 _(assign_cv, IR_FUNCTION | IR_FASTCALL_FUNC) \
246 _(new_array, IR_FUNCTION | IR_FASTCALL_FUNC) \
247
248 #define JIT_STUB_ID(name, flags) \
249 jit_stub_ ## name,
250
251 #define JIT_STUB_FORWARD(name, flags) \
252 static int zend_jit_ ## name ## _stub(zend_jit_ctx *jit);
253
254 #define JIT_STUB(name, flags) \
255 {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub, flags},
256
257 typedef enum _jit_stub_id {
258 JIT_STUBS(JIT_STUB_ID)
259 jit_last_stub
260 } jit_stub_id;
261
262 typedef struct _zend_jit_reg_var {
263 ir_ref ref;
264 uint32_t flags;
265 } zend_jit_reg_var;
266
267 typedef struct _zend_jit_ctx {
268 ir_ctx ctx;
269 const zend_op *last_valid_opline;
270 bool use_last_valid_opline;
271 bool track_last_valid_opline;
272 bool reuse_ip;
273 uint32_t delayed_call_level;
274 int b; /* current basic block number or -1 */
275 #ifdef ZTS
276 ir_ref tls;
277 #endif
278 ir_ref fp;
279 ir_ref trace_loop_ref;
280 ir_ref return_inputs;
281 const zend_op_array *op_array;
282 const zend_op_array *current_op_array;
283 zend_ssa *ssa;
284 zend_string *name;
285 ir_ref *bb_start_ref; /* PHP BB -> IR ref mapping */
286 ir_ref *bb_predecessors; /* PHP BB -> index in bb_edges -> IR refs of predessors */
287 ir_ref *bb_edges;
288 zend_jit_trace_info *trace;
289 zend_jit_reg_var *ra;
290 int delay_var;
291 ir_refs *delay_refs;
292 ir_ref eg_exception_addr;
293 HashTable addr_hash;
294 ir_ref stub_addr[jit_last_stub];
295 } zend_jit_ctx;
296
297 typedef int8_t zend_reg;
298
299 typedef struct _zend_jit_registers_buf {
300 #if defined(IR_TARGET_X64)
301 uint64_t gpr[16]; /* general purpose integer register */
302 double fpr[16]; /* floating point registers */
303 #elif defined(IR_TARGET_X86)
304 uint32_t gpr[8]; /* general purpose integer register */
305 double fpr[8]; /* floating point registers */
306 #elif defined (IR_TARGET_AARCH64)
307 uint64_t gpr[32]; /* general purpose integer register */
308 double fpr[32]; /* floating point registers */
309 #else
310 # error "Unknown IR target"
311 #endif
312 } zend_jit_registers_buf;
313
314 /* Keep 32 exit points in a single code block */
315 #define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes
316 #define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points
317
318 static uint32_t zend_jit_exit_point_by_addr(const void *addr);
319 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs);
320
321 static int zend_jit_assign_to_variable(zend_jit_ctx *jit,
322 const zend_op *opline,
323 zend_jit_addr var_use_addr,
324 zend_jit_addr var_addr,
325 uint32_t var_info,
326 uint32_t var_def_info,
327 uint8_t val_type,
328 zend_jit_addr val_addr,
329 uint32_t val_info,
330 zend_jit_addr res_addr,
331 zend_jit_addr ref_addr,
332 bool check_exception);
333
334 typedef struct _zend_jit_stub {
335 const char *name;
336 int (*stub)(zend_jit_ctx *jit);
337 uint32_t flags;
338 } zend_jit_stub;
339
340 JIT_STUBS(JIT_STUB_FORWARD)
341
342 static const zend_jit_stub zend_jit_stubs[] = {
343 JIT_STUBS(JIT_STUB)
344 };
345
346 #if defined(_WIN32) || defined(IR_TARGET_AARCH64)
347 /* We keep addresses in SHM to share them between sepaeate processes (on Windows) or to support veneers (on AArch64) */
348 static void** zend_jit_stub_handlers = NULL;
349 #else
350 static void* zend_jit_stub_handlers[sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])];
351 #endif
352
353 #if defined(IR_TARGET_AARCH64)
354
355 #define IR_HAS_VENEERS (1U<<31) /* IR_RESERVED_FLAG_1 */
356
zend_jit_get_veneer(ir_ctx * ctx,const void * addr)357 static const void *zend_jit_get_veneer(ir_ctx *ctx, const void *addr)
358 {
359 int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
360
361 for (i = 0; i < count; i++) {
362 if (zend_jit_stub_handlers[i] == addr) {
363 return zend_jit_stub_handlers[count + i];
364 }
365 }
366
367 if (((zend_jit_ctx*)ctx)->trace
368 && (void*)addr >= dasm_buf && (void*)addr < dasm_end) {
369 uint32_t exit_point = zend_jit_exit_point_by_addr(addr);
370
371 if (exit_point != (uint32_t)-1) {
372 zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;
373
374 ZEND_ASSERT(exit_point < t->exit_count);
375 return (const void*)((char*)ctx->deoptimization_exits_base + (exit_point * 4));
376 }
377 }
378
379 return NULL;
380 }
381
zend_jit_set_veneer(ir_ctx * ctx,const void * addr,const void * veneer)382 static bool zend_jit_set_veneer(ir_ctx *ctx, const void *addr, const void *veneer)
383 {
384 int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
385 uint32_t exit_point = zend_jit_exit_point_by_addr(addr);
386
387 if (exit_point != (uint32_t)-1) {
388 return 1;
389 }
390 for (i = 0; i < count; i++) {
391 if (zend_jit_stub_handlers[i] == addr) {
392 const void **ptr = (const void**)&zend_jit_stub_handlers[count + i];
393 *ptr = veneer;
394 ctx->flags2 |= IR_HAS_VENEERS;
395 #ifdef HAVE_CAPSTONE
396 int64_t offset;
397 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
398 const char *name = ir_disasm_find_symbol((uint64_t)(uintptr_t)addr, &offset);
399
400 if (name && !offset) {
401 if (strstr(name, "@veneer") == NULL) {
402 char *new_name;
403
404 zend_spprintf(&new_name, 0, "%s@veneer", name);
405 ir_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4);
406 efree(new_name);
407 } else {
408 ir_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4);
409 }
410 }
411 }
412 #endif
413 return 1;
414 }
415 }
416
417 return 0;
418 }
419
zend_jit_commit_veneers(void)420 static void zend_jit_commit_veneers(void)
421 {
422 int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
423
424 for (i = 0; i < count; i++) {
425 if (zend_jit_stub_handlers[count + i]) {
426 zend_jit_stub_handlers[i] = zend_jit_stub_handlers[count + i];
427 zend_jit_stub_handlers[count + i] = NULL;
428 }
429 }
430 }
431 #endif
432
zend_jit_prefer_const_addr_load(zend_jit_ctx * jit,uintptr_t addr)433 static bool zend_jit_prefer_const_addr_load(zend_jit_ctx *jit, uintptr_t addr)
434 {
435 #if defined(IR_TARGET_X86)
436 return 0; /* always use immediate value */
437 #elif defined(IR_TARGET_X64)
438 return addr > 0xffffffff; /* prefer loading long constant from memery */
439 #elif defined(IR_TARGET_AARCH64)
440 return addr > 0xffff;
441 #else
442 # error "Unknown IR target"
443 #endif
444 }
445
zend_reg_name(int8_t reg)446 static const char* zend_reg_name(int8_t reg)
447 {
448 return ir_reg_name(reg, ir_reg_is_int(reg) ? IR_LONG : IR_DOUBLE);
449 }
450
451 /* IR helpers */
452
453 #ifdef ZTS
jit_TLS(zend_jit_ctx * jit)454 static ir_ref jit_TLS(zend_jit_ctx *jit)
455 {
456 ZEND_ASSERT(jit->ctx.control);
457 if (jit->tls) {
458 /* Emit "TLS" once for basic block */
459 ir_insn *insn;
460 ir_ref ref = jit->ctx.control;
461
462 while (1) {
463 if (ref == jit->tls) {
464 return jit->tls;
465 }
466 insn = &jit->ctx.ir_base[ref];
467 if (insn->op >= IR_START || insn->op == IR_CALL) {
468 break;
469 }
470 ref = insn->op1;
471 }
472 }
473 jit->tls = ir_TLS(
474 tsrm_ls_cache_tcb_offset ? tsrm_ls_cache_tcb_offset : tsrm_tls_index,
475 tsrm_ls_cache_tcb_offset ? IR_NULL : tsrm_tls_offset);
476 return jit->tls;
477 }
478 #endif
479
jit_CONST_ADDR(zend_jit_ctx * jit,uintptr_t addr)480 static ir_ref jit_CONST_ADDR(zend_jit_ctx *jit, uintptr_t addr)
481 {
482 ir_ref ref;
483 zval *zv;
484
485 if (addr == 0) {
486 return IR_NULL;
487 }
488 zv = zend_hash_index_lookup(&jit->addr_hash, addr);
489 if (Z_TYPE_P(zv) == IS_LONG) {
490 ref = Z_LVAL_P(zv);
491 ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_ADDR, IR_ADDR));
492 } else {
493 ref = ir_unique_const_addr(&jit->ctx, addr);
494 ZVAL_LONG(zv, ref);
495 }
496 return ref;
497 }
498
jit_CONST_FUNC_PROTO(zend_jit_ctx * jit,uintptr_t addr,ir_ref proto)499 static ir_ref jit_CONST_FUNC_PROTO(zend_jit_ctx *jit, uintptr_t addr, ir_ref proto)
500 {
501 ir_ref ref;
502 ir_insn *insn;
503 zval *zv;
504
505 ZEND_ASSERT(addr != 0);
506 zv = zend_hash_index_lookup(&jit->addr_hash, addr);
507 if (Z_TYPE_P(zv) == IS_LONG) {
508 ref = Z_LVAL_P(zv);
509 ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_FUNC_ADDR, IR_ADDR) && jit->ctx.ir_base[ref].proto == proto);
510 } else {
511 ref = ir_unique_const_addr(&jit->ctx, addr);
512 insn = &jit->ctx.ir_base[ref];
513 insn->optx = IR_OPT(IR_FUNC_ADDR, IR_ADDR);
514 insn->proto = proto;
515 ZVAL_LONG(zv, ref);
516 }
517 return ref;
518 }
519
jit_CONST_FUNC(zend_jit_ctx * jit,uintptr_t addr,uint16_t flags)520 static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags)
521 {
522 #if defined(IR_TARGET_X86)
523 /* TODO: dummy prototype (only flags matter) ??? */
524 ir_ref proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0;
525 #else
526 ir_ref proto = 0;
527 #endif
528
529 return jit_CONST_FUNC_PROTO(jit, addr, proto);
530 }
531
jit_ADD_OFFSET(zend_jit_ctx * jit,ir_ref addr,uintptr_t offset)532 static ir_ref jit_ADD_OFFSET(zend_jit_ctx *jit, ir_ref addr, uintptr_t offset)
533 {
534 if (offset) {
535 addr = ir_ADD_A(addr, ir_CONST_ADDR(offset));
536 }
537 return addr;
538 }
539
jit_EG_exception(zend_jit_ctx * jit)540 static ir_ref jit_EG_exception(zend_jit_ctx *jit)
541 {
542 #ifdef ZTS
543 return jit_EG(exception);
544 #else
545 ir_ref ref = jit->eg_exception_addr;
546
547 if (UNEXPECTED(!ref)) {
548 ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)&EG(exception));
549 jit->eg_exception_addr = ref;
550 }
551 return ref;
552 #endif
553 }
554
jit_STUB_ADDR(zend_jit_ctx * jit,jit_stub_id id)555 static ir_ref jit_STUB_ADDR(zend_jit_ctx *jit, jit_stub_id id)
556 {
557 ir_ref ref = jit->stub_addr[id];
558
559 if (UNEXPECTED(!ref)) {
560 ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]);
561 jit->stub_addr[id] = ref;
562 }
563 return ref;
564 }
565
jit_STUB_FUNC_ADDR(zend_jit_ctx * jit,jit_stub_id id,uint16_t flags)566 static ir_ref jit_STUB_FUNC_ADDR(zend_jit_ctx *jit, jit_stub_id id, uint16_t flags)
567 {
568 ir_ref ref = jit->stub_addr[id];
569 ir_insn *insn;
570
571 if (UNEXPECTED(!ref)) {
572 ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]);
573 insn = &jit->ctx.ir_base[ref];
574 insn->optx = IR_OPT(IR_FUNC_ADDR, IR_ADDR);
575 #if defined(IR_TARGET_X86)
576 /* TODO: dummy prototype (only flags matter) ??? */
577 insn->proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0;
578 #else
579 insn->proto = 0;
580 #endif
581 jit->stub_addr[id] = ref;
582 }
583 return ref;
584 }
585
jit_SNAPSHOT(zend_jit_ctx * jit,ir_ref addr)586 static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr)
587 {
588 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) {
589 const void *ptr = (const void*)jit->ctx.ir_base[addr].val.addr;
590 const zend_op_array *op_array = &JIT_G(current_frame)->func->op_array;
591 uint32_t stack_size = op_array->last_var + op_array->T;
592
593 if (ptr == zend_jit_stub_handlers[jit_stub_exception_handler]
594 || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_undef]
595 || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op1_op2]
596 || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op2]
597 || ptr == zend_jit_stub_handlers[jit_stub_interrupt_handler]
598 || ptr == zend_jit_stub_handlers[jit_stub_leave_function_handler]
599 || ptr == zend_jit_stub_handlers[jit_stub_negative_shift]
600 || ptr == zend_jit_stub_handlers[jit_stub_mod_by_zero]
601 || ptr == zend_jit_stub_handlers[jit_stub_invalid_this]
602 || ptr == zend_jit_stub_handlers[jit_stub_undefined_function]
603 || ptr == zend_jit_stub_handlers[jit_stub_throw_cannot_pass_by_ref]
604 || ptr == zend_jit_stub_handlers[jit_stub_icall_throw]
605 || ptr == zend_jit_stub_handlers[jit_stub_leave_throw]
606 || ptr == zend_jit_stub_handlers[jit_stub_trace_halt]
607 || ptr == zend_jit_stub_handlers[jit_stub_trace_escape]) {
608 /* This is a GUARD that trigger exit through a stub code (without deoptimization) */
609 return;
610 }
611
612 /* Check if we need snapshot entries for polymorphic method call */
613 zend_jit_trace_info *t = jit->trace;
614 uint32_t exit_point = 0, n = 0;
615
616 if (addr < 0) {
617 if (t->exit_count > 0
618 && jit->ctx.ir_base[addr].val.u64 == (uintptr_t)zend_jit_trace_get_exit_addr(t->exit_count - 1)) {
619 exit_point = t->exit_count - 1;
620 if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) {
621 n = 2;
622 }
623 }
624 }
625
626 if (stack_size || n) {
627 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
628 uint32_t snapshot_size, i;
629
630 snapshot_size = stack_size;
631 while (snapshot_size > 0) {
632 ir_ref ref = STACK_REF(stack, snapshot_size - 1);
633
634 if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, snapshot_size - 1) & (/*ZREG_LOAD|*/ZREG_STORE))) {
635 snapshot_size--;
636 } else {
637 break;
638 }
639 }
640 if (snapshot_size || n) {
641 ir_ref snapshot;
642
643 snapshot = ir_SNAPSHOT(snapshot_size + n);
644 for (i = 0; i < snapshot_size; i++) {
645 ir_ref ref = STACK_REF(stack, i);
646
647 if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, i) & (/*ZREG_LOAD|*/ZREG_STORE))) {
648 ref = IR_UNUSED;
649 }
650 ir_SNAPSHOT_SET_OP(snapshot, i + 1, ref);
651 }
652 if (n) {
653 ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func_ref);
654 ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this_ref);
655 }
656 }
657 }
658 }
659 }
660
_add_trace_const(zend_jit_trace_info * t,int64_t val)661 static int32_t _add_trace_const(zend_jit_trace_info *t, int64_t val)
662 {
663 int32_t i;
664
665 for (i = 0; i < t->consts_count; i++) {
666 if (t->constants[i].i == val) {
667 return i;
668 }
669 }
670 ZEND_ASSERT(i < 0x7fffffff);
671 t->consts_count = i + 1;
672 t->constants = erealloc(t->constants, (i + 1) * sizeof(zend_jit_exit_const));
673 t->constants[i].i = val;
674 return i;
675 }
676
zend_jit_duplicate_exit_point(ir_ctx * ctx,zend_jit_trace_info * t,uint32_t exit_point,ir_ref snapshot_ref)677 uint32_t zend_jit_duplicate_exit_point(ir_ctx *ctx, zend_jit_trace_info *t, uint32_t exit_point, ir_ref snapshot_ref)
678 {
679 uint32_t stack_size, stack_offset;
680 uint32_t new_exit_point = t->exit_count;
681
682 if (new_exit_point >= ZEND_JIT_TRACE_MAX_EXITS) {
683 ctx->status = -ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
684 return exit_point;
685 }
686
687 t->exit_count++;
688 memcpy(&t->exit_info[new_exit_point], &t->exit_info[exit_point], sizeof(zend_jit_trace_exit_info));
689 stack_size = t->exit_info[new_exit_point].stack_size;
690 if (stack_size != 0) {
691 stack_offset = t->stack_map_size;
692 t->stack_map_size += stack_size;
693 // TODO: reduce number of reallocations ???
694 t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
695 memcpy(t->stack_map + stack_offset, t->stack_map + t->exit_info[new_exit_point].stack_offset, stack_size * sizeof(zend_jit_trace_stack));
696 t->exit_info[new_exit_point].stack_offset = stack_offset;
697 }
698 t->exit_info[new_exit_point].flags &= ~ZEND_JIT_EXIT_FIXED;
699
700 return new_exit_point;
701 }
702
zend_jit_snapshot_handler(ir_ctx * ctx,ir_ref snapshot_ref,ir_insn * snapshot,void * addr)703 void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, void *addr)
704 {
705 zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;
706 uint32_t exit_point, exit_flags;
707 ir_ref n = snapshot->inputs_count;
708 ir_ref i;
709
710 exit_point = zend_jit_exit_point_by_addr(addr);
711 ZEND_ASSERT(exit_point < t->exit_count);
712 exit_flags = t->exit_info[exit_point].flags;
713
714 if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) {
715 int8_t *reg_ops = ctx->regs[snapshot_ref];
716
717 ZEND_ASSERT(reg_ops[n - 1] != -1 && reg_ops[n] != -1);
718 if ((exit_flags & ZEND_JIT_EXIT_FIXED)
719 && (t->exit_info[exit_point].poly_func_reg != reg_ops[n - 1]
720 || t->exit_info[exit_point].poly_this_reg != reg_ops[n])) {
721 exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
722 addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
723 exit_flags &= ~ZEND_JIT_EXIT_FIXED;
724 }
725 t->exit_info[exit_point].poly_func_reg = reg_ops[n - 1];
726 t->exit_info[exit_point].poly_this_reg = reg_ops[n];
727 n -= 2;
728 }
729
730 for (i = 2; i <= n; i++) {
731 ir_ref ref = ir_insn_op(snapshot, i);
732
733 if (ref) {
734 int8_t *reg_ops = ctx->regs[snapshot_ref];
735 int8_t reg = reg_ops[i];
736 ir_ref var = i - 2;
737
738 ZEND_ASSERT(var < t->exit_info[exit_point].stack_size);
739 if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags == ZREG_ZVAL_COPY) {
740 ZEND_ASSERT(reg != ZREG_NONE);
741 if ((exit_flags & ZEND_JIT_EXIT_FIXED)
742 && t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)) {
743 exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
744 addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
745 exit_flags &= ~ZEND_JIT_EXIT_FIXED;
746 }
747 t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);
748 } else if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_CONST) {
749 ZEND_ASSERT(t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_LONG ||
750 t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_DOUBLE);
751
752 if (ref > 0) {
753 if (reg != ZREG_NONE) {
754 if (reg & IR_REG_SPILL_LOAD) {
755 ZEND_ASSERT(!(reg & IR_REG_SPILL_SPECIAL));
756 /* spill slot on a CPU stack */
757 if ((exit_flags & ZEND_JIT_EXIT_FIXED)
758 && (t->stack_map[t->exit_info[exit_point].stack_offset + var].ref != ref
759 || t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
760 || !(t->stack_map[t->exit_info[exit_point].stack_offset + var].flags & ZREG_SPILL_SLOT))) {
761 exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
762 addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
763 exit_flags &= ~ZEND_JIT_EXIT_FIXED;
764 }
765 t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = ref;
766 t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;
767 t->stack_map[t->exit_info[exit_point].stack_offset + var].flags |= ZREG_SPILL_SLOT;
768 } else if (reg & IR_REG_SPILL_SPECIAL) {
769 /* spill slot on a VM stack */
770 if ((exit_flags & ZEND_JIT_EXIT_FIXED)
771 && (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
772 || t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) {
773 exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
774 addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
775 exit_flags &= ~ZEND_JIT_EXIT_FIXED;
776 }
777 t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;
778 t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;
779 } else {
780 if ((exit_flags & ZEND_JIT_EXIT_FIXED)
781 && t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)) {
782 exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
783 addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
784 exit_flags &= ~ZEND_JIT_EXIT_FIXED;
785 }
786 t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);
787 }
788 } else {
789 if ((exit_flags & ZEND_JIT_EXIT_FIXED)
790 && (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
791 || t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) {
792 exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
793 addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
794 exit_flags &= ~ZEND_JIT_EXIT_FIXED;
795 }
796 t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;
797 }
798 } else if (!(exit_flags & ZEND_JIT_EXIT_FIXED)) {
799 int32_t idx = _add_trace_const(t, ctx->ir_base[ref].val.i64);
800 t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_CONST;
801 t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = idx;
802 }
803 }
804 }
805 }
806 t->exit_info[exit_point].flags |= ZEND_JIT_EXIT_FIXED;
807 return addr;
808 }
809
jit_SIDE_EXIT(zend_jit_ctx * jit,ir_ref addr)810 static void jit_SIDE_EXIT(zend_jit_ctx *jit, ir_ref addr)
811 {
812 jit_SNAPSHOT(jit, addr);
813 ir_IJMP(addr);
814 }
815
816 /* PHP JIT helpers */
817
jit_EMALLOC(zend_jit_ctx * jit,size_t size,const zend_op_array * op_array,const zend_op * opline)818 static ir_ref jit_EMALLOC(zend_jit_ctx *jit, size_t size, const zend_op_array *op_array, const zend_op *opline)
819 {
820 #if ZEND_DEBUG
821 return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_emalloc),
822 ir_CONST_ADDR(size),
823 op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL,
824 ir_CONST_U32(opline ? opline->lineno : 0),
825 IR_NULL,
826 ir_CONST_U32(0));
827 #elif defined(HAVE_BUILTIN_CONSTANT_P)
828 if (size > 24 && size <= 32) {
829 return ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(_emalloc_32));
830 } else {
831 return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size));
832 }
833 #else
834 return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size));
835 #endif
836 }
837
jit_EFREE(zend_jit_ctx * jit,ir_ref ptr,size_t size,const zend_op_array * op_array,const zend_op * opline)838 static ir_ref jit_EFREE(zend_jit_ctx *jit, ir_ref ptr, size_t size, const zend_op_array *op_array, const zend_op *opline)
839 {
840 #if ZEND_DEBUG
841 return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_efree),
842 ptr,
843 op_array && op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL,
844 ir_CONST_U32(opline ? opline->lineno : 0),
845 IR_NULL,
846 ir_CONST_U32(0));
847 #elif defined(HAVE_BUILTIN_CONSTANT_P)
848 if (size > 24 && size <= 32) {
849 return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree_32), ptr);
850 } else {
851 return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr);
852 }
853 #else
854 return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr);
855 #endif
856 }
857
jit_FP(zend_jit_ctx * jit)858 static ir_ref jit_FP(zend_jit_ctx *jit)
859 {
860 ZEND_ASSERT(jit->ctx.control);
861 if (jit->fp == IR_UNUSED) {
862 /* Emit "RLOAD FP" once for basic block */
863 jit->fp = ir_RLOAD_A(ZREG_FP);
864 } else {
865 ir_insn *insn;
866 ir_ref ref = jit->ctx.control;
867
868 while (1) {
869 if (ref == jit->fp) {
870 break;
871 }
872 insn = &jit->ctx.ir_base[ref];
873 if (insn->op >= IR_START || insn->op == IR_CALL) {
874 jit->fp = ir_RLOAD_A(ZREG_FP);
875 break;
876 }
877 ref = insn->op1;
878 }
879 }
880 return jit->fp;
881 }
882
jit_STORE_FP(zend_jit_ctx * jit,ir_ref ref)883 static void jit_STORE_FP(zend_jit_ctx *jit, ir_ref ref)
884 {
885 ir_RSTORE(ZREG_FP, ref);
886 jit->fp = IR_UNUSED;
887 }
888
jit_IP(zend_jit_ctx * jit)889 static ir_ref jit_IP(zend_jit_ctx *jit)
890 {
891 return ir_RLOAD_A(ZREG_IP);
892 }
893
jit_STORE_IP(zend_jit_ctx * jit,ir_ref ref)894 static void jit_STORE_IP(zend_jit_ctx *jit, ir_ref ref)
895 {
896 ir_RSTORE(ZREG_IP, ref);
897 }
898
jit_IP32(zend_jit_ctx * jit)899 static ir_ref jit_IP32(zend_jit_ctx *jit)
900 {
901 return ir_RLOAD_U32(ZREG_IP);
902 }
903
jit_LOAD_IP(zend_jit_ctx * jit,ir_ref ref)904 static void jit_LOAD_IP(zend_jit_ctx *jit, ir_ref ref)
905 {
906 if (GCC_GLOBAL_REGS) {
907 jit_STORE_IP(jit, ref);
908 } else {
909 ir_STORE(jit_EX(opline), ref);
910 }
911 }
912
jit_LOAD_IP_ADDR(zend_jit_ctx * jit,const zend_op * target)913 static void jit_LOAD_IP_ADDR(zend_jit_ctx *jit, const zend_op *target)
914 {
915 jit_LOAD_IP(jit, ir_CONST_ADDR(target));
916 }
917
zend_jit_track_last_valid_opline(zend_jit_ctx * jit)918 static void zend_jit_track_last_valid_opline(zend_jit_ctx *jit)
919 {
920 jit->use_last_valid_opline = 0;
921 jit->track_last_valid_opline = 1;
922 }
923
zend_jit_use_last_valid_opline(zend_jit_ctx * jit)924 static void zend_jit_use_last_valid_opline(zend_jit_ctx *jit)
925 {
926 if (jit->track_last_valid_opline) {
927 jit->use_last_valid_opline = 1;
928 jit->track_last_valid_opline = 0;
929 }
930 }
931
zend_jit_trace_uses_initial_ip(zend_jit_ctx * jit)932 static bool zend_jit_trace_uses_initial_ip(zend_jit_ctx *jit)
933 {
934 return jit->use_last_valid_opline;
935 }
936
zend_jit_set_last_valid_opline(zend_jit_ctx * jit,const zend_op * opline)937 static void zend_jit_set_last_valid_opline(zend_jit_ctx *jit, const zend_op *opline)
938 {
939 if (!jit->reuse_ip) {
940 jit->track_last_valid_opline = 1;
941 jit->last_valid_opline = opline;
942 }
943 }
944
zend_jit_reset_last_valid_opline(zend_jit_ctx * jit)945 static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit)
946 {
947 jit->track_last_valid_opline = 0;
948 jit->last_valid_opline = NULL;
949 }
950
zend_jit_start_reuse_ip(zend_jit_ctx * jit)951 static void zend_jit_start_reuse_ip(zend_jit_ctx *jit)
952 {
953 zend_jit_reset_last_valid_opline(jit);
954 jit->reuse_ip = 1;
955 }
956
zend_jit_reuse_ip(zend_jit_ctx * jit)957 static int zend_jit_reuse_ip(zend_jit_ctx *jit)
958 {
959 if (!jit->reuse_ip) {
960 zend_jit_start_reuse_ip(jit);
961 // RX = EX(call);
962 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call)));
963 }
964 return 1;
965 }
966
zend_jit_stop_reuse_ip(zend_jit_ctx * jit)967 static void zend_jit_stop_reuse_ip(zend_jit_ctx *jit)
968 {
969 jit->reuse_ip = 0;
970 }
971
zend_jit_save_call_chain(zend_jit_ctx * jit,uint32_t call_level)972 static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level)
973 {
974 ir_ref rx, call;
975
976 if (call_level == 1) {
977 // JIT: call = NULL;
978 call = IR_NULL;
979 } else {
980 // JIT: call = EX(call);
981 call = ir_LOAD_A(jit_EX(call));
982 }
983
984 rx = jit_IP(jit);
985
986 // JIT: call->prev_execute_data = call;
987 ir_STORE(jit_CALL(rx, prev_execute_data), call);
988
989 // JIT: EX(call) = call;
990 ir_STORE(jit_EX(call), rx);
991
992 jit->delayed_call_level = 0;
993 delayed_call_chain = 0;
994
995 return 1;
996 }
997
zend_jit_set_ip(zend_jit_ctx * jit,const zend_op * target)998 static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target)
999 {
1000 ir_ref ref;
1001 ir_ref addr = IR_UNUSED;
1002
1003 if (jit->delayed_call_level) {
1004 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
1005 return 0;
1006 }
1007 }
1008
1009 if (jit->last_valid_opline) {
1010 zend_jit_use_last_valid_opline(jit);
1011 if (jit->last_valid_opline != target) {
1012 if (GCC_GLOBAL_REGS) {
1013 ref = jit_IP(jit);
1014 } else {
1015 addr = jit_EX(opline);
1016 ref = ir_LOAD_A(addr);
1017 }
1018 if (target > jit->last_valid_opline) {
1019 ref = ir_ADD_OFFSET(ref, (uintptr_t)target - (uintptr_t)jit->last_valid_opline);
1020 } else {
1021 ref = ir_SUB_A(ref, ir_CONST_ADDR((uintptr_t)jit->last_valid_opline - (uintptr_t)target));
1022 }
1023 if (GCC_GLOBAL_REGS) {
1024 jit_STORE_IP(jit, ref);
1025 } else {
1026 ir_STORE(addr, ref);
1027 }
1028 }
1029 } else {
1030 if (GCC_GLOBAL_REGS) {
1031 jit_STORE_IP(jit, ir_CONST_ADDR(target));
1032 } else {
1033 ir_STORE(jit_EX(opline), ir_CONST_ADDR(target));
1034 }
1035 }
1036 jit->reuse_ip = 0;
1037 zend_jit_set_last_valid_opline(jit, target);
1038 return 1;
1039 }
1040
zend_jit_set_ip_ex(zend_jit_ctx * jit,const zend_op * target,bool set_ip_reg)1041 static int zend_jit_set_ip_ex(zend_jit_ctx *jit, const zend_op *target, bool set_ip_reg)
1042 {
1043 if (!GCC_GLOBAL_REGS && set_ip_reg && !jit->last_valid_opline) {
1044 /* Optimization to avoid duplicate constant load */
1045 ir_STORE(jit_EX(opline), ir_HARD_COPY_A(ir_CONST_ADDR(target)));
1046 return 1;
1047 }
1048 return zend_jit_set_ip(jit, target);
1049 }
1050
jit_SET_EX_OPLINE(zend_jit_ctx * jit,const zend_op * target)1051 static void jit_SET_EX_OPLINE(zend_jit_ctx *jit, const zend_op *target)
1052 {
1053 if (jit->last_valid_opline == target) {
1054 zend_jit_use_last_valid_opline(jit);
1055 if (GCC_GLOBAL_REGS) {
1056 // EX(opline) = opline
1057 ir_STORE(jit_EX(opline), jit_IP(jit));
1058 }
1059 } else {
1060 ir_STORE(jit_EX(opline), ir_CONST_ADDR(target));
1061 if (!GCC_GLOBAL_REGS) {
1062 zend_jit_reset_last_valid_opline(jit);
1063 }
1064 }
1065 }
1066
jit_ZVAL_ADDR(zend_jit_ctx * jit,zend_jit_addr addr)1067 static ir_ref jit_ZVAL_ADDR(zend_jit_ctx *jit, zend_jit_addr addr)
1068 {
1069 if (Z_MODE(addr) == IS_MEM_ZVAL) {
1070 ir_ref reg;
1071
1072 if (Z_REG(addr) == ZREG_FP) {
1073 reg = jit_FP(jit);
1074 } else if (Z_REG(addr) == ZREG_RX) {
1075 reg = jit_IP(jit);
1076 } else {
1077 ZEND_UNREACHABLE();
1078 }
1079 return ir_ADD_OFFSET(reg, Z_OFFSET(addr));
1080 } else if (Z_MODE(addr) == IS_REF_ZVAL) {
1081 return Z_IR_REF(addr);
1082 } else {
1083 ZEND_ASSERT(Z_MODE(addr) == IS_CONST_ZVAL);
1084 return ir_CONST_ADDR(Z_ZV(addr));
1085 }
1086 }
1087
jit_Z_TYPE_ref(zend_jit_ctx * jit,ir_ref ref)1088 static ir_ref jit_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref)
1089 {
1090 return ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zval, u1.v.type)));
1091 }
1092
jit_Z_TYPE(zend_jit_ctx * jit,zend_jit_addr addr)1093 static ir_ref jit_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr)
1094 {
1095 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1096 return ir_CONST_U8(Z_TYPE_P(Z_ZV(addr)));
1097 } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
1098 ir_ref reg;
1099
1100 ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
1101 if (Z_REG(addr) == ZREG_FP) {
1102 reg = jit_FP(jit);
1103 } else if (Z_REG(addr) == ZREG_RX) {
1104 reg = jit_IP(jit);
1105 } else {
1106 ZEND_UNREACHABLE();
1107 }
1108 return ir_LOAD_U8(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.v.type)));
1109 } else {
1110 return jit_Z_TYPE_ref(jit, jit_ZVAL_ADDR(jit, addr));
1111 }
1112 }
1113
jit_Z_TYPE_FLAGS_ref(zend_jit_ctx * jit,ir_ref ref)1114 static ir_ref jit_Z_TYPE_FLAGS_ref(zend_jit_ctx *jit, ir_ref ref)
1115 {
1116 return ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zval, u1.v.type_flags)));
1117 }
1118
jit_Z_TYPE_FLAGS(zend_jit_ctx * jit,zend_jit_addr addr)1119 static ir_ref jit_Z_TYPE_FLAGS(zend_jit_ctx *jit, zend_jit_addr addr)
1120 {
1121 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1122 return ir_CONST_U8(Z_TYPE_FLAGS_P(Z_ZV(addr)));
1123 } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
1124 ir_ref reg;
1125
1126 ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
1127 if (Z_REG(addr) == ZREG_FP) {
1128 reg = jit_FP(jit);
1129 } else if (Z_REG(addr) == ZREG_RX) {
1130 reg = jit_IP(jit);
1131 } else {
1132 ZEND_UNREACHABLE();
1133 }
1134 return ir_LOAD_U8(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.v.type_flags)));
1135 } else {
1136 return jit_Z_TYPE_FLAGS_ref(jit, jit_ZVAL_ADDR(jit, addr));
1137 }
1138 }
1139
jit_Z_TYPE_INFO_ref(zend_jit_ctx * jit,ir_ref ref)1140 static ir_ref jit_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref)
1141 {
1142 return ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zval, u1.type_info)));
1143 }
1144
jit_Z_TYPE_INFO(zend_jit_ctx * jit,zend_jit_addr addr)1145 static ir_ref jit_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr)
1146 {
1147 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1148 return ir_CONST_U32(Z_TYPE_INFO_P(Z_ZV(addr)));
1149 } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
1150 ir_ref reg;
1151
1152 ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
1153 if (Z_REG(addr) == ZREG_FP) {
1154 reg = jit_FP(jit);
1155 } else if (Z_REG(addr) == ZREG_RX) {
1156 reg = jit_IP(jit);
1157 } else {
1158 ZEND_UNREACHABLE();
1159 }
1160 return ir_LOAD_U32(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.type_info)));
1161 } else {
1162 return jit_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr));
1163 }
1164 }
1165
jit_set_Z_TYPE_INFO_ref(zend_jit_ctx * jit,ir_ref ref,ir_ref type_info)1166 static void jit_set_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type_info)
1167 {
1168 ir_STORE(ir_ADD_OFFSET(ref, offsetof(zval, u1.type_info)), type_info);
1169 }
1170
jit_set_Z_TYPE_INFO_ex(zend_jit_ctx * jit,zend_jit_addr addr,ir_ref type_info)1171 static void jit_set_Z_TYPE_INFO_ex(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref type_info)
1172 {
1173 if (Z_MODE(addr) == IS_MEM_ZVAL) {
1174 ir_ref reg;
1175
1176 ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
1177 if (Z_REG(addr) == ZREG_FP) {
1178 reg = jit_FP(jit);
1179 } else if (Z_REG(addr) == ZREG_RX) {
1180 reg = jit_IP(jit);
1181 } else {
1182 ZEND_UNREACHABLE();
1183 }
1184 ir_STORE(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.type_info)), type_info);
1185 } else {
1186 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr), type_info);
1187 }
1188 }
1189
jit_set_Z_TYPE_INFO(zend_jit_ctx * jit,zend_jit_addr addr,uint32_t type_info)1190 static void jit_set_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr, uint32_t type_info)
1191 {
1192 if (type_info < IS_STRING
1193 && Z_MODE(addr) == IS_MEM_ZVAL
1194 && Z_REG(addr) == ZREG_FP
1195 && JIT_G(current_frame)
1196 && STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(addr))) == type_info) {
1197 /* type is already set */
1198 return;
1199 }
1200 jit_set_Z_TYPE_INFO_ex(jit, addr, ir_CONST_U32(type_info));
1201 }
1202
jit_if_Z_TYPE_ref(zend_jit_ctx * jit,ir_ref ref,ir_ref type)1203 static ir_ref jit_if_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type)
1204 {
1205 return ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), type));
1206 }
1207
jit_if_Z_TYPE(zend_jit_ctx * jit,zend_jit_addr addr,uint8_t type)1208 static ir_ref jit_if_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type)
1209 {
1210 ZEND_ASSERT(type != IS_UNDEF);
1211 return ir_IF(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type)));
1212 }
1213
jit_if_not_Z_TYPE(zend_jit_ctx * jit,zend_jit_addr addr,uint8_t type)1214 static ir_ref jit_if_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type)
1215 {
1216 ir_ref ref = jit_Z_TYPE(jit, addr);
1217
1218 if (type != IS_UNDEF) {
1219 ref = ir_NE(ref, ir_CONST_U8(type));
1220 }
1221 return ir_IF(ref);
1222 }
1223
jit_guard_Z_TYPE(zend_jit_ctx * jit,zend_jit_addr addr,uint8_t type,const void * exit_addr)1224 static void jit_guard_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_addr)
1225 {
1226 ir_ref ref = jit_Z_TYPE(jit, addr);
1227
1228 if (type != IS_UNDEF) {
1229 ir_GUARD(ir_EQ(ref, ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr));
1230 } else {
1231 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
1232 }
1233 }
1234
jit_guard_not_Z_TYPE(zend_jit_ctx * jit,zend_jit_addr addr,uint8_t type,const void * exit_addr)1235 static void jit_guard_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_addr)
1236 {
1237 ir_ref ref = jit_Z_TYPE(jit, addr);
1238
1239 if (type != IS_UNDEF) {
1240 ref = ir_NE(ref, ir_CONST_U8(type));
1241 }
1242 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
1243 }
1244
jit_if_REFCOUNTED(zend_jit_ctx * jit,zend_jit_addr addr)1245 static ir_ref jit_if_REFCOUNTED(zend_jit_ctx *jit, zend_jit_addr addr)
1246 {
1247 return ir_IF(jit_Z_TYPE_FLAGS(jit, addr));
1248 }
1249
jit_if_COLLECTABLE_ref(zend_jit_ctx * jit,ir_ref addr_ref)1250 static ir_ref jit_if_COLLECTABLE_ref(zend_jit_ctx *jit, ir_ref addr_ref)
1251 {
1252 return ir_IF(ir_AND_U8(jit_Z_TYPE_FLAGS_ref(jit, addr_ref), ir_CONST_U8(IS_TYPE_COLLECTABLE)));
1253 }
1254
jit_Z_LVAL_ref(zend_jit_ctx * jit,ir_ref ref)1255 static ir_ref jit_Z_LVAL_ref(zend_jit_ctx *jit, ir_ref ref)
1256 {
1257 return ir_LOAD_L(ref);
1258 }
1259
jit_Z_DVAL_ref(zend_jit_ctx * jit,ir_ref ref)1260 static ir_ref jit_Z_DVAL_ref(zend_jit_ctx *jit, ir_ref ref)
1261 {
1262 return ir_LOAD_D(ref);
1263 }
1264
zend_jit_spilling_may_cause_conflict(zend_jit_ctx * jit,int var,ir_ref val)1265 static bool zend_jit_spilling_may_cause_conflict(zend_jit_ctx *jit, int var, ir_ref val)
1266 {
1267 if (jit->ctx.ir_base[val].op == IR_RLOAD) {
1268 /* Deoptimization */
1269 return 0;
1270 }
1271 // if (jit->ctx.ir_base[val].op == IR_LOAD
1272 // && jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op == IR_ADD
1273 // && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op == IR_RLOAD
1274 // && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op2 == ZREG_FP
1275 // && IR_IS_CONST_REF(jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2)
1276 // && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr == (uintptr_t)EX_NUM_TO_VAR(jit->ssa->vars[var].var)) {
1277 // /* LOAD from the same location (the LOAD is pinned) */
1278 // // TODO: should be anti-dependent with the following stores ???
1279 // return 0;
1280 // }
1281 if (jit->ssa->vars[var].var < jit->current_op_array->last_var) {
1282 /* IS_CV */
1283 return 0;
1284 }
1285 return 1;
1286 }
1287
zend_jit_def_reg(zend_jit_ctx * jit,zend_jit_addr addr,ir_ref val)1288 static void zend_jit_def_reg(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref val)
1289 {
1290 int var;
1291
1292 ZEND_ASSERT(Z_MODE(addr) == IS_REG);
1293 var = Z_SSA_VAR(addr);
1294 if (var == jit->delay_var) {
1295 ir_refs_add(jit->delay_refs, val);
1296 return;
1297 }
1298 ZEND_ASSERT(jit->ra && jit->ra[var].ref == IR_NULL);
1299
1300 /* Negative "var" has special meaning for IR */
1301 if (val > 0 && !zend_jit_spilling_may_cause_conflict(jit, var, val)) {
1302 val = ir_bind(&jit->ctx, -EX_NUM_TO_VAR(jit->ssa->vars[var].var), val);
1303 }
1304 jit->ra[var].ref = val;
1305
1306 if (jit->ra[var].flags & ZREG_FORWARD) {
1307 zend_ssa_phi *phi = jit->ssa->vars[var].phi_use_chain;
1308 zend_basic_block *bb;
1309 int n, j, *p;
1310 ir_ref *q;
1311
1312 jit->ra[var].flags &= ~ZREG_FORWARD;
1313 while (phi != NULL) {
1314 zend_ssa_phi *dst_phi = phi;
1315 int src_var = var;
1316
1317 if (dst_phi->pi >= 0) {
1318 jit->ra[src_var].ref = val;
1319 src_var = dst_phi->ssa_var;
1320 if (!(jit->ra[src_var].flags & ZREG_FORWARD)) {
1321 phi = zend_ssa_next_use_phi(jit->ssa, var, phi);
1322 continue;
1323 }
1324 dst_phi = jit->ssa->vars[src_var].phi_use_chain;
1325 ZEND_ASSERT(dst_phi != NULL && "reg forwarding");
1326 ZEND_ASSERT(!zend_ssa_next_use_phi(jit->ssa, src_var, dst_phi) && "reg forwarding");
1327 jit->ra[src_var].flags &= ~ZREG_FORWARD;
1328 }
1329
1330 if (jit->ra[dst_phi->ssa_var].ref > 0) {
1331 ir_insn *phi_insn = &jit->ctx.ir_base[jit->ra[dst_phi->ssa_var].ref];
1332 if (phi_insn->op == IR_PHI) {
1333 // ZEND_ASSERT(ir_operands_count(ctx, phi_insn) == n + 1);
1334 bb = &jit->ssa->cfg.blocks[dst_phi->block];
1335 n = bb->predecessors_count;
1336 for (j = 0, p = &dst_phi->sources[0], q = phi_insn->ops + 2; j < n; j++, p++, q++) {
1337 if (*p == src_var) {
1338 *q = val;
1339 }
1340 }
1341 }
1342 }
1343
1344 phi = zend_ssa_next_use_phi(jit->ssa, var, phi);
1345 }
1346 }
1347 }
1348
zend_jit_use_reg(zend_jit_ctx * jit,zend_jit_addr addr)1349 static ir_ref zend_jit_use_reg(zend_jit_ctx *jit, zend_jit_addr addr)
1350 {
1351 int var = Z_SSA_VAR(addr);
1352
1353 ZEND_ASSERT(Z_MODE(addr) == IS_REG);
1354 ZEND_ASSERT(jit->ra && jit->ra[var].ref);
1355 if (jit->ra[var].ref == IR_NULL) {
1356 zend_jit_addr mem_addr;
1357 ir_ref ref;
1358
1359 ZEND_ASSERT(jit->ra[var].flags & ZREG_LOAD);
1360 mem_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(jit->ssa->vars[var].var));
1361 if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_LONG) {
1362 ref = jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr));
1363 } else if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) {
1364 ref = jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr));
1365 } else {
1366 ZEND_UNREACHABLE();
1367 }
1368 zend_jit_def_reg(jit, addr, ref);
1369 return ref;
1370 }
1371 return jit->ra[Z_SSA_VAR(addr)].ref;
1372 }
1373
zend_jit_gen_pi(zend_jit_ctx * jit,zend_ssa_phi * phi)1374 static void zend_jit_gen_pi(zend_jit_ctx *jit, zend_ssa_phi *phi)
1375 {
1376 int src_var = phi->sources[0];
1377 int dst_var = phi->ssa_var;
1378
1379 ZEND_ASSERT(phi->pi >= 0);
1380 ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
1381 ZEND_ASSERT(jit->ra[src_var].ref);
1382
1383 if (jit->ra[src_var].ref == IR_NULL) {
1384 /* Not defined yet */
1385 if (jit->ssa->vars[dst_var].use_chain < 0
1386 && jit->ssa->vars[dst_var].phi_use_chain) {
1387 zend_ssa_phi *phi = jit->ssa->vars[dst_var].phi_use_chain;
1388 if (!zend_ssa_next_use_phi(jit->ssa, dst_var, phi)) {
1389 /* This is a Pi forwarded to Phi */
1390 jit->ra[src_var].flags |= ZREG_FORWARD;
1391 return;
1392 }
1393 }
1394 ZEND_ASSERT(0 && "Not defined Pi source");
1395 }
1396 /* Reuse register */
1397 zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var),
1398 zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)));
1399 }
1400
zend_jit_gen_phi(zend_jit_ctx * jit,zend_ssa_phi * phi)1401 static void zend_jit_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi)
1402 {
1403 int dst_var = phi->ssa_var;
1404 zend_basic_block *bb = &jit->ssa->cfg.blocks[phi->block];
1405 int n = bb->predecessors_count;
1406 int i;
1407 ir_type type = (jit->ssa->var_info[phi->ssa_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE;
1408 ir_ref merge = jit->bb_start_ref[phi->block];
1409 ir_ref ref;
1410 ir_ref old_insns_count = jit->ctx.insns_count;
1411 ir_ref same_src_ref = IR_UNUSED;
1412 bool phi_inputs_are_the_same = 1;
1413
1414 ZEND_ASSERT(phi->pi < 0);
1415 ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
1416 ZEND_ASSERT(merge);
1417 ZEND_ASSERT(jit->ctx.ir_base[merge].op == IR_MERGE || jit->ctx.ir_base[merge].op == IR_LOOP_BEGIN);
1418 ZEND_ASSERT(n == jit->ctx.ir_base[merge].inputs_count);
1419
1420 ref = ir_emit_N(&jit->ctx, IR_OPT(IR_PHI, type), n + 1);
1421 ir_set_op(&jit->ctx, ref, 1, merge);
1422
1423 for (i = 0; i < n; i++) {
1424 int src_var = phi->sources[i];
1425
1426 ZEND_ASSERT(jit->ra[src_var].ref);
1427 if (jit->ra[src_var].ref == IR_NULL) {
1428 jit->ra[src_var].flags |= ZREG_FORWARD;
1429 phi_inputs_are_the_same = 0;
1430 } else {
1431 ir_ref src_ref = zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var));
1432 if (i == 0) {
1433 same_src_ref = src_ref;
1434 } else if (same_src_ref != src_ref) {
1435 phi_inputs_are_the_same = 0;
1436 }
1437 ir_set_op(&jit->ctx, ref, i + 2, src_ref);
1438 }
1439 }
1440 if (phi_inputs_are_the_same) {
1441 ref = same_src_ref;
1442 jit->ctx.insns_count = old_insns_count;
1443 }
1444
1445 zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref);
1446 }
1447
jit_Z_LVAL(zend_jit_ctx * jit,zend_jit_addr addr)1448 static ir_ref jit_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr)
1449 {
1450 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1451 return ir_CONST_LONG(Z_LVAL_P(Z_ZV(addr)));
1452 } else if (Z_MODE(addr) == IS_REG) {
1453 return zend_jit_use_reg(jit, addr);
1454 } else {
1455 return jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, addr));
1456 }
1457 }
1458
jit_set_Z_LVAL(zend_jit_ctx * jit,zend_jit_addr addr,ir_ref lval)1459 static void jit_set_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval)
1460 {
1461 if (Z_MODE(addr) == IS_REG) {
1462 zend_jit_def_reg(jit, addr, lval);
1463 } else {
1464 ir_STORE(jit_ZVAL_ADDR(jit, addr), lval);
1465 }
1466 }
1467
1468 #if SIZEOF_ZEND_LONG == 4
jit_Z_W2(zend_jit_ctx * jit,zend_jit_addr addr)1469 static ir_ref jit_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr)
1470 {
1471 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1472 return ir_CONST_U32((Z_ZV(addr))->value.ww.w2);
1473 } else {
1474 return ir_LOAD_L(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2)));
1475 }
1476 }
1477
jit_set_Z_W2(zend_jit_ctx * jit,zend_jit_addr addr,ir_ref lval)1478 static void jit_set_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval)
1479 {
1480 ir_STORE(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2)), lval);
1481 }
1482 #endif
1483
jit_Z_DVAL(zend_jit_ctx * jit,zend_jit_addr addr)1484 static ir_ref jit_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr)
1485 {
1486 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1487 return ir_CONST_DOUBLE(Z_DVAL_P(Z_ZV(addr)));
1488 } else if (Z_MODE(addr) == IS_REG) {
1489 return zend_jit_use_reg(jit, addr);
1490 } else {
1491 return jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, addr));
1492 }
1493 }
1494
jit_set_Z_DVAL(zend_jit_ctx * jit,zend_jit_addr addr,ir_ref dval)1495 static void jit_set_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref dval)
1496 {
1497 if (Z_MODE(addr) == IS_REG) {
1498 zend_jit_def_reg(jit, addr, dval);
1499 } else {
1500 ir_STORE(jit_ZVAL_ADDR(jit, addr), dval);
1501 }
1502 }
1503
jit_Z_PTR_ref(zend_jit_ctx * jit,ir_ref ref)1504 static ir_ref jit_Z_PTR_ref(zend_jit_ctx *jit, ir_ref ref)
1505 {
1506 return ir_LOAD_A(ref);
1507 }
1508
jit_Z_PTR(zend_jit_ctx * jit,zend_jit_addr addr)1509 static ir_ref jit_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr)
1510 {
1511 if (Z_MODE(addr) == IS_CONST_ZVAL) {
1512 return ir_CONST_ADDR(Z_PTR_P(Z_ZV(addr)));
1513 } else {
1514 return jit_Z_PTR_ref(jit, jit_ZVAL_ADDR(jit, addr));
1515 }
1516 }
1517
jit_set_Z_PTR(zend_jit_ctx * jit,zend_jit_addr addr,ir_ref ptr)1518 static void jit_set_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref ptr)
1519 {
1520 ir_STORE(jit_ZVAL_ADDR(jit, addr), ptr);
1521 }
1522
jit_GC_REFCOUNT(zend_jit_ctx * jit,ir_ref ref)1523 static ir_ref jit_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref)
1524 {
1525 return ir_LOAD_U32(ref);
1526 }
1527
jit_set_GC_REFCOUNT(zend_jit_ctx * jit,ir_ref ref,uint32_t refcount)1528 static void jit_set_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref, uint32_t refcount)
1529 {
1530 ir_STORE(ref, ir_CONST_U32(refcount));
1531 }
1532
jit_GC_ADDREF(zend_jit_ctx * jit,ir_ref ref)1533 static void jit_GC_ADDREF(zend_jit_ctx *jit, ir_ref ref)
1534 {
1535 ir_STORE(ref, ir_ADD_U32(ir_LOAD_U32(ref), ir_CONST_U32(1)));
1536 }
1537
jit_GC_ADDREF2(zend_jit_ctx * jit,ir_ref ref)1538 static void jit_GC_ADDREF2(zend_jit_ctx *jit, ir_ref ref)
1539 {
1540 ir_ref counter = ir_LOAD_U32(ref);
1541 ir_STORE(ref, ir_ADD_U32(counter, ir_CONST_U32(2)));
1542 }
1543
jit_GC_DELREF(zend_jit_ctx * jit,ir_ref ref)1544 static ir_ref jit_GC_DELREF(zend_jit_ctx *jit, ir_ref ref)
1545 {
1546 ir_ref counter = ir_LOAD_U32(ref);
1547 counter = ir_SUB_U32(counter, ir_CONST_U32(1));
1548 ir_STORE(ref, counter);
1549 return counter;
1550 }
1551
jit_if_GC_MAY_NOT_LEAK(zend_jit_ctx * jit,ir_ref ref)1552 static ir_ref jit_if_GC_MAY_NOT_LEAK(zend_jit_ctx *jit, ir_ref ref)
1553 {
1554 return ir_IF(
1555 ir_AND_U32(
1556 ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_refcounted, gc.u.type_info))),
1557 ir_CONST_U32(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT))));
1558 }
1559
jit_ZVAL_COPY_CONST(zend_jit_ctx * jit,zend_jit_addr dst,uint32_t dst_info,uint32_t dst_def_info,zval * zv,bool addref)1560 static void jit_ZVAL_COPY_CONST(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, uint32_t dst_def_info, zval *zv, bool addref)
1561 {
1562 ir_ref ref = IR_UNUSED;
1563
1564 if (Z_TYPE_P(zv) > IS_TRUE) {
1565 if (Z_TYPE_P(zv) == IS_DOUBLE) {
1566 jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE(Z_DVAL_P(zv)));
1567 } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) {
1568 jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE((double)Z_LVAL_P(zv)));
1569 } else if (Z_TYPE_P(zv) == IS_LONG) {
1570 jit_set_Z_LVAL(jit, dst, ir_CONST_LONG(Z_LVAL_P(zv)));
1571 } else {
1572 ref = ir_CONST_ADDR(Z_PTR_P(zv));
1573 jit_set_Z_PTR(jit, dst, ref);
1574 if (addref && Z_REFCOUNTED_P(zv)) {
1575 jit_GC_ADDREF(jit, ref);
1576 }
1577 }
1578 }
1579 if (Z_MODE(dst) != IS_REG) {
1580 if (dst_def_info == MAY_BE_DOUBLE) {
1581 if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
1582 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
1583 }
1584 } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
1585 jit_set_Z_TYPE_INFO(jit, dst, Z_TYPE_INFO_P(zv));
1586 }
1587 }
1588 }
1589
jit_if_TYPED_REF(zend_jit_ctx * jit,ir_ref ref)1590 static ir_ref jit_if_TYPED_REF(zend_jit_ctx *jit, ir_ref ref)
1591 {
1592 return ir_IF(ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_reference, sources.ptr))));
1593 }
1594
jit_ZVAL_COPY(zend_jit_ctx * jit,zend_jit_addr dst,uint32_t dst_info,zend_jit_addr src,uint32_t src_info,bool addref)1595 static void jit_ZVAL_COPY(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, zend_jit_addr src, uint32_t src_info, bool addref)
1596 {
1597 ir_ref ref = IR_UNUSED;
1598
1599 if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
1600 if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
1601 jit_set_Z_LVAL(jit, dst, jit_Z_LVAL(jit, src));
1602 } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
1603 jit_set_Z_DVAL(jit, dst, jit_Z_DVAL(jit, src));
1604 } else {
1605 #if SIZEOF_ZEND_LONG == 4
1606 if (src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) {
1607 jit_set_Z_W2(jit, dst, jit_Z_W2(jit, src));
1608 }
1609 #endif
1610 ref = jit_Z_PTR(jit, src);
1611 jit_set_Z_PTR(jit, dst, ref);
1612 }
1613 }
1614 if (has_concrete_type(src_info & MAY_BE_ANY)
1615 && (src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))
1616 && !(src_info & MAY_BE_GUARD)) {
1617 if (Z_MODE(dst) != IS_REG
1618 && (dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
1619 uint8_t type = concrete_type(src_info);
1620 jit_set_Z_TYPE_INFO(jit, dst, type);
1621 }
1622 } else {
1623 ir_ref type = jit_Z_TYPE_INFO(jit, src);
1624 jit_set_Z_TYPE_INFO_ex(jit, dst, type);
1625 if (addref) {
1626 if (src_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
1627 ir_ref if_refcounted = IR_UNUSED;
1628
1629 if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
1630 if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(0xff00)));
1631 ir_IF_TRUE(if_refcounted);
1632 }
1633
1634 jit_GC_ADDREF(jit, ref);
1635
1636 if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
1637 ir_MERGE_WITH_EMPTY_FALSE(if_refcounted);
1638 }
1639 }
1640 }
1641 }
1642 }
1643
jit_ZVAL_COPY_2(zend_jit_ctx * jit,zend_jit_addr dst2,zend_jit_addr dst,uint32_t dst_info,zend_jit_addr src,uint32_t src_info,int addref)1644 static void jit_ZVAL_COPY_2(zend_jit_ctx *jit, zend_jit_addr dst2, zend_jit_addr dst, uint32_t dst_info, zend_jit_addr src, uint32_t src_info, int addref)
1645 {
1646 ir_ref ref = IR_UNUSED;
1647
1648 if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
1649 if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
1650 ref = jit_Z_LVAL(jit, src);
1651 jit_set_Z_LVAL(jit, dst, ref);
1652 jit_set_Z_LVAL(jit, dst2, ref);
1653 } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
1654 ref = jit_Z_DVAL(jit, src);
1655 jit_set_Z_DVAL(jit, dst, ref);
1656 jit_set_Z_DVAL(jit, dst2, ref);
1657 } else {
1658 #if SIZEOF_ZEND_LONG == 4
1659 if (src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) {
1660 ref = jit_Z_W2(jit, src);
1661 jit_set_Z_W2(jit, dst, ref);
1662 jit_set_Z_W2(jit, dst2, ref);
1663 }
1664 #endif
1665 ref = jit_Z_PTR(jit, src);
1666 jit_set_Z_PTR(jit, dst, ref);
1667 jit_set_Z_PTR(jit, dst2, ref);
1668 }
1669 }
1670 if (has_concrete_type(src_info & MAY_BE_ANY)
1671 && (src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))
1672 && !(src_info & MAY_BE_GUARD)) {
1673 uint8_t type = concrete_type(src_info);
1674 ir_ref type_ref = ir_CONST_U32(type);
1675
1676 if (Z_MODE(dst) != IS_REG
1677 && (dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
1678 jit_set_Z_TYPE_INFO_ex(jit, dst, type_ref);
1679 }
1680 if (Z_MODE(dst2) != IS_REG) {
1681 jit_set_Z_TYPE_INFO_ex(jit, dst2, type_ref);
1682 }
1683 } else {
1684 ir_ref type = jit_Z_TYPE_INFO(jit, src);
1685 jit_set_Z_TYPE_INFO_ex(jit, dst, type);
1686 jit_set_Z_TYPE_INFO_ex(jit, dst2, type);
1687 if (addref) {
1688 if (src_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
1689 ir_ref if_refcounted = IR_UNUSED;
1690
1691 if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
1692 if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(0xff00)));
1693 ir_IF_TRUE(if_refcounted);
1694 }
1695
1696 if (addref == 2) {
1697 jit_GC_ADDREF2(jit, ref);
1698 } else {
1699 jit_GC_ADDREF(jit, ref);
1700 }
1701
1702 if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
1703 ir_MERGE_WITH_EMPTY_FALSE(if_refcounted);
1704 }
1705 }
1706 }
1707 }
1708 }
1709
jit_ZVAL_DTOR(zend_jit_ctx * jit,ir_ref ref,uint32_t op_info,const zend_op * opline)1710 static void jit_ZVAL_DTOR(zend_jit_ctx *jit, ir_ref ref, uint32_t op_info, const zend_op *opline)
1711 {
1712 if (!((op_info) & MAY_BE_GUARD)
1713 && has_concrete_type((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
1714 uint8_t type = concrete_type((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE));
1715 if (type == IS_STRING && !ZEND_DEBUG) {
1716 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(_efree), ref);
1717 return;
1718 } else if (type == IS_ARRAY) {
1719 if ((op_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) {
1720 if (opline && ((op_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) {
1721 jit_SET_EX_OPLINE(jit, opline);
1722 }
1723 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_array_destroy), ref);
1724 } else {
1725 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_array_free), ref);
1726 }
1727 return;
1728 } else if (type == IS_OBJECT) {
1729 if (opline) {
1730 jit_SET_EX_OPLINE(jit, opline);
1731 }
1732 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_objects_store_del), ref);
1733 return;
1734 }
1735 }
1736 if (opline) {
1737 jit_SET_EX_OPLINE(jit, opline);
1738 }
1739 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(rc_dtor_func), ref);
1740 }
1741
jit_ZVAL_PTR_DTOR(zend_jit_ctx * jit,zend_jit_addr addr,uint32_t op_info,bool gc,const zend_op * opline)1742 static void jit_ZVAL_PTR_DTOR(zend_jit_ctx *jit,
1743 zend_jit_addr addr,
1744 uint32_t op_info,
1745 bool gc,
1746 const zend_op *opline)
1747 {
1748 ir_ref ref, ref2;
1749 ir_ref if_refcounted = IR_UNUSED;
1750 ir_ref if_not_zero = IR_UNUSED;
1751 ir_ref end_inputs = IR_UNUSED;
1752
1753 if (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) {
1754 if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
1755 if_refcounted = jit_if_REFCOUNTED(jit, addr);
1756 ir_IF_FALSE(if_refcounted);
1757 ir_END_list(end_inputs);
1758 ir_IF_TRUE(if_refcounted);
1759 }
1760 ref = jit_Z_PTR(jit, addr);
1761 ref2 = jit_GC_DELREF(jit, ref);
1762
1763 if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) {
1764 if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) {
1765 if_not_zero = ir_IF(ref2);
1766 ir_IF_FALSE(if_not_zero);
1767 }
1768 // zval_dtor_func(r);
1769 jit_ZVAL_DTOR(jit, ref, op_info, opline);
1770 if (if_not_zero) {
1771 ir_END_list(end_inputs);
1772 ir_IF_TRUE(if_not_zero);
1773 }
1774 }
1775 if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
1776 ir_ref if_may_not_leak;
1777
1778 if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) {
1779 ir_ref if_ref, if_collectable;
1780
1781 if_ref = jit_if_Z_TYPE(jit, addr, IS_REFERENCE);
1782 ir_IF_TRUE(if_ref);
1783
1784 ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
1785
1786 if_collectable = jit_if_COLLECTABLE_ref(jit, ref2);
1787 ir_IF_FALSE(if_collectable);
1788 ir_END_list(end_inputs);
1789 ir_IF_TRUE(if_collectable);
1790
1791 ref2 = jit_Z_PTR_ref(jit, ref2);
1792
1793 ir_MERGE_WITH_EMPTY_FALSE(if_ref);
1794 ref = ir_PHI_2(IR_ADDR, ref2, ref);
1795 }
1796
1797 if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
1798 ir_IF_TRUE(if_may_not_leak);
1799 ir_END_list(end_inputs);
1800 ir_IF_FALSE(if_may_not_leak);
1801
1802 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
1803 }
1804
1805 if (end_inputs) {
1806 ir_END_list(end_inputs);
1807 ir_MERGE_list(end_inputs);
1808 }
1809 }
1810 }
1811
jit_FREE_OP(zend_jit_ctx * jit,uint8_t op_type,znode_op op,uint32_t op_info,const zend_op * opline)1812 static void jit_FREE_OP(zend_jit_ctx *jit,
1813 uint8_t op_type,
1814 znode_op op,
1815 uint32_t op_info,
1816 const zend_op *opline)
1817 {
1818 if (op_type & (IS_VAR|IS_TMP_VAR)) {
1819 jit_ZVAL_PTR_DTOR(jit,
1820 ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var),
1821 op_info, 0, opline);
1822 }
1823 }
1824
jit_OBJ_RELEASE(zend_jit_ctx * jit,ir_ref ref)1825 static void jit_OBJ_RELEASE(zend_jit_ctx *jit, ir_ref ref)
1826 {
1827 ir_ref end_inputs = IR_UNUSED;
1828 ir_ref if_not_zero, if_may_not_leak;
1829
1830 // JIT: if (GC_DELREF(obj) == 0) {
1831 if_not_zero = ir_IF(jit_GC_DELREF(jit, ref));
1832 ir_IF_FALSE(if_not_zero);
1833
1834 // JIT: zend_objects_store_del(obj)
1835 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_objects_store_del), ref);
1836 ir_END_list(end_inputs);
1837
1838 ir_IF_TRUE(if_not_zero);
1839 if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
1840
1841 ir_IF_TRUE(if_may_not_leak);
1842 ir_END_list(end_inputs);
1843
1844 ir_IF_FALSE(if_may_not_leak);
1845 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
1846 ir_END_list(end_inputs);
1847
1848 ir_MERGE_list(end_inputs);
1849 }
1850
zend_jit_check_timeout(zend_jit_ctx * jit,const zend_op * opline,const void * exit_addr)1851 static int zend_jit_check_timeout(zend_jit_ctx *jit, const zend_op *opline, const void *exit_addr)
1852 {
1853 ir_ref ref = ir_LOAD_U8(jit_EG(vm_interrupt));
1854
1855 if (exit_addr) {
1856 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
1857 } else if (!opline || jit->last_valid_opline == opline) {
1858 ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_interrupt_handler));
1859 } else {
1860 ir_ref if_timeout = ir_IF(ref);
1861
1862 ir_IF_TRUE_cold(if_timeout);
1863 jit_LOAD_IP_ADDR(jit, opline);
1864 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_interrupt_handler));
1865 ir_IF_FALSE(if_timeout);
1866 }
1867 return 1;
1868 }
1869
1870 /* stubs */
1871
zend_jit_exception_handler_stub(zend_jit_ctx * jit)1872 static int zend_jit_exception_handler_stub(zend_jit_ctx *jit)
1873 {
1874 const void *handler;
1875
1876 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
1877 handler = zend_get_opcode_handler_func(EG(exception_op));
1878
1879 ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
1880 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
1881 } else {
1882 handler = EG(exception_op)->handler;
1883
1884 if (GCC_GLOBAL_REGS) {
1885 ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
1886 } else {
1887 ir_ref ref, if_negative;
1888
1889 ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), jit_FP(jit));
1890 if_negative = ir_IF(ir_LT(ref, ir_CONST_U32(0)));
1891 ir_IF_TRUE(if_negative);
1892 ir_MERGE_WITH_EMPTY_FALSE(if_negative);
1893 ref = ir_PHI_2(IR_I32, ref, ir_CONST_I32(1));
1894 ir_RETURN(ref);
1895 }
1896 }
1897 return 1;
1898 }
1899
zend_jit_exception_handler_undef_stub(zend_jit_ctx * jit)1900 static int zend_jit_exception_handler_undef_stub(zend_jit_ctx *jit)
1901 {
1902 ir_ref ref, result_type, if_result_used;
1903
1904 ref = jit_EG(opline_before_exception);
1905 result_type = ir_LOAD_U8(ir_ADD_OFFSET(ir_LOAD_A(ref), offsetof(zend_op, result_type)));
1906
1907 if_result_used = ir_IF(ir_AND_U8(result_type, ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
1908 ir_IF_TRUE(if_result_used);
1909
1910 ref = ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(ref), offsetof(zend_op, result.var)));
1911 if (sizeof(void*) == 8) {
1912 ref = ir_ZEXT_A(ref);
1913 }
1914 ir_STORE(ir_ADD_OFFSET(ir_ADD_A(jit_FP(jit), ref), offsetof(zval, u1.type_info)), ir_CONST_U32(IS_UNDEF));
1915 ir_MERGE_WITH_EMPTY_FALSE(if_result_used);
1916
1917 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
1918
1919 return 1;
1920 }
1921
zend_jit_exception_handler_free_op1_op2_stub(zend_jit_ctx * jit)1922 static int zend_jit_exception_handler_free_op1_op2_stub(zend_jit_ctx *jit)
1923 {
1924 ir_ref ref, if_dtor;
1925 zend_jit_addr var_addr;
1926
1927 ref = ir_LOAD_A(jit_EG(opline_before_exception));
1928 if_dtor = ir_IF(ir_AND_U8(ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zend_op, op1_type))),
1929 ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
1930 ir_IF_TRUE(if_dtor);
1931 ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op1.var)));
1932 if (sizeof(void*) == 8) {
1933 ref = ir_ZEXT_A(ref);
1934 }
1935 ref = ir_ADD_A(jit_FP(jit), ref);
1936 var_addr = ZEND_ADDR_REF_ZVAL(ref);
1937 jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
1938 ir_MERGE_WITH_EMPTY_FALSE(if_dtor);
1939
1940 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
1941
1942 return 1;
1943 }
1944
zend_jit_exception_handler_free_op2_stub(zend_jit_ctx * jit)1945 static int zend_jit_exception_handler_free_op2_stub(zend_jit_ctx *jit)
1946 {
1947 ir_ref ref, if_dtor;
1948 zend_jit_addr var_addr;
1949
1950 ref = ir_LOAD_A(jit_EG(opline_before_exception));
1951 if_dtor = ir_IF(ir_AND_U8(ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zend_op, op2_type))),
1952 ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
1953 ir_IF_TRUE(if_dtor);
1954 ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op2.var)));
1955 if (sizeof(void*) == 8) {
1956 ref = ir_ZEXT_A(ref);
1957 }
1958 ref = ir_ADD_A(jit_FP(jit), ref);
1959 var_addr = ZEND_ADDR_REF_ZVAL(ref);
1960 jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
1961 ir_MERGE_WITH_EMPTY_FALSE(if_dtor);
1962
1963 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
1964
1965 return 1;
1966 }
1967
zend_jit_interrupt_handler_stub(zend_jit_ctx * jit)1968 static int zend_jit_interrupt_handler_stub(zend_jit_ctx *jit)
1969 {
1970 ir_ref if_timeout, if_exception;
1971
1972 if (GCC_GLOBAL_REGS) {
1973 // EX(opline) = opline
1974 ir_STORE(jit_EX(opline), jit_IP(jit));
1975 }
1976
1977 ir_STORE(jit_EG(vm_interrupt), ir_CONST_U8(0));
1978 if_timeout = ir_IF(ir_EQ(ir_LOAD_U8(jit_EG(timed_out)), ir_CONST_U8(0)));
1979 ir_IF_FALSE(if_timeout);
1980 ir_CALL(IR_VOID, ir_CONST_FUNC(zend_timeout));
1981 ir_MERGE_WITH_EMPTY_TRUE(if_timeout);
1982
1983 if (zend_interrupt_function) {
1984 ir_CALL_1(IR_VOID, ir_CONST_FUNC(zend_interrupt_function), jit_FP(jit));
1985 if_exception = ir_IF(ir_LOAD_A(jit_EG(exception)));
1986 ir_IF_TRUE(if_exception);
1987 ir_CALL(IR_VOID, ir_CONST_FUNC(zend_jit_exception_in_interrupt_handler_helper));
1988 ir_MERGE_WITH_EMPTY_FALSE(if_exception);
1989
1990 jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data)));
1991 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
1992 }
1993
1994 if (GCC_GLOBAL_REGS) {
1995 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
1996 } else {
1997 ir_RETURN(ir_CONST_I32(1));
1998 }
1999 return 1;
2000 }
2001
zend_jit_leave_function_handler_stub(zend_jit_ctx * jit)2002 static int zend_jit_leave_function_handler_stub(zend_jit_ctx *jit)
2003 {
2004 ir_ref call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
2005 ir_ref if_top = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_TOP)));
2006
2007 ir_IF_FALSE(if_top);
2008
2009 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
2010 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info);
2011 jit_STORE_IP(jit,
2012 ir_LOAD_A(jit_EX(opline)));
2013 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
2014 } else if (GCC_GLOBAL_REGS) {
2015 ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info);
2016 } else {
2017 ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info, jit_FP(jit));
2018 }
2019
2020 ir_IF_TRUE(if_top);
2021
2022 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
2023 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info);
2024 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
2025 } else if (GCC_GLOBAL_REGS) {
2026 ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info);
2027 } else {
2028 ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info, jit_FP(jit));
2029 }
2030
2031 return 1;
2032 }
2033
zend_jit_negative_shift_stub(zend_jit_ctx * jit)2034 static int zend_jit_negative_shift_stub(zend_jit_ctx *jit)
2035 {
2036 ir_CALL_2(IR_VOID,
2037 ir_CONST_FUNC_PROTO(zend_throw_error,
2038 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
2039 ir_CONST_ADDR(zend_ce_arithmetic_error),
2040 ir_CONST_ADDR("Bit shift by negative number"));
2041 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2));
2042 return 1;
2043 }
2044
zend_jit_mod_by_zero_stub(zend_jit_ctx * jit)2045 static int zend_jit_mod_by_zero_stub(zend_jit_ctx *jit)
2046 {
2047 ir_CALL_2(IR_VOID,
2048 ir_CONST_FUNC_PROTO(zend_throw_error,
2049 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
2050 ir_CONST_ADDR(zend_ce_division_by_zero_error),
2051 ir_CONST_ADDR("Modulo by zero"));
2052 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2));
2053 return 1;
2054 }
2055
zend_jit_invalid_this_stub(zend_jit_ctx * jit)2056 static int zend_jit_invalid_this_stub(zend_jit_ctx *jit)
2057 {
2058 ir_CALL_2(IR_VOID,
2059 ir_CONST_FUNC_PROTO(zend_throw_error,
2060 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
2061 IR_NULL,
2062 ir_CONST_ADDR("Using $this when not in object context"));
2063 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
2064 return 1;
2065 }
2066
zend_jit_undefined_function_stub(zend_jit_ctx * jit)2067 static int zend_jit_undefined_function_stub(zend_jit_ctx *jit)
2068 {
2069 // JIT: load EX(opline)
2070 ir_ref ref = ir_LOAD_A(jit_FP(jit));
2071 ir_ref arg3 = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op2.constant)));
2072
2073 if (sizeof(void*) == 8) {
2074 arg3 = ir_LOAD_A(ir_ADD_A(ref, ir_SEXT_A(arg3)));
2075 } else {
2076 arg3 = ir_LOAD_A(arg3);
2077 }
2078 arg3 = ir_ADD_OFFSET(arg3, offsetof(zend_string, val));
2079
2080 ir_CALL_3(IR_VOID,
2081 ir_CONST_FUNC_PROTO(zend_throw_error,
2082 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
2083 IR_NULL,
2084 ir_CONST_ADDR("Call to undefined function %s()"),
2085 arg3);
2086
2087 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
2088
2089 return 1;
2090 }
2091
zend_jit_throw_cannot_pass_by_ref_stub(zend_jit_ctx * jit)2092 static int zend_jit_throw_cannot_pass_by_ref_stub(zend_jit_ctx *jit)
2093 {
2094 ir_ref opline, ref, rx, if_eq, if_tmp;
2095
2096 // JIT: opline = EX(opline)
2097 opline = ir_LOAD_A(jit_FP(jit));
2098
2099 // JIT: ZVAL_UNDEF(ZEND_CALL_VAR(RX, opline->result.var))
2100 ref = ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, result.var)));
2101 if (sizeof(void*) == 8) {
2102 ref = ir_ZEXT_A(ref);
2103 }
2104 rx = jit_IP(jit);
2105 jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(rx, ref), ir_CONST_U32(IS_UNDEF));
2106
2107 // last EX(call) frame may be delayed
2108 // JIT: if (EX(call) == RX)
2109 ref = ir_LOAD_A(jit_EX(call));
2110 if_eq = ir_IF(ir_EQ(rx, ref));
2111 ir_IF_FALSE(if_eq);
2112
2113 // JIT: RX->prev_execute_data == EX(call)
2114 ir_STORE(jit_CALL(rx, prev_execute_data), ref);
2115
2116 // JIT: EX(call) = RX
2117 ir_STORE(jit_EX(call), rx);
2118 ir_MERGE_WITH_EMPTY_TRUE(if_eq);
2119
2120 // JIT: IP = opline
2121 jit_STORE_IP(jit, opline);
2122
2123 // JIT: zend_cannot_pass_by_reference(opline->op2.num)
2124 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_cannot_pass_by_reference),
2125 ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, op2.num))));
2126
2127 // JIT: if (IP->op1_type == IS_TMP_VAR)
2128 ref = ir_LOAD_U8(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1_type)));
2129 if_tmp = ir_IF(ir_EQ(ref, ir_CONST_U8(IS_TMP_VAR)));
2130 ir_IF_TRUE(if_tmp);
2131
2132 // JIT: zval_ptr_dtor(EX_VAR(IP->op1.var))
2133 ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1.var)));
2134 if (sizeof(void*) == 8) {
2135 ref = ir_ZEXT_A(ref);
2136 }
2137 ref = ir_ADD_A(jit_FP(jit), ref);
2138 jit_ZVAL_PTR_DTOR(jit,
2139 ZEND_ADDR_REF_ZVAL(ref),
2140 MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
2141 ir_MERGE_WITH_EMPTY_FALSE(if_tmp);
2142
2143 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
2144
2145 return 1;
2146 }
2147
zend_jit_icall_throw_stub(zend_jit_ctx * jit)2148 static int zend_jit_icall_throw_stub(zend_jit_ctx *jit)
2149 {
2150 ir_ref ip, if_set;
2151
2152 // JIT: zend_rethrow_exception(zend_execute_data *execute_data)
2153 // JIT: if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) {
2154 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
2155 ip = jit_IP(jit);
2156 if_set = ir_IF(ir_EQ(ir_LOAD_U8(ir_ADD_OFFSET(ip, offsetof(zend_op, opcode))),
2157 ir_CONST_U8(ZEND_HANDLE_EXCEPTION)));
2158 ir_IF_FALSE(if_set);
2159
2160 // JIT: EG(opline_before_exception) = opline;
2161 ir_STORE(jit_EG(opline_before_exception), ip);
2162 ir_MERGE_WITH_EMPTY_TRUE(if_set);
2163
2164 // JIT: opline = EG(exception_op);
2165 jit_STORE_IP(jit, jit_EG(exception_op));
2166
2167 if (GCC_GLOBAL_REGS) {
2168 ir_STORE(jit_EX(opline), jit_IP(jit));
2169 }
2170
2171 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
2172
2173 return 1;
2174 }
2175
zend_jit_leave_throw_stub(zend_jit_ctx * jit)2176 static int zend_jit_leave_throw_stub(zend_jit_ctx *jit)
2177 {
2178 ir_ref ip, if_set;
2179
2180 // JIT: if (opline->opcode != ZEND_HANDLE_EXCEPTION) {
2181 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
2182 ip = jit_IP(jit);
2183 if_set = ir_IF(ir_EQ(ir_LOAD_U8(ir_ADD_OFFSET(ip, offsetof(zend_op, opcode))),
2184 ir_CONST_U8(ZEND_HANDLE_EXCEPTION)));
2185 ir_IF_FALSE(if_set);
2186
2187 // JIT: EG(opline_before_exception) = opline;
2188 ir_STORE(jit_EG(opline_before_exception), ip);
2189 ir_MERGE_WITH_EMPTY_TRUE(if_set);
2190
2191 // JIT: opline = EG(exception_op);
2192 jit_LOAD_IP(jit, jit_EG(exception_op));
2193
2194 if (GCC_GLOBAL_REGS) {
2195 ir_STORE(jit_EX(opline), jit_IP(jit));
2196
2197 // JIT: HANDLE_EXCEPTION()
2198 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
2199 } else {
2200 ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
2201 }
2202
2203 return 1;
2204 }
2205
zend_jit_hybrid_runtime_jit_stub(zend_jit_ctx * jit)2206 static int zend_jit_hybrid_runtime_jit_stub(zend_jit_ctx *jit)
2207 {
2208 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
2209 return 0;
2210 }
2211
2212 ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_runtime_jit));
2213 ir_IJMP(ir_LOAD_A(jit_IP(jit)));
2214 return 1;
2215 }
2216
zend_jit_hybrid_profile_jit_stub(zend_jit_ctx * jit)2217 static int zend_jit_hybrid_profile_jit_stub(zend_jit_ctx *jit)
2218 {
2219 ir_ref addr, func, run_time_cache, jit_extension;
2220
2221 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
2222 return 0;
2223 }
2224
2225 addr = ir_CONST_ADDR(&zend_jit_profile_counter),
2226 ir_STORE(addr, ir_ADD_L(ir_LOAD_L(addr), ir_CONST_LONG(1)));
2227
2228 func = ir_LOAD_A(jit_EX(func));
2229 run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
2230 jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
2231
2232 if (zend_jit_profile_counter_rid) {
2233 addr = ir_ADD_OFFSET(run_time_cache, zend_jit_profile_counter_rid * sizeof(void*));
2234 } else {
2235 addr = run_time_cache;
2236 }
2237 ir_STORE(addr, ir_ADD_L(ir_LOAD_L(addr), ir_CONST_LONG(1)));
2238
2239 addr = ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_extension, orig_handler));
2240 ir_IJMP(ir_LOAD_A(addr));
2241
2242 return 1;
2243 }
2244
_zend_jit_hybrid_hot_counter_stub(zend_jit_ctx * jit,uint32_t cost)2245 static int _zend_jit_hybrid_hot_counter_stub(zend_jit_ctx *jit, uint32_t cost)
2246 {
2247 ir_ref func, jit_extension, addr, ref, if_overflow;
2248
2249 func = ir_LOAD_A(jit_EX(func));
2250 jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
2251 addr = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_hot_extension, counter)));
2252 ref = ir_SUB_I16(ir_LOAD_I16(addr), ir_CONST_I16(cost));
2253 ir_STORE(addr, ref);
2254 if_overflow = ir_IF(ir_LE(ref, ir_CONST_I16(0)));
2255
2256 ir_IF_TRUE_cold(if_overflow);
2257 ir_STORE(addr, ir_CONST_I16(ZEND_JIT_COUNTER_INIT));
2258 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_hot_func),
2259 jit_FP(jit),
2260 jit_IP(jit));
2261 ir_IJMP(ir_LOAD_A(jit_IP(jit)));
2262
2263 ir_IF_FALSE(if_overflow);
2264 ref = ir_SUB_A(jit_IP(jit),
2265 ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, opcodes))));
2266 ref = ir_DIV_A(ref, ir_CONST_ADDR(sizeof(zend_op) / sizeof(void*)));
2267
2268 addr = ir_ADD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_hot_extension, orig_handlers)),
2269 ref);
2270 ir_IJMP(ir_LOAD_A(addr));
2271
2272 return 1;
2273 }
2274
zend_jit_hybrid_func_hot_counter_stub(zend_jit_ctx * jit)2275 static int zend_jit_hybrid_func_hot_counter_stub(zend_jit_ctx *jit)
2276 {
2277 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
2278 return 0;
2279 }
2280
2281 return _zend_jit_hybrid_hot_counter_stub(jit,
2282 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
2283 }
2284
zend_jit_hybrid_loop_hot_counter_stub(zend_jit_ctx * jit)2285 static int zend_jit_hybrid_loop_hot_counter_stub(zend_jit_ctx *jit)
2286 {
2287 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
2288 return 0;
2289 }
2290
2291 return _zend_jit_hybrid_hot_counter_stub(jit,
2292 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
2293 }
2294
_zend_jit_orig_opline_handler(zend_jit_ctx * jit,ir_ref offset)2295 static ir_ref _zend_jit_orig_opline_handler(zend_jit_ctx *jit, ir_ref offset)
2296 {
2297 ir_ref addr;
2298
2299 if (GCC_GLOBAL_REGS) {
2300 addr = ir_ADD_A(offset, jit_IP(jit));
2301 } else {
2302 addr = ir_ADD_A(offset, ir_LOAD_A(jit_EX(opline)));
2303 }
2304
2305 return ir_LOAD_A(addr);
2306 }
2307
zend_jit_orig_opline_handler(zend_jit_ctx * jit)2308 static ir_ref zend_jit_orig_opline_handler(zend_jit_ctx *jit)
2309 {
2310 ir_ref func, jit_extension, offset;
2311
2312 func = ir_LOAD_A(jit_EX(func));
2313 jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
2314 offset = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_trace_extension, offset)));
2315 return _zend_jit_orig_opline_handler(jit, offset);
2316 }
2317
_zend_jit_hybrid_trace_counter_stub(zend_jit_ctx * jit,uint32_t cost)2318 static int _zend_jit_hybrid_trace_counter_stub(zend_jit_ctx *jit, uint32_t cost)
2319 {
2320 ir_ref func, jit_extension, offset, addr, ref, if_overflow, ret, if_halt;
2321
2322 func = ir_LOAD_A(jit_EX(func));
2323 jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
2324 offset = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_trace_extension, offset)));
2325 addr = ir_LOAD_A(ir_ADD_OFFSET(ir_ADD_A(offset, jit_IP(jit)), offsetof(zend_op_trace_info, counter)));
2326 ref = ir_SUB_I16(ir_LOAD_I16(addr), ir_CONST_I16(cost));
2327 ir_STORE(addr, ref);
2328 if_overflow = ir_IF(ir_LE(ref, ir_CONST_I16(0)));
2329
2330 ir_IF_TRUE_cold(if_overflow);
2331 ir_STORE(addr, ir_CONST_I16(ZEND_JIT_COUNTER_INIT));
2332 ret = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_trace_hot_root),
2333 jit_FP(jit),
2334 jit_IP(jit));
2335 if_halt = ir_IF(ir_LT(ret, ir_CONST_I32(0)));
2336 ir_IF_FALSE(if_halt);
2337
2338 ref = jit_EG(current_execute_data);
2339 jit_STORE_FP(jit, ir_LOAD_A(ref));
2340 ref = ir_LOAD_A(jit_EX(opline));
2341 jit_STORE_IP(jit, ref);
2342 ir_IJMP(ir_LOAD_A(jit_IP(jit)));
2343
2344 ir_IF_FALSE(if_overflow);
2345 ir_IJMP(_zend_jit_orig_opline_handler(jit, offset));
2346
2347 ir_IF_TRUE(if_halt);
2348 ir_IJMP(ir_CONST_FC_FUNC(zend_jit_halt_op->handler));
2349
2350 return 1;
2351 }
2352
zend_jit_hybrid_func_trace_counter_stub(zend_jit_ctx * jit)2353 static int zend_jit_hybrid_func_trace_counter_stub(zend_jit_ctx *jit)
2354 {
2355 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
2356 return 0;
2357 }
2358
2359 return _zend_jit_hybrid_trace_counter_stub(jit,
2360 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
2361 }
2362
zend_jit_hybrid_ret_trace_counter_stub(zend_jit_ctx * jit)2363 static int zend_jit_hybrid_ret_trace_counter_stub(zend_jit_ctx *jit)
2364 {
2365 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) {
2366 return 0;
2367 }
2368
2369 return _zend_jit_hybrid_trace_counter_stub(jit,
2370 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
2371 }
2372
zend_jit_hybrid_loop_trace_counter_stub(zend_jit_ctx * jit)2373 static int zend_jit_hybrid_loop_trace_counter_stub(zend_jit_ctx *jit)
2374 {
2375 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
2376 return 0;
2377 }
2378
2379 return _zend_jit_hybrid_trace_counter_stub(jit,
2380 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
2381 }
2382
zend_jit_trace_halt_stub(zend_jit_ctx * jit)2383 static int zend_jit_trace_halt_stub(zend_jit_ctx *jit)
2384 {
2385 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
2386 ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_halt_op->handler));
2387 } else if (GCC_GLOBAL_REGS) {
2388 jit_STORE_IP(jit, IR_NULL);
2389 ir_RETURN(IR_VOID);
2390 } else {
2391 ir_RETURN(ir_CONST_I32(-1)); // ZEND_VM_RETURN
2392 }
2393 return 1;
2394 }
2395
zend_jit_trace_escape_stub(zend_jit_ctx * jit)2396 static int zend_jit_trace_escape_stub(zend_jit_ctx *jit)
2397 {
2398 if (GCC_GLOBAL_REGS) {
2399 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
2400 } else {
2401 ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
2402 }
2403
2404 return 1;
2405 }
2406
zend_jit_trace_exit_stub(zend_jit_ctx * jit)2407 static int zend_jit_trace_exit_stub(zend_jit_ctx *jit)
2408 {
2409 ir_ref ref, ret, if_zero, addr;
2410
2411 if (GCC_GLOBAL_REGS) {
2412 // EX(opline) = opline
2413 ir_STORE(jit_EX(opline), jit_IP(jit));
2414 }
2415
2416 ret = ir_EXITCALL(ir_CONST_FC_FUNC(zend_jit_trace_exit));
2417
2418 if_zero = ir_IF(ir_EQ(ret, ir_CONST_I32(0)));
2419
2420 ir_IF_TRUE(if_zero);
2421
2422 if (GCC_GLOBAL_REGS) {
2423 ref = jit_EG(current_execute_data);
2424 jit_STORE_FP(jit, ir_LOAD_A(ref));
2425 ref = ir_LOAD_A(jit_EX(opline));
2426 jit_STORE_IP(jit, ref);
2427 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
2428 } else {
2429 ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
2430 }
2431
2432 ir_IF_FALSE(if_zero);
2433
2434 ir_GUARD(ir_GE(ret, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
2435
2436 ref = jit_EG(current_execute_data);
2437 jit_STORE_FP(jit, ir_LOAD_A(ref));
2438
2439 if (GCC_GLOBAL_REGS) {
2440 ref = ir_LOAD_A(jit_EX(opline));
2441 jit_STORE_IP(jit, ref);
2442 }
2443
2444 // check for interrupt (try to avoid this ???)
2445 if (!zend_jit_check_timeout(jit, NULL, NULL)) {
2446 return 0;
2447 }
2448
2449 addr = zend_jit_orig_opline_handler(jit);
2450 if (GCC_GLOBAL_REGS) {
2451 ir_TAILCALL(IR_VOID, addr);
2452 } else {
2453 #if defined(IR_TARGET_X86)
2454 addr = ir_CAST_FC_FUNC(addr);
2455 #endif
2456 ref = ir_CALL_1(IR_I32, addr, jit_FP(jit));
2457 ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
2458 ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
2459 }
2460
2461 return 1;
2462 }
2463
zend_jit_undefined_offset_stub(zend_jit_ctx * jit)2464 static int zend_jit_undefined_offset_stub(zend_jit_ctx *jit)
2465 {
2466 if (GCC_GLOBAL_REGS) {
2467 ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key));
2468 } else {
2469 ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key), jit_FP(jit));
2470 }
2471
2472 return 1;
2473 }
2474
zend_jit_undefined_key_stub(zend_jit_ctx * jit)2475 static int zend_jit_undefined_key_stub(zend_jit_ctx *jit)
2476 {
2477 if (GCC_GLOBAL_REGS) {
2478 ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key));
2479 } else {
2480 ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key), jit_FP(jit));
2481 }
2482
2483 return 1;
2484 }
2485
zend_jit_cannot_add_element_stub(zend_jit_ctx * jit)2486 static int zend_jit_cannot_add_element_stub(zend_jit_ctx *jit)
2487 {
2488 ir_ref opline = ir_LOAD_A(jit_EX(opline));
2489 ir_ref ref, if_result_used;
2490
2491 if_result_used = ir_IF(ir_AND_U8(
2492 ir_LOAD_U8(ir_ADD_OFFSET(opline, offsetof(zend_op, result_type))),
2493 ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
2494 ir_IF_TRUE(if_result_used);
2495
2496 ref = ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, result.var)));
2497 if (sizeof(void*) == 8) {
2498 ref = ir_ZEXT_A(ref);
2499 }
2500 jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(jit_FP(jit), ref), ir_CONST_U32(IS_UNDEF));
2501 ir_MERGE_WITH_EMPTY_FALSE(if_result_used);
2502
2503 ir_CALL_2(IR_VOID,
2504 ir_CONST_FUNC_PROTO(zend_throw_error,
2505 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
2506 IR_NULL,
2507 ir_CONST_ADDR("Cannot add element to the array as the next element is already occupied"));
2508 ir_RETURN(IR_VOID);
2509
2510 return 1;
2511 }
2512
zend_jit_assign_const_stub(zend_jit_ctx * jit)2513 static int zend_jit_assign_const_stub(zend_jit_ctx *jit)
2514 {
2515 ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
2516 ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
2517
2518 zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
2519 zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
2520 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
2521
2522 if (!zend_jit_assign_to_variable(
2523 jit, NULL,
2524 var_addr, var_addr, -1, -1,
2525 IS_CONST, val_addr, val_info,
2526 0, 0, 0)) {
2527 return 0;
2528 }
2529 ir_RETURN(IR_VOID);
2530 return 1;
2531 }
2532
zend_jit_assign_tmp_stub(zend_jit_ctx * jit)2533 static int zend_jit_assign_tmp_stub(zend_jit_ctx *jit)
2534 {
2535 ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
2536 ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
2537
2538 zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
2539 zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
2540 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
2541
2542 if (!zend_jit_assign_to_variable(
2543 jit, NULL,
2544 var_addr, var_addr, -1, -1,
2545 IS_TMP_VAR, val_addr, val_info,
2546 0, 0, 0)) {
2547 return 0;
2548 }
2549 ir_RETURN(IR_VOID);
2550 return 1;
2551 }
2552
zend_jit_assign_var_stub(zend_jit_ctx * jit)2553 static int zend_jit_assign_var_stub(zend_jit_ctx *jit)
2554 {
2555 ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
2556 ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
2557
2558 zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
2559 zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
2560 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF;
2561
2562 if (!zend_jit_assign_to_variable(
2563 jit, NULL,
2564 var_addr, var_addr, -1, -1,
2565 IS_VAR, val_addr, val_info,
2566 0, 0, 0)) {
2567 return 0;
2568 }
2569 ir_RETURN(IR_VOID);
2570 return 1;
2571 }
2572
zend_jit_assign_cv_noref_stub(zend_jit_ctx * jit)2573 static int zend_jit_assign_cv_noref_stub(zend_jit_ctx *jit)
2574 {
2575 ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
2576 ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
2577
2578 zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
2579 zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
2580 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/;
2581
2582 if (!zend_jit_assign_to_variable(
2583 jit, NULL,
2584 var_addr, var_addr, -1, -1,
2585 IS_CV, val_addr, val_info,
2586 0, 0, 0)) {
2587 return 0;
2588 }
2589 ir_RETURN(IR_VOID);
2590 return 1;
2591 }
2592
zend_jit_new_array_stub(zend_jit_ctx * jit)2593 static int zend_jit_new_array_stub(zend_jit_ctx *jit)
2594 {
2595 ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
2596 zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
2597 ir_ref ref = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(_zend_new_array_0));
2598
2599 jit_set_Z_PTR(jit, var_addr, ref);
2600 jit_set_Z_TYPE_INFO(jit, var_addr, IS_ARRAY_EX);
2601 ir_RETURN(ref);
2602 return 1;
2603 }
2604
zend_jit_assign_cv_stub(zend_jit_ctx * jit)2605 static int zend_jit_assign_cv_stub(zend_jit_ctx *jit)
2606 {
2607 ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
2608 ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
2609
2610 zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
2611 zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
2612 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/;
2613
2614 if (!zend_jit_assign_to_variable(
2615 jit, NULL,
2616 var_addr, var_addr, -1, -1,
2617 IS_CV, val_addr, val_info,
2618 0, 0, 0)) {
2619 return 0;
2620 }
2621 ir_RETURN(IR_VOID);
2622 return 1;
2623 }
2624
zend_jit_init_ctx(zend_jit_ctx * jit,uint32_t flags)2625 static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags)
2626 {
2627 #if defined (__CET__) && (__CET__ & 1) != 0
2628 flags |= IR_GEN_ENDBR;
2629 #endif
2630 flags |= IR_OPT_FOLDING | IR_OPT_CFG | IR_OPT_CODEGEN;
2631
2632 ir_init(&jit->ctx, flags, 256, 1024);
2633 jit->ctx.ret_type = -1;
2634
2635 #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
2636 jit->ctx.mflags |= default_mflags;
2637 if (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) {
2638 jit->ctx.mflags |= IR_X86_AVX;
2639 }
2640 #elif defined(IR_TARGET_AARCH64)
2641 jit->ctx.get_veneer = zend_jit_get_veneer;
2642 jit->ctx.set_veneer = zend_jit_set_veneer;
2643 #endif
2644
2645 jit->ctx.fixed_regset = (1<<ZREG_FP) | (1<<ZREG_IP);
2646 if (!(flags & IR_FUNCTION)) {
2647 jit->ctx.flags |= IR_NO_STACK_COMBINE;
2648 if (zend_jit_vm_kind == ZEND_VM_KIND_CALL) {
2649 jit->ctx.flags |= IR_FUNCTION;
2650 /* Stack must be 16 byte aligned */
2651 /* TODO: select stack size ??? */
2652 #if defined(IR_TARGET_AARCH64)
2653 jit->ctx.flags |= IR_USE_FRAME_POINTER;
2654 jit->ctx.fixed_stack_frame_size = sizeof(void*) * 16; /* 10 saved registers and 6 spill slots (8 bytes) */
2655 #elif defined(_WIN64)
2656 jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 8 saved registers and 3 spill slots (8 bytes) */
2657 #elif defined(IR_TARGET_X86_64)
2658 jit->ctx.fixed_stack_frame_size = sizeof(void*) * 9; /* 6 saved registers and 3 spill slots (8 bytes) */
2659 #else /* IR_TARGET_x86 */
2660 jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 4 saved registers and 7 spill slots (4 bytes) */
2661 #endif
2662 if (GCC_GLOBAL_REGS) {
2663 jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED & ~((1<<ZREG_FP) | (1<<ZREG_IP));
2664 } else {
2665 jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED;
2666 //#ifdef _WIN64
2667 // jit->ctx.fixed_save_regset &= 0xffff; // TODO: don't save FP registers ???
2668 //#endif
2669 }
2670 jit->ctx.fixed_call_stack_size = 16;
2671 } else {
2672 #ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
2673 jit->ctx.fixed_stack_red_zone = ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE;
2674 if (jit->ctx.fixed_stack_red_zone > 16) {
2675 jit->ctx.fixed_stack_frame_size = jit->ctx.fixed_stack_red_zone - 16;
2676 jit->ctx.fixed_call_stack_size = 16;
2677 }
2678 jit->ctx.flags |= IR_MERGE_EMPTY_ENTRIES;
2679 #else
2680 jit->ctx.fixed_stack_red_zone = 0;
2681 jit->ctx.fixed_stack_frame_size = 32; /* 4 spill slots (8 bytes) or 8 spill slots (4 bytes) */
2682 jit->ctx.fixed_call_stack_size = 16;
2683 #endif
2684 #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
2685 jit->ctx.fixed_regset |= (1<<IR_REG_FP); /* prevent %rbp (%r5) usage */
2686 #endif
2687 }
2688 }
2689
2690 jit->ctx.snapshot_create = (ir_snapshot_create_t)jit_SNAPSHOT;
2691
2692 jit->op_array = NULL;
2693 jit->current_op_array = NULL;
2694 jit->ssa = NULL;
2695 jit->name = NULL;
2696 jit->last_valid_opline = NULL;
2697 jit->use_last_valid_opline = 0;
2698 jit->track_last_valid_opline = 0;
2699 jit->reuse_ip = 0;
2700 jit->delayed_call_level = 0;
2701 delayed_call_chain = 0;
2702 jit->b = -1;
2703 #ifdef ZTS
2704 jit->tls = IR_UNUSED;
2705 #endif
2706 jit->fp = IR_UNUSED;
2707 jit->trace_loop_ref = IR_UNUSED;
2708 jit->return_inputs = IR_UNUSED;
2709 jit->bb_start_ref = NULL;
2710 jit->bb_predecessors = NULL;
2711 jit->bb_edges = NULL;
2712 jit->trace = NULL;
2713 jit->ra = NULL;
2714 jit->delay_var = -1;
2715 jit->delay_refs = NULL;
2716 jit->eg_exception_addr = 0;
2717 zend_hash_init(&jit->addr_hash, 64, NULL, NULL, 0);
2718 memset(jit->stub_addr, 0, sizeof(jit->stub_addr));
2719
2720 ir_START();
2721 }
2722
zend_jit_free_ctx(zend_jit_ctx * jit)2723 static int zend_jit_free_ctx(zend_jit_ctx *jit)
2724 {
2725 if (jit->name) {
2726 zend_string_release(jit->name);
2727 }
2728 zend_hash_destroy(&jit->addr_hash);
2729 ir_free(&jit->ctx);
2730 return 1;
2731 }
2732
zend_jit_ir_compile(ir_ctx * ctx,size_t * size,const char * name)2733 static void *zend_jit_ir_compile(ir_ctx *ctx, size_t *size, const char *name)
2734 {
2735 void *entry;
2736 ir_code_buffer code_buffer;
2737
2738 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_SRC) {
2739 if (name) fprintf(stderr, "%s: ; after folding\n", name);
2740 ir_save(ctx, 0, stderr);
2741 }
2742
2743 #if ZEND_DEBUG
2744 ir_check(ctx);
2745 #endif
2746
2747 ir_build_def_use_lists(ctx);
2748
2749 #if ZEND_DEBUG
2750 ir_check(ctx);
2751 #endif
2752
2753 #if 1
2754 ir_sccp(ctx);
2755 #endif
2756
2757 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_SCCP) {
2758 if (name) fprintf(stderr, "%s: ; after SCCP\n", name);
2759 ir_save(ctx, 0, stderr);
2760 }
2761
2762 ir_build_cfg(ctx);
2763 ir_build_dominators_tree(ctx);
2764 ir_find_loops(ctx);
2765
2766 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_CFG) {
2767 if (name) fprintf(stderr, "%s: ; after CFG\n", name);
2768 ir_save(ctx, IR_SAVE_CFG, stderr);
2769 }
2770
2771 ir_gcm(ctx);
2772
2773 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_GCM) {
2774 if (name) fprintf(stderr, "%s: ; after GCM\n", name);
2775 ir_save(ctx, IR_SAVE_CFG|IR_SAVE_CFG_MAP, stderr);
2776 }
2777
2778 ir_schedule(ctx);
2779
2780 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE) {
2781 if (name) fprintf(stderr, "%s: ; after schedule\n", name);
2782 ir_save(ctx, IR_SAVE_CFG, stderr);
2783 }
2784
2785 ir_match(ctx);
2786 #if !defined(IR_TARGET_AARCH64)
2787 ctx->flags &= ~IR_USE_FRAME_POINTER; /* don't use FRAME_POINTER even with ALLOCA, TODO: cleanup this ??? */
2788 #endif
2789 ir_assign_virtual_registers(ctx);
2790 ir_compute_live_ranges(ctx);
2791 ir_coalesce(ctx);
2792 ir_reg_alloc(ctx);
2793
2794 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_REGS) {
2795 if (name) fprintf(stderr, "%s: ; after register allocation\n", name);
2796 ir_save(ctx, IR_SAVE_CFG|IR_SAVE_RULES|IR_SAVE_REGS, stderr);
2797 ir_dump_live_ranges(ctx, stderr);
2798 }
2799
2800 ir_schedule_blocks(ctx);
2801
2802 if (JIT_G(debug) & (ZEND_JIT_DEBUG_IR_FINAL|ZEND_JIT_DEBUG_IR_CODEGEN)) {
2803 if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_CODEGEN) {
2804 if (name) fprintf(stderr, "%s: ; codegen\n", name);
2805 ir_dump_codegen(ctx, stderr);
2806 } else {
2807 if (name) fprintf(stderr, "%s: ; final\n", name);
2808 ir_save(ctx, IR_SAVE_CFG|IR_SAVE_RULES|IR_SAVE_REGS, stderr);
2809 }
2810 }
2811
2812 #if ZEND_DEBUG
2813 ir_check(ctx);
2814 #endif
2815
2816 code_buffer.start = dasm_buf;
2817 code_buffer.end = dasm_end;
2818 code_buffer.pos = *dasm_ptr;
2819 ctx->code_buffer = &code_buffer;
2820
2821 entry = ir_emit_code(ctx, size);
2822
2823 *dasm_ptr = code_buffer.pos;
2824
2825 #if defined(IR_TARGET_AARCH64)
2826 if (ctx->flags2 & IR_HAS_VENEERS) {
2827 zend_jit_commit_veneers();
2828 }
2829 #endif
2830
2831 return entry;
2832 }
2833
zend_jit_setup_stubs(void)2834 static void zend_jit_setup_stubs(void)
2835 {
2836 zend_jit_ctx jit;
2837 void *entry;
2838 size_t size;
2839 uint32_t i;
2840
2841 for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) {
2842 zend_jit_init_ctx(&jit, zend_jit_stubs[i].flags);
2843
2844 if (!zend_jit_stubs[i].stub(&jit)) {
2845 zend_jit_free_ctx(&jit);
2846 zend_jit_stub_handlers[i] = NULL;
2847 continue;
2848 }
2849
2850 entry = zend_jit_ir_compile(&jit.ctx, &size, zend_jit_stubs[i].name);
2851 if (!entry) {
2852 zend_jit_free_ctx(&jit);
2853 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: could not compile stub");
2854 }
2855
2856 zend_jit_stub_handlers[i] = entry;
2857
2858 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
2859 #ifdef HAVE_CAPSTONE
2860 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
2861 ir_disasm_add_symbol(zend_jit_stubs[i].name, (uintptr_t)entry, size);
2862 }
2863 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_STUBS) {
2864 ir_disasm(zend_jit_stubs[i].name,
2865 entry, size, (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0, &jit.ctx, stderr);
2866 }
2867 #endif
2868 #ifndef _WIN32
2869 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
2870 // ir_mem_unprotect(entry, size);
2871 ir_gdb_register(zend_jit_stubs[i].name, entry, size, 0, 0);
2872 // ir_mem_protect(entry, size);
2873 }
2874
2875 if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
2876 ir_perf_map_register(zend_jit_stubs[i].name, entry, size);
2877 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
2878 ir_perf_jitdump_register(zend_jit_stubs[i].name, entry, size);
2879 }
2880 }
2881 #endif
2882 }
2883 zend_jit_free_ctx(&jit);
2884 }
2885 }
2886
2887 #define REGISTER_HELPER(n) \
2888 ir_disasm_add_symbol(#n, (uint64_t)(uintptr_t)n, sizeof(void*));
2889 #define REGISTER_DATA(n) \
2890 ir_disasm_add_symbol(#n, (uint64_t)(uintptr_t)&n, sizeof(n));
2891
zend_jit_setup_disasm(void)2892 static void zend_jit_setup_disasm(void)
2893 {
2894 #ifdef HAVE_CAPSTONE
2895 ir_disasm_init();
2896
2897 if (zend_vm_kind() == ZEND_VM_KIND_HYBRID) {
2898 zend_op opline;
2899
2900 memset(&opline, 0, sizeof(opline));
2901
2902 opline.opcode = ZEND_DO_UCALL;
2903 opline.result_type = IS_UNUSED;
2904 zend_vm_set_opcode_handler(&opline);
2905 ir_disasm_add_symbol("ZEND_DO_UCALL_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2906
2907 opline.opcode = ZEND_DO_UCALL;
2908 opline.result_type = IS_VAR;
2909 zend_vm_set_opcode_handler(&opline);
2910 ir_disasm_add_symbol("ZEND_DO_UCALL_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2911
2912 opline.opcode = ZEND_DO_FCALL_BY_NAME;
2913 opline.result_type = IS_UNUSED;
2914 zend_vm_set_opcode_handler(&opline);
2915 ir_disasm_add_symbol("ZEND_DO_FCALL_BY_NAME_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2916
2917 opline.opcode = ZEND_DO_FCALL_BY_NAME;
2918 opline.result_type = IS_VAR;
2919 zend_vm_set_opcode_handler(&opline);
2920 ir_disasm_add_symbol("ZEND_DO_FCALL_BY_NAME_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2921
2922 opline.opcode = ZEND_DO_FCALL;
2923 opline.result_type = IS_UNUSED;
2924 zend_vm_set_opcode_handler(&opline);
2925 ir_disasm_add_symbol("ZEND_DO_FCALL_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2926
2927 opline.opcode = ZEND_DO_FCALL;
2928 opline.result_type = IS_VAR;
2929 zend_vm_set_opcode_handler(&opline);
2930 ir_disasm_add_symbol("ZEND_DO_FCALL_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2931
2932 opline.opcode = ZEND_RETURN;
2933 opline.op1_type = IS_CONST;
2934 zend_vm_set_opcode_handler(&opline);
2935 ir_disasm_add_symbol("ZEND_RETURN_SPEC_CONST_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2936
2937 opline.opcode = ZEND_RETURN;
2938 opline.op1_type = IS_TMP_VAR;
2939 zend_vm_set_opcode_handler(&opline);
2940 ir_disasm_add_symbol("ZEND_RETURN_SPEC_TMP_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2941
2942 opline.opcode = ZEND_RETURN;
2943 opline.op1_type = IS_VAR;
2944 zend_vm_set_opcode_handler(&opline);
2945 ir_disasm_add_symbol("ZEND_RETURN_SPEC_VAR_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2946
2947 opline.opcode = ZEND_RETURN;
2948 opline.op1_type = IS_CV;
2949 zend_vm_set_opcode_handler(&opline);
2950 ir_disasm_add_symbol("ZEND_RETURN_SPEC_CV_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
2951
2952 ir_disasm_add_symbol("ZEND_HYBRID_HALT_LABEL", (uint64_t)(uintptr_t)zend_jit_halt_op->handler, sizeof(void*));
2953 }
2954
2955 REGISTER_DATA(zend_jit_profile_counter);
2956
2957 REGISTER_HELPER(zend_runtime_jit);
2958 REGISTER_HELPER(zend_jit_hot_func);
2959 REGISTER_HELPER(zend_jit_trace_hot_root);
2960 REGISTER_HELPER(zend_jit_trace_exit);
2961
2962 REGISTER_HELPER(zend_jit_array_free);
2963 REGISTER_HELPER(zend_jit_undefined_op_helper);
2964 REGISTER_HELPER(zend_jit_pre_inc_typed_ref);
2965 REGISTER_HELPER(zend_jit_pre_dec_typed_ref);
2966 REGISTER_HELPER(zend_jit_post_inc_typed_ref);
2967 REGISTER_HELPER(zend_jit_post_dec_typed_ref);
2968 REGISTER_HELPER(zend_jit_pre_inc);
2969 REGISTER_HELPER(zend_jit_pre_dec);
2970 REGISTER_HELPER(zend_jit_add_arrays_helper);
2971 REGISTER_HELPER(zend_jit_fast_assign_concat_helper);
2972 REGISTER_HELPER(zend_jit_fast_concat_helper);
2973 REGISTER_HELPER(zend_jit_fast_concat_tmp_helper);
2974 REGISTER_HELPER(zend_jit_assign_op_to_typed_ref_tmp);
2975 REGISTER_HELPER(zend_jit_assign_op_to_typed_ref);
2976 REGISTER_HELPER(zend_jit_assign_const_to_typed_ref);
2977 REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref);
2978 REGISTER_HELPER(zend_jit_assign_var_to_typed_ref);
2979 REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref);
2980 REGISTER_HELPER(zend_jit_assign_const_to_typed_ref2);
2981 REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref2);
2982 REGISTER_HELPER(zend_jit_assign_var_to_typed_ref2);
2983 REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref2);
2984 REGISTER_HELPER(zend_jit_check_constant);
2985 REGISTER_HELPER(zend_jit_get_constant);
2986 REGISTER_HELPER(zend_jit_int_extend_stack_helper);
2987 REGISTER_HELPER(zend_jit_extend_stack_helper);
2988 REGISTER_HELPER(zend_jit_init_func_run_time_cache_helper);
2989 REGISTER_HELPER(zend_jit_find_func_helper);
2990 REGISTER_HELPER(zend_jit_find_ns_func_helper);
2991 REGISTER_HELPER(zend_jit_jmp_frameless_helper);
2992 REGISTER_HELPER(zend_jit_unref_helper);
2993 REGISTER_HELPER(zend_jit_invalid_method_call);
2994 REGISTER_HELPER(zend_jit_invalid_method_call_tmp);
2995 REGISTER_HELPER(zend_jit_find_method_helper);
2996 REGISTER_HELPER(zend_jit_find_method_tmp_helper);
2997 REGISTER_HELPER(zend_jit_push_static_metod_call_frame);
2998 REGISTER_HELPER(zend_jit_push_static_metod_call_frame_tmp);
2999 REGISTER_HELPER(zend_jit_free_trampoline_helper);
3000 REGISTER_HELPER(zend_jit_verify_return_slow);
3001 REGISTER_HELPER(zend_jit_deprecated_helper);
3002 REGISTER_HELPER(zend_jit_undefined_long_key);
3003 REGISTER_HELPER(zend_jit_undefined_string_key);
3004 REGISTER_HELPER(zend_jit_copy_extra_args_helper);
3005 REGISTER_HELPER(zend_jit_vm_stack_free_args_helper);
3006 REGISTER_HELPER(zend_free_extra_named_params);
3007 REGISTER_HELPER(zend_jit_free_call_frame);
3008 REGISTER_HELPER(zend_jit_exception_in_interrupt_handler_helper);
3009 REGISTER_HELPER(zend_jit_verify_arg_slow);
3010 REGISTER_HELPER(zend_missing_arg_error);
3011 REGISTER_HELPER(zend_jit_only_vars_by_reference);
3012 REGISTER_HELPER(zend_jit_leave_func_helper);
3013 REGISTER_HELPER(zend_jit_leave_nested_func_helper);
3014 REGISTER_HELPER(zend_jit_leave_top_func_helper);
3015 REGISTER_HELPER(zend_jit_fetch_global_helper);
3016 REGISTER_HELPER(zend_jit_hash_index_lookup_rw_no_packed);
3017 REGISTER_HELPER(zend_jit_hash_index_lookup_rw);
3018 REGISTER_HELPER(zend_jit_hash_lookup_rw);
3019 REGISTER_HELPER(zend_jit_symtable_find);
3020 REGISTER_HELPER(zend_jit_symtable_lookup_w);
3021 REGISTER_HELPER(zend_jit_symtable_lookup_rw);
3022 REGISTER_HELPER(zend_jit_fetch_dim_r_helper);
3023 REGISTER_HELPER(zend_jit_fetch_dim_is_helper);
3024 REGISTER_HELPER(zend_jit_fetch_dim_isset_helper);
3025 REGISTER_HELPER(zend_jit_fetch_dim_rw_helper);
3026 REGISTER_HELPER(zend_jit_fetch_dim_w_helper);
3027 REGISTER_HELPER(zend_jit_fetch_dim_str_offset_r_helper);
3028 REGISTER_HELPER(zend_jit_fetch_dim_str_r_helper);
3029 REGISTER_HELPER(zend_jit_fetch_dim_str_is_helper);
3030 REGISTER_HELPER(zend_jit_fetch_dim_obj_r_helper);
3031 REGISTER_HELPER(zend_jit_fetch_dim_obj_is_helper);
3032 REGISTER_HELPER(zend_jit_invalid_array_access);
3033 REGISTER_HELPER(zend_jit_zval_array_dup);
3034 REGISTER_HELPER(zend_jit_prepare_assign_dim_ref);
3035 REGISTER_HELPER(zend_jit_fetch_dim_obj_w_helper);
3036 REGISTER_HELPER(zend_jit_fetch_dim_obj_rw_helper);
3037 REGISTER_HELPER(zend_jit_isset_dim_helper);
3038 REGISTER_HELPER(zend_jit_assign_dim_helper);
3039 REGISTER_HELPER(zend_jit_assign_dim_op_helper);
3040 REGISTER_HELPER(zend_jit_fetch_obj_w_slow);
3041 REGISTER_HELPER(zend_jit_fetch_obj_r_slow);
3042 REGISTER_HELPER(zend_jit_fetch_obj_is_slow);
3043 REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic);
3044 REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic);
3045 REGISTER_HELPER(zend_jit_check_array_promotion);
3046 REGISTER_HELPER(zend_jit_create_typed_ref);
3047 REGISTER_HELPER(zend_jit_invalid_property_write);
3048 REGISTER_HELPER(zend_jit_invalid_property_read);
3049 REGISTER_HELPER(zend_jit_extract_helper);
3050 REGISTER_HELPER(zend_jit_invalid_property_assign);
3051 REGISTER_HELPER(zend_jit_assign_to_typed_prop);
3052 REGISTER_HELPER(zend_jit_assign_obj_helper);
3053 REGISTER_HELPER(zend_jit_invalid_property_assign_op);
3054 REGISTER_HELPER(zend_jit_assign_op_to_typed_prop);
3055 REGISTER_HELPER(zend_jit_assign_obj_op_helper);
3056 REGISTER_HELPER(zend_jit_invalid_property_incdec);
3057 REGISTER_HELPER(zend_jit_inc_typed_prop);
3058 REGISTER_HELPER(zend_jit_dec_typed_prop);
3059 REGISTER_HELPER(zend_jit_pre_inc_typed_prop);
3060 REGISTER_HELPER(zend_jit_post_inc_typed_prop);
3061 REGISTER_HELPER(zend_jit_pre_dec_typed_prop);
3062 REGISTER_HELPER(zend_jit_post_dec_typed_prop);
3063 REGISTER_HELPER(zend_jit_pre_inc_obj_helper);
3064 REGISTER_HELPER(zend_jit_post_inc_obj_helper);
3065 REGISTER_HELPER(zend_jit_pre_dec_obj_helper);
3066 REGISTER_HELPER(zend_jit_post_dec_obj_helper);
3067 REGISTER_HELPER(zend_jit_rope_end);
3068
3069 #ifndef ZTS
3070 REGISTER_DATA(EG(current_execute_data));
3071 REGISTER_DATA(EG(exception));
3072 REGISTER_DATA(EG(opline_before_exception));
3073 REGISTER_DATA(EG(vm_interrupt));
3074 REGISTER_DATA(EG(timed_out));
3075 REGISTER_DATA(EG(uninitialized_zval));
3076 REGISTER_DATA(EG(zend_constants));
3077 REGISTER_DATA(EG(jit_trace_num));
3078 REGISTER_DATA(EG(vm_stack_top));
3079 REGISTER_DATA(EG(vm_stack_end));
3080 REGISTER_DATA(EG(exception_op));
3081 REGISTER_DATA(EG(symbol_table));
3082
3083 REGISTER_DATA(CG(map_ptr_base));
3084 #endif
3085 #endif
3086 }
3087
zend_jit_calc_trace_prologue_size(void)3088 static void zend_jit_calc_trace_prologue_size(void)
3089 {
3090 zend_jit_ctx jit_ctx;
3091 zend_jit_ctx *jit = &jit_ctx;
3092 void *entry;
3093 size_t size;
3094
3095 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
3096
3097 if (!GCC_GLOBAL_REGS) {
3098 ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
3099 jit_STORE_FP(jit, ref);
3100 jit->ctx.flags |= IR_FASTCALL_FUNC;
3101 }
3102
3103 ir_UNREACHABLE();
3104
3105 entry = zend_jit_ir_compile(&jit->ctx, &size, "JIT$trace_prologue");
3106 zend_jit_free_ctx(jit);
3107
3108 if (!entry) {
3109 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: could not compile prologue");
3110 }
3111
3112 zend_jit_trace_prologue_size = size;
3113 }
3114
3115 #if !ZEND_WIN32 && !defined(IR_TARGET_AARCH64)
3116 static uintptr_t zend_jit_hybrid_vm_sp_adj = 0;
3117
3118 typedef struct _Unwind_Context _Unwind_Context;
3119 typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *);
3120 extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *);
3121 extern uintptr_t _Unwind_GetCFA(_Unwind_Context *);
3122
3123 typedef struct _zend_jit_unwind_arg {
3124 int cnt;
3125 uintptr_t cfa[3];
3126 } zend_jit_unwind_arg;
3127
zend_jit_unwind_cb(_Unwind_Context * ctx,void * a)3128 static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a)
3129 {
3130 zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a;
3131 arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx);
3132 arg->cnt++;
3133 if (arg->cnt == 3) {
3134 return 5; // _URC_END_OF_STACK
3135 }
3136 return 0; // _URC_NO_REASON;
3137 }
3138
zend_jit_touch_vm_stack_data(void * vm_stack_data)3139 static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data)
3140 {
3141 zend_jit_unwind_arg arg;
3142
3143 memset(&arg, 0, sizeof(arg));
3144 _Unwind_Backtrace(zend_jit_unwind_cb, &arg);
3145 if (arg.cnt == 3) {
3146 zend_jit_hybrid_vm_sp_adj = arg.cfa[2] - arg.cfa[1];
3147 }
3148 }
3149
3150 extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data);
3151
zend_jit_set_sp_adj_vm(void)3152 static zend_never_inline void zend_jit_set_sp_adj_vm(void)
3153 {
3154 void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *);
3155
3156 orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data;
3157 zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data;
3158 execute_ex(NULL); // set sp_adj[SP_ADJ_VM]
3159 zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data;
3160 }
3161 #endif
3162
zend_jit_setup(void)3163 static void zend_jit_setup(void)
3164 {
3165 #if defined(IR_TARGET_X86)
3166 if (!zend_cpu_supports_sse2()) {
3167 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: CPU doesn't support SSE2");
3168 }
3169 #endif
3170 #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
3171 allowed_opt_flags = 0;
3172 if (zend_cpu_supports_avx()) {
3173 allowed_opt_flags |= ZEND_JIT_CPU_AVX;
3174 }
3175 # if PHP_HAVE_BUILTIN_CPU_SUPPORTS && defined(__GNUC__) && (ZEND_GCC_VERSION >= 11000)
3176 if (zend_cpu_supports_cldemote()) {
3177 default_mflags |= IR_X86_CLDEMOTE;
3178 }
3179 # endif
3180 #endif
3181 #ifdef ZTS
3182 #if defined(IR_TARGET_AARCH64)
3183 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
3184 ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0);
3185 # elif defined(_WIN64)
3186 tsrm_tls_index = _tls_index * sizeof(void*);
3187
3188 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
3189 /* Probably, it might be better solution */
3190 do {
3191 void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index];
3192 void *val = _tsrm_ls_cache;
3193 size_t offset = 0;
3194 size_t size = (char*)&_tls_end - (char*)&_tls_start;
3195
3196 while (offset < size) {
3197 if (*tls_mem == val) {
3198 tsrm_tls_offset = offset;
3199 break;
3200 }
3201 tls_mem++;
3202 offset += sizeof(void*);
3203 }
3204 if (offset >= size) {
3205 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size");
3206 }
3207 } while(0);
3208 # elif ZEND_WIN32
3209 tsrm_tls_index = _tls_index * sizeof(void*);
3210
3211 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
3212 /* Probably, it might be better solution */
3213 do {
3214 void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index];
3215 void *val = _tsrm_ls_cache;
3216 size_t offset = 0;
3217 size_t size = (char*)&_tls_end - (char*)&_tls_start;
3218
3219 while (offset < size) {
3220 if (*tls_mem == val) {
3221 tsrm_tls_offset = offset;
3222 break;
3223 }
3224 tls_mem++;
3225 offset += sizeof(void*);
3226 }
3227 if (offset >= size) {
3228 zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size");
3229 }
3230 } while(0);
3231 # elif defined(__APPLE__) && defined(__x86_64__)
3232 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
3233 if (tsrm_ls_cache_tcb_offset == 0) {
3234 size_t *ti;
3235 __asm__(
3236 "leaq __tsrm_ls_cache(%%rip),%0"
3237 : "=r" (ti));
3238 tsrm_tls_offset = ti[2];
3239 tsrm_tls_index = ti[1] * 8;
3240 }
3241 # elif defined(__GNUC__) && defined(__x86_64__)
3242 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
3243 if (tsrm_ls_cache_tcb_offset == 0) {
3244 #if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && \
3245 !defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
3246 size_t ret;
3247
3248 asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0"
3249 : "=r" (ret));
3250 tsrm_ls_cache_tcb_offset = ret;
3251 #elif defined(__MUSL__)
3252 size_t *ti;
3253
3254 __asm__(
3255 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
3256 : "=a" (ti));
3257 tsrm_tls_offset = ti[1];
3258 tsrm_tls_index = ti[0] * 8;
3259 #else
3260 size_t *ti;
3261
3262 __asm__(
3263 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
3264 : "=a" (ti));
3265 tsrm_tls_offset = ti[1];
3266 tsrm_tls_index = ti[0] * 16;
3267 #endif
3268 }
3269 # elif defined(__GNUC__) && defined(__i386__)
3270 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
3271 if (tsrm_ls_cache_tcb_offset == 0) {
3272 #if !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
3273 size_t ret;
3274
3275 asm ("leal _tsrm_ls_cache@ntpoff,%0\n"
3276 : "=a" (ret));
3277 tsrm_ls_cache_tcb_offset = ret;
3278 #else
3279 size_t *ti, _ebx, _ecx, _edx;
3280
3281 __asm__(
3282 "call 1f\n"
3283 ".subsection 1\n"
3284 "1:\tmovl (%%esp), %%ebx\n\t"
3285 "ret\n"
3286 ".previous\n\t"
3287 "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t"
3288 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t"
3289 "call ___tls_get_addr@plt\n\t"
3290 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n"
3291 : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx));
3292 tsrm_tls_offset = ti[1];
3293 tsrm_tls_index = ti[0] * 8;
3294 #endif
3295 }
3296 # endif
3297 #endif
3298
3299 #if !ZEND_WIN32 && !defined(IR_TARGET_AARCH64)
3300 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
3301 zend_jit_set_sp_adj_vm(); // set zend_jit_hybrid_vm_sp_adj
3302 }
3303 #endif
3304
3305 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
3306 zend_jit_setup_disasm();
3307 }
3308
3309 #ifndef _WIN32
3310 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
3311 ir_perf_jitdump_open();
3312 }
3313
3314 #endif
3315 zend_long debug = JIT_G(debug);
3316 if (!(debug & ZEND_JIT_DEBUG_ASM_STUBS)) {
3317 JIT_G(debug) &= ~(ZEND_JIT_DEBUG_IR_SRC|ZEND_JIT_DEBUG_IR_FINAL|
3318 ZEND_JIT_DEBUG_IR_CODEGEN|
3319 ZEND_JIT_DEBUG_IR_AFTER_SCCP|ZEND_JIT_DEBUG_IR_AFTER_CFG|ZEND_JIT_DEBUG_IR_AFTER_GCM|
3320 ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE|ZEND_JIT_DEBUG_IR_AFTER_REGS);
3321 }
3322
3323 zend_jit_calc_trace_prologue_size();
3324 zend_jit_setup_stubs();
3325 JIT_G(debug) = debug;
3326 }
3327
zend_jit_shutdown_ir(void)3328 static void zend_jit_shutdown_ir(void)
3329 {
3330 #ifndef _WIN32
3331 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
3332 ir_perf_jitdump_close();
3333 }
3334 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
3335 ir_gdb_unregister_all();
3336 }
3337 #endif
3338 #ifdef HAVE_CAPSTONE
3339 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
3340 ir_disasm_free();
3341 }
3342 #endif
3343 }
3344
3345 /* PHP control flow reconstruction helpers */
jit_IF_ex(zend_jit_ctx * jit,ir_ref condition,ir_ref true_block)3346 static ir_ref jit_IF_ex(zend_jit_ctx *jit, ir_ref condition, ir_ref true_block)
3347 {
3348 ir_ref ref = ir_IF(condition);
3349 /* op3 is used as a temporary storage for PHP BB number to reconstruct PHP control flow.
3350 *
3351 * It's used in jit_IF_TRUE_FALSE_ex() to select IF_TRUE or IF_FALSE instructions
3352 * to start target block
3353 */
3354 ir_set_op(&jit->ctx, ref, 3, true_block);
3355 return ref;
3356 }
3357
jit_IF_TRUE_FALSE_ex(zend_jit_ctx * jit,ir_ref if_ref,ir_ref true_block)3358 static void jit_IF_TRUE_FALSE_ex(zend_jit_ctx *jit, ir_ref if_ref, ir_ref true_block)
3359 {
3360 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
3361 ZEND_ASSERT(if_ref);
3362 ZEND_ASSERT(jit->ctx.ir_base[if_ref].op == IR_IF);
3363 ZEND_ASSERT(jit->ctx.ir_base[if_ref].op3);
3364 if (jit->ctx.ir_base[if_ref].op3 == true_block) {
3365 ir_IF_TRUE(if_ref);
3366 } else {
3367 ir_IF_FALSE(if_ref);
3368 }
3369 }
3370
_zend_jit_add_predecessor_ref(zend_jit_ctx * jit,int b,int pred,ir_ref ref)3371 static void _zend_jit_add_predecessor_ref(zend_jit_ctx *jit, int b, int pred, ir_ref ref)
3372 {
3373 int i, *p;
3374 zend_basic_block *bb;
3375 ir_ref *r, header;
3376
3377 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
3378 bb = &jit->ssa->cfg.blocks[b];
3379 p = &jit->ssa->cfg.predecessors[bb->predecessor_offset];
3380 r = &jit->bb_edges[jit->bb_predecessors[b]];
3381 for (i = 0; i < bb->predecessors_count; i++, p++, r++) {
3382 if (*p == pred) {
3383 ZEND_ASSERT(*r == IR_UNUSED || *r == ref);
3384 header = jit->bb_start_ref[b];
3385 if (header) {
3386 /* this is back edge */
3387 ZEND_ASSERT(jit->ctx.ir_base[header].op == IR_LOOP_BEGIN);
3388 if (jit->ctx.ir_base[ref].op == IR_END) {
3389 jit->ctx.ir_base[ref].op = IR_LOOP_END;
3390 } else if (jit->ctx.ir_base[ref].op == IR_IF) {
3391 jit_IF_TRUE_FALSE_ex(jit, ref, b);
3392 ref = ir_LOOP_END();
3393 } else if (jit->ctx.ir_base[ref].op == IR_UNREACHABLE) {
3394 ir_BEGIN(ref);
3395 ref = ir_LOOP_END();
3396 } else {
3397 ZEND_UNREACHABLE();
3398 }
3399 ir_MERGE_SET_OP(header, i + 1, ref);
3400 }
3401 *r = ref;
3402 return;
3403 }
3404 }
3405 ZEND_UNREACHABLE();
3406 }
3407
_zend_jit_merge_smart_branch_inputs(zend_jit_ctx * jit,uint32_t true_label,uint32_t false_label,ir_ref true_inputs,ir_ref false_inputs)3408 static void _zend_jit_merge_smart_branch_inputs(zend_jit_ctx *jit,
3409 uint32_t true_label,
3410 uint32_t false_label,
3411 ir_ref true_inputs,
3412 ir_ref false_inputs)
3413 {
3414 ir_ref true_path = IR_UNUSED, false_path = IR_UNUSED;
3415
3416 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
3417 if (true_inputs) {
3418 ZEND_ASSERT(jit->ctx.ir_base[true_inputs].op == IR_END);
3419 if (!jit->ctx.ir_base[true_inputs].op2) {
3420 true_path = true_inputs;
3421 } else {
3422 ir_MERGE_list(true_inputs);
3423 true_path = ir_END();
3424 }
3425 }
3426 if (false_inputs) {
3427 ZEND_ASSERT(jit->ctx.ir_base[false_inputs].op == IR_END);
3428 if (!jit->ctx.ir_base[false_inputs].op2) {
3429 false_path = false_inputs;
3430 } else {
3431 ir_MERGE_list(false_inputs);
3432 false_path = ir_END();
3433 }
3434 }
3435
3436 if (true_label == false_label && true_path && false_path) {
3437 ir_MERGE_2(true_path, false_path);
3438 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END());
3439 } else if (!true_path && !false_path) {
3440 /* dead code */
3441 true_path = ir_END();
3442 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path);
3443 } else {
3444 if (true_path) {
3445 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path);
3446 }
3447 if (false_path) {
3448 _zend_jit_add_predecessor_ref(jit, false_label, jit->b, false_path);
3449 }
3450 }
3451
3452 jit->b = -1;
3453 }
3454
_zend_jit_fix_merges(zend_jit_ctx * jit)3455 static void _zend_jit_fix_merges(zend_jit_ctx *jit)
3456 {
3457 int i, count;
3458 ir_ref j, k, n, *p, *q, *r;
3459 ir_ref ref;
3460 ir_insn *insn, *phi;
3461
3462 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
3463 count = jit->ssa->cfg.blocks_count;
3464 for (i = 0, p = jit->bb_start_ref; i < count; i++, p++) {
3465 ref = *p;
3466 if (ref) {
3467 insn = &jit->ctx.ir_base[ref];
3468 if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN) {
3469 n = insn->inputs_count;
3470 /* Remove IS_UNUSED inputs */
3471 for (j = k = 0, q = r = insn->ops + 1; j < n; j++, q++) {
3472 if (*q) {
3473 if (q != r) {
3474 *r = *q;
3475 phi = insn + 1 + (n >> 2);
3476 while (phi->op == IR_PI) {
3477 phi++;
3478 }
3479 while (phi->op == IR_PHI) {
3480 ir_insn_set_op(phi, k + 2, ir_insn_op(phi, j + 2));
3481 phi += 1 + ((n + 1) >> 2);
3482 }
3483 }
3484 k++;
3485 r++;
3486 }
3487 }
3488 if (k != n) {
3489 ir_ref n2, k2;
3490
3491 if (k <= 1) {
3492 insn->op = IR_BEGIN;
3493 insn->inputs_count = 0;
3494 } else {
3495 insn->inputs_count = k;
3496 }
3497 n2 = 1 + (n >> 2);
3498 k2 = 1 + (k >> 2);
3499 while (k2 != n2) {
3500 (insn+k2)->optx = IR_NOP;
3501 k2++;
3502 }
3503 phi = insn + 1 + (n >> 2);
3504 while (phi->op == IR_PI) {
3505 phi++;
3506 }
3507 while (phi->op == IR_PHI) {
3508 if (k <= 1) {
3509 phi->op = IR_COPY;
3510 phi->op1 = phi->op2;
3511 phi->op2 = 1;
3512 phi->inputs_count = 0;
3513 } else {
3514 phi->inputs_count = k + 1;
3515 }
3516 n2 = 1 + ((n + 1) >> 2);
3517 k2 = 1 + ((k + 1) >> 2);
3518 while (k2 != n2) {
3519 (phi+k2)->optx = IR_NOP;
3520 k2++;
3521 }
3522 phi += 1 + ((n + 1) >> 2);
3523 }
3524 }
3525 }
3526 }
3527 }
3528 }
3529
zend_jit_case_start(zend_jit_ctx * jit,int switch_b,int case_b,ir_ref switch_ref)3530 static void zend_jit_case_start(zend_jit_ctx *jit, int switch_b, int case_b, ir_ref switch_ref)
3531 {
3532 zend_basic_block *bb = &jit->ssa->cfg.blocks[switch_b];
3533 const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1];
3534
3535 if (opline->opcode == ZEND_SWITCH_LONG
3536 || opline->opcode == ZEND_SWITCH_STRING
3537 || opline->opcode == ZEND_MATCH) {
3538 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
3539 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
3540 int default_b = jit->ssa->cfg.map[default_opline - jit->op_array->opcodes];
3541 zval *zv;
3542 ir_ref list = IR_UNUSED, idx;
3543 bool first = 1;
3544
3545 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
3546 const zend_op *target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
3547 int b = jit->ssa->cfg.map[target - jit->op_array->opcodes];
3548
3549 if (b == case_b) {
3550 if (!first) {
3551 ir_END_list(list);
3552 }
3553 if (HT_IS_PACKED(jumptable)) {
3554 idx = ir_CONST_LONG(zv - jumptable->arPacked);
3555 } else {
3556 idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
3557 }
3558 ir_CASE_VAL(switch_ref, idx);
3559 first = 0;
3560 }
3561 } ZEND_HASH_FOREACH_END();
3562 if (default_b == case_b) {
3563 if (!first) {
3564 ir_END_list(list);
3565 }
3566 if (jit->ctx.ir_base[switch_ref].op3) {
3567 /* op3 may contain a list of additional "default" path inputs for MATCH */
3568 ir_ref ref = jit->ctx.ir_base[switch_ref].op3;
3569 jit->ctx.ir_base[switch_ref].op3 = IS_UNDEF;
3570 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END);
3571 ir_ref end = ref;
3572 while (jit->ctx.ir_base[end].op2) {
3573 ZEND_ASSERT(jit->ctx.ir_base[end].op == IR_END);
3574 end = jit->ctx.ir_base[end].op2;
3575 }
3576 jit->ctx.ir_base[end].op2 = list;
3577 list = ref;
3578 }
3579 ir_CASE_DEFAULT(switch_ref);
3580 }
3581 if (list) {
3582 ir_END_list(list);
3583 ir_MERGE_list(list);
3584 }
3585 } else {
3586 ZEND_UNREACHABLE();
3587 }
3588 }
3589
zend_jit_bb_start(zend_jit_ctx * jit,int b)3590 static int zend_jit_bb_start(zend_jit_ctx *jit, int b)
3591 {
3592 zend_basic_block *bb;
3593 int i, n, *p, pred;
3594 ir_ref ref, bb_start;
3595
3596 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
3597 ZEND_ASSERT(b < jit->ssa->cfg.blocks_count);
3598 bb = &jit->ssa->cfg.blocks[b];
3599 ZEND_ASSERT((bb->flags & ZEND_BB_REACHABLE) != 0);
3600 n = bb->predecessors_count;
3601
3602 if (n == 0) {
3603 /* pass */
3604 ZEND_ASSERT(jit->ctx.control);
3605 #if ZEND_DEBUG
3606 ref = jit->ctx.control;
3607 ir_insn *insn = &jit->ctx.ir_base[ref];
3608 while (insn->op >= IR_CALL && insn->op <= IR_TRAP) {
3609 ref = insn->op1;
3610 insn = &jit->ctx.ir_base[ref];
3611 }
3612 ZEND_ASSERT(insn->op == IR_START);
3613 ZEND_ASSERT(ref == 1);
3614 #endif
3615 bb_start = 1;
3616 if (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY) {
3617 /* prvent END/BEGIN merging */
3618 jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ir_END());
3619 bb_start = jit->ctx.control;
3620 }
3621 } else if (n == 1) {
3622 ZEND_ASSERT(!jit->ctx.control);
3623 pred = jit->ssa->cfg.predecessors[bb->predecessor_offset];
3624 ref = jit->bb_edges[jit->bb_predecessors[b]];
3625 if (ref == IR_UNUSED) {
3626 if (!jit->ctx.control) {
3627 ir_BEGIN(IR_UNUSED); /* unreachable block */
3628 }
3629 } else {
3630 ir_op op = jit->ctx.ir_base[ref].op;
3631
3632 if (op == IR_IF) {
3633 if (!jit->ctx.control) {
3634 jit_IF_TRUE_FALSE_ex(jit, ref, b);
3635 } else {
3636 ir_ref entry_path = ir_END();
3637 jit_IF_TRUE_FALSE_ex(jit, ref, b);
3638 ir_MERGE_WITH(entry_path);
3639 }
3640 } else if (op == IR_SWITCH) {
3641 zend_jit_case_start(jit, pred, b, ref);
3642 } else {
3643 if (!jit->ctx.control) {
3644 ZEND_ASSERT(op == IR_END || op == IR_UNREACHABLE || op == IR_RETURN);
3645 if ((jit->ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY)
3646 && (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)) {
3647 /* prvent END/BEGIN merging */
3648 jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ref);
3649 } else {
3650 ir_BEGIN(ref);
3651 }
3652 } else {
3653 ir_MERGE_WITH(ref);
3654 }
3655 }
3656 }
3657 bb_start = jit->ctx.control;
3658 } else {
3659 int forward_edges_count = 0;
3660 int back_edges_count = 0;
3661 ir_ref *pred_refs;
3662 ir_ref entry_path = IR_UNUSED;
3663 ALLOCA_FLAG(use_heap);
3664
3665 ZEND_ASSERT(!jit->ctx.control);
3666 if (jit->ctx.control) {
3667 entry_path = ir_END();
3668 }
3669 pred_refs = (ir_ref *)do_alloca(sizeof(ir_ref) * n, use_heap);
3670 for (i = 0, p = jit->ssa->cfg.predecessors + bb->predecessor_offset; i < n; p++, i++) {
3671 pred = *p;
3672 if (jit->bb_start_ref[pred]) {
3673 /* forward edge */
3674 forward_edges_count++;
3675 ref = jit->bb_edges[jit->bb_predecessors[b] + i];
3676 if (ref == IR_UNUSED) {
3677 /* dead edge */
3678 pred_refs[i] = IR_UNUSED;
3679 } else {
3680 ir_op op = jit->ctx.ir_base[ref].op;
3681
3682 if (op == IR_IF) {
3683 jit_IF_TRUE_FALSE_ex(jit, ref, b);
3684 pred_refs[i] = ir_END();
3685 } else if (op == IR_SWITCH) {
3686 zend_jit_case_start(jit, pred, b, ref);
3687 pred_refs[i] = ir_END();
3688 } else {
3689 ZEND_ASSERT(op == IR_END || op == IR_UNREACHABLE || op == IR_RETURN);
3690 pred_refs[i] = ref;
3691 }
3692 }
3693 } else {
3694 /* backward edge */
3695 back_edges_count++;
3696 pred_refs[i] = IR_UNUSED;
3697 }
3698 }
3699
3700 if (bb->flags & ZEND_BB_LOOP_HEADER) {
3701 ZEND_ASSERT(back_edges_count != 0);
3702 ZEND_ASSERT(forward_edges_count != 0);
3703 ir_MERGE_N(n, pred_refs);
3704 jit->ctx.ir_base[jit->ctx.control].op = IR_LOOP_BEGIN;
3705 bb_start = jit->ctx.control;
3706 if (entry_path) {
3707 ir_MERGE_WITH(entry_path);
3708 }
3709 } else {
3710 // ZEND_ASSERT(back_edges_count != 0);
3711 /* edges from exceptional blocks may be counted as back edges */
3712 ir_MERGE_N(n, pred_refs);
3713 bb_start = jit->ctx.control;
3714 if (entry_path) {
3715 ir_MERGE_WITH(entry_path);
3716 }
3717 }
3718 free_alloca(pred_refs, use_heap);
3719 }
3720 jit->b = b;
3721 jit->bb_start_ref[b] = bb_start;
3722
3723 if ((bb->flags & ZEND_BB_ENTRY) || (bb->idom >= 0 && jit->bb_start_ref[bb->idom] < jit->ctx.fold_cse_limit)) {
3724 jit->ctx.fold_cse_limit = bb_start;
3725 }
3726
3727 return 1;
3728 }
3729
zend_jit_bb_end(zend_jit_ctx * jit,int b)3730 static int zend_jit_bb_end(zend_jit_ctx *jit, int b)
3731 {
3732 int succ;
3733 zend_basic_block *bb;
3734
3735 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
3736 if (jit->b != b) {
3737 return 1;
3738 }
3739
3740 bb = &jit->ssa->cfg.blocks[b];
3741 ZEND_ASSERT(bb->successors_count != 0);
3742 if (bb->successors_count == 1) {
3743 succ = bb->successors[0];
3744 } else {
3745 const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1];
3746
3747 /* Use only the following successor of SWITCH and FE_RESET_R */
3748 ZEND_ASSERT(opline->opcode == ZEND_SWITCH_LONG
3749 || opline->opcode == ZEND_SWITCH_STRING
3750 || opline->opcode == ZEND_MATCH
3751 || opline->opcode == ZEND_FE_RESET_R);
3752 succ = b + 1;
3753 }
3754 _zend_jit_add_predecessor_ref(jit, succ, b, ir_END());
3755 jit->b = -1;
3756 return 1;
3757 }
3758
jit_CMP_IP(zend_jit_ctx * jit,ir_op op,const zend_op * next_opline)3759 static int jit_CMP_IP(zend_jit_ctx *jit, ir_op op, const zend_op *next_opline)
3760 {
3761 ir_ref ref;
3762
3763 #if 1
3764 if (GCC_GLOBAL_REGS) {
3765 ref = jit_IP32(jit);
3766 } else {
3767 ref = ir_LOAD_U32(jit_EX(opline));
3768 }
3769 ref = ir_CMP_OP(op, ref, ir_CONST_U32((uint32_t)(uintptr_t)next_opline));
3770 #else
3771 if (GCC_GLOBAL_REGS) {
3772 ref = jit_IP(jit);
3773 } else {
3774 ref = ir_LOAD_A(jit_EX(opline));
3775 }
3776 ref = ir_CMP_OP(op, ref, ir_CONST_ADDR(next_opline));
3777 #endif
3778 return ref;
3779 }
3780
zend_jit_jmp_frameless(zend_jit_ctx * jit,const zend_op * opline,const void * exit_addr,zend_jmp_fl_result guard)3781 static int zend_jit_jmp_frameless(
3782 zend_jit_ctx *jit,
3783 const zend_op *opline,
3784 const void *exit_addr,
3785 zend_jmp_fl_result guard
3786 ) {
3787 ir_ref ref, if_ref, cache_result, function_result, phi_result, cache_slot_ref;
3788 zend_basic_block *bb;
3789
3790 // JIT: CACHED_PTR(opline->extended_value)
3791 cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value);
3792 cache_result = ir_LOAD_L(cache_slot_ref);
3793
3794 // JIT: if (UNEXPECTED(!result))
3795 if_ref = ir_IF(cache_result);
3796 ir_IF_FALSE_cold(if_ref);
3797 zval *func_name_zv = RT_CONSTANT(opline, opline->op1);
3798 function_result = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_jit_jmp_frameless_helper),
3799 ir_CONST_ADDR(func_name_zv),
3800 cache_slot_ref);
3801 ir_MERGE_WITH_EMPTY_TRUE(if_ref);
3802
3803 phi_result = ir_PHI_2(IR_LONG, function_result, cache_result);
3804
3805 if (exit_addr) {
3806 ir_GUARD(ir_EQ(phi_result, ir_CONST_LONG(guard)), ir_CONST_ADDR(exit_addr));
3807 } else {
3808 ZEND_ASSERT(jit->b >= 0);
3809 bb = &jit->ssa->cfg.blocks[jit->b];
3810 // JIT: if (result == ZEND_JMP_FL_HIT)
3811 ref = jit_IF_ex(jit, ir_EQ(phi_result, ir_CONST_LONG(ZEND_JMP_FL_HIT)), bb->successors[0]);
3812 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
3813 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
3814 jit->b = -1;
3815 }
3816
3817 return 1;
3818 }
3819
zend_jit_cond_jmp(zend_jit_ctx * jit,const zend_op * next_opline,int target_block)3820 static int zend_jit_cond_jmp(zend_jit_ctx *jit, const zend_op *next_opline, int target_block)
3821 {
3822 ir_ref ref;
3823 zend_basic_block *bb;
3824
3825 ZEND_ASSERT(jit->b >= 0);
3826 bb = &jit->ssa->cfg.blocks[jit->b];
3827
3828 ZEND_ASSERT(bb->successors_count == 2);
3829 if (bb->successors[0] == bb->successors[1]) {
3830 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
3831 jit->b = -1;
3832 zend_jit_set_last_valid_opline(jit, next_opline);
3833 return 1;
3834 }
3835
3836 ref = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, next_opline), target_block);
3837
3838 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
3839 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
3840
3841 jit->b = -1;
3842 zend_jit_set_last_valid_opline(jit, next_opline);
3843
3844 return 1;
3845 }
3846
zend_jit_set_cond(zend_jit_ctx * jit,const zend_op * next_opline,uint32_t var)3847 static int zend_jit_set_cond(zend_jit_ctx *jit, const zend_op *next_opline, uint32_t var)
3848 {
3849 ir_ref ref;
3850
3851 ref = ir_ADD_U32(ir_ZEXT_U32(jit_CMP_IP(jit, IR_EQ, next_opline)), ir_CONST_U32(IS_FALSE));
3852
3853 // EX_VAR(var) = ...
3854 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), var + offsetof(zval, u1.type_info)), ref);
3855
3856 zend_jit_reset_last_valid_opline(jit);
3857 return zend_jit_set_ip(jit, next_opline - 1);
3858 }
3859
3860 /* PHP JIT handlers */
zend_jit_check_exception(zend_jit_ctx * jit)3861 static void zend_jit_check_exception(zend_jit_ctx *jit)
3862 {
3863 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
3864 jit_STUB_ADDR(jit, jit_stub_exception_handler));
3865 }
3866
zend_jit_check_exception_undef_result(zend_jit_ctx * jit,const zend_op * opline)3867 static void zend_jit_check_exception_undef_result(zend_jit_ctx *jit, const zend_op *opline)
3868 {
3869 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
3870 jit_STUB_ADDR(jit,
3871 (opline->result_type & (IS_TMP_VAR|IS_VAR)) ? jit_stub_exception_handler_undef : jit_stub_exception_handler));
3872 }
3873
zend_jit_type_check_undef(zend_jit_ctx * jit,ir_ref type,uint32_t var,const zend_op * opline,bool check_exception,bool in_cold_path,bool undef_result)3874 static void zend_jit_type_check_undef(zend_jit_ctx *jit,
3875 ir_ref type,
3876 uint32_t var,
3877 const zend_op *opline,
3878 bool check_exception,
3879 bool in_cold_path,
3880 bool undef_result)
3881 {
3882 ir_ref if_def = ir_IF(type);
3883
3884 if (!in_cold_path) {
3885 ir_IF_FALSE_cold(if_def);
3886 } else {
3887 ir_IF_FALSE(if_def);
3888 }
3889 if (opline) {
3890 jit_SET_EX_OPLINE(jit, opline);
3891 }
3892 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(var));
3893 if (check_exception) {
3894 if (undef_result) {
3895 zend_jit_check_exception_undef_result(jit, opline);
3896 } else {
3897 zend_jit_check_exception(jit);
3898 }
3899 }
3900 ir_MERGE_WITH_EMPTY_TRUE(if_def);
3901 }
3902
zend_jit_zval_check_undef(zend_jit_ctx * jit,ir_ref ref,uint32_t var,const zend_op * opline,bool check_exception)3903 static ir_ref zend_jit_zval_check_undef(zend_jit_ctx *jit,
3904 ir_ref ref,
3905 uint32_t var,
3906 const zend_op *opline,
3907 bool check_exception)
3908 {
3909 ir_ref if_def, ref2;
3910
3911 if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
3912 ir_IF_FALSE_cold(if_def);
3913
3914 if (opline) {
3915 jit_SET_EX_OPLINE(jit, opline);
3916 }
3917
3918 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(var));
3919
3920 if (check_exception) {
3921 zend_jit_check_exception(jit);
3922 }
3923
3924 ref2 = jit_EG(uninitialized_zval);
3925
3926 ir_MERGE_WITH_EMPTY_TRUE(if_def);
3927
3928 return ir_PHI_2(IR_ADDR, ref2, ref);
3929 }
3930
zend_jit_recv_entry(zend_jit_ctx * jit,int b)3931 static void zend_jit_recv_entry(zend_jit_ctx *jit, int b)
3932 {
3933 zend_basic_block *bb = &jit->ssa->cfg.blocks[b];
3934 int pred;
3935 ir_ref ref;
3936
3937 ZEND_ASSERT(bb->predecessors_count > 0);
3938
3939 pred = jit->bb_predecessors[b];
3940 ref = jit->bb_edges[pred];
3941
3942 ZEND_ASSERT(ref);
3943 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END);
3944
3945 /* Insert a MERGE block with additional ENTRY input between predecessor and this one */
3946 ir_ENTRY(ref, bb->start);
3947 if (!GCC_GLOBAL_REGS) {
3948 /* 2 is hardcoded reference to IR_PARAM */
3949 ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
3950 ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
3951 jit_STORE_FP(jit, 2);
3952 }
3953
3954 ir_MERGE_WITH(ref);
3955 jit->bb_edges[pred] = ir_END();
3956 }
3957
zend_jit_osr_entry(zend_jit_ctx * jit,int b)3958 static void zend_jit_osr_entry(zend_jit_ctx *jit, int b)
3959 {
3960 zend_basic_block *bb = &jit->ssa->cfg.blocks[b];
3961 ir_ref ref = ir_END();
3962
3963 /* Insert a MERGE block with additional ENTRY input between predecessor and this one */
3964 ir_ENTRY(ref, bb->start);
3965 if (!GCC_GLOBAL_REGS) {
3966 /* 2 is hardcoded reference to IR_PARAM */
3967 ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
3968 ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
3969 jit_STORE_FP(jit, 2);
3970 }
3971
3972 ir_MERGE_WITH(ref);
3973 }
3974
zend_jit_continue_entry(zend_jit_ctx * jit,ir_ref src,unsigned int label)3975 static ir_ref zend_jit_continue_entry(zend_jit_ctx *jit, ir_ref src, unsigned int label)
3976 {
3977 ir_ENTRY(src, label);
3978 if (!GCC_GLOBAL_REGS) {
3979 /* 2 is hardcoded reference to IR_PARAM */
3980 ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
3981 ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
3982 jit_STORE_FP(jit, 2);
3983 }
3984 return ir_END();
3985 }
3986
zend_jit_handler(zend_jit_ctx * jit,const zend_op * opline,int may_throw)3987 static int zend_jit_handler(zend_jit_ctx *jit, const zend_op *opline, int may_throw)
3988 {
3989 ir_ref ref;
3990 const void *handler;
3991
3992 zend_jit_set_ip(jit, opline);
3993 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
3994 handler = zend_get_opcode_handler_func(opline);
3995 } else {
3996 handler = opline->handler;
3997 }
3998 if (GCC_GLOBAL_REGS) {
3999 ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
4000 } else {
4001 ref = jit_FP(jit);
4002 ref = ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(handler), ref);
4003 }
4004 if (may_throw) {
4005 zend_jit_check_exception(jit);
4006 }
4007 /* Skip the following OP_DATA */
4008 switch (opline->opcode) {
4009 case ZEND_ASSIGN_DIM:
4010 case ZEND_ASSIGN_OBJ:
4011 case ZEND_ASSIGN_STATIC_PROP:
4012 case ZEND_ASSIGN_DIM_OP:
4013 case ZEND_ASSIGN_OBJ_OP:
4014 case ZEND_ASSIGN_STATIC_PROP_OP:
4015 case ZEND_ASSIGN_STATIC_PROP_REF:
4016 case ZEND_ASSIGN_OBJ_REF:
4017 zend_jit_set_last_valid_opline(jit, opline + 2);
4018 break;
4019 default:
4020 zend_jit_set_last_valid_opline(jit, opline + 1);
4021 break;
4022 }
4023 return 1;
4024 }
4025
zend_jit_tail_handler(zend_jit_ctx * jit,const zend_op * opline)4026 static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline)
4027 {
4028 const void *handler;
4029 ir_ref ref;
4030 zend_basic_block *bb;
4031
4032 zend_jit_set_ip(jit, opline);
4033 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
4034 if (opline->opcode == ZEND_DO_UCALL ||
4035 opline->opcode == ZEND_DO_FCALL_BY_NAME ||
4036 opline->opcode == ZEND_DO_FCALL ||
4037 opline->opcode == ZEND_RETURN) {
4038
4039 /* Use inlined HYBRID VM handler */
4040 handler = opline->handler;
4041 ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
4042 } else {
4043 handler = zend_get_opcode_handler_func(opline);
4044 ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
4045 ref = ir_LOAD_A(jit_IP(jit));
4046 ir_TAILCALL(IR_VOID, ref);
4047 }
4048 } else {
4049 handler = opline->handler;
4050 if (GCC_GLOBAL_REGS) {
4051 ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
4052 } else if ((jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)
4053 && (opline->opcode == ZEND_CATCH
4054 || opline->opcode == ZEND_FAST_CALL
4055 || opline->opcode == ZEND_FAST_RET
4056 || opline->opcode == ZEND_MATCH_ERROR
4057 || opline->opcode == ZEND_THROW
4058 || opline->opcode == ZEND_VERIFY_NEVER_TYPE)) {
4059 ref = jit_FP(jit);
4060 ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
4061 ir_RETURN(ir_CONST_I32(1));
4062 } else {
4063 ref = jit_FP(jit);
4064 ir_TAILCALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
4065 }
4066 }
4067 if (jit->b >= 0) {
4068 bb = &jit->ssa->cfg.blocks[jit->b];
4069 if (bb->successors_count > 0
4070 && (opline->opcode == ZEND_DO_FCALL
4071 || opline->opcode == ZEND_DO_UCALL
4072 || opline->opcode == ZEND_DO_FCALL_BY_NAME
4073 || opline->opcode == ZEND_INCLUDE_OR_EVAL
4074 || opline->opcode == ZEND_GENERATOR_CREATE
4075 || opline->opcode == ZEND_YIELD
4076 || opline->opcode == ZEND_YIELD_FROM
4077 || opline->opcode == ZEND_FAST_CALL)) {
4078 /* Add a fake control edge from UNREACHABLE to the following ENTRY */
4079 int succ;
4080
4081 if (bb->successors_count == 1) {
4082 succ = bb->successors[0];
4083 ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY);
4084 } else {
4085 /* Use only the following successor of FAST_CALL */
4086 ZEND_ASSERT(opline->opcode == ZEND_FAST_CALL);
4087 succ = jit->b + 1;
4088 /* we need an entry */
4089 jit->ssa->cfg.blocks[succ].flags |= ZEND_BB_ENTRY;
4090 }
4091 ref = jit->ctx.insns_count - 1;
4092 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE || jit->ctx.ir_base[ref].op == IR_RETURN);
4093 ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start);
4094 _zend_jit_add_predecessor_ref(jit, succ, jit->b, ref);
4095 }
4096 jit->b = -1;
4097 zend_jit_reset_last_valid_opline(jit);
4098 }
4099 return 1;
4100 }
4101
zend_jit_call(zend_jit_ctx * jit,const zend_op * opline,unsigned int next_block)4102 static int zend_jit_call(zend_jit_ctx *jit, const zend_op *opline, unsigned int next_block)
4103 {
4104 return zend_jit_tail_handler(jit, opline);
4105 }
4106
zend_jit_spill_store(zend_jit_ctx * jit,zend_jit_addr src,zend_jit_addr dst,uint32_t info,bool set_type)4107 static int zend_jit_spill_store(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type)
4108 {
4109 ZEND_ASSERT(Z_MODE(src) == IS_REG);
4110 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
4111
4112 if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
4113 jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src));
4114 if (set_type &&
4115 (Z_REG(dst) != ZREG_FP ||
4116 !JIT_G(current_frame) ||
4117 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
4118 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4119 }
4120 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4121 jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src));
4122 if (set_type &&
4123 (Z_REG(dst) != ZREG_FP ||
4124 !JIT_G(current_frame) ||
4125 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
4126 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4127 }
4128 } else {
4129 ZEND_UNREACHABLE();
4130 }
4131 return 1;
4132 }
4133
zend_jit_spill_store_inv(zend_jit_ctx * jit,zend_jit_addr src,zend_jit_addr dst,uint32_t info)4134 static int zend_jit_spill_store_inv(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
4135 {
4136 ZEND_ASSERT(Z_MODE(src) == IS_REG);
4137 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
4138
4139 if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
4140 jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src));
4141 if (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame)) {
4142 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4143 } else if (STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG) {
4144 /* invalidate memory type */
4145 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) = IS_UNKNOWN;
4146 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4147 }
4148 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4149 jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src));
4150 if (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame)) {
4151 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4152 } else if (STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE) {
4153 /* invalidate memory type */
4154 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) = IS_UNKNOWN;
4155 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4156 }
4157 } else {
4158 ZEND_UNREACHABLE();
4159 }
4160 return 1;
4161 }
4162
zend_jit_load_reg(zend_jit_ctx * jit,zend_jit_addr src,zend_jit_addr dst,uint32_t info)4163 static int zend_jit_load_reg(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
4164 {
4165 ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL);
4166 ZEND_ASSERT(Z_MODE(dst) == IS_REG);
4167
4168 if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
4169 zend_jit_def_reg(jit, dst, jit_Z_LVAL(jit, src));
4170 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4171 zend_jit_def_reg(jit, dst, jit_Z_DVAL(jit, src));
4172 } else {
4173 ZEND_UNREACHABLE();
4174 }
4175 return 1;
4176 }
4177
zend_jit_store_var(zend_jit_ctx * jit,uint32_t info,int var,int ssa_var,bool set_type)4178 static int zend_jit_store_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var, bool set_type)
4179 {
4180 zend_jit_addr src = ZEND_ADDR_REG(ssa_var);
4181 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4182
4183 return zend_jit_spill_store(jit, src, dst, info, set_type);
4184 }
4185
zend_jit_store_ref(zend_jit_ctx * jit,uint32_t info,int var,int32_t src,bool set_type)4186 static int zend_jit_store_ref(zend_jit_ctx *jit, uint32_t info, int var, int32_t src, bool set_type)
4187 {
4188 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4189
4190 if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
4191 jit_set_Z_LVAL(jit, dst, src);
4192 if (set_type &&
4193 (Z_REG(dst) != ZREG_FP ||
4194 !JIT_G(current_frame) ||
4195 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
4196 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4197 }
4198 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4199 jit_set_Z_DVAL(jit, dst, src);
4200 if (set_type &&
4201 (Z_REG(dst) != ZREG_FP ||
4202 !JIT_G(current_frame) ||
4203 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
4204 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4205 }
4206 } else {
4207 ZEND_UNREACHABLE();
4208 }
4209 return 1;
4210 }
4211
zend_jit_deopt_rload(zend_jit_ctx * jit,ir_type type,int32_t reg)4212 static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t reg)
4213 {
4214 ir_ref ref = jit->ctx.control;
4215 ir_insn *insn;
4216
4217 while (1) {
4218 insn = &jit->ctx.ir_base[ref];
4219 if (insn->op == IR_RLOAD && insn->op2 == reg) {
4220 ZEND_ASSERT(insn->type == type);
4221 return ref;
4222 } else if (insn->op == IR_START) {
4223 break;
4224 }
4225 ref = insn->op1;
4226 }
4227 return ir_RLOAD(type, reg);
4228 }
4229
zend_jit_store_const_long(zend_jit_ctx * jit,int var,zend_long val)4230 static int zend_jit_store_const_long(zend_jit_ctx *jit, int var, zend_long val)
4231 {
4232 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4233 ir_ref src = ir_CONST_LONG(val);
4234
4235 if (jit->ra && jit->ra[var].ref == IR_NULL) {
4236 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4237 }
4238 jit_set_Z_LVAL(jit, dst, src);
4239 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4240 return 1;
4241 }
4242
zend_jit_store_const_double(zend_jit_ctx * jit,int var,double val)4243 static int zend_jit_store_const_double(zend_jit_ctx *jit, int var, double val)
4244 {
4245 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4246 ir_ref src = ir_CONST_DOUBLE(val);
4247
4248 if (jit->ra && jit->ra[var].ref == IR_NULL) {
4249 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4250 }
4251 jit_set_Z_DVAL(jit, dst, src);
4252 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4253 return 1;
4254 }
4255
zend_jit_store_type(zend_jit_ctx * jit,int var,uint8_t type)4256 static int zend_jit_store_type(zend_jit_ctx *jit, int var, uint8_t type)
4257 {
4258 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4259
4260 ZEND_ASSERT(type <= IS_DOUBLE);
4261 jit_set_Z_TYPE_INFO(jit, dst, type);
4262 return 1;
4263 }
4264
zend_jit_store_reg(zend_jit_ctx * jit,uint32_t info,int var,int8_t reg,bool in_mem,bool set_type)4265 static int zend_jit_store_reg(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, bool in_mem, bool set_type)
4266 {
4267 zend_jit_addr src;
4268 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4269 ir_type type;
4270
4271 if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
4272 type = IR_LONG;
4273 src = zend_jit_deopt_rload(jit, type, reg);
4274 if (jit->ra && jit->ra[var].ref == IR_NULL) {
4275 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4276 } else if (!in_mem) {
4277 jit_set_Z_LVAL(jit, dst, src);
4278 if (set_type &&
4279 (Z_REG(dst) != ZREG_FP ||
4280 !JIT_G(current_frame) ||
4281 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
4282 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4283 }
4284 }
4285 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4286 type = IR_DOUBLE;
4287 src = zend_jit_deopt_rload(jit, type, reg);
4288 if (jit->ra && jit->ra[var].ref == IR_NULL) {
4289 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4290 } else if (!in_mem) {
4291 jit_set_Z_DVAL(jit, dst, src);
4292 if (set_type &&
4293 (Z_REG(dst) != ZREG_FP ||
4294 !JIT_G(current_frame) ||
4295 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
4296 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4297 }
4298 }
4299 } else {
4300 ZEND_UNREACHABLE();
4301 }
4302 return 1;
4303 }
4304
zend_jit_store_spill_slot(zend_jit_ctx * jit,uint32_t info,int var,int8_t reg,int32_t offset,bool set_type)4305 static int zend_jit_store_spill_slot(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, int32_t offset, bool set_type)
4306 {
4307 zend_jit_addr src;
4308 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4309
4310 if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
4311 src = ir_LOAD_L(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset));
4312 if (jit->ra && jit->ra[var].ref == IR_NULL) {
4313 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4314 } else {
4315 jit_set_Z_LVAL(jit, dst, src);
4316 if (set_type &&
4317 (Z_REG(dst) != ZREG_FP ||
4318 !JIT_G(current_frame) ||
4319 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
4320 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
4321 }
4322 }
4323 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4324 src = ir_LOAD_D(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset));
4325 if (jit->ra && jit->ra[var].ref == IR_NULL) {
4326 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
4327 } else {
4328 jit_set_Z_DVAL(jit, dst, src);
4329 if (set_type &&
4330 (Z_REG(dst) != ZREG_FP ||
4331 !JIT_G(current_frame) ||
4332 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
4333 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
4334 }
4335 }
4336 } else {
4337 ZEND_UNREACHABLE();
4338 }
4339 return 1;
4340 }
4341
zend_jit_store_var_type(zend_jit_ctx * jit,int var,uint32_t type)4342 static int zend_jit_store_var_type(zend_jit_ctx *jit, int var, uint32_t type)
4343 {
4344 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4345
4346 jit_set_Z_TYPE_INFO(jit, dst, type);
4347 return 1;
4348 }
4349
zend_jit_zval_try_addref(zend_jit_ctx * jit,zend_jit_addr var_addr)4350 static int zend_jit_zval_try_addref(zend_jit_ctx *jit, zend_jit_addr var_addr)
4351 {
4352 ir_ref if_refcounted, end1;
4353
4354 if_refcounted = jit_if_REFCOUNTED(jit, var_addr);
4355 ir_IF_FALSE(if_refcounted);
4356 end1 = ir_END();
4357 ir_IF_TRUE(if_refcounted);
4358 jit_GC_ADDREF(jit, jit_Z_PTR(jit, var_addr));
4359 ir_MERGE_WITH(end1);
4360 return 1;
4361 }
4362
zend_jit_store_var_if_necessary(zend_jit_ctx * jit,int var,zend_jit_addr src,uint32_t info)4363 static int zend_jit_store_var_if_necessary(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t info)
4364 {
4365 if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
4366 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
4367 return zend_jit_spill_store(jit, src, dst, info, 1);
4368 }
4369 return 1;
4370 }
4371
zend_jit_store_var_if_necessary_ex(zend_jit_ctx * jit,int var,zend_jit_addr src,uint32_t info,zend_jit_addr old,uint32_t old_info)4372 static int zend_jit_store_var_if_necessary_ex(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info)
4373 {
4374 if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
4375 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
4376 bool set_type = 1;
4377
4378 if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) ==
4379 (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) {
4380 if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) {
4381 if (JIT_G(current_frame)) {
4382 uint32_t mem_type = STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var));
4383
4384 if (mem_type != IS_UNKNOWN
4385 && (info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == (1 << mem_type)) {
4386 set_type = 0;
4387 }
4388 } else {
4389 set_type = 0;
4390 }
4391 }
4392 }
4393 return zend_jit_spill_store(jit, src, dst, info, set_type);
4394 }
4395 return 1;
4396 }
4397
zend_jit_load_var(zend_jit_ctx * jit,uint32_t info,int var,int ssa_var)4398 static int zend_jit_load_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var)
4399 {
4400 zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
4401 zend_jit_addr dst = ZEND_ADDR_REG(ssa_var);
4402
4403 return zend_jit_load_reg(jit, src, dst, info);
4404 }
4405
zend_jit_invalidate_var_if_necessary(zend_jit_ctx * jit,uint8_t op_type,zend_jit_addr addr,znode_op op)4406 static int zend_jit_invalidate_var_if_necessary(zend_jit_ctx *jit, uint8_t op_type, zend_jit_addr addr, znode_op op)
4407 {
4408 if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) {
4409 /* Invalidate operand type to prevent incorrect destuction by exception_handler_free_op1_op2() */
4410 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var);
4411 jit_set_Z_TYPE_INFO(jit, dst, IS_UNDEF);
4412 }
4413 return 1;
4414 }
4415
zend_jit_update_regs(zend_jit_ctx * jit,uint32_t var,zend_jit_addr src,zend_jit_addr dst,uint32_t info)4416 static int zend_jit_update_regs(zend_jit_ctx *jit, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
4417 {
4418 if (!zend_jit_same_addr(src, dst)) {
4419 if (Z_MODE(src) == IS_REG) {
4420 if (Z_MODE(dst) == IS_REG) {
4421 zend_jit_def_reg(jit, dst, zend_jit_use_reg(jit, src));
4422 if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) {
4423 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
4424
4425 if (!zend_jit_spill_store(jit, dst, var_addr, info,
4426 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
4427 JIT_G(current_frame) == NULL ||
4428 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
4429 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
4430 )) {
4431 return 0;
4432 }
4433 }
4434 } else if (Z_MODE(dst) == IS_MEM_ZVAL) {
4435 if (!Z_LOAD(src) && !Z_STORE(src)) {
4436 if (!zend_jit_spill_store(jit, src, dst, info,
4437 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
4438 JIT_G(current_frame) == NULL ||
4439 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
4440 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
4441 )) {
4442 return 0;
4443 }
4444 }
4445 } else {
4446 ZEND_UNREACHABLE();
4447 }
4448 } else if (Z_MODE(src) == IS_MEM_ZVAL) {
4449 if (Z_MODE(dst) == IS_REG) {
4450 if (!zend_jit_load_reg(jit, src, dst, info)) {
4451 return 0;
4452 }
4453 } else {
4454 ZEND_UNREACHABLE();
4455 }
4456 } else {
4457 ZEND_UNREACHABLE();
4458 }
4459 } else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) {
4460 dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
4461 if (!zend_jit_spill_store(jit, src, dst, info,
4462 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
4463 JIT_G(current_frame) == NULL ||
4464 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
4465 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
4466 )) {
4467 return 0;
4468 }
4469 }
4470 return 1;
4471 }
4472
zend_jit_inc_dec(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_addr,uint32_t op1_def_info,zend_jit_addr op1_def_addr,uint32_t res_use_info,uint32_t res_info,zend_jit_addr res_addr,int may_overflow,int may_throw)4473 static int zend_jit_inc_dec(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
4474 {
4475 ir_ref if_long = IR_UNUSED;
4476 ir_ref op1_lval_ref = IR_UNUSED;
4477 ir_ref ref;
4478 ir_op op;
4479
4480 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) {
4481 if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
4482 ir_IF_TRUE(if_long);
4483 }
4484 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
4485 op1_lval_ref = jit_Z_LVAL(jit, op1_addr);
4486 jit_set_Z_LVAL(jit, res_addr, op1_lval_ref);
4487 if (Z_MODE(res_addr) != IS_REG) {
4488 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
4489 }
4490 }
4491 if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL
4492 && Z_MODE(op1_addr) == IS_REG
4493 && !Z_LOAD(op1_addr)
4494 && !Z_STORE(op1_addr)) {
4495 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_LONG);
4496 }
4497 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
4498 op = may_overflow ? IR_ADD_OV : IR_ADD;
4499 } else {
4500 op = may_overflow ? IR_SUB_OV : IR_SUB;
4501 }
4502 if (!op1_lval_ref) {
4503 op1_lval_ref = jit_Z_LVAL(jit, op1_addr);
4504 }
4505 ref = ir_BINARY_OP_L(op, op1_lval_ref, ir_CONST_LONG(1));
4506 if (op1_def_info & MAY_BE_LONG) {
4507 jit_set_Z_LVAL(jit, op1_def_addr, ref);
4508 }
4509 if (may_overflow &&
4510 (((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)) ||
4511 ((opline->result_type != IS_UNUSED && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD))))) {
4512 int32_t exit_point;
4513 const void *exit_addr;
4514 zend_jit_trace_stack *stack;
4515 uint32_t old_op1_info, old_res_info = 0;
4516
4517 stack = JIT_G(current_frame)->stack;
4518 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
4519 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0);
4520 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
4521 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
4522 } else {
4523 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
4524 }
4525 if (opline->result_type != IS_UNUSED) {
4526 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
4527 if (opline->opcode == ZEND_PRE_INC) {
4528 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
4529 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
4530 } else if (opline->opcode == ZEND_PRE_DEC) {
4531 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
4532 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
4533 } else if (opline->opcode == ZEND_POST_INC) {
4534 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
4535 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_LONG(ZEND_LONG_MAX));
4536 } else if (opline->opcode == ZEND_POST_DEC) {
4537 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
4538 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_LONG(ZEND_LONG_MIN));
4539 }
4540 }
4541
4542 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
4543 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4544 ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
4545
4546 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
4547 opline->result_type != IS_UNUSED) {
4548 jit_set_Z_LVAL(jit, res_addr, ref);
4549 if (Z_MODE(res_addr) != IS_REG) {
4550 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
4551 }
4552 }
4553
4554 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
4555 if (opline->result_type != IS_UNUSED) {
4556 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
4557 }
4558 } else if (may_overflow) {
4559 ir_ref if_overflow;
4560 ir_ref merge_inputs = IR_UNUSED;
4561
4562 if (((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
4563 || (opline->result_type != IS_UNUSED && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))) {
4564 int32_t exit_point;
4565 const void *exit_addr;
4566 zend_jit_trace_stack *stack;
4567 uint32_t old_res_info = 0;
4568
4569 stack = JIT_G(current_frame)->stack;
4570 if (opline->result_type != IS_UNUSED) {
4571 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
4572 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) {
4573 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
4574 }
4575 }
4576 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
4577 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4578 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
4579 opline->result_type != IS_UNUSED) {
4580 if_overflow = ir_IF(ir_OVERFLOW(ref));
4581 ir_IF_FALSE_cold(if_overflow);
4582 jit_set_Z_LVAL(jit, res_addr, ref);
4583 if (Z_MODE(res_addr) != IS_REG) {
4584 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
4585 }
4586 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
4587 ir_IF_TRUE(if_overflow);
4588 } else {
4589 ir_GUARD(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
4590 }
4591 if (opline->result_type != IS_UNUSED) {
4592 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
4593 }
4594 } else {
4595 if_overflow = ir_IF(ir_OVERFLOW(ref));
4596 ir_IF_FALSE(if_overflow);
4597 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
4598 opline->result_type != IS_UNUSED) {
4599 jit_set_Z_LVAL(jit, res_addr, ref);
4600 if (Z_MODE(res_addr) != IS_REG) {
4601 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
4602 }
4603 }
4604 ir_END_list(merge_inputs);
4605
4606 /* overflow => cold path */
4607 ir_IF_TRUE_cold(if_overflow);
4608 }
4609
4610 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
4611 if (Z_MODE(op1_def_addr) == IS_REG) {
4612 jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
4613 } else {
4614 #if SIZEOF_ZEND_LONG == 4
4615 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0));
4616 jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0x41e00000));
4617 #else
4618 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x43e0000000000000));
4619 #endif
4620 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE);
4621 }
4622 } else {
4623 if (Z_MODE(op1_def_addr) == IS_REG) {
4624 jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
4625 } else {
4626 #if SIZEOF_ZEND_LONG == 4
4627 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x00200000));
4628 jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0xc1e00000));
4629 #else
4630 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0xc3e0000000000000));
4631 #endif
4632 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE);
4633 }
4634 }
4635 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
4636 opline->result_type != IS_UNUSED) {
4637 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
4638 if (Z_MODE(res_addr) == IS_REG) {
4639 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
4640 } else {
4641 #if SIZEOF_ZEND_LONG == 4
4642 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
4643 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
4644 #else
4645 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
4646 #endif
4647 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4648 }
4649 } else {
4650 if (Z_MODE(res_addr) == IS_REG) {
4651 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
4652 } else {
4653 #if SIZEOF_ZEND_LONG == 4
4654 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
4655 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
4656 #else
4657 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
4658 #endif
4659 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4660 }
4661 }
4662 }
4663
4664 if (merge_inputs) {
4665 ir_END_list(merge_inputs);
4666 ir_MERGE_list(merge_inputs);
4667 }
4668 } else {
4669 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
4670 opline->result_type != IS_UNUSED) {
4671 jit_set_Z_LVAL(jit, res_addr, ref);
4672 if (Z_MODE(res_addr) != IS_REG) {
4673 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
4674 }
4675 }
4676 }
4677 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
4678 ir_ref merge_inputs = ir_END();
4679
4680 /* !is_long => cold path */
4681 ir_IF_FALSE_cold(if_long);
4682 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
4683 jit_SET_EX_OPLINE(jit, opline);
4684 if (op1_info & MAY_BE_UNDEF) {
4685 ir_ref if_def;
4686
4687 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
4688 ir_IF_FALSE_cold(if_def);
4689
4690 // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
4691 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
4692
4693 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_NULL);
4694 ir_MERGE_WITH_EMPTY_TRUE(if_def);
4695
4696 op1_info |= MAY_BE_NULL;
4697 }
4698
4699 ref = jit_ZVAL_ADDR(jit, op1_addr);
4700
4701 if (op1_info & MAY_BE_REF) {
4702 ir_ref if_ref, if_typed, func, ref2, arg2;
4703
4704 if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
4705 ir_IF_TRUE(if_ref);
4706 ref2 = jit_Z_PTR_ref(jit, ref);
4707
4708 if_typed = jit_if_TYPED_REF(jit, ref2);
4709 ir_IF_TRUE(if_typed);
4710
4711 if (RETURN_VALUE_USED(opline)) {
4712 ZEND_ASSERT(Z_MODE(res_addr) != IS_REG);
4713 arg2 = jit_ZVAL_ADDR(jit, res_addr);
4714 } else {
4715 arg2 = IR_NULL;
4716 }
4717 if (opline->opcode == ZEND_PRE_INC) {
4718 func = ir_CONST_FC_FUNC(zend_jit_pre_inc_typed_ref);
4719 } else if (opline->opcode == ZEND_PRE_DEC) {
4720 func = ir_CONST_FC_FUNC(zend_jit_pre_dec_typed_ref);
4721 } else if (opline->opcode == ZEND_POST_INC) {
4722 func = ir_CONST_FC_FUNC(zend_jit_post_inc_typed_ref);
4723 } else if (opline->opcode == ZEND_POST_DEC) {
4724 func = ir_CONST_FC_FUNC(zend_jit_post_dec_typed_ref);
4725 } else {
4726 ZEND_UNREACHABLE();
4727 }
4728
4729 ir_CALL_2(IR_VOID, func, ref2, arg2);
4730 zend_jit_check_exception(jit);
4731 ir_END_list(merge_inputs);
4732
4733 ir_IF_FALSE(if_typed);
4734 ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
4735 ir_MERGE_WITH_EMPTY_FALSE(if_ref);
4736 ref = ir_PHI_2(IR_ADDR, ref2, ref);
4737 }
4738
4739 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
4740 jit_ZVAL_COPY(jit,
4741 res_addr,
4742 res_use_info,
4743 ZEND_ADDR_REF_ZVAL(ref), op1_info, 1);
4744 }
4745 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
4746 if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) {
4747 ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr);
4748 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_inc), ref, arg2);
4749 } else {
4750 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(increment_function), ref);
4751 }
4752 } else {
4753 if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) {
4754 ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr);
4755 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_dec), ref, arg2);
4756 } else {
4757 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(decrement_function), ref);
4758 }
4759 }
4760 if (may_throw) {
4761 zend_jit_check_exception(jit);
4762 }
4763 } else {
4764 ref = jit_Z_DVAL(jit, op1_addr);
4765 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
4766 jit_set_Z_DVAL(jit, res_addr, ref);
4767 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4768 }
4769 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
4770 op = IR_ADD;
4771 } else {
4772 op = IR_SUB;
4773 }
4774 ref = ir_BINARY_OP_D(op, ref, ir_CONST_DOUBLE(1.0));
4775 jit_set_Z_DVAL(jit, op1_def_addr, ref);
4776 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
4777 opline->result_type != IS_UNUSED) {
4778 jit_set_Z_DVAL(jit, res_addr, ref);
4779 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4780 }
4781 }
4782 ir_END_list(merge_inputs);
4783 ir_MERGE_list(merge_inputs);
4784 }
4785 if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) {
4786 return 0;
4787 }
4788 if (opline->result_type != IS_UNUSED) {
4789 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
4790 return 0;
4791 }
4792 }
4793 return 1;
4794 }
4795
zend_jit_math_long_long(zend_jit_ctx * jit,const zend_op * opline,uint8_t opcode,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint32_t res_info,uint32_t res_use_info,int may_overflow)4796 static int zend_jit_math_long_long(zend_jit_ctx *jit,
4797 const zend_op *opline,
4798 uint8_t opcode,
4799 zend_jit_addr op1_addr,
4800 zend_jit_addr op2_addr,
4801 zend_jit_addr res_addr,
4802 uint32_t res_info,
4803 uint32_t res_use_info,
4804 int may_overflow)
4805 {
4806 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
4807 ir_op op;
4808 ir_ref op1, op2, ref, if_overflow = IR_UNUSED;
4809
4810 if (opcode == ZEND_ADD) {
4811 op = may_overflow ? IR_ADD_OV : IR_ADD;
4812 } else if (opcode == ZEND_SUB) {
4813 op = may_overflow ? IR_SUB_OV : IR_SUB;
4814 } else if (opcode == ZEND_MUL) {
4815 op = may_overflow ? IR_MUL_OV : IR_MUL;
4816 } else {
4817 ZEND_UNREACHABLE();
4818 }
4819 op1 = jit_Z_LVAL(jit, op1_addr);
4820 op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
4821 ref = ir_BINARY_OP_L(op, op1, op2);
4822
4823 if (may_overflow) {
4824 if (res_info & MAY_BE_GUARD) {
4825 if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) {
4826 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
4827 uint32_t old_res_info;
4828 int32_t exit_point;
4829 const void *exit_addr;
4830
4831 if (opline->opcode == ZEND_ADD
4832 && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
4833 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
4834 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
4835 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
4836 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
4837 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
4838 } else if (opline->opcode == ZEND_SUB
4839 && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
4840 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
4841 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
4842 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
4843 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
4844 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
4845 } else {
4846 exit_point = zend_jit_trace_get_exit_point(opline, 0);
4847 }
4848
4849 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4850 if (!exit_addr) {
4851 return 0;
4852 }
4853 ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
4854 may_overflow = 0;
4855 } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
4856 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
4857 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
4858
4859 if (!exit_addr) {
4860 return 0;
4861 }
4862 ir_GUARD(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
4863 } else {
4864 ZEND_UNREACHABLE();
4865 }
4866 } else {
4867 if_overflow = ir_IF(ir_OVERFLOW(ref));
4868 ir_IF_FALSE(if_overflow);
4869 }
4870 }
4871
4872 if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
4873 jit_set_Z_LVAL(jit, res_addr, ref);
4874
4875 if (Z_MODE(res_addr) != IS_REG) {
4876 if (!zend_jit_same_addr(op1_addr, res_addr)) {
4877 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
4878 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
4879 }
4880 }
4881 }
4882 }
4883
4884 if (may_overflow) {
4885 ir_ref fast_path = IR_UNUSED;
4886
4887 if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
4888 fast_path = ir_END();
4889 ir_IF_TRUE_cold(if_overflow);
4890 }
4891 if (opcode == ZEND_ADD) {
4892 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
4893 if (Z_MODE(res_addr) == IS_REG) {
4894 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
4895 } else {
4896 #if SIZEOF_ZEND_LONG == 4
4897 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
4898 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
4899 #else
4900 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
4901 #endif
4902 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4903 }
4904 if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
4905 ir_MERGE_WITH(fast_path);
4906 }
4907 return 1;
4908 }
4909 op = IR_ADD;
4910 } else if (opcode == ZEND_SUB) {
4911 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
4912 if (Z_MODE(res_addr) == IS_REG) {
4913 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
4914 } else {
4915 #if SIZEOF_ZEND_LONG == 4
4916 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
4917 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
4918 #else
4919 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
4920 #endif
4921 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4922 }
4923 if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
4924 ir_MERGE_WITH(fast_path);
4925 }
4926 return 1;
4927 }
4928 op = IR_SUB;
4929 } else if (opcode == ZEND_MUL) {
4930 op = IR_MUL;
4931 } else {
4932 ZEND_UNREACHABLE();
4933 }
4934 #if 1
4935 /* reload */
4936 op1 = jit_Z_LVAL(jit, op1_addr);
4937 op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
4938 #endif
4939 #if 1
4940 /* disable CSE */
4941 ir_ref old_cse_limit = jit->ctx.fold_cse_limit;
4942 jit->ctx.fold_cse_limit = 0x7fffffff;
4943 #endif
4944 op1 = ir_INT2D(op1);
4945 op2 = ir_INT2D(op2);
4946 #if 1
4947 jit->ctx.fold_cse_limit = old_cse_limit;
4948 #endif
4949 ref = ir_BINARY_OP_D(op, op1, op2);
4950 jit_set_Z_DVAL(jit, res_addr, ref);
4951 if (Z_MODE(res_addr) != IS_REG) {
4952 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4953 }
4954 if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
4955 ir_MERGE_WITH(fast_path);
4956 }
4957 }
4958
4959 return 1;
4960 }
4961
zend_jit_math_long_double(zend_jit_ctx * jit,uint8_t opcode,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint32_t res_use_info)4962 static int zend_jit_math_long_double(zend_jit_ctx *jit,
4963 uint8_t opcode,
4964 zend_jit_addr op1_addr,
4965 zend_jit_addr op2_addr,
4966 zend_jit_addr res_addr,
4967 uint32_t res_use_info)
4968 {
4969 ir_op op;
4970 ir_ref op1, op2, ref;
4971
4972 if (opcode == ZEND_ADD) {
4973 op = IR_ADD;
4974 } else if (opcode == ZEND_SUB) {
4975 op = IR_SUB;
4976 } else if (opcode == ZEND_MUL) {
4977 op = IR_MUL;
4978 } else if (opcode == ZEND_DIV) {
4979 op = IR_DIV;
4980 } else {
4981 ZEND_UNREACHABLE();
4982 }
4983 op1 = jit_Z_LVAL(jit, op1_addr);
4984 op2 = jit_Z_DVAL(jit, op2_addr);
4985 ref = ir_BINARY_OP_D(op, ir_INT2D(op1), op2);
4986 jit_set_Z_DVAL(jit, res_addr, ref);
4987
4988 if (Z_MODE(res_addr) != IS_REG) {
4989 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
4990 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
4991 }
4992 }
4993 return 1;
4994 }
4995
zend_jit_math_double_long(zend_jit_ctx * jit,uint8_t opcode,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint32_t res_use_info)4996 static int zend_jit_math_double_long(zend_jit_ctx *jit,
4997 uint8_t opcode,
4998 zend_jit_addr op1_addr,
4999 zend_jit_addr op2_addr,
5000 zend_jit_addr res_addr,
5001 uint32_t res_use_info)
5002 {
5003 ir_op op;
5004 ir_ref op1, op2, ref;
5005
5006 if (opcode == ZEND_ADD) {
5007 op = IR_ADD;
5008 } else if (opcode == ZEND_SUB) {
5009 op = IR_SUB;
5010 } else if (opcode == ZEND_MUL) {
5011 op = IR_MUL;
5012 } else if (opcode == ZEND_DIV) {
5013 op = IR_DIV;
5014 } else {
5015 ZEND_UNREACHABLE();
5016 }
5017 op1 = jit_Z_DVAL(jit, op1_addr);
5018 op2 = jit_Z_LVAL(jit, op2_addr);
5019 ref = ir_BINARY_OP_D(op, op1, ir_INT2D(op2));
5020 jit_set_Z_DVAL(jit, res_addr, ref);
5021
5022 if (Z_MODE(res_addr) != IS_REG) {
5023 if (!zend_jit_same_addr(op1_addr, res_addr)) {
5024 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
5025 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
5026 }
5027 }
5028 }
5029 return 1;
5030 }
5031
zend_jit_math_double_double(zend_jit_ctx * jit,uint8_t opcode,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint32_t res_use_info)5032 static int zend_jit_math_double_double(zend_jit_ctx *jit,
5033 uint8_t opcode,
5034 zend_jit_addr op1_addr,
5035 zend_jit_addr op2_addr,
5036 zend_jit_addr res_addr,
5037 uint32_t res_use_info)
5038 {
5039 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
5040 ir_op op;
5041 ir_ref op1, op2, ref;
5042
5043 if (opcode == ZEND_ADD) {
5044 op = IR_ADD;
5045 } else if (opcode == ZEND_SUB) {
5046 op = IR_SUB;
5047 } else if (opcode == ZEND_MUL) {
5048 op = IR_MUL;
5049 } else if (opcode == ZEND_DIV) {
5050 op = IR_DIV;
5051 } else {
5052 ZEND_UNREACHABLE();
5053 }
5054 op1 = jit_Z_DVAL(jit, op1_addr);
5055 op2 = (same_ops) ? op1 : jit_Z_DVAL(jit, op2_addr);
5056 ref = ir_BINARY_OP_D(op, op1, op2);
5057 jit_set_Z_DVAL(jit, res_addr, ref);
5058
5059 if (Z_MODE(res_addr) != IS_REG) {
5060 if (!zend_jit_same_addr(op1_addr, res_addr)) {
5061 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
5062 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
5063 }
5064 }
5065 }
5066 return 1;
5067 }
5068
zend_jit_math_helper(zend_jit_ctx * jit,const zend_op * opline,uint8_t opcode,uint8_t op1_type,znode_op op1,zend_jit_addr op1_addr,uint32_t op1_info,uint8_t op2_type,znode_op op2,zend_jit_addr op2_addr,uint32_t op2_info,uint32_t res_var,zend_jit_addr res_addr,uint32_t res_info,uint32_t res_use_info,int may_overflow,int may_throw)5069 static int zend_jit_math_helper(zend_jit_ctx *jit,
5070 const zend_op *opline,
5071 uint8_t opcode,
5072 uint8_t op1_type,
5073 znode_op op1,
5074 zend_jit_addr op1_addr,
5075 uint32_t op1_info,
5076 uint8_t op2_type,
5077 znode_op op2,
5078 zend_jit_addr op2_addr,
5079 uint32_t op2_info,
5080 uint32_t res_var,
5081 zend_jit_addr res_addr,
5082 uint32_t res_info,
5083 uint32_t res_use_info,
5084 int may_overflow,
5085 int may_throw)
5086 {
5087 ir_ref if_op1_long = IR_UNUSED;
5088 ir_ref if_op1_double = IR_UNUSED;
5089 ir_ref if_op2_double = IR_UNUSED;
5090 ir_ref if_op1_long_op2_long = IR_UNUSED;
5091 ir_ref if_op1_long_op2_double = IR_UNUSED;
5092 ir_ref if_op1_double_op2_double = IR_UNUSED;
5093 ir_ref if_op1_double_op2_long = IR_UNUSED;
5094 ir_ref slow_inputs = IR_UNUSED;
5095 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
5096 ir_refs *end_inputs;
5097 ir_refs *res_inputs;
5098
5099 ir_refs_init(end_inputs, 6);
5100 ir_refs_init(res_inputs, 6);
5101
5102 if (Z_MODE(op1_addr) == IS_REG) {
5103 if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) {
5104 /* Force load */
5105 zend_jit_use_reg(jit, op1_addr);
5106 }
5107 } else if (Z_MODE(op2_addr) == IS_REG) {
5108 if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) {
5109 /* Force load */
5110 zend_jit_use_reg(jit, op2_addr);
5111 }
5112 }
5113
5114 if (Z_MODE(res_addr) == IS_REG) {
5115 jit->delay_var = Z_SSA_VAR(res_addr);
5116 jit->delay_refs = res_inputs;
5117 }
5118
5119 if ((res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG) && (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
5120 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
5121 if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
5122 ir_IF_TRUE(if_op1_long);
5123 }
5124 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
5125 if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
5126 ir_IF_TRUE(if_op1_long_op2_long);
5127 }
5128 if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
5129 return 0;
5130 }
5131 ir_refs_add(end_inputs, ir_END());
5132 if (if_op1_long) {
5133 ir_IF_FALSE_cold(if_op1_long);
5134 ir_END_list(slow_inputs);
5135 }
5136 if (if_op1_long_op2_long) {
5137 ir_IF_FALSE_cold(if_op1_long_op2_long);
5138 ir_END_list(slow_inputs);
5139 }
5140 } else if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
5141 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
5142 if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
5143 ir_IF_TRUE(if_op1_long);
5144 }
5145 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
5146 if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
5147 ir_IF_FALSE_cold(if_op1_long_op2_long);
5148 if (op2_info & MAY_BE_DOUBLE) {
5149 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
5150 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
5151 ir_IF_FALSE_cold(if_op1_long_op2_double);
5152 ir_END_list(slow_inputs);
5153 ir_IF_TRUE(if_op1_long_op2_double);
5154 }
5155 if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5156 return 0;
5157 }
5158 ir_refs_add(end_inputs, ir_END());
5159 } else {
5160 ir_END_list(slow_inputs);
5161 }
5162 ir_IF_TRUE(if_op1_long_op2_long);
5163 }
5164 if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
5165 return 0;
5166 }
5167 ir_refs_add(end_inputs, ir_END());
5168
5169 if (if_op1_long) {
5170 ir_IF_FALSE_cold(if_op1_long);
5171 }
5172
5173 if (op1_info & MAY_BE_DOUBLE) {
5174 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
5175 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
5176 ir_IF_FALSE_cold(if_op1_double);
5177 ir_END_list(slow_inputs);
5178 ir_IF_TRUE(if_op1_double);
5179 }
5180 if (op2_info & MAY_BE_DOUBLE) {
5181 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
5182 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
5183 ir_IF_TRUE(if_op1_double_op2_double);
5184 }
5185 if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5186 return 0;
5187 }
5188 ir_refs_add(end_inputs, ir_END());
5189 if (if_op1_double_op2_double) {
5190 ir_IF_FALSE_cold(if_op1_double_op2_double);
5191 }
5192 }
5193 if (!same_ops) {
5194 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
5195 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
5196 ir_IF_FALSE_cold(if_op1_double_op2_long);
5197 ir_END_list(slow_inputs);
5198 ir_IF_TRUE(if_op1_double_op2_long);
5199 }
5200 if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5201 return 0;
5202 }
5203 ir_refs_add(end_inputs, ir_END());
5204 } else if (if_op1_double_op2_double) {
5205 ir_END_list(slow_inputs);
5206 }
5207 } else if (if_op1_long) {
5208 ir_END_list(slow_inputs);
5209 }
5210 } else if ((op1_info & MAY_BE_DOUBLE) &&
5211 !(op1_info & MAY_BE_LONG) &&
5212 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
5213 (res_info & MAY_BE_DOUBLE)) {
5214 if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
5215 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
5216 ir_IF_FALSE_cold(if_op1_double);
5217 ir_END_list(slow_inputs);
5218 ir_IF_TRUE(if_op1_double);
5219 }
5220 if (op2_info & MAY_BE_DOUBLE) {
5221 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
5222 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
5223 ir_IF_TRUE(if_op1_double_op2_double);
5224 }
5225 if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5226 return 0;
5227 }
5228 ir_refs_add(end_inputs, ir_END());
5229 if (if_op1_double_op2_double) {
5230 ir_IF_FALSE_cold(if_op1_double_op2_double);
5231 }
5232 }
5233 if (!same_ops && (op2_info & MAY_BE_LONG)) {
5234 if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
5235 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
5236 ir_IF_FALSE_cold(if_op1_double_op2_long);
5237 ir_END_list(slow_inputs);
5238 ir_IF_TRUE(if_op1_double_op2_long);
5239 }
5240 if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5241 return 0;
5242 }
5243 ir_refs_add(end_inputs, ir_END());
5244 } else if (if_op1_double_op2_double) {
5245 ir_END_list(slow_inputs);
5246 }
5247 } else if ((op2_info & MAY_BE_DOUBLE) &&
5248 !(op2_info & MAY_BE_LONG) &&
5249 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
5250 (res_info & MAY_BE_DOUBLE)) {
5251 if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
5252 if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
5253 ir_IF_FALSE_cold(if_op2_double);
5254 ir_END_list(slow_inputs);
5255 ir_IF_TRUE(if_op2_double);
5256 }
5257 if (op1_info & MAY_BE_DOUBLE) {
5258 if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
5259 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
5260 ir_IF_TRUE(if_op1_double_op2_double);
5261 }
5262 if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5263 return 0;
5264 }
5265 ir_refs_add(end_inputs, ir_END());
5266 if (if_op1_double_op2_double) {
5267 ir_IF_FALSE_cold(if_op1_double_op2_double);
5268 }
5269 }
5270 if (!same_ops && (op1_info & MAY_BE_LONG)) {
5271 if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
5272 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
5273 ir_IF_FALSE_cold(if_op1_long_op2_double);
5274 ir_END_list(slow_inputs);
5275 ir_IF_TRUE(if_op1_long_op2_double);
5276 }
5277 if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
5278 return 0;
5279 }
5280 ir_refs_add(end_inputs, ir_END());
5281 } else if (if_op1_double_op2_double) {
5282 ir_END_list(slow_inputs);
5283 }
5284 }
5285
5286 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
5287 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
5288 ir_ref func, arg1, arg2, arg3;
5289
5290 if (slow_inputs) {
5291 ir_MERGE_list(slow_inputs);
5292 }
5293
5294 if (Z_MODE(op1_addr) == IS_REG) {
5295 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
5296 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
5297 return 0;
5298 }
5299 op1_addr = real_addr;
5300 }
5301 if (Z_MODE(op2_addr) == IS_REG) {
5302 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
5303 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
5304 return 0;
5305 }
5306 op2_addr = real_addr;
5307 }
5308 if (Z_MODE(res_addr) == IS_REG) {
5309 arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var));
5310 } else {
5311 arg1 = jit_ZVAL_ADDR(jit, res_addr);
5312 }
5313 arg2 = jit_ZVAL_ADDR(jit, op1_addr);
5314 arg3 = jit_ZVAL_ADDR(jit, op2_addr);
5315 jit_SET_EX_OPLINE(jit, opline);
5316 if (opcode == ZEND_ADD) {
5317 func = ir_CONST_FC_FUNC(add_function);
5318 } else if (opcode == ZEND_SUB) {
5319 func = ir_CONST_FC_FUNC(sub_function);
5320 } else if (opcode == ZEND_MUL) {
5321 func = ir_CONST_FC_FUNC(mul_function);
5322 } else if (opcode == ZEND_DIV) {
5323 func = ir_CONST_FC_FUNC(div_function);
5324 } else {
5325 ZEND_UNREACHABLE();
5326 }
5327 ir_CALL_3(IR_VOID, func, arg1, arg2, arg3);
5328
5329 jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
5330 jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
5331
5332 if (may_throw) {
5333 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
5334 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
5335 jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
5336 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
5337 zend_jit_check_exception_undef_result(jit, opline);
5338 } else {
5339 zend_jit_check_exception(jit);
5340 }
5341 }
5342 if (Z_MODE(res_addr) == IS_REG) {
5343 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
5344 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
5345 return 0;
5346 }
5347 }
5348 ir_refs_add(end_inputs, ir_END());
5349 }
5350
5351 if (end_inputs->count) {
5352 ir_MERGE_N(end_inputs->count, end_inputs->refs);
5353 }
5354
5355 if (Z_MODE(res_addr) == IS_REG) {
5356 ZEND_ASSERT(jit->delay_refs == res_inputs);
5357 ZEND_ASSERT(end_inputs->count == res_inputs->count);
5358 jit->delay_var = -1;
5359 jit->delay_refs = NULL;
5360 if (res_inputs->count == 1) {
5361 zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]);
5362 } else {
5363 ir_ref phi = ir_PHI_N((res_info & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE, res_inputs->count, res_inputs->refs);
5364 zend_jit_def_reg(jit, res_addr, phi);
5365 }
5366 }
5367
5368 return 1;
5369 }
5370
zend_jit_math(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_addr,uint32_t op2_info,zend_jit_addr op2_addr,uint32_t res_use_info,uint32_t res_info,zend_jit_addr res_addr,int may_overflow,int may_throw)5371 static int zend_jit_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
5372 {
5373 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
5374
5375 if (!zend_jit_math_helper(jit, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) {
5376 return 0;
5377 }
5378 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
5379 return 0;
5380 }
5381 return 1;
5382 }
5383
zend_jit_add_arrays(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_addr,uint32_t op2_info,zend_jit_addr op2_addr,zend_jit_addr res_addr)5384 static int zend_jit_add_arrays(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr)
5385 {
5386 ir_ref ref;
5387 ir_ref arg1 = jit_Z_PTR(jit, op1_addr);
5388 ir_ref arg2 = jit_Z_PTR(jit, op2_addr);
5389
5390 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_add_arrays_helper), arg1, arg2);
5391 jit_set_Z_PTR(jit, res_addr, ref);
5392 jit_set_Z_TYPE_INFO(jit, res_addr, IS_ARRAY_EX);
5393 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
5394 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
5395 return 1;
5396 }
5397
zend_jit_long_math_helper(zend_jit_ctx * jit,const zend_op * opline,uint8_t opcode,uint8_t op1_type,znode_op op1,zend_jit_addr op1_addr,uint32_t op1_info,zend_ssa_range * op1_range,uint8_t op2_type,znode_op op2,zend_jit_addr op2_addr,uint32_t op2_info,zend_ssa_range * op2_range,uint32_t res_var,zend_jit_addr res_addr,uint32_t res_info,uint32_t res_use_info,int may_throw)5398 static int zend_jit_long_math_helper(zend_jit_ctx *jit,
5399 const zend_op *opline,
5400 uint8_t opcode,
5401 uint8_t op1_type,
5402 znode_op op1,
5403 zend_jit_addr op1_addr,
5404 uint32_t op1_info,
5405 zend_ssa_range *op1_range,
5406 uint8_t op2_type,
5407 znode_op op2,
5408 zend_jit_addr op2_addr,
5409 uint32_t op2_info,
5410 zend_ssa_range *op2_range,
5411 uint32_t res_var,
5412 zend_jit_addr res_addr,
5413 uint32_t res_info,
5414 uint32_t res_use_info,
5415 int may_throw)
5416 {
5417 ir_ref ref = IR_UNUSED;
5418 ir_ref if_long1 = IR_UNUSED;
5419 ir_ref if_long2 = IR_UNUSED;
5420 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
5421 ir_refs *res_inputs;
5422
5423 ir_refs_init(res_inputs, 2);
5424
5425 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
5426 if_long1 = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
5427 ir_IF_TRUE(if_long1);
5428 }
5429 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
5430 if_long2 = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
5431 ir_IF_TRUE(if_long2);
5432 }
5433
5434 if (opcode == ZEND_SL) {
5435 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
5436 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
5437
5438 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
5439 if (EXPECTED(op2_lval > 0)) {
5440 ref = ir_CONST_LONG(0);
5441 } else {
5442 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
5443 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
5444 jit_SET_EX_OPLINE(jit, opline);
5445 ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift));
5446 if (Z_MODE(res_addr) == IS_REG) {
5447 ref = ir_CONST_LONG(0); // dead code
5448 }
5449 }
5450 } else {
5451 ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
5452 }
5453 } else {
5454 ref = jit_Z_LVAL(jit, op2_addr);
5455 if (!op2_range ||
5456 op2_range->min < 0 ||
5457 op2_range->max >= SIZEOF_ZEND_LONG * 8) {
5458
5459 ir_ref if_wrong, cold_path, ref2, if_ok;
5460 ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr);
5461
5462 if_wrong = ir_IF(ir_UGT(ref, ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1)));
5463 ir_IF_TRUE_cold(if_wrong);
5464 if_ok = ir_IF(ir_GE(ref, ir_CONST_LONG(0)));
5465 ir_IF_FALSE(if_ok);
5466 jit_SET_EX_OPLINE(jit, opline);
5467 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
5468 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
5469 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift));
5470 ir_IF_TRUE(if_ok);
5471 ref2 = ir_CONST_LONG(0);
5472 cold_path = ir_END();
5473 ir_IF_FALSE(if_wrong);
5474 ref = ir_SHL_L(op1_ref, ref);
5475 ir_MERGE_WITH(cold_path);
5476 ref = ir_PHI_2(IR_LONG, ref, ref2);
5477 } else {
5478 ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ref);
5479 }
5480 }
5481 } else if (opcode == ZEND_SR) {
5482 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
5483 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
5484
5485 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
5486 if (EXPECTED(op2_lval > 0)) {
5487 ref = ir_SAR_L(
5488 jit_Z_LVAL(jit, op1_addr),
5489 ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1));
5490 } else {
5491 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
5492 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
5493 jit_SET_EX_OPLINE(jit, opline);
5494 ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift));
5495 if (Z_MODE(res_addr) == IS_REG) {
5496 ref = ir_CONST_LONG(0); // dead code
5497 }
5498 }
5499 } else {
5500 ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
5501 }
5502 } else {
5503 ref = jit_Z_LVAL(jit, op2_addr);
5504 if (!op2_range ||
5505 op2_range->min < 0 ||
5506 op2_range->max >= SIZEOF_ZEND_LONG * 8) {
5507
5508 ir_ref if_wrong, cold_path, ref2, if_ok;
5509
5510 if_wrong = ir_IF(ir_UGT(ref, ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1)));
5511 ir_IF_TRUE_cold(if_wrong);
5512 if_ok = ir_IF(ir_GE(ref, ir_CONST_LONG(0)));
5513 ir_IF_FALSE(if_ok);
5514 jit_SET_EX_OPLINE(jit, opline);
5515 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
5516 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
5517 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift));
5518 ir_IF_TRUE(if_ok);
5519 ref2 = ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1);
5520 cold_path = ir_END();
5521 ir_IF_FALSE(if_wrong);
5522 ir_MERGE_WITH(cold_path);
5523 ref = ir_PHI_2(IR_LONG, ref, ref2);
5524 }
5525 ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ref);
5526 }
5527 } else if (opcode == ZEND_MOD) {
5528 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
5529 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
5530
5531 if (op2_lval == 0) {
5532 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
5533 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
5534 jit_SET_EX_OPLINE(jit, opline);
5535 ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_mod_by_zero));
5536 if (Z_MODE(res_addr) == IS_REG) {
5537 ref = ir_CONST_LONG(0); // dead code
5538 }
5539 } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) {
5540 ref = ir_AND_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval - 1));
5541 } else {
5542 ref = ir_MOD_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
5543 }
5544 } else {
5545 ir_ref zero_path = 0;
5546 ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr);
5547
5548 ref = jit_Z_LVAL(jit, op2_addr);
5549 if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) {
5550 ir_ref if_ok = ir_IF(ref);
5551 ir_IF_FALSE(if_ok);
5552 jit_SET_EX_OPLINE(jit, opline);
5553 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
5554 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
5555 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_mod_by_zero));
5556 ir_IF_TRUE(if_ok);
5557 }
5558
5559 /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */
5560 if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) {
5561 ir_ref if_minus_one = ir_IF(ir_EQ(ref, ir_CONST_LONG(-1)));
5562 ir_IF_TRUE_cold(if_minus_one);
5563 zero_path = ir_END();
5564 ir_IF_FALSE(if_minus_one);
5565 }
5566 ref = ir_MOD_L(op1_ref, ref);
5567
5568 if (zero_path) {
5569 ir_MERGE_WITH(zero_path);
5570 ref = ir_PHI_2(IR_LONG, ref, ir_CONST_LONG(0));
5571 }
5572 }
5573 } else {
5574 ir_op op;
5575 ir_ref op1, op2;
5576
5577 if (opcode == ZEND_BW_OR) {
5578 op = IR_OR;
5579 } else if (opcode == ZEND_BW_AND) {
5580 op = IR_AND;
5581 } else if (opcode == ZEND_BW_XOR) {
5582 op = IR_XOR;
5583 } else {
5584 ZEND_UNREACHABLE();
5585 }
5586 op1 = jit_Z_LVAL(jit, op1_addr);
5587 op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
5588 ref = ir_BINARY_OP_L(op, op1, op2);
5589 }
5590
5591 if (ref) {
5592 if (Z_MODE(res_addr) == IS_REG
5593 && ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))
5594 || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
5595 jit->delay_var = Z_SSA_VAR(res_addr);
5596 jit->delay_refs = res_inputs;
5597 }
5598 jit_set_Z_LVAL(jit, res_addr, ref);
5599 if (Z_MODE(res_addr) != IS_REG) {
5600 if (!zend_jit_same_addr(op1_addr, res_addr)) {
5601 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
5602 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
5603 }
5604 }
5605 }
5606 }
5607
5608 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) ||
5609 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
5610 ir_ref fast_path = ir_END();
5611 ir_ref func, arg1, arg2, arg3;
5612
5613 if (if_long2 && if_long1) {
5614 ir_ref ref;
5615 ir_IF_FALSE_cold(if_long2);
5616 ref = ir_END();
5617 ir_IF_FALSE_cold(if_long1);
5618 ir_MERGE_2(ref, ir_END());
5619 } else if (if_long1) {
5620 ir_IF_FALSE_cold(if_long1);
5621 } else if (if_long2) {
5622 ir_IF_FALSE_cold(if_long2);
5623 }
5624
5625 if (op1_info & MAY_BE_UNDEF) {
5626 ir_ref if_def, ref, ref2;
5627
5628 ref = jit_ZVAL_ADDR(jit, op1_addr);
5629 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
5630 ir_IF_FALSE_cold(if_def);
5631
5632 // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
5633 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
5634
5635 ref2 = jit_EG(uninitialized_zval);
5636 ir_MERGE_WITH_EMPTY_TRUE(if_def);
5637 ref = ir_PHI_2(IR_ADDR, ref2, ref);
5638 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
5639 }
5640
5641 if (op2_info & MAY_BE_UNDEF) {
5642 ir_ref if_def, ref, ref2;
5643
5644 ref = jit_ZVAL_ADDR(jit, op2_addr);
5645 if_def = jit_if_not_Z_TYPE(jit, op2_addr, IS_UNDEF);
5646 ir_IF_FALSE_cold(if_def);
5647
5648 // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op2.var)));
5649 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op2.var));
5650
5651 ref2 = jit_EG(uninitialized_zval);
5652 ir_MERGE_WITH_EMPTY_TRUE(if_def);
5653 ref = ir_PHI_2(IR_ADDR, ref2, ref);
5654 op2_addr = ZEND_ADDR_REF_ZVAL(ref);
5655 }
5656
5657 if (Z_MODE(op1_addr) == IS_REG) {
5658 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
5659 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
5660 return 0;
5661 }
5662 op1_addr = real_addr;
5663 }
5664 if (Z_MODE(op2_addr) == IS_REG) {
5665 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
5666 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
5667 return 0;
5668 }
5669 op2_addr = real_addr;
5670 }
5671 if (Z_MODE(res_addr) == IS_REG) {
5672 arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var));
5673 } else {
5674 arg1 = jit_ZVAL_ADDR(jit, res_addr);
5675 }
5676 arg2 = jit_ZVAL_ADDR(jit, op1_addr);
5677 arg3 = jit_ZVAL_ADDR(jit, op2_addr);
5678 jit_SET_EX_OPLINE(jit, opline);
5679 if (opcode == ZEND_BW_OR) {
5680 func = ir_CONST_FC_FUNC(bitwise_or_function);
5681 } else if (opcode == ZEND_BW_AND) {
5682 func = ir_CONST_FC_FUNC(bitwise_and_function);
5683 } else if (opcode == ZEND_BW_XOR) {
5684 func = ir_CONST_FC_FUNC(bitwise_xor_function);
5685 } else if (opcode == ZEND_SL) {
5686 func = ir_CONST_FC_FUNC(shift_left_function);
5687 } else if (opcode == ZEND_SR) {
5688 func = ir_CONST_FC_FUNC(shift_right_function);
5689 } else if (opcode == ZEND_MOD) {
5690 func = ir_CONST_FC_FUNC(mod_function);
5691 } else {
5692 ZEND_UNREACHABLE();
5693 }
5694 ir_CALL_3(IR_VOID, func, arg1, arg2, arg3);
5695
5696 if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) {
5697 /* compound assignment may decrement "op2" refcount */
5698 op2_info |= MAY_BE_RC1;
5699 }
5700
5701 jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
5702 jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
5703
5704 if (may_throw) {
5705 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
5706 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
5707 jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
5708 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
5709 zend_jit_check_exception_undef_result(jit, opline);
5710 } else {
5711 zend_jit_check_exception(jit);
5712 }
5713 }
5714
5715 if (Z_MODE(res_addr) == IS_REG) {
5716 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
5717 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
5718 return 0;
5719 }
5720 }
5721
5722 ir_MERGE_2(fast_path, ir_END());
5723
5724 if (Z_MODE(res_addr) == IS_REG) {
5725 ZEND_ASSERT(jit->delay_refs == res_inputs);
5726 ZEND_ASSERT(res_inputs->count == 2);
5727 jit->delay_var = -1;
5728 jit->delay_refs = NULL;
5729 if (res_inputs->count == 1) {
5730 zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]);
5731 } else {
5732 ir_ref phi = ir_PHI_N(IR_LONG, res_inputs->count, res_inputs->refs);
5733 zend_jit_def_reg(jit, res_addr, phi);
5734 }
5735 }
5736 }
5737
5738 return 1;
5739 }
5740
zend_jit_long_math(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_ssa_range * op1_range,zend_jit_addr op1_addr,uint32_t op2_info,zend_ssa_range * op2_range,zend_jit_addr op2_addr,uint32_t res_use_info,uint32_t res_info,zend_jit_addr res_addr,int may_throw)5741 static int zend_jit_long_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw)
5742 {
5743 ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG));
5744
5745 if (!zend_jit_long_math_helper(jit, opline, opline->opcode,
5746 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
5747 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
5748 opline->result.var, res_addr, res_info, res_use_info, may_throw)) {
5749 return 0;
5750 }
5751 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
5752 return 0;
5753 }
5754 return 1;
5755 }
5756
zend_jit_concat_helper(zend_jit_ctx * jit,const zend_op * opline,uint8_t op1_type,znode_op op1,zend_jit_addr op1_addr,uint32_t op1_info,uint8_t op2_type,znode_op op2,zend_jit_addr op2_addr,uint32_t op2_info,zend_jit_addr res_addr,int may_throw)5757 static int zend_jit_concat_helper(zend_jit_ctx *jit,
5758 const zend_op *opline,
5759 uint8_t op1_type,
5760 znode_op op1,
5761 zend_jit_addr op1_addr,
5762 uint32_t op1_info,
5763 uint8_t op2_type,
5764 znode_op op2,
5765 zend_jit_addr op2_addr,
5766 uint32_t op2_info,
5767 zend_jit_addr res_addr,
5768 int may_throw)
5769 {
5770 ir_ref if_op1_string = IR_UNUSED;
5771 ir_ref if_op2_string = IR_UNUSED;
5772 ir_ref fast_path = IR_UNUSED;
5773
5774 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
5775 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
5776 if_op1_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
5777 ir_IF_TRUE(if_op1_string);
5778 }
5779 if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
5780 if_op2_string = jit_if_Z_TYPE(jit, op2_addr, IS_STRING);
5781 ir_IF_TRUE(if_op2_string);
5782 }
5783 if (zend_jit_same_addr(op1_addr, res_addr)) {
5784 ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
5785 ir_ref arg2 = jit_ZVAL_ADDR(jit, op2_addr);
5786
5787 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_assign_concat_helper), arg1, arg2);
5788 /* concatenation with itself may reduce refcount */
5789 op2_info |= MAY_BE_RC1;
5790 } else {
5791 ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
5792 ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr);
5793 ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr);
5794
5795 if (op1_type == IS_CV || op1_type == IS_CONST) {
5796 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_concat_helper), arg1, arg2, arg3);
5797 } else {
5798 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_concat_tmp_helper), arg1, arg2, arg3);
5799 }
5800 }
5801 /* concatenation with empty string may increase refcount */
5802 op2_info |= MAY_BE_RCN;
5803 jit_FREE_OP(jit, op2_type, op2, op2_info, opline);
5804 if (if_op1_string || if_op2_string) {
5805 fast_path = ir_END();
5806 }
5807 }
5808 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) ||
5809 (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) {
5810 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
5811 if (if_op1_string && if_op2_string) {
5812 ir_IF_FALSE(if_op1_string);
5813 ir_MERGE_WITH_EMPTY_FALSE(if_op2_string);
5814 } else if (if_op1_string) {
5815 ir_IF_FALSE_cold(if_op1_string);
5816 } else if (if_op2_string) {
5817 ir_IF_FALSE_cold(if_op2_string);
5818 }
5819 }
5820 ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
5821 ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr);
5822 ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr);
5823
5824 jit_SET_EX_OPLINE(jit, opline);
5825 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(concat_function), arg1, arg2, arg3);
5826 /* concatenation with empty string may increase refcount */
5827 op1_info |= MAY_BE_RCN;
5828 op2_info |= MAY_BE_RCN;
5829 jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
5830 jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
5831 if (may_throw) {
5832 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
5833 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
5834 jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
5835 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
5836 zend_jit_check_exception_undef_result(jit, opline);
5837 } else {
5838 zend_jit_check_exception(jit);
5839 }
5840 }
5841 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
5842 ir_MERGE_WITH(fast_path);
5843 }
5844 }
5845 return 1;
5846 }
5847
zend_jit_concat(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,uint32_t op2_info,zend_jit_addr res_addr,int may_throw)5848 static int zend_jit_concat(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw)
5849 {
5850 zend_jit_addr op1_addr, op2_addr;
5851
5852 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
5853 ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING));
5854
5855 op1_addr = OP1_ADDR();
5856 op2_addr = OP2_ADDR();
5857
5858 return zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw);
5859 }
5860
zend_jit_assign_op(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,uint32_t op1_def_info,zend_ssa_range * op1_range,uint32_t op2_info,zend_ssa_range * op2_range,int may_overflow,int may_throw)5861 static int zend_jit_assign_op(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw)
5862 {
5863 int result = 1;
5864 zend_jit_addr op1_addr, op2_addr;
5865 ir_ref slow_path = IR_UNUSED;
5866
5867
5868 ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED);
5869 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
5870
5871 op1_addr = OP1_ADDR();
5872 op2_addr = OP2_ADDR();
5873
5874 if (op1_info & MAY_BE_REF) {
5875 ir_ref ref, ref2, arg2, op1_noref_path;
5876 ir_ref if_op1_ref = IR_UNUSED;
5877 ir_ref if_op1_typed = IR_UNUSED;
5878 binary_op_type binary_op = get_binary_op(opline->extended_value);
5879
5880 ref = jit_ZVAL_ADDR(jit, op1_addr);
5881 if_op1_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
5882 ir_IF_FALSE(if_op1_ref);
5883 op1_noref_path = ir_END();
5884 ir_IF_TRUE(if_op1_ref);
5885 ref2 = jit_Z_PTR_ref(jit, ref);
5886
5887 if_op1_typed = jit_if_TYPED_REF(jit, ref2);
5888 ir_IF_TRUE_cold(if_op1_typed);
5889
5890 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
5891 jit_SET_EX_OPLINE(jit, opline);
5892 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
5893 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
5894 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref_tmp),
5895 ref2, arg2, ir_CONST_FC_FUNC(binary_op));
5896 } else {
5897 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
5898 ref2, arg2, ir_CONST_FC_FUNC(binary_op));
5899 }
5900 zend_jit_check_exception(jit);
5901 slow_path = ir_END();
5902
5903 ir_IF_FALSE(if_op1_typed);
5904 ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
5905
5906 ir_MERGE_WITH(op1_noref_path);
5907 ref = ir_PHI_2(IR_ADDR, ref2, ref);
5908 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
5909 }
5910
5911 switch (opline->extended_value) {
5912 case ZEND_ADD:
5913 case ZEND_SUB:
5914 case ZEND_MUL:
5915 case ZEND_DIV:
5916 result = zend_jit_math_helper(jit, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw);
5917 break;
5918 case ZEND_BW_OR:
5919 case ZEND_BW_AND:
5920 case ZEND_BW_XOR:
5921 case ZEND_SL:
5922 case ZEND_SR:
5923 case ZEND_MOD:
5924 result = zend_jit_long_math_helper(jit, opline, opline->extended_value,
5925 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
5926 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
5927 opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw);
5928 break;
5929 case ZEND_CONCAT:
5930 result = zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw);
5931 break;
5932 default:
5933 ZEND_UNREACHABLE();
5934 }
5935
5936 if (op1_info & MAY_BE_REF) {
5937 ir_MERGE_WITH(slow_path);
5938 }
5939
5940 return result;
5941 }
5942
jit_ZVAL_DEREF_ref(zend_jit_ctx * jit,ir_ref ref)5943 static ir_ref jit_ZVAL_DEREF_ref(zend_jit_ctx *jit, ir_ref ref)
5944 {
5945 ir_ref if_ref, ref2;
5946
5947 if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_REFERENCE)));
5948 ir_IF_TRUE(if_ref);
5949 ref2 = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val));
5950 ir_MERGE_WITH_EMPTY_FALSE(if_ref);
5951 return ir_PHI_2(IR_ADDR, ref2, ref);
5952 }
5953
jit_ZVAL_DEREF(zend_jit_ctx * jit,zend_jit_addr addr)5954 static zend_jit_addr jit_ZVAL_DEREF(zend_jit_ctx *jit, zend_jit_addr addr)
5955 {
5956 ir_ref ref = jit_ZVAL_ADDR(jit, addr);
5957 ref = jit_ZVAL_DEREF_ref(jit, ref);
5958 return ZEND_ADDR_REF_ZVAL(ref);
5959 }
5960
jit_ZVAL_INDIRECT_DEREF_ref(zend_jit_ctx * jit,ir_ref ref)5961 static ir_ref jit_ZVAL_INDIRECT_DEREF_ref(zend_jit_ctx *jit, ir_ref ref)
5962 {
5963 ir_ref if_ref, ref2;
5964
5965 if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_INDIRECT)));
5966 ir_IF_TRUE(if_ref);
5967 ref2 = jit_Z_PTR_ref(jit, ref);
5968 ir_MERGE_WITH_EMPTY_FALSE(if_ref);
5969 return ir_PHI_2(IR_ADDR, ref2, ref);
5970 }
5971
jit_ZVAL_INDIRECT_DEREF(zend_jit_ctx * jit,zend_jit_addr addr)5972 static zend_jit_addr jit_ZVAL_INDIRECT_DEREF(zend_jit_ctx *jit, zend_jit_addr addr)
5973 {
5974 ir_ref ref = jit_ZVAL_ADDR(jit, addr);
5975 ref = jit_ZVAL_INDIRECT_DEREF_ref(jit, ref);
5976 return ZEND_ADDR_REF_ZVAL(ref);
5977 }
5978
zend_jit_simple_assign(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr var_addr,uint32_t var_info,uint32_t var_def_info,uint8_t val_type,zend_jit_addr val_addr,uint32_t val_info,zend_jit_addr res_addr,bool check_exception)5979 static int zend_jit_simple_assign(zend_jit_ctx *jit,
5980 const zend_op *opline,
5981 zend_jit_addr var_addr,
5982 uint32_t var_info,
5983 uint32_t var_def_info,
5984 uint8_t val_type,
5985 zend_jit_addr val_addr,
5986 uint32_t val_info,
5987 zend_jit_addr res_addr,
5988 bool check_exception)
5989 {
5990 ir_ref end_inputs = IR_UNUSED;
5991
5992 if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
5993 zval *zv = Z_ZV(val_addr);
5994
5995 if (!res_addr) {
5996 jit_ZVAL_COPY_CONST(jit,
5997 var_addr,
5998 var_info, var_def_info,
5999 zv, 1);
6000 } else {
6001 jit_ZVAL_COPY_CONST(jit,
6002 var_addr,
6003 var_info, var_def_info,
6004 zv, 1);
6005 jit_ZVAL_COPY_CONST(jit,
6006 res_addr,
6007 -1, var_def_info,
6008 zv, 1);
6009 }
6010 } else {
6011 if (val_info & MAY_BE_UNDEF) {
6012 ir_ref if_def, ret;
6013
6014 if_def = jit_if_not_Z_TYPE(jit, val_addr, IS_UNDEF);
6015 ir_IF_FALSE_cold(if_def);
6016
6017 jit_set_Z_TYPE_INFO(jit, var_addr, IS_NULL);
6018 if (res_addr) {
6019 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
6020 }
6021 jit_SET_EX_OPLINE(jit, opline);
6022
6023 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL);
6024 // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
6025 ret = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(Z_OFFSET(val_addr)));
6026
6027 if (check_exception) {
6028 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
6029 }
6030
6031 ir_END_list(end_inputs);
6032 ir_IF_TRUE(if_def);
6033 }
6034 if (val_info & MAY_BE_REF) {
6035 if (val_type == IS_CV) {
6036 ir_ref ref = jit_ZVAL_ADDR(jit, val_addr);
6037 ref = jit_ZVAL_DEREF_ref(jit, ref);
6038 val_addr = ZEND_ADDR_REF_ZVAL(ref);
6039 } else {
6040 ir_ref ref, type, if_ref, ref2, refcount, if_not_zero;
6041
6042 ref = jit_ZVAL_ADDR(jit, val_addr);
6043 type = jit_Z_TYPE_ref(jit, ref);
6044 if_ref = ir_IF(ir_EQ(type, ir_CONST_U8(IS_REFERENCE)));
6045
6046 ir_IF_TRUE_cold(if_ref);
6047 ref = jit_Z_PTR_ref(jit, ref);
6048 ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
6049 if (!res_addr) {
6050 jit_ZVAL_COPY(jit,
6051 var_addr,
6052 var_info,
6053 ZEND_ADDR_REF_ZVAL(ref2), val_info, 1);
6054 } else {
6055 jit_ZVAL_COPY_2(jit,
6056 res_addr,
6057 var_addr,
6058 var_info,
6059 ZEND_ADDR_REF_ZVAL(ref2), val_info, 2);
6060 }
6061
6062 refcount = jit_GC_DELREF(jit, ref);
6063 if_not_zero = ir_IF(refcount);
6064 ir_IF_FALSE(if_not_zero);
6065 // TODO: instead of dtor() call and ADDREF above, we may call efree() and move addref at "true" path ???
6066 // This is related to GH-10168 (keep this before GH-10168 is completely closed)
6067 // jit_EFREE(jit, ref, sizeof(zend_reference), NULL, NULL);
6068 jit_ZVAL_DTOR(jit, ref, val_info, opline);
6069 ir_END_list(end_inputs);
6070 ir_IF_TRUE(if_not_zero);
6071 ir_END_list(end_inputs);
6072
6073 ir_IF_FALSE(if_ref);
6074 }
6075 }
6076
6077 if (!res_addr) {
6078 jit_ZVAL_COPY(jit,
6079 var_addr,
6080 var_info,
6081 val_addr, val_info, val_type == IS_CV);
6082 } else {
6083 jit_ZVAL_COPY_2(jit,
6084 res_addr,
6085 var_addr,
6086 var_info,
6087 val_addr, val_info, val_type == IS_CV ? 2 : 1);
6088 }
6089 }
6090
6091 if (end_inputs) {
6092 ir_END_list(end_inputs);
6093 ir_MERGE_list(end_inputs);
6094 }
6095
6096 return 1;
6097 }
6098
zend_jit_assign_to_variable_call(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr __var_use_addr,zend_jit_addr var_addr,uint32_t __var_info,uint32_t __var_def_info,uint8_t val_type,zend_jit_addr val_addr,uint32_t val_info,zend_jit_addr __res_addr,bool __check_exception)6099 static int zend_jit_assign_to_variable_call(zend_jit_ctx *jit,
6100 const zend_op *opline,
6101 zend_jit_addr __var_use_addr,
6102 zend_jit_addr var_addr,
6103 uint32_t __var_info,
6104 uint32_t __var_def_info,
6105 uint8_t val_type,
6106 zend_jit_addr val_addr,
6107 uint32_t val_info,
6108 zend_jit_addr __res_addr,
6109 bool __check_exception)
6110 {
6111 jit_stub_id func;
6112 ir_ref undef_path = IR_UNUSED;
6113
6114 if (val_info & MAY_BE_UNDEF) {
6115 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
6116 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
6117 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
6118
6119 if (!exit_addr) {
6120 return 0;
6121 }
6122
6123 jit_guard_not_Z_TYPE(jit, val_addr, IS_UNDEF, exit_addr);
6124 } else {
6125 ir_ref if_def;
6126
6127 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP);
6128 if_def = ir_IF(jit_Z_TYPE(jit, val_addr));
6129 ir_IF_FALSE_cold(if_def);
6130 jit_SET_EX_OPLINE(jit, opline);
6131 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(Z_OFFSET(val_addr)));
6132
6133 ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_assign_const, IR_FASTCALL_FUNC),
6134 jit_ZVAL_ADDR(jit, var_addr),
6135 jit_EG(uninitialized_zval));
6136
6137 undef_path = ir_END();
6138 ir_IF_TRUE(if_def);
6139 }
6140 }
6141
6142 if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
6143 func = jit_stub_assign_tmp;
6144 } else if (val_type == IS_CONST) {
6145 func = jit_stub_assign_const;
6146 } else if (val_type == IS_TMP_VAR) {
6147 func = jit_stub_assign_tmp;
6148 } else if (val_type == IS_VAR) {
6149 if (!(val_info & MAY_BE_REF)) {
6150 func = jit_stub_assign_tmp;
6151 } else {
6152 func = jit_stub_assign_var;
6153 }
6154 } else if (val_type == IS_CV) {
6155 if (!(val_info & MAY_BE_REF)) {
6156 func = jit_stub_assign_cv_noref;
6157 } else {
6158 func = jit_stub_assign_cv;
6159 }
6160 } else {
6161 ZEND_UNREACHABLE();
6162 }
6163
6164 if (opline) {
6165 jit_SET_EX_OPLINE(jit, opline);
6166 }
6167
6168 ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, func, IR_FASTCALL_FUNC),
6169 jit_ZVAL_ADDR(jit, var_addr),
6170 jit_ZVAL_ADDR(jit, val_addr));
6171
6172 if (undef_path) {
6173 ir_MERGE_WITH(undef_path);
6174 }
6175
6176 return 1;
6177 }
6178
zend_jit_assign_to_variable(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr var_use_addr,zend_jit_addr var_addr,uint32_t var_info,uint32_t var_def_info,uint8_t val_type,zend_jit_addr val_addr,uint32_t val_info,zend_jit_addr res_addr,zend_jit_addr ref_addr,bool check_exception)6179 static int zend_jit_assign_to_variable(zend_jit_ctx *jit,
6180 const zend_op *opline,
6181 zend_jit_addr var_use_addr,
6182 zend_jit_addr var_addr,
6183 uint32_t var_info,
6184 uint32_t var_def_info,
6185 uint8_t val_type,
6186 zend_jit_addr val_addr,
6187 uint32_t val_info,
6188 zend_jit_addr res_addr,
6189 zend_jit_addr ref_addr,
6190 bool check_exception)
6191 {
6192 ir_ref if_refcounted = IR_UNUSED;
6193 ir_ref simple_inputs = IR_UNUSED;
6194 bool done = 0;
6195 zend_jit_addr real_res_addr = 0;
6196 ir_refs *end_inputs;
6197 ir_refs *res_inputs;
6198
6199 ir_refs_init(end_inputs, 6);
6200 ir_refs_init(res_inputs, 6);
6201
6202 if (Z_MODE(val_addr) == IS_REG && jit->ra[Z_SSA_VAR(val_addr)].ref == IR_NULL) {
6203 /* Force load */
6204 zend_jit_use_reg(jit, val_addr);
6205 }
6206
6207 if (Z_MODE(var_addr) == IS_REG) {
6208 jit->delay_var = Z_SSA_VAR(var_addr);
6209 jit->delay_refs = res_inputs;
6210 if (Z_MODE(res_addr) == IS_REG) {
6211 real_res_addr = res_addr;
6212 res_addr = 0;
6213 }
6214 } else if (Z_MODE(res_addr) == IS_REG) {
6215 jit->delay_var = Z_SSA_VAR(res_addr);
6216 jit->delay_refs = res_inputs;
6217 }
6218
6219 if ((var_info & MAY_BE_REF) || ref_addr) {
6220 ir_ref ref = 0, if_ref = 0, ref2, arg2, if_typed, non_ref_path;
6221 uintptr_t func;
6222
6223 if (!ref_addr) {
6224 ref = jit_ZVAL_ADDR(jit, var_use_addr);
6225 if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
6226 ir_IF_TRUE(if_ref);
6227 ref2 = jit_Z_PTR_ref(jit, ref);
6228 } else {
6229 ref2 = jit_ZVAL_ADDR(jit, ref_addr);
6230 }
6231 if_typed = jit_if_TYPED_REF(jit, ref2);
6232 ir_IF_TRUE_cold(if_typed);
6233 jit_SET_EX_OPLINE(jit, opline);
6234 if (Z_MODE(val_addr) == IS_REG) {
6235 ZEND_ASSERT(opline->opcode == ZEND_ASSIGN);
6236 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
6237 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
6238 return 0;
6239 }
6240 arg2 = jit_ZVAL_ADDR(jit, real_addr);
6241 } else {
6242 arg2 = jit_ZVAL_ADDR(jit, val_addr);
6243 }
6244 if (!res_addr) {
6245 if (val_type == IS_CONST) {
6246 func = (uintptr_t)zend_jit_assign_const_to_typed_ref;
6247 } else if (val_type == IS_TMP_VAR) {
6248 func = (uintptr_t)zend_jit_assign_tmp_to_typed_ref;
6249 } else if (val_type == IS_VAR) {
6250 func = (uintptr_t)zend_jit_assign_var_to_typed_ref;
6251 } else if (val_type == IS_CV) {
6252 func = (uintptr_t)zend_jit_assign_cv_to_typed_ref;
6253 } else {
6254 ZEND_UNREACHABLE();
6255 }
6256 ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2);
6257 } else {
6258 if (val_type == IS_CONST) {
6259 func = (uintptr_t)zend_jit_assign_const_to_typed_ref2;
6260 } else if (val_type == IS_TMP_VAR) {
6261 func = (uintptr_t)zend_jit_assign_tmp_to_typed_ref2;
6262 } else if (val_type == IS_VAR) {
6263 func = (uintptr_t)zend_jit_assign_var_to_typed_ref2;
6264 } else if (val_type == IS_CV) {
6265 func = (uintptr_t)zend_jit_assign_cv_to_typed_ref2;
6266 } else {
6267 ZEND_UNREACHABLE();
6268 }
6269 ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2, jit_ZVAL_ADDR(jit, res_addr));
6270 }
6271 if (check_exception) {
6272 zend_jit_check_exception(jit);
6273 }
6274 ir_refs_add(end_inputs, ir_END());
6275
6276 if (!ref_addr) {
6277 ir_IF_FALSE(if_ref);
6278 non_ref_path = ir_END();
6279 ir_IF_FALSE(if_typed);
6280 ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
6281 ir_MERGE_WITH(non_ref_path);
6282 ref = ir_PHI_2(IR_ADDR, ref2, ref);
6283 var_addr = var_use_addr = ZEND_ADDR_REF_ZVAL(ref);
6284 } else {
6285 ir_IF_FALSE(if_typed);
6286 }
6287 }
6288
6289 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
6290 ir_ref ref, counter, if_not_zero;
6291
6292 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
6293 if_refcounted = jit_if_REFCOUNTED(jit, var_use_addr);
6294 ir_IF_FALSE(if_refcounted);
6295 ir_END_list(simple_inputs);
6296 ir_IF_TRUE_cold(if_refcounted);
6297 } else if (RC_MAY_BE_1(var_info)) {
6298 done = 1;
6299 }
6300 ref = jit_Z_PTR(jit, var_use_addr);
6301 if (RC_MAY_BE_1(var_info)) {
6302 if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0)) {
6303 return 0;
6304 }
6305 counter = jit_GC_DELREF(jit, ref);
6306
6307 if_not_zero = ir_IF(counter);
6308 ir_IF_FALSE(if_not_zero);
6309 jit_ZVAL_DTOR(jit, ref, var_info, opline);
6310 if (check_exception) {
6311 zend_jit_check_exception(jit);
6312 }
6313 ir_refs_add(end_inputs, ir_END());
6314 ir_IF_TRUE(if_not_zero);
6315 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
6316 ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
6317 ir_IF_FALSE(if_may_leak);
6318 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
6319
6320 if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
6321 ZEND_ASSERT(jit->delay_refs == res_inputs);
6322 ZEND_ASSERT(res_inputs->count > 0);
6323 ir_refs_add(res_inputs, res_inputs->refs[res_inputs->count - 1]);
6324 }
6325 if (check_exception && (val_info & MAY_BE_UNDEF)) {
6326 zend_jit_check_exception(jit);
6327 }
6328 ir_refs_add(end_inputs, ir_END());
6329 ir_IF_TRUE(if_may_leak);
6330 }
6331 if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
6332 ZEND_ASSERT(jit->delay_refs == res_inputs);
6333 ZEND_ASSERT(res_inputs->count > 0);
6334 ir_refs_add(res_inputs, res_inputs->refs[res_inputs->count - 1]);
6335 }
6336 if (check_exception && (val_info & MAY_BE_UNDEF)) {
6337 zend_jit_check_exception(jit);
6338 }
6339 ir_refs_add(end_inputs, ir_END());
6340 } else /* if (RC_MAY_BE_N(var_info)) */ {
6341 jit_GC_DELREF(jit, ref);
6342 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) {
6343 ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
6344 ir_IF_FALSE(if_may_leak);
6345 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
6346 ir_END_list(simple_inputs);
6347 ir_IF_TRUE(if_may_leak);
6348 }
6349 ir_END_list(simple_inputs);
6350 }
6351 }
6352
6353 if (simple_inputs) {
6354 ir_MERGE_list(simple_inputs);
6355 }
6356
6357 if (!done) {
6358 if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, check_exception)) {
6359 return 0;
6360 }
6361 if (end_inputs->count) {
6362 ir_refs_add(end_inputs, ir_END());
6363 }
6364 }
6365
6366 if (end_inputs->count) {
6367 ir_MERGE_N(end_inputs->count, end_inputs->refs);
6368 }
6369
6370 if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
6371 ir_ref phi;
6372
6373 ZEND_ASSERT(jit->delay_refs == res_inputs);
6374 ZEND_ASSERT(end_inputs->count == res_inputs->count || (end_inputs->count == 0 && res_inputs->count == 1));
6375 jit->delay_var = -1;
6376 jit->delay_refs = NULL;
6377 if (res_inputs->count == 1) {
6378 phi = res_inputs->refs[0];
6379 } else {
6380 phi = ir_PHI_N((var_def_info & MAY_BE_LONG & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE,
6381 res_inputs->count, res_inputs->refs);
6382 }
6383 if (Z_MODE(var_addr) == IS_REG) {
6384 if ((var_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || ref_addr) {
6385 phi = ir_emit2(&jit->ctx, IR_OPT(IR_COPY, jit->ctx.ir_base[phi].type), phi, 1);
6386 }
6387 zend_jit_def_reg(jit, var_addr, phi);
6388 if (real_res_addr) {
6389 if (var_def_info & MAY_BE_LONG) {
6390 jit_set_Z_LVAL(jit, real_res_addr, jit_Z_LVAL(jit, var_addr));
6391 } else {
6392 jit_set_Z_DVAL(jit, real_res_addr, jit_Z_DVAL(jit, var_addr));
6393 }
6394 }
6395 } else {
6396 zend_jit_def_reg(jit, res_addr, phi);
6397 }
6398 }
6399
6400 return 1;
6401 }
6402
zend_jit_qm_assign(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_addr,zend_jit_addr op1_def_addr,uint32_t res_use_info,uint32_t res_info,zend_jit_addr res_addr)6403 static int zend_jit_qm_assign(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr)
6404 {
6405 if (op1_addr != op1_def_addr) {
6406 if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
6407 return 0;
6408 }
6409 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
6410 op1_addr = op1_def_addr;
6411 }
6412 }
6413
6414 if (!zend_jit_simple_assign(jit, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 1)) {
6415 return 0;
6416 }
6417 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
6418 return 0;
6419 }
6420 return 1;
6421 }
6422
zend_jit_assign(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_use_addr,uint32_t op1_def_info,zend_jit_addr op1_addr,uint32_t op2_info,zend_jit_addr op2_addr,zend_jit_addr op2_def_addr,uint32_t res_info,zend_jit_addr res_addr,zend_jit_addr ref_addr,int may_throw)6423 static int zend_jit_assign(zend_jit_ctx *jit,
6424 const zend_op *opline,
6425 uint32_t op1_info,
6426 zend_jit_addr op1_use_addr,
6427 uint32_t op1_def_info,
6428 zend_jit_addr op1_addr,
6429 uint32_t op2_info,
6430 zend_jit_addr op2_addr,
6431 zend_jit_addr op2_def_addr,
6432 uint32_t res_info,
6433 zend_jit_addr res_addr,
6434 zend_jit_addr ref_addr,
6435 int may_throw)
6436 {
6437 ZEND_ASSERT(opline->op1_type == IS_CV);
6438
6439 if (op2_addr != op2_def_addr) {
6440 if (!zend_jit_update_regs(jit, opline->op2.var, op2_addr, op2_def_addr, op2_info)) {
6441 return 0;
6442 }
6443 if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) {
6444 op2_addr = op2_def_addr;
6445 }
6446 }
6447
6448 if (Z_MODE(op1_addr) != IS_REG
6449 && Z_MODE(op1_use_addr) == IS_REG
6450 && !Z_LOAD(op1_use_addr)
6451 && !Z_STORE(op1_use_addr)) {
6452 /* Force type update */
6453 op1_info |= MAY_BE_UNDEF;
6454 }
6455 if (!zend_jit_assign_to_variable(jit, opline, op1_use_addr, op1_addr, op1_info, op1_def_info,
6456 opline->op2_type, op2_addr, op2_info, res_addr, ref_addr, may_throw)) {
6457 return 0;
6458 }
6459 if (Z_MODE(op1_addr) == IS_REG) {
6460 if (Z_STORE(op1_addr)) {
6461 if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) {
6462 return 0;
6463 }
6464 } else if ((op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
6465 && Z_MODE(op1_use_addr) == IS_MEM_ZVAL
6466 && Z_REG(op1_use_addr) == ZREG_FP
6467 && EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)) < jit->current_op_array->last_var) {
6468 /* We have to update type of CV because it may be captured by exception backtrace or released on RETURN */
6469 if ((op1_def_info & MAY_BE_ANY) == MAY_BE_LONG) {
6470 jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_LONG);
6471 if (JIT_G(current_frame)) {
6472 SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)), IS_LONG, 1);
6473 }
6474 } else if ((op1_def_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
6475 jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_DOUBLE);
6476 if (JIT_G(current_frame)) {
6477 SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)), IS_DOUBLE, 1);
6478 }
6479 } else {
6480 ZEND_UNREACHABLE();
6481 }
6482 }
6483 }
6484 if (opline->result_type != IS_UNUSED) {
6485 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
6486 return 0;
6487 }
6488 }
6489
6490 return 1;
6491 }
6492
zend_jit_cmp_op(const zend_op * opline)6493 static ir_op zend_jit_cmp_op(const zend_op *opline)
6494 {
6495 ir_op op;
6496
6497 switch (opline->opcode) {
6498 case ZEND_IS_EQUAL:
6499 case ZEND_IS_IDENTICAL:
6500 case ZEND_CASE:
6501 case ZEND_CASE_STRICT:
6502 op = IR_EQ;
6503 break;
6504 case ZEND_IS_NOT_EQUAL:
6505 case ZEND_IS_NOT_IDENTICAL:
6506 op = IR_NE;
6507 break;
6508 case ZEND_IS_SMALLER:
6509 op = IR_LT;
6510 break;
6511 case ZEND_IS_SMALLER_OR_EQUAL:
6512 op = IR_LE;
6513 break;
6514 default:
6515 ZEND_UNREACHABLE();
6516 }
6517 return op;
6518 }
6519
zend_jit_cmp_long_long(zend_jit_ctx * jit,const zend_op * opline,zend_ssa_range * op1_range,zend_jit_addr op1_addr,zend_ssa_range * op2_range,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr,bool skip_comparison)6520 static ir_ref zend_jit_cmp_long_long(zend_jit_ctx *jit,
6521 const zend_op *opline,
6522 zend_ssa_range *op1_range,
6523 zend_jit_addr op1_addr,
6524 zend_ssa_range *op2_range,
6525 zend_jit_addr op2_addr,
6526 zend_jit_addr res_addr,
6527 uint8_t smart_branch_opcode,
6528 uint32_t target_label,
6529 uint32_t target_label2,
6530 const void *exit_addr,
6531 bool skip_comparison)
6532 {
6533 ir_ref ref;
6534 bool result;
6535
6536 if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) {
6537 if (!smart_branch_opcode ||
6538 smart_branch_opcode == ZEND_JMPZ_EX ||
6539 smart_branch_opcode == ZEND_JMPNZ_EX) {
6540 jit_set_Z_TYPE_INFO(jit, res_addr, result ? IS_TRUE : IS_FALSE);
6541 }
6542 if (smart_branch_opcode && !exit_addr) {
6543 if (smart_branch_opcode == ZEND_JMPZ ||
6544 smart_branch_opcode == ZEND_JMPZ_EX) {
6545 return jit_IF_ex(jit, IR_FALSE, result ? target_label : target_label2);
6546 } else if (smart_branch_opcode == ZEND_JMPNZ ||
6547 smart_branch_opcode == ZEND_JMPNZ_EX) {
6548 return jit_IF_ex(jit, IR_TRUE, result ? target_label : target_label2);
6549 } else {
6550 ZEND_UNREACHABLE();
6551 }
6552 }
6553 if (opline->opcode != ZEND_IS_IDENTICAL
6554 && opline->opcode != ZEND_IS_NOT_IDENTICAL
6555 && opline->opcode != ZEND_CASE_STRICT) {
6556 return ir_END();
6557 } else {
6558 return IR_NULL; /* success */
6559 }
6560 }
6561
6562 ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_LVAL(jit, op1_addr), jit_Z_LVAL(jit, op2_addr));
6563
6564 if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
6565 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
6566 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
6567 }
6568 if (exit_addr) {
6569 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
6570 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
6571 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6572 } else {
6573 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6574 }
6575 } else {
6576 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
6577 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6578 } else {
6579 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6580 }
6581 }
6582 } else if (smart_branch_opcode) {
6583 return jit_IF_ex(jit, ref,
6584 (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
6585 }
6586
6587 if (opline->opcode != ZEND_IS_IDENTICAL
6588 && opline->opcode != ZEND_IS_NOT_IDENTICAL
6589 && opline->opcode != ZEND_CASE_STRICT) {
6590 return ir_END();
6591 } else {
6592 return IR_NULL; /* success */
6593 }
6594 }
6595
zend_jit_cmp_long_double(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)6596 static ir_ref zend_jit_cmp_long_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
6597 {
6598 ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), ir_INT2D(jit_Z_LVAL(jit, op1_addr)), jit_Z_DVAL(jit, op2_addr));
6599
6600 if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
6601 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
6602 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
6603 }
6604 if (exit_addr) {
6605 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
6606 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6607 } else {
6608 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6609 }
6610 } else if (smart_branch_opcode) {
6611 return jit_IF_ex(jit, ref,
6612 (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
6613 }
6614 return ir_END();
6615 }
6616
zend_jit_cmp_double_long(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)6617 static ir_ref zend_jit_cmp_double_long(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
6618 {
6619 ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), ir_INT2D(jit_Z_LVAL(jit, op2_addr)));
6620
6621 if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
6622 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
6623 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
6624 }
6625 if (exit_addr) {
6626 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
6627 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6628 } else {
6629 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6630 }
6631 } else if (smart_branch_opcode) {
6632 return jit_IF_ex(jit, ref,
6633 (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
6634 }
6635 return ir_END();
6636 }
6637
zend_jit_cmp_double_double(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr op1_addr,zend_jit_addr op2_addr,zend_jit_addr res_addr,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)6638 static ir_ref zend_jit_cmp_double_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
6639 {
6640 ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), jit_Z_DVAL(jit, op2_addr));
6641
6642 if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
6643 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
6644 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
6645 }
6646 if (exit_addr) {
6647 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
6648 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
6649 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6650 } else {
6651 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6652 }
6653 } else {
6654 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
6655 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6656 } else {
6657 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6658 }
6659 }
6660 } else if (smart_branch_opcode) {
6661 return jit_IF_ex(jit, ref,
6662 (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
6663 }
6664 if (opline->opcode != ZEND_IS_IDENTICAL
6665 && opline->opcode != ZEND_IS_NOT_IDENTICAL
6666 && opline->opcode != ZEND_CASE_STRICT) {
6667 return ir_END();
6668 } else {
6669 return IR_NULL; /* success */
6670 }
6671 }
6672
zend_jit_cmp_slow(zend_jit_ctx * jit,ir_ref ref,const zend_op * opline,zend_jit_addr res_addr,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)6673 static ir_ref zend_jit_cmp_slow(zend_jit_ctx *jit, ir_ref ref, const zend_op *opline, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
6674 {
6675 ref = ir_CMP_OP(zend_jit_cmp_op(opline), ref, ir_CONST_I32(0));
6676
6677 if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
6678 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
6679 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
6680 }
6681 if (exit_addr) {
6682 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
6683 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
6684 } else {
6685 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
6686 }
6687 } else if (smart_branch_opcode) {
6688 return jit_IF_ex(jit, ref,
6689 (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
6690 }
6691
6692 return ir_END();
6693 }
6694
zend_jit_cmp(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_ssa_range * op1_range,zend_jit_addr op1_addr,uint32_t op2_info,zend_ssa_range * op2_range,zend_jit_addr op2_addr,zend_jit_addr res_addr,int may_throw,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr,bool skip_comparison)6695 static int zend_jit_cmp(zend_jit_ctx *jit,
6696 const zend_op *opline,
6697 uint32_t op1_info,
6698 zend_ssa_range *op1_range,
6699 zend_jit_addr op1_addr,
6700 uint32_t op2_info,
6701 zend_ssa_range *op2_range,
6702 zend_jit_addr op2_addr,
6703 zend_jit_addr res_addr,
6704 int may_throw,
6705 uint8_t smart_branch_opcode,
6706 uint32_t target_label,
6707 uint32_t target_label2,
6708 const void *exit_addr,
6709 bool skip_comparison)
6710 {
6711 ir_ref ref = IR_UNUSED;
6712 ir_ref if_op1_long = IR_UNUSED;
6713 ir_ref if_op1_double = IR_UNUSED;
6714 ir_ref if_op2_double = IR_UNUSED;
6715 ir_ref if_op1_long_op2_long = IR_UNUSED;
6716 ir_ref if_op1_long_op2_double = IR_UNUSED;
6717 ir_ref if_op1_double_op2_double = IR_UNUSED;
6718 ir_ref if_op1_double_op2_long = IR_UNUSED;
6719 ir_ref slow_inputs = IR_UNUSED;
6720 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
6721 bool has_slow =
6722 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
6723 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
6724 ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
6725 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))));
6726 ir_refs *end_inputs;
6727
6728 ir_refs_init(end_inputs, 8);
6729
6730 if (Z_MODE(op1_addr) == IS_REG) {
6731 if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) {
6732 /* Force load */
6733 zend_jit_use_reg(jit, op1_addr);
6734 }
6735 } else if (Z_MODE(op2_addr) == IS_REG) {
6736 if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) {
6737 /* Force load */
6738 zend_jit_use_reg(jit, op2_addr);
6739 }
6740 }
6741
6742 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
6743 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
6744 if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
6745 ir_IF_TRUE(if_op1_long);
6746 }
6747 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
6748 if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
6749 ir_IF_FALSE_cold(if_op1_long_op2_long);
6750 if (op2_info & MAY_BE_DOUBLE) {
6751 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
6752 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
6753 ir_IF_FALSE_cold(if_op1_long_op2_double);
6754 ir_END_list(slow_inputs);
6755 ir_IF_TRUE(if_op1_long_op2_double);
6756 }
6757 ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6758 if (!ref) {
6759 return 0;
6760 }
6761 ir_refs_add(end_inputs, ref);
6762 } else {
6763 ir_END_list(slow_inputs);
6764 }
6765 ir_IF_TRUE(if_op1_long_op2_long);
6766 }
6767 ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison);
6768 if (!ref) {
6769 return 0;
6770 }
6771 ir_refs_add(end_inputs, ref);
6772
6773 if (if_op1_long) {
6774 ir_IF_FALSE_cold(if_op1_long);
6775 }
6776 if (op1_info & MAY_BE_DOUBLE) {
6777 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
6778 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
6779 ir_IF_FALSE_cold(if_op1_double);
6780 ir_END_list(slow_inputs);
6781 ir_IF_TRUE(if_op1_double);
6782 }
6783 if (op2_info & MAY_BE_DOUBLE) {
6784 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
6785 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
6786 ir_IF_TRUE(if_op1_double_op2_double);
6787 }
6788 ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6789 if (!ref) {
6790 return 0;
6791 }
6792 ir_refs_add(end_inputs, ref);
6793 if (if_op1_double_op2_double) {
6794 ir_IF_FALSE_cold(if_op1_double_op2_double);
6795 }
6796 }
6797 if (!same_ops) {
6798 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
6799 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
6800 ir_IF_FALSE_cold(if_op1_double_op2_long);
6801 ir_END_list(slow_inputs);
6802 ir_IF_TRUE(if_op1_double_op2_long);
6803 }
6804 ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6805 if (!ref) {
6806 return 0;
6807 }
6808 ir_refs_add(end_inputs, ref);
6809 } else if (if_op1_double_op2_double) {
6810 ir_END_list(slow_inputs);
6811 }
6812 } else if (if_op1_long) {
6813 ir_END_list(slow_inputs);
6814 }
6815 } else if ((op1_info & MAY_BE_DOUBLE) &&
6816 !(op1_info & MAY_BE_LONG) &&
6817 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
6818 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
6819 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
6820 ir_IF_FALSE_cold(if_op1_double);
6821 ir_END_list(slow_inputs);
6822 ir_IF_TRUE(if_op1_double);
6823 }
6824 if (op2_info & MAY_BE_DOUBLE) {
6825 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
6826 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
6827 ir_IF_TRUE(if_op1_double_op2_double);
6828 }
6829 ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6830 if (!ref) {
6831 return 0;
6832 }
6833 ir_refs_add(end_inputs, ref);
6834 if (if_op1_double_op2_double) {
6835 ir_IF_FALSE_cold(if_op1_double_op2_double);
6836 }
6837 }
6838 if (!same_ops && (op2_info & MAY_BE_LONG)) {
6839 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
6840 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
6841 ir_IF_FALSE_cold(if_op1_double_op2_long);
6842 ir_END_list(slow_inputs);
6843 ir_IF_TRUE(if_op1_double_op2_long);
6844 }
6845 ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6846 if (!ref) {
6847 return 0;
6848 }
6849 ir_refs_add(end_inputs, ref);
6850 } else if (if_op1_double_op2_double) {
6851 ir_END_list(slow_inputs);
6852 }
6853 } else if ((op2_info & MAY_BE_DOUBLE) &&
6854 !(op2_info & MAY_BE_LONG) &&
6855 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
6856 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
6857 if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
6858 ir_IF_FALSE_cold(if_op2_double);
6859 ir_END_list(slow_inputs);
6860 ir_IF_TRUE(if_op2_double);
6861 }
6862 if (op1_info & MAY_BE_DOUBLE) {
6863 if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
6864 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
6865 ir_IF_TRUE(if_op1_double_op2_double);
6866 }
6867 ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6868 if (!ref) {
6869 return 0;
6870 }
6871 ir_refs_add(end_inputs, ref);
6872 if (if_op1_double_op2_double) {
6873 ir_IF_FALSE_cold(if_op1_double_op2_double);
6874 }
6875 }
6876 if (!same_ops && (op1_info & MAY_BE_LONG)) {
6877 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
6878 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
6879 ir_IF_FALSE_cold(if_op1_long_op2_double);
6880 ir_END_list(slow_inputs);
6881 ir_IF_TRUE(if_op1_long_op2_double);
6882 }
6883 ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6884 if (!ref) {
6885 return 0;
6886 }
6887 ir_refs_add(end_inputs, ref);
6888 } else if (if_op1_double_op2_double) {
6889 ir_END_list(slow_inputs);
6890 }
6891 }
6892
6893 if (has_slow ||
6894 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
6895 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
6896 ir_ref op1, op2, ref;
6897
6898 if (slow_inputs) {
6899 ir_MERGE_list(slow_inputs);
6900 }
6901 jit_SET_EX_OPLINE(jit, opline);
6902
6903 if (Z_MODE(op1_addr) == IS_REG) {
6904 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
6905 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
6906 return 0;
6907 }
6908 op1_addr = real_addr;
6909 }
6910 if (Z_MODE(op2_addr) == IS_REG) {
6911 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
6912 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
6913 return 0;
6914 }
6915 op2_addr = real_addr;
6916 }
6917
6918 op1 = jit_ZVAL_ADDR(jit, op1_addr);
6919 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
6920 op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, NULL, 0);
6921 }
6922 op2 = jit_ZVAL_ADDR(jit, op2_addr);
6923 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
6924 op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, NULL, 0);
6925 }
6926 ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_compare), op1, op2);
6927 if (opline->opcode != ZEND_CASE) {
6928 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
6929 }
6930 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
6931 if (may_throw) {
6932 zend_jit_check_exception_undef_result(jit, opline);
6933 }
6934
6935 ref = zend_jit_cmp_slow(jit, ref, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
6936 if (!ref) {
6937 return 0;
6938 }
6939 ir_refs_add(end_inputs, ref);
6940 }
6941
6942 if (end_inputs->count) {
6943 uint32_t n = end_inputs->count;
6944
6945 if (smart_branch_opcode && !exit_addr) {
6946 zend_basic_block *bb;
6947 ir_ref ref;
6948 uint32_t label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
6949 target_label2 : target_label;
6950 uint32_t label2 = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
6951 target_label : target_label2;
6952
6953 ZEND_ASSERT(jit->b >= 0);
6954 bb = &jit->ssa->cfg.blocks[jit->b];
6955 ZEND_ASSERT(bb->successors_count == 2);
6956
6957 if (UNEXPECTED(bb->successors[0] == bb->successors[1])) {
6958 ir_ref merge_inputs = IR_UNUSED;
6959
6960 while (n) {
6961 n--;
6962 ir_IF_TRUE(end_inputs->refs[n]);
6963 ir_END_list(merge_inputs);
6964 ir_IF_FALSE(end_inputs->refs[n]);
6965 ir_END_list(merge_inputs);
6966 }
6967 ir_MERGE_list(merge_inputs);
6968 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
6969 } else if (n == 1) {
6970 ref = end_inputs->refs[0];
6971 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
6972 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
6973 } else {
6974 ir_ref true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
6975
6976 while (n) {
6977 n--;
6978 ir_IF_TRUE(end_inputs->refs[n]);
6979 ir_END_list(true_inputs);
6980 ir_IF_FALSE(end_inputs->refs[n]);
6981 ir_END_list(false_inputs);
6982 }
6983 ir_MERGE_list(true_inputs);
6984 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
6985 ir_MERGE_list(false_inputs);
6986 _zend_jit_add_predecessor_ref(jit, label2, jit->b, ir_END());
6987 }
6988 jit->b = -1;
6989 } else {
6990 ir_MERGE_N(n, end_inputs->refs);
6991 }
6992 } else if (smart_branch_opcode && !exit_addr) {
6993 /* dead code */
6994 _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
6995 jit->b = -1;
6996 }
6997
6998 return 1;
6999 }
7000
zend_jit_identical(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_ssa_range * op1_range,zend_jit_addr op1_addr,uint32_t op2_info,zend_ssa_range * op2_range,zend_jit_addr op2_addr,zend_jit_addr res_addr,int may_throw,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr,bool skip_comparison)7001 static int zend_jit_identical(zend_jit_ctx *jit,
7002 const zend_op *opline,
7003 uint32_t op1_info,
7004 zend_ssa_range *op1_range,
7005 zend_jit_addr op1_addr,
7006 uint32_t op2_info,
7007 zend_ssa_range *op2_range,
7008 zend_jit_addr op2_addr,
7009 zend_jit_addr res_addr,
7010 int may_throw,
7011 uint8_t smart_branch_opcode,
7012 uint32_t target_label,
7013 uint32_t target_label2,
7014 const void *exit_addr,
7015 bool skip_comparison)
7016 {
7017 bool always_false = 0, always_true = 0;
7018 ir_ref ref = IR_UNUSED;
7019
7020 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
7021 ir_ref op1 = jit_ZVAL_ADDR(jit, op1_addr);
7022 op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, opline, 0);
7023 op1_info |= MAY_BE_NULL;
7024 op1_addr = ZEND_ADDR_REF_ZVAL(op1);
7025 }
7026 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
7027 ir_ref op2 = jit_ZVAL_ADDR(jit, op2_addr);
7028 op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, opline, 0);
7029 op2_info |= MAY_BE_NULL;
7030 op2_addr = ZEND_ADDR_REF_ZVAL(op2);
7031 }
7032
7033 if ((op1_info & op2_info & MAY_BE_ANY) == 0) {
7034 always_false = 1;
7035 } else if (has_concrete_type(op1_info)
7036 && has_concrete_type(op2_info)
7037 && concrete_type(op1_info) == concrete_type(op2_info)
7038 && concrete_type(op1_info) <= IS_TRUE) {
7039 always_true = 1;
7040 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) {
7041 if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) {
7042 always_true = 1;
7043 } else {
7044 always_false = 1;
7045 }
7046 }
7047
7048 if (always_true) {
7049 if (opline->opcode != ZEND_CASE_STRICT) {
7050 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
7051 }
7052 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
7053 if (!smart_branch_opcode
7054 || smart_branch_opcode == ZEND_JMPZ_EX
7055 || smart_branch_opcode == ZEND_JMPNZ_EX) {
7056 jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE);
7057 }
7058 if (may_throw) {
7059 zend_jit_check_exception(jit);
7060 }
7061 if (exit_addr) {
7062 if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) {
7063 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7064 }
7065 } else if (smart_branch_opcode) {
7066 uint32_t label;
7067
7068 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
7069 label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
7070 target_label : target_label2;
7071 } else {
7072 label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
7073 target_label2 : target_label;
7074 }
7075 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
7076 jit->b = -1;
7077 }
7078 return 1;
7079 } else if (always_false) {
7080 if (opline->opcode != ZEND_CASE_STRICT) {
7081 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
7082 }
7083 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
7084 if (!smart_branch_opcode
7085 || smart_branch_opcode == ZEND_JMPZ_EX
7086 || smart_branch_opcode == ZEND_JMPNZ_EX) {
7087 jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE);
7088 }
7089 if (may_throw) {
7090 zend_jit_check_exception(jit);
7091 }
7092 if (exit_addr) {
7093 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
7094 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7095 }
7096 } else if (smart_branch_opcode) {
7097 uint32_t label;
7098
7099 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
7100 label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
7101 target_label2 : target_label;
7102 } else {
7103 label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
7104 target_label : target_label2;
7105 }
7106 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
7107 jit->b = -1;
7108 }
7109 return 1;
7110 }
7111
7112 if ((opline->op1_type & (IS_CV|IS_VAR)) && (op1_info & MAY_BE_REF)) {
7113 ref = jit_ZVAL_ADDR(jit, op1_addr);
7114 ref = jit_ZVAL_DEREF_ref(jit, ref);
7115 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
7116 }
7117 if ((opline->op2_type & (IS_CV|IS_VAR)) && (op2_info & MAY_BE_REF)) {
7118 ref = jit_ZVAL_ADDR(jit, op2_addr);
7119 ref = jit_ZVAL_DEREF_ref(jit, ref);
7120 op2_addr = ZEND_ADDR_REF_ZVAL(ref);
7121 }
7122
7123 if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG &&
7124 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) {
7125 ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison);
7126 if (!ref) {
7127 return 0;
7128 }
7129 } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE &&
7130 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) {
7131 ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
7132 if (!ref) {
7133 return 0;
7134 }
7135 } else {
7136 if (opline->op1_type != IS_CONST) {
7137 if (Z_MODE(op1_addr) == IS_REG) {
7138 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
7139 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
7140 return 0;
7141 }
7142 op1_addr = real_addr;
7143 }
7144 }
7145 if (opline->op2_type != IS_CONST) {
7146 if (Z_MODE(op2_addr) == IS_REG) {
7147 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
7148 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
7149 return 0;
7150 }
7151 }
7152 }
7153
7154 if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) {
7155 zval *val = Z_ZV(op1_addr);
7156
7157 ref = ir_EQ(jit_Z_TYPE(jit, op2_addr), ir_CONST_U8(Z_TYPE_P(val)));
7158 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) {
7159 zval *val = Z_ZV(op2_addr);
7160
7161 ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(Z_TYPE_P(val)));
7162 } else {
7163 if (Z_MODE(op1_addr) == IS_REG) {
7164 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
7165 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
7166 return 0;
7167 }
7168 op1_addr = real_addr;
7169 }
7170 if (Z_MODE(op2_addr) == IS_REG) {
7171 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
7172 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
7173 return 0;
7174 }
7175 op2_addr = real_addr;
7176 }
7177
7178 ref = ir_CALL_2(IR_BOOL, ir_CONST_FC_FUNC(zend_is_identical),
7179 jit_ZVAL_ADDR(jit, op1_addr),
7180 jit_ZVAL_ADDR(jit, op2_addr));
7181 }
7182
7183 if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
7184 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
7185 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7186 ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
7187 } else {
7188 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7189 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
7190 }
7191 }
7192 if (opline->opcode != ZEND_CASE_STRICT) {
7193 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
7194 }
7195 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
7196 if (may_throw) {
7197 zend_jit_check_exception_undef_result(jit, opline);
7198 }
7199 if (exit_addr) {
7200 if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
7201 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
7202 } else {
7203 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
7204 }
7205 } else if (smart_branch_opcode) {
7206 if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
7207 /* swap labels */
7208 uint32_t tmp = target_label;
7209 target_label = target_label2;
7210 target_label2 = tmp;
7211 }
7212 ref = jit_IF_ex(jit, ref,
7213 (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
7214 }
7215 }
7216
7217 if (smart_branch_opcode && !exit_addr) {
7218 zend_basic_block *bb;
7219
7220 ZEND_ASSERT(jit->b >= 0);
7221 bb = &jit->ssa->cfg.blocks[jit->b];
7222 ZEND_ASSERT(bb->successors_count == 2);
7223
7224 if (bb->successors_count == 2 && bb->successors[0] == bb->successors[1]) {
7225 ir_IF_TRUE(ref);
7226 ir_MERGE_WITH_EMPTY_FALSE(ref);
7227 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
7228 } else {
7229 ZEND_ASSERT(bb->successors_count == 2);
7230 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
7231 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
7232 }
7233 jit->b = -1;
7234 }
7235
7236 return 1;
7237 }
7238
zend_jit_bool_jmpznz(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_addr,zend_jit_addr res_addr,uint32_t target_label,uint32_t target_label2,int may_throw,uint8_t branch_opcode,const void * exit_addr)7239 static int zend_jit_bool_jmpznz(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, uint8_t branch_opcode, const void *exit_addr)
7240 {
7241 uint32_t true_label = -1;
7242 uint32_t false_label = -1;
7243 bool set_bool = 0;
7244 bool set_bool_not = 0;
7245 bool always_true = 0, always_false = 0;
7246 ir_ref ref, end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
7247 ir_type type = IR_UNUSED;
7248
7249 if (branch_opcode == ZEND_BOOL) {
7250 set_bool = 1;
7251 } else if (branch_opcode == ZEND_BOOL_NOT) {
7252 set_bool = 1;
7253 set_bool_not = 1;
7254 } else if (branch_opcode == ZEND_JMPZ) {
7255 true_label = target_label2;
7256 false_label = target_label;
7257 } else if (branch_opcode == ZEND_JMPNZ) {
7258 true_label = target_label;
7259 false_label = target_label2;
7260 } else if (branch_opcode == ZEND_JMPZ_EX) {
7261 set_bool = 1;
7262 true_label = target_label2;
7263 false_label = target_label;
7264 } else if (branch_opcode == ZEND_JMPNZ_EX) {
7265 set_bool = 1;
7266 true_label = target_label;
7267 false_label = target_label2;
7268 } else {
7269 ZEND_UNREACHABLE();
7270 }
7271
7272 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) {
7273 ref = jit_ZVAL_ADDR(jit, op1_addr);
7274 ref = jit_ZVAL_DEREF_ref(jit, ref);
7275 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
7276 }
7277
7278 if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
7279 if (zend_is_true(Z_ZV(op1_addr))) {
7280 always_true = 1;
7281 } else {
7282 always_false = 1;
7283 }
7284 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
7285 if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) {
7286 always_true = 1;
7287 } else if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) {
7288 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
7289 ref = jit_ZVAL_ADDR(jit, op1_addr);
7290 zend_jit_zval_check_undef(jit, ref, opline->op1.var, opline, 0);
7291 }
7292 always_false = 1;
7293 }
7294 }
7295
7296 if (always_true) {
7297 if (set_bool) {
7298 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
7299 }
7300 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
7301 if (may_throw) {
7302 zend_jit_check_exception(jit);
7303 }
7304 if (true_label != (uint32_t)-1) {
7305 ZEND_ASSERT(exit_addr == NULL);
7306 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END());
7307 jit->b = -1;
7308 }
7309 return 1;
7310 } else if (always_false) {
7311 if (set_bool) {
7312 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
7313 }
7314 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
7315 if (may_throw) {
7316 zend_jit_check_exception(jit);
7317 }
7318 if (false_label != (uint32_t)-1) {
7319 ZEND_ASSERT(exit_addr == NULL);
7320 _zend_jit_add_predecessor_ref(jit, false_label, jit->b, ir_END());
7321 jit->b = -1;
7322 }
7323 return 1;
7324 }
7325
7326 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
7327 type = jit_Z_TYPE(jit, op1_addr);
7328 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) {
7329 ir_ref if_type = ir_IF(ir_LT(type, ir_CONST_U8(IS_TRUE)));
7330
7331 ir_IF_TRUE_cold(if_type);
7332
7333 if (op1_info & MAY_BE_UNDEF) {
7334 zend_jit_type_check_undef(jit,
7335 type,
7336 opline->op1.var,
7337 opline, 1, 0, 1);
7338 }
7339 if (set_bool) {
7340 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
7341 }
7342 if (exit_addr) {
7343 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
7344 ir_END_list(end_inputs);
7345 } else {
7346 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7347 }
7348 } else if (false_label != (uint32_t)-1) {
7349 ir_END_list(false_inputs);
7350 } else {
7351 ir_END_list(end_inputs);
7352 }
7353 ir_IF_FALSE(if_type);
7354 }
7355
7356 if (op1_info & MAY_BE_TRUE) {
7357 ir_ref if_type = IR_UNUSED;
7358
7359 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
7360 if_type = ir_IF(ir_EQ(type, ir_CONST_U8(IS_TRUE)));
7361
7362 ir_IF_TRUE(if_type);
7363 }
7364 if (set_bool) {
7365 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
7366 }
7367 if (exit_addr) {
7368 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
7369 ir_END_list(end_inputs);
7370 } else {
7371 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7372 }
7373 } else if (true_label != (uint32_t)-1) {
7374 ir_END_list(true_inputs);
7375 } else {
7376 ir_END_list(end_inputs);
7377 }
7378 if (if_type) {
7379 ir_IF_FALSE(if_type);
7380 }
7381 }
7382 }
7383
7384 if (op1_info & MAY_BE_LONG) {
7385 ir_ref if_long = IR_UNUSED;
7386 ir_ref ref;
7387
7388 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
7389 if (!type) {
7390 type = jit_Z_TYPE(jit, op1_addr);
7391 }
7392 if_long = ir_IF(ir_EQ(type, ir_CONST_U8(IS_LONG)));
7393 ir_IF_TRUE(if_long);
7394 }
7395 ref = jit_Z_LVAL(jit, op1_addr);
7396 if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
7397 ref = ir_NE(ref, ir_CONST_LONG(0));
7398 if (set_bool_not) {
7399 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7400 ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
7401 } else {
7402 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7403 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
7404 }
7405 ir_END_list(end_inputs);
7406 } else if (exit_addr) {
7407 if (set_bool) {
7408 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7409 ir_ADD_U32(ir_ZEXT_U32(ir_NE(ref, ir_CONST_LONG(0))), ir_CONST_U32(IS_FALSE)));
7410 }
7411 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
7412 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
7413 } else {
7414 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
7415 }
7416 ir_END_list(end_inputs);
7417 } else {
7418 ir_ref if_val = ir_IF(ref);
7419 ir_IF_TRUE(if_val);
7420 if (set_bool) {
7421 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
7422 }
7423 ir_END_list(true_inputs);
7424 ir_IF_FALSE(if_val);
7425 if (set_bool) {
7426 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
7427 }
7428 ir_END_list(false_inputs);
7429 }
7430 if (if_long) {
7431 ir_IF_FALSE(if_long);
7432 }
7433 }
7434
7435 if (op1_info & MAY_BE_DOUBLE) {
7436 ir_ref if_double = IR_UNUSED;
7437 ir_ref ref;
7438
7439 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))) {
7440 if (!type) {
7441 type = jit_Z_TYPE(jit, op1_addr);
7442 }
7443 if_double = ir_IF(ir_EQ(type, ir_CONST_U8(IS_DOUBLE)));
7444 ir_IF_TRUE(if_double);
7445 }
7446 ref = ir_NE(jit_Z_DVAL(jit, op1_addr), ir_CONST_DOUBLE(0.0));
7447 if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
7448 if (set_bool_not) {
7449 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7450 ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
7451 } else {
7452 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7453 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
7454 }
7455 ir_END_list(end_inputs);
7456 } else if (exit_addr) {
7457 if (set_bool) {
7458 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7459 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
7460 }
7461 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
7462 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
7463 } else {
7464 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
7465 }
7466 ir_END_list(end_inputs);
7467 } else {
7468 ir_ref if_val = ir_IF(ref);
7469 ir_IF_TRUE(if_val);
7470 if (set_bool) {
7471 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
7472 }
7473 ir_END_list(true_inputs);
7474 ir_IF_FALSE(if_val);
7475 if (set_bool) {
7476 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
7477 }
7478 ir_END_list(false_inputs);
7479 }
7480 if (if_double) {
7481 ir_IF_FALSE(if_double);
7482 }
7483 }
7484
7485 if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))) {
7486 jit_SET_EX_OPLINE(jit, opline);
7487 ref = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_is_true), jit_ZVAL_ADDR(jit, op1_addr));
7488 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
7489 if (may_throw) {
7490 zend_jit_check_exception_undef_result(jit, opline);
7491 }
7492 if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
7493 if (set_bool_not) {
7494 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7495 ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
7496 } else {
7497 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7498 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
7499 }
7500 if (end_inputs) {
7501 ir_END_list(end_inputs);
7502 }
7503 } else if (exit_addr) {
7504 if (set_bool) {
7505 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7506 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
7507 }
7508 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
7509 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
7510 } else {
7511 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
7512 }
7513 if (end_inputs) {
7514 ir_END_list(end_inputs);
7515 }
7516 } else {
7517 ir_ref if_val = ir_IF(ref);
7518 ir_IF_TRUE(if_val);
7519 if (set_bool) {
7520 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
7521 }
7522 ir_END_list(true_inputs);
7523 ir_IF_FALSE(if_val);
7524 if (set_bool) {
7525 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
7526 }
7527 ir_END_list(false_inputs);
7528 }
7529 }
7530
7531 if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT || exit_addr) {
7532 if (end_inputs) {
7533 ir_MERGE_list(end_inputs);
7534 }
7535 } else {
7536 _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
7537 }
7538
7539 return 1;
7540 }
7541
zend_jit_defined(zend_jit_ctx * jit,const zend_op * opline,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)7542 static int zend_jit_defined(zend_jit_ctx *jit, const zend_op *opline, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
7543 {
7544 uint32_t defined_label = (uint32_t)-1;
7545 uint32_t undefined_label = (uint32_t)-1;
7546 zval *zv = RT_CONSTANT(opline, opline->op1);
7547 zend_jit_addr res_addr = 0;
7548 ir_ref ref, ref2, if_set, if_zero, if_set2;
7549 ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
7550
7551 if (smart_branch_opcode && !exit_addr) {
7552 if (smart_branch_opcode == ZEND_JMPZ) {
7553 defined_label = target_label2;
7554 undefined_label = target_label;
7555 } else if (smart_branch_opcode == ZEND_JMPNZ) {
7556 defined_label = target_label;
7557 undefined_label = target_label2;
7558 } else {
7559 ZEND_UNREACHABLE();
7560 }
7561 } else {
7562 res_addr = RES_ADDR();
7563 }
7564
7565 // if (CACHED_PTR(opline->extended_value)) {
7566 ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value));
7567
7568 if_set = ir_IF(ref);
7569
7570 ir_IF_FALSE_cold(if_set);
7571 if_zero = ir_END();
7572
7573 ir_IF_TRUE(if_set);
7574 if_set2 = ir_IF(ir_AND_A(ref, ir_CONST_ADDR(CACHE_SPECIAL)));
7575 ir_IF_FALSE(if_set2);
7576
7577 if (exit_addr) {
7578 if (smart_branch_opcode == ZEND_JMPNZ) {
7579 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7580 } else {
7581 ir_END_list(end_inputs);
7582 }
7583 } else if (smart_branch_opcode) {
7584 ir_END_list(true_inputs);
7585 } else {
7586 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
7587 ir_END_list(end_inputs);
7588 }
7589
7590 ir_IF_TRUE_cold(if_set2);
7591
7592 ref2 = jit_EG(zend_constants);
7593 ref = ir_SHR_A(ref, ir_CONST_ADDR(1));
7594 if (sizeof(void*) == 8) {
7595 ref = ir_TRUNC_U32(ref);
7596 }
7597 ref2 = ir_EQ(ref, ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(ref2), offsetof(HashTable, nNumOfElements))));
7598 ref2 = ir_IF(ref2);
7599 ir_IF_TRUE(ref2);
7600
7601 if (exit_addr) {
7602 if (smart_branch_opcode == ZEND_JMPZ) {
7603 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7604 } else {
7605 ir_END_list(end_inputs);
7606 }
7607 } else if (smart_branch_opcode) {
7608 ir_END_list(false_inputs);
7609 } else {
7610 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
7611 ir_END_list(end_inputs);
7612 }
7613
7614 ir_IF_FALSE(ref2);
7615 ir_MERGE_2(if_zero, ir_END());
7616
7617 jit_SET_EX_OPLINE(jit, opline);
7618 ref2 = ir_NE(ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_check_constant), ir_CONST_ADDR(zv)), IR_NULL);
7619 if (exit_addr) {
7620 if (smart_branch_opcode == ZEND_JMPZ) {
7621 ir_GUARD(ref2, ir_CONST_ADDR(exit_addr));
7622 } else {
7623 ir_GUARD_NOT(ref2, ir_CONST_ADDR(exit_addr));
7624 }
7625 ir_END_list(end_inputs);
7626 } else if (smart_branch_opcode) {
7627 ref2 = ir_IF(ref2);
7628 ir_IF_TRUE(ref2);
7629 ir_END_list(true_inputs);
7630 ir_IF_FALSE(ref2);
7631 ir_END_list(false_inputs);
7632 } else {
7633 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
7634 ir_ADD_U32(ir_ZEXT_U32(ref2), ir_CONST_U32(IS_FALSE)));
7635 ir_END_list(end_inputs);
7636 }
7637
7638 if (!smart_branch_opcode || exit_addr) {
7639 if (end_inputs) {
7640 ir_MERGE_list(end_inputs);
7641 }
7642 } else {
7643 _zend_jit_merge_smart_branch_inputs(jit, defined_label, undefined_label, true_inputs, false_inputs);
7644 }
7645
7646 return 1;
7647 }
7648
zend_jit_escape_if_undef(zend_jit_ctx * jit,int var,uint32_t flags,const zend_op * opline,int8_t reg)7649 static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opline, int8_t reg)
7650 {
7651 zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg));
7652 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, reg_addr));
7653
7654 ir_IF_FALSE_cold(if_def);
7655
7656 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
7657 if (!zend_jit_save_call_chain(jit, -1)) {
7658 return 0;
7659 }
7660 }
7661
7662 if ((opline-1)->opcode != ZEND_FETCH_CONSTANT
7663 && (opline-1)->opcode != ZEND_FETCH_LIST_R
7664 && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR))
7665 && !(flags & ZEND_JIT_EXIT_FREE_OP1)) {
7666 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var);
7667
7668 zend_jit_zval_try_addref(jit, val_addr);
7669 }
7670
7671 jit_LOAD_IP_ADDR(jit, opline - 1);
7672 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape));
7673
7674 ir_IF_TRUE(if_def);
7675
7676 return 1;
7677 }
7678
zend_jit_restore_zval(zend_jit_ctx * jit,int var,int8_t reg)7679 static int zend_jit_restore_zval(zend_jit_ctx *jit, int var, int8_t reg)
7680 {
7681 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
7682 zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg));
7683
7684 // JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
7685 jit_ZVAL_COPY(jit, var_addr, MAY_BE_ANY, reg_addr, MAY_BE_ANY, 1);
7686 return 1;
7687 }
7688
zend_jit_guard_fetch_result_type(zend_jit_ctx * jit,const zend_op * opline,zend_jit_addr val_addr,uint8_t type,bool deref,uint32_t flags,bool op1_avoid_refcounting)7689 static zend_jit_addr zend_jit_guard_fetch_result_type(zend_jit_ctx *jit,
7690 const zend_op *opline,
7691 zend_jit_addr val_addr,
7692 uint8_t type,
7693 bool deref,
7694 uint32_t flags,
7695 bool op1_avoid_refcounting)
7696 {
7697 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
7698 int32_t exit_point;
7699 const void *res_exit_addr = NULL;
7700 ir_ref end1 = IR_UNUSED, ref1 = IR_UNUSED;
7701 ir_ref ref = jit_ZVAL_ADDR(jit, val_addr);
7702 uint32_t old_op1_info = 0;
7703 uint32_t old_info;
7704 ir_ref old_ref;
7705
7706
7707 if (opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) {
7708 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
7709 if (op1_avoid_refcounting
7710 || ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
7711 && STACK_FLAGS(stack, EX_VAR_TO_NUM(opline->op1.var)) & (ZREG_ZVAL_ADDREF|ZREG_THIS))) {
7712 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
7713 }
7714 }
7715 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
7716 old_ref = STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
7717 CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
7718 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
7719
7720 if (deref) {
7721 ir_ref if_type = jit_if_Z_TYPE(jit, val_addr, type);
7722
7723 ir_IF_TRUE(if_type);
7724 end1 = ir_END();
7725 ref1 = ref;
7726 ir_IF_FALSE_cold(if_type);
7727
7728 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY);
7729 exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
7730 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
7731 if (!res_exit_addr) {
7732 return 0;
7733 }
7734
7735 jit_guard_Z_TYPE(jit, val_addr, IS_REFERENCE, res_exit_addr);
7736 ref = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val));
7737 val_addr = ZEND_ADDR_REF_ZVAL(ref);
7738 }
7739
7740 SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY);
7741 exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
7742 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
7743 if (!res_exit_addr) {
7744 return 0;
7745 }
7746
7747 jit_guard_Z_TYPE(jit, val_addr, type, res_exit_addr);
7748
7749 if (deref) {
7750 ir_MERGE_WITH(end1);
7751 ref = ir_PHI_2(IR_ADDR, ref, ref1);
7752 }
7753
7754 val_addr = ZEND_ADDR_REF_ZVAL(ref);
7755
7756 SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), old_ref);
7757 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
7758 if (opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) {
7759 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
7760 }
7761
7762 return val_addr;
7763 }
7764
zend_jit_fetch_constant(zend_jit_ctx * jit,const zend_op * opline,const zend_op_array * op_array,zend_ssa * ssa,const zend_ssa_op * ssa_op,zend_jit_addr res_addr)7765 static int zend_jit_fetch_constant(zend_jit_ctx *jit,
7766 const zend_op *opline,
7767 const zend_op_array *op_array,
7768 zend_ssa *ssa,
7769 const zend_ssa_op *ssa_op,
7770 zend_jit_addr res_addr)
7771 {
7772 zval *zv = RT_CONSTANT(opline, opline->op2) + 1;
7773 uint32_t res_info = RES_INFO();
7774 ir_ref ref, ref2, if_set, if_special, not_set_path, special_path, fast_path;
7775
7776 // JIT: c = CACHED_PTR(opline->extended_value);
7777 ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value));
7778
7779 // JIT: if (c != NULL)
7780 if_set = ir_IF(ref);
7781
7782 if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) {
7783 // JIT: if (!IS_SPECIAL_CACHE_VAL(c))
7784 ir_IF_FALSE_cold(if_set);
7785 not_set_path = ir_END();
7786 ir_IF_TRUE(if_set);
7787 if_special = ir_IF(ir_AND_A(ref, ir_CONST_ADDR(CACHE_SPECIAL)));
7788 ir_IF_TRUE_cold(if_special);
7789 special_path = ir_END();
7790 ir_IF_FALSE(if_special);
7791 fast_path = ir_END();
7792 ir_MERGE_2(not_set_path, special_path);
7793 } else {
7794 ir_IF_TRUE(if_set);
7795 fast_path = ir_END();
7796 ir_IF_FALSE_cold(if_set);
7797 }
7798
7799 // JIT: zend_jit_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num);
7800 jit_SET_EX_OPLINE(jit, opline);
7801 ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_constant),
7802 ir_CONST_ADDR(zv),
7803 ir_CONST_U32(opline->op1.num));
7804 ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
7805
7806 ir_MERGE_WITH(fast_path);
7807 ref = ir_PHI_2(IR_ADDR, ref2, ref);
7808
7809 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) {
7810 uint8_t type = concrete_type(res_info);
7811 zend_jit_addr const_addr = ZEND_ADDR_REF_ZVAL(ref);
7812
7813 const_addr = zend_jit_guard_fetch_result_type(jit, opline, const_addr, type, 0, 0, 0);
7814 if (!const_addr) {
7815 return 0;
7816 }
7817
7818 res_info &= ~MAY_BE_GUARD;
7819 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
7820
7821 // JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
7822 jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, res_info, 1);
7823 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
7824 return 0;
7825 }
7826 } else {
7827 ir_ref const_addr = ZEND_ADDR_REF_ZVAL(ref);
7828
7829 // JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
7830 jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, 1);
7831 }
7832
7833
7834 return 1;
7835 }
7836
zend_jit_type_check(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)7837 static int zend_jit_type_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
7838 {
7839 uint32_t mask;
7840 zend_jit_addr op1_addr = OP1_ADDR();
7841 zend_jit_addr res_addr = 0;
7842 uint32_t true_label = -1, false_label = -1;
7843 ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
7844
7845 // TODO: support for is_resource() ???
7846 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);
7847
7848 if (smart_branch_opcode && !exit_addr) {
7849 if (smart_branch_opcode == ZEND_JMPZ) {
7850 true_label = target_label2;
7851 false_label = target_label;
7852 } else if (smart_branch_opcode == ZEND_JMPNZ) {
7853 true_label = target_label;
7854 false_label = target_label2;
7855 } else {
7856 ZEND_UNREACHABLE();
7857 }
7858 } else {
7859 res_addr = RES_ADDR();
7860 }
7861
7862 if (op1_info & MAY_BE_UNDEF) {
7863 ir_ref if_def = IR_UNUSED;
7864
7865 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
7866 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
7867 ir_IF_FALSE_cold(if_def);
7868 }
7869
7870 jit_SET_EX_OPLINE(jit, opline);
7871 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
7872 zend_jit_check_exception_undef_result(jit, opline);
7873 if (opline->extended_value & MAY_BE_NULL) {
7874 if (exit_addr) {
7875 if (smart_branch_opcode == ZEND_JMPNZ) {
7876 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7877 } else {
7878 ir_END_list(end_inputs);
7879 }
7880 } else if (smart_branch_opcode) {
7881 ir_END_list(true_inputs);
7882 } else {
7883 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
7884 ir_END_list(end_inputs);
7885 }
7886 } else {
7887 if (exit_addr) {
7888 if (smart_branch_opcode == ZEND_JMPZ) {
7889 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7890 } else {
7891 ir_END_list(end_inputs);
7892 }
7893 } else if (smart_branch_opcode) {
7894 ir_END_list(false_inputs);
7895 } else {
7896 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
7897 if (if_def) {
7898 ir_END_list(end_inputs);
7899 }
7900 }
7901 }
7902
7903 if (if_def) {
7904 ir_IF_TRUE(if_def);
7905 op1_info |= MAY_BE_NULL;
7906 }
7907 }
7908
7909 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
7910 mask = opline->extended_value;
7911 if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) {
7912 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
7913 if (exit_addr) {
7914 if (smart_branch_opcode == ZEND_JMPNZ) {
7915 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7916 } else if (end_inputs) {
7917 ir_END_list(end_inputs);
7918 }
7919 } else if (smart_branch_opcode) {
7920 ir_END_list(true_inputs);
7921 } else {
7922 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
7923 ir_END_list(end_inputs);
7924 }
7925 } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) {
7926 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
7927 if (exit_addr) {
7928 if (smart_branch_opcode == ZEND_JMPZ) {
7929 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
7930 } else if (end_inputs) {
7931 ir_END_list(end_inputs);
7932 }
7933 } else if (smart_branch_opcode) {
7934 ir_END_list(false_inputs);
7935 } else {
7936 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
7937 ir_END_list(end_inputs);
7938 }
7939 } else {
7940 ir_ref ref;
7941 bool invert = 0;
7942 uint8_t type;
7943
7944 switch (mask) {
7945 case MAY_BE_NULL: type = IS_NULL; break;
7946 case MAY_BE_FALSE: type = IS_FALSE; break;
7947 case MAY_BE_TRUE: type = IS_TRUE; break;
7948 case MAY_BE_LONG: type = IS_LONG; break;
7949 case MAY_BE_DOUBLE: type = IS_DOUBLE; break;
7950 case MAY_BE_STRING: type = IS_STRING; break;
7951 case MAY_BE_ARRAY: type = IS_ARRAY; break;
7952 case MAY_BE_OBJECT: type = IS_OBJECT; break;
7953 case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break;
7954 case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break;
7955 case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break;
7956 case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break;
7957 case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break;
7958 case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break;
7959 case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break;
7960 case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break;
7961 case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break;
7962 default:
7963 type = 0;
7964 }
7965
7966 if (op1_info & MAY_BE_REF) {
7967 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
7968 ref = jit_ZVAL_DEREF_ref(jit, ref);
7969 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
7970 }
7971 if (type == 0) {
7972 ref = ir_AND_U32(ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)), ir_CONST_U32(mask));
7973 if (!smart_branch_opcode) {
7974 ref = ir_NE(ref, ir_CONST_U32(0));
7975 }
7976 } else if (invert) {
7977 ref = ir_NE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type));
7978 } else {
7979 ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type));
7980 }
7981
7982 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
7983
7984 if (exit_addr) {
7985 if (smart_branch_opcode == ZEND_JMPZ) {
7986 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
7987 } else {
7988 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
7989 }
7990 if (end_inputs) {
7991 ir_END_list(end_inputs);
7992 }
7993 } else if (smart_branch_opcode) {
7994 ir_ref if_val = ir_IF(ref);
7995 ir_IF_TRUE(if_val);
7996 ir_END_list(true_inputs);
7997 ir_IF_FALSE(if_val);
7998 ir_END_list(false_inputs);
7999 } else {
8000 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
8001 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
8002 ir_END_list(end_inputs);
8003 }
8004 }
8005 }
8006
8007 if (!smart_branch_opcode || exit_addr) {
8008 if (end_inputs) {
8009 ir_MERGE_list(end_inputs);
8010 }
8011 } else {
8012 _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
8013 }
8014
8015 return 1;
8016 }
8017
zend_jit_isset_isempty_cv(zend_jit_ctx * jit,const zend_op * opline,uint32_t op1_info,zend_jit_addr op1_addr,uint8_t smart_branch_opcode,uint32_t target_label,uint32_t target_label2,const void * exit_addr)8018 static int zend_jit_isset_isempty_cv(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
8019 {
8020 zend_jit_addr res_addr = RES_ADDR();
8021 uint32_t true_label = -1, false_label = -1;
8022 ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
8023
8024 // TODO: support for empty() ???
8025 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);
8026
8027 if (smart_branch_opcode && !exit_addr) {
8028 if (smart_branch_opcode == ZEND_JMPZ) {
8029 true_label = target_label2;
8030 false_label = target_label;
8031 } else if (smart_branch_opcode == ZEND_JMPNZ) {
8032 true_label = target_label;
8033 false_label = target_label2;
8034 } else {
8035 ZEND_UNREACHABLE();
8036 }
8037 } else {
8038 res_addr = RES_ADDR();
8039 }
8040
8041 if (op1_info & MAY_BE_REF) {
8042 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
8043 ref = jit_ZVAL_DEREF_ref(jit, ref);
8044 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
8045 }
8046
8047 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
8048 if (exit_addr) {
8049 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ);
8050 } else if (smart_branch_opcode) {
8051 ir_END_list(true_inputs);
8052 } else {
8053 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
8054 ir_END_list(end_inputs);
8055 }
8056 } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) {
8057 if (exit_addr) {
8058 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ);
8059 } else if (smart_branch_opcode) {
8060 ir_END_list(false_inputs);
8061 } else {
8062 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
8063 ir_END_list(end_inputs);
8064 }
8065 } else {
8066 ir_ref ref = ir_GT(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL));
8067 if (exit_addr) {
8068 if (smart_branch_opcode == ZEND_JMPNZ) {
8069 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
8070 } else {
8071 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
8072 }
8073 } else if (smart_branch_opcode) {
8074 ir_ref if_val = ir_IF(ref);
8075 ir_IF_TRUE(if_val);
8076 ir_END_list(true_inputs);
8077 ir_IF_FALSE(if_val);
8078 ir_END_list(false_inputs);
8079 } else {
8080 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
8081 ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
8082 ir_END_list(end_inputs);
8083 }
8084 }
8085
8086 if (!smart_branch_opcode || exit_addr) {
8087 if (end_inputs) {
8088 ir_MERGE_list(end_inputs);
8089 }
8090 } else {
8091 _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
8092 }
8093
8094 return 1;
8095 }
8096
8097 /* copy of hidden zend_closure */
8098 typedef struct _zend_closure {
8099 zend_object std;
8100 zend_function func;
8101 zval this_ptr;
8102 zend_class_entry *called_scope;
8103 zif_handler orig_internal_handler;
8104 } zend_closure;
8105
zend_jit_stack_check(zend_jit_ctx * jit,const zend_op * opline,uint32_t used_stack)8106 static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t used_stack)
8107 {
8108 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
8109 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8110
8111 if (!exit_addr) {
8112 return 0;
8113 }
8114
8115 // JIT: if (EG(vm_stack_end) - EG(vm_stack_top) < used_stack)
8116 ir_GUARD(
8117 ir_UGE(
8118 ir_SUB_A(ir_LOAD_A(jit_EG(vm_stack_end)), ir_LOAD_A(jit_EG(vm_stack_top))),
8119 ir_CONST_ADDR(used_stack)),
8120 ir_CONST_ADDR(exit_addr));
8121
8122 return 1;
8123 }
8124
zend_jit_free_trampoline(zend_jit_ctx * jit,int8_t func_reg)8125 static int zend_jit_free_trampoline(zend_jit_ctx *jit, int8_t func_reg)
8126 {
8127 // JIT: if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
8128 ir_ref func = ir_RLOAD_A(func_reg);
8129 ir_ref if_trampoline = ir_IF(ir_AND_U32(
8130 ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, common.fn_flags))),
8131 ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE)));
8132
8133 ir_IF_TRUE(if_trampoline);
8134 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_free_trampoline_helper), func);
8135 ir_MERGE_WITH_EMPTY_FALSE(if_trampoline);
8136
8137 return 1;
8138 }
8139
zend_jit_push_call_frame(zend_jit_ctx * jit,const zend_op * opline,const zend_op_array * op_array,zend_function * func,bool is_closure,bool delayed_fetch_this,int checked_stack,ir_ref func_ref,ir_ref this_ref)8140 static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack, ir_ref func_ref, ir_ref this_ref)
8141 {
8142 uint32_t used_stack;
8143 ir_ref used_stack_ref = IR_UNUSED;
8144 bool stack_check = 1;
8145 ir_ref rx, ref, top, if_enough_stack, cold_path = IR_UNUSED;
8146
8147 ZEND_ASSERT(func_ref != IR_NULL);
8148 if (func) {
8149 used_stack = zend_vm_calc_used_stack(opline->extended_value, func);
8150 if ((int)used_stack <= checked_stack) {
8151 stack_check = 0;
8152 }
8153 used_stack_ref = ir_CONST_ADDR(used_stack);
8154 } else {
8155 ir_ref num_args_ref;
8156 ir_ref if_internal_func = IR_UNUSED;
8157
8158 used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value + ZEND_OBSERVER_ENABLED) * sizeof(zval);
8159 used_stack_ref = ir_CONST_ADDR(used_stack);
8160
8161 if (!is_closure) {
8162 used_stack_ref = ir_HARD_COPY_A(used_stack_ref); /* load constant once */
8163
8164 // JIT: if (EXPECTED(ZEND_USER_CODE(func->type))) {
8165 ir_ref tmp = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type)));
8166 if_internal_func = ir_IF(ir_AND_U8(tmp, ir_CONST_U8(1)));
8167 ir_IF_FALSE(if_internal_func);
8168 }
8169
8170 // JIT: used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval);
8171 num_args_ref = ir_CONST_U32(opline->extended_value);
8172 if (!is_closure) {
8173 ref = ir_SUB_U32(
8174 ir_SUB_U32(
8175 ir_MIN_U32(
8176 num_args_ref,
8177 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.num_args)))),
8178 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.last_var)))),
8179 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.T))));
8180 } else {
8181 ref = ir_SUB_U32(
8182 ir_SUB_U32(
8183 ir_MIN_U32(
8184 num_args_ref,
8185 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.num_args)))),
8186 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.last_var)))),
8187 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.T))));
8188 }
8189 ref = ir_MUL_U32(ref, ir_CONST_U32(sizeof(zval)));
8190 if (sizeof(void*) == 8) {
8191 ref = ir_SEXT_A(ref);
8192 }
8193 ref = ir_SUB_A(used_stack_ref, ref);
8194
8195 if (is_closure) {
8196 used_stack_ref = ref;
8197 } else {
8198 ir_MERGE_WITH_EMPTY_TRUE(if_internal_func);
8199 used_stack_ref = ir_PHI_2(IR_ADDR, ref, used_stack_ref);
8200 }
8201 }
8202
8203 zend_jit_start_reuse_ip(jit);
8204
8205 // JIT: if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) {
8206 jit_STORE_IP(jit, ir_LOAD_A(jit_EG(vm_stack_top)));
8207
8208 if (stack_check) {
8209 // JIT: Check Stack Overflow
8210 ref = ir_UGE(
8211 ir_SUB_A(
8212 ir_LOAD_A(jit_EG(vm_stack_end)),
8213 jit_IP(jit)),
8214 used_stack_ref);
8215
8216 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
8217 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
8218 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8219
8220 if (!exit_addr) {
8221 return 0;
8222 }
8223
8224 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
8225 } else {
8226 if_enough_stack = ir_IF(ref);
8227 ir_IF_FALSE_cold(if_enough_stack);
8228
8229 #ifdef _WIN32
8230 if (0) {
8231 #else
8232 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
8233 #endif
8234 jit_SET_EX_OPLINE(jit, opline);
8235 ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_int_extend_stack_helper), used_stack_ref);
8236 } else {
8237 if (!is_closure) {
8238 ref = func_ref;
8239 } else {
8240 ref = ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func));
8241 }
8242 jit_SET_EX_OPLINE(jit, opline);
8243 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_extend_stack_helper),
8244 used_stack_ref, ref);
8245 }
8246 jit_STORE_IP(jit, ref);
8247
8248 cold_path = ir_END();
8249 ir_IF_TRUE(if_enough_stack);
8250 }
8251 }
8252
8253 ref = jit_EG(vm_stack_top);
8254 rx = jit_IP(jit);
8255 #if !OPTIMIZE_FOR_SIZE
8256 /* JIT: EG(vm_stack_top) = (zval*)((char*)call + used_stack);
8257 * This vesions is longer but faster
8258 * mov EG(vm_stack_top), %CALL
8259 * lea size(%call), %tmp
8260 * mov %tmp, EG(vm_stack_top)
8261 */
8262 top = rx;
8263 #else
8264 /* JIT: EG(vm_stack_top) += used_stack;
8265 * Use ir_emit() because ir_LOAD() makes load forwarding and doesn't allow load/store fusion
8266 * mov EG(vm_stack_top), %CALL
8267 * add $size, EG(vm_stack_top)
8268 */
8269 top = jit->ctx.control = ir_emit2(&jit->ctx, IR_OPT(IR_LOAD, IR_ADDR), jit->ctx.control, ref);
8270 #endif
8271 ir_STORE(ref, ir_ADD_A(top, used_stack_ref));
8272
8273 // JIT: zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object);
8274 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) {
8275 // JIT: ZEND_SET_CALL_INFO(call, 0, call_info);
8276 ir_STORE(jit_CALL(rx, This.u1.type_info), ir_CONST_U32(IS_UNDEF | ZEND_CALL_NESTED_FUNCTION));
8277 }
8278 #ifdef _WIN32
8279 if (0) {
8280 #else
8281 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
8282 #endif
8283 if (cold_path) {
8284 ir_MERGE_WITH(cold_path);
8285 rx = jit_IP(jit);
8286 }
8287
8288 // JIT: call->func = func;
8289 ir_STORE(jit_CALL(rx, func), func_ref);
8290 } else {
8291 if (!is_closure) {
8292 // JIT: call->func = func;
8293 ir_STORE(jit_CALL(rx, func), func_ref);
8294 } else {
8295 // JIT: call->func = &closure->func;
8296 ir_STORE(jit_CALL(rx, func), ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func)));
8297 }
8298 if (cold_path) {
8299 ir_MERGE_WITH(cold_path);
8300 rx = jit_IP(jit);
8301 }
8302 }
8303 if (opline->opcode == ZEND_INIT_METHOD_CALL) {
8304 // JIT: Z_PTR(call->This) = obj;
8305 ZEND_ASSERT(this_ref != IR_NULL);
8306 ir_STORE(jit_CALL(rx, This.value.ptr), this_ref);
8307 if (opline->op1_type == IS_UNUSED || delayed_fetch_this) {
8308 // JIT: call->call_info |= ZEND_CALL_HAS_THIS;
8309 ref = jit_CALL(rx, This.u1.type_info);
8310 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
8311 ir_STORE(ref, ir_CONST_U32( ZEND_CALL_HAS_THIS));
8312 } else {
8313 ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_HAS_THIS)));
8314 }
8315 } else {
8316 if (opline->op1_type == IS_CV) {
8317 // JIT: GC_ADDREF(obj);
8318 jit_GC_ADDREF(jit, this_ref);
8319 }
8320
8321 // JIT: call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS;
8322 ref = jit_CALL(rx, This.u1.type_info);
8323 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
8324 ir_STORE(ref, ir_CONST_U32( ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS));
8325 } else {
8326 ir_STORE(ref,
8327 ir_OR_U32(ir_LOAD_U32(ref),
8328 ir_CONST_U32(ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS)));
8329 }
8330 }
8331 } else if (!is_closure) {
8332 // JIT: Z_CE(call->This) = called_scope;
8333 ir_STORE(jit_CALL(rx, This), IR_NULL);
8334 } else {
8335 ir_ref object_or_called_scope, call_info, call_info2, object, if_cond;
8336
8337 if (opline->op2_type == IS_CV) {
8338 // JIT: GC_ADDREF(closure);
8339 jit_GC_ADDREF(jit, func_ref);
8340 }
8341
8342 // JIT: RX(object_or_called_scope) = closure->called_scope;
8343 object_or_called_scope = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, called_scope)));
8344
8345 // JIT: call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE |
8346 // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE);
8347 call_info = ir_OR_U32(
8348 ir_AND_U32(
8349 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.common.fn_flags))),
8350 ir_CONST_U32(ZEND_ACC_FAKE_CLOSURE)),
8351 ir_CONST_U32(ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE));
8352 // JIT: if (Z_TYPE(closure->this_ptr) != IS_UNDEF) {
8353 if_cond = ir_IF(ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, this_ptr.u1.v.type))));
8354 ir_IF_TRUE(if_cond);
8355
8356 // JIT: call_info |= ZEND_CALL_HAS_THIS;
8357 call_info2 = ir_OR_U32(call_info, ir_CONST_U32(ZEND_CALL_HAS_THIS));
8358
8359 // JIT: object_or_called_scope = Z_OBJ(closure->this_ptr);
8360 object = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, this_ptr.value.ptr)));
8361
8362 ir_MERGE_WITH_EMPTY_FALSE(if_cond);
8363 call_info = ir_PHI_2(IR_U32, call_info2, call_info);
8364 object_or_called_scope = ir_PHI_2(IR_ADDR, object, object_or_called_scope);
8365
8366 // JIT: ZEND_SET_CALL_INFO(call, 0, call_info);
8367 ref = jit_CALL(rx, This.u1.type_info);
8368 ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), call_info));
8369
8370 // JIT: Z_PTR(call->This) = object_or_called_scope;
8371 ir_STORE(jit_CALL(rx, This.value.ptr), object_or_called_scope);
8372
8373 // JIT: if (closure->func.op_array.run_time_cache__ptr)
8374 if_cond = ir_IF(ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.run_time_cache__ptr))));
8375 ir_IF_FALSE(if_cond);
8376
8377 // JIT: zend_jit_init_func_run_time_cache_helper(closure->func);
8378 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_init_func_run_time_cache_helper),
8379 ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func)));
8380
8381 ir_MERGE_WITH_EMPTY_TRUE(if_cond);
8382 }
8383
8384 // JIT: ZEND_CALL_NUM_ARGS(call) = num_args;
8385 ir_STORE(jit_CALL(rx, This.u2.num_args), ir_CONST_U32(opline->extended_value));
8386
8387 return 1;
8388 }
8389
8390 static int zend_jit_init_fcall_guard(zend_jit_ctx *jit, uint32_t level, const zend_function *func, const zend_op *to_opline)
8391 {
8392 int32_t exit_point;
8393 const void *exit_addr;
8394 ir_ref call;
8395
8396 if (func->type == ZEND_INTERNAL_FUNCTION) {
8397 #ifdef ZEND_WIN32
8398 // TODO: ASLR may cause different addresses in different workers ???
8399 return 0;
8400 #endif
8401 } else if (func->type == ZEND_USER_FUNCTION) {
8402 if (!zend_accel_in_shm(func->op_array.opcodes)) {
8403 /* op_array and op_array->opcodes are not persistent. We can't link. */
8404 return 0;
8405 }
8406 } else {
8407 ZEND_UNREACHABLE();
8408 return 0;
8409 }
8410
8411 exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM);
8412 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8413 if (!exit_addr) {
8414 return 0;
8415 }
8416
8417 // call = EX(call);
8418 call = ir_LOAD_A(jit_EX(call));
8419 while (level > 0) {
8420 // call = call->prev_execute_data
8421 call = ir_LOAD_A(jit_CALL(call, prev_execute_data));
8422 level--;
8423 }
8424
8425 if (func->type == ZEND_USER_FUNCTION &&
8426 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
8427 (func->common.fn_flags & ZEND_ACC_CLOSURE) ||
8428 !func->common.function_name)) {
8429 const zend_op *opcodes = func->op_array.opcodes;
8430
8431 // JIT: if (call->func.op_array.opcodes != opcodes) goto exit_addr;
8432 ir_GUARD(
8433 ir_EQ(
8434 ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(call, func)), offsetof(zend_op_array, opcodes))),
8435 ir_CONST_ADDR(opcodes)),
8436 ir_CONST_ADDR(exit_addr));
8437 } else {
8438 // JIT: if (call->func != func) goto exit_addr;
8439 ir_GUARD(ir_EQ(ir_LOAD_A(jit_CALL(call, func)), ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
8440 }
8441
8442 return 1;
8443 }
8444
8445 static int zend_jit_init_fcall(zend_jit_ctx *jit, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack)
8446 {
8447 zend_func_info *info = ZEND_FUNC_INFO(op_array);
8448 zend_call_info *call_info = NULL;
8449 zend_function *func = NULL;
8450 ir_ref func_ref = IR_UNUSED;
8451
8452 if (jit->delayed_call_level) {
8453 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
8454 return 0;
8455 }
8456 }
8457
8458 if (info) {
8459 call_info = info->callee_info;
8460 while (call_info && call_info->caller_init_opline != opline) {
8461 call_info = call_info->next_callee;
8462 }
8463 if (call_info && call_info->callee_func && !call_info->is_prototype) {
8464 func = call_info->callee_func;
8465 }
8466 }
8467
8468 if (!func
8469 && trace
8470 && trace->op == ZEND_JIT_TRACE_INIT_CALL) {
8471 #ifdef _WIN32
8472 /* ASLR */
8473 if (trace->func->type != ZEND_INTERNAL_FUNCTION) {
8474 func = (zend_function*)trace->func;
8475 }
8476 #else
8477 func = (zend_function*)trace->func;
8478 #endif
8479 }
8480
8481 #ifdef _WIN32
8482 if (0) {
8483 #else
8484 if (opline->opcode == ZEND_INIT_FCALL
8485 && func
8486 && func->type == ZEND_INTERNAL_FUNCTION) {
8487 #endif
8488 /* load constant address later */
8489 func_ref = ir_CONST_ADDR(func);
8490 } else if (func && op_array == &func->op_array) {
8491 /* recursive call */
8492 if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)
8493 || zend_jit_prefer_const_addr_load(jit, (uintptr_t)func)) {
8494 func_ref = ir_LOAD_A(jit_EX(func));
8495 } else {
8496 func_ref = ir_CONST_ADDR(func);
8497 }
8498 } else {
8499 ir_ref if_func, cache_slot_ref, ref;
8500
8501 // JIT: if (CACHED_PTR(opline->result.num))
8502 cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num);
8503 func_ref = ir_LOAD_A(cache_slot_ref);
8504 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
8505 && func
8506 && (func->common.fn_flags & ZEND_ACC_IMMUTABLE)
8507 && opline->opcode != ZEND_INIT_FCALL) {
8508 /* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */
8509 if_func = ir_IF(ir_EQ(func_ref, ir_CONST_ADDR(func)));
8510 } else {
8511 if_func = ir_IF(func_ref);
8512 }
8513 ir_IF_FALSE_cold(if_func);
8514 if (opline->opcode == ZEND_INIT_FCALL
8515 && func
8516 && func->type == ZEND_USER_FUNCTION
8517 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) {
8518 ref = ir_HARD_COPY_A(ir_CONST_ADDR(func)); /* load constant once */
8519 ir_STORE(cache_slot_ref, ref);
8520 ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_init_func_run_time_cache_helper), ref);
8521 } else {
8522 zval *zv = RT_CONSTANT(opline, opline->op2);
8523
8524 if (opline->opcode == ZEND_INIT_FCALL) {
8525 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_func_helper),
8526 ir_CONST_ADDR(Z_STR_P(zv)),
8527 cache_slot_ref);
8528 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) {
8529 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_func_helper),
8530 ir_CONST_ADDR(Z_STR_P(zv + 1)),
8531 cache_slot_ref);
8532 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) {
8533 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_ns_func_helper),
8534 ir_CONST_ADDR(zv),
8535 cache_slot_ref);
8536 } else {
8537 ZEND_UNREACHABLE();
8538 }
8539 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
8540 int32_t exit_point = zend_jit_trace_get_exit_point(opline,
8541 func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0);
8542 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8543
8544 if (!exit_addr) {
8545 return 0;
8546 }
8547 if (!func || opline->opcode == ZEND_INIT_FCALL) {
8548 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
8549 } else if (func->type == ZEND_USER_FUNCTION
8550 && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) {
8551 const zend_op *opcodes = func->op_array.opcodes;
8552
8553 ir_GUARD(
8554 ir_EQ(
8555 ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_op_array, opcodes))),
8556 ir_CONST_ADDR(opcodes)),
8557 ir_CONST_ADDR(exit_addr));
8558 } else {
8559 ir_GUARD(ir_EQ(ref, ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
8560 }
8561 } else {
8562 jit_SET_EX_OPLINE(jit, opline);
8563 ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_undefined_function));
8564 }
8565 }
8566 ir_MERGE_WITH_EMPTY_TRUE(if_func);
8567 func_ref = ir_PHI_2(IR_ADDR, ref, func_ref);
8568 }
8569
8570 if (!zend_jit_push_call_frame(jit, opline, op_array, func, 0, 0, checked_stack, func_ref, IR_UNUSED)) {
8571 return 0;
8572 }
8573
8574 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
8575 if (!zend_jit_save_call_chain(jit, call_level)) {
8576 return 0;
8577 }
8578 } else {
8579 ZEND_ASSERT(call_level > 0);
8580 jit->delayed_call_level = call_level;
8581 delayed_call_chain = 1;
8582 }
8583
8584 return 1;
8585 }
8586
8587 static int zend_jit_init_method_call(zend_jit_ctx *jit,
8588 const zend_op *opline,
8589 uint32_t b,
8590 const zend_op_array *op_array,
8591 zend_ssa *ssa,
8592 const zend_ssa_op *ssa_op,
8593 int call_level,
8594 uint32_t op1_info,
8595 zend_jit_addr op1_addr,
8596 zend_class_entry *ce,
8597 bool ce_is_instanceof,
8598 bool on_this,
8599 bool delayed_fetch_this,
8600 zend_class_entry *trace_ce,
8601 zend_jit_trace_rec *trace,
8602 int checked_stack,
8603 int8_t func_reg,
8604 int8_t this_reg,
8605 bool polymorphic_side_trace)
8606 {
8607 zend_func_info *info = ZEND_FUNC_INFO(op_array);
8608 zend_call_info *call_info = NULL;
8609 zend_function *func = NULL;
8610 zval *function_name;
8611 ir_ref if_static = IR_UNUSED, cold_path, this_ref = IR_NULL, func_ref = IR_NULL;
8612
8613 ZEND_ASSERT(opline->op2_type == IS_CONST);
8614 ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
8615
8616 function_name = RT_CONSTANT(opline, opline->op2);
8617
8618 if (info) {
8619 call_info = info->callee_info;
8620 while (call_info && call_info->caller_init_opline != opline) {
8621 call_info = call_info->next_callee;
8622 }
8623 if (call_info && call_info->callee_func && !call_info->is_prototype) {
8624 func = call_info->callee_func;
8625 }
8626 }
8627
8628 if (polymorphic_side_trace) {
8629 /* function is passed in r0 from parent_trace */
8630 ZEND_ASSERT(func_reg >= 0 && this_reg >= 0);
8631 func_ref = zend_jit_deopt_rload(jit, IR_ADDR, func_reg);
8632 this_ref = zend_jit_deopt_rload(jit, IR_ADDR, this_reg);
8633 } else {
8634 ir_ref ref, ref2, if_found, fast_path, run_time_cache, this_ref2;
8635
8636 if (on_this) {
8637 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
8638 this_ref = jit_Z_PTR(jit, this_addr);
8639 } else {
8640 if (op1_info & MAY_BE_REF) {
8641 if (opline->op1_type == IS_CV) {
8642 // JIT: ZVAL_DEREF(op1)
8643 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
8644 ref = jit_ZVAL_DEREF_ref(jit, ref);
8645 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
8646 } else {
8647 ir_ref if_ref;
8648
8649 /* Hack: Convert reference to regular value to simplify JIT code */
8650 ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP);
8651
8652 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
8653 ir_IF_TRUE(if_ref);
8654 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper), jit_ZVAL_ADDR(jit, op1_addr));
8655
8656 ir_MERGE_WITH_EMPTY_FALSE(if_ref);
8657 }
8658 }
8659 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
8660 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
8661 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
8662 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8663
8664 if (!exit_addr) {
8665 return 0;
8666 }
8667 ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_OBJECT)),
8668 ir_CONST_ADDR(exit_addr));
8669 } else {
8670 ir_ref if_object = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
8671
8672 ir_IF_FALSE_cold(if_object);
8673
8674 jit_SET_EX_OPLINE(jit, opline);
8675 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
8676 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_method_call_tmp),
8677 jit_ZVAL_ADDR(jit, op1_addr));
8678 } else {
8679 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_method_call),
8680 jit_ZVAL_ADDR(jit, op1_addr));
8681 }
8682 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
8683 ir_IF_TRUE(if_object);
8684 }
8685 }
8686
8687 this_ref = jit_Z_PTR(jit, op1_addr);
8688 }
8689
8690 if (jit->delayed_call_level) {
8691 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
8692 return 0;
8693 }
8694 }
8695
8696 if (func) {
8697 // JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*));
8698 ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num + sizeof(void*)));
8699
8700 if_found = ir_IF(ref);
8701 ir_IF_TRUE(if_found);
8702 fast_path = ir_END();
8703 } else {
8704 // JIT: if (CACHED_PTR(opline->result.num) == obj->ce)) {
8705 run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
8706 ref = ir_EQ(
8707 ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->result.num)),
8708 ir_LOAD_A(ir_ADD_OFFSET(this_ref, offsetof(zend_object, ce))));
8709 if_found = ir_IF(ref);
8710 ir_IF_TRUE(if_found);
8711
8712 // JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*));
8713 ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->result.num + sizeof(void*)));
8714 fast_path = ir_END();
8715
8716 }
8717
8718 ir_IF_FALSE_cold(if_found);
8719 jit_SET_EX_OPLINE(jit, opline);
8720
8721 if (!jit->ctx.fixed_call_stack_size) {
8722 // JIT: alloca(sizeof(void*));
8723 this_ref2 = ir_ALLOCA(ir_CONST_ADDR(0x10));
8724 } else {
8725 this_ref2 = ir_HARD_COPY_A(ir_RLOAD_A(IR_REG_SP));
8726 }
8727 ir_STORE(this_ref2, this_ref);
8728
8729 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
8730 ref2 = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_method_tmp_helper),
8731 this_ref,
8732 ir_CONST_ADDR(function_name),
8733 this_ref2);
8734 } else {
8735 ref2 = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_method_helper),
8736 this_ref,
8737 ir_CONST_ADDR(function_name),
8738 this_ref2);
8739 }
8740
8741 this_ref2 = ir_LOAD_A(ir_RLOAD_A(IR_REG_SP));
8742 if (!jit->ctx.fixed_call_stack_size) {
8743 // JIT: revert alloca
8744 ir_AFREE(ir_CONST_ADDR(0x10));
8745 }
8746
8747 ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
8748
8749 ir_MERGE_WITH(fast_path);
8750 func_ref = ir_PHI_2(IR_ADDR, ref2, ref);
8751 this_ref = ir_PHI_2(IR_ADDR, this_ref2, this_ref);
8752 }
8753
8754 if ((!func || zend_jit_may_be_modified(func, op_array))
8755 && trace
8756 && trace->op == ZEND_JIT_TRACE_INIT_CALL
8757 && trace->func
8758 #ifdef _WIN32
8759 && trace->func->type != ZEND_INTERNAL_FUNCTION
8760 #endif
8761 ) {
8762 int32_t exit_point;
8763 const void *exit_addr;
8764
8765 exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL);
8766 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8767 if (!exit_addr) {
8768 return 0;
8769 }
8770
8771 jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
8772 jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
8773
8774 func = (zend_function*)trace->func;
8775
8776 if (func->type == ZEND_USER_FUNCTION &&
8777 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
8778 (func->common.fn_flags & ZEND_ACC_CLOSURE) ||
8779 !func->common.function_name)) {
8780 const zend_op *opcodes = func->op_array.opcodes;
8781
8782 ir_GUARD(
8783 ir_EQ(
8784 ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes))),
8785 ir_CONST_ADDR(opcodes)),
8786 ir_CONST_ADDR(exit_addr));
8787 } else {
8788 ir_GUARD(ir_EQ(func_ref, ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
8789 }
8790 }
8791
8792 if (!func) {
8793 // JIT: if (fbc->common.fn_flags & ZEND_ACC_STATIC) {
8794 if_static = ir_IF(ir_AND_U32(
8795 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, common.fn_flags))),
8796 ir_CONST_U32(ZEND_ACC_STATIC)));
8797 ir_IF_TRUE_cold(if_static);
8798 }
8799
8800 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) {
8801 ir_ref ret;
8802
8803 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
8804 ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_metod_call_frame_tmp),
8805 this_ref,
8806 func_ref,
8807 ir_CONST_U32(opline->extended_value));
8808 } else {
8809 ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_metod_call_frame),
8810 this_ref,
8811 func_ref,
8812 ir_CONST_U32(opline->extended_value));
8813 }
8814
8815 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) {
8816 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
8817 }
8818 jit_STORE_IP(jit, ret);
8819 }
8820
8821 if (!func) {
8822 cold_path = ir_END();
8823 ir_IF_FALSE(if_static);
8824 }
8825
8826 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) {
8827 if (!zend_jit_push_call_frame(jit, opline, NULL, func, 0, delayed_fetch_this, checked_stack, func_ref, this_ref)) {
8828 return 0;
8829 }
8830 }
8831
8832 if (!func) {
8833 ir_MERGE_WITH(cold_path);
8834 }
8835 zend_jit_start_reuse_ip(jit);
8836
8837 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
8838 if (!zend_jit_save_call_chain(jit, call_level)) {
8839 return 0;
8840 }
8841 } else {
8842 ZEND_ASSERT(call_level > 0);
8843 delayed_call_chain = 1;
8844 jit->delayed_call_level = call_level;
8845 }
8846
8847 return 1;
8848 }
8849
8850 static int zend_jit_init_closure_call(zend_jit_ctx *jit,
8851 const zend_op *opline,
8852 uint32_t b,
8853 const zend_op_array *op_array,
8854 zend_ssa *ssa,
8855 const zend_ssa_op *ssa_op,
8856 int call_level,
8857 zend_jit_trace_rec *trace,
8858 int checked_stack)
8859 {
8860 zend_function *func = NULL;
8861 zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
8862 ir_ref ref;
8863
8864 ref = jit_Z_PTR(jit, op2_addr);
8865
8866 if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure
8867 && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) {
8868 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
8869 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8870
8871 if (!exit_addr) {
8872 return 0;
8873 }
8874
8875 ir_GUARD(
8876 ir_EQ(
8877 ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_object, ce))),
8878 ir_CONST_ADDR(zend_ce_closure)),
8879 ir_CONST_ADDR(exit_addr));
8880
8881 if (ssa->var_info && ssa_op->op2_use >= 0) {
8882 ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD;
8883 ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure;
8884 ssa->var_info[ssa_op->op2_use].is_instanceof = 0;
8885 }
8886 }
8887
8888 if (trace
8889 && trace->op == ZEND_JIT_TRACE_INIT_CALL
8890 && trace->func
8891 && trace->func->type == ZEND_USER_FUNCTION) {
8892 const zend_op *opcodes;
8893 int32_t exit_point;
8894 const void *exit_addr;
8895
8896 func = (zend_function*)trace->func;
8897 opcodes = func->op_array.opcodes;
8898 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL);
8899 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8900 if (!exit_addr) {
8901 return 0;
8902 }
8903
8904 ir_GUARD(
8905 ir_EQ(
8906 ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_closure, func.op_array.opcodes))),
8907 ir_CONST_ADDR(opcodes)),
8908 ir_CONST_ADDR(exit_addr));
8909 }
8910
8911 if (jit->delayed_call_level) {
8912 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
8913 return 0;
8914 }
8915 }
8916
8917 if (!zend_jit_push_call_frame(jit, opline, NULL, func, 1, 0, checked_stack, ref, IR_UNUSED)) {
8918 return 0;
8919 }
8920
8921 if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
8922 if (!zend_jit_save_call_chain(jit, call_level)) {
8923 return 0;
8924 }
8925 } else {
8926 ZEND_ASSERT(call_level > 0);
8927 delayed_call_chain = 1;
8928 jit->delayed_call_level = call_level;
8929 }
8930
8931 if (trace
8932 && trace->op == ZEND_JIT_TRACE_END
8933 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
8934 if (!zend_jit_set_ip(jit, opline + 1)) {
8935 return 0;
8936 }
8937 }
8938
8939 return 1;
8940 }
8941
8942 static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr)
8943 {
8944 uint32_t arg_num = opline->op2.num;
8945 zend_jit_addr arg_addr;
8946
8947 ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM);
8948
8949 if (!zend_jit_reuse_ip(jit)) {
8950 return 0;
8951 }
8952
8953 if (opline->opcode == ZEND_SEND_VAL_EX) {
8954 uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2);
8955
8956 ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM);
8957
8958 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
8959 && JIT_G(current_frame)
8960 && JIT_G(current_frame)->call
8961 && JIT_G(current_frame)->call->func) {
8962 if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
8963 /* Don't generate code that always throws exception */
8964 return 0;
8965 }
8966 } else {
8967 ir_ref cond = ir_AND_U32(
8968 ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))),
8969 ir_CONST_U32(mask));
8970
8971 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
8972 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
8973 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
8974 if (!exit_addr) {
8975 return 0;
8976 }
8977 ir_GUARD_NOT(cond, ir_CONST_ADDR(exit_addr));
8978 } else {
8979 ir_ref if_pass_by_ref;
8980
8981 if_pass_by_ref = ir_IF(cond);
8982
8983 ir_IF_TRUE_cold(if_pass_by_ref);
8984 if (Z_MODE(op1_addr) == IS_REG) {
8985 /* set type to avoid zval_ptr_dtor() on uninitialized value */
8986 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
8987 jit_set_Z_TYPE_INFO(jit, addr, IS_UNDEF);
8988 }
8989 jit_SET_EX_OPLINE(jit, opline);
8990 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_throw_cannot_pass_by_ref));
8991
8992 ir_IF_FALSE(if_pass_by_ref);
8993 }
8994 }
8995 }
8996
8997 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
8998
8999 if (opline->op1_type == IS_CONST) {
9000 zval *zv = RT_CONSTANT(opline, opline->op1);
9001
9002 jit_ZVAL_COPY_CONST(jit,
9003 arg_addr,
9004 MAY_BE_ANY, MAY_BE_ANY,
9005 zv, 1);
9006 } else {
9007 jit_ZVAL_COPY(jit,
9008 arg_addr,
9009 MAY_BE_ANY,
9010 op1_addr, op1_info, 0);
9011 }
9012
9013 return 1;
9014 }
9015
9016 static int zend_jit_send_ref(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold)
9017 {
9018 zend_jit_addr op1_addr, arg_addr, ref_addr;
9019 ir_ref ref_path = IR_UNUSED;
9020
9021 op1_addr = OP1_ADDR();
9022 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
9023
9024 if (!zend_jit_reuse_ip(jit)) {
9025 return 0;
9026 }
9027
9028 if (opline->op1_type == IS_VAR) {
9029 if (op1_info & MAY_BE_INDIRECT) {
9030 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
9031 }
9032 } else if (opline->op1_type == IS_CV) {
9033 if (op1_info & MAY_BE_UNDEF) {
9034 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
9035 // JIT: if (Z_TYPE_P(op1) == IS_UNDEF)
9036 ir_ref if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
9037 ir_IF_FALSE(if_def);
9038 // JIT: ZVAL_NULL(op1)
9039 jit_set_Z_TYPE_INFO(jit,op1_addr, IS_NULL);
9040 ir_MERGE_WITH_EMPTY_TRUE(if_def);
9041 }
9042 op1_info &= ~MAY_BE_UNDEF;
9043 op1_info |= MAY_BE_NULL;
9044 }
9045 } else {
9046 ZEND_UNREACHABLE();
9047 }
9048
9049 if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) {
9050 ir_ref ref, ref2;
9051
9052 if (op1_info & MAY_BE_REF) {
9053 ir_ref if_ref;
9054
9055 // JIT: if (Z_TYPE_P(op1) == IS_UNDEF)
9056 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
9057 ir_IF_TRUE(if_ref);
9058 // JIT: ref = Z_PTR_P(op1)
9059 ref = jit_Z_PTR(jit, op1_addr);
9060 // JIT: GC_ADDREF(ref)
9061 jit_GC_ADDREF(jit, ref);
9062 // JIT: ZVAL_REFERENCE(arg, ref)
9063 jit_set_Z_PTR(jit, arg_addr, ref);
9064 jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
9065 ref_path = ir_END();
9066 ir_IF_FALSE(if_ref);
9067 }
9068
9069 // JIT: ZVAL_NEW_REF(arg, varptr);
9070 // JIT: ref = emalloc(sizeof(zend_reference));
9071 ref = jit_EMALLOC(jit, sizeof(zend_reference), op_array, opline);
9072 // JIT: GC_REFCOUNT(ref) = 2
9073 jit_set_GC_REFCOUNT(jit, ref, 2);
9074 // JIT: GC_TYPE(ref) = GC_REFERENCE
9075 ir_STORE(ir_ADD_OFFSET(ref, offsetof(zend_reference, gc.u.type_info)), ir_CONST_U32(GC_REFERENCE));
9076 ir_STORE(ir_ADD_OFFSET(ref, offsetof(zend_reference, sources.ptr)), IR_NULL);
9077 ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
9078 ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
9079
9080 // JIT: ZVAL_COPY_VALUE(&ref->val, op1)
9081 jit_ZVAL_COPY(jit,
9082 ref_addr,
9083 MAY_BE_ANY,
9084 op1_addr, op1_info, 0);
9085
9086 // JIT: ZVAL_REFERENCE(arg, ref)
9087 jit_set_Z_PTR(jit, op1_addr, ref);
9088 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
9089
9090 // JIT: ZVAL_REFERENCE(arg, ref)
9091 jit_set_Z_PTR(jit, arg_addr, ref);
9092 jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
9093 }
9094
9095 if (ref_path) {
9096 ir_MERGE_WITH(ref_path);
9097 }
9098
9099 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
9100
9101 return 1;
9102 }
9103
9104 static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr)
9105 {
9106 uint32_t arg_num = opline->op2.num;
9107 zend_jit_addr arg_addr;
9108 ir_ref end_inputs = IR_UNUSED;
9109
9110 ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX &&
9111 opline->opcode != ZEND_SEND_VAR_NO_REF_EX) ||
9112 arg_num <= MAX_ARG_FLAG_NUM);
9113
9114 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
9115
9116 if (!zend_jit_reuse_ip(jit)) {
9117 return 0;
9118 }
9119
9120 if (opline->opcode == ZEND_SEND_VAR_EX) {
9121 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
9122 && JIT_G(current_frame)
9123 && JIT_G(current_frame)->call
9124 && JIT_G(current_frame)->call->func) {
9125 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
9126 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
9127 return 0;
9128 }
9129 return 1;
9130 }
9131 } else {
9132 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
9133
9134 // JIT: if (RX->func->quick_arg_flags & mask)
9135 ir_ref if_send_by_ref = ir_IF(ir_AND_U32(
9136 ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))),
9137 ir_CONST_U32(mask)));
9138 ir_IF_TRUE_cold(if_send_by_ref);
9139
9140 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
9141 return 0;
9142 }
9143
9144 ir_END_list(end_inputs);
9145 ir_IF_FALSE(if_send_by_ref);
9146 }
9147 } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
9148 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
9149 && JIT_G(current_frame)
9150 && JIT_G(current_frame)->call
9151 && JIT_G(current_frame)->call->func) {
9152 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
9153
9154 // JIT: ZVAL_COPY_VALUE(arg, op1)
9155 jit_ZVAL_COPY(jit,
9156 arg_addr,
9157 MAY_BE_ANY,
9158 op1_addr, op1_info, 0);
9159
9160 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
9161 if (!(op1_info & MAY_BE_REF)) {
9162 /* Don't generate code that always throws exception */
9163 return 0;
9164 } else {
9165 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
9166 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
9167 if (!exit_addr) {
9168 return 0;
9169 }
9170
9171 // JIT: if (Z_TYPE_P(op1) != IS_REFERENCE)
9172 ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U32(IS_REFERENCE)),
9173 ir_CONST_ADDR(exit_addr));
9174 }
9175 }
9176 return 1;
9177 }
9178 } else {
9179 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
9180 ir_ref func, if_send_by_ref, if_prefer_ref;
9181
9182 // JIT: if (RX->func->quick_arg_flags & mask)
9183 func = ir_LOAD_A(jit_RX(func));
9184 if_send_by_ref = ir_IF(ir_AND_U32(
9185 ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))),
9186 ir_CONST_U32(mask)));
9187 ir_IF_TRUE_cold(if_send_by_ref);
9188
9189 mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2);
9190
9191 // JIT: ZVAL_COPY_VALUE(arg, op1)
9192 jit_ZVAL_COPY(jit,
9193 arg_addr,
9194 MAY_BE_ANY,
9195 op1_addr, op1_info, 0);
9196
9197 if (op1_info & MAY_BE_REF) {
9198 ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
9199 ir_IF_TRUE(if_ref);
9200 ir_END_list(end_inputs);
9201 ir_IF_FALSE(if_ref);
9202 }
9203
9204 // JIT: if (RX->func->quick_arg_flags & mask)
9205 if_prefer_ref = ir_IF(ir_AND_U32(
9206 ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))),
9207 ir_CONST_U32(mask)));
9208 ir_IF_TRUE(if_prefer_ref);
9209 ir_END_list(end_inputs);
9210 ir_IF_FALSE(if_prefer_ref);
9211
9212 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
9213 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
9214 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
9215 if (!exit_addr) {
9216 return 0;
9217 }
9218 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
9219 } else {
9220 jit_SET_EX_OPLINE(jit, opline);
9221 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_only_vars_by_reference),
9222 jit_ZVAL_ADDR(jit, arg_addr));
9223 zend_jit_check_exception(jit);
9224 ir_END_list(end_inputs);
9225 }
9226
9227 ir_IF_FALSE(if_send_by_ref);
9228 }
9229 } else if (opline->opcode == ZEND_SEND_FUNC_ARG) {
9230 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
9231 && JIT_G(current_frame)
9232 && JIT_G(current_frame)->call
9233 && JIT_G(current_frame)->call->func) {
9234 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
9235 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
9236 return 0;
9237 }
9238 return 1;
9239 }
9240 } else {
9241 // JIT: if (RX->This.u1.type_info & ZEND_CALL_SEND_ARG_BY_REF)
9242 ir_ref if_send_by_ref = ir_IF(ir_AND_U32(
9243 ir_LOAD_U32(jit_RX(This.u1.type_info)),
9244 ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
9245 ir_IF_TRUE_cold(if_send_by_ref);
9246
9247 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
9248 return 0;
9249 }
9250
9251 ir_END_list(end_inputs);
9252 ir_IF_FALSE(if_send_by_ref);
9253 }
9254 }
9255
9256 if (op1_info & MAY_BE_UNDEF) {
9257 ir_ref ref, if_def = IR_UNUSED;
9258
9259 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
9260 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
9261 ir_IF_FALSE_cold(if_def);
9262 }
9263
9264 // JIT: zend_jit_undefined_op_helper(opline->op1.var)
9265 jit_SET_EX_OPLINE(jit, opline);
9266 ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
9267 ir_CONST_U32(opline->op1.var));
9268
9269 // JIT: ZVAL_NULL(arg)
9270 jit_set_Z_TYPE_INFO(jit, arg_addr, IS_NULL);
9271
9272 // JIT: check_exception
9273 ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9274
9275 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
9276 ir_END_list(end_inputs);
9277 ir_IF_TRUE(if_def);
9278 } else {
9279 if (end_inputs) {
9280 ir_END_list(end_inputs);
9281 ir_MERGE_list(end_inputs);
9282 }
9283 return 1;
9284 }
9285 }
9286
9287 if (opline->opcode == ZEND_SEND_VAR_NO_REF) {
9288 // JIT: ZVAL_COPY_VALUE(arg, op1)
9289 jit_ZVAL_COPY(jit,
9290 arg_addr,
9291 MAY_BE_ANY,
9292 op1_addr, op1_info, 0);
9293 if (op1_info & MAY_BE_REF) {
9294 // JIT: if (Z_TYPE_P(arg) == IS_REFERENCE)
9295 ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
9296 ir_IF_TRUE(if_ref);
9297 ir_END_list(end_inputs);
9298 ir_IF_FALSE(if_ref);
9299 }
9300 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
9301 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
9302 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
9303 if (!exit_addr) {
9304 return 0;
9305 }
9306 ir_GUARD(IR_FALSE, ir_CONST_ADDR(exit_addr));
9307 } else {
9308 jit_SET_EX_OPLINE(jit, opline);
9309 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_only_vars_by_reference),
9310 jit_ZVAL_ADDR(jit, arg_addr));
9311 zend_jit_check_exception(jit);
9312 }
9313 } else {
9314 if (op1_info & MAY_BE_REF) {
9315 if (opline->op1_type == IS_CV) {
9316 ir_ref ref;
9317
9318 // JIT: ZVAL_DEREF(op1)
9319 ref = jit_ZVAL_ADDR(jit, op1_addr);
9320 ref = jit_ZVAL_DEREF_ref(jit, ref);
9321 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
9322
9323 // JIT: ZVAL_COPY(arg, op1)
9324 jit_ZVAL_COPY(jit,
9325 arg_addr,
9326 MAY_BE_ANY,
9327 op1_addr, op1_info, 1);
9328 } else {
9329 ir_ref if_ref, ref, ref2, refcount, if_not_zero, if_refcounted;
9330 zend_jit_addr ref_addr;
9331
9332 // JIT: if (Z_TYPE_P(op1) == IS_REFERENCE)
9333 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
9334 ir_IF_TRUE_cold(if_ref);
9335
9336 // JIT: ref = Z_COUNTED_P(op1);
9337 ref = jit_Z_PTR(jit, op1_addr);
9338 ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
9339 ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
9340
9341 // JIT: ZVAL_COPY_VALUE(arg, op1);
9342 jit_ZVAL_COPY(jit,
9343 arg_addr,
9344 MAY_BE_ANY,
9345 ref_addr, op1_info, 0);
9346
9347 // JIT: if (GC_DELREF(ref) != 0)
9348 refcount = jit_GC_DELREF(jit, ref);
9349 if_not_zero = ir_IF(refcount);
9350 ir_IF_TRUE(if_not_zero);
9351
9352 // JIT: if (Z_REFCOUNTED_P(arg)
9353 if_refcounted = jit_if_REFCOUNTED(jit, arg_addr);
9354 ir_IF_TRUE(if_refcounted);
9355 // JIT: Z_ADDREF_P(arg)
9356 jit_GC_ADDREF(jit, jit_Z_PTR(jit, arg_addr));
9357 ir_END_list(end_inputs);
9358 ir_IF_FALSE(if_refcounted);
9359 ir_END_list(end_inputs);
9360
9361 ir_IF_FALSE(if_not_zero);
9362
9363 // JIT: efree(ref)
9364 jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
9365 ir_END_list(end_inputs);
9366
9367 ir_IF_FALSE(if_ref);
9368
9369 // JIT: ZVAL_COPY_VALUE(arg, op1);
9370 jit_ZVAL_COPY(jit,
9371 arg_addr,
9372 MAY_BE_ANY,
9373 op1_addr, op1_info, 0);
9374 }
9375 } else {
9376 if (op1_addr != op1_def_addr) {
9377 if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
9378 return 0;
9379 }
9380 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
9381 op1_addr = op1_def_addr;
9382 }
9383 }
9384
9385 // JIT: ZVAL_COPY_VALUE(arg, op1)
9386 jit_ZVAL_COPY(jit,
9387 arg_addr,
9388 MAY_BE_ANY,
9389 op1_addr, op1_info, opline->op1_type == IS_CV);
9390 }
9391 }
9392
9393 if (end_inputs) {
9394 ir_END_list(end_inputs);
9395 ir_MERGE_list(end_inputs);
9396 }
9397
9398 return 1;
9399 }
9400
9401 static int zend_jit_check_func_arg(zend_jit_ctx *jit, const zend_op *opline)
9402 {
9403 uint32_t arg_num = opline->op2.num;
9404 ir_ref ref;
9405
9406 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
9407 && JIT_G(current_frame)
9408 && JIT_G(current_frame)->call
9409 && JIT_G(current_frame)->call->func) {
9410 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
9411 if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) {
9412 TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call);
9413 // JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
9414 if (jit->reuse_ip) {
9415 ref = jit_IP(jit);
9416 } else {
9417 ref = ir_LOAD_A(jit_EX(call));
9418 }
9419 ref = jit_CALL(ref, This.u1.type_info);
9420 ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
9421 }
9422 } else {
9423 if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
9424 TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call);
9425 // JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
9426 if (jit->reuse_ip) {
9427 ref = jit_IP(jit);
9428 } else {
9429 ref = ir_LOAD_A(jit_EX(call));
9430 }
9431 ref = jit_CALL(ref, This.u1.type_info);
9432 ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF)));
9433 }
9434 }
9435 } else {
9436 // JIT: if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) {
9437 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
9438 ir_ref rx, if_ref, cold_path;
9439
9440 if (!zend_jit_reuse_ip(jit)) {
9441 return 0;
9442 }
9443
9444 rx = jit_IP(jit);
9445
9446 ref = ir_AND_U32(
9447 ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(rx, func)), offsetof(zend_function, quick_arg_flags))),
9448 ir_CONST_U32(mask));
9449 if_ref = ir_IF(ref);
9450 ir_IF_TRUE_cold(if_ref);
9451
9452 // JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
9453 ref = jit_CALL(rx, This.u1.type_info);
9454 ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
9455
9456 cold_path = ir_END();
9457 ir_IF_FALSE(if_ref);
9458
9459 // JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
9460 ref = jit_CALL(rx, This.u1.type_info);
9461 ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF)));
9462
9463 ir_MERGE_WITH(cold_path);
9464 }
9465
9466 return 1;
9467 }
9468
9469 static int zend_jit_check_undef_args(zend_jit_ctx *jit, const zend_op *opline)
9470 {
9471 ir_ref call, if_may_have_undef, ret;
9472
9473 if (jit->reuse_ip) {
9474 call = jit_IP(jit);
9475 } else {
9476 call = ir_LOAD_A(jit_EX(call));
9477 }
9478
9479 if_may_have_undef = ir_IF(ir_AND_U8(
9480 ir_LOAD_U8(ir_ADD_OFFSET(call, offsetof(zend_execute_data, This.u1.type_info) + 3)),
9481 ir_CONST_U8(ZEND_CALL_MAY_HAVE_UNDEF >> 24)));
9482
9483 ir_IF_TRUE_cold(if_may_have_undef);
9484 jit_SET_EX_OPLINE(jit, opline);
9485 ret = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_handle_undef_args), call);
9486 ir_GUARD_NOT(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9487 ir_MERGE_WITH_EMPTY_FALSE(if_may_have_undef);
9488
9489 return 1;
9490 }
9491
9492 static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace)
9493 {
9494 zend_func_info *info = ZEND_FUNC_INFO(op_array);
9495 zend_call_info *call_info = NULL;
9496 const zend_function *func = NULL;
9497 uint32_t i;
9498 uint32_t call_num_args = 0;
9499 bool unknown_num_args = 0;
9500 const void *exit_addr = NULL;
9501 const zend_op *prev_opline;
9502 ir_ref rx, func_ref = IR_UNUSED, if_user = IR_UNUSED, user_path = IR_UNUSED;
9503
9504 prev_opline = opline - 1;
9505 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
9506 prev_opline--;
9507 }
9508 if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY ||
9509 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
9510 unknown_num_args = 1;
9511 }
9512
9513 if (info) {
9514 call_info = info->callee_info;
9515 while (call_info && call_info->caller_call_opline != opline) {
9516 call_info = call_info->next_callee;
9517 }
9518 if (call_info && call_info->callee_func && !call_info->is_prototype) {
9519 func = call_info->callee_func;
9520 }
9521 if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)
9522 && JIT_G(current_frame)
9523 && JIT_G(current_frame)->call
9524 && !JIT_G(current_frame)->call->func) {
9525 call_info = NULL; func = NULL; /* megamorphic call from trait */
9526 }
9527 }
9528 if (!func) {
9529 /* resolve function at run time */
9530 } else if (func->type == ZEND_USER_FUNCTION) {
9531 ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL);
9532 call_num_args = call_info->num_args;
9533 } else if (func->type == ZEND_INTERNAL_FUNCTION) {
9534 ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL);
9535 call_num_args = call_info->num_args;
9536 } else {
9537 ZEND_UNREACHABLE();
9538 }
9539
9540 if (trace && !func) {
9541 if (trace->op == ZEND_JIT_TRACE_DO_ICALL) {
9542 ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION);
9543 #ifndef ZEND_WIN32
9544 // TODO: ASLR may cause different addresses in different workers ???
9545 func = trace->func;
9546 if (JIT_G(current_frame) &&
9547 JIT_G(current_frame)->call &&
9548 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
9549 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
9550 } else {
9551 unknown_num_args = 1;
9552 }
9553 #endif
9554 } else if (trace->op == ZEND_JIT_TRACE_ENTER) {
9555 ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION);
9556 if (zend_accel_in_shm(trace->func->op_array.opcodes)) {
9557 func = trace->func;
9558 if (JIT_G(current_frame) &&
9559 JIT_G(current_frame)->call &&
9560 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
9561 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
9562 } else {
9563 unknown_num_args = 1;
9564 }
9565 }
9566 }
9567 }
9568
9569 bool may_have_extra_named_params =
9570 opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS &&
9571 (!func || func->common.fn_flags & ZEND_ACC_VARIADIC);
9572
9573 if (!jit->reuse_ip) {
9574 zend_jit_start_reuse_ip(jit);
9575 // JIT: call = EX(call);
9576 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call)));
9577 }
9578 rx = jit_IP(jit);
9579 zend_jit_stop_reuse_ip(jit);
9580
9581 jit_SET_EX_OPLINE(jit, opline);
9582
9583 if (opline->opcode == ZEND_DO_FCALL) {
9584 if (!func) {
9585 if (trace) {
9586 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
9587
9588 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
9589 if (!exit_addr) {
9590 return 0;
9591 }
9592
9593 func_ref = ir_LOAD_A(jit_CALL(rx, func));
9594 ir_GUARD_NOT(
9595 ir_AND_U32(
9596 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
9597 ir_CONST_U32(ZEND_ACC_DEPRECATED)),
9598 ir_CONST_ADDR(exit_addr));
9599 }
9600 }
9601 }
9602
9603 if (!jit->delayed_call_level) {
9604 // JIT: EX(call) = call->prev_execute_data;
9605 ir_STORE(jit_EX(call),
9606 (call_level == 1) ? IR_NULL : ir_LOAD_A(jit_CALL(rx, prev_execute_data)));
9607 }
9608 delayed_call_chain = 0;
9609 jit->delayed_call_level = 0;
9610
9611 // JIT: call->prev_execute_data = execute_data;
9612 ir_STORE(jit_CALL(rx, prev_execute_data), jit_FP(jit));
9613
9614 if (!func) {
9615 if (!func_ref) {
9616 func_ref = ir_LOAD_A(jit_CALL(rx, func));
9617 }
9618 }
9619
9620 if (opline->opcode == ZEND_DO_FCALL) {
9621 if (!func) {
9622 if (!trace) {
9623 ir_ref if_deprecated, ret;
9624
9625 if_deprecated = ir_IF(ir_AND_U32(
9626 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
9627 ir_CONST_U32(ZEND_ACC_DEPRECATED)));
9628 ir_IF_TRUE_cold(if_deprecated);
9629
9630 if (GCC_GLOBAL_REGS) {
9631 ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
9632 } else {
9633 ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
9634 }
9635 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9636 ir_MERGE_WITH_EMPTY_FALSE(if_deprecated);
9637 }
9638 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
9639 ir_ref ret;
9640
9641 if (GCC_GLOBAL_REGS) {
9642 ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
9643 } else {
9644 ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
9645 }
9646 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9647 }
9648 }
9649
9650 if (!func
9651 && opline->opcode != ZEND_DO_UCALL
9652 && opline->opcode != ZEND_DO_ICALL) {
9653 ir_ref type_ref = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type)));
9654 if_user = ir_IF(ir_EQ(type_ref, ir_CONST_U8(ZEND_USER_FUNCTION)));
9655 ir_IF_TRUE(if_user);
9656 }
9657
9658 if ((!func || func->type == ZEND_USER_FUNCTION)
9659 && opline->opcode != ZEND_DO_ICALL) {
9660 bool recursive_call_through_jmp = 0;
9661
9662 // JIT: EX(call) = NULL;
9663 ir_STORE(jit_CALL(rx, call), IR_NULL);
9664
9665 // JIT: EX(return_value) = RETURN_VALUE_USED(opline) ? EX_VAR(opline->result.var) : 0;
9666 ir_STORE(jit_CALL(rx, return_value),
9667 RETURN_VALUE_USED(opline) ?
9668 jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var)) :
9669 IR_NULL);
9670
9671 // JIT: EX_LOAD_RUN_TIME_CACHE(op_array);
9672 if (!func || func->op_array.cache_size) {
9673 ir_ref run_time_cache;
9674
9675 if (func && op_array == &func->op_array) {
9676 /* recursive call */
9677 run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
9678 } else if (func
9679 && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)
9680 && ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) {
9681 run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CG(map_ptr_base)),
9682 (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache)));
9683 } else if ((func && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) ||
9684 (JIT_G(current_frame) &&
9685 JIT_G(current_frame)->call &&
9686 TRACE_FRAME_IS_CLOSURE_CALL(JIT_G(current_frame)->call))) {
9687 /* Closures always use direct pointers */
9688 ir_ref local_func_ref = func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func));
9689
9690 run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(local_func_ref, offsetof(zend_op_array, run_time_cache__ptr)));
9691 } else {
9692 ir_ref if_odd, run_time_cache2;
9693 ir_ref local_func_ref = func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func));
9694
9695 run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(local_func_ref, offsetof(zend_op_array, run_time_cache__ptr)));
9696 if_odd = ir_IF(ir_AND_A(run_time_cache, ir_CONST_ADDR(1)));
9697 ir_IF_TRUE(if_odd);
9698
9699 run_time_cache2 = ir_LOAD_A(ir_ADD_A(run_time_cache, ir_LOAD_A(jit_CG(map_ptr_base))));
9700
9701 ir_MERGE_WITH_EMPTY_FALSE(if_odd);
9702 run_time_cache = ir_PHI_2(IR_ADDR, run_time_cache2, run_time_cache);
9703 }
9704
9705 ir_STORE(jit_CALL(rx, run_time_cache), run_time_cache);
9706 }
9707
9708 // JIT: EG(current_execute_data) = execute_data = call;
9709 ir_STORE(jit_EG(current_execute_data), rx);
9710 jit_STORE_FP(jit, rx);
9711
9712 // JIT: opline = op_array->opcodes;
9713 if (func && !unknown_num_args) {
9714
9715 for (i = call_num_args; i < func->op_array.last_var; i++) {
9716 uint32_t n = EX_NUM_TO_VAR(i);
9717 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, n);
9718
9719 jit_set_Z_TYPE_INFO_ex(jit, var_addr, ir_CONST_U32(IS_UNDEF));
9720 }
9721
9722 if (call_num_args <= func->op_array.num_args) {
9723 if (!trace || (trace->op == ZEND_JIT_TRACE_END
9724 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
9725 uint32_t num_args;
9726
9727 if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) {
9728 if (trace) {
9729 num_args = 0;
9730 } else if (call_info) {
9731 num_args = skip_valid_arguments(op_array, ssa, call_info);
9732 } else {
9733 num_args = call_num_args;
9734 }
9735 } else {
9736 num_args = call_num_args;
9737 }
9738 if (zend_accel_in_shm(func->op_array.opcodes)) {
9739 jit_LOAD_IP_ADDR(jit, func->op_array.opcodes + num_args);
9740 } else {
9741 if (!func_ref) {
9742 func_ref = ir_LOAD_A(jit_CALL(rx, func));
9743 }
9744 ir_ref ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
9745 if (num_args) {
9746 ip = ir_ADD_OFFSET(ip, num_args * sizeof(zend_op));
9747 }
9748 jit_LOAD_IP(jit, ip);
9749 }
9750
9751 if (!trace && op_array == &func->op_array && call_num_args >= op_array->required_num_args) {
9752 /* recursive call */
9753 recursive_call_through_jmp = 1;
9754 }
9755 }
9756 } else {
9757 if (!trace || (trace->op == ZEND_JIT_TRACE_END
9758 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
9759 ir_ref ip;
9760
9761 if (zend_accel_in_shm(func->op_array.opcodes)) {
9762 ip = ir_CONST_ADDR(func->op_array.opcodes);
9763 } else {
9764 if (!func_ref) {
9765 func_ref = ir_LOAD_A(jit_CALL(rx, func));
9766 }
9767 ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
9768 }
9769 jit_LOAD_IP(jit, ip);
9770 }
9771 if (GCC_GLOBAL_REGS) {
9772 ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper));
9773 } else {
9774 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper), jit_FP(jit));
9775 }
9776 }
9777 } else {
9778 ir_ref ip;
9779 ir_ref merge_inputs = IR_UNUSED;
9780
9781 // JIT: opline = op_array->opcodes
9782 if (func && zend_accel_in_shm(func->op_array.opcodes)) {
9783 ip = ir_CONST_ADDR(func->op_array.opcodes);
9784 } else {
9785 if (!func_ref) {
9786 func_ref = ir_LOAD_A(jit_CALL(rx, func));
9787 }
9788 ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
9789 }
9790 jit_LOAD_IP(jit, ip);
9791
9792 // JIT: num_args = EX_NUM_ARGS();
9793 ir_ref num_args, first_extra_arg;
9794
9795 num_args = ir_LOAD_U32(jit_EX(This.u2.num_args));
9796 if (func) {
9797 first_extra_arg = ir_CONST_U32(func->op_array.num_args);
9798 } else {
9799 // JIT: first_extra_arg = op_array->num_args;
9800 ZEND_ASSERT(func_ref);
9801 first_extra_arg = ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, num_args)));
9802 }
9803
9804 // JIT: if (UNEXPECTED(num_args > first_extra_arg))
9805 ir_ref if_extra_args = ir_IF(ir_GT(num_args, first_extra_arg));
9806 ir_IF_TRUE_cold(if_extra_args);
9807 if (GCC_GLOBAL_REGS) {
9808 ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper));
9809 } else {
9810 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper), jit_FP(jit));
9811 }
9812 ir_END_list(merge_inputs);
9813 ir_IF_FALSE(if_extra_args);
9814 if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) {
9815 if (!func) {
9816 // JIT: if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0))
9817 ir_ref if_has_type_hints = ir_IF(ir_AND_U32(
9818 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
9819 ir_CONST_U32(ZEND_ACC_HAS_TYPE_HINTS)));
9820 ir_IF_TRUE(if_has_type_hints);
9821 ir_END_list(merge_inputs);
9822 ir_IF_FALSE(if_has_type_hints);
9823 }
9824 // JIT: opline += num_args;
9825
9826 ir_ref ref = ir_MUL_U32(num_args, ir_CONST_U32(sizeof(zend_op)));
9827
9828 if (sizeof(void*) == 8) {
9829 ref = ir_ZEXT_A(ref);
9830 }
9831
9832 if (GCC_GLOBAL_REGS) {
9833 jit_STORE_IP(jit, ir_ADD_A(jit_IP(jit), ref));
9834 } else {
9835 ir_ref addr = jit_EX(opline);
9836
9837 ir_STORE(addr, ir_ADD_A(ir_LOAD_A(addr), ref));
9838 }
9839 }
9840
9841 ir_END_list(merge_inputs);
9842 ir_MERGE_list(merge_inputs);
9843
9844 // JIT: if (EXPECTED((int)num_args < op_array->last_var)) {
9845 ir_ref last_var;
9846
9847 if (func) {
9848 last_var = ir_CONST_U32(func->op_array.last_var);
9849 } else {
9850 ZEND_ASSERT(func_ref);
9851 last_var = ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, last_var)));
9852 }
9853
9854 ir_ref idx = ir_SUB_U32(last_var, num_args);
9855 ir_ref if_need = ir_IF(ir_GT(idx, ir_CONST_U32(0)));
9856 ir_IF_TRUE(if_need);
9857
9858 // JIT: zval *var = EX_VAR_NUM(num_args);
9859 if (sizeof(void*) == 8) {
9860 num_args = ir_ZEXT_A(num_args);
9861 }
9862 ir_ref var_ref = ir_ADD_OFFSET(
9863 ir_ADD_A(jit_FP(jit), ir_MUL_A(num_args, ir_CONST_ADDR(sizeof(zval)))),
9864 (ZEND_CALL_FRAME_SLOT * sizeof(zval)) + offsetof(zval, u1.type_info));
9865
9866 ir_ref loop = ir_LOOP_BEGIN(ir_END());
9867 var_ref = ir_PHI_2(IR_ADDR, var_ref, IR_UNUSED);
9868 idx = ir_PHI_2(IR_U32, idx, IR_UNUSED);
9869 ir_STORE(var_ref, ir_CONST_I32(IS_UNDEF));
9870 ir_PHI_SET_OP(var_ref, 2, ir_ADD_OFFSET(var_ref, sizeof(zval)));
9871 ir_ref idx2 = ir_SUB_U32(idx, ir_CONST_U32(1));
9872 ir_PHI_SET_OP(idx, 2, idx2);
9873 ir_ref if_not_zero = ir_IF(idx2);
9874 ir_IF_TRUE(if_not_zero);
9875 ir_MERGE_SET_OP(loop, 2, ir_LOOP_END());
9876 ir_IF_FALSE(if_not_zero);
9877 ir_MERGE_WITH_EMPTY_FALSE(if_need);
9878 }
9879
9880 if (ZEND_OBSERVER_ENABLED) {
9881 if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
9882 ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END);
9883 jit_SET_EX_OPLINE(jit, trace[1].opline);
9884 } else if (GCC_GLOBAL_REGS) {
9885 // EX(opline) = opline
9886 ir_STORE(jit_EX(opline), jit_IP(jit));
9887 }
9888 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_begin), jit_FP(jit));
9889 }
9890
9891 if (trace) {
9892 if (!func && (opline->opcode != ZEND_DO_UCALL)) {
9893 user_path = ir_END();
9894 }
9895 } else {
9896 zend_basic_block *bb;
9897
9898 do {
9899 if (recursive_call_through_jmp) {
9900 ir_ref begin, end;
9901 ir_insn *insn;
9902
9903 /* attempt to convert direct recursive call into loop */
9904 begin = jit->bb_start_ref[call_num_args];
9905 ZEND_ASSERT(begin != IR_UNUSED);
9906 insn = &jit->ctx.ir_base[begin];
9907 if (insn->op == IR_BEGIN) {
9908 end = ir_LOOP_END();
9909 insn = &jit->ctx.ir_base[begin];
9910 insn->op = IR_LOOP_BEGIN;
9911 insn->inputs_count = 2;
9912 insn->op2 = end;
9913 break;
9914 } else if ((insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN)
9915 && insn->inputs_count == 2) {
9916 end = ir_LOOP_END();
9917 insn = &jit->ctx.ir_base[begin];
9918 insn->op = IR_LOOP_BEGIN;
9919 insn->inputs_count = 3;
9920 insn->op3 = end;
9921 break;
9922 } else if (insn->op == IR_LOOP_BEGIN && insn->inputs_count == 3) {
9923 ZEND_ASSERT(jit->ctx.ir_base[insn->op3].op == IR_LOOP_END);
9924 jit->ctx.ir_base[insn->op3].op = IR_END;
9925 ir_MERGE_2(insn->op3, ir_END());
9926 end = ir_LOOP_END();
9927 insn = &jit->ctx.ir_base[begin];
9928 insn->op3 = end;
9929 break;
9930 }
9931 }
9932 /* fallback to indirect JMP or RETURN */
9933 if (GCC_GLOBAL_REGS) {
9934 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
9935 } else {
9936 ir_RETURN(ir_CONST_I32(1));
9937 }
9938 } while (0);
9939
9940 bb = &jit->ssa->cfg.blocks[jit->b];
9941 if (bb->successors_count > 0) {
9942 int succ;
9943 ir_ref ref;
9944
9945 ZEND_ASSERT(bb->successors_count == 1);
9946 succ = bb->successors[0];
9947 /* Add a fake control edge from UNREACHABLE/RETURN to the following ENTRY */
9948 ref = jit->ctx.insns_count - 1;
9949 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE
9950 || jit->ctx.ir_base[ref].op == IR_RETURN
9951 || jit->ctx.ir_base[ref].op == IR_LOOP_END);
9952 ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY);
9953 ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start);
9954 if (func || (opline->opcode == ZEND_DO_UCALL)) {
9955 _zend_jit_add_predecessor_ref(jit, succ, jit->b, ref);
9956 jit->b = -1;
9957 } else {
9958 user_path = ref;
9959 }
9960 }
9961 }
9962 }
9963
9964 if ((!func || func->type == ZEND_INTERNAL_FUNCTION)
9965 && (opline->opcode != ZEND_DO_UCALL)) {
9966 if (!func && (opline->opcode != ZEND_DO_ICALL)) {
9967 ir_IF_FALSE(if_user);
9968 }
9969 if (opline->opcode == ZEND_DO_FCALL_BY_NAME) {
9970 if (!func) {
9971 if (trace) {
9972 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
9973
9974 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
9975 if (!exit_addr) {
9976 return 0;
9977 }
9978 ZEND_ASSERT(func_ref);
9979 ir_GUARD_NOT(
9980 ir_AND_U32(
9981 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
9982 ir_CONST_U32(ZEND_ACC_DEPRECATED)),
9983 ir_CONST_ADDR(exit_addr));
9984 } else {
9985 ir_ref if_deprecated, ret;
9986
9987 if_deprecated = ir_IF(ir_AND_U32(
9988 ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
9989 ir_CONST_U32(ZEND_ACC_DEPRECATED)));
9990 ir_IF_TRUE_cold(if_deprecated);
9991
9992 if (GCC_GLOBAL_REGS) {
9993 ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
9994 } else {
9995 ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
9996 }
9997 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9998 ir_MERGE_WITH_EMPTY_FALSE(if_deprecated);
9999 }
10000 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
10001 ir_ref ret;
10002
10003 if (GCC_GLOBAL_REGS) {
10004 ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
10005 } else {
10006 ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
10007 }
10008 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
10009 }
10010 }
10011
10012 // JIT: EG(current_execute_data) = execute_data;
10013 ir_STORE(jit_EG(current_execute_data), rx);
10014
10015 if (ZEND_OBSERVER_ENABLED) {
10016 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_begin), rx);
10017 }
10018
10019 // JIT: ZVAL_NULL(EX_VAR(opline->result.var));
10020 ir_ref res_addr = IR_UNUSED, func_ptr;
10021
10022 if (RETURN_VALUE_USED(opline)) {
10023 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
10024 } else {
10025 /* CPU stack allocated temporary zval */
10026 ir_ref ptr;
10027
10028 if (!jit->ctx.fixed_call_stack_size) {
10029 // JIT: alloca(sizeof(void*));
10030 ptr = ir_ALLOCA(ir_CONST_ADDR(sizeof(zval)));
10031 } else {
10032 ptr = ir_HARD_COPY_A(ir_RLOAD_A(IR_REG_SP));
10033 }
10034 res_addr = ZEND_ADDR_REF_ZVAL(ptr);
10035 }
10036
10037 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
10038
10039 zend_jit_reset_last_valid_opline(jit);
10040
10041 // JIT: (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret);
10042 if (zend_execute_internal) {
10043 ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_execute_internal), rx, jit_ZVAL_ADDR(jit, res_addr));
10044 } else {
10045 if (func) {
10046 func_ptr = ir_CONST_FC_FUNC(func->internal_function.handler);
10047 } else {
10048 func_ptr = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_internal_function, handler)));
10049 #if defined(IR_TARGET_X86)
10050 func_ptr = ir_CAST_FC_FUNC(func_ptr);
10051 #endif
10052 }
10053 ir_CALL_2(IR_VOID, func_ptr, rx, jit_ZVAL_ADDR(jit, res_addr));
10054 }
10055
10056 if (ZEND_OBSERVER_ENABLED) {
10057 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_end),
10058 rx, jit_ZVAL_ADDR(jit, res_addr));
10059 }
10060
10061 // JIT: EG(current_execute_data) = execute_data;
10062 ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
10063
10064 // JIT: zend_vm_stack_free_args(call);
10065 if (func && !unknown_num_args) {
10066 for (i = 0; i < call_num_args; i++ ) {
10067 if (zend_jit_needs_arg_dtor(func, i, call_info)) {
10068 uint32_t offset = EX_NUM_TO_VAR(i);
10069 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset);
10070
10071 jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, opline);
10072 }
10073 }
10074 } else {
10075 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_vm_stack_free_args_helper), rx);
10076 }
10077
10078 if (may_have_extra_named_params) {
10079 // JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS))
10080 ir_ref if_has_named = ir_IF(ir_AND_U8(
10081 ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 3)),
10082 ir_CONST_U8(ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24)));
10083 ir_IF_TRUE_cold(if_has_named);
10084
10085 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_extra_named_params),
10086 ir_LOAD_A(jit_CALL(rx, extra_named_params)));
10087
10088 ir_MERGE_WITH_EMPTY_FALSE(if_has_named);
10089 }
10090
10091 if (opline->opcode == ZEND_DO_FCALL) {
10092 // TODO: optimize ???
10093 // JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS))
10094 ir_ref if_release_this = ir_IF(ir_AND_U8(
10095 ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 2)),
10096 ir_CONST_U8(ZEND_CALL_RELEASE_THIS >> 16)));
10097 ir_IF_TRUE_cold(if_release_this);
10098
10099 // JIT: OBJ_RELEASE(Z_OBJ(RX->This));
10100 jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_CALL(rx, This.value.obj)));
10101
10102 ir_MERGE_WITH_EMPTY_FALSE(if_release_this);
10103 }
10104
10105
10106 ir_ref allocated_path = IR_UNUSED;
10107
10108 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
10109 !JIT_G(current_frame) ||
10110 !JIT_G(current_frame)->call ||
10111 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) ||
10112 prev_opline->opcode == ZEND_SEND_UNPACK ||
10113 prev_opline->opcode == ZEND_SEND_ARRAY ||
10114 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
10115
10116 // JIT: zend_vm_stack_free_call_frame(call);
10117 // JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_ALLOCATED))
10118 ir_ref if_allocated = ir_IF(ir_AND_U8(
10119 ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 2)),
10120 ir_CONST_U8(ZEND_CALL_ALLOCATED >> 16)));
10121 ir_IF_TRUE_cold(if_allocated);
10122
10123 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_free_call_frame), rx);
10124
10125 allocated_path = ir_END();
10126 ir_IF_FALSE(if_allocated);
10127 }
10128
10129 ir_STORE(jit_EG(vm_stack_top), rx);
10130
10131 if (allocated_path) {
10132 ir_MERGE_WITH(allocated_path);
10133 }
10134
10135 if (!RETURN_VALUE_USED(opline)) {
10136 zend_class_entry *ce;
10137 bool ce_is_instanceof;
10138 uint32_t func_info = call_info ?
10139 zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) :
10140 (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN);
10141
10142 /* If an exception is thrown, the return_value may stay at the
10143 * original value of null. */
10144 func_info |= MAY_BE_NULL;
10145
10146 if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
10147 ir_ref sp = ir_RLOAD_A(IR_REG_SP);
10148 res_addr = ZEND_ADDR_REF_ZVAL(sp);
10149 jit_ZVAL_PTR_DTOR(jit, res_addr, func_info, 1, opline);
10150 }
10151 if (!jit->ctx.fixed_call_stack_size) {
10152 // JIT: revert alloca
10153 ir_AFREE(ir_CONST_ADDR(sizeof(zval)));
10154 }
10155 }
10156
10157 // JIT: if (UNEXPECTED(EG(exception) != NULL)) {
10158 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
10159 jit_STUB_ADDR(jit, jit_stub_icall_throw));
10160
10161 // TODO: Can we avoid checking for interrupts after each call ???
10162 if (trace && jit->last_valid_opline != opline) {
10163 int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM);
10164
10165 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
10166 if (!exit_addr) {
10167 return 0;
10168 }
10169 } else {
10170 exit_addr = NULL;
10171 }
10172
10173 if (!zend_jit_check_timeout(jit, opline + 1, exit_addr)) {
10174 return 0;
10175 }
10176
10177 if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) {
10178 jit_LOAD_IP_ADDR(jit, opline + 1);
10179 } else if (trace
10180 && trace->op == ZEND_JIT_TRACE_END
10181 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
10182 jit_LOAD_IP_ADDR(jit, opline + 1);
10183 }
10184 }
10185
10186 if (user_path) {
10187 ir_MERGE_WITH(user_path);
10188 }
10189
10190 return 1;
10191 }
10192
10193 static int zend_jit_constructor(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, int next_block)
10194 {
10195 ir_ref if_skip_constructor = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, opline), next_block);
10196
10197 ir_IF_FALSE(if_skip_constructor);
10198
10199 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
10200 if (!zend_jit_tail_handler(jit, opline)) {
10201 return 0;
10202 }
10203 } else {
10204 if (!zend_jit_do_fcall(jit, opline, op_array, ssa, call_level, next_block, NULL)) {
10205 return 0;
10206 }
10207 }
10208
10209 /* override predecessors of the next block */
10210 ZEND_ASSERT(jit->ssa->cfg.blocks[next_block].predecessors_count == 1);
10211 if (!jit->ctx.control) {
10212 ZEND_ASSERT(jit->bb_edges[jit->bb_predecessors[next_block]]);
10213 ir_IF_TRUE(if_skip_constructor);
10214 ir_MERGE_2(jit->bb_edges[jit->bb_predecessors[next_block]], ir_END());
10215 jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
10216 } else {
10217 ZEND_ASSERT(!jit->bb_edges[jit->bb_predecessors[next_block]]);
10218 /* merge current control path with the true branch of constructor skip condition */
10219 ir_MERGE_WITH_EMPTY_TRUE(if_skip_constructor);
10220 jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
10221
10222 jit->b = -1;
10223 }
10224
10225 return 1;
10226 }
10227
10228 static int zend_jit_verify_arg_type(zend_jit_ctx *jit, const zend_op *opline, zend_arg_info *arg_info, bool check_exception)
10229 {
10230 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
10231 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
10232 ir_ref ref, fast_path = IR_UNUSED;
10233
10234 ref = jit_ZVAL_ADDR(jit, res_addr);
10235 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
10236 && JIT_G(current_frame)
10237 && JIT_G(current_frame)->prev) {
10238 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
10239 uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
10240
10241 if (type != IS_UNKNOWN && (type_mask & (1u << type))) {
10242 return 1;
10243 }
10244 }
10245
10246 if (ZEND_ARG_SEND_MODE(arg_info)) {
10247 if (opline->opcode == ZEND_RECV_INIT) {
10248 ref = jit_ZVAL_DEREF_ref(jit, ref);
10249 } else {
10250 ref = jit_Z_PTR_ref(jit, ref);
10251 ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
10252 }
10253 }
10254
10255 if (type_mask != 0) {
10256 if (is_power_of_two(type_mask)) {
10257 uint32_t type_code = concrete_type(type_mask);
10258 ir_ref if_ok = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(type_code));
10259 ir_IF_TRUE(if_ok);
10260 fast_path = ir_END();
10261 ir_IF_FALSE_cold(if_ok);
10262 } else {
10263 ir_ref if_ok = ir_IF(ir_AND_U32(
10264 ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE_ref(jit, ref)),
10265 ir_CONST_U32(type_mask)));
10266 ir_IF_TRUE(if_ok);
10267 fast_path = ir_END();
10268 ir_IF_FALSE_cold(if_ok);
10269 }
10270 }
10271
10272 jit_SET_EX_OPLINE(jit, opline);
10273 ref = ir_CALL_2(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_verify_arg_slow),
10274 ref, ir_CONST_ADDR(arg_info));
10275
10276 if (check_exception) {
10277 ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
10278 }
10279
10280 if (fast_path) {
10281 ir_MERGE_WITH(fast_path);
10282 }
10283
10284 return 1;
10285 }
10286
10287 static int zend_jit_recv(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array)
10288 {
10289 uint32_t arg_num = opline->op1.num;
10290 zend_arg_info *arg_info = NULL;
10291
10292 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
10293 if (EXPECTED(arg_num <= op_array->num_args)) {
10294 arg_info = &op_array->arg_info[arg_num-1];
10295 } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) {
10296 arg_info = &op_array->arg_info[op_array->num_args];
10297 }
10298 if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) {
10299 arg_info = NULL;
10300 }
10301 }
10302
10303 if (arg_info || (opline+1)->opcode != ZEND_RECV) {
10304 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
10305 if (!JIT_G(current_frame) ||
10306 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 ||
10307 arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
10308 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
10309 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
10310
10311 if (!exit_addr) {
10312 return 0;
10313 }
10314 ir_GUARD(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)),
10315 ir_CONST_ADDR(exit_addr));
10316 }
10317 } else {
10318 ir_ref if_ok =ir_IF(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)));
10319 ir_IF_FALSE_cold(if_ok);
10320
10321 jit_SET_EX_OPLINE(jit, opline);
10322 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_missing_arg_error), jit_FP(jit));
10323 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
10324 ir_IF_TRUE(if_ok);
10325 }
10326 }
10327
10328 if (arg_info) {
10329 if (!zend_jit_verify_arg_type(jit, opline, arg_info, 1)) {
10330 return 0;
10331 }
10332 }
10333
10334 return 1;
10335 }
10336
10337 static int zend_jit_recv_init(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw)
10338 {
10339 uint32_t arg_num = opline->op1.num;
10340 zval *zv = RT_CONSTANT(opline, opline->op2);
10341 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
10342 ir_ref ref, if_fail, skip_path = IR_UNUSED;
10343
10344 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
10345 && JIT_G(current_frame)
10346 && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) {
10347 if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
10348 jit_ZVAL_COPY_CONST(jit,
10349 res_addr,
10350 -1, -1,
10351 zv, 1);
10352 }
10353 } else {
10354 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
10355 (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
10356 ir_ref if_skip = ir_IF(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)));
10357 ir_IF_TRUE(if_skip);
10358 skip_path = ir_END();
10359 ir_IF_FALSE(if_skip);
10360 }
10361 jit_ZVAL_COPY_CONST(jit,
10362 res_addr,
10363 -1, -1,
10364 zv, 1);
10365 }
10366
10367 if (Z_CONSTANT_P(zv)) {
10368 jit_SET_EX_OPLINE(jit, opline);
10369 ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zval_update_constant_ex),
10370 jit_ZVAL_ADDR(jit, res_addr),
10371 ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), offsetof(zend_op_array, scope))));
10372
10373 if_fail = ir_IF(ref);
10374 ir_IF_TRUE_cold(if_fail);
10375 jit_ZVAL_PTR_DTOR(jit, res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, opline);
10376 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
10377 ir_IF_FALSE(if_fail);
10378 }
10379
10380 if (skip_path) {
10381 ir_MERGE_WITH(skip_path);
10382 }
10383
10384 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
10385 do {
10386 zend_arg_info *arg_info;
10387
10388 if (arg_num <= op_array->num_args) {
10389 arg_info = &op_array->arg_info[arg_num-1];
10390 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
10391 arg_info = &op_array->arg_info[op_array->num_args];
10392 } else {
10393 break;
10394 }
10395 if (!ZEND_TYPE_IS_SET(arg_info->type)) {
10396 break;
10397 }
10398 if (!zend_jit_verify_arg_type(jit, opline, arg_info, may_throw)) {
10399 return 0;
10400 }
10401 } while (0);
10402 }
10403
10404 return 1;
10405 }
10406
10407 static bool zend_jit_verify_return_type(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info)
10408 {
10409 zend_arg_info *arg_info = &op_array->arg_info[-1];
10410 ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type));
10411 zend_jit_addr op1_addr = OP1_ADDR();
10412 bool needs_slow_check = 1;
10413 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
10414 ir_ref fast_path = IR_UNUSED;
10415
10416 if (type_mask != 0) {
10417 if (((op1_info & MAY_BE_ANY) & type_mask) == 0) {
10418 /* pass */
10419 } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) {
10420 needs_slow_check = 0;
10421 } else if (is_power_of_two(type_mask)) {
10422 uint32_t type_code = concrete_type(type_mask);
10423 ir_ref if_ok = jit_if_Z_TYPE(jit, op1_addr, type_code);
10424
10425 ir_IF_TRUE(if_ok);
10426 fast_path = ir_END();
10427 ir_IF_FALSE_cold(if_ok);
10428 } else {
10429 ir_ref if_ok = ir_IF(ir_AND_U32(
10430 ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)),
10431 ir_CONST_U32(type_mask)));
10432
10433 ir_IF_TRUE(if_ok);
10434 fast_path = ir_END();
10435 ir_IF_FALSE_cold(if_ok);
10436 }
10437 }
10438 if (needs_slow_check) {
10439 ir_ref ref;
10440
10441 jit_SET_EX_OPLINE(jit, opline);
10442 ref = jit_ZVAL_ADDR(jit, op1_addr);
10443 if (op1_info & MAY_BE_UNDEF) {
10444 ref = zend_jit_zval_check_undef(jit, ref, opline->op1.var, NULL, 1);
10445 }
10446
10447 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_verify_return_slow),
10448 ref,
10449 ir_LOAD_A(jit_EX(func)),
10450 ir_CONST_ADDR(arg_info),
10451 ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->op2.num));
10452
10453 zend_jit_check_exception(jit);
10454
10455 if (fast_path) {
10456 ir_MERGE_WITH(fast_path);
10457 }
10458 }
10459
10460 return 1;
10461 }
10462
10463 static int zend_jit_leave_frame(zend_jit_ctx *jit)
10464 {
10465 // JIT: EG(current_execute_data) = EX(prev_execute_data);
10466 ir_STORE(jit_EG(current_execute_data), ir_LOAD_A(jit_EX(prev_execute_data)));
10467 return 1;
10468 }
10469
10470 static int zend_jit_free_cvs(zend_jit_ctx *jit)
10471 {
10472 // JIT: EG(current_execute_data) = EX(prev_execute_data);
10473 ir_STORE(jit_EG(current_execute_data), ir_LOAD_A(jit_EX(prev_execute_data)));
10474
10475 // JIT: zend_free_compiled_variables(execute_data);
10476 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_compiled_variables), jit_FP(jit));
10477 return 1;
10478 }
10479
10480 static int zend_jit_free_cv(zend_jit_ctx *jit, uint32_t info, uint32_t var)
10481 {
10482 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
10483 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
10484
10485 jit_ZVAL_PTR_DTOR(jit, var_addr, info, 1, NULL);
10486 }
10487 return 1;
10488 }
10489
10490 static int zend_jit_free_op(zend_jit_ctx *jit, const zend_op *opline, uint32_t info, uint32_t var_offset)
10491 {
10492 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
10493 jit_ZVAL_PTR_DTOR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, opline);
10494 }
10495 return 1;
10496 }
10497
10498 static int zend_jit_leave_func(zend_jit_ctx *jit,
10499 const zend_op_array *op_array,
10500 const zend_op *opline,
10501 uint32_t op1_info,
10502 bool left_frame,
10503 zend_jit_trace_rec *trace,
10504 zend_jit_trace_info *trace_info,
10505 int indirect_var_access,
10506 int may_throw)
10507 {
10508 bool may_be_top_frame =
10509 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
10510 !JIT_G(current_frame) ||
10511 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame));
10512 bool may_need_call_helper =
10513 indirect_var_access || /* may have symbol table */
10514 !op_array->function_name || /* may have symbol table */
10515 may_be_top_frame ||
10516 (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */
10517 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
10518 !JIT_G(current_frame) ||
10519 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */
10520 (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */
10521 bool may_need_release_this =
10522 !(op_array->fn_flags & ZEND_ACC_CLOSURE) &&
10523 op_array->scope &&
10524 !(op_array->fn_flags & ZEND_ACC_STATIC) &&
10525 (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
10526 !JIT_G(current_frame) ||
10527 !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame)));
10528 ir_ref call_info = IR_UNUSED, ref, cold_path = IR_UNUSED;
10529
10530 if (may_need_call_helper) {
10531 if (!left_frame) {
10532 left_frame = 1;
10533 if (!zend_jit_leave_frame(jit)) {
10534 return 0;
10535 }
10536 }
10537 /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */
10538 call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
10539 ref = ir_AND_U32(call_info,
10540 ir_CONST_U32(ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE));
10541 if (trace && trace->op != ZEND_JIT_TRACE_END) {
10542 ir_ref if_slow = ir_IF(ref);
10543
10544 ir_IF_TRUE_cold(if_slow);
10545 if (!GCC_GLOBAL_REGS) {
10546 ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_func_helper), jit_FP(jit));
10547 } else {
10548 ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_func_helper));
10549 }
10550
10551 if (may_be_top_frame) {
10552 // TODO: try to avoid this check ???
10553 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
10554 #if 0
10555 /* this check should be handled by the following OPLINE guard */
10556 | cmp IP, zend_jit_halt_op
10557 | je ->trace_halt
10558 #endif
10559 } else if (GCC_GLOBAL_REGS) {
10560 ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
10561 } else {
10562 ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
10563 }
10564 }
10565
10566 if (!GCC_GLOBAL_REGS) {
10567 // execute_data = EG(current_execute_data)
10568 jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data)));
10569 }
10570 cold_path = ir_END();
10571 ir_IF_FALSE(if_slow);
10572 } else {
10573 ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_leave_function_handler));
10574 }
10575 }
10576
10577 if ((op_array->fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) == ZEND_ACC_CLOSURE) {
10578 if (!left_frame) {
10579 left_frame = 1;
10580 if (!zend_jit_leave_frame(jit)) {
10581 return 0;
10582 }
10583 }
10584 // JIT: OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
10585 jit_OBJ_RELEASE(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), -sizeof(zend_object)));
10586 } else if (may_need_release_this) {
10587 ir_ref if_release, fast_path = IR_UNUSED;
10588
10589 if (!left_frame) {
10590 left_frame = 1;
10591 if (!zend_jit_leave_frame(jit)) {
10592 return 0;
10593 }
10594 }
10595 if (!JIT_G(current_frame) || !TRACE_FRAME_ALWAYS_RELEASE_THIS(JIT_G(current_frame))) {
10596 // JIT: if (call_info & ZEND_CALL_RELEASE_THIS)
10597 if (!call_info) {
10598 call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
10599 }
10600 if_release = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_RELEASE_THIS)));
10601 ir_IF_FALSE(if_release);
10602 fast_path = ir_END();
10603 ir_IF_TRUE(if_release);
10604 }
10605 // JIT: OBJ_RELEASE(execute_data->This))
10606 jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_EX(This.value.obj)));
10607 if (fast_path) {
10608 ir_MERGE_WITH(fast_path);
10609 }
10610 // TODO: avoid EG(excption) check for $this->foo() calls
10611 may_throw = 1;
10612 }
10613
10614 // JIT: EG(vm_stack_top) = (zval*)execute_data
10615 ir_STORE(jit_EG(vm_stack_top), jit_FP(jit));
10616
10617 // JITL execute_data = EX(prev_execute_data)
10618 jit_STORE_FP(jit, ir_LOAD_A(jit_EX(prev_execute_data)));
10619
10620 if (!left_frame) {
10621 // JIT: EG(current_execute_data) = execute_data
10622 ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
10623 }
10624
10625 if (trace) {
10626 if (trace->op != ZEND_JIT_TRACE_END
10627 && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
10628 zend_jit_reset_last_valid_opline(jit);
10629 } else {
10630 if (GCC_GLOBAL_REGS) {
10631 /* We add extra RLOAD and RSTORE to make fusion for persistent register
10632 * mov (%FP), %IP
10633 * add $0x1c, %IP
10634 * The naive (commented) code leads to extra register allocation and move.
10635 * mov (%FP), %tmp
10636 * add $0x1c, %tmp
10637 * mov %tmp, %FP
10638 */
10639 #if 0
10640 jit_STORE_IP(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(opline)), sizeof(zend_op)));
10641 #else
10642 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
10643 jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
10644 #endif
10645 } else {
10646 ir_ref ref = jit_EX(opline);
10647
10648 ir_STORE(ref, ir_ADD_OFFSET(ir_LOAD_A(ref), sizeof(zend_op)));
10649 }
10650 }
10651
10652 if (cold_path) {
10653 ir_MERGE_WITH(cold_path);
10654 }
10655
10656 if (trace->op == ZEND_JIT_TRACE_BACK
10657 && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
10658 const zend_op *next_opline = trace->opline;
10659
10660 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
10661 && (op1_info & MAY_BE_RC1)
10662 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) {
10663 /* exception might be thrown during destruction of unused return value */
10664 // JIT: if (EG(exception))
10665 ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
10666 }
10667 do {
10668 trace++;
10669 } while (trace->op == ZEND_JIT_TRACE_INIT_CALL);
10670 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
10671 next_opline = trace->opline;
10672 ZEND_ASSERT(next_opline != NULL);
10673
10674 if (trace->op == ZEND_JIT_TRACE_END
10675 && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
10676 trace_info->flags |= ZEND_JIT_TRACE_LOOP;
10677
10678 ir_ref if_eq = ir_IF(jit_CMP_IP(jit, IR_EQ, next_opline));
10679
10680 ir_IF_TRUE(if_eq);
10681 ZEND_ASSERT(jit->trace_loop_ref);
10682 ZEND_ASSERT(jit->ctx.ir_base[jit->trace_loop_ref].op2 == IR_UNUSED);
10683 ir_MERGE_SET_OP(jit->trace_loop_ref, 2, ir_END());
10684 ir_IF_FALSE(if_eq);
10685
10686 #ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
10687 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
10688 #else
10689 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape));
10690 #endif
10691 } else {
10692 ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), jit_STUB_ADDR(jit, jit_stub_trace_escape));
10693 }
10694
10695 zend_jit_set_last_valid_opline(jit, trace->opline);
10696
10697 return 1;
10698 } else if (may_throw ||
10699 (((opline->op1_type & (IS_VAR|IS_TMP_VAR))
10700 && (op1_info & MAY_BE_RC1)
10701 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)))
10702 && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) {
10703 // JIT: if (EG(exception))
10704 ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
10705 }
10706
10707 return 1;
10708 } else {
10709 // JIT: if (EG(exception))
10710 ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
10711 // JIT: opline = EX(opline) + 1
10712 if (GCC_GLOBAL_REGS) {
10713 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
10714 jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
10715 } else {
10716 ir_ref ref = jit_EX(opline);
10717
10718 ir_STORE(ref, ir_ADD_OFFSET(ir_LOAD_A(ref), sizeof(zend_op)));
10719 }
10720 }
10721
10722 if (GCC_GLOBAL_REGS) {
10723 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
10724 } else {
10725 ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
10726 }
10727
10728 jit->b = -1;
10729
10730 return 1;
10731 }
10732
10733 static void zend_jit_common_return(zend_jit_ctx *jit)
10734 {
10735 ZEND_ASSERT(jit->return_inputs);
10736 ir_MERGE_list(jit->return_inputs);
10737 }
10738
10739 static int zend_jit_return(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr)
10740 {
10741 zend_jit_addr ret_addr;
10742 int8_t return_value_used = -1;
10743 ir_ref return_value = IR_UNUSED, ref, refcount, if_return_value_used = IR_UNUSED;
10744
10745 ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name);
10746 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF));
10747
10748 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
10749 jit->return_inputs = IR_UNUSED;
10750 if (JIT_G(current_frame)) {
10751 if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
10752 return_value_used = 1;
10753 } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) {
10754 return_value_used = 0;
10755 } else {
10756 return_value_used = -1;
10757 }
10758 }
10759 }
10760
10761 if (ZEND_OBSERVER_ENABLED) {
10762 if (Z_MODE(op1_addr) == IS_REG) {
10763 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
10764
10765 if (!zend_jit_spill_store_inv(jit, op1_addr, dst, op1_info)) {
10766 return 0;
10767 }
10768 op1_addr = dst;
10769 }
10770 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_end),
10771 jit_FP(jit),
10772 jit_ZVAL_ADDR(jit, op1_addr));
10773 }
10774
10775 // JIT: if (!EX(return_value))
10776 return_value = ir_LOAD_A(jit_EX(return_value));
10777 ret_addr = ZEND_ADDR_REF_ZVAL(return_value);
10778 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
10779 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
10780 if (return_value_used == -1) {
10781 if_return_value_used = ir_IF(return_value);
10782 ir_IF_FALSE_cold(if_return_value_used);
10783 }
10784 if (return_value_used != 1) {
10785 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
10786 ir_ref if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
10787 ir_IF_FALSE(if_refcounted);
10788 ir_END_list(jit->return_inputs);
10789 ir_IF_TRUE(if_refcounted);
10790 }
10791 ref = jit_Z_PTR(jit, op1_addr);
10792 refcount = jit_GC_DELREF(jit, ref);
10793
10794 if (RC_MAY_BE_1(op1_info)) {
10795 if (RC_MAY_BE_N(op1_info)) {
10796 ir_ref if_non_zero = ir_IF(refcount);
10797 ir_IF_TRUE(if_non_zero);
10798 ir_END_list(jit->return_inputs);
10799 ir_IF_FALSE(if_non_zero);
10800 }
10801 jit_ZVAL_DTOR(jit, ref, op1_info, opline);
10802 }
10803 if (return_value_used == -1) {
10804 ir_END_list(jit->return_inputs);
10805 }
10806 }
10807 } else if (return_value_used == -1) {
10808 if_return_value_used = ir_IF(return_value);
10809 ir_IF_FALSE_cold(if_return_value_used);
10810 ir_END_list(jit->return_inputs);
10811 }
10812
10813 if (if_return_value_used) {
10814 ir_IF_TRUE(if_return_value_used);
10815 }
10816
10817 if (return_value_used == 0) {
10818 if (jit->return_inputs) {
10819 ZEND_ASSERT(JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE);
10820 ir_END_list(jit->return_inputs);
10821 ir_MERGE_list(jit->return_inputs);
10822 jit->return_inputs = IR_UNUSED;
10823 }
10824 return 1;
10825 }
10826
10827 if (opline->op1_type == IS_CONST) {
10828 zval *zv = RT_CONSTANT(opline, opline->op1);
10829
10830 jit_ZVAL_COPY_CONST(jit, ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
10831 } else if (opline->op1_type == IS_TMP_VAR) {
10832 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
10833 } else if (opline->op1_type == IS_CV) {
10834 if (op1_info & MAY_BE_REF) {
10835 ref = jit_ZVAL_ADDR(jit, op1_addr);
10836 ref = jit_ZVAL_DEREF_ref(jit, ref);
10837 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
10838 }
10839
10840 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
10841 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
10842 (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) ||
10843 !op_array->function_name) {
10844 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 1);
10845 } else if (return_value_used != 1) {
10846 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
10847 // JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr);
10848 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_NULL);
10849 } else {
10850 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
10851 }
10852 } else {
10853 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
10854 }
10855 } else {
10856 if (op1_info & MAY_BE_REF) {
10857 ir_ref if_ref, ref2, if_non_zero;
10858 zend_jit_addr ref_addr;
10859
10860 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
10861 ir_IF_TRUE_cold(if_ref);
10862
10863 // JIT: zend_refcounted *ref = Z_COUNTED_P(retval_ptr)
10864 ref = jit_Z_PTR(jit, op1_addr);
10865
10866 // JIT: ZVAL_COPY_VALUE(return_value, &ref->value)
10867 ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
10868 ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
10869 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, ref_addr, op1_info, 0);
10870 ref2 = jit_GC_DELREF(jit, ref);
10871 if_non_zero = ir_IF(ref2);
10872 ir_IF_TRUE(if_non_zero);
10873
10874 // JIT: if (IS_REFCOUNTED())
10875 ir_ref if_refcounted = jit_if_REFCOUNTED(jit, ret_addr);
10876 ir_IF_FALSE(if_refcounted);
10877 ir_END_list(jit->return_inputs);
10878 ir_IF_TRUE(if_refcounted);
10879
10880 // JIT: ADDREF
10881 ref2 = jit_Z_PTR(jit, ret_addr);
10882 jit_GC_ADDREF(jit, ref2);
10883 ir_END_list(jit->return_inputs);
10884
10885 ir_IF_FALSE(if_non_zero);
10886
10887 jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
10888 ir_END_list(jit->return_inputs);
10889
10890 ir_IF_FALSE(if_ref);
10891 }
10892 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
10893 }
10894
10895 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
10896 if (jit->return_inputs) {
10897 ir_END_list(jit->return_inputs);
10898 ir_MERGE_list(jit->return_inputs);
10899 jit->return_inputs = IR_UNUSED;
10900 }
10901 } else {
10902 ir_END_list(jit->return_inputs);
10903 jit->b = -1;
10904 }
10905
10906 return 1;
10907 }
10908
10909 static int zend_jit_bind_global(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
10910 {
10911 zend_jit_addr op1_addr = OP1_ADDR();
10912 zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2));
10913 ir_ref cache_slot_ref, idx_ref, num_used_ref, bucket_ref, ref, ref2;
10914 ir_ref if_fit, if_reference, if_same_key, fast_path;
10915 ir_ref slow_inputs = IR_UNUSED, end_inputs = IR_UNUSED;
10916
10917 // JIT: idx = (uintptr_t)CACHED_PTR(opline->extended_value) - 1;
10918 cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value);
10919 idx_ref = ir_SUB_A(ir_LOAD_A(cache_slot_ref), ir_CONST_ADDR(1));
10920
10921 // JIT: if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket)))
10922 num_used_ref = ir_MUL_U32(ir_LOAD_U32(jit_EG(symbol_table.nNumUsed)),
10923 ir_CONST_U32(sizeof(Bucket)));
10924 if (sizeof(void*) == 8) {
10925 num_used_ref = ir_ZEXT_A(num_used_ref);
10926 }
10927 if_fit = ir_IF(ir_ULT(idx_ref, num_used_ref));
10928 ir_IF_FALSE_cold(if_fit);
10929 ir_END_list(slow_inputs);
10930 ir_IF_TRUE(if_fit);
10931
10932 // JIT: Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx);
10933 bucket_ref = ir_ADD_A(ir_LOAD_A(jit_EG(symbol_table.arData)), idx_ref);
10934 if_reference = jit_if_Z_TYPE_ref(jit, bucket_ref, ir_CONST_U8(IS_REFERENCE));
10935 ir_IF_FALSE_cold(if_reference);
10936 ir_END_list(slow_inputs);
10937 ir_IF_TRUE(if_reference);
10938
10939 // JIT: (EXPECTED(p->key == varname))
10940 if_same_key = ir_IF(ir_EQ(ir_LOAD_A(ir_ADD_OFFSET(bucket_ref, offsetof(Bucket, key))), ir_CONST_ADDR(varname)));
10941 ir_IF_FALSE_cold(if_same_key);
10942 ir_END_list(slow_inputs);
10943 ir_IF_TRUE(if_same_key);
10944
10945 // JIT: GC_ADDREF(Z_PTR(p->val))
10946 ref = jit_Z_PTR_ref(jit, bucket_ref);
10947 jit_GC_ADDREF(jit, ref);
10948
10949 fast_path = ir_END();
10950 ir_MERGE_list(slow_inputs);
10951
10952 ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_global_helper),
10953 ir_CONST_ADDR(varname),
10954 cache_slot_ref);
10955
10956 ir_MERGE_WITH(fast_path);
10957 ref = ir_PHI_2(IR_ADDR, ref2, ref);
10958
10959 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
10960 ir_ref if_refcounted = IR_UNUSED, refcount, if_non_zero, if_may_not_leak;
10961
10962 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
10963 // JIT: if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr)))
10964 if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
10965 ir_IF_TRUE_cold(if_refcounted);
10966 }
10967
10968 // JIT:zend_refcounted *garbage = Z_COUNTED_P(variable_ptr);
10969 ref2 = jit_Z_PTR(jit, op1_addr);
10970
10971 // JIT: ZVAL_REF(variable_ptr, ref)
10972 jit_set_Z_PTR(jit, op1_addr, ref);
10973 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
10974
10975 // JIT: if (GC_DELREF(garbage) == 0)
10976 refcount = jit_GC_DELREF(jit, ref2);
10977 if_non_zero = ir_IF(refcount);
10978 if (!(op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
10979 ir_IF_TRUE(if_non_zero);
10980 ir_END_list(end_inputs);
10981 }
10982 ir_IF_FALSE(if_non_zero);
10983
10984 jit_ZVAL_DTOR(jit, ref2, op1_info, opline);
10985 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) {
10986 ir_END_list(end_inputs);
10987 ir_IF_TRUE(if_non_zero);
10988
10989 // JIT: GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr)
10990 if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref2);
10991 ir_IF_TRUE(if_may_not_leak);
10992 ir_END_list(end_inputs);
10993 ir_IF_FALSE(if_may_not_leak);
10994 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref2);
10995 }
10996 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
10997 ir_END_list(end_inputs);
10998 ir_IF_FALSE(if_refcounted);
10999 }
11000 }
11001
11002 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
11003 // JIT: ZVAL_REF(variable_ptr, ref)
11004 jit_set_Z_PTR(jit, op1_addr, ref);
11005 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
11006 }
11007
11008 if (end_inputs) {
11009 ir_END_list(end_inputs);
11010 ir_MERGE_list(end_inputs);
11011 }
11012
11013 return 1;
11014 }
11015
11016 static int zend_jit_free(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, int may_throw)
11017 {
11018 zend_jit_addr op1_addr = OP1_ADDR();
11019
11020 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
11021 if (may_throw) {
11022 jit_SET_EX_OPLINE(jit, opline);
11023 }
11024 if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) {
11025 ir_ref ref, if_array, if_exists, end_inputs = IR_UNUSED;
11026
11027 if (op1_info & MAY_BE_ARRAY) {
11028 if_array = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
11029 ir_IF_TRUE(if_array);
11030 ir_END_list(end_inputs);
11031 ir_IF_FALSE(if_array);
11032 }
11033 ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_iter_idx)));
11034 if_exists = ir_IF(ir_EQ(ref, ir_CONST_U32(-1)));
11035 ir_IF_TRUE(if_exists);
11036 ir_END_list(end_inputs);
11037 ir_IF_FALSE(if_exists);
11038
11039 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_hash_iterator_del), ref);
11040
11041 ir_END_list(end_inputs);
11042 ir_MERGE_list(end_inputs);
11043 }
11044
11045 jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
11046
11047 if (may_throw) {
11048 zend_jit_check_exception(jit);
11049 }
11050 }
11051
11052 return 1;
11053 }
11054
11055 static int zend_jit_echo(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
11056 {
11057 if (opline->op1_type == IS_CONST) {
11058 zval *zv;
11059 size_t len;
11060
11061 zv = RT_CONSTANT(opline, opline->op1);
11062 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
11063 len = Z_STRLEN_P(zv);
11064
11065 if (len > 0) {
11066 const char *str = Z_STRVAL_P(zv);
11067
11068 jit_SET_EX_OPLINE(jit, opline);
11069 ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_write),
11070 ir_CONST_ADDR(str), ir_CONST_ADDR(len));
11071
11072 zend_jit_check_exception(jit);
11073 }
11074 } else {
11075 zend_jit_addr op1_addr = OP1_ADDR();
11076 ir_ref ref;
11077
11078 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
11079
11080 jit_SET_EX_OPLINE(jit, opline);
11081
11082 ref = jit_Z_PTR(jit, op1_addr);
11083 ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_write),
11084 ir_ADD_OFFSET(ref, offsetof(zend_string, val)),
11085 ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_string, len))));
11086
11087 if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) {
11088 jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
11089 }
11090
11091 zend_jit_check_exception(jit);
11092 }
11093 return 1;
11094 }
11095
11096 static int zend_jit_strlen(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr)
11097 {
11098 if (opline->op1_type == IS_CONST) {
11099 zval *zv;
11100 size_t len;
11101
11102 zv = RT_CONSTANT(opline, opline->op1);
11103 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
11104 len = Z_STRLEN_P(zv);
11105
11106 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(len));
11107 if (Z_MODE(res_addr) != IS_REG) {
11108 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11109 } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11110 return 0;
11111 }
11112 } else {
11113 ir_ref ref;
11114
11115 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
11116
11117 ref = jit_Z_PTR(jit, op1_addr);
11118 ref = ir_LOAD_L(ir_ADD_OFFSET(ref, offsetof(zend_string, len)));
11119 jit_set_Z_LVAL(jit, res_addr, ref);
11120
11121 if (Z_MODE(res_addr) == IS_REG) {
11122 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11123 return 0;
11124 }
11125 } else {
11126 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11127 }
11128 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
11129 }
11130 return 1;
11131 }
11132
11133 static int zend_jit_count(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw)
11134 {
11135 if (opline->op1_type == IS_CONST) {
11136 zval *zv;
11137 zend_long count;
11138
11139 zv = RT_CONSTANT(opline, opline->op1);
11140 ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY);
11141 count = zend_hash_num_elements(Z_ARRVAL_P(zv));
11142
11143 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(count));
11144 if (Z_MODE(res_addr) != IS_REG) {
11145 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11146 } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11147 return 0;
11148 }
11149 } else {
11150 ir_ref ref;
11151
11152 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY);
11153 // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+.
11154
11155 ref = jit_Z_PTR(jit, op1_addr);
11156 if (sizeof(void*) == 8) {
11157 ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(HashTable, nNumOfElements)));
11158 ref = ir_ZEXT_L(ref);
11159 } else {
11160 ref = ir_LOAD_L(ir_ADD_OFFSET(ref, offsetof(HashTable, nNumOfElements)));
11161 }
11162 jit_set_Z_LVAL(jit, res_addr, ref);
11163
11164 if (Z_MODE(res_addr) == IS_REG) {
11165 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11166 return 0;
11167 }
11168 } else {
11169 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11170 }
11171 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
11172 }
11173
11174 if (may_throw) {
11175 zend_jit_check_exception(jit);
11176 }
11177 return 1;
11178 }
11179
11180 static int zend_jit_in_array(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
11181 {
11182 HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
11183 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
11184 ir_ref ref;
11185
11186 ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR);
11187 ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING);
11188
11189 // JIT: result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST);
11190 if (opline->op1_type != IS_CONST) {
11191 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find),
11192 ir_CONST_ADDR(ht),
11193 jit_Z_PTR(jit, op1_addr));
11194 } else {
11195 zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1));
11196
11197 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash),
11198 ir_CONST_ADDR(ht), ir_CONST_ADDR(str));
11199 }
11200
11201 if (exit_addr) {
11202 if (smart_branch_opcode == ZEND_JMPZ) {
11203 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
11204 } else {
11205 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
11206 }
11207 } else if (smart_branch_opcode) {
11208 zend_basic_block *bb;
11209
11210 ZEND_ASSERT(jit->b >= 0);
11211 bb = &jit->ssa->cfg.blocks[jit->b];
11212 ZEND_ASSERT(bb->successors_count == 2);
11213 ref = jit_IF_ex(jit, ref,
11214 (smart_branch_opcode == ZEND_JMPZ) ? target_label2 : target_label);
11215 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
11216 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
11217 jit->b = -1;
11218 } else {
11219 jit_set_Z_TYPE_INFO_ex(jit, res_addr,
11220 ir_ADD_U32(ir_ZEXT_U32(ir_NE(ref, IR_NULL)), ir_CONST_U32(IS_FALSE)));
11221 }
11222
11223 return 1;
11224 }
11225
11226 static int zend_jit_rope(zend_jit_ctx *jit, const zend_op *opline, uint32_t op2_info)
11227 {
11228 uint32_t offset;
11229
11230 offset = (opline->opcode == ZEND_ROPE_INIT) ?
11231 opline->result.var :
11232 opline->op1.var + opline->extended_value * sizeof(zend_string*);
11233
11234 if (opline->op2_type == IS_CONST) {
11235 zval *zv = RT_CONSTANT(opline, opline->op2);
11236 zend_string *str;
11237
11238 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
11239 str = Z_STR_P(zv);
11240
11241 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ir_CONST_ADDR(str));
11242 } else {
11243 zend_jit_addr op2_addr = OP2_ADDR();
11244 ir_ref ref;
11245
11246 ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
11247
11248 ref = jit_Z_PTR(jit, op2_addr);
11249 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ref);
11250 if (opline->op2_type == IS_CV) {
11251 ir_ref if_refcounted, long_path;
11252
11253 if_refcounted = jit_if_REFCOUNTED(jit, op2_addr);
11254 ir_IF_TRUE(if_refcounted);
11255 jit_GC_ADDREF(jit, ref);
11256 long_path = ir_END();
11257
11258 ir_IF_FALSE(if_refcounted);
11259 ir_MERGE_WITH(long_path);
11260 }
11261 }
11262
11263 if (opline->opcode == ZEND_ROPE_END) {
11264 zend_jit_addr res_addr = RES_ADDR();
11265 ir_ref ref;
11266
11267 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_rope_end),
11268 ir_ADD_OFFSET(jit_FP(jit), opline->op1.var),
11269 ir_CONST_U32(opline->extended_value));
11270
11271 jit_set_Z_PTR(jit, res_addr, ref);
11272 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
11273 }
11274
11275 return 1;
11276 }
11277
11278 static int zend_jit_zval_copy_deref(zend_jit_ctx *jit, zend_jit_addr res_addr, zend_jit_addr val_addr, ir_ref type)
11279 {
11280 ir_ref if_refcounted, if_reference, if_refcounted2, ptr, val2, ptr2, type2;
11281 ir_refs *merge_inputs, *types, *ptrs;
11282 #if SIZEOF_ZEND_LONG == 4
11283 ir_ref val = jit_ZVAL_ADDR(jit, val_addr);
11284 ir_refs *values; /* we need this only for zval.w2 copy */
11285 #endif
11286
11287 ir_refs_init(merge_inputs, 4);
11288 ir_refs_init(types, 4);
11289 ir_refs_init(ptrs, 4);
11290 #if SIZEOF_ZEND_LONG == 4
11291 ir_refs_init(values, 4);
11292 #endif
11293
11294 // JIT: ptr = Z_PTR_P(val);
11295 ptr = jit_Z_PTR(jit, val_addr);
11296
11297 // JIT: if (Z_OPT_REFCOUNTED_P(val)) {
11298 if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(Z_TYPE_FLAGS_MASK)));
11299 ir_IF_FALSE_cold(if_refcounted);
11300 ir_refs_add(merge_inputs, ir_END());
11301 ir_refs_add(types, type);
11302 ir_refs_add(ptrs, ptr);
11303 #if SIZEOF_ZEND_LONG == 4
11304 ir_refs_add(values, val);
11305 #endif
11306
11307 ir_IF_TRUE(if_refcounted);
11308
11309 // JIT: if (UNEXPECTED(Z_OPT_ISREF_P(val))) {
11310 if_reference = ir_IF(ir_EQ(type, ir_CONST_U32(IS_REFERENCE_EX)));
11311 // if_reference = ir_IF(ir_EQ(ir_TRUNC_U8(type), ir_CONST_U8(IS_REFERENCE))); // TODO: fix IR to avoid need for extra register ???
11312 ir_IF_TRUE(if_reference);
11313
11314 // JIT: val = Z_REFVAL_P(val);
11315 val2 = ir_ADD_OFFSET(ptr, offsetof(zend_reference, val));
11316 type2 = jit_Z_TYPE_INFO_ref(jit, val2);
11317 ptr2 = jit_Z_PTR_ref(jit, val2);
11318
11319 // JIT: if (Z_OPT_REFCOUNTED_P(val)) {
11320 if_refcounted2 = ir_IF(ir_AND_U32(type2, ir_CONST_U32(Z_TYPE_FLAGS_MASK)));
11321 ir_IF_FALSE_cold(if_refcounted2);
11322 ir_refs_add(merge_inputs, ir_END());
11323 ir_refs_add(types, type2);
11324 ir_refs_add(ptrs, ptr2);
11325 #if SIZEOF_ZEND_LONG == 4
11326 ir_refs_add(values, val2);
11327 #endif
11328
11329 ir_IF_TRUE(if_refcounted2);
11330 ir_MERGE_WITH_EMPTY_FALSE(if_reference);
11331 type = ir_PHI_2(IR_U32, type2, type);
11332 ptr = ir_PHI_2(IR_ADDR, ptr2, ptr);
11333 #if SIZEOF_ZEND_LONG == 4
11334 val = ir_PHI_2(IR_ADDR, val2, val);
11335 #endif
11336
11337 // JIT: Z_ADDREF_P(val);
11338 jit_GC_ADDREF(jit, ptr);
11339 ir_refs_add(merge_inputs, ir_END());
11340 ir_refs_add(types, type);
11341 ir_refs_add(ptrs, ptr);
11342 #if SIZEOF_ZEND_LONG == 4
11343 ir_refs_add(values, val);
11344 #endif
11345
11346 ir_MERGE_N(merge_inputs->count, merge_inputs->refs);
11347 type = ir_PHI_N(IR_U32, types->count, types->refs);
11348 ptr = ir_PHI_N(IR_ADDR, ptrs->count, ptrs->refs);
11349 #if SIZEOF_ZEND_LONG == 4
11350 val = ir_PHI_N(IR_ADDR, values->count, values->refs);
11351 val_addr = ZEND_ADDR_REF_ZVAL(val);
11352 #endif
11353
11354 // JIT: Z_PTR_P(res) = ptr;
11355 jit_set_Z_PTR(jit, res_addr, ptr);
11356 #if SIZEOF_ZEND_LONG == 4
11357 jit_set_Z_W2(jit, res_addr, jit_Z_W2(jit, val_addr));
11358 #endif
11359 jit_set_Z_TYPE_INFO_ex(jit, res_addr, type);
11360
11361 return 1;
11362 }
11363
11364 static int zend_jit_fetch_dimension_address_inner(zend_jit_ctx *jit,
11365 const zend_op *opline,
11366 uint32_t type,
11367 uint32_t op1_info,
11368 uint32_t op2_info,
11369 uint8_t dim_type,
11370 const void *found_exit_addr,
11371 const void *not_found_exit_addr,
11372 const void *exit_addr,
11373 bool result_type_guard,
11374 ir_ref ht_ref,
11375 ir_refs *found_inputs,
11376 ir_refs *found_vals,
11377 ir_ref *end_inputs,
11378 ir_ref *not_found_inputs)
11379 {
11380 zend_jit_addr op2_addr = OP2_ADDR();
11381 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
11382 ir_ref ref = IR_UNUSED, cond, if_found;
11383 ir_ref if_type = IS_UNUSED;
11384 ir_refs *test_zval_inputs, *test_zval_values;
11385
11386 ir_refs_init(test_zval_inputs, 4);
11387 ir_refs_init(test_zval_values, 4);
11388
11389 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
11390 && type == BP_VAR_R
11391 && !exit_addr) {
11392 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
11393 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
11394 if (!exit_addr) {
11395 return 0;
11396 }
11397 }
11398
11399 if (op2_info & MAY_BE_LONG) {
11400 bool op2_loaded = 0;
11401 bool packed_loaded = 0;
11402 bool bad_packed_key = 0;
11403 ir_ref if_packed = IS_UNDEF;
11404 ir_ref h = IR_UNUSED;
11405 ir_ref idx_not_found_inputs = IR_UNUSED;
11406
11407 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) {
11408 // JIT: if (EXPECTED(Z_TYPE_P(dim) == IS_LONG))
11409 if_type = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
11410 ir_IF_TRUE(if_type);
11411 }
11412 if (op1_info & MAY_BE_PACKED_GUARD) {
11413 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
11414 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
11415
11416 if (!exit_addr) {
11417 return 0;
11418 }
11419 cond = ir_AND_U32(
11420 ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
11421 ir_CONST_U32(HASH_FLAG_PACKED));
11422 if (op1_info & MAY_BE_ARRAY_PACKED) {
11423 ir_GUARD(cond, ir_CONST_ADDR(exit_addr));
11424 } else {
11425 ir_GUARD_NOT(cond, ir_CONST_ADDR(exit_addr));
11426 }
11427 }
11428 if (type == BP_VAR_W) {
11429 // JIT: hval = Z_LVAL_P(dim);
11430 h = jit_Z_LVAL(jit, op2_addr);
11431 op2_loaded = 1;
11432 }
11433 if (op1_info & MAY_BE_ARRAY_PACKED) {
11434 zend_long val = -1;
11435
11436 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
11437 val = Z_LVAL_P(Z_ZV(op2_addr));
11438 if (val >= 0 && val < HT_MAX_SIZE) {
11439 packed_loaded = 1;
11440 } else {
11441 bad_packed_key = 1;
11442 }
11443 h = ir_CONST_LONG(val);
11444 } else {
11445 if (!op2_loaded) {
11446 // JIT: hval = Z_LVAL_P(dim);
11447 h = jit_Z_LVAL(jit, op2_addr);
11448 op2_loaded = 1;
11449 }
11450 packed_loaded = 1;
11451 }
11452
11453 if (dim_type == IS_UNDEF && type == BP_VAR_W && packed_loaded) {
11454 /* don't generate "fast" code for packed array */
11455 packed_loaded = 0;
11456 }
11457
11458 if (packed_loaded) {
11459 // JIT: ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef);
11460 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
11461 if_packed = ir_IF(
11462 ir_AND_U32(
11463 ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
11464 ir_CONST_U32(HASH_FLAG_PACKED)));
11465 ir_IF_TRUE(if_packed);
11466 }
11467 // JIT: if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed))
11468 ref = ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed)));
11469 #if SIZEOF_ZEND_LONG == 8
11470 ref = ir_ZEXT_L(ref);
11471 #endif
11472 cond = ir_ULT(h, ref);
11473 if (type == BP_JIT_IS) {
11474 if (not_found_exit_addr) {
11475 ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
11476 } else {
11477 ir_ref if_fit = ir_IF(cond);
11478 ir_IF_FALSE(if_fit);
11479 ir_END_list(*end_inputs);
11480 ir_IF_TRUE(if_fit);
11481 }
11482 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
11483 ir_GUARD(cond, ir_CONST_ADDR(exit_addr));
11484 } else if (type == BP_VAR_IS && not_found_exit_addr) {
11485 ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
11486 } else if (type == BP_VAR_RW && not_found_exit_addr) {
11487 ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
11488 } else if (type == BP_VAR_IS && result_type_guard) {
11489 ir_ref if_fit = ir_IF(cond);
11490 ir_IF_FALSE(if_fit);
11491 ir_END_list(*not_found_inputs);
11492 ir_IF_TRUE(if_fit);
11493 } else {
11494 ir_ref if_fit = ir_IF(cond);
11495 ir_IF_FALSE(if_fit);
11496 ir_END_list(idx_not_found_inputs);
11497 ir_IF_TRUE(if_fit);
11498 }
11499 // JIT: _ret = &_ht->arPacked[h];
11500 ref = ir_MUL_L(h, ir_CONST_LONG(sizeof(zval)));
11501 ref = ir_BITCAST_A(ref);
11502 ref = ir_ADD_A(ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arPacked))), ref);
11503 if (type == BP_JIT_IS) {
11504 ir_refs_add(test_zval_values, ref);
11505 ir_refs_add(test_zval_inputs, ir_END());
11506 }
11507 }
11508 }
11509 switch (type) {
11510 case BP_JIT_IS:
11511 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
11512 if (if_packed) {
11513 ir_IF_FALSE(if_packed);
11514 if_packed = IR_UNUSED;
11515 }
11516 if (!op2_loaded) {
11517 // JIT: hval = Z_LVAL_P(dim);
11518 h = jit_Z_LVAL(jit, op2_addr);
11519 }
11520 if (packed_loaded) {
11521 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(_zend_hash_index_find), ht_ref, h);
11522 } else {
11523 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_find), ht_ref, h);
11524 }
11525 if (not_found_exit_addr) {
11526 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11527 } else {
11528 if_found = ir_IF(ref);
11529 ir_IF_FALSE(if_found);
11530 ir_END_list(*end_inputs);
11531 ir_IF_TRUE(if_found);
11532 }
11533 ir_refs_add(test_zval_values, ref);
11534 ir_refs_add(test_zval_inputs, ir_END());
11535 } else if (!not_found_exit_addr && !packed_loaded) {
11536 ir_END_list(*end_inputs);
11537 }
11538 break;
11539 case BP_VAR_R:
11540 case BP_VAR_IS:
11541 case BP_VAR_UNSET:
11542 if (packed_loaded) {
11543 ir_ref type_ref = jit_Z_TYPE_ref(jit, ref);
11544
11545 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
11546 ir_ref if_def = ir_IF(type_ref);
11547 ir_IF_TRUE(if_def);
11548 ir_refs_add(found_inputs, ir_END());
11549 ir_refs_add(found_vals, ref);
11550 ir_IF_FALSE(if_def);
11551 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
11552 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
11553 } else if (type == BP_VAR_IS && not_found_exit_addr) {
11554 jit_SIDE_EXIT(jit, ir_CONST_ADDR(not_found_exit_addr));
11555 } else if (type == BP_VAR_IS && result_type_guard) {
11556 ir_END_list(*not_found_inputs);
11557 } else {
11558 ir_END_list(idx_not_found_inputs);
11559 }
11560 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
11561 /* perform IS_UNDEF check only after result type guard (during deoptimization) */
11562 if (!result_type_guard) {
11563 ir_GUARD(type_ref, ir_CONST_ADDR(exit_addr));
11564 }
11565 } else if (type == BP_VAR_IS && not_found_exit_addr) {
11566 ir_GUARD(type_ref, ir_CONST_ADDR(not_found_exit_addr));
11567 } else if (type == BP_VAR_IS && result_type_guard) {
11568 ir_ref if_def = ir_IF(type_ref);
11569 ir_IF_FALSE(if_def);
11570 ir_END_list(*not_found_inputs);
11571 ir_IF_TRUE(if_def);
11572 } else {
11573 ir_ref if_def = ir_IF(type_ref);
11574 ir_IF_FALSE(if_def);
11575 ir_END_list(idx_not_found_inputs);
11576 ir_IF_TRUE(if_def);
11577 }
11578 }
11579 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
11580 if (if_packed) {
11581 ir_IF_FALSE(if_packed);
11582 if_packed = IR_UNUSED;
11583 }
11584 if (!op2_loaded) {
11585 // JIT: hval = Z_LVAL_P(dim);
11586 h = jit_Z_LVAL(jit, op2_addr);
11587 }
11588 if (packed_loaded) {
11589 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(_zend_hash_index_find), ht_ref, h);
11590 } else {
11591 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_find), ht_ref, h);
11592 }
11593 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
11594 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
11595 } else if (type == BP_VAR_IS && not_found_exit_addr) {
11596 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11597 } else if (type == BP_VAR_IS && result_type_guard) {
11598 if_found = ir_IF(ref);
11599 ir_IF_FALSE(if_found);
11600 ir_END_list(*not_found_inputs);
11601 ir_IF_TRUE(if_found);
11602 } else {
11603 if_found = ir_IF(ref);
11604 ir_IF_FALSE(if_found);
11605 ir_END_list(idx_not_found_inputs);
11606 ir_IF_TRUE(if_found);
11607 }
11608 ir_refs_add(found_inputs, ir_END());
11609 ir_refs_add(found_vals, ref);
11610 } else if (packed_loaded) {
11611 ir_refs_add(found_inputs, ir_END());
11612 ir_refs_add(found_vals, ref);
11613 } else {
11614 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
11615 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
11616 } else if (type == BP_VAR_IS && not_found_exit_addr) {
11617 jit_SIDE_EXIT(jit, ir_CONST_ADDR(not_found_exit_addr));
11618 } else if (type == BP_VAR_IS && result_type_guard) {
11619 ir_END_list(*not_found_inputs);
11620 } else {
11621 ir_END_list(idx_not_found_inputs);
11622 }
11623 }
11624
11625 if (idx_not_found_inputs) {
11626 ir_MERGE_list(idx_not_found_inputs);
11627 switch (type) {
11628 case BP_VAR_R:
11629 ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
11630 // JIT: zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval);
11631 // JIT: retval = &EG(uninitialized_zval);
11632 jit_SET_EX_OPLINE(jit, opline);
11633 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_offset, IR_FASTCALL_FUNC));
11634 ir_END_list(*end_inputs);
11635 break;
11636 case BP_VAR_IS:
11637 case BP_VAR_UNSET:
11638 if (!not_found_exit_addr) {
11639 // JIT: retval = &EG(uninitialized_zval);
11640 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
11641 ir_END_list(*end_inputs);
11642 }
11643 break;
11644 default:
11645 ZEND_UNREACHABLE();
11646 }
11647 }
11648 break;
11649 case BP_VAR_RW:
11650 if (packed_loaded) {
11651 if (not_found_exit_addr) {
11652 ir_refs_add(found_inputs, ir_END());
11653 ir_refs_add(found_vals, ref);
11654 } else {
11655 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
11656 ir_IF_TRUE(if_def);
11657 ir_refs_add(found_inputs, ir_END());
11658 ir_refs_add(found_vals, ref);
11659 ir_IF_FALSE_cold(if_def);
11660 ir_END_list(idx_not_found_inputs);
11661 }
11662 }
11663 if (!packed_loaded ||
11664 !not_found_exit_addr ||
11665 (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
11666 if (if_packed) {
11667 ir_IF_FALSE(if_packed);
11668 if_packed = IR_UNUSED;
11669 ir_END_list(idx_not_found_inputs);
11670 } else if (!packed_loaded) {
11671 ir_END_list(idx_not_found_inputs);
11672 }
11673
11674 ir_MERGE_list(idx_not_found_inputs);
11675 if (!op2_loaded) {
11676 // JIT: hval = Z_LVAL_P(dim);
11677 h = jit_Z_LVAL(jit, op2_addr);
11678 }
11679 if (packed_loaded) {
11680 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_index_lookup_rw_no_packed),
11681 ht_ref, h);
11682 } else {
11683 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_index_lookup_rw), ht_ref, h);
11684 }
11685 if (not_found_exit_addr) {
11686 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11687 } else {
11688 if_found = ir_IF(ref);
11689 ir_IF_FALSE(if_found);
11690 ir_END_list(*end_inputs);
11691 ir_IF_TRUE(if_found);
11692 }
11693 ir_refs_add(found_inputs, ir_END());
11694 ir_refs_add(found_vals, ref);
11695 }
11696 break;
11697 case BP_VAR_W:
11698 if (packed_loaded) {
11699 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
11700 ir_IF_TRUE_cold(if_def);
11701 ir_refs_add(found_inputs, ir_END());
11702 ir_refs_add(found_vals, ref);
11703 ir_IF_FALSE(if_def);
11704 ir_END_list(idx_not_found_inputs);
11705 }
11706 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) {
11707 if (if_packed) {
11708 ir_IF_FALSE(if_packed);
11709 if_packed = IR_UNUSED;
11710 ir_END_list(idx_not_found_inputs);
11711 } else if (!packed_loaded) {
11712 ir_END_list(idx_not_found_inputs);
11713 }
11714 ir_MERGE_list(idx_not_found_inputs);
11715 if (!op2_loaded) {
11716 // JIT: hval = Z_LVAL_P(dim);
11717 h = jit_Z_LVAL(jit, op2_addr);
11718 }
11719 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_lookup), ht_ref, h);
11720 ir_refs_add(found_inputs, ir_END());
11721 ir_refs_add(found_vals, ref);
11722 }
11723 break;
11724 default:
11725 ZEND_UNREACHABLE();
11726 }
11727 }
11728
11729 if (op2_info & MAY_BE_STRING) {
11730 ir_ref key;
11731
11732 if (if_type) {
11733 ir_IF_FALSE(if_type);
11734 if_type = IS_UNUSED;
11735 }
11736
11737 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
11738 // JIT: if (EXPECTED(Z_TYPE_P(dim) == IS_STRING))
11739 if_type = jit_if_Z_TYPE(jit, op2_addr, IS_STRING);
11740 ir_IF_TRUE(if_type);
11741 }
11742
11743 // JIT: offset_key = Z_STR_P(dim);
11744 key = jit_Z_PTR(jit, op2_addr);
11745
11746 // JIT: retval = zend_hash_find(ht, offset_key);
11747 switch (type) {
11748 case BP_JIT_IS:
11749 if (opline->op2_type != IS_CONST) {
11750 ir_ref if_num, end1, ref2;
11751
11752 if_num = ir_IF(
11753 ir_ULE(
11754 ir_LOAD_C(ir_ADD_OFFSET(key, offsetof(zend_string, val))),
11755 ir_CONST_CHAR('9')));
11756 ir_IF_TRUE_cold(if_num);
11757 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_find), ht_ref, key);
11758 end1 = ir_END();
11759 ir_IF_FALSE(if_num);
11760 ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find), ht_ref, key);
11761 ir_MERGE_WITH(end1);
11762 ref = ir_PHI_2(IR_ADDR, ref2, ref);
11763 } else {
11764 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash), ht_ref, key);
11765 }
11766 if (not_found_exit_addr) {
11767 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11768 } else {
11769 if_found = ir_IF(ref);
11770 ir_IF_FALSE(if_found);
11771 ir_END_list(*end_inputs);
11772 ir_IF_TRUE(if_found);
11773 }
11774 ir_refs_add(test_zval_values, ref);
11775 ir_refs_add(test_zval_inputs, ir_END());
11776 break;
11777 case BP_VAR_R:
11778 case BP_VAR_IS:
11779 case BP_VAR_UNSET:
11780 if (opline->op2_type != IS_CONST) {
11781 ir_ref if_num, end1, ref2;
11782
11783 if_num = ir_IF(
11784 ir_ULE(
11785 ir_LOAD_C(ir_ADD_OFFSET(key, offsetof(zend_string, val))),
11786 ir_CONST_CHAR('9')));
11787 ir_IF_TRUE_cold(if_num);
11788 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_find), ht_ref, key);
11789 end1 = ir_END();
11790 ir_IF_FALSE(if_num);
11791 ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find), ht_ref, key);
11792 ir_MERGE_WITH(end1);
11793 ref = ir_PHI_2(IR_ADDR, ref2, ref);
11794 } else {
11795 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash), ht_ref, key);
11796 }
11797 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
11798 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
11799 } else if (type == BP_VAR_IS && not_found_exit_addr) {
11800 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11801 } else if (type == BP_VAR_IS && result_type_guard) {
11802 if_found = ir_IF(ref);
11803 ir_IF_FALSE(if_found);
11804 ir_END_list(*not_found_inputs);
11805 ir_IF_TRUE(if_found);
11806 } else {
11807 if_found = ir_IF(ref);
11808 switch (type) {
11809 case BP_VAR_R:
11810 ir_IF_FALSE_cold(if_found);
11811 // JIT: zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key));
11812 jit_SET_EX_OPLINE(jit, opline);
11813 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_key, IR_FASTCALL_FUNC));
11814 ir_END_list(*end_inputs);
11815 break;
11816 case BP_VAR_IS:
11817 case BP_VAR_UNSET:
11818 ir_IF_FALSE(if_found);
11819 // JIT: retval = &EG(uninitialized_zval);
11820 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
11821 ir_END_list(*end_inputs);
11822 break;
11823 default:
11824 ZEND_UNREACHABLE();
11825 }
11826 ir_IF_TRUE(if_found);
11827 }
11828 ir_refs_add(found_inputs, ir_END());
11829 ir_refs_add(found_vals, ref);
11830 break;
11831 case BP_VAR_RW:
11832 if (opline->op2_type != IS_CONST) {
11833 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_lookup_rw), ht_ref, key);
11834 } else {
11835 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_lookup_rw), ht_ref, key);
11836 }
11837 if (not_found_exit_addr) {
11838 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11839 } else {
11840 if_found = ir_IF(ref);
11841 ir_IF_FALSE(if_found);
11842 ir_END_list(*end_inputs);
11843 ir_IF_TRUE(if_found);
11844 }
11845 ir_refs_add(found_inputs, ir_END());
11846 ir_refs_add(found_vals, ref);
11847 break;
11848 case BP_VAR_W:
11849 if (opline->op2_type != IS_CONST) {
11850 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_lookup_w), ht_ref, key);
11851 } else {
11852 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_lookup), ht_ref, key);
11853 }
11854 ir_refs_add(found_inputs, ir_END());
11855 ir_refs_add(found_vals, ref);
11856 break;
11857 default:
11858 ZEND_UNREACHABLE();
11859 }
11860 }
11861
11862 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
11863 if (if_type) {
11864 ir_IF_FALSE_cold(if_type);
11865 if_type = IS_UNDEF;
11866 }
11867 if (type != BP_VAR_RW) {
11868 jit_SET_EX_OPLINE(jit, opline);
11869 }
11870 ref = jit_ZVAL_ADDR(jit, op2_addr);
11871 switch (type) {
11872 case BP_VAR_R:
11873 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_r_helper),
11874 ht_ref,
11875 ref,
11876 jit_ZVAL_ADDR(jit, res_addr));
11877 ir_END_list(*end_inputs);
11878 break;
11879 case BP_JIT_IS:
11880 ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_fetch_dim_isset_helper), ht_ref, ref);
11881 if (not_found_exit_addr) {
11882 ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
11883 ir_refs_add(found_inputs, ir_END());
11884 } else if (found_exit_addr) {
11885 ir_GUARD_NOT(ref, ir_CONST_ADDR(found_exit_addr));
11886 ir_END_list(*end_inputs);
11887 } else {
11888 if_found = ir_IF(ref);
11889 ir_IF_TRUE(if_found);
11890 ir_refs_add(found_inputs, ir_END());
11891 ir_IF_FALSE(if_found);
11892 ir_END_list(*end_inputs);
11893 }
11894 break;
11895 case BP_VAR_IS:
11896 case BP_VAR_UNSET:
11897 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_is_helper),
11898 ht_ref,
11899 ref,
11900 jit_ZVAL_ADDR(jit, res_addr));
11901 ir_END_list(*end_inputs);
11902 break;
11903 case BP_VAR_RW:
11904 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_rw_helper), ht_ref, ref);
11905 if_found = ir_IF(ref);
11906 ir_IF_TRUE(if_found);
11907 ir_refs_add(found_inputs, ir_END());
11908 ir_refs_add(found_vals, ref);
11909 ir_IF_FALSE(if_found);
11910 ir_END_list(*end_inputs);
11911 break;
11912 case BP_VAR_W:
11913 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_w_helper), ht_ref, ref);
11914 if_found = ir_IF(ref);
11915 ir_IF_TRUE(if_found);
11916 ir_refs_add(found_inputs, ir_END());
11917 ir_refs_add(found_vals, ref);
11918 ir_IF_FALSE(if_found);
11919 ir_END_list(*end_inputs);
11920 break;
11921 default:
11922 ZEND_UNREACHABLE();
11923 }
11924 }
11925
11926 if (type == BP_JIT_IS
11927 && !(op2_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
11928 /* dead code */
11929 ir_END_list(*end_inputs);
11930 } else if (type == BP_JIT_IS
11931 && (op1_info & MAY_BE_ARRAY)
11932 && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))
11933 && test_zval_inputs->count) {
11934
11935 ir_MERGE_N(test_zval_inputs->count, test_zval_inputs->refs);
11936 ref = ir_PHI_N(IR_ADDR, test_zval_values->count, test_zval_values->refs);
11937
11938 if (op1_info & MAY_BE_ARRAY_OF_REF) {
11939 ref = jit_ZVAL_DEREF_ref(jit, ref);
11940 }
11941 cond = ir_GT(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_NULL));
11942 if (not_found_exit_addr) {
11943 ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
11944 ir_refs_add(found_inputs, ir_END());
11945 } else if (found_exit_addr) {
11946 ir_GUARD_NOT(cond, ir_CONST_ADDR(found_exit_addr));
11947 ir_END_list(*end_inputs);
11948 } else {
11949 ir_ref if_set = ir_IF(cond);
11950 ir_IF_FALSE(if_set);
11951 ir_END_list(*end_inputs);
11952 ir_IF_TRUE(if_set);
11953 ir_refs_add(found_inputs, ir_END());
11954 }
11955 }
11956
11957 return 1;
11958 }
11959
11960 static int zend_jit_fetch_dim_read(zend_jit_ctx *jit,
11961 const zend_op *opline,
11962 zend_ssa *ssa,
11963 const zend_ssa_op *ssa_op,
11964 uint32_t op1_info,
11965 zend_jit_addr op1_addr,
11966 bool op1_avoid_refcounting,
11967 uint32_t op2_info,
11968 uint32_t res_info,
11969 zend_jit_addr res_addr,
11970 uint8_t dim_type)
11971 {
11972 zend_jit_addr orig_op1_addr, op2_addr;
11973 const void *exit_addr = NULL;
11974 const void *not_found_exit_addr = NULL;
11975 bool result_type_guard = 0;
11976 bool result_avoid_refcounting = 0;
11977 uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0;
11978 int may_throw = 0;
11979 ir_ref if_type = IR_UNUSED;
11980 ir_ref end_inputs = IR_UNUSED;
11981 ir_ref not_found_inputs = IR_UNUSED;
11982
11983 orig_op1_addr = OP1_ADDR();
11984 op2_addr = OP2_ADDR();
11985
11986 if (opline->opcode != ZEND_FETCH_DIM_IS
11987 && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
11988 && !has_concrete_type(op1_info)) {
11989 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
11990 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
11991 if (!exit_addr) {
11992 return 0;
11993 }
11994 }
11995
11996 if ((res_info & MAY_BE_GUARD)
11997 && JIT_G(current_frame)
11998 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
11999
12000 if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) {
12001 result_type_guard = 1;
12002 res_info &= ~MAY_BE_GUARD;
12003 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
12004 }
12005
12006 if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
12007 && (opline->opcode == ZEND_FETCH_LIST_R
12008 || !(opline->op1_type & (IS_VAR|IS_TMP_VAR))
12009 || op1_avoid_refcounting)
12010 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
12011 && (ssa_op+1)->op1_use == ssa_op->result_def
12012 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))
12013 && zend_jit_may_avoid_refcounting(opline+1, res_info)) {
12014 result_avoid_refcounting = 1;
12015 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
12016 }
12017
12018 if (opline->opcode == ZEND_FETCH_DIM_IS
12019 && !(res_info & MAY_BE_NULL)) {
12020 uint32_t flags = 0;
12021 uint32_t old_op1_info = 0;
12022 uint32_t old_info;
12023 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
12024 int32_t exit_point;
12025
12026 if (opline->opcode != ZEND_FETCH_LIST_R
12027 && (opline->op1_type & (IS_VAR|IS_TMP_VAR))
12028 && !op1_avoid_refcounting) {
12029 flags |= ZEND_JIT_EXIT_FREE_OP1;
12030 }
12031 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
12032 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
12033 flags |= ZEND_JIT_EXIT_FREE_OP2;
12034 }
12035
12036 if (op1_avoid_refcounting) {
12037 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
12038 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
12039 }
12040
12041 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
12042 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0);
12043 SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_TYPE_ONLY);
12044 exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
12045 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
12046 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
12047 if (!not_found_exit_addr) {
12048 return 0;
12049 }
12050
12051 if (op1_avoid_refcounting) {
12052 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
12053 }
12054 }
12055 }
12056
12057 if (op1_info & MAY_BE_REF) {
12058 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
12059 ref = jit_ZVAL_DEREF_ref(jit, ref);
12060 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
12061 }
12062
12063 if (op1_info & MAY_BE_ARRAY) {
12064 ir_ref ht_ref, ref;
12065 zend_jit_addr val_addr;
12066 ir_refs *found_inputs, *found_vals;
12067
12068 ir_refs_init(found_inputs, 10);
12069 ir_refs_init(found_vals, 10);
12070
12071 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
12072 if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) {
12073 jit_guard_Z_TYPE(jit, op1_addr, IS_ARRAY, exit_addr);
12074 } else {
12075 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
12076 ir_IF_TRUE(if_type);
12077 }
12078 }
12079
12080 ht_ref = jit_Z_PTR(jit, op1_addr);
12081
12082 if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) ||
12083 (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) {
12084 may_throw = 1;
12085 }
12086
12087 if (!zend_jit_fetch_dimension_address_inner(jit, opline,
12088 (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS,
12089 op1_info, op2_info, dim_type, NULL, not_found_exit_addr, exit_addr,
12090 result_type_guard, ht_ref, found_inputs, found_vals,
12091 &end_inputs, ¬_found_inputs)) {
12092 return 0;
12093 }
12094
12095 if (found_inputs->count) {
12096 ir_MERGE_N(found_inputs->count, found_inputs->refs);
12097 ref = ir_PHI_N(IR_ADDR, found_vals->count, found_vals->refs);
12098 val_addr = ZEND_ADDR_REF_ZVAL(ref);
12099
12100 if (result_type_guard) {
12101 uint8_t type = concrete_type(res_info);
12102 uint32_t flags = 0;
12103
12104 if (opline->opcode != ZEND_FETCH_LIST_R
12105 && (opline->op1_type & (IS_VAR|IS_TMP_VAR))
12106 && !op1_avoid_refcounting) {
12107 flags |= ZEND_JIT_EXIT_FREE_OP1;
12108 }
12109 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
12110 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
12111 flags |= ZEND_JIT_EXIT_FREE_OP2;
12112 }
12113
12114 val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
12115 (op1_info & MAY_BE_ARRAY_OF_REF) != 0, flags, op1_avoid_refcounting);
12116 if (!val_addr) {
12117 return 0;
12118 }
12119
12120 if (not_found_inputs) {
12121 ir_END_list(not_found_inputs);
12122 ir_MERGE_list(not_found_inputs);
12123 }
12124
12125 // ZVAL_COPY
12126 jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
12127 if (Z_MODE(res_addr) != IS_REG) {
12128 } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
12129 return 0;
12130 }
12131 } else if (op1_info & MAY_BE_ARRAY_OF_REF) {
12132 // ZVAL_COPY_DEREF
12133 ir_ref type_info = jit_Z_TYPE_INFO(jit, val_addr);
12134 if (!zend_jit_zval_copy_deref(jit, res_addr, val_addr, type_info)) {
12135 return 0;
12136 }
12137 } else {
12138 // ZVAL_COPY
12139 jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, 1);
12140 }
12141
12142 ir_END_list(end_inputs);
12143 } else if (not_found_inputs) {
12144 ir_MERGE_list(not_found_inputs);
12145 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12146 ir_END_list(end_inputs);
12147 } else if (!end_inputs && jit->ctx.control) {
12148 ir_END_list(end_inputs); /* dead code */
12149 }
12150 }
12151
12152 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
12153 if (if_type) {
12154 ir_IF_FALSE_cold(if_type);
12155 if_type = IS_UNDEF;
12156 }
12157
12158 if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) {
12159 ir_ref str_ref;
12160
12161 may_throw = 1;
12162 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) {
12163 if (exit_addr && !(op1_info & MAY_BE_OBJECT)) {
12164 jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, exit_addr);
12165 } else {
12166 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
12167 ir_IF_TRUE(if_type);
12168 }
12169 }
12170 jit_SET_EX_OPLINE(jit, opline);
12171 str_ref = jit_Z_PTR(jit, op1_addr);
12172 if (opline->opcode != ZEND_FETCH_DIM_IS) {
12173 ir_ref ref;
12174
12175 if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) {
12176 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_offset_r_helper),
12177 str_ref, jit_Z_LVAL(jit, op2_addr));
12178 } else {
12179 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_r_helper),
12180 str_ref, jit_ZVAL_ADDR(jit, op2_addr));
12181 }
12182 jit_set_Z_PTR(jit, res_addr, ref);
12183 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
12184 } else {
12185 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_is_helper),
12186 str_ref,
12187 jit_ZVAL_ADDR(jit, op2_addr),
12188 jit_ZVAL_ADDR(jit, res_addr));
12189 }
12190 ir_END_list(end_inputs);
12191 }
12192
12193 if (op1_info & MAY_BE_OBJECT) {
12194 ir_ref arg2;
12195
12196 if (if_type) {
12197 ir_IF_FALSE_cold(if_type);
12198 if_type = IS_UNDEF;
12199 }
12200
12201 may_throw = 1;
12202 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) {
12203 if (exit_addr) {
12204 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
12205 } else {
12206 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
12207 ir_IF_TRUE(if_type);
12208 }
12209 }
12210
12211 jit_SET_EX_OPLINE(jit, opline);
12212 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
12213 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
12214 arg2 = ir_CONST_ADDR(Z_ZV(op2_addr)+1);
12215 } else {
12216 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
12217 }
12218
12219 if (opline->opcode != ZEND_FETCH_DIM_IS) {
12220 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_r_helper),
12221 jit_ZVAL_ADDR(jit, op1_addr),
12222 arg2,
12223 jit_ZVAL_ADDR(jit, res_addr));
12224 } else {
12225 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_is_helper),
12226 jit_ZVAL_ADDR(jit, op1_addr),
12227 arg2,
12228 jit_ZVAL_ADDR(jit, res_addr));
12229 }
12230
12231 ir_END_list(end_inputs);
12232 }
12233
12234 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))
12235 && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) {
12236
12237 if (if_type) {
12238 ir_IF_FALSE_cold(if_type);
12239 if_type = IS_UNDEF;
12240 }
12241
12242 if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) {
12243 jit_SET_EX_OPLINE(jit, opline);
12244 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) {
12245 may_throw = 1;
12246 zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op1_addr), opline->op1.var, NULL, 0, 1, 0);
12247 }
12248
12249 if (op2_info & MAY_BE_UNDEF) {
12250 may_throw = 1;
12251 zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op2_addr), opline->op2.var, NULL, 0, 1, 0);
12252 }
12253 }
12254
12255 if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) {
12256 ir_ref ref;
12257
12258 may_throw = 1;
12259 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
12260 ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
12261 } else {
12262 jit_SET_EX_OPLINE(jit, opline);
12263 ref = jit_ZVAL_ADDR(jit, op1_addr);
12264 }
12265 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_array_access), ref);
12266 }
12267
12268 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12269 ir_END_list(end_inputs);
12270 }
12271 }
12272
12273 if (end_inputs) {
12274 ir_MERGE_list(end_inputs);
12275
12276 #ifdef ZEND_JIT_USE_RC_INFERENCE
12277 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
12278 /* Magic offsetGet() may increase refcount of the key */
12279 op2_info |= MAY_BE_RCN;
12280 }
12281 #endif
12282
12283 if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) {
12284 if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) {
12285 may_throw = 1;
12286 }
12287 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
12288 }
12289 if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) {
12290 if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) {
12291 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
12292 may_throw = 1;
12293 }
12294 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
12295 }
12296 }
12297
12298 if (may_throw) {
12299 zend_jit_check_exception(jit);
12300 }
12301 } else if (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) {
12302 ir_BEGIN(IR_UNUSED); /* unreachable tail */
12303 }
12304
12305 return 1;
12306 }
12307
12308 static zend_jit_addr zend_jit_prepare_array_update(zend_jit_ctx *jit,
12309 const zend_op *opline,
12310 uint32_t op1_info,
12311 zend_jit_addr op1_addr,
12312 ir_ref *if_type,
12313 ir_ref *ht_ref,
12314 int *may_throw)
12315 {
12316 ir_ref ref = IR_UNUSED;
12317 ir_ref array_reference_end = IR_UNUSED, array_reference_ref = IR_UNUSED;
12318 ir_refs *array_inputs, *array_values;
12319
12320 ir_refs_init(array_inputs, 4);
12321 ir_refs_init(array_values, 4);
12322
12323 ref = jit_ZVAL_ADDR(jit, op1_addr);
12324 if (op1_info & MAY_BE_REF) {
12325 ir_ref if_reference, if_array, end1, ref2;
12326
12327 *may_throw = 1;
12328 if_reference = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
12329 ir_IF_FALSE(if_reference);
12330 end1 = ir_END();
12331 ir_IF_TRUE_cold(if_reference);
12332 array_reference_ref = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val));
12333 if_array = jit_if_Z_TYPE_ref(jit, array_reference_ref, ir_CONST_U8(IS_ARRAY));
12334 ir_IF_TRUE(if_array);
12335 array_reference_end = ir_END();
12336 ir_IF_FALSE_cold(if_array);
12337 if (opline->opcode != ZEND_FETCH_DIM_RW && opline->opcode != ZEND_ASSIGN_DIM_OP) {
12338 jit_SET_EX_OPLINE(jit, opline);
12339 }
12340 ref2 = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_prepare_assign_dim_ref), ref);
12341 ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
12342
12343 ir_MERGE_WITH(end1);
12344 ref = ir_PHI_2(IR_ADDR, ref2, ref);
12345 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
12346 }
12347
12348 if (op1_info & MAY_BE_ARRAY) {
12349 ir_ref op1_ref = ref;
12350
12351 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
12352 *if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
12353 ir_IF_TRUE(*if_type);
12354 }
12355 if (array_reference_end) {
12356 ir_MERGE_WITH(array_reference_end);
12357 op1_ref = ir_PHI_2(IR_ADDR, ref, array_reference_ref);
12358 }
12359 // JIT: SEPARATE_ARRAY()
12360 ref = jit_Z_PTR_ref(jit, op1_ref);
12361 if (RC_MAY_BE_N(op1_info)) {
12362 if (RC_MAY_BE_1(op1_info)) {
12363 ir_ref if_refcount_1 = ir_IF(ir_EQ(jit_GC_REFCOUNT(jit, ref), ir_CONST_U32(1)));
12364 ir_IF_TRUE(if_refcount_1);
12365 ir_refs_add(array_inputs, ir_END());
12366 ir_refs_add(array_values, ref);
12367 ir_IF_FALSE(if_refcount_1);
12368 }
12369 ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_zval_array_dup), op1_ref);
12370 }
12371 if (array_inputs->count || (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
12372 ir_refs_add(array_inputs, ir_END());
12373 ir_refs_add(array_values, ref);
12374 }
12375 }
12376
12377 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
12378 if (*if_type) {
12379 ir_IF_FALSE_cold(*if_type);
12380 *if_type = IR_UNUSED;
12381 }
12382 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
12383 *if_type = ir_IF(ir_LE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL)));
12384 ir_IF_TRUE(*if_type);
12385 }
12386 if ((op1_info & MAY_BE_UNDEF)
12387 && (opline->opcode == ZEND_FETCH_DIM_RW || opline->opcode == ZEND_ASSIGN_DIM_OP)) {
12388 ir_ref end1 = IR_UNUSED;
12389
12390 *may_throw = 1;
12391 if (op1_info & MAY_BE_NULL) {
12392 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
12393 ir_IF_TRUE(if_def);
12394 end1 = ir_END();
12395 ir_IF_FALSE(if_def);
12396 }
12397 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
12398 if (end1) {
12399 ir_MERGE_WITH(end1);
12400 }
12401 }
12402 // JIT: ZVAL_ARR(container, zend_new_array(8));
12403 ref = ir_CALL_1(IR_ADDR,
12404 jit_STUB_FUNC_ADDR(jit, jit_stub_new_array, IR_FASTCALL_FUNC),
12405 jit_ZVAL_ADDR(jit, op1_addr));
12406 if (array_inputs->count) {
12407 ir_refs_add(array_inputs, ir_END());
12408 ir_refs_add(array_values, ref);
12409 }
12410 }
12411
12412 if (array_inputs->count) {
12413 ir_MERGE_N(array_inputs->count, array_inputs->refs);
12414 ref = ir_PHI_N(IR_ADDR, array_values->count, array_values->refs);
12415 }
12416
12417 *ht_ref = ref;
12418 return op1_addr;
12419 }
12420
12421 static int zend_jit_fetch_dim(zend_jit_ctx *jit,
12422 const zend_op *opline,
12423 uint32_t op1_info,
12424 zend_jit_addr op1_addr,
12425 uint32_t op2_info,
12426 zend_jit_addr res_addr,
12427 uint8_t dim_type)
12428 {
12429 zend_jit_addr op2_addr;
12430 int may_throw = 0;
12431 ir_ref end_inputs = IR_UNUSED;
12432 ir_ref ref, if_type = IR_UNUSED, ht_ref;
12433
12434 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
12435
12436 if (opline->opcode == ZEND_FETCH_DIM_RW) {
12437 jit_SET_EX_OPLINE(jit, opline);
12438 }
12439
12440 op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
12441
12442 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
12443 ir_refs *found_inputs, *found_vals;
12444
12445 ir_refs_init(found_inputs, 8);
12446 ir_refs_init(found_vals, 8);
12447
12448 if (opline->op2_type == IS_UNUSED) {
12449 ir_ref if_ok;
12450
12451 may_throw = 1;
12452 // JIT:var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
12453 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
12454 ht_ref, jit_EG(uninitialized_zval));
12455
12456 // JIT: if (UNEXPECTED(!var_ptr)) {
12457 if_ok = ir_IF(ref);
12458 ir_IF_FALSE_cold(if_ok);
12459 if (opline->opcode != ZEND_FETCH_DIM_RW) {
12460 jit_SET_EX_OPLINE(jit, opline);
12461 }
12462 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
12463 ir_END_list(end_inputs);
12464
12465 ir_IF_TRUE(if_ok);
12466 jit_set_Z_PTR(jit, res_addr, ref);
12467 jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
12468
12469 ir_END_list(end_inputs);
12470 } else {
12471 uint32_t type;
12472
12473 switch (opline->opcode) {
12474 case ZEND_FETCH_DIM_W:
12475 case ZEND_FETCH_LIST_W:
12476 type = BP_VAR_W;
12477 break;
12478 case ZEND_FETCH_DIM_RW:
12479 may_throw = 1;
12480 type = BP_VAR_RW;
12481 break;
12482 case ZEND_FETCH_DIM_UNSET:
12483 type = BP_VAR_UNSET;
12484 break;
12485 default:
12486 ZEND_UNREACHABLE();
12487 }
12488
12489 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
12490 may_throw = 1;
12491 }
12492 if (!zend_jit_fetch_dimension_address_inner(jit, opline, type, op1_info, op2_info, dim_type, NULL, NULL, NULL,
12493 0, ht_ref, found_inputs, found_vals, &end_inputs, NULL)) {
12494 return 0;
12495 }
12496
12497 if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
12498 if (end_inputs) {
12499 ir_MERGE_list(end_inputs);
12500 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12501 end_inputs = ir_END();
12502 }
12503 } else if (!(op2_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
12504 /* impossible dead path */
12505 end_inputs = ir_END();
12506 } else {
12507 ZEND_ASSERT(end_inputs == IR_UNUSED);
12508 }
12509
12510 if (found_inputs->count) {
12511 ir_MERGE_N(found_inputs->count, found_inputs->refs);
12512 ref = ir_PHI_N(IR_ADDR, found_vals->count, found_vals->refs);
12513 jit_set_Z_PTR(jit, res_addr, ref);
12514 jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
12515 ir_END_list(end_inputs);
12516 }
12517
12518 }
12519 }
12520
12521 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
12522 ir_ref arg2;
12523
12524 may_throw = 1;
12525
12526 if (if_type) {
12527 ir_IF_FALSE(if_type);
12528 if_type = IR_UNUSED;
12529 }
12530
12531 if (opline->opcode != ZEND_FETCH_DIM_RW) {
12532 jit_SET_EX_OPLINE(jit, opline);
12533 }
12534
12535 if (opline->op2_type == IS_UNUSED) {
12536 arg2 = IR_NULL;
12537 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
12538 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
12539 arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
12540 } else {
12541 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
12542 }
12543
12544 switch (opline->opcode) {
12545 case ZEND_FETCH_DIM_W:
12546 case ZEND_FETCH_LIST_W:
12547 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_w_helper),
12548 jit_ZVAL_ADDR(jit, op1_addr),
12549 arg2,
12550 jit_ZVAL_ADDR(jit, res_addr));
12551 break;
12552 case ZEND_FETCH_DIM_RW:
12553 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_rw_helper),
12554 jit_ZVAL_ADDR(jit, op1_addr),
12555 arg2,
12556 jit_ZVAL_ADDR(jit, res_addr));
12557 break;
12558 // case ZEND_FETCH_DIM_UNSET:
12559 // | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0
12560 // break;
12561 default:
12562 ZEND_UNREACHABLE();
12563 }
12564
12565 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
12566 ir_END_list(end_inputs);
12567 }
12568 }
12569
12570 #ifdef ZEND_JIT_USE_RC_INFERENCE
12571 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
12572 /* ASSIGN_DIM may increase refcount of the key */
12573 op2_info |= MAY_BE_RCN;
12574 }
12575 #endif
12576
12577 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
12578 && (op2_info & MAY_HAVE_DTOR)
12579 && (op2_info & MAY_BE_RC1)) {
12580 may_throw = 1;
12581 }
12582
12583 if (end_inputs) {
12584 ir_MERGE_list(end_inputs);
12585 }
12586
12587 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
12588
12589 if (may_throw) {
12590 zend_jit_check_exception(jit);
12591 }
12592
12593 return 1;
12594 }
12595
12596 static int zend_jit_isset_isempty_dim(zend_jit_ctx *jit,
12597 const zend_op *opline,
12598 uint32_t op1_info,
12599 zend_jit_addr op1_addr,
12600 bool op1_avoid_refcounting,
12601 uint32_t op2_info,
12602 uint8_t dim_type,
12603 int may_throw,
12604 uint8_t smart_branch_opcode,
12605 uint32_t target_label,
12606 uint32_t target_label2,
12607 const void *exit_addr)
12608 {
12609 zend_jit_addr op2_addr, res_addr;
12610 ir_ref if_type = IR_UNUSED;
12611 ir_ref false_inputs = IR_UNUSED, end_inputs = IR_UNUSED;
12612 ir_refs *true_inputs;
12613
12614 ir_refs_init(true_inputs, 8);
12615
12616 // TODO: support for empty() ???
12617 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY));
12618
12619 op2_addr = OP2_ADDR();
12620 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
12621
12622 if (op1_info & MAY_BE_REF) {
12623 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
12624 ref = jit_ZVAL_DEREF_ref(jit, ref);
12625 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
12626 }
12627
12628 if (op1_info & MAY_BE_ARRAY) {
12629 const void *found_exit_addr = NULL;
12630 const void *not_found_exit_addr = NULL;
12631 ir_ref ht_ref;
12632
12633 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
12634 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
12635 ir_IF_TRUE(if_type);
12636 }
12637
12638 ht_ref = jit_Z_PTR(jit, op1_addr);
12639
12640 if (exit_addr
12641 && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY))
12642 && !may_throw
12643 && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting)
12644 && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
12645 if (smart_branch_opcode == ZEND_JMPNZ) {
12646 found_exit_addr = exit_addr;
12647 } else {
12648 not_found_exit_addr = exit_addr;
12649 }
12650 }
12651 if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_JIT_IS, op1_info, op2_info, dim_type, found_exit_addr, not_found_exit_addr, NULL,
12652 0, ht_ref, true_inputs, NULL, &false_inputs, NULL)) {
12653 return 0;
12654 }
12655
12656 if (found_exit_addr) {
12657 ir_MERGE_list(false_inputs);
12658 return 1;
12659 } else if (not_found_exit_addr) {
12660 ir_MERGE_N(true_inputs->count, true_inputs->refs);
12661 return 1;
12662 }
12663 }
12664
12665 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
12666 if (if_type) {
12667 ir_IF_FALSE(if_type);
12668 if_type = IR_UNUSED;
12669 }
12670
12671 if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) {
12672 ir_ref ref, arg1, arg2, if_true;
12673
12674 jit_SET_EX_OPLINE(jit, opline);
12675 arg1 = jit_ZVAL_ADDR(jit, op1_addr);
12676 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
12677 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
12678 arg2 = ir_CONST_ADDR(Z_ZV(op2_addr)+1);
12679 } else {
12680 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
12681 }
12682 ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_isset_dim_helper), arg1, arg2);
12683 if_true = ir_IF(ref);
12684 ir_IF_TRUE(if_true);
12685 ir_refs_add(true_inputs, ir_END());
12686 ir_IF_FALSE(if_true);
12687 ir_END_list(false_inputs);
12688 } else {
12689 if (op2_info & MAY_BE_UNDEF) {
12690 ir_ref end1 = IR_UNUSED;
12691
12692 if (op2_info & MAY_BE_ANY) {
12693 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op2_addr));
12694 ir_IF_TRUE(if_def);
12695 end1 = ir_END();
12696 ir_IF_FALSE(if_def);
12697 }
12698 jit_SET_EX_OPLINE(jit, opline);
12699 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op2.var));
12700 if (end1) {
12701 ir_MERGE_WITH(end1);
12702 }
12703 }
12704 ir_END_list(false_inputs);
12705 }
12706 }
12707
12708 #ifdef ZEND_JIT_USE_RC_INFERENCE
12709 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
12710 /* Magic offsetExists() may increase refcount of the key */
12711 op2_info |= MAY_BE_RCN;
12712 }
12713 #endif
12714
12715 if (true_inputs->count) {
12716 ir_MERGE_N(true_inputs->count, true_inputs->refs);
12717
12718 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
12719 if (!op1_avoid_refcounting) {
12720 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
12721 }
12722 if (may_throw) {
12723 zend_jit_check_exception_undef_result(jit, opline);
12724 }
12725 if (!(opline->extended_value & ZEND_ISEMPTY)) {
12726 if (exit_addr) {
12727 if (smart_branch_opcode == ZEND_JMPNZ) {
12728 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
12729 } else {
12730 ir_END_list(end_inputs);
12731 }
12732 } else if (smart_branch_opcode) {
12733 if (smart_branch_opcode == ZEND_JMPZ) {
12734 _zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
12735 } else if (smart_branch_opcode == ZEND_JMPNZ) {
12736 _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
12737 } else {
12738 ZEND_UNREACHABLE();
12739 }
12740 } else {
12741 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
12742 ir_END_list(end_inputs);
12743 }
12744 } else {
12745 ZEND_UNREACHABLE(); // TODO: support for empty()
12746 }
12747 }
12748
12749 ir_MERGE_list(false_inputs);
12750 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
12751 if (!op1_avoid_refcounting) {
12752 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
12753 }
12754 if (may_throw) {
12755 zend_jit_check_exception_undef_result(jit, opline);
12756 }
12757 if (!(opline->extended_value & ZEND_ISEMPTY)) {
12758 if (exit_addr) {
12759 if (smart_branch_opcode == ZEND_JMPZ) {
12760 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
12761 } else {
12762 ir_END_list(end_inputs);
12763 }
12764 } else if (smart_branch_opcode) {
12765 if (smart_branch_opcode == ZEND_JMPZ) {
12766 _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
12767 } else if (smart_branch_opcode == ZEND_JMPNZ) {
12768 _zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
12769 } else {
12770 ZEND_UNREACHABLE();
12771 }
12772 } else {
12773 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
12774 ir_END_list(end_inputs);
12775 }
12776 } else {
12777 ZEND_UNREACHABLE(); // TODO: support for empty()
12778 }
12779
12780 if (!exit_addr && smart_branch_opcode) {
12781 jit->b = -1;
12782 } else {
12783 ir_MERGE_list(end_inputs);
12784 }
12785
12786 return 1;
12787 }
12788
12789 static int zend_jit_assign_dim(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw)
12790 {
12791 zend_jit_addr op2_addr, op3_addr, res_addr;
12792 ir_ref if_type = IR_UNUSED;
12793 ir_ref end_inputs = IR_UNUSED, ht_ref;
12794
12795 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
12796 op3_addr = OP1_DATA_ADDR();
12797 if (opline->result_type == IS_UNUSED) {
12798 res_addr = 0;
12799 } else {
12800 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
12801 }
12802
12803 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) {
12804 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
12805 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
12806
12807 if (!exit_addr) {
12808 return 0;
12809 }
12810
12811 jit_guard_not_Z_TYPE(jit, op3_addr, IS_UNDEF, exit_addr);
12812
12813 val_info &= ~MAY_BE_UNDEF;
12814 }
12815
12816 op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
12817
12818 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
12819 if (opline->op2_type == IS_UNUSED) {
12820 uint32_t var_info = MAY_BE_NULL;
12821 ir_ref if_ok, ref;
12822 zend_jit_addr var_addr;
12823
12824 // JIT: var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
12825 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
12826 ht_ref, jit_EG(uninitialized_zval));
12827
12828 // JIT: if (UNEXPECTED(!var_ptr)) {
12829 if_ok = ir_IF(ref);
12830 ir_IF_FALSE_cold(if_ok);
12831
12832 // JIT: zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
12833 jit_SET_EX_OPLINE(jit, opline);
12834 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
12835
12836 ir_END_list(end_inputs);
12837
12838 ir_IF_TRUE(if_ok);
12839 var_addr = ZEND_ADDR_REF_ZVAL(ref);
12840 if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
12841 return 0;
12842 }
12843 } else {
12844 uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
12845 zend_jit_addr var_addr;
12846 ir_ref ref;
12847 ir_refs *found_inputs, *found_values;
12848
12849 ir_refs_init(found_inputs, 8);
12850 ir_refs_init(found_values, 8);
12851
12852 if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL,
12853 0, ht_ref, found_inputs, found_values, &end_inputs, NULL)) {
12854 return 0;
12855 }
12856
12857 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
12858 var_info |= MAY_BE_REF;
12859 }
12860 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
12861 var_info |= MAY_BE_RC1;
12862 }
12863
12864 if (found_inputs->count) {
12865 ir_MERGE_N(found_inputs->count, found_inputs->refs);
12866 ref = ir_PHI_N(IR_ADDR, found_values->count, found_values->refs);
12867 var_addr = ZEND_ADDR_REF_ZVAL(ref);
12868
12869 // JIT: value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE);
12870 if (opline->op1_type == IS_VAR) {
12871 ZEND_ASSERT(opline->result_type == IS_UNUSED);
12872 if (!zend_jit_assign_to_variable_call(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
12873 return 0;
12874 }
12875 } else {
12876 if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) {
12877 return 0;
12878 }
12879 }
12880 }
12881 }
12882
12883 ir_END_list(end_inputs);
12884 }
12885
12886 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
12887 ir_ref arg2, arg4;
12888
12889 if (if_type) {
12890 ir_IF_FALSE_cold(if_type);
12891 if_type = IR_UNUSED;
12892 }
12893
12894 jit_SET_EX_OPLINE(jit, opline);
12895
12896 if (opline->op2_type == IS_UNUSED) {
12897 arg2 = IR_NULL;
12898 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
12899 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
12900 arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
12901 } else {
12902 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
12903 }
12904
12905 if (opline->result_type == IS_UNUSED) {
12906 arg4 = IR_NULL;
12907 } else {
12908 arg4 = jit_ZVAL_ADDR(jit, res_addr);
12909 }
12910 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_dim_helper),
12911 jit_ZVAL_ADDR(jit, op1_addr),
12912 arg2,
12913 jit_ZVAL_ADDR(jit, op3_addr),
12914 arg4);
12915
12916 #ifdef ZEND_JIT_USE_RC_INFERENCE
12917 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) {
12918 /* ASSIGN_DIM may increase refcount of the value */
12919 val_info |= MAY_BE_RCN;
12920 }
12921 #endif
12922
12923 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, NULL);
12924
12925 ir_END_list(end_inputs);
12926 }
12927
12928 #ifdef ZEND_JIT_USE_RC_INFERENCE
12929 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
12930 /* ASSIGN_DIM may increase refcount of the key */
12931 op2_info |= MAY_BE_RCN;
12932 }
12933 #endif
12934
12935 ir_MERGE_list(end_inputs);
12936 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
12937
12938 if (may_throw) {
12939 zend_jit_check_exception(jit);
12940 }
12941
12942 return 1;
12943 }
12944
12945 static int zend_jit_assign_dim_op(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw)
12946 {
12947 zend_jit_addr op2_addr, op3_addr, var_addr = IS_UNUSED;
12948 const void *not_found_exit_addr = NULL;
12949 uint32_t var_info = MAY_BE_NULL;
12950 ir_ref if_type = IS_UNUSED;
12951 ir_ref end_inputs = IR_UNUSED, ht_ref;
12952 bool emit_fast_path = 1;
12953
12954 ZEND_ASSERT(opline->result_type == IS_UNUSED);
12955
12956 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
12957 op3_addr = OP1_DATA_ADDR();
12958
12959 jit_SET_EX_OPLINE(jit, opline);
12960
12961 op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
12962
12963 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
12964 uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0);
12965
12966 if (opline->op2_type == IS_UNUSED) {
12967 var_info = MAY_BE_NULL;
12968 ir_ref if_ok, ref;
12969
12970 // JIT: var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
12971 ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
12972 ht_ref, jit_EG(uninitialized_zval));
12973
12974 // JIT: if (UNEXPECTED(!var_ptr)) {
12975 if_ok = ir_IF(ref);
12976 ir_IF_FALSE_cold(if_ok);
12977
12978 // JIT: zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
12979 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
12980
12981 ir_END_list(end_inputs);
12982
12983 ir_IF_TRUE(if_ok);
12984 var_addr = ZEND_ADDR_REF_ZVAL(ref);
12985 } else {
12986 ir_ref ref;
12987 ir_refs *found_inputs, *found_values;
12988
12989 ir_refs_init(found_inputs, 8);
12990 ir_refs_init(found_values, 8);
12991
12992 var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
12993 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
12994 var_info |= MAY_BE_REF;
12995 }
12996 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
12997 var_info |= MAY_BE_RC1;
12998 }
12999
13000 if (dim_type != IS_UNKNOWN
13001 && dim_type != IS_UNDEF
13002 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
13003 && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))
13004 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
13005 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
13006 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
13007 if (!not_found_exit_addr) {
13008 return 0;
13009 }
13010 }
13011
13012 if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, not_found_exit_addr, NULL,
13013 0, ht_ref, found_inputs, found_values, &end_inputs, NULL)) {
13014 return 0;
13015 }
13016
13017 if (found_inputs->count) {
13018 ir_MERGE_N(found_inputs->count, found_inputs->refs);
13019 ref = ir_PHI_N(IR_ADDR, found_values->count, found_values->refs);
13020 var_addr = ZEND_ADDR_REF_ZVAL(ref);
13021
13022 if (not_found_exit_addr && dim_type != IS_REFERENCE) {
13023 jit_guard_Z_TYPE(jit, var_addr, dim_type, not_found_exit_addr);
13024 var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
13025 }
13026 if (var_info & MAY_BE_REF) {
13027 binary_op_type binary_op = get_binary_op(opline->extended_value);
13028 ir_ref if_ref, if_typed, noref_path, ref_path, ref, reference, ref2, arg2;
13029
13030 ref = jit_ZVAL_ADDR(jit, var_addr);
13031 if_ref = jit_if_Z_TYPE(jit, var_addr, IS_REFERENCE);
13032 ir_IF_FALSE(if_ref);
13033 noref_path = ir_END();
13034 ir_IF_TRUE(if_ref);
13035
13036 reference = jit_Z_PTR_ref(jit, ref);
13037 ref2 = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
13038 if_typed = jit_if_TYPED_REF(jit, reference);
13039 ir_IF_FALSE(if_typed);
13040 ref_path = ir_END();
13041 ir_IF_TRUE_cold(if_typed);
13042
13043 arg2 = jit_ZVAL_ADDR(jit, op3_addr);
13044 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
13045 reference, arg2, ir_CONST_FC_FUNC(binary_op));
13046
13047 ir_END_list(end_inputs);
13048
13049 ir_MERGE_2(noref_path, ref_path);
13050 ref = ir_PHI_2(IR_ADDR, ref, ref2);
13051 var_addr = ZEND_ADDR_REF_ZVAL(ref);
13052 }
13053 } else {
13054 emit_fast_path = 0;
13055 }
13056 }
13057
13058 if (emit_fast_path) {
13059 uint8_t val_op_type = (opline+1)->op1_type;
13060
13061 if (val_op_type & (IS_TMP_VAR|IS_VAR)) {
13062 /* prevent FREE_OP in the helpers */
13063 val_op_type = IS_CV;
13064 }
13065
13066 switch (opline->extended_value) {
13067 case ZEND_ADD:
13068 case ZEND_SUB:
13069 case ZEND_MUL:
13070 case ZEND_DIV:
13071 if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info,
13072 1 /* may overflow */, may_throw)) {
13073 return 0;
13074 }
13075 break;
13076 case ZEND_BW_OR:
13077 case ZEND_BW_AND:
13078 case ZEND_BW_XOR:
13079 case ZEND_SL:
13080 case ZEND_SR:
13081 case ZEND_MOD:
13082 if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
13083 IS_CV, opline->op1, var_addr, var_info, NULL,
13084 val_op_type, (opline+1)->op1, op3_addr, op1_data_info,
13085 op1_data_range,
13086 0, var_addr, var_def_info, var_info, may_throw)) {
13087 return 0;
13088 }
13089 break;
13090 case ZEND_CONCAT:
13091 if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr,
13092 may_throw)) {
13093 return 0;
13094 }
13095 break;
13096 default:
13097 ZEND_UNREACHABLE();
13098 }
13099
13100 ir_END_list(end_inputs);
13101 }
13102 }
13103
13104 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
13105 binary_op_type binary_op;
13106 ir_ref arg2;
13107
13108 if (if_type) {
13109 ir_IF_FALSE_cold(if_type);
13110 if_type = IS_UNUSED;
13111 }
13112
13113 if (opline->op2_type == IS_UNUSED) {
13114 arg2 = IR_NULL;
13115 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
13116 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
13117 arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
13118 } else {
13119 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
13120 }
13121 binary_op = get_binary_op(opline->extended_value);
13122 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_dim_op_helper),
13123 jit_ZVAL_ADDR(jit, op1_addr),
13124 arg2,
13125 jit_ZVAL_ADDR(jit, op3_addr),
13126 ir_CONST_FC_FUNC(binary_op));
13127 ir_END_list(end_inputs);
13128 }
13129
13130 if (end_inputs) {
13131 ir_MERGE_list(end_inputs);
13132 }
13133
13134 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
13135 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
13136 if (may_throw) {
13137 zend_jit_check_exception(jit);
13138 }
13139
13140 return 1;
13141 }
13142
13143 static int zend_jit_fe_reset(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
13144 {
13145 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
13146
13147 // JIT: ZVAL_COPY(res, value);
13148 if (opline->op1_type == IS_CONST) {
13149 zval *zv = RT_CONSTANT(opline, opline->op1);
13150
13151 jit_ZVAL_COPY_CONST(jit, res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
13152 } else {
13153 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
13154
13155 jit_ZVAL_COPY(jit, res_addr, -1, op1_addr, op1_info, opline->op1_type == IS_CV);
13156 }
13157
13158 // JIT: Z_FE_POS_P(res) = 0;
13159 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->result.var + offsetof(zval, u2.fe_pos)), ir_CONST_U32(0));
13160
13161 return 1;
13162 }
13163
13164 static int zend_jit_packed_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint32_t op_info)
13165 {
13166 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
13167 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
13168 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
13169 ir_ref ref;
13170
13171 if (!exit_addr) {
13172 return 0;
13173 }
13174
13175 ref = ir_AND_U32(
13176 ir_LOAD_U32(ir_ADD_OFFSET(jit_Z_PTR(jit, addr), offsetof(zend_array, u.flags))),
13177 ir_CONST_U32(HASH_FLAG_PACKED));
13178 if (op_info & MAY_BE_ARRAY_PACKED) {
13179 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
13180 } else {
13181 ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
13182 }
13183
13184 return 1;
13185 }
13186
13187 static int zend_jit_fe_fetch(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, uint8_t exit_opcode, const void *exit_addr)
13188 {
13189 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
13190 ir_ref ref, ht_ref, hash_pos_ref, packed_pos_ref, hash_p_ref = IR_UNUSED, packed_p_ref = IR_UNUSED, if_packed = IR_UNUSED;
13191 ir_ref if_def_hash = IR_UNUSED, if_def_packed = IR_UNUSED;
13192 ir_ref exit_inputs = IR_UNUSED;
13193
13194 if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) {
13195 /* empty array */
13196 if (exit_addr) {
13197 if (exit_opcode == ZEND_JMP) {
13198 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
13199 }
13200 } else {
13201 zend_basic_block *bb;
13202
13203 ZEND_ASSERT(jit->b >= 0);
13204 bb = &jit->ssa->cfg.blocks[jit->b];
13205 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
13206 jit->b = -1;
13207 }
13208 return 1;
13209 }
13210
13211 // JIT: array = EX_VAR(opline->op1.var);
13212 // JIT: fe_ht = Z_ARRVAL_P(array);
13213 ht_ref = jit_Z_PTR(jit, op1_addr);
13214
13215 if (op1_info & MAY_BE_PACKED_GUARD) {
13216 if (!zend_jit_packed_guard(jit, opline, opline->op1.var, op1_info)) {
13217 return 0;
13218 }
13219 }
13220
13221 // JIT: pos = Z_FE_POS_P(array);
13222 hash_pos_ref = packed_pos_ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)));
13223
13224 if (MAY_BE_HASH(op1_info)) {
13225 ir_ref loop_ref, pos2_ref, p2_ref;
13226
13227 if (MAY_BE_PACKED(op1_info)) {
13228 ref = ir_AND_U32(
13229 ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
13230 ir_CONST_U32(HASH_FLAG_PACKED));
13231 if_packed = ir_IF(ref);
13232 ir_IF_FALSE(if_packed);
13233 }
13234
13235 // JIT: p = fe_ht->arData + pos;
13236 if (sizeof(void*) == 8) {
13237 ref = ir_ZEXT_A(hash_pos_ref);
13238 } else {
13239 ref = ir_BITCAST_A(hash_pos_ref);
13240 }
13241 hash_p_ref = ir_ADD_A(
13242 ir_MUL_A(ref, ir_CONST_ADDR(sizeof(Bucket))),
13243 ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arData))));
13244
13245 loop_ref = ir_LOOP_BEGIN(ir_END());
13246 hash_pos_ref = ir_PHI_2(IR_U32, hash_pos_ref, IR_UNUSED);
13247 hash_p_ref = ir_PHI_2(IR_ADDR, hash_p_ref, IR_UNUSED);
13248
13249 // JIT: if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
13250 ref = ir_ULT(hash_pos_ref,
13251 ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed))));
13252
13253 // JIT: ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
13254 // JIT: ZEND_VM_CONTINUE();
13255
13256 if (exit_addr) {
13257 if (exit_opcode == ZEND_JMP) {
13258 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
13259 } else {
13260 ir_ref if_fit = ir_IF(ref);
13261 ir_IF_FALSE(if_fit);
13262 ir_END_list(exit_inputs);
13263 ir_IF_TRUE(if_fit);
13264 }
13265 } else {
13266 ir_ref if_fit = ir_IF(ref);
13267 ir_IF_FALSE(if_fit);
13268 ir_END_list(exit_inputs);
13269 ir_IF_TRUE(if_fit);
13270 }
13271
13272 // JIT: pos++;
13273 pos2_ref = ir_ADD_U32(hash_pos_ref, ir_CONST_U32(1));
13274
13275 // JIT: value_type = Z_TYPE_INFO_P(value);
13276 // JIT: if (EXPECTED(value_type != IS_UNDEF)) {
13277 if (!exit_addr || exit_opcode == ZEND_JMP) {
13278 if_def_hash = ir_IF(jit_Z_TYPE_ref(jit, hash_p_ref));
13279 ir_IF_FALSE(if_def_hash);
13280 } else {
13281 ir_GUARD_NOT(jit_Z_TYPE_ref(jit, hash_p_ref), ir_CONST_ADDR(exit_addr));
13282 }
13283
13284 // JIT: p++;
13285 p2_ref = ir_ADD_OFFSET(hash_p_ref, sizeof(Bucket));
13286
13287 ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
13288 ir_PHI_SET_OP(hash_pos_ref, 2, pos2_ref);
13289 ir_PHI_SET_OP(hash_p_ref, 2, p2_ref);
13290
13291 if (MAY_BE_PACKED(op1_info)) {
13292 ir_IF_TRUE(if_packed);
13293 }
13294 }
13295 if (MAY_BE_PACKED(op1_info)) {
13296 ir_ref loop_ref, pos2_ref, p2_ref;
13297
13298 // JIT: p = fe_ht->arPacked + pos;
13299 if (sizeof(void*) == 8) {
13300 ref = ir_ZEXT_A(packed_pos_ref);
13301 } else {
13302 ref = ir_BITCAST_A(packed_pos_ref);
13303 }
13304 packed_p_ref = ir_ADD_A(
13305 ir_MUL_A(ref, ir_CONST_ADDR(sizeof(zval))),
13306 ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arPacked))));
13307
13308 loop_ref = ir_LOOP_BEGIN(ir_END());
13309 packed_pos_ref = ir_PHI_2(IR_U32, packed_pos_ref, IR_UNUSED);
13310 packed_p_ref = ir_PHI_2(IR_ADDR, packed_p_ref, IR_UNUSED);
13311
13312 // JIT: if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
13313 ref = ir_ULT(packed_pos_ref,
13314 ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed))));
13315
13316 // JIT: ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
13317 // JIT: ZEND_VM_CONTINUE();
13318 if (exit_addr) {
13319 if (exit_opcode == ZEND_JMP) {
13320 ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
13321 } else {
13322 ir_ref if_fit = ir_IF(ref);
13323 ir_IF_FALSE(if_fit);
13324 ir_END_list(exit_inputs);
13325 ir_IF_TRUE(if_fit);
13326 }
13327 } else {
13328 ir_ref if_fit = ir_IF(ref);
13329 ir_IF_FALSE(if_fit);
13330 ir_END_list(exit_inputs);
13331 ir_IF_TRUE(if_fit);
13332 }
13333
13334 // JIT: pos++;
13335 pos2_ref = ir_ADD_U32(packed_pos_ref, ir_CONST_U32(1));
13336
13337 // JIT: value_type = Z_TYPE_INFO_P(value);
13338 // JIT: if (EXPECTED(value_type != IS_UNDEF)) {
13339 if (!exit_addr || exit_opcode == ZEND_JMP) {
13340 if_def_packed = ir_IF(jit_Z_TYPE_ref(jit, packed_p_ref));
13341 ir_IF_FALSE(if_def_packed);
13342 } else {
13343 ir_GUARD_NOT(jit_Z_TYPE_ref(jit, packed_p_ref), ir_CONST_ADDR(exit_addr));
13344 }
13345
13346 // JIT: p++;
13347 p2_ref = ir_ADD_OFFSET(packed_p_ref, sizeof(zval));
13348
13349 ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
13350 ir_PHI_SET_OP(packed_pos_ref, 2, pos2_ref);
13351 ir_PHI_SET_OP(packed_p_ref, 2, p2_ref);
13352 }
13353
13354 if (!exit_addr || exit_opcode == ZEND_JMP) {
13355 zend_jit_addr val_addr;
13356 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
13357 uint32_t val_info;
13358 ir_ref p_ref = IR_UNUSED, hash_path = IR_UNUSED;
13359
13360 if (RETURN_VALUE_USED(opline)) {
13361 zend_jit_addr res_addr = RES_ADDR();
13362
13363 if (MAY_BE_HASH(op1_info)) {
13364 ir_ref key_ref = IR_UNUSED, if_key = IR_UNUSED, key_path = IR_UNUSED;
13365
13366 ZEND_ASSERT(if_def_hash);
13367 ir_IF_TRUE(if_def_hash);
13368
13369 // JIT: Z_FE_POS_P(array) = pos + 1;
13370 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
13371 ir_ADD_U32(hash_pos_ref, ir_CONST_U32(1)));
13372
13373 if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
13374 key_ref = ir_LOAD_A(ir_ADD_OFFSET(hash_p_ref, offsetof(Bucket, key)));
13375 }
13376 if ((op1_info & MAY_BE_ARRAY_KEY_LONG)
13377 && (op1_info & MAY_BE_ARRAY_KEY_STRING)) {
13378 // JIT: if (!p->key) {
13379 if_key = ir_IF(key_ref);
13380 ir_IF_TRUE(if_key);
13381 }
13382 if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
13383 ir_ref if_interned, interned_path;
13384
13385 // JIT: ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key);
13386 jit_set_Z_PTR(jit, res_addr, key_ref);
13387 ref = ir_AND_U32(
13388 ir_LOAD_U32(ir_ADD_OFFSET(key_ref, offsetof(zend_refcounted, gc.u.type_info))),
13389 ir_CONST_U32(IS_STR_INTERNED));
13390 if_interned = ir_IF(ref);
13391 ir_IF_TRUE(if_interned);
13392
13393 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
13394
13395 interned_path = ir_END();
13396 ir_IF_FALSE(if_interned);
13397
13398 jit_GC_ADDREF(jit, key_ref);
13399 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
13400
13401 ir_MERGE_WITH(interned_path);
13402
13403 if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
13404 key_path = ir_END();
13405 }
13406 }
13407 if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
13408 if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
13409 ir_IF_FALSE(if_key);
13410 }
13411 // JIT: ZVAL_LONG(EX_VAR(opline->result.var), p->h);
13412 ref = ir_LOAD_L(ir_ADD_OFFSET(hash_p_ref, offsetof(Bucket, h)));
13413 jit_set_Z_LVAL(jit, res_addr, ref);
13414 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
13415
13416 if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
13417 ir_MERGE_WITH(key_path);
13418 }
13419 }
13420 if (MAY_BE_PACKED(op1_info)) {
13421 hash_path = ir_END();
13422 } else {
13423 p_ref = hash_p_ref;
13424 }
13425 }
13426 if (MAY_BE_PACKED(op1_info)) {
13427 ZEND_ASSERT(if_def_packed);
13428 ir_IF_TRUE(if_def_packed);
13429
13430 // JIT: Z_FE_POS_P(array) = pos + 1;
13431 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
13432 ir_ADD_U32(packed_pos_ref, ir_CONST_U32(1)));
13433
13434 // JIT: ZVAL_LONG(EX_VAR(opline->result.var), pos);
13435 if (sizeof(zend_long) == 8) {
13436 packed_pos_ref = ir_ZEXT_L(packed_pos_ref);
13437 } else {
13438 packed_pos_ref = ir_BITCAST_L(packed_pos_ref);
13439 }
13440 jit_set_Z_LVAL(jit, res_addr, packed_pos_ref);
13441 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
13442
13443 if (MAY_BE_HASH(op1_info)) {
13444 ir_MERGE_WITH(hash_path);
13445 p_ref = ir_PHI_2(IR_ADDR, packed_p_ref, hash_p_ref);
13446 } else {
13447 p_ref = packed_p_ref;
13448 }
13449 }
13450 } else {
13451 ir_ref pos_ref = IR_UNUSED;
13452
13453 if (if_def_hash && if_def_packed) {
13454 ir_IF_TRUE(if_def_hash);
13455 ir_MERGE_WITH_EMPTY_TRUE(if_def_packed);
13456 pos_ref = ir_PHI_2(IR_U32, hash_pos_ref, packed_pos_ref);
13457 p_ref = ir_PHI_2(IR_ADDR, hash_p_ref, packed_p_ref);
13458 } else if (if_def_hash) {
13459 ir_IF_TRUE(if_def_hash);
13460 pos_ref = hash_pos_ref;
13461 p_ref = hash_p_ref;
13462 } else if (if_def_packed) {
13463 ir_IF_TRUE(if_def_packed);
13464 pos_ref = packed_pos_ref;
13465 p_ref = packed_p_ref;
13466 } else {
13467 ZEND_UNREACHABLE();
13468 }
13469
13470 // JIT: Z_FE_POS_P(array) = pos + 1;
13471 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
13472 ir_ADD_U32(pos_ref, ir_CONST_U32(1)));
13473 }
13474
13475 val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT);
13476 if (val_info & MAY_BE_ARRAY) {
13477 val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
13478 }
13479 if (op1_info & MAY_BE_ARRAY_OF_REF) {
13480 val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY |
13481 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
13482 } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
13483 val_info |= MAY_BE_RC1 | MAY_BE_RCN;
13484 }
13485
13486 val_addr = ZEND_ADDR_REF_ZVAL(p_ref);
13487 if (opline->op2_type == IS_CV) {
13488 // JIT: zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES());
13489 if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 0, 1)) {
13490 return 0;
13491 }
13492 } else {
13493 // JIT: ZVAL_COPY(res, value);
13494 jit_ZVAL_COPY(jit, var_addr, -1, val_addr, val_info, 1);
13495 }
13496
13497 if (!exit_addr) {
13498 zend_basic_block *bb;
13499
13500 ZEND_ASSERT(jit->b >= 0);
13501 bb = &jit->ssa->cfg.blocks[jit->b];
13502 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ir_END());
13503 ZEND_ASSERT(exit_inputs);
13504 if (!jit->ctx.ir_base[exit_inputs].op2) {
13505 ref = exit_inputs;
13506 } else {
13507 ir_MERGE_list(exit_inputs);
13508 ref = ir_END();
13509 }
13510 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
13511 jit->b = -1;
13512 }
13513 } else {
13514 ZEND_ASSERT(exit_inputs);
13515 ir_MERGE_list(exit_inputs);
13516 }
13517
13518 return 1;
13519 }
13520
13521 static int zend_jit_load_this(zend_jit_ctx *jit, uint32_t var)
13522 {
13523 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
13524 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
13525 ir_ref ref = jit_Z_PTR(jit, this_addr);
13526
13527 jit_set_Z_PTR(jit, var_addr, ref);
13528 jit_set_Z_TYPE_INFO(jit, var_addr, IS_OBJECT_EX);
13529 jit_GC_ADDREF(jit, ref);
13530
13531 return 1;
13532 }
13533
13534 static int zend_jit_fetch_this(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, bool check_only)
13535 {
13536 if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) {
13537 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
13538 if (!JIT_G(current_frame) ||
13539 !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) {
13540
13541 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
13542 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
13543 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
13544
13545 if (!exit_addr) {
13546 return 0;
13547 }
13548
13549 jit_guard_Z_TYPE(jit, this_addr, IS_OBJECT, exit_addr);
13550
13551 if (JIT_G(current_frame)) {
13552 TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame));
13553 }
13554 }
13555 } else {
13556 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
13557 ir_ref if_object = jit_if_Z_TYPE(jit, this_addr, IS_OBJECT);
13558
13559 ir_IF_FALSE_cold(if_object);
13560 jit_SET_EX_OPLINE(jit, opline);
13561 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_invalid_this));
13562
13563 ir_IF_TRUE(if_object);
13564 }
13565 }
13566
13567 if (!check_only) {
13568 if (!zend_jit_load_this(jit, opline->result.var)) {
13569 return 0;
13570 }
13571 }
13572
13573 return 1;
13574 }
13575
13576 static int zend_jit_class_guard(zend_jit_ctx *jit, const zend_op *opline, ir_ref obj_ref, zend_class_entry *ce)
13577 {
13578 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
13579 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
13580
13581 if (!exit_addr) {
13582 return 0;
13583 }
13584
13585 ir_GUARD(ir_EQ(ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))), ir_CONST_ADDR(ce)),
13586 ir_CONST_ADDR(exit_addr));
13587
13588 return 1;
13589 }
13590
13591 static int zend_jit_fetch_obj(zend_jit_ctx *jit,
13592 const zend_op *opline,
13593 const zend_op_array *op_array,
13594 zend_ssa *ssa,
13595 const zend_ssa_op *ssa_op,
13596 uint32_t op1_info,
13597 zend_jit_addr op1_addr,
13598 bool op1_indirect,
13599 zend_class_entry *ce,
13600 bool ce_is_instanceof,
13601 bool on_this,
13602 bool delayed_fetch_this,
13603 bool op1_avoid_refcounting,
13604 zend_class_entry *trace_ce,
13605 uint8_t prop_type,
13606 int may_throw)
13607 {
13608 zval *member;
13609 zend_property_info *prop_info;
13610 bool may_be_dynamic = 1;
13611 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
13612 zend_jit_addr prop_addr;
13613 uint32_t res_info = RES_INFO();
13614 ir_ref prop_type_ref = IR_UNUSED;
13615 ir_ref obj_ref = IR_UNUSED;
13616 ir_ref prop_ref = IR_UNUSED;
13617 ir_ref end_inputs = IR_UNUSED;
13618 ir_ref slow_inputs = IR_UNUSED;
13619
13620 ZEND_ASSERT(opline->op2_type == IS_CONST);
13621 ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
13622
13623 member = RT_CONSTANT(opline, opline->op2);
13624 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
13625 prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename);
13626
13627 if (on_this) {
13628 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
13629 obj_ref = jit_Z_PTR(jit, this_addr);
13630 } else {
13631 if (opline->op1_type == IS_VAR
13632 && opline->opcode == ZEND_FETCH_OBJ_W
13633 && (op1_info & MAY_BE_INDIRECT)
13634 && Z_REG(op1_addr) == ZREG_FP) {
13635 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
13636 }
13637 if (op1_info & MAY_BE_REF) {
13638 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
13639 }
13640 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
13641 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
13642 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
13643 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
13644
13645 if (!exit_addr) {
13646 return 0;
13647 }
13648 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
13649 } else {
13650 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
13651
13652 ir_IF_FALSE_cold(if_obj);
13653 if (opline->opcode != ZEND_FETCH_OBJ_IS) {
13654 ir_ref op1_ref = IR_UNUSED;
13655
13656 jit_SET_EX_OPLINE(jit, opline);
13657 if (opline->opcode != ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_UNDEF)) {
13658 zend_jit_addr orig_op1_addr = OP1_ADDR();
13659 ir_ref fast_path = IR_UNUSED;
13660
13661 if (op1_info & MAY_BE_ANY) {
13662 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
13663 ir_IF_TRUE(if_def);
13664 fast_path = ir_END();
13665 ir_IF_FALSE_cold(if_def);
13666 }
13667 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
13668 ir_CONST_U32(opline->op1.var));
13669 if (fast_path) {
13670 ir_MERGE_WITH(fast_path);
13671 }
13672 op1_ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
13673 } else {
13674 op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
13675 }
13676 if (opline->opcode == ZEND_FETCH_OBJ_W) {
13677 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_write),
13678 op1_ref, ir_CONST_ADDR(Z_STRVAL_P(member)));
13679 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
13680 } else {
13681 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_read),
13682 op1_ref, ir_CONST_ADDR(Z_STRVAL_P(member)));
13683 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
13684 }
13685 } else {
13686 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
13687 }
13688 ir_END_list(end_inputs);
13689
13690 ir_IF_TRUE(if_obj);
13691 }
13692 }
13693 obj_ref = jit_Z_PTR(jit, op1_addr);
13694 }
13695
13696 ZEND_ASSERT(obj_ref);
13697 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
13698 prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename);
13699 if (prop_info) {
13700 ce = trace_ce;
13701 ce_is_instanceof = 0;
13702 if (!(op1_info & MAY_BE_CLASS_GUARD)) {
13703 if (on_this && JIT_G(current_frame)
13704 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
13705 ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
13706 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
13707 if (on_this && JIT_G(current_frame)) {
13708 JIT_G(current_frame)->ce = ce;
13709 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
13710 }
13711 } else {
13712 return 0;
13713 }
13714 if (ssa->var_info && ssa_op->op1_use >= 0) {
13715 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
13716 ssa->var_info[ssa_op->op1_use].ce = ce;
13717 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
13718 }
13719 }
13720 }
13721 }
13722
13723 if (!prop_info) {
13724 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
13725 ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
13726 ir_ref if_same = ir_IF(ir_EQ(ref,
13727 ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
13728
13729 ir_IF_FALSE_cold(if_same);
13730 ir_END_list(slow_inputs);
13731
13732 ir_IF_TRUE(if_same);
13733 ir_ref offset_ref = ir_LOAD_A(
13734 ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
13735
13736 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename);
13737 if (may_be_dynamic) {
13738 ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, IR_NULL));
13739 if (opline->opcode == ZEND_FETCH_OBJ_W) {
13740 ir_IF_TRUE_cold(if_dynamic);
13741 ir_END_list(slow_inputs);
13742 } else {
13743 ir_IF_TRUE_cold(if_dynamic);
13744 jit_SET_EX_OPLINE(jit, opline);
13745
13746 if (opline->opcode != ZEND_FETCH_OBJ_IS) {
13747 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_dynamic),
13748 obj_ref, offset_ref);
13749 } else {
13750 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_dynamic),
13751 obj_ref, offset_ref);
13752 }
13753 ir_END_list(end_inputs);
13754 }
13755 ir_IF_FALSE(if_dynamic);
13756 }
13757 prop_ref = ir_ADD_A(obj_ref, offset_ref);
13758 prop_type_ref = jit_Z_TYPE_ref(jit, prop_ref);
13759 ir_ref if_def = ir_IF(prop_type_ref);
13760 ir_IF_FALSE_cold(if_def);
13761 ir_END_list(slow_inputs);
13762 ir_IF_TRUE(if_def);
13763 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
13764 if (opline->opcode == ZEND_FETCH_OBJ_W
13765 && (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) {
13766 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
13767
13768 ir_ref prop_info_ref = ir_LOAD_A(
13769 ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
13770 ir_ref if_has_prop_info = ir_IF(prop_info_ref);
13771
13772 ir_IF_TRUE_cold(if_has_prop_info);
13773
13774 ir_ref if_readonly = ir_IF(
13775 ir_AND_U32(ir_LOAD_U32(ir_ADD_OFFSET(prop_info_ref, offsetof(zend_property_info, flags))),
13776 ir_CONST_U32(ZEND_ACC_READONLY)));
13777 ir_IF_TRUE(if_readonly);
13778
13779 ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
13780 ir_IF_TRUE(if_prop_obj);
13781 ref = jit_Z_PTR(jit, prop_addr);
13782 jit_GC_ADDREF(jit, ref);
13783 jit_set_Z_PTR(jit, res_addr, ref);
13784 jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
13785 ir_END_list(end_inputs);
13786
13787 ir_IF_FALSE_cold(if_prop_obj);
13788
13789 ir_ref extra_addr = ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, prop_addr), offsetof(zval, u2.extra));
13790 ir_ref extra = ir_LOAD_U32(extra_addr);
13791 ir_ref if_reinitable = ir_IF(ir_AND_U32(extra, ir_CONST_U32(IS_PROP_REINITABLE)));
13792 ir_IF_TRUE(if_reinitable);
13793 ir_STORE(extra_addr, ir_AND_U32(extra, ir_CONST_U32(~IS_PROP_REINITABLE)));
13794 ir_ref reinit_path = ir_END();
13795
13796 ir_IF_FALSE(if_reinitable);
13797
13798 jit_SET_EX_OPLINE(jit, opline);
13799 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_readonly_property_modification_error), prop_info_ref);
13800 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
13801 ir_END_list(end_inputs);
13802
13803 if (flags == ZEND_FETCH_DIM_WRITE) {
13804 ir_IF_FALSE_cold(if_readonly);
13805 ir_MERGE_WITH(reinit_path);
13806 jit_SET_EX_OPLINE(jit, opline);
13807 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_check_array_promotion),
13808 prop_ref, prop_info_ref);
13809 ir_END_list(end_inputs);
13810 ir_IF_FALSE(if_has_prop_info);
13811 } else if (flags == ZEND_FETCH_REF) {
13812 ir_IF_FALSE_cold(if_readonly);
13813 ir_MERGE_WITH(reinit_path);
13814 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_create_typed_ref),
13815 prop_ref,
13816 prop_info_ref,
13817 jit_ZVAL_ADDR(jit, res_addr));
13818 ir_END_list(end_inputs);
13819 ir_IF_FALSE(if_has_prop_info);
13820 } else {
13821 ir_ref list = reinit_path;
13822
13823 ZEND_ASSERT(flags == 0);
13824 ir_IF_FALSE(if_has_prop_info);
13825 ir_END_list(list);
13826 ir_IF_FALSE(if_readonly);
13827 ir_END_list(list);
13828 ir_MERGE_list(list);
13829 }
13830 }
13831 } else {
13832 prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
13833 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
13834 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
13835 if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) {
13836 /* perform IS_UNDEF check only after result type guard (during deoptimization) */
13837 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
13838 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
13839
13840 if (!exit_addr) {
13841 return 0;
13842 }
13843 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
13844 ir_GUARD(prop_type_ref, ir_CONST_ADDR(exit_addr));
13845 }
13846 } else {
13847 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
13848 ir_ref if_def = ir_IF(prop_type_ref);
13849 ir_IF_FALSE_cold(if_def);
13850 ir_END_list(slow_inputs);
13851 ir_IF_TRUE(if_def);
13852 }
13853 if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) {
13854 if (!prop_type_ref) {
13855 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
13856 }
13857 ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
13858 ir_IF_TRUE(if_prop_obj);
13859 ir_ref ref = jit_Z_PTR(jit, prop_addr);
13860 jit_GC_ADDREF(jit, ref);
13861 jit_set_Z_PTR(jit, res_addr, ref);
13862 jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
13863 ir_END_list(end_inputs);
13864
13865 ir_IF_FALSE_cold(if_prop_obj);
13866
13867 ir_ref extra_addr = ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, prop_addr), offsetof(zval, u2.extra));
13868 ir_ref extra = ir_LOAD_U32(extra_addr);
13869 ir_ref if_reinitable = ir_IF(ir_AND_U32(extra, ir_CONST_U32(IS_PROP_REINITABLE)));
13870
13871 ir_IF_FALSE(if_reinitable);
13872 jit_SET_EX_OPLINE(jit, opline);
13873 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_readonly_property_modification_error), ir_CONST_ADDR(prop_info));
13874 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
13875 ir_END_list(end_inputs);
13876
13877 ir_IF_TRUE(if_reinitable);
13878 ir_STORE(extra_addr, ir_AND_U32(extra, ir_CONST_U32(~IS_PROP_REINITABLE)));
13879 }
13880
13881 if (opline->opcode == ZEND_FETCH_OBJ_W
13882 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS)
13883 && ZEND_TYPE_IS_SET(prop_info->type)) {
13884 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
13885
13886 if (flags == ZEND_FETCH_DIM_WRITE) {
13887 if ((ZEND_TYPE_FULL_MASK(prop_info->type) & MAY_BE_ARRAY) == 0) {
13888 if (!prop_type_ref) {
13889 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
13890 }
13891 ir_ref if_null_or_flase = ir_IF(ir_LE(prop_type_ref, ir_CONST_U32(IR_FALSE)));
13892 ir_IF_TRUE_cold(if_null_or_flase);
13893 jit_SET_EX_OPLINE(jit, opline);
13894 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_check_array_promotion),
13895 prop_ref, ir_CONST_ADDR(prop_info));
13896 ir_END_list(end_inputs);
13897 ir_IF_FALSE(if_null_or_flase);
13898 }
13899 } else if (flags == ZEND_FETCH_REF) {
13900 ir_ref ref;
13901
13902 if (!prop_type_ref) {
13903 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
13904 }
13905
13906 ir_ref if_reference = ir_IF(ir_EQ(prop_type_ref, ir_CONST_U32(IS_REFERENCE_EX)));
13907 ir_IF_FALSE(if_reference);
13908 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
13909 ref = ir_CONST_ADDR(prop_info);
13910 } else {
13911 int prop_info_offset =
13912 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
13913
13914 ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
13915 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
13916 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
13917 }
13918 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_create_typed_ref),
13919 prop_ref,
13920 ref,
13921 jit_ZVAL_ADDR(jit, res_addr));
13922 ir_END_list(end_inputs);
13923 ir_IF_TRUE(if_reference);
13924 } else {
13925 ZEND_UNREACHABLE();
13926 }
13927 }
13928 }
13929
13930 if (opline->opcode == ZEND_FETCH_OBJ_W) {
13931 ZEND_ASSERT(prop_ref);
13932 jit_set_Z_PTR(jit, res_addr, prop_ref);
13933 jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
13934 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) {
13935 ssa->var_info[ssa_op->result_def].indirect_reference = 1;
13936 }
13937 ir_END_list(end_inputs);
13938 } else {
13939 bool result_avoid_refcounting = 0;
13940
13941 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) {
13942 uint8_t type = concrete_type(res_info);
13943 uint32_t flags = 0;
13944 zend_jit_addr val_addr = prop_addr;
13945
13946 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
13947 && !delayed_fetch_this
13948 && !op1_avoid_refcounting) {
13949 flags = ZEND_JIT_EXIT_FREE_OP1;
13950 }
13951
13952 if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
13953 && !(flags & ZEND_JIT_EXIT_FREE_OP1)
13954 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
13955 && (ssa_op+1)->op1_use == ssa_op->result_def
13956 && zend_jit_may_avoid_refcounting(opline+1, res_info)) {
13957 result_avoid_refcounting = 1;
13958 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
13959 }
13960
13961 val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
13962 1, flags, op1_avoid_refcounting);
13963 if (!val_addr) {
13964 return 0;
13965 }
13966
13967 res_info &= ~MAY_BE_GUARD;
13968 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
13969
13970 // ZVAL_COPY
13971 jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
13972 } else {
13973 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
13974
13975 if (!zend_jit_zval_copy_deref(jit, res_addr, prop_addr, prop_type_ref)) {
13976 return 0;
13977 }
13978 }
13979 ir_END_list(end_inputs);
13980 }
13981
13982 if (op1_avoid_refcounting) {
13983 SET_STACK_REG(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
13984 }
13985
13986 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) {
13987 ir_MERGE_list(slow_inputs);
13988 jit_SET_EX_OPLINE(jit, opline);
13989
13990 if (opline->opcode == ZEND_FETCH_OBJ_W) {
13991 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_w_slow), obj_ref);
13992 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) {
13993 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_slow), obj_ref);
13994 } else {
13995 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_slow), obj_ref);
13996 }
13997 ir_END_list(end_inputs);
13998 }
13999
14000 ir_MERGE_list(end_inputs);
14001
14002 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
14003 if (opline->op1_type == IS_VAR
14004 && opline->opcode == ZEND_FETCH_OBJ_W
14005 && (op1_info & MAY_BE_RC1)) {
14006 zend_jit_addr orig_op1_addr = OP1_ADDR();
14007 ir_ref if_refcounted, ptr, refcount, if_non_zero;
14008 ir_ref merge_inputs = IR_UNUSED;
14009
14010 if_refcounted = jit_if_REFCOUNTED(jit, orig_op1_addr);
14011 ir_IF_FALSE( if_refcounted);
14012 ir_END_list(merge_inputs);
14013 ir_IF_TRUE( if_refcounted);
14014 ptr = jit_Z_PTR(jit, orig_op1_addr);
14015 refcount = jit_GC_DELREF(jit, ptr);
14016 if_non_zero = ir_IF(refcount);
14017 ir_IF_TRUE( if_non_zero);
14018 ir_END_list(merge_inputs);
14019 ir_IF_FALSE( if_non_zero);
14020 jit_SET_EX_OPLINE(jit, opline);
14021 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_extract_helper), ptr);
14022 ir_END_list(merge_inputs);
14023 ir_MERGE_list(merge_inputs);
14024 } else if (!op1_avoid_refcounting) {
14025 if (on_this) {
14026 op1_info &= ~MAY_BE_RC1;
14027 }
14028 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
14029 }
14030 }
14031
14032 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
14033 && prop_info
14034 && (opline->opcode != ZEND_FETCH_OBJ_W ||
14035 !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) ||
14036 !ZEND_TYPE_IS_SET(prop_info->type))
14037 && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) {
14038 may_throw = 0;
14039 }
14040
14041 if (may_throw) {
14042 zend_jit_check_exception(jit);
14043 }
14044
14045 return 1;
14046 }
14047
14048 static int zend_jit_assign_obj(zend_jit_ctx *jit,
14049 const zend_op *opline,
14050 const zend_op_array *op_array,
14051 zend_ssa *ssa,
14052 const zend_ssa_op *ssa_op,
14053 uint32_t op1_info,
14054 zend_jit_addr op1_addr,
14055 uint32_t val_info,
14056 bool op1_indirect,
14057 zend_class_entry *ce,
14058 bool ce_is_instanceof,
14059 bool on_this,
14060 bool delayed_fetch_this,
14061 zend_class_entry *trace_ce,
14062 uint8_t prop_type,
14063 int may_throw)
14064 {
14065 zval *member;
14066 zend_string *name;
14067 zend_property_info *prop_info;
14068 zend_jit_addr val_addr = OP1_DATA_ADDR();
14069 zend_jit_addr res_addr = 0;
14070 zend_jit_addr prop_addr;
14071 ir_ref obj_ref = IR_UNUSED;
14072 ir_ref prop_ref = IR_UNUSED;
14073 ir_ref delayed_end_input = IR_UNUSED;
14074 ir_ref end_inputs = IR_UNUSED;
14075 ir_ref slow_inputs = IR_UNUSED;
14076
14077 if (RETURN_VALUE_USED(opline)) {
14078 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
14079 }
14080
14081 ZEND_ASSERT(opline->op2_type == IS_CONST);
14082 ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
14083
14084 member = RT_CONSTANT(opline, opline->op2);
14085 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
14086 name = Z_STR_P(member);
14087 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
14088
14089 if (on_this) {
14090 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
14091 obj_ref = jit_Z_PTR(jit, this_addr);
14092 } else {
14093 if (opline->op1_type == IS_VAR
14094 && (op1_info & MAY_BE_INDIRECT)
14095 && Z_REG(op1_addr) == ZREG_FP) {
14096 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
14097 }
14098 if (op1_info & MAY_BE_REF) {
14099 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
14100 }
14101 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
14102 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
14103 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
14104 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
14105
14106 if (!exit_addr) {
14107 return 0;
14108 }
14109 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
14110 } else {
14111 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
14112 ir_IF_FALSE_cold(if_obj);
14113
14114 jit_SET_EX_OPLINE(jit, opline);
14115 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_assign),
14116 jit_ZVAL_ADDR(jit, op1_addr),
14117 ir_CONST_ADDR(ZSTR_VAL(name)));
14118
14119 if (RETURN_VALUE_USED(opline)) {
14120 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
14121 }
14122
14123 ir_END_list(end_inputs);
14124
14125 ir_IF_TRUE(if_obj);
14126 }
14127 }
14128 obj_ref = jit_Z_PTR(jit, op1_addr);
14129 }
14130
14131 ZEND_ASSERT(obj_ref);
14132 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
14133 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
14134 if (prop_info) {
14135 ce = trace_ce;
14136 ce_is_instanceof = 0;
14137 if (!(op1_info & MAY_BE_CLASS_GUARD)) {
14138 if (on_this && JIT_G(current_frame)
14139 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
14140 ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
14141 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
14142 if (on_this && JIT_G(current_frame)) {
14143 JIT_G(current_frame)->ce = ce;
14144 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
14145 }
14146 } else {
14147 return 0;
14148 }
14149 if (ssa->var_info && ssa_op->op1_use >= 0) {
14150 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
14151 ssa->var_info[ssa_op->op1_use].ce = ce;
14152 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
14153 }
14154 if (ssa->var_info && ssa_op->op1_def >= 0) {
14155 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
14156 ssa->var_info[ssa_op->op1_def].ce = ce;
14157 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
14158 }
14159 }
14160 }
14161 }
14162
14163 if (!prop_info) {
14164 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
14165 ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
14166 ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
14167
14168 ir_IF_FALSE_cold(if_same);
14169 ir_END_list(slow_inputs);
14170
14171 ir_IF_TRUE(if_same);
14172 ir_ref offset_ref = ir_LOAD_A(
14173 ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
14174
14175 ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, IR_NULL));
14176 ir_IF_TRUE_cold(if_dynamic);
14177 ir_END_list(slow_inputs);
14178
14179 ir_IF_FALSE(if_dynamic);
14180 prop_ref = ir_ADD_A(obj_ref, offset_ref);
14181 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
14182 ir_IF_FALSE_cold(if_def);
14183 ir_END_list(slow_inputs);
14184
14185 ir_IF_TRUE(if_def);
14186 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14187
14188 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
14189 ir_ref prop_info_ref = ir_LOAD_A(
14190 ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
14191 ir_ref if_has_prop_info = ir_IF(prop_info_ref);
14192 ir_IF_TRUE_cold(if_has_prop_info);
14193
14194 // JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
14195 jit_SET_EX_OPLINE(jit, opline);
14196 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_to_typed_prop),
14197 prop_ref,
14198 prop_info_ref,
14199 jit_ZVAL_ADDR(jit, val_addr),
14200 RETURN_VALUE_USED(opline) ? jit_ZVAL_ADDR(jit, res_addr) : IR_NULL);
14201
14202 if ((opline+1)->op1_type == IS_CONST) {
14203 // TODO: ???
14204 // if (Z_TYPE_P(value) == orig_type) {
14205 // CACHE_PTR_EX(cache_slot + 2, NULL);
14206 }
14207
14208 ir_END_list(end_inputs);
14209 ir_IF_FALSE(if_has_prop_info);
14210 }
14211 } else {
14212 prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
14213 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14214 if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) {
14215 // Undefined property with magic __get()/__set()
14216 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
14217 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
14218 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
14219
14220 if (!exit_addr) {
14221 return 0;
14222 }
14223 ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
14224 } else {
14225 ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
14226 ir_IF_FALSE_cold(if_def);
14227 ir_END_list(slow_inputs);
14228 ir_IF_TRUE(if_def);
14229 }
14230 }
14231 if (ZEND_TYPE_IS_SET(prop_info->type)) {
14232 ir_ref ref;
14233
14234 // JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
14235 jit_SET_EX_OPLINE(jit, opline);
14236 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
14237 ref = ir_CONST_ADDR(prop_info);
14238 } else {
14239 int prop_info_offset =
14240 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
14241
14242 ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
14243 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
14244 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
14245 }
14246 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_to_typed_prop),
14247 prop_ref,
14248 ref,
14249 jit_ZVAL_ADDR(jit, val_addr),
14250 RETURN_VALUE_USED(opline) ? jit_ZVAL_ADDR(jit, res_addr) : IR_NULL);
14251
14252 ir_END_list(end_inputs);
14253 }
14254 }
14255
14256 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
14257 if (opline->result_type == IS_UNUSED) {
14258 if (!zend_jit_assign_to_variable_call(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) {
14259 return 0;
14260 }
14261 } else {
14262 if (!zend_jit_assign_to_variable(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0, 0)) {
14263 return 0;
14264 }
14265 }
14266 if (end_inputs || slow_inputs) {
14267 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
14268 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
14269 /* skip FREE_OP_DATA() */
14270 delayed_end_input = ir_END();
14271 } else {
14272 ir_END_list(end_inputs);
14273 }
14274 }
14275 }
14276
14277 if (slow_inputs) {
14278 ir_MERGE_list(slow_inputs);
14279 jit_SET_EX_OPLINE(jit, opline);
14280
14281 // JIT: value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
14282 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
14283 ir_CALL_5(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_obj_helper),
14284 obj_ref,
14285 ir_CONST_ADDR(name),
14286 jit_ZVAL_ADDR(jit, val_addr),
14287 ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
14288 RETURN_VALUE_USED(opline) ? jit_ZVAL_ADDR(jit, res_addr) : IR_NULL);
14289
14290 ir_END_list(end_inputs);
14291 }
14292
14293 if (end_inputs) {
14294 ir_MERGE_list(end_inputs);
14295
14296 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
14297 val_info |= MAY_BE_RC1|MAY_BE_RCN;
14298 }
14299 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
14300
14301 if (delayed_end_input) {
14302 ir_MERGE_WITH(delayed_end_input);
14303 }
14304 }
14305
14306 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
14307 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
14308 }
14309
14310 if (may_throw) {
14311 zend_jit_check_exception(jit);
14312 }
14313
14314 return 1;
14315 }
14316
14317 static int zend_jit_assign_obj_op(zend_jit_ctx *jit,
14318 const zend_op *opline,
14319 const zend_op_array *op_array,
14320 zend_ssa *ssa,
14321 const zend_ssa_op *ssa_op,
14322 uint32_t op1_info,
14323 zend_jit_addr op1_addr,
14324 uint32_t val_info,
14325 zend_ssa_range *val_range,
14326 bool op1_indirect,
14327 zend_class_entry *ce,
14328 bool ce_is_instanceof,
14329 bool on_this,
14330 bool delayed_fetch_this,
14331 zend_class_entry *trace_ce,
14332 uint8_t prop_type)
14333 {
14334 zval *member;
14335 zend_string *name;
14336 zend_property_info *prop_info;
14337 zend_jit_addr val_addr = OP1_DATA_ADDR();
14338 zend_jit_addr prop_addr;
14339 bool use_prop_guard = 0;
14340 bool may_throw = 0;
14341 binary_op_type binary_op = get_binary_op(opline->extended_value);
14342 ir_ref obj_ref = IR_UNUSED;
14343 ir_ref prop_ref = IR_UNUSED;
14344 ir_ref end_inputs = IR_UNUSED;
14345 ir_ref slow_inputs = IR_UNUSED;
14346
14347 ZEND_ASSERT(opline->op2_type == IS_CONST);
14348 ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
14349 ZEND_ASSERT(opline->result_type == IS_UNUSED);
14350
14351 member = RT_CONSTANT(opline, opline->op2);
14352 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
14353 name = Z_STR_P(member);
14354 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
14355
14356 if (on_this) {
14357 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
14358 obj_ref = jit_Z_PTR(jit, this_addr);
14359 } else {
14360 if (opline->op1_type == IS_VAR
14361 && (op1_info & MAY_BE_INDIRECT)
14362 && Z_REG(op1_addr) == ZREG_FP) {
14363 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
14364 }
14365 if (op1_info & MAY_BE_REF) {
14366 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
14367 }
14368 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
14369 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
14370 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
14371 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
14372
14373 if (!exit_addr) {
14374 return 0;
14375 }
14376 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
14377 } else {
14378 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
14379 ir_IF_FALSE_cold(if_obj);
14380
14381 jit_SET_EX_OPLINE(jit, opline);
14382 ir_CALL_2(IR_VOID,
14383 (op1_info & MAY_BE_UNDEF) ?
14384 ir_CONST_FC_FUNC(zend_jit_invalid_property_assign_op) :
14385 ir_CONST_FC_FUNC(zend_jit_invalid_property_assign),
14386 jit_ZVAL_ADDR(jit, op1_addr),
14387 ir_CONST_ADDR(ZSTR_VAL(name)));
14388
14389 may_throw = 1;
14390
14391 ir_END_list(end_inputs);
14392 ir_IF_TRUE(if_obj);
14393 }
14394 }
14395 obj_ref = jit_Z_PTR(jit, op1_addr);
14396 }
14397
14398 ZEND_ASSERT(obj_ref);
14399 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
14400 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
14401 if (prop_info) {
14402 ce = trace_ce;
14403 ce_is_instanceof = 0;
14404 if (!(op1_info & MAY_BE_CLASS_GUARD)) {
14405 if (on_this && JIT_G(current_frame)
14406 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
14407 ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
14408 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
14409 if (on_this && JIT_G(current_frame)) {
14410 JIT_G(current_frame)->ce = ce;
14411 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
14412 }
14413 } else {
14414 return 0;
14415 }
14416 if (ssa->var_info && ssa_op->op1_use >= 0) {
14417 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
14418 ssa->var_info[ssa_op->op1_use].ce = ce;
14419 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
14420 }
14421 if (ssa->var_info && ssa_op->op1_def >= 0) {
14422 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
14423 ssa->var_info[ssa_op->op1_def].ce = ce;
14424 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
14425 }
14426 }
14427 }
14428 }
14429
14430 use_prop_guard = (prop_type != IS_UNKNOWN
14431 && prop_type != IS_UNDEF
14432 && prop_type != IS_REFERENCE
14433 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
14434
14435 if (!prop_info) {
14436 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
14437 ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, (opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
14438 ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
14439
14440 ir_IF_FALSE_cold(if_same);
14441 ir_END_list(slow_inputs);
14442
14443 ir_IF_TRUE(if_same);
14444 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
14445 ir_ref prop_info_ref = ir_LOAD_A(
14446 ir_ADD_OFFSET(run_time_cache, ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
14447 ir_ref if_has_prop_info = ir_IF(prop_info_ref);
14448 ir_IF_TRUE_cold(if_has_prop_info);
14449 ir_END_list(slow_inputs);
14450
14451 ir_IF_FALSE(if_has_prop_info);
14452 }
14453 ir_ref offset_ref = ir_LOAD_A(
14454 ir_ADD_OFFSET(run_time_cache, ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
14455
14456 ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, IR_NULL));
14457 ir_IF_TRUE_cold(if_dynamic);
14458 ir_END_list(slow_inputs);
14459
14460 ir_IF_FALSE(if_dynamic);
14461
14462 prop_ref = ir_ADD_A(obj_ref, offset_ref);
14463 if (!use_prop_guard) {
14464 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
14465 ir_IF_FALSE_cold(if_def);
14466 ir_END_list(slow_inputs);
14467
14468 ir_IF_TRUE(if_def);
14469 }
14470 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14471 } else {
14472 prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
14473 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14474
14475 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
14476 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
14477 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
14478 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
14479
14480 if (!exit_addr) {
14481 return 0;
14482 }
14483 ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
14484 } else {
14485 ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
14486 ir_IF_FALSE_cold(if_def);
14487 ir_END_list(slow_inputs);
14488 ir_IF_TRUE(if_def);
14489 }
14490 }
14491 if (ZEND_TYPE_IS_SET(prop_info->type)) {
14492 ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref;
14493
14494 may_throw = 1;
14495
14496 jit_SET_EX_OPLINE(jit, opline);
14497
14498 if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
14499 ir_IF_FALSE(if_ref);
14500 noref_path = ir_END();
14501 ir_IF_TRUE(if_ref);
14502
14503 reference = jit_Z_PTR(jit, prop_addr);
14504 ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
14505 if_typed = jit_if_TYPED_REF(jit, reference);
14506 ir_IF_FALSE(if_typed);
14507 ref_path = ir_END();
14508 ir_IF_TRUE_cold(if_typed);
14509
14510 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
14511 reference,
14512 jit_ZVAL_ADDR(jit, val_addr),
14513 ir_CONST_FC_FUNC(binary_op));
14514
14515 ir_END_list(end_inputs);
14516
14517 ir_MERGE_2(noref_path, ref_path);
14518 prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
14519 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14520
14521 // JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
14522 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
14523 ref = ir_CONST_ADDR(prop_info);
14524 } else {
14525 int prop_info_offset =
14526 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
14527
14528 ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
14529 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
14530 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
14531 }
14532
14533 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_prop),
14534 prop_ref,
14535 ref,
14536 jit_ZVAL_ADDR(jit, val_addr),
14537 ir_CONST_FC_FUNC(binary_op));
14538
14539 ir_END_list(end_inputs);
14540 }
14541 }
14542
14543 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
14544 zend_jit_addr var_addr = prop_addr;
14545 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
14546 uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
14547
14548 if (use_prop_guard) {
14549 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
14550 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
14551 if (!exit_addr) {
14552 return 0;
14553 }
14554
14555 jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
14556 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
14557 }
14558
14559 if (var_info & MAY_BE_REF) {
14560 ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref;
14561
14562 may_throw = 1;
14563
14564 if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
14565 ir_IF_FALSE(if_ref);
14566 noref_path = ir_END();
14567 ir_IF_TRUE(if_ref);
14568
14569 reference = jit_Z_PTR(jit, var_addr);
14570 ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
14571 if_typed = jit_if_TYPED_REF(jit, reference);
14572 ir_IF_FALSE(if_typed);
14573 ref_path = ir_END();
14574 ir_IF_TRUE_cold(if_typed);
14575
14576 jit_SET_EX_OPLINE(jit, opline);
14577
14578 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
14579 reference,
14580 jit_ZVAL_ADDR(jit, val_addr),
14581 ir_CONST_FC_FUNC(binary_op));
14582
14583 ir_END_list(end_inputs);
14584
14585 ir_MERGE_2(noref_path, ref_path);
14586 prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
14587 var_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14588
14589 var_info &= ~MAY_BE_REF;
14590 }
14591
14592 uint8_t val_op_type = (opline+1)->op1_type;
14593 if (val_op_type & (IS_TMP_VAR|IS_VAR)) {
14594 /* prevent FREE_OP in the helpers */
14595 val_op_type = IS_CV;
14596 }
14597
14598 switch (opline->extended_value) {
14599 case ZEND_ADD:
14600 case ZEND_SUB:
14601 case ZEND_MUL:
14602 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
14603 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
14604 if (opline->extended_value != ZEND_ADD ||
14605 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
14606 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) {
14607 may_throw = 1;
14608 }
14609 }
14610 if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info,
14611 1 /* may overflow */, 0)) {
14612 return 0;
14613 }
14614 break;
14615 case ZEND_BW_OR:
14616 case ZEND_BW_AND:
14617 case ZEND_BW_XOR:
14618 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
14619 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
14620 if ((var_info & MAY_BE_ANY) != MAY_BE_STRING ||
14621 (val_info & MAY_BE_ANY) != MAY_BE_STRING) {
14622 may_throw = 1;
14623 }
14624 }
14625 goto long_math;
14626 case ZEND_SL:
14627 case ZEND_SR:
14628 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
14629 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
14630 may_throw = 1;
14631 }
14632 if (val_op_type != IS_CONST ||
14633 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
14634 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) {
14635 may_throw = 1;
14636 }
14637 goto long_math;
14638 case ZEND_MOD:
14639 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
14640 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
14641 may_throw = 1;
14642 }
14643 if (val_op_type != IS_CONST ||
14644 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
14645 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) {
14646 may_throw = 1;
14647 }
14648 long_math:
14649 if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
14650 IS_CV, opline->op1, var_addr, var_info, NULL,
14651 val_op_type, (opline+1)->op1, val_addr, val_info,
14652 val_range,
14653 0, var_addr, var_def_info, var_info, /* may throw */ 1)) {
14654 return 0;
14655 }
14656 break;
14657 case ZEND_CONCAT:
14658 may_throw = 1;
14659 if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, val_addr, val_info, var_addr,
14660 0)) {
14661 return 0;
14662 }
14663 break;
14664 default:
14665 ZEND_UNREACHABLE();
14666 }
14667 if (end_inputs || slow_inputs) {
14668 ir_END_list(end_inputs);
14669 }
14670 }
14671
14672 if (slow_inputs) {
14673 ir_MERGE_list(slow_inputs);
14674
14675 may_throw = 1;
14676
14677 jit_SET_EX_OPLINE(jit, opline);
14678 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
14679 ir_CALL_5(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_obj_op_helper),
14680 obj_ref,
14681 ir_CONST_ADDR(name),
14682 jit_ZVAL_ADDR(jit, val_addr),
14683 ir_ADD_OFFSET(run_time_cache, (opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
14684 ir_CONST_FC_FUNC(binary_op));
14685
14686 ir_END_list(end_inputs);
14687 }
14688
14689 if (end_inputs) {
14690 ir_MERGE_list(end_inputs);
14691 }
14692
14693 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
14694 val_info |= MAY_BE_RC1|MAY_BE_RCN;
14695 }
14696
14697 // JIT: FREE_OP_DATA();
14698 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
14699
14700 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
14701 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
14702 may_throw = 1;
14703 }
14704 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
14705 }
14706
14707 if (may_throw) {
14708 zend_jit_check_exception(jit);
14709 }
14710
14711 return 1;
14712 }
14713
14714 static int zend_jit_incdec_obj(zend_jit_ctx *jit,
14715 const zend_op *opline,
14716 const zend_op_array *op_array,
14717 zend_ssa *ssa,
14718 const zend_ssa_op *ssa_op,
14719 uint32_t op1_info,
14720 zend_jit_addr op1_addr,
14721 bool op1_indirect,
14722 zend_class_entry *ce,
14723 bool ce_is_instanceof,
14724 bool on_this,
14725 bool delayed_fetch_this,
14726 zend_class_entry *trace_ce,
14727 uint8_t prop_type)
14728 {
14729 zval *member;
14730 zend_string *name;
14731 zend_property_info *prop_info;
14732 zend_jit_addr res_addr = 0;
14733 zend_jit_addr prop_addr;
14734 bool use_prop_guard = 0;
14735 bool may_throw = 0;
14736 uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0;
14737 ir_ref obj_ref = IR_UNUSED;
14738 ir_ref prop_ref = IR_UNUSED;
14739 ir_ref end_inputs = IR_UNUSED;
14740 ir_ref slow_inputs = IR_UNUSED;
14741
14742 ZEND_ASSERT(opline->op2_type == IS_CONST);
14743 ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
14744
14745 if (opline->result_type != IS_UNUSED) {
14746 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
14747 }
14748
14749 member = RT_CONSTANT(opline, opline->op2);
14750 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
14751 name = Z_STR_P(member);
14752 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
14753
14754 if (on_this) {
14755 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
14756 obj_ref = jit_Z_PTR(jit, this_addr);
14757 } else {
14758 if (opline->op1_type == IS_VAR
14759 && (op1_info & MAY_BE_INDIRECT)
14760 && Z_REG(op1_addr) == ZREG_FP) {
14761 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
14762 }
14763 if (op1_info & MAY_BE_REF) {
14764 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
14765 }
14766 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
14767 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
14768 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
14769 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
14770
14771 if (!exit_addr) {
14772 return 0;
14773 }
14774 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
14775 } else {
14776 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
14777 ir_IF_FALSE_cold(if_obj);
14778
14779 jit_SET_EX_OPLINE(jit, opline);
14780 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_incdec),
14781 jit_ZVAL_ADDR(jit, op1_addr),
14782 ir_CONST_ADDR(ZSTR_VAL(name)));
14783
14784 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
14785 ir_IF_TRUE(if_obj);
14786 }
14787 }
14788 obj_ref = jit_Z_PTR(jit, op1_addr);
14789 }
14790
14791 ZEND_ASSERT(obj_ref);
14792 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
14793 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
14794 if (prop_info) {
14795 ce = trace_ce;
14796 ce_is_instanceof = 0;
14797 if (!(op1_info & MAY_BE_CLASS_GUARD)) {
14798 if (on_this && JIT_G(current_frame)
14799 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
14800 ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
14801 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
14802 if (on_this && JIT_G(current_frame)) {
14803 JIT_G(current_frame)->ce = ce;
14804 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
14805 }
14806 } else {
14807 return 0;
14808 }
14809 if (ssa->var_info && ssa_op->op1_use >= 0) {
14810 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
14811 ssa->var_info[ssa_op->op1_use].ce = ce;
14812 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
14813 }
14814 if (ssa->var_info && ssa_op->op1_def >= 0) {
14815 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
14816 ssa->var_info[ssa_op->op1_def].ce = ce;
14817 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
14818 }
14819 }
14820 }
14821 }
14822
14823 use_prop_guard = (prop_type != IS_UNKNOWN
14824 && prop_type != IS_UNDEF
14825 && prop_type != IS_REFERENCE
14826 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
14827
14828 if (!prop_info) {
14829 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
14830 ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
14831 ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
14832
14833 ir_IF_FALSE_cold(if_same);
14834 ir_END_list(slow_inputs);
14835
14836 ir_IF_TRUE(if_same);
14837 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
14838 ir_ref prop_info_ref = ir_LOAD_A(
14839 ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
14840 ir_ref if_has_prop_info = ir_IF(prop_info_ref);
14841 ir_IF_TRUE_cold(if_has_prop_info);
14842 ir_END_list(slow_inputs);
14843
14844 ir_IF_FALSE(if_has_prop_info);
14845 }
14846 ir_ref offset_ref = ir_LOAD_A(
14847 ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
14848
14849 ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, IR_NULL));
14850 ir_IF_TRUE_cold(if_dynamic);
14851 ir_END_list(slow_inputs);
14852
14853 ir_IF_FALSE(if_dynamic);
14854
14855 prop_ref = ir_ADD_A(obj_ref, offset_ref);
14856 if (!use_prop_guard) {
14857 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
14858 ir_IF_FALSE_cold(if_def);
14859 ir_END_list(slow_inputs);
14860
14861 ir_IF_TRUE(if_def);
14862 }
14863 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14864 } else {
14865 prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
14866 prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
14867
14868 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
14869 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
14870 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
14871 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
14872
14873 if (!exit_addr) {
14874 return 0;
14875 }
14876 ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
14877 } else {
14878 ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
14879 ir_IF_FALSE_cold(if_def);
14880 ir_END_list(slow_inputs);
14881 ir_IF_TRUE(if_def);
14882 }
14883 }
14884
14885 if (ZEND_TYPE_IS_SET(prop_info->type)) {
14886 const void *func;
14887 ir_ref ref;
14888
14889 may_throw = 1;
14890 jit_SET_EX_OPLINE(jit, opline);
14891
14892 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
14893 ref = ir_CONST_ADDR(prop_info);
14894 } else {
14895 int prop_info_offset =
14896 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
14897
14898 ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
14899 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
14900 ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
14901 }
14902
14903 if (opline->result_type == IS_UNUSED) {
14904 switch (opline->opcode) {
14905 case ZEND_PRE_INC_OBJ:
14906 case ZEND_POST_INC_OBJ:
14907 func = zend_jit_inc_typed_prop;
14908 break;
14909 case ZEND_PRE_DEC_OBJ:
14910 case ZEND_POST_DEC_OBJ:
14911 func = zend_jit_dec_typed_prop;
14912 break;
14913 default:
14914 ZEND_UNREACHABLE();
14915 }
14916
14917 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(func), prop_ref, ref);
14918 } else {
14919 switch (opline->opcode) {
14920 case ZEND_PRE_INC_OBJ:
14921 func = zend_jit_pre_inc_typed_prop;
14922 break;
14923 case ZEND_PRE_DEC_OBJ:
14924 func = zend_jit_pre_dec_typed_prop;
14925 break;
14926 case ZEND_POST_INC_OBJ:
14927 func = zend_jit_post_inc_typed_prop;
14928 break;
14929 case ZEND_POST_DEC_OBJ:
14930 func = zend_jit_post_dec_typed_prop;
14931 break;
14932 default:
14933 ZEND_UNREACHABLE();
14934 }
14935 ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(func),
14936 prop_ref,
14937 ref,
14938 jit_ZVAL_ADDR(jit, res_addr));
14939 }
14940 ir_END_list(end_inputs);
14941 }
14942 }
14943
14944 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
14945 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
14946 zend_jit_addr var_addr = prop_addr;
14947 ir_ref if_long = IR_UNUSED;
14948 ir_ref if_overflow = IR_UNUSED;
14949
14950 if (use_prop_guard) {
14951 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
14952 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
14953 if (!exit_addr) {
14954 return 0;
14955 }
14956
14957 jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
14958 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
14959 }
14960
14961 if (var_info & MAY_BE_REF) {
14962 const void *func;
14963 ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref;
14964
14965 if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
14966 ir_IF_FALSE(if_ref);
14967 noref_path = ir_END();
14968 ir_IF_TRUE(if_ref);
14969
14970 reference = jit_Z_PTR(jit, var_addr);
14971 ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
14972 if_typed = jit_if_TYPED_REF(jit, reference);
14973 ir_IF_FALSE(if_typed);
14974 ref_path = ir_END();
14975 ir_IF_TRUE_cold(if_typed);
14976
14977 switch (opline->opcode) {
14978 case ZEND_PRE_INC_OBJ:
14979 func = zend_jit_pre_inc_typed_ref;
14980 break;
14981 case ZEND_PRE_DEC_OBJ:
14982 func = zend_jit_pre_dec_typed_ref;
14983 break;
14984 case ZEND_POST_INC_OBJ:
14985 func = zend_jit_post_inc_typed_ref;
14986 break;
14987 case ZEND_POST_DEC_OBJ:
14988 func = zend_jit_post_dec_typed_ref;
14989 break;
14990 default:
14991 ZEND_UNREACHABLE();
14992 }
14993
14994 may_throw = 1;
14995 jit_SET_EX_OPLINE(jit, opline);
14996 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(func),
14997 reference,
14998 (opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
14999
15000 ir_END_list(end_inputs);
15001
15002 ir_MERGE_2(noref_path, ref_path);
15003 prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
15004 var_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
15005
15006 var_info &= ~MAY_BE_REF;
15007 }
15008
15009 if (var_info & MAY_BE_LONG) {
15010 ir_ref addr, ref;
15011
15012 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
15013 if_long = jit_if_Z_TYPE(jit, var_addr, IS_LONG);
15014 ir_IF_TRUE(if_long);
15015 }
15016
15017 addr = jit_ZVAL_ADDR(jit, var_addr);
15018 ref = ir_LOAD_L(addr);
15019 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
15020 if (opline->result_type != IS_UNUSED) {
15021 jit_set_Z_LVAL(jit, res_addr, ref);
15022 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
15023 }
15024 }
15025 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
15026 ref = ir_ADD_OV_L(ref, ir_CONST_LONG(1));
15027 } else {
15028 ref = ir_SUB_OV_L(ref, ir_CONST_LONG(1));
15029 }
15030
15031 ir_STORE(addr, ref);
15032 if_overflow = ir_IF(ir_OVERFLOW(ref));
15033 ir_IF_FALSE(if_overflow);
15034
15035 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) {
15036 if (opline->result_type != IS_UNUSED) {
15037 jit_set_Z_LVAL(jit, res_addr, ref);
15038 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
15039 }
15040 }
15041 ir_END_list(end_inputs);
15042 }
15043
15044 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
15045 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
15046 may_throw = 1;
15047 }
15048 if (if_long) {
15049 ir_IF_FALSE_cold(if_long);
15050 }
15051 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
15052 jit_ZVAL_COPY(jit, res_addr, -1, var_addr, var_info, 1);
15053 }
15054 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
15055 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
15056 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_inc),
15057 jit_ZVAL_ADDR(jit, var_addr),
15058 jit_ZVAL_ADDR(jit, res_addr));
15059 } else {
15060 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(increment_function),
15061 jit_ZVAL_ADDR(jit, var_addr));
15062 }
15063 } else {
15064 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
15065 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_dec),
15066 jit_ZVAL_ADDR(jit, var_addr),
15067 jit_ZVAL_ADDR(jit, res_addr));
15068 } else {
15069 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(decrement_function),
15070 jit_ZVAL_ADDR(jit, var_addr));
15071 }
15072 }
15073
15074 ir_END_list(end_inputs);
15075 }
15076 if (var_info & MAY_BE_LONG) {
15077 ir_IF_TRUE_cold(if_overflow);
15078 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
15079 #if SIZEOF_ZEND_LONG == 4
15080 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0));
15081 jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0x41e00000));
15082 #else
15083 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x43e0000000000000));
15084 #endif
15085 jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
15086 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
15087 #if SIZEOF_ZEND_LONG == 4
15088 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
15089 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
15090 #else
15091 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
15092 #endif
15093 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
15094 }
15095 } else {
15096 #if SIZEOF_ZEND_LONG == 4
15097 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x00200000));
15098 jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0xc1e00000));
15099 #else
15100 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0xc3e0000000000000));
15101 #endif
15102 jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
15103 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
15104 #if SIZEOF_ZEND_LONG == 4
15105 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
15106 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
15107 #else
15108 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
15109 #endif
15110 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
15111 }
15112 }
15113 if (opline->result_type != IS_UNUSED
15114 && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ)
15115 && prop_info
15116 && !ZEND_TYPE_IS_SET(prop_info->type)
15117 && (res_info & MAY_BE_GUARD)
15118 && (res_info & MAY_BE_LONG)) {
15119 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
15120 uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
15121 int32_t exit_point;
15122 const void *exit_addr;
15123
15124 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
15125 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
15126 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15127 if (!exit_addr) {
15128 return 0;
15129 }
15130 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
15131 ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD;
15132 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
15133 } else {
15134 ir_END_list(end_inputs);
15135 }
15136 }
15137 }
15138
15139 if (slow_inputs) {
15140 const void *func;
15141
15142 ir_MERGE_list(slow_inputs);
15143
15144 // JIT: zend_jit_pre_inc_obj_helper(zobj, name, CACHE_ADDR(opline->extended_value), result);
15145 switch (opline->opcode) {
15146 case ZEND_PRE_INC_OBJ:
15147 func = zend_jit_pre_inc_obj_helper;
15148 break;
15149 case ZEND_PRE_DEC_OBJ:
15150 func = zend_jit_pre_dec_obj_helper;
15151 break;
15152 case ZEND_POST_INC_OBJ:
15153 func = zend_jit_post_inc_obj_helper;
15154 break;
15155 case ZEND_POST_DEC_OBJ:
15156 func = zend_jit_post_dec_obj_helper;
15157 break;
15158 default:
15159 ZEND_UNREACHABLE();
15160 }
15161
15162 may_throw = 1;
15163 jit_SET_EX_OPLINE(jit, opline);
15164 ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
15165 ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(func),
15166 obj_ref,
15167 ir_CONST_ADDR(name),
15168 ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
15169 (opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
15170
15171 ir_END_list(end_inputs);
15172 }
15173
15174 if (end_inputs) {
15175 ir_MERGE_list(end_inputs);
15176 }
15177
15178 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this && !op1_indirect) {
15179 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
15180 may_throw = 1;
15181 }
15182 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
15183 }
15184
15185 if (may_throw) {
15186 zend_jit_check_exception(jit);
15187 }
15188
15189 return 1;
15190 }
15191
15192 static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info)
15193 {
15194 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
15195 const zend_op *next_opline = NULL;
15196 ir_refs *slow_inputs;
15197
15198 ir_refs_init(slow_inputs, 8);
15199
15200 if (trace) {
15201 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
15202 ZEND_ASSERT(trace->opline != NULL);
15203 next_opline = trace->opline;
15204 }
15205
15206 if (opline->op1_type == IS_CONST) {
15207 zval *zv = RT_CONSTANT(opline, opline->op1);
15208 zval *jump_zv = NULL;
15209 int b;
15210
15211 if (opline->opcode == ZEND_SWITCH_LONG) {
15212 if (Z_TYPE_P(zv) == IS_LONG) {
15213 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
15214 }
15215 } else if (opline->opcode == ZEND_SWITCH_STRING) {
15216 if (Z_TYPE_P(zv) == IS_STRING) {
15217 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
15218 }
15219 } else if (opline->opcode == ZEND_MATCH) {
15220 if (Z_TYPE_P(zv) == IS_LONG) {
15221 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
15222 } else if (Z_TYPE_P(zv) == IS_STRING) {
15223 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
15224 }
15225 } else {
15226 ZEND_UNREACHABLE();
15227 }
15228 if (next_opline) {
15229 const zend_op *target;
15230
15231 if (jump_zv != NULL) {
15232 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv));
15233 } else {
15234 target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
15235 }
15236 ZEND_ASSERT(target == next_opline);
15237 } else {
15238 if (jump_zv != NULL) {
15239 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
15240 } else {
15241 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
15242 }
15243 _zend_jit_add_predecessor_ref(jit, b, jit->b, ir_END());
15244 jit->b = -1;
15245 }
15246 } else {
15247 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[opline - op_array->opcodes] : NULL;
15248 uint32_t op1_info = OP1_INFO();
15249 zend_jit_addr op1_addr = OP1_ADDR();
15250 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
15251 const zend_op *target;
15252 int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes];
15253 int b;
15254 int32_t exit_point;
15255 const void *exit_addr;
15256 const void *default_label = NULL;
15257 zval *zv;
15258
15259 if (next_opline) {
15260 if (next_opline != default_opline) {
15261 exit_point = zend_jit_trace_get_exit_point(default_opline, 0);
15262 default_label = zend_jit_trace_get_exit_addr(exit_point);
15263 if (!default_label) {
15264 return 0;
15265 }
15266 }
15267 }
15268
15269 if (opline->opcode == ZEND_SWITCH_LONG) {
15270 if (op1_info & MAY_BE_LONG) {
15271 const void *fallback_label = NULL;
15272
15273 if (next_opline) {
15274 if (next_opline != opline + 1) {
15275 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
15276 fallback_label = zend_jit_trace_get_exit_addr(exit_point);
15277 if (!fallback_label) {
15278 return 0;
15279 }
15280 }
15281 }
15282 if (op1_info & MAY_BE_REF) {
15283 ir_ref ref, if_long, fast_path, ref2;
15284
15285 ref = jit_ZVAL_ADDR(jit, op1_addr);
15286 if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15287 ir_IF_TRUE(if_long);
15288 fast_path = ir_END();
15289 ir_IF_FALSE_cold(if_long);
15290
15291 // JIT: ZVAL_DEREF(op)
15292 if (fallback_label) {
15293 jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
15294 } else {
15295 ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
15296 ir_IF_FALSE_cold(if_ref);
15297 ir_refs_add(slow_inputs, ir_END());
15298 ir_IF_TRUE(if_ref);
15299 }
15300
15301 ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
15302 op1_addr = ZEND_ADDR_REF_ZVAL(ref2);
15303
15304 if (fallback_label) {
15305 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
15306 } else {
15307 if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15308 ir_IF_FALSE_cold(if_long);
15309 ir_refs_add(slow_inputs, ir_END());
15310 ir_IF_TRUE(if_long);
15311 }
15312
15313 ir_MERGE_2(fast_path, ir_END());
15314 ref = ir_PHI_2(IR_ADDR, ref, ref2);
15315 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
15316 } else if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
15317 if (fallback_label) {
15318 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
15319 } else {
15320 ir_ref if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15321 ir_IF_FALSE_cold(if_long);
15322 ir_refs_add(slow_inputs, ir_END());
15323 ir_IF_TRUE(if_long);
15324 }
15325 }
15326 ir_ref ref = jit_Z_LVAL(jit, op1_addr);
15327
15328 if (!HT_IS_PACKED(jumptable)) {
15329 ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_index_find),
15330 ir_CONST_ADDR(jumptable), ref);
15331 ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
15332 /* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
15333 if (sizeof(Bucket) == 32) {
15334 ref = ir_SHR_L(ref, ir_CONST_LONG(5));
15335 } else {
15336 ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
15337 }
15338 }
15339 ref = ir_SWITCH(ref);
15340
15341 if (next_opline) {
15342 ir_ref continue_list = IR_UNUSED;
15343
15344 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
15345 ir_ref idx;
15346 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
15347
15348 if (HT_IS_PACKED(jumptable)) {
15349 idx = ir_CONST_LONG(zv - jumptable->arPacked);
15350 } else {
15351 idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
15352 }
15353 ir_CASE_VAL(ref, idx);
15354 if (target == next_opline) {
15355 ir_END_list(continue_list);
15356 } else {
15357 exit_point = zend_jit_trace_get_exit_point(target, 0);
15358 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15359 if (!exit_addr) {
15360 return 0;
15361 }
15362 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
15363 }
15364 } ZEND_HASH_FOREACH_END();
15365
15366 ir_CASE_DEFAULT(ref);
15367 if (next_opline == default_opline) {
15368 ir_END_list(continue_list);
15369 } else {
15370 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
15371 }
15372 if (continue_list) {
15373 ir_MERGE_list(continue_list);
15374 } else {
15375 ZEND_ASSERT(slow_inputs->count);
15376 ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
15377 }
15378 } else {
15379 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
15380 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
15381 b = ssa->cfg.map[target - op_array->opcodes];
15382 _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
15383 } ZEND_HASH_FOREACH_END();
15384
15385 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
15386 if (slow_inputs->count) {
15387 ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
15388 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
15389 }
15390 jit->b = -1;
15391 }
15392 } else if (!next_opline) {
15393 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
15394 jit->b = -1;
15395 }
15396 } else if (opline->opcode == ZEND_SWITCH_STRING) {
15397 if (op1_info & MAY_BE_STRING) {
15398 const void *fallback_label = NULL;
15399
15400 if (next_opline) {
15401 if (next_opline != opline + 1) {
15402 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
15403 fallback_label = zend_jit_trace_get_exit_addr(exit_point);
15404 if (!fallback_label) {
15405 return 0;
15406 }
15407 }
15408 }
15409 if (op1_info & MAY_BE_REF) {
15410 ir_ref ref, if_string, fast_path, ref2;
15411
15412 ref = jit_ZVAL_ADDR(jit, op1_addr);
15413 if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15414 ir_IF_TRUE(if_string);
15415 fast_path = ir_END();
15416 ir_IF_FALSE_cold(if_string);
15417
15418 // JIT: ZVAL_DEREF(op)
15419 if (fallback_label) {
15420 jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
15421 } else {
15422 ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15423 ir_IF_FALSE_cold(if_ref);
15424 ir_refs_add(slow_inputs, ir_END());
15425 ir_IF_TRUE(if_ref);
15426 }
15427
15428 ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
15429 op1_addr = ZEND_ADDR_REF_ZVAL(ref2);
15430
15431 if (fallback_label) {
15432 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
15433 } else {
15434 if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15435 ir_IF_FALSE_cold(if_string);
15436 ir_refs_add(slow_inputs, ir_END());
15437 ir_IF_TRUE(if_string);
15438 }
15439
15440 ir_MERGE_2(fast_path, ir_END());
15441 ref = ir_PHI_2(IR_ADDR, ref, ref2);
15442 op1_addr = ZEND_ADDR_REF_ZVAL(ref);
15443 } else if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) {
15444 if (fallback_label) {
15445 jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, fallback_label);
15446 } else {
15447 ir_ref if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15448 ir_IF_FALSE_cold(if_string);
15449 ir_refs_add(slow_inputs, ir_END());
15450 ir_IF_TRUE(if_string);
15451 }
15452 }
15453
15454 ir_ref ref = jit_Z_PTR(jit, op1_addr);
15455 ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_find),
15456 ir_CONST_ADDR(jumptable), ref);
15457 ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
15458 /* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
15459 if (sizeof(Bucket) == 32) {
15460 ref = ir_SHR_L(ref, ir_CONST_LONG(5));
15461 } else {
15462 ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
15463 }
15464 ref = ir_SWITCH(ref);
15465
15466 if (next_opline) {
15467 ir_ref continue_list = IR_UNUSED;
15468
15469 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
15470 ir_ref idx;
15471 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
15472
15473 if (HT_IS_PACKED(jumptable)) {
15474 idx = ir_CONST_LONG(zv - jumptable->arPacked);
15475 } else {
15476 idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
15477 }
15478 ir_CASE_VAL(ref, idx);
15479 if (target == next_opline) {
15480 ir_END_list(continue_list);
15481 } else {
15482 exit_point = zend_jit_trace_get_exit_point(target, 0);
15483 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15484 if (!exit_addr) {
15485 return 0;
15486 }
15487 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
15488 }
15489 } ZEND_HASH_FOREACH_END();
15490
15491 ir_CASE_DEFAULT(ref);
15492 if (next_opline == default_opline) {
15493 ir_END_list(continue_list);
15494 } else {
15495 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
15496 }
15497 if (continue_list) {
15498 ir_MERGE_list(continue_list);
15499 } else {
15500 ZEND_ASSERT(slow_inputs->count);
15501 ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
15502 }
15503 } else {
15504 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
15505 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
15506 b = ssa->cfg.map[target - op_array->opcodes];
15507 _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
15508 } ZEND_HASH_FOREACH_END();
15509 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
15510 if (slow_inputs->count) {
15511 ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
15512 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
15513 }
15514 jit->b = -1;
15515 }
15516 } else if (!next_opline) {
15517 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
15518 jit->b = -1;
15519 }
15520 } else if (opline->opcode == ZEND_MATCH) {
15521 ir_ref if_type = IR_UNUSED, default_input_list = IR_UNUSED, ref = IR_UNUSED;
15522 ir_ref continue_list = IR_UNUSED;
15523
15524 if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) {
15525 ir_ref long_path = IR_UNUSED;
15526
15527 if (op1_info & MAY_BE_REF) {
15528 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
15529 }
15530 if (op1_info & MAY_BE_LONG) {
15531 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
15532 if (op1_info & (MAY_BE_STRING|MAY_BE_UNDEF)) {
15533 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15534 ir_IF_TRUE(if_type);
15535 } else if (default_label) {
15536 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, default_label);
15537 } else if (next_opline) {
15538 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15539 ir_IF_FALSE(if_type);
15540 ir_END_list(continue_list);
15541 ir_IF_TRUE(if_type);
15542 } else {
15543 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15544 ir_IF_FALSE(if_type);
15545 ir_END_list(default_input_list);
15546 ir_IF_TRUE(if_type);
15547 }
15548 }
15549 ref = jit_Z_LVAL(jit, op1_addr);
15550 ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_index_find),
15551 ir_CONST_ADDR(jumptable), ref);
15552 if (op1_info & MAY_BE_STRING) {
15553 long_path = ir_END();
15554 }
15555 }
15556 if (op1_info & MAY_BE_STRING) {
15557 if (if_type) {
15558 ir_IF_FALSE(if_type);
15559 if_type = IS_UNUSED;
15560 }
15561 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) {
15562 if (op1_info & MAY_BE_UNDEF) {
15563 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15564 ir_IF_TRUE(if_type);
15565 } else if (default_label) {
15566 jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, default_label);
15567 } else if (next_opline) {
15568 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15569 ir_IF_FALSE(if_type);
15570 ir_END_list(continue_list);
15571 ir_IF_TRUE(if_type);
15572 } else {
15573 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
15574 ir_IF_FALSE(if_type);
15575 ir_END_list(default_input_list);
15576 ir_IF_TRUE(if_type);
15577 }
15578 }
15579 ir_ref ref2 = jit_Z_PTR(jit, op1_addr);
15580 ref2 = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_find),
15581 ir_CONST_ADDR(jumptable), ref2);
15582 if (op1_info & MAY_BE_LONG) {
15583 ir_MERGE_WITH(long_path);
15584 ref = ir_PHI_2(IR_LONG, ref2, ref);
15585 } else {
15586 ref = ref2;
15587 }
15588 }
15589
15590 ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
15591 /* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
15592 if (HT_IS_PACKED(jumptable)) {
15593 ZEND_ASSERT(sizeof(zval) == 16);
15594 ref = ir_SHR_L(ref, ir_CONST_LONG(4));
15595 } else {
15596 if (sizeof(Bucket) == 32) {
15597 ref = ir_SHR_L(ref, ir_CONST_LONG(5));
15598 } else {
15599 ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
15600 }
15601 }
15602 ref = ir_SWITCH(ref);
15603
15604 if (next_opline) {
15605 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
15606 ir_ref idx;
15607 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
15608
15609 if (HT_IS_PACKED(jumptable)) {
15610 idx = ir_CONST_LONG(zv - jumptable->arPacked);
15611 } else {
15612 idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
15613 }
15614 ir_CASE_VAL(ref, idx);
15615 if (target == next_opline) {
15616 ir_END_list(continue_list);
15617 } else {
15618 exit_point = zend_jit_trace_get_exit_point(target, 0);
15619 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15620 if (!exit_addr) {
15621 return 0;
15622 }
15623 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
15624 }
15625 } ZEND_HASH_FOREACH_END();
15626
15627 ir_CASE_DEFAULT(ref);
15628 if (next_opline == default_opline) {
15629 ir_END_list(continue_list);
15630 } else {
15631 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
15632 }
15633 } else {
15634 ZEND_HASH_FOREACH_VAL(jumptable, zv) {
15635 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
15636 b = ssa->cfg.map[target - op_array->opcodes];
15637 _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
15638 } ZEND_HASH_FOREACH_END();
15639 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
15640 }
15641 } else if (!(op1_info & MAY_BE_UNDEF)) {
15642 if (next_opline) {
15643 if (next_opline == default_opline) {
15644 ir_END_list(continue_list);
15645 } else {
15646 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
15647 }
15648 } else {
15649 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
15650 }
15651 }
15652
15653 if (op1_info & MAY_BE_UNDEF) {
15654 if (if_type) {
15655 ir_IF_FALSE(if_type);
15656 if_type = IS_UNUSED;
15657 }
15658 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) {
15659 if (default_label) {
15660 jit_guard_Z_TYPE(jit, op1_addr, IS_UNDEF, default_label);
15661 } else if (next_opline) {
15662 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
15663 ir_IF_TRUE(if_def);
15664 ir_END_list(continue_list);
15665 ir_IF_FALSE_cold(if_def);
15666 } else {
15667 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
15668 ir_IF_TRUE(if_def);
15669 ir_END_list(default_input_list);
15670 ir_IF_FALSE_cold(if_def);
15671 }
15672 }
15673
15674 jit_SET_EX_OPLINE(jit, opline);
15675 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
15676 ir_CONST_U32(opline->op1.var));
15677 zend_jit_check_exception_undef_result(jit, opline);
15678 if (default_label) {
15679 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
15680 } else if (next_opline) {
15681 ir_END_list(continue_list);
15682 } else {
15683 ir_END_list(default_input_list);
15684 }
15685 }
15686 if (next_opline) {
15687 ZEND_ASSERT(continue_list);
15688 ir_MERGE_list(continue_list);
15689 } else {
15690 if (default_input_list) {
15691 if (jit->ctx.ir_base[ref].op == IR_SWITCH) {
15692 ZEND_ASSERT(jit->ctx.ir_base[ref].op3 == IR_UNUSED);
15693 jit->ctx.ir_base[ref].op3 = default_input_list;
15694 } else {
15695 ir_MERGE_list(default_input_list);
15696 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
15697 }
15698 }
15699 jit->b = -1;
15700 }
15701 } else {
15702 ZEND_UNREACHABLE();
15703 }
15704 }
15705 return 1;
15706 }
15707
15708 static int zend_jit_start(zend_jit_ctx *jit, const zend_op_array *op_array, zend_ssa *ssa)
15709 {
15710 int i, count;
15711 zend_basic_block *bb;
15712
15713 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : (IR_START_BR_TARGET|IR_ENTRY_BR_TARGET));
15714
15715 jit->ctx.spill_base = ZREG_FP;
15716
15717 jit->op_array = jit->current_op_array = op_array;
15718 jit->ssa = ssa;
15719 jit->bb_start_ref = zend_arena_calloc(&CG(arena), ssa->cfg.blocks_count * 2, sizeof(ir_ref));
15720 jit->bb_predecessors = jit->bb_start_ref + ssa->cfg.blocks_count;
15721
15722 count = 0;
15723 for (i = 0, bb = ssa->cfg.blocks; i < ssa->cfg.blocks_count; i++, bb++) {
15724 jit->bb_predecessors[i] = count;
15725 count += bb->predecessors_count;
15726 }
15727 jit->bb_edges = zend_arena_calloc(&CG(arena), count, sizeof(ir_ref));
15728
15729 if (!GCC_GLOBAL_REGS) {
15730 ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
15731 jit_STORE_FP(jit, ref);
15732 jit->ctx.flags |= IR_FASTCALL_FUNC;
15733 }
15734
15735 return 1;
15736 }
15737
15738 static void *zend_jit_finish(zend_jit_ctx *jit)
15739 {
15740 void *entry;
15741 size_t size;
15742 zend_string *str = NULL;
15743
15744 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP|
15745 ZEND_JIT_DEBUG_IR_SRC|ZEND_JIT_DEBUG_IR_AFTER_SCCP|ZEND_JIT_DEBUG_IR_AFTER_SCCP|
15746 ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE|ZEND_JIT_DEBUG_IR_AFTER_REGS|ZEND_JIT_DEBUG_IR_FINAL|ZEND_JIT_DEBUG_IR_CODEGEN)) {
15747 if (jit->name) {
15748 str = zend_string_copy(jit->name);
15749 } else {
15750 str = zend_jit_func_name(jit->op_array);
15751 }
15752 }
15753
15754 if (jit->op_array) {
15755 /* Only for function JIT */
15756 _zend_jit_fix_merges(jit);
15757 #if defined(IR_TARGET_AARCH64)
15758 } else if (jit->trace) {
15759 jit->ctx.deoptimization_exits = jit->trace->exit_count;
15760 jit->ctx.get_exit_addr = zend_jit_trace_get_exit_addr;
15761 #endif
15762 } else {
15763 #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
15764 jit->ctx.flags |= IR_GEN_CACHE_DEMOTE;
15765 #endif
15766 }
15767
15768 entry = zend_jit_ir_compile(&jit->ctx, &size, str ? ZSTR_VAL(str) : NULL);
15769 if (entry) {
15770 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
15771 #if HAVE_CAPSTONE
15772 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
15773 if (str) {
15774 ir_disasm_add_symbol(ZSTR_VAL(str), (uintptr_t)entry, size);
15775 }
15776 ir_disasm(str ? ZSTR_VAL(str) : "unknown",
15777 entry, size,
15778 (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0,
15779 &jit->ctx, stderr);
15780 }
15781 #endif
15782 #ifndef _WIN32
15783 if (str) {
15784 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
15785 uintptr_t sp_offset = 0;
15786
15787 // ir_mem_unprotect(entry, size);
15788 if (!(jit->ctx.flags & IR_FUNCTION)
15789 && zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
15790 #if !ZEND_WIN32 && !defined(IR_TARGET_AARCH64)
15791 sp_offset = zend_jit_hybrid_vm_sp_adj;
15792 #else
15793 sp_offset = sizeof(void*);
15794 #endif
15795 } else {
15796 sp_offset = sizeof(void*);
15797 }
15798 ir_gdb_register(ZSTR_VAL(str), entry, size, sp_offset, 0);
15799 // ir_mem_protect(entry, size);
15800 }
15801
15802 if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
15803 ir_perf_map_register(ZSTR_VAL(str), entry, size);
15804 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
15805 ir_perf_jitdump_register(ZSTR_VAL(str), entry, size);
15806 }
15807 }
15808 }
15809 #endif
15810 }
15811
15812 if (jit->op_array) {
15813 /* Only for function JIT */
15814 const zend_op_array *op_array = jit->op_array;
15815 zend_op *opline = (zend_op*)op_array->opcodes;
15816
15817 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
15818 while (opline->opcode == ZEND_RECV) {
15819 opline++;
15820 }
15821 }
15822 opline->handler = entry;
15823
15824 if (jit->ctx.entries_count) {
15825 /* For all entries */
15826 int i = jit->ctx.entries_count;
15827 do {
15828 ir_insn *insn = &jit->ctx.ir_base[jit->ctx.entries[--i]];
15829 op_array->opcodes[insn->op2].handler = (char*)entry + insn->op3;
15830 } while (i != 0);
15831 }
15832 } else {
15833 /* Only for tracing JIT */
15834 zend_jit_trace_info *t = jit->trace;
15835 zend_jit_trace_stack *stack;
15836 uint32_t i;
15837
15838 if (t) {
15839 for (i = 0; i < t->stack_map_size; i++) {
15840 stack = t->stack_map + i;
15841 if (stack->flags & ZREG_SPILL_SLOT) {
15842 stack->reg = (jit->ctx.flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP;
15843 stack->ref = ir_get_spill_slot_offset(&jit->ctx, stack->ref);
15844 }
15845 }
15846 }
15847
15848 zend_jit_trace_add_code(entry, size);
15849 }
15850 }
15851
15852 if (str) {
15853 zend_string_release(str);
15854 }
15855
15856 return entry;
15857 }
15858
15859 static const void *zend_jit_trace_allocate_exit_group(uint32_t n)
15860 {
15861 const void *entry;
15862 size_t size;
15863 ir_code_buffer code_buffer;
15864
15865 code_buffer.start = dasm_buf;
15866 code_buffer.end = dasm_end;
15867 code_buffer.pos = *dasm_ptr;
15868
15869 entry = ir_emit_exitgroup(n, ZEND_JIT_EXIT_POINTS_PER_GROUP, zend_jit_stub_handlers[jit_stub_trace_exit],
15870 &code_buffer, &size);
15871
15872 *dasm_ptr = code_buffer.pos;
15873
15874 if (entry) {
15875 #ifdef HAVE_CAPSTONE
15876 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
15877 uint32_t i;
15878 char name[32];
15879
15880 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
15881 sprintf(name, "jit$$trace_exit_%d", n + i);
15882 ir_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING);
15883 }
15884 }
15885 #endif
15886 }
15887
15888 return entry;
15889 }
15890
15891 static int zend_jit_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint8_t type)
15892 {
15893 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
15894 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15895 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
15896
15897 if (!exit_addr) {
15898 return 0;
15899 }
15900 ir_GUARD(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr));
15901
15902 return 1;
15903 }
15904
15905 static int zend_jit_scalar_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var)
15906 {
15907 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
15908 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15909 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
15910
15911 if (!exit_addr) {
15912 return 0;
15913 }
15914 ir_GUARD(ir_LT(jit_Z_TYPE(jit, addr), ir_CONST_U8(IS_STRING)), ir_CONST_ADDR(exit_addr));
15915
15916 return 1;
15917 }
15918
15919 static bool zend_jit_noref_guard(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_addr)
15920 {
15921 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
15922 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15923
15924 if (!exit_addr) {
15925 return 0;
15926 }
15927 ir_GUARD(ir_NE(jit_Z_TYPE(jit, var_addr), ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
15928
15929 return 1;
15930 }
15931
15932 static int zend_jit_trace_opline_guard(zend_jit_ctx *jit, const zend_op *opline)
15933 {
15934 uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0);
15935 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15936
15937 if (!exit_addr) {
15938 return 0;
15939 }
15940
15941 ir_GUARD(jit_CMP_IP(jit, IR_EQ, opline), ir_CONST_ADDR(exit_addr));
15942 zend_jit_set_last_valid_opline(jit, opline);
15943
15944 return 1;
15945 }
15946
15947 static bool zend_jit_guard_reference(zend_jit_ctx *jit,
15948 const zend_op *opline,
15949 zend_jit_addr *var_addr_ptr,
15950 zend_jit_addr *ref_addr_ptr,
15951 bool add_ref_guard)
15952 {
15953 zend_jit_addr var_addr = *var_addr_ptr;
15954 const void *exit_addr = NULL;
15955 ir_ref ref;
15956
15957 if (add_ref_guard) {
15958 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
15959
15960 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15961 if (!exit_addr) {
15962 return 0;
15963 }
15964
15965 ref = jit_Z_TYPE(jit, var_addr);
15966 ir_GUARD(ir_EQ(ref, ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
15967 }
15968
15969 ref = jit_Z_PTR(jit, var_addr);
15970 *ref_addr_ptr = ZEND_ADDR_REF_ZVAL(ref);
15971 ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
15972 var_addr = ZEND_ADDR_REF_ZVAL(ref);
15973 *var_addr_ptr = var_addr;
15974
15975 return 1;
15976 }
15977
15978 static bool zend_jit_fetch_reference(zend_jit_ctx *jit,
15979 const zend_op *opline,
15980 uint8_t var_type,
15981 uint32_t *var_info_ptr,
15982 zend_jit_addr *var_addr_ptr,
15983 bool add_ref_guard,
15984 bool add_type_guard)
15985 {
15986 zend_jit_addr var_addr = *var_addr_ptr;
15987 uint32_t var_info = *var_info_ptr;
15988 const void *exit_addr = NULL;
15989 ir_ref ref;
15990
15991 if (add_ref_guard || add_type_guard) {
15992 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
15993
15994 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
15995 if (!exit_addr) {
15996 return 0;
15997 }
15998 }
15999
16000 if (add_ref_guard) {
16001 ref = jit_Z_TYPE(jit, var_addr);
16002 ir_GUARD(ir_EQ(ref, ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
16003 }
16004 if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) {
16005 /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */
16006 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper),
16007 jit_ZVAL_ADDR(jit, var_addr));
16008 *var_addr_ptr = var_addr;
16009 } else {
16010 ref = jit_Z_PTR(jit, var_addr);
16011 ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
16012 var_addr = ZEND_ADDR_REF_ZVAL(ref);
16013 *var_addr_ptr = var_addr;
16014 }
16015
16016 if (var_type != IS_UNKNOWN) {
16017 var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED);
16018 }
16019 if (add_type_guard
16020 && var_type != IS_UNKNOWN
16021 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
16022 ref = jit_Z_TYPE(jit, var_addr);
16023 ir_GUARD(ir_EQ(ref, ir_CONST_U8(var_type)), ir_CONST_ADDR(exit_addr));
16024
16025 ZEND_ASSERT(var_info & (1 << var_type));
16026 if (var_type < IS_STRING) {
16027 var_info = (1 << var_type);
16028 } else if (var_type != IS_ARRAY) {
16029 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
16030 } else {
16031 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
16032 }
16033
16034 *var_info_ptr = var_info;
16035 } else {
16036 var_info &= ~MAY_BE_REF;
16037 *var_info_ptr = var_info;
16038 }
16039 *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */
16040
16041 return 1;
16042 }
16043
16044 static bool zend_jit_fetch_indirect_var(zend_jit_ctx *jit, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard)
16045 {
16046 zend_jit_addr var_addr = *var_addr_ptr;
16047 uint32_t var_info = *var_info_ptr;
16048 int32_t exit_point;
16049 const void *exit_addr;
16050 ir_ref ref = IR_UNUSED;
16051
16052 if (add_indirect_guard) {
16053 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
16054 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
16055
16056 if (!exit_addr) {
16057 return 0;
16058 }
16059 jit_guard_Z_TYPE(jit, var_addr, IS_INDIRECT, exit_addr);
16060 ref = jit_Z_PTR(jit, var_addr);
16061 } else {
16062 /* This LOAD of INDIRECT VAR, stored by the previous FETCH_(DIM/OBJ)_W,
16063 * is eliminated by store forwarding (S2L) */
16064 ref = jit_Z_PTR(jit, var_addr);
16065 }
16066 *var_info_ptr &= ~MAY_BE_INDIRECT;
16067 var_addr = ZEND_ADDR_REF_ZVAL(ref);
16068 *var_addr_ptr = var_addr;
16069
16070 if (var_type != IS_UNKNOWN) {
16071 var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED);
16072 }
16073 if (!(var_type & IS_TRACE_REFERENCE)
16074 && var_type != IS_UNKNOWN
16075 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
16076 exit_point = zend_jit_trace_get_exit_point(opline, 0);
16077 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
16078
16079 if (!exit_addr) {
16080 return 0;
16081 }
16082
16083 jit_guard_Z_TYPE(jit, var_addr, var_type, exit_addr);
16084
16085 //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info);
16086 ZEND_ASSERT(var_info & (1 << var_type));
16087 if (var_type < IS_STRING) {
16088 var_info = (1 << var_type);
16089 } else if (var_type != IS_ARRAY) {
16090 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
16091 } else {
16092 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
16093 }
16094
16095 *var_info_ptr = var_info;
16096 }
16097
16098 return 1;
16099 }
16100
16101 static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace)
16102 {
16103 zend_jit_op_array_trace_extension *jit_extension =
16104 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
16105 size_t offset = jit_extension->offset;
16106 const void *handler =
16107 (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
16108 ir_ref ref;
16109
16110 zend_jit_set_ip(jit, opline);
16111 if (GCC_GLOBAL_REGS) {
16112 ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
16113 } else {
16114 ref = jit_FP(jit);
16115 ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
16116 }
16117 if (may_throw
16118 && opline->opcode != ZEND_RETURN
16119 && opline->opcode != ZEND_RETURN_BY_REF) {
16120 zend_jit_check_exception(jit);
16121 }
16122
16123 while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) {
16124 trace++;
16125 }
16126
16127 if (!GCC_GLOBAL_REGS
16128 && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) {
16129 if (opline->opcode == ZEND_RETURN ||
16130 opline->opcode == ZEND_RETURN_BY_REF ||
16131 opline->opcode == ZEND_DO_UCALL ||
16132 opline->opcode == ZEND_DO_FCALL_BY_NAME ||
16133 opline->opcode == ZEND_DO_FCALL ||
16134 opline->opcode == ZEND_GENERATOR_CREATE) {
16135
16136 ir_ref addr = jit_EG(current_execute_data);
16137
16138 jit_STORE_FP(jit, ir_LOAD_A(addr));
16139 }
16140 }
16141
16142 if (zend_jit_trace_may_exit(op_array, opline)) {
16143 if (opline->opcode == ZEND_RETURN ||
16144 opline->opcode == ZEND_RETURN_BY_REF ||
16145 opline->opcode == ZEND_GENERATOR_CREATE) {
16146
16147 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
16148 if (trace->op != ZEND_JIT_TRACE_END ||
16149 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
16150 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
16151 /* this check may be handled by the following OPLINE guard or jmp [IP] */
16152 ir_GUARD(ir_NE(jit_IP(jit), ir_CONST_ADDR(zend_jit_halt_op)),
16153 jit_STUB_ADDR(jit, jit_stub_trace_halt));
16154 }
16155 } else if (GCC_GLOBAL_REGS) {
16156 ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
16157 } else {
16158 ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
16159 }
16160 } else if (opline->opcode == ZEND_EXIT ||
16161 opline->opcode == ZEND_GENERATOR_RETURN ||
16162 opline->opcode == ZEND_YIELD ||
16163 opline->opcode == ZEND_YIELD_FROM) {
16164 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_halt));
16165 ir_BEGIN(IR_UNUSED); /* unreachable block */
16166 }
16167 if (trace->op != ZEND_JIT_TRACE_END ||
16168 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
16169 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
16170
16171 const zend_op *next_opline = trace->opline;
16172 const zend_op *exit_opline = NULL;
16173 uint32_t exit_point;
16174 const void *exit_addr;
16175 uint32_t old_info = 0;
16176 uint32_t old_res_info = 0;
16177 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
16178
16179 if (zend_is_smart_branch(opline)) {
16180 bool exit_if_true = 0;
16181 exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true);
16182 } else {
16183 switch (opline->opcode) {
16184 case ZEND_JMPZ:
16185 case ZEND_JMPNZ:
16186 case ZEND_JMPZ_EX:
16187 case ZEND_JMPNZ_EX:
16188 case ZEND_JMP_SET:
16189 case ZEND_COALESCE:
16190 case ZEND_JMP_NULL:
16191 case ZEND_FE_RESET_R:
16192 case ZEND_FE_RESET_RW:
16193 exit_opline = (trace->opline == opline + 1) ?
16194 OP_JMP_ADDR(opline, opline->op2) :
16195 opline + 1;
16196 break;
16197 case ZEND_FE_FETCH_R:
16198 case ZEND_FE_FETCH_RW:
16199 exit_opline = (trace->opline == opline + 1) ?
16200 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
16201 opline + 1;
16202 break;
16203
16204 }
16205 }
16206
16207 switch (opline->opcode) {
16208 case ZEND_FE_FETCH_R:
16209 case ZEND_FE_FETCH_RW:
16210 if (opline->op2_type != IS_UNUSED) {
16211 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var));
16212 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1);
16213 }
16214 break;
16215 case ZEND_BIND_INIT_STATIC_OR_JMP:
16216 if (opline->op1_type == IS_CV) {
16217 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
16218 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1);
16219 }
16220 break;
16221 }
16222 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
16223 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
16224 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
16225 }
16226 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
16227 exit_addr = zend_jit_trace_get_exit_addr(exit_point);
16228
16229 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
16230 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
16231 }
16232 switch (opline->opcode) {
16233 case ZEND_FE_FETCH_R:
16234 case ZEND_FE_FETCH_RW:
16235 if (opline->op2_type != IS_UNUSED) {
16236 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info);
16237 }
16238 break;
16239 case ZEND_BIND_INIT_STATIC_OR_JMP:
16240 if (opline->op1_type == IS_CV) {
16241 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
16242 }
16243 break;
16244 }
16245
16246 if (!exit_addr) {
16247 return 0;
16248 }
16249 ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), ir_CONST_ADDR(exit_addr));
16250 }
16251 }
16252
16253 zend_jit_set_last_valid_opline(jit, trace->opline);
16254
16255 return 1;
16256 }
16257
16258 static int zend_jit_deoptimizer_start(zend_jit_ctx *jit,
16259 zend_string *name,
16260 uint32_t trace_num,
16261 uint32_t exit_num)
16262 {
16263 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
16264
16265 jit->ctx.spill_base = ZREG_FP;
16266
16267 jit->op_array = NULL;
16268 jit->ssa = NULL;
16269 jit->name = zend_string_copy(name);
16270
16271 jit->ctx.flags |= IR_SKIP_PROLOGUE;
16272
16273 return 1;
16274 }
16275
16276 static int zend_jit_trace_start(zend_jit_ctx *jit,
16277 const zend_op_array *op_array,
16278 zend_ssa *ssa,
16279 zend_string *name,
16280 uint32_t trace_num,
16281 zend_jit_trace_info *parent,
16282 uint32_t exit_num)
16283 {
16284 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
16285
16286 jit->ctx.spill_base = ZREG_FP;
16287
16288 jit->op_array = NULL;
16289 jit->current_op_array = op_array;
16290 jit->ssa = ssa;
16291 jit->name = zend_string_copy(name);
16292
16293 if (!GCC_GLOBAL_REGS) {
16294 if (!parent) {
16295 ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
16296 jit_STORE_FP(jit, ref);
16297 jit->ctx.flags |= IR_FASTCALL_FUNC;
16298 }
16299 }
16300
16301 if (parent) {
16302 jit->ctx.flags |= IR_SKIP_PROLOGUE;
16303 }
16304
16305 if (parent) {
16306 int i;
16307 int parent_vars_count = parent->exit_info[exit_num].stack_size;
16308 zend_jit_trace_stack *parent_stack = parent_vars_count == 0 ? NULL :
16309 parent->stack_map +
16310 parent->exit_info[exit_num].stack_offset;
16311
16312 /* prevent clobbering of registers used for deoptimization */
16313 for (i = 0; i < parent_vars_count; i++) {
16314 if (STACK_FLAGS(parent_stack, i) != ZREG_CONST
16315 && STACK_REG(parent_stack, i) != ZREG_NONE) {
16316 int32_t reg = STACK_REG(parent_stack, i);
16317 ir_type type;
16318
16319 if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_COPY) {
16320 type = IR_ADDR;
16321 } else if (STACK_TYPE(parent_stack, i) == IS_LONG) {
16322 type = IR_LONG;
16323 } else if (STACK_TYPE(parent_stack, i) == IS_DOUBLE) {
16324 type = IR_DOUBLE;
16325 } else {
16326 ZEND_UNREACHABLE();
16327 }
16328 if (ssa && ssa->vars[i].no_val) {
16329 /* pass */
16330 } else {
16331 ir_ref ref = ir_RLOAD(type, reg);
16332
16333 if (STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) {
16334 /* op3 is used as a flag that the value is already stored in memory.
16335 * In case the IR framework decides to spill the result of IR_LOAD,
16336 * it doesn't have to store the value once again.
16337 *
16338 * See: insn->op3 check in ir_emit_rload()
16339 */
16340 ir_set_op(&jit->ctx, ref, 3, EX_NUM_TO_VAR(i));
16341 }
16342 }
16343 }
16344 }
16345 }
16346
16347 if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
16348 ZEND_ASSERT(parent->exit_info[exit_num].poly_func_reg >= 0 && parent->exit_info[exit_num].poly_this_reg >= 0);
16349 ir_RLOAD_A(parent->exit_info[exit_num].poly_func_reg);
16350 ir_RLOAD_A(parent->exit_info[exit_num].poly_this_reg);
16351 }
16352
16353 ir_STORE(jit_EG(jit_trace_num), ir_CONST_U32(trace_num));
16354
16355 return 1;
16356 }
16357
16358 static int zend_jit_trace_begin_loop(zend_jit_ctx *jit)
16359 {
16360 return ir_LOOP_BEGIN(ir_END());
16361 }
16362
16363 static void zend_jit_trace_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi)
16364 {
16365 int dst_var = phi->ssa_var;
16366 int src_var = phi->sources[0];
16367 ir_ref ref;
16368
16369 ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
16370 ZEND_ASSERT(jit->ra[src_var].ref != IR_UNUSED && jit->ra[src_var].ref != IR_NULL);
16371
16372 ref = ir_PHI_2(
16373 (jit->ssa->var_info[src_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE,
16374 zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)), IR_UNUSED);
16375
16376 src_var = phi->sources[1];
16377 ZEND_ASSERT(jit->ra[src_var].ref == IR_NULL);
16378 jit->ra[src_var].flags |= ZREG_FORWARD;
16379
16380 zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref);
16381 }
16382
16383 static int zend_jit_trace_end_loop(zend_jit_ctx *jit, int loop_ref, const void *timeout_exit_addr)
16384 {
16385 if (timeout_exit_addr) {
16386 zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
16387 }
16388 ZEND_ASSERT(jit->ctx.ir_base[loop_ref].op2 == IR_UNUSED);
16389 ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
16390 return 1;
16391 }
16392
16393 static int zend_jit_trace_return(zend_jit_ctx *jit, bool original_handler, const zend_op *opline)
16394 {
16395 if (GCC_GLOBAL_REGS) {
16396 if (!original_handler) {
16397 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
16398 } else {
16399 ir_TAILCALL(IR_VOID, zend_jit_orig_opline_handler(jit));
16400 }
16401 } else {
16402 if (original_handler) {
16403 ir_ref ref;
16404 ir_ref addr = zend_jit_orig_opline_handler(jit);
16405
16406 #if defined(IR_TARGET_X86)
16407 addr = ir_CAST_FC_FUNC(addr);
16408 #endif
16409 ref = ir_CALL_1(IR_I32, addr, jit_FP(jit));
16410 if (opline &&
16411 (opline->opcode == ZEND_RETURN
16412 || opline->opcode == ZEND_RETURN_BY_REF
16413 || opline->opcode == ZEND_GENERATOR_RETURN
16414 || opline->opcode == ZEND_GENERATOR_CREATE
16415 || opline->opcode == ZEND_YIELD
16416 || opline->opcode == ZEND_YIELD_FROM)) {
16417 ir_RETURN(ref);
16418 return 1;
16419 }
16420 }
16421 ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
16422 }
16423 return 1;
16424 }
16425
16426 static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr)
16427 {
16428 return ir_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
16429 }
16430
16431 static int zend_jit_trace_link_to_root(zend_jit_ctx *jit, zend_jit_trace_info *t, const void *timeout_exit_addr)
16432 {
16433 const void *link_addr;
16434
16435 /* Skip prologue. */
16436 ZEND_ASSERT(zend_jit_trace_prologue_size != (size_t)-1);
16437 link_addr = (const void*)((const char*)t->code_start + zend_jit_trace_prologue_size);
16438
16439 if (timeout_exit_addr) {
16440 zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
16441 }
16442 ir_IJMP(ir_CONST_ADDR(link_addr));
16443
16444 return 1;
16445 }
16446
16447 static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace)
16448 {
16449 uint32_t op1_info, op2_info;
16450
16451 switch (opline->opcode) {
16452 case ZEND_SEND_VAR:
16453 case ZEND_SEND_VAL:
16454 case ZEND_SEND_VAL_EX:
16455 return (opline->op2_type != IS_CONST) && (opline->opcode != ZEND_SEND_VAL_EX || opline->op2.num <= MAX_ARG_FLAG_NUM);
16456 case ZEND_QM_ASSIGN:
16457 case ZEND_IS_SMALLER:
16458 case ZEND_IS_SMALLER_OR_EQUAL:
16459 case ZEND_IS_EQUAL:
16460 case ZEND_IS_NOT_EQUAL:
16461 case ZEND_IS_IDENTICAL:
16462 case ZEND_IS_NOT_IDENTICAL:
16463 case ZEND_CASE:
16464 return 1;
16465 case ZEND_RETURN:
16466 return (op_array->type != ZEND_EVAL_CODE && op_array->function_name);
16467 case ZEND_ASSIGN:
16468 return (opline->op1_type == IS_CV);
16469 case ZEND_ADD:
16470 case ZEND_SUB:
16471 case ZEND_MUL:
16472 op1_info = OP1_INFO();
16473 op2_info = OP2_INFO();
16474 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
16475 return 0;
16476 }
16477 if (trace && trace->op1_type != IS_UNKNOWN) {
16478 op1_info &= 1U << (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
16479 }
16480 if (trace && trace->op2_type != IS_UNKNOWN) {
16481 op2_info &= 1U << (trace->op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
16482 }
16483 return !(op1_info & MAY_BE_UNDEF)
16484 && !(op2_info & MAY_BE_UNDEF)
16485 && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
16486 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
16487 case ZEND_BW_OR:
16488 case ZEND_BW_AND:
16489 case ZEND_BW_XOR:
16490 case ZEND_SL:
16491 case ZEND_SR:
16492 case ZEND_MOD:
16493 op1_info = OP1_INFO();
16494 op2_info = OP2_INFO();
16495 if (trace && trace->op1_type != IS_UNKNOWN) {
16496 op1_info &= 1U << (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
16497 }
16498 if (trace && trace->op2_type != IS_UNKNOWN) {
16499 op2_info &= 1U << (trace->op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
16500 }
16501 return (op1_info & MAY_BE_LONG)
16502 && (op2_info & MAY_BE_LONG);
16503 case ZEND_PRE_INC:
16504 case ZEND_PRE_DEC:
16505 case ZEND_POST_INC:
16506 case ZEND_POST_DEC:
16507 op1_info = OP1_INFO();
16508 return opline->op1_type == IS_CV
16509 && (op1_info & MAY_BE_LONG)
16510 && !(op1_info & MAY_BE_REF);
16511 case ZEND_STRLEN:
16512 op1_info = OP1_INFO();
16513 return (opline->op1_type & (IS_CV|IS_CONST))
16514 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING;
16515 case ZEND_COUNT:
16516 op1_info = OP1_INFO();
16517 return (opline->op1_type & (IS_CV|IS_CONST))
16518 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY;
16519 case ZEND_JMPZ:
16520 case ZEND_JMPNZ:
16521 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
16522 if (!ssa->cfg.map) {
16523 return 0;
16524 }
16525 if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start &&
16526 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
16527 return 0;
16528 }
16529 }
16530 ZEND_FALLTHROUGH;
16531 case ZEND_BOOL:
16532 case ZEND_BOOL_NOT:
16533 case ZEND_JMPZ_EX:
16534 case ZEND_JMPNZ_EX:
16535 return 1;
16536 case ZEND_FETCH_CONSTANT:
16537 return 1;
16538 case ZEND_FETCH_DIM_R:
16539 op1_info = OP1_INFO();
16540 op2_info = OP2_INFO();
16541 if (trace
16542 && trace->op1_type != IS_UNKNOWN
16543 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) {
16544 op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY);
16545 }
16546 return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) &&
16547 (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) &&
16548 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) ||
16549 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) &&
16550 (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1))));
16551 }
16552 return 0;
16553 }
16554
16555 static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var)
16556 {
16557 if (ssa->vars[var].no_val) {
16558 /* we don't need the value */
16559 return 0;
16560 }
16561
16562 if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) {
16563 /* Disable global register allocation,
16564 * register allocation for SSA variables connected through Phi functions
16565 */
16566 if (ssa->vars[var].definition_phi) {
16567 return 0;
16568 }
16569 if (ssa->vars[var].phi_use_chain) {
16570 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
16571 do {
16572 if (!ssa->vars[phi->ssa_var].no_val) {
16573 return 0;
16574 }
16575 phi = zend_ssa_next_use_phi(ssa, var, phi);
16576 } while (phi);
16577 }
16578 }
16579
16580 if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) &&
16581 ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) {
16582 /* bad type */
16583 return 0;
16584 }
16585
16586 return 1;
16587 }
16588
16589 static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var)
16590 {
16591 if (!zend_jit_var_supports_reg(ssa, var)) {
16592 return 0;
16593 }
16594
16595 if (ssa->vars[var].definition >= 0) {
16596 uint32_t def = ssa->vars[var].definition;
16597 if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) {
16598 return 0;
16599 }
16600 }
16601
16602 if (ssa->vars[var].use_chain >= 0) {
16603 int use = ssa->vars[var].use_chain;
16604
16605 do {
16606 if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) &&
16607 !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) {
16608 return 0;
16609 }
16610 /* Quick workaround to disable register allocation for unsupported operand */
16611 // TODO: Find a general solution ???
16612 if (op_array->opcodes[use].opcode == ZEND_FETCH_DIM_R) {
16613 return 0;
16614 }
16615 use = zend_ssa_next_use(ssa->ops, var, use);
16616 } while (use >= 0);
16617 }
16618
16619 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
16620 int def_block, use_block, b, use, j;
16621 zend_basic_block *bb;
16622 zend_ssa_phi *p;
16623 bool ret = 1;
16624 zend_worklist worklist;
16625 ALLOCA_FLAG(use_heap)
16626
16627 /* Check if live range is split by ENTRY block */
16628 if (ssa->vars[var].definition >= 0) {
16629 def_block =ssa->cfg.map[ssa->vars[var].definition];
16630 } else {
16631 ZEND_ASSERT(ssa->vars[var].definition_phi);
16632 def_block = ssa->vars[var].definition_phi->block;
16633 }
16634
16635 ZEND_WORKLIST_ALLOCA(&worklist, ssa->cfg.blocks_count, use_heap);
16636
16637 if (ssa->vars[var].use_chain >= 0) {
16638 use = ssa->vars[var].use_chain;
16639 do {
16640 use_block = ssa->cfg.map[use];
16641 if (use_block != def_block) {
16642 zend_worklist_push(&worklist, use_block);
16643 }
16644 use = zend_ssa_next_use(ssa->ops, var, use);
16645 } while (use >= 0);
16646 }
16647
16648 p = ssa->vars[var].phi_use_chain;
16649 while (p) {
16650 use_block = p->block;
16651 if (use_block != def_block) {
16652 bb = &ssa->cfg.blocks[use_block];
16653 for (j = 0; j < bb->predecessors_count; j++) {
16654 if (p->sources[j] == var) {
16655 use_block = ssa->cfg.predecessors[bb->predecessor_offset + j];
16656 if (use_block != def_block) {
16657 zend_worklist_push(&worklist, use_block);
16658 }
16659 }
16660 }
16661 }
16662 p = zend_ssa_next_use_phi(ssa, var, p);
16663 }
16664
16665 while (zend_worklist_len(&worklist) != 0) {
16666 b = zend_worklist_pop(&worklist);
16667 bb = &ssa->cfg.blocks[b];
16668 if (bb->flags & (ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) {
16669 ret = 0;
16670 break;
16671 }
16672 for (j = 0; j < bb->predecessors_count; j++) {
16673 b = ssa->cfg.predecessors[bb->predecessor_offset + j];
16674 if (b != def_block) {
16675 zend_worklist_push(&worklist, b);
16676 }
16677 }
16678 }
16679
16680 ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
16681
16682 return ret;
16683 }
16684
16685 return 1;
16686 }
16687
16688 static void jit_frameless_icall0(zend_jit_ctx *jit, const zend_op *opline)
16689 {
16690 jit_SET_EX_OPLINE(jit, opline);
16691
16692 void *function = ZEND_FLF_HANDLER(opline);
16693 zend_jit_addr res_addr = RES_ADDR();
16694 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
16695 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
16696 ir_CALL_1(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref);
16697 zend_jit_check_exception(jit);
16698 }
16699
16700 static void jit_frameless_icall1(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
16701 {
16702 jit_SET_EX_OPLINE(jit, opline);
16703
16704 /* Avoid dropping RC check in case op escapes. */
16705 if (op1_info & MAY_BE_RC1) {
16706 op1_info |= MAY_BE_RCN;
16707 }
16708
16709 void *function = ZEND_FLF_HANDLER(opline);
16710 zend_jit_addr res_addr = RES_ADDR();
16711 zend_jit_addr op1_addr = OP1_ADDR();
16712 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
16713 ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
16714 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
16715 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
16716 zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
16717 }
16718 if (op1_info & MAY_BE_REF) {
16719 op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
16720 }
16721 ir_CALL_2(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref);
16722 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
16723 zend_jit_check_exception(jit);
16724 }
16725
16726 static void jit_frameless_icall2(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info)
16727 {
16728 jit_SET_EX_OPLINE(jit, opline);
16729
16730 /* Avoid dropping RC check in case op escapes. */
16731 if (op1_info & MAY_BE_RC1) {
16732 op1_info |= MAY_BE_RCN;
16733 }
16734 if (op2_info & MAY_BE_RC1) {
16735 op2_info |= MAY_BE_RCN;
16736 }
16737
16738 void *function = ZEND_FLF_HANDLER(opline);
16739 zend_jit_addr res_addr = RES_ADDR();
16740 zend_jit_addr op1_addr = OP1_ADDR();
16741 zend_jit_addr op2_addr = OP2_ADDR();
16742 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
16743 ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
16744 ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
16745 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
16746 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
16747 zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
16748 }
16749 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
16750 zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
16751 }
16752 if (op1_info & MAY_BE_REF) {
16753 op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
16754 }
16755 if (op2_info & MAY_BE_REF) {
16756 op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
16757 }
16758 ir_CALL_3(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref, op2_ref);
16759 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
16760 /* Set OP1 to UNDEF in case FREE_OP2() throws. */
16761 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) != 0 && (opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0) {
16762 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
16763 }
16764 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
16765 zend_jit_check_exception(jit);
16766 }
16767
16768 static void jit_frameless_icall3(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, uint32_t op1_data_info)
16769 {
16770 jit_SET_EX_OPLINE(jit, opline);
16771
16772 /* Avoid dropping RC check in case op escapes. */
16773 if (op1_info & MAY_BE_RC1) {
16774 op1_info |= MAY_BE_RCN;
16775 }
16776 if (op2_info & MAY_BE_RC1) {
16777 op2_info |= MAY_BE_RCN;
16778 }
16779 if (op1_data_info & MAY_BE_RC1) {
16780 op1_data_info |= MAY_BE_RCN;
16781 }
16782
16783 void *function = ZEND_FLF_HANDLER(opline);
16784 uint8_t op_data_type = (opline + 1)->op1_type;
16785 zend_jit_addr res_addr = RES_ADDR();
16786 zend_jit_addr op1_addr = OP1_ADDR();
16787 zend_jit_addr op2_addr = OP2_ADDR();
16788 zend_jit_addr op3_addr = OP1_DATA_ADDR();
16789 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
16790 ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
16791 ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
16792 ir_ref op3_ref = jit_ZVAL_ADDR(jit, op3_addr);
16793 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
16794 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
16795 zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
16796 }
16797 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
16798 zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
16799 }
16800 if ((opline+1)->op1_type == IS_CV && (op1_data_info & MAY_BE_UNDEF)) {
16801 zend_jit_zval_check_undef(jit, op3_ref, (opline+1)->op1.var, opline, 1);
16802 }
16803 if (op1_info & MAY_BE_REF) {
16804 op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
16805 }
16806 if (op2_info & MAY_BE_REF) {
16807 op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
16808 }
16809 if (op1_data_info & MAY_BE_REF) {
16810 op3_ref = jit_ZVAL_DEREF_ref(jit, op3_ref);
16811 }
16812 ir_CALL_4(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref, op2_ref, op3_ref);
16813 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
16814 /* Set OP1 to UNDEF in case FREE_OP2() throws. */
16815 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
16816 && ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
16817 || (op_data_type & (IS_VAR|IS_TMP_VAR)))) {
16818 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
16819 }
16820 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
16821 /* If OP1 is a TMP|VAR, we don't need to set OP2 to UNDEF on free because
16822 * zend_fetch_debug_backtrace aborts when it encounters the first UNDEF TMP|VAR. */
16823 if (!(opline->op1_type & (IS_VAR|IS_TMP_VAR))
16824 && (opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0
16825 && (op_data_type & (IS_VAR|IS_TMP_VAR)) != 0) {
16826 jit_set_Z_TYPE_INFO(jit, op2_addr, IS_UNDEF);
16827 }
16828 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
16829 zend_jit_check_exception(jit);
16830 }
16831
16832 /*
16833 * Local variables:
16834 * tab-width: 4
16835 * c-basic-offset: 4
16836 * indent-tabs-mode: t
16837 * End:
16838 */
16839