Lines Matching refs:jit
64 #define _ir_CTX (&jit->ctx)
67 #define ir_CONST_ADDR(_addr) jit_CONST_ADDR(jit, (uintptr_t)(_addr))
68 #define ir_CONST_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), 0)
69 #define ir_CONST_FC_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), IR_FASTCALL_FUNC)
74 jit_CONST_FUNC_PROTO(jit, (uintptr_t)(_addr), (_proto))
78 jit_ADD_OFFSET(jit, _addr, _offset)
191 ir_ADD_OFFSET(jit_TLS(jit), EG_TLS_OFFSET(_field))
194 ir_ADD_OFFSET(jit_TLS(jit), CG_TLS_OFFSET(_field))
210 jit_CALL(jit_FP(jit), _field)
213 jit_CALL(jit_IP(jit), _field)
253 static int zend_jit_ ## name ## _stub(zend_jit_ctx *jit);
322 static int zend_jit_assign_to_variable(zend_jit_ctx *jit,
337 int (*stub)(zend_jit_ctx *jit);
443 static bool zend_jit_prefer_const_addr_load(zend_jit_ctx *jit, uintptr_t addr) in zend_jit_prefer_const_addr_load() argument
464 static ir_ref jit_TLS(zend_jit_ctx *jit) in jit_TLS() argument
466 ZEND_ASSERT(jit->ctx.control); in jit_TLS()
467 if (jit->tls) { in jit_TLS()
470 ir_ref ref = jit->ctx.control; in jit_TLS()
473 if (ref == jit->tls) { in jit_TLS()
474 return jit->tls; in jit_TLS()
476 insn = &jit->ctx.ir_base[ref]; in jit_TLS()
483 jit->tls = ir_TLS( in jit_TLS()
486 return jit->tls; in jit_TLS()
490 static ir_ref jit_CONST_ADDR(zend_jit_ctx *jit, uintptr_t addr) in jit_CONST_ADDR() argument
498 zv = zend_hash_index_lookup(&jit->addr_hash, addr); in jit_CONST_ADDR()
501 ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_ADDR, IR_ADDR)); in jit_CONST_ADDR()
503 ref = ir_unique_const_addr(&jit->ctx, addr); in jit_CONST_ADDR()
509 static ir_ref jit_CONST_FUNC_PROTO(zend_jit_ctx *jit, uintptr_t addr, ir_ref proto) in jit_CONST_FUNC_PROTO() argument
516 zv = zend_hash_index_lookup(&jit->addr_hash, addr); in jit_CONST_FUNC_PROTO()
519 …ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_FUNC_ADDR, IR_ADDR) && jit->ctx.ir_base[ref].pr… in jit_CONST_FUNC_PROTO()
521 ref = ir_unique_const_addr(&jit->ctx, addr); in jit_CONST_FUNC_PROTO()
522 insn = &jit->ctx.ir_base[ref]; in jit_CONST_FUNC_PROTO()
530 static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags) in jit_CONST_FUNC() argument
534 ir_ref proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0; in jit_CONST_FUNC()
539 return jit_CONST_FUNC_PROTO(jit, addr, proto); in jit_CONST_FUNC()
542 static ir_ref jit_ADD_OFFSET(zend_jit_ctx *jit, ir_ref addr, uintptr_t offset) in jit_ADD_OFFSET() argument
550 static ir_ref jit_EG_exception(zend_jit_ctx *jit) in jit_EG_exception() argument
555 ir_ref ref = jit->eg_exception_addr; in jit_EG_exception()
558 ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)&EG(exception)); in jit_EG_exception()
559 jit->eg_exception_addr = ref; in jit_EG_exception()
565 static ir_ref jit_STUB_ADDR(zend_jit_ctx *jit, jit_stub_id id) in jit_STUB_ADDR() argument
567 ir_ref ref = jit->stub_addr[id]; in jit_STUB_ADDR()
570 ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]); in jit_STUB_ADDR()
571 jit->stub_addr[id] = ref; in jit_STUB_ADDR()
576 static ir_ref jit_STUB_FUNC_ADDR(zend_jit_ctx *jit, jit_stub_id id, uint16_t flags) in jit_STUB_FUNC_ADDR() argument
578 ir_ref ref = jit->stub_addr[id]; in jit_STUB_FUNC_ADDR()
582 ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]); in jit_STUB_FUNC_ADDR()
583 insn = &jit->ctx.ir_base[ref]; in jit_STUB_FUNC_ADDR()
587 insn->proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0; in jit_STUB_FUNC_ADDR()
591 jit->stub_addr[id] = ref; in jit_STUB_FUNC_ADDR()
596 static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr) in jit_SNAPSHOT() argument
599 const void *ptr = (const void*)jit->ctx.ir_base[addr].val.addr; in jit_SNAPSHOT()
623 zend_jit_trace_info *t = jit->trace; in jit_SNAPSHOT()
628 … && jit->ctx.ir_base[addr].val.u64 == (uintptr_t)zend_jit_trace_get_exit_addr(t->exit_count - 1)) { in jit_SNAPSHOT()
820 static void jit_SIDE_EXIT(zend_jit_ctx *jit, ir_ref addr) in jit_SIDE_EXIT() argument
822 jit_SNAPSHOT(jit, addr); in jit_SIDE_EXIT()
828 static ir_ref jit_EMALLOC(zend_jit_ctx *jit, size_t size, const zend_op_array *op_array, const zend… in jit_EMALLOC() argument
848 static ir_ref jit_EFREE(zend_jit_ctx *jit, ir_ref ptr, size_t size, const zend_op_array *op_array, … in jit_EFREE() argument
868 static ir_ref jit_FP(zend_jit_ctx *jit) in jit_FP() argument
870 ZEND_ASSERT(jit->ctx.control); in jit_FP()
871 if (jit->fp == IR_UNUSED) { in jit_FP()
873 jit->fp = ir_RLOAD_A(ZREG_FP); in jit_FP()
876 ir_ref ref = jit->ctx.control; in jit_FP()
879 if (ref == jit->fp) { in jit_FP()
882 insn = &jit->ctx.ir_base[ref]; in jit_FP()
884 jit->fp = ir_RLOAD_A(ZREG_FP); in jit_FP()
890 return jit->fp; in jit_FP()
893 static void jit_STORE_FP(zend_jit_ctx *jit, ir_ref ref) in jit_STORE_FP() argument
896 jit->fp = IR_UNUSED; in jit_STORE_FP()
899 static ir_ref jit_IP(zend_jit_ctx *jit) in jit_IP() argument
904 static void jit_STORE_IP(zend_jit_ctx *jit, ir_ref ref) in jit_STORE_IP() argument
909 static ir_ref jit_IP32(zend_jit_ctx *jit) in jit_IP32() argument
914 static void jit_LOAD_IP(zend_jit_ctx *jit, ir_ref ref) in jit_LOAD_IP() argument
917 jit_STORE_IP(jit, ref); in jit_LOAD_IP()
923 static void jit_LOAD_IP_ADDR(zend_jit_ctx *jit, const zend_op *target) in jit_LOAD_IP_ADDR() argument
925 jit_LOAD_IP(jit, ir_CONST_ADDR(target)); in jit_LOAD_IP_ADDR()
928 static void zend_jit_track_last_valid_opline(zend_jit_ctx *jit) in zend_jit_track_last_valid_opline() argument
930 jit->use_last_valid_opline = 0; in zend_jit_track_last_valid_opline()
931 jit->track_last_valid_opline = 1; in zend_jit_track_last_valid_opline()
934 static void zend_jit_use_last_valid_opline(zend_jit_ctx *jit) in zend_jit_use_last_valid_opline() argument
936 if (jit->track_last_valid_opline) { in zend_jit_use_last_valid_opline()
937 jit->use_last_valid_opline = 1; in zend_jit_use_last_valid_opline()
938 jit->track_last_valid_opline = 0; in zend_jit_use_last_valid_opline()
942 static bool zend_jit_trace_uses_initial_ip(zend_jit_ctx *jit) in zend_jit_trace_uses_initial_ip() argument
944 return jit->use_last_valid_opline; in zend_jit_trace_uses_initial_ip()
947 static void zend_jit_set_last_valid_opline(zend_jit_ctx *jit, const zend_op *opline) in zend_jit_set_last_valid_opline() argument
949 if (!jit->reuse_ip) { in zend_jit_set_last_valid_opline()
950 jit->track_last_valid_opline = 1; in zend_jit_set_last_valid_opline()
951 jit->last_valid_opline = opline; in zend_jit_set_last_valid_opline()
955 static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit) in zend_jit_reset_last_valid_opline() argument
957 jit->track_last_valid_opline = 0; in zend_jit_reset_last_valid_opline()
958 jit->last_valid_opline = NULL; in zend_jit_reset_last_valid_opline()
961 static void zend_jit_start_reuse_ip(zend_jit_ctx *jit) in zend_jit_start_reuse_ip() argument
963 zend_jit_reset_last_valid_opline(jit); in zend_jit_start_reuse_ip()
964 jit->reuse_ip = 1; in zend_jit_start_reuse_ip()
967 static int zend_jit_reuse_ip(zend_jit_ctx *jit) in zend_jit_reuse_ip() argument
969 if (!jit->reuse_ip) { in zend_jit_reuse_ip()
970 zend_jit_start_reuse_ip(jit); in zend_jit_reuse_ip()
972 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call))); in zend_jit_reuse_ip()
977 static void zend_jit_stop_reuse_ip(zend_jit_ctx *jit) in zend_jit_stop_reuse_ip() argument
979 jit->reuse_ip = 0; in zend_jit_stop_reuse_ip()
982 static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level) in zend_jit_save_call_chain() argument
994 rx = jit_IP(jit); in zend_jit_save_call_chain()
1002 jit->delayed_call_level = 0; in zend_jit_save_call_chain()
1008 static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target) in zend_jit_set_ip() argument
1013 if (jit->delayed_call_level) { in zend_jit_set_ip()
1014 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { in zend_jit_set_ip()
1019 if (jit->last_valid_opline) { in zend_jit_set_ip()
1020 zend_jit_use_last_valid_opline(jit); in zend_jit_set_ip()
1021 if (jit->last_valid_opline != target) { in zend_jit_set_ip()
1023 ref = jit_IP(jit); in zend_jit_set_ip()
1028 if (target > jit->last_valid_opline) { in zend_jit_set_ip()
1029 ref = ir_ADD_OFFSET(ref, (uintptr_t)target - (uintptr_t)jit->last_valid_opline); in zend_jit_set_ip()
1031 ref = ir_SUB_A(ref, ir_CONST_ADDR((uintptr_t)jit->last_valid_opline - (uintptr_t)target)); in zend_jit_set_ip()
1034 jit_STORE_IP(jit, ref); in zend_jit_set_ip()
1041 jit_STORE_IP(jit, ir_CONST_ADDR(target)); in zend_jit_set_ip()
1046 jit->reuse_ip = 0; in zend_jit_set_ip()
1047 zend_jit_set_last_valid_opline(jit, target); in zend_jit_set_ip()
1051 static int zend_jit_set_ip_ex(zend_jit_ctx *jit, const zend_op *target, bool set_ip_reg) in zend_jit_set_ip_ex() argument
1053 if (!GCC_GLOBAL_REGS && set_ip_reg && !jit->last_valid_opline) { in zend_jit_set_ip_ex()
1058 return zend_jit_set_ip(jit, target); in zend_jit_set_ip_ex()
1061 static void jit_SET_EX_OPLINE(zend_jit_ctx *jit, const zend_op *target) in jit_SET_EX_OPLINE() argument
1063 if (jit->last_valid_opline == target) { in jit_SET_EX_OPLINE()
1064 zend_jit_use_last_valid_opline(jit); in jit_SET_EX_OPLINE()
1067 ir_STORE(jit_EX(opline), jit_IP(jit)); in jit_SET_EX_OPLINE()
1072 zend_jit_reset_last_valid_opline(jit); in jit_SET_EX_OPLINE()
1077 static ir_ref jit_ZVAL_ADDR(zend_jit_ctx *jit, zend_jit_addr addr) in jit_ZVAL_ADDR() argument
1083 reg = jit_FP(jit); in jit_ZVAL_ADDR()
1085 reg = jit_IP(jit); in jit_ZVAL_ADDR()
1098 static ir_ref jit_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref) in jit_Z_TYPE_ref() argument
1103 static ir_ref jit_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_TYPE() argument
1112 reg = jit_FP(jit); in jit_Z_TYPE()
1114 reg = jit_IP(jit); in jit_Z_TYPE()
1120 return jit_Z_TYPE_ref(jit, jit_ZVAL_ADDR(jit, addr)); in jit_Z_TYPE()
1124 static ir_ref jit_Z_TYPE_FLAGS_ref(zend_jit_ctx *jit, ir_ref ref) in jit_Z_TYPE_FLAGS_ref() argument
1129 static ir_ref jit_Z_TYPE_FLAGS(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_TYPE_FLAGS() argument
1138 reg = jit_FP(jit); in jit_Z_TYPE_FLAGS()
1140 reg = jit_IP(jit); in jit_Z_TYPE_FLAGS()
1146 return jit_Z_TYPE_FLAGS_ref(jit, jit_ZVAL_ADDR(jit, addr)); in jit_Z_TYPE_FLAGS()
1150 static ir_ref jit_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref) in jit_Z_TYPE_INFO_ref() argument
1155 static ir_ref jit_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_TYPE_INFO() argument
1164 reg = jit_FP(jit); in jit_Z_TYPE_INFO()
1166 reg = jit_IP(jit); in jit_Z_TYPE_INFO()
1172 return jit_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr)); in jit_Z_TYPE_INFO()
1176 static void jit_set_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type_info) in jit_set_Z_TYPE_INFO_ref() argument
1181 static void jit_set_Z_TYPE_INFO_ex(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref type_info) in jit_set_Z_TYPE_INFO_ex() argument
1188 reg = jit_FP(jit); in jit_set_Z_TYPE_INFO_ex()
1190 reg = jit_IP(jit); in jit_set_Z_TYPE_INFO_ex()
1196 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr), type_info); in jit_set_Z_TYPE_INFO_ex()
1200 static void jit_set_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr, uint32_t type_info) in jit_set_Z_TYPE_INFO() argument
1210 jit_set_Z_TYPE_INFO_ex(jit, addr, ir_CONST_U32(type_info)); in jit_set_Z_TYPE_INFO()
1213 static ir_ref jit_if_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type) in jit_if_Z_TYPE_ref() argument
1215 return ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), type)); in jit_if_Z_TYPE_ref()
1218 static ir_ref jit_if_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type) in jit_if_Z_TYPE() argument
1221 return ir_IF(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type))); in jit_if_Z_TYPE()
1224 static ir_ref jit_if_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type) in jit_if_not_Z_TYPE() argument
1226 ir_ref ref = jit_Z_TYPE(jit, addr); in jit_if_not_Z_TYPE()
1234 static void jit_guard_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_… in jit_guard_Z_TYPE() argument
1236 ir_ref ref = jit_Z_TYPE(jit, addr); in jit_guard_Z_TYPE()
1245 static void jit_guard_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *e… in jit_guard_not_Z_TYPE() argument
1247 ir_ref ref = jit_Z_TYPE(jit, addr); in jit_guard_not_Z_TYPE()
1255 static ir_ref jit_if_REFCOUNTED(zend_jit_ctx *jit, zend_jit_addr addr) in jit_if_REFCOUNTED() argument
1257 return ir_IF(jit_Z_TYPE_FLAGS(jit, addr)); in jit_if_REFCOUNTED()
1260 static ir_ref jit_if_COLLECTABLE_ref(zend_jit_ctx *jit, ir_ref addr_ref) in jit_if_COLLECTABLE_ref() argument
1262 return ir_IF(ir_AND_U8(jit_Z_TYPE_FLAGS_ref(jit, addr_ref), ir_CONST_U8(IS_TYPE_COLLECTABLE))); in jit_if_COLLECTABLE_ref()
1265 static ir_ref jit_Z_LVAL_ref(zend_jit_ctx *jit, ir_ref ref) in jit_Z_LVAL_ref() argument
1270 static ir_ref jit_Z_DVAL_ref(zend_jit_ctx *jit, ir_ref ref) in jit_Z_DVAL_ref() argument
1275 static bool zend_jit_spilling_may_cause_conflict(zend_jit_ctx *jit, int var, ir_ref val) in zend_jit_spilling_may_cause_conflict() argument
1277 if (jit->ctx.ir_base[val].op == IR_RLOAD) { in zend_jit_spilling_may_cause_conflict()
1291 if (jit->ssa->vars[var].var < jit->current_op_array->last_var) { in zend_jit_spilling_may_cause_conflict()
1293 if (jit->ctx.ir_base[val].op == IR_LOAD in zend_jit_spilling_may_cause_conflict()
1294 && jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op == IR_ADD in zend_jit_spilling_may_cause_conflict()
1295 && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op == IR_RLOAD in zend_jit_spilling_may_cause_conflict()
1296 && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op2 == ZREG_FP in zend_jit_spilling_may_cause_conflict()
1297 && IR_IS_CONST_REF(jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2) in zend_jit_spilling_may_cause_conflict()
1298 …&& jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr != (uintptr_t)EX_NUM… in zend_jit_spilling_may_cause_conflict()
1299 …&& EX_VAR_TO_NUM(jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr) < jit… in zend_jit_spilling_may_cause_conflict()
1302 } else if (jit->ssa->vars[var].definition >= 0 in zend_jit_spilling_may_cause_conflict()
1303 && jit->ssa->ops[jit->ssa->vars[var].definition].op1_def == var in zend_jit_spilling_may_cause_conflict()
1304 && jit->ssa->ops[jit->ssa->vars[var].definition].op1_use >= 0 in zend_jit_spilling_may_cause_conflict()
1305 && jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].no_val in zend_jit_spilling_may_cause_conflict()
1306 && jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].definition_phi in zend_jit_spilling_may_cause_conflict()
1307 …&& (jit->ssa->cfg.blocks[jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].def… in zend_jit_spilling_may_cause_conflict()
1310 } else if (jit->ssa->vars[var].definition >= 0 in zend_jit_spilling_may_cause_conflict()
1311 && jit->ssa->ops[jit->ssa->vars[var].definition].op1_def == var in zend_jit_spilling_may_cause_conflict()
1312 && jit->ssa->ops[jit->ssa->vars[var].definition].op1_use >= 0 in zend_jit_spilling_may_cause_conflict()
1313 && jit->ssa->ops[jit->ssa->vars[var].definition].op2_use >= 0 in zend_jit_spilling_may_cause_conflict()
1314 && jit->ra[jit->ssa->ops[jit->ssa->vars[var].definition].op2_use].ref == val) { in zend_jit_spilling_may_cause_conflict()
1323 static void zend_jit_def_reg(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref val) in zend_jit_def_reg() argument
1329 if (var == jit->delay_var) { in zend_jit_def_reg()
1330 ir_refs_add(jit->delay_refs, val); in zend_jit_def_reg()
1333 ZEND_ASSERT(jit->ra && jit->ra[var].ref == IR_NULL); in zend_jit_def_reg()
1337 if (jit->ctx.binding) { in zend_jit_def_reg()
1338 ir_ref old = ir_binding_find(&jit->ctx, val); in zend_jit_def_reg()
1339 if (old && old != -EX_NUM_TO_VAR(jit->ssa->vars[var].var)) { in zend_jit_def_reg()
1340 val = ir_emit2(&jit->ctx, IR_OPT(IR_COPY, jit->ctx.ir_base[val].type), val, 1); in zend_jit_def_reg()
1343 if (!zend_jit_spilling_may_cause_conflict(jit, var, val)) { in zend_jit_def_reg()
1344 val = ir_bind(&jit->ctx, -EX_NUM_TO_VAR(jit->ssa->vars[var].var), val); in zend_jit_def_reg()
1347 jit->ra[var].ref = val; in zend_jit_def_reg()
1349 if (jit->ra[var].flags & ZREG_FORWARD) { in zend_jit_def_reg()
1350 zend_ssa_phi *phi = jit->ssa->vars[var].phi_use_chain; in zend_jit_def_reg()
1355 jit->ra[var].flags &= ~ZREG_FORWARD; in zend_jit_def_reg()
1361 jit->ra[src_var].ref = val; in zend_jit_def_reg()
1363 if (!(jit->ra[src_var].flags & ZREG_FORWARD)) { in zend_jit_def_reg()
1364 phi = zend_ssa_next_use_phi(jit->ssa, var, phi); in zend_jit_def_reg()
1367 dst_phi = jit->ssa->vars[src_var].phi_use_chain; in zend_jit_def_reg()
1369 ZEND_ASSERT(!zend_ssa_next_use_phi(jit->ssa, src_var, dst_phi) && "reg forwarding"); in zend_jit_def_reg()
1370 jit->ra[src_var].flags &= ~ZREG_FORWARD; in zend_jit_def_reg()
1373 if (jit->ra[dst_phi->ssa_var].ref > 0) { in zend_jit_def_reg()
1374 ir_insn *phi_insn = &jit->ctx.ir_base[jit->ra[dst_phi->ssa_var].ref]; in zend_jit_def_reg()
1377 bb = &jit->ssa->cfg.blocks[dst_phi->block]; in zend_jit_def_reg()
1387 phi = zend_ssa_next_use_phi(jit->ssa, var, phi); in zend_jit_def_reg()
1392 static ir_ref zend_jit_use_reg(zend_jit_ctx *jit, zend_jit_addr addr) in zend_jit_use_reg() argument
1397 ZEND_ASSERT(jit->ra && jit->ra[var].ref); in zend_jit_use_reg()
1398 if (jit->ra[var].ref == IR_NULL) { in zend_jit_use_reg()
1402 ZEND_ASSERT(jit->ra[var].flags & ZREG_LOAD); in zend_jit_use_reg()
1403 mem_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(jit->ssa->vars[var].var)); in zend_jit_use_reg()
1404 if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_LONG) { in zend_jit_use_reg()
1405 ref = jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr)); in zend_jit_use_reg()
1406 } else if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { in zend_jit_use_reg()
1407 ref = jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr)); in zend_jit_use_reg()
1411 zend_jit_def_reg(jit, addr, ref); in zend_jit_use_reg()
1414 return jit->ra[Z_SSA_VAR(addr)].ref; in zend_jit_use_reg()
1417 static void zend_jit_gen_pi(zend_jit_ctx *jit, zend_ssa_phi *phi) in zend_jit_gen_pi() argument
1423 ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD)); in zend_jit_gen_pi()
1424 ZEND_ASSERT(jit->ra[src_var].ref); in zend_jit_gen_pi()
1426 if (jit->ra[src_var].ref == IR_NULL) { in zend_jit_gen_pi()
1428 if (jit->ssa->vars[dst_var].use_chain < 0 in zend_jit_gen_pi()
1429 && jit->ssa->vars[dst_var].phi_use_chain) { in zend_jit_gen_pi()
1430 zend_ssa_phi *phi = jit->ssa->vars[dst_var].phi_use_chain; in zend_jit_gen_pi()
1431 if (!zend_ssa_next_use_phi(jit->ssa, dst_var, phi)) { in zend_jit_gen_pi()
1433 jit->ra[src_var].flags |= ZREG_FORWARD; in zend_jit_gen_pi()
1440 zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), in zend_jit_gen_pi()
1441 zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var))); in zend_jit_gen_pi()
1444 static void zend_jit_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi) in zend_jit_gen_phi() argument
1447 zend_basic_block *bb = &jit->ssa->cfg.blocks[phi->block]; in zend_jit_gen_phi()
1450 ir_type type = (jit->ssa->var_info[phi->ssa_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE; in zend_jit_gen_phi()
1451 ir_ref merge = jit->bb_start_ref[phi->block]; in zend_jit_gen_phi()
1453 ir_ref old_insns_count = jit->ctx.insns_count; in zend_jit_gen_phi()
1458 ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD)); in zend_jit_gen_phi()
1460 ZEND_ASSERT(jit->ctx.ir_base[merge].op == IR_MERGE || jit->ctx.ir_base[merge].op == IR_LOOP_BEGIN); in zend_jit_gen_phi()
1461 ZEND_ASSERT(n == jit->ctx.ir_base[merge].inputs_count); in zend_jit_gen_phi()
1463 ref = ir_emit_N(&jit->ctx, IR_OPT(IR_PHI, type), n + 1); in zend_jit_gen_phi()
1464 ir_set_op(&jit->ctx, ref, 1, merge); in zend_jit_gen_phi()
1469 ZEND_ASSERT(jit->ra[src_var].ref); in zend_jit_gen_phi()
1470 if (jit->ra[src_var].ref == IR_NULL) { in zend_jit_gen_phi()
1471 jit->ra[src_var].flags |= ZREG_FORWARD; in zend_jit_gen_phi()
1474 ir_ref src_ref = zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)); in zend_jit_gen_phi()
1480 ir_set_op(&jit->ctx, ref, i + 2, src_ref); in zend_jit_gen_phi()
1485 jit->ctx.insns_count = old_insns_count; in zend_jit_gen_phi()
1488 zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref); in zend_jit_gen_phi()
1491 static ir_ref jit_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_LVAL() argument
1496 return zend_jit_use_reg(jit, addr); in jit_Z_LVAL()
1498 return jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, addr)); in jit_Z_LVAL()
1502 static void jit_set_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval) in jit_set_Z_LVAL() argument
1505 zend_jit_def_reg(jit, addr, lval); in jit_set_Z_LVAL()
1507 ir_STORE(jit_ZVAL_ADDR(jit, addr), lval); in jit_set_Z_LVAL()
1512 static ir_ref jit_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_W2() argument
1517 return ir_LOAD_L(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2))); in jit_Z_W2()
1521 static void jit_set_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval) in jit_set_Z_W2() argument
1523 ir_STORE(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2)), lval); in jit_set_Z_W2()
1527 static ir_ref jit_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_DVAL() argument
1532 return zend_jit_use_reg(jit, addr); in jit_Z_DVAL()
1534 return jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, addr)); in jit_Z_DVAL()
1538 static void jit_set_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref dval) in jit_set_Z_DVAL() argument
1541 zend_jit_def_reg(jit, addr, dval); in jit_set_Z_DVAL()
1543 ir_STORE(jit_ZVAL_ADDR(jit, addr), dval); in jit_set_Z_DVAL()
1547 static ir_ref jit_Z_PTR_ref(zend_jit_ctx *jit, ir_ref ref) in jit_Z_PTR_ref() argument
1552 static ir_ref jit_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_PTR() argument
1557 return jit_Z_PTR_ref(jit, jit_ZVAL_ADDR(jit, addr)); in jit_Z_PTR()
1561 static void jit_set_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref ptr) in jit_set_Z_PTR() argument
1563 ir_STORE(jit_ZVAL_ADDR(jit, addr), ptr); in jit_set_Z_PTR()
1566 static ir_ref jit_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref) in jit_GC_REFCOUNT() argument
1571 static void jit_set_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref, uint32_t refcount) in jit_set_GC_REFCOUNT() argument
1576 static void jit_GC_ADDREF(zend_jit_ctx *jit, ir_ref ref) in jit_GC_ADDREF() argument
1581 static void jit_GC_ADDREF2(zend_jit_ctx *jit, ir_ref ref) in jit_GC_ADDREF2() argument
1587 static ir_ref jit_GC_DELREF(zend_jit_ctx *jit, ir_ref ref) in jit_GC_DELREF() argument
1595 static ir_ref jit_if_GC_MAY_NOT_LEAK(zend_jit_ctx *jit, ir_ref ref) in jit_if_GC_MAY_NOT_LEAK() argument
1603 static void jit_ZVAL_COPY_CONST(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, uint32_t d… in jit_ZVAL_COPY_CONST() argument
1609 jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE(Z_DVAL_P(zv))); in jit_ZVAL_COPY_CONST()
1611 jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE((double)Z_LVAL_P(zv))); in jit_ZVAL_COPY_CONST()
1613 jit_set_Z_LVAL(jit, dst, ir_CONST_LONG(Z_LVAL_P(zv))); in jit_ZVAL_COPY_CONST()
1616 jit_set_Z_PTR(jit, dst, ref); in jit_ZVAL_COPY_CONST()
1618 jit_GC_ADDREF(jit, ref); in jit_ZVAL_COPY_CONST()
1625 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in jit_ZVAL_COPY_CONST()
1628 jit_set_Z_TYPE_INFO(jit, dst, Z_TYPE_INFO_P(zv)); in jit_ZVAL_COPY_CONST()
1633 static ir_ref jit_if_TYPED_REF(zend_jit_ctx *jit, ir_ref ref) in jit_if_TYPED_REF() argument
1638 static void jit_ZVAL_COPY(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, zend_jit_addr sr… in jit_ZVAL_COPY() argument
1644 jit_set_Z_LVAL(jit, dst, jit_Z_LVAL(jit, src)); in jit_ZVAL_COPY()
1646 jit_set_Z_DVAL(jit, dst, jit_Z_DVAL(jit, src)); in jit_ZVAL_COPY()
1650 jit_set_Z_W2(jit, dst, jit_Z_W2(jit, src)); in jit_ZVAL_COPY()
1653 ref = jit_Z_PTR(jit, src); in jit_ZVAL_COPY()
1654 jit_set_Z_PTR(jit, dst, ref); in jit_ZVAL_COPY()
1663 jit_set_Z_TYPE_INFO(jit, dst, type); in jit_ZVAL_COPY()
1666 ir_ref type = jit_Z_TYPE_INFO(jit, src); in jit_ZVAL_COPY()
1667 jit_set_Z_TYPE_INFO_ex(jit, dst, type); in jit_ZVAL_COPY()
1677 jit_GC_ADDREF(jit, ref); in jit_ZVAL_COPY()
1687 static void jit_ZVAL_COPY_2(zend_jit_ctx *jit, zend_jit_addr dst2, zend_jit_addr dst, uint32_t dst_… in jit_ZVAL_COPY_2() argument
1693 ref = jit_Z_LVAL(jit, src); in jit_ZVAL_COPY_2()
1694 jit_set_Z_LVAL(jit, dst, ref); in jit_ZVAL_COPY_2()
1695 jit_set_Z_LVAL(jit, dst2, ref); in jit_ZVAL_COPY_2()
1697 ref = jit_Z_DVAL(jit, src); in jit_ZVAL_COPY_2()
1698 jit_set_Z_DVAL(jit, dst, ref); in jit_ZVAL_COPY_2()
1699 jit_set_Z_DVAL(jit, dst2, ref); in jit_ZVAL_COPY_2()
1703 ref = jit_Z_W2(jit, src); in jit_ZVAL_COPY_2()
1704 jit_set_Z_W2(jit, dst, ref); in jit_ZVAL_COPY_2()
1705 jit_set_Z_W2(jit, dst2, ref); in jit_ZVAL_COPY_2()
1708 ref = jit_Z_PTR(jit, src); in jit_ZVAL_COPY_2()
1709 jit_set_Z_PTR(jit, dst, ref); in jit_ZVAL_COPY_2()
1710 jit_set_Z_PTR(jit, dst2, ref); in jit_ZVAL_COPY_2()
1721 jit_set_Z_TYPE_INFO_ex(jit, dst, type_ref); in jit_ZVAL_COPY_2()
1724 jit_set_Z_TYPE_INFO_ex(jit, dst2, type_ref); in jit_ZVAL_COPY_2()
1727 ir_ref type = jit_Z_TYPE_INFO(jit, src); in jit_ZVAL_COPY_2()
1728 jit_set_Z_TYPE_INFO_ex(jit, dst, type); in jit_ZVAL_COPY_2()
1729 jit_set_Z_TYPE_INFO_ex(jit, dst2, type); in jit_ZVAL_COPY_2()
1740 jit_GC_ADDREF2(jit, ref); in jit_ZVAL_COPY_2()
1742 jit_GC_ADDREF(jit, ref); in jit_ZVAL_COPY_2()
1753 static void jit_ZVAL_DTOR(zend_jit_ctx *jit, ir_ref ref, uint32_t op_info, const zend_op *opline) in jit_ZVAL_DTOR() argument
1764 jit_SET_EX_OPLINE(jit, opline); in jit_ZVAL_DTOR()
1773 jit_SET_EX_OPLINE(jit, opline); in jit_ZVAL_DTOR()
1780 jit_SET_EX_OPLINE(jit, opline); in jit_ZVAL_DTOR()
1785 static void jit_ZVAL_PTR_DTOR(zend_jit_ctx *jit, in jit_ZVAL_PTR_DTOR() argument
1798 if_refcounted = jit_if_REFCOUNTED(jit, addr); in jit_ZVAL_PTR_DTOR()
1803 ref = jit_Z_PTR(jit, addr); in jit_ZVAL_PTR_DTOR()
1804 ref2 = jit_GC_DELREF(jit, ref); in jit_ZVAL_PTR_DTOR()
1812 jit_ZVAL_DTOR(jit, ref, op_info, opline); in jit_ZVAL_PTR_DTOR()
1824 if_ref = jit_if_Z_TYPE(jit, addr, IS_REFERENCE); in jit_ZVAL_PTR_DTOR()
1829 if_collectable = jit_if_COLLECTABLE_ref(jit, ref2); in jit_ZVAL_PTR_DTOR()
1834 ref2 = jit_Z_PTR_ref(jit, ref2); in jit_ZVAL_PTR_DTOR()
1840 if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref); in jit_ZVAL_PTR_DTOR()
1846 jit_SET_EX_OPLINE(jit, opline); in jit_ZVAL_PTR_DTOR()
1858 static void jit_FREE_OP(zend_jit_ctx *jit, in jit_FREE_OP() argument
1865 jit_ZVAL_PTR_DTOR(jit, in jit_FREE_OP()
1871 static void jit_OBJ_RELEASE(zend_jit_ctx *jit, ir_ref ref) in jit_OBJ_RELEASE() argument
1877 if_not_zero = ir_IF(jit_GC_DELREF(jit, ref)); in jit_OBJ_RELEASE()
1885 if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref); in jit_OBJ_RELEASE()
1897 static void zend_jit_check_timeout(zend_jit_ctx *jit, const zend_op *opline, const void *exit_addr) in zend_jit_check_timeout() argument
1903 } else if (!opline || jit->last_valid_opline == opline) { in zend_jit_check_timeout()
1904 ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_interrupt_handler)); in zend_jit_check_timeout()
1909 jit_LOAD_IP_ADDR(jit, opline); in zend_jit_check_timeout()
1910 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_interrupt_handler)); in zend_jit_check_timeout()
1917 static int zend_jit_exception_handler_stub(zend_jit_ctx *jit) in zend_jit_exception_handler_stub() argument
1925 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); in zend_jit_exception_handler_stub()
1934 ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), jit_FP(jit)); in zend_jit_exception_handler_stub()
1945 static int zend_jit_exception_handler_undef_stub(zend_jit_ctx *jit) in zend_jit_exception_handler_undef_stub() argument
1959 …ir_STORE(ir_ADD_OFFSET(ir_ADD_A(jit_FP(jit), ref), offsetof(zval, u1.type_info)), ir_CONST_U32(IS_… in zend_jit_exception_handler_undef_stub()
1962 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_exception_handler_undef_stub()
1967 static int zend_jit_exception_handler_free_op1_op2_stub(zend_jit_ctx *jit) in zend_jit_exception_handler_free_op1_op2_stub() argument
1980 ref = ir_ADD_A(jit_FP(jit), ref); in zend_jit_exception_handler_free_op1_op2_stub()
1982 jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL); in zend_jit_exception_handler_free_op1_op2_stub()
1985 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); in zend_jit_exception_handler_free_op1_op2_stub()
1990 static int zend_jit_exception_handler_free_op2_stub(zend_jit_ctx *jit) in zend_jit_exception_handler_free_op2_stub() argument
2003 ref = ir_ADD_A(jit_FP(jit), ref); in zend_jit_exception_handler_free_op2_stub()
2005 jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL); in zend_jit_exception_handler_free_op2_stub()
2008 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef)); in zend_jit_exception_handler_free_op2_stub()
2013 static int zend_jit_interrupt_handler_stub(zend_jit_ctx *jit) in zend_jit_interrupt_handler_stub() argument
2019 ir_STORE(jit_EX(opline), jit_IP(jit)); in zend_jit_interrupt_handler_stub()
2029 ir_CALL_1(IR_VOID, ir_CONST_FUNC(zend_interrupt_function), jit_FP(jit)); in zend_jit_interrupt_handler_stub()
2035 jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data))); in zend_jit_interrupt_handler_stub()
2036 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline))); in zend_jit_interrupt_handler_stub()
2040 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); in zend_jit_interrupt_handler_stub()
2047 static int zend_jit_leave_function_handler_stub(zend_jit_ctx *jit) in zend_jit_leave_function_handler_stub() argument
2056 jit_STORE_IP(jit, in zend_jit_leave_function_handler_stub()
2058 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); in zend_jit_leave_function_handler_stub()
2062 …ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info, jit_FP(jit)); in zend_jit_leave_function_handler_stub()
2069 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); in zend_jit_leave_function_handler_stub()
2073 ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info, jit_FP(jit)); in zend_jit_leave_function_handler_stub()
2079 static int zend_jit_negative_shift_stub(zend_jit_ctx *jit) in zend_jit_negative_shift_stub() argument
2083 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), in zend_jit_negative_shift_stub()
2086 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2)); in zend_jit_negative_shift_stub()
2090 static int zend_jit_mod_by_zero_stub(zend_jit_ctx *jit) in zend_jit_mod_by_zero_stub() argument
2094 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), in zend_jit_mod_by_zero_stub()
2097 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2)); in zend_jit_mod_by_zero_stub()
2101 static int zend_jit_invalid_this_stub(zend_jit_ctx *jit) in zend_jit_invalid_this_stub() argument
2105 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), in zend_jit_invalid_this_stub()
2108 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef)); in zend_jit_invalid_this_stub()
2112 static int zend_jit_undefined_function_stub(zend_jit_ctx *jit) in zend_jit_undefined_function_stub() argument
2115 ir_ref ref = ir_LOAD_A(jit_FP(jit)); in zend_jit_undefined_function_stub()
2127 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), in zend_jit_undefined_function_stub()
2132 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_undefined_function_stub()
2137 static int zend_jit_throw_cannot_pass_by_ref_stub(zend_jit_ctx *jit) in zend_jit_throw_cannot_pass_by_ref_stub() argument
2142 opline = ir_LOAD_A(jit_FP(jit)); in zend_jit_throw_cannot_pass_by_ref_stub()
2149 rx = jit_IP(jit); in zend_jit_throw_cannot_pass_by_ref_stub()
2150 jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(rx, ref), ir_CONST_U32(IS_UNDEF)); in zend_jit_throw_cannot_pass_by_ref_stub()
2166 jit_STORE_IP(jit, opline); in zend_jit_throw_cannot_pass_by_ref_stub()
2173 ref = ir_LOAD_U8(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1_type))); in zend_jit_throw_cannot_pass_by_ref_stub()
2178 ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1.var))); in zend_jit_throw_cannot_pass_by_ref_stub()
2182 ref = ir_ADD_A(jit_FP(jit), ref); in zend_jit_throw_cannot_pass_by_ref_stub()
2183 jit_ZVAL_PTR_DTOR(jit, in zend_jit_throw_cannot_pass_by_ref_stub()
2188 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_throw_cannot_pass_by_ref_stub()
2193 static int zend_jit_icall_throw_stub(zend_jit_ctx *jit) in zend_jit_icall_throw_stub() argument
2199 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline))); in zend_jit_icall_throw_stub()
2200 ip = jit_IP(jit); in zend_jit_icall_throw_stub()
2210 jit_STORE_IP(jit, jit_EG(exception_op)); in zend_jit_icall_throw_stub()
2213 ir_STORE(jit_EX(opline), jit_IP(jit)); in zend_jit_icall_throw_stub()
2216 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_icall_throw_stub()
2221 static int zend_jit_leave_throw_stub(zend_jit_ctx *jit) in zend_jit_leave_throw_stub() argument
2226 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline))); in zend_jit_leave_throw_stub()
2227 ip = jit_IP(jit); in zend_jit_leave_throw_stub()
2237 jit_LOAD_IP(jit, jit_EG(exception_op)); in zend_jit_leave_throw_stub()
2240 ir_STORE(jit_EX(opline), jit_IP(jit)); in zend_jit_leave_throw_stub()
2243 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_leave_throw_stub()
2251 static int zend_jit_hybrid_runtime_jit_stub(zend_jit_ctx *jit) in zend_jit_hybrid_runtime_jit_stub() argument
2258 ir_IJMP(ir_LOAD_A(jit_IP(jit))); in zend_jit_hybrid_runtime_jit_stub()
2262 static int zend_jit_hybrid_profile_jit_stub(zend_jit_ctx *jit) in zend_jit_hybrid_profile_jit_stub() argument
2290 static int _zend_jit_hybrid_hot_counter_stub(zend_jit_ctx *jit, uint32_t cost) in _zend_jit_hybrid_hot_counter_stub() argument
2304 jit_FP(jit), in _zend_jit_hybrid_hot_counter_stub()
2305 jit_IP(jit)); in _zend_jit_hybrid_hot_counter_stub()
2306 ir_IJMP(ir_LOAD_A(jit_IP(jit))); in _zend_jit_hybrid_hot_counter_stub()
2309 ref = ir_SUB_A(jit_IP(jit), in _zend_jit_hybrid_hot_counter_stub()
2320 static int zend_jit_hybrid_func_hot_counter_stub(zend_jit_ctx *jit) in zend_jit_hybrid_func_hot_counter_stub() argument
2326 return _zend_jit_hybrid_hot_counter_stub(jit, in zend_jit_hybrid_func_hot_counter_stub()
2330 static int zend_jit_hybrid_loop_hot_counter_stub(zend_jit_ctx *jit) in zend_jit_hybrid_loop_hot_counter_stub() argument
2336 return _zend_jit_hybrid_hot_counter_stub(jit, in zend_jit_hybrid_loop_hot_counter_stub()
2340 static ir_ref _zend_jit_orig_opline_handler(zend_jit_ctx *jit, ir_ref offset) in _zend_jit_orig_opline_handler() argument
2345 addr = ir_ADD_A(offset, jit_IP(jit)); in _zend_jit_orig_opline_handler()
2353 static ir_ref zend_jit_orig_opline_handler(zend_jit_ctx *jit) in zend_jit_orig_opline_handler() argument
2360 return _zend_jit_orig_opline_handler(jit, offset); in zend_jit_orig_opline_handler()
2363 static int _zend_jit_hybrid_trace_counter_stub(zend_jit_ctx *jit, uint32_t cost) in _zend_jit_hybrid_trace_counter_stub() argument
2370 …addr = ir_LOAD_A(ir_ADD_OFFSET(ir_ADD_A(offset, jit_IP(jit)), offsetof(zend_op_trace_info, counter… in _zend_jit_hybrid_trace_counter_stub()
2378 jit_FP(jit), in _zend_jit_hybrid_trace_counter_stub()
2379 jit_IP(jit)); in _zend_jit_hybrid_trace_counter_stub()
2384 jit_STORE_FP(jit, ir_LOAD_A(ref)); in _zend_jit_hybrid_trace_counter_stub()
2386 jit_STORE_IP(jit, ref); in _zend_jit_hybrid_trace_counter_stub()
2387 ir_IJMP(ir_LOAD_A(jit_IP(jit))); in _zend_jit_hybrid_trace_counter_stub()
2390 ir_IJMP(_zend_jit_orig_opline_handler(jit, offset)); in _zend_jit_hybrid_trace_counter_stub()
2398 static int zend_jit_hybrid_func_trace_counter_stub(zend_jit_ctx *jit) in zend_jit_hybrid_func_trace_counter_stub() argument
2404 return _zend_jit_hybrid_trace_counter_stub(jit, in zend_jit_hybrid_func_trace_counter_stub()
2408 static int zend_jit_hybrid_ret_trace_counter_stub(zend_jit_ctx *jit) in zend_jit_hybrid_ret_trace_counter_stub() argument
2414 return _zend_jit_hybrid_trace_counter_stub(jit, in zend_jit_hybrid_ret_trace_counter_stub()
2418 static int zend_jit_hybrid_loop_trace_counter_stub(zend_jit_ctx *jit) in zend_jit_hybrid_loop_trace_counter_stub() argument
2424 return _zend_jit_hybrid_trace_counter_stub(jit, in zend_jit_hybrid_loop_trace_counter_stub()
2428 static int zend_jit_trace_halt_stub(zend_jit_ctx *jit) in zend_jit_trace_halt_stub() argument
2433 jit_STORE_IP(jit, IR_NULL); in zend_jit_trace_halt_stub()
2441 static int zend_jit_trace_escape_stub(zend_jit_ctx *jit) in zend_jit_trace_escape_stub() argument
2444 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); in zend_jit_trace_escape_stub()
2452 static int zend_jit_trace_exit_stub(zend_jit_ctx *jit) in zend_jit_trace_exit_stub() argument
2458 ir_STORE(jit_EX(opline), jit_IP(jit)); in zend_jit_trace_exit_stub()
2469 jit_STORE_FP(jit, ir_LOAD_A(ref)); in zend_jit_trace_exit_stub()
2471 jit_STORE_IP(jit, ref); in zend_jit_trace_exit_stub()
2472 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); in zend_jit_trace_exit_stub()
2479 ir_GUARD(ir_GE(ret, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt)); in zend_jit_trace_exit_stub()
2482 jit_STORE_FP(jit, ir_LOAD_A(ref)); in zend_jit_trace_exit_stub()
2486 jit_STORE_IP(jit, ref); in zend_jit_trace_exit_stub()
2490 zend_jit_check_timeout(jit, NULL, NULL); in zend_jit_trace_exit_stub()
2492 addr = zend_jit_orig_opline_handler(jit); in zend_jit_trace_exit_stub()
2499 ref = ir_CALL_1(IR_I32, addr, jit_FP(jit)); in zend_jit_trace_exit_stub()
2500 ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt)); in zend_jit_trace_exit_stub()
2507 static int zend_jit_undefined_offset_stub(zend_jit_ctx *jit) in zend_jit_undefined_offset_stub() argument
2512 ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key), jit_FP(jit)); in zend_jit_undefined_offset_stub()
2518 static int zend_jit_undefined_key_stub(zend_jit_ctx *jit) in zend_jit_undefined_key_stub() argument
2523 ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key), jit_FP(jit)); in zend_jit_undefined_key_stub()
2529 static int zend_jit_cannot_add_element_stub(zend_jit_ctx *jit) in zend_jit_cannot_add_element_stub() argument
2543 jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(jit_FP(jit), ref), ir_CONST_U32(IS_UNDEF)); in zend_jit_cannot_add_element_stub()
2548 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), in zend_jit_cannot_add_element_stub()
2556 static int zend_jit_assign_const_stub(zend_jit_ctx *jit) in zend_jit_assign_const_stub() argument
2566 jit, NULL, in zend_jit_assign_const_stub()
2576 static int zend_jit_assign_tmp_stub(zend_jit_ctx *jit) in zend_jit_assign_tmp_stub() argument
2586 jit, NULL, in zend_jit_assign_tmp_stub()
2596 static int zend_jit_assign_var_stub(zend_jit_ctx *jit) in zend_jit_assign_var_stub() argument
2606 jit, NULL, in zend_jit_assign_var_stub()
2616 static int zend_jit_assign_cv_noref_stub(zend_jit_ctx *jit) in zend_jit_assign_cv_noref_stub() argument
2626 jit, NULL, in zend_jit_assign_cv_noref_stub()
2636 static int zend_jit_new_array_stub(zend_jit_ctx *jit) in zend_jit_new_array_stub() argument
2642 jit_set_Z_PTR(jit, var_addr, ref); in zend_jit_new_array_stub()
2643 jit_set_Z_TYPE_INFO(jit, var_addr, IS_ARRAY_EX); in zend_jit_new_array_stub()
2648 static int zend_jit_assign_cv_stub(zend_jit_ctx *jit) in zend_jit_assign_cv_stub() argument
2658 jit, NULL, in zend_jit_assign_cv_stub()
2668 static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags) in zend_jit_init_ctx() argument
2675 ir_init(&jit->ctx, flags, 256, 1024); in zend_jit_init_ctx()
2676 jit->ctx.ret_type = -1; in zend_jit_init_ctx()
2679 jit->ctx.mflags |= default_mflags; in zend_jit_init_ctx()
2681 jit->ctx.mflags |= IR_X86_AVX; in zend_jit_init_ctx()
2684 jit->ctx.get_veneer = zend_jit_get_veneer; in zend_jit_init_ctx()
2685 jit->ctx.set_veneer = zend_jit_set_veneer; in zend_jit_init_ctx()
2688 jit->ctx.fixed_regset = (1<<ZREG_FP) | (1<<ZREG_IP); in zend_jit_init_ctx()
2690 jit->ctx.flags |= IR_NO_STACK_COMBINE; in zend_jit_init_ctx()
2692 jit->ctx.flags |= IR_FUNCTION; in zend_jit_init_ctx()
2696 jit->ctx.flags |= IR_USE_FRAME_POINTER; in zend_jit_init_ctx()
2697 …jit->ctx.fixed_stack_frame_size = sizeof(void*) * 16; /* 10 saved registers and 6 spill slots (8 b… in zend_jit_init_ctx()
2699 …jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 8 saved registers and 3 spill slots (8 by… in zend_jit_init_ctx()
2701 …jit->ctx.fixed_stack_frame_size = sizeof(void*) * 9; /* 6 saved registers and 3 spill slots (8 by… in zend_jit_init_ctx()
2703 …jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 4 saved registers and 7 spill slots (4 by… in zend_jit_init_ctx()
2706 jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED & ~((1<<ZREG_FP) | (1<<ZREG_IP)); in zend_jit_init_ctx()
2708 jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED; in zend_jit_init_ctx()
2714 jit->ctx.fixed_call_stack_size = 16 + IR_SHADOW_ARGS; in zend_jit_init_ctx()
2716 jit->ctx.fixed_call_stack_size = 16; in zend_jit_init_ctx()
2720 jit->ctx.fixed_stack_red_zone = ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE; in zend_jit_init_ctx()
2721 if (jit->ctx.fixed_stack_red_zone > 16) { in zend_jit_init_ctx()
2722 jit->ctx.fixed_stack_frame_size = jit->ctx.fixed_stack_red_zone - 16; in zend_jit_init_ctx()
2723 jit->ctx.fixed_call_stack_size = 16; in zend_jit_init_ctx()
2725 jit->ctx.flags |= IR_MERGE_EMPTY_ENTRIES; in zend_jit_init_ctx()
2727 jit->ctx.fixed_stack_red_zone = 0; in zend_jit_init_ctx()
2728 jit->ctx.fixed_stack_frame_size = 32; /* 4 spill slots (8 bytes) or 8 spill slots (4 bytes) */ in zend_jit_init_ctx()
2729 jit->ctx.fixed_call_stack_size = 16; in zend_jit_init_ctx()
2732 jit->ctx.fixed_regset |= (1<<IR_REG_FP); /* prevent %rbp (%r5) usage */ in zend_jit_init_ctx()
2737 jit->ctx.snapshot_create = (ir_snapshot_create_t)jit_SNAPSHOT; in zend_jit_init_ctx()
2739 jit->op_array = NULL; in zend_jit_init_ctx()
2740 jit->current_op_array = NULL; in zend_jit_init_ctx()
2741 jit->ssa = NULL; in zend_jit_init_ctx()
2742 jit->name = NULL; in zend_jit_init_ctx()
2743 jit->last_valid_opline = NULL; in zend_jit_init_ctx()
2744 jit->use_last_valid_opline = 0; in zend_jit_init_ctx()
2745 jit->track_last_valid_opline = 0; in zend_jit_init_ctx()
2746 jit->reuse_ip = 0; in zend_jit_init_ctx()
2747 jit->delayed_call_level = 0; in zend_jit_init_ctx()
2749 jit->b = -1; in zend_jit_init_ctx()
2751 jit->tls = IR_UNUSED; in zend_jit_init_ctx()
2753 jit->fp = IR_UNUSED; in zend_jit_init_ctx()
2754 jit->trace_loop_ref = IR_UNUSED; in zend_jit_init_ctx()
2755 jit->return_inputs = IR_UNUSED; in zend_jit_init_ctx()
2756 jit->bb_start_ref = NULL; in zend_jit_init_ctx()
2757 jit->bb_predecessors = NULL; in zend_jit_init_ctx()
2758 jit->bb_edges = NULL; in zend_jit_init_ctx()
2759 jit->trace = NULL; in zend_jit_init_ctx()
2760 jit->ra = NULL; in zend_jit_init_ctx()
2761 jit->delay_var = -1; in zend_jit_init_ctx()
2762 jit->delay_refs = NULL; in zend_jit_init_ctx()
2763 jit->eg_exception_addr = 0; in zend_jit_init_ctx()
2764 zend_hash_init(&jit->addr_hash, 64, NULL, NULL, 0); in zend_jit_init_ctx()
2765 memset(jit->stub_addr, 0, sizeof(jit->stub_addr)); in zend_jit_init_ctx()
2770 static int zend_jit_free_ctx(zend_jit_ctx *jit) in zend_jit_free_ctx() argument
2772 if (jit->name) { in zend_jit_free_ctx()
2773 zend_string_release(jit->name); in zend_jit_free_ctx()
2775 zend_hash_destroy(&jit->addr_hash); in zend_jit_free_ctx()
2776 ir_free(&jit->ctx); in zend_jit_free_ctx()
2883 zend_jit_ctx jit; in zend_jit_setup_stubs() local
2889 zend_jit_init_ctx(&jit, zend_jit_stubs[i].flags); in zend_jit_setup_stubs()
2891 if (!zend_jit_stubs[i].stub(&jit)) { in zend_jit_setup_stubs()
2892 zend_jit_free_ctx(&jit); in zend_jit_setup_stubs()
2897 entry = zend_jit_ir_compile(&jit.ctx, &size, zend_jit_stubs[i].name); in zend_jit_setup_stubs()
2899 zend_jit_free_ctx(&jit); in zend_jit_setup_stubs()
2912 entry, size, (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0, &jit.ctx, stderr); in zend_jit_setup_stubs()
2930 zend_jit_free_ctx(&jit); in zend_jit_setup_stubs()
3145 zend_jit_ctx *jit = &jit_ctx; in zend_jit_calc_trace_prologue_size() local
3149 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET); in zend_jit_calc_trace_prologue_size()
3153 jit_STORE_FP(jit, ref); in zend_jit_calc_trace_prologue_size()
3154 jit->ctx.flags |= IR_FASTCALL_FUNC; in zend_jit_calc_trace_prologue_size()
3159 entry = zend_jit_ir_compile(&jit->ctx, &size, "JIT$trace_prologue"); in zend_jit_calc_trace_prologue_size()
3160 zend_jit_free_ctx(jit); in zend_jit_calc_trace_prologue_size()
3536 static ir_ref jit_IF_ex(zend_jit_ctx *jit, ir_ref condition, ir_ref true_block) in jit_IF_ex() argument
3544 ir_set_op(&jit->ctx, ref, 3, true_block); in jit_IF_ex()
3548 static void jit_IF_TRUE_FALSE_ex(zend_jit_ctx *jit, ir_ref if_ref, ir_ref true_block) in jit_IF_TRUE_FALSE_ex() argument
3552 ZEND_ASSERT(jit->ctx.ir_base[if_ref].op == IR_IF); in jit_IF_TRUE_FALSE_ex()
3553 ZEND_ASSERT(jit->ctx.ir_base[if_ref].op3); in jit_IF_TRUE_FALSE_ex()
3554 if (jit->ctx.ir_base[if_ref].op3 == true_block) { in jit_IF_TRUE_FALSE_ex()
3561 static void _zend_jit_add_predecessor_ref(zend_jit_ctx *jit, int b, int pred, ir_ref ref) in _zend_jit_add_predecessor_ref() argument
3568 bb = &jit->ssa->cfg.blocks[b]; in _zend_jit_add_predecessor_ref()
3569 p = &jit->ssa->cfg.predecessors[bb->predecessor_offset]; in _zend_jit_add_predecessor_ref()
3570 r = &jit->bb_edges[jit->bb_predecessors[b]]; in _zend_jit_add_predecessor_ref()
3574 header = jit->bb_start_ref[b]; in _zend_jit_add_predecessor_ref()
3577 ZEND_ASSERT(jit->ctx.ir_base[header].op == IR_LOOP_BEGIN); in _zend_jit_add_predecessor_ref()
3578 if (jit->ctx.ir_base[ref].op == IR_END) { in _zend_jit_add_predecessor_ref()
3579 jit->ctx.ir_base[ref].op = IR_LOOP_END; in _zend_jit_add_predecessor_ref()
3580 } else if (jit->ctx.ir_base[ref].op == IR_IF) { in _zend_jit_add_predecessor_ref()
3581 jit_IF_TRUE_FALSE_ex(jit, ref, b); in _zend_jit_add_predecessor_ref()
3583 } else if (jit->ctx.ir_base[ref].op == IR_UNREACHABLE) { in _zend_jit_add_predecessor_ref()
3598 static void _zend_jit_merge_smart_branch_inputs(zend_jit_ctx *jit, in _zend_jit_merge_smart_branch_inputs() argument
3608 ZEND_ASSERT(jit->ctx.ir_base[true_inputs].op == IR_END); in _zend_jit_merge_smart_branch_inputs()
3609 if (!jit->ctx.ir_base[true_inputs].op2) { in _zend_jit_merge_smart_branch_inputs()
3617 ZEND_ASSERT(jit->ctx.ir_base[false_inputs].op == IR_END); in _zend_jit_merge_smart_branch_inputs()
3618 if (!jit->ctx.ir_base[false_inputs].op2) { in _zend_jit_merge_smart_branch_inputs()
3628 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END()); in _zend_jit_merge_smart_branch_inputs()
3632 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path); in _zend_jit_merge_smart_branch_inputs()
3635 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path); in _zend_jit_merge_smart_branch_inputs()
3638 _zend_jit_add_predecessor_ref(jit, false_label, jit->b, false_path); in _zend_jit_merge_smart_branch_inputs()
3642 jit->b = -1; in _zend_jit_merge_smart_branch_inputs()
3645 static void _zend_jit_fix_merges(zend_jit_ctx *jit) in _zend_jit_fix_merges() argument
3653 count = jit->ssa->cfg.blocks_count; in _zend_jit_fix_merges()
3654 for (i = 0, p = jit->bb_start_ref; i < count; i++, p++) { in _zend_jit_fix_merges()
3657 insn = &jit->ctx.ir_base[ref]; in _zend_jit_fix_merges()
3720 static void zend_jit_case_start(zend_jit_ctx *jit, int switch_b, int case_b, ir_ref switch_ref) in zend_jit_case_start() argument
3722 zend_basic_block *bb = &jit->ssa->cfg.blocks[switch_b]; in zend_jit_case_start()
3723 const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1]; in zend_jit_case_start()
3730 int default_b = jit->ssa->cfg.map[default_opline - jit->op_array->opcodes]; in zend_jit_case_start()
3737 int b = jit->ssa->cfg.map[target - jit->op_array->opcodes]; in zend_jit_case_start()
3756 if (jit->ctx.ir_base[switch_ref].op3) { in zend_jit_case_start()
3758 ir_ref ref = jit->ctx.ir_base[switch_ref].op3; in zend_jit_case_start()
3759 jit->ctx.ir_base[switch_ref].op3 = IS_UNDEF; in zend_jit_case_start()
3760 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END); in zend_jit_case_start()
3762 while (jit->ctx.ir_base[end].op2) { in zend_jit_case_start()
3763 ZEND_ASSERT(jit->ctx.ir_base[end].op == IR_END); in zend_jit_case_start()
3764 end = jit->ctx.ir_base[end].op2; in zend_jit_case_start()
3766 jit->ctx.ir_base[end].op2 = list; in zend_jit_case_start()
3780 static int zend_jit_bb_start(zend_jit_ctx *jit, int b) in zend_jit_bb_start() argument
3787 ZEND_ASSERT(b < jit->ssa->cfg.blocks_count); in zend_jit_bb_start()
3788 bb = &jit->ssa->cfg.blocks[b]; in zend_jit_bb_start()
3794 ZEND_ASSERT(jit->ctx.control); in zend_jit_bb_start()
3796 ref = jit->ctx.control; in zend_jit_bb_start()
3797 ir_insn *insn = &jit->ctx.ir_base[ref]; in zend_jit_bb_start()
3800 insn = &jit->ctx.ir_base[ref]; in zend_jit_bb_start()
3806 if (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY) { in zend_jit_bb_start()
3808 jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ir_END()); in zend_jit_bb_start()
3809 bb_start = jit->ctx.control; in zend_jit_bb_start()
3812 ZEND_ASSERT(!jit->ctx.control); in zend_jit_bb_start()
3813 pred = jit->ssa->cfg.predecessors[bb->predecessor_offset]; in zend_jit_bb_start()
3814 ref = jit->bb_edges[jit->bb_predecessors[b]]; in zend_jit_bb_start()
3816 if (!jit->ctx.control) { in zend_jit_bb_start()
3820 ir_op op = jit->ctx.ir_base[ref].op; in zend_jit_bb_start()
3823 if (!jit->ctx.control) { in zend_jit_bb_start()
3824 jit_IF_TRUE_FALSE_ex(jit, ref, b); in zend_jit_bb_start()
3827 jit_IF_TRUE_FALSE_ex(jit, ref, b); in zend_jit_bb_start()
3831 zend_jit_case_start(jit, pred, b, ref); in zend_jit_bb_start()
3833 if (!jit->ctx.control) { in zend_jit_bb_start()
3835 if ((jit->ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY) in zend_jit_bb_start()
3836 && (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)) { in zend_jit_bb_start()
3838 jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ref); in zend_jit_bb_start()
3847 bb_start = jit->ctx.control; in zend_jit_bb_start()
3855 ZEND_ASSERT(!jit->ctx.control); in zend_jit_bb_start()
3856 if (jit->ctx.control) { in zend_jit_bb_start()
3860 for (i = 0, p = jit->ssa->cfg.predecessors + bb->predecessor_offset; i < n; p++, i++) { in zend_jit_bb_start()
3862 if (jit->bb_start_ref[pred]) { in zend_jit_bb_start()
3865 ref = jit->bb_edges[jit->bb_predecessors[b] + i]; in zend_jit_bb_start()
3870 ir_op op = jit->ctx.ir_base[ref].op; in zend_jit_bb_start()
3873 jit_IF_TRUE_FALSE_ex(jit, ref, b); in zend_jit_bb_start()
3876 zend_jit_case_start(jit, pred, b, ref); in zend_jit_bb_start()
3894 jit->ctx.ir_base[jit->ctx.control].op = IR_LOOP_BEGIN; in zend_jit_bb_start()
3895 bb_start = jit->ctx.control; in zend_jit_bb_start()
3903 bb_start = jit->ctx.control; in zend_jit_bb_start()
3910 jit->b = b; in zend_jit_bb_start()
3911 jit->bb_start_ref[b] = bb_start; in zend_jit_bb_start()
3913 …if ((bb->flags & ZEND_BB_ENTRY) || (bb->idom >= 0 && jit->bb_start_ref[bb->idom] < jit->ctx.fold_c… in zend_jit_bb_start()
3914 jit->ctx.fold_cse_limit = bb_start; in zend_jit_bb_start()
3920 static int zend_jit_bb_end(zend_jit_ctx *jit, int b) in zend_jit_bb_end() argument
3926 if (jit->b != b) { in zend_jit_bb_end()
3930 bb = &jit->ssa->cfg.blocks[b]; in zend_jit_bb_end()
3935 const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1]; in zend_jit_bb_end()
3944 _zend_jit_add_predecessor_ref(jit, succ, b, ir_END()); in zend_jit_bb_end()
3945 jit->b = -1; in zend_jit_bb_end()
3949 static int jit_CMP_IP(zend_jit_ctx *jit, ir_op op, const zend_op *next_opline) in jit_CMP_IP() argument
3955 ref = jit_IP32(jit); in jit_CMP_IP()
3962 ref = jit_IP(jit); in jit_CMP_IP()
3972 zend_jit_ctx *jit, in zend_jit_jmp_frameless() argument
3998 ZEND_ASSERT(jit->b >= 0); in zend_jit_jmp_frameless()
3999 bb = &jit->ssa->cfg.blocks[jit->b]; in zend_jit_jmp_frameless()
4001 ref = jit_IF_ex(jit, ir_EQ(phi_result, ir_CONST_LONG(ZEND_JMP_FL_HIT)), bb->successors[0]); in zend_jit_jmp_frameless()
4002 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); in zend_jit_jmp_frameless()
4003 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref); in zend_jit_jmp_frameless()
4004 jit->b = -1; in zend_jit_jmp_frameless()
4010 static int zend_jit_cond_jmp(zend_jit_ctx *jit, const zend_op *next_opline, int target_block) in zend_jit_cond_jmp() argument
4015 ZEND_ASSERT(jit->b >= 0); in zend_jit_cond_jmp()
4016 bb = &jit->ssa->cfg.blocks[jit->b]; in zend_jit_cond_jmp()
4020 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END()); in zend_jit_cond_jmp()
4021 jit->b = -1; in zend_jit_cond_jmp()
4022 zend_jit_set_last_valid_opline(jit, next_opline); in zend_jit_cond_jmp()
4026 ref = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, next_opline), target_block); in zend_jit_cond_jmp()
4028 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); in zend_jit_cond_jmp()
4029 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref); in zend_jit_cond_jmp()
4031 jit->b = -1; in zend_jit_cond_jmp()
4032 zend_jit_set_last_valid_opline(jit, next_opline); in zend_jit_cond_jmp()
4037 static int zend_jit_set_cond(zend_jit_ctx *jit, const zend_op *next_opline, uint32_t var) in zend_jit_set_cond() argument
4041 ref = ir_ADD_U32(ir_ZEXT_U32(jit_CMP_IP(jit, IR_EQ, next_opline)), ir_CONST_U32(IS_FALSE)); in zend_jit_set_cond()
4044 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), var + offsetof(zval, u1.type_info)), ref); in zend_jit_set_cond()
4046 zend_jit_reset_last_valid_opline(jit); in zend_jit_set_cond()
4047 return zend_jit_set_ip(jit, next_opline - 1); in zend_jit_set_cond()
4051 static void zend_jit_check_exception(zend_jit_ctx *jit) in zend_jit_check_exception() argument
4053 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), in zend_jit_check_exception()
4054 jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_check_exception()
4057 static void zend_jit_check_exception_undef_result(zend_jit_ctx *jit, const zend_op *opline) in zend_jit_check_exception_undef_result() argument
4059 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), in zend_jit_check_exception_undef_result()
4060 jit_STUB_ADDR(jit, in zend_jit_check_exception_undef_result()
4064 static void zend_jit_type_check_undef(zend_jit_ctx *jit, in zend_jit_type_check_undef() argument
4080 jit_SET_EX_OPLINE(jit, opline); in zend_jit_type_check_undef()
4085 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_type_check_undef()
4087 zend_jit_check_exception(jit); in zend_jit_type_check_undef()
4093 static ir_ref zend_jit_zval_check_undef(zend_jit_ctx *jit, in zend_jit_zval_check_undef() argument
4101 if_def = ir_IF(jit_Z_TYPE_ref(jit, ref)); in zend_jit_zval_check_undef()
4105 jit_SET_EX_OPLINE(jit, opline); in zend_jit_zval_check_undef()
4111 zend_jit_check_exception(jit); in zend_jit_zval_check_undef()
4121 static void zend_jit_recv_entry(zend_jit_ctx *jit, int b) in zend_jit_recv_entry() argument
4123 zend_basic_block *bb = &jit->ssa->cfg.blocks[b]; in zend_jit_recv_entry()
4129 pred = jit->bb_predecessors[b]; in zend_jit_recv_entry()
4130 ref = jit->bb_edges[pred]; in zend_jit_recv_entry()
4133 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END); in zend_jit_recv_entry()
4139 ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM); in zend_jit_recv_entry()
4140 ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1); in zend_jit_recv_entry()
4141 jit_STORE_FP(jit, 2); in zend_jit_recv_entry()
4145 jit->bb_edges[pred] = ir_END(); in zend_jit_recv_entry()
4148 static void zend_jit_osr_entry(zend_jit_ctx *jit, int b) in zend_jit_osr_entry() argument
4150 zend_basic_block *bb = &jit->ssa->cfg.blocks[b]; in zend_jit_osr_entry()
4157 ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM); in zend_jit_osr_entry()
4158 ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1); in zend_jit_osr_entry()
4159 jit_STORE_FP(jit, 2); in zend_jit_osr_entry()
4165 static ir_ref zend_jit_continue_entry(zend_jit_ctx *jit, ir_ref src, unsigned int label) in zend_jit_continue_entry() argument
4170 ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM); in zend_jit_continue_entry()
4171 ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1); in zend_jit_continue_entry()
4172 jit_STORE_FP(jit, 2); in zend_jit_continue_entry()
4177 static int zend_jit_handler(zend_jit_ctx *jit, const zend_op *opline, int may_throw) in zend_jit_handler() argument
4182 zend_jit_set_ip(jit, opline); in zend_jit_handler()
4191 ref = jit_FP(jit); in zend_jit_handler()
4195 zend_jit_check_exception(jit); in zend_jit_handler()
4207 zend_jit_set_last_valid_opline(jit, opline + 2); in zend_jit_handler()
4210 zend_jit_set_last_valid_opline(jit, opline + 1); in zend_jit_handler()
4216 static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline) in zend_jit_tail_handler() argument
4222 zend_jit_set_ip(jit, opline); in zend_jit_tail_handler()
4235 ref = ir_LOAD_A(jit_IP(jit)); in zend_jit_tail_handler()
4242 } else if ((jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY) in zend_jit_tail_handler()
4249 ref = jit_FP(jit); in zend_jit_tail_handler()
4253 ref = jit_FP(jit); in zend_jit_tail_handler()
4257 if (jit->b >= 0) { in zend_jit_tail_handler()
4258 bb = &jit->ssa->cfg.blocks[jit->b]; in zend_jit_tail_handler()
4273 ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY); in zend_jit_tail_handler()
4277 succ = jit->b + 1; in zend_jit_tail_handler()
4279 jit->ssa->cfg.blocks[succ].flags |= ZEND_BB_ENTRY; in zend_jit_tail_handler()
4281 ref = jit->ctx.insns_count - 1; in zend_jit_tail_handler()
4282 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE || jit->ctx.ir_base[ref].op == IR_RETURN); in zend_jit_tail_handler()
4283 ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start); in zend_jit_tail_handler()
4284 _zend_jit_add_predecessor_ref(jit, succ, jit->b, ref); in zend_jit_tail_handler()
4286 jit->b = -1; in zend_jit_tail_handler()
4287 zend_jit_reset_last_valid_opline(jit); in zend_jit_tail_handler()
4292 static int zend_jit_call(zend_jit_ctx *jit, const zend_op *opline, unsigned int next_block) in zend_jit_call() argument
4294 return zend_jit_tail_handler(jit, opline); in zend_jit_call()
4297 static int zend_jit_spill_store(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t i… in zend_jit_spill_store() argument
4303 jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src)); in zend_jit_spill_store()
4308 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_spill_store()
4311 jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src)); in zend_jit_spill_store()
4316 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_spill_store()
4324 static int zend_jit_spill_store_inv(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32… in zend_jit_spill_store_inv() argument
4330 jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src)); in zend_jit_spill_store_inv()
4332 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_spill_store_inv()
4336 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_spill_store_inv()
4339 jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src)); in zend_jit_spill_store_inv()
4341 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_spill_store_inv()
4345 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_spill_store_inv()
4353 static int zend_jit_load_reg(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info) in zend_jit_load_reg() argument
4359 zend_jit_def_reg(jit, dst, jit_Z_LVAL(jit, src)); in zend_jit_load_reg()
4361 zend_jit_def_reg(jit, dst, jit_Z_DVAL(jit, src)); in zend_jit_load_reg()
4368 static int zend_jit_store_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var, bool set_type) in zend_jit_store_var() argument
4373 return zend_jit_spill_store(jit, src, dst, info, set_type); in zend_jit_store_var()
4376 static int zend_jit_store_ref(zend_jit_ctx *jit, uint32_t info, int var, int32_t src, bool set_type) in zend_jit_store_ref() argument
4381 jit_set_Z_LVAL(jit, dst, src); in zend_jit_store_ref()
4386 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_store_ref()
4389 jit_set_Z_DVAL(jit, dst, src); in zend_jit_store_ref()
4394 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_store_ref()
4402 static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t reg) in zend_jit_deopt_rload() argument
4404 ir_ref ref = jit->ctx.control; in zend_jit_deopt_rload()
4408 insn = &jit->ctx.ir_base[ref]; in zend_jit_deopt_rload()
4420 static int zend_jit_store_const_long(zend_jit_ctx *jit, int var, zend_long val) in zend_jit_store_const_long() argument
4425 if (jit->ra && jit->ra[var].ref == IR_NULL) { in zend_jit_store_const_long()
4426 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); in zend_jit_store_const_long()
4428 jit_set_Z_LVAL(jit, dst, src); in zend_jit_store_const_long()
4429 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_store_const_long()
4433 static int zend_jit_store_const_double(zend_jit_ctx *jit, int var, double val) in zend_jit_store_const_double() argument
4438 if (jit->ra && jit->ra[var].ref == IR_NULL) { in zend_jit_store_const_double()
4439 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); in zend_jit_store_const_double()
4441 jit_set_Z_DVAL(jit, dst, src); in zend_jit_store_const_double()
4442 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_store_const_double()
4446 static int zend_jit_store_type(zend_jit_ctx *jit, int var, uint8_t type) in zend_jit_store_type() argument
4451 jit_set_Z_TYPE_INFO(jit, dst, type); in zend_jit_store_type()
4455 static int zend_jit_store_reg(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, bool in_mem, b… in zend_jit_store_reg() argument
4463 src = zend_jit_deopt_rload(jit, type, reg); in zend_jit_store_reg()
4464 if (jit->ra && jit->ra[var].ref == IR_NULL) { in zend_jit_store_reg()
4465 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); in zend_jit_store_reg()
4467 jit_set_Z_LVAL(jit, dst, src); in zend_jit_store_reg()
4472 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_store_reg()
4477 src = zend_jit_deopt_rload(jit, type, reg); in zend_jit_store_reg()
4478 if (jit->ra && jit->ra[var].ref == IR_NULL) { in zend_jit_store_reg()
4479 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); in zend_jit_store_reg()
4481 jit_set_Z_DVAL(jit, dst, src); in zend_jit_store_reg()
4486 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_store_reg()
4495 static int zend_jit_store_spill_slot(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, int32_t… in zend_jit_store_spill_slot() argument
4502 if (jit->ra && jit->ra[var].ref == IR_NULL) { in zend_jit_store_spill_slot()
4503 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); in zend_jit_store_spill_slot()
4505 jit_set_Z_LVAL(jit, dst, src); in zend_jit_store_spill_slot()
4510 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_store_spill_slot()
4515 if (jit->ra && jit->ra[var].ref == IR_NULL) { in zend_jit_store_spill_slot()
4516 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); in zend_jit_store_spill_slot()
4518 jit_set_Z_DVAL(jit, dst, src); in zend_jit_store_spill_slot()
4523 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_store_spill_slot()
4532 static int zend_jit_store_var_type(zend_jit_ctx *jit, int var, uint32_t type) in zend_jit_store_var_type() argument
4536 jit_set_Z_TYPE_INFO(jit, dst, type); in zend_jit_store_var_type()
4540 static int zend_jit_zval_try_addref(zend_jit_ctx *jit, zend_jit_addr var_addr) in zend_jit_zval_try_addref() argument
4544 if_refcounted = jit_if_REFCOUNTED(jit, var_addr); in zend_jit_zval_try_addref()
4548 jit_GC_ADDREF(jit, jit_Z_PTR(jit, var_addr)); in zend_jit_zval_try_addref()
4553 static int zend_jit_store_var_if_necessary(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t … in zend_jit_store_var_if_necessary() argument
4557 return zend_jit_spill_store(jit, src, dst, info, 1); in zend_jit_store_var_if_necessary()
4562 static int zend_jit_store_var_if_necessary_ex(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32… in zend_jit_store_var_if_necessary_ex() argument
4583 return zend_jit_spill_store(jit, src, dst, info, set_type); in zend_jit_store_var_if_necessary_ex()
4588 static int zend_jit_load_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var) in zend_jit_load_var() argument
4593 return zend_jit_load_reg(jit, src, dst, info); in zend_jit_load_var()
4596 static int zend_jit_invalidate_var_if_necessary(zend_jit_ctx *jit, uint8_t op_type, zend_jit_addr a… in zend_jit_invalidate_var_if_necessary() argument
4601 jit_set_Z_TYPE_INFO(jit, dst, IS_UNDEF); in zend_jit_invalidate_var_if_necessary()
4606 static int zend_jit_update_regs(zend_jit_ctx *jit, uint32_t var, zend_jit_addr src, zend_jit_addr d… in zend_jit_update_regs() argument
4611 zend_jit_def_reg(jit, dst, zend_jit_use_reg(jit, src)); in zend_jit_update_regs()
4615 if (!zend_jit_spill_store(jit, dst, var_addr, info, in zend_jit_update_regs()
4626 if (!zend_jit_spill_store(jit, src, dst, info, in zend_jit_update_regs()
4640 if (!zend_jit_load_reg(jit, src, dst, info)) { in zend_jit_update_regs()
4651 if (!zend_jit_spill_store(jit, src, dst, info, in zend_jit_update_regs()
4668 …ll_is_unobserved_data jit_observer_fcall_is_unobserved_start(zend_jit_ctx *jit, const zend_functio… in jit_observer_fcall_is_unobserved_start() argument
4743 static void jit_observer_fcall_is_unobserved_end(zend_jit_ctx *jit, struct jit_observer_fcall_is_un… in jit_observer_fcall_is_unobserved_end() argument
4750 static void jit_observer_fcall_begin(zend_jit_ctx *jit, ir_ref rx, ir_ref observer_handler) { in jit_observer_fcall_begin() argument
4754 static void jit_observer_fcall_end(zend_jit_ctx *jit, ir_ref rx, ir_ref res_ref) { in jit_observer_fcall_end() argument
4763 static int zend_jit_inc_dec(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_a… in zend_jit_inc_dec() argument
4771 if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_inc_dec()
4775 op1_lval_ref = jit_Z_LVAL(jit, op1_addr); in zend_jit_inc_dec()
4776 jit_set_Z_LVAL(jit, res_addr, op1_lval_ref); in zend_jit_inc_dec()
4778 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_inc_dec()
4785 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_LONG); in zend_jit_inc_dec()
4793 op1_lval_ref = jit_Z_LVAL(jit, op1_addr); in zend_jit_inc_dec()
4797 jit_set_Z_LVAL(jit, op1_def_addr, ref); in zend_jit_inc_dec()
4838 jit_set_Z_LVAL(jit, res_addr, ref); in zend_jit_inc_dec()
4840 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_inc_dec()
4872 jit_set_Z_LVAL(jit, res_addr, ref); in zend_jit_inc_dec()
4874 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_inc_dec()
4876 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_inc_dec()
4889 jit_set_Z_LVAL(jit, res_addr, ref); in zend_jit_inc_dec()
4891 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_inc_dec()
4902 jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); in zend_jit_inc_dec()
4905 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0)); in zend_jit_inc_dec()
4906 jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0x41e00000)); in zend_jit_inc_dec()
4908 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x43e0000000000000)); in zend_jit_inc_dec()
4910 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE); in zend_jit_inc_dec()
4914 jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); in zend_jit_inc_dec()
4917 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x00200000)); in zend_jit_inc_dec()
4918 jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0xc1e00000)); in zend_jit_inc_dec()
4920 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0xc3e0000000000000)); in zend_jit_inc_dec()
4922 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE); in zend_jit_inc_dec()
4929 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); in zend_jit_inc_dec()
4932 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0)); in zend_jit_inc_dec()
4933 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000)); in zend_jit_inc_dec()
4935 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000)); in zend_jit_inc_dec()
4937 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_inc_dec()
4941 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); in zend_jit_inc_dec()
4944 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000)); in zend_jit_inc_dec()
4945 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000)); in zend_jit_inc_dec()
4947 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000)); in zend_jit_inc_dec()
4949 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_inc_dec()
4961 jit_set_Z_LVAL(jit, res_addr, ref); in zend_jit_inc_dec()
4963 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_inc_dec()
4973 jit_SET_EX_OPLINE(jit, opline); in zend_jit_inc_dec()
4977 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF); in zend_jit_inc_dec()
4983 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_NULL); in zend_jit_inc_dec()
4989 ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_inc_dec()
4994 if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE)); in zend_jit_inc_dec()
4996 ref2 = jit_Z_PTR_ref(jit, ref); in zend_jit_inc_dec()
4998 if_typed = jit_if_TYPED_REF(jit, ref2); in zend_jit_inc_dec()
5003 arg2 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_inc_dec()
5020 zend_jit_check_exception(jit); in zend_jit_inc_dec()
5030 jit_ZVAL_COPY(jit, in zend_jit_inc_dec()
5037 ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_inc_dec()
5044 ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_inc_dec()
5051 zend_jit_check_exception(jit); in zend_jit_inc_dec()
5054 ref = jit_Z_DVAL(jit, op1_addr); in zend_jit_inc_dec()
5056 jit_set_Z_DVAL(jit, res_addr, ref); in zend_jit_inc_dec()
5057 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_inc_dec()
5065 jit_set_Z_DVAL(jit, op1_def_addr, ref); in zend_jit_inc_dec()
5068 jit_set_Z_DVAL(jit, res_addr, ref); in zend_jit_inc_dec()
5069 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_inc_dec()
5075 …if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_def_addr, op1_def_info, op1_addr… in zend_jit_inc_dec()
5079 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { in zend_jit_inc_dec()
5086 static int zend_jit_math_long_long(zend_jit_ctx *jit, in zend_jit_math_long_long() argument
5109 op1 = jit_Z_LVAL(jit, op1_addr); in zend_jit_math_long_long()
5110 op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr); in zend_jit_math_long_long()
5163 jit_set_Z_LVAL(jit, res_addr, ref); in zend_jit_math_long_long()
5168 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_math_long_long()
5184 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); in zend_jit_math_long_long()
5187 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0)); in zend_jit_math_long_long()
5188 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000)); in zend_jit_math_long_long()
5190 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000)); in zend_jit_math_long_long()
5192 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_math_long_long()
5203 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); in zend_jit_math_long_long()
5206 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000)); in zend_jit_math_long_long()
5207 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000)); in zend_jit_math_long_long()
5209 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000)); in zend_jit_math_long_long()
5211 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_math_long_long()
5226 op1 = jit_Z_LVAL(jit, op1_addr); in zend_jit_math_long_long()
5227 op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr); in zend_jit_math_long_long()
5231 ir_ref old_cse_limit = jit->ctx.fold_cse_limit; in zend_jit_math_long_long()
5232 jit->ctx.fold_cse_limit = 0x7fffffff; in zend_jit_math_long_long()
5237 jit->ctx.fold_cse_limit = old_cse_limit; in zend_jit_math_long_long()
5240 jit_set_Z_DVAL(jit, res_addr, ref); in zend_jit_math_long_long()
5242 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_math_long_long()
5252 static int zend_jit_math_long_double(zend_jit_ctx *jit, in zend_jit_math_long_double() argument
5273 op1 = jit_Z_LVAL(jit, op1_addr); in zend_jit_math_long_double()
5274 op2 = jit_Z_DVAL(jit, op2_addr); in zend_jit_math_long_double()
5276 jit_set_Z_DVAL(jit, res_addr, ref); in zend_jit_math_long_double()
5280 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_math_long_double()
5286 static int zend_jit_math_double_long(zend_jit_ctx *jit, in zend_jit_math_double_long() argument
5307 op1 = jit_Z_DVAL(jit, op1_addr); in zend_jit_math_double_long()
5308 op2 = jit_Z_LVAL(jit, op2_addr); in zend_jit_math_double_long()
5310 jit_set_Z_DVAL(jit, res_addr, ref); in zend_jit_math_double_long()
5315 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_math_double_long()
5322 static int zend_jit_math_double_double(zend_jit_ctx *jit, in zend_jit_math_double_double() argument
5344 op1 = jit_Z_DVAL(jit, op1_addr); in zend_jit_math_double_double()
5345 op2 = (same_ops) ? op1 : jit_Z_DVAL(jit, op2_addr); in zend_jit_math_double_double()
5347 jit_set_Z_DVAL(jit, res_addr, ref); in zend_jit_math_double_double()
5352 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_math_double_double()
5359 static int zend_jit_math_helper(zend_jit_ctx *jit, in zend_jit_math_helper() argument
5393 if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) { in zend_jit_math_helper()
5395 zend_jit_use_reg(jit, op1_addr); in zend_jit_math_helper()
5398 if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) { in zend_jit_math_helper()
5400 zend_jit_use_reg(jit, op2_addr); in zend_jit_math_helper()
5405 jit->delay_var = Z_SSA_VAR(res_addr); in zend_jit_math_helper()
5406 jit->delay_refs = res_inputs; in zend_jit_math_helper()
5411 if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_math_helper()
5415 if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_math_helper()
5418 …if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_… in zend_jit_math_helper()
5432 if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_math_helper()
5436 if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_math_helper()
5440 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_math_helper()
5445 if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5454 …if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_… in zend_jit_math_helper()
5465 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); in zend_jit_math_helper()
5472 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_math_helper()
5475 if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5485 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_math_helper()
5490 if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5505 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); in zend_jit_math_helper()
5512 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_math_helper()
5515 if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5525 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_math_helper()
5530 if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5542 if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_math_helper()
5549 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); in zend_jit_math_helper()
5552 if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5562 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_math_helper()
5567 if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5586 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { in zend_jit_math_helper()
5593 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { in zend_jit_math_helper()
5599 arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var)); in zend_jit_math_helper()
5601 arg1 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_math_helper()
5603 arg2 = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_math_helper()
5604 arg3 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_math_helper()
5605 jit_SET_EX_OPLINE(jit, opline); in zend_jit_math_helper()
5619 jit_FREE_OP(jit, op1_type, op1, op1_info, NULL); in zend_jit_math_helper()
5620 jit_FREE_OP(jit, op2_type, op2, op2_info, NULL); in zend_jit_math_helper()
5624 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), in zend_jit_math_helper()
5625 jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); in zend_jit_math_helper()
5627 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_math_helper()
5629 zend_jit_check_exception(jit); in zend_jit_math_helper()
5634 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) { in zend_jit_math_helper()
5646 ZEND_ASSERT(jit->delay_refs == res_inputs); in zend_jit_math_helper()
5648 jit->delay_var = -1; in zend_jit_math_helper()
5649 jit->delay_refs = NULL; in zend_jit_math_helper()
5651 zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]); in zend_jit_math_helper()
5654 zend_jit_def_reg(jit, res_addr, phi); in zend_jit_math_helper()
5661 static int zend_jit_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr… in zend_jit_math() argument
5665 …if (!zend_jit_math_helper(jit, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op… in zend_jit_math()
5668 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { in zend_jit_math()
5674 static int zend_jit_add_arrays(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_ji… in zend_jit_add_arrays() argument
5677 ir_ref arg1 = jit_Z_PTR(jit, op1_addr); in zend_jit_add_arrays()
5678 ir_ref arg2 = jit_Z_PTR(jit, op2_addr); in zend_jit_add_arrays()
5681 jit_set_Z_PTR(jit, res_addr, ref); in zend_jit_add_arrays()
5682 jit_set_Z_TYPE_INFO(jit, res_addr, IS_ARRAY_EX); in zend_jit_add_arrays()
5683 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_add_arrays()
5684 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); in zend_jit_add_arrays()
5688 static int zend_jit_long_math_helper(zend_jit_ctx *jit, in zend_jit_long_math_helper() argument
5717 && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) { in zend_jit_long_math_helper()
5719 zend_jit_use_reg(jit, op1_addr); in zend_jit_long_math_helper()
5723 && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) { in zend_jit_long_math_helper()
5725 zend_jit_use_reg(jit, op2_addr); in zend_jit_long_math_helper()
5729 if_long1 = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_long_math_helper()
5733 if_long2 = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_long_math_helper()
5745 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); in zend_jit_long_math_helper()
5746 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); in zend_jit_long_math_helper()
5747 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
5748 ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift)); in zend_jit_long_math_helper()
5754 ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval)); in zend_jit_long_math_helper()
5757 ref = jit_Z_LVAL(jit, op2_addr); in zend_jit_long_math_helper()
5763 ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr); in zend_jit_long_math_helper()
5769 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
5770 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); in zend_jit_long_math_helper()
5771 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); in zend_jit_long_math_helper()
5772 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift)); in zend_jit_long_math_helper()
5781 ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ref); in zend_jit_long_math_helper()
5791 jit_Z_LVAL(jit, op1_addr), in zend_jit_long_math_helper()
5794 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); in zend_jit_long_math_helper()
5795 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); in zend_jit_long_math_helper()
5796 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
5797 ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift)); in zend_jit_long_math_helper()
5803 ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval)); in zend_jit_long_math_helper()
5806 ref = jit_Z_LVAL(jit, op2_addr); in zend_jit_long_math_helper()
5817 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
5818 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); in zend_jit_long_math_helper()
5819 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); in zend_jit_long_math_helper()
5820 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift)); in zend_jit_long_math_helper()
5828 ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ref); in zend_jit_long_math_helper()
5835 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); in zend_jit_long_math_helper()
5836 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); in zend_jit_long_math_helper()
5837 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
5838 ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_mod_by_zero)); in zend_jit_long_math_helper()
5843 ref = ir_AND_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval - 1)); in zend_jit_long_math_helper()
5845 ref = ir_MOD_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval)); in zend_jit_long_math_helper()
5849 ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr); in zend_jit_long_math_helper()
5851 ref = jit_Z_LVAL(jit, op2_addr); in zend_jit_long_math_helper()
5855 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
5856 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); in zend_jit_long_math_helper()
5857 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); in zend_jit_long_math_helper()
5858 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_mod_by_zero)); in zend_jit_long_math_helper()
5889 op1 = jit_Z_LVAL(jit, op1_addr); in zend_jit_long_math_helper()
5890 op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr); in zend_jit_long_math_helper()
5898 jit->delay_var = Z_SSA_VAR(res_addr); in zend_jit_long_math_helper()
5899 jit->delay_refs = res_inputs; in zend_jit_long_math_helper()
5901 jit_set_Z_LVAL(jit, res_addr, ref); in zend_jit_long_math_helper()
5905 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_long_math_helper()
5931 ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_long_math_helper()
5932 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF); in zend_jit_long_math_helper()
5947 ref = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_long_math_helper()
5948 if_def = jit_if_not_Z_TYPE(jit, op2_addr, IS_UNDEF); in zend_jit_long_math_helper()
5962 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { in zend_jit_long_math_helper()
5969 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { in zend_jit_long_math_helper()
5975 arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var)); in zend_jit_long_math_helper()
5977 arg1 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_long_math_helper()
5979 arg2 = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_long_math_helper()
5980 arg3 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_long_math_helper()
5981 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
6004 jit_FREE_OP(jit, op1_type, op1, op1_info, NULL); in zend_jit_long_math_helper()
6005 jit_FREE_OP(jit, op2_type, op2, op2_info, NULL); in zend_jit_long_math_helper()
6009 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), in zend_jit_long_math_helper()
6010 jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); in zend_jit_long_math_helper()
6012 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_long_math_helper()
6014 zend_jit_check_exception(jit); in zend_jit_long_math_helper()
6020 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) { in zend_jit_long_math_helper()
6028 ZEND_ASSERT(jit->delay_refs == res_inputs); in zend_jit_long_math_helper()
6030 jit->delay_var = -1; in zend_jit_long_math_helper()
6031 jit->delay_refs = NULL; in zend_jit_long_math_helper()
6033 zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]); in zend_jit_long_math_helper()
6036 zend_jit_def_reg(jit, res_addr, phi); in zend_jit_long_math_helper()
6044 static int zend_jit_long_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_ssa… in zend_jit_long_math() argument
6048 if (!zend_jit_long_math_helper(jit, opline, opline->opcode, in zend_jit_long_math()
6054 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { in zend_jit_long_math()
6060 static int zend_jit_concat_helper(zend_jit_ctx *jit, in zend_jit_concat_helper() argument
6079 if_op1_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING); in zend_jit_concat_helper()
6083 if_op2_string = jit_if_Z_TYPE(jit, op2_addr, IS_STRING); in zend_jit_concat_helper()
6087 ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_concat_helper()
6088 ir_ref arg2 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_concat_helper()
6094 ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_concat_helper()
6095 ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_concat_helper()
6096 ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_concat_helper()
6106 jit_FREE_OP(jit, op2_type, op2, op2_info, opline); in zend_jit_concat_helper()
6123 ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_concat_helper()
6124 ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_concat_helper()
6125 ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_concat_helper()
6127 jit_SET_EX_OPLINE(jit, opline); in zend_jit_concat_helper()
6132 jit_FREE_OP(jit, op1_type, op1, op1_info, NULL); in zend_jit_concat_helper()
6133 jit_FREE_OP(jit, op2_type, op2, op2_info, NULL); in zend_jit_concat_helper()
6136 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), in zend_jit_concat_helper()
6137 jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); in zend_jit_concat_helper()
6139 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_concat_helper()
6141 zend_jit_check_exception(jit); in zend_jit_concat_helper()
6151 static int zend_jit_concat(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op… in zend_jit_concat() argument
6161 …return zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opli… in zend_jit_concat()
6164 static int zend_jit_assign_op(zend_jit_ctx *jit, in zend_jit_assign_op() argument
6190 ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_assign_op()
6191 if_op1_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE)); in zend_jit_assign_op()
6195 ref2 = jit_Z_PTR_ref(jit, ref); in zend_jit_assign_op()
6197 if_op1_typed = jit_if_TYPED_REF(jit, ref2); in zend_jit_assign_op()
6202 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { in zend_jit_assign_op()
6205 arg2 = jit_ZVAL_ADDR(jit, real_addr); in zend_jit_assign_op()
6207 arg2 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_assign_op()
6209 jit_SET_EX_OPLINE(jit, opline); in zend_jit_assign_op()
6218 zend_jit_check_exception(jit); in zend_jit_assign_op()
6235 …result = zend_jit_math_helper(jit, opline, opline->extended_value, opline->op1_type, opline->op1, … in zend_jit_assign_op()
6243 result = zend_jit_long_math_helper(jit, opline, opline->extended_value, in zend_jit_assign_op()
6249 …result = zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, op… in zend_jit_assign_op()
6255 …if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_def_addr, op1_def_info, op1_addr… in zend_jit_assign_op()
6266 static ir_ref jit_ZVAL_DEREF_ref(zend_jit_ctx *jit, ir_ref ref) in jit_ZVAL_DEREF_ref() argument
6270 if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_REFERENCE))); in jit_ZVAL_DEREF_ref()
6272 ref2 = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val)); in jit_ZVAL_DEREF_ref()
6277 static zend_jit_addr jit_ZVAL_DEREF(zend_jit_ctx *jit, zend_jit_addr addr) in jit_ZVAL_DEREF() argument
6279 ir_ref ref = jit_ZVAL_ADDR(jit, addr); in jit_ZVAL_DEREF()
6280 ref = jit_ZVAL_DEREF_ref(jit, ref); in jit_ZVAL_DEREF()
6284 static ir_ref jit_ZVAL_INDIRECT_DEREF_ref(zend_jit_ctx *jit, ir_ref ref) in jit_ZVAL_INDIRECT_DEREF_ref() argument
6288 if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_INDIRECT))); in jit_ZVAL_INDIRECT_DEREF_ref()
6290 ref2 = jit_Z_PTR_ref(jit, ref); in jit_ZVAL_INDIRECT_DEREF_ref()
6295 static zend_jit_addr jit_ZVAL_INDIRECT_DEREF(zend_jit_ctx *jit, zend_jit_addr addr) in jit_ZVAL_INDIRECT_DEREF() argument
6297 ir_ref ref = jit_ZVAL_ADDR(jit, addr); in jit_ZVAL_INDIRECT_DEREF()
6298 ref = jit_ZVAL_INDIRECT_DEREF_ref(jit, ref); in jit_ZVAL_INDIRECT_DEREF()
6302 static int zend_jit_simple_assign(zend_jit_ctx *jit, in zend_jit_simple_assign() argument
6319 jit_ZVAL_COPY_CONST(jit, in zend_jit_simple_assign()
6324 jit_ZVAL_COPY_CONST(jit, in zend_jit_simple_assign()
6328 jit_ZVAL_COPY_CONST(jit, in zend_jit_simple_assign()
6337 if_def = jit_if_not_Z_TYPE(jit, val_addr, IS_UNDEF); in zend_jit_simple_assign()
6340 jit_set_Z_TYPE_INFO(jit, var_addr, IS_NULL); in zend_jit_simple_assign()
6342 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); in zend_jit_simple_assign()
6344 jit_SET_EX_OPLINE(jit, opline); in zend_jit_simple_assign()
6351 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef)); in zend_jit_simple_assign()
6359 ir_ref ref = jit_ZVAL_ADDR(jit, val_addr); in zend_jit_simple_assign()
6360 ref = jit_ZVAL_DEREF_ref(jit, ref); in zend_jit_simple_assign()
6365 ref = jit_ZVAL_ADDR(jit, val_addr); in zend_jit_simple_assign()
6366 type = jit_Z_TYPE_ref(jit, ref); in zend_jit_simple_assign()
6370 ref = jit_Z_PTR_ref(jit, ref); in zend_jit_simple_assign()
6373 jit_ZVAL_COPY(jit, in zend_jit_simple_assign()
6378 jit_ZVAL_COPY_2(jit, in zend_jit_simple_assign()
6385 refcount = jit_GC_DELREF(jit, ref); in zend_jit_simple_assign()
6391 jit_ZVAL_DTOR(jit, ref, val_info, opline); in zend_jit_simple_assign()
6401 jit_ZVAL_COPY(jit, in zend_jit_simple_assign()
6406 jit_ZVAL_COPY_2(jit, in zend_jit_simple_assign()
6422 static int zend_jit_assign_to_variable_call(zend_jit_ctx *jit, in zend_jit_assign_to_variable_call() argument
6446 jit_guard_not_Z_TYPE(jit, val_addr, IS_UNDEF, exit_addr); in zend_jit_assign_to_variable_call()
6451 if_def = ir_IF(jit_Z_TYPE(jit, val_addr)); in zend_jit_assign_to_variable_call()
6453 jit_SET_EX_OPLINE(jit, opline); in zend_jit_assign_to_variable_call()
6456 ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_assign_const, IR_FASTCALL_FUNC), in zend_jit_assign_to_variable_call()
6457 jit_ZVAL_ADDR(jit, var_addr), in zend_jit_assign_to_variable_call()
6488 jit_SET_EX_OPLINE(jit, opline); in zend_jit_assign_to_variable_call()
6491 ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, func, IR_FASTCALL_FUNC), in zend_jit_assign_to_variable_call()
6492 jit_ZVAL_ADDR(jit, var_addr), in zend_jit_assign_to_variable_call()
6493 jit_ZVAL_ADDR(jit, val_addr)); in zend_jit_assign_to_variable_call()
6502 static int zend_jit_assign_to_variable(zend_jit_ctx *jit, in zend_jit_assign_to_variable() argument
6525 if (Z_MODE(val_addr) == IS_REG && jit->ra[Z_SSA_VAR(val_addr)].ref == IR_NULL) { in zend_jit_assign_to_variable()
6527 zend_jit_use_reg(jit, val_addr); in zend_jit_assign_to_variable()
6531 jit->delay_var = Z_SSA_VAR(var_addr); in zend_jit_assign_to_variable()
6532 jit->delay_refs = res_inputs; in zend_jit_assign_to_variable()
6538 jit->delay_var = Z_SSA_VAR(res_addr); in zend_jit_assign_to_variable()
6539 jit->delay_refs = res_inputs; in zend_jit_assign_to_variable()
6547 ref = jit_ZVAL_ADDR(jit, var_use_addr); in zend_jit_assign_to_variable()
6548 if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE)); in zend_jit_assign_to_variable()
6550 ref2 = jit_Z_PTR_ref(jit, ref); in zend_jit_assign_to_variable()
6552 ref2 = jit_ZVAL_ADDR(jit, ref_addr); in zend_jit_assign_to_variable()
6554 if_typed = jit_if_TYPED_REF(jit, ref2); in zend_jit_assign_to_variable()
6556 jit_SET_EX_OPLINE(jit, opline); in zend_jit_assign_to_variable()
6566 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) { in zend_jit_assign_to_variable()
6569 arg2 = jit_ZVAL_ADDR(jit, real_addr); in zend_jit_assign_to_variable()
6571 arg2 = jit_ZVAL_ADDR(jit, val_addr); in zend_jit_assign_to_variable()
6598 ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2, jit_ZVAL_ADDR(jit, res_addr)); in zend_jit_assign_to_variable()
6601 zend_jit_check_exception(jit); in zend_jit_assign_to_variable()
6622 if_refcounted = jit_if_REFCOUNTED(jit, var_use_addr); in zend_jit_assign_to_variable()
6629 ref = jit_Z_PTR(jit, var_use_addr); in zend_jit_assign_to_variable()
6631 …if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val… in zend_jit_assign_to_variable()
6634 counter = jit_GC_DELREF(jit, ref); in zend_jit_assign_to_variable()
6638 jit_ZVAL_DTOR(jit, ref, var_info, opline); in zend_jit_assign_to_variable()
6640 zend_jit_check_exception(jit); in zend_jit_assign_to_variable()
6645 ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref); in zend_jit_assign_to_variable()
6648 jit_SET_EX_OPLINE(jit, opline); in zend_jit_assign_to_variable()
6653 ZEND_ASSERT(jit->delay_refs == res_inputs); in zend_jit_assign_to_variable()
6658 zend_jit_check_exception(jit); in zend_jit_assign_to_variable()
6664 ZEND_ASSERT(jit->delay_refs == res_inputs); in zend_jit_assign_to_variable()
6669 zend_jit_check_exception(jit); in zend_jit_assign_to_variable()
6673 jit_GC_DELREF(jit, ref); in zend_jit_assign_to_variable()
6675 ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref); in zend_jit_assign_to_variable()
6678 jit_SET_EX_OPLINE(jit, opline); in zend_jit_assign_to_variable()
6693 …if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val… in zend_jit_assign_to_variable()
6708 ZEND_ASSERT(jit->delay_refs == res_inputs); in zend_jit_assign_to_variable()
6710 jit->delay_var = -1; in zend_jit_assign_to_variable()
6711 jit->delay_refs = NULL; in zend_jit_assign_to_variable()
6720 phi = ir_emit2(&jit->ctx, IR_OPT(IR_COPY, jit->ctx.ir_base[phi].type), phi, 1); in zend_jit_assign_to_variable()
6722 zend_jit_def_reg(jit, var_addr, phi); in zend_jit_assign_to_variable()
6725 jit_set_Z_LVAL(jit, real_res_addr, jit_Z_LVAL(jit, var_addr)); in zend_jit_assign_to_variable()
6727 jit_set_Z_DVAL(jit, real_res_addr, jit_Z_DVAL(jit, var_addr)); in zend_jit_assign_to_variable()
6731 zend_jit_def_reg(jit, res_addr, phi); in zend_jit_assign_to_variable()
6738 static int zend_jit_qm_assign(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit… in zend_jit_qm_assign() argument
6741 if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { in zend_jit_qm_assign()
6749 …if (!zend_jit_simple_assign(jit, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_a… in zend_jit_qm_assign()
6752 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { in zend_jit_qm_assign()
6758 static int zend_jit_assign(zend_jit_ctx *jit, in zend_jit_assign() argument
6775 if (!zend_jit_update_regs(jit, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { in zend_jit_assign()
6790 if (!zend_jit_assign_to_variable(jit, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, in zend_jit_assign()
6796 …if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_addr, op1_def_info, op1_use_addr… in zend_jit_assign()
6802 && EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)) < jit->current_op_array->last_var) { in zend_jit_assign()
6805 jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_LONG); in zend_jit_assign()
6810 jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_DOUBLE); in zend_jit_assign()
6820 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { in zend_jit_assign()
6855 static ir_ref zend_jit_cmp_long_long(zend_jit_ctx *jit, in zend_jit_cmp_long_long() argument
6875 jit_set_Z_TYPE_INFO(jit, res_addr, result ? IS_TRUE : IS_FALSE); in zend_jit_cmp_long_long()
6880 return jit_IF_ex(jit, IR_FALSE, result ? target_label : target_label2); in zend_jit_cmp_long_long()
6883 return jit_IF_ex(jit, IR_TRUE, result ? target_label : target_label2); in zend_jit_cmp_long_long()
6897 ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_LVAL(jit, op1_addr), jit_Z_LVAL(jit, op2_addr)); in zend_jit_cmp_long_long()
6900 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_cmp_long_long()
6918 return jit_IF_ex(jit, ref, in zend_jit_cmp_long_long()
6931 static ir_ref zend_jit_cmp_long_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_… in zend_jit_cmp_long_double() argument
6933 …ref = ir_CMP_OP(zend_jit_cmp_op(opline), ir_INT2D(jit_Z_LVAL(jit, op1_addr)), jit_Z_DVAL(jit, op2_… in zend_jit_cmp_long_double()
6936 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_cmp_long_double()
6946 return jit_IF_ex(jit, ref, in zend_jit_cmp_long_double()
6952 static ir_ref zend_jit_cmp_double_long(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_… in zend_jit_cmp_double_long() argument
6954 …ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), ir_INT2D(jit_Z_LVAL(jit… in zend_jit_cmp_double_long()
6957 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_cmp_double_long()
6967 return jit_IF_ex(jit, ref, in zend_jit_cmp_double_long()
6973 static ir_ref zend_jit_cmp_double_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op… in zend_jit_cmp_double_double() argument
6975 …ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), jit_Z_DVAL(jit, op2_add… in zend_jit_cmp_double_double()
6978 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_cmp_double_double()
6996 return jit_IF_ex(jit, ref, in zend_jit_cmp_double_double()
7008 static ir_ref zend_jit_cmp_slow(zend_jit_ctx *jit, ir_ref ref, const zend_op *opline, zend_jit_addr… in zend_jit_cmp_slow() argument
7013 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_cmp_slow()
7023 return jit_IF_ex(jit, ref, in zend_jit_cmp_slow()
7030 static int zend_jit_cmp(zend_jit_ctx *jit, in zend_jit_cmp() argument
7066 if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) { in zend_jit_cmp()
7068 zend_jit_use_reg(jit, op1_addr); in zend_jit_cmp()
7071 if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) { in zend_jit_cmp()
7073 zend_jit_use_reg(jit, op2_addr); in zend_jit_cmp()
7079 if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_cmp()
7083 if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_cmp()
7087 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_cmp()
7092 …ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, tar… in zend_jit_cmp()
7102 …ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smar… in zend_jit_cmp()
7113 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); in zend_jit_cmp()
7120 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_cmp()
7123 …ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, t… in zend_jit_cmp()
7134 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_cmp()
7139 …ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, tar… in zend_jit_cmp()
7154 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); in zend_jit_cmp()
7161 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_cmp()
7164 …ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, t… in zend_jit_cmp()
7175 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_cmp()
7180 …ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, tar… in zend_jit_cmp()
7192 if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_cmp()
7199 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); in zend_jit_cmp()
7202 …ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, t… in zend_jit_cmp()
7213 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_cmp()
7218 …ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, tar… in zend_jit_cmp()
7236 jit_SET_EX_OPLINE(jit, opline); in zend_jit_cmp()
7240 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { in zend_jit_cmp()
7247 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { in zend_jit_cmp()
7253 op1 = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_cmp()
7255 op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, NULL, 0); in zend_jit_cmp()
7257 op2 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_cmp()
7259 op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, NULL, 0); in zend_jit_cmp()
7263 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL); in zend_jit_cmp()
7265 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL); in zend_jit_cmp()
7267 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_cmp()
7270 …ref = zend_jit_cmp_slow(jit, ref, opline, res_addr, smart_branch_opcode, target_label, target_labe… in zend_jit_cmp()
7288 ZEND_ASSERT(jit->b >= 0); in zend_jit_cmp()
7289 bb = &jit->ssa->cfg.blocks[jit->b]; in zend_jit_cmp()
7303 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END()); in zend_jit_cmp()
7306 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); in zend_jit_cmp()
7307 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref); in zend_jit_cmp()
7313 jit_IF_TRUE_FALSE_ex(jit, end_inputs->refs[n], label); in zend_jit_cmp()
7315 jit_IF_TRUE_FALSE_ex(jit, end_inputs->refs[n], label2); in zend_jit_cmp()
7319 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END()); in zend_jit_cmp()
7321 _zend_jit_add_predecessor_ref(jit, label2, jit->b, ir_END()); in zend_jit_cmp()
7323 jit->b = -1; in zend_jit_cmp()
7329 _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END()); in zend_jit_cmp()
7330 jit->b = -1; in zend_jit_cmp()
7336 static int zend_jit_identical(zend_jit_ctx *jit, in zend_jit_identical() argument
7356 ir_ref op1 = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_identical()
7357 op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, opline, 0); in zend_jit_identical()
7362 ir_ref op2 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_identical()
7363 op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, opline, 0); in zend_jit_identical()
7385 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_identical()
7387 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); in zend_jit_identical()
7391 jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE); in zend_jit_identical()
7394 zend_jit_check_exception(jit); in zend_jit_identical()
7398 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_identical()
7410 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END()); in zend_jit_identical()
7411 jit->b = -1; in zend_jit_identical()
7416 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_identical()
7418 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); in zend_jit_identical()
7422 jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE); in zend_jit_identical()
7425 zend_jit_check_exception(jit); in zend_jit_identical()
7429 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_identical()
7441 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END()); in zend_jit_identical()
7442 jit->b = -1; in zend_jit_identical()
7448 ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_identical()
7449 ref = jit_ZVAL_DEREF_ref(jit, ref); in zend_jit_identical()
7453 ref = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_identical()
7454 ref = jit_ZVAL_DEREF_ref(jit, ref); in zend_jit_identical()
7460 …ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smar… in zend_jit_identical()
7466 …ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, t… in zend_jit_identical()
7474 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { in zend_jit_identical()
7483 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { in zend_jit_identical()
7492 ref = ir_EQ(jit_Z_TYPE(jit, op2_addr), ir_CONST_U8(Z_TYPE_P(val))); in zend_jit_identical()
7496 ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(Z_TYPE_P(val))); in zend_jit_identical()
7500 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { in zend_jit_identical()
7507 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { in zend_jit_identical()
7513 jit_SET_EX_OPLINE(jit, opline); in zend_jit_identical()
7517 jit_ZVAL_ADDR(jit, op1_addr), in zend_jit_identical()
7518 jit_ZVAL_ADDR(jit, op2_addr)); in zend_jit_identical()
7523 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_identical()
7526 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_identical()
7531 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL); in zend_jit_identical()
7533 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL); in zend_jit_identical()
7535 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_identical()
7550 ref = jit_IF_ex(jit, ref, in zend_jit_identical()
7558 ZEND_ASSERT(jit->b >= 0); in zend_jit_identical()
7559 bb = &jit->ssa->cfg.blocks[jit->b]; in zend_jit_identical()
7565 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END()); in zend_jit_identical()
7568 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); in zend_jit_identical()
7569 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref); in zend_jit_identical()
7571 jit->b = -1; in zend_jit_identical()
7577 static int zend_jit_bool_jmpznz(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_j… in zend_jit_bool_jmpznz() argument
7611 ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_bool_jmpznz()
7612 ref = jit_ZVAL_DEREF_ref(jit, ref); in zend_jit_bool_jmpznz()
7627 ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_bool_jmpznz()
7628 zend_jit_zval_check_undef(jit, ref, opline->op1.var, opline, 0); in zend_jit_bool_jmpznz()
7636 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); in zend_jit_bool_jmpznz()
7638 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_bool_jmpznz()
7640 zend_jit_check_exception(jit); in zend_jit_bool_jmpznz()
7644 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END()); in zend_jit_bool_jmpznz()
7645 jit->b = -1; in zend_jit_bool_jmpznz()
7650 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); in zend_jit_bool_jmpznz()
7652 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_bool_jmpznz()
7654 zend_jit_check_exception(jit); in zend_jit_bool_jmpznz()
7658 _zend_jit_add_predecessor_ref(jit, false_label, jit->b, ir_END()); in zend_jit_bool_jmpznz()
7659 jit->b = -1; in zend_jit_bool_jmpznz()
7665 type = jit_Z_TYPE(jit, op1_addr); in zend_jit_bool_jmpznz()
7672 zend_jit_type_check_undef(jit, in zend_jit_bool_jmpznz()
7678 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); in zend_jit_bool_jmpznz()
7684 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_bool_jmpznz()
7703 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); in zend_jit_bool_jmpznz()
7709 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_bool_jmpznz()
7728 type = jit_Z_TYPE(jit, op1_addr); in zend_jit_bool_jmpznz()
7733 ref = jit_Z_LVAL(jit, op1_addr); in zend_jit_bool_jmpznz()
7737 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7740 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7746 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7759 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); in zend_jit_bool_jmpznz()
7764 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); in zend_jit_bool_jmpznz()
7779 type = jit_Z_TYPE(jit, op1_addr); in zend_jit_bool_jmpznz()
7784 ref = ir_NE(jit_Z_DVAL(jit, op1_addr), ir_CONST_DOUBLE(0.0)); in zend_jit_bool_jmpznz()
7787 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7790 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7796 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7809 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); in zend_jit_bool_jmpznz()
7814 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); in zend_jit_bool_jmpznz()
7824 jit_SET_EX_OPLINE(jit, opline); in zend_jit_bool_jmpznz()
7825 ref = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_is_true), jit_ZVAL_ADDR(jit, op1_addr)); in zend_jit_bool_jmpznz()
7826 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL); in zend_jit_bool_jmpznz()
7828 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_bool_jmpznz()
7832 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7835 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7843 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7858 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); in zend_jit_bool_jmpznz()
7863 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); in zend_jit_bool_jmpznz()
7874 _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs); in zend_jit_bool_jmpznz()
7880 static int zend_jit_defined(zend_jit_ctx *jit, const zend_op *opline, uint8_t smart_branch_opcode, … in zend_jit_defined() argument
7917 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_defined()
7924 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE); in zend_jit_defined()
7941 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_defined()
7948 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE); in zend_jit_defined()
7955 jit_SET_EX_OPLINE(jit, opline); in zend_jit_defined()
7971 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_defined()
7981 …_zend_jit_merge_smart_branch_inputs(jit, defined_label, undefined_label, true_inputs, false_inputs… in zend_jit_defined()
7987 static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opli… in zend_jit_escape_if_undef() argument
7989 zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg)); in zend_jit_escape_if_undef()
7990 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, reg_addr)); in zend_jit_escape_if_undef()
7995 if (!zend_jit_save_call_chain(jit, -1)) { in zend_jit_escape_if_undef()
8006 zend_jit_zval_try_addref(jit, val_addr); in zend_jit_escape_if_undef()
8009 jit_LOAD_IP_ADDR(jit, opline - 1); in zend_jit_escape_if_undef()
8010 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape)); in zend_jit_escape_if_undef()
8017 static int zend_jit_restore_zval(zend_jit_ctx *jit, int var, int8_t reg) in zend_jit_restore_zval() argument
8020 zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg)); in zend_jit_restore_zval()
8023 jit_ZVAL_COPY(jit, var_addr, MAY_BE_ANY, reg_addr, MAY_BE_ANY, 1); in zend_jit_restore_zval()
8027 static zend_jit_addr zend_jit_guard_fetch_result_type(zend_jit_ctx *jit, in zend_jit_guard_fetch_result_type() argument
8039 ir_ref ref = jit_ZVAL_ADDR(jit, val_addr); in zend_jit_guard_fetch_result_type()
8062 if_type = ir_IF(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type))); in zend_jit_guard_fetch_result_type()
8064 if_type = jit_if_Z_TYPE(jit, val_addr, type); in zend_jit_guard_fetch_result_type()
8078 jit_guard_Z_TYPE(jit, val_addr, IS_REFERENCE, res_exit_addr); in zend_jit_guard_fetch_result_type()
8079 ref = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val)); in zend_jit_guard_fetch_result_type()
8091 ir_GUARD(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type)), ir_CONST_ADDR(res_exit_addr)); in zend_jit_guard_fetch_result_type()
8093 jit_guard_Z_TYPE(jit, val_addr, type, res_exit_addr); in zend_jit_guard_fetch_result_type()
8112 static int zend_jit_fetch_constant(zend_jit_ctx *jit, in zend_jit_fetch_constant() argument
8147 jit_SET_EX_OPLINE(jit, opline); in zend_jit_fetch_constant()
8151 ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_fetch_constant()
8160 const_addr = zend_jit_guard_fetch_result_type(jit, opline, const_addr, type, 0, 0, 0); in zend_jit_fetch_constant()
8169 jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, res_info, 1); in zend_jit_fetch_constant()
8170 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { in zend_jit_fetch_constant()
8177 jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, 1); in zend_jit_fetch_constant()
8184 static int zend_jit_type_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint8_t… in zend_jit_type_check() argument
8213 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF); in zend_jit_type_check()
8217 jit_SET_EX_OPLINE(jit, opline); in zend_jit_type_check()
8219 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_type_check()
8223 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_type_check()
8230 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE); in zend_jit_type_check()
8236 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_type_check()
8243 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE); in zend_jit_type_check()
8259 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_type_check()
8262 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_type_check()
8269 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE); in zend_jit_type_check()
8273 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_type_check()
8276 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_type_check()
8283 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE); in zend_jit_type_check()
8314 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_type_check()
8315 ref = jit_ZVAL_DEREF_ref(jit, ref); in zend_jit_type_check()
8319 ref = ir_AND_U32(ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)), ir_CONST_U32(mask)); in zend_jit_type_check()
8324 ref = ir_NE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type)); in zend_jit_type_check()
8326 ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type)); in zend_jit_type_check()
8329 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_type_check()
8347 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_type_check()
8357 } else if (exit_addr && !jit->ctx.control) { in zend_jit_type_check()
8361 _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs); in zend_jit_type_check()
8367 static int zend_jit_isset_isempty_cv(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, z… in zend_jit_isset_isempty_cv() argument
8391 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_isset_isempty_cv()
8392 ref = jit_ZVAL_DEREF_ref(jit, ref); in zend_jit_isset_isempty_cv()
8402 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE); in zend_jit_isset_isempty_cv()
8411 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE); in zend_jit_isset_isempty_cv()
8415 ir_ref ref = ir_GT(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL)); in zend_jit_isset_isempty_cv()
8429 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_isset_isempty_cv()
8440 _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs); in zend_jit_isset_isempty_cv()
8455 static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t used_stack) in zend_jit_stack_check() argument
8474 static int zend_jit_free_trampoline(zend_jit_ctx *jit, int8_t func_reg) in zend_jit_free_trampoline() argument
8489 static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *… argument
8552 zend_jit_start_reuse_ip(jit);
8555 jit_STORE_IP(jit, ir_LOAD_A(jit_EG(vm_stack_top)));
8562 jit_IP(jit)),
8583 jit_SET_EX_OPLINE(jit, opline);
8591 jit_SET_EX_OPLINE(jit, opline);
8595 jit_STORE_IP(jit, ref);
8603 rx = jit_IP(jit);
8618 top = jit->ctx.control = ir_emit2(&jit->ctx, IR_OPT(IR_LOAD, IR_ADDR), jit->ctx.control, ref);
8634 rx = jit_IP(jit);
8649 rx = jit_IP(jit);
8667 jit_GC_ADDREF(jit, this_ref);
8688 jit_GC_ADDREF(jit, func_ref);
8739 static int zend_jit_func_guard(zend_jit_ctx *jit, ir_ref func_ref, const zend_function *func, const… argument
8771 static int zend_jit_init_fcall_guard(zend_jit_ctx *jit, uint32_t level, const zend_function *func, … argument
8797 return zend_jit_func_guard(jit, ir_LOAD_A(jit_CALL(call, func)), func, exit_addr);
8800 static int zend_jit_init_fcall(zend_jit_ctx *jit, const zend_op *opline, uint32_t b, const zend_op_… argument
8807 if (jit->delayed_call_level) {
8808 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
8848 || zend_jit_prefer_const_addr_load(jit, (uintptr_t)func)) {
8904 } else if (!zend_jit_func_guard(jit, ref, func, exit_addr)) {
8908 jit_SET_EX_OPLINE(jit, opline);
8909 ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_undefined_function));
8916 …if (!zend_jit_push_call_frame(jit, opline, op_array, func, 0, 0, checked_stack, func_ref, IR_UNUSE…
8921 if (!zend_jit_save_call_chain(jit, call_level)) {
8926 jit->delayed_call_level = call_level;
8933 if (!zend_jit_set_ip(jit, opline + 1)) {
8941 static int zend_jit_init_method_call(zend_jit_ctx *jit, argument
8985 func_ref = zend_jit_deopt_rload(jit, IR_ADDR, func_reg);
8986 this_ref = zend_jit_deopt_rload(jit, IR_ADDR, this_reg);
8992 this_ref = jit_Z_PTR(jit, this_addr);
8997 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
8998 ref = jit_ZVAL_DEREF_ref(jit, ref);
9006 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
9008 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper), jit_ZVAL_ADDR(jit, op1_addr));
9021 ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_OBJECT)),
9024 ir_ref if_object = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
9028 jit_SET_EX_OPLINE(jit, opline);
9031 jit_ZVAL_ADDR(jit, op1_addr));
9034 jit_ZVAL_ADDR(jit, op1_addr));
9036 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
9041 this_ref = jit_Z_PTR(jit, op1_addr);
9044 if (jit->delayed_call_level) {
9045 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
9073 jit_SET_EX_OPLINE(jit, opline);
9075 if (!jit->ctx.fixed_call_stack_size) {
9080 this_ref2 = ir_HARD_COPY_A(jit_ADD_OFFSET(jit, ir_RLOAD_A(IR_REG_SP), IR_SHADOW_ARGS));
9100 if (!jit->ctx.fixed_call_stack_size) {
9106 this_ref2 = ir_LOAD_A(jit_ADD_OFFSET(jit, ir_RLOAD_A(IR_REG_SP), IR_SHADOW_ARGS));
9112 ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9132 jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
9133 jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
9137 if (!zend_jit_func_guard(jit, func_ref, func, exit_addr)) {
9166 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9168 jit_STORE_IP(jit, ret);
9177 …if (!zend_jit_push_call_frame(jit, opline, NULL, func, 0, delayed_fetch_this, checked_stack, func_…
9185 zend_jit_start_reuse_ip(jit);
9188 if (!zend_jit_save_call_chain(jit, call_level)) {
9194 jit->delayed_call_level = call_level;
9200 if (!zend_jit_set_ip(jit, opline + 1)) {
9208 static int zend_jit_init_closure_call(zend_jit_ctx *jit, argument
9222 ref = jit_Z_PTR(jit, op2_addr);
9269 if (jit->delayed_call_level) {
9270 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
9275 if (!zend_jit_push_call_frame(jit, opline, NULL, func, 1, 0, checked_stack, ref, IR_UNUSED)) {
9280 if (!zend_jit_save_call_chain(jit, call_level)) {
9286 jit->delayed_call_level = call_level;
9292 if (!zend_jit_set_ip(jit, opline + 1)) {
9300 static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_… argument
9307 if (!zend_jit_reuse_ip(jit)) {
9345 jit_set_Z_TYPE_INFO(jit, addr, IS_UNDEF);
9347 jit_SET_EX_OPLINE(jit, opline);
9348 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_throw_cannot_pass_by_ref));
9360 jit_ZVAL_COPY_CONST(jit,
9365 jit_ZVAL_COPY(jit,
9374 static int zend_jit_send_ref(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_arra… argument
9382 if (!zend_jit_reuse_ip(jit)) {
9388 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
9394 ir_ref if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
9397 jit_set_Z_TYPE_INFO(jit,op1_addr, IS_NULL);
9414 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
9417 ref = jit_Z_PTR(jit, op1_addr);
9419 jit_GC_ADDREF(jit, ref);
9421 jit_set_Z_PTR(jit, arg_addr, ref);
9422 jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
9429 ref = jit_EMALLOC(jit, sizeof(zend_reference), op_array, opline);
9431 jit_set_GC_REFCOUNT(jit, ref, 2);
9439 jit_ZVAL_COPY(jit,
9445 jit_set_Z_PTR(jit, op1_addr, ref);
9446 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
9449 jit_set_Z_PTR(jit, arg_addr, ref);
9450 jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
9457 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
9462 static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_arra… argument
9474 if (!zend_jit_reuse_ip(jit)) {
9484 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
9498 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
9513 jit_ZVAL_COPY(jit,
9530 ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U32(IS_REFERENCE)),
9550 jit_ZVAL_COPY(jit,
9556 ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
9576 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
9578 jit_SET_EX_OPLINE(jit, opline);
9580 jit_ZVAL_ADDR(jit, arg_addr));
9581 zend_jit_check_exception(jit);
9593 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
9605 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
9618 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
9623 jit_SET_EX_OPLINE(jit, opline);
9628 jit_set_Z_TYPE_INFO(jit, arg_addr, IS_NULL);
9631 ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9647 jit_ZVAL_COPY(jit,
9653 ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
9666 jit_SET_EX_OPLINE(jit, opline);
9668 jit_ZVAL_ADDR(jit, arg_addr));
9669 zend_jit_check_exception(jit);
9677 ref = jit_ZVAL_ADDR(jit, op1_addr);
9678 ref = jit_ZVAL_DEREF_ref(jit, ref);
9682 jit_ZVAL_COPY(jit,
9691 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
9695 ref = jit_Z_PTR(jit, op1_addr);
9700 jit_ZVAL_COPY(jit,
9706 refcount = jit_GC_DELREF(jit, ref);
9711 if_refcounted = jit_if_REFCOUNTED(jit, arg_addr);
9714 jit_GC_ADDREF(jit, jit_Z_PTR(jit, arg_addr));
9722 jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
9728 jit_ZVAL_COPY(jit,
9735 if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
9744 jit_ZVAL_COPY(jit,
9759 static int zend_jit_check_func_arg(zend_jit_ctx *jit, const zend_op *opline) argument
9772 if (jit->reuse_ip) {
9773 ref = jit_IP(jit);
9784 if (jit->reuse_ip) {
9785 ref = jit_IP(jit);
9798 if (!zend_jit_reuse_ip(jit)) {
9802 rx = jit_IP(jit);
9827 static int zend_jit_check_undef_args(zend_jit_ctx *jit, const zend_op *opline) argument
9831 if (jit->reuse_ip) {
9832 call = jit_IP(jit);
9842 jit_SET_EX_OPLINE(jit, opline);
9844 ir_GUARD_NOT(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9850 static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_arra… argument
9931 if (!jit->reuse_ip) {
9932 zend_jit_start_reuse_ip(jit);
9934 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call)));
9936 rx = jit_IP(jit);
9937 zend_jit_stop_reuse_ip(jit);
9939 jit_SET_EX_OPLINE(jit, opline);
9961 if (!jit->delayed_call_level) {
9967 jit->delayed_call_level = 0;
9970 ir_STORE(jit_CALL(rx, prev_execute_data), jit_FP(jit));
9993 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
10004 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
10027 jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var)) :
10069 jit_STORE_FP(jit, rx);
10078 jit_set_Z_TYPE_INFO_ex(jit, var_addr, ir_CONST_U32(IS_UNDEF));
10096 jit_LOAD_IP_ADDR(jit, func->op_array.opcodes + num_args);
10105 jit_LOAD_IP(jit, ip);
10127 jit_LOAD_IP(jit, ip);
10135 ir_CALL_1(IR_VOID, helper, jit_FP(jit));
10151 jit_LOAD_IP(jit, ip);
10171 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper), jit_FP(jit));
10194 jit_STORE_IP(jit, ir_ADD_A(jit_IP(jit), ref));
10224 ir_ADD_A(jit_FP(jit), ir_MUL_A(num_args, ir_CONST_ADDR(sizeof(zval)))),
10243 ir_ref rx = jit_FP(jit);
10244 …s_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_ha…
10247 jit_SET_EX_OPLINE(jit, trace[1].opline);
10250 ir_STORE(jit_EX(opline), jit_IP(jit));
10252 jit_observer_fcall_begin(jit, rx, observer_handler);
10265 zend_jit_check_timeout(jit, NULL /* we're inside the called function */, exit_addr);
10267 jit_observer_fcall_is_unobserved_end(jit, &unobserved_data);
10283 begin = jit->bb_start_ref[num_args];
10285 insn = &jit->ctx.ir_base[begin];
10288 insn = &jit->ctx.ir_base[begin];
10296 insn = &jit->ctx.ir_base[begin];
10302 ZEND_ASSERT(jit->ctx.ir_base[insn->op3].op == IR_LOOP_END);
10303 jit->ctx.ir_base[insn->op3].op = IR_END;
10306 insn = &jit->ctx.ir_base[begin];
10313 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
10319 bb = &jit->ssa->cfg.blocks[jit->b];
10327 ref = jit->ctx.insns_count - 1;
10328 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE
10329 || jit->ctx.ir_base[ref].op == IR_RETURN
10330 || jit->ctx.ir_base[ref].op == IR_LOOP_END);
10331 ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY);
10332 ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start);
10334 _zend_jit_add_predecessor_ref(jit, succ, jit->b, ref);
10335 jit->b = -1;
10355 …s_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_ha…
10356 jit_observer_fcall_begin(jit, rx, observer_handler);
10357 jit_observer_fcall_is_unobserved_end(jit, &unobserved_data);
10369 if (!jit->ctx.fixed_call_stack_size) {
10374 ptr = ir_HARD_COPY_A(jit_ADD_OFFSET(jit, ir_RLOAD_A(IR_REG_SP), IR_SHADOW_ARGS));
10382 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
10384 zend_jit_reset_last_valid_opline(jit);
10387 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
10403 jit_observer_fcall_end(jit, rx, res_ref);
10420 ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
10429 jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, opline);
10458 jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_CALL(rx, This.value.obj)));
10506 if (!jit->ctx.fixed_call_stack_size) {
10510 sp = jit_ADD_OFFSET(jit, ir_RLOAD_A(IR_REG_SP), IR_SHADOW_ARGS);
10516 jit_ZVAL_PTR_DTOR(jit, res_addr, func_info, 1, opline);
10518 if (!jit->ctx.fixed_call_stack_size) {
10525 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
10526 jit_STUB_ADDR(jit, jit_stub_icall_throw));
10533 if (trace && jit->last_valid_opline != opline) {
10544 zend_jit_check_timeout(jit, opline + 1, exit_addr);
10548 jit_LOAD_IP_ADDR(jit, opline + 1);
10552 jit_LOAD_IP_ADDR(jit, opline + 1);
10563 static int zend_jit_constructor(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_a… argument
10565 ir_ref if_skip_constructor = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, opline), next_block);
10570 if (!zend_jit_tail_handler(jit, opline)) {
10574 if (!zend_jit_do_fcall(jit, opline, op_array, ssa, call_level, next_block, NULL)) {
10580 ZEND_ASSERT(jit->ssa->cfg.blocks[next_block].predecessors_count == 1);
10581 if (!jit->ctx.control) {
10582 ZEND_ASSERT(jit->bb_edges[jit->bb_predecessors[next_block]]);
10584 ir_MERGE_2(jit->bb_edges[jit->bb_predecessors[next_block]], ir_END());
10585 jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
10587 ZEND_ASSERT(!jit->bb_edges[jit->bb_predecessors[next_block]]);
10590 jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
10592 jit->b = -1;
10598 static int zend_jit_verify_arg_type(zend_jit_ctx *jit, const zend_op *opline, zend_arg_info *arg_in… argument
10604 ref = jit_ZVAL_ADDR(jit, res_addr);
10618 ref = jit_ZVAL_DEREF_ref(jit, ref);
10620 ref = jit_Z_PTR_ref(jit, ref);
10628 ir_ref if_ok = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(type_code));
10634 ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE_ref(jit, ref)),
10642 jit_SET_EX_OPLINE(jit, opline);
10647 ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
10657 static int zend_jit_recv(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array) argument
10691 jit_SET_EX_OPLINE(jit, opline);
10692 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_missing_arg_error), jit_FP(jit));
10693 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
10699 if (!zend_jit_verify_arg_type(jit, opline, arg_info, 1)) {
10707 static int zend_jit_recv_init(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_arr… argument
10718 jit_ZVAL_COPY_CONST(jit,
10731 jit_ZVAL_COPY_CONST(jit,
10738 jit_SET_EX_OPLINE(jit, opline);
10740 jit_ZVAL_ADDR(jit, res_addr),
10745 jit_ZVAL_PTR_DTOR(jit, res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, opline);
10746 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
10768 if (!zend_jit_verify_arg_type(jit, opline, arg_info, may_throw)) {
10777 static bool zend_jit_verify_return_type(zend_jit_ctx *jit, const zend_op *opline, const zend_op_arr… argument
10793 ir_ref if_ok = jit_if_Z_TYPE(jit, op1_addr, type_code);
10800 ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)),
10811 jit_SET_EX_OPLINE(jit, opline);
10812 ref = jit_ZVAL_ADDR(jit, op1_addr);
10814 ref = zend_jit_zval_check_undef(jit, ref, opline->op1.var, NULL, 1);
10823 zend_jit_check_exception(jit);
10833 static int zend_jit_leave_frame(zend_jit_ctx *jit) argument
10840 static int zend_jit_free_cvs(zend_jit_ctx *jit) argument
10846 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_compiled_variables), jit_FP(jit));
10850 static int zend_jit_free_cv(zend_jit_ctx *jit, uint32_t info, uint32_t var) argument
10855 jit_ZVAL_PTR_DTOR(jit, var_addr, info, 1, NULL);
10860 static int zend_jit_free_op(zend_jit_ctx *jit, const zend_op *opline, uint32_t info, uint32_t var_o… argument
10863 jit_ZVAL_PTR_DTOR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, opline);
10868 static int zend_jit_leave_func(zend_jit_ctx *jit, argument
10903 if (!zend_jit_leave_frame(jit)) {
10916 ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_func_helper), jit_FP(jit));
10930 ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
10932 ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
10938 jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data)));
10943 ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_leave_function_handler));
10950 if (!zend_jit_leave_frame(jit)) {
10955 jit_OBJ_RELEASE(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), -sizeof(zend_object)));
10961 if (!zend_jit_leave_frame(jit)) {
10976 jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_EX(This.value.obj)));
10985 ir_STORE(jit_EG(vm_stack_top), jit_FP(jit));
10988 jit_STORE_FP(jit, ir_LOAD_A(jit_EX(prev_execute_data)));
10992 ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
10998 zend_jit_reset_last_valid_opline(jit);
11010 jit_STORE_IP(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(opline)), sizeof(zend_op)));
11012 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
11013 jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
11035 ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
11048 ir_ref if_eq = ir_IF(jit_CMP_IP(jit, IR_EQ, next_opline));
11051 ZEND_ASSERT(jit->trace_loop_ref);
11052 ZEND_ASSERT(jit->ctx.ir_base[jit->trace_loop_ref].op2 == IR_UNUSED);
11053 ir_MERGE_SET_OP(jit->trace_loop_ref, 2, ir_END());
11057 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
11059 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape));
11062 ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), jit_STUB_ADDR(jit, jit_stub_trace_escape));
11065 zend_jit_set_last_valid_opline(jit, trace->opline);
11074 ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
11080 ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
11083 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
11084 jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
11093 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
11098 jit->b = -1;
11103 static void zend_jit_common_return(zend_jit_ctx *jit) argument
11105 ZEND_ASSERT(jit->return_inputs);
11106 ir_MERGE_list(jit->return_inputs);
11109 static int zend_jit_return(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array,… argument
11119 jit->return_inputs = IR_UNUSED;
11135 if (!zend_jit_spill_store_inv(jit, op1_addr, dst, op1_info)) {
11140 jit_observer_fcall_end(jit, jit_FP(jit), jit_ZVAL_ADDR(jit, op1_addr));
11154 ir_ref if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
11156 ir_END_list(jit->return_inputs);
11159 ref = jit_Z_PTR(jit, op1_addr);
11160 refcount = jit_GC_DELREF(jit, ref);
11166 ir_END_list(jit->return_inputs);
11169 jit_ZVAL_DTOR(jit, ref, op1_info, opline);
11172 ir_END_list(jit->return_inputs);
11178 ir_END_list(jit->return_inputs);
11186 if (jit->return_inputs) {
11188 ir_END_list(jit->return_inputs);
11189 ir_MERGE_list(jit->return_inputs);
11190 jit->return_inputs = IR_UNUSED;
11198 jit_ZVAL_COPY_CONST(jit, ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
11200 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
11203 ref = jit_ZVAL_ADDR(jit, op1_addr);
11204 ref = jit_ZVAL_DEREF_ref(jit, ref);
11212 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 1);
11214 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
11216 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_NULL);
11218 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
11221 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
11228 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
11232 ref = jit_Z_PTR(jit, op1_addr);
11237 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, ref_addr, op1_info, 0);
11238 ref2 = jit_GC_DELREF(jit, ref);
11243 ir_ref if_refcounted = jit_if_REFCOUNTED(jit, ret_addr);
11245 ir_END_list(jit->return_inputs);
11249 ref2 = jit_Z_PTR(jit, ret_addr);
11250 jit_GC_ADDREF(jit, ref2);
11251 ir_END_list(jit->return_inputs);
11255 jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
11256 ir_END_list(jit->return_inputs);
11260 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
11264 if (jit->return_inputs) {
11265 ir_END_list(jit->return_inputs);
11266 ir_MERGE_list(jit->return_inputs);
11267 jit->return_inputs = IR_UNUSED;
11270 ir_END_list(jit->return_inputs);
11271 jit->b = -1;
11277 static int zend_jit_bind_global(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info) argument
11302 if_reference = jit_if_Z_TYPE_ref(jit, bucket_ref, ir_CONST_U8(IS_REFERENCE));
11314 ref = jit_Z_PTR_ref(jit, bucket_ref);
11315 jit_GC_ADDREF(jit, ref);
11332 if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
11337 ref2 = jit_Z_PTR(jit, op1_addr);
11340 jit_set_Z_PTR(jit, op1_addr, ref);
11341 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
11344 refcount = jit_GC_DELREF(jit, ref2);
11352 jit_ZVAL_DTOR(jit, ref2, op1_info, opline);
11358 if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref2);
11363 jit_SET_EX_OPLINE(jit, opline);
11375 jit_set_Z_PTR(jit, op1_addr, ref);
11376 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
11387 static int zend_jit_free(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, int may_throw) argument
11393 jit_SET_EX_OPLINE(jit, opline);
11399 if_array = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
11404 ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_iter_idx)));
11416 jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
11419 zend_jit_check_exception(jit);
11426 static int zend_jit_echo(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info) argument
11439 jit_SET_EX_OPLINE(jit, opline);
11443 zend_jit_check_exception(jit);
11451 jit_SET_EX_OPLINE(jit, opline);
11453 ref = jit_Z_PTR(jit, op1_addr);
11459 jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
11462 zend_jit_check_exception(jit);
11467 static int zend_jit_strlen(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_ad… argument
11477 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(len));
11479 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11480 } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11488 ref = jit_Z_PTR(jit, op1_addr);
11490 jit_set_Z_LVAL(jit, res_addr, ref);
11493 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11497 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11499 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
11504 static int zend_jit_count(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_add… argument
11514 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(count));
11516 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11517 } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11526 ref = jit_Z_PTR(jit, op1_addr);
11533 jit_set_Z_LVAL(jit, res_addr, ref);
11536 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11540 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11542 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
11546 zend_jit_check_exception(jit);
11551 static int zend_jit_in_array(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_… argument
11564 jit_Z_PTR(jit, op1_addr));
11581 ZEND_ASSERT(jit->b >= 0);
11582 bb = &jit->ssa->cfg.blocks[jit->b];
11584 ref = jit_IF_ex(jit, ref,
11586 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
11587 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
11588 jit->b = -1;
11590 jit_set_Z_TYPE_INFO_ex(jit, res_addr,
11597 static int zend_jit_rope(zend_jit_ctx *jit, const zend_op *opline, uint32_t op2_info) argument
11612 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ir_CONST_ADDR(str));
11619 ref = jit_Z_PTR(jit, op2_addr);
11620 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ref);
11624 if_refcounted = jit_if_REFCOUNTED(jit, op2_addr);
11626 jit_GC_ADDREF(jit, ref);
11639 ir_ADD_OFFSET(jit_FP(jit), opline->op1.var),
11642 jit_set_Z_PTR(jit, res_addr, ref);
11643 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
11649 static int zend_jit_zval_copy_deref_reg(zend_jit_ctx *jit, zend_jit_addr res_addr, uint32_t res_inf… argument
11656 val = jit_ZVAL_ADDR(jit, val_addr);
11659 val = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val));
11664 val = jit_ZVAL_ADDR(jit, val_addr);
11667 val = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val));
11675 static int zend_jit_zval_copy_deref(zend_jit_ctx *jit, zend_jit_addr res_addr, zend_jit_addr val_ad… argument
11680 ir_ref val = jit_ZVAL_ADDR(jit, val_addr);
11692 ptr = jit_Z_PTR(jit, val_addr);
11713 type2 = jit_Z_TYPE_INFO_ref(jit, val2);
11714 ptr2 = jit_Z_PTR_ref(jit, val2);
11735 jit_GC_ADDREF(jit, ptr);
11752 jit_set_Z_PTR(jit, res_addr, ptr);
11754 jit_set_Z_W2(jit, res_addr, jit_Z_W2(jit, val_addr));
11756 jit_set_Z_TYPE_INFO_ex(jit, res_addr, type);
11761 static int zend_jit_fetch_dimension_address_inner(zend_jit_ctx *jit, argument
11807 if_type = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
11828 h = jit_Z_LVAL(jit, op2_addr);
11845 h = jit_Z_LVAL(jit, op2_addr);
11923 h = jit_Z_LVAL(jit, op2_addr);
11949 ir_ref type_ref = jit_Z_TYPE_ref(jit, ref);
11973 h = jit_Z_LVAL(jit, op2_addr);
12000 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
12002 jit_SIDE_EXIT(jit, ir_CONST_ADDR(not_found_exit_addr));
12017 jit_SET_EX_OPLINE(jit, opline);
12021 h = jit_Z_LVAL(jit, op2_addr);
12026 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key_ex), h, jit_FP(jit));
12029 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_offset, IR_FASTCALL_FUNC));
12037 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12052 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
12074 h = jit_Z_LVAL(jit, op2_addr);
12096 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
12114 h = jit_Z_LVAL(jit, op2_addr);
12136 if_type = jit_if_Z_TYPE(jit, op2_addr, IS_STRING);
12141 key = jit_Z_PTR(jit, op2_addr);
12209 jit_SET_EX_OPLINE(jit, opline);
12210 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_key, IR_FASTCALL_FUNC));
12217 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12265 jit_SET_EX_OPLINE(jit, opline);
12267 ref = jit_ZVAL_ADDR(jit, op2_addr);
12273 jit_ZVAL_ADDR(jit, res_addr));
12297 jit_ZVAL_ADDR(jit, res_addr));
12336 ref = jit_ZVAL_DEREF_ref(jit, ref);
12338 cond = ir_GT(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_NULL));
12357 static int zend_jit_fetch_dim_read(zend_jit_ctx *jit, argument
12455 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
12456 ref = jit_ZVAL_DEREF_ref(jit, ref);
12470 jit_guard_Z_TYPE(jit, op1_addr, IS_ARRAY, exit_addr);
12472 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
12477 ht_ref = jit_Z_PTR(jit, op1_addr);
12484 if (!zend_jit_fetch_dimension_address_inner(jit, opline,
12511 val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
12523 jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
12525 } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
12530 ir_ref type_info = jit_Z_TYPE_INFO(jit, val_addr);
12531 if (!zend_jit_zval_copy_deref(jit, res_addr, val_addr, type_info)) {
12536 jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, 1);
12542 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12544 } else if (!end_inputs && jit->ctx.control) {
12561 jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, exit_addr);
12563 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
12567 jit_SET_EX_OPLINE(jit, opline);
12568 str_ref = jit_Z_PTR(jit, op1_addr);
12574 str_ref, jit_Z_LVAL(jit, op2_addr));
12577 str_ref, jit_ZVAL_ADDR(jit, op2_addr));
12579 jit_set_Z_PTR(jit, res_addr, ref);
12580 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
12584 jit_ZVAL_ADDR(jit, op2_addr),
12585 jit_ZVAL_ADDR(jit, res_addr));
12601 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
12603 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
12608 jit_SET_EX_OPLINE(jit, opline);
12613 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
12618 jit_ZVAL_ADDR(jit, op1_addr),
12620 jit_ZVAL_ADDR(jit, res_addr));
12623 jit_ZVAL_ADDR(jit, op1_addr),
12625 jit_ZVAL_ADDR(jit, res_addr));
12640 jit_SET_EX_OPLINE(jit, opline);
12643 zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op1_addr), opline->op1.var, NULL, 0, 1, 0);
12648 zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op2_addr), opline->op2.var, NULL, 0, 1, 0);
12657 ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
12659 jit_SET_EX_OPLINE(jit, opline);
12660 ref = jit_ZVAL_ADDR(jit, op1_addr);
12665 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12684 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
12691 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
12696 zend_jit_check_exception(jit);
12705 static zend_jit_addr zend_jit_prepare_array_update(zend_jit_ctx *jit, argument
12720 ref = jit_ZVAL_ADDR(jit, op1_addr);
12725 if_reference = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
12729 array_reference_ref = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val));
12730 if_array = jit_if_Z_TYPE_ref(jit, array_reference_ref, ir_CONST_U8(IS_ARRAY));
12735 jit_SET_EX_OPLINE(jit, opline);
12738 ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
12749 *if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
12757 ref = jit_Z_PTR_ref(jit, op1_ref);
12760 ir_ref if_refcount_1 = ir_IF(ir_EQ(jit_GC_REFCOUNT(jit, ref), ir_CONST_U32(1)));
12780 *if_type = ir_IF(ir_LE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL)));
12789 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
12801 jit_STUB_FUNC_ADDR(jit, jit_stub_new_array, IR_FASTCALL_FUNC),
12802 jit_ZVAL_ADDR(jit, op1_addr));
12818 static int zend_jit_fetch_dim(zend_jit_ctx *jit, argument
12833 jit_SET_EX_OPLINE(jit, opline);
12836 …op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_…
12856 jit_SET_EX_OPLINE(jit, opline);
12858 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
12862 jit_set_Z_PTR(jit, res_addr, ref);
12863 jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
12888 if (!zend_jit_fetch_dimension_address_inner(jit, opline, type, op1_info,
12897 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12910 jit_set_Z_PTR(jit, res_addr, ref);
12911 jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
12929 jit_SET_EX_OPLINE(jit, opline);
12938 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
12945 jit_ZVAL_ADDR(jit, op1_addr),
12947 jit_ZVAL_ADDR(jit, res_addr));
12951 jit_ZVAL_ADDR(jit, op1_addr),
12953 jit_ZVAL_ADDR(jit, res_addr));
12984 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
12987 zend_jit_check_exception(jit);
12993 static int zend_jit_isset_isempty_dim(zend_jit_ctx *jit, argument
13021 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
13022 ref = jit_ZVAL_DEREF_ref(jit, ref);
13032 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
13036 ht_ref = jit_Z_PTR(jit, op1_addr);
13049 if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_JIT_IS, op1_info,
13073 jit_SET_EX_OPLINE(jit, opline);
13074 arg1 = jit_ZVAL_ADDR(jit, op1_addr);
13079 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
13092 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op2_addr));
13097 jit_SET_EX_OPLINE(jit, opline);
13117 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
13119 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
13122 zend_jit_check_exception_undef_result(jit, opline);
13127 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
13133 _zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
13135 _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
13140 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
13149 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
13151 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
13154 zend_jit_check_exception_undef_result(jit, opline);
13159 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
13165 _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
13167 _zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
13172 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
13180 jit->b = -1;
13188 static int zend_jit_assign_dim(zend_jit_ctx *jit, argument
13206 if (!zend_jit_update_regs(jit, (opline+1)->op1.var, op3_addr, op3_def_addr, val_info)) {
13222 jit_guard_not_Z_TYPE(jit, op3_addr, IS_UNDEF, exit_addr);
13227 …op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_…
13244 jit_SET_EX_OPLINE(jit, opline);
13245 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
13251 …if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, v…
13263 if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_W, op1_info,
13286 …if (!zend_jit_assign_to_variable_call(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->o…
13290 …if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_ty…
13308 jit_SET_EX_OPLINE(jit, opline);
13316 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
13322 arg4 = jit_ZVAL_ADDR(jit, res_addr);
13325 jit_ZVAL_ADDR(jit, op1_addr),
13327 jit_ZVAL_ADDR(jit, op3_addr),
13337 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, NULL);
13350 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
13353 zend_jit_check_exception(jit);
13359 static int zend_jit_assign_dim_op(zend_jit_ctx *jit, argument
13383 jit_SET_EX_OPLINE(jit, opline);
13386 …op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_…
13390 && jit->ra[Z_SSA_VAR(op3_addr)].ref == IR_NULL) {
13392 zend_jit_use_reg(jit, op3_addr);
13412 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
13445 if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_RW, op1_info,
13457 jit_guard_Z_TYPE(jit, var_addr, dim_type, not_found_exit_addr);
13464 ref = jit_ZVAL_ADDR(jit, var_addr);
13465 if_ref = jit_if_Z_TYPE(jit, var_addr, IS_REFERENCE);
13470 reference = jit_Z_PTR_ref(jit, ref);
13472 if_typed = jit_if_TYPED_REF(jit, reference);
13477 arg2 = jit_ZVAL_ADDR(jit, op3_addr);
13505 …if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_i…
13516 if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
13525 …if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opl…
13553 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
13557 jit_ZVAL_ADDR(jit, op1_addr),
13559 jit_ZVAL_ADDR(jit, op3_addr),
13568 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
13569 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
13571 zend_jit_check_exception(jit);
13577 static int zend_jit_fe_reset(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info) argument
13585 jit_ZVAL_COPY_CONST(jit, res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
13589 jit_ZVAL_COPY(jit, res_addr, -1, op1_addr, op1_info, opline->op1_type == IS_CV);
13593 …ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->result.var + offsetof(zval, u2.fe_pos)), ir_CONST_U32(…
13598 static int zend_jit_packed_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint32_t o… argument
13610 ir_LOAD_U32(ir_ADD_OFFSET(jit_Z_PTR(jit, addr), offsetof(zend_array, u.flags))),
13621 static int zend_jit_fe_fetch(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t … argument
13632 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
13637 ZEND_ASSERT(jit->b >= 0);
13638 bb = &jit->ssa->cfg.blocks[jit->b];
13639 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
13640 jit->b = -1;
13647 ht_ref = jit_Z_PTR(jit, op1_addr);
13650 if (!zend_jit_packed_guard(jit, opline, opline->op1.var, op1_info)) {
13656 …hash_pos_ref = packed_pos_ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(…
13712 if_def_hash = ir_IF(jit_Z_TYPE_ref(jit, hash_p_ref));
13715 ir_GUARD_NOT(jit_Z_TYPE_ref(jit, hash_p_ref), ir_CONST_ADDR(exit_addr));
13774 if_def_packed = ir_IF(jit_Z_TYPE_ref(jit, packed_p_ref));
13777 ir_GUARD_NOT(jit_Z_TYPE_ref(jit, packed_p_ref), ir_CONST_ADDR(exit_addr));
13804 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
13820 jit_set_Z_PTR(jit, res_addr, key_ref);
13827 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
13832 jit_GC_ADDREF(jit, key_ref);
13833 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
13847 jit_set_Z_LVAL(jit, res_addr, ref);
13848 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
13865 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
13874 jit_set_Z_LVAL(jit, res_addr, packed_pos_ref);
13875 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
13905 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
13923 …if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, v…
13928 jit_ZVAL_COPY(jit, var_addr, -1, val_addr, val_info, 1);
13934 ZEND_ASSERT(jit->b >= 0);
13935 bb = &jit->ssa->cfg.blocks[jit->b];
13936 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ir_END());
13938 if (!jit->ctx.ir_base[exit_inputs].op2) {
13944 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
13945 jit->b = -1;
13955 static int zend_jit_load_this(zend_jit_ctx *jit, uint32_t var) argument
13959 ir_ref ref = jit_Z_PTR(jit, this_addr);
13961 jit_set_Z_PTR(jit, var_addr, ref);
13962 jit_set_Z_TYPE_INFO(jit, var_addr, IS_OBJECT_EX);
13963 jit_GC_ADDREF(jit, ref);
13968 static int zend_jit_fetch_this(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_ar… argument
13985 jit_guard_Z_TYPE(jit, this_addr, IS_OBJECT, exit_addr);
13993 ir_ref if_object = jit_if_Z_TYPE(jit, this_addr, IS_OBJECT);
13996 jit_SET_EX_OPLINE(jit, opline);
13997 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_invalid_this));
14004 if (!zend_jit_load_this(jit, opline->result.var)) {
14012 static int zend_jit_class_guard(zend_jit_ctx *jit, const zend_op *opline, ir_ref obj_ref, zend_clas… argument
14027 static int zend_jit_fetch_obj(zend_jit_ctx *jit, argument
14066 obj_ref = jit_Z_PTR(jit, this_addr);
14072 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
14075 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
14085 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
14087 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
14093 jit_SET_EX_OPLINE(jit, opline);
14099 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
14109 op1_ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
14111 op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
14116 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
14120 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
14123 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
14130 obj_ref = jit_Z_PTR(jit, op1_addr);
14143 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
14181 jit_SET_EX_OPLINE(jit, opline);
14208 prop_type_ref = jit_Z_TYPE_ref(jit, prop_ref);
14250 ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
14252 ref = jit_Z_PTR(jit, prop_addr);
14253 jit_GC_ADDREF(jit, ref);
14254 jit_set_Z_PTR(jit, res_addr, ref);
14255 jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
14260 jit_SET_EX_OPLINE(jit, opline);
14264 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
14270 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
14276 jit_SET_EX_OPLINE(jit, opline);
14285 jit_ZVAL_ADDR(jit, res_addr));
14305 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
14309 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
14316 ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
14318 ir_ref ref = jit_Z_PTR(jit, prop_addr);
14319 jit_GC_ADDREF(jit, ref);
14320 jit_set_Z_PTR(jit, res_addr, ref);
14321 jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
14325 jit_SET_EX_OPLINE(jit, opline);
14327 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
14339 ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
14341 ir_ref ref = jit_Z_PTR(jit, prop_addr);
14342 jit_GC_ADDREF(jit, ref);
14343 jit_set_Z_PTR(jit, res_addr, ref);
14344 jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
14363 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
14367 jit_SET_EX_OPLINE(jit, opline);
14377 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
14395 jit_ZVAL_ADDR(jit, res_addr));
14406 jit_set_Z_PTR(jit, res_addr, prop_ref);
14407 jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
14414 ir_END_PHI_list(end_values, jit_ZVAL_ADDR(jit, prop_addr));
14416 ir_END_PHI_list(end_values, jit_ZVAL_ADDR(jit, prop_addr));
14418 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
14420 …if (!zend_jit_zval_copy_deref_reg(jit, res_addr, res_info & ~MAY_BE_GUARD, prop_addr, prop_type_re…
14424 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
14426 if (!zend_jit_zval_copy_deref(jit, res_addr, prop_addr, prop_type_ref)) {
14440 jit_SET_EX_OPLINE(jit, opline);
14488 val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
14499 jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
14501 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
14516 if_refcounted = jit_if_REFCOUNTED(jit, orig_op1_addr);
14520 ptr = jit_Z_PTR(jit, orig_op1_addr);
14521 refcount = jit_GC_DELREF(jit, ptr);
14526 jit_SET_EX_OPLINE(jit, opline);
14534 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
14548 zend_jit_check_exception(jit);
14554 static int zend_jit_assign_obj(zend_jit_ctx *jit, argument
14586 if (!zend_jit_update_regs(jit, (opline+1)->op1.var, val_addr, val_def_addr, val_info)) {
14604 obj_ref = jit_Z_PTR(jit, this_addr);
14609 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
14612 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
14622 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
14624 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
14627 jit_SET_EX_OPLINE(jit, opline);
14629 jit_ZVAL_ADDR(jit, op1_addr),
14633 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
14641 obj_ref = jit_Z_PTR(jit, op1_addr);
14654 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
14694 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
14710 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
14713 arg3 = jit_ZVAL_ADDR(jit, real_addr);
14715 arg3 = jit_ZVAL_ADDR(jit, val_addr);
14722 arg4 = jit_ZVAL_ADDR(jit, real_addr);
14724 arg4 = jit_ZVAL_ADDR(jit, res_addr);
14727 jit_SET_EX_OPLINE(jit, opline);
14742 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
14767 ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
14769 ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
14778 jit_SET_EX_OPLINE(jit, opline);
14791 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
14794 arg3 = jit_ZVAL_ADDR(jit, real_addr);
14796 arg3 = jit_ZVAL_ADDR(jit, val_addr);
14802 arg4 = jit_ZVAL_ADDR(jit, real_addr);
14804 arg4 = jit_ZVAL_ADDR(jit, res_addr);
14814 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
14826 …if (!zend_jit_assign_to_variable_call(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_t…
14830 …if (!zend_jit_assign_to_variable(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, …
14851 jit_SET_EX_OPLINE(jit, opline);
14855 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
14858 arg3 = jit_ZVAL_ADDR(jit, real_addr);
14860 arg3 = jit_ZVAL_ADDR(jit, val_addr);
14866 arg5 = jit_ZVAL_ADDR(jit, real_addr);
14868 arg5 = jit_ZVAL_ADDR(jit, res_addr);
14882 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
14895 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
14903 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
14907 zend_jit_check_exception(jit);
14913 static int zend_jit_assign_obj_op(zend_jit_ctx *jit, argument
14954 obj_ref = jit_Z_PTR(jit, this_addr);
14959 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
14962 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
14972 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
14974 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
14977 jit_SET_EX_OPLINE(jit, opline);
14982 jit_ZVAL_ADDR(jit, op1_addr),
14991 obj_ref = jit_Z_PTR(jit, op1_addr);
15004 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
15033 && jit->ra[Z_SSA_VAR(val_addr)].ref == IR_NULL) {
15035 zend_jit_use_reg(jit, val_addr);
15067 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
15086 ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
15088 ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
15099 jit_SET_EX_OPLINE(jit, opline);
15103 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
15106 arg2 = jit_ZVAL_ADDR(jit, real_addr);
15108 arg2 = jit_ZVAL_ADDR(jit, val_addr);
15111 if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
15116 reference = jit_Z_PTR(jit, prop_addr);
15118 if_typed = jit_if_TYPED_REF(jit, reference);
15168 jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
15177 if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
15182 reference = jit_Z_PTR(jit, var_addr);
15184 if_typed = jit_if_TYPED_REF(jit, reference);
15189 jit_SET_EX_OPLINE(jit, opline);
15193 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
15196 arg2 = jit_ZVAL_ADDR(jit, real_addr);
15198 arg2 = jit_ZVAL_ADDR(jit, val_addr);
15232 …if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_i…
15271 if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
15281 …if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opl…
15303 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
15306 arg3 = jit_ZVAL_ADDR(jit, real_addr);
15308 arg3 = jit_ZVAL_ADDR(jit, val_addr);
15310 jit_SET_EX_OPLINE(jit, opline);
15331 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
15337 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
15341 zend_jit_check_exception(jit);
15347 static int zend_jit_incdec_obj(zend_jit_ctx *jit, argument
15389 obj_ref = jit_Z_PTR(jit, this_addr);
15394 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
15397 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
15407 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
15409 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
15412 jit_SET_EX_OPLINE(jit, opline);
15414 jit_ZVAL_ADDR(jit, op1_addr),
15417 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
15421 obj_ref = jit_Z_PTR(jit, op1_addr);
15434 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
15490 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
15509 ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
15511 ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
15523 jit_SET_EX_OPLINE(jit, opline);
15571 jit_ZVAL_ADDR(jit, res_addr));
15590 jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
15598 if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
15603 reference = jit_Z_PTR(jit, var_addr);
15605 if_typed = jit_if_TYPED_REF(jit, reference);
15628 jit_SET_EX_OPLINE(jit, opline);
15631 (opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
15646 if_long = jit_if_Z_TYPE(jit, var_addr, IS_LONG);
15650 addr = jit_ZVAL_ADDR(jit, var_addr);
15654 jit_set_Z_LVAL(jit, res_addr, ref);
15655 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
15670 jit_set_Z_LVAL(jit, res_addr, ref);
15671 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
15685 jit_ZVAL_COPY(jit, res_addr, -1, var_addr, var_info, 1);
15690 jit_ZVAL_ADDR(jit, var_addr),
15691 jit_ZVAL_ADDR(jit, res_addr));
15694 jit_ZVAL_ADDR(jit, var_addr));
15699 jit_ZVAL_ADDR(jit, var_addr),
15700 jit_ZVAL_ADDR(jit, res_addr));
15703 jit_ZVAL_ADDR(jit, var_addr));
15713 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0));
15714 jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0x41e00000));
15716 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x43e0000000000000));
15718 jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
15721 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
15722 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
15724 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
15726 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
15730 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x00200000));
15731 jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0xc1e00000));
15733 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0xc3e0000000000000));
15735 jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
15738 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
15739 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
15741 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
15743 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
15765 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
15796 jit_SET_EX_OPLINE(jit, opline);
15802 (opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
15815 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
15819 zend_jit_check_exception(jit);
15825 static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array,… argument
15876 _zend_jit_add_predecessor_ref(jit, b, jit->b, ir_END());
15877 jit->b = -1;
15918 ref = jit_ZVAL_ADDR(jit, op1_addr);
15919 if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15926 jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
15928 ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
15934 ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
15938 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
15940 if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15951 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
15953 ir_ref if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
15959 ir_ref ref = jit_Z_LVAL(jit, op1_addr);
15995 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
16003 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
16015 _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
16018 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
16021 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
16023 jit->b = -1;
16026 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
16027 jit->b = -1;
16045 ref = jit_ZVAL_ADDR(jit, op1_addr);
16046 if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16053 jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
16055 ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16061 ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
16065 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
16067 if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16078 jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, fallback_label);
16080 ir_ref if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16087 ir_ref ref = jit_Z_PTR(jit, op1_addr);
16120 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
16128 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
16140 _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
16142 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
16145 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
16147 jit->b = -1;
16150 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
16151 jit->b = -1;
16161 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
16166 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
16169 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, default_label);
16171 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
16176 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
16182 ref = jit_Z_LVAL(jit, op1_addr);
16196 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16199 jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, default_label);
16201 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16206 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16212 ir_ref ref2 = jit_Z_PTR(jit, op1_addr);
16256 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
16264 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
16270 _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
16272 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
16279 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
16282 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
16293 jit_guard_Z_TYPE(jit, op1_addr, IS_UNDEF, default_label);
16295 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
16300 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
16307 jit_SET_EX_OPLINE(jit, opline);
16310 zend_jit_check_exception_undef_result(jit, opline);
16312 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
16324 if (jit->ctx.ir_base[ref].op == IR_SWITCH) {
16325 ZEND_ASSERT(jit->ctx.ir_base[ref].op3 == IR_UNUSED);
16326 jit->ctx.ir_base[ref].op3 = default_input_list;
16329 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
16332 jit->b = -1;
16341 static int zend_jit_start(zend_jit_ctx *jit, const zend_op_array *op_array, zend_ssa *ssa) argument
16346 …zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : (IR_START_BR_TARGET|IR_ENTRY_…
16348 jit->ctx.spill_base = ZREG_FP;
16350 jit->op_array = jit->current_op_array = op_array;
16351 jit->ssa = ssa;
16352 jit->bb_start_ref = zend_arena_calloc(&CG(arena), ssa->cfg.blocks_count * 2, sizeof(ir_ref));
16353 jit->bb_predecessors = jit->bb_start_ref + ssa->cfg.blocks_count;
16357 jit->bb_predecessors[i] = count;
16360 jit->bb_edges = zend_arena_calloc(&CG(arena), count, sizeof(ir_ref));
16364 jit_STORE_FP(jit, ref);
16365 jit->ctx.flags |= IR_FASTCALL_FUNC;
16371 static void *zend_jit_finish(zend_jit_ctx *jit) argument
16380 if (jit->name) {
16381 str = zend_string_copy(jit->name);
16383 str = zend_jit_func_name(jit->op_array);
16387 if (jit->op_array) {
16389 _zend_jit_fix_merges(jit);
16391 } else if (jit->trace) {
16392 jit->ctx.deoptimization_exits = jit->trace->exit_count;
16393 jit->ctx.get_exit_addr = zend_jit_trace_get_exit_addr;
16397 jit->ctx.flags |= IR_GEN_CACHE_DEMOTE;
16401 entry = zend_jit_ir_compile(&jit->ctx, &size, str ? ZSTR_VAL(str) : NULL);
16412 &jit->ctx, stderr);
16421 if (!(jit->ctx.flags & IR_FUNCTION)
16445 if (jit->op_array) {
16447 const zend_op_array *op_array = jit->op_array;
16457 if (jit->ctx.entries_count) {
16459 int i = jit->ctx.entries_count;
16461 ir_insn *insn = &jit->ctx.ir_base[jit->ctx.entries[--i]];
16467 zend_jit_trace_info *t = jit->trace;
16475 stack->reg = (jit->ctx.flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP;
16476 stack->ref = ir_get_spill_slot_offset(&jit->ctx, stack->ref);
16524 static int zend_jit_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint8_t type) argument
16533 ir_GUARD(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr));
16538 static int zend_jit_scalar_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var) argument
16547 ir_GUARD(ir_LT(jit_Z_TYPE(jit, addr), ir_CONST_U8(IS_STRING)), ir_CONST_ADDR(exit_addr));
16552 static bool zend_jit_noref_guard(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_addr) argument
16560 ir_GUARD(ir_NE(jit_Z_TYPE(jit, var_addr), ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
16565 static int zend_jit_trace_opline_guard(zend_jit_ctx *jit, const zend_op *opline) argument
16574 ir_GUARD(jit_CMP_IP(jit, IR_EQ, opline), ir_CONST_ADDR(exit_addr));
16575 zend_jit_set_last_valid_opline(jit, opline);
16580 static bool zend_jit_guard_reference(zend_jit_ctx *jit, argument
16598 ref = jit_Z_TYPE(jit, var_addr);
16602 ref = jit_Z_PTR(jit, var_addr);
16611 static bool zend_jit_fetch_reference(zend_jit_ctx *jit, argument
16634 ref = jit_Z_TYPE(jit, var_addr);
16640 jit_ZVAL_ADDR(jit, var_addr));
16643 ref = jit_Z_PTR(jit, var_addr);
16655 ref = jit_Z_TYPE(jit, var_addr);
16677 static bool zend_jit_fetch_indirect_var(zend_jit_ctx *jit, const zend_op *opline, uint8_t var_type,… argument
16692 jit_guard_Z_TYPE(jit, var_addr, IS_INDIRECT, exit_addr);
16693 ref = jit_Z_PTR(jit, var_addr);
16697 ref = jit_Z_PTR(jit, var_addr);
16716 jit_guard_Z_TYPE(jit, var_addr, var_type, exit_addr);
16734 static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_array, const zend_op *… argument
16743 zend_jit_set_ip(jit, opline);
16747 ref = jit_FP(jit);
16753 zend_jit_check_exception(jit);
16771 jit_STORE_FP(jit, ir_LOAD_A(addr));
16785 ir_GUARD(ir_NE(jit_IP(jit), ir_CONST_ADDR(zend_jit_halt_op)),
16786 jit_STUB_ADDR(jit, jit_stub_trace_halt));
16789 ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
16791 ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
16796 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_halt));
16881 ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), ir_CONST_ADDR(exit_addr));
16885 zend_jit_set_last_valid_opline(jit, trace->opline);
16890 static int zend_jit_deoptimizer_start(zend_jit_ctx *jit, argument
16895 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
16897 jit->ctx.spill_base = ZREG_FP;
16899 jit->op_array = NULL;
16900 jit->ssa = NULL;
16901 jit->name = zend_string_copy(name);
16903 jit->ctx.flags |= IR_SKIP_PROLOGUE;
16908 static int zend_jit_trace_start(zend_jit_ctx *jit, argument
16916 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
16918 jit->ctx.spill_base = ZREG_FP;
16920 jit->op_array = NULL;
16921 jit->current_op_array = op_array;
16922 jit->ssa = ssa;
16923 jit->name = zend_string_copy(name);
16928 jit_STORE_FP(jit, ref);
16929 jit->ctx.flags |= IR_FASTCALL_FUNC;
16934 jit->ctx.flags |= IR_SKIP_PROLOGUE;
16972 ir_set_op(&jit->ctx, ref, 3, EX_NUM_TO_VAR(i));
16990 static int zend_jit_trace_begin_loop(zend_jit_ctx *jit) argument
16995 static void zend_jit_trace_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi) argument
17001 ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
17002 ZEND_ASSERT(jit->ra[src_var].ref != IR_UNUSED && jit->ra[src_var].ref != IR_NULL);
17005 (jit->ssa->var_info[src_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE,
17006 zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)), IR_UNUSED);
17009 ZEND_ASSERT(jit->ra[src_var].ref == IR_NULL);
17010 jit->ra[src_var].flags |= ZREG_FORWARD;
17012 zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref);
17015 static int zend_jit_trace_end_loop(zend_jit_ctx *jit, int loop_ref, const void *timeout_exit_addr) argument
17018 zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
17020 ZEND_ASSERT(jit->ctx.ir_base[loop_ref].op2 == IR_UNUSED);
17025 static int zend_jit_trace_return(zend_jit_ctx *jit, bool original_handler, const zend_op *opline) argument
17029 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
17031 ir_TAILCALL(IR_VOID, zend_jit_orig_opline_handler(jit));
17036 ir_ref addr = zend_jit_orig_opline_handler(jit);
17041 ref = ir_CALL_1(IR_I32, addr, jit_FP(jit));
17063 static int zend_jit_trace_link_to_root(zend_jit_ctx *jit, zend_jit_trace_info *t, const void *timeo… argument
17072 zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
17386 static ir_ref jit_frameless_observer(zend_jit_ctx *jit, const zend_op *opline) { argument
17391 …ir_ref if_unobserved = jit_observer_fcall_is_unobserved_start(jit, fbc, &observer_handler, IR_UNUS…
17394 ir_CALL_1(IR_VOID, ir_CONST_ADDR((size_t)zend_frameless_observed_call), jit_FP(jit));
17401 static void jit_frameless_icall0(zend_jit_ctx *jit, const zend_op *opline) argument
17403 jit_SET_EX_OPLINE(jit, opline);
17407 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
17408 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
17412 skip_observer = jit_frameless_observer(jit, opline);
17421 zend_jit_check_exception(jit);
17424 static void jit_frameless_icall1(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info) argument
17426 jit_SET_EX_OPLINE(jit, opline);
17436 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
17437 ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
17438 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
17440 op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
17446 op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
17451 skip_observer = jit_frameless_observer(jit, opline);
17460 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
17461 zend_jit_check_exception(jit);
17464 static void jit_frameless_icall2(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint3… argument
17466 jit_SET_EX_OPLINE(jit, opline);
17480 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
17481 ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
17482 ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
17483 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
17485 op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
17491 op2_ref = zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
17497 op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
17500 op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
17505 skip_observer = jit_frameless_observer(jit, opline);
17514 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
17520 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
17526 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
17527 zend_jit_check_exception(jit);
17530 static void jit_frameless_icall3(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint3… argument
17532 jit_SET_EX_OPLINE(jit, opline);
17551 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
17552 ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
17553 ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
17554 ir_ref op3_ref = jit_ZVAL_ADDR(jit, op3_addr);
17555 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
17557 op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
17563 op2_ref = zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
17569 op3_ref = zend_jit_zval_check_undef(jit, op3_ref, (opline+1)->op1.var, opline, 1);
17575 op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
17578 op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
17581 op3_ref = jit_ZVAL_DEREF_ref(jit, op3_ref);
17586 skip_observer = jit_frameless_observer(jit, opline);
17595 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
17606 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
17612 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
17620 jit_set_Z_TYPE_INFO(jit, op2_addr, IS_UNDEF);
17626 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
17627 zend_jit_check_exception(jit);
