Lines Matching refs:jit
63 #define _ir_CTX (&jit->ctx)
66 #define ir_CONST_ADDR(_addr) jit_CONST_ADDR(jit, (uintptr_t)(_addr))
67 #define ir_CONST_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), 0)
68 #define ir_CONST_FC_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), IR_FASTCALL_FUNC)
73 jit_CONST_FUNC_PROTO(jit, (uintptr_t)(_addr), (_proto))
77 jit_ADD_OFFSET(jit, _addr, _offset)
190 ir_ADD_OFFSET(jit_TLS(jit), EG_TLS_OFFSET(_field))
193 ir_ADD_OFFSET(jit_TLS(jit), CG_TLS_OFFSET(_field))
209 jit_CALL(jit_FP(jit), _field)
212 jit_CALL(jit_IP(jit), _field)
252 static int zend_jit_ ## name ## _stub(zend_jit_ctx *jit);
321 static int zend_jit_assign_to_variable(zend_jit_ctx *jit,
336 int (*stub)(zend_jit_ctx *jit);
442 static bool zend_jit_prefer_const_addr_load(zend_jit_ctx *jit, uintptr_t addr) in zend_jit_prefer_const_addr_load() argument
463 static ir_ref jit_TLS(zend_jit_ctx *jit) in jit_TLS() argument
465 ZEND_ASSERT(jit->ctx.control); in jit_TLS()
466 if (jit->tls) { in jit_TLS()
469 ir_ref ref = jit->ctx.control; in jit_TLS()
472 if (ref == jit->tls) { in jit_TLS()
473 return jit->tls; in jit_TLS()
475 insn = &jit->ctx.ir_base[ref]; in jit_TLS()
482 jit->tls = ir_TLS( in jit_TLS()
485 return jit->tls; in jit_TLS()
489 static ir_ref jit_CONST_ADDR(zend_jit_ctx *jit, uintptr_t addr) in jit_CONST_ADDR() argument
497 zv = zend_hash_index_lookup(&jit->addr_hash, addr); in jit_CONST_ADDR()
500 ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_ADDR, IR_ADDR)); in jit_CONST_ADDR()
502 ref = ir_unique_const_addr(&jit->ctx, addr); in jit_CONST_ADDR()
508 static ir_ref jit_CONST_FUNC_PROTO(zend_jit_ctx *jit, uintptr_t addr, ir_ref proto) in jit_CONST_FUNC_PROTO() argument
515 zv = zend_hash_index_lookup(&jit->addr_hash, addr); in jit_CONST_FUNC_PROTO()
518 …ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_FUNC_ADDR, IR_ADDR) && jit->ctx.ir_base[ref].pr… in jit_CONST_FUNC_PROTO()
520 ref = ir_unique_const_addr(&jit->ctx, addr); in jit_CONST_FUNC_PROTO()
521 insn = &jit->ctx.ir_base[ref]; in jit_CONST_FUNC_PROTO()
529 static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags) in jit_CONST_FUNC() argument
533 ir_ref proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0; in jit_CONST_FUNC()
538 return jit_CONST_FUNC_PROTO(jit, addr, proto); in jit_CONST_FUNC()
541 static ir_ref jit_ADD_OFFSET(zend_jit_ctx *jit, ir_ref addr, uintptr_t offset) in jit_ADD_OFFSET() argument
549 static ir_ref jit_EG_exception(zend_jit_ctx *jit) in jit_EG_exception() argument
554 ir_ref ref = jit->eg_exception_addr; in jit_EG_exception()
557 ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)&EG(exception)); in jit_EG_exception()
558 jit->eg_exception_addr = ref; in jit_EG_exception()
564 static ir_ref jit_STUB_ADDR(zend_jit_ctx *jit, jit_stub_id id) in jit_STUB_ADDR() argument
566 ir_ref ref = jit->stub_addr[id]; in jit_STUB_ADDR()
569 ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]); in jit_STUB_ADDR()
570 jit->stub_addr[id] = ref; in jit_STUB_ADDR()
575 static ir_ref jit_STUB_FUNC_ADDR(zend_jit_ctx *jit, jit_stub_id id, uint16_t flags) in jit_STUB_FUNC_ADDR() argument
577 ir_ref ref = jit->stub_addr[id]; in jit_STUB_FUNC_ADDR()
581 ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]); in jit_STUB_FUNC_ADDR()
582 insn = &jit->ctx.ir_base[ref]; in jit_STUB_FUNC_ADDR()
586 insn->proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0; in jit_STUB_FUNC_ADDR()
590 jit->stub_addr[id] = ref; in jit_STUB_FUNC_ADDR()
595 static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr) in jit_SNAPSHOT() argument
598 const void *ptr = (const void*)jit->ctx.ir_base[addr].val.addr; in jit_SNAPSHOT()
622 zend_jit_trace_info *t = jit->trace; in jit_SNAPSHOT()
627 … && jit->ctx.ir_base[addr].val.u64 == (uintptr_t)zend_jit_trace_get_exit_addr(t->exit_count - 1)) { in jit_SNAPSHOT()
819 static void jit_SIDE_EXIT(zend_jit_ctx *jit, ir_ref addr) in jit_SIDE_EXIT() argument
821 jit_SNAPSHOT(jit, addr); in jit_SIDE_EXIT()
827 static ir_ref jit_EMALLOC(zend_jit_ctx *jit, size_t size, const zend_op_array *op_array, const zend… in jit_EMALLOC() argument
847 static ir_ref jit_EFREE(zend_jit_ctx *jit, ir_ref ptr, size_t size, const zend_op_array *op_array, … in jit_EFREE() argument
867 static ir_ref jit_FP(zend_jit_ctx *jit) in jit_FP() argument
869 ZEND_ASSERT(jit->ctx.control); in jit_FP()
870 if (jit->fp == IR_UNUSED) { in jit_FP()
872 jit->fp = ir_RLOAD_A(ZREG_FP); in jit_FP()
875 ir_ref ref = jit->ctx.control; in jit_FP()
878 if (ref == jit->fp) { in jit_FP()
881 insn = &jit->ctx.ir_base[ref]; in jit_FP()
883 jit->fp = ir_RLOAD_A(ZREG_FP); in jit_FP()
889 return jit->fp; in jit_FP()
892 static void jit_STORE_FP(zend_jit_ctx *jit, ir_ref ref) in jit_STORE_FP() argument
895 jit->fp = IR_UNUSED; in jit_STORE_FP()
898 static ir_ref jit_IP(zend_jit_ctx *jit) in jit_IP() argument
903 static void jit_STORE_IP(zend_jit_ctx *jit, ir_ref ref) in jit_STORE_IP() argument
908 static ir_ref jit_IP32(zend_jit_ctx *jit) in jit_IP32() argument
913 static void jit_LOAD_IP(zend_jit_ctx *jit, ir_ref ref) in jit_LOAD_IP() argument
916 jit_STORE_IP(jit, ref); in jit_LOAD_IP()
922 static void jit_LOAD_IP_ADDR(zend_jit_ctx *jit, const zend_op *target) in jit_LOAD_IP_ADDR() argument
924 jit_LOAD_IP(jit, ir_CONST_ADDR(target)); in jit_LOAD_IP_ADDR()
927 static void zend_jit_track_last_valid_opline(zend_jit_ctx *jit) in zend_jit_track_last_valid_opline() argument
929 jit->use_last_valid_opline = 0; in zend_jit_track_last_valid_opline()
930 jit->track_last_valid_opline = 1; in zend_jit_track_last_valid_opline()
933 static void zend_jit_use_last_valid_opline(zend_jit_ctx *jit) in zend_jit_use_last_valid_opline() argument
935 if (jit->track_last_valid_opline) { in zend_jit_use_last_valid_opline()
936 jit->use_last_valid_opline = 1; in zend_jit_use_last_valid_opline()
937 jit->track_last_valid_opline = 0; in zend_jit_use_last_valid_opline()
941 static bool zend_jit_trace_uses_initial_ip(zend_jit_ctx *jit) in zend_jit_trace_uses_initial_ip() argument
943 return jit->use_last_valid_opline; in zend_jit_trace_uses_initial_ip()
946 static void zend_jit_set_last_valid_opline(zend_jit_ctx *jit, const zend_op *opline) in zend_jit_set_last_valid_opline() argument
948 if (!jit->reuse_ip) { in zend_jit_set_last_valid_opline()
949 jit->track_last_valid_opline = 1; in zend_jit_set_last_valid_opline()
950 jit->last_valid_opline = opline; in zend_jit_set_last_valid_opline()
954 static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit) in zend_jit_reset_last_valid_opline() argument
956 jit->track_last_valid_opline = 0; in zend_jit_reset_last_valid_opline()
957 jit->last_valid_opline = NULL; in zend_jit_reset_last_valid_opline()
960 static void zend_jit_start_reuse_ip(zend_jit_ctx *jit) in zend_jit_start_reuse_ip() argument
962 zend_jit_reset_last_valid_opline(jit); in zend_jit_start_reuse_ip()
963 jit->reuse_ip = 1; in zend_jit_start_reuse_ip()
966 static int zend_jit_reuse_ip(zend_jit_ctx *jit) in zend_jit_reuse_ip() argument
968 if (!jit->reuse_ip) { in zend_jit_reuse_ip()
969 zend_jit_start_reuse_ip(jit); in zend_jit_reuse_ip()
971 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call))); in zend_jit_reuse_ip()
976 static void zend_jit_stop_reuse_ip(zend_jit_ctx *jit) in zend_jit_stop_reuse_ip() argument
978 jit->reuse_ip = 0; in zend_jit_stop_reuse_ip()
981 static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level) in zend_jit_save_call_chain() argument
993 rx = jit_IP(jit); in zend_jit_save_call_chain()
1001 jit->delayed_call_level = 0; in zend_jit_save_call_chain()
1007 static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target) in zend_jit_set_ip() argument
1012 if (jit->delayed_call_level) { in zend_jit_set_ip()
1013 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { in zend_jit_set_ip()
1018 if (jit->last_valid_opline) { in zend_jit_set_ip()
1019 zend_jit_use_last_valid_opline(jit); in zend_jit_set_ip()
1020 if (jit->last_valid_opline != target) { in zend_jit_set_ip()
1022 ref = jit_IP(jit); in zend_jit_set_ip()
1027 if (target > jit->last_valid_opline) { in zend_jit_set_ip()
1028 ref = ir_ADD_OFFSET(ref, (uintptr_t)target - (uintptr_t)jit->last_valid_opline); in zend_jit_set_ip()
1030 ref = ir_SUB_A(ref, ir_CONST_ADDR((uintptr_t)jit->last_valid_opline - (uintptr_t)target)); in zend_jit_set_ip()
1033 jit_STORE_IP(jit, ref); in zend_jit_set_ip()
1040 jit_STORE_IP(jit, ir_CONST_ADDR(target)); in zend_jit_set_ip()
1045 jit->reuse_ip = 0; in zend_jit_set_ip()
1046 zend_jit_set_last_valid_opline(jit, target); in zend_jit_set_ip()
1050 static int zend_jit_set_ip_ex(zend_jit_ctx *jit, const zend_op *target, bool set_ip_reg) in zend_jit_set_ip_ex() argument
1052 if (!GCC_GLOBAL_REGS && set_ip_reg && !jit->last_valid_opline) { in zend_jit_set_ip_ex()
1057 return zend_jit_set_ip(jit, target); in zend_jit_set_ip_ex()
1060 static void jit_SET_EX_OPLINE(zend_jit_ctx *jit, const zend_op *target) in jit_SET_EX_OPLINE() argument
1062 if (jit->last_valid_opline == target) { in jit_SET_EX_OPLINE()
1063 zend_jit_use_last_valid_opline(jit); in jit_SET_EX_OPLINE()
1066 ir_STORE(jit_EX(opline), jit_IP(jit)); in jit_SET_EX_OPLINE()
1071 zend_jit_reset_last_valid_opline(jit); in jit_SET_EX_OPLINE()
1076 static ir_ref jit_ZVAL_ADDR(zend_jit_ctx *jit, zend_jit_addr addr) in jit_ZVAL_ADDR() argument
1082 reg = jit_FP(jit); in jit_ZVAL_ADDR()
1084 reg = jit_IP(jit); in jit_ZVAL_ADDR()
1097 static ir_ref jit_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref) in jit_Z_TYPE_ref() argument
1102 static ir_ref jit_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_TYPE() argument
1111 reg = jit_FP(jit); in jit_Z_TYPE()
1113 reg = jit_IP(jit); in jit_Z_TYPE()
1119 return jit_Z_TYPE_ref(jit, jit_ZVAL_ADDR(jit, addr)); in jit_Z_TYPE()
1123 static ir_ref jit_Z_TYPE_FLAGS_ref(zend_jit_ctx *jit, ir_ref ref) in jit_Z_TYPE_FLAGS_ref() argument
1128 static ir_ref jit_Z_TYPE_FLAGS(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_TYPE_FLAGS() argument
1137 reg = jit_FP(jit); in jit_Z_TYPE_FLAGS()
1139 reg = jit_IP(jit); in jit_Z_TYPE_FLAGS()
1145 return jit_Z_TYPE_FLAGS_ref(jit, jit_ZVAL_ADDR(jit, addr)); in jit_Z_TYPE_FLAGS()
1149 static ir_ref jit_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref) in jit_Z_TYPE_INFO_ref() argument
1154 static ir_ref jit_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_TYPE_INFO() argument
1163 reg = jit_FP(jit); in jit_Z_TYPE_INFO()
1165 reg = jit_IP(jit); in jit_Z_TYPE_INFO()
1171 return jit_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr)); in jit_Z_TYPE_INFO()
1175 static void jit_set_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type_info) in jit_set_Z_TYPE_INFO_ref() argument
1180 static void jit_set_Z_TYPE_INFO_ex(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref type_info) in jit_set_Z_TYPE_INFO_ex() argument
1187 reg = jit_FP(jit); in jit_set_Z_TYPE_INFO_ex()
1189 reg = jit_IP(jit); in jit_set_Z_TYPE_INFO_ex()
1195 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr), type_info); in jit_set_Z_TYPE_INFO_ex()
1199 static void jit_set_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr, uint32_t type_info) in jit_set_Z_TYPE_INFO() argument
1209 jit_set_Z_TYPE_INFO_ex(jit, addr, ir_CONST_U32(type_info)); in jit_set_Z_TYPE_INFO()
1212 static ir_ref jit_if_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type) in jit_if_Z_TYPE_ref() argument
1214 return ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), type)); in jit_if_Z_TYPE_ref()
1217 static ir_ref jit_if_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type) in jit_if_Z_TYPE() argument
1220 return ir_IF(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type))); in jit_if_Z_TYPE()
1223 static ir_ref jit_if_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type) in jit_if_not_Z_TYPE() argument
1225 ir_ref ref = jit_Z_TYPE(jit, addr); in jit_if_not_Z_TYPE()
1233 static void jit_guard_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_… in jit_guard_Z_TYPE() argument
1235 ir_ref ref = jit_Z_TYPE(jit, addr); in jit_guard_Z_TYPE()
1244 static void jit_guard_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *e… in jit_guard_not_Z_TYPE() argument
1246 ir_ref ref = jit_Z_TYPE(jit, addr); in jit_guard_not_Z_TYPE()
1254 static ir_ref jit_if_REFCOUNTED(zend_jit_ctx *jit, zend_jit_addr addr) in jit_if_REFCOUNTED() argument
1256 return ir_IF(jit_Z_TYPE_FLAGS(jit, addr)); in jit_if_REFCOUNTED()
1259 static ir_ref jit_if_COLLECTABLE_ref(zend_jit_ctx *jit, ir_ref addr_ref) in jit_if_COLLECTABLE_ref() argument
1261 return ir_IF(ir_AND_U8(jit_Z_TYPE_FLAGS_ref(jit, addr_ref), ir_CONST_U8(IS_TYPE_COLLECTABLE))); in jit_if_COLLECTABLE_ref()
1264 static ir_ref jit_Z_LVAL_ref(zend_jit_ctx *jit, ir_ref ref) in jit_Z_LVAL_ref() argument
1269 static ir_ref jit_Z_DVAL_ref(zend_jit_ctx *jit, ir_ref ref) in jit_Z_DVAL_ref() argument
1274 static bool zend_jit_spilling_may_cause_conflict(zend_jit_ctx *jit, int var, ir_ref val) in zend_jit_spilling_may_cause_conflict() argument
1276 if (jit->ctx.ir_base[val].op == IR_RLOAD) { in zend_jit_spilling_may_cause_conflict()
1290 if (jit->ssa->vars[var].var < jit->current_op_array->last_var) { in zend_jit_spilling_may_cause_conflict()
1292 if (jit->ctx.ir_base[val].op == IR_LOAD in zend_jit_spilling_may_cause_conflict()
1293 && jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op == IR_ADD in zend_jit_spilling_may_cause_conflict()
1294 && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op == IR_RLOAD in zend_jit_spilling_may_cause_conflict()
1295 && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op2 == ZREG_FP in zend_jit_spilling_may_cause_conflict()
1296 && IR_IS_CONST_REF(jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2) in zend_jit_spilling_may_cause_conflict()
1297 …&& jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr != (uintptr_t)EX_NUM… in zend_jit_spilling_may_cause_conflict()
1298 …&& EX_VAR_TO_NUM(jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr) < jit… in zend_jit_spilling_may_cause_conflict()
1301 } else if (jit->ssa->vars[var].definition >= 0 in zend_jit_spilling_may_cause_conflict()
1302 && jit->ssa->ops[jit->ssa->vars[var].definition].op1_def == var in zend_jit_spilling_may_cause_conflict()
1303 && jit->ssa->ops[jit->ssa->vars[var].definition].op1_use >= 0 in zend_jit_spilling_may_cause_conflict()
1304 && jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].no_val in zend_jit_spilling_may_cause_conflict()
1305 && jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].definition_phi in zend_jit_spilling_may_cause_conflict()
1306 …&& (jit->ssa->cfg.blocks[jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].def… in zend_jit_spilling_may_cause_conflict()
1315 static void zend_jit_def_reg(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref val) in zend_jit_def_reg() argument
1321 if (var == jit->delay_var) { in zend_jit_def_reg()
1322 ir_refs_add(jit->delay_refs, val); in zend_jit_def_reg()
1325 ZEND_ASSERT(jit->ra && jit->ra[var].ref == IR_NULL); in zend_jit_def_reg()
1328 if (val > 0 && !zend_jit_spilling_may_cause_conflict(jit, var, val)) { in zend_jit_def_reg()
1329 val = ir_bind(&jit->ctx, -EX_NUM_TO_VAR(jit->ssa->vars[var].var), val); in zend_jit_def_reg()
1331 jit->ra[var].ref = val; in zend_jit_def_reg()
1333 if (jit->ra[var].flags & ZREG_FORWARD) { in zend_jit_def_reg()
1334 zend_ssa_phi *phi = jit->ssa->vars[var].phi_use_chain; in zend_jit_def_reg()
1339 jit->ra[var].flags &= ~ZREG_FORWARD; in zend_jit_def_reg()
1345 jit->ra[src_var].ref = val; in zend_jit_def_reg()
1347 if (!(jit->ra[src_var].flags & ZREG_FORWARD)) { in zend_jit_def_reg()
1348 phi = zend_ssa_next_use_phi(jit->ssa, var, phi); in zend_jit_def_reg()
1351 dst_phi = jit->ssa->vars[src_var].phi_use_chain; in zend_jit_def_reg()
1353 ZEND_ASSERT(!zend_ssa_next_use_phi(jit->ssa, src_var, dst_phi) && "reg forwarding"); in zend_jit_def_reg()
1354 jit->ra[src_var].flags &= ~ZREG_FORWARD; in zend_jit_def_reg()
1357 if (jit->ra[dst_phi->ssa_var].ref > 0) { in zend_jit_def_reg()
1358 ir_insn *phi_insn = &jit->ctx.ir_base[jit->ra[dst_phi->ssa_var].ref]; in zend_jit_def_reg()
1361 bb = &jit->ssa->cfg.blocks[dst_phi->block]; in zend_jit_def_reg()
1371 phi = zend_ssa_next_use_phi(jit->ssa, var, phi); in zend_jit_def_reg()
1376 static ir_ref zend_jit_use_reg(zend_jit_ctx *jit, zend_jit_addr addr) in zend_jit_use_reg() argument
1381 ZEND_ASSERT(jit->ra && jit->ra[var].ref); in zend_jit_use_reg()
1382 if (jit->ra[var].ref == IR_NULL) { in zend_jit_use_reg()
1386 ZEND_ASSERT(jit->ra[var].flags & ZREG_LOAD); in zend_jit_use_reg()
1387 mem_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(jit->ssa->vars[var].var)); in zend_jit_use_reg()
1388 if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_LONG) { in zend_jit_use_reg()
1389 ref = jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr)); in zend_jit_use_reg()
1390 } else if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { in zend_jit_use_reg()
1391 ref = jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr)); in zend_jit_use_reg()
1395 zend_jit_def_reg(jit, addr, ref); in zend_jit_use_reg()
1398 return jit->ra[Z_SSA_VAR(addr)].ref; in zend_jit_use_reg()
1401 static void zend_jit_gen_pi(zend_jit_ctx *jit, zend_ssa_phi *phi) in zend_jit_gen_pi() argument
1407 ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD)); in zend_jit_gen_pi()
1408 ZEND_ASSERT(jit->ra[src_var].ref); in zend_jit_gen_pi()
1410 if (jit->ra[src_var].ref == IR_NULL) { in zend_jit_gen_pi()
1412 if (jit->ssa->vars[dst_var].use_chain < 0 in zend_jit_gen_pi()
1413 && jit->ssa->vars[dst_var].phi_use_chain) { in zend_jit_gen_pi()
1414 zend_ssa_phi *phi = jit->ssa->vars[dst_var].phi_use_chain; in zend_jit_gen_pi()
1415 if (!zend_ssa_next_use_phi(jit->ssa, dst_var, phi)) { in zend_jit_gen_pi()
1417 jit->ra[src_var].flags |= ZREG_FORWARD; in zend_jit_gen_pi()
1424 zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), in zend_jit_gen_pi()
1425 zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var))); in zend_jit_gen_pi()
1428 static void zend_jit_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi) in zend_jit_gen_phi() argument
1431 zend_basic_block *bb = &jit->ssa->cfg.blocks[phi->block]; in zend_jit_gen_phi()
1434 ir_type type = (jit->ssa->var_info[phi->ssa_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE; in zend_jit_gen_phi()
1435 ir_ref merge = jit->bb_start_ref[phi->block]; in zend_jit_gen_phi()
1437 ir_ref old_insns_count = jit->ctx.insns_count; in zend_jit_gen_phi()
1442 ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD)); in zend_jit_gen_phi()
1444 ZEND_ASSERT(jit->ctx.ir_base[merge].op == IR_MERGE || jit->ctx.ir_base[merge].op == IR_LOOP_BEGIN); in zend_jit_gen_phi()
1445 ZEND_ASSERT(n == jit->ctx.ir_base[merge].inputs_count); in zend_jit_gen_phi()
1447 ref = ir_emit_N(&jit->ctx, IR_OPT(IR_PHI, type), n + 1); in zend_jit_gen_phi()
1448 ir_set_op(&jit->ctx, ref, 1, merge); in zend_jit_gen_phi()
1453 ZEND_ASSERT(jit->ra[src_var].ref); in zend_jit_gen_phi()
1454 if (jit->ra[src_var].ref == IR_NULL) { in zend_jit_gen_phi()
1455 jit->ra[src_var].flags |= ZREG_FORWARD; in zend_jit_gen_phi()
1458 ir_ref src_ref = zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)); in zend_jit_gen_phi()
1464 ir_set_op(&jit->ctx, ref, i + 2, src_ref); in zend_jit_gen_phi()
1469 jit->ctx.insns_count = old_insns_count; in zend_jit_gen_phi()
1472 zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref); in zend_jit_gen_phi()
1475 static ir_ref jit_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_LVAL() argument
1480 return zend_jit_use_reg(jit, addr); in jit_Z_LVAL()
1482 return jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, addr)); in jit_Z_LVAL()
1486 static void jit_set_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval) in jit_set_Z_LVAL() argument
1489 zend_jit_def_reg(jit, addr, lval); in jit_set_Z_LVAL()
1491 ir_STORE(jit_ZVAL_ADDR(jit, addr), lval); in jit_set_Z_LVAL()
1496 static ir_ref jit_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_W2() argument
1501 return ir_LOAD_L(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2))); in jit_Z_W2()
1505 static void jit_set_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval) in jit_set_Z_W2() argument
1507 ir_STORE(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2)), lval); in jit_set_Z_W2()
1511 static ir_ref jit_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_DVAL() argument
1516 return zend_jit_use_reg(jit, addr); in jit_Z_DVAL()
1518 return jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, addr)); in jit_Z_DVAL()
1522 static void jit_set_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref dval) in jit_set_Z_DVAL() argument
1525 zend_jit_def_reg(jit, addr, dval); in jit_set_Z_DVAL()
1527 ir_STORE(jit_ZVAL_ADDR(jit, addr), dval); in jit_set_Z_DVAL()
1531 static ir_ref jit_Z_PTR_ref(zend_jit_ctx *jit, ir_ref ref) in jit_Z_PTR_ref() argument
1536 static ir_ref jit_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr) in jit_Z_PTR() argument
1541 return jit_Z_PTR_ref(jit, jit_ZVAL_ADDR(jit, addr)); in jit_Z_PTR()
1545 static void jit_set_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref ptr) in jit_set_Z_PTR() argument
1547 ir_STORE(jit_ZVAL_ADDR(jit, addr), ptr); in jit_set_Z_PTR()
1550 static ir_ref jit_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref) in jit_GC_REFCOUNT() argument
1555 static void jit_set_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref, uint32_t refcount) in jit_set_GC_REFCOUNT() argument
1560 static void jit_GC_ADDREF(zend_jit_ctx *jit, ir_ref ref) in jit_GC_ADDREF() argument
1565 static void jit_GC_ADDREF2(zend_jit_ctx *jit, ir_ref ref) in jit_GC_ADDREF2() argument
1571 static ir_ref jit_GC_DELREF(zend_jit_ctx *jit, ir_ref ref) in jit_GC_DELREF() argument
1579 static ir_ref jit_if_GC_MAY_NOT_LEAK(zend_jit_ctx *jit, ir_ref ref) in jit_if_GC_MAY_NOT_LEAK() argument
1587 static void jit_ZVAL_COPY_CONST(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, uint32_t d… in jit_ZVAL_COPY_CONST() argument
1593 jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE(Z_DVAL_P(zv))); in jit_ZVAL_COPY_CONST()
1595 jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE((double)Z_LVAL_P(zv))); in jit_ZVAL_COPY_CONST()
1597 jit_set_Z_LVAL(jit, dst, ir_CONST_LONG(Z_LVAL_P(zv))); in jit_ZVAL_COPY_CONST()
1600 jit_set_Z_PTR(jit, dst, ref); in jit_ZVAL_COPY_CONST()
1602 jit_GC_ADDREF(jit, ref); in jit_ZVAL_COPY_CONST()
1609 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in jit_ZVAL_COPY_CONST()
1612 jit_set_Z_TYPE_INFO(jit, dst, Z_TYPE_INFO_P(zv)); in jit_ZVAL_COPY_CONST()
1617 static ir_ref jit_if_TYPED_REF(zend_jit_ctx *jit, ir_ref ref) in jit_if_TYPED_REF() argument
1622 static void jit_ZVAL_COPY(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, zend_jit_addr sr… in jit_ZVAL_COPY() argument
1628 jit_set_Z_LVAL(jit, dst, jit_Z_LVAL(jit, src)); in jit_ZVAL_COPY()
1630 jit_set_Z_DVAL(jit, dst, jit_Z_DVAL(jit, src)); in jit_ZVAL_COPY()
1634 jit_set_Z_W2(jit, dst, jit_Z_W2(jit, src)); in jit_ZVAL_COPY()
1637 ref = jit_Z_PTR(jit, src); in jit_ZVAL_COPY()
1638 jit_set_Z_PTR(jit, dst, ref); in jit_ZVAL_COPY()
1647 jit_set_Z_TYPE_INFO(jit, dst, type); in jit_ZVAL_COPY()
1650 ir_ref type = jit_Z_TYPE_INFO(jit, src); in jit_ZVAL_COPY()
1651 jit_set_Z_TYPE_INFO_ex(jit, dst, type); in jit_ZVAL_COPY()
1661 jit_GC_ADDREF(jit, ref); in jit_ZVAL_COPY()
1671 static void jit_ZVAL_COPY_2(zend_jit_ctx *jit, zend_jit_addr dst2, zend_jit_addr dst, uint32_t dst_… in jit_ZVAL_COPY_2() argument
1677 ref = jit_Z_LVAL(jit, src); in jit_ZVAL_COPY_2()
1678 jit_set_Z_LVAL(jit, dst, ref); in jit_ZVAL_COPY_2()
1679 jit_set_Z_LVAL(jit, dst2, ref); in jit_ZVAL_COPY_2()
1681 ref = jit_Z_DVAL(jit, src); in jit_ZVAL_COPY_2()
1682 jit_set_Z_DVAL(jit, dst, ref); in jit_ZVAL_COPY_2()
1683 jit_set_Z_DVAL(jit, dst2, ref); in jit_ZVAL_COPY_2()
1687 ref = jit_Z_W2(jit, src); in jit_ZVAL_COPY_2()
1688 jit_set_Z_W2(jit, dst, ref); in jit_ZVAL_COPY_2()
1689 jit_set_Z_W2(jit, dst2, ref); in jit_ZVAL_COPY_2()
1692 ref = jit_Z_PTR(jit, src); in jit_ZVAL_COPY_2()
1693 jit_set_Z_PTR(jit, dst, ref); in jit_ZVAL_COPY_2()
1694 jit_set_Z_PTR(jit, dst2, ref); in jit_ZVAL_COPY_2()
1705 jit_set_Z_TYPE_INFO_ex(jit, dst, type_ref); in jit_ZVAL_COPY_2()
1708 jit_set_Z_TYPE_INFO_ex(jit, dst2, type_ref); in jit_ZVAL_COPY_2()
1711 ir_ref type = jit_Z_TYPE_INFO(jit, src); in jit_ZVAL_COPY_2()
1712 jit_set_Z_TYPE_INFO_ex(jit, dst, type); in jit_ZVAL_COPY_2()
1713 jit_set_Z_TYPE_INFO_ex(jit, dst2, type); in jit_ZVAL_COPY_2()
1724 jit_GC_ADDREF2(jit, ref); in jit_ZVAL_COPY_2()
1726 jit_GC_ADDREF(jit, ref); in jit_ZVAL_COPY_2()
1737 static void jit_ZVAL_DTOR(zend_jit_ctx *jit, ir_ref ref, uint32_t op_info, const zend_op *opline) in jit_ZVAL_DTOR() argument
1748 jit_SET_EX_OPLINE(jit, opline); in jit_ZVAL_DTOR()
1757 jit_SET_EX_OPLINE(jit, opline); in jit_ZVAL_DTOR()
1764 jit_SET_EX_OPLINE(jit, opline); in jit_ZVAL_DTOR()
1769 static void jit_ZVAL_PTR_DTOR(zend_jit_ctx *jit, in jit_ZVAL_PTR_DTOR() argument
1782 if_refcounted = jit_if_REFCOUNTED(jit, addr); in jit_ZVAL_PTR_DTOR()
1787 ref = jit_Z_PTR(jit, addr); in jit_ZVAL_PTR_DTOR()
1788 ref2 = jit_GC_DELREF(jit, ref); in jit_ZVAL_PTR_DTOR()
1796 jit_ZVAL_DTOR(jit, ref, op_info, opline); in jit_ZVAL_PTR_DTOR()
1808 if_ref = jit_if_Z_TYPE(jit, addr, IS_REFERENCE); in jit_ZVAL_PTR_DTOR()
1813 if_collectable = jit_if_COLLECTABLE_ref(jit, ref2); in jit_ZVAL_PTR_DTOR()
1818 ref2 = jit_Z_PTR_ref(jit, ref2); in jit_ZVAL_PTR_DTOR()
1824 if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref); in jit_ZVAL_PTR_DTOR()
1830 jit_SET_EX_OPLINE(jit, opline); in jit_ZVAL_PTR_DTOR()
1842 static void jit_FREE_OP(zend_jit_ctx *jit, in jit_FREE_OP() argument
1849 jit_ZVAL_PTR_DTOR(jit, in jit_FREE_OP()
1855 static void jit_OBJ_RELEASE(zend_jit_ctx *jit, ir_ref ref) in jit_OBJ_RELEASE() argument
1861 if_not_zero = ir_IF(jit_GC_DELREF(jit, ref)); in jit_OBJ_RELEASE()
1869 if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref); in jit_OBJ_RELEASE()
1881 static void zend_jit_check_timeout(zend_jit_ctx *jit, const zend_op *opline, const void *exit_addr) in zend_jit_check_timeout() argument
1887 } else if (!opline || jit->last_valid_opline == opline) { in zend_jit_check_timeout()
1888 ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_interrupt_handler)); in zend_jit_check_timeout()
1893 jit_LOAD_IP_ADDR(jit, opline); in zend_jit_check_timeout()
1894 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_interrupt_handler)); in zend_jit_check_timeout()
1901 static int zend_jit_exception_handler_stub(zend_jit_ctx *jit) in zend_jit_exception_handler_stub() argument
1909 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); in zend_jit_exception_handler_stub()
1918 ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), jit_FP(jit)); in zend_jit_exception_handler_stub()
1929 static int zend_jit_exception_handler_undef_stub(zend_jit_ctx *jit) in zend_jit_exception_handler_undef_stub() argument
1943 …ir_STORE(ir_ADD_OFFSET(ir_ADD_A(jit_FP(jit), ref), offsetof(zval, u1.type_info)), ir_CONST_U32(IS_… in zend_jit_exception_handler_undef_stub()
1946 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_exception_handler_undef_stub()
1951 static int zend_jit_exception_handler_free_op1_op2_stub(zend_jit_ctx *jit) in zend_jit_exception_handler_free_op1_op2_stub() argument
1964 ref = ir_ADD_A(jit_FP(jit), ref); in zend_jit_exception_handler_free_op1_op2_stub()
1966 jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL); in zend_jit_exception_handler_free_op1_op2_stub()
1969 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); in zend_jit_exception_handler_free_op1_op2_stub()
1974 static int zend_jit_exception_handler_free_op2_stub(zend_jit_ctx *jit) in zend_jit_exception_handler_free_op2_stub() argument
1987 ref = ir_ADD_A(jit_FP(jit), ref); in zend_jit_exception_handler_free_op2_stub()
1989 jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL); in zend_jit_exception_handler_free_op2_stub()
1992 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef)); in zend_jit_exception_handler_free_op2_stub()
1997 static int zend_jit_interrupt_handler_stub(zend_jit_ctx *jit) in zend_jit_interrupt_handler_stub() argument
2003 ir_STORE(jit_EX(opline), jit_IP(jit)); in zend_jit_interrupt_handler_stub()
2013 ir_CALL_1(IR_VOID, ir_CONST_FUNC(zend_interrupt_function), jit_FP(jit)); in zend_jit_interrupt_handler_stub()
2019 jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data))); in zend_jit_interrupt_handler_stub()
2020 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline))); in zend_jit_interrupt_handler_stub()
2024 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); in zend_jit_interrupt_handler_stub()
2031 static int zend_jit_leave_function_handler_stub(zend_jit_ctx *jit) in zend_jit_leave_function_handler_stub() argument
2040 jit_STORE_IP(jit, in zend_jit_leave_function_handler_stub()
2042 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); in zend_jit_leave_function_handler_stub()
2046 …ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info, jit_FP(jit)); in zend_jit_leave_function_handler_stub()
2053 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); in zend_jit_leave_function_handler_stub()
2057 ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info, jit_FP(jit)); in zend_jit_leave_function_handler_stub()
2063 static int zend_jit_negative_shift_stub(zend_jit_ctx *jit) in zend_jit_negative_shift_stub() argument
2067 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), in zend_jit_negative_shift_stub()
2070 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2)); in zend_jit_negative_shift_stub()
2074 static int zend_jit_mod_by_zero_stub(zend_jit_ctx *jit) in zend_jit_mod_by_zero_stub() argument
2078 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), in zend_jit_mod_by_zero_stub()
2081 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2)); in zend_jit_mod_by_zero_stub()
2085 static int zend_jit_invalid_this_stub(zend_jit_ctx *jit) in zend_jit_invalid_this_stub() argument
2089 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), in zend_jit_invalid_this_stub()
2092 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef)); in zend_jit_invalid_this_stub()
2096 static int zend_jit_undefined_function_stub(zend_jit_ctx *jit) in zend_jit_undefined_function_stub() argument
2099 ir_ref ref = ir_LOAD_A(jit_FP(jit)); in zend_jit_undefined_function_stub()
2111 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), in zend_jit_undefined_function_stub()
2116 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_undefined_function_stub()
2121 static int zend_jit_throw_cannot_pass_by_ref_stub(zend_jit_ctx *jit) in zend_jit_throw_cannot_pass_by_ref_stub() argument
2126 opline = ir_LOAD_A(jit_FP(jit)); in zend_jit_throw_cannot_pass_by_ref_stub()
2133 rx = jit_IP(jit); in zend_jit_throw_cannot_pass_by_ref_stub()
2134 jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(rx, ref), ir_CONST_U32(IS_UNDEF)); in zend_jit_throw_cannot_pass_by_ref_stub()
2150 jit_STORE_IP(jit, opline); in zend_jit_throw_cannot_pass_by_ref_stub()
2157 ref = ir_LOAD_U8(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1_type))); in zend_jit_throw_cannot_pass_by_ref_stub()
2162 ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1.var))); in zend_jit_throw_cannot_pass_by_ref_stub()
2166 ref = ir_ADD_A(jit_FP(jit), ref); in zend_jit_throw_cannot_pass_by_ref_stub()
2167 jit_ZVAL_PTR_DTOR(jit, in zend_jit_throw_cannot_pass_by_ref_stub()
2172 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_throw_cannot_pass_by_ref_stub()
2177 static int zend_jit_icall_throw_stub(zend_jit_ctx *jit) in zend_jit_icall_throw_stub() argument
2183 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline))); in zend_jit_icall_throw_stub()
2184 ip = jit_IP(jit); in zend_jit_icall_throw_stub()
2194 jit_STORE_IP(jit, jit_EG(exception_op)); in zend_jit_icall_throw_stub()
2197 ir_STORE(jit_EX(opline), jit_IP(jit)); in zend_jit_icall_throw_stub()
2200 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_icall_throw_stub()
2205 static int zend_jit_leave_throw_stub(zend_jit_ctx *jit) in zend_jit_leave_throw_stub() argument
2210 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline))); in zend_jit_leave_throw_stub()
2211 ip = jit_IP(jit); in zend_jit_leave_throw_stub()
2221 jit_LOAD_IP(jit, jit_EG(exception_op)); in zend_jit_leave_throw_stub()
2224 ir_STORE(jit_EX(opline), jit_IP(jit)); in zend_jit_leave_throw_stub()
2227 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_leave_throw_stub()
2235 static int zend_jit_hybrid_runtime_jit_stub(zend_jit_ctx *jit) in zend_jit_hybrid_runtime_jit_stub() argument
2242 ir_IJMP(ir_LOAD_A(jit_IP(jit))); in zend_jit_hybrid_runtime_jit_stub()
2246 static int zend_jit_hybrid_profile_jit_stub(zend_jit_ctx *jit) in zend_jit_hybrid_profile_jit_stub() argument
2274 static int _zend_jit_hybrid_hot_counter_stub(zend_jit_ctx *jit, uint32_t cost) in _zend_jit_hybrid_hot_counter_stub() argument
2288 jit_FP(jit), in _zend_jit_hybrid_hot_counter_stub()
2289 jit_IP(jit)); in _zend_jit_hybrid_hot_counter_stub()
2290 ir_IJMP(ir_LOAD_A(jit_IP(jit))); in _zend_jit_hybrid_hot_counter_stub()
2293 ref = ir_SUB_A(jit_IP(jit), in _zend_jit_hybrid_hot_counter_stub()
2304 static int zend_jit_hybrid_func_hot_counter_stub(zend_jit_ctx *jit) in zend_jit_hybrid_func_hot_counter_stub() argument
2310 return _zend_jit_hybrid_hot_counter_stub(jit, in zend_jit_hybrid_func_hot_counter_stub()
2314 static int zend_jit_hybrid_loop_hot_counter_stub(zend_jit_ctx *jit) in zend_jit_hybrid_loop_hot_counter_stub() argument
2320 return _zend_jit_hybrid_hot_counter_stub(jit, in zend_jit_hybrid_loop_hot_counter_stub()
2324 static ir_ref _zend_jit_orig_opline_handler(zend_jit_ctx *jit, ir_ref offset) in _zend_jit_orig_opline_handler() argument
2329 addr = ir_ADD_A(offset, jit_IP(jit)); in _zend_jit_orig_opline_handler()
2337 static ir_ref zend_jit_orig_opline_handler(zend_jit_ctx *jit) in zend_jit_orig_opline_handler() argument
2344 return _zend_jit_orig_opline_handler(jit, offset); in zend_jit_orig_opline_handler()
2347 static int _zend_jit_hybrid_trace_counter_stub(zend_jit_ctx *jit, uint32_t cost) in _zend_jit_hybrid_trace_counter_stub() argument
2354 …addr = ir_LOAD_A(ir_ADD_OFFSET(ir_ADD_A(offset, jit_IP(jit)), offsetof(zend_op_trace_info, counter… in _zend_jit_hybrid_trace_counter_stub()
2362 jit_FP(jit), in _zend_jit_hybrid_trace_counter_stub()
2363 jit_IP(jit)); in _zend_jit_hybrid_trace_counter_stub()
2368 jit_STORE_FP(jit, ir_LOAD_A(ref)); in _zend_jit_hybrid_trace_counter_stub()
2370 jit_STORE_IP(jit, ref); in _zend_jit_hybrid_trace_counter_stub()
2371 ir_IJMP(ir_LOAD_A(jit_IP(jit))); in _zend_jit_hybrid_trace_counter_stub()
2374 ir_IJMP(_zend_jit_orig_opline_handler(jit, offset)); in _zend_jit_hybrid_trace_counter_stub()
2382 static int zend_jit_hybrid_func_trace_counter_stub(zend_jit_ctx *jit) in zend_jit_hybrid_func_trace_counter_stub() argument
2388 return _zend_jit_hybrid_trace_counter_stub(jit, in zend_jit_hybrid_func_trace_counter_stub()
2392 static int zend_jit_hybrid_ret_trace_counter_stub(zend_jit_ctx *jit) in zend_jit_hybrid_ret_trace_counter_stub() argument
2398 return _zend_jit_hybrid_trace_counter_stub(jit, in zend_jit_hybrid_ret_trace_counter_stub()
2402 static int zend_jit_hybrid_loop_trace_counter_stub(zend_jit_ctx *jit) in zend_jit_hybrid_loop_trace_counter_stub() argument
2408 return _zend_jit_hybrid_trace_counter_stub(jit, in zend_jit_hybrid_loop_trace_counter_stub()
2412 static int zend_jit_trace_halt_stub(zend_jit_ctx *jit) in zend_jit_trace_halt_stub() argument
2417 jit_STORE_IP(jit, IR_NULL); in zend_jit_trace_halt_stub()
2425 static int zend_jit_trace_escape_stub(zend_jit_ctx *jit) in zend_jit_trace_escape_stub() argument
2428 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); in zend_jit_trace_escape_stub()
2436 static int zend_jit_trace_exit_stub(zend_jit_ctx *jit) in zend_jit_trace_exit_stub() argument
2442 ir_STORE(jit_EX(opline), jit_IP(jit)); in zend_jit_trace_exit_stub()
2453 jit_STORE_FP(jit, ir_LOAD_A(ref)); in zend_jit_trace_exit_stub()
2455 jit_STORE_IP(jit, ref); in zend_jit_trace_exit_stub()
2456 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); in zend_jit_trace_exit_stub()
2463 ir_GUARD(ir_GE(ret, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt)); in zend_jit_trace_exit_stub()
2466 jit_STORE_FP(jit, ir_LOAD_A(ref)); in zend_jit_trace_exit_stub()
2470 jit_STORE_IP(jit, ref); in zend_jit_trace_exit_stub()
2474 zend_jit_check_timeout(jit, NULL, NULL); in zend_jit_trace_exit_stub()
2476 addr = zend_jit_orig_opline_handler(jit); in zend_jit_trace_exit_stub()
2483 ref = ir_CALL_1(IR_I32, addr, jit_FP(jit)); in zend_jit_trace_exit_stub()
2484 ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt)); in zend_jit_trace_exit_stub()
2491 static int zend_jit_undefined_offset_stub(zend_jit_ctx *jit) in zend_jit_undefined_offset_stub() argument
2496 ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key), jit_FP(jit)); in zend_jit_undefined_offset_stub()
2502 static int zend_jit_undefined_key_stub(zend_jit_ctx *jit) in zend_jit_undefined_key_stub() argument
2507 ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key), jit_FP(jit)); in zend_jit_undefined_key_stub()
2513 static int zend_jit_cannot_add_element_stub(zend_jit_ctx *jit) in zend_jit_cannot_add_element_stub() argument
2527 jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(jit_FP(jit), ref), ir_CONST_U32(IS_UNDEF)); in zend_jit_cannot_add_element_stub()
2532 ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), in zend_jit_cannot_add_element_stub()
2540 static int zend_jit_assign_const_stub(zend_jit_ctx *jit) in zend_jit_assign_const_stub() argument
2550 jit, NULL, in zend_jit_assign_const_stub()
2560 static int zend_jit_assign_tmp_stub(zend_jit_ctx *jit) in zend_jit_assign_tmp_stub() argument
2570 jit, NULL, in zend_jit_assign_tmp_stub()
2580 static int zend_jit_assign_var_stub(zend_jit_ctx *jit) in zend_jit_assign_var_stub() argument
2590 jit, NULL, in zend_jit_assign_var_stub()
2600 static int zend_jit_assign_cv_noref_stub(zend_jit_ctx *jit) in zend_jit_assign_cv_noref_stub() argument
2610 jit, NULL, in zend_jit_assign_cv_noref_stub()
2620 static int zend_jit_new_array_stub(zend_jit_ctx *jit) in zend_jit_new_array_stub() argument
2626 jit_set_Z_PTR(jit, var_addr, ref); in zend_jit_new_array_stub()
2627 jit_set_Z_TYPE_INFO(jit, var_addr, IS_ARRAY_EX); in zend_jit_new_array_stub()
2632 static int zend_jit_assign_cv_stub(zend_jit_ctx *jit) in zend_jit_assign_cv_stub() argument
2642 jit, NULL, in zend_jit_assign_cv_stub()
2652 static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags) in zend_jit_init_ctx() argument
2659 ir_init(&jit->ctx, flags, 256, 1024); in zend_jit_init_ctx()
2660 jit->ctx.ret_type = -1; in zend_jit_init_ctx()
2663 jit->ctx.mflags |= default_mflags; in zend_jit_init_ctx()
2665 jit->ctx.mflags |= IR_X86_AVX; in zend_jit_init_ctx()
2668 jit->ctx.get_veneer = zend_jit_get_veneer; in zend_jit_init_ctx()
2669 jit->ctx.set_veneer = zend_jit_set_veneer; in zend_jit_init_ctx()
2672 jit->ctx.fixed_regset = (1<<ZREG_FP) | (1<<ZREG_IP); in zend_jit_init_ctx()
2674 jit->ctx.flags |= IR_NO_STACK_COMBINE; in zend_jit_init_ctx()
2676 jit->ctx.flags |= IR_FUNCTION; in zend_jit_init_ctx()
2680 jit->ctx.flags |= IR_USE_FRAME_POINTER; in zend_jit_init_ctx()
2681 …jit->ctx.fixed_stack_frame_size = sizeof(void*) * 16; /* 10 saved registers and 6 spill slots (8 b… in zend_jit_init_ctx()
2683 …jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 8 saved registers and 3 spill slots (8 by… in zend_jit_init_ctx()
2685 …jit->ctx.fixed_stack_frame_size = sizeof(void*) * 9; /* 6 saved registers and 3 spill slots (8 by… in zend_jit_init_ctx()
2687 …jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 4 saved registers and 7 spill slots (4 by… in zend_jit_init_ctx()
2690 jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED & ~((1<<ZREG_FP) | (1<<ZREG_IP)); in zend_jit_init_ctx()
2692 jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED; in zend_jit_init_ctx()
2697 jit->ctx.fixed_call_stack_size = 16; in zend_jit_init_ctx()
2700 jit->ctx.fixed_stack_red_zone = ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE; in zend_jit_init_ctx()
2701 if (jit->ctx.fixed_stack_red_zone > 16) { in zend_jit_init_ctx()
2702 jit->ctx.fixed_stack_frame_size = jit->ctx.fixed_stack_red_zone - 16; in zend_jit_init_ctx()
2703 jit->ctx.fixed_call_stack_size = 16; in zend_jit_init_ctx()
2705 jit->ctx.flags |= IR_MERGE_EMPTY_ENTRIES; in zend_jit_init_ctx()
2707 jit->ctx.fixed_stack_red_zone = 0; in zend_jit_init_ctx()
2708 jit->ctx.fixed_stack_frame_size = 32; /* 4 spill slots (8 bytes) or 8 spill slots (4 bytes) */ in zend_jit_init_ctx()
2709 jit->ctx.fixed_call_stack_size = 16; in zend_jit_init_ctx()
2712 jit->ctx.fixed_regset |= (1<<IR_REG_FP); /* prevent %rbp (%r5) usage */ in zend_jit_init_ctx()
2717 jit->ctx.snapshot_create = (ir_snapshot_create_t)jit_SNAPSHOT; in zend_jit_init_ctx()
2719 jit->op_array = NULL; in zend_jit_init_ctx()
2720 jit->current_op_array = NULL; in zend_jit_init_ctx()
2721 jit->ssa = NULL; in zend_jit_init_ctx()
2722 jit->name = NULL; in zend_jit_init_ctx()
2723 jit->last_valid_opline = NULL; in zend_jit_init_ctx()
2724 jit->use_last_valid_opline = 0; in zend_jit_init_ctx()
2725 jit->track_last_valid_opline = 0; in zend_jit_init_ctx()
2726 jit->reuse_ip = 0; in zend_jit_init_ctx()
2727 jit->delayed_call_level = 0; in zend_jit_init_ctx()
2729 jit->b = -1; in zend_jit_init_ctx()
2731 jit->tls = IR_UNUSED; in zend_jit_init_ctx()
2733 jit->fp = IR_UNUSED; in zend_jit_init_ctx()
2734 jit->trace_loop_ref = IR_UNUSED; in zend_jit_init_ctx()
2735 jit->return_inputs = IR_UNUSED; in zend_jit_init_ctx()
2736 jit->bb_start_ref = NULL; in zend_jit_init_ctx()
2737 jit->bb_predecessors = NULL; in zend_jit_init_ctx()
2738 jit->bb_edges = NULL; in zend_jit_init_ctx()
2739 jit->trace = NULL; in zend_jit_init_ctx()
2740 jit->ra = NULL; in zend_jit_init_ctx()
2741 jit->delay_var = -1; in zend_jit_init_ctx()
2742 jit->delay_refs = NULL; in zend_jit_init_ctx()
2743 jit->eg_exception_addr = 0; in zend_jit_init_ctx()
2744 zend_hash_init(&jit->addr_hash, 64, NULL, NULL, 0); in zend_jit_init_ctx()
2745 memset(jit->stub_addr, 0, sizeof(jit->stub_addr)); in zend_jit_init_ctx()
2750 static int zend_jit_free_ctx(zend_jit_ctx *jit) in zend_jit_free_ctx() argument
2752 if (jit->name) { in zend_jit_free_ctx()
2753 zend_string_release(jit->name); in zend_jit_free_ctx()
2755 zend_hash_destroy(&jit->addr_hash); in zend_jit_free_ctx()
2756 ir_free(&jit->ctx); in zend_jit_free_ctx()
2863 zend_jit_ctx jit; in zend_jit_setup_stubs() local
2869 zend_jit_init_ctx(&jit, zend_jit_stubs[i].flags); in zend_jit_setup_stubs()
2871 if (!zend_jit_stubs[i].stub(&jit)) { in zend_jit_setup_stubs()
2872 zend_jit_free_ctx(&jit); in zend_jit_setup_stubs()
2877 entry = zend_jit_ir_compile(&jit.ctx, &size, zend_jit_stubs[i].name); in zend_jit_setup_stubs()
2879 zend_jit_free_ctx(&jit); in zend_jit_setup_stubs()
2892 entry, size, (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0, &jit.ctx, stderr); in zend_jit_setup_stubs()
2910 zend_jit_free_ctx(&jit); in zend_jit_setup_stubs()
3128 zend_jit_ctx *jit = &jit_ctx; in zend_jit_calc_trace_prologue_size() local
3132 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET); in zend_jit_calc_trace_prologue_size()
3136 jit_STORE_FP(jit, ref); in zend_jit_calc_trace_prologue_size()
3137 jit->ctx.flags |= IR_FASTCALL_FUNC; in zend_jit_calc_trace_prologue_size()
3142 entry = zend_jit_ir_compile(&jit->ctx, &size, "JIT$trace_prologue"); in zend_jit_calc_trace_prologue_size()
3143 zend_jit_free_ctx(jit); in zend_jit_calc_trace_prologue_size()
3413 static ir_ref jit_IF_ex(zend_jit_ctx *jit, ir_ref condition, ir_ref true_block) in jit_IF_ex() argument
3421 ir_set_op(&jit->ctx, ref, 3, true_block); in jit_IF_ex()
3425 static void jit_IF_TRUE_FALSE_ex(zend_jit_ctx *jit, ir_ref if_ref, ir_ref true_block) in jit_IF_TRUE_FALSE_ex() argument
3429 ZEND_ASSERT(jit->ctx.ir_base[if_ref].op == IR_IF); in jit_IF_TRUE_FALSE_ex()
3430 ZEND_ASSERT(jit->ctx.ir_base[if_ref].op3); in jit_IF_TRUE_FALSE_ex()
3431 if (jit->ctx.ir_base[if_ref].op3 == true_block) { in jit_IF_TRUE_FALSE_ex()
3438 static void _zend_jit_add_predecessor_ref(zend_jit_ctx *jit, int b, int pred, ir_ref ref) in _zend_jit_add_predecessor_ref() argument
3445 bb = &jit->ssa->cfg.blocks[b]; in _zend_jit_add_predecessor_ref()
3446 p = &jit->ssa->cfg.predecessors[bb->predecessor_offset]; in _zend_jit_add_predecessor_ref()
3447 r = &jit->bb_edges[jit->bb_predecessors[b]]; in _zend_jit_add_predecessor_ref()
3451 header = jit->bb_start_ref[b]; in _zend_jit_add_predecessor_ref()
3454 ZEND_ASSERT(jit->ctx.ir_base[header].op == IR_LOOP_BEGIN); in _zend_jit_add_predecessor_ref()
3455 if (jit->ctx.ir_base[ref].op == IR_END) { in _zend_jit_add_predecessor_ref()
3456 jit->ctx.ir_base[ref].op = IR_LOOP_END; in _zend_jit_add_predecessor_ref()
3457 } else if (jit->ctx.ir_base[ref].op == IR_IF) { in _zend_jit_add_predecessor_ref()
3458 jit_IF_TRUE_FALSE_ex(jit, ref, b); in _zend_jit_add_predecessor_ref()
3460 } else if (jit->ctx.ir_base[ref].op == IR_UNREACHABLE) { in _zend_jit_add_predecessor_ref()
3475 static void _zend_jit_merge_smart_branch_inputs(zend_jit_ctx *jit, in _zend_jit_merge_smart_branch_inputs() argument
3485 ZEND_ASSERT(jit->ctx.ir_base[true_inputs].op == IR_END); in _zend_jit_merge_smart_branch_inputs()
3486 if (!jit->ctx.ir_base[true_inputs].op2) { in _zend_jit_merge_smart_branch_inputs()
3494 ZEND_ASSERT(jit->ctx.ir_base[false_inputs].op == IR_END); in _zend_jit_merge_smart_branch_inputs()
3495 if (!jit->ctx.ir_base[false_inputs].op2) { in _zend_jit_merge_smart_branch_inputs()
3505 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END()); in _zend_jit_merge_smart_branch_inputs()
3509 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path); in _zend_jit_merge_smart_branch_inputs()
3512 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path); in _zend_jit_merge_smart_branch_inputs()
3515 _zend_jit_add_predecessor_ref(jit, false_label, jit->b, false_path); in _zend_jit_merge_smart_branch_inputs()
3519 jit->b = -1; in _zend_jit_merge_smart_branch_inputs()
3522 static void _zend_jit_fix_merges(zend_jit_ctx *jit) in _zend_jit_fix_merges() argument
3530 count = jit->ssa->cfg.blocks_count; in _zend_jit_fix_merges()
3531 for (i = 0, p = jit->bb_start_ref; i < count; i++, p++) { in _zend_jit_fix_merges()
3534 insn = &jit->ctx.ir_base[ref]; in _zend_jit_fix_merges()
3597 static void zend_jit_case_start(zend_jit_ctx *jit, int switch_b, int case_b, ir_ref switch_ref) in zend_jit_case_start() argument
3599 zend_basic_block *bb = &jit->ssa->cfg.blocks[switch_b]; in zend_jit_case_start()
3600 const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1]; in zend_jit_case_start()
3607 int default_b = jit->ssa->cfg.map[default_opline - jit->op_array->opcodes]; in zend_jit_case_start()
3614 int b = jit->ssa->cfg.map[target - jit->op_array->opcodes]; in zend_jit_case_start()
3633 if (jit->ctx.ir_base[switch_ref].op3) { in zend_jit_case_start()
3635 ir_ref ref = jit->ctx.ir_base[switch_ref].op3; in zend_jit_case_start()
3636 jit->ctx.ir_base[switch_ref].op3 = IS_UNDEF; in zend_jit_case_start()
3637 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END); in zend_jit_case_start()
3639 while (jit->ctx.ir_base[end].op2) { in zend_jit_case_start()
3640 ZEND_ASSERT(jit->ctx.ir_base[end].op == IR_END); in zend_jit_case_start()
3641 end = jit->ctx.ir_base[end].op2; in zend_jit_case_start()
3643 jit->ctx.ir_base[end].op2 = list; in zend_jit_case_start()
3657 static int zend_jit_bb_start(zend_jit_ctx *jit, int b) in zend_jit_bb_start() argument
3664 ZEND_ASSERT(b < jit->ssa->cfg.blocks_count); in zend_jit_bb_start()
3665 bb = &jit->ssa->cfg.blocks[b]; in zend_jit_bb_start()
3671 ZEND_ASSERT(jit->ctx.control); in zend_jit_bb_start()
3673 ref = jit->ctx.control; in zend_jit_bb_start()
3674 ir_insn *insn = &jit->ctx.ir_base[ref]; in zend_jit_bb_start()
3677 insn = &jit->ctx.ir_base[ref]; in zend_jit_bb_start()
3683 if (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY) { in zend_jit_bb_start()
3685 jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ir_END()); in zend_jit_bb_start()
3686 bb_start = jit->ctx.control; in zend_jit_bb_start()
3689 ZEND_ASSERT(!jit->ctx.control); in zend_jit_bb_start()
3690 pred = jit->ssa->cfg.predecessors[bb->predecessor_offset]; in zend_jit_bb_start()
3691 ref = jit->bb_edges[jit->bb_predecessors[b]]; in zend_jit_bb_start()
3693 if (!jit->ctx.control) { in zend_jit_bb_start()
3697 ir_op op = jit->ctx.ir_base[ref].op; in zend_jit_bb_start()
3700 if (!jit->ctx.control) { in zend_jit_bb_start()
3701 jit_IF_TRUE_FALSE_ex(jit, ref, b); in zend_jit_bb_start()
3704 jit_IF_TRUE_FALSE_ex(jit, ref, b); in zend_jit_bb_start()
3708 zend_jit_case_start(jit, pred, b, ref); in zend_jit_bb_start()
3710 if (!jit->ctx.control) { in zend_jit_bb_start()
3712 if ((jit->ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY) in zend_jit_bb_start()
3713 && (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)) { in zend_jit_bb_start()
3715 jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ref); in zend_jit_bb_start()
3724 bb_start = jit->ctx.control; in zend_jit_bb_start()
3732 ZEND_ASSERT(!jit->ctx.control); in zend_jit_bb_start()
3733 if (jit->ctx.control) { in zend_jit_bb_start()
3737 for (i = 0, p = jit->ssa->cfg.predecessors + bb->predecessor_offset; i < n; p++, i++) { in zend_jit_bb_start()
3739 if (jit->bb_start_ref[pred]) { in zend_jit_bb_start()
3742 ref = jit->bb_edges[jit->bb_predecessors[b] + i]; in zend_jit_bb_start()
3747 ir_op op = jit->ctx.ir_base[ref].op; in zend_jit_bb_start()
3750 jit_IF_TRUE_FALSE_ex(jit, ref, b); in zend_jit_bb_start()
3753 zend_jit_case_start(jit, pred, b, ref); in zend_jit_bb_start()
3771 jit->ctx.ir_base[jit->ctx.control].op = IR_LOOP_BEGIN; in zend_jit_bb_start()
3772 bb_start = jit->ctx.control; in zend_jit_bb_start()
3780 bb_start = jit->ctx.control; in zend_jit_bb_start()
3787 jit->b = b; in zend_jit_bb_start()
3788 jit->bb_start_ref[b] = bb_start; in zend_jit_bb_start()
3790 …if ((bb->flags & ZEND_BB_ENTRY) || (bb->idom >= 0 && jit->bb_start_ref[bb->idom] < jit->ctx.fold_c… in zend_jit_bb_start()
3791 jit->ctx.fold_cse_limit = bb_start; in zend_jit_bb_start()
3797 static int zend_jit_bb_end(zend_jit_ctx *jit, int b) in zend_jit_bb_end() argument
3803 if (jit->b != b) { in zend_jit_bb_end()
3807 bb = &jit->ssa->cfg.blocks[b]; in zend_jit_bb_end()
3812 const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1]; in zend_jit_bb_end()
3821 _zend_jit_add_predecessor_ref(jit, succ, b, ir_END()); in zend_jit_bb_end()
3822 jit->b = -1; in zend_jit_bb_end()
3826 static int jit_CMP_IP(zend_jit_ctx *jit, ir_op op, const zend_op *next_opline) in jit_CMP_IP() argument
3832 ref = jit_IP32(jit); in jit_CMP_IP()
3839 ref = jit_IP(jit); in jit_CMP_IP()
3849 zend_jit_ctx *jit, in zend_jit_jmp_frameless() argument
3875 ZEND_ASSERT(jit->b >= 0); in zend_jit_jmp_frameless()
3876 bb = &jit->ssa->cfg.blocks[jit->b]; in zend_jit_jmp_frameless()
3878 ref = jit_IF_ex(jit, ir_EQ(phi_result, ir_CONST_LONG(ZEND_JMP_FL_HIT)), bb->successors[0]); in zend_jit_jmp_frameless()
3879 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); in zend_jit_jmp_frameless()
3880 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref); in zend_jit_jmp_frameless()
3881 jit->b = -1; in zend_jit_jmp_frameless()
3887 static int zend_jit_cond_jmp(zend_jit_ctx *jit, const zend_op *next_opline, int target_block) in zend_jit_cond_jmp() argument
3892 ZEND_ASSERT(jit->b >= 0); in zend_jit_cond_jmp()
3893 bb = &jit->ssa->cfg.blocks[jit->b]; in zend_jit_cond_jmp()
3897 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END()); in zend_jit_cond_jmp()
3898 jit->b = -1; in zend_jit_cond_jmp()
3899 zend_jit_set_last_valid_opline(jit, next_opline); in zend_jit_cond_jmp()
3903 ref = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, next_opline), target_block); in zend_jit_cond_jmp()
3905 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); in zend_jit_cond_jmp()
3906 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref); in zend_jit_cond_jmp()
3908 jit->b = -1; in zend_jit_cond_jmp()
3909 zend_jit_set_last_valid_opline(jit, next_opline); in zend_jit_cond_jmp()
3914 static int zend_jit_set_cond(zend_jit_ctx *jit, const zend_op *next_opline, uint32_t var) in zend_jit_set_cond() argument
3918 ref = ir_ADD_U32(ir_ZEXT_U32(jit_CMP_IP(jit, IR_EQ, next_opline)), ir_CONST_U32(IS_FALSE)); in zend_jit_set_cond()
3921 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), var + offsetof(zval, u1.type_info)), ref); in zend_jit_set_cond()
3923 zend_jit_reset_last_valid_opline(jit); in zend_jit_set_cond()
3924 return zend_jit_set_ip(jit, next_opline - 1); in zend_jit_set_cond()
3928 static void zend_jit_check_exception(zend_jit_ctx *jit) in zend_jit_check_exception() argument
3930 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), in zend_jit_check_exception()
3931 jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_check_exception()
3934 static void zend_jit_check_exception_undef_result(zend_jit_ctx *jit, const zend_op *opline) in zend_jit_check_exception_undef_result() argument
3936 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), in zend_jit_check_exception_undef_result()
3937 jit_STUB_ADDR(jit, in zend_jit_check_exception_undef_result()
3941 static void zend_jit_type_check_undef(zend_jit_ctx *jit, in zend_jit_type_check_undef() argument
3957 jit_SET_EX_OPLINE(jit, opline); in zend_jit_type_check_undef()
3962 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_type_check_undef()
3964 zend_jit_check_exception(jit); in zend_jit_type_check_undef()
3970 static ir_ref zend_jit_zval_check_undef(zend_jit_ctx *jit, in zend_jit_zval_check_undef() argument
3978 if_def = ir_IF(jit_Z_TYPE_ref(jit, ref)); in zend_jit_zval_check_undef()
3982 jit_SET_EX_OPLINE(jit, opline); in zend_jit_zval_check_undef()
3988 zend_jit_check_exception(jit); in zend_jit_zval_check_undef()
3998 static void zend_jit_recv_entry(zend_jit_ctx *jit, int b) in zend_jit_recv_entry() argument
4000 zend_basic_block *bb = &jit->ssa->cfg.blocks[b]; in zend_jit_recv_entry()
4006 pred = jit->bb_predecessors[b]; in zend_jit_recv_entry()
4007 ref = jit->bb_edges[pred]; in zend_jit_recv_entry()
4010 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END); in zend_jit_recv_entry()
4016 ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM); in zend_jit_recv_entry()
4017 ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1); in zend_jit_recv_entry()
4018 jit_STORE_FP(jit, 2); in zend_jit_recv_entry()
4022 jit->bb_edges[pred] = ir_END(); in zend_jit_recv_entry()
4025 static void zend_jit_osr_entry(zend_jit_ctx *jit, int b) in zend_jit_osr_entry() argument
4027 zend_basic_block *bb = &jit->ssa->cfg.blocks[b]; in zend_jit_osr_entry()
4034 ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM); in zend_jit_osr_entry()
4035 ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1); in zend_jit_osr_entry()
4036 jit_STORE_FP(jit, 2); in zend_jit_osr_entry()
4042 static ir_ref zend_jit_continue_entry(zend_jit_ctx *jit, ir_ref src, unsigned int label) in zend_jit_continue_entry() argument
4047 ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM); in zend_jit_continue_entry()
4048 ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1); in zend_jit_continue_entry()
4049 jit_STORE_FP(jit, 2); in zend_jit_continue_entry()
4054 static int zend_jit_handler(zend_jit_ctx *jit, const zend_op *opline, int may_throw) in zend_jit_handler() argument
4059 zend_jit_set_ip(jit, opline); in zend_jit_handler()
4068 ref = jit_FP(jit); in zend_jit_handler()
4072 zend_jit_check_exception(jit); in zend_jit_handler()
4084 zend_jit_set_last_valid_opline(jit, opline + 2); in zend_jit_handler()
4087 zend_jit_set_last_valid_opline(jit, opline + 1); in zend_jit_handler()
4093 static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline) in zend_jit_tail_handler() argument
4099 zend_jit_set_ip(jit, opline); in zend_jit_tail_handler()
4112 ref = ir_LOAD_A(jit_IP(jit)); in zend_jit_tail_handler()
4119 } else if ((jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY) in zend_jit_tail_handler()
4126 ref = jit_FP(jit); in zend_jit_tail_handler()
4130 ref = jit_FP(jit); in zend_jit_tail_handler()
4134 if (jit->b >= 0) { in zend_jit_tail_handler()
4135 bb = &jit->ssa->cfg.blocks[jit->b]; in zend_jit_tail_handler()
4150 ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY); in zend_jit_tail_handler()
4154 succ = jit->b + 1; in zend_jit_tail_handler()
4156 jit->ssa->cfg.blocks[succ].flags |= ZEND_BB_ENTRY; in zend_jit_tail_handler()
4158 ref = jit->ctx.insns_count - 1; in zend_jit_tail_handler()
4159 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE || jit->ctx.ir_base[ref].op == IR_RETURN); in zend_jit_tail_handler()
4160 ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start); in zend_jit_tail_handler()
4161 _zend_jit_add_predecessor_ref(jit, succ, jit->b, ref); in zend_jit_tail_handler()
4163 jit->b = -1; in zend_jit_tail_handler()
4164 zend_jit_reset_last_valid_opline(jit); in zend_jit_tail_handler()
4169 static int zend_jit_call(zend_jit_ctx *jit, const zend_op *opline, unsigned int next_block) in zend_jit_call() argument
4171 return zend_jit_tail_handler(jit, opline); in zend_jit_call()
4174 static int zend_jit_spill_store(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t i… in zend_jit_spill_store() argument
4180 jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src)); in zend_jit_spill_store()
4185 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_spill_store()
4188 jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src)); in zend_jit_spill_store()
4193 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_spill_store()
4201 static int zend_jit_spill_store_inv(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32… in zend_jit_spill_store_inv() argument
4207 jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src)); in zend_jit_spill_store_inv()
4209 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_spill_store_inv()
4213 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_spill_store_inv()
4216 jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src)); in zend_jit_spill_store_inv()
4218 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_spill_store_inv()
4222 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_spill_store_inv()
4230 static int zend_jit_load_reg(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info) in zend_jit_load_reg() argument
4236 zend_jit_def_reg(jit, dst, jit_Z_LVAL(jit, src)); in zend_jit_load_reg()
4238 zend_jit_def_reg(jit, dst, jit_Z_DVAL(jit, src)); in zend_jit_load_reg()
4245 static int zend_jit_store_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var, bool set_type) in zend_jit_store_var() argument
4250 return zend_jit_spill_store(jit, src, dst, info, set_type); in zend_jit_store_var()
4253 static int zend_jit_store_ref(zend_jit_ctx *jit, uint32_t info, int var, int32_t src, bool set_type) in zend_jit_store_ref() argument
4258 jit_set_Z_LVAL(jit, dst, src); in zend_jit_store_ref()
4263 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_store_ref()
4266 jit_set_Z_DVAL(jit, dst, src); in zend_jit_store_ref()
4271 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_store_ref()
4279 static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t reg) in zend_jit_deopt_rload() argument
4281 ir_ref ref = jit->ctx.control; in zend_jit_deopt_rload()
4285 insn = &jit->ctx.ir_base[ref]; in zend_jit_deopt_rload()
4297 static int zend_jit_store_const_long(zend_jit_ctx *jit, int var, zend_long val) in zend_jit_store_const_long() argument
4302 if (jit->ra && jit->ra[var].ref == IR_NULL) { in zend_jit_store_const_long()
4303 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); in zend_jit_store_const_long()
4305 jit_set_Z_LVAL(jit, dst, src); in zend_jit_store_const_long()
4306 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_store_const_long()
4310 static int zend_jit_store_const_double(zend_jit_ctx *jit, int var, double val) in zend_jit_store_const_double() argument
4315 if (jit->ra && jit->ra[var].ref == IR_NULL) { in zend_jit_store_const_double()
4316 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); in zend_jit_store_const_double()
4318 jit_set_Z_DVAL(jit, dst, src); in zend_jit_store_const_double()
4319 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_store_const_double()
4323 static int zend_jit_store_type(zend_jit_ctx *jit, int var, uint8_t type) in zend_jit_store_type() argument
4328 jit_set_Z_TYPE_INFO(jit, dst, type); in zend_jit_store_type()
4332 static int zend_jit_store_reg(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, bool in_mem, b… in zend_jit_store_reg() argument
4340 src = zend_jit_deopt_rload(jit, type, reg); in zend_jit_store_reg()
4341 if (jit->ra && jit->ra[var].ref == IR_NULL) { in zend_jit_store_reg()
4342 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); in zend_jit_store_reg()
4344 jit_set_Z_LVAL(jit, dst, src); in zend_jit_store_reg()
4349 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_store_reg()
4354 src = zend_jit_deopt_rload(jit, type, reg); in zend_jit_store_reg()
4355 if (jit->ra && jit->ra[var].ref == IR_NULL) { in zend_jit_store_reg()
4356 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); in zend_jit_store_reg()
4358 jit_set_Z_DVAL(jit, dst, src); in zend_jit_store_reg()
4363 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_store_reg()
4372 static int zend_jit_store_spill_slot(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, int32_t… in zend_jit_store_spill_slot() argument
4379 if (jit->ra && jit->ra[var].ref == IR_NULL) { in zend_jit_store_spill_slot()
4380 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); in zend_jit_store_spill_slot()
4382 jit_set_Z_LVAL(jit, dst, src); in zend_jit_store_spill_slot()
4387 jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); in zend_jit_store_spill_slot()
4392 if (jit->ra && jit->ra[var].ref == IR_NULL) { in zend_jit_store_spill_slot()
4393 zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); in zend_jit_store_spill_slot()
4395 jit_set_Z_DVAL(jit, dst, src); in zend_jit_store_spill_slot()
4400 jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); in zend_jit_store_spill_slot()
4409 static int zend_jit_store_var_type(zend_jit_ctx *jit, int var, uint32_t type) in zend_jit_store_var_type() argument
4413 jit_set_Z_TYPE_INFO(jit, dst, type); in zend_jit_store_var_type()
4417 static int zend_jit_zval_try_addref(zend_jit_ctx *jit, zend_jit_addr var_addr) in zend_jit_zval_try_addref() argument
4421 if_refcounted = jit_if_REFCOUNTED(jit, var_addr); in zend_jit_zval_try_addref()
4425 jit_GC_ADDREF(jit, jit_Z_PTR(jit, var_addr)); in zend_jit_zval_try_addref()
4430 static int zend_jit_store_var_if_necessary(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t … in zend_jit_store_var_if_necessary() argument
4434 return zend_jit_spill_store(jit, src, dst, info, 1); in zend_jit_store_var_if_necessary()
4439 static int zend_jit_store_var_if_necessary_ex(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32… in zend_jit_store_var_if_necessary_ex() argument
4460 return zend_jit_spill_store(jit, src, dst, info, set_type); in zend_jit_store_var_if_necessary_ex()
4465 static int zend_jit_load_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var) in zend_jit_load_var() argument
4470 return zend_jit_load_reg(jit, src, dst, info); in zend_jit_load_var()
4473 static int zend_jit_invalidate_var_if_necessary(zend_jit_ctx *jit, uint8_t op_type, zend_jit_addr a… in zend_jit_invalidate_var_if_necessary() argument
4478 jit_set_Z_TYPE_INFO(jit, dst, IS_UNDEF); in zend_jit_invalidate_var_if_necessary()
4483 static int zend_jit_update_regs(zend_jit_ctx *jit, uint32_t var, zend_jit_addr src, zend_jit_addr d… in zend_jit_update_regs() argument
4488 zend_jit_def_reg(jit, dst, zend_jit_use_reg(jit, src)); in zend_jit_update_regs()
4492 if (!zend_jit_spill_store(jit, dst, var_addr, info, in zend_jit_update_regs()
4503 if (!zend_jit_spill_store(jit, src, dst, info, in zend_jit_update_regs()
4517 if (!zend_jit_load_reg(jit, src, dst, info)) { in zend_jit_update_regs()
4528 if (!zend_jit_spill_store(jit, src, dst, info, in zend_jit_update_regs()
4545 …ll_is_unobserved_data jit_observer_fcall_is_unobserved_start(zend_jit_ctx *jit, const zend_functio… in jit_observer_fcall_is_unobserved_start() argument
4620 static void jit_observer_fcall_is_unobserved_end(zend_jit_ctx *jit, struct jit_observer_fcall_is_un… in jit_observer_fcall_is_unobserved_end() argument
4627 static void jit_observer_fcall_begin(zend_jit_ctx *jit, ir_ref rx, ir_ref observer_handler) { in jit_observer_fcall_begin() argument
4631 static void jit_observer_fcall_end(zend_jit_ctx *jit, ir_ref rx, ir_ref res_ref) { in jit_observer_fcall_end() argument
4640 static int zend_jit_inc_dec(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_a… in zend_jit_inc_dec() argument
4648 if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_inc_dec()
4652 op1_lval_ref = jit_Z_LVAL(jit, op1_addr); in zend_jit_inc_dec()
4653 jit_set_Z_LVAL(jit, res_addr, op1_lval_ref); in zend_jit_inc_dec()
4655 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_inc_dec()
4662 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_LONG); in zend_jit_inc_dec()
4670 op1_lval_ref = jit_Z_LVAL(jit, op1_addr); in zend_jit_inc_dec()
4674 jit_set_Z_LVAL(jit, op1_def_addr, ref); in zend_jit_inc_dec()
4715 jit_set_Z_LVAL(jit, res_addr, ref); in zend_jit_inc_dec()
4717 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_inc_dec()
4749 jit_set_Z_LVAL(jit, res_addr, ref); in zend_jit_inc_dec()
4751 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_inc_dec()
4753 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_inc_dec()
4766 jit_set_Z_LVAL(jit, res_addr, ref); in zend_jit_inc_dec()
4768 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_inc_dec()
4779 jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); in zend_jit_inc_dec()
4782 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0)); in zend_jit_inc_dec()
4783 jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0x41e00000)); in zend_jit_inc_dec()
4785 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x43e0000000000000)); in zend_jit_inc_dec()
4787 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE); in zend_jit_inc_dec()
4791 jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); in zend_jit_inc_dec()
4794 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x00200000)); in zend_jit_inc_dec()
4795 jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0xc1e00000)); in zend_jit_inc_dec()
4797 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0xc3e0000000000000)); in zend_jit_inc_dec()
4799 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE); in zend_jit_inc_dec()
4806 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); in zend_jit_inc_dec()
4809 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0)); in zend_jit_inc_dec()
4810 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000)); in zend_jit_inc_dec()
4812 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000)); in zend_jit_inc_dec()
4814 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_inc_dec()
4818 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); in zend_jit_inc_dec()
4821 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000)); in zend_jit_inc_dec()
4822 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000)); in zend_jit_inc_dec()
4824 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000)); in zend_jit_inc_dec()
4826 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_inc_dec()
4838 jit_set_Z_LVAL(jit, res_addr, ref); in zend_jit_inc_dec()
4840 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_inc_dec()
4850 jit_SET_EX_OPLINE(jit, opline); in zend_jit_inc_dec()
4854 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF); in zend_jit_inc_dec()
4860 jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_NULL); in zend_jit_inc_dec()
4866 ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_inc_dec()
4871 if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE)); in zend_jit_inc_dec()
4873 ref2 = jit_Z_PTR_ref(jit, ref); in zend_jit_inc_dec()
4875 if_typed = jit_if_TYPED_REF(jit, ref2); in zend_jit_inc_dec()
4880 arg2 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_inc_dec()
4897 zend_jit_check_exception(jit); in zend_jit_inc_dec()
4907 jit_ZVAL_COPY(jit, in zend_jit_inc_dec()
4914 ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_inc_dec()
4921 ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_inc_dec()
4928 zend_jit_check_exception(jit); in zend_jit_inc_dec()
4931 ref = jit_Z_DVAL(jit, op1_addr); in zend_jit_inc_dec()
4933 jit_set_Z_DVAL(jit, res_addr, ref); in zend_jit_inc_dec()
4934 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_inc_dec()
4942 jit_set_Z_DVAL(jit, op1_def_addr, ref); in zend_jit_inc_dec()
4945 jit_set_Z_DVAL(jit, res_addr, ref); in zend_jit_inc_dec()
4946 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_inc_dec()
4952 …if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_def_addr, op1_def_info, op1_addr… in zend_jit_inc_dec()
4956 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { in zend_jit_inc_dec()
4963 static int zend_jit_math_long_long(zend_jit_ctx *jit, in zend_jit_math_long_long() argument
4986 op1 = jit_Z_LVAL(jit, op1_addr); in zend_jit_math_long_long()
4987 op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr); in zend_jit_math_long_long()
5040 jit_set_Z_LVAL(jit, res_addr, ref); in zend_jit_math_long_long()
5045 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_math_long_long()
5061 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); in zend_jit_math_long_long()
5064 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0)); in zend_jit_math_long_long()
5065 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000)); in zend_jit_math_long_long()
5067 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000)); in zend_jit_math_long_long()
5069 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_math_long_long()
5080 jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); in zend_jit_math_long_long()
5083 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000)); in zend_jit_math_long_long()
5084 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000)); in zend_jit_math_long_long()
5086 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000)); in zend_jit_math_long_long()
5088 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_math_long_long()
5103 op1 = jit_Z_LVAL(jit, op1_addr); in zend_jit_math_long_long()
5104 op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr); in zend_jit_math_long_long()
5108 ir_ref old_cse_limit = jit->ctx.fold_cse_limit; in zend_jit_math_long_long()
5109 jit->ctx.fold_cse_limit = 0x7fffffff; in zend_jit_math_long_long()
5114 jit->ctx.fold_cse_limit = old_cse_limit; in zend_jit_math_long_long()
5117 jit_set_Z_DVAL(jit, res_addr, ref); in zend_jit_math_long_long()
5119 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_math_long_long()
5129 static int zend_jit_math_long_double(zend_jit_ctx *jit, in zend_jit_math_long_double() argument
5150 op1 = jit_Z_LVAL(jit, op1_addr); in zend_jit_math_long_double()
5151 op2 = jit_Z_DVAL(jit, op2_addr); in zend_jit_math_long_double()
5153 jit_set_Z_DVAL(jit, res_addr, ref); in zend_jit_math_long_double()
5157 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_math_long_double()
5163 static int zend_jit_math_double_long(zend_jit_ctx *jit, in zend_jit_math_double_long() argument
5184 op1 = jit_Z_DVAL(jit, op1_addr); in zend_jit_math_double_long()
5185 op2 = jit_Z_LVAL(jit, op2_addr); in zend_jit_math_double_long()
5187 jit_set_Z_DVAL(jit, res_addr, ref); in zend_jit_math_double_long()
5192 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_math_double_long()
5199 static int zend_jit_math_double_double(zend_jit_ctx *jit, in zend_jit_math_double_double() argument
5221 op1 = jit_Z_DVAL(jit, op1_addr); in zend_jit_math_double_double()
5222 op2 = (same_ops) ? op1 : jit_Z_DVAL(jit, op2_addr); in zend_jit_math_double_double()
5224 jit_set_Z_DVAL(jit, res_addr, ref); in zend_jit_math_double_double()
5229 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); in zend_jit_math_double_double()
5236 static int zend_jit_math_helper(zend_jit_ctx *jit, in zend_jit_math_helper() argument
5270 if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) { in zend_jit_math_helper()
5272 zend_jit_use_reg(jit, op1_addr); in zend_jit_math_helper()
5275 if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) { in zend_jit_math_helper()
5277 zend_jit_use_reg(jit, op2_addr); in zend_jit_math_helper()
5282 jit->delay_var = Z_SSA_VAR(res_addr); in zend_jit_math_helper()
5283 jit->delay_refs = res_inputs; in zend_jit_math_helper()
5288 if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_math_helper()
5292 if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_math_helper()
5295 …if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_… in zend_jit_math_helper()
5309 if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_math_helper()
5313 if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_math_helper()
5317 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_math_helper()
5322 if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5331 …if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_… in zend_jit_math_helper()
5342 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); in zend_jit_math_helper()
5349 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_math_helper()
5352 if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5362 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_math_helper()
5367 if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5382 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); in zend_jit_math_helper()
5389 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_math_helper()
5392 if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5402 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_math_helper()
5407 if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5419 if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_math_helper()
5426 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); in zend_jit_math_helper()
5429 if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5439 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_math_helper()
5444 if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { in zend_jit_math_helper()
5463 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { in zend_jit_math_helper()
5470 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { in zend_jit_math_helper()
5476 arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var)); in zend_jit_math_helper()
5478 arg1 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_math_helper()
5480 arg2 = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_math_helper()
5481 arg3 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_math_helper()
5482 jit_SET_EX_OPLINE(jit, opline); in zend_jit_math_helper()
5496 jit_FREE_OP(jit, op1_type, op1, op1_info, NULL); in zend_jit_math_helper()
5497 jit_FREE_OP(jit, op2_type, op2, op2_info, NULL); in zend_jit_math_helper()
5501 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), in zend_jit_math_helper()
5502 jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); in zend_jit_math_helper()
5504 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_math_helper()
5506 zend_jit_check_exception(jit); in zend_jit_math_helper()
5511 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) { in zend_jit_math_helper()
5523 ZEND_ASSERT(jit->delay_refs == res_inputs); in zend_jit_math_helper()
5525 jit->delay_var = -1; in zend_jit_math_helper()
5526 jit->delay_refs = NULL; in zend_jit_math_helper()
5528 zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]); in zend_jit_math_helper()
5531 zend_jit_def_reg(jit, res_addr, phi); in zend_jit_math_helper()
5538 static int zend_jit_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr… in zend_jit_math() argument
5542 …if (!zend_jit_math_helper(jit, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op… in zend_jit_math()
5545 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { in zend_jit_math()
5551 static int zend_jit_add_arrays(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_ji… in zend_jit_add_arrays() argument
5554 ir_ref arg1 = jit_Z_PTR(jit, op1_addr); in zend_jit_add_arrays()
5555 ir_ref arg2 = jit_Z_PTR(jit, op2_addr); in zend_jit_add_arrays()
5558 jit_set_Z_PTR(jit, res_addr, ref); in zend_jit_add_arrays()
5559 jit_set_Z_TYPE_INFO(jit, res_addr, IS_ARRAY_EX); in zend_jit_add_arrays()
5560 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_add_arrays()
5561 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); in zend_jit_add_arrays()
5565 static int zend_jit_long_math_helper(zend_jit_ctx *jit, in zend_jit_long_math_helper() argument
5594 && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) { in zend_jit_long_math_helper()
5596 zend_jit_use_reg(jit, op1_addr); in zend_jit_long_math_helper()
5600 && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) { in zend_jit_long_math_helper()
5602 zend_jit_use_reg(jit, op2_addr); in zend_jit_long_math_helper()
5606 if_long1 = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_long_math_helper()
5610 if_long2 = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_long_math_helper()
5622 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); in zend_jit_long_math_helper()
5623 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); in zend_jit_long_math_helper()
5624 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
5625 ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift)); in zend_jit_long_math_helper()
5631 ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval)); in zend_jit_long_math_helper()
5634 ref = jit_Z_LVAL(jit, op2_addr); in zend_jit_long_math_helper()
5640 ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr); in zend_jit_long_math_helper()
5646 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
5647 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); in zend_jit_long_math_helper()
5648 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); in zend_jit_long_math_helper()
5649 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift)); in zend_jit_long_math_helper()
5658 ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ref); in zend_jit_long_math_helper()
5668 jit_Z_LVAL(jit, op1_addr), in zend_jit_long_math_helper()
5671 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); in zend_jit_long_math_helper()
5672 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); in zend_jit_long_math_helper()
5673 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
5674 ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift)); in zend_jit_long_math_helper()
5680 ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval)); in zend_jit_long_math_helper()
5683 ref = jit_Z_LVAL(jit, op2_addr); in zend_jit_long_math_helper()
5694 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
5695 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); in zend_jit_long_math_helper()
5696 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); in zend_jit_long_math_helper()
5697 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift)); in zend_jit_long_math_helper()
5705 ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ref); in zend_jit_long_math_helper()
5712 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); in zend_jit_long_math_helper()
5713 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); in zend_jit_long_math_helper()
5714 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
5715 ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_mod_by_zero)); in zend_jit_long_math_helper()
5720 ref = ir_AND_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval - 1)); in zend_jit_long_math_helper()
5722 ref = ir_MOD_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval)); in zend_jit_long_math_helper()
5726 ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr); in zend_jit_long_math_helper()
5728 ref = jit_Z_LVAL(jit, op2_addr); in zend_jit_long_math_helper()
5732 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
5733 zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); in zend_jit_long_math_helper()
5734 zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); in zend_jit_long_math_helper()
5735 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_mod_by_zero)); in zend_jit_long_math_helper()
5766 op1 = jit_Z_LVAL(jit, op1_addr); in zend_jit_long_math_helper()
5767 op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr); in zend_jit_long_math_helper()
5775 jit->delay_var = Z_SSA_VAR(res_addr); in zend_jit_long_math_helper()
5776 jit->delay_refs = res_inputs; in zend_jit_long_math_helper()
5778 jit_set_Z_LVAL(jit, res_addr, ref); in zend_jit_long_math_helper()
5782 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); in zend_jit_long_math_helper()
5808 ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_long_math_helper()
5809 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF); in zend_jit_long_math_helper()
5824 ref = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_long_math_helper()
5825 if_def = jit_if_not_Z_TYPE(jit, op2_addr, IS_UNDEF); in zend_jit_long_math_helper()
5839 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { in zend_jit_long_math_helper()
5846 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { in zend_jit_long_math_helper()
5852 arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var)); in zend_jit_long_math_helper()
5854 arg1 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_long_math_helper()
5856 arg2 = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_long_math_helper()
5857 arg3 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_long_math_helper()
5858 jit_SET_EX_OPLINE(jit, opline); in zend_jit_long_math_helper()
5881 jit_FREE_OP(jit, op1_type, op1, op1_info, NULL); in zend_jit_long_math_helper()
5882 jit_FREE_OP(jit, op2_type, op2, op2_info, NULL); in zend_jit_long_math_helper()
5886 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), in zend_jit_long_math_helper()
5887 jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); in zend_jit_long_math_helper()
5889 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_long_math_helper()
5891 zend_jit_check_exception(jit); in zend_jit_long_math_helper()
5897 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) { in zend_jit_long_math_helper()
5905 ZEND_ASSERT(jit->delay_refs == res_inputs); in zend_jit_long_math_helper()
5907 jit->delay_var = -1; in zend_jit_long_math_helper()
5908 jit->delay_refs = NULL; in zend_jit_long_math_helper()
5910 zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]); in zend_jit_long_math_helper()
5913 zend_jit_def_reg(jit, res_addr, phi); in zend_jit_long_math_helper()
5921 static int zend_jit_long_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_ssa… in zend_jit_long_math() argument
5925 if (!zend_jit_long_math_helper(jit, opline, opline->opcode, in zend_jit_long_math()
5931 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { in zend_jit_long_math()
5937 static int zend_jit_concat_helper(zend_jit_ctx *jit, in zend_jit_concat_helper() argument
5956 if_op1_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING); in zend_jit_concat_helper()
5960 if_op2_string = jit_if_Z_TYPE(jit, op2_addr, IS_STRING); in zend_jit_concat_helper()
5964 ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_concat_helper()
5965 ir_ref arg2 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_concat_helper()
5971 ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_concat_helper()
5972 ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_concat_helper()
5973 ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_concat_helper()
5983 jit_FREE_OP(jit, op2_type, op2, op2_info, opline); in zend_jit_concat_helper()
6000 ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr); in zend_jit_concat_helper()
6001 ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_concat_helper()
6002 ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_concat_helper()
6004 jit_SET_EX_OPLINE(jit, opline); in zend_jit_concat_helper()
6009 jit_FREE_OP(jit, op1_type, op1, op1_info, NULL); in zend_jit_concat_helper()
6010 jit_FREE_OP(jit, op2_type, op2, op2_info, NULL); in zend_jit_concat_helper()
6013 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), in zend_jit_concat_helper()
6014 jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); in zend_jit_concat_helper()
6016 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_concat_helper()
6018 zend_jit_check_exception(jit); in zend_jit_concat_helper()
6028 static int zend_jit_concat(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op… in zend_jit_concat() argument
6038 …return zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opli… in zend_jit_concat()
6041 static int zend_jit_assign_op(zend_jit_ctx *jit, in zend_jit_assign_op() argument
6067 ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_assign_op()
6068 if_op1_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE)); in zend_jit_assign_op()
6072 ref2 = jit_Z_PTR_ref(jit, ref); in zend_jit_assign_op()
6074 if_op1_typed = jit_if_TYPED_REF(jit, ref2); in zend_jit_assign_op()
6079 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { in zend_jit_assign_op()
6082 arg2 = jit_ZVAL_ADDR(jit, real_addr); in zend_jit_assign_op()
6084 arg2 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_assign_op()
6086 jit_SET_EX_OPLINE(jit, opline); in zend_jit_assign_op()
6095 zend_jit_check_exception(jit); in zend_jit_assign_op()
6112 …result = zend_jit_math_helper(jit, opline, opline->extended_value, opline->op1_type, opline->op1, … in zend_jit_assign_op()
6120 result = zend_jit_long_math_helper(jit, opline, opline->extended_value, in zend_jit_assign_op()
6126 …result = zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, op… in zend_jit_assign_op()
6132 …if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_def_addr, op1_def_info, op1_addr… in zend_jit_assign_op()
6143 static ir_ref jit_ZVAL_DEREF_ref(zend_jit_ctx *jit, ir_ref ref) in jit_ZVAL_DEREF_ref() argument
6147 if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_REFERENCE))); in jit_ZVAL_DEREF_ref()
6149 ref2 = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val)); in jit_ZVAL_DEREF_ref()
6154 static zend_jit_addr jit_ZVAL_DEREF(zend_jit_ctx *jit, zend_jit_addr addr) in jit_ZVAL_DEREF() argument
6156 ir_ref ref = jit_ZVAL_ADDR(jit, addr); in jit_ZVAL_DEREF()
6157 ref = jit_ZVAL_DEREF_ref(jit, ref); in jit_ZVAL_DEREF()
6161 static ir_ref jit_ZVAL_INDIRECT_DEREF_ref(zend_jit_ctx *jit, ir_ref ref) in jit_ZVAL_INDIRECT_DEREF_ref() argument
6165 if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_INDIRECT))); in jit_ZVAL_INDIRECT_DEREF_ref()
6167 ref2 = jit_Z_PTR_ref(jit, ref); in jit_ZVAL_INDIRECT_DEREF_ref()
6172 static zend_jit_addr jit_ZVAL_INDIRECT_DEREF(zend_jit_ctx *jit, zend_jit_addr addr) in jit_ZVAL_INDIRECT_DEREF() argument
6174 ir_ref ref = jit_ZVAL_ADDR(jit, addr); in jit_ZVAL_INDIRECT_DEREF()
6175 ref = jit_ZVAL_INDIRECT_DEREF_ref(jit, ref); in jit_ZVAL_INDIRECT_DEREF()
6179 static int zend_jit_simple_assign(zend_jit_ctx *jit, in zend_jit_simple_assign() argument
6196 jit_ZVAL_COPY_CONST(jit, in zend_jit_simple_assign()
6201 jit_ZVAL_COPY_CONST(jit, in zend_jit_simple_assign()
6205 jit_ZVAL_COPY_CONST(jit, in zend_jit_simple_assign()
6214 if_def = jit_if_not_Z_TYPE(jit, val_addr, IS_UNDEF); in zend_jit_simple_assign()
6217 jit_set_Z_TYPE_INFO(jit, var_addr, IS_NULL); in zend_jit_simple_assign()
6219 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); in zend_jit_simple_assign()
6221 jit_SET_EX_OPLINE(jit, opline); in zend_jit_simple_assign()
6228 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef)); in zend_jit_simple_assign()
6236 ir_ref ref = jit_ZVAL_ADDR(jit, val_addr); in zend_jit_simple_assign()
6237 ref = jit_ZVAL_DEREF_ref(jit, ref); in zend_jit_simple_assign()
6242 ref = jit_ZVAL_ADDR(jit, val_addr); in zend_jit_simple_assign()
6243 type = jit_Z_TYPE_ref(jit, ref); in zend_jit_simple_assign()
6247 ref = jit_Z_PTR_ref(jit, ref); in zend_jit_simple_assign()
6250 jit_ZVAL_COPY(jit, in zend_jit_simple_assign()
6255 jit_ZVAL_COPY_2(jit, in zend_jit_simple_assign()
6262 refcount = jit_GC_DELREF(jit, ref); in zend_jit_simple_assign()
6268 jit_ZVAL_DTOR(jit, ref, val_info, opline); in zend_jit_simple_assign()
6278 jit_ZVAL_COPY(jit, in zend_jit_simple_assign()
6283 jit_ZVAL_COPY_2(jit, in zend_jit_simple_assign()
6299 static int zend_jit_assign_to_variable_call(zend_jit_ctx *jit, in zend_jit_assign_to_variable_call() argument
6323 jit_guard_not_Z_TYPE(jit, val_addr, IS_UNDEF, exit_addr); in zend_jit_assign_to_variable_call()
6328 if_def = ir_IF(jit_Z_TYPE(jit, val_addr)); in zend_jit_assign_to_variable_call()
6330 jit_SET_EX_OPLINE(jit, opline); in zend_jit_assign_to_variable_call()
6333 ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_assign_const, IR_FASTCALL_FUNC), in zend_jit_assign_to_variable_call()
6334 jit_ZVAL_ADDR(jit, var_addr), in zend_jit_assign_to_variable_call()
6365 jit_SET_EX_OPLINE(jit, opline); in zend_jit_assign_to_variable_call()
6368 ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, func, IR_FASTCALL_FUNC), in zend_jit_assign_to_variable_call()
6369 jit_ZVAL_ADDR(jit, var_addr), in zend_jit_assign_to_variable_call()
6370 jit_ZVAL_ADDR(jit, val_addr)); in zend_jit_assign_to_variable_call()
6379 static int zend_jit_assign_to_variable(zend_jit_ctx *jit, in zend_jit_assign_to_variable() argument
6402 if (Z_MODE(val_addr) == IS_REG && jit->ra[Z_SSA_VAR(val_addr)].ref == IR_NULL) { in zend_jit_assign_to_variable()
6404 zend_jit_use_reg(jit, val_addr); in zend_jit_assign_to_variable()
6408 jit->delay_var = Z_SSA_VAR(var_addr); in zend_jit_assign_to_variable()
6409 jit->delay_refs = res_inputs; in zend_jit_assign_to_variable()
6415 jit->delay_var = Z_SSA_VAR(res_addr); in zend_jit_assign_to_variable()
6416 jit->delay_refs = res_inputs; in zend_jit_assign_to_variable()
6424 ref = jit_ZVAL_ADDR(jit, var_use_addr); in zend_jit_assign_to_variable()
6425 if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE)); in zend_jit_assign_to_variable()
6427 ref2 = jit_Z_PTR_ref(jit, ref); in zend_jit_assign_to_variable()
6429 ref2 = jit_ZVAL_ADDR(jit, ref_addr); in zend_jit_assign_to_variable()
6431 if_typed = jit_if_TYPED_REF(jit, ref2); in zend_jit_assign_to_variable()
6433 jit_SET_EX_OPLINE(jit, opline); in zend_jit_assign_to_variable()
6443 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) { in zend_jit_assign_to_variable()
6446 arg2 = jit_ZVAL_ADDR(jit, real_addr); in zend_jit_assign_to_variable()
6448 arg2 = jit_ZVAL_ADDR(jit, val_addr); in zend_jit_assign_to_variable()
6475 ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2, jit_ZVAL_ADDR(jit, res_addr)); in zend_jit_assign_to_variable()
6478 zend_jit_check_exception(jit); in zend_jit_assign_to_variable()
6499 if_refcounted = jit_if_REFCOUNTED(jit, var_use_addr); in zend_jit_assign_to_variable()
6506 ref = jit_Z_PTR(jit, var_use_addr); in zend_jit_assign_to_variable()
6508 …if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val… in zend_jit_assign_to_variable()
6511 counter = jit_GC_DELREF(jit, ref); in zend_jit_assign_to_variable()
6515 jit_ZVAL_DTOR(jit, ref, var_info, opline); in zend_jit_assign_to_variable()
6517 zend_jit_check_exception(jit); in zend_jit_assign_to_variable()
6522 ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref); in zend_jit_assign_to_variable()
6525 jit_SET_EX_OPLINE(jit, opline); in zend_jit_assign_to_variable()
6530 ZEND_ASSERT(jit->delay_refs == res_inputs); in zend_jit_assign_to_variable()
6535 zend_jit_check_exception(jit); in zend_jit_assign_to_variable()
6541 ZEND_ASSERT(jit->delay_refs == res_inputs); in zend_jit_assign_to_variable()
6546 zend_jit_check_exception(jit); in zend_jit_assign_to_variable()
6550 jit_GC_DELREF(jit, ref); in zend_jit_assign_to_variable()
6552 ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref); in zend_jit_assign_to_variable()
6555 jit_SET_EX_OPLINE(jit, opline); in zend_jit_assign_to_variable()
6570 …if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val… in zend_jit_assign_to_variable()
6585 ZEND_ASSERT(jit->delay_refs == res_inputs); in zend_jit_assign_to_variable()
6587 jit->delay_var = -1; in zend_jit_assign_to_variable()
6588 jit->delay_refs = NULL; in zend_jit_assign_to_variable()
6597 phi = ir_emit2(&jit->ctx, IR_OPT(IR_COPY, jit->ctx.ir_base[phi].type), phi, 1); in zend_jit_assign_to_variable()
6599 zend_jit_def_reg(jit, var_addr, phi); in zend_jit_assign_to_variable()
6602 jit_set_Z_LVAL(jit, real_res_addr, jit_Z_LVAL(jit, var_addr)); in zend_jit_assign_to_variable()
6604 jit_set_Z_DVAL(jit, real_res_addr, jit_Z_DVAL(jit, var_addr)); in zend_jit_assign_to_variable()
6608 zend_jit_def_reg(jit, res_addr, phi); in zend_jit_assign_to_variable()
6615 static int zend_jit_qm_assign(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit… in zend_jit_qm_assign() argument
6618 if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { in zend_jit_qm_assign()
6626 …if (!zend_jit_simple_assign(jit, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_a… in zend_jit_qm_assign()
6629 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { in zend_jit_qm_assign()
6635 static int zend_jit_assign(zend_jit_ctx *jit, in zend_jit_assign() argument
6652 if (!zend_jit_update_regs(jit, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { in zend_jit_assign()
6667 if (!zend_jit_assign_to_variable(jit, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, in zend_jit_assign()
6673 …if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_addr, op1_def_info, op1_use_addr… in zend_jit_assign()
6679 && EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)) < jit->current_op_array->last_var) { in zend_jit_assign()
6682 jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_LONG); in zend_jit_assign()
6687 jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_DOUBLE); in zend_jit_assign()
6697 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { in zend_jit_assign()
6732 static ir_ref zend_jit_cmp_long_long(zend_jit_ctx *jit, in zend_jit_cmp_long_long() argument
6752 jit_set_Z_TYPE_INFO(jit, res_addr, result ? IS_TRUE : IS_FALSE); in zend_jit_cmp_long_long()
6757 return jit_IF_ex(jit, IR_FALSE, result ? target_label : target_label2); in zend_jit_cmp_long_long()
6760 return jit_IF_ex(jit, IR_TRUE, result ? target_label : target_label2); in zend_jit_cmp_long_long()
6774 ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_LVAL(jit, op1_addr), jit_Z_LVAL(jit, op2_addr)); in zend_jit_cmp_long_long()
6777 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_cmp_long_long()
6795 return jit_IF_ex(jit, ref, in zend_jit_cmp_long_long()
6808 static ir_ref zend_jit_cmp_long_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_… in zend_jit_cmp_long_double() argument
6810 …ref = ir_CMP_OP(zend_jit_cmp_op(opline), ir_INT2D(jit_Z_LVAL(jit, op1_addr)), jit_Z_DVAL(jit, op2_… in zend_jit_cmp_long_double()
6813 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_cmp_long_double()
6823 return jit_IF_ex(jit, ref, in zend_jit_cmp_long_double()
6829 static ir_ref zend_jit_cmp_double_long(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_… in zend_jit_cmp_double_long() argument
6831 …ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), ir_INT2D(jit_Z_LVAL(jit… in zend_jit_cmp_double_long()
6834 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_cmp_double_long()
6844 return jit_IF_ex(jit, ref, in zend_jit_cmp_double_long()
6850 static ir_ref zend_jit_cmp_double_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op… in zend_jit_cmp_double_double() argument
6852 …ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), jit_Z_DVAL(jit, op2_add… in zend_jit_cmp_double_double()
6855 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_cmp_double_double()
6873 return jit_IF_ex(jit, ref, in zend_jit_cmp_double_double()
6885 static ir_ref zend_jit_cmp_slow(zend_jit_ctx *jit, ir_ref ref, const zend_op *opline, zend_jit_addr… in zend_jit_cmp_slow() argument
6890 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_cmp_slow()
6900 return jit_IF_ex(jit, ref, in zend_jit_cmp_slow()
6907 static int zend_jit_cmp(zend_jit_ctx *jit, in zend_jit_cmp() argument
6943 if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) { in zend_jit_cmp()
6945 zend_jit_use_reg(jit, op1_addr); in zend_jit_cmp()
6948 if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) { in zend_jit_cmp()
6950 zend_jit_use_reg(jit, op2_addr); in zend_jit_cmp()
6956 if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_cmp()
6960 if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_cmp()
6964 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_cmp()
6969 …ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, tar… in zend_jit_cmp()
6979 …ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smar… in zend_jit_cmp()
6990 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); in zend_jit_cmp()
6997 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_cmp()
7000 …ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, t… in zend_jit_cmp()
7011 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_cmp()
7016 …ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, tar… in zend_jit_cmp()
7031 if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); in zend_jit_cmp()
7038 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_cmp()
7041 …ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, t… in zend_jit_cmp()
7052 if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); in zend_jit_cmp()
7057 …ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, tar… in zend_jit_cmp()
7069 if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); in zend_jit_cmp()
7076 if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); in zend_jit_cmp()
7079 …ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, t… in zend_jit_cmp()
7090 if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); in zend_jit_cmp()
7095 …ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, tar… in zend_jit_cmp()
7113 jit_SET_EX_OPLINE(jit, opline); in zend_jit_cmp()
7117 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { in zend_jit_cmp()
7124 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { in zend_jit_cmp()
7130 op1 = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_cmp()
7132 op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, NULL, 0); in zend_jit_cmp()
7134 op2 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_cmp()
7136 op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, NULL, 0); in zend_jit_cmp()
7140 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL); in zend_jit_cmp()
7142 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL); in zend_jit_cmp()
7144 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_cmp()
7147 …ref = zend_jit_cmp_slow(jit, ref, opline, res_addr, smart_branch_opcode, target_label, target_labe… in zend_jit_cmp()
7165 ZEND_ASSERT(jit->b >= 0); in zend_jit_cmp()
7166 bb = &jit->ssa->cfg.blocks[jit->b]; in zend_jit_cmp()
7180 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END()); in zend_jit_cmp()
7183 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); in zend_jit_cmp()
7184 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref); in zend_jit_cmp()
7196 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END()); in zend_jit_cmp()
7198 _zend_jit_add_predecessor_ref(jit, label2, jit->b, ir_END()); in zend_jit_cmp()
7200 jit->b = -1; in zend_jit_cmp()
7206 _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END()); in zend_jit_cmp()
7207 jit->b = -1; in zend_jit_cmp()
7213 static int zend_jit_identical(zend_jit_ctx *jit, in zend_jit_identical() argument
7233 ir_ref op1 = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_identical()
7234 op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, opline, 0); in zend_jit_identical()
7239 ir_ref op2 = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_identical()
7240 op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, opline, 0); in zend_jit_identical()
7262 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_identical()
7264 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); in zend_jit_identical()
7268 jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE); in zend_jit_identical()
7271 zend_jit_check_exception(jit); in zend_jit_identical()
7275 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_identical()
7287 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END()); in zend_jit_identical()
7288 jit->b = -1; in zend_jit_identical()
7293 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_identical()
7295 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); in zend_jit_identical()
7299 jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE); in zend_jit_identical()
7302 zend_jit_check_exception(jit); in zend_jit_identical()
7306 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_identical()
7318 _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END()); in zend_jit_identical()
7319 jit->b = -1; in zend_jit_identical()
7325 ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_identical()
7326 ref = jit_ZVAL_DEREF_ref(jit, ref); in zend_jit_identical()
7330 ref = jit_ZVAL_ADDR(jit, op2_addr); in zend_jit_identical()
7331 ref = jit_ZVAL_DEREF_ref(jit, ref); in zend_jit_identical()
7337 …ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smar… in zend_jit_identical()
7343 …ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, t… in zend_jit_identical()
7351 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { in zend_jit_identical()
7360 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { in zend_jit_identical()
7369 ref = ir_EQ(jit_Z_TYPE(jit, op2_addr), ir_CONST_U8(Z_TYPE_P(val))); in zend_jit_identical()
7373 ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(Z_TYPE_P(val))); in zend_jit_identical()
7377 if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { in zend_jit_identical()
7384 if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { in zend_jit_identical()
7390 jit_SET_EX_OPLINE(jit, opline); in zend_jit_identical()
7394 jit_ZVAL_ADDR(jit, op1_addr), in zend_jit_identical()
7395 jit_ZVAL_ADDR(jit, op2_addr)); in zend_jit_identical()
7400 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_identical()
7403 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_identical()
7408 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL); in zend_jit_identical()
7410 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL); in zend_jit_identical()
7412 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_identical()
7427 ref = jit_IF_ex(jit, ref, in zend_jit_identical()
7435 ZEND_ASSERT(jit->b >= 0); in zend_jit_identical()
7436 bb = &jit->ssa->cfg.blocks[jit->b]; in zend_jit_identical()
7442 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END()); in zend_jit_identical()
7445 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); in zend_jit_identical()
7446 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref); in zend_jit_identical()
7448 jit->b = -1; in zend_jit_identical()
7454 static int zend_jit_bool_jmpznz(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_j… in zend_jit_bool_jmpznz() argument
7488 ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_bool_jmpznz()
7489 ref = jit_ZVAL_DEREF_ref(jit, ref); in zend_jit_bool_jmpznz()
7504 ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_bool_jmpznz()
7505 zend_jit_zval_check_undef(jit, ref, opline->op1.var, opline, 0); in zend_jit_bool_jmpznz()
7513 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); in zend_jit_bool_jmpznz()
7515 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_bool_jmpznz()
7517 zend_jit_check_exception(jit); in zend_jit_bool_jmpznz()
7521 _zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END()); in zend_jit_bool_jmpznz()
7522 jit->b = -1; in zend_jit_bool_jmpznz()
7527 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); in zend_jit_bool_jmpznz()
7529 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_bool_jmpznz()
7531 zend_jit_check_exception(jit); in zend_jit_bool_jmpznz()
7535 _zend_jit_add_predecessor_ref(jit, false_label, jit->b, ir_END()); in zend_jit_bool_jmpznz()
7536 jit->b = -1; in zend_jit_bool_jmpznz()
7542 type = jit_Z_TYPE(jit, op1_addr); in zend_jit_bool_jmpznz()
7549 zend_jit_type_check_undef(jit, in zend_jit_bool_jmpznz()
7555 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); in zend_jit_bool_jmpznz()
7561 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_bool_jmpznz()
7580 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); in zend_jit_bool_jmpznz()
7586 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_bool_jmpznz()
7605 type = jit_Z_TYPE(jit, op1_addr); in zend_jit_bool_jmpznz()
7610 ref = jit_Z_LVAL(jit, op1_addr); in zend_jit_bool_jmpznz()
7614 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7617 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7623 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7636 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); in zend_jit_bool_jmpznz()
7641 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); in zend_jit_bool_jmpznz()
7656 type = jit_Z_TYPE(jit, op1_addr); in zend_jit_bool_jmpznz()
7661 ref = ir_NE(jit_Z_DVAL(jit, op1_addr), ir_CONST_DOUBLE(0.0)); in zend_jit_bool_jmpznz()
7664 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7667 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7673 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7686 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); in zend_jit_bool_jmpznz()
7691 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); in zend_jit_bool_jmpznz()
7701 jit_SET_EX_OPLINE(jit, opline); in zend_jit_bool_jmpznz()
7702 ref = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_is_true), jit_ZVAL_ADDR(jit, op1_addr)); in zend_jit_bool_jmpznz()
7703 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL); in zend_jit_bool_jmpznz()
7705 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_bool_jmpznz()
7709 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7712 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7720 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_bool_jmpznz()
7735 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); in zend_jit_bool_jmpznz()
7740 jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); in zend_jit_bool_jmpznz()
7751 _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs); in zend_jit_bool_jmpznz()
7757 static int zend_jit_defined(zend_jit_ctx *jit, const zend_op *opline, uint8_t smart_branch_opcode, … in zend_jit_defined() argument
7794 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_defined()
7801 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE); in zend_jit_defined()
7818 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_defined()
7825 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE); in zend_jit_defined()
7832 jit_SET_EX_OPLINE(jit, opline); in zend_jit_defined()
7848 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_defined()
7858 …_zend_jit_merge_smart_branch_inputs(jit, defined_label, undefined_label, true_inputs, false_inputs… in zend_jit_defined()
7864 static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opli… in zend_jit_escape_if_undef() argument
7866 zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg)); in zend_jit_escape_if_undef()
7867 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, reg_addr)); in zend_jit_escape_if_undef()
7872 if (!zend_jit_save_call_chain(jit, -1)) { in zend_jit_escape_if_undef()
7883 zend_jit_zval_try_addref(jit, val_addr); in zend_jit_escape_if_undef()
7886 jit_LOAD_IP_ADDR(jit, opline - 1); in zend_jit_escape_if_undef()
7887 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape)); in zend_jit_escape_if_undef()
7894 static int zend_jit_restore_zval(zend_jit_ctx *jit, int var, int8_t reg) in zend_jit_restore_zval() argument
7897 zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg)); in zend_jit_restore_zval()
7900 jit_ZVAL_COPY(jit, var_addr, MAY_BE_ANY, reg_addr, MAY_BE_ANY, 1); in zend_jit_restore_zval()
7904 static zend_jit_addr zend_jit_guard_fetch_result_type(zend_jit_ctx *jit, in zend_jit_guard_fetch_result_type() argument
7916 ir_ref ref = jit_ZVAL_ADDR(jit, val_addr); in zend_jit_guard_fetch_result_type()
7939 if_type = ir_IF(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type))); in zend_jit_guard_fetch_result_type()
7941 if_type = jit_if_Z_TYPE(jit, val_addr, type); in zend_jit_guard_fetch_result_type()
7955 jit_guard_Z_TYPE(jit, val_addr, IS_REFERENCE, res_exit_addr); in zend_jit_guard_fetch_result_type()
7956 ref = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val)); in zend_jit_guard_fetch_result_type()
7968 ir_GUARD(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type)), ir_CONST_ADDR(res_exit_addr)); in zend_jit_guard_fetch_result_type()
7970 jit_guard_Z_TYPE(jit, val_addr, type, res_exit_addr); in zend_jit_guard_fetch_result_type()
7989 static int zend_jit_fetch_constant(zend_jit_ctx *jit, in zend_jit_fetch_constant() argument
8024 jit_SET_EX_OPLINE(jit, opline); in zend_jit_fetch_constant()
8028 ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler)); in zend_jit_fetch_constant()
8037 const_addr = zend_jit_guard_fetch_result_type(jit, opline, const_addr, type, 0, 0, 0); in zend_jit_fetch_constant()
8046 jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, res_info, 1); in zend_jit_fetch_constant()
8047 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { in zend_jit_fetch_constant()
8054 jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, 1); in zend_jit_fetch_constant()
8061 static int zend_jit_type_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint8_t… in zend_jit_type_check() argument
8090 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF); in zend_jit_type_check()
8094 jit_SET_EX_OPLINE(jit, opline); in zend_jit_type_check()
8096 zend_jit_check_exception_undef_result(jit, opline); in zend_jit_type_check()
8100 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_type_check()
8107 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE); in zend_jit_type_check()
8113 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_type_check()
8120 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE); in zend_jit_type_check()
8136 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_type_check()
8139 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_type_check()
8146 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE); in zend_jit_type_check()
8150 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_type_check()
8153 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); in zend_jit_type_check()
8160 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE); in zend_jit_type_check()
8191 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_type_check()
8192 ref = jit_ZVAL_DEREF_ref(jit, ref); in zend_jit_type_check()
8196 ref = ir_AND_U32(ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)), ir_CONST_U32(mask)); in zend_jit_type_check()
8201 ref = ir_NE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type)); in zend_jit_type_check()
8203 ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type)); in zend_jit_type_check()
8206 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); in zend_jit_type_check()
8224 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_type_check()
8234 } else if (exit_addr && !jit->ctx.control) { in zend_jit_type_check()
8238 _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs); in zend_jit_type_check()
8244 static int zend_jit_isset_isempty_cv(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, z… in zend_jit_isset_isempty_cv() argument
8268 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr); in zend_jit_isset_isempty_cv()
8269 ref = jit_ZVAL_DEREF_ref(jit, ref); in zend_jit_isset_isempty_cv()
8279 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE); in zend_jit_isset_isempty_cv()
8288 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE); in zend_jit_isset_isempty_cv()
8292 ir_ref ref = ir_GT(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL)); in zend_jit_isset_isempty_cv()
8306 jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), in zend_jit_isset_isempty_cv()
8317 _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs); in zend_jit_isset_isempty_cv()
8332 static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t used_stack) in zend_jit_stack_check() argument
8351 static int zend_jit_free_trampoline(zend_jit_ctx *jit, int8_t func_reg) in zend_jit_free_trampoline() argument
8366 static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *… argument
8429 zend_jit_start_reuse_ip(jit);
8432 jit_STORE_IP(jit, ir_LOAD_A(jit_EG(vm_stack_top)));
8439 jit_IP(jit)),
8454 jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
8455 jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
8468 jit_SET_EX_OPLINE(jit, opline);
8476 jit_SET_EX_OPLINE(jit, opline);
8480 jit_STORE_IP(jit, ref);
8488 rx = jit_IP(jit);
8503 top = jit->ctx.control = ir_emit2(&jit->ctx, IR_OPT(IR_LOAD, IR_ADDR), jit->ctx.control, ref);
8519 rx = jit_IP(jit);
8534 rx = jit_IP(jit);
8552 jit_GC_ADDREF(jit, this_ref);
8576 jit_GC_ADDREF(jit, func_ref);
8627 static int zend_jit_init_fcall_guard(zend_jit_ctx *jit, uint32_t level, const zend_function *func, … argument
8682 static int zend_jit_init_fcall(zend_jit_ctx *jit, const zend_op *opline, uint32_t b, const zend_op_… argument
8689 if (jit->delayed_call_level) {
8690 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
8730 || zend_jit_prefer_const_addr_load(jit, (uintptr_t)func)) {
8799 jit_SET_EX_OPLINE(jit, opline);
8800 ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_undefined_function));
8807 …if (!zend_jit_push_call_frame(jit, opline, op_array, func, 0, 0, checked_stack, func_ref, IR_UNUSE…
8812 if (!zend_jit_save_call_chain(jit, call_level)) {
8817 jit->delayed_call_level = call_level;
8824 if (!zend_jit_set_ip(jit, opline + 1)) {
8832 static int zend_jit_init_method_call(zend_jit_ctx *jit, argument
8876 func_ref = zend_jit_deopt_rload(jit, IR_ADDR, func_reg);
8877 this_ref = zend_jit_deopt_rload(jit, IR_ADDR, this_reg);
8883 this_ref = jit_Z_PTR(jit, this_addr);
8888 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
8889 ref = jit_ZVAL_DEREF_ref(jit, ref);
8897 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
8899 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper), jit_ZVAL_ADDR(jit, op1_addr));
8912 ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_OBJECT)),
8915 ir_ref if_object = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
8919 jit_SET_EX_OPLINE(jit, opline);
8922 jit_ZVAL_ADDR(jit, op1_addr));
8925 jit_ZVAL_ADDR(jit, op1_addr));
8927 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
8932 this_ref = jit_Z_PTR(jit, op1_addr);
8935 if (jit->delayed_call_level) {
8936 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
8964 jit_SET_EX_OPLINE(jit, opline);
8966 if (!jit->ctx.fixed_call_stack_size) {
8987 if (!jit->ctx.fixed_call_stack_size) {
8992 ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9016 jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
9017 jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
9061 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9063 jit_STORE_IP(jit, ret);
9072 …if (!zend_jit_push_call_frame(jit, opline, NULL, func, 0, delayed_fetch_this, checked_stack, func_…
9080 zend_jit_start_reuse_ip(jit);
9083 if (!zend_jit_save_call_chain(jit, call_level)) {
9089 jit->delayed_call_level = call_level;
9095 if (!zend_jit_set_ip(jit, opline + 1)) {
9103 static int zend_jit_init_static_method_call(zend_jit_ctx *jit, argument
9151 if (jit->delayed_call_level) {
9152 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
9164 jit_SET_EX_OPLINE(jit, opline);
9165 scope_ref2 = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_class_helper), jit_FP(jit));
9166 ir_GUARD(scope_ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9168 …func_ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_static_method_helper), jit_FP(jit), …
9169 ir_GUARD(func_ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9235 ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9236 jit_STORE_IP(jit, ref);
9254 …if (!zend_jit_push_call_frame(jit, opline, op_array, func, 0, 0, checked_stack, func_ref, scope_re…
9263 zend_jit_start_reuse_ip(jit);
9265 if (!zend_jit_save_call_chain(jit, call_level)) {
9270 jit->delayed_call_level = call_level;
9277 if (!zend_jit_set_ip(jit, opline + 1)) {
9285 static int zend_jit_init_closure_call(zend_jit_ctx *jit, argument
9299 ref = jit_Z_PTR(jit, op2_addr);
9346 if (jit->delayed_call_level) {
9347 if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
9352 if (!zend_jit_push_call_frame(jit, opline, NULL, func, 1, 0, checked_stack, ref, IR_UNUSED)) {
9357 if (!zend_jit_save_call_chain(jit, call_level)) {
9363 jit->delayed_call_level = call_level;
9369 if (!zend_jit_set_ip(jit, opline + 1)) {
9377 static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_… argument
9384 if (!zend_jit_reuse_ip(jit)) {
9422 jit_set_Z_TYPE_INFO(jit, addr, IS_UNDEF);
9424 jit_SET_EX_OPLINE(jit, opline);
9425 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_throw_cannot_pass_by_ref));
9437 jit_ZVAL_COPY_CONST(jit,
9442 jit_ZVAL_COPY(jit,
9451 static int zend_jit_send_ref(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_arra… argument
9459 if (!zend_jit_reuse_ip(jit)) {
9465 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
9471 ir_ref if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
9474 jit_set_Z_TYPE_INFO(jit,op1_addr, IS_NULL);
9491 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
9494 ref = jit_Z_PTR(jit, op1_addr);
9496 jit_GC_ADDREF(jit, ref);
9498 jit_set_Z_PTR(jit, arg_addr, ref);
9499 jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
9506 ref = jit_EMALLOC(jit, sizeof(zend_reference), op_array, opline);
9508 jit_set_GC_REFCOUNT(jit, ref, 2);
9516 jit_ZVAL_COPY(jit,
9522 jit_set_Z_PTR(jit, op1_addr, ref);
9523 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
9526 jit_set_Z_PTR(jit, arg_addr, ref);
9527 jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
9534 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
9539 static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_arra… argument
9551 if (!zend_jit_reuse_ip(jit)) {
9561 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
9575 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
9590 jit_ZVAL_COPY(jit,
9607 ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U32(IS_REFERENCE)),
9627 jit_ZVAL_COPY(jit,
9633 ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
9653 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
9655 jit_SET_EX_OPLINE(jit, opline);
9657 jit_ZVAL_ADDR(jit, arg_addr));
9658 zend_jit_check_exception(jit);
9670 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
9682 if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
9695 if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
9700 jit_SET_EX_OPLINE(jit, opline);
9705 jit_set_Z_TYPE_INFO(jit, arg_addr, IS_NULL);
9708 ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9724 jit_ZVAL_COPY(jit,
9730 ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
9743 jit_SET_EX_OPLINE(jit, opline);
9745 jit_ZVAL_ADDR(jit, arg_addr));
9746 zend_jit_check_exception(jit);
9754 ref = jit_ZVAL_ADDR(jit, op1_addr);
9755 ref = jit_ZVAL_DEREF_ref(jit, ref);
9759 jit_ZVAL_COPY(jit,
9768 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
9772 ref = jit_Z_PTR(jit, op1_addr);
9777 jit_ZVAL_COPY(jit,
9783 refcount = jit_GC_DELREF(jit, ref);
9788 if_refcounted = jit_if_REFCOUNTED(jit, arg_addr);
9791 jit_GC_ADDREF(jit, jit_Z_PTR(jit, arg_addr));
9799 jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
9805 jit_ZVAL_COPY(jit,
9812 if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
9821 jit_ZVAL_COPY(jit,
9836 static int zend_jit_check_func_arg(zend_jit_ctx *jit, const zend_op *opline) argument
9849 if (jit->reuse_ip) {
9850 ref = jit_IP(jit);
9861 if (jit->reuse_ip) {
9862 ref = jit_IP(jit);
9875 if (!zend_jit_reuse_ip(jit)) {
9879 rx = jit_IP(jit);
9904 static int zend_jit_check_undef_args(zend_jit_ctx *jit, const zend_op *opline) argument
9908 if (jit->reuse_ip) {
9909 call = jit_IP(jit);
9919 jit_SET_EX_OPLINE(jit, opline);
9921 ir_GUARD_NOT(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
9927 static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_arra… argument
10008 if (!jit->reuse_ip) {
10009 zend_jit_start_reuse_ip(jit);
10011 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call)));
10013 rx = jit_IP(jit);
10014 zend_jit_stop_reuse_ip(jit);
10016 jit_SET_EX_OPLINE(jit, opline);
10038 if (!jit->delayed_call_level) {
10044 jit->delayed_call_level = 0;
10047 ir_STORE(jit_CALL(rx, prev_execute_data), jit_FP(jit));
10070 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
10081 ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
10104 jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var)) :
10146 jit_STORE_FP(jit, rx);
10155 jit_set_Z_TYPE_INFO_ex(jit, var_addr, ir_CONST_U32(IS_UNDEF));
10173 jit_LOAD_IP_ADDR(jit, func->op_array.opcodes + num_args);
10182 jit_LOAD_IP(jit, ip);
10203 jit_LOAD_IP(jit, ip);
10208 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper), jit_FP(jit));
10224 jit_LOAD_IP(jit, ip);
10244 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper), jit_FP(jit));
10267 jit_STORE_IP(jit, ir_ADD_A(jit_IP(jit), ref));
10297 ir_ADD_A(jit_FP(jit), ir_MUL_A(num_args, ir_CONST_ADDR(sizeof(zval)))),
10316 ir_ref rx = jit_FP(jit);
10317 …s_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_ha…
10320 jit_SET_EX_OPLINE(jit, trace[1].opline);
10323 ir_STORE(jit_EX(opline), jit_IP(jit));
10325 jit_observer_fcall_begin(jit, rx, observer_handler);
10338 zend_jit_check_timeout(jit, NULL /* we're inside the called function */, exit_addr);
10340 jit_observer_fcall_is_unobserved_end(jit, &unobserved_data);
10356 begin = jit->bb_start_ref[num_args];
10358 insn = &jit->ctx.ir_base[begin];
10361 insn = &jit->ctx.ir_base[begin];
10369 insn = &jit->ctx.ir_base[begin];
10375 ZEND_ASSERT(jit->ctx.ir_base[insn->op3].op == IR_LOOP_END);
10376 jit->ctx.ir_base[insn->op3].op = IR_END;
10379 insn = &jit->ctx.ir_base[begin];
10386 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
10392 bb = &jit->ssa->cfg.blocks[jit->b];
10400 ref = jit->ctx.insns_count - 1;
10401 ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE
10402 || jit->ctx.ir_base[ref].op == IR_RETURN
10403 || jit->ctx.ir_base[ref].op == IR_LOOP_END);
10404 ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY);
10405 ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start);
10407 _zend_jit_add_predecessor_ref(jit, succ, jit->b, ref);
10408 jit->b = -1;
10428 …s_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_ha…
10429 jit_observer_fcall_begin(jit, rx, observer_handler);
10430 jit_observer_fcall_is_unobserved_end(jit, &unobserved_data);
10442 if (!jit->ctx.fixed_call_stack_size) {
10451 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
10453 zend_jit_reset_last_valid_opline(jit);
10456 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
10472 jit_observer_fcall_end(jit, rx, res_ref);
10489 ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
10498 jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, opline);
10527 jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_CALL(rx, This.value.obj)));
10576 jit_ZVAL_PTR_DTOR(jit, res_addr, func_info, 1, opline);
10578 if (!jit->ctx.fixed_call_stack_size) {
10585 ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
10586 jit_STUB_ADDR(jit, jit_stub_icall_throw));
10593 if (trace && jit->last_valid_opline != opline) {
10604 zend_jit_check_timeout(jit, opline + 1, exit_addr);
10608 jit_LOAD_IP_ADDR(jit, opline + 1);
10612 jit_LOAD_IP_ADDR(jit, opline + 1);
10623 static int zend_jit_constructor(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_a… argument
10625 ir_ref if_skip_constructor = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, opline), next_block);
10630 if (!zend_jit_tail_handler(jit, opline)) {
10634 if (!zend_jit_do_fcall(jit, opline, op_array, ssa, call_level, next_block, NULL)) {
10640 ZEND_ASSERT(jit->ssa->cfg.blocks[next_block].predecessors_count == 1);
10641 if (!jit->ctx.control) {
10642 ZEND_ASSERT(jit->bb_edges[jit->bb_predecessors[next_block]]);
10644 ir_MERGE_2(jit->bb_edges[jit->bb_predecessors[next_block]], ir_END());
10645 jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
10647 ZEND_ASSERT(!jit->bb_edges[jit->bb_predecessors[next_block]]);
10650 jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
10652 jit->b = -1;
10658 static int zend_jit_verify_arg_type(zend_jit_ctx *jit, const zend_op *opline, zend_arg_info *arg_in… argument
10664 ref = jit_ZVAL_ADDR(jit, res_addr);
10678 ref = jit_ZVAL_DEREF_ref(jit, ref);
10680 ref = jit_Z_PTR_ref(jit, ref);
10688 ir_ref if_ok = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(type_code));
10694 ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE_ref(jit, ref)),
10702 jit_SET_EX_OPLINE(jit, opline);
10707 ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
10717 static int zend_jit_recv(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array) argument
10751 jit_SET_EX_OPLINE(jit, opline);
10752 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_missing_arg_error), jit_FP(jit));
10753 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
10759 if (!zend_jit_verify_arg_type(jit, opline, arg_info, 1)) {
10767 static int zend_jit_recv_init(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_arr… argument
10778 jit_ZVAL_COPY_CONST(jit,
10791 jit_ZVAL_COPY_CONST(jit,
10798 jit_SET_EX_OPLINE(jit, opline);
10800 jit_ZVAL_ADDR(jit, res_addr),
10805 jit_ZVAL_PTR_DTOR(jit, res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, opline);
10806 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
10828 if (!zend_jit_verify_arg_type(jit, opline, arg_info, may_throw)) {
10837 static bool zend_jit_verify_return_type(zend_jit_ctx *jit, const zend_op *opline, const zend_op_arr… argument
10853 ir_ref if_ok = jit_if_Z_TYPE(jit, op1_addr, type_code);
10860 ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)),
10871 jit_SET_EX_OPLINE(jit, opline);
10872 ref = jit_ZVAL_ADDR(jit, op1_addr);
10874 ref = zend_jit_zval_check_undef(jit, ref, opline->op1.var, NULL, 1);
10883 zend_jit_check_exception(jit);
10893 static int zend_jit_leave_frame(zend_jit_ctx *jit) argument
10900 static int zend_jit_free_cvs(zend_jit_ctx *jit) argument
10906 ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_compiled_variables), jit_FP(jit));
10910 static int zend_jit_free_cv(zend_jit_ctx *jit, uint32_t info, uint32_t var) argument
10915 jit_ZVAL_PTR_DTOR(jit, var_addr, info, 1, NULL);
10920 static int zend_jit_free_op(zend_jit_ctx *jit, const zend_op *opline, uint32_t info, uint32_t var_o… argument
10923 jit_ZVAL_PTR_DTOR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, opline);
10928 static int zend_jit_leave_func(zend_jit_ctx *jit, argument
10963 if (!zend_jit_leave_frame(jit)) {
10976 ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_func_helper), jit_FP(jit));
10990 ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
10992 ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
10998 jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data)));
11003 ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_leave_function_handler));
11010 if (!zend_jit_leave_frame(jit)) {
11015 jit_OBJ_RELEASE(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), -sizeof(zend_object)));
11021 if (!zend_jit_leave_frame(jit)) {
11036 jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_EX(This.value.obj)));
11045 ir_STORE(jit_EG(vm_stack_top), jit_FP(jit));
11048 jit_STORE_FP(jit, ir_LOAD_A(jit_EX(prev_execute_data)));
11052 ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
11058 zend_jit_reset_last_valid_opline(jit);
11070 jit_STORE_IP(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(opline)), sizeof(zend_op)));
11072 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
11073 jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
11095 ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
11108 ir_ref if_eq = ir_IF(jit_CMP_IP(jit, IR_EQ, next_opline));
11111 ZEND_ASSERT(jit->trace_loop_ref);
11112 ZEND_ASSERT(jit->ctx.ir_base[jit->trace_loop_ref].op2 == IR_UNUSED);
11113 ir_MERGE_SET_OP(jit->trace_loop_ref, 2, ir_END());
11117 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
11119 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape));
11122 ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), jit_STUB_ADDR(jit, jit_stub_trace_escape));
11125 zend_jit_set_last_valid_opline(jit, trace->opline);
11134 ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
11140 ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
11143 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
11144 jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
11153 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
11158 jit->b = -1;
11163 static void zend_jit_common_return(zend_jit_ctx *jit) argument
11165 ZEND_ASSERT(jit->return_inputs);
11166 ir_MERGE_list(jit->return_inputs);
11169 static int zend_jit_return(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array,… argument
11179 jit->return_inputs = IR_UNUSED;
11195 if (!zend_jit_spill_store_inv(jit, op1_addr, dst, op1_info)) {
11200 jit_observer_fcall_end(jit, jit_FP(jit), jit_ZVAL_ADDR(jit, op1_addr));
11214 ir_ref if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
11216 ir_END_list(jit->return_inputs);
11219 ref = jit_Z_PTR(jit, op1_addr);
11220 refcount = jit_GC_DELREF(jit, ref);
11226 ir_END_list(jit->return_inputs);
11229 jit_ZVAL_DTOR(jit, ref, op1_info, opline);
11232 ir_END_list(jit->return_inputs);
11238 ir_END_list(jit->return_inputs);
11246 if (jit->return_inputs) {
11248 ir_END_list(jit->return_inputs);
11249 ir_MERGE_list(jit->return_inputs);
11250 jit->return_inputs = IR_UNUSED;
11258 jit_ZVAL_COPY_CONST(jit, ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
11260 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
11263 ref = jit_ZVAL_ADDR(jit, op1_addr);
11264 ref = jit_ZVAL_DEREF_ref(jit, ref);
11272 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 1);
11274 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
11276 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_NULL);
11278 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
11281 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
11288 if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
11292 ref = jit_Z_PTR(jit, op1_addr);
11297 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, ref_addr, op1_info, 0);
11298 ref2 = jit_GC_DELREF(jit, ref);
11303 ir_ref if_refcounted = jit_if_REFCOUNTED(jit, ret_addr);
11305 ir_END_list(jit->return_inputs);
11309 ref2 = jit_Z_PTR(jit, ret_addr);
11310 jit_GC_ADDREF(jit, ref2);
11311 ir_END_list(jit->return_inputs);
11315 jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
11316 ir_END_list(jit->return_inputs);
11320 jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
11324 if (jit->return_inputs) {
11325 ir_END_list(jit->return_inputs);
11326 ir_MERGE_list(jit->return_inputs);
11327 jit->return_inputs = IR_UNUSED;
11330 ir_END_list(jit->return_inputs);
11331 jit->b = -1;
11337 static int zend_jit_bind_global(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info) argument
11362 if_reference = jit_if_Z_TYPE_ref(jit, bucket_ref, ir_CONST_U8(IS_REFERENCE));
11374 ref = jit_Z_PTR_ref(jit, bucket_ref);
11375 jit_GC_ADDREF(jit, ref);
11392 if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
11397 ref2 = jit_Z_PTR(jit, op1_addr);
11400 jit_set_Z_PTR(jit, op1_addr, ref);
11401 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
11404 refcount = jit_GC_DELREF(jit, ref2);
11412 jit_ZVAL_DTOR(jit, ref2, op1_info, opline);
11418 if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref2);
11423 jit_SET_EX_OPLINE(jit, opline);
11435 jit_set_Z_PTR(jit, op1_addr, ref);
11436 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
11447 static int zend_jit_free(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, int may_throw) argument
11453 jit_SET_EX_OPLINE(jit, opline);
11459 if_array = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
11464 ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_iter_idx)));
11476 jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
11479 zend_jit_check_exception(jit);
11486 static int zend_jit_echo(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info) argument
11499 jit_SET_EX_OPLINE(jit, opline);
11503 zend_jit_check_exception(jit);
11511 jit_SET_EX_OPLINE(jit, opline);
11513 ref = jit_Z_PTR(jit, op1_addr);
11519 jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
11522 zend_jit_check_exception(jit);
11527 static int zend_jit_strlen(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_ad… argument
11537 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(len));
11539 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11540 } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11548 ref = jit_Z_PTR(jit, op1_addr);
11550 jit_set_Z_LVAL(jit, res_addr, ref);
11553 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11557 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11559 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
11564 static int zend_jit_count(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_add… argument
11574 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(count));
11576 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11577 } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11586 ref = jit_Z_PTR(jit, op1_addr);
11593 jit_set_Z_LVAL(jit, res_addr, ref);
11596 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
11600 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
11602 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
11606 zend_jit_check_exception(jit);
11611 static int zend_jit_in_array(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_… argument
11624 jit_Z_PTR(jit, op1_addr));
11641 ZEND_ASSERT(jit->b >= 0);
11642 bb = &jit->ssa->cfg.blocks[jit->b];
11644 ref = jit_IF_ex(jit, ref,
11646 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
11647 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
11648 jit->b = -1;
11650 jit_set_Z_TYPE_INFO_ex(jit, res_addr,
11657 static int zend_jit_rope(zend_jit_ctx *jit, const zend_op *opline, uint32_t op2_info) argument
11672 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ir_CONST_ADDR(str));
11679 ref = jit_Z_PTR(jit, op2_addr);
11680 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ref);
11684 if_refcounted = jit_if_REFCOUNTED(jit, op2_addr);
11686 jit_GC_ADDREF(jit, ref);
11699 ir_ADD_OFFSET(jit_FP(jit), opline->op1.var),
11702 jit_set_Z_PTR(jit, res_addr, ref);
11703 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
11709 static int zend_jit_zval_copy_deref(zend_jit_ctx *jit, zend_jit_addr res_addr, zend_jit_addr val_ad… argument
11714 ir_ref val = jit_ZVAL_ADDR(jit, val_addr);
11726 ptr = jit_Z_PTR(jit, val_addr);
11747 type2 = jit_Z_TYPE_INFO_ref(jit, val2);
11748 ptr2 = jit_Z_PTR_ref(jit, val2);
11769 jit_GC_ADDREF(jit, ptr);
11786 jit_set_Z_PTR(jit, res_addr, ptr);
11788 jit_set_Z_W2(jit, res_addr, jit_Z_W2(jit, val_addr));
11790 jit_set_Z_TYPE_INFO_ex(jit, res_addr, type);
11795 static int zend_jit_fetch_dimension_address_inner(zend_jit_ctx *jit, argument
11841 if_type = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
11862 h = jit_Z_LVAL(jit, op2_addr);
11879 h = jit_Z_LVAL(jit, op2_addr);
11957 h = jit_Z_LVAL(jit, op2_addr);
11983 ir_ref type_ref = jit_Z_TYPE_ref(jit, ref);
12007 h = jit_Z_LVAL(jit, op2_addr);
12034 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
12036 jit_SIDE_EXIT(jit, ir_CONST_ADDR(not_found_exit_addr));
12051 jit_SET_EX_OPLINE(jit, opline);
12055 h = jit_Z_LVAL(jit, op2_addr);
12060 ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key_ex), h, jit_FP(jit));
12063 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_offset, IR_FASTCALL_FUNC));
12071 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12086 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
12108 h = jit_Z_LVAL(jit, op2_addr);
12130 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
12148 h = jit_Z_LVAL(jit, op2_addr);
12170 if_type = jit_if_Z_TYPE(jit, op2_addr, IS_STRING);
12175 key = jit_Z_PTR(jit, op2_addr);
12243 jit_SET_EX_OPLINE(jit, opline);
12244 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_key, IR_FASTCALL_FUNC));
12251 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12299 jit_SET_EX_OPLINE(jit, opline);
12301 ref = jit_ZVAL_ADDR(jit, op2_addr);
12307 jit_ZVAL_ADDR(jit, res_addr));
12331 jit_ZVAL_ADDR(jit, res_addr));
12370 ref = jit_ZVAL_DEREF_ref(jit, ref);
12372 cond = ir_GT(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_NULL));
12391 static int zend_jit_fetch_dim_read(zend_jit_ctx *jit, argument
12489 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
12490 ref = jit_ZVAL_DEREF_ref(jit, ref);
12504 jit_guard_Z_TYPE(jit, op1_addr, IS_ARRAY, exit_addr);
12506 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
12511 ht_ref = jit_Z_PTR(jit, op1_addr);
12518 if (!zend_jit_fetch_dimension_address_inner(jit, opline,
12545 val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
12557 jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
12559 } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
12564 ir_ref type_info = jit_Z_TYPE_INFO(jit, val_addr);
12565 if (!zend_jit_zval_copy_deref(jit, res_addr, val_addr, type_info)) {
12570 jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, 1);
12576 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12578 } else if (!end_inputs && jit->ctx.control) {
12595 jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, exit_addr);
12597 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
12601 jit_SET_EX_OPLINE(jit, opline);
12602 str_ref = jit_Z_PTR(jit, op1_addr);
12608 str_ref, jit_Z_LVAL(jit, op2_addr));
12611 str_ref, jit_ZVAL_ADDR(jit, op2_addr));
12613 jit_set_Z_PTR(jit, res_addr, ref);
12614 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
12618 jit_ZVAL_ADDR(jit, op2_addr),
12619 jit_ZVAL_ADDR(jit, res_addr));
12635 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
12637 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
12642 jit_SET_EX_OPLINE(jit, opline);
12647 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
12652 jit_ZVAL_ADDR(jit, op1_addr),
12654 jit_ZVAL_ADDR(jit, res_addr));
12657 jit_ZVAL_ADDR(jit, op1_addr),
12659 jit_ZVAL_ADDR(jit, res_addr));
12674 jit_SET_EX_OPLINE(jit, opline);
12677 zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op1_addr), opline->op1.var, NULL, 0, 1, 0);
12682 zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op2_addr), opline->op2.var, NULL, 0, 1, 0);
12691 ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
12693 jit_SET_EX_OPLINE(jit, opline);
12694 ref = jit_ZVAL_ADDR(jit, op1_addr);
12699 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12718 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
12725 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
12730 zend_jit_check_exception(jit);
12739 static zend_jit_addr zend_jit_prepare_array_update(zend_jit_ctx *jit, argument
12755 ref = jit_ZVAL_ADDR(jit, op1_addr);
12760 if_reference = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
12764 array_reference_ref = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val));
12765 if_array = jit_if_Z_TYPE_ref(jit, array_reference_ref, ir_CONST_U8(IS_ARRAY));
12770 jit_SET_EX_OPLINE(jit, opline);
12773 ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
12784 *if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
12792 ref = jit_Z_PTR_ref(jit, op1_ref);
12795 ir_ref if_refcount_1 = ir_IF(ir_EQ(jit_GC_REFCOUNT(jit, ref), ir_CONST_U32(1)));
12815 *if_type = ir_IF(ir_LE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL)));
12824 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
12836 jit_STUB_FUNC_ADDR(jit, jit_stub_new_array, IR_FASTCALL_FUNC),
12837 jit_ZVAL_ADDR(jit, op1_addr));
12856 static int zend_jit_fetch_dim(zend_jit_ctx *jit, argument
12871 jit_SET_EX_OPLINE(jit, opline);
12874 …op1_addr = zend_jit_prepare_array_update(jit, opline, &op1_info, op1_addr, &if_type, &ht_ref, &may…
12894 jit_SET_EX_OPLINE(jit, opline);
12896 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
12900 jit_set_Z_PTR(jit, res_addr, ref);
12901 jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
12926 if (!zend_jit_fetch_dimension_address_inner(jit, opline, type, op1_info,
12935 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
12948 jit_set_Z_PTR(jit, res_addr, ref);
12949 jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
12967 jit_SET_EX_OPLINE(jit, opline);
12976 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
12983 jit_ZVAL_ADDR(jit, op1_addr),
12985 jit_ZVAL_ADDR(jit, res_addr));
12989 jit_ZVAL_ADDR(jit, op1_addr),
12991 jit_ZVAL_ADDR(jit, res_addr));
13022 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
13025 zend_jit_check_exception(jit);
13031 static int zend_jit_isset_isempty_dim(zend_jit_ctx *jit, argument
13059 ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
13060 ref = jit_ZVAL_DEREF_ref(jit, ref);
13070 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
13074 ht_ref = jit_Z_PTR(jit, op1_addr);
13087 if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_JIT_IS, op1_info,
13111 jit_SET_EX_OPLINE(jit, opline);
13112 arg1 = jit_ZVAL_ADDR(jit, op1_addr);
13117 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
13130 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op2_addr));
13135 jit_SET_EX_OPLINE(jit, opline);
13155 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
13157 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
13160 zend_jit_check_exception_undef_result(jit, opline);
13165 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
13171 _zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
13173 _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
13178 jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
13187 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
13189 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
13192 zend_jit_check_exception_undef_result(jit, opline);
13197 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
13203 _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
13205 _zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
13210 jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
13218 jit->b = -1;
13226 static int zend_jit_assign_dim(zend_jit_ctx *jit, argument
13245 if (!zend_jit_update_regs(jit, (opline+1)->op1.var, op3_addr, op3_def_addr, val_info)) {
13261 jit_guard_not_Z_TYPE(jit, op3_addr, IS_UNDEF, exit_addr);
13266 …op1_addr = zend_jit_prepare_array_update(jit, opline, &op1_info, op1_addr, &if_type, &ht_ref, &may…
13283 jit_SET_EX_OPLINE(jit, opline);
13284 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
13290 …if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, v…
13302 if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_W, op1_info,
13325 …if (!zend_jit_assign_to_variable_call(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->o…
13329 …if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_ty…
13347 jit_SET_EX_OPLINE(jit, opline);
13355 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
13361 arg4 = jit_ZVAL_ADDR(jit, res_addr);
13364 jit_ZVAL_ADDR(jit, op1_addr),
13366 jit_ZVAL_ADDR(jit, op3_addr),
13376 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, NULL);
13389 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
13392 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
13396 zend_jit_check_exception(jit);
13402 static int zend_jit_assign_dim_op(zend_jit_ctx *jit, argument
13427 jit_SET_EX_OPLINE(jit, opline);
13430 …op1_addr = zend_jit_prepare_array_update(jit, opline, &op1_info, op1_addr, &if_type, &ht_ref, &may…
13448 ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
13481 if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_RW, op1_info,
13493 jit_guard_Z_TYPE(jit, var_addr, dim_type, not_found_exit_addr);
13500 ref = jit_ZVAL_ADDR(jit, var_addr);
13501 if_ref = jit_if_Z_TYPE(jit, var_addr, IS_REFERENCE);
13506 reference = jit_Z_PTR_ref(jit, ref);
13508 if_typed = jit_if_TYPED_REF(jit, reference);
13513 arg2 = jit_ZVAL_ADDR(jit, op3_addr);
13541 …if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_i…
13552 if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
13561 …if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opl…
13589 arg2 = jit_ZVAL_ADDR(jit, op2_addr);
13593 jit_ZVAL_ADDR(jit, op1_addr),
13595 jit_ZVAL_ADDR(jit, op3_addr),
13604 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
13605 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
13607 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
13610 zend_jit_check_exception(jit);
13616 static int zend_jit_fe_reset(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info) argument
13624 jit_ZVAL_COPY_CONST(jit, res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
13628 jit_ZVAL_COPY(jit, res_addr, -1, op1_addr, op1_info, opline->op1_type == IS_CV);
13632 …ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->result.var + offsetof(zval, u2.fe_pos)), ir_CONST_U32(…
13637 static int zend_jit_packed_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint32_t o… argument
13649 ir_LOAD_U32(ir_ADD_OFFSET(jit_Z_PTR(jit, addr), offsetof(zend_array, u.flags))),
13660 static int zend_jit_fe_fetch(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t … argument
13671 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
13676 ZEND_ASSERT(jit->b >= 0);
13677 bb = &jit->ssa->cfg.blocks[jit->b];
13678 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
13679 jit->b = -1;
13686 ht_ref = jit_Z_PTR(jit, op1_addr);
13689 if (!zend_jit_packed_guard(jit, opline, opline->op1.var, op1_info)) {
13695 …hash_pos_ref = packed_pos_ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(…
13751 if_def_hash = ir_IF(jit_Z_TYPE_ref(jit, hash_p_ref));
13754 ir_GUARD_NOT(jit_Z_TYPE_ref(jit, hash_p_ref), ir_CONST_ADDR(exit_addr));
13813 if_def_packed = ir_IF(jit_Z_TYPE_ref(jit, packed_p_ref));
13816 ir_GUARD_NOT(jit_Z_TYPE_ref(jit, packed_p_ref), ir_CONST_ADDR(exit_addr));
13843 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
13859 jit_set_Z_PTR(jit, res_addr, key_ref);
13866 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
13871 jit_GC_ADDREF(jit, key_ref);
13872 jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
13886 jit_set_Z_LVAL(jit, res_addr, ref);
13887 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
13904 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
13913 jit_set_Z_LVAL(jit, res_addr, packed_pos_ref);
13914 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
13944 ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
13962 …if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, v…
13967 jit_ZVAL_COPY(jit, var_addr, -1, val_addr, val_info, 1);
13973 ZEND_ASSERT(jit->b >= 0);
13974 bb = &jit->ssa->cfg.blocks[jit->b];
13975 _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ir_END());
13977 if (!jit->ctx.ir_base[exit_inputs].op2) {
13983 _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
13984 jit->b = -1;
13994 static int zend_jit_load_this(zend_jit_ctx *jit, uint32_t var) argument
13998 ir_ref ref = jit_Z_PTR(jit, this_addr);
14000 jit_set_Z_PTR(jit, var_addr, ref);
14001 jit_set_Z_TYPE_INFO(jit, var_addr, IS_OBJECT_EX);
14002 jit_GC_ADDREF(jit, ref);
14007 static int zend_jit_fetch_this(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_ar… argument
14024 jit_guard_Z_TYPE(jit, this_addr, IS_OBJECT, exit_addr);
14032 ir_ref if_object = jit_if_Z_TYPE(jit, this_addr, IS_OBJECT);
14035 jit_SET_EX_OPLINE(jit, opline);
14036 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_invalid_this));
14043 if (!zend_jit_load_this(jit, opline->result.var)) {
14051 static int zend_jit_class_guard(zend_jit_ctx *jit, const zend_op *opline, ir_ref obj_ref, zend_clas… argument
14066 static int zend_jit_fetch_obj(zend_jit_ctx *jit, argument
14105 obj_ref = jit_Z_PTR(jit, this_addr);
14111 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
14114 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
14124 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
14126 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
14132 jit_SET_EX_OPLINE(jit, opline);
14138 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
14148 op1_ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
14150 op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
14155 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
14159 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
14162 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
14169 obj_ref = jit_Z_PTR(jit, op1_addr);
14182 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
14220 jit_SET_EX_OPLINE(jit, opline);
14247 prop_type_ref = jit_Z_TYPE_ref(jit, prop_ref);
14289 ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
14291 ref = jit_Z_PTR(jit, prop_addr);
14292 jit_GC_ADDREF(jit, ref);
14293 jit_set_Z_PTR(jit, res_addr, ref);
14294 jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
14299 jit_SET_EX_OPLINE(jit, opline);
14303 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
14309 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
14315 jit_SET_EX_OPLINE(jit, opline);
14324 jit_ZVAL_ADDR(jit, res_addr));
14344 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
14348 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
14355 ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
14357 ir_ref ref = jit_Z_PTR(jit, prop_addr);
14358 jit_GC_ADDREF(jit, ref);
14359 jit_set_Z_PTR(jit, res_addr, ref);
14360 jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
14364 jit_SET_EX_OPLINE(jit, opline);
14366 jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
14378 ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
14380 ir_ref ref = jit_Z_PTR(jit, prop_addr);
14381 jit_GC_ADDREF(jit, ref);
14382 jit_set_Z_PTR(jit, res_addr, ref);
14383 jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
14402 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
14406 jit_SET_EX_OPLINE(jit, opline);
14416 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
14434 jit_ZVAL_ADDR(jit, res_addr));
14445 jit_set_Z_PTR(jit, res_addr, prop_ref);
14446 jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
14454 ir_END_PHI_list(end_values, jit_ZVAL_ADDR(jit, prop_addr));
14456 prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
14458 if (!zend_jit_zval_copy_deref(jit, res_addr, prop_addr, prop_type_ref)) {
14472 jit_SET_EX_OPLINE(jit, opline);
14519 val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
14530 jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
14532 if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
14547 if_refcounted = jit_if_REFCOUNTED(jit, orig_op1_addr);
14551 ptr = jit_Z_PTR(jit, orig_op1_addr);
14552 refcount = jit_GC_DELREF(jit, ptr);
14557 jit_SET_EX_OPLINE(jit, opline);
14565 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
14579 zend_jit_check_exception(jit);
14585 static int zend_jit_assign_obj(zend_jit_ctx *jit, argument
14617 if (!zend_jit_update_regs(jit, (opline+1)->op1.var, val_addr, val_def_addr, val_info)) {
14635 obj_ref = jit_Z_PTR(jit, this_addr);
14640 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
14643 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
14653 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
14655 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
14658 jit_SET_EX_OPLINE(jit, opline);
14660 jit_ZVAL_ADDR(jit, op1_addr),
14664 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
14672 obj_ref = jit_Z_PTR(jit, op1_addr);
14685 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
14725 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
14741 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
14744 arg3 = jit_ZVAL_ADDR(jit, real_addr);
14746 arg3 = jit_ZVAL_ADDR(jit, val_addr);
14753 arg4 = jit_ZVAL_ADDR(jit, real_addr);
14755 arg4 = jit_ZVAL_ADDR(jit, res_addr);
14758 jit_SET_EX_OPLINE(jit, opline);
14773 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
14798 ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
14800 ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
14809 jit_SET_EX_OPLINE(jit, opline);
14822 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
14825 arg3 = jit_ZVAL_ADDR(jit, real_addr);
14827 arg3 = jit_ZVAL_ADDR(jit, val_addr);
14833 arg4 = jit_ZVAL_ADDR(jit, real_addr);
14835 arg4 = jit_ZVAL_ADDR(jit, res_addr);
14845 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
14857 …if (!zend_jit_assign_to_variable_call(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_t…
14861 …if (!zend_jit_assign_to_variable(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, …
14882 jit_SET_EX_OPLINE(jit, opline);
14886 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
14889 arg3 = jit_ZVAL_ADDR(jit, real_addr);
14891 arg3 = jit_ZVAL_ADDR(jit, val_addr);
14897 arg5 = jit_ZVAL_ADDR(jit, real_addr);
14899 arg5 = jit_ZVAL_ADDR(jit, res_addr);
14913 if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
14926 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
14934 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
14938 zend_jit_check_exception(jit);
14944 static int zend_jit_assign_obj_op(zend_jit_ctx *jit, argument
14985 obj_ref = jit_Z_PTR(jit, this_addr);
14990 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
14993 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
15003 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
15005 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
15008 jit_SET_EX_OPLINE(jit, opline);
15013 jit_ZVAL_ADDR(jit, op1_addr),
15022 obj_ref = jit_Z_PTR(jit, op1_addr);
15035 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
15091 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
15110 ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
15112 ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
15123 jit_SET_EX_OPLINE(jit, opline);
15127 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
15130 arg2 = jit_ZVAL_ADDR(jit, real_addr);
15132 arg2 = jit_ZVAL_ADDR(jit, val_addr);
15135 if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
15140 reference = jit_Z_PTR(jit, prop_addr);
15142 if_typed = jit_if_TYPED_REF(jit, reference);
15192 jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
15201 if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
15206 reference = jit_Z_PTR(jit, var_addr);
15208 if_typed = jit_if_TYPED_REF(jit, reference);
15213 jit_SET_EX_OPLINE(jit, opline);
15217 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
15220 arg2 = jit_ZVAL_ADDR(jit, real_addr);
15222 arg2 = jit_ZVAL_ADDR(jit, val_addr);
15256 …if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_i…
15295 if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
15305 …if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opl…
15327 if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
15330 arg3 = jit_ZVAL_ADDR(jit, real_addr);
15332 arg3 = jit_ZVAL_ADDR(jit, val_addr);
15334 jit_SET_EX_OPLINE(jit, opline);
15355 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
15361 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
15365 zend_jit_check_exception(jit);
15371 static int zend_jit_incdec_obj(zend_jit_ctx *jit, argument
15413 obj_ref = jit_Z_PTR(jit, this_addr);
15418 op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
15421 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
15431 jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
15433 ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
15436 jit_SET_EX_OPLINE(jit, opline);
15438 jit_ZVAL_ADDR(jit, op1_addr),
15441 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
15445 obj_ref = jit_Z_PTR(jit, op1_addr);
15458 } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
15514 ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
15533 ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
15535 ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
15547 jit_SET_EX_OPLINE(jit, opline);
15595 jit_ZVAL_ADDR(jit, res_addr));
15614 jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
15622 if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
15627 reference = jit_Z_PTR(jit, var_addr);
15629 if_typed = jit_if_TYPED_REF(jit, reference);
15652 jit_SET_EX_OPLINE(jit, opline);
15655 (opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
15670 if_long = jit_if_Z_TYPE(jit, var_addr, IS_LONG);
15674 addr = jit_ZVAL_ADDR(jit, var_addr);
15678 jit_set_Z_LVAL(jit, res_addr, ref);
15679 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
15694 jit_set_Z_LVAL(jit, res_addr, ref);
15695 jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
15709 jit_ZVAL_COPY(jit, res_addr, -1, var_addr, var_info, 1);
15714 jit_ZVAL_ADDR(jit, var_addr),
15715 jit_ZVAL_ADDR(jit, res_addr));
15718 jit_ZVAL_ADDR(jit, var_addr));
15723 jit_ZVAL_ADDR(jit, var_addr),
15724 jit_ZVAL_ADDR(jit, res_addr));
15727 jit_ZVAL_ADDR(jit, var_addr));
15737 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0));
15738 jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0x41e00000));
15740 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x43e0000000000000));
15742 jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
15745 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
15746 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
15748 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
15750 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
15754 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x00200000));
15755 jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0xc1e00000));
15757 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0xc3e0000000000000));
15759 jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
15762 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
15763 jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
15765 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
15767 jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
15789 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
15820 jit_SET_EX_OPLINE(jit, opline);
15826 (opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
15839 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
15843 zend_jit_check_exception(jit);
15849 static int zend_jit_fetch_static_prop(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array… argument
15916 if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
15932 jit_SET_EX_OPLINE(jit, opline);
15934 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
15965 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
15977 jit_SET_EX_OPLINE(jit, opline);
15978 …ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_fetch_static_property), jit_FP(jit), ir_CONST_I32(…
15979 zend_jit_check_exception_undef_result(jit, opline);
15987 if (!zend_jit_zval_copy_deref(jit, res_addr, ZEND_ADDR_REF_ZVAL(ref),
15988 jit_Z_TYPE_INFO_ref(jit, ref))) {
15993 jit_set_Z_PTR(jit, res_addr, ref);
15994 jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
16000 static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array,… argument
16051 _zend_jit_add_predecessor_ref(jit, b, jit->b, ir_END());
16052 jit->b = -1;
16093 ref = jit_ZVAL_ADDR(jit, op1_addr);
16094 if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
16101 jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
16103 ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
16109 ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
16113 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
16115 if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
16126 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
16128 ir_ref if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
16134 ir_ref ref = jit_Z_LVAL(jit, op1_addr);
16170 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
16178 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
16190 _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
16193 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
16196 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
16198 jit->b = -1;
16201 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
16202 jit->b = -1;
16220 ref = jit_ZVAL_ADDR(jit, op1_addr);
16221 if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16228 jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
16230 ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16236 ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
16240 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
16242 if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16253 jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, fallback_label);
16255 ir_ref if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16262 ir_ref ref = jit_Z_PTR(jit, op1_addr);
16295 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
16303 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
16315 _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
16317 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
16320 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
16322 jit->b = -1;
16325 _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
16326 jit->b = -1;
16336 op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
16341 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
16344 jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, default_label);
16346 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
16351 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
16357 ref = jit_Z_LVAL(jit, op1_addr);
16371 if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16374 jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, default_label);
16376 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16381 ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
16387 ir_ref ref2 = jit_Z_PTR(jit, op1_addr);
16431 jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
16439 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
16445 _zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
16447 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
16454 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
16457 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
16468 jit_guard_Z_TYPE(jit, op1_addr, IS_UNDEF, default_label);
16470 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
16475 ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
16482 jit_SET_EX_OPLINE(jit, opline);
16485 zend_jit_check_exception_undef_result(jit, opline);
16487 jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
16499 if (jit->ctx.ir_base[ref].op == IR_SWITCH) {
16500 ZEND_ASSERT(jit->ctx.ir_base[ref].op3 == IR_UNUSED);
16501 jit->ctx.ir_base[ref].op3 = default_input_list;
16504 _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
16507 jit->b = -1;
16516 static int zend_jit_start(zend_jit_ctx *jit, const zend_op_array *op_array, zend_ssa *ssa) argument
16521 …zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : (IR_START_BR_TARGET|IR_ENTRY_…
16523 jit->ctx.spill_base = ZREG_FP;
16525 jit->op_array = jit->current_op_array = op_array;
16526 jit->ssa = ssa;
16527 jit->bb_start_ref = zend_arena_calloc(&CG(arena), ssa->cfg.blocks_count * 2, sizeof(ir_ref));
16528 jit->bb_predecessors = jit->bb_start_ref + ssa->cfg.blocks_count;
16532 jit->bb_predecessors[i] = count;
16535 jit->bb_edges = zend_arena_calloc(&CG(arena), count, sizeof(ir_ref));
16539 jit_STORE_FP(jit, ref);
16540 jit->ctx.flags |= IR_FASTCALL_FUNC;
16546 static void *zend_jit_finish(zend_jit_ctx *jit) argument
16555 if (jit->name) {
16556 str = zend_string_copy(jit->name);
16558 str = zend_jit_func_name(jit->op_array);
16562 if (jit->op_array) {
16564 _zend_jit_fix_merges(jit);
16566 } else if (jit->trace) {
16567 jit->ctx.deoptimization_exits = jit->trace->exit_count;
16568 jit->ctx.get_exit_addr = zend_jit_trace_get_exit_addr;
16572 jit->ctx.flags |= IR_GEN_CACHE_DEMOTE;
16576 entry = zend_jit_ir_compile(&jit->ctx, &size, str ? ZSTR_VAL(str) : NULL);
16587 &jit->ctx, stderr);
16596 if (!(jit->ctx.flags & IR_FUNCTION)
16620 if (jit->op_array) {
16622 const zend_op_array *op_array = jit->op_array;
16632 if (jit->ctx.entries_count) {
16634 int i = jit->ctx.entries_count;
16636 ir_insn *insn = &jit->ctx.ir_base[jit->ctx.entries[--i]];
16642 zend_jit_trace_info *t = jit->trace;
16650 stack->reg = (jit->ctx.flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP;
16651 stack->ref = ir_get_spill_slot_offset(&jit->ctx, stack->ref);
16699 static int zend_jit_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint8_t type) argument
16708 ir_GUARD(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr));
16713 static int zend_jit_scalar_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var) argument
16722 ir_GUARD(ir_LT(jit_Z_TYPE(jit, addr), ir_CONST_U8(IS_STRING)), ir_CONST_ADDR(exit_addr));
16727 static bool zend_jit_noref_guard(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_addr) argument
16735 ir_GUARD(ir_NE(jit_Z_TYPE(jit, var_addr), ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
16740 static int zend_jit_trace_opline_guard(zend_jit_ctx *jit, const zend_op *opline) argument
16749 ir_GUARD(jit_CMP_IP(jit, IR_EQ, opline), ir_CONST_ADDR(exit_addr));
16750 zend_jit_set_last_valid_opline(jit, opline);
16755 static bool zend_jit_guard_reference(zend_jit_ctx *jit, argument
16773 ref = jit_Z_TYPE(jit, var_addr);
16777 ref = jit_Z_PTR(jit, var_addr);
16786 static bool zend_jit_fetch_reference(zend_jit_ctx *jit, argument
16809 ref = jit_Z_TYPE(jit, var_addr);
16815 jit_ZVAL_ADDR(jit, var_addr));
16818 ref = jit_Z_PTR(jit, var_addr);
16830 ref = jit_Z_TYPE(jit, var_addr);
16852 static bool zend_jit_fetch_indirect_var(zend_jit_ctx *jit, const zend_op *opline, uint8_t var_type,… argument
16867 jit_guard_Z_TYPE(jit, var_addr, IS_INDIRECT, exit_addr);
16868 ref = jit_Z_PTR(jit, var_addr);
16872 ref = jit_Z_PTR(jit, var_addr);
16891 jit_guard_Z_TYPE(jit, var_addr, var_type, exit_addr);
16909 static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_array, const zend_op *… argument
16918 zend_jit_set_ip(jit, opline);
16922 ref = jit_FP(jit);
16928 zend_jit_check_exception(jit);
16946 jit_STORE_FP(jit, ir_LOAD_A(addr));
16960 ir_GUARD(ir_NE(jit_IP(jit), ir_CONST_ADDR(zend_jit_halt_op)),
16961 jit_STUB_ADDR(jit, jit_stub_trace_halt));
16964 ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
16966 ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
16971 ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_halt));
17056 ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), ir_CONST_ADDR(exit_addr));
17060 zend_jit_set_last_valid_opline(jit, trace->opline);
17065 static int zend_jit_deoptimizer_start(zend_jit_ctx *jit, argument
17070 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
17072 jit->ctx.spill_base = ZREG_FP;
17074 jit->op_array = NULL;
17075 jit->ssa = NULL;
17076 jit->name = zend_string_copy(name);
17078 jit->ctx.flags |= IR_SKIP_PROLOGUE;
17083 static int zend_jit_trace_start(zend_jit_ctx *jit, argument
17091 zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
17093 jit->ctx.spill_base = ZREG_FP;
17095 jit->op_array = NULL;
17096 jit->current_op_array = op_array;
17097 jit->ssa = ssa;
17098 jit->name = zend_string_copy(name);
17103 jit_STORE_FP(jit, ref);
17104 jit->ctx.flags |= IR_FASTCALL_FUNC;
17109 jit->ctx.flags |= IR_SKIP_PROLOGUE;
17147 ir_set_op(&jit->ctx, ref, 3, EX_NUM_TO_VAR(i));
17165 static int zend_jit_trace_begin_loop(zend_jit_ctx *jit) argument
17170 static void zend_jit_trace_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi) argument
17176 ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
17177 ZEND_ASSERT(jit->ra[src_var].ref != IR_UNUSED && jit->ra[src_var].ref != IR_NULL);
17180 (jit->ssa->var_info[src_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE,
17181 zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)), IR_UNUSED);
17184 ZEND_ASSERT(jit->ra[src_var].ref == IR_NULL);
17185 jit->ra[src_var].flags |= ZREG_FORWARD;
17187 zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref);
17190 static int zend_jit_trace_end_loop(zend_jit_ctx *jit, int loop_ref, const void *timeout_exit_addr) argument
17193 zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
17195 ZEND_ASSERT(jit->ctx.ir_base[loop_ref].op2 == IR_UNUSED);
17200 static int zend_jit_trace_return(zend_jit_ctx *jit, bool original_handler, const zend_op *opline) argument
17204 ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
17206 ir_TAILCALL(IR_VOID, zend_jit_orig_opline_handler(jit));
17211 ir_ref addr = zend_jit_orig_opline_handler(jit);
17216 ref = ir_CALL_1(IR_I32, addr, jit_FP(jit));
17238 static int zend_jit_trace_link_to_root(zend_jit_ctx *jit, zend_jit_trace_info *t, const void *timeo… argument
17247 zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
17561 static ir_ref jit_frameless_observer(zend_jit_ctx *jit, const zend_op *opline) { argument
17566 …ir_ref if_unobserved = jit_observer_fcall_is_unobserved_start(jit, fbc, &observer_handler, IR_UNUS…
17569 ir_CALL_1(IR_VOID, ir_CONST_ADDR((size_t)zend_frameless_observed_call), jit_FP(jit));
17576 static void jit_frameless_icall0(zend_jit_ctx *jit, const zend_op *opline) argument
17578 jit_SET_EX_OPLINE(jit, opline);
17582 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
17583 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
17587 skip_observer = jit_frameless_observer(jit, opline);
17596 zend_jit_check_exception(jit);
17599 static void jit_frameless_icall1(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info) argument
17601 jit_SET_EX_OPLINE(jit, opline);
17611 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
17612 ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
17613 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
17615 op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
17621 op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
17626 skip_observer = jit_frameless_observer(jit, opline);
17635 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
17636 zend_jit_check_exception(jit);
17639 static void jit_frameless_icall2(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint3… argument
17641 jit_SET_EX_OPLINE(jit, opline);
17655 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
17656 ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
17657 ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
17658 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
17660 op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
17666 op2_ref = zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
17672 op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
17675 op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
17680 skip_observer = jit_frameless_observer(jit, opline);
17689 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
17692 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
17694 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
17695 zend_jit_check_exception(jit);
17698 static void jit_frameless_icall3(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint3… argument
17700 jit_SET_EX_OPLINE(jit, opline);
17719 ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
17720 ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
17721 ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
17722 ir_ref op3_ref = jit_ZVAL_ADDR(jit, op3_addr);
17723 jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
17725 op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
17731 op2_ref = zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
17737 op3_ref = zend_jit_zval_check_undef(jit, op3_ref, (opline+1)->op1.var, opline, 1);
17743 op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
17746 op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
17749 op3_ref = jit_ZVAL_DEREF_ref(jit, op3_ref);
17754 skip_observer = jit_frameless_observer(jit, opline);
17763 jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
17768 jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
17770 jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
17776 jit_set_Z_TYPE_INFO(jit, op2_addr, IS_UNDEF);
17778 jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
17779 zend_jit_check_exception(jit);
