Lines Matching refs:insn
61 #define IR_IS_FP_ZERO(insn) ((insn.type == IR_DOUBLE) ? (insn.val.u64 == 0) : (insn.val.u32…
1134 const ir_insn *insn;
1142 insn = &ctx->ir_base[ref];
1144 if (ctx->ir_base[insn->op1].op == IR_RLOAD) {
1152 if (IR_IS_CONST_REF(insn->op2) && insn->op1 != insn->op2) {
1153 n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints);
1154 } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) {
1206 insn = &ctx->ir_base[ref];
1207 if (IR_IS_CONST_REF(insn->op2) && insn->op1 != insn->op2) {
1208 constraints->tmp_regs[n] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
1214 insn = &ctx->ir_base[ref];
1216 if (IR_IS_CONST_REF(insn->op1)) {
1217 const ir_insn *val_insn = &ctx->ir_base[insn->op1];
1220 } else if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) {
1223 } else if (ir_rule(ctx, insn->op1) & IR_FUSED) {
1226 if (IR_IS_CONST_REF(insn->op2) && insn->op1 != insn->op2) {
1228 n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints);
1229 } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) {
1235 insn = &ctx->ir_base[ref];
1241 if (IR_IS_CONST_REF(insn->op1)) {
1242 const ir_insn *val_insn = &ctx->ir_base[insn->op1];
1249 insn = &ctx->ir_base[ref];
1250 if (IR_IS_CONST_REF(insn->op1)) {
1251 constraints->tmp_regs[0] = IR_TMP_REG(1, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
1256 insn = &ctx->ir_base[ref];
1257 if (!IR_IS_TYPE_INT(ctx->ir_base[insn->op1].type)) {
1263 insn = &ctx->ir_base[ref];
1264 if (IR_IS_TYPE_INT(insn->type)) {
1265 if (IR_IS_CONST_REF(insn->op3) || ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA) {
1267 constraints->tmp_regs[0] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF);
1269 } else if (IR_IS_CONST_REF(insn->op2) || ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) {
1271 constraints->tmp_regs[0] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_SAVE_SUB_REF);
1285 insn = &ctx->ir_base[ref];
1286 if (IR_IS_CONST_REF(insn->op3)) {
1287 n = ir_add_const_tmp_reg(ctx, insn->op3, 3, n, constraints);
1288 } else if (ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA) {
1295 insn = &ctx->ir_base[ref];
1296 if (IR_IS_CONST_REF(insn->op2)) {
1297 n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints);
1299 if (IR_IS_CONST_REF(insn->op3)) {
1300 n = ir_add_const_tmp_reg(ctx, insn->op3, 3, n, constraints);
1301 } else if (ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA) {
1308 insn = &ctx->ir_base[ref];
1309 if (IR_IS_CONST_REF(insn->op3)) {
1310 insn = &ctx->ir_base[insn->op3];
1311 constraints->tmp_regs[0] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
1328 insn = &ctx->ir_base[ref];
1329 if (IR_IS_CONST_REF(insn->op2)) {
1330 n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints);
1335 insn = &ctx->ir_base[ref];
1336 if (IR_IS_CONST_REF(insn->op2)) {
1337 n = ir_add_const_tmp_reg(ctx, insn->op2, 2, n, constraints);
1339 if (IR_IS_CONST_REF(insn->op3)) {
1340 insn = &ctx->ir_base[insn->op3];
1341 constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
1347 insn = &ctx->ir_base[ref];
1348 if (IR_IS_CONST_REF(insn->op2)) {
1349 insn = &ctx->ir_base[insn->op2];
1350 constraints->tmp_regs[0] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
1358 insn = &ctx->ir_base[ref];
1359 if (IR_IS_TYPE_INT(insn->type)) {
1370 insn = &ctx->ir_base[ref];
1371 if (insn->inputs_count > 2) {
1373 constraints->hints_count = ir_get_args_regs(ctx, insn, constraints->hints);
1374 if (!IR_IS_CONST_REF(insn->op2)) {
1395 insn = &ctx->ir_base[ref];
1396 if (ir_type_size[insn->type] == 8) {
1397 int64_t offset = ctx->ir_base[insn->op2].val.u64 - 1;
1399 constraints->tmp_regs[n] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
1406 insn = &ctx->ir_base[ref];
1407 if (ir_type_size[insn->type] == 8) {
1408 int64_t offset = ctx->ir_base[insn->op2].val.u64 - 1;
1410 constraints->tmp_regs[n] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
1414 constraints->tmp_regs[n] = IR_TMP_REG(3, insn->type, IR_USE_SUB_REF, IR_SAVE_SUB_REF);
1419 insn = &ctx->ir_base[ref];
1420 if (ir_type_size[insn->type] == 8) {
1421 int64_t offset = ctx->ir_base[insn->op2].val.u64 - 1;
1423 constraints->tmp_regs[n] = IR_TMP_REG(2, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
1429 insn = &ctx->ir_base[ref];
1430 if (ir_type_size[insn->type] == 1) {
1435 if (IR_IS_CONST_REF(insn->op1)) {
1436 constraints->tmp_regs[0] = IR_TMP_REG(1, insn->type, IR_LOAD_SUB_REF, IR_DEF_SUB_REF);
1442 insn = &ctx->ir_base[ref];
1443 constraints->tmp_regs[0] = IR_TMP_REG(2, insn->type, IR_USE_SUB_REF, IR_SAVE_SUB_REF);
1445 if (ir_type_size[insn->type] == 8) {
1446 constraints->tmp_regs[1] = IR_TMP_REG(3, insn->type, IR_USE_SUB_REF, IR_SAVE_SUB_REF);
1530 static void ir_swap_ops(ir_insn *insn)
1532 SWAP_REFS(insn->op1, insn->op2);
1537 ir_insn *insn = &ctx->ir_base[ref];
1540 if (insn->op1 == insn->op2) {
1542 } else if (ir_match_try_fuse_load(ctx, insn->op2, ref)) {
1545 } else if (ir_match_try_fuse_load(ctx, insn->op1, ref)) {
1547 ir_swap_ops(insn);
1577 ir_insn *insn = &ctx->ir_base[*p];
1578 if (insn->op != IR_LOAD && (insn->op != IR_STORE || insn->op3 == addr_ref)) {
1591 ir_insn *op2_insn, *insn = &ctx->ir_base[use];
1593 if (insn->op == IR_ADD) {
1594 if (insn->op1 == ref) {
1595 if (IR_IS_CONST_REF(insn->op2)) {
1596 op2_insn = &ctx->ir_base[insn->op2];
1604 } else if (insn->op2 != ref) {
1607 } else if (insn->op2 == ref && insn->op1 != insn->op2) {
1623 ir_insn *insn = &ctx->ir_base[ref];
1625 IR_ASSERT(IR_IS_TYPE_INT(insn->type) && ir_type_size[insn->type] >= 4);
1626 if (insn->op == IR_MUL
1627 && IR_IS_CONST_REF(insn->op2)) {
1628 insn = &ctx->ir_base[insn->op2];
1629 if (!IR_IS_SYM_CONST(insn->op)
1630 && (insn->val.u64 == 2 || insn->val.u64 == 4 || insn->val.u64 == 8)) {
1659 ir_insn *insn = &ctx->ir_base[pos];
1661 if (insn->op == IR_STORE) {
1664 } else if (insn->op == IR_CALL) {
1698 ir_insn *insn = &ctx->ir_base[ref];
1701 && insn->op == IR_LOAD) {
1718 } else if (insn->op == IR_PARAM) {
1729 static void ir_match_fuse_load_commutative_int(ir_ctx *ctx, ir_insn *insn, ir_ref root)
1731 if (IR_IS_CONST_REF(insn->op2)
1732 && ir_may_fuse_imm(ctx, &ctx->ir_base[insn->op2])) {
1734 } else if (ir_match_try_fuse_load(ctx, insn->op2, root)) {
1736 } else if (ir_match_try_fuse_load(ctx, insn->op1, root)) {
1737 ir_swap_ops(insn);
1741 static void ir_match_fuse_load_commutative_fp(ir_ctx *ctx, ir_insn *insn, ir_ref root)
1743 if (!IR_IS_CONST_REF(insn->op2)
1744 && !ir_match_try_fuse_load(ctx, insn->op2, root)
1745 && (IR_IS_CONST_REF(insn->op1) || ir_match_try_fuse_load(ctx, insn->op1, root))) {
1746 ir_swap_ops(insn);
1750 static void ir_match_fuse_load_cmp_int(ir_ctx *ctx, ir_insn *insn, ir_ref root)
1752 if (IR_IS_CONST_REF(insn->op2)
1753 && ir_may_fuse_imm(ctx, &ctx->ir_base[insn->op2])) {
1754 ir_match_fuse_load(ctx, insn->op1, root);
1755 } else if (!ir_match_try_fuse_load(ctx, insn->op2, root)
1756 && ir_match_try_fuse_load(ctx, insn->op1, root)) {
1757 ir_swap_ops(insn);
1758 if (insn->op != IR_EQ && insn->op != IR_NE) {
1759 insn->op ^= 3;
1764 static void ir_match_fuse_load_test_int(ir_ctx *ctx, ir_insn *insn, ir_ref root)
1766 if (IR_IS_CONST_REF(insn->op2)
1767 && ir_may_fuse_imm(ctx, &ctx->ir_base[insn->op2])) {
1768 ir_match_fuse_load(ctx, insn->op1, root);
1769 } else if (!ir_match_try_fuse_load(ctx, insn->op2, root)
1770 && ir_match_try_fuse_load(ctx, insn->op1, root)) {
1771 ir_swap_ops(insn);
1775 static void ir_match_fuse_load_cmp_fp(ir_ctx *ctx, ir_insn *insn, ir_ref root)
1777 if (insn->op != IR_EQ && insn->op != IR_NE) {
1778 if (insn->op == IR_LT || insn->op == IR_LE) {
1780 ir_swap_ops(insn);
1781 insn->op ^= 3;
1783 ir_match_fuse_load(ctx, insn->op2, root);
1784 } else if (IR_IS_CONST_REF(insn->op2) && !IR_IS_FP_ZERO(ctx->ir_base[insn->op2])) {
1786 } else if (ir_match_try_fuse_load(ctx, insn->op2, root)) {
1788 …} else if ((IR_IS_CONST_REF(insn->op1) && !IR_IS_FP_ZERO(ctx->ir_base[insn->op1])) || ir_match_try…
1789 ir_swap_ops(insn);
1790 if (insn->op != IR_EQ && insn->op != IR_NE) {
1791 insn->op ^= 3;
1796 static void ir_match_fuse_load_cmp_fp_br(ir_ctx *ctx, ir_insn *insn, ir_ref root, bool direct)
1799 if (insn->op == IR_LT || insn->op == IR_LE) {
1801 ir_swap_ops(insn);
1802 insn->op ^= 3;
1805 if (insn->op == IR_GT || insn->op == IR_GE) {
1807 ir_swap_ops(insn);
1808 insn->op ^= 3;
1811 if (IR_IS_CONST_REF(insn->op2) && !IR_IS_FP_ZERO(ctx->ir_base[insn->op2])) {
1813 } else if (ir_match_try_fuse_load(ctx, insn->op2, root)) {
1815 …} else if ((IR_IS_CONST_REF(insn->op1) && !IR_IS_FP_ZERO(ctx->ir_base[insn->op1])) || ir_match_try…
1816 ir_swap_ops(insn);
1817 if (insn->op != IR_EQ && insn->op != IR_NE) {
1818 insn->op ^= 3;
1894 ir_insn *insn = &ctx->ir_base[ref];
1898 switch (insn->op) {
1909 if (IR_IS_TYPE_INT(ctx->ir_base[insn->op1].type)) {
1910 if (IR_IS_CONST_REF(insn->op2)
1911 && !IR_IS_SYM_CONST(ctx->ir_base[insn->op2].op)
1912 && ctx->ir_base[insn->op2].val.i64 == 0
1913 && insn->op1 == ref - 1) { /* previous instruction */
1914 ir_insn *op1_insn = &ctx->ir_base[insn->op1];
1916 if (op1_insn->op == IR_AND && ctx->use_lists[insn->op1].count == 1) {
1919 ctx->rules[insn->op1] = IR_FUSED | IR_TEST_INT;
1924 (insn->op == IR_EQ || insn->op == IR_NE))) {
1928 ctx->rules[insn->op1] = IR_BINOP_INT | IR_MAY_SWAP;
1931 ctx->rules[insn->op1] = IR_BINOP_INT;
1936 ir_match_fuse_load_cmp_int(ctx, insn, ref);
1939 ir_match_fuse_load_cmp_fp(ctx, insn, ref);
1945 if (IR_IS_TYPE_INT(insn->type)) {
1946 if ((ctx->flags & IR_OPT_CODEGEN) && IR_IS_CONST_REF(insn->op2)) {
1947 op2_insn = &ctx->ir_base[insn->op2];
1948 if (IR_IS_CONST_REF(insn->op1)) {
1952 if (insn->op == IR_ADD && ir_may_fuse_addr(ctx, op2_insn)) {
1958 …} else if ((ir_type_size[insn->type] >= 4 && insn->op == IR_ADD && IR_IS_SIGNED_32BIT(op2_insn->va…
1959 …(ir_type_size[insn->type] >= 4 && insn->op == IR_SUB && IR_IS_SIGNED_NEG_32BIT(op2_insn->val.i64))…
1961 if (ctx->use_lists[insn->op1].count == 1 || ir_match_fuse_addr_all_useges(ctx, insn->op1)) {
1962 uint32_t rule = ctx->rules[insn->op1];
1965 ctx->rules[insn->op1] = rule = ir_match_insn(ctx, insn->op1);
1969 ctx->rules[insn->op1] = IR_FUSED | IR_SIMPLE | IR_LEA_SI;
1973 ctx->rules[insn->op1] = IR_FUSED | IR_SIMPLE | IR_LEA_SIB;
1977 ctx->rules[insn->op1] = IR_FUSED | IR_SIMPLE | IR_LEA_IB;
1984 if (insn->op == IR_ADD) {
2002 … } else if ((ctx->flags & IR_OPT_CODEGEN) && insn->op == IR_ADD && ir_type_size[insn->type] >= 4) {
2003 if (insn->op1 != insn->op2) {
2004 if (ctx->use_lists[insn->op1].count == 1 || ir_match_fuse_addr_all_useges(ctx, insn->op1)) {
2005 uint32_t rule =ctx->rules[insn->op1];
2007 ctx->rules[insn->op1] = rule = ir_match_insn(ctx, insn->op1);
2010 ctx->rules[insn->op1] = IR_FUSED | IR_SIMPLE | IR_LEA_OB;
2011 if (ctx->use_lists[insn->op2].count == 1 || ir_match_fuse_addr_all_useges(ctx, insn->op2)) {
2012 rule = ctx->rules[insn->op2];
2014 ctx->rules[insn->op2] = rule = ir_match_insn(ctx, insn->op2);
2018 ctx->rules[insn->op2] = IR_FUSED | IR_SIMPLE | IR_LEA_SI;
2025 ctx->rules[insn->op1] = IR_FUSED | IR_SIMPLE | IR_LEA_SI;
2026 if (ctx->use_lists[insn->op2].count == 1) {
2027 rule = ctx->rules[insn->op2];
2029 ctx->rules[insn->op2] = rule = ir_match_insn(ctx, insn->op2);
2033 ctx->rules[insn->op2] = IR_FUSED | IR_SIMPLE | IR_LEA_OB;
2041 if (ctx->use_lists[insn->op2].count == 1 || ir_match_fuse_addr_all_useges(ctx, insn->op2)) {
2042 uint32_t rule = ctx->rules[insn->op2];
2044 ctx->rules[insn->op2] = rule = ir_match_insn(ctx, insn->op2);
2047 ctx->rules[insn->op2] = IR_FUSED | IR_SIMPLE | IR_LEA_OB;
2051 ctx->rules[insn->op2] = IR_FUSED | IR_SIMPLE | IR_LEA_SI;
2061 if (ir_op_flags[insn->op] & IR_OP_FLAG_COMMUTATIVE) {
2062 ir_match_fuse_load_commutative_int(ctx, insn, ref);
2065 ir_match_fuse_load(ctx, insn->op2, ref);
2070 if (ir_op_flags[insn->op] & IR_OP_FLAG_COMMUTATIVE) {
2071 ir_match_fuse_load_commutative_fp(ctx, insn, ref);
2078 ir_match_fuse_load(ctx, insn->op2, ref);
2088 if (IR_IS_TYPE_INT(insn->type)) {
2089 if ((ctx->flags & IR_OPT_CODEGEN) && IR_IS_CONST_REF(insn->op2)) {
2090 op2_insn = &ctx->ir_base[insn->op2];
2093 } else if (IR_IS_CONST_REF(insn->op1)) {
2099 } else if (ir_type_size[insn->type] >= 4 &&
2103 } else if (ir_type_size[insn->type] >= 4 &&
2110 } else if (IR_IS_TYPE_SIGNED(insn->type)
2111 && ir_type_size[insn->type] != 1
2113 && !IR_IS_CONST_REF(insn->op1)) {
2115 ir_match_fuse_load(ctx, insn->op1, ref);
2120 // if (IR_IS_TYPE_SIGNED(insn->type) && ir_type_size[insn->type] != 1) {
2121 if (ir_type_size[insn->type] != 1) {
2124 ir_match_fuse_load(ctx, insn->op2, ref);
2132 IR_ASSERT(IR_IS_TYPE_INT(insn->type));
2135 IR_ASSERT(IR_IS_TYPE_INT(insn->type));
2136 if (IR_IS_TYPE_SIGNED(insn->type) && ir_type_size[insn->type] != 1) {
2137 if ((ctx->flags & IR_OPT_CODEGEN) && IR_IS_CONST_REF(insn->op2)) {
2138 op2_insn = &ctx->ir_base[insn->op2];
2141 && !IR_IS_CONST_REF(insn->op1)) {
2143 ir_match_fuse_load(ctx, insn->op1, ref);
2149 ir_match_fuse_load(ctx, insn->op2, ref);
2152 if (IR_IS_TYPE_INT(insn->type)) {
2153 if ((ctx->flags & IR_OPT_CODEGEN) && IR_IS_CONST_REF(insn->op2)) {
2154 op2_insn = &ctx->ir_base[insn->op2];
2157 } else if (IR_IS_CONST_REF(insn->op1)) {
2163 if (IR_IS_TYPE_UNSIGNED(insn->type)) {
2170 ir_match_fuse_load(ctx, insn->op2, ref);
2177 if ((ctx->flags & IR_OPT_CODEGEN) && IR_IS_CONST_REF(insn->op2)) {
2178 op2_insn = &ctx->ir_base[insn->op2];
2181 } else if (IR_IS_CONST_REF(insn->op1)) {
2185 if (IR_IS_TYPE_UNSIGNED(insn->type)) {
2192 ir_match_fuse_load(ctx, insn->op2, ref);
2196 if (insn->type == IR_BOOL) {
2197 IR_ASSERT(IR_IS_TYPE_INT(ctx->ir_base[insn->op1].type)); // TODO: IR_BOOL_NOT_FP
2200 IR_ASSERT(IR_IS_TYPE_INT(insn->type));
2205 if (IR_IS_TYPE_INT(insn->type)) {
2211 if (IR_IS_TYPE_INT(insn->type)) {
2217 if ((ctx->flags & IR_OPT_CODEGEN) && IR_IS_CONST_REF(insn->op2)) {
2218 op2_insn = &ctx->ir_base[insn->op2];
2221 } else if (IR_IS_CONST_REF(insn->op1)) {
2231 if ((ctx->flags & IR_OPT_CODEGEN) && IR_IS_CONST_REF(insn->op2)) {
2232 op2_insn = &ctx->ir_base[insn->op2];
2235 } else if (IR_IS_CONST_REF(insn->op1)) {
2245 if ((ctx->flags & IR_OPT_CODEGEN) && IR_IS_CONST_REF(insn->op2)) {
2246 op2_insn = &ctx->ir_base[insn->op2];
2249 } else if (IR_IS_CONST_REF(insn->op1)) {
2255 if (IR_IS_CONST_REF(insn->op2)) {
2257 op2_insn = &ctx->ir_base[insn->op2];
2260 } else if (IR_IS_CONST_REF(insn->op1)) {
2264 } else if (ir_type_size[insn->type] >= 4) {
2281 if (IR_IS_CONST_REF(insn->op2)) {
2283 op2_insn = &ctx->ir_base[insn->op2];
2286 } else if (IR_IS_CONST_REF(insn->op1)) {
2297 if (IR_IS_TYPE_INT(insn->type)) {
2304 if (IR_IS_TYPE_INT(insn->type)) {
2311 if (IR_IS_CONST_REF(insn->op2)) {
2312 const ir_insn *func = &ctx->ir_base[insn->op2];
2324 if (IR_IS_TYPE_FP(insn->type)) {
2331 ir_match_fuse_load(ctx, insn->op2, ref);
2332 return insn->op;
2340 if (IR_IS_CONST_REF(insn->op2) && ctx->cfg_map[ref] == 1) {
2341 ir_insn *val = &ctx->ir_base[insn->op2];
2352 if (IR_IS_TYPE_INT(ctx->ir_base[insn->op3].type)) {
2357 && ir_in_same_block(ctx, insn->op3)
2358 && (ctx->use_lists[insn->op3].count == 1 ||
2359 (ctx->use_lists[insn->op3].count == 2
2360 && (ctx->ir_base[insn->op3].op == IR_ADD_OV ||
2361 ctx->ir_base[insn->op3].op == IR_SUB_OV)))) {
2362 ir_insn *op_insn = &ctx->ir_base[insn->op3];
2363 uint32_t rule = ctx->rules[insn->op3];
2366 ctx->rules[insn->op3] = rule = ir_match_insn(ctx, insn->op3);
2369 if (insn->op1 == op_insn->op1
2371 && ctx->ir_base[op_insn->op1].op2 == insn->op2
2374 ctx->rules[insn->op3] = IR_FUSED | IR_BINOP_INT;
2382 && insn->op1 == op_insn->op2
2384 && ctx->ir_base[op_insn->op2].op2 == insn->op2
2388 ctx->rules[insn->op3] = IR_FUSED | IR_BINOP_INT;
2393 if (insn->op1 == op_insn->op1
2395 && ctx->ir_base[op_insn->op1].op2 == insn->op2
2398 ctx->rules[insn->op3] = IR_SKIPPED | IR_INC;
2403 if (insn->op1 == op_insn->op1
2405 && ctx->ir_base[op_insn->op1].op2 == insn->op2
2408 ctx->rules[insn->op3] = IR_SKIPPED | IR_DEC;
2413 if (insn->op1 == op_insn->op1
2415 && ctx->ir_base[op_insn->op1].op2 == insn->op2
2418 ctx->rules[insn->op3] = IR_SKIPPED | IR_MUL_PWR2;
2423 if (insn->op1 == op_insn->op1
2425 && ctx->ir_base[op_insn->op1].op2 == insn->op2
2428 ctx->rules[insn->op3] = IR_SKIPPED | IR_DIV_PWR2;
2433 if (insn->op1 == op_insn->op1
2435 && ctx->ir_base[op_insn->op1].op2 == insn->op2
2438 ctx->rules[insn->op3] = IR_SKIPPED | IR_MOD_PWR2;
2443 if (insn->op1 == op_insn->op1
2445 && ctx->ir_base[op_insn->op1].op2 == insn->op2
2448 ctx->rules[insn->op3] = IR_FUSED | IR_SHIFT;
2453 if (insn->op1 == op_insn->op1
2455 && ctx->ir_base[op_insn->op1].op2 == insn->op2
2458 ctx->rules[insn->op3] = IR_SKIPPED | IR_SHIFT_CONST;
2463 if (insn->op1 == op_insn->op1
2465 && ctx->ir_base[op_insn->op1].op2 == insn->op2
2468 ctx->rules[insn->op3] = IR_SKIPPED | IR_OP_INT;
2474 ctx->rules[insn->op3] = IR_FUSED | IR_CMP_INT;
2484 ir_match_fuse_addr(ctx, insn->op2);
2485 if (IR_IS_TYPE_INT(insn->type)) {
2492 ir_match_fuse_addr(ctx, insn->op2);
2493 if (IR_IS_TYPE_INT(ctx->ir_base[insn->op3].type)) {
2502 if (IR_REGSET_IN(IR_REGSET_UNION((ir_regset)ctx->fixed_regset, IR_REGSET_FIXED), insn->op2)) {
2507 if (IR_IS_TYPE_INT(ctx->ir_base[insn->op2].type)) {
2509 && ir_in_same_block(ctx, insn->op2)
2510 && ctx->use_lists[insn->op2].count == 1
2511 && IR_IS_TYPE_INT(ctx->ir_base[insn->op2].type)) {
2512 ir_insn *op_insn = &ctx->ir_base[insn->op2];
2520 if (insn->op1 == op_insn->op1
2522 && ctx->ir_base[op_insn->op1].op2 == insn->op3
2525 ctx->rules[insn->op2] = IR_FUSED | IR_BINOP_INT;
2529 && insn->op1 == op_insn->op2
2531 && ctx->ir_base[op_insn->op2].op2 == insn->op3
2535 ctx->rules[insn->op2] = IR_FUSED | IR_BINOP_INT;
2542 ir_match_fuse_load(ctx, insn->op2, ref);
2553 return IR_SKIPPED | insn->op;
2555 if (!insn->op2) {
2557 } else if (IR_IS_TYPE_INT(ctx->ir_base[insn->op2].type)) {
2563 if (!IR_IS_CONST_REF(insn->op2) && ctx->use_lists[insn->op2].count == 1) {
2564 op2_insn = &ctx->ir_base[insn->op2];
2570 && op2_insn->op1 == insn->op2 - 1) { /* previous instruction */
2577 ctx->rules[insn->op2] = IR_FUSED | IR_SIMPLE | IR_NOP;
2579 } else if (insn->op2 == ref - 1 && /* previous instruction */
2592 ctx->rules[insn->op2] = IR_FUSED | IR_CMP_INT;
2598 ctx->rules[insn->op2] = IR_FUSED | IR_CMP_INT;
2603 ctx->rules[insn->op2] = IR_FUSED | IR_CMP_FP;
2609 ctx->rules[insn->op2] = IR_FUSED | IR_TEST_INT;
2611 } else if (op2_insn->op == IR_OVERFLOW && ir_in_same_block(ctx, insn->op2)) {
2613 ctx->rules[insn->op2] = IR_FUSED | IR_SIMPLE | IR_OVERFLOW;
2617 if (IR_IS_TYPE_INT(ctx->ir_base[insn->op2].type)) {
2618 if (insn->op2 == ref - 1) { /* previous instruction */
2619 op2_insn = &ctx->ir_base[insn->op2];
2630 ctx->rules[insn->op2] = IR_BINOP_INT | IR_MAY_SWAP;
2633 ctx->rules[insn->op2] = IR_BINOP_INT;
2638 && insn->op1 == ref - 1 /* previous instruction */
2639 && insn->op2 == ref - 2 /* previous instruction */
2640 && ctx->use_lists[insn->op2].count == 2
2641 && IR_IS_TYPE_INT(ctx->ir_base[insn->op2].type)) {
2642 ir_insn *store_insn = &ctx->ir_base[insn->op1];
2644 if (store_insn->op == IR_STORE && store_insn->op3 == insn->op2) {
2645 ir_insn *op_insn = &ctx->ir_base[insn->op2];
2659 ctx->rules[insn->op2] = IR_FUSED | IR_BINOP_INT;
2662 ctx->rules[insn->op1] = IR_MEM_BINOP_INT;
2673 ctx->rules[insn->op2] = IR_FUSED | IR_BINOP_INT;
2676 ctx->rules[insn->op1] = IR_MEM_BINOP_INT;
2683 ir_match_fuse_load(ctx, insn->op2, ref);
2690 if (!IR_IS_CONST_REF(insn->op1) && ctx->use_lists[insn->op1].count == 1) {
2691 ir_insn *op1_insn = &ctx->ir_base[insn->op1];
2696 ctx->rules[insn->op1] = IR_FUSED | IR_CMP_INT;
2700 ctx->rules[insn->op1] = IR_FUSED | IR_CMP_FP;
2708 if (!IR_IS_CONST_REF(insn->op2) && ctx->use_lists[insn->op2].count == 1) {
2709 op2_insn = &ctx->ir_base[insn->op2];
2712 && (insn->op2 == ref - 1 ||
2713 (insn->op2 == ctx->prev_ref[ref] - 1
2719 if (op2_insn->op1 == insn->op2 - 1) { /* previous instruction */
2734 ctx->rules[insn->op2] = IR_FUSED | IR_CMP_INT;
2738 && op2_insn->op1 == insn->op2 - 2 /* before previous instruction */
2741 ir_insn *store_insn = &ctx->ir_base[insn->op2 - 1];
2759 ctx->rules[insn->op2 - 1] = IR_MEM_BINOP_INT;
2760 ctx->rules[insn->op2] = IR_SKIPPED | IR_NOP;
2774 ctx->rules[insn->op2 - 1] = IR_MEM_BINOP_INT;
2775 ctx->rules[insn->op2] = IR_SKIPPED | IR_NOP;
2785 ctx->rules[insn->op2] = IR_FUSED | IR_CMP_INT;
2789 ir_match_fuse_load_cmp_fp_br(ctx, op2_insn, ref, insn->op == IR_GUARD_NOT);
2790 ctx->rules[insn->op2] = IR_FUSED | IR_CMP_FP;
2796 ctx->rules[insn->op2] = IR_FUSED | IR_TEST_INT;
2798 } else if (op2_insn->op == IR_OVERFLOW && ir_in_same_block(ctx, insn->op2)) {
2800 ctx->rules[insn->op2] = IR_FUSED | IR_SIMPLE | IR_OVERFLOW;
2804 ir_match_fuse_load(ctx, insn->op2, ref);
2805 return insn->op;
2807 …if (ir_type_size[ctx->ir_base[insn->op1].type] > (IR_IS_TYPE_SIGNED(ctx->ir_base[insn->op1].type) …
2808 ir_match_fuse_load(ctx, insn->op1, ref);
2810 return insn->op;
2815 ir_match_fuse_load(ctx, insn->op1, ref);
2816 return insn->op;
2819 ir_match_fuse_load(ctx, insn->op1, ref);
2820 return insn->op | IR_MAY_REUSE;
2822 ir_match_fuse_load(ctx, insn->op1, ref);
2823 if (IR_IS_TYPE_INT(insn->type) && IR_IS_TYPE_INT(ctx->ir_base[insn->op1].type)) {
2824 return insn->op | IR_MAY_REUSE;
2826 return insn->op;
2830 ir_match_fuse_load(ctx, insn->op1, ref);
2833 ir_match_fuse_load(ctx, insn->op1, ref);
2837 if ((ctx->ir_base[insn->op2].op == IR_ALLOCA) || (ctx->ir_base[insn->op2].op == IR_VADDR)) {
2838 ir_use_list *use_list = &ctx->use_lists[insn->op2];
2844 if (use_insn->op3 == insn->op2) {
2848 if (use_insn->op2 == insn->op2) {
2881 return insn->op;
3014 ir_insn *insn = &ctx->ir_base[src];
3017 if (type == IR_FLOAT && insn->val.u32 == 0) {
3023 } else if (type == IR_DOUBLE && insn->val.u64 == 0) {
3071 ir_insn *insn = &ctx->ir_base[src];
3073 if (insn->op == IR_SYM || insn->op == IR_FUNC) {
3074 void *addr = ir_sym_val(ctx, insn);
3076 } else if (insn->op == IR_STR) {
3083 ir_emit_load_imm_int(ctx, type, reg, insn->val.i64);
3258 ir_insn *insn = &ctx->ir_base[ref];
3271 offset_insn = insn;
3272 if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) {
3273 offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op1].op3);
3283 scale = ctx->ir_base[insn->op2].val.i32;
3290 scale = ctx->ir_base[insn->op2].val.i32 - 1;
3294 if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) {
3295 offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op1].op3);
3299 } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) {
3300 offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3);
3312 op1_insn = &ctx->ir_base[insn->op1];
3315 if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) {
3316 offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3);
3319 index_reg_ref = insn->op1 * sizeof(ir_ref) + 1;
3326 base_reg_ref = insn->op1 * sizeof(ir_ref) + 1;
3331 op2_insn = &ctx->ir_base[insn->op2];
3334 if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) {
3335 offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op1].op3);
3338 index_reg_ref = insn->op2 * sizeof(ir_ref) + 1;
3346 index_reg_ref = insn->op2 * sizeof(ir_ref) + 1;
3350 index_reg_ref = insn->op1 * sizeof(ir_ref) + 1;
3351 op1_insn = &ctx->ir_base[insn->op1];
3353 offset_insn = insn;
3357 base_reg_ref = index_reg_ref = insn->op1 * sizeof(ir_ref) + 1;
3358 op1_insn = &ctx->ir_base[insn->op1];
3360 offset_insn = insn;
3363 base_reg_ref = insn->op1 * sizeof(ir_ref) + 1;
3364 index_reg_ref = insn->op1 * sizeof(ir_ref) + 2;
3365 offset_insn = insn;
3369 index_reg_ref = insn->op2 * sizeof(ir_ref) + 1;
3370 op1_insn = &ctx->ir_base[insn->op1];
3372 op2_insn = &ctx->ir_base[insn->op2];
3379 base_reg_ref = insn->op1 * sizeof(ir_ref) + 1;
3383 index_reg_ref = insn->op1 * sizeof(ir_ref) + 1;
3384 op1_insn = &ctx->ir_base[insn->op1];
3386 op2_insn = &ctx->ir_base[insn->op2];
3393 base_reg_ref = insn->op2 * sizeof(ir_ref) + 1;
3397 if (ir_rule(ctx, insn->op1) == IR_STATIC_ALLOCA) {
3398 offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op1].op3);
3404 index_reg_ref = insn->op2 * sizeof(ir_ref) + 1;
3405 op2_insn = &ctx->ir_base[insn->op2];
3410 index_reg_ref = insn->op1 * sizeof(ir_ref) + 1;
3411 if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) {
3412 offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3);
3418 op1_insn = &ctx->ir_base[insn->op1];
3423 offset = IR_SPILL_POS_TO_OFFSET(insn->op3);
3456 ir_emit_load(ctx, insn->type, base_reg, ((ir_ref*)ctx->ir_base)[base_reg_ref]);
3475 ir_emit_load(ctx, insn->type, index_reg, ((ir_ref*)ctx->ir_base)[index_reg_ref]);
3531 ir_insn *insn = &ctx->ir_base[src];
3533 if (insn->op == IR_SYM || insn->op == IR_FUNC) {
3534 void *addr = ir_sym_val(ctx, insn);
3536 } else if (insn->op == IR_STR) {
3543 ir_emit_load_imm_int(ctx, type, reg, insn->val.i64);
3748 static void ir_emit_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
3752 ir_type type = insn->type;
3753 ir_ref op1 = insn->op1;
3754 ir_ref op2 = insn->op2;
3783 switch (insn->op) {
3811 switch (insn->op) {
3844 switch (insn->op) {
3875 static void ir_emit_imul3(ir_ctx *ctx, ir_ref def, ir_insn *insn)
3879 ir_type type = insn->type;
3880 ir_ref op1 = insn->op1;
3881 ir_ref op2 = insn->op2;
3924 static void ir_emit_min_max_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
3928 ir_type type = insn->type;
3929 ir_ref op1 = insn->op1;
3930 ir_ref op2 = insn->op2;
3961 if (insn->op == IR_MIN) {
3968 IR_ASSERT(insn->op == IR_MAX);
3981 static void ir_emit_overflow(ir_ctx *ctx, ir_ref def, ir_insn *insn)
3986 ir_type type = ctx->ir_base[insn->op1].type;
3996 ir_emit_store(ctx, insn->type, def, def_reg);
4000 static void ir_emit_overflow_and_branch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_…
4004 ir_insn *overflow_insn = &ctx->ir_base[insn->op2];
4036 static void ir_emit_mem_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4040 ir_insn *op_insn = &ctx->ir_base[insn->op3];
4043 ir_reg op2_reg = ctx->regs[insn->op3][2];
4046 if (insn->op == IR_STORE) {
4047 mem = ir_fuse_mem(ctx, def, def, insn, ctx->regs[def][2]);
4049 IR_ASSERT(insn->op == IR_VSTORE);
4050 mem = ir_var_spill_slot(ctx, insn->op2);
4106 static void ir_emit_reg_binop_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4110 ir_insn *op_insn = &ctx->ir_base[insn->op2];
4113 ir_reg op2_reg = ctx->regs[insn->op2][2];
4116 IR_ASSERT(insn->op == IR_RSTORE);
4117 reg = insn->op3;
4168 static void ir_emit_mul_div_mod_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4172 ir_type type = insn->type;
4173 ir_ref op1 = insn->op1;
4177 IR_ASSERT(IR_IS_CONST_REF(insn->op2));
4178 IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[insn->op2].op));
4192 if (insn->op == IR_MUL) {
4193 uint32_t shift = IR_LOG2(ctx->ir_base[insn->op2].val.u64);
4200 } else if (insn->op == IR_DIV) {
4201 uint32_t shift = IR_LOG2(ctx->ir_base[insn->op2].val.u64);
4205 IR_ASSERT(insn->op == IR_MOD);
4206 uint64_t mask = ctx->ir_base[insn->op2].val.u64 - 1;
4227 static void ir_emit_sdiv_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4231 ir_type type = insn->type;
4232 ir_ref op1 = insn->op1;
4235 uint32_t shift = IR_LOG2(ctx->ir_base[insn->op2].val.u64);
4236 int64_t offset = ctx->ir_base[insn->op2].val.u64 - 1;
4239 IR_ASSERT(IR_IS_CONST_REF(insn->op2));
4240 IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[insn->op2].op));
4290 static void ir_emit_smod_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4294 ir_type type = insn->type;
4295 ir_ref op1 = insn->op1;
4299 uint32_t shift = IR_LOG2(ctx->ir_base[insn->op2].val.u64);
4300 uint64_t mask = ctx->ir_base[insn->op2].val.u64 - 1;
4303 IR_ASSERT(IR_IS_CONST_REF(insn->op2));
4304 IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[insn->op2].op));
4352 static void ir_emit_mem_mul_div_mod_pwr2(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4356 ir_insn *op_insn = &ctx->ir_base[insn->op3];
4363 if (insn->op == IR_STORE) {
4364 mem = ir_fuse_mem(ctx, def, def, insn, ctx->regs[def][2]);
4366 IR_ASSERT(insn->op == IR_VSTORE);
4367 mem = ir_var_spill_slot(ctx, insn->op2);
4384 static void ir_emit_shift(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4388 ir_type type = insn->type;
4396 ir_emit_load(ctx, type, op1_reg, insn->op1);
4400 ir_emit_load(ctx, type, op2_reg, insn->op2);
4410 ir_emit_load(ctx, type, IR_REG_RCX, insn->op2);
4417 ir_emit_load(ctx, type, def_reg, insn->op1);
4420 switch (insn->op) {
4424 | ASM_REG_TXT_OP shl, insn->type, def_reg, cl
4427 | ASM_REG_TXT_OP shr, insn->type, def_reg, cl
4430 | ASM_REG_TXT_OP sar, insn->type, def_reg, cl
4433 | ASM_REG_TXT_OP rol, insn->type, def_reg, cl
4436 | ASM_REG_TXT_OP ror, insn->type, def_reg, cl
4444 static void ir_emit_mem_shift(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4448 ir_insn *op_insn = &ctx->ir_base[insn->op3];
4451 ir_reg op2_reg = ctx->regs[insn->op3][2];
4454 if (insn->op == IR_STORE) {
4455 mem = ir_fuse_mem(ctx, def, def, insn, ctx->regs[def][2]);
4457 IR_ASSERT(insn->op == IR_VSTORE);
4458 mem = ir_var_spill_slot(ctx, insn->op2);
4493 static void ir_emit_shift_const(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4498 ir_type type = insn->type;
4499 ir_ref op1 = insn->op1;
4503 IR_ASSERT(!IR_IS_SYM_CONST(ctx->ir_base[insn->op2].op));
4504 IR_ASSERT(IR_IS_SIGNED_32BIT(ctx->ir_base[insn->op2].val.i64));
4505 shift = ctx->ir_base[insn->op2].val.i32;
4519 switch (insn->op) {
4523 | ASM_REG_IMM_OP shl, insn->type, def_reg, shift
4526 | ASM_REG_IMM_OP shr, insn->type, def_reg, shift
4529 | ASM_REG_IMM_OP sar, insn->type, def_reg, shift
4532 | ASM_REG_IMM_OP rol, insn->type, def_reg, shift
4535 | ASM_REG_IMM_OP ror, insn->type, def_reg, shift
4543 static void ir_emit_mem_shift_const(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4547 ir_insn *op_insn = &ctx->ir_base[insn->op3];
4556 if (insn->op == IR_STORE) {
4557 mem = ir_fuse_mem(ctx, def, def, insn, ctx->regs[def][2]);
4559 IR_ASSERT(insn->op == IR_VSTORE);
4560 mem = ir_var_spill_slot(ctx, insn->op2);
4584 static void ir_emit_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn, uint32_t rule)
4588 ir_type type = insn->type;
4589 ir_ref op1 = insn->op1;
4607 | ASM_REG_OP inc, insn->type, def_reg
4609 | ASM_REG_OP dec, insn->type, def_reg
4610 } else if (insn->op == IR_NOT) {
4611 | ASM_REG_OP not, insn->type, def_reg
4612 } else if (insn->op == IR_NEG) {
4613 | ASM_REG_OP neg, insn->type, def_reg
4615 IR_ASSERT(insn->op == IR_BSWAP);
4616 switch (ir_type_size[insn->type]) {
4635 static void ir_emit_bit_count(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4639 ir_type type = insn->type;
4640 ir_ref op1 = insn->op1;
4651 switch (ir_type_size[insn->type]) {
4655 if (insn->op == IR_CTLZ) {
4662 } else if (insn->op == IR_CTTZ) {
4669 IR_ASSERT(insn->op == IR_CTPOP);
4675 if (insn->op == IR_CTLZ) {
4687 if (insn->op == IR_CTLZ) {
4694 } else if (insn->op == IR_CTTZ) {
4701 IR_ASSERT(insn->op == IR_CTPOP);
4707 if (insn->op == IR_CTLZ) {
4714 } else if (insn->op == IR_CTTZ) {
4721 IR_ASSERT(insn->op == IR_CTPOP);
4735 switch (ir_type_size[insn->type]) {
4739 if (insn->op == IR_CTLZ) {
4746 } else if (insn->op == IR_CTTZ) {
4757 if (insn->op == IR_CTLZ) {
4764 } else if (insn->op == IR_CTTZ) {
4776 if (insn->op == IR_CTLZ) {
4783 } else if (insn->op == IR_CTTZ) {
4802 static void ir_emit_ctpop(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4806 ir_type type = insn->type;
4807 ir_ref op1 = insn->op1;
4818 if (ir_type_size[insn->type] == 1) {
4820 } else if (ir_type_size[insn->type] == 2) {
4828 switch (ir_type_size[insn->type]) {
4847 switch (ir_type_size[insn->type]) {
4932 static void ir_emit_mem_op_int(ir_ctx *ctx, ir_ref def, ir_insn *insn, uint32_t rule)
4936 ir_insn *op_insn = &ctx->ir_base[insn->op3];
4940 if (insn->op == IR_STORE) {
4941 mem = ir_fuse_mem(ctx, def, def, insn, ctx->regs[def][2]);
4943 IR_ASSERT(insn->op == IR_VSTORE);
4944 mem = ir_var_spill_slot(ctx, insn->op2);
4959 static void ir_emit_abs_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4963 ir_type type = insn->type;
4964 ir_ref op1 = insn->op1;
4977 ir_emit_mov(ctx, insn->type, def_reg, op1_reg);
4978 | ASM_REG_OP neg, insn->type, def_reg
4985 static void ir_emit_bool_not_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
4989 ir_type type = ctx->ir_base[insn->op1].type;
4990 ir_ref op1 = insn->op1;
5015 static void ir_emit_mul_div_mod(ir_ctx *ctx, ir_ref def, ir_insn *insn)
5019 ir_type type = insn->type;
5020 ir_ref op1 = insn->op1;
5021 ir_ref op2 = insn->op2;
5046 && (insn->op == IR_MUL || insn->op == IR_MUL_OV)) {
5050 if (insn->op == IR_MUL || insn->op == IR_MUL_OV) {
5051 if (IR_IS_TYPE_SIGNED(insn->type)) {
5114 if (insn->op == IR_MUL || insn->op == IR_MUL_OV || insn->op == IR_DIV) {
5126 IR_ASSERT(insn->op == IR_MOD);
5170 static void ir_emit_op_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
5174 ir_type type = insn->type;
5175 ir_ref op1 = insn->op1;
5192 if (insn->op == IR_NEG) {
5193 if (insn->type == IR_DOUBLE) {
5208 IR_ASSERT(insn->type == IR_FLOAT);
5224 IR_ASSERT(insn->op == IR_ABS);
5225 if (insn->type == IR_DOUBLE) {
5240 IR_ASSERT(insn->type == IR_FLOAT);
5257 ir_emit_store(ctx, insn->type, def, def_reg);
5261 static void ir_emit_binop_sse2(ir_ctx *ctx, ir_ref def, ir_insn *insn)
5265 ir_type type = insn->type;
5266 ir_ref op1 = insn->op1;
5267 ir_ref op2 = insn->op2;
5295 switch (insn->op) {
5320 switch (insn->op) {
5350 switch (insn->op) {
5374 ir_emit_store(ctx, insn->type, def, def_reg);
5378 static void ir_emit_binop_avx(ir_ctx *ctx, ir_ref def, ir_insn *insn)
5382 ir_type type = insn->type;
5383 ir_ref op1 = insn->op1;
5384 ir_ref op2 = insn->op2;
5402 switch (insn->op) {
5427 switch (insn->op) {
5457 switch (insn->op) {
5481 ir_emit_store(ctx, insn->type, def, def_reg);
5485 static void ir_emit_cmp_int_common(ir_ctx *ctx, ir_type type, ir_ref root, ir_insn *insn, ir_reg op…
5632 static void ir_emit_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
5636 ir_type type = ctx->ir_base[insn->op1].type;
5637 ir_op op = insn->op;
5638 ir_ref op1 = insn->op1;
5639 ir_ref op2 = insn->op2;
5660 ir_emit_store(ctx, insn->type, def, def_reg);
5665 | ASM_REG_IMM_OP mov, insn->type, def_reg, 1
5667 ir_emit_store(ctx, insn->type, def, def_reg);
5676 ir_emit_cmp_int_common(ctx, type, def, insn, op1_reg, op1, op2_reg, op2);
5679 ir_emit_store(ctx, insn->type, def, def_reg);
5768 static void ir_emit_testcc_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
5773 ir_emit_test_int_common(ctx, def, insn->op1, insn->op);
5774 _ir_emit_setcc_int(ctx, insn->op, def_reg);
5776 ir_emit_store(ctx, insn->type, def, def_reg);
5780 static void ir_emit_setcc_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
5785 _ir_emit_setcc_int(ctx, insn->op, def_reg);
5787 ir_emit_store(ctx, insn->type, def, def_reg);
5845 static void ir_emit_cmp_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
5849 ir_op op = ir_emit_cmp_fp_common(ctx, def, def, insn);
5901 ir_emit_store(ctx, insn->type, def, def_reg);
5929 static void ir_emit_jcc(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_t next_block, ui…
6049 static void ir_emit_cmp_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_t…
6051 ir_insn *cmp_insn = &ctx->ir_base[insn->op2];
6056 ir_reg op1_reg = ctx->regs[insn->op2][1];
6057 ir_reg op2_reg = ctx->regs[insn->op2][2];
6086 ir_insn *prev_insn = &ctx->ir_base[insn->op1];
6099 ir_emit_jcc(ctx, b, def, insn, next_block, op, 1);
6102 static void ir_emit_test_and_branch_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_…
6104 ir_ref op2 = insn->op2;
6115 ir_emit_jcc(ctx, b, def, insn, next_block, op, 1);
6118 static void ir_emit_cmp_and_branch_fp(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_t …
6120 ir_op op = ir_emit_cmp_fp_common(ctx, def, insn->op2, &ctx->ir_base[insn->op2]);
6121 ir_emit_jcc(ctx, b, def, insn, next_block, op, 0);
6124 static void ir_emit_if_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_t next_block)
6126 ir_type type = ctx->ir_base[insn->op2].type;
6134 ir_emit_load(ctx, type, op2_reg, insn->op2);
6137 } else if (IR_IS_CONST_REF(insn->op2)) {
6141 if (ir_const_is_true(&ctx->ir_base[insn->op2])) {
6151 } else if (ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA) {
6162 if (ir_rule(ctx, insn->op2) & IR_FUSED) {
6163 mem = ir_fuse_load(ctx, def, insn->op2);
6165 mem = ir_ref_spill_slot(ctx, insn->op2);
6169 ir_emit_jcc(ctx, b, def, insn, next_block, IR_NE, 1);
6172 static void ir_emit_cond(ir_ctx *ctx, ir_ref def, ir_insn *insn)
6176 ir_type type = insn->type;
6177 ir_ref op1 = insn->op1;
6178 ir_ref op2 = insn->op2;
6179 ir_ref op3 = insn->op3;
6304 static void ir_emit_cond_cmp_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
6308 ir_type type = insn->type;
6309 ir_ref op2 = insn->op2;
6310 ir_ref op3 = insn->op3;
6328 ir_emit_cmp_int_common2(ctx, def, insn->op1, &ctx->ir_base[insn->op1]);
6329 op = ctx->ir_base[insn->op1].op;
6468 static void ir_emit_cond_cmp_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
6472 ir_type type = insn->type;
6473 ir_ref op2 = insn->op2;
6474 ir_ref op3 = insn->op3;
6492 op = ir_emit_cmp_fp_common(ctx, def, insn->op1, &ctx->ir_base[insn->op1]);
6584 static void ir_emit_return_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
6589 ir_type type = ctx->ir_base[insn->op2].type;
6594 ir_emit_load(ctx, type, IR_REG_INT_RET1, insn->op2);
6600 static void ir_emit_return_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
6603 ir_type type = ctx->ir_base[insn->op2].type;
6610 ir_emit_load(ctx, type, IR_REG_FP_RET1, insn->op2);
6619 int32_t offset = ir_ref_spill_slot_offset(ctx, insn->op2, &fp);
6646 static void ir_emit_sext(ir_ctx *ctx, ir_ref def, ir_insn *insn)
6648 ir_type dst_type = insn->type;
6649 ir_type src_type = ctx->ir_base[insn->op1].type;
6663 ir_emit_load(ctx, src_type, op1_reg, insn->op1);
6695 } else if (IR_IS_CONST_REF(insn->op1)) {
6700 if (ir_rule(ctx, insn->op1) & IR_FUSED) {
6701 mem = ir_fuse_load(ctx, def, insn->op1);
6703 mem = ir_ref_spill_slot(ctx, insn->op1);
6742 static void ir_emit_zext(ir_ctx *ctx, ir_ref def, ir_insn *insn)
6744 ir_type dst_type = insn->type;
6745 ir_type src_type = ctx->ir_base[insn->op1].type;
6759 ir_emit_load(ctx, src_type, op1_reg, insn->op1);
6794 } else if (IR_IS_CONST_REF(insn->op1)) {
6799 if (ir_rule(ctx, insn->op1) & IR_FUSED) {
6800 mem = ir_fuse_load(ctx, def, insn->op1);
6802 mem = ir_ref_spill_slot(ctx, insn->op1);
6840 static void ir_emit_trunc(ir_ctx *ctx, ir_ref def, ir_insn *insn)
6842 ir_type dst_type = insn->type;
6843 ir_type src_type = ctx->ir_base[insn->op1].type;
6854 ir_emit_load(ctx, src_type, op1_reg, insn->op1);
6860 ir_emit_load_ex(ctx, dst_type, def_reg, insn->op1, def);
6867 static void ir_emit_bitcast(ir_ctx *ctx, ir_ref def, ir_insn *insn)
6869 ir_type dst_type = insn->type;
6870 ir_type src_type = ctx->ir_base[insn->op1].type;
6882 ir_emit_load(ctx, src_type, op1_reg, insn->op1);
6888 ir_emit_load_ex(ctx, dst_type, def_reg, insn->op1, def);
6894 ir_emit_load(ctx, src_type, op1_reg, insn->op1);
6900 ir_emit_load_ex(ctx, dst_type, def_reg, insn->op1, def);
6907 ir_emit_load(ctx, src_type, op1_reg, insn->op1);
6926 } else if (IR_IS_CONST_REF(insn->op1)) {
6927 ir_insn *_insn = &ctx->ir_base[insn->op1];
6941 if (ir_rule(ctx, insn->op1) & IR_FUSED) {
6942 mem = ir_fuse_load(ctx, def, insn->op1);
6944 mem = ir_ref_spill_slot(ctx, insn->op1);
6962 ir_emit_load(ctx, src_type, op1_reg, insn->op1);
6981 } else if (IR_IS_CONST_REF(insn->op1)) {
6982 int label = ir_const_label(ctx, insn->op1);
6988 if (ir_rule(ctx, insn->op1) & IR_FUSED) {
6989 mem = ir_fuse_load(ctx, def, insn->op1);
6991 mem = ir_ref_spill_slot(ctx, insn->op1);
7002 static void ir_emit_int2fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7004 ir_type dst_type = insn->type;
7005 ir_type src_type = ctx->ir_base[insn->op1].type;
7019 ir_emit_load(ctx, src_type, op1_reg, insn->op1);
7107 if (ir_rule(ctx, insn->op1) & IR_FUSED) {
7108 mem = ir_fuse_load(ctx, def, insn->op1);
7110 mem = ir_ref_spill_slot(ctx, insn->op1);
7161 static void ir_emit_fp2int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7163 ir_type dst_type = insn->type;
7164 ir_type src_type = ctx->ir_base[insn->op1].type;
7181 ir_emit_load(ctx, src_type, op1_reg, insn->op1);
7217 } else if (IR_IS_CONST_REF(insn->op1)) {
7218 int label = ir_const_label(ctx, insn->op1);
7257 if (ir_rule(ctx, insn->op1) & IR_FUSED) {
7258 mem = ir_fuse_load(ctx, def, insn->op1);
7260 mem = ir_ref_spill_slot(ctx, insn->op1);
7303 static void ir_emit_fp2fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7305 ir_type dst_type = insn->type;
7306 ir_type src_type = ctx->ir_base[insn->op1].type;
7318 ir_emit_load(ctx, src_type, op1_reg, insn->op1);
7338 } else if (IR_IS_CONST_REF(insn->op1)) {
7339 int label = ir_const_label(ctx, insn->op1);
7358 if (ir_rule(ctx, insn->op1) & IR_FUSED) {
7359 mem = ir_fuse_load(ctx, def, insn->op1);
7361 mem = ir_ref_spill_slot(ctx, insn->op1);
7384 static void ir_emit_copy_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7386 ir_ref type = insn->type;
7393 ir_emit_load(ctx, type, op1_reg, insn->op1);
7400 ir_emit_load(ctx, type, def_reg, insn->op1);
7411 static void ir_emit_copy_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7413 ir_type type = insn->type;
7420 ir_emit_load(ctx, type, op1_reg, insn->op1);
7427 ir_emit_load(ctx, type, def_reg, insn->op1);
7438 static void ir_emit_vaddr(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7442 ir_ref type = insn->type;
7449 mem = ir_var_spill_slot(ctx, insn->op1);
7458 static void ir_emit_vload(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7460 ir_insn *var_insn = &ctx->ir_base[insn->op2];
7461 ir_ref type = insn->type;
7480 static void ir_emit_vstore_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
7482 ir_insn *var_insn = &ctx->ir_base[insn->op2];
7483 ir_insn *val_insn = &ctx->ir_base[insn->op3];
7493 && !IR_IS_CONST_REF(insn->op3)
7494 && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA
7495 && ir_is_same_mem_var(ctx, insn->op3, var_insn->op3)) {
7498 if (IR_IS_CONST_REF(insn->op3)) {
7499 ir_emit_store_mem_int_const(ctx, type, mem, insn->op3, op3_reg, 0);
7504 ir_emit_load(ctx, type, op3_reg, insn->op3);
7510 static void ir_emit_vstore_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
7512 ir_insn *var_insn = &ctx->ir_base[insn->op2];
7513 ir_ref type = ctx->ir_base[insn->op3].type;
7522 && !IR_IS_CONST_REF(insn->op3)
7523 && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA
7524 && ir_is_same_mem_var(ctx, insn->op3, var_insn->op3)) {
7527 if (IR_IS_CONST_REF(insn->op3)) {
7528 ir_emit_store_mem_fp_const(ctx, type, mem, insn->op3, IR_REG_NONE, op3_reg);
7533 ir_emit_load(ctx, type, op3_reg, insn->op3);
7539 static void ir_emit_load_int(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7541 ir_ref type = insn->type;
7554 IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR);
7555 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
7558 } else if (IR_IS_CONST_REF(insn->op2)) {
7559 mem = ir_fuse_addr_const(ctx, insn->op2);
7561 IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED);
7562 mem = ir_fuse_addr(ctx, def, insn->op2);
7578 static void ir_emit_load_fp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7580 ir_ref type = insn->type;
7593 IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR);
7594 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
7597 } else if (IR_IS_CONST_REF(insn->op2)) {
7598 mem = ir_fuse_addr_const(ctx, insn->op2);
7600 IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED);
7601 mem = ir_fuse_addr(ctx, def, insn->op2);
7617 static void ir_emit_store_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
7619 ir_insn *val_insn = &ctx->ir_base[insn->op3];
7628 IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR);
7629 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
7632 } else if (IR_IS_CONST_REF(insn->op2)) {
7633 mem = ir_fuse_addr_const(ctx, insn->op2);
7635 IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED);
7636 mem = ir_fuse_addr(ctx, ref, insn->op2);
7637 if (!IR_IS_CONST_REF(insn->op3)
7639 && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA
7640 && ir_is_same_spill_slot(ctx, insn->op3, mem)) {
7641 if (!ir_may_avoid_spill_load(ctx, insn->op3, ref)) {
7643 ir_emit_load(ctx, type, op3_reg, insn->op3);
7650 if (IR_IS_CONST_REF(insn->op3)) {
7651 ir_emit_store_mem_int_const(ctx, type, mem, insn->op3, op3_reg, 0);
7656 ir_emit_load(ctx, type, op3_reg, insn->op3);
7662 static void ir_emit_cmp_and_store_int(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
7666 ir_insn *cmp_insn = &ctx->ir_base[insn->op3];
7671 ir_reg op1_reg = ctx->regs[insn->op3][1];
7672 ir_reg op2_reg = ctx->regs[insn->op3][2];
7677 IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR);
7678 ir_emit_load(ctx, IR_ADDR, addr_reg, insn->op2);
7681 } else if (IR_IS_CONST_REF(insn->op2)) {
7682 mem = ir_fuse_addr_const(ctx, insn->op2);
7684 IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED);
7685 mem = ir_fuse_addr(ctx, ref, insn->op2);
7703 static void ir_emit_store_fp(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
7705 ir_ref type = ctx->ir_base[insn->op3].type;
7714 IR_ASSERT(ctx->ir_base[insn->op2].type == IR_ADDR);
7715 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
7718 } else if (IR_IS_CONST_REF(insn->op2)) {
7719 mem = ir_fuse_addr_const(ctx, insn->op2);
7721 IR_ASSERT(ir_rule(ctx, insn->op2) & IR_FUSED);
7722 mem = ir_fuse_addr(ctx, ref, insn->op2);
7723 if (!IR_IS_CONST_REF(insn->op3)
7725 && ir_rule(ctx, insn->op3) != IR_STATIC_ALLOCA
7726 && ir_is_same_spill_slot(ctx, insn->op3, mem)) {
7727 if (!ir_may_avoid_spill_load(ctx, insn->op3, ref)) {
7729 ir_emit_load(ctx, type, op3_reg, insn->op3);
7736 if (IR_IS_CONST_REF(insn->op3)) {
7737 ir_emit_store_mem_fp_const(ctx, type, mem, insn->op3, IR_REG_NONE, op3_reg);
7742 ir_emit_load(ctx, type, op3_reg, insn->op3);
7748 static void ir_emit_rload(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7750 ir_reg src_reg = insn->op2;
7751 ir_type type = insn->type;
7766 if (!insn->op3 || !ir_is_same_spill_slot(ctx, def, IR_MEM_BO(ctx->spill_base, insn->op3))) {
7779 && (!insn->op3 || !ir_is_same_spill_slot(ctx, def, IR_MEM_BO(ctx->spill_base, insn->op3)))) {
7786 static void ir_emit_rstore(ir_ctx *ctx, ir_ref ref, ir_insn *insn)
7788 ir_ref type = ctx->ir_base[insn->op2].type;
7790 ir_reg dst_reg = insn->op3;
7795 ir_emit_load(ctx, type, op2_reg, insn->op2);
7806 ir_emit_load_ex(ctx, type, dst_reg, insn->op2, ref);
7810 static void ir_emit_alloca(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7820 if (IR_IS_CONST_REF(insn->op2)) {
7821 ir_insn *val = &ctx->ir_base[insn->op2];
7838 ir_type type = ctx->ir_base[insn->op2].type;
7845 ir_emit_load(ctx, type, op2_reg, insn->op2);
7851 ir_emit_load(ctx, type, def_reg, insn->op2);
7862 ir_emit_store(ctx, insn->type, def, def_reg);
7869 static void ir_emit_afree(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7874 if (IR_IS_CONST_REF(insn->op2)) {
7875 ir_insn *val = &ctx->ir_base[insn->op2];
7892 ir_type type = ctx->ir_base[insn->op2].type;
7897 ir_emit_load(ctx, type, op2_reg, insn->op2);
7906 static void ir_emit_block_begin(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7919 static void ir_emit_block_end(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7928 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
7950 static void ir_emit_va_start(ir_ctx *ctx, ir_ref def, ir_insn *insn)
7965 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
7969 IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
7971 offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3);
7999 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
8003 IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
8005 offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3);
8052 static void ir_emit_va_copy(ir_ctx *ctx, ir_ref def, ir_insn *insn)
8066 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
8070 IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
8072 op2_offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3);
8077 ir_emit_load(ctx, IR_ADDR, op3_reg, insn->op3);
8081 IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA);
8083 op3_offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op3].op3);
8100 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
8104 IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
8106 op2_offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3);
8111 ir_emit_load(ctx, IR_ADDR, op3_reg, insn->op3);
8115 IR_ASSERT(ir_rule(ctx, insn->op3) == IR_STATIC_ALLOCA);
8117 op3_offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op3].op3);
8133 static void ir_emit_va_arg(ir_ctx *ctx, ir_ref def, ir_insn *insn)
8138 ir_type type = insn->type;
8148 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
8152 IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
8154 offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3);
8167 ir_type type = insn->type;
8177 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
8181 IR_ASSERT(ir_rule(ctx, insn->op2) == IR_STATIC_ALLOCA);
8183 offset = IR_SPILL_POS_TO_OFFSET(ctx->ir_base[insn->op2].op3);
8224 static void ir_emit_switch(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn)
8239 type = ctx->ir_base[insn->op2].type;
8276 ir_emit_load(ctx, type, op2_reg, insn->op2);
8397 ir_insn *insn = &ctx->ir_base[bb->end];
8399 if (insn->op == IR_IJMP && IR_IS_CONST_REF(insn->op2)) {
8405 void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op2]);
8448 static int32_t ir_call_used_stack(ir_ctx *ctx, ir_insn *insn)
8459 if (sizeof(void*) == 4 && ir_is_fastcall(ctx, insn)) {
8465 n = insn->inputs_count;
8467 type = ctx->ir_base[ir_insn_op(insn, j)].type;
8496 static int32_t ir_emit_arguments(ir_ctx *ctx, ir_ref def, ir_insn *insn, ir_reg tmp_reg)
8518 n = insn->inputs_count;
8528 if (sizeof(void*) == 4 && ir_is_fastcall(ctx, insn)) {
8536 if (insn->op == IR_CALL
8539 && !ir_is_fastcall(ctx, insn) /* fast call functions restore stack pointer */
8545 used_stack = ir_call_used_stack(ctx, insn);
8547 && insn->op == IR_TAILCALL
8554 && !ir_is_fastcall(ctx, insn) /* fast call functions restore stack pointer */
8572 arg = ir_insn_op(insn, j);
8640 arg = ir_insn_op(insn, j);
8739 if (ir_is_vararg(ctx, insn)) {
8742 arg = ir_insn_op(insn, j);
8761 if (ir_is_vararg(ctx, insn)) {
8770 static void ir_emit_call_ex(ir_ctx *ctx, ir_ref def, ir_insn *insn, int32_t used_stack)
8776 if (IR_IS_CONST_REF(insn->op2)) {
8777 void *addr = ir_call_addr(ctx, insn, &ctx->ir_base[insn->op2]);
8786 || if (ir_is_vararg(ctx, insn)) {
8804 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
8810 if (ir_rule(ctx, insn->op2) & IR_FUSED) {
8811 mem = ir_fuse_load(ctx, def, insn->op2);
8813 mem = ir_ref_spill_slot(ctx, insn->op2);
8824 if (ir_is_fastcall(ctx, insn)) {
8834 if (insn->type != IR_VOID) {
8835 if (IR_IS_TYPE_INT(insn->type)) {
8839 ir_emit_mov(ctx, insn->type, def_reg, IR_REG_INT_RET1);
8842 ir_emit_store(ctx, insn->type, def, def_reg);
8845 ir_emit_store(ctx, insn->type, def, IR_REG_INT_RET1);
8848 IR_ASSERT(IR_IS_TYPE_FP(insn->type));
8853 ir_emit_fp_mov(ctx, insn->type, def_reg, IR_REG_FP_RET1);
8856 ir_emit_store(ctx, insn->type, def, def_reg);
8859 ir_emit_store(ctx, insn->type, def, IR_REG_FP_RET1);
8868 if (insn->type == IR_DOUBLE) {
8871 IR_ASSERT(insn->type == IR_FLOAT);
8879 if (insn->type == IR_DOUBLE) {
8882 IR_ASSERT(insn->type == IR_FLOAT);
8885 ir_emit_load_mem_fp(ctx, insn->type, def_reg, IR_MEM_BO(fp, offset));
8887 ir_emit_store(ctx, insn->type, def, def_reg);
8896 static void ir_emit_call(ir_ctx *ctx, ir_ref def, ir_insn *insn)
8898 int32_t used_stack = ir_emit_arguments(ctx, def, insn, ctx->regs[def][1]);
8899 ir_emit_call_ex(ctx, def, insn, used_stack);
8902 static void ir_emit_tailcall(ir_ctx *ctx, ir_ref def, ir_insn *insn)
8906 int32_t used_stack = ir_emit_arguments(ctx, def, insn, ctx->regs[def][1]);
8909 ir_emit_call_ex(ctx, def, insn, used_stack);
8916 if (IR_IS_CONST_REF(insn->op2)) {
8917 void *addr = ir_call_addr(ctx, insn, &ctx->ir_base[insn->op2]);
8926 || if (ir_is_vararg(ctx, insn)) {
8944 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
8950 if (ir_rule(ctx, insn->op2) & IR_FUSED) {
8951 mem = ir_fuse_load(ctx, def, insn->op2);
8953 mem = ir_ref_spill_slot(ctx, insn->op2);
8960 static void ir_emit_ijmp(ir_ctx *ctx, ir_ref def, ir_insn *insn)
8966 if (IR_IS_CONST_REF(insn->op2)) {
8967 void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op2]);
8981 } else if (ir_rule(ctx, insn->op2) & IR_FUSED) {
8982 ir_mem mem = ir_fuse_load(ctx, def, insn->op2);
8987 ir_emit_load(ctx, IR_ADDR, op2_reg, insn->op2);
8991 ir_mem mem = ir_ref_spill_slot(ctx, insn->op2);
9232 static bool ir_emit_guard(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_t next_block)
9237 ir_type type = ctx->ir_base[insn->op2].type;
9241 if (IR_IS_CONST_REF(insn->op2)) {
9242 bool is_true = ir_ref_is_true(ctx, insn->op2);
9244 if ((insn->op == IR_GUARD && !is_true) || (insn->op == IR_GUARD_NOT && is_true)) {
9245 addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]);
9265 ir_emit_load(ctx, type, op2_reg, insn->op2);
9271 if (ir_rule(ctx, insn->op2) & IR_FUSED) {
9272 mem = ir_fuse_load(ctx, def, insn->op2);
9274 mem = ir_ref_spill_slot(ctx, insn->op2);
9279 addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]);
9283 if (insn->op == IR_GUARD) {
9291 if (insn->op == IR_GUARD) {
9310 static bool ir_emit_guard_cmp_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_t next…
9314 ir_insn *cmp_insn = &ctx->ir_base[insn->op2];
9319 ir_reg op1_reg = ctx->regs[insn->op2][1];
9320 ir_reg op2_reg = ctx->regs[insn->op2][2];
9334 addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]);
9362 if (insn->op == IR_GUARD) {
9369 static bool ir_emit_guard_cmp_fp(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_t next_…
9371 ir_op op = ir_emit_cmp_fp_common(ctx, def, insn->op2, &ctx->ir_base[insn->op2]);
9372 void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]);
9374 if (insn->op == IR_GUARD) {
9380 static bool ir_emit_guard_test_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_t nex…
9382 void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]);
9383 ir_op op = (insn->op == IR_GUARD) ? IR_EQ : IR_NE;
9385 ir_emit_test_int_common(ctx, def, insn->op2, op);
9389 static bool ir_emit_guard_jcc_int(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn, uint32_t next…
9391 void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]);
9392 ir_op op = ctx->ir_base[insn->op2].op;
9394 if (insn->op == IR_GUARD) {
9400 static bool ir_emit_guard_overflow(ir_ctx *ctx, uint32_t b, ir_ref def, ir_insn *insn)
9405 void *addr = ir_jmp_addr(ctx, insn, &ctx->ir_base[insn->op3]);
9407 type = ctx->ir_base[ctx->ir_base[insn->op2].op1].type;
9411 if (insn->op == IR_GUARD) {
9417 if (insn->op == IR_GUARD) {
9472 static void ir_emit_tls(ir_ctx *ctx, ir_ref def, ir_insn *insn)
9486 | mov Ra(reg), aword [Ra(reg)+insn->op2]
9487 | mov Ra(reg), aword [Ra(reg)+insn->op3]
9491 | mov Ra(reg), aword [Ra(reg)+insn->op2]
9492 | mov Ra(reg), aword [Ra(reg)+insn->op3]
9495 || if (insn->op3 == IR_NULL) {
9496 | mov Ra(reg), aword [insn->op2]
9498 | mov Ra(reg), aword [insn->op2]
9499 | mov Ra(reg), aword [Ra(reg)+insn->op3]
9503 || if (insn->op3 == IR_NULL) {
9504 | mov Ra(reg), aword [insn->op2]
9507 | mov Ra(reg), aword [Ra(reg)+insn->op2]
9508 | mov Ra(reg), aword [Ra(reg)+insn->op3]
9512 || if (insn->op3 == IR_NULL) {
9513 | mov Ra(reg), aword [insn->op2]
9516 | mov Ra(reg), aword [Ra(reg)+insn->op2]
9517 | mov Ra(reg), aword [Ra(reg)+insn->op3]
9525 static void ir_emit_sse_sqrt(ir_ctx *ctx, ir_ref def, ir_insn *insn)
9532 IR_ASSERT(IR_IS_TYPE_FP(insn->type));
9537 ir_emit_load(ctx, IR_ADDR, op3_reg, insn->op3);
9540 | ASM_FP_REG_REG_OP sqrts, insn->type, def_reg, op3_reg
9543 ir_emit_store(ctx, insn->type, def, def_reg);
9547 static void ir_emit_sse_round(ir_ctx *ctx, ir_ref def, ir_insn *insn, int round_op)
9554 IR_ASSERT(IR_IS_TYPE_FP(insn->type));
9559 ir_emit_load(ctx, IR_ADDR, op3_reg, insn->op3);
9563 | ASM_SSE2_REG_REG_REG_TXT_OP vrounds, insn->type, def_reg, def_reg, op3_reg, round_op
9565 | ASM_SSE2_REG_REG_TXT_OP rounds, insn->type, def_reg, op3_reg, round_op
9569 ir_emit_store(ctx, insn->type, def, def_reg);
9573 static void ir_emit_exitcall(ir_ctx *ctx, ir_ref def, ir_insn *insn)
9646 if (IR_IS_CONST_REF(insn->op2)) {
9647 void *addr = ir_call_addr(ctx, insn, &ctx->ir_base[insn->op2]);
9675 ir_emit_mov(ctx, insn->type, def_reg, IR_REG_INT_RET1);
9678 ir_emit_store(ctx, insn->type, def, def_reg);
9714 ir_insn *insn;
9744 insn = &ctx->ir_base[use];
9745 if (insn->op == IR_PARAM) {
9746 if (IR_IS_TYPE_INT(insn->type)) {
9777 ir_emit_param_move(ctx, insn->type, src_reg, dst_reg, use, stack_offset);
9780 ir_emit_store(ctx, insn->type, use, dst_reg);
9787 stack_offset += IR_MAX(sizeof(void*), ir_type_size[insn->type]);
9840 ir_insn *insn;
9872 insn = &ctx->ir_base[use];
9873 if (insn->op == IR_PARAM) {
9874 if (IR_IS_TYPE_INT(insn->type)) {
9909 stack_offset += IR_MAX(sizeof(void*), ir_type_size[insn->type]);
9928 ir_insn *insn;
9959 for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) {
9960 switch (ctx->rules ? *rule : insn->op) {
9974 if (ctx->ret_slot == -1 && (insn->type == IR_FLOAT || insn->type == IR_DOUBLE)) {
9995 if (insn->op == IR_VLOAD
9999 } else if (insn->op != IR_PARAM) {
10000 reg = ir_get_free_reg(insn->type, available);
10009 ival->type = insn->type;
10013 if (insn->op == IR_PARAM && reg == IR_REG_NONE) {
10018 } else if (insn->op == IR_PARAM) {
10022 } else if (insn->op == IR_VAR) {
10027 … int32_t stack_spill_pos = insn->op3 = ir_allocate_spill_slot(ctx, insn->type, &data->ra_data);
10040 ival->type = insn->type;
10052 ival->type = insn->type;
10062 insn_flags = ir_op_flags[insn->op];
10069 ir_ref *ops = insn->ops;
10084 n = insn->inputs_count;
10085 for (j = 1, p = insn->ops + 1; j <= n; j++, p++) {
10098 } else if (j > 1 && input == insn->op1 && ctx->regs[i][1] != IR_REG_NONE) {
10110 n = ir_insn_len(insn);
10112 insn += n;
10130 ir_insn *insn;
10132 for (i = 1, insn = ctx->ir_base + 1; i < ctx->insns_count;) {
10133 if (insn->op == IR_CALL) {
10134 call_stack_size = ir_call_used_stack(ctx, insn);
10137 && !ir_is_fastcall(ctx, insn) /* fast call functions restore stack pointer */
10143 n = ir_insn_len(insn);
10145 insn += n;
10233 ir_insn *insn;
10326 insn = ctx->ir_base + i;
10328 uint32_t label = ctx->cfg_blocks_count + ctx->consts_count + 4 + insn->op3;
10339 ctx->entries[insn->op3] = i;
10343 n = ir_insn_len(insn);
10345 insn += n;
10371 ir_emit_lea(ctx, i, insn->type);
10376 ir_emit_mul_div_mod_pwr2(ctx, i, insn);
10379 ir_emit_sdiv_pwr2(ctx, i, insn);
10382 ir_emit_smod_pwr2(ctx, i, insn);
10385 ir_emit_shift(ctx, i, insn);
10388 ir_emit_shift_const(ctx, i, insn);
10391 ir_emit_bit_count(ctx, i, insn);
10394 ir_emit_ctpop(ctx, i, insn);
10399 ir_emit_op_int(ctx, i, insn, *rule);
10402 ir_emit_abs_int(ctx, i, insn);
10405 ir_emit_bool_not_int(ctx, i, insn);
10408 ir_emit_op_fp(ctx, i, insn);
10411 ir_emit_imul3(ctx, i, insn);
10414 ir_emit_binop_int(ctx, i, insn);
10417 ir_emit_binop_sse2(ctx, i, insn);
10420 ir_emit_binop_avx(ctx, i, insn);
10425 ir_emit_mul_div_mod(ctx, i, insn);
10428 ir_emit_cmp_int(ctx, i, insn);
10431 ir_emit_testcc_int(ctx, i, insn);
10434 ir_emit_setcc_int(ctx, i, insn);
10437 ir_emit_cmp_fp(ctx, i, insn);
10440 ir_emit_sext(ctx, i, insn);
10443 ir_emit_zext(ctx, i, insn);
10446 ir_emit_trunc(ctx, i, insn);
10450 ir_emit_bitcast(ctx, i, insn);
10453 ir_emit_int2fp(ctx, i, insn);
10456 ir_emit_fp2int(ctx, i, insn);
10459 ir_emit_fp2fp(ctx, i, insn);
10462 ir_emit_copy_int(ctx, i, insn);
10465 ir_emit_copy_fp(ctx, i, insn);
10468 ir_emit_cmp_and_store_int(ctx, i, insn);
10471 ir_emit_cmp_and_branch_int(ctx, b, i, insn, _ir_next_block(ctx, _b));
10474 ir_emit_cmp_and_branch_fp(ctx, b, i, insn, _ir_next_block(ctx, _b));
10477 ir_emit_test_and_branch_int(ctx, b, i, insn, _ir_next_block(ctx, _b));
10481 ir_op op = ctx->ir_base[insn->op2].op;
10493 ir_emit_jcc(ctx, b, i, insn, _ir_next_block(ctx, _b), op, 1);
10497 if (ir_emit_guard_cmp_int(ctx, b, i, insn, _ir_next_block(ctx, _b))) {
10502 if (ir_emit_guard_cmp_fp(ctx, b, i, insn, _ir_next_block(ctx, _b))) {
10507 if (ir_emit_guard_test_int(ctx, b, i, insn, _ir_next_block(ctx, _b))) {
10512 if (ir_emit_guard_jcc_int(ctx, b, i, insn, _ir_next_block(ctx, _b))) {
10517 ir_emit_if_int(ctx, b, i, insn, _ir_next_block(ctx, _b));
10520 ir_emit_cond(ctx, i, insn);
10523 ir_emit_cond_cmp_int(ctx, i, insn);
10526 ir_emit_cond_cmp_fp(ctx, i, insn);
10529 ir_emit_switch(ctx, b, i, insn);
10532 ir_emit_min_max_int(ctx, i, insn);
10535 ir_emit_overflow(ctx, i, insn);
10538 ir_emit_overflow_and_branch(ctx, b, i, insn, _ir_next_block(ctx, _b));
10570 ir_emit_return_int(ctx, i, insn);
10573 ir_emit_return_fp(ctx, i, insn);
10576 ir_emit_call(ctx, i, insn);
10579 ir_emit_tailcall(ctx, i, insn);
10582 ir_emit_ijmp(ctx, i, insn);
10587 ir_emit_mem_op_int(ctx, i, insn, *rule);
10590 ir_emit_mem_binop_int(ctx, i, insn);
10595 ir_emit_mem_mul_div_mod_pwr2(ctx, i, insn);
10598 ir_emit_mem_shift(ctx, i, insn);
10601 ir_emit_mem_shift_const(ctx, i, insn);
10604 ir_emit_reg_binop_int(ctx, i, insn);
10607 ir_emit_vaddr(ctx, i, insn);
10610 ir_emit_vload(ctx, i, insn);
10613 ir_emit_vstore_int(ctx, i, insn);
10616 ir_emit_vstore_fp(ctx, i, insn);
10619 ir_emit_rload(ctx, i, insn);
10622 ir_emit_rstore(ctx, i, insn);
10625 ir_emit_load_int(ctx, i, insn);
10628 ir_emit_load_fp(ctx, i, insn);
10631 ir_emit_store_int(ctx, i, insn);
10634 ir_emit_store_fp(ctx, i, insn);
10637 ir_emit_alloca(ctx, i, insn);
10640 ir_emit_va_start(ctx, i, insn);
10643 ir_emit_va_copy(ctx, i, insn);
10646 ir_emit_va_arg(ctx, i, insn);
10649 ir_emit_afree(ctx, i, insn);
10652 ir_emit_block_begin(ctx, i, insn);
10655 ir_emit_block_end(ctx, i, insn);
10661 ir_emit_exitcall(ctx, i, insn);
10665 if (ir_emit_guard(ctx, b, i, insn, _ir_next_block(ctx, _b))) {
10670 if (ir_emit_guard_overflow(ctx, b, i, insn)) {
10675 ir_emit_sse_sqrt(ctx, i, insn);
10678 ir_emit_sse_round(ctx, i, insn, 4);
10681 ir_emit_sse_round(ctx, i, insn, 9);
10684 ir_emit_sse_round(ctx, i, insn, 10);
10687 ir_emit_sse_round(ctx, i, insn, 11);
10690 ir_emit_sse_round(ctx, i, insn, 12);
10693 ir_emit_tls(ctx, i, insn);
10706 n = ir_insn_len(insn);
10708 insn += n;
10718 insn = &ctx->ir_base[-i];
10719 if (IR_IS_TYPE_FP(insn->type)) {
10728 if (insn->type == IR_DOUBLE) {
10731 |.dword insn->val.u32, insn->val.u32_hi
10733 IR_ASSERT(insn->type == IR_FLOAT);
10736 |.dword insn->val.u32
10738 } else if (insn->op == IR_STR) {
10740 const char *str = ir_get_str(ctx, insn->val.str);
10838 ir_insn *insn = &ctx->ir_base[ctx->entries[--i]];
10839 …set = dasm_getpclabel(&data.dasm_state, ctx->cfg_blocks_count + ctx->consts_count + 4 + insn->op3);
10840 insn->op3 = offset;
