Lines Matching refs:ival
120 ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); in ir_new_live_range() local
122 ival->type = IR_VOID; in ir_new_live_range()
123 ival->reg = IR_REG_NONE; in ir_new_live_range()
124 ival->flags = 0; in ir_new_live_range()
125 ival->vreg = v; in ir_new_live_range()
126 ival->stack_spill_pos = -1; // not allocated in ir_new_live_range()
127 ival->range.start = start; in ir_new_live_range()
128 ival->range.end = ival->end = end; in ir_new_live_range()
129 ival->range.next = NULL; in ir_new_live_range()
130 ival->use_pos = NULL; in ir_new_live_range()
131 ival->next = NULL; in ir_new_live_range()
133 ctx->live_intervals[v] = ival; in ir_new_live_range()
134 return ival; in ir_new_live_range()
139 ir_live_interval *ival = ctx->live_intervals[v]; in ir_add_live_range() local
142 if (!ival) { in ir_add_live_range()
146 p = &ival->range; in ir_add_live_range()
171 ival->end = p->end; in ir_add_live_range()
174 return ival; in ir_add_live_range()
180 ival->end = end; in ir_add_live_range()
194 return ival; in ir_add_live_range()
211 return ival; in ir_add_live_range()
216 ir_live_interval *ival = ctx->live_intervals[v]; in ir_add_prev_live_range() local
218 if (ival && ival->range.start == end) { in ir_add_prev_live_range()
219 ival->range.start = start; in ir_add_prev_live_range()
220 return ival; in ir_add_prev_live_range()
228 ir_live_interval *ival = ctx->live_intervals[v]; in ir_add_fixed_live_range() local
231 if (!ival) { in ir_add_fixed_live_range()
232 ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); in ir_add_fixed_live_range()
233 ival->type = IR_VOID; in ir_add_fixed_live_range()
234 ival->reg = reg; in ir_add_fixed_live_range()
235 ival->flags = IR_LIVE_INTERVAL_FIXED; in ir_add_fixed_live_range()
236 ival->vreg = v; in ir_add_fixed_live_range()
237 ival->stack_spill_pos = -1; // not allocated in ir_add_fixed_live_range()
238 ival->range.start = start; in ir_add_fixed_live_range()
239 ival->range.end = ival->end = end; in ir_add_fixed_live_range()
240 ival->range.next = NULL; in ir_add_fixed_live_range()
241 ival->use_pos = NULL; in ir_add_fixed_live_range()
242 ival->next = NULL; in ir_add_fixed_live_range()
244 ctx->live_intervals[v] = ival; in ir_add_fixed_live_range()
245 } else if (EXPECTED(end < ival->range.start)) { in ir_add_fixed_live_range()
254 q->start = ival->range.start; in ir_add_fixed_live_range()
255 q->end = ival->range.end; in ir_add_fixed_live_range()
256 q->next = ival->range.next; in ir_add_fixed_live_range()
257 ival->range.start = start; in ir_add_fixed_live_range()
258 ival->range.end = end; in ir_add_fixed_live_range()
259 ival->range.next = q; in ir_add_fixed_live_range()
260 } else if (end == ival->range.start) { in ir_add_fixed_live_range()
261 ival->range.start = start; in ir_add_fixed_live_range()
269 ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); in ir_add_tmp() local
271 ival->type = tmp_reg.type; in ir_add_tmp()
272 ival->reg = IR_REG_NONE; in ir_add_tmp()
273 ival->flags = IR_LIVE_INTERVAL_TEMP; in ir_add_tmp()
274 ival->tmp_ref = tmp_ref; in ir_add_tmp()
275 ival->tmp_op_num = tmp_op_num; in ir_add_tmp()
276 ival->range.start = IR_START_LIVE_POS_FROM_REF(ref) + tmp_reg.start; in ir_add_tmp()
277 ival->range.end = ival->end = IR_START_LIVE_POS_FROM_REF(ref) + tmp_reg.end; in ir_add_tmp()
278 ival->range.next = NULL; in ir_add_tmp()
279 ival->use_pos = NULL; in ir_add_tmp()
282 ival->next = NULL; in ir_add_tmp()
283 ctx->live_intervals[0] = ival; in ir_add_tmp()
284 } else if (ival->range.start >= ctx->live_intervals[0]->range.start) { in ir_add_tmp()
287 while (prev->next && ival->range.start >= prev->next->range.start) { in ir_add_tmp()
290 ival->next = prev->next; in ir_add_tmp()
291 prev->next = ival; in ir_add_tmp()
295 ival->next = next; in ir_add_tmp()
296 ctx->live_intervals[0] = ival; in ir_add_tmp()
303 ir_live_interval *ival = ctx->live_intervals[0]; in ir_has_tmp() local
305 if (ival) { in ir_has_tmp()
306 while (ival && IR_LIVE_POS_TO_REF(ival->range.start) <= ref) { in ir_has_tmp()
307 if (ival->tmp_ref == ref && ival->tmp_op_num == op_num) { in ir_has_tmp()
310 ival = ival->next; in ir_has_tmp()
318 ir_live_interval *ival = ctx->live_intervals[v]; in ir_fix_live_range() local
319 ir_live_range *p = &ival->range; in ir_fix_live_range()
326 IR_ASSERT(ival && p->start == old_start); in ir_fix_live_range()
328 return ival; in ir_fix_live_range()
331 static void ir_add_use_pos(ir_ctx *ctx, ir_live_interval *ival, ir_use_pos *use_pos) in ir_add_use_pos() argument
333 ir_use_pos *p = ival->use_pos; in ir_add_use_pos()
337 ival->use_pos = use_pos; in ir_add_use_pos()
352 IR_ALWAYS_INLINE void ir_add_use(ir_ctx *ctx, ir_live_interval *ival, int op_num, ir_live_pos pos, … in ir_add_use() argument
364 ival->flags |= IR_LIVE_INTERVAL_HAS_HINT_REGS; in ir_add_use()
367 ival->flags |= IR_LIVE_INTERVAL_HAS_HINT_REFS; in ir_add_use()
370 ir_add_use_pos(ctx, ival, use_pos); in ir_add_use()
373 static void ir_add_phi_use(ir_ctx *ctx, ir_live_interval *ival, int op_num, ir_live_pos pos, ir_ref… in ir_add_phi_use() argument
385 ir_add_use_pos(ctx, ival, use_pos); in ir_add_phi_use()
390 ir_live_interval *ival = ctx->live_intervals[ctx->vregs[ref]]; in ir_add_hint() local
392 if (!(ival->flags & IR_LIVE_INTERVAL_HAS_HINT_REGS)) { in ir_add_hint()
393 ir_use_pos *use_pos = ival->use_pos; in ir_add_hint()
399 ival->flags |= IR_LIVE_INTERVAL_HAS_HINT_REGS; in ir_add_hint()
410 ir_live_interval *ival; in ir_hint_propagation() local
415 ival = ctx->live_intervals[i]; in ir_hint_propagation()
416 if (ival in ir_hint_propagation()
417 …&& (ival->flags & (IR_LIVE_INTERVAL_HAS_HINT_REGS|IR_LIVE_INTERVAL_HAS_HINT_REFS)) == (IR_LIVE_INT… in ir_hint_propagation()
418 use_pos = ival->use_pos; in ir_hint_propagation()
509 ir_live_interval *ival; in ir_add_fusion_ranges() local
560 ival = ir_add_live_range(ctx, v, in ir_add_fusion_ranges()
563 ival = ctx->live_intervals[v]; in ir_add_fusion_ranges()
565 ir_add_use(ctx, ival, j, use_pos, reg, use_flags, -input); in ir_add_fusion_ranges()
594 ir_live_interval *ival; in ir_compute_live_ranges() local
694 ival = ir_add_prev_live_range(ctx, v, in ir_compute_live_ranges()
697 ir_add_phi_use(ctx, ival, k, IR_DEF_LIVE_POS_FROM_REF(bb->end), use); in ir_compute_live_ranges()
788 ival = ir_fix_live_range(ctx, v, in ir_compute_live_ranges()
790 ival->type = insn->type; in ir_compute_live_ranges()
791 ir_add_use(ctx, ival, 0, def_pos, reg, def_flags, hint_ref); in ir_compute_live_ranges()
796 ival = ctx->live_intervals[v]; in ir_compute_live_ranges()
797 if (UNEXPECTED(!ival)) { in ir_compute_live_ranges()
799 … ival = ir_add_live_range(ctx, v, IR_DEF_LIVE_POS_FROM_REF(ref), IR_USE_LIVE_POS_FROM_REF(ref)); in ir_compute_live_ranges()
801 ival->type = insn->type; in ir_compute_live_ranges()
802 … ir_add_use(ctx, ival, 0, IR_DEF_LIVE_POS_FROM_REF(ref), IR_REG_NONE, IR_USE_SHOULD_BE_IN_REG, 0); in ir_compute_live_ranges()
843 ival = ir_add_live_range(ctx, v, IR_START_LIVE_POS_FROM_REF(bb->start), use_pos); in ir_compute_live_ranges()
845 ival = ctx->live_intervals[v]; in ir_compute_live_ranges()
847 ir_add_use(ctx, ival, j, use_pos, reg, IR_USE_FLAGS(def_flags, j), hint_ref); in ir_compute_live_ranges()
1166 ir_live_interval *ival; in ir_add_fusion_ranges() local
1217 ival = ir_add_live_range(ctx, v, in ir_add_fusion_ranges()
1220 ival = ctx->live_intervals[v]; in ir_add_fusion_ranges()
1223 ir_add_use(ctx, ival, j, use_pos, reg, use_flags, -input); in ir_add_fusion_ranges()
1248 ir_live_interval *ival; in ir_compute_live_ranges() local
1313 ival = ctx->live_intervals[v]; in ir_compute_live_ranges()
1314 ir_add_phi_use(ctx, ival, k, IR_DEF_LIVE_POS_FROM_REF(bb->end), use); in ir_compute_live_ranges()
1406 ival = ir_fix_live_range(ctx, v, in ir_compute_live_ranges()
1408 ival->type = insn->type; in ir_compute_live_ranges()
1409 ir_add_use(ctx, ival, 0, def_pos, reg, def_flags, hint_ref); in ir_compute_live_ranges()
1412 ival = ctx->live_intervals[v]; in ir_compute_live_ranges()
1413 if (UNEXPECTED(!ival)) { in ir_compute_live_ranges()
1415 … ival = ir_add_live_range(ctx, v, IR_DEF_LIVE_POS_FROM_REF(ref), IR_USE_LIVE_POS_FROM_REF(ref)); in ir_compute_live_ranges()
1417 ival->type = insn->type; in ir_compute_live_ranges()
1418 … ir_add_use(ctx, ival, 0, IR_DEF_LIVE_POS_FROM_REF(ref), IR_REG_NONE, IR_USE_SHOULD_BE_IN_REG, 0); in ir_compute_live_ranges()
1463 ival = ir_add_live_range(ctx, v, IR_START_LIVE_POS_FROM_REF(bb->start), use_pos); in ir_compute_live_ranges()
1465 ival = ctx->live_intervals[v]; in ir_compute_live_ranges()
1467 ir_add_use(ctx, ival, j, use_pos, reg, IR_USE_FLAGS(def_flags, j), hint_ref); in ir_compute_live_ranges()
1580 ir_live_interval *ival = ctx->live_intervals[r2]; in ir_vregs_join() local
1581 ir_live_range *live_range = &ival->range; in ir_vregs_join()
1601 use_pos = ival->use_pos; in ir_vregs_join()
1629 …IR_LIVE_INTERVAL_COALESCED | (ival->flags & (IR_LIVE_INTERVAL_HAS_HINT_REGS|IR_LIVE_INTERVAL_HAS_H… in ir_vregs_join()
1722 ir_live_interval *ival; in ir_swap_operands() local
1731 ival = ctx->live_intervals[ctx->vregs[insn->op1]]; in ir_swap_operands()
1732 p = ival->use_pos; in ir_swap_operands()
1743 ival = ctx->live_intervals[ctx->vregs[i]]; in ir_swap_operands()
1744 p = ival->use_pos; in ir_swap_operands()
1754 ival = ctx->live_intervals[ctx->vregs[insn->op2]]; in ir_swap_operands()
1755 r = &ival->range; in ir_swap_operands()
1760 ival->end = pos; in ir_swap_operands()
1766 p = ival->use_pos; in ir_swap_operands()
1825 ir_live_interval *ival = ctx->live_intervals[ctx->vregs[insn->op2]]; in ir_try_swap_operands() local
1826 ir_live_range *r = &ival->range; in ir_try_swap_operands()
1828 if ((ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) && ctx->use_lists[insn->op2].count == 1) { in ir_try_swap_operands()
1835 ival->end = load_pos; in ir_try_swap_operands()
1844 ival->end = pos; in ir_try_swap_operands()
2226 # define IR_LOG_LSRA(action, ival, comment) do { \ argument
2228 ir_live_interval *_ival = (ival); \
2237 # define IR_LOG_LSRA_ASSIGN(action, ival, comment) do { \ argument
2239 ir_live_interval *_ival = (ival); \
2249 # define IR_LOG_LSRA_SPLIT(ival, pos) do { \ argument
2251 ir_live_interval *_ival = (ival); \
2262 # define IR_LOG_LSRA_CONFLICT(action, ival, pos) do { \ argument
2264 ir_live_interval *_ival = (ival); \
2277 # define IR_LOG_LSRA(action, ival, comment) argument
2278 # define IR_LOG_LSRA_ASSIGN(action, ival, comment) argument
2279 # define IR_LOG_LSRA_SPLIT(ival, pos) argument
2280 # define IR_LOG_LSRA_CONFLICT(action, ival, pos); argument
2283 static bool ir_ival_covers(ir_live_interval *ival, ir_live_pos position) in ir_ival_covers() argument
2285 ir_live_range *live_range = &ival->range; in ir_ival_covers()
2297 static bool ir_ival_has_hole_between(ir_live_interval *ival, ir_live_pos from, ir_live_pos to) in ir_ival_has_hole_between() argument
2299 ir_live_range *r = &ival->range; in ir_ival_has_hole_between()
2313 static ir_live_pos ir_last_use_pos_before(ir_live_interval *ival, ir_live_pos pos, uint8_t flags) in ir_last_use_pos_before() argument
2316 ir_use_pos *p = ival->use_pos; in ir_last_use_pos_before()
2327 static ir_live_pos ir_first_use_pos_after(ir_live_interval *ival, ir_live_pos pos, uint8_t flags) in ir_first_use_pos_after() argument
2329 ir_use_pos *p = ival->use_pos; in ir_first_use_pos_after()
2343 static ir_live_pos ir_first_use_pos(ir_live_interval *ival, uint8_t flags) in ir_first_use_pos() argument
2345 ir_use_pos *p = ival->use_pos; in ir_first_use_pos()
2367 static ir_live_pos ir_find_optimal_split_position(ir_ctx *ctx, ir_live_interval *ival, ir_live_pos … in ir_find_optimal_split_position() argument
2376 IR_ASSERT(min_pos >= ival->range.start); in ir_find_optimal_split_position()
2377 IR_ASSERT(max_pos < ival->end); in ir_find_optimal_split_position()
2383 || ir_ival_has_hole_between(ival, min_pos, max_pos)) { // TODO: ??? in ir_find_optimal_split_position()
2418 static ir_live_interval *ir_split_interval_at(ir_ctx *ctx, ir_live_interval *ival, ir_live_pos pos) in ir_split_interval_at() argument
2424 IR_LOG_LSRA_SPLIT(ival, pos); in ir_split_interval_at()
2425 IR_ASSERT(pos > ival->range.start); in ir_split_interval_at()
2428 p = &ival->range; in ir_split_interval_at()
2441 use_pos = ival->use_pos; in ir_split_interval_at()
2444 ival->flags &= ~(IR_LIVE_INTERVAL_HAS_HINT_REGS|IR_LIVE_INTERVAL_HAS_HINT_REFS); in ir_split_interval_at()
2448 ival->flags |= IR_LIVE_INTERVAL_HAS_HINT_REGS; in ir_split_interval_at()
2451 ival->flags |= IR_LIVE_INTERVAL_HAS_HINT_REFS; in ir_split_interval_at()
2459 ival->flags |= IR_LIVE_INTERVAL_HAS_HINT_REGS; in ir_split_interval_at()
2462 ival->flags |= IR_LIVE_INTERVAL_HAS_HINT_REFS; in ir_split_interval_at()
2470 child->type = ival->type; in ir_split_interval_at()
2473 child->vreg = ival->vreg; in ir_split_interval_at()
2478 child->end = ival->end; in ir_split_interval_at()
2481 child->next = ival->next; in ir_split_interval_at()
2482 ival->next = child; in ir_split_interval_at()
2486 ival->end = prev->end; in ir_split_interval_at()
2491 p->end = ival->end = pos; in ir_split_interval_at()
2497 ival->use_pos = NULL; in ir_split_interval_at()
2641 static ir_reg ir_get_first_reg_hint(ir_ctx *ctx, ir_live_interval *ival, ir_regset available) in ir_get_first_reg_hint() argument
2646 use_pos = ival->use_pos; in ir_get_first_reg_hint()
2658 static ir_reg ir_try_allocate_preferred_reg(ir_ctx *ctx, ir_live_interval *ival, ir_regset availabl… in ir_try_allocate_preferred_reg() argument
2663 if (ival->flags & IR_LIVE_INTERVAL_HAS_HINT_REGS) { in ir_try_allocate_preferred_reg()
2664 use_pos = ival->use_pos; in ir_try_allocate_preferred_reg()
2668 if (ival->end <= freeUntilPos[reg]) { in ir_try_allocate_preferred_reg()
2677 if (ival->flags & IR_LIVE_INTERVAL_HAS_HINT_REFS) { in ir_try_allocate_preferred_reg()
2678 use_pos = ival->use_pos; in ir_try_allocate_preferred_reg()
2683 if (ival->end <= freeUntilPos[reg]) { in ir_try_allocate_preferred_reg()
2696 static ir_reg ir_get_preferred_reg(ir_ctx *ctx, ir_live_interval *ival, ir_regset available) in ir_get_preferred_reg() argument
2701 use_pos = ival->use_pos; in ir_get_preferred_reg()
2718 static void ir_add_to_unhandled(ir_live_interval **unhandled, ir_live_interval *ival) in ir_add_to_unhandled() argument
2720 ir_live_pos pos = ival->range.start; in ir_add_to_unhandled()
2725 && (ival->flags & (IR_LIVE_INTERVAL_HAS_HINT_REGS|IR_LIVE_INTERVAL_HAS_HINT_REFS)) in ir_add_to_unhandled()
2728 && ival->vreg > (*unhandled)->vreg)) { in ir_add_to_unhandled()
2729 ival->list_next = *unhandled; in ir_add_to_unhandled()
2730 *unhandled = ival; in ir_add_to_unhandled()
2737 && (ival->flags & (IR_LIVE_INTERVAL_HAS_HINT_REGS|IR_LIVE_INTERVAL_HAS_HINT_REFS)) in ir_add_to_unhandled()
2740 && ival->vreg > prev->list_next->vreg)) { in ir_add_to_unhandled()
2745 ival->list_next = prev->list_next; in ir_add_to_unhandled()
2746 prev->list_next = ival; in ir_add_to_unhandled()
2751 static void ir_merge_to_unhandled(ir_live_interval **unhandled, ir_live_interval *ival) in ir_merge_to_unhandled() argument
2757 *unhandled = ival; in ir_merge_to_unhandled()
2758 while (ival) { in ir_merge_to_unhandled()
2759 ival = ival->list_next = ival->next; in ir_merge_to_unhandled()
2763 while (ival) { in ir_merge_to_unhandled()
2764 pos = ival->range.start; in ir_merge_to_unhandled()
2768 ival->list_next = *prev; in ir_merge_to_unhandled()
2769 *prev = ival; in ir_merge_to_unhandled()
2770 prev = &ival->list_next; in ir_merge_to_unhandled()
2771 ival = ival->next; in ir_merge_to_unhandled()
2775 ival = *unhandled; in ir_merge_to_unhandled()
2778 while (ival) { in ir_merge_to_unhandled()
2779 IR_ASSERT(ival->range.start >= pos); in ir_merge_to_unhandled()
2780 pos = ival->range.start; in ir_merge_to_unhandled()
2781 ival = ival->list_next; in ir_merge_to_unhandled()
2786 static void ir_add_to_unhandled_spill(ir_live_interval **unhandled, ir_live_interval *ival) in ir_add_to_unhandled_spill() argument
2788 ir_live_pos pos = ival->range.start; in ir_add_to_unhandled_spill()
2792 ival->list_next = *unhandled; in ir_add_to_unhandled_spill()
2793 *unhandled = ival; in ir_add_to_unhandled_spill()
2803 ival->list_next = prev->list_next; in ir_add_to_unhandled_spill()
2804 prev->list_next = ival; in ir_add_to_unhandled_spill()
2808 static ir_reg ir_try_allocate_free_reg(ir_ctx *ctx, ir_live_interval *ival, ir_live_interval **acti… in ir_try_allocate_free_reg() argument
2816 if (IR_IS_TYPE_FP(ival->type)) { in ir_try_allocate_free_reg()
2828 if (ir_type_size[ival->type] == 1) { in ir_try_allocate_free_reg()
2869 pos = ival->end; in ir_try_allocate_free_reg()
2873 next = ir_ivals_overlap(&ival->range, other->current_range); in ir_try_allocate_free_reg()
2906 if (ival->flags & (IR_LIVE_INTERVAL_HAS_HINT_REGS|IR_LIVE_INTERVAL_HAS_HINT_REFS)) { in ir_try_allocate_free_reg()
2908 reg = ir_try_allocate_preferred_reg(ctx, ival, available, freeUntilPos); in ir_try_allocate_free_reg()
2910 ival->reg = reg; in ir_try_allocate_free_reg()
2911 IR_LOG_LSRA_ASSIGN(" ---- Assign", ival, " (hint available without spilling)"); in ir_try_allocate_free_reg()
2912 if (*unhandled && ival->end > (*unhandled)->range.start) { in ir_try_allocate_free_reg()
2913 ival->list_next = *active; in ir_try_allocate_free_reg()
2914 *active = ival; in ir_try_allocate_free_reg()
2920 if (ival->flags & IR_LIVE_INTERVAL_SPLIT_CHILD) { in ir_try_allocate_free_reg()
2922 reg = ctx->live_intervals[ival->vreg]->reg; in ir_try_allocate_free_reg()
2924 ival->reg = reg; in ir_try_allocate_free_reg()
2925 IR_LOG_LSRA_ASSIGN(" ---- Assign", ival, " (available without spilling)"); in ir_try_allocate_free_reg()
2926 if (*unhandled && ival->end > (*unhandled)->range.start) { in ir_try_allocate_free_reg()
2927 ival->list_next = *active; in ir_try_allocate_free_reg()
2928 *active = ival; in ir_try_allocate_free_reg()
2942 while (other && other->range.start < ival->range.end) { in ir_try_allocate_free_reg()
2966 ival->reg = reg; in ir_try_allocate_free_reg()
2967 IR_LOG_LSRA_ASSIGN(" ---- Assign", ival, " (available without spilling)"); in ir_try_allocate_free_reg()
2968 if (*unhandled && ival->end > (*unhandled)->range.start) { in ir_try_allocate_free_reg()
2969 ival->list_next = *active; in ir_try_allocate_free_reg()
2970 *active = ival; in ir_try_allocate_free_reg()
2991 if (pos > ival->range.start) { in ir_try_allocate_free_reg()
2994 ir_live_pos split_pos = ir_last_use_pos_before(ival, pos, in ir_try_allocate_free_reg()
2996 if (split_pos > ival->range.start) { in ir_try_allocate_free_reg()
2997 split_pos = ir_find_optimal_split_position(ctx, ival, split_pos, pos, 0); in ir_try_allocate_free_reg()
2998 other = ir_split_interval_at(ctx, ival, split_pos); in ir_try_allocate_free_reg()
2999 if (ival->flags & (IR_LIVE_INTERVAL_HAS_HINT_REGS|IR_LIVE_INTERVAL_HAS_HINT_REFS)) { in ir_try_allocate_free_reg()
3000 …ir_reg pref_reg = ir_try_allocate_preferred_reg(ctx, ival, IR_REGSET_UNION(available, overlapped),… in ir_try_allocate_free_reg()
3003 ival->reg = pref_reg; in ir_try_allocate_free_reg()
3005 ival->reg = reg; in ir_try_allocate_free_reg()
3008 ival->reg = reg; in ir_try_allocate_free_reg()
3010 IR_LOG_LSRA_ASSIGN(" ---- Assign", ival, " (available without spilling for the first part)"); in ir_try_allocate_free_reg()
3011 if (*unhandled && ival->end > (*unhandled)->range.start) { in ir_try_allocate_free_reg()
3012 ival->list_next = *active; in ir_try_allocate_free_reg()
3013 *active = ival; in ir_try_allocate_free_reg()
3023 static ir_reg ir_allocate_blocked_reg(ir_ctx *ctx, ir_live_interval *ival, ir_live_interval **activ… in ir_allocate_blocked_reg() argument
3033 if (!(ival->flags & IR_LIVE_INTERVAL_TEMP)) { in ir_allocate_blocked_reg()
3034 use_pos = ival->use_pos; in ir_allocate_blocked_reg()
3040 IR_LOG_LSRA(" ---- Spill", ival, " (no use pos that must be in reg)"); in ir_allocate_blocked_reg()
3046 next_use_pos = ival->range.end; in ir_allocate_blocked_reg()
3049 if (IR_IS_TYPE_FP(ival->type)) { in ir_allocate_blocked_reg()
3062 if (ir_type_size[ival->type] == 1) { in ir_allocate_blocked_reg()
3106 pos = ir_first_use_pos_after(other, ival->range.start, in ir_allocate_blocked_reg()
3123 ir_live_pos overlap = ir_ivals_overlap(&ival->range, other->current_range); in ir_allocate_blocked_reg()
3144 ir_live_pos overlap = ir_ivals_overlap(&ival->range, other->current_range); in ir_allocate_blocked_reg()
3155 pos = ir_first_use_pos_after(other, ival->range.start, in ir_allocate_blocked_reg()
3168 if (ival->flags & (IR_LIVE_INTERVAL_HAS_HINT_REGS|IR_LIVE_INTERVAL_HAS_HINT_REFS)) { in ir_allocate_blocked_reg()
3169 reg = ir_get_preferred_reg(ctx, ival, available); in ir_allocate_blocked_reg()
3188 if (next_use_pos > pos && !(ival->flags & IR_LIVE_INTERVAL_TEMP)) { in ir_allocate_blocked_reg()
3195 if (next_use_pos == ival->range.start) { in ir_allocate_blocked_reg()
3196 IR_ASSERT(ival->use_pos && ival->use_pos->op_num == 0); in ir_allocate_blocked_reg()
3200 split_pos = ir_find_optimal_split_position(ctx, ival, ival->range.start, next_use_pos - 1, 1); in ir_allocate_blocked_reg()
3203 if (split_pos > ival->range.start) { in ir_allocate_blocked_reg()
3204 IR_LOG_LSRA(" ---- Conflict with others", ival, " (all others are used before)"); in ir_allocate_blocked_reg()
3205 other = ir_split_interval_at(ctx, ival, split_pos); in ir_allocate_blocked_reg()
3206 IR_LOG_LSRA(" ---- Spill", ival, ""); in ir_allocate_blocked_reg()
3213 if (ival->end > blockPos[reg]) { in ir_allocate_blocked_reg()
3215 …IR_LOG_LSRA(" ---- Conflict with others", ival, " (spilling make a register free only for the f… in ir_allocate_blocked_reg()
3217 ir_live_pos split_pos = ir_last_use_pos_before(ival, blockPos[reg] + 1, in ir_allocate_blocked_reg()
3220 split_pos = ir_first_use_pos_after(ival, blockPos[reg], in ir_allocate_blocked_reg()
3222 other = ir_split_interval_at(ctx, ival, split_pos); in ir_allocate_blocked_reg()
3235 IR_LOG_LSRA(" ---- Restart", ival, ""); in ir_allocate_blocked_reg()
3238 split_pos = ir_find_optimal_split_position(ctx, ival, split_pos, blockPos[reg], 1); in ir_allocate_blocked_reg()
3239 other = ir_split_interval_at(ctx, ival, split_pos); in ir_allocate_blocked_reg()
3252 ir_live_pos overlap = ir_ivals_overlap(&ival->range, other->current_range); in ir_allocate_blocked_reg()
3260 …split_pos = ir_last_use_pos_before(other, ival->range.start, IR_USE_MUST_BE_IN_REG | IR_USE_SHOULD… in ir_allocate_blocked_reg()
3262 split_pos = ival->range.start; in ir_allocate_blocked_reg()
3264 split_pos = ir_find_optimal_split_position(ctx, other, split_pos, ival->range.start, 1); in ir_allocate_blocked_reg()
3275 if (!(ival->flags & IR_LIVE_INTERVAL_TEMP)) { in ir_allocate_blocked_reg()
3276 next_use_pos = ir_first_use_pos(ival, IR_USE_MUST_BE_IN_REG); in ir_allocate_blocked_reg()
3277 if (next_use_pos == ival->range.start) { in ir_allocate_blocked_reg()
3278 IR_ASSERT(ival->use_pos && ival->use_pos->op_num == 0); in ir_allocate_blocked_reg()
3282 … split_pos = ir_find_optimal_split_position(ctx, ival, ival->range.start, next_use_pos - 1, 1); in ir_allocate_blocked_reg()
3285 if (split_pos > ival->range.start) { in ir_allocate_blocked_reg()
3301 …split_pos = ir_first_use_pos_after(child, ival->range.start, IR_USE_MUST_BE_IN_REG | IR_USE_SHOULD… in ir_allocate_blocked_reg()
3303 …ir_live_pos opt_split_pos = ir_find_optimal_split_position(ctx, child, ival->range.start, split_po… in ir_allocate_blocked_reg()
3328 ir_live_pos overlap = ir_ivals_overlap(&ival->range, other->current_range); in ir_allocate_blocked_reg()
3347 ival->reg = reg; in ir_allocate_blocked_reg()
3348 IR_LOG_LSRA_ASSIGN(" ---- Assign", ival, " (after splitting others)"); in ir_allocate_blocked_reg()
3350 if (*unhandled && ival->end > (*unhandled)->range.start) { in ir_allocate_blocked_reg()
3351 ival->list_next = *active; in ir_allocate_blocked_reg()
3352 *active = ival; in ir_allocate_blocked_reg()
3397 static bool ir_ival_spill_for_fuse_load(ir_ctx *ctx, ir_live_interval *ival, ir_reg_alloc_data *dat… in ir_ival_spill_for_fuse_load() argument
3399 ir_use_pos *use_pos = ival->use_pos; in ir_ival_spill_for_fuse_load()
3402 if (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM) { in ir_ival_spill_for_fuse_load()
3403 IR_ASSERT(!ival->next && use_pos && use_pos->op_num == 0); in ir_ival_spill_for_fuse_load()
3419 } else if (ival->flags & IR_LIVE_INTERVAL_MEM_LOAD) { in ir_ival_spill_for_fuse_load()
3430 if (bb->loop_depth && bb != ir_block_from_live_pos(ctx, ival->use_pos->pos)) { in ir_ival_spill_for_fuse_load()
3440 if (use > IR_LIVE_POS_TO_REF(ival->use_pos->pos) && use < IR_LIVE_POS_TO_REF(use_pos->pos)) { in ir_ival_spill_for_fuse_load()
3448 ival->stack_spill_pos = ctx->ir_base[insn->op2].op3; in ir_ival_spill_for_fuse_load()
3460 ir_live_interval *ival; in ir_assign_bound_spill_slots() local
3465 ival = ctx->live_intervals[v]; in ir_assign_bound_spill_slots()
3466 if (ival in ir_assign_bound_spill_slots()
3467 && ival->stack_spill_pos == -1 in ir_assign_bound_spill_slots()
3468 && (ival->next || ival->reg == IR_REG_NONE)) { in ir_assign_bound_spill_slots()
3471 ival->stack_spill_pos = -b->val; in ir_assign_bound_spill_slots()
3472 ival->flags |= IR_LIVE_INTERVAL_SPILLED | IR_LIVE_INTERVAL_SPILL_SPECIAL; in ir_assign_bound_spill_slots()
3487 ir_live_interval *ival, *other, *prev; in ir_linear_scan() local
3552 ival = ctx->live_intervals[j]; in ir_linear_scan()
3553 if (ival) { in ir_linear_scan()
3554 if (!(ival->flags & (IR_LIVE_INTERVAL_MEM_PARAM|IR_LIVE_INTERVAL_MEM_LOAD)) in ir_linear_scan()
3555 || !ir_ival_spill_for_fuse_load(ctx, ival, &data)) { in ir_linear_scan()
3556 ir_add_to_unhandled(&unhandled, ival); in ir_linear_scan()
3561 ival = ctx->live_intervals[0]; in ir_linear_scan()
3562 if (ival) { in ir_linear_scan()
3563 ir_merge_to_unhandled(&unhandled, ival); in ir_linear_scan()
3568 ival = ctx->live_intervals[j]; in ir_linear_scan()
3569 if (ival) { in ir_linear_scan()
3570 ival->current_range = &ival->range; in ir_linear_scan()
3571 ival->list_next = inactive; in ir_linear_scan()
3572 inactive = ival; in ir_linear_scan()
3587 ival = unhandled; in ir_linear_scan()
3588 ival->current_range = &ival->range; in ir_linear_scan()
3589 unhandled = ival->list_next; in ir_linear_scan()
3590 position = ival->range.start; in ir_linear_scan()
3592 IR_LOG_LSRA(" ---- Processing", ival, "..."); in ir_linear_scan()
3670 reg = ir_try_allocate_free_reg(ctx, ival, &active, inactive, &unhandled); in ir_linear_scan()
3672 reg = ir_allocate_blocked_reg(ctx, ival, &active, &inactive, &unhandled); in ir_linear_scan()
3678 ival = active; in ir_linear_scan()
3679 while (ival) { in ir_linear_scan()
3680 IR_ASSERT(!ival->next); in ir_linear_scan()
3681 ival = ival->list_next; in ir_linear_scan()
3683 ival = inactive; in ir_linear_scan()
3684 while (ival) { in ir_linear_scan()
3685 IR_ASSERT(!ival->next); in ir_linear_scan()
3686 ival = ival->list_next; in ir_linear_scan()
3699 ival = ctx->live_intervals[j]; in ir_linear_scan()
3700 if (ival in ir_linear_scan()
3701 && (ival->next || ival->reg == IR_REG_NONE) in ir_linear_scan()
3702 && ival->stack_spill_pos == -1) { in ir_linear_scan()
3703 ival->flags |= IR_LIVE_INTERVAL_SPILLED; in ir_linear_scan()
3704 if (!(ival->flags & IR_LIVE_INTERVAL_MEM_PARAM)) { in ir_linear_scan()
3707 other = ival; in ir_linear_scan()
3715 ival->end = r->end; in ir_linear_scan()
3716 ir_add_to_unhandled_spill(&unhandled, ival); in ir_linear_scan()
3729 ival = unhandled; in ir_linear_scan()
3730 ival->current_range = &ival->range; in ir_linear_scan()
3731 unhandled = ival->list_next; in ir_linear_scan()
3732 position = ival->range.start; in ir_linear_scan()
3764 ival->stack_spill_pos = ir_allocate_spill_slot(ctx, ival->type, &data); in ir_linear_scan()
3765 if (unhandled && ival->end > unhandled->range.start) { in ir_linear_scan()
3766 ival->list_next = active; in ir_linear_scan()
3767 active = ival; in ir_linear_scan()
3769 size = ir_type_size[ival->type]; in ir_linear_scan()
3773 if (old->stack_spill_pos == ival->stack_spill_pos) { in ir_linear_scan()
3779 ival->list_next = handled[size]; in ir_linear_scan()
3780 handled[size] = ival; in ir_linear_scan()
3809 static bool needs_spill_reload(ir_ctx *ctx, ir_live_interval *ival, uint32_t b0, ir_bitset availabl… in needs_spill_reload() argument
3829 if (!ir_ival_covers(ival, IR_SAVE_LIVE_POS_FROM_REF(bb->end))) { in needs_spill_reload()
3841 static bool needs_spill_load(ir_ctx *ctx, ir_live_interval *ival, ir_use_pos *use_pos) in needs_spill_load() argument
3870 ir_live_interval *ival, *top_ival; in assign_regs() local
3883 ival = ctx->live_intervals[i]; in assign_regs()
3884 if (ival) { in assign_regs()
3886 if (ival->reg != IR_REG_NONE) { in assign_regs()
3887 reg = ival->reg; in assign_regs()
3889 use_pos = ival->use_pos; in assign_regs()
3896 ival = ival->next; in assign_regs()
3897 } while (ival); in assign_regs()
3904 top_ival = ival = ctx->live_intervals[i]; in assign_regs()
3905 if (ival) { in assign_regs()
3906 if (!(ival->flags & IR_LIVE_INTERVAL_SPILLED)) { in assign_regs()
3908 if (ival->reg != IR_REG_NONE) { in assign_regs()
3909 IR_REGSET_INCL(used_regs, ival->reg); in assign_regs()
3910 use_pos = ival->use_pos; in assign_regs()
3912 reg = ival->reg; in assign_regs()
3922 ival = ival->next; in assign_regs()
3923 } while (ival); in assign_regs()
3926 if (ival->reg != IR_REG_NONE) { in assign_regs()
3930 IR_REGSET_INCL(used_regs, ival->reg); in assign_regs()
3931 use_pos = ival->use_pos; in assign_regs()
3933 reg = ival->reg; in assign_regs()
3954 && (ival->flags & IR_LIVE_INTERVAL_MEM_PARAM)) { in assign_regs()
3960 if (ir_ival_covers(ival, IR_SAVE_LIVE_POS_FROM_REF(ctx->cfg_blocks[use_b].end))) { in assign_regs()
3971 && needs_spill_reload(ctx, ival, ctx->cfg_map[ref], available)) { in assign_regs()
3977 && !needs_spill_load(ctx, ival, use_pos)) { in assign_regs()
4000 if (ir_ival_covers(ival, IR_SAVE_LIVE_POS_FROM_REF(ctx->cfg_blocks[use_b].end))) { in assign_regs()
4049 use_pos = ival->use_pos; in assign_regs()
4061 ival = ival->next; in assign_regs()
4062 } while (ival); in assign_regs()
4070 ival = ctx->live_intervals[0]; in assign_regs()
4071 if (ival) { in assign_regs()
4073 IR_ASSERT(ival->reg != IR_REG_NONE); in assign_regs()
4074 IR_REGSET_INCL(used_regs, ival->reg); in assign_regs()
4075 reg = ival->reg; in assign_regs()
4076 if (ival->tmp_op_num > 0) { in assign_regs()
4077 ir_insn *insn = &ctx->ir_base[ival->tmp_ref]; in assign_regs()
4079 if (ival->tmp_op_num <= insn->inputs_count) { in assign_regs()
4081 if (IR_IS_CONST_REF(ops[ival->tmp_op_num])) { in assign_regs()
4084 } else if (ctx->ir_base[ops[ival->tmp_op_num]].op == IR_ALLOCA in assign_regs()
4085 || ctx->ir_base[ops[ival->tmp_op_num]].op == IR_VADDR) { in assign_regs()
4091 ir_set_alocated_reg(ctx, ival->tmp_ref, ival->tmp_op_num, reg); in assign_regs()
4092 ival = ival->next; in assign_regs()
4093 } while (ival); in assign_regs()
