1 /*
2 * IR - Lightweight JIT Compilation Framework
3 * (IR construction, folding, utilities)
4 * Copyright (C) 2022 Zend by Perforce.
5 * Authors: Dmitry Stogov <dmitry@php.net>
6 *
7 * The logical IR representation is based on Cliff Click's Sea of Nodes.
8 * See: C. Click, M. Paleczny. "A Simple Graph-Based Intermediate
9 * Representation" In ACM SIGPLAN Workshop on Intermediate Representations
10 * (IR '95), pages 35-49, Jan. 1995.
11 *
12 * The physical IR representation is based on Mike Pall's LuaJIT IR.
13 * See: M. Pall. "LuaJIT 2.0 intellectual property disclosure and research
14 * opportunities" November 2009 http://lua-users.org/lists/lua-l/2009-11/msg00089.html
15 */
16
17 #ifndef _GNU_SOURCE
18 # define _GNU_SOURCE
19 #endif
20
21 #ifndef _WIN32
22 # include <sys/mman.h>
23 # if defined(__linux__) || defined(__sun)
24 # include <alloca.h>
25 # endif
26 # if defined(__APPLE__) && defined(__aarch64__)
27 # include <libkern/OSCacheControl.h>
28 # endif
29 #else
30 # define WIN32_LEAN_AND_MEAN
31 # include <windows.h>
32 #endif
33
34 #include "ir.h"
35 #include "ir_private.h"
36
37 #include <stddef.h>
38 #include <stdlib.h>
39 #include <math.h>
40
41 #ifdef HAVE_VALGRIND
42 # include <valgrind/valgrind.h>
43 #endif
44
45 #define IR_TYPE_FLAGS(name, type, field, flags) ((flags)|sizeof(type)),
46 #define IR_TYPE_NAME(name, type, field, flags) #name,
47 #define IR_TYPE_CNAME(name, type, field, flags) #type,
48 #define IR_TYPE_SIZE(name, type, field, flags) sizeof(type),
49 #define IR_OP_NAME(name, flags, op1, op2, op3) #name,
50
51 const uint8_t ir_type_flags[IR_LAST_TYPE] = {
52 0,
53 IR_TYPES(IR_TYPE_FLAGS)
54 };
55
56 const char *ir_type_name[IR_LAST_TYPE] = {
57 "void",
58 IR_TYPES(IR_TYPE_NAME)
59 };
60
61 const uint8_t ir_type_size[IR_LAST_TYPE] = {
62 0,
63 IR_TYPES(IR_TYPE_SIZE)
64 };
65
66 const char *ir_type_cname[IR_LAST_TYPE] = {
67 "void",
68 IR_TYPES(IR_TYPE_CNAME)
69 };
70
71 const char *ir_op_name[IR_LAST_OP] = {
72 IR_OPS(IR_OP_NAME)
73 #ifdef IR_PHP
74 IR_PHP_OPS(IR_OP_NAME)
75 #endif
76 };
77
ir_print_escaped_str(const char * s,size_t len,FILE * f)78 static void ir_print_escaped_str(const char *s, size_t len, FILE *f)
79 {
80 char ch;
81
82 while (len > 0) {
83 ch = *s;
84 switch (ch) {
85 case '\\': fputs("\\\\", f); break;
86 case '\'': fputs("'", f); break;
87 case '\"': fputs("\\\"", f); break;
88 case '\a': fputs("\\a", f); break;
89 case '\b': fputs("\\b", f); break;
90 case '\033': fputs("\\e", f); break;
91 case '\f': fputs("\\f", f); break;
92 case '\n': fputs("\\n", f); break;
93 case '\r': fputs("\\r", f); break;
94 case '\t': fputs("\\t", f); break;
95 case '\v': fputs("\\v", f); break;
96 case '\?': fputs("\\?", f); break;
97 default:
98 if (ch < 32) {
99 fprintf(f, "\\%c%c%c",
100 '0' + ((ch >> 3) % 8),
101 '0' + ((ch >> 6) % 8),
102 '0' + (ch % 8));
103 break;
104 } else {
105 fputc(ch, f);
106 }
107 }
108 s++;
109 len--;
110 }
111 }
112
ir_print_const(const ir_ctx * ctx,const ir_insn * insn,FILE * f,bool quoted)113 void ir_print_const(const ir_ctx *ctx, const ir_insn *insn, FILE *f, bool quoted)
114 {
115 char buf[128];
116
117 if (insn->op == IR_FUNC || insn->op == IR_SYM) {
118 fprintf(f, "%s", ir_get_str(ctx, insn->val.name));
119 return;
120 } else if (insn->op == IR_STR) {
121 size_t len;
122 const char *str = ir_get_strl(ctx, insn->val.str, &len);
123
124 if (quoted) {
125 fprintf(f, "\"");
126 ir_print_escaped_str(str, len, f);
127 fprintf(f, "\"");
128 } else {
129 ir_print_escaped_str(str, len, f);
130 }
131 return;
132 }
133 IR_ASSERT(IR_IS_CONST_OP(insn->op) || insn->op == IR_FUNC_ADDR);
134 switch (insn->type) {
135 case IR_BOOL:
136 fprintf(f, "%u", insn->val.b);
137 break;
138 case IR_U8:
139 fprintf(f, "%u", insn->val.u8);
140 break;
141 case IR_U16:
142 fprintf(f, "%u", insn->val.u16);
143 break;
144 case IR_U32:
145 fprintf(f, "%u", insn->val.u32);
146 break;
147 case IR_U64:
148 fprintf(f, "%" PRIu64, insn->val.u64);
149 break;
150 case IR_ADDR:
151 if (insn->val.addr) {
152 fprintf(f, "0x%" PRIxPTR, insn->val.addr);
153 } else {
154 fprintf(f, "0");
155 }
156 break;
157 case IR_CHAR:
158 if (insn->val.c == '\\') {
159 fprintf(f, "'\\\\'");
160 } else if (insn->val.c >= ' ') {
161 fprintf(f, "'%c'", insn->val.c);
162 } else if (insn->val.c == '\t') {
163 fprintf(f, "'\\t'");
164 } else if (insn->val.c == '\r') {
165 fprintf(f, "'\\r'");
166 } else if (insn->val.c == '\n') {
167 fprintf(f, "'\\n'");
168 } else if (insn->val.c == '\0') {
169 fprintf(f, "'\\0'");
170 } else {
171 fprintf(f, "%u", insn->val.c);
172 }
173 break;
174 case IR_I8:
175 fprintf(f, "%d", insn->val.i8);
176 break;
177 case IR_I16:
178 fprintf(f, "%d", insn->val.i16);
179 break;
180 case IR_I32:
181 fprintf(f, "%d", insn->val.i32);
182 break;
183 case IR_I64:
184 fprintf(f, "%" PRIi64, insn->val.i64);
185 break;
186 case IR_DOUBLE:
187 if (isnan(insn->val.d)) {
188 fprintf(f, "nan");
189 } else {
190 snprintf(buf, sizeof(buf), "%g", insn->val.d);
191 if (strtod(buf, NULL) != insn->val.d) {
192 snprintf(buf, sizeof(buf), "%.53e", insn->val.d);
193 if (strtod(buf, NULL) != insn->val.d) {
194 IR_ASSERT(0 && "can't format double");
195 }
196 }
197 fprintf(f, "%s", buf);
198 }
199 break;
200 case IR_FLOAT:
201 if (isnan(insn->val.f)) {
202 fprintf(f, "nan");
203 } else {
204 snprintf(buf, sizeof(buf), "%g", insn->val.f);
205 if (strtod(buf, NULL) != insn->val.f) {
206 snprintf(buf, sizeof(buf), "%.24e", insn->val.f);
207 if (strtod(buf, NULL) != insn->val.f) {
208 IR_ASSERT(0 && "can't format float");
209 }
210 }
211 fprintf(f, "%s", buf);
212 }
213 break;
214 default:
215 IR_ASSERT(0);
216 break;
217 }
218 }
219
220 #define ir_op_flag_v 0
221 #define ir_op_flag_v0X3 (0 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
222 #define ir_op_flag_d IR_OP_FLAG_DATA
223 #define ir_op_flag_d0 ir_op_flag_d
224 #define ir_op_flag_d1 (ir_op_flag_d | 1 | (1 << IR_OP_FLAG_OPERANDS_SHIFT))
225 #define ir_op_flag_d1X1 (ir_op_flag_d | 1 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
226 #define ir_op_flag_d2 (ir_op_flag_d | 2 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
227 #define ir_op_flag_d2C (ir_op_flag_d | IR_OP_FLAG_COMMUTATIVE | 2 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
228 #define ir_op_flag_d3 (ir_op_flag_d | 3 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
229 #define ir_op_flag_r IR_OP_FLAG_DATA // "d" and "r" are the same now
230 #define ir_op_flag_r0 ir_op_flag_r
231 #define ir_op_flag_p (IR_OP_FLAG_DATA | IR_OP_FLAG_PINNED)
232 #define ir_op_flag_p1 (ir_op_flag_p | 1 | (1 << IR_OP_FLAG_OPERANDS_SHIFT))
233 #define ir_op_flag_p1X1 (ir_op_flag_p | 1 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
234 #define ir_op_flag_p1X2 (ir_op_flag_p | 1 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
235 #define ir_op_flag_p2 (ir_op_flag_p | 2 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
236 #define ir_op_flag_pN (ir_op_flag_p | IR_OP_FLAG_VAR_INPUTS)
237 #define ir_op_flag_c IR_OP_FLAG_CONTROL
238 #define ir_op_flag_c1X2 (ir_op_flag_c | 1 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
239 #define ir_op_flag_c3 (ir_op_flag_c | 3 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
240 #define ir_op_flag_S (IR_OP_FLAG_CONTROL|IR_OP_FLAG_BB_START)
241 #define ir_op_flag_S0X1 (ir_op_flag_S | 0 | (1 << IR_OP_FLAG_OPERANDS_SHIFT))
242 #define ir_op_flag_S1 (ir_op_flag_S | 1 | (1 << IR_OP_FLAG_OPERANDS_SHIFT))
243 #define ir_op_flag_S1X1 (ir_op_flag_S | 1 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
244 #define ir_op_flag_S2 (ir_op_flag_S | 2 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
245 #define ir_op_flag_S2X1 (ir_op_flag_S | 2 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
246 #define ir_op_flag_SN (ir_op_flag_S | IR_OP_FLAG_VAR_INPUTS)
247 #define ir_op_flag_E (IR_OP_FLAG_CONTROL|IR_OP_FLAG_BB_END)
248 #define ir_op_flag_E1 (ir_op_flag_E | 1 | (1 << IR_OP_FLAG_OPERANDS_SHIFT))
249 #define ir_op_flag_E2 (ir_op_flag_E | 2 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
250 #define ir_op_flag_T (IR_OP_FLAG_CONTROL|IR_OP_FLAG_BB_END|IR_OP_FLAG_TERMINATOR)
251 #define ir_op_flag_T2X1 (ir_op_flag_T | 2 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
252 #define ir_op_flag_T1X2 (ir_op_flag_T | 1 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
253 #define ir_op_flag_l (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_LOAD)
254 #define ir_op_flag_l0 ir_op_flag_l
255 #define ir_op_flag_l1 (ir_op_flag_l | 1 | (1 << IR_OP_FLAG_OPERANDS_SHIFT))
256 #define ir_op_flag_l1X1 (ir_op_flag_l | 1 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
257 #define ir_op_flag_l1X2 (ir_op_flag_l | 1 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
258 #define ir_op_flag_l2 (ir_op_flag_l | 2 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
259 #define ir_op_flag_l3 (ir_op_flag_l | 3 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
260 #define ir_op_flag_s (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_STORE)
261 #define ir_op_flag_s0 ir_op_flag_s
262 #define ir_op_flag_s1 (ir_op_flag_s | 1 | (1 << IR_OP_FLAG_OPERANDS_SHIFT))
263 #define ir_op_flag_s2 (ir_op_flag_s | 2 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
264 #define ir_op_flag_s2X1 (ir_op_flag_s | 2 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
265 #define ir_op_flag_s3 (ir_op_flag_s | 3 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
266 #define ir_op_flag_x1 (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_CALL | 1 | (1 << IR_OP_FLAG_OPERANDS_SHIFT))
267 #define ir_op_flag_x2 (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_CALL | 2 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
268 #define ir_op_flag_x3 (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_CALL | 3 | (3 << IR_OP_FLAG_OPERANDS_SHIFT))
269 #define ir_op_flag_xN (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_CALL | IR_OP_FLAG_VAR_INPUTS)
270 #define ir_op_flag_a2 (IR_OP_FLAG_CONTROL|IR_OP_FLAG_MEM|IR_OP_FLAG_MEM_ALLOC | 2 | (2 << IR_OP_FLAG_OPERANDS_SHIFT))
271
272 #define ir_op_kind____ IR_OPND_UNUSED
273 #define ir_op_kind_def IR_OPND_DATA
274 #define ir_op_kind_ref IR_OPND_DATA
275 #define ir_op_kind_src IR_OPND_CONTROL
276 #define ir_op_kind_reg IR_OPND_CONTROL_DEP
277 #define ir_op_kind_ret IR_OPND_CONTROL_REF
278 #define ir_op_kind_str IR_OPND_STR
279 #define ir_op_kind_num IR_OPND_NUM
280 #define ir_op_kind_fld IR_OPND_STR
281 #define ir_op_kind_var IR_OPND_DATA
282 #define ir_op_kind_prb IR_OPND_PROB
283 #define ir_op_kind_opt IR_OPND_PROB
284 #define ir_op_kind_pro IR_OPND_PROTO
285
286 #define _IR_OP_FLAGS(name, flags, op1, op2, op3) \
287 IR_OP_FLAGS(ir_op_flag_ ## flags, ir_op_kind_ ## op1, ir_op_kind_ ## op2, ir_op_kind_ ## op3),
288
289 const uint32_t ir_op_flags[IR_LAST_OP] = {
290 IR_OPS(_IR_OP_FLAGS)
291 #ifdef IR_PHP
292 IR_PHP_OPS(_IR_OP_FLAGS)
293 #endif
294 };
295
ir_grow_bottom(ir_ctx * ctx)296 static void ir_grow_bottom(ir_ctx *ctx)
297 {
298 ir_insn *buf = ctx->ir_base - ctx->consts_limit;
299 ir_ref old_consts_limit = ctx->consts_limit;
300
301 if (ctx->consts_limit < 1024 * 4) {
302 ctx->consts_limit *= 2;
303 } else if (ctx->consts_limit < 1024 * 4 * 2) {
304 ctx->consts_limit = 1024 * 4 * 2;
305 } else {
306 ctx->consts_limit += 1024 * 4;
307 }
308 buf = ir_mem_realloc(buf, (ctx->consts_limit + ctx->insns_limit) * sizeof(ir_insn));
309 memmove(buf + (ctx->consts_limit - old_consts_limit),
310 buf,
311 (old_consts_limit + ctx->insns_count) * sizeof(ir_insn));
312 ctx->ir_base = buf + ctx->consts_limit;
313 }
314
ir_next_const(ir_ctx * ctx)315 static ir_ref ir_next_const(ir_ctx *ctx)
316 {
317 ir_ref ref = ctx->consts_count;
318
319 if (UNEXPECTED(ref >= ctx->consts_limit)) {
320 ir_grow_bottom(ctx);
321 }
322 ctx->consts_count = ref + 1;
323 return -ref;
324 }
325
ir_grow_top(ir_ctx * ctx)326 static void ir_grow_top(ir_ctx *ctx)
327 {
328 ir_insn *buf = ctx->ir_base - ctx->consts_limit;
329
330 if (ctx->insns_limit < 1024 * 4) {
331 ctx->insns_limit *= 2;
332 } else if (ctx->insns_limit < 1024 * 4 * 2) {
333 ctx->insns_limit = 1024 * 4 * 2;
334 } else {
335 ctx->insns_limit += 1024 * 4;
336 }
337 buf = ir_mem_realloc(buf, (ctx->consts_limit + ctx->insns_limit) * sizeof(ir_insn));
338 ctx->ir_base = buf + ctx->consts_limit;
339 }
340
ir_next_insn(ir_ctx * ctx)341 static ir_ref ir_next_insn(ir_ctx *ctx)
342 {
343 ir_ref ref = ctx->insns_count;
344
345 if (UNEXPECTED(ref >= ctx->insns_limit)) {
346 ir_grow_top(ctx);
347 }
348 ctx->insns_count = ref + 1;
349 return ref;
350 }
351
ir_truncate(ir_ctx * ctx)352 void ir_truncate(ir_ctx *ctx)
353 {
354 ir_insn *buf = ir_mem_malloc((ctx->consts_count + ctx->insns_count) * sizeof(ir_insn));
355
356 memcpy(buf, ctx->ir_base - ctx->consts_count, (ctx->consts_count + ctx->insns_count) * sizeof(ir_insn));
357 ir_mem_free(ctx->ir_base - ctx->consts_limit);
358 ctx->insns_limit = ctx->insns_count;
359 ctx->consts_limit = ctx->consts_count;
360 ctx->ir_base = buf + ctx->consts_limit;
361 }
362
ir_init(ir_ctx * ctx,uint32_t flags,ir_ref consts_limit,ir_ref insns_limit)363 void ir_init(ir_ctx *ctx, uint32_t flags, ir_ref consts_limit, ir_ref insns_limit)
364 {
365 ir_insn *buf;
366
367 IR_ASSERT(consts_limit >= IR_CONSTS_LIMIT_MIN);
368 IR_ASSERT(insns_limit >= IR_INSNS_LIMIT_MIN);
369
370 memset(ctx, 0, sizeof(ir_ctx));
371
372 ctx->insns_count = IR_UNUSED + 1;
373 ctx->insns_limit = insns_limit;
374 ctx->consts_count = -(IR_TRUE - 1);
375 ctx->consts_limit = consts_limit;
376 ctx->fold_cse_limit = IR_UNUSED + 1;
377 ctx->flags = flags;
378
379 ctx->spill_base = -1;
380 ctx->fixed_stack_frame_size = -1;
381
382 buf = ir_mem_malloc((consts_limit + insns_limit) * sizeof(ir_insn));
383 ctx->ir_base = buf + consts_limit;
384
385 ctx->ir_base[IR_UNUSED].optx = IR_NOP;
386 ctx->ir_base[IR_NULL].optx = IR_OPT(IR_C_ADDR, IR_ADDR);
387 ctx->ir_base[IR_NULL].val.u64 = 0;
388 ctx->ir_base[IR_FALSE].optx = IR_OPT(IR_C_BOOL, IR_BOOL);
389 ctx->ir_base[IR_FALSE].val.u64 = 0;
390 ctx->ir_base[IR_TRUE].optx = IR_OPT(IR_C_BOOL, IR_BOOL);
391 ctx->ir_base[IR_TRUE].val.u64 = 1;
392 }
393
ir_free(ir_ctx * ctx)394 void ir_free(ir_ctx *ctx)
395 {
396 ir_insn *buf = ctx->ir_base - ctx->consts_limit;
397 ir_mem_free(buf);
398 if (ctx->strtab.data) {
399 ir_strtab_free(&ctx->strtab);
400 }
401 if (ctx->binding) {
402 ir_hashtab_free(ctx->binding);
403 ir_mem_free(ctx->binding);
404 }
405 if (ctx->use_lists) {
406 ir_mem_free(ctx->use_lists);
407 }
408 if (ctx->use_edges) {
409 ir_mem_free(ctx->use_edges);
410 }
411 if (ctx->cfg_blocks) {
412 ir_mem_free(ctx->cfg_blocks);
413 }
414 if (ctx->cfg_edges) {
415 ir_mem_free(ctx->cfg_edges);
416 }
417 if (ctx->cfg_map) {
418 ir_mem_free(ctx->cfg_map);
419 }
420 if (ctx->rules) {
421 ir_mem_free(ctx->rules);
422 }
423 if (ctx->vregs) {
424 ir_mem_free(ctx->vregs);
425 }
426 if (ctx->live_intervals) {
427 ir_mem_free(ctx->live_intervals);
428 }
429 if (ctx->arena) {
430 ir_arena_free(ctx->arena);
431 }
432 if (ctx->regs) {
433 ir_mem_free(ctx->regs);
434 if (ctx->fused_regs) {
435 ir_strtab_free(ctx->fused_regs);
436 ir_mem_free(ctx->fused_regs);
437 }
438 }
439 if (ctx->prev_ref) {
440 ir_mem_free(ctx->prev_ref);
441 }
442 if (ctx->entries) {
443 ir_mem_free(ctx->entries);
444 }
445 if (ctx->osr_entry_loads) {
446 ir_list_free((ir_list*)ctx->osr_entry_loads);
447 ir_mem_free(ctx->osr_entry_loads);
448 }
449 }
450
ir_unique_const_addr(ir_ctx * ctx,uintptr_t addr)451 ir_ref ir_unique_const_addr(ir_ctx *ctx, uintptr_t addr)
452 {
453 ir_ref ref = ir_next_const(ctx);
454 ir_insn *insn = &ctx->ir_base[ref];
455
456 insn->optx = IR_OPT(IR_ADDR, IR_ADDR);
457 insn->val.u64 = addr;
458 /* don't insert into constants chain */
459 insn->prev_const = IR_UNUSED;
460 #if 0
461 insn->prev_const = ctx->prev_const_chain[IR_ADDR];
462 ctx->prev_const_chain[IR_ADDR] = ref;
463 #endif
464 #if 0
465 ir_insn *prev_insn, *next_insn;
466 ir_ref next;
467
468 prev_insn = NULL;
469 next = ctx->prev_const_chain[IR_ADDR];
470 while (next) {
471 next_insn = &ctx->ir_base[next];
472 if (UNEXPECTED(next_insn->val.u64 >= addr)) {
473 break;
474 }
475 prev_insn = next_insn;
476 next = next_insn->prev_const;
477 }
478
479 if (prev_insn) {
480 insn->prev_const = prev_insn->prev_const;
481 prev_insn->prev_const = ref;
482 } else {
483 insn->prev_const = ctx->prev_const_chain[IR_ADDR];
484 ctx->prev_const_chain[IR_ADDR] = ref;
485 }
486 #endif
487
488 return ref;
489 }
490
ir_const_ex(ir_ctx * ctx,ir_val val,uint8_t type,uint32_t optx)491 static IR_NEVER_INLINE ir_ref ir_const_ex(ir_ctx *ctx, ir_val val, uint8_t type, uint32_t optx)
492 {
493 ir_insn *insn, *prev_insn;
494 ir_ref ref, prev;
495
496 if (type == IR_BOOL) {
497 return val.u64 ? IR_TRUE : IR_FALSE;
498 } else if (type == IR_ADDR && val.u64 == 0) {
499 return IR_NULL;
500 }
501 prev_insn = NULL;
502 ref = ctx->prev_const_chain[type];
503 while (ref) {
504 insn = &ctx->ir_base[ref];
505 if (UNEXPECTED(insn->val.u64 >= val.u64)) {
506 if (insn->val.u64 == val.u64) {
507 if (insn->optx == optx) {
508 return ref;
509 }
510 } else {
511 break;
512 }
513 }
514 prev_insn = insn;
515 ref = insn->prev_const;
516 }
517
518 if (prev_insn) {
519 prev = prev_insn->prev_const;
520 prev_insn->prev_const = -ctx->consts_count;
521 } else {
522 prev = ctx->prev_const_chain[type];
523 ctx->prev_const_chain[type] = -ctx->consts_count;
524 }
525
526 ref = ir_next_const(ctx);
527 insn = &ctx->ir_base[ref];
528 insn->prev_const = prev;
529
530 insn->optx = optx;
531 insn->val.u64 = val.u64;
532
533 return ref;
534 }
535
ir_const(ir_ctx * ctx,ir_val val,uint8_t type)536 ir_ref ir_const(ir_ctx *ctx, ir_val val, uint8_t type)
537 {
538 return ir_const_ex(ctx, val, type, IR_OPT(type, type));
539 }
540
ir_const_i8(ir_ctx * ctx,int8_t c)541 ir_ref ir_const_i8(ir_ctx *ctx, int8_t c)
542 {
543 ir_val val;
544 val.i64 = c;
545 return ir_const(ctx, val, IR_I8);
546 }
547
ir_const_i16(ir_ctx * ctx,int16_t c)548 ir_ref ir_const_i16(ir_ctx *ctx, int16_t c)
549 {
550 ir_val val;
551 val.i64 = c;
552 return ir_const(ctx, val, IR_I16);
553 }
554
ir_const_i32(ir_ctx * ctx,int32_t c)555 ir_ref ir_const_i32(ir_ctx *ctx, int32_t c)
556 {
557 ir_val val;
558 val.i64 = c;
559 return ir_const(ctx, val, IR_I32);
560 }
561
ir_const_i64(ir_ctx * ctx,int64_t c)562 ir_ref ir_const_i64(ir_ctx *ctx, int64_t c)
563 {
564 ir_val val;
565 val.i64 = c;
566 return ir_const(ctx, val, IR_I64);
567 }
568
ir_const_u8(ir_ctx * ctx,uint8_t c)569 ir_ref ir_const_u8(ir_ctx *ctx, uint8_t c)
570 {
571 ir_val val;
572 val.u64 = c;
573 return ir_const(ctx, val, IR_U8);
574 }
575
ir_const_u16(ir_ctx * ctx,uint16_t c)576 ir_ref ir_const_u16(ir_ctx *ctx, uint16_t c)
577 {
578 ir_val val;
579 val.u64 = c;
580 return ir_const(ctx, val, IR_U16);
581 }
582
ir_const_u32(ir_ctx * ctx,uint32_t c)583 ir_ref ir_const_u32(ir_ctx *ctx, uint32_t c)
584 {
585 ir_val val;
586 val.u64 = c;
587 return ir_const(ctx, val, IR_U32);
588 }
589
ir_const_u64(ir_ctx * ctx,uint64_t c)590 ir_ref ir_const_u64(ir_ctx *ctx, uint64_t c)
591 {
592 ir_val val;
593 val.u64 = c;
594 return ir_const(ctx, val, IR_U64);
595 }
596
ir_const_bool(ir_ctx * ctx,bool c)597 ir_ref ir_const_bool(ir_ctx *ctx, bool c)
598 {
599 return (c) ? IR_TRUE : IR_FALSE;
600 }
601
ir_const_char(ir_ctx * ctx,char c)602 ir_ref ir_const_char(ir_ctx *ctx, char c)
603 {
604 ir_val val;
605 val.i64 = c;
606 return ir_const(ctx, val, IR_CHAR);
607 }
608
ir_const_float(ir_ctx * ctx,float c)609 ir_ref ir_const_float(ir_ctx *ctx, float c)
610 {
611 ir_val val;
612 val.u32_hi = 0;
613 val.f = c;
614 return ir_const(ctx, val, IR_FLOAT);
615 }
616
ir_const_double(ir_ctx * ctx,double c)617 ir_ref ir_const_double(ir_ctx *ctx, double c)
618 {
619 ir_val val;
620 val.d = c;
621 return ir_const(ctx, val, IR_DOUBLE);
622 }
623
ir_const_addr(ir_ctx * ctx,uintptr_t c)624 ir_ref ir_const_addr(ir_ctx *ctx, uintptr_t c)
625 {
626 if (c == 0) {
627 return IR_NULL;
628 }
629 ir_val val;
630 val.u64 = c;
631 return ir_const(ctx, val, IR_ADDR);
632 }
633
ir_const_func_addr(ir_ctx * ctx,uintptr_t c,ir_ref proto)634 ir_ref ir_const_func_addr(ir_ctx *ctx, uintptr_t c, ir_ref proto)
635 {
636 if (c == 0) {
637 return IR_NULL;
638 }
639 ir_val val;
640 val.u64 = c;
641 IR_ASSERT(proto >= 0 && proto < 0xffff);
642 return ir_const_ex(ctx, val, IR_ADDR, IR_OPTX(IR_FUNC_ADDR, IR_ADDR, proto));
643 }
644
ir_const_func(ir_ctx * ctx,ir_ref str,ir_ref proto)645 ir_ref ir_const_func(ir_ctx *ctx, ir_ref str, ir_ref proto)
646 {
647 ir_val val;
648 val.u64 = str;
649 IR_ASSERT(proto >= 0 && proto < 0xffff);
650 return ir_const_ex(ctx, val, IR_ADDR, IR_OPTX(IR_FUNC, IR_ADDR, proto));
651 }
652
ir_const_sym(ir_ctx * ctx,ir_ref str)653 ir_ref ir_const_sym(ir_ctx *ctx, ir_ref str)
654 {
655 ir_val val;
656 val.u64 = str;
657 return ir_const_ex(ctx, val, IR_ADDR, IR_OPTX(IR_SYM, IR_ADDR, 0));
658 }
659
ir_const_str(ir_ctx * ctx,ir_ref str)660 ir_ref ir_const_str(ir_ctx *ctx, ir_ref str)
661 {
662 ir_val val;
663 val.u64 = str;
664 return ir_const_ex(ctx, val, IR_ADDR, IR_OPTX(IR_STR, IR_ADDR, 0));
665 }
666
ir_str(ir_ctx * ctx,const char * s)667 ir_ref ir_str(ir_ctx *ctx, const char *s)
668 {
669 size_t len;
670
671 if (!ctx->strtab.data) {
672 ir_strtab_init(&ctx->strtab, 64, 4096);
673 }
674 len = strlen(s);
675 IR_ASSERT(len <= 0xffffffff);
676 return ir_strtab_lookup(&ctx->strtab, s, (uint32_t)len, ir_strtab_count(&ctx->strtab) + 1);
677 }
678
ir_strl(ir_ctx * ctx,const char * s,size_t len)679 ir_ref ir_strl(ir_ctx *ctx, const char *s, size_t len)
680 {
681 if (!ctx->strtab.data) {
682 ir_strtab_init(&ctx->strtab, 64, 4096);
683 }
684 IR_ASSERT(len <= 0xffffffff);
685 return ir_strtab_lookup(&ctx->strtab, s, (uint32_t)len, ir_strtab_count(&ctx->strtab) + 1);
686 }
687
ir_get_str(const ir_ctx * ctx,ir_ref idx)688 const char *ir_get_str(const ir_ctx *ctx, ir_ref idx)
689 {
690 IR_ASSERT(ctx->strtab.data);
691 return ir_strtab_str(&ctx->strtab, idx - 1);
692 }
693
ir_get_strl(const ir_ctx * ctx,ir_ref idx,size_t * len)694 const char *ir_get_strl(const ir_ctx *ctx, ir_ref idx, size_t *len)
695 {
696 IR_ASSERT(ctx->strtab.data);
697 return ir_strtab_strl(&ctx->strtab, idx - 1, len);
698 }
699
ir_proto_0(ir_ctx * ctx,uint8_t flags,ir_type ret_type)700 ir_ref ir_proto_0(ir_ctx *ctx, uint8_t flags, ir_type ret_type)
701 {
702 ir_proto_t proto;
703
704 proto.flags = flags;
705 proto.ret_type = ret_type;
706 proto.params_count = 0;
707 return ir_strl(ctx, (const char *)&proto, offsetof(ir_proto_t, param_types) + 0);
708 }
709
ir_proto_1(ir_ctx * ctx,uint8_t flags,ir_type ret_type,ir_type t1)710 ir_ref ir_proto_1(ir_ctx *ctx, uint8_t flags, ir_type ret_type, ir_type t1)
711 {
712 ir_proto_t proto;
713
714 proto.flags = flags;
715 proto.ret_type = ret_type;
716 proto.params_count = 1;
717 proto.param_types[0] = t1;
718 return ir_strl(ctx, (const char *)&proto, offsetof(ir_proto_t, param_types) + 1);
719 }
720
ir_proto_2(ir_ctx * ctx,uint8_t flags,ir_type ret_type,ir_type t1,ir_type t2)721 ir_ref ir_proto_2(ir_ctx *ctx, uint8_t flags, ir_type ret_type, ir_type t1, ir_type t2)
722 {
723 ir_proto_t proto;
724
725 proto.flags = flags;
726 proto.ret_type = ret_type;
727 proto.params_count = 2;
728 proto.param_types[0] = t1;
729 proto.param_types[1] = t2;
730 return ir_strl(ctx, (const char *)&proto, offsetof(ir_proto_t, param_types) + 2);
731 }
732
ir_proto_3(ir_ctx * ctx,uint8_t flags,ir_type ret_type,ir_type t1,ir_type t2,ir_type t3)733 ir_ref ir_proto_3(ir_ctx *ctx, uint8_t flags, ir_type ret_type, ir_type t1, ir_type t2, ir_type t3)
734 {
735 ir_proto_t proto;
736
737 proto.flags = flags;
738 proto.ret_type = ret_type;
739 proto.params_count = 3;
740 proto.param_types[0] = t1;
741 proto.param_types[1] = t2;
742 proto.param_types[2] = t3;
743 return ir_strl(ctx, (const char *)&proto, offsetof(ir_proto_t, param_types) + 3);
744 }
745
ir_proto_4(ir_ctx * ctx,uint8_t flags,ir_type ret_type,ir_type t1,ir_type t2,ir_type t3,ir_type t4)746 ir_ref ir_proto_4(ir_ctx *ctx, uint8_t flags, ir_type ret_type, ir_type t1, ir_type t2, ir_type t3,
747 ir_type t4)
748 {
749 ir_proto_t proto;
750
751 proto.flags = flags;
752 proto.ret_type = ret_type;
753 proto.params_count = 4;
754 proto.param_types[0] = t1;
755 proto.param_types[1] = t2;
756 proto.param_types[2] = t3;
757 proto.param_types[3] = t4;
758 return ir_strl(ctx, (const char *)&proto, offsetof(ir_proto_t, param_types) + 4);
759 }
760
ir_proto_5(ir_ctx * ctx,uint8_t flags,ir_type ret_type,ir_type t1,ir_type t2,ir_type t3,ir_type t4,ir_type t5)761 ir_ref ir_proto_5(ir_ctx *ctx, uint8_t flags, ir_type ret_type, ir_type t1, ir_type t2, ir_type t3,
762 ir_type t4, ir_type t5)
763 {
764 ir_proto_t proto;
765
766 proto.flags = flags;
767 proto.ret_type = ret_type;
768 proto.params_count = 5;
769 proto.param_types[0] = t1;
770 proto.param_types[1] = t2;
771 proto.param_types[2] = t3;
772 proto.param_types[3] = t4;
773 proto.param_types[4] = t5;
774 return ir_strl(ctx, (const char *)&proto, offsetof(ir_proto_t, param_types) + 5);
775 }
776
ir_proto(ir_ctx * ctx,uint8_t flags,ir_type ret_type,uint32_t params_count,uint8_t * param_types)777 ir_ref ir_proto(ir_ctx *ctx, uint8_t flags, ir_type ret_type, uint32_t params_count, uint8_t *param_types)
778 {
779 ir_proto_t *proto = alloca(offsetof(ir_proto_t, param_types) + params_count);
780
781 IR_ASSERT(params_count <= IR_MAX_PROTO_PARAMS);
782 proto->flags = flags;
783 proto->ret_type = ret_type;
784 proto->params_count = params_count;
785 memcpy(proto->param_types, param_types, params_count);
786 return ir_strl(ctx, (const char *)proto, offsetof(ir_proto_t, param_types) + params_count);
787 }
788
789 /* IR construction */
ir_emit(ir_ctx * ctx,uint32_t opt,ir_ref op1,ir_ref op2,ir_ref op3)790 ir_ref ir_emit(ir_ctx *ctx, uint32_t opt, ir_ref op1, ir_ref op2, ir_ref op3)
791 {
792 ir_ref ref = ir_next_insn(ctx);
793 ir_insn *insn = &ctx->ir_base[ref];
794
795 insn->optx = opt;
796 insn->op1 = op1;
797 insn->op2 = op2;
798 insn->op3 = op3;
799
800 return ref;
801 }
802
ir_emit0(ir_ctx * ctx,uint32_t opt)803 ir_ref ir_emit0(ir_ctx *ctx, uint32_t opt)
804 {
805 return ir_emit(ctx, opt, IR_UNUSED, IR_UNUSED, IR_UNUSED);
806 }
807
ir_emit1(ir_ctx * ctx,uint32_t opt,ir_ref op1)808 ir_ref ir_emit1(ir_ctx *ctx, uint32_t opt, ir_ref op1)
809 {
810 return ir_emit(ctx, opt, op1, IR_UNUSED, IR_UNUSED);
811 }
812
ir_emit2(ir_ctx * ctx,uint32_t opt,ir_ref op1,ir_ref op2)813 ir_ref ir_emit2(ir_ctx *ctx, uint32_t opt, ir_ref op1, ir_ref op2)
814 {
815 return ir_emit(ctx, opt, op1, op2, IR_UNUSED);
816 }
817
ir_emit3(ir_ctx * ctx,uint32_t opt,ir_ref op1,ir_ref op2,ir_ref op3)818 ir_ref ir_emit3(ir_ctx *ctx, uint32_t opt, ir_ref op1, ir_ref op2, ir_ref op3)
819 {
820 return ir_emit(ctx, opt, op1, op2, op3);
821 }
822
_ir_fold_cse(ir_ctx * ctx,uint32_t opt,ir_ref op1,ir_ref op2,ir_ref op3)823 static ir_ref _ir_fold_cse(ir_ctx *ctx, uint32_t opt, ir_ref op1, ir_ref op2, ir_ref op3)
824 {
825 ir_ref ref = ctx->prev_insn_chain[opt & IR_OPT_OP_MASK];
826 ir_insn *insn;
827
828 if (ref) {
829 ir_ref limit = ctx->fold_cse_limit;
830
831 if (op1 > limit) {
832 limit = op1;
833 }
834 if (op2 > limit) {
835 limit = op2;
836 }
837 if (op3 > limit) {
838 limit = op3;
839 }
840 while (ref >= limit) {
841 insn = &ctx->ir_base[ref];
842 if (insn->opt == opt && insn->op1 == op1 && insn->op2 == op2 && insn->op3 == op3) {
843 return ref;
844 }
845 if (!insn->prev_insn_offset) {
846 break;
847 }
848 ref = ref - (ir_ref)(uint32_t)insn->prev_insn_offset;
849 }
850 }
851
852 return IR_UNUSED;
853 }
854
855 #define IR_FOLD(X) IR_FOLD1(X, __LINE__)
856 #define IR_FOLD1(X, Y) IR_FOLD2(X, Y)
857 #define IR_FOLD2(X, Y) case IR_RULE_ ## Y:
858
859 #define IR_FOLD_ERROR(msg) do { \
860 IR_ASSERT(0 && (msg)); \
861 goto ir_fold_emit; \
862 } while (0)
863
864 #define IR_FOLD_CONST_U(_val) do { \
865 val.u64 = (_val); \
866 goto ir_fold_const; \
867 } while (0)
868
869 #define IR_FOLD_CONST_I(_val) do { \
870 val.i64 = (_val); \
871 goto ir_fold_const; \
872 } while (0)
873
874 #define IR_FOLD_CONST_D(_val) do { \
875 val.d = (_val); \
876 goto ir_fold_const; \
877 } while (0)
878
879 #define IR_FOLD_CONST_F(_val) do { \
880 val.f = (_val); \
881 val.u32_hi = 0; \
882 goto ir_fold_const; \
883 } while (0)
884
885 #define IR_FOLD_COPY(op) do { \
886 ref = (op); \
887 goto ir_fold_copy; \
888 } while (0)
889
890 #define IR_FOLD_BOOL(cond) \
891 IR_FOLD_COPY((cond) ? IR_TRUE : IR_FALSE)
892
893 #define IR_FOLD_NAMED(name) ir_fold_ ## name:
894 #define IR_FOLD_DO_NAMED(name) goto ir_fold_ ## name
895 #define IR_FOLD_RESTART goto ir_fold_restart
896 #define IR_FOLD_CSE goto ir_fold_cse
897 #define IR_FOLD_EMIT goto ir_fold_emit
898 #define IR_FOLD_NEXT break
899
900 #include "ir_fold_hash.h"
901
902 #define IR_FOLD_RULE(x) ((x) >> 21)
903 #define IR_FOLD_KEY(x) ((x) & 0x1fffff)
904
905 /*
906 * key = insn->op | (insn->op1->op << 7) | (insn->op2->op << 14)
907 *
908 * ANY and UNUSED ops are represented by 0
909 */
910
ir_folding(ir_ctx * ctx,uint32_t opt,ir_ref op1,ir_ref op2,ir_ref op3,ir_insn * op1_insn,ir_insn * op2_insn,ir_insn * op3_insn)911 ir_ref ir_folding(ir_ctx *ctx, uint32_t opt, ir_ref op1, ir_ref op2, ir_ref op3, ir_insn *op1_insn, ir_insn *op2_insn, ir_insn *op3_insn)
912 {
913 uint8_t op;
914 ir_ref ref;
915 ir_val val;
916 uint32_t key, any;
917 (void) op3_insn;
918
919 restart:
920 key = (opt & IR_OPT_OP_MASK) + ((uint32_t)op1_insn->op << 7) + ((uint32_t)op2_insn->op << 14);
921 any = 0x1fffff;
922 do {
923 uint32_t k = key & any;
924 uint32_t h = _ir_fold_hashkey(k);
925 uint32_t fh = _ir_fold_hash[h];
926 if (IR_FOLD_KEY(fh) == k /*|| (fh = _ir_fold_hash[h+1], (fh & 0x1fffff) == k)*/) {
927 switch (IR_FOLD_RULE(fh)) {
928 #include "ir_fold.h"
929 default:
930 break;
931 }
932 }
933 if (any == 0x7f) {
934 /* All parrerns are checked. Pass on to CSE. */
935 goto ir_fold_cse;
936 }
937 /* op2/op1/op op2/_/op _/op1/op _/_/op
938 * 0x1fffff -> 0x1fc07f -> 0x003fff -> 0x00007f
939 * from masks to bis: 11 -> 10 -> 01 -> 00
940 *
941 * a b => x y
942 * 1 1 1 0
943 * 1 0 0 1
944 * 0 1 0 0
945 *
946 * x = a & b; y = !b
947 */
948 any = ((any & (any << 7)) & 0x1fc000) | (~any & 0x3f80) | 0x7f;
949 } while (1);
950
951 ir_fold_restart:
952 if (!(ctx->flags2 & IR_OPT_IN_SCCP)) {
953 op1_insn = ctx->ir_base + op1;
954 op2_insn = ctx->ir_base + op2;
955 op3_insn = ctx->ir_base + op3;
956 goto restart;
957 } else {
958 ctx->fold_insn.optx = opt;
959 ctx->fold_insn.op1 = op1;
960 ctx->fold_insn.op2 = op2;
961 ctx->fold_insn.op3 = op3;
962 return IR_FOLD_DO_RESTART;
963 }
964 ir_fold_cse:
965 if (!(ctx->flags2 & IR_OPT_IN_SCCP)) {
966 /* Local CSE */
967 ref = _ir_fold_cse(ctx, opt, op1, op2, op3);
968 if (ref) {
969 return ref;
970 }
971
972 ref = ir_emit(ctx, opt, op1, op2, op3);
973
974 /* Update local CSE chain */
975 op = opt & IR_OPT_OP_MASK;
976 ir_ref prev = ctx->prev_insn_chain[op];
977 ir_insn *insn = ctx->ir_base + ref;
978 if (!prev || ref - prev > 0xffff) {
979 /* can't fit into 16-bit */
980 insn->prev_insn_offset = 0;
981 } else {
982 insn->prev_insn_offset = ref - prev;
983 }
984 ctx->prev_insn_chain[op] = ref;
985
986 return ref;
987 }
988 ir_fold_emit:
989 if (!(ctx->flags2 & IR_OPT_IN_SCCP)) {
990 return ir_emit(ctx, opt, op1, op2, op3);
991 } else {
992 ctx->fold_insn.optx = opt;
993 ctx->fold_insn.op1 = op1;
994 ctx->fold_insn.op2 = op2;
995 ctx->fold_insn.op3 = op3;
996 return IR_FOLD_DO_EMIT;
997 }
998 ir_fold_copy:
999 if (!(ctx->flags2 & IR_OPT_IN_SCCP)) {
1000 return ref;
1001 } else {
1002 ctx->fold_insn.op1 = ref;
1003 return IR_FOLD_DO_COPY;
1004 }
1005 ir_fold_const:
1006 if (!(ctx->flags2 & IR_OPT_IN_SCCP)) {
1007 return ir_const(ctx, val, IR_OPT_TYPE(opt));
1008 } else {
1009 ctx->fold_insn.type = IR_OPT_TYPE(opt);
1010 ctx->fold_insn.val.u64 = val.u64;
1011 return IR_FOLD_DO_CONST;
1012 }
1013 }
1014
ir_fold(ir_ctx * ctx,uint32_t opt,ir_ref op1,ir_ref op2,ir_ref op3)1015 ir_ref ir_fold(ir_ctx *ctx, uint32_t opt, ir_ref op1, ir_ref op2, ir_ref op3)
1016 {
1017 if (UNEXPECTED(!(ctx->flags & IR_OPT_FOLDING))) {
1018 if ((opt & IR_OPT_OP_MASK) == IR_PHI) {
1019 opt |= (3 << IR_OPT_INPUTS_SHIFT);
1020 }
1021 return ir_emit(ctx, opt, op1, op2, op3);
1022 }
1023 return ir_folding(ctx, opt, op1, op2, op3, ctx->ir_base + op1, ctx->ir_base + op2, ctx->ir_base + op3);
1024 }
1025
ir_fold0(ir_ctx * ctx,uint32_t opt)1026 ir_ref ir_fold0(ir_ctx *ctx, uint32_t opt)
1027 {
1028 return ir_fold(ctx, opt, IR_UNUSED, IR_UNUSED, IR_UNUSED);
1029 }
1030
ir_fold1(ir_ctx * ctx,uint32_t opt,ir_ref op1)1031 ir_ref ir_fold1(ir_ctx *ctx, uint32_t opt, ir_ref op1)
1032 {
1033 return ir_fold(ctx, opt, op1, IR_UNUSED, IR_UNUSED);
1034 }
1035
ir_fold2(ir_ctx * ctx,uint32_t opt,ir_ref op1,ir_ref op2)1036 ir_ref ir_fold2(ir_ctx *ctx, uint32_t opt, ir_ref op1, ir_ref op2)
1037 {
1038 return ir_fold(ctx, opt, op1, op2, IR_UNUSED);
1039 }
1040
ir_fold3(ir_ctx * ctx,uint32_t opt,ir_ref op1,ir_ref op2,ir_ref op3)1041 ir_ref ir_fold3(ir_ctx *ctx, uint32_t opt, ir_ref op1, ir_ref op2, ir_ref op3)
1042 {
1043 return ir_fold(ctx, opt, op1, op2, op3);
1044 }
1045
ir_emit_N(ir_ctx * ctx,uint32_t opt,int32_t count)1046 ir_ref ir_emit_N(ir_ctx *ctx, uint32_t opt, int32_t count)
1047 {
1048 int i;
1049 ir_ref *p, ref = ctx->insns_count;
1050 ir_insn *insn;
1051
1052 IR_ASSERT(count >= 0);
1053 while (UNEXPECTED(ref + count/4 >= ctx->insns_limit)) {
1054 ir_grow_top(ctx);
1055 }
1056 ctx->insns_count = ref + 1 + count/4;
1057
1058 insn = &ctx->ir_base[ref];
1059 insn->optx = opt | (count << IR_OPT_INPUTS_SHIFT);
1060 for (i = 1, p = insn->ops + i; i <= (count|3); i++, p++) {
1061 *p = IR_UNUSED;
1062 }
1063
1064 return ref;
1065 }
1066
ir_set_op(ir_ctx * ctx,ir_ref ref,int32_t n,ir_ref val)1067 void ir_set_op(ir_ctx *ctx, ir_ref ref, int32_t n, ir_ref val)
1068 {
1069 ir_insn *insn = &ctx->ir_base[ref];
1070
1071 #ifdef IR_DEBUG
1072 if (n > 3) {
1073 int32_t count;
1074
1075 IR_ASSERT(IR_OP_HAS_VAR_INPUTS(ir_op_flags[insn->op]));
1076 count = insn->inputs_count;
1077 IR_ASSERT(n <= count);
1078 }
1079 #endif
1080 ir_insn_set_op(insn, n, val);
1081 }
1082
ir_param(ir_ctx * ctx,ir_type type,ir_ref region,const char * name,int pos)1083 ir_ref ir_param(ir_ctx *ctx, ir_type type, ir_ref region, const char *name, int pos)
1084 {
1085 return ir_emit(ctx, IR_OPT(IR_PARAM, type), region, ir_str(ctx, name), pos);
1086 }
1087
ir_var(ir_ctx * ctx,ir_type type,ir_ref region,const char * name)1088 ir_ref ir_var(ir_ctx *ctx, ir_type type, ir_ref region, const char *name)
1089 {
1090 return ir_emit(ctx, IR_OPT(IR_VAR, type), region, ir_str(ctx, name), IR_UNUSED);
1091 }
1092
ir_bind(ir_ctx * ctx,ir_ref var,ir_ref def)1093 ir_ref ir_bind(ir_ctx *ctx, ir_ref var, ir_ref def)
1094 {
1095 if (IR_IS_CONST_REF(def)) {
1096 return def;
1097 }
1098 if (!ctx->binding) {
1099 ctx->binding = ir_mem_malloc(sizeof(ir_hashtab));;
1100 ir_hashtab_init(ctx->binding, 16);
1101 }
1102 /* Node may be bound to some special spill slot (using negative "var") */
1103 IR_ASSERT(var < 0);
1104 if (!ir_hashtab_add(ctx->binding, def, var)) {
1105 /* Add a copy with different binding */
1106 def = ir_emit2(ctx, IR_OPT(IR_COPY, ctx->ir_base[def].type), def, 1);
1107 ir_hashtab_add(ctx->binding, def, var);
1108 }
1109 return def;
1110 }
1111
1112 /* Batch construction of def->use edges */
1113 #if 0
1114 void ir_build_def_use_lists(ir_ctx *ctx)
1115 {
1116 ir_ref n, i, j, *p, def;
1117 ir_insn *insn;
1118 uint32_t edges_count;
1119 ir_use_list *lists = ir_mem_calloc(ctx->insns_count, sizeof(ir_use_list));
1120 ir_ref *edges;
1121 ir_use_list *use_list;
1122
1123 for (i = IR_UNUSED + 1, insn = ctx->ir_base + i; i < ctx->insns_count;) {
1124 uint32_t flags = ir_op_flags[insn->op];
1125
1126 if (UNEXPECTED(IR_OP_HAS_VAR_INPUTS(flags))) {
1127 n = insn->inputs_count;
1128 } else {
1129 n = insn->inputs_count = IR_INPUT_EDGES_COUNT(flags);
1130 }
1131 for (j = n, p = insn->ops + 1; j > 0; j--, p++) {
1132 def = *p;
1133 if (def > 0) {
1134 lists[def].count++;
1135 }
1136 }
1137 n = ir_insn_inputs_to_len(n);
1138 i += n;
1139 insn += n;
1140 }
1141
1142 edges_count = 0;
1143 for (i = IR_UNUSED + 1, use_list = &lists[i]; i < ctx->insns_count; i++, use_list++) {
1144 use_list->refs = edges_count;
1145 edges_count += use_list->count;
1146 use_list->count = 0;
1147 }
1148
1149 edges = ir_mem_malloc(edges_count * sizeof(ir_ref));
1150 for (i = IR_UNUSED + 1, insn = ctx->ir_base + i; i < ctx->insns_count;) {
1151 n = insn->inputs_count;
1152 for (j = n, p = insn->ops + 1; j > 0; j--, p++) {
1153 def = *p;
1154 if (def > 0) {
1155 use_list = &lists[def];
1156 edges[use_list->refs + use_list->count++] = i;
1157 }
1158 }
1159 n = ir_insn_inputs_to_len(n);
1160 i += n;
1161 insn += n;
1162 }
1163
1164 ctx->use_edges = edges;
1165 ctx->use_edges_count = edges_count;
1166 ctx->use_lists = lists;
1167 }
1168 #else
ir_build_def_use_lists(ir_ctx * ctx)1169 void ir_build_def_use_lists(ir_ctx *ctx)
1170 {
1171 ir_ref n, i, j, *p, def;
1172 ir_insn *insn;
1173 size_t linked_lists_size, linked_lists_top = 0, edges_count = 0;
1174 ir_use_list *lists = ir_mem_calloc(ctx->insns_count, sizeof(ir_use_list));
1175 ir_ref *edges;
1176 ir_use_list *use_list;
1177 ir_ref *linked_lists;
1178
1179 linked_lists_size = IR_ALIGNED_SIZE(ctx->insns_count, 1024);
1180 linked_lists = ir_mem_malloc(linked_lists_size * sizeof(ir_ref));
1181 for (i = IR_UNUSED + 1, insn = ctx->ir_base + i; i < ctx->insns_count;) {
1182 uint32_t flags = ir_op_flags[insn->op];
1183
1184 if (UNEXPECTED(IR_OP_HAS_VAR_INPUTS(flags))) {
1185 n = insn->inputs_count;
1186 } else {
1187 n = insn->inputs_count = IR_INPUT_EDGES_COUNT(flags);
1188 }
1189 for (j = n, p = insn->ops + 1; j > 0; j--, p++) {
1190 def = *p;
1191 if (def > 0) {
1192 use_list = &lists[def];
1193 edges_count++;
1194 if (!use_list->refs) {
1195 /* store a single "use" directly in "refs" using a positive number */
1196 use_list->refs = i;
1197 use_list->count = 1;
1198 } else {
1199 if (UNEXPECTED(linked_lists_top >= linked_lists_size)) {
1200 linked_lists_size += 1024;
1201 linked_lists = ir_mem_realloc(linked_lists, linked_lists_size * sizeof(ir_ref));
1202 }
1203 /* form a linked list of "uses" (like in binsort) */
1204 linked_lists[linked_lists_top] = i; /* store the "use" */
1205 linked_lists[linked_lists_top + 1] = use_list->refs; /* store list next */
1206 use_list->refs = -(linked_lists_top + 1); /* store a head of the list using a negative number */
1207 linked_lists_top += 2;
1208 use_list->count++;
1209 }
1210 }
1211 }
1212 n = ir_insn_inputs_to_len(n);
1213 i += n;
1214 insn += n;
1215 }
1216
1217 ctx->use_edges_count = edges_count;
1218 edges = ir_mem_malloc(edges_count * sizeof(ir_ref));
1219 for (use_list = lists + ctx->insns_count - 1; use_list != lists; use_list--) {
1220 n = use_list->refs;
1221 if (n) {
1222 /* transform linked list to plain array */
1223 while (n < 0) {
1224 n = -n;
1225 edges[--edges_count] = linked_lists[n - 1];
1226 n = linked_lists[n];
1227 }
1228 IR_ASSERT(n > 0);
1229 edges[--edges_count] = n;
1230 use_list->refs = edges_count;
1231 }
1232 }
1233
1234 ctx->use_edges = edges;
1235 ctx->use_lists = lists;
1236 ir_mem_free(linked_lists);
1237 }
1238 #endif
1239
1240 /* Helper Data Types */
ir_array_grow(ir_array * a,uint32_t size)1241 void ir_array_grow(ir_array *a, uint32_t size)
1242 {
1243 IR_ASSERT(size > a->size);
1244 a->refs = ir_mem_realloc(a->refs, size * sizeof(ir_ref));
1245 a->size = size;
1246 }
1247
ir_array_insert(ir_array * a,uint32_t i,ir_ref val)1248 void ir_array_insert(ir_array *a, uint32_t i, ir_ref val)
1249 {
1250 IR_ASSERT(i < a->size);
1251 if (a->refs[a->size - 1]) {
1252 ir_array_grow(a, a->size + 1);
1253 }
1254 memmove(a->refs + i + 1, a->refs + i, (a->size - i - 1) * sizeof(ir_ref));
1255 a->refs[i] = val;
1256 }
1257
ir_array_remove(ir_array * a,uint32_t i)1258 void ir_array_remove(ir_array *a, uint32_t i)
1259 {
1260 IR_ASSERT(i < a->size);
1261 memmove(a->refs + i, a->refs + i + 1, (a->size - i - 1) * sizeof(ir_ref));
1262 a->refs[a->size - 1] = IR_UNUSED;
1263 }
1264
ir_list_insert(ir_list * l,uint32_t i,ir_ref val)1265 void ir_list_insert(ir_list *l, uint32_t i, ir_ref val)
1266 {
1267 IR_ASSERT(i < l->len);
1268 if (l->len >= l->a.size) {
1269 ir_array_grow(&l->a, l->a.size + 1);
1270 }
1271 memmove(l->a.refs + i + 1, l->a.refs + i, (l->len - i) * sizeof(ir_ref));
1272 l->a.refs[i] = val;
1273 l->len++;
1274 }
1275
ir_list_remove(ir_list * l,uint32_t i)1276 void ir_list_remove(ir_list *l, uint32_t i)
1277 {
1278 IR_ASSERT(i < l->len);
1279 memmove(l->a.refs + i, l->a.refs + i + 1, (l->len - i) * sizeof(ir_ref));
1280 l->len--;
1281 }
1282
ir_list_contains(const ir_list * l,ir_ref val)1283 bool ir_list_contains(const ir_list *l, ir_ref val)
1284 {
1285 uint32_t i;
1286
1287 for (i = 0; i < l->len; i++) {
1288 if (ir_array_at(&l->a, i) == val) {
1289 return 1;
1290 }
1291 }
1292 return 0;
1293 }
1294
ir_hashtab_hash_size(uint32_t size)1295 static uint32_t ir_hashtab_hash_size(uint32_t size)
1296 {
1297 size -= 1;
1298 size |= (size >> 1);
1299 size |= (size >> 2);
1300 size |= (size >> 4);
1301 size |= (size >> 8);
1302 size |= (size >> 16);
1303 return IR_MAX(size + 1, 4);
1304 }
1305
ir_hashtab_resize(ir_hashtab * tab)1306 static void ir_hashtab_resize(ir_hashtab *tab)
1307 {
1308 uint32_t old_hash_size = (uint32_t)(-(int32_t)tab->mask);
1309 char *old_data = tab->data;
1310 uint32_t size = tab->size * 2;
1311 uint32_t hash_size = ir_hashtab_hash_size(size);
1312 char *data = ir_mem_malloc(hash_size * sizeof(uint32_t) + size * sizeof(ir_hashtab_bucket));
1313 ir_hashtab_bucket *p;
1314 uint32_t pos, i;
1315
1316 memset(data, -1, hash_size * sizeof(uint32_t));
1317 tab->data = data + (hash_size * sizeof(uint32_t));
1318 tab->mask = (uint32_t)(-(int32_t)hash_size);
1319 tab->size = size;
1320
1321 memcpy(tab->data, old_data, tab->count * sizeof(ir_hashtab_bucket));
1322 ir_mem_free(old_data - (old_hash_size * sizeof(uint32_t)));
1323
1324 i = tab->count;
1325 pos = 0;
1326 p = (ir_hashtab_bucket*)tab->data;
1327 do {
1328 uint32_t key = p->key | tab->mask;
1329 p->next = ((uint32_t*)tab->data)[(int32_t)key];
1330 ((uint32_t*)tab->data)[(int32_t)key] = pos;
1331 pos += sizeof(ir_hashtab_bucket);
1332 p++;
1333 } while (--i);
1334 }
1335
ir_hashtab_init(ir_hashtab * tab,uint32_t size)1336 void ir_hashtab_init(ir_hashtab *tab, uint32_t size)
1337 {
1338 IR_ASSERT(size > 0);
1339 uint32_t hash_size = ir_hashtab_hash_size(size);
1340 char *data = ir_mem_malloc(hash_size * sizeof(uint32_t) + size * sizeof(ir_hashtab_bucket));
1341 memset(data, -1, hash_size * sizeof(uint32_t));
1342 tab->data = (data + (hash_size * sizeof(uint32_t)));
1343 tab->mask = (uint32_t)(-(int32_t)hash_size);
1344 tab->size = size;
1345 tab->count = 0;
1346 tab->pos = 0;
1347 }
1348
ir_hashtab_free(ir_hashtab * tab)1349 void ir_hashtab_free(ir_hashtab *tab)
1350 {
1351 uint32_t hash_size = (uint32_t)(-(int32_t)tab->mask);
1352 char *data = (char*)tab->data - (hash_size * sizeof(uint32_t));
1353 ir_mem_free(data);
1354 tab->data = NULL;
1355 }
1356
ir_hashtab_find(const ir_hashtab * tab,uint32_t key)1357 ir_ref ir_hashtab_find(const ir_hashtab *tab, uint32_t key)
1358 {
1359 const char *data = (const char*)tab->data;
1360 uint32_t pos = ((uint32_t*)data)[(int32_t)(key | tab->mask)];
1361 ir_hashtab_bucket *p;
1362
1363 while (pos != IR_INVALID_IDX) {
1364 p = (ir_hashtab_bucket*)(data + pos);
1365 if (p->key == key) {
1366 return p->val;
1367 }
1368 pos = p->next;
1369 }
1370 return IR_INVALID_VAL;
1371 }
1372
ir_hashtab_add(ir_hashtab * tab,uint32_t key,ir_ref val)1373 bool ir_hashtab_add(ir_hashtab *tab, uint32_t key, ir_ref val)
1374 {
1375 char *data = (char*)tab->data;
1376 uint32_t pos = ((uint32_t*)data)[(int32_t)(key | tab->mask)];
1377 ir_hashtab_bucket *p;
1378
1379 while (pos != IR_INVALID_IDX) {
1380 p = (ir_hashtab_bucket*)(data + pos);
1381 if (p->key == key) {
1382 return p->val == val;
1383 }
1384 pos = p->next;
1385 }
1386
1387 if (UNEXPECTED(tab->count >= tab->size)) {
1388 ir_hashtab_resize(tab);
1389 data = tab->data;
1390 }
1391
1392 pos = tab->pos;
1393 tab->pos += sizeof(ir_hashtab_bucket);
1394 tab->count++;
1395 p = (ir_hashtab_bucket*)(data + pos);
1396 p->key = key;
1397 p->val = val;
1398 key |= tab->mask;
1399 p->next = ((uint32_t*)data)[(int32_t)key];
1400 ((uint32_t*)data)[(int32_t)key] = pos;
1401 return 1;
1402 }
1403
ir_hashtab_key_cmp(const void * b1,const void * b2)1404 static int ir_hashtab_key_cmp(const void *b1, const void *b2)
1405 {
1406 return ((ir_hashtab_bucket*)b1)->key - ((ir_hashtab_bucket*)b2)->key;
1407 }
1408
ir_hashtab_key_sort(ir_hashtab * tab)1409 void ir_hashtab_key_sort(ir_hashtab *tab)
1410 {
1411 ir_hashtab_bucket *p;
1412 uint32_t hash_size, pos, i;
1413
1414 if (!tab->count) {
1415 return;
1416 }
1417
1418 qsort(tab->data, tab->count, sizeof(ir_hashtab_bucket), ir_hashtab_key_cmp);
1419
1420 hash_size = ir_hashtab_hash_size(tab->size);
1421 memset((char*)tab->data - (hash_size * sizeof(uint32_t)), -1, hash_size * sizeof(uint32_t));
1422
1423 i = tab->count;
1424 pos = 0;
1425 p = (ir_hashtab_bucket*)tab->data;
1426 do {
1427 uint32_t key = p->key | tab->mask;
1428 p->next = ((uint32_t*)tab->data)[(int32_t)key];
1429 ((uint32_t*)tab->data)[(int32_t)key] = pos;
1430 pos += sizeof(ir_hashtab_bucket);
1431 p++;
1432 } while (--i);
1433 }
1434
ir_addrtab_resize(ir_hashtab * tab)1435 static void ir_addrtab_resize(ir_hashtab *tab)
1436 {
1437 uint32_t old_hash_size = (uint32_t)(-(int32_t)tab->mask);
1438 char *old_data = tab->data;
1439 uint32_t size = tab->size * 2;
1440 uint32_t hash_size = ir_hashtab_hash_size(size);
1441 char *data = ir_mem_malloc(hash_size * sizeof(uint32_t) + size * sizeof(ir_addrtab_bucket));
1442 ir_addrtab_bucket *p;
1443 uint32_t pos, i;
1444
1445 memset(data, -1, hash_size * sizeof(uint32_t));
1446 tab->data = data + (hash_size * sizeof(uint32_t));
1447 tab->mask = (uint32_t)(-(int32_t)hash_size);
1448 tab->size = size;
1449
1450 memcpy(tab->data, old_data, tab->count * sizeof(ir_addrtab_bucket));
1451 ir_mem_free(old_data - (old_hash_size * sizeof(uint32_t)));
1452
1453 i = tab->count;
1454 pos = 0;
1455 p = (ir_addrtab_bucket*)tab->data;
1456 do {
1457 uint32_t key = (uint32_t)p->key | tab->mask;
1458 p->next = ((uint32_t*)tab->data)[(int32_t)key];
1459 ((uint32_t*)tab->data)[(int32_t)key] = pos;
1460 pos += sizeof(ir_addrtab_bucket);
1461 p++;
1462 } while (--i);
1463 }
1464
ir_addrtab_init(ir_hashtab * tab,uint32_t size)1465 void ir_addrtab_init(ir_hashtab *tab, uint32_t size)
1466 {
1467 IR_ASSERT(size > 0);
1468 uint32_t hash_size = ir_hashtab_hash_size(size);
1469 char *data = ir_mem_malloc(hash_size * sizeof(uint32_t) + size * sizeof(ir_addrtab_bucket));
1470 memset(data, -1, hash_size * sizeof(uint32_t));
1471 tab->data = (data + (hash_size * sizeof(uint32_t)));
1472 tab->mask = (uint32_t)(-(int32_t)hash_size);
1473 tab->size = size;
1474 tab->count = 0;
1475 tab->pos = 0;
1476 }
1477
ir_addrtab_free(ir_hashtab * tab)1478 void ir_addrtab_free(ir_hashtab *tab)
1479 {
1480 uint32_t hash_size = (uint32_t)(-(int32_t)tab->mask);
1481 char *data = (char*)tab->data - (hash_size * sizeof(uint32_t));
1482 ir_mem_free(data);
1483 tab->data = NULL;
1484 }
1485
ir_addrtab_find(const ir_hashtab * tab,uint64_t key)1486 ir_ref ir_addrtab_find(const ir_hashtab *tab, uint64_t key)
1487 {
1488 const char *data = (const char*)tab->data;
1489 uint32_t pos = ((uint32_t*)data)[(int32_t)(key | tab->mask)];
1490 ir_addrtab_bucket *p;
1491
1492 while (pos != IR_INVALID_IDX) {
1493 p = (ir_addrtab_bucket*)(data + pos);
1494 if (p->key == key) {
1495 return p->val;
1496 }
1497 pos = p->next;
1498 }
1499 return IR_INVALID_VAL;
1500 }
1501
ir_addrtab_add(ir_hashtab * tab,uint64_t key,ir_ref val)1502 bool ir_addrtab_add(ir_hashtab *tab, uint64_t key, ir_ref val)
1503 {
1504 char *data = (char*)tab->data;
1505 uint32_t pos = ((uint32_t*)data)[(int32_t)(key | tab->mask)];
1506 ir_addrtab_bucket *p;
1507
1508 while (pos != IR_INVALID_IDX) {
1509 p = (ir_addrtab_bucket*)(data + pos);
1510 if (p->key == key) {
1511 return p->val == val;
1512 }
1513 pos = p->next;
1514 }
1515
1516 if (UNEXPECTED(tab->count >= tab->size)) {
1517 ir_addrtab_resize(tab);
1518 data = tab->data;
1519 }
1520
1521 pos = tab->pos;
1522 tab->pos += sizeof(ir_addrtab_bucket);
1523 tab->count++;
1524 p = (ir_addrtab_bucket*)(data + pos);
1525 p->key = key;
1526 p->val = val;
1527 key |= tab->mask;
1528 p->next = ((uint32_t*)data)[(int32_t)key];
1529 ((uint32_t*)data)[(int32_t)key] = pos;
1530 return 1;
1531 }
1532
1533 /* Memory API */
1534 #ifdef _WIN32
ir_mem_mmap(size_t size)1535 void *ir_mem_mmap(size_t size)
1536 {
1537 void *ret;
1538
1539 #ifdef _M_X64
1540 DWORD size_hi = size >> 32, size_lo = size & 0xffffffff;
1541 #else
1542 DWORD size_hi = 0, size_lo = size;
1543 #endif
1544
1545 HANDLE h = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, size_hi, size_lo, NULL);
1546
1547 ret = MapViewOfFile(h, FILE_MAP_READ | FILE_MAP_WRITE | FILE_MAP_EXECUTE, 0, 0, size);
1548 if (!ret) {
1549 CloseHandle(h);
1550 }
1551
1552 return ret;
1553 }
1554
ir_mem_unmap(void * ptr,size_t size)1555 int ir_mem_unmap(void *ptr, size_t size)
1556 {
1557 /* XXX file handle is leaked. */
1558 UnmapViewOfFile(ptr);
1559 return 1;
1560 }
1561
ir_mem_protect(void * ptr,size_t size)1562 int ir_mem_protect(void *ptr, size_t size)
1563 {
1564 return 1;
1565 }
1566
ir_mem_unprotect(void * ptr,size_t size)1567 int ir_mem_unprotect(void *ptr, size_t size)
1568 {
1569 return 1;
1570 }
1571
ir_mem_flush(void * ptr,size_t size)1572 int ir_mem_flush(void *ptr, size_t size)
1573 {
1574 return 1;
1575 }
1576 #else
ir_mem_mmap(size_t size)1577 void *ir_mem_mmap(size_t size)
1578 {
1579 int prot_flags = PROT_EXEC;
1580 #if defined(__NetBSD__)
1581 prot_flags |= PROT_MPROTECT(PROT_READ|PROT_WRITE);
1582 #endif
1583 void *ret = mmap(NULL, size, prot_flags, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1584 if (ret == MAP_FAILED) {
1585 ret = NULL;
1586 }
1587 return ret;
1588 }
1589
ir_mem_unmap(void * ptr,size_t size)1590 int ir_mem_unmap(void *ptr, size_t size)
1591 {
1592 munmap(ptr, size);
1593 return 1;
1594 }
1595
ir_mem_protect(void * ptr,size_t size)1596 int ir_mem_protect(void *ptr, size_t size)
1597 {
1598 if (mprotect(ptr, size, PROT_READ | PROT_EXEC) != 0) {
1599 #ifdef IR_DEBUG
1600 fprintf(stderr, "mprotect() failed\n");
1601 #endif
1602 return 0;
1603 }
1604 return 1;
1605 }
1606
ir_mem_unprotect(void * ptr,size_t size)1607 int ir_mem_unprotect(void *ptr, size_t size)
1608 {
1609 if (mprotect(ptr, size, PROT_READ | PROT_WRITE) != 0) {
1610 #ifdef IR_DEBUG
1611 fprintf(stderr, "mprotect() failed\n");
1612 #endif
1613 return 0;
1614 }
1615 return 1;
1616 }
1617
ir_mem_flush(void * ptr,size_t size)1618 int ir_mem_flush(void *ptr, size_t size)
1619 {
1620 #if ((defined(__GNUC__) && ZEND_GCC_VERSION >= 4003) || __has_builtin(__builtin___clear_cache))
1621 __builtin___clear_cache((char*)(ptr), (char*)(ptr) + size);
1622 #endif
1623 #if defined(__APPLE__) && defined(__aarch64__)
1624 sys_icache_invalidate(ptr, size);
1625 #endif
1626 #ifdef HAVE_VALGRIND
1627 VALGRIND_DISCARD_TRANSLATIONS(ptr, size);
1628 #endif
1629 return 1;
1630 }
1631 #endif
1632
1633 /* Alias Analyses */
1634 typedef enum _ir_alias {
1635 IR_MAY_ALIAS = -1,
1636 IR_NO_ALIAS = 0,
1637 IR_MUST_ALIAS = 1,
1638 } ir_alias;
1639
1640 #if 0
1641 static ir_alias ir_check_aliasing(ir_ctx *ctx, ir_ref addr1, ir_ref addr2)
1642 {
1643 ir_insn *insn1, *insn2;
1644
1645 if (addr1 == addr2) {
1646 return IR_MUST_ALIAS;
1647 }
1648
1649 insn1 = &ctx->ir_base[addr1];
1650 insn2 = &ctx->ir_base[addr2];
1651 if (insn1->op == IR_ADD && IR_IS_CONST_REF(insn1->op2)) {
1652 if (insn1->op1 == addr2) {
1653 uintptr_t offset1 = ctx->ir_base[insn1->op2].val.u64;
1654 return (offset1 != 0) ? IR_MUST_ALIAS : IR_NO_ALIAS;
1655 } else if (insn2->op == IR_ADD && IR_IS_CONST_REF(insn1->op2) && insn1->op1 == insn2->op1) {
1656 if (insn1->op2 == insn2->op2) {
1657 return IR_MUST_ALIAS;
1658 } else if (IR_IS_CONST_REF(insn1->op2) && IR_IS_CONST_REF(insn2->op2)) {
1659 uintptr_t offset1 = ctx->ir_base[insn1->op2].val.u64;
1660 uintptr_t offset2 = ctx->ir_base[insn2->op2].val.u64;
1661
1662 return (offset1 == offset2) ? IR_MUST_ALIAS : IR_NO_ALIAS;
1663 }
1664 }
1665 } else if (insn2->op == IR_ADD && IR_IS_CONST_REF(insn2->op2)) {
1666 if (insn2->op1 == addr1) {
1667 uintptr_t offset2 = ctx->ir_base[insn2->op2].val.u64;
1668
1669 return (offset2 != 0) ? IR_MUST_ALIAS : IR_NO_ALIAS;
1670 }
1671 }
1672 return IR_MAY_ALIAS;
1673 }
1674 #endif
1675
ir_check_partial_aliasing(const ir_ctx * ctx,ir_ref addr1,ir_ref addr2,ir_type type1,ir_type type2)1676 static ir_alias ir_check_partial_aliasing(const ir_ctx *ctx, ir_ref addr1, ir_ref addr2, ir_type type1, ir_type type2)
1677 {
1678 ir_insn *insn1, *insn2;
1679
1680 /* this must be already check */
1681 IR_ASSERT(addr1 != addr2);
1682
1683 insn1 = &ctx->ir_base[addr1];
1684 insn2 = &ctx->ir_base[addr2];
1685 if (insn1->op == IR_ADD && IR_IS_CONST_REF(insn1->op2)) {
1686 if (insn1->op1 == addr2) {
1687 uintptr_t offset1 = ctx->ir_base[insn1->op2].val.addr;
1688 uintptr_t size2 = ir_type_size[type2];
1689
1690 return (offset1 < size2) ? IR_MUST_ALIAS : IR_NO_ALIAS;
1691 } else if (insn2->op == IR_ADD && IR_IS_CONST_REF(insn1->op2) && insn1->op1 == insn2->op1) {
1692 if (insn1->op2 == insn2->op2) {
1693 return IR_MUST_ALIAS;
1694 } else if (IR_IS_CONST_REF(insn1->op2) && IR_IS_CONST_REF(insn2->op2)) {
1695 uintptr_t offset1 = ctx->ir_base[insn1->op2].val.addr;
1696 uintptr_t offset2 = ctx->ir_base[insn2->op2].val.addr;
1697
1698 if (offset1 == offset2) {
1699 return IR_MUST_ALIAS;
1700 } else if (type1 == type2) {
1701 return IR_NO_ALIAS;
1702 } else {
1703 /* check for partail intersection */
1704 uintptr_t size1 = ir_type_size[type1];
1705 uintptr_t size2 = ir_type_size[type2];
1706
1707 if (offset1 > offset2) {
1708 return offset1 < offset2 + size2 ? IR_MUST_ALIAS : IR_NO_ALIAS;
1709 } else {
1710 return offset2 < offset1 + size1 ? IR_MUST_ALIAS : IR_NO_ALIAS;
1711 }
1712 }
1713 }
1714 }
1715 } else if (insn2->op == IR_ADD && IR_IS_CONST_REF(insn2->op2)) {
1716 if (insn2->op1 == addr1) {
1717 uintptr_t offset2 = ctx->ir_base[insn2->op2].val.addr;
1718 uintptr_t size1 = ir_type_size[type1];
1719
1720 return (offset2 < size1) ? IR_MUST_ALIAS : IR_NO_ALIAS;
1721 }
1722 }
1723 return IR_MAY_ALIAS;
1724 }
1725
ir_find_aliasing_load(ir_ctx * ctx,ir_ref ref,ir_type type,ir_ref addr)1726 static ir_ref ir_find_aliasing_load(ir_ctx *ctx, ir_ref ref, ir_type type, ir_ref addr)
1727 {
1728 ir_ref limit = (addr > 0) ? addr : 1;
1729 ir_insn *insn;
1730 uint32_t modified_regset = 0;
1731
1732 while (ref > limit) {
1733 insn = &ctx->ir_base[ref];
1734 if (insn->op == IR_LOAD) {
1735 if (insn->type == type && insn->op2 == addr) {
1736 return ref; /* load forwarding (L2L) */
1737 }
1738 } else if (insn->op == IR_STORE) {
1739 ir_type type2 = ctx->ir_base[insn->op3].type;
1740
1741 if (insn->op2 == addr) {
1742 if (type2 == type) {
1743 ref = insn->op3;
1744 insn = &ctx->ir_base[ref];
1745 if (insn->op == IR_RLOAD && (modified_regset & (1 << insn->op2))) {
1746 /* anti-dependency */
1747 return IR_UNUSED;
1748 }
1749 return ref; /* store forwarding (S2L) */
1750 } else if (IR_IS_TYPE_INT(type) && ir_type_size[type2] > ir_type_size[type]) {
1751 return ir_fold1(ctx, IR_OPT(IR_TRUNC, type), insn->op3); /* partial store forwarding (S2L) */
1752 } else {
1753 return IR_UNUSED;
1754 }
1755 } else if (ir_check_partial_aliasing(ctx, addr, insn->op2, type, type2) != IR_NO_ALIAS) {
1756 return IR_UNUSED;
1757 }
1758 } else if (insn->op == IR_RSTORE) {
1759 modified_regset |= (1 << insn->op3);
1760 } else if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN || insn->op == IR_CALL || insn->op == IR_VSTORE) {
1761 return IR_UNUSED;
1762 }
1763 ref = insn->op1;
1764 }
1765 return IR_UNUSED;
1766 }
1767
1768 /* IR Construction API */
1769
_ir_PARAM(ir_ctx * ctx,ir_type type,const char * name,ir_ref num)1770 ir_ref _ir_PARAM(ir_ctx *ctx, ir_type type, const char* name, ir_ref num)
1771 {
1772 IR_ASSERT(ctx->control);
1773 IR_ASSERT(ctx->ir_base[ctx->control].op == IR_START);
1774 IR_ASSERT(ctx->insns_count == num + 1);
1775 return ir_param(ctx, type, ctx->control, name, num);
1776 }
1777
_ir_VAR(ir_ctx * ctx,ir_type type,const char * name)1778 ir_ref _ir_VAR(ir_ctx *ctx, ir_type type, const char* name)
1779 {
1780 // IR_ASSERT(ctx->control);
1781 // IR_ASSERT(IR_IS_BB_START(ctx->ir_base[ctx->control].op));
1782 // TODO: VAR may be insterted after some "memory" instruction
1783 ir_ref ref = ctx->control;
1784
1785 while (1) {
1786 IR_ASSERT(ctx->control);
1787 if (IR_IS_BB_START(ctx->ir_base[ref].op)) {
1788 break;
1789 }
1790 ref = ctx->ir_base[ref].op1;
1791 }
1792 return ir_var(ctx, type, ref, name);
1793 }
1794
_ir_PHI_2(ir_ctx * ctx,ir_type type,ir_ref src1,ir_ref src2)1795 ir_ref _ir_PHI_2(ir_ctx *ctx, ir_type type, ir_ref src1, ir_ref src2)
1796 {
1797 IR_ASSERT(ctx->control);
1798 IR_ASSERT(ctx->ir_base[ctx->control].op == IR_MERGE || ctx->ir_base[ctx->control].op == IR_LOOP_BEGIN);
1799 if (src1 == src2 && src1 != IR_UNUSED) {
1800 return src1;
1801 }
1802 return ir_emit3(ctx, IR_OPTX(IR_PHI, type, 3), ctx->control, src1, src2);
1803 }
1804
_ir_PHI_N(ir_ctx * ctx,ir_type type,ir_ref n,ir_ref * inputs)1805 ir_ref _ir_PHI_N(ir_ctx *ctx, ir_type type, ir_ref n, ir_ref *inputs)
1806 {
1807 IR_ASSERT(ctx->control);
1808 IR_ASSERT(n > 0);
1809 if (n == 1) {
1810 return inputs[0];
1811 } else {
1812 ir_ref i;
1813 ir_ref ref = inputs[0];
1814
1815 IR_ASSERT(ctx->ir_base[ctx->control].op == IR_MERGE || ctx->ir_base[ctx->control].op == IR_LOOP_BEGIN);
1816 if (ref != IR_UNUSED) {
1817 for (i = 1; i < n; i++) {
1818 if (inputs[i] != ref) {
1819 break;
1820 }
1821 }
1822 if (i == n) {
1823 /* all the same */
1824 return ref;
1825 }
1826 }
1827
1828 ref = ir_emit_N(ctx, IR_OPT(IR_PHI, type), n + 1);
1829 ir_set_op(ctx, ref, 1, ctx->control);
1830 for (i = 0; i < n; i++) {
1831 ir_set_op(ctx, ref, i + 2, inputs[i]);
1832 }
1833 return ref;
1834 }
1835 }
1836
_ir_PHI_SET_OP(ir_ctx * ctx,ir_ref phi,ir_ref pos,ir_ref src)1837 void _ir_PHI_SET_OP(ir_ctx *ctx, ir_ref phi, ir_ref pos, ir_ref src)
1838 {
1839 ir_insn *insn = &ctx->ir_base[phi];
1840 ir_ref *ops = insn->ops;
1841
1842 IR_ASSERT(insn->op == IR_PHI);
1843 IR_ASSERT(ctx->ir_base[insn->op1].op == IR_MERGE || ctx->ir_base[insn->op1].op == IR_LOOP_BEGIN);
1844 IR_ASSERT(pos > 0 && pos < insn->inputs_count);
1845 pos++; /* op1 is used for control */
1846 ops[pos] = src;
1847 }
1848
_ir_START(ir_ctx * ctx)1849 void _ir_START(ir_ctx *ctx)
1850 {
1851 IR_ASSERT(!ctx->control);
1852 IR_ASSERT(ctx->insns_count == 1);
1853 ctx->control = ir_emit0(ctx, IR_START);
1854 }
1855
_ir_ENTRY(ir_ctx * ctx,ir_ref src,ir_ref num)1856 void _ir_ENTRY(ir_ctx *ctx, ir_ref src, ir_ref num)
1857 {
1858 IR_ASSERT(!ctx->control);
1859 /* fake control edge */
1860 IR_ASSERT((ir_op_flags[ctx->ir_base[src].op] & IR_OP_FLAG_TERMINATOR)
1861 || ctx->ir_base[src].op == IR_END
1862 || ctx->ir_base[src].op == IR_LOOP_END); /* return from a recursive call */
1863 ctx->control = ir_emit2(ctx, IR_ENTRY, src, num);
1864 }
1865
_ir_BEGIN(ir_ctx * ctx,ir_ref src)1866 void _ir_BEGIN(ir_ctx *ctx, ir_ref src)
1867 {
1868 IR_ASSERT(!ctx->control);
1869 if (src
1870 && src + 1 == ctx->insns_count
1871 && ctx->ir_base[src].op == IR_END) {
1872 /* merge with the last END */
1873 ctx->control = ctx->ir_base[src].op1;
1874 ctx->insns_count--;
1875 } else {
1876 ctx->control = ir_emit1(ctx, IR_BEGIN, src);
1877 }
1878 }
1879
_ir_IF(ir_ctx * ctx,ir_ref condition)1880 ir_ref _ir_IF(ir_ctx *ctx, ir_ref condition)
1881 {
1882 ir_ref if_ref;
1883
1884 IR_ASSERT(ctx->control);
1885 if_ref = ir_emit2(ctx, IR_IF, ctx->control, condition);
1886 ctx->control = IR_UNUSED;
1887 return if_ref;
1888 }
1889
_ir_IF_TRUE(ir_ctx * ctx,ir_ref if_ref)1890 void _ir_IF_TRUE(ir_ctx *ctx, ir_ref if_ref)
1891 {
1892 IR_ASSERT(!ctx->control);
1893 IR_ASSERT(if_ref);
1894 IR_ASSERT(ctx->ir_base[if_ref].op == IR_IF);
1895 ctx->control = ir_emit1(ctx, IR_IF_TRUE, if_ref);
1896 }
1897
_ir_IF_TRUE_cold(ir_ctx * ctx,ir_ref if_ref)1898 void _ir_IF_TRUE_cold(ir_ctx *ctx, ir_ref if_ref)
1899 {
1900 IR_ASSERT(!ctx->control);
1901 IR_ASSERT(if_ref);
1902 IR_ASSERT(ctx->ir_base[if_ref].op == IR_IF);
1903 /* op2 is used as an indicator of low-probability branch */
1904 ctx->control = ir_emit2(ctx, IR_IF_TRUE, if_ref, 1);
1905 }
1906
_ir_IF_FALSE(ir_ctx * ctx,ir_ref if_ref)1907 void _ir_IF_FALSE(ir_ctx *ctx, ir_ref if_ref)
1908 {
1909 IR_ASSERT(!ctx->control);
1910 IR_ASSERT(if_ref);
1911 IR_ASSERT(ctx->ir_base[if_ref].op == IR_IF);
1912 ctx->control = ir_emit1(ctx, IR_IF_FALSE, if_ref);
1913 }
1914
_ir_IF_FALSE_cold(ir_ctx * ctx,ir_ref if_ref)1915 void _ir_IF_FALSE_cold(ir_ctx *ctx, ir_ref if_ref)
1916 {
1917 IR_ASSERT(!ctx->control);
1918 IR_ASSERT(if_ref);
1919 IR_ASSERT(ctx->ir_base[if_ref].op == IR_IF);
1920 /* op2 is used as an indicator of low-probability branch */
1921 ctx->control = ir_emit2(ctx, IR_IF_FALSE, if_ref, 1);
1922 }
1923
_ir_END(ir_ctx * ctx)1924 ir_ref _ir_END(ir_ctx *ctx)
1925 {
1926 ir_ref ref;
1927
1928 IR_ASSERT(ctx->control);
1929 ref = ir_emit1(ctx, IR_END, ctx->control);
1930 ctx->control = IR_UNUSED;
1931 return ref;
1932 }
1933
_ir_MERGE_2(ir_ctx * ctx,ir_ref src1,ir_ref src2)1934 void _ir_MERGE_2(ir_ctx *ctx, ir_ref src1, ir_ref src2)
1935 {
1936 IR_ASSERT(!ctx->control);
1937 ctx->control = ir_emit2(ctx, IR_OPTX(IR_MERGE, IR_VOID, 2), src1, src2);
1938 }
1939
_ir_MERGE_N(ir_ctx * ctx,ir_ref n,ir_ref * inputs)1940 void _ir_MERGE_N(ir_ctx *ctx, ir_ref n, ir_ref *inputs)
1941 {
1942 IR_ASSERT(!ctx->control);
1943 IR_ASSERT(n > 0);
1944 if (n == 1) {
1945 _ir_BEGIN(ctx, inputs[0]);
1946 } else {
1947 ir_ref *ops;
1948
1949 ctx->control = ir_emit_N(ctx, IR_MERGE, n);
1950 ops = ctx->ir_base[ctx->control].ops;
1951 while (n) {
1952 n--;
1953 ops[n + 1] = inputs[n];
1954 }
1955 }
1956 }
1957
_ir_MERGE_SET_OP(ir_ctx * ctx,ir_ref merge,ir_ref pos,ir_ref src)1958 void _ir_MERGE_SET_OP(ir_ctx *ctx, ir_ref merge, ir_ref pos, ir_ref src)
1959 {
1960 ir_insn *insn = &ctx->ir_base[merge];
1961 ir_ref *ops = insn->ops;
1962
1963 IR_ASSERT(insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN);
1964 IR_ASSERT(pos > 0 && pos <= insn->inputs_count);
1965 ops[pos] = src;
1966 }
1967
_ir_END_LIST(ir_ctx * ctx,ir_ref list)1968 ir_ref _ir_END_LIST(ir_ctx *ctx, ir_ref list)
1969 {
1970 ir_ref ref;
1971
1972 IR_ASSERT(ctx->control);
1973 IR_ASSERT(!list || ctx->ir_base[list].op == IR_END);
1974 /* create a liked list of END nodes with the same destination through END.op2 */
1975 ref = ir_emit2(ctx, IR_END, ctx->control, list);
1976 ctx->control = IR_UNUSED;
1977 return ref;
1978 }
1979
_ir_MERGE_LIST(ir_ctx * ctx,ir_ref list)1980 void _ir_MERGE_LIST(ir_ctx *ctx, ir_ref list)
1981 {
1982 ir_ref ref = list;
1983
1984 if (list != IR_UNUSED) {
1985 uint32_t n = 0;
1986
1987 IR_ASSERT(!ctx->control);
1988
1989 /* count inputs count */
1990 do {
1991 ir_insn *insn = &ctx->ir_base[ref];
1992
1993 IR_ASSERT(insn->op == IR_END);
1994 ref = insn->op2;
1995 n++;
1996 } while (ref != IR_UNUSED);
1997
1998
1999 /* create MERGE node */
2000 IR_ASSERT(n > 0);
2001 if (n == 1) {
2002 ctx->ir_base[list].op2 = IR_UNUSED;
2003 _ir_BEGIN(ctx, list);
2004 } else {
2005 ctx->control = ir_emit_N(ctx, IR_MERGE, n);
2006 ref = list;
2007 while (n) {
2008 ir_insn *insn = &ctx->ir_base[ref];
2009
2010 ir_set_op(ctx, ctx->control, n, ref);
2011 ref = insn->op2;
2012 insn->op2 = IR_UNUSED;
2013 n--;
2014 }
2015 }
2016 }
2017 }
2018
_ir_LOOP_BEGIN(ir_ctx * ctx,ir_ref src1)2019 ir_ref _ir_LOOP_BEGIN(ir_ctx *ctx, ir_ref src1)
2020 {
2021 IR_ASSERT(!ctx->control);
2022 ctx->control = ir_emit2(ctx, IR_OPTX(IR_LOOP_BEGIN, IR_VOID, 2), src1, IR_UNUSED);
2023 return ctx->control;
2024 }
2025
_ir_LOOP_END(ir_ctx * ctx)2026 ir_ref _ir_LOOP_END(ir_ctx *ctx)
2027 {
2028 ir_ref ref;
2029
2030 IR_ASSERT(ctx->control);
2031 ref = ir_emit1(ctx, IR_LOOP_END, ctx->control);
2032 ctx->control = IR_UNUSED;
2033 return ref;
2034 }
2035
_ir_CALL(ir_ctx * ctx,ir_type type,ir_ref func)2036 ir_ref _ir_CALL(ir_ctx *ctx, ir_type type, ir_ref func)
2037 {
2038 IR_ASSERT(ctx->control);
2039 return ctx->control = ir_emit2(ctx, IR_OPTX(IR_CALL, type, 2), ctx->control, func);
2040 }
2041
_ir_CALL_1(ir_ctx * ctx,ir_type type,ir_ref func,ir_ref arg1)2042 ir_ref _ir_CALL_1(ir_ctx *ctx, ir_type type, ir_ref func, ir_ref arg1)
2043 {
2044 IR_ASSERT(ctx->control);
2045 return ctx->control = ir_emit3(ctx, IR_OPTX(IR_CALL, type, 3), ctx->control, func, arg1);
2046 }
2047
_ir_CALL_2(ir_ctx * ctx,ir_type type,ir_ref func,ir_ref arg1,ir_ref arg2)2048 ir_ref _ir_CALL_2(ir_ctx *ctx, ir_type type, ir_ref func, ir_ref arg1, ir_ref arg2)
2049 {
2050 ir_ref call;
2051
2052 IR_ASSERT(ctx->control);
2053 call = ir_emit_N(ctx, IR_OPT(IR_CALL, type), 4);
2054 ir_set_op(ctx, call, 1, ctx->control);
2055 ir_set_op(ctx, call, 2, func);
2056 ir_set_op(ctx, call, 3, arg1);
2057 ir_set_op(ctx, call, 4, arg2);
2058 ctx->control = call;
2059 return call;
2060 }
2061
_ir_CALL_3(ir_ctx * ctx,ir_type type,ir_ref func,ir_ref arg1,ir_ref arg2,ir_ref arg3)2062 ir_ref _ir_CALL_3(ir_ctx *ctx, ir_type type, ir_ref func, ir_ref arg1, ir_ref arg2, ir_ref arg3)
2063 {
2064 ir_ref call;
2065
2066 IR_ASSERT(ctx->control);
2067 call = ir_emit_N(ctx, IR_OPT(IR_CALL, type), 5);
2068 ir_set_op(ctx, call, 1, ctx->control);
2069 ir_set_op(ctx, call, 2, func);
2070 ir_set_op(ctx, call, 3, arg1);
2071 ir_set_op(ctx, call, 4, arg2);
2072 ir_set_op(ctx, call, 5, arg3);
2073 ctx->control = call;
2074 return call;
2075 }
2076
_ir_CALL_4(ir_ctx * ctx,ir_type type,ir_ref func,ir_ref arg1,ir_ref arg2,ir_ref arg3,ir_ref arg4)2077 ir_ref _ir_CALL_4(ir_ctx *ctx, ir_type type, ir_ref func, ir_ref arg1, ir_ref arg2, ir_ref arg3, ir_ref arg4)
2078 {
2079 ir_ref call;
2080
2081 IR_ASSERT(ctx->control);
2082 call = ir_emit_N(ctx, IR_OPT(IR_CALL, type), 6);
2083 ir_set_op(ctx, call, 1, ctx->control);
2084 ir_set_op(ctx, call, 2, func);
2085 ir_set_op(ctx, call, 3, arg1);
2086 ir_set_op(ctx, call, 4, arg2);
2087 ir_set_op(ctx, call, 5, arg3);
2088 ir_set_op(ctx, call, 6, arg4);
2089 ctx->control = call;
2090 return call;
2091 }
2092
_ir_CALL_5(ir_ctx * ctx,ir_type type,ir_ref func,ir_ref arg1,ir_ref arg2,ir_ref arg3,ir_ref arg4,ir_ref arg5)2093 ir_ref _ir_CALL_5(ir_ctx *ctx, ir_type type, ir_ref func, ir_ref arg1, ir_ref arg2, ir_ref arg3, ir_ref arg4, ir_ref arg5)
2094 {
2095 ir_ref call;
2096
2097 IR_ASSERT(ctx->control);
2098 call = ir_emit_N(ctx, IR_OPT(IR_CALL, type), 7);
2099 ir_set_op(ctx, call, 1, ctx->control);
2100 ir_set_op(ctx, call, 2, func);
2101 ir_set_op(ctx, call, 3, arg1);
2102 ir_set_op(ctx, call, 4, arg2);
2103 ir_set_op(ctx, call, 5, arg3);
2104 ir_set_op(ctx, call, 6, arg4);
2105 ir_set_op(ctx, call, 7, arg5);
2106 ctx->control = call;
2107 return call;
2108 }
2109
_ir_CALL_N(ir_ctx * ctx,ir_type type,ir_ref func,uint32_t count,ir_ref * args)2110 ir_ref _ir_CALL_N(ir_ctx *ctx, ir_type type, ir_ref func, uint32_t count, ir_ref *args)
2111 {
2112 ir_ref call;
2113 uint32_t i;
2114
2115 IR_ASSERT(ctx->control);
2116 call = ir_emit_N(ctx, IR_OPT(IR_CALL, type), count + 2);
2117 ir_set_op(ctx, call, 1, ctx->control);
2118 ir_set_op(ctx, call, 2, func);
2119 for (i = 0; i < count; i++) {
2120 ir_set_op(ctx, call, i + 3, args[i]);
2121 }
2122 ctx->control = call;
2123 return call;
2124 }
2125
_ir_UNREACHABLE(ir_ctx * ctx)2126 void _ir_UNREACHABLE(ir_ctx *ctx)
2127 {
2128 IR_ASSERT(ctx->control);
2129 ctx->control = ir_emit3(ctx, IR_UNREACHABLE, ctx->control, IR_UNUSED, ctx->ir_base[1].op1);
2130 ctx->ir_base[1].op1 = ctx->control;
2131 ctx->control = IR_UNUSED;
2132 }
2133
_ir_TAILCALL(ir_ctx * ctx,ir_type type,ir_ref func)2134 void _ir_TAILCALL(ir_ctx *ctx, ir_type type, ir_ref func)
2135 {
2136 IR_ASSERT(ctx->control);
2137 if (ctx->ret_type == (ir_type)-1) {
2138 ctx->ret_type = type;
2139 }
2140 IR_ASSERT(ctx->ret_type == type && "conflicting return type");
2141 ctx->control = ir_emit2(ctx, IR_OPTX(IR_TAILCALL, type, 2), ctx->control, func);
2142 _ir_UNREACHABLE(ctx);
2143 }
2144
_ir_TAILCALL_1(ir_ctx * ctx,ir_type type,ir_ref func,ir_ref arg1)2145 void _ir_TAILCALL_1(ir_ctx *ctx, ir_type type, ir_ref func, ir_ref arg1)
2146 {
2147 IR_ASSERT(ctx->control);
2148 if (ctx->ret_type == (ir_type)-1) {
2149 ctx->ret_type = type;
2150 }
2151 IR_ASSERT(ctx->ret_type == type && "conflicting return type");
2152 ctx->control = ir_emit3(ctx, IR_OPTX(IR_TAILCALL, type, 3), ctx->control, func, arg1);
2153 _ir_UNREACHABLE(ctx);
2154 }
2155
_ir_TAILCALL_2(ir_ctx * ctx,ir_type type,ir_ref func,ir_ref arg1,ir_ref arg2)2156 void _ir_TAILCALL_2(ir_ctx *ctx, ir_type type, ir_ref func, ir_ref arg1, ir_ref arg2)
2157 {
2158 ir_ref call;
2159
2160 IR_ASSERT(ctx->control);
2161 if (ctx->ret_type == (ir_type)-1) {
2162 ctx->ret_type = type;
2163 }
2164 IR_ASSERT(ctx->ret_type == type && "conflicting return type");
2165 call = ir_emit_N(ctx, IR_OPT(IR_TAILCALL, type), 4);
2166 ir_set_op(ctx, call, 1, ctx->control);
2167 ir_set_op(ctx, call, 2, func);
2168 ir_set_op(ctx, call, 3, arg1);
2169 ir_set_op(ctx, call, 4, arg2);
2170 ctx->control = call;
2171 _ir_UNREACHABLE(ctx);
2172 }
2173
_ir_TAILCALL_3(ir_ctx * ctx,ir_type type,ir_ref func,ir_ref arg1,ir_ref arg2,ir_ref arg3)2174 void _ir_TAILCALL_3(ir_ctx *ctx, ir_type type, ir_ref func, ir_ref arg1, ir_ref arg2, ir_ref arg3)
2175 {
2176 ir_ref call;
2177
2178 IR_ASSERT(ctx->control);
2179 if (ctx->ret_type == (ir_type)-1) {
2180 ctx->ret_type = type;
2181 }
2182 IR_ASSERT(ctx->ret_type == type && "conflicting return type");
2183 call = ir_emit_N(ctx, IR_OPT(IR_TAILCALL, type), 5);
2184 ir_set_op(ctx, call, 1, ctx->control);
2185 ir_set_op(ctx, call, 2, func);
2186 ir_set_op(ctx, call, 3, arg1);
2187 ir_set_op(ctx, call, 4, arg2);
2188 ir_set_op(ctx, call, 5, arg3);
2189 ctx->control = call;
2190 _ir_UNREACHABLE(ctx);
2191 }
2192
_ir_TAILCALL_4(ir_ctx * ctx,ir_type type,ir_ref func,ir_ref arg1,ir_ref arg2,ir_ref arg3,ir_ref arg4)2193 void _ir_TAILCALL_4(ir_ctx *ctx, ir_type type, ir_ref func, ir_ref arg1, ir_ref arg2, ir_ref arg3, ir_ref arg4)
2194 {
2195 ir_ref call;
2196
2197 IR_ASSERT(ctx->control);
2198 if (ctx->ret_type == (ir_type)-1) {
2199 ctx->ret_type = type;
2200 }
2201 IR_ASSERT(ctx->ret_type == type && "conflicting return type");
2202 call = ir_emit_N(ctx, IR_OPT(IR_TAILCALL, type), 6);
2203 ir_set_op(ctx, call, 1, ctx->control);
2204 ir_set_op(ctx, call, 2, func);
2205 ir_set_op(ctx, call, 3, arg1);
2206 ir_set_op(ctx, call, 4, arg2);
2207 ir_set_op(ctx, call, 5, arg3);
2208 ir_set_op(ctx, call, 6, arg4);
2209 ctx->control = call;
2210 _ir_UNREACHABLE(ctx);
2211 }
2212
_ir_TAILCALL_5(ir_ctx * ctx,ir_type type,ir_ref func,ir_ref arg1,ir_ref arg2,ir_ref arg3,ir_ref arg4,ir_ref arg5)2213 void _ir_TAILCALL_5(ir_ctx *ctx, ir_type type, ir_ref func, ir_ref arg1, ir_ref arg2, ir_ref arg3, ir_ref arg4, ir_ref arg5)
2214 {
2215 ir_ref call;
2216
2217 IR_ASSERT(ctx->control);
2218 if (ctx->ret_type == (ir_type)-1) {
2219 ctx->ret_type = type;
2220 }
2221 IR_ASSERT(ctx->ret_type == type && "conflicting return type");
2222 call = ir_emit_N(ctx, IR_OPT(IR_TAILCALL, type), 7);
2223 ir_set_op(ctx, call, 1, ctx->control);
2224 ir_set_op(ctx, call, 2, func);
2225 ir_set_op(ctx, call, 3, arg1);
2226 ir_set_op(ctx, call, 4, arg2);
2227 ir_set_op(ctx, call, 5, arg3);
2228 ir_set_op(ctx, call, 6, arg4);
2229 ir_set_op(ctx, call, 7, arg5);
2230 ctx->control = call;
2231 _ir_UNREACHABLE(ctx);
2232 }
2233
_ir_TAILCALL_N(ir_ctx * ctx,ir_type type,ir_ref func,uint32_t count,ir_ref * args)2234 void _ir_TAILCALL_N(ir_ctx *ctx, ir_type type, ir_ref func, uint32_t count, ir_ref *args)
2235 {
2236 ir_ref call;
2237 uint32_t i;
2238
2239 IR_ASSERT(ctx->control);
2240 if (ctx->ret_type == (ir_type)-1) {
2241 ctx->ret_type = type;
2242 }
2243 IR_ASSERT(ctx->ret_type == type && "conflicting return type");
2244 call = ir_emit_N(ctx, IR_OPT(IR_TAILCALL, type), count + 2);
2245 ir_set_op(ctx, call, 1, ctx->control);
2246 ir_set_op(ctx, call, 2, func);
2247 for (i = 0; i < count; i++) {
2248 ir_set_op(ctx, call, i + 3, args[i]);
2249 }
2250 ctx->control = call;
2251 _ir_UNREACHABLE(ctx);
2252 }
2253
_ir_SWITCH(ir_ctx * ctx,ir_ref val)2254 ir_ref _ir_SWITCH(ir_ctx *ctx, ir_ref val)
2255 {
2256 ir_ref ref;
2257
2258 IR_ASSERT(ctx->control);
2259 ref = ir_emit2(ctx, IR_SWITCH, ctx->control, val);
2260 ctx->control = IR_UNUSED;
2261 return ref;
2262 }
2263
_ir_CASE_VAL(ir_ctx * ctx,ir_ref switch_ref,ir_ref val)2264 void _ir_CASE_VAL(ir_ctx *ctx, ir_ref switch_ref, ir_ref val)
2265 {
2266 IR_ASSERT(!ctx->control);
2267 ctx->control = ir_emit2(ctx, IR_CASE_VAL, switch_ref, val);
2268 }
2269
_ir_CASE_DEFAULT(ir_ctx * ctx,ir_ref switch_ref)2270 void _ir_CASE_DEFAULT(ir_ctx *ctx, ir_ref switch_ref)
2271 {
2272 IR_ASSERT(!ctx->control);
2273 ctx->control = ir_emit1(ctx, IR_CASE_DEFAULT, switch_ref);
2274 }
2275
_ir_RETURN(ir_ctx * ctx,ir_ref val)2276 void _ir_RETURN(ir_ctx *ctx, ir_ref val)
2277 {
2278 ir_type type = (val != IR_UNUSED) ? ctx->ir_base[val].type : IR_VOID;
2279
2280 IR_ASSERT(ctx->control);
2281 if (ctx->ret_type == (ir_type)-1) {
2282 ctx->ret_type = type;
2283 }
2284 IR_ASSERT(ctx->ret_type == type && "conflicting return type");
2285 ctx->control = ir_emit3(ctx, IR_RETURN, ctx->control, val, ctx->ir_base[1].op1);
2286 ctx->ir_base[1].op1 = ctx->control;
2287 ctx->control = IR_UNUSED;
2288 }
2289
_ir_IJMP(ir_ctx * ctx,ir_ref addr)2290 void _ir_IJMP(ir_ctx *ctx, ir_ref addr)
2291 {
2292 IR_ASSERT(ctx->control);
2293 ctx->control = ir_emit3(ctx, IR_IJMP, ctx->control, addr, ctx->ir_base[1].op1);
2294 ctx->ir_base[1].op1 = ctx->control;
2295 ctx->control = IR_UNUSED;
2296 }
2297
_ir_ADD_OFFSET(ir_ctx * ctx,ir_ref addr,uintptr_t offset)2298 ir_ref _ir_ADD_OFFSET(ir_ctx *ctx, ir_ref addr, uintptr_t offset)
2299 {
2300 if (offset) {
2301 addr = ir_fold2(ctx, IR_OPT(IR_ADD, IR_ADDR), addr, ir_const_addr(ctx, offset));
2302 }
2303 return addr;
2304 }
2305
_ir_GUARD(ir_ctx * ctx,ir_ref condition,ir_ref addr)2306 void _ir_GUARD(ir_ctx *ctx, ir_ref condition, ir_ref addr)
2307 {
2308 IR_ASSERT(ctx->control);
2309 if (condition == IR_TRUE) {
2310 return;
2311 } else {
2312 ir_ref ref = ctx->control;
2313 ir_insn *insn;
2314
2315 while (ref > condition) {
2316 insn = &ctx->ir_base[ref];
2317 if (insn->op == IR_GUARD) {
2318 if (insn->op2 == condition) {
2319 return;
2320 }
2321 } else if (insn->op == IR_GUARD_NOT) {
2322 if (insn->op2 == condition) {
2323 condition = IR_FALSE;
2324 break;
2325 }
2326 } else if (insn->op == IR_START || insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN) {
2327 break;
2328 }
2329 ref = insn->op1;
2330 }
2331 }
2332 if (ctx->snapshot_create) {
2333 ctx->snapshot_create(ctx, addr);
2334 }
2335 ctx->control = ir_emit3(ctx, IR_GUARD, ctx->control, condition, addr);
2336 }
2337
_ir_GUARD_NOT(ir_ctx * ctx,ir_ref condition,ir_ref addr)2338 void _ir_GUARD_NOT(ir_ctx *ctx, ir_ref condition, ir_ref addr)
2339 {
2340 IR_ASSERT(ctx->control);
2341 if (condition == IR_FALSE) {
2342 return;
2343 } else {
2344 ir_ref ref = ctx->control;
2345 ir_insn *insn;
2346
2347 while (ref > condition) {
2348 insn = &ctx->ir_base[ref];
2349 if (insn->op == IR_GUARD_NOT) {
2350 if (insn->op2 == condition) {
2351 return;
2352 }
2353 } else if (insn->op == IR_GUARD) {
2354 if (insn->op2 == condition) {
2355 condition = IR_TRUE;
2356 break;
2357 }
2358 } else if (insn->op == IR_START || insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN) {
2359 break;
2360 }
2361 ref = insn->op1;
2362 }
2363 }
2364 if (ctx->snapshot_create) {
2365 ctx->snapshot_create(ctx, addr);
2366 }
2367 ctx->control = ir_emit3(ctx, IR_GUARD_NOT, ctx->control, condition, addr);
2368 }
2369
_ir_SNAPSHOT(ir_ctx * ctx,ir_ref n)2370 ir_ref _ir_SNAPSHOT(ir_ctx *ctx, ir_ref n)
2371 {
2372 ir_ref snapshot;
2373
2374 IR_ASSERT(ctx->control);
2375 snapshot = ir_emit_N(ctx, IR_SNAPSHOT, 1 + n); /* op1 is used for control */
2376 ctx->ir_base[snapshot].op1 = ctx->control;
2377 ctx->control = snapshot;
2378 return snapshot;
2379 }
2380
_ir_SNAPSHOT_SET_OP(ir_ctx * ctx,ir_ref snapshot,ir_ref pos,ir_ref val)2381 void _ir_SNAPSHOT_SET_OP(ir_ctx *ctx, ir_ref snapshot, ir_ref pos, ir_ref val)
2382 {
2383 ir_insn *insn = &ctx->ir_base[snapshot];
2384 ir_ref *ops = insn->ops;
2385
2386 IR_ASSERT(val < snapshot);
2387 IR_ASSERT(insn->op == IR_SNAPSHOT);
2388 pos++; /* op1 is used for control */
2389 IR_ASSERT(pos > 1 && pos <= insn->inputs_count);
2390 ops[pos] = val;
2391 }
2392
_ir_EXITCALL(ir_ctx * ctx,ir_ref func)2393 ir_ref _ir_EXITCALL(ir_ctx *ctx, ir_ref func)
2394 {
2395 IR_ASSERT(ctx->control);
2396 return ctx->control = ir_emit2(ctx, IR_OPT(IR_EXITCALL, IR_I32), ctx->control, func);
2397 }
2398
_ir_ALLOCA(ir_ctx * ctx,ir_ref size)2399 ir_ref _ir_ALLOCA(ir_ctx *ctx, ir_ref size)
2400 {
2401 IR_ASSERT(ctx->control);
2402 return ctx->control = ir_emit2(ctx, IR_OPT(IR_ALLOCA, IR_ADDR), ctx->control, size);
2403 }
2404
_ir_AFREE(ir_ctx * ctx,ir_ref size)2405 void _ir_AFREE(ir_ctx *ctx, ir_ref size)
2406 {
2407 IR_ASSERT(ctx->control);
2408 ctx->control = ir_emit2(ctx, IR_AFREE, ctx->control, size);
2409 }
2410
_ir_VLOAD(ir_ctx * ctx,ir_type type,ir_ref var)2411 ir_ref _ir_VLOAD(ir_ctx *ctx, ir_type type, ir_ref var)
2412 {
2413 IR_ASSERT(ctx->control);
2414 return ctx->control = ir_emit2(ctx, IR_OPT(IR_VLOAD, type), ctx->control, var);
2415 }
2416
_ir_VSTORE(ir_ctx * ctx,ir_ref var,ir_ref val)2417 void _ir_VSTORE(ir_ctx *ctx, ir_ref var, ir_ref val)
2418 {
2419 IR_ASSERT(ctx->control);
2420 ctx->control = ir_emit3(ctx, IR_VSTORE, ctx->control, var, val);
2421 }
2422
_ir_TLS(ir_ctx * ctx,ir_ref index,ir_ref offset)2423 ir_ref _ir_TLS(ir_ctx *ctx, ir_ref index, ir_ref offset)
2424 {
2425 IR_ASSERT(ctx->control);
2426 return ctx->control = ir_emit3(ctx, IR_OPT(IR_TLS, IR_ADDR), ctx->control, index, offset);
2427 }
2428
_ir_RLOAD(ir_ctx * ctx,ir_type type,ir_ref reg)2429 ir_ref _ir_RLOAD(ir_ctx *ctx, ir_type type, ir_ref reg)
2430 {
2431 IR_ASSERT(ctx->control);
2432 return ctx->control = ir_emit2(ctx, IR_OPT(IR_RLOAD, type), ctx->control, reg);
2433 }
2434
_ir_RSTORE(ir_ctx * ctx,ir_ref reg,ir_ref val)2435 void _ir_RSTORE(ir_ctx *ctx, ir_ref reg, ir_ref val)
2436 {
2437 IR_ASSERT(ctx->control);
2438 ctx->control = ir_emit3(ctx, IR_RSTORE, ctx->control, val, reg);
2439 }
2440
_ir_LOAD(ir_ctx * ctx,ir_type type,ir_ref addr)2441 ir_ref _ir_LOAD(ir_ctx *ctx, ir_type type, ir_ref addr)
2442 {
2443 ir_ref ref = ir_find_aliasing_load(ctx, ctx->control, type, addr);
2444
2445 IR_ASSERT(ctx->control);
2446 if (!ref) {
2447 ctx->control = ref = ir_emit2(ctx, IR_OPT(IR_LOAD, type), ctx->control, addr);
2448 }
2449 return ref;
2450 }
2451
_ir_STORE(ir_ctx * ctx,ir_ref addr,ir_ref val)2452 void _ir_STORE(ir_ctx *ctx, ir_ref addr, ir_ref val)
2453 {
2454 ir_ref limit = (addr > 0) ? addr : 1;
2455 ir_ref ref = ctx->control;
2456 ir_ref prev = IR_UNUSED;
2457 ir_insn *insn;
2458 ir_type type = ctx->ir_base[val].type;
2459 ir_type type2;
2460 bool guarded = 0;
2461
2462 IR_ASSERT(ctx->control);
2463 while (ref > limit) {
2464 insn = &ctx->ir_base[ref];
2465 if (insn->op == IR_STORE) {
2466 if (insn->op2 == addr) {
2467 if (ctx->ir_base[insn->op3].type == type) {
2468 if (insn->op3 == val) {
2469 return;
2470 } else {
2471 if (!guarded) {
2472 if (prev) {
2473 ctx->ir_base[prev].op1 = insn->op1;
2474 } else {
2475 ctx->control = insn->op1;
2476 }
2477 insn->optx = IR_NOP;
2478 insn->op1 = IR_NOP;
2479 insn->op2 = IR_NOP;
2480 insn->op3 = IR_NOP;
2481 }
2482 break;
2483 }
2484 } else {
2485 break;
2486 }
2487 } else {
2488 type2 = ctx->ir_base[insn->op3].type;
2489 goto check_aliasing;
2490 }
2491 } else if (insn->op == IR_LOAD) {
2492 if (insn->op2 == addr) {
2493 break;
2494 }
2495 type2 = insn->type;
2496 check_aliasing:
2497 if (ir_check_partial_aliasing(ctx, addr, insn->op2, type, type2) != IR_NO_ALIAS) {
2498 break;
2499 }
2500 } else if (insn->op == IR_GUARD || insn->op == IR_GUARD_NOT) {
2501 guarded = 1;
2502 } else if (insn->op >= IR_START || insn->op == IR_CALL) {
2503 break;
2504 }
2505 prev = ref;
2506 ref = insn->op1;
2507 }
2508 ctx->control = ir_emit3(ctx, IR_STORE, ctx->control, addr, val);
2509 }
2510
_ir_VA_START(ir_ctx * ctx,ir_ref list)2511 void _ir_VA_START(ir_ctx *ctx, ir_ref list)
2512 {
2513 IR_ASSERT(ctx->control);
2514 ctx->control = ir_emit2(ctx, IR_VA_START, ctx->control, list);
2515 }
2516
_ir_VA_END(ir_ctx * ctx,ir_ref list)2517 void _ir_VA_END(ir_ctx *ctx, ir_ref list)
2518 {
2519 IR_ASSERT(ctx->control);
2520 ctx->control = ir_emit2(ctx, IR_VA_END, ctx->control, list);
2521 }
2522
_ir_VA_COPY(ir_ctx * ctx,ir_ref dst,ir_ref src)2523 void _ir_VA_COPY(ir_ctx *ctx, ir_ref dst, ir_ref src)
2524 {
2525 IR_ASSERT(ctx->control);
2526 ctx->control = ir_emit3(ctx, IR_VA_COPY, ctx->control, dst, src);
2527 }
2528
_ir_VA_ARG(ir_ctx * ctx,ir_type type,ir_ref list)2529 ir_ref _ir_VA_ARG(ir_ctx *ctx, ir_type type, ir_ref list)
2530 {
2531 IR_ASSERT(ctx->control);
2532 return ctx->control = ir_emit2(ctx, IR_OPT(IR_VA_ARG, type), ctx->control, list);
2533 }
2534