1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
sljit_get_platform_name(void)27 SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
28 {
29 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
30 return "RISC-V-32" SLJIT_CPUINFO;
31 #else /* !SLJIT_CONFIG_RISCV_32 */
32 return "RISC-V-64" SLJIT_CPUINFO;
33 #endif /* SLJIT_CONFIG_RISCV_32 */
34 }
35
36 /* Length of an instruction word
37 Both for riscv-32 and riscv-64 */
38 typedef sljit_u32 sljit_ins;
39
40 #define TMP_REG1 (SLJIT_NUMBER_OF_REGISTERS + 2)
41 #define TMP_REG2 (SLJIT_NUMBER_OF_REGISTERS + 3)
42 #define TMP_REG3 (SLJIT_NUMBER_OF_REGISTERS + 4)
43 #define TMP_ZERO 0
44
45 /* Flags are kept in volatile registers. */
46 #define EQUAL_FLAG (SLJIT_NUMBER_OF_REGISTERS + 5)
47 #define RETURN_ADDR_REG TMP_REG2
48 #define OTHER_FLAG (SLJIT_NUMBER_OF_REGISTERS + 6)
49
50 #define TMP_FREG1 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1)
51 #define TMP_FREG2 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2)
52
53 static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 7] = {
54 0, 10, 11, 12, 13, 14, 15, 16, 17, 29, 30, 31, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 9, 8, 2, 6, 1, 7, 5, 28
55 };
56
57 static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = {
58 0, 10, 11, 12, 13, 14, 15, 16, 17, 2, 3, 4, 5, 6, 7, 28, 29, 30, 31, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 9, 8, 0, 1,
59 };
60
61 /* --------------------------------------------------------------------- */
62 /* Instrucion forms */
63 /* --------------------------------------------------------------------- */
64
65 #define RD(rd) ((sljit_ins)reg_map[rd] << 7)
66 #define RS1(rs1) ((sljit_ins)reg_map[rs1] << 15)
67 #define RS2(rs2) ((sljit_ins)reg_map[rs2] << 20)
68 #define FRD(rd) ((sljit_ins)freg_map[rd] << 7)
69 #define FRS1(rs1) ((sljit_ins)freg_map[rs1] << 15)
70 #define FRS2(rs2) ((sljit_ins)freg_map[rs2] << 20)
71 #define IMM_I(imm) ((sljit_ins)(imm) << 20)
72 #define IMM_S(imm) ((((sljit_ins)(imm) & 0xfe0) << 20) | (((sljit_ins)(imm) & 0x1f) << 7))
73
74 /* Represents funct(i) parts of the instructions. */
75 #define OPC(o) ((sljit_ins)(o))
76 #define F3(f) ((sljit_ins)(f) << 12)
77 #define F12(f) ((sljit_ins)(f) << 20)
78 #define F7(f) ((sljit_ins)(f) << 25)
79
80 #define ADD (F7(0x0) | F3(0x0) | OPC(0x33))
81 #define ADDI (F3(0x0) | OPC(0x13))
82 #define AND (F7(0x0) | F3(0x7) | OPC(0x33))
83 #define ANDI (F3(0x7) | OPC(0x13))
84 #define AUIPC (OPC(0x17))
85 #define BEQ (F3(0x0) | OPC(0x63))
86 #define BNE (F3(0x1) | OPC(0x63))
87 #define BLT (F3(0x4) | OPC(0x63))
88 #define BGE (F3(0x5) | OPC(0x63))
89 #define BLTU (F3(0x6) | OPC(0x63))
90 #define BGEU (F3(0x7) | OPC(0x63))
91 #define DIV (F7(0x1) | F3(0x4) | OPC(0x33))
92 #define DIVU (F7(0x1) | F3(0x5) | OPC(0x33))
93 #define EBREAK (F12(0x1) | F3(0x0) | OPC(0x73))
94 #define FADD_S (F7(0x0) | F3(0x7) | OPC(0x53))
95 #define FDIV_S (F7(0xc) | F3(0x7) | OPC(0x53))
96 #define FEQ_S (F7(0x50) | F3(0x2) | OPC(0x53))
97 #define FLD (F3(0x3) | OPC(0x7))
98 #define FLE_S (F7(0x50) | F3(0x0) | OPC(0x53))
99 #define FLT_S (F7(0x50) | F3(0x1) | OPC(0x53))
100 /* These conversion opcodes are partly defined. */
101 #define FCVT_S_D (F7(0x20) | OPC(0x53))
102 #define FCVT_S_W (F7(0x68) | OPC(0x53))
103 #define FCVT_S_WU (F7(0x68) | F12(0x1) | OPC(0x53))
104 #define FCVT_W_S (F7(0x60) | F3(0x1) | OPC(0x53))
105 #define FMUL_S (F7(0x8) | F3(0x7) | OPC(0x53))
106 #define FMV_X_W (F7(0x70) | F3(0x0) | OPC(0x53))
107 #define FMV_W_X (F7(0x78) | F3(0x0) | OPC(0x53))
108 #define FSD (F3(0x3) | OPC(0x27))
109 #define FSGNJ_S (F7(0x10) | F3(0x0) | OPC(0x53))
110 #define FSGNJN_S (F7(0x10) | F3(0x1) | OPC(0x53))
111 #define FSGNJX_S (F7(0x10) | F3(0x2) | OPC(0x53))
112 #define FSUB_S (F7(0x4) | F3(0x7) | OPC(0x53))
113 #define FSW (F3(0x2) | OPC(0x27))
114 #define JAL (OPC(0x6f))
115 #define JALR (F3(0x0) | OPC(0x67))
116 #define LD (F3(0x3) | OPC(0x3))
117 #define LUI (OPC(0x37))
118 #define LW (F3(0x2) | OPC(0x3))
119 #define MUL (F7(0x1) | F3(0x0) | OPC(0x33))
120 #define MULH (F7(0x1) | F3(0x1) | OPC(0x33))
121 #define MULHU (F7(0x1) | F3(0x3) | OPC(0x33))
122 #define OR (F7(0x0) | F3(0x6) | OPC(0x33))
123 #define ORI (F3(0x6) | OPC(0x13))
124 #define REM (F7(0x1) | F3(0x6) | OPC(0x33))
125 #define REMU (F7(0x1) | F3(0x7) | OPC(0x33))
126 #define SD (F3(0x3) | OPC(0x23))
127 #define SLL (F7(0x0) | F3(0x1) | OPC(0x33))
128 #define SLLI (IMM_I(0x0) | F3(0x1) | OPC(0x13))
129 #define SLT (F7(0x0) | F3(0x2) | OPC(0x33))
130 #define SLTI (F3(0x2) | OPC(0x13))
131 #define SLTU (F7(0x0) | F3(0x3) | OPC(0x33))
132 #define SLTUI (F3(0x3) | OPC(0x13))
133 #define SRL (F7(0x0) | F3(0x5) | OPC(0x33))
134 #define SRLI (IMM_I(0x0) | F3(0x5) | OPC(0x13))
135 #define SRA (F7(0x20) | F3(0x5) | OPC(0x33))
136 #define SRAI (IMM_I(0x400) | F3(0x5) | OPC(0x13))
137 #define SUB (F7(0x20) | F3(0x0) | OPC(0x33))
138 #define SW (F3(0x2) | OPC(0x23))
139 #define XOR (F7(0x0) | F3(0x4) | OPC(0x33))
140 #define XORI (F3(0x4) | OPC(0x13))
141
142 #define SIMM_MAX (0x7ff)
143 #define SIMM_MIN (-0x800)
144 #define BRANCH_MAX (0xfff)
145 #define BRANCH_MIN (-0x1000)
146 #define JUMP_MAX (0xfffff)
147 #define JUMP_MIN (-0x100000)
148
149 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
150 #define S32_MAX (0x7ffff7ffl)
151 #define S32_MIN (-0x80000000l)
152 #define S44_MAX (0x7fffffff7ffl)
153 #define S52_MAX (0x7ffffffffffffl)
154 #endif
155
push_inst(struct sljit_compiler * compiler,sljit_ins ins)156 static sljit_s32 push_inst(struct sljit_compiler *compiler, sljit_ins ins)
157 {
158 sljit_ins *ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins));
159 FAIL_IF(!ptr);
160 *ptr = ins;
161 compiler->size++;
162 return SLJIT_SUCCESS;
163 }
164
push_imm_s_inst(struct sljit_compiler * compiler,sljit_ins ins,sljit_sw imm)165 static sljit_s32 push_imm_s_inst(struct sljit_compiler *compiler, sljit_ins ins, sljit_sw imm)
166 {
167 return push_inst(compiler, ins | IMM_S(imm));
168 }
169
detect_jump_type(struct sljit_jump * jump,sljit_ins * code,sljit_sw executable_offset)170 static SLJIT_INLINE sljit_ins* detect_jump_type(struct sljit_jump *jump, sljit_ins *code, sljit_sw executable_offset)
171 {
172 sljit_sw diff;
173 sljit_uw target_addr;
174 sljit_ins *inst;
175
176 inst = (sljit_ins *)jump->addr;
177
178 if (jump->flags & SLJIT_REWRITABLE_JUMP)
179 goto exit;
180
181 if (jump->flags & JUMP_ADDR)
182 target_addr = jump->u.target;
183 else {
184 SLJIT_ASSERT(jump->flags & JUMP_LABEL);
185 target_addr = (sljit_uw)(code + jump->u.label->size) + (sljit_uw)executable_offset;
186 }
187
188 diff = (sljit_sw)target_addr - (sljit_sw)inst - executable_offset;
189
190 if (jump->flags & IS_COND) {
191 inst--;
192 diff += SSIZE_OF(ins);
193
194 if (diff >= BRANCH_MIN && diff <= BRANCH_MAX) {
195 jump->flags |= PATCH_B;
196 inst[0] = (inst[0] & 0x1fff07f) ^ 0x1000;
197 jump->addr = (sljit_uw)inst;
198 return inst;
199 }
200
201 inst++;
202 diff -= SSIZE_OF(ins);
203 }
204
205 if (diff >= JUMP_MIN && diff <= JUMP_MAX) {
206 if (jump->flags & IS_COND) {
207 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
208 inst[-1] -= (sljit_ins)(1 * sizeof(sljit_ins)) << 7;
209 #else
210 inst[-1] -= (sljit_ins)(5 * sizeof(sljit_ins)) << 7;
211 #endif
212 }
213
214 jump->flags |= PATCH_J;
215 return inst;
216 }
217
218 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
219 if (diff >= S32_MIN && diff <= S32_MAX) {
220 if (jump->flags & IS_COND)
221 inst[-1] -= (sljit_ins)(4 * sizeof(sljit_ins)) << 7;
222
223 jump->flags |= PATCH_REL32;
224 inst[1] = inst[0];
225 return inst + 1;
226 }
227
228 if (target_addr <= (sljit_uw)S32_MAX) {
229 if (jump->flags & IS_COND)
230 inst[-1] -= (sljit_ins)(4 * sizeof(sljit_ins)) << 7;
231
232 jump->flags |= PATCH_ABS32;
233 inst[1] = inst[0];
234 return inst + 1;
235 }
236
237 if (target_addr <= S44_MAX) {
238 if (jump->flags & IS_COND)
239 inst[-1] -= (sljit_ins)(2 * sizeof(sljit_ins)) << 7;
240
241 jump->flags |= PATCH_ABS44;
242 inst[3] = inst[0];
243 return inst + 3;
244 }
245
246 if (target_addr <= S52_MAX) {
247 if (jump->flags & IS_COND)
248 inst[-1] -= (sljit_ins)(1 * sizeof(sljit_ins)) << 7;
249
250 jump->flags |= PATCH_ABS52;
251 inst[4] = inst[0];
252 return inst + 4;
253 }
254 #endif
255
256 exit:
257 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
258 inst[1] = inst[0];
259 return inst + 1;
260 #else
261 inst[5] = inst[0];
262 return inst + 5;
263 #endif
264 }
265
266 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
267
put_label_get_length(struct sljit_put_label * put_label,sljit_uw max_label)268 static SLJIT_INLINE sljit_sw put_label_get_length(struct sljit_put_label *put_label, sljit_uw max_label)
269 {
270 if (max_label <= (sljit_uw)S32_MAX) {
271 put_label->flags = PATCH_ABS32;
272 return 1;
273 }
274
275 if (max_label <= S44_MAX) {
276 put_label->flags = PATCH_ABS44;
277 return 3;
278 }
279
280 if (max_label <= S52_MAX) {
281 put_label->flags = PATCH_ABS52;
282 return 4;
283 }
284
285 put_label->flags = 0;
286 return 5;
287 }
288
289 #endif /* SLJIT_CONFIG_RISCV_64 */
290
load_addr_to_reg(void * dst,sljit_u32 reg)291 static SLJIT_INLINE void load_addr_to_reg(void *dst, sljit_u32 reg)
292 {
293 struct sljit_jump *jump = NULL;
294 struct sljit_put_label *put_label;
295 sljit_uw flags;
296 sljit_ins *inst;
297 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
298 sljit_sw high;
299 #endif
300 sljit_uw addr;
301
302 if (reg != 0) {
303 jump = (struct sljit_jump*)dst;
304 flags = jump->flags;
305 inst = (sljit_ins*)jump->addr;
306 addr = (flags & JUMP_LABEL) ? jump->u.label->addr : jump->u.target;
307 } else {
308 put_label = (struct sljit_put_label*)dst;
309 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
310 flags = put_label->flags;
311 #endif
312 inst = (sljit_ins*)put_label->addr;
313 addr = put_label->label->addr;
314 reg = *inst;
315 }
316
317 if ((addr & 0x800) != 0)
318 addr += 0x1000;
319
320 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
321 inst[0] = LUI | RD(reg) | (sljit_ins)((sljit_sw)addr & ~0xfff);
322 #else /* !SLJIT_CONFIG_RISCV_32 */
323
324 if (flags & PATCH_ABS32) {
325 SLJIT_ASSERT(addr <= S32_MAX);
326 inst[0] = LUI | RD(reg) | (sljit_ins)((sljit_sw)addr & ~0xfff);
327 } else if (flags & PATCH_ABS44) {
328 high = (sljit_sw)addr >> 12;
329 SLJIT_ASSERT((sljit_uw)high <= 0x7fffffff);
330
331 if (high > S32_MAX) {
332 SLJIT_ASSERT((high & 0x800) != 0);
333 inst[0] = LUI | RD(reg) | (sljit_ins)0x80000000u;
334 inst[1] = XORI | RD(reg) | RS1(reg) | IMM_I(high);
335 } else {
336 if ((high & 0x800) != 0)
337 high += 0x1000;
338
339 inst[0] = LUI | RD(reg) | (sljit_ins)(high & ~0xfff);
340 inst[1] = ADDI | RD(reg) | RS1(reg) | IMM_I(high);
341 }
342
343 inst[2] = SLLI | RD(reg) | RS1(reg) | IMM_I(12);
344 inst += 2;
345 } else {
346 high = (sljit_sw)addr >> 32;
347
348 if ((addr & 0x80000000l) != 0)
349 high = ~high;
350
351 if (flags & PATCH_ABS52) {
352 SLJIT_ASSERT(addr <= S52_MAX);
353 inst[0] = LUI | RD(TMP_REG3) | (sljit_ins)(high << 12);
354 } else {
355 if ((high & 0x800) != 0)
356 high += 0x1000;
357 inst[0] = LUI | RD(TMP_REG3) | (sljit_ins)(high & ~0xfff);
358 inst[1] = ADDI | RD(TMP_REG3) | RS1(TMP_REG3) | IMM_I(high);
359 inst++;
360 }
361
362 inst[1] = LUI | RD(reg) | (sljit_ins)((sljit_sw)addr & ~0xfff);
363 inst[2] = SLLI | RD(TMP_REG3) | RS1(TMP_REG3) | IMM_I((flags & PATCH_ABS52) ? 20 : 32);
364 inst[3] = XOR | RD(reg) | RS1(reg) | RS2(TMP_REG3);
365 inst += 3;
366 }
367 #endif /* !SLJIT_CONFIG_RISCV_32 */
368
369 if (jump != NULL) {
370 SLJIT_ASSERT((inst[1] & 0x707f) == JALR);
371 inst[1] = (inst[1] & 0xfffff) | IMM_I(addr);
372 } else
373 inst[1] = ADDI | RD(reg) | RS1(reg) | IMM_I(addr);
374 }
375
sljit_generate_code(struct sljit_compiler * compiler)376 SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler)
377 {
378 struct sljit_memory_fragment *buf;
379 sljit_ins *code;
380 sljit_ins *code_ptr;
381 sljit_ins *buf_ptr;
382 sljit_ins *buf_end;
383 sljit_uw word_count;
384 sljit_uw next_addr;
385 sljit_sw executable_offset;
386 sljit_uw addr;
387
388 struct sljit_label *label;
389 struct sljit_jump *jump;
390 struct sljit_const *const_;
391 struct sljit_put_label *put_label;
392
393 CHECK_ERROR_PTR();
394 CHECK_PTR(check_sljit_generate_code(compiler));
395 reverse_buf(compiler);
396
397 code = (sljit_ins*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_ins), compiler->exec_allocator_data);
398 PTR_FAIL_WITH_EXEC_IF(code);
399 buf = compiler->buf;
400
401 code_ptr = code;
402 word_count = 0;
403 next_addr = 0;
404 executable_offset = SLJIT_EXEC_OFFSET(code);
405
406 label = compiler->labels;
407 jump = compiler->jumps;
408 const_ = compiler->consts;
409 put_label = compiler->put_labels;
410
411 do {
412 buf_ptr = (sljit_ins*)buf->memory;
413 buf_end = buf_ptr + (buf->used_size >> 2);
414 do {
415 *code_ptr = *buf_ptr++;
416 if (next_addr == word_count) {
417 SLJIT_ASSERT(!label || label->size >= word_count);
418 SLJIT_ASSERT(!jump || jump->addr >= word_count);
419 SLJIT_ASSERT(!const_ || const_->addr >= word_count);
420 SLJIT_ASSERT(!put_label || put_label->addr >= word_count);
421
422 /* These structures are ordered by their address. */
423 if (label && label->size == word_count) {
424 label->addr = (sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
425 label->size = (sljit_uw)(code_ptr - code);
426 label = label->next;
427 }
428 if (jump && jump->addr == word_count) {
429 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
430 word_count += 1;
431 #else
432 word_count += 5;
433 #endif
434 jump->addr = (sljit_uw)code_ptr;
435 code_ptr = detect_jump_type(jump, code, executable_offset);
436 jump = jump->next;
437 }
438 if (const_ && const_->addr == word_count) {
439 const_->addr = (sljit_uw)code_ptr;
440 const_ = const_->next;
441 }
442 if (put_label && put_label->addr == word_count) {
443 SLJIT_ASSERT(put_label->label);
444 put_label->addr = (sljit_uw)code_ptr;
445 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
446 code_ptr += 1;
447 word_count += 1;
448 #else
449 code_ptr += put_label_get_length(put_label, (sljit_uw)(SLJIT_ADD_EXEC_OFFSET(code, executable_offset) + put_label->label->size));
450 word_count += 5;
451 #endif
452 put_label = put_label->next;
453 }
454 next_addr = compute_next_addr(label, jump, const_, put_label);
455 }
456 code_ptr++;
457 word_count++;
458 } while (buf_ptr < buf_end);
459
460 buf = buf->next;
461 } while (buf);
462
463 if (label && label->size == word_count) {
464 label->addr = (sljit_uw)code_ptr;
465 label->size = (sljit_uw)(code_ptr - code);
466 label = label->next;
467 }
468
469 SLJIT_ASSERT(!label);
470 SLJIT_ASSERT(!jump);
471 SLJIT_ASSERT(!const_);
472 SLJIT_ASSERT(!put_label);
473 SLJIT_ASSERT(code_ptr - code <= (sljit_sw)compiler->size);
474
475 jump = compiler->jumps;
476 while (jump) {
477 do {
478 if (!(jump->flags & (PATCH_B | PATCH_J | PATCH_REL32))) {
479 load_addr_to_reg(jump, TMP_REG1);
480 break;
481 }
482
483 addr = (jump->flags & JUMP_LABEL) ? jump->u.label->addr : jump->u.target;
484 buf_ptr = (sljit_ins *)jump->addr;
485 addr -= (sljit_uw)SLJIT_ADD_EXEC_OFFSET(buf_ptr, executable_offset);
486
487 if (jump->flags & PATCH_B) {
488 SLJIT_ASSERT((sljit_sw)addr >= BRANCH_MIN && (sljit_sw)addr <= BRANCH_MAX);
489 addr = ((addr & 0x800) >> 4) | ((addr & 0x1e) << 7) | ((addr & 0x7e0) << 20) | ((addr & 0x1000) << 19);
490 buf_ptr[0] |= (sljit_ins)addr;
491 break;
492 }
493
494 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
495 if (jump->flags & PATCH_REL32) {
496 SLJIT_ASSERT((sljit_sw)addr >= S32_MIN && (sljit_sw)addr <= S32_MAX);
497
498 if ((addr & 0x800) != 0)
499 addr += 0x1000;
500
501 buf_ptr[0] = AUIPC | RD(TMP_REG1) | (sljit_ins)((sljit_sw)addr & ~0xfff);
502 SLJIT_ASSERT((buf_ptr[1] & 0x707f) == JALR);
503 buf_ptr[1] |= IMM_I(addr);
504 break;
505 }
506 #endif
507
508 SLJIT_ASSERT((sljit_sw)addr >= JUMP_MIN && (sljit_sw)addr <= JUMP_MAX);
509 addr = (addr & 0xff000) | ((addr & 0x800) << 9) | ((addr & 0x7fe) << 20) | ((addr & 0x100000) << 11);
510 buf_ptr[0] = JAL | RD((jump->flags & IS_CALL) ? RETURN_ADDR_REG : TMP_ZERO) | (sljit_ins)addr;
511 } while (0);
512 jump = jump->next;
513 }
514
515 put_label = compiler->put_labels;
516 while (put_label) {
517 load_addr_to_reg(put_label, 0);
518 put_label = put_label->next;
519 }
520
521 compiler->error = SLJIT_ERR_COMPILED;
522 compiler->executable_offset = executable_offset;
523 compiler->executable_size = (sljit_uw)(code_ptr - code) * sizeof(sljit_ins);
524
525 code = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
526 code_ptr = (sljit_ins *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
527
528 SLJIT_CACHE_FLUSH(code, code_ptr);
529 SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1);
530 return code;
531 }
532
sljit_has_cpu_feature(sljit_s32 feature_type)533 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
534 {
535 switch (feature_type) {
536 case SLJIT_HAS_FPU:
537 #ifdef SLJIT_IS_FPU_AVAILABLE
538 return (SLJIT_IS_FPU_AVAILABLE) != 0;
539 #elif defined(__riscv_float_abi_soft)
540 return 0;
541 #else
542 return 1;
543 #endif /* SLJIT_IS_FPU_AVAILABLE */
544 case SLJIT_HAS_ZERO_REGISTER:
545 case SLJIT_HAS_COPY_F32:
546 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
547 case SLJIT_HAS_COPY_F64:
548 #endif /* !SLJIT_CONFIG_RISCV_64 */
549 return 1;
550 default:
551 return 0;
552 }
553 }
554
sljit_cmp_info(sljit_s32 type)555 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
556 {
557 switch (type) {
558 case SLJIT_UNORDERED_OR_EQUAL:
559 case SLJIT_ORDERED_NOT_EQUAL:
560 return 2;
561
562 case SLJIT_UNORDERED:
563 case SLJIT_ORDERED:
564 return 1;
565 }
566
567 return 0;
568 }
569
570 /* --------------------------------------------------------------------- */
571 /* Entry, exit */
572 /* --------------------------------------------------------------------- */
573
574 /* Creates an index in data_transfer_insts array. */
575 #define LOAD_DATA 0x01
576 #define WORD_DATA 0x00
577 #define BYTE_DATA 0x02
578 #define HALF_DATA 0x04
579 #define INT_DATA 0x06
580 #define SIGNED_DATA 0x08
581 /* Separates integer and floating point registers */
582 #define GPR_REG 0x0f
583 #define DOUBLE_DATA 0x10
584 #define SINGLE_DATA 0x12
585
586 #define MEM_MASK 0x1f
587
588 #define ARG_TEST 0x00020
589 #define ALT_KEEP_CACHE 0x00040
590 #define CUMULATIVE_OP 0x00080
591 #define IMM_OP 0x00100
592 #define MOVE_OP 0x00200
593 #define SRC2_IMM 0x00400
594
595 #define UNUSED_DEST 0x00800
596 #define REG_DEST 0x01000
597 #define REG1_SOURCE 0x02000
598 #define REG2_SOURCE 0x04000
599 #define SLOW_SRC1 0x08000
600 #define SLOW_SRC2 0x10000
601 #define SLOW_DEST 0x20000
602
603 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
604 #define STACK_STORE SW
605 #define STACK_LOAD LW
606 #else
607 #define STACK_STORE SD
608 #define STACK_LOAD LD
609 #endif
610
611 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
612 #include "sljitNativeRISCV_32.c"
613 #else
614 #include "sljitNativeRISCV_64.c"
615 #endif
616
617 #define STACK_MAX_DISTANCE (-SIMM_MIN)
618
619 static sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg, sljit_sw argw);
620
sljit_emit_enter(struct sljit_compiler * compiler,sljit_s32 options,sljit_s32 arg_types,sljit_s32 scratches,sljit_s32 saveds,sljit_s32 fscratches,sljit_s32 fsaveds,sljit_s32 local_size)621 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
622 sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
623 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
624 {
625 sljit_s32 i, tmp, offset;
626 sljit_s32 saved_arg_count = SLJIT_KEPT_SAVEDS_COUNT(options);
627
628 CHECK_ERROR();
629 CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
630 set_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
631
632 local_size += GET_SAVED_REGISTERS_SIZE(scratches, saveds - saved_arg_count, 1);
633 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
634 if (fsaveds > 0 || fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG) {
635 if ((local_size & SSIZE_OF(sw)) != 0)
636 local_size += SSIZE_OF(sw);
637 local_size += GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
638 }
639 #else
640 local_size += GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
641 #endif
642 local_size = (local_size + SLJIT_LOCALS_OFFSET + 15) & ~0xf;
643 compiler->local_size = local_size;
644
645 if (local_size <= STACK_MAX_DISTANCE) {
646 /* Frequent case. */
647 FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_SP) | RS1(SLJIT_SP) | IMM_I(-local_size)));
648 offset = local_size - SSIZE_OF(sw);
649 local_size = 0;
650 } else {
651 FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_SP) | RS1(SLJIT_SP) | IMM_I(STACK_MAX_DISTANCE)));
652 local_size -= STACK_MAX_DISTANCE;
653
654 if (local_size > STACK_MAX_DISTANCE)
655 FAIL_IF(load_immediate(compiler, TMP_REG1, local_size, TMP_REG3));
656 offset = STACK_MAX_DISTANCE - SSIZE_OF(sw);
657 }
658
659 FAIL_IF(push_imm_s_inst(compiler, STACK_STORE | RS1(SLJIT_SP) | RS2(RETURN_ADDR_REG), offset));
660
661 tmp = SLJIT_S0 - saveds;
662 for (i = SLJIT_S0 - saved_arg_count; i > tmp; i--) {
663 offset -= SSIZE_OF(sw);
664 FAIL_IF(push_imm_s_inst(compiler, STACK_STORE | RS1(SLJIT_SP) | RS2(i), offset));
665 }
666
667 for (i = scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
668 offset -= SSIZE_OF(sw);
669 FAIL_IF(push_imm_s_inst(compiler, STACK_STORE | RS1(SLJIT_SP) | RS2(i), offset));
670 }
671
672 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
673 /* This alignment is valid because offset is not used after storing FPU regs. */
674 if ((offset & SSIZE_OF(sw)) != 0)
675 offset -= SSIZE_OF(sw);
676 #endif
677
678 tmp = SLJIT_FS0 - fsaveds;
679 for (i = SLJIT_FS0; i > tmp; i--) {
680 offset -= SSIZE_OF(f64);
681 FAIL_IF(push_imm_s_inst(compiler, FSD | RS1(SLJIT_SP) | FRS2(i), offset));
682 }
683
684 for (i = fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
685 offset -= SSIZE_OF(f64);
686 FAIL_IF(push_imm_s_inst(compiler, FSD | RS1(SLJIT_SP) | FRS2(i), offset));
687 }
688
689 if (local_size > STACK_MAX_DISTANCE)
690 FAIL_IF(push_inst(compiler, SUB | RD(SLJIT_SP) | RS1(SLJIT_SP) | RS2(TMP_REG1)));
691 else if (local_size > 0)
692 FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_SP) | RS1(SLJIT_SP) | IMM_I(-local_size)));
693
694 if (options & SLJIT_ENTER_REG_ARG)
695 return SLJIT_SUCCESS;
696
697 arg_types >>= SLJIT_ARG_SHIFT;
698 saved_arg_count = 0;
699 tmp = SLJIT_R0;
700
701 while (arg_types > 0) {
702 if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64) {
703 if (!(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG)) {
704 FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_S0 - saved_arg_count) | RS1(tmp) | IMM_I(0)));
705 saved_arg_count++;
706 }
707 tmp++;
708 }
709
710 arg_types >>= SLJIT_ARG_SHIFT;
711 }
712
713 return SLJIT_SUCCESS;
714 }
715
716 #undef STACK_MAX_DISTANCE
717
sljit_set_context(struct sljit_compiler * compiler,sljit_s32 options,sljit_s32 arg_types,sljit_s32 scratches,sljit_s32 saveds,sljit_s32 fscratches,sljit_s32 fsaveds,sljit_s32 local_size)718 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
719 sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
720 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
721 {
722 CHECK_ERROR();
723 CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
724 set_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
725
726 local_size += GET_SAVED_REGISTERS_SIZE(scratches, saveds - SLJIT_KEPT_SAVEDS_COUNT(options), 1);
727 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
728 if (fsaveds > 0 || fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG) {
729 if ((local_size & SSIZE_OF(sw)) != 0)
730 local_size += SSIZE_OF(sw);
731 local_size += GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
732 }
733 #else
734 local_size += GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, f64);
735 #endif
736 compiler->local_size = (local_size + SLJIT_LOCALS_OFFSET + 15) & ~0xf;
737
738 return SLJIT_SUCCESS;
739 }
740
741 #define STACK_MAX_DISTANCE (-SIMM_MIN - 16)
742
emit_stack_frame_release(struct sljit_compiler * compiler,sljit_s32 is_return_to)743 static sljit_s32 emit_stack_frame_release(struct sljit_compiler *compiler, sljit_s32 is_return_to)
744 {
745 sljit_s32 i, tmp, offset;
746 sljit_s32 local_size = compiler->local_size;
747
748 if (local_size > STACK_MAX_DISTANCE) {
749 local_size -= STACK_MAX_DISTANCE;
750
751 if (local_size > STACK_MAX_DISTANCE) {
752 FAIL_IF(load_immediate(compiler, TMP_REG2, local_size, TMP_REG3));
753 FAIL_IF(push_inst(compiler, ADD | RD(SLJIT_SP) | RS1(SLJIT_SP) | RS2(TMP_REG2)));
754 } else
755 FAIL_IF(push_inst(compiler, ADDI | RD(SLJIT_SP) | RS1(SLJIT_SP) | IMM_I(local_size)));
756
757 local_size = STACK_MAX_DISTANCE;
758 }
759
760 SLJIT_ASSERT(local_size > 0);
761
762 offset = local_size - SSIZE_OF(sw);
763 if (!is_return_to)
764 FAIL_IF(push_inst(compiler, STACK_LOAD | RD(RETURN_ADDR_REG) | RS1(SLJIT_SP) | IMM_I(offset)));
765
766 tmp = SLJIT_S0 - compiler->saveds;
767 for (i = SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options); i > tmp; i--) {
768 offset -= SSIZE_OF(sw);
769 FAIL_IF(push_inst(compiler, STACK_LOAD | RD(i) | RS1(SLJIT_SP) | IMM_I(offset)));
770 }
771
772 for (i = compiler->scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
773 offset -= SSIZE_OF(sw);
774 FAIL_IF(push_inst(compiler, STACK_LOAD | RD(i) | RS1(SLJIT_SP) | IMM_I(offset)));
775 }
776
777 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
778 /* This alignment is valid because offset is not used after storing FPU regs. */
779 if ((offset & SSIZE_OF(sw)) != 0)
780 offset -= SSIZE_OF(sw);
781 #endif
782
783 tmp = SLJIT_FS0 - compiler->fsaveds;
784 for (i = SLJIT_FS0; i > tmp; i--) {
785 offset -= SSIZE_OF(f64);
786 FAIL_IF(push_inst(compiler, FLD | FRD(i) | RS1(SLJIT_SP) | IMM_I(offset)));
787 }
788
789 for (i = compiler->fscratches; i >= SLJIT_FIRST_SAVED_FLOAT_REG; i--) {
790 offset -= SSIZE_OF(f64);
791 FAIL_IF(push_inst(compiler, FLD | FRD(i) | RS1(SLJIT_SP) | IMM_I(offset)));
792 }
793
794 return push_inst(compiler, ADDI | RD(SLJIT_SP) | RS1(SLJIT_SP) | IMM_I(local_size));
795 }
796
797 #undef STACK_MAX_DISTANCE
798
sljit_emit_return_void(struct sljit_compiler * compiler)799 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_void(struct sljit_compiler *compiler)
800 {
801 CHECK_ERROR();
802 CHECK(check_sljit_emit_return_void(compiler));
803
804 FAIL_IF(emit_stack_frame_release(compiler, 0));
805 return push_inst(compiler, JALR | RD(TMP_ZERO) | RS1(RETURN_ADDR_REG) | IMM_I(0));
806 }
807
sljit_emit_return_to(struct sljit_compiler * compiler,sljit_s32 src,sljit_sw srcw)808 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_to(struct sljit_compiler *compiler,
809 sljit_s32 src, sljit_sw srcw)
810 {
811 CHECK_ERROR();
812 CHECK(check_sljit_emit_return_to(compiler, src, srcw));
813
814 if (src & SLJIT_MEM) {
815 ADJUST_LOCAL_OFFSET(src, srcw);
816 FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_REG1, src, srcw));
817 src = TMP_REG1;
818 srcw = 0;
819 } else if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
820 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(src) | IMM_I(0)));
821 src = TMP_REG1;
822 srcw = 0;
823 }
824
825 FAIL_IF(emit_stack_frame_release(compiler, 1));
826
827 SLJIT_SKIP_CHECKS(compiler);
828 return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
829 }
830
831 /* --------------------------------------------------------------------- */
832 /* Operators */
833 /* --------------------------------------------------------------------- */
834
835 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
836 #define ARCH_32_64(a, b) a
837 #else
838 #define ARCH_32_64(a, b) b
839 #endif
840
841 static const sljit_ins data_transfer_insts[16 + 4] = {
842 /* u w s */ ARCH_32_64(F3(0x2) | OPC(0x23) /* sw */, F3(0x3) | OPC(0x23) /* sd */),
843 /* u w l */ ARCH_32_64(F3(0x2) | OPC(0x3) /* lw */, F3(0x3) | OPC(0x3) /* ld */),
844 /* u b s */ F3(0x0) | OPC(0x23) /* sb */,
845 /* u b l */ F3(0x4) | OPC(0x3) /* lbu */,
846 /* u h s */ F3(0x1) | OPC(0x23) /* sh */,
847 /* u h l */ F3(0x5) | OPC(0x3) /* lhu */,
848 /* u i s */ F3(0x2) | OPC(0x23) /* sw */,
849 /* u i l */ ARCH_32_64(F3(0x2) | OPC(0x3) /* lw */, F3(0x6) | OPC(0x3) /* lwu */),
850
851 /* s w s */ ARCH_32_64(F3(0x2) | OPC(0x23) /* sw */, F3(0x3) | OPC(0x23) /* sd */),
852 /* s w l */ ARCH_32_64(F3(0x2) | OPC(0x3) /* lw */, F3(0x3) | OPC(0x3) /* ld */),
853 /* s b s */ F3(0x0) | OPC(0x23) /* sb */,
854 /* s b l */ F3(0x0) | OPC(0x3) /* lb */,
855 /* s h s */ F3(0x1) | OPC(0x23) /* sh */,
856 /* s h l */ F3(0x1) | OPC(0x3) /* lh */,
857 /* s i s */ F3(0x2) | OPC(0x23) /* sw */,
858 /* s i l */ F3(0x2) | OPC(0x3) /* lw */,
859
860 /* d s */ F3(0x3) | OPC(0x27) /* fsd */,
861 /* d l */ F3(0x3) | OPC(0x7) /* fld */,
862 /* s s */ F3(0x2) | OPC(0x27) /* fsw */,
863 /* s l */ F3(0x2) | OPC(0x7) /* flw */,
864 };
865
866 #undef ARCH_32_64
867
push_mem_inst(struct sljit_compiler * compiler,sljit_s32 flags,sljit_s32 reg,sljit_s32 base,sljit_sw offset)868 static sljit_s32 push_mem_inst(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 base, sljit_sw offset)
869 {
870 sljit_ins ins;
871
872 SLJIT_ASSERT(FAST_IS_REG(base) && offset <= 0xfff && offset >= SIMM_MIN);
873
874 ins = data_transfer_insts[flags & MEM_MASK] | RS1(base);
875 if (flags & LOAD_DATA)
876 ins |= ((flags & MEM_MASK) <= GPR_REG ? RD(reg) : FRD(reg)) | IMM_I(offset);
877 else
878 ins |= ((flags & MEM_MASK) <= GPR_REG ? RS2(reg) : FRS2(reg)) | IMM_S(offset);
879
880 return push_inst(compiler, ins);
881 }
882
883 /* Can perform an operation using at most 1 instruction. */
getput_arg_fast(struct sljit_compiler * compiler,sljit_s32 flags,sljit_s32 reg,sljit_s32 arg,sljit_sw argw)884 static sljit_s32 getput_arg_fast(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg, sljit_sw argw)
885 {
886
887 SLJIT_ASSERT(arg & SLJIT_MEM);
888
889 if (!(arg & OFFS_REG_MASK) && argw <= SIMM_MAX && argw >= SIMM_MIN) {
890 /* Works for both absoulte and relative addresses. */
891 if (SLJIT_UNLIKELY(flags & ARG_TEST))
892 return 1;
893
894 FAIL_IF(push_mem_inst(compiler, flags, reg, arg & REG_MASK, argw));
895 return -1;
896 }
897 return 0;
898 }
899
900 #define TO_ARGW_HI(argw) (((argw) & ~0xfff) + (((argw) & 0x800) ? 0x1000 : 0))
901
902 /* See getput_arg below.
903 Note: can_cache is called only for binary operators. */
can_cache(sljit_s32 arg,sljit_sw argw,sljit_s32 next_arg,sljit_sw next_argw)904 static sljit_s32 can_cache(sljit_s32 arg, sljit_sw argw, sljit_s32 next_arg, sljit_sw next_argw)
905 {
906 SLJIT_ASSERT((arg & SLJIT_MEM) && (next_arg & SLJIT_MEM));
907
908 /* Simple operation except for updates. */
909 if (arg & OFFS_REG_MASK) {
910 argw &= 0x3;
911 next_argw &= 0x3;
912 if (argw && argw == next_argw && (arg == next_arg || (arg & OFFS_REG_MASK) == (next_arg & OFFS_REG_MASK)))
913 return 1;
914 return 0;
915 }
916
917 if (arg == next_arg) {
918 if (((next_argw - argw) <= SIMM_MAX && (next_argw - argw) >= SIMM_MIN)
919 || TO_ARGW_HI(argw) == TO_ARGW_HI(next_argw))
920 return 1;
921 return 0;
922 }
923
924 return 0;
925 }
926
927 /* Emit the necessary instructions. See can_cache above. */
getput_arg(struct sljit_compiler * compiler,sljit_s32 flags,sljit_s32 reg,sljit_s32 arg,sljit_sw argw,sljit_s32 next_arg,sljit_sw next_argw)928 static sljit_s32 getput_arg(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg, sljit_sw argw, sljit_s32 next_arg, sljit_sw next_argw)
929 {
930 sljit_s32 base = arg & REG_MASK;
931 sljit_s32 tmp_r = TMP_REG1;
932 sljit_sw offset, argw_hi;
933
934 SLJIT_ASSERT(arg & SLJIT_MEM);
935 if (!(next_arg & SLJIT_MEM)) {
936 next_arg = 0;
937 next_argw = 0;
938 }
939
940 /* Since tmp can be the same as base or offset registers,
941 * these might be unavailable after modifying tmp. */
942 if ((flags & MEM_MASK) <= GPR_REG && (flags & LOAD_DATA) && reg == TMP_REG2)
943 tmp_r = reg;
944
945 if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
946 argw &= 0x3;
947
948 /* Using the cache. */
949 if (argw == compiler->cache_argw) {
950 if (arg == compiler->cache_arg)
951 return push_mem_inst(compiler, flags, reg, TMP_REG3, 0);
952
953 if ((SLJIT_MEM | (arg & OFFS_REG_MASK)) == compiler->cache_arg) {
954 if (arg == next_arg && argw == (next_argw & 0x3)) {
955 compiler->cache_arg = arg;
956 compiler->cache_argw = argw;
957 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG3) | RS1(TMP_REG3) | RS2(base)));
958 return push_mem_inst(compiler, flags, reg, TMP_REG3, 0);
959 }
960 FAIL_IF(push_inst(compiler, ADD | RD(tmp_r) | RS1(base) | RS2(TMP_REG3)));
961 return push_mem_inst(compiler, flags, reg, tmp_r, 0);
962 }
963 }
964
965 if (SLJIT_UNLIKELY(argw)) {
966 compiler->cache_arg = SLJIT_MEM | (arg & OFFS_REG_MASK);
967 compiler->cache_argw = argw;
968 FAIL_IF(push_inst(compiler, SLLI | RD(TMP_REG3) | RS1(OFFS_REG(arg)) | IMM_I(argw)));
969 }
970
971 if (arg == next_arg && argw == (next_argw & 0x3)) {
972 compiler->cache_arg = arg;
973 compiler->cache_argw = argw;
974 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG3) | RS1(base) | RS2(!argw ? OFFS_REG(arg) : TMP_REG3)));
975 tmp_r = TMP_REG3;
976 }
977 else
978 FAIL_IF(push_inst(compiler, ADD | RD(tmp_r) | RS1(base) | RS2(!argw ? OFFS_REG(arg) : TMP_REG3)));
979 return push_mem_inst(compiler, flags, reg, tmp_r, 0);
980 }
981
982 if (compiler->cache_arg == arg && argw - compiler->cache_argw <= SIMM_MAX && argw - compiler->cache_argw >= SIMM_MIN)
983 return push_mem_inst(compiler, flags, reg, TMP_REG3, argw - compiler->cache_argw);
984
985 if (compiler->cache_arg == SLJIT_MEM && (argw - compiler->cache_argw <= SIMM_MAX) && (argw - compiler->cache_argw >= SIMM_MIN)) {
986 offset = argw - compiler->cache_argw;
987 } else {
988 compiler->cache_arg = SLJIT_MEM;
989
990 argw_hi = TO_ARGW_HI(argw);
991
992 if (next_arg && next_argw - argw <= SIMM_MAX && next_argw - argw >= SIMM_MIN && argw_hi != TO_ARGW_HI(next_argw)) {
993 FAIL_IF(load_immediate(compiler, TMP_REG3, argw, tmp_r));
994 compiler->cache_argw = argw;
995 offset = 0;
996 } else {
997 FAIL_IF(load_immediate(compiler, TMP_REG3, argw_hi, tmp_r));
998 compiler->cache_argw = argw_hi;
999 offset = argw & 0xfff;
1000 argw = argw_hi;
1001 }
1002 }
1003
1004 if (!base)
1005 return push_mem_inst(compiler, flags, reg, TMP_REG3, offset);
1006
1007 if (arg == next_arg && next_argw - argw <= SIMM_MAX && next_argw - argw >= SIMM_MIN) {
1008 compiler->cache_arg = arg;
1009 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG3) | RS1(TMP_REG3) | RS2(base)));
1010 return push_mem_inst(compiler, flags, reg, TMP_REG3, offset);
1011 }
1012
1013 FAIL_IF(push_inst(compiler, ADD | RD(tmp_r) | RS1(TMP_REG3) | RS2(base)));
1014 return push_mem_inst(compiler, flags, reg, tmp_r, offset);
1015 }
1016
emit_op_mem(struct sljit_compiler * compiler,sljit_s32 flags,sljit_s32 reg,sljit_s32 arg,sljit_sw argw)1017 static sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg, sljit_sw argw)
1018 {
1019 sljit_s32 base = arg & REG_MASK;
1020 sljit_s32 tmp_r = TMP_REG1;
1021
1022 if (getput_arg_fast(compiler, flags, reg, arg, argw))
1023 return compiler->error;
1024
1025 if ((flags & MEM_MASK) <= GPR_REG && (flags & LOAD_DATA))
1026 tmp_r = reg;
1027
1028 if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
1029 argw &= 0x3;
1030
1031 if (SLJIT_UNLIKELY(argw)) {
1032 FAIL_IF(push_inst(compiler, SLLI | RD(tmp_r) | RS1(OFFS_REG(arg)) | IMM_I(argw)));
1033 FAIL_IF(push_inst(compiler, ADD | RD(tmp_r) | RS1(tmp_r) | RS2(base)));
1034 }
1035 else
1036 FAIL_IF(push_inst(compiler, ADD | RD(tmp_r) | RS1(base) | RS2(OFFS_REG(arg))));
1037
1038 argw = 0;
1039 } else {
1040 FAIL_IF(load_immediate(compiler, tmp_r, TO_ARGW_HI(argw), TMP_REG3));
1041
1042 if (base != 0)
1043 FAIL_IF(push_inst(compiler, ADD | RD(tmp_r) | RS1(tmp_r) | RS2(base)));
1044 }
1045
1046 return push_mem_inst(compiler, flags, reg, tmp_r, argw & 0xfff);
1047 }
1048
emit_op_mem2(struct sljit_compiler * compiler,sljit_s32 flags,sljit_s32 reg,sljit_s32 arg1,sljit_sw arg1w,sljit_s32 arg2,sljit_sw arg2w)1049 static SLJIT_INLINE sljit_s32 emit_op_mem2(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg1, sljit_sw arg1w, sljit_s32 arg2, sljit_sw arg2w)
1050 {
1051 if (getput_arg_fast(compiler, flags, reg, arg1, arg1w))
1052 return compiler->error;
1053 return getput_arg(compiler, flags, reg, arg1, arg1w, arg2, arg2w);
1054 }
1055
1056 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1057 #define WORD 0
1058 #define WORD_32 0
1059 #define IMM_EXTEND(v) (IMM_I(v))
1060 #else /* !SLJIT_CONFIG_RISCV_32 */
1061 #define WORD word
1062 #define WORD_32 0x08
1063 #define IMM_EXTEND(v) (IMM_I((op & SLJIT_32) ? (v) : (32 + (v))))
1064 #endif /* SLJIT_CONFIG_RISCV_32 */
1065
emit_clz_ctz(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw src)1066 static sljit_s32 emit_clz_ctz(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 dst, sljit_sw src)
1067 {
1068 sljit_s32 is_clz = (GET_OPCODE(op) == SLJIT_CLZ);
1069 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1070 sljit_ins word = (sljit_ins)(op & SLJIT_32) >> 5;
1071 sljit_ins word_size = (op & SLJIT_32) ? 32 : 64;
1072 #else /* !SLJIT_CONFIG_RISCV_64 */
1073 sljit_ins word_size = 32;
1074 #endif /* SLJIT_CONFIG_RISCV_64 */
1075
1076 SLJIT_ASSERT(WORD == 0 || WORD == 0x8);
1077
1078 /* The OTHER_FLAG is the counter. */
1079 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(OTHER_FLAG) | RS1(TMP_ZERO) | IMM_I(word_size)));
1080
1081 /* The TMP_REG2 is the next value. */
1082 if (src != TMP_REG2)
1083 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(TMP_REG2) | RS1(src) | IMM_I(0)));
1084
1085 FAIL_IF(push_inst(compiler, BEQ | RS1(TMP_REG2) | RS2(TMP_ZERO) | ((sljit_ins)((is_clz ? 4 : 5) * SSIZE_OF(ins)) << 7) | ((sljit_ins)(8 * SSIZE_OF(ins)) << 20)));
1086
1087 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(OTHER_FLAG) | RS1(TMP_ZERO) | IMM_I(0)));
1088 if (!is_clz) {
1089 FAIL_IF(push_inst(compiler, ANDI | RD(TMP_REG1) | RS1(TMP_REG2) | IMM_I(1)));
1090 FAIL_IF(push_inst(compiler, BNE | RS1(TMP_REG1) | RS2(TMP_ZERO) | ((sljit_ins)(2 * SSIZE_OF(ins)) << 7) | ((sljit_ins)(8 * SSIZE_OF(ins)) << 20)));
1091 } else
1092 FAIL_IF(push_inst(compiler, BLT | RS1(TMP_REG2) | RS2(TMP_ZERO) | ((sljit_ins)(2 * SSIZE_OF(ins)) << 7) | ((sljit_ins)(8 * SSIZE_OF(ins)) << 20)));
1093
1094 /* The TMP_REG1 is the next shift. */
1095 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(TMP_REG1) | RS1(TMP_ZERO) | IMM_I(word_size)));
1096
1097 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(EQUAL_FLAG) | RS1(TMP_REG2) | IMM_I(0)));
1098 FAIL_IF(push_inst(compiler, SRLI | WORD | RD(TMP_REG1) | RS1(TMP_REG1) | IMM_I(1)));
1099
1100 FAIL_IF(push_inst(compiler, (is_clz ? SRL : SLL) | WORD | RD(TMP_REG2) | RS1(EQUAL_FLAG) | RS2(TMP_REG1)));
1101 FAIL_IF(push_inst(compiler, BNE | RS1(TMP_REG2) | RS2(TMP_ZERO) | ((sljit_ins)0xfe000e80 - ((2 * SSIZE_OF(ins)) << 7))));
1102 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(TMP_REG2) | RS1(TMP_REG1) | IMM_I(-1)));
1103 FAIL_IF(push_inst(compiler, (is_clz ? SRL : SLL) | WORD | RD(TMP_REG2) | RS1(EQUAL_FLAG) | RS2(TMP_REG2)));
1104 FAIL_IF(push_inst(compiler, OR | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | RS2(TMP_REG1)));
1105 FAIL_IF(push_inst(compiler, BEQ | RS1(TMP_REG2) | RS2(TMP_ZERO) | ((sljit_ins)0xfe000e80 - ((5 * SSIZE_OF(ins)) << 7))));
1106
1107 return push_inst(compiler, ADDI | WORD | RD(dst) | RS1(OTHER_FLAG) | IMM_I(0));
1108 }
1109
emit_rev(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw src)1110 static sljit_s32 emit_rev(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 dst, sljit_sw src)
1111 {
1112 SLJIT_UNUSED_ARG(op);
1113
1114 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1115 if (!(op & SLJIT_32)) {
1116 FAIL_IF(push_inst(compiler, LUI | RD(OTHER_FLAG) | 0x10000));
1117 FAIL_IF(push_inst(compiler, SRLI | RD(TMP_REG1) | RS1(src) | IMM_I(32)));
1118 FAIL_IF(push_inst(compiler, ADDI | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | IMM_I(0xfff)));
1119 FAIL_IF(push_inst(compiler, SLLI | RD(dst) | RS1(src) | IMM_I(32)));
1120 FAIL_IF(push_inst(compiler, SLLI | RD(EQUAL_FLAG) | RS1(OTHER_FLAG) | IMM_I(32)));
1121 FAIL_IF(push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(TMP_REG1)));
1122 FAIL_IF(push_inst(compiler, OR | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | RS2(EQUAL_FLAG)));
1123
1124 FAIL_IF(push_inst(compiler, SRLI | RD(TMP_REG1) | RS1(dst) | IMM_I(16)));
1125 FAIL_IF(push_inst(compiler, AND | RD(dst) | RS1(dst) | RS2(OTHER_FLAG)));
1126 FAIL_IF(push_inst(compiler, AND | RD(TMP_REG1) | RS1(TMP_REG1) | RS2(OTHER_FLAG)));
1127 FAIL_IF(push_inst(compiler, SLLI | RD(EQUAL_FLAG) | RS1(OTHER_FLAG) | IMM_I(8)));
1128 FAIL_IF(push_inst(compiler, SLLI | RD(dst) | RS1(dst) | IMM_I(16)));
1129 FAIL_IF(push_inst(compiler, XOR | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | RS2(EQUAL_FLAG)));
1130 FAIL_IF(push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(TMP_REG1)));
1131
1132 FAIL_IF(push_inst(compiler, SRLI | RD(TMP_REG1) | RS1(dst) | IMM_I(8)));
1133 FAIL_IF(push_inst(compiler, AND | RD(dst) | RS1(dst) | RS2(OTHER_FLAG)));
1134 FAIL_IF(push_inst(compiler, AND | RD(TMP_REG1) | RS1(TMP_REG1) | RS2(OTHER_FLAG)));
1135 FAIL_IF(push_inst(compiler, SLLI | RD(dst) | RS1(dst) | IMM_I(8)));
1136 return push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(TMP_REG1));
1137 }
1138 #endif /* SLJIT_CONFIG_RISCV_64 */
1139
1140 FAIL_IF(push_inst(compiler, SRLI | WORD_32 | RD(TMP_REG1) | RS1(src) | IMM_I(16)));
1141 FAIL_IF(push_inst(compiler, LUI | RD(OTHER_FLAG) | 0xff0000));
1142 FAIL_IF(push_inst(compiler, SLLI | WORD_32 | RD(dst) | RS1(src) | IMM_I(16)));
1143 FAIL_IF(push_inst(compiler, ORI | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | IMM_I(0xff)));
1144 FAIL_IF(push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(TMP_REG1)));
1145
1146 FAIL_IF(push_inst(compiler, SRLI | WORD_32 | RD(TMP_REG1) | RS1(dst) | IMM_I(8)));
1147 FAIL_IF(push_inst(compiler, AND | RD(dst) | RS1(dst) | RS2(OTHER_FLAG)));
1148 FAIL_IF(push_inst(compiler, AND | RD(TMP_REG1) | RS1(TMP_REG1) | RS2(OTHER_FLAG)));
1149 FAIL_IF(push_inst(compiler, SLLI | WORD_32 | RD(dst) | RS1(dst) | IMM_I(8)));
1150 return push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(TMP_REG1));
1151 }
1152
emit_rev16(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw src)1153 static sljit_s32 emit_rev16(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 dst, sljit_sw src)
1154 {
1155 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1156 sljit_ins word = (sljit_ins)(op & SLJIT_32) >> 5;
1157 sljit_ins word_size = (op & SLJIT_32) ? 32 : 64;
1158 #else /* !SLJIT_CONFIG_RISCV_64 */
1159 sljit_ins word_size = 32;
1160 #endif /* SLJIT_CONFIG_RISCV_64 */
1161
1162 FAIL_IF(push_inst(compiler, SRLI | WORD | RD(TMP_REG1) | RS1(src) | IMM_I(8)));
1163 FAIL_IF(push_inst(compiler, SLLI | WORD | RD(dst) | RS1(src) | IMM_I(word_size - 8)));
1164 FAIL_IF(push_inst(compiler, ANDI | RD(TMP_REG1) | RS1(TMP_REG1) | IMM_I(0xff)));
1165 FAIL_IF(push_inst(compiler, (GET_OPCODE(op) == SLJIT_REV_U16 ? SRLI : SRAI) | WORD | RD(dst) | RS1(dst) | IMM_I(word_size - 16)));
1166 return push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(TMP_REG1));
1167 }
1168
1169 #define EMIT_LOGICAL(op_imm, op_reg) \
1170 if (flags & SRC2_IMM) { \
1171 if (op & SLJIT_SET_Z) \
1172 FAIL_IF(push_inst(compiler, op_imm | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(src2))); \
1173 if (!(flags & UNUSED_DEST)) \
1174 FAIL_IF(push_inst(compiler, op_imm | RD(dst) | RS1(src1) | IMM_I(src2))); \
1175 } \
1176 else { \
1177 if (op & SLJIT_SET_Z) \
1178 FAIL_IF(push_inst(compiler, op_reg | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2))); \
1179 if (!(flags & UNUSED_DEST)) \
1180 FAIL_IF(push_inst(compiler, op_reg | RD(dst) | RS1(src1) | RS2(src2))); \
1181 }
1182
1183 #define EMIT_SHIFT(imm, reg) \
1184 op_imm = (imm); \
1185 op_reg = (reg);
1186
emit_single_op(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 flags,sljit_s32 dst,sljit_s32 src1,sljit_sw src2)1187 static SLJIT_INLINE sljit_s32 emit_single_op(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 flags,
1188 sljit_s32 dst, sljit_s32 src1, sljit_sw src2)
1189 {
1190 sljit_s32 is_overflow, is_carry, carry_src_r, is_handled;
1191 sljit_ins op_imm, op_reg;
1192 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1193 sljit_ins word = (sljit_ins)(op & SLJIT_32) >> 5;
1194 #endif /* SLJIT_CONFIG_RISCV_64 */
1195
1196 SLJIT_ASSERT(WORD == 0 || WORD == 0x8);
1197
1198 switch (GET_OPCODE(op)) {
1199 case SLJIT_MOV:
1200 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM));
1201 if (dst != src2)
1202 return push_inst(compiler, ADDI | RD(dst) | RS1(src2) | IMM_I(0));
1203 return SLJIT_SUCCESS;
1204
1205 case SLJIT_MOV_U8:
1206 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM));
1207 if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE))
1208 return push_inst(compiler, ANDI | RD(dst) | RS1(src2) | IMM_I(0xff));
1209 SLJIT_ASSERT(dst == src2);
1210 return SLJIT_SUCCESS;
1211
1212 case SLJIT_MOV_S8:
1213 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM));
1214 if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) {
1215 FAIL_IF(push_inst(compiler, SLLI | WORD | RD(dst) | RS1(src2) | IMM_EXTEND(24)));
1216 return push_inst(compiler, SRAI | WORD | RD(dst) | RS1(dst) | IMM_EXTEND(24));
1217 }
1218 SLJIT_ASSERT(dst == src2);
1219 return SLJIT_SUCCESS;
1220
1221 case SLJIT_MOV_U16:
1222 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM));
1223 if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) {
1224 FAIL_IF(push_inst(compiler, SLLI | WORD | RD(dst) | RS1(src2) | IMM_EXTEND(16)));
1225 return push_inst(compiler, SRLI | WORD | RD(dst) | RS1(dst) | IMM_EXTEND(16));
1226 }
1227 SLJIT_ASSERT(dst == src2);
1228 return SLJIT_SUCCESS;
1229
1230 case SLJIT_MOV_S16:
1231 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM));
1232 if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) {
1233 FAIL_IF(push_inst(compiler, SLLI | WORD | RD(dst) | RS1(src2) | IMM_EXTEND(16)));
1234 return push_inst(compiler, SRAI | WORD | RD(dst) | RS1(dst) | IMM_EXTEND(16));
1235 }
1236 SLJIT_ASSERT(dst == src2);
1237 return SLJIT_SUCCESS;
1238
1239 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1240 case SLJIT_MOV_U32:
1241 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM));
1242 if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE)) {
1243 FAIL_IF(push_inst(compiler, SLLI | RD(dst) | RS1(src2) | IMM_I(32)));
1244 return push_inst(compiler, SRLI | RD(dst) | RS1(dst) | IMM_I(32));
1245 }
1246 SLJIT_ASSERT(dst == src2);
1247 return SLJIT_SUCCESS;
1248
1249 case SLJIT_MOV_S32:
1250 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM));
1251 if ((flags & (REG_DEST | REG2_SOURCE)) == (REG_DEST | REG2_SOURCE))
1252 return push_inst(compiler, ADDI | 0x8 | RD(dst) | RS1(src2) | IMM_I(0));
1253 SLJIT_ASSERT(dst == src2);
1254 return SLJIT_SUCCESS;
1255 #endif /* SLJIT_CONFIG_RISCV_64 */
1256
1257 case SLJIT_CLZ:
1258 case SLJIT_CTZ:
1259 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM));
1260 return emit_clz_ctz(compiler, op, dst, src2);
1261
1262 case SLJIT_REV:
1263 case SLJIT_REV_S32:
1264 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1265 case SLJIT_REV_U32:
1266 #endif /* SLJIT_CONFIG_RISCV_32 */
1267 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM));
1268 return emit_rev(compiler, op, dst, src2);
1269
1270 case SLJIT_REV_U16:
1271 case SLJIT_REV_S16:
1272 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM));
1273 return emit_rev16(compiler, op, dst, src2);
1274
1275 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1276 case SLJIT_REV_U32:
1277 SLJIT_ASSERT(src1 == TMP_REG1 && !(flags & SRC2_IMM) && dst != TMP_REG1);
1278 FAIL_IF(emit_rev(compiler, op, dst, src2));
1279 if (dst == TMP_REG2)
1280 return SLJIT_SUCCESS;
1281 FAIL_IF(push_inst(compiler, SLLI | RD(dst) | RS1(dst) | IMM_I(32)));
1282 return push_inst(compiler, SRLI | RD(dst) | RS1(dst) | IMM_I(32));
1283 #endif /* SLJIT_CONFIG_RISCV_32 */
1284
1285 case SLJIT_ADD:
1286 /* Overflow computation (both add and sub): overflow = src1_sign ^ src2_sign ^ result_sign ^ carry_flag */
1287 is_overflow = GET_FLAG_TYPE(op) == SLJIT_OVERFLOW;
1288 carry_src_r = GET_FLAG_TYPE(op) == SLJIT_CARRY;
1289
1290 if (flags & SRC2_IMM) {
1291 if (is_overflow) {
1292 if (src2 >= 0)
1293 FAIL_IF(push_inst(compiler, ADDI | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(0)));
1294 else
1295 FAIL_IF(push_inst(compiler, XORI | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(-1)));
1296 }
1297 else if (op & SLJIT_SET_Z)
1298 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(src2)));
1299
1300 /* Only the zero flag is needed. */
1301 if (!(flags & UNUSED_DEST) || (op & VARIABLE_FLAG_MASK))
1302 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(dst) | RS1(src1) | IMM_I(src2)));
1303 }
1304 else {
1305 if (is_overflow)
1306 FAIL_IF(push_inst(compiler, XOR | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
1307 else if (op & SLJIT_SET_Z)
1308 FAIL_IF(push_inst(compiler, ADD | WORD | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
1309
1310 if (is_overflow || carry_src_r != 0) {
1311 if (src1 != dst)
1312 carry_src_r = (sljit_s32)src1;
1313 else if (src2 != dst)
1314 carry_src_r = (sljit_s32)src2;
1315 else {
1316 FAIL_IF(push_inst(compiler, ADDI | RD(OTHER_FLAG) | RS1(src1) | IMM_I(0)));
1317 carry_src_r = OTHER_FLAG;
1318 }
1319 }
1320
1321 /* Only the zero flag is needed. */
1322 if (!(flags & UNUSED_DEST) || (op & VARIABLE_FLAG_MASK))
1323 FAIL_IF(push_inst(compiler, ADD | WORD | RD(dst) | RS1(src1) | RS2(src2)));
1324 }
1325
1326 /* Carry is zero if a + b >= a or a + b >= b, otherwise it is 1. */
1327 if (is_overflow || carry_src_r != 0) {
1328 if (flags & SRC2_IMM)
1329 FAIL_IF(push_inst(compiler, SLTUI | RD(OTHER_FLAG) | RS1(dst) | IMM_I(src2)));
1330 else
1331 FAIL_IF(push_inst(compiler, SLTU | RD(OTHER_FLAG) | RS1(dst) | RS2(carry_src_r)));
1332 }
1333
1334 if (!is_overflow)
1335 return SLJIT_SUCCESS;
1336
1337 FAIL_IF(push_inst(compiler, XOR | RD(TMP_REG1) | RS1(dst) | RS2(EQUAL_FLAG)));
1338 if (op & SLJIT_SET_Z)
1339 FAIL_IF(push_inst(compiler, ADDI | RD(EQUAL_FLAG) | RS1(dst) | IMM_I(0)));
1340 FAIL_IF(push_inst(compiler, SRLI | WORD | RD(TMP_REG1) | RS1(TMP_REG1) | IMM_EXTEND(31)));
1341 return push_inst(compiler, XOR | RD(OTHER_FLAG) | RS1(TMP_REG1) | RS2(OTHER_FLAG));
1342
1343 case SLJIT_ADDC:
1344 carry_src_r = GET_FLAG_TYPE(op) == SLJIT_CARRY;
1345
1346 if (flags & SRC2_IMM) {
1347 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(dst) | RS1(src1) | IMM_I(src2)));
1348 } else {
1349 if (carry_src_r != 0) {
1350 if (src1 != dst)
1351 carry_src_r = (sljit_s32)src1;
1352 else if (src2 != dst)
1353 carry_src_r = (sljit_s32)src2;
1354 else {
1355 FAIL_IF(push_inst(compiler, ADDI | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(0)));
1356 carry_src_r = EQUAL_FLAG;
1357 }
1358 }
1359
1360 FAIL_IF(push_inst(compiler, ADD | WORD | RD(dst) | RS1(src1) | RS2(src2)));
1361 }
1362
1363 /* Carry is zero if a + b >= a or a + b >= b, otherwise it is 1. */
1364 if (carry_src_r != 0) {
1365 if (flags & SRC2_IMM)
1366 FAIL_IF(push_inst(compiler, SLTUI | RD(EQUAL_FLAG) | RS1(dst) | IMM_I(src2)));
1367 else
1368 FAIL_IF(push_inst(compiler, SLTU | RD(EQUAL_FLAG) | RS1(dst) | RS2(carry_src_r)));
1369 }
1370
1371 FAIL_IF(push_inst(compiler, ADD | WORD | RD(dst) | RS1(dst) | RS2(OTHER_FLAG)));
1372
1373 if (carry_src_r == 0)
1374 return SLJIT_SUCCESS;
1375
1376 /* Set ULESS_FLAG (dst == 0) && (OTHER_FLAG == 1). */
1377 FAIL_IF(push_inst(compiler, SLTU | RD(OTHER_FLAG) | RS1(dst) | RS2(OTHER_FLAG)));
1378 /* Set carry flag. */
1379 return push_inst(compiler, OR | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | RS2(EQUAL_FLAG));
1380
1381 case SLJIT_SUB:
1382 if ((flags & SRC2_IMM) && src2 == SIMM_MIN) {
1383 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG2) | RS1(TMP_ZERO) | IMM_I(src2)));
1384 src2 = TMP_REG2;
1385 flags &= ~SRC2_IMM;
1386 }
1387
1388 is_handled = 0;
1389
1390 if (flags & SRC2_IMM) {
1391 if (GET_FLAG_TYPE(op) == SLJIT_LESS) {
1392 FAIL_IF(push_inst(compiler, SLTUI | RD(OTHER_FLAG) | RS1(src1) | IMM_I(src2)));
1393 is_handled = 1;
1394 }
1395 else if (GET_FLAG_TYPE(op) == SLJIT_SIG_LESS) {
1396 FAIL_IF(push_inst(compiler, SLTI | RD(OTHER_FLAG) | RS1(src1) | IMM_I(src2)));
1397 is_handled = 1;
1398 }
1399 }
1400
1401 if (!is_handled && GET_FLAG_TYPE(op) >= SLJIT_LESS && GET_FLAG_TYPE(op) <= SLJIT_SIG_LESS_EQUAL) {
1402 is_handled = 1;
1403
1404 if (flags & SRC2_IMM) {
1405 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG2) | RS1(TMP_ZERO) | IMM_I(src2)));
1406 src2 = TMP_REG2;
1407 flags &= ~SRC2_IMM;
1408 }
1409
1410 switch (GET_FLAG_TYPE(op)) {
1411 case SLJIT_LESS:
1412 FAIL_IF(push_inst(compiler, SLTU | RD(OTHER_FLAG) | RS1(src1) | RS2(src2)));
1413 break;
1414 case SLJIT_GREATER:
1415 FAIL_IF(push_inst(compiler, SLTU | RD(OTHER_FLAG) | RS1(src2) | RS2(src1)));
1416 break;
1417 case SLJIT_SIG_LESS:
1418 FAIL_IF(push_inst(compiler, SLT | RD(OTHER_FLAG) | RS1(src1) | RS2(src2)));
1419 break;
1420 case SLJIT_SIG_GREATER:
1421 FAIL_IF(push_inst(compiler, SLT | RD(OTHER_FLAG) | RS1(src2) | RS2(src1)));
1422 break;
1423 }
1424 }
1425
1426 if (is_handled) {
1427 if (flags & SRC2_IMM) {
1428 if (op & SLJIT_SET_Z)
1429 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(-src2)));
1430 if (!(flags & UNUSED_DEST))
1431 return push_inst(compiler, ADDI | WORD | RD(dst) | RS1(src1) | IMM_I(-src2));
1432 }
1433 else {
1434 if (op & SLJIT_SET_Z)
1435 FAIL_IF(push_inst(compiler, SUB | WORD | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
1436 if (!(flags & UNUSED_DEST))
1437 return push_inst(compiler, SUB | WORD | RD(dst) | RS1(src1) | RS2(src2));
1438 }
1439 return SLJIT_SUCCESS;
1440 }
1441
1442 is_overflow = GET_FLAG_TYPE(op) == SLJIT_OVERFLOW;
1443 is_carry = GET_FLAG_TYPE(op) == SLJIT_CARRY;
1444
1445 if (flags & SRC2_IMM) {
1446 if (is_overflow) {
1447 if (src2 >= 0)
1448 FAIL_IF(push_inst(compiler, ADDI | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(0)));
1449 else
1450 FAIL_IF(push_inst(compiler, XORI | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(-1)));
1451 }
1452 else if (op & SLJIT_SET_Z)
1453 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(-src2)));
1454
1455 if (is_overflow || is_carry)
1456 FAIL_IF(push_inst(compiler, SLTUI | RD(OTHER_FLAG) | RS1(src1) | IMM_I(src2)));
1457
1458 /* Only the zero flag is needed. */
1459 if (!(flags & UNUSED_DEST) || (op & VARIABLE_FLAG_MASK))
1460 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(dst) | RS1(src1) | IMM_I(-src2)));
1461 }
1462 else {
1463 if (is_overflow)
1464 FAIL_IF(push_inst(compiler, XOR | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
1465 else if (op & SLJIT_SET_Z)
1466 FAIL_IF(push_inst(compiler, SUB | WORD | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
1467
1468 if (is_overflow || is_carry)
1469 FAIL_IF(push_inst(compiler, SLTU | RD(OTHER_FLAG) | RS1(src1) | RS2(src2)));
1470
1471 /* Only the zero flag is needed. */
1472 if (!(flags & UNUSED_DEST) || (op & VARIABLE_FLAG_MASK))
1473 FAIL_IF(push_inst(compiler, SUB | WORD | RD(dst) | RS1(src1) | RS2(src2)));
1474 }
1475
1476 if (!is_overflow)
1477 return SLJIT_SUCCESS;
1478
1479 FAIL_IF(push_inst(compiler, XOR | RD(TMP_REG1) | RS1(dst) | RS2(EQUAL_FLAG)));
1480 if (op & SLJIT_SET_Z)
1481 FAIL_IF(push_inst(compiler, ADDI | RD(EQUAL_FLAG) | RS1(dst) | IMM_I(0)));
1482 FAIL_IF(push_inst(compiler, SRLI | WORD | RD(TMP_REG1) | RS1(TMP_REG1) | IMM_EXTEND(31)));
1483 return push_inst(compiler, XOR | RD(OTHER_FLAG) | RS1(TMP_REG1) | RS2(OTHER_FLAG));
1484
1485 case SLJIT_SUBC:
1486 if ((flags & SRC2_IMM) && src2 == SIMM_MIN) {
1487 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG2) | RS1(TMP_ZERO) | IMM_I(src2)));
1488 src2 = TMP_REG2;
1489 flags &= ~SRC2_IMM;
1490 }
1491
1492 is_carry = GET_FLAG_TYPE(op) == SLJIT_CARRY;
1493
1494 if (flags & SRC2_IMM) {
1495 if (is_carry)
1496 FAIL_IF(push_inst(compiler, SLTUI | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(src2)));
1497
1498 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(dst) | RS1(src1) | IMM_I(-src2)));
1499 }
1500 else {
1501 if (is_carry)
1502 FAIL_IF(push_inst(compiler, SLTU | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
1503
1504 FAIL_IF(push_inst(compiler, SUB | WORD | RD(dst) | RS1(src1) | RS2(src2)));
1505 }
1506
1507 if (is_carry)
1508 FAIL_IF(push_inst(compiler, SLTU | RD(TMP_REG1) | RS1(dst) | RS2(OTHER_FLAG)));
1509
1510 FAIL_IF(push_inst(compiler, SUB | WORD | RD(dst) | RS1(dst) | RS2(OTHER_FLAG)));
1511
1512 if (!is_carry)
1513 return SLJIT_SUCCESS;
1514
1515 return push_inst(compiler, OR | RD(OTHER_FLAG) | RS1(EQUAL_FLAG) | RS2(TMP_REG1));
1516
1517 case SLJIT_MUL:
1518 SLJIT_ASSERT(!(flags & SRC2_IMM));
1519
1520 if (GET_FLAG_TYPE(op) != SLJIT_OVERFLOW)
1521 return push_inst(compiler, MUL | WORD | RD(dst) | RS1(src1) | RS2(src2));
1522
1523 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1524 if (word) {
1525 FAIL_IF(push_inst(compiler, MUL | RD(OTHER_FLAG) | RS1(src1) | RS2(src2)));
1526 FAIL_IF(push_inst(compiler, MUL | 0x8 | RD(dst) | RS1(src1) | RS2(src2)));
1527 return push_inst(compiler, SUB | RD(OTHER_FLAG) | RS1(dst) | RS2(OTHER_FLAG));
1528 }
1529 #endif /* SLJIT_CONFIG_RISCV_64 */
1530
1531 FAIL_IF(push_inst(compiler, MULH | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
1532 FAIL_IF(push_inst(compiler, MUL | RD(dst) | RS1(src1) | RS2(src2)));
1533 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1534 FAIL_IF(push_inst(compiler, SRAI | RD(OTHER_FLAG) | RS1(dst) | IMM_I(31)));
1535 #else /* !SLJIT_CONFIG_RISCV_32 */
1536 FAIL_IF(push_inst(compiler, SRAI | RD(OTHER_FLAG) | RS1(dst) | IMM_I(63)));
1537 #endif /* SLJIT_CONFIG_RISCV_32 */
1538 return push_inst(compiler, SUB | RD(OTHER_FLAG) | RS1(EQUAL_FLAG) | RS2(OTHER_FLAG));
1539
1540 case SLJIT_AND:
1541 EMIT_LOGICAL(ANDI, AND);
1542 return SLJIT_SUCCESS;
1543
1544 case SLJIT_OR:
1545 EMIT_LOGICAL(ORI, OR);
1546 return SLJIT_SUCCESS;
1547
1548 case SLJIT_XOR:
1549 EMIT_LOGICAL(XORI, XOR);
1550 return SLJIT_SUCCESS;
1551
1552 case SLJIT_SHL:
1553 case SLJIT_MSHL:
1554 EMIT_SHIFT(SLLI, SLL);
1555 break;
1556
1557 case SLJIT_LSHR:
1558 case SLJIT_MLSHR:
1559 EMIT_SHIFT(SRLI, SRL);
1560 break;
1561
1562 case SLJIT_ASHR:
1563 case SLJIT_MASHR:
1564 EMIT_SHIFT(SRAI, SRA);
1565 break;
1566
1567 case SLJIT_ROTL:
1568 case SLJIT_ROTR:
1569 if (flags & SRC2_IMM) {
1570 SLJIT_ASSERT(src2 != 0);
1571
1572 op_imm = (GET_OPCODE(op) == SLJIT_ROTL) ? SLLI : SRLI;
1573 FAIL_IF(push_inst(compiler, op_imm | WORD | RD(OTHER_FLAG) | RS1(src1) | IMM_I(src2)));
1574
1575 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1576 src2 = ((op & SLJIT_32) ? 32 : 64) - src2;
1577 #else /* !SLJIT_CONFIG_RISCV_64 */
1578 src2 = 32 - src2;
1579 #endif /* SLJIT_CONFIG_RISCV_64 */
1580 op_imm = (GET_OPCODE(op) == SLJIT_ROTL) ? SRLI : SLLI;
1581 FAIL_IF(push_inst(compiler, op_imm | WORD | RD(dst) | RS1(src1) | IMM_I(src2)));
1582 return push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(OTHER_FLAG));
1583 }
1584
1585 if (src2 == TMP_ZERO) {
1586 if (dst != src1)
1587 return push_inst(compiler, ADDI | WORD | RD(dst) | RS1(src1) | IMM_I(0));
1588 return SLJIT_SUCCESS;
1589 }
1590
1591 FAIL_IF(push_inst(compiler, SUB | WORD | RD(EQUAL_FLAG) | RS1(TMP_ZERO) | RS2(src2)));
1592 op_reg = (GET_OPCODE(op) == SLJIT_ROTL) ? SLL : SRL;
1593 FAIL_IF(push_inst(compiler, op_reg | WORD | RD(OTHER_FLAG) | RS1(src1) | RS2(src2)));
1594 op_reg = (GET_OPCODE(op) == SLJIT_ROTL) ? SRL : SLL;
1595 FAIL_IF(push_inst(compiler, op_reg | WORD | RD(dst) | RS1(src1) | RS2(EQUAL_FLAG)));
1596 return push_inst(compiler, OR | RD(dst) | RS1(dst) | RS2(OTHER_FLAG));
1597
1598 default:
1599 SLJIT_UNREACHABLE();
1600 return SLJIT_SUCCESS;
1601 }
1602
1603 if (flags & SRC2_IMM) {
1604 if (op & SLJIT_SET_Z)
1605 FAIL_IF(push_inst(compiler, op_imm | WORD | RD(EQUAL_FLAG) | RS1(src1) | IMM_I(src2)));
1606
1607 if (flags & UNUSED_DEST)
1608 return SLJIT_SUCCESS;
1609 return push_inst(compiler, op_imm | WORD | RD(dst) | RS1(src1) | IMM_I(src2));
1610 }
1611
1612 if (op & SLJIT_SET_Z)
1613 FAIL_IF(push_inst(compiler, op_reg | WORD | RD(EQUAL_FLAG) | RS1(src1) | RS2(src2)));
1614
1615 if (flags & UNUSED_DEST)
1616 return SLJIT_SUCCESS;
1617 return push_inst(compiler, op_reg | WORD | RD(dst) | RS1(src1) | RS2(src2));
1618 }
1619
1620 #undef IMM_EXTEND
1621
emit_op(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 flags,sljit_s32 dst,sljit_sw dstw,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)1622 static sljit_s32 emit_op(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 flags,
1623 sljit_s32 dst, sljit_sw dstw,
1624 sljit_s32 src1, sljit_sw src1w,
1625 sljit_s32 src2, sljit_sw src2w)
1626 {
1627 /* arg1 goes to TMP_REG1 or src reg
1628 arg2 goes to TMP_REG2, imm or src reg
1629 TMP_REG3 can be used for caching
1630 result goes to TMP_REG2, so put result can use TMP_REG1 and TMP_REG3. */
1631 sljit_s32 dst_r = TMP_REG2;
1632 sljit_s32 src1_r;
1633 sljit_sw src2_r = 0;
1634 sljit_s32 sugg_src2_r = TMP_REG2;
1635
1636 if (!(flags & ALT_KEEP_CACHE)) {
1637 compiler->cache_arg = 0;
1638 compiler->cache_argw = 0;
1639 }
1640
1641 if (dst == 0) {
1642 SLJIT_ASSERT(HAS_FLAGS(op));
1643 flags |= UNUSED_DEST;
1644 dst = TMP_REG2;
1645 }
1646 else if (FAST_IS_REG(dst)) {
1647 dst_r = dst;
1648 flags |= REG_DEST;
1649 if (flags & MOVE_OP)
1650 sugg_src2_r = dst_r;
1651 }
1652 else if ((dst & SLJIT_MEM) && !getput_arg_fast(compiler, flags | ARG_TEST, TMP_REG1, dst, dstw))
1653 flags |= SLOW_DEST;
1654
1655 if (flags & IMM_OP) {
1656 if (src2 == SLJIT_IMM && src2w != 0 && src2w <= SIMM_MAX && src2w >= SIMM_MIN) {
1657 flags |= SRC2_IMM;
1658 src2_r = src2w;
1659 }
1660 else if ((flags & CUMULATIVE_OP) && src1 == SLJIT_IMM && src1w != 0 && src1w <= SIMM_MAX && src1w >= SIMM_MIN) {
1661 flags |= SRC2_IMM;
1662 src2_r = src1w;
1663
1664 /* And swap arguments. */
1665 src1 = src2;
1666 src1w = src2w;
1667 src2 = SLJIT_IMM;
1668 /* src2w = src2_r unneeded. */
1669 }
1670 }
1671
1672 /* Source 1. */
1673 if (FAST_IS_REG(src1)) {
1674 src1_r = src1;
1675 flags |= REG1_SOURCE;
1676 }
1677 else if (src1 == SLJIT_IMM) {
1678 if (src1w) {
1679 FAIL_IF(load_immediate(compiler, TMP_REG1, src1w, TMP_REG3));
1680 src1_r = TMP_REG1;
1681 }
1682 else
1683 src1_r = TMP_ZERO;
1684 }
1685 else {
1686 if (getput_arg_fast(compiler, flags | LOAD_DATA, TMP_REG1, src1, src1w))
1687 FAIL_IF(compiler->error);
1688 else
1689 flags |= SLOW_SRC1;
1690 src1_r = TMP_REG1;
1691 }
1692
1693 /* Source 2. */
1694 if (FAST_IS_REG(src2)) {
1695 src2_r = src2;
1696 flags |= REG2_SOURCE;
1697 if ((flags & (REG_DEST | MOVE_OP)) == MOVE_OP)
1698 dst_r = (sljit_s32)src2_r;
1699 }
1700 else if (src2 == SLJIT_IMM) {
1701 if (!(flags & SRC2_IMM)) {
1702 if (src2w) {
1703 FAIL_IF(load_immediate(compiler, sugg_src2_r, src2w, TMP_REG3));
1704 src2_r = sugg_src2_r;
1705 }
1706 else {
1707 src2_r = TMP_ZERO;
1708 if (flags & MOVE_OP) {
1709 if (dst & SLJIT_MEM)
1710 dst_r = 0;
1711 else
1712 op = SLJIT_MOV;
1713 }
1714 }
1715 }
1716 }
1717 else {
1718 if (getput_arg_fast(compiler, flags | LOAD_DATA, sugg_src2_r, src2, src2w))
1719 FAIL_IF(compiler->error);
1720 else
1721 flags |= SLOW_SRC2;
1722 src2_r = sugg_src2_r;
1723 }
1724
1725 if ((flags & (SLOW_SRC1 | SLOW_SRC2)) == (SLOW_SRC1 | SLOW_SRC2)) {
1726 SLJIT_ASSERT(src2_r == TMP_REG2);
1727 if (!can_cache(src1, src1w, src2, src2w) && can_cache(src1, src1w, dst, dstw)) {
1728 FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG2, src2, src2w, src1, src1w));
1729 FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG1, src1, src1w, dst, dstw));
1730 }
1731 else {
1732 FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG1, src1, src1w, src2, src2w));
1733 FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG2, src2, src2w, dst, dstw));
1734 }
1735 }
1736 else if (flags & SLOW_SRC1)
1737 FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, TMP_REG1, src1, src1w, dst, dstw));
1738 else if (flags & SLOW_SRC2)
1739 FAIL_IF(getput_arg(compiler, flags | LOAD_DATA, sugg_src2_r, src2, src2w, dst, dstw));
1740
1741 FAIL_IF(emit_single_op(compiler, op, flags, dst_r, src1_r, src2_r));
1742
1743 if (dst & SLJIT_MEM) {
1744 if (!(flags & SLOW_DEST)) {
1745 getput_arg_fast(compiler, flags, dst_r, dst, dstw);
1746 return compiler->error;
1747 }
1748 return getput_arg(compiler, flags, dst_r, dst, dstw, 0, 0);
1749 }
1750
1751 return SLJIT_SUCCESS;
1752 }
1753
sljit_emit_op0(struct sljit_compiler * compiler,sljit_s32 op)1754 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1755 {
1756 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1757 sljit_ins word = (sljit_ins)(op & SLJIT_32) >> 5;
1758
1759 SLJIT_ASSERT(word == 0 || word == 0x8);
1760 #endif /* SLJIT_CONFIG_RISCV_64 */
1761
1762 CHECK_ERROR();
1763 CHECK(check_sljit_emit_op0(compiler, op));
1764
1765 switch (GET_OPCODE(op)) {
1766 case SLJIT_BREAKPOINT:
1767 return push_inst(compiler, EBREAK);
1768 case SLJIT_NOP:
1769 return push_inst(compiler, ADDI | RD(TMP_ZERO) | RS1(TMP_ZERO) | IMM_I(0));
1770 case SLJIT_LMUL_UW:
1771 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(SLJIT_R1) | IMM_I(0)));
1772 FAIL_IF(push_inst(compiler, MULHU | RD(SLJIT_R1) | RS1(SLJIT_R0) | RS2(SLJIT_R1)));
1773 return push_inst(compiler, MUL | RD(SLJIT_R0) | RS1(SLJIT_R0) | RS2(TMP_REG1));
1774 case SLJIT_LMUL_SW:
1775 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(SLJIT_R1) | IMM_I(0)));
1776 FAIL_IF(push_inst(compiler, MULH | RD(SLJIT_R1) | RS1(SLJIT_R0) | RS2(SLJIT_R1)));
1777 return push_inst(compiler, MUL | RD(SLJIT_R0) | RS1(SLJIT_R0) | RS2(TMP_REG1));
1778 case SLJIT_DIVMOD_UW:
1779 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(SLJIT_R0) | IMM_I(0)));
1780 FAIL_IF(push_inst(compiler, DIVU | WORD | RD(SLJIT_R0) | RS1(SLJIT_R0) | RS2(SLJIT_R1)));
1781 return push_inst(compiler, REMU | WORD | RD(SLJIT_R1) | RS1(TMP_REG1) | RS2(SLJIT_R1));
1782 case SLJIT_DIVMOD_SW:
1783 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(SLJIT_R0) | IMM_I(0)));
1784 FAIL_IF(push_inst(compiler, DIV | WORD | RD(SLJIT_R0) | RS1(SLJIT_R0) | RS2(SLJIT_R1)));
1785 return push_inst(compiler, REM | WORD | RD(SLJIT_R1) | RS1(TMP_REG1) | RS2(SLJIT_R1));
1786 case SLJIT_DIV_UW:
1787 return push_inst(compiler, DIVU | WORD | RD(SLJIT_R0) | RS1(SLJIT_R0) | RS2(SLJIT_R1));
1788 case SLJIT_DIV_SW:
1789 return push_inst(compiler, DIV | WORD | RD(SLJIT_R0) | RS1(SLJIT_R0) | RS2(SLJIT_R1));
1790 case SLJIT_ENDBR:
1791 case SLJIT_SKIP_FRAMES_BEFORE_RETURN:
1792 return SLJIT_SUCCESS;
1793 }
1794
1795 return SLJIT_SUCCESS;
1796 }
1797
sljit_emit_op1(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)1798 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1799 sljit_s32 dst, sljit_sw dstw,
1800 sljit_s32 src, sljit_sw srcw)
1801 {
1802 sljit_s32 flags = 0;
1803
1804 CHECK_ERROR();
1805 CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
1806 ADJUST_LOCAL_OFFSET(dst, dstw);
1807 ADJUST_LOCAL_OFFSET(src, srcw);
1808
1809 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1810 if (op & SLJIT_32)
1811 flags = INT_DATA | SIGNED_DATA;
1812 #endif
1813
1814 switch (GET_OPCODE(op)) {
1815 case SLJIT_MOV:
1816 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1817 case SLJIT_MOV_U32:
1818 case SLJIT_MOV_S32:
1819 case SLJIT_MOV32:
1820 #endif
1821 case SLJIT_MOV_P:
1822 return emit_op(compiler, SLJIT_MOV, WORD_DATA | MOVE_OP, dst, dstw, TMP_REG1, 0, src, srcw);
1823
1824 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1825 case SLJIT_MOV_U32:
1826 return emit_op(compiler, SLJIT_MOV_U32, INT_DATA | MOVE_OP, dst, dstw, TMP_REG1, 0, src, (src == SLJIT_IMM) ? (sljit_u32)srcw : srcw);
1827
1828 case SLJIT_MOV_S32:
1829 /* Logical operators have no W variant, so sign extended input is necessary for them. */
1830 case SLJIT_MOV32:
1831 return emit_op(compiler, SLJIT_MOV_S32, INT_DATA | SIGNED_DATA | MOVE_OP, dst, dstw, TMP_REG1, 0, src, (src == SLJIT_IMM) ? (sljit_s32)srcw : srcw);
1832 #endif
1833
1834 case SLJIT_MOV_U8:
1835 return emit_op(compiler, op, BYTE_DATA | MOVE_OP, dst, dstw, TMP_REG1, 0, src, (src == SLJIT_IMM) ? (sljit_u8)srcw : srcw);
1836
1837 case SLJIT_MOV_S8:
1838 return emit_op(compiler, op, BYTE_DATA | SIGNED_DATA | MOVE_OP, dst, dstw, TMP_REG1, 0, src, (src == SLJIT_IMM) ? (sljit_s8)srcw : srcw);
1839
1840 case SLJIT_MOV_U16:
1841 return emit_op(compiler, op, HALF_DATA | MOVE_OP, dst, dstw, TMP_REG1, 0, src, (src == SLJIT_IMM) ? (sljit_u16)srcw : srcw);
1842
1843 case SLJIT_MOV_S16:
1844 return emit_op(compiler, op, HALF_DATA | SIGNED_DATA | MOVE_OP, dst, dstw, TMP_REG1, 0, src, (src == SLJIT_IMM) ? (sljit_s16)srcw : srcw);
1845
1846 case SLJIT_CLZ:
1847 case SLJIT_CTZ:
1848 case SLJIT_REV:
1849 return emit_op(compiler, op, flags, dst, dstw, TMP_REG1, 0, src, srcw);
1850
1851 case SLJIT_REV_U16:
1852 case SLJIT_REV_S16:
1853 return emit_op(compiler, op, HALF_DATA, dst, dstw, TMP_REG1, 0, src, srcw);
1854
1855 case SLJIT_REV_U32:
1856 case SLJIT_REV_S32:
1857 return emit_op(compiler, op | SLJIT_32, INT_DATA, dst, dstw, TMP_REG1, 0, src, srcw);
1858 }
1859
1860 SLJIT_UNREACHABLE();
1861 return SLJIT_SUCCESS;
1862 }
1863
sljit_emit_op2(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)1864 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
1865 sljit_s32 dst, sljit_sw dstw,
1866 sljit_s32 src1, sljit_sw src1w,
1867 sljit_s32 src2, sljit_sw src2w)
1868 {
1869 sljit_s32 flags = 0;
1870
1871 CHECK_ERROR();
1872 CHECK(check_sljit_emit_op2(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w));
1873 ADJUST_LOCAL_OFFSET(dst, dstw);
1874 ADJUST_LOCAL_OFFSET(src1, src1w);
1875 ADJUST_LOCAL_OFFSET(src2, src2w);
1876
1877 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1878 if (op & SLJIT_32) {
1879 flags |= INT_DATA | SIGNED_DATA;
1880 if (src1 == SLJIT_IMM)
1881 src1w = (sljit_s32)src1w;
1882 if (src2 == SLJIT_IMM)
1883 src2w = (sljit_s32)src2w;
1884 }
1885 #endif
1886
1887 switch (GET_OPCODE(op)) {
1888 case SLJIT_ADD:
1889 case SLJIT_ADDC:
1890 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
1891 return emit_op(compiler, op, flags | CUMULATIVE_OP | IMM_OP, dst, dstw, src1, src1w, src2, src2w);
1892
1893 case SLJIT_SUB:
1894 case SLJIT_SUBC:
1895 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
1896 return emit_op(compiler, op, flags | IMM_OP, dst, dstw, src1, src1w, src2, src2w);
1897
1898 case SLJIT_MUL:
1899 compiler->status_flags_state = 0;
1900 return emit_op(compiler, op, flags | CUMULATIVE_OP, dst, dstw, src1, src1w, src2, src2w);
1901
1902 case SLJIT_AND:
1903 case SLJIT_OR:
1904 case SLJIT_XOR:
1905 return emit_op(compiler, op, flags | CUMULATIVE_OP | IMM_OP, dst, dstw, src1, src1w, src2, src2w);
1906
1907 case SLJIT_SHL:
1908 case SLJIT_MSHL:
1909 case SLJIT_LSHR:
1910 case SLJIT_MLSHR:
1911 case SLJIT_ASHR:
1912 case SLJIT_MASHR:
1913 case SLJIT_ROTL:
1914 case SLJIT_ROTR:
1915 if (src2 == SLJIT_IMM) {
1916 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
1917 src2w &= 0x1f;
1918 #else /* !SLJIT_CONFIG_RISCV_32 */
1919 if (op & SLJIT_32)
1920 src2w &= 0x1f;
1921 else
1922 src2w &= 0x3f;
1923 #endif /* SLJIT_CONFIG_RISCV_32 */
1924 }
1925
1926 return emit_op(compiler, op, flags | IMM_OP, dst, dstw, src1, src1w, src2, src2w);
1927 }
1928
1929 SLJIT_UNREACHABLE();
1930 return SLJIT_SUCCESS;
1931 }
1932
sljit_emit_op2u(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)1933 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2u(struct sljit_compiler *compiler, sljit_s32 op,
1934 sljit_s32 src1, sljit_sw src1w,
1935 sljit_s32 src2, sljit_sw src2w)
1936 {
1937 CHECK_ERROR();
1938 CHECK(check_sljit_emit_op2(compiler, op, 1, 0, 0, src1, src1w, src2, src2w));
1939
1940 SLJIT_SKIP_CHECKS(compiler);
1941 return sljit_emit_op2(compiler, op, 0, 0, src1, src1w, src2, src2w);
1942 }
1943
sljit_emit_shift_into(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst_reg,sljit_s32 src1_reg,sljit_s32 src2_reg,sljit_s32 src3,sljit_sw src3w)1944 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
1945 sljit_s32 dst_reg,
1946 sljit_s32 src1_reg,
1947 sljit_s32 src2_reg,
1948 sljit_s32 src3, sljit_sw src3w)
1949 {
1950 sljit_s32 is_left;
1951 sljit_ins ins1, ins2, ins3;
1952 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
1953 sljit_ins word = (sljit_ins)(op & SLJIT_32) >> 5;
1954 sljit_s32 inp_flags = ((op & SLJIT_32) ? INT_DATA : WORD_DATA) | LOAD_DATA;
1955 sljit_sw bit_length = (op & SLJIT_32) ? 32 : 64;
1956 #else /* !SLJIT_CONFIG_RISCV_64 */
1957 sljit_s32 inp_flags = WORD_DATA | LOAD_DATA;
1958 sljit_sw bit_length = 32;
1959 #endif /* SLJIT_CONFIG_RISCV_64 */
1960
1961 SLJIT_ASSERT(WORD == 0 || WORD == 0x8);
1962
1963 CHECK_ERROR();
1964 CHECK(check_sljit_emit_shift_into(compiler, op, dst_reg, src1_reg, src2_reg, src3, src3w));
1965
1966 is_left = (GET_OPCODE(op) == SLJIT_SHL || GET_OPCODE(op) == SLJIT_MSHL);
1967
1968 if (src1_reg == src2_reg) {
1969 SLJIT_SKIP_CHECKS(compiler);
1970 return sljit_emit_op2(compiler, (is_left ? SLJIT_ROTL : SLJIT_ROTR) | (op & SLJIT_32), dst_reg, 0, src1_reg, 0, src3, src3w);
1971 }
1972
1973 ADJUST_LOCAL_OFFSET(src3, src3w);
1974
1975 if (src3 == SLJIT_IMM) {
1976 src3w &= bit_length - 1;
1977
1978 if (src3w == 0)
1979 return SLJIT_SUCCESS;
1980
1981 if (is_left) {
1982 ins1 = SLLI | WORD | IMM_I(src3w);
1983 src3w = bit_length - src3w;
1984 ins2 = SRLI | WORD | IMM_I(src3w);
1985 } else {
1986 ins1 = SRLI | WORD | IMM_I(src3w);
1987 src3w = bit_length - src3w;
1988 ins2 = SLLI | WORD | IMM_I(src3w);
1989 }
1990
1991 FAIL_IF(push_inst(compiler, ins1 | RD(dst_reg) | RS1(src1_reg)));
1992 FAIL_IF(push_inst(compiler, ins2 | RD(TMP_REG1) | RS1(src2_reg)));
1993 return push_inst(compiler, OR | RD(dst_reg) | RS1(dst_reg) | RS2(TMP_REG1));
1994 }
1995
1996 if (src3 & SLJIT_MEM) {
1997 FAIL_IF(emit_op_mem(compiler, inp_flags, TMP_REG2, src3, src3w));
1998 src3 = TMP_REG2;
1999 } else if (dst_reg == src3) {
2000 push_inst(compiler, ADDI | WORD | RD(TMP_REG2) | RS1(src3) | IMM_I(0));
2001 src3 = TMP_REG2;
2002 }
2003
2004 if (is_left) {
2005 ins1 = SLL;
2006 ins2 = SRLI;
2007 ins3 = SRL;
2008 } else {
2009 ins1 = SRL;
2010 ins2 = SLLI;
2011 ins3 = SLL;
2012 }
2013
2014 FAIL_IF(push_inst(compiler, ins1 | WORD | RD(dst_reg) | RS1(src1_reg) | RS2(src3)));
2015
2016 if (!(op & SLJIT_SHIFT_INTO_NON_ZERO)) {
2017 FAIL_IF(push_inst(compiler, ins2 | WORD | RD(TMP_REG1) | RS1(src2_reg) | IMM_I(1)));
2018 FAIL_IF(push_inst(compiler, XORI | RD(TMP_REG2) | RS1(src3) | IMM_I((sljit_ins)bit_length - 1)));
2019 src2_reg = TMP_REG1;
2020 } else
2021 FAIL_IF(push_inst(compiler, SUB | WORD | RD(TMP_REG2) | RS1(TMP_ZERO) | RS2(src3)));
2022
2023 FAIL_IF(push_inst(compiler, ins3 | WORD | RD(TMP_REG1) | RS1(src2_reg) | RS2(TMP_REG2)));
2024 return push_inst(compiler, OR | RD(dst_reg) | RS1(dst_reg) | RS2(TMP_REG1));
2025 }
2026
sljit_emit_op_src(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src,sljit_sw srcw)2027 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
2028 sljit_s32 src, sljit_sw srcw)
2029 {
2030 CHECK_ERROR();
2031 CHECK(check_sljit_emit_op_src(compiler, op, src, srcw));
2032 ADJUST_LOCAL_OFFSET(src, srcw);
2033
2034 switch (op) {
2035 case SLJIT_FAST_RETURN:
2036 if (FAST_IS_REG(src))
2037 FAIL_IF(push_inst(compiler, ADDI | RD(RETURN_ADDR_REG) | RS1(src) | IMM_I(0)));
2038 else
2039 FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, RETURN_ADDR_REG, src, srcw));
2040
2041 return push_inst(compiler, JALR | RD(TMP_ZERO) | RS1(RETURN_ADDR_REG) | IMM_I(0));
2042 case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN:
2043 return SLJIT_SUCCESS;
2044 case SLJIT_PREFETCH_L1:
2045 case SLJIT_PREFETCH_L2:
2046 case SLJIT_PREFETCH_L3:
2047 case SLJIT_PREFETCH_ONCE:
2048 return SLJIT_SUCCESS;
2049 }
2050
2051 return SLJIT_SUCCESS;
2052 }
2053
sljit_emit_op_dst(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw)2054 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_dst(struct sljit_compiler *compiler, sljit_s32 op,
2055 sljit_s32 dst, sljit_sw dstw)
2056 {
2057 sljit_s32 dst_r;
2058
2059 CHECK_ERROR();
2060 CHECK(check_sljit_emit_op_dst(compiler, op, dst, dstw));
2061 ADJUST_LOCAL_OFFSET(dst, dstw);
2062
2063 switch (op) {
2064 case SLJIT_FAST_ENTER:
2065 if (FAST_IS_REG(dst))
2066 return push_inst(compiler, ADDI | RD(dst) | RS1(RETURN_ADDR_REG) | IMM_I(0));
2067
2068 SLJIT_ASSERT(RETURN_ADDR_REG == TMP_REG2);
2069 break;
2070 case SLJIT_GET_RETURN_ADDRESS:
2071 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
2072 FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, dst_r, SLJIT_MEM1(SLJIT_SP), compiler->local_size - SSIZE_OF(sw)));
2073 break;
2074 }
2075
2076 if (dst & SLJIT_MEM)
2077 return emit_op_mem(compiler, WORD_DATA, TMP_REG2, dst, dstw);
2078
2079 return SLJIT_SUCCESS;
2080 }
2081
sljit_get_register_index(sljit_s32 type,sljit_s32 reg)2082 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 type, sljit_s32 reg)
2083 {
2084 CHECK_REG_INDEX(check_sljit_get_register_index(type, reg));
2085
2086 if (type == SLJIT_GP_REGISTER)
2087 return reg_map[reg];
2088
2089 if (type != SLJIT_FLOAT_REGISTER)
2090 return -1;
2091
2092 return freg_map[reg];
2093 }
2094
sljit_emit_op_custom(struct sljit_compiler * compiler,void * instruction,sljit_u32 size)2095 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
2096 void *instruction, sljit_u32 size)
2097 {
2098 SLJIT_UNUSED_ARG(size);
2099
2100 CHECK_ERROR();
2101 CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
2102
2103 return push_inst(compiler, *(sljit_ins*)instruction);
2104 }
2105
2106 /* --------------------------------------------------------------------- */
2107 /* Floating point operators */
2108 /* --------------------------------------------------------------------- */
2109
2110 #define FLOAT_DATA(op) (DOUBLE_DATA | ((op & SLJIT_32) >> 7))
2111 #define FMT(op) ((sljit_ins)((op & SLJIT_32) ^ SLJIT_32) << 17)
2112
sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)2113 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
2114 sljit_s32 dst, sljit_sw dstw,
2115 sljit_s32 src, sljit_sw srcw)
2116 {
2117 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2118 # define flags (sljit_u32)0
2119 #else
2120 sljit_u32 flags = ((sljit_u32)(GET_OPCODE(op) == SLJIT_CONV_SW_FROM_F64)) << 21;
2121 #endif
2122 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
2123
2124 if (src & SLJIT_MEM) {
2125 FAIL_IF(emit_op_mem2(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src, srcw, dst, dstw));
2126 src = TMP_FREG1;
2127 }
2128
2129 FAIL_IF(push_inst(compiler, FCVT_W_S | FMT(op) | flags | RD(dst_r) | FRS1(src)));
2130
2131 /* Store the integer value from a VFP register. */
2132 if (dst & SLJIT_MEM) {
2133 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2134 return emit_op_mem2(compiler, WORD_DATA, TMP_REG2, dst, dstw, 0, 0);
2135 #else
2136 return emit_op_mem2(compiler, flags ? WORD_DATA : INT_DATA, TMP_REG2, dst, dstw, 0, 0);
2137 #endif
2138 }
2139 return SLJIT_SUCCESS;
2140
2141 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2142 # undef flags
2143 #endif
2144 }
2145
sljit_emit_fop1_conv_f64_from_w(struct sljit_compiler * compiler,sljit_ins ins,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)2146 static sljit_s32 sljit_emit_fop1_conv_f64_from_w(struct sljit_compiler *compiler, sljit_ins ins,
2147 sljit_s32 dst, sljit_sw dstw,
2148 sljit_s32 src, sljit_sw srcw)
2149 {
2150 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2151
2152 if (src & SLJIT_MEM) {
2153 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2154 FAIL_IF(emit_op_mem2(compiler, WORD_DATA | LOAD_DATA, TMP_REG1, src, srcw, dst, dstw));
2155 #else /* SLJIT_CONFIG_RISCV_32 */
2156 FAIL_IF(emit_op_mem2(compiler, ((ins & (1 << 21)) ? WORD_DATA : INT_DATA) | LOAD_DATA, TMP_REG1, src, srcw, dst, dstw));
2157 #endif /* !SLJIT_CONFIG_RISCV_32 */
2158 src = TMP_REG1;
2159 } else if (src == SLJIT_IMM) {
2160 FAIL_IF(load_immediate(compiler, TMP_REG1, srcw, TMP_REG3));
2161 src = TMP_REG1;
2162 }
2163
2164 FAIL_IF(push_inst(compiler, ins | FRD(dst_r) | RS1(src)));
2165
2166 if (dst & SLJIT_MEM)
2167 return emit_op_mem2(compiler, DOUBLE_DATA | ((sljit_s32)(~ins >> 24) & 0x2), TMP_FREG1, dst, dstw, 0, 0);
2168 return SLJIT_SUCCESS;
2169 }
2170
sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)2171 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler *compiler, sljit_s32 op,
2172 sljit_s32 dst, sljit_sw dstw,
2173 sljit_s32 src, sljit_sw srcw)
2174 {
2175 sljit_ins ins = FCVT_S_W | FMT(op);
2176
2177 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2178 if (op & SLJIT_32)
2179 ins |= F3(0x7);
2180 #else /* !SLJIT_CONFIG_RISCV_32 */
2181 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_SW)
2182 ins |= (1 << 21);
2183 else if (src == SLJIT_IMM)
2184 srcw = (sljit_s32)srcw;
2185
2186 if (op != SLJIT_CONV_F64_FROM_S32)
2187 ins |= F3(0x7);
2188 #endif /* SLJIT_CONFIG_RISCV_32 */
2189
2190 return sljit_emit_fop1_conv_f64_from_w(compiler, ins, dst, dstw, src, srcw);
2191 }
2192
sljit_emit_fop1_conv_f64_from_uw(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)2193 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_uw(struct sljit_compiler *compiler, sljit_s32 op,
2194 sljit_s32 dst, sljit_sw dstw,
2195 sljit_s32 src, sljit_sw srcw)
2196 {
2197 sljit_ins ins = FCVT_S_WU | FMT(op);
2198
2199 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2200 if (op & SLJIT_32)
2201 ins |= F3(0x7);
2202 #else /* !SLJIT_CONFIG_RISCV_32 */
2203 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_UW)
2204 ins |= (1 << 21);
2205 else if (src == SLJIT_IMM)
2206 srcw = (sljit_u32)srcw;
2207
2208 if (op != SLJIT_CONV_F64_FROM_S32)
2209 ins |= F3(0x7);
2210 #endif /* SLJIT_CONFIG_RISCV_32 */
2211
2212 return sljit_emit_fop1_conv_f64_from_w(compiler, ins, dst, dstw, src, srcw);
2213 }
2214
sljit_emit_fop1_cmp(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)2215 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
2216 sljit_s32 src1, sljit_sw src1w,
2217 sljit_s32 src2, sljit_sw src2w)
2218 {
2219 sljit_ins inst;
2220
2221 if (src1 & SLJIT_MEM) {
2222 FAIL_IF(emit_op_mem2(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w, src2, src2w));
2223 src1 = TMP_FREG1;
2224 }
2225
2226 if (src2 & SLJIT_MEM) {
2227 FAIL_IF(emit_op_mem2(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w, 0, 0));
2228 src2 = TMP_FREG2;
2229 }
2230
2231 switch (GET_FLAG_TYPE(op)) {
2232 case SLJIT_F_EQUAL:
2233 case SLJIT_ORDERED_EQUAL:
2234 inst = FEQ_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src1) | FRS2(src2);
2235 break;
2236 case SLJIT_F_LESS:
2237 case SLJIT_ORDERED_LESS:
2238 inst = FLT_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src1) | FRS2(src2);
2239 break;
2240 case SLJIT_ORDERED_GREATER:
2241 inst = FLT_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src2) | FRS2(src1);
2242 break;
2243 case SLJIT_F_GREATER:
2244 case SLJIT_UNORDERED_OR_GREATER:
2245 inst = FLE_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src1) | FRS2(src2);
2246 break;
2247 case SLJIT_UNORDERED_OR_LESS:
2248 inst = FLE_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src2) | FRS2(src1);
2249 break;
2250 case SLJIT_UNORDERED_OR_EQUAL:
2251 FAIL_IF(push_inst(compiler, FLT_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src1) | FRS2(src2)));
2252 FAIL_IF(push_inst(compiler, FLT_S | FMT(op) | RD(TMP_REG1) | FRS1(src2) | FRS2(src1)));
2253 inst = OR | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | RS2(TMP_REG1);
2254 break;
2255 default: /* SLJIT_UNORDERED */
2256 if (src1 == src2) {
2257 inst = FEQ_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src1) | FRS2(src1);
2258 break;
2259 }
2260 FAIL_IF(push_inst(compiler, FEQ_S | FMT(op) | RD(OTHER_FLAG) | FRS1(src1) | FRS2(src1)));
2261 FAIL_IF(push_inst(compiler, FEQ_S | FMT(op) | RD(TMP_REG1) | FRS1(src2) | FRS2(src2)));
2262 inst = AND | RD(OTHER_FLAG) | RS1(OTHER_FLAG) | RS2(TMP_REG1);
2263 break;
2264 }
2265
2266 return push_inst(compiler, inst);
2267 }
2268
sljit_emit_fop1(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)2269 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
2270 sljit_s32 dst, sljit_sw dstw,
2271 sljit_s32 src, sljit_sw srcw)
2272 {
2273 sljit_s32 dst_r;
2274
2275 CHECK_ERROR();
2276 compiler->cache_arg = 0;
2277 compiler->cache_argw = 0;
2278
2279 SLJIT_COMPILE_ASSERT((SLJIT_32 == 0x100) && !(DOUBLE_DATA & 0x2), float_transfer_bit_error);
2280 SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
2281
2282 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32)
2283 op ^= SLJIT_32;
2284
2285 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2286
2287 if (src & SLJIT_MEM) {
2288 FAIL_IF(emit_op_mem2(compiler, FLOAT_DATA(op) | LOAD_DATA, dst_r, src, srcw, dst, dstw));
2289 src = dst_r;
2290 }
2291
2292 switch (GET_OPCODE(op)) {
2293 case SLJIT_MOV_F64:
2294 if (src != dst_r) {
2295 if (dst_r != TMP_FREG1)
2296 FAIL_IF(push_inst(compiler, FSGNJ_S | FMT(op) | FRD(dst_r) | FRS1(src) | FRS2(src)));
2297 else
2298 dst_r = src;
2299 }
2300 break;
2301 case SLJIT_NEG_F64:
2302 FAIL_IF(push_inst(compiler, FSGNJN_S | FMT(op) | FRD(dst_r) | FRS1(src) | FRS2(src)));
2303 break;
2304 case SLJIT_ABS_F64:
2305 FAIL_IF(push_inst(compiler, FSGNJX_S | FMT(op) | FRD(dst_r) | FRS1(src) | FRS2(src)));
2306 break;
2307 case SLJIT_CONV_F64_FROM_F32:
2308 /* The SLJIT_32 bit is inverted because sljit_f32 needs to be loaded from the memory. */
2309 FAIL_IF(push_inst(compiler, FCVT_S_D | ((op & SLJIT_32) ? (1 << 25) : ((1 << 20) | F3(7))) | FRD(dst_r) | FRS1(src)));
2310 op ^= SLJIT_32;
2311 break;
2312 }
2313
2314 if (dst & SLJIT_MEM)
2315 return emit_op_mem2(compiler, FLOAT_DATA(op), dst_r, dst, dstw, 0, 0);
2316 return SLJIT_SUCCESS;
2317 }
2318
sljit_emit_fop2(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)2319 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
2320 sljit_s32 dst, sljit_sw dstw,
2321 sljit_s32 src1, sljit_sw src1w,
2322 sljit_s32 src2, sljit_sw src2w)
2323 {
2324 sljit_s32 dst_r, flags = 0;
2325
2326 CHECK_ERROR();
2327 CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
2328 ADJUST_LOCAL_OFFSET(dst, dstw);
2329 ADJUST_LOCAL_OFFSET(src1, src1w);
2330 ADJUST_LOCAL_OFFSET(src2, src2w);
2331
2332 compiler->cache_arg = 0;
2333 compiler->cache_argw = 0;
2334
2335 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG2;
2336
2337 if (src1 & SLJIT_MEM) {
2338 if (getput_arg_fast(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w)) {
2339 FAIL_IF(compiler->error);
2340 src1 = TMP_FREG1;
2341 } else
2342 flags |= SLOW_SRC1;
2343 }
2344
2345 if (src2 & SLJIT_MEM) {
2346 if (getput_arg_fast(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w)) {
2347 FAIL_IF(compiler->error);
2348 src2 = TMP_FREG2;
2349 } else
2350 flags |= SLOW_SRC2;
2351 }
2352
2353 if ((flags & (SLOW_SRC1 | SLOW_SRC2)) == (SLOW_SRC1 | SLOW_SRC2)) {
2354 if (!can_cache(src1, src1w, src2, src2w) && can_cache(src1, src1w, dst, dstw)) {
2355 FAIL_IF(getput_arg(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w, src1, src1w));
2356 FAIL_IF(getput_arg(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w, dst, dstw));
2357 }
2358 else {
2359 FAIL_IF(getput_arg(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w, src2, src2w));
2360 FAIL_IF(getput_arg(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w, dst, dstw));
2361 }
2362 }
2363 else if (flags & SLOW_SRC1)
2364 FAIL_IF(getput_arg(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG1, src1, src1w, dst, dstw));
2365 else if (flags & SLOW_SRC2)
2366 FAIL_IF(getput_arg(compiler, FLOAT_DATA(op) | LOAD_DATA, TMP_FREG2, src2, src2w, dst, dstw));
2367
2368 if (flags & SLOW_SRC1)
2369 src1 = TMP_FREG1;
2370 if (flags & SLOW_SRC2)
2371 src2 = TMP_FREG2;
2372
2373 switch (GET_OPCODE(op)) {
2374 case SLJIT_ADD_F64:
2375 FAIL_IF(push_inst(compiler, FADD_S | FMT(op) | FRD(dst_r) | FRS1(src1) | FRS2(src2)));
2376 break;
2377
2378 case SLJIT_SUB_F64:
2379 FAIL_IF(push_inst(compiler, FSUB_S | FMT(op) | FRD(dst_r) | FRS1(src1) | FRS2(src2)));
2380 break;
2381
2382 case SLJIT_MUL_F64:
2383 FAIL_IF(push_inst(compiler, FMUL_S | FMT(op) | FRD(dst_r) | FRS1(src1) | FRS2(src2)));
2384 break;
2385
2386 case SLJIT_DIV_F64:
2387 FAIL_IF(push_inst(compiler, FDIV_S | FMT(op) | FRD(dst_r) | FRS1(src1) | FRS2(src2)));
2388 break;
2389
2390 case SLJIT_COPYSIGN_F64:
2391 return push_inst(compiler, FSGNJ_S | FMT(op) | FRD(dst_r) | FRS1(src1) | FRS2(src2));
2392 }
2393
2394 if (dst_r == TMP_FREG2)
2395 FAIL_IF(emit_op_mem2(compiler, FLOAT_DATA(op), TMP_FREG2, dst, dstw, 0, 0));
2396
2397 return SLJIT_SUCCESS;
2398 }
2399
sljit_emit_fset32(struct sljit_compiler * compiler,sljit_s32 freg,sljit_f32 value)2400 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset32(struct sljit_compiler *compiler,
2401 sljit_s32 freg, sljit_f32 value)
2402 {
2403 union {
2404 sljit_s32 imm;
2405 sljit_f32 value;
2406 } u;
2407
2408 CHECK_ERROR();
2409 CHECK(check_sljit_emit_fset32(compiler, freg, value));
2410
2411 u.value = value;
2412
2413 if (u.imm == 0)
2414 return push_inst(compiler, FMV_W_X | RS1(TMP_ZERO) | FRD(freg));
2415
2416 FAIL_IF(load_immediate(compiler, TMP_REG1, u.imm, TMP_REG3));
2417 return push_inst(compiler, FMV_W_X | RS1(TMP_REG1) | FRD(freg));
2418 }
2419
2420 /* --------------------------------------------------------------------- */
2421 /* Conditional instructions */
2422 /* --------------------------------------------------------------------- */
2423
sljit_emit_label(struct sljit_compiler * compiler)2424 SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
2425 {
2426 struct sljit_label *label;
2427
2428 CHECK_ERROR_PTR();
2429 CHECK_PTR(check_sljit_emit_label(compiler));
2430
2431 if (compiler->last_label && compiler->last_label->size == compiler->size)
2432 return compiler->last_label;
2433
2434 label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
2435 PTR_FAIL_IF(!label);
2436 set_label(label, compiler);
2437 return label;
2438 }
2439
2440 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2441 #define BRANCH_LENGTH ((sljit_ins)(3 * sizeof(sljit_ins)) << 7)
2442 #else
2443 #define BRANCH_LENGTH ((sljit_ins)(7 * sizeof(sljit_ins)) << 7)
2444 #endif
2445
get_jump_instruction(sljit_s32 type)2446 static sljit_ins get_jump_instruction(sljit_s32 type)
2447 {
2448 switch (type) {
2449 case SLJIT_EQUAL:
2450 return BNE | RS1(EQUAL_FLAG) | RS2(TMP_ZERO);
2451 case SLJIT_NOT_EQUAL:
2452 return BEQ | RS1(EQUAL_FLAG) | RS2(TMP_ZERO);
2453 case SLJIT_LESS:
2454 case SLJIT_GREATER:
2455 case SLJIT_SIG_LESS:
2456 case SLJIT_SIG_GREATER:
2457 case SLJIT_OVERFLOW:
2458 case SLJIT_CARRY:
2459 case SLJIT_F_EQUAL:
2460 case SLJIT_ORDERED_EQUAL:
2461 case SLJIT_ORDERED_NOT_EQUAL:
2462 case SLJIT_F_LESS:
2463 case SLJIT_ORDERED_LESS:
2464 case SLJIT_ORDERED_GREATER:
2465 case SLJIT_F_LESS_EQUAL:
2466 case SLJIT_ORDERED_LESS_EQUAL:
2467 case SLJIT_ORDERED_GREATER_EQUAL:
2468 case SLJIT_ORDERED:
2469 return BEQ | RS1(OTHER_FLAG) | RS2(TMP_ZERO);
2470 break;
2471 case SLJIT_GREATER_EQUAL:
2472 case SLJIT_LESS_EQUAL:
2473 case SLJIT_SIG_GREATER_EQUAL:
2474 case SLJIT_SIG_LESS_EQUAL:
2475 case SLJIT_NOT_OVERFLOW:
2476 case SLJIT_NOT_CARRY:
2477 case SLJIT_F_NOT_EQUAL:
2478 case SLJIT_UNORDERED_OR_NOT_EQUAL:
2479 case SLJIT_UNORDERED_OR_EQUAL:
2480 case SLJIT_F_GREATER_EQUAL:
2481 case SLJIT_UNORDERED_OR_GREATER_EQUAL:
2482 case SLJIT_UNORDERED_OR_LESS_EQUAL:
2483 case SLJIT_F_GREATER:
2484 case SLJIT_UNORDERED_OR_GREATER:
2485 case SLJIT_UNORDERED_OR_LESS:
2486 case SLJIT_UNORDERED:
2487 return BNE | RS1(OTHER_FLAG) | RS2(TMP_ZERO);
2488 default:
2489 /* Not conditional branch. */
2490 return 0;
2491 }
2492 }
2493
sljit_emit_jump(struct sljit_compiler * compiler,sljit_s32 type)2494 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2495 {
2496 struct sljit_jump *jump;
2497 sljit_ins inst;
2498
2499 CHECK_ERROR_PTR();
2500 CHECK_PTR(check_sljit_emit_jump(compiler, type));
2501
2502 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2503 PTR_FAIL_IF(!jump);
2504 set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP);
2505 type &= 0xff;
2506
2507 inst = get_jump_instruction(type);
2508
2509 if (inst != 0) {
2510 PTR_FAIL_IF(push_inst(compiler, inst | BRANCH_LENGTH));
2511 jump->flags |= IS_COND;
2512 }
2513
2514 jump->addr = compiler->size;
2515 inst = JALR | RS1(TMP_REG1) | IMM_I(0);
2516
2517 if (type >= SLJIT_FAST_CALL) {
2518 jump->flags |= IS_CALL;
2519 inst |= RD(RETURN_ADDR_REG);
2520 }
2521
2522 PTR_FAIL_IF(push_inst(compiler, inst));
2523
2524 /* Maximum number of instructions required for generating a constant. */
2525 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2526 compiler->size += 1;
2527 #else
2528 compiler->size += 5;
2529 #endif
2530 return jump;
2531 }
2532
sljit_emit_call(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 arg_types)2533 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
2534 sljit_s32 arg_types)
2535 {
2536 SLJIT_UNUSED_ARG(arg_types);
2537 CHECK_ERROR_PTR();
2538 CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
2539
2540 if (type & SLJIT_CALL_RETURN) {
2541 PTR_FAIL_IF(emit_stack_frame_release(compiler, 0));
2542 type = SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP);
2543 }
2544
2545 SLJIT_SKIP_CHECKS(compiler);
2546 return sljit_emit_jump(compiler, type);
2547 }
2548
sljit_emit_cmp(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)2549 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_cmp(struct sljit_compiler *compiler, sljit_s32 type,
2550 sljit_s32 src1, sljit_sw src1w,
2551 sljit_s32 src2, sljit_sw src2w)
2552 {
2553 struct sljit_jump *jump;
2554 sljit_s32 flags;
2555 sljit_ins inst;
2556
2557 CHECK_ERROR_PTR();
2558 CHECK_PTR(check_sljit_emit_cmp(compiler, type, src1, src1w, src2, src2w));
2559 ADJUST_LOCAL_OFFSET(src1, src1w);
2560 ADJUST_LOCAL_OFFSET(src2, src2w);
2561
2562 compiler->cache_arg = 0;
2563 compiler->cache_argw = 0;
2564 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2565 flags = WORD_DATA | LOAD_DATA;
2566 #else /* !SLJIT_CONFIG_RISCV_32 */
2567 flags = ((type & SLJIT_32) ? INT_DATA : WORD_DATA) | LOAD_DATA;
2568 #endif /* SLJIT_CONFIG_RISCV_32 */
2569
2570 if (src1 & SLJIT_MEM) {
2571 PTR_FAIL_IF(emit_op_mem2(compiler, flags, TMP_REG1, src1, src1w, src2, src2w));
2572 src1 = TMP_REG1;
2573 }
2574
2575 if (src2 & SLJIT_MEM) {
2576 PTR_FAIL_IF(emit_op_mem2(compiler, flags, TMP_REG2, src2, src2w, 0, 0));
2577 src2 = TMP_REG2;
2578 }
2579
2580 if (src1 == SLJIT_IMM) {
2581 if (src1w != 0) {
2582 PTR_FAIL_IF(load_immediate(compiler, TMP_REG1, src1w, TMP_REG3));
2583 src1 = TMP_REG1;
2584 }
2585 else
2586 src1 = TMP_ZERO;
2587 }
2588
2589 if (src2 == SLJIT_IMM) {
2590 if (src2w != 0) {
2591 PTR_FAIL_IF(load_immediate(compiler, TMP_REG2, src2w, TMP_REG3));
2592 src2 = TMP_REG2;
2593 }
2594 else
2595 src2 = TMP_ZERO;
2596 }
2597
2598 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2599 PTR_FAIL_IF(!jump);
2600 set_jump(jump, compiler, (sljit_u32)((type & SLJIT_REWRITABLE_JUMP) | IS_COND));
2601 type &= 0xff;
2602
2603 switch (type) {
2604 case SLJIT_EQUAL:
2605 inst = BNE | RS1(src1) | RS2(src2) | BRANCH_LENGTH;
2606 break;
2607 case SLJIT_NOT_EQUAL:
2608 inst = BEQ | RS1(src1) | RS2(src2) | BRANCH_LENGTH;
2609 break;
2610 case SLJIT_LESS:
2611 inst = BGEU | RS1(src1) | RS2(src2) | BRANCH_LENGTH;
2612 break;
2613 case SLJIT_GREATER_EQUAL:
2614 inst = BLTU | RS1(src1) | RS2(src2) | BRANCH_LENGTH;
2615 break;
2616 case SLJIT_GREATER:
2617 inst = BGEU | RS1(src2) | RS2(src1) | BRANCH_LENGTH;
2618 break;
2619 case SLJIT_LESS_EQUAL:
2620 inst = BLTU | RS1(src2) | RS2(src1) | BRANCH_LENGTH;
2621 break;
2622 case SLJIT_SIG_LESS:
2623 inst = BGE | RS1(src1) | RS2(src2) | BRANCH_LENGTH;
2624 break;
2625 case SLJIT_SIG_GREATER_EQUAL:
2626 inst = BLT | RS1(src1) | RS2(src2) | BRANCH_LENGTH;
2627 break;
2628 case SLJIT_SIG_GREATER:
2629 inst = BGE | RS1(src2) | RS2(src1) | BRANCH_LENGTH;
2630 break;
2631 case SLJIT_SIG_LESS_EQUAL:
2632 inst = BLT | RS1(src2) | RS2(src1) | BRANCH_LENGTH;
2633 break;
2634 }
2635
2636 PTR_FAIL_IF(push_inst(compiler, inst));
2637
2638 jump->addr = compiler->size;
2639 PTR_FAIL_IF(push_inst(compiler, JALR | RD(TMP_ZERO) | RS1(TMP_REG1) | IMM_I(0)));
2640
2641 /* Maximum number of instructions required for generating a constant. */
2642 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2643 compiler->size += 1;
2644 #else
2645 compiler->size += 5;
2646 #endif
2647 return jump;
2648 }
2649
2650 #undef BRANCH_LENGTH
2651
sljit_emit_ijump(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 src,sljit_sw srcw)2652 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
2653 {
2654 struct sljit_jump *jump;
2655
2656 CHECK_ERROR();
2657 CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
2658
2659 if (src != SLJIT_IMM) {
2660 if (src & SLJIT_MEM) {
2661 ADJUST_LOCAL_OFFSET(src, srcw);
2662 FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_REG1, src, srcw));
2663 src = TMP_REG1;
2664 }
2665 return push_inst(compiler, JALR | RD((type >= SLJIT_FAST_CALL) ? RETURN_ADDR_REG : TMP_ZERO) | RS1(src) | IMM_I(0));
2666 }
2667
2668 /* These jumps are converted to jump/call instructions when possible. */
2669 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2670 FAIL_IF(!jump);
2671 set_jump(jump, compiler, JUMP_ADDR | ((type >= SLJIT_FAST_CALL) ? IS_CALL : 0));
2672 jump->u.target = (sljit_uw)srcw;
2673
2674 jump->addr = compiler->size;
2675 FAIL_IF(push_inst(compiler, JALR | RD((type >= SLJIT_FAST_CALL) ? RETURN_ADDR_REG : TMP_ZERO) | RS1(TMP_REG1) | IMM_I(0)));
2676
2677 /* Maximum number of instructions required for generating a constant. */
2678 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2679 compiler->size += 1;
2680 #else
2681 compiler->size += 5;
2682 #endif
2683 return SLJIT_SUCCESS;
2684 }
2685
sljit_emit_icall(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 arg_types,sljit_s32 src,sljit_sw srcw)2686 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
2687 sljit_s32 arg_types,
2688 sljit_s32 src, sljit_sw srcw)
2689 {
2690 SLJIT_UNUSED_ARG(arg_types);
2691 CHECK_ERROR();
2692 CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
2693
2694 if (src & SLJIT_MEM) {
2695 ADJUST_LOCAL_OFFSET(src, srcw);
2696 FAIL_IF(emit_op_mem(compiler, WORD_DATA | LOAD_DATA, TMP_REG1, src, srcw));
2697 src = TMP_REG1;
2698 }
2699
2700 if (type & SLJIT_CALL_RETURN) {
2701 if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
2702 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG1) | RS1(src) | IMM_I(0)));
2703 src = TMP_REG1;
2704 }
2705
2706 FAIL_IF(emit_stack_frame_release(compiler, 0));
2707 type = SLJIT_JUMP;
2708 }
2709
2710 SLJIT_SKIP_CHECKS(compiler);
2711 return sljit_emit_ijump(compiler, type, src, srcw);
2712 }
2713
sljit_emit_op_flags(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 type)2714 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
2715 sljit_s32 dst, sljit_sw dstw,
2716 sljit_s32 type)
2717 {
2718 sljit_s32 src_r, dst_r, invert;
2719 sljit_s32 saved_op = op;
2720 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2721 sljit_s32 mem_type = WORD_DATA;
2722 #else
2723 sljit_s32 mem_type = ((op & SLJIT_32) || op == SLJIT_MOV32) ? (INT_DATA | SIGNED_DATA) : WORD_DATA;
2724 #endif
2725
2726 CHECK_ERROR();
2727 CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
2728 ADJUST_LOCAL_OFFSET(dst, dstw);
2729
2730 op = GET_OPCODE(op);
2731 dst_r = (op < SLJIT_ADD && FAST_IS_REG(dst)) ? dst : TMP_REG2;
2732
2733 compiler->cache_arg = 0;
2734 compiler->cache_argw = 0;
2735
2736 if (op >= SLJIT_ADD && (dst & SLJIT_MEM))
2737 FAIL_IF(emit_op_mem2(compiler, mem_type | LOAD_DATA, TMP_REG1, dst, dstw, dst, dstw));
2738
2739 if (type < SLJIT_F_EQUAL) {
2740 src_r = OTHER_FLAG;
2741 invert = type & 0x1;
2742
2743 switch (type) {
2744 case SLJIT_EQUAL:
2745 case SLJIT_NOT_EQUAL:
2746 FAIL_IF(push_inst(compiler, SLTUI | RD(dst_r) | RS1(EQUAL_FLAG) | IMM_I(1)));
2747 src_r = dst_r;
2748 break;
2749 case SLJIT_OVERFLOW:
2750 case SLJIT_NOT_OVERFLOW:
2751 if (compiler->status_flags_state & (SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB)) {
2752 src_r = OTHER_FLAG;
2753 break;
2754 }
2755 FAIL_IF(push_inst(compiler, SLTUI | RD(dst_r) | RS1(OTHER_FLAG) | IMM_I(1)));
2756 src_r = dst_r;
2757 invert ^= 0x1;
2758 break;
2759 }
2760 } else {
2761 invert = 0;
2762 src_r = OTHER_FLAG;
2763
2764 switch (type) {
2765 case SLJIT_F_NOT_EQUAL:
2766 case SLJIT_UNORDERED_OR_NOT_EQUAL:
2767 case SLJIT_UNORDERED_OR_EQUAL: /* Not supported. */
2768 case SLJIT_F_GREATER_EQUAL:
2769 case SLJIT_UNORDERED_OR_GREATER_EQUAL:
2770 case SLJIT_UNORDERED_OR_LESS_EQUAL:
2771 case SLJIT_F_GREATER:
2772 case SLJIT_UNORDERED_OR_GREATER:
2773 case SLJIT_UNORDERED_OR_LESS:
2774 case SLJIT_UNORDERED:
2775 invert = 1;
2776 break;
2777 }
2778 }
2779
2780 if (invert) {
2781 FAIL_IF(push_inst(compiler, XORI | RD(dst_r) | RS1(src_r) | IMM_I(1)));
2782 src_r = dst_r;
2783 }
2784
2785 if (op < SLJIT_ADD) {
2786 if (dst & SLJIT_MEM)
2787 return emit_op_mem(compiler, mem_type, src_r, dst, dstw);
2788
2789 if (src_r != dst_r)
2790 return push_inst(compiler, ADDI | RD(dst_r) | RS1(src_r) | IMM_I(0));
2791 return SLJIT_SUCCESS;
2792 }
2793
2794 mem_type |= CUMULATIVE_OP | IMM_OP | ALT_KEEP_CACHE;
2795
2796 if (dst & SLJIT_MEM)
2797 return emit_op(compiler, saved_op, mem_type, dst, dstw, TMP_REG1, 0, src_r, 0);
2798 return emit_op(compiler, saved_op, mem_type, dst, dstw, dst, dstw, src_r, 0);
2799 }
2800
sljit_emit_select(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 dst_reg,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2_reg)2801 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_select(struct sljit_compiler *compiler, sljit_s32 type,
2802 sljit_s32 dst_reg,
2803 sljit_s32 src1, sljit_sw src1w,
2804 sljit_s32 src2_reg)
2805 {
2806 sljit_ins *ptr;
2807 sljit_uw size;
2808 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
2809 sljit_ins word = (sljit_ins)(type & SLJIT_32) >> 5;
2810 sljit_s32 inp_flags = ((type & SLJIT_32) ? INT_DATA : WORD_DATA) | LOAD_DATA;
2811 #else /* !SLJIT_CONFIG_RISCV_64 */
2812 sljit_s32 inp_flags = WORD_DATA | LOAD_DATA;
2813 #endif /* SLJIT_CONFIG_RISCV_64 */
2814
2815 SLJIT_ASSERT(WORD == 0 || WORD == 0x8);
2816
2817 CHECK_ERROR();
2818 CHECK(check_sljit_emit_select(compiler, type, dst_reg, src1, src1w, src2_reg));
2819
2820 ADJUST_LOCAL_OFFSET(src1, src1w);
2821
2822 if (dst_reg != src2_reg) {
2823 if (dst_reg == src1) {
2824 src1 = src2_reg;
2825 src1w = 0;
2826 type ^= 0x1;
2827 } else {
2828 if (ADDRESSING_DEPENDS_ON(src1, dst_reg)) {
2829 FAIL_IF(push_inst(compiler, ADDI | RD(TMP_REG2) | RS1(dst_reg) | IMM_I(0)));
2830
2831 if ((src1 & REG_MASK) == dst_reg)
2832 src1 = (src1 & ~REG_MASK) | TMP_REG2;
2833
2834 if (OFFS_REG(src1) == dst_reg)
2835 src1 = (src1 & ~OFFS_REG_MASK) | TO_OFFS_REG(TMP_REG2);
2836 }
2837
2838 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(dst_reg) | RS1(src2_reg) | IMM_I(0)));
2839 }
2840 }
2841
2842 size = compiler->size;
2843
2844 ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins));
2845 FAIL_IF(!ptr);
2846 compiler->size++;
2847
2848 if (src1 & SLJIT_MEM) {
2849 FAIL_IF(emit_op_mem(compiler, inp_flags, dst_reg, src1, src1w));
2850 } else if (src1 == SLJIT_IMM) {
2851 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
2852 if (word)
2853 src1w = (sljit_s32)src1w;
2854 #endif /* SLJIT_CONFIG_RISCV_64 */
2855 FAIL_IF(load_immediate(compiler, dst_reg, src1w, TMP_REG1));
2856 } else
2857 FAIL_IF(push_inst(compiler, ADDI | WORD | RD(dst_reg) | RS1(src1) | IMM_I(0)));
2858
2859 *ptr = get_jump_instruction(type & ~SLJIT_32) | (sljit_ins)((compiler->size - size) << 9);
2860 return SLJIT_SUCCESS;
2861 }
2862
2863 #undef WORD
2864
sljit_emit_fselect(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 dst_freg,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2_freg)2865 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fselect(struct sljit_compiler *compiler, sljit_s32 type,
2866 sljit_s32 dst_freg,
2867 sljit_s32 src1, sljit_sw src1w,
2868 sljit_s32 src2_freg)
2869 {
2870 sljit_ins *ptr;
2871 sljit_uw size;
2872
2873 CHECK_ERROR();
2874 CHECK(check_sljit_emit_fselect(compiler, type, dst_freg, src1, src1w, src2_freg));
2875
2876 ADJUST_LOCAL_OFFSET(src1, src1w);
2877
2878 if (dst_freg != src2_freg) {
2879 if (dst_freg == src1) {
2880 src1 = src2_freg;
2881 src1w = 0;
2882 type ^= 0x1;
2883 } else
2884 FAIL_IF(push_inst(compiler, FSGNJ_S | FMT(type) | FRD(dst_freg) | FRS1(src2_freg) | FRS2(src2_freg)));
2885 }
2886
2887 size = compiler->size;
2888
2889 ptr = (sljit_ins*)ensure_buf(compiler, sizeof(sljit_ins));
2890 FAIL_IF(!ptr);
2891 compiler->size++;
2892
2893 if (src1 & SLJIT_MEM)
2894 FAIL_IF(emit_op_mem(compiler, FLOAT_DATA(type) | LOAD_DATA, dst_freg, src1, src1w));
2895 else
2896 FAIL_IF(push_inst(compiler, FSGNJ_S | FMT(type) | FRD(dst_freg) | FRS1(src1) | FRS2(src1)));
2897
2898 *ptr = get_jump_instruction(type & ~SLJIT_32) | (sljit_ins)((compiler->size - size) << 9);
2899 return SLJIT_SUCCESS;
2900 }
2901
2902 #undef FLOAT_DATA
2903 #undef FMT
2904
sljit_emit_mem(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 reg,sljit_s32 mem,sljit_sw memw)2905 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
2906 sljit_s32 reg,
2907 sljit_s32 mem, sljit_sw memw)
2908 {
2909 sljit_s32 flags;
2910
2911 CHECK_ERROR();
2912 CHECK(check_sljit_emit_mem(compiler, type, reg, mem, memw));
2913
2914 if (!(reg & REG_PAIR_MASK))
2915 return sljit_emit_mem_unaligned(compiler, type, reg, mem, memw);
2916
2917 if (SLJIT_UNLIKELY(mem & OFFS_REG_MASK)) {
2918 memw &= 0x3;
2919
2920 if (SLJIT_UNLIKELY(memw != 0)) {
2921 FAIL_IF(push_inst(compiler, SLLI | RD(TMP_REG1) | RS1(OFFS_REG(mem)) | IMM_I(memw)));
2922 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG1) | RS1(TMP_REG1) | RS2(mem & REG_MASK)));
2923 } else
2924 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG1) | RS1(mem & REG_MASK) | RS2(OFFS_REG(mem))));
2925
2926 mem = TMP_REG1;
2927 memw = 0;
2928 } else if (memw > SIMM_MAX - SSIZE_OF(sw) || memw < SIMM_MIN) {
2929 if (((memw + 0x800) & 0xfff) <= 0xfff - SSIZE_OF(sw)) {
2930 FAIL_IF(load_immediate(compiler, TMP_REG1, TO_ARGW_HI(memw), TMP_REG3));
2931 memw &= 0xfff;
2932 } else {
2933 FAIL_IF(load_immediate(compiler, TMP_REG1, memw, TMP_REG3));
2934 memw = 0;
2935 }
2936
2937 if (mem & REG_MASK)
2938 FAIL_IF(push_inst(compiler, ADD | RD(TMP_REG1) | RS1(TMP_REG1) | RS2(mem & REG_MASK)));
2939
2940 mem = TMP_REG1;
2941 } else {
2942 mem &= REG_MASK;
2943 memw &= 0xfff;
2944 }
2945
2946 SLJIT_ASSERT((memw >= 0 && memw <= SIMM_MAX - SSIZE_OF(sw)) || (memw > SIMM_MAX && memw <= 0xfff));
2947
2948 if (!(type & SLJIT_MEM_STORE) && mem == REG_PAIR_FIRST(reg)) {
2949 FAIL_IF(push_mem_inst(compiler, WORD_DATA | LOAD_DATA, REG_PAIR_SECOND(reg), mem, (memw + SSIZE_OF(sw)) & 0xfff));
2950 return push_mem_inst(compiler, WORD_DATA | LOAD_DATA, REG_PAIR_FIRST(reg), mem, memw);
2951 }
2952
2953 flags = WORD_DATA | (!(type & SLJIT_MEM_STORE) ? LOAD_DATA : 0);
2954
2955 FAIL_IF(push_mem_inst(compiler, flags, REG_PAIR_FIRST(reg), mem, memw));
2956 return push_mem_inst(compiler, flags, REG_PAIR_SECOND(reg), mem, (memw + SSIZE_OF(sw)) & 0xfff);
2957 }
2958
2959 #undef TO_ARGW_HI
2960
sljit_emit_const(struct sljit_compiler * compiler,sljit_s32 dst,sljit_sw dstw,sljit_sw init_value)2961 SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value)
2962 {
2963 struct sljit_const *const_;
2964 sljit_s32 dst_r;
2965
2966 CHECK_ERROR_PTR();
2967 CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value));
2968 ADJUST_LOCAL_OFFSET(dst, dstw);
2969
2970 const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
2971 PTR_FAIL_IF(!const_);
2972 set_const(const_, compiler);
2973
2974 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
2975 PTR_FAIL_IF(emit_const(compiler, dst_r, init_value, ADDI | RD(dst_r)));
2976
2977 if (dst & SLJIT_MEM)
2978 PTR_FAIL_IF(emit_op_mem(compiler, WORD_DATA, TMP_REG2, dst, dstw));
2979
2980 return const_;
2981 }
2982
sljit_emit_put_label(struct sljit_compiler * compiler,sljit_s32 dst,sljit_sw dstw)2983 SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
2984 {
2985 struct sljit_put_label *put_label;
2986 sljit_s32 dst_r;
2987
2988 CHECK_ERROR_PTR();
2989 CHECK_PTR(check_sljit_emit_put_label(compiler, dst, dstw));
2990 ADJUST_LOCAL_OFFSET(dst, dstw);
2991
2992 put_label = (struct sljit_put_label*)ensure_abuf(compiler, sizeof(struct sljit_put_label));
2993 PTR_FAIL_IF(!put_label);
2994 set_put_label(put_label, compiler, 0);
2995
2996 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG2;
2997 PTR_FAIL_IF(push_inst(compiler, (sljit_ins)dst_r));
2998 #if (defined SLJIT_CONFIG_RISCV_32 && SLJIT_CONFIG_RISCV_32)
2999 compiler->size += 1;
3000 #else
3001 compiler->size += 5;
3002 #endif
3003
3004 if (dst & SLJIT_MEM)
3005 PTR_FAIL_IF(emit_op_mem(compiler, WORD_DATA, TMP_REG2, dst, dstw));
3006
3007 return put_label;
3008 }
3009
sljit_set_const(sljit_uw addr,sljit_sw new_constant,sljit_sw executable_offset)3010 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset)
3011 {
3012 sljit_set_jump_addr(addr, (sljit_uw)new_constant, executable_offset);
3013 }
3014