1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
sljit_get_platform_name(void)27 SLJIT_API_FUNC_ATTRIBUTE const char* sljit_get_platform_name(void)
28 {
29 #ifdef __SOFTFP__
30 return "ARM-Thumb2" SLJIT_CPUINFO " ABI:softfp";
31 #else
32 return "ARM-Thumb2" SLJIT_CPUINFO " ABI:hardfp";
33 #endif
34 }
35
36 /* Length of an instruction word. */
37 typedef sljit_u32 sljit_ins;
38
39 /* Last register + 1. */
40 #define TMP_REG1 (SLJIT_NUMBER_OF_REGISTERS + 2)
41 #define TMP_REG2 (SLJIT_NUMBER_OF_REGISTERS + 3)
42 #define TMP_PC (SLJIT_NUMBER_OF_REGISTERS + 4)
43
44 #define TMP_FREG1 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 1)
45 #define TMP_FREG2 (SLJIT_NUMBER_OF_FLOAT_REGISTERS + 2)
46
47 /* See sljit_emit_enter and sljit_emit_op0 if you want to change them. */
48 static const sljit_u8 reg_map[SLJIT_NUMBER_OF_REGISTERS + 5] = {
49 0, 0, 1, 2, 3, 11, 10, 9, 8, 7, 6, 5, 4, 13, 12, 14, 15
50 };
51
52 static const sljit_u8 freg_map[SLJIT_NUMBER_OF_FLOAT_REGISTERS + 3] = {
53 0, 0, 1, 2, 3, 4, 5, 15, 14, 13, 12, 11, 10, 9, 8, 6, 7
54 };
55
56 #define COPY_BITS(src, from, to, bits) \
57 ((from >= to ? ((sljit_ins)(src) >> (from - to)) : ((sljit_ins)(src) << (to - from))) & (((1 << bits) - 1) << to))
58
59 #define NEGATE(uimm) ((sljit_uw)-(sljit_sw)(uimm))
60
61 /* Thumb16 encodings. */
62 #define RD3(rd) ((sljit_ins)reg_map[rd])
63 #define RN3(rn) ((sljit_ins)reg_map[rn] << 3)
64 #define RM3(rm) ((sljit_ins)reg_map[rm] << 6)
65 #define RDN3(rdn) ((sljit_ins)reg_map[rdn] << 8)
66 #define IMM3(imm) ((sljit_ins)imm << 6)
67 #define IMM8(imm) ((sljit_ins)imm)
68
69 /* Thumb16 helpers. */
70 #define SET_REGS44(rd, rn) \
71 (((sljit_ins)reg_map[rn] << 3) | ((sljit_ins)reg_map[rd] & 0x7) | (((sljit_ins)reg_map[rd] & 0x8) << 4))
72 #define IS_2_LO_REGS(reg1, reg2) \
73 (reg_map[reg1] <= 7 && reg_map[reg2] <= 7)
74 #define IS_3_LO_REGS(reg1, reg2, reg3) \
75 (reg_map[reg1] <= 7 && reg_map[reg2] <= 7 && reg_map[reg3] <= 7)
76
77 /* Thumb32 encodings. */
78 #define RD4(rd) ((sljit_ins)reg_map[rd] << 8)
79 #define RN4(rn) ((sljit_ins)reg_map[rn] << 16)
80 #define RM4(rm) ((sljit_ins)reg_map[rm])
81 #define RT4(rt) ((sljit_ins)reg_map[rt] << 12)
82 #define DD4(dd) ((sljit_ins)freg_map[dd] << 12)
83 #define DN4(dn) ((sljit_ins)freg_map[dn] << 16)
84 #define DM4(dm) ((sljit_ins)freg_map[dm])
85 #define IMM5(imm) \
86 (COPY_BITS(imm, 2, 12, 3) | (((sljit_ins)imm & 0x3) << 6))
87 #define IMM12(imm) \
88 (COPY_BITS(imm, 11, 26, 1) | COPY_BITS(imm, 8, 12, 3) | ((sljit_ins)imm & 0xff))
89
90 /* --------------------------------------------------------------------- */
91 /* Instrucion forms */
92 /* --------------------------------------------------------------------- */
93
94 /* dot '.' changed to _
95 I immediate form (possibly followed by number of immediate bits). */
96 #define ADCI 0xf1400000
97 #define ADCS 0x4140
98 #define ADC_W 0xeb400000
99 #define ADD 0x4400
100 #define ADDS 0x1800
101 #define ADDSI3 0x1c00
102 #define ADDSI8 0x3000
103 #define ADDWI 0xf2000000
104 #define ADD_SP 0x4485
105 #define ADD_SP_I 0xb000
106 #define ADD_W 0xeb000000
107 #define ADD_WI 0xf1000000
108 #define ANDI 0xf0000000
109 #define ANDS 0x4000
110 #define AND_W 0xea000000
111 #define ASRS 0x4100
112 #define ASRSI 0x1000
113 #define ASR_W 0xfa40f000
114 #define ASR_WI 0xea4f0020
115 #define BCC 0xd000
116 #define BICI 0xf0200000
117 #define BKPT 0xbe00
118 #define BLX 0x4780
119 #define BX 0x4700
120 #define CLZ 0xfab0f080
121 #define CMNI_W 0xf1100f00
122 #define CMP 0x4280
123 #define CMPI 0x2800
124 #define CMPI_W 0xf1b00f00
125 #define CMP_X 0x4500
126 #define CMP_W 0xebb00f00
127 #define EORI 0xf0800000
128 #define EORS 0x4040
129 #define EOR_W 0xea800000
130 #define IT 0xbf00
131 #define LDR_SP 0x9800
132 #define LDR 0xf8d00000
133 #define LDRD 0xe9500000
134 #define LDRI 0xf8500800
135 #define LSLS 0x4080
136 #define LSLSI 0x0000
137 #define LSL_W 0xfa00f000
138 #define LSL_WI 0xea4f0000
139 #define LSRS 0x40c0
140 #define LSRSI 0x0800
141 #define LSR_W 0xfa20f000
142 #define LSR_WI 0xea4f0010
143 #define MOV 0x4600
144 #define MOVS 0x0000
145 #define MOVSI 0x2000
146 #define MOVT 0xf2c00000
147 #define MOVW 0xf2400000
148 #define MOV_W 0xea4f0000
149 #define MOV_WI 0xf04f0000
150 #define MUL 0xfb00f000
151 #define MVNS 0x43c0
152 #define MVN_W 0xea6f0000
153 #define MVN_WI 0xf06f0000
154 #define NOP 0xbf00
155 #define ORNI 0xf0600000
156 #define ORRI 0xf0400000
157 #define ORRS 0x4300
158 #define ORR_W 0xea400000
159 #define POP 0xbc00
160 #define POP_W 0xe8bd0000
161 #define PUSH 0xb400
162 #define PUSH_W 0xe92d0000
163 #define RBIT 0xfa90f0a0
164 #define RORS 0x41c0
165 #define ROR_W 0xfa60f000
166 #define ROR_WI 0xea4f0030
167 #define RSB_WI 0xf1c00000
168 #define RSBSI 0x4240
169 #define SBCI 0xf1600000
170 #define SBCS 0x4180
171 #define SBC_W 0xeb600000
172 #define SDIV 0xfb90f0f0
173 #define SMULL 0xfb800000
174 #define STRD 0xe9400000
175 #define STR_SP 0x9000
176 #define SUBS 0x1a00
177 #define SUBSI3 0x1e00
178 #define SUBSI8 0x3800
179 #define SUB_W 0xeba00000
180 #define SUBWI 0xf2a00000
181 #define SUB_SP_I 0xb080
182 #define SUB_WI 0xf1a00000
183 #define SXTB 0xb240
184 #define SXTB_W 0xfa4ff080
185 #define SXTH 0xb200
186 #define SXTH_W 0xfa0ff080
187 #define TST 0x4200
188 #define TSTI 0xf0000f00
189 #define TST_W 0xea000f00
190 #define UDIV 0xfbb0f0f0
191 #define UMULL 0xfba00000
192 #define UXTB 0xb2c0
193 #define UXTB_W 0xfa5ff080
194 #define UXTH 0xb280
195 #define UXTH_W 0xfa1ff080
196 #define VABS_F32 0xeeb00ac0
197 #define VADD_F32 0xee300a00
198 #define VCMP_F32 0xeeb40a40
199 #define VCVT_F32_S32 0xeeb80ac0
200 #define VCVT_F64_F32 0xeeb70ac0
201 #define VCVT_S32_F32 0xeebd0ac0
202 #define VDIV_F32 0xee800a00
203 #define VLDR_F32 0xed100a00
204 #define VMOV_F32 0xeeb00a40
205 #define VMOV 0xee000a10
206 #define VMOV2 0xec400a10
207 #define VMRS 0xeef1fa10
208 #define VMUL_F32 0xee200a00
209 #define VNEG_F32 0xeeb10a40
210 #define VPOP 0xecbd0b00
211 #define VPUSH 0xed2d0b00
212 #define VSTR_F32 0xed000a00
213 #define VSUB_F32 0xee300a40
214
push_inst16(struct sljit_compiler * compiler,sljit_ins inst)215 static sljit_s32 push_inst16(struct sljit_compiler *compiler, sljit_ins inst)
216 {
217 sljit_u16 *ptr;
218 SLJIT_ASSERT(!(inst & 0xffff0000));
219
220 ptr = (sljit_u16*)ensure_buf(compiler, sizeof(sljit_u16));
221 FAIL_IF(!ptr);
222 *ptr = (sljit_u16)(inst);
223 compiler->size++;
224 return SLJIT_SUCCESS;
225 }
226
push_inst32(struct sljit_compiler * compiler,sljit_ins inst)227 static sljit_s32 push_inst32(struct sljit_compiler *compiler, sljit_ins inst)
228 {
229 sljit_u16 *ptr = (sljit_u16*)ensure_buf(compiler, sizeof(sljit_ins));
230 FAIL_IF(!ptr);
231 *ptr++ = (sljit_u16)(inst >> 16);
232 *ptr = (sljit_u16)(inst);
233 compiler->size += 2;
234 return SLJIT_SUCCESS;
235 }
236
emit_imm32_const(struct sljit_compiler * compiler,sljit_s32 dst,sljit_uw imm)237 static SLJIT_INLINE sljit_s32 emit_imm32_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_uw imm)
238 {
239 FAIL_IF(push_inst32(compiler, MOVW | RD4(dst)
240 | COPY_BITS(imm, 12, 16, 4) | COPY_BITS(imm, 11, 26, 1) | COPY_BITS(imm, 8, 12, 3) | (imm & 0xff)));
241 return push_inst32(compiler, MOVT | RD4(dst)
242 | COPY_BITS(imm, 12 + 16, 16, 4) | COPY_BITS(imm, 11 + 16, 26, 1) | COPY_BITS(imm, 8 + 16, 12, 3) | ((imm & 0xff0000) >> 16));
243 }
244
modify_imm32_const(sljit_u16 * inst,sljit_uw new_imm)245 static SLJIT_INLINE void modify_imm32_const(sljit_u16 *inst, sljit_uw new_imm)
246 {
247 sljit_ins dst = inst[1] & 0x0f00;
248 SLJIT_ASSERT(((inst[0] & 0xfbf0) == (MOVW >> 16)) && ((inst[2] & 0xfbf0) == (MOVT >> 16)) && dst == (inst[3] & 0x0f00));
249 inst[0] = (sljit_u16)((MOVW >> 16) | COPY_BITS(new_imm, 12, 0, 4) | COPY_BITS(new_imm, 11, 10, 1));
250 inst[1] = (sljit_u16)(dst | COPY_BITS(new_imm, 8, 12, 3) | (new_imm & 0xff));
251 inst[2] = (sljit_u16)((MOVT >> 16) | COPY_BITS(new_imm, 12 + 16, 0, 4) | COPY_BITS(new_imm, 11 + 16, 10, 1));
252 inst[3] = (sljit_u16)(dst | COPY_BITS(new_imm, 8 + 16, 12, 3) | ((new_imm & 0xff0000) >> 16));
253 }
254
detect_jump_type(struct sljit_jump * jump,sljit_u16 * code_ptr,sljit_u16 * code,sljit_sw executable_offset)255 static SLJIT_INLINE sljit_s32 detect_jump_type(struct sljit_jump *jump, sljit_u16 *code_ptr, sljit_u16 *code, sljit_sw executable_offset)
256 {
257 sljit_sw diff;
258
259 if (jump->flags & SLJIT_REWRITABLE_JUMP)
260 return 0;
261
262 if (jump->flags & JUMP_ADDR) {
263 /* Branch to ARM code is not optimized yet. */
264 if (!(jump->u.target & 0x1))
265 return 0;
266 diff = ((sljit_sw)jump->u.target - (sljit_sw)(code_ptr + 2) - executable_offset) >> 1;
267 }
268 else {
269 SLJIT_ASSERT(jump->flags & JUMP_LABEL);
270 diff = ((sljit_sw)(code + jump->u.label->size) - (sljit_sw)(code_ptr + 2)) >> 1;
271 }
272
273 if (jump->flags & IS_COND) {
274 SLJIT_ASSERT(!(jump->flags & IS_BL));
275 if (diff <= 127 && diff >= -128) {
276 jump->flags |= PATCH_TYPE1;
277 return 5;
278 }
279 if (diff <= 524287 && diff >= -524288) {
280 jump->flags |= PATCH_TYPE2;
281 return 4;
282 }
283 /* +1 comes from the prefix IT instruction. */
284 diff--;
285 if (diff <= 8388607 && diff >= -8388608) {
286 jump->flags |= PATCH_TYPE3;
287 return 3;
288 }
289 }
290 else if (jump->flags & IS_BL) {
291 if (diff <= 8388607 && diff >= -8388608) {
292 jump->flags |= PATCH_BL;
293 return 3;
294 }
295 }
296 else {
297 if (diff <= 1023 && diff >= -1024) {
298 jump->flags |= PATCH_TYPE4;
299 return 4;
300 }
301 if (diff <= 8388607 && diff >= -8388608) {
302 jump->flags |= PATCH_TYPE5;
303 return 3;
304 }
305 }
306
307 return 0;
308 }
309
set_jump_instruction(struct sljit_jump * jump,sljit_sw executable_offset)310 static SLJIT_INLINE void set_jump_instruction(struct sljit_jump *jump, sljit_sw executable_offset)
311 {
312 sljit_s32 type = (jump->flags >> 4) & 0xf;
313 sljit_sw diff;
314 sljit_u16 *jump_inst;
315 sljit_s32 s, j1, j2;
316
317 if (SLJIT_UNLIKELY(type == 0)) {
318 modify_imm32_const((sljit_u16*)jump->addr, (jump->flags & JUMP_LABEL) ? jump->u.label->addr : jump->u.target);
319 return;
320 }
321
322 if (jump->flags & JUMP_ADDR) {
323 SLJIT_ASSERT(jump->u.target & 0x1);
324 diff = ((sljit_sw)jump->u.target - (sljit_sw)(jump->addr + sizeof(sljit_u32)) - executable_offset) >> 1;
325 }
326 else {
327 SLJIT_ASSERT(jump->u.label->addr & 0x1);
328 diff = ((sljit_sw)(jump->u.label->addr) - (sljit_sw)(jump->addr + sizeof(sljit_u32)) - executable_offset) >> 1;
329 }
330 jump_inst = (sljit_u16*)jump->addr;
331
332 switch (type) {
333 case 1:
334 /* Encoding T1 of 'B' instruction */
335 SLJIT_ASSERT(diff <= 127 && diff >= -128 && (jump->flags & IS_COND));
336 jump_inst[0] = (sljit_u16)(0xd000 | (jump->flags & 0xf00) | ((sljit_ins)diff & 0xff));
337 return;
338 case 2:
339 /* Encoding T3 of 'B' instruction */
340 SLJIT_ASSERT(diff <= 524287 && diff >= -524288 && (jump->flags & IS_COND));
341 jump_inst[0] = (sljit_u16)(0xf000 | COPY_BITS(jump->flags, 8, 6, 4) | COPY_BITS(diff, 11, 0, 6) | COPY_BITS(diff, 19, 10, 1));
342 jump_inst[1] = (sljit_u16)(0x8000 | COPY_BITS(diff, 17, 13, 1) | COPY_BITS(diff, 18, 11, 1) | ((sljit_ins)diff & 0x7ff));
343 return;
344 case 3:
345 SLJIT_ASSERT(jump->flags & IS_COND);
346 *jump_inst++ = (sljit_u16)(IT | ((jump->flags >> 4) & 0xf0) | 0x8);
347 diff--;
348 type = 5;
349 break;
350 case 4:
351 /* Encoding T2 of 'B' instruction */
352 SLJIT_ASSERT(diff <= 1023 && diff >= -1024 && !(jump->flags & IS_COND));
353 jump_inst[0] = (sljit_u16)(0xe000 | (diff & 0x7ff));
354 return;
355 }
356
357 SLJIT_ASSERT(diff <= 8388607 && diff >= -8388608);
358
359 /* Really complex instruction form for branches. */
360 s = (diff >> 23) & 0x1;
361 j1 = (~(diff >> 22) ^ s) & 0x1;
362 j2 = (~(diff >> 21) ^ s) & 0x1;
363 jump_inst[0] = (sljit_u16)(0xf000 | ((sljit_ins)s << 10) | COPY_BITS(diff, 11, 0, 10));
364 jump_inst[1] = (sljit_u16)((j1 << 13) | (j2 << 11) | (diff & 0x7ff));
365
366 /* The others have a common form. */
367 if (type == 5) /* Encoding T4 of 'B' instruction */
368 jump_inst[1] |= 0x9000;
369 else if (type == 6) /* Encoding T1 of 'BL' instruction */
370 jump_inst[1] |= 0xd000;
371 else
372 SLJIT_UNREACHABLE();
373 }
374
sljit_generate_code(struct sljit_compiler * compiler)375 SLJIT_API_FUNC_ATTRIBUTE void* sljit_generate_code(struct sljit_compiler *compiler)
376 {
377 struct sljit_memory_fragment *buf;
378 sljit_u16 *code;
379 sljit_u16 *code_ptr;
380 sljit_u16 *buf_ptr;
381 sljit_u16 *buf_end;
382 sljit_uw half_count;
383 sljit_uw next_addr;
384 sljit_sw executable_offset;
385
386 struct sljit_label *label;
387 struct sljit_jump *jump;
388 struct sljit_const *const_;
389 struct sljit_put_label *put_label;
390
391 CHECK_ERROR_PTR();
392 CHECK_PTR(check_sljit_generate_code(compiler));
393 reverse_buf(compiler);
394
395 code = (sljit_u16*)SLJIT_MALLOC_EXEC(compiler->size * sizeof(sljit_u16), compiler->exec_allocator_data);
396 PTR_FAIL_WITH_EXEC_IF(code);
397 buf = compiler->buf;
398
399 code_ptr = code;
400 half_count = 0;
401 next_addr = 0;
402 executable_offset = SLJIT_EXEC_OFFSET(code);
403
404 label = compiler->labels;
405 jump = compiler->jumps;
406 const_ = compiler->consts;
407 put_label = compiler->put_labels;
408
409 do {
410 buf_ptr = (sljit_u16*)buf->memory;
411 buf_end = buf_ptr + (buf->used_size >> 1);
412 do {
413 *code_ptr = *buf_ptr++;
414 if (next_addr == half_count) {
415 SLJIT_ASSERT(!label || label->size >= half_count);
416 SLJIT_ASSERT(!jump || jump->addr >= half_count);
417 SLJIT_ASSERT(!const_ || const_->addr >= half_count);
418 SLJIT_ASSERT(!put_label || put_label->addr >= half_count);
419
420 /* These structures are ordered by their address. */
421 if (label && label->size == half_count) {
422 label->addr = ((sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset)) | 0x1;
423 label->size = (sljit_uw)(code_ptr - code);
424 label = label->next;
425 }
426 if (jump && jump->addr == half_count) {
427 jump->addr = (sljit_uw)code_ptr - ((jump->flags & IS_COND) ? 10 : 8);
428 code_ptr -= detect_jump_type(jump, code_ptr, code, executable_offset);
429 jump = jump->next;
430 }
431 if (const_ && const_->addr == half_count) {
432 const_->addr = (sljit_uw)code_ptr;
433 const_ = const_->next;
434 }
435 if (put_label && put_label->addr == half_count) {
436 SLJIT_ASSERT(put_label->label);
437 put_label->addr = (sljit_uw)code_ptr;
438 put_label = put_label->next;
439 }
440 next_addr = compute_next_addr(label, jump, const_, put_label);
441 }
442 code_ptr++;
443 half_count++;
444 } while (buf_ptr < buf_end);
445
446 buf = buf->next;
447 } while (buf);
448
449 if (label && label->size == half_count) {
450 label->addr = ((sljit_uw)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset)) | 0x1;
451 label->size = (sljit_uw)(code_ptr - code);
452 label = label->next;
453 }
454
455 SLJIT_ASSERT(!label);
456 SLJIT_ASSERT(!jump);
457 SLJIT_ASSERT(!const_);
458 SLJIT_ASSERT(!put_label);
459 SLJIT_ASSERT(code_ptr - code <= (sljit_sw)compiler->size);
460
461 jump = compiler->jumps;
462 while (jump) {
463 set_jump_instruction(jump, executable_offset);
464 jump = jump->next;
465 }
466
467 put_label = compiler->put_labels;
468 while (put_label) {
469 modify_imm32_const((sljit_u16 *)put_label->addr, put_label->label->addr);
470 put_label = put_label->next;
471 }
472
473 compiler->error = SLJIT_ERR_COMPILED;
474 compiler->executable_offset = executable_offset;
475 compiler->executable_size = (sljit_uw)(code_ptr - code) * sizeof(sljit_u16);
476
477 code = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(code, executable_offset);
478 code_ptr = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(code_ptr, executable_offset);
479
480 SLJIT_CACHE_FLUSH(code, code_ptr);
481 SLJIT_UPDATE_WX_FLAGS(code, code_ptr, 1);
482
483 /* Set thumb mode flag. */
484 return (void*)((sljit_uw)code | 0x1);
485 }
486
sljit_has_cpu_feature(sljit_s32 feature_type)487 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_has_cpu_feature(sljit_s32 feature_type)
488 {
489 switch (feature_type) {
490 case SLJIT_HAS_FPU:
491 #ifdef SLJIT_IS_FPU_AVAILABLE
492 return SLJIT_IS_FPU_AVAILABLE;
493 #else
494 /* Available by default. */
495 return 1;
496 #endif
497
498 case SLJIT_HAS_CLZ:
499 case SLJIT_HAS_CTZ:
500 case SLJIT_HAS_ROT:
501 case SLJIT_HAS_CMOV:
502 case SLJIT_HAS_PREFETCH:
503 return 1;
504
505 default:
506 return 0;
507 }
508 }
509
510 /* --------------------------------------------------------------------- */
511 /* Core code generator functions. */
512 /* --------------------------------------------------------------------- */
513
514 #define INVALID_IMM 0x80000000
get_imm(sljit_uw imm)515 static sljit_uw get_imm(sljit_uw imm)
516 {
517 /* Thumb immediate form. */
518 sljit_s32 counter;
519
520 if (imm <= 0xff)
521 return imm;
522
523 if ((imm & 0xffff) == (imm >> 16)) {
524 /* Some special cases. */
525 if (!(imm & 0xff00))
526 return (1 << 12) | (imm & 0xff);
527 if (!(imm & 0xff))
528 return (2 << 12) | ((imm >> 8) & 0xff);
529 if ((imm & 0xff00) == ((imm & 0xff) << 8))
530 return (3 << 12) | (imm & 0xff);
531 }
532
533 /* Assembly optimization: count leading zeroes? */
534 counter = 8;
535 if (!(imm & 0xffff0000)) {
536 counter += 16;
537 imm <<= 16;
538 }
539 if (!(imm & 0xff000000)) {
540 counter += 8;
541 imm <<= 8;
542 }
543 if (!(imm & 0xf0000000)) {
544 counter += 4;
545 imm <<= 4;
546 }
547 if (!(imm & 0xc0000000)) {
548 counter += 2;
549 imm <<= 2;
550 }
551 if (!(imm & 0x80000000)) {
552 counter += 1;
553 imm <<= 1;
554 }
555 /* Since imm >= 128, this must be true. */
556 SLJIT_ASSERT(counter <= 31);
557
558 if (imm & 0x00ffffff)
559 return INVALID_IMM; /* Cannot be encoded. */
560
561 return ((imm >> 24) & 0x7f) | COPY_BITS(counter, 4, 26, 1) | COPY_BITS(counter, 1, 12, 3) | COPY_BITS(counter, 0, 7, 1);
562 }
563
load_immediate(struct sljit_compiler * compiler,sljit_s32 dst,sljit_uw imm)564 static sljit_s32 load_immediate(struct sljit_compiler *compiler, sljit_s32 dst, sljit_uw imm)
565 {
566 sljit_uw tmp;
567
568 /* MOVS cannot be used since it destroy flags. */
569
570 if (imm >= 0x10000) {
571 tmp = get_imm(imm);
572 if (tmp != INVALID_IMM)
573 return push_inst32(compiler, MOV_WI | RD4(dst) | tmp);
574 tmp = get_imm(~imm);
575 if (tmp != INVALID_IMM)
576 return push_inst32(compiler, MVN_WI | RD4(dst) | tmp);
577 }
578
579 /* set low 16 bits, set hi 16 bits to 0. */
580 FAIL_IF(push_inst32(compiler, MOVW | RD4(dst)
581 | COPY_BITS(imm, 12, 16, 4) | COPY_BITS(imm, 11, 26, 1) | COPY_BITS(imm, 8, 12, 3) | (imm & 0xff)));
582
583 /* set hi 16 bit if needed. */
584 if (imm >= 0x10000)
585 return push_inst32(compiler, MOVT | RD4(dst)
586 | COPY_BITS(imm, 12 + 16, 16, 4) | COPY_BITS(imm, 11 + 16, 26, 1) | COPY_BITS(imm, 8 + 16, 12, 3) | ((imm & 0xff0000) >> 16));
587 return SLJIT_SUCCESS;
588 }
589
590 #define ARG1_IMM 0x0010000
591 #define ARG2_IMM 0x0020000
592 /* SET_FLAGS must be 0x100000 as it is also the value of S bit (can be used for optimization). */
593 #define SET_FLAGS 0x0100000
594 #define UNUSED_RETURN 0x0200000
595
emit_op_imm(struct sljit_compiler * compiler,sljit_s32 flags,sljit_s32 dst,sljit_uw arg1,sljit_uw arg2)596 static sljit_s32 emit_op_imm(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 dst, sljit_uw arg1, sljit_uw arg2)
597 {
598 /* dst must be register, TMP_REG1
599 arg1 must be register, imm
600 arg2 must be register, imm */
601 sljit_s32 reg;
602 sljit_uw imm, imm2;
603
604 if (SLJIT_UNLIKELY((flags & (ARG1_IMM | ARG2_IMM)) == (ARG1_IMM | ARG2_IMM))) {
605 /* Both are immediates, no temporaries are used. */
606 flags &= ~ARG1_IMM;
607 FAIL_IF(load_immediate(compiler, TMP_REG1, arg1));
608 arg1 = TMP_REG1;
609 }
610
611 if (flags & (ARG1_IMM | ARG2_IMM)) {
612 reg = (sljit_s32)((flags & ARG2_IMM) ? arg1 : arg2);
613 imm = (flags & ARG2_IMM) ? arg2 : arg1;
614
615 switch (flags & 0xffff) {
616 case SLJIT_CLZ:
617 case SLJIT_CTZ:
618 case SLJIT_MUL:
619 /* No form with immediate operand. */
620 break;
621 case SLJIT_MOV:
622 SLJIT_ASSERT(!(flags & SET_FLAGS) && (flags & ARG2_IMM) && arg1 == TMP_REG2);
623 return load_immediate(compiler, dst, imm);
624 case SLJIT_NOT:
625 if (!(flags & SET_FLAGS))
626 return load_immediate(compiler, dst, ~imm);
627 /* Since the flags should be set, we just fallback to the register mode.
628 Although some clever things could be done here, "NOT IMM" does not worth the efforts. */
629 break;
630 case SLJIT_ADD:
631 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
632 imm2 = NEGATE(imm);
633 if (IS_2_LO_REGS(reg, dst)) {
634 if (imm <= 0x7)
635 return push_inst16(compiler, ADDSI3 | IMM3(imm) | RD3(dst) | RN3(reg));
636 if (imm2 <= 0x7)
637 return push_inst16(compiler, SUBSI3 | IMM3(imm2) | RD3(dst) | RN3(reg));
638 if (reg == dst) {
639 if (imm <= 0xff)
640 return push_inst16(compiler, ADDSI8 | IMM8(imm) | RDN3(dst));
641 if (imm2 <= 0xff)
642 return push_inst16(compiler, SUBSI8 | IMM8(imm2) | RDN3(dst));
643 }
644 }
645 if (!(flags & SET_FLAGS)) {
646 if (imm <= 0xfff)
647 return push_inst32(compiler, ADDWI | RD4(dst) | RN4(reg) | IMM12(imm));
648 if (imm2 <= 0xfff)
649 return push_inst32(compiler, SUBWI | RD4(dst) | RN4(reg) | IMM12(imm2));
650 }
651 imm2 = get_imm(imm);
652 if (imm2 != INVALID_IMM)
653 return push_inst32(compiler, ADD_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm2);
654 imm = get_imm(NEGATE(imm));
655 if (imm != INVALID_IMM)
656 return push_inst32(compiler, SUB_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
657 break;
658 case SLJIT_ADDC:
659 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
660 imm = get_imm(imm);
661 if (imm != INVALID_IMM)
662 return push_inst32(compiler, ADCI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
663 break;
664 case SLJIT_SUB:
665 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
666 if (flags & ARG1_IMM) {
667 if (imm == 0 && IS_2_LO_REGS(reg, dst))
668 return push_inst16(compiler, RSBSI | RD3(dst) | RN3(reg));
669 imm = get_imm(imm);
670 if (imm != INVALID_IMM)
671 return push_inst32(compiler, RSB_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
672 break;
673 }
674 if (flags & UNUSED_RETURN) {
675 if (imm <= 0xff && reg_map[reg] <= 7)
676 return push_inst16(compiler, CMPI | IMM8(imm) | RDN3(reg));
677 imm2 = get_imm(imm);
678 if (imm2 != INVALID_IMM)
679 return push_inst32(compiler, CMPI_W | RN4(reg) | imm2);
680 imm = get_imm(NEGATE(imm));
681 if (imm != INVALID_IMM)
682 return push_inst32(compiler, CMNI_W | RN4(reg) | imm);
683 break;
684 }
685 imm2 = NEGATE(imm);
686 if (IS_2_LO_REGS(reg, dst)) {
687 if (imm <= 0x7)
688 return push_inst16(compiler, SUBSI3 | IMM3(imm) | RD3(dst) | RN3(reg));
689 if (imm2 <= 0x7)
690 return push_inst16(compiler, ADDSI3 | IMM3(imm2) | RD3(dst) | RN3(reg));
691 if (reg == dst) {
692 if (imm <= 0xff)
693 return push_inst16(compiler, SUBSI8 | IMM8(imm) | RDN3(dst));
694 if (imm2 <= 0xff)
695 return push_inst16(compiler, ADDSI8 | IMM8(imm2) | RDN3(dst));
696 }
697 }
698 if (!(flags & SET_FLAGS)) {
699 if (imm <= 0xfff)
700 return push_inst32(compiler, SUBWI | RD4(dst) | RN4(reg) | IMM12(imm));
701 if (imm2 <= 0xfff)
702 return push_inst32(compiler, ADDWI | RD4(dst) | RN4(reg) | IMM12(imm2));
703 }
704 imm2 = get_imm(imm);
705 if (imm2 != INVALID_IMM)
706 return push_inst32(compiler, SUB_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm2);
707 imm = get_imm(NEGATE(imm));
708 if (imm != INVALID_IMM)
709 return push_inst32(compiler, ADD_WI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
710 break;
711 case SLJIT_SUBC:
712 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
713 if (flags & ARG1_IMM)
714 break;
715 imm = get_imm(imm);
716 if (imm != INVALID_IMM)
717 return push_inst32(compiler, SBCI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
718 break;
719 case SLJIT_AND:
720 imm2 = get_imm(imm);
721 if (imm2 != INVALID_IMM)
722 return push_inst32(compiler, ((flags & UNUSED_RETURN) ? TSTI : ANDI) | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm2);
723 imm = get_imm(~imm);
724 if (imm != INVALID_IMM)
725 return push_inst32(compiler, BICI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
726 break;
727 case SLJIT_OR:
728 imm2 = get_imm(imm);
729 if (imm2 != INVALID_IMM)
730 return push_inst32(compiler, ORRI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm2);
731 imm = get_imm(~imm);
732 if (imm != INVALID_IMM)
733 return push_inst32(compiler, ORNI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
734 break;
735 case SLJIT_XOR:
736 imm = get_imm(imm);
737 if (imm != INVALID_IMM)
738 return push_inst32(compiler, EORI | (flags & SET_FLAGS) | RD4(dst) | RN4(reg) | imm);
739 break;
740 case SLJIT_SHL:
741 case SLJIT_MSHL:
742 case SLJIT_LSHR:
743 case SLJIT_MLSHR:
744 case SLJIT_ASHR:
745 case SLJIT_MASHR:
746 case SLJIT_ROTL:
747 case SLJIT_ROTR:
748 if (flags & ARG1_IMM)
749 break;
750 imm &= 0x1f;
751
752 if (imm == 0) {
753 if (!(flags & SET_FLAGS))
754 return push_inst16(compiler, MOV | SET_REGS44(dst, reg));
755 if (IS_2_LO_REGS(dst, reg))
756 return push_inst16(compiler, MOVS | RD3(dst) | RN3(reg));
757 return push_inst32(compiler, MOV_W | SET_FLAGS | RD4(dst) | RM4(reg));
758 }
759
760 switch (flags & 0xffff) {
761 case SLJIT_SHL:
762 case SLJIT_MSHL:
763 if (IS_2_LO_REGS(dst, reg))
764 return push_inst16(compiler, LSLSI | RD3(dst) | RN3(reg) | (imm << 6));
765 return push_inst32(compiler, LSL_WI | (flags & SET_FLAGS) | RD4(dst) | RM4(reg) | IMM5(imm));
766 case SLJIT_LSHR:
767 case SLJIT_MLSHR:
768 if (IS_2_LO_REGS(dst, reg))
769 return push_inst16(compiler, LSRSI | RD3(dst) | RN3(reg) | (imm << 6));
770 return push_inst32(compiler, LSR_WI | (flags & SET_FLAGS) | RD4(dst) | RM4(reg) | IMM5(imm));
771 case SLJIT_ASHR:
772 case SLJIT_MASHR:
773 if (IS_2_LO_REGS(dst, reg))
774 return push_inst16(compiler, ASRSI | RD3(dst) | RN3(reg) | (imm << 6));
775 return push_inst32(compiler, ASR_WI | (flags & SET_FLAGS) | RD4(dst) | RM4(reg) | IMM5(imm));
776 case SLJIT_ROTL:
777 imm = (imm ^ 0x1f) + 1;
778 /* fallthrough */
779 default: /* SLJIT_ROTR */
780 return push_inst32(compiler, ROR_WI | RD4(dst) | RM4(reg) | IMM5(imm));
781 }
782 default:
783 SLJIT_UNREACHABLE();
784 break;
785 }
786
787 if (flags & ARG2_IMM) {
788 imm = arg2;
789 arg2 = (arg1 == TMP_REG1) ? TMP_REG2 : TMP_REG1;
790 FAIL_IF(load_immediate(compiler, (sljit_s32)arg2, imm));
791 }
792 else {
793 imm = arg1;
794 arg1 = (arg2 == TMP_REG1) ? TMP_REG2 : TMP_REG1;
795 FAIL_IF(load_immediate(compiler, (sljit_s32)arg1, imm));
796 }
797
798 SLJIT_ASSERT(arg1 != arg2);
799 }
800
801 /* Both arguments are registers. */
802 switch (flags & 0xffff) {
803 case SLJIT_MOV:
804 case SLJIT_MOV_U32:
805 case SLJIT_MOV_S32:
806 case SLJIT_MOV32:
807 case SLJIT_MOV_P:
808 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG2);
809 if (dst == (sljit_s32)arg2)
810 return SLJIT_SUCCESS;
811 return push_inst16(compiler, MOV | SET_REGS44(dst, arg2));
812 case SLJIT_MOV_U8:
813 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG2);
814 if (IS_2_LO_REGS(dst, arg2))
815 return push_inst16(compiler, UXTB | RD3(dst) | RN3(arg2));
816 return push_inst32(compiler, UXTB_W | RD4(dst) | RM4(arg2));
817 case SLJIT_MOV_S8:
818 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG2);
819 if (IS_2_LO_REGS(dst, arg2))
820 return push_inst16(compiler, SXTB | RD3(dst) | RN3(arg2));
821 return push_inst32(compiler, SXTB_W | RD4(dst) | RM4(arg2));
822 case SLJIT_MOV_U16:
823 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG2);
824 if (IS_2_LO_REGS(dst, arg2))
825 return push_inst16(compiler, UXTH | RD3(dst) | RN3(arg2));
826 return push_inst32(compiler, UXTH_W | RD4(dst) | RM4(arg2));
827 case SLJIT_MOV_S16:
828 SLJIT_ASSERT(!(flags & SET_FLAGS) && arg1 == TMP_REG2);
829 if (IS_2_LO_REGS(dst, arg2))
830 return push_inst16(compiler, SXTH | RD3(dst) | RN3(arg2));
831 return push_inst32(compiler, SXTH_W | RD4(dst) | RM4(arg2));
832 case SLJIT_NOT:
833 SLJIT_ASSERT(arg1 == TMP_REG2);
834 if (IS_2_LO_REGS(dst, arg2))
835 return push_inst16(compiler, MVNS | RD3(dst) | RN3(arg2));
836 return push_inst32(compiler, MVN_W | (flags & SET_FLAGS) | RD4(dst) | RM4(arg2));
837 case SLJIT_CLZ:
838 SLJIT_ASSERT(arg1 == TMP_REG2);
839 return push_inst32(compiler, CLZ | RN4(arg2) | RD4(dst) | RM4(arg2));
840 case SLJIT_CTZ:
841 SLJIT_ASSERT(arg1 == TMP_REG2);
842 FAIL_IF(push_inst32(compiler, RBIT | RN4(arg2) | RD4(dst) | RM4(arg2)));
843 return push_inst32(compiler, CLZ | RN4(dst) | RD4(dst) | RM4(dst));
844 case SLJIT_ADD:
845 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
846 if (IS_3_LO_REGS(dst, arg1, arg2))
847 return push_inst16(compiler, ADDS | RD3(dst) | RN3(arg1) | RM3(arg2));
848 if (dst == (sljit_s32)arg1 && !(flags & SET_FLAGS))
849 return push_inst16(compiler, ADD | SET_REGS44(dst, arg2));
850 return push_inst32(compiler, ADD_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
851 case SLJIT_ADDC:
852 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_ADD;
853 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
854 return push_inst16(compiler, ADCS | RD3(dst) | RN3(arg2));
855 return push_inst32(compiler, ADC_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
856 case SLJIT_SUB:
857 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
858 if (flags & UNUSED_RETURN) {
859 if (IS_2_LO_REGS(arg1, arg2))
860 return push_inst16(compiler, CMP | RD3(arg1) | RN3(arg2));
861 return push_inst16(compiler, CMP_X | SET_REGS44(arg1, arg2));
862 }
863 if (IS_3_LO_REGS(dst, arg1, arg2))
864 return push_inst16(compiler, SUBS | RD3(dst) | RN3(arg1) | RM3(arg2));
865 return push_inst32(compiler, SUB_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
866 case SLJIT_SUBC:
867 compiler->status_flags_state = SLJIT_CURRENT_FLAGS_SUB;
868 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
869 return push_inst16(compiler, SBCS | RD3(dst) | RN3(arg2));
870 return push_inst32(compiler, SBC_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
871 case SLJIT_MUL:
872 compiler->status_flags_state = 0;
873 if (!(flags & SET_FLAGS))
874 return push_inst32(compiler, MUL | RD4(dst) | RN4(arg1) | RM4(arg2));
875 SLJIT_ASSERT(dst != TMP_REG2);
876 FAIL_IF(push_inst32(compiler, SMULL | RT4(dst) | RD4(TMP_REG2) | RN4(arg1) | RM4(arg2)));
877 /* cmp TMP_REG2, dst asr #31. */
878 return push_inst32(compiler, CMP_W | RN4(TMP_REG2) | 0x70e0 | RM4(dst));
879 case SLJIT_AND:
880 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
881 return push_inst16(compiler, ANDS | RD3(dst) | RN3(arg2));
882 if ((flags & UNUSED_RETURN) && IS_2_LO_REGS(arg1, arg2))
883 return push_inst16(compiler, TST | RD3(arg1) | RN3(arg2));
884 return push_inst32(compiler, ((flags & UNUSED_RETURN) ? TST_W : AND_W) | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
885 case SLJIT_OR:
886 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
887 return push_inst16(compiler, ORRS | RD3(dst) | RN3(arg2));
888 return push_inst32(compiler, ORR_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
889 case SLJIT_XOR:
890 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
891 return push_inst16(compiler, EORS | RD3(dst) | RN3(arg2));
892 return push_inst32(compiler, EOR_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
893 case SLJIT_MSHL:
894 FAIL_IF(push_inst32(compiler, ANDI | RD4(TMP_REG2) | RN4(arg2) | 0x1f));
895 arg2 = TMP_REG2;
896 /* fallthrough */
897 case SLJIT_SHL:
898 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
899 return push_inst16(compiler, LSLS | RD3(dst) | RN3(arg2));
900 return push_inst32(compiler, LSL_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
901 case SLJIT_MLSHR:
902 FAIL_IF(push_inst32(compiler, ANDI | RD4(TMP_REG2) | RN4(arg2) | 0x1f));
903 arg2 = TMP_REG2;
904 /* fallthrough */
905 case SLJIT_LSHR:
906 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
907 return push_inst16(compiler, LSRS | RD3(dst) | RN3(arg2));
908 return push_inst32(compiler, LSR_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
909 case SLJIT_MASHR:
910 FAIL_IF(push_inst32(compiler, ANDI | RD4(TMP_REG2) | RN4(arg2) | 0x1f));
911 arg2 = TMP_REG2;
912 /* fallthrough */
913 case SLJIT_ASHR:
914 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
915 return push_inst16(compiler, ASRS | RD3(dst) | RN3(arg2));
916 return push_inst32(compiler, ASR_W | (flags & SET_FLAGS) | RD4(dst) | RN4(arg1) | RM4(arg2));
917 case SLJIT_ROTL:
918 FAIL_IF(push_inst32(compiler, RSB_WI | RD4(TMP_REG2) | RN4(arg2) | 0));
919 arg2 = TMP_REG2;
920 /* fallthrough */
921 case SLJIT_ROTR:
922 if (dst == (sljit_s32)arg1 && IS_2_LO_REGS(dst, arg2))
923 return push_inst16(compiler, RORS | RD3(dst) | RN3(arg2));
924 return push_inst32(compiler, ROR_W | RD4(dst) | RN4(arg1) | RM4(arg2));
925 }
926
927 SLJIT_UNREACHABLE();
928 return SLJIT_SUCCESS;
929 }
930
931 #define STORE 0x01
932 #define SIGNED 0x02
933
934 #define WORD_SIZE 0x00
935 #define BYTE_SIZE 0x04
936 #define HALF_SIZE 0x08
937 #define PRELOAD 0x0c
938
939 #define IS_WORD_SIZE(flags) (!((flags) & (BYTE_SIZE | HALF_SIZE)))
940 #define ALIGN_CHECK(argw, imm, shift) (!((argw) & ~((imm) << (shift))))
941
942 /*
943 1st letter:
944 w = word
945 b = byte
946 h = half
947
948 2nd letter:
949 s = signed
950 u = unsigned
951
952 3rd letter:
953 l = load
954 s = store
955 */
956
957 static const sljit_ins sljit_mem16[12] = {
958 /* w u l */ 0x5800 /* ldr */,
959 /* w u s */ 0x5000 /* str */,
960 /* w s l */ 0x5800 /* ldr */,
961 /* w s s */ 0x5000 /* str */,
962
963 /* b u l */ 0x5c00 /* ldrb */,
964 /* b u s */ 0x5400 /* strb */,
965 /* b s l */ 0x5600 /* ldrsb */,
966 /* b s s */ 0x5400 /* strb */,
967
968 /* h u l */ 0x5a00 /* ldrh */,
969 /* h u s */ 0x5200 /* strh */,
970 /* h s l */ 0x5e00 /* ldrsh */,
971 /* h s s */ 0x5200 /* strh */,
972 };
973
974 static const sljit_ins sljit_mem16_imm5[12] = {
975 /* w u l */ 0x6800 /* ldr imm5 */,
976 /* w u s */ 0x6000 /* str imm5 */,
977 /* w s l */ 0x6800 /* ldr imm5 */,
978 /* w s s */ 0x6000 /* str imm5 */,
979
980 /* b u l */ 0x7800 /* ldrb imm5 */,
981 /* b u s */ 0x7000 /* strb imm5 */,
982 /* b s l */ 0x0000 /* not allowed */,
983 /* b s s */ 0x7000 /* strb imm5 */,
984
985 /* h u l */ 0x8800 /* ldrh imm5 */,
986 /* h u s */ 0x8000 /* strh imm5 */,
987 /* h s l */ 0x0000 /* not allowed */,
988 /* h s s */ 0x8000 /* strh imm5 */,
989 };
990
991 #define MEM_IMM8 0xc00
992 #define MEM_IMM12 0x800000
993 static const sljit_ins sljit_mem32[13] = {
994 /* w u l */ 0xf8500000 /* ldr.w */,
995 /* w u s */ 0xf8400000 /* str.w */,
996 /* w s l */ 0xf8500000 /* ldr.w */,
997 /* w s s */ 0xf8400000 /* str.w */,
998
999 /* b u l */ 0xf8100000 /* ldrb.w */,
1000 /* b u s */ 0xf8000000 /* strb.w */,
1001 /* b s l */ 0xf9100000 /* ldrsb.w */,
1002 /* b s s */ 0xf8000000 /* strb.w */,
1003
1004 /* h u l */ 0xf8300000 /* ldrh.w */,
1005 /* h u s */ 0xf8200000 /* strsh.w */,
1006 /* h s l */ 0xf9300000 /* ldrsh.w */,
1007 /* h s s */ 0xf8200000 /* strsh.w */,
1008
1009 /* p u l */ 0xf8100000 /* pld */,
1010 };
1011
1012 /* Helper function. Dst should be reg + value, using at most 1 instruction, flags does not set. */
emit_set_delta(struct sljit_compiler * compiler,sljit_s32 dst,sljit_s32 reg,sljit_sw value)1013 static sljit_s32 emit_set_delta(struct sljit_compiler *compiler, sljit_s32 dst, sljit_s32 reg, sljit_sw value)
1014 {
1015 sljit_uw imm;
1016
1017 if (value >= 0) {
1018 if (value <= 0xfff)
1019 return push_inst32(compiler, ADDWI | RD4(dst) | RN4(reg) | IMM12(value));
1020 imm = get_imm((sljit_uw)value);
1021 if (imm != INVALID_IMM)
1022 return push_inst32(compiler, ADD_WI | RD4(dst) | RN4(reg) | imm);
1023 }
1024 else {
1025 value = -value;
1026 if (value <= 0xfff)
1027 return push_inst32(compiler, SUBWI | RD4(dst) | RN4(reg) | IMM12(value));
1028 imm = get_imm((sljit_uw)value);
1029 if (imm != INVALID_IMM)
1030 return push_inst32(compiler, SUB_WI | RD4(dst) | RN4(reg) | imm);
1031 }
1032 return SLJIT_ERR_UNSUPPORTED;
1033 }
1034
emit_op_mem(struct sljit_compiler * compiler,sljit_s32 flags,sljit_s32 reg,sljit_s32 arg,sljit_sw argw,sljit_s32 tmp_reg)1035 static SLJIT_INLINE sljit_s32 emit_op_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg,
1036 sljit_s32 arg, sljit_sw argw, sljit_s32 tmp_reg)
1037 {
1038 sljit_s32 other_r;
1039 sljit_uw imm, tmp;
1040
1041 SLJIT_ASSERT(arg & SLJIT_MEM);
1042 SLJIT_ASSERT((arg & REG_MASK) != tmp_reg || (arg == SLJIT_MEM1(tmp_reg) && argw >= -0xff && argw <= 0xfff));
1043
1044 if (SLJIT_UNLIKELY(!(arg & REG_MASK))) {
1045 imm = get_imm((sljit_uw)argw & ~(sljit_uw)0xfff);
1046 if (imm != INVALID_IMM) {
1047 FAIL_IF(push_inst32(compiler, MOV_WI | RD4(tmp_reg) | imm));
1048 return push_inst32(compiler, sljit_mem32[flags] | MEM_IMM12 | RT4(reg) | RN4(tmp_reg) | (argw & 0xfff));
1049 }
1050
1051 FAIL_IF(load_immediate(compiler, tmp_reg, (sljit_uw)argw));
1052 if (IS_2_LO_REGS(reg, tmp_reg) && sljit_mem16_imm5[flags])
1053 return push_inst16(compiler, sljit_mem16_imm5[flags] | RD3(reg) | RN3(tmp_reg));
1054 return push_inst32(compiler, sljit_mem32[flags] | MEM_IMM12 | RT4(reg) | RN4(tmp_reg));
1055 }
1056
1057 if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
1058 argw &= 0x3;
1059 other_r = OFFS_REG(arg);
1060 arg &= REG_MASK;
1061
1062 if (!argw && IS_3_LO_REGS(reg, arg, other_r))
1063 return push_inst16(compiler, sljit_mem16[flags] | RD3(reg) | RN3(arg) | RM3(other_r));
1064 return push_inst32(compiler, sljit_mem32[flags] | RT4(reg) | RN4(arg) | RM4(other_r) | ((sljit_ins)argw << 4));
1065 }
1066
1067 arg &= REG_MASK;
1068
1069 if (argw > 0xfff) {
1070 imm = get_imm((sljit_uw)(argw & ~0xfff));
1071 if (imm != INVALID_IMM) {
1072 push_inst32(compiler, ADD_WI | RD4(tmp_reg) | RN4(arg) | imm);
1073 arg = tmp_reg;
1074 argw = argw & 0xfff;
1075 }
1076 }
1077 else if (argw < -0xff) {
1078 tmp = (sljit_uw)((-argw + 0xfff) & ~0xfff);
1079 SLJIT_ASSERT(tmp >= (sljit_uw)-argw);
1080 imm = get_imm(tmp);
1081
1082 if (imm != INVALID_IMM) {
1083 push_inst32(compiler, SUB_WI | RD4(tmp_reg) | RN4(arg) | imm);
1084 arg = tmp_reg;
1085 argw += (sljit_sw)tmp;
1086
1087 SLJIT_ASSERT(argw >= 0 && argw <= 0xfff);
1088 }
1089 }
1090
1091 /* 16 bit instruction forms. */
1092 if (IS_2_LO_REGS(reg, arg) && sljit_mem16_imm5[flags]) {
1093 tmp = 3;
1094 if (IS_WORD_SIZE(flags)) {
1095 if (ALIGN_CHECK(argw, 0x1f, 2))
1096 tmp = 2;
1097 }
1098 else if (flags & BYTE_SIZE)
1099 {
1100 if (ALIGN_CHECK(argw, 0x1f, 0))
1101 tmp = 0;
1102 }
1103 else {
1104 SLJIT_ASSERT(flags & HALF_SIZE);
1105 if (ALIGN_CHECK(argw, 0x1f, 1))
1106 tmp = 1;
1107 }
1108
1109 if (tmp < 3)
1110 return push_inst16(compiler, sljit_mem16_imm5[flags] | RD3(reg) | RN3(arg) | ((sljit_ins)argw << (6 - tmp)));
1111 }
1112 else if (SLJIT_UNLIKELY(arg == SLJIT_SP) && IS_WORD_SIZE(flags) && ALIGN_CHECK(argw, 0xff, 2) && reg_map[reg] <= 7) {
1113 /* SP based immediate. */
1114 return push_inst16(compiler, STR_SP | (sljit_ins)((flags & STORE) ? 0 : 0x800) | RDN3(reg) | ((sljit_ins)argw >> 2));
1115 }
1116
1117 if (argw >= 0 && argw <= 0xfff)
1118 return push_inst32(compiler, sljit_mem32[flags] | MEM_IMM12 | RT4(reg) | RN4(arg) | (sljit_ins)argw);
1119 else if (argw < 0 && argw >= -0xff)
1120 return push_inst32(compiler, sljit_mem32[flags] | MEM_IMM8 | RT4(reg) | RN4(arg) | (sljit_ins)-argw);
1121
1122 SLJIT_ASSERT(arg != tmp_reg);
1123
1124 FAIL_IF(load_immediate(compiler, tmp_reg, (sljit_uw)argw));
1125 if (IS_3_LO_REGS(reg, arg, tmp_reg))
1126 return push_inst16(compiler, sljit_mem16[flags] | RD3(reg) | RN3(arg) | RM3(tmp_reg));
1127 return push_inst32(compiler, sljit_mem32[flags] | RT4(reg) | RN4(arg) | RM4(tmp_reg));
1128 }
1129
1130 #undef ALIGN_CHECK
1131 #undef IS_WORD_SIZE
1132
1133 /* --------------------------------------------------------------------- */
1134 /* Entry, exit */
1135 /* --------------------------------------------------------------------- */
1136
sljit_emit_enter(struct sljit_compiler * compiler,sljit_s32 options,sljit_s32 arg_types,sljit_s32 scratches,sljit_s32 saveds,sljit_s32 fscratches,sljit_s32 fsaveds,sljit_s32 local_size)1137 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
1138 sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
1139 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
1140 {
1141 sljit_s32 size, i, tmp, word_arg_count;
1142 sljit_s32 saved_arg_count = SLJIT_KEPT_SAVEDS_COUNT(options);
1143 sljit_uw offset;
1144 sljit_uw imm = 0;
1145 #ifdef __SOFTFP__
1146 sljit_u32 float_arg_count;
1147 #else
1148 sljit_u32 old_offset, f32_offset;
1149 sljit_u32 remap[3];
1150 sljit_u32 *remap_ptr = remap;
1151 #endif
1152
1153 CHECK_ERROR();
1154 CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
1155 set_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
1156
1157 tmp = SLJIT_S0 - saveds;
1158 for (i = SLJIT_S0 - saved_arg_count; i > tmp; i--)
1159 imm |= (sljit_uw)1 << reg_map[i];
1160
1161 for (i = scratches; i >= SLJIT_FIRST_SAVED_REG; i--)
1162 imm |= (sljit_uw)1 << reg_map[i];
1163
1164 /* At least two registers must be set for PUSH_W and one for PUSH instruction. */
1165 FAIL_IF((imm & 0xff00)
1166 ? push_inst32(compiler, PUSH_W | (1 << 14) | imm)
1167 : push_inst16(compiler, PUSH | (1 << 8) | imm));
1168
1169 /* Stack must be aligned to 8 bytes: (LR, R4) */
1170 size = GET_SAVED_REGISTERS_SIZE(scratches, saveds - saved_arg_count, 1);
1171
1172 if (fsaveds > 0 || fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG) {
1173 if ((size & SSIZE_OF(sw)) != 0) {
1174 FAIL_IF(push_inst16(compiler, SUB_SP_I | (sizeof(sljit_sw) >> 2)));
1175 size += SSIZE_OF(sw);
1176 }
1177
1178 if (fsaveds + fscratches >= SLJIT_NUMBER_OF_FLOAT_REGISTERS) {
1179 FAIL_IF(push_inst32(compiler, VPUSH | DD4(SLJIT_FS0) | ((sljit_uw)SLJIT_NUMBER_OF_SAVED_FLOAT_REGISTERS << 1)));
1180 } else {
1181 if (fsaveds > 0)
1182 FAIL_IF(push_inst32(compiler, VPUSH | DD4(SLJIT_FS0) | ((sljit_uw)fsaveds << 1)));
1183 if (fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG)
1184 FAIL_IF(push_inst32(compiler, VPUSH | DD4(fscratches) | ((sljit_uw)(fscratches - (SLJIT_FIRST_SAVED_FLOAT_REG - 1)) << 1)));
1185 }
1186 }
1187
1188 local_size = ((size + local_size + 0x7) & ~0x7) - size;
1189 compiler->local_size = local_size;
1190
1191 if (options & SLJIT_ENTER_REG_ARG)
1192 arg_types = 0;
1193
1194 arg_types >>= SLJIT_ARG_SHIFT;
1195 word_arg_count = 0;
1196 saved_arg_count = 0;
1197 #ifdef __SOFTFP__
1198 SLJIT_COMPILE_ASSERT(SLJIT_FR0 == 1, float_register_index_start);
1199
1200 offset = 0;
1201 float_arg_count = 0;
1202
1203 while (arg_types) {
1204 switch (arg_types & SLJIT_ARG_MASK) {
1205 case SLJIT_ARG_TYPE_F64:
1206 if (offset & 0x7)
1207 offset += sizeof(sljit_sw);
1208
1209 if (offset < 4 * sizeof(sljit_sw))
1210 FAIL_IF(push_inst32(compiler, VMOV2 | (offset << 10) | ((offset + sizeof(sljit_sw)) << 14) | float_arg_count));
1211 else
1212 FAIL_IF(push_inst32(compiler, VLDR_F32 | 0x800100 | RN4(SLJIT_SP)
1213 | (float_arg_count << 12) | ((offset + (sljit_uw)size - 4 * sizeof(sljit_sw)) >> 2)));
1214 float_arg_count++;
1215 offset += sizeof(sljit_f64) - sizeof(sljit_sw);
1216 break;
1217 case SLJIT_ARG_TYPE_F32:
1218 if (offset < 4 * sizeof(sljit_sw))
1219 FAIL_IF(push_inst32(compiler, VMOV | (float_arg_count << 16) | (offset << 10)));
1220 else
1221 FAIL_IF(push_inst32(compiler, VLDR_F32 | 0x800000 | RN4(SLJIT_SP)
1222 | (float_arg_count << 12) | ((offset + (sljit_uw)size - 4 * sizeof(sljit_sw)) >> 2)));
1223 float_arg_count++;
1224 break;
1225 default:
1226 word_arg_count++;
1227
1228 if (!(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG)) {
1229 tmp = SLJIT_S0 - saved_arg_count;
1230 saved_arg_count++;
1231 } else if (word_arg_count - 1 != (sljit_s32)(offset >> 2))
1232 tmp = word_arg_count;
1233 else
1234 break;
1235
1236 if (offset < 4 * sizeof(sljit_sw))
1237 FAIL_IF(push_inst16(compiler, MOV | ((sljit_ins)reg_map[tmp] & 0x7) | (((sljit_ins)reg_map[tmp] & 0x8) << 4) | (offset << 1)));
1238 else if (reg_map[tmp] <= 7)
1239 FAIL_IF(push_inst16(compiler, LDR_SP | RDN3(tmp)
1240 | ((offset + (sljit_uw)size - 4 * sizeof(sljit_sw)) >> 2)));
1241 else
1242 FAIL_IF(push_inst32(compiler, LDR | RT4(tmp) | RN4(SLJIT_SP)
1243 | ((offset + (sljit_uw)size - 4 * sizeof(sljit_sw)))));
1244 break;
1245 }
1246
1247 offset += sizeof(sljit_sw);
1248 arg_types >>= SLJIT_ARG_SHIFT;
1249 }
1250
1251 compiler->args_size = offset;
1252 #else
1253 offset = SLJIT_FR0;
1254 old_offset = SLJIT_FR0;
1255 f32_offset = 0;
1256
1257 while (arg_types) {
1258 switch (arg_types & SLJIT_ARG_MASK) {
1259 case SLJIT_ARG_TYPE_F64:
1260 if (offset != old_offset)
1261 *remap_ptr++ = VMOV_F32 | SLJIT_32 | DD4(offset) | DM4(old_offset);
1262 old_offset++;
1263 offset++;
1264 break;
1265 case SLJIT_ARG_TYPE_F32:
1266 if (f32_offset != 0) {
1267 *remap_ptr++ = VMOV_F32 | 0x20 | DD4(offset) | DM4(f32_offset);
1268 f32_offset = 0;
1269 } else {
1270 if (offset != old_offset)
1271 *remap_ptr++ = VMOV_F32 | DD4(offset) | DM4(old_offset);
1272 f32_offset = old_offset;
1273 old_offset++;
1274 }
1275 offset++;
1276 break;
1277 default:
1278 if (!(arg_types & SLJIT_ARG_TYPE_SCRATCH_REG)) {
1279 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(SLJIT_S0 - saved_arg_count, SLJIT_R0 + word_arg_count)));
1280 saved_arg_count++;
1281 }
1282
1283 word_arg_count++;
1284 break;
1285 }
1286 arg_types >>= SLJIT_ARG_SHIFT;
1287 }
1288
1289 SLJIT_ASSERT((sljit_uw)(remap_ptr - remap) <= sizeof(remap));
1290
1291 while (remap_ptr > remap)
1292 FAIL_IF(push_inst32(compiler, *(--remap_ptr)));
1293 #endif
1294
1295 #ifdef _WIN32
1296 if (local_size >= 4096) {
1297 imm = get_imm(4096);
1298 SLJIT_ASSERT(imm != INVALID_IMM);
1299
1300 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(SLJIT_SP) | RN4(SLJIT_SP) | imm));
1301
1302 if (local_size < 4 * 4096) {
1303 if (local_size > 2 * 4096) {
1304 if (local_size > 3 * 4096) {
1305 FAIL_IF(push_inst32(compiler, LDRI | 0x400 | RT4(TMP_REG1) | RN4(SLJIT_SP)));
1306 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(SLJIT_SP) | RN4(SLJIT_SP) | imm));
1307 }
1308
1309 FAIL_IF(push_inst32(compiler, LDRI | 0x400 | RT4(TMP_REG1) | RN4(SLJIT_SP)));
1310 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(SLJIT_SP) | RN4(SLJIT_SP) | imm));
1311 }
1312 } else {
1313 FAIL_IF(load_immediate(compiler, TMP_REG2, ((sljit_uw)local_size >> 12) - 1));
1314 FAIL_IF(push_inst32(compiler, LDRI | 0x400 | RT4(TMP_REG1) | RN4(SLJIT_SP)));
1315 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(SLJIT_SP) | RN4(SLJIT_SP) | imm));
1316 FAIL_IF(push_inst32(compiler, SUB_WI | SET_FLAGS | RD4(TMP_REG2) | RN4(TMP_REG2) | 1));
1317 FAIL_IF(push_inst16(compiler, BCC | (0x1 << 8) /* not-equal */ | (-8 & 0xff)));
1318 }
1319
1320 FAIL_IF(push_inst32(compiler, LDRI | 0x400 | RT4(TMP_REG1) | RN4(SLJIT_SP)));
1321 local_size &= 0xfff;
1322 }
1323
1324 if (local_size >= 256) {
1325 SLJIT_ASSERT(local_size < 4096);
1326
1327 if (local_size <= (127 << 2))
1328 FAIL_IF(push_inst16(compiler, SUB_SP_I | ((sljit_uw)local_size >> 2)));
1329 else
1330 FAIL_IF(emit_op_imm(compiler, SLJIT_SUB | ARG2_IMM, SLJIT_SP, SLJIT_SP, (sljit_uw)local_size));
1331
1332 FAIL_IF(push_inst32(compiler, LDRI | 0x400 | RT4(TMP_REG1) | RN4(SLJIT_SP)));
1333 } else if (local_size > 0)
1334 FAIL_IF(push_inst32(compiler, LDRI | 0x500 | RT4(TMP_REG1) | RN4(SLJIT_SP) | (sljit_uw)local_size));
1335 #else /* !_WIN32 */
1336 if (local_size > 0) {
1337 if (local_size <= (127 << 2))
1338 FAIL_IF(push_inst16(compiler, SUB_SP_I | ((sljit_uw)local_size >> 2)));
1339 else
1340 FAIL_IF(emit_op_imm(compiler, SLJIT_SUB | ARG2_IMM, SLJIT_SP, SLJIT_SP, (sljit_uw)local_size));
1341 }
1342 #endif /* _WIN32 */
1343
1344 return SLJIT_SUCCESS;
1345 }
1346
sljit_set_context(struct sljit_compiler * compiler,sljit_s32 options,sljit_s32 arg_types,sljit_s32 scratches,sljit_s32 saveds,sljit_s32 fscratches,sljit_s32 fsaveds,sljit_s32 local_size)1347 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
1348 sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
1349 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
1350 {
1351 sljit_s32 size;
1352
1353 CHECK_ERROR();
1354 CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
1355 set_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
1356
1357 size = GET_SAVED_REGISTERS_SIZE(scratches, saveds - SLJIT_KEPT_SAVEDS_COUNT(options), 1);
1358
1359 if ((size & SSIZE_OF(sw)) != 0 && (fsaveds > 0 || fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG))
1360 size += SSIZE_OF(sw);
1361
1362 compiler->local_size = ((size + local_size + 0x7) & ~0x7) - size;
1363 return SLJIT_SUCCESS;
1364 }
1365
emit_add_sp(struct sljit_compiler * compiler,sljit_uw imm)1366 static sljit_s32 emit_add_sp(struct sljit_compiler *compiler, sljit_uw imm)
1367 {
1368 sljit_uw imm2;
1369
1370 /* The TMP_REG1 register must keep its value. */
1371 if (imm <= (127u << 2))
1372 return push_inst16(compiler, ADD_SP_I | (imm >> 2));
1373
1374 if (imm <= 0xfff)
1375 return push_inst32(compiler, ADDWI | RD4(SLJIT_SP) | RN4(SLJIT_SP) | IMM12(imm));
1376
1377 imm2 = get_imm(imm);
1378
1379 if (imm2 != INVALID_IMM)
1380 return push_inst32(compiler, ADD_WI | RD4(SLJIT_SP) | RN4(SLJIT_SP) | imm2);
1381
1382 FAIL_IF(load_immediate(compiler, TMP_REG2, imm));
1383 return push_inst16(compiler, ADD_SP | RN3(TMP_REG2));
1384 }
1385
emit_stack_frame_release(struct sljit_compiler * compiler,sljit_s32 frame_size)1386 static sljit_s32 emit_stack_frame_release(struct sljit_compiler *compiler, sljit_s32 frame_size)
1387 {
1388 sljit_s32 local_size, fscratches, fsaveds, i, tmp;
1389 sljit_s32 restored_reg = 0;
1390 sljit_s32 lr_dst = TMP_PC;
1391 sljit_uw reg_list = 0;
1392
1393 SLJIT_ASSERT(reg_map[TMP_REG2] == 14 && frame_size <= 128);
1394
1395 local_size = compiler->local_size;
1396 fscratches = compiler->fscratches;
1397 fsaveds = compiler->fsaveds;
1398
1399 if (fsaveds > 0 || fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG) {
1400 if (local_size > 0)
1401 FAIL_IF(emit_add_sp(compiler, (sljit_uw)local_size));
1402
1403 if (fsaveds + fscratches >= SLJIT_NUMBER_OF_FLOAT_REGISTERS) {
1404 FAIL_IF(push_inst32(compiler, VPOP | DD4(SLJIT_FS0) | ((sljit_uw)SLJIT_NUMBER_OF_SAVED_FLOAT_REGISTERS << 1)));
1405 } else {
1406 if (fscratches >= SLJIT_FIRST_SAVED_FLOAT_REG)
1407 FAIL_IF(push_inst32(compiler, VPOP | DD4(fscratches) | ((sljit_uw)(fscratches - (SLJIT_FIRST_SAVED_FLOAT_REG - 1)) << 1)));
1408 if (fsaveds > 0)
1409 FAIL_IF(push_inst32(compiler, VPOP | DD4(SLJIT_FS0) | ((sljit_uw)fsaveds << 1)));
1410 }
1411
1412 local_size = GET_SAVED_REGISTERS_SIZE(compiler->scratches, compiler->saveds, 1) & 0x7;
1413 }
1414
1415 if (frame_size < 0) {
1416 lr_dst = TMP_REG2;
1417 frame_size = 0;
1418 } else if (frame_size > 0) {
1419 SLJIT_ASSERT(frame_size == 1 || (frame_size & 0x7) == 0);
1420 lr_dst = 0;
1421 frame_size &= ~0x7;
1422 }
1423
1424 tmp = SLJIT_S0 - compiler->saveds;
1425 i = SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options);
1426 if (tmp < i) {
1427 restored_reg = i;
1428 do {
1429 reg_list |= (sljit_uw)1 << reg_map[i];
1430 } while (--i > tmp);
1431 }
1432
1433 i = compiler->scratches;
1434 if (i >= SLJIT_FIRST_SAVED_REG) {
1435 restored_reg = i;
1436 do {
1437 reg_list |= (sljit_uw)1 << reg_map[i];
1438 } while (--i >= SLJIT_FIRST_SAVED_REG);
1439 }
1440
1441 if (lr_dst == TMP_REG2 && reg_list == 0) {
1442 reg_list |= (sljit_uw)1 << reg_map[TMP_REG2];
1443 restored_reg = TMP_REG2;
1444 lr_dst = 0;
1445 }
1446
1447 if (lr_dst == 0 && (reg_list & (reg_list - 1)) == 0) {
1448 /* The local_size does not include the saved registers. */
1449 tmp = 0;
1450 if (reg_list != 0) {
1451 tmp = 2;
1452 if (local_size <= 0xfff) {
1453 if (local_size == 0) {
1454 SLJIT_ASSERT(restored_reg != TMP_REG2);
1455 if (frame_size == 0)
1456 return push_inst32(compiler, LDRI | RT4(restored_reg) | RN4(SLJIT_SP) | 0x308);
1457 if (frame_size > 2 * SSIZE_OF(sw))
1458 return push_inst32(compiler, LDRI | RT4(restored_reg) | RN4(SLJIT_SP) | 0x100 | (sljit_ins)(frame_size - (2 * SSIZE_OF(sw))));
1459 }
1460
1461 if (reg_map[restored_reg] <= 7 && local_size <= 0x3fc)
1462 FAIL_IF(push_inst16(compiler, STR_SP | 0x800 | RDN3(restored_reg) | (sljit_ins)(local_size >> 2)));
1463 else
1464 FAIL_IF(push_inst32(compiler, LDR | RT4(restored_reg) | RN4(SLJIT_SP) | (sljit_ins)local_size));
1465 tmp = 1;
1466 } else if (frame_size == 0) {
1467 frame_size = (restored_reg == TMP_REG2) ? SSIZE_OF(sw) : 2 * SSIZE_OF(sw);
1468 tmp = 3;
1469 }
1470
1471 /* Place for the saved register. */
1472 if (restored_reg != TMP_REG2)
1473 local_size += SSIZE_OF(sw);
1474 }
1475
1476 /* Place for the lr register. */
1477 local_size += SSIZE_OF(sw);
1478
1479 if (frame_size > local_size)
1480 FAIL_IF(push_inst16(compiler, SUB_SP_I | ((sljit_ins)(frame_size - local_size) >> 2)));
1481 else if (frame_size < local_size)
1482 FAIL_IF(emit_add_sp(compiler, (sljit_uw)(local_size - frame_size)));
1483
1484 if (tmp <= 1)
1485 return SLJIT_SUCCESS;
1486
1487 if (tmp == 2) {
1488 frame_size -= SSIZE_OF(sw);
1489 if (restored_reg != TMP_REG2)
1490 frame_size -= SSIZE_OF(sw);
1491
1492 if (reg_map[restored_reg] <= 7)
1493 return push_inst16(compiler, STR_SP | 0x800 | RDN3(restored_reg) | (sljit_ins)(frame_size >> 2));
1494
1495 return push_inst32(compiler, LDR | RT4(restored_reg) | RN4(SLJIT_SP) | (sljit_ins)frame_size);
1496 }
1497
1498 tmp = (restored_reg == TMP_REG2) ? 0x304 : 0x308;
1499 return push_inst32(compiler, LDRI | RT4(restored_reg) | RN4(SLJIT_SP) | (sljit_ins)tmp);
1500 }
1501
1502 if (local_size > 0)
1503 FAIL_IF(emit_add_sp(compiler, (sljit_uw)local_size));
1504
1505 if (!(reg_list & 0xff00) && lr_dst != TMP_REG2) {
1506 if (lr_dst == TMP_PC)
1507 reg_list |= 1u << 8;
1508
1509 /* At least one register must be set for POP instruction. */
1510 SLJIT_ASSERT(reg_list != 0);
1511
1512 FAIL_IF(push_inst16(compiler, POP | reg_list));
1513 } else {
1514 if (lr_dst != 0)
1515 reg_list |= (sljit_uw)1 << reg_map[lr_dst];
1516
1517 /* At least two registers must be set for POP_W instruction. */
1518 SLJIT_ASSERT((reg_list & (reg_list - 1)) != 0);
1519
1520 FAIL_IF(push_inst32(compiler, POP_W | reg_list));
1521 }
1522
1523 if (frame_size > 0)
1524 return push_inst16(compiler, SUB_SP_I | (((sljit_ins)frame_size - sizeof(sljit_sw)) >> 2));
1525
1526 if (lr_dst != 0)
1527 return SLJIT_SUCCESS;
1528
1529 return push_inst16(compiler, ADD_SP_I | 1);
1530 }
1531
sljit_emit_return_void(struct sljit_compiler * compiler)1532 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_void(struct sljit_compiler *compiler)
1533 {
1534 CHECK_ERROR();
1535 CHECK(check_sljit_emit_return_void(compiler));
1536
1537 return emit_stack_frame_release(compiler, 0);
1538 }
1539
sljit_emit_return_to(struct sljit_compiler * compiler,sljit_s32 src,sljit_sw srcw)1540 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_to(struct sljit_compiler *compiler,
1541 sljit_s32 src, sljit_sw srcw)
1542 {
1543 CHECK_ERROR();
1544 CHECK(check_sljit_emit_return_to(compiler, src, srcw));
1545
1546 if (src & SLJIT_MEM) {
1547 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, src, srcw, TMP_REG1));
1548 src = TMP_REG1;
1549 srcw = 0;
1550 } else if (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options))) {
1551 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(TMP_REG1, src)));
1552 src = TMP_REG1;
1553 srcw = 0;
1554 }
1555
1556 FAIL_IF(emit_stack_frame_release(compiler, 1));
1557
1558 SLJIT_SKIP_CHECKS(compiler);
1559 return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
1560 }
1561
1562 /* --------------------------------------------------------------------- */
1563 /* Operators */
1564 /* --------------------------------------------------------------------- */
1565
1566 #if !(defined __ARM_FEATURE_IDIV) && !(defined __ARM_ARCH_EXT_IDIV__)
1567
1568 #ifdef __cplusplus
1569 extern "C" {
1570 #endif
1571
1572 #ifdef _WIN32
1573 extern unsigned long long __rt_udiv(unsigned int denominator, unsigned int numerator);
1574 extern long long __rt_sdiv(int denominator, int numerator);
1575 #elif defined(__GNUC__)
1576 extern unsigned int __aeabi_uidivmod(unsigned int numerator, int unsigned denominator);
1577 extern int __aeabi_idivmod(int numerator, int denominator);
1578 #else
1579 #error "Software divmod functions are needed"
1580 #endif
1581
1582 #ifdef __cplusplus
1583 }
1584 #endif
1585
1586 #endif /* !__ARM_FEATURE_IDIV && !__ARM_ARCH_EXT_IDIV__ */
1587
sljit_emit_op0(struct sljit_compiler * compiler,sljit_s32 op)1588 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1589 {
1590 #if !(defined __ARM_FEATURE_IDIV) && !(defined __ARM_ARCH_EXT_IDIV__)
1591 sljit_uw saved_reg_list[3];
1592 sljit_uw saved_reg_count;
1593 #endif
1594
1595 CHECK_ERROR();
1596 CHECK(check_sljit_emit_op0(compiler, op));
1597
1598 op = GET_OPCODE(op);
1599 switch (op) {
1600 case SLJIT_BREAKPOINT:
1601 return push_inst16(compiler, BKPT);
1602 case SLJIT_NOP:
1603 return push_inst16(compiler, NOP);
1604 case SLJIT_LMUL_UW:
1605 case SLJIT_LMUL_SW:
1606 return push_inst32(compiler, (op == SLJIT_LMUL_UW ? UMULL : SMULL)
1607 | RD4(SLJIT_R1) | RT4(SLJIT_R0) | RN4(SLJIT_R0) | RM4(SLJIT_R1));
1608 #if (defined __ARM_FEATURE_IDIV) || (defined __ARM_ARCH_EXT_IDIV__)
1609 case SLJIT_DIVMOD_UW:
1610 case SLJIT_DIVMOD_SW:
1611 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(TMP_REG1, SLJIT_R0)));
1612 FAIL_IF(push_inst32(compiler, (op == SLJIT_DIVMOD_UW ? UDIV : SDIV) | RD4(SLJIT_R0) | RN4(SLJIT_R0) | RM4(SLJIT_R1)));
1613 FAIL_IF(push_inst32(compiler, MUL | RD4(SLJIT_R1) | RN4(SLJIT_R0) | RM4(SLJIT_R1)));
1614 return push_inst32(compiler, SUB_W | RD4(SLJIT_R1) | RN4(TMP_REG1) | RM4(SLJIT_R1));
1615 case SLJIT_DIV_UW:
1616 case SLJIT_DIV_SW:
1617 return push_inst32(compiler, (op == SLJIT_DIV_UW ? UDIV : SDIV) | RD4(SLJIT_R0) | RN4(SLJIT_R0) | RM4(SLJIT_R1));
1618 #else /* !__ARM_FEATURE_IDIV && !__ARM_ARCH_EXT_IDIV__ */
1619 case SLJIT_DIVMOD_UW:
1620 case SLJIT_DIVMOD_SW:
1621 case SLJIT_DIV_UW:
1622 case SLJIT_DIV_SW:
1623 SLJIT_COMPILE_ASSERT((SLJIT_DIVMOD_UW & 0x2) == 0 && SLJIT_DIV_UW - 0x2 == SLJIT_DIVMOD_UW, bad_div_opcode_assignments);
1624 SLJIT_ASSERT(reg_map[2] == 1 && reg_map[3] == 2 && reg_map[4] == 3);
1625
1626 saved_reg_count = 0;
1627 if (compiler->scratches >= 4)
1628 saved_reg_list[saved_reg_count++] = 3;
1629 if (compiler->scratches >= 3)
1630 saved_reg_list[saved_reg_count++] = 2;
1631 if (op >= SLJIT_DIV_UW)
1632 saved_reg_list[saved_reg_count++] = 1;
1633
1634 if (saved_reg_count > 0) {
1635 FAIL_IF(push_inst32(compiler, 0xf84d0d00 | (saved_reg_count >= 3 ? 16 : 8)
1636 | (saved_reg_list[0] << 12) /* str rX, [sp, #-8/-16]! */));
1637 if (saved_reg_count >= 2) {
1638 SLJIT_ASSERT(saved_reg_list[1] < 8);
1639 FAIL_IF(push_inst16(compiler, 0x9001 | (saved_reg_list[1] << 8) /* str rX, [sp, #4] */));
1640 }
1641 if (saved_reg_count >= 3) {
1642 SLJIT_ASSERT(saved_reg_list[2] < 8);
1643 FAIL_IF(push_inst16(compiler, 0x9002 | (saved_reg_list[2] << 8) /* str rX, [sp, #8] */));
1644 }
1645 }
1646
1647 #ifdef _WIN32
1648 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(TMP_REG1, SLJIT_R0)));
1649 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(SLJIT_R0, SLJIT_R1)));
1650 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(SLJIT_R1, TMP_REG1)));
1651 FAIL_IF(sljit_emit_ijump(compiler, SLJIT_FAST_CALL, SLJIT_IMM,
1652 ((op | 0x2) == SLJIT_DIV_UW ? SLJIT_FUNC_ADDR(__rt_udiv) : SLJIT_FUNC_ADDR(__rt_sdiv))));
1653 #elif defined(__GNUC__)
1654 FAIL_IF(sljit_emit_ijump(compiler, SLJIT_FAST_CALL, SLJIT_IMM,
1655 ((op | 0x2) == SLJIT_DIV_UW ? SLJIT_FUNC_ADDR(__aeabi_uidivmod) : SLJIT_FUNC_ADDR(__aeabi_idivmod))));
1656 #else
1657 #error "Software divmod functions are needed"
1658 #endif
1659
1660 if (saved_reg_count > 0) {
1661 if (saved_reg_count >= 3) {
1662 SLJIT_ASSERT(saved_reg_list[2] < 8);
1663 FAIL_IF(push_inst16(compiler, 0x9802 | (saved_reg_list[2] << 8) /* ldr rX, [sp, #8] */));
1664 }
1665 if (saved_reg_count >= 2) {
1666 SLJIT_ASSERT(saved_reg_list[1] < 8);
1667 FAIL_IF(push_inst16(compiler, 0x9801 | (saved_reg_list[1] << 8) /* ldr rX, [sp, #4] */));
1668 }
1669 return push_inst32(compiler, 0xf85d0b00 | (saved_reg_count >= 3 ? 16 : 8)
1670 | (saved_reg_list[0] << 12) /* ldr rX, [sp], #8/16 */);
1671 }
1672 return SLJIT_SUCCESS;
1673 #endif /* __ARM_FEATURE_IDIV || __ARM_ARCH_EXT_IDIV__ */
1674 case SLJIT_ENDBR:
1675 case SLJIT_SKIP_FRAMES_BEFORE_RETURN:
1676 return SLJIT_SUCCESS;
1677 }
1678
1679 return SLJIT_SUCCESS;
1680 }
1681
sljit_emit_op1(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)1682 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1683 sljit_s32 dst, sljit_sw dstw,
1684 sljit_s32 src, sljit_sw srcw)
1685 {
1686 sljit_s32 dst_r, flags;
1687 sljit_s32 op_flags = GET_ALL_FLAGS(op);
1688
1689 CHECK_ERROR();
1690 CHECK(check_sljit_emit_op1(compiler, op, dst, dstw, src, srcw));
1691 ADJUST_LOCAL_OFFSET(dst, dstw);
1692 ADJUST_LOCAL_OFFSET(src, srcw);
1693
1694 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
1695
1696 op = GET_OPCODE(op);
1697 if (op >= SLJIT_MOV && op <= SLJIT_MOV_P) {
1698 switch (op) {
1699 case SLJIT_MOV:
1700 case SLJIT_MOV_U32:
1701 case SLJIT_MOV_S32:
1702 case SLJIT_MOV32:
1703 case SLJIT_MOV_P:
1704 flags = WORD_SIZE;
1705 break;
1706 case SLJIT_MOV_U8:
1707 flags = BYTE_SIZE;
1708 if (src & SLJIT_IMM)
1709 srcw = (sljit_u8)srcw;
1710 break;
1711 case SLJIT_MOV_S8:
1712 flags = BYTE_SIZE | SIGNED;
1713 if (src & SLJIT_IMM)
1714 srcw = (sljit_s8)srcw;
1715 break;
1716 case SLJIT_MOV_U16:
1717 flags = HALF_SIZE;
1718 if (src & SLJIT_IMM)
1719 srcw = (sljit_u16)srcw;
1720 break;
1721 case SLJIT_MOV_S16:
1722 flags = HALF_SIZE | SIGNED;
1723 if (src & SLJIT_IMM)
1724 srcw = (sljit_s16)srcw;
1725 break;
1726 default:
1727 SLJIT_UNREACHABLE();
1728 flags = 0;
1729 break;
1730 }
1731
1732 if (src & SLJIT_IMM)
1733 FAIL_IF(emit_op_imm(compiler, SLJIT_MOV | ARG2_IMM, dst_r, TMP_REG2, (sljit_uw)srcw));
1734 else if (src & SLJIT_MEM) {
1735 FAIL_IF(emit_op_mem(compiler, flags, dst_r, src, srcw, TMP_REG1));
1736 } else {
1737 if (dst_r != TMP_REG1)
1738 return emit_op_imm(compiler, op, dst_r, TMP_REG2, (sljit_uw)src);
1739 dst_r = src;
1740 }
1741
1742 if (!(dst & SLJIT_MEM))
1743 return SLJIT_SUCCESS;
1744
1745 return emit_op_mem(compiler, flags | STORE, dst_r, dst, dstw, TMP_REG2);
1746 }
1747
1748 flags = HAS_FLAGS(op_flags) ? SET_FLAGS : 0;
1749
1750 if (src & SLJIT_MEM) {
1751 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, src, srcw, TMP_REG1));
1752 src = TMP_REG1;
1753 }
1754
1755 emit_op_imm(compiler, flags | op, dst_r, TMP_REG2, (sljit_uw)src);
1756
1757 if (SLJIT_UNLIKELY(dst & SLJIT_MEM))
1758 return emit_op_mem(compiler, flags | STORE, dst_r, dst, dstw, TMP_REG2);
1759 return SLJIT_SUCCESS;
1760 }
1761
sljit_emit_op2(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)1762 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op,
1763 sljit_s32 dst, sljit_sw dstw,
1764 sljit_s32 src1, sljit_sw src1w,
1765 sljit_s32 src2, sljit_sw src2w)
1766 {
1767 sljit_s32 dst_reg, flags, src2_reg;
1768
1769 CHECK_ERROR();
1770 CHECK(check_sljit_emit_op2(compiler, op, 0, dst, dstw, src1, src1w, src2, src2w));
1771 ADJUST_LOCAL_OFFSET(dst, dstw);
1772 ADJUST_LOCAL_OFFSET(src1, src1w);
1773 ADJUST_LOCAL_OFFSET(src2, src2w);
1774
1775 dst_reg = FAST_IS_REG(dst) ? dst : TMP_REG1;
1776 flags = HAS_FLAGS(op) ? SET_FLAGS : 0;
1777
1778 if (dst == TMP_REG1)
1779 flags |= UNUSED_RETURN;
1780
1781 if (src1 & SLJIT_IMM)
1782 flags |= ARG1_IMM;
1783 else if (src1 & SLJIT_MEM) {
1784 emit_op_mem(compiler, WORD_SIZE, TMP_REG1, src1, src1w, TMP_REG1);
1785 src1w = TMP_REG1;
1786 }
1787 else
1788 src1w = src1;
1789
1790 if (src2 & SLJIT_IMM)
1791 flags |= ARG2_IMM;
1792 else if (src2 & SLJIT_MEM) {
1793 src2_reg = (!(flags & ARG1_IMM) && (src1w == TMP_REG1)) ? TMP_REG2 : TMP_REG1;
1794 emit_op_mem(compiler, WORD_SIZE, src2_reg, src2, src2w, src2_reg);
1795 src2w = src2_reg;
1796 }
1797 else
1798 src2w = src2;
1799
1800 emit_op_imm(compiler, flags | GET_OPCODE(op), dst_reg, (sljit_uw)src1w, (sljit_uw)src2w);
1801
1802 if (!(dst & SLJIT_MEM))
1803 return SLJIT_SUCCESS;
1804 return emit_op_mem(compiler, WORD_SIZE | STORE, dst_reg, dst, dstw, TMP_REG2);
1805 }
1806
sljit_emit_op2u(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)1807 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op2u(struct sljit_compiler *compiler, sljit_s32 op,
1808 sljit_s32 src1, sljit_sw src1w,
1809 sljit_s32 src2, sljit_sw src2w)
1810 {
1811 CHECK_ERROR();
1812 CHECK(check_sljit_emit_op2(compiler, op, 1, 0, 0, src1, src1w, src2, src2w));
1813
1814 SLJIT_SKIP_CHECKS(compiler);
1815 return sljit_emit_op2(compiler, op, TMP_REG1, 0, src1, src1w, src2, src2w);
1816 }
1817
sljit_emit_shift_into(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src_dst,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)1818 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
1819 sljit_s32 src_dst,
1820 sljit_s32 src1, sljit_sw src1w,
1821 sljit_s32 src2, sljit_sw src2w)
1822 {
1823 sljit_s32 is_left;
1824
1825 CHECK_ERROR();
1826 CHECK(check_sljit_emit_shift_into(compiler, op, src_dst, src1, src1w, src2, src2w));
1827
1828 op = GET_OPCODE(op);
1829 is_left = (op == SLJIT_SHL || op == SLJIT_MSHL);
1830
1831 if (src_dst == src1) {
1832 SLJIT_SKIP_CHECKS(compiler);
1833 return sljit_emit_op2(compiler, is_left ? SLJIT_ROTL : SLJIT_ROTR, src_dst, 0, src_dst, 0, src2, src2w);
1834 }
1835
1836 ADJUST_LOCAL_OFFSET(src1, src1w);
1837 ADJUST_LOCAL_OFFSET(src2, src2w);
1838
1839 if (src2 & SLJIT_IMM) {
1840 src2w &= 0x1f;
1841
1842 if (src2w == 0)
1843 return SLJIT_SUCCESS;
1844 } else if (src2 & SLJIT_MEM) {
1845 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, src2, src2w, TMP_REG2));
1846 src2 = TMP_REG2;
1847 }
1848
1849 if (src1 & SLJIT_MEM) {
1850 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, src1, src1w, TMP_REG1));
1851 src1 = TMP_REG1;
1852 } else if (src1 & SLJIT_IMM) {
1853 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)src1w));
1854 src1 = TMP_REG1;
1855 }
1856
1857 if (src2 & SLJIT_IMM) {
1858 if (reg_map[src_dst] <= 7)
1859 FAIL_IF(push_inst16(compiler, (is_left ? LSLSI : LSRSI) | RD3(src_dst) | RN3(src_dst) | ((sljit_ins)src2w << 6)));
1860 else
1861 FAIL_IF(push_inst32(compiler, (is_left ? LSL_WI : LSR_WI) | RD4(src_dst) | RM4(src_dst) | IMM5(src2w)));
1862
1863 src2w = (src2w ^ 0x1f) + 1;
1864 return push_inst32(compiler, ORR_W | RD4(src_dst) | RN4(src_dst) | RM4(src1) | (is_left ? 0x10 : 0x0) | IMM5(src2w));
1865 }
1866
1867 if (op == SLJIT_MSHL || op == SLJIT_MLSHR) {
1868 FAIL_IF(push_inst32(compiler, ANDI | RD4(TMP_REG2) | RN4(src2) | 0x1f));
1869 src2 = TMP_REG2;
1870 }
1871
1872 if (IS_2_LO_REGS(src_dst, src2))
1873 FAIL_IF(push_inst16(compiler, (is_left ? LSLS : LSRS) | RD3(src_dst) | RN3(src2)));
1874 else
1875 FAIL_IF(push_inst32(compiler, (is_left ? LSL_W : LSR_W) | RD4(src_dst) | RN4(src_dst) | RM4(src2)));
1876
1877 FAIL_IF(push_inst32(compiler, (is_left ? LSR_WI : LSL_WI) | RD4(TMP_REG1) | RM4(src1) | (1 << 6)));
1878 FAIL_IF(push_inst32(compiler, EORI | RD4(TMP_REG2) | RN4(src2) | 0x1f));
1879 FAIL_IF(push_inst32(compiler, (is_left ? LSR_W : LSL_W) | RD4(TMP_REG1) | RN4(TMP_REG1) | RM4(TMP_REG2)));
1880 return push_inst32(compiler, ORR_W | RD4(src_dst) | RN4(src_dst) | RM4(TMP_REG1));
1881 }
1882
sljit_emit_op_src(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src,sljit_sw srcw)1883 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
1884 sljit_s32 src, sljit_sw srcw)
1885 {
1886 CHECK_ERROR();
1887 CHECK(check_sljit_emit_op_src(compiler, op, src, srcw));
1888 ADJUST_LOCAL_OFFSET(src, srcw);
1889
1890 switch (op) {
1891 case SLJIT_FAST_RETURN:
1892 SLJIT_ASSERT(reg_map[TMP_REG2] == 14);
1893
1894 if (FAST_IS_REG(src))
1895 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(TMP_REG2, src)));
1896 else
1897 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, src, srcw, TMP_REG2));
1898
1899 return push_inst16(compiler, BX | RN3(TMP_REG2));
1900 case SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN:
1901 return SLJIT_SUCCESS;
1902 case SLJIT_PREFETCH_L1:
1903 case SLJIT_PREFETCH_L2:
1904 case SLJIT_PREFETCH_L3:
1905 case SLJIT_PREFETCH_ONCE:
1906 return emit_op_mem(compiler, PRELOAD, TMP_PC, src, srcw, TMP_REG1);
1907 }
1908
1909 return SLJIT_SUCCESS;
1910 }
1911
sljit_get_register_index(sljit_s32 reg)1912 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_register_index(sljit_s32 reg)
1913 {
1914 CHECK_REG_INDEX(check_sljit_get_register_index(reg));
1915 return reg_map[reg];
1916 }
1917
sljit_get_float_register_index(sljit_s32 reg)1918 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_float_register_index(sljit_s32 reg)
1919 {
1920 CHECK_REG_INDEX(check_sljit_get_float_register_index(reg));
1921 return (freg_map[reg] << 1);
1922 }
1923
sljit_emit_op_custom(struct sljit_compiler * compiler,void * instruction,sljit_u32 size)1924 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_custom(struct sljit_compiler *compiler,
1925 void *instruction, sljit_u32 size)
1926 {
1927 CHECK_ERROR();
1928 CHECK(check_sljit_emit_op_custom(compiler, instruction, size));
1929
1930 if (size == 2)
1931 return push_inst16(compiler, *(sljit_u16*)instruction);
1932 return push_inst32(compiler, *(sljit_ins*)instruction);
1933 }
1934
1935 /* --------------------------------------------------------------------- */
1936 /* Floating point operators */
1937 /* --------------------------------------------------------------------- */
1938
1939 #define FPU_LOAD (1 << 20)
1940
emit_fop_mem(struct sljit_compiler * compiler,sljit_s32 flags,sljit_s32 reg,sljit_s32 arg,sljit_sw argw)1941 static sljit_s32 emit_fop_mem(struct sljit_compiler *compiler, sljit_s32 flags, sljit_s32 reg, sljit_s32 arg, sljit_sw argw)
1942 {
1943 sljit_uw imm;
1944 sljit_ins inst = VSTR_F32 | (flags & (SLJIT_32 | FPU_LOAD));
1945
1946 SLJIT_ASSERT(arg & SLJIT_MEM);
1947
1948 /* Fast loads and stores. */
1949 if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
1950 FAIL_IF(push_inst32(compiler, ADD_W | RD4(TMP_REG1) | RN4(arg & REG_MASK) | RM4(OFFS_REG(arg)) | (((sljit_uw)argw & 0x3) << 6)));
1951 arg = SLJIT_MEM | TMP_REG1;
1952 argw = 0;
1953 }
1954
1955 if ((arg & REG_MASK) && (argw & 0x3) == 0) {
1956 if (!(argw & ~0x3fc))
1957 return push_inst32(compiler, inst | 0x800000 | RN4(arg & REG_MASK) | DD4(reg) | ((sljit_uw)argw >> 2));
1958 if (!(-argw & ~0x3fc))
1959 return push_inst32(compiler, inst | RN4(arg & REG_MASK) | DD4(reg) | ((sljit_uw)-argw >> 2));
1960 }
1961
1962 if (arg & REG_MASK) {
1963 if (emit_set_delta(compiler, TMP_REG1, arg & REG_MASK, argw) != SLJIT_ERR_UNSUPPORTED) {
1964 FAIL_IF(compiler->error);
1965 return push_inst32(compiler, inst | 0x800000 | RN4(TMP_REG1) | DD4(reg));
1966 }
1967
1968 imm = get_imm((sljit_uw)argw & ~(sljit_uw)0x3fc);
1969 if (imm != INVALID_IMM) {
1970 FAIL_IF(push_inst32(compiler, ADD_WI | RD4(TMP_REG1) | RN4(arg & REG_MASK) | imm));
1971 return push_inst32(compiler, inst | 0x800000 | RN4(TMP_REG1) | DD4(reg) | (((sljit_uw)argw & 0x3fc) >> 2));
1972 }
1973
1974 imm = get_imm((sljit_uw)-argw & ~(sljit_uw)0x3fc);
1975 if (imm != INVALID_IMM) {
1976 argw = -argw;
1977 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(TMP_REG1) | RN4(arg & REG_MASK) | imm));
1978 return push_inst32(compiler, inst | RN4(TMP_REG1) | DD4(reg) | (((sljit_uw)argw & 0x3fc) >> 2));
1979 }
1980 }
1981
1982 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)argw));
1983 if (arg & REG_MASK)
1984 FAIL_IF(push_inst16(compiler, ADD | SET_REGS44(TMP_REG1, (arg & REG_MASK))));
1985 return push_inst32(compiler, inst | 0x800000 | RN4(TMP_REG1) | DD4(reg));
1986 }
1987
sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)1988 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
1989 sljit_s32 dst, sljit_sw dstw,
1990 sljit_s32 src, sljit_sw srcw)
1991 {
1992 op ^= SLJIT_32;
1993
1994 if (src & SLJIT_MEM) {
1995 FAIL_IF(emit_fop_mem(compiler, (op & SLJIT_32) | FPU_LOAD, TMP_FREG1, src, srcw));
1996 src = TMP_FREG1;
1997 }
1998
1999 FAIL_IF(push_inst32(compiler, VCVT_S32_F32 | (op & SLJIT_32) | DD4(TMP_FREG1) | DM4(src)));
2000
2001 if (FAST_IS_REG(dst))
2002 return push_inst32(compiler, VMOV | (1 << 20) | RT4(dst) | DN4(TMP_FREG1));
2003
2004 /* Store the integer value from a VFP register. */
2005 return emit_fop_mem(compiler, 0, TMP_FREG1, dst, dstw);
2006 }
2007
sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)2008 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_sw(struct sljit_compiler *compiler, sljit_s32 op,
2009 sljit_s32 dst, sljit_sw dstw,
2010 sljit_s32 src, sljit_sw srcw)
2011 {
2012 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2013
2014 op ^= SLJIT_32;
2015
2016 if (FAST_IS_REG(src))
2017 FAIL_IF(push_inst32(compiler, VMOV | RT4(src) | DN4(TMP_FREG1)));
2018 else if (src & SLJIT_MEM) {
2019 /* Load the integer value into a VFP register. */
2020 FAIL_IF(emit_fop_mem(compiler, FPU_LOAD, TMP_FREG1, src, srcw));
2021 }
2022 else {
2023 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)srcw));
2024 FAIL_IF(push_inst32(compiler, VMOV | RT4(TMP_REG1) | DN4(TMP_FREG1)));
2025 }
2026
2027 FAIL_IF(push_inst32(compiler, VCVT_F32_S32 | (op & SLJIT_32) | DD4(dst_r) | DM4(TMP_FREG1)));
2028
2029 if (dst & SLJIT_MEM)
2030 return emit_fop_mem(compiler, (op & SLJIT_32), TMP_FREG1, dst, dstw);
2031 return SLJIT_SUCCESS;
2032 }
2033
sljit_emit_fop1_cmp(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)2034 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
2035 sljit_s32 src1, sljit_sw src1w,
2036 sljit_s32 src2, sljit_sw src2w)
2037 {
2038 op ^= SLJIT_32;
2039
2040 if (src1 & SLJIT_MEM) {
2041 emit_fop_mem(compiler, (op & SLJIT_32) | FPU_LOAD, TMP_FREG1, src1, src1w);
2042 src1 = TMP_FREG1;
2043 }
2044
2045 if (src2 & SLJIT_MEM) {
2046 emit_fop_mem(compiler, (op & SLJIT_32) | FPU_LOAD, TMP_FREG2, src2, src2w);
2047 src2 = TMP_FREG2;
2048 }
2049
2050 FAIL_IF(push_inst32(compiler, VCMP_F32 | (op & SLJIT_32) | DD4(src1) | DM4(src2)));
2051 return push_inst32(compiler, VMRS);
2052 }
2053
sljit_emit_fop1(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)2054 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
2055 sljit_s32 dst, sljit_sw dstw,
2056 sljit_s32 src, sljit_sw srcw)
2057 {
2058 sljit_s32 dst_r;
2059
2060 CHECK_ERROR();
2061
2062 SLJIT_COMPILE_ASSERT((SLJIT_32 == 0x100), float_transfer_bit_error);
2063 SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw);
2064
2065 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2066
2067 if (GET_OPCODE(op) != SLJIT_CONV_F64_FROM_F32)
2068 op ^= SLJIT_32;
2069
2070 if (src & SLJIT_MEM) {
2071 emit_fop_mem(compiler, (op & SLJIT_32) | FPU_LOAD, dst_r, src, srcw);
2072 src = dst_r;
2073 }
2074
2075 switch (GET_OPCODE(op)) {
2076 case SLJIT_MOV_F64:
2077 if (src != dst_r) {
2078 if (dst_r != TMP_FREG1)
2079 FAIL_IF(push_inst32(compiler, VMOV_F32 | (op & SLJIT_32) | DD4(dst_r) | DM4(src)));
2080 else
2081 dst_r = src;
2082 }
2083 break;
2084 case SLJIT_NEG_F64:
2085 FAIL_IF(push_inst32(compiler, VNEG_F32 | (op & SLJIT_32) | DD4(dst_r) | DM4(src)));
2086 break;
2087 case SLJIT_ABS_F64:
2088 FAIL_IF(push_inst32(compiler, VABS_F32 | (op & SLJIT_32) | DD4(dst_r) | DM4(src)));
2089 break;
2090 case SLJIT_CONV_F64_FROM_F32:
2091 FAIL_IF(push_inst32(compiler, VCVT_F64_F32 | (op & SLJIT_32) | DD4(dst_r) | DM4(src)));
2092 op ^= SLJIT_32;
2093 break;
2094 }
2095
2096 if (dst & SLJIT_MEM)
2097 return emit_fop_mem(compiler, (op & SLJIT_32), dst_r, dst, dstw);
2098 return SLJIT_SUCCESS;
2099 }
2100
sljit_emit_fop2(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)2101 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
2102 sljit_s32 dst, sljit_sw dstw,
2103 sljit_s32 src1, sljit_sw src1w,
2104 sljit_s32 src2, sljit_sw src2w)
2105 {
2106 sljit_s32 dst_r;
2107
2108 CHECK_ERROR();
2109 CHECK(check_sljit_emit_fop2(compiler, op, dst, dstw, src1, src1w, src2, src2w));
2110 ADJUST_LOCAL_OFFSET(dst, dstw);
2111 ADJUST_LOCAL_OFFSET(src1, src1w);
2112 ADJUST_LOCAL_OFFSET(src2, src2w);
2113
2114 op ^= SLJIT_32;
2115
2116 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG1;
2117 if (src1 & SLJIT_MEM) {
2118 emit_fop_mem(compiler, (op & SLJIT_32) | FPU_LOAD, TMP_FREG1, src1, src1w);
2119 src1 = TMP_FREG1;
2120 }
2121 if (src2 & SLJIT_MEM) {
2122 emit_fop_mem(compiler, (op & SLJIT_32) | FPU_LOAD, TMP_FREG2, src2, src2w);
2123 src2 = TMP_FREG2;
2124 }
2125
2126 switch (GET_OPCODE(op)) {
2127 case SLJIT_ADD_F64:
2128 FAIL_IF(push_inst32(compiler, VADD_F32 | (op & SLJIT_32) | DD4(dst_r) | DN4(src1) | DM4(src2)));
2129 break;
2130 case SLJIT_SUB_F64:
2131 FAIL_IF(push_inst32(compiler, VSUB_F32 | (op & SLJIT_32) | DD4(dst_r) | DN4(src1) | DM4(src2)));
2132 break;
2133 case SLJIT_MUL_F64:
2134 FAIL_IF(push_inst32(compiler, VMUL_F32 | (op & SLJIT_32) | DD4(dst_r) | DN4(src1) | DM4(src2)));
2135 break;
2136 case SLJIT_DIV_F64:
2137 FAIL_IF(push_inst32(compiler, VDIV_F32 | (op & SLJIT_32) | DD4(dst_r) | DN4(src1) | DM4(src2)));
2138 break;
2139 }
2140
2141 if (!(dst & SLJIT_MEM))
2142 return SLJIT_SUCCESS;
2143 return emit_fop_mem(compiler, (op & SLJIT_32), TMP_FREG1, dst, dstw);
2144 }
2145
2146 /* --------------------------------------------------------------------- */
2147 /* Other instructions */
2148 /* --------------------------------------------------------------------- */
2149
sljit_emit_fast_enter(struct sljit_compiler * compiler,sljit_s32 dst,sljit_sw dstw)2150 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
2151 {
2152 CHECK_ERROR();
2153 CHECK(check_sljit_emit_fast_enter(compiler, dst, dstw));
2154 ADJUST_LOCAL_OFFSET(dst, dstw);
2155
2156 SLJIT_ASSERT(reg_map[TMP_REG2] == 14);
2157
2158 if (FAST_IS_REG(dst))
2159 return push_inst16(compiler, MOV | SET_REGS44(dst, TMP_REG2));
2160
2161 /* Memory. */
2162 return emit_op_mem(compiler, WORD_SIZE | STORE, TMP_REG2, dst, dstw, TMP_REG1);
2163 }
2164
2165 /* --------------------------------------------------------------------- */
2166 /* Conditional instructions */
2167 /* --------------------------------------------------------------------- */
2168
get_cc(struct sljit_compiler * compiler,sljit_s32 type)2169 static sljit_uw get_cc(struct sljit_compiler *compiler, sljit_s32 type)
2170 {
2171 switch (type) {
2172 case SLJIT_EQUAL:
2173 case SLJIT_F_EQUAL:
2174 case SLJIT_ORDERED_EQUAL:
2175 case SLJIT_UNORDERED_OR_EQUAL: /* Not supported. */
2176 return 0x0;
2177
2178 case SLJIT_NOT_EQUAL:
2179 case SLJIT_F_NOT_EQUAL:
2180 case SLJIT_UNORDERED_OR_NOT_EQUAL:
2181 case SLJIT_ORDERED_NOT_EQUAL: /* Not supported. */
2182 return 0x1;
2183
2184 case SLJIT_CARRY:
2185 if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_ADD)
2186 return 0x2;
2187 /* fallthrough */
2188
2189 case SLJIT_LESS:
2190 return 0x3;
2191
2192 case SLJIT_NOT_CARRY:
2193 if (compiler->status_flags_state & SLJIT_CURRENT_FLAGS_ADD)
2194 return 0x3;
2195 /* fallthrough */
2196
2197 case SLJIT_GREATER_EQUAL:
2198 return 0x2;
2199
2200 case SLJIT_GREATER:
2201 case SLJIT_UNORDERED_OR_GREATER:
2202 return 0x8;
2203
2204 case SLJIT_LESS_EQUAL:
2205 case SLJIT_F_LESS_EQUAL:
2206 case SLJIT_ORDERED_LESS_EQUAL:
2207 return 0x9;
2208
2209 case SLJIT_SIG_LESS:
2210 case SLJIT_UNORDERED_OR_LESS:
2211 return 0xb;
2212
2213 case SLJIT_SIG_GREATER_EQUAL:
2214 case SLJIT_F_GREATER_EQUAL:
2215 case SLJIT_ORDERED_GREATER_EQUAL:
2216 return 0xa;
2217
2218 case SLJIT_SIG_GREATER:
2219 case SLJIT_F_GREATER:
2220 case SLJIT_ORDERED_GREATER:
2221 return 0xc;
2222
2223 case SLJIT_SIG_LESS_EQUAL:
2224 case SLJIT_UNORDERED_OR_LESS_EQUAL:
2225 return 0xd;
2226
2227 case SLJIT_OVERFLOW:
2228 if (!(compiler->status_flags_state & (SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB)))
2229 return 0x1;
2230 /* fallthrough */
2231
2232 case SLJIT_UNORDERED:
2233 return 0x6;
2234
2235 case SLJIT_NOT_OVERFLOW:
2236 if (!(compiler->status_flags_state & (SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB)))
2237 return 0x0;
2238 /* fallthrough */
2239
2240 case SLJIT_ORDERED:
2241 return 0x7;
2242
2243 case SLJIT_F_LESS:
2244 case SLJIT_ORDERED_LESS:
2245 return 0x4;
2246
2247 case SLJIT_UNORDERED_OR_GREATER_EQUAL:
2248 return 0x5;
2249
2250 default: /* SLJIT_JUMP */
2251 SLJIT_UNREACHABLE();
2252 return 0xe;
2253 }
2254 }
2255
sljit_emit_label(struct sljit_compiler * compiler)2256 SLJIT_API_FUNC_ATTRIBUTE struct sljit_label* sljit_emit_label(struct sljit_compiler *compiler)
2257 {
2258 struct sljit_label *label;
2259
2260 CHECK_ERROR_PTR();
2261 CHECK_PTR(check_sljit_emit_label(compiler));
2262
2263 if (compiler->last_label && compiler->last_label->size == compiler->size)
2264 return compiler->last_label;
2265
2266 label = (struct sljit_label*)ensure_abuf(compiler, sizeof(struct sljit_label));
2267 PTR_FAIL_IF(!label);
2268 set_label(label, compiler);
2269 return label;
2270 }
2271
sljit_emit_jump(struct sljit_compiler * compiler,sljit_s32 type)2272 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2273 {
2274 struct sljit_jump *jump;
2275 sljit_ins cc;
2276
2277 CHECK_ERROR_PTR();
2278 CHECK_PTR(check_sljit_emit_jump(compiler, type));
2279
2280 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2281 PTR_FAIL_IF(!jump);
2282 set_jump(jump, compiler, type & SLJIT_REWRITABLE_JUMP);
2283 type &= 0xff;
2284
2285 PTR_FAIL_IF(emit_imm32_const(compiler, TMP_REG1, 0));
2286 if (type < SLJIT_JUMP) {
2287 jump->flags |= IS_COND;
2288 cc = get_cc(compiler, type);
2289 jump->flags |= cc << 8;
2290 PTR_FAIL_IF(push_inst16(compiler, IT | (cc << 4) | 0x8));
2291 }
2292
2293 jump->addr = compiler->size;
2294 if (type <= SLJIT_JUMP)
2295 PTR_FAIL_IF(push_inst16(compiler, BX | RN3(TMP_REG1)));
2296 else {
2297 jump->flags |= IS_BL;
2298 PTR_FAIL_IF(push_inst16(compiler, BLX | RN3(TMP_REG1)));
2299 }
2300
2301 return jump;
2302 }
2303
2304 #ifdef __SOFTFP__
2305
softfloat_call_with_args(struct sljit_compiler * compiler,sljit_s32 arg_types,sljit_s32 * src,sljit_u32 * extra_space)2306 static sljit_s32 softfloat_call_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types, sljit_s32 *src, sljit_u32 *extra_space)
2307 {
2308 sljit_u32 is_tail_call = *extra_space & SLJIT_CALL_RETURN;
2309 sljit_u32 offset = 0;
2310 sljit_u32 word_arg_offset = 0;
2311 sljit_u32 float_arg_count = 0;
2312 sljit_s32 types = 0;
2313 sljit_u32 src_offset = 4 * sizeof(sljit_sw);
2314 sljit_u8 offsets[4];
2315 sljit_u8 *offset_ptr = offsets;
2316
2317 if (src && FAST_IS_REG(*src))
2318 src_offset = (sljit_u32)reg_map[*src] * sizeof(sljit_sw);
2319
2320 arg_types >>= SLJIT_ARG_SHIFT;
2321
2322 while (arg_types) {
2323 types = (types << SLJIT_ARG_SHIFT) | (arg_types & SLJIT_ARG_MASK);
2324
2325 switch (arg_types & SLJIT_ARG_MASK) {
2326 case SLJIT_ARG_TYPE_F64:
2327 if (offset & 0x7)
2328 offset += sizeof(sljit_sw);
2329 *offset_ptr++ = (sljit_u8)offset;
2330 offset += sizeof(sljit_f64);
2331 float_arg_count++;
2332 break;
2333 case SLJIT_ARG_TYPE_F32:
2334 *offset_ptr++ = (sljit_u8)offset;
2335 offset += sizeof(sljit_f32);
2336 float_arg_count++;
2337 break;
2338 default:
2339 *offset_ptr++ = (sljit_u8)offset;
2340 offset += sizeof(sljit_sw);
2341 word_arg_offset += sizeof(sljit_sw);
2342 break;
2343 }
2344
2345 arg_types >>= SLJIT_ARG_SHIFT;
2346 }
2347
2348 if (offset > 4 * sizeof(sljit_sw) && (!is_tail_call || offset > compiler->args_size)) {
2349 /* Keep lr register on the stack. */
2350 if (is_tail_call)
2351 offset += sizeof(sljit_sw);
2352
2353 offset = ((offset - 4 * sizeof(sljit_sw)) + 0x7) & ~(sljit_uw)0x7;
2354
2355 *extra_space = offset;
2356
2357 if (is_tail_call)
2358 FAIL_IF(emit_stack_frame_release(compiler, (sljit_s32)offset));
2359 else
2360 FAIL_IF(push_inst16(compiler, SUB_SP_I | (offset >> 2)));
2361 } else {
2362 if (is_tail_call)
2363 FAIL_IF(emit_stack_frame_release(compiler, -1));
2364 *extra_space = 0;
2365 }
2366
2367 SLJIT_ASSERT(reg_map[TMP_REG1] == 12);
2368
2369 /* Process arguments in reversed direction. */
2370 while (types) {
2371 switch (types & SLJIT_ARG_MASK) {
2372 case SLJIT_ARG_TYPE_F64:
2373 float_arg_count--;
2374 offset = *(--offset_ptr);
2375
2376 SLJIT_ASSERT((offset & 0x7) == 0);
2377
2378 if (offset < 4 * sizeof(sljit_sw)) {
2379 if (src_offset == offset || src_offset == offset + sizeof(sljit_sw)) {
2380 FAIL_IF(push_inst16(compiler, MOV | (src_offset << 1) | 4 | (1 << 7)));
2381 *src = TMP_REG1;
2382 }
2383 FAIL_IF(push_inst32(compiler, VMOV2 | 0x100000 | (offset << 10) | ((offset + sizeof(sljit_sw)) << 14) | float_arg_count));
2384 } else
2385 FAIL_IF(push_inst32(compiler, VSTR_F32 | 0x800100 | RN4(SLJIT_SP)
2386 | (float_arg_count << 12) | ((offset - 4 * sizeof(sljit_sw)) >> 2)));
2387 break;
2388 case SLJIT_ARG_TYPE_F32:
2389 float_arg_count--;
2390 offset = *(--offset_ptr);
2391
2392 if (offset < 4 * sizeof(sljit_sw)) {
2393 if (src_offset == offset) {
2394 FAIL_IF(push_inst16(compiler, MOV | (src_offset << 1) | 4 | (1 << 7)));
2395 *src = TMP_REG1;
2396 }
2397 FAIL_IF(push_inst32(compiler, VMOV | 0x100000 | (float_arg_count << 16) | (offset << 10)));
2398 } else
2399 FAIL_IF(push_inst32(compiler, VSTR_F32 | 0x800000 | RN4(SLJIT_SP)
2400 | (float_arg_count << 12) | ((offset - 4 * sizeof(sljit_sw)) >> 2)));
2401 break;
2402 default:
2403 word_arg_offset -= sizeof(sljit_sw);
2404 offset = *(--offset_ptr);
2405
2406 SLJIT_ASSERT(offset >= word_arg_offset);
2407
2408 if (offset != word_arg_offset) {
2409 if (offset < 4 * sizeof(sljit_sw)) {
2410 if (src_offset == offset) {
2411 FAIL_IF(push_inst16(compiler, MOV | (src_offset << 1) | 4 | (1 << 7)));
2412 *src = TMP_REG1;
2413 }
2414 else if (src_offset == word_arg_offset) {
2415 *src = (sljit_s32)(1 + (offset >> 2));
2416 src_offset = offset;
2417 }
2418 FAIL_IF(push_inst16(compiler, MOV | (offset >> 2) | (word_arg_offset << 1)));
2419 } else
2420 FAIL_IF(push_inst16(compiler, STR_SP | (word_arg_offset << 6) | ((offset - 4 * sizeof(sljit_sw)) >> 2)));
2421 }
2422 break;
2423 }
2424
2425 types >>= SLJIT_ARG_SHIFT;
2426 }
2427
2428 return SLJIT_SUCCESS;
2429 }
2430
softfloat_post_call_with_args(struct sljit_compiler * compiler,sljit_s32 arg_types)2431 static sljit_s32 softfloat_post_call_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types)
2432 {
2433 if ((arg_types & SLJIT_ARG_MASK) == SLJIT_ARG_TYPE_F64)
2434 FAIL_IF(push_inst32(compiler, VMOV2 | (1 << 16) | (0 << 12) | 0));
2435 if ((arg_types & SLJIT_ARG_MASK) == SLJIT_ARG_TYPE_F32)
2436 FAIL_IF(push_inst32(compiler, VMOV | (0 << 16) | (0 << 12)));
2437
2438 return SLJIT_SUCCESS;
2439 }
2440
2441 #else
2442
hardfloat_call_with_args(struct sljit_compiler * compiler,sljit_s32 arg_types)2443 static sljit_s32 hardfloat_call_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types)
2444 {
2445 sljit_u32 offset = SLJIT_FR0;
2446 sljit_u32 new_offset = SLJIT_FR0;
2447 sljit_u32 f32_offset = 0;
2448
2449 /* Remove return value. */
2450 arg_types >>= SLJIT_ARG_SHIFT;
2451
2452 while (arg_types) {
2453 switch (arg_types & SLJIT_ARG_MASK) {
2454 case SLJIT_ARG_TYPE_F64:
2455 if (offset != new_offset)
2456 FAIL_IF(push_inst32(compiler, VMOV_F32 | SLJIT_32 | DD4(new_offset) | DM4(offset)));
2457
2458 new_offset++;
2459 offset++;
2460 break;
2461 case SLJIT_ARG_TYPE_F32:
2462 if (f32_offset != 0) {
2463 FAIL_IF(push_inst32(compiler, VMOV_F32 | 0x400000 | DD4(f32_offset) | DM4(offset)));
2464 f32_offset = 0;
2465 } else {
2466 if (offset != new_offset)
2467 FAIL_IF(push_inst32(compiler, VMOV_F32 | 0x400000 | DD4(new_offset) | DM4(offset)));
2468 f32_offset = new_offset;
2469 new_offset++;
2470 }
2471 offset++;
2472 break;
2473 }
2474 arg_types >>= SLJIT_ARG_SHIFT;
2475 }
2476
2477 return SLJIT_SUCCESS;
2478 }
2479
2480 #endif
2481
sljit_emit_call(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 arg_types)2482 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
2483 sljit_s32 arg_types)
2484 {
2485 #ifdef __SOFTFP__
2486 struct sljit_jump *jump;
2487 sljit_u32 extra_space = (sljit_u32)type;
2488 #endif
2489
2490 CHECK_ERROR_PTR();
2491 CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
2492
2493 #ifdef __SOFTFP__
2494 if ((type & 0xff) != SLJIT_CALL_REG_ARG) {
2495 PTR_FAIL_IF(softfloat_call_with_args(compiler, arg_types, NULL, &extra_space));
2496 SLJIT_ASSERT((extra_space & 0x7) == 0);
2497
2498 if ((type & SLJIT_CALL_RETURN) && extra_space == 0)
2499 type = SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP);
2500
2501 SLJIT_SKIP_CHECKS(compiler);
2502 jump = sljit_emit_jump(compiler, type);
2503 PTR_FAIL_IF(jump == NULL);
2504
2505 if (extra_space > 0) {
2506 if (type & SLJIT_CALL_RETURN)
2507 PTR_FAIL_IF(push_inst32(compiler, LDR | RT4(TMP_REG2)
2508 | RN4(SLJIT_SP) | (extra_space - sizeof(sljit_sw))));
2509
2510 PTR_FAIL_IF(push_inst16(compiler, ADD_SP_I | (extra_space >> 2)));
2511
2512 if (type & SLJIT_CALL_RETURN) {
2513 PTR_FAIL_IF(push_inst16(compiler, BX | RN3(TMP_REG2)));
2514 return jump;
2515 }
2516 }
2517
2518 SLJIT_ASSERT(!(type & SLJIT_CALL_RETURN));
2519 PTR_FAIL_IF(softfloat_post_call_with_args(compiler, arg_types));
2520 return jump;
2521 }
2522 #endif /* __SOFTFP__ */
2523
2524 if (type & SLJIT_CALL_RETURN) {
2525 /* ldmia sp!, {..., lr} */
2526 PTR_FAIL_IF(emit_stack_frame_release(compiler, -1));
2527 type = SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP);
2528 }
2529
2530 #ifndef __SOFTFP__
2531 if ((type & 0xff) != SLJIT_CALL_REG_ARG)
2532 PTR_FAIL_IF(hardfloat_call_with_args(compiler, arg_types));
2533 #endif /* !__SOFTFP__ */
2534
2535 SLJIT_SKIP_CHECKS(compiler);
2536 return sljit_emit_jump(compiler, type);
2537 }
2538
sljit_emit_ijump(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 src,sljit_sw srcw)2539 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type, sljit_s32 src, sljit_sw srcw)
2540 {
2541 struct sljit_jump *jump;
2542
2543 CHECK_ERROR();
2544 CHECK(check_sljit_emit_ijump(compiler, type, src, srcw));
2545 ADJUST_LOCAL_OFFSET(src, srcw);
2546
2547 SLJIT_ASSERT(reg_map[TMP_REG1] != 14);
2548
2549 if (!(src & SLJIT_IMM)) {
2550 if (FAST_IS_REG(src)) {
2551 SLJIT_ASSERT(reg_map[src] != 14);
2552 return push_inst16(compiler, (type <= SLJIT_JUMP ? BX : BLX) | RN3(src));
2553 }
2554
2555 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, type <= SLJIT_JUMP ? TMP_PC : TMP_REG1, src, srcw, TMP_REG1));
2556 if (type >= SLJIT_FAST_CALL)
2557 return push_inst16(compiler, BLX | RN3(TMP_REG1));
2558 }
2559
2560 /* These jumps are converted to jump/call instructions when possible. */
2561 jump = (struct sljit_jump*)ensure_abuf(compiler, sizeof(struct sljit_jump));
2562 FAIL_IF(!jump);
2563 set_jump(jump, compiler, JUMP_ADDR | ((type >= SLJIT_FAST_CALL) ? IS_BL : 0));
2564 jump->u.target = (sljit_uw)srcw;
2565
2566 FAIL_IF(emit_imm32_const(compiler, TMP_REG1, 0));
2567 jump->addr = compiler->size;
2568 return push_inst16(compiler, (type <= SLJIT_JUMP ? BX : BLX) | RN3(TMP_REG1));
2569 }
2570
sljit_emit_icall(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 arg_types,sljit_s32 src,sljit_sw srcw)2571 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
2572 sljit_s32 arg_types,
2573 sljit_s32 src, sljit_sw srcw)
2574 {
2575 #ifdef __SOFTFP__
2576 sljit_u32 extra_space = (sljit_u32)type;
2577 #endif
2578
2579 CHECK_ERROR();
2580 CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
2581
2582 if (src & SLJIT_MEM) {
2583 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, src, srcw, TMP_REG1));
2584 src = TMP_REG1;
2585 }
2586
2587 if ((type & SLJIT_CALL_RETURN) && (src >= SLJIT_FIRST_SAVED_REG && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options)))) {
2588 FAIL_IF(push_inst16(compiler, MOV | SET_REGS44(TMP_REG1, src)));
2589 src = TMP_REG1;
2590 }
2591
2592 #ifdef __SOFTFP__
2593 if ((type & 0xff) != SLJIT_CALL_REG_ARG) {
2594 FAIL_IF(softfloat_call_with_args(compiler, arg_types, &src, &extra_space));
2595 SLJIT_ASSERT((extra_space & 0x7) == 0);
2596
2597 if ((type & SLJIT_CALL_RETURN) && extra_space == 0)
2598 type = SLJIT_JUMP;
2599
2600 SLJIT_SKIP_CHECKS(compiler);
2601 FAIL_IF(sljit_emit_ijump(compiler, type, src, srcw));
2602
2603 if (extra_space > 0) {
2604 if (type & SLJIT_CALL_RETURN)
2605 FAIL_IF(push_inst32(compiler, LDR | RT4(TMP_REG2)
2606 | RN4(SLJIT_SP) | (extra_space - sizeof(sljit_sw))));
2607
2608 FAIL_IF(push_inst16(compiler, ADD_SP_I | (extra_space >> 2)));
2609
2610 if (type & SLJIT_CALL_RETURN)
2611 return push_inst16(compiler, BX | RN3(TMP_REG2));
2612 }
2613
2614 SLJIT_ASSERT(!(type & SLJIT_CALL_RETURN));
2615 return softfloat_post_call_with_args(compiler, arg_types);
2616 }
2617 #endif /* __SOFTFP__ */
2618
2619 if (type & SLJIT_CALL_RETURN) {
2620 /* ldmia sp!, {..., lr} */
2621 FAIL_IF(emit_stack_frame_release(compiler, -1));
2622 type = SLJIT_JUMP;
2623 }
2624
2625 #ifndef __SOFTFP__
2626 if ((type & 0xff) != SLJIT_CALL_REG_ARG)
2627 FAIL_IF(hardfloat_call_with_args(compiler, arg_types));
2628 #endif /* !__SOFTFP__ */
2629
2630 SLJIT_SKIP_CHECKS(compiler);
2631 return sljit_emit_ijump(compiler, type, src, srcw);
2632 }
2633
2634 #ifdef __SOFTFP__
2635
emit_fmov_before_return(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src,sljit_sw srcw)2636 static SLJIT_INLINE sljit_s32 emit_fmov_before_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
2637 {
2638 if (compiler->options & SLJIT_ENTER_REG_ARG) {
2639 if (src == SLJIT_FR0)
2640 return SLJIT_SUCCESS;
2641
2642 SLJIT_SKIP_CHECKS(compiler);
2643 return sljit_emit_fop1(compiler, op, SLJIT_RETURN_FREG, 0, src, srcw);
2644 }
2645
2646 if (FAST_IS_REG(src)) {
2647 if (op & SLJIT_32)
2648 return push_inst32(compiler, VMOV | (1 << 20) | DN4(src) | RT4(SLJIT_R0));
2649 return push_inst32(compiler, VMOV2 | (1 << 20) | DM4(src) | RT4(SLJIT_R0) | RN4(SLJIT_R1));
2650 }
2651
2652 SLJIT_SKIP_CHECKS(compiler);
2653
2654 if (op & SLJIT_32)
2655 return sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R0, 0, src, srcw);
2656 return sljit_emit_mem(compiler, SLJIT_MOV, SLJIT_REG_PAIR(SLJIT_R0, SLJIT_R1), src, srcw);
2657 }
2658
2659 #endif /* __SOFTFP__ */
2660
sljit_emit_op_flags(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 type)2661 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
2662 sljit_s32 dst, sljit_sw dstw,
2663 sljit_s32 type)
2664 {
2665 sljit_s32 dst_r, flags = GET_ALL_FLAGS(op);
2666 sljit_ins cc;
2667
2668 CHECK_ERROR();
2669 CHECK(check_sljit_emit_op_flags(compiler, op, dst, dstw, type));
2670 ADJUST_LOCAL_OFFSET(dst, dstw);
2671
2672 op = GET_OPCODE(op);
2673 cc = get_cc(compiler, type);
2674 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
2675
2676 if (op < SLJIT_ADD) {
2677 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | (((cc & 0x1) ^ 0x1) << 3) | 0x4));
2678 if (reg_map[dst_r] > 7) {
2679 FAIL_IF(push_inst32(compiler, MOV_WI | RD4(dst_r) | 1));
2680 FAIL_IF(push_inst32(compiler, MOV_WI | RD4(dst_r) | 0));
2681 } else {
2682 /* The movsi (immediate) instruction does not set flags in IT block. */
2683 FAIL_IF(push_inst16(compiler, MOVSI | RDN3(dst_r) | 1));
2684 FAIL_IF(push_inst16(compiler, MOVSI | RDN3(dst_r) | 0));
2685 }
2686 if (!(dst & SLJIT_MEM))
2687 return SLJIT_SUCCESS;
2688 return emit_op_mem(compiler, WORD_SIZE | STORE, TMP_REG1, dst, dstw, TMP_REG2);
2689 }
2690
2691 if (dst & SLJIT_MEM)
2692 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, dst, dstw, TMP_REG2));
2693
2694 if (op == SLJIT_AND) {
2695 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | (((cc & 0x1) ^ 0x1) << 3) | 0x4));
2696 FAIL_IF(push_inst32(compiler, ANDI | RN4(dst_r) | RD4(dst_r) | 1));
2697 FAIL_IF(push_inst32(compiler, ANDI | RN4(dst_r) | RD4(dst_r) | 0));
2698 }
2699 else {
2700 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | 0x8));
2701 FAIL_IF(push_inst32(compiler, ((op == SLJIT_OR) ? ORRI : EORI) | RN4(dst_r) | RD4(dst_r) | 1));
2702 }
2703
2704 if (dst & SLJIT_MEM)
2705 FAIL_IF(emit_op_mem(compiler, WORD_SIZE | STORE, TMP_REG1, dst, dstw, TMP_REG2));
2706
2707 if (!(flags & SLJIT_SET_Z))
2708 return SLJIT_SUCCESS;
2709
2710 /* The condition must always be set, even if the ORR/EORI is not executed above. */
2711 return push_inst32(compiler, MOV_W | SET_FLAGS | RD4(TMP_REG1) | RM4(dst_r));
2712 }
2713
sljit_emit_cmov(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 dst_reg,sljit_s32 src,sljit_sw srcw)2714 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_cmov(struct sljit_compiler *compiler, sljit_s32 type,
2715 sljit_s32 dst_reg,
2716 sljit_s32 src, sljit_sw srcw)
2717 {
2718 sljit_uw cc, tmp;
2719
2720 CHECK_ERROR();
2721 CHECK(check_sljit_emit_cmov(compiler, type, dst_reg, src, srcw));
2722
2723 cc = get_cc(compiler, type & ~SLJIT_32);
2724
2725 if (!(src & SLJIT_IMM)) {
2726 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | 0x8));
2727 return push_inst16(compiler, MOV | SET_REGS44(dst_reg, src));
2728 }
2729
2730 tmp = (sljit_uw) srcw;
2731
2732 if (tmp < 0x10000) {
2733 /* set low 16 bits, set hi 16 bits to 0. */
2734 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | 0x8));
2735 return push_inst32(compiler, MOVW | RD4(dst_reg)
2736 | COPY_BITS(tmp, 12, 16, 4) | COPY_BITS(tmp, 11, 26, 1) | COPY_BITS(tmp, 8, 12, 3) | (tmp & 0xff));
2737 }
2738
2739 tmp = get_imm((sljit_uw)srcw);
2740 if (tmp != INVALID_IMM) {
2741 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | 0x8));
2742 return push_inst32(compiler, MOV_WI | RD4(dst_reg) | tmp);
2743 }
2744
2745 tmp = get_imm(~(sljit_uw)srcw);
2746 if (tmp != INVALID_IMM) {
2747 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | 0x8));
2748 return push_inst32(compiler, MVN_WI | RD4(dst_reg) | tmp);
2749 }
2750
2751 FAIL_IF(push_inst16(compiler, IT | (cc << 4) | ((cc & 0x1) << 3) | 0x4));
2752
2753 tmp = (sljit_uw) srcw;
2754 FAIL_IF(push_inst32(compiler, MOVW | RD4(dst_reg)
2755 | COPY_BITS(tmp, 12, 16, 4) | COPY_BITS(tmp, 11, 26, 1) | COPY_BITS(tmp, 8, 12, 3) | (tmp & 0xff)));
2756 return push_inst32(compiler, MOVT | RD4(dst_reg)
2757 | COPY_BITS(tmp, 12 + 16, 16, 4) | COPY_BITS(tmp, 11 + 16, 26, 1) | COPY_BITS(tmp, 8 + 16, 12, 3) | ((tmp & 0xff0000) >> 16));
2758 }
2759
sljit_emit_mem(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 reg,sljit_s32 mem,sljit_sw memw)2760 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
2761 sljit_s32 reg,
2762 sljit_s32 mem, sljit_sw memw)
2763 {
2764 sljit_s32 flags;
2765 sljit_uw imm, tmp;
2766
2767 CHECK_ERROR();
2768 CHECK(check_sljit_emit_mem(compiler, type, reg, mem, memw));
2769
2770 if (!(reg & REG_PAIR_MASK))
2771 return sljit_emit_mem_unaligned(compiler, type, reg, mem, memw);
2772
2773 if (type & (SLJIT_MEM_UNALIGNED | SLJIT_MEM_UNALIGNED_16 | SLJIT_MEM_UNALIGNED_32)) {
2774 if ((mem & REG_MASK) == 0) {
2775 if ((memw & 0xfff) >= (0x1000 - SSIZE_OF(sw))) {
2776 imm = get_imm((sljit_uw)((memw + 0x1000) & ~0xfff));
2777
2778 if (imm != INVALID_IMM)
2779 memw = (memw & 0xfff) - 0x1000;
2780 } else {
2781 imm = get_imm((sljit_uw)(memw & ~0xfff));
2782
2783 if (imm != INVALID_IMM)
2784 memw &= 0xff;
2785 }
2786
2787 if (imm == INVALID_IMM) {
2788 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)memw));
2789 memw = 0;
2790 } else
2791 FAIL_IF(push_inst32(compiler, MOV_WI | RD4(TMP_REG1) | imm));
2792
2793 mem = SLJIT_MEM1(TMP_REG1);
2794 } else if (mem & OFFS_REG_MASK) {
2795 FAIL_IF(push_inst32(compiler, ADD_W | RD4(TMP_REG1) | RN4(mem & REG_MASK) | RM4(OFFS_REG(mem)) | ((sljit_uw)(memw & 0x3) << 6)));
2796 memw = 0;
2797 mem = SLJIT_MEM1(TMP_REG1);
2798 } else if (memw < -0xff) {
2799 /* Zero value can be included in the first case. */
2800 if ((-memw & 0xfff) <= SSIZE_OF(sw))
2801 tmp = (sljit_uw)((-memw + 0x7ff) & ~0x7ff);
2802 else
2803 tmp = (sljit_uw)((-memw + 0xfff) & ~0xfff);
2804
2805 SLJIT_ASSERT(tmp >= (sljit_uw)-memw);
2806 imm = get_imm(tmp);
2807
2808 if (imm != INVALID_IMM) {
2809 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(TMP_REG1) | RN4(mem & REG_MASK) | imm));
2810 memw += (sljit_sw)tmp;
2811 SLJIT_ASSERT(memw >= 0 && memw <= 0xfff - SSIZE_OF(sw));
2812 } else {
2813 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)memw));
2814 FAIL_IF(push_inst16(compiler, ADD | SET_REGS44(TMP_REG1, mem & REG_MASK)));
2815 memw = 0;
2816 }
2817
2818 mem = SLJIT_MEM1(TMP_REG1);
2819 } else if (memw >= (0x1000 - SSIZE_OF(sw))) {
2820 if ((memw & 0xfff) >= (0x1000 - SSIZE_OF(sw))) {
2821 imm = get_imm((sljit_uw)((memw + 0x1000) & ~0xfff));
2822
2823 if (imm != INVALID_IMM)
2824 memw = (memw & 0xfff) - 0x1000;
2825 } else {
2826 imm = get_imm((sljit_uw)(memw & ~0xfff));
2827
2828 if (imm != INVALID_IMM)
2829 memw &= 0xfff;
2830 }
2831
2832 if (imm != INVALID_IMM) {
2833 SLJIT_ASSERT(memw >= -0xff && memw <= 0xfff);
2834 FAIL_IF(push_inst32(compiler, ADD_WI | RD4(TMP_REG1) | RN4(mem & REG_MASK) | imm));
2835 } else {
2836 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)memw));
2837 FAIL_IF(push_inst16(compiler, ADD | SET_REGS44(TMP_REG1, mem & REG_MASK)));
2838 memw = 0;
2839 }
2840
2841 mem = SLJIT_MEM1(TMP_REG1);
2842 }
2843
2844 flags = WORD_SIZE;
2845
2846 SLJIT_ASSERT(memw <= 0xfff - SSIZE_OF(sw) && memw >= -0xff);
2847
2848 if (type & SLJIT_MEM_STORE) {
2849 flags |= STORE;
2850 } else if (REG_PAIR_FIRST(reg) == (mem & REG_MASK)) {
2851 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, REG_PAIR_SECOND(reg), mem, memw + SSIZE_OF(sw), TMP_REG2));
2852 return emit_op_mem(compiler, WORD_SIZE, REG_PAIR_FIRST(reg), mem, memw, TMP_REG2);
2853 }
2854
2855 FAIL_IF(emit_op_mem(compiler, flags, REG_PAIR_FIRST(reg), mem, memw, TMP_REG2));
2856 return emit_op_mem(compiler, flags, REG_PAIR_SECOND(reg), mem, memw + SSIZE_OF(sw), TMP_REG2);
2857 }
2858
2859 flags = 1 << 23;
2860
2861 if ((mem & REG_MASK) == 0) {
2862 tmp = (sljit_uw)(memw & 0x7fc);
2863 imm = get_imm((sljit_uw)((memw + (tmp <= 0x400 ? 0 : 0x400)) & ~0x3fc));
2864
2865 if (imm == INVALID_IMM) {
2866 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)memw));
2867 memw = 0;
2868 } else {
2869 FAIL_IF(push_inst32(compiler, MOV_WI | RD4(TMP_REG1) | imm));
2870 memw = (memw & 0x3fc) >> 2;
2871
2872 if (tmp > 0x400) {
2873 memw = 0x100 - memw;
2874 flags = 0;
2875 }
2876
2877 SLJIT_ASSERT(memw >= 0 && memw <= 0xff);
2878 }
2879
2880 mem = SLJIT_MEM1(TMP_REG1);
2881 } else if (mem & OFFS_REG_MASK) {
2882 FAIL_IF(push_inst32(compiler, ADD_W | RD4(TMP_REG1) | RN4(mem & REG_MASK) | RM4(OFFS_REG(mem)) | ((sljit_uw)(memw & 0x3) << 6)));
2883 memw = 0;
2884 mem = SLJIT_MEM1(TMP_REG1);
2885 } else if (memw < 0) {
2886 if ((-memw & ~0x3fc) == 0) {
2887 flags = 0;
2888 memw = -memw >> 2;
2889 } else {
2890 tmp = (sljit_uw)(-memw & 0x7fc);
2891 imm = get_imm((sljit_uw)((-memw + (tmp <= 0x400 ? 0 : 0x400)) & ~0x3fc));
2892
2893 if (imm != INVALID_IMM) {
2894 FAIL_IF(push_inst32(compiler, SUB_WI | RD4(TMP_REG1) | RN4(mem & REG_MASK) | imm));
2895 memw = (-memw & 0x3fc) >> 2;
2896
2897 if (tmp <= 0x400)
2898 flags = 0;
2899 else
2900 memw = 0x100 - memw;
2901 } else {
2902 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)memw));
2903 FAIL_IF(push_inst16(compiler, ADD | SET_REGS44(TMP_REG1, mem & REG_MASK)));
2904 memw = 0;
2905 }
2906
2907 mem = SLJIT_MEM1(TMP_REG1);
2908 }
2909 } else if ((memw & ~0x3fc) != 0) {
2910 tmp = (sljit_uw)(memw & 0x7fc);
2911 imm = get_imm((sljit_uw)((memw + (tmp <= 0x400 ? 0 : 0x400)) & ~0x3fc));
2912
2913 if (imm != INVALID_IMM) {
2914 FAIL_IF(push_inst32(compiler, ADD_WI | RD4(TMP_REG1) | RN4(mem & REG_MASK) | imm));
2915 memw = (memw & 0x3fc) >> 2;
2916
2917 if (tmp > 0x400) {
2918 memw = 0x100 - memw;
2919 flags = 0;
2920 }
2921 } else {
2922 FAIL_IF(load_immediate(compiler, TMP_REG1, (sljit_uw)memw));
2923 FAIL_IF(push_inst16(compiler, ADD | SET_REGS44(TMP_REG1, mem & REG_MASK)));
2924 memw = 0;
2925 }
2926
2927 mem = SLJIT_MEM1(TMP_REG1);
2928 } else
2929 memw >>= 2;
2930
2931 SLJIT_ASSERT(memw >= 0 && memw <= 0xff);
2932 return push_inst32(compiler, ((type & SLJIT_MEM_STORE) ? STRD : LDRD) | (sljit_ins)flags | RN4(mem & REG_MASK) | RT4(REG_PAIR_FIRST(reg)) | RD4(REG_PAIR_SECOND(reg)) | (sljit_ins)memw);
2933 }
2934
sljit_emit_mem_update(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 reg,sljit_s32 mem,sljit_sw memw)2935 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
2936 sljit_s32 reg,
2937 sljit_s32 mem, sljit_sw memw)
2938 {
2939 sljit_s32 flags;
2940 sljit_ins inst;
2941
2942 CHECK_ERROR();
2943 CHECK(check_sljit_emit_mem_update(compiler, type, reg, mem, memw));
2944
2945 if ((mem & OFFS_REG_MASK) || (memw > 255 || memw < -255))
2946 return SLJIT_ERR_UNSUPPORTED;
2947
2948 if (type & SLJIT_MEM_SUPP)
2949 return SLJIT_SUCCESS;
2950
2951 switch (type & 0xff) {
2952 case SLJIT_MOV:
2953 case SLJIT_MOV_U32:
2954 case SLJIT_MOV_S32:
2955 case SLJIT_MOV32:
2956 case SLJIT_MOV_P:
2957 flags = WORD_SIZE;
2958 break;
2959 case SLJIT_MOV_U8:
2960 flags = BYTE_SIZE;
2961 break;
2962 case SLJIT_MOV_S8:
2963 flags = BYTE_SIZE | SIGNED;
2964 break;
2965 case SLJIT_MOV_U16:
2966 flags = HALF_SIZE;
2967 break;
2968 case SLJIT_MOV_S16:
2969 flags = HALF_SIZE | SIGNED;
2970 break;
2971 default:
2972 SLJIT_UNREACHABLE();
2973 flags = WORD_SIZE;
2974 break;
2975 }
2976
2977 if (type & SLJIT_MEM_STORE)
2978 flags |= STORE;
2979
2980 inst = sljit_mem32[flags] | 0x900;
2981
2982 if (!(type & SLJIT_MEM_POST))
2983 inst |= 0x400;
2984
2985 if (memw >= 0)
2986 inst |= 0x200;
2987 else
2988 memw = -memw;
2989
2990 return push_inst32(compiler, inst | RT4(reg) | RN4(mem & REG_MASK) | (sljit_ins)memw);
2991 }
2992
update_mem_addr(struct sljit_compiler * compiler,sljit_s32 * mem,sljit_sw * memw,sljit_s32 max_offset)2993 static sljit_s32 update_mem_addr(struct sljit_compiler *compiler, sljit_s32 *mem, sljit_sw *memw, sljit_s32 max_offset)
2994 {
2995 sljit_s32 arg = *mem;
2996 sljit_sw argw = *memw;
2997 sljit_uw imm;
2998
2999 *mem = TMP_REG1;
3000
3001 if (SLJIT_UNLIKELY(arg & OFFS_REG_MASK)) {
3002 *memw = 0;
3003 return push_inst32(compiler, ADD_W | RD4(TMP_REG1) | RN4(arg & REG_MASK) | RM4(OFFS_REG(arg)) | ((sljit_uw)(argw & 0x3) << 6));
3004 }
3005
3006 arg &= REG_MASK;
3007
3008 if (arg) {
3009 if (argw <= max_offset && argw >= -0xff) {
3010 *mem = arg;
3011 return SLJIT_SUCCESS;
3012 }
3013
3014 if (argw < 0) {
3015 imm = get_imm((sljit_uw)(-argw & ~0xff));
3016
3017 if (imm) {
3018 *memw = -(-argw & 0xff);
3019 return push_inst32(compiler, SUB_WI | RD4(TMP_REG1) | RN4(arg) | imm);
3020 }
3021 } else if ((argw & 0xfff) <= max_offset) {
3022 imm = get_imm((sljit_uw)(argw & ~0xfff));
3023
3024 if (imm) {
3025 *memw = argw & 0xfff;
3026 return push_inst32(compiler, ADD_WI | RD4(TMP_REG1) | RN4(arg) | imm);
3027 }
3028 } else {
3029 imm = get_imm((sljit_uw)((argw | 0xfff) + 1));
3030
3031 if (imm) {
3032 *memw = (argw & 0xfff) - 0x1000;
3033 return push_inst32(compiler, ADD_WI | RD4(TMP_REG1) | RN4(arg) | imm);
3034 }
3035 }
3036 }
3037
3038 imm = (sljit_uw)(argw & ~0xfff);
3039
3040 if ((argw & 0xfff) > max_offset) {
3041 imm += 0x1000;
3042 *memw = (argw & 0xfff) - 0x1000;
3043 } else
3044 *memw = argw & 0xfff;
3045
3046 FAIL_IF(load_immediate(compiler, TMP_REG1, imm));
3047
3048 if (arg == 0)
3049 return SLJIT_SUCCESS;
3050
3051 return push_inst16(compiler, ADD | SET_REGS44(TMP_REG1, arg));
3052 }
3053
sljit_emit_fmem(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 mem,sljit_sw memw)3054 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem(struct sljit_compiler *compiler, sljit_s32 type,
3055 sljit_s32 freg,
3056 sljit_s32 mem, sljit_sw memw)
3057 {
3058 CHECK_ERROR();
3059 CHECK(check_sljit_emit_fmem(compiler, type, freg, mem, memw));
3060
3061 if (type & SLJIT_MEM_UNALIGNED_32)
3062 return emit_fop_mem(compiler, ((type ^ SLJIT_32) & SLJIT_32) | ((type & SLJIT_MEM_STORE) ? 0 : FPU_LOAD), freg, mem, memw);
3063
3064 if (type & SLJIT_MEM_STORE) {
3065 FAIL_IF(push_inst32(compiler, VMOV | (1 << 20) | DN4(freg) | RT4(TMP_REG2)));
3066
3067 if (type & SLJIT_32)
3068 return emit_op_mem(compiler, WORD_SIZE | STORE, TMP_REG2, mem, memw, TMP_REG1);
3069
3070 FAIL_IF(update_mem_addr(compiler, &mem, &memw, 0xfff - 4));
3071 mem |= SLJIT_MEM;
3072
3073 FAIL_IF(emit_op_mem(compiler, WORD_SIZE | STORE, TMP_REG2, mem, memw, TMP_REG1));
3074 FAIL_IF(push_inst32(compiler, VMOV | (1 << 20) | DN4(freg) | 0x80 | RT4(TMP_REG2)));
3075 return emit_op_mem(compiler, WORD_SIZE | STORE, TMP_REG2, mem, memw + 4, TMP_REG1);
3076 }
3077
3078 if (type & SLJIT_32) {
3079 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, mem, memw, TMP_REG1));
3080 return push_inst32(compiler, VMOV | DN4(freg) | RT4(TMP_REG2));
3081 }
3082
3083 FAIL_IF(update_mem_addr(compiler, &mem, &memw, 0xfff - 4));
3084 mem |= SLJIT_MEM;
3085
3086 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG2, mem, memw, TMP_REG1));
3087 FAIL_IF(emit_op_mem(compiler, WORD_SIZE, TMP_REG1, mem, memw + 4, TMP_REG1));
3088 return push_inst32(compiler, VMOV2 | DM4(freg) | RT4(TMP_REG2) | RN4(TMP_REG1));
3089 }
3090
3091 #undef FPU_LOAD
3092
sljit_emit_const(struct sljit_compiler * compiler,sljit_s32 dst,sljit_sw dstw,sljit_sw init_value)3093 SLJIT_API_FUNC_ATTRIBUTE struct sljit_const* sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value)
3094 {
3095 struct sljit_const *const_;
3096 sljit_s32 dst_r;
3097
3098 CHECK_ERROR_PTR();
3099 CHECK_PTR(check_sljit_emit_const(compiler, dst, dstw, init_value));
3100 ADJUST_LOCAL_OFFSET(dst, dstw);
3101
3102 const_ = (struct sljit_const*)ensure_abuf(compiler, sizeof(struct sljit_const));
3103 PTR_FAIL_IF(!const_);
3104 set_const(const_, compiler);
3105
3106 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
3107 PTR_FAIL_IF(emit_imm32_const(compiler, dst_r, (sljit_uw)init_value));
3108
3109 if (dst & SLJIT_MEM)
3110 PTR_FAIL_IF(emit_op_mem(compiler, WORD_SIZE | STORE, dst_r, dst, dstw, TMP_REG2));
3111 return const_;
3112 }
3113
sljit_emit_put_label(struct sljit_compiler * compiler,sljit_s32 dst,sljit_sw dstw)3114 SLJIT_API_FUNC_ATTRIBUTE struct sljit_put_label* sljit_emit_put_label(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
3115 {
3116 struct sljit_put_label *put_label;
3117 sljit_s32 dst_r;
3118
3119 CHECK_ERROR_PTR();
3120 CHECK_PTR(check_sljit_emit_put_label(compiler, dst, dstw));
3121 ADJUST_LOCAL_OFFSET(dst, dstw);
3122
3123 put_label = (struct sljit_put_label*)ensure_abuf(compiler, sizeof(struct sljit_put_label));
3124 PTR_FAIL_IF(!put_label);
3125 set_put_label(put_label, compiler, 0);
3126
3127 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
3128 PTR_FAIL_IF(emit_imm32_const(compiler, dst_r, 0));
3129
3130 if (dst & SLJIT_MEM)
3131 PTR_FAIL_IF(emit_op_mem(compiler, WORD_SIZE | STORE, dst_r, dst, dstw, TMP_REG2));
3132 return put_label;
3133 }
3134
sljit_set_jump_addr(sljit_uw addr,sljit_uw new_target,sljit_sw executable_offset)3135 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_jump_addr(sljit_uw addr, sljit_uw new_target, sljit_sw executable_offset)
3136 {
3137 sljit_u16 *inst = (sljit_u16*)addr;
3138 SLJIT_UNUSED_ARG(executable_offset);
3139
3140 SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 0);
3141 modify_imm32_const(inst, new_target);
3142 SLJIT_UPDATE_WX_FLAGS(inst, inst + 4, 1);
3143 inst = (sljit_u16 *)SLJIT_ADD_EXEC_OFFSET(inst, executable_offset);
3144 SLJIT_CACHE_FLUSH(inst, inst + 4);
3145 }
3146
sljit_set_const(sljit_uw addr,sljit_sw new_constant,sljit_sw executable_offset)3147 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_const(sljit_uw addr, sljit_sw new_constant, sljit_sw executable_offset)
3148 {
3149 sljit_set_jump_addr(addr, (sljit_uw)new_constant, executable_offset);
3150 }
3151