1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 /* x86 32-bit arch dependent functions. */
28
29 /* --------------------------------------------------------------------- */
30 /* Operators */
31 /* --------------------------------------------------------------------- */
32
emit_do_imm(struct sljit_compiler * compiler,sljit_u8 opcode,sljit_sw imm)33 static sljit_s32 emit_do_imm(struct sljit_compiler *compiler, sljit_u8 opcode, sljit_sw imm)
34 {
35 sljit_u8 *inst;
36
37 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + sizeof(sljit_sw));
38 FAIL_IF(!inst);
39 INC_SIZE(1 + sizeof(sljit_sw));
40 *inst++ = opcode;
41 sljit_unaligned_store_sw(inst, imm);
42 return SLJIT_SUCCESS;
43 }
44
45 /* Size contains the flags as well. */
emit_x86_instruction(struct sljit_compiler * compiler,sljit_uw size,sljit_s32 a,sljit_sw imma,sljit_s32 b,sljit_sw immb)46 static sljit_u8* emit_x86_instruction(struct sljit_compiler *compiler, sljit_uw size,
47 /* The register or immediate operand. */
48 sljit_s32 a, sljit_sw imma,
49 /* The general operand (not immediate). */
50 sljit_s32 b, sljit_sw immb)
51 {
52 sljit_u8 *inst;
53 sljit_u8 *buf_ptr;
54 sljit_u8 reg_map_b;
55 sljit_uw flags = size;
56 sljit_uw inst_size;
57
58 /* Both cannot be switched on. */
59 SLJIT_ASSERT((flags & (EX86_BIN_INS | EX86_SHIFT_INS)) != (EX86_BIN_INS | EX86_SHIFT_INS));
60 /* Size flags not allowed for typed instructions. */
61 SLJIT_ASSERT(!(flags & (EX86_BIN_INS | EX86_SHIFT_INS)) || (flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) == 0);
62 /* Both size flags cannot be switched on. */
63 SLJIT_ASSERT((flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) != (EX86_BYTE_ARG | EX86_HALF_ARG));
64 /* SSE2 and immediate is not possible. */
65 SLJIT_ASSERT(a != SLJIT_IMM || !(flags & EX86_SSE2));
66 SLJIT_ASSERT(((flags & (EX86_PREF_F2 | EX86_PREF_F3 | EX86_PREF_66))
67 & ((flags & (EX86_PREF_F2 | EX86_PREF_F3 | EX86_PREF_66)) - 1)) == 0);
68 SLJIT_ASSERT((flags & (EX86_VEX_EXT | EX86_REX)) != EX86_VEX_EXT);
69
70 size &= 0xf;
71 /* The mod r/m byte is always present. */
72 inst_size = size + 1;
73
74 if (flags & (EX86_PREF_F2 | EX86_PREF_F3 | EX86_PREF_66))
75 inst_size++;
76
77 /* Calculate size of b. */
78 if (b & SLJIT_MEM) {
79 if (!(b & REG_MASK))
80 inst_size += sizeof(sljit_sw);
81 else {
82 if (immb != 0 && !(b & OFFS_REG_MASK)) {
83 /* Immediate operand. */
84 if (immb <= 127 && immb >= -128)
85 inst_size += sizeof(sljit_s8);
86 else
87 inst_size += sizeof(sljit_sw);
88 } else if (reg_map[b & REG_MASK] == 5) {
89 /* Swap registers if possible. */
90 if ((b & OFFS_REG_MASK) && (immb & 0x3) == 0 && reg_map[OFFS_REG(b)] != 5)
91 b = SLJIT_MEM | OFFS_REG(b) | TO_OFFS_REG(b & REG_MASK);
92 else
93 inst_size += sizeof(sljit_s8);
94 }
95
96 if (reg_map[b & REG_MASK] == 4 && !(b & OFFS_REG_MASK))
97 b |= TO_OFFS_REG(SLJIT_SP);
98
99 if (b & OFFS_REG_MASK)
100 inst_size += 1; /* SIB byte. */
101 }
102 }
103
104 /* Calculate size of a. */
105 if (a == SLJIT_IMM) {
106 if (flags & EX86_BIN_INS) {
107 if (imma <= 127 && imma >= -128) {
108 inst_size += 1;
109 flags |= EX86_BYTE_ARG;
110 } else
111 inst_size += 4;
112 } else if (flags & EX86_SHIFT_INS) {
113 SLJIT_ASSERT(imma <= 0x1f);
114 if (imma != 1) {
115 inst_size++;
116 flags |= EX86_BYTE_ARG;
117 }
118 } else if (flags & EX86_BYTE_ARG)
119 inst_size++;
120 else if (flags & EX86_HALF_ARG)
121 inst_size += sizeof(short);
122 else
123 inst_size += sizeof(sljit_sw);
124 } else
125 SLJIT_ASSERT(!(flags & EX86_SHIFT_INS) || a == SLJIT_PREF_SHIFT_REG);
126
127 inst = (sljit_u8*)ensure_buf(compiler, 1 + inst_size);
128 PTR_FAIL_IF(!inst);
129
130 /* Encoding the byte. */
131 INC_SIZE(inst_size);
132 if (flags & EX86_PREF_F2)
133 *inst++ = 0xf2;
134 else if (flags & EX86_PREF_F3)
135 *inst++ = 0xf3;
136 else if (flags & EX86_PREF_66)
137 *inst++ = 0x66;
138
139 buf_ptr = inst + size;
140
141 /* Encode mod/rm byte. */
142 if (!(flags & EX86_SHIFT_INS)) {
143 if ((flags & EX86_BIN_INS) && a == SLJIT_IMM)
144 *inst = (flags & EX86_BYTE_ARG) ? GROUP_BINARY_83 : GROUP_BINARY_81;
145
146 if (a == SLJIT_IMM)
147 *buf_ptr = 0;
148 else if (!(flags & EX86_SSE2_OP1))
149 *buf_ptr = U8(reg_map[a] << 3);
150 else
151 *buf_ptr = U8(freg_map[a] << 3);
152 } else {
153 if (a == SLJIT_IMM) {
154 if (imma == 1)
155 *inst = GROUP_SHIFT_1;
156 else
157 *inst = GROUP_SHIFT_N;
158 } else
159 *inst = GROUP_SHIFT_CL;
160 *buf_ptr = 0;
161 }
162
163 if (!(b & SLJIT_MEM)) {
164 *buf_ptr = U8(*buf_ptr | MOD_REG | (!(flags & EX86_SSE2_OP2) ? reg_map[b] : freg_map[b]));
165 buf_ptr++;
166 } else if (b & REG_MASK) {
167 reg_map_b = reg_map[b & REG_MASK];
168
169 if (!(b & OFFS_REG_MASK) || (b & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_SP)) {
170 if (immb != 0 || reg_map_b == 5) {
171 if (immb <= 127 && immb >= -128)
172 *buf_ptr |= 0x40;
173 else
174 *buf_ptr |= 0x80;
175 }
176
177 if (!(b & OFFS_REG_MASK))
178 *buf_ptr++ |= reg_map_b;
179 else {
180 buf_ptr[0] |= 0x04;
181 buf_ptr[1] = U8(reg_map_b | (reg_map[OFFS_REG(b)] << 3));
182 buf_ptr += 2;
183 }
184
185 if (immb != 0 || reg_map_b == 5) {
186 if (immb <= 127 && immb >= -128)
187 *buf_ptr++ = U8(immb); /* 8 bit displacement. */
188 else {
189 sljit_unaligned_store_sw(buf_ptr, immb); /* 32 bit displacement. */
190 buf_ptr += sizeof(sljit_sw);
191 }
192 }
193 } else {
194 if (reg_map_b == 5)
195 *buf_ptr |= 0x40;
196
197 buf_ptr[0] |= 0x04;
198 buf_ptr[1] = U8(reg_map_b | (reg_map[OFFS_REG(b)] << 3) | (immb << 6));
199 buf_ptr += 2;
200
201 if (reg_map_b == 5)
202 *buf_ptr++ = 0;
203 }
204 } else {
205 *buf_ptr++ |= 0x05;
206 sljit_unaligned_store_sw(buf_ptr, immb); /* 32 bit displacement. */
207 buf_ptr += sizeof(sljit_sw);
208 }
209
210 if (a == SLJIT_IMM) {
211 if (flags & EX86_BYTE_ARG)
212 *buf_ptr = U8(imma);
213 else if (flags & EX86_HALF_ARG)
214 sljit_unaligned_store_s16(buf_ptr, (sljit_s16)imma);
215 else if (!(flags & EX86_SHIFT_INS))
216 sljit_unaligned_store_sw(buf_ptr, imma);
217 }
218
219 return inst;
220 }
221
emit_vex_instruction(struct sljit_compiler * compiler,sljit_uw op,sljit_s32 a,sljit_s32 v,sljit_s32 b,sljit_sw immb)222 static sljit_s32 emit_vex_instruction(struct sljit_compiler *compiler, sljit_uw op,
223 /* The first and second register operand. */
224 sljit_s32 a, sljit_s32 v,
225 /* The general operand (not immediate). */
226 sljit_s32 b, sljit_sw immb)
227 {
228 sljit_u8 *inst;
229 sljit_u8 vex = 0;
230 sljit_u8 vex_m = 0;
231 sljit_uw size;
232
233 SLJIT_ASSERT(((op & (EX86_PREF_F2 | EX86_PREF_F3 | EX86_PREF_66))
234 & ((op & (EX86_PREF_F2 | EX86_PREF_F3 | EX86_PREF_66)) - 1)) == 0);
235
236 if (op & VEX_OP_0F38)
237 vex_m = 0x2;
238 else if (op & VEX_OP_0F3A)
239 vex_m = 0x3;
240
241 if (op & VEX_W) {
242 if (vex_m == 0)
243 vex_m = 0x1;
244
245 vex |= 0x80;
246 }
247
248 if (op & EX86_PREF_66)
249 vex |= 0x1;
250 else if (op & EX86_PREF_F2)
251 vex |= 0x3;
252 else if (op & EX86_PREF_F3)
253 vex |= 0x2;
254
255 op &= ~(EX86_PREF_66 | EX86_PREF_F2 | EX86_PREF_F3);
256
257 if (op & VEX_256)
258 vex |= 0x4;
259
260 vex = U8(vex | ((((op & VEX_SSE2_OPV) ? freg_map[v] : reg_map[v]) ^ 0xf) << 3));
261
262 size = op & ~(sljit_uw)0xff;
263 size |= (vex_m == 0) ? 3 : 4;
264
265 inst = emit_x86_instruction(compiler, size, a, 0, b, immb);
266 FAIL_IF(!inst);
267
268 if (vex_m == 0) {
269 inst[0] = 0xc5;
270 inst[1] = U8(vex | 0x80);
271 inst[2] = U8(op);
272 return SLJIT_SUCCESS;
273 }
274
275 inst[0] = 0xc4;
276 inst[1] = U8(vex_m | 0xe0);
277 inst[2] = vex;
278 inst[3] = U8(op);
279 return SLJIT_SUCCESS;
280 }
281
282 /* --------------------------------------------------------------------- */
283 /* Enter / return */
284 /* --------------------------------------------------------------------- */
285
detect_far_jump_type(struct sljit_jump * jump,sljit_u8 * code_ptr,sljit_sw executable_offset)286 static sljit_u8* detect_far_jump_type(struct sljit_jump *jump, sljit_u8 *code_ptr, sljit_sw executable_offset)
287 {
288 sljit_uw type = jump->flags >> TYPE_SHIFT;
289
290 if (type == SLJIT_JUMP) {
291 *code_ptr++ = JMP_i32;
292 } else if (type >= SLJIT_FAST_CALL) {
293 *code_ptr++ = CALL_i32;
294 } else {
295 *code_ptr++ = GROUP_0F;
296 *code_ptr++ = get_jump_code(type);
297 }
298
299 jump->addr = (sljit_uw)code_ptr;
300
301 if (jump->flags & JUMP_ADDR)
302 sljit_unaligned_store_sw(code_ptr, (sljit_sw)(jump->u.target - (jump->addr + 4) - (sljit_uw)executable_offset));
303 else
304 jump->flags |= PATCH_MW;
305 code_ptr += 4;
306
307 return code_ptr;
308 }
309
310 #define ENTER_TMP_TO_R4 0x00001
311 #define ENTER_TMP_TO_S 0x00002
312
sljit_emit_enter(struct sljit_compiler * compiler,sljit_s32 options,sljit_s32 arg_types,sljit_s32 scratches,sljit_s32 saveds,sljit_s32 fscratches,sljit_s32 fsaveds,sljit_s32 local_size)313 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
314 sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
315 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
316 {
317 sljit_s32 word_arg_count, saved_arg_count, float_arg_count;
318 sljit_s32 size, args_size, types, status;
319 sljit_s32 kept_saveds_count = SLJIT_KEPT_SAVEDS_COUNT(options);
320 sljit_u8 *inst;
321 #ifdef _WIN32
322 sljit_s32 r2_offset = -1;
323 #endif
324
325 CHECK_ERROR();
326 CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
327 set_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
328
329 /* Emit ENDBR32 at function entry if needed. */
330 FAIL_IF(emit_endbranch(compiler));
331
332 SLJIT_COMPILE_ASSERT(SLJIT_FR0 == 1, float_register_index_start);
333
334 arg_types >>= SLJIT_ARG_SHIFT;
335 word_arg_count = 0;
336 status = 0;
337
338 if (options & SLJIT_ENTER_REG_ARG) {
339 args_size = 3 * SSIZE_OF(sw);
340
341 while (arg_types) {
342 if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64) {
343 word_arg_count++;
344 if (word_arg_count >= 4)
345 status |= ENTER_TMP_TO_R4;
346 }
347
348 arg_types >>= SLJIT_ARG_SHIFT;
349 }
350
351 compiler->args_size = 0;
352 } else {
353 types = arg_types;
354 saved_arg_count = 0;
355 float_arg_count = 0;
356 args_size = SSIZE_OF(sw);
357 while (types) {
358 switch (types & SLJIT_ARG_MASK) {
359 case SLJIT_ARG_TYPE_F64:
360 float_arg_count++;
361 FAIL_IF(emit_sse2_load(compiler, 0, float_arg_count, SLJIT_MEM1(SLJIT_SP), args_size));
362 args_size += SSIZE_OF(f64);
363 break;
364 case SLJIT_ARG_TYPE_F32:
365 float_arg_count++;
366 FAIL_IF(emit_sse2_load(compiler, 1, float_arg_count, SLJIT_MEM1(SLJIT_SP), args_size));
367 args_size += SSIZE_OF(f32);
368 break;
369 default:
370 word_arg_count++;
371
372 if (!(types & SLJIT_ARG_TYPE_SCRATCH_REG))
373 saved_arg_count++;
374
375 if (word_arg_count == 4) {
376 if (types & SLJIT_ARG_TYPE_SCRATCH_REG) {
377 status |= ENTER_TMP_TO_R4;
378 arg_types &= ~(SLJIT_ARG_FULL_MASK << 3 * SLJIT_ARG_SHIFT);
379 } else if (saved_arg_count == 4) {
380 status |= ENTER_TMP_TO_S;
381 arg_types &= ~(SLJIT_ARG_FULL_MASK << 3 * SLJIT_ARG_SHIFT);
382 }
383 }
384
385 args_size += SSIZE_OF(sw);
386 break;
387 }
388 types >>= SLJIT_ARG_SHIFT;
389 }
390
391 args_size -= SSIZE_OF(sw);
392 compiler->args_size = args_size;
393 }
394
395 size = (scratches > 9 ? (scratches - 9) : 0) + (saveds <= 3 ? saveds : 3) - kept_saveds_count;
396 if (!(options & SLJIT_ENTER_REG_ARG))
397 size++;
398
399 if (size != 0) {
400 inst = (sljit_u8*)ensure_buf(compiler, (sljit_uw)(size + 1));
401 FAIL_IF(!inst);
402
403 INC_SIZE((sljit_uw)size);
404
405 if (!(options & SLJIT_ENTER_REG_ARG))
406 PUSH_REG(reg_map[TMP_REG1]);
407
408 if ((saveds > 2 && kept_saveds_count <= 2) || scratches > 9)
409 PUSH_REG(reg_map[SLJIT_S2]);
410 if ((saveds > 1 && kept_saveds_count <= 1) || scratches > 10)
411 PUSH_REG(reg_map[SLJIT_S1]);
412 if ((saveds > 0 && kept_saveds_count == 0) || scratches > 11)
413 PUSH_REG(reg_map[SLJIT_S0]);
414
415 size *= SSIZE_OF(sw);
416 }
417
418 if (status & (ENTER_TMP_TO_R4 | ENTER_TMP_TO_S))
419 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), args_size + size);
420
421 size += SSIZE_OF(sw);
422
423 local_size = ((SLJIT_LOCALS_OFFSET_BASE + local_size + size + 0xf) & ~0xf) - size;
424 compiler->local_size = local_size;
425
426 word_arg_count = 0;
427 saved_arg_count = 0;
428 args_size = size;
429 while (arg_types) {
430 switch (arg_types & SLJIT_ARG_MASK) {
431 case SLJIT_ARG_TYPE_F64:
432 args_size += SSIZE_OF(f64);
433 break;
434 case SLJIT_ARG_TYPE_F32:
435 args_size += SSIZE_OF(f32);
436 break;
437 default:
438 word_arg_count++;
439 SLJIT_ASSERT(word_arg_count <= 3 || (word_arg_count == 4 && !(status & (ENTER_TMP_TO_R4 | ENTER_TMP_TO_S))));
440
441 if (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) {
442 #ifdef _WIN32
443 if (word_arg_count == 3 && local_size > 4 * 4096)
444 r2_offset = local_size + args_size;
445 else
446 #endif
447 EMIT_MOV(compiler, word_arg_count, 0, SLJIT_MEM1(SLJIT_SP), args_size);
448
449 } else {
450 EMIT_MOV(compiler, SLJIT_S0 - saved_arg_count, 0, SLJIT_MEM1(SLJIT_SP), args_size);
451 saved_arg_count++;
452 }
453
454 args_size += SSIZE_OF(sw);
455 break;
456 }
457 arg_types >>= SLJIT_ARG_SHIFT;
458 }
459
460 SLJIT_ASSERT(SLJIT_LOCALS_OFFSET > 0);
461
462 #ifdef _WIN32
463 SLJIT_ASSERT(r2_offset == -1 || local_size > 4 * 4096);
464
465 if (local_size > 4096) {
466 if (local_size <= 4 * 4096) {
467 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -4096);
468
469 if (local_size > 2 * 4096)
470 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -4096 * 2);
471 if (local_size > 3 * 4096)
472 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -4096 * 3);
473 }
474 else {
475 if (options & SLJIT_ENTER_REG_ARG) {
476 SLJIT_ASSERT(r2_offset == -1);
477
478 inst = (sljit_u8*)ensure_buf(compiler, (sljit_uw)(1 + 1));
479 FAIL_IF(!inst);
480 INC_SIZE(1);
481 PUSH_REG(reg_map[SLJIT_R2]);
482
483 local_size -= SSIZE_OF(sw);
484 r2_offset = local_size;
485 }
486
487 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_IMM, local_size >> 12);
488
489 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -4096);
490 BINARY_IMM32(SUB, 4096, SLJIT_SP, 0);
491
492 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
493 FAIL_IF(!inst);
494
495 INC_SIZE(2);
496 inst[0] = LOOP_i8;
497 inst[1] = (sljit_u8)-16;
498 local_size &= 0xfff;
499 }
500 }
501
502 if (local_size > 0) {
503 BINARY_IMM32(OR, 0, SLJIT_MEM1(SLJIT_SP), -local_size);
504 BINARY_IMM32(SUB, local_size, SLJIT_SP, 0);
505 }
506
507 if (r2_offset != -1)
508 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), r2_offset);
509
510 #else /* !_WIN32 */
511
512 SLJIT_ASSERT(local_size > 0);
513
514 BINARY_IMM32(SUB, local_size, SLJIT_SP, 0);
515
516 #endif /* _WIN32 */
517
518 size = SLJIT_LOCALS_OFFSET_BASE - SSIZE_OF(sw);
519 kept_saveds_count = SLJIT_R3 - kept_saveds_count;
520
521 while (saved_arg_count > 3) {
522 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), size, kept_saveds_count, 0);
523 kept_saveds_count++;
524 size -= SSIZE_OF(sw);
525 saved_arg_count--;
526 }
527
528 if (status & (ENTER_TMP_TO_R4 | ENTER_TMP_TO_S)) {
529 if (status & ENTER_TMP_TO_R4)
530 size = 2 * SSIZE_OF(sw);
531
532 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), size, TMP_REG1, 0);
533 }
534
535 return SLJIT_SUCCESS;
536 }
537
sljit_set_context(struct sljit_compiler * compiler,sljit_s32 options,sljit_s32 arg_types,sljit_s32 scratches,sljit_s32 saveds,sljit_s32 fscratches,sljit_s32 fsaveds,sljit_s32 local_size)538 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
539 sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
540 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
541 {
542 sljit_s32 args_size;
543
544 CHECK_ERROR();
545 CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
546 set_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
547
548 arg_types >>= SLJIT_ARG_SHIFT;
549 args_size = 0;
550
551 if (!(options & SLJIT_ENTER_REG_ARG)) {
552 while (arg_types) {
553 switch (arg_types & SLJIT_ARG_MASK) {
554 case SLJIT_ARG_TYPE_F64:
555 args_size += SSIZE_OF(f64);
556 break;
557 case SLJIT_ARG_TYPE_F32:
558 args_size += SSIZE_OF(f32);
559 break;
560 default:
561 args_size += SSIZE_OF(sw);
562 break;
563 }
564 arg_types >>= SLJIT_ARG_SHIFT;
565 }
566 }
567
568 compiler->args_size = args_size;
569
570 /* [esp+0] for saving temporaries and for function calls. */
571
572 saveds = (1 + (scratches > 9 ? (scratches - 9) : 0) + (saveds <= 3 ? saveds : 3) - SLJIT_KEPT_SAVEDS_COUNT(options)) * SSIZE_OF(sw);
573
574 /* Saving ebp. */
575 if (!(options & SLJIT_ENTER_REG_ARG))
576 saveds += SSIZE_OF(sw);
577
578 compiler->local_size = ((SLJIT_LOCALS_OFFSET_BASE + local_size + saveds + 0xf) & ~0xf) - saveds;
579 return SLJIT_SUCCESS;
580 }
581
emit_stack_frame_release(struct sljit_compiler * compiler,sljit_s32 is_return_to)582 static sljit_s32 emit_stack_frame_release(struct sljit_compiler *compiler, sljit_s32 is_return_to)
583 {
584 sljit_s32 kept_saveds_count = SLJIT_KEPT_SAVEDS_COUNT(compiler->options);
585 sljit_s32 local_size, saveds;
586 sljit_uw size;
587 sljit_u8 *inst;
588
589 size = (sljit_uw)((compiler->scratches > 9 ? (compiler->scratches - 9) : 0) +
590 (compiler->saveds <= 3 ? compiler->saveds : 3) - kept_saveds_count);
591
592 local_size = compiler->local_size;
593
594 if (!(compiler->options & SLJIT_ENTER_REG_ARG))
595 size++;
596 else if (is_return_to && size == 0) {
597 local_size += SSIZE_OF(sw);
598 is_return_to = 0;
599 }
600
601 if (local_size > 0)
602 BINARY_IMM32(ADD, local_size, SLJIT_SP, 0);
603
604 if (size == 0)
605 return SLJIT_SUCCESS;
606
607 inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
608 FAIL_IF(!inst);
609
610 INC_SIZE(size);
611
612 saveds = compiler->saveds;
613
614 if ((saveds > 0 && kept_saveds_count == 0) || compiler->scratches > 11)
615 POP_REG(reg_map[SLJIT_S0]);
616 if ((saveds > 1 && kept_saveds_count <= 1) || compiler->scratches > 10)
617 POP_REG(reg_map[SLJIT_S1]);
618 if ((saveds > 2 && kept_saveds_count <= 2) || compiler->scratches > 9)
619 POP_REG(reg_map[SLJIT_S2]);
620
621 if (!(compiler->options & SLJIT_ENTER_REG_ARG))
622 POP_REG(reg_map[TMP_REG1]);
623
624 if (is_return_to)
625 BINARY_IMM32(ADD, sizeof(sljit_sw), SLJIT_SP, 0);
626
627 return SLJIT_SUCCESS;
628 }
629
sljit_emit_return_void(struct sljit_compiler * compiler)630 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_void(struct sljit_compiler *compiler)
631 {
632 CHECK_ERROR();
633 CHECK(check_sljit_emit_return_void(compiler));
634
635 SLJIT_ASSERT(compiler->args_size >= 0);
636 SLJIT_ASSERT(compiler->local_size > 0);
637
638 FAIL_IF(emit_stack_frame_release(compiler, 0));
639
640 return emit_byte(compiler, RET_near);
641 }
642
sljit_emit_return_to(struct sljit_compiler * compiler,sljit_s32 src,sljit_sw srcw)643 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return_to(struct sljit_compiler *compiler,
644 sljit_s32 src, sljit_sw srcw)
645 {
646 sljit_s32 src_r;
647
648 CHECK_ERROR();
649 CHECK(check_sljit_emit_return_to(compiler, src, srcw));
650
651 if ((src & SLJIT_MEM) || (src > SLJIT_R2 && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options)))) {
652 ADJUST_LOCAL_OFFSET(src, srcw);
653 CHECK_EXTRA_REGS(src, srcw, (void)0);
654
655 src_r = (compiler->options & SLJIT_ENTER_REG_ARG) ? TMP_REG1 : SLJIT_R1;
656
657 EMIT_MOV(compiler, src_r, 0, src, srcw);
658 src = src_r;
659 srcw = 0;
660 }
661
662 FAIL_IF(emit_stack_frame_release(compiler, 1));
663
664 SLJIT_SKIP_CHECKS(compiler);
665 return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
666 }
667
668 /* --------------------------------------------------------------------- */
669 /* Call / return instructions */
670 /* --------------------------------------------------------------------- */
671
call_get_stack_size(sljit_s32 arg_types,sljit_s32 * word_arg_count_ptr)672 static sljit_s32 call_get_stack_size(sljit_s32 arg_types, sljit_s32 *word_arg_count_ptr)
673 {
674 sljit_sw stack_size = 0;
675 sljit_s32 word_arg_count = 0;
676
677 arg_types >>= SLJIT_ARG_SHIFT;
678
679 while (arg_types) {
680 switch (arg_types & SLJIT_ARG_MASK) {
681 case SLJIT_ARG_TYPE_F64:
682 stack_size += SSIZE_OF(f64);
683 break;
684 case SLJIT_ARG_TYPE_F32:
685 stack_size += SSIZE_OF(f32);
686 break;
687 default:
688 word_arg_count++;
689 stack_size += SSIZE_OF(sw);
690 break;
691 }
692
693 arg_types >>= SLJIT_ARG_SHIFT;
694 }
695
696 if (word_arg_count_ptr)
697 *word_arg_count_ptr = word_arg_count;
698
699 if (stack_size <= 4 * SSIZE_OF(sw))
700 return 0;
701
702 return ((stack_size - (4 * SSIZE_OF(sw)) + 0xf) & ~0xf);
703 }
704
call_with_args(struct sljit_compiler * compiler,sljit_s32 arg_types,sljit_sw stack_size,sljit_s32 word_arg_count,sljit_s32 keep_tmp1)705 static sljit_s32 call_with_args(struct sljit_compiler *compiler,
706 sljit_s32 arg_types, sljit_sw stack_size, sljit_s32 word_arg_count, sljit_s32 keep_tmp1)
707 {
708 sljit_s32 float_arg_count = 0, arg4_reg = 0, arg_offset;
709 sljit_u8 *inst;
710
711 if (word_arg_count >= 4) {
712 arg4_reg = SLJIT_R0;
713
714 if (!keep_tmp1) {
715 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), 2 * SSIZE_OF(sw));
716 arg4_reg = TMP_REG1;
717 }
718 }
719
720 if (stack_size > 0)
721 BINARY_IMM32(SUB, stack_size, SLJIT_SP, 0);
722
723 arg_offset = 0;
724 word_arg_count = 0;
725 arg_types >>= SLJIT_ARG_SHIFT;
726
727 while (arg_types) {
728 switch (arg_types & SLJIT_ARG_MASK) {
729 case SLJIT_ARG_TYPE_F64:
730 float_arg_count++;
731 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), arg_offset, float_arg_count));
732 arg_offset += SSIZE_OF(f64);
733 break;
734 case SLJIT_ARG_TYPE_F32:
735 float_arg_count++;
736 FAIL_IF(emit_sse2_store(compiler, 1, SLJIT_MEM1(SLJIT_SP), arg_offset, float_arg_count));
737 arg_offset += SSIZE_OF(f32);
738 break;
739 default:
740 word_arg_count++;
741 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), arg_offset, (word_arg_count >= 4) ? arg4_reg : word_arg_count, 0);
742
743 if (word_arg_count == 1 && arg4_reg == SLJIT_R0)
744 EMIT_MOV(compiler, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_SP), 2 * SSIZE_OF(sw) + stack_size);
745
746 arg_offset += SSIZE_OF(sw);
747 break;
748 }
749
750 arg_types >>= SLJIT_ARG_SHIFT;
751 }
752
753 return SLJIT_SUCCESS;
754 }
755
post_call_with_args(struct sljit_compiler * compiler,sljit_s32 arg_types,sljit_s32 stack_size)756 static sljit_s32 post_call_with_args(struct sljit_compiler *compiler,
757 sljit_s32 arg_types, sljit_s32 stack_size)
758 {
759 sljit_u8 *inst;
760 sljit_s32 single;
761
762 if (stack_size > 0)
763 BINARY_IMM32(ADD, stack_size, SLJIT_SP, 0);
764
765 if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64)
766 return SLJIT_SUCCESS;
767
768 single = ((arg_types & SLJIT_ARG_MASK) == SLJIT_ARG_TYPE_F32);
769
770 inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
771 FAIL_IF(!inst);
772 INC_SIZE(3);
773 inst[0] = single ? FSTPS : FSTPD;
774 inst[1] = (0x03 << 3) | 0x04;
775 inst[2] = (0x04 << 3) | reg_map[SLJIT_SP];
776
777 return emit_sse2_load(compiler, single, SLJIT_FR0, SLJIT_MEM1(SLJIT_SP), 0);
778 }
779
tail_call_with_args(struct sljit_compiler * compiler,sljit_s32 * extra_space,sljit_s32 arg_types,sljit_s32 src,sljit_sw srcw)780 static sljit_s32 tail_call_with_args(struct sljit_compiler *compiler,
781 sljit_s32 *extra_space, sljit_s32 arg_types,
782 sljit_s32 src, sljit_sw srcw)
783 {
784 sljit_sw args_size, saved_regs_size;
785 sljit_sw types, word_arg_count, float_arg_count;
786 sljit_sw stack_size, prev_stack_size, min_size, offset;
787 sljit_sw word_arg4_offset;
788 sljit_u8 r2_offset = 0;
789 sljit_s32 kept_saveds_count = SLJIT_KEPT_SAVEDS_COUNT(compiler->options);
790 sljit_u8* inst;
791
792 ADJUST_LOCAL_OFFSET(src, srcw);
793 CHECK_EXTRA_REGS(src, srcw, (void)0);
794
795 saved_regs_size = (1 + (compiler->scratches > 9 ? (compiler->scratches - 9) : 0)
796 + (compiler->saveds <= 3 ? compiler->saveds : 3) - kept_saveds_count) * SSIZE_OF(sw);
797
798 word_arg_count = 0;
799 float_arg_count = 0;
800 arg_types >>= SLJIT_ARG_SHIFT;
801 types = 0;
802 args_size = 0;
803
804 while (arg_types != 0) {
805 types = (types << SLJIT_ARG_SHIFT) | (arg_types & SLJIT_ARG_MASK);
806
807 switch (arg_types & SLJIT_ARG_MASK) {
808 case SLJIT_ARG_TYPE_F64:
809 args_size += SSIZE_OF(f64);
810 float_arg_count++;
811 break;
812 case SLJIT_ARG_TYPE_F32:
813 args_size += SSIZE_OF(f32);
814 float_arg_count++;
815 break;
816 default:
817 word_arg_count++;
818 args_size += SSIZE_OF(sw);
819 break;
820 }
821 arg_types >>= SLJIT_ARG_SHIFT;
822 }
823
824 if (args_size <= compiler->args_size) {
825 *extra_space = 0;
826 stack_size = args_size + SSIZE_OF(sw) + saved_regs_size;
827
828 offset = stack_size + compiler->local_size;
829
830 if (src != SLJIT_IMM && src != SLJIT_R0) {
831 if (word_arg_count >= 1) {
832 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_R0, 0);
833 r2_offset = sizeof(sljit_sw);
834 }
835 EMIT_MOV(compiler, SLJIT_R0, 0, src, srcw);
836 }
837
838 while (types != 0) {
839 switch (types & SLJIT_ARG_MASK) {
840 case SLJIT_ARG_TYPE_F64:
841 offset -= SSIZE_OF(f64);
842 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), offset, float_arg_count));
843 float_arg_count--;
844 break;
845 case SLJIT_ARG_TYPE_F32:
846 offset -= SSIZE_OF(f32);
847 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), offset, float_arg_count));
848 float_arg_count--;
849 break;
850 default:
851 switch (word_arg_count) {
852 case 1:
853 offset -= SSIZE_OF(sw);
854 if (r2_offset != 0) {
855 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), 0);
856 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
857 } else
858 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R0, 0);
859 break;
860 case 2:
861 offset -= SSIZE_OF(sw);
862 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R1, 0);
863 break;
864 case 3:
865 offset -= SSIZE_OF(sw);
866 break;
867 case 4:
868 offset -= SSIZE_OF(sw);
869 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), 2 * SSIZE_OF(sw));
870 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
871 break;
872 }
873 word_arg_count--;
874 break;
875 }
876 types >>= SLJIT_ARG_SHIFT;
877 }
878
879 return emit_stack_frame_release(compiler, 0);
880 }
881
882 stack_size = args_size + SSIZE_OF(sw);
883
884 if (word_arg_count >= 1 && src != SLJIT_IMM && src != SLJIT_R0) {
885 r2_offset = SSIZE_OF(sw);
886 stack_size += SSIZE_OF(sw);
887 }
888
889 if (word_arg_count >= 3)
890 stack_size += SSIZE_OF(sw);
891
892 prev_stack_size = SSIZE_OF(sw) + saved_regs_size;
893 min_size = prev_stack_size + compiler->local_size;
894
895 word_arg4_offset = 2 * SSIZE_OF(sw);
896
897 if (stack_size > min_size) {
898 BINARY_IMM32(SUB, stack_size - min_size, SLJIT_SP, 0);
899 if (src == SLJIT_MEM1(SLJIT_SP))
900 srcw += stack_size - min_size;
901 word_arg4_offset += stack_size - min_size;
902 }
903 else
904 stack_size = min_size;
905
906 if (word_arg_count >= 3) {
907 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), r2_offset, SLJIT_R2, 0);
908
909 if (word_arg_count >= 4)
910 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), word_arg4_offset);
911 }
912
913 if (src != SLJIT_IMM && src != SLJIT_R0) {
914 if (word_arg_count >= 1) {
915 SLJIT_ASSERT(r2_offset == sizeof(sljit_sw));
916 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_R0, 0);
917 }
918 EMIT_MOV(compiler, SLJIT_R0, 0, src, srcw);
919 }
920
921 /* Restore saved registers. */
922 offset = stack_size - 2 * SSIZE_OF(sw);
923 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), offset);
924
925 if (compiler->saveds > 2 || compiler->scratches > 9) {
926 offset -= SSIZE_OF(sw);
927 EMIT_MOV(compiler, SLJIT_S2, 0, SLJIT_MEM1(SLJIT_SP), offset);
928 }
929 if ((compiler->saveds > 1 && kept_saveds_count <= 1) || compiler->scratches > 10) {
930 offset -= SSIZE_OF(sw);
931 EMIT_MOV(compiler, SLJIT_S1, 0, SLJIT_MEM1(SLJIT_SP), offset);
932 }
933 if ((compiler->saveds > 0 && kept_saveds_count == 0) || compiler->scratches > 11) {
934 offset -= SSIZE_OF(sw);
935 EMIT_MOV(compiler, SLJIT_S0, 0, SLJIT_MEM1(SLJIT_SP), offset);
936 }
937
938 /* Copy fourth argument and return address. */
939 offset = stack_size - SSIZE_OF(sw);
940 *extra_space = args_size;
941
942 if (word_arg_count >= 4) {
943 offset -= SSIZE_OF(sw);
944 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
945 }
946
947 while (types != 0) {
948 switch (types & SLJIT_ARG_MASK) {
949 case SLJIT_ARG_TYPE_F64:
950 offset -= SSIZE_OF(f64);
951 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), offset, float_arg_count));
952 float_arg_count--;
953 break;
954 case SLJIT_ARG_TYPE_F32:
955 offset -= SSIZE_OF(f32);
956 FAIL_IF(emit_sse2_store(compiler, 0, SLJIT_MEM1(SLJIT_SP), offset, float_arg_count));
957 float_arg_count--;
958 break;
959 default:
960 switch (word_arg_count) {
961 case 1:
962 offset -= SSIZE_OF(sw);
963 if (r2_offset != 0) {
964 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), 0);
965 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
966 } else
967 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R0, 0);
968 break;
969 case 2:
970 offset -= SSIZE_OF(sw);
971 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R1, 0);
972 break;
973 case 3:
974 offset -= SSIZE_OF(sw);
975 EMIT_MOV(compiler, SLJIT_R2, 0, SLJIT_MEM1(SLJIT_SP), r2_offset);
976 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, SLJIT_R2, 0);
977 break;
978 }
979 word_arg_count--;
980 break;
981 }
982 types >>= SLJIT_ARG_SHIFT;
983 }
984
985 SLJIT_ASSERT(offset >= 0);
986
987 if (offset == 0)
988 return SLJIT_SUCCESS;
989
990 BINARY_IMM32(ADD, offset, SLJIT_SP, 0);
991 return SLJIT_SUCCESS;
992 }
993
emit_tail_call_end(struct sljit_compiler * compiler,sljit_s32 extra_space)994 static sljit_s32 emit_tail_call_end(struct sljit_compiler *compiler, sljit_s32 extra_space)
995 {
996 /* Called when stack consumption cannot be reduced to 0. */
997 sljit_u8 *inst;
998
999 BINARY_IMM32(ADD, extra_space, SLJIT_SP, 0);
1000 return emit_byte(compiler, RET_near);
1001 }
1002
tail_call_reg_arg_with_args(struct sljit_compiler * compiler,sljit_s32 arg_types)1003 static sljit_s32 tail_call_reg_arg_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types)
1004 {
1005 sljit_s32 word_arg_count = 0;
1006 sljit_s32 kept_saveds_count, offset;
1007
1008 arg_types >>= SLJIT_ARG_SHIFT;
1009
1010 while (arg_types) {
1011 if ((arg_types & SLJIT_ARG_MASK) < SLJIT_ARG_TYPE_F64)
1012 word_arg_count++;
1013
1014 arg_types >>= SLJIT_ARG_SHIFT;
1015 }
1016
1017 if (word_arg_count < 4)
1018 return SLJIT_SUCCESS;
1019
1020 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), 2 * SSIZE_OF(sw));
1021
1022 kept_saveds_count = SLJIT_KEPT_SAVEDS_COUNT(compiler->options);
1023 offset = compiler->local_size + 3 * SSIZE_OF(sw);
1024
1025 if ((compiler->saveds > 0 && kept_saveds_count == 0) || compiler->scratches > 11)
1026 offset += SSIZE_OF(sw);
1027 if ((compiler->saveds > 1 && kept_saveds_count <= 1) || compiler->scratches > 10)
1028 offset += SSIZE_OF(sw);
1029 if ((compiler->saveds > 2 && kept_saveds_count <= 2) || compiler->scratches > 9)
1030 offset += SSIZE_OF(sw);
1031
1032 return emit_mov(compiler, SLJIT_MEM1(SLJIT_SP), offset, TMP_REG1, 0);
1033 }
1034
sljit_emit_call(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 arg_types)1035 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
1036 sljit_s32 arg_types)
1037 {
1038 struct sljit_jump *jump;
1039 sljit_sw stack_size = 0;
1040 sljit_s32 word_arg_count;
1041
1042 CHECK_ERROR_PTR();
1043 CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
1044
1045 if (type & SLJIT_CALL_RETURN) {
1046 if ((type & 0xff) == SLJIT_CALL_REG_ARG) {
1047 PTR_FAIL_IF(tail_call_reg_arg_with_args(compiler, arg_types));
1048 PTR_FAIL_IF(emit_stack_frame_release(compiler, 0));
1049
1050 SLJIT_SKIP_CHECKS(compiler);
1051 return sljit_emit_jump(compiler, SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP));
1052 }
1053
1054 stack_size = type;
1055 PTR_FAIL_IF(tail_call_with_args(compiler, &stack_size, arg_types, SLJIT_IMM, 0));
1056
1057 SLJIT_SKIP_CHECKS(compiler);
1058
1059 if (stack_size == 0)
1060 return sljit_emit_jump(compiler, SLJIT_JUMP | (type & SLJIT_REWRITABLE_JUMP));
1061
1062 jump = sljit_emit_jump(compiler, type);
1063 PTR_FAIL_IF(jump == NULL);
1064
1065 PTR_FAIL_IF(emit_tail_call_end(compiler, stack_size));
1066 return jump;
1067 }
1068
1069 if ((type & 0xff) == SLJIT_CALL_REG_ARG) {
1070 SLJIT_SKIP_CHECKS(compiler);
1071 return sljit_emit_jump(compiler, type);
1072 }
1073
1074 stack_size = call_get_stack_size(arg_types, &word_arg_count);
1075 PTR_FAIL_IF(call_with_args(compiler, arg_types, stack_size, word_arg_count, 0));
1076
1077 SLJIT_SKIP_CHECKS(compiler);
1078 jump = sljit_emit_jump(compiler, type);
1079 PTR_FAIL_IF(jump == NULL);
1080
1081 PTR_FAIL_IF(post_call_with_args(compiler, arg_types, stack_size));
1082 return jump;
1083 }
1084
sljit_emit_icall(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 arg_types,sljit_s32 src,sljit_sw srcw)1085 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
1086 sljit_s32 arg_types,
1087 sljit_s32 src, sljit_sw srcw)
1088 {
1089 sljit_sw stack_size = 0;
1090 sljit_s32 word_arg_count;
1091
1092 CHECK_ERROR();
1093 CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
1094
1095 if (type & SLJIT_CALL_RETURN) {
1096 if ((type & 0xff) == SLJIT_CALL_REG_ARG) {
1097 FAIL_IF(tail_call_reg_arg_with_args(compiler, arg_types));
1098
1099 if ((src & SLJIT_MEM) || (src > SLJIT_R2 && src <= (SLJIT_S0 - SLJIT_KEPT_SAVEDS_COUNT(compiler->options)))) {
1100 ADJUST_LOCAL_OFFSET(src, srcw);
1101 CHECK_EXTRA_REGS(src, srcw, (void)0);
1102
1103 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1104 src = TMP_REG1;
1105 srcw = 0;
1106 }
1107
1108 FAIL_IF(emit_stack_frame_release(compiler, 0));
1109
1110 SLJIT_SKIP_CHECKS(compiler);
1111 return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
1112 }
1113
1114 stack_size = type;
1115 FAIL_IF(tail_call_with_args(compiler, &stack_size, arg_types, src, srcw));
1116
1117 if (src != SLJIT_IMM) {
1118 src = SLJIT_R0;
1119 srcw = 0;
1120 }
1121
1122 SLJIT_SKIP_CHECKS(compiler);
1123
1124 if (stack_size == 0)
1125 return sljit_emit_ijump(compiler, SLJIT_JUMP, src, srcw);
1126
1127 FAIL_IF(sljit_emit_ijump(compiler, type, src, srcw));
1128 return emit_tail_call_end(compiler, stack_size);
1129 }
1130
1131 if ((type & 0xff) == SLJIT_CALL_REG_ARG) {
1132 SLJIT_SKIP_CHECKS(compiler);
1133 return sljit_emit_ijump(compiler, type, src, srcw);
1134 }
1135
1136 ADJUST_LOCAL_OFFSET(src, srcw);
1137 CHECK_EXTRA_REGS(src, srcw, (void)0);
1138
1139 if (src & SLJIT_MEM) {
1140 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1141 src = TMP_REG1;
1142 srcw = 0;
1143 }
1144
1145 stack_size = call_get_stack_size(arg_types, &word_arg_count);
1146 FAIL_IF(call_with_args(compiler, arg_types, stack_size, word_arg_count, src == TMP_REG1));
1147
1148 if (stack_size > 0 && src == SLJIT_MEM1(SLJIT_SP))
1149 srcw += stack_size;
1150
1151 SLJIT_SKIP_CHECKS(compiler);
1152 FAIL_IF(sljit_emit_ijump(compiler, type, src, srcw));
1153
1154 return post_call_with_args(compiler, arg_types, stack_size);
1155 }
1156
emit_fmov_before_return(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src,sljit_sw srcw)1157 static SLJIT_INLINE sljit_s32 emit_fmov_before_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
1158 {
1159 sljit_u8* inst;
1160
1161 if (compiler->options & SLJIT_ENTER_REG_ARG) {
1162 if (src == SLJIT_FR0)
1163 return SLJIT_SUCCESS;
1164
1165 SLJIT_SKIP_CHECKS(compiler);
1166 return sljit_emit_fop1(compiler, op, SLJIT_RETURN_FREG, 0, src, srcw);
1167 }
1168
1169 if (FAST_IS_REG(src)) {
1170 FAIL_IF(emit_sse2_store(compiler, op & SLJIT_32, SLJIT_MEM1(SLJIT_SP), 0, src));
1171
1172 src = SLJIT_MEM1(SLJIT_SP);
1173 srcw = 0;
1174 } else {
1175 ADJUST_LOCAL_OFFSET(src, srcw);
1176 }
1177
1178 inst = emit_x86_instruction(compiler, 1 | EX86_SSE2_OP1, 0, 0, src, srcw);
1179 *inst = (op & SLJIT_32) ? FLDS : FLDL;
1180
1181 return SLJIT_SUCCESS;
1182 }
1183
emit_fast_enter(struct sljit_compiler * compiler,sljit_s32 dst,sljit_sw dstw)1184 static sljit_s32 emit_fast_enter(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
1185 {
1186 sljit_u8 *inst;
1187
1188 CHECK_EXTRA_REGS(dst, dstw, (void)0);
1189
1190 /* Unused dest is possible here. */
1191 if (FAST_IS_REG(dst))
1192 return emit_byte(compiler, U8(POP_r + reg_map[dst]));
1193
1194 /* Memory. */
1195 inst = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
1196 FAIL_IF(!inst);
1197 *inst = POP_rm;
1198 return SLJIT_SUCCESS;
1199 }
1200
emit_fast_return(struct sljit_compiler * compiler,sljit_s32 src,sljit_sw srcw)1201 static sljit_s32 emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw)
1202 {
1203 sljit_u8 *inst;
1204
1205 CHECK_EXTRA_REGS(src, srcw, (void)0);
1206
1207 if (FAST_IS_REG(src)) {
1208 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + 1);
1209 FAIL_IF(!inst);
1210
1211 INC_SIZE(1 + 1);
1212 PUSH_REG(reg_map[src]);
1213 }
1214 else {
1215 inst = emit_x86_instruction(compiler, 1, 0, 0, src, srcw);
1216 FAIL_IF(!inst);
1217 inst[0] = GROUP_FF;
1218 inst[1] |= PUSH_rm;
1219
1220 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
1221 FAIL_IF(!inst);
1222 INC_SIZE(1);
1223 }
1224
1225 RET();
1226 return SLJIT_SUCCESS;
1227 }
1228
sljit_emit_get_return_address(struct sljit_compiler * compiler,sljit_s32 dst,sljit_sw dstw)1229 static sljit_s32 sljit_emit_get_return_address(struct sljit_compiler *compiler,
1230 sljit_s32 dst, sljit_sw dstw)
1231 {
1232 sljit_s32 options = compiler->options;
1233 sljit_s32 saveds = compiler->saveds;
1234 sljit_s32 scratches = compiler->scratches;
1235
1236 saveds = ((scratches > 9 ? (scratches - 9) : 0) + (saveds <= 3 ? saveds : 3) - SLJIT_KEPT_SAVEDS_COUNT(options)) * SSIZE_OF(sw);
1237
1238 /* Saving ebp. */
1239 if (!(options & SLJIT_ENTER_REG_ARG))
1240 saveds += SSIZE_OF(sw);
1241
1242 return emit_mov(compiler, dst, dstw, SLJIT_MEM1(SLJIT_SP), compiler->local_size + saveds);
1243 }
1244
1245 /* --------------------------------------------------------------------- */
1246 /* Other operations */
1247 /* --------------------------------------------------------------------- */
1248
sljit_emit_select(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 dst_reg,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2_reg)1249 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_select(struct sljit_compiler *compiler, sljit_s32 type,
1250 sljit_s32 dst_reg,
1251 sljit_s32 src1, sljit_sw src1w,
1252 sljit_s32 src2_reg)
1253 {
1254 sljit_s32 dst = dst_reg;
1255 sljit_sw dstw = 0;
1256 sljit_sw src2w = 0;
1257
1258 CHECK_ERROR();
1259 CHECK(check_sljit_emit_select(compiler, type, dst_reg, src1, src1w, src2_reg));
1260
1261 ADJUST_LOCAL_OFFSET(src1, src1w);
1262
1263 CHECK_EXTRA_REGS(dst, dstw, (void)0);
1264 CHECK_EXTRA_REGS(src1, src1w, (void)0);
1265 CHECK_EXTRA_REGS(src2_reg, src2w, (void)0);
1266
1267 type &= ~SLJIT_32;
1268
1269 if (dst & SLJIT_MEM) {
1270 if (src1 == SLJIT_IMM || (!(src1 & SLJIT_MEM) && (src2_reg & SLJIT_MEM))) {
1271 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
1272 src1 = src2_reg;
1273 src1w = src2w;
1274 type ^= 0x1;
1275 } else
1276 EMIT_MOV(compiler, TMP_REG1, 0, src2_reg, src2w);
1277
1278 dst_reg = TMP_REG1;
1279 } else {
1280 if (dst_reg != src2_reg) {
1281 if (dst_reg == src1) {
1282 src1 = src2_reg;
1283 src1w = src2w;
1284 type ^= 0x1;
1285 } else if (ADDRESSING_DEPENDS_ON(src1, dst_reg)) {
1286 EMIT_MOV(compiler, dst_reg, 0, src1, src1w);
1287 src1 = src2_reg;
1288 src1w = src2w;
1289 type ^= 0x1;
1290 } else
1291 EMIT_MOV(compiler, dst_reg, 0, src2_reg, src2w);
1292 }
1293 }
1294
1295 if (sljit_has_cpu_feature(SLJIT_HAS_CMOV) && (src1 != SLJIT_IMM || dst_reg != TMP_REG1)) {
1296 if (SLJIT_UNLIKELY(src1 == SLJIT_IMM)) {
1297 EMIT_MOV(compiler, TMP_REG1, 0, src1, src1w);
1298 src1 = TMP_REG1;
1299 src1w = 0;
1300 }
1301
1302 FAIL_IF(emit_groupf(compiler, U8(get_jump_code((sljit_uw)type) - 0x40), dst_reg, src1, src1w));
1303 } else
1304 FAIL_IF(emit_cmov_generic(compiler, type, dst_reg, src1, src1w));
1305
1306 if (dst & SLJIT_MEM)
1307 return emit_mov(compiler, dst, dstw, TMP_REG1, 0);
1308 return SLJIT_SUCCESS;
1309 }
1310
sljit_emit_mem(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 reg,sljit_s32 mem,sljit_sw memw)1311 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
1312 sljit_s32 reg,
1313 sljit_s32 mem, sljit_sw memw)
1314 {
1315 sljit_u8* inst;
1316 sljit_s32 i, next, reg_idx, offset;
1317 sljit_u8 regs[2];
1318
1319 CHECK_ERROR();
1320 CHECK(check_sljit_emit_mem(compiler, type, reg, mem, memw));
1321
1322 if (!(reg & REG_PAIR_MASK))
1323 return sljit_emit_mem_unaligned(compiler, type, reg, mem, memw);
1324
1325 ADJUST_LOCAL_OFFSET(mem, memw);
1326
1327 regs[0] = U8(REG_PAIR_FIRST(reg));
1328 regs[1] = U8(REG_PAIR_SECOND(reg));
1329
1330 next = SSIZE_OF(sw);
1331
1332 if (!(type & SLJIT_MEM_STORE) && (regs[0] == (mem & REG_MASK) || regs[0] == OFFS_REG(mem))) {
1333 if (regs[1] == (mem & REG_MASK) || regs[1] == OFFS_REG(mem)) {
1334 /* None of them are virtual register so TMP_REG1 will not be used. */
1335 EMIT_MOV(compiler, TMP_REG1, 0, OFFS_REG(mem), 0);
1336
1337 if (regs[1] == OFFS_REG(mem))
1338 next = -SSIZE_OF(sw);
1339
1340 mem = (mem & ~OFFS_REG_MASK) | TO_OFFS_REG(TMP_REG1);
1341 } else {
1342 next = -SSIZE_OF(sw);
1343
1344 if (!(mem & OFFS_REG_MASK))
1345 memw += SSIZE_OF(sw);
1346 }
1347 }
1348
1349 for (i = 0; i < 2; i++) {
1350 reg_idx = next > 0 ? i : (i ^ 0x1);
1351 reg = regs[reg_idx];
1352
1353 offset = -1;
1354
1355 if (reg >= SLJIT_R3 && reg <= SLJIT_S3) {
1356 offset = (2 * SSIZE_OF(sw)) + ((reg) - SLJIT_R3) * SSIZE_OF(sw);
1357 reg = TMP_REG1;
1358
1359 if (type & SLJIT_MEM_STORE)
1360 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), offset);
1361 }
1362
1363 if ((mem & OFFS_REG_MASK) && (reg_idx == 1)) {
1364 inst = (sljit_u8*)ensure_buf(compiler, (sljit_uw)(1 + 4));
1365 FAIL_IF(!inst);
1366
1367 INC_SIZE(4);
1368
1369 inst[0] = (type & SLJIT_MEM_STORE) ? MOV_rm_r : MOV_r_rm;
1370 inst[1] = 0x44 | U8(reg_map[reg] << 3);
1371 inst[2] = U8(memw << 6) | U8(reg_map[OFFS_REG(mem)] << 3) | reg_map[mem & REG_MASK];
1372 inst[3] = sizeof(sljit_sw);
1373 } else if (type & SLJIT_MEM_STORE) {
1374 EMIT_MOV(compiler, mem, memw, reg, 0);
1375 } else {
1376 EMIT_MOV(compiler, reg, 0, mem, memw);
1377 }
1378
1379 if (!(mem & OFFS_REG_MASK))
1380 memw += next;
1381
1382 if (!(type & SLJIT_MEM_STORE) && offset != -1)
1383 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), offset, TMP_REG1, 0);
1384 }
1385
1386 return SLJIT_SUCCESS;
1387 }
1388
sljit_emit_fop1_conv_f64_from_uw(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)1389 static SLJIT_INLINE sljit_s32 sljit_emit_fop1_conv_f64_from_uw(struct sljit_compiler *compiler, sljit_s32 op,
1390 sljit_s32 dst, sljit_sw dstw,
1391 sljit_s32 src, sljit_sw srcw)
1392 {
1393 sljit_s32 dst_r = FAST_IS_REG(dst) ? dst : TMP_FREG;
1394 sljit_u8 *inst, *jump_inst1, *jump_inst2;
1395 sljit_uw size1, size2;
1396
1397 /* Binary representation of 0x80000000. */
1398 static const sljit_f64 f64_high_bit = (sljit_f64)0x80000000ul;
1399
1400 CHECK_EXTRA_REGS(src, srcw, (void)0);
1401
1402 if (!(op & SLJIT_32)) {
1403 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1404
1405 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 1, TMP_REG1, 0);
1406 FAIL_IF(!inst);
1407 inst[1] |= ROL;
1408
1409 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 1, TMP_REG1, 0);
1410 FAIL_IF(!inst);
1411 inst[1] |= SHR;
1412
1413 FAIL_IF(emit_groupf(compiler, CVTSI2SD_x_rm | EX86_PREF_F2 | EX86_SSE2_OP1, dst_r, TMP_REG1, 0));
1414
1415 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1416 FAIL_IF(!inst);
1417 INC_SIZE(2);
1418 inst[0] = U8(get_jump_code(SLJIT_NOT_CARRY) - 0x10);
1419
1420 size1 = compiler->size;
1421 FAIL_IF(emit_groupf(compiler, ADDSD_x_xm | EX86_PREF_F2 | EX86_SSE2, dst_r, SLJIT_MEM0(), (sljit_sw)&f64_high_bit));
1422
1423 inst[1] = U8(compiler->size - size1);
1424
1425 if (dst_r == TMP_FREG)
1426 return emit_sse2_store(compiler, 0, dst, dstw, TMP_FREG);
1427 return SLJIT_SUCCESS;
1428 }
1429
1430 if (!FAST_IS_REG(src)) {
1431 EMIT_MOV(compiler, TMP_REG1, 0, src, srcw);
1432 src = TMP_REG1;
1433 }
1434
1435 BINARY_IMM32(CMP, 0, src, 0);
1436
1437 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1438 FAIL_IF(!inst);
1439 INC_SIZE(2);
1440 inst[0] = JL_i8;
1441 jump_inst1 = inst;
1442
1443 size1 = compiler->size;
1444
1445 FAIL_IF(emit_groupf(compiler, CVTSI2SD_x_rm | EX86_SELECT_F2_F3(op) | EX86_SSE2_OP1, dst_r, src, 0));
1446
1447 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1448 FAIL_IF(!inst);
1449 INC_SIZE(2);
1450 inst[0] = JMP_i8;
1451 jump_inst2 = inst;
1452
1453 size2 = compiler->size;
1454
1455 jump_inst1[1] = U8(size2 - size1);
1456
1457 if (src != TMP_REG1)
1458 EMIT_MOV(compiler, TMP_REG1, 0, src, 0);
1459
1460 inst = emit_x86_instruction(compiler, 1 | EX86_SHIFT_INS, SLJIT_IMM, 1, TMP_REG1, 0);
1461 FAIL_IF(!inst);
1462 inst[1] |= SHR;
1463
1464 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
1465 FAIL_IF(!inst);
1466 INC_SIZE(2);
1467 inst[0] = JNC_i8;
1468 jump_inst1 = inst;
1469
1470 size1 = compiler->size;
1471
1472 BINARY_IMM32(OR, 1, TMP_REG1, 0);
1473 jump_inst1[1] = U8(compiler->size - size1);
1474
1475 FAIL_IF(emit_groupf(compiler, CVTSI2SD_x_rm | EX86_SELECT_F2_F3(op) | EX86_SSE2_OP1, dst_r, TMP_REG1, 0));
1476 FAIL_IF(emit_groupf(compiler, ADDSD_x_xm | EX86_SELECT_F2_F3(op) | EX86_SSE2, dst_r, dst_r, 0));
1477
1478 jump_inst2[1] = U8(compiler->size - size2);
1479
1480 if (dst_r == TMP_FREG)
1481 return emit_sse2_store(compiler, op & SLJIT_32, dst, dstw, TMP_FREG);
1482 return SLJIT_SUCCESS;
1483 }
1484
sljit_emit_fset32(struct sljit_compiler * compiler,sljit_s32 freg,sljit_f32 value)1485 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset32(struct sljit_compiler *compiler,
1486 sljit_s32 freg, sljit_f32 value)
1487 {
1488 sljit_u8 *inst;
1489 union {
1490 sljit_s32 imm;
1491 sljit_f32 value;
1492 } u;
1493
1494 CHECK_ERROR();
1495 CHECK(check_sljit_emit_fset32(compiler, freg, value));
1496
1497 u.value = value;
1498
1499 if (u.imm != 0)
1500 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, u.imm);
1501
1502 inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1503 FAIL_IF(!inst);
1504 INC_SIZE(4);
1505
1506 inst[0] = GROUP_66;
1507 inst[1] = GROUP_0F;
1508
1509 if (u.imm == 0) {
1510 inst[2] = PXOR_x_xm;
1511 inst[3] = U8(freg_map[freg] | (freg_map[freg] << 3) | MOD_REG);
1512 } else {
1513 inst[2] = MOVD_x_rm;
1514 inst[3] = U8(reg_map[TMP_REG1] | (freg_map[freg] << 3) | MOD_REG);
1515 }
1516
1517 return SLJIT_SUCCESS;
1518 }
1519
sljit_emit_fset64(struct sljit_compiler * compiler,sljit_s32 freg,sljit_f64 value)1520 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fset64(struct sljit_compiler *compiler,
1521 sljit_s32 freg, sljit_f64 value)
1522 {
1523 sljit_u8 *inst;
1524 union {
1525 sljit_s32 imm[2];
1526 sljit_f64 value;
1527 } u;
1528
1529 CHECK_ERROR();
1530 CHECK(check_sljit_emit_fset64(compiler, freg, value));
1531
1532 u.value = value;
1533
1534 if (u.imm[0] == 0) {
1535 if (u.imm[1] == 0)
1536 return emit_groupf(compiler, PXOR_x_xm | EX86_PREF_66 | EX86_SSE2, freg, freg, 0);
1537
1538 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, u.imm[1]);
1539 } else {
1540 SLJIT_ASSERT(cpu_feature_list != 0);
1541
1542 if (!(cpu_feature_list & CPU_FEATURE_SSE41) && u.imm[1] != 0 && u.imm[0] != u.imm[1]) {
1543 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), 0, SLJIT_IMM, u.imm[0]);
1544 EMIT_MOV(compiler, SLJIT_MEM1(SLJIT_SP), sizeof(sljit_sw), SLJIT_IMM, u.imm[1]);
1545
1546 return emit_groupf(compiler, MOVLPD_x_m | EX86_SSE2, freg, SLJIT_MEM1(SLJIT_SP), 0);
1547 }
1548
1549 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, u.imm[0]);
1550 }
1551
1552 FAIL_IF(emit_groupf(compiler, MOVD_x_rm | EX86_PREF_66 | EX86_SSE2_OP1, freg, TMP_REG1, 0));
1553
1554 if (u.imm[1] == 0)
1555 return SLJIT_SUCCESS;
1556
1557 if (u.imm[0] == 0) {
1558 inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
1559 FAIL_IF(!inst);
1560 INC_SIZE(4);
1561
1562 inst[0] = GROUP_0F;
1563 inst[1] = SHUFPS_x_xm;
1564 inst[2] = U8(MOD_REG | (freg_map[freg] << 3) | freg_map[freg]);
1565 inst[3] = 0x51;
1566 return SLJIT_SUCCESS;
1567 }
1568
1569 if (u.imm[0] != u.imm[1]) {
1570 SLJIT_ASSERT(cpu_feature_list & CPU_FEATURE_SSE41);
1571 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, u.imm[1]);
1572
1573 FAIL_IF(emit_groupf_ext(compiler, PINSRD_x_rm_i8 | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2_OP1, freg, TMP_REG1, 0));
1574 return emit_byte(compiler, 1);
1575 }
1576
1577 inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
1578 FAIL_IF(!inst);
1579 INC_SIZE(3);
1580
1581 inst[0] = GROUP_0F;
1582 inst[1] = UNPCKLPS_x_xm;
1583 inst[2] = U8(MOD_REG | (freg_map[freg] << 3) | freg_map[freg]);
1584 return SLJIT_SUCCESS;
1585 }
1586
sljit_emit_fcopy(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 freg,sljit_s32 reg)1587 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fcopy(struct sljit_compiler *compiler, sljit_s32 op,
1588 sljit_s32 freg, sljit_s32 reg)
1589 {
1590 sljit_u8 *inst;
1591 sljit_s32 reg2;
1592 sljit_sw regw, reg2w;
1593
1594 CHECK_ERROR();
1595 CHECK(check_sljit_emit_fcopy(compiler, op, freg, reg));
1596
1597 regw = 0;
1598 reg2 = 0;
1599 reg2w = 0;
1600
1601 SLJIT_ASSERT(cpu_feature_list != 0);
1602
1603 if (!(op & SLJIT_32) && (cpu_feature_list & CPU_FEATURE_SSE41)) {
1604 if (reg & REG_PAIR_MASK) {
1605 reg2 = REG_PAIR_FIRST(reg);
1606 reg = REG_PAIR_SECOND(reg);
1607
1608 CHECK_EXTRA_REGS(reg, regw, (void)0);
1609
1610 FAIL_IF(emit_groupf(compiler, (GET_OPCODE(op) == SLJIT_COPY_TO_F64 ? MOVD_x_rm : MOVD_rm_x)
1611 | EX86_PREF_66 | EX86_SSE2_OP1, freg, reg, regw));
1612 } else
1613 reg2 = reg;
1614
1615 CHECK_EXTRA_REGS(reg2, reg2w, (void)0);
1616
1617 FAIL_IF(emit_groupf_ext(compiler, (GET_OPCODE(op) == SLJIT_COPY_TO_F64 ? PINSRD_x_rm_i8 : PEXTRD_rm_x_i8)
1618 | EX86_PREF_66 | VEX_OP_0F3A | EX86_SSE2_OP1, freg, reg2, reg2w));
1619 return emit_byte(compiler, 1);
1620 }
1621
1622 if (reg & REG_PAIR_MASK) {
1623 reg2 = REG_PAIR_SECOND(reg);
1624 reg = REG_PAIR_FIRST(reg);
1625
1626 if (reg == reg2)
1627 reg = 0;
1628
1629 CHECK_EXTRA_REGS(reg2, reg2w, (void)0);
1630 }
1631
1632 CHECK_EXTRA_REGS(reg, regw, (void)0);
1633
1634 if (op & SLJIT_32)
1635 return emit_groupf(compiler, (GET_OPCODE(op) == SLJIT_COPY_TO_F64 ? MOVD_x_rm : MOVD_rm_x)
1636 | EX86_PREF_66 | EX86_SSE2_OP1, freg, reg, regw);
1637
1638 if (op == SLJIT_COPY_FROM_F64) {
1639 inst = (sljit_u8*)ensure_buf(compiler, 1 + 5);
1640 FAIL_IF(!inst);
1641 INC_SIZE(5);
1642
1643 inst[0] = GROUP_66;
1644 inst[1] = GROUP_0F;
1645 inst[2] = PSHUFD_x_xm;
1646 inst[3] = U8(MOD_REG | (TMP_FREG << 3) | freg_map[freg]);
1647 inst[4] = 1;
1648 } else if (reg != 0)
1649 FAIL_IF(emit_groupf(compiler, MOVD_x_rm | EX86_PREF_66 | EX86_SSE2_OP1, TMP_FREG, reg, regw));
1650
1651 if (reg2 != 0)
1652 FAIL_IF(emit_groupf(compiler, (GET_OPCODE(op) == SLJIT_COPY_TO_F64 ? MOVD_x_rm : MOVD_rm_x)
1653 | EX86_PREF_66 | EX86_SSE2_OP1, freg, reg2, reg2w));
1654
1655 if (GET_OPCODE(op) == SLJIT_COPY_TO_F64) {
1656 inst = (sljit_u8*)ensure_buf(compiler, 1 + 3);
1657 FAIL_IF(!inst);
1658 INC_SIZE(3);
1659
1660 inst[0] = GROUP_0F;
1661 inst[1] = UNPCKLPS_x_xm;
1662 inst[2] = U8(MOD_REG | (freg_map[freg] << 3) | freg_map[reg == 0 ? freg : TMP_FREG]);
1663 } else
1664 FAIL_IF(emit_groupf(compiler, MOVD_rm_x | EX86_PREF_66 | EX86_SSE2_OP1, TMP_FREG, reg, regw));
1665
1666 return SLJIT_SUCCESS;
1667 }
1668
skip_frames_before_return(struct sljit_compiler * compiler)1669 static sljit_s32 skip_frames_before_return(struct sljit_compiler *compiler)
1670 {
1671 sljit_sw size;
1672
1673 /* Don't adjust shadow stack if it isn't enabled. */
1674 if (!cpu_has_shadow_stack())
1675 return SLJIT_SUCCESS;
1676
1677 SLJIT_ASSERT(compiler->args_size >= 0);
1678 SLJIT_ASSERT(compiler->local_size > 0);
1679
1680 size = compiler->local_size;
1681 size += (1 + (compiler->scratches > 9 ? (compiler->scratches - 9) : 0)
1682 + (compiler->saveds <= 3 ? compiler->saveds : 3)) * SSIZE_OF(sw);
1683
1684 return adjust_shadow_stack(compiler, SLJIT_MEM1(SLJIT_SP), size);
1685 }
1686
