1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 /* x86 64-bit arch dependent functions. */
28
emit_load_imm64(struct sljit_compiler * compiler,sljit_s32 reg,sljit_sw imm)29 static sljit_s32 emit_load_imm64(struct sljit_compiler *compiler, sljit_s32 reg, sljit_sw imm)
30 {
31 sljit_u8 *inst;
32
33 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2 + sizeof(sljit_sw));
34 FAIL_IF(!inst);
35 INC_SIZE(2 + sizeof(sljit_sw));
36 *inst++ = REX_W | ((reg_map[reg] <= 7) ? 0 : REX_B);
37 *inst++ = MOV_r_i32 + (reg_map[reg] & 0x7);
38 sljit_unaligned_store_sw(inst, imm);
39 return SLJIT_SUCCESS;
40 }
41
generate_far_jump_code(struct sljit_jump * jump,sljit_u8 * code_ptr)42 static sljit_u8* generate_far_jump_code(struct sljit_jump *jump, sljit_u8 *code_ptr)
43 {
44 sljit_s32 type = jump->flags >> TYPE_SHIFT;
45
46 int short_addr = !(jump->flags & SLJIT_REWRITABLE_JUMP) && !(jump->flags & JUMP_LABEL) && (jump->u.target <= 0xffffffff);
47
48 /* The relative jump below specialized for this case. */
49 SLJIT_ASSERT(reg_map[TMP_REG2] >= 8);
50
51 if (type < SLJIT_JUMP) {
52 /* Invert type. */
53 *code_ptr++ = get_jump_code(type ^ 0x1) - 0x10;
54 *code_ptr++ = short_addr ? (6 + 3) : (10 + 3);
55 }
56
57 *code_ptr++ = short_addr ? REX_B : (REX_W | REX_B);
58 *code_ptr++ = MOV_r_i32 | reg_lmap[TMP_REG2];
59 jump->addr = (sljit_uw)code_ptr;
60
61 if (jump->flags & JUMP_LABEL)
62 jump->flags |= PATCH_MD;
63 else if (short_addr)
64 sljit_unaligned_store_s32(code_ptr, (sljit_s32)jump->u.target);
65 else
66 sljit_unaligned_store_sw(code_ptr, jump->u.target);
67
68 code_ptr += short_addr ? sizeof(sljit_s32) : sizeof(sljit_sw);
69
70 *code_ptr++ = REX_B;
71 *code_ptr++ = GROUP_FF;
72 *code_ptr++ = MOD_REG | (type >= SLJIT_FAST_CALL ? CALL_rm : JMP_rm) | reg_lmap[TMP_REG2];
73
74 return code_ptr;
75 }
76
generate_put_label_code(struct sljit_put_label * put_label,sljit_u8 * code_ptr,sljit_uw max_label)77 static sljit_u8* generate_put_label_code(struct sljit_put_label *put_label, sljit_u8 *code_ptr, sljit_uw max_label)
78 {
79 if (max_label > HALFWORD_MAX) {
80 put_label->addr -= put_label->flags;
81 put_label->flags = PATCH_MD;
82 return code_ptr;
83 }
84
85 if (put_label->flags == 0) {
86 /* Destination is register. */
87 code_ptr = (sljit_u8*)put_label->addr - 2 - sizeof(sljit_uw);
88
89 SLJIT_ASSERT((code_ptr[0] & 0xf8) == REX_W);
90 SLJIT_ASSERT((code_ptr[1] & 0xf8) == MOV_r_i32);
91
92 if ((code_ptr[0] & 0x07) != 0) {
93 code_ptr[0] = (sljit_u8)(code_ptr[0] & ~0x08);
94 code_ptr += 2 + sizeof(sljit_s32);
95 }
96 else {
97 code_ptr[0] = code_ptr[1];
98 code_ptr += 1 + sizeof(sljit_s32);
99 }
100
101 put_label->addr = (sljit_uw)code_ptr;
102 return code_ptr;
103 }
104
105 code_ptr -= put_label->flags + (2 + sizeof(sljit_uw));
106 SLJIT_MEMMOVE(code_ptr, code_ptr + (2 + sizeof(sljit_uw)), put_label->flags);
107
108 SLJIT_ASSERT((code_ptr[0] & 0xf8) == REX_W);
109
110 if ((code_ptr[1] & 0xf8) == MOV_r_i32) {
111 code_ptr += 2 + sizeof(sljit_uw);
112 SLJIT_ASSERT((code_ptr[0] & 0xf8) == REX_W);
113 }
114
115 SLJIT_ASSERT(code_ptr[1] == MOV_rm_r);
116
117 code_ptr[0] = (sljit_u8)(code_ptr[0] & ~0x4);
118 code_ptr[1] = MOV_rm_i32;
119 code_ptr[2] = (sljit_u8)(code_ptr[2] & ~(0x7 << 3));
120
121 code_ptr = (sljit_u8*)(put_label->addr - (2 + sizeof(sljit_uw)) + sizeof(sljit_s32));
122 put_label->addr = (sljit_uw)code_ptr;
123 put_label->flags = 0;
124 return code_ptr;
125 }
126
sljit_emit_enter(struct sljit_compiler * compiler,sljit_s32 options,sljit_s32 arg_types,sljit_s32 scratches,sljit_s32 saveds,sljit_s32 fscratches,sljit_s32 fsaveds,sljit_s32 local_size)127 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_enter(struct sljit_compiler *compiler,
128 sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
129 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
130 {
131 sljit_s32 args, i, tmp, size, saved_register_size;
132 sljit_u8 *inst;
133
134 CHECK_ERROR();
135 CHECK(check_sljit_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
136 set_emit_enter(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
137
138 /* Emit ENDBR64 at function entry if needed. */
139 FAIL_IF(emit_endbranch(compiler));
140
141 compiler->mode32 = 0;
142
143 #ifdef _WIN64
144 /* Two/four register slots for parameters plus space for xmm6 register if needed. */
145 if (fscratches >= 6 || fsaveds >= 1)
146 compiler->locals_offset = 6 * sizeof(sljit_sw);
147 else
148 compiler->locals_offset = ((scratches > 2) ? 4 : 2) * sizeof(sljit_sw);
149 #endif
150
151 /* Including the return address saved by the call instruction. */
152 saved_register_size = GET_SAVED_REGISTERS_SIZE(scratches, saveds, 1);
153
154 tmp = saveds < SLJIT_NUMBER_OF_SAVED_REGISTERS ? (SLJIT_S0 + 1 - saveds) : SLJIT_FIRST_SAVED_REG;
155 for (i = SLJIT_S0; i >= tmp; i--) {
156 size = reg_map[i] >= 8 ? 2 : 1;
157 inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
158 FAIL_IF(!inst);
159 INC_SIZE(size);
160 if (reg_map[i] >= 8)
161 *inst++ = REX_B;
162 PUSH_REG(reg_lmap[i]);
163 }
164
165 for (i = scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
166 size = reg_map[i] >= 8 ? 2 : 1;
167 inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
168 FAIL_IF(!inst);
169 INC_SIZE(size);
170 if (reg_map[i] >= 8)
171 *inst++ = REX_B;
172 PUSH_REG(reg_lmap[i]);
173 }
174
175 args = get_arg_count(arg_types);
176
177 if (args > 0) {
178 size = args * 3;
179 inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
180 FAIL_IF(!inst);
181
182 INC_SIZE(size);
183
184 #ifndef _WIN64
185 if (args > 0) {
186 inst[0] = REX_W;
187 inst[1] = MOV_r_rm;
188 inst[2] = MOD_REG | (reg_map[SLJIT_S0] << 3) | 0x7 /* rdi */;
189 inst += 3;
190 }
191 if (args > 1) {
192 inst[0] = REX_W | REX_R;
193 inst[1] = MOV_r_rm;
194 inst[2] = MOD_REG | (reg_lmap[SLJIT_S1] << 3) | 0x6 /* rsi */;
195 inst += 3;
196 }
197 if (args > 2) {
198 inst[0] = REX_W | REX_R;
199 inst[1] = MOV_r_rm;
200 inst[2] = MOD_REG | (reg_lmap[SLJIT_S2] << 3) | 0x2 /* rdx */;
201 }
202 #else
203 if (args > 0) {
204 inst[0] = REX_W;
205 inst[1] = MOV_r_rm;
206 inst[2] = MOD_REG | (reg_map[SLJIT_S0] << 3) | 0x1 /* rcx */;
207 inst += 3;
208 }
209 if (args > 1) {
210 inst[0] = REX_W;
211 inst[1] = MOV_r_rm;
212 inst[2] = MOD_REG | (reg_map[SLJIT_S1] << 3) | 0x2 /* rdx */;
213 inst += 3;
214 }
215 if (args > 2) {
216 inst[0] = REX_W | REX_B;
217 inst[1] = MOV_r_rm;
218 inst[2] = MOD_REG | (reg_map[SLJIT_S2] << 3) | 0x0 /* r8 */;
219 }
220 #endif
221 }
222
223 local_size = ((local_size + SLJIT_LOCALS_OFFSET + saved_register_size + 15) & ~15) - saved_register_size;
224 compiler->local_size = local_size;
225
226 #ifdef _WIN64
227 if (local_size > 0) {
228 if (local_size <= 4 * 4096) {
229 if (local_size > 4096)
230 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), -4096);
231 if (local_size > 2 * 4096)
232 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), -4096 * 2);
233 if (local_size > 3 * 4096)
234 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), -4096 * 3);
235 }
236 else {
237 EMIT_MOV(compiler, SLJIT_R0, 0, SLJIT_SP, 0);
238 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_IMM, (local_size - 1) >> 12);
239
240 SLJIT_ASSERT (reg_map[SLJIT_R0] == 0);
241
242 EMIT_MOV(compiler, TMP_REG2, 0, SLJIT_MEM1(SLJIT_R0), -4096);
243 FAIL_IF(emit_non_cum_binary(compiler, BINARY_OPCODE(SUB),
244 SLJIT_R0, 0, SLJIT_R0, 0, SLJIT_IMM, 4096));
245 FAIL_IF(emit_non_cum_binary(compiler, BINARY_OPCODE(SUB),
246 TMP_REG1, 0, TMP_REG1, 0, SLJIT_IMM, 1));
247
248 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
249 FAIL_IF(!inst);
250
251 INC_SIZE(2);
252 inst[0] = JNE_i8;
253 inst[1] = (sljit_s8) -19;
254 }
255
256 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_MEM1(SLJIT_SP), -local_size);
257 }
258 #endif
259
260 if (local_size > 0) {
261 FAIL_IF(emit_non_cum_binary(compiler, BINARY_OPCODE(SUB),
262 SLJIT_SP, 0, SLJIT_SP, 0, SLJIT_IMM, local_size));
263 }
264
265 #ifdef _WIN64
266 /* Save xmm6 register: movaps [rsp + 0x20], xmm6 */
267 if (fscratches >= 6 || fsaveds >= 1) {
268 inst = (sljit_u8*)ensure_buf(compiler, 1 + 5);
269 FAIL_IF(!inst);
270 INC_SIZE(5);
271 *inst++ = GROUP_0F;
272 sljit_unaligned_store_s32(inst, 0x20247429);
273 }
274 #endif
275
276 return SLJIT_SUCCESS;
277 }
278
sljit_set_context(struct sljit_compiler * compiler,sljit_s32 options,sljit_s32 arg_types,sljit_s32 scratches,sljit_s32 saveds,sljit_s32 fscratches,sljit_s32 fsaveds,sljit_s32 local_size)279 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_set_context(struct sljit_compiler *compiler,
280 sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
281 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
282 {
283 sljit_s32 saved_register_size;
284
285 CHECK_ERROR();
286 CHECK(check_sljit_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size));
287 set_set_context(compiler, options, arg_types, scratches, saveds, fscratches, fsaveds, local_size);
288
289 #ifdef _WIN64
290 /* Two/four register slots for parameters plus space for xmm6 register if needed. */
291 if (fscratches >= 6 || fsaveds >= 1)
292 compiler->locals_offset = 6 * sizeof(sljit_sw);
293 else
294 compiler->locals_offset = ((scratches > 2) ? 4 : 2) * sizeof(sljit_sw);
295 #endif
296
297 /* Including the return address saved by the call instruction. */
298 saved_register_size = GET_SAVED_REGISTERS_SIZE(scratches, saveds, 1);
299 compiler->local_size = ((local_size + SLJIT_LOCALS_OFFSET + saved_register_size + 15) & ~15) - saved_register_size;
300 return SLJIT_SUCCESS;
301 }
302
sljit_emit_return(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src,sljit_sw srcw)303 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
304 {
305 sljit_s32 i, tmp, size;
306 sljit_u8 *inst;
307
308 CHECK_ERROR();
309 CHECK(check_sljit_emit_return(compiler, op, src, srcw));
310
311 FAIL_IF(emit_mov_before_return(compiler, op, src, srcw));
312
313 #ifdef _WIN64
314 /* Restore xmm6 register: movaps xmm6, [rsp + 0x20] */
315 if (compiler->fscratches >= 6 || compiler->fsaveds >= 1) {
316 inst = (sljit_u8*)ensure_buf(compiler, 1 + 5);
317 FAIL_IF(!inst);
318 INC_SIZE(5);
319 *inst++ = GROUP_0F;
320 sljit_unaligned_store_s32(inst, 0x20247428);
321 }
322 #endif
323
324 if (compiler->local_size > 0) {
325 if (compiler->local_size <= 127) {
326 inst = (sljit_u8*)ensure_buf(compiler, 1 + 4);
327 FAIL_IF(!inst);
328 INC_SIZE(4);
329 *inst++ = REX_W;
330 *inst++ = GROUP_BINARY_83;
331 *inst++ = MOD_REG | ADD | 4;
332 *inst = compiler->local_size;
333 }
334 else {
335 inst = (sljit_u8*)ensure_buf(compiler, 1 + 7);
336 FAIL_IF(!inst);
337 INC_SIZE(7);
338 *inst++ = REX_W;
339 *inst++ = GROUP_BINARY_81;
340 *inst++ = MOD_REG | ADD | 4;
341 sljit_unaligned_store_s32(inst, compiler->local_size);
342 }
343 }
344
345 tmp = compiler->scratches;
346 for (i = SLJIT_FIRST_SAVED_REG; i <= tmp; i++) {
347 size = reg_map[i] >= 8 ? 2 : 1;
348 inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
349 FAIL_IF(!inst);
350 INC_SIZE(size);
351 if (reg_map[i] >= 8)
352 *inst++ = REX_B;
353 POP_REG(reg_lmap[i]);
354 }
355
356 tmp = compiler->saveds < SLJIT_NUMBER_OF_SAVED_REGISTERS ? (SLJIT_S0 + 1 - compiler->saveds) : SLJIT_FIRST_SAVED_REG;
357 for (i = tmp; i <= SLJIT_S0; i++) {
358 size = reg_map[i] >= 8 ? 2 : 1;
359 inst = (sljit_u8*)ensure_buf(compiler, 1 + size);
360 FAIL_IF(!inst);
361 INC_SIZE(size);
362 if (reg_map[i] >= 8)
363 *inst++ = REX_B;
364 POP_REG(reg_lmap[i]);
365 }
366
367 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
368 FAIL_IF(!inst);
369 INC_SIZE(1);
370 RET();
371 return SLJIT_SUCCESS;
372 }
373
374 /* --------------------------------------------------------------------- */
375 /* Operators */
376 /* --------------------------------------------------------------------- */
377
emit_do_imm32(struct sljit_compiler * compiler,sljit_u8 rex,sljit_u8 opcode,sljit_sw imm)378 static sljit_s32 emit_do_imm32(struct sljit_compiler *compiler, sljit_u8 rex, sljit_u8 opcode, sljit_sw imm)
379 {
380 sljit_u8 *inst;
381 sljit_s32 length = 1 + (rex ? 1 : 0) + sizeof(sljit_s32);
382
383 inst = (sljit_u8*)ensure_buf(compiler, 1 + length);
384 FAIL_IF(!inst);
385 INC_SIZE(length);
386 if (rex)
387 *inst++ = rex;
388 *inst++ = opcode;
389 sljit_unaligned_store_s32(inst, imm);
390 return SLJIT_SUCCESS;
391 }
392
emit_x86_instruction(struct sljit_compiler * compiler,sljit_s32 size,sljit_s32 a,sljit_sw imma,sljit_s32 b,sljit_sw immb)393 static sljit_u8* emit_x86_instruction(struct sljit_compiler *compiler, sljit_s32 size,
394 /* The register or immediate operand. */
395 sljit_s32 a, sljit_sw imma,
396 /* The general operand (not immediate). */
397 sljit_s32 b, sljit_sw immb)
398 {
399 sljit_u8 *inst;
400 sljit_u8 *buf_ptr;
401 sljit_u8 rex = 0;
402 sljit_s32 flags = size & ~0xf;
403 sljit_s32 inst_size;
404
405 /* The immediate operand must be 32 bit. */
406 SLJIT_ASSERT(!(a & SLJIT_IMM) || compiler->mode32 || IS_HALFWORD(imma));
407 /* Both cannot be switched on. */
408 SLJIT_ASSERT((flags & (EX86_BIN_INS | EX86_SHIFT_INS)) != (EX86_BIN_INS | EX86_SHIFT_INS));
409 /* Size flags not allowed for typed instructions. */
410 SLJIT_ASSERT(!(flags & (EX86_BIN_INS | EX86_SHIFT_INS)) || (flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) == 0);
411 /* Both size flags cannot be switched on. */
412 SLJIT_ASSERT((flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) != (EX86_BYTE_ARG | EX86_HALF_ARG));
413 /* SSE2 and immediate is not possible. */
414 SLJIT_ASSERT(!(a & SLJIT_IMM) || !(flags & EX86_SSE2));
415 SLJIT_ASSERT((flags & (EX86_PREF_F2 | EX86_PREF_F3)) != (EX86_PREF_F2 | EX86_PREF_F3)
416 && (flags & (EX86_PREF_F2 | EX86_PREF_66)) != (EX86_PREF_F2 | EX86_PREF_66)
417 && (flags & (EX86_PREF_F3 | EX86_PREF_66)) != (EX86_PREF_F3 | EX86_PREF_66));
418
419 size &= 0xf;
420 inst_size = size;
421
422 if (!compiler->mode32 && !(flags & EX86_NO_REXW))
423 rex |= REX_W;
424 else if (flags & EX86_REX)
425 rex |= REX;
426
427 if (flags & (EX86_PREF_F2 | EX86_PREF_F3))
428 inst_size++;
429 if (flags & EX86_PREF_66)
430 inst_size++;
431
432 /* Calculate size of b. */
433 inst_size += 1; /* mod r/m byte. */
434 if (b & SLJIT_MEM) {
435 if (!(b & OFFS_REG_MASK)) {
436 if (NOT_HALFWORD(immb)) {
437 PTR_FAIL_IF(emit_load_imm64(compiler, TMP_REG2, immb));
438 immb = 0;
439 if (b & REG_MASK)
440 b |= TO_OFFS_REG(TMP_REG2);
441 else
442 b |= TMP_REG2;
443 }
444 else if (reg_lmap[b & REG_MASK] == 4)
445 b |= TO_OFFS_REG(SLJIT_SP);
446 }
447
448 if ((b & REG_MASK) == SLJIT_UNUSED)
449 inst_size += 1 + sizeof(sljit_s32); /* SIB byte required to avoid RIP based addressing. */
450 else {
451 if (reg_map[b & REG_MASK] >= 8)
452 rex |= REX_B;
453
454 if (immb != 0 && (!(b & OFFS_REG_MASK) || (b & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_SP))) {
455 /* Immediate operand. */
456 if (immb <= 127 && immb >= -128)
457 inst_size += sizeof(sljit_s8);
458 else
459 inst_size += sizeof(sljit_s32);
460 }
461 else if (reg_lmap[b & REG_MASK] == 5)
462 inst_size += sizeof(sljit_s8);
463
464 if ((b & OFFS_REG_MASK) != SLJIT_UNUSED) {
465 inst_size += 1; /* SIB byte. */
466 if (reg_map[OFFS_REG(b)] >= 8)
467 rex |= REX_X;
468 }
469 }
470 }
471 else if (!(flags & EX86_SSE2_OP2)) {
472 if (reg_map[b] >= 8)
473 rex |= REX_B;
474 }
475 else if (freg_map[b] >= 8)
476 rex |= REX_B;
477
478 if (a & SLJIT_IMM) {
479 if (flags & EX86_BIN_INS) {
480 if (imma <= 127 && imma >= -128) {
481 inst_size += 1;
482 flags |= EX86_BYTE_ARG;
483 } else
484 inst_size += 4;
485 }
486 else if (flags & EX86_SHIFT_INS) {
487 imma &= compiler->mode32 ? 0x1f : 0x3f;
488 if (imma != 1) {
489 inst_size ++;
490 flags |= EX86_BYTE_ARG;
491 }
492 } else if (flags & EX86_BYTE_ARG)
493 inst_size++;
494 else if (flags & EX86_HALF_ARG)
495 inst_size += sizeof(short);
496 else
497 inst_size += sizeof(sljit_s32);
498 }
499 else {
500 SLJIT_ASSERT(!(flags & EX86_SHIFT_INS) || a == SLJIT_PREF_SHIFT_REG);
501 /* reg_map[SLJIT_PREF_SHIFT_REG] is less than 8. */
502 if (!(flags & EX86_SSE2_OP1)) {
503 if (reg_map[a] >= 8)
504 rex |= REX_R;
505 }
506 else if (freg_map[a] >= 8)
507 rex |= REX_R;
508 }
509
510 if (rex)
511 inst_size++;
512
513 inst = (sljit_u8*)ensure_buf(compiler, 1 + inst_size);
514 PTR_FAIL_IF(!inst);
515
516 /* Encoding the byte. */
517 INC_SIZE(inst_size);
518 if (flags & EX86_PREF_F2)
519 *inst++ = 0xf2;
520 if (flags & EX86_PREF_F3)
521 *inst++ = 0xf3;
522 if (flags & EX86_PREF_66)
523 *inst++ = 0x66;
524 if (rex)
525 *inst++ = rex;
526 buf_ptr = inst + size;
527
528 /* Encode mod/rm byte. */
529 if (!(flags & EX86_SHIFT_INS)) {
530 if ((flags & EX86_BIN_INS) && (a & SLJIT_IMM))
531 *inst = (flags & EX86_BYTE_ARG) ? GROUP_BINARY_83 : GROUP_BINARY_81;
532
533 if (a & SLJIT_IMM)
534 *buf_ptr = 0;
535 else if (!(flags & EX86_SSE2_OP1))
536 *buf_ptr = reg_lmap[a] << 3;
537 else
538 *buf_ptr = freg_lmap[a] << 3;
539 }
540 else {
541 if (a & SLJIT_IMM) {
542 if (imma == 1)
543 *inst = GROUP_SHIFT_1;
544 else
545 *inst = GROUP_SHIFT_N;
546 } else
547 *inst = GROUP_SHIFT_CL;
548 *buf_ptr = 0;
549 }
550
551 if (!(b & SLJIT_MEM))
552 *buf_ptr++ |= MOD_REG + ((!(flags & EX86_SSE2_OP2)) ? reg_lmap[b] : freg_lmap[b]);
553 else if ((b & REG_MASK) != SLJIT_UNUSED) {
554 if ((b & OFFS_REG_MASK) == SLJIT_UNUSED || (b & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_SP)) {
555 if (immb != 0 || reg_lmap[b & REG_MASK] == 5) {
556 if (immb <= 127 && immb >= -128)
557 *buf_ptr |= 0x40;
558 else
559 *buf_ptr |= 0x80;
560 }
561
562 if ((b & OFFS_REG_MASK) == SLJIT_UNUSED)
563 *buf_ptr++ |= reg_lmap[b & REG_MASK];
564 else {
565 *buf_ptr++ |= 0x04;
566 *buf_ptr++ = reg_lmap[b & REG_MASK] | (reg_lmap[OFFS_REG(b)] << 3);
567 }
568
569 if (immb != 0 || reg_lmap[b & REG_MASK] == 5) {
570 if (immb <= 127 && immb >= -128)
571 *buf_ptr++ = immb; /* 8 bit displacement. */
572 else {
573 sljit_unaligned_store_s32(buf_ptr, immb); /* 32 bit displacement. */
574 buf_ptr += sizeof(sljit_s32);
575 }
576 }
577 }
578 else {
579 if (reg_lmap[b & REG_MASK] == 5)
580 *buf_ptr |= 0x40;
581 *buf_ptr++ |= 0x04;
582 *buf_ptr++ = reg_lmap[b & REG_MASK] | (reg_lmap[OFFS_REG(b)] << 3) | (immb << 6);
583 if (reg_lmap[b & REG_MASK] == 5)
584 *buf_ptr++ = 0;
585 }
586 }
587 else {
588 *buf_ptr++ |= 0x04;
589 *buf_ptr++ = 0x25;
590 sljit_unaligned_store_s32(buf_ptr, immb); /* 32 bit displacement. */
591 buf_ptr += sizeof(sljit_s32);
592 }
593
594 if (a & SLJIT_IMM) {
595 if (flags & EX86_BYTE_ARG)
596 *buf_ptr = imma;
597 else if (flags & EX86_HALF_ARG)
598 sljit_unaligned_store_s16(buf_ptr, imma);
599 else if (!(flags & EX86_SHIFT_INS))
600 sljit_unaligned_store_s32(buf_ptr, imma);
601 }
602
603 return !(flags & EX86_SHIFT_INS) ? inst : (inst + 1);
604 }
605
606 /* --------------------------------------------------------------------- */
607 /* Call / return instructions */
608 /* --------------------------------------------------------------------- */
609
610 #ifndef _WIN64
611
call_with_args(struct sljit_compiler * compiler,sljit_s32 arg_types,sljit_s32 * src_ptr,sljit_sw srcw)612 static sljit_s32 call_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types, sljit_s32 *src_ptr, sljit_sw srcw)
613 {
614 sljit_s32 src = src_ptr ? (*src_ptr) : 0;
615 sljit_s32 word_arg_count = 0;
616
617 SLJIT_ASSERT(reg_map[SLJIT_R1] == 6 && reg_map[SLJIT_R3] == 1 && reg_map[TMP_REG1] == 2);
618
619 compiler->mode32 = 0;
620
621 /* Remove return value. */
622 arg_types >>= SLJIT_DEF_SHIFT;
623
624 while (arg_types) {
625 if ((arg_types & SLJIT_DEF_MASK) < SLJIT_ARG_TYPE_F32)
626 word_arg_count++;
627 arg_types >>= SLJIT_DEF_SHIFT;
628 }
629
630 if (word_arg_count == 0)
631 return SLJIT_SUCCESS;
632
633 if (src & SLJIT_MEM) {
634 ADJUST_LOCAL_OFFSET(src, srcw);
635 EMIT_MOV(compiler, TMP_REG2, 0, src, srcw);
636 *src_ptr = TMP_REG2;
637 }
638 else if (src == SLJIT_R2 && word_arg_count >= SLJIT_R2)
639 *src_ptr = TMP_REG1;
640
641 if (word_arg_count >= 3)
642 EMIT_MOV(compiler, TMP_REG1, 0, SLJIT_R2, 0);
643 return emit_mov(compiler, SLJIT_R2, 0, SLJIT_R0, 0);
644 }
645
646 #else
647
call_with_args(struct sljit_compiler * compiler,sljit_s32 arg_types,sljit_s32 * src_ptr,sljit_sw srcw)648 static sljit_s32 call_with_args(struct sljit_compiler *compiler, sljit_s32 arg_types, sljit_s32 *src_ptr, sljit_sw srcw)
649 {
650 sljit_s32 src = src_ptr ? (*src_ptr) : 0;
651 sljit_s32 arg_count = 0;
652 sljit_s32 word_arg_count = 0;
653 sljit_s32 float_arg_count = 0;
654 sljit_s32 types = 0;
655 sljit_s32 data_trandfer = 0;
656 static sljit_u8 word_arg_regs[5] = { 0, SLJIT_R3, SLJIT_R1, SLJIT_R2, TMP_REG1 };
657
658 SLJIT_ASSERT(reg_map[SLJIT_R3] == 1 && reg_map[SLJIT_R1] == 2 && reg_map[SLJIT_R2] == 8 && reg_map[TMP_REG1] == 9);
659
660 compiler->mode32 = 0;
661 arg_types >>= SLJIT_DEF_SHIFT;
662
663 while (arg_types) {
664 types = (types << SLJIT_DEF_SHIFT) | (arg_types & SLJIT_DEF_MASK);
665
666 switch (arg_types & SLJIT_DEF_MASK) {
667 case SLJIT_ARG_TYPE_F32:
668 case SLJIT_ARG_TYPE_F64:
669 arg_count++;
670 float_arg_count++;
671
672 if (arg_count != float_arg_count)
673 data_trandfer = 1;
674 break;
675 default:
676 arg_count++;
677 word_arg_count++;
678
679 if (arg_count != word_arg_count || arg_count != word_arg_regs[arg_count]) {
680 data_trandfer = 1;
681
682 if (src == word_arg_regs[arg_count]) {
683 EMIT_MOV(compiler, TMP_REG2, 0, src, 0);
684 *src_ptr = TMP_REG2;
685 }
686 }
687 break;
688 }
689
690 arg_types >>= SLJIT_DEF_SHIFT;
691 }
692
693 if (!data_trandfer)
694 return SLJIT_SUCCESS;
695
696 if (src & SLJIT_MEM) {
697 ADJUST_LOCAL_OFFSET(src, srcw);
698 EMIT_MOV(compiler, TMP_REG2, 0, src, srcw);
699 *src_ptr = TMP_REG2;
700 }
701
702 while (types) {
703 switch (types & SLJIT_DEF_MASK) {
704 case SLJIT_ARG_TYPE_F32:
705 if (arg_count != float_arg_count)
706 FAIL_IF(emit_sse2_load(compiler, 1, arg_count, float_arg_count, 0));
707 arg_count--;
708 float_arg_count--;
709 break;
710 case SLJIT_ARG_TYPE_F64:
711 if (arg_count != float_arg_count)
712 FAIL_IF(emit_sse2_load(compiler, 0, arg_count, float_arg_count, 0));
713 arg_count--;
714 float_arg_count--;
715 break;
716 default:
717 if (arg_count != word_arg_count || arg_count != word_arg_regs[arg_count])
718 EMIT_MOV(compiler, word_arg_regs[arg_count], 0, word_arg_count, 0);
719 arg_count--;
720 word_arg_count--;
721 break;
722 }
723
724 types >>= SLJIT_DEF_SHIFT;
725 }
726
727 return SLJIT_SUCCESS;
728 }
729
730 #endif
731
sljit_emit_call(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 arg_types)732 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
733 sljit_s32 arg_types)
734 {
735 CHECK_ERROR_PTR();
736 CHECK_PTR(check_sljit_emit_call(compiler, type, arg_types));
737
738 PTR_FAIL_IF(call_with_args(compiler, arg_types, NULL, 0));
739
740 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) \
741 || (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
742 compiler->skip_checks = 1;
743 #endif
744
745 return sljit_emit_jump(compiler, type);
746 }
747
sljit_emit_icall(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 arg_types,sljit_s32 src,sljit_sw srcw)748 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
749 sljit_s32 arg_types,
750 sljit_s32 src, sljit_sw srcw)
751 {
752 CHECK_ERROR();
753 CHECK(check_sljit_emit_icall(compiler, type, arg_types, src, srcw));
754
755 FAIL_IF(call_with_args(compiler, arg_types, &src, srcw));
756
757 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) \
758 || (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
759 compiler->skip_checks = 1;
760 #endif
761
762 return sljit_emit_ijump(compiler, type, src, srcw);
763 }
764
sljit_emit_fast_enter(struct sljit_compiler * compiler,sljit_s32 dst,sljit_sw dstw)765 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fast_enter(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
766 {
767 sljit_u8 *inst;
768
769 CHECK_ERROR();
770 CHECK(check_sljit_emit_fast_enter(compiler, dst, dstw));
771 ADJUST_LOCAL_OFFSET(dst, dstw);
772
773 /* For UNUSED dst. Uncommon, but possible. */
774 if (dst == SLJIT_UNUSED)
775 dst = TMP_REG1;
776
777 if (FAST_IS_REG(dst)) {
778 if (reg_map[dst] < 8) {
779 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
780 FAIL_IF(!inst);
781 INC_SIZE(1);
782 POP_REG(reg_lmap[dst]);
783 return SLJIT_SUCCESS;
784 }
785
786 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2);
787 FAIL_IF(!inst);
788 INC_SIZE(2);
789 *inst++ = REX_B;
790 POP_REG(reg_lmap[dst]);
791 return SLJIT_SUCCESS;
792 }
793
794 /* REX_W is not necessary (src is not immediate). */
795 compiler->mode32 = 1;
796 inst = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
797 FAIL_IF(!inst);
798 *inst++ = POP_rm;
799 return SLJIT_SUCCESS;
800 }
801
emit_fast_return(struct sljit_compiler * compiler,sljit_s32 src,sljit_sw srcw)802 static sljit_s32 emit_fast_return(struct sljit_compiler *compiler, sljit_s32 src, sljit_sw srcw)
803 {
804 sljit_u8 *inst;
805
806 if (FAST_IS_REG(src)) {
807 if (reg_map[src] < 8) {
808 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1 + 1);
809 FAIL_IF(!inst);
810
811 INC_SIZE(1 + 1);
812 PUSH_REG(reg_lmap[src]);
813 }
814 else {
815 inst = (sljit_u8*)ensure_buf(compiler, 1 + 2 + 1);
816 FAIL_IF(!inst);
817
818 INC_SIZE(2 + 1);
819 *inst++ = REX_B;
820 PUSH_REG(reg_lmap[src]);
821 }
822 }
823 else {
824 /* REX_W is not necessary (src is not immediate). */
825 compiler->mode32 = 1;
826 inst = emit_x86_instruction(compiler, 1, 0, 0, src, srcw);
827 FAIL_IF(!inst);
828 *inst++ = GROUP_FF;
829 *inst |= PUSH_rm;
830
831 inst = (sljit_u8*)ensure_buf(compiler, 1 + 1);
832 FAIL_IF(!inst);
833 INC_SIZE(1);
834 }
835
836 RET();
837 return SLJIT_SUCCESS;
838 }
839
840 /* --------------------------------------------------------------------- */
841 /* Extend input */
842 /* --------------------------------------------------------------------- */
843
emit_mov_int(struct sljit_compiler * compiler,sljit_s32 sign,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)844 static sljit_s32 emit_mov_int(struct sljit_compiler *compiler, sljit_s32 sign,
845 sljit_s32 dst, sljit_sw dstw,
846 sljit_s32 src, sljit_sw srcw)
847 {
848 sljit_u8* inst;
849 sljit_s32 dst_r;
850
851 compiler->mode32 = 0;
852
853 if (dst == SLJIT_UNUSED && !(src & SLJIT_MEM))
854 return SLJIT_SUCCESS; /* Empty instruction. */
855
856 if (src & SLJIT_IMM) {
857 if (FAST_IS_REG(dst)) {
858 if (sign || ((sljit_uw)srcw <= 0x7fffffff)) {
859 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, (sljit_sw)(sljit_s32)srcw, dst, dstw);
860 FAIL_IF(!inst);
861 *inst = MOV_rm_i32;
862 return SLJIT_SUCCESS;
863 }
864 return emit_load_imm64(compiler, dst, srcw);
865 }
866 compiler->mode32 = 1;
867 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, (sljit_sw)(sljit_s32)srcw, dst, dstw);
868 FAIL_IF(!inst);
869 *inst = MOV_rm_i32;
870 compiler->mode32 = 0;
871 return SLJIT_SUCCESS;
872 }
873
874 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
875
876 if ((dst & SLJIT_MEM) && FAST_IS_REG(src))
877 dst_r = src;
878 else {
879 if (sign) {
880 inst = emit_x86_instruction(compiler, 1, dst_r, 0, src, srcw);
881 FAIL_IF(!inst);
882 *inst++ = MOVSXD_r_rm;
883 } else {
884 compiler->mode32 = 1;
885 FAIL_IF(emit_mov(compiler, dst_r, 0, src, srcw));
886 compiler->mode32 = 0;
887 }
888 }
889
890 if (dst & SLJIT_MEM) {
891 compiler->mode32 = 1;
892 inst = emit_x86_instruction(compiler, 1, dst_r, 0, dst, dstw);
893 FAIL_IF(!inst);
894 *inst = MOV_rm_r;
895 compiler->mode32 = 0;
896 }
897
898 return SLJIT_SUCCESS;
899 }
900
skip_frames_before_return(struct sljit_compiler * compiler)901 static sljit_s32 skip_frames_before_return(struct sljit_compiler *compiler)
902 {
903 sljit_s32 tmp, size;
904
905 /* Don't adjust shadow stack if it isn't enabled. */
906 if (!cpu_has_shadow_stack ())
907 return SLJIT_SUCCESS;
908
909 size = compiler->local_size;
910 tmp = compiler->scratches;
911 if (tmp >= SLJIT_FIRST_SAVED_REG)
912 size += (tmp - SLJIT_FIRST_SAVED_REG + 1) * sizeof(sljit_uw);
913 tmp = compiler->saveds < SLJIT_NUMBER_OF_SAVED_REGISTERS ? (SLJIT_S0 + 1 - compiler->saveds) : SLJIT_FIRST_SAVED_REG;
914 if (SLJIT_S0 >= tmp)
915 size += (SLJIT_S0 - tmp + 1) * sizeof(sljit_uw);
916
917 return adjust_shadow_stack(compiler, SLJIT_UNUSED, 0, SLJIT_SP, size);
918 }
919
