1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include "sljitLir.h"
28
29 #ifdef _WIN32
30
31 #include <windows.h>
32
33 #endif /* _WIN32 */
34
35 #if !(defined SLJIT_STD_MACROS_DEFINED && SLJIT_STD_MACROS_DEFINED)
36
37 /* These libraries are needed for the macros below. */
38 #include <stdlib.h>
39 #include <string.h>
40
41 #endif /* SLJIT_STD_MACROS_DEFINED */
42
43 #define CHECK_ERROR() \
44 do { \
45 if (SLJIT_UNLIKELY(compiler->error)) \
46 return compiler->error; \
47 } while (0)
48
49 #define CHECK_ERROR_PTR() \
50 do { \
51 if (SLJIT_UNLIKELY(compiler->error)) \
52 return NULL; \
53 } while (0)
54
55 #define FAIL_IF(expr) \
56 do { \
57 if (SLJIT_UNLIKELY(expr)) \
58 return compiler->error; \
59 } while (0)
60
61 #define PTR_FAIL_IF(expr) \
62 do { \
63 if (SLJIT_UNLIKELY(expr)) \
64 return NULL; \
65 } while (0)
66
67 #define FAIL_IF_NULL(ptr) \
68 do { \
69 if (SLJIT_UNLIKELY(!(ptr))) { \
70 compiler->error = SLJIT_ERR_ALLOC_FAILED; \
71 return SLJIT_ERR_ALLOC_FAILED; \
72 } \
73 } while (0)
74
75 #define PTR_FAIL_IF_NULL(ptr) \
76 do { \
77 if (SLJIT_UNLIKELY(!(ptr))) { \
78 compiler->error = SLJIT_ERR_ALLOC_FAILED; \
79 return NULL; \
80 } \
81 } while (0)
82
83 #define PTR_FAIL_WITH_EXEC_IF(ptr) \
84 do { \
85 if (SLJIT_UNLIKELY(!(ptr))) { \
86 compiler->error = SLJIT_ERR_EX_ALLOC_FAILED; \
87 return NULL; \
88 } \
89 } while (0)
90
91 #if !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED)
92
93 #define SSIZE_OF(type) ((sljit_s32)sizeof(sljit_ ## type))
94
95 #define VARIABLE_FLAG_SHIFT (10)
96 /* All variable flags are even. */
97 #define VARIABLE_FLAG_MASK (0x3e << VARIABLE_FLAG_SHIFT)
98 #define GET_FLAG_TYPE(op) ((op) >> VARIABLE_FLAG_SHIFT)
99
100 #define GET_OPCODE(op) \
101 ((op) & ~(SLJIT_32 | SLJIT_SET_Z | VARIABLE_FLAG_MASK))
102
103 #define HAS_FLAGS(op) \
104 ((op) & (SLJIT_SET_Z | VARIABLE_FLAG_MASK))
105
106 #define GET_ALL_FLAGS(op) \
107 ((op) & (SLJIT_32 | SLJIT_SET_Z | VARIABLE_FLAG_MASK))
108
109 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
110 #define TYPE_CAST_NEEDED(op) \
111 ((op) >= SLJIT_MOV_U8 && (op) <= SLJIT_MOV_S32)
112 #else /* !SLJIT_64BIT_ARCHITECTURE */
113 #define TYPE_CAST_NEEDED(op) \
114 ((op) >= SLJIT_MOV_U8 && (op) <= SLJIT_MOV_S16)
115 #endif /* SLJIT_64BIT_ARCHITECTURE */
116
117 #define BUF_SIZE 4096
118
119 #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE)
120 #define ABUF_SIZE 2048
121 #else
122 #define ABUF_SIZE 4096
123 #endif
124
125 /* Parameter parsing. */
126 #define REG_MASK 0x7f
127 #define OFFS_REG(reg) (((reg) >> 8) & REG_MASK)
128 #define OFFS_REG_MASK (REG_MASK << 8)
129 #define TO_OFFS_REG(reg) ((reg) << 8)
130 #define FAST_IS_REG(reg) ((reg) < REG_MASK)
131
132 /* Mask for argument types. */
133 #define SLJIT_ARG_MASK 0x7
134 #define SLJIT_ARG_FULL_MASK (SLJIT_ARG_MASK | SLJIT_ARG_TYPE_SCRATCH_REG)
135
136 /* Mask for register pairs. */
137 #define REG_PAIR_MASK 0x7f00
138 #define REG_PAIR_FIRST(reg) ((reg) & 0x7f)
139 #define REG_PAIR_SECOND(reg) ((reg) >> 8)
140
141 /* Mask for sljit_emit_enter. */
142 #define SLJIT_KEPT_SAVEDS_COUNT(options) ((options) & 0x3)
143
144 /* Getters for simd operations, which returns with log2(size). */
145 #define SLJIT_SIMD_GET_OPCODE(type) ((type) & 0xff)
146 #define SLJIT_SIMD_GET_REG_SIZE(type) (((type) >> 12) & 0x3f)
147 #define SLJIT_SIMD_GET_ELEM_SIZE(type) (((type) >> 18) & 0x3f)
148 #define SLJIT_SIMD_GET_ELEM2_SIZE(type) (((type) >> 24) & 0x3f)
149
150 #define SLJIT_SIMD_CHECK_REG(type) (((type) & 0x3f000) >= SLJIT_SIMD_REG_64 && ((type) & 0x3f000) <= SLJIT_SIMD_REG_512)
151 #define SLJIT_SIMD_TYPE_MASK(m) ((sljit_s32)0xff000fff & ~(SLJIT_SIMD_FLOAT | SLJIT_SIMD_TEST | (m)))
152 #define SLJIT_SIMD_TYPE_MASK2(m) ((sljit_s32)0xc0000fff & ~(SLJIT_SIMD_FLOAT | SLJIT_SIMD_TEST | (m)))
153
154 /* Jump flags. */
155 #define JUMP_ADDR 0x1
156 #define JUMP_MOV_ADDR 0x2
157 /* SLJIT_REWRITABLE_JUMP is 0x1000. */
158
159 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
160 # define PATCH_MB 0x04
161 # define PATCH_MW 0x08
162 #if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)
163 # define PATCH_MD 0x10
164 # define MOV_ADDR_HI 0x20
165 # define JUMP_MAX_SIZE ((sljit_uw)(10 + 3))
166 # define CJUMP_MAX_SIZE ((sljit_uw)(2 + 10 + 3))
167 #else /* !SLJIT_CONFIG_X86_64 */
168 # define JUMP_MAX_SIZE ((sljit_uw)5)
169 # define CJUMP_MAX_SIZE ((sljit_uw)6)
170 #endif /* SLJIT_CONFIG_X86_64 */
171 # define TYPE_SHIFT 13
172 #if (defined SLJIT_DEBUG && SLJIT_DEBUG)
173 /* Bits 7..12 is for debug jump size, SLJIT_REWRITABLE_JUMP is 0x1000 */
174 # define JUMP_SIZE_SHIFT 7
175 #endif /* SLJIT_DEBUG */
176 #endif /* SLJIT_CONFIG_X86 */
177
178 #if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6) || (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
179 # define IS_BL 0x04
180 # define PATCH_B 0x08
181 #endif /* SLJIT_CONFIG_ARM_V6 || SLJIT_CONFIG_ARM_V7 */
182
183 #if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
184 # define CPOOL_SIZE 512
185 #endif /* SLJIT_CONFIG_ARM_V6 */
186
187 #if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
188 # define JUMP_SIZE_SHIFT 26
189 # define JUMP_MAX_SIZE ((sljit_uw)3)
190 #endif /* SLJIT_CONFIG_ARM_V7 */
191
192 #if (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
193 # define IS_COND 0x04
194 # define IS_BL 0x08
195 /* conditional + imm8 */
196 # define PATCH_TYPE1 0x10
197 /* conditional + imm20 */
198 # define PATCH_TYPE2 0x20
199 /* imm11 */
200 # define PATCH_TYPE3 0x30
201 /* imm24 */
202 # define PATCH_TYPE4 0x40
203 /* BL + imm24 */
204 # define PATCH_TYPE5 0x50
205 /* addwi/subwi */
206 # define PATCH_TYPE6 0x60
207 /* 0xf00 cc code for branches */
208 # define JUMP_SIZE_SHIFT 26
209 # define JUMP_MAX_SIZE ((sljit_uw)5)
210 #endif /* SLJIT_CONFIG_ARM_THUMB2 */
211
212 #if (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
213 # define IS_COND 0x004
214 # define IS_CBZ 0x008
215 # define IS_BL 0x010
216 # define PATCH_COND 0x020
217 # define PATCH_B 0x040
218 # define PATCH_B32 0x080
219 # define PATCH_ABS48 0x100
220 # define PATCH_ABS64 0x200
221 # define JUMP_SIZE_SHIFT 58
222 # define JUMP_MAX_SIZE ((sljit_uw)5)
223 #endif /* SLJIT_CONFIG_ARM_64 */
224
225 #if (defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
226 # define IS_COND 0x004
227 # define IS_CALL 0x008
228 # define PATCH_B 0x010
229 # define PATCH_ABS_B 0x020
230 #if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)
231 # define PATCH_ABS32 0x040
232 # define PATCH_ABS48 0x080
233 # define JUMP_SIZE_SHIFT 58
234 # define JUMP_MAX_SIZE ((sljit_uw)7)
235 #else /* !SLJIT_CONFIG_PPC_64 */
236 # define JUMP_SIZE_SHIFT 26
237 # define JUMP_MAX_SIZE ((sljit_uw)4)
238 #endif /* SLJIT_CONFIG_PPC_64 */
239 #endif /* SLJIT_CONFIG_PPC */
240
241 #if (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
242 # define IS_MOVABLE 0x004
243 # define IS_JAL 0x008
244 # define IS_CALL 0x010
245 # define IS_BIT26_COND 0x020
246 # define IS_BIT16_COND 0x040
247 # define IS_BIT23_COND 0x080
248
249 # define IS_COND (IS_BIT26_COND | IS_BIT16_COND | IS_BIT23_COND)
250
251 # define PATCH_B 0x100
252 # define PATCH_J 0x200
253
254 #if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64)
255 # define PATCH_ABS32 0x400
256 # define PATCH_ABS48 0x800
257 #endif /* SLJIT_CONFIG_MIPS_64 */
258
259 /* instruction types */
260 # define MOVABLE_INS 0
261 /* 1 - 31 last destination register */
262 /* no destination (i.e: store) */
263 # define UNMOVABLE_INS 32
264 /* FPU status register */
265 # define FCSR_FCC 33
266 #endif /* SLJIT_CONFIG_MIPS */
267
268 #if (defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV)
269 # define IS_COND 0x004
270 # define IS_CALL 0x008
271
272 # define PATCH_B 0x010
273 # define PATCH_J 0x020
274
275 #if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)
276 # define PATCH_REL32 0x040
277 # define PATCH_ABS32 0x080
278 # define PATCH_ABS44 0x100
279 # define PATCH_ABS52 0x200
280 # define JUMP_SIZE_SHIFT 58
281 # define JUMP_MAX_SIZE ((sljit_uw)6)
282 #else /* !SLJIT_CONFIG_RISCV_64 */
283 # define JUMP_SIZE_SHIFT 26
284 # define JUMP_MAX_SIZE ((sljit_uw)2)
285 #endif /* SLJIT_CONFIG_RISCV_64 */
286 #endif /* SLJIT_CONFIG_RISCV */
287
288 #if (defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
289 # define IS_COND 0x004
290 # define IS_CALL 0x008
291
292 # define PATCH_B 0x010
293 # define PATCH_J 0x020
294
295 # define PATCH_REL32 0x040
296 # define PATCH_ABS32 0x080
297 # define PATCH_ABS52 0x100
298 # define JUMP_SIZE_SHIFT 58
299 # define JUMP_MAX_SIZE ((sljit_uw)4)
300
301 #endif /* SLJIT_CONFIG_LOONGARCH */
302 /* Stack management. */
303
304 #define GET_SAVED_REGISTERS_SIZE(scratches, saveds, extra) \
305 (((scratches < SLJIT_NUMBER_OF_SCRATCH_REGISTERS ? 0 : (scratches - SLJIT_NUMBER_OF_SCRATCH_REGISTERS)) + \
306 (saveds) + (sljit_s32)(extra)) * (sljit_s32)sizeof(sljit_sw))
307
308 #define GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, type) \
309 (((fscratches < SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS ? 0 : (fscratches - SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS)) + \
310 (fsaveds)) * SSIZE_OF(type))
311
312 #define ADJUST_LOCAL_OFFSET(p, i) \
313 if ((p) == (SLJIT_MEM1(SLJIT_SP))) \
314 (i) += SLJIT_LOCALS_OFFSET;
315
316 #endif /* !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED) */
317
318 /* Utils can still be used even if SLJIT_CONFIG_UNSUPPORTED is set. */
319 #include "sljitUtils.c"
320
321 #if (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
322 #define SLJIT_CODE_TO_PTR(code) ((void*)((sljit_up)(code) & ~(sljit_up)0x1))
323 #elif (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)
324 #define SLJIT_CODE_TO_PTR(code) ((void*)(*(sljit_up*)code))
325 #else /* !SLJIT_CONFIG_ARM_THUMB2 && !SLJIT_INDIRECT_CALL */
326 #define SLJIT_CODE_TO_PTR(code) ((void*)(code))
327 #endif /* SLJIT_CONFIG_ARM_THUMB2 || SLJIT_INDIRECT_CALL */
328
329 #if !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED)
330
331 #if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR)
332
333 #if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR)
334
335 #if defined(__NetBSD__)
336 #include "allocator_src/sljitProtExecAllocatorNetBSD.c"
337 #else
338 #include "allocator_src/sljitProtExecAllocatorPosix.c"
339 #endif
340
341 #elif (defined SLJIT_WX_EXECUTABLE_ALLOCATOR && SLJIT_WX_EXECUTABLE_ALLOCATOR)
342
343 #if defined(_WIN32)
344 #include "allocator_src/sljitWXExecAllocatorWindows.c"
345 #else
346 #include "allocator_src/sljitWXExecAllocatorPosix.c"
347 #endif
348
349 #else
350
351 #if defined(_WIN32)
352 #include "allocator_src/sljitExecAllocatorWindows.c"
353 #elif defined(__APPLE__)
354 #include "allocator_src/sljitExecAllocatorApple.c"
355 #elif defined(__FreeBSD__)
356 #include "allocator_src/sljitExecAllocatorFreeBSD.c"
357 #else
358 #include "allocator_src/sljitExecAllocatorPosix.c"
359 #endif
360
361 #endif
362
363 #else /* !SLJIT_EXECUTABLE_ALLOCATOR */
364
365 #ifndef SLJIT_UPDATE_WX_FLAGS
366 #define SLJIT_UPDATE_WX_FLAGS(from, to, enable_exec)
367 #endif
368
369 #endif /* SLJIT_EXECUTABLE_ALLOCATOR */
370
371 #if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR)
372 #define SLJIT_ADD_EXEC_OFFSET(ptr, exec_offset) ((sljit_u8 *)(ptr) + (exec_offset))
373 #else
374 #define SLJIT_ADD_EXEC_OFFSET(ptr, exec_offset) ((sljit_u8 *)(ptr))
375 #endif
376
377 /* Argument checking features. */
378
379 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
380
381 /* Returns with error when an invalid argument is passed. */
382
383 #define CHECK_ARGUMENT(x) \
384 do { \
385 if (SLJIT_UNLIKELY(!(x))) \
386 return 1; \
387 } while (0)
388
389 #define CHECK_RETURN_TYPE sljit_s32
390 #define CHECK_RETURN_OK return 0
391
392 #define CHECK(x) \
393 do { \
394 if (SLJIT_UNLIKELY(x)) { \
395 compiler->error = SLJIT_ERR_BAD_ARGUMENT; \
396 return SLJIT_ERR_BAD_ARGUMENT; \
397 } \
398 } while (0)
399
400 #define CHECK_PTR(x) \
401 do { \
402 if (SLJIT_UNLIKELY(x)) { \
403 compiler->error = SLJIT_ERR_BAD_ARGUMENT; \
404 return NULL; \
405 } \
406 } while (0)
407
408 #define CHECK_REG_INDEX(x) \
409 do { \
410 if (SLJIT_UNLIKELY(x)) { \
411 return -2; \
412 } \
413 } while (0)
414
415 #elif (defined SLJIT_DEBUG && SLJIT_DEBUG)
416
417 /* Assertion failure occures if an invalid argument is passed. */
418 #undef SLJIT_ARGUMENT_CHECKS
419 #define SLJIT_ARGUMENT_CHECKS 1
420
421 #define CHECK_ARGUMENT(x) SLJIT_ASSERT(x)
422 #define CHECK_RETURN_TYPE void
423 #define CHECK_RETURN_OK return
424 #define CHECK(x) x
425 #define CHECK_PTR(x) x
426 #define CHECK_REG_INDEX(x) x
427
428 #elif (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
429
430 /* Arguments are not checked. */
431 #define CHECK_RETURN_TYPE void
432 #define CHECK_RETURN_OK return
433 #define CHECK(x) x
434 #define CHECK_PTR(x) x
435 #define CHECK_REG_INDEX(x) x
436
437 #else
438
439 /* Arguments are not checked. */
440 #define CHECK(x)
441 #define CHECK_PTR(x)
442 #define CHECK_REG_INDEX(x)
443
444 #endif /* SLJIT_ARGUMENT_CHECKS */
445
446 /* --------------------------------------------------------------------- */
447 /* Public functions */
448 /* --------------------------------------------------------------------- */
449
450 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
451 #define SLJIT_NEEDS_COMPILER_INIT 1
452 static sljit_s32 compiler_initialized = 0;
453 /* A thread safe initialization. */
454 static void init_compiler(void);
455 #endif
456
sljit_create_compiler(void * allocator_data)457 SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data)
458 {
459 struct sljit_compiler *compiler = (struct sljit_compiler*)SLJIT_MALLOC(sizeof(struct sljit_compiler), allocator_data);
460 if (!compiler)
461 return NULL;
462 SLJIT_ZEROMEM(compiler, sizeof(struct sljit_compiler));
463
464 SLJIT_COMPILE_ASSERT(
465 sizeof(sljit_s8) == 1 && sizeof(sljit_u8) == 1
466 && sizeof(sljit_s16) == 2 && sizeof(sljit_u16) == 2
467 && sizeof(sljit_s32) == 4 && sizeof(sljit_u32) == 4
468 && (sizeof(sljit_up) == 4 || sizeof(sljit_up) == 8)
469 && sizeof(sljit_up) <= sizeof(sljit_sw)
470 && sizeof(sljit_up) == sizeof(sljit_sp)
471 && (sizeof(sljit_sw) == 4 || sizeof(sljit_sw) == 8)
472 && (sizeof(sljit_uw) == sizeof(sljit_sw)),
473 invalid_integer_types);
474 SLJIT_COMPILE_ASSERT(SLJIT_REWRITABLE_JUMP != SLJIT_32,
475 rewritable_jump_and_single_op_must_not_be_the_same);
476 SLJIT_COMPILE_ASSERT(!(SLJIT_EQUAL & 0x1) && !(SLJIT_LESS & 0x1) && !(SLJIT_F_EQUAL & 0x1) && !(SLJIT_JUMP & 0x1),
477 conditional_flags_must_be_even_numbers);
478
479 /* Only the non-zero members must be set. */
480 compiler->error = SLJIT_SUCCESS;
481
482 compiler->allocator_data = allocator_data;
483 compiler->buf = (struct sljit_memory_fragment*)SLJIT_MALLOC(BUF_SIZE, allocator_data);
484 compiler->abuf = (struct sljit_memory_fragment*)SLJIT_MALLOC(ABUF_SIZE, allocator_data);
485
486 if (!compiler->buf || !compiler->abuf) {
487 if (compiler->buf)
488 SLJIT_FREE(compiler->buf, allocator_data);
489 if (compiler->abuf)
490 SLJIT_FREE(compiler->abuf, allocator_data);
491 SLJIT_FREE(compiler, allocator_data);
492 return NULL;
493 }
494
495 compiler->buf->next = NULL;
496 compiler->buf->used_size = 0;
497 compiler->abuf->next = NULL;
498 compiler->abuf->used_size = 0;
499
500 compiler->scratches = -1;
501 compiler->saveds = -1;
502 compiler->fscratches = -1;
503 compiler->fsaveds = -1;
504 compiler->local_size = -1;
505
506 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
507 compiler->args_size = -1;
508 #endif /* SLJIT_CONFIG_X86_32 */
509
510 #if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
511 compiler->cpool = (sljit_uw*)SLJIT_MALLOC(CPOOL_SIZE * sizeof(sljit_uw)
512 + CPOOL_SIZE * sizeof(sljit_u8), allocator_data);
513 if (!compiler->cpool) {
514 SLJIT_FREE(compiler->buf, allocator_data);
515 SLJIT_FREE(compiler->abuf, allocator_data);
516 SLJIT_FREE(compiler, allocator_data);
517 return NULL;
518 }
519 compiler->cpool_unique = (sljit_u8*)(compiler->cpool + CPOOL_SIZE);
520 compiler->cpool_diff = 0xffffffff;
521 #endif /* SLJIT_CONFIG_ARM_V6 */
522
523 #if (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
524 compiler->delay_slot = UNMOVABLE_INS;
525 #endif /* SLJIT_CONFIG_MIPS */
526
527 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
528 || (defined SLJIT_DEBUG && SLJIT_DEBUG)
529 compiler->last_flags = 0;
530 compiler->last_return = -1;
531 compiler->logical_local_size = 0;
532 #endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_DEBUG */
533
534 #if (defined SLJIT_NEEDS_COMPILER_INIT && SLJIT_NEEDS_COMPILER_INIT)
535 if (!compiler_initialized) {
536 init_compiler();
537 compiler_initialized = 1;
538 }
539 #endif
540
541 return compiler;
542 }
543
sljit_free_compiler(struct sljit_compiler * compiler)544 SLJIT_API_FUNC_ATTRIBUTE void sljit_free_compiler(struct sljit_compiler *compiler)
545 {
546 struct sljit_memory_fragment *buf;
547 struct sljit_memory_fragment *curr;
548 void *allocator_data = compiler->allocator_data;
549 SLJIT_UNUSED_ARG(allocator_data);
550
551 buf = compiler->buf;
552 while (buf) {
553 curr = buf;
554 buf = buf->next;
555 SLJIT_FREE(curr, allocator_data);
556 }
557
558 buf = compiler->abuf;
559 while (buf) {
560 curr = buf;
561 buf = buf->next;
562 SLJIT_FREE(curr, allocator_data);
563 }
564
565 #if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
566 SLJIT_FREE(compiler->cpool, allocator_data);
567 #endif
568 SLJIT_FREE(compiler, allocator_data);
569 }
570
sljit_set_compiler_memory_error(struct sljit_compiler * compiler)571 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_compiler_memory_error(struct sljit_compiler *compiler)
572 {
573 if (compiler->error == SLJIT_SUCCESS)
574 compiler->error = SLJIT_ERR_ALLOC_FAILED;
575 }
576
sljit_free_code(void * code,void * exec_allocator_data)577 SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data)
578 {
579 SLJIT_UNUSED_ARG(exec_allocator_data);
580
581 SLJIT_FREE_EXEC(SLJIT_CODE_TO_PTR(code), exec_allocator_data);
582 }
583
sljit_set_label(struct sljit_jump * jump,struct sljit_label * label)584 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_label(struct sljit_jump *jump, struct sljit_label* label)
585 {
586 if (SLJIT_LIKELY(!!jump) && SLJIT_LIKELY(!!label)) {
587 jump->flags &= (sljit_uw)~JUMP_ADDR;
588 jump->u.label = label;
589 }
590 }
591
sljit_set_target(struct sljit_jump * jump,sljit_uw target)592 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_target(struct sljit_jump *jump, sljit_uw target)
593 {
594 if (SLJIT_LIKELY(!!jump)) {
595 jump->flags |= JUMP_ADDR;
596 jump->u.target = target;
597 }
598 }
599
600 #define SLJIT_CURRENT_FLAGS_ALL \
601 (SLJIT_CURRENT_FLAGS_32 | SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB | SLJIT_CURRENT_FLAGS_COMPARE)
602
sljit_set_current_flags(struct sljit_compiler * compiler,sljit_s32 current_flags)603 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_current_flags(struct sljit_compiler *compiler, sljit_s32 current_flags)
604 {
605 SLJIT_UNUSED_ARG(compiler);
606 SLJIT_UNUSED_ARG(current_flags);
607
608 #if (defined SLJIT_HAS_STATUS_FLAGS_STATE && SLJIT_HAS_STATUS_FLAGS_STATE)
609 compiler->status_flags_state = current_flags;
610 #endif
611
612 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
613 compiler->last_flags = 0;
614 if ((current_flags & ~(VARIABLE_FLAG_MASK | SLJIT_SET_Z | SLJIT_CURRENT_FLAGS_ALL)) == 0) {
615 compiler->last_flags = GET_FLAG_TYPE(current_flags) | (current_flags & (SLJIT_32 | SLJIT_SET_Z));
616 }
617 #endif
618 }
619
620 /* --------------------------------------------------------------------- */
621 /* Private functions */
622 /* --------------------------------------------------------------------- */
623
ensure_buf(struct sljit_compiler * compiler,sljit_uw size)624 static void* ensure_buf(struct sljit_compiler *compiler, sljit_uw size)
625 {
626 sljit_u8 *ret;
627 struct sljit_memory_fragment *new_frag;
628
629 SLJIT_ASSERT(size <= 256);
630 if (compiler->buf->used_size + size <= (BUF_SIZE - (sljit_uw)SLJIT_OFFSETOF(struct sljit_memory_fragment, memory))) {
631 ret = compiler->buf->memory + compiler->buf->used_size;
632 compiler->buf->used_size += size;
633 return ret;
634 }
635 new_frag = (struct sljit_memory_fragment*)SLJIT_MALLOC(BUF_SIZE, compiler->allocator_data);
636 PTR_FAIL_IF_NULL(new_frag);
637 new_frag->next = compiler->buf;
638 compiler->buf = new_frag;
639 new_frag->used_size = size;
640 return new_frag->memory;
641 }
642
ensure_abuf(struct sljit_compiler * compiler,sljit_uw size)643 static void* ensure_abuf(struct sljit_compiler *compiler, sljit_uw size)
644 {
645 sljit_u8 *ret;
646 struct sljit_memory_fragment *new_frag;
647
648 SLJIT_ASSERT(size <= 256);
649 if (compiler->abuf->used_size + size <= (ABUF_SIZE - (sljit_uw)SLJIT_OFFSETOF(struct sljit_memory_fragment, memory))) {
650 ret = compiler->abuf->memory + compiler->abuf->used_size;
651 compiler->abuf->used_size += size;
652 return ret;
653 }
654 new_frag = (struct sljit_memory_fragment*)SLJIT_MALLOC(ABUF_SIZE, compiler->allocator_data);
655 PTR_FAIL_IF_NULL(new_frag);
656 new_frag->next = compiler->abuf;
657 compiler->abuf = new_frag;
658 new_frag->used_size = size;
659 return new_frag->memory;
660 }
661
sljit_alloc_memory(struct sljit_compiler * compiler,sljit_s32 size)662 SLJIT_API_FUNC_ATTRIBUTE void* sljit_alloc_memory(struct sljit_compiler *compiler, sljit_s32 size)
663 {
664 CHECK_ERROR_PTR();
665
666 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
667 if (size <= 0 || size > 128)
668 return NULL;
669 size = (size + 7) & ~7;
670 #else
671 if (size <= 0 || size > 64)
672 return NULL;
673 size = (size + 3) & ~3;
674 #endif
675 return ensure_abuf(compiler, (sljit_uw)size);
676 }
677
reverse_buf(struct sljit_compiler * compiler)678 static SLJIT_INLINE void reverse_buf(struct sljit_compiler *compiler)
679 {
680 struct sljit_memory_fragment *buf = compiler->buf;
681 struct sljit_memory_fragment *prev = NULL;
682 struct sljit_memory_fragment *tmp;
683
684 do {
685 tmp = buf->next;
686 buf->next = prev;
687 prev = buf;
688 buf = tmp;
689 } while (buf != NULL);
690
691 compiler->buf = prev;
692 }
693
allocate_executable_memory(sljit_uw size,sljit_s32 options,void * exec_allocator_data,sljit_sw * executable_offset)694 static SLJIT_INLINE void* allocate_executable_memory(sljit_uw size, sljit_s32 options,
695 void *exec_allocator_data, sljit_sw *executable_offset)
696 {
697 void *code;
698 struct sljit_generate_code_buffer *buffer;
699
700 if (SLJIT_LIKELY(!(options & SLJIT_GENERATE_CODE_BUFFER))) {
701 code = SLJIT_MALLOC_EXEC(size, exec_allocator_data);
702 *executable_offset = SLJIT_EXEC_OFFSET(code);
703 return code;
704 }
705
706 buffer = (struct sljit_generate_code_buffer*)exec_allocator_data;
707
708 if (size <= buffer->size) {
709 *executable_offset = buffer->executable_offset;
710 return buffer->buffer;
711 }
712
713 return NULL;
714 }
715
716 #define SLJIT_MAX_ADDRESS ~(sljit_uw)0
717
718 #define SLJIT_GET_NEXT_SIZE(ptr) (ptr != NULL) ? ((ptr)->size) : SLJIT_MAX_ADDRESS
719 #define SLJIT_GET_NEXT_ADDRESS(ptr) (ptr != NULL) ? ((ptr)->addr) : SLJIT_MAX_ADDRESS
720
721 #if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
722
723 #define SLJIT_NEXT_DEFINE_TYPES \
724 sljit_uw next_label_size; \
725 sljit_uw next_jump_addr; \
726 sljit_uw next_const_addr; \
727 sljit_uw next_min_addr
728
729 #define SLJIT_NEXT_INIT_TYPES() \
730 next_label_size = SLJIT_GET_NEXT_SIZE(label); \
731 next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump); \
732 next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);
733
734 #define SLJIT_GET_NEXT_MIN() \
735 next_min_addr = sljit_get_next_min(next_label_size, next_jump_addr, next_const_addr);
736
sljit_get_next_min(sljit_uw next_label_size,sljit_uw next_jump_addr,sljit_uw next_const_addr)737 static SLJIT_INLINE sljit_uw sljit_get_next_min(sljit_uw next_label_size,
738 sljit_uw next_jump_addr, sljit_uw next_const_addr)
739 {
740 sljit_uw result = next_jump_addr;
741
742 SLJIT_ASSERT(result == SLJIT_MAX_ADDRESS || result != next_const_addr);
743
744 if (next_const_addr < result)
745 result = next_const_addr;
746
747 if (next_label_size < result)
748 result = next_label_size;
749
750 return result;
751 }
752
753 #endif /* !SLJIT_CONFIG_X86 */
754
set_emit_enter(struct sljit_compiler * compiler,sljit_s32 options,sljit_s32 args,sljit_s32 scratches,sljit_s32 saveds,sljit_s32 fscratches,sljit_s32 fsaveds,sljit_s32 local_size)755 static SLJIT_INLINE void set_emit_enter(struct sljit_compiler *compiler,
756 sljit_s32 options, sljit_s32 args, sljit_s32 scratches, sljit_s32 saveds,
757 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
758 {
759 SLJIT_UNUSED_ARG(args);
760 SLJIT_UNUSED_ARG(local_size);
761
762 compiler->options = options;
763 compiler->scratches = scratches;
764 compiler->saveds = saveds;
765 compiler->fscratches = fscratches;
766 compiler->fsaveds = fsaveds;
767 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
768 compiler->last_return = args & SLJIT_ARG_MASK;
769 compiler->logical_local_size = local_size;
770 #endif
771 }
772
set_set_context(struct sljit_compiler * compiler,sljit_s32 options,sljit_s32 args,sljit_s32 scratches,sljit_s32 saveds,sljit_s32 fscratches,sljit_s32 fsaveds,sljit_s32 local_size)773 static SLJIT_INLINE void set_set_context(struct sljit_compiler *compiler,
774 sljit_s32 options, sljit_s32 args, sljit_s32 scratches, sljit_s32 saveds,
775 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
776 {
777 SLJIT_UNUSED_ARG(args);
778 SLJIT_UNUSED_ARG(local_size);
779
780 compiler->options = options;
781 compiler->scratches = scratches;
782 compiler->saveds = saveds;
783 compiler->fscratches = fscratches;
784 compiler->fsaveds = fsaveds;
785 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
786 compiler->last_return = args & SLJIT_ARG_MASK;
787 compiler->logical_local_size = local_size;
788 #endif
789 }
790
set_label(struct sljit_label * label,struct sljit_compiler * compiler)791 static SLJIT_INLINE void set_label(struct sljit_label *label, struct sljit_compiler *compiler)
792 {
793 label->next = NULL;
794 label->u.index = compiler->label_count++;
795 label->size = compiler->size;
796 if (compiler->last_label != NULL)
797 compiler->last_label->next = label;
798 else
799 compiler->labels = label;
800 compiler->last_label = label;
801 }
802
set_jump(struct sljit_jump * jump,struct sljit_compiler * compiler,sljit_u32 flags)803 static SLJIT_INLINE void set_jump(struct sljit_jump *jump, struct sljit_compiler *compiler, sljit_u32 flags)
804 {
805 jump->next = NULL;
806 jump->flags = flags;
807 jump->u.label = NULL;
808 if (compiler->last_jump != NULL)
809 compiler->last_jump->next = jump;
810 else
811 compiler->jumps = jump;
812 compiler->last_jump = jump;
813 }
814
set_mov_addr(struct sljit_jump * jump,struct sljit_compiler * compiler,sljit_uw offset)815 static SLJIT_INLINE void set_mov_addr(struct sljit_jump *jump, struct sljit_compiler *compiler, sljit_uw offset)
816 {
817 jump->next = NULL;
818 jump->addr = compiler->size - offset;
819 jump->flags = JUMP_MOV_ADDR;
820 jump->u.label = NULL;
821 if (compiler->last_jump != NULL)
822 compiler->last_jump->next = jump;
823 else
824 compiler->jumps = jump;
825 compiler->last_jump = jump;
826 }
827
set_const(struct sljit_const * const_,struct sljit_compiler * compiler)828 static SLJIT_INLINE void set_const(struct sljit_const *const_, struct sljit_compiler *compiler)
829 {
830 const_->next = NULL;
831 const_->addr = compiler->size;
832 if (compiler->last_const != NULL)
833 compiler->last_const->next = const_;
834 else
835 compiler->consts = const_;
836 compiler->last_const = const_;
837 }
838
839 #define ADDRESSING_DEPENDS_ON(exp, reg) \
840 (((exp) & SLJIT_MEM) && (((exp) & REG_MASK) == reg || OFFS_REG(exp) == reg))
841
842 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
843
function_check_arguments(sljit_s32 arg_types,sljit_s32 scratches,sljit_s32 saveds,sljit_s32 fscratches)844 static sljit_s32 function_check_arguments(sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds, sljit_s32 fscratches)
845 {
846 sljit_s32 word_arg_count, scratch_arg_end, saved_arg_count, float_arg_count, curr_type;
847
848 curr_type = (arg_types & SLJIT_ARG_FULL_MASK);
849
850 if (curr_type >= SLJIT_ARG_TYPE_F64) {
851 if (curr_type > SLJIT_ARG_TYPE_F32 || fscratches == 0)
852 return 0;
853 } else if (curr_type >= SLJIT_ARG_TYPE_W) {
854 if (scratches == 0)
855 return 0;
856 }
857
858 arg_types >>= SLJIT_ARG_SHIFT;
859
860 word_arg_count = 0;
861 scratch_arg_end = 0;
862 saved_arg_count = 0;
863 float_arg_count = 0;
864 while (arg_types != 0) {
865 if (word_arg_count + float_arg_count >= 4)
866 return 0;
867
868 curr_type = (arg_types & SLJIT_ARG_MASK);
869
870 if (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) {
871 if (saveds == -1 || curr_type < SLJIT_ARG_TYPE_W || curr_type > SLJIT_ARG_TYPE_P)
872 return 0;
873
874 word_arg_count++;
875 scratch_arg_end = word_arg_count;
876 } else {
877 if (curr_type < SLJIT_ARG_TYPE_W || curr_type > SLJIT_ARG_TYPE_F32)
878 return 0;
879
880 if (curr_type < SLJIT_ARG_TYPE_F64) {
881 word_arg_count++;
882 saved_arg_count++;
883 } else
884 float_arg_count++;
885 }
886
887 arg_types >>= SLJIT_ARG_SHIFT;
888 }
889
890 if (saveds == -1)
891 return (word_arg_count <= scratches && float_arg_count <= fscratches);
892
893 return (saved_arg_count <= saveds && scratch_arg_end <= scratches && float_arg_count <= fscratches);
894 }
895
896 #define FUNCTION_CHECK_IS_REG(r) \
897 (((r) >= SLJIT_R0 && (r) < (SLJIT_R0 + compiler->scratches)) \
898 || ((r) > (SLJIT_S0 - compiler->saveds) && (r) <= SLJIT_S0) \
899 || ((r) >= SLJIT_TMP_REGISTER_BASE && (r) < (SLJIT_TMP_REGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_REGISTERS)))
900
901 #if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)
902 #define CHECK_IF_VIRTUAL_REGISTER(p) ((p) <= SLJIT_S3 && (p) >= SLJIT_S8)
903 #else
904 #define CHECK_IF_VIRTUAL_REGISTER(p) 0
905 #endif
906
function_check_src_mem(struct sljit_compiler * compiler,sljit_s32 p,sljit_sw i)907 static sljit_s32 function_check_src_mem(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
908 {
909 if (compiler->scratches == -1)
910 return 0;
911
912 if (!(p & SLJIT_MEM))
913 return 0;
914
915 if (p == SLJIT_MEM1(SLJIT_SP))
916 return (i >= 0 && i < compiler->logical_local_size);
917
918 if (!(!(p & REG_MASK) || FUNCTION_CHECK_IS_REG(p & REG_MASK)))
919 return 0;
920
921 if (CHECK_IF_VIRTUAL_REGISTER(p & REG_MASK))
922 return 0;
923
924 if (p & OFFS_REG_MASK) {
925 if (!(p & REG_MASK))
926 return 0;
927
928 if (!(FUNCTION_CHECK_IS_REG(OFFS_REG(p))))
929 return 0;
930
931 if (CHECK_IF_VIRTUAL_REGISTER(OFFS_REG(p)))
932 return 0;
933
934 if ((i & ~0x3) != 0)
935 return 0;
936 }
937
938 return (p & ~(SLJIT_MEM | REG_MASK | OFFS_REG_MASK)) == 0;
939 }
940
941 #define FUNCTION_CHECK_SRC_MEM(p, i) \
942 CHECK_ARGUMENT(function_check_src_mem(compiler, p, i));
943
function_check_src(struct sljit_compiler * compiler,sljit_s32 p,sljit_sw i)944 static sljit_s32 function_check_src(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
945 {
946 if (compiler->scratches == -1)
947 return 0;
948
949 if (FUNCTION_CHECK_IS_REG(p))
950 return (i == 0);
951
952 if (p == SLJIT_IMM)
953 return 1;
954
955 return function_check_src_mem(compiler, p, i);
956 }
957
958 #define FUNCTION_CHECK_SRC(p, i) \
959 CHECK_ARGUMENT(function_check_src(compiler, p, i));
960
function_check_dst(struct sljit_compiler * compiler,sljit_s32 p,sljit_sw i)961 static sljit_s32 function_check_dst(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
962 {
963 if (compiler->scratches == -1)
964 return 0;
965
966 if (FUNCTION_CHECK_IS_REG(p))
967 return (i == 0);
968
969 return function_check_src_mem(compiler, p, i);
970 }
971
972 #define FUNCTION_CHECK_DST(p, i) \
973 CHECK_ARGUMENT(function_check_dst(compiler, p, i));
974
975 #if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
976 || (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32)
977
978 #define FUNCTION_CHECK_IS_FREG(fr, is_32) \
979 function_check_is_freg(compiler, (fr), (is_32))
980
981 static sljit_s32 function_check_is_freg(struct sljit_compiler *compiler, sljit_s32 fr, sljit_s32 is_32);
982
983 #define FUNCTION_FCHECK(p, i, is_32) \
984 CHECK_ARGUMENT(function_fcheck(compiler, (p), (i), (is_32)));
985
function_fcheck(struct sljit_compiler * compiler,sljit_s32 p,sljit_sw i,sljit_s32 is_32)986 static sljit_s32 function_fcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i, sljit_s32 is_32)
987 {
988 if (compiler->scratches == -1)
989 return 0;
990
991 if (FUNCTION_CHECK_IS_FREG(p, is_32))
992 return (i == 0);
993
994 return function_check_src_mem(compiler, p, i);
995 }
996
997 #else /* !SLJIT_CONFIG_ARM_32 && !SLJIT_CONFIG_MIPS_32 */
998 #define FUNCTION_CHECK_IS_FREG(fr, is_32) \
999 function_check_is_freg(compiler, (fr))
1000
function_check_is_freg(struct sljit_compiler * compiler,sljit_s32 fr)1001 static sljit_s32 function_check_is_freg(struct sljit_compiler *compiler, sljit_s32 fr)
1002 {
1003 if (compiler->scratches == -1)
1004 return 0;
1005
1006 return (fr >= SLJIT_FR0 && fr < (SLJIT_FR0 + compiler->fscratches))
1007 || (fr > (SLJIT_FS0 - compiler->fsaveds) && fr <= SLJIT_FS0)
1008 || (fr >= SLJIT_TMP_FREGISTER_BASE && fr < (SLJIT_TMP_FREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_FLOAT_REGISTERS));
1009 }
1010
1011 #define FUNCTION_FCHECK(p, i, is_32) \
1012 CHECK_ARGUMENT(function_fcheck(compiler, (p), (i)));
1013
function_fcheck(struct sljit_compiler * compiler,sljit_s32 p,sljit_sw i)1014 static sljit_s32 function_fcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1015 {
1016 if (compiler->scratches == -1)
1017 return 0;
1018
1019 if ((p >= SLJIT_FR0 && p < (SLJIT_FR0 + compiler->fscratches))
1020 || (p > (SLJIT_FS0 - compiler->fsaveds) && p <= SLJIT_FS0)
1021 || (p >= SLJIT_TMP_FREGISTER_BASE && p < (SLJIT_TMP_FREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_FLOAT_REGISTERS)))
1022 return (i == 0);
1023
1024 return function_check_src_mem(compiler, p, i);
1025 }
1026
1027 #endif /* SLJIT_CONFIG_ARM_32 || SLJIT_CONFIG_MIPS_32 */
1028
1029 #endif /* SLJIT_ARGUMENT_CHECKS */
1030
1031 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1032
sljit_compiler_verbose(struct sljit_compiler * compiler,FILE * verbose)1033 SLJIT_API_FUNC_ATTRIBUTE void sljit_compiler_verbose(struct sljit_compiler *compiler, FILE* verbose)
1034 {
1035 compiler->verbose = verbose;
1036 }
1037
1038 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
1039 #ifdef _WIN64
1040 #ifdef __GNUC__
1041 # define SLJIT_PRINT_D "ll"
1042 #else
1043 # define SLJIT_PRINT_D "I64"
1044 #endif
1045 #else
1046 # define SLJIT_PRINT_D "l"
1047 #endif
1048 #else
1049 # define SLJIT_PRINT_D ""
1050 #endif
1051
sljit_verbose_reg(struct sljit_compiler * compiler,sljit_s32 r)1052 static void sljit_verbose_reg(struct sljit_compiler *compiler, sljit_s32 r)
1053 {
1054 if (r < (SLJIT_R0 + compiler->scratches))
1055 fprintf(compiler->verbose, "r%d", r - SLJIT_R0);
1056 else if (r < SLJIT_SP)
1057 fprintf(compiler->verbose, "s%d", SLJIT_NUMBER_OF_REGISTERS - r);
1058 else if (r == SLJIT_SP)
1059 fprintf(compiler->verbose, "sp");
1060 else
1061 fprintf(compiler->verbose, "t%d", r - SLJIT_TMP_REGISTER_BASE);
1062 }
1063
sljit_verbose_freg(struct sljit_compiler * compiler,sljit_s32 r)1064 static void sljit_verbose_freg(struct sljit_compiler *compiler, sljit_s32 r)
1065 {
1066 #if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
1067 || (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32)
1068 if (r >= SLJIT_F64_SECOND(SLJIT_FR0)) {
1069 fprintf(compiler->verbose, "^");
1070 r -= SLJIT_F64_SECOND(0);
1071 }
1072 #endif /* SLJIT_CONFIG_ARM_32 || SLJIT_CONFIG_MIPS_32 */
1073
1074 if (r < (SLJIT_FR0 + compiler->fscratches))
1075 fprintf(compiler->verbose, "fr%d", r - SLJIT_FR0);
1076 else if (r < SLJIT_TMP_FREGISTER_BASE)
1077 fprintf(compiler->verbose, "fs%d", SLJIT_NUMBER_OF_FLOAT_REGISTERS - r);
1078 else
1079 fprintf(compiler->verbose, "ft%d", r - SLJIT_TMP_FREGISTER_BASE);
1080 }
1081
sljit_verbose_param(struct sljit_compiler * compiler,sljit_s32 p,sljit_sw i)1082 static void sljit_verbose_param(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1083 {
1084 if ((p) == SLJIT_IMM)
1085 fprintf(compiler->verbose, "#%" SLJIT_PRINT_D "d", (i));
1086 else if ((p) & SLJIT_MEM) {
1087 if ((p) & REG_MASK) {
1088 fputc('[', compiler->verbose);
1089 sljit_verbose_reg(compiler, (p) & REG_MASK);
1090 if ((p) & OFFS_REG_MASK) {
1091 fprintf(compiler->verbose, " + ");
1092 sljit_verbose_reg(compiler, OFFS_REG(p));
1093 if (i)
1094 fprintf(compiler->verbose, " * %d", 1 << (i));
1095 }
1096 else if (i)
1097 fprintf(compiler->verbose, " + %" SLJIT_PRINT_D "d", (i));
1098 fputc(']', compiler->verbose);
1099 }
1100 else
1101 fprintf(compiler->verbose, "[#%" SLJIT_PRINT_D "d]", (i));
1102 } else
1103 sljit_verbose_reg(compiler, p);
1104 }
1105
sljit_verbose_fparam(struct sljit_compiler * compiler,sljit_s32 p,sljit_sw i)1106 static void sljit_verbose_fparam(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)
1107 {
1108 if ((p) & SLJIT_MEM) {
1109 if ((p) & REG_MASK) {
1110 fputc('[', compiler->verbose);
1111 sljit_verbose_reg(compiler, (p) & REG_MASK);
1112 if ((p) & OFFS_REG_MASK) {
1113 fprintf(compiler->verbose, " + ");
1114 sljit_verbose_reg(compiler, OFFS_REG(p));
1115 if (i)
1116 fprintf(compiler->verbose, "%d", 1 << (i));
1117 }
1118 else if (i)
1119 fprintf(compiler->verbose, " + %" SLJIT_PRINT_D "d", (i));
1120 fputc(']', compiler->verbose);
1121 }
1122 else
1123 fprintf(compiler->verbose, "[#%" SLJIT_PRINT_D "d]", (i));
1124 }
1125 else
1126 sljit_verbose_freg(compiler, p);
1127 }
1128
1129 static const char* op0_names[] = {
1130 "breakpoint", "nop", "lmul.uw", "lmul.sw",
1131 "divmod.u", "divmod.s", "div.u", "div.s",
1132 "endbr", "skip_frames_before_return"
1133 };
1134
1135 static const char* op1_names[] = {
1136 "mov", "mov", "mov", "mov",
1137 "mov", "mov", "mov", "mov",
1138 "mov", "clz", "ctz", "rev",
1139 "rev", "rev", "rev", "rev"
1140 };
1141
1142 static const char* op1_types[] = {
1143 "", ".u8", ".s8", ".u16",
1144 ".s16", ".u32", ".s32", "32",
1145 ".p", "", "", "",
1146 ".u16", ".s16", ".u32", ".s32"
1147 };
1148
1149 static const char* op2_names[] = {
1150 "add", "addc", "sub", "subc",
1151 "mul", "and", "or", "xor",
1152 "shl", "mshl", "lshr", "mlshr",
1153 "ashr", "mashr", "rotl", "rotr"
1154 };
1155
1156 static const char* op2r_names[] = {
1157 "muladd"
1158 };
1159
1160 static const char* op_src_dst_names[] = {
1161 "fast_return", "skip_frames_before_fast_return",
1162 "prefetch_l1", "prefetch_l2",
1163 "prefetch_l3", "prefetch_once",
1164 "fast_enter", "get_return_address"
1165 };
1166
1167 static const char* fop1_names[] = {
1168 "mov", "conv", "conv", "conv",
1169 "conv", "conv", "conv", "conv",
1170 "cmp", "neg", "abs",
1171 };
1172
1173 static const char* fop1_conv_types[] = {
1174 "sw", "s32", "sw", "s32",
1175 "uw", "u32"
1176 };
1177
1178 static const char* fop2_names[] = {
1179 "add", "sub", "mul", "div"
1180 };
1181
1182 static const char* fop2r_names[] = {
1183 "copysign"
1184 };
1185
1186 static const char* simd_op2_names[] = {
1187 "and", "or", "xor"
1188 };
1189
1190 static const char* jump_names[] = {
1191 "equal", "not_equal",
1192 "less", "greater_equal",
1193 "greater", "less_equal",
1194 "sig_less", "sig_greater_equal",
1195 "sig_greater", "sig_less_equal",
1196 "overflow", "not_overflow",
1197 "carry", "not_carry",
1198 "atomic_stored", "atomic_not_stored",
1199 "f_equal", "f_not_equal",
1200 "f_less", "f_greater_equal",
1201 "f_greater", "f_less_equal",
1202 "unordered", "ordered",
1203 "ordered_equal", "unordered_or_not_equal",
1204 "ordered_less", "unordered_or_greater_equal",
1205 "ordered_greater", "unordered_or_less_equal",
1206 "unordered_or_equal", "ordered_not_equal",
1207 "unordered_or_less", "ordered_greater_equal",
1208 "unordered_or_greater", "ordered_less_equal",
1209 "jump", "fast_call",
1210 "call", "call_reg_arg"
1211 };
1212
1213 static const char* call_arg_names[] = {
1214 "void", "w", "32", "p", "f64", "f32"
1215 };
1216
1217 #endif /* SLJIT_VERBOSE */
1218
1219 /* --------------------------------------------------------------------- */
1220 /* Arch dependent */
1221 /* --------------------------------------------------------------------- */
1222
1223 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \
1224 || (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1225
1226 #define SLJIT_SKIP_CHECKS(compiler) (compiler)->skip_checks = 1
1227
check_sljit_generate_code(struct sljit_compiler * compiler)1228 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_generate_code(struct sljit_compiler *compiler)
1229 {
1230 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1231 struct sljit_jump *jump;
1232 #endif
1233
1234 SLJIT_UNUSED_ARG(compiler);
1235
1236 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1237 CHECK_ARGUMENT(compiler->size > 0);
1238 jump = compiler->jumps;
1239 while (jump) {
1240 /* All jumps have target. */
1241 CHECK_ARGUMENT((jump->flags & JUMP_ADDR) || jump->u.label != NULL);
1242 jump = jump->next;
1243 }
1244 #endif
1245 CHECK_RETURN_OK;
1246 }
1247
1248 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
1249 #define SLJIT_ENTER_CPU_SPECIFIC_OPTIONS (SLJIT_ENTER_USE_VEX)
1250 #else /* !SLJIT_CONFIG_X86 */
1251 #define SLJIT_ENTER_CPU_SPECIFIC_OPTIONS (0)
1252 #endif /* !SLJIT_CONFIG_X86 */
1253
check_sljit_emit_enter(struct sljit_compiler * compiler,sljit_s32 options,sljit_s32 arg_types,sljit_s32 scratches,sljit_s32 saveds,sljit_s32 fscratches,sljit_s32 fsaveds,sljit_s32 local_size)1254 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_enter(struct sljit_compiler *compiler,
1255 sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
1256 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
1257 {
1258 SLJIT_UNUSED_ARG(compiler);
1259
1260 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1261 if (options & SLJIT_ENTER_REG_ARG) {
1262 CHECK_ARGUMENT(!(options & ~(0x3 | SLJIT_ENTER_REG_ARG | SLJIT_ENTER_CPU_SPECIFIC_OPTIONS)));
1263 } else {
1264 CHECK_ARGUMENT((options & ~SLJIT_ENTER_CPU_SPECIFIC_OPTIONS) == 0);
1265 }
1266 CHECK_ARGUMENT(SLJIT_KEPT_SAVEDS_COUNT(options) <= 3 && SLJIT_KEPT_SAVEDS_COUNT(options) <= saveds);
1267 CHECK_ARGUMENT(scratches >= 0 && scratches <= SLJIT_NUMBER_OF_REGISTERS);
1268 CHECK_ARGUMENT(saveds >= 0 && saveds <= SLJIT_NUMBER_OF_SAVED_REGISTERS);
1269 CHECK_ARGUMENT(scratches + saveds <= SLJIT_NUMBER_OF_REGISTERS);
1270 CHECK_ARGUMENT(fscratches >= 0 && fscratches <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1271 CHECK_ARGUMENT(fsaveds >= 0 && fsaveds <= SLJIT_NUMBER_OF_SAVED_FLOAT_REGISTERS);
1272 CHECK_ARGUMENT(fscratches + fsaveds <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1273 CHECK_ARGUMENT(local_size >= 0 && local_size <= SLJIT_MAX_LOCAL_SIZE);
1274 CHECK_ARGUMENT((arg_types & SLJIT_ARG_FULL_MASK) <= SLJIT_ARG_TYPE_F32);
1275 CHECK_ARGUMENT(function_check_arguments(arg_types, scratches, (options & SLJIT_ENTER_REG_ARG) ? 0 : saveds, fscratches));
1276
1277 compiler->last_flags = 0;
1278 #endif
1279 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1280 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1281 fprintf(compiler->verbose, " enter ret[%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
1282
1283 arg_types >>= SLJIT_ARG_SHIFT;
1284 if (arg_types) {
1285 fprintf(compiler->verbose, "], args[");
1286 do {
1287 fprintf(compiler->verbose, "%s%s", call_arg_names[arg_types & SLJIT_ARG_MASK],
1288 (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) ? "_r" : "");
1289 arg_types >>= SLJIT_ARG_SHIFT;
1290 if (arg_types)
1291 fprintf(compiler->verbose, ",");
1292 } while (arg_types);
1293 }
1294
1295 fprintf(compiler->verbose, "],");
1296
1297 if (options & SLJIT_ENTER_REG_ARG) {
1298 if (SLJIT_KEPT_SAVEDS_COUNT(options) > 0)
1299 fprintf(compiler->verbose, " opt:reg_arg(%d),", SLJIT_KEPT_SAVEDS_COUNT(options));
1300 else
1301 fprintf(compiler->verbose, " opt:reg_arg,");
1302 }
1303
1304 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
1305 if (options & SLJIT_ENTER_USE_VEX) {
1306 fprintf(compiler->verbose, " opt:use_vex,");
1307 }
1308 #endif /* !SLJIT_CONFIG_X86 */
1309
1310 fprintf(compiler->verbose, " scratches:%d, saveds:%d, fscratches:%d, fsaveds:%d, local_size:%d\n",
1311 scratches, saveds, fscratches, fsaveds, local_size);
1312 }
1313 #endif
1314 CHECK_RETURN_OK;
1315 }
1316
check_sljit_set_context(struct sljit_compiler * compiler,sljit_s32 options,sljit_s32 arg_types,sljit_s32 scratches,sljit_s32 saveds,sljit_s32 fscratches,sljit_s32 fsaveds,sljit_s32 local_size)1317 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_set_context(struct sljit_compiler *compiler,
1318 sljit_s32 options, sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds,
1319 sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)
1320 {
1321 SLJIT_UNUSED_ARG(compiler);
1322
1323 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1324 if (options & SLJIT_ENTER_REG_ARG) {
1325 CHECK_ARGUMENT(!(options & ~(0x3 | SLJIT_ENTER_REG_ARG | SLJIT_ENTER_CPU_SPECIFIC_OPTIONS)));
1326 } else {
1327 CHECK_ARGUMENT((options & ~SLJIT_ENTER_CPU_SPECIFIC_OPTIONS) == 0);
1328 }
1329 CHECK_ARGUMENT(SLJIT_KEPT_SAVEDS_COUNT(options) <= 3 && SLJIT_KEPT_SAVEDS_COUNT(options) <= saveds);
1330 CHECK_ARGUMENT(scratches >= 0 && scratches <= SLJIT_NUMBER_OF_REGISTERS);
1331 CHECK_ARGUMENT(saveds >= 0 && saveds <= SLJIT_NUMBER_OF_SAVED_REGISTERS);
1332 CHECK_ARGUMENT(scratches + saveds <= SLJIT_NUMBER_OF_REGISTERS);
1333 CHECK_ARGUMENT(fscratches >= 0 && fscratches <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1334 CHECK_ARGUMENT(fsaveds >= 0 && fsaveds <= SLJIT_NUMBER_OF_SAVED_FLOAT_REGISTERS);
1335 CHECK_ARGUMENT(fscratches + fsaveds <= SLJIT_NUMBER_OF_FLOAT_REGISTERS);
1336 CHECK_ARGUMENT(local_size >= 0 && local_size <= SLJIT_MAX_LOCAL_SIZE);
1337 CHECK_ARGUMENT((arg_types & SLJIT_ARG_FULL_MASK) < SLJIT_ARG_TYPE_F64);
1338 CHECK_ARGUMENT(function_check_arguments(arg_types, scratches, (options & SLJIT_ENTER_REG_ARG) ? 0 : saveds, fscratches));
1339
1340 compiler->last_flags = 0;
1341 #endif
1342 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1343 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1344 fprintf(compiler->verbose, " set_context ret[%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
1345
1346 arg_types >>= SLJIT_ARG_SHIFT;
1347 if (arg_types) {
1348 fprintf(compiler->verbose, "], args[");
1349 do {
1350 fprintf(compiler->verbose, "%s%s", call_arg_names[arg_types & SLJIT_ARG_MASK],
1351 (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) ? "_r" : "");
1352 arg_types >>= SLJIT_ARG_SHIFT;
1353 if (arg_types)
1354 fprintf(compiler->verbose, ",");
1355 } while (arg_types);
1356 }
1357
1358 fprintf(compiler->verbose, "],");
1359
1360 if (options & SLJIT_ENTER_REG_ARG) {
1361 if (SLJIT_KEPT_SAVEDS_COUNT(options) > 0)
1362 fprintf(compiler->verbose, " opt:reg_arg(%d),", SLJIT_KEPT_SAVEDS_COUNT(options));
1363 else
1364 fprintf(compiler->verbose, " opt:reg_arg,");
1365 }
1366
1367 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
1368 if (options & SLJIT_ENTER_USE_VEX) {
1369 fprintf(compiler->verbose, " opt:use_vex,");
1370 }
1371 #endif /* !SLJIT_CONFIG_X86 */
1372
1373 fprintf(compiler->verbose, " scratches:%d, saveds:%d, fscratches:%d, fsaveds:%d, local_size:%d\n",
1374 scratches, saveds, fscratches, fsaveds, local_size);
1375 }
1376 #endif
1377 CHECK_RETURN_OK;
1378 }
1379
1380 #undef SLJIT_ENTER_CPU_SPECIFIC_OPTIONS
1381
check_sljit_emit_return_void(struct sljit_compiler * compiler)1382 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_return_void(struct sljit_compiler *compiler)
1383 {
1384 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1385 compiler->skip_checks = 0;
1386 CHECK_RETURN_OK;
1387 }
1388
1389 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1390 CHECK_ARGUMENT(compiler->last_return == SLJIT_ARG_TYPE_RET_VOID);
1391 #endif
1392
1393 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1394 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1395 fprintf(compiler->verbose, " return_void\n");
1396 }
1397 #endif
1398 CHECK_RETURN_OK;
1399 }
1400
check_sljit_emit_return(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src,sljit_sw srcw)1401 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
1402 {
1403 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1404 CHECK_ARGUMENT(compiler->scratches >= 0);
1405
1406 switch (compiler->last_return) {
1407 case SLJIT_ARG_TYPE_W:
1408 CHECK_ARGUMENT(op >= SLJIT_MOV && op <= SLJIT_MOV_S32);
1409 break;
1410 case SLJIT_ARG_TYPE_32:
1411 CHECK_ARGUMENT(op == SLJIT_MOV32 || (op >= SLJIT_MOV32_U8 && op <= SLJIT_MOV32_S16));
1412 break;
1413 case SLJIT_ARG_TYPE_P:
1414 CHECK_ARGUMENT(op == SLJIT_MOV_P);
1415 break;
1416 case SLJIT_ARG_TYPE_F64:
1417 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
1418 CHECK_ARGUMENT(op == SLJIT_MOV_F64);
1419 break;
1420 case SLJIT_ARG_TYPE_F32:
1421 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
1422 CHECK_ARGUMENT(op == SLJIT_MOV_F32);
1423 break;
1424 default:
1425 /* Context not initialized, void, etc. */
1426 CHECK_ARGUMENT(0);
1427 break;
1428 }
1429
1430 if (GET_OPCODE(op) < SLJIT_MOV_F64) {
1431 FUNCTION_CHECK_SRC(src, srcw);
1432 } else {
1433 FUNCTION_FCHECK(src, srcw, op & SLJIT_32);
1434 }
1435 compiler->last_flags = 0;
1436 #endif
1437 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1438 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1439 if (GET_OPCODE(op) < SLJIT_MOV_F64) {
1440 fprintf(compiler->verbose, " return%s%s ", !(op & SLJIT_32) ? "" : "32",
1441 op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
1442 sljit_verbose_param(compiler, src, srcw);
1443 } else {
1444 fprintf(compiler->verbose, " return%s ", !(op & SLJIT_32) ? ".f64" : ".f32");
1445 sljit_verbose_fparam(compiler, src, srcw);
1446 }
1447 fprintf(compiler->verbose, "\n");
1448 }
1449 #endif
1450 CHECK_RETURN_OK;
1451 }
1452
check_sljit_emit_return_to(struct sljit_compiler * compiler,sljit_s32 src,sljit_sw srcw)1453 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_return_to(struct sljit_compiler *compiler,
1454 sljit_s32 src, sljit_sw srcw)
1455 {
1456 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1457 FUNCTION_CHECK_SRC(src, srcw);
1458 #endif
1459 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1460 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1461 fprintf(compiler->verbose, " return_to ");
1462 sljit_verbose_param(compiler, src, srcw);
1463 fprintf(compiler->verbose, "\n");
1464 }
1465 #endif
1466 CHECK_RETURN_OK;
1467 }
1468
check_sljit_emit_op0(struct sljit_compiler * compiler,sljit_s32 op)1469 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op0(struct sljit_compiler *compiler, sljit_s32 op)
1470 {
1471 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1472 CHECK_ARGUMENT((op >= SLJIT_BREAKPOINT && op <= SLJIT_LMUL_SW)
1473 || ((op & ~SLJIT_32) >= SLJIT_DIVMOD_UW && (op & ~SLJIT_32) <= SLJIT_DIV_SW)
1474 || (op >= SLJIT_ENDBR && op <= SLJIT_SKIP_FRAMES_BEFORE_RETURN));
1475 CHECK_ARGUMENT(GET_OPCODE(op) < SLJIT_LMUL_UW || GET_OPCODE(op) >= SLJIT_ENDBR || compiler->scratches >= 2);
1476 if ((GET_OPCODE(op) >= SLJIT_LMUL_UW && GET_OPCODE(op) <= SLJIT_DIV_SW) || op == SLJIT_SKIP_FRAMES_BEFORE_RETURN)
1477 compiler->last_flags = 0;
1478 #endif
1479 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1480 if (SLJIT_UNLIKELY(!!compiler->verbose))
1481 {
1482 fprintf(compiler->verbose, " %s", op0_names[GET_OPCODE(op) - SLJIT_OP0_BASE]);
1483 if (GET_OPCODE(op) >= SLJIT_DIVMOD_UW && GET_OPCODE(op) <= SLJIT_DIV_SW) {
1484 fprintf(compiler->verbose, (op & SLJIT_32) ? "32" : "w");
1485 }
1486 fprintf(compiler->verbose, "\n");
1487 }
1488 #endif
1489 CHECK_RETURN_OK;
1490 }
1491
check_sljit_emit_op1(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)1492 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op1(struct sljit_compiler *compiler, sljit_s32 op,
1493 sljit_s32 dst, sljit_sw dstw,
1494 sljit_s32 src, sljit_sw srcw)
1495 {
1496 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1497 compiler->skip_checks = 0;
1498 CHECK_RETURN_OK;
1499 }
1500
1501 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1502 CHECK_ARGUMENT(GET_OPCODE(op) >= SLJIT_MOV && GET_OPCODE(op) <= SLJIT_REV_S32);
1503
1504 switch (GET_OPCODE(op)) {
1505 case SLJIT_MOV:
1506 case SLJIT_MOV_U32:
1507 case SLJIT_MOV_S32:
1508 case SLJIT_MOV32:
1509 case SLJIT_MOV_P:
1510 case SLJIT_REV_U32:
1511 case SLJIT_REV_S32:
1512 /* Nothing allowed */
1513 CHECK_ARGUMENT(!(op & (SLJIT_32 | SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
1514 break;
1515 default:
1516 /* Only SLJIT_32 is allowed. */
1517 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
1518 break;
1519 }
1520
1521 FUNCTION_CHECK_DST(dst, dstw);
1522 FUNCTION_CHECK_SRC(src, srcw);
1523 #endif
1524 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1525 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1526 fprintf(compiler->verbose, " %s%s%s ", op1_names[GET_OPCODE(op) - SLJIT_OP1_BASE],
1527 !(op & SLJIT_32) ? "" : "32", op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
1528
1529 sljit_verbose_param(compiler, dst, dstw);
1530 fprintf(compiler->verbose, ", ");
1531 sljit_verbose_param(compiler, src, srcw);
1532 fprintf(compiler->verbose, "\n");
1533 }
1534 #endif
1535 CHECK_RETURN_OK;
1536 }
1537
check_sljit_emit_atomic_load(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst_reg,sljit_s32 mem_reg)1538 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_atomic_load(struct sljit_compiler *compiler, sljit_s32 op,
1539 sljit_s32 dst_reg,
1540 sljit_s32 mem_reg)
1541 {
1542 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1543 compiler->skip_checks = 0;
1544 CHECK_RETURN_OK;
1545 }
1546
1547 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1548 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_ATOMIC));
1549 CHECK_ARGUMENT(GET_OPCODE(op) >= SLJIT_MOV && GET_OPCODE(op) <= SLJIT_MOV_P);
1550 CHECK_ARGUMENT(GET_OPCODE(op) != SLJIT_MOV_S8 && GET_OPCODE(op) != SLJIT_MOV_S16 && GET_OPCODE(op) != SLJIT_MOV_S32);
1551
1552 /* All arguments must be valid registers. */
1553 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
1554 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(mem_reg) && !CHECK_IF_VIRTUAL_REGISTER(mem_reg));
1555
1556 if (op == SLJIT_MOV32_U8 || op == SLJIT_MOV32_U16) {
1557 /* Only SLJIT_32 is allowed. */
1558 CHECK_ARGUMENT(!(op & (VARIABLE_FLAG_MASK | SLJIT_SET_Z)));
1559 } else {
1560 /* Nothing allowed. */
1561 CHECK_ARGUMENT(!(op & (SLJIT_32 | SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
1562 }
1563
1564 compiler->last_flags = 0;
1565 #endif /* SLJIT_ARGUMENT_CHECKS */
1566 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1567 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1568 fprintf(compiler->verbose, " atomic_load%s%s ", !(op & SLJIT_32) ? "" : "32",
1569 op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE]);
1570 sljit_verbose_reg(compiler, dst_reg);
1571 fprintf(compiler->verbose, ", [");
1572 sljit_verbose_reg(compiler, mem_reg);
1573 fprintf(compiler->verbose, "]\n");
1574 }
1575 #endif /* SLJIT_VERBOSE */
1576 CHECK_RETURN_OK;
1577 }
1578
check_sljit_emit_atomic_store(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src_reg,sljit_s32 mem_reg,sljit_s32 temp_reg)1579 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_atomic_store(struct sljit_compiler *compiler, sljit_s32 op,
1580 sljit_s32 src_reg,
1581 sljit_s32 mem_reg,
1582 sljit_s32 temp_reg)
1583 {
1584 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1585 compiler->skip_checks = 0;
1586 CHECK_RETURN_OK;
1587 }
1588
1589 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1590 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_ATOMIC));
1591 CHECK_ARGUMENT(GET_OPCODE(op) >= SLJIT_MOV && GET_OPCODE(op) <= SLJIT_MOV_P);
1592 CHECK_ARGUMENT(GET_OPCODE(op) != SLJIT_MOV_S8 && GET_OPCODE(op) != SLJIT_MOV_S16 && GET_OPCODE(op) != SLJIT_MOV_S32);
1593
1594 /* All arguments must be valid registers. */
1595 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src_reg));
1596 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(mem_reg) && !CHECK_IF_VIRTUAL_REGISTER(mem_reg));
1597 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(temp_reg) && src_reg != temp_reg);
1598
1599 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK) || GET_FLAG_TYPE(op) == SLJIT_ATOMIC_STORED);
1600
1601 if (GET_OPCODE(op) == SLJIT_MOV_U8 || GET_OPCODE(op) == SLJIT_MOV_U16) {
1602 /* Only SLJIT_32, SLJIT_ATOMIC_STORED are allowed. */
1603 CHECK_ARGUMENT(!(op & SLJIT_SET_Z));
1604 } else {
1605 /* Only SLJIT_ATOMIC_STORED is allowed. */
1606 CHECK_ARGUMENT(!(op & (SLJIT_32 | SLJIT_SET_Z)));
1607 }
1608
1609 compiler->last_flags = GET_FLAG_TYPE(op) | (op & SLJIT_32);
1610 #endif /* SLJIT_ARGUMENT_CHECKS */
1611 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1612 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1613 fprintf(compiler->verbose, " atomic_store%s%s%s ", !(op & SLJIT_32) ? "" : "32",
1614 op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE], !(op & VARIABLE_FLAG_MASK) ? "" : ".stored");
1615 sljit_verbose_reg(compiler, src_reg);
1616 fprintf(compiler->verbose, ", [");
1617 sljit_verbose_reg(compiler, mem_reg);
1618 fprintf(compiler->verbose, "], ");
1619 sljit_verbose_reg(compiler, temp_reg);
1620 fprintf(compiler->verbose, "\n");
1621 }
1622 #endif /* SLJIT_VERBOSE */
1623 CHECK_RETURN_OK;
1624 }
1625
check_sljit_emit_op2(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 unset,sljit_s32 dst,sljit_sw dstw,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)1626 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 unset,
1627 sljit_s32 dst, sljit_sw dstw,
1628 sljit_s32 src1, sljit_sw src1w,
1629 sljit_s32 src2, sljit_sw src2w)
1630 {
1631 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1632 compiler->skip_checks = 0;
1633 CHECK_RETURN_OK;
1634 }
1635
1636 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1637 CHECK_ARGUMENT(GET_OPCODE(op) >= SLJIT_ADD && GET_OPCODE(op) <= SLJIT_ROTR);
1638
1639 switch (GET_OPCODE(op)) {
1640 case SLJIT_AND:
1641 case SLJIT_OR:
1642 case SLJIT_XOR:
1643 case SLJIT_SHL:
1644 case SLJIT_MSHL:
1645 case SLJIT_LSHR:
1646 case SLJIT_MLSHR:
1647 case SLJIT_ASHR:
1648 case SLJIT_MASHR:
1649 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK));
1650 break;
1651 case SLJIT_MUL:
1652 CHECK_ARGUMENT(!(op & SLJIT_SET_Z));
1653 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK)
1654 || GET_FLAG_TYPE(op) == SLJIT_OVERFLOW);
1655 break;
1656 case SLJIT_ADD:
1657 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK)
1658 || GET_FLAG_TYPE(op) == GET_FLAG_TYPE(SLJIT_SET_CARRY)
1659 || GET_FLAG_TYPE(op) == SLJIT_OVERFLOW);
1660 break;
1661 case SLJIT_SUB:
1662 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK)
1663 || (GET_FLAG_TYPE(op) >= SLJIT_LESS && GET_FLAG_TYPE(op) <= SLJIT_OVERFLOW)
1664 || GET_FLAG_TYPE(op) == GET_FLAG_TYPE(SLJIT_SET_CARRY));
1665 break;
1666 case SLJIT_ADDC:
1667 case SLJIT_SUBC:
1668 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK)
1669 || GET_FLAG_TYPE(op) == GET_FLAG_TYPE(SLJIT_SET_CARRY));
1670 CHECK_ARGUMENT((compiler->last_flags & 0xff) == GET_FLAG_TYPE(SLJIT_SET_CARRY));
1671 CHECK_ARGUMENT((op & SLJIT_32) == (compiler->last_flags & SLJIT_32));
1672 break;
1673 case SLJIT_ROTL:
1674 case SLJIT_ROTR:
1675 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
1676 break;
1677 default:
1678 SLJIT_UNREACHABLE();
1679 break;
1680 }
1681
1682 if (unset) {
1683 CHECK_ARGUMENT(HAS_FLAGS(op));
1684 } else {
1685 FUNCTION_CHECK_DST(dst, dstw);
1686 }
1687 FUNCTION_CHECK_SRC(src1, src1w);
1688 FUNCTION_CHECK_SRC(src2, src2w);
1689 compiler->last_flags = GET_FLAG_TYPE(op) | (op & (SLJIT_32 | SLJIT_SET_Z));
1690 #endif
1691 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1692 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1693 fprintf(compiler->verbose, " %s%s%s%s%s ", op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE], !(op & SLJIT_32) ? "" : "32",
1694 !(op & SLJIT_SET_Z) ? "" : ".z", !(op & VARIABLE_FLAG_MASK) ? "" : ".",
1695 !(op & VARIABLE_FLAG_MASK) ? "" : jump_names[GET_FLAG_TYPE(op)]);
1696 if (unset)
1697 fprintf(compiler->verbose, "unset");
1698 else
1699 sljit_verbose_param(compiler, dst, dstw);
1700 fprintf(compiler->verbose, ", ");
1701 sljit_verbose_param(compiler, src1, src1w);
1702 fprintf(compiler->verbose, ", ");
1703 sljit_verbose_param(compiler, src2, src2w);
1704 fprintf(compiler->verbose, "\n");
1705 }
1706 #endif
1707 CHECK_RETURN_OK;
1708 }
1709
check_sljit_emit_op2r(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst_reg,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)1710 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op2r(struct sljit_compiler *compiler, sljit_s32 op,
1711 sljit_s32 dst_reg,
1712 sljit_s32 src1, sljit_sw src1w,
1713 sljit_s32 src2, sljit_sw src2w)
1714 {
1715 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1716 CHECK_ARGUMENT((op | SLJIT_32) == SLJIT_MULADD32);
1717 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
1718 FUNCTION_CHECK_SRC(src1, src1w);
1719 FUNCTION_CHECK_SRC(src2, src2w);
1720 compiler->last_flags = 0;
1721 #endif
1722 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1723 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1724 fprintf(compiler->verbose, " %s%s ", op2r_names[GET_OPCODE(op) - SLJIT_OP2R_BASE], !(op & SLJIT_32) ? "" : "32");
1725
1726 sljit_verbose_reg(compiler, dst_reg);
1727 fprintf(compiler->verbose, ", ");
1728 sljit_verbose_param(compiler, src1, src1w);
1729 fprintf(compiler->verbose, ", ");
1730 sljit_verbose_param(compiler, src2, src2w);
1731 fprintf(compiler->verbose, "\n");
1732 }
1733 #endif
1734 CHECK_RETURN_OK;
1735 }
1736
check_sljit_emit_shift_into(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst_reg,sljit_s32 src1_reg,sljit_s32 src2_reg,sljit_s32 src3,sljit_sw src3w)1737 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_shift_into(struct sljit_compiler *compiler, sljit_s32 op,
1738 sljit_s32 dst_reg,
1739 sljit_s32 src1_reg,
1740 sljit_s32 src2_reg,
1741 sljit_s32 src3, sljit_sw src3w)
1742 {
1743 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1744 CHECK_ARGUMENT(GET_OPCODE(op) == SLJIT_SHL || GET_OPCODE(op) == SLJIT_LSHR
1745 || GET_OPCODE(op) == SLJIT_MSHL || GET_OPCODE(op) == SLJIT_MLSHR);
1746 CHECK_ARGUMENT((op & ~(0xff | SLJIT_32 | SLJIT_SHIFT_INTO_NON_ZERO)) == 0);
1747 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
1748 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src1_reg));
1749 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src2_reg));
1750 FUNCTION_CHECK_SRC(src3, src3w);
1751 CHECK_ARGUMENT(dst_reg != src2_reg);
1752 #endif
1753 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1754 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1755 fprintf(compiler->verbose, " %s%s.into%s ", op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE], !(op & SLJIT_32) ? "" : "32",
1756 (op & SLJIT_SHIFT_INTO_NON_ZERO) ? ".nz" : "");
1757
1758 sljit_verbose_reg(compiler, dst_reg);
1759 fprintf(compiler->verbose, ", ");
1760 sljit_verbose_reg(compiler, src1_reg);
1761 fprintf(compiler->verbose, ", ");
1762 sljit_verbose_reg(compiler, src2_reg);
1763 fprintf(compiler->verbose, ", ");
1764 sljit_verbose_param(compiler, src3, src3w);
1765 fprintf(compiler->verbose, "\n");
1766 }
1767 #endif
1768 CHECK_RETURN_OK;
1769 }
1770
check_sljit_emit_op_src(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src,sljit_sw srcw)1771 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_src(struct sljit_compiler *compiler, sljit_s32 op,
1772 sljit_s32 src, sljit_sw srcw)
1773 {
1774 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1775 CHECK_ARGUMENT(op >= SLJIT_FAST_RETURN && op <= SLJIT_PREFETCH_ONCE);
1776 FUNCTION_CHECK_SRC(src, srcw);
1777
1778 if (op == SLJIT_FAST_RETURN || op == SLJIT_SKIP_FRAMES_BEFORE_FAST_RETURN) {
1779 CHECK_ARGUMENT(src != SLJIT_IMM);
1780 compiler->last_flags = 0;
1781 } else if (op >= SLJIT_PREFETCH_L1 && op <= SLJIT_PREFETCH_ONCE) {
1782 CHECK_ARGUMENT(src & SLJIT_MEM);
1783 }
1784 #endif
1785 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1786 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1787 fprintf(compiler->verbose, " %s ", op_src_dst_names[op - SLJIT_OP_SRC_DST_BASE]);
1788 sljit_verbose_param(compiler, src, srcw);
1789 fprintf(compiler->verbose, "\n");
1790 }
1791 #endif
1792 CHECK_RETURN_OK;
1793 }
1794
check_sljit_emit_op_dst(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw)1795 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_dst(struct sljit_compiler *compiler, sljit_s32 op,
1796 sljit_s32 dst, sljit_sw dstw)
1797 {
1798 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1799 CHECK_ARGUMENT(op >= SLJIT_FAST_ENTER && op <= SLJIT_GET_RETURN_ADDRESS);
1800 FUNCTION_CHECK_DST(dst, dstw);
1801
1802 if (op == SLJIT_FAST_ENTER)
1803 compiler->last_flags = 0;
1804 #endif
1805 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1806 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1807 fprintf(compiler->verbose, " %s ", op_src_dst_names[op - SLJIT_OP_SRC_DST_BASE]);
1808 sljit_verbose_param(compiler, dst, dstw);
1809 fprintf(compiler->verbose, "\n");
1810 }
1811 #endif
1812 CHECK_RETURN_OK;
1813 }
1814
check_sljit_get_register_index(sljit_s32 type,sljit_s32 reg)1815 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_get_register_index(sljit_s32 type, sljit_s32 reg)
1816 {
1817 SLJIT_UNUSED_ARG(type);
1818 SLJIT_UNUSED_ARG(reg);
1819 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1820 if (type == SLJIT_GP_REGISTER) {
1821 CHECK_ARGUMENT((reg > 0 && reg <= SLJIT_NUMBER_OF_REGISTERS)
1822 || (reg >= SLJIT_TMP_REGISTER_BASE && reg < (SLJIT_TMP_REGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_REGISTERS)));
1823 } else {
1824 CHECK_ARGUMENT(type == SLJIT_FLOAT_REGISTER || ((type >> 12) == 0 || ((type >> 12) >= 3 && (type >> 12) <= 6)));
1825 CHECK_ARGUMENT((reg > 0 && reg <= SLJIT_NUMBER_OF_FLOAT_REGISTERS)
1826 || (reg >= SLJIT_TMP_FREGISTER_BASE && reg < (SLJIT_TMP_FREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_FLOAT_REGISTERS)));
1827 }
1828 #endif
1829 CHECK_RETURN_OK;
1830 }
1831
check_sljit_emit_op_custom(struct sljit_compiler * compiler,void * instruction,sljit_u32 size)1832 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_custom(struct sljit_compiler *compiler,
1833 void *instruction, sljit_u32 size)
1834 {
1835 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1836 sljit_u32 i;
1837 #endif
1838
1839 SLJIT_UNUSED_ARG(compiler);
1840
1841 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1842 CHECK_ARGUMENT(instruction);
1843
1844 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
1845 CHECK_ARGUMENT(size > 0 && size < 16);
1846 #elif (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
1847 CHECK_ARGUMENT((size == 2 && (((sljit_sw)instruction) & 0x1) == 0)
1848 || (size == 4 && (((sljit_sw)instruction) & 0x3) == 0));
1849 #elif (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X)
1850 CHECK_ARGUMENT(size == 2 || size == 4 || size == 6);
1851 #else
1852 CHECK_ARGUMENT(size == 4 && (((sljit_sw)instruction) & 0x3) == 0);
1853 #endif
1854
1855 compiler->last_flags = 0;
1856 #endif
1857 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1858 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1859 fprintf(compiler->verbose, " op_custom");
1860 for (i = 0; i < size; i++)
1861 fprintf(compiler->verbose, " 0x%x", ((sljit_u8*)instruction)[i]);
1862 fprintf(compiler->verbose, "\n");
1863 }
1864 #endif
1865 CHECK_RETURN_OK;
1866 }
1867
check_sljit_emit_fop1(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)1868 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1(struct sljit_compiler *compiler, sljit_s32 op,
1869 sljit_s32 dst, sljit_sw dstw,
1870 sljit_s32 src, sljit_sw srcw)
1871 {
1872 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1873 compiler->skip_checks = 0;
1874 CHECK_RETURN_OK;
1875 }
1876
1877 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1878 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
1879 CHECK_ARGUMENT(GET_OPCODE(op) >= SLJIT_MOV_F64 && GET_OPCODE(op) <= SLJIT_ABS_F64);
1880 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
1881 FUNCTION_FCHECK(src, srcw, op & SLJIT_32);
1882 FUNCTION_FCHECK(dst, dstw, op & SLJIT_32);
1883 #endif
1884 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1885 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1886 if (GET_OPCODE(op) == SLJIT_CONV_F64_FROM_F32)
1887 fprintf(compiler->verbose, " %s%s ", fop1_names[SLJIT_CONV_F64_FROM_F32 - SLJIT_FOP1_BASE],
1888 (op & SLJIT_32) ? ".f32.from.f64" : ".f64.from.f32");
1889 else
1890 fprintf(compiler->verbose, " %s%s ", fop1_names[GET_OPCODE(op) - SLJIT_FOP1_BASE],
1891 (op & SLJIT_32) ? ".f32" : ".f64");
1892
1893 sljit_verbose_fparam(compiler, dst, dstw);
1894 fprintf(compiler->verbose, ", ");
1895 sljit_verbose_fparam(compiler, src, srcw);
1896 fprintf(compiler->verbose, "\n");
1897 }
1898 #endif
1899 CHECK_RETURN_OK;
1900 }
1901
check_sljit_emit_fop1_cmp(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)1902 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1_cmp(struct sljit_compiler *compiler, sljit_s32 op,
1903 sljit_s32 src1, sljit_sw src1w,
1904 sljit_s32 src2, sljit_sw src2w)
1905 {
1906 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1907 compiler->last_flags = GET_FLAG_TYPE(op) | (op & SLJIT_32);
1908 #endif
1909
1910 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1911 compiler->skip_checks = 0;
1912 CHECK_RETURN_OK;
1913 }
1914
1915 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1916 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
1917 CHECK_ARGUMENT(GET_OPCODE(op) == SLJIT_CMP_F64);
1918 CHECK_ARGUMENT(!(op & SLJIT_SET_Z));
1919 CHECK_ARGUMENT((op & VARIABLE_FLAG_MASK)
1920 || (GET_FLAG_TYPE(op) >= SLJIT_F_EQUAL && GET_FLAG_TYPE(op) <= SLJIT_ORDERED_LESS_EQUAL));
1921 FUNCTION_FCHECK(src1, src1w, op & SLJIT_32);
1922 FUNCTION_FCHECK(src2, src2w, op & SLJIT_32);
1923 #endif
1924 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1925 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1926 fprintf(compiler->verbose, " %s%s", fop1_names[SLJIT_CMP_F64 - SLJIT_FOP1_BASE], (op & SLJIT_32) ? ".f32" : ".f64");
1927 if (op & VARIABLE_FLAG_MASK) {
1928 fprintf(compiler->verbose, ".%s", jump_names[GET_FLAG_TYPE(op)]);
1929 }
1930 fprintf(compiler->verbose, " ");
1931 sljit_verbose_fparam(compiler, src1, src1w);
1932 fprintf(compiler->verbose, ", ");
1933 sljit_verbose_fparam(compiler, src2, src2w);
1934 fprintf(compiler->verbose, "\n");
1935 }
1936 #endif
1937 CHECK_RETURN_OK;
1938 }
1939
check_sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)1940 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1_conv_sw_from_f64(struct sljit_compiler *compiler, sljit_s32 op,
1941 sljit_s32 dst, sljit_sw dstw,
1942 sljit_s32 src, sljit_sw srcw)
1943 {
1944 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1945 compiler->skip_checks = 0;
1946 CHECK_RETURN_OK;
1947 }
1948
1949 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1950 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
1951 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
1952 FUNCTION_FCHECK(src, srcw, op & SLJIT_32);
1953 FUNCTION_CHECK_DST(dst, dstw);
1954 #endif
1955 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1956 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1957 fprintf(compiler->verbose, " %s%s.from%s ", fop1_names[GET_OPCODE(op) - SLJIT_FOP1_BASE],
1958 fop1_conv_types[GET_OPCODE(op) - SLJIT_CONV_SW_FROM_F64],
1959 (op & SLJIT_32) ? ".f32" : ".f64");
1960 sljit_verbose_param(compiler, dst, dstw);
1961 fprintf(compiler->verbose, ", ");
1962 sljit_verbose_fparam(compiler, src, srcw);
1963 fprintf(compiler->verbose, "\n");
1964 }
1965 #endif
1966 CHECK_RETURN_OK;
1967 }
1968
check_sljit_emit_fop1_conv_f64_from_w(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src,sljit_sw srcw)1969 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop1_conv_f64_from_w(struct sljit_compiler *compiler, sljit_s32 op,
1970 sljit_s32 dst, sljit_sw dstw,
1971 sljit_s32 src, sljit_sw srcw)
1972 {
1973 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
1974 compiler->skip_checks = 0;
1975 CHECK_RETURN_OK;
1976 }
1977
1978 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
1979 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
1980 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
1981 FUNCTION_CHECK_SRC(src, srcw);
1982 FUNCTION_FCHECK(dst, dstw, op & SLJIT_32);
1983 #endif
1984 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
1985 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
1986 fprintf(compiler->verbose, " %s%s.from.%s ", fop1_names[GET_OPCODE(op) - SLJIT_FOP1_BASE],
1987 (op & SLJIT_32) ? ".f32" : ".f64",
1988 fop1_conv_types[GET_OPCODE(op) - SLJIT_CONV_SW_FROM_F64]);
1989 sljit_verbose_fparam(compiler, dst, dstw);
1990 fprintf(compiler->verbose, ", ");
1991 sljit_verbose_param(compiler, src, srcw);
1992 fprintf(compiler->verbose, "\n");
1993 }
1994 #endif
1995 CHECK_RETURN_OK;
1996 }
1997
check_sljit_emit_fop2(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)1998 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop2(struct sljit_compiler *compiler, sljit_s32 op,
1999 sljit_s32 dst, sljit_sw dstw,
2000 sljit_s32 src1, sljit_sw src1w,
2001 sljit_s32 src2, sljit_sw src2w)
2002 {
2003 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2004 compiler->skip_checks = 0;
2005 CHECK_RETURN_OK;
2006 }
2007
2008 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2009 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2010 CHECK_ARGUMENT(GET_OPCODE(op) >= SLJIT_ADD_F64 && GET_OPCODE(op) <= SLJIT_DIV_F64);
2011 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
2012 FUNCTION_FCHECK(src1, src1w, op & SLJIT_32);
2013 FUNCTION_FCHECK(src2, src2w, op & SLJIT_32);
2014 FUNCTION_FCHECK(dst, dstw, op & SLJIT_32);
2015 #endif
2016 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2017 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2018 fprintf(compiler->verbose, " %s%s ", fop2_names[GET_OPCODE(op) - SLJIT_FOP2_BASE], (op & SLJIT_32) ? ".f32" : ".f64");
2019 sljit_verbose_fparam(compiler, dst, dstw);
2020 fprintf(compiler->verbose, ", ");
2021 sljit_verbose_fparam(compiler, src1, src1w);
2022 fprintf(compiler->verbose, ", ");
2023 sljit_verbose_fparam(compiler, src2, src2w);
2024 fprintf(compiler->verbose, "\n");
2025 }
2026 #endif
2027 CHECK_RETURN_OK;
2028 }
2029
check_sljit_emit_fop2r(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst_freg,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)2030 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fop2r(struct sljit_compiler *compiler, sljit_s32 op,
2031 sljit_s32 dst_freg,
2032 sljit_s32 src1, sljit_sw src1w,
2033 sljit_s32 src2, sljit_sw src2w)
2034 {
2035 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2036 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2037 CHECK_ARGUMENT(GET_OPCODE(op) == SLJIT_COPYSIGN_F64);
2038 FUNCTION_FCHECK(src1, src1w, op & SLJIT_32);
2039 FUNCTION_FCHECK(src2, src2w, op & SLJIT_32);
2040 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(dst_freg, op & SLJIT_32));
2041 #endif
2042 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2043 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2044 fprintf(compiler->verbose, " %s%s ", fop2r_names[GET_OPCODE(op) - SLJIT_FOP2R_BASE], (op & SLJIT_32) ? ".f32" : ".f64");
2045 sljit_verbose_freg(compiler, dst_freg);
2046 fprintf(compiler->verbose, ", ");
2047 sljit_verbose_fparam(compiler, src1, src1w);
2048 fprintf(compiler->verbose, ", ");
2049 sljit_verbose_fparam(compiler, src2, src2w);
2050 fprintf(compiler->verbose, "\n");
2051 }
2052 #endif
2053 CHECK_RETURN_OK;
2054 }
2055
check_sljit_emit_fset32(struct sljit_compiler * compiler,sljit_s32 freg,sljit_f32 value)2056 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fset32(struct sljit_compiler *compiler,
2057 sljit_s32 freg, sljit_f32 value)
2058 {
2059 SLJIT_UNUSED_ARG(value);
2060
2061 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2062 compiler->skip_checks = 0;
2063 CHECK_RETURN_OK;
2064 }
2065
2066 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2067 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2068 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 1));
2069 #endif
2070 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2071 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2072 fprintf(compiler->verbose, " fset32 ");
2073 sljit_verbose_freg(compiler, freg);
2074 fprintf(compiler->verbose, ", %f\n", value);
2075 }
2076 #endif
2077 CHECK_RETURN_OK;
2078 }
2079
check_sljit_emit_fset64(struct sljit_compiler * compiler,sljit_s32 freg,sljit_f64 value)2080 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fset64(struct sljit_compiler *compiler,
2081 sljit_s32 freg, sljit_f64 value)
2082 {
2083 SLJIT_UNUSED_ARG(value);
2084
2085 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2086 compiler->skip_checks = 0;
2087 CHECK_RETURN_OK;
2088 }
2089
2090 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2091 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2092 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 0));
2093 #endif
2094 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2095 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2096 fprintf(compiler->verbose, " fset64 ");
2097 sljit_verbose_freg(compiler, freg);
2098 fprintf(compiler->verbose, ", %f\n", value);
2099 }
2100 #endif
2101 CHECK_RETURN_OK;
2102 }
2103
check_sljit_emit_fcopy(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 freg,sljit_s32 reg)2104 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fcopy(struct sljit_compiler *compiler, sljit_s32 op,
2105 sljit_s32 freg, sljit_s32 reg)
2106 {
2107 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2108 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2109 CHECK_ARGUMENT(GET_OPCODE(op) >= SLJIT_COPY_TO_F64 && GET_OPCODE(op) <= SLJIT_COPY_FROM_F64);
2110 CHECK_ARGUMENT(!(op & (SLJIT_SET_Z | VARIABLE_FLAG_MASK)));
2111 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, op & SLJIT_32));
2112
2113 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
2114 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2115 #else /* !SLJIT_64BIT_ARCHITECTURE */
2116 switch (op) {
2117 case SLJIT_COPY32_TO_F32:
2118 case SLJIT_COPY32_FROM_F32:
2119 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2120 break;
2121 case SLJIT_COPY_TO_F64:
2122 case SLJIT_COPY_FROM_F64:
2123 if (reg & REG_PAIR_MASK) {
2124 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_FIRST(reg)));
2125 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_SECOND(reg)));
2126
2127 if (op == SLJIT_COPY_TO_F64)
2128 break;
2129
2130 CHECK_ARGUMENT(REG_PAIR_FIRST(reg) != REG_PAIR_SECOND(reg));
2131 break;
2132 }
2133
2134 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2135 break;
2136 }
2137 #endif /* SLJIT_64BIT_ARCHITECTURE */
2138 #endif
2139 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2140 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2141 fprintf(compiler->verbose, " copy%s_%s_f%s ", (op & SLJIT_32) ? "32" : "",
2142 GET_OPCODE(op) == SLJIT_COPY_TO_F64 ? "to" : "from", (op & SLJIT_32) ? "32" : "64");
2143
2144 sljit_verbose_freg(compiler, freg);
2145
2146 if (reg & REG_PAIR_MASK) {
2147 fprintf(compiler->verbose, ", {");
2148 sljit_verbose_reg(compiler, REG_PAIR_FIRST(reg));
2149 fprintf(compiler->verbose, ", ");
2150 sljit_verbose_reg(compiler, REG_PAIR_SECOND(reg));
2151 fprintf(compiler->verbose, "}\n");
2152 } else {
2153 fprintf(compiler->verbose, ", ");
2154 sljit_verbose_reg(compiler, reg);
2155 fprintf(compiler->verbose, "\n");
2156 }
2157 }
2158 #endif
2159 CHECK_RETURN_OK;
2160 }
2161
check_sljit_emit_label(struct sljit_compiler * compiler)2162 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_label(struct sljit_compiler *compiler)
2163 {
2164 SLJIT_UNUSED_ARG(compiler);
2165
2166 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2167 compiler->skip_checks = 0;
2168 CHECK_RETURN_OK;
2169 }
2170
2171 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2172 compiler->last_flags = 0;
2173 #endif
2174
2175 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2176 if (SLJIT_UNLIKELY(!!compiler->verbose))
2177 fprintf(compiler->verbose, "label:\n");
2178 #endif
2179 CHECK_RETURN_OK;
2180 }
2181
2182 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2183 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
2184 || (defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM)
2185 #define CHECK_UNORDERED(type, last_flags) \
2186 ((((type) & 0xfe) == SLJIT_ORDERED) && \
2187 ((last_flags) & 0xff) >= SLJIT_UNORDERED && ((last_flags) & 0xff) <= SLJIT_ORDERED_LESS_EQUAL)
2188 #else
2189 #define CHECK_UNORDERED(type, last_flags) 0
2190 #endif
2191 #endif /* SLJIT_ARGUMENT_CHECKS */
2192
check_sljit_emit_jump(struct sljit_compiler * compiler,sljit_s32 type)2193 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_jump(struct sljit_compiler *compiler, sljit_s32 type)
2194 {
2195 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2196 compiler->skip_checks = 0;
2197 CHECK_RETURN_OK;
2198 }
2199
2200 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2201 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP)));
2202 CHECK_ARGUMENT((type & 0xff) >= SLJIT_EQUAL && (type & 0xff) <= SLJIT_FAST_CALL);
2203
2204 if ((type & 0xff) < SLJIT_JUMP) {
2205 if ((type & 0xff) <= SLJIT_NOT_ZERO)
2206 CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2207 else if ((compiler->last_flags & 0xff) == SLJIT_CARRY) {
2208 CHECK_ARGUMENT((type & 0xfe) == SLJIT_CARRY);
2209 compiler->last_flags = 0;
2210 } else
2211 CHECK_ARGUMENT((type & 0xfe) == (compiler->last_flags & 0xff)
2212 || CHECK_UNORDERED(type, compiler->last_flags));
2213 }
2214 #endif
2215 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2216 if (SLJIT_UNLIKELY(!!compiler->verbose))
2217 fprintf(compiler->verbose, " jump%s %s\n", !(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r",
2218 jump_names[type & 0xff]);
2219 #endif
2220 CHECK_RETURN_OK;
2221 }
2222
check_sljit_emit_call(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 arg_types)2223 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_call(struct sljit_compiler *compiler, sljit_s32 type,
2224 sljit_s32 arg_types)
2225 {
2226 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2227 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP | SLJIT_CALL_RETURN)));
2228 CHECK_ARGUMENT((type & 0xff) >= SLJIT_CALL && (type & 0xff) <= SLJIT_CALL_REG_ARG);
2229 CHECK_ARGUMENT(function_check_arguments(arg_types, compiler->scratches, -1, compiler->fscratches));
2230
2231 if (type & SLJIT_CALL_RETURN) {
2232 CHECK_ARGUMENT((arg_types & SLJIT_ARG_MASK) == compiler->last_return);
2233
2234 if (compiler->options & SLJIT_ENTER_REG_ARG) {
2235 CHECK_ARGUMENT((type & 0xff) == SLJIT_CALL_REG_ARG);
2236 } else {
2237 CHECK_ARGUMENT((type & 0xff) != SLJIT_CALL_REG_ARG);
2238 }
2239 }
2240 #endif
2241 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2242 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2243 fprintf(compiler->verbose, " %s%s%s ret[%s", jump_names[type & 0xff],
2244 !(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r",
2245 !(type & SLJIT_CALL_RETURN) ? "" : ".ret",
2246 call_arg_names[arg_types & SLJIT_ARG_MASK]);
2247
2248 arg_types >>= SLJIT_ARG_SHIFT;
2249 if (arg_types) {
2250 fprintf(compiler->verbose, "], args[");
2251 do {
2252 fprintf(compiler->verbose, "%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
2253 arg_types >>= SLJIT_ARG_SHIFT;
2254 if (arg_types)
2255 fprintf(compiler->verbose, ",");
2256 } while (arg_types);
2257 }
2258 fprintf(compiler->verbose, "]\n");
2259 }
2260 #endif
2261 CHECK_RETURN_OK;
2262 }
2263
check_sljit_emit_cmp(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)2264 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_cmp(struct sljit_compiler *compiler, sljit_s32 type,
2265 sljit_s32 src1, sljit_sw src1w,
2266 sljit_s32 src2, sljit_sw src2w)
2267 {
2268 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2269 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP | SLJIT_32)));
2270 CHECK_ARGUMENT((type & 0xff) >= SLJIT_EQUAL && (type & 0xff) <= SLJIT_SIG_LESS_EQUAL);
2271 FUNCTION_CHECK_SRC(src1, src1w);
2272 FUNCTION_CHECK_SRC(src2, src2w);
2273 compiler->last_flags = 0;
2274 #endif
2275 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2276 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2277 fprintf(compiler->verbose, " cmp%s%s %s, ", (type & SLJIT_32) ? "32" : "",
2278 !(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r", jump_names[type & 0xff]);
2279 sljit_verbose_param(compiler, src1, src1w);
2280 fprintf(compiler->verbose, ", ");
2281 sljit_verbose_param(compiler, src2, src2w);
2282 fprintf(compiler->verbose, "\n");
2283 }
2284 #endif
2285 CHECK_RETURN_OK;
2286 }
2287
check_sljit_emit_fcmp(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)2288 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fcmp(struct sljit_compiler *compiler, sljit_s32 type,
2289 sljit_s32 src1, sljit_sw src1w,
2290 sljit_s32 src2, sljit_sw src2w)
2291 {
2292 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2293 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2294 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_REWRITABLE_JUMP | SLJIT_32)));
2295 CHECK_ARGUMENT((type & 0xff) >= SLJIT_F_EQUAL && (type & 0xff) <= SLJIT_ORDERED_LESS_EQUAL);
2296 FUNCTION_FCHECK(src1, src1w, type & SLJIT_32);
2297 FUNCTION_FCHECK(src2, src2w, type & SLJIT_32);
2298 compiler->last_flags = 0;
2299 #endif
2300 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2301 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2302 fprintf(compiler->verbose, " fcmp%s%s %s, ", (type & SLJIT_32) ? ".f32" : ".f64",
2303 !(type & SLJIT_REWRITABLE_JUMP) ? "" : ".r", jump_names[type & 0xff]);
2304 sljit_verbose_fparam(compiler, src1, src1w);
2305 fprintf(compiler->verbose, ", ");
2306 sljit_verbose_fparam(compiler, src2, src2w);
2307 fprintf(compiler->verbose, "\n");
2308 }
2309 #endif
2310 CHECK_RETURN_OK;
2311 }
2312
check_sljit_emit_ijump(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 src,sljit_sw srcw)2313 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_ijump(struct sljit_compiler *compiler, sljit_s32 type,
2314 sljit_s32 src, sljit_sw srcw)
2315 {
2316 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2317 compiler->skip_checks = 0;
2318 CHECK_RETURN_OK;
2319 }
2320
2321 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2322 CHECK_ARGUMENT(type >= SLJIT_JUMP && type <= SLJIT_FAST_CALL);
2323 FUNCTION_CHECK_SRC(src, srcw);
2324 #endif
2325 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2326 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2327 fprintf(compiler->verbose, " ijump.%s ", jump_names[type]);
2328 sljit_verbose_param(compiler, src, srcw);
2329 fprintf(compiler->verbose, "\n");
2330 }
2331 #endif
2332 CHECK_RETURN_OK;
2333 }
2334
check_sljit_emit_icall(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 arg_types,sljit_s32 src,sljit_sw srcw)2335 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_icall(struct sljit_compiler *compiler, sljit_s32 type,
2336 sljit_s32 arg_types,
2337 sljit_s32 src, sljit_sw srcw)
2338 {
2339 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2340 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_CALL_RETURN)));
2341 CHECK_ARGUMENT((type & 0xff) >= SLJIT_CALL && (type & 0xff) <= SLJIT_CALL_REG_ARG);
2342 CHECK_ARGUMENT(function_check_arguments(arg_types, compiler->scratches, -1, compiler->fscratches));
2343 FUNCTION_CHECK_SRC(src, srcw);
2344
2345 if (type & SLJIT_CALL_RETURN) {
2346 CHECK_ARGUMENT((arg_types & SLJIT_ARG_MASK) == compiler->last_return);
2347
2348 if (compiler->options & SLJIT_ENTER_REG_ARG) {
2349 CHECK_ARGUMENT((type & 0xff) == SLJIT_CALL_REG_ARG);
2350 } else {
2351 CHECK_ARGUMENT((type & 0xff) != SLJIT_CALL_REG_ARG);
2352 }
2353 }
2354 #endif
2355 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2356 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2357 fprintf(compiler->verbose, " i%s%s ret[%s", jump_names[type & 0xff],
2358 !(type & SLJIT_CALL_RETURN) ? "" : ".ret",
2359 call_arg_names[arg_types & SLJIT_ARG_MASK]);
2360
2361 arg_types >>= SLJIT_ARG_SHIFT;
2362 if (arg_types) {
2363 fprintf(compiler->verbose, "], args[");
2364 do {
2365 fprintf(compiler->verbose, "%s", call_arg_names[arg_types & SLJIT_ARG_MASK]);
2366 arg_types >>= SLJIT_ARG_SHIFT;
2367 if (arg_types)
2368 fprintf(compiler->verbose, ",");
2369 } while (arg_types);
2370 }
2371 fprintf(compiler->verbose, "], ");
2372 sljit_verbose_param(compiler, src, srcw);
2373 fprintf(compiler->verbose, "\n");
2374 }
2375 #endif
2376 CHECK_RETURN_OK;
2377 }
2378
check_sljit_emit_op_flags(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst,sljit_sw dstw,sljit_s32 type)2379 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_op_flags(struct sljit_compiler *compiler, sljit_s32 op,
2380 sljit_s32 dst, sljit_sw dstw,
2381 sljit_s32 type)
2382 {
2383 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2384 CHECK_ARGUMENT(type >= SLJIT_EQUAL && type <= SLJIT_ORDERED_LESS_EQUAL);
2385 CHECK_ARGUMENT(op == SLJIT_MOV || op == SLJIT_MOV32
2386 || (GET_OPCODE(op) >= SLJIT_AND && GET_OPCODE(op) <= SLJIT_XOR));
2387 CHECK_ARGUMENT(!(op & VARIABLE_FLAG_MASK));
2388
2389 if (type <= SLJIT_NOT_ZERO)
2390 CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2391 else
2392 CHECK_ARGUMENT((type & 0xfe) == (compiler->last_flags & 0xff)
2393 || CHECK_UNORDERED(type, compiler->last_flags));
2394
2395 FUNCTION_CHECK_DST(dst, dstw);
2396
2397 if (GET_OPCODE(op) >= SLJIT_ADD)
2398 compiler->last_flags = GET_FLAG_TYPE(op) | (op & (SLJIT_32 | SLJIT_SET_Z));
2399 #endif
2400 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2401 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2402 fprintf(compiler->verbose, " flags.%s%s%s ",
2403 GET_OPCODE(op) < SLJIT_OP2_BASE ? "mov" : op2_names[GET_OPCODE(op) - SLJIT_OP2_BASE],
2404 GET_OPCODE(op) < SLJIT_OP2_BASE ? op1_types[GET_OPCODE(op) - SLJIT_OP1_BASE] : ((op & SLJIT_32) ? "32" : ""),
2405 !(op & SLJIT_SET_Z) ? "" : ".z");
2406 sljit_verbose_param(compiler, dst, dstw);
2407 fprintf(compiler->verbose, ", %s\n", jump_names[type]);
2408 }
2409 #endif
2410 CHECK_RETURN_OK;
2411 }
2412
check_sljit_emit_select(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 dst_reg,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2_reg)2413 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_select(struct sljit_compiler *compiler, sljit_s32 type,
2414 sljit_s32 dst_reg,
2415 sljit_s32 src1, sljit_sw src1w,
2416 sljit_s32 src2_reg)
2417 {
2418 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2419 sljit_s32 cond = type & ~SLJIT_32;
2420
2421 CHECK_ARGUMENT(cond >= SLJIT_EQUAL && cond <= SLJIT_ORDERED_LESS_EQUAL);
2422
2423 CHECK_ARGUMENT(compiler->scratches != -1 && compiler->saveds != -1);
2424 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(dst_reg));
2425 FUNCTION_CHECK_SRC(src1, src1w);
2426 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(src2_reg));
2427
2428 if (cond <= SLJIT_NOT_ZERO)
2429 CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2430 else if ((compiler->last_flags & 0xff) == SLJIT_CARRY) {
2431 CHECK_ARGUMENT((type & 0xfe) == SLJIT_CARRY);
2432 compiler->last_flags = 0;
2433 } else
2434 CHECK_ARGUMENT((cond & 0xfe) == (compiler->last_flags & 0xff)
2435 || CHECK_UNORDERED(cond, compiler->last_flags));
2436 #endif
2437 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2438 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2439 fprintf(compiler->verbose, " select%s %s, ",
2440 !(type & SLJIT_32) ? "" : "32",
2441 jump_names[type & ~SLJIT_32]);
2442 sljit_verbose_reg(compiler, dst_reg);
2443 fprintf(compiler->verbose, ", ");
2444 sljit_verbose_param(compiler, src1, src1w);
2445 fprintf(compiler->verbose, ", ");
2446 sljit_verbose_reg(compiler, src2_reg);
2447 fprintf(compiler->verbose, "\n");
2448 }
2449 #endif
2450 CHECK_RETURN_OK;
2451 }
2452
check_sljit_emit_fselect(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 dst_freg,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2_freg)2453 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fselect(struct sljit_compiler *compiler, sljit_s32 type,
2454 sljit_s32 dst_freg,
2455 sljit_s32 src1, sljit_sw src1w,
2456 sljit_s32 src2_freg)
2457 {
2458 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2459 sljit_s32 cond = type & ~SLJIT_32;
2460
2461 CHECK_ARGUMENT(cond >= SLJIT_EQUAL && cond <= SLJIT_ORDERED_LESS_EQUAL);
2462
2463 CHECK_ARGUMENT(compiler->fscratches != -1 && compiler->fsaveds != -1);
2464 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(dst_freg, type & SLJIT_32));
2465 FUNCTION_FCHECK(src1, src1w, type & SLJIT_32);
2466 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(src2_freg, type & SLJIT_32));
2467
2468 if (cond <= SLJIT_NOT_ZERO)
2469 CHECK_ARGUMENT(compiler->last_flags & SLJIT_SET_Z);
2470 else if ((compiler->last_flags & 0xff) == SLJIT_CARRY) {
2471 CHECK_ARGUMENT((type & 0xfe) == SLJIT_CARRY);
2472 compiler->last_flags = 0;
2473 } else
2474 CHECK_ARGUMENT((cond & 0xfe) == (compiler->last_flags & 0xff)
2475 || CHECK_UNORDERED(cond, compiler->last_flags));
2476 #endif
2477 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2478 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2479 fprintf(compiler->verbose, " fselect%s %s, ",
2480 !(type & SLJIT_32) ? "" : "32",
2481 jump_names[type & ~SLJIT_32]);
2482 sljit_verbose_freg(compiler, dst_freg);
2483 fprintf(compiler->verbose, ", ");
2484 sljit_verbose_fparam(compiler, src1, src1w);
2485 fprintf(compiler->verbose, ", ");
2486 sljit_verbose_freg(compiler, src2_freg);
2487 fprintf(compiler->verbose, "\n");
2488 }
2489 #endif
2490 CHECK_RETURN_OK;
2491 }
2492
check_sljit_emit_mem(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 reg,sljit_s32 mem,sljit_sw memw)2493 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_mem(struct sljit_compiler *compiler, sljit_s32 type,
2494 sljit_s32 reg,
2495 sljit_s32 mem, sljit_sw memw)
2496 {
2497 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2498 sljit_s32 allowed_flags;
2499 #endif /* SLJIT_ARGUMENT_CHECKS */
2500
2501 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2502 compiler->skip_checks = 0;
2503 CHECK_RETURN_OK;
2504 }
2505
2506 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2507 if (type & SLJIT_MEM_UNALIGNED) {
2508 CHECK_ARGUMENT(!(type & (SLJIT_MEM_ALIGNED_16 | SLJIT_MEM_ALIGNED_32)));
2509 } else if (type & SLJIT_MEM_ALIGNED_16) {
2510 CHECK_ARGUMENT(!(type & SLJIT_MEM_ALIGNED_32));
2511 } else {
2512 CHECK_ARGUMENT((reg & REG_PAIR_MASK) || (type & SLJIT_MEM_ALIGNED_32));
2513 }
2514
2515 allowed_flags = SLJIT_MEM_UNALIGNED;
2516
2517 switch (type & 0xff) {
2518 case SLJIT_MOV_P:
2519 case SLJIT_MOV:
2520 allowed_flags |= SLJIT_MEM_ALIGNED_32;
2521 /* fallthrough */
2522 case SLJIT_MOV_U32:
2523 case SLJIT_MOV_S32:
2524 case SLJIT_MOV32:
2525 allowed_flags |= SLJIT_MEM_ALIGNED_16;
2526 break;
2527 }
2528
2529 CHECK_ARGUMENT((type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | allowed_flags)) == 0);
2530
2531 if (reg & REG_PAIR_MASK) {
2532 CHECK_ARGUMENT((type & 0xff) == SLJIT_MOV);
2533 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_FIRST(reg)));
2534 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(REG_PAIR_SECOND(reg)));
2535 CHECK_ARGUMENT(REG_PAIR_FIRST(reg) != REG_PAIR_SECOND(reg));
2536 } else {
2537 CHECK_ARGUMENT((type & 0xff) >= SLJIT_MOV && (type & 0xff) <= SLJIT_MOV_P);
2538 CHECK_ARGUMENT(!(type & SLJIT_32) || ((type & 0xff) >= SLJIT_MOV_U8 && (type & 0xff) <= SLJIT_MOV_S16));
2539 CHECK_ARGUMENT(FUNCTION_CHECK_IS_REG(reg));
2540 }
2541
2542 FUNCTION_CHECK_SRC_MEM(mem, memw);
2543 #endif
2544 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2545 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2546 if ((type & 0xff) == SLJIT_MOV32)
2547 fprintf(compiler->verbose, " %s32",
2548 (type & SLJIT_MEM_STORE) ? "store" : "load");
2549 else
2550 fprintf(compiler->verbose, " %s%s%s",
2551 (type & SLJIT_MEM_STORE) ? "store" : "load",
2552 !(type & SLJIT_32) ? "" : "32", op1_types[(type & 0xff) - SLJIT_OP1_BASE]);
2553
2554 if (type & SLJIT_MEM_UNALIGNED)
2555 printf(".unal");
2556 else if (type & SLJIT_MEM_ALIGNED_16)
2557 printf(".al16");
2558 else if (type & SLJIT_MEM_ALIGNED_32)
2559 printf(".al32");
2560
2561 if (reg & REG_PAIR_MASK) {
2562 fprintf(compiler->verbose, " {");
2563 sljit_verbose_reg(compiler, REG_PAIR_FIRST(reg));
2564 fprintf(compiler->verbose, ", ");
2565 sljit_verbose_reg(compiler, REG_PAIR_SECOND(reg));
2566 fprintf(compiler->verbose, "}, ");
2567 } else {
2568 fprintf(compiler->verbose, " ");
2569 sljit_verbose_reg(compiler, reg);
2570 fprintf(compiler->verbose, ", ");
2571 }
2572 sljit_verbose_param(compiler, mem, memw);
2573 fprintf(compiler->verbose, "\n");
2574 }
2575 #endif
2576 CHECK_RETURN_OK;
2577 }
2578
check_sljit_emit_mem_update(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 reg,sljit_s32 mem,sljit_sw memw)2579 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
2580 sljit_s32 reg,
2581 sljit_s32 mem, sljit_sw memw)
2582 {
2583 if (SLJIT_UNLIKELY(compiler->skip_checks)) {
2584 compiler->skip_checks = 0;
2585 CHECK_RETURN_OK;
2586 }
2587
2588 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2589 CHECK_ARGUMENT((type & 0xff) >= SLJIT_MOV && (type & 0xff) <= SLJIT_MOV_P);
2590 CHECK_ARGUMENT((type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | SLJIT_MEM_SUPP | SLJIT_MEM_POST)) == 0);
2591 CHECK_ARGUMENT((mem & REG_MASK) != 0 && (mem & REG_MASK) != reg);
2592
2593 FUNCTION_CHECK_SRC_MEM(mem, memw);
2594 #endif
2595 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2596 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2597 if (type & SLJIT_MEM_SUPP)
2598 CHECK_RETURN_OK;
2599 if (sljit_emit_mem_update(compiler, type | SLJIT_MEM_SUPP, reg, mem, memw) == SLJIT_ERR_UNSUPPORTED) {
2600 fprintf(compiler->verbose, " # mem: unsupported form, no instructions are emitted\n");
2601 CHECK_RETURN_OK;
2602 }
2603
2604 if ((type & 0xff) == SLJIT_MOV32)
2605 fprintf(compiler->verbose, " %s32.%s ",
2606 (type & SLJIT_MEM_STORE) ? "store" : "load",
2607 (type & SLJIT_MEM_POST) ? "post" : "pre");
2608 else
2609 fprintf(compiler->verbose, " %s%s%s.%s ",
2610 (type & SLJIT_MEM_STORE) ? "store" : "load",
2611 !(type & SLJIT_32) ? "" : "32",
2612 op1_types[(type & 0xff) - SLJIT_OP1_BASE],
2613 (type & SLJIT_MEM_POST) ? "post" : "pre");
2614
2615 sljit_verbose_reg(compiler, reg);
2616 fprintf(compiler->verbose, ", ");
2617 sljit_verbose_param(compiler, mem, memw);
2618 fprintf(compiler->verbose, "\n");
2619 }
2620 #endif
2621 CHECK_RETURN_OK;
2622 }
2623
check_sljit_emit_fmem(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 mem,sljit_sw memw)2624 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fmem(struct sljit_compiler *compiler, sljit_s32 type,
2625 sljit_s32 freg,
2626 sljit_s32 mem, sljit_sw memw)
2627 {
2628 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2629 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2630 CHECK_ARGUMENT((type & 0xff) == SLJIT_MOV_F64);
2631
2632 if (type & SLJIT_MEM_UNALIGNED) {
2633 CHECK_ARGUMENT(!(type & (SLJIT_MEM_ALIGNED_16 | SLJIT_MEM_ALIGNED_32)));
2634 } else if (type & SLJIT_MEM_ALIGNED_16) {
2635 CHECK_ARGUMENT(!(type & SLJIT_MEM_ALIGNED_32));
2636 } else {
2637 CHECK_ARGUMENT(type & SLJIT_MEM_ALIGNED_32);
2638 CHECK_ARGUMENT(!(type & SLJIT_32));
2639 }
2640
2641 CHECK_ARGUMENT(!(type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | SLJIT_MEM_UNALIGNED | SLJIT_MEM_ALIGNED_16 | SLJIT_MEM_ALIGNED_32)));
2642 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, type & SLJIT_32));
2643 FUNCTION_CHECK_SRC_MEM(mem, memw);
2644 #endif
2645 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2646 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2647 fprintf(compiler->verbose, " %s.%s",
2648 (type & SLJIT_MEM_STORE) ? "store" : "load",
2649 !(type & SLJIT_32) ? "f64" : "f32");
2650
2651 if (type & SLJIT_MEM_UNALIGNED)
2652 printf(".unal");
2653 else if (type & SLJIT_MEM_ALIGNED_16)
2654 printf(".al16");
2655 else if (type & SLJIT_MEM_ALIGNED_32)
2656 printf(".al32");
2657
2658 fprintf(compiler->verbose, " ");
2659 sljit_verbose_freg(compiler, freg);
2660 fprintf(compiler->verbose, ", ");
2661 sljit_verbose_param(compiler, mem, memw);
2662 fprintf(compiler->verbose, "\n");
2663 }
2664 #endif
2665 CHECK_RETURN_OK;
2666 }
2667
check_sljit_emit_fmem_update(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 mem,sljit_sw memw)2668 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_fmem_update(struct sljit_compiler *compiler, sljit_s32 type,
2669 sljit_s32 freg,
2670 sljit_s32 mem, sljit_sw memw)
2671 {
2672 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2673 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_FPU));
2674 CHECK_ARGUMENT((type & 0xff) == SLJIT_MOV_F64);
2675 CHECK_ARGUMENT((type & ~(0xff | SLJIT_32 | SLJIT_MEM_STORE | SLJIT_MEM_SUPP | SLJIT_MEM_POST)) == 0);
2676 FUNCTION_CHECK_SRC_MEM(mem, memw);
2677 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, type & SLJIT_32));
2678 #endif
2679 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2680 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2681 if (type & SLJIT_MEM_SUPP)
2682 CHECK_RETURN_OK;
2683 if (sljit_emit_fmem_update(compiler, type | SLJIT_MEM_SUPP, freg, mem, memw) == SLJIT_ERR_UNSUPPORTED) {
2684 fprintf(compiler->verbose, " # fmem: unsupported form, no instructions are emitted\n");
2685 CHECK_RETURN_OK;
2686 }
2687
2688 fprintf(compiler->verbose, " %s.%s.%s ",
2689 (type & SLJIT_MEM_STORE) ? "store" : "load",
2690 !(type & SLJIT_32) ? "f64" : "f32",
2691 (type & SLJIT_MEM_POST) ? "post" : "pre");
2692
2693 sljit_verbose_freg(compiler, freg);
2694 fprintf(compiler->verbose, ", ");
2695 sljit_verbose_param(compiler, mem, memw);
2696 fprintf(compiler->verbose, "\n");
2697 }
2698 #endif
2699 CHECK_RETURN_OK;
2700 }
2701
check_sljit_emit_simd_mov(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 srcdst,sljit_sw srcdstw)2702 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
2703 sljit_s32 freg,
2704 sljit_s32 srcdst, sljit_sw srcdstw)
2705 {
2706 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2707 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
2708 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK2(SLJIT_SIMD_STORE)) == 0);
2709 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
2710 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) <= SLJIT_SIMD_GET_REG_SIZE(type));
2711 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM2_SIZE(type) <= (srcdst & SLJIT_MEM) ? SLJIT_SIMD_GET_REG_SIZE(type) : 0);
2712 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 0));
2713 FUNCTION_FCHECK(srcdst, srcdstw, 0);
2714 #endif
2715 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2716 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2717 if (type & SLJIT_SIMD_TEST)
2718 CHECK_RETURN_OK;
2719 if (sljit_emit_simd_mov(compiler, type | SLJIT_SIMD_TEST, freg, srcdst, srcdstw) == SLJIT_ERR_UNSUPPORTED) {
2720 fprintf(compiler->verbose, " # simd_mem: unsupported form, no instructions are emitted\n");
2721 CHECK_RETURN_OK;
2722 }
2723
2724 fprintf(compiler->verbose, " simd_%s.%d.%s%d",
2725 (type & SLJIT_SIMD_STORE) ? "store" : "load",
2726 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
2727 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
2728 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
2729
2730 if ((type & 0x3f000000) == SLJIT_SIMD_MEM_UNALIGNED)
2731 fprintf(compiler->verbose, ".unal ");
2732 else
2733 fprintf(compiler->verbose, ".al%d ", (8 << SLJIT_SIMD_GET_ELEM2_SIZE(type)));
2734
2735 sljit_verbose_freg(compiler, freg);
2736 fprintf(compiler->verbose, ", ");
2737 sljit_verbose_fparam(compiler, srcdst, srcdstw);
2738 fprintf(compiler->verbose, "\n");
2739 }
2740 #endif
2741 CHECK_RETURN_OK;
2742 }
2743
check_sljit_emit_simd_replicate(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 src,sljit_sw srcw)2744 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
2745 sljit_s32 freg,
2746 sljit_s32 src, sljit_sw srcw)
2747 {
2748 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2749 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
2750 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(0)) == 0);
2751 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
2752 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
2753 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 0));
2754
2755 if (type & SLJIT_SIMD_FLOAT) {
2756 if (src == SLJIT_IMM) {
2757 CHECK_ARGUMENT(srcw == 0);
2758 } else {
2759 FUNCTION_FCHECK(src, srcw, SLJIT_SIMD_GET_ELEM_SIZE(type) == 2);
2760 }
2761 } else if (src != SLJIT_IMM) {
2762 FUNCTION_CHECK_DST(src, srcw);
2763 }
2764 #endif
2765 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2766 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2767 if (type & SLJIT_SIMD_TEST)
2768 CHECK_RETURN_OK;
2769 if (sljit_emit_simd_replicate(compiler, type | SLJIT_SIMD_TEST, freg, src, srcw) == SLJIT_ERR_UNSUPPORTED) {
2770 fprintf(compiler->verbose, " # simd_dup: unsupported form, no instructions are emitted\n");
2771 CHECK_RETURN_OK;
2772 }
2773
2774 fprintf(compiler->verbose, " simd_replicate.%d.%s%d ",
2775 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
2776 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
2777 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
2778
2779 sljit_verbose_freg(compiler, freg);
2780 fprintf(compiler->verbose, ", ");
2781 if (type & SLJIT_SIMD_FLOAT)
2782 sljit_verbose_fparam(compiler, src, srcw);
2783 else
2784 sljit_verbose_param(compiler, src, srcw);
2785 fprintf(compiler->verbose, "\n");
2786 }
2787 #endif
2788 CHECK_RETURN_OK;
2789 }
2790
check_sljit_emit_simd_lane_mov(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 lane_index,sljit_s32 srcdst,sljit_sw srcdstw)2791 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
2792 sljit_s32 freg, sljit_s32 lane_index,
2793 sljit_s32 srcdst, sljit_sw srcdstw)
2794 {
2795 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2796 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
2797 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_ZERO | SLJIT_SIMD_LANE_SIGNED | SLJIT_32)) == 0);
2798 CHECK_ARGUMENT((type & (SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_ZERO)) != (SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_ZERO));
2799 CHECK_ARGUMENT((type & (SLJIT_SIMD_STORE | SLJIT_SIMD_LANE_SIGNED)) != SLJIT_SIMD_LANE_SIGNED);
2800 CHECK_ARGUMENT(!(type & SLJIT_SIMD_FLOAT) || !(type & (SLJIT_SIMD_LANE_SIGNED | SLJIT_32)));
2801 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
2802 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
2803 CHECK_ARGUMENT(!(type & SLJIT_32) || SLJIT_SIMD_GET_ELEM_SIZE(type) <= 2);
2804 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 0));
2805 CHECK_ARGUMENT(lane_index >= 0 && lane_index < (1 << (SLJIT_SIMD_GET_REG_SIZE(type) - SLJIT_SIMD_GET_ELEM_SIZE(type))));
2806
2807 if (type & SLJIT_SIMD_FLOAT) {
2808 FUNCTION_FCHECK(srcdst, srcdstw, SLJIT_SIMD_GET_ELEM_SIZE(type) == 2);
2809 } else if ((type & SLJIT_SIMD_STORE) || srcdst != SLJIT_IMM) {
2810 FUNCTION_CHECK_DST(srcdst, srcdstw);
2811 }
2812 #endif
2813 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2814 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2815 if (type & SLJIT_SIMD_TEST)
2816 CHECK_RETURN_OK;
2817 if (sljit_emit_simd_lane_mov(compiler, type | SLJIT_SIMD_TEST, freg, lane_index, srcdst, srcdstw) == SLJIT_ERR_UNSUPPORTED) {
2818 fprintf(compiler->verbose, " # simd_move_lane: unsupported form, no instructions are emitted\n");
2819 CHECK_RETURN_OK;
2820 }
2821
2822 fprintf(compiler->verbose, " simd_%s_lane%s%s%s.%d.%s%d ",
2823 (type & SLJIT_SIMD_STORE) ? "store" : "load",
2824 (type & SLJIT_32) ? "32" : "",
2825 (type & SLJIT_SIMD_LANE_ZERO) ? "_z" : "",
2826 (type & SLJIT_SIMD_LANE_SIGNED) ? "_s" : "",
2827 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
2828 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
2829 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
2830
2831 sljit_verbose_freg(compiler, freg);
2832 fprintf(compiler->verbose, "[%d], ", lane_index);
2833 if (type & SLJIT_SIMD_FLOAT)
2834 sljit_verbose_fparam(compiler, srcdst, srcdstw);
2835 else
2836 sljit_verbose_param(compiler, srcdst, srcdstw);
2837 fprintf(compiler->verbose, "\n");
2838 }
2839 #endif
2840 CHECK_RETURN_OK;
2841 }
2842
check_sljit_emit_simd_lane_replicate(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 src,sljit_s32 src_lane_index)2843 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
2844 sljit_s32 freg,
2845 sljit_s32 src, sljit_s32 src_lane_index)
2846 {
2847 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2848 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
2849 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(0)) == 0);
2850 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
2851 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
2852 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 0));
2853 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(src, 0));
2854 CHECK_ARGUMENT(src_lane_index >= 0 && src_lane_index < (1 << (SLJIT_SIMD_GET_REG_SIZE(type) - SLJIT_SIMD_GET_ELEM_SIZE(type))));
2855 #endif
2856 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2857 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2858 if (type & SLJIT_SIMD_TEST)
2859 CHECK_RETURN_OK;
2860 if (sljit_emit_simd_lane_replicate(compiler, type | SLJIT_SIMD_TEST, freg, src, src_lane_index) == SLJIT_ERR_UNSUPPORTED) {
2861 fprintf(compiler->verbose, " # simd_lane_replicate: unsupported form, no instructions are emitted\n");
2862 CHECK_RETURN_OK;
2863 }
2864
2865 fprintf(compiler->verbose, " simd_lane_replicate.%d.%s%d ",
2866 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
2867 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
2868 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
2869
2870 sljit_verbose_freg(compiler, freg);
2871 fprintf(compiler->verbose, ", ");
2872 sljit_verbose_freg(compiler, src);
2873 fprintf(compiler->verbose, "[%d]\n", src_lane_index);
2874 }
2875 #endif
2876 CHECK_RETURN_OK;
2877 }
2878
check_sljit_emit_simd_extend(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 src,sljit_sw srcw)2879 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
2880 sljit_s32 freg,
2881 sljit_s32 src, sljit_sw srcw)
2882 {
2883 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2884 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
2885 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK2(SLJIT_SIMD_EXTEND_SIGNED)) == 0);
2886 CHECK_ARGUMENT((type & (SLJIT_SIMD_EXTEND_SIGNED | SLJIT_SIMD_FLOAT)) != (SLJIT_SIMD_EXTEND_SIGNED | SLJIT_SIMD_FLOAT));
2887 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
2888 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM2_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
2889 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_ELEM2_SIZE(type));
2890 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 0));
2891 FUNCTION_FCHECK(src, srcw, SLJIT_SIMD_GET_ELEM_SIZE(type) == 2);
2892 #endif
2893 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2894 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2895 if (type & SLJIT_SIMD_TEST)
2896 CHECK_RETURN_OK;
2897 if (sljit_emit_simd_extend(compiler, type | SLJIT_SIMD_TEST, freg, src, srcw) == SLJIT_ERR_UNSUPPORTED) {
2898 fprintf(compiler->verbose, " # simd_extend: unsupported form, no instructions are emitted\n");
2899 CHECK_RETURN_OK;
2900 }
2901
2902 fprintf(compiler->verbose, " simd_load_extend%s.%d.%s%d.%s%d ",
2903 (type & SLJIT_SIMD_EXTEND_SIGNED) ? "_s" : "",
2904 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
2905 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
2906 (8 << SLJIT_SIMD_GET_ELEM2_SIZE(type)),
2907 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
2908 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
2909
2910 sljit_verbose_freg(compiler, freg);
2911 fprintf(compiler->verbose, ", ");
2912 sljit_verbose_fparam(compiler, src, srcw);
2913 fprintf(compiler->verbose, "\n");
2914 }
2915 #endif
2916 CHECK_RETURN_OK;
2917 }
2918
check_sljit_emit_simd_sign(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 dst,sljit_sw dstw)2919 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
2920 sljit_s32 freg,
2921 sljit_s32 dst, sljit_sw dstw)
2922 {
2923 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2924 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
2925 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(SLJIT_32)) == SLJIT_SIMD_STORE);
2926 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
2927 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) < SLJIT_SIMD_GET_REG_SIZE(type));
2928 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(freg, 0));
2929 FUNCTION_CHECK_DST(dst, dstw);
2930 #endif
2931 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2932 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2933 if (type & SLJIT_SIMD_TEST)
2934 CHECK_RETURN_OK;
2935 if (sljit_emit_simd_sign(compiler, type | SLJIT_SIMD_TEST, freg, dst, dstw) == SLJIT_ERR_UNSUPPORTED) {
2936 fprintf(compiler->verbose, " # simd_sign: unsupported form, no instructions are emitted\n");
2937 CHECK_RETURN_OK;
2938 }
2939
2940 fprintf(compiler->verbose, " simd_store_sign%s.%d.%s%d ",
2941 (type & SLJIT_32) ? "32" : "",
2942 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
2943 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
2944 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
2945
2946 sljit_verbose_freg(compiler, freg);
2947 fprintf(compiler->verbose, ", ");
2948 sljit_verbose_param(compiler, dst, dstw);
2949 fprintf(compiler->verbose, "\n");
2950 }
2951 #endif
2952 CHECK_RETURN_OK;
2953 }
2954
check_sljit_emit_simd_op2(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 dst_freg,sljit_s32 src1_freg,sljit_s32 src2_freg)2955 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
2956 sljit_s32 dst_freg, sljit_s32 src1_freg, sljit_s32 src2_freg)
2957 {
2958 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2959 CHECK_ARGUMENT(sljit_has_cpu_feature(SLJIT_HAS_SIMD));
2960 CHECK_ARGUMENT((type & SLJIT_SIMD_TYPE_MASK(0)) >= SLJIT_SIMD_OP2_AND && (type & SLJIT_SIMD_TYPE_MASK(0)) <= SLJIT_SIMD_OP2_XOR);
2961 CHECK_ARGUMENT(SLJIT_SIMD_CHECK_REG(type));
2962 CHECK_ARGUMENT(SLJIT_SIMD_GET_ELEM_SIZE(type) <= SLJIT_SIMD_GET_REG_SIZE(type));
2963 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(dst_freg, 0));
2964 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(src1_freg, 0));
2965 CHECK_ARGUMENT(FUNCTION_CHECK_IS_FREG(src2_freg, 0));
2966 #endif
2967 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
2968 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
2969 if (type & SLJIT_SIMD_TEST)
2970 CHECK_RETURN_OK;
2971 if (sljit_emit_simd_op2(compiler, type | SLJIT_SIMD_TEST, dst_freg, src1_freg, src2_freg) == SLJIT_ERR_UNSUPPORTED) {
2972 fprintf(compiler->verbose, " # simd_op2: unsupported form, no instructions are emitted\n");
2973 CHECK_RETURN_OK;
2974 }
2975
2976 fprintf(compiler->verbose, " simd_%s.%d.%s%d ",
2977 simd_op2_names[SLJIT_SIMD_GET_OPCODE(type) - 1],
2978 (8 << SLJIT_SIMD_GET_REG_SIZE(type)),
2979 (type & SLJIT_SIMD_FLOAT) ? "f" : "",
2980 (8 << SLJIT_SIMD_GET_ELEM_SIZE(type)));
2981
2982 sljit_verbose_freg(compiler, dst_freg);
2983 fprintf(compiler->verbose, ", ");
2984 sljit_verbose_freg(compiler, src1_freg);
2985 fprintf(compiler->verbose, ", ");
2986 sljit_verbose_freg(compiler, src2_freg);
2987 fprintf(compiler->verbose, "\n");
2988 }
2989 #endif
2990 CHECK_RETURN_OK;
2991 }
2992
check_sljit_get_local_base(struct sljit_compiler * compiler,sljit_s32 dst,sljit_sw dstw,sljit_sw offset)2993 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
2994 {
2995 /* Any offset is allowed. */
2996 SLJIT_UNUSED_ARG(offset);
2997
2998 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
2999 FUNCTION_CHECK_DST(dst, dstw);
3000 #endif
3001 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3002 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3003 fprintf(compiler->verbose, " local_base ");
3004 sljit_verbose_param(compiler, dst, dstw);
3005 fprintf(compiler->verbose, ", #%" SLJIT_PRINT_D "d\n", offset);
3006 }
3007 #endif
3008 CHECK_RETURN_OK;
3009 }
3010
check_sljit_emit_const(struct sljit_compiler * compiler,sljit_s32 dst,sljit_sw dstw,sljit_sw init_value)3011 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_const(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw init_value)
3012 {
3013 SLJIT_UNUSED_ARG(init_value);
3014
3015 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3016 FUNCTION_CHECK_DST(dst, dstw);
3017 #endif
3018 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3019 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3020 fprintf(compiler->verbose, " const ");
3021 sljit_verbose_param(compiler, dst, dstw);
3022 fprintf(compiler->verbose, ", #%" SLJIT_PRINT_D "d\n", init_value);
3023 }
3024 #endif
3025 CHECK_RETURN_OK;
3026 }
3027
check_sljit_emit_mov_addr(struct sljit_compiler * compiler,sljit_s32 dst,sljit_sw dstw)3028 static SLJIT_INLINE CHECK_RETURN_TYPE check_sljit_emit_mov_addr(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw)
3029 {
3030 #if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)
3031 FUNCTION_CHECK_DST(dst, dstw);
3032 #endif
3033 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE)
3034 if (SLJIT_UNLIKELY(!!compiler->verbose)) {
3035 fprintf(compiler->verbose, " mov_addr ");
3036 sljit_verbose_param(compiler, dst, dstw);
3037 fprintf(compiler->verbose, "\n");
3038 }
3039 #endif
3040 CHECK_RETURN_OK;
3041 }
3042
3043 #else /* !SLJIT_ARGUMENT_CHECKS && !SLJIT_VERBOSE */
3044
3045 #define SLJIT_SKIP_CHECKS(compiler)
3046
3047 #endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_VERBOSE */
3048
3049 #define SELECT_FOP1_OPERATION_WITH_CHECKS(compiler, op, dst, dstw, src, srcw) \
3050 SLJIT_COMPILE_ASSERT(!(SLJIT_CONV_SW_FROM_F64 & 0x1) && !(SLJIT_CONV_F64_FROM_SW & 0x1) && !(SLJIT_CONV_F64_FROM_UW & 0x1), \
3051 invalid_float_opcodes); \
3052 if (GET_OPCODE(op) >= SLJIT_CONV_SW_FROM_F64 && GET_OPCODE(op) <= SLJIT_CMP_F64) { \
3053 if (GET_OPCODE(op) == SLJIT_CMP_F64) { \
3054 CHECK(check_sljit_emit_fop1_cmp(compiler, op, dst, dstw, src, srcw)); \
3055 ADJUST_LOCAL_OFFSET(dst, dstw); \
3056 ADJUST_LOCAL_OFFSET(src, srcw); \
3057 return sljit_emit_fop1_cmp(compiler, op, dst, dstw, src, srcw); \
3058 } \
3059 if ((GET_OPCODE(op) | 0x1) == SLJIT_CONV_S32_FROM_F64) { \
3060 CHECK(check_sljit_emit_fop1_conv_sw_from_f64(compiler, op, dst, dstw, src, srcw)); \
3061 ADJUST_LOCAL_OFFSET(dst, dstw); \
3062 ADJUST_LOCAL_OFFSET(src, srcw); \
3063 return sljit_emit_fop1_conv_sw_from_f64(compiler, op, dst, dstw, src, srcw); \
3064 } \
3065 if ((GET_OPCODE(op) | 0x1) == SLJIT_CONV_F64_FROM_S32) { \
3066 CHECK(check_sljit_emit_fop1_conv_f64_from_w(compiler, op, dst, dstw, src, srcw)); \
3067 ADJUST_LOCAL_OFFSET(dst, dstw); \
3068 ADJUST_LOCAL_OFFSET(src, srcw); \
3069 return sljit_emit_fop1_conv_f64_from_sw(compiler, op, dst, dstw, src, srcw); \
3070 } \
3071 CHECK(check_sljit_emit_fop1_conv_f64_from_w(compiler, op, dst, dstw, src, srcw)); \
3072 ADJUST_LOCAL_OFFSET(dst, dstw); \
3073 ADJUST_LOCAL_OFFSET(src, srcw); \
3074 return sljit_emit_fop1_conv_f64_from_uw(compiler, op, dst, dstw, src, srcw); \
3075 } \
3076 CHECK(check_sljit_emit_fop1(compiler, op, dst, dstw, src, srcw)); \
3077 ADJUST_LOCAL_OFFSET(dst, dstw); \
3078 ADJUST_LOCAL_OFFSET(src, srcw);
3079
3080 #if (!(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) || (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6))
3081
sljit_emit_mem_unaligned(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 reg,sljit_s32 mem,sljit_sw memw)3082 static sljit_s32 sljit_emit_mem_unaligned(struct sljit_compiler *compiler, sljit_s32 type,
3083 sljit_s32 reg,
3084 sljit_s32 mem, sljit_sw memw)
3085 {
3086 SLJIT_SKIP_CHECKS(compiler);
3087
3088 if (type & SLJIT_MEM_STORE)
3089 return sljit_emit_op1(compiler, type & (0xff | SLJIT_32), mem, memw, reg, 0);
3090 return sljit_emit_op1(compiler, type & (0xff | SLJIT_32), reg, 0, mem, memw);
3091 }
3092
3093 #endif /* (!SLJIT_CONFIG_MIPS || SLJIT_MIPS_REV >= 6) */
3094
3095 #if (!(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) || (defined SLJIT_MIPS_REV && SLJIT_MIPS_REV >= 6)) \
3096 && !(defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32)
3097
sljit_emit_fmem_unaligned(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 mem,sljit_sw memw)3098 static sljit_s32 sljit_emit_fmem_unaligned(struct sljit_compiler *compiler, sljit_s32 type,
3099 sljit_s32 freg,
3100 sljit_s32 mem, sljit_sw memw)
3101 {
3102 SLJIT_SKIP_CHECKS(compiler);
3103
3104 if (type & SLJIT_MEM_STORE)
3105 return sljit_emit_fop1(compiler, type & (0xff | SLJIT_32), mem, memw, freg, 0);
3106 return sljit_emit_fop1(compiler, type & (0xff | SLJIT_32), freg, 0, mem, memw);
3107 }
3108
3109 #endif /* (!SLJIT_CONFIG_MIPS || SLJIT_MIPS_REV >= 6) && !SLJIT_CONFIG_ARM */
3110
3111 /* CPU description section */
3112
3113 #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE)
3114 #define SLJIT_CPUINFO_PART1 " 32bit ("
3115 #elif (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
3116 #define SLJIT_CPUINFO_PART1 " 64bit ("
3117 #else
3118 #error "Internal error: CPU type info missing"
3119 #endif
3120
3121 #if (defined SLJIT_LITTLE_ENDIAN && SLJIT_LITTLE_ENDIAN)
3122 #define SLJIT_CPUINFO_PART2 "little endian + "
3123 #elif (defined SLJIT_BIG_ENDIAN && SLJIT_BIG_ENDIAN)
3124 #define SLJIT_CPUINFO_PART2 "big endian + "
3125 #else
3126 #error "Internal error: CPU type info missing"
3127 #endif
3128
3129 #if (defined SLJIT_UNALIGNED && SLJIT_UNALIGNED)
3130 #define SLJIT_CPUINFO_PART3 "unaligned)"
3131 #else
3132 #define SLJIT_CPUINFO_PART3 "aligned)"
3133 #endif
3134
3135 #define SLJIT_CPUINFO SLJIT_CPUINFO_PART1 SLJIT_CPUINFO_PART2 SLJIT_CPUINFO_PART3
3136
3137 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
3138 # include "sljitNativeX86_common.c"
3139 #elif (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)
3140 # include "sljitNativeARM_32.c"
3141 #elif (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)
3142 # include "sljitNativeARM_32.c"
3143 #elif (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)
3144 # include "sljitNativeARM_T2_32.c"
3145 #elif (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
3146 # include "sljitNativeARM_64.c"
3147 #elif (defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
3148 # include "sljitNativePPC_common.c"
3149 #elif (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
3150 # include "sljitNativeMIPS_common.c"
3151 #elif (defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV)
3152 # include "sljitNativeRISCV_common.c"
3153 #elif (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X)
3154 # include "sljitNativeS390X.c"
3155 #elif (defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3156 # include "sljitNativeLOONGARCH_64.c"
3157 #endif
3158
3159 #include "sljitSerialize.c"
3160
emit_mov_before_return(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src,sljit_sw srcw)3161 static SLJIT_INLINE sljit_s32 emit_mov_before_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
3162 {
3163 #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)
3164 /* At the moment the pointer size is always equal to sljit_sw. May be changed in the future. */
3165 if (src == SLJIT_RETURN_REG && (op == SLJIT_MOV || op == SLJIT_MOV_P))
3166 return SLJIT_SUCCESS;
3167 #else
3168 if (src == SLJIT_RETURN_REG && (op == SLJIT_MOV || op == SLJIT_MOV_U32 || op == SLJIT_MOV_S32 || op == SLJIT_MOV_P))
3169 return SLJIT_SUCCESS;
3170 #endif
3171
3172 SLJIT_SKIP_CHECKS(compiler);
3173 return sljit_emit_op1(compiler, op, SLJIT_RETURN_REG, 0, src, srcw);
3174 }
3175
3176 #if !(defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32) \
3177 && !((defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) && defined __SOFTFP__)
3178
emit_fmov_before_return(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src,sljit_sw srcw)3179 static SLJIT_INLINE sljit_s32 emit_fmov_before_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
3180 {
3181 if (src == SLJIT_FR0)
3182 return SLJIT_SUCCESS;
3183
3184 SLJIT_SKIP_CHECKS(compiler);
3185 return sljit_emit_fop1(compiler, op, SLJIT_RETURN_FREG, 0, src, srcw);
3186 }
3187
3188 #endif /* !SLJIT_CONFIG_X86_32 && !(SLJIT_CONFIG_ARM_32 && __SOFTFP__) */
3189
sljit_emit_return(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src,sljit_sw srcw)3190 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_return(struct sljit_compiler *compiler, sljit_s32 op, sljit_s32 src, sljit_sw srcw)
3191 {
3192 CHECK_ERROR();
3193 CHECK(check_sljit_emit_return(compiler, op, src, srcw));
3194
3195 if (GET_OPCODE(op) < SLJIT_MOV_F64) {
3196 FAIL_IF(emit_mov_before_return(compiler, op, src, srcw));
3197 } else {
3198 FAIL_IF(emit_fmov_before_return(compiler, op, src, srcw));
3199 }
3200
3201 SLJIT_SKIP_CHECKS(compiler);
3202 return sljit_emit_return_void(compiler);
3203 }
3204
3205 #if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
3206 && !(defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) \
3207 && !(defined(SLJIT_CONFIG_LOONGARCH_64) && SLJIT_CONFIG_LOONGARCH_64)
3208
sljit_emit_fop2r(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst_freg,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)3209 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fop2r(struct sljit_compiler *compiler, sljit_s32 op,
3210 sljit_s32 dst_freg,
3211 sljit_s32 src1, sljit_sw src1w,
3212 sljit_s32 src2, sljit_sw src2w)
3213 {
3214 CHECK_ERROR();
3215 CHECK(check_sljit_emit_fop2r(compiler, op, dst_freg, src1, src1w, src2, src2w));
3216 ADJUST_LOCAL_OFFSET(src1, src1w);
3217 ADJUST_LOCAL_OFFSET(src2, src2w);
3218
3219 SLJIT_SKIP_CHECKS(compiler);
3220 return sljit_emit_fop2(compiler, op, dst_freg, 0, src1, src1w, src2, src2w);
3221 }
3222
3223 #endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_S390X && !SLJIT_CONFIG_LOONGARCH_64 */
3224
3225 #if !(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS) \
3226 && !(defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV) \
3227 && !(defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3228
sljit_emit_cmp(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)3229 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_cmp(struct sljit_compiler *compiler, sljit_s32 type,
3230 sljit_s32 src1, sljit_sw src1w,
3231 sljit_s32 src2, sljit_sw src2w)
3232 {
3233 /* Default compare for most architectures. */
3234 sljit_s32 flags, tmp_src, condition;
3235 sljit_sw tmp_srcw;
3236
3237 CHECK_ERROR_PTR();
3238 CHECK_PTR(check_sljit_emit_cmp(compiler, type, src1, src1w, src2, src2w));
3239
3240 condition = type & 0xff;
3241 #if (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
3242 if ((condition == SLJIT_EQUAL || condition == SLJIT_NOT_EQUAL)) {
3243 if (src1 == SLJIT_IMM && !src1w) {
3244 src1 = src2;
3245 src1w = src2w;
3246 src2 = SLJIT_IMM;
3247 src2w = 0;
3248 }
3249 if (src2 == SLJIT_IMM && !src2w)
3250 return emit_cmp_to0(compiler, type, src1, src1w);
3251 }
3252 #endif
3253
3254 if (SLJIT_UNLIKELY(src1 == SLJIT_IMM && src2 != SLJIT_IMM)) {
3255 /* Immediate is preferred as second argument by most architectures. */
3256 switch (condition) {
3257 case SLJIT_LESS:
3258 condition = SLJIT_GREATER;
3259 break;
3260 case SLJIT_GREATER_EQUAL:
3261 condition = SLJIT_LESS_EQUAL;
3262 break;
3263 case SLJIT_GREATER:
3264 condition = SLJIT_LESS;
3265 break;
3266 case SLJIT_LESS_EQUAL:
3267 condition = SLJIT_GREATER_EQUAL;
3268 break;
3269 case SLJIT_SIG_LESS:
3270 condition = SLJIT_SIG_GREATER;
3271 break;
3272 case SLJIT_SIG_GREATER_EQUAL:
3273 condition = SLJIT_SIG_LESS_EQUAL;
3274 break;
3275 case SLJIT_SIG_GREATER:
3276 condition = SLJIT_SIG_LESS;
3277 break;
3278 case SLJIT_SIG_LESS_EQUAL:
3279 condition = SLJIT_SIG_GREATER_EQUAL;
3280 break;
3281 }
3282
3283 type = condition | (type & (SLJIT_32 | SLJIT_REWRITABLE_JUMP));
3284 tmp_src = src1;
3285 src1 = src2;
3286 src2 = tmp_src;
3287 tmp_srcw = src1w;
3288 src1w = src2w;
3289 src2w = tmp_srcw;
3290 }
3291
3292 if (condition <= SLJIT_NOT_ZERO)
3293 flags = SLJIT_SET_Z;
3294 else
3295 flags = (condition & 0xfe) << VARIABLE_FLAG_SHIFT;
3296
3297 SLJIT_SKIP_CHECKS(compiler);
3298 PTR_FAIL_IF(sljit_emit_op2u(compiler,
3299 SLJIT_SUB | flags | (type & SLJIT_32), src1, src1w, src2, src2w));
3300
3301 SLJIT_SKIP_CHECKS(compiler);
3302 return sljit_emit_jump(compiler, condition | (type & (SLJIT_REWRITABLE_JUMP | SLJIT_32)));
3303 }
3304
3305 #endif /* !SLJIT_CONFIG_MIPS */
3306
3307 #if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32)
3308
sljit_cmp_info(sljit_s32 type)3309 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_cmp_info(sljit_s32 type)
3310 {
3311 switch (type) {
3312 case SLJIT_UNORDERED_OR_EQUAL:
3313 case SLJIT_ORDERED_NOT_EQUAL:
3314 return 1;
3315 }
3316
3317 return 0;
3318 }
3319
3320 #endif /* SLJIT_CONFIG_ARM */
3321
sljit_emit_fcmp(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 src1,sljit_sw src1w,sljit_s32 src2,sljit_sw src2w)3322 SLJIT_API_FUNC_ATTRIBUTE struct sljit_jump* sljit_emit_fcmp(struct sljit_compiler *compiler, sljit_s32 type,
3323 sljit_s32 src1, sljit_sw src1w,
3324 sljit_s32 src2, sljit_sw src2w)
3325 {
3326 CHECK_ERROR_PTR();
3327 CHECK_PTR(check_sljit_emit_fcmp(compiler, type, src1, src1w, src2, src2w));
3328
3329 SLJIT_SKIP_CHECKS(compiler);
3330 sljit_emit_fop1(compiler, SLJIT_CMP_F64 | ((type & 0xfe) << VARIABLE_FLAG_SHIFT) | (type & SLJIT_32), src1, src1w, src2, src2w);
3331
3332 SLJIT_SKIP_CHECKS(compiler);
3333 return sljit_emit_jump(compiler, type);
3334 }
3335
3336 #if !(defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM) \
3337 && !(defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
3338
sljit_emit_mem_update(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 reg,sljit_s32 mem,sljit_sw memw)3339 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_mem_update(struct sljit_compiler *compiler, sljit_s32 type,
3340 sljit_s32 reg,
3341 sljit_s32 mem, sljit_sw memw)
3342 {
3343 CHECK_ERROR();
3344 CHECK(check_sljit_emit_mem_update(compiler, type, reg, mem, memw));
3345 SLJIT_UNUSED_ARG(type);
3346 SLJIT_UNUSED_ARG(reg);
3347 SLJIT_UNUSED_ARG(mem);
3348 SLJIT_UNUSED_ARG(memw);
3349
3350 return SLJIT_ERR_UNSUPPORTED;
3351 }
3352
3353 #endif /* !SLJIT_CONFIG_ARM && !SLJIT_CONFIG_PPC */
3354
3355 #if !(defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \
3356 && !(defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)
3357
sljit_emit_fmem(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 mem,sljit_sw memw)3358 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem(struct sljit_compiler *compiler, sljit_s32 type,
3359 sljit_s32 freg,
3360 sljit_s32 mem, sljit_sw memw)
3361 {
3362 CHECK_ERROR();
3363 CHECK(check_sljit_emit_fmem(compiler, type, freg, mem, memw));
3364
3365 return sljit_emit_fmem_unaligned(compiler, type, freg, mem, memw);
3366 }
3367
3368 #endif /* !SLJIT_CONFIG_ARM_32 && !SLJIT_CONFIG_MIPS */
3369
3370 #if !(defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64) \
3371 && !(defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)
3372
sljit_emit_fmem_update(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 mem,sljit_sw memw)3373 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_fmem_update(struct sljit_compiler *compiler, sljit_s32 type,
3374 sljit_s32 freg,
3375 sljit_s32 mem, sljit_sw memw)
3376 {
3377 CHECK_ERROR();
3378 CHECK(check_sljit_emit_fmem_update(compiler, type, freg, mem, memw));
3379 SLJIT_UNUSED_ARG(type);
3380 SLJIT_UNUSED_ARG(freg);
3381 SLJIT_UNUSED_ARG(mem);
3382 SLJIT_UNUSED_ARG(memw);
3383
3384 return SLJIT_ERR_UNSUPPORTED;
3385 }
3386
3387 #endif /* !SLJIT_CONFIG_ARM_64 && !SLJIT_CONFIG_PPC */
3388
3389 #if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
3390 && !(defined SLJIT_CONFIG_ARM && SLJIT_CONFIG_ARM) \
3391 && !(defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X) \
3392 && !(defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)
3393
sljit_emit_simd_mov(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 srcdst,sljit_sw srcdstw)3394 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_mov(struct sljit_compiler *compiler, sljit_s32 type,
3395 sljit_s32 freg,
3396 sljit_s32 srcdst, sljit_sw srcdstw)
3397 {
3398 CHECK_ERROR();
3399 CHECK(check_sljit_emit_simd_mov(compiler, type, freg, srcdst, srcdstw));
3400 SLJIT_UNUSED_ARG(compiler);
3401 SLJIT_UNUSED_ARG(type);
3402 SLJIT_UNUSED_ARG(freg);
3403 SLJIT_UNUSED_ARG(srcdst);
3404 SLJIT_UNUSED_ARG(srcdstw);
3405
3406 return SLJIT_ERR_UNSUPPORTED;
3407 }
3408
sljit_emit_simd_replicate(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 src,sljit_sw srcw)3409 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3410 sljit_s32 freg,
3411 sljit_s32 src, sljit_sw srcw)
3412 {
3413 CHECK_ERROR();
3414 CHECK(check_sljit_emit_simd_replicate(compiler, type, freg, src, srcw));
3415 SLJIT_UNUSED_ARG(compiler);
3416 SLJIT_UNUSED_ARG(type);
3417 SLJIT_UNUSED_ARG(freg);
3418 SLJIT_UNUSED_ARG(src);
3419 SLJIT_UNUSED_ARG(srcw);
3420
3421 return SLJIT_ERR_UNSUPPORTED;
3422 }
3423
sljit_emit_simd_lane_mov(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 lane_index,sljit_s32 srcdst,sljit_sw srcdstw)3424 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_mov(struct sljit_compiler *compiler, sljit_s32 type,
3425 sljit_s32 freg, sljit_s32 lane_index,
3426 sljit_s32 srcdst, sljit_sw srcdstw)
3427 {
3428 CHECK_ERROR();
3429 CHECK(check_sljit_emit_simd_lane_mov(compiler, type, freg, lane_index, srcdst, srcdstw));
3430 SLJIT_UNUSED_ARG(compiler);
3431 SLJIT_UNUSED_ARG(type);
3432 SLJIT_UNUSED_ARG(freg);
3433 SLJIT_UNUSED_ARG(lane_index);
3434 SLJIT_UNUSED_ARG(srcdst);
3435 SLJIT_UNUSED_ARG(srcdstw);
3436
3437 return SLJIT_ERR_UNSUPPORTED;
3438 }
3439
sljit_emit_simd_lane_replicate(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 src,sljit_s32 src_lane_index)3440 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_lane_replicate(struct sljit_compiler *compiler, sljit_s32 type,
3441 sljit_s32 freg,
3442 sljit_s32 src, sljit_s32 src_lane_index)
3443 {
3444 CHECK_ERROR();
3445 CHECK(check_sljit_emit_simd_lane_replicate(compiler, type, freg, src, src_lane_index));
3446 SLJIT_UNUSED_ARG(compiler);
3447 SLJIT_UNUSED_ARG(type);
3448 SLJIT_UNUSED_ARG(freg);
3449 SLJIT_UNUSED_ARG(src);
3450 SLJIT_UNUSED_ARG(src_lane_index);
3451
3452 return SLJIT_ERR_UNSUPPORTED;
3453 }
3454
sljit_emit_simd_extend(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 src,sljit_sw srcw)3455 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_extend(struct sljit_compiler *compiler, sljit_s32 type,
3456 sljit_s32 freg,
3457 sljit_s32 src, sljit_sw srcw)
3458 {
3459 CHECK_ERROR();
3460 CHECK(check_sljit_emit_simd_extend(compiler, type, freg, src, srcw));
3461 SLJIT_UNUSED_ARG(compiler);
3462 SLJIT_UNUSED_ARG(type);
3463 SLJIT_UNUSED_ARG(freg);
3464 SLJIT_UNUSED_ARG(src);
3465 SLJIT_UNUSED_ARG(srcw);
3466
3467 return SLJIT_ERR_UNSUPPORTED;
3468 }
3469
sljit_emit_simd_sign(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 freg,sljit_s32 dst,sljit_sw dstw)3470 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_sign(struct sljit_compiler *compiler, sljit_s32 type,
3471 sljit_s32 freg,
3472 sljit_s32 dst, sljit_sw dstw)
3473 {
3474 CHECK_ERROR();
3475 CHECK(check_sljit_emit_simd_sign(compiler, type, freg, dst, dstw));
3476 SLJIT_UNUSED_ARG(compiler);
3477 SLJIT_UNUSED_ARG(type);
3478 SLJIT_UNUSED_ARG(freg);
3479 SLJIT_UNUSED_ARG(dst);
3480 SLJIT_UNUSED_ARG(dstw);
3481
3482 return SLJIT_ERR_UNSUPPORTED;
3483 }
3484
sljit_emit_simd_op2(struct sljit_compiler * compiler,sljit_s32 type,sljit_s32 dst_freg,sljit_s32 src1_freg,sljit_s32 src2_freg)3485 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_simd_op2(struct sljit_compiler *compiler, sljit_s32 type,
3486 sljit_s32 dst_freg, sljit_s32 src1_freg, sljit_s32 src2_freg)
3487 {
3488 CHECK_ERROR();
3489 CHECK(check_sljit_emit_simd_op2(compiler, type, dst_freg, src1_freg, src2_freg));
3490 SLJIT_UNUSED_ARG(compiler);
3491 SLJIT_UNUSED_ARG(type);
3492 SLJIT_UNUSED_ARG(dst_freg);
3493 SLJIT_UNUSED_ARG(src1_freg);
3494 SLJIT_UNUSED_ARG(src2_freg);
3495
3496 return SLJIT_ERR_UNSUPPORTED;
3497 }
3498
3499 #endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_ARM */
3500
3501 #if !(defined(SLJIT_CONFIG_X86) && SLJIT_CONFIG_X86) \
3502 && !(defined(SLJIT_CONFIG_ARM) && SLJIT_CONFIG_ARM) \
3503 && !(defined(SLJIT_CONFIG_S390X) && SLJIT_CONFIG_S390X) \
3504 && !(defined(SLJIT_CONFIG_LOONGARCH) && SLJIT_CONFIG_LOONGARCH)
3505
sljit_emit_atomic_load(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 dst_reg,sljit_s32 mem_reg)3506 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_load(struct sljit_compiler *compiler,
3507 sljit_s32 op,
3508 sljit_s32 dst_reg,
3509 sljit_s32 mem_reg)
3510 {
3511 SLJIT_UNUSED_ARG(compiler);
3512 SLJIT_UNUSED_ARG(op);
3513 SLJIT_UNUSED_ARG(dst_reg);
3514 SLJIT_UNUSED_ARG(mem_reg);
3515
3516 CHECK_ERROR();
3517 CHECK(check_sljit_emit_atomic_load(compiler, op, dst_reg, mem_reg));
3518
3519 return SLJIT_ERR_UNSUPPORTED;
3520 }
3521
sljit_emit_atomic_store(struct sljit_compiler * compiler,sljit_s32 op,sljit_s32 src_reg,sljit_s32 mem_reg,sljit_s32 temp_reg)3522 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_emit_atomic_store(struct sljit_compiler *compiler,
3523 sljit_s32 op,
3524 sljit_s32 src_reg,
3525 sljit_s32 mem_reg,
3526 sljit_s32 temp_reg)
3527 {
3528 SLJIT_UNUSED_ARG(compiler);
3529 SLJIT_UNUSED_ARG(op);
3530 SLJIT_UNUSED_ARG(src_reg);
3531 SLJIT_UNUSED_ARG(mem_reg);
3532 SLJIT_UNUSED_ARG(temp_reg);
3533
3534 CHECK_ERROR();
3535 CHECK(check_sljit_emit_atomic_store(compiler, op, src_reg, mem_reg, temp_reg));
3536
3537 return SLJIT_ERR_UNSUPPORTED;
3538 }
3539
3540 #endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_ARM && !SLJIT_CONFIG_S390X && !SLJIT_CONFIG_LOONGARCH */
3541
3542 #if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
3543 && !(defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)
3544
sljit_get_local_base(struct sljit_compiler * compiler,sljit_s32 dst,sljit_sw dstw,sljit_sw offset)3545 SLJIT_API_FUNC_ATTRIBUTE sljit_s32 sljit_get_local_base(struct sljit_compiler *compiler, sljit_s32 dst, sljit_sw dstw, sljit_sw offset)
3546 {
3547 CHECK_ERROR();
3548 CHECK(check_sljit_get_local_base(compiler, dst, dstw, offset));
3549
3550 ADJUST_LOCAL_OFFSET(SLJIT_MEM1(SLJIT_SP), offset);
3551
3552 SLJIT_SKIP_CHECKS(compiler);
3553
3554 if (offset != 0)
3555 return sljit_emit_op2(compiler, SLJIT_ADD, dst, dstw, SLJIT_SP, 0, SLJIT_IMM, offset);
3556 return sljit_emit_op1(compiler, SLJIT_MOV, dst, dstw, SLJIT_SP, 0);
3557 }
3558
3559 #endif /* !SLJIT_CONFIG_X86 && !SLJIT_CONFIG_ARM_64 */
3560
3561 #endif /* !SLJIT_CONFIG_UNSUPPORTED */
3562
