1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 | Xinchen Hui <laruence@php.net> |
17 | Hao Sun <hao.sun@arm.com> |
18 +----------------------------------------------------------------------+
19 */
20
21 #ifndef ZEND_JIT_INTERNAL_H
22 #define ZEND_JIT_INTERNAL_H
23
24 #include "zend_bitset.h"
25
26 /* Register Set */
27 #define ZEND_REGSET_EMPTY 0
28
29 #define ZEND_REGSET_IS_EMPTY(regset) \
30 (regset == ZEND_REGSET_EMPTY)
31
32 #define ZEND_REGSET_IS_SINGLETON(regset) \
33 (regset && !(regset & (regset - 1)))
34
35 #if (!ZEND_REGSET_64BIT)
36 #define ZEND_REGSET(reg) \
37 (1u << (reg))
38 #else
39 #define ZEND_REGSET(reg) \
40 (1ull << (reg))
41 #endif
42
43 #if (!ZEND_REGSET_64BIT)
44 #define ZEND_REGSET_INTERVAL(reg1, reg2) \
45 (((1u << ((reg2) - (reg1) + 1)) - 1) << (reg1))
46 #else
47 #define ZEND_REGSET_INTERVAL(reg1, reg2) \
48 (((1ull << ((reg2) - (reg1) + 1)) - 1) << (reg1))
49 #endif
50
51 #define ZEND_REGSET_IN(regset, reg) \
52 (((regset) & ZEND_REGSET(reg)) != 0)
53
54 #define ZEND_REGSET_INCL(regset, reg) \
55 (regset) |= ZEND_REGSET(reg)
56
57 #define ZEND_REGSET_EXCL(regset, reg) \
58 (regset) &= ~ZEND_REGSET(reg)
59
60 #define ZEND_REGSET_UNION(set1, set2) \
61 ((set1) | (set2))
62
63 #define ZEND_REGSET_INTERSECTION(set1, set2) \
64 ((set1) & (set2))
65
66 #define ZEND_REGSET_DIFFERENCE(set1, set2) \
67 ((set1) & ~(set2))
68
69 #if !defined(_WIN32)
70 # if (!ZEND_REGSET_64BIT)
71 # define ZEND_REGSET_FIRST(set) ((zend_reg)__builtin_ctz(set))
72 # define ZEND_REGSET_LAST(set) ((zend_reg)(__builtin_clz(set)^31))
73 # else
74 # define ZEND_REGSET_FIRST(set) ((zend_reg)__builtin_ctzll(set))
75 # define ZEND_REGSET_LAST(set) ((zend_reg)(__builtin_clzll(set)^63))
76 # endif
77 #else
78 # include <intrin.h>
__zend_jit_ctz(uint32_t value)79 uint32_t __inline __zend_jit_ctz(uint32_t value) {
80 DWORD trailing_zero = 0;
81 if (_BitScanForward(&trailing_zero, value)) {
82 return trailing_zero;
83 }
84 return 32;
85 }
__zend_jit_clz(uint32_t value)86 uint32_t __inline __zend_jit_clz(uint32_t value) {
87 DWORD leading_zero = 0;
88 if (_BitScanReverse(&leading_zero, value)) {
89 return 31 - leading_zero;
90 }
91 return 32;
92 }
93 # define ZEND_REGSET_FIRST(set) ((zend_reg)__zend_jit_ctz(set))
94 # define ZEND_REGSET_LAST(set) ((zend_reg)(__zend_jit_clz(set)^31))
95 #endif
96
97 #define ZEND_REGSET_FOREACH(set, reg) \
98 do { \
99 zend_regset _tmp = (set); \
100 while (!ZEND_REGSET_IS_EMPTY(_tmp)) { \
101 zend_reg _reg = ZEND_REGSET_FIRST(_tmp); \
102 ZEND_REGSET_EXCL(_tmp, _reg); \
103 reg = _reg; \
104
105 #define ZEND_REGSET_FOREACH_END() \
106 } \
107 } while (0)
108
109 /* Register Names */
110 extern const char *zend_reg_name[];
111
112 /* Address Encoding */
113 typedef uintptr_t zend_jit_addr;
114
115 #define IS_CONST_ZVAL 0
116 #define IS_MEM_ZVAL 1
117 #define IS_REG 2
118
119 #define _ZEND_ADDR_MODE_MASK 0x3
120 #define _ZEND_ADDR_REG_SHIFT 2
121 #define _ZEND_ADDR_REG_MASK 0x3f /* no more than 64 registers */
122 #define _ZEND_ADDR_OFFSET_SHIFT 8
123 #define _ZEND_ADDR_REG_STORE_BIT 8
124 #define _ZEND_ADDR_REG_LOAD_BIT 9
125 #define _ZEND_ADDR_REG_LAST_USE_BIT 10
126
127 #define ZEND_ADDR_CONST_ZVAL(zv) \
128 (((zend_jit_addr)(uintptr_t)(zv)) | IS_CONST_ZVAL)
129 #define ZEND_ADDR_MEM_ZVAL(reg, offset) \
130 ((((zend_jit_addr)(uintptr_t)(offset)) << _ZEND_ADDR_OFFSET_SHIFT) | \
131 (((zend_jit_addr)(uintptr_t)(reg)) << _ZEND_ADDR_REG_SHIFT) | \
132 IS_MEM_ZVAL)
133 #define ZEND_ADDR_REG(reg) \
134 ((((zend_jit_addr)(uintptr_t)(reg)) << _ZEND_ADDR_REG_SHIFT) | \
135 IS_REG)
136
137 #define Z_MODE(addr) (((addr) & _ZEND_ADDR_MODE_MASK))
138 #define Z_ZV(addr) ((zval*)(addr))
139 #define Z_OFFSET(addr) ((uint32_t)((addr)>>_ZEND_ADDR_OFFSET_SHIFT))
140 #define Z_REG(addr) ((zend_reg)(((addr)>>_ZEND_ADDR_REG_SHIFT) & _ZEND_ADDR_REG_MASK))
141 #define Z_STORE(addr) ((zend_reg)(((addr)>>_ZEND_ADDR_REG_STORE_BIT) & 1))
142 #define Z_LOAD(addr) ((zend_reg)(((addr)>>_ZEND_ADDR_REG_LOAD_BIT) & 1))
143 #define Z_LAST_USE(addr) ((zend_reg)(((addr)>>_ZEND_ADDR_REG_LAST_USE_BIT) & 1))
144
145 #define OP_REG_EX(reg, store, load, last_use) \
146 ((reg) | \
147 ((store) ? (1 << (_ZEND_ADDR_REG_STORE_BIT-_ZEND_ADDR_REG_SHIFT)) : 0) | \
148 ((load) ? (1 << (_ZEND_ADDR_REG_LOAD_BIT-_ZEND_ADDR_REG_SHIFT)) : 0) | \
149 ((last_use) ? (1 << (_ZEND_ADDR_REG_LAST_USE_BIT-_ZEND_ADDR_REG_SHIFT)) : 0) \
150 )
151
152 #define OP_REG(ssa_op, op) \
153 (ra && ssa_op->op >= 0 && ra[ssa_op->op] ? \
154 OP_REG_EX(ra[ssa_op->op]->reg, \
155 (ra[ssa_op->op]->flags & ZREG_STORE), \
156 (ra[ssa_op->op]->flags & ZREG_LOAD), \
157 zend_ival_is_last_use(ra[ssa_op->op], ssa_op - ssa->ops) \
158 ) : ZREG_NONE)
159
_zend_jit_decode_op(uint8_t op_type,znode_op op,const zend_op * opline,zend_reg reg)160 static zend_always_inline zend_jit_addr _zend_jit_decode_op(uint8_t op_type, znode_op op, const zend_op *opline, zend_reg reg)
161 {
162 if (op_type == IS_CONST) {
163 #if ZEND_USE_ABS_CONST_ADDR
164 return ZEND_ADDR_CONST_ZVAL(op.zv);
165 #else
166 return ZEND_ADDR_CONST_ZVAL(RT_CONSTANT(opline, op));
167 #endif
168 } else {
169 ZEND_ASSERT(op_type & (IS_CV|IS_TMP_VAR|IS_VAR));
170 if (reg != ZREG_NONE) {
171 return ZEND_ADDR_REG(reg);
172 } else {
173 return ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var);
174 }
175 }
176 }
177
178 #define OP_ADDR(opline, type, op) \
179 _zend_jit_decode_op((opline)->type, (opline)->op, opline, ZREG_NONE)
180
181 #define OP1_ADDR() \
182 OP_ADDR(opline, op1_type, op1)
183 #define OP2_ADDR() \
184 OP_ADDR(opline, op2_type, op2)
185 #define RES_ADDR() \
186 OP_ADDR(opline, result_type, result)
187 #define OP1_DATA_ADDR() \
188 OP_ADDR(opline + 1, op1_type, op1)
189
190 #define OP_REG_ADDR(opline, type, _op, _ssa_op) \
191 _zend_jit_decode_op((opline)->type, (opline)->_op, opline, \
192 OP_REG(ssa_op, _ssa_op))
193
194 #define OP1_REG_ADDR() \
195 OP_REG_ADDR(opline, op1_type, op1, op1_use)
196 #define OP2_REG_ADDR() \
197 OP_REG_ADDR(opline, op2_type, op2, op2_use)
198 #define RES_REG_ADDR() \
199 OP_REG_ADDR(opline, result_type, result, result_def)
200 #define OP1_DATA_REG_ADDR() \
201 OP_REG_ADDR(opline + 1, op1_type, op1, op1_use)
202
203 #define OP1_DEF_REG_ADDR() \
204 OP_REG_ADDR(opline, op1_type, op1, op1_def)
205 #define OP2_DEF_REG_ADDR() \
206 OP_REG_ADDR(opline, op2_type, op2, op2_def)
207 #define RES_USE_REG_ADDR() \
208 OP_REG_ADDR(opline, result_type, result, result_use)
209 #define OP1_DATA_DEF_REG_ADDR() \
210 OP_REG_ADDR(opline + 1, op1_type, op1, op1_def)
211
zend_jit_same_addr(zend_jit_addr addr1,zend_jit_addr addr2)212 static zend_always_inline bool zend_jit_same_addr(zend_jit_addr addr1, zend_jit_addr addr2)
213 {
214 if (addr1 == addr2) {
215 return 1;
216 } else if (Z_MODE(addr1) == IS_REG && Z_MODE(addr2) == IS_REG) {
217 return Z_REG(addr1) == Z_REG(addr2);
218 }
219 return 0;
220 }
221
222 typedef struct _zend_jit_op_array_extension {
223 zend_func_info func_info;
224 const void *orig_handler;
225 } zend_jit_op_array_extension;
226
227 /* Profiler */
228 extern zend_ulong zend_jit_profile_counter;
229 extern int zend_jit_profile_counter_rid;
230
231 #define ZEND_COUNTER_INFO(op_array) \
232 ZEND_OP_ARRAY_EXTENSION(op_array, zend_jit_profile_counter_rid)
233
234 /* Hot Counters */
235
236 #define ZEND_HOT_COUNTERS_COUNT 128
237
238 extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
239
zend_jit_hash(const void * ptr)240 static zend_always_inline zend_long zend_jit_hash(const void *ptr)
241 {
242 uintptr_t x;
243
244 x = (uintptr_t)ptr >> 3;
245 #if SIZEOF_SIZE_T == 4
246 x = ((x >> 16) ^ x) * 0x45d9f3b;
247 x = ((x >> 16) ^ x) * 0x45d9f3b;
248 x = (x >> 16) ^ x;
249 #elif SIZEOF_SIZE_T == 8
250 x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
251 x = (x ^ (x >> 27)) * 0x94d049bb133111eb;
252 x = x ^ (x >> 31);
253 #endif
254 return x;
255 }
256
257 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline);
258
259 typedef struct _zend_jit_op_array_hot_extension {
260 zend_func_info func_info;
261 int16_t *counter;
262 const void *orig_handlers[1];
263 } zend_jit_op_array_hot_extension;
264
265 #define zend_jit_op_array_hash(op_array) \
266 zend_jit_hash((op_array)->opcodes)
267
268 extern const zend_op *zend_jit_halt_op;
269
270 #ifdef HAVE_GCC_GLOBAL_REGS
271 # define EXECUTE_DATA_D void
272 # define EXECUTE_DATA_C
273 # define EXECUTE_DATA_DC
274 # define EXECUTE_DATA_CC
275 # define OPLINE_D void
276 # define OPLINE_C
277 # define OPLINE_DC
278 # define OPLINE_CC
279 # define ZEND_OPCODE_HANDLER_RET void
280 # define ZEND_OPCODE_HANDLER_ARGS EXECUTE_DATA_D
281 # define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU
282 # define ZEND_OPCODE_HANDLER_ARGS_DC
283 # define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC
284 # define ZEND_OPCODE_RETURN() return
285 # define ZEND_OPCODE_TAIL_CALL(handler) do { \
286 handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
287 return; \
288 } while(0)
289 # define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \
290 handler(arg ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC); \
291 return; \
292 } while(0)
293 #else
294 # define EXECUTE_DATA_D zend_execute_data* execute_data
295 # define EXECUTE_DATA_C execute_data
296 # define EXECUTE_DATA_DC , EXECUTE_DATA_D
297 # define EXECUTE_DATA_CC , EXECUTE_DATA_C
298 # define OPLINE_D const zend_op* opline
299 # define OPLINE_C opline
300 # define OPLINE_DC , OPLINE_D
301 # define OPLINE_CC , OPLINE_C
302 # define ZEND_OPCODE_HANDLER_RET int
303 # define ZEND_OPCODE_HANDLER_ARGS EXECUTE_DATA_D
304 # define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU EXECUTE_DATA_C
305 # define ZEND_OPCODE_HANDLER_ARGS_DC EXECUTE_DATA_DC
306 # define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC EXECUTE_DATA_CC
307 # define ZEND_OPCODE_RETURN() return 0
308 # define ZEND_OPCODE_TAIL_CALL(handler) do { \
309 return handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
310 } while(0)
311 # define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \
312 return handler(arg ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC); \
313 } while(0)
314 #endif
315
316 /* VM handlers */
317 typedef ZEND_OPCODE_HANDLER_RET (ZEND_FASTCALL *zend_vm_opcode_handler_t)(ZEND_OPCODE_HANDLER_ARGS);
318
319 /* VM helpers */
320 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC);
321 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC);
322 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(EXECUTE_DATA_D);
323
324 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS);
325
326 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS);
327 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS);
328
329 void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D);
330 bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D);
331 void ZEND_FASTCALL zend_jit_undefined_long_key(EXECUTE_DATA_D);
332 void ZEND_FASTCALL zend_jit_undefined_string_key(EXECUTE_DATA_D);
333
334 zend_constant* ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags);
335 zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key);
336
337 /* Tracer */
338 #define zend_jit_opline_hash(opline) \
339 zend_jit_hash(opline)
340
341 #define ZEND_JIT_TRACE_STOP(_) \
342 _(LOOP, "loop") \
343 _(RECURSIVE_CALL, "recursive call") \
344 _(RECURSIVE_RET, "recursive return") \
345 _(RETURN, "return") \
346 _(INTERPRETER, "exit to VM interpreter") \
347 _(LINK, "link to another trace") \
348 /* compilation and linking successful */ \
349 _(COMPILED, "compiled") \
350 _(ALREADY_DONE, "already prcessed") \
351 /* failures */ \
352 _(ERROR, "error") /* not used */ \
353 _(NOT_SUPPORTED, "not supported instructions") \
354 _(EXCEPTION, "exception") \
355 _(TOO_LONG, "trace too long") \
356 _(TOO_DEEP, "trace too deep") \
357 _(TOO_DEEP_RET, "trace too deep return") \
358 _(DEEP_RECURSION, "deep recursion") \
359 _(LOOP_UNROLL, "loop unroll limit reached") \
360 _(LOOP_EXIT, "exit from loop") \
361 _(RECURSION_EXIT, "return from recursive function") \
362 _(BLACK_LIST, "trace blacklisted") \
363 _(INNER_LOOP, "inner loop") /* trace it */ \
364 _(COMPILED_LOOP, "compiled loop") \
365 _(TRAMPOLINE, "trampoline call") \
366 _(BAD_FUNC, "bad function call") \
367 _(COMPILER_ERROR, "JIT compilation error") \
368 /* no recoverable error (blacklist immediately) */ \
369 _(NO_SHM, "insufficient shared memory") \
370 _(TOO_MANY_TRACES, "too many traces") \
371 _(TOO_MANY_CHILDREN, "too many side traces") \
372 _(TOO_MANY_EXITS, "too many side exits") \
373
374 #define ZEND_JIT_TRACE_STOP_NAME(name, description) \
375 ZEND_JIT_TRACE_STOP_ ## name,
376
377 typedef enum _zend_jit_trace_stop {
378 ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_NAME)
379 ZEND_JIT_TRACE_HALT = 0x40
380 } zend_jit_trace_stop;
381
382 #define ZEND_JIT_TRACE_STOP_OK(ret) \
383 (ret < ZEND_JIT_TRACE_STOP_COMPILED)
384
385 #define ZEND_JIT_TRACE_STOP_DONE(ret) \
386 (ret < ZEND_JIT_TRACE_STOP_ERROR)
387
388 #define ZEND_JIT_TRACE_STOP_REPEAT(ret) \
389 (ret == ZEND_JIT_TRACE_STOP_INNER_LOOP)
390
391 #define ZEND_JIT_TRACE_STOP_MAY_RECOVER(ret) \
392 (ret <= ZEND_JIT_TRACE_STOP_COMPILER_ERROR)
393
394 #define ZEND_JIT_TRACE_START_MASK 0xf
395
396 #define ZEND_JIT_TRACE_START_LOOP (1<<0)
397 #define ZEND_JIT_TRACE_START_ENTER (1<<1)
398 #define ZEND_JIT_TRACE_START_RETURN (1<<2)
399 #define ZEND_JIT_TRACE_START_SIDE (1<<3) /* used for side traces */
400
401 #define ZEND_JIT_TRACE_JITED (1<<4)
402 #define ZEND_JIT_TRACE_BLACKLISTED (1<<5)
403 #define ZEND_JIT_TRACE_UNSUPPORTED (1<<6)
404
405 #define ZEND_JIT_TRACE_SUPPORTED 0
406
407 #define ZEND_JIT_EXIT_JITED (1<<0)
408 #define ZEND_JIT_EXIT_BLACKLISTED (1<<1)
409 #define ZEND_JIT_EXIT_TO_VM (1<<2) /* exit to VM without attempt to create a side trace */
410 #define ZEND_JIT_EXIT_RESTORE_CALL (1<<3) /* deoptimizer should restore EX(call) chain */
411 #define ZEND_JIT_EXIT_POLYMORPHISM (1<<4) /* exit because of polymorphic call */
412 #define ZEND_JIT_EXIT_FREE_OP1 (1<<5)
413 #define ZEND_JIT_EXIT_FREE_OP2 (1<<6)
414 #define ZEND_JIT_EXIT_PACKED_GUARD (1<<7)
415 #define ZEND_JIT_EXIT_CLOSURE_CALL (1<<8) /* exit because of polymorphic INIT_DYNAMIC_CALL call */
416 #define ZEND_JIT_EXIT_METHOD_CALL (1<<9) /* exit because of polymorphic INIT_METHOD_CALL call */
417 #define ZEND_JIT_EXIT_INVALIDATE (1<<10) /* invalidate current trace */
418
419 typedef union _zend_op_trace_info {
420 zend_op dummy; /* the size of this structure must be the same as zend_op */
421 struct {
422 const void *orig_handler;
423 const void *call_handler;
424 int16_t *counter;
425 uint8_t trace_flags;
426 };
427 } zend_op_trace_info;
428
429 typedef struct _zend_jit_op_array_trace_extension {
430 zend_func_info func_info;
431 const zend_op_array *op_array;
432 size_t offset; /* offset from "zend_op" to corresponding "op_info" */
433 zend_op_trace_info trace_info[1];
434 } zend_jit_op_array_trace_extension;
435
436 #define ZEND_OP_TRACE_INFO(opline, offset) \
437 ((zend_op_trace_info*)(((char*)opline) + offset))
438
439 /* Recorder */
440 typedef enum _zend_jit_trace_op {
441 ZEND_JIT_TRACE_VM,
442 ZEND_JIT_TRACE_OP1_TYPE,
443 ZEND_JIT_TRACE_OP2_TYPE,
444 ZEND_JIT_TRACE_VAL_INFO,
445 ZEND_JIT_TRACE_INIT_CALL,
446 ZEND_JIT_TRACE_DO_ICALL,
447 ZEND_JIT_TRACE_ENTER,
448 ZEND_JIT_TRACE_BACK,
449 ZEND_JIT_TRACE_END,
450 ZEND_JIT_TRACE_START,
451 } zend_jit_trace_op;
452
453 #define IS_UNKNOWN 255 /* may be used for zend_jit_trace_rec.op?_type */
454 #define IS_TRACE_PACKED (1<<4)
455 #define IS_TRACE_REFERENCE (1<<5)
456 #define IS_TRACE_INDIRECT (1<<6)
457
458 #define IS_TRACE_TYPE_MASK 0xf
459
460 #define ZEND_JIT_TRACE_FAKE_INIT_CALL 0x00000100
461 #define ZEND_JIT_TRACE_RETURN_VALUE_USED 0x00000100
462
463 #define ZEND_JIT_TRACE_MAX_SSA_VAR 0x7ffffe
464 #define ZEND_JIT_TRACE_SSA_VAR_SHIFT 9
465
466 #define ZEND_JIT_TRACE_FAKE_LEVEL_MASK 0xffff0000
467 #define ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT 16
468
469 #define ZEND_JIT_TRACE_FAKE_LEVEL(info) \
470 (((info) & ZEND_JIT_TRACE_FAKE_LEVEL_MASK) >> ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT)
471
472 #define ZEND_JIT_TRACE_FAKE_INFO(level) \
473 (((level) << ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT) | ZEND_JIT_TRACE_FAKE_INIT_CALL)
474
475 #define ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(_info, var) do { \
476 _info |= (var << ZEND_JIT_TRACE_SSA_VAR_SHIFT); \
477 } while (0)
478 #define ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(_info) \
479 (_info >> ZEND_JIT_TRACE_SSA_VAR_SHIFT)
480
481 struct _zend_jit_trace_rec {
482 union {
483 struct { ZEND_ENDIAN_LOHI(
484 uint8_t op, /* zend_jit_trace_op */
485 union {
486 struct {
487 uint8_t op1_type;/* recorded zval op1_type for ZEND_JIT_TRACE_VM */
488 uint8_t op2_type;/* recorded zval op2_type for ZEND_JIT_TRACE_VM */
489 uint8_t op3_type;/* recorded zval for op_data.op1_type for ZEND_JIT_TRACE_VM */
490 };
491 struct {
492 uint8_t start; /* ZEND_JIT_TRACE_START_MASK for ZEND_JIT_TRACE_START/END */
493 uint8_t stop; /* zend_jit_trace_stop for ZEND_JIT_TRACE_START/END */
494 uint8_t level; /* recursive return level for ZEND_JIT_TRACE_START */
495 };
496 })
497 };
498 uint32_t last;
499 uint32_t info; /* "first_ssa_var" for ZEND_JIT_TRACE_ENTER and ZEND_JIT_TRACE_BACK,
500 * "return_value_used" for ZEND_JIT_TRACE_ENTER,
501 * "fake" for ZEND_JIT_TRACE_INIT_CALL */
502 };
503 union {
504 const void *ptr;
505 const zend_function *func;
506 const zend_op_array *op_array;
507 const zend_op *opline;
508 const zend_class_entry *ce;
509 };
510 };
511
512 #define ZEND_JIT_TRACE_START_REC_SIZE 2
513
514 typedef struct _zend_jit_trace_exit_info {
515 const zend_op *opline; /* opline where VM should continue execution */
516 const zend_op_array *op_array;
517 uint32_t flags; /* set of ZEND_JIT_EXIT_... */
518 uint32_t stack_size;
519 uint32_t stack_offset;
520 } zend_jit_trace_exit_info;
521
522 typedef union _zend_jit_trace_stack {
523 int32_t ssa_var;
524 uint32_t info;
525 struct {
526 uint8_t type; /* variable type (for type inference) */
527 uint8_t mem_type; /* stack slot type (for eliminate dead type store) */
528 int8_t reg;
529 uint8_t flags;
530 };
531 } zend_jit_trace_stack;
532
533 #define STACK_VAR(_stack, _slot) \
534 (_stack)[_slot].ssa_var
535 #define STACK_INFO(_stack, _slot) \
536 (_stack)[_slot].info
537 #define STACK_TYPE(_stack, _slot) \
538 (_stack)[_slot].type
539 #define STACK_MEM_TYPE(_stack, _slot) \
540 (_stack)[_slot].mem_type
541 #define STACK_REG(_stack, _slot) \
542 (_stack)[_slot].reg
543 #define STACK_FLAGS(_stack, _slot) \
544 (_stack)[_slot].flags
545 #define SET_STACK_VAR(_stack, _slot, _ssa_var) do { \
546 (_stack)[_slot].ssa_var = _ssa_var; \
547 } while (0)
548 #define SET_STACK_INFO(_stack, _slot, _info) do { \
549 (_stack)[_slot].info = _info; \
550 } while (0)
551 #define SET_STACK_TYPE(_stack, _slot, _type, _set_mem_type) do { \
552 uint8_t __type = (_type); \
553 (_stack)[_slot].type = __type; \
554 if (_set_mem_type) { \
555 (_stack)[_slot].mem_type = __type; \
556 } \
557 (_stack)[_slot].reg = ZREG_NONE; \
558 (_stack)[_slot].flags = 0; \
559 } while (0)
560 #define SET_STACK_REG(_stack, _slot, _reg) do { \
561 (_stack)[_slot].reg = _reg; \
562 (_stack)[_slot].flags = 0; \
563 } while (0)
564 #define SET_STACK_REG_EX(_stack, _slot, _reg, _flags) do { \
565 (_stack)[_slot].reg = _reg; \
566 (_stack)[_slot].flags = _flags; \
567 } while (0)
568 #define RESET_STACK_MEM_TYPE(_stack, _slot) do { \
569 (_stack)[_slot].mem_type = IS_UNKNOWN; \
570 } while (0)
571
572 /* trace info flags */
573 #define ZEND_JIT_TRACE_CHECK_INTERRUPT (1<<0)
574 #define ZEND_JIT_TRACE_LOOP (1<<1)
575 #define ZEND_JIT_TRACE_USES_INITIAL_IP (1<<2)
576
577 typedef struct _zend_jit_trace_info {
578 uint32_t id; /* trace id */
579 uint32_t root; /* root trace id or self id for root traces */
580 uint32_t parent; /* parent trace id or 0 for root traces */
581 uint32_t link; /* link trace id or self id for loop) */
582 uint32_t exit_count; /* number of side exits */
583 uint32_t child_count; /* number of side traces for root traces */
584 uint32_t code_size; /* size of native code */
585 uint32_t exit_counters; /* offset in exit counters array */
586 uint32_t stack_map_size;
587 uint32_t flags; /* See ZEND_JIT_TRACE_... defines above */
588 uint32_t polymorphism; /* Counter of polymorphic calls */
589 uint32_t jmp_table_size;/* number of jmp_table slots */
590 const zend_op_array *op_array; /* function */
591 const zend_op *opline; /* first opline */
592 const void *code_start; /* address of native code */
593 zend_jit_trace_exit_info *exit_info; /* info about side exits */
594 zend_jit_trace_stack *stack_map;
595 //uint32_t loop_offset;
596 } zend_jit_trace_info;
597
598 struct _zend_jit_trace_stack_frame {
599 zend_jit_trace_stack_frame *call;
600 zend_jit_trace_stack_frame *prev;
601 const zend_function *func;
602 const zend_op *call_opline;
603 zend_class_entry *ce; /* $this */
604 uint32_t call_level;
605 uint32_t _info;
606 int used_stack;
607 int old_checked_stack;
608 int old_peek_checked_stack;
609 zend_jit_trace_stack stack[1];
610 };
611
612 #define TRACE_FRAME_SHIFT_NUM_ARGS 16
613 #define TRACE_FRAME_MAX_NUM_ARGS 32767
614
615 #define TRACE_FRAME_MASK_NUM_ARGS 0xffff0000
616 #define TRACE_FRAME_MASK_NESTED 0x00000001
617 #define TRACE_FRAME_MASK_LAST_SEND_BY_REF 0x00000002
618 #define TRACE_FRAME_MASK_LAST_SEND_BY_VAL 0x00000004
619 #define TRACE_FRAME_MASK_RETURN_VALUE_USED 0x00000008
620 #define TRACE_FRAME_MASK_RETURN_VALUE_UNUSED 0x00000010
621 #define TRACE_FRAME_MASK_THIS_CHECKED 0x00000020
622 #define TRACE_FRAME_MASK_UNKNOWN_RETURN 0x00000040
623 #define TRACE_FRAME_MASK_NO_NEED_RELEASE_THIS 0x00000080
624 #define TRACE_FRAME_MASK_THIS_CLASS_CHECKED 0x00000100
625 #define TRACE_FRAME_MASK_CLOSURE_CALL 0x00000200
626 #define TRACE_FRAME_MASK_ALWAYS_RELEASE_THIS 0x00000400
627
628
629 #define TRACE_FRAME_INIT(frame, _func, _flags, num_args) do { \
630 zend_jit_trace_stack_frame *_frame = (frame); \
631 _frame->call = NULL; \
632 _frame->prev = NULL; \
633 _frame->func = (const zend_function*)_func; \
634 _frame->call_opline = NULL; \
635 _frame->call_level = 0; \
636 _frame->_info = (((uint32_t)(num_args)) << TRACE_FRAME_SHIFT_NUM_ARGS) & TRACE_FRAME_MASK_NUM_ARGS; \
637 _frame->_info |= _flags; \
638 } while (0)
639
640 #define TRACE_FRAME_RETURN_SSA_VAR(frame) \
641 ((int)(frame)->_info)
642 #define TRACE_FRAME_NUM_ARGS(frame) \
643 ((int)((frame)->_info) >> TRACE_FRAME_SHIFT_NUM_ARGS)
644 #define TRACE_FRAME_IS_NESTED(frame) \
645 ((frame)->_info & TRACE_FRAME_MASK_NESTED)
646 #define TRACE_FRAME_IS_LAST_SEND_BY_REF(frame) \
647 ((frame)->_info & TRACE_FRAME_MASK_LAST_SEND_BY_REF)
648 #define TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame) \
649 ((frame)->_info & TRACE_FRAME_MASK_LAST_SEND_BY_VAL)
650 #define TRACE_FRAME_IS_RETURN_VALUE_USED(frame) \
651 ((frame)->_info & TRACE_FRAME_MASK_RETURN_VALUE_USED)
652 #define TRACE_FRAME_IS_RETURN_VALUE_UNUSED(frame) \
653 ((frame)->_info & TRACE_FRAME_MASK_RETURN_VALUE_UNUSED)
654 #define TRACE_FRAME_IS_THIS_CHECKED(frame) \
655 ((frame)->_info & TRACE_FRAME_MASK_THIS_CHECKED)
656 #define TRACE_FRAME_IS_UNKNOWN_RETURN(frame) \
657 ((frame)->_info & TRACE_FRAME_MASK_UNKNOWN_RETURN)
658 #define TRACE_FRAME_NO_NEED_RELEASE_THIS(frame) \
659 ((frame)->_info & TRACE_FRAME_MASK_NO_NEED_RELEASE_THIS)
660 #define TRACE_FRAME_IS_THIS_CLASS_CHECKED(frame) \
661 ((frame)->_info & TRACE_FRAME_MASK_THIS_CLASS_CHECKED)
662 #define TRACE_FRAME_IS_CLOSURE_CALL(frame) \
663 ((frame)->_info & TRACE_FRAME_MASK_CLOSURE_CALL)
664 #define TRACE_FRAME_ALWAYS_RELEASE_THIS(frame) \
665 ((frame)->_info & TRACE_FRAME_MASK_ALWAYS_RELEASE_THIS)
666
667 #define TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(frame) do { \
668 (frame)->_info |= (0xffffu << TRACE_FRAME_SHIFT_NUM_ARGS); \
669 } while (0)
670 #define TRACE_FRAME_SET_RETURN_SSA_VAR(frame, var) do { \
671 (frame)->_info = var; \
672 } while (0)
673 #define TRACE_FRAME_SET_LAST_SEND_BY_REF(frame) do { \
674 (frame)->_info |= TRACE_FRAME_MASK_LAST_SEND_BY_REF; \
675 (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_VAL; \
676 } while (0)
677 #define TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame) do { \
678 (frame)->_info |= TRACE_FRAME_MASK_LAST_SEND_BY_VAL; \
679 (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_REF; \
680 } while (0)
681 #define TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame) do { \
682 (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_VAL; \
683 (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_REF; \
684 } while (0)
685 #define TRACE_FRAME_SET_RETURN_VALUE_USED(frame) do { \
686 (frame)->_info |= TRACE_FRAME_MASK_RETURN_VALUE_USED; \
687 (frame)->_info &= ~TRACE_FRAME_MASK_RETURN_VALUE_UNUSED; \
688 } while (0)
689 #define TRACE_FRAME_SET_RETURN_VALUE_UNUSED(frame) do { \
690 (frame)->_info |= TRACE_FRAME_MASK_RETURN_VALUE_UNUSED; \
691 (frame)->_info &= ~TRACE_FRAME_MASK_RETURN_VALUE_USED; \
692 } while (0)
693 #define TRACE_FRAME_SET_THIS_CHECKED(frame) do { \
694 (frame)->_info |= TRACE_FRAME_MASK_THIS_CHECKED; \
695 } while (0)
696 #define TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(frame) do { \
697 (frame)->_info |= TRACE_FRAME_MASK_NO_NEED_RELEASE_THIS; \
698 } while (0)
699 #define TRACE_FRAME_SET_THIS_CLASS_CHECKED(frame) do { \
700 (frame)->_info |= TRACE_FRAME_MASK_THIS_CLASS_CHECKED; \
701 } while (0)
702 #define TRACE_FRAME_SET_CLOSURE_CALL(frame) do { \
703 (frame)->_info |= TRACE_FRAME_MASK_CLOSURE_CALL; \
704 } while (0)
705 #define TRACE_FRAME_SET_ALWAYS_RELEASE_THIS(frame) do { \
706 (frame)->_info |= TRACE_FRAME_MASK_ALWAYS_RELEASE_THIS; \
707 } while (0)
708
709 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
710 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
711 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
712
713 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline);
714 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs);
715 zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start, uint32_t is_megamorphc);
716
zend_jit_trace_get_exit_opline(zend_jit_trace_rec * trace,const zend_op * opline,bool * exit_if_true)717 static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit_trace_rec *trace, const zend_op *opline, bool *exit_if_true)
718 {
719 if (trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END) {
720 if (trace->opline == opline + 1) {
721 /* not taken branch */
722 *exit_if_true = opline->opcode == ZEND_JMPNZ;
723 return OP_JMP_ADDR(opline, opline->op2);
724 } else if (trace->opline == OP_JMP_ADDR(opline, opline->op2)) {
725 /* taken branch */
726 *exit_if_true = opline->opcode == ZEND_JMPZ;
727 return opline + 1;
728 } else {
729 ZEND_UNREACHABLE();
730 }
731 } else {
732 ZEND_UNREACHABLE();
733 }
734 *exit_if_true = 0;
735 return NULL;
736 }
737
zend_jit_may_be_modified(const zend_function * func,const zend_op_array * called_from)738 static inline bool zend_jit_may_be_modified(const zend_function *func, const zend_op_array *called_from)
739 {
740 if (func->type == ZEND_INTERNAL_FUNCTION) {
741 #ifdef _WIN32
742 /* ASLR */
743 return 1;
744 #else
745 return 0;
746 #endif
747 } else if (func->type == ZEND_USER_FUNCTION) {
748 if (func->common.fn_flags & ZEND_ACC_PRELOADED) {
749 return 0;
750 }
751 if (func->op_array.filename == called_from->filename && !func->op_array.scope) {
752 return 0;
753 }
754 }
755 return 1;
756 }
757
zend_jit_may_be_polymorphic_call(const zend_op * opline)758 static zend_always_inline bool zend_jit_may_be_polymorphic_call(const zend_op *opline)
759 {
760 if (opline->opcode == ZEND_INIT_FCALL
761 || opline->opcode == ZEND_INIT_FCALL_BY_NAME
762 || opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) {
763 return 0;
764 } else if (opline->opcode == ZEND_INIT_METHOD_CALL
765 || opline->opcode == ZEND_INIT_DYNAMIC_CALL) {
766 return 1;
767 } else if (opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
768 return (opline->op1_type != IS_CONST || opline->op2_type != IS_CONST);
769 } else if (opline->opcode == ZEND_INIT_USER_CALL) {
770 return (opline->op2_type != IS_CONST);
771 } else if (opline->opcode == ZEND_NEW) {
772 return (opline->op1_type != IS_CONST);
773 } else {
774 ZEND_UNREACHABLE();
775 return 0;
776 }
777 }
778
779 /* Instruction cache flush */
780 #ifndef JIT_CACHE_FLUSH
781 # if ZEND_JIT_TARGET_ARM64
782 # if ((defined(__GNUC__) && ZEND_GCC_VERSION >= 4003) || __has_builtin(__builtin___clear_cache))
783 # define JIT_CACHE_FLUSH(from, to) __builtin___clear_cache((char*)(from), (char*)(to))
784 # else
785 # error "Missing builtin to flush instruction cache for AArch64"
786 # endif
787 # else /* Not required to implement on archs with unified caches */
788 # define JIT_CACHE_FLUSH(from, to)
789 # endif
790 #endif /* !JIT_CACHE_FLUSH */
791
792 /* bit helpers */
793
zend_long_is_power_of_two(zend_long x)794 static zend_always_inline bool zend_long_is_power_of_two(zend_long x)
795 {
796 return (x > 0) && !(x & (x - 1));
797 }
798
zend_long_floor_log2(zend_long x)799 static zend_always_inline uint32_t zend_long_floor_log2(zend_long x)
800 {
801 ZEND_ASSERT(zend_long_is_power_of_two(x));
802 return zend_ulong_ntz(x);
803 }
804
805 /* from http://aggregate.org/MAGIC/ */
ones32(uint32_t x)806 static zend_always_inline uint32_t ones32(uint32_t x)
807 {
808 x -= ((x >> 1) & 0x55555555);
809 x = (((x >> 2) & 0x33333333) + (x & 0x33333333));
810 x = (((x >> 4) + x) & 0x0f0f0f0f);
811 x += (x >> 8);
812 x += (x >> 16);
813 return x & 0x0000003f;
814 }
815
floor_log2(uint32_t x)816 static zend_always_inline uint32_t floor_log2(uint32_t x)
817 {
818 ZEND_ASSERT(x != 0);
819 x |= (x >> 1);
820 x |= (x >> 2);
821 x |= (x >> 4);
822 x |= (x >> 8);
823 x |= (x >> 16);
824 return ones32(x) - 1;
825 }
826
is_power_of_two(uint32_t x)827 static zend_always_inline bool is_power_of_two(uint32_t x)
828 {
829 return !(x & (x - 1)) && x != 0;
830 }
831
has_concrete_type(uint32_t value_type)832 static zend_always_inline bool has_concrete_type(uint32_t value_type)
833 {
834 return is_power_of_two (value_type & (MAY_BE_ANY|MAY_BE_UNDEF));
835 }
836
concrete_type(uint32_t value_type)837 static zend_always_inline uint32_t concrete_type(uint32_t value_type)
838 {
839 return floor_log2(value_type & (MAY_BE_ANY|MAY_BE_UNDEF));
840 }
841
is_signed(double d)842 static zend_always_inline bool is_signed(double d)
843 {
844 return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0;
845 }
846
847 #endif /* ZEND_JIT_INTERNAL_H */
848
