1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 | Xinchen Hui <laruence@php.net> |
17 | Hao Sun <hao.sun@arm.com> |
18 +----------------------------------------------------------------------+
19 */
20
21 #ifndef ZEND_JIT_INTERNAL_H
22 #define ZEND_JIT_INTERNAL_H
23
24 /* Address Encoding */
25 typedef uintptr_t zend_jit_addr;
26
27 #define IS_CONST_ZVAL 0
28 #define IS_MEM_ZVAL 1
29 #define IS_REG 2 /* CPU register or PHP SSA variable number (for IR JIT) */
30 #define IS_REF_ZVAL 3 /* IR reference */
31
32 #define _ZEND_ADDR_MODE_MASK 0x3
33 #define _ZEND_ADDR_REG_SHIFT 2
34 #define _ZEND_ADDR_REG_MASK 0x3f /* no more than 64 registers */
35 #define _ZEND_ADDR_OFFSET_SHIFT 8
36
37 #define ZEND_ADDR_CONST_ZVAL(zv) \
38 (((zend_jit_addr)(uintptr_t)(zv)) | IS_CONST_ZVAL)
39 #define ZEND_ADDR_MEM_ZVAL(reg, offset) \
40 ((((zend_jit_addr)(uintptr_t)(offset)) << _ZEND_ADDR_OFFSET_SHIFT) | \
41 (((zend_jit_addr)(uintptr_t)(reg)) << _ZEND_ADDR_REG_SHIFT) | \
42 IS_MEM_ZVAL)
43 #define ZEND_ADDR_REG(reg) \
44 ((((zend_jit_addr)(uintptr_t)(reg)) << _ZEND_ADDR_REG_SHIFT) | \
45 IS_REG)
46
47 #define Z_MODE(addr) (((addr) & _ZEND_ADDR_MODE_MASK))
48 #define Z_ZV(addr) ((zval*)(addr))
49 #define Z_OFFSET(addr) ((uint32_t)((addr)>>_ZEND_ADDR_OFFSET_SHIFT))
50 #define Z_REG(addr) ((zend_reg)(((addr)>>_ZEND_ADDR_REG_SHIFT) & _ZEND_ADDR_REG_MASK))
51
52 #define ZEND_ADDR_REF_ZVAL(ref) \
53 ((((zend_jit_addr)(uintptr_t)(ref)) << _ZEND_ADDR_REG_SHIFT) | \
54 IS_REF_ZVAL)
55
56 #define Z_SSA_VAR(addr) ((addr)>>_ZEND_ADDR_REG_SHIFT)
57 #define Z_IR_REF(addr) ((addr)>>_ZEND_ADDR_REG_SHIFT)
58
59 #define Z_STORE(addr) \
60 ((jit->ra && jit->ra[Z_SSA_VAR(addr)].ref) ? \
61 (jit->ra[Z_SSA_VAR(addr)].flags & ZREG_STORE) : \
62 0)
63 #define Z_LOAD(addr) \
64 ((jit->ra && jit->ra[Z_SSA_VAR(addr)].ref) ? \
65 (jit->ra[Z_SSA_VAR(addr)].flags & ZREG_LOAD) : \
66 0)
67
68 #if ZEND_USE_ABS_CONST_ADDR
69 # define OP_ADDR(opline, type, op) \
70 (((opline)->type == IS_CONST) ? \
71 ZEND_ADDR_CONST_ZVAL((opline)->op.zv) : \
72 ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline)->op.var))
73 #else
74 # define OP_ADDR(opline, type, op) \
75 (((opline)->type == IS_CONST) ? \
76 ZEND_ADDR_CONST_ZVAL(RT_CONSTANT(opline, (opline)->op)) : \
77 ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline)->op.var))
78 #endif
79
80 #define OP_REG_ADDR(opline, ssa_op, type, op, _ssa_op) \
81 ((ctx.ra && (ssa_op)->_ssa_op >= 0 && ctx.ra[(ssa_op)->_ssa_op].ref) ? \
82 ZEND_ADDR_REG((ssa_op)->_ssa_op) : \
83 OP_ADDR(opline, type, op))
84
85 #define OP1_ADDR() \
86 OP_ADDR(opline, op1_type, op1)
87 #define OP2_ADDR() \
88 OP_ADDR(opline, op2_type, op2)
89 #define RES_ADDR() \
90 OP_ADDR(opline, result_type, result)
91 #define OP1_DATA_ADDR() \
92 OP_ADDR(opline + 1, op1_type, op1)
93
94 #define OP1_REG_ADDR() \
95 OP_REG_ADDR(opline, ssa_op, op1_type, op1, op1_use)
96 #define OP2_REG_ADDR() \
97 OP_REG_ADDR(opline, ssa_op, op2_type, op2, op2_use)
98 #define RES_REG_ADDR() \
99 OP_REG_ADDR(opline, ssa_op, result_type, result, result_def)
100 #define OP1_DATA_REG_ADDR() \
101 OP_REG_ADDR(opline + 1, ssa_op + 1, op1_type, op1, op1_use)
102
103 #define OP1_DEF_REG_ADDR() \
104 OP_REG_ADDR(opline, ssa_op, op1_type, op1, op1_def)
105 #define OP2_DEF_REG_ADDR() \
106 OP_REG_ADDR(opline, ssa_op, op2_type, op2, op2_def)
107 #define RES_USE_REG_ADDR() \
108 OP_REG_ADDR(opline, ssa_op, result_type, result, result_use)
109 #define OP1_DATA_DEF_REG_ADDR() \
110 OP_REG_ADDR(opline + 1, ssa_op + 1, op1_type, op1, op1_def)
111
zend_jit_same_addr(zend_jit_addr addr1,zend_jit_addr addr2)112 static zend_always_inline bool zend_jit_same_addr(zend_jit_addr addr1, zend_jit_addr addr2)
113 {
114 if (addr1 == addr2) {
115 return 1;
116 } else if (Z_MODE(addr1) == IS_REG && Z_MODE(addr2) == IS_REG) {
117 return Z_SSA_VAR(addr1) == Z_SSA_VAR(addr2);
118 } else if (Z_MODE(addr1) == IS_REF_ZVAL && Z_MODE(addr2) == IS_REF_ZVAL) {
119 return Z_IR_REF(addr1) == Z_IR_REF(addr2);
120 }
121 return 0;
122 }
123
124 typedef struct _zend_jit_op_array_extension {
125 zend_func_info func_info;
126 const void *orig_handler;
127 } zend_jit_op_array_extension;
128
129 /* Profiler */
130 extern zend_ulong zend_jit_profile_counter;
131 extern int zend_jit_profile_counter_rid;
132
133 #define ZEND_COUNTER_INFO(op_array) \
134 ZEND_OP_ARRAY_EXTENSION(op_array, zend_jit_profile_counter_rid)
135
136 /* Hot Counters */
137
138 #define ZEND_HOT_COUNTERS_COUNT 128
139
140 extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
141
zend_jit_hash(const void * ptr)142 static zend_always_inline zend_long zend_jit_hash(const void *ptr)
143 {
144 uintptr_t x;
145
146 x = (uintptr_t)ptr >> 3;
147 #if SIZEOF_SIZE_T == 4
148 x = ((x >> 16) ^ x) * 0x45d9f3b;
149 x = ((x >> 16) ^ x) * 0x45d9f3b;
150 x = (x >> 16) ^ x;
151 #elif SIZEOF_SIZE_T == 8
152 x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
153 x = (x ^ (x >> 27)) * 0x94d049bb133111eb;
154 x = x ^ (x >> 31);
155 #endif
156 return x;
157 }
158
159 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline);
160
161 typedef struct _zend_jit_op_array_hot_extension {
162 zend_func_info func_info;
163 int16_t *counter;
164 const void *orig_handlers[1];
165 } zend_jit_op_array_hot_extension;
166
167 #define zend_jit_op_array_hash(op_array) \
168 zend_jit_hash((op_array)->opcodes)
169
170 extern const zend_op *zend_jit_halt_op;
171
172 #ifdef HAVE_GCC_GLOBAL_REGS
173 # define EXECUTE_DATA_D void
174 # define EXECUTE_DATA_C
175 # define EXECUTE_DATA_DC
176 # define EXECUTE_DATA_CC
177 # define OPLINE_D void
178 # define OPLINE_C
179 # define OPLINE_DC
180 # define OPLINE_CC
181 # define ZEND_OPCODE_HANDLER_RET void
182 # define ZEND_OPCODE_HANDLER_ARGS EXECUTE_DATA_D
183 # define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU
184 # define ZEND_OPCODE_HANDLER_ARGS_DC
185 # define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC
186 # define ZEND_OPCODE_RETURN() return
187 # define ZEND_OPCODE_TAIL_CALL(handler) do { \
188 handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
189 return; \
190 } while(0)
191 # define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \
192 handler(arg ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC); \
193 return; \
194 } while(0)
195 #else
196 # define EXECUTE_DATA_D zend_execute_data* execute_data
197 # define EXECUTE_DATA_C execute_data
198 # define EXECUTE_DATA_DC , EXECUTE_DATA_D
199 # define EXECUTE_DATA_CC , EXECUTE_DATA_C
200 # define OPLINE_D const zend_op* opline
201 # define OPLINE_C opline
202 # define OPLINE_DC , OPLINE_D
203 # define OPLINE_CC , OPLINE_C
204 # define ZEND_OPCODE_HANDLER_RET int
205 # define ZEND_OPCODE_HANDLER_ARGS EXECUTE_DATA_D
206 # define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU EXECUTE_DATA_C
207 # define ZEND_OPCODE_HANDLER_ARGS_DC EXECUTE_DATA_DC
208 # define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC EXECUTE_DATA_CC
209 # define ZEND_OPCODE_RETURN() return 0
210 # define ZEND_OPCODE_TAIL_CALL(handler) do { \
211 return handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
212 } while(0)
213 # define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \
214 return handler(arg ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC); \
215 } while(0)
216 #endif
217
218 /* VM handlers */
219 typedef ZEND_OPCODE_HANDLER_RET (ZEND_FASTCALL *zend_vm_opcode_handler_t)(ZEND_OPCODE_HANDLER_ARGS);
220
221 /* VM helpers */
222 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC);
223 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC);
224 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(EXECUTE_DATA_D);
225
226 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS);
227
228 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS);
229 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS);
230
231 void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D);
232 bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D);
233 void ZEND_FASTCALL zend_jit_undefined_long_key(EXECUTE_DATA_D);
234 void ZEND_FASTCALL zend_jit_undefined_long_key_ex(zend_long key EXECUTE_DATA_DC);
235 void ZEND_FASTCALL zend_jit_undefined_string_key(EXECUTE_DATA_D);
236
237 zend_constant* ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags);
238 zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key);
239
240 /* Tracer */
241 #define zend_jit_opline_hash(opline) \
242 zend_jit_hash(opline)
243
244 #define ZEND_JIT_TRACE_STOP(_) \
245 _(LOOP, "loop") \
246 _(RECURSIVE_CALL, "recursive call") \
247 _(RECURSIVE_RET, "recursive return") \
248 _(RETURN, "return") \
249 _(INTERPRETER, "exit to VM interpreter") \
250 _(LINK, "link to another trace") \
251 /* compilation and linking successful */ \
252 _(COMPILED, "compiled") \
253 _(ALREADY_DONE, "already prcessed") \
254 /* failures */ \
255 _(ERROR, "error") /* not used */ \
256 _(NOT_SUPPORTED, "not supported instructions") \
257 _(EXCEPTION, "exception") \
258 _(TOO_LONG, "trace too long") \
259 _(TOO_DEEP, "trace too deep") \
260 _(TOO_DEEP_RET, "trace too deep return") \
261 _(DEEP_RECURSION, "deep recursion") \
262 _(LOOP_UNROLL, "loop unroll limit reached") \
263 _(LOOP_EXIT, "exit from loop") \
264 _(RECURSION_EXIT, "return from recursive function") \
265 _(BLACK_LIST, "trace blacklisted") \
266 _(INNER_LOOP, "inner loop") /* trace it */ \
267 _(COMPILED_LOOP, "compiled loop") \
268 _(TRAMPOLINE, "trampoline call") \
269 _(PROP_HOOK_CALL, "property hook call") \
270 _(BAD_FUNC, "bad function call") \
271 _(COMPILER_ERROR, "JIT compilation error") \
272 /* no recoverable error (blacklist immediately) */ \
273 _(NO_SHM, "insufficient shared memory") \
274 _(TOO_MANY_TRACES, "too many traces") \
275 _(TOO_MANY_CHILDREN, "too many side traces") \
276 _(TOO_MANY_EXITS, "too many side exits") \
277
278 #define ZEND_JIT_TRACE_STOP_NAME(name, description) \
279 ZEND_JIT_TRACE_STOP_ ## name,
280
281 typedef enum _zend_jit_trace_stop {
282 ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_NAME)
283 ZEND_JIT_TRACE_HALT = 0x40
284 } zend_jit_trace_stop;
285
286 #define ZEND_JIT_TRACE_STOP_OK(ret) \
287 (ret < ZEND_JIT_TRACE_STOP_COMPILED)
288
289 #define ZEND_JIT_TRACE_STOP_DONE(ret) \
290 (ret < ZEND_JIT_TRACE_STOP_ERROR)
291
292 #define ZEND_JIT_TRACE_STOP_REPEAT(ret) \
293 (ret == ZEND_JIT_TRACE_STOP_INNER_LOOP)
294
295 #define ZEND_JIT_TRACE_STOP_MAY_RECOVER(ret) \
296 (ret <= ZEND_JIT_TRACE_STOP_COMPILER_ERROR)
297
298 #define ZEND_JIT_TRACE_START_MASK 0xf
299
300 #define ZEND_JIT_TRACE_START_LOOP (1<<0)
301 #define ZEND_JIT_TRACE_START_ENTER (1<<1)
302 #define ZEND_JIT_TRACE_START_RETURN (1<<2)
303 #define ZEND_JIT_TRACE_START_SIDE (1<<3) /* used for side traces */
304
305 #define ZEND_JIT_TRACE_JITED (1<<4)
306 #define ZEND_JIT_TRACE_BLACKLISTED (1<<5)
307 #define ZEND_JIT_TRACE_UNSUPPORTED (1<<6)
308
309 #define ZEND_JIT_TRACE_SUPPORTED 0
310
311 #define ZEND_JIT_EXIT_JITED (1<<0)
312 #define ZEND_JIT_EXIT_BLACKLISTED (1<<1)
313 #define ZEND_JIT_EXIT_TO_VM (1<<2) /* exit to VM without attempt to create a side trace */
314 #define ZEND_JIT_EXIT_RESTORE_CALL (1<<3) /* deoptimizer should restore EX(call) chain */
315 #define ZEND_JIT_EXIT_POLYMORPHISM (1<<4) /* exit because of polymorphic call */
316 #define ZEND_JIT_EXIT_FREE_OP1 (1<<5)
317 #define ZEND_JIT_EXIT_FREE_OP2 (1<<6)
318 #define ZEND_JIT_EXIT_PACKED_GUARD (1<<7)
319 #define ZEND_JIT_EXIT_CLOSURE_CALL (1<<8) /* exit because of polymorphic INIT_DYNAMIC_CALL call */
320 #define ZEND_JIT_EXIT_METHOD_CALL (1<<9) /* exit because of polymorphic INIT_METHOD_CALL call */
321 #define ZEND_JIT_EXIT_INVALIDATE (1<<10) /* invalidate current trace */
322
323 #define ZEND_JIT_EXIT_FIXED (1U<<31) /* the exit_info can't be changed by zend_jit_snapshot_handler() */
324
325 typedef union _zend_op_trace_info {
326 zend_op dummy; /* the size of this structure must be the same as zend_op */
327 struct {
328 const void *orig_handler;
329 const void *call_handler;
330 int16_t *counter;
331 uint8_t trace_flags;
332 };
333 } zend_op_trace_info;
334
335 typedef struct _zend_jit_op_array_trace_extension {
336 zend_func_info func_info;
337 const zend_op_array *op_array;
338 size_t offset; /* offset from "zend_op" to corresponding "op_info" */
339 zend_op_trace_info trace_info[1];
340 } zend_jit_op_array_trace_extension;
341
342 #define ZEND_OP_TRACE_INFO(opline, offset) \
343 ((zend_op_trace_info*)(((char*)opline) + offset))
344
345 /* Recorder */
346 typedef enum _zend_jit_trace_op {
347 ZEND_JIT_TRACE_VM,
348 ZEND_JIT_TRACE_OP1_TYPE,
349 ZEND_JIT_TRACE_OP2_TYPE,
350 ZEND_JIT_TRACE_VAL_INFO,
351 ZEND_JIT_TRACE_INIT_CALL,
352 ZEND_JIT_TRACE_DO_ICALL,
353 ZEND_JIT_TRACE_ENTER,
354 ZEND_JIT_TRACE_BACK,
355 ZEND_JIT_TRACE_END,
356 ZEND_JIT_TRACE_START,
357 } zend_jit_trace_op;
358
359 #define IS_UNKNOWN 255 /* may be used for zend_jit_trace_rec.op?_type */
360 #define IS_TRACE_PACKED (1<<4)
361 #define IS_TRACE_REFERENCE (1<<5)
362 #define IS_TRACE_INDIRECT (1<<6)
363
364 #define IS_TRACE_TYPE_MASK 0xf
365
366 #define ZEND_JIT_TRACE_FAKE_INIT_CALL 0x00000100
367 #define ZEND_JIT_TRACE_RETURN_VALUE_USED 0x00000100
368
369 #define ZEND_JIT_TRACE_MAX_SSA_VAR 0x7ffffe
370 #define ZEND_JIT_TRACE_SSA_VAR_SHIFT 9
371
372 #define ZEND_JIT_TRACE_FAKE_LEVEL_MASK 0xffff0000
373 #define ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT 16
374
375 #define ZEND_JIT_TRACE_FAKE_LEVEL(info) \
376 (((info) & ZEND_JIT_TRACE_FAKE_LEVEL_MASK) >> ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT)
377
378 #define ZEND_JIT_TRACE_FAKE_INFO(level) \
379 (((level) << ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT) | ZEND_JIT_TRACE_FAKE_INIT_CALL)
380
381 #define ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(_info, var) do { \
382 _info |= (var << ZEND_JIT_TRACE_SSA_VAR_SHIFT); \
383 } while (0)
384 #define ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(_info) \
385 (_info >> ZEND_JIT_TRACE_SSA_VAR_SHIFT)
386
387 struct _zend_jit_trace_rec {
388 union {
389 struct { ZEND_ENDIAN_LOHI(
390 uint8_t op, /* zend_jit_trace_op */
391 union {
392 struct {
393 uint8_t op1_type;/* recorded zval op1_type for ZEND_JIT_TRACE_VM */
394 uint8_t op2_type;/* recorded zval op2_type for ZEND_JIT_TRACE_VM */
395 uint8_t op3_type;/* recorded zval for op_data.op1_type for ZEND_JIT_TRACE_VM */
396 };
397 struct {
398 uint8_t start; /* ZEND_JIT_TRACE_START_MASK for ZEND_JIT_TRACE_START/END */
399 uint8_t stop; /* zend_jit_trace_stop for ZEND_JIT_TRACE_START/END */
400 uint8_t level; /* recursive return level for ZEND_JIT_TRACE_START */
401 };
402 })
403 };
404 uint32_t last;
405 uint32_t info; /* "first_ssa_var" for ZEND_JIT_TRACE_ENTER and ZEND_JIT_TRACE_BACK,
406 * "return_value_used" for ZEND_JIT_TRACE_ENTER,
407 * "fake" for ZEND_JIT_TRACE_INIT_CALL */
408 };
409 union {
410 const void *ptr;
411 const zend_function *func;
412 const zend_op_array *op_array;
413 const zend_op *opline;
414 const zend_class_entry *ce;
415 };
416 };
417
418 #define ZEND_JIT_TRACE_START_REC_SIZE 2
419
420 typedef struct _zend_jit_trace_exit_info {
421 const zend_op *opline; /* opline where VM should continue execution */
422 const zend_op_array *op_array;
423 uint32_t flags; /* set of ZEND_JIT_EXIT_... */
424 uint32_t stack_size;
425 uint32_t stack_offset;
426 int32_t poly_func_ref;
427 int32_t poly_this_ref;
428 int8_t poly_func_reg;
429 int8_t poly_this_reg;
430 } zend_jit_trace_exit_info;
431
432 typedef struct _zend_jit_trace_stack {
433 union {
434 uint32_t info;
435 struct {
436 uint8_t type; /* variable type (for type inference) */
437 uint8_t mem_type; /* stack slot type (for eliminate dead type store) */
438 int8_t reg;
439 uint8_t flags;
440 };
441 };
442 int32_t ref;
443 } zend_jit_trace_stack;
444
445 #define STACK_VAR(_stack, _slot) \
446 ((int32_t*)(_stack))[_slot]
447 #define SET_STACK_VAR(_stack, _slot, _ssa_var) do { \
448 ((int32_t*)(_stack))[_slot] = _ssa_var; \
449 } while (0)
450
451 #define CLEAR_STACK_REF(_stack, _slot) do { \
452 (_stack)[_slot].ref = IR_UNUSED; \
453 (_stack)[_slot].flags = 0; \
454 } while (0)
455 #define STACK_REF(_stack, _slot) \
456 (_stack)[_slot].ref
457 #define SET_STACK_REF(_stack, _slot, _ref) do { \
458 (_stack)[_slot].ref = (_ref); \
459 (_stack)[_slot].flags = 0; \
460 } while (0)
461 #define SET_STACK_REF_EX(_stack, _slot, _ref, _flags) do { \
462 (_stack)[_slot].ref = (_ref); \
463 (_stack)[_slot].flags = _flags; \
464 } while (0)
465
466 #define STACK_INFO(_stack, _slot) \
467 (_stack)[_slot].info
468 #define STACK_TYPE(_stack, _slot) \
469 (_stack)[_slot].type
470 #define STACK_MEM_TYPE(_stack, _slot) \
471 (_stack)[_slot].mem_type
472 #define STACK_REG(_stack, _slot) \
473 (_stack)[_slot].reg
474 #define STACK_FLAGS(_stack, _slot) \
475 (_stack)[_slot].flags
476 #define SET_STACK_INFO(_stack, _slot, _info) do { \
477 (_stack)[_slot].info = _info; \
478 } while (0)
479 #define SET_STACK_TYPE(_stack, _slot, _type, _set_mem_type) do { \
480 uint8_t __type = (_type); \
481 (_stack)[_slot].type = __type; \
482 if (_set_mem_type) { \
483 (_stack)[_slot].mem_type = __type; \
484 } \
485 (_stack)[_slot].reg = ZREG_NONE; \
486 (_stack)[_slot].flags = 0; \
487 CLEAR_STACK_REF(_stack, _slot); \
488 } while (0)
489 #define SET_STACK_REG(_stack, _slot, _reg) do { \
490 (_stack)[_slot].reg = _reg; \
491 (_stack)[_slot].flags = 0; \
492 } while (0)
493 #define SET_STACK_REG_EX(_stack, _slot, _reg, _flags) do { \
494 (_stack)[_slot].reg = _reg; \
495 (_stack)[_slot].flags = _flags; \
496 } while (0)
497 #define RESET_STACK_MEM_TYPE(_stack, _slot) do { \
498 (_stack)[_slot].mem_type = IS_UNKNOWN; \
499 } while (0)
500
501 /* trace info flags */
502 #define ZEND_JIT_TRACE_CHECK_INTERRUPT (1<<0)
503 #define ZEND_JIT_TRACE_LOOP (1<<1)
504 #define ZEND_JIT_TRACE_USES_INITIAL_IP (1<<2)
505
506 typedef union _zend_jit_exit_const {
507 int64_t i;
508 double d;
509 } zend_jit_exit_const;
510
511 typedef struct _zend_jit_trace_info {
512 uint32_t id; /* trace id */
513 uint32_t root; /* root trace id or self id for root traces */
514 uint32_t parent; /* parent trace id or 0 for root traces */
515 uint32_t link; /* link trace id or self id for loop) */
516 uint32_t exit_count; /* number of side exits */
517 uint32_t child_count; /* number of side traces for root traces */
518 uint32_t code_size; /* size of native code */
519 uint32_t exit_counters; /* offset in exit counters array */
520 uint32_t stack_map_size;
521 uint32_t flags; /* See ZEND_JIT_TRACE_... defines above */
522 uint32_t polymorphism; /* Counter of polymorphic calls */
523 uint32_t jmp_table_size;/* number of jmp_table slots */
524 const zend_op_array *op_array; /* function */
525 const zend_op *opline; /* first opline */
526 const void *code_start; /* address of native code */
527 zend_jit_trace_exit_info *exit_info; /* info about side exits */
528 zend_jit_trace_stack *stack_map;
529 //uint32_t loop_offset;
530 uint32_t consts_count; /* number of side exits */
531 zend_jit_exit_const *constants;
532 } zend_jit_trace_info;
533
534 struct _zend_jit_trace_stack_frame {
535 zend_jit_trace_stack_frame *call;
536 zend_jit_trace_stack_frame *prev;
537 const zend_function *func;
538 const zend_op *call_opline;
539 zend_class_entry *ce; /* $this */
540 uint32_t call_level;
541 uint32_t _info;
542 int used_stack;
543 int old_checked_stack;
544 int old_peek_checked_stack;
545 zend_jit_trace_stack stack[1];
546 };
547
548 #define TRACE_FRAME_SHIFT_NUM_ARGS 16
549 #define TRACE_FRAME_MAX_NUM_ARGS 32767
550
551 #define TRACE_FRAME_MASK_NUM_ARGS 0xffff0000
552 #define TRACE_FRAME_MASK_NESTED 0x00000001
553 #define TRACE_FRAME_MASK_LAST_SEND_BY_REF 0x00000002
554 #define TRACE_FRAME_MASK_LAST_SEND_BY_VAL 0x00000004
555 #define TRACE_FRAME_MASK_RETURN_VALUE_USED 0x00000008
556 #define TRACE_FRAME_MASK_RETURN_VALUE_UNUSED 0x00000010
557 #define TRACE_FRAME_MASK_THIS_CHECKED 0x00000020
558 #define TRACE_FRAME_MASK_UNKNOWN_RETURN 0x00000040
559 #define TRACE_FRAME_MASK_NO_NEED_RELEASE_THIS 0x00000080
560 #define TRACE_FRAME_MASK_THIS_CLASS_CHECKED 0x00000100
561 #define TRACE_FRAME_MASK_CLOSURE_CALL 0x00000200
562 #define TRACE_FRAME_MASK_ALWAYS_RELEASE_THIS 0x00000400
563
564
565 #define TRACE_FRAME_INIT(frame, _func, _flags, num_args) do { \
566 zend_jit_trace_stack_frame *_frame = (frame); \
567 _frame->call = NULL; \
568 _frame->prev = NULL; \
569 _frame->func = (const zend_function*)_func; \
570 _frame->call_opline = NULL; \
571 _frame->call_level = 0; \
572 _frame->_info = (((uint32_t)(num_args)) << TRACE_FRAME_SHIFT_NUM_ARGS) & TRACE_FRAME_MASK_NUM_ARGS; \
573 _frame->_info |= _flags; \
574 } while (0)
575
576 #define TRACE_FRAME_RETURN_SSA_VAR(frame) \
577 ((int)(frame)->_info)
578 #define TRACE_FRAME_NUM_ARGS(frame) \
579 ((int)((frame)->_info) >> TRACE_FRAME_SHIFT_NUM_ARGS)
580 #define TRACE_FRAME_IS_NESTED(frame) \
581 ((frame)->_info & TRACE_FRAME_MASK_NESTED)
582 #define TRACE_FRAME_IS_LAST_SEND_BY_REF(frame) \
583 ((frame)->_info & TRACE_FRAME_MASK_LAST_SEND_BY_REF)
584 #define TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame) \
585 ((frame)->_info & TRACE_FRAME_MASK_LAST_SEND_BY_VAL)
586 #define TRACE_FRAME_IS_RETURN_VALUE_USED(frame) \
587 ((frame)->_info & TRACE_FRAME_MASK_RETURN_VALUE_USED)
588 #define TRACE_FRAME_IS_RETURN_VALUE_UNUSED(frame) \
589 ((frame)->_info & TRACE_FRAME_MASK_RETURN_VALUE_UNUSED)
590 #define TRACE_FRAME_IS_THIS_CHECKED(frame) \
591 ((frame)->_info & TRACE_FRAME_MASK_THIS_CHECKED)
592 #define TRACE_FRAME_IS_UNKNOWN_RETURN(frame) \
593 ((frame)->_info & TRACE_FRAME_MASK_UNKNOWN_RETURN)
594 #define TRACE_FRAME_NO_NEED_RELEASE_THIS(frame) \
595 ((frame)->_info & TRACE_FRAME_MASK_NO_NEED_RELEASE_THIS)
596 #define TRACE_FRAME_IS_THIS_CLASS_CHECKED(frame) \
597 ((frame)->_info & TRACE_FRAME_MASK_THIS_CLASS_CHECKED)
598 #define TRACE_FRAME_IS_CLOSURE_CALL(frame) \
599 ((frame)->_info & TRACE_FRAME_MASK_CLOSURE_CALL)
600 #define TRACE_FRAME_ALWAYS_RELEASE_THIS(frame) \
601 ((frame)->_info & TRACE_FRAME_MASK_ALWAYS_RELEASE_THIS)
602
603 #define TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(frame) do { \
604 (frame)->_info |= (0xffffu << TRACE_FRAME_SHIFT_NUM_ARGS); \
605 } while (0)
606 #define TRACE_FRAME_SET_RETURN_SSA_VAR(frame, var) do { \
607 (frame)->_info = var; \
608 } while (0)
609 #define TRACE_FRAME_SET_LAST_SEND_BY_REF(frame) do { \
610 (frame)->_info |= TRACE_FRAME_MASK_LAST_SEND_BY_REF; \
611 (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_VAL; \
612 } while (0)
613 #define TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame) do { \
614 (frame)->_info |= TRACE_FRAME_MASK_LAST_SEND_BY_VAL; \
615 (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_REF; \
616 } while (0)
617 #define TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame) do { \
618 (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_VAL; \
619 (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_REF; \
620 } while (0)
621 #define TRACE_FRAME_SET_RETURN_VALUE_USED(frame) do { \
622 (frame)->_info |= TRACE_FRAME_MASK_RETURN_VALUE_USED; \
623 (frame)->_info &= ~TRACE_FRAME_MASK_RETURN_VALUE_UNUSED; \
624 } while (0)
625 #define TRACE_FRAME_SET_RETURN_VALUE_UNUSED(frame) do { \
626 (frame)->_info |= TRACE_FRAME_MASK_RETURN_VALUE_UNUSED; \
627 (frame)->_info &= ~TRACE_FRAME_MASK_RETURN_VALUE_USED; \
628 } while (0)
629 #define TRACE_FRAME_SET_THIS_CHECKED(frame) do { \
630 (frame)->_info |= TRACE_FRAME_MASK_THIS_CHECKED; \
631 } while (0)
632 #define TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(frame) do { \
633 (frame)->_info |= TRACE_FRAME_MASK_NO_NEED_RELEASE_THIS; \
634 } while (0)
635 #define TRACE_FRAME_SET_THIS_CLASS_CHECKED(frame) do { \
636 (frame)->_info |= TRACE_FRAME_MASK_THIS_CLASS_CHECKED; \
637 } while (0)
638 #define TRACE_FRAME_SET_CLOSURE_CALL(frame) do { \
639 (frame)->_info |= TRACE_FRAME_MASK_CLOSURE_CALL; \
640 } while (0)
641 #define TRACE_FRAME_SET_ALWAYS_RELEASE_THIS(frame) do { \
642 (frame)->_info |= TRACE_FRAME_MASK_ALWAYS_RELEASE_THIS; \
643 } while (0)
644
645 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
646 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
647 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
648
649 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline);
650 zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start, uint32_t is_megamorphc);
651
zend_jit_trace_get_exit_opline(zend_jit_trace_rec * trace,const zend_op * opline,bool * exit_if_true)652 static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit_trace_rec *trace, const zend_op *opline, bool *exit_if_true)
653 {
654 if (trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END) {
655 if (trace->opline == opline + 1) {
656 /* not taken branch */
657 *exit_if_true = opline->opcode == ZEND_JMPNZ;
658 return OP_JMP_ADDR(opline, opline->op2);
659 } else if (trace->opline == OP_JMP_ADDR(opline, opline->op2)) {
660 /* taken branch */
661 *exit_if_true = opline->opcode == ZEND_JMPZ;
662 return opline + 1;
663 } else {
664 ZEND_UNREACHABLE();
665 }
666 } else {
667 ZEND_UNREACHABLE();
668 }
669 *exit_if_true = 0;
670 return NULL;
671 }
672
zend_jit_may_be_modified(const zend_function * func,const zend_op_array * called_from)673 static inline bool zend_jit_may_be_modified(const zend_function *func, const zend_op_array *called_from)
674 {
675 if (func->type == ZEND_INTERNAL_FUNCTION) {
676 #ifdef _WIN32
677 /* ASLR */
678 return 1;
679 #else
680 return 0;
681 #endif
682 } else if (func->type == ZEND_USER_FUNCTION) {
683 if (func->common.fn_flags & ZEND_ACC_PRELOADED) {
684 return 0;
685 }
686 if (func->op_array.filename == called_from->filename && !func->op_array.scope) {
687 return 0;
688 }
689 }
690 return 1;
691 }
692
zend_jit_may_be_polymorphic_call(const zend_op * opline)693 static zend_always_inline bool zend_jit_may_be_polymorphic_call(const zend_op *opline)
694 {
695 if (opline->opcode == ZEND_INIT_FCALL
696 || opline->opcode == ZEND_INIT_FCALL_BY_NAME
697 || opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME
698 || opline->opcode == ZEND_INIT_PARENT_PROPERTY_HOOK_CALL) {
699 return 0;
700 } else if (opline->opcode == ZEND_INIT_METHOD_CALL
701 || opline->opcode == ZEND_INIT_DYNAMIC_CALL) {
702 return 1;
703 } else if (opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
704 return (opline->op1_type != IS_CONST || opline->op2_type != IS_CONST);
705 } else if (opline->opcode == ZEND_INIT_USER_CALL) {
706 return (opline->op2_type != IS_CONST);
707 } else if (opline->opcode == ZEND_NEW) {
708 return (opline->op1_type != IS_CONST);
709 } else {
710 ZEND_UNREACHABLE();
711 return 0;
712 }
713 }
714
715 /* Instruction cache flush */
716 #ifndef JIT_CACHE_FLUSH
717 # if ZEND_JIT_TARGET_ARM64
718 # if ((defined(__GNUC__) && ZEND_GCC_VERSION >= 4003) || __has_builtin(__builtin___clear_cache))
719 # define JIT_CACHE_FLUSH(from, to) __builtin___clear_cache((char*)(from), (char*)(to))
720 # else
721 # error "Missing builtin to flush instruction cache for AArch64"
722 # endif
723 # else /* Not required to implement on archs with unified caches */
724 # define JIT_CACHE_FLUSH(from, to)
725 # endif
726 #endif /* !JIT_CACHE_FLUSH */
727
728 /* bit helpers */
729
zend_long_is_power_of_two(zend_long x)730 static zend_always_inline bool zend_long_is_power_of_two(zend_long x)
731 {
732 return (x > 0) && !(x & (x - 1));
733 }
734
735 /* from http://aggregate.org/MAGIC/ */
ones32(uint32_t x)736 static zend_always_inline uint32_t ones32(uint32_t x)
737 {
738 x -= ((x >> 1) & 0x55555555);
739 x = (((x >> 2) & 0x33333333) + (x & 0x33333333));
740 x = (((x >> 4) + x) & 0x0f0f0f0f);
741 x += (x >> 8);
742 x += (x >> 16);
743 return x & 0x0000003f;
744 }
745
floor_log2(uint32_t x)746 static zend_always_inline uint32_t floor_log2(uint32_t x)
747 {
748 ZEND_ASSERT(x != 0);
749 x |= (x >> 1);
750 x |= (x >> 2);
751 x |= (x >> 4);
752 x |= (x >> 8);
753 x |= (x >> 16);
754 return ones32(x) - 1;
755 }
756
is_power_of_two(uint32_t x)757 static zend_always_inline bool is_power_of_two(uint32_t x)
758 {
759 return !(x & (x - 1)) && x != 0;
760 }
761
has_concrete_type(uint32_t value_type)762 static zend_always_inline bool has_concrete_type(uint32_t value_type)
763 {
764 return is_power_of_two (value_type & (MAY_BE_ANY|MAY_BE_UNDEF));
765 }
766
concrete_type(uint32_t value_type)767 static zend_always_inline uint32_t concrete_type(uint32_t value_type)
768 {
769 return floor_log2(value_type & (MAY_BE_ANY|MAY_BE_UNDEF));
770 }
771
is_signed(double d)772 static zend_always_inline bool is_signed(double d)
773 {
774 return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0;
775 }
776
777 #endif /* ZEND_JIT_INTERNAL_H */
778
