1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 | Xinchen Hui <laruence@php.net> |
17 +----------------------------------------------------------------------+
18 */
19
20 #ifndef ZEND_JIT_INTERNAL_H
21 #define ZEND_JIT_INTERNAL_H
22
23 typedef struct _zend_jit_op_array_extension {
24 zend_func_info func_info;
25 const void *orig_handler;
26 } zend_jit_op_array_extension;
27
28 /* Profiler */
29 extern zend_ulong zend_jit_profile_counter;
30 extern int zend_jit_profile_counter_rid;
31
32 #define ZEND_COUNTER_INFO(op_array) \
33 ZEND_OP_ARRAY_EXTENSION(op_array, zend_jit_profile_counter_rid)
34
35 /* Hot Counters */
36
37 #define ZEND_HOT_COUNTERS_COUNT 128
38
39 extern int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
40
zend_jit_hash(const void * ptr)41 static zend_always_inline zend_long zend_jit_hash(const void *ptr)
42 {
43 uintptr_t x;
44
45 x = (uintptr_t)ptr >> 3;
46 #if SIZEOF_SIZE_T == 4
47 x = ((x >> 16) ^ x) * 0x45d9f3b;
48 x = ((x >> 16) ^ x) * 0x45d9f3b;
49 x = (x >> 16) ^ x;
50 #elif SIZEOF_SIZE_T == 8
51 x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
52 x = (x ^ (x >> 27)) * 0x94d049bb133111eb;
53 x = x ^ (x >> 31);
54 #endif
55 return x;
56 }
57
58 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline);
59
60 typedef struct _zend_jit_op_array_hot_extension {
61 zend_func_info func_info;
62 int16_t *counter;
63 const void *orig_handlers[1];
64 } zend_jit_op_array_hot_extension;
65
66 #define zend_jit_op_array_hash(op_array) \
67 zend_jit_hash((op_array)->opcodes)
68
69 extern const zend_op *zend_jit_halt_op;
70
71 #ifdef HAVE_GCC_GLOBAL_REGS
72 # define EXECUTE_DATA_D void
73 # define EXECUTE_DATA_C
74 # define EXECUTE_DATA_DC
75 # define EXECUTE_DATA_CC
76 # define OPLINE_D void
77 # define OPLINE_C
78 # define OPLINE_DC
79 # define OPLINE_CC
80 # define ZEND_OPCODE_HANDLER_RET void
81 # define ZEND_OPCODE_HANDLER_ARGS EXECUTE_DATA_D
82 # define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU
83 # define ZEND_OPCODE_HANDLER_ARGS_DC
84 # define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC
85 # define ZEND_OPCODE_RETURN() return
86 # define ZEND_OPCODE_TAIL_CALL(handler) do { \
87 handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
88 return; \
89 } while(0)
90 # define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \
91 handler(arg ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC); \
92 return; \
93 } while(0)
94 #else
95 # define EXECUTE_DATA_D zend_execute_data* execute_data
96 # define EXECUTE_DATA_C execute_data
97 # define EXECUTE_DATA_DC , EXECUTE_DATA_D
98 # define EXECUTE_DATA_CC , EXECUTE_DATA_C
99 # define OPLINE_D const zend_op* opline
100 # define OPLINE_C opline
101 # define OPLINE_DC , OPLINE_D
102 # define OPLINE_CC , OPLINE_C
103 # define ZEND_OPCODE_HANDLER_RET int
104 # define ZEND_OPCODE_HANDLER_ARGS EXECUTE_DATA_D
105 # define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU EXECUTE_DATA_C
106 # define ZEND_OPCODE_HANDLER_ARGS_DC EXECUTE_DATA_DC
107 # define ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC EXECUTE_DATA_CC
108 # define ZEND_OPCODE_RETURN() return 0
109 # define ZEND_OPCODE_TAIL_CALL(handler) do { \
110 return handler(ZEND_OPCODE_HANDLER_ARGS_PASSTHRU); \
111 } while(0)
112 # define ZEND_OPCODE_TAIL_CALL_EX(handler, arg) do { \
113 return handler(arg ZEND_OPCODE_HANDLER_ARGS_PASSTHRU_CC); \
114 } while(0)
115 #endif
116
117 /* VM handlers */
118 typedef ZEND_OPCODE_HANDLER_RET (ZEND_FASTCALL *zend_vm_opcode_handler_t)(ZEND_OPCODE_HANDLER_ARGS);
119
120 /* VM helpers */
121 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_nested_func_helper(uint32_t call_info EXECUTE_DATA_DC);
122 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_top_func_helper(uint32_t call_info EXECUTE_DATA_DC);
123 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_leave_func_helper(uint32_t call_info EXECUTE_DATA_DC);
124
125 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_profile_helper(ZEND_OPCODE_HANDLER_ARGS);
126
127 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_counter_helper(ZEND_OPCODE_HANDLER_ARGS);
128 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_counter_helper(ZEND_OPCODE_HANDLER_ARGS);
129
130 void ZEND_FASTCALL zend_jit_copy_extra_args_helper(EXECUTE_DATA_D);
131 zend_bool ZEND_FASTCALL zend_jit_deprecated_helper(OPLINE_D);
132
133 zend_constant* ZEND_FASTCALL zend_jit_get_constant(const zval *key, uint32_t flags);
134 zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key);
135
136 /* Tracer */
137 #define zend_jit_opline_hash(opline) \
138 zend_jit_hash(opline)
139
140 #define ZEND_JIT_TRACE_STOP(_) \
141 _(LOOP, "loop") \
142 _(RECURSIVE_CALL, "recursive call") \
143 _(RECURSIVE_RET, "recursive return") \
144 _(RETURN, "return") \
145 _(INTERPRETER, "exit to VM interpreter") \
146 _(LINK, "link to another trace") \
147 /* compilation and linking successful */ \
148 _(COMPILED, "compiled") \
149 _(ALREADY_DONE, "already prcessed") \
150 /* failures */ \
151 _(ERROR, "error") /* not used */ \
152 _(NOT_SUPPORTED, "not supported instructions") \
153 _(EXCEPTION, "exception") \
154 _(TOO_LONG, "trace too long") \
155 _(TOO_DEEP, "trace too deep") \
156 _(TOO_DEEP_RET, "trace too deep return") \
157 _(DEEP_RECURSION, "deep recursion") \
158 _(LOOP_UNROLL, "loop unroll limit reached") \
159 _(LOOP_EXIT, "exit from loop") \
160 _(RECURSION_EXIT, "return from recursive function") \
161 _(BLACK_LIST, "trace blacklisted") \
162 _(INNER_LOOP, "inner loop") /* trace it */ \
163 _(COMPILED_LOOP, "compiled loop") \
164 _(TRAMPOLINE, "trampoline call") \
165 _(BAD_FUNC, "bad function call") \
166 _(COMPILER_ERROR, "JIT compilation error") \
167 /* no recoverable error (blacklist immediately) */ \
168 _(NO_SHM, "insufficient shared memory") \
169 _(TOO_MANY_TRACES, "too many traces") \
170 _(TOO_MANY_CHILDREN, "too many side traces") \
171 _(TOO_MANY_EXITS, "too many side exits") \
172
173 #define ZEND_JIT_TRACE_STOP_NAME(name, description) \
174 ZEND_JIT_TRACE_STOP_ ## name,
175
176 typedef enum _zend_jit_trace_stop {
177 ZEND_JIT_TRACE_STOP(ZEND_JIT_TRACE_STOP_NAME)
178 ZEND_JIT_TRACE_HALT = 0x40
179 } zend_jit_trace_stop;
180
181 #define ZEND_JIT_TRACE_STOP_OK(ret) \
182 (ret < ZEND_JIT_TRACE_STOP_COMPILED)
183
184 #define ZEND_JIT_TRACE_STOP_DONE(ret) \
185 (ret < ZEND_JIT_TRACE_STOP_ERROR)
186
187 #define ZEND_JIT_TRACE_STOP_REPEAT(ret) \
188 (ret == ZEND_JIT_TRACE_STOP_INNER_LOOP)
189
190 #define ZEND_JIT_TRACE_STOP_MAY_RECOVER(ret) \
191 (ret <= ZEND_JIT_TRACE_STOP_COMPILER_ERROR)
192
193 #define ZEND_JIT_TRACE_START_MASK 0xf
194
195 #define ZEND_JIT_TRACE_START_LOOP (1<<0)
196 #define ZEND_JIT_TRACE_START_ENTER (1<<1)
197 #define ZEND_JIT_TRACE_START_RETURN (1<<2)
198 #define ZEND_JIT_TRACE_START_SIDE (1<<3) /* used for side traces */
199
200 #define ZEND_JIT_TRACE_JITED (1<<4)
201 #define ZEND_JIT_TRACE_BLACKLISTED (1<<5)
202 #define ZEND_JIT_TRACE_UNSUPPORTED (1<<6)
203
204 #define ZEND_JIT_TRACE_SUPPORTED 0
205
206 #define ZEND_JIT_EXIT_JITED (1<<0)
207 #define ZEND_JIT_EXIT_BLACKLISTED (1<<1)
208 #define ZEND_JIT_EXIT_TO_VM (1<<2) /* exit to VM without attempt to create a side trace */
209 #define ZEND_JIT_EXIT_RESTORE_CALL (1<<3) /* deoptimizer should restore EX(call) chain */
210 #define ZEND_JIT_EXIT_POLYMORPHISM (1<<4) /* exit because of polymorphic call */
211 #define ZEND_JIT_EXIT_FREE_OP1 (1<<5)
212 #define ZEND_JIT_EXIT_FREE_OP2 (1<<6)
213 #define ZEND_JIT_EXIT_PACKED_GUARD (1<<7)
214 #define ZEND_JIT_EXIT_CLOSURE_CALL (1<<8) /* exit because of polymorphic INIT_DYNAMIC_CALL call */
215 #define ZEND_JIT_EXIT_METHOD_CALL (1<<9) /* exit because of polymorphic INIT_METHOD_CALL call */
216
217 typedef union _zend_op_trace_info {
218 zend_op dummy; /* the size of this structure must be the same as zend_op */
219 struct {
220 const void *orig_handler;
221 const void *call_handler;
222 int16_t *counter;
223 uint8_t trace_flags;
224 };
225 } zend_op_trace_info;
226
227 typedef struct _zend_jit_op_array_trace_extension {
228 zend_func_info func_info;
229 const zend_op_array *op_array;
230 size_t offset; /* offset from "zend_op" to corresponding "op_info" */
231 zend_op_trace_info trace_info[1];
232 } zend_jit_op_array_trace_extension;
233
234 #define ZEND_OP_TRACE_INFO(opline, offset) \
235 ((zend_op_trace_info*)(((char*)opline) + offset))
236
237 /* Recorder */
238 typedef enum _zend_jit_trace_op {
239 ZEND_JIT_TRACE_VM,
240 ZEND_JIT_TRACE_OP1_TYPE,
241 ZEND_JIT_TRACE_OP2_TYPE,
242 ZEND_JIT_TRACE_INIT_CALL,
243 ZEND_JIT_TRACE_DO_ICALL,
244 ZEND_JIT_TRACE_ENTER,
245 ZEND_JIT_TRACE_BACK,
246 ZEND_JIT_TRACE_END,
247 ZEND_JIT_TRACE_START,
248 } zend_jit_trace_op;
249
250 #define IS_UNKNOWN 255 /* may be used for zend_jit_trace_rec.op?_type */
251 #define IS_TRACE_PACKED (1<<4)
252 #define IS_TRACE_REFERENCE (1<<5)
253 #define IS_TRACE_INDIRECT (1<<6)
254
255 #define ZEND_JIT_TRACE_FAKE_INIT_CALL 0x00000100
256 #define ZEND_JIT_TRACE_RETRUN_VALUE_USED 0x00000100
257
258 #define ZEND_JIT_TRACE_MAX_SSA_VAR 0x7ffffe
259 #define ZEND_JIT_TRACE_SSA_VAR_SHIFT 9
260
261 #define ZEND_JIT_TRACE_FAKE_LEVEL_MASK 0xffff0000
262 #define ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT 16
263
264 #define ZEND_JIT_TRACE_FAKE_LEVEL(info) \
265 (((info) & ZEND_JIT_TRACE_FAKE_LEVEL_MASK) >> ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT)
266
267 #define ZEND_JIT_TRACE_FAKE_INFO(level) \
268 (((level) << ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT) | ZEND_JIT_TRACE_FAKE_INIT_CALL)
269
270 #define ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(_info, var) do { \
271 _info |= (var << ZEND_JIT_TRACE_SSA_VAR_SHIFT); \
272 } while (0)
273 #define ZEND_JIT_TRACE_GET_FIRST_SSA_VAR(_info) \
274 (_info >> ZEND_JIT_TRACE_SSA_VAR_SHIFT)
275
276 struct _zend_jit_trace_rec {
277 union {
278 struct { ZEND_ENDIAN_LOHI(
279 uint8_t op, /* zend_jit_trace_op */
280 union {
281 struct {
282 uint8_t op1_type;/* recorded zval op1_type for ZEND_JIT_TRACE_VM */
283 uint8_t op2_type;/* recorded zval op2_type for ZEND_JIT_TRACE_VM */
284 uint8_t op3_type;/* recorded zval for op_data.op1_type for ZEND_JIT_TRACE_VM */
285 };
286 struct {
287 uint8_t start; /* ZEND_JIT_TRACE_START_MASK for ZEND_JIT_TRACE_START/END */
288 uint8_t stop; /* zend_jit_trace_stop for ZEND_JIT_TRACE_START/END */
289 uint8_t level; /* recursive return level for ZEND_JIT_TRACE_START */
290 };
291 })
292 };
293 uint32_t last;
294 uint32_t info; /* "first_ssa_var" for ZEND_JIT_TRACE_ENTER and ZEND_JIT_TRACE_BACK,
295 * "return_value_used" for ZEND_JIT_TRACE_ENTER,
296 * "fake" for ZEND_JIT_TRACE_INIT_CALL */
297 };
298 union {
299 const void *ptr;
300 const zend_function *func;
301 const zend_op_array *op_array;
302 const zend_op *opline;
303 const zend_class_entry *ce;
304 };
305 };
306
307 #define ZEND_JIT_TRACE_START_REC_SIZE 2
308
309 typedef struct _zend_jit_trace_exit_info {
310 const zend_op *opline; /* opline where VM should continue execution */
311 const zend_op_array *op_array;
312 uint32_t flags; /* set of ZEND_JIT_EXIT_... */
313 uint32_t stack_size;
314 uint32_t stack_offset;
315 } zend_jit_trace_exit_info;
316
317 typedef union _zend_jit_trace_stack {
318 int32_t ssa_var;
319 uint32_t info;
320 struct {
321 uint8_t type; /* variable type (for type inference) */
322 uint8_t mem_type; /* stack slot type (for eliminate dead type store) */
323 int8_t reg;
324 uint8_t flags;
325 };
326 } zend_jit_trace_stack;
327
328 #define STACK_VAR(_stack, _slot) \
329 (_stack)[_slot].ssa_var
330 #define STACK_INFO(_stack, _slot) \
331 (_stack)[_slot].info
332 #define STACK_TYPE(_stack, _slot) \
333 (_stack)[_slot].type
334 #define STACK_MEM_TYPE(_stack, _slot) \
335 (_stack)[_slot].mem_type
336 #define STACK_REG(_stack, _slot) \
337 (_stack)[_slot].reg
338 #define STACK_FLAGS(_stack, _slot) \
339 (_stack)[_slot].flags
340 #define SET_STACK_VAR(_stack, _slot, _ssa_var) do { \
341 (_stack)[_slot].ssa_var = _ssa_var; \
342 } while (0)
343 #define SET_STACK_INFO(_stack, _slot, _info) do { \
344 (_stack)[_slot].info = _info; \
345 } while (0)
346 #define SET_STACK_TYPE(_stack, _slot, _type, _set_mem_type) do { \
347 uint8_t __type = (_type); \
348 (_stack)[_slot].type = __type; \
349 if (_set_mem_type) { \
350 (_stack)[_slot].mem_type = __type; \
351 } \
352 (_stack)[_slot].reg = ZREG_NONE; \
353 (_stack)[_slot].flags = 0; \
354 } while (0)
355 #define SET_STACK_REG(_stack, _slot, _reg) do { \
356 (_stack)[_slot].reg = _reg; \
357 (_stack)[_slot].flags = 0; \
358 } while (0)
359 #define SET_STACK_REG_EX(_stack, _slot, _reg, _flags) do { \
360 (_stack)[_slot].reg = _reg; \
361 (_stack)[_slot].flags = _flags; \
362 } while (0)
363 #define RESET_STACK_MEM_TYPE(_stack, _slot) do { \
364 (_stack)[_slot].mem_type = IS_UNKNOWN; \
365 } while (0)
366
367 /* trace info flags */
368 #define ZEND_JIT_TRACE_CHECK_INTERRUPT (1<<0)
369 #define ZEND_JIT_TRACE_LOOP (1<<1)
370 #define ZEND_JIT_TRACE_USES_INITIAL_IP (1<<2)
371
372 typedef struct _zend_jit_trace_info {
373 uint32_t id; /* trace id */
374 uint32_t root; /* root trace id or self id for root traces */
375 uint32_t parent; /* parent trace id or 0 for root traces */
376 uint32_t link; /* link trace id or self id for loop) */
377 uint32_t exit_count; /* number of side exits */
378 uint32_t child_count; /* number of side traces for root traces */
379 uint32_t code_size; /* size of native code */
380 uint32_t exit_counters; /* offset in exit counters array */
381 uint32_t stack_map_size;
382 uint32_t flags; /* See ZEND_JIT_TRACE_... defines above */
383 uint32_t polymorphism; /* Counter of polymorphic calls */
384 uint32_t jmp_table_size;/* number of jmp_table slots */
385 const zend_op *opline; /* first opline */
386 const void *code_start; /* address of native code */
387 zend_jit_trace_exit_info *exit_info; /* info about side exits */
388 zend_jit_trace_stack *stack_map;
389 //uint32_t loop_offset;
390 } zend_jit_trace_info;
391
392 struct _zend_jit_trace_stack_frame {
393 zend_jit_trace_stack_frame *call;
394 zend_jit_trace_stack_frame *prev;
395 const zend_function *func;
396 const zend_op *call_opline;
397 uint32_t call_level;
398 uint32_t _info;
399 int used_stack;
400 zend_jit_trace_stack stack[1];
401 };
402
403 #define TRACE_FRAME_SHIFT_NUM_ARGS 16
404 #define TRACE_FRAME_MAX_NUM_ARGS 32767
405
406 #define TRACE_FRAME_MASK_NUM_ARGS 0xffff0000
407 #define TRACE_FRAME_MASK_NESTED 0x00000001
408 #define TRACE_FRAME_MASK_LAST_SEND_BY_REF 0x00000002
409 #define TRACE_FRAME_MASK_LAST_SEND_BY_VAL 0x00000004
410 #define TRACE_FRAME_MASK_RETURN_VALUE_USED 0x00000008
411 #define TRACE_FRAME_MASK_RETURN_VALUE_UNUSED 0x00000010
412 #define TRACE_FRAME_MASK_THIS_CHECKED 0x00000020
413 #define TRACE_FRAME_MASK_UNKNOWN_RETURN 0x00000040
414 #define TRACE_FRAME_MASK_NO_NEED_RELEASE_THIS 0x00000080
415
416
417 #define TRACE_FRAME_INIT(frame, _func, _flags, num_args) do { \
418 zend_jit_trace_stack_frame *_frame = (frame); \
419 _frame->call = NULL; \
420 _frame->prev = NULL; \
421 _frame->func = (const zend_function*)_func; \
422 _frame->call_opline = NULL; \
423 _frame->call_level = 0; \
424 _frame->_info = (((uint32_t)(num_args)) << TRACE_FRAME_SHIFT_NUM_ARGS) & TRACE_FRAME_MASK_NUM_ARGS; \
425 _frame->_info |= _flags; \
426 } while (0)
427
428 #define TRACE_FRAME_RETURN_SSA_VAR(frame) \
429 ((int)(frame)->_info)
430 #define TRACE_FRAME_NUM_ARGS(frame) \
431 ((int)((frame)->_info) >> TRACE_FRAME_SHIFT_NUM_ARGS)
432 #define TRACE_FRAME_IS_NESTED(frame) \
433 ((frame)->_info & TRACE_FRAME_MASK_NESTED)
434 #define TRACE_FRAME_IS_LAST_SEND_BY_REF(frame) \
435 ((frame)->_info & TRACE_FRAME_MASK_LAST_SEND_BY_REF)
436 #define TRACE_FRAME_IS_LAST_SEND_BY_VAL(frame) \
437 ((frame)->_info & TRACE_FRAME_MASK_LAST_SEND_BY_VAL)
438 #define TRACE_FRAME_IS_RETURN_VALUE_USED(frame) \
439 ((frame)->_info & TRACE_FRAME_MASK_RETURN_VALUE_USED)
440 #define TRACE_FRAME_IS_RETURN_VALUE_UNUSED(frame) \
441 ((frame)->_info & TRACE_FRAME_MASK_RETURN_VALUE_UNUSED)
442 #define TRACE_FRAME_IS_THIS_CHECKED(frame) \
443 ((frame)->_info & TRACE_FRAME_MASK_THIS_CHECKED)
444 #define TRACE_FRAME_IS_UNKNOWN_RETURN(frame) \
445 ((frame)->_info & TRACE_FRAME_MASK_UNKNOWN_RETURN)
446 #define TRACE_FRAME_NO_NEED_RELEASE_THIS(frame) \
447 ((frame)->_info & TRACE_FRAME_MASK_NO_NEED_RELEASE_THIS)
448
449 #define TRACE_FRAME_SET_UNKNOWN_NUM_ARGS(frame) do { \
450 (frame)->_info |= (0xffffu << TRACE_FRAME_SHIFT_NUM_ARGS); \
451 } while (0)
452 #define TRACE_FRAME_SET_RETURN_SSA_VAR(frame, var) do { \
453 (frame)->_info = var; \
454 } while (0)
455 #define TRACE_FRAME_SET_LAST_SEND_BY_REF(frame) do { \
456 (frame)->_info |= TRACE_FRAME_MASK_LAST_SEND_BY_REF; \
457 (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_VAL; \
458 } while (0)
459 #define TRACE_FRAME_SET_LAST_SEND_BY_VAL(frame) do { \
460 (frame)->_info |= TRACE_FRAME_MASK_LAST_SEND_BY_VAL; \
461 (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_REF; \
462 } while (0)
463 #define TRACE_FRAME_SET_LAST_SEND_UNKNOWN(frame) do { \
464 (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_VAL; \
465 (frame)->_info &= ~TRACE_FRAME_MASK_LAST_SEND_BY_REF; \
466 } while (0)
467 #define TRACE_FRAME_SET_RETURN_VALUE_USED(frame) do { \
468 (frame)->_info |= TRACE_FRAME_MASK_RETURN_VALUE_USED; \
469 (frame)->_info &= ~TRACE_FRAME_MASK_RETURN_VALUE_UNUSED; \
470 } while (0)
471 #define TRACE_FRAME_SET_RETURN_VALUE_UNUSED(frame) do { \
472 (frame)->_info |= TRACE_FRAME_MASK_RETURN_VALUE_UNUSED; \
473 (frame)->_info &= ~TRACE_FRAME_MASK_RETURN_VALUE_USED; \
474 } while (0)
475 #define TRACE_FRAME_SET_THIS_CHECKED(frame) do { \
476 (frame)->_info |= TRACE_FRAME_MASK_THIS_CHECKED; \
477 } while (0)
478 #define TRACE_FRAME_SET_NO_NEED_RELEASE_THIS(frame) do { \
479 (frame)->_info |= TRACE_FRAME_MASK_NO_NEED_RELEASE_THIS; \
480 } while (0)
481
482 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_func_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
483 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_ret_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
484 ZEND_OPCODE_HANDLER_RET ZEND_FASTCALL zend_jit_loop_trace_helper(ZEND_OPCODE_HANDLER_ARGS);
485
486 int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const zend_op *opline);
487 int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs);
488 zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *execute_data, const zend_op *opline, zend_jit_trace_rec *trace_buffer, uint8_t start, uint32_t is_megamorphc);
489
zend_jit_trace_get_exit_opline(zend_jit_trace_rec * trace,const zend_op * opline,zend_bool * exit_if_true)490 static zend_always_inline const zend_op* zend_jit_trace_get_exit_opline(zend_jit_trace_rec *trace, const zend_op *opline, zend_bool *exit_if_true)
491 {
492 if (trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END) {
493 if (trace->opline == opline + 1) {
494 /* not taken branch */
495 *exit_if_true = opline->opcode == ZEND_JMPNZ;
496 return OP_JMP_ADDR(opline, opline->op2);
497 } else if (trace->opline == OP_JMP_ADDR(opline, opline->op2)) {
498 /* taken branch */
499 *exit_if_true = opline->opcode == ZEND_JMPZ;
500 return opline + 1;
501 } else {
502 ZEND_UNREACHABLE();
503 }
504 } else {
505 ZEND_UNREACHABLE();
506 }
507 *exit_if_true = 0;
508 return NULL;
509 }
510
zend_jit_may_be_polymorphic_call(const zend_op * opline)511 static zend_always_inline zend_bool zend_jit_may_be_polymorphic_call(const zend_op *opline)
512 {
513 if (opline->opcode == ZEND_INIT_FCALL
514 || opline->opcode == ZEND_INIT_FCALL_BY_NAME
515 || opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) {
516 return 0;
517 } else if (opline->opcode == ZEND_INIT_METHOD_CALL
518 || opline->opcode == ZEND_INIT_DYNAMIC_CALL) {
519 return 1;
520 } else if (opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
521 return (opline->op1_type != IS_CONST || opline->op2_type != IS_CONST);
522 } else if (opline->opcode == ZEND_INIT_USER_CALL) {
523 return (opline->op2_type != IS_CONST);
524 } else if (opline->opcode == ZEND_NEW) {
525 return (opline->op1_type != IS_CONST);
526 } else {
527 ZEND_UNREACHABLE();
528 return 0;
529 }
530 }
531
532 #endif /* ZEND_JIT_INTERNAL_H */
533
