1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 #include "main/php.h"
20 #include "main/SAPI.h"
21 #include "php_version.h"
22 #include <ZendAccelerator.h>
23 #include "zend_shared_alloc.h"
24 #include "Zend/zend_execute.h"
25 #include "Zend/zend_vm.h"
26 #include "Zend/zend_exceptions.h"
27 #include "Zend/zend_constants.h"
28 #include "Zend/zend_closures.h"
29 #include "Zend/zend_ini.h"
30 #include "Zend/zend_observer.h"
31 #include "zend_smart_str.h"
32 #include "jit/zend_jit.h"
33
34 #ifdef HAVE_JIT
35
36 #include "Optimizer/zend_func_info.h"
37 #include "Optimizer/zend_ssa.h"
38 #include "Optimizer/zend_inference.h"
39 #include "Optimizer/zend_call_graph.h"
40 #include "Optimizer/zend_dump.h"
41
42 #if ZEND_JIT_TARGET_X86
43 # include "jit/zend_jit_x86.h"
44 #elif ZEND_JIT_TARGET_ARM64
45 # include "jit/zend_jit_arm64.h"
46 #endif
47
48 #include "jit/zend_jit_internal.h"
49
50 #ifdef ZTS
51 int jit_globals_id;
52 #else
53 zend_jit_globals jit_globals;
54 #endif
55
56 //#define CONTEXT_THREADED_JIT
57 #define ZEND_JIT_USE_RC_INFERENCE
58
59 #ifdef ZEND_JIT_USE_RC_INFERENCE
60 # define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE
61 # define RC_MAY_BE_1(info) (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0)
62 # define RC_MAY_BE_N(info) (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0)
63 #else
64 # define ZEND_SSA_RC_INFERENCE_FLAG 0
65 # define RC_MAY_BE_1(info) 1
66 # define RC_MAY_BE_N(info) 1
67 #endif
68
69 #define JIT_PREFIX "JIT$"
70 #define JIT_STUB_PREFIX "JIT$$"
71 #define TRACE_PREFIX "TRACE-"
72
73 #define DASM_M_GROW(ctx, t, p, sz, need) \
74 do { \
75 size_t _sz = (sz), _need = (need); \
76 if (_sz < _need) { \
77 if (_sz < 16) _sz = 16; \
78 while (_sz < _need) _sz += _sz; \
79 (p) = (t *)erealloc((p), _sz); \
80 (sz) = _sz; \
81 } \
82 } while(0)
83
84 #define DASM_M_FREE(ctx, p, sz) efree(p)
85
86 #if ZEND_DEBUG
87 # define DASM_CHECKS 1
88 #endif
89
90 #include "dynasm/dasm_proto.h"
91
92 typedef struct _zend_jit_stub {
93 const char *name;
94 int (*stub)(dasm_State **Dst);
95 uint32_t offset;
96 uint32_t adjustment;
97 } zend_jit_stub;
98
99 #define JIT_STUB(name, offset, adjustment) \
100 {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub, offset, adjustment}
101
102 zend_ulong zend_jit_profile_counter = 0;
103 int zend_jit_profile_counter_rid = -1;
104
105 int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
106
107 const zend_op *zend_jit_halt_op = NULL;
108 static int zend_jit_vm_kind = 0;
109
110 static void *dasm_buf = NULL;
111 static void *dasm_end = NULL;
112 static void **dasm_ptr = NULL;
113
114 static size_t dasm_size = 0;
115
116 static zend_long jit_bisect_pos = 0;
117
118 static const void *zend_jit_runtime_jit_handler = NULL;
119 static const void *zend_jit_profile_jit_handler = NULL;
120 static const void *zend_jit_func_hot_counter_handler = NULL;
121 static const void *zend_jit_loop_hot_counter_handler = NULL;
122 static const void *zend_jit_func_trace_counter_handler = NULL;
123 static const void *zend_jit_ret_trace_counter_handler = NULL;
124 static const void *zend_jit_loop_trace_counter_handler = NULL;
125
126 static int ZEND_FASTCALL zend_runtime_jit(void);
127
128 static int zend_jit_trace_op_len(const zend_op *opline);
129 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
130 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags);
131 static const void *zend_jit_trace_get_exit_addr(uint32_t n);
132 static void zend_jit_trace_add_code(const void *start, uint32_t size);
133 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info);
134
135 #if ZEND_JIT_TARGET_ARM64
136 static zend_jit_trace_info *zend_jit_get_current_trace_info(void);
137 static uint32_t zend_jit_trace_find_exit_point(const void* addr);
138 #endif
139
140 static int zend_jit_assign_to_variable(dasm_State **Dst,
141 const zend_op *opline,
142 zend_jit_addr var_use_addr,
143 zend_jit_addr var_addr,
144 uint32_t var_info,
145 uint32_t var_def_info,
146 zend_uchar val_type,
147 zend_jit_addr val_addr,
148 uint32_t val_info,
149 zend_jit_addr res_addr,
150 bool check_exception);
151
dominates(const zend_basic_block * blocks,int a,int b)152 static bool dominates(const zend_basic_block *blocks, int a, int b) {
153 while (blocks[b].level > blocks[a].level) {
154 b = blocks[b].idom;
155 }
156 return a == b;
157 }
158
zend_ssa_is_last_use(const zend_op_array * op_array,const zend_ssa * ssa,int var,int use)159 static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
160 {
161 int next_use;
162
163 if (ssa->vars[var].phi_use_chain) {
164 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
165 do {
166 if (!ssa->vars[phi->ssa_var].no_val) {
167 return 0;
168 }
169 phi = zend_ssa_next_use_phi(ssa, var, phi);
170 } while (phi);
171 }
172
173 if (ssa->cfg.blocks[ssa->cfg.map[use]].loop_header > 0
174 || (ssa->cfg.blocks[ssa->cfg.map[use]].flags & ZEND_BB_LOOP_HEADER)) {
175 int b = ssa->cfg.map[use];
176 int prev_use = ssa->vars[var].use_chain;
177
178 while (prev_use >= 0 && prev_use != use) {
179 if (b != ssa->cfg.map[prev_use]
180 && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use])
181 && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) {
182 return 0;
183 }
184 prev_use = zend_ssa_next_use(ssa->ops, var, prev_use);
185 }
186 }
187
188 next_use = zend_ssa_next_use(ssa->ops, var, use);
189 if (next_use < 0) {
190 return 1;
191 } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) {
192 return 1;
193 }
194 return 0;
195 }
196
zend_ival_is_last_use(const zend_lifetime_interval * ival,int use)197 static bool zend_ival_is_last_use(const zend_lifetime_interval *ival, int use)
198 {
199 if (ival->flags & ZREG_LAST_USE) {
200 const zend_life_range *range = &ival->range;
201
202 while (range->next) {
203 range = range->next;
204 }
205 return range->end == use;
206 }
207 return 0;
208 }
209
zend_is_commutative(zend_uchar opcode)210 static bool zend_is_commutative(zend_uchar opcode)
211 {
212 return
213 opcode == ZEND_ADD ||
214 opcode == ZEND_MUL ||
215 opcode == ZEND_BW_OR ||
216 opcode == ZEND_BW_AND ||
217 opcode == ZEND_BW_XOR;
218 }
219
zend_jit_is_constant_cmp_long_long(const zend_op * opline,zend_ssa_range * op1_range,zend_jit_addr op1_addr,zend_ssa_range * op2_range,zend_jit_addr op2_addr,bool * result)220 static int zend_jit_is_constant_cmp_long_long(const zend_op *opline,
221 zend_ssa_range *op1_range,
222 zend_jit_addr op1_addr,
223 zend_ssa_range *op2_range,
224 zend_jit_addr op2_addr,
225 bool *result)
226 {
227 zend_long op1_min;
228 zend_long op1_max;
229 zend_long op2_min;
230 zend_long op2_max;
231
232 if (op1_range) {
233 op1_min = op1_range->min;
234 op1_max = op1_range->max;
235 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
236 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG);
237 op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr));
238 } else {
239 return 0;
240 }
241
242 if (op2_range) {
243 op2_min = op2_range->min;
244 op2_max = op2_range->max;
245 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
246 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG);
247 op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr));
248 } else {
249 return 0;
250 }
251
252 switch (opline->opcode) {
253 case ZEND_IS_EQUAL:
254 case ZEND_IS_IDENTICAL:
255 case ZEND_CASE:
256 case ZEND_CASE_STRICT:
257 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
258 *result = 1;
259 return 1;
260 } else if (op1_max < op2_min || op1_min > op2_max) {
261 *result = 0;
262 return 1;
263 }
264 return 0;
265 case ZEND_IS_NOT_EQUAL:
266 case ZEND_IS_NOT_IDENTICAL:
267 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
268 *result = 0;
269 return 1;
270 } else if (op1_max < op2_min || op1_min > op2_max) {
271 *result = 1;
272 return 1;
273 }
274 return 0;
275 case ZEND_IS_SMALLER:
276 if (op1_max < op2_min) {
277 *result = 1;
278 return 1;
279 } else if (op1_min >= op2_max) {
280 *result = 0;
281 return 1;
282 }
283 return 0;
284 case ZEND_IS_SMALLER_OR_EQUAL:
285 if (op1_max <= op2_min) {
286 *result = 1;
287 return 1;
288 } else if (op1_min > op2_max) {
289 *result = 0;
290 return 1;
291 }
292 return 0;
293 default:
294 ZEND_UNREACHABLE();
295 }
296 return 0;
297 }
298
zend_jit_needs_call_chain(zend_call_info * call_info,uint32_t b,const zend_op_array * op_array,zend_ssa * ssa,const zend_ssa_op * ssa_op,const zend_op * opline,int call_level,zend_jit_trace_rec * trace)299 static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace)
300 {
301 int skip;
302
303 if (trace) {
304 zend_jit_trace_rec *p = trace;
305
306 ssa_op++;
307 while (1) {
308 if (p->op == ZEND_JIT_TRACE_VM) {
309 switch (p->opline->opcode) {
310 case ZEND_SEND_ARRAY:
311 case ZEND_SEND_USER:
312 case ZEND_SEND_UNPACK:
313 case ZEND_INIT_FCALL:
314 case ZEND_INIT_METHOD_CALL:
315 case ZEND_INIT_STATIC_METHOD_CALL:
316 case ZEND_INIT_FCALL_BY_NAME:
317 case ZEND_INIT_NS_FCALL_BY_NAME:
318 case ZEND_INIT_DYNAMIC_CALL:
319 case ZEND_NEW:
320 case ZEND_INIT_USER_CALL:
321 case ZEND_FAST_CALL:
322 case ZEND_JMP:
323 case ZEND_JMPZNZ:
324 case ZEND_JMPZ:
325 case ZEND_JMPNZ:
326 case ZEND_JMPZ_EX:
327 case ZEND_JMPNZ_EX:
328 case ZEND_FE_RESET_R:
329 case ZEND_FE_RESET_RW:
330 case ZEND_JMP_SET:
331 case ZEND_COALESCE:
332 case ZEND_JMP_NULL:
333 case ZEND_ASSERT_CHECK:
334 case ZEND_CATCH:
335 case ZEND_DECLARE_ANON_CLASS:
336 case ZEND_FE_FETCH_R:
337 case ZEND_FE_FETCH_RW:
338 return 1;
339 case ZEND_DO_ICALL:
340 case ZEND_DO_UCALL:
341 case ZEND_DO_FCALL_BY_NAME:
342 case ZEND_DO_FCALL:
343 case ZEND_CALLABLE_CONVERT:
344 return 0;
345 case ZEND_SEND_VAL:
346 case ZEND_SEND_VAR:
347 case ZEND_SEND_VAL_EX:
348 case ZEND_SEND_VAR_EX:
349 case ZEND_SEND_FUNC_ARG:
350 case ZEND_SEND_REF:
351 case ZEND_SEND_VAR_NO_REF:
352 case ZEND_SEND_VAR_NO_REF_EX:
353 /* skip */
354 break;
355 default:
356 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
357 return 1;
358 }
359 }
360 ssa_op += zend_jit_trace_op_len(opline);
361 } else if (p->op == ZEND_JIT_TRACE_ENTER ||
362 p->op == ZEND_JIT_TRACE_BACK ||
363 p->op == ZEND_JIT_TRACE_END) {
364 return 1;
365 }
366 p++;
367 }
368 }
369
370 if (!call_info) {
371 const zend_op *end = op_array->opcodes + op_array->last;
372
373 opline++;
374 ssa_op++;
375 skip = (call_level == 1);
376 while (opline != end) {
377 if (!skip) {
378 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
379 return 1;
380 }
381 }
382 switch (opline->opcode) {
383 case ZEND_SEND_VAL:
384 case ZEND_SEND_VAR:
385 case ZEND_SEND_VAL_EX:
386 case ZEND_SEND_VAR_EX:
387 case ZEND_SEND_FUNC_ARG:
388 case ZEND_SEND_REF:
389 case ZEND_SEND_VAR_NO_REF:
390 case ZEND_SEND_VAR_NO_REF_EX:
391 skip = 0;
392 break;
393 case ZEND_SEND_ARRAY:
394 case ZEND_SEND_USER:
395 case ZEND_SEND_UNPACK:
396 case ZEND_INIT_FCALL:
397 case ZEND_INIT_METHOD_CALL:
398 case ZEND_INIT_STATIC_METHOD_CALL:
399 case ZEND_INIT_FCALL_BY_NAME:
400 case ZEND_INIT_NS_FCALL_BY_NAME:
401 case ZEND_INIT_DYNAMIC_CALL:
402 case ZEND_NEW:
403 case ZEND_INIT_USER_CALL:
404 case ZEND_FAST_CALL:
405 case ZEND_JMP:
406 case ZEND_JMPZNZ:
407 case ZEND_JMPZ:
408 case ZEND_JMPNZ:
409 case ZEND_JMPZ_EX:
410 case ZEND_JMPNZ_EX:
411 case ZEND_FE_RESET_R:
412 case ZEND_FE_RESET_RW:
413 case ZEND_JMP_SET:
414 case ZEND_COALESCE:
415 case ZEND_JMP_NULL:
416 case ZEND_ASSERT_CHECK:
417 case ZEND_CATCH:
418 case ZEND_DECLARE_ANON_CLASS:
419 case ZEND_FE_FETCH_R:
420 case ZEND_FE_FETCH_RW:
421 return 1;
422 case ZEND_DO_ICALL:
423 case ZEND_DO_UCALL:
424 case ZEND_DO_FCALL_BY_NAME:
425 case ZEND_DO_FCALL:
426 case ZEND_CALLABLE_CONVERT:
427 end = opline;
428 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
429 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
430 return 1;
431 }
432 return 0;
433 }
434 opline++;
435 ssa_op++;
436 }
437
438 return 1;
439 } else {
440 const zend_op *end = call_info->caller_call_opline;
441
442 /* end may be null if an opcode like EXIT is part of the argument list. */
443 if (!end || end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
444 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
445 return 1;
446 }
447
448 opline++;
449 ssa_op++;
450 skip = (call_level == 1);
451 while (opline != end) {
452 if (skip) {
453 switch (opline->opcode) {
454 case ZEND_SEND_VAL:
455 case ZEND_SEND_VAR:
456 case ZEND_SEND_VAL_EX:
457 case ZEND_SEND_VAR_EX:
458 case ZEND_SEND_FUNC_ARG:
459 case ZEND_SEND_REF:
460 case ZEND_SEND_VAR_NO_REF:
461 case ZEND_SEND_VAR_NO_REF_EX:
462 skip = 0;
463 break;
464 case ZEND_SEND_ARRAY:
465 case ZEND_SEND_USER:
466 case ZEND_SEND_UNPACK:
467 return 1;
468 }
469 } else {
470 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
471 return 1;
472 }
473 }
474 opline++;
475 ssa_op++;
476 }
477
478 return 0;
479 }
480 }
481
skip_valid_arguments(const zend_op_array * op_array,zend_ssa * ssa,const zend_call_info * call_info)482 static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
483 {
484 uint32_t num_args = 0;
485 zend_function *func = call_info->callee_func;
486
487 /* It's okay to handle prototypes here, because they can only increase the accepted arguments.
488 * Anything legal for the parent method is also legal for the parent method. */
489 while (num_args < call_info->num_args) {
490 zend_arg_info *arg_info = func->op_array.arg_info + num_args;
491
492 if (ZEND_TYPE_IS_SET(arg_info->type)) {
493 if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) {
494 zend_op *opline = call_info->arg_info[num_args].opline;
495 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
496 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
497 if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) {
498 break;
499 }
500 } else {
501 break;
502 }
503 }
504 num_args++;
505 }
506 return num_args;
507 }
508
zend_ssa_cv_info(const zend_op_array * op_array,zend_ssa * ssa,uint32_t var)509 static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var)
510 {
511 uint32_t j, info;
512
513 if (ssa->vars && ssa->var_info) {
514 info = ssa->var_info[var].type;
515 for (j = op_array->last_var; j < ssa->vars_count; j++) {
516 if (ssa->vars[j].var == var) {
517 info |= ssa->var_info[j].type;
518 }
519 }
520 } else {
521 info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF |
522 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
523 }
524
525 #ifdef ZEND_JIT_USE_RC_INFERENCE
526 /* Refcount may be increased by RETURN opcode */
527 if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
528 for (j = 0; j < ssa->cfg.blocks_count; j++) {
529 if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
530 ssa->cfg.blocks[j].len > 0) {
531 const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
532
533 if (opline->opcode == ZEND_RETURN) {
534 if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) {
535 info |= MAY_BE_RCN;
536 break;
537 }
538 }
539 }
540 }
541 }
542 #endif
543
544 return info;
545 }
546
zend_jit_may_avoid_refcounting(const zend_op * opline,uint32_t op1_info)547 static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_info)
548 {
549 switch (opline->opcode) {
550 case ZEND_FETCH_OBJ_FUNC_ARG:
551 if (!JIT_G(current_frame) ||
552 !JIT_G(current_frame)->call->func ||
553 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
554 return 0;
555 }
556 /* break missing intentionally */
557 case ZEND_FETCH_OBJ_R:
558 case ZEND_FETCH_OBJ_IS:
559 if ((op1_info & MAY_BE_OBJECT)
560 && opline->op2_type == IS_CONST
561 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
562 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
563 return 1;
564 }
565 break;
566 case ZEND_FETCH_DIM_FUNC_ARG:
567 if (!JIT_G(current_frame) ||
568 !JIT_G(current_frame)->call->func ||
569 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
570 return 0;
571 }
572 /* break missing intentionally */
573 case ZEND_FETCH_DIM_R:
574 case ZEND_FETCH_DIM_IS:
575 return 1;
576 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
577 if (!(opline->extended_value & ZEND_ISEMPTY)) {
578 return 1;
579 }
580 break;
581 }
582 return 0;
583 }
584
zend_jit_is_persistent_constant(zval * key,uint32_t flags)585 static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags)
586 {
587 zval *zv;
588 zend_constant *c = NULL;
589
590 /* null/true/false are resolved during compilation, so don't check for them here. */
591 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
592 if (zv) {
593 c = (zend_constant*)Z_PTR_P(zv);
594 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
595 key++;
596 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
597 if (zv) {
598 c = (zend_constant*)Z_PTR_P(zv);
599 }
600 }
601 return c && (ZEND_CONSTANT_FLAGS(c) & CONST_PERSISTENT);
602 }
603
zend_get_known_property_info(const zend_op_array * op_array,zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)604 static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
605 {
606 zend_property_info *info = NULL;
607
608 if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) ||
609 !ce ||
610 !(ce->ce_flags & ZEND_ACC_LINKED) ||
611 (ce->ce_flags & ZEND_ACC_TRAIT) ||
612 ce->create_object) {
613 return NULL;
614 }
615
616 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
617 if (ce->info.user.filename != filename) {
618 /* class declaration might be changed independently */
619 return NULL;
620 }
621
622 if (ce->parent) {
623 zend_class_entry *parent = ce->parent;
624
625 do {
626 if (parent->type == ZEND_INTERNAL_CLASS) {
627 break;
628 } else if (parent->info.user.filename != filename) {
629 /* some of parents class declarations might be changed independently */
630 /* TODO: this check may be not enough, because even
631 * in the same it's possible to conditionally define
632 * few classes with the same name, and "parent" may
633 * change from request to request.
634 */
635 return NULL;
636 }
637 parent = parent->parent;
638 } while (parent);
639 }
640 }
641
642 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
643 if (info == NULL ||
644 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
645 (info->flags & ZEND_ACC_STATIC)) {
646 return NULL;
647 }
648
649 if (info->flags & ZEND_ACC_PUBLIC) {
650 return info;
651 } else if (on_this) {
652 if (ce == info->ce) {
653 if (ce == op_array->scope) {
654 return info;
655 } else {
656 return NULL;
657 }
658 } else if ((info->flags & ZEND_ACC_PROTECTED)
659 && instanceof_function_slow(ce, info->ce)) {
660 return info;
661 }
662 }
663
664 return NULL;
665 }
666
zend_may_be_dynamic_property(zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)667 static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
668 {
669 zend_property_info *info;
670
671 if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) {
672 return 1;
673 }
674
675 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
676 if (ce->info.user.filename != filename) {
677 /* class declaration might be changed independently */
678 return 1;
679 }
680 }
681
682 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
683 if (info == NULL ||
684 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
685 (info->flags & ZEND_ACC_STATIC)) {
686 return 1;
687 }
688
689 if (!(info->flags & ZEND_ACC_PUBLIC) &&
690 (!on_this || info->ce != ce)) {
691 return 1;
692 }
693
694 return 0;
695 }
696
697 #define OP_RANGE(ssa_op, opN) \
698 (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
699 ssa->var_info && \
700 (ssa_op)->opN##_use >= 0 && \
701 ssa->var_info[(ssa_op)->opN##_use].has_range) ? \
702 &ssa->var_info[(ssa_op)->opN##_use].range : NULL)
703
704 #define OP1_RANGE() OP_RANGE(ssa_op, op1)
705 #define OP2_RANGE() OP_RANGE(ssa_op, op2)
706 #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)
707
708 #if ZEND_JIT_TARGET_X86
709 # include "dynasm/dasm_x86.h"
710 #elif ZEND_JIT_TARGET_ARM64
711 static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset);
712 # define DASM_ADD_VENEER zend_jit_add_veneer
713 # include "dynasm/dasm_arm64.h"
714 #endif
715
716 #include "jit/zend_jit_helpers.c"
717 #include "jit/zend_jit_disasm.c"
718 #ifndef _WIN32
719 # include "jit/zend_jit_gdb.h"
720 # include "jit/zend_jit_perf_dump.c"
721 #endif
722 #ifdef HAVE_OPROFILE
723 # include "jit/zend_jit_oprofile.c"
724 #endif
725
726 #include "Zend/zend_cpuinfo.h"
727
728 #ifdef HAVE_VALGRIND
729 # include <valgrind/valgrind.h>
730 #endif
731
732 #ifdef HAVE_GCC_GLOBAL_REGS
733 # define GCC_GLOBAL_REGS 1
734 #else
735 # define GCC_GLOBAL_REGS 0
736 #endif
737
738 /* By default avoid JITing inline handlers if it does not seem profitable due to lack of
739 * type information. Disabling this option allows testing some JIT handlers in the
740 * presence of try/catch blocks, which prevent SSA construction. */
741 #ifndef PROFITABILITY_CHECKS
742 # define PROFITABILITY_CHECKS 1
743 #endif
744
745 #define BP_JIT_IS 6 /* Used for ISSET_ISEMPTY_DIM_OBJ. see BP_VAR_*defines in Zend/zend_compile.h */
746
747 typedef enum _sp_adj_kind {
748 SP_ADJ_NONE,
749 SP_ADJ_RET,
750 SP_ADJ_VM,
751 SP_ADJ_JIT,
752 SP_ADJ_ASSIGN,
753 SP_ADJ_LAST
754 } sp_adj_kind;
755
756 static int sp_adj[SP_ADJ_LAST];
757
758 /* The generated code may contain tautological comparisons, ignore them. */
759 #if defined(__clang__)
760 # pragma clang diagnostic push
761 # pragma clang diagnostic ignored "-Wtautological-compare"
762 # pragma clang diagnostic ignored "-Wstring-compare"
763 #endif
764
765 #if ZEND_JIT_TARGET_X86
766 # include "jit/zend_jit_vtune.c"
767 # include "jit/zend_jit_x86.c"
768 #elif ZEND_JIT_TARGET_ARM64
769 # include "jit/zend_jit_arm64.c"
770 #endif
771
772 #if defined(__clang__)
773 # pragma clang diagnostic pop
774 #endif
775
776 #if _WIN32
777 # include <Windows.h>
778 #else
779 # include <sys/mman.h>
780 # if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
781 # define MAP_ANONYMOUS MAP_ANON
782 # endif
783 #endif
784
zend_jit_status(zval * ret)785 ZEND_EXT_API void zend_jit_status(zval *ret)
786 {
787 zval stats;
788 array_init(&stats);
789 add_assoc_bool(&stats, "enabled", JIT_G(enabled));
790 add_assoc_bool(&stats, "on", JIT_G(on));
791 add_assoc_long(&stats, "kind", JIT_G(trigger));
792 add_assoc_long(&stats, "opt_level", JIT_G(opt_level));
793 add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags));
794 if (dasm_buf) {
795 add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
796 add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
797 } else {
798 add_assoc_long(&stats, "buffer_size", 0);
799 add_assoc_long(&stats, "buffer_free", 0);
800 }
801 add_assoc_zval(ret, "jit", &stats);
802 }
803
zend_jit_func_name(const zend_op_array * op_array)804 static zend_string *zend_jit_func_name(const zend_op_array *op_array)
805 {
806 smart_str buf = {0};
807
808 if (op_array->function_name) {
809 if (op_array->scope) {
810 smart_str_appends(&buf, JIT_PREFIX);
811 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
812 smart_str_appends(&buf, "::");
813 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
814 smart_str_0(&buf);
815 return buf.s;
816 } else {
817 smart_str_appends(&buf, JIT_PREFIX);
818 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
819 smart_str_0(&buf);
820 return buf.s;
821 }
822 } else if (op_array->filename) {
823 smart_str_appends(&buf, JIT_PREFIX);
824 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
825 smart_str_0(&buf);
826 return buf.s;
827 } else {
828 return NULL;
829 }
830 }
831
832 #if ZEND_DEBUG
handle_dasm_error(int ret)833 static void handle_dasm_error(int ret) {
834 switch (ret & 0xff000000u) {
835 case DASM_S_NOMEM:
836 fprintf(stderr, "DASM_S_NOMEM\n");
837 break;
838 case DASM_S_PHASE:
839 fprintf(stderr, "DASM_S_PHASE\n");
840 break;
841 case DASM_S_MATCH_SEC:
842 fprintf(stderr, "DASM_S_MATCH_SEC\n");
843 break;
844 case DASM_S_RANGE_I:
845 fprintf(stderr, "DASM_S_RANGE_I\n");
846 break;
847 case DASM_S_RANGE_SEC:
848 fprintf(stderr, "DASM_S_RANGE_SEC\n");
849 break;
850 case DASM_S_RANGE_LG:
851 fprintf(stderr, "DASM_S_RANGE_LG\n");
852 break;
853 case DASM_S_RANGE_PC:
854 fprintf(stderr, "DASM_S_RANGE_PC %d\n", ret & 0xffffffu);
855 break;
856 #ifdef DASM_S_RANGE_VREG
857 case DASM_S_RANGE_VREG:
858 fprintf(stderr, "DASM_S_RANGE_VREG\n");
859 break;
860 #endif
861 #ifdef DASM_S_UNDEF_L
862 case DASM_S_UNDEF_L:
863 fprintf(stderr, "DASM_S_UNDEF_L\n");
864 break;
865 #endif
866 #ifdef DASM_S_UNDEF_LG
867 case DASM_S_UNDEF_LG:
868 fprintf(stderr, "DASM_S_UNDEF_LG\n");
869 break;
870 #endif
871 #ifdef DASM_S_RANGE_REL
872 case DASM_S_RANGE_REL:
873 fprintf(stderr, "DASM_S_RANGE_REL\n");
874 break;
875 #endif
876 case DASM_S_UNDEF_PC:
877 fprintf(stderr, "DASM_S_UNDEF_PC\n");
878 break;
879 default:
880 fprintf(stderr, "DASM_S_%0x\n", ret & 0xff000000u);
881 break;
882 }
883 ZEND_UNREACHABLE();
884 }
885 #endif
886
dasm_link_and_encode(dasm_State ** dasm_state,const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline,zend_lifetime_interval ** ra,const char * name,uint32_t trace_num,uint32_t sp_offset,uint32_t sp_adjustment)887 static void *dasm_link_and_encode(dasm_State **dasm_state,
888 const zend_op_array *op_array,
889 zend_ssa *ssa,
890 const zend_op *rt_opline,
891 zend_lifetime_interval **ra,
892 const char *name,
893 uint32_t trace_num,
894 uint32_t sp_offset,
895 uint32_t sp_adjustment)
896 {
897 size_t size;
898 int ret;
899 void *entry;
900 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
901 zend_string *str = NULL;
902 #endif
903
904 if (rt_opline && ssa && ssa->cfg.map) {
905 /* Create additional entry point, to switch from interpreter to JIT-ed
906 * code at run-time.
907 */
908 int b = ssa->cfg.map[rt_opline - op_array->opcodes];
909
910 //#ifdef CONTEXT_THREADED_JIT
911 // if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY))) {
912 //#else
913 if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY))) {
914 //#endif
915 zend_jit_label(dasm_state, ssa->cfg.blocks_count + b);
916 zend_jit_prologue(dasm_state);
917 if (ra) {
918 int i;
919 zend_lifetime_interval *ival;
920 zend_life_range *range;
921 uint32_t pos = rt_opline - op_array->opcodes;
922
923 for (i = 0; i < ssa->vars_count; i++) {
924 ival = ra[i];
925
926 if (ival && ival->reg != ZREG_NONE) {
927 range = &ival->range;
928
929 if (pos >= range->start && pos <= range->end) {
930 if (!zend_jit_load_var(dasm_state, ssa->var_info[i].type, ssa->vars[i].var, ival->reg)) {
931 return NULL;
932 }
933 break;
934 }
935 range = range->next;
936 }
937 }
938 }
939 zend_jit_jmp(dasm_state, b);
940 }
941 }
942
943 ret = dasm_link(dasm_state, &size);
944 if (ret != DASM_S_OK) {
945 #if ZEND_DEBUG
946 handle_dasm_error(ret);
947 #endif
948 return NULL;
949 }
950
951 if ((void*)((char*)*dasm_ptr + size) > dasm_end) {
952 *dasm_ptr = dasm_end; //prevent further try
953 // TODO: jit_buffer_size overflow ???
954 return NULL;
955 }
956
957 #if ZEND_JIT_TARGET_ARM64
958 dasm_venners_size = 0;
959 #endif
960
961 ret = dasm_encode(dasm_state, *dasm_ptr);
962 if (ret != DASM_S_OK) {
963 #if ZEND_DEBUG
964 handle_dasm_error(ret);
965 #endif
966 return NULL;
967 }
968
969 #if ZEND_JIT_TARGET_ARM64
970 size += dasm_venners_size;
971 #endif
972
973 entry = *dasm_ptr;
974 *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT));
975
976 /* flush the hardware I-cache */
977 JIT_CACHE_FLUSH(entry, entry + size);
978
979 if (trace_num) {
980 zend_jit_trace_add_code(entry, dasm_getpclabel(dasm_state, 1));
981 }
982
983 if (op_array && ssa) {
984 int b;
985
986 for (b = 0; b < ssa->cfg.blocks_count; b++) {
987 //#ifdef CONTEXT_THREADED_JIT
988 // if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
989 //#else
990 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) {
991 //#endif
992 zend_op *opline = op_array->opcodes + ssa->cfg.blocks[b].start;
993 int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
994
995 if (offset >= 0) {
996 opline->handler = (void*)(((char*)entry) + offset);
997 }
998 }
999 }
1000 if (rt_opline && ssa && ssa->cfg.map) {
1001 int b = ssa->cfg.map[rt_opline - op_array->opcodes];
1002 zend_op *opline = (zend_op*)rt_opline;
1003 int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
1004
1005 if (offset >= 0) {
1006 opline->handler = (void*)(((char*)entry) + offset);
1007 }
1008 }
1009 }
1010
1011 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
1012 if (!name) {
1013 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) {
1014 str = zend_jit_func_name(op_array);
1015 if (str) {
1016 name = ZSTR_VAL(str);
1017 }
1018 }
1019 #ifdef HAVE_DISASM
1020 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
1021 zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
1022 zend_jit_disasm(
1023 name,
1024 (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
1025 op_array,
1026 &ssa->cfg,
1027 entry,
1028 size);
1029 }
1030 } else {
1031 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) {
1032 zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
1033 if ((JIT_G(debug) & (trace_num ? ZEND_JIT_DEBUG_ASM : ZEND_JIT_DEBUG_ASM_STUBS)) != 0) {
1034 zend_jit_disasm(
1035 name,
1036 (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
1037 op_array,
1038 ssa ? &ssa->cfg : NULL,
1039 entry,
1040 size);
1041 }
1042 }
1043 # endif
1044 }
1045 #endif
1046
1047 #ifdef HAVE_GDB
1048 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
1049 if (name) {
1050 zend_jit_gdb_register(
1051 name,
1052 op_array,
1053 entry,
1054 size,
1055 sp_adj[sp_offset],
1056 sp_adj[sp_adjustment]);
1057 }
1058 }
1059 #endif
1060
1061 #ifdef HAVE_OPROFILE
1062 if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
1063 zend_jit_oprofile_register(
1064 name,
1065 entry,
1066 size);
1067 }
1068 #endif
1069
1070 #ifdef HAVE_PERFTOOLS
1071 if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
1072 if (name) {
1073 zend_jit_perf_map_register(
1074 name,
1075 entry,
1076 size);
1077 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
1078 zend_jit_perf_jitdump_register(
1079 name,
1080 entry,
1081 size);
1082 }
1083 }
1084 }
1085 #endif
1086
1087 #ifdef HAVE_VTUNE
1088 if (JIT_G(debug) & ZEND_JIT_DEBUG_VTUNE) {
1089 if (name) {
1090 zend_jit_vtune_register(
1091 name,
1092 entry,
1093 size);
1094 }
1095 }
1096 #endif
1097
1098 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
1099 if (str) {
1100 zend_string_release(str);
1101 }
1102 #endif
1103
1104 return entry;
1105 }
1106
zend_may_overflow(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa)1107 static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
1108 {
1109 int res;
1110 zend_long op1_min, op1_max, op2_min, op2_max;
1111
1112 if (!ssa->ops || !ssa->var_info) {
1113 return 1;
1114 }
1115 switch (opline->opcode) {
1116 case ZEND_PRE_INC:
1117 case ZEND_POST_INC:
1118 res = ssa_op->op1_def;
1119 if (res < 0
1120 || !ssa->var_info[res].has_range
1121 || ssa->var_info[res].range.overflow) {
1122 if (!OP1_HAS_RANGE()) {
1123 return 1;
1124 }
1125 op1_max = OP1_MAX_RANGE();
1126 if (op1_max == ZEND_LONG_MAX) {
1127 return 1;
1128 }
1129 }
1130 return 0;
1131 case ZEND_PRE_DEC:
1132 case ZEND_POST_DEC:
1133 res = ssa_op->op1_def;
1134 if (res < 0
1135 || !ssa->var_info[res].has_range
1136 || ssa->var_info[res].range.underflow) {
1137 if (!OP1_HAS_RANGE()) {
1138 return 1;
1139 }
1140 op1_min = OP1_MIN_RANGE();
1141 if (op1_min == ZEND_LONG_MIN) {
1142 return 1;
1143 }
1144 }
1145 return 0;
1146 case ZEND_ADD:
1147 res = ssa_op->result_def;
1148 if (res < 0
1149 || !ssa->var_info[res].has_range
1150 || ssa->var_info[res].range.underflow) {
1151 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1152 return 1;
1153 }
1154 op1_min = OP1_MIN_RANGE();
1155 op2_min = OP2_MIN_RANGE();
1156 if (zend_add_will_overflow(op1_min, op2_min)) {
1157 return 1;
1158 }
1159 }
1160 if (res < 0
1161 || !ssa->var_info[res].has_range
1162 || ssa->var_info[res].range.overflow) {
1163 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1164 return 1;
1165 }
1166 op1_max = OP1_MAX_RANGE();
1167 op2_max = OP2_MAX_RANGE();
1168 if (zend_add_will_overflow(op1_max, op2_max)) {
1169 return 1;
1170 }
1171 }
1172 return 0;
1173 case ZEND_SUB:
1174 res = ssa_op->result_def;
1175 if (res < 0
1176 || !ssa->var_info[res].has_range
1177 || ssa->var_info[res].range.underflow) {
1178 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1179 return 1;
1180 }
1181 op1_min = OP1_MIN_RANGE();
1182 op2_max = OP2_MAX_RANGE();
1183 if (zend_sub_will_overflow(op1_min, op2_max)) {
1184 return 1;
1185 }
1186 }
1187 if (res < 0
1188 || !ssa->var_info[res].has_range
1189 || ssa->var_info[res].range.overflow) {
1190 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1191 return 1;
1192 }
1193 op1_max = OP1_MAX_RANGE();
1194 op2_min = OP2_MIN_RANGE();
1195 if (zend_sub_will_overflow(op1_max, op2_min)) {
1196 return 1;
1197 }
1198 }
1199 return 0;
1200 case ZEND_MUL:
1201 res = ssa_op->result_def;
1202 return (res < 0 ||
1203 !ssa->var_info[res].has_range ||
1204 ssa->var_info[res].range.underflow ||
1205 ssa->var_info[res].range.overflow);
1206 case ZEND_ASSIGN_OP:
1207 if (opline->extended_value == ZEND_ADD) {
1208 res = ssa_op->op1_def;
1209 if (res < 0
1210 || !ssa->var_info[res].has_range
1211 || ssa->var_info[res].range.underflow) {
1212 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1213 return 1;
1214 }
1215 op1_min = OP1_MIN_RANGE();
1216 op2_min = OP2_MIN_RANGE();
1217 if (zend_add_will_overflow(op1_min, op2_min)) {
1218 return 1;
1219 }
1220 }
1221 if (res < 0
1222 || !ssa->var_info[res].has_range
1223 || ssa->var_info[res].range.overflow) {
1224 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1225 return 1;
1226 }
1227 op1_max = OP1_MAX_RANGE();
1228 op2_max = OP2_MAX_RANGE();
1229 if (zend_add_will_overflow(op1_max, op2_max)) {
1230 return 1;
1231 }
1232 }
1233 return 0;
1234 } else if (opline->extended_value == ZEND_SUB) {
1235 res = ssa_op->op1_def;
1236 if (res < 0
1237 || !ssa->var_info[res].has_range
1238 || ssa->var_info[res].range.underflow) {
1239 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1240 return 1;
1241 }
1242 op1_min = OP1_MIN_RANGE();
1243 op2_max = OP2_MAX_RANGE();
1244 if (zend_sub_will_overflow(op1_min, op2_max)) {
1245 return 1;
1246 }
1247 }
1248 if (res < 0
1249 || !ssa->var_info[res].has_range
1250 || ssa->var_info[res].range.overflow) {
1251 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1252 return 1;
1253 }
1254 op1_max = OP1_MAX_RANGE();
1255 op2_min = OP2_MIN_RANGE();
1256 if (zend_sub_will_overflow(op1_max, op2_min)) {
1257 return 1;
1258 }
1259 }
1260 return 0;
1261 } else if (opline->extended_value == ZEND_MUL) {
1262 res = ssa_op->op1_def;
1263 return (res < 0 ||
1264 !ssa->var_info[res].has_range ||
1265 ssa->var_info[res].range.underflow ||
1266 ssa->var_info[res].range.overflow);
1267 }
1268 ZEND_FALLTHROUGH;
1269 default:
1270 return 1;
1271 }
1272 }
1273
zend_jit_build_cfg(const zend_op_array * op_array,zend_cfg * cfg)1274 static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg)
1275 {
1276 uint32_t flags;
1277
1278 flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY;
1279
1280 if (zend_build_cfg(&CG(arena), op_array, flags, cfg) != SUCCESS) {
1281 return FAILURE;
1282 }
1283
1284 /* Don't JIT huge functions. Apart from likely being detrimental due to the amount of
1285 * generated code, some of our analysis is recursive and will stack overflow with many
1286 * blocks. */
1287 if (cfg->blocks_count > 100000) {
1288 return FAILURE;
1289 }
1290
1291 if (zend_cfg_build_predecessors(&CG(arena), cfg) != SUCCESS) {
1292 return FAILURE;
1293 }
1294
1295 /* Compute Dominators Tree */
1296 if (zend_cfg_compute_dominators_tree(op_array, cfg) != SUCCESS) {
1297 return FAILURE;
1298 }
1299
1300 /* Identify reducible and irreducible loops */
1301 if (zend_cfg_identify_loops(op_array, cfg) != SUCCESS) {
1302 return FAILURE;
1303 }
1304
1305 return SUCCESS;
1306 }
1307
zend_jit_op_array_analyze1(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa)1308 static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
1309 {
1310 if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) {
1311 return FAILURE;
1312 }
1313
1314 #if 0
1315 /* TODO: debugger and profiler supports? */
1316 if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) {
1317 return FAILURE;
1318 }
1319 #endif
1320
1321 /* TODO: move this to zend_cfg.c ? */
1322 if (!op_array->function_name) {
1323 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1324 }
1325
1326 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
1327 && ssa->cfg.blocks
1328 && op_array->last_try_catch == 0
1329 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
1330 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
1331 if (zend_build_ssa(&CG(arena), script, op_array, ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) {
1332 return FAILURE;
1333 }
1334
1335 if (zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa) != SUCCESS) {
1336 return FAILURE;
1337 }
1338
1339 if (zend_ssa_find_false_dependencies(op_array, ssa) != SUCCESS) {
1340 return FAILURE;
1341 }
1342
1343 if (zend_ssa_find_sccs(op_array, ssa) != SUCCESS){
1344 return FAILURE;
1345 }
1346 }
1347
1348 return SUCCESS;
1349 }
1350
zend_jit_op_array_analyze2(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa,uint32_t optimization_level)1351 static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t optimization_level)
1352 {
1353 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
1354 && ssa->cfg.blocks
1355 && op_array->last_try_catch == 0
1356 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
1357 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
1358 if (zend_ssa_inference(&CG(arena), op_array, script, ssa,
1359 optimization_level & ~ZEND_OPTIMIZER_NARROW_TO_DOUBLE) != SUCCESS) {
1360 return FAILURE;
1361 }
1362 }
1363
1364 return SUCCESS;
1365 }
1366
zend_jit_add_range(zend_lifetime_interval ** intervals,int var,uint32_t from,uint32_t to)1367 static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint32_t from, uint32_t to)
1368 {
1369 zend_lifetime_interval *ival = intervals[var];
1370
1371 if (!ival) {
1372 ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
1373 if (!ival) {
1374 return FAILURE;
1375 }
1376 ival->ssa_var = var;
1377 ival->reg = ZREG_NONE;
1378 ival->flags = 0;
1379 ival->range.start = from;
1380 ival->range.end = to;
1381 ival->range.next = NULL;
1382 ival->hint = NULL;
1383 ival->used_as_hint = NULL;
1384 intervals[var] = ival;
1385 } else if (ival->range.start > to + 1) {
1386 zend_life_range *range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1387
1388 if (!range) {
1389 return FAILURE;
1390 }
1391 range->start = ival->range.start;
1392 range->end = ival->range.end;
1393 range->next = ival->range.next;
1394 ival->range.start = from;
1395 ival->range.end = to;
1396 ival->range.next = range;
1397 } else if (ival->range.start == to + 1) {
1398 ival->range.start = from;
1399 } else {
1400 zend_life_range *range = &ival->range;
1401 zend_life_range *last = NULL;
1402
1403 do {
1404 if (range->start > to + 1) {
1405 break;
1406 } else if (range->end + 1 >= from) {
1407 if (range->start > from) {
1408 range->start = from;
1409 }
1410 last = range;
1411 range = range->next;
1412 while (range) {
1413 if (range->start > to + 1) {
1414 break;
1415 }
1416 last->end = range->end;
1417 range = range->next;
1418 last->next = range;
1419 }
1420 if (to > last->end) {
1421 last->end = to;
1422 }
1423 return SUCCESS;
1424 }
1425 last = range;
1426 range = range->next;
1427 } while (range);
1428
1429 range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1430 if (!range) {
1431 return FAILURE;
1432 }
1433 range->start = from;
1434 range->end = to;
1435 range->next = last->next;
1436 last->next = range;
1437 }
1438
1439 return SUCCESS;
1440 }
1441
zend_jit_begin_range(zend_lifetime_interval ** intervals,int var,uint32_t block_start,uint32_t from)1442 static int zend_jit_begin_range(zend_lifetime_interval **intervals, int var, uint32_t block_start, uint32_t from)
1443 {
1444 if (block_start != from && intervals[var]) {
1445 zend_life_range *range = &intervals[var]->range;
1446
1447 do {
1448 if (from >= range->start && from <= range->end) {
1449 if (range->start == block_start) {
1450 range->start = from;
1451 } else {
1452 zend_life_range *r = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1453 if (!r) {
1454 return FAILURE;
1455 }
1456 r->start = from;
1457 r->end = range->end;
1458 r->next = range->next;
1459 range->end = block_start - 1;
1460 range->next = r;
1461 }
1462 return SUCCESS;
1463 }
1464 range = range->next;
1465 } while (range);
1466 }
1467
1468 // dead store
1469 return zend_jit_add_range(intervals, var, from, from);
1470 }
1471
zend_jit_insert_interval(zend_lifetime_interval ** list,zend_lifetime_interval * ival)1472 static void zend_jit_insert_interval(zend_lifetime_interval **list, zend_lifetime_interval *ival)
1473 {
1474 while (1) {
1475 if (*list == NULL) {
1476 *list = ival;
1477 ival->list_next = NULL;
1478 return;
1479 } else if (ival->range.start < (*list)->range.start) {
1480 ival->list_next = *list;
1481 *list = ival;
1482 return;
1483 }
1484 list = &(*list)->list_next;
1485 }
1486 }
1487
zend_jit_split_interval(zend_lifetime_interval * current,uint32_t pos,zend_lifetime_interval ** list,zend_lifetime_interval ** free)1488 static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos, zend_lifetime_interval **list, zend_lifetime_interval **free)
1489 {
1490 zend_lifetime_interval *ival;
1491 zend_life_range *range = ¤t->range;
1492 zend_life_range *prev = NULL;
1493
1494 if (*free) {
1495 ival = *free;
1496 *free = ival->list_next;
1497 } else {
1498 ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
1499
1500 if (!ival) {
1501 return FAILURE;
1502 }
1503 }
1504
1505 current->flags |= ZREG_STORE;
1506
1507 ival->ssa_var = current->ssa_var;
1508 ival->reg = ZREG_NONE;
1509 ival->flags |= ZREG_SPLIT | ZREG_LOAD;
1510 ival->flags &= ~ZREG_STORE;
1511 ival->hint = NULL;
1512
1513 do {
1514 if (pos >= range->start && pos <= range->end) {
1515 break;
1516 }
1517 prev = range;
1518 range = range->next;
1519 } while(range);
1520
1521 ZEND_ASSERT(range != NULL);
1522
1523 ival->range.start = pos;
1524 ival->range.end = range->end;
1525 ival->range.next = range->next;
1526
1527 if (pos == range->start) {
1528 ZEND_ASSERT(prev != NULL);
1529 prev->next = NULL;
1530 } else {
1531 range->end = pos - 1;
1532 }
1533
1534 zend_jit_insert_interval(list, ival);
1535
1536 return SUCCESS;
1537 }
1538
zend_jit_sort_intervals(zend_lifetime_interval ** intervals,int count)1539 static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval **intervals, int count)
1540 {
1541 zend_lifetime_interval *list, *last;
1542 int i;
1543
1544 list = NULL;
1545 i = 0;
1546 while (i < count) {
1547 list = intervals[i];
1548 i++;
1549 if (list) {
1550 last = list;
1551 last->list_next = NULL;
1552 break;
1553 }
1554 }
1555
1556 while (i < count) {
1557 zend_lifetime_interval *ival = intervals[i];
1558
1559 i++;
1560 if (ival) {
1561 if ((ival->range.start > last->range.start) ||
1562 (ival->range.start == last->range.start &&
1563 ((!ival->hint && last->hint && last->hint != ival) ||
1564 ival->range.end > last->range.end))) {
1565 last->list_next = ival;
1566 last = ival;
1567 ival->list_next = NULL;
1568 } else {
1569 zend_lifetime_interval **p = &list;
1570
1571 while (1) {
1572 if (*p == NULL) {
1573 *p = last = ival;
1574 ival->list_next = NULL;
1575 break;
1576 } else if ((ival->range.start < (*p)->range.start) ||
1577 (ival->range.start == (*p)->range.start &&
1578 ((ival->hint && !(*p)->hint && ival->hint != *p) ||
1579 ival->range.end < (*p)->range.end))) {
1580 ival->list_next = *p;
1581 *p = ival;
1582 break;
1583 }
1584 p = &(*p)->list_next;
1585 }
1586 }
1587 }
1588 }
1589
1590 return list;
1591 }
1592
zend_jit_print_regset(zend_regset regset)1593 static ZEND_ATTRIBUTE_UNUSED void zend_jit_print_regset(zend_regset regset)
1594 {
1595 zend_reg reg;
1596 int first = 1;
1597
1598 ZEND_REGSET_FOREACH(regset, reg) {
1599 if (first) {
1600 first = 0;
1601 fprintf(stderr, "%s", zend_reg_name[reg]);
1602 } else {
1603 fprintf(stderr, ", %s", zend_reg_name[reg]);
1604 }
1605 } ZEND_REGSET_FOREACH_END();
1606 }
1607
zend_jit_compute_block_order_int(zend_ssa * ssa,int n,int * block_order)1608 static int *zend_jit_compute_block_order_int(zend_ssa *ssa, int n, int *block_order)
1609 {
1610 zend_basic_block *b = ssa->cfg.blocks + n;
1611
1612 tail_call:
1613 *block_order = n;
1614 block_order++;
1615
1616 n = b->children;
1617 while (n >= 0) {
1618 b = ssa->cfg.blocks + n;
1619 if (b->next_child < 0) {
1620 goto tail_call;
1621 }
1622 block_order = zend_jit_compute_block_order_int(ssa, n, block_order);
1623 n = b->next_child;
1624 }
1625
1626 return block_order;
1627 }
1628
zend_jit_compute_block_order(zend_ssa * ssa,int * block_order)1629 static int zend_jit_compute_block_order(zend_ssa *ssa, int *block_order)
1630 {
1631 int *end = zend_jit_compute_block_order_int(ssa, 0, block_order);
1632
1633 return end - block_order;
1634 }
1635
zend_jit_in_loop(zend_ssa * ssa,int header,zend_basic_block * b)1636 static bool zend_jit_in_loop(zend_ssa *ssa, int header, zend_basic_block *b)
1637 {
1638 while (b->loop_header >= 0) {
1639 if (b->loop_header == header) {
1640 return 1;
1641 }
1642 b = ssa->cfg.blocks + b->loop_header;
1643 }
1644 return 0;
1645 }
1646
zend_jit_compute_loop_body(zend_ssa * ssa,int header,int n,zend_bitset loop_body)1647 static void zend_jit_compute_loop_body(zend_ssa *ssa, int header, int n, zend_bitset loop_body)
1648 {
1649 zend_basic_block *b = ssa->cfg.blocks + n;
1650 uint32_t i;
1651
1652 tail_call:
1653 if (b->len) {
1654 for (i = b->start; i < b->start + b->len; i++) {
1655 zend_bitset_incl(loop_body, i);
1656 }
1657 }
1658
1659 n = b->children;
1660 while (n >= 0) {
1661 b = ssa->cfg.blocks + n;
1662 if (zend_jit_in_loop(ssa, header, b)) {
1663 if (b->next_child < 0) {
1664 goto tail_call;
1665 }
1666 zend_jit_compute_loop_body(ssa, header, n, loop_body);
1667 }
1668 n = b->next_child;
1669 }
1670 }
1671
zend_jit_add_hint(zend_lifetime_interval ** intervals,int dst,int src)1672 static void zend_jit_add_hint(zend_lifetime_interval **intervals, int dst, int src)
1673 {
1674 if (intervals[dst]->range.start < intervals[src]->range.start) {
1675 int tmp = src;
1676 src = dst;
1677 dst = tmp;
1678 }
1679 while (dst != src && intervals[dst]->hint) {
1680 if (intervals[dst]->hint->range.start < intervals[src]->range.start) {
1681 int tmp = src;
1682 src = intervals[dst]->hint->ssa_var;
1683 dst = tmp;
1684 } else {
1685 dst = intervals[dst]->hint->ssa_var;
1686 }
1687 }
1688 if (dst != src) {
1689 intervals[dst]->hint = intervals[src];
1690 }
1691 }
1692
1693 /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
1694 Michael Franz, CGO'10 (2010), Figure 4. */
zend_jit_compute_liveness(const zend_op_array * op_array,zend_ssa * ssa,zend_bitset candidates,zend_lifetime_interval ** list)1695 static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list)
1696 {
1697 int set_size, i, j, k, l;
1698 uint32_t n;
1699 zend_bitset live, live_in, pi_vars, loop_body;
1700 int *block_order;
1701 zend_ssa_phi *phi;
1702 zend_lifetime_interval **intervals;
1703 size_t mem_size;
1704 ALLOCA_FLAG(use_heap);
1705
1706 set_size = zend_bitset_len(ssa->vars_count);
1707 mem_size =
1708 ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)) +
1709 ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE) +
1710 ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
1711 ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
1712 ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE) +
1713 ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(int));
1714 intervals = do_alloca(mem_size, use_heap);
1715 if (!intervals) {
1716 *list = NULL;
1717 return FAILURE;
1718 }
1719
1720 live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)));
1721 live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE));
1722 pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
1723 loop_body = (zend_bitset)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
1724 block_order = (int*)((char*)loop_body + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE));
1725
1726 memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*));
1727 zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count);
1728
1729 /* TODO: Provide a linear block order where all dominators of a block
1730 * are before this block, and where all blocks belonging to the same loop
1731 * are contiguous ???
1732 */
1733 for (l = zend_jit_compute_block_order(ssa, block_order) - 1; l >= 0; l--) {
1734 zend_basic_block *b;
1735
1736 i = block_order[l];
1737 b = ssa->cfg.blocks + i;
1738
1739 /* live = UNION of successor.liveIn for each successor of b */
1740 /* live.add(phi.inputOf(b)) for each phi of successors of b */
1741 zend_bitset_clear(live, set_size);
1742 for (j = 0; j < b->successors_count; j++) {
1743 int succ = b->successors[j];
1744
1745 zend_bitset_union(live, live_in + set_size * succ, set_size);
1746 zend_bitset_clear(pi_vars, set_size);
1747 for (phi = ssa->blocks[succ].phis; phi; phi = phi->next) {
1748 if (ssa->vars[phi->ssa_var].no_val) {
1749 /* skip */
1750 } else if (phi->pi >= 0) {
1751 if (phi->pi == i && phi->sources[0] >= 0) {
1752 if (zend_bitset_in(candidates, phi->sources[0])) {
1753 zend_bitset_incl(live, phi->sources[0]);
1754 }
1755 zend_bitset_incl(pi_vars, phi->var);
1756 }
1757 } else if (!zend_bitset_in(pi_vars, phi->var)) {
1758 for (k = 0; k < ssa->cfg.blocks[succ].predecessors_count; k++) {
1759 if (ssa->cfg.predecessors[ssa->cfg.blocks[succ].predecessor_offset + k] == i) {
1760 if (phi->sources[k] >= 0 && zend_bitset_in(candidates, phi->sources[k])) {
1761 zend_bitset_incl(live, phi->sources[k]);
1762 }
1763 break;
1764 }
1765 }
1766 }
1767 }
1768 }
1769
1770 /* addRange(var, b.from, b.to) for each var in live */
1771 ZEND_BITSET_FOREACH(live, set_size, j) {
1772 if (zend_bitset_in(candidates, j)) {
1773 if (zend_jit_add_range(intervals, j, b->start, b->start + b->len - 1) != SUCCESS) {
1774 goto failure;
1775 }
1776 }
1777 } ZEND_BITSET_FOREACH_END();
1778
1779 /* for each operation op of b in reverse order */
1780 for (n = b->start + b->len; n > b->start;) {
1781 zend_ssa_op *op;
1782 const zend_op *opline;
1783 uint32_t num;
1784
1785 n--;
1786 op = ssa->ops + n;
1787 opline = op_array->opcodes + n;
1788
1789 if (UNEXPECTED(opline->opcode == ZEND_OP_DATA)) {
1790 num = n - 1;
1791 } else {
1792 num = n;
1793 }
1794
1795 /* for each output operand opd of op do */
1796 /* setFrom(opd, op) */
1797 /* live.remove(opd) */
1798 if (op->op1_def >= 0 && zend_bitset_in(candidates, op->op1_def)) {
1799 if (zend_jit_begin_range(intervals, op->op1_def, b->start, num) != SUCCESS) {
1800 goto failure;
1801 }
1802 zend_bitset_excl(live, op->op1_def);
1803 }
1804 if (op->op2_def >= 0 && zend_bitset_in(candidates, op->op2_def)) {
1805 if (zend_jit_begin_range(intervals, op->op2_def, b->start, num) != SUCCESS) {
1806 goto failure;
1807 }
1808 zend_bitset_excl(live, op->op2_def);
1809 }
1810 if (op->result_def >= 0 && zend_bitset_in(candidates, op->result_def)) {
1811 if (zend_jit_begin_range(intervals, op->result_def, b->start, num) != SUCCESS) {
1812 goto failure;
1813 }
1814 zend_bitset_excl(live, op->result_def);
1815 }
1816
1817 /* for each input operand opd of op do */
1818 /* live.add(opd) */
1819 /* addRange(opd, b.from, op) */
1820 if (op->op1_use >= 0
1821 && zend_bitset_in(candidates, op->op1_use)
1822 && !zend_ssa_is_no_val_use(opline, op, op->op1_use)) {
1823 zend_bitset_incl(live, op->op1_use);
1824 if (zend_jit_add_range(intervals, op->op1_use, b->start, num) != SUCCESS) {
1825 goto failure;
1826 }
1827 }
1828 if (op->op2_use >= 0
1829 && zend_bitset_in(candidates, op->op2_use)
1830 && !zend_ssa_is_no_val_use(opline, op, op->op2_use)) {
1831 zend_bitset_incl(live, op->op2_use);
1832 if (zend_jit_add_range(intervals, op->op2_use, b->start, num) != SUCCESS) {
1833 goto failure;
1834 }
1835 }
1836 if (op->result_use >= 0
1837 && zend_bitset_in(candidates, op->result_use)
1838 && !zend_ssa_is_no_val_use(opline, op, op->result_use)) {
1839 zend_bitset_incl(live, op->result_use);
1840 if (zend_jit_add_range(intervals, op->result_use, b->start, num) != SUCCESS) {
1841 goto failure;
1842 }
1843 }
1844 }
1845
1846 /* live.remove(phi.output) for each phi of b */
1847 for (phi = ssa->blocks[i].phis; phi; phi = phi->next) {
1848 zend_bitset_excl(live, phi->ssa_var);
1849 }
1850
1851 /* b.liveIn = live */
1852 zend_bitset_copy(live_in + set_size * i, live, set_size);
1853 }
1854
1855 for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) {
1856 zend_basic_block *b = ssa->cfg.blocks + i;
1857
1858 /* if b is loop header */
1859 if ((b->flags & ZEND_BB_LOOP_HEADER)) {
1860 live = live_in + set_size * i;
1861
1862 if (!zend_bitset_empty(live, set_size)) {
1863 uint32_t set_size2 = zend_bitset_len(op_array->last);
1864
1865 zend_bitset_clear(loop_body, set_size2);
1866 zend_jit_compute_loop_body(ssa, i, i, loop_body);
1867 while (!zend_bitset_empty(loop_body, set_size2)) {
1868 uint32_t from = zend_bitset_first(loop_body, set_size2);
1869 uint32_t to = from;
1870
1871 do {
1872 zend_bitset_excl(loop_body, to);
1873 to++;
1874 } while (zend_bitset_in(loop_body, to));
1875 to--;
1876
1877 ZEND_BITSET_FOREACH(live, set_size, j) {
1878 if (zend_jit_add_range(intervals, j, from, to) != SUCCESS) {
1879 goto failure;
1880 }
1881 } ZEND_BITSET_FOREACH_END();
1882 }
1883 }
1884 }
1885
1886 }
1887
1888 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
1889 /* Register hinting (a cheap way for register coalescing) */
1890 for (i = 0; i < ssa->vars_count; i++) {
1891 if (intervals[i]) {
1892 int src;
1893
1894 if (ssa->vars[i].definition_phi) {
1895 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
1896
1897 if (phi->pi >= 0) {
1898 src = phi->sources[0];
1899 if (intervals[src]) {
1900 zend_jit_add_hint(intervals, i, src);
1901 }
1902 } else {
1903 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1904 src = phi->sources[k];
1905 if (src >= 0) {
1906 if (ssa->vars[src].definition_phi
1907 && ssa->vars[src].definition_phi->pi >= 0
1908 && phi->block == ssa->vars[src].definition_phi->block) {
1909 /* Skip zero-length interval for Pi variable */
1910 src = ssa->vars[src].definition_phi->sources[0];
1911 }
1912 if (intervals[src]) {
1913 zend_jit_add_hint(intervals, i, src);
1914 }
1915 }
1916 }
1917 }
1918 }
1919 }
1920 }
1921 for (i = 0; i < ssa->vars_count; i++) {
1922 if (intervals[i] && !intervals[i]->hint) {
1923
1924 if (ssa->vars[i].definition >= 0) {
1925 uint32_t line = ssa->vars[i].definition;
1926 const zend_op *opline = op_array->opcodes + line;
1927
1928 switch (opline->opcode) {
1929 case ZEND_QM_ASSIGN:
1930 case ZEND_POST_INC:
1931 case ZEND_POST_DEC:
1932 if (ssa->ops[line].op1_use >= 0 &&
1933 intervals[ssa->ops[line].op1_use] &&
1934 (i == ssa->ops[line].op1_def ||
1935 (i == ssa->ops[line].result_def &&
1936 (ssa->ops[line].op1_def < 0 ||
1937 !intervals[ssa->ops[line].op1_def])))) {
1938 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1939 }
1940 break;
1941 case ZEND_SEND_VAR:
1942 case ZEND_PRE_INC:
1943 case ZEND_PRE_DEC:
1944 if (i == ssa->ops[line].op1_def &&
1945 ssa->ops[line].op1_use >= 0 &&
1946 intervals[ssa->ops[line].op1_use]) {
1947 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1948 }
1949 break;
1950 case ZEND_ASSIGN:
1951 if (ssa->ops[line].op2_use >= 0 &&
1952 intervals[ssa->ops[line].op2_use] &&
1953 (i == ssa->ops[line].op2_def ||
1954 (i == ssa->ops[line].op1_def &&
1955 (ssa->ops[line].op2_def < 0 ||
1956 !intervals[ssa->ops[line].op2_def])) ||
1957 (i == ssa->ops[line].result_def &&
1958 (ssa->ops[line].op2_def < 0 ||
1959 !intervals[ssa->ops[line].op2_def]) &&
1960 (ssa->ops[line].op1_def < 0 ||
1961 !intervals[ssa->ops[line].op1_def])))) {
1962 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
1963 }
1964 break;
1965 case ZEND_SUB:
1966 case ZEND_ADD:
1967 case ZEND_MUL:
1968 case ZEND_BW_OR:
1969 case ZEND_BW_AND:
1970 case ZEND_BW_XOR:
1971 if (i == ssa->ops[line].result_def) {
1972 if (ssa->ops[line].op1_use >= 0 &&
1973 intervals[ssa->ops[line].op1_use] &&
1974 ssa->ops[line].op1_use_chain < 0 &&
1975 !ssa->vars[ssa->ops[line].op1_use].phi_use_chain &&
1976 (ssa->var_info[i].type & MAY_BE_ANY) ==
1977 (ssa->var_info[ssa->ops[line].op1_use].type & MAY_BE_ANY)) {
1978 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1979 } else if (opline->opcode != ZEND_SUB &&
1980 ssa->ops[line].op2_use >= 0 &&
1981 intervals[ssa->ops[line].op2_use] &&
1982 ssa->ops[line].op2_use_chain < 0 &&
1983 !ssa->vars[ssa->ops[line].op2_use].phi_use_chain &&
1984 (ssa->var_info[i].type & MAY_BE_ANY) ==
1985 (ssa->var_info[ssa->ops[line].op2_use].type & MAY_BE_ANY)) {
1986 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
1987 }
1988 }
1989 break;
1990 }
1991 }
1992 }
1993 }
1994 }
1995
1996 *list = zend_jit_sort_intervals(intervals, ssa->vars_count);
1997
1998 if (*list) {
1999 zend_lifetime_interval *ival = *list;
2000 while (ival) {
2001 if (ival->hint) {
2002 ival->hint->used_as_hint = ival;
2003 }
2004 ival = ival->list_next;
2005 }
2006 }
2007
2008 free_alloca(intervals, use_heap);
2009 return SUCCESS;
2010
2011 failure:
2012 *list = NULL;
2013 free_alloca(intervals, use_heap);
2014 return FAILURE;
2015 }
2016
zend_interval_end(zend_lifetime_interval * ival)2017 static uint32_t zend_interval_end(zend_lifetime_interval *ival)
2018 {
2019 zend_life_range *range = &ival->range;
2020
2021 while (range->next) {
2022 range = range->next;
2023 }
2024 return range->end;
2025 }
2026
zend_interval_covers(zend_lifetime_interval * ival,uint32_t position)2027 static bool zend_interval_covers(zend_lifetime_interval *ival, uint32_t position)
2028 {
2029 zend_life_range *range = &ival->range;
2030
2031 do {
2032 if (position >= range->start && position <= range->end) {
2033 return 1;
2034 }
2035 range = range->next;
2036 } while (range);
2037
2038 return 0;
2039 }
2040
zend_interval_intersection(zend_lifetime_interval * ival1,zend_lifetime_interval * ival2)2041 static uint32_t zend_interval_intersection(zend_lifetime_interval *ival1, zend_lifetime_interval *ival2)
2042 {
2043 zend_life_range *r1 = &ival1->range;
2044 zend_life_range *r2 = &ival2->range;
2045
2046 do {
2047 if (r1->start <= r2->end) {
2048 if (r2->start <= r1->end) {
2049 return MAX(r1->start, r2->start);
2050 } else {
2051 r2 = r2->next;
2052 }
2053 } else {
2054 r1 = r1->next;
2055 }
2056 } while (r1 && r2);
2057
2058 return 0xffffffff;
2059 }
2060
2061 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2062 Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling */
zend_jit_try_allocate_free_reg(const zend_op_array * op_array,const zend_op ** ssa_opcodes,zend_ssa * ssa,zend_lifetime_interval * current,zend_regset available,zend_regset * hints,zend_lifetime_interval * active,zend_lifetime_interval * inactive,zend_lifetime_interval ** list,zend_lifetime_interval ** free)2063 static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_regset *hints, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free)
2064 {
2065 zend_lifetime_interval *it;
2066 uint32_t freeUntilPos[ZREG_NUM];
2067 uint32_t pos, pos2;
2068 zend_reg i, reg, reg2;
2069 zend_reg hint = ZREG_NONE;
2070 zend_regset low_priority_regs;
2071 zend_life_range *range;
2072
2073 if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) {
2074 available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_FP);
2075 } else {
2076 available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_GP);
2077 }
2078
2079 /* TODO: Allow usage of preserved registers ???
2080 * Their values have to be stored in prologue and restored in epilogue
2081 */
2082 available = ZEND_REGSET_DIFFERENCE(available, ZEND_REGSET_PRESERVED);
2083
2084 /* Set freeUntilPos of all physical registers to maxInt */
2085 for (i = 0; i < ZREG_NUM; i++) {
2086 freeUntilPos[i] = 0xffffffff;
2087 }
2088
2089 /* for each interval it in active do */
2090 /* freeUntilPos[it.reg] = 0 */
2091 it = active;
2092 if (ssa->vars[current->ssa_var].definition == current->range.start) {
2093 while (it) {
2094 if (current->range.start != zend_interval_end(it)) {
2095 freeUntilPos[it->reg] = 0;
2096 } else if (zend_jit_may_reuse_reg(
2097 ssa_opcodes ? ssa_opcodes[current->range.start] : op_array->opcodes + current->range.start,
2098 ssa->ops + current->range.start, ssa, current->ssa_var, it->ssa_var)) {
2099 if (!ZEND_REGSET_IN(*hints, it->reg) &&
2100 /* TODO: Avoid most often scratch registers. Find a better way ??? */
2101 (!current->used_as_hint ||
2102 !ZEND_REGSET_IN(ZEND_REGSET_LOW_PRIORITY, it->reg))) {
2103 hint = it->reg;
2104 }
2105 } else {
2106 freeUntilPos[it->reg] = 0;
2107 }
2108 it = it->list_next;
2109 }
2110 } else {
2111 while (it) {
2112 freeUntilPos[it->reg] = 0;
2113 it = it->list_next;
2114 }
2115 }
2116 if (current->hint) {
2117 hint = current->hint->reg;
2118 if (hint != ZREG_NONE && current->hint->used_as_hint == current) {
2119 ZEND_REGSET_EXCL(*hints, hint);
2120 }
2121 }
2122
2123 if (hint == ZREG_NONE && ZEND_REGSET_IS_EMPTY(available)) {
2124 return 0;
2125 }
2126
2127 /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
2128 Michael Franz, CGO'10 (2010), Figure 6. */
2129 if (current->flags & ZREG_SPLIT) {
2130 /* for each interval it in inactive intersecting with current do */
2131 /* freeUntilPos[it.reg] = next intersection of it with current */
2132 it = inactive;
2133 while (it) {
2134 uint32_t next = zend_interval_intersection(current, it);
2135
2136 //ZEND_ASSERT(next != 0xffffffff && !current->split);
2137 if (next < freeUntilPos[it->reg]) {
2138 freeUntilPos[it->reg] = next;
2139 }
2140 it = it->list_next;
2141 }
2142 }
2143
2144 /* Handle Scratch Registers */
2145 /* TODO: Optimize ??? */
2146 range = ¤t->range;
2147 do {
2148 uint32_t line = range->start;
2149 uint32_t last_use_line = (uint32_t)-1;
2150 zend_regset regset;
2151 zend_reg reg;
2152
2153 if ((current->flags & ZREG_LAST_USE) && !range->next) {
2154 last_use_line = range->end;
2155 }
2156 if (ssa->ops[line].op1_def == current->ssa_var ||
2157 ssa->ops[line].op2_def == current->ssa_var ||
2158 ssa->ops[line].result_def == current->ssa_var) {
2159 regset = zend_jit_get_def_scratch_regset(
2160 ssa_opcodes ? ssa_opcodes[line] : op_array->opcodes + line,
2161 ssa->ops + line,
2162 op_array, ssa, current->ssa_var, line == last_use_line);
2163 ZEND_REGSET_FOREACH(regset, reg) {
2164 if (line < freeUntilPos[reg]) {
2165 freeUntilPos[reg] = line;
2166 }
2167 } ZEND_REGSET_FOREACH_END();
2168 line++;
2169 }
2170 while (line <= range->end) {
2171 regset = zend_jit_get_scratch_regset(
2172 ssa_opcodes ? ssa_opcodes[line] : op_array->opcodes + line,
2173 ssa->ops + line,
2174 op_array, ssa, current->ssa_var, line == last_use_line);
2175 ZEND_REGSET_FOREACH(regset, reg) {
2176 if (line < freeUntilPos[reg]) {
2177 freeUntilPos[reg] = line;
2178 }
2179 } ZEND_REGSET_FOREACH_END();
2180 line++;
2181 }
2182 range = range->next;
2183 } while (range);
2184
2185 #if 0
2186 /* Coalescing */
2187 if (ssa->vars[current->ssa_var].definition == current->start) {
2188 zend_op *opline = op_array->opcodes + current->start;
2189 int hint = -1;
2190
2191 switch (opline->opcode) {
2192 case ZEND_ASSIGN:
2193 hint = ssa->ops[current->start].op2_use;
2194 case ZEND_QM_ASSIGN:
2195 hint = ssa->ops[current->start].op1_use;
2196 break;
2197 case ZEND_ADD:
2198 case ZEND_SUB:
2199 case ZEND_MUL:
2200 hint = ssa->ops[current->start].op1_use;
2201 break;
2202 case ZEND_ASSIGN_OP:
2203 if (opline->extended_value == ZEND_ADD
2204 || opline->extended_value == ZEND_SUB
2205 || opline->extended_value == ZEND_MUL) {
2206 hint = ssa->ops[current->start].op1_use;
2207 }
2208 break;
2209 }
2210 if (hint >= 0) {
2211 }
2212 }
2213 #endif
2214
2215 if (hint != ZREG_NONE && freeUntilPos[hint] > zend_interval_end(current)) {
2216 current->reg = hint;
2217 if (current->used_as_hint) {
2218 ZEND_REGSET_INCL(*hints, hint);
2219 }
2220 return 1;
2221 }
2222
2223 if (ZEND_REGSET_IS_EMPTY(available)) {
2224 return 0;
2225 }
2226
2227 pos = 0; reg = ZREG_NONE;
2228 pos2 = 0; reg2 = ZREG_NONE;
2229 low_priority_regs = *hints;
2230 if (current->used_as_hint) {
2231 /* TODO: Avoid most often scratch registers. Find a better way ??? */
2232 low_priority_regs = ZEND_REGSET_UNION(low_priority_regs, ZEND_REGSET_LOW_PRIORITY);
2233 }
2234
2235 ZEND_REGSET_FOREACH(available, i) {
2236 if (ZEND_REGSET_IN(low_priority_regs, i)) {
2237 if (freeUntilPos[i] > pos2) {
2238 reg2 = i;
2239 pos2 = freeUntilPos[i];
2240 }
2241 } else if (freeUntilPos[i] > pos) {
2242 reg = i;
2243 pos = freeUntilPos[i];
2244 }
2245 } ZEND_REGSET_FOREACH_END();
2246
2247 if (reg == ZREG_NONE) {
2248 if (reg2 != ZREG_NONE) {
2249 reg = reg2;
2250 pos = pos2;
2251 reg2 = ZREG_NONE;
2252 }
2253 }
2254
2255 if (reg == ZREG_NONE) {
2256 /* no register available without spilling */
2257 return 0;
2258 } else if (zend_interval_end(current) < pos) {
2259 /* register available for the whole interval */
2260 current->reg = reg;
2261 if (current->used_as_hint) {
2262 ZEND_REGSET_INCL(*hints, reg);
2263 }
2264 return 1;
2265 #if 0
2266 // TODO: allow low priority register usage
2267 } else if (reg2 != ZREG_NONE && zend_interval_end(current) < pos2) {
2268 /* register available for the whole interval */
2269 current->reg = reg2;
2270 if (current->used_as_hint) {
2271 ZEND_REGSET_INCL(*hints, reg2);
2272 }
2273 return 1;
2274 #endif
2275 } else {
2276 /* TODO: enable interval splitting ??? */
2277 /* register available for the first part of the interval */
2278 if (1 || zend_jit_split_interval(current, pos, list, free) != SUCCESS) {
2279 return 0;
2280 }
2281 current->reg = reg;
2282 if (current->used_as_hint) {
2283 ZEND_REGSET_INCL(*hints, reg);
2284 }
2285 return 1;
2286 }
2287 }
2288
2289 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2290 Christian Wimmer VEE'05 (2005), Figure 5. Allocation with spilling.
2291 and "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
2292 Michael Franz, CGO'10 (2010), Figure 6. */
zend_jit_allocate_blocked_reg(void)2293 static int zend_jit_allocate_blocked_reg(void)
2294 {
2295 /* TODO: ??? */
2296 return 0;
2297 }
2298
2299 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2300 Christian Wimmer VEE'10 (2005), Figure 2. */
zend_jit_linear_scan(const zend_op_array * op_array,const zend_op ** ssa_opcodes,zend_ssa * ssa,zend_lifetime_interval * list)2301 static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *list)
2302 {
2303 zend_lifetime_interval *unhandled, *active, *inactive, *handled, *free;
2304 zend_lifetime_interval *current, **p, *q;
2305 uint32_t position;
2306 zend_regset available = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP);
2307 zend_regset hints = ZEND_REGSET_EMPTY;
2308
2309 unhandled = list;
2310 /* active = inactive = handled = free = {} */
2311 active = inactive = handled = free = NULL;
2312 while (unhandled != NULL) {
2313 current = unhandled;
2314 unhandled = unhandled->list_next;
2315 position = current->range.start;
2316
2317 p = &active;
2318 while (*p) {
2319 uint32_t end = zend_interval_end(*p);
2320
2321 q = *p;
2322 if (end < position) {
2323 /* move ival from active to handled */
2324 ZEND_REGSET_INCL(available, q->reg);
2325 *p = q->list_next;
2326 q->list_next = handled;
2327 handled = q;
2328 } else if (!zend_interval_covers(q, position)) {
2329 /* move ival from active to inactive */
2330 ZEND_REGSET_INCL(available, q->reg);
2331 *p = q->list_next;
2332 q->list_next = inactive;
2333 inactive = q;
2334 } else {
2335 p = &q->list_next;
2336 }
2337 }
2338
2339 p = &inactive;
2340 while (*p) {
2341 uint32_t end = zend_interval_end(*p);
2342
2343 q = *p;
2344 if (end < position) {
2345 /* move ival from inactive to handled */
2346 *p = q->list_next;
2347 q->list_next = handled;
2348 handled = q;
2349 } else if (zend_interval_covers(q, position)) {
2350 /* move ival from inactive to active */
2351 ZEND_REGSET_EXCL(available, q->reg);
2352 *p = q->list_next;
2353 q->list_next = active;
2354 active = q;
2355 } else {
2356 p = &q->list_next;
2357 }
2358 }
2359
2360 if (zend_jit_try_allocate_free_reg(op_array, ssa_opcodes, ssa, current, available, &hints, active, inactive, &unhandled, &free) ||
2361 zend_jit_allocate_blocked_reg()) {
2362 ZEND_REGSET_EXCL(available, current->reg);
2363 current->list_next = active;
2364 active = current;
2365 } else {
2366 current->list_next = free;
2367 free = current;
2368 }
2369 }
2370
2371 /* move active to handled */
2372 while (active) {
2373 current = active;
2374 active = active->list_next;
2375 current->list_next = handled;
2376 handled = current;
2377 }
2378
2379 /* move inactive to handled */
2380 while (inactive) {
2381 current = inactive;
2382 inactive = inactive->list_next;
2383 current->list_next = handled;
2384 handled = current;
2385 }
2386
2387 return handled;
2388 }
2389
zend_jit_dump_lifetime_interval(const zend_op_array * op_array,const zend_ssa * ssa,const zend_lifetime_interval * ival)2390 static void zend_jit_dump_lifetime_interval(const zend_op_array *op_array, const zend_ssa *ssa, const zend_lifetime_interval *ival)
2391 {
2392 zend_life_range *range;
2393 int var_num = ssa->vars[ival->ssa_var].var;
2394
2395 fprintf(stderr, "#%d.", ival->ssa_var);
2396 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
2397 fprintf(stderr, ": %u-%u", ival->range.start, ival->range.end);
2398 range = ival->range.next;
2399 while (range) {
2400 fprintf(stderr, ", %u-%u", range->start, range->end);
2401 range = range->next;
2402 }
2403 if (ival->reg != ZREG_NONE) {
2404 fprintf(stderr, " (%s)", zend_reg_name[ival->reg]);
2405 }
2406 if (ival->flags & ZREG_LAST_USE) {
2407 fprintf(stderr, " last_use");
2408 }
2409 if (ival->flags & ZREG_LOAD) {
2410 fprintf(stderr, " load");
2411 }
2412 if (ival->flags & ZREG_STORE) {
2413 fprintf(stderr, " store");
2414 }
2415 if (ival->hint) {
2416 fprintf(stderr, " hint");
2417 if (ival->hint->ssa_var >= 0) {
2418 var_num = ssa->vars[ival->hint->ssa_var].var;
2419 fprintf(stderr, "=#%d.", ival->hint->ssa_var);
2420 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
2421 }
2422 if (ival->hint->reg != ZREG_NONE) {
2423 fprintf(stderr, " (%s)", zend_reg_name[ival->hint->reg]);
2424 }
2425 }
2426 fprintf(stderr, "\n");
2427 }
2428
zend_jit_allocate_registers(const zend_op_array * op_array,zend_ssa * ssa)2429 static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array *op_array, zend_ssa *ssa)
2430 {
2431 void *checkpoint;
2432 int set_size, candidates_count, i;
2433 zend_bitset candidates = NULL;
2434 zend_lifetime_interval *list, *ival;
2435 zend_lifetime_interval **intervals;
2436 ALLOCA_FLAG(use_heap);
2437
2438 if (!ssa->var_info) {
2439 return NULL;
2440 }
2441
2442 /* Identify SSA variables suitable for register allocation */
2443 set_size = zend_bitset_len(ssa->vars_count);
2444 candidates = ZEND_BITSET_ALLOCA(set_size, use_heap);
2445 if (!candidates) {
2446 return NULL;
2447 }
2448 candidates_count = 0;
2449 zend_bitset_clear(candidates, set_size);
2450 for (i = 0; i < ssa->vars_count; i++) {
2451 if (zend_jit_may_be_in_reg(op_array, ssa, i)) {
2452 zend_bitset_incl(candidates, i);
2453 candidates_count++;
2454 }
2455 }
2456 if (!candidates_count) {
2457 free_alloca(candidates, use_heap);
2458 return NULL;
2459 }
2460
2461 checkpoint = zend_arena_checkpoint(CG(arena));
2462
2463 /* Find life-time intervals */
2464 if (zend_jit_compute_liveness(op_array, ssa, candidates, &list) != SUCCESS) {
2465 goto failure;
2466 }
2467
2468 if (list) {
2469 /* Set ZREG_LAST_USE flags */
2470 ival = list;
2471 while (ival) {
2472 zend_life_range *range = &ival->range;
2473
2474 while (range->next) {
2475 range = range->next;
2476 }
2477 if (zend_ssa_is_last_use(op_array, ssa, ival->ssa_var, range->end)) {
2478 ival->flags |= ZREG_LAST_USE;
2479 }
2480 ival = ival->list_next;
2481 }
2482 }
2483
2484 if (list) {
2485 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
2486 fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
2487 ival = list;
2488 while (ival) {
2489 zend_jit_dump_lifetime_interval(op_array, ssa, ival);
2490 ival = ival->list_next;
2491 }
2492 fprintf(stderr, "\n");
2493 }
2494
2495 /* Linear Scan Register Allocation */
2496 list = zend_jit_linear_scan(op_array, NULL, ssa, list);
2497
2498 if (list) {
2499 intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*));
2500 if (!intervals) {
2501 goto failure;
2502 }
2503
2504 ival = list;
2505 while (ival != NULL) {
2506 zend_lifetime_interval *next = ival->list_next;
2507
2508 ival->list_next = intervals[ival->ssa_var];
2509 intervals[ival->ssa_var] = ival;
2510 ival = next;
2511 }
2512
2513 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
2514 /* Naive SSA resolution */
2515 for (i = 0; i < ssa->vars_count; i++) {
2516 if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
2517 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
2518 int k, src;
2519
2520 if (phi->pi >= 0) {
2521 if (!ssa->vars[i].phi_use_chain
2522 || ssa->vars[i].phi_use_chain->block != phi->block) {
2523 src = phi->sources[0];
2524 if (intervals[i]) {
2525 if (!intervals[src]) {
2526 intervals[i]->flags |= ZREG_LOAD;
2527 } else if (intervals[i]->reg != intervals[src]->reg) {
2528 intervals[i]->flags |= ZREG_LOAD;
2529 intervals[src]->flags |= ZREG_STORE;
2530 }
2531 } else if (intervals[src]) {
2532 intervals[src]->flags |= ZREG_STORE;
2533 }
2534 }
2535 } else {
2536 int need_move = 0;
2537
2538 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
2539 src = phi->sources[k];
2540 if (src >= 0) {
2541 if (ssa->vars[src].definition_phi
2542 && ssa->vars[src].definition_phi->pi >= 0
2543 && phi->block == ssa->vars[src].definition_phi->block) {
2544 /* Skip zero-length interval for Pi variable */
2545 src = ssa->vars[src].definition_phi->sources[0];
2546 }
2547 if (intervals[i]) {
2548 if (!intervals[src]) {
2549 need_move = 1;
2550 } else if (intervals[i]->reg != intervals[src]->reg) {
2551 need_move = 1;
2552 }
2553 } else if (intervals[src]) {
2554 need_move = 1;
2555 }
2556 }
2557 }
2558 if (need_move) {
2559 if (intervals[i]) {
2560 intervals[i]->flags |= ZREG_LOAD;
2561 }
2562 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
2563 src = phi->sources[k];
2564 if (src >= 0) {
2565 if (ssa->vars[src].definition_phi
2566 && ssa->vars[src].definition_phi->pi >= 0
2567 && phi->block == ssa->vars[src].definition_phi->block) {
2568 /* Skip zero-length interval for Pi variable */
2569 src = ssa->vars[src].definition_phi->sources[0];
2570 }
2571 if (intervals[src]) {
2572 intervals[src]->flags |= ZREG_STORE;
2573 }
2574 }
2575 }
2576 }
2577 }
2578 }
2579 }
2580 /* Remove useless register allocation */
2581 for (i = 0; i < ssa->vars_count; i++) {
2582 if (intervals[i] &&
2583 ((intervals[i]->flags & ZREG_LOAD) ||
2584 ((intervals[i]->flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) &&
2585 ssa->vars[i].use_chain < 0) {
2586 bool may_remove = 1;
2587 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
2588
2589 while (phi) {
2590 if (intervals[phi->ssa_var] &&
2591 !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) {
2592 may_remove = 0;
2593 break;
2594 }
2595 phi = zend_ssa_next_use_phi(ssa, i, phi);
2596 }
2597 if (may_remove) {
2598 intervals[i] = NULL;
2599 }
2600 }
2601 }
2602 /* Remove intervals used once */
2603 for (i = 0; i < ssa->vars_count; i++) {
2604 if (intervals[i] &&
2605 (intervals[i]->flags & ZREG_LOAD) &&
2606 (intervals[i]->flags & ZREG_STORE) &&
2607 (ssa->vars[i].use_chain < 0 ||
2608 zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) {
2609 bool may_remove = 1;
2610 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
2611
2612 while (phi) {
2613 if (intervals[phi->ssa_var] &&
2614 !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) {
2615 may_remove = 0;
2616 break;
2617 }
2618 phi = zend_ssa_next_use_phi(ssa, i, phi);
2619 }
2620 if (may_remove) {
2621 intervals[i] = NULL;
2622 }
2623 }
2624 }
2625 }
2626
2627 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
2628 fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
2629 for (i = 0; i < ssa->vars_count; i++) {
2630 ival = intervals[i];
2631 while (ival) {
2632 zend_jit_dump_lifetime_interval(op_array, ssa, ival);
2633 ival = ival->list_next;
2634 }
2635 }
2636 fprintf(stderr, "\n");
2637 }
2638
2639 free_alloca(candidates, use_heap);
2640 return intervals;
2641 }
2642 }
2643
2644 failure:
2645 zend_arena_release(&CG(arena), checkpoint);
2646 free_alloca(candidates, use_heap);
2647 return NULL;
2648 }
2649
zend_jit_next_is_send_result(const zend_op * opline)2650 static bool zend_jit_next_is_send_result(const zend_op *opline)
2651 {
2652 if (opline->result_type == IS_TMP_VAR
2653 && (opline+1)->opcode == ZEND_SEND_VAL
2654 && (opline+1)->op1_type == IS_TMP_VAR
2655 && (opline+1)->op2_type != IS_CONST
2656 && (opline+1)->op1.var == opline->result.var) {
2657 return 1;
2658 }
2659 return 0;
2660 }
2661
zend_jit_supported_binary_op(zend_uchar op,uint32_t op1_info,uint32_t op2_info)2662 static bool zend_jit_supported_binary_op(zend_uchar op, uint32_t op1_info, uint32_t op2_info)
2663 {
2664 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2665 return false;
2666 }
2667 switch (op) {
2668 case ZEND_POW:
2669 case ZEND_DIV:
2670 // TODO: check for division by zero ???
2671 return false;
2672 case ZEND_ADD:
2673 case ZEND_SUB:
2674 case ZEND_MUL:
2675 return (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
2676 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
2677 case ZEND_BW_OR:
2678 case ZEND_BW_AND:
2679 case ZEND_BW_XOR:
2680 case ZEND_SL:
2681 case ZEND_SR:
2682 case ZEND_MOD:
2683 return (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG);
2684 case ZEND_CONCAT:
2685 return (op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING);
2686 EMPTY_SWITCH_DEFAULT_CASE()
2687 }
2688 }
2689
zend_jit(const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline)2690 static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline)
2691 {
2692 int b, i, end;
2693 zend_op *opline;
2694 dasm_State* dasm_state = NULL;
2695 void *handler;
2696 int call_level = 0;
2697 void *checkpoint = NULL;
2698 zend_lifetime_interval **ra = NULL;
2699 bool is_terminated = 1; /* previous basic block is terminated by jump */
2700 bool recv_emitted = 0; /* emitted at least one RECV opcode */
2701 zend_uchar smart_branch_opcode;
2702 uint32_t target_label, target_label2;
2703 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info;
2704 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
2705 zend_class_entry *ce;
2706 bool ce_is_instanceof;
2707 bool on_this;
2708
2709 if (JIT_G(bisect_limit)) {
2710 jit_bisect_pos++;
2711 if (jit_bisect_pos >= JIT_G(bisect_limit)) {
2712 if (jit_bisect_pos == JIT_G(bisect_limit)) {
2713 fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n",
2714 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
2715 op_array->scope ? "::" : "",
2716 op_array->function_name ? ZSTR_VAL(op_array->function_name) : "{main}",
2717 ZSTR_VAL(op_array->filename), op_array->line_start);
2718 }
2719 return FAILURE;
2720 }
2721 }
2722
2723 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2724 checkpoint = zend_arena_checkpoint(CG(arena));
2725 ra = zend_jit_allocate_registers(op_array, ssa);
2726 }
2727
2728 /* mark hidden branch targets */
2729 for (b = 0; b < ssa->cfg.blocks_count; b++) {
2730 if (ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE &&
2731 ssa->cfg.blocks[b].len > 1) {
2732
2733 opline = op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
2734 if (opline->opcode == ZEND_DO_FCALL &&
2735 (opline-1)->opcode == ZEND_NEW) {
2736 ssa->cfg.blocks[ssa->cfg.blocks[b].successors[0]].flags |= ZEND_BB_TARGET;
2737 }
2738 }
2739 }
2740
2741 dasm_init(&dasm_state, DASM_MAXSECTION);
2742 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
2743 dasm_setup(&dasm_state, dasm_actions);
2744
2745 dasm_growpc(&dasm_state, ssa->cfg.blocks_count * 2 + 1);
2746
2747 zend_jit_align_func(&dasm_state);
2748 for (b = 0; b < ssa->cfg.blocks_count; b++) {
2749 if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) {
2750 continue;
2751 }
2752 //#ifndef CONTEXT_THREADED_JIT
2753 if (ssa->cfg.blocks[b].flags & ZEND_BB_ENTRY) {
2754 if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
2755 /* pass */
2756 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2757 ssa->cfg.blocks[b].len == 1 &&
2758 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT) &&
2759 op_array->opcodes[ssa->cfg.blocks[b].start].opcode != ZEND_JMP) {
2760 /* don't generate code for BB with single opcode */
2761 continue;
2762 }
2763 if (ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW) {
2764 if (!is_terminated) {
2765 zend_jit_jmp(&dasm_state, b);
2766 }
2767 }
2768 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2769 zend_jit_prologue(&dasm_state);
2770 } else
2771 //#endif
2772 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
2773 opline = op_array->opcodes + ssa->cfg.blocks[b].start;
2774 if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) {
2775 if (opline->opcode == ZEND_RECV_INIT) {
2776 if (opline == op_array->opcodes ||
2777 (opline-1)->opcode != ZEND_RECV_INIT) {
2778 if (recv_emitted) {
2779 zend_jit_jmp(&dasm_state, b);
2780 }
2781 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2782 for (i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) {
2783 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b + i);
2784 }
2785 zend_jit_prologue(&dasm_state);
2786 }
2787 recv_emitted = 1;
2788 } else if (opline->opcode == ZEND_RECV) {
2789 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2790 /* skip */
2791 continue;
2792 } else if (recv_emitted) {
2793 zend_jit_jmp(&dasm_state, b);
2794 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2795 zend_jit_prologue(&dasm_state);
2796 } else {
2797 zend_arg_info *arg_info;
2798
2799 if (opline->op1.num <= op_array->num_args) {
2800 arg_info = &op_array->arg_info[opline->op1.num - 1];
2801 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
2802 arg_info = &op_array->arg_info[op_array->num_args];
2803 } else {
2804 /* skip */
2805 continue;
2806 }
2807 if (!ZEND_TYPE_IS_SET(arg_info->type)) {
2808 /* skip */
2809 continue;
2810 }
2811 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2812 zend_jit_prologue(&dasm_state);
2813 recv_emitted = 1;
2814 }
2815 } else {
2816 if (recv_emitted) {
2817 zend_jit_jmp(&dasm_state, b);
2818 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2819 ssa->cfg.blocks[b].len == 1 &&
2820 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
2821 /* don't generate code for BB with single opcode */
2822 dasm_free(&dasm_state);
2823
2824 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2825 zend_arena_release(&CG(arena), checkpoint);
2826 }
2827 return SUCCESS;
2828 }
2829 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2830 zend_jit_prologue(&dasm_state);
2831 recv_emitted = 1;
2832 }
2833 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2834 ssa->cfg.blocks[b].len == 1 &&
2835 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
2836 /* don't generate code for BB with single opcode */
2837 dasm_free(&dasm_state);
2838
2839 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2840 zend_arena_release(&CG(arena), checkpoint);
2841 }
2842 return SUCCESS;
2843 } else {
2844 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2845 zend_jit_prologue(&dasm_state);
2846 }
2847 }
2848
2849 is_terminated = 0;
2850
2851 zend_jit_label(&dasm_state, b);
2852 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
2853 if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
2854 && ssa->cfg.blocks[b].start != 0
2855 && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
2856 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_LONG
2857 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_STRING
2858 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_MATCH)) {
2859 zend_jit_reset_last_valid_opline();
2860 if (!zend_jit_set_ip(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) {
2861 goto jit_failure;
2862 }
2863 } else {
2864 zend_jit_set_last_valid_opline(op_array->opcodes + ssa->cfg.blocks[b].start);
2865 }
2866 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
2867 zend_jit_reset_last_valid_opline();
2868 } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY|ZEND_BB_ENTRY)) {
2869 zend_jit_set_last_valid_opline(op_array->opcodes + ssa->cfg.blocks[b].start);
2870 }
2871 if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) {
2872 if (!zend_jit_check_timeout(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start, NULL)) {
2873 goto jit_failure;
2874 }
2875 }
2876 if (!ssa->cfg.blocks[b].len) {
2877 continue;
2878 }
2879 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
2880 zend_ssa_phi *phi = ssa->blocks[b].phis;
2881
2882 while (phi) {
2883 zend_lifetime_interval *ival = ra[phi->ssa_var];
2884
2885 if (ival) {
2886 if (ival->flags & ZREG_LOAD) {
2887 ZEND_ASSERT(ival->reg != ZREG_NONE);
2888
2889 if (!zend_jit_load_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg)) {
2890 goto jit_failure;
2891 }
2892 } else if (ival->flags & ZREG_STORE) {
2893 ZEND_ASSERT(ival->reg != ZREG_NONE);
2894
2895 if (!zend_jit_store_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg, 1)) {
2896 goto jit_failure;
2897 }
2898 }
2899 }
2900 phi = phi->next;
2901 }
2902 }
2903 end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
2904 for (i = ssa->cfg.blocks[b].start; i <= end; i++) {
2905 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[i] : NULL;
2906 opline = op_array->opcodes + i;
2907 switch (opline->opcode) {
2908 case ZEND_INIT_FCALL:
2909 case ZEND_INIT_FCALL_BY_NAME:
2910 case ZEND_INIT_NS_FCALL_BY_NAME:
2911 case ZEND_INIT_METHOD_CALL:
2912 case ZEND_INIT_DYNAMIC_CALL:
2913 case ZEND_INIT_STATIC_METHOD_CALL:
2914 case ZEND_INIT_USER_CALL:
2915 case ZEND_NEW:
2916 call_level++;
2917 }
2918
2919 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
2920 switch (opline->opcode) {
2921 case ZEND_PRE_INC:
2922 case ZEND_PRE_DEC:
2923 case ZEND_POST_INC:
2924 case ZEND_POST_DEC:
2925 if (opline->op1_type != IS_CV) {
2926 break;
2927 }
2928 op1_info = OP1_INFO();
2929 if (!(op1_info & MAY_BE_LONG)) {
2930 break;
2931 }
2932 if (opline->result_type != IS_UNUSED) {
2933 res_use_info = -1;
2934
2935 if (opline->result_type == IS_CV
2936 && ssa->vars
2937 && ssa_op->result_use >= 0
2938 && !ssa->vars[ssa_op->result_use].no_val) {
2939 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
2940
2941 if (Z_MODE(res_use_addr) != IS_REG
2942 || Z_LOAD(res_use_addr)
2943 || Z_STORE(res_use_addr)) {
2944 res_use_info = RES_USE_INFO();
2945 }
2946 }
2947 res_info = RES_INFO();
2948 res_addr = RES_REG_ADDR();
2949 } else {
2950 res_use_info = -1;
2951 res_info = -1;
2952 res_addr = 0;
2953 }
2954 op1_def_info = OP1_DEF_INFO();
2955 if (!zend_jit_inc_dec(&dasm_state, opline,
2956 op1_info, OP1_REG_ADDR(),
2957 op1_def_info, OP1_DEF_REG_ADDR(),
2958 res_use_info, res_info,
2959 res_addr,
2960 (op1_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
2961 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2962 goto jit_failure;
2963 }
2964 goto done;
2965 case ZEND_BW_OR:
2966 case ZEND_BW_AND:
2967 case ZEND_BW_XOR:
2968 case ZEND_SL:
2969 case ZEND_SR:
2970 case ZEND_MOD:
2971 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2972 break;
2973 }
2974 op1_info = OP1_INFO();
2975 op2_info = OP2_INFO();
2976 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2977 break;
2978 }
2979 if (!(op1_info & MAY_BE_LONG)
2980 || !(op2_info & MAY_BE_LONG)) {
2981 break;
2982 }
2983 res_addr = RES_REG_ADDR();
2984 if (Z_MODE(res_addr) != IS_REG
2985 && (i + 1) <= end
2986 && zend_jit_next_is_send_result(opline)) {
2987 i++;
2988 res_use_info = -1;
2989 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
2990 if (!zend_jit_reuse_ip(&dasm_state)) {
2991 goto jit_failure;
2992 }
2993 } else {
2994 res_use_info = -1;
2995
2996 if (opline->result_type == IS_CV
2997 && ssa->vars
2998 && ssa_op->result_use >= 0
2999 && !ssa->vars[ssa_op->result_use].no_val) {
3000 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
3001
3002 if (Z_MODE(res_use_addr) != IS_REG
3003 || Z_LOAD(res_use_addr)
3004 || Z_STORE(res_use_addr)) {
3005 res_use_info = RES_USE_INFO();
3006 }
3007 }
3008 }
3009 if (!zend_jit_long_math(&dasm_state, opline,
3010 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
3011 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
3012 res_use_info, RES_INFO(), res_addr,
3013 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3014 goto jit_failure;
3015 }
3016 goto done;
3017 case ZEND_ADD:
3018 case ZEND_SUB:
3019 case ZEND_MUL:
3020 // case ZEND_DIV: // TODO: check for division by zero ???
3021 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3022 break;
3023 }
3024 op1_info = OP1_INFO();
3025 op2_info = OP2_INFO();
3026 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
3027 break;
3028 }
3029 if (opline->opcode == ZEND_ADD &&
3030 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
3031 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
3032 /* pass */
3033 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
3034 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
3035 break;
3036 }
3037 res_addr = RES_REG_ADDR();
3038 if (Z_MODE(res_addr) != IS_REG
3039 && (i + 1) <= end
3040 && zend_jit_next_is_send_result(opline)) {
3041 i++;
3042 res_use_info = -1;
3043 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3044 if (!zend_jit_reuse_ip(&dasm_state)) {
3045 goto jit_failure;
3046 }
3047 } else {
3048 res_use_info = -1;
3049
3050 if (opline->result_type == IS_CV
3051 && ssa->vars
3052 && ssa_op->result_use >= 0
3053 && !ssa->vars[ssa_op->result_use].no_val) {
3054 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
3055
3056 if (Z_MODE(res_use_addr) != IS_REG
3057 || Z_LOAD(res_use_addr)
3058 || Z_STORE(res_use_addr)) {
3059 res_use_info = RES_USE_INFO();
3060 }
3061 }
3062 }
3063 res_info = RES_INFO();
3064 if (opline->opcode == ZEND_ADD &&
3065 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
3066 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
3067 if (!zend_jit_add_arrays(&dasm_state, opline, op1_info, OP1_REG_ADDR(), op2_info, OP2_REG_ADDR(), res_addr)) {
3068 goto jit_failure;
3069 }
3070 } else {
3071 if (!zend_jit_math(&dasm_state, opline,
3072 op1_info, OP1_REG_ADDR(),
3073 op2_info, OP2_REG_ADDR(),
3074 res_use_info, res_info, res_addr,
3075 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
3076 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3077 goto jit_failure;
3078 }
3079 }
3080 goto done;
3081 case ZEND_CONCAT:
3082 case ZEND_FAST_CONCAT:
3083 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3084 break;
3085 }
3086 op1_info = OP1_INFO();
3087 op2_info = OP2_INFO();
3088 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
3089 break;
3090 }
3091 if (!(op1_info & MAY_BE_STRING) ||
3092 !(op2_info & MAY_BE_STRING)) {
3093 break;
3094 }
3095 res_addr = RES_REG_ADDR();
3096 if ((i + 1) <= end
3097 && zend_jit_next_is_send_result(opline)) {
3098 i++;
3099 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3100 if (!zend_jit_reuse_ip(&dasm_state)) {
3101 goto jit_failure;
3102 }
3103 }
3104 if (!zend_jit_concat(&dasm_state, opline,
3105 op1_info, op2_info, res_addr,
3106 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3107 goto jit_failure;
3108 }
3109 goto done;
3110 case ZEND_ASSIGN_OP:
3111 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
3112 break;
3113 }
3114 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3115 break;
3116 }
3117 op1_info = OP1_INFO();
3118 op2_info = OP2_INFO();
3119 if (!zend_jit_supported_binary_op(
3120 opline->extended_value, op1_info, op2_info)) {
3121 break;
3122 }
3123 op1_def_info = OP1_DEF_INFO();
3124 if (!zend_jit_assign_op(&dasm_state, opline,
3125 op1_info, op1_def_info, OP1_RANGE(),
3126 op2_info, OP2_RANGE(),
3127 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
3128 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3129 goto jit_failure;
3130 }
3131 goto done;
3132 case ZEND_ASSIGN_DIM_OP:
3133 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
3134 break;
3135 }
3136 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3137 break;
3138 }
3139 if (!zend_jit_supported_binary_op(
3140 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3141 break;
3142 }
3143 if (!zend_jit_assign_dim_op(&dasm_state, opline,
3144 OP1_INFO(), OP1_DEF_INFO(), OP1_REG_ADDR(), OP2_INFO(),
3145 OP1_DATA_INFO(), OP1_DATA_RANGE(), IS_UNKNOWN,
3146 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3147 goto jit_failure;
3148 }
3149 goto done;
3150 case ZEND_ASSIGN_DIM:
3151 if (opline->op1_type != IS_CV) {
3152 break;
3153 }
3154 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3155 break;
3156 }
3157 if (!zend_jit_assign_dim(&dasm_state, opline,
3158 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), OP1_DATA_INFO(), IS_UNKNOWN,
3159 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3160 goto jit_failure;
3161 }
3162 goto done;
3163 case ZEND_PRE_INC_OBJ:
3164 case ZEND_PRE_DEC_OBJ:
3165 case ZEND_POST_INC_OBJ:
3166 case ZEND_POST_DEC_OBJ:
3167 if (opline->op2_type != IS_CONST
3168 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3169 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3170 break;
3171 }
3172 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3173 break;
3174 }
3175 ce = NULL;
3176 ce_is_instanceof = 0;
3177 on_this = 0;
3178 if (opline->op1_type == IS_UNUSED) {
3179 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3180 ce = op_array->scope;
3181 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3182 op1_addr = 0;
3183 on_this = 1;
3184 } else {
3185 op1_info = OP1_INFO();
3186 if (!(op1_info & MAY_BE_OBJECT)) {
3187 break;
3188 }
3189 op1_addr = OP1_REG_ADDR();
3190 if (ssa->var_info && ssa->ops) {
3191 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3192 if (ssa_op->op1_use >= 0) {
3193 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3194 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3195 ce = op1_ssa->ce;
3196 ce_is_instanceof = op1_ssa->is_instanceof;
3197 }
3198 }
3199 }
3200 }
3201 if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3202 op1_info, op1_addr,
3203 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
3204 goto jit_failure;
3205 }
3206 goto done;
3207 case ZEND_ASSIGN_OBJ_OP:
3208 if (opline->result_type != IS_UNUSED) {
3209 break;
3210 }
3211 if (opline->op2_type != IS_CONST
3212 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3213 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3214 break;
3215 }
3216 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3217 break;
3218 }
3219 if (!zend_jit_supported_binary_op(
3220 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3221 break;
3222 }
3223 ce = NULL;
3224 ce_is_instanceof = 0;
3225 on_this = 0;
3226 if (opline->op1_type == IS_UNUSED) {
3227 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3228 ce = op_array->scope;
3229 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3230 op1_addr = 0;
3231 on_this = 1;
3232 } else {
3233 op1_info = OP1_INFO();
3234 if (!(op1_info & MAY_BE_OBJECT)) {
3235 break;
3236 }
3237 op1_addr = OP1_REG_ADDR();
3238 if (ssa->var_info && ssa->ops) {
3239 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3240 if (ssa_op->op1_use >= 0) {
3241 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3242 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3243 ce = op1_ssa->ce;
3244 ce_is_instanceof = op1_ssa->is_instanceof;
3245 }
3246 }
3247 }
3248 }
3249 if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op,
3250 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_RANGE(),
3251 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
3252 goto jit_failure;
3253 }
3254 goto done;
3255 case ZEND_ASSIGN_OBJ:
3256 if (opline->op2_type != IS_CONST
3257 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3258 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3259 break;
3260 }
3261 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3262 break;
3263 }
3264 ce = NULL;
3265 ce_is_instanceof = 0;
3266 on_this = 0;
3267 if (opline->op1_type == IS_UNUSED) {
3268 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3269 ce = op_array->scope;
3270 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3271 op1_addr = 0;
3272 on_this = 1;
3273 } else {
3274 op1_info = OP1_INFO();
3275 if (!(op1_info & MAY_BE_OBJECT)) {
3276 break;
3277 }
3278 op1_addr = OP1_REG_ADDR();
3279 if (ssa->var_info && ssa->ops) {
3280 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3281 if (ssa_op->op1_use >= 0) {
3282 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3283 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3284 ce = op1_ssa->ce;
3285 ce_is_instanceof = op1_ssa->is_instanceof;
3286 }
3287 }
3288 }
3289 }
3290 if (!zend_jit_assign_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3291 op1_info, op1_addr, OP1_DATA_INFO(),
3292 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN,
3293 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3294 goto jit_failure;
3295 }
3296 goto done;
3297 case ZEND_ASSIGN:
3298 if (opline->op1_type != IS_CV) {
3299 break;
3300 }
3301 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3302 break;
3303 }
3304 op2_addr = OP2_REG_ADDR();
3305 if (ra
3306 && ssa->ops[opline - op_array->opcodes].op2_def >= 0
3307 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) {
3308 op2_def_addr = OP2_DEF_REG_ADDR();
3309 } else {
3310 op2_def_addr = op2_addr;
3311 }
3312 op1_info = OP1_INFO();
3313 if (ra && ssa->vars[ssa_op->op1_use].no_val) {
3314 op1_info |= MAY_BE_UNDEF; // requres type assignment
3315 }
3316 if (opline->result_type == IS_UNUSED) {
3317 res_addr = 0;
3318 res_info = -1;
3319 } else {
3320 res_addr = RES_REG_ADDR();
3321 res_info = RES_INFO();
3322 if (Z_MODE(res_addr) != IS_REG
3323 && (i + 1) <= end
3324 && zend_jit_next_is_send_result(opline)
3325 && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) {
3326 i++;
3327 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3328 if (!zend_jit_reuse_ip(&dasm_state)) {
3329 goto jit_failure;
3330 }
3331 }
3332 }
3333 if (!zend_jit_assign(&dasm_state, opline,
3334 op1_info, OP1_REG_ADDR(),
3335 OP1_DEF_INFO(), OP1_DEF_REG_ADDR(),
3336 OP2_INFO(), op2_addr, op2_def_addr,
3337 res_info, res_addr,
3338 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3339 goto jit_failure;
3340 }
3341 goto done;
3342 case ZEND_QM_ASSIGN:
3343 op1_addr = OP1_REG_ADDR();
3344 if (ra
3345 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
3346 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
3347 op1_def_addr = OP1_DEF_REG_ADDR();
3348 } else {
3349 op1_def_addr = op1_addr;
3350 }
3351 if (!zend_jit_qm_assign(&dasm_state, opline,
3352 OP1_INFO(), op1_addr, op1_def_addr,
3353 -1, RES_INFO(), RES_REG_ADDR())) {
3354 goto jit_failure;
3355 }
3356 goto done;
3357 case ZEND_INIT_FCALL:
3358 case ZEND_INIT_FCALL_BY_NAME:
3359 case ZEND_INIT_NS_FCALL_BY_NAME:
3360 if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level, NULL, 0)) {
3361 goto jit_failure;
3362 }
3363 goto done;
3364 case ZEND_SEND_VAL:
3365 case ZEND_SEND_VAL_EX:
3366 if (opline->op2_type == IS_CONST) {
3367 /* Named parameters not supported in JIT (yet) */
3368 break;
3369 }
3370 if (opline->opcode == ZEND_SEND_VAL_EX
3371 && opline->op2.num > MAX_ARG_FLAG_NUM) {
3372 break;
3373 }
3374 if (!zend_jit_send_val(&dasm_state, opline,
3375 OP1_INFO(), OP1_REG_ADDR())) {
3376 goto jit_failure;
3377 }
3378 goto done;
3379 case ZEND_SEND_REF:
3380 if (opline->op2_type == IS_CONST) {
3381 /* Named parameters not supported in JIT (yet) */
3382 break;
3383 }
3384 if (!zend_jit_send_ref(&dasm_state, opline, op_array,
3385 OP1_INFO(), 0)) {
3386 goto jit_failure;
3387 }
3388 goto done;
3389 case ZEND_SEND_VAR:
3390 case ZEND_SEND_VAR_EX:
3391 case ZEND_SEND_VAR_NO_REF:
3392 case ZEND_SEND_VAR_NO_REF_EX:
3393 case ZEND_SEND_FUNC_ARG:
3394 if (opline->op2_type == IS_CONST) {
3395 /* Named parameters not supported in JIT (yet) */
3396 break;
3397 }
3398 if ((opline->opcode == ZEND_SEND_VAR_EX
3399 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
3400 && opline->op2.num > MAX_ARG_FLAG_NUM) {
3401 break;
3402 }
3403 op1_addr = OP1_REG_ADDR();
3404 if (ra
3405 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
3406 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
3407 op1_def_addr = OP1_DEF_REG_ADDR();
3408 } else {
3409 op1_def_addr = op1_addr;
3410 }
3411 if (!zend_jit_send_var(&dasm_state, opline, op_array,
3412 OP1_INFO(), op1_addr, op1_def_addr)) {
3413 goto jit_failure;
3414 }
3415 goto done;
3416 case ZEND_CHECK_FUNC_ARG:
3417 if (opline->op2_type == IS_CONST) {
3418 /* Named parameters not supported in JIT (yet) */
3419 break;
3420 }
3421 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
3422 break;
3423 }
3424 if (!zend_jit_check_func_arg(&dasm_state, opline)) {
3425 goto jit_failure;
3426 }
3427 goto done;
3428 case ZEND_CHECK_UNDEF_ARGS:
3429 if (!zend_jit_check_undef_args(&dasm_state, opline)) {
3430 goto jit_failure;
3431 }
3432 goto done;
3433 case ZEND_DO_UCALL:
3434 is_terminated = 1;
3435 ZEND_FALLTHROUGH;
3436 case ZEND_DO_ICALL:
3437 case ZEND_DO_FCALL_BY_NAME:
3438 case ZEND_DO_FCALL:
3439 if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level, b + 1, NULL)) {
3440 goto jit_failure;
3441 }
3442 goto done;
3443 case ZEND_IS_EQUAL:
3444 case ZEND_IS_NOT_EQUAL:
3445 case ZEND_IS_SMALLER:
3446 case ZEND_IS_SMALLER_OR_EQUAL:
3447 case ZEND_CASE: {
3448 res_addr = RES_REG_ADDR();
3449 if ((opline->result_type & IS_TMP_VAR)
3450 && (i + 1) <= end
3451 && ((opline+1)->opcode == ZEND_JMPZ
3452 || (opline+1)->opcode == ZEND_JMPNZ
3453 || (opline+1)->opcode == ZEND_JMPZ_EX
3454 || (opline+1)->opcode == ZEND_JMPNZ_EX
3455 || (opline+1)->opcode == ZEND_JMPZNZ)
3456 && (opline+1)->op1_type == IS_TMP_VAR
3457 && (opline+1)->op1.var == opline->result.var) {
3458 i++;
3459 smart_branch_opcode = (opline+1)->opcode;
3460 target_label = ssa->cfg.blocks[b].successors[0];
3461 target_label2 = ssa->cfg.blocks[b].successors[1];
3462 /* For EX variant write into the result of EX opcode. */
3463 if ((opline+1)->opcode == ZEND_JMPZ_EX
3464 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
3465 res_addr = OP_REG_ADDR(opline + 1, result_type, result, result_def);
3466 }
3467 } else {
3468 smart_branch_opcode = 0;
3469 target_label = target_label2 = (uint32_t)-1;
3470 }
3471 if (!zend_jit_cmp(&dasm_state, opline,
3472 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
3473 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
3474 res_addr,
3475 zend_may_throw(opline, ssa_op, op_array, ssa),
3476 smart_branch_opcode, target_label, target_label2,
3477 NULL, 0)) {
3478 goto jit_failure;
3479 }
3480 goto done;
3481 }
3482 case ZEND_IS_IDENTICAL:
3483 case ZEND_IS_NOT_IDENTICAL:
3484 case ZEND_CASE_STRICT:
3485 if ((opline->result_type & IS_TMP_VAR)
3486 && (i + 1) <= end
3487 && ((opline+1)->opcode == ZEND_JMPZ
3488 || (opline+1)->opcode == ZEND_JMPNZ
3489 || (opline+1)->opcode == ZEND_JMPZNZ)
3490 && (opline+1)->op1_type == IS_TMP_VAR
3491 && (opline+1)->op1.var == opline->result.var) {
3492 i++;
3493 smart_branch_opcode = (opline+1)->opcode;
3494 target_label = ssa->cfg.blocks[b].successors[0];
3495 target_label2 = ssa->cfg.blocks[b].successors[1];
3496 } else {
3497 smart_branch_opcode = 0;
3498 target_label = target_label2 = (uint32_t)-1;
3499 }
3500 if (!zend_jit_identical(&dasm_state, opline,
3501 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
3502 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
3503 RES_REG_ADDR(),
3504 zend_may_throw(opline, ssa_op, op_array, ssa),
3505 smart_branch_opcode, target_label, target_label2,
3506 NULL, 0)) {
3507 goto jit_failure;
3508 }
3509 goto done;
3510 case ZEND_DEFINED:
3511 if ((opline->result_type & IS_TMP_VAR)
3512 && (i + 1) <= end
3513 && ((opline+1)->opcode == ZEND_JMPZ
3514 || (opline+1)->opcode == ZEND_JMPNZ
3515 || (opline+1)->opcode == ZEND_JMPZNZ)
3516 && (opline+1)->op1_type == IS_TMP_VAR
3517 && (opline+1)->op1.var == opline->result.var) {
3518 i++;
3519 smart_branch_opcode = (opline+1)->opcode;
3520 target_label = ssa->cfg.blocks[b].successors[0];
3521 target_label2 = ssa->cfg.blocks[b].successors[1];
3522 } else {
3523 smart_branch_opcode = 0;
3524 target_label = target_label2 = (uint32_t)-1;
3525 }
3526 if (!zend_jit_defined(&dasm_state, opline, smart_branch_opcode, target_label, target_label2, NULL)) {
3527 goto jit_failure;
3528 }
3529 goto done;
3530 case ZEND_TYPE_CHECK:
3531 if (opline->extended_value == MAY_BE_RESOURCE) {
3532 // TODO: support for is_resource() ???
3533 break;
3534 }
3535 if ((opline->result_type & IS_TMP_VAR)
3536 && (i + 1) <= end
3537 && ((opline+1)->opcode == ZEND_JMPZ
3538 || (opline+1)->opcode == ZEND_JMPNZ
3539 || (opline+1)->opcode == ZEND_JMPZNZ)
3540 && (opline+1)->op1_type == IS_TMP_VAR
3541 && (opline+1)->op1.var == opline->result.var) {
3542 i++;
3543 smart_branch_opcode = (opline+1)->opcode;
3544 target_label = ssa->cfg.blocks[b].successors[0];
3545 target_label2 = ssa->cfg.blocks[b].successors[1];
3546 } else {
3547 smart_branch_opcode = 0;
3548 target_label = target_label2 = (uint32_t)-1;
3549 }
3550 if (!zend_jit_type_check(&dasm_state, opline, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) {
3551 goto jit_failure;
3552 }
3553 goto done;
3554 case ZEND_RETURN:
3555 op1_info = OP1_INFO();
3556 if ((PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info))
3557 || op_array->type == ZEND_EVAL_CODE
3558 // TODO: support for top-level code
3559 || !op_array->function_name
3560 // TODO: support for IS_UNDEF ???
3561 || (op1_info & MAY_BE_UNDEF)) {
3562 if (!zend_jit_tail_handler(&dasm_state, opline)) {
3563 goto jit_failure;
3564 }
3565 } else {
3566 int j;
3567 bool left_frame = 0;
3568
3569 if (!zend_jit_return(&dasm_state, opline, op_array,
3570 op1_info, OP1_REG_ADDR())) {
3571 goto jit_failure;
3572 }
3573 if (jit_return_label >= 0) {
3574 if (!zend_jit_jmp(&dasm_state, jit_return_label)) {
3575 goto jit_failure;
3576 }
3577 goto done;
3578 }
3579 jit_return_label = ssa->cfg.blocks_count * 2;
3580 if (!zend_jit_label(&dasm_state, jit_return_label)) {
3581 goto jit_failure;
3582 }
3583 if (op_array->last_var > 100) {
3584 /* To many CVs to unroll */
3585 if (!zend_jit_free_cvs(&dasm_state)) {
3586 goto jit_failure;
3587 }
3588 left_frame = 1;
3589 }
3590 if (!left_frame) {
3591 for (j = 0 ; j < op_array->last_var; j++) {
3592 uint32_t info = zend_ssa_cv_info(op_array, ssa, j);
3593
3594 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
3595 if (!left_frame) {
3596 left_frame = 1;
3597 if (!zend_jit_leave_frame(&dasm_state)) {
3598 goto jit_failure;
3599 }
3600 }
3601 if (!zend_jit_free_cv(&dasm_state, info, j)) {
3602 goto jit_failure;
3603 }
3604 }
3605 }
3606 }
3607 if (!zend_jit_leave_func(&dasm_state, op_array, opline, op1_info, left_frame,
3608 NULL, NULL, (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, 1)) {
3609 goto jit_failure;
3610 }
3611 }
3612 goto done;
3613 case ZEND_BOOL:
3614 case ZEND_BOOL_NOT:
3615 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
3616 OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(),
3617 -1, -1,
3618 zend_may_throw(opline, ssa_op, op_array, ssa),
3619 opline->opcode, NULL)) {
3620 goto jit_failure;
3621 }
3622 goto done;
3623 case ZEND_JMPZ:
3624 case ZEND_JMPNZ:
3625 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
3626 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
3627 /* smart branch */
3628 if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
3629 goto jit_failure;
3630 }
3631 goto done;
3632 }
3633 ZEND_FALLTHROUGH;
3634 case ZEND_JMPZNZ:
3635 case ZEND_JMPZ_EX:
3636 case ZEND_JMPNZ_EX:
3637 if (opline->result_type == IS_UNDEF) {
3638 res_addr = 0;
3639 } else {
3640 res_addr = RES_REG_ADDR();
3641 }
3642 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
3643 OP1_INFO(), OP1_REG_ADDR(), res_addr,
3644 ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1],
3645 zend_may_throw(opline, ssa_op, op_array, ssa),
3646 opline->opcode, NULL)) {
3647 goto jit_failure;
3648 }
3649 goto done;
3650 case ZEND_ISSET_ISEMPTY_CV:
3651 if ((opline->extended_value & ZEND_ISEMPTY)) {
3652 // TODO: support for empty() ???
3653 break;
3654 }
3655 if ((opline->result_type & IS_TMP_VAR)
3656 && (i + 1) <= end
3657 && ((opline+1)->opcode == ZEND_JMPZ
3658 || (opline+1)->opcode == ZEND_JMPNZ
3659 || (opline+1)->opcode == ZEND_JMPZNZ)
3660 && (opline+1)->op1_type == IS_TMP_VAR
3661 && (opline+1)->op1.var == opline->result.var) {
3662 i++;
3663 smart_branch_opcode = (opline+1)->opcode;
3664 target_label = ssa->cfg.blocks[b].successors[0];
3665 target_label2 = ssa->cfg.blocks[b].successors[1];
3666 } else {
3667 smart_branch_opcode = 0;
3668 target_label = target_label2 = (uint32_t)-1;
3669 }
3670 if (!zend_jit_isset_isempty_cv(&dasm_state, opline,
3671 OP1_INFO(), OP1_REG_ADDR(),
3672 smart_branch_opcode, target_label, target_label2,
3673 NULL)) {
3674 goto jit_failure;
3675 }
3676 goto done;
3677 case ZEND_IN_ARRAY:
3678 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
3679 break;
3680 }
3681 op1_info = OP1_INFO();
3682 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
3683 break;
3684 }
3685 if ((opline->result_type & IS_TMP_VAR)
3686 && (i + 1) <= end
3687 && ((opline+1)->opcode == ZEND_JMPZ
3688 || (opline+1)->opcode == ZEND_JMPNZ
3689 || (opline+1)->opcode == ZEND_JMPZNZ)
3690 && (opline+1)->op1_type == IS_TMP_VAR
3691 && (opline+1)->op1.var == opline->result.var) {
3692 i++;
3693 smart_branch_opcode = (opline+1)->opcode;
3694 target_label = ssa->cfg.blocks[b].successors[0];
3695 target_label2 = ssa->cfg.blocks[b].successors[1];
3696 } else {
3697 smart_branch_opcode = 0;
3698 target_label = target_label2 = (uint32_t)-1;
3699 }
3700 if (!zend_jit_in_array(&dasm_state, opline,
3701 op1_info, OP1_REG_ADDR(),
3702 smart_branch_opcode, target_label, target_label2,
3703 NULL)) {
3704 goto jit_failure;
3705 }
3706 goto done;
3707 case ZEND_FETCH_DIM_R:
3708 case ZEND_FETCH_DIM_IS:
3709 case ZEND_FETCH_LIST_R:
3710 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3711 break;
3712 }
3713 if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op,
3714 OP1_INFO(), OP1_REG_ADDR(), 0,
3715 OP2_INFO(), RES_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
3716 goto jit_failure;
3717 }
3718 goto done;
3719 case ZEND_FETCH_DIM_W:
3720 case ZEND_FETCH_DIM_RW:
3721 // case ZEND_FETCH_DIM_UNSET:
3722 case ZEND_FETCH_LIST_W:
3723 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3724 break;
3725 }
3726 if (opline->op1_type != IS_CV) {
3727 break;
3728 }
3729 if (!zend_jit_fetch_dim(&dasm_state, opline,
3730 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
3731 goto jit_failure;
3732 }
3733 goto done;
3734 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
3735 if ((opline->extended_value & ZEND_ISEMPTY)) {
3736 // TODO: support for empty() ???
3737 break;
3738 }
3739 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3740 break;
3741 }
3742 if ((opline->result_type & IS_TMP_VAR)
3743 && (i + 1) <= end
3744 && ((opline+1)->opcode == ZEND_JMPZ
3745 || (opline+1)->opcode == ZEND_JMPNZ
3746 || (opline+1)->opcode == ZEND_JMPZNZ)
3747 && (opline+1)->op1_type == IS_TMP_VAR
3748 && (opline+1)->op1.var == opline->result.var) {
3749 i++;
3750 smart_branch_opcode = (opline+1)->opcode;
3751 target_label = ssa->cfg.blocks[b].successors[0];
3752 target_label2 = ssa->cfg.blocks[b].successors[1];
3753 } else {
3754 smart_branch_opcode = 0;
3755 target_label = target_label2 = (uint32_t)-1;
3756 }
3757 if (!zend_jit_isset_isempty_dim(&dasm_state, opline,
3758 OP1_INFO(), OP1_REG_ADDR(), 0,
3759 OP2_INFO(), IS_UNKNOWN,
3760 zend_may_throw(opline, ssa_op, op_array, ssa),
3761 smart_branch_opcode, target_label, target_label2,
3762 NULL)) {
3763 goto jit_failure;
3764 }
3765 goto done;
3766 case ZEND_FETCH_OBJ_R:
3767 case ZEND_FETCH_OBJ_IS:
3768 case ZEND_FETCH_OBJ_W:
3769 if (opline->op2_type != IS_CONST
3770 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3771 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3772 break;
3773 }
3774 ce = NULL;
3775 ce_is_instanceof = 0;
3776 on_this = 0;
3777 if (opline->op1_type == IS_UNUSED) {
3778 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3779 op1_addr = 0;
3780 ce = op_array->scope;
3781 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3782 on_this = 1;
3783 } else {
3784 op1_info = OP1_INFO();
3785 if (!(op1_info & MAY_BE_OBJECT)) {
3786 break;
3787 }
3788 op1_addr = OP1_REG_ADDR();
3789 if (ssa->var_info && ssa->ops) {
3790 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3791 if (ssa_op->op1_use >= 0) {
3792 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3793 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3794 ce = op1_ssa->ce;
3795 ce_is_instanceof = op1_ssa->is_instanceof;
3796 }
3797 }
3798 }
3799 }
3800 if (!zend_jit_fetch_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3801 op1_info, op1_addr, 0, ce, ce_is_instanceof, on_this, 0, 0, NULL,
3802 IS_UNKNOWN,
3803 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3804 goto jit_failure;
3805 }
3806 goto done;
3807 case ZEND_BIND_GLOBAL:
3808 if (!ssa->ops || !ssa->var_info) {
3809 op1_info = MAY_BE_ANY|MAY_BE_REF;
3810 } else {
3811 op1_info = OP1_INFO();
3812 }
3813 if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) {
3814 goto jit_failure;
3815 }
3816 goto done;
3817 case ZEND_RECV:
3818 if (!zend_jit_recv(&dasm_state, opline, op_array)) {
3819 goto jit_failure;
3820 }
3821 goto done;
3822 case ZEND_RECV_INIT:
3823 if (!zend_jit_recv_init(&dasm_state, opline, op_array,
3824 (opline + 1)->opcode != ZEND_RECV_INIT,
3825 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3826 goto jit_failure;
3827 }
3828 goto done;
3829 case ZEND_FREE:
3830 case ZEND_FE_FREE:
3831 if (!zend_jit_free(&dasm_state, opline, OP1_INFO(),
3832 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3833 goto jit_failure;
3834 }
3835 goto done;
3836 case ZEND_ECHO:
3837 op1_info = OP1_INFO();
3838 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3839 break;
3840 }
3841 if (!zend_jit_echo(&dasm_state, opline, op1_info)) {
3842 goto jit_failure;
3843 }
3844 goto done;
3845 case ZEND_STRLEN:
3846 op1_info = OP1_INFO();
3847 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3848 break;
3849 }
3850 if (!zend_jit_strlen(&dasm_state, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR())) {
3851 goto jit_failure;
3852 }
3853 goto done;
3854 case ZEND_COUNT:
3855 op1_info = OP1_INFO();
3856 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
3857 break;
3858 }
3859 if (!zend_jit_count(&dasm_state, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
3860 goto jit_failure;
3861 }
3862 goto done;
3863 case ZEND_FETCH_THIS:
3864 if (!zend_jit_fetch_this(&dasm_state, opline, op_array, 0)) {
3865 goto jit_failure;
3866 }
3867 goto done;
3868 case ZEND_SWITCH_LONG:
3869 case ZEND_SWITCH_STRING:
3870 case ZEND_MATCH:
3871 if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL, NULL)) {
3872 goto jit_failure;
3873 }
3874 goto done;
3875 case ZEND_VERIFY_RETURN_TYPE:
3876 if (opline->op1_type == IS_UNUSED) {
3877 /* Always throws */
3878 break;
3879 }
3880 if (opline->op1_type == IS_CONST) {
3881 /* TODO Different instruction format, has return value */
3882 break;
3883 }
3884 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
3885 /* Not worth bothering with */
3886 break;
3887 }
3888 if (OP1_INFO() & MAY_BE_REF) {
3889 /* TODO May need reference unwrapping. */
3890 break;
3891 }
3892 if (!zend_jit_verify_return_type(&dasm_state, opline, op_array, OP1_INFO())) {
3893 goto jit_failure;
3894 }
3895 goto done;
3896 case ZEND_FE_RESET_R:
3897 op1_info = OP1_INFO();
3898 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
3899 break;
3900 }
3901 if (!zend_jit_fe_reset(&dasm_state, opline, op1_info)) {
3902 goto jit_failure;
3903 }
3904 goto done;
3905 case ZEND_FE_FETCH_R:
3906 op1_info = OP1_INFO();
3907 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
3908 break;
3909 }
3910 if (!zend_jit_fe_fetch(&dasm_state, opline, op1_info, OP2_INFO(),
3911 ssa->cfg.blocks[b].successors[0], opline->opcode, NULL)) {
3912 goto jit_failure;
3913 }
3914 goto done;
3915 case ZEND_FETCH_CONSTANT:
3916 if (!zend_jit_fetch_constant(&dasm_state, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
3917 goto jit_failure;
3918 }
3919 goto done;
3920 case ZEND_INIT_METHOD_CALL:
3921 if (opline->op2_type != IS_CONST
3922 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
3923 break;
3924 }
3925 ce = NULL;
3926 ce_is_instanceof = 0;
3927 on_this = 0;
3928 if (opline->op1_type == IS_UNUSED) {
3929 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3930 op1_addr = 0;
3931 ce = op_array->scope;
3932 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3933 on_this = 1;
3934 } else {
3935 op1_info = OP1_INFO();
3936 if (!(op1_info & MAY_BE_OBJECT)) {
3937 break;
3938 }
3939 op1_addr = OP1_REG_ADDR();
3940 if (ssa->var_info && ssa->ops) {
3941 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3942 if (ssa_op->op1_use >= 0) {
3943 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3944 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3945 ce = op1_ssa->ce;
3946 ce_is_instanceof = op1_ssa->is_instanceof;
3947 }
3948 }
3949 }
3950 }
3951 if (!zend_jit_init_method_call(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level,
3952 op1_info, op1_addr, ce, ce_is_instanceof, on_this, 0, NULL,
3953 NULL, 0, 0)) {
3954 goto jit_failure;
3955 }
3956 goto done;
3957 case ZEND_ROPE_INIT:
3958 case ZEND_ROPE_ADD:
3959 case ZEND_ROPE_END:
3960 op2_info = OP2_INFO();
3961 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3962 break;
3963 }
3964 if (!zend_jit_rope(&dasm_state, opline, op2_info)) {
3965 goto jit_failure;
3966 }
3967 goto done;
3968 default:
3969 break;
3970 }
3971 }
3972
3973 switch (opline->opcode) {
3974 case ZEND_RECV_INIT:
3975 case ZEND_BIND_GLOBAL:
3976 if (opline == op_array->opcodes ||
3977 opline->opcode != op_array->opcodes[i-1].opcode) {
3978 /* repeatable opcodes */
3979 if (!zend_jit_handler(&dasm_state, opline,
3980 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3981 goto jit_failure;
3982 }
3983 }
3984 zend_jit_set_last_valid_opline(opline+1);
3985 break;
3986 case ZEND_NOP:
3987 case ZEND_OP_DATA:
3988 case ZEND_SWITCH_LONG:
3989 case ZEND_SWITCH_STRING:
3990 case ZEND_MATCH:
3991 break;
3992 case ZEND_JMP:
3993 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
3994 const zend_op *target = OP_JMP_ADDR(opline, opline->op1);
3995
3996 if (!zend_jit_set_ip(&dasm_state, target)) {
3997 goto jit_failure;
3998 }
3999 }
4000 if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) {
4001 goto jit_failure;
4002 }
4003 is_terminated = 1;
4004 break;
4005 case ZEND_CATCH:
4006 case ZEND_FAST_CALL:
4007 case ZEND_FAST_RET:
4008 case ZEND_GENERATOR_CREATE:
4009 case ZEND_GENERATOR_RETURN:
4010 case ZEND_RETURN_BY_REF:
4011 case ZEND_RETURN:
4012 case ZEND_EXIT:
4013 case ZEND_MATCH_ERROR:
4014 /* switch through trampoline */
4015 case ZEND_YIELD:
4016 case ZEND_YIELD_FROM:
4017 if (!zend_jit_tail_handler(&dasm_state, opline)) {
4018 goto jit_failure;
4019 }
4020 is_terminated = 1;
4021 break;
4022 /* stackless execution */
4023 case ZEND_INCLUDE_OR_EVAL:
4024 case ZEND_DO_FCALL:
4025 case ZEND_DO_UCALL:
4026 case ZEND_DO_FCALL_BY_NAME:
4027 if (!zend_jit_call(&dasm_state, opline, b + 1)) {
4028 goto jit_failure;
4029 }
4030 is_terminated = 1;
4031 break;
4032 case ZEND_JMPZNZ:
4033 if (!zend_jit_handler(&dasm_state, opline,
4034 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
4035 !zend_jit_cond_jmp(&dasm_state, OP_JMP_ADDR(opline, opline->op2), ssa->cfg.blocks[b].successors[1]) ||
4036 !zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) {
4037 goto jit_failure;
4038 }
4039 is_terminated = 1;
4040 break;
4041 case ZEND_JMPZ:
4042 case ZEND_JMPNZ:
4043 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
4044 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
4045 /* smart branch */
4046 if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
4047 goto jit_failure;
4048 }
4049 goto done;
4050 }
4051 ZEND_FALLTHROUGH;
4052 case ZEND_JMPZ_EX:
4053 case ZEND_JMPNZ_EX:
4054 case ZEND_JMP_SET:
4055 case ZEND_COALESCE:
4056 case ZEND_JMP_NULL:
4057 case ZEND_FE_RESET_R:
4058 case ZEND_FE_RESET_RW:
4059 case ZEND_ASSERT_CHECK:
4060 case ZEND_FE_FETCH_R:
4061 case ZEND_FE_FETCH_RW:
4062 if (!zend_jit_handler(&dasm_state, opline,
4063 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
4064 !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
4065 goto jit_failure;
4066 }
4067 break;
4068 case ZEND_NEW:
4069 if (!zend_jit_handler(&dasm_state, opline, 1)) {
4070 return 0;
4071 }
4072 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
4073 zend_class_entry *ce = NULL;
4074
4075 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
4076 if (ssa->ops && ssa->var_info) {
4077 zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
4078 if (res_ssa->ce && !res_ssa->is_instanceof) {
4079 ce = res_ssa->ce;
4080 }
4081 }
4082 } else {
4083 if (opline->op1_type == IS_CONST) {
4084 zval *zv = RT_CONSTANT(opline, opline->op1);
4085 if (Z_TYPE_P(zv) == IS_STRING) {
4086 zval *lc = zv + 1;
4087 ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc));
4088 }
4089 }
4090 }
4091
4092 i++;
4093
4094 if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) {
4095 const zend_op *next_opline = opline + 1;
4096
4097 zend_jit_cond_jmp(&dasm_state, next_opline, ssa->cfg.blocks[b].successors[0]);
4098 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
4099 zend_jit_call(&dasm_state, next_opline, b + 1);
4100 is_terminated = 1;
4101 } else {
4102 zend_jit_do_fcall(&dasm_state, next_opline, op_array, ssa, call_level, b + 1, NULL);
4103 }
4104 }
4105
4106 /* We skip over the DO_FCALL, so decrement call_level ourselves. */
4107 call_level--;
4108 }
4109 break;
4110 default:
4111 if (!zend_jit_handler(&dasm_state, opline,
4112 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4113 goto jit_failure;
4114 }
4115 if (i == end
4116 && (opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
4117 /* smart branch split across basic blocks */
4118 if (!zend_jit_cond_jmp(&dasm_state, opline + 2, ssa->cfg.blocks[b+1].successors[0])) {
4119 goto jit_failure;
4120 }
4121 if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b+1].successors[1])) {
4122 goto jit_failure;
4123 }
4124 is_terminated = 1;
4125 }
4126 }
4127 done:
4128 switch (opline->opcode) {
4129 case ZEND_DO_FCALL:
4130 case ZEND_DO_ICALL:
4131 case ZEND_DO_UCALL:
4132 case ZEND_DO_FCALL_BY_NAME:
4133 case ZEND_CALLABLE_CONVERT:
4134 call_level--;
4135 }
4136 }
4137 }
4138
4139 handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0,
4140 (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET, SP_ADJ_JIT);
4141 if (!handler) {
4142 goto jit_failure;
4143 }
4144 dasm_free(&dasm_state);
4145
4146 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
4147 zend_arena_release(&CG(arena), checkpoint);
4148 }
4149 return SUCCESS;
4150
4151 jit_failure:
4152 if (dasm_state) {
4153 dasm_free(&dasm_state);
4154 }
4155 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
4156 zend_arena_release(&CG(arena), checkpoint);
4157 }
4158 return FAILURE;
4159 }
4160
zend_jit_collect_calls(zend_op_array * op_array,zend_script * script)4161 static int zend_jit_collect_calls(zend_op_array *op_array, zend_script *script)
4162 {
4163 zend_func_info *func_info;
4164
4165 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4166 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4167 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4168 func_info = ZEND_FUNC_INFO(op_array);
4169 } else {
4170 func_info = zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info));
4171 ZEND_SET_FUNC_INFO(op_array, func_info);
4172 }
4173 return zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, op_array, func_info);
4174 }
4175
zend_jit_cleanup_func_info(zend_op_array * op_array)4176 static void zend_jit_cleanup_func_info(zend_op_array *op_array)
4177 {
4178 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
4179 zend_call_info *caller_info, *callee_info;
4180
4181 if (func_info) {
4182 caller_info = func_info->caller_info;
4183 callee_info = func_info->callee_info;
4184
4185 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4186 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4187 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4188 func_info->num = 0;
4189 func_info->flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
4190 | ZEND_FUNC_JIT_ON_PROF_REQUEST
4191 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
4192 | ZEND_FUNC_JIT_ON_HOT_TRACE;
4193 memset(&func_info->ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
4194 } else {
4195 ZEND_SET_FUNC_INFO(op_array, NULL);
4196 }
4197
4198 while (caller_info) {
4199 if (caller_info->caller_op_array) {
4200 zend_jit_cleanup_func_info(caller_info->caller_op_array);
4201 }
4202 caller_info = caller_info->next_caller;
4203 }
4204 while (callee_info) {
4205 if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) {
4206 zend_jit_cleanup_func_info(&callee_info->callee_func->op_array);
4207 }
4208 callee_info = callee_info->next_callee;
4209 }
4210 }
4211 }
4212
zend_real_jit_func(zend_op_array * op_array,zend_script * script,const zend_op * rt_opline)4213 static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline)
4214 {
4215 zend_ssa ssa;
4216 void *checkpoint;
4217 zend_func_info *func_info;
4218
4219 if (*dasm_ptr == dasm_end) {
4220 return FAILURE;
4221 }
4222
4223 checkpoint = zend_arena_checkpoint(CG(arena));
4224
4225 /* Build SSA */
4226 memset(&ssa, 0, sizeof(zend_ssa));
4227
4228 if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) {
4229 goto jit_failure;
4230 }
4231
4232 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
4233 if (zend_jit_collect_calls(op_array, script) != SUCCESS) {
4234 goto jit_failure;
4235 }
4236 func_info = ZEND_FUNC_INFO(op_array);
4237 func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array);
4238 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
4239 zend_init_func_return_info(op_array, script, &func_info->return_info);
4240 }
4241 }
4242
4243 if (zend_jit_op_array_analyze2(op_array, script, &ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
4244 goto jit_failure;
4245 }
4246
4247 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
4248 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa);
4249 }
4250
4251 if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) {
4252 goto jit_failure;
4253 }
4254
4255 zend_jit_cleanup_func_info(op_array);
4256 zend_arena_release(&CG(arena), checkpoint);
4257 return SUCCESS;
4258
4259 jit_failure:
4260 zend_jit_cleanup_func_info(op_array);
4261 zend_arena_release(&CG(arena), checkpoint);
4262 return FAILURE;
4263 }
4264
4265 /* Run-time JIT handler */
zend_runtime_jit(void)4266 static int ZEND_FASTCALL zend_runtime_jit(void)
4267 {
4268 zend_execute_data *execute_data = EG(current_execute_data);
4269 zend_op_array *op_array = &EX(func)->op_array;
4270 zend_op *opline = op_array->opcodes;
4271 zend_jit_op_array_extension *jit_extension;
4272 bool do_bailout = 0;
4273
4274 zend_shared_alloc_lock();
4275
4276 if (ZEND_FUNC_INFO(op_array)) {
4277
4278 SHM_UNPROTECT();
4279 zend_jit_unprotect();
4280
4281 zend_try {
4282 /* restore original opcode handlers */
4283 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4284 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4285 opline++;
4286 }
4287 }
4288 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
4289 opline->handler = jit_extension->orig_handler;
4290
4291 /* perform real JIT for this function */
4292 zend_real_jit_func(op_array, NULL, NULL);
4293 } zend_catch {
4294 do_bailout = true;
4295 } zend_end_try();
4296
4297 zend_jit_protect();
4298 SHM_PROTECT();
4299 }
4300
4301 zend_shared_alloc_unlock();
4302
4303 if (do_bailout) {
4304 zend_bailout();
4305 }
4306
4307 /* JIT-ed code is going to be called by VM */
4308 return 0;
4309 }
4310
zend_jit_check_funcs(HashTable * function_table,bool is_method)4311 void zend_jit_check_funcs(HashTable *function_table, bool is_method) {
4312 zend_op *opline;
4313 zend_function *func;
4314 zend_op_array *op_array;
4315 uintptr_t counter;
4316 zend_jit_op_array_extension *jit_extension;
4317
4318 ZEND_HASH_REVERSE_FOREACH_PTR(function_table, func) {
4319 if (func->type == ZEND_INTERNAL_FUNCTION) {
4320 break;
4321 }
4322 op_array = &func->op_array;
4323 opline = op_array->opcodes;
4324 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4325 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4326 opline++;
4327 }
4328 }
4329 if (opline->handler == zend_jit_profile_jit_handler) {
4330 if (!RUN_TIME_CACHE(op_array)) {
4331 continue;
4332 }
4333 counter = (uintptr_t)ZEND_COUNTER_INFO(op_array);
4334 ZEND_COUNTER_INFO(op_array) = 0;
4335 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
4336 opline->handler = jit_extension->orig_handler;
4337 if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) {
4338 zend_real_jit_func(op_array, NULL, NULL);
4339 }
4340 }
4341 } ZEND_HASH_FOREACH_END();
4342 }
4343
zend_jit_hot_func(zend_execute_data * execute_data,const zend_op * opline)4344 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
4345 {
4346 zend_op_array *op_array = &EX(func)->op_array;
4347 zend_jit_op_array_hot_extension *jit_extension;
4348 uint32_t i;
4349 bool do_bailout = 0;
4350
4351 zend_shared_alloc_lock();
4352 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
4353
4354 if (jit_extension) {
4355 SHM_UNPROTECT();
4356 zend_jit_unprotect();
4357
4358 zend_try {
4359 for (i = 0; i < op_array->last; i++) {
4360 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
4361 }
4362
4363 /* perform real JIT for this function */
4364 zend_real_jit_func(op_array, NULL, opline);
4365 } zend_catch {
4366 do_bailout = 1;
4367 } zend_end_try();
4368
4369 zend_jit_protect();
4370 SHM_PROTECT();
4371 }
4372
4373 zend_shared_alloc_unlock();
4374
4375 if (do_bailout) {
4376 zend_bailout();
4377 }
4378 /* JIT-ed code is going to be called by VM */
4379 }
4380
zend_jit_setup_hot_counters_ex(zend_op_array * op_array,zend_cfg * cfg)4381 static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg)
4382 {
4383 if (JIT_G(hot_func)) {
4384 zend_op *opline = op_array->opcodes;
4385
4386 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4387 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4388 opline++;
4389 }
4390 }
4391
4392 opline->handler = (const void*)zend_jit_func_hot_counter_handler;
4393 }
4394
4395 if (JIT_G(hot_loop)) {
4396 uint32_t i;
4397
4398 for (i = 0; i < cfg->blocks_count; i++) {
4399 if ((cfg->blocks[i].flags & ZEND_BB_REACHABLE) &&
4400 (cfg->blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
4401 op_array->opcodes[cfg->blocks[i].start].handler =
4402 (const void*)zend_jit_loop_hot_counter_handler;
4403 }
4404 }
4405 }
4406 }
4407
zend_jit_restart_hot_counters(zend_op_array * op_array)4408 static int zend_jit_restart_hot_counters(zend_op_array *op_array)
4409 {
4410 zend_jit_op_array_hot_extension *jit_extension;
4411 zend_cfg cfg;
4412 uint32_t i;
4413
4414 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
4415 for (i = 0; i < op_array->last; i++) {
4416 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
4417 }
4418
4419 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
4420 return FAILURE;
4421 }
4422
4423 zend_jit_setup_hot_counters_ex(op_array, &cfg);
4424
4425 return SUCCESS;
4426 }
4427
zend_jit_setup_hot_counters(zend_op_array * op_array)4428 static int zend_jit_setup_hot_counters(zend_op_array *op_array)
4429 {
4430 zend_jit_op_array_hot_extension *jit_extension;
4431 zend_cfg cfg;
4432 uint32_t i;
4433
4434 ZEND_ASSERT(zend_jit_func_hot_counter_handler != NULL);
4435 ZEND_ASSERT(zend_jit_loop_hot_counter_handler != NULL);
4436
4437 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
4438 return FAILURE;
4439 }
4440
4441 jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*));
4442 if (!jit_extension) {
4443 return FAILURE;
4444 }
4445 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4446 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS;
4447 jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
4448 for (i = 0; i < op_array->last; i++) {
4449 jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
4450 }
4451 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4452
4453 zend_jit_setup_hot_counters_ex(op_array, &cfg);
4454
4455 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4456
4457 return SUCCESS;
4458 }
4459
4460 #include "jit/zend_jit_trace.c"
4461
zend_jit_op_array(zend_op_array * op_array,zend_script * script)4462 ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
4463 {
4464 if (dasm_ptr == NULL) {
4465 return FAILURE;
4466 }
4467
4468 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) {
4469 zend_jit_op_array_extension *jit_extension;
4470 zend_op *opline = op_array->opcodes;
4471
4472 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
4473 ZEND_SET_FUNC_INFO(op_array, NULL);
4474 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
4475 return SUCCESS;
4476 }
4477
4478 /* Set run-time JIT handler */
4479 ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL);
4480 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4481 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4482 opline++;
4483 }
4484 }
4485 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
4486 if (!jit_extension) {
4487 return FAILURE;
4488 }
4489 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4490 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC;
4491 jit_extension->orig_handler = (void*)opline->handler;
4492 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4493 opline->handler = (const void*)zend_jit_runtime_jit_handler;
4494 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4495
4496 return SUCCESS;
4497 } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) {
4498 zend_jit_op_array_extension *jit_extension;
4499 zend_op *opline = op_array->opcodes;
4500
4501 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
4502 ZEND_SET_FUNC_INFO(op_array, NULL);
4503 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
4504 return SUCCESS;
4505 }
4506
4507 ZEND_ASSERT(zend_jit_profile_jit_handler != NULL);
4508 if (op_array->function_name) {
4509 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4510 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4511 opline++;
4512 }
4513 }
4514 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
4515 if (!jit_extension) {
4516 return FAILURE;
4517 }
4518 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4519 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST;
4520 jit_extension->orig_handler = (void*)opline->handler;
4521 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4522 opline->handler = (const void*)zend_jit_profile_jit_handler;
4523 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4524 }
4525
4526 return SUCCESS;
4527 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4528 return zend_jit_setup_hot_counters(op_array);
4529 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4530 return zend_jit_setup_hot_trace_counters(op_array);
4531 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4532 return zend_real_jit_func(op_array, script, NULL);
4533 } else {
4534 ZEND_UNREACHABLE();
4535 }
4536 }
4537
zend_jit_script(zend_script * script)4538 ZEND_EXT_API int zend_jit_script(zend_script *script)
4539 {
4540 void *checkpoint;
4541 zend_call_graph call_graph;
4542 zend_func_info *info;
4543 int i;
4544
4545 if (dasm_ptr == NULL || *dasm_ptr == dasm_end) {
4546 return FAILURE;
4547 }
4548
4549 checkpoint = zend_arena_checkpoint(CG(arena));
4550
4551 call_graph.op_arrays_count = 0;
4552 if (zend_build_call_graph(&CG(arena), script, &call_graph) != SUCCESS) {
4553 goto jit_failure;
4554 }
4555
4556 zend_analyze_call_graph(&CG(arena), script, &call_graph);
4557
4558 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4559 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4560 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS ||
4561 JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4562 for (i = 0; i < call_graph.op_arrays_count; i++) {
4563 if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
4564 goto jit_failure;
4565 }
4566 }
4567 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4568 for (i = 0; i < call_graph.op_arrays_count; i++) {
4569 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4570 if (info) {
4571 if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
4572 goto jit_failure;
4573 }
4574 info->flags = info->ssa.cfg.flags;
4575 }
4576 }
4577
4578 for (i = 0; i < call_graph.op_arrays_count; i++) {
4579 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4580 if (info) {
4581 info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]);
4582 if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
4583 zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info);
4584 }
4585 }
4586 }
4587
4588 for (i = 0; i < call_graph.op_arrays_count; i++) {
4589 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4590 if (info) {
4591 if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
4592 goto jit_failure;
4593 }
4594 info->flags = info->ssa.cfg.flags;
4595 }
4596 }
4597
4598 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
4599 for (i = 0; i < call_graph.op_arrays_count; i++) {
4600 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4601 if (info) {
4602 zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa);
4603 }
4604 }
4605 }
4606
4607 for (i = 0; i < call_graph.op_arrays_count; i++) {
4608 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4609 if (info) {
4610 if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) {
4611 goto jit_failure;
4612 }
4613 }
4614 }
4615
4616 for (i = 0; i < call_graph.op_arrays_count; i++) {
4617 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
4618 }
4619 } else {
4620 ZEND_UNREACHABLE();
4621 }
4622
4623 zend_arena_release(&CG(arena), checkpoint);
4624
4625 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC
4626 || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST
4627 || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS
4628 || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4629 zend_class_entry *ce;
4630 zend_op_array *op_array;
4631
4632 ZEND_HASH_FOREACH_PTR(&script->class_table, ce) {
4633 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
4634 if (!ZEND_FUNC_INFO(op_array)) {
4635 void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes);
4636
4637 if (jit_extension) {
4638 ZEND_SET_FUNC_INFO(op_array, jit_extension);
4639 }
4640 }
4641 } ZEND_HASH_FOREACH_END();
4642 } ZEND_HASH_FOREACH_END();
4643 }
4644
4645 return SUCCESS;
4646
4647 jit_failure:
4648 if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4649 for (i = 0; i < call_graph.op_arrays_count; i++) {
4650 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
4651 }
4652 }
4653 zend_arena_release(&CG(arena), checkpoint);
4654 return FAILURE;
4655 }
4656
zend_jit_unprotect(void)4657 ZEND_EXT_API void zend_jit_unprotect(void)
4658 {
4659 #ifdef HAVE_MPROTECT
4660 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4661 int opts = PROT_READ | PROT_WRITE;
4662 #ifdef ZTS
4663 /* TODO: EXEC+WRITE is not supported in macOS. Removing EXEC is still buggy as
4664 * other threads, which are executing the JITed code, would crash anyway. */
4665 # ifndef __APPLE__
4666 /* Another thread may be executing JITed code. */
4667 opts |= PROT_EXEC;
4668 # endif
4669 #endif
4670 if (mprotect(dasm_buf, dasm_size, opts) != 0) {
4671 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4672 }
4673 }
4674 #elif _WIN32
4675 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4676 DWORD old, new;
4677 #ifdef ZTS
4678 new = PAGE_EXECUTE_READWRITE;
4679 #else
4680 new = PAGE_READWRITE;
4681 #endif
4682 if (!VirtualProtect(dasm_buf, dasm_size, new, &old)) {
4683 DWORD err = GetLastError();
4684 char *msg = php_win32_error_to_msg(err);
4685 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4686 php_win32_error_msg_free(msg);
4687 }
4688 }
4689 #endif
4690 }
4691
zend_jit_protect(void)4692 ZEND_EXT_API void zend_jit_protect(void)
4693 {
4694 #ifdef HAVE_MPROTECT
4695 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4696 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
4697 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4698 }
4699 }
4700 #elif _WIN32
4701 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4702 DWORD old;
4703
4704 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
4705 DWORD err = GetLastError();
4706 char *msg = php_win32_error_to_msg(err);
4707 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4708 php_win32_error_msg_free(msg);
4709 }
4710 }
4711 #endif
4712 }
4713
zend_jit_init_handlers(void)4714 static void zend_jit_init_handlers(void)
4715 {
4716 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
4717 zend_jit_runtime_jit_handler = dasm_labels[zend_lbhybrid_runtime_jit];
4718 zend_jit_profile_jit_handler = dasm_labels[zend_lbhybrid_profile_jit];
4719 zend_jit_func_hot_counter_handler = dasm_labels[zend_lbhybrid_func_hot_counter];
4720 zend_jit_loop_hot_counter_handler = dasm_labels[zend_lbhybrid_loop_hot_counter];
4721 zend_jit_func_trace_counter_handler = dasm_labels[zend_lbhybrid_func_trace_counter];
4722 zend_jit_ret_trace_counter_handler = dasm_labels[zend_lbhybrid_ret_trace_counter];
4723 zend_jit_loop_trace_counter_handler = dasm_labels[zend_lbhybrid_loop_trace_counter];
4724 } else {
4725 zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
4726 zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
4727 zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper;
4728 zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper;
4729 zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper;
4730 zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper;
4731 zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper;
4732 }
4733 }
4734
zend_jit_make_stubs(void)4735 static int zend_jit_make_stubs(void)
4736 {
4737 dasm_State* dasm_state = NULL;
4738 uint32_t i;
4739
4740 dasm_init(&dasm_state, DASM_MAXSECTION);
4741 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
4742
4743 for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) {
4744 dasm_setup(&dasm_state, dasm_actions);
4745 if (!zend_jit_stubs[i].stub(&dasm_state)) {
4746 return 0;
4747 }
4748 if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name, 0,
4749 zend_jit_stubs[i].offset, zend_jit_stubs[i].adjustment)) {
4750 return 0;
4751 }
4752 }
4753
4754 zend_jit_init_handlers();
4755
4756 dasm_free(&dasm_state);
4757 return 1;
4758 }
4759
zend_jit_globals_ctor(zend_jit_globals * jit_globals)4760 static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
4761 {
4762 memset(jit_globals, 0, sizeof(zend_jit_globals));
4763 zend_jit_trace_init_caches();
4764 }
4765
4766 #ifdef ZTS
zend_jit_globals_dtor(zend_jit_globals * jit_globals)4767 static void zend_jit_globals_dtor(zend_jit_globals *jit_globals)
4768 {
4769 zend_jit_trace_free_caches(jit_globals);
4770 }
4771 #endif
4772
zend_jit_parse_config_num(zend_long jit)4773 static int zend_jit_parse_config_num(zend_long jit)
4774 {
4775 if (jit == 0) {
4776 JIT_G(on) = 0;
4777 return SUCCESS;
4778 }
4779
4780 if (jit < 0) return FAILURE;
4781
4782 if (jit % 10 == 0 || jit % 10 > 5) return FAILURE;
4783 JIT_G(opt_level) = jit % 10;
4784
4785 jit /= 10;
4786 if (jit % 10 > 5) return FAILURE;
4787 JIT_G(trigger) = jit % 10;
4788
4789 jit /= 10;
4790 if (jit % 10 > 2) return FAILURE;
4791 JIT_G(opt_flags) = jit % 10;
4792
4793 jit /= 10;
4794 if (jit % 10 > 1) return FAILURE;
4795 JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0);
4796
4797 if (jit / 10 != 0) return FAILURE;
4798
4799 JIT_G(on) = 1;
4800
4801 return SUCCESS;
4802 }
4803
zend_jit_config(zend_string * jit,int stage)4804 ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage)
4805 {
4806 if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) {
4807 if (stage == ZEND_INI_STAGE_RUNTIME) {
4808 zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)");
4809 }
4810 return FAILURE;
4811 }
4812
4813 if (ZSTR_LEN(jit) == 0
4814 || zend_string_equals_literal_ci(jit, "disable")) {
4815 JIT_G(enabled) = 0;
4816 JIT_G(on) = 0;
4817 return SUCCESS;
4818 } else if (zend_string_equals_literal_ci(jit, "0")
4819 || zend_string_equals_literal_ci(jit, "off")
4820 || zend_string_equals_literal_ci(jit, "no")
4821 || zend_string_equals_literal_ci(jit, "false")) {
4822 JIT_G(enabled) = 1;
4823 JIT_G(on) = 0;
4824 return SUCCESS;
4825 } else if (zend_string_equals_literal_ci(jit, "1")
4826 || zend_string_equals_literal_ci(jit, "on")
4827 || zend_string_equals_literal_ci(jit, "yes")
4828 || zend_string_equals_literal_ci(jit, "true")
4829 || zend_string_equals_literal_ci(jit, "tracing")) {
4830 JIT_G(enabled) = 1;
4831 JIT_G(on) = 1;
4832 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS;
4833 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
4834 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
4835 return SUCCESS;
4836 } else if (zend_string_equals_literal_ci(jit, "function")) {
4837 JIT_G(enabled) = 1;
4838 JIT_G(on) = 1;
4839 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT;
4840 JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD;
4841 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
4842 return SUCCESS;
4843 } else {
4844 char *end;
4845 zend_long num = ZEND_STRTOL(ZSTR_VAL(jit), &end, 10);
4846 if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) {
4847 goto failure;
4848 }
4849 JIT_G(enabled) = 1;
4850 return SUCCESS;
4851 }
4852
4853 failure:
4854 zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number");
4855 JIT_G(enabled) = 0;
4856 JIT_G(on) = 0;
4857 return FAILURE;
4858 }
4859
zend_jit_debug_config(zend_long old_val,zend_long new_val,int stage)4860 ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage)
4861 {
4862 if (stage != ZEND_INI_STAGE_STARTUP) {
4863 if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) {
4864 if (stage == ZEND_INI_STAGE_RUNTIME) {
4865 zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup");
4866 }
4867 return FAILURE;
4868 }
4869 #ifdef HAVE_DISASM
4870 if (new_val & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
4871 if (JIT_G(enabled) && !JIT_G(symbols) && !zend_jit_disasm_init()) {
4872 // TODO: error reporting and cleanup ???
4873 return FAILURE;
4874 }
4875 // TODO: symbols for JIT-ed code compiled before are missing ???
4876 }
4877 #endif
4878 }
4879 return SUCCESS;
4880 }
4881
zend_jit_init(void)4882 ZEND_EXT_API void zend_jit_init(void)
4883 {
4884 #ifdef ZTS
4885 jit_globals_id = ts_allocate_id(&jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, (ts_allocate_dtor) zend_jit_globals_dtor);
4886 #else
4887 zend_jit_globals_ctor(&jit_globals);
4888 #endif
4889 }
4890
zend_jit_check_support(void)4891 ZEND_EXT_API int zend_jit_check_support(void)
4892 {
4893 int i;
4894
4895 zend_jit_vm_kind = zend_vm_kind();
4896 if (zend_jit_vm_kind != ZEND_VM_KIND_CALL &&
4897 zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
4898 zend_error(E_WARNING, "JIT is compatible only with CALL and HYBRID VM. JIT disabled.");
4899 JIT_G(enabled) = 0;
4900 JIT_G(on) = 0;
4901 return FAILURE;
4902 }
4903
4904 if (zend_execute_ex != execute_ex) {
4905 if (strcmp(sapi_module.name, "phpdbg") != 0) {
4906 zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled.");
4907 }
4908 JIT_G(enabled) = 0;
4909 JIT_G(on) = 0;
4910 return FAILURE;
4911 }
4912
4913 for (i = 0; i <= 256; i++) {
4914 switch (i) {
4915 /* JIT has no effect on these opcodes */
4916 case ZEND_BEGIN_SILENCE:
4917 case ZEND_END_SILENCE:
4918 case ZEND_EXIT:
4919 break;
4920 default:
4921 if (zend_get_user_opcode_handler(i) != NULL) {
4922 zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled.");
4923 JIT_G(enabled) = 0;
4924 JIT_G(on) = 0;
4925 return FAILURE;
4926 }
4927 }
4928 }
4929
4930 return SUCCESS;
4931 }
4932
zend_jit_startup(void * buf,size_t size,bool reattached)4933 ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, bool reattached)
4934 {
4935 int ret;
4936
4937 zend_jit_halt_op = zend_get_halt_op();
4938
4939 if (zend_jit_setup() != SUCCESS) {
4940 // TODO: error reporting and cleanup ???
4941 return FAILURE;
4942 }
4943
4944 zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(ACCELERATOR_PRODUCT_NAME);
4945
4946 #ifdef HAVE_GDB
4947 zend_jit_gdb_init();
4948 #endif
4949
4950 #ifdef HAVE_OPROFILE
4951 if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
4952 if (!zend_jit_oprofile_startup()) {
4953 // TODO: error reporting and cleanup ???
4954 return FAILURE;
4955 }
4956 }
4957 #endif
4958
4959 dasm_buf = buf;
4960 dasm_size = size;
4961
4962 #ifdef HAVE_MPROTECT
4963 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
4964 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
4965 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4966 }
4967 } else {
4968 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
4969 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4970 }
4971 }
4972 #elif _WIN32
4973 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
4974 DWORD old;
4975
4976 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
4977 DWORD err = GetLastError();
4978 char *msg = php_win32_error_to_msg(err);
4979 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4980 php_win32_error_msg_free(msg);
4981 }
4982 } else {
4983 DWORD old;
4984
4985 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
4986 DWORD err = GetLastError();
4987 char *msg = php_win32_error_to_msg(err);
4988 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4989 php_win32_error_msg_free(msg);
4990 }
4991 }
4992 #endif
4993
4994 dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr) * 2);
4995 if (!reattached) {
4996 zend_jit_unprotect();
4997 *dasm_ptr = dasm_buf;
4998 #if _WIN32
4999 /* reserve space for global labels */
5000 *dasm_ptr = (void**)*dasm_ptr + zend_lb_MAX;
5001 #endif
5002 zend_jit_protect();
5003 }
5004
5005 #ifdef HAVE_DISASM
5006 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
5007 if (!zend_jit_disasm_init()) {
5008 // TODO: error reporting and cleanup ???
5009 return FAILURE;
5010 }
5011 }
5012 #endif
5013
5014 #ifdef HAVE_PERFTOOLS
5015 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
5016 zend_jit_perf_jitdump_open();
5017 }
5018 #endif
5019
5020 if (!reattached) {
5021 zend_jit_unprotect();
5022 ret = zend_jit_make_stubs();
5023 #if _WIN32
5024 /* save global labels */
5025 memcpy(dasm_buf, dasm_labels, sizeof(void*) * zend_lb_MAX);
5026 #endif
5027 zend_jit_protect();
5028 if (!ret) {
5029 // TODO: error reporting and cleanup ???
5030 return FAILURE;
5031 }
5032 } else {
5033 #if _WIN32
5034 /* restore global labels */
5035 memcpy(dasm_labels, dasm_buf, sizeof(void*) * zend_lb_MAX);
5036 zend_jit_init_handlers();
5037 #endif
5038 }
5039
5040 if (zend_jit_trace_startup(reattached) != SUCCESS) {
5041 return FAILURE;
5042 }
5043
5044 zend_jit_unprotect();
5045 #if ZEND_JIT_TARGET_ARM64
5046 /* reserve space for global labels veneers */
5047 dasm_labels_veneers = *dasm_ptr;
5048 *dasm_ptr = (void**)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT);
5049 memset(dasm_labels_veneers, 0, sizeof(void*) * ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT));
5050 #endif
5051 /* save JIT buffer pos */
5052 dasm_ptr[1] = dasm_ptr[0];
5053 zend_jit_protect();
5054
5055 return SUCCESS;
5056 }
5057
zend_jit_shutdown(void)5058 ZEND_EXT_API void zend_jit_shutdown(void)
5059 {
5060 if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE) {
5061 fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf));
5062 }
5063
5064 #ifdef HAVE_OPROFILE
5065 if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
5066 zend_jit_oprofile_shutdown();
5067 }
5068 #endif
5069
5070 #ifdef HAVE_GDB
5071 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
5072 zend_jit_gdb_unregister();
5073 }
5074 #endif
5075
5076 #ifdef HAVE_DISASM
5077 zend_jit_disasm_shutdown();
5078 #endif
5079
5080 #ifdef HAVE_PERFTOOLS
5081 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
5082 zend_jit_perf_jitdump_close();
5083 }
5084 #endif
5085 #ifdef ZTS
5086 ts_free_id(jit_globals_id);
5087 #else
5088 zend_jit_trace_free_caches(&jit_globals);
5089 #endif
5090 }
5091
zend_jit_reset_counters(void)5092 static void zend_jit_reset_counters(void)
5093 {
5094 int i;
5095
5096 for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
5097 zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT;
5098 }
5099 }
5100
zend_jit_activate(void)5101 ZEND_EXT_API void zend_jit_activate(void)
5102 {
5103 zend_jit_profile_counter = 0;
5104 if (JIT_G(on)) {
5105 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
5106 zend_jit_reset_counters();
5107 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
5108 zend_jit_reset_counters();
5109 zend_jit_trace_reset_caches();
5110 }
5111 }
5112 }
5113
zend_jit_deactivate(void)5114 ZEND_EXT_API void zend_jit_deactivate(void)
5115 {
5116 if (zend_jit_profile_counter && !CG(unclean_shutdown)) {
5117 zend_class_entry *ce;
5118
5119 zend_shared_alloc_lock();
5120 SHM_UNPROTECT();
5121 zend_jit_unprotect();
5122
5123 zend_jit_check_funcs(EG(function_table), 0);
5124 ZEND_HASH_REVERSE_FOREACH_PTR(EG(class_table), ce) {
5125 if (ce->type == ZEND_INTERNAL_CLASS) {
5126 break;
5127 }
5128 zend_jit_check_funcs(&ce->function_table, 1);
5129 } ZEND_HASH_FOREACH_END();
5130
5131 zend_jit_protect();
5132 SHM_PROTECT();
5133 zend_shared_alloc_unlock();
5134 }
5135
5136 zend_jit_profile_counter = 0;
5137 }
5138
zend_jit_restart_preloaded_op_array(zend_op_array * op_array)5139 static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array)
5140 {
5141 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
5142
5143 if (!func_info) {
5144 return;
5145 }
5146
5147 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
5148 zend_jit_restart_hot_trace_counters(op_array);
5149 } else if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_COUNTERS) {
5150 zend_jit_restart_hot_counters(op_array);
5151 #if 0
5152 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ???
5153 } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) {
5154 zend_op *opline = op_array->opcodes;
5155 zend_jit_op_array_extension *jit_extension =
5156 (zend_jit_op_array_extension*)func_info;
5157
5158 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
5159 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
5160 opline++;
5161 }
5162 }
5163 if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) {
5164 opline->handler = (const void*)zend_jit_runtime_jit_handler;
5165 } else {
5166 opline->handler = (const void*)zend_jit_profile_jit_handler;
5167 }
5168 #endif
5169 }
5170 if (op_array->num_dynamic_func_defs) {
5171 for (uint32_t i = 0; i < op_array->num_dynamic_func_defs; i++) {
5172 zend_jit_restart_preloaded_op_array(op_array->dynamic_func_defs[i]);
5173 }
5174 }
5175 }
5176
zend_jit_restart_preloaded_script(zend_persistent_script * script)5177 static void zend_jit_restart_preloaded_script(zend_persistent_script *script)
5178 {
5179 zend_class_entry *ce;
5180 zend_op_array *op_array;
5181
5182 zend_jit_restart_preloaded_op_array(&script->script.main_op_array);
5183
5184 ZEND_HASH_FOREACH_PTR(&script->script.function_table, op_array) {
5185 zend_jit_restart_preloaded_op_array(op_array);
5186 } ZEND_HASH_FOREACH_END();
5187
5188 ZEND_HASH_FOREACH_PTR(&script->script.class_table, ce) {
5189 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
5190 if (op_array->type == ZEND_USER_FUNCTION) {
5191 zend_jit_restart_preloaded_op_array(op_array);
5192 }
5193 } ZEND_HASH_FOREACH_END();
5194 } ZEND_HASH_FOREACH_END();
5195 }
5196
zend_jit_restart(void)5197 ZEND_EXT_API void zend_jit_restart(void)
5198 {
5199 if (dasm_buf) {
5200 zend_jit_unprotect();
5201
5202 #if ZEND_JIT_TARGET_ARM64
5203 memset(dasm_labels_veneers, 0, sizeof(void*) * ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT));
5204 #endif
5205
5206 /* restore JIT buffer pos */
5207 dasm_ptr[0] = dasm_ptr[1];
5208
5209 zend_jit_trace_restart();
5210
5211 if (ZCSG(preload_script)) {
5212 zend_jit_restart_preloaded_script(ZCSG(preload_script));
5213 if (ZCSG(saved_scripts)) {
5214 zend_persistent_script **p = ZCSG(saved_scripts);
5215
5216 while (*p) {
5217 zend_jit_restart_preloaded_script(*p);
5218 p++;
5219 }
5220 }
5221 }
5222
5223 zend_jit_protect();
5224
5225 #ifdef HAVE_DISASM
5226 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
5227 zend_jit_disasm_shutdown();
5228 zend_jit_disasm_init();
5229 }
5230 #endif
5231 }
5232 }
5233
5234 #endif /* HAVE_JIT */
5235
