1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 #include "main/php.h"
20 #include "main/SAPI.h"
21 #include "php_version.h"
22 #include <ZendAccelerator.h>
23 #include "zend_shared_alloc.h"
24 #include "Zend/zend_execute.h"
25 #include "Zend/zend_vm.h"
26 #include "Zend/zend_exceptions.h"
27 #include "Zend/zend_constants.h"
28 #include "Zend/zend_closures.h"
29 #include "Zend/zend_ini.h"
30 #include "Zend/zend_observer.h"
31 #include "zend_smart_str.h"
32 #include "jit/zend_jit.h"
33
34 #ifdef HAVE_JIT
35
36 #include "Optimizer/zend_func_info.h"
37 #include "Optimizer/zend_ssa.h"
38 #include "Optimizer/zend_inference.h"
39 #include "Optimizer/zend_call_graph.h"
40 #include "Optimizer/zend_dump.h"
41
42 #if ZEND_JIT_TARGET_X86
43 # include "jit/zend_jit_x86.h"
44 #elif ZEND_JIT_TARGET_ARM64
45 # include "jit/zend_jit_arm64.h"
46 #endif
47
48 #include "jit/zend_jit_internal.h"
49
50 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
51 #include <pthread.h>
52 #endif
53
54 #ifdef ZTS
55 int jit_globals_id;
56 #else
57 zend_jit_globals jit_globals;
58 #endif
59
60 //#define CONTEXT_THREADED_JIT
61 #define ZEND_JIT_USE_RC_INFERENCE
62
63 #ifdef ZEND_JIT_USE_RC_INFERENCE
64 # define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE
65 # define RC_MAY_BE_1(info) (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0)
66 # define RC_MAY_BE_N(info) (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0)
67 #else
68 # define ZEND_SSA_RC_INFERENCE_FLAG 0
69 # define RC_MAY_BE_1(info) 1
70 # define RC_MAY_BE_N(info) 1
71 #endif
72
73 #define JIT_PREFIX "JIT$"
74 #define JIT_STUB_PREFIX "JIT$$"
75 #define TRACE_PREFIX "TRACE-"
76
77 #define DASM_M_GROW(ctx, t, p, sz, need) \
78 do { \
79 size_t _sz = (sz), _need = (need); \
80 if (_sz < _need) { \
81 if (_sz < 16) _sz = 16; \
82 while (_sz < _need) _sz += _sz; \
83 (p) = (t *)erealloc((p), _sz); \
84 (sz) = _sz; \
85 } \
86 } while(0)
87
88 #define DASM_M_FREE(ctx, p, sz) efree(p)
89
90 #if ZEND_DEBUG
91 # define DASM_CHECKS 1
92 #endif
93
94 #include "dynasm/dasm_proto.h"
95
96 typedef struct _zend_jit_stub {
97 const char *name;
98 int (*stub)(dasm_State **Dst);
99 uint32_t offset;
100 uint32_t adjustment;
101 } zend_jit_stub;
102
103 #define JIT_STUB(name, offset, adjustment) \
104 {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub, offset, adjustment}
105
106 zend_ulong zend_jit_profile_counter = 0;
107 int zend_jit_profile_counter_rid = -1;
108
109 int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
110
111 const zend_op *zend_jit_halt_op = NULL;
112 static int zend_jit_vm_kind = 0;
113 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
114 static int zend_write_protect = 1;
115 #endif
116
117 static void *dasm_buf = NULL;
118 static void *dasm_end = NULL;
119 static void **dasm_ptr = NULL;
120
121 static size_t dasm_size = 0;
122
123 static zend_long jit_bisect_pos = 0;
124
125 static const void *zend_jit_runtime_jit_handler = NULL;
126 static const void *zend_jit_profile_jit_handler = NULL;
127 static const void *zend_jit_func_hot_counter_handler = NULL;
128 static const void *zend_jit_loop_hot_counter_handler = NULL;
129 static const void *zend_jit_func_trace_counter_handler = NULL;
130 static const void *zend_jit_ret_trace_counter_handler = NULL;
131 static const void *zend_jit_loop_trace_counter_handler = NULL;
132
133 static int ZEND_FASTCALL zend_runtime_jit(void);
134
135 static int zend_jit_trace_op_len(const zend_op *opline);
136 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
137 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags);
138 static const void *zend_jit_trace_get_exit_addr(uint32_t n);
139 static void zend_jit_trace_add_code(const void *start, uint32_t size);
140 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info);
141
142 #if ZEND_JIT_TARGET_ARM64
143 static zend_jit_trace_info *zend_jit_get_current_trace_info(void);
144 static uint32_t zend_jit_trace_find_exit_point(const void* addr);
145 #endif
146
147 static int zend_jit_assign_to_variable(dasm_State **Dst,
148 const zend_op *opline,
149 zend_jit_addr var_use_addr,
150 zend_jit_addr var_addr,
151 uint32_t var_info,
152 uint32_t var_def_info,
153 zend_uchar val_type,
154 zend_jit_addr val_addr,
155 uint32_t val_info,
156 zend_jit_addr res_addr,
157 bool check_exception);
158
dominates(const zend_basic_block * blocks,int a,int b)159 static bool dominates(const zend_basic_block *blocks, int a, int b) {
160 while (blocks[b].level > blocks[a].level) {
161 b = blocks[b].idom;
162 }
163 return a == b;
164 }
165
zend_ssa_is_last_use(const zend_op_array * op_array,const zend_ssa * ssa,int var,int use)166 static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
167 {
168 int next_use;
169
170 if (ssa->vars[var].phi_use_chain) {
171 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
172 do {
173 if (!ssa->vars[phi->ssa_var].no_val) {
174 return 0;
175 }
176 phi = zend_ssa_next_use_phi(ssa, var, phi);
177 } while (phi);
178 }
179
180 if (ssa->cfg.blocks[ssa->cfg.map[use]].loop_header > 0
181 || (ssa->cfg.blocks[ssa->cfg.map[use]].flags & ZEND_BB_LOOP_HEADER)) {
182 int b = ssa->cfg.map[use];
183 int prev_use = ssa->vars[var].use_chain;
184
185 while (prev_use >= 0 && prev_use != use) {
186 if (b != ssa->cfg.map[prev_use]
187 && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use])
188 && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) {
189 return 0;
190 }
191 prev_use = zend_ssa_next_use(ssa->ops, var, prev_use);
192 }
193 }
194
195 next_use = zend_ssa_next_use(ssa->ops, var, use);
196 if (next_use < 0) {
197 return 1;
198 } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) {
199 return 1;
200 }
201 return 0;
202 }
203
zend_ival_is_last_use(const zend_lifetime_interval * ival,int use)204 static bool zend_ival_is_last_use(const zend_lifetime_interval *ival, int use)
205 {
206 if (ival->flags & ZREG_LAST_USE) {
207 const zend_life_range *range = &ival->range;
208
209 while (range->next) {
210 range = range->next;
211 }
212 return range->end == use;
213 }
214 return 0;
215 }
216
zend_is_commutative(zend_uchar opcode)217 static bool zend_is_commutative(zend_uchar opcode)
218 {
219 return
220 opcode == ZEND_ADD ||
221 opcode == ZEND_MUL ||
222 opcode == ZEND_BW_OR ||
223 opcode == ZEND_BW_AND ||
224 opcode == ZEND_BW_XOR;
225 }
226
zend_jit_is_constant_cmp_long_long(const zend_op * opline,zend_ssa_range * op1_range,zend_jit_addr op1_addr,zend_ssa_range * op2_range,zend_jit_addr op2_addr,bool * result)227 static int zend_jit_is_constant_cmp_long_long(const zend_op *opline,
228 zend_ssa_range *op1_range,
229 zend_jit_addr op1_addr,
230 zend_ssa_range *op2_range,
231 zend_jit_addr op2_addr,
232 bool *result)
233 {
234 zend_long op1_min;
235 zend_long op1_max;
236 zend_long op2_min;
237 zend_long op2_max;
238
239 if (op1_range) {
240 op1_min = op1_range->min;
241 op1_max = op1_range->max;
242 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
243 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG);
244 op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr));
245 } else {
246 return 0;
247 }
248
249 if (op2_range) {
250 op2_min = op2_range->min;
251 op2_max = op2_range->max;
252 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
253 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG);
254 op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr));
255 } else {
256 return 0;
257 }
258
259 switch (opline->opcode) {
260 case ZEND_IS_EQUAL:
261 case ZEND_IS_IDENTICAL:
262 case ZEND_CASE:
263 case ZEND_CASE_STRICT:
264 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
265 *result = 1;
266 return 1;
267 } else if (op1_max < op2_min || op1_min > op2_max) {
268 *result = 0;
269 return 1;
270 }
271 return 0;
272 case ZEND_IS_NOT_EQUAL:
273 case ZEND_IS_NOT_IDENTICAL:
274 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
275 *result = 0;
276 return 1;
277 } else if (op1_max < op2_min || op1_min > op2_max) {
278 *result = 1;
279 return 1;
280 }
281 return 0;
282 case ZEND_IS_SMALLER:
283 if (op1_max < op2_min) {
284 *result = 1;
285 return 1;
286 } else if (op1_min >= op2_max) {
287 *result = 0;
288 return 1;
289 }
290 return 0;
291 case ZEND_IS_SMALLER_OR_EQUAL:
292 if (op1_max <= op2_min) {
293 *result = 1;
294 return 1;
295 } else if (op1_min > op2_max) {
296 *result = 0;
297 return 1;
298 }
299 return 0;
300 default:
301 ZEND_UNREACHABLE();
302 }
303 return 0;
304 }
305
zend_jit_needs_call_chain(zend_call_info * call_info,uint32_t b,const zend_op_array * op_array,zend_ssa * ssa,const zend_ssa_op * ssa_op,const zend_op * opline,int call_level,zend_jit_trace_rec * trace)306 static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace)
307 {
308 int skip;
309
310 if (trace) {
311 zend_jit_trace_rec *p = trace;
312
313 ssa_op++;
314 while (1) {
315 if (p->op == ZEND_JIT_TRACE_VM) {
316 switch (p->opline->opcode) {
317 case ZEND_SEND_ARRAY:
318 case ZEND_SEND_USER:
319 case ZEND_SEND_UNPACK:
320 case ZEND_INIT_FCALL:
321 case ZEND_INIT_METHOD_CALL:
322 case ZEND_INIT_STATIC_METHOD_CALL:
323 case ZEND_INIT_FCALL_BY_NAME:
324 case ZEND_INIT_NS_FCALL_BY_NAME:
325 case ZEND_INIT_DYNAMIC_CALL:
326 case ZEND_NEW:
327 case ZEND_INIT_USER_CALL:
328 case ZEND_FAST_CALL:
329 case ZEND_JMP:
330 case ZEND_JMPZ:
331 case ZEND_JMPNZ:
332 case ZEND_JMPZ_EX:
333 case ZEND_JMPNZ_EX:
334 case ZEND_FE_RESET_R:
335 case ZEND_FE_RESET_RW:
336 case ZEND_JMP_SET:
337 case ZEND_COALESCE:
338 case ZEND_JMP_NULL:
339 case ZEND_ASSERT_CHECK:
340 case ZEND_CATCH:
341 case ZEND_DECLARE_ANON_CLASS:
342 case ZEND_FE_FETCH_R:
343 case ZEND_FE_FETCH_RW:
344 return 1;
345 case ZEND_DO_ICALL:
346 case ZEND_DO_UCALL:
347 case ZEND_DO_FCALL_BY_NAME:
348 case ZEND_DO_FCALL:
349 case ZEND_CALLABLE_CONVERT:
350 return 0;
351 case ZEND_SEND_VAL:
352 case ZEND_SEND_VAR:
353 case ZEND_SEND_VAL_EX:
354 case ZEND_SEND_VAR_EX:
355 case ZEND_SEND_FUNC_ARG:
356 case ZEND_SEND_REF:
357 case ZEND_SEND_VAR_NO_REF:
358 case ZEND_SEND_VAR_NO_REF_EX:
359 /* skip */
360 break;
361 default:
362 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
363 return 1;
364 }
365 }
366 ssa_op += zend_jit_trace_op_len(opline);
367 } else if (p->op == ZEND_JIT_TRACE_ENTER ||
368 p->op == ZEND_JIT_TRACE_BACK ||
369 p->op == ZEND_JIT_TRACE_END) {
370 return 1;
371 }
372 p++;
373 }
374 }
375
376 if (!call_info) {
377 const zend_op *end = op_array->opcodes + op_array->last;
378
379 opline++;
380 ssa_op++;
381 skip = (call_level == 1);
382 while (opline != end) {
383 if (!skip) {
384 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
385 return 1;
386 }
387 }
388 switch (opline->opcode) {
389 case ZEND_SEND_VAL:
390 case ZEND_SEND_VAR:
391 case ZEND_SEND_VAL_EX:
392 case ZEND_SEND_VAR_EX:
393 case ZEND_SEND_FUNC_ARG:
394 case ZEND_SEND_REF:
395 case ZEND_SEND_VAR_NO_REF:
396 case ZEND_SEND_VAR_NO_REF_EX:
397 skip = 0;
398 break;
399 case ZEND_SEND_ARRAY:
400 case ZEND_SEND_USER:
401 case ZEND_SEND_UNPACK:
402 case ZEND_INIT_FCALL:
403 case ZEND_INIT_METHOD_CALL:
404 case ZEND_INIT_STATIC_METHOD_CALL:
405 case ZEND_INIT_FCALL_BY_NAME:
406 case ZEND_INIT_NS_FCALL_BY_NAME:
407 case ZEND_INIT_DYNAMIC_CALL:
408 case ZEND_NEW:
409 case ZEND_INIT_USER_CALL:
410 case ZEND_FAST_CALL:
411 case ZEND_JMP:
412 case ZEND_JMPZ:
413 case ZEND_JMPNZ:
414 case ZEND_JMPZ_EX:
415 case ZEND_JMPNZ_EX:
416 case ZEND_FE_RESET_R:
417 case ZEND_FE_RESET_RW:
418 case ZEND_JMP_SET:
419 case ZEND_COALESCE:
420 case ZEND_JMP_NULL:
421 case ZEND_ASSERT_CHECK:
422 case ZEND_CATCH:
423 case ZEND_DECLARE_ANON_CLASS:
424 case ZEND_FE_FETCH_R:
425 case ZEND_FE_FETCH_RW:
426 return 1;
427 case ZEND_DO_ICALL:
428 case ZEND_DO_UCALL:
429 case ZEND_DO_FCALL_BY_NAME:
430 case ZEND_DO_FCALL:
431 case ZEND_CALLABLE_CONVERT:
432 end = opline;
433 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
434 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
435 return 1;
436 }
437 return 0;
438 }
439 opline++;
440 ssa_op++;
441 }
442
443 return 1;
444 } else {
445 const zend_op *end = call_info->caller_call_opline;
446
447 /* end may be null if an opcode like EXIT is part of the argument list. */
448 if (!end || end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
449 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
450 return 1;
451 }
452
453 opline++;
454 ssa_op++;
455 skip = (call_level == 1);
456 while (opline != end) {
457 if (skip) {
458 switch (opline->opcode) {
459 case ZEND_SEND_VAL:
460 case ZEND_SEND_VAR:
461 case ZEND_SEND_VAL_EX:
462 case ZEND_SEND_VAR_EX:
463 case ZEND_SEND_FUNC_ARG:
464 case ZEND_SEND_REF:
465 case ZEND_SEND_VAR_NO_REF:
466 case ZEND_SEND_VAR_NO_REF_EX:
467 skip = 0;
468 break;
469 case ZEND_SEND_ARRAY:
470 case ZEND_SEND_USER:
471 case ZEND_SEND_UNPACK:
472 return 1;
473 }
474 } else {
475 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
476 return 1;
477 }
478 }
479 opline++;
480 ssa_op++;
481 }
482
483 return 0;
484 }
485 }
486
skip_valid_arguments(const zend_op_array * op_array,zend_ssa * ssa,const zend_call_info * call_info)487 static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
488 {
489 uint32_t num_args = 0;
490 zend_function *func = call_info->callee_func;
491
492 /* It's okay to handle prototypes here, because they can only increase the accepted arguments.
493 * Anything legal for the parent method is also legal for the parent method. */
494 while (num_args < call_info->num_args) {
495 zend_arg_info *arg_info = func->op_array.arg_info + num_args;
496
497 if (ZEND_TYPE_IS_SET(arg_info->type)) {
498 if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) {
499 zend_op *opline = call_info->arg_info[num_args].opline;
500 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
501 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
502 if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) {
503 break;
504 }
505 } else {
506 break;
507 }
508 }
509 num_args++;
510 }
511 return num_args;
512 }
513
zend_ssa_cv_info(const zend_op_array * op_array,zend_ssa * ssa,uint32_t var)514 static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var)
515 {
516 uint32_t j, info;
517
518 if (ssa->vars && ssa->var_info) {
519 info = ssa->var_info[var].type;
520 for (j = op_array->last_var; j < ssa->vars_count; j++) {
521 if (ssa->vars[j].var == var) {
522 info |= ssa->var_info[j].type;
523 }
524 }
525 } else {
526 info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF |
527 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
528 }
529
530 #ifdef ZEND_JIT_USE_RC_INFERENCE
531 /* Refcount may be increased by RETURN opcode */
532 if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
533 for (j = 0; j < ssa->cfg.blocks_count; j++) {
534 if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
535 ssa->cfg.blocks[j].len > 0) {
536 const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
537
538 if (opline->opcode == ZEND_RETURN) {
539 if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) {
540 info |= MAY_BE_RCN;
541 break;
542 }
543 }
544 }
545 }
546 }
547 #endif
548
549 return info;
550 }
551
zend_jit_may_avoid_refcounting(const zend_op * opline,uint32_t op1_info)552 static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_info)
553 {
554 switch (opline->opcode) {
555 case ZEND_FETCH_OBJ_FUNC_ARG:
556 if (!JIT_G(current_frame) ||
557 !JIT_G(current_frame)->call->func ||
558 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
559 return 0;
560 }
561 /* break missing intentionally */
562 case ZEND_FETCH_OBJ_R:
563 case ZEND_FETCH_OBJ_IS:
564 if ((op1_info & MAY_BE_OBJECT)
565 && opline->op2_type == IS_CONST
566 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
567 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
568 return 1;
569 }
570 break;
571 case ZEND_FETCH_DIM_FUNC_ARG:
572 if (!JIT_G(current_frame) ||
573 !JIT_G(current_frame)->call->func ||
574 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
575 return 0;
576 }
577 /* break missing intentionally */
578 case ZEND_FETCH_DIM_R:
579 case ZEND_FETCH_DIM_IS:
580 return 1;
581 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
582 if (!(opline->extended_value & ZEND_ISEMPTY)) {
583 return 1;
584 }
585 break;
586 }
587 return 0;
588 }
589
zend_jit_is_persistent_constant(zval * key,uint32_t flags)590 static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags)
591 {
592 zval *zv;
593 zend_constant *c = NULL;
594
595 /* null/true/false are resolved during compilation, so don't check for them here. */
596 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
597 if (zv) {
598 c = (zend_constant*)Z_PTR_P(zv);
599 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
600 key++;
601 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
602 if (zv) {
603 c = (zend_constant*)Z_PTR_P(zv);
604 }
605 }
606 return c && (ZEND_CONSTANT_FLAGS(c) & CONST_PERSISTENT);
607 }
608
zend_get_known_property_info(const zend_op_array * op_array,zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)609 static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
610 {
611 zend_property_info *info = NULL;
612
613 if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) ||
614 !ce ||
615 !(ce->ce_flags & ZEND_ACC_LINKED) ||
616 (ce->ce_flags & ZEND_ACC_TRAIT) ||
617 ce->create_object) {
618 return NULL;
619 }
620
621 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
622 if (ce->info.user.filename != filename) {
623 /* class declaration might be changed independently */
624 return NULL;
625 }
626
627 if (ce->parent) {
628 zend_class_entry *parent = ce->parent;
629
630 do {
631 if (parent->type == ZEND_INTERNAL_CLASS) {
632 break;
633 } else if (parent->info.user.filename != filename) {
634 /* some of parents class declarations might be changed independently */
635 /* TODO: this check may be not enough, because even
636 * in the same it's possible to conditionally define
637 * few classes with the same name, and "parent" may
638 * change from request to request.
639 */
640 return NULL;
641 }
642 parent = parent->parent;
643 } while (parent);
644 }
645 }
646
647 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
648 if (info == NULL ||
649 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
650 (info->flags & ZEND_ACC_STATIC)) {
651 return NULL;
652 }
653
654 if (info->flags & ZEND_ACC_PUBLIC) {
655 return info;
656 } else if (on_this) {
657 if (ce == info->ce) {
658 if (ce == op_array->scope) {
659 return info;
660 } else {
661 return NULL;
662 }
663 } else if ((info->flags & ZEND_ACC_PROTECTED)
664 && instanceof_function_slow(ce, info->ce)) {
665 return info;
666 }
667 }
668
669 return NULL;
670 }
671
zend_may_be_dynamic_property(zend_class_entry * ce,zend_string * member,bool on_this,const zend_op_array * op_array)672 static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, const zend_op_array *op_array)
673 {
674 zend_property_info *info;
675
676 if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT) || (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) {
677 return 1;
678 }
679
680 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
681 if (ce->info.user.filename != op_array->filename) {
682 /* class declaration might be changed independently */
683 return 1;
684 }
685 }
686
687 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
688 if (info == NULL ||
689 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
690 (info->flags & ZEND_ACC_STATIC)) {
691 return 1;
692 }
693
694 if (!(info->flags & ZEND_ACC_PUBLIC) &&
695 (!on_this || info->ce != ce)) {
696 return 1;
697 }
698
699 return 0;
700 }
701
702 #define OP_RANGE(ssa_op, opN) \
703 (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
704 ssa->var_info && \
705 (ssa_op)->opN##_use >= 0 && \
706 ssa->var_info[(ssa_op)->opN##_use].has_range) ? \
707 &ssa->var_info[(ssa_op)->opN##_use].range : NULL)
708
709 #define OP1_RANGE() OP_RANGE(ssa_op, op1)
710 #define OP2_RANGE() OP_RANGE(ssa_op, op2)
711 #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)
712
713 #if ZEND_JIT_TARGET_X86
714 # include "dynasm/dasm_x86.h"
715 #elif ZEND_JIT_TARGET_ARM64
716 static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset);
717 # define DASM_ADD_VENEER zend_jit_add_veneer
718 # include "dynasm/dasm_arm64.h"
719 #endif
720
721 #include "jit/zend_jit_helpers.c"
722 #include "jit/zend_jit_disasm.c"
723 #ifndef _WIN32
724 # include "jit/zend_jit_gdb.h"
725 # include "jit/zend_jit_perf_dump.c"
726 #endif
727 #ifdef HAVE_OPROFILE
728 # include "jit/zend_jit_oprofile.c"
729 #endif
730
731 #include "Zend/zend_cpuinfo.h"
732
733 #ifdef HAVE_VALGRIND
734 # include <valgrind/valgrind.h>
735 #endif
736
737 #ifdef HAVE_GCC_GLOBAL_REGS
738 # define GCC_GLOBAL_REGS 1
739 #else
740 # define GCC_GLOBAL_REGS 0
741 #endif
742
743 /* By default avoid JITing inline handlers if it does not seem profitable due to lack of
744 * type information. Disabling this option allows testing some JIT handlers in the
745 * presence of try/catch blocks, which prevent SSA construction. */
746 #ifndef PROFITABILITY_CHECKS
747 # define PROFITABILITY_CHECKS 1
748 #endif
749
750 #define BP_JIT_IS 6 /* Used for ISSET_ISEMPTY_DIM_OBJ. see BP_VAR_*defines in Zend/zend_compile.h */
751
752 typedef enum _sp_adj_kind {
753 SP_ADJ_NONE,
754 SP_ADJ_RET,
755 SP_ADJ_VM,
756 SP_ADJ_JIT,
757 SP_ADJ_ASSIGN,
758 SP_ADJ_LAST
759 } sp_adj_kind;
760
761 static int sp_adj[SP_ADJ_LAST];
762
763 /* The generated code may contain tautological comparisons, ignore them. */
764 #if defined(__clang__)
765 # pragma clang diagnostic push
766 # pragma clang diagnostic ignored "-Wtautological-compare"
767 # pragma clang diagnostic ignored "-Wstring-compare"
768 #endif
769
770 #if ZEND_JIT_TARGET_X86
771 # include "jit/zend_jit_vtune.c"
772 # include "jit/zend_jit_x86.c"
773 #elif ZEND_JIT_TARGET_ARM64
774 # include "jit/zend_jit_arm64.c"
775 #endif
776
777 #if defined(__clang__)
778 # pragma clang diagnostic pop
779 #endif
780
781 #if _WIN32
782 # include <Windows.h>
783 #else
784 # include <sys/mman.h>
785 # if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
786 # define MAP_ANONYMOUS MAP_ANON
787 # endif
788 #endif
789
zend_jit_status(zval * ret)790 ZEND_EXT_API void zend_jit_status(zval *ret)
791 {
792 zval stats;
793 array_init(&stats);
794 add_assoc_bool(&stats, "enabled", JIT_G(enabled));
795 add_assoc_bool(&stats, "on", JIT_G(on));
796 add_assoc_long(&stats, "kind", JIT_G(trigger));
797 add_assoc_long(&stats, "opt_level", JIT_G(opt_level));
798 add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags));
799 if (dasm_buf) {
800 add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
801 add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
802 } else {
803 add_assoc_long(&stats, "buffer_size", 0);
804 add_assoc_long(&stats, "buffer_free", 0);
805 }
806 add_assoc_zval(ret, "jit", &stats);
807 }
808
zend_jit_func_name(const zend_op_array * op_array)809 static zend_string *zend_jit_func_name(const zend_op_array *op_array)
810 {
811 smart_str buf = {0};
812
813 if (op_array->function_name) {
814 if (op_array->scope) {
815 smart_str_appends(&buf, JIT_PREFIX);
816 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
817 smart_str_appends(&buf, "::");
818 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
819 smart_str_0(&buf);
820 return buf.s;
821 } else {
822 smart_str_appends(&buf, JIT_PREFIX);
823 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
824 smart_str_0(&buf);
825 return buf.s;
826 }
827 } else if (op_array->filename) {
828 smart_str_appends(&buf, JIT_PREFIX);
829 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
830 smart_str_0(&buf);
831 return buf.s;
832 } else {
833 return NULL;
834 }
835 }
836
837 #if ZEND_DEBUG
handle_dasm_error(int ret)838 static void handle_dasm_error(int ret) {
839 switch (ret & 0xff000000u) {
840 case DASM_S_NOMEM:
841 fprintf(stderr, "DASM_S_NOMEM\n");
842 break;
843 case DASM_S_PHASE:
844 fprintf(stderr, "DASM_S_PHASE\n");
845 break;
846 case DASM_S_MATCH_SEC:
847 fprintf(stderr, "DASM_S_MATCH_SEC\n");
848 break;
849 case DASM_S_RANGE_I:
850 fprintf(stderr, "DASM_S_RANGE_I\n");
851 break;
852 case DASM_S_RANGE_SEC:
853 fprintf(stderr, "DASM_S_RANGE_SEC\n");
854 break;
855 case DASM_S_RANGE_LG:
856 fprintf(stderr, "DASM_S_RANGE_LG\n");
857 break;
858 case DASM_S_RANGE_PC:
859 fprintf(stderr, "DASM_S_RANGE_PC %d\n", ret & 0xffffffu);
860 break;
861 #ifdef DASM_S_RANGE_VREG
862 case DASM_S_RANGE_VREG:
863 fprintf(stderr, "DASM_S_RANGE_VREG\n");
864 break;
865 #endif
866 #ifdef DASM_S_UNDEF_L
867 case DASM_S_UNDEF_L:
868 fprintf(stderr, "DASM_S_UNDEF_L\n");
869 break;
870 #endif
871 #ifdef DASM_S_UNDEF_LG
872 case DASM_S_UNDEF_LG:
873 fprintf(stderr, "DASM_S_UNDEF_LG\n");
874 break;
875 #endif
876 #ifdef DASM_S_RANGE_REL
877 case DASM_S_RANGE_REL:
878 fprintf(stderr, "DASM_S_RANGE_REL\n");
879 break;
880 #endif
881 case DASM_S_UNDEF_PC:
882 fprintf(stderr, "DASM_S_UNDEF_PC %d\n", ret & 0xffffffu);
883 break;
884 default:
885 fprintf(stderr, "DASM_S_%0x\n", ret & 0xff000000u);
886 break;
887 }
888 ZEND_UNREACHABLE();
889 }
890 #endif
891
dasm_link_and_encode(dasm_State ** dasm_state,const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline,zend_lifetime_interval ** ra,const char * name,uint32_t trace_num,uint32_t sp_offset,uint32_t sp_adjustment)892 static void *dasm_link_and_encode(dasm_State **dasm_state,
893 const zend_op_array *op_array,
894 zend_ssa *ssa,
895 const zend_op *rt_opline,
896 zend_lifetime_interval **ra,
897 const char *name,
898 uint32_t trace_num,
899 uint32_t sp_offset,
900 uint32_t sp_adjustment)
901 {
902 size_t size;
903 int ret;
904 void *entry;
905 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
906 zend_string *str = NULL;
907 #endif
908
909 if (rt_opline && ssa && ssa->cfg.map) {
910 /* Create additional entry point, to switch from interpreter to JIT-ed
911 * code at run-time.
912 */
913 int b = ssa->cfg.map[rt_opline - op_array->opcodes];
914
915 //#ifdef CONTEXT_THREADED_JIT
916 // if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY))) {
917 //#else
918 if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY))) {
919 //#endif
920 zend_jit_label(dasm_state, ssa->cfg.blocks_count + b);
921 zend_jit_prologue(dasm_state);
922 if (ra) {
923 int i;
924 zend_lifetime_interval *ival;
925 zend_life_range *range;
926 uint32_t pos = rt_opline - op_array->opcodes;
927
928 for (i = 0; i < ssa->vars_count; i++) {
929 ival = ra[i];
930
931 if (ival && ival->reg != ZREG_NONE) {
932 range = &ival->range;
933
934 if (pos >= range->start && pos <= range->end) {
935 if (!zend_jit_load_var(dasm_state, ssa->var_info[i].type, ssa->vars[i].var, ival->reg)) {
936 return NULL;
937 }
938 break;
939 }
940 range = range->next;
941 }
942 }
943 }
944 zend_jit_jmp(dasm_state, b);
945 }
946 }
947
948 ret = dasm_link(dasm_state, &size);
949 if (ret != DASM_S_OK) {
950 #if ZEND_DEBUG
951 handle_dasm_error(ret);
952 #endif
953 return NULL;
954 }
955
956 if ((void*)((char*)*dasm_ptr + size) > dasm_end) {
957 *dasm_ptr = dasm_end; //prevent further try
958 // TODO: jit_buffer_size overflow ???
959 return NULL;
960 }
961
962 #if ZEND_JIT_TARGET_ARM64
963 dasm_venners_size = 0;
964 #endif
965
966 ret = dasm_encode(dasm_state, *dasm_ptr);
967 if (ret != DASM_S_OK) {
968 #if ZEND_DEBUG
969 handle_dasm_error(ret);
970 #endif
971 return NULL;
972 }
973
974 #if ZEND_JIT_TARGET_ARM64
975 size += dasm_venners_size;
976 #endif
977
978 entry = *dasm_ptr;
979 *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT));
980
981 /* flush the hardware I-cache */
982 JIT_CACHE_FLUSH(entry, entry + size);
983
984 if (trace_num) {
985 zend_jit_trace_add_code(entry, dasm_getpclabel(dasm_state, 1));
986 }
987
988 if (op_array && ssa) {
989 int b;
990
991 for (b = 0; b < ssa->cfg.blocks_count; b++) {
992 //#ifdef CONTEXT_THREADED_JIT
993 // if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
994 //#else
995 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) {
996 //#endif
997 zend_op *opline = op_array->opcodes + ssa->cfg.blocks[b].start;
998 int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
999
1000 if (offset >= 0) {
1001 opline->handler = (void*)(((char*)entry) + offset);
1002 }
1003 }
1004 }
1005 if (rt_opline && ssa && ssa->cfg.map) {
1006 int b = ssa->cfg.map[rt_opline - op_array->opcodes];
1007 zend_op *opline = (zend_op*)rt_opline;
1008 int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
1009
1010 if (offset >= 0) {
1011 opline->handler = (void*)(((char*)entry) + offset);
1012 }
1013 }
1014 }
1015
1016 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
1017 if (!name) {
1018 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) {
1019 str = zend_jit_func_name(op_array);
1020 if (str) {
1021 name = ZSTR_VAL(str);
1022 }
1023 }
1024 #ifdef HAVE_DISASM
1025 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
1026 zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
1027 zend_jit_disasm(
1028 name,
1029 (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
1030 op_array,
1031 &ssa->cfg,
1032 entry,
1033 size);
1034 }
1035 } else {
1036 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) {
1037 zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
1038 if ((JIT_G(debug) & (trace_num ? ZEND_JIT_DEBUG_ASM : ZEND_JIT_DEBUG_ASM_STUBS)) != 0) {
1039 zend_jit_disasm(
1040 name,
1041 (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
1042 op_array,
1043 ssa ? &ssa->cfg : NULL,
1044 entry,
1045 size);
1046 }
1047 }
1048 # endif
1049 }
1050 #endif
1051
1052 #ifdef HAVE_GDB
1053 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
1054 if (name) {
1055 zend_jit_gdb_register(
1056 name,
1057 op_array,
1058 entry,
1059 size,
1060 sp_adj[sp_offset],
1061 sp_adj[sp_adjustment]);
1062 }
1063 }
1064 #endif
1065
1066 #ifdef HAVE_OPROFILE
1067 if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
1068 zend_jit_oprofile_register(
1069 name,
1070 entry,
1071 size);
1072 }
1073 #endif
1074
1075 #ifdef HAVE_PERFTOOLS
1076 if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
1077 if (name) {
1078 zend_jit_perf_map_register(
1079 name,
1080 entry,
1081 size);
1082 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
1083 zend_jit_perf_jitdump_register(
1084 name,
1085 entry,
1086 size);
1087 }
1088 }
1089 }
1090 #endif
1091
1092 #ifdef HAVE_VTUNE
1093 if (JIT_G(debug) & ZEND_JIT_DEBUG_VTUNE) {
1094 if (name) {
1095 zend_jit_vtune_register(
1096 name,
1097 entry,
1098 size);
1099 }
1100 }
1101 #endif
1102
1103 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
1104 if (str) {
1105 zend_string_release(str);
1106 }
1107 #endif
1108
1109 return entry;
1110 }
1111
zend_may_overflow(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa)1112 static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
1113 {
1114 int res;
1115 zend_long op1_min, op1_max, op2_min, op2_max;
1116
1117 if (!ssa->ops || !ssa->var_info) {
1118 return 1;
1119 }
1120 switch (opline->opcode) {
1121 case ZEND_PRE_INC:
1122 case ZEND_POST_INC:
1123 res = ssa_op->op1_def;
1124 if (res < 0
1125 || !ssa->var_info[res].has_range
1126 || ssa->var_info[res].range.overflow) {
1127 if (!OP1_HAS_RANGE()) {
1128 return 1;
1129 }
1130 op1_max = OP1_MAX_RANGE();
1131 if (op1_max == ZEND_LONG_MAX) {
1132 return 1;
1133 }
1134 }
1135 return 0;
1136 case ZEND_PRE_DEC:
1137 case ZEND_POST_DEC:
1138 res = ssa_op->op1_def;
1139 if (res < 0
1140 || !ssa->var_info[res].has_range
1141 || ssa->var_info[res].range.underflow) {
1142 if (!OP1_HAS_RANGE()) {
1143 return 1;
1144 }
1145 op1_min = OP1_MIN_RANGE();
1146 if (op1_min == ZEND_LONG_MIN) {
1147 return 1;
1148 }
1149 }
1150 return 0;
1151 case ZEND_ADD:
1152 res = ssa_op->result_def;
1153 if (res < 0
1154 || !ssa->var_info[res].has_range
1155 || ssa->var_info[res].range.underflow) {
1156 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1157 return 1;
1158 }
1159 op1_min = OP1_MIN_RANGE();
1160 op2_min = OP2_MIN_RANGE();
1161 if (zend_add_will_overflow(op1_min, op2_min)) {
1162 return 1;
1163 }
1164 }
1165 if (res < 0
1166 || !ssa->var_info[res].has_range
1167 || ssa->var_info[res].range.overflow) {
1168 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1169 return 1;
1170 }
1171 op1_max = OP1_MAX_RANGE();
1172 op2_max = OP2_MAX_RANGE();
1173 if (zend_add_will_overflow(op1_max, op2_max)) {
1174 return 1;
1175 }
1176 }
1177 return 0;
1178 case ZEND_SUB:
1179 res = ssa_op->result_def;
1180 if (res < 0
1181 || !ssa->var_info[res].has_range
1182 || ssa->var_info[res].range.underflow) {
1183 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1184 return 1;
1185 }
1186 op1_min = OP1_MIN_RANGE();
1187 op2_max = OP2_MAX_RANGE();
1188 if (zend_sub_will_overflow(op1_min, op2_max)) {
1189 return 1;
1190 }
1191 }
1192 if (res < 0
1193 || !ssa->var_info[res].has_range
1194 || ssa->var_info[res].range.overflow) {
1195 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1196 return 1;
1197 }
1198 op1_max = OP1_MAX_RANGE();
1199 op2_min = OP2_MIN_RANGE();
1200 if (zend_sub_will_overflow(op1_max, op2_min)) {
1201 return 1;
1202 }
1203 }
1204 return 0;
1205 case ZEND_MUL:
1206 res = ssa_op->result_def;
1207 return (res < 0 ||
1208 !ssa->var_info[res].has_range ||
1209 ssa->var_info[res].range.underflow ||
1210 ssa->var_info[res].range.overflow);
1211 case ZEND_ASSIGN_OP:
1212 if (opline->extended_value == ZEND_ADD) {
1213 res = ssa_op->op1_def;
1214 if (res < 0
1215 || !ssa->var_info[res].has_range
1216 || ssa->var_info[res].range.underflow) {
1217 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1218 return 1;
1219 }
1220 op1_min = OP1_MIN_RANGE();
1221 op2_min = OP2_MIN_RANGE();
1222 if (zend_add_will_overflow(op1_min, op2_min)) {
1223 return 1;
1224 }
1225 }
1226 if (res < 0
1227 || !ssa->var_info[res].has_range
1228 || ssa->var_info[res].range.overflow) {
1229 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1230 return 1;
1231 }
1232 op1_max = OP1_MAX_RANGE();
1233 op2_max = OP2_MAX_RANGE();
1234 if (zend_add_will_overflow(op1_max, op2_max)) {
1235 return 1;
1236 }
1237 }
1238 return 0;
1239 } else if (opline->extended_value == ZEND_SUB) {
1240 res = ssa_op->op1_def;
1241 if (res < 0
1242 || !ssa->var_info[res].has_range
1243 || ssa->var_info[res].range.underflow) {
1244 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1245 return 1;
1246 }
1247 op1_min = OP1_MIN_RANGE();
1248 op2_max = OP2_MAX_RANGE();
1249 if (zend_sub_will_overflow(op1_min, op2_max)) {
1250 return 1;
1251 }
1252 }
1253 if (res < 0
1254 || !ssa->var_info[res].has_range
1255 || ssa->var_info[res].range.overflow) {
1256 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1257 return 1;
1258 }
1259 op1_max = OP1_MAX_RANGE();
1260 op2_min = OP2_MIN_RANGE();
1261 if (zend_sub_will_overflow(op1_max, op2_min)) {
1262 return 1;
1263 }
1264 }
1265 return 0;
1266 } else if (opline->extended_value == ZEND_MUL) {
1267 res = ssa_op->op1_def;
1268 return (res < 0 ||
1269 !ssa->var_info[res].has_range ||
1270 ssa->var_info[res].range.underflow ||
1271 ssa->var_info[res].range.overflow);
1272 }
1273 ZEND_FALLTHROUGH;
1274 default:
1275 return 1;
1276 }
1277 }
1278
zend_jit_build_cfg(const zend_op_array * op_array,zend_cfg * cfg)1279 static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg)
1280 {
1281 uint32_t flags;
1282
1283 flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY;
1284
1285 zend_build_cfg(&CG(arena), op_array, flags, cfg);
1286
1287 /* Don't JIT huge functions. Apart from likely being detrimental due to the amount of
1288 * generated code, some of our analysis is recursive and will stack overflow with many
1289 * blocks. */
1290 if (cfg->blocks_count > 100000) {
1291 return FAILURE;
1292 }
1293
1294 zend_cfg_build_predecessors(&CG(arena), cfg);
1295
1296 /* Compute Dominators Tree */
1297 zend_cfg_compute_dominators_tree(op_array, cfg);
1298
1299 /* Identify reducible and irreducible loops */
1300 zend_cfg_identify_loops(op_array, cfg);
1301
1302 return SUCCESS;
1303 }
1304
zend_jit_op_array_analyze1(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa)1305 static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
1306 {
1307 if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) {
1308 return FAILURE;
1309 }
1310
1311 #if 0
1312 /* TODO: debugger and profiler supports? */
1313 if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) {
1314 return FAILURE;
1315 }
1316 #endif
1317
1318 /* TODO: move this to zend_cfg.c ? */
1319 if (!op_array->function_name) {
1320 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1321 }
1322
1323 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
1324 && ssa->cfg.blocks
1325 && op_array->last_try_catch == 0
1326 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
1327 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
1328 if (zend_build_ssa(&CG(arena), script, op_array, ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) {
1329 return FAILURE;
1330 }
1331
1332 zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa);
1333
1334 zend_ssa_find_false_dependencies(op_array, ssa);
1335
1336 zend_ssa_find_sccs(op_array, ssa);
1337 }
1338
1339 return SUCCESS;
1340 }
1341
zend_jit_op_array_analyze2(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa,uint32_t optimization_level)1342 static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t optimization_level)
1343 {
1344 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
1345 && ssa->cfg.blocks
1346 && op_array->last_try_catch == 0
1347 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
1348 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
1349 if (zend_ssa_inference(&CG(arena), op_array, script, ssa,
1350 optimization_level & ~ZEND_OPTIMIZER_NARROW_TO_DOUBLE) != SUCCESS) {
1351 return FAILURE;
1352 }
1353 }
1354
1355 return SUCCESS;
1356 }
1357
zend_jit_add_range(zend_lifetime_interval ** intervals,int var,uint32_t from,uint32_t to)1358 static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint32_t from, uint32_t to)
1359 {
1360 zend_lifetime_interval *ival = intervals[var];
1361
1362 if (!ival) {
1363 ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
1364 if (!ival) {
1365 return FAILURE;
1366 }
1367 ival->ssa_var = var;
1368 ival->reg = ZREG_NONE;
1369 ival->flags = 0;
1370 ival->range.start = from;
1371 ival->range.end = to;
1372 ival->range.next = NULL;
1373 ival->hint = NULL;
1374 ival->used_as_hint = NULL;
1375 intervals[var] = ival;
1376 } else if (ival->range.start > to + 1) {
1377 zend_life_range *range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1378
1379 if (!range) {
1380 return FAILURE;
1381 }
1382 range->start = ival->range.start;
1383 range->end = ival->range.end;
1384 range->next = ival->range.next;
1385 ival->range.start = from;
1386 ival->range.end = to;
1387 ival->range.next = range;
1388 } else if (ival->range.start == to + 1) {
1389 ival->range.start = from;
1390 } else {
1391 zend_life_range *range = &ival->range;
1392 zend_life_range *last = NULL;
1393
1394 do {
1395 if (range->start > to + 1) {
1396 break;
1397 } else if (range->end + 1 >= from) {
1398 if (range->start > from) {
1399 range->start = from;
1400 }
1401 last = range;
1402 range = range->next;
1403 while (range) {
1404 if (range->start > to + 1) {
1405 break;
1406 }
1407 last->end = range->end;
1408 range = range->next;
1409 last->next = range;
1410 }
1411 if (to > last->end) {
1412 last->end = to;
1413 }
1414 return SUCCESS;
1415 }
1416 last = range;
1417 range = range->next;
1418 } while (range);
1419
1420 range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1421 if (!range) {
1422 return FAILURE;
1423 }
1424 range->start = from;
1425 range->end = to;
1426 range->next = last->next;
1427 last->next = range;
1428 }
1429
1430 return SUCCESS;
1431 }
1432
zend_jit_begin_range(zend_lifetime_interval ** intervals,int var,uint32_t block_start,uint32_t from)1433 static int zend_jit_begin_range(zend_lifetime_interval **intervals, int var, uint32_t block_start, uint32_t from)
1434 {
1435 if (block_start != from && intervals[var]) {
1436 zend_life_range *range = &intervals[var]->range;
1437
1438 do {
1439 if (from >= range->start && from <= range->end) {
1440 if (range->start == block_start) {
1441 range->start = from;
1442 } else {
1443 zend_life_range *r = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1444 if (!r) {
1445 return FAILURE;
1446 }
1447 r->start = from;
1448 r->end = range->end;
1449 r->next = range->next;
1450 range->end = block_start - 1;
1451 range->next = r;
1452 }
1453 return SUCCESS;
1454 }
1455 range = range->next;
1456 } while (range);
1457 }
1458
1459 // dead store
1460 return zend_jit_add_range(intervals, var, from, from);
1461 }
1462
zend_jit_insert_interval(zend_lifetime_interval ** list,zend_lifetime_interval * ival)1463 static void zend_jit_insert_interval(zend_lifetime_interval **list, zend_lifetime_interval *ival)
1464 {
1465 while (1) {
1466 if (*list == NULL) {
1467 *list = ival;
1468 ival->list_next = NULL;
1469 return;
1470 } else if (ival->range.start < (*list)->range.start) {
1471 ival->list_next = *list;
1472 *list = ival;
1473 return;
1474 }
1475 list = &(*list)->list_next;
1476 }
1477 }
1478
zend_jit_split_interval(zend_lifetime_interval * current,uint32_t pos,zend_lifetime_interval ** list,zend_lifetime_interval ** free)1479 static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos, zend_lifetime_interval **list, zend_lifetime_interval **free)
1480 {
1481 zend_lifetime_interval *ival;
1482 zend_life_range *range = ¤t->range;
1483 zend_life_range *prev = NULL;
1484
1485 if (*free) {
1486 ival = *free;
1487 *free = ival->list_next;
1488 } else {
1489 ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
1490
1491 if (!ival) {
1492 return FAILURE;
1493 }
1494 }
1495
1496 current->flags |= ZREG_STORE;
1497
1498 ival->ssa_var = current->ssa_var;
1499 ival->reg = ZREG_NONE;
1500 ival->flags |= ZREG_SPLIT | ZREG_LOAD;
1501 ival->flags &= ~ZREG_STORE;
1502 ival->hint = NULL;
1503
1504 do {
1505 if (pos >= range->start && pos <= range->end) {
1506 break;
1507 }
1508 prev = range;
1509 range = range->next;
1510 } while(range);
1511
1512 ZEND_ASSERT(range != NULL);
1513
1514 ival->range.start = pos;
1515 ival->range.end = range->end;
1516 ival->range.next = range->next;
1517
1518 if (pos == range->start) {
1519 ZEND_ASSERT(prev != NULL);
1520 prev->next = NULL;
1521 } else {
1522 range->end = pos - 1;
1523 }
1524
1525 zend_jit_insert_interval(list, ival);
1526
1527 return SUCCESS;
1528 }
1529
zend_jit_sort_intervals(zend_lifetime_interval ** intervals,int count)1530 static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval **intervals, int count)
1531 {
1532 zend_lifetime_interval *list, *last;
1533 int i;
1534
1535 list = NULL;
1536 i = 0;
1537 while (i < count) {
1538 list = intervals[i];
1539 i++;
1540 if (list) {
1541 last = list;
1542 last->list_next = NULL;
1543 break;
1544 }
1545 }
1546
1547 while (i < count) {
1548 zend_lifetime_interval *ival = intervals[i];
1549
1550 i++;
1551 if (ival) {
1552 if ((ival->range.start > last->range.start) ||
1553 (ival->range.start == last->range.start &&
1554 ((!ival->hint && last->hint && last->hint != ival) ||
1555 ival->range.end > last->range.end))) {
1556 last->list_next = ival;
1557 last = ival;
1558 ival->list_next = NULL;
1559 } else {
1560 zend_lifetime_interval **p = &list;
1561
1562 while (1) {
1563 if (*p == NULL) {
1564 *p = last = ival;
1565 ival->list_next = NULL;
1566 break;
1567 } else if ((ival->range.start < (*p)->range.start) ||
1568 (ival->range.start == (*p)->range.start &&
1569 ((ival->hint && !(*p)->hint && ival->hint != *p) ||
1570 ival->range.end < (*p)->range.end))) {
1571 ival->list_next = *p;
1572 *p = ival;
1573 break;
1574 }
1575 p = &(*p)->list_next;
1576 }
1577 }
1578 }
1579 }
1580
1581 return list;
1582 }
1583
zend_jit_print_regset(zend_regset regset)1584 static ZEND_ATTRIBUTE_UNUSED void zend_jit_print_regset(zend_regset regset)
1585 {
1586 zend_reg reg;
1587 int first = 1;
1588
1589 ZEND_REGSET_FOREACH(regset, reg) {
1590 if (first) {
1591 first = 0;
1592 fprintf(stderr, "%s", zend_reg_name[reg]);
1593 } else {
1594 fprintf(stderr, ", %s", zend_reg_name[reg]);
1595 }
1596 } ZEND_REGSET_FOREACH_END();
1597 }
1598
zend_jit_compute_block_order_int(zend_ssa * ssa,int n,int * block_order)1599 static int *zend_jit_compute_block_order_int(zend_ssa *ssa, int n, int *block_order)
1600 {
1601 zend_basic_block *b = ssa->cfg.blocks + n;
1602
1603 tail_call:
1604 *block_order = n;
1605 block_order++;
1606
1607 n = b->children;
1608 while (n >= 0) {
1609 b = ssa->cfg.blocks + n;
1610 if (b->next_child < 0) {
1611 goto tail_call;
1612 }
1613 block_order = zend_jit_compute_block_order_int(ssa, n, block_order);
1614 n = b->next_child;
1615 }
1616
1617 return block_order;
1618 }
1619
zend_jit_compute_block_order(zend_ssa * ssa,int * block_order)1620 static int zend_jit_compute_block_order(zend_ssa *ssa, int *block_order)
1621 {
1622 int *end = zend_jit_compute_block_order_int(ssa, 0, block_order);
1623
1624 return end - block_order;
1625 }
1626
zend_jit_in_loop(zend_ssa * ssa,int header,zend_basic_block * b)1627 static bool zend_jit_in_loop(zend_ssa *ssa, int header, zend_basic_block *b)
1628 {
1629 while (b->loop_header >= 0) {
1630 if (b->loop_header == header) {
1631 return 1;
1632 }
1633 b = ssa->cfg.blocks + b->loop_header;
1634 }
1635 return 0;
1636 }
1637
zend_jit_compute_loop_body(zend_ssa * ssa,int header,int n,zend_bitset loop_body)1638 static void zend_jit_compute_loop_body(zend_ssa *ssa, int header, int n, zend_bitset loop_body)
1639 {
1640 zend_basic_block *b = ssa->cfg.blocks + n;
1641 uint32_t i;
1642
1643 tail_call:
1644 if (b->len) {
1645 for (i = b->start; i < b->start + b->len; i++) {
1646 zend_bitset_incl(loop_body, i);
1647 }
1648 }
1649
1650 n = b->children;
1651 while (n >= 0) {
1652 b = ssa->cfg.blocks + n;
1653 if (zend_jit_in_loop(ssa, header, b)) {
1654 if (b->next_child < 0) {
1655 goto tail_call;
1656 }
1657 zend_jit_compute_loop_body(ssa, header, n, loop_body);
1658 }
1659 n = b->next_child;
1660 }
1661 }
1662
zend_jit_add_hint(zend_lifetime_interval ** intervals,int dst,int src)1663 static void zend_jit_add_hint(zend_lifetime_interval **intervals, int dst, int src)
1664 {
1665 if (intervals[dst]->range.start < intervals[src]->range.start) {
1666 int tmp = src;
1667 src = dst;
1668 dst = tmp;
1669 }
1670 while (dst != src && intervals[dst]->hint) {
1671 if (intervals[dst]->hint->range.start < intervals[src]->range.start) {
1672 int tmp = src;
1673 src = intervals[dst]->hint->ssa_var;
1674 dst = tmp;
1675 } else {
1676 dst = intervals[dst]->hint->ssa_var;
1677 }
1678 }
1679 if (dst != src) {
1680 intervals[dst]->hint = intervals[src];
1681 }
1682 }
1683
1684 /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
1685 Michael Franz, CGO'10 (2010), Figure 4. */
zend_jit_compute_liveness(const zend_op_array * op_array,zend_ssa * ssa,zend_bitset candidates,zend_lifetime_interval ** list)1686 static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list)
1687 {
1688 int set_size, i, j, k, l;
1689 uint32_t n;
1690 zend_bitset live, live_in, pi_vars, loop_body;
1691 int *block_order;
1692 zend_ssa_phi *phi;
1693 zend_lifetime_interval **intervals;
1694 size_t mem_size;
1695 ALLOCA_FLAG(use_heap);
1696
1697 set_size = zend_bitset_len(ssa->vars_count);
1698 mem_size =
1699 ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)) +
1700 ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE) +
1701 ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
1702 ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
1703 ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE) +
1704 ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(int));
1705 intervals = do_alloca(mem_size, use_heap);
1706 if (!intervals) {
1707 *list = NULL;
1708 return FAILURE;
1709 }
1710
1711 live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)));
1712 live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE));
1713 pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
1714 loop_body = (zend_bitset)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
1715 block_order = (int*)((char*)loop_body + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE));
1716
1717 memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*));
1718 zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count);
1719
1720 /* TODO: Provide a linear block order where all dominators of a block
1721 * are before this block, and where all blocks belonging to the same loop
1722 * are contiguous ???
1723 */
1724 for (l = zend_jit_compute_block_order(ssa, block_order) - 1; l >= 0; l--) {
1725 zend_basic_block *b;
1726
1727 i = block_order[l];
1728 b = ssa->cfg.blocks + i;
1729
1730 /* live = UNION of successor.liveIn for each successor of b */
1731 /* live.add(phi.inputOf(b)) for each phi of successors of b */
1732 zend_bitset_clear(live, set_size);
1733 for (j = 0; j < b->successors_count; j++) {
1734 int succ = b->successors[j];
1735
1736 zend_bitset_union(live, live_in + set_size * succ, set_size);
1737 zend_bitset_clear(pi_vars, set_size);
1738 for (phi = ssa->blocks[succ].phis; phi; phi = phi->next) {
1739 if (ssa->vars[phi->ssa_var].no_val) {
1740 /* skip */
1741 } else if (phi->pi >= 0) {
1742 if (phi->pi == i && phi->sources[0] >= 0) {
1743 if (zend_bitset_in(candidates, phi->sources[0])) {
1744 zend_bitset_incl(live, phi->sources[0]);
1745 }
1746 zend_bitset_incl(pi_vars, phi->var);
1747 }
1748 } else if (!zend_bitset_in(pi_vars, phi->var)) {
1749 for (k = 0; k < ssa->cfg.blocks[succ].predecessors_count; k++) {
1750 if (ssa->cfg.predecessors[ssa->cfg.blocks[succ].predecessor_offset + k] == i) {
1751 if (phi->sources[k] >= 0 && zend_bitset_in(candidates, phi->sources[k])) {
1752 zend_bitset_incl(live, phi->sources[k]);
1753 }
1754 break;
1755 }
1756 }
1757 }
1758 }
1759 }
1760
1761 /* addRange(var, b.from, b.to) for each var in live */
1762 ZEND_BITSET_FOREACH(live, set_size, j) {
1763 if (zend_bitset_in(candidates, j)) {
1764 if (zend_jit_add_range(intervals, j, b->start, b->start + b->len - 1) != SUCCESS) {
1765 goto failure;
1766 }
1767 }
1768 } ZEND_BITSET_FOREACH_END();
1769
1770 /* for each operation op of b in reverse order */
1771 for (n = b->start + b->len; n > b->start;) {
1772 zend_ssa_op *op;
1773 const zend_op *opline;
1774 uint32_t num;
1775
1776 n--;
1777 op = ssa->ops + n;
1778 opline = op_array->opcodes + n;
1779
1780 if (UNEXPECTED(opline->opcode == ZEND_OP_DATA)) {
1781 num = n - 1;
1782 } else {
1783 num = n;
1784 }
1785
1786 /* for each output operand opd of op do */
1787 /* setFrom(opd, op) */
1788 /* live.remove(opd) */
1789 if (op->op1_def >= 0 && zend_bitset_in(candidates, op->op1_def)) {
1790 if (zend_jit_begin_range(intervals, op->op1_def, b->start, num) != SUCCESS) {
1791 goto failure;
1792 }
1793 zend_bitset_excl(live, op->op1_def);
1794 }
1795 if (op->op2_def >= 0 && zend_bitset_in(candidates, op->op2_def)) {
1796 if (zend_jit_begin_range(intervals, op->op2_def, b->start, num) != SUCCESS) {
1797 goto failure;
1798 }
1799 zend_bitset_excl(live, op->op2_def);
1800 }
1801 if (op->result_def >= 0 && zend_bitset_in(candidates, op->result_def)) {
1802 if (zend_jit_begin_range(intervals, op->result_def, b->start, num) != SUCCESS) {
1803 goto failure;
1804 }
1805 zend_bitset_excl(live, op->result_def);
1806 }
1807
1808 /* for each input operand opd of op do */
1809 /* live.add(opd) */
1810 /* addRange(opd, b.from, op) */
1811 if (op->op1_use >= 0
1812 && zend_bitset_in(candidates, op->op1_use)
1813 && !zend_ssa_is_no_val_use(opline, op, op->op1_use)) {
1814 zend_bitset_incl(live, op->op1_use);
1815 if (zend_jit_add_range(intervals, op->op1_use, b->start, num) != SUCCESS) {
1816 goto failure;
1817 }
1818 }
1819 if (op->op2_use >= 0
1820 && zend_bitset_in(candidates, op->op2_use)
1821 && !zend_ssa_is_no_val_use(opline, op, op->op2_use)) {
1822 zend_bitset_incl(live, op->op2_use);
1823 if (zend_jit_add_range(intervals, op->op2_use, b->start, num) != SUCCESS) {
1824 goto failure;
1825 }
1826 }
1827 if (op->result_use >= 0
1828 && zend_bitset_in(candidates, op->result_use)
1829 && !zend_ssa_is_no_val_use(opline, op, op->result_use)) {
1830 zend_bitset_incl(live, op->result_use);
1831 if (zend_jit_add_range(intervals, op->result_use, b->start, num) != SUCCESS) {
1832 goto failure;
1833 }
1834 }
1835 }
1836
1837 /* live.remove(phi.output) for each phi of b */
1838 for (phi = ssa->blocks[i].phis; phi; phi = phi->next) {
1839 zend_bitset_excl(live, phi->ssa_var);
1840 }
1841
1842 /* b.liveIn = live */
1843 zend_bitset_copy(live_in + set_size * i, live, set_size);
1844 }
1845
1846 for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) {
1847 zend_basic_block *b = ssa->cfg.blocks + i;
1848
1849 /* if b is loop header */
1850 if ((b->flags & ZEND_BB_LOOP_HEADER)) {
1851 live = live_in + set_size * i;
1852
1853 if (!zend_bitset_empty(live, set_size)) {
1854 uint32_t set_size2 = zend_bitset_len(op_array->last);
1855
1856 zend_bitset_clear(loop_body, set_size2);
1857 zend_jit_compute_loop_body(ssa, i, i, loop_body);
1858 while (!zend_bitset_empty(loop_body, set_size2)) {
1859 uint32_t from = zend_bitset_first(loop_body, set_size2);
1860 uint32_t to = from;
1861
1862 do {
1863 zend_bitset_excl(loop_body, to);
1864 to++;
1865 } while (zend_bitset_in(loop_body, to));
1866 to--;
1867
1868 ZEND_BITSET_FOREACH(live, set_size, j) {
1869 if (zend_jit_add_range(intervals, j, from, to) != SUCCESS) {
1870 goto failure;
1871 }
1872 } ZEND_BITSET_FOREACH_END();
1873 }
1874 }
1875 }
1876
1877 }
1878
1879 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
1880 /* Register hinting (a cheap way for register coalescing) */
1881 for (i = 0; i < ssa->vars_count; i++) {
1882 if (intervals[i]) {
1883 int src;
1884
1885 if (ssa->vars[i].definition_phi) {
1886 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
1887
1888 if (phi->pi >= 0) {
1889 src = phi->sources[0];
1890 if (intervals[src]) {
1891 zend_jit_add_hint(intervals, i, src);
1892 }
1893 } else {
1894 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1895 src = phi->sources[k];
1896 if (src >= 0) {
1897 if (ssa->vars[src].definition_phi
1898 && ssa->vars[src].definition_phi->pi >= 0
1899 && phi->block == ssa->vars[src].definition_phi->block) {
1900 /* Skip zero-length interval for Pi variable */
1901 src = ssa->vars[src].definition_phi->sources[0];
1902 }
1903 if (intervals[src]) {
1904 zend_jit_add_hint(intervals, i, src);
1905 }
1906 }
1907 }
1908 }
1909 }
1910 }
1911 }
1912 for (i = 0; i < ssa->vars_count; i++) {
1913 if (intervals[i] && !intervals[i]->hint) {
1914
1915 if (ssa->vars[i].definition >= 0) {
1916 uint32_t line = ssa->vars[i].definition;
1917 const zend_op *opline = op_array->opcodes + line;
1918
1919 switch (opline->opcode) {
1920 case ZEND_QM_ASSIGN:
1921 case ZEND_POST_INC:
1922 case ZEND_POST_DEC:
1923 if (ssa->ops[line].op1_use >= 0 &&
1924 intervals[ssa->ops[line].op1_use] &&
1925 (i == ssa->ops[line].op1_def ||
1926 (i == ssa->ops[line].result_def &&
1927 (ssa->ops[line].op1_def < 0 ||
1928 !intervals[ssa->ops[line].op1_def])))) {
1929 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1930 }
1931 break;
1932 case ZEND_SEND_VAR:
1933 case ZEND_PRE_INC:
1934 case ZEND_PRE_DEC:
1935 if (i == ssa->ops[line].op1_def &&
1936 ssa->ops[line].op1_use >= 0 &&
1937 intervals[ssa->ops[line].op1_use]) {
1938 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1939 }
1940 break;
1941 case ZEND_ASSIGN:
1942 if (ssa->ops[line].op2_use >= 0 &&
1943 intervals[ssa->ops[line].op2_use] &&
1944 (i == ssa->ops[line].op2_def ||
1945 (i == ssa->ops[line].op1_def &&
1946 (ssa->ops[line].op2_def < 0 ||
1947 !intervals[ssa->ops[line].op2_def])) ||
1948 (i == ssa->ops[line].result_def &&
1949 (ssa->ops[line].op2_def < 0 ||
1950 !intervals[ssa->ops[line].op2_def]) &&
1951 (ssa->ops[line].op1_def < 0 ||
1952 !intervals[ssa->ops[line].op1_def])))) {
1953 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
1954 }
1955 break;
1956 case ZEND_SUB:
1957 case ZEND_ADD:
1958 case ZEND_MUL:
1959 case ZEND_BW_OR:
1960 case ZEND_BW_AND:
1961 case ZEND_BW_XOR:
1962 if (i == ssa->ops[line].result_def) {
1963 if (ssa->ops[line].op1_use >= 0 &&
1964 intervals[ssa->ops[line].op1_use] &&
1965 ssa->ops[line].op1_use_chain < 0 &&
1966 !ssa->vars[ssa->ops[line].op1_use].phi_use_chain &&
1967 (ssa->var_info[i].type & MAY_BE_ANY) ==
1968 (ssa->var_info[ssa->ops[line].op1_use].type & MAY_BE_ANY)) {
1969 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1970 } else if (opline->opcode != ZEND_SUB &&
1971 ssa->ops[line].op2_use >= 0 &&
1972 intervals[ssa->ops[line].op2_use] &&
1973 ssa->ops[line].op2_use_chain < 0 &&
1974 !ssa->vars[ssa->ops[line].op2_use].phi_use_chain &&
1975 (ssa->var_info[i].type & MAY_BE_ANY) ==
1976 (ssa->var_info[ssa->ops[line].op2_use].type & MAY_BE_ANY)) {
1977 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
1978 }
1979 }
1980 break;
1981 }
1982 }
1983 }
1984 }
1985 }
1986
1987 *list = zend_jit_sort_intervals(intervals, ssa->vars_count);
1988
1989 if (*list) {
1990 zend_lifetime_interval *ival = *list;
1991 while (ival) {
1992 if (ival->hint) {
1993 ival->hint->used_as_hint = ival;
1994 }
1995 ival = ival->list_next;
1996 }
1997 }
1998
1999 free_alloca(intervals, use_heap);
2000 return SUCCESS;
2001
2002 failure:
2003 *list = NULL;
2004 free_alloca(intervals, use_heap);
2005 return FAILURE;
2006 }
2007
zend_interval_end(zend_lifetime_interval * ival)2008 static uint32_t zend_interval_end(zend_lifetime_interval *ival)
2009 {
2010 zend_life_range *range = &ival->range;
2011
2012 while (range->next) {
2013 range = range->next;
2014 }
2015 return range->end;
2016 }
2017
zend_interval_covers(zend_lifetime_interval * ival,uint32_t position)2018 static bool zend_interval_covers(zend_lifetime_interval *ival, uint32_t position)
2019 {
2020 zend_life_range *range = &ival->range;
2021
2022 do {
2023 if (position >= range->start && position <= range->end) {
2024 return 1;
2025 }
2026 range = range->next;
2027 } while (range);
2028
2029 return 0;
2030 }
2031
zend_interval_intersection(zend_lifetime_interval * ival1,zend_lifetime_interval * ival2)2032 static uint32_t zend_interval_intersection(zend_lifetime_interval *ival1, zend_lifetime_interval *ival2)
2033 {
2034 zend_life_range *r1 = &ival1->range;
2035 zend_life_range *r2 = &ival2->range;
2036
2037 do {
2038 if (r1->start <= r2->end) {
2039 if (r2->start <= r1->end) {
2040 return MAX(r1->start, r2->start);
2041 } else {
2042 r2 = r2->next;
2043 }
2044 } else {
2045 r1 = r1->next;
2046 }
2047 } while (r1 && r2);
2048
2049 return 0xffffffff;
2050 }
2051
2052 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2053 Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling */
zend_jit_try_allocate_free_reg(const zend_op_array * op_array,const zend_op ** ssa_opcodes,zend_ssa * ssa,zend_lifetime_interval * current,zend_regset available,zend_regset * hints,zend_lifetime_interval * active,zend_lifetime_interval * inactive,zend_lifetime_interval ** list,zend_lifetime_interval ** free)2054 static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_regset *hints, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free)
2055 {
2056 zend_lifetime_interval *it;
2057 uint32_t freeUntilPos[ZREG_NUM];
2058 uint32_t pos, pos2;
2059 zend_reg i, reg, reg2;
2060 zend_reg hint = ZREG_NONE;
2061 zend_regset low_priority_regs;
2062 zend_life_range *range;
2063
2064 if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) {
2065 available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_FP);
2066 } else {
2067 available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_GP);
2068 }
2069
2070 /* TODO: Allow usage of preserved registers ???
2071 * Their values have to be stored in prologue and restored in epilogue
2072 */
2073 available = ZEND_REGSET_DIFFERENCE(available, ZEND_REGSET_PRESERVED);
2074
2075 /* Set freeUntilPos of all physical registers to maxInt */
2076 for (i = 0; i < ZREG_NUM; i++) {
2077 freeUntilPos[i] = 0xffffffff;
2078 }
2079
2080 /* for each interval it in active do */
2081 /* freeUntilPos[it.reg] = 0 */
2082 it = active;
2083 if (ssa->vars[current->ssa_var].definition == current->range.start) {
2084 while (it) {
2085 if (current->range.start != zend_interval_end(it)) {
2086 freeUntilPos[it->reg] = 0;
2087 } else if (zend_jit_may_reuse_reg(
2088 ssa_opcodes ? ssa_opcodes[current->range.start] : op_array->opcodes + current->range.start,
2089 ssa->ops + current->range.start, ssa, current->ssa_var, it->ssa_var)) {
2090 if (!ZEND_REGSET_IN(*hints, it->reg) &&
2091 /* TODO: Avoid most often scratch registers. Find a better way ??? */
2092 (!current->used_as_hint ||
2093 !ZEND_REGSET_IN(ZEND_REGSET_LOW_PRIORITY, it->reg))) {
2094 hint = it->reg;
2095 }
2096 } else {
2097 freeUntilPos[it->reg] = 0;
2098 }
2099 it = it->list_next;
2100 }
2101 } else {
2102 while (it) {
2103 freeUntilPos[it->reg] = 0;
2104 it = it->list_next;
2105 }
2106 }
2107 if (current->hint) {
2108 hint = current->hint->reg;
2109 if (hint != ZREG_NONE && current->hint->used_as_hint == current) {
2110 ZEND_REGSET_EXCL(*hints, hint);
2111 }
2112 }
2113
2114 if (hint == ZREG_NONE && ZEND_REGSET_IS_EMPTY(available)) {
2115 return 0;
2116 }
2117
2118 /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
2119 Michael Franz, CGO'10 (2010), Figure 6. */
2120 if (current->flags & ZREG_SPLIT) {
2121 /* for each interval it in inactive intersecting with current do */
2122 /* freeUntilPos[it.reg] = next intersection of it with current */
2123 it = inactive;
2124 while (it) {
2125 uint32_t next = zend_interval_intersection(current, it);
2126
2127 //ZEND_ASSERT(next != 0xffffffff && !current->split);
2128 if (next < freeUntilPos[it->reg]) {
2129 freeUntilPos[it->reg] = next;
2130 }
2131 it = it->list_next;
2132 }
2133 }
2134
2135 /* Handle Scratch Registers */
2136 /* TODO: Optimize ??? */
2137 range = ¤t->range;
2138 do {
2139 uint32_t line = range->start;
2140 uint32_t last_use_line = (uint32_t)-1;
2141 zend_regset regset;
2142 zend_reg reg;
2143
2144 if ((current->flags & ZREG_LAST_USE) && !range->next) {
2145 last_use_line = range->end;
2146 }
2147 if (ssa->ops[line].op1_def == current->ssa_var ||
2148 ssa->ops[line].op2_def == current->ssa_var ||
2149 ssa->ops[line].result_def == current->ssa_var) {
2150 regset = zend_jit_get_def_scratch_regset(
2151 ssa_opcodes ? ssa_opcodes[line] : op_array->opcodes + line,
2152 ssa->ops + line,
2153 op_array, ssa, current->ssa_var, line == last_use_line);
2154 ZEND_REGSET_FOREACH(regset, reg) {
2155 if (line < freeUntilPos[reg]) {
2156 freeUntilPos[reg] = line;
2157 }
2158 } ZEND_REGSET_FOREACH_END();
2159 line++;
2160 }
2161 while (line <= range->end) {
2162 regset = zend_jit_get_scratch_regset(
2163 ssa_opcodes ? ssa_opcodes[line] : op_array->opcodes + line,
2164 ssa->ops + line,
2165 op_array, ssa, current->ssa_var, line == last_use_line);
2166 ZEND_REGSET_FOREACH(regset, reg) {
2167 if (line < freeUntilPos[reg]) {
2168 freeUntilPos[reg] = line;
2169 }
2170 } ZEND_REGSET_FOREACH_END();
2171 line++;
2172 }
2173 range = range->next;
2174 } while (range);
2175
2176 #if 0
2177 /* Coalescing */
2178 if (ssa->vars[current->ssa_var].definition == current->start) {
2179 zend_op *opline = op_array->opcodes + current->start;
2180 int hint = -1;
2181
2182 switch (opline->opcode) {
2183 case ZEND_ASSIGN:
2184 hint = ssa->ops[current->start].op2_use;
2185 case ZEND_QM_ASSIGN:
2186 hint = ssa->ops[current->start].op1_use;
2187 break;
2188 case ZEND_ADD:
2189 case ZEND_SUB:
2190 case ZEND_MUL:
2191 hint = ssa->ops[current->start].op1_use;
2192 break;
2193 case ZEND_ASSIGN_OP:
2194 if (opline->extended_value == ZEND_ADD
2195 || opline->extended_value == ZEND_SUB
2196 || opline->extended_value == ZEND_MUL) {
2197 hint = ssa->ops[current->start].op1_use;
2198 }
2199 break;
2200 }
2201 if (hint >= 0) {
2202 }
2203 }
2204 #endif
2205
2206 if (hint != ZREG_NONE && freeUntilPos[hint] > zend_interval_end(current)) {
2207 current->reg = hint;
2208 if (current->used_as_hint) {
2209 ZEND_REGSET_INCL(*hints, hint);
2210 }
2211 return 1;
2212 }
2213
2214 if (ZEND_REGSET_IS_EMPTY(available)) {
2215 return 0;
2216 }
2217
2218 pos = 0; reg = ZREG_NONE;
2219 pos2 = 0; reg2 = ZREG_NONE;
2220 low_priority_regs = *hints;
2221 if (current->used_as_hint) {
2222 /* TODO: Avoid most often scratch registers. Find a better way ??? */
2223 low_priority_regs = ZEND_REGSET_UNION(low_priority_regs, ZEND_REGSET_LOW_PRIORITY);
2224 }
2225
2226 ZEND_REGSET_FOREACH(available, i) {
2227 if (ZEND_REGSET_IN(low_priority_regs, i)) {
2228 if (freeUntilPos[i] > pos2) {
2229 reg2 = i;
2230 pos2 = freeUntilPos[i];
2231 }
2232 } else if (freeUntilPos[i] > pos) {
2233 reg = i;
2234 pos = freeUntilPos[i];
2235 }
2236 } ZEND_REGSET_FOREACH_END();
2237
2238 if (reg == ZREG_NONE) {
2239 if (reg2 != ZREG_NONE) {
2240 reg = reg2;
2241 pos = pos2;
2242 reg2 = ZREG_NONE;
2243 }
2244 }
2245
2246 if (reg == ZREG_NONE) {
2247 /* no register available without spilling */
2248 return 0;
2249 } else if (zend_interval_end(current) < pos) {
2250 /* register available for the whole interval */
2251 current->reg = reg;
2252 if (current->used_as_hint) {
2253 ZEND_REGSET_INCL(*hints, reg);
2254 }
2255 return 1;
2256 #if 0
2257 // TODO: allow low priority register usage
2258 } else if (reg2 != ZREG_NONE && zend_interval_end(current) < pos2) {
2259 /* register available for the whole interval */
2260 current->reg = reg2;
2261 if (current->used_as_hint) {
2262 ZEND_REGSET_INCL(*hints, reg2);
2263 }
2264 return 1;
2265 #endif
2266 } else {
2267 /* TODO: enable interval splitting ??? */
2268 /* register available for the first part of the interval */
2269 if (1 || zend_jit_split_interval(current, pos, list, free) != SUCCESS) {
2270 return 0;
2271 }
2272 current->reg = reg;
2273 if (current->used_as_hint) {
2274 ZEND_REGSET_INCL(*hints, reg);
2275 }
2276 return 1;
2277 }
2278 }
2279
2280 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2281 Christian Wimmer VEE'05 (2005), Figure 5. Allocation with spilling.
2282 and "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
2283 Michael Franz, CGO'10 (2010), Figure 6. */
zend_jit_allocate_blocked_reg(void)2284 static int zend_jit_allocate_blocked_reg(void)
2285 {
2286 /* TODO: ??? */
2287 return 0;
2288 }
2289
2290 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2291 Christian Wimmer VEE'10 (2005), Figure 2. */
zend_jit_linear_scan(const zend_op_array * op_array,const zend_op ** ssa_opcodes,zend_ssa * ssa,zend_lifetime_interval * list)2292 static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *list)
2293 {
2294 zend_lifetime_interval *unhandled, *active, *inactive, *handled, *free;
2295 zend_lifetime_interval *current, **p, *q;
2296 uint32_t position;
2297 zend_regset available = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP);
2298 zend_regset hints = ZEND_REGSET_EMPTY;
2299
2300 unhandled = list;
2301 /* active = inactive = handled = free = {} */
2302 active = inactive = handled = free = NULL;
2303 while (unhandled != NULL) {
2304 current = unhandled;
2305 unhandled = unhandled->list_next;
2306 position = current->range.start;
2307
2308 p = &active;
2309 while (*p) {
2310 uint32_t end = zend_interval_end(*p);
2311
2312 q = *p;
2313 if (end < position) {
2314 /* move ival from active to handled */
2315 ZEND_REGSET_INCL(available, q->reg);
2316 *p = q->list_next;
2317 q->list_next = handled;
2318 handled = q;
2319 } else if (!zend_interval_covers(q, position)) {
2320 /* move ival from active to inactive */
2321 ZEND_REGSET_INCL(available, q->reg);
2322 *p = q->list_next;
2323 q->list_next = inactive;
2324 inactive = q;
2325 } else {
2326 p = &q->list_next;
2327 }
2328 }
2329
2330 p = &inactive;
2331 while (*p) {
2332 uint32_t end = zend_interval_end(*p);
2333
2334 q = *p;
2335 if (end < position) {
2336 /* move ival from inactive to handled */
2337 *p = q->list_next;
2338 q->list_next = handled;
2339 handled = q;
2340 } else if (zend_interval_covers(q, position)) {
2341 /* move ival from inactive to active */
2342 ZEND_REGSET_EXCL(available, q->reg);
2343 *p = q->list_next;
2344 q->list_next = active;
2345 active = q;
2346 } else {
2347 p = &q->list_next;
2348 }
2349 }
2350
2351 if (zend_jit_try_allocate_free_reg(op_array, ssa_opcodes, ssa, current, available, &hints, active, inactive, &unhandled, &free) ||
2352 zend_jit_allocate_blocked_reg()) {
2353 ZEND_REGSET_EXCL(available, current->reg);
2354 current->list_next = active;
2355 active = current;
2356 } else {
2357 current->list_next = free;
2358 free = current;
2359 }
2360 }
2361
2362 /* move active to handled */
2363 while (active) {
2364 current = active;
2365 active = active->list_next;
2366 current->list_next = handled;
2367 handled = current;
2368 }
2369
2370 /* move inactive to handled */
2371 while (inactive) {
2372 current = inactive;
2373 inactive = inactive->list_next;
2374 current->list_next = handled;
2375 handled = current;
2376 }
2377
2378 return handled;
2379 }
2380
zend_jit_dump_lifetime_interval(const zend_op_array * op_array,const zend_ssa * ssa,const zend_lifetime_interval * ival)2381 static void zend_jit_dump_lifetime_interval(const zend_op_array *op_array, const zend_ssa *ssa, const zend_lifetime_interval *ival)
2382 {
2383 zend_life_range *range;
2384 int var_num = ssa->vars[ival->ssa_var].var;
2385
2386 fprintf(stderr, "#%d.", ival->ssa_var);
2387 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
2388 fprintf(stderr, ": %u-%u", ival->range.start, ival->range.end);
2389 range = ival->range.next;
2390 while (range) {
2391 fprintf(stderr, ", %u-%u", range->start, range->end);
2392 range = range->next;
2393 }
2394 if (ival->reg != ZREG_NONE) {
2395 fprintf(stderr, " (%s)", zend_reg_name[ival->reg]);
2396 }
2397 if (ival->flags & ZREG_LAST_USE) {
2398 fprintf(stderr, " last_use");
2399 }
2400 if (ival->flags & ZREG_LOAD) {
2401 fprintf(stderr, " load");
2402 }
2403 if (ival->flags & ZREG_STORE) {
2404 fprintf(stderr, " store");
2405 }
2406 if (ival->hint) {
2407 fprintf(stderr, " hint");
2408 if (ival->hint->ssa_var >= 0) {
2409 var_num = ssa->vars[ival->hint->ssa_var].var;
2410 fprintf(stderr, "=#%d.", ival->hint->ssa_var);
2411 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
2412 }
2413 if (ival->hint->reg != ZREG_NONE) {
2414 fprintf(stderr, " (%s)", zend_reg_name[ival->hint->reg]);
2415 }
2416 }
2417 fprintf(stderr, "\n");
2418 }
2419
zend_jit_allocate_registers(const zend_op_array * op_array,zend_ssa * ssa)2420 static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array *op_array, zend_ssa *ssa)
2421 {
2422 void *checkpoint;
2423 int set_size, candidates_count, i;
2424 zend_bitset candidates = NULL;
2425 zend_lifetime_interval *list, *ival;
2426 zend_lifetime_interval **intervals;
2427 ALLOCA_FLAG(use_heap);
2428
2429 if (!ssa->var_info) {
2430 return NULL;
2431 }
2432
2433 /* Identify SSA variables suitable for register allocation */
2434 set_size = zend_bitset_len(ssa->vars_count);
2435 candidates = ZEND_BITSET_ALLOCA(set_size, use_heap);
2436 if (!candidates) {
2437 return NULL;
2438 }
2439 candidates_count = 0;
2440 zend_bitset_clear(candidates, set_size);
2441 for (i = 0; i < ssa->vars_count; i++) {
2442 if (zend_jit_may_be_in_reg(op_array, ssa, i)) {
2443 zend_bitset_incl(candidates, i);
2444 candidates_count++;
2445 }
2446 }
2447 if (!candidates_count) {
2448 free_alloca(candidates, use_heap);
2449 return NULL;
2450 }
2451
2452 checkpoint = zend_arena_checkpoint(CG(arena));
2453
2454 /* Find life-time intervals */
2455 if (zend_jit_compute_liveness(op_array, ssa, candidates, &list) != SUCCESS) {
2456 goto failure;
2457 }
2458
2459 if (list) {
2460 /* Set ZREG_LAST_USE flags */
2461 ival = list;
2462 while (ival) {
2463 zend_life_range *range = &ival->range;
2464
2465 while (range->next) {
2466 range = range->next;
2467 }
2468 if (zend_ssa_is_last_use(op_array, ssa, ival->ssa_var, range->end)) {
2469 ival->flags |= ZREG_LAST_USE;
2470 }
2471 ival = ival->list_next;
2472 }
2473 }
2474
2475 if (list) {
2476 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
2477 fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
2478 ival = list;
2479 while (ival) {
2480 zend_jit_dump_lifetime_interval(op_array, ssa, ival);
2481 ival = ival->list_next;
2482 }
2483 fprintf(stderr, "\n");
2484 }
2485
2486 /* Linear Scan Register Allocation */
2487 list = zend_jit_linear_scan(op_array, NULL, ssa, list);
2488
2489 if (list) {
2490 intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*));
2491 if (!intervals) {
2492 goto failure;
2493 }
2494
2495 ival = list;
2496 while (ival != NULL) {
2497 zend_lifetime_interval *next = ival->list_next;
2498
2499 ival->list_next = intervals[ival->ssa_var];
2500 intervals[ival->ssa_var] = ival;
2501 ival = next;
2502 }
2503
2504 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
2505 /* Naive SSA resolution */
2506 for (i = 0; i < ssa->vars_count; i++) {
2507 if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
2508 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
2509 int k, src;
2510
2511 if (phi->pi >= 0) {
2512 if (!ssa->vars[i].phi_use_chain
2513 || ssa->vars[i].phi_use_chain->block != phi->block) {
2514 src = phi->sources[0];
2515 if (intervals[i]) {
2516 if (!intervals[src]) {
2517 intervals[i]->flags |= ZREG_LOAD;
2518 } else if (intervals[i]->reg != intervals[src]->reg) {
2519 intervals[i]->flags |= ZREG_LOAD;
2520 intervals[src]->flags |= ZREG_STORE;
2521 }
2522 } else if (intervals[src]) {
2523 intervals[src]->flags |= ZREG_STORE;
2524 }
2525 }
2526 } else {
2527 int need_move = 0;
2528
2529 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
2530 src = phi->sources[k];
2531 if (src >= 0) {
2532 if (ssa->vars[src].definition_phi
2533 && ssa->vars[src].definition_phi->pi >= 0
2534 && phi->block == ssa->vars[src].definition_phi->block) {
2535 /* Skip zero-length interval for Pi variable */
2536 src = ssa->vars[src].definition_phi->sources[0];
2537 }
2538 if (intervals[i]) {
2539 if (!intervals[src]) {
2540 need_move = 1;
2541 } else if (intervals[i]->reg != intervals[src]->reg) {
2542 need_move = 1;
2543 }
2544 } else if (intervals[src]) {
2545 need_move = 1;
2546 }
2547 }
2548 }
2549 if (need_move) {
2550 if (intervals[i]) {
2551 intervals[i]->flags |= ZREG_LOAD;
2552 }
2553 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
2554 src = phi->sources[k];
2555 if (src >= 0) {
2556 if (ssa->vars[src].definition_phi
2557 && ssa->vars[src].definition_phi->pi >= 0
2558 && phi->block == ssa->vars[src].definition_phi->block) {
2559 /* Skip zero-length interval for Pi variable */
2560 src = ssa->vars[src].definition_phi->sources[0];
2561 }
2562 if (intervals[src]) {
2563 intervals[src]->flags |= ZREG_STORE;
2564 }
2565 }
2566 }
2567 }
2568 }
2569 }
2570 }
2571 /* Remove useless register allocation */
2572 for (i = 0; i < ssa->vars_count; i++) {
2573 if (intervals[i] &&
2574 ((intervals[i]->flags & ZREG_LOAD) ||
2575 ((intervals[i]->flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) &&
2576 ssa->vars[i].use_chain < 0) {
2577 bool may_remove = 1;
2578 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
2579
2580 while (phi) {
2581 if (intervals[phi->ssa_var] &&
2582 !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) {
2583 may_remove = 0;
2584 break;
2585 }
2586 phi = zend_ssa_next_use_phi(ssa, i, phi);
2587 }
2588 if (may_remove) {
2589 intervals[i] = NULL;
2590 }
2591 }
2592 }
2593 /* Remove intervals used once */
2594 for (i = 0; i < ssa->vars_count; i++) {
2595 if (intervals[i] &&
2596 (intervals[i]->flags & ZREG_LOAD) &&
2597 (intervals[i]->flags & ZREG_STORE) &&
2598 (ssa->vars[i].use_chain < 0 ||
2599 zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) {
2600 bool may_remove = 1;
2601 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
2602
2603 while (phi) {
2604 if (intervals[phi->ssa_var] &&
2605 !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) {
2606 may_remove = 0;
2607 break;
2608 }
2609 phi = zend_ssa_next_use_phi(ssa, i, phi);
2610 }
2611 if (may_remove) {
2612 intervals[i] = NULL;
2613 }
2614 }
2615 }
2616 }
2617
2618 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
2619 fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
2620 for (i = 0; i < ssa->vars_count; i++) {
2621 ival = intervals[i];
2622 while (ival) {
2623 zend_jit_dump_lifetime_interval(op_array, ssa, ival);
2624 ival = ival->list_next;
2625 }
2626 }
2627 fprintf(stderr, "\n");
2628 }
2629
2630 free_alloca(candidates, use_heap);
2631 return intervals;
2632 }
2633 }
2634
2635 failure:
2636 zend_arena_release(&CG(arena), checkpoint);
2637 free_alloca(candidates, use_heap);
2638 return NULL;
2639 }
2640
zend_jit_next_is_send_result(const zend_op * opline)2641 static bool zend_jit_next_is_send_result(const zend_op *opline)
2642 {
2643 if (opline->result_type == IS_TMP_VAR
2644 && (opline+1)->opcode == ZEND_SEND_VAL
2645 && (opline+1)->op1_type == IS_TMP_VAR
2646 && (opline+1)->op2_type != IS_CONST
2647 && (opline+1)->op1.var == opline->result.var) {
2648 return 1;
2649 }
2650 return 0;
2651 }
2652
zend_jit_supported_binary_op(zend_uchar op,uint32_t op1_info,uint32_t op2_info)2653 static bool zend_jit_supported_binary_op(zend_uchar op, uint32_t op1_info, uint32_t op2_info)
2654 {
2655 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2656 return false;
2657 }
2658 switch (op) {
2659 case ZEND_POW:
2660 case ZEND_DIV:
2661 // TODO: check for division by zero ???
2662 return false;
2663 case ZEND_ADD:
2664 case ZEND_SUB:
2665 case ZEND_MUL:
2666 return (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
2667 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
2668 case ZEND_BW_OR:
2669 case ZEND_BW_AND:
2670 case ZEND_BW_XOR:
2671 case ZEND_SL:
2672 case ZEND_SR:
2673 case ZEND_MOD:
2674 return (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG);
2675 case ZEND_CONCAT:
2676 return (op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING);
2677 EMPTY_SWITCH_DEFAULT_CASE()
2678 }
2679 }
2680
zend_jit(const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline)2681 static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline)
2682 {
2683 int b, i, end;
2684 zend_op *opline;
2685 dasm_State* dasm_state = NULL;
2686 void *handler;
2687 int call_level = 0;
2688 void *checkpoint = NULL;
2689 zend_lifetime_interval **ra = NULL;
2690 bool is_terminated = 1; /* previous basic block is terminated by jump */
2691 bool recv_emitted = 0; /* emitted at least one RECV opcode */
2692 zend_uchar smart_branch_opcode;
2693 uint32_t target_label, target_label2;
2694 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info;
2695 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
2696 zend_class_entry *ce;
2697 bool ce_is_instanceof;
2698 bool on_this;
2699
2700 if (JIT_G(bisect_limit)) {
2701 jit_bisect_pos++;
2702 if (jit_bisect_pos >= JIT_G(bisect_limit)) {
2703 if (jit_bisect_pos == JIT_G(bisect_limit)) {
2704 fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n",
2705 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
2706 op_array->scope ? "::" : "",
2707 op_array->function_name ? ZSTR_VAL(op_array->function_name) : "{main}",
2708 ZSTR_VAL(op_array->filename), op_array->line_start);
2709 }
2710 return FAILURE;
2711 }
2712 }
2713
2714 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2715 checkpoint = zend_arena_checkpoint(CG(arena));
2716 ra = zend_jit_allocate_registers(op_array, ssa);
2717 }
2718
2719 /* mark hidden branch targets */
2720 for (b = 0; b < ssa->cfg.blocks_count; b++) {
2721 if (ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE &&
2722 ssa->cfg.blocks[b].len > 1) {
2723
2724 opline = op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
2725 if (opline->opcode == ZEND_DO_FCALL &&
2726 (opline-1)->opcode == ZEND_NEW) {
2727 ssa->cfg.blocks[ssa->cfg.blocks[b].successors[0]].flags |= ZEND_BB_TARGET;
2728 }
2729 }
2730 }
2731
2732 dasm_init(&dasm_state, DASM_MAXSECTION);
2733 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
2734 dasm_setup(&dasm_state, dasm_actions);
2735
2736 dasm_growpc(&dasm_state, ssa->cfg.blocks_count * 2 + 1);
2737
2738 zend_jit_align_func(&dasm_state);
2739 for (b = 0; b < ssa->cfg.blocks_count; b++) {
2740 if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) {
2741 continue;
2742 }
2743 //#ifndef CONTEXT_THREADED_JIT
2744 if (ssa->cfg.blocks[b].flags & ZEND_BB_ENTRY) {
2745 if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
2746 /* pass */
2747 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2748 ssa->cfg.blocks[b].len == 1 &&
2749 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT) &&
2750 op_array->opcodes[ssa->cfg.blocks[b].start].opcode != ZEND_JMP) {
2751 /* don't generate code for BB with single opcode */
2752 continue;
2753 }
2754 if (ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW) {
2755 if (!is_terminated) {
2756 zend_jit_jmp(&dasm_state, b);
2757 }
2758 }
2759 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2760 zend_jit_prologue(&dasm_state);
2761 } else
2762 //#endif
2763 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
2764 opline = op_array->opcodes + ssa->cfg.blocks[b].start;
2765 if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) {
2766 if (opline->opcode == ZEND_RECV_INIT) {
2767 if (opline == op_array->opcodes ||
2768 (opline-1)->opcode != ZEND_RECV_INIT) {
2769 if (recv_emitted) {
2770 zend_jit_jmp(&dasm_state, b);
2771 }
2772 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2773 for (i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) {
2774 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b + i);
2775 }
2776 zend_jit_prologue(&dasm_state);
2777 }
2778 recv_emitted = 1;
2779 } else if (opline->opcode == ZEND_RECV) {
2780 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2781 /* skip */
2782 continue;
2783 } else if (recv_emitted) {
2784 zend_jit_jmp(&dasm_state, b);
2785 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2786 zend_jit_prologue(&dasm_state);
2787 } else {
2788 zend_arg_info *arg_info;
2789
2790 if (opline->op1.num <= op_array->num_args) {
2791 arg_info = &op_array->arg_info[opline->op1.num - 1];
2792 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
2793 arg_info = &op_array->arg_info[op_array->num_args];
2794 } else {
2795 /* skip */
2796 continue;
2797 }
2798 if (!ZEND_TYPE_IS_SET(arg_info->type)) {
2799 /* skip */
2800 continue;
2801 }
2802 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2803 zend_jit_prologue(&dasm_state);
2804 recv_emitted = 1;
2805 }
2806 } else {
2807 if (recv_emitted) {
2808 zend_jit_jmp(&dasm_state, b);
2809 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2810 ssa->cfg.blocks[b].len == 1 &&
2811 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
2812 /* don't generate code for BB with single opcode */
2813 dasm_free(&dasm_state);
2814
2815 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2816 zend_arena_release(&CG(arena), checkpoint);
2817 }
2818 return SUCCESS;
2819 }
2820 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2821 zend_jit_prologue(&dasm_state);
2822 recv_emitted = 1;
2823 }
2824 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2825 ssa->cfg.blocks[b].len == 1 &&
2826 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
2827 /* don't generate code for BB with single opcode */
2828 dasm_free(&dasm_state);
2829
2830 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2831 zend_arena_release(&CG(arena), checkpoint);
2832 }
2833 return SUCCESS;
2834 } else {
2835 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2836 zend_jit_prologue(&dasm_state);
2837 }
2838 }
2839
2840 is_terminated = 0;
2841
2842 zend_jit_label(&dasm_state, b);
2843 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
2844 if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
2845 && ssa->cfg.blocks[b].start != 0
2846 && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
2847 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_LONG
2848 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_STRING
2849 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_MATCH)) {
2850 zend_jit_reset_last_valid_opline();
2851 if (!zend_jit_set_ip(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) {
2852 goto jit_failure;
2853 }
2854 } else {
2855 zend_jit_set_last_valid_opline(op_array->opcodes + ssa->cfg.blocks[b].start);
2856 }
2857 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
2858 zend_jit_reset_last_valid_opline();
2859 } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY|ZEND_BB_ENTRY)) {
2860 zend_jit_set_last_valid_opline(op_array->opcodes + ssa->cfg.blocks[b].start);
2861 }
2862 if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) {
2863 if (!zend_jit_check_timeout(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start, NULL)) {
2864 goto jit_failure;
2865 }
2866 }
2867 if (!ssa->cfg.blocks[b].len) {
2868 continue;
2869 }
2870 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
2871 zend_ssa_phi *phi = ssa->blocks[b].phis;
2872
2873 while (phi) {
2874 zend_lifetime_interval *ival = ra[phi->ssa_var];
2875
2876 if (ival) {
2877 if (ival->flags & ZREG_LOAD) {
2878 ZEND_ASSERT(ival->reg != ZREG_NONE);
2879
2880 if (!zend_jit_load_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg)) {
2881 goto jit_failure;
2882 }
2883 } else if (ival->flags & ZREG_STORE) {
2884 ZEND_ASSERT(ival->reg != ZREG_NONE);
2885
2886 if (!zend_jit_store_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg, 1)) {
2887 goto jit_failure;
2888 }
2889 }
2890 }
2891 phi = phi->next;
2892 }
2893 }
2894 end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
2895 for (i = ssa->cfg.blocks[b].start; i <= end; i++) {
2896 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[i] : NULL;
2897 opline = op_array->opcodes + i;
2898 switch (opline->opcode) {
2899 case ZEND_INIT_FCALL:
2900 case ZEND_INIT_FCALL_BY_NAME:
2901 case ZEND_INIT_NS_FCALL_BY_NAME:
2902 case ZEND_INIT_METHOD_CALL:
2903 case ZEND_INIT_DYNAMIC_CALL:
2904 case ZEND_INIT_STATIC_METHOD_CALL:
2905 case ZEND_INIT_USER_CALL:
2906 case ZEND_NEW:
2907 call_level++;
2908 }
2909
2910 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
2911 switch (opline->opcode) {
2912 case ZEND_PRE_INC:
2913 case ZEND_PRE_DEC:
2914 case ZEND_POST_INC:
2915 case ZEND_POST_DEC:
2916 if (opline->op1_type != IS_CV) {
2917 break;
2918 }
2919 op1_info = OP1_INFO();
2920 if (!(op1_info & MAY_BE_LONG)) {
2921 break;
2922 }
2923 if (opline->result_type != IS_UNUSED) {
2924 res_use_info = -1;
2925
2926 if (opline->result_type == IS_CV
2927 && ssa->vars
2928 && ssa_op->result_use >= 0
2929 && !ssa->vars[ssa_op->result_use].no_val) {
2930 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
2931
2932 if (Z_MODE(res_use_addr) != IS_REG
2933 || Z_LOAD(res_use_addr)
2934 || Z_STORE(res_use_addr)) {
2935 res_use_info = RES_USE_INFO();
2936 }
2937 }
2938 res_info = RES_INFO();
2939 res_addr = RES_REG_ADDR();
2940 } else {
2941 res_use_info = -1;
2942 res_info = -1;
2943 res_addr = 0;
2944 }
2945 op1_def_info = OP1_DEF_INFO();
2946 if (!zend_jit_inc_dec(&dasm_state, opline,
2947 op1_info, OP1_REG_ADDR(),
2948 op1_def_info, OP1_DEF_REG_ADDR(),
2949 res_use_info, res_info,
2950 res_addr,
2951 (op1_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
2952 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2953 goto jit_failure;
2954 }
2955 goto done;
2956 case ZEND_BW_OR:
2957 case ZEND_BW_AND:
2958 case ZEND_BW_XOR:
2959 case ZEND_SL:
2960 case ZEND_SR:
2961 case ZEND_MOD:
2962 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2963 break;
2964 }
2965 op1_info = OP1_INFO();
2966 op2_info = OP2_INFO();
2967 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2968 break;
2969 }
2970 if (!(op1_info & MAY_BE_LONG)
2971 || !(op2_info & MAY_BE_LONG)) {
2972 break;
2973 }
2974 res_addr = RES_REG_ADDR();
2975 if (Z_MODE(res_addr) != IS_REG
2976 && (i + 1) <= end
2977 && zend_jit_next_is_send_result(opline)) {
2978 i++;
2979 res_use_info = -1;
2980 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
2981 if (!zend_jit_reuse_ip(&dasm_state)) {
2982 goto jit_failure;
2983 }
2984 } else {
2985 res_use_info = -1;
2986
2987 if (opline->result_type == IS_CV
2988 && ssa->vars
2989 && ssa_op->result_use >= 0
2990 && !ssa->vars[ssa_op->result_use].no_val) {
2991 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
2992
2993 if (Z_MODE(res_use_addr) != IS_REG
2994 || Z_LOAD(res_use_addr)
2995 || Z_STORE(res_use_addr)) {
2996 res_use_info = RES_USE_INFO();
2997 }
2998 }
2999 }
3000 if (!zend_jit_long_math(&dasm_state, opline,
3001 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
3002 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
3003 res_use_info, RES_INFO(), res_addr,
3004 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3005 goto jit_failure;
3006 }
3007 goto done;
3008 case ZEND_ADD:
3009 case ZEND_SUB:
3010 case ZEND_MUL:
3011 // case ZEND_DIV: // TODO: check for division by zero ???
3012 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3013 break;
3014 }
3015 op1_info = OP1_INFO();
3016 op2_info = OP2_INFO();
3017 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
3018 break;
3019 }
3020 if (opline->opcode == ZEND_ADD &&
3021 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
3022 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
3023 /* pass */
3024 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
3025 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
3026 break;
3027 }
3028 res_addr = RES_REG_ADDR();
3029 if (Z_MODE(res_addr) != IS_REG
3030 && (i + 1) <= end
3031 && zend_jit_next_is_send_result(opline)) {
3032 i++;
3033 res_use_info = -1;
3034 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3035 if (!zend_jit_reuse_ip(&dasm_state)) {
3036 goto jit_failure;
3037 }
3038 } else {
3039 res_use_info = -1;
3040
3041 if (opline->result_type == IS_CV
3042 && ssa->vars
3043 && ssa_op->result_use >= 0
3044 && !ssa->vars[ssa_op->result_use].no_val) {
3045 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
3046
3047 if (Z_MODE(res_use_addr) != IS_REG
3048 || Z_LOAD(res_use_addr)
3049 || Z_STORE(res_use_addr)) {
3050 res_use_info = RES_USE_INFO();
3051 }
3052 }
3053 }
3054 res_info = RES_INFO();
3055 if (opline->opcode == ZEND_ADD &&
3056 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
3057 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
3058 if (!zend_jit_add_arrays(&dasm_state, opline, op1_info, OP1_REG_ADDR(), op2_info, OP2_REG_ADDR(), res_addr)) {
3059 goto jit_failure;
3060 }
3061 } else {
3062 if (!zend_jit_math(&dasm_state, opline,
3063 op1_info, OP1_REG_ADDR(),
3064 op2_info, OP2_REG_ADDR(),
3065 res_use_info, res_info, res_addr,
3066 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
3067 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3068 goto jit_failure;
3069 }
3070 }
3071 goto done;
3072 case ZEND_CONCAT:
3073 case ZEND_FAST_CONCAT:
3074 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3075 break;
3076 }
3077 op1_info = OP1_INFO();
3078 op2_info = OP2_INFO();
3079 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
3080 break;
3081 }
3082 if (!(op1_info & MAY_BE_STRING) ||
3083 !(op2_info & MAY_BE_STRING)) {
3084 break;
3085 }
3086 res_addr = RES_REG_ADDR();
3087 if ((i + 1) <= end
3088 && zend_jit_next_is_send_result(opline)) {
3089 i++;
3090 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3091 if (!zend_jit_reuse_ip(&dasm_state)) {
3092 goto jit_failure;
3093 }
3094 }
3095 if (!zend_jit_concat(&dasm_state, opline,
3096 op1_info, op2_info, res_addr,
3097 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3098 goto jit_failure;
3099 }
3100 goto done;
3101 case ZEND_ASSIGN_OP:
3102 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
3103 break;
3104 }
3105 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3106 break;
3107 }
3108 op1_info = OP1_INFO();
3109 op2_info = OP2_INFO();
3110 if (!zend_jit_supported_binary_op(
3111 opline->extended_value, op1_info, op2_info)) {
3112 break;
3113 }
3114 op1_def_info = OP1_DEF_INFO();
3115 if (!zend_jit_assign_op(&dasm_state, opline,
3116 op1_info, op1_def_info, OP1_RANGE(),
3117 op2_info, OP2_RANGE(),
3118 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
3119 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3120 goto jit_failure;
3121 }
3122 goto done;
3123 case ZEND_ASSIGN_DIM_OP:
3124 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
3125 break;
3126 }
3127 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3128 break;
3129 }
3130 if (!zend_jit_supported_binary_op(
3131 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3132 break;
3133 }
3134 if (!zend_jit_assign_dim_op(&dasm_state, opline,
3135 OP1_INFO(), OP1_DEF_INFO(), OP1_REG_ADDR(), OP2_INFO(),
3136 OP1_DATA_INFO(), OP1_DATA_RANGE(), IS_UNKNOWN,
3137 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3138 goto jit_failure;
3139 }
3140 goto done;
3141 case ZEND_ASSIGN_DIM:
3142 if (opline->op1_type != IS_CV) {
3143 break;
3144 }
3145 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3146 break;
3147 }
3148 if (!zend_jit_assign_dim(&dasm_state, opline,
3149 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), OP1_DATA_INFO(), IS_UNKNOWN,
3150 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3151 goto jit_failure;
3152 }
3153 goto done;
3154 case ZEND_PRE_INC_OBJ:
3155 case ZEND_PRE_DEC_OBJ:
3156 case ZEND_POST_INC_OBJ:
3157 case ZEND_POST_DEC_OBJ:
3158 if (opline->op2_type != IS_CONST
3159 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3160 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3161 break;
3162 }
3163 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3164 break;
3165 }
3166 ce = NULL;
3167 ce_is_instanceof = 0;
3168 on_this = 0;
3169 if (opline->op1_type == IS_UNUSED) {
3170 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3171 ce = op_array->scope;
3172 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3173 op1_addr = 0;
3174 on_this = 1;
3175 } else {
3176 op1_info = OP1_INFO();
3177 if (!(op1_info & MAY_BE_OBJECT)) {
3178 break;
3179 }
3180 op1_addr = OP1_REG_ADDR();
3181 if (ssa->var_info && ssa->ops) {
3182 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3183 if (ssa_op->op1_use >= 0) {
3184 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3185 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3186 ce = op1_ssa->ce;
3187 ce_is_instanceof = op1_ssa->is_instanceof;
3188 }
3189 }
3190 }
3191 }
3192 if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3193 op1_info, op1_addr,
3194 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
3195 goto jit_failure;
3196 }
3197 goto done;
3198 case ZEND_ASSIGN_OBJ_OP:
3199 if (opline->result_type != IS_UNUSED) {
3200 break;
3201 }
3202 if (opline->op2_type != IS_CONST
3203 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3204 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3205 break;
3206 }
3207 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3208 break;
3209 }
3210 if (!zend_jit_supported_binary_op(
3211 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3212 break;
3213 }
3214 ce = NULL;
3215 ce_is_instanceof = 0;
3216 on_this = 0;
3217 if (opline->op1_type == IS_UNUSED) {
3218 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3219 ce = op_array->scope;
3220 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3221 op1_addr = 0;
3222 on_this = 1;
3223 } else {
3224 op1_info = OP1_INFO();
3225 if (!(op1_info & MAY_BE_OBJECT)) {
3226 break;
3227 }
3228 op1_addr = OP1_REG_ADDR();
3229 if (ssa->var_info && ssa->ops) {
3230 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3231 if (ssa_op->op1_use >= 0) {
3232 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3233 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3234 ce = op1_ssa->ce;
3235 ce_is_instanceof = op1_ssa->is_instanceof;
3236 }
3237 }
3238 }
3239 }
3240 if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op,
3241 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_RANGE(),
3242 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
3243 goto jit_failure;
3244 }
3245 goto done;
3246 case ZEND_ASSIGN_OBJ:
3247 if (opline->op2_type != IS_CONST
3248 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3249 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3250 break;
3251 }
3252 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3253 break;
3254 }
3255 ce = NULL;
3256 ce_is_instanceof = 0;
3257 on_this = 0;
3258 if (opline->op1_type == IS_UNUSED) {
3259 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3260 ce = op_array->scope;
3261 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3262 op1_addr = 0;
3263 on_this = 1;
3264 } else {
3265 op1_info = OP1_INFO();
3266 if (!(op1_info & MAY_BE_OBJECT)) {
3267 break;
3268 }
3269 op1_addr = OP1_REG_ADDR();
3270 if (ssa->var_info && ssa->ops) {
3271 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3272 if (ssa_op->op1_use >= 0) {
3273 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3274 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3275 ce = op1_ssa->ce;
3276 ce_is_instanceof = op1_ssa->is_instanceof;
3277 }
3278 }
3279 }
3280 }
3281 if (!zend_jit_assign_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3282 op1_info, op1_addr, OP1_DATA_INFO(),
3283 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN,
3284 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3285 goto jit_failure;
3286 }
3287 goto done;
3288 case ZEND_ASSIGN:
3289 if (opline->op1_type != IS_CV) {
3290 break;
3291 }
3292 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3293 break;
3294 }
3295 op2_addr = OP2_REG_ADDR();
3296 if (ra
3297 && ssa->ops[opline - op_array->opcodes].op2_def >= 0
3298 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) {
3299 op2_def_addr = OP2_DEF_REG_ADDR();
3300 } else {
3301 op2_def_addr = op2_addr;
3302 }
3303 op1_info = OP1_INFO();
3304 if (ra && ssa->vars[ssa_op->op1_use].no_val) {
3305 op1_info |= MAY_BE_UNDEF; // requres type assignment
3306 }
3307 if (opline->result_type == IS_UNUSED) {
3308 res_addr = 0;
3309 res_info = -1;
3310 } else {
3311 res_addr = RES_REG_ADDR();
3312 res_info = RES_INFO();
3313 if (Z_MODE(res_addr) != IS_REG
3314 && (i + 1) <= end
3315 && zend_jit_next_is_send_result(opline)
3316 && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) {
3317 i++;
3318 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3319 if (!zend_jit_reuse_ip(&dasm_state)) {
3320 goto jit_failure;
3321 }
3322 }
3323 }
3324 if (!zend_jit_assign(&dasm_state, opline,
3325 op1_info, OP1_REG_ADDR(),
3326 OP1_DEF_INFO(), OP1_DEF_REG_ADDR(),
3327 OP2_INFO(), op2_addr, op2_def_addr,
3328 res_info, res_addr,
3329 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3330 goto jit_failure;
3331 }
3332 goto done;
3333 case ZEND_QM_ASSIGN:
3334 op1_addr = OP1_REG_ADDR();
3335 if (ra
3336 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
3337 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
3338 op1_def_addr = OP1_DEF_REG_ADDR();
3339 } else {
3340 op1_def_addr = op1_addr;
3341 }
3342 if (!zend_jit_qm_assign(&dasm_state, opline,
3343 OP1_INFO(), op1_addr, op1_def_addr,
3344 -1, RES_INFO(), RES_REG_ADDR())) {
3345 goto jit_failure;
3346 }
3347 goto done;
3348 case ZEND_INIT_FCALL:
3349 case ZEND_INIT_FCALL_BY_NAME:
3350 case ZEND_INIT_NS_FCALL_BY_NAME:
3351 if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level, NULL, 0)) {
3352 goto jit_failure;
3353 }
3354 goto done;
3355 case ZEND_SEND_VAL:
3356 case ZEND_SEND_VAL_EX:
3357 if (opline->op2_type == IS_CONST) {
3358 /* Named parameters not supported in JIT (yet) */
3359 break;
3360 }
3361 if (opline->opcode == ZEND_SEND_VAL_EX
3362 && opline->op2.num > MAX_ARG_FLAG_NUM) {
3363 break;
3364 }
3365 if (!zend_jit_send_val(&dasm_state, opline,
3366 OP1_INFO(), OP1_REG_ADDR())) {
3367 goto jit_failure;
3368 }
3369 goto done;
3370 case ZEND_SEND_REF:
3371 if (opline->op2_type == IS_CONST) {
3372 /* Named parameters not supported in JIT (yet) */
3373 break;
3374 }
3375 if (!zend_jit_send_ref(&dasm_state, opline, op_array,
3376 OP1_INFO(), 0)) {
3377 goto jit_failure;
3378 }
3379 goto done;
3380 case ZEND_SEND_VAR:
3381 case ZEND_SEND_VAR_EX:
3382 case ZEND_SEND_VAR_NO_REF:
3383 case ZEND_SEND_VAR_NO_REF_EX:
3384 case ZEND_SEND_FUNC_ARG:
3385 if (opline->op2_type == IS_CONST) {
3386 /* Named parameters not supported in JIT (yet) */
3387 break;
3388 }
3389 if ((opline->opcode == ZEND_SEND_VAR_EX
3390 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
3391 && opline->op2.num > MAX_ARG_FLAG_NUM) {
3392 break;
3393 }
3394 op1_addr = OP1_REG_ADDR();
3395 if (ra
3396 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
3397 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
3398 op1_def_addr = OP1_DEF_REG_ADDR();
3399 } else {
3400 op1_def_addr = op1_addr;
3401 }
3402 if (!zend_jit_send_var(&dasm_state, opline, op_array,
3403 OP1_INFO(), op1_addr, op1_def_addr)) {
3404 goto jit_failure;
3405 }
3406 goto done;
3407 case ZEND_CHECK_FUNC_ARG:
3408 if (opline->op2_type == IS_CONST) {
3409 /* Named parameters not supported in JIT (yet) */
3410 break;
3411 }
3412 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
3413 break;
3414 }
3415 if (!zend_jit_check_func_arg(&dasm_state, opline)) {
3416 goto jit_failure;
3417 }
3418 goto done;
3419 case ZEND_CHECK_UNDEF_ARGS:
3420 if (!zend_jit_check_undef_args(&dasm_state, opline)) {
3421 goto jit_failure;
3422 }
3423 goto done;
3424 case ZEND_DO_UCALL:
3425 is_terminated = 1;
3426 ZEND_FALLTHROUGH;
3427 case ZEND_DO_ICALL:
3428 case ZEND_DO_FCALL_BY_NAME:
3429 case ZEND_DO_FCALL:
3430 if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level, b + 1, NULL)) {
3431 goto jit_failure;
3432 }
3433 goto done;
3434 case ZEND_IS_EQUAL:
3435 case ZEND_IS_NOT_EQUAL:
3436 case ZEND_IS_SMALLER:
3437 case ZEND_IS_SMALLER_OR_EQUAL:
3438 case ZEND_CASE: {
3439 res_addr = RES_REG_ADDR();
3440 if ((opline->result_type & IS_TMP_VAR)
3441 && (i + 1) <= end
3442 && ((opline+1)->opcode == ZEND_JMPZ
3443 || (opline+1)->opcode == ZEND_JMPNZ
3444 || (opline+1)->opcode == ZEND_JMPZ_EX
3445 || (opline+1)->opcode == ZEND_JMPNZ_EX)
3446 && (opline+1)->op1_type == IS_TMP_VAR
3447 && (opline+1)->op1.var == opline->result.var) {
3448 i++;
3449 smart_branch_opcode = (opline+1)->opcode;
3450 target_label = ssa->cfg.blocks[b].successors[0];
3451 target_label2 = ssa->cfg.blocks[b].successors[1];
3452 /* For EX variant write into the result of EX opcode. */
3453 if ((opline+1)->opcode == ZEND_JMPZ_EX
3454 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
3455 res_addr = OP_REG_ADDR(opline + 1, result_type, result, result_def);
3456 }
3457 } else {
3458 smart_branch_opcode = 0;
3459 target_label = target_label2 = (uint32_t)-1;
3460 }
3461 if (!zend_jit_cmp(&dasm_state, opline,
3462 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
3463 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
3464 res_addr,
3465 zend_may_throw(opline, ssa_op, op_array, ssa),
3466 smart_branch_opcode, target_label, target_label2,
3467 NULL, 0)) {
3468 goto jit_failure;
3469 }
3470 goto done;
3471 }
3472 case ZEND_IS_IDENTICAL:
3473 case ZEND_IS_NOT_IDENTICAL:
3474 case ZEND_CASE_STRICT:
3475 if ((opline->result_type & IS_TMP_VAR)
3476 && (i + 1) <= end
3477 && ((opline+1)->opcode == ZEND_JMPZ
3478 || (opline+1)->opcode == ZEND_JMPNZ)
3479 && (opline+1)->op1_type == IS_TMP_VAR
3480 && (opline+1)->op1.var == opline->result.var) {
3481 i++;
3482 smart_branch_opcode = (opline+1)->opcode;
3483 target_label = ssa->cfg.blocks[b].successors[0];
3484 target_label2 = ssa->cfg.blocks[b].successors[1];
3485 } else {
3486 smart_branch_opcode = 0;
3487 target_label = target_label2 = (uint32_t)-1;
3488 }
3489 if (!zend_jit_identical(&dasm_state, opline,
3490 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
3491 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
3492 RES_REG_ADDR(),
3493 zend_may_throw(opline, ssa_op, op_array, ssa),
3494 smart_branch_opcode, target_label, target_label2,
3495 NULL, 0)) {
3496 goto jit_failure;
3497 }
3498 goto done;
3499 case ZEND_DEFINED:
3500 if ((opline->result_type & IS_TMP_VAR)
3501 && (i + 1) <= end
3502 && ((opline+1)->opcode == ZEND_JMPZ
3503 || (opline+1)->opcode == ZEND_JMPNZ)
3504 && (opline+1)->op1_type == IS_TMP_VAR
3505 && (opline+1)->op1.var == opline->result.var) {
3506 i++;
3507 smart_branch_opcode = (opline+1)->opcode;
3508 target_label = ssa->cfg.blocks[b].successors[0];
3509 target_label2 = ssa->cfg.blocks[b].successors[1];
3510 } else {
3511 smart_branch_opcode = 0;
3512 target_label = target_label2 = (uint32_t)-1;
3513 }
3514 if (!zend_jit_defined(&dasm_state, opline, smart_branch_opcode, target_label, target_label2, NULL)) {
3515 goto jit_failure;
3516 }
3517 goto done;
3518 case ZEND_TYPE_CHECK:
3519 if (opline->extended_value == MAY_BE_RESOURCE) {
3520 // TODO: support for is_resource() ???
3521 break;
3522 }
3523 if ((opline->result_type & IS_TMP_VAR)
3524 && (i + 1) <= end
3525 && ((opline+1)->opcode == ZEND_JMPZ
3526 || (opline+1)->opcode == ZEND_JMPNZ)
3527 && (opline+1)->op1_type == IS_TMP_VAR
3528 && (opline+1)->op1.var == opline->result.var) {
3529 i++;
3530 smart_branch_opcode = (opline+1)->opcode;
3531 target_label = ssa->cfg.blocks[b].successors[0];
3532 target_label2 = ssa->cfg.blocks[b].successors[1];
3533 } else {
3534 smart_branch_opcode = 0;
3535 target_label = target_label2 = (uint32_t)-1;
3536 }
3537 if (!zend_jit_type_check(&dasm_state, opline, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) {
3538 goto jit_failure;
3539 }
3540 goto done;
3541 case ZEND_RETURN:
3542 op1_info = OP1_INFO();
3543 if ((PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info))
3544 || op_array->type == ZEND_EVAL_CODE
3545 // TODO: support for top-level code
3546 || !op_array->function_name
3547 // TODO: support for IS_UNDEF ???
3548 || (op1_info & MAY_BE_UNDEF)) {
3549 if (!zend_jit_tail_handler(&dasm_state, opline)) {
3550 goto jit_failure;
3551 }
3552 } else {
3553 int j;
3554 bool left_frame = 0;
3555
3556 if (!zend_jit_return(&dasm_state, opline, op_array,
3557 op1_info, OP1_REG_ADDR())) {
3558 goto jit_failure;
3559 }
3560 if (jit_return_label >= 0) {
3561 if (!zend_jit_jmp(&dasm_state, jit_return_label)) {
3562 goto jit_failure;
3563 }
3564 goto done;
3565 }
3566 jit_return_label = ssa->cfg.blocks_count * 2;
3567 if (!zend_jit_label(&dasm_state, jit_return_label)) {
3568 goto jit_failure;
3569 }
3570 if (op_array->last_var > 100) {
3571 /* To many CVs to unroll */
3572 if (!zend_jit_free_cvs(&dasm_state)) {
3573 goto jit_failure;
3574 }
3575 left_frame = 1;
3576 }
3577 if (!left_frame) {
3578 for (j = 0 ; j < op_array->last_var; j++) {
3579 uint32_t info = zend_ssa_cv_info(op_array, ssa, j);
3580
3581 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
3582 if (!left_frame) {
3583 left_frame = 1;
3584 if (!zend_jit_leave_frame(&dasm_state)) {
3585 goto jit_failure;
3586 }
3587 }
3588 if (!zend_jit_free_cv(&dasm_state, info, j)) {
3589 goto jit_failure;
3590 }
3591 }
3592 }
3593 }
3594 if (!zend_jit_leave_func(&dasm_state, op_array, opline, op1_info, left_frame,
3595 NULL, NULL, (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, 1)) {
3596 goto jit_failure;
3597 }
3598 }
3599 goto done;
3600 case ZEND_BOOL:
3601 case ZEND_BOOL_NOT:
3602 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
3603 OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(),
3604 -1, -1,
3605 zend_may_throw(opline, ssa_op, op_array, ssa),
3606 opline->opcode, NULL)) {
3607 goto jit_failure;
3608 }
3609 goto done;
3610 case ZEND_JMPZ:
3611 case ZEND_JMPNZ:
3612 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
3613 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
3614 /* smart branch */
3615 if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
3616 goto jit_failure;
3617 }
3618 goto done;
3619 }
3620 ZEND_FALLTHROUGH;
3621 case ZEND_JMPZ_EX:
3622 case ZEND_JMPNZ_EX:
3623 if (opline->result_type == IS_UNDEF) {
3624 res_addr = 0;
3625 } else {
3626 res_addr = RES_REG_ADDR();
3627 }
3628 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
3629 OP1_INFO(), OP1_REG_ADDR(), res_addr,
3630 ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1],
3631 zend_may_throw(opline, ssa_op, op_array, ssa),
3632 opline->opcode, NULL)) {
3633 goto jit_failure;
3634 }
3635 goto done;
3636 case ZEND_ISSET_ISEMPTY_CV:
3637 if ((opline->extended_value & ZEND_ISEMPTY)) {
3638 // TODO: support for empty() ???
3639 break;
3640 }
3641 if ((opline->result_type & IS_TMP_VAR)
3642 && (i + 1) <= end
3643 && ((opline+1)->opcode == ZEND_JMPZ
3644 || (opline+1)->opcode == ZEND_JMPNZ)
3645 && (opline+1)->op1_type == IS_TMP_VAR
3646 && (opline+1)->op1.var == opline->result.var) {
3647 i++;
3648 smart_branch_opcode = (opline+1)->opcode;
3649 target_label = ssa->cfg.blocks[b].successors[0];
3650 target_label2 = ssa->cfg.blocks[b].successors[1];
3651 } else {
3652 smart_branch_opcode = 0;
3653 target_label = target_label2 = (uint32_t)-1;
3654 }
3655 if (!zend_jit_isset_isempty_cv(&dasm_state, opline,
3656 OP1_INFO(), OP1_REG_ADDR(),
3657 smart_branch_opcode, target_label, target_label2,
3658 NULL)) {
3659 goto jit_failure;
3660 }
3661 goto done;
3662 case ZEND_IN_ARRAY:
3663 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
3664 break;
3665 }
3666 op1_info = OP1_INFO();
3667 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
3668 break;
3669 }
3670 if ((opline->result_type & IS_TMP_VAR)
3671 && (i + 1) <= end
3672 && ((opline+1)->opcode == ZEND_JMPZ
3673 || (opline+1)->opcode == ZEND_JMPNZ)
3674 && (opline+1)->op1_type == IS_TMP_VAR
3675 && (opline+1)->op1.var == opline->result.var) {
3676 i++;
3677 smart_branch_opcode = (opline+1)->opcode;
3678 target_label = ssa->cfg.blocks[b].successors[0];
3679 target_label2 = ssa->cfg.blocks[b].successors[1];
3680 } else {
3681 smart_branch_opcode = 0;
3682 target_label = target_label2 = (uint32_t)-1;
3683 }
3684 if (!zend_jit_in_array(&dasm_state, opline,
3685 op1_info, OP1_REG_ADDR(),
3686 smart_branch_opcode, target_label, target_label2,
3687 NULL)) {
3688 goto jit_failure;
3689 }
3690 goto done;
3691 case ZEND_FETCH_DIM_R:
3692 case ZEND_FETCH_DIM_IS:
3693 case ZEND_FETCH_LIST_R:
3694 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3695 break;
3696 }
3697 if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op,
3698 OP1_INFO(), OP1_REG_ADDR(), 0,
3699 OP2_INFO(), RES_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
3700 goto jit_failure;
3701 }
3702 goto done;
3703 case ZEND_FETCH_DIM_W:
3704 case ZEND_FETCH_DIM_RW:
3705 // case ZEND_FETCH_DIM_UNSET:
3706 case ZEND_FETCH_LIST_W:
3707 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3708 break;
3709 }
3710 if (opline->op1_type != IS_CV) {
3711 break;
3712 }
3713 if (!zend_jit_fetch_dim(&dasm_state, opline,
3714 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
3715 goto jit_failure;
3716 }
3717 goto done;
3718 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
3719 if ((opline->extended_value & ZEND_ISEMPTY)) {
3720 // TODO: support for empty() ???
3721 break;
3722 }
3723 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3724 break;
3725 }
3726 if ((opline->result_type & IS_TMP_VAR)
3727 && (i + 1) <= end
3728 && ((opline+1)->opcode == ZEND_JMPZ
3729 || (opline+1)->opcode == ZEND_JMPNZ)
3730 && (opline+1)->op1_type == IS_TMP_VAR
3731 && (opline+1)->op1.var == opline->result.var) {
3732 i++;
3733 smart_branch_opcode = (opline+1)->opcode;
3734 target_label = ssa->cfg.blocks[b].successors[0];
3735 target_label2 = ssa->cfg.blocks[b].successors[1];
3736 } else {
3737 smart_branch_opcode = 0;
3738 target_label = target_label2 = (uint32_t)-1;
3739 }
3740 if (!zend_jit_isset_isempty_dim(&dasm_state, opline,
3741 OP1_INFO(), OP1_REG_ADDR(), 0,
3742 OP2_INFO(), IS_UNKNOWN,
3743 zend_may_throw(opline, ssa_op, op_array, ssa),
3744 smart_branch_opcode, target_label, target_label2,
3745 NULL)) {
3746 goto jit_failure;
3747 }
3748 goto done;
3749 case ZEND_FETCH_OBJ_R:
3750 case ZEND_FETCH_OBJ_IS:
3751 case ZEND_FETCH_OBJ_W:
3752 if (opline->op2_type != IS_CONST
3753 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3754 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3755 break;
3756 }
3757 ce = NULL;
3758 ce_is_instanceof = 0;
3759 on_this = 0;
3760 if (opline->op1_type == IS_UNUSED) {
3761 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3762 op1_addr = 0;
3763 ce = op_array->scope;
3764 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3765 on_this = 1;
3766 } else {
3767 op1_info = OP1_INFO();
3768 if (!(op1_info & MAY_BE_OBJECT)) {
3769 break;
3770 }
3771 op1_addr = OP1_REG_ADDR();
3772 if (ssa->var_info && ssa->ops) {
3773 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3774 if (ssa_op->op1_use >= 0) {
3775 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3776 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3777 ce = op1_ssa->ce;
3778 ce_is_instanceof = op1_ssa->is_instanceof;
3779 }
3780 }
3781 }
3782 }
3783 if (!zend_jit_fetch_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3784 op1_info, op1_addr, 0, ce, ce_is_instanceof, on_this, 0, 0, NULL,
3785 IS_UNKNOWN,
3786 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3787 goto jit_failure;
3788 }
3789 goto done;
3790 case ZEND_BIND_GLOBAL:
3791 if (!ssa->ops || !ssa->var_info) {
3792 op1_info = MAY_BE_ANY|MAY_BE_REF;
3793 } else {
3794 op1_info = OP1_INFO();
3795 }
3796 if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) {
3797 goto jit_failure;
3798 }
3799 goto done;
3800 case ZEND_RECV:
3801 if (!zend_jit_recv(&dasm_state, opline, op_array)) {
3802 goto jit_failure;
3803 }
3804 goto done;
3805 case ZEND_RECV_INIT:
3806 if (!zend_jit_recv_init(&dasm_state, opline, op_array,
3807 (opline + 1)->opcode != ZEND_RECV_INIT,
3808 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3809 goto jit_failure;
3810 }
3811 goto done;
3812 case ZEND_FREE:
3813 case ZEND_FE_FREE:
3814 if (!zend_jit_free(&dasm_state, opline, OP1_INFO(),
3815 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3816 goto jit_failure;
3817 }
3818 goto done;
3819 case ZEND_ECHO:
3820 op1_info = OP1_INFO();
3821 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3822 break;
3823 }
3824 if (!zend_jit_echo(&dasm_state, opline, op1_info)) {
3825 goto jit_failure;
3826 }
3827 goto done;
3828 case ZEND_STRLEN:
3829 op1_info = OP1_INFO();
3830 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3831 break;
3832 }
3833 if (!zend_jit_strlen(&dasm_state, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR())) {
3834 goto jit_failure;
3835 }
3836 goto done;
3837 case ZEND_COUNT:
3838 op1_info = OP1_INFO();
3839 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
3840 break;
3841 }
3842 if (!zend_jit_count(&dasm_state, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
3843 goto jit_failure;
3844 }
3845 goto done;
3846 case ZEND_FETCH_THIS:
3847 if (!zend_jit_fetch_this(&dasm_state, opline, op_array, 0)) {
3848 goto jit_failure;
3849 }
3850 goto done;
3851 case ZEND_SWITCH_LONG:
3852 case ZEND_SWITCH_STRING:
3853 case ZEND_MATCH:
3854 if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL, NULL)) {
3855 goto jit_failure;
3856 }
3857 goto done;
3858 case ZEND_VERIFY_RETURN_TYPE:
3859 if (opline->op1_type == IS_UNUSED) {
3860 /* Always throws */
3861 break;
3862 }
3863 if (opline->op1_type == IS_CONST) {
3864 /* TODO Different instruction format, has return value */
3865 break;
3866 }
3867 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
3868 /* Not worth bothering with */
3869 break;
3870 }
3871 if (OP1_INFO() & MAY_BE_REF) {
3872 /* TODO May need reference unwrapping. */
3873 break;
3874 }
3875 if (!zend_jit_verify_return_type(&dasm_state, opline, op_array, OP1_INFO())) {
3876 goto jit_failure;
3877 }
3878 goto done;
3879 case ZEND_FE_RESET_R:
3880 op1_info = OP1_INFO();
3881 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
3882 break;
3883 }
3884 if (!zend_jit_fe_reset(&dasm_state, opline, op1_info)) {
3885 goto jit_failure;
3886 }
3887 goto done;
3888 case ZEND_FE_FETCH_R:
3889 op1_info = OP1_INFO();
3890 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
3891 break;
3892 }
3893 if (!zend_jit_fe_fetch(&dasm_state, opline, op1_info, OP2_INFO(),
3894 ssa->cfg.blocks[b].successors[0], opline->opcode, NULL)) {
3895 goto jit_failure;
3896 }
3897 goto done;
3898 case ZEND_FETCH_CONSTANT:
3899 if (!zend_jit_fetch_constant(&dasm_state, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
3900 goto jit_failure;
3901 }
3902 goto done;
3903 case ZEND_INIT_METHOD_CALL:
3904 if (opline->op2_type != IS_CONST
3905 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
3906 break;
3907 }
3908 ce = NULL;
3909 ce_is_instanceof = 0;
3910 on_this = 0;
3911 if (opline->op1_type == IS_UNUSED) {
3912 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3913 op1_addr = 0;
3914 ce = op_array->scope;
3915 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3916 on_this = 1;
3917 } else {
3918 op1_info = OP1_INFO();
3919 if (!(op1_info & MAY_BE_OBJECT)) {
3920 break;
3921 }
3922 op1_addr = OP1_REG_ADDR();
3923 if (ssa->var_info && ssa->ops) {
3924 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3925 if (ssa_op->op1_use >= 0) {
3926 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3927 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3928 ce = op1_ssa->ce;
3929 ce_is_instanceof = op1_ssa->is_instanceof;
3930 }
3931 }
3932 }
3933 }
3934 if (!zend_jit_init_method_call(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level,
3935 op1_info, op1_addr, ce, ce_is_instanceof, on_this, 0, NULL,
3936 NULL, 0, 0)) {
3937 goto jit_failure;
3938 }
3939 goto done;
3940 case ZEND_ROPE_INIT:
3941 case ZEND_ROPE_ADD:
3942 case ZEND_ROPE_END:
3943 op2_info = OP2_INFO();
3944 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3945 break;
3946 }
3947 if (!zend_jit_rope(&dasm_state, opline, op2_info)) {
3948 goto jit_failure;
3949 }
3950 goto done;
3951 default:
3952 break;
3953 }
3954 }
3955
3956 switch (opline->opcode) {
3957 case ZEND_RECV_INIT:
3958 case ZEND_BIND_GLOBAL:
3959 if (opline == op_array->opcodes ||
3960 opline->opcode != op_array->opcodes[i-1].opcode) {
3961 /* repeatable opcodes */
3962 if (!zend_jit_handler(&dasm_state, opline,
3963 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3964 goto jit_failure;
3965 }
3966 }
3967 zend_jit_set_last_valid_opline(opline+1);
3968 break;
3969 case ZEND_NOP:
3970 case ZEND_OP_DATA:
3971 case ZEND_SWITCH_LONG:
3972 case ZEND_SWITCH_STRING:
3973 break;
3974 case ZEND_MATCH:
3975 /* We have to exit to the VM because the MATCH handler performs an N-way jump for
3976 * which we can't generate simple (opcache.jit=1201) JIT code. */
3977 if (!zend_jit_tail_handler(&dasm_state, opline)) {
3978 goto jit_failure;
3979 }
3980 break;
3981 case ZEND_JMP:
3982 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
3983 const zend_op *target = OP_JMP_ADDR(opline, opline->op1);
3984
3985 if (!zend_jit_set_ip(&dasm_state, target)) {
3986 goto jit_failure;
3987 }
3988 }
3989 if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) {
3990 goto jit_failure;
3991 }
3992 is_terminated = 1;
3993 break;
3994 case ZEND_CATCH:
3995 case ZEND_FAST_CALL:
3996 case ZEND_FAST_RET:
3997 case ZEND_GENERATOR_CREATE:
3998 case ZEND_GENERATOR_RETURN:
3999 case ZEND_RETURN_BY_REF:
4000 case ZEND_RETURN:
4001 case ZEND_EXIT:
4002 case ZEND_MATCH_ERROR:
4003 /* switch through trampoline */
4004 case ZEND_YIELD:
4005 case ZEND_YIELD_FROM:
4006 if (!zend_jit_tail_handler(&dasm_state, opline)) {
4007 goto jit_failure;
4008 }
4009 is_terminated = 1;
4010 break;
4011 /* stackless execution */
4012 case ZEND_INCLUDE_OR_EVAL:
4013 case ZEND_DO_FCALL:
4014 case ZEND_DO_UCALL:
4015 case ZEND_DO_FCALL_BY_NAME:
4016 if (!zend_jit_call(&dasm_state, opline, b + 1)) {
4017 goto jit_failure;
4018 }
4019 is_terminated = 1;
4020 break;
4021 case ZEND_JMPZ:
4022 case ZEND_JMPNZ:
4023 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
4024 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
4025 /* smart branch */
4026 if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
4027 goto jit_failure;
4028 }
4029 goto done;
4030 }
4031 ZEND_FALLTHROUGH;
4032 case ZEND_JMPZ_EX:
4033 case ZEND_JMPNZ_EX:
4034 case ZEND_JMP_SET:
4035 case ZEND_COALESCE:
4036 case ZEND_JMP_NULL:
4037 case ZEND_FE_RESET_R:
4038 case ZEND_FE_RESET_RW:
4039 case ZEND_ASSERT_CHECK:
4040 case ZEND_FE_FETCH_R:
4041 case ZEND_FE_FETCH_RW:
4042 if (!zend_jit_handler(&dasm_state, opline,
4043 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
4044 !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
4045 goto jit_failure;
4046 }
4047 break;
4048 case ZEND_NEW:
4049 if (!zend_jit_handler(&dasm_state, opline, 1)) {
4050 return 0;
4051 }
4052 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
4053 zend_class_entry *ce = NULL;
4054
4055 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
4056 if (ssa->ops && ssa->var_info) {
4057 zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
4058 if (res_ssa->ce && !res_ssa->is_instanceof) {
4059 ce = res_ssa->ce;
4060 }
4061 }
4062 } else {
4063 if (opline->op1_type == IS_CONST) {
4064 zval *zv = RT_CONSTANT(opline, opline->op1);
4065 if (Z_TYPE_P(zv) == IS_STRING) {
4066 zval *lc = zv + 1;
4067 ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc));
4068 }
4069 }
4070 }
4071
4072 i++;
4073
4074 if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) {
4075 const zend_op *next_opline = opline + 1;
4076
4077 zend_jit_cond_jmp(&dasm_state, next_opline, ssa->cfg.blocks[b].successors[0]);
4078 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
4079 zend_jit_call(&dasm_state, next_opline, b + 1);
4080 is_terminated = 1;
4081 } else {
4082 zend_jit_do_fcall(&dasm_state, next_opline, op_array, ssa, call_level, b + 1, NULL);
4083 }
4084 }
4085
4086 /* We skip over the DO_FCALL, so decrement call_level ourselves. */
4087 call_level--;
4088 }
4089 break;
4090 default:
4091 if (!zend_jit_handler(&dasm_state, opline,
4092 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4093 goto jit_failure;
4094 }
4095 if (i == end
4096 && (opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
4097 /* smart branch split across basic blocks */
4098 if (!zend_jit_cond_jmp(&dasm_state, opline + 2, ssa->cfg.blocks[b+1].successors[0])) {
4099 goto jit_failure;
4100 }
4101 if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b+1].successors[1])) {
4102 goto jit_failure;
4103 }
4104 is_terminated = 1;
4105 }
4106 }
4107 done:
4108 switch (opline->opcode) {
4109 case ZEND_DO_FCALL:
4110 case ZEND_DO_ICALL:
4111 case ZEND_DO_UCALL:
4112 case ZEND_DO_FCALL_BY_NAME:
4113 case ZEND_CALLABLE_CONVERT:
4114 call_level--;
4115 }
4116 }
4117 }
4118
4119 handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0,
4120 (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET, SP_ADJ_JIT);
4121 if (!handler) {
4122 goto jit_failure;
4123 }
4124 dasm_free(&dasm_state);
4125
4126 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
4127 zend_arena_release(&CG(arena), checkpoint);
4128 }
4129 return SUCCESS;
4130
4131 jit_failure:
4132 if (dasm_state) {
4133 dasm_free(&dasm_state);
4134 }
4135 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
4136 zend_arena_release(&CG(arena), checkpoint);
4137 }
4138 return FAILURE;
4139 }
4140
zend_jit_collect_calls(zend_op_array * op_array,zend_script * script)4141 static void zend_jit_collect_calls(zend_op_array *op_array, zend_script *script)
4142 {
4143 zend_func_info *func_info;
4144
4145 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4146 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4147 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4148 func_info = ZEND_FUNC_INFO(op_array);
4149 } else {
4150 func_info = zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info));
4151 ZEND_SET_FUNC_INFO(op_array, func_info);
4152 }
4153 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, op_array, func_info);
4154 }
4155
zend_jit_cleanup_func_info(zend_op_array * op_array)4156 static void zend_jit_cleanup_func_info(zend_op_array *op_array)
4157 {
4158 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
4159 zend_call_info *caller_info, *callee_info;
4160
4161 if (func_info) {
4162 caller_info = func_info->caller_info;
4163 callee_info = func_info->callee_info;
4164
4165 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4166 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4167 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4168 func_info->num = 0;
4169 func_info->flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
4170 | ZEND_FUNC_JIT_ON_PROF_REQUEST
4171 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
4172 | ZEND_FUNC_JIT_ON_HOT_TRACE;
4173 memset(&func_info->ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
4174 } else {
4175 ZEND_SET_FUNC_INFO(op_array, NULL);
4176 }
4177
4178 while (caller_info) {
4179 if (caller_info->caller_op_array) {
4180 zend_jit_cleanup_func_info(caller_info->caller_op_array);
4181 }
4182 caller_info = caller_info->next_caller;
4183 }
4184 while (callee_info) {
4185 if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) {
4186 zend_jit_cleanup_func_info(&callee_info->callee_func->op_array);
4187 }
4188 callee_info = callee_info->next_callee;
4189 }
4190 }
4191 }
4192
zend_real_jit_func(zend_op_array * op_array,zend_script * script,const zend_op * rt_opline,uint8_t trigger)4193 static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline, uint8_t trigger)
4194 {
4195 zend_ssa ssa;
4196 void *checkpoint;
4197 zend_func_info *func_info;
4198 uint8_t orig_trigger;
4199
4200 if (*dasm_ptr == dasm_end) {
4201 return FAILURE;
4202 }
4203
4204 orig_trigger = JIT_G(trigger);
4205 JIT_G(trigger) = trigger;
4206 checkpoint = zend_arena_checkpoint(CG(arena));
4207
4208 /* Build SSA */
4209 memset(&ssa, 0, sizeof(zend_ssa));
4210
4211 if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) {
4212 goto jit_failure;
4213 }
4214
4215 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
4216 zend_jit_collect_calls(op_array, script);
4217 func_info = ZEND_FUNC_INFO(op_array);
4218 func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array);
4219 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
4220 zend_init_func_return_info(op_array, script, &func_info->return_info);
4221 }
4222 }
4223
4224 if (zend_jit_op_array_analyze2(op_array, script, &ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
4225 goto jit_failure;
4226 }
4227
4228 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
4229 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa);
4230 }
4231
4232 if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) {
4233 goto jit_failure;
4234 }
4235
4236 zend_jit_cleanup_func_info(op_array);
4237 zend_arena_release(&CG(arena), checkpoint);
4238 JIT_G(trigger) = orig_trigger;
4239 return SUCCESS;
4240
4241 jit_failure:
4242 zend_jit_cleanup_func_info(op_array);
4243 zend_arena_release(&CG(arena), checkpoint);
4244 JIT_G(trigger) = orig_trigger;
4245 return FAILURE;
4246 }
4247
4248 /* Run-time JIT handler */
zend_runtime_jit(void)4249 static int ZEND_FASTCALL zend_runtime_jit(void)
4250 {
4251 zend_execute_data *execute_data = EG(current_execute_data);
4252 zend_op_array *op_array = &EX(func)->op_array;
4253 zend_op *opline = op_array->opcodes;
4254 zend_jit_op_array_extension *jit_extension;
4255 bool do_bailout = 0;
4256
4257 zend_shared_alloc_lock();
4258
4259 if (ZEND_FUNC_INFO(op_array)) {
4260
4261 SHM_UNPROTECT();
4262 zend_jit_unprotect();
4263
4264 zend_try {
4265 /* restore original opcode handlers */
4266 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4267 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4268 opline++;
4269 }
4270 }
4271 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
4272 opline->handler = jit_extension->orig_handler;
4273
4274 /* perform real JIT for this function */
4275 zend_real_jit_func(op_array, NULL, NULL, ZEND_JIT_ON_FIRST_EXEC);
4276 } zend_catch {
4277 do_bailout = true;
4278 } zend_end_try();
4279
4280 zend_jit_protect();
4281 SHM_PROTECT();
4282 }
4283
4284 zend_shared_alloc_unlock();
4285
4286 if (do_bailout) {
4287 zend_bailout();
4288 }
4289
4290 /* JIT-ed code is going to be called by VM */
4291 return 0;
4292 }
4293
zend_jit_check_funcs(HashTable * function_table,bool is_method)4294 void zend_jit_check_funcs(HashTable *function_table, bool is_method) {
4295 zend_op *opline;
4296 zend_function *func;
4297 zend_op_array *op_array;
4298 uintptr_t counter;
4299 zend_jit_op_array_extension *jit_extension;
4300
4301 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(function_table, func) {
4302 if (func->type == ZEND_INTERNAL_FUNCTION) {
4303 break;
4304 }
4305 op_array = &func->op_array;
4306 opline = op_array->opcodes;
4307 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4308 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4309 opline++;
4310 }
4311 }
4312 if (opline->handler == zend_jit_profile_jit_handler) {
4313 if (!RUN_TIME_CACHE(op_array)) {
4314 continue;
4315 }
4316 counter = (uintptr_t)ZEND_COUNTER_INFO(op_array);
4317 ZEND_COUNTER_INFO(op_array) = 0;
4318 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
4319 opline->handler = jit_extension->orig_handler;
4320 if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) {
4321 zend_real_jit_func(op_array, NULL, NULL, ZEND_JIT_ON_PROF_REQUEST);
4322 }
4323 }
4324 } ZEND_HASH_FOREACH_END();
4325 }
4326
zend_jit_hot_func(zend_execute_data * execute_data,const zend_op * opline)4327 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
4328 {
4329 zend_op_array *op_array = &EX(func)->op_array;
4330 zend_jit_op_array_hot_extension *jit_extension;
4331 uint32_t i;
4332 bool do_bailout = 0;
4333
4334 zend_shared_alloc_lock();
4335 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
4336
4337 if (jit_extension) {
4338 SHM_UNPROTECT();
4339 zend_jit_unprotect();
4340
4341 zend_try {
4342 for (i = 0; i < op_array->last; i++) {
4343 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
4344 }
4345
4346 /* perform real JIT for this function */
4347 zend_real_jit_func(op_array, NULL, opline, ZEND_JIT_ON_HOT_COUNTERS);
4348 } zend_catch {
4349 do_bailout = 1;
4350 } zend_end_try();
4351
4352 zend_jit_protect();
4353 SHM_PROTECT();
4354 }
4355
4356 zend_shared_alloc_unlock();
4357
4358 if (do_bailout) {
4359 zend_bailout();
4360 }
4361 /* JIT-ed code is going to be called by VM */
4362 }
4363
zend_jit_setup_hot_counters_ex(zend_op_array * op_array,zend_cfg * cfg)4364 static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg)
4365 {
4366 if (JIT_G(hot_func)) {
4367 zend_op *opline = op_array->opcodes;
4368
4369 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4370 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4371 opline++;
4372 }
4373 }
4374
4375 opline->handler = (const void*)zend_jit_func_hot_counter_handler;
4376 }
4377
4378 if (JIT_G(hot_loop)) {
4379 uint32_t i;
4380
4381 for (i = 0; i < cfg->blocks_count; i++) {
4382 if ((cfg->blocks[i].flags & ZEND_BB_REACHABLE) &&
4383 (cfg->blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
4384 op_array->opcodes[cfg->blocks[i].start].handler =
4385 (const void*)zend_jit_loop_hot_counter_handler;
4386 }
4387 }
4388 }
4389 }
4390
zend_jit_restart_hot_counters(zend_op_array * op_array)4391 static int zend_jit_restart_hot_counters(zend_op_array *op_array)
4392 {
4393 zend_jit_op_array_hot_extension *jit_extension;
4394 zend_cfg cfg;
4395 uint32_t i;
4396
4397 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
4398 for (i = 0; i < op_array->last; i++) {
4399 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
4400 }
4401
4402 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
4403 return FAILURE;
4404 }
4405
4406 zend_jit_setup_hot_counters_ex(op_array, &cfg);
4407
4408 return SUCCESS;
4409 }
4410
zend_jit_setup_hot_counters(zend_op_array * op_array)4411 static int zend_jit_setup_hot_counters(zend_op_array *op_array)
4412 {
4413 zend_jit_op_array_hot_extension *jit_extension;
4414 zend_cfg cfg;
4415 uint32_t i;
4416
4417 ZEND_ASSERT(zend_jit_func_hot_counter_handler != NULL);
4418 ZEND_ASSERT(zend_jit_loop_hot_counter_handler != NULL);
4419
4420 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
4421 return FAILURE;
4422 }
4423
4424 jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*));
4425 if (!jit_extension) {
4426 return FAILURE;
4427 }
4428 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4429 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS;
4430 jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
4431 for (i = 0; i < op_array->last; i++) {
4432 jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
4433 }
4434 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4435
4436 zend_jit_setup_hot_counters_ex(op_array, &cfg);
4437
4438 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4439
4440 return SUCCESS;
4441 }
4442
4443 #include "jit/zend_jit_trace.c"
4444
zend_jit_op_array(zend_op_array * op_array,zend_script * script)4445 ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
4446 {
4447 if (dasm_ptr == NULL) {
4448 return FAILURE;
4449 }
4450
4451 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) {
4452 zend_jit_op_array_extension *jit_extension;
4453 zend_op *opline = op_array->opcodes;
4454
4455 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
4456 ZEND_SET_FUNC_INFO(op_array, NULL);
4457 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
4458 return SUCCESS;
4459 }
4460
4461 /* Set run-time JIT handler */
4462 ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL);
4463 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4464 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4465 opline++;
4466 }
4467 }
4468 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
4469 if (!jit_extension) {
4470 return FAILURE;
4471 }
4472 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4473 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC;
4474 jit_extension->orig_handler = (void*)opline->handler;
4475 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4476 opline->handler = (const void*)zend_jit_runtime_jit_handler;
4477 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4478
4479 return SUCCESS;
4480 } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) {
4481 zend_jit_op_array_extension *jit_extension;
4482 zend_op *opline = op_array->opcodes;
4483
4484 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
4485 ZEND_SET_FUNC_INFO(op_array, NULL);
4486 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
4487 return SUCCESS;
4488 }
4489
4490 ZEND_ASSERT(zend_jit_profile_jit_handler != NULL);
4491 if (op_array->function_name) {
4492 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4493 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4494 opline++;
4495 }
4496 }
4497 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
4498 if (!jit_extension) {
4499 return FAILURE;
4500 }
4501 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4502 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST;
4503 jit_extension->orig_handler = (void*)opline->handler;
4504 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4505 opline->handler = (const void*)zend_jit_profile_jit_handler;
4506 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4507 }
4508
4509 return SUCCESS;
4510 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4511 return zend_jit_setup_hot_counters(op_array);
4512 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4513 return zend_jit_setup_hot_trace_counters(op_array);
4514 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4515 return zend_real_jit_func(op_array, script, NULL, ZEND_JIT_ON_SCRIPT_LOAD);
4516 } else {
4517 ZEND_UNREACHABLE();
4518 }
4519 }
4520
zend_jit_script(zend_script * script)4521 ZEND_EXT_API int zend_jit_script(zend_script *script)
4522 {
4523 void *checkpoint;
4524 zend_call_graph call_graph;
4525 zend_func_info *info;
4526 int i;
4527
4528 if (dasm_ptr == NULL || *dasm_ptr == dasm_end) {
4529 return FAILURE;
4530 }
4531
4532 checkpoint = zend_arena_checkpoint(CG(arena));
4533
4534 call_graph.op_arrays_count = 0;
4535 zend_build_call_graph(&CG(arena), script, &call_graph);
4536
4537 zend_analyze_call_graph(&CG(arena), script, &call_graph);
4538
4539 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4540 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4541 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS ||
4542 JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4543 for (i = 0; i < call_graph.op_arrays_count; i++) {
4544 if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
4545 goto jit_failure;
4546 }
4547 }
4548 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4549 for (i = 0; i < call_graph.op_arrays_count; i++) {
4550 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4551 if (info) {
4552 if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
4553 goto jit_failure;
4554 }
4555 info->flags = info->ssa.cfg.flags;
4556 }
4557 }
4558
4559 for (i = 0; i < call_graph.op_arrays_count; i++) {
4560 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4561 if (info) {
4562 info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]);
4563 if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
4564 zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info);
4565 }
4566 }
4567 }
4568
4569 for (i = 0; i < call_graph.op_arrays_count; i++) {
4570 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4571 if (info) {
4572 if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
4573 goto jit_failure;
4574 }
4575 info->flags = info->ssa.cfg.flags;
4576 }
4577 }
4578
4579 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
4580 for (i = 0; i < call_graph.op_arrays_count; i++) {
4581 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4582 if (info) {
4583 zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa);
4584 }
4585 }
4586 }
4587
4588 for (i = 0; i < call_graph.op_arrays_count; i++) {
4589 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4590 if (info) {
4591 if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) {
4592 goto jit_failure;
4593 }
4594 }
4595 }
4596
4597 for (i = 0; i < call_graph.op_arrays_count; i++) {
4598 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
4599 }
4600 } else {
4601 ZEND_UNREACHABLE();
4602 }
4603
4604 zend_arena_release(&CG(arena), checkpoint);
4605
4606 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC
4607 || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST
4608 || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS
4609 || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4610 zend_class_entry *ce;
4611 zend_op_array *op_array;
4612
4613 ZEND_HASH_MAP_FOREACH_PTR(&script->class_table, ce) {
4614 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
4615 if (!ZEND_FUNC_INFO(op_array)) {
4616 void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes);
4617
4618 if (jit_extension) {
4619 ZEND_SET_FUNC_INFO(op_array, jit_extension);
4620 }
4621 }
4622 } ZEND_HASH_FOREACH_END();
4623 } ZEND_HASH_FOREACH_END();
4624 }
4625
4626 return SUCCESS;
4627
4628 jit_failure:
4629 if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4630 for (i = 0; i < call_graph.op_arrays_count; i++) {
4631 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
4632 }
4633 }
4634 zend_arena_release(&CG(arena), checkpoint);
4635 return FAILURE;
4636 }
4637
zend_jit_unprotect(void)4638 ZEND_EXT_API void zend_jit_unprotect(void)
4639 {
4640 #ifdef HAVE_MPROTECT
4641 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4642 int opts = PROT_READ | PROT_WRITE;
4643 #ifdef ZTS
4644 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
4645 if (zend_write_protect) {
4646 pthread_jit_write_protect_np(0);
4647 }
4648 #endif
4649 opts |= PROT_EXEC;
4650 #endif
4651 if (mprotect(dasm_buf, dasm_size, opts) != 0) {
4652 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4653 }
4654 }
4655 #elif _WIN32
4656 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4657 DWORD old, new;
4658 #ifdef ZTS
4659 new = PAGE_EXECUTE_READWRITE;
4660 #else
4661 new = PAGE_READWRITE;
4662 #endif
4663 if (!VirtualProtect(dasm_buf, dasm_size, new, &old)) {
4664 DWORD err = GetLastError();
4665 char *msg = php_win32_error_to_msg(err);
4666 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4667 php_win32_error_msg_free(msg);
4668 }
4669 }
4670 #endif
4671 }
4672
zend_jit_protect(void)4673 ZEND_EXT_API void zend_jit_protect(void)
4674 {
4675 #ifdef HAVE_MPROTECT
4676 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4677 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
4678 if (zend_write_protect) {
4679 pthread_jit_write_protect_np(1);
4680 }
4681 #endif
4682 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
4683 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4684 }
4685 }
4686 #elif _WIN32
4687 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4688 DWORD old;
4689
4690 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
4691 DWORD err = GetLastError();
4692 char *msg = php_win32_error_to_msg(err);
4693 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4694 php_win32_error_msg_free(msg);
4695 }
4696 }
4697 #endif
4698 }
4699
zend_jit_init_handlers(void)4700 static void zend_jit_init_handlers(void)
4701 {
4702 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
4703 zend_jit_runtime_jit_handler = dasm_labels[zend_lbhybrid_runtime_jit];
4704 zend_jit_profile_jit_handler = dasm_labels[zend_lbhybrid_profile_jit];
4705 zend_jit_func_hot_counter_handler = dasm_labels[zend_lbhybrid_func_hot_counter];
4706 zend_jit_loop_hot_counter_handler = dasm_labels[zend_lbhybrid_loop_hot_counter];
4707 zend_jit_func_trace_counter_handler = dasm_labels[zend_lbhybrid_func_trace_counter];
4708 zend_jit_ret_trace_counter_handler = dasm_labels[zend_lbhybrid_ret_trace_counter];
4709 zend_jit_loop_trace_counter_handler = dasm_labels[zend_lbhybrid_loop_trace_counter];
4710 } else {
4711 zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
4712 zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
4713 zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper;
4714 zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper;
4715 zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper;
4716 zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper;
4717 zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper;
4718 }
4719 }
4720
zend_jit_make_stubs(void)4721 static int zend_jit_make_stubs(void)
4722 {
4723 dasm_State* dasm_state = NULL;
4724 uint32_t i;
4725
4726 dasm_init(&dasm_state, DASM_MAXSECTION);
4727 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
4728
4729 for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) {
4730 dasm_setup(&dasm_state, dasm_actions);
4731 if (!zend_jit_stubs[i].stub(&dasm_state)) {
4732 return 0;
4733 }
4734 if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name, 0,
4735 zend_jit_stubs[i].offset, zend_jit_stubs[i].adjustment)) {
4736 return 0;
4737 }
4738 }
4739
4740 zend_jit_init_handlers();
4741
4742 dasm_free(&dasm_state);
4743 return 1;
4744 }
4745
zend_jit_globals_ctor(zend_jit_globals * jit_globals)4746 static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
4747 {
4748 memset(jit_globals, 0, sizeof(zend_jit_globals));
4749 zend_jit_trace_init_caches();
4750 }
4751
4752 #ifdef ZTS
zend_jit_globals_dtor(zend_jit_globals * jit_globals)4753 static void zend_jit_globals_dtor(zend_jit_globals *jit_globals)
4754 {
4755 zend_jit_trace_free_caches(jit_globals);
4756 }
4757 #endif
4758
zend_jit_parse_config_num(zend_long jit)4759 static int zend_jit_parse_config_num(zend_long jit)
4760 {
4761 if (jit == 0) {
4762 JIT_G(on) = 0;
4763 return SUCCESS;
4764 }
4765
4766 if (jit < 0) return FAILURE;
4767
4768 if (jit % 10 == 0 || jit % 10 > 5) return FAILURE;
4769 JIT_G(opt_level) = jit % 10;
4770
4771 jit /= 10;
4772 if (jit % 10 > 5 || jit % 10 == 4) return FAILURE;
4773 JIT_G(trigger) = jit % 10;
4774
4775 jit /= 10;
4776 if (jit % 10 > 2) return FAILURE;
4777 JIT_G(opt_flags) = jit % 10;
4778
4779 jit /= 10;
4780 if (jit % 10 > 1) return FAILURE;
4781 JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0);
4782
4783 if (jit / 10 != 0) return FAILURE;
4784
4785 JIT_G(on) = 1;
4786
4787 return SUCCESS;
4788 }
4789
zend_jit_config(zend_string * jit,int stage)4790 ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage)
4791 {
4792 if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) {
4793 if (stage == ZEND_INI_STAGE_RUNTIME) {
4794 zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)");
4795 }
4796 return FAILURE;
4797 }
4798
4799 if (zend_string_equals_literal_ci(jit, "disable")) {
4800 JIT_G(enabled) = 0;
4801 JIT_G(on) = 0;
4802 return SUCCESS;
4803 } else if (ZSTR_LEN(jit) == 0
4804 || zend_string_equals_literal_ci(jit, "0")
4805 || zend_string_equals_literal_ci(jit, "off")
4806 || zend_string_equals_literal_ci(jit, "no")
4807 || zend_string_equals_literal_ci(jit, "false")) {
4808 JIT_G(enabled) = 1;
4809 JIT_G(on) = 0;
4810 return SUCCESS;
4811 } else if (zend_string_equals_literal_ci(jit, "1")
4812 || zend_string_equals_literal_ci(jit, "on")
4813 || zend_string_equals_literal_ci(jit, "yes")
4814 || zend_string_equals_literal_ci(jit, "true")
4815 || zend_string_equals_literal_ci(jit, "tracing")) {
4816 JIT_G(enabled) = 1;
4817 JIT_G(on) = 1;
4818 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS;
4819 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
4820 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
4821 return SUCCESS;
4822 } else if (zend_string_equals_literal_ci(jit, "function")) {
4823 JIT_G(enabled) = 1;
4824 JIT_G(on) = 1;
4825 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT;
4826 JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD;
4827 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
4828 return SUCCESS;
4829 } else {
4830 char *end;
4831 zend_long num = ZEND_STRTOL(ZSTR_VAL(jit), &end, 10);
4832 if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) {
4833 goto failure;
4834 }
4835 JIT_G(enabled) = 1;
4836 return SUCCESS;
4837 }
4838
4839 failure:
4840 zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number");
4841 JIT_G(enabled) = 0;
4842 JIT_G(on) = 0;
4843 return FAILURE;
4844 }
4845
zend_jit_debug_config(zend_long old_val,zend_long new_val,int stage)4846 ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage)
4847 {
4848 if (stage != ZEND_INI_STAGE_STARTUP) {
4849 if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) {
4850 if (stage == ZEND_INI_STAGE_RUNTIME) {
4851 zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup");
4852 }
4853 return FAILURE;
4854 }
4855 #ifdef HAVE_DISASM
4856 if (new_val & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
4857 if (JIT_G(enabled) && !JIT_G(symbols) && !zend_jit_disasm_init()) {
4858 // TODO: error reporting and cleanup ???
4859 return FAILURE;
4860 }
4861 // TODO: symbols for JIT-ed code compiled before are missing ???
4862 }
4863 #endif
4864 }
4865 return SUCCESS;
4866 }
4867
zend_jit_init(void)4868 ZEND_EXT_API void zend_jit_init(void)
4869 {
4870 #ifdef ZTS
4871 jit_globals_id = ts_allocate_id(&jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, (ts_allocate_dtor) zend_jit_globals_dtor);
4872 #else
4873 zend_jit_globals_ctor(&jit_globals);
4874 #endif
4875 }
4876
zend_jit_check_support(void)4877 ZEND_EXT_API int zend_jit_check_support(void)
4878 {
4879 int i;
4880
4881 zend_jit_vm_kind = zend_vm_kind();
4882 if (zend_jit_vm_kind != ZEND_VM_KIND_CALL &&
4883 zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
4884 zend_error(E_WARNING, "JIT is compatible only with CALL and HYBRID VM. JIT disabled.");
4885 JIT_G(enabled) = 0;
4886 JIT_G(on) = 0;
4887 return FAILURE;
4888 }
4889
4890 if (zend_execute_ex != execute_ex) {
4891 if (zend_dtrace_enabled) {
4892 zend_error(E_WARNING, "JIT is incompatible with DTrace. JIT disabled.");
4893 } else if (strcmp(sapi_module.name, "phpdbg") != 0) {
4894 zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled.");
4895 }
4896 JIT_G(enabled) = 0;
4897 JIT_G(on) = 0;
4898 return FAILURE;
4899 }
4900
4901 for (i = 0; i <= 256; i++) {
4902 switch (i) {
4903 /* JIT has no effect on these opcodes */
4904 case ZEND_BEGIN_SILENCE:
4905 case ZEND_END_SILENCE:
4906 case ZEND_EXIT:
4907 break;
4908 default:
4909 if (zend_get_user_opcode_handler(i) != NULL) {
4910 zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled.");
4911 JIT_G(enabled) = 0;
4912 JIT_G(on) = 0;
4913 return FAILURE;
4914 }
4915 }
4916 }
4917
4918 return SUCCESS;
4919 }
4920
zend_jit_startup(void * buf,size_t size,bool reattached)4921 ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, bool reattached)
4922 {
4923 int ret;
4924
4925 zend_jit_halt_op = zend_get_halt_op();
4926
4927 if (zend_jit_setup() != SUCCESS) {
4928 // TODO: error reporting and cleanup ???
4929 return FAILURE;
4930 }
4931
4932 zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(ACCELERATOR_PRODUCT_NAME);
4933
4934 #ifdef HAVE_GDB
4935 zend_jit_gdb_init();
4936 #endif
4937
4938 #ifdef HAVE_OPROFILE
4939 if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
4940 if (!zend_jit_oprofile_startup()) {
4941 // TODO: error reporting and cleanup ???
4942 return FAILURE;
4943 }
4944 }
4945 #endif
4946 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
4947 zend_write_protect = pthread_jit_write_protect_supported_np();
4948 #endif
4949
4950 dasm_buf = buf;
4951 dasm_size = size;
4952
4953 #ifdef HAVE_MPROTECT
4954 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
4955 if (zend_write_protect) {
4956 pthread_jit_write_protect_np(1);
4957 }
4958 #endif
4959 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
4960 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
4961 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4962 }
4963 } else {
4964 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
4965 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4966 }
4967 }
4968 #elif _WIN32
4969 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
4970 DWORD old;
4971
4972 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
4973 DWORD err = GetLastError();
4974 char *msg = php_win32_error_to_msg(err);
4975 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4976 php_win32_error_msg_free(msg);
4977 }
4978 } else {
4979 DWORD old;
4980
4981 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
4982 DWORD err = GetLastError();
4983 char *msg = php_win32_error_to_msg(err);
4984 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4985 php_win32_error_msg_free(msg);
4986 }
4987 }
4988 #endif
4989
4990 dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr) * 2);
4991 if (!reattached) {
4992 zend_jit_unprotect();
4993 *dasm_ptr = dasm_buf;
4994 #if _WIN32
4995 /* reserve space for global labels */
4996 *dasm_ptr = (void**)*dasm_ptr + zend_lb_MAX;
4997 #endif
4998 zend_jit_protect();
4999 }
5000
5001 #ifdef HAVE_DISASM
5002 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
5003 if (!zend_jit_disasm_init()) {
5004 // TODO: error reporting and cleanup ???
5005 return FAILURE;
5006 }
5007 }
5008 #endif
5009
5010 #ifdef HAVE_PERFTOOLS
5011 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
5012 zend_jit_perf_jitdump_open();
5013 }
5014 #endif
5015
5016 if (!reattached) {
5017 zend_jit_unprotect();
5018 ret = zend_jit_make_stubs();
5019 #if _WIN32
5020 /* save global labels */
5021 memcpy(dasm_buf, dasm_labels, sizeof(void*) * zend_lb_MAX);
5022 #endif
5023 zend_jit_protect();
5024 if (!ret) {
5025 // TODO: error reporting and cleanup ???
5026 return FAILURE;
5027 }
5028 } else {
5029 #if _WIN32
5030 /* restore global labels */
5031 memcpy(dasm_labels, dasm_buf, sizeof(void*) * zend_lb_MAX);
5032 zend_jit_init_handlers();
5033 #endif
5034 }
5035
5036 if (zend_jit_trace_startup(reattached) != SUCCESS) {
5037 return FAILURE;
5038 }
5039
5040 zend_jit_unprotect();
5041 #if ZEND_JIT_TARGET_ARM64
5042 /* reserve space for global labels veneers */
5043 dasm_labels_veneers = *dasm_ptr;
5044 *dasm_ptr = (void**)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT);
5045 memset(dasm_labels_veneers, 0, sizeof(void*) * ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT));
5046 #endif
5047 /* save JIT buffer pos */
5048 dasm_ptr[1] = dasm_ptr[0];
5049 zend_jit_protect();
5050
5051 return SUCCESS;
5052 }
5053
zend_jit_shutdown(void)5054 ZEND_EXT_API void zend_jit_shutdown(void)
5055 {
5056 if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE && dasm_ptr != NULL) {
5057 fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf));
5058 }
5059
5060 #ifdef HAVE_OPROFILE
5061 if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
5062 zend_jit_oprofile_shutdown();
5063 }
5064 #endif
5065
5066 #ifdef HAVE_GDB
5067 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
5068 zend_jit_gdb_unregister();
5069 }
5070 #endif
5071
5072 #ifdef HAVE_DISASM
5073 zend_jit_disasm_shutdown();
5074 #endif
5075
5076 #ifdef HAVE_PERFTOOLS
5077 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
5078 zend_jit_perf_jitdump_close();
5079 }
5080 #endif
5081 #ifdef ZTS
5082 ts_free_id(jit_globals_id);
5083 #else
5084 zend_jit_trace_free_caches(&jit_globals);
5085 #endif
5086 }
5087
zend_jit_reset_counters(void)5088 static void zend_jit_reset_counters(void)
5089 {
5090 int i;
5091
5092 for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
5093 zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT;
5094 }
5095 }
5096
zend_jit_activate(void)5097 ZEND_EXT_API void zend_jit_activate(void)
5098 {
5099 zend_jit_profile_counter = 0;
5100 if (JIT_G(on)) {
5101 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
5102 zend_jit_reset_counters();
5103 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
5104 zend_jit_reset_counters();
5105 zend_jit_trace_reset_caches();
5106 }
5107 }
5108 }
5109
zend_jit_deactivate(void)5110 ZEND_EXT_API void zend_jit_deactivate(void)
5111 {
5112 if (zend_jit_profile_counter && !CG(unclean_shutdown)) {
5113 zend_class_entry *ce;
5114
5115 zend_shared_alloc_lock();
5116 SHM_UNPROTECT();
5117 zend_jit_unprotect();
5118
5119 zend_jit_check_funcs(EG(function_table), 0);
5120 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(EG(class_table), ce) {
5121 if (ce->type == ZEND_INTERNAL_CLASS) {
5122 break;
5123 }
5124 zend_jit_check_funcs(&ce->function_table, 1);
5125 } ZEND_HASH_FOREACH_END();
5126
5127 zend_jit_protect();
5128 SHM_PROTECT();
5129 zend_shared_alloc_unlock();
5130 }
5131
5132 zend_jit_profile_counter = 0;
5133 }
5134
zend_jit_restart_preloaded_op_array(zend_op_array * op_array)5135 static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array)
5136 {
5137 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
5138
5139 if (!func_info) {
5140 return;
5141 }
5142
5143 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
5144 zend_jit_restart_hot_trace_counters(op_array);
5145 } else if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_COUNTERS) {
5146 zend_jit_restart_hot_counters(op_array);
5147 #if 0
5148 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ???
5149 } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) {
5150 zend_op *opline = op_array->opcodes;
5151 zend_jit_op_array_extension *jit_extension =
5152 (zend_jit_op_array_extension*)func_info;
5153
5154 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
5155 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
5156 opline++;
5157 }
5158 }
5159 if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) {
5160 opline->handler = (const void*)zend_jit_runtime_jit_handler;
5161 } else {
5162 opline->handler = (const void*)zend_jit_profile_jit_handler;
5163 }
5164 #endif
5165 }
5166 if (op_array->num_dynamic_func_defs) {
5167 for (uint32_t i = 0; i < op_array->num_dynamic_func_defs; i++) {
5168 zend_jit_restart_preloaded_op_array(op_array->dynamic_func_defs[i]);
5169 }
5170 }
5171 }
5172
zend_jit_restart_preloaded_script(zend_persistent_script * script)5173 static void zend_jit_restart_preloaded_script(zend_persistent_script *script)
5174 {
5175 zend_class_entry *ce;
5176 zend_op_array *op_array;
5177
5178 zend_jit_restart_preloaded_op_array(&script->script.main_op_array);
5179
5180 ZEND_HASH_MAP_FOREACH_PTR(&script->script.function_table, op_array) {
5181 zend_jit_restart_preloaded_op_array(op_array);
5182 } ZEND_HASH_FOREACH_END();
5183
5184 ZEND_HASH_MAP_FOREACH_PTR(&script->script.class_table, ce) {
5185 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
5186 if (op_array->type == ZEND_USER_FUNCTION) {
5187 zend_jit_restart_preloaded_op_array(op_array);
5188 }
5189 } ZEND_HASH_FOREACH_END();
5190 } ZEND_HASH_FOREACH_END();
5191 }
5192
zend_jit_restart(void)5193 ZEND_EXT_API void zend_jit_restart(void)
5194 {
5195 if (dasm_buf) {
5196 zend_jit_unprotect();
5197
5198 #if ZEND_JIT_TARGET_ARM64
5199 memset(dasm_labels_veneers, 0, sizeof(void*) * ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT));
5200 #endif
5201
5202 /* restore JIT buffer pos */
5203 dasm_ptr[0] = dasm_ptr[1];
5204
5205 zend_jit_trace_restart();
5206
5207 if (ZCSG(preload_script)) {
5208 zend_jit_restart_preloaded_script(ZCSG(preload_script));
5209 if (ZCSG(saved_scripts)) {
5210 zend_persistent_script **p = ZCSG(saved_scripts);
5211
5212 while (*p) {
5213 zend_jit_restart_preloaded_script(*p);
5214 p++;
5215 }
5216 }
5217 }
5218
5219 zend_jit_protect();
5220
5221 #ifdef HAVE_DISASM
5222 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
5223 zend_jit_disasm_shutdown();
5224 zend_jit_disasm_init();
5225 }
5226 #endif
5227 }
5228 }
5229
5230 #endif /* HAVE_JIT */
5231
