1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 #include "main/php.h"
20 #include "main/SAPI.h"
21 #include "php_version.h"
22 #include <ZendAccelerator.h>
23 #include "zend_shared_alloc.h"
24 #include "Zend/zend_execute.h"
25 #include "Zend/zend_vm.h"
26 #include "Zend/zend_exceptions.h"
27 #include "Zend/zend_constants.h"
28 #include "Zend/zend_closures.h"
29 #include "Zend/zend_ini.h"
30 #include "Zend/zend_observer.h"
31 #include "zend_smart_str.h"
32 #include "jit/zend_jit.h"
33
34 #ifdef HAVE_JIT
35
36 #include "Optimizer/zend_func_info.h"
37 #include "Optimizer/zend_ssa.h"
38 #include "Optimizer/zend_inference.h"
39 #include "Optimizer/zend_call_graph.h"
40 #include "Optimizer/zend_dump.h"
41
42 #if ZEND_JIT_TARGET_X86
43 # include "jit/zend_jit_x86.h"
44 #elif ZEND_JIT_TARGET_ARM64
45 # include "jit/zend_jit_arm64.h"
46 #endif
47
48 #include "jit/zend_jit_internal.h"
49
50 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
51 #include <pthread.h>
52 #endif
53
54 #ifdef ZTS
55 int jit_globals_id;
56 #else
57 zend_jit_globals jit_globals;
58 #endif
59
60 //#define CONTEXT_THREADED_JIT
61 #define ZEND_JIT_USE_RC_INFERENCE
62
63 #ifdef ZEND_JIT_USE_RC_INFERENCE
64 # define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE
65 # define RC_MAY_BE_1(info) (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0)
66 # define RC_MAY_BE_N(info) (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0)
67 #else
68 # define ZEND_SSA_RC_INFERENCE_FLAG 0
69 # define RC_MAY_BE_1(info) 1
70 # define RC_MAY_BE_N(info) 1
71 #endif
72
73 #define JIT_PREFIX "JIT$"
74 #define JIT_STUB_PREFIX "JIT$$"
75 #define TRACE_PREFIX "TRACE-"
76
77 #define DASM_M_GROW(ctx, t, p, sz, need) \
78 do { \
79 size_t _sz = (sz), _need = (need); \
80 if (_sz < _need) { \
81 if (_sz < 16) _sz = 16; \
82 while (_sz < _need) _sz += _sz; \
83 (p) = (t *)erealloc((p), _sz); \
84 (sz) = _sz; \
85 } \
86 } while(0)
87
88 #define DASM_M_FREE(ctx, p, sz) efree(p)
89
90 #if ZEND_DEBUG
91 # define DASM_CHECKS 1
92 #endif
93
94 #include "dynasm/dasm_proto.h"
95
96 typedef struct _zend_jit_stub {
97 const char *name;
98 int (*stub)(dasm_State **Dst);
99 uint32_t offset;
100 uint32_t adjustment;
101 } zend_jit_stub;
102
103 #define JIT_STUB(name, offset, adjustment) \
104 {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub, offset, adjustment}
105
106 zend_ulong zend_jit_profile_counter = 0;
107 int zend_jit_profile_counter_rid = -1;
108
109 int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
110
111 const zend_op *zend_jit_halt_op = NULL;
112 static int zend_jit_vm_kind = 0;
113 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
114 static int zend_write_protect = 1;
115 #endif
116
117 static void *dasm_buf = NULL;
118 static void *dasm_end = NULL;
119 static void **dasm_ptr = NULL;
120
121 static size_t dasm_size = 0;
122
123 static zend_long jit_bisect_pos = 0;
124
125 static const void *zend_jit_runtime_jit_handler = NULL;
126 static const void *zend_jit_profile_jit_handler = NULL;
127 static const void *zend_jit_func_hot_counter_handler = NULL;
128 static const void *zend_jit_loop_hot_counter_handler = NULL;
129 static const void *zend_jit_func_trace_counter_handler = NULL;
130 static const void *zend_jit_ret_trace_counter_handler = NULL;
131 static const void *zend_jit_loop_trace_counter_handler = NULL;
132
133 static int ZEND_FASTCALL zend_runtime_jit(void);
134
135 static int zend_jit_trace_op_len(const zend_op *opline);
136 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
137 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags);
138 static const void *zend_jit_trace_get_exit_addr(uint32_t n);
139 static void zend_jit_trace_add_code(const void *start, uint32_t size);
140 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info);
141
142 #if ZEND_JIT_TARGET_ARM64
143 static zend_jit_trace_info *zend_jit_get_current_trace_info(void);
144 static uint32_t zend_jit_trace_find_exit_point(const void* addr);
145 #endif
146
147 #if ZEND_JIT_TARGET_X86 && defined(__linux__)
148 # if PHP_HAVE_BUILTIN_CPU_SUPPORTS && defined(__GNUC__) && (ZEND_GCC_VERSION >= 11000)
149 # define ZEND_JIT_SUPPORT_CLDEMOTE 1
150 # else
151 # define ZEND_JIT_SUPPORT_CLDEMOTE 0
152 # endif
153 #endif
154
155 #if ZEND_JIT_SUPPORT_CLDEMOTE
156 #include <immintrin.h>
157 #pragma GCC push_options
158 #pragma GCC target("cldemote")
159 // check cldemote by CPUID when JIT startup
160 static int cpu_support_cldemote = 0;
shared_cacheline_demote(uintptr_t start,size_t size)161 static inline void shared_cacheline_demote(uintptr_t start, size_t size) {
162 uintptr_t cache_line_base = start & ~0x3F;
163 do {
164 _cldemote((void *)cache_line_base);
165 // next cacheline start size
166 cache_line_base += 64;
167 } while (cache_line_base < start + size);
168 }
169 #pragma GCC pop_options
170 #endif
171
172 static int zend_jit_assign_to_variable(dasm_State **Dst,
173 const zend_op *opline,
174 zend_jit_addr var_use_addr,
175 zend_jit_addr var_addr,
176 uint32_t var_info,
177 uint32_t var_def_info,
178 uint8_t val_type,
179 zend_jit_addr val_addr,
180 uint32_t val_info,
181 zend_jit_addr res_addr,
182 bool check_exception);
183
dominates(const zend_basic_block * blocks,int a,int b)184 static bool dominates(const zend_basic_block *blocks, int a, int b) {
185 while (blocks[b].level > blocks[a].level) {
186 b = blocks[b].idom;
187 }
188 return a == b;
189 }
190
zend_ssa_is_last_use(const zend_op_array * op_array,const zend_ssa * ssa,int var,int use)191 static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
192 {
193 int next_use;
194
195 if (ssa->vars[var].phi_use_chain) {
196 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
197 do {
198 if (!ssa->vars[phi->ssa_var].no_val) {
199 return 0;
200 }
201 phi = zend_ssa_next_use_phi(ssa, var, phi);
202 } while (phi);
203 }
204
205 if (ssa->cfg.blocks[ssa->cfg.map[use]].loop_header > 0
206 || (ssa->cfg.blocks[ssa->cfg.map[use]].flags & ZEND_BB_LOOP_HEADER)) {
207 int b = ssa->cfg.map[use];
208 int prev_use = ssa->vars[var].use_chain;
209
210 while (prev_use >= 0 && prev_use != use) {
211 if (b != ssa->cfg.map[prev_use]
212 && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use])
213 && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) {
214 return 0;
215 }
216 prev_use = zend_ssa_next_use(ssa->ops, var, prev_use);
217 }
218 }
219
220 next_use = zend_ssa_next_use(ssa->ops, var, use);
221 if (next_use < 0) {
222 return 1;
223 } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) {
224 return 1;
225 }
226 return 0;
227 }
228
zend_ival_is_last_use(const zend_lifetime_interval * ival,int use)229 static bool zend_ival_is_last_use(const zend_lifetime_interval *ival, int use)
230 {
231 if (ival->flags & ZREG_LAST_USE) {
232 const zend_life_range *range = &ival->range;
233
234 while (range->next) {
235 range = range->next;
236 }
237 return range->end == use;
238 }
239 return 0;
240 }
241
zend_is_commutative(uint8_t opcode)242 static bool zend_is_commutative(uint8_t opcode)
243 {
244 return
245 opcode == ZEND_ADD ||
246 opcode == ZEND_MUL ||
247 opcode == ZEND_BW_OR ||
248 opcode == ZEND_BW_AND ||
249 opcode == ZEND_BW_XOR;
250 }
251
zend_jit_is_constant_cmp_long_long(const zend_op * opline,zend_ssa_range * op1_range,zend_jit_addr op1_addr,zend_ssa_range * op2_range,zend_jit_addr op2_addr,bool * result)252 static int zend_jit_is_constant_cmp_long_long(const zend_op *opline,
253 zend_ssa_range *op1_range,
254 zend_jit_addr op1_addr,
255 zend_ssa_range *op2_range,
256 zend_jit_addr op2_addr,
257 bool *result)
258 {
259 zend_long op1_min;
260 zend_long op1_max;
261 zend_long op2_min;
262 zend_long op2_max;
263
264 if (op1_range) {
265 op1_min = op1_range->min;
266 op1_max = op1_range->max;
267 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
268 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG);
269 op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr));
270 } else {
271 return 0;
272 }
273
274 if (op2_range) {
275 op2_min = op2_range->min;
276 op2_max = op2_range->max;
277 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
278 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG);
279 op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr));
280 } else {
281 return 0;
282 }
283
284 switch (opline->opcode) {
285 case ZEND_IS_EQUAL:
286 case ZEND_IS_IDENTICAL:
287 case ZEND_CASE:
288 case ZEND_CASE_STRICT:
289 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
290 *result = 1;
291 return 1;
292 } else if (op1_max < op2_min || op1_min > op2_max) {
293 *result = 0;
294 return 1;
295 }
296 return 0;
297 case ZEND_IS_NOT_EQUAL:
298 case ZEND_IS_NOT_IDENTICAL:
299 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
300 *result = 0;
301 return 1;
302 } else if (op1_max < op2_min || op1_min > op2_max) {
303 *result = 1;
304 return 1;
305 }
306 return 0;
307 case ZEND_IS_SMALLER:
308 if (op1_max < op2_min) {
309 *result = 1;
310 return 1;
311 } else if (op1_min >= op2_max) {
312 *result = 0;
313 return 1;
314 }
315 return 0;
316 case ZEND_IS_SMALLER_OR_EQUAL:
317 if (op1_max <= op2_min) {
318 *result = 1;
319 return 1;
320 } else if (op1_min > op2_max) {
321 *result = 0;
322 return 1;
323 }
324 return 0;
325 default:
326 ZEND_UNREACHABLE();
327 }
328 return 0;
329 }
330
zend_jit_needs_call_chain(zend_call_info * call_info,uint32_t b,const zend_op_array * op_array,zend_ssa * ssa,const zend_ssa_op * ssa_op,const zend_op * opline,int call_level,zend_jit_trace_rec * trace)331 static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace)
332 {
333 int skip;
334
335 if (trace) {
336 zend_jit_trace_rec *p = trace;
337
338 ssa_op++;
339 while (1) {
340 if (p->op == ZEND_JIT_TRACE_VM) {
341 switch (p->opline->opcode) {
342 case ZEND_SEND_ARRAY:
343 case ZEND_SEND_USER:
344 case ZEND_SEND_UNPACK:
345 case ZEND_INIT_FCALL:
346 case ZEND_INIT_METHOD_CALL:
347 case ZEND_INIT_STATIC_METHOD_CALL:
348 case ZEND_INIT_FCALL_BY_NAME:
349 case ZEND_INIT_NS_FCALL_BY_NAME:
350 case ZEND_INIT_DYNAMIC_CALL:
351 case ZEND_NEW:
352 case ZEND_INIT_USER_CALL:
353 case ZEND_FAST_CALL:
354 case ZEND_JMP:
355 case ZEND_JMPZ:
356 case ZEND_JMPNZ:
357 case ZEND_JMPZ_EX:
358 case ZEND_JMPNZ_EX:
359 case ZEND_FE_RESET_R:
360 case ZEND_FE_RESET_RW:
361 case ZEND_JMP_SET:
362 case ZEND_COALESCE:
363 case ZEND_JMP_NULL:
364 case ZEND_ASSERT_CHECK:
365 case ZEND_CATCH:
366 case ZEND_DECLARE_ANON_CLASS:
367 case ZEND_FE_FETCH_R:
368 case ZEND_FE_FETCH_RW:
369 case ZEND_BIND_INIT_STATIC_OR_JMP:
370 return 1;
371 case ZEND_DO_ICALL:
372 case ZEND_DO_UCALL:
373 case ZEND_DO_FCALL_BY_NAME:
374 case ZEND_DO_FCALL:
375 case ZEND_CALLABLE_CONVERT:
376 return 0;
377 case ZEND_SEND_VAL:
378 case ZEND_SEND_VAR:
379 case ZEND_SEND_VAL_EX:
380 case ZEND_SEND_VAR_EX:
381 case ZEND_SEND_FUNC_ARG:
382 case ZEND_SEND_REF:
383 case ZEND_SEND_VAR_NO_REF:
384 case ZEND_SEND_VAR_NO_REF_EX:
385 /* skip */
386 break;
387 default:
388 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
389 return 1;
390 }
391 }
392 ssa_op += zend_jit_trace_op_len(opline);
393 } else if (p->op == ZEND_JIT_TRACE_ENTER ||
394 p->op == ZEND_JIT_TRACE_BACK ||
395 p->op == ZEND_JIT_TRACE_END) {
396 return 1;
397 }
398 p++;
399 }
400 }
401
402 if (!call_info) {
403 const zend_op *end = op_array->opcodes + op_array->last;
404
405 opline++;
406 ssa_op++;
407 skip = (call_level == 1);
408 while (opline != end) {
409 if (!skip) {
410 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
411 return 1;
412 }
413 }
414 switch (opline->opcode) {
415 case ZEND_SEND_VAL:
416 case ZEND_SEND_VAR:
417 case ZEND_SEND_VAL_EX:
418 case ZEND_SEND_VAR_EX:
419 case ZEND_SEND_FUNC_ARG:
420 case ZEND_SEND_REF:
421 case ZEND_SEND_VAR_NO_REF:
422 case ZEND_SEND_VAR_NO_REF_EX:
423 skip = 0;
424 break;
425 case ZEND_SEND_ARRAY:
426 case ZEND_SEND_USER:
427 case ZEND_SEND_UNPACK:
428 case ZEND_INIT_FCALL:
429 case ZEND_INIT_METHOD_CALL:
430 case ZEND_INIT_STATIC_METHOD_CALL:
431 case ZEND_INIT_FCALL_BY_NAME:
432 case ZEND_INIT_NS_FCALL_BY_NAME:
433 case ZEND_INIT_DYNAMIC_CALL:
434 case ZEND_NEW:
435 case ZEND_INIT_USER_CALL:
436 case ZEND_FAST_CALL:
437 case ZEND_JMP:
438 case ZEND_JMPZ:
439 case ZEND_JMPNZ:
440 case ZEND_JMPZ_EX:
441 case ZEND_JMPNZ_EX:
442 case ZEND_FE_RESET_R:
443 case ZEND_FE_RESET_RW:
444 case ZEND_JMP_SET:
445 case ZEND_COALESCE:
446 case ZEND_JMP_NULL:
447 case ZEND_ASSERT_CHECK:
448 case ZEND_CATCH:
449 case ZEND_DECLARE_ANON_CLASS:
450 case ZEND_FE_FETCH_R:
451 case ZEND_FE_FETCH_RW:
452 case ZEND_BIND_INIT_STATIC_OR_JMP:
453 return 1;
454 case ZEND_DO_ICALL:
455 case ZEND_DO_UCALL:
456 case ZEND_DO_FCALL_BY_NAME:
457 case ZEND_DO_FCALL:
458 case ZEND_CALLABLE_CONVERT:
459 end = opline;
460 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
461 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
462 return 1;
463 }
464 return 0;
465 }
466 opline++;
467 ssa_op++;
468 }
469
470 return 1;
471 } else {
472 const zend_op *end = call_info->caller_call_opline;
473
474 /* end may be null if an opcode like EXIT is part of the argument list. */
475 if (!end || end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
476 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
477 return 1;
478 }
479
480 opline++;
481 ssa_op++;
482 skip = (call_level == 1);
483 while (opline != end) {
484 if (skip) {
485 switch (opline->opcode) {
486 case ZEND_SEND_VAL:
487 case ZEND_SEND_VAR:
488 case ZEND_SEND_VAL_EX:
489 case ZEND_SEND_VAR_EX:
490 case ZEND_SEND_FUNC_ARG:
491 case ZEND_SEND_REF:
492 case ZEND_SEND_VAR_NO_REF:
493 case ZEND_SEND_VAR_NO_REF_EX:
494 skip = 0;
495 break;
496 case ZEND_SEND_ARRAY:
497 case ZEND_SEND_USER:
498 case ZEND_SEND_UNPACK:
499 return 1;
500 }
501 } else {
502 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
503 return 1;
504 }
505 }
506 opline++;
507 ssa_op++;
508 }
509
510 return 0;
511 }
512 }
513
skip_valid_arguments(const zend_op_array * op_array,zend_ssa * ssa,const zend_call_info * call_info)514 static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
515 {
516 uint32_t num_args = 0;
517 zend_function *func = call_info->callee_func;
518
519 /* It's okay to handle prototypes here, because they can only increase the accepted arguments.
520 * Anything legal for the parent method is also legal for the parent method. */
521 while (num_args < call_info->num_args) {
522 zend_arg_info *arg_info = func->op_array.arg_info + num_args;
523
524 if (ZEND_TYPE_IS_SET(arg_info->type)) {
525 if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) {
526 zend_op *opline = call_info->arg_info[num_args].opline;
527 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
528 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
529 if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) {
530 break;
531 }
532 } else {
533 break;
534 }
535 }
536 num_args++;
537 }
538 return num_args;
539 }
540
zend_ssa_cv_info(const zend_op_array * op_array,zend_ssa * ssa,uint32_t var)541 static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var)
542 {
543 uint32_t j, info;
544
545 if (ssa->vars && ssa->var_info) {
546 info = ssa->var_info[var].type;
547 for (j = op_array->last_var; j < ssa->vars_count; j++) {
548 if (ssa->vars[j].var == var) {
549 info |= ssa->var_info[j].type;
550 }
551 }
552 } else {
553 info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF |
554 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
555 }
556
557 #ifdef ZEND_JIT_USE_RC_INFERENCE
558 /* Refcount may be increased by RETURN opcode */
559 if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
560 for (j = 0; j < ssa->cfg.blocks_count; j++) {
561 if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
562 ssa->cfg.blocks[j].len > 0) {
563 const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
564
565 if (opline->opcode == ZEND_RETURN) {
566 if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) {
567 info |= MAY_BE_RCN;
568 break;
569 }
570 }
571 }
572 }
573 }
574 #endif
575
576 return info;
577 }
578
zend_jit_may_avoid_refcounting(const zend_op * opline,uint32_t op1_info)579 static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_info)
580 {
581 switch (opline->opcode) {
582 case ZEND_FETCH_OBJ_FUNC_ARG:
583 if (!JIT_G(current_frame) ||
584 !JIT_G(current_frame)->call->func ||
585 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
586 return 0;
587 }
588 /* break missing intentionally */
589 case ZEND_FETCH_OBJ_R:
590 case ZEND_FETCH_OBJ_IS:
591 if ((op1_info & MAY_BE_OBJECT)
592 && opline->op2_type == IS_CONST
593 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
594 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
595 return 1;
596 }
597 break;
598 case ZEND_FETCH_DIM_FUNC_ARG:
599 if (!JIT_G(current_frame) ||
600 !JIT_G(current_frame)->call->func ||
601 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
602 return 0;
603 }
604 /* break missing intentionally */
605 case ZEND_FETCH_DIM_R:
606 case ZEND_FETCH_DIM_IS:
607 return 1;
608 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
609 if (!(opline->extended_value & ZEND_ISEMPTY)) {
610 return 1;
611 }
612 break;
613 }
614 return 0;
615 }
616
zend_jit_is_persistent_constant(zval * key,uint32_t flags)617 static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags)
618 {
619 zval *zv;
620 zend_constant *c = NULL;
621
622 /* null/true/false are resolved during compilation, so don't check for them here. */
623 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
624 if (zv) {
625 c = (zend_constant*)Z_PTR_P(zv);
626 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
627 key++;
628 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
629 if (zv) {
630 c = (zend_constant*)Z_PTR_P(zv);
631 }
632 }
633 return c && (ZEND_CONSTANT_FLAGS(c) & CONST_PERSISTENT);
634 }
635
zend_get_known_property_info(const zend_op_array * op_array,zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)636 static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
637 {
638 zend_property_info *info = NULL;
639
640 if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) ||
641 !ce ||
642 !(ce->ce_flags & ZEND_ACC_LINKED) ||
643 (ce->ce_flags & ZEND_ACC_TRAIT) ||
644 ce->create_object) {
645 return NULL;
646 }
647
648 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
649 if (ce->info.user.filename != filename) {
650 /* class declaration might be changed independently */
651 return NULL;
652 }
653
654 if (ce->parent) {
655 zend_class_entry *parent = ce->parent;
656
657 do {
658 if (parent->type == ZEND_INTERNAL_CLASS) {
659 break;
660 } else if (parent->info.user.filename != filename) {
661 /* some of parents class declarations might be changed independently */
662 /* TODO: this check may be not enough, because even
663 * in the same it's possible to conditionally define
664 * few classes with the same name, and "parent" may
665 * change from request to request.
666 */
667 return NULL;
668 }
669 parent = parent->parent;
670 } while (parent);
671 }
672 }
673
674 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
675 if (info == NULL ||
676 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
677 (info->flags & ZEND_ACC_STATIC)) {
678 return NULL;
679 }
680
681 if (info->flags & ZEND_ACC_PUBLIC) {
682 return info;
683 } else if (on_this) {
684 if (ce == info->ce) {
685 if (ce == op_array->scope) {
686 return info;
687 } else {
688 return NULL;
689 }
690 } else if ((info->flags & ZEND_ACC_PROTECTED)
691 && instanceof_function_slow(ce, info->ce)) {
692 return info;
693 }
694 }
695
696 return NULL;
697 }
698
zend_may_be_dynamic_property(zend_class_entry * ce,zend_string * member,bool on_this,const zend_op_array * op_array)699 static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, const zend_op_array *op_array)
700 {
701 zend_property_info *info;
702
703 if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT) || (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) {
704 return 1;
705 }
706
707 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
708 if (ce->info.user.filename != op_array->filename) {
709 /* class declaration might be changed independently */
710 return 1;
711 }
712 }
713
714 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
715 if (info == NULL ||
716 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
717 (info->flags & ZEND_ACC_STATIC)) {
718 return 1;
719 }
720
721 if (!(info->flags & ZEND_ACC_PUBLIC) &&
722 (!on_this || info->ce != ce)) {
723 return 1;
724 }
725
726 return 0;
727 }
728
729 #define OP_RANGE(ssa_op, opN) \
730 (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
731 ssa->var_info && \
732 (ssa_op)->opN##_use >= 0 && \
733 ssa->var_info[(ssa_op)->opN##_use].has_range) ? \
734 &ssa->var_info[(ssa_op)->opN##_use].range : NULL)
735
736 #define OP1_RANGE() OP_RANGE(ssa_op, op1)
737 #define OP2_RANGE() OP_RANGE(ssa_op, op2)
738 #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)
739
740 #if ZEND_JIT_TARGET_X86
741 # include "dynasm/dasm_x86.h"
742 #elif ZEND_JIT_TARGET_ARM64
743 static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset);
744 # define DASM_ADD_VENEER zend_jit_add_veneer
745 # include "dynasm/dasm_arm64.h"
746 #endif
747
748 #include "jit/zend_jit_helpers.c"
749 #include "jit/zend_jit_disasm.c"
750 #ifndef _WIN32
751 # include "jit/zend_jit_gdb.h"
752 # include "jit/zend_jit_perf_dump.c"
753 #endif
754
755 #include "Zend/zend_cpuinfo.h"
756
757 #ifdef HAVE_VALGRIND
758 # include <valgrind/valgrind.h>
759 #endif
760
761 #ifdef HAVE_GCC_GLOBAL_REGS
762 # define GCC_GLOBAL_REGS 1
763 #else
764 # define GCC_GLOBAL_REGS 0
765 #endif
766
767 /* By default avoid JITing inline handlers if it does not seem profitable due to lack of
768 * type information. Disabling this option allows testing some JIT handlers in the
769 * presence of try/catch blocks, which prevent SSA construction. */
770 #ifndef PROFITABILITY_CHECKS
771 # define PROFITABILITY_CHECKS 1
772 #endif
773
774 #define BP_JIT_IS 6 /* Used for ISSET_ISEMPTY_DIM_OBJ. see BP_VAR_*defines in Zend/zend_compile.h */
775
776 typedef enum _sp_adj_kind {
777 SP_ADJ_NONE,
778 SP_ADJ_RET,
779 SP_ADJ_VM,
780 SP_ADJ_JIT,
781 SP_ADJ_ASSIGN,
782 SP_ADJ_LAST
783 } sp_adj_kind;
784
785 static int sp_adj[SP_ADJ_LAST];
786
787 /* The generated code may contain tautological comparisons, ignore them. */
788 #if defined(__clang__)
789 # pragma clang diagnostic push
790 # pragma clang diagnostic ignored "-Wtautological-compare"
791 # pragma clang diagnostic ignored "-Wstring-compare"
792 #endif
793
794 #if ZEND_JIT_TARGET_X86
795 # include "jit/zend_jit_vtune.c"
796 # include "jit/zend_jit_x86.c"
797 #elif ZEND_JIT_TARGET_ARM64
798 # include "jit/zend_jit_arm64.c"
799 #endif
800
801 #if defined(__clang__)
802 # pragma clang diagnostic pop
803 #endif
804
805 #if _WIN32
806 # include <Windows.h>
807 #else
808 # include <sys/mman.h>
809 # if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
810 # define MAP_ANONYMOUS MAP_ANON
811 # endif
812 #endif
813
zend_jit_status(zval * ret)814 ZEND_EXT_API void zend_jit_status(zval *ret)
815 {
816 zval stats;
817 array_init(&stats);
818 add_assoc_bool(&stats, "enabled", JIT_G(enabled));
819 add_assoc_bool(&stats, "on", JIT_G(on));
820 add_assoc_long(&stats, "kind", JIT_G(trigger));
821 add_assoc_long(&stats, "opt_level", JIT_G(opt_level));
822 add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags));
823 if (dasm_buf) {
824 add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
825 add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
826 } else {
827 add_assoc_long(&stats, "buffer_size", 0);
828 add_assoc_long(&stats, "buffer_free", 0);
829 }
830 add_assoc_zval(ret, "jit", &stats);
831 }
832
zend_jit_func_name(const zend_op_array * op_array)833 static zend_string *zend_jit_func_name(const zend_op_array *op_array)
834 {
835 smart_str buf = {0};
836
837 if (op_array->function_name) {
838 if (op_array->scope) {
839 smart_str_appends(&buf, JIT_PREFIX);
840 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
841 smart_str_appends(&buf, "::");
842 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
843 smart_str_0(&buf);
844 return buf.s;
845 } else {
846 smart_str_appends(&buf, JIT_PREFIX);
847 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
848 smart_str_0(&buf);
849 return buf.s;
850 }
851 } else if (op_array->filename) {
852 smart_str_appends(&buf, JIT_PREFIX);
853 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
854 smart_str_0(&buf);
855 return buf.s;
856 } else {
857 return NULL;
858 }
859 }
860
861 #if ZEND_DEBUG
handle_dasm_error(int ret)862 static void handle_dasm_error(int ret) {
863 switch (ret & 0xff000000u) {
864 case DASM_S_NOMEM:
865 fprintf(stderr, "DASM_S_NOMEM\n");
866 break;
867 case DASM_S_PHASE:
868 fprintf(stderr, "DASM_S_PHASE\n");
869 break;
870 case DASM_S_MATCH_SEC:
871 fprintf(stderr, "DASM_S_MATCH_SEC\n");
872 break;
873 case DASM_S_RANGE_I:
874 fprintf(stderr, "DASM_S_RANGE_I\n");
875 break;
876 case DASM_S_RANGE_SEC:
877 fprintf(stderr, "DASM_S_RANGE_SEC\n");
878 break;
879 case DASM_S_RANGE_LG:
880 fprintf(stderr, "DASM_S_RANGE_LG\n");
881 break;
882 case DASM_S_RANGE_PC:
883 fprintf(stderr, "DASM_S_RANGE_PC %d\n", ret & 0xffffffu);
884 break;
885 #ifdef DASM_S_RANGE_VREG
886 case DASM_S_RANGE_VREG:
887 fprintf(stderr, "DASM_S_RANGE_VREG\n");
888 break;
889 #endif
890 #ifdef DASM_S_UNDEF_L
891 case DASM_S_UNDEF_L:
892 fprintf(stderr, "DASM_S_UNDEF_L\n");
893 break;
894 #endif
895 #ifdef DASM_S_UNDEF_LG
896 case DASM_S_UNDEF_LG:
897 fprintf(stderr, "DASM_S_UNDEF_LG\n");
898 break;
899 #endif
900 #ifdef DASM_S_RANGE_REL
901 case DASM_S_RANGE_REL:
902 fprintf(stderr, "DASM_S_RANGE_REL\n");
903 break;
904 #endif
905 case DASM_S_UNDEF_PC:
906 fprintf(stderr, "DASM_S_UNDEF_PC %d\n", ret & 0xffffffu);
907 break;
908 default:
909 fprintf(stderr, "DASM_S_%0x\n", ret & 0xff000000u);
910 break;
911 }
912 ZEND_UNREACHABLE();
913 }
914 #endif
915
dasm_link_and_encode(dasm_State ** dasm_state,const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline,zend_lifetime_interval ** ra,const char * name,uint32_t trace_num,uint32_t sp_offset,uint32_t sp_adjustment)916 static void *dasm_link_and_encode(dasm_State **dasm_state,
917 const zend_op_array *op_array,
918 zend_ssa *ssa,
919 const zend_op *rt_opline,
920 zend_lifetime_interval **ra,
921 const char *name,
922 uint32_t trace_num,
923 uint32_t sp_offset,
924 uint32_t sp_adjustment)
925 {
926 size_t size;
927 int ret;
928 void *entry;
929 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
930 zend_string *str = NULL;
931 #endif
932
933 if (rt_opline && ssa && ssa->cfg.map) {
934 /* Create additional entry point, to switch from interpreter to JIT-ed
935 * code at run-time.
936 */
937 int b = ssa->cfg.map[rt_opline - op_array->opcodes];
938
939 //#ifdef CONTEXT_THREADED_JIT
940 // if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY))) {
941 //#else
942 if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY))) {
943 //#endif
944 zend_jit_label(dasm_state, ssa->cfg.blocks_count + b);
945 zend_jit_prologue(dasm_state);
946 if (ra) {
947 int i;
948 zend_lifetime_interval *ival;
949 zend_life_range *range;
950 uint32_t pos = rt_opline - op_array->opcodes;
951
952 for (i = 0; i < ssa->vars_count; i++) {
953 ival = ra[i];
954
955 if (ival && ival->reg != ZREG_NONE) {
956 range = &ival->range;
957
958 if (pos >= range->start && pos <= range->end) {
959 if (!zend_jit_load_var(dasm_state, ssa->var_info[i].type, ssa->vars[i].var, ival->reg)) {
960 return NULL;
961 }
962 break;
963 }
964 range = range->next;
965 }
966 }
967 }
968 zend_jit_jmp(dasm_state, b);
969 }
970 }
971
972 ret = dasm_link(dasm_state, &size);
973 if (ret != DASM_S_OK) {
974 #if ZEND_DEBUG
975 handle_dasm_error(ret);
976 #endif
977 return NULL;
978 }
979
980 if ((void*)((char*)*dasm_ptr + size) > dasm_end) {
981 *dasm_ptr = dasm_end; //prevent further try
982 // TODO: jit_buffer_size overflow ???
983 return NULL;
984 }
985
986 #if ZEND_JIT_TARGET_ARM64
987 dasm_venners_size = 0;
988 #endif
989
990 ret = dasm_encode(dasm_state, *dasm_ptr);
991 if (ret != DASM_S_OK) {
992 #if ZEND_DEBUG
993 handle_dasm_error(ret);
994 #endif
995 return NULL;
996 }
997
998 #if ZEND_JIT_TARGET_ARM64
999 size += dasm_venners_size;
1000 #endif
1001
1002 entry = *dasm_ptr;
1003 *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT));
1004
1005 /* flush the hardware I-cache */
1006 JIT_CACHE_FLUSH(entry, entry + size);
1007 /* hint to the hardware to push out the cache line that contains the linear address */
1008 #if ZEND_JIT_SUPPORT_CLDEMOTE
1009 if (cpu_support_cldemote && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
1010 shared_cacheline_demote((uintptr_t)entry, size);
1011 }
1012 #endif
1013
1014 if (trace_num) {
1015 zend_jit_trace_add_code(entry, dasm_getpclabel(dasm_state, 1));
1016 }
1017
1018 if (op_array && ssa) {
1019 int b;
1020
1021 for (b = 0; b < ssa->cfg.blocks_count; b++) {
1022 //#ifdef CONTEXT_THREADED_JIT
1023 // if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
1024 //#else
1025 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) {
1026 //#endif
1027 zend_op *opline = op_array->opcodes + ssa->cfg.blocks[b].start;
1028 int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
1029
1030 if (offset >= 0) {
1031 opline->handler = (void*)(((char*)entry) + offset);
1032 }
1033 }
1034 }
1035 if (rt_opline && ssa && ssa->cfg.map) {
1036 int b = ssa->cfg.map[rt_opline - op_array->opcodes];
1037 zend_op *opline = (zend_op*)rt_opline;
1038 int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
1039
1040 if (offset >= 0) {
1041 opline->handler = (void*)(((char*)entry) + offset);
1042 }
1043 }
1044 }
1045
1046 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
1047 if (!name) {
1048 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) {
1049 str = zend_jit_func_name(op_array);
1050 if (str) {
1051 name = ZSTR_VAL(str);
1052 }
1053 }
1054 #ifdef HAVE_DISASM
1055 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
1056 zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
1057 zend_jit_disasm(
1058 name,
1059 (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
1060 op_array,
1061 &ssa->cfg,
1062 entry,
1063 size);
1064 }
1065 } else {
1066 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) {
1067 zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
1068 if ((JIT_G(debug) & (trace_num ? ZEND_JIT_DEBUG_ASM : ZEND_JIT_DEBUG_ASM_STUBS)) != 0) {
1069 zend_jit_disasm(
1070 name,
1071 (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
1072 op_array,
1073 ssa ? &ssa->cfg : NULL,
1074 entry,
1075 size);
1076 }
1077 }
1078 # endif
1079 }
1080 #endif
1081
1082 #ifdef HAVE_GDB
1083 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
1084 if (name) {
1085 zend_jit_gdb_register(
1086 name,
1087 op_array,
1088 entry,
1089 size,
1090 sp_adj[sp_offset],
1091 sp_adj[sp_adjustment]);
1092 }
1093 }
1094 #endif
1095
1096
1097 #ifdef HAVE_PERFTOOLS
1098 if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
1099 if (name) {
1100 zend_jit_perf_map_register(
1101 name,
1102 entry,
1103 size);
1104 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
1105 zend_jit_perf_jitdump_register(
1106 name,
1107 entry,
1108 size);
1109 }
1110 }
1111 }
1112 #endif
1113
1114 #ifdef HAVE_VTUNE
1115 if (JIT_G(debug) & ZEND_JIT_DEBUG_VTUNE) {
1116 if (name) {
1117 zend_jit_vtune_register(
1118 name,
1119 entry,
1120 size);
1121 }
1122 }
1123 #endif
1124
1125 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
1126 if (str) {
1127 zend_string_release(str);
1128 }
1129 #endif
1130
1131 return entry;
1132 }
1133
zend_may_overflow(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa)1134 static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
1135 {
1136 int res;
1137 zend_long op1_min, op1_max, op2_min, op2_max;
1138
1139 if (!ssa->ops || !ssa->var_info) {
1140 return 1;
1141 }
1142 switch (opline->opcode) {
1143 case ZEND_PRE_INC:
1144 case ZEND_POST_INC:
1145 res = ssa_op->op1_def;
1146 if (res < 0
1147 || !ssa->var_info[res].has_range
1148 || ssa->var_info[res].range.overflow) {
1149 if (!OP1_HAS_RANGE()) {
1150 return 1;
1151 }
1152 op1_max = OP1_MAX_RANGE();
1153 if (op1_max == ZEND_LONG_MAX) {
1154 return 1;
1155 }
1156 }
1157 return 0;
1158 case ZEND_PRE_DEC:
1159 case ZEND_POST_DEC:
1160 res = ssa_op->op1_def;
1161 if (res < 0
1162 || !ssa->var_info[res].has_range
1163 || ssa->var_info[res].range.underflow) {
1164 if (!OP1_HAS_RANGE()) {
1165 return 1;
1166 }
1167 op1_min = OP1_MIN_RANGE();
1168 if (op1_min == ZEND_LONG_MIN) {
1169 return 1;
1170 }
1171 }
1172 return 0;
1173 case ZEND_ADD:
1174 res = ssa_op->result_def;
1175 if (res < 0
1176 || !ssa->var_info[res].has_range
1177 || ssa->var_info[res].range.underflow) {
1178 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1179 return 1;
1180 }
1181 op1_min = OP1_MIN_RANGE();
1182 op2_min = OP2_MIN_RANGE();
1183 if (zend_add_will_overflow(op1_min, op2_min)) {
1184 return 1;
1185 }
1186 }
1187 if (res < 0
1188 || !ssa->var_info[res].has_range
1189 || ssa->var_info[res].range.overflow) {
1190 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1191 return 1;
1192 }
1193 op1_max = OP1_MAX_RANGE();
1194 op2_max = OP2_MAX_RANGE();
1195 if (zend_add_will_overflow(op1_max, op2_max)) {
1196 return 1;
1197 }
1198 }
1199 return 0;
1200 case ZEND_SUB:
1201 res = ssa_op->result_def;
1202 if (res < 0
1203 || !ssa->var_info[res].has_range
1204 || ssa->var_info[res].range.underflow) {
1205 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1206 return 1;
1207 }
1208 op1_min = OP1_MIN_RANGE();
1209 op2_max = OP2_MAX_RANGE();
1210 if (zend_sub_will_overflow(op1_min, op2_max)) {
1211 return 1;
1212 }
1213 }
1214 if (res < 0
1215 || !ssa->var_info[res].has_range
1216 || ssa->var_info[res].range.overflow) {
1217 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1218 return 1;
1219 }
1220 op1_max = OP1_MAX_RANGE();
1221 op2_min = OP2_MIN_RANGE();
1222 if (zend_sub_will_overflow(op1_max, op2_min)) {
1223 return 1;
1224 }
1225 }
1226 return 0;
1227 case ZEND_MUL:
1228 res = ssa_op->result_def;
1229 return (res < 0 ||
1230 !ssa->var_info[res].has_range ||
1231 ssa->var_info[res].range.underflow ||
1232 ssa->var_info[res].range.overflow);
1233 case ZEND_ASSIGN_OP:
1234 if (opline->extended_value == ZEND_ADD) {
1235 res = ssa_op->op1_def;
1236 if (res < 0
1237 || !ssa->var_info[res].has_range
1238 || ssa->var_info[res].range.underflow) {
1239 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1240 return 1;
1241 }
1242 op1_min = OP1_MIN_RANGE();
1243 op2_min = OP2_MIN_RANGE();
1244 if (zend_add_will_overflow(op1_min, op2_min)) {
1245 return 1;
1246 }
1247 }
1248 if (res < 0
1249 || !ssa->var_info[res].has_range
1250 || ssa->var_info[res].range.overflow) {
1251 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1252 return 1;
1253 }
1254 op1_max = OP1_MAX_RANGE();
1255 op2_max = OP2_MAX_RANGE();
1256 if (zend_add_will_overflow(op1_max, op2_max)) {
1257 return 1;
1258 }
1259 }
1260 return 0;
1261 } else if (opline->extended_value == ZEND_SUB) {
1262 res = ssa_op->op1_def;
1263 if (res < 0
1264 || !ssa->var_info[res].has_range
1265 || ssa->var_info[res].range.underflow) {
1266 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1267 return 1;
1268 }
1269 op1_min = OP1_MIN_RANGE();
1270 op2_max = OP2_MAX_RANGE();
1271 if (zend_sub_will_overflow(op1_min, op2_max)) {
1272 return 1;
1273 }
1274 }
1275 if (res < 0
1276 || !ssa->var_info[res].has_range
1277 || ssa->var_info[res].range.overflow) {
1278 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1279 return 1;
1280 }
1281 op1_max = OP1_MAX_RANGE();
1282 op2_min = OP2_MIN_RANGE();
1283 if (zend_sub_will_overflow(op1_max, op2_min)) {
1284 return 1;
1285 }
1286 }
1287 return 0;
1288 } else if (opline->extended_value == ZEND_MUL) {
1289 res = ssa_op->op1_def;
1290 return (res < 0 ||
1291 !ssa->var_info[res].has_range ||
1292 ssa->var_info[res].range.underflow ||
1293 ssa->var_info[res].range.overflow);
1294 }
1295 ZEND_FALLTHROUGH;
1296 default:
1297 return 1;
1298 }
1299 }
1300
zend_jit_build_cfg(const zend_op_array * op_array,zend_cfg * cfg)1301 static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg)
1302 {
1303 uint32_t flags;
1304
1305 flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY;
1306
1307 zend_build_cfg(&CG(arena), op_array, flags, cfg);
1308
1309 /* Don't JIT huge functions. Apart from likely being detrimental due to the amount of
1310 * generated code, some of our analysis is recursive and will stack overflow with many
1311 * blocks. */
1312 if (cfg->blocks_count > 100000) {
1313 return FAILURE;
1314 }
1315
1316 zend_cfg_build_predecessors(&CG(arena), cfg);
1317
1318 /* Compute Dominators Tree */
1319 zend_cfg_compute_dominators_tree(op_array, cfg);
1320
1321 /* Identify reducible and irreducible loops */
1322 zend_cfg_identify_loops(op_array, cfg);
1323
1324 return SUCCESS;
1325 }
1326
zend_jit_op_array_analyze1(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa)1327 static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
1328 {
1329 if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) {
1330 return FAILURE;
1331 }
1332
1333 #if 0
1334 /* TODO: debugger and profiler supports? */
1335 if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) {
1336 return FAILURE;
1337 }
1338 #endif
1339
1340 /* TODO: move this to zend_cfg.c ? */
1341 if (!op_array->function_name) {
1342 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1343 }
1344
1345 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
1346 && ssa->cfg.blocks
1347 && op_array->last_try_catch == 0
1348 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
1349 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
1350 if (zend_build_ssa(&CG(arena), script, op_array, ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) {
1351 return FAILURE;
1352 }
1353
1354 zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa);
1355
1356 zend_ssa_find_false_dependencies(op_array, ssa);
1357
1358 zend_ssa_find_sccs(op_array, ssa);
1359 }
1360
1361 return SUCCESS;
1362 }
1363
zend_jit_op_array_analyze2(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa,uint32_t optimization_level)1364 static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t optimization_level)
1365 {
1366 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
1367 && ssa->cfg.blocks
1368 && op_array->last_try_catch == 0
1369 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
1370 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
1371 if (zend_ssa_inference(&CG(arena), op_array, script, ssa,
1372 optimization_level & ~ZEND_OPTIMIZER_NARROW_TO_DOUBLE) != SUCCESS) {
1373 return FAILURE;
1374 }
1375 }
1376
1377 return SUCCESS;
1378 }
1379
zend_jit_add_range(zend_lifetime_interval ** intervals,int var,uint32_t from,uint32_t to)1380 static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint32_t from, uint32_t to)
1381 {
1382 zend_lifetime_interval *ival = intervals[var];
1383
1384 if (!ival) {
1385 ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
1386 if (!ival) {
1387 return FAILURE;
1388 }
1389 ival->ssa_var = var;
1390 ival->reg = ZREG_NONE;
1391 ival->flags = 0;
1392 ival->range.start = from;
1393 ival->range.end = to;
1394 ival->range.next = NULL;
1395 ival->hint = NULL;
1396 ival->used_as_hint = NULL;
1397 intervals[var] = ival;
1398 } else if (ival->range.start > to + 1) {
1399 zend_life_range *range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1400
1401 if (!range) {
1402 return FAILURE;
1403 }
1404 range->start = ival->range.start;
1405 range->end = ival->range.end;
1406 range->next = ival->range.next;
1407 ival->range.start = from;
1408 ival->range.end = to;
1409 ival->range.next = range;
1410 } else if (ival->range.start == to + 1) {
1411 ival->range.start = from;
1412 } else {
1413 zend_life_range *range = &ival->range;
1414 zend_life_range *last = NULL;
1415
1416 do {
1417 if (range->start > to + 1) {
1418 break;
1419 } else if (range->end + 1 >= from) {
1420 if (range->start > from) {
1421 range->start = from;
1422 }
1423 last = range;
1424 range = range->next;
1425 while (range) {
1426 if (range->start > to + 1) {
1427 break;
1428 }
1429 last->end = range->end;
1430 range = range->next;
1431 last->next = range;
1432 }
1433 if (to > last->end) {
1434 last->end = to;
1435 }
1436 return SUCCESS;
1437 }
1438 last = range;
1439 range = range->next;
1440 } while (range);
1441
1442 range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1443 if (!range) {
1444 return FAILURE;
1445 }
1446 range->start = from;
1447 range->end = to;
1448 range->next = last->next;
1449 last->next = range;
1450 }
1451
1452 return SUCCESS;
1453 }
1454
zend_jit_begin_range(zend_lifetime_interval ** intervals,int var,uint32_t block_start,uint32_t from)1455 static int zend_jit_begin_range(zend_lifetime_interval **intervals, int var, uint32_t block_start, uint32_t from)
1456 {
1457 if (block_start != from && intervals[var]) {
1458 zend_life_range *range = &intervals[var]->range;
1459
1460 do {
1461 if (from >= range->start && from <= range->end) {
1462 if (range->start == block_start) {
1463 range->start = from;
1464 } else {
1465 zend_life_range *r = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1466 if (!r) {
1467 return FAILURE;
1468 }
1469 r->start = from;
1470 r->end = range->end;
1471 r->next = range->next;
1472 range->end = block_start - 1;
1473 range->next = r;
1474 }
1475 return SUCCESS;
1476 }
1477 range = range->next;
1478 } while (range);
1479 }
1480
1481 // dead store
1482 return zend_jit_add_range(intervals, var, from, from);
1483 }
1484
zend_jit_insert_interval(zend_lifetime_interval ** list,zend_lifetime_interval * ival)1485 static void zend_jit_insert_interval(zend_lifetime_interval **list, zend_lifetime_interval *ival)
1486 {
1487 while (1) {
1488 if (*list == NULL) {
1489 *list = ival;
1490 ival->list_next = NULL;
1491 return;
1492 } else if (ival->range.start < (*list)->range.start) {
1493 ival->list_next = *list;
1494 *list = ival;
1495 return;
1496 }
1497 list = &(*list)->list_next;
1498 }
1499 }
1500
zend_jit_split_interval(zend_lifetime_interval * current,uint32_t pos,zend_lifetime_interval ** list,zend_lifetime_interval ** free)1501 static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos, zend_lifetime_interval **list, zend_lifetime_interval **free)
1502 {
1503 zend_lifetime_interval *ival;
1504 zend_life_range *range = ¤t->range;
1505 zend_life_range *prev = NULL;
1506
1507 if (*free) {
1508 ival = *free;
1509 *free = ival->list_next;
1510 } else {
1511 ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
1512
1513 if (!ival) {
1514 return FAILURE;
1515 }
1516 }
1517
1518 current->flags |= ZREG_STORE;
1519
1520 ival->ssa_var = current->ssa_var;
1521 ival->reg = ZREG_NONE;
1522 ival->flags |= ZREG_SPLIT | ZREG_LOAD;
1523 ival->flags &= ~ZREG_STORE;
1524 ival->hint = NULL;
1525
1526 do {
1527 if (pos >= range->start && pos <= range->end) {
1528 break;
1529 }
1530 prev = range;
1531 range = range->next;
1532 } while(range);
1533
1534 ZEND_ASSERT(range != NULL);
1535
1536 ival->range.start = pos;
1537 ival->range.end = range->end;
1538 ival->range.next = range->next;
1539
1540 if (pos == range->start) {
1541 ZEND_ASSERT(prev != NULL);
1542 prev->next = NULL;
1543 } else {
1544 range->end = pos - 1;
1545 }
1546
1547 zend_jit_insert_interval(list, ival);
1548
1549 return SUCCESS;
1550 }
1551
zend_jit_sort_intervals(zend_lifetime_interval ** intervals,int count)1552 static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval **intervals, int count)
1553 {
1554 zend_lifetime_interval *list, *last;
1555 int i;
1556
1557 list = NULL;
1558 i = 0;
1559 while (i < count) {
1560 list = intervals[i];
1561 i++;
1562 if (list) {
1563 last = list;
1564 last->list_next = NULL;
1565 break;
1566 }
1567 }
1568
1569 while (i < count) {
1570 zend_lifetime_interval *ival = intervals[i];
1571
1572 i++;
1573 if (ival) {
1574 if ((ival->range.start > last->range.start) ||
1575 (ival->range.start == last->range.start &&
1576 ((!ival->hint && last->hint && last->hint != ival) ||
1577 ival->range.end > last->range.end))) {
1578 last->list_next = ival;
1579 last = ival;
1580 ival->list_next = NULL;
1581 } else {
1582 zend_lifetime_interval **p = &list;
1583
1584 while (1) {
1585 if (*p == NULL) {
1586 *p = last = ival;
1587 ival->list_next = NULL;
1588 break;
1589 } else if ((ival->range.start < (*p)->range.start) ||
1590 (ival->range.start == (*p)->range.start &&
1591 ((ival->hint && !(*p)->hint && ival->hint != *p) ||
1592 ival->range.end < (*p)->range.end))) {
1593 ival->list_next = *p;
1594 *p = ival;
1595 break;
1596 }
1597 p = &(*p)->list_next;
1598 }
1599 }
1600 }
1601 }
1602
1603 return list;
1604 }
1605
zend_jit_print_regset(zend_regset regset)1606 static ZEND_ATTRIBUTE_UNUSED void zend_jit_print_regset(zend_regset regset)
1607 {
1608 zend_reg reg;
1609 int first = 1;
1610
1611 ZEND_REGSET_FOREACH(regset, reg) {
1612 if (first) {
1613 first = 0;
1614 fprintf(stderr, "%s", zend_reg_name[reg]);
1615 } else {
1616 fprintf(stderr, ", %s", zend_reg_name[reg]);
1617 }
1618 } ZEND_REGSET_FOREACH_END();
1619 }
1620
zend_jit_compute_block_order_int(zend_ssa * ssa,int n,int * block_order)1621 static int *zend_jit_compute_block_order_int(zend_ssa *ssa, int n, int *block_order)
1622 {
1623 zend_basic_block *b = ssa->cfg.blocks + n;
1624
1625 tail_call:
1626 *block_order = n;
1627 block_order++;
1628
1629 n = b->children;
1630 while (n >= 0) {
1631 b = ssa->cfg.blocks + n;
1632 if (b->next_child < 0) {
1633 goto tail_call;
1634 }
1635 block_order = zend_jit_compute_block_order_int(ssa, n, block_order);
1636 n = b->next_child;
1637 }
1638
1639 return block_order;
1640 }
1641
zend_jit_compute_block_order(zend_ssa * ssa,int * block_order)1642 static int zend_jit_compute_block_order(zend_ssa *ssa, int *block_order)
1643 {
1644 int *end = zend_jit_compute_block_order_int(ssa, 0, block_order);
1645
1646 return end - block_order;
1647 }
1648
zend_jit_in_loop(zend_ssa * ssa,int header,zend_basic_block * b)1649 static bool zend_jit_in_loop(zend_ssa *ssa, int header, zend_basic_block *b)
1650 {
1651 while (b->loop_header >= 0) {
1652 if (b->loop_header == header) {
1653 return 1;
1654 }
1655 b = ssa->cfg.blocks + b->loop_header;
1656 }
1657 return 0;
1658 }
1659
zend_jit_compute_loop_body(zend_ssa * ssa,int header,int n,zend_bitset loop_body)1660 static void zend_jit_compute_loop_body(zend_ssa *ssa, int header, int n, zend_bitset loop_body)
1661 {
1662 zend_basic_block *b = ssa->cfg.blocks + n;
1663 uint32_t i;
1664
1665 tail_call:
1666 if (b->len) {
1667 for (i = b->start; i < b->start + b->len; i++) {
1668 zend_bitset_incl(loop_body, i);
1669 }
1670 }
1671
1672 n = b->children;
1673 while (n >= 0) {
1674 b = ssa->cfg.blocks + n;
1675 if (zend_jit_in_loop(ssa, header, b)) {
1676 if (b->next_child < 0) {
1677 goto tail_call;
1678 }
1679 zend_jit_compute_loop_body(ssa, header, n, loop_body);
1680 }
1681 n = b->next_child;
1682 }
1683 }
1684
zend_jit_add_hint(zend_lifetime_interval ** intervals,int dst,int src)1685 static void zend_jit_add_hint(zend_lifetime_interval **intervals, int dst, int src)
1686 {
1687 if (intervals[dst]->range.start < intervals[src]->range.start) {
1688 int tmp = src;
1689 src = dst;
1690 dst = tmp;
1691 }
1692 while (dst != src && intervals[dst]->hint) {
1693 if (intervals[dst]->hint->range.start < intervals[src]->range.start) {
1694 int tmp = src;
1695 src = intervals[dst]->hint->ssa_var;
1696 dst = tmp;
1697 } else {
1698 dst = intervals[dst]->hint->ssa_var;
1699 }
1700 }
1701 if (dst != src) {
1702 intervals[dst]->hint = intervals[src];
1703 }
1704 }
1705
1706 /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
1707 Michael Franz, CGO'10 (2010), Figure 4. */
zend_jit_compute_liveness(const zend_op_array * op_array,zend_ssa * ssa,zend_bitset candidates,zend_lifetime_interval ** list)1708 static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list)
1709 {
1710 int set_size, i, j, k, l;
1711 uint32_t n;
1712 zend_bitset live, live_in, pi_vars, loop_body;
1713 int *block_order;
1714 zend_ssa_phi *phi;
1715 zend_lifetime_interval **intervals;
1716 size_t mem_size;
1717 ALLOCA_FLAG(use_heap);
1718
1719 set_size = zend_bitset_len(ssa->vars_count);
1720 mem_size =
1721 ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)) +
1722 ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE) +
1723 ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
1724 ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
1725 ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE) +
1726 ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(int));
1727 intervals = do_alloca(mem_size, use_heap);
1728 if (!intervals) {
1729 *list = NULL;
1730 return FAILURE;
1731 }
1732
1733 live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)));
1734 live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE));
1735 pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
1736 loop_body = (zend_bitset)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
1737 block_order = (int*)((char*)loop_body + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE));
1738
1739 memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*));
1740 zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count);
1741
1742 /* TODO: Provide a linear block order where all dominators of a block
1743 * are before this block, and where all blocks belonging to the same loop
1744 * are contiguous ???
1745 */
1746 for (l = zend_jit_compute_block_order(ssa, block_order) - 1; l >= 0; l--) {
1747 zend_basic_block *b;
1748
1749 i = block_order[l];
1750 b = ssa->cfg.blocks + i;
1751
1752 /* live = UNION of successor.liveIn for each successor of b */
1753 /* live.add(phi.inputOf(b)) for each phi of successors of b */
1754 zend_bitset_clear(live, set_size);
1755 for (j = 0; j < b->successors_count; j++) {
1756 int succ = b->successors[j];
1757
1758 zend_bitset_union(live, live_in + set_size * succ, set_size);
1759 zend_bitset_clear(pi_vars, set_size);
1760 for (phi = ssa->blocks[succ].phis; phi; phi = phi->next) {
1761 if (ssa->vars[phi->ssa_var].no_val) {
1762 /* skip */
1763 } else if (phi->pi >= 0) {
1764 if (phi->pi == i && phi->sources[0] >= 0) {
1765 if (zend_bitset_in(candidates, phi->sources[0])) {
1766 zend_bitset_incl(live, phi->sources[0]);
1767 }
1768 zend_bitset_incl(pi_vars, phi->var);
1769 }
1770 } else if (!zend_bitset_in(pi_vars, phi->var)) {
1771 for (k = 0; k < ssa->cfg.blocks[succ].predecessors_count; k++) {
1772 if (ssa->cfg.predecessors[ssa->cfg.blocks[succ].predecessor_offset + k] == i) {
1773 if (phi->sources[k] >= 0 && zend_bitset_in(candidates, phi->sources[k])) {
1774 zend_bitset_incl(live, phi->sources[k]);
1775 }
1776 break;
1777 }
1778 }
1779 }
1780 }
1781 }
1782
1783 /* addRange(var, b.from, b.to) for each var in live */
1784 ZEND_BITSET_FOREACH(live, set_size, j) {
1785 if (zend_bitset_in(candidates, j)) {
1786 if (zend_jit_add_range(intervals, j, b->start, b->start + b->len - 1) != SUCCESS) {
1787 goto failure;
1788 }
1789 }
1790 } ZEND_BITSET_FOREACH_END();
1791
1792 /* for each operation op of b in reverse order */
1793 for (n = b->start + b->len; n > b->start;) {
1794 zend_ssa_op *op;
1795 const zend_op *opline;
1796 uint32_t num;
1797
1798 n--;
1799 op = ssa->ops + n;
1800 opline = op_array->opcodes + n;
1801
1802 if (UNEXPECTED(opline->opcode == ZEND_OP_DATA)) {
1803 num = n - 1;
1804 } else {
1805 num = n;
1806 }
1807
1808 /* for each output operand opd of op do */
1809 /* setFrom(opd, op) */
1810 /* live.remove(opd) */
1811 if (op->op1_def >= 0 && zend_bitset_in(candidates, op->op1_def)) {
1812 if (zend_jit_begin_range(intervals, op->op1_def, b->start, num) != SUCCESS) {
1813 goto failure;
1814 }
1815 zend_bitset_excl(live, op->op1_def);
1816 }
1817 if (op->op2_def >= 0 && zend_bitset_in(candidates, op->op2_def)) {
1818 if (zend_jit_begin_range(intervals, op->op2_def, b->start, num) != SUCCESS) {
1819 goto failure;
1820 }
1821 zend_bitset_excl(live, op->op2_def);
1822 }
1823 if (op->result_def >= 0 && zend_bitset_in(candidates, op->result_def)) {
1824 if (zend_jit_begin_range(intervals, op->result_def, b->start, num) != SUCCESS) {
1825 goto failure;
1826 }
1827 zend_bitset_excl(live, op->result_def);
1828 }
1829
1830 /* for each input operand opd of op do */
1831 /* live.add(opd) */
1832 /* addRange(opd, b.from, op) */
1833 if (op->op1_use >= 0
1834 && zend_bitset_in(candidates, op->op1_use)
1835 && !zend_ssa_is_no_val_use(opline, op, op->op1_use)) {
1836 zend_bitset_incl(live, op->op1_use);
1837 if (zend_jit_add_range(intervals, op->op1_use, b->start, num) != SUCCESS) {
1838 goto failure;
1839 }
1840 }
1841 if (op->op2_use >= 0
1842 && zend_bitset_in(candidates, op->op2_use)
1843 && !zend_ssa_is_no_val_use(opline, op, op->op2_use)) {
1844 zend_bitset_incl(live, op->op2_use);
1845 if (zend_jit_add_range(intervals, op->op2_use, b->start, num) != SUCCESS) {
1846 goto failure;
1847 }
1848 }
1849 if (op->result_use >= 0
1850 && zend_bitset_in(candidates, op->result_use)
1851 && !zend_ssa_is_no_val_use(opline, op, op->result_use)) {
1852 zend_bitset_incl(live, op->result_use);
1853 if (zend_jit_add_range(intervals, op->result_use, b->start, num) != SUCCESS) {
1854 goto failure;
1855 }
1856 }
1857 }
1858
1859 /* live.remove(phi.output) for each phi of b */
1860 for (phi = ssa->blocks[i].phis; phi; phi = phi->next) {
1861 zend_bitset_excl(live, phi->ssa_var);
1862 }
1863
1864 /* b.liveIn = live */
1865 zend_bitset_copy(live_in + set_size * i, live, set_size);
1866 }
1867
1868 for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) {
1869 zend_basic_block *b = ssa->cfg.blocks + i;
1870
1871 /* if b is loop header */
1872 if ((b->flags & ZEND_BB_LOOP_HEADER)) {
1873 live = live_in + set_size * i;
1874
1875 if (!zend_bitset_empty(live, set_size)) {
1876 uint32_t set_size2 = zend_bitset_len(op_array->last);
1877
1878 zend_bitset_clear(loop_body, set_size2);
1879 zend_jit_compute_loop_body(ssa, i, i, loop_body);
1880 while (!zend_bitset_empty(loop_body, set_size2)) {
1881 uint32_t from = zend_bitset_first(loop_body, set_size2);
1882 uint32_t to = from;
1883
1884 do {
1885 zend_bitset_excl(loop_body, to);
1886 to++;
1887 } while (zend_bitset_in(loop_body, to));
1888 to--;
1889
1890 ZEND_BITSET_FOREACH(live, set_size, j) {
1891 if (zend_jit_add_range(intervals, j, from, to) != SUCCESS) {
1892 goto failure;
1893 }
1894 } ZEND_BITSET_FOREACH_END();
1895 }
1896 }
1897 }
1898
1899 }
1900
1901 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
1902 /* Register hinting (a cheap way for register coalescing) */
1903 for (i = 0; i < ssa->vars_count; i++) {
1904 if (intervals[i]) {
1905 int src;
1906
1907 if (ssa->vars[i].definition_phi) {
1908 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
1909
1910 if (phi->pi >= 0) {
1911 src = phi->sources[0];
1912 if (intervals[src]) {
1913 zend_jit_add_hint(intervals, i, src);
1914 }
1915 } else {
1916 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1917 src = phi->sources[k];
1918 if (src >= 0) {
1919 if (ssa->vars[src].definition_phi
1920 && ssa->vars[src].definition_phi->pi >= 0
1921 && phi->block == ssa->vars[src].definition_phi->block) {
1922 /* Skip zero-length interval for Pi variable */
1923 src = ssa->vars[src].definition_phi->sources[0];
1924 }
1925 if (intervals[src]) {
1926 zend_jit_add_hint(intervals, i, src);
1927 }
1928 }
1929 }
1930 }
1931 }
1932 }
1933 }
1934 for (i = 0; i < ssa->vars_count; i++) {
1935 if (intervals[i] && !intervals[i]->hint) {
1936
1937 if (ssa->vars[i].definition >= 0) {
1938 uint32_t line = ssa->vars[i].definition;
1939 const zend_op *opline = op_array->opcodes + line;
1940
1941 switch (opline->opcode) {
1942 case ZEND_QM_ASSIGN:
1943 case ZEND_POST_INC:
1944 case ZEND_POST_DEC:
1945 if (ssa->ops[line].op1_use >= 0 &&
1946 intervals[ssa->ops[line].op1_use] &&
1947 (i == ssa->ops[line].op1_def ||
1948 (i == ssa->ops[line].result_def &&
1949 (ssa->ops[line].op1_def < 0 ||
1950 !intervals[ssa->ops[line].op1_def])))) {
1951 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1952 }
1953 break;
1954 case ZEND_SEND_VAR:
1955 case ZEND_PRE_INC:
1956 case ZEND_PRE_DEC:
1957 if (i == ssa->ops[line].op1_def &&
1958 ssa->ops[line].op1_use >= 0 &&
1959 intervals[ssa->ops[line].op1_use]) {
1960 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1961 }
1962 break;
1963 case ZEND_ASSIGN:
1964 if (ssa->ops[line].op2_use >= 0 &&
1965 intervals[ssa->ops[line].op2_use] &&
1966 (i == ssa->ops[line].op2_def ||
1967 (i == ssa->ops[line].op1_def &&
1968 (ssa->ops[line].op2_def < 0 ||
1969 !intervals[ssa->ops[line].op2_def])) ||
1970 (i == ssa->ops[line].result_def &&
1971 (ssa->ops[line].op2_def < 0 ||
1972 !intervals[ssa->ops[line].op2_def]) &&
1973 (ssa->ops[line].op1_def < 0 ||
1974 !intervals[ssa->ops[line].op1_def])))) {
1975 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
1976 }
1977 break;
1978 case ZEND_SUB:
1979 case ZEND_ADD:
1980 case ZEND_MUL:
1981 case ZEND_BW_OR:
1982 case ZEND_BW_AND:
1983 case ZEND_BW_XOR:
1984 if (i == ssa->ops[line].result_def) {
1985 if (ssa->ops[line].op1_use >= 0 &&
1986 intervals[ssa->ops[line].op1_use] &&
1987 ssa->ops[line].op1_use_chain < 0 &&
1988 !ssa->vars[ssa->ops[line].op1_use].phi_use_chain &&
1989 (ssa->var_info[i].type & MAY_BE_ANY) ==
1990 (ssa->var_info[ssa->ops[line].op1_use].type & MAY_BE_ANY)) {
1991 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1992 } else if (opline->opcode != ZEND_SUB &&
1993 ssa->ops[line].op2_use >= 0 &&
1994 intervals[ssa->ops[line].op2_use] &&
1995 ssa->ops[line].op2_use_chain < 0 &&
1996 !ssa->vars[ssa->ops[line].op2_use].phi_use_chain &&
1997 (ssa->var_info[i].type & MAY_BE_ANY) ==
1998 (ssa->var_info[ssa->ops[line].op2_use].type & MAY_BE_ANY)) {
1999 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
2000 }
2001 }
2002 break;
2003 }
2004 }
2005 }
2006 }
2007 }
2008
2009 *list = zend_jit_sort_intervals(intervals, ssa->vars_count);
2010
2011 if (*list) {
2012 zend_lifetime_interval *ival = *list;
2013 while (ival) {
2014 if (ival->hint) {
2015 ival->hint->used_as_hint = ival;
2016 }
2017 ival = ival->list_next;
2018 }
2019 }
2020
2021 free_alloca(intervals, use_heap);
2022 return SUCCESS;
2023
2024 failure:
2025 *list = NULL;
2026 free_alloca(intervals, use_heap);
2027 return FAILURE;
2028 }
2029
zend_interval_end(zend_lifetime_interval * ival)2030 static uint32_t zend_interval_end(zend_lifetime_interval *ival)
2031 {
2032 zend_life_range *range = &ival->range;
2033
2034 while (range->next) {
2035 range = range->next;
2036 }
2037 return range->end;
2038 }
2039
zend_interval_covers(zend_lifetime_interval * ival,uint32_t position)2040 static bool zend_interval_covers(zend_lifetime_interval *ival, uint32_t position)
2041 {
2042 zend_life_range *range = &ival->range;
2043
2044 do {
2045 if (position >= range->start && position <= range->end) {
2046 return 1;
2047 }
2048 range = range->next;
2049 } while (range);
2050
2051 return 0;
2052 }
2053
zend_interval_intersection(zend_lifetime_interval * ival1,zend_lifetime_interval * ival2)2054 static uint32_t zend_interval_intersection(zend_lifetime_interval *ival1, zend_lifetime_interval *ival2)
2055 {
2056 zend_life_range *r1 = &ival1->range;
2057 zend_life_range *r2 = &ival2->range;
2058
2059 do {
2060 if (r1->start <= r2->end) {
2061 if (r2->start <= r1->end) {
2062 return MAX(r1->start, r2->start);
2063 } else {
2064 r2 = r2->next;
2065 }
2066 } else {
2067 r1 = r1->next;
2068 }
2069 } while (r1 && r2);
2070
2071 return 0xffffffff;
2072 }
2073
2074 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2075 Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling */
zend_jit_try_allocate_free_reg(const zend_op_array * op_array,const zend_op ** ssa_opcodes,zend_ssa * ssa,zend_lifetime_interval * current,zend_regset available,zend_regset * hints,zend_lifetime_interval * active,zend_lifetime_interval * inactive,zend_lifetime_interval ** list,zend_lifetime_interval ** free)2076 static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_regset *hints, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free)
2077 {
2078 zend_lifetime_interval *it;
2079 uint32_t freeUntilPos[ZREG_NUM];
2080 uint32_t pos, pos2;
2081 zend_reg i, reg, reg2;
2082 zend_reg hint = ZREG_NONE;
2083 zend_regset low_priority_regs;
2084 zend_life_range *range;
2085
2086 if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) {
2087 available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_FP);
2088 } else {
2089 available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_GP);
2090 }
2091
2092 /* TODO: Allow usage of preserved registers ???
2093 * Their values have to be stored in prologue and restored in epilogue
2094 */
2095 available = ZEND_REGSET_DIFFERENCE(available, ZEND_REGSET_PRESERVED);
2096
2097 /* Set freeUntilPos of all physical registers to maxInt */
2098 for (i = 0; i < ZREG_NUM; i++) {
2099 freeUntilPos[i] = 0xffffffff;
2100 }
2101
2102 /* for each interval it in active do */
2103 /* freeUntilPos[it.reg] = 0 */
2104 it = active;
2105 if (ssa->vars[current->ssa_var].definition == current->range.start) {
2106 while (it) {
2107 if (current->range.start != zend_interval_end(it)) {
2108 freeUntilPos[it->reg] = 0;
2109 } else if (zend_jit_may_reuse_reg(
2110 ssa_opcodes ? ssa_opcodes[current->range.start] : op_array->opcodes + current->range.start,
2111 ssa->ops + current->range.start, ssa, current->ssa_var, it->ssa_var)) {
2112 if (!ZEND_REGSET_IN(*hints, it->reg) &&
2113 /* TODO: Avoid most often scratch registers. Find a better way ??? */
2114 (!current->used_as_hint ||
2115 !ZEND_REGSET_IN(ZEND_REGSET_LOW_PRIORITY, it->reg))) {
2116 hint = it->reg;
2117 }
2118 } else {
2119 freeUntilPos[it->reg] = 0;
2120 }
2121 it = it->list_next;
2122 }
2123 } else {
2124 while (it) {
2125 freeUntilPos[it->reg] = 0;
2126 it = it->list_next;
2127 }
2128 }
2129 if (current->hint) {
2130 hint = current->hint->reg;
2131 if (hint != ZREG_NONE && current->hint->used_as_hint == current) {
2132 ZEND_REGSET_EXCL(*hints, hint);
2133 }
2134 }
2135
2136 if (hint == ZREG_NONE && ZEND_REGSET_IS_EMPTY(available)) {
2137 return 0;
2138 }
2139
2140 /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
2141 Michael Franz, CGO'10 (2010), Figure 6. */
2142 if (current->flags & ZREG_SPLIT) {
2143 /* for each interval it in inactive intersecting with current do */
2144 /* freeUntilPos[it.reg] = next intersection of it with current */
2145 it = inactive;
2146 while (it) {
2147 uint32_t next = zend_interval_intersection(current, it);
2148
2149 //ZEND_ASSERT(next != 0xffffffff && !current->split);
2150 if (next < freeUntilPos[it->reg]) {
2151 freeUntilPos[it->reg] = next;
2152 }
2153 it = it->list_next;
2154 }
2155 }
2156
2157 /* Handle Scratch Registers */
2158 /* TODO: Optimize ??? */
2159 range = ¤t->range;
2160 do {
2161 uint32_t line = range->start;
2162 uint32_t last_use_line = (uint32_t)-1;
2163 zend_regset regset;
2164 zend_reg reg;
2165
2166 if ((current->flags & ZREG_LAST_USE) && !range->next) {
2167 last_use_line = range->end;
2168 }
2169 if (ssa->ops[line].op1_def == current->ssa_var ||
2170 ssa->ops[line].op2_def == current->ssa_var ||
2171 ssa->ops[line].result_def == current->ssa_var) {
2172 regset = zend_jit_get_def_scratch_regset(
2173 ssa_opcodes ? ssa_opcodes[line] : op_array->opcodes + line,
2174 ssa->ops + line,
2175 op_array, ssa, current->ssa_var, line == last_use_line);
2176 ZEND_REGSET_FOREACH(regset, reg) {
2177 if (line < freeUntilPos[reg]) {
2178 freeUntilPos[reg] = line;
2179 }
2180 } ZEND_REGSET_FOREACH_END();
2181 line++;
2182 }
2183 while (line <= range->end) {
2184 regset = zend_jit_get_scratch_regset(
2185 ssa_opcodes ? ssa_opcodes[line] : op_array->opcodes + line,
2186 ssa->ops + line,
2187 op_array, ssa, current->ssa_var, line == last_use_line);
2188 ZEND_REGSET_FOREACH(regset, reg) {
2189 if (line < freeUntilPos[reg]) {
2190 freeUntilPos[reg] = line;
2191 }
2192 } ZEND_REGSET_FOREACH_END();
2193 line++;
2194 }
2195 range = range->next;
2196 } while (range);
2197
2198 #if 0
2199 /* Coalescing */
2200 if (ssa->vars[current->ssa_var].definition == current->start) {
2201 zend_op *opline = op_array->opcodes + current->start;
2202 int hint = -1;
2203
2204 switch (opline->opcode) {
2205 case ZEND_ASSIGN:
2206 hint = ssa->ops[current->start].op2_use;
2207 case ZEND_QM_ASSIGN:
2208 hint = ssa->ops[current->start].op1_use;
2209 break;
2210 case ZEND_ADD:
2211 case ZEND_SUB:
2212 case ZEND_MUL:
2213 hint = ssa->ops[current->start].op1_use;
2214 break;
2215 case ZEND_ASSIGN_OP:
2216 if (opline->extended_value == ZEND_ADD
2217 || opline->extended_value == ZEND_SUB
2218 || opline->extended_value == ZEND_MUL) {
2219 hint = ssa->ops[current->start].op1_use;
2220 }
2221 break;
2222 }
2223 if (hint >= 0) {
2224 }
2225 }
2226 #endif
2227
2228 if (hint != ZREG_NONE && freeUntilPos[hint] > zend_interval_end(current)) {
2229 current->reg = hint;
2230 if (current->used_as_hint) {
2231 ZEND_REGSET_INCL(*hints, hint);
2232 }
2233 return 1;
2234 }
2235
2236 if (ZEND_REGSET_IS_EMPTY(available)) {
2237 return 0;
2238 }
2239
2240 pos = 0; reg = ZREG_NONE;
2241 pos2 = 0; reg2 = ZREG_NONE;
2242 low_priority_regs = *hints;
2243 if (current->used_as_hint) {
2244 /* TODO: Avoid most often scratch registers. Find a better way ??? */
2245 low_priority_regs = ZEND_REGSET_UNION(low_priority_regs, ZEND_REGSET_LOW_PRIORITY);
2246 }
2247
2248 ZEND_REGSET_FOREACH(available, i) {
2249 if (ZEND_REGSET_IN(low_priority_regs, i)) {
2250 if (freeUntilPos[i] > pos2) {
2251 reg2 = i;
2252 pos2 = freeUntilPos[i];
2253 }
2254 } else if (freeUntilPos[i] > pos) {
2255 reg = i;
2256 pos = freeUntilPos[i];
2257 }
2258 } ZEND_REGSET_FOREACH_END();
2259
2260 if (reg == ZREG_NONE) {
2261 if (reg2 != ZREG_NONE) {
2262 reg = reg2;
2263 pos = pos2;
2264 reg2 = ZREG_NONE;
2265 }
2266 }
2267
2268 if (reg == ZREG_NONE) {
2269 /* no register available without spilling */
2270 return 0;
2271 } else if (zend_interval_end(current) < pos) {
2272 /* register available for the whole interval */
2273 current->reg = reg;
2274 if (current->used_as_hint) {
2275 ZEND_REGSET_INCL(*hints, reg);
2276 }
2277 return 1;
2278 #if 0
2279 // TODO: allow low priority register usage
2280 } else if (reg2 != ZREG_NONE && zend_interval_end(current) < pos2) {
2281 /* register available for the whole interval */
2282 current->reg = reg2;
2283 if (current->used_as_hint) {
2284 ZEND_REGSET_INCL(*hints, reg2);
2285 }
2286 return 1;
2287 #endif
2288 } else {
2289 /* TODO: enable interval splitting ??? */
2290 /* register available for the first part of the interval */
2291 if (1 || zend_jit_split_interval(current, pos, list, free) != SUCCESS) {
2292 return 0;
2293 }
2294 current->reg = reg;
2295 if (current->used_as_hint) {
2296 ZEND_REGSET_INCL(*hints, reg);
2297 }
2298 return 1;
2299 }
2300 }
2301
2302 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2303 Christian Wimmer VEE'05 (2005), Figure 5. Allocation with spilling.
2304 and "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
2305 Michael Franz, CGO'10 (2010), Figure 6. */
zend_jit_allocate_blocked_reg(void)2306 static int zend_jit_allocate_blocked_reg(void)
2307 {
2308 /* TODO: ??? */
2309 return 0;
2310 }
2311
2312 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2313 Christian Wimmer VEE'10 (2005), Figure 2. */
zend_jit_linear_scan(const zend_op_array * op_array,const zend_op ** ssa_opcodes,zend_ssa * ssa,zend_lifetime_interval * list)2314 static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *list)
2315 {
2316 zend_lifetime_interval *unhandled, *active, *inactive, *handled, *free;
2317 zend_lifetime_interval *current, **p, *q;
2318 uint32_t position;
2319 zend_regset available = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP);
2320 zend_regset hints = ZEND_REGSET_EMPTY;
2321
2322 unhandled = list;
2323 /* active = inactive = handled = free = {} */
2324 active = inactive = handled = free = NULL;
2325 while (unhandled != NULL) {
2326 current = unhandled;
2327 unhandled = unhandled->list_next;
2328 position = current->range.start;
2329
2330 p = &active;
2331 while (*p) {
2332 uint32_t end = zend_interval_end(*p);
2333
2334 q = *p;
2335 if (end < position) {
2336 /* move ival from active to handled */
2337 ZEND_REGSET_INCL(available, q->reg);
2338 *p = q->list_next;
2339 q->list_next = handled;
2340 handled = q;
2341 } else if (!zend_interval_covers(q, position)) {
2342 /* move ival from active to inactive */
2343 ZEND_REGSET_INCL(available, q->reg);
2344 *p = q->list_next;
2345 q->list_next = inactive;
2346 inactive = q;
2347 } else {
2348 p = &q->list_next;
2349 }
2350 }
2351
2352 p = &inactive;
2353 while (*p) {
2354 uint32_t end = zend_interval_end(*p);
2355
2356 q = *p;
2357 if (end < position) {
2358 /* move ival from inactive to handled */
2359 *p = q->list_next;
2360 q->list_next = handled;
2361 handled = q;
2362 } else if (zend_interval_covers(q, position)) {
2363 /* move ival from inactive to active */
2364 ZEND_REGSET_EXCL(available, q->reg);
2365 *p = q->list_next;
2366 q->list_next = active;
2367 active = q;
2368 } else {
2369 p = &q->list_next;
2370 }
2371 }
2372
2373 if (zend_jit_try_allocate_free_reg(op_array, ssa_opcodes, ssa, current, available, &hints, active, inactive, &unhandled, &free) ||
2374 zend_jit_allocate_blocked_reg()) {
2375 ZEND_REGSET_EXCL(available, current->reg);
2376 current->list_next = active;
2377 active = current;
2378 } else {
2379 current->list_next = free;
2380 free = current;
2381 }
2382 }
2383
2384 /* move active to handled */
2385 while (active) {
2386 current = active;
2387 active = active->list_next;
2388 current->list_next = handled;
2389 handled = current;
2390 }
2391
2392 /* move inactive to handled */
2393 while (inactive) {
2394 current = inactive;
2395 inactive = inactive->list_next;
2396 current->list_next = handled;
2397 handled = current;
2398 }
2399
2400 return handled;
2401 }
2402
zend_jit_dump_lifetime_interval(const zend_op_array * op_array,const zend_ssa * ssa,const zend_lifetime_interval * ival)2403 static void zend_jit_dump_lifetime_interval(const zend_op_array *op_array, const zend_ssa *ssa, const zend_lifetime_interval *ival)
2404 {
2405 zend_life_range *range;
2406 int var_num = ssa->vars[ival->ssa_var].var;
2407
2408 fprintf(stderr, "#%d.", ival->ssa_var);
2409 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
2410 fprintf(stderr, ": %u-%u", ival->range.start, ival->range.end);
2411 range = ival->range.next;
2412 while (range) {
2413 fprintf(stderr, ", %u-%u", range->start, range->end);
2414 range = range->next;
2415 }
2416 if (ival->reg != ZREG_NONE) {
2417 fprintf(stderr, " (%s)", zend_reg_name[ival->reg]);
2418 }
2419 if (ival->flags & ZREG_LAST_USE) {
2420 fprintf(stderr, " last_use");
2421 }
2422 if (ival->flags & ZREG_LOAD) {
2423 fprintf(stderr, " load");
2424 }
2425 if (ival->flags & ZREG_STORE) {
2426 fprintf(stderr, " store");
2427 }
2428 if (ival->hint) {
2429 fprintf(stderr, " hint");
2430 if (ival->hint->ssa_var >= 0) {
2431 var_num = ssa->vars[ival->hint->ssa_var].var;
2432 fprintf(stderr, "=#%d.", ival->hint->ssa_var);
2433 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
2434 }
2435 if (ival->hint->reg != ZREG_NONE) {
2436 fprintf(stderr, " (%s)", zend_reg_name[ival->hint->reg]);
2437 }
2438 }
2439 fprintf(stderr, "\n");
2440 }
2441
zend_jit_allocate_registers(const zend_op_array * op_array,zend_ssa * ssa)2442 static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array *op_array, zend_ssa *ssa)
2443 {
2444 void *checkpoint;
2445 int set_size, candidates_count, i;
2446 zend_bitset candidates = NULL;
2447 zend_lifetime_interval *list, *ival;
2448 zend_lifetime_interval **intervals;
2449 ALLOCA_FLAG(use_heap);
2450
2451 if (!ssa->var_info) {
2452 return NULL;
2453 }
2454
2455 /* Identify SSA variables suitable for register allocation */
2456 set_size = zend_bitset_len(ssa->vars_count);
2457 candidates = ZEND_BITSET_ALLOCA(set_size, use_heap);
2458 if (!candidates) {
2459 return NULL;
2460 }
2461 candidates_count = 0;
2462 zend_bitset_clear(candidates, set_size);
2463 for (i = 0; i < ssa->vars_count; i++) {
2464 if (zend_jit_may_be_in_reg(op_array, ssa, i)) {
2465 zend_bitset_incl(candidates, i);
2466 candidates_count++;
2467 }
2468 }
2469 if (!candidates_count) {
2470 free_alloca(candidates, use_heap);
2471 return NULL;
2472 }
2473
2474 checkpoint = zend_arena_checkpoint(CG(arena));
2475
2476 /* Find life-time intervals */
2477 if (zend_jit_compute_liveness(op_array, ssa, candidates, &list) != SUCCESS) {
2478 goto failure;
2479 }
2480
2481 if (list) {
2482 /* Set ZREG_LAST_USE flags */
2483 ival = list;
2484 while (ival) {
2485 zend_life_range *range = &ival->range;
2486
2487 while (range->next) {
2488 range = range->next;
2489 }
2490 if (zend_ssa_is_last_use(op_array, ssa, ival->ssa_var, range->end)) {
2491 ival->flags |= ZREG_LAST_USE;
2492 }
2493 ival = ival->list_next;
2494 }
2495 }
2496
2497 if (list) {
2498 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
2499 fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
2500 ival = list;
2501 while (ival) {
2502 zend_jit_dump_lifetime_interval(op_array, ssa, ival);
2503 ival = ival->list_next;
2504 }
2505 fprintf(stderr, "\n");
2506 }
2507
2508 /* Linear Scan Register Allocation */
2509 list = zend_jit_linear_scan(op_array, NULL, ssa, list);
2510
2511 if (list) {
2512 intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*));
2513 if (!intervals) {
2514 goto failure;
2515 }
2516
2517 ival = list;
2518 while (ival != NULL) {
2519 zend_lifetime_interval *next = ival->list_next;
2520
2521 ival->list_next = intervals[ival->ssa_var];
2522 intervals[ival->ssa_var] = ival;
2523 ival = next;
2524 }
2525
2526 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
2527 /* Naive SSA resolution */
2528 for (i = 0; i < ssa->vars_count; i++) {
2529 if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
2530 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
2531 int k, src;
2532
2533 if (phi->pi >= 0) {
2534 if (!ssa->vars[i].phi_use_chain
2535 || ssa->vars[i].phi_use_chain->block != phi->block) {
2536 src = phi->sources[0];
2537 if (intervals[i]) {
2538 if (!intervals[src]) {
2539 intervals[i]->flags |= ZREG_LOAD;
2540 } else if (intervals[i]->reg != intervals[src]->reg) {
2541 intervals[i]->flags |= ZREG_LOAD;
2542 intervals[src]->flags |= ZREG_STORE;
2543 }
2544 } else if (intervals[src]) {
2545 intervals[src]->flags |= ZREG_STORE;
2546 }
2547 }
2548 } else {
2549 int need_move = 0;
2550
2551 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
2552 src = phi->sources[k];
2553 if (src >= 0) {
2554 if (ssa->vars[src].definition_phi
2555 && ssa->vars[src].definition_phi->pi >= 0
2556 && phi->block == ssa->vars[src].definition_phi->block) {
2557 /* Skip zero-length interval for Pi variable */
2558 src = ssa->vars[src].definition_phi->sources[0];
2559 }
2560 if (intervals[i]) {
2561 if (!intervals[src]) {
2562 need_move = 1;
2563 } else if (intervals[i]->reg != intervals[src]->reg) {
2564 need_move = 1;
2565 }
2566 } else if (intervals[src]) {
2567 need_move = 1;
2568 }
2569 }
2570 }
2571 if (need_move) {
2572 if (intervals[i]) {
2573 intervals[i]->flags |= ZREG_LOAD;
2574 }
2575 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
2576 src = phi->sources[k];
2577 if (src >= 0) {
2578 if (ssa->vars[src].definition_phi
2579 && ssa->vars[src].definition_phi->pi >= 0
2580 && phi->block == ssa->vars[src].definition_phi->block) {
2581 /* Skip zero-length interval for Pi variable */
2582 src = ssa->vars[src].definition_phi->sources[0];
2583 }
2584 if (intervals[src]) {
2585 intervals[src]->flags |= ZREG_STORE;
2586 }
2587 }
2588 }
2589 }
2590 }
2591 }
2592 }
2593 /* Remove useless register allocation */
2594 for (i = 0; i < ssa->vars_count; i++) {
2595 if (intervals[i] &&
2596 ((intervals[i]->flags & ZREG_LOAD) ||
2597 ((intervals[i]->flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) &&
2598 ssa->vars[i].use_chain < 0) {
2599 bool may_remove = 1;
2600 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
2601
2602 while (phi) {
2603 if (intervals[phi->ssa_var] &&
2604 !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) {
2605 may_remove = 0;
2606 break;
2607 }
2608 phi = zend_ssa_next_use_phi(ssa, i, phi);
2609 }
2610 if (may_remove) {
2611 intervals[i] = NULL;
2612 }
2613 }
2614 }
2615 /* Remove intervals used once */
2616 for (i = 0; i < ssa->vars_count; i++) {
2617 if (intervals[i] &&
2618 (intervals[i]->flags & ZREG_LOAD) &&
2619 (intervals[i]->flags & ZREG_STORE) &&
2620 (ssa->vars[i].use_chain < 0 ||
2621 zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) {
2622 bool may_remove = 1;
2623 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
2624
2625 while (phi) {
2626 if (intervals[phi->ssa_var] &&
2627 !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) {
2628 may_remove = 0;
2629 break;
2630 }
2631 phi = zend_ssa_next_use_phi(ssa, i, phi);
2632 }
2633 if (may_remove) {
2634 intervals[i] = NULL;
2635 }
2636 }
2637 }
2638 }
2639
2640 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
2641 fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
2642 for (i = 0; i < ssa->vars_count; i++) {
2643 ival = intervals[i];
2644 while (ival) {
2645 zend_jit_dump_lifetime_interval(op_array, ssa, ival);
2646 ival = ival->list_next;
2647 }
2648 }
2649 fprintf(stderr, "\n");
2650 }
2651
2652 free_alloca(candidates, use_heap);
2653 return intervals;
2654 }
2655 }
2656
2657 failure:
2658 zend_arena_release(&CG(arena), checkpoint);
2659 free_alloca(candidates, use_heap);
2660 return NULL;
2661 }
2662
zend_jit_next_is_send_result(const zend_op * opline)2663 static bool zend_jit_next_is_send_result(const zend_op *opline)
2664 {
2665 if (opline->result_type == IS_TMP_VAR
2666 && (opline+1)->opcode == ZEND_SEND_VAL
2667 && (opline+1)->op1_type == IS_TMP_VAR
2668 && (opline+1)->op2_type != IS_CONST
2669 && (opline+1)->op1.var == opline->result.var) {
2670 return 1;
2671 }
2672 return 0;
2673 }
2674
zend_jit_supported_binary_op(uint8_t op,uint32_t op1_info,uint32_t op2_info)2675 static bool zend_jit_supported_binary_op(uint8_t op, uint32_t op1_info, uint32_t op2_info)
2676 {
2677 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2678 return false;
2679 }
2680 switch (op) {
2681 case ZEND_POW:
2682 case ZEND_DIV:
2683 // TODO: check for division by zero ???
2684 return false;
2685 case ZEND_ADD:
2686 case ZEND_SUB:
2687 case ZEND_MUL:
2688 return (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
2689 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
2690 case ZEND_BW_OR:
2691 case ZEND_BW_AND:
2692 case ZEND_BW_XOR:
2693 case ZEND_SL:
2694 case ZEND_SR:
2695 case ZEND_MOD:
2696 return (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG);
2697 case ZEND_CONCAT:
2698 return (op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING);
2699 EMPTY_SWITCH_DEFAULT_CASE()
2700 }
2701 }
2702
zend_jit(const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline)2703 static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline)
2704 {
2705 int b, i, end;
2706 zend_op *opline;
2707 dasm_State* dasm_state = NULL;
2708 void *handler;
2709 int call_level = 0;
2710 void *checkpoint = NULL;
2711 zend_lifetime_interval **ra = NULL;
2712 bool is_terminated = 1; /* previous basic block is terminated by jump */
2713 bool recv_emitted = 0; /* emitted at least one RECV opcode */
2714 uint8_t smart_branch_opcode;
2715 uint32_t target_label, target_label2;
2716 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info;
2717 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
2718 zend_class_entry *ce;
2719 bool ce_is_instanceof;
2720 bool on_this;
2721
2722 if (JIT_G(bisect_limit)) {
2723 jit_bisect_pos++;
2724 if (jit_bisect_pos >= JIT_G(bisect_limit)) {
2725 if (jit_bisect_pos == JIT_G(bisect_limit)) {
2726 fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n",
2727 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
2728 op_array->scope ? "::" : "",
2729 op_array->function_name ? ZSTR_VAL(op_array->function_name) : "{main}",
2730 ZSTR_VAL(op_array->filename), op_array->line_start);
2731 }
2732 return FAILURE;
2733 }
2734 }
2735
2736 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2737 checkpoint = zend_arena_checkpoint(CG(arena));
2738 ra = zend_jit_allocate_registers(op_array, ssa);
2739 }
2740
2741 /* mark hidden branch targets */
2742 for (b = 0; b < ssa->cfg.blocks_count; b++) {
2743 if (ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE &&
2744 ssa->cfg.blocks[b].len > 1) {
2745
2746 opline = op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
2747 if (opline->opcode == ZEND_DO_FCALL &&
2748 (opline-1)->opcode == ZEND_NEW) {
2749 ssa->cfg.blocks[ssa->cfg.blocks[b].successors[0]].flags |= ZEND_BB_TARGET;
2750 }
2751 }
2752 }
2753
2754 dasm_init(&dasm_state, DASM_MAXSECTION);
2755 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
2756 dasm_setup(&dasm_state, dasm_actions);
2757
2758 dasm_growpc(&dasm_state, ssa->cfg.blocks_count * 2 + 1);
2759
2760 zend_jit_align_func(&dasm_state);
2761 for (b = 0; b < ssa->cfg.blocks_count; b++) {
2762 if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) {
2763 continue;
2764 }
2765 //#ifndef CONTEXT_THREADED_JIT
2766 if (ssa->cfg.blocks[b].flags & ZEND_BB_ENTRY) {
2767 if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
2768 /* pass */
2769 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2770 ssa->cfg.blocks[b].len == 1 &&
2771 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT) &&
2772 op_array->opcodes[ssa->cfg.blocks[b].start].opcode != ZEND_JMP) {
2773 /* don't generate code for BB with single opcode */
2774 continue;
2775 }
2776 if (ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW) {
2777 if (!is_terminated) {
2778 zend_jit_jmp(&dasm_state, b);
2779 }
2780 }
2781 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2782 zend_jit_prologue(&dasm_state);
2783 } else
2784 //#endif
2785 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
2786 opline = op_array->opcodes + ssa->cfg.blocks[b].start;
2787 if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) {
2788 if (opline->opcode == ZEND_RECV_INIT) {
2789 if (opline == op_array->opcodes ||
2790 (opline-1)->opcode != ZEND_RECV_INIT) {
2791 if (recv_emitted) {
2792 zend_jit_jmp(&dasm_state, b);
2793 }
2794 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2795 for (i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) {
2796 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b + i);
2797 }
2798 zend_jit_prologue(&dasm_state);
2799 }
2800 recv_emitted = 1;
2801 } else if (opline->opcode == ZEND_RECV) {
2802 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2803 /* skip */
2804 continue;
2805 } else if (recv_emitted) {
2806 zend_jit_jmp(&dasm_state, b);
2807 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2808 zend_jit_prologue(&dasm_state);
2809 } else {
2810 zend_arg_info *arg_info;
2811
2812 if (opline->op1.num <= op_array->num_args) {
2813 arg_info = &op_array->arg_info[opline->op1.num - 1];
2814 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
2815 arg_info = &op_array->arg_info[op_array->num_args];
2816 } else {
2817 /* skip */
2818 continue;
2819 }
2820 if (!ZEND_TYPE_IS_SET(arg_info->type)) {
2821 /* skip */
2822 continue;
2823 }
2824 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2825 zend_jit_prologue(&dasm_state);
2826 recv_emitted = 1;
2827 }
2828 } else {
2829 if (recv_emitted) {
2830 zend_jit_jmp(&dasm_state, b);
2831 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2832 ssa->cfg.blocks[b].len == 1 &&
2833 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
2834 /* don't generate code for BB with single opcode */
2835 dasm_free(&dasm_state);
2836
2837 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2838 zend_arena_release(&CG(arena), checkpoint);
2839 }
2840 return SUCCESS;
2841 }
2842 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2843 zend_jit_prologue(&dasm_state);
2844 recv_emitted = 1;
2845 }
2846 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2847 ssa->cfg.blocks[b].len == 1 &&
2848 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
2849 /* don't generate code for BB with single opcode */
2850 dasm_free(&dasm_state);
2851
2852 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2853 zend_arena_release(&CG(arena), checkpoint);
2854 }
2855 return SUCCESS;
2856 } else {
2857 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2858 zend_jit_prologue(&dasm_state);
2859 }
2860 }
2861
2862 is_terminated = 0;
2863
2864 zend_jit_label(&dasm_state, b);
2865 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
2866 if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
2867 && ssa->cfg.blocks[b].start != 0
2868 && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
2869 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_LONG
2870 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_STRING
2871 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_MATCH)) {
2872 zend_jit_reset_last_valid_opline();
2873 if (!zend_jit_set_ip(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) {
2874 goto jit_failure;
2875 }
2876 } else {
2877 zend_jit_set_last_valid_opline(op_array->opcodes + ssa->cfg.blocks[b].start);
2878 }
2879 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
2880 zend_jit_reset_last_valid_opline();
2881 } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY|ZEND_BB_ENTRY)) {
2882 zend_jit_set_last_valid_opline(op_array->opcodes + ssa->cfg.blocks[b].start);
2883 }
2884 if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) {
2885 if (!zend_jit_check_timeout(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start, NULL)) {
2886 goto jit_failure;
2887 }
2888 }
2889 if (!ssa->cfg.blocks[b].len) {
2890 continue;
2891 }
2892 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
2893 zend_ssa_phi *phi = ssa->blocks[b].phis;
2894
2895 while (phi) {
2896 zend_lifetime_interval *ival = ra[phi->ssa_var];
2897
2898 if (ival) {
2899 if (ival->flags & ZREG_LOAD) {
2900 ZEND_ASSERT(ival->reg != ZREG_NONE);
2901
2902 if (!zend_jit_load_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg)) {
2903 goto jit_failure;
2904 }
2905 } else if (ival->flags & ZREG_STORE) {
2906 ZEND_ASSERT(ival->reg != ZREG_NONE);
2907
2908 if (!zend_jit_store_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg, 1)) {
2909 goto jit_failure;
2910 }
2911 }
2912 }
2913 phi = phi->next;
2914 }
2915 }
2916 end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
2917 for (i = ssa->cfg.blocks[b].start; i <= end; i++) {
2918 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[i] : NULL;
2919 opline = op_array->opcodes + i;
2920 switch (opline->opcode) {
2921 case ZEND_INIT_FCALL:
2922 case ZEND_INIT_FCALL_BY_NAME:
2923 case ZEND_INIT_NS_FCALL_BY_NAME:
2924 case ZEND_INIT_METHOD_CALL:
2925 case ZEND_INIT_DYNAMIC_CALL:
2926 case ZEND_INIT_STATIC_METHOD_CALL:
2927 case ZEND_INIT_USER_CALL:
2928 case ZEND_NEW:
2929 call_level++;
2930 }
2931
2932 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
2933 switch (opline->opcode) {
2934 case ZEND_PRE_INC:
2935 case ZEND_PRE_DEC:
2936 case ZEND_POST_INC:
2937 case ZEND_POST_DEC:
2938 if (opline->op1_type != IS_CV) {
2939 break;
2940 }
2941 op1_info = OP1_INFO();
2942 if (!(op1_info & MAY_BE_LONG)) {
2943 break;
2944 }
2945 if (opline->result_type != IS_UNUSED) {
2946 res_use_info = -1;
2947
2948 if (opline->result_type == IS_CV
2949 && ssa->vars
2950 && ssa_op->result_use >= 0
2951 && !ssa->vars[ssa_op->result_use].no_val) {
2952 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
2953
2954 if (Z_MODE(res_use_addr) != IS_REG
2955 || Z_LOAD(res_use_addr)
2956 || Z_STORE(res_use_addr)) {
2957 res_use_info = RES_USE_INFO();
2958 }
2959 }
2960 res_info = RES_INFO();
2961 res_addr = RES_REG_ADDR();
2962 } else {
2963 res_use_info = -1;
2964 res_info = -1;
2965 res_addr = 0;
2966 }
2967 op1_def_info = OP1_DEF_INFO();
2968 if (!zend_jit_inc_dec(&dasm_state, opline,
2969 op1_info, OP1_REG_ADDR(),
2970 op1_def_info, OP1_DEF_REG_ADDR(),
2971 res_use_info, res_info,
2972 res_addr,
2973 (op1_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
2974 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2975 goto jit_failure;
2976 }
2977 goto done;
2978 case ZEND_BW_OR:
2979 case ZEND_BW_AND:
2980 case ZEND_BW_XOR:
2981 case ZEND_SL:
2982 case ZEND_SR:
2983 case ZEND_MOD:
2984 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2985 break;
2986 }
2987 op1_info = OP1_INFO();
2988 op2_info = OP2_INFO();
2989 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2990 break;
2991 }
2992 if (!(op1_info & MAY_BE_LONG)
2993 || !(op2_info & MAY_BE_LONG)) {
2994 break;
2995 }
2996 res_addr = RES_REG_ADDR();
2997 if (Z_MODE(res_addr) != IS_REG
2998 && (i + 1) <= end
2999 && zend_jit_next_is_send_result(opline)) {
3000 i++;
3001 res_use_info = -1;
3002 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3003 if (!zend_jit_reuse_ip(&dasm_state)) {
3004 goto jit_failure;
3005 }
3006 } else {
3007 res_use_info = -1;
3008
3009 if (opline->result_type == IS_CV
3010 && ssa->vars
3011 && ssa_op->result_use >= 0
3012 && !ssa->vars[ssa_op->result_use].no_val) {
3013 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
3014
3015 if (Z_MODE(res_use_addr) != IS_REG
3016 || Z_LOAD(res_use_addr)
3017 || Z_STORE(res_use_addr)) {
3018 res_use_info = RES_USE_INFO();
3019 }
3020 }
3021 }
3022 if (!zend_jit_long_math(&dasm_state, opline,
3023 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
3024 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
3025 res_use_info, RES_INFO(), res_addr,
3026 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3027 goto jit_failure;
3028 }
3029 goto done;
3030 case ZEND_ADD:
3031 case ZEND_SUB:
3032 case ZEND_MUL:
3033 // case ZEND_DIV: // TODO: check for division by zero ???
3034 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3035 break;
3036 }
3037 op1_info = OP1_INFO();
3038 op2_info = OP2_INFO();
3039 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
3040 break;
3041 }
3042 if (opline->opcode == ZEND_ADD &&
3043 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
3044 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
3045 /* pass */
3046 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
3047 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
3048 break;
3049 }
3050 res_addr = RES_REG_ADDR();
3051 if (Z_MODE(res_addr) != IS_REG
3052 && (i + 1) <= end
3053 && zend_jit_next_is_send_result(opline)) {
3054 i++;
3055 res_use_info = -1;
3056 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3057 if (!zend_jit_reuse_ip(&dasm_state)) {
3058 goto jit_failure;
3059 }
3060 } else {
3061 res_use_info = -1;
3062
3063 if (opline->result_type == IS_CV
3064 && ssa->vars
3065 && ssa_op->result_use >= 0
3066 && !ssa->vars[ssa_op->result_use].no_val) {
3067 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
3068
3069 if (Z_MODE(res_use_addr) != IS_REG
3070 || Z_LOAD(res_use_addr)
3071 || Z_STORE(res_use_addr)) {
3072 res_use_info = RES_USE_INFO();
3073 }
3074 }
3075 }
3076 res_info = RES_INFO();
3077 if (opline->opcode == ZEND_ADD &&
3078 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
3079 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
3080 if (!zend_jit_add_arrays(&dasm_state, opline, op1_info, OP1_REG_ADDR(), op2_info, OP2_REG_ADDR(), res_addr)) {
3081 goto jit_failure;
3082 }
3083 } else {
3084 if (!zend_jit_math(&dasm_state, opline,
3085 op1_info, OP1_REG_ADDR(),
3086 op2_info, OP2_REG_ADDR(),
3087 res_use_info, res_info, res_addr,
3088 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
3089 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3090 goto jit_failure;
3091 }
3092 }
3093 goto done;
3094 case ZEND_CONCAT:
3095 case ZEND_FAST_CONCAT:
3096 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3097 break;
3098 }
3099 op1_info = OP1_INFO();
3100 op2_info = OP2_INFO();
3101 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
3102 break;
3103 }
3104 if (!(op1_info & MAY_BE_STRING) ||
3105 !(op2_info & MAY_BE_STRING)) {
3106 break;
3107 }
3108 res_addr = RES_REG_ADDR();
3109 if ((i + 1) <= end
3110 && zend_jit_next_is_send_result(opline)) {
3111 i++;
3112 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3113 if (!zend_jit_reuse_ip(&dasm_state)) {
3114 goto jit_failure;
3115 }
3116 }
3117 if (!zend_jit_concat(&dasm_state, opline,
3118 op1_info, op2_info, res_addr,
3119 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3120 goto jit_failure;
3121 }
3122 goto done;
3123 case ZEND_ASSIGN_OP:
3124 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
3125 break;
3126 }
3127 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3128 break;
3129 }
3130 op1_info = OP1_INFO();
3131 op2_info = OP2_INFO();
3132 if (!zend_jit_supported_binary_op(
3133 opline->extended_value, op1_info, op2_info)) {
3134 break;
3135 }
3136 op1_def_info = OP1_DEF_INFO();
3137 if (!zend_jit_assign_op(&dasm_state, opline,
3138 op1_info, op1_def_info, OP1_RANGE(),
3139 op2_info, OP2_RANGE(),
3140 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
3141 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3142 goto jit_failure;
3143 }
3144 goto done;
3145 case ZEND_ASSIGN_DIM_OP:
3146 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
3147 break;
3148 }
3149 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3150 break;
3151 }
3152 if (!zend_jit_supported_binary_op(
3153 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3154 break;
3155 }
3156 if (!zend_jit_assign_dim_op(&dasm_state, opline,
3157 OP1_INFO(), OP1_DEF_INFO(), OP1_REG_ADDR(), OP2_INFO(),
3158 OP1_DATA_INFO(), OP1_DATA_RANGE(), IS_UNKNOWN,
3159 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3160 goto jit_failure;
3161 }
3162 goto done;
3163 case ZEND_ASSIGN_DIM:
3164 if (opline->op1_type != IS_CV) {
3165 break;
3166 }
3167 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3168 break;
3169 }
3170 if (!zend_jit_assign_dim(&dasm_state, opline,
3171 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), OP1_DATA_INFO(), IS_UNKNOWN,
3172 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3173 goto jit_failure;
3174 }
3175 goto done;
3176 case ZEND_PRE_INC_OBJ:
3177 case ZEND_PRE_DEC_OBJ:
3178 case ZEND_POST_INC_OBJ:
3179 case ZEND_POST_DEC_OBJ:
3180 if (opline->op2_type != IS_CONST
3181 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3182 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3183 break;
3184 }
3185 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3186 break;
3187 }
3188 ce = NULL;
3189 ce_is_instanceof = 0;
3190 on_this = 0;
3191 if (opline->op1_type == IS_UNUSED) {
3192 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3193 ce = op_array->scope;
3194 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3195 op1_addr = 0;
3196 on_this = 1;
3197 } else {
3198 op1_info = OP1_INFO();
3199 if (!(op1_info & MAY_BE_OBJECT)) {
3200 break;
3201 }
3202 op1_addr = OP1_REG_ADDR();
3203 if (ssa->var_info && ssa->ops) {
3204 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3205 if (ssa_op->op1_use >= 0) {
3206 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3207 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3208 ce = op1_ssa->ce;
3209 ce_is_instanceof = op1_ssa->is_instanceof;
3210 }
3211 }
3212 }
3213 }
3214 if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3215 op1_info, op1_addr,
3216 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
3217 goto jit_failure;
3218 }
3219 goto done;
3220 case ZEND_ASSIGN_OBJ_OP:
3221 if (opline->result_type != IS_UNUSED) {
3222 break;
3223 }
3224 if (opline->op2_type != IS_CONST
3225 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3226 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3227 break;
3228 }
3229 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3230 break;
3231 }
3232 if (!zend_jit_supported_binary_op(
3233 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3234 break;
3235 }
3236 ce = NULL;
3237 ce_is_instanceof = 0;
3238 on_this = 0;
3239 if (opline->op1_type == IS_UNUSED) {
3240 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3241 ce = op_array->scope;
3242 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3243 op1_addr = 0;
3244 on_this = 1;
3245 } else {
3246 op1_info = OP1_INFO();
3247 if (!(op1_info & MAY_BE_OBJECT)) {
3248 break;
3249 }
3250 op1_addr = OP1_REG_ADDR();
3251 if (ssa->var_info && ssa->ops) {
3252 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3253 if (ssa_op->op1_use >= 0) {
3254 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3255 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3256 ce = op1_ssa->ce;
3257 ce_is_instanceof = op1_ssa->is_instanceof;
3258 }
3259 }
3260 }
3261 }
3262 if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op,
3263 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_RANGE(),
3264 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
3265 goto jit_failure;
3266 }
3267 goto done;
3268 case ZEND_ASSIGN_OBJ:
3269 if (opline->op2_type != IS_CONST
3270 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3271 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3272 break;
3273 }
3274 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3275 break;
3276 }
3277 ce = NULL;
3278 ce_is_instanceof = 0;
3279 on_this = 0;
3280 if (opline->op1_type == IS_UNUSED) {
3281 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3282 ce = op_array->scope;
3283 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3284 op1_addr = 0;
3285 on_this = 1;
3286 } else {
3287 op1_info = OP1_INFO();
3288 if (!(op1_info & MAY_BE_OBJECT)) {
3289 break;
3290 }
3291 op1_addr = OP1_REG_ADDR();
3292 if (ssa->var_info && ssa->ops) {
3293 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3294 if (ssa_op->op1_use >= 0) {
3295 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3296 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3297 ce = op1_ssa->ce;
3298 ce_is_instanceof = op1_ssa->is_instanceof;
3299 }
3300 }
3301 }
3302 }
3303 if (!zend_jit_assign_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3304 op1_info, op1_addr, OP1_DATA_INFO(),
3305 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN,
3306 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3307 goto jit_failure;
3308 }
3309 goto done;
3310 case ZEND_ASSIGN:
3311 if (opline->op1_type != IS_CV) {
3312 break;
3313 }
3314 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3315 break;
3316 }
3317 op2_addr = OP2_REG_ADDR();
3318 if (ra
3319 && ssa->ops[opline - op_array->opcodes].op2_def >= 0
3320 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) {
3321 op2_def_addr = OP2_DEF_REG_ADDR();
3322 } else {
3323 op2_def_addr = op2_addr;
3324 }
3325 op1_info = OP1_INFO();
3326 if (ra && ssa->vars[ssa_op->op1_use].no_val) {
3327 op1_info |= MAY_BE_UNDEF; // requres type assignment
3328 }
3329 if (opline->result_type == IS_UNUSED) {
3330 res_addr = 0;
3331 res_info = -1;
3332 } else {
3333 res_addr = RES_REG_ADDR();
3334 res_info = RES_INFO();
3335 if (Z_MODE(res_addr) != IS_REG
3336 && (i + 1) <= end
3337 && zend_jit_next_is_send_result(opline)
3338 && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) {
3339 i++;
3340 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3341 if (!zend_jit_reuse_ip(&dasm_state)) {
3342 goto jit_failure;
3343 }
3344 }
3345 }
3346 if (!zend_jit_assign(&dasm_state, opline,
3347 op1_info, OP1_REG_ADDR(),
3348 OP1_DEF_INFO(), OP1_DEF_REG_ADDR(),
3349 OP2_INFO(), op2_addr, op2_def_addr,
3350 res_info, res_addr,
3351 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3352 goto jit_failure;
3353 }
3354 goto done;
3355 case ZEND_QM_ASSIGN:
3356 op1_addr = OP1_REG_ADDR();
3357 if (ra
3358 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
3359 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
3360 op1_def_addr = OP1_DEF_REG_ADDR();
3361 } else {
3362 op1_def_addr = op1_addr;
3363 }
3364 if (!zend_jit_qm_assign(&dasm_state, opline,
3365 OP1_INFO(), op1_addr, op1_def_addr,
3366 -1, RES_INFO(), RES_REG_ADDR())) {
3367 goto jit_failure;
3368 }
3369 goto done;
3370 case ZEND_INIT_FCALL:
3371 case ZEND_INIT_FCALL_BY_NAME:
3372 case ZEND_INIT_NS_FCALL_BY_NAME:
3373 if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level, NULL, 0)) {
3374 goto jit_failure;
3375 }
3376 goto done;
3377 case ZEND_SEND_VAL:
3378 case ZEND_SEND_VAL_EX:
3379 if (opline->op2_type == IS_CONST) {
3380 /* Named parameters not supported in JIT (yet) */
3381 break;
3382 }
3383 if (opline->opcode == ZEND_SEND_VAL_EX
3384 && opline->op2.num > MAX_ARG_FLAG_NUM) {
3385 break;
3386 }
3387 if (!zend_jit_send_val(&dasm_state, opline,
3388 OP1_INFO(), OP1_REG_ADDR())) {
3389 goto jit_failure;
3390 }
3391 goto done;
3392 case ZEND_SEND_REF:
3393 if (opline->op2_type == IS_CONST) {
3394 /* Named parameters not supported in JIT (yet) */
3395 break;
3396 }
3397 if (!zend_jit_send_ref(&dasm_state, opline, op_array,
3398 OP1_INFO(), 0)) {
3399 goto jit_failure;
3400 }
3401 goto done;
3402 case ZEND_SEND_VAR:
3403 case ZEND_SEND_VAR_EX:
3404 case ZEND_SEND_VAR_NO_REF:
3405 case ZEND_SEND_VAR_NO_REF_EX:
3406 case ZEND_SEND_FUNC_ARG:
3407 if (opline->op2_type == IS_CONST) {
3408 /* Named parameters not supported in JIT (yet) */
3409 break;
3410 }
3411 if ((opline->opcode == ZEND_SEND_VAR_EX
3412 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
3413 && opline->op2.num > MAX_ARG_FLAG_NUM) {
3414 break;
3415 }
3416 op1_addr = OP1_REG_ADDR();
3417 if (ra
3418 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
3419 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
3420 op1_def_addr = OP1_DEF_REG_ADDR();
3421 } else {
3422 op1_def_addr = op1_addr;
3423 }
3424 if (!zend_jit_send_var(&dasm_state, opline, op_array,
3425 OP1_INFO(), op1_addr, op1_def_addr)) {
3426 goto jit_failure;
3427 }
3428 goto done;
3429 case ZEND_CHECK_FUNC_ARG:
3430 if (opline->op2_type == IS_CONST) {
3431 /* Named parameters not supported in JIT (yet) */
3432 break;
3433 }
3434 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
3435 break;
3436 }
3437 if (!zend_jit_check_func_arg(&dasm_state, opline)) {
3438 goto jit_failure;
3439 }
3440 goto done;
3441 case ZEND_CHECK_UNDEF_ARGS:
3442 if (!zend_jit_check_undef_args(&dasm_state, opline)) {
3443 goto jit_failure;
3444 }
3445 goto done;
3446 case ZEND_DO_UCALL:
3447 is_terminated = 1;
3448 ZEND_FALLTHROUGH;
3449 case ZEND_DO_ICALL:
3450 case ZEND_DO_FCALL_BY_NAME:
3451 case ZEND_DO_FCALL:
3452 if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level, b + 1, NULL)) {
3453 goto jit_failure;
3454 }
3455 goto done;
3456 case ZEND_IS_EQUAL:
3457 case ZEND_IS_NOT_EQUAL:
3458 case ZEND_IS_SMALLER:
3459 case ZEND_IS_SMALLER_OR_EQUAL:
3460 case ZEND_CASE: {
3461 res_addr = RES_REG_ADDR();
3462 if ((opline->result_type & IS_TMP_VAR)
3463 && (i + 1) <= end
3464 && ((opline+1)->opcode == ZEND_JMPZ
3465 || (opline+1)->opcode == ZEND_JMPNZ
3466 || (opline+1)->opcode == ZEND_JMPZ_EX
3467 || (opline+1)->opcode == ZEND_JMPNZ_EX)
3468 && (opline+1)->op1_type == IS_TMP_VAR
3469 && (opline+1)->op1.var == opline->result.var) {
3470 i++;
3471 smart_branch_opcode = (opline+1)->opcode;
3472 target_label = ssa->cfg.blocks[b].successors[0];
3473 target_label2 = ssa->cfg.blocks[b].successors[1];
3474 /* For EX variant write into the result of EX opcode. */
3475 if ((opline+1)->opcode == ZEND_JMPZ_EX
3476 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
3477 res_addr = OP_REG_ADDR(opline + 1, result_type, result, result_def);
3478 }
3479 } else {
3480 smart_branch_opcode = 0;
3481 target_label = target_label2 = (uint32_t)-1;
3482 }
3483 if (!zend_jit_cmp(&dasm_state, opline,
3484 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
3485 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
3486 res_addr,
3487 zend_may_throw(opline, ssa_op, op_array, ssa),
3488 smart_branch_opcode, target_label, target_label2,
3489 NULL, 0)) {
3490 goto jit_failure;
3491 }
3492 goto done;
3493 }
3494 case ZEND_IS_IDENTICAL:
3495 case ZEND_IS_NOT_IDENTICAL:
3496 case ZEND_CASE_STRICT:
3497 if ((opline->result_type & IS_TMP_VAR)
3498 && (i + 1) <= end
3499 && ((opline+1)->opcode == ZEND_JMPZ
3500 || (opline+1)->opcode == ZEND_JMPNZ)
3501 && (opline+1)->op1_type == IS_TMP_VAR
3502 && (opline+1)->op1.var == opline->result.var) {
3503 i++;
3504 smart_branch_opcode = (opline+1)->opcode;
3505 target_label = ssa->cfg.blocks[b].successors[0];
3506 target_label2 = ssa->cfg.blocks[b].successors[1];
3507 } else {
3508 smart_branch_opcode = 0;
3509 target_label = target_label2 = (uint32_t)-1;
3510 }
3511 if (!zend_jit_identical(&dasm_state, opline,
3512 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
3513 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
3514 RES_REG_ADDR(),
3515 zend_may_throw(opline, ssa_op, op_array, ssa),
3516 smart_branch_opcode, target_label, target_label2,
3517 NULL, 0)) {
3518 goto jit_failure;
3519 }
3520 goto done;
3521 case ZEND_DEFINED:
3522 if ((opline->result_type & IS_TMP_VAR)
3523 && (i + 1) <= end
3524 && ((opline+1)->opcode == ZEND_JMPZ
3525 || (opline+1)->opcode == ZEND_JMPNZ)
3526 && (opline+1)->op1_type == IS_TMP_VAR
3527 && (opline+1)->op1.var == opline->result.var) {
3528 i++;
3529 smart_branch_opcode = (opline+1)->opcode;
3530 target_label = ssa->cfg.blocks[b].successors[0];
3531 target_label2 = ssa->cfg.blocks[b].successors[1];
3532 } else {
3533 smart_branch_opcode = 0;
3534 target_label = target_label2 = (uint32_t)-1;
3535 }
3536 if (!zend_jit_defined(&dasm_state, opline, smart_branch_opcode, target_label, target_label2, NULL)) {
3537 goto jit_failure;
3538 }
3539 goto done;
3540 case ZEND_TYPE_CHECK:
3541 if (opline->extended_value == MAY_BE_RESOURCE) {
3542 // TODO: support for is_resource() ???
3543 break;
3544 }
3545 if ((opline->result_type & IS_TMP_VAR)
3546 && (i + 1) <= end
3547 && ((opline+1)->opcode == ZEND_JMPZ
3548 || (opline+1)->opcode == ZEND_JMPNZ)
3549 && (opline+1)->op1_type == IS_TMP_VAR
3550 && (opline+1)->op1.var == opline->result.var) {
3551 i++;
3552 smart_branch_opcode = (opline+1)->opcode;
3553 target_label = ssa->cfg.blocks[b].successors[0];
3554 target_label2 = ssa->cfg.blocks[b].successors[1];
3555 } else {
3556 smart_branch_opcode = 0;
3557 target_label = target_label2 = (uint32_t)-1;
3558 }
3559 if (!zend_jit_type_check(&dasm_state, opline, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) {
3560 goto jit_failure;
3561 }
3562 goto done;
3563 case ZEND_RETURN:
3564 op1_info = OP1_INFO();
3565 if ((PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info))
3566 || op_array->type == ZEND_EVAL_CODE
3567 // TODO: support for top-level code
3568 || !op_array->function_name
3569 // TODO: support for IS_UNDEF ???
3570 || (op1_info & MAY_BE_UNDEF)) {
3571 if (!zend_jit_tail_handler(&dasm_state, opline)) {
3572 goto jit_failure;
3573 }
3574 } else {
3575 int j;
3576 bool left_frame = 0;
3577
3578 if (!zend_jit_return(&dasm_state, opline, op_array,
3579 op1_info, OP1_REG_ADDR())) {
3580 goto jit_failure;
3581 }
3582 if (jit_return_label >= 0) {
3583 if (!zend_jit_jmp(&dasm_state, jit_return_label)) {
3584 goto jit_failure;
3585 }
3586 goto done;
3587 }
3588 jit_return_label = ssa->cfg.blocks_count * 2;
3589 if (!zend_jit_label(&dasm_state, jit_return_label)) {
3590 goto jit_failure;
3591 }
3592 if (op_array->last_var > 100) {
3593 /* To many CVs to unroll */
3594 if (!zend_jit_free_cvs(&dasm_state)) {
3595 goto jit_failure;
3596 }
3597 left_frame = 1;
3598 }
3599 if (!left_frame) {
3600 for (j = 0 ; j < op_array->last_var; j++) {
3601 uint32_t info = zend_ssa_cv_info(op_array, ssa, j);
3602
3603 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
3604 if (!left_frame) {
3605 left_frame = 1;
3606 if (!zend_jit_leave_frame(&dasm_state)) {
3607 goto jit_failure;
3608 }
3609 }
3610 if (!zend_jit_free_cv(&dasm_state, info, j)) {
3611 goto jit_failure;
3612 }
3613 }
3614 }
3615 }
3616 if (!zend_jit_leave_func(&dasm_state, op_array, opline, op1_info, left_frame,
3617 NULL, NULL, (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, 1)) {
3618 goto jit_failure;
3619 }
3620 }
3621 goto done;
3622 case ZEND_BOOL:
3623 case ZEND_BOOL_NOT:
3624 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
3625 OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(),
3626 -1, -1,
3627 zend_may_throw(opline, ssa_op, op_array, ssa),
3628 opline->opcode, NULL)) {
3629 goto jit_failure;
3630 }
3631 goto done;
3632 case ZEND_JMPZ:
3633 case ZEND_JMPNZ:
3634 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
3635 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
3636 /* smart branch */
3637 if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
3638 goto jit_failure;
3639 }
3640 goto done;
3641 }
3642 ZEND_FALLTHROUGH;
3643 case ZEND_JMPZ_EX:
3644 case ZEND_JMPNZ_EX:
3645 if (opline->result_type == IS_UNDEF) {
3646 res_addr = 0;
3647 } else {
3648 res_addr = RES_REG_ADDR();
3649 }
3650 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
3651 OP1_INFO(), OP1_REG_ADDR(), res_addr,
3652 ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1],
3653 zend_may_throw(opline, ssa_op, op_array, ssa),
3654 opline->opcode, NULL)) {
3655 goto jit_failure;
3656 }
3657 goto done;
3658 case ZEND_ISSET_ISEMPTY_CV:
3659 if ((opline->extended_value & ZEND_ISEMPTY)) {
3660 // TODO: support for empty() ???
3661 break;
3662 }
3663 if ((opline->result_type & IS_TMP_VAR)
3664 && (i + 1) <= end
3665 && ((opline+1)->opcode == ZEND_JMPZ
3666 || (opline+1)->opcode == ZEND_JMPNZ)
3667 && (opline+1)->op1_type == IS_TMP_VAR
3668 && (opline+1)->op1.var == opline->result.var) {
3669 i++;
3670 smart_branch_opcode = (opline+1)->opcode;
3671 target_label = ssa->cfg.blocks[b].successors[0];
3672 target_label2 = ssa->cfg.blocks[b].successors[1];
3673 } else {
3674 smart_branch_opcode = 0;
3675 target_label = target_label2 = (uint32_t)-1;
3676 }
3677 if (!zend_jit_isset_isempty_cv(&dasm_state, opline,
3678 OP1_INFO(), OP1_REG_ADDR(),
3679 smart_branch_opcode, target_label, target_label2,
3680 NULL)) {
3681 goto jit_failure;
3682 }
3683 goto done;
3684 case ZEND_IN_ARRAY:
3685 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
3686 break;
3687 }
3688 op1_info = OP1_INFO();
3689 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
3690 break;
3691 }
3692 if ((opline->result_type & IS_TMP_VAR)
3693 && (i + 1) <= end
3694 && ((opline+1)->opcode == ZEND_JMPZ
3695 || (opline+1)->opcode == ZEND_JMPNZ)
3696 && (opline+1)->op1_type == IS_TMP_VAR
3697 && (opline+1)->op1.var == opline->result.var) {
3698 i++;
3699 smart_branch_opcode = (opline+1)->opcode;
3700 target_label = ssa->cfg.blocks[b].successors[0];
3701 target_label2 = ssa->cfg.blocks[b].successors[1];
3702 } else {
3703 smart_branch_opcode = 0;
3704 target_label = target_label2 = (uint32_t)-1;
3705 }
3706 if (!zend_jit_in_array(&dasm_state, opline,
3707 op1_info, OP1_REG_ADDR(),
3708 smart_branch_opcode, target_label, target_label2,
3709 NULL)) {
3710 goto jit_failure;
3711 }
3712 goto done;
3713 case ZEND_FETCH_DIM_R:
3714 case ZEND_FETCH_DIM_IS:
3715 case ZEND_FETCH_LIST_R:
3716 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3717 break;
3718 }
3719 if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op,
3720 OP1_INFO(), OP1_REG_ADDR(), 0,
3721 OP2_INFO(), RES_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
3722 goto jit_failure;
3723 }
3724 goto done;
3725 case ZEND_FETCH_DIM_W:
3726 case ZEND_FETCH_DIM_RW:
3727 // case ZEND_FETCH_DIM_UNSET:
3728 case ZEND_FETCH_LIST_W:
3729 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3730 break;
3731 }
3732 if (opline->op1_type != IS_CV) {
3733 break;
3734 }
3735 if (!zend_jit_fetch_dim(&dasm_state, opline,
3736 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
3737 goto jit_failure;
3738 }
3739 goto done;
3740 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
3741 if ((opline->extended_value & ZEND_ISEMPTY)) {
3742 // TODO: support for empty() ???
3743 break;
3744 }
3745 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3746 break;
3747 }
3748 if ((opline->result_type & IS_TMP_VAR)
3749 && (i + 1) <= end
3750 && ((opline+1)->opcode == ZEND_JMPZ
3751 || (opline+1)->opcode == ZEND_JMPNZ)
3752 && (opline+1)->op1_type == IS_TMP_VAR
3753 && (opline+1)->op1.var == opline->result.var) {
3754 i++;
3755 smart_branch_opcode = (opline+1)->opcode;
3756 target_label = ssa->cfg.blocks[b].successors[0];
3757 target_label2 = ssa->cfg.blocks[b].successors[1];
3758 } else {
3759 smart_branch_opcode = 0;
3760 target_label = target_label2 = (uint32_t)-1;
3761 }
3762 if (!zend_jit_isset_isempty_dim(&dasm_state, opline,
3763 OP1_INFO(), OP1_REG_ADDR(), 0,
3764 OP2_INFO(), IS_UNKNOWN,
3765 zend_may_throw(opline, ssa_op, op_array, ssa),
3766 smart_branch_opcode, target_label, target_label2,
3767 NULL)) {
3768 goto jit_failure;
3769 }
3770 goto done;
3771 case ZEND_FETCH_OBJ_R:
3772 case ZEND_FETCH_OBJ_IS:
3773 case ZEND_FETCH_OBJ_W:
3774 if (opline->op2_type != IS_CONST
3775 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3776 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3777 break;
3778 }
3779 ce = NULL;
3780 ce_is_instanceof = 0;
3781 on_this = 0;
3782 if (opline->op1_type == IS_UNUSED) {
3783 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3784 op1_addr = 0;
3785 ce = op_array->scope;
3786 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3787 on_this = 1;
3788 } else {
3789 op1_info = OP1_INFO();
3790 if (!(op1_info & MAY_BE_OBJECT)) {
3791 break;
3792 }
3793 op1_addr = OP1_REG_ADDR();
3794 if (ssa->var_info && ssa->ops) {
3795 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3796 if (ssa_op->op1_use >= 0) {
3797 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3798 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3799 ce = op1_ssa->ce;
3800 ce_is_instanceof = op1_ssa->is_instanceof;
3801 }
3802 }
3803 }
3804 }
3805 if (!zend_jit_fetch_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3806 op1_info, op1_addr, 0, ce, ce_is_instanceof, on_this, 0, 0, NULL,
3807 IS_UNKNOWN,
3808 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3809 goto jit_failure;
3810 }
3811 goto done;
3812 case ZEND_BIND_GLOBAL:
3813 if (!ssa->ops || !ssa->var_info) {
3814 op1_info = MAY_BE_ANY|MAY_BE_REF;
3815 } else {
3816 op1_info = OP1_INFO();
3817 }
3818 if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) {
3819 goto jit_failure;
3820 }
3821 goto done;
3822 case ZEND_RECV:
3823 if (!zend_jit_recv(&dasm_state, opline, op_array)) {
3824 goto jit_failure;
3825 }
3826 goto done;
3827 case ZEND_RECV_INIT:
3828 if (!zend_jit_recv_init(&dasm_state, opline, op_array,
3829 (opline + 1)->opcode != ZEND_RECV_INIT,
3830 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3831 goto jit_failure;
3832 }
3833 goto done;
3834 case ZEND_FREE:
3835 case ZEND_FE_FREE:
3836 if (!zend_jit_free(&dasm_state, opline, OP1_INFO(),
3837 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3838 goto jit_failure;
3839 }
3840 goto done;
3841 case ZEND_ECHO:
3842 op1_info = OP1_INFO();
3843 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3844 break;
3845 }
3846 if (!zend_jit_echo(&dasm_state, opline, op1_info)) {
3847 goto jit_failure;
3848 }
3849 goto done;
3850 case ZEND_STRLEN:
3851 op1_info = OP1_INFO();
3852 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3853 break;
3854 }
3855 if (!zend_jit_strlen(&dasm_state, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR())) {
3856 goto jit_failure;
3857 }
3858 goto done;
3859 case ZEND_COUNT:
3860 op1_info = OP1_INFO();
3861 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
3862 break;
3863 }
3864 if (!zend_jit_count(&dasm_state, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
3865 goto jit_failure;
3866 }
3867 goto done;
3868 case ZEND_FETCH_THIS:
3869 if (!zend_jit_fetch_this(&dasm_state, opline, op_array, 0)) {
3870 goto jit_failure;
3871 }
3872 goto done;
3873 case ZEND_SWITCH_LONG:
3874 case ZEND_SWITCH_STRING:
3875 case ZEND_MATCH:
3876 if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL, NULL)) {
3877 goto jit_failure;
3878 }
3879 goto done;
3880 case ZEND_VERIFY_RETURN_TYPE:
3881 if (opline->op1_type == IS_UNUSED) {
3882 /* Always throws */
3883 break;
3884 }
3885 if (opline->op1_type == IS_CONST) {
3886 /* TODO Different instruction format, has return value */
3887 break;
3888 }
3889 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
3890 /* Not worth bothering with */
3891 break;
3892 }
3893 if (OP1_INFO() & MAY_BE_REF) {
3894 /* TODO May need reference unwrapping. */
3895 break;
3896 }
3897 if (!zend_jit_verify_return_type(&dasm_state, opline, op_array, OP1_INFO())) {
3898 goto jit_failure;
3899 }
3900 goto done;
3901 case ZEND_FE_RESET_R:
3902 op1_info = OP1_INFO();
3903 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
3904 break;
3905 }
3906 if (!zend_jit_fe_reset(&dasm_state, opline, op1_info)) {
3907 goto jit_failure;
3908 }
3909 goto done;
3910 case ZEND_FE_FETCH_R:
3911 op1_info = OP1_INFO();
3912 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
3913 break;
3914 }
3915 if (!zend_jit_fe_fetch(&dasm_state, opline, op1_info, OP2_INFO(),
3916 ssa->cfg.blocks[b].successors[0], opline->opcode, NULL)) {
3917 goto jit_failure;
3918 }
3919 goto done;
3920 case ZEND_FETCH_CONSTANT:
3921 if (!zend_jit_fetch_constant(&dasm_state, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
3922 goto jit_failure;
3923 }
3924 goto done;
3925 case ZEND_INIT_METHOD_CALL:
3926 if (opline->op2_type != IS_CONST
3927 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
3928 break;
3929 }
3930 ce = NULL;
3931 ce_is_instanceof = 0;
3932 on_this = 0;
3933 if (opline->op1_type == IS_UNUSED) {
3934 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3935 op1_addr = 0;
3936 ce = op_array->scope;
3937 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3938 on_this = 1;
3939 } else {
3940 op1_info = OP1_INFO();
3941 if (!(op1_info & MAY_BE_OBJECT)) {
3942 break;
3943 }
3944 op1_addr = OP1_REG_ADDR();
3945 if (ssa->var_info && ssa->ops) {
3946 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3947 if (ssa_op->op1_use >= 0) {
3948 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3949 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3950 ce = op1_ssa->ce;
3951 ce_is_instanceof = op1_ssa->is_instanceof;
3952 }
3953 }
3954 }
3955 }
3956 if (!zend_jit_init_method_call(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level,
3957 op1_info, op1_addr, ce, ce_is_instanceof, on_this, 0, NULL,
3958 NULL, 0, 0)) {
3959 goto jit_failure;
3960 }
3961 goto done;
3962 case ZEND_ROPE_INIT:
3963 case ZEND_ROPE_ADD:
3964 case ZEND_ROPE_END:
3965 op2_info = OP2_INFO();
3966 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3967 break;
3968 }
3969 if (!zend_jit_rope(&dasm_state, opline, op2_info)) {
3970 goto jit_failure;
3971 }
3972 goto done;
3973 default:
3974 break;
3975 }
3976 }
3977
3978 switch (opline->opcode) {
3979 case ZEND_RECV_INIT:
3980 case ZEND_BIND_GLOBAL:
3981 if (opline == op_array->opcodes ||
3982 opline->opcode != op_array->opcodes[i-1].opcode) {
3983 /* repeatable opcodes */
3984 if (!zend_jit_handler(&dasm_state, opline,
3985 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3986 goto jit_failure;
3987 }
3988 }
3989 zend_jit_set_last_valid_opline(opline+1);
3990 break;
3991 case ZEND_NOP:
3992 case ZEND_OP_DATA:
3993 case ZEND_SWITCH_LONG:
3994 case ZEND_SWITCH_STRING:
3995 break;
3996 case ZEND_MATCH:
3997 /* We have to exit to the VM because the MATCH handler performs an N-way jump for
3998 * which we can't generate simple (opcache.jit=1201) JIT code. */
3999 if (!zend_jit_tail_handler(&dasm_state, opline)) {
4000 goto jit_failure;
4001 }
4002 break;
4003 case ZEND_JMP:
4004 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
4005 const zend_op *target = OP_JMP_ADDR(opline, opline->op1);
4006
4007 if (!zend_jit_set_ip(&dasm_state, target)) {
4008 goto jit_failure;
4009 }
4010 }
4011 if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) {
4012 goto jit_failure;
4013 }
4014 is_terminated = 1;
4015 break;
4016 case ZEND_CATCH:
4017 case ZEND_FAST_CALL:
4018 case ZEND_FAST_RET:
4019 case ZEND_GENERATOR_CREATE:
4020 case ZEND_GENERATOR_RETURN:
4021 case ZEND_RETURN_BY_REF:
4022 case ZEND_RETURN:
4023 case ZEND_EXIT:
4024 case ZEND_MATCH_ERROR:
4025 /* switch through trampoline */
4026 case ZEND_YIELD:
4027 case ZEND_YIELD_FROM:
4028 if (!zend_jit_tail_handler(&dasm_state, opline)) {
4029 goto jit_failure;
4030 }
4031 is_terminated = 1;
4032 break;
4033 /* stackless execution */
4034 case ZEND_INCLUDE_OR_EVAL:
4035 case ZEND_DO_FCALL:
4036 case ZEND_DO_UCALL:
4037 case ZEND_DO_FCALL_BY_NAME:
4038 if (!zend_jit_call(&dasm_state, opline, b + 1)) {
4039 goto jit_failure;
4040 }
4041 is_terminated = 1;
4042 break;
4043 case ZEND_JMPZ:
4044 case ZEND_JMPNZ:
4045 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
4046 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
4047 /* smart branch */
4048 if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
4049 goto jit_failure;
4050 }
4051 goto done;
4052 }
4053 ZEND_FALLTHROUGH;
4054 case ZEND_JMPZ_EX:
4055 case ZEND_JMPNZ_EX:
4056 case ZEND_JMP_SET:
4057 case ZEND_COALESCE:
4058 case ZEND_JMP_NULL:
4059 case ZEND_FE_RESET_R:
4060 case ZEND_FE_RESET_RW:
4061 case ZEND_ASSERT_CHECK:
4062 case ZEND_FE_FETCH_R:
4063 case ZEND_FE_FETCH_RW:
4064 case ZEND_BIND_INIT_STATIC_OR_JMP:
4065 if (!zend_jit_handler(&dasm_state, opline,
4066 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
4067 !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
4068 goto jit_failure;
4069 }
4070 break;
4071 case ZEND_NEW:
4072 if (!zend_jit_handler(&dasm_state, opline, 1)) {
4073 return 0;
4074 }
4075 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
4076 zend_class_entry *ce = NULL;
4077
4078 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
4079 if (ssa->ops && ssa->var_info) {
4080 zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
4081 if (res_ssa->ce && !res_ssa->is_instanceof) {
4082 ce = res_ssa->ce;
4083 }
4084 }
4085 } else {
4086 if (opline->op1_type == IS_CONST) {
4087 zval *zv = RT_CONSTANT(opline, opline->op1);
4088 if (Z_TYPE_P(zv) == IS_STRING) {
4089 zval *lc = zv + 1;
4090 ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc));
4091 }
4092 }
4093 }
4094
4095 i++;
4096
4097 if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) {
4098 const zend_op *next_opline = opline + 1;
4099
4100 zend_jit_cond_jmp(&dasm_state, next_opline, ssa->cfg.blocks[b].successors[0]);
4101 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
4102 zend_jit_call(&dasm_state, next_opline, b + 1);
4103 is_terminated = 1;
4104 } else {
4105 zend_jit_do_fcall(&dasm_state, next_opline, op_array, ssa, call_level, b + 1, NULL);
4106 }
4107 }
4108
4109 /* We skip over the DO_FCALL, so decrement call_level ourselves. */
4110 call_level--;
4111 }
4112 break;
4113 default:
4114 if (!zend_jit_handler(&dasm_state, opline,
4115 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4116 goto jit_failure;
4117 }
4118 if (i == end
4119 && (opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
4120 /* smart branch split across basic blocks */
4121 if (!zend_jit_cond_jmp(&dasm_state, opline + 2, ssa->cfg.blocks[b+1].successors[0])) {
4122 goto jit_failure;
4123 }
4124 if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b+1].successors[1])) {
4125 goto jit_failure;
4126 }
4127 is_terminated = 1;
4128 }
4129 }
4130 done:
4131 switch (opline->opcode) {
4132 case ZEND_DO_FCALL:
4133 case ZEND_DO_ICALL:
4134 case ZEND_DO_UCALL:
4135 case ZEND_DO_FCALL_BY_NAME:
4136 case ZEND_CALLABLE_CONVERT:
4137 call_level--;
4138 }
4139 }
4140 }
4141
4142 handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0,
4143 (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET, SP_ADJ_JIT);
4144 if (!handler) {
4145 goto jit_failure;
4146 }
4147 dasm_free(&dasm_state);
4148
4149 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
4150 zend_arena_release(&CG(arena), checkpoint);
4151 }
4152 return SUCCESS;
4153
4154 jit_failure:
4155 if (dasm_state) {
4156 dasm_free(&dasm_state);
4157 }
4158 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
4159 zend_arena_release(&CG(arena), checkpoint);
4160 }
4161 return FAILURE;
4162 }
4163
zend_jit_collect_calls(zend_op_array * op_array,zend_script * script)4164 static void zend_jit_collect_calls(zend_op_array *op_array, zend_script *script)
4165 {
4166 zend_func_info *func_info;
4167
4168 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4169 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4170 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4171 func_info = ZEND_FUNC_INFO(op_array);
4172 } else {
4173 func_info = zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info));
4174 ZEND_SET_FUNC_INFO(op_array, func_info);
4175 }
4176 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, op_array, func_info);
4177 }
4178
zend_jit_cleanup_func_info(zend_op_array * op_array)4179 static void zend_jit_cleanup_func_info(zend_op_array *op_array)
4180 {
4181 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
4182 zend_call_info *caller_info, *callee_info;
4183
4184 if (func_info) {
4185 caller_info = func_info->caller_info;
4186 callee_info = func_info->callee_info;
4187
4188 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4189 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4190 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4191 func_info->num = 0;
4192 func_info->flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
4193 | ZEND_FUNC_JIT_ON_PROF_REQUEST
4194 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
4195 | ZEND_FUNC_JIT_ON_HOT_TRACE;
4196 memset(&func_info->ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
4197 } else {
4198 ZEND_SET_FUNC_INFO(op_array, NULL);
4199 }
4200
4201 while (caller_info) {
4202 if (caller_info->caller_op_array) {
4203 zend_jit_cleanup_func_info(caller_info->caller_op_array);
4204 }
4205 caller_info = caller_info->next_caller;
4206 }
4207 while (callee_info) {
4208 if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) {
4209 zend_jit_cleanup_func_info(&callee_info->callee_func->op_array);
4210 }
4211 callee_info = callee_info->next_callee;
4212 }
4213 }
4214 }
4215
zend_real_jit_func(zend_op_array * op_array,zend_script * script,const zend_op * rt_opline,uint8_t trigger)4216 static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline, uint8_t trigger)
4217 {
4218 zend_ssa ssa;
4219 void *checkpoint;
4220 zend_func_info *func_info;
4221 uint8_t orig_trigger;
4222
4223 if (*dasm_ptr == dasm_end) {
4224 return FAILURE;
4225 }
4226
4227 orig_trigger = JIT_G(trigger);
4228 JIT_G(trigger) = trigger;
4229 checkpoint = zend_arena_checkpoint(CG(arena));
4230
4231 /* Build SSA */
4232 memset(&ssa, 0, sizeof(zend_ssa));
4233
4234 if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) {
4235 goto jit_failure;
4236 }
4237
4238 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
4239 zend_jit_collect_calls(op_array, script);
4240 func_info = ZEND_FUNC_INFO(op_array);
4241 func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array);
4242 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
4243 zend_init_func_return_info(op_array, script, &func_info->return_info);
4244 }
4245 }
4246
4247 if (zend_jit_op_array_analyze2(op_array, script, &ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
4248 goto jit_failure;
4249 }
4250
4251 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
4252 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa);
4253 }
4254
4255 if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) {
4256 goto jit_failure;
4257 }
4258
4259 zend_jit_cleanup_func_info(op_array);
4260 zend_arena_release(&CG(arena), checkpoint);
4261 JIT_G(trigger) = orig_trigger;
4262 return SUCCESS;
4263
4264 jit_failure:
4265 zend_jit_cleanup_func_info(op_array);
4266 zend_arena_release(&CG(arena), checkpoint);
4267 JIT_G(trigger) = orig_trigger;
4268 return FAILURE;
4269 }
4270
4271 /* Run-time JIT handler */
zend_runtime_jit(void)4272 static int ZEND_FASTCALL zend_runtime_jit(void)
4273 {
4274 zend_execute_data *execute_data = EG(current_execute_data);
4275 zend_op_array *op_array = &EX(func)->op_array;
4276 zend_op *opline = op_array->opcodes;
4277 zend_jit_op_array_extension *jit_extension;
4278 bool do_bailout = 0;
4279
4280 zend_shared_alloc_lock();
4281
4282 if (ZEND_FUNC_INFO(op_array)) {
4283
4284 SHM_UNPROTECT();
4285 zend_jit_unprotect();
4286
4287 zend_try {
4288 /* restore original opcode handlers */
4289 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4290 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4291 opline++;
4292 }
4293 }
4294 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
4295 opline->handler = jit_extension->orig_handler;
4296
4297 /* perform real JIT for this function */
4298 zend_real_jit_func(op_array, NULL, NULL, ZEND_JIT_ON_FIRST_EXEC);
4299 } zend_catch {
4300 do_bailout = true;
4301 } zend_end_try();
4302
4303 zend_jit_protect();
4304 SHM_PROTECT();
4305 }
4306
4307 zend_shared_alloc_unlock();
4308
4309 if (do_bailout) {
4310 zend_bailout();
4311 }
4312
4313 /* JIT-ed code is going to be called by VM */
4314 return 0;
4315 }
4316
zend_jit_check_funcs(HashTable * function_table,bool is_method)4317 void zend_jit_check_funcs(HashTable *function_table, bool is_method) {
4318 zend_op *opline;
4319 zend_function *func;
4320 zend_op_array *op_array;
4321 uintptr_t counter;
4322 zend_jit_op_array_extension *jit_extension;
4323
4324 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(function_table, func) {
4325 if (func->type == ZEND_INTERNAL_FUNCTION) {
4326 break;
4327 }
4328 op_array = &func->op_array;
4329 opline = op_array->opcodes;
4330 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4331 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4332 opline++;
4333 }
4334 }
4335 if (opline->handler == zend_jit_profile_jit_handler) {
4336 if (!RUN_TIME_CACHE(op_array)) {
4337 continue;
4338 }
4339 counter = (uintptr_t)ZEND_COUNTER_INFO(op_array);
4340 ZEND_COUNTER_INFO(op_array) = 0;
4341 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
4342 opline->handler = jit_extension->orig_handler;
4343 if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) {
4344 zend_real_jit_func(op_array, NULL, NULL, ZEND_JIT_ON_PROF_REQUEST);
4345 }
4346 }
4347 } ZEND_HASH_FOREACH_END();
4348 }
4349
zend_jit_hot_func(zend_execute_data * execute_data,const zend_op * opline)4350 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
4351 {
4352 zend_op_array *op_array = &EX(func)->op_array;
4353 zend_jit_op_array_hot_extension *jit_extension;
4354 uint32_t i;
4355 bool do_bailout = 0;
4356
4357 zend_shared_alloc_lock();
4358 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
4359
4360 if (jit_extension) {
4361 SHM_UNPROTECT();
4362 zend_jit_unprotect();
4363
4364 zend_try {
4365 for (i = 0; i < op_array->last; i++) {
4366 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
4367 }
4368
4369 /* perform real JIT for this function */
4370 zend_real_jit_func(op_array, NULL, opline, ZEND_JIT_ON_HOT_COUNTERS);
4371 } zend_catch {
4372 do_bailout = 1;
4373 } zend_end_try();
4374
4375 zend_jit_protect();
4376 SHM_PROTECT();
4377 }
4378
4379 zend_shared_alloc_unlock();
4380
4381 if (do_bailout) {
4382 zend_bailout();
4383 }
4384 /* JIT-ed code is going to be called by VM */
4385 }
4386
zend_jit_setup_hot_counters_ex(zend_op_array * op_array,zend_cfg * cfg)4387 static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg)
4388 {
4389 if (JIT_G(hot_func)) {
4390 zend_op *opline = op_array->opcodes;
4391
4392 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4393 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4394 opline++;
4395 }
4396 }
4397
4398 opline->handler = (const void*)zend_jit_func_hot_counter_handler;
4399 }
4400
4401 if (JIT_G(hot_loop)) {
4402 uint32_t i;
4403
4404 for (i = 0; i < cfg->blocks_count; i++) {
4405 if ((cfg->blocks[i].flags & ZEND_BB_REACHABLE) &&
4406 (cfg->blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
4407 op_array->opcodes[cfg->blocks[i].start].handler =
4408 (const void*)zend_jit_loop_hot_counter_handler;
4409 }
4410 }
4411 }
4412 }
4413
zend_jit_restart_hot_counters(zend_op_array * op_array)4414 static int zend_jit_restart_hot_counters(zend_op_array *op_array)
4415 {
4416 zend_jit_op_array_hot_extension *jit_extension;
4417 zend_cfg cfg;
4418 uint32_t i;
4419
4420 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
4421 for (i = 0; i < op_array->last; i++) {
4422 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
4423 }
4424
4425 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
4426 return FAILURE;
4427 }
4428
4429 zend_jit_setup_hot_counters_ex(op_array, &cfg);
4430
4431 return SUCCESS;
4432 }
4433
zend_jit_setup_hot_counters(zend_op_array * op_array)4434 static int zend_jit_setup_hot_counters(zend_op_array *op_array)
4435 {
4436 zend_jit_op_array_hot_extension *jit_extension;
4437 zend_cfg cfg;
4438 uint32_t i;
4439
4440 ZEND_ASSERT(zend_jit_func_hot_counter_handler != NULL);
4441 ZEND_ASSERT(zend_jit_loop_hot_counter_handler != NULL);
4442
4443 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
4444 return FAILURE;
4445 }
4446
4447 jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*));
4448 if (!jit_extension) {
4449 return FAILURE;
4450 }
4451 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4452 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS;
4453 jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
4454 for (i = 0; i < op_array->last; i++) {
4455 jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
4456 }
4457 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4458
4459 zend_jit_setup_hot_counters_ex(op_array, &cfg);
4460
4461 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4462
4463 return SUCCESS;
4464 }
4465
4466 #include "jit/zend_jit_trace.c"
4467
zend_jit_op_array(zend_op_array * op_array,zend_script * script)4468 ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
4469 {
4470 if (dasm_ptr == NULL) {
4471 return FAILURE;
4472 }
4473
4474 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) {
4475 zend_jit_op_array_extension *jit_extension;
4476 zend_op *opline = op_array->opcodes;
4477
4478 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
4479 ZEND_SET_FUNC_INFO(op_array, NULL);
4480 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
4481 return SUCCESS;
4482 }
4483
4484 /* Set run-time JIT handler */
4485 ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL);
4486 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4487 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4488 opline++;
4489 }
4490 }
4491 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
4492 if (!jit_extension) {
4493 return FAILURE;
4494 }
4495 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4496 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC;
4497 jit_extension->orig_handler = (void*)opline->handler;
4498 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4499 opline->handler = (const void*)zend_jit_runtime_jit_handler;
4500 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4501
4502 return SUCCESS;
4503 } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) {
4504 zend_jit_op_array_extension *jit_extension;
4505 zend_op *opline = op_array->opcodes;
4506
4507 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
4508 ZEND_SET_FUNC_INFO(op_array, NULL);
4509 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
4510 return SUCCESS;
4511 }
4512
4513 ZEND_ASSERT(zend_jit_profile_jit_handler != NULL);
4514 if (op_array->function_name) {
4515 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4516 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4517 opline++;
4518 }
4519 }
4520 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
4521 if (!jit_extension) {
4522 return FAILURE;
4523 }
4524 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4525 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST;
4526 jit_extension->orig_handler = (void*)opline->handler;
4527 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4528 opline->handler = (const void*)zend_jit_profile_jit_handler;
4529 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4530 }
4531
4532 return SUCCESS;
4533 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4534 return zend_jit_setup_hot_counters(op_array);
4535 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4536 return zend_jit_setup_hot_trace_counters(op_array);
4537 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4538 return zend_real_jit_func(op_array, script, NULL, ZEND_JIT_ON_SCRIPT_LOAD);
4539 } else {
4540 ZEND_UNREACHABLE();
4541 }
4542 }
4543
zend_jit_script(zend_script * script)4544 ZEND_EXT_API int zend_jit_script(zend_script *script)
4545 {
4546 void *checkpoint;
4547 zend_call_graph call_graph;
4548 zend_func_info *info;
4549 int i;
4550
4551 if (dasm_ptr == NULL || *dasm_ptr == dasm_end) {
4552 return FAILURE;
4553 }
4554
4555 checkpoint = zend_arena_checkpoint(CG(arena));
4556
4557 call_graph.op_arrays_count = 0;
4558 zend_build_call_graph(&CG(arena), script, &call_graph);
4559
4560 zend_analyze_call_graph(&CG(arena), script, &call_graph);
4561
4562 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4563 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4564 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS ||
4565 JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4566 for (i = 0; i < call_graph.op_arrays_count; i++) {
4567 if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
4568 goto jit_failure;
4569 }
4570 }
4571 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4572 for (i = 0; i < call_graph.op_arrays_count; i++) {
4573 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4574 if (info) {
4575 if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
4576 goto jit_failure;
4577 }
4578 info->flags = info->ssa.cfg.flags;
4579 }
4580 }
4581
4582 for (i = 0; i < call_graph.op_arrays_count; i++) {
4583 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4584 if (info) {
4585 info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]);
4586 if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
4587 zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info);
4588 }
4589 }
4590 }
4591
4592 for (i = 0; i < call_graph.op_arrays_count; i++) {
4593 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4594 if (info) {
4595 if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
4596 goto jit_failure;
4597 }
4598 info->flags = info->ssa.cfg.flags;
4599 }
4600 }
4601
4602 for (i = 0; i < call_graph.op_arrays_count; i++) {
4603 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4604 if (info) {
4605 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
4606 zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa);
4607 }
4608 if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) {
4609 goto jit_failure;
4610 }
4611 }
4612 }
4613
4614 for (i = 0; i < call_graph.op_arrays_count; i++) {
4615 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
4616 }
4617 } else {
4618 ZEND_UNREACHABLE();
4619 }
4620
4621 zend_arena_release(&CG(arena), checkpoint);
4622
4623 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC
4624 || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST
4625 || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS
4626 || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4627 zend_class_entry *ce;
4628 zend_op_array *op_array;
4629
4630 ZEND_HASH_MAP_FOREACH_PTR(&script->class_table, ce) {
4631 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
4632 if (!ZEND_FUNC_INFO(op_array)) {
4633 void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes);
4634
4635 if (jit_extension) {
4636 ZEND_SET_FUNC_INFO(op_array, jit_extension);
4637 }
4638 }
4639 } ZEND_HASH_FOREACH_END();
4640 } ZEND_HASH_FOREACH_END();
4641 }
4642
4643 return SUCCESS;
4644
4645 jit_failure:
4646 if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4647 for (i = 0; i < call_graph.op_arrays_count; i++) {
4648 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
4649 }
4650 }
4651 zend_arena_release(&CG(arena), checkpoint);
4652 return FAILURE;
4653 }
4654
zend_jit_unprotect(void)4655 ZEND_EXT_API void zend_jit_unprotect(void)
4656 {
4657 #ifdef HAVE_MPROTECT
4658 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4659 int opts = PROT_READ | PROT_WRITE;
4660 #ifdef ZTS
4661 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
4662 if (zend_write_protect) {
4663 pthread_jit_write_protect_np(0);
4664 }
4665 #endif
4666 opts |= PROT_EXEC;
4667 #endif
4668 if (mprotect(dasm_buf, dasm_size, opts) != 0) {
4669 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4670 }
4671 }
4672 #elif _WIN32
4673 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4674 DWORD old, new;
4675 #ifdef ZTS
4676 new = PAGE_EXECUTE_READWRITE;
4677 #else
4678 new = PAGE_READWRITE;
4679 #endif
4680 if (!VirtualProtect(dasm_buf, dasm_size, new, &old)) {
4681 DWORD err = GetLastError();
4682 char *msg = php_win32_error_to_msg(err);
4683 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4684 php_win32_error_msg_free(msg);
4685 }
4686 }
4687 #endif
4688 }
4689
zend_jit_protect(void)4690 ZEND_EXT_API void zend_jit_protect(void)
4691 {
4692 #ifdef HAVE_MPROTECT
4693 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4694 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
4695 if (zend_write_protect) {
4696 pthread_jit_write_protect_np(1);
4697 }
4698 #endif
4699 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
4700 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4701 }
4702 }
4703 #elif _WIN32
4704 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4705 DWORD old;
4706
4707 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
4708 DWORD err = GetLastError();
4709 char *msg = php_win32_error_to_msg(err);
4710 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4711 php_win32_error_msg_free(msg);
4712 }
4713 }
4714 #endif
4715 }
4716
zend_jit_init_handlers(void)4717 static void zend_jit_init_handlers(void)
4718 {
4719 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
4720 zend_jit_runtime_jit_handler = dasm_labels[zend_lbhybrid_runtime_jit];
4721 zend_jit_profile_jit_handler = dasm_labels[zend_lbhybrid_profile_jit];
4722 zend_jit_func_hot_counter_handler = dasm_labels[zend_lbhybrid_func_hot_counter];
4723 zend_jit_loop_hot_counter_handler = dasm_labels[zend_lbhybrid_loop_hot_counter];
4724 zend_jit_func_trace_counter_handler = dasm_labels[zend_lbhybrid_func_trace_counter];
4725 zend_jit_ret_trace_counter_handler = dasm_labels[zend_lbhybrid_ret_trace_counter];
4726 zend_jit_loop_trace_counter_handler = dasm_labels[zend_lbhybrid_loop_trace_counter];
4727 } else {
4728 zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
4729 zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
4730 zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper;
4731 zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper;
4732 zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper;
4733 zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper;
4734 zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper;
4735 }
4736 }
4737
zend_jit_make_stubs(void)4738 static int zend_jit_make_stubs(void)
4739 {
4740 dasm_State* dasm_state = NULL;
4741 uint32_t i;
4742
4743 dasm_init(&dasm_state, DASM_MAXSECTION);
4744 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
4745
4746 for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) {
4747 dasm_setup(&dasm_state, dasm_actions);
4748 zend_jit_align_stub(&dasm_state);
4749 if (!zend_jit_stubs[i].stub(&dasm_state)) {
4750 return 0;
4751 }
4752 if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name, 0,
4753 zend_jit_stubs[i].offset, zend_jit_stubs[i].adjustment)) {
4754 return 0;
4755 }
4756 }
4757
4758 zend_jit_init_handlers();
4759
4760 dasm_free(&dasm_state);
4761 return 1;
4762 }
4763
zend_jit_globals_ctor(zend_jit_globals * jit_globals)4764 static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
4765 {
4766 memset(jit_globals, 0, sizeof(zend_jit_globals));
4767 zend_jit_trace_init_caches();
4768 }
4769
4770 #ifdef ZTS
zend_jit_globals_dtor(zend_jit_globals * jit_globals)4771 static void zend_jit_globals_dtor(zend_jit_globals *jit_globals)
4772 {
4773 zend_jit_trace_free_caches(jit_globals);
4774 }
4775 #endif
4776
zend_jit_parse_config_num(zend_long jit)4777 static int zend_jit_parse_config_num(zend_long jit)
4778 {
4779 if (jit == 0) {
4780 JIT_G(on) = 0;
4781 return SUCCESS;
4782 }
4783
4784 if (jit < 0) return FAILURE;
4785
4786 if (jit % 10 == 0 || jit % 10 > 5) return FAILURE;
4787 JIT_G(opt_level) = jit % 10;
4788
4789 jit /= 10;
4790 if (jit % 10 > 5 || jit % 10 == 4) return FAILURE;
4791 JIT_G(trigger) = jit % 10;
4792
4793 jit /= 10;
4794 if (jit % 10 > 2) return FAILURE;
4795 JIT_G(opt_flags) = jit % 10;
4796
4797 jit /= 10;
4798 if (jit % 10 > 1) return FAILURE;
4799 JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0);
4800
4801 if (jit / 10 != 0) return FAILURE;
4802
4803 JIT_G(on) = 1;
4804
4805 return SUCCESS;
4806 }
4807
zend_jit_config(zend_string * jit,int stage)4808 ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage)
4809 {
4810 if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) {
4811 if (stage == ZEND_INI_STAGE_RUNTIME) {
4812 zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)");
4813 }
4814 return FAILURE;
4815 }
4816
4817 if (zend_string_equals_literal_ci(jit, "disable")) {
4818 JIT_G(enabled) = 0;
4819 JIT_G(on) = 0;
4820 return SUCCESS;
4821 } else if (ZSTR_LEN(jit) == 0
4822 || zend_string_equals_literal_ci(jit, "0")
4823 || zend_string_equals_literal_ci(jit, "off")
4824 || zend_string_equals_literal_ci(jit, "no")
4825 || zend_string_equals_literal_ci(jit, "false")) {
4826 JIT_G(enabled) = 1;
4827 JIT_G(on) = 0;
4828 return SUCCESS;
4829 } else if (zend_string_equals_literal_ci(jit, "1")
4830 || zend_string_equals_literal_ci(jit, "on")
4831 || zend_string_equals_literal_ci(jit, "yes")
4832 || zend_string_equals_literal_ci(jit, "true")
4833 || zend_string_equals_literal_ci(jit, "tracing")) {
4834 JIT_G(enabled) = 1;
4835 JIT_G(on) = 1;
4836 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS;
4837 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
4838 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
4839 return SUCCESS;
4840 } else if (zend_string_equals_ci(jit, ZSTR_KNOWN(ZEND_STR_FUNCTION))) {
4841 JIT_G(enabled) = 1;
4842 JIT_G(on) = 1;
4843 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT;
4844 JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD;
4845 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
4846 return SUCCESS;
4847 } else {
4848 char *end;
4849 zend_long num = ZEND_STRTOL(ZSTR_VAL(jit), &end, 10);
4850 if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) {
4851 goto failure;
4852 }
4853 JIT_G(enabled) = 1;
4854 return SUCCESS;
4855 }
4856
4857 failure:
4858 zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number");
4859 JIT_G(enabled) = 0;
4860 JIT_G(on) = 0;
4861 return FAILURE;
4862 }
4863
zend_jit_debug_config(zend_long old_val,zend_long new_val,int stage)4864 ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage)
4865 {
4866 if (stage != ZEND_INI_STAGE_STARTUP) {
4867 if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) {
4868 if (stage == ZEND_INI_STAGE_RUNTIME) {
4869 zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup");
4870 }
4871 return FAILURE;
4872 }
4873 #ifdef HAVE_DISASM
4874 if (new_val & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
4875 if (JIT_G(enabled) && !JIT_G(symbols) && !zend_jit_disasm_init()) {
4876 // TODO: error reporting and cleanup ???
4877 return FAILURE;
4878 }
4879 // TODO: symbols for JIT-ed code compiled before are missing ???
4880 }
4881 #endif
4882 }
4883 return SUCCESS;
4884 }
4885
zend_jit_init(void)4886 ZEND_EXT_API void zend_jit_init(void)
4887 {
4888 #ifdef ZTS
4889 jit_globals_id = ts_allocate_id(&jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, (ts_allocate_dtor) zend_jit_globals_dtor);
4890 #else
4891 zend_jit_globals_ctor(&jit_globals);
4892 #endif
4893 }
4894
zend_jit_check_support(void)4895 ZEND_EXT_API int zend_jit_check_support(void)
4896 {
4897 int i;
4898
4899 zend_jit_vm_kind = zend_vm_kind();
4900 if (zend_jit_vm_kind != ZEND_VM_KIND_CALL &&
4901 zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
4902 zend_error(E_WARNING, "JIT is compatible only with CALL and HYBRID VM. JIT disabled.");
4903 JIT_G(enabled) = 0;
4904 JIT_G(on) = 0;
4905 return FAILURE;
4906 }
4907
4908 if (zend_execute_ex != execute_ex) {
4909 if (zend_dtrace_enabled) {
4910 zend_error(E_WARNING, "JIT is incompatible with DTrace. JIT disabled.");
4911 } else if (strcmp(sapi_module.name, "phpdbg") != 0) {
4912 zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled.");
4913 }
4914 JIT_G(enabled) = 0;
4915 JIT_G(on) = 0;
4916 return FAILURE;
4917 }
4918
4919 for (i = 0; i <= 256; i++) {
4920 switch (i) {
4921 /* JIT has no effect on these opcodes */
4922 case ZEND_BEGIN_SILENCE:
4923 case ZEND_END_SILENCE:
4924 case ZEND_EXIT:
4925 break;
4926 default:
4927 if (zend_get_user_opcode_handler(i) != NULL) {
4928 zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled.");
4929 JIT_G(enabled) = 0;
4930 JIT_G(on) = 0;
4931 return FAILURE;
4932 }
4933 }
4934 }
4935
4936 return SUCCESS;
4937 }
4938
zend_jit_startup(void * buf,size_t size,bool reattached)4939 ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, bool reattached)
4940 {
4941 int ret;
4942
4943 zend_jit_halt_op = zend_get_halt_op();
4944
4945 if (zend_jit_setup() != SUCCESS) {
4946 // TODO: error reporting and cleanup ???
4947 return FAILURE;
4948 }
4949
4950 zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(ACCELERATOR_PRODUCT_NAME);
4951
4952 #ifdef HAVE_GDB
4953 zend_jit_gdb_init();
4954 #endif
4955
4956 #if ZEND_JIT_SUPPORT_CLDEMOTE
4957 cpu_support_cldemote = zend_cpu_supports_cldemote();
4958 #endif
4959
4960 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
4961 zend_write_protect = pthread_jit_write_protect_supported_np();
4962 #endif
4963
4964 dasm_buf = buf;
4965 dasm_size = size;
4966
4967 #ifdef HAVE_MPROTECT
4968 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
4969 if (zend_write_protect) {
4970 pthread_jit_write_protect_np(1);
4971 }
4972 #endif
4973 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
4974 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
4975 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4976 }
4977 } else {
4978 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
4979 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4980 }
4981 }
4982 #elif _WIN32
4983 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
4984 DWORD old;
4985
4986 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
4987 DWORD err = GetLastError();
4988 char *msg = php_win32_error_to_msg(err);
4989 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4990 php_win32_error_msg_free(msg);
4991 }
4992 } else {
4993 DWORD old;
4994
4995 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
4996 DWORD err = GetLastError();
4997 char *msg = php_win32_error_to_msg(err);
4998 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4999 php_win32_error_msg_free(msg);
5000 }
5001 }
5002 #endif
5003
5004 dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr) * 2);
5005 if (!reattached) {
5006 zend_jit_unprotect();
5007 *dasm_ptr = dasm_buf;
5008 #if _WIN32
5009 /* reserve space for global labels */
5010 *dasm_ptr = (void**)*dasm_ptr + zend_lb_MAX;
5011 #endif
5012 zend_jit_protect();
5013 }
5014
5015 #ifdef HAVE_DISASM
5016 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
5017 if (!zend_jit_disasm_init()) {
5018 // TODO: error reporting and cleanup ???
5019 return FAILURE;
5020 }
5021 }
5022 #endif
5023
5024 #ifdef HAVE_PERFTOOLS
5025 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
5026 zend_jit_perf_jitdump_open();
5027 }
5028 #endif
5029
5030 if (!reattached) {
5031 zend_jit_unprotect();
5032 ret = zend_jit_make_stubs();
5033 #if _WIN32
5034 /* save global labels */
5035 memcpy(dasm_buf, dasm_labels, sizeof(void*) * zend_lb_MAX);
5036 #endif
5037 zend_jit_protect();
5038 if (!ret) {
5039 // TODO: error reporting and cleanup ???
5040 return FAILURE;
5041 }
5042 } else {
5043 #if _WIN32
5044 /* restore global labels */
5045 memcpy(dasm_labels, dasm_buf, sizeof(void*) * zend_lb_MAX);
5046 zend_jit_init_handlers();
5047 #endif
5048 }
5049
5050 if (zend_jit_trace_startup(reattached) != SUCCESS) {
5051 return FAILURE;
5052 }
5053
5054 zend_jit_unprotect();
5055 #if ZEND_JIT_TARGET_ARM64
5056 /* reserve space for global labels veneers */
5057 dasm_labels_veneers = *dasm_ptr;
5058 *dasm_ptr = (void**)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT);
5059 memset(dasm_labels_veneers, 0, sizeof(void*) * ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT));
5060 #endif
5061 /* save JIT buffer pos */
5062 dasm_ptr[1] = dasm_ptr[0];
5063 zend_jit_protect();
5064
5065 return SUCCESS;
5066 }
5067
zend_jit_shutdown(void)5068 ZEND_EXT_API void zend_jit_shutdown(void)
5069 {
5070 if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE && dasm_ptr != NULL) {
5071 fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf));
5072 }
5073
5074 #ifdef HAVE_GDB
5075 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
5076 zend_jit_gdb_unregister();
5077 }
5078 #endif
5079
5080 #ifdef HAVE_DISASM
5081 zend_jit_disasm_shutdown();
5082 #endif
5083
5084 #ifdef HAVE_PERFTOOLS
5085 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
5086 zend_jit_perf_jitdump_close();
5087 }
5088 #endif
5089 #ifdef ZTS
5090 ts_free_id(jit_globals_id);
5091 #else
5092 zend_jit_trace_free_caches(&jit_globals);
5093 #endif
5094 }
5095
zend_jit_reset_counters(void)5096 static void zend_jit_reset_counters(void)
5097 {
5098 int i;
5099
5100 for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
5101 zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT;
5102 }
5103 }
5104
zend_jit_activate(void)5105 ZEND_EXT_API void zend_jit_activate(void)
5106 {
5107 zend_jit_profile_counter = 0;
5108 if (JIT_G(on)) {
5109 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
5110 zend_jit_reset_counters();
5111 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
5112 zend_jit_reset_counters();
5113 zend_jit_trace_reset_caches();
5114 }
5115 }
5116 }
5117
zend_jit_deactivate(void)5118 ZEND_EXT_API void zend_jit_deactivate(void)
5119 {
5120 if (zend_jit_profile_counter && !CG(unclean_shutdown)) {
5121 zend_class_entry *ce;
5122
5123 zend_shared_alloc_lock();
5124 SHM_UNPROTECT();
5125 zend_jit_unprotect();
5126
5127 zend_jit_check_funcs(EG(function_table), 0);
5128 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(EG(class_table), ce) {
5129 if (ce->type == ZEND_INTERNAL_CLASS) {
5130 break;
5131 }
5132 zend_jit_check_funcs(&ce->function_table, 1);
5133 } ZEND_HASH_FOREACH_END();
5134
5135 zend_jit_protect();
5136 SHM_PROTECT();
5137 zend_shared_alloc_unlock();
5138 }
5139
5140 zend_jit_profile_counter = 0;
5141 }
5142
zend_jit_restart_preloaded_op_array(zend_op_array * op_array)5143 static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array)
5144 {
5145 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
5146
5147 if (!func_info) {
5148 return;
5149 }
5150
5151 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
5152 zend_jit_restart_hot_trace_counters(op_array);
5153 } else if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_COUNTERS) {
5154 zend_jit_restart_hot_counters(op_array);
5155 #if 0
5156 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ???
5157 } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) {
5158 zend_op *opline = op_array->opcodes;
5159 zend_jit_op_array_extension *jit_extension =
5160 (zend_jit_op_array_extension*)func_info;
5161
5162 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
5163 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
5164 opline++;
5165 }
5166 }
5167 if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) {
5168 opline->handler = (const void*)zend_jit_runtime_jit_handler;
5169 } else {
5170 opline->handler = (const void*)zend_jit_profile_jit_handler;
5171 }
5172 #endif
5173 }
5174 if (op_array->num_dynamic_func_defs) {
5175 for (uint32_t i = 0; i < op_array->num_dynamic_func_defs; i++) {
5176 zend_jit_restart_preloaded_op_array(op_array->dynamic_func_defs[i]);
5177 }
5178 }
5179 }
5180
zend_jit_restart_preloaded_script(zend_persistent_script * script)5181 static void zend_jit_restart_preloaded_script(zend_persistent_script *script)
5182 {
5183 zend_class_entry *ce;
5184 zend_op_array *op_array;
5185
5186 zend_jit_restart_preloaded_op_array(&script->script.main_op_array);
5187
5188 ZEND_HASH_MAP_FOREACH_PTR(&script->script.function_table, op_array) {
5189 zend_jit_restart_preloaded_op_array(op_array);
5190 } ZEND_HASH_FOREACH_END();
5191
5192 ZEND_HASH_MAP_FOREACH_PTR(&script->script.class_table, ce) {
5193 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
5194 if (op_array->type == ZEND_USER_FUNCTION) {
5195 zend_jit_restart_preloaded_op_array(op_array);
5196 }
5197 } ZEND_HASH_FOREACH_END();
5198 } ZEND_HASH_FOREACH_END();
5199 }
5200
zend_jit_restart(void)5201 ZEND_EXT_API void zend_jit_restart(void)
5202 {
5203 if (dasm_buf) {
5204 zend_jit_unprotect();
5205
5206 #if ZEND_JIT_TARGET_ARM64
5207 memset(dasm_labels_veneers, 0, sizeof(void*) * ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT));
5208 #endif
5209
5210 /* restore JIT buffer pos */
5211 dasm_ptr[0] = dasm_ptr[1];
5212
5213 zend_jit_trace_restart();
5214
5215 if (ZCSG(preload_script)) {
5216 zend_jit_restart_preloaded_script(ZCSG(preload_script));
5217 if (ZCSG(saved_scripts)) {
5218 zend_persistent_script **p = ZCSG(saved_scripts);
5219
5220 while (*p) {
5221 zend_jit_restart_preloaded_script(*p);
5222 p++;
5223 }
5224 }
5225 }
5226
5227 zend_jit_protect();
5228
5229 #ifdef HAVE_DISASM
5230 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
5231 zend_jit_disasm_shutdown();
5232 zend_jit_disasm_init();
5233 }
5234 #endif
5235 }
5236 }
5237
5238 #endif /* HAVE_JIT */
5239
