1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 #include "main/php.h"
20 #include "main/SAPI.h"
21 #include "php_version.h"
22 #include <ZendAccelerator.h>
23 #include "zend_shared_alloc.h"
24 #include "Zend/zend_execute.h"
25 #include "Zend/zend_vm.h"
26 #include "Zend/zend_exceptions.h"
27 #include "Zend/zend_constants.h"
28 #include "Zend/zend_closures.h"
29 #include "Zend/zend_ini.h"
30 #include "Zend/zend_observer.h"
31 #include "zend_smart_str.h"
32 #include "jit/zend_jit.h"
33
34 #ifdef HAVE_JIT
35
36 #include "Optimizer/zend_func_info.h"
37 #include "Optimizer/zend_ssa.h"
38 #include "Optimizer/zend_inference.h"
39 #include "Optimizer/zend_call_graph.h"
40 #include "Optimizer/zend_dump.h"
41
42 #if ZEND_JIT_TARGET_X86
43 # include "jit/zend_jit_x86.h"
44 #elif ZEND_JIT_TARGET_ARM64
45 # include "jit/zend_jit_arm64.h"
46 #endif
47
48 #include "jit/zend_jit_internal.h"
49
50 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
51 #include <pthread.h>
52 #endif
53
54 #ifdef ZTS
55 int jit_globals_id;
56 #else
57 zend_jit_globals jit_globals;
58 #endif
59
60 //#define CONTEXT_THREADED_JIT
61 #define ZEND_JIT_USE_RC_INFERENCE
62
63 #ifdef ZEND_JIT_USE_RC_INFERENCE
64 # define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE
65 # define RC_MAY_BE_1(info) (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0)
66 # define RC_MAY_BE_N(info) (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0)
67 #else
68 # define ZEND_SSA_RC_INFERENCE_FLAG 0
69 # define RC_MAY_BE_1(info) 1
70 # define RC_MAY_BE_N(info) 1
71 #endif
72
73 #define JIT_PREFIX "JIT$"
74 #define JIT_STUB_PREFIX "JIT$$"
75 #define TRACE_PREFIX "TRACE-"
76
77 #define DASM_M_GROW(ctx, t, p, sz, need) \
78 do { \
79 size_t _sz = (sz), _need = (need); \
80 if (_sz < _need) { \
81 if (_sz < 16) _sz = 16; \
82 while (_sz < _need) _sz += _sz; \
83 (p) = (t *)erealloc((p), _sz); \
84 (sz) = _sz; \
85 } \
86 } while(0)
87
88 #define DASM_M_FREE(ctx, p, sz) efree(p)
89
90 #if ZEND_DEBUG
91 # define DASM_CHECKS 1
92 #endif
93
94 #include "dynasm/dasm_proto.h"
95
96 typedef struct _zend_jit_stub {
97 const char *name;
98 int (*stub)(dasm_State **Dst);
99 uint32_t offset;
100 uint32_t adjustment;
101 } zend_jit_stub;
102
103 #define JIT_STUB(name, offset, adjustment) \
104 {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub, offset, adjustment}
105
106 zend_ulong zend_jit_profile_counter = 0;
107 int zend_jit_profile_counter_rid = -1;
108
109 int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
110
111 const zend_op *zend_jit_halt_op = NULL;
112 static int zend_jit_vm_kind = 0;
113 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
114 static int zend_write_protect = 1;
115 #endif
116
117 static void *dasm_buf = NULL;
118 static void *dasm_end = NULL;
119 static void **dasm_ptr = NULL;
120
121 static size_t dasm_size = 0;
122
123 static zend_long jit_bisect_pos = 0;
124
125 static const void *zend_jit_runtime_jit_handler = NULL;
126 static const void *zend_jit_profile_jit_handler = NULL;
127 static const void *zend_jit_func_hot_counter_handler = NULL;
128 static const void *zend_jit_loop_hot_counter_handler = NULL;
129 static const void *zend_jit_func_trace_counter_handler = NULL;
130 static const void *zend_jit_ret_trace_counter_handler = NULL;
131 static const void *zend_jit_loop_trace_counter_handler = NULL;
132
133 static int ZEND_FASTCALL zend_runtime_jit(void);
134
135 static int zend_jit_trace_op_len(const zend_op *opline);
136 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
137 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags);
138 static const void *zend_jit_trace_get_exit_addr(uint32_t n);
139 static void zend_jit_trace_add_code(const void *start, uint32_t size);
140 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info);
141
142 #if ZEND_JIT_TARGET_ARM64
143 static zend_jit_trace_info *zend_jit_get_current_trace_info(void);
144 static uint32_t zend_jit_trace_find_exit_point(const void* addr);
145 #endif
146
147 #if ZEND_JIT_TARGET_X86 && defined(__linux__)
148 # if PHP_HAVE_BUILTIN_CPU_SUPPORTS && defined(__GNUC__) && (ZEND_GCC_VERSION >= 11000)
149 # define ZEND_JIT_SUPPORT_CLDEMOTE 1
150 # else
151 # define ZEND_JIT_SUPPORT_CLDEMOTE 0
152 # endif
153 #endif
154
155 #if ZEND_JIT_SUPPORT_CLDEMOTE
156 #include <immintrin.h>
157 #pragma GCC push_options
158 #pragma GCC target("cldemote")
159 // check cldemote by CPUID when JIT startup
160 static int cpu_support_cldemote = 0;
shared_cacheline_demote(uintptr_t start,size_t size)161 static inline void shared_cacheline_demote(uintptr_t start, size_t size) {
162 uintptr_t cache_line_base = start & ~0x3F;
163 do {
164 _cldemote((void *)cache_line_base);
165 // next cacheline start size
166 cache_line_base += 64;
167 } while (cache_line_base < start + size);
168 }
169 #pragma GCC pop_options
170 #endif
171
172 static int zend_jit_assign_to_variable(dasm_State **Dst,
173 const zend_op *opline,
174 zend_jit_addr var_use_addr,
175 zend_jit_addr var_addr,
176 uint32_t var_info,
177 uint32_t var_def_info,
178 uint8_t val_type,
179 zend_jit_addr val_addr,
180 uint32_t val_info,
181 zend_jit_addr res_addr,
182 bool check_exception);
183
dominates(const zend_basic_block * blocks,int a,int b)184 static bool dominates(const zend_basic_block *blocks, int a, int b) {
185 while (blocks[b].level > blocks[a].level) {
186 b = blocks[b].idom;
187 }
188 return a == b;
189 }
190
zend_ssa_is_last_use(const zend_op_array * op_array,const zend_ssa * ssa,int var,int use)191 static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
192 {
193 int next_use;
194
195 if (ssa->vars[var].phi_use_chain) {
196 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
197 do {
198 if (!ssa->vars[phi->ssa_var].no_val) {
199 return 0;
200 }
201 phi = zend_ssa_next_use_phi(ssa, var, phi);
202 } while (phi);
203 }
204
205 if (ssa->cfg.blocks[ssa->cfg.map[use]].loop_header > 0
206 || (ssa->cfg.blocks[ssa->cfg.map[use]].flags & ZEND_BB_LOOP_HEADER)) {
207 int b = ssa->cfg.map[use];
208 int prev_use = ssa->vars[var].use_chain;
209
210 while (prev_use >= 0 && prev_use != use) {
211 if (b != ssa->cfg.map[prev_use]
212 && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use])
213 && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) {
214 return 0;
215 }
216 prev_use = zend_ssa_next_use(ssa->ops, var, prev_use);
217 }
218 }
219
220 next_use = zend_ssa_next_use(ssa->ops, var, use);
221 if (next_use < 0) {
222 return 1;
223 } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) {
224 return 1;
225 }
226 return 0;
227 }
228
zend_ival_is_last_use(const zend_lifetime_interval * ival,int use)229 static bool zend_ival_is_last_use(const zend_lifetime_interval *ival, int use)
230 {
231 if (ival->flags & ZREG_LAST_USE) {
232 const zend_life_range *range = &ival->range;
233
234 while (range->next) {
235 range = range->next;
236 }
237 return range->end == use;
238 }
239 return 0;
240 }
241
zend_is_commutative(uint8_t opcode)242 static bool zend_is_commutative(uint8_t opcode)
243 {
244 return
245 opcode == ZEND_ADD ||
246 opcode == ZEND_MUL ||
247 opcode == ZEND_BW_OR ||
248 opcode == ZEND_BW_AND ||
249 opcode == ZEND_BW_XOR;
250 }
251
zend_jit_is_constant_cmp_long_long(const zend_op * opline,zend_ssa_range * op1_range,zend_jit_addr op1_addr,zend_ssa_range * op2_range,zend_jit_addr op2_addr,bool * result)252 static int zend_jit_is_constant_cmp_long_long(const zend_op *opline,
253 zend_ssa_range *op1_range,
254 zend_jit_addr op1_addr,
255 zend_ssa_range *op2_range,
256 zend_jit_addr op2_addr,
257 bool *result)
258 {
259 zend_long op1_min;
260 zend_long op1_max;
261 zend_long op2_min;
262 zend_long op2_max;
263
264 if (op1_range) {
265 op1_min = op1_range->min;
266 op1_max = op1_range->max;
267 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
268 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG);
269 op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr));
270 } else {
271 return 0;
272 }
273
274 if (op2_range) {
275 op2_min = op2_range->min;
276 op2_max = op2_range->max;
277 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
278 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG);
279 op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr));
280 } else {
281 return 0;
282 }
283
284 switch (opline->opcode) {
285 case ZEND_IS_EQUAL:
286 case ZEND_IS_IDENTICAL:
287 case ZEND_CASE:
288 case ZEND_CASE_STRICT:
289 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
290 *result = 1;
291 return 1;
292 } else if (op1_max < op2_min || op1_min > op2_max) {
293 *result = 0;
294 return 1;
295 }
296 return 0;
297 case ZEND_IS_NOT_EQUAL:
298 case ZEND_IS_NOT_IDENTICAL:
299 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
300 *result = 0;
301 return 1;
302 } else if (op1_max < op2_min || op1_min > op2_max) {
303 *result = 1;
304 return 1;
305 }
306 return 0;
307 case ZEND_IS_SMALLER:
308 if (op1_max < op2_min) {
309 *result = 1;
310 return 1;
311 } else if (op1_min >= op2_max) {
312 *result = 0;
313 return 1;
314 }
315 return 0;
316 case ZEND_IS_SMALLER_OR_EQUAL:
317 if (op1_max <= op2_min) {
318 *result = 1;
319 return 1;
320 } else if (op1_min > op2_max) {
321 *result = 0;
322 return 1;
323 }
324 return 0;
325 default:
326 ZEND_UNREACHABLE();
327 }
328 return 0;
329 }
330
zend_jit_needs_call_chain(zend_call_info * call_info,uint32_t b,const zend_op_array * op_array,zend_ssa * ssa,const zend_ssa_op * ssa_op,const zend_op * opline,int call_level,zend_jit_trace_rec * trace)331 static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace)
332 {
333 int skip;
334
335 if (trace) {
336 zend_jit_trace_rec *p = trace;
337
338 ssa_op++;
339 while (1) {
340 if (p->op == ZEND_JIT_TRACE_VM) {
341 switch (p->opline->opcode) {
342 case ZEND_SEND_ARRAY:
343 case ZEND_SEND_USER:
344 case ZEND_SEND_UNPACK:
345 case ZEND_INIT_FCALL:
346 case ZEND_INIT_METHOD_CALL:
347 case ZEND_INIT_STATIC_METHOD_CALL:
348 case ZEND_INIT_FCALL_BY_NAME:
349 case ZEND_INIT_NS_FCALL_BY_NAME:
350 case ZEND_INIT_DYNAMIC_CALL:
351 case ZEND_NEW:
352 case ZEND_INIT_USER_CALL:
353 case ZEND_FAST_CALL:
354 case ZEND_JMP:
355 case ZEND_JMPZ:
356 case ZEND_JMPNZ:
357 case ZEND_JMPZ_EX:
358 case ZEND_JMPNZ_EX:
359 case ZEND_FE_RESET_R:
360 case ZEND_FE_RESET_RW:
361 case ZEND_JMP_SET:
362 case ZEND_COALESCE:
363 case ZEND_JMP_NULL:
364 case ZEND_ASSERT_CHECK:
365 case ZEND_CATCH:
366 case ZEND_DECLARE_ANON_CLASS:
367 case ZEND_FE_FETCH_R:
368 case ZEND_FE_FETCH_RW:
369 case ZEND_BIND_INIT_STATIC_OR_JMP:
370 return 1;
371 case ZEND_DO_ICALL:
372 case ZEND_DO_UCALL:
373 case ZEND_DO_FCALL_BY_NAME:
374 case ZEND_DO_FCALL:
375 case ZEND_CALLABLE_CONVERT:
376 return 0;
377 case ZEND_SEND_VAL:
378 case ZEND_SEND_VAR:
379 case ZEND_SEND_VAL_EX:
380 case ZEND_SEND_VAR_EX:
381 case ZEND_SEND_FUNC_ARG:
382 case ZEND_SEND_REF:
383 case ZEND_SEND_VAR_NO_REF:
384 case ZEND_SEND_VAR_NO_REF_EX:
385 /* skip */
386 break;
387 default:
388 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
389 return 1;
390 }
391 }
392 ssa_op += zend_jit_trace_op_len(opline);
393 } else if (p->op == ZEND_JIT_TRACE_ENTER ||
394 p->op == ZEND_JIT_TRACE_BACK ||
395 p->op == ZEND_JIT_TRACE_END) {
396 return 1;
397 }
398 p++;
399 }
400 }
401
402 if (!call_info) {
403 const zend_op *end = op_array->opcodes + op_array->last;
404
405 opline++;
406 ssa_op++;
407 skip = (call_level == 1);
408 while (opline != end) {
409 if (!skip) {
410 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
411 return 1;
412 }
413 }
414 switch (opline->opcode) {
415 case ZEND_SEND_VAL:
416 case ZEND_SEND_VAR:
417 case ZEND_SEND_VAL_EX:
418 case ZEND_SEND_VAR_EX:
419 case ZEND_SEND_FUNC_ARG:
420 case ZEND_SEND_REF:
421 case ZEND_SEND_VAR_NO_REF:
422 case ZEND_SEND_VAR_NO_REF_EX:
423 skip = 0;
424 break;
425 case ZEND_SEND_ARRAY:
426 case ZEND_SEND_USER:
427 case ZEND_SEND_UNPACK:
428 case ZEND_INIT_FCALL:
429 case ZEND_INIT_METHOD_CALL:
430 case ZEND_INIT_STATIC_METHOD_CALL:
431 case ZEND_INIT_FCALL_BY_NAME:
432 case ZEND_INIT_NS_FCALL_BY_NAME:
433 case ZEND_INIT_DYNAMIC_CALL:
434 case ZEND_NEW:
435 case ZEND_INIT_USER_CALL:
436 case ZEND_FAST_CALL:
437 case ZEND_JMP:
438 case ZEND_JMPZ:
439 case ZEND_JMPNZ:
440 case ZEND_JMPZ_EX:
441 case ZEND_JMPNZ_EX:
442 case ZEND_FE_RESET_R:
443 case ZEND_FE_RESET_RW:
444 case ZEND_JMP_SET:
445 case ZEND_COALESCE:
446 case ZEND_JMP_NULL:
447 case ZEND_ASSERT_CHECK:
448 case ZEND_CATCH:
449 case ZEND_DECLARE_ANON_CLASS:
450 case ZEND_FE_FETCH_R:
451 case ZEND_FE_FETCH_RW:
452 case ZEND_BIND_INIT_STATIC_OR_JMP:
453 return 1;
454 case ZEND_DO_ICALL:
455 case ZEND_DO_UCALL:
456 case ZEND_DO_FCALL_BY_NAME:
457 case ZEND_DO_FCALL:
458 case ZEND_CALLABLE_CONVERT:
459 end = opline;
460 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
461 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
462 return 1;
463 }
464 return 0;
465 }
466 opline++;
467 ssa_op++;
468 }
469
470 return 1;
471 } else {
472 const zend_op *end = call_info->caller_call_opline;
473
474 /* end may be null if an opcode like EXIT is part of the argument list. */
475 if (!end || end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
476 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
477 return 1;
478 }
479
480 opline++;
481 ssa_op++;
482 skip = (call_level == 1);
483 while (opline != end) {
484 if (skip) {
485 switch (opline->opcode) {
486 case ZEND_SEND_VAL:
487 case ZEND_SEND_VAR:
488 case ZEND_SEND_VAL_EX:
489 case ZEND_SEND_VAR_EX:
490 case ZEND_SEND_FUNC_ARG:
491 case ZEND_SEND_REF:
492 case ZEND_SEND_VAR_NO_REF:
493 case ZEND_SEND_VAR_NO_REF_EX:
494 skip = 0;
495 break;
496 case ZEND_SEND_ARRAY:
497 case ZEND_SEND_USER:
498 case ZEND_SEND_UNPACK:
499 return 1;
500 }
501 } else {
502 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
503 return 1;
504 }
505 }
506 opline++;
507 ssa_op++;
508 }
509
510 return 0;
511 }
512 }
513
skip_valid_arguments(const zend_op_array * op_array,zend_ssa * ssa,const zend_call_info * call_info)514 static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
515 {
516 uint32_t num_args = 0;
517 zend_function *func = call_info->callee_func;
518
519 /* It's okay to handle prototypes here, because they can only increase the accepted arguments.
520 * Anything legal for the parent method is also legal for the parent method. */
521 while (num_args < call_info->num_args) {
522 zend_arg_info *arg_info = func->op_array.arg_info + num_args;
523
524 if (ZEND_TYPE_IS_SET(arg_info->type)) {
525 if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) {
526 zend_op *opline = call_info->arg_info[num_args].opline;
527 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
528 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
529 if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) {
530 break;
531 }
532 } else {
533 break;
534 }
535 }
536 num_args++;
537 }
538 return num_args;
539 }
540
zend_ssa_cv_info(const zend_op_array * op_array,zend_ssa * ssa,uint32_t var)541 static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var)
542 {
543 uint32_t j, info;
544
545 if (ssa->vars && ssa->var_info) {
546 info = ssa->var_info[var].type;
547 for (j = op_array->last_var; j < ssa->vars_count; j++) {
548 if (ssa->vars[j].var == var) {
549 info |= ssa->var_info[j].type;
550 }
551 }
552 } else {
553 info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF |
554 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
555 }
556
557 #ifdef ZEND_JIT_USE_RC_INFERENCE
558 /* Refcount may be increased by RETURN opcode */
559 if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
560 for (j = 0; j < ssa->cfg.blocks_count; j++) {
561 if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
562 ssa->cfg.blocks[j].len > 0) {
563 const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
564
565 if (opline->opcode == ZEND_RETURN) {
566 if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) {
567 info |= MAY_BE_RCN;
568 break;
569 }
570 }
571 }
572 }
573 }
574 #endif
575
576 return info;
577 }
578
zend_jit_may_avoid_refcounting(const zend_op * opline,uint32_t op1_info)579 static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_info)
580 {
581 switch (opline->opcode) {
582 case ZEND_FETCH_OBJ_FUNC_ARG:
583 if (!JIT_G(current_frame) ||
584 !JIT_G(current_frame)->call->func ||
585 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
586 return 0;
587 }
588 /* break missing intentionally */
589 case ZEND_FETCH_OBJ_R:
590 case ZEND_FETCH_OBJ_IS:
591 if ((op1_info & MAY_BE_OBJECT)
592 && opline->op2_type == IS_CONST
593 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
594 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
595 return 1;
596 }
597 break;
598 case ZEND_FETCH_DIM_FUNC_ARG:
599 if (!JIT_G(current_frame) ||
600 !JIT_G(current_frame)->call->func ||
601 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
602 return 0;
603 }
604 /* break missing intentionally */
605 case ZEND_FETCH_DIM_R:
606 case ZEND_FETCH_DIM_IS:
607 return 1;
608 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
609 if (!(opline->extended_value & ZEND_ISEMPTY)) {
610 return 1;
611 }
612 break;
613 }
614 return 0;
615 }
616
zend_jit_is_persistent_constant(zval * key,uint32_t flags)617 static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags)
618 {
619 zval *zv;
620 zend_constant *c = NULL;
621
622 /* null/true/false are resolved during compilation, so don't check for them here. */
623 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
624 if (zv) {
625 c = (zend_constant*)Z_PTR_P(zv);
626 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
627 key++;
628 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
629 if (zv) {
630 c = (zend_constant*)Z_PTR_P(zv);
631 }
632 }
633 return c && (ZEND_CONSTANT_FLAGS(c) & CONST_PERSISTENT);
634 }
635
zend_get_known_property_info(const zend_op_array * op_array,zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)636 static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
637 {
638 zend_property_info *info = NULL;
639
640 if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) ||
641 !ce ||
642 !(ce->ce_flags & ZEND_ACC_LINKED) ||
643 (ce->ce_flags & ZEND_ACC_TRAIT) ||
644 ce->create_object) {
645 return NULL;
646 }
647
648 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
649 if (ce->info.user.filename != filename) {
650 /* class declaration might be changed independently */
651 return NULL;
652 }
653
654 if (ce->parent) {
655 zend_class_entry *parent = ce->parent;
656
657 do {
658 if (parent->type == ZEND_INTERNAL_CLASS) {
659 break;
660 } else if (parent->info.user.filename != filename) {
661 /* some of parents class declarations might be changed independently */
662 /* TODO: this check may be not enough, because even
663 * in the same it's possible to conditionally define
664 * few classes with the same name, and "parent" may
665 * change from request to request.
666 */
667 return NULL;
668 }
669 parent = parent->parent;
670 } while (parent);
671 }
672 }
673
674 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
675 if (info == NULL ||
676 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
677 (info->flags & ZEND_ACC_STATIC)) {
678 return NULL;
679 }
680
681 if (info->flags & ZEND_ACC_PUBLIC) {
682 return info;
683 } else if (on_this) {
684 if (ce == info->ce) {
685 if (ce == op_array->scope) {
686 return info;
687 } else {
688 return NULL;
689 }
690 } else if ((info->flags & ZEND_ACC_PROTECTED)
691 && instanceof_function_slow(ce, info->ce)) {
692 return info;
693 }
694 }
695
696 return NULL;
697 }
698
zend_may_be_dynamic_property(zend_class_entry * ce,zend_string * member,bool on_this,const zend_op_array * op_array)699 static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, const zend_op_array *op_array)
700 {
701 zend_property_info *info;
702
703 if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT) || (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) {
704 return 1;
705 }
706
707 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
708 if (ce->info.user.filename != op_array->filename) {
709 /* class declaration might be changed independently */
710 return 1;
711 }
712 }
713
714 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
715 if (info == NULL ||
716 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
717 (info->flags & ZEND_ACC_STATIC)) {
718 return 1;
719 }
720
721 if (!(info->flags & ZEND_ACC_PUBLIC) &&
722 (!on_this || info->ce != ce)) {
723 return 1;
724 }
725
726 return 0;
727 }
728
729 #define OP_RANGE(ssa_op, opN) \
730 (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
731 ssa->var_info && \
732 (ssa_op)->opN##_use >= 0 && \
733 ssa->var_info[(ssa_op)->opN##_use].has_range) ? \
734 &ssa->var_info[(ssa_op)->opN##_use].range : NULL)
735
736 #define OP1_RANGE() OP_RANGE(ssa_op, op1)
737 #define OP2_RANGE() OP_RANGE(ssa_op, op2)
738 #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)
739
740 #if ZEND_JIT_TARGET_X86
741 # include "dynasm/dasm_x86.h"
742 #elif ZEND_JIT_TARGET_ARM64
743 static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset);
744 # define DASM_ADD_VENEER zend_jit_add_veneer
745 # include "dynasm/dasm_arm64.h"
746 #endif
747
748 #include "jit/zend_jit_helpers.c"
749 #include "jit/zend_jit_disasm.c"
750 #ifndef _WIN32
751 # include "jit/zend_jit_gdb.h"
752 # include "jit/zend_jit_perf_dump.c"
753 #endif
754
755 #include "Zend/zend_cpuinfo.h"
756
757 #ifdef HAVE_VALGRIND
758 # include <valgrind/valgrind.h>
759 #endif
760
761 #ifdef HAVE_GCC_GLOBAL_REGS
762 # define GCC_GLOBAL_REGS 1
763 #else
764 # define GCC_GLOBAL_REGS 0
765 #endif
766
767 /* By default avoid JITing inline handlers if it does not seem profitable due to lack of
768 * type information. Disabling this option allows testing some JIT handlers in the
769 * presence of try/catch blocks, which prevent SSA construction. */
770 #ifndef PROFITABILITY_CHECKS
771 # define PROFITABILITY_CHECKS 1
772 #endif
773
774 #define BP_JIT_IS 6 /* Used for ISSET_ISEMPTY_DIM_OBJ. see BP_VAR_*defines in Zend/zend_compile.h */
775
776 typedef enum _sp_adj_kind {
777 SP_ADJ_NONE,
778 SP_ADJ_RET,
779 SP_ADJ_VM,
780 SP_ADJ_JIT,
781 SP_ADJ_ASSIGN,
782 SP_ADJ_LAST
783 } sp_adj_kind;
784
785 static int sp_adj[SP_ADJ_LAST];
786
787 /* The generated code may contain tautological comparisons, ignore them. */
788 #if defined(__clang__)
789 # pragma clang diagnostic push
790 # pragma clang diagnostic ignored "-Wtautological-compare"
791 # pragma clang diagnostic ignored "-Wstring-compare"
792 #endif
793
794 #if ZEND_JIT_TARGET_X86
795 # include "jit/zend_jit_vtune.c"
796 # include "jit/zend_jit_x86.c"
797 #elif ZEND_JIT_TARGET_ARM64
798 # include "jit/zend_jit_arm64.c"
799 #endif
800
801 #if defined(__clang__)
802 # pragma clang diagnostic pop
803 #endif
804
805 #if _WIN32
806 # include <Windows.h>
807 #else
808 # include <sys/mman.h>
809 # if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
810 # define MAP_ANONYMOUS MAP_ANON
811 # endif
812 #endif
813
zend_jit_status(zval * ret)814 ZEND_EXT_API void zend_jit_status(zval *ret)
815 {
816 zval stats;
817 array_init(&stats);
818 add_assoc_bool(&stats, "enabled", JIT_G(enabled));
819 add_assoc_bool(&stats, "on", JIT_G(on));
820 add_assoc_long(&stats, "kind", JIT_G(trigger));
821 add_assoc_long(&stats, "opt_level", JIT_G(opt_level));
822 add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags));
823 if (dasm_buf) {
824 add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
825 add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
826 } else {
827 add_assoc_long(&stats, "buffer_size", 0);
828 add_assoc_long(&stats, "buffer_free", 0);
829 }
830 add_assoc_zval(ret, "jit", &stats);
831 }
832
zend_jit_func_name(const zend_op_array * op_array)833 static zend_string *zend_jit_func_name(const zend_op_array *op_array)
834 {
835 smart_str buf = {0};
836
837 if (op_array->function_name) {
838 if (op_array->scope) {
839 smart_str_appends(&buf, JIT_PREFIX);
840 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
841 smart_str_appends(&buf, "::");
842 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
843 smart_str_0(&buf);
844 return buf.s;
845 } else {
846 smart_str_appends(&buf, JIT_PREFIX);
847 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
848 smart_str_0(&buf);
849 return buf.s;
850 }
851 } else if (op_array->filename) {
852 smart_str_appends(&buf, JIT_PREFIX);
853 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
854 smart_str_0(&buf);
855 return buf.s;
856 } else {
857 return NULL;
858 }
859 }
860
861 #if ZEND_DEBUG
handle_dasm_error(int ret)862 static void handle_dasm_error(int ret) {
863 switch (ret & 0xff000000u) {
864 case DASM_S_NOMEM:
865 fprintf(stderr, "DASM_S_NOMEM\n");
866 break;
867 case DASM_S_PHASE:
868 fprintf(stderr, "DASM_S_PHASE\n");
869 break;
870 case DASM_S_MATCH_SEC:
871 fprintf(stderr, "DASM_S_MATCH_SEC\n");
872 break;
873 case DASM_S_RANGE_I:
874 fprintf(stderr, "DASM_S_RANGE_I\n");
875 break;
876 case DASM_S_RANGE_SEC:
877 fprintf(stderr, "DASM_S_RANGE_SEC\n");
878 break;
879 case DASM_S_RANGE_LG:
880 fprintf(stderr, "DASM_S_RANGE_LG\n");
881 break;
882 case DASM_S_RANGE_PC:
883 fprintf(stderr, "DASM_S_RANGE_PC %d\n", ret & 0xffffffu);
884 break;
885 #ifdef DASM_S_RANGE_VREG
886 case DASM_S_RANGE_VREG:
887 fprintf(stderr, "DASM_S_RANGE_VREG\n");
888 break;
889 #endif
890 #ifdef DASM_S_UNDEF_L
891 case DASM_S_UNDEF_L:
892 fprintf(stderr, "DASM_S_UNDEF_L\n");
893 break;
894 #endif
895 #ifdef DASM_S_UNDEF_LG
896 case DASM_S_UNDEF_LG:
897 fprintf(stderr, "DASM_S_UNDEF_LG\n");
898 break;
899 #endif
900 #ifdef DASM_S_RANGE_REL
901 case DASM_S_RANGE_REL:
902 fprintf(stderr, "DASM_S_RANGE_REL\n");
903 break;
904 #endif
905 case DASM_S_UNDEF_PC:
906 fprintf(stderr, "DASM_S_UNDEF_PC %d\n", ret & 0xffffffu);
907 break;
908 default:
909 fprintf(stderr, "DASM_S_%0x\n", ret & 0xff000000u);
910 break;
911 }
912 ZEND_UNREACHABLE();
913 }
914 #endif
915
dasm_link_and_encode(dasm_State ** dasm_state,const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline,zend_lifetime_interval ** ra,const char * name,uint32_t trace_num,uint32_t sp_offset,uint32_t sp_adjustment)916 static void *dasm_link_and_encode(dasm_State **dasm_state,
917 const zend_op_array *op_array,
918 zend_ssa *ssa,
919 const zend_op *rt_opline,
920 zend_lifetime_interval **ra,
921 const char *name,
922 uint32_t trace_num,
923 uint32_t sp_offset,
924 uint32_t sp_adjustment)
925 {
926 size_t size;
927 int ret;
928 void *entry;
929 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
930 zend_string *str = NULL;
931 #endif
932
933 if (rt_opline && ssa && ssa->cfg.map) {
934 /* Create additional entry point, to switch from interpreter to JIT-ed
935 * code at run-time.
936 */
937 int b = ssa->cfg.map[rt_opline - op_array->opcodes];
938
939 //#ifdef CONTEXT_THREADED_JIT
940 // if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY))) {
941 //#else
942 if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY))) {
943 //#endif
944 zend_jit_label(dasm_state, ssa->cfg.blocks_count + b);
945 zend_jit_prologue(dasm_state);
946 if (ra) {
947 int i;
948 zend_lifetime_interval *ival;
949 zend_life_range *range;
950 uint32_t pos = rt_opline - op_array->opcodes;
951
952 for (i = 0; i < ssa->vars_count; i++) {
953 ival = ra[i];
954
955 if (ival && ival->reg != ZREG_NONE) {
956 range = &ival->range;
957
958 if (pos >= range->start && pos <= range->end) {
959 if (!zend_jit_load_var(dasm_state, ssa->var_info[i].type, ssa->vars[i].var, ival->reg)) {
960 return NULL;
961 }
962 break;
963 }
964 range = range->next;
965 }
966 }
967 }
968 zend_jit_jmp(dasm_state, b);
969 }
970 }
971
972 ret = dasm_link(dasm_state, &size);
973 if (ret != DASM_S_OK) {
974 #if ZEND_DEBUG
975 handle_dasm_error(ret);
976 #endif
977 return NULL;
978 }
979
980 if ((void*)((char*)*dasm_ptr + size) > dasm_end) {
981 *dasm_ptr = dasm_end; //prevent further try
982 // TODO: jit_buffer_size overflow ???
983 return NULL;
984 }
985
986 #if ZEND_JIT_TARGET_ARM64
987 dasm_venners_size = 0;
988 #endif
989
990 ret = dasm_encode(dasm_state, *dasm_ptr);
991 if (ret != DASM_S_OK) {
992 #if ZEND_DEBUG
993 handle_dasm_error(ret);
994 #endif
995 return NULL;
996 }
997
998 #if ZEND_JIT_TARGET_ARM64
999 size += dasm_venners_size;
1000 #endif
1001
1002 entry = *dasm_ptr;
1003 *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT));
1004
1005 /* flush the hardware I-cache */
1006 JIT_CACHE_FLUSH(entry, entry + size);
1007 /* hint to the hardware to push out the cache line that contains the linear address */
1008 #if ZEND_JIT_SUPPORT_CLDEMOTE
1009 if (cpu_support_cldemote && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
1010 shared_cacheline_demote((uintptr_t)entry, size);
1011 }
1012 #endif
1013
1014 if (trace_num) {
1015 zend_jit_trace_add_code(entry, dasm_getpclabel(dasm_state, 1));
1016 }
1017
1018 if (op_array && ssa) {
1019 int b;
1020
1021 for (b = 0; b < ssa->cfg.blocks_count; b++) {
1022 //#ifdef CONTEXT_THREADED_JIT
1023 // if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
1024 //#else
1025 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) {
1026 //#endif
1027 zend_op *opline = op_array->opcodes + ssa->cfg.blocks[b].start;
1028 int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
1029
1030 if (offset >= 0) {
1031 opline->handler = (void*)(((char*)entry) + offset);
1032 }
1033 }
1034 }
1035 if (rt_opline && ssa && ssa->cfg.map) {
1036 int b = ssa->cfg.map[rt_opline - op_array->opcodes];
1037 zend_op *opline = (zend_op*)rt_opline;
1038 int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
1039
1040 if (offset >= 0) {
1041 opline->handler = (void*)(((char*)entry) + offset);
1042 }
1043 }
1044 }
1045
1046 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
1047 if (!name) {
1048 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) {
1049 str = zend_jit_func_name(op_array);
1050 if (str) {
1051 name = ZSTR_VAL(str);
1052 }
1053 }
1054 #ifdef HAVE_DISASM
1055 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
1056 zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
1057 zend_jit_disasm(
1058 name,
1059 (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
1060 op_array,
1061 &ssa->cfg,
1062 entry,
1063 size);
1064 }
1065 } else {
1066 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) {
1067 zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
1068 if ((JIT_G(debug) & (trace_num ? ZEND_JIT_DEBUG_ASM : ZEND_JIT_DEBUG_ASM_STUBS)) != 0) {
1069 zend_jit_disasm(
1070 name,
1071 (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
1072 op_array,
1073 ssa ? &ssa->cfg : NULL,
1074 entry,
1075 size);
1076 }
1077 }
1078 # endif
1079 }
1080 #endif
1081
1082 #ifdef HAVE_GDB
1083 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
1084 if (name) {
1085 zend_jit_gdb_register(
1086 name,
1087 op_array,
1088 entry,
1089 size,
1090 sp_adj[sp_offset],
1091 sp_adj[sp_adjustment]);
1092 }
1093 }
1094 #endif
1095
1096
1097 #ifdef HAVE_PERFTOOLS
1098 if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
1099 if (name) {
1100 zend_jit_perf_map_register(
1101 name,
1102 entry,
1103 size);
1104 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
1105 zend_jit_perf_jitdump_register(
1106 name,
1107 entry,
1108 size);
1109 }
1110 }
1111 }
1112 #endif
1113
1114 #ifdef HAVE_VTUNE
1115 if (JIT_G(debug) & ZEND_JIT_DEBUG_VTUNE) {
1116 if (name) {
1117 zend_jit_vtune_register(
1118 name,
1119 entry,
1120 size);
1121 }
1122 }
1123 #endif
1124
1125 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
1126 if (str) {
1127 zend_string_release(str);
1128 }
1129 #endif
1130
1131 return entry;
1132 }
1133
zend_may_overflow(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa)1134 static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
1135 {
1136 int res;
1137 zend_long op1_min, op1_max, op2_min, op2_max;
1138
1139 if (!ssa->ops || !ssa->var_info) {
1140 return 1;
1141 }
1142 switch (opline->opcode) {
1143 case ZEND_PRE_INC:
1144 case ZEND_POST_INC:
1145 res = ssa_op->op1_def;
1146 if (res < 0
1147 || !ssa->var_info[res].has_range
1148 || ssa->var_info[res].range.overflow) {
1149 if (!OP1_HAS_RANGE()) {
1150 return 1;
1151 }
1152 op1_max = OP1_MAX_RANGE();
1153 if (op1_max == ZEND_LONG_MAX) {
1154 return 1;
1155 }
1156 }
1157 return 0;
1158 case ZEND_PRE_DEC:
1159 case ZEND_POST_DEC:
1160 res = ssa_op->op1_def;
1161 if (res < 0
1162 || !ssa->var_info[res].has_range
1163 || ssa->var_info[res].range.underflow) {
1164 if (!OP1_HAS_RANGE()) {
1165 return 1;
1166 }
1167 op1_min = OP1_MIN_RANGE();
1168 if (op1_min == ZEND_LONG_MIN) {
1169 return 1;
1170 }
1171 }
1172 return 0;
1173 case ZEND_ADD:
1174 res = ssa_op->result_def;
1175 if (res < 0
1176 || !ssa->var_info[res].has_range
1177 || ssa->var_info[res].range.underflow) {
1178 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1179 return 1;
1180 }
1181 op1_min = OP1_MIN_RANGE();
1182 op2_min = OP2_MIN_RANGE();
1183 if (zend_add_will_overflow(op1_min, op2_min)) {
1184 return 1;
1185 }
1186 }
1187 if (res < 0
1188 || !ssa->var_info[res].has_range
1189 || ssa->var_info[res].range.overflow) {
1190 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1191 return 1;
1192 }
1193 op1_max = OP1_MAX_RANGE();
1194 op2_max = OP2_MAX_RANGE();
1195 if (zend_add_will_overflow(op1_max, op2_max)) {
1196 return 1;
1197 }
1198 }
1199 return 0;
1200 case ZEND_SUB:
1201 res = ssa_op->result_def;
1202 if (res < 0
1203 || !ssa->var_info[res].has_range
1204 || ssa->var_info[res].range.underflow) {
1205 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1206 return 1;
1207 }
1208 op1_min = OP1_MIN_RANGE();
1209 op2_max = OP2_MAX_RANGE();
1210 if (zend_sub_will_overflow(op1_min, op2_max)) {
1211 return 1;
1212 }
1213 }
1214 if (res < 0
1215 || !ssa->var_info[res].has_range
1216 || ssa->var_info[res].range.overflow) {
1217 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1218 return 1;
1219 }
1220 op1_max = OP1_MAX_RANGE();
1221 op2_min = OP2_MIN_RANGE();
1222 if (zend_sub_will_overflow(op1_max, op2_min)) {
1223 return 1;
1224 }
1225 }
1226 return 0;
1227 case ZEND_MUL:
1228 res = ssa_op->result_def;
1229 return (res < 0 ||
1230 !ssa->var_info[res].has_range ||
1231 ssa->var_info[res].range.underflow ||
1232 ssa->var_info[res].range.overflow);
1233 case ZEND_ASSIGN_OP:
1234 if (opline->extended_value == ZEND_ADD) {
1235 res = ssa_op->op1_def;
1236 if (res < 0
1237 || !ssa->var_info[res].has_range
1238 || ssa->var_info[res].range.underflow) {
1239 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1240 return 1;
1241 }
1242 op1_min = OP1_MIN_RANGE();
1243 op2_min = OP2_MIN_RANGE();
1244 if (zend_add_will_overflow(op1_min, op2_min)) {
1245 return 1;
1246 }
1247 }
1248 if (res < 0
1249 || !ssa->var_info[res].has_range
1250 || ssa->var_info[res].range.overflow) {
1251 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1252 return 1;
1253 }
1254 op1_max = OP1_MAX_RANGE();
1255 op2_max = OP2_MAX_RANGE();
1256 if (zend_add_will_overflow(op1_max, op2_max)) {
1257 return 1;
1258 }
1259 }
1260 return 0;
1261 } else if (opline->extended_value == ZEND_SUB) {
1262 res = ssa_op->op1_def;
1263 if (res < 0
1264 || !ssa->var_info[res].has_range
1265 || ssa->var_info[res].range.underflow) {
1266 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1267 return 1;
1268 }
1269 op1_min = OP1_MIN_RANGE();
1270 op2_max = OP2_MAX_RANGE();
1271 if (zend_sub_will_overflow(op1_min, op2_max)) {
1272 return 1;
1273 }
1274 }
1275 if (res < 0
1276 || !ssa->var_info[res].has_range
1277 || ssa->var_info[res].range.overflow) {
1278 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1279 return 1;
1280 }
1281 op1_max = OP1_MAX_RANGE();
1282 op2_min = OP2_MIN_RANGE();
1283 if (zend_sub_will_overflow(op1_max, op2_min)) {
1284 return 1;
1285 }
1286 }
1287 return 0;
1288 } else if (opline->extended_value == ZEND_MUL) {
1289 res = ssa_op->op1_def;
1290 return (res < 0 ||
1291 !ssa->var_info[res].has_range ||
1292 ssa->var_info[res].range.underflow ||
1293 ssa->var_info[res].range.overflow);
1294 }
1295 ZEND_FALLTHROUGH;
1296 default:
1297 return 1;
1298 }
1299 }
1300
zend_jit_build_cfg(const zend_op_array * op_array,zend_cfg * cfg)1301 static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg)
1302 {
1303 uint32_t flags;
1304
1305 flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY;
1306
1307 zend_build_cfg(&CG(arena), op_array, flags, cfg);
1308
1309 /* Don't JIT huge functions. Apart from likely being detrimental due to the amount of
1310 * generated code, some of our analysis is recursive and will stack overflow with many
1311 * blocks. */
1312 if (cfg->blocks_count > 100000) {
1313 return FAILURE;
1314 }
1315
1316 zend_cfg_build_predecessors(&CG(arena), cfg);
1317
1318 /* Compute Dominators Tree */
1319 zend_cfg_compute_dominators_tree(op_array, cfg);
1320
1321 /* Identify reducible and irreducible loops */
1322 zend_cfg_identify_loops(op_array, cfg);
1323
1324 return SUCCESS;
1325 }
1326
zend_jit_op_array_analyze1(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa)1327 static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
1328 {
1329 if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) {
1330 return FAILURE;
1331 }
1332
1333 #if 0
1334 /* TODO: debugger and profiler supports? */
1335 if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) {
1336 return FAILURE;
1337 }
1338 #endif
1339
1340 /* TODO: move this to zend_cfg.c ? */
1341 if (!op_array->function_name) {
1342 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1343 }
1344
1345 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
1346 && ssa->cfg.blocks
1347 && op_array->last_try_catch == 0
1348 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
1349 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
1350 if (zend_build_ssa(&CG(arena), script, op_array, ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) {
1351 return FAILURE;
1352 }
1353
1354 zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa);
1355
1356 zend_ssa_find_false_dependencies(op_array, ssa);
1357
1358 zend_ssa_find_sccs(op_array, ssa);
1359 }
1360
1361 return SUCCESS;
1362 }
1363
zend_jit_op_array_analyze2(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa,uint32_t optimization_level)1364 static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t optimization_level)
1365 {
1366 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
1367 && ssa->cfg.blocks
1368 && op_array->last_try_catch == 0
1369 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
1370 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
1371 if (zend_ssa_inference(&CG(arena), op_array, script, ssa,
1372 optimization_level & ~ZEND_OPTIMIZER_NARROW_TO_DOUBLE) != SUCCESS) {
1373 return FAILURE;
1374 }
1375 }
1376
1377 return SUCCESS;
1378 }
1379
zend_jit_add_range(zend_lifetime_interval ** intervals,int var,uint32_t from,uint32_t to)1380 static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint32_t from, uint32_t to)
1381 {
1382 zend_lifetime_interval *ival = intervals[var];
1383
1384 if (!ival) {
1385 ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
1386 if (!ival) {
1387 return FAILURE;
1388 }
1389 ival->ssa_var = var;
1390 ival->reg = ZREG_NONE;
1391 ival->flags = 0;
1392 ival->range.start = from;
1393 ival->range.end = to;
1394 ival->range.next = NULL;
1395 ival->hint = NULL;
1396 ival->used_as_hint = NULL;
1397 intervals[var] = ival;
1398 } else if (ival->range.start > to + 1) {
1399 zend_life_range *range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1400
1401 if (!range) {
1402 return FAILURE;
1403 }
1404 range->start = ival->range.start;
1405 range->end = ival->range.end;
1406 range->next = ival->range.next;
1407 ival->range.start = from;
1408 ival->range.end = to;
1409 ival->range.next = range;
1410 } else if (ival->range.start == to + 1) {
1411 ival->range.start = from;
1412 } else {
1413 zend_life_range *range = &ival->range;
1414 zend_life_range *last = NULL;
1415
1416 do {
1417 if (range->start > to + 1) {
1418 break;
1419 } else if (range->end + 1 >= from) {
1420 if (range->start > from) {
1421 range->start = from;
1422 }
1423 last = range;
1424 range = range->next;
1425 while (range) {
1426 if (range->start > to + 1) {
1427 break;
1428 }
1429 last->end = range->end;
1430 range = range->next;
1431 last->next = range;
1432 }
1433 if (to > last->end) {
1434 last->end = to;
1435 }
1436 return SUCCESS;
1437 }
1438 last = range;
1439 range = range->next;
1440 } while (range);
1441
1442 range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1443 if (!range) {
1444 return FAILURE;
1445 }
1446 range->start = from;
1447 range->end = to;
1448 range->next = last->next;
1449 last->next = range;
1450 }
1451
1452 return SUCCESS;
1453 }
1454
zend_jit_begin_range(zend_lifetime_interval ** intervals,int var,uint32_t block_start,uint32_t from)1455 static int zend_jit_begin_range(zend_lifetime_interval **intervals, int var, uint32_t block_start, uint32_t from)
1456 {
1457 if (block_start != from && intervals[var]) {
1458 zend_life_range *range = &intervals[var]->range;
1459
1460 do {
1461 if (from >= range->start && from <= range->end) {
1462 if (range->start == block_start) {
1463 range->start = from;
1464 } else {
1465 zend_life_range *r = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
1466 if (!r) {
1467 return FAILURE;
1468 }
1469 r->start = from;
1470 r->end = range->end;
1471 r->next = range->next;
1472 range->end = block_start - 1;
1473 range->next = r;
1474 }
1475 return SUCCESS;
1476 }
1477 range = range->next;
1478 } while (range);
1479 }
1480
1481 // dead store
1482 return zend_jit_add_range(intervals, var, from, from);
1483 }
1484
zend_jit_insert_interval(zend_lifetime_interval ** list,zend_lifetime_interval * ival)1485 static void zend_jit_insert_interval(zend_lifetime_interval **list, zend_lifetime_interval *ival)
1486 {
1487 while (1) {
1488 if (*list == NULL) {
1489 *list = ival;
1490 ival->list_next = NULL;
1491 return;
1492 } else if (ival->range.start < (*list)->range.start) {
1493 ival->list_next = *list;
1494 *list = ival;
1495 return;
1496 }
1497 list = &(*list)->list_next;
1498 }
1499 }
1500
zend_jit_split_interval(zend_lifetime_interval * current,uint32_t pos,zend_lifetime_interval ** list,zend_lifetime_interval ** free)1501 static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos, zend_lifetime_interval **list, zend_lifetime_interval **free)
1502 {
1503 zend_lifetime_interval *ival;
1504 zend_life_range *range = ¤t->range;
1505 zend_life_range *prev = NULL;
1506
1507 if (*free) {
1508 ival = *free;
1509 *free = ival->list_next;
1510 } else {
1511 ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
1512
1513 if (!ival) {
1514 return FAILURE;
1515 }
1516 }
1517
1518 current->flags |= ZREG_STORE;
1519
1520 ival->ssa_var = current->ssa_var;
1521 ival->reg = ZREG_NONE;
1522 ival->flags |= ZREG_SPLIT | ZREG_LOAD;
1523 ival->flags &= ~ZREG_STORE;
1524 ival->hint = NULL;
1525
1526 do {
1527 if (pos >= range->start && pos <= range->end) {
1528 break;
1529 }
1530 prev = range;
1531 range = range->next;
1532 } while(range);
1533
1534 ZEND_ASSERT(range != NULL);
1535
1536 ival->range.start = pos;
1537 ival->range.end = range->end;
1538 ival->range.next = range->next;
1539
1540 if (pos == range->start) {
1541 ZEND_ASSERT(prev != NULL);
1542 prev->next = NULL;
1543 } else {
1544 range->end = pos - 1;
1545 }
1546
1547 zend_jit_insert_interval(list, ival);
1548
1549 return SUCCESS;
1550 }
1551
zend_jit_sort_intervals(zend_lifetime_interval ** intervals,int count)1552 static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval **intervals, int count)
1553 {
1554 zend_lifetime_interval *list, *last;
1555 int i;
1556
1557 list = NULL;
1558 i = 0;
1559 while (i < count) {
1560 list = intervals[i];
1561 i++;
1562 if (list) {
1563 last = list;
1564 last->list_next = NULL;
1565 break;
1566 }
1567 }
1568
1569 while (i < count) {
1570 zend_lifetime_interval *ival = intervals[i];
1571
1572 i++;
1573 if (ival) {
1574 if ((ival->range.start > last->range.start) ||
1575 (ival->range.start == last->range.start &&
1576 ((!ival->hint && last->hint && last->hint != ival) ||
1577 ival->range.end > last->range.end))) {
1578 last->list_next = ival;
1579 last = ival;
1580 ival->list_next = NULL;
1581 } else {
1582 zend_lifetime_interval **p = &list;
1583
1584 while (1) {
1585 if (*p == NULL) {
1586 *p = last = ival;
1587 ival->list_next = NULL;
1588 break;
1589 } else if ((ival->range.start < (*p)->range.start) ||
1590 (ival->range.start == (*p)->range.start &&
1591 ((ival->hint && !(*p)->hint && ival->hint != *p) ||
1592 ival->range.end < (*p)->range.end))) {
1593 ival->list_next = *p;
1594 *p = ival;
1595 break;
1596 }
1597 p = &(*p)->list_next;
1598 }
1599 }
1600 }
1601 }
1602
1603 return list;
1604 }
1605
zend_jit_print_regset(zend_regset regset)1606 static ZEND_ATTRIBUTE_UNUSED void zend_jit_print_regset(zend_regset regset)
1607 {
1608 zend_reg reg;
1609 int first = 1;
1610
1611 ZEND_REGSET_FOREACH(regset, reg) {
1612 if (first) {
1613 first = 0;
1614 fprintf(stderr, "%s", zend_reg_name[reg]);
1615 } else {
1616 fprintf(stderr, ", %s", zend_reg_name[reg]);
1617 }
1618 } ZEND_REGSET_FOREACH_END();
1619 }
1620
zend_jit_compute_block_order_int(zend_ssa * ssa,int n,int * block_order)1621 static int *zend_jit_compute_block_order_int(zend_ssa *ssa, int n, int *block_order)
1622 {
1623 zend_basic_block *b = ssa->cfg.blocks + n;
1624
1625 tail_call:
1626 *block_order = n;
1627 block_order++;
1628
1629 n = b->children;
1630 while (n >= 0) {
1631 b = ssa->cfg.blocks + n;
1632 if (b->next_child < 0) {
1633 goto tail_call;
1634 }
1635 block_order = zend_jit_compute_block_order_int(ssa, n, block_order);
1636 n = b->next_child;
1637 }
1638
1639 return block_order;
1640 }
1641
zend_jit_compute_block_order(zend_ssa * ssa,int * block_order)1642 static int zend_jit_compute_block_order(zend_ssa *ssa, int *block_order)
1643 {
1644 int *end = zend_jit_compute_block_order_int(ssa, 0, block_order);
1645
1646 return end - block_order;
1647 }
1648
zend_jit_in_loop(zend_ssa * ssa,int header,zend_basic_block * b)1649 static bool zend_jit_in_loop(zend_ssa *ssa, int header, zend_basic_block *b)
1650 {
1651 while (b->loop_header >= 0) {
1652 if (b->loop_header == header) {
1653 return 1;
1654 }
1655 b = ssa->cfg.blocks + b->loop_header;
1656 }
1657 return 0;
1658 }
1659
zend_jit_compute_loop_body(zend_ssa * ssa,int header,int n,zend_bitset loop_body)1660 static void zend_jit_compute_loop_body(zend_ssa *ssa, int header, int n, zend_bitset loop_body)
1661 {
1662 zend_basic_block *b = ssa->cfg.blocks + n;
1663 uint32_t i;
1664
1665 tail_call:
1666 if (b->len) {
1667 for (i = b->start; i < b->start + b->len; i++) {
1668 zend_bitset_incl(loop_body, i);
1669 }
1670 }
1671
1672 n = b->children;
1673 while (n >= 0) {
1674 b = ssa->cfg.blocks + n;
1675 if (zend_jit_in_loop(ssa, header, b)) {
1676 if (b->next_child < 0) {
1677 goto tail_call;
1678 }
1679 zend_jit_compute_loop_body(ssa, header, n, loop_body);
1680 }
1681 n = b->next_child;
1682 }
1683 }
1684
zend_jit_add_hint(zend_lifetime_interval ** intervals,int dst,int src)1685 static void zend_jit_add_hint(zend_lifetime_interval **intervals, int dst, int src)
1686 {
1687 if (intervals[dst]->range.start < intervals[src]->range.start) {
1688 int tmp = src;
1689 src = dst;
1690 dst = tmp;
1691 }
1692 while (dst != src && intervals[dst]->hint) {
1693 if (intervals[dst]->hint->range.start < intervals[src]->range.start) {
1694 int tmp = src;
1695 src = intervals[dst]->hint->ssa_var;
1696 dst = tmp;
1697 } else {
1698 dst = intervals[dst]->hint->ssa_var;
1699 }
1700 }
1701 if (dst != src) {
1702 intervals[dst]->hint = intervals[src];
1703 }
1704 }
1705
1706 /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
1707 Michael Franz, CGO'10 (2010), Figure 4. */
zend_jit_compute_liveness(const zend_op_array * op_array,zend_ssa * ssa,zend_bitset candidates,zend_lifetime_interval ** list)1708 static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list)
1709 {
1710 int set_size, i, j, k, l;
1711 uint32_t n;
1712 zend_bitset live, live_in, pi_vars, loop_body;
1713 int *block_order;
1714 zend_ssa_phi *phi;
1715 zend_lifetime_interval **intervals;
1716 size_t mem_size;
1717 ALLOCA_FLAG(use_heap);
1718
1719 set_size = zend_bitset_len(ssa->vars_count);
1720 mem_size =
1721 ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)) +
1722 ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE) +
1723 ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
1724 ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
1725 ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE) +
1726 ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(int));
1727 intervals = do_alloca(mem_size, use_heap);
1728 if (!intervals) {
1729 *list = NULL;
1730 return FAILURE;
1731 }
1732
1733 live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)));
1734 live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE));
1735 pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
1736 loop_body = (zend_bitset)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
1737 block_order = (int*)((char*)loop_body + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE));
1738
1739 memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*));
1740 zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count);
1741
1742 /* TODO: Provide a linear block order where all dominators of a block
1743 * are before this block, and where all blocks belonging to the same loop
1744 * are contiguous ???
1745 */
1746 for (l = zend_jit_compute_block_order(ssa, block_order) - 1; l >= 0; l--) {
1747 zend_basic_block *b;
1748
1749 i = block_order[l];
1750 b = ssa->cfg.blocks + i;
1751
1752 /* live = UNION of successor.liveIn for each successor of b */
1753 /* live.add(phi.inputOf(b)) for each phi of successors of b */
1754 zend_bitset_clear(live, set_size);
1755 for (j = 0; j < b->successors_count; j++) {
1756 int succ = b->successors[j];
1757
1758 zend_bitset_union(live, live_in + set_size * succ, set_size);
1759 zend_bitset_clear(pi_vars, set_size);
1760 for (phi = ssa->blocks[succ].phis; phi; phi = phi->next) {
1761 if (ssa->vars[phi->ssa_var].no_val) {
1762 /* skip */
1763 } else if (phi->pi >= 0) {
1764 if (phi->pi == i && phi->sources[0] >= 0) {
1765 if (zend_bitset_in(candidates, phi->sources[0])) {
1766 zend_bitset_incl(live, phi->sources[0]);
1767 }
1768 zend_bitset_incl(pi_vars, phi->var);
1769 }
1770 } else if (!zend_bitset_in(pi_vars, phi->var)) {
1771 for (k = 0; k < ssa->cfg.blocks[succ].predecessors_count; k++) {
1772 if (ssa->cfg.predecessors[ssa->cfg.blocks[succ].predecessor_offset + k] == i) {
1773 if (phi->sources[k] >= 0 && zend_bitset_in(candidates, phi->sources[k])) {
1774 zend_bitset_incl(live, phi->sources[k]);
1775 }
1776 break;
1777 }
1778 }
1779 }
1780 }
1781 }
1782
1783 /* addRange(var, b.from, b.to) for each var in live */
1784 ZEND_BITSET_FOREACH(live, set_size, j) {
1785 if (zend_bitset_in(candidates, j)) {
1786 if (zend_jit_add_range(intervals, j, b->start, b->start + b->len - 1) != SUCCESS) {
1787 goto failure;
1788 }
1789 }
1790 } ZEND_BITSET_FOREACH_END();
1791
1792 /* for each operation op of b in reverse order */
1793 for (n = b->start + b->len; n > b->start;) {
1794 zend_ssa_op *op;
1795 const zend_op *opline;
1796 uint32_t num;
1797
1798 n--;
1799 op = ssa->ops + n;
1800 opline = op_array->opcodes + n;
1801
1802 if (UNEXPECTED(opline->opcode == ZEND_OP_DATA)) {
1803 num = n - 1;
1804 } else {
1805 num = n;
1806 }
1807
1808 /* for each output operand opd of op do */
1809 /* setFrom(opd, op) */
1810 /* live.remove(opd) */
1811 if (op->op1_def >= 0 && zend_bitset_in(candidates, op->op1_def)) {
1812 if (zend_jit_begin_range(intervals, op->op1_def, b->start, num) != SUCCESS) {
1813 goto failure;
1814 }
1815 zend_bitset_excl(live, op->op1_def);
1816 }
1817 if (op->op2_def >= 0 && zend_bitset_in(candidates, op->op2_def)) {
1818 if (zend_jit_begin_range(intervals, op->op2_def, b->start, num) != SUCCESS) {
1819 goto failure;
1820 }
1821 zend_bitset_excl(live, op->op2_def);
1822 }
1823 if (op->result_def >= 0 && zend_bitset_in(candidates, op->result_def)) {
1824 if (zend_jit_begin_range(intervals, op->result_def, b->start, num) != SUCCESS) {
1825 goto failure;
1826 }
1827 zend_bitset_excl(live, op->result_def);
1828 }
1829
1830 /* for each input operand opd of op do */
1831 /* live.add(opd) */
1832 /* addRange(opd, b.from, op) */
1833 if (op->op1_use >= 0
1834 && zend_bitset_in(candidates, op->op1_use)
1835 && !zend_ssa_is_no_val_use(opline, op, op->op1_use)) {
1836 zend_bitset_incl(live, op->op1_use);
1837 if (zend_jit_add_range(intervals, op->op1_use, b->start, num) != SUCCESS) {
1838 goto failure;
1839 }
1840 }
1841 if (op->op2_use >= 0
1842 && zend_bitset_in(candidates, op->op2_use)
1843 && !zend_ssa_is_no_val_use(opline, op, op->op2_use)) {
1844 zend_bitset_incl(live, op->op2_use);
1845 if (zend_jit_add_range(intervals, op->op2_use, b->start, num) != SUCCESS) {
1846 goto failure;
1847 }
1848 }
1849 if (op->result_use >= 0
1850 && zend_bitset_in(candidates, op->result_use)
1851 && !zend_ssa_is_no_val_use(opline, op, op->result_use)) {
1852 zend_bitset_incl(live, op->result_use);
1853 if (zend_jit_add_range(intervals, op->result_use, b->start, num) != SUCCESS) {
1854 goto failure;
1855 }
1856 }
1857 }
1858
1859 /* live.remove(phi.output) for each phi of b */
1860 for (phi = ssa->blocks[i].phis; phi; phi = phi->next) {
1861 zend_bitset_excl(live, phi->ssa_var);
1862 }
1863
1864 /* b.liveIn = live */
1865 zend_bitset_copy(live_in + set_size * i, live, set_size);
1866 }
1867
1868 for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) {
1869 zend_basic_block *b = ssa->cfg.blocks + i;
1870
1871 /* if b is loop header */
1872 if ((b->flags & ZEND_BB_LOOP_HEADER)) {
1873 live = live_in + set_size * i;
1874
1875 if (!zend_bitset_empty(live, set_size)) {
1876 uint32_t set_size2 = zend_bitset_len(op_array->last);
1877
1878 zend_bitset_clear(loop_body, set_size2);
1879 zend_jit_compute_loop_body(ssa, i, i, loop_body);
1880 while (!zend_bitset_empty(loop_body, set_size2)) {
1881 uint32_t from = zend_bitset_first(loop_body, set_size2);
1882 uint32_t to = from;
1883
1884 do {
1885 zend_bitset_excl(loop_body, to);
1886 to++;
1887 } while (zend_bitset_in(loop_body, to));
1888 to--;
1889
1890 ZEND_BITSET_FOREACH(live, set_size, j) {
1891 if (zend_jit_add_range(intervals, j, from, to) != SUCCESS) {
1892 goto failure;
1893 }
1894 } ZEND_BITSET_FOREACH_END();
1895 }
1896 }
1897 }
1898
1899 }
1900
1901 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
1902 /* Register hinting (a cheap way for register coalescing) */
1903 for (i = 0; i < ssa->vars_count; i++) {
1904 if (intervals[i]) {
1905 int src;
1906
1907 if (ssa->vars[i].definition_phi) {
1908 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
1909
1910 if (phi->pi >= 0) {
1911 src = phi->sources[0];
1912 if (intervals[src]) {
1913 zend_jit_add_hint(intervals, i, src);
1914 }
1915 } else {
1916 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1917 src = phi->sources[k];
1918 if (src >= 0) {
1919 if (ssa->vars[src].definition_phi
1920 && ssa->vars[src].definition_phi->pi >= 0
1921 && phi->block == ssa->vars[src].definition_phi->block) {
1922 /* Skip zero-length interval for Pi variable */
1923 src = ssa->vars[src].definition_phi->sources[0];
1924 }
1925 if (intervals[src]) {
1926 zend_jit_add_hint(intervals, i, src);
1927 }
1928 }
1929 }
1930 }
1931 }
1932 }
1933 }
1934 for (i = 0; i < ssa->vars_count; i++) {
1935 if (intervals[i] && !intervals[i]->hint) {
1936
1937 if (ssa->vars[i].definition >= 0) {
1938 uint32_t line = ssa->vars[i].definition;
1939 const zend_op *opline = op_array->opcodes + line;
1940
1941 switch (opline->opcode) {
1942 case ZEND_QM_ASSIGN:
1943 case ZEND_POST_INC:
1944 case ZEND_POST_DEC:
1945 if (ssa->ops[line].op1_use >= 0 &&
1946 intervals[ssa->ops[line].op1_use] &&
1947 (i == ssa->ops[line].op1_def ||
1948 (i == ssa->ops[line].result_def &&
1949 (ssa->ops[line].op1_def < 0 ||
1950 !intervals[ssa->ops[line].op1_def])))) {
1951 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1952 }
1953 break;
1954 case ZEND_SEND_VAR:
1955 case ZEND_PRE_INC:
1956 case ZEND_PRE_DEC:
1957 if (i == ssa->ops[line].op1_def &&
1958 ssa->ops[line].op1_use >= 0 &&
1959 intervals[ssa->ops[line].op1_use]) {
1960 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1961 }
1962 break;
1963 case ZEND_ASSIGN:
1964 if (ssa->ops[line].op2_use >= 0 &&
1965 intervals[ssa->ops[line].op2_use] &&
1966 (i == ssa->ops[line].op2_def ||
1967 (i == ssa->ops[line].op1_def &&
1968 (ssa->ops[line].op2_def < 0 ||
1969 !intervals[ssa->ops[line].op2_def])) ||
1970 (i == ssa->ops[line].result_def &&
1971 (ssa->ops[line].op2_def < 0 ||
1972 !intervals[ssa->ops[line].op2_def]) &&
1973 (ssa->ops[line].op1_def < 0 ||
1974 !intervals[ssa->ops[line].op1_def])))) {
1975 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
1976 }
1977 break;
1978 case ZEND_SUB:
1979 case ZEND_ADD:
1980 case ZEND_MUL:
1981 case ZEND_BW_OR:
1982 case ZEND_BW_AND:
1983 case ZEND_BW_XOR:
1984 if (i == ssa->ops[line].result_def) {
1985 if (ssa->ops[line].op1_use >= 0 &&
1986 intervals[ssa->ops[line].op1_use] &&
1987 ssa->ops[line].op1_use_chain < 0 &&
1988 !ssa->vars[ssa->ops[line].op1_use].phi_use_chain &&
1989 (ssa->var_info[i].type & MAY_BE_ANY) ==
1990 (ssa->var_info[ssa->ops[line].op1_use].type & MAY_BE_ANY)) {
1991 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1992 } else if (opline->opcode != ZEND_SUB &&
1993 ssa->ops[line].op2_use >= 0 &&
1994 intervals[ssa->ops[line].op2_use] &&
1995 ssa->ops[line].op2_use_chain < 0 &&
1996 !ssa->vars[ssa->ops[line].op2_use].phi_use_chain &&
1997 (ssa->var_info[i].type & MAY_BE_ANY) ==
1998 (ssa->var_info[ssa->ops[line].op2_use].type & MAY_BE_ANY)) {
1999 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
2000 }
2001 }
2002 break;
2003 }
2004 }
2005 }
2006 }
2007 }
2008
2009 *list = zend_jit_sort_intervals(intervals, ssa->vars_count);
2010
2011 if (*list) {
2012 zend_lifetime_interval *ival = *list;
2013 while (ival) {
2014 if (ival->hint) {
2015 ival->hint->used_as_hint = ival;
2016 }
2017 ival = ival->list_next;
2018 }
2019 }
2020
2021 free_alloca(intervals, use_heap);
2022 return SUCCESS;
2023
2024 failure:
2025 *list = NULL;
2026 free_alloca(intervals, use_heap);
2027 return FAILURE;
2028 }
2029
zend_interval_end(zend_lifetime_interval * ival)2030 static uint32_t zend_interval_end(zend_lifetime_interval *ival)
2031 {
2032 zend_life_range *range = &ival->range;
2033
2034 while (range->next) {
2035 range = range->next;
2036 }
2037 return range->end;
2038 }
2039
zend_interval_covers(zend_lifetime_interval * ival,uint32_t position)2040 static bool zend_interval_covers(zend_lifetime_interval *ival, uint32_t position)
2041 {
2042 zend_life_range *range = &ival->range;
2043
2044 do {
2045 if (position >= range->start && position <= range->end) {
2046 return 1;
2047 }
2048 range = range->next;
2049 } while (range);
2050
2051 return 0;
2052 }
2053
zend_interval_intersection(zend_lifetime_interval * ival1,zend_lifetime_interval * ival2)2054 static uint32_t zend_interval_intersection(zend_lifetime_interval *ival1, zend_lifetime_interval *ival2)
2055 {
2056 zend_life_range *r1 = &ival1->range;
2057 zend_life_range *r2 = &ival2->range;
2058
2059 do {
2060 if (r1->start <= r2->end) {
2061 if (r2->start <= r1->end) {
2062 return MAX(r1->start, r2->start);
2063 } else {
2064 r2 = r2->next;
2065 }
2066 } else {
2067 r1 = r1->next;
2068 }
2069 } while (r1 && r2);
2070
2071 return 0xffffffff;
2072 }
2073
2074 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2075 Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling */
zend_jit_try_allocate_free_reg(const zend_op_array * op_array,const zend_op ** ssa_opcodes,zend_ssa * ssa,zend_lifetime_interval * current,zend_regset available,zend_regset * hints,zend_lifetime_interval * active,zend_lifetime_interval * inactive,zend_lifetime_interval ** list,zend_lifetime_interval ** free)2076 static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_regset *hints, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free)
2077 {
2078 zend_lifetime_interval *it;
2079 uint32_t freeUntilPos[ZREG_NUM];
2080 uint32_t pos, pos2;
2081 zend_reg i, reg, reg2;
2082 zend_reg hint = ZREG_NONE;
2083 zend_regset low_priority_regs;
2084 zend_life_range *range;
2085
2086 if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) {
2087 available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_FP);
2088 } else {
2089 available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_GP);
2090 }
2091
2092 /* TODO: Allow usage of preserved registers ???
2093 * Their values have to be stored in prologue and restored in epilogue
2094 */
2095 available = ZEND_REGSET_DIFFERENCE(available, ZEND_REGSET_PRESERVED);
2096
2097 /* Set freeUntilPos of all physical registers to maxInt */
2098 for (i = 0; i < ZREG_NUM; i++) {
2099 freeUntilPos[i] = 0xffffffff;
2100 }
2101
2102 /* for each interval it in active do */
2103 /* freeUntilPos[it.reg] = 0 */
2104 it = active;
2105 if (ssa->vars[current->ssa_var].definition == current->range.start) {
2106 while (it) {
2107 if (current->range.start != zend_interval_end(it)) {
2108 freeUntilPos[it->reg] = 0;
2109 } else if (zend_jit_may_reuse_reg(
2110 ssa_opcodes ? ssa_opcodes[current->range.start] : op_array->opcodes + current->range.start,
2111 ssa->ops + current->range.start, ssa, current->ssa_var, it->ssa_var)) {
2112 if (!ZEND_REGSET_IN(*hints, it->reg) &&
2113 /* TODO: Avoid most often scratch registers. Find a better way ??? */
2114 (!current->used_as_hint ||
2115 !ZEND_REGSET_IN(ZEND_REGSET_LOW_PRIORITY, it->reg))) {
2116 hint = it->reg;
2117 }
2118 } else {
2119 freeUntilPos[it->reg] = 0;
2120 }
2121 it = it->list_next;
2122 }
2123 } else {
2124 while (it) {
2125 freeUntilPos[it->reg] = 0;
2126 it = it->list_next;
2127 }
2128 }
2129 if (current->hint) {
2130 hint = current->hint->reg;
2131 if (hint != ZREG_NONE && current->hint->used_as_hint == current) {
2132 ZEND_REGSET_EXCL(*hints, hint);
2133 }
2134 }
2135
2136 if (hint == ZREG_NONE && ZEND_REGSET_IS_EMPTY(available)) {
2137 return 0;
2138 }
2139
2140 /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
2141 Michael Franz, CGO'10 (2010), Figure 6. */
2142 if (current->flags & ZREG_SPLIT) {
2143 /* for each interval it in inactive intersecting with current do */
2144 /* freeUntilPos[it.reg] = next intersection of it with current */
2145 it = inactive;
2146 while (it) {
2147 uint32_t next = zend_interval_intersection(current, it);
2148
2149 //ZEND_ASSERT(next != 0xffffffff && !current->split);
2150 if (next < freeUntilPos[it->reg]) {
2151 freeUntilPos[it->reg] = next;
2152 }
2153 it = it->list_next;
2154 }
2155 }
2156
2157 /* Handle Scratch Registers */
2158 /* TODO: Optimize ??? */
2159 range = ¤t->range;
2160 do {
2161 uint32_t line = range->start;
2162 uint32_t last_use_line = (uint32_t)-1;
2163 zend_regset regset;
2164 zend_reg reg;
2165
2166 if ((current->flags & ZREG_LAST_USE) && !range->next) {
2167 last_use_line = range->end;
2168 }
2169 if (ssa->ops[line].op1_def == current->ssa_var ||
2170 ssa->ops[line].op2_def == current->ssa_var ||
2171 ssa->ops[line].result_def == current->ssa_var) {
2172 regset = zend_jit_get_def_scratch_regset(
2173 ssa_opcodes ? ssa_opcodes[line] : op_array->opcodes + line,
2174 ssa->ops + line,
2175 op_array, ssa, current->ssa_var, line == last_use_line);
2176 ZEND_REGSET_FOREACH(regset, reg) {
2177 if (line < freeUntilPos[reg]) {
2178 freeUntilPos[reg] = line;
2179 }
2180 } ZEND_REGSET_FOREACH_END();
2181 line++;
2182 }
2183 while (line <= range->end) {
2184 regset = zend_jit_get_scratch_regset(
2185 ssa_opcodes ? ssa_opcodes[line] : op_array->opcodes + line,
2186 ssa->ops + line,
2187 op_array, ssa, current->ssa_var, line == last_use_line);
2188 ZEND_REGSET_FOREACH(regset, reg) {
2189 if (line < freeUntilPos[reg]) {
2190 freeUntilPos[reg] = line;
2191 }
2192 } ZEND_REGSET_FOREACH_END();
2193 line++;
2194 }
2195 range = range->next;
2196 } while (range);
2197
2198 #if 0
2199 /* Coalescing */
2200 if (ssa->vars[current->ssa_var].definition == current->start) {
2201 zend_op *opline = op_array->opcodes + current->start;
2202 int hint = -1;
2203
2204 switch (opline->opcode) {
2205 case ZEND_ASSIGN:
2206 hint = ssa->ops[current->start].op2_use;
2207 case ZEND_QM_ASSIGN:
2208 hint = ssa->ops[current->start].op1_use;
2209 break;
2210 case ZEND_ADD:
2211 case ZEND_SUB:
2212 case ZEND_MUL:
2213 hint = ssa->ops[current->start].op1_use;
2214 break;
2215 case ZEND_ASSIGN_OP:
2216 if (opline->extended_value == ZEND_ADD
2217 || opline->extended_value == ZEND_SUB
2218 || opline->extended_value == ZEND_MUL) {
2219 hint = ssa->ops[current->start].op1_use;
2220 }
2221 break;
2222 }
2223 if (hint >= 0) {
2224 }
2225 }
2226 #endif
2227
2228 if (hint != ZREG_NONE && freeUntilPos[hint] > zend_interval_end(current)) {
2229 current->reg = hint;
2230 if (current->used_as_hint) {
2231 ZEND_REGSET_INCL(*hints, hint);
2232 }
2233 return 1;
2234 }
2235
2236 if (ZEND_REGSET_IS_EMPTY(available)) {
2237 return 0;
2238 }
2239
2240 pos = 0; reg = ZREG_NONE;
2241 pos2 = 0; reg2 = ZREG_NONE;
2242 low_priority_regs = *hints;
2243 if (current->used_as_hint) {
2244 /* TODO: Avoid most often scratch registers. Find a better way ??? */
2245 low_priority_regs = ZEND_REGSET_UNION(low_priority_regs, ZEND_REGSET_LOW_PRIORITY);
2246 }
2247
2248 ZEND_REGSET_FOREACH(available, i) {
2249 if (ZEND_REGSET_IN(low_priority_regs, i)) {
2250 if (freeUntilPos[i] > pos2) {
2251 reg2 = i;
2252 pos2 = freeUntilPos[i];
2253 }
2254 } else if (freeUntilPos[i] > pos) {
2255 reg = i;
2256 pos = freeUntilPos[i];
2257 }
2258 } ZEND_REGSET_FOREACH_END();
2259
2260 if (reg == ZREG_NONE) {
2261 if (reg2 != ZREG_NONE) {
2262 reg = reg2;
2263 pos = pos2;
2264 reg2 = ZREG_NONE;
2265 }
2266 }
2267
2268 if (reg == ZREG_NONE) {
2269 /* no register available without spilling */
2270 return 0;
2271 } else if (zend_interval_end(current) < pos) {
2272 /* register available for the whole interval */
2273 current->reg = reg;
2274 if (current->used_as_hint) {
2275 ZEND_REGSET_INCL(*hints, reg);
2276 }
2277 return 1;
2278 #if 0
2279 // TODO: allow low priority register usage
2280 } else if (reg2 != ZREG_NONE && zend_interval_end(current) < pos2) {
2281 /* register available for the whole interval */
2282 current->reg = reg2;
2283 if (current->used_as_hint) {
2284 ZEND_REGSET_INCL(*hints, reg2);
2285 }
2286 return 1;
2287 #endif
2288 } else {
2289 /* TODO: enable interval splitting ??? */
2290 /* register available for the first part of the interval */
2291 if (1 || zend_jit_split_interval(current, pos, list, free) != SUCCESS) {
2292 return 0;
2293 }
2294 current->reg = reg;
2295 if (current->used_as_hint) {
2296 ZEND_REGSET_INCL(*hints, reg);
2297 }
2298 return 1;
2299 }
2300 }
2301
2302 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2303 Christian Wimmer VEE'05 (2005), Figure 5. Allocation with spilling.
2304 and "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
2305 Michael Franz, CGO'10 (2010), Figure 6. */
zend_jit_allocate_blocked_reg(void)2306 static int zend_jit_allocate_blocked_reg(void)
2307 {
2308 /* TODO: ??? */
2309 return 0;
2310 }
2311
2312 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
2313 Christian Wimmer VEE'10 (2005), Figure 2. */
zend_jit_linear_scan(const zend_op_array * op_array,const zend_op ** ssa_opcodes,zend_ssa * ssa,zend_lifetime_interval * list)2314 static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *list)
2315 {
2316 zend_lifetime_interval *unhandled, *active, *inactive, *handled, *free;
2317 zend_lifetime_interval *current, **p, *q;
2318 uint32_t position;
2319 zend_regset available = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP);
2320 zend_regset hints = ZEND_REGSET_EMPTY;
2321
2322 unhandled = list;
2323 /* active = inactive = handled = free = {} */
2324 active = inactive = handled = free = NULL;
2325 while (unhandled != NULL) {
2326 current = unhandled;
2327 unhandled = unhandled->list_next;
2328 position = current->range.start;
2329
2330 p = &active;
2331 while (*p) {
2332 uint32_t end = zend_interval_end(*p);
2333
2334 q = *p;
2335 if (end < position) {
2336 /* move ival from active to handled */
2337 ZEND_REGSET_INCL(available, q->reg);
2338 *p = q->list_next;
2339 q->list_next = handled;
2340 handled = q;
2341 } else if (!zend_interval_covers(q, position)) {
2342 /* move ival from active to inactive */
2343 ZEND_REGSET_INCL(available, q->reg);
2344 *p = q->list_next;
2345 q->list_next = inactive;
2346 inactive = q;
2347 } else {
2348 p = &q->list_next;
2349 }
2350 }
2351
2352 p = &inactive;
2353 while (*p) {
2354 uint32_t end = zend_interval_end(*p);
2355
2356 q = *p;
2357 if (end < position) {
2358 /* move ival from inactive to handled */
2359 *p = q->list_next;
2360 q->list_next = handled;
2361 handled = q;
2362 } else if (zend_interval_covers(q, position)) {
2363 /* move ival from inactive to active */
2364 ZEND_REGSET_EXCL(available, q->reg);
2365 *p = q->list_next;
2366 q->list_next = active;
2367 active = q;
2368 } else {
2369 p = &q->list_next;
2370 }
2371 }
2372
2373 if (zend_jit_try_allocate_free_reg(op_array, ssa_opcodes, ssa, current, available, &hints, active, inactive, &unhandled, &free) ||
2374 zend_jit_allocate_blocked_reg()) {
2375 ZEND_REGSET_EXCL(available, current->reg);
2376 current->list_next = active;
2377 active = current;
2378 } else {
2379 current->list_next = free;
2380 free = current;
2381 }
2382 }
2383
2384 /* move active to handled */
2385 while (active) {
2386 current = active;
2387 active = active->list_next;
2388 current->list_next = handled;
2389 handled = current;
2390 }
2391
2392 /* move inactive to handled */
2393 while (inactive) {
2394 current = inactive;
2395 inactive = inactive->list_next;
2396 current->list_next = handled;
2397 handled = current;
2398 }
2399
2400 return handled;
2401 }
2402
zend_jit_dump_lifetime_interval(const zend_op_array * op_array,const zend_ssa * ssa,const zend_lifetime_interval * ival)2403 static void zend_jit_dump_lifetime_interval(const zend_op_array *op_array, const zend_ssa *ssa, const zend_lifetime_interval *ival)
2404 {
2405 zend_life_range *range;
2406 int var_num = ssa->vars[ival->ssa_var].var;
2407
2408 fprintf(stderr, "#%d.", ival->ssa_var);
2409 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
2410 fprintf(stderr, ": %u-%u", ival->range.start, ival->range.end);
2411 range = ival->range.next;
2412 while (range) {
2413 fprintf(stderr, ", %u-%u", range->start, range->end);
2414 range = range->next;
2415 }
2416 if (ival->reg != ZREG_NONE) {
2417 fprintf(stderr, " (%s)", zend_reg_name[ival->reg]);
2418 }
2419 if (ival->flags & ZREG_LAST_USE) {
2420 fprintf(stderr, " last_use");
2421 }
2422 if (ival->flags & ZREG_LOAD) {
2423 fprintf(stderr, " load");
2424 }
2425 if (ival->flags & ZREG_STORE) {
2426 fprintf(stderr, " store");
2427 }
2428 if (ival->hint) {
2429 fprintf(stderr, " hint");
2430 if (ival->hint->ssa_var >= 0) {
2431 var_num = ssa->vars[ival->hint->ssa_var].var;
2432 fprintf(stderr, "=#%d.", ival->hint->ssa_var);
2433 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
2434 }
2435 if (ival->hint->reg != ZREG_NONE) {
2436 fprintf(stderr, " (%s)", zend_reg_name[ival->hint->reg]);
2437 }
2438 }
2439 fprintf(stderr, "\n");
2440 }
2441
zend_jit_allocate_registers(const zend_op_array * op_array,zend_ssa * ssa)2442 static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array *op_array, zend_ssa *ssa)
2443 {
2444 void *checkpoint;
2445 int set_size, candidates_count, i;
2446 zend_bitset candidates = NULL;
2447 zend_lifetime_interval *list, *ival;
2448 zend_lifetime_interval **intervals;
2449 ALLOCA_FLAG(use_heap);
2450
2451 if (!ssa->var_info) {
2452 return NULL;
2453 }
2454
2455 /* Identify SSA variables suitable for register allocation */
2456 set_size = zend_bitset_len(ssa->vars_count);
2457 candidates = ZEND_BITSET_ALLOCA(set_size, use_heap);
2458 if (!candidates) {
2459 return NULL;
2460 }
2461 candidates_count = 0;
2462 zend_bitset_clear(candidates, set_size);
2463 for (i = 0; i < ssa->vars_count; i++) {
2464 if (zend_jit_may_be_in_reg(op_array, ssa, i)) {
2465 zend_bitset_incl(candidates, i);
2466 candidates_count++;
2467 }
2468 }
2469 if (!candidates_count) {
2470 free_alloca(candidates, use_heap);
2471 return NULL;
2472 }
2473
2474 checkpoint = zend_arena_checkpoint(CG(arena));
2475
2476 /* Find life-time intervals */
2477 if (zend_jit_compute_liveness(op_array, ssa, candidates, &list) != SUCCESS) {
2478 goto failure;
2479 }
2480
2481 if (list) {
2482 /* Set ZREG_LAST_USE flags */
2483 ival = list;
2484 while (ival) {
2485 zend_life_range *range = &ival->range;
2486
2487 while (range->next) {
2488 range = range->next;
2489 }
2490 if (zend_ssa_is_last_use(op_array, ssa, ival->ssa_var, range->end)) {
2491 ival->flags |= ZREG_LAST_USE;
2492 }
2493 ival = ival->list_next;
2494 }
2495 }
2496
2497 if (list) {
2498 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
2499 fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
2500 ival = list;
2501 while (ival) {
2502 zend_jit_dump_lifetime_interval(op_array, ssa, ival);
2503 ival = ival->list_next;
2504 }
2505 fprintf(stderr, "\n");
2506 }
2507
2508 /* Linear Scan Register Allocation */
2509 list = zend_jit_linear_scan(op_array, NULL, ssa, list);
2510
2511 if (list) {
2512 intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*));
2513 if (!intervals) {
2514 goto failure;
2515 }
2516
2517 ival = list;
2518 while (ival != NULL) {
2519 zend_lifetime_interval *next = ival->list_next;
2520
2521 ival->list_next = intervals[ival->ssa_var];
2522 intervals[ival->ssa_var] = ival;
2523 ival = next;
2524 }
2525
2526 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
2527 /* Naive SSA resolution */
2528 for (i = 0; i < ssa->vars_count; i++) {
2529 if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
2530 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
2531 int k, src;
2532
2533 if (phi->pi >= 0) {
2534 if (!ssa->vars[i].phi_use_chain
2535 || ssa->vars[i].phi_use_chain->block != phi->block) {
2536 src = phi->sources[0];
2537 if (intervals[i]) {
2538 if (!intervals[src]) {
2539 intervals[i]->flags |= ZREG_LOAD;
2540 } else if (intervals[i]->reg != intervals[src]->reg) {
2541 intervals[i]->flags |= ZREG_LOAD;
2542 intervals[src]->flags |= ZREG_STORE;
2543 }
2544 } else if (intervals[src]) {
2545 intervals[src]->flags |= ZREG_STORE;
2546 }
2547 }
2548 } else {
2549 int need_move = 0;
2550
2551 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
2552 src = phi->sources[k];
2553 if (src >= 0) {
2554 if (ssa->vars[src].definition_phi
2555 && ssa->vars[src].definition_phi->pi >= 0
2556 && phi->block == ssa->vars[src].definition_phi->block) {
2557 /* Skip zero-length interval for Pi variable */
2558 src = ssa->vars[src].definition_phi->sources[0];
2559 }
2560 if (intervals[i]) {
2561 if (!intervals[src]) {
2562 need_move = 1;
2563 } else if (intervals[i]->reg != intervals[src]->reg) {
2564 need_move = 1;
2565 }
2566 } else if (intervals[src]) {
2567 need_move = 1;
2568 }
2569 }
2570 }
2571 if (need_move) {
2572 if (intervals[i]) {
2573 intervals[i]->flags |= ZREG_LOAD;
2574 }
2575 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
2576 src = phi->sources[k];
2577 if (src >= 0) {
2578 if (ssa->vars[src].definition_phi
2579 && ssa->vars[src].definition_phi->pi >= 0
2580 && phi->block == ssa->vars[src].definition_phi->block) {
2581 /* Skip zero-length interval for Pi variable */
2582 src = ssa->vars[src].definition_phi->sources[0];
2583 }
2584 if (intervals[src]) {
2585 intervals[src]->flags |= ZREG_STORE;
2586 }
2587 }
2588 }
2589 }
2590 }
2591 }
2592 }
2593 /* Remove useless register allocation */
2594 for (i = 0; i < ssa->vars_count; i++) {
2595 if (intervals[i] &&
2596 ((intervals[i]->flags & ZREG_LOAD) ||
2597 ((intervals[i]->flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) &&
2598 ssa->vars[i].use_chain < 0) {
2599 bool may_remove = 1;
2600 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
2601
2602 while (phi) {
2603 if (intervals[phi->ssa_var] &&
2604 !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) {
2605 may_remove = 0;
2606 break;
2607 }
2608 phi = zend_ssa_next_use_phi(ssa, i, phi);
2609 }
2610 if (may_remove) {
2611 intervals[i] = NULL;
2612 }
2613 }
2614 }
2615 /* Remove intervals used once */
2616 for (i = 0; i < ssa->vars_count; i++) {
2617 if (intervals[i] &&
2618 (intervals[i]->flags & ZREG_LOAD) &&
2619 (intervals[i]->flags & ZREG_STORE) &&
2620 (ssa->vars[i].use_chain < 0 ||
2621 zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) {
2622 bool may_remove = 1;
2623 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
2624
2625 while (phi) {
2626 if (intervals[phi->ssa_var] &&
2627 !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) {
2628 may_remove = 0;
2629 break;
2630 }
2631 phi = zend_ssa_next_use_phi(ssa, i, phi);
2632 }
2633 if (may_remove) {
2634 intervals[i] = NULL;
2635 }
2636 }
2637 }
2638 }
2639
2640 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
2641 fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
2642 for (i = 0; i < ssa->vars_count; i++) {
2643 ival = intervals[i];
2644 while (ival) {
2645 zend_jit_dump_lifetime_interval(op_array, ssa, ival);
2646 ival = ival->list_next;
2647 }
2648 }
2649 fprintf(stderr, "\n");
2650 }
2651
2652 free_alloca(candidates, use_heap);
2653 return intervals;
2654 }
2655 }
2656
2657 failure:
2658 zend_arena_release(&CG(arena), checkpoint);
2659 free_alloca(candidates, use_heap);
2660 return NULL;
2661 }
2662
zend_jit_next_is_send_result(const zend_op * opline)2663 static bool zend_jit_next_is_send_result(const zend_op *opline)
2664 {
2665 if (opline->result_type == IS_TMP_VAR
2666 && (opline+1)->opcode == ZEND_SEND_VAL
2667 && (opline+1)->op1_type == IS_TMP_VAR
2668 && (opline+1)->op2_type != IS_CONST
2669 && (opline+1)->op1.var == opline->result.var) {
2670 return 1;
2671 }
2672 return 0;
2673 }
2674
zend_jit_supported_binary_op(uint8_t op,uint32_t op1_info,uint32_t op2_info)2675 static bool zend_jit_supported_binary_op(uint8_t op, uint32_t op1_info, uint32_t op2_info)
2676 {
2677 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2678 return false;
2679 }
2680 switch (op) {
2681 case ZEND_POW:
2682 case ZEND_DIV:
2683 // TODO: check for division by zero ???
2684 return false;
2685 case ZEND_ADD:
2686 case ZEND_SUB:
2687 case ZEND_MUL:
2688 return (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
2689 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
2690 case ZEND_BW_OR:
2691 case ZEND_BW_AND:
2692 case ZEND_BW_XOR:
2693 case ZEND_SL:
2694 case ZEND_SR:
2695 case ZEND_MOD:
2696 return (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG);
2697 case ZEND_CONCAT:
2698 return (op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING);
2699 EMPTY_SWITCH_DEFAULT_CASE()
2700 }
2701 }
2702
zend_jit(const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline)2703 static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline)
2704 {
2705 int b, i, end;
2706 zend_op *opline;
2707 dasm_State* dasm_state = NULL;
2708 void *handler;
2709 int call_level = 0;
2710 void *checkpoint = NULL;
2711 zend_lifetime_interval **ra = NULL;
2712 bool is_terminated = 1; /* previous basic block is terminated by jump */
2713 bool recv_emitted = 0; /* emitted at least one RECV opcode */
2714 uint8_t smart_branch_opcode;
2715 uint32_t target_label, target_label2;
2716 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info;
2717 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
2718 zend_class_entry *ce;
2719 bool ce_is_instanceof;
2720 bool on_this;
2721
2722 if (JIT_G(bisect_limit)) {
2723 jit_bisect_pos++;
2724 if (jit_bisect_pos >= JIT_G(bisect_limit)) {
2725 if (jit_bisect_pos == JIT_G(bisect_limit)) {
2726 fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n",
2727 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
2728 op_array->scope ? "::" : "",
2729 op_array->function_name ? ZSTR_VAL(op_array->function_name) : "{main}",
2730 ZSTR_VAL(op_array->filename), op_array->line_start);
2731 }
2732 return FAILURE;
2733 }
2734 }
2735
2736 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2737 checkpoint = zend_arena_checkpoint(CG(arena));
2738 ra = zend_jit_allocate_registers(op_array, ssa);
2739 }
2740
2741 /* mark hidden branch targets */
2742 for (b = 0; b < ssa->cfg.blocks_count; b++) {
2743 if (ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE &&
2744 ssa->cfg.blocks[b].len > 1) {
2745
2746 opline = op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
2747 if (opline->opcode == ZEND_DO_FCALL &&
2748 (opline-1)->opcode == ZEND_NEW) {
2749 ssa->cfg.blocks[ssa->cfg.blocks[b].successors[0]].flags |= ZEND_BB_TARGET;
2750 }
2751 }
2752 }
2753
2754 dasm_init(&dasm_state, DASM_MAXSECTION);
2755 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
2756 dasm_setup(&dasm_state, dasm_actions);
2757
2758 dasm_growpc(&dasm_state, ssa->cfg.blocks_count * 2 + 1);
2759
2760 zend_jit_align_func(&dasm_state);
2761 for (b = 0; b < ssa->cfg.blocks_count; b++) {
2762 if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) {
2763 continue;
2764 }
2765 //#ifndef CONTEXT_THREADED_JIT
2766 if (ssa->cfg.blocks[b].flags & ZEND_BB_ENTRY) {
2767 if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
2768 /* pass */
2769 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2770 ssa->cfg.blocks[b].len == 1 &&
2771 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT) &&
2772 op_array->opcodes[ssa->cfg.blocks[b].start].opcode != ZEND_JMP) {
2773 /* don't generate code for BB with single opcode */
2774 continue;
2775 }
2776 if (ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW) {
2777 if (!is_terminated) {
2778 zend_jit_jmp(&dasm_state, b);
2779 }
2780 }
2781 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2782 zend_jit_prologue(&dasm_state);
2783 } else
2784 //#endif
2785 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
2786 opline = op_array->opcodes + ssa->cfg.blocks[b].start;
2787 if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) {
2788 if (opline->opcode == ZEND_RECV_INIT) {
2789 if (opline == op_array->opcodes ||
2790 (opline-1)->opcode != ZEND_RECV_INIT) {
2791 if (recv_emitted) {
2792 zend_jit_jmp(&dasm_state, b);
2793 }
2794 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2795 for (i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) {
2796 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b + i);
2797 }
2798 zend_jit_prologue(&dasm_state);
2799 }
2800 recv_emitted = 1;
2801 } else if (opline->opcode == ZEND_RECV) {
2802 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2803 /* skip */
2804 continue;
2805 } else if (recv_emitted) {
2806 zend_jit_jmp(&dasm_state, b);
2807 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2808 zend_jit_prologue(&dasm_state);
2809 } else {
2810 zend_arg_info *arg_info;
2811
2812 if (opline->op1.num <= op_array->num_args) {
2813 arg_info = &op_array->arg_info[opline->op1.num - 1];
2814 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
2815 arg_info = &op_array->arg_info[op_array->num_args];
2816 } else {
2817 /* skip */
2818 continue;
2819 }
2820 if (!ZEND_TYPE_IS_SET(arg_info->type)) {
2821 /* skip */
2822 continue;
2823 }
2824 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2825 zend_jit_prologue(&dasm_state);
2826 recv_emitted = 1;
2827 }
2828 } else {
2829 if (recv_emitted) {
2830 zend_jit_jmp(&dasm_state, b);
2831 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2832 ssa->cfg.blocks[b].len == 1 &&
2833 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
2834 /* don't generate code for BB with single opcode */
2835 dasm_free(&dasm_state);
2836
2837 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2838 zend_arena_release(&CG(arena), checkpoint);
2839 }
2840 return SUCCESS;
2841 }
2842 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2843 zend_jit_prologue(&dasm_state);
2844 recv_emitted = 1;
2845 }
2846 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2847 ssa->cfg.blocks[b].len == 1 &&
2848 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
2849 /* don't generate code for BB with single opcode */
2850 dasm_free(&dasm_state);
2851
2852 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2853 zend_arena_release(&CG(arena), checkpoint);
2854 }
2855 return SUCCESS;
2856 } else {
2857 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2858 zend_jit_prologue(&dasm_state);
2859 }
2860 }
2861
2862 is_terminated = 0;
2863
2864 zend_jit_label(&dasm_state, b);
2865 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
2866 if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
2867 && ssa->cfg.blocks[b].start != 0
2868 && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
2869 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_LONG
2870 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_STRING
2871 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_MATCH)) {
2872 zend_jit_reset_last_valid_opline();
2873 if (!zend_jit_set_ip(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) {
2874 goto jit_failure;
2875 }
2876 } else {
2877 zend_jit_set_last_valid_opline(op_array->opcodes + ssa->cfg.blocks[b].start);
2878 }
2879 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
2880 zend_jit_reset_last_valid_opline();
2881 } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY|ZEND_BB_ENTRY)) {
2882 zend_jit_set_last_valid_opline(op_array->opcodes + ssa->cfg.blocks[b].start);
2883 }
2884 if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) {
2885 if (!zend_jit_check_timeout(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start, NULL)) {
2886 goto jit_failure;
2887 }
2888 }
2889 if (!ssa->cfg.blocks[b].len) {
2890 continue;
2891 }
2892 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
2893 zend_ssa_phi *phi = ssa->blocks[b].phis;
2894
2895 while (phi) {
2896 zend_lifetime_interval *ival = ra[phi->ssa_var];
2897
2898 if (ival) {
2899 if (ival->flags & ZREG_LOAD) {
2900 ZEND_ASSERT(ival->reg != ZREG_NONE);
2901
2902 if (!zend_jit_load_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg)) {
2903 goto jit_failure;
2904 }
2905 } else if (ival->flags & ZREG_STORE) {
2906 ZEND_ASSERT(ival->reg != ZREG_NONE);
2907
2908 if (!zend_jit_store_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg, 1)) {
2909 goto jit_failure;
2910 }
2911 }
2912 }
2913 phi = phi->next;
2914 }
2915 }
2916 end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
2917 for (i = ssa->cfg.blocks[b].start; i <= end; i++) {
2918 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[i] : NULL;
2919 opline = op_array->opcodes + i;
2920 switch (opline->opcode) {
2921 case ZEND_INIT_FCALL:
2922 case ZEND_INIT_FCALL_BY_NAME:
2923 case ZEND_INIT_NS_FCALL_BY_NAME:
2924 case ZEND_INIT_METHOD_CALL:
2925 case ZEND_INIT_DYNAMIC_CALL:
2926 case ZEND_INIT_STATIC_METHOD_CALL:
2927 case ZEND_INIT_USER_CALL:
2928 case ZEND_NEW:
2929 call_level++;
2930 }
2931
2932 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
2933 switch (opline->opcode) {
2934 case ZEND_PRE_INC:
2935 case ZEND_PRE_DEC:
2936 case ZEND_POST_INC:
2937 case ZEND_POST_DEC:
2938 if (opline->op1_type != IS_CV) {
2939 break;
2940 }
2941 op1_info = OP1_INFO();
2942 if (!(op1_info & MAY_BE_LONG)) {
2943 break;
2944 }
2945 if (opline->result_type != IS_UNUSED) {
2946 res_use_info = -1;
2947
2948 if (opline->result_type == IS_CV
2949 && ssa->vars
2950 && ssa_op->result_use >= 0
2951 && !ssa->vars[ssa_op->result_use].no_val) {
2952 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
2953
2954 if (Z_MODE(res_use_addr) != IS_REG
2955 || Z_LOAD(res_use_addr)
2956 || Z_STORE(res_use_addr)) {
2957 res_use_info = RES_USE_INFO();
2958 }
2959 }
2960 res_info = RES_INFO();
2961 res_addr = RES_REG_ADDR();
2962 } else {
2963 res_use_info = -1;
2964 res_info = -1;
2965 res_addr = 0;
2966 }
2967 op1_def_info = OP1_DEF_INFO();
2968 if (!zend_jit_inc_dec(&dasm_state, opline,
2969 op1_info, OP1_REG_ADDR(),
2970 op1_def_info, OP1_DEF_REG_ADDR(),
2971 res_use_info, res_info,
2972 res_addr,
2973 (op1_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
2974 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2975 goto jit_failure;
2976 }
2977 goto done;
2978 case ZEND_BW_OR:
2979 case ZEND_BW_AND:
2980 case ZEND_BW_XOR:
2981 case ZEND_SL:
2982 case ZEND_SR:
2983 case ZEND_MOD:
2984 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2985 break;
2986 }
2987 op1_info = OP1_INFO();
2988 op2_info = OP2_INFO();
2989 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2990 break;
2991 }
2992 if (!(op1_info & MAY_BE_LONG)
2993 || !(op2_info & MAY_BE_LONG)) {
2994 break;
2995 }
2996 res_addr = RES_REG_ADDR();
2997 if (Z_MODE(res_addr) != IS_REG
2998 && (i + 1) <= end
2999 && zend_jit_next_is_send_result(opline)) {
3000 i++;
3001 res_use_info = -1;
3002 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3003 if (!zend_jit_reuse_ip(&dasm_state)) {
3004 goto jit_failure;
3005 }
3006 } else {
3007 res_use_info = -1;
3008
3009 if (opline->result_type == IS_CV
3010 && ssa->vars
3011 && ssa_op->result_use >= 0
3012 && !ssa->vars[ssa_op->result_use].no_val) {
3013 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
3014
3015 if (Z_MODE(res_use_addr) != IS_REG
3016 || Z_LOAD(res_use_addr)
3017 || Z_STORE(res_use_addr)) {
3018 res_use_info = RES_USE_INFO();
3019 }
3020 }
3021 }
3022 if (!zend_jit_long_math(&dasm_state, opline,
3023 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
3024 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
3025 res_use_info, RES_INFO(), res_addr,
3026 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3027 goto jit_failure;
3028 }
3029 goto done;
3030 case ZEND_ADD:
3031 case ZEND_SUB:
3032 case ZEND_MUL:
3033 // case ZEND_DIV: // TODO: check for division by zero ???
3034 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3035 break;
3036 }
3037 op1_info = OP1_INFO();
3038 op2_info = OP2_INFO();
3039 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
3040 break;
3041 }
3042 if (opline->opcode == ZEND_ADD &&
3043 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
3044 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
3045 /* pass */
3046 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
3047 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
3048 break;
3049 }
3050 res_addr = RES_REG_ADDR();
3051 if (Z_MODE(res_addr) != IS_REG
3052 && (i + 1) <= end
3053 && zend_jit_next_is_send_result(opline)) {
3054 i++;
3055 res_use_info = -1;
3056 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3057 if (!zend_jit_reuse_ip(&dasm_state)) {
3058 goto jit_failure;
3059 }
3060 } else {
3061 res_use_info = -1;
3062
3063 if (opline->result_type == IS_CV
3064 && ssa->vars
3065 && ssa_op->result_use >= 0
3066 && !ssa->vars[ssa_op->result_use].no_val) {
3067 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
3068
3069 if (Z_MODE(res_use_addr) != IS_REG
3070 || Z_LOAD(res_use_addr)
3071 || Z_STORE(res_use_addr)) {
3072 res_use_info = RES_USE_INFO();
3073 }
3074 }
3075 }
3076 res_info = RES_INFO();
3077 if (opline->opcode == ZEND_ADD &&
3078 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
3079 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
3080 if (!zend_jit_add_arrays(&dasm_state, opline, op1_info, OP1_REG_ADDR(), op2_info, OP2_REG_ADDR(), res_addr)) {
3081 goto jit_failure;
3082 }
3083 } else {
3084 if (!zend_jit_math(&dasm_state, opline,
3085 op1_info, OP1_REG_ADDR(),
3086 op2_info, OP2_REG_ADDR(),
3087 res_use_info, res_info, res_addr,
3088 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
3089 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3090 goto jit_failure;
3091 }
3092 }
3093 goto done;
3094 case ZEND_CONCAT:
3095 case ZEND_FAST_CONCAT:
3096 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3097 break;
3098 }
3099 op1_info = OP1_INFO();
3100 op2_info = OP2_INFO();
3101 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
3102 break;
3103 }
3104 if (!(op1_info & MAY_BE_STRING) ||
3105 !(op2_info & MAY_BE_STRING)) {
3106 break;
3107 }
3108 res_addr = RES_REG_ADDR();
3109 if ((i + 1) <= end
3110 && zend_jit_next_is_send_result(opline)) {
3111 i++;
3112 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3113 if (!zend_jit_reuse_ip(&dasm_state)) {
3114 goto jit_failure;
3115 }
3116 }
3117 if (!zend_jit_concat(&dasm_state, opline,
3118 op1_info, op2_info, res_addr,
3119 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3120 goto jit_failure;
3121 }
3122 goto done;
3123 case ZEND_ASSIGN_OP:
3124 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
3125 break;
3126 }
3127 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3128 break;
3129 }
3130 op1_info = OP1_INFO();
3131 op2_info = OP2_INFO();
3132 if (!zend_jit_supported_binary_op(
3133 opline->extended_value, op1_info, op2_info)) {
3134 break;
3135 }
3136 op1_def_info = OP1_DEF_INFO();
3137 if (!zend_jit_assign_op(&dasm_state, opline,
3138 op1_info, op1_def_info, OP1_RANGE(),
3139 op2_info, OP2_RANGE(),
3140 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
3141 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3142 goto jit_failure;
3143 }
3144 goto done;
3145 case ZEND_ASSIGN_DIM_OP:
3146 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
3147 break;
3148 }
3149 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3150 break;
3151 }
3152 if (!zend_jit_supported_binary_op(
3153 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3154 break;
3155 }
3156 if (!zend_jit_assign_dim_op(&dasm_state, opline,
3157 OP1_INFO(), OP1_DEF_INFO(), OP1_REG_ADDR(), OP2_INFO(),
3158 OP1_DATA_INFO(), OP1_DATA_RANGE(), IS_UNKNOWN,
3159 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3160 goto jit_failure;
3161 }
3162 goto done;
3163 case ZEND_ASSIGN_DIM:
3164 if (opline->op1_type != IS_CV) {
3165 break;
3166 }
3167 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3168 break;
3169 }
3170 if (!zend_jit_assign_dim(&dasm_state, opline,
3171 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), OP1_DATA_INFO(), IS_UNKNOWN,
3172 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3173 goto jit_failure;
3174 }
3175 goto done;
3176 case ZEND_PRE_INC_OBJ:
3177 case ZEND_PRE_DEC_OBJ:
3178 case ZEND_POST_INC_OBJ:
3179 case ZEND_POST_DEC_OBJ:
3180 if (opline->op2_type != IS_CONST
3181 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3182 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3183 break;
3184 }
3185 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3186 break;
3187 }
3188 ce = NULL;
3189 ce_is_instanceof = 0;
3190 on_this = 0;
3191 if (opline->op1_type == IS_UNUSED) {
3192 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3193 ce = op_array->scope;
3194 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3195 op1_addr = 0;
3196 on_this = 1;
3197 } else {
3198 op1_info = OP1_INFO();
3199 if (!(op1_info & MAY_BE_OBJECT)) {
3200 break;
3201 }
3202 op1_addr = OP1_REG_ADDR();
3203 if (ssa->var_info && ssa->ops) {
3204 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3205 if (ssa_op->op1_use >= 0) {
3206 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3207 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3208 ce = op1_ssa->ce;
3209 ce_is_instanceof = op1_ssa->is_instanceof;
3210 }
3211 }
3212 }
3213 }
3214 if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3215 op1_info, op1_addr,
3216 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
3217 goto jit_failure;
3218 }
3219 goto done;
3220 case ZEND_ASSIGN_OBJ_OP:
3221 if (opline->result_type != IS_UNUSED) {
3222 break;
3223 }
3224 if (opline->op2_type != IS_CONST
3225 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3226 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3227 break;
3228 }
3229 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3230 break;
3231 }
3232 if (!zend_jit_supported_binary_op(
3233 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
3234 break;
3235 }
3236 ce = NULL;
3237 ce_is_instanceof = 0;
3238 on_this = 0;
3239 if (opline->op1_type == IS_UNUSED) {
3240 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3241 ce = op_array->scope;
3242 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3243 op1_addr = 0;
3244 on_this = 1;
3245 } else {
3246 op1_info = OP1_INFO();
3247 if (!(op1_info & MAY_BE_OBJECT)) {
3248 break;
3249 }
3250 op1_addr = OP1_REG_ADDR();
3251 if (ssa->var_info && ssa->ops) {
3252 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3253 if (ssa_op->op1_use >= 0) {
3254 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3255 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3256 ce = op1_ssa->ce;
3257 ce_is_instanceof = op1_ssa->is_instanceof;
3258 }
3259 }
3260 }
3261 }
3262 if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op,
3263 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_RANGE(),
3264 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
3265 goto jit_failure;
3266 }
3267 goto done;
3268 case ZEND_ASSIGN_OBJ:
3269 if (opline->op2_type != IS_CONST
3270 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3271 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3272 break;
3273 }
3274 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3275 break;
3276 }
3277 ce = NULL;
3278 ce_is_instanceof = 0;
3279 on_this = 0;
3280 if (opline->op1_type == IS_UNUSED) {
3281 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3282 ce = op_array->scope;
3283 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3284 op1_addr = 0;
3285 on_this = 1;
3286 } else {
3287 op1_info = OP1_INFO();
3288 if (!(op1_info & MAY_BE_OBJECT)) {
3289 break;
3290 }
3291 op1_addr = OP1_REG_ADDR();
3292 if (ssa->var_info && ssa->ops) {
3293 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3294 if (ssa_op->op1_use >= 0) {
3295 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3296 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3297 ce = op1_ssa->ce;
3298 ce_is_instanceof = op1_ssa->is_instanceof;
3299 }
3300 }
3301 }
3302 }
3303 if (!zend_jit_assign_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3304 op1_info, op1_addr, OP1_DATA_INFO(),
3305 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN,
3306 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3307 goto jit_failure;
3308 }
3309 goto done;
3310 case ZEND_ASSIGN:
3311 if (opline->op1_type != IS_CV) {
3312 break;
3313 }
3314 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3315 break;
3316 }
3317 op2_addr = OP2_REG_ADDR();
3318 if (ra
3319 && ssa->ops[opline - op_array->opcodes].op2_def >= 0
3320 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) {
3321 op2_def_addr = OP2_DEF_REG_ADDR();
3322 } else {
3323 op2_def_addr = op2_addr;
3324 }
3325 op1_info = OP1_INFO();
3326 if (ra && ssa->vars[ssa_op->op1_use].no_val) {
3327 op1_info |= MAY_BE_UNDEF; // requres type assignment
3328 }
3329 if (opline->result_type == IS_UNUSED) {
3330 res_addr = 0;
3331 res_info = -1;
3332 } else {
3333 res_addr = RES_REG_ADDR();
3334 res_info = RES_INFO();
3335 if (Z_MODE(res_addr) != IS_REG
3336 && (i + 1) <= end
3337 && zend_jit_next_is_send_result(opline)
3338 && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) {
3339 i++;
3340 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
3341 if (!zend_jit_reuse_ip(&dasm_state)) {
3342 goto jit_failure;
3343 }
3344 }
3345 }
3346 if (!zend_jit_assign(&dasm_state, opline,
3347 op1_info, OP1_REG_ADDR(),
3348 OP1_DEF_INFO(), OP1_DEF_REG_ADDR(),
3349 OP2_INFO(), op2_addr, op2_def_addr,
3350 res_info, res_addr,
3351 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3352 goto jit_failure;
3353 }
3354 goto done;
3355 case ZEND_QM_ASSIGN:
3356 op1_addr = OP1_REG_ADDR();
3357 if (ra
3358 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
3359 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
3360 op1_def_addr = OP1_DEF_REG_ADDR();
3361 } else {
3362 op1_def_addr = op1_addr;
3363 }
3364 if (!zend_jit_qm_assign(&dasm_state, opline,
3365 OP1_INFO(), op1_addr, op1_def_addr,
3366 -1, RES_INFO(), RES_REG_ADDR())) {
3367 goto jit_failure;
3368 }
3369 goto done;
3370 case ZEND_INIT_FCALL:
3371 case ZEND_INIT_FCALL_BY_NAME:
3372 case ZEND_INIT_NS_FCALL_BY_NAME:
3373 if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level, NULL, 0)) {
3374 goto jit_failure;
3375 }
3376 goto done;
3377 case ZEND_SEND_VAL:
3378 case ZEND_SEND_VAL_EX:
3379 if (opline->op2_type == IS_CONST) {
3380 /* Named parameters not supported in JIT (yet) */
3381 break;
3382 }
3383 if (opline->opcode == ZEND_SEND_VAL_EX
3384 && opline->op2.num > MAX_ARG_FLAG_NUM) {
3385 break;
3386 }
3387 if (!zend_jit_send_val(&dasm_state, opline,
3388 OP1_INFO(), OP1_REG_ADDR())) {
3389 goto jit_failure;
3390 }
3391 goto done;
3392 case ZEND_SEND_REF:
3393 if (opline->op2_type == IS_CONST) {
3394 /* Named parameters not supported in JIT (yet) */
3395 break;
3396 }
3397 if (!zend_jit_send_ref(&dasm_state, opline, op_array,
3398 OP1_INFO(), 0)) {
3399 goto jit_failure;
3400 }
3401 goto done;
3402 case ZEND_SEND_VAR:
3403 case ZEND_SEND_VAR_EX:
3404 case ZEND_SEND_VAR_NO_REF:
3405 case ZEND_SEND_VAR_NO_REF_EX:
3406 case ZEND_SEND_FUNC_ARG:
3407 if (opline->op2_type == IS_CONST) {
3408 /* Named parameters not supported in JIT (yet) */
3409 break;
3410 }
3411 if ((opline->opcode == ZEND_SEND_VAR_EX
3412 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
3413 && opline->op2.num > MAX_ARG_FLAG_NUM) {
3414 break;
3415 }
3416 op1_addr = OP1_REG_ADDR();
3417 if (ra
3418 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
3419 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
3420 op1_def_addr = OP1_DEF_REG_ADDR();
3421 } else {
3422 op1_def_addr = op1_addr;
3423 }
3424 if (!zend_jit_send_var(&dasm_state, opline, op_array,
3425 OP1_INFO(), op1_addr, op1_def_addr)) {
3426 goto jit_failure;
3427 }
3428 goto done;
3429 case ZEND_CHECK_FUNC_ARG:
3430 if (opline->op2_type == IS_CONST) {
3431 /* Named parameters not supported in JIT (yet) */
3432 break;
3433 }
3434 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
3435 break;
3436 }
3437 if (!zend_jit_check_func_arg(&dasm_state, opline)) {
3438 goto jit_failure;
3439 }
3440 goto done;
3441 case ZEND_CHECK_UNDEF_ARGS:
3442 if (!zend_jit_check_undef_args(&dasm_state, opline)) {
3443 goto jit_failure;
3444 }
3445 goto done;
3446 case ZEND_DO_UCALL:
3447 is_terminated = 1;
3448 ZEND_FALLTHROUGH;
3449 case ZEND_DO_ICALL:
3450 case ZEND_DO_FCALL_BY_NAME:
3451 case ZEND_DO_FCALL:
3452 if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level, b + 1, NULL)) {
3453 goto jit_failure;
3454 }
3455 goto done;
3456 case ZEND_IS_EQUAL:
3457 case ZEND_IS_NOT_EQUAL:
3458 case ZEND_IS_SMALLER:
3459 case ZEND_IS_SMALLER_OR_EQUAL:
3460 case ZEND_CASE: {
3461 res_addr = RES_REG_ADDR();
3462 if ((opline->result_type & IS_TMP_VAR)
3463 && (i + 1) <= end
3464 && ((opline+1)->opcode == ZEND_JMPZ
3465 || (opline+1)->opcode == ZEND_JMPNZ
3466 || (opline+1)->opcode == ZEND_JMPZ_EX
3467 || (opline+1)->opcode == ZEND_JMPNZ_EX)
3468 && (opline+1)->op1_type == IS_TMP_VAR
3469 && (opline+1)->op1.var == opline->result.var) {
3470 i++;
3471 smart_branch_opcode = (opline+1)->opcode;
3472 target_label = ssa->cfg.blocks[b].successors[0];
3473 target_label2 = ssa->cfg.blocks[b].successors[1];
3474 /* For EX variant write into the result of EX opcode. */
3475 if ((opline+1)->opcode == ZEND_JMPZ_EX
3476 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
3477 res_addr = OP_REG_ADDR(opline + 1, result_type, result, result_def);
3478 }
3479 } else {
3480 smart_branch_opcode = 0;
3481 target_label = target_label2 = (uint32_t)-1;
3482 }
3483 if (!zend_jit_cmp(&dasm_state, opline,
3484 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
3485 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
3486 res_addr,
3487 zend_may_throw(opline, ssa_op, op_array, ssa),
3488 smart_branch_opcode, target_label, target_label2,
3489 NULL, 0)) {
3490 goto jit_failure;
3491 }
3492 goto done;
3493 }
3494 case ZEND_IS_IDENTICAL:
3495 case ZEND_IS_NOT_IDENTICAL:
3496 case ZEND_CASE_STRICT:
3497 if ((opline->result_type & IS_TMP_VAR)
3498 && (i + 1) <= end
3499 && ((opline+1)->opcode == ZEND_JMPZ
3500 || (opline+1)->opcode == ZEND_JMPNZ)
3501 && (opline+1)->op1_type == IS_TMP_VAR
3502 && (opline+1)->op1.var == opline->result.var) {
3503 i++;
3504 smart_branch_opcode = (opline+1)->opcode;
3505 target_label = ssa->cfg.blocks[b].successors[0];
3506 target_label2 = ssa->cfg.blocks[b].successors[1];
3507 } else {
3508 smart_branch_opcode = 0;
3509 target_label = target_label2 = (uint32_t)-1;
3510 }
3511 if (!zend_jit_identical(&dasm_state, opline,
3512 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
3513 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
3514 RES_REG_ADDR(),
3515 zend_may_throw(opline, ssa_op, op_array, ssa),
3516 smart_branch_opcode, target_label, target_label2,
3517 NULL, 0)) {
3518 goto jit_failure;
3519 }
3520 goto done;
3521 case ZEND_DEFINED:
3522 if ((opline->result_type & IS_TMP_VAR)
3523 && (i + 1) <= end
3524 && ((opline+1)->opcode == ZEND_JMPZ
3525 || (opline+1)->opcode == ZEND_JMPNZ)
3526 && (opline+1)->op1_type == IS_TMP_VAR
3527 && (opline+1)->op1.var == opline->result.var) {
3528 i++;
3529 smart_branch_opcode = (opline+1)->opcode;
3530 target_label = ssa->cfg.blocks[b].successors[0];
3531 target_label2 = ssa->cfg.blocks[b].successors[1];
3532 } else {
3533 smart_branch_opcode = 0;
3534 target_label = target_label2 = (uint32_t)-1;
3535 }
3536 if (!zend_jit_defined(&dasm_state, opline, smart_branch_opcode, target_label, target_label2, NULL)) {
3537 goto jit_failure;
3538 }
3539 goto done;
3540 case ZEND_TYPE_CHECK:
3541 if (opline->extended_value == MAY_BE_RESOURCE) {
3542 // TODO: support for is_resource() ???
3543 break;
3544 }
3545 if ((opline->result_type & IS_TMP_VAR)
3546 && (i + 1) <= end
3547 && ((opline+1)->opcode == ZEND_JMPZ
3548 || (opline+1)->opcode == ZEND_JMPNZ)
3549 && (opline+1)->op1_type == IS_TMP_VAR
3550 && (opline+1)->op1.var == opline->result.var) {
3551 i++;
3552 smart_branch_opcode = (opline+1)->opcode;
3553 target_label = ssa->cfg.blocks[b].successors[0];
3554 target_label2 = ssa->cfg.blocks[b].successors[1];
3555 } else {
3556 smart_branch_opcode = 0;
3557 target_label = target_label2 = (uint32_t)-1;
3558 }
3559 if (!zend_jit_type_check(&dasm_state, opline, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) {
3560 goto jit_failure;
3561 }
3562 goto done;
3563 case ZEND_RETURN:
3564 op1_info = OP1_INFO();
3565 if ((PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info))
3566 || op_array->type == ZEND_EVAL_CODE
3567 // TODO: support for top-level code
3568 || !op_array->function_name
3569 // TODO: support for IS_UNDEF ???
3570 || (op1_info & MAY_BE_UNDEF)) {
3571 if (!zend_jit_tail_handler(&dasm_state, opline)) {
3572 goto jit_failure;
3573 }
3574 } else {
3575 int j;
3576 bool left_frame = 0;
3577
3578 if (!zend_jit_return(&dasm_state, opline, op_array,
3579 op1_info, OP1_REG_ADDR())) {
3580 goto jit_failure;
3581 }
3582 if (jit_return_label >= 0) {
3583 if (!zend_jit_jmp(&dasm_state, jit_return_label)) {
3584 goto jit_failure;
3585 }
3586 goto done;
3587 }
3588 jit_return_label = ssa->cfg.blocks_count * 2;
3589 if (!zend_jit_label(&dasm_state, jit_return_label)) {
3590 goto jit_failure;
3591 }
3592 if (op_array->last_var > 100) {
3593 /* To many CVs to unroll */
3594 if (!zend_jit_free_cvs(&dasm_state)) {
3595 goto jit_failure;
3596 }
3597 left_frame = 1;
3598 }
3599 if (!left_frame) {
3600 for (j = 0 ; j < op_array->last_var; j++) {
3601 uint32_t info = zend_ssa_cv_info(op_array, ssa, j);
3602
3603 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
3604 if (!left_frame) {
3605 left_frame = 1;
3606 if (!zend_jit_leave_frame(&dasm_state)) {
3607 goto jit_failure;
3608 }
3609 }
3610 if (!zend_jit_free_cv(&dasm_state, info, j)) {
3611 goto jit_failure;
3612 }
3613 }
3614 }
3615 }
3616 if (!zend_jit_leave_func(&dasm_state, op_array, opline, op1_info, left_frame,
3617 NULL, NULL, (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, 1)) {
3618 goto jit_failure;
3619 }
3620 }
3621 goto done;
3622 case ZEND_BOOL:
3623 case ZEND_BOOL_NOT:
3624 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
3625 OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(),
3626 -1, -1,
3627 zend_may_throw(opline, ssa_op, op_array, ssa),
3628 opline->opcode, NULL)) {
3629 goto jit_failure;
3630 }
3631 goto done;
3632 case ZEND_JMPZ:
3633 case ZEND_JMPNZ:
3634 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
3635 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
3636 /* smart branch */
3637 if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
3638 goto jit_failure;
3639 }
3640 goto done;
3641 }
3642 ZEND_FALLTHROUGH;
3643 case ZEND_JMPZ_EX:
3644 case ZEND_JMPNZ_EX:
3645 if (opline->result_type == IS_UNDEF) {
3646 res_addr = 0;
3647 } else {
3648 res_addr = RES_REG_ADDR();
3649 }
3650 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
3651 OP1_INFO(), OP1_REG_ADDR(), res_addr,
3652 ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1],
3653 zend_may_throw(opline, ssa_op, op_array, ssa),
3654 opline->opcode, NULL)) {
3655 goto jit_failure;
3656 }
3657 goto done;
3658 case ZEND_ISSET_ISEMPTY_CV:
3659 if ((opline->extended_value & ZEND_ISEMPTY)) {
3660 // TODO: support for empty() ???
3661 break;
3662 }
3663 if ((opline->result_type & IS_TMP_VAR)
3664 && (i + 1) <= end
3665 && ((opline+1)->opcode == ZEND_JMPZ
3666 || (opline+1)->opcode == ZEND_JMPNZ)
3667 && (opline+1)->op1_type == IS_TMP_VAR
3668 && (opline+1)->op1.var == opline->result.var) {
3669 i++;
3670 smart_branch_opcode = (opline+1)->opcode;
3671 target_label = ssa->cfg.blocks[b].successors[0];
3672 target_label2 = ssa->cfg.blocks[b].successors[1];
3673 } else {
3674 smart_branch_opcode = 0;
3675 target_label = target_label2 = (uint32_t)-1;
3676 }
3677 if (!zend_jit_isset_isempty_cv(&dasm_state, opline,
3678 OP1_INFO(), OP1_REG_ADDR(),
3679 smart_branch_opcode, target_label, target_label2,
3680 NULL)) {
3681 goto jit_failure;
3682 }
3683 goto done;
3684 case ZEND_IN_ARRAY:
3685 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
3686 break;
3687 }
3688 op1_info = OP1_INFO();
3689 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
3690 break;
3691 }
3692 if ((opline->result_type & IS_TMP_VAR)
3693 && (i + 1) <= end
3694 && ((opline+1)->opcode == ZEND_JMPZ
3695 || (opline+1)->opcode == ZEND_JMPNZ)
3696 && (opline+1)->op1_type == IS_TMP_VAR
3697 && (opline+1)->op1.var == opline->result.var) {
3698 i++;
3699 smart_branch_opcode = (opline+1)->opcode;
3700 target_label = ssa->cfg.blocks[b].successors[0];
3701 target_label2 = ssa->cfg.blocks[b].successors[1];
3702 } else {
3703 smart_branch_opcode = 0;
3704 target_label = target_label2 = (uint32_t)-1;
3705 }
3706 if (!zend_jit_in_array(&dasm_state, opline,
3707 op1_info, OP1_REG_ADDR(),
3708 smart_branch_opcode, target_label, target_label2,
3709 NULL)) {
3710 goto jit_failure;
3711 }
3712 goto done;
3713 case ZEND_FETCH_DIM_R:
3714 case ZEND_FETCH_DIM_IS:
3715 case ZEND_FETCH_LIST_R:
3716 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3717 break;
3718 }
3719 if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op,
3720 OP1_INFO(), OP1_REG_ADDR(), 0,
3721 OP2_INFO(), RES_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
3722 goto jit_failure;
3723 }
3724 goto done;
3725 case ZEND_FETCH_DIM_W:
3726 case ZEND_FETCH_DIM_RW:
3727 // case ZEND_FETCH_DIM_UNSET:
3728 case ZEND_FETCH_LIST_W:
3729 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3730 break;
3731 }
3732 if (opline->op1_type != IS_CV) {
3733 break;
3734 }
3735 if (!zend_jit_fetch_dim(&dasm_state, opline,
3736 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
3737 goto jit_failure;
3738 }
3739 goto done;
3740 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
3741 if ((opline->extended_value & ZEND_ISEMPTY)) {
3742 // TODO: support for empty() ???
3743 break;
3744 }
3745 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3746 break;
3747 }
3748 if ((opline->result_type & IS_TMP_VAR)
3749 && (i + 1) <= end
3750 && ((opline+1)->opcode == ZEND_JMPZ
3751 || (opline+1)->opcode == ZEND_JMPNZ)
3752 && (opline+1)->op1_type == IS_TMP_VAR
3753 && (opline+1)->op1.var == opline->result.var) {
3754 i++;
3755 smart_branch_opcode = (opline+1)->opcode;
3756 target_label = ssa->cfg.blocks[b].successors[0];
3757 target_label2 = ssa->cfg.blocks[b].successors[1];
3758 } else {
3759 smart_branch_opcode = 0;
3760 target_label = target_label2 = (uint32_t)-1;
3761 }
3762 if (!zend_jit_isset_isempty_dim(&dasm_state, opline,
3763 OP1_INFO(), OP1_REG_ADDR(), 0,
3764 OP2_INFO(), IS_UNKNOWN,
3765 zend_may_throw(opline, ssa_op, op_array, ssa),
3766 smart_branch_opcode, target_label, target_label2,
3767 NULL)) {
3768 goto jit_failure;
3769 }
3770 goto done;
3771 case ZEND_FETCH_OBJ_R:
3772 case ZEND_FETCH_OBJ_IS:
3773 case ZEND_FETCH_OBJ_W:
3774 if (opline->op2_type != IS_CONST
3775 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3776 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3777 break;
3778 }
3779 ce = NULL;
3780 ce_is_instanceof = 0;
3781 on_this = 0;
3782 if (opline->op1_type == IS_UNUSED) {
3783 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3784 op1_addr = 0;
3785 ce = op_array->scope;
3786 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3787 on_this = 1;
3788 } else {
3789 op1_info = OP1_INFO();
3790 if (!(op1_info & MAY_BE_OBJECT)) {
3791 break;
3792 }
3793 op1_addr = OP1_REG_ADDR();
3794 if (ssa->var_info && ssa->ops) {
3795 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3796 if (ssa_op->op1_use >= 0) {
3797 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3798 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3799 ce = op1_ssa->ce;
3800 ce_is_instanceof = op1_ssa->is_instanceof;
3801 }
3802 }
3803 }
3804 }
3805 if (!zend_jit_fetch_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3806 op1_info, op1_addr, 0, ce, ce_is_instanceof, on_this, 0, 0, NULL,
3807 IS_UNKNOWN,
3808 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3809 goto jit_failure;
3810 }
3811 goto done;
3812 case ZEND_BIND_GLOBAL:
3813 if (!ssa->ops || !ssa->var_info) {
3814 op1_info = MAY_BE_ANY|MAY_BE_REF;
3815 } else {
3816 op1_info = OP1_INFO();
3817 }
3818 if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) {
3819 goto jit_failure;
3820 }
3821 goto done;
3822 case ZEND_RECV:
3823 if (!zend_jit_recv(&dasm_state, opline, op_array)) {
3824 goto jit_failure;
3825 }
3826 goto done;
3827 case ZEND_RECV_INIT:
3828 if (!zend_jit_recv_init(&dasm_state, opline, op_array,
3829 (opline + 1)->opcode != ZEND_RECV_INIT,
3830 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3831 goto jit_failure;
3832 }
3833 goto done;
3834 case ZEND_FREE:
3835 case ZEND_FE_FREE:
3836 if (!zend_jit_free(&dasm_state, opline, OP1_INFO(),
3837 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3838 goto jit_failure;
3839 }
3840 goto done;
3841 case ZEND_ECHO:
3842 op1_info = OP1_INFO();
3843 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3844 break;
3845 }
3846 if (!zend_jit_echo(&dasm_state, opline, op1_info)) {
3847 goto jit_failure;
3848 }
3849 goto done;
3850 case ZEND_STRLEN:
3851 op1_info = OP1_INFO();
3852 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3853 break;
3854 }
3855 if (!zend_jit_strlen(&dasm_state, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR())) {
3856 goto jit_failure;
3857 }
3858 goto done;
3859 case ZEND_COUNT:
3860 op1_info = OP1_INFO();
3861 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
3862 break;
3863 }
3864 if (!zend_jit_count(&dasm_state, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
3865 goto jit_failure;
3866 }
3867 goto done;
3868 case ZEND_FETCH_THIS:
3869 if (!zend_jit_fetch_this(&dasm_state, opline, op_array, 0)) {
3870 goto jit_failure;
3871 }
3872 goto done;
3873 case ZEND_SWITCH_LONG:
3874 case ZEND_SWITCH_STRING:
3875 case ZEND_MATCH:
3876 if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL, NULL)) {
3877 goto jit_failure;
3878 }
3879 goto done;
3880 case ZEND_VERIFY_RETURN_TYPE:
3881 if (opline->op1_type == IS_UNUSED) {
3882 /* Always throws */
3883 break;
3884 }
3885 if (opline->op1_type == IS_CONST) {
3886 /* TODO Different instruction format, has return value */
3887 break;
3888 }
3889 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
3890 /* Not worth bothering with */
3891 break;
3892 }
3893 if (OP1_INFO() & MAY_BE_REF) {
3894 /* TODO May need reference unwrapping. */
3895 break;
3896 }
3897 if (!zend_jit_verify_return_type(&dasm_state, opline, op_array, OP1_INFO())) {
3898 goto jit_failure;
3899 }
3900 goto done;
3901 case ZEND_FE_RESET_R:
3902 op1_info = OP1_INFO();
3903 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
3904 break;
3905 }
3906 if (!zend_jit_fe_reset(&dasm_state, opline, op1_info)) {
3907 goto jit_failure;
3908 }
3909 goto done;
3910 case ZEND_FE_FETCH_R:
3911 op1_info = OP1_INFO();
3912 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
3913 break;
3914 }
3915 if (!zend_jit_fe_fetch(&dasm_state, opline, op1_info, OP2_INFO(),
3916 ssa->cfg.blocks[b].successors[0], opline->opcode, NULL)) {
3917 goto jit_failure;
3918 }
3919 goto done;
3920 case ZEND_FETCH_CONSTANT:
3921 if (!zend_jit_fetch_constant(&dasm_state, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
3922 goto jit_failure;
3923 }
3924 goto done;
3925 case ZEND_INIT_METHOD_CALL:
3926 if (opline->op2_type != IS_CONST
3927 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
3928 break;
3929 }
3930 ce = NULL;
3931 ce_is_instanceof = 0;
3932 on_this = 0;
3933 if (opline->op1_type == IS_UNUSED) {
3934 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3935 op1_addr = 0;
3936 ce = op_array->scope;
3937 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3938 on_this = 1;
3939 } else {
3940 op1_info = OP1_INFO();
3941 if (!(op1_info & MAY_BE_OBJECT)) {
3942 break;
3943 }
3944 op1_addr = OP1_REG_ADDR();
3945 if (ssa->var_info && ssa->ops) {
3946 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3947 if (ssa_op->op1_use >= 0) {
3948 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3949 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3950 ce = op1_ssa->ce;
3951 ce_is_instanceof = op1_ssa->is_instanceof;
3952 }
3953 }
3954 }
3955 }
3956 if (!zend_jit_init_method_call(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level,
3957 op1_info, op1_addr, ce, ce_is_instanceof, on_this, 0, NULL,
3958 NULL, 0, 0)) {
3959 goto jit_failure;
3960 }
3961 goto done;
3962 case ZEND_ROPE_INIT:
3963 case ZEND_ROPE_ADD:
3964 case ZEND_ROPE_END:
3965 op2_info = OP2_INFO();
3966 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3967 break;
3968 }
3969 if (!zend_jit_rope(&dasm_state, opline, op2_info)) {
3970 goto jit_failure;
3971 }
3972 goto done;
3973 default:
3974 break;
3975 }
3976 }
3977
3978 switch (opline->opcode) {
3979 case ZEND_RECV_INIT:
3980 case ZEND_BIND_GLOBAL:
3981 if (opline == op_array->opcodes ||
3982 opline->opcode != op_array->opcodes[i-1].opcode) {
3983 /* repeatable opcodes */
3984 if (!zend_jit_handler(&dasm_state, opline,
3985 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3986 goto jit_failure;
3987 }
3988 }
3989 zend_jit_set_last_valid_opline(opline+1);
3990 break;
3991 case ZEND_NOP:
3992 case ZEND_OP_DATA:
3993 case ZEND_SWITCH_LONG:
3994 case ZEND_SWITCH_STRING:
3995 break;
3996 case ZEND_MATCH:
3997 /* We have to exit to the VM because the MATCH handler performs an N-way jump for
3998 * which we can't generate simple (opcache.jit=1201) JIT code. */
3999 if (!zend_jit_tail_handler(&dasm_state, opline)) {
4000 goto jit_failure;
4001 }
4002 break;
4003 case ZEND_JMP:
4004 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
4005 const zend_op *target = OP_JMP_ADDR(opline, opline->op1);
4006
4007 if (!zend_jit_set_ip(&dasm_state, target)) {
4008 goto jit_failure;
4009 }
4010 }
4011 if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) {
4012 goto jit_failure;
4013 }
4014 is_terminated = 1;
4015 break;
4016 case ZEND_CATCH:
4017 case ZEND_FAST_CALL:
4018 case ZEND_FAST_RET:
4019 case ZEND_GENERATOR_CREATE:
4020 case ZEND_GENERATOR_RETURN:
4021 case ZEND_RETURN_BY_REF:
4022 case ZEND_RETURN:
4023 case ZEND_EXIT:
4024 case ZEND_MATCH_ERROR:
4025 /* switch through trampoline */
4026 case ZEND_YIELD:
4027 case ZEND_YIELD_FROM:
4028 if (!zend_jit_tail_handler(&dasm_state, opline)) {
4029 goto jit_failure;
4030 }
4031 is_terminated = 1;
4032 break;
4033 /* stackless execution */
4034 case ZEND_INCLUDE_OR_EVAL:
4035 case ZEND_DO_FCALL:
4036 case ZEND_DO_UCALL:
4037 case ZEND_DO_FCALL_BY_NAME:
4038 if (!zend_jit_call(&dasm_state, opline, b + 1)) {
4039 goto jit_failure;
4040 }
4041 is_terminated = 1;
4042 break;
4043 case ZEND_JMPZ:
4044 case ZEND_JMPNZ:
4045 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
4046 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
4047 /* smart branch */
4048 if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
4049 goto jit_failure;
4050 }
4051 goto done;
4052 }
4053 ZEND_FALLTHROUGH;
4054 case ZEND_JMPZ_EX:
4055 case ZEND_JMPNZ_EX:
4056 case ZEND_JMP_SET:
4057 case ZEND_COALESCE:
4058 case ZEND_JMP_NULL:
4059 case ZEND_FE_RESET_R:
4060 case ZEND_FE_RESET_RW:
4061 case ZEND_ASSERT_CHECK:
4062 case ZEND_FE_FETCH_R:
4063 case ZEND_FE_FETCH_RW:
4064 case ZEND_BIND_INIT_STATIC_OR_JMP:
4065 if (!zend_jit_handler(&dasm_state, opline,
4066 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
4067 !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
4068 goto jit_failure;
4069 }
4070 break;
4071 case ZEND_NEW:
4072 if (!zend_jit_handler(&dasm_state, opline, 1)) {
4073 return 0;
4074 }
4075 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
4076 zend_class_entry *ce = NULL;
4077
4078 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
4079 if (ssa->ops && ssa->var_info) {
4080 zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
4081 if (res_ssa->ce && !res_ssa->is_instanceof) {
4082 ce = res_ssa->ce;
4083 }
4084 }
4085 } else {
4086 if (opline->op1_type == IS_CONST) {
4087 zval *zv = RT_CONSTANT(opline, opline->op1);
4088 if (Z_TYPE_P(zv) == IS_STRING) {
4089 zval *lc = zv + 1;
4090 ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc));
4091 }
4092 }
4093 }
4094
4095 i++;
4096
4097 if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) {
4098 const zend_op *next_opline = opline + 1;
4099
4100 zend_jit_cond_jmp(&dasm_state, next_opline, ssa->cfg.blocks[b].successors[0]);
4101 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
4102 zend_jit_call(&dasm_state, next_opline, b + 1);
4103 is_terminated = 1;
4104 } else {
4105 zend_jit_do_fcall(&dasm_state, next_opline, op_array, ssa, call_level, b + 1, NULL);
4106 }
4107 }
4108
4109 /* We skip over the DO_FCALL, so decrement call_level ourselves. */
4110 call_level--;
4111 }
4112 break;
4113 default:
4114 if (!zend_jit_handler(&dasm_state, opline,
4115 zend_may_throw(opline, ssa_op, op_array, ssa))) {
4116 goto jit_failure;
4117 }
4118 if (i == end
4119 && (opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
4120 /* smart branch split across basic blocks */
4121 if (!zend_jit_cond_jmp(&dasm_state, opline + 2, ssa->cfg.blocks[b+1].successors[0])) {
4122 goto jit_failure;
4123 }
4124 if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b+1].successors[1])) {
4125 goto jit_failure;
4126 }
4127 is_terminated = 1;
4128 }
4129 }
4130 done:
4131 switch (opline->opcode) {
4132 case ZEND_DO_FCALL:
4133 case ZEND_DO_ICALL:
4134 case ZEND_DO_UCALL:
4135 case ZEND_DO_FCALL_BY_NAME:
4136 case ZEND_CALLABLE_CONVERT:
4137 call_level--;
4138 }
4139 }
4140 }
4141
4142 handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0,
4143 (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) ? SP_ADJ_VM : SP_ADJ_RET, SP_ADJ_JIT);
4144 if (!handler) {
4145 goto jit_failure;
4146 }
4147 dasm_free(&dasm_state);
4148
4149 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
4150 zend_arena_release(&CG(arena), checkpoint);
4151 }
4152 return SUCCESS;
4153
4154 jit_failure:
4155 if (dasm_state) {
4156 dasm_free(&dasm_state);
4157 }
4158 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
4159 zend_arena_release(&CG(arena), checkpoint);
4160 }
4161 return FAILURE;
4162 }
4163
zend_jit_collect_calls(zend_op_array * op_array,zend_script * script)4164 static void zend_jit_collect_calls(zend_op_array *op_array, zend_script *script)
4165 {
4166 zend_func_info *func_info;
4167
4168 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4169 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4170 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4171 func_info = ZEND_FUNC_INFO(op_array);
4172 } else {
4173 func_info = zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info));
4174 ZEND_SET_FUNC_INFO(op_array, func_info);
4175 }
4176 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, op_array, func_info);
4177 }
4178
zend_jit_cleanup_func_info(zend_op_array * op_array)4179 static void zend_jit_cleanup_func_info(zend_op_array *op_array)
4180 {
4181 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
4182 zend_call_info *caller_info, *callee_info;
4183
4184 if (func_info) {
4185 caller_info = func_info->caller_info;
4186 callee_info = func_info->callee_info;
4187
4188 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4189 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4190 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4191 func_info->num = 0;
4192 func_info->flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
4193 | ZEND_FUNC_JIT_ON_PROF_REQUEST
4194 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
4195 | ZEND_FUNC_JIT_ON_HOT_TRACE;
4196 memset(&func_info->ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
4197 } else {
4198 ZEND_SET_FUNC_INFO(op_array, NULL);
4199 }
4200
4201 while (caller_info) {
4202 if (caller_info->caller_op_array) {
4203 zend_jit_cleanup_func_info(caller_info->caller_op_array);
4204 }
4205 caller_info = caller_info->next_caller;
4206 }
4207 while (callee_info) {
4208 if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) {
4209 zend_jit_cleanup_func_info(&callee_info->callee_func->op_array);
4210 }
4211 callee_info = callee_info->next_callee;
4212 }
4213 }
4214 }
4215
zend_real_jit_func(zend_op_array * op_array,zend_script * script,const zend_op * rt_opline)4216 static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline)
4217 {
4218 zend_ssa ssa;
4219 void *checkpoint;
4220 zend_func_info *func_info;
4221
4222 if (*dasm_ptr == dasm_end) {
4223 return FAILURE;
4224 }
4225
4226 checkpoint = zend_arena_checkpoint(CG(arena));
4227
4228 /* Build SSA */
4229 memset(&ssa, 0, sizeof(zend_ssa));
4230
4231 if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) {
4232 goto jit_failure;
4233 }
4234
4235 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
4236 zend_jit_collect_calls(op_array, script);
4237 func_info = ZEND_FUNC_INFO(op_array);
4238 func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array);
4239 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
4240 zend_init_func_return_info(op_array, script, &func_info->return_info);
4241 }
4242 }
4243
4244 if (zend_jit_op_array_analyze2(op_array, script, &ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
4245 goto jit_failure;
4246 }
4247
4248 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
4249 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa);
4250 }
4251
4252 if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) {
4253 goto jit_failure;
4254 }
4255
4256 zend_jit_cleanup_func_info(op_array);
4257 zend_arena_release(&CG(arena), checkpoint);
4258 return SUCCESS;
4259
4260 jit_failure:
4261 zend_jit_cleanup_func_info(op_array);
4262 zend_arena_release(&CG(arena), checkpoint);
4263 return FAILURE;
4264 }
4265
4266 /* Run-time JIT handler */
zend_runtime_jit(void)4267 static int ZEND_FASTCALL zend_runtime_jit(void)
4268 {
4269 zend_execute_data *execute_data = EG(current_execute_data);
4270 zend_op_array *op_array = &EX(func)->op_array;
4271 zend_op *opline = op_array->opcodes;
4272 zend_jit_op_array_extension *jit_extension;
4273 bool do_bailout = 0;
4274
4275 zend_shared_alloc_lock();
4276
4277 if (ZEND_FUNC_INFO(op_array)) {
4278
4279 SHM_UNPROTECT();
4280 zend_jit_unprotect();
4281
4282 zend_try {
4283 /* restore original opcode handlers */
4284 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4285 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4286 opline++;
4287 }
4288 }
4289 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
4290 opline->handler = jit_extension->orig_handler;
4291
4292 /* perform real JIT for this function */
4293 zend_real_jit_func(op_array, NULL, NULL);
4294 } zend_catch {
4295 do_bailout = true;
4296 } zend_end_try();
4297
4298 zend_jit_protect();
4299 SHM_PROTECT();
4300 }
4301
4302 zend_shared_alloc_unlock();
4303
4304 if (do_bailout) {
4305 zend_bailout();
4306 }
4307
4308 /* JIT-ed code is going to be called by VM */
4309 return 0;
4310 }
4311
zend_jit_check_funcs(HashTable * function_table,bool is_method)4312 void zend_jit_check_funcs(HashTable *function_table, bool is_method) {
4313 zend_op *opline;
4314 zend_function *func;
4315 zend_op_array *op_array;
4316 uintptr_t counter;
4317 zend_jit_op_array_extension *jit_extension;
4318
4319 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(function_table, func) {
4320 if (func->type == ZEND_INTERNAL_FUNCTION) {
4321 break;
4322 }
4323 op_array = &func->op_array;
4324 opline = op_array->opcodes;
4325 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4326 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4327 opline++;
4328 }
4329 }
4330 if (opline->handler == zend_jit_profile_jit_handler) {
4331 if (!RUN_TIME_CACHE(op_array)) {
4332 continue;
4333 }
4334 counter = (uintptr_t)ZEND_COUNTER_INFO(op_array);
4335 ZEND_COUNTER_INFO(op_array) = 0;
4336 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
4337 opline->handler = jit_extension->orig_handler;
4338 if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) {
4339 zend_real_jit_func(op_array, NULL, NULL);
4340 }
4341 }
4342 } ZEND_HASH_FOREACH_END();
4343 }
4344
zend_jit_hot_func(zend_execute_data * execute_data,const zend_op * opline)4345 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
4346 {
4347 zend_op_array *op_array = &EX(func)->op_array;
4348 zend_jit_op_array_hot_extension *jit_extension;
4349 uint32_t i;
4350 bool do_bailout = 0;
4351
4352 zend_shared_alloc_lock();
4353 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
4354
4355 if (jit_extension) {
4356 SHM_UNPROTECT();
4357 zend_jit_unprotect();
4358
4359 zend_try {
4360 for (i = 0; i < op_array->last; i++) {
4361 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
4362 }
4363
4364 /* perform real JIT for this function */
4365 zend_real_jit_func(op_array, NULL, opline);
4366 } zend_catch {
4367 do_bailout = 1;
4368 } zend_end_try();
4369
4370 zend_jit_protect();
4371 SHM_PROTECT();
4372 }
4373
4374 zend_shared_alloc_unlock();
4375
4376 if (do_bailout) {
4377 zend_bailout();
4378 }
4379 /* JIT-ed code is going to be called by VM */
4380 }
4381
zend_jit_setup_hot_counters_ex(zend_op_array * op_array,zend_cfg * cfg)4382 static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg)
4383 {
4384 if (JIT_G(hot_func)) {
4385 zend_op *opline = op_array->opcodes;
4386
4387 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4388 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4389 opline++;
4390 }
4391 }
4392
4393 opline->handler = (const void*)zend_jit_func_hot_counter_handler;
4394 }
4395
4396 if (JIT_G(hot_loop)) {
4397 uint32_t i;
4398
4399 for (i = 0; i < cfg->blocks_count; i++) {
4400 if ((cfg->blocks[i].flags & ZEND_BB_REACHABLE) &&
4401 (cfg->blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
4402 op_array->opcodes[cfg->blocks[i].start].handler =
4403 (const void*)zend_jit_loop_hot_counter_handler;
4404 }
4405 }
4406 }
4407 }
4408
zend_jit_restart_hot_counters(zend_op_array * op_array)4409 static int zend_jit_restart_hot_counters(zend_op_array *op_array)
4410 {
4411 zend_jit_op_array_hot_extension *jit_extension;
4412 zend_cfg cfg;
4413 uint32_t i;
4414
4415 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
4416 for (i = 0; i < op_array->last; i++) {
4417 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
4418 }
4419
4420 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
4421 return FAILURE;
4422 }
4423
4424 zend_jit_setup_hot_counters_ex(op_array, &cfg);
4425
4426 return SUCCESS;
4427 }
4428
zend_jit_setup_hot_counters(zend_op_array * op_array)4429 static int zend_jit_setup_hot_counters(zend_op_array *op_array)
4430 {
4431 zend_jit_op_array_hot_extension *jit_extension;
4432 zend_cfg cfg;
4433 uint32_t i;
4434
4435 ZEND_ASSERT(zend_jit_func_hot_counter_handler != NULL);
4436 ZEND_ASSERT(zend_jit_loop_hot_counter_handler != NULL);
4437
4438 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
4439 return FAILURE;
4440 }
4441
4442 jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*));
4443 if (!jit_extension) {
4444 return FAILURE;
4445 }
4446 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4447 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS;
4448 jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
4449 for (i = 0; i < op_array->last; i++) {
4450 jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
4451 }
4452 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4453
4454 zend_jit_setup_hot_counters_ex(op_array, &cfg);
4455
4456 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4457
4458 return SUCCESS;
4459 }
4460
4461 #include "jit/zend_jit_trace.c"
4462
zend_jit_op_array(zend_op_array * op_array,zend_script * script)4463 ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
4464 {
4465 if (dasm_ptr == NULL) {
4466 return FAILURE;
4467 }
4468
4469 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) {
4470 zend_jit_op_array_extension *jit_extension;
4471 zend_op *opline = op_array->opcodes;
4472
4473 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
4474 ZEND_SET_FUNC_INFO(op_array, NULL);
4475 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
4476 return SUCCESS;
4477 }
4478
4479 /* Set run-time JIT handler */
4480 ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL);
4481 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4482 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4483 opline++;
4484 }
4485 }
4486 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
4487 if (!jit_extension) {
4488 return FAILURE;
4489 }
4490 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4491 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC;
4492 jit_extension->orig_handler = (void*)opline->handler;
4493 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4494 opline->handler = (const void*)zend_jit_runtime_jit_handler;
4495 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4496
4497 return SUCCESS;
4498 } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) {
4499 zend_jit_op_array_extension *jit_extension;
4500 zend_op *opline = op_array->opcodes;
4501
4502 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
4503 ZEND_SET_FUNC_INFO(op_array, NULL);
4504 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
4505 return SUCCESS;
4506 }
4507
4508 ZEND_ASSERT(zend_jit_profile_jit_handler != NULL);
4509 if (op_array->function_name) {
4510 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4511 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4512 opline++;
4513 }
4514 }
4515 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
4516 if (!jit_extension) {
4517 return FAILURE;
4518 }
4519 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
4520 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST;
4521 jit_extension->orig_handler = (void*)opline->handler;
4522 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
4523 opline->handler = (const void*)zend_jit_profile_jit_handler;
4524 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
4525 }
4526
4527 return SUCCESS;
4528 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4529 return zend_jit_setup_hot_counters(op_array);
4530 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4531 return zend_jit_setup_hot_trace_counters(op_array);
4532 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4533 return zend_real_jit_func(op_array, script, NULL);
4534 } else {
4535 ZEND_UNREACHABLE();
4536 }
4537 }
4538
zend_jit_script(zend_script * script)4539 ZEND_EXT_API int zend_jit_script(zend_script *script)
4540 {
4541 void *checkpoint;
4542 zend_call_graph call_graph;
4543 zend_func_info *info;
4544 int i;
4545
4546 if (dasm_ptr == NULL || *dasm_ptr == dasm_end) {
4547 return FAILURE;
4548 }
4549
4550 checkpoint = zend_arena_checkpoint(CG(arena));
4551
4552 call_graph.op_arrays_count = 0;
4553 zend_build_call_graph(&CG(arena), script, &call_graph);
4554
4555 zend_analyze_call_graph(&CG(arena), script, &call_graph);
4556
4557 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
4558 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
4559 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS ||
4560 JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4561 for (i = 0; i < call_graph.op_arrays_count; i++) {
4562 if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
4563 goto jit_failure;
4564 }
4565 }
4566 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4567 for (i = 0; i < call_graph.op_arrays_count; i++) {
4568 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4569 if (info) {
4570 if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
4571 goto jit_failure;
4572 }
4573 info->flags = info->ssa.cfg.flags;
4574 }
4575 }
4576
4577 for (i = 0; i < call_graph.op_arrays_count; i++) {
4578 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4579 if (info) {
4580 info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]);
4581 if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
4582 zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info);
4583 }
4584 }
4585 }
4586
4587 for (i = 0; i < call_graph.op_arrays_count; i++) {
4588 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4589 if (info) {
4590 if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
4591 goto jit_failure;
4592 }
4593 info->flags = info->ssa.cfg.flags;
4594 }
4595 }
4596
4597 for (i = 0; i < call_graph.op_arrays_count; i++) {
4598 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
4599 if (info) {
4600 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
4601 zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa);
4602 }
4603 if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) {
4604 goto jit_failure;
4605 }
4606 }
4607 }
4608
4609 for (i = 0; i < call_graph.op_arrays_count; i++) {
4610 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
4611 }
4612 } else {
4613 ZEND_UNREACHABLE();
4614 }
4615
4616 zend_arena_release(&CG(arena), checkpoint);
4617
4618 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC
4619 || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST
4620 || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS
4621 || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4622 zend_class_entry *ce;
4623 zend_op_array *op_array;
4624
4625 ZEND_HASH_MAP_FOREACH_PTR(&script->class_table, ce) {
4626 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
4627 if (!ZEND_FUNC_INFO(op_array)) {
4628 void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes);
4629
4630 if (jit_extension) {
4631 ZEND_SET_FUNC_INFO(op_array, jit_extension);
4632 }
4633 }
4634 } ZEND_HASH_FOREACH_END();
4635 } ZEND_HASH_FOREACH_END();
4636 }
4637
4638 return SUCCESS;
4639
4640 jit_failure:
4641 if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4642 for (i = 0; i < call_graph.op_arrays_count; i++) {
4643 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
4644 }
4645 }
4646 zend_arena_release(&CG(arena), checkpoint);
4647 return FAILURE;
4648 }
4649
zend_jit_unprotect(void)4650 ZEND_EXT_API void zend_jit_unprotect(void)
4651 {
4652 #ifdef HAVE_MPROTECT
4653 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4654 int opts = PROT_READ | PROT_WRITE;
4655 #ifdef ZTS
4656 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
4657 if (zend_write_protect) {
4658 pthread_jit_write_protect_np(0);
4659 }
4660 #endif
4661 opts |= PROT_EXEC;
4662 #endif
4663 if (mprotect(dasm_buf, dasm_size, opts) != 0) {
4664 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4665 }
4666 }
4667 #elif _WIN32
4668 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4669 DWORD old, new;
4670 #ifdef ZTS
4671 new = PAGE_EXECUTE_READWRITE;
4672 #else
4673 new = PAGE_READWRITE;
4674 #endif
4675 if (!VirtualProtect(dasm_buf, dasm_size, new, &old)) {
4676 DWORD err = GetLastError();
4677 char *msg = php_win32_error_to_msg(err);
4678 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4679 php_win32_error_msg_free(msg);
4680 }
4681 }
4682 #endif
4683 }
4684
zend_jit_protect(void)4685 ZEND_EXT_API void zend_jit_protect(void)
4686 {
4687 #ifdef HAVE_MPROTECT
4688 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4689 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
4690 if (zend_write_protect) {
4691 pthread_jit_write_protect_np(1);
4692 }
4693 #endif
4694 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
4695 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4696 }
4697 }
4698 #elif _WIN32
4699 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4700 DWORD old;
4701
4702 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
4703 DWORD err = GetLastError();
4704 char *msg = php_win32_error_to_msg(err);
4705 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4706 php_win32_error_msg_free(msg);
4707 }
4708 }
4709 #endif
4710 }
4711
zend_jit_init_handlers(void)4712 static void zend_jit_init_handlers(void)
4713 {
4714 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
4715 zend_jit_runtime_jit_handler = dasm_labels[zend_lbhybrid_runtime_jit];
4716 zend_jit_profile_jit_handler = dasm_labels[zend_lbhybrid_profile_jit];
4717 zend_jit_func_hot_counter_handler = dasm_labels[zend_lbhybrid_func_hot_counter];
4718 zend_jit_loop_hot_counter_handler = dasm_labels[zend_lbhybrid_loop_hot_counter];
4719 zend_jit_func_trace_counter_handler = dasm_labels[zend_lbhybrid_func_trace_counter];
4720 zend_jit_ret_trace_counter_handler = dasm_labels[zend_lbhybrid_ret_trace_counter];
4721 zend_jit_loop_trace_counter_handler = dasm_labels[zend_lbhybrid_loop_trace_counter];
4722 } else {
4723 zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
4724 zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
4725 zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper;
4726 zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper;
4727 zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper;
4728 zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper;
4729 zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper;
4730 }
4731 }
4732
zend_jit_make_stubs(void)4733 static int zend_jit_make_stubs(void)
4734 {
4735 dasm_State* dasm_state = NULL;
4736 uint32_t i;
4737
4738 dasm_init(&dasm_state, DASM_MAXSECTION);
4739 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
4740
4741 for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) {
4742 dasm_setup(&dasm_state, dasm_actions);
4743 zend_jit_align_stub(&dasm_state);
4744 if (!zend_jit_stubs[i].stub(&dasm_state)) {
4745 return 0;
4746 }
4747 if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name, 0,
4748 zend_jit_stubs[i].offset, zend_jit_stubs[i].adjustment)) {
4749 return 0;
4750 }
4751 }
4752
4753 zend_jit_init_handlers();
4754
4755 dasm_free(&dasm_state);
4756 return 1;
4757 }
4758
zend_jit_globals_ctor(zend_jit_globals * jit_globals)4759 static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
4760 {
4761 memset(jit_globals, 0, sizeof(zend_jit_globals));
4762 zend_jit_trace_init_caches();
4763 }
4764
4765 #ifdef ZTS
zend_jit_globals_dtor(zend_jit_globals * jit_globals)4766 static void zend_jit_globals_dtor(zend_jit_globals *jit_globals)
4767 {
4768 zend_jit_trace_free_caches(jit_globals);
4769 }
4770 #endif
4771
zend_jit_parse_config_num(zend_long jit)4772 static int zend_jit_parse_config_num(zend_long jit)
4773 {
4774 if (jit == 0) {
4775 JIT_G(on) = 0;
4776 return SUCCESS;
4777 }
4778
4779 if (jit < 0) return FAILURE;
4780
4781 if (jit % 10 == 0 || jit % 10 > 5) return FAILURE;
4782 JIT_G(opt_level) = jit % 10;
4783
4784 jit /= 10;
4785 if (jit % 10 > 5 || jit % 10 == 4) return FAILURE;
4786 JIT_G(trigger) = jit % 10;
4787
4788 jit /= 10;
4789 if (jit % 10 > 2) return FAILURE;
4790 JIT_G(opt_flags) = jit % 10;
4791
4792 jit /= 10;
4793 if (jit % 10 > 1) return FAILURE;
4794 JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0);
4795
4796 if (jit / 10 != 0) return FAILURE;
4797
4798 JIT_G(on) = 1;
4799
4800 return SUCCESS;
4801 }
4802
zend_jit_config(zend_string * jit,int stage)4803 ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage)
4804 {
4805 if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) {
4806 if (stage == ZEND_INI_STAGE_RUNTIME) {
4807 zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)");
4808 }
4809 return FAILURE;
4810 }
4811
4812 if (zend_string_equals_literal_ci(jit, "disable")) {
4813 JIT_G(enabled) = 0;
4814 JIT_G(on) = 0;
4815 return SUCCESS;
4816 } else if (ZSTR_LEN(jit) == 0
4817 || zend_string_equals_literal_ci(jit, "0")
4818 || zend_string_equals_literal_ci(jit, "off")
4819 || zend_string_equals_literal_ci(jit, "no")
4820 || zend_string_equals_literal_ci(jit, "false")) {
4821 JIT_G(enabled) = 1;
4822 JIT_G(on) = 0;
4823 return SUCCESS;
4824 } else if (zend_string_equals_literal_ci(jit, "1")
4825 || zend_string_equals_literal_ci(jit, "on")
4826 || zend_string_equals_literal_ci(jit, "yes")
4827 || zend_string_equals_literal_ci(jit, "true")
4828 || zend_string_equals_literal_ci(jit, "tracing")) {
4829 JIT_G(enabled) = 1;
4830 JIT_G(on) = 1;
4831 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS;
4832 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
4833 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
4834 return SUCCESS;
4835 } else if (zend_string_equals_ci(jit, ZSTR_KNOWN(ZEND_STR_FUNCTION))) {
4836 JIT_G(enabled) = 1;
4837 JIT_G(on) = 1;
4838 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT;
4839 JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD;
4840 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
4841 return SUCCESS;
4842 } else {
4843 char *end;
4844 zend_long num = ZEND_STRTOL(ZSTR_VAL(jit), &end, 10);
4845 if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) {
4846 goto failure;
4847 }
4848 JIT_G(enabled) = 1;
4849 return SUCCESS;
4850 }
4851
4852 failure:
4853 zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number");
4854 JIT_G(enabled) = 0;
4855 JIT_G(on) = 0;
4856 return FAILURE;
4857 }
4858
zend_jit_debug_config(zend_long old_val,zend_long new_val,int stage)4859 ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage)
4860 {
4861 if (stage != ZEND_INI_STAGE_STARTUP) {
4862 if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) {
4863 if (stage == ZEND_INI_STAGE_RUNTIME) {
4864 zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup");
4865 }
4866 return FAILURE;
4867 }
4868 #ifdef HAVE_DISASM
4869 if (new_val & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
4870 if (JIT_G(enabled) && !JIT_G(symbols) && !zend_jit_disasm_init()) {
4871 // TODO: error reporting and cleanup ???
4872 return FAILURE;
4873 }
4874 // TODO: symbols for JIT-ed code compiled before are missing ???
4875 }
4876 #endif
4877 }
4878 return SUCCESS;
4879 }
4880
zend_jit_init(void)4881 ZEND_EXT_API void zend_jit_init(void)
4882 {
4883 #ifdef ZTS
4884 jit_globals_id = ts_allocate_id(&jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, (ts_allocate_dtor) zend_jit_globals_dtor);
4885 #else
4886 zend_jit_globals_ctor(&jit_globals);
4887 #endif
4888 }
4889
zend_jit_check_support(void)4890 ZEND_EXT_API int zend_jit_check_support(void)
4891 {
4892 int i;
4893
4894 zend_jit_vm_kind = zend_vm_kind();
4895 if (zend_jit_vm_kind != ZEND_VM_KIND_CALL &&
4896 zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
4897 zend_error(E_WARNING, "JIT is compatible only with CALL and HYBRID VM. JIT disabled.");
4898 JIT_G(enabled) = 0;
4899 JIT_G(on) = 0;
4900 return FAILURE;
4901 }
4902
4903 if (zend_execute_ex != execute_ex) {
4904 if (zend_dtrace_enabled) {
4905 zend_error(E_WARNING, "JIT is incompatible with DTrace. JIT disabled.");
4906 } else if (strcmp(sapi_module.name, "phpdbg") != 0) {
4907 zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled.");
4908 }
4909 JIT_G(enabled) = 0;
4910 JIT_G(on) = 0;
4911 return FAILURE;
4912 }
4913
4914 for (i = 0; i <= 256; i++) {
4915 switch (i) {
4916 /* JIT has no effect on these opcodes */
4917 case ZEND_BEGIN_SILENCE:
4918 case ZEND_END_SILENCE:
4919 case ZEND_EXIT:
4920 break;
4921 default:
4922 if (zend_get_user_opcode_handler(i) != NULL) {
4923 zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled.");
4924 JIT_G(enabled) = 0;
4925 JIT_G(on) = 0;
4926 return FAILURE;
4927 }
4928 }
4929 }
4930
4931 return SUCCESS;
4932 }
4933
zend_jit_startup(void * buf,size_t size,bool reattached)4934 ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, bool reattached)
4935 {
4936 int ret;
4937
4938 zend_jit_halt_op = zend_get_halt_op();
4939
4940 if (zend_jit_setup() != SUCCESS) {
4941 // TODO: error reporting and cleanup ???
4942 return FAILURE;
4943 }
4944
4945 zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(ACCELERATOR_PRODUCT_NAME);
4946
4947 #ifdef HAVE_GDB
4948 zend_jit_gdb_init();
4949 #endif
4950
4951 #if ZEND_JIT_SUPPORT_CLDEMOTE
4952 cpu_support_cldemote = zend_cpu_supports_cldemote();
4953 #endif
4954
4955 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
4956 zend_write_protect = pthread_jit_write_protect_supported_np();
4957 #endif
4958
4959 dasm_buf = buf;
4960 dasm_size = size;
4961
4962 #ifdef HAVE_MPROTECT
4963 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
4964 if (zend_write_protect) {
4965 pthread_jit_write_protect_np(1);
4966 }
4967 #endif
4968 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
4969 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
4970 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4971 }
4972 } else {
4973 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
4974 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4975 }
4976 }
4977 #elif _WIN32
4978 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
4979 DWORD old;
4980
4981 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
4982 DWORD err = GetLastError();
4983 char *msg = php_win32_error_to_msg(err);
4984 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4985 php_win32_error_msg_free(msg);
4986 }
4987 } else {
4988 DWORD old;
4989
4990 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
4991 DWORD err = GetLastError();
4992 char *msg = php_win32_error_to_msg(err);
4993 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4994 php_win32_error_msg_free(msg);
4995 }
4996 }
4997 #endif
4998
4999 dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr) * 2);
5000 if (!reattached) {
5001 zend_jit_unprotect();
5002 *dasm_ptr = dasm_buf;
5003 #if _WIN32
5004 /* reserve space for global labels */
5005 *dasm_ptr = (void**)*dasm_ptr + zend_lb_MAX;
5006 #endif
5007 zend_jit_protect();
5008 }
5009
5010 #ifdef HAVE_DISASM
5011 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
5012 if (!zend_jit_disasm_init()) {
5013 // TODO: error reporting and cleanup ???
5014 return FAILURE;
5015 }
5016 }
5017 #endif
5018
5019 #ifdef HAVE_PERFTOOLS
5020 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
5021 zend_jit_perf_jitdump_open();
5022 }
5023 #endif
5024
5025 if (!reattached) {
5026 zend_jit_unprotect();
5027 ret = zend_jit_make_stubs();
5028 #if _WIN32
5029 /* save global labels */
5030 memcpy(dasm_buf, dasm_labels, sizeof(void*) * zend_lb_MAX);
5031 #endif
5032 zend_jit_protect();
5033 if (!ret) {
5034 // TODO: error reporting and cleanup ???
5035 return FAILURE;
5036 }
5037 } else {
5038 #if _WIN32
5039 /* restore global labels */
5040 memcpy(dasm_labels, dasm_buf, sizeof(void*) * zend_lb_MAX);
5041 zend_jit_init_handlers();
5042 #endif
5043 }
5044
5045 if (zend_jit_trace_startup(reattached) != SUCCESS) {
5046 return FAILURE;
5047 }
5048
5049 zend_jit_unprotect();
5050 #if ZEND_JIT_TARGET_ARM64
5051 /* reserve space for global labels veneers */
5052 dasm_labels_veneers = *dasm_ptr;
5053 *dasm_ptr = (void**)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT);
5054 memset(dasm_labels_veneers, 0, sizeof(void*) * ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT));
5055 #endif
5056 /* save JIT buffer pos */
5057 dasm_ptr[1] = dasm_ptr[0];
5058 zend_jit_protect();
5059
5060 return SUCCESS;
5061 }
5062
zend_jit_shutdown(void)5063 ZEND_EXT_API void zend_jit_shutdown(void)
5064 {
5065 if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE && dasm_ptr != NULL) {
5066 fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf));
5067 }
5068
5069 #ifdef HAVE_GDB
5070 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
5071 zend_jit_gdb_unregister();
5072 }
5073 #endif
5074
5075 #ifdef HAVE_DISASM
5076 zend_jit_disasm_shutdown();
5077 #endif
5078
5079 #ifdef HAVE_PERFTOOLS
5080 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
5081 zend_jit_perf_jitdump_close();
5082 }
5083 #endif
5084 #ifdef ZTS
5085 ts_free_id(jit_globals_id);
5086 #else
5087 zend_jit_trace_free_caches(&jit_globals);
5088 #endif
5089 }
5090
zend_jit_reset_counters(void)5091 static void zend_jit_reset_counters(void)
5092 {
5093 int i;
5094
5095 for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
5096 zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT;
5097 }
5098 }
5099
zend_jit_activate(void)5100 ZEND_EXT_API void zend_jit_activate(void)
5101 {
5102 zend_jit_profile_counter = 0;
5103 if (JIT_G(on)) {
5104 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
5105 zend_jit_reset_counters();
5106 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
5107 zend_jit_reset_counters();
5108 zend_jit_trace_reset_caches();
5109 }
5110 }
5111 }
5112
zend_jit_deactivate(void)5113 ZEND_EXT_API void zend_jit_deactivate(void)
5114 {
5115 if (zend_jit_profile_counter && !CG(unclean_shutdown)) {
5116 zend_class_entry *ce;
5117
5118 zend_shared_alloc_lock();
5119 SHM_UNPROTECT();
5120 zend_jit_unprotect();
5121
5122 zend_jit_check_funcs(EG(function_table), 0);
5123 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(EG(class_table), ce) {
5124 if (ce->type == ZEND_INTERNAL_CLASS) {
5125 break;
5126 }
5127 zend_jit_check_funcs(&ce->function_table, 1);
5128 } ZEND_HASH_FOREACH_END();
5129
5130 zend_jit_protect();
5131 SHM_PROTECT();
5132 zend_shared_alloc_unlock();
5133 }
5134
5135 zend_jit_profile_counter = 0;
5136 }
5137
zend_jit_restart_preloaded_op_array(zend_op_array * op_array)5138 static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array)
5139 {
5140 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
5141
5142 if (!func_info) {
5143 return;
5144 }
5145
5146 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
5147 zend_jit_restart_hot_trace_counters(op_array);
5148 } else if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_COUNTERS) {
5149 zend_jit_restart_hot_counters(op_array);
5150 #if 0
5151 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ???
5152 } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) {
5153 zend_op *opline = op_array->opcodes;
5154 zend_jit_op_array_extension *jit_extension =
5155 (zend_jit_op_array_extension*)func_info;
5156
5157 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
5158 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
5159 opline++;
5160 }
5161 }
5162 if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) {
5163 opline->handler = (const void*)zend_jit_runtime_jit_handler;
5164 } else {
5165 opline->handler = (const void*)zend_jit_profile_jit_handler;
5166 }
5167 #endif
5168 }
5169 if (op_array->num_dynamic_func_defs) {
5170 for (uint32_t i = 0; i < op_array->num_dynamic_func_defs; i++) {
5171 zend_jit_restart_preloaded_op_array(op_array->dynamic_func_defs[i]);
5172 }
5173 }
5174 }
5175
zend_jit_restart_preloaded_script(zend_persistent_script * script)5176 static void zend_jit_restart_preloaded_script(zend_persistent_script *script)
5177 {
5178 zend_class_entry *ce;
5179 zend_op_array *op_array;
5180
5181 zend_jit_restart_preloaded_op_array(&script->script.main_op_array);
5182
5183 ZEND_HASH_MAP_FOREACH_PTR(&script->script.function_table, op_array) {
5184 zend_jit_restart_preloaded_op_array(op_array);
5185 } ZEND_HASH_FOREACH_END();
5186
5187 ZEND_HASH_MAP_FOREACH_PTR(&script->script.class_table, ce) {
5188 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
5189 if (op_array->type == ZEND_USER_FUNCTION) {
5190 zend_jit_restart_preloaded_op_array(op_array);
5191 }
5192 } ZEND_HASH_FOREACH_END();
5193 } ZEND_HASH_FOREACH_END();
5194 }
5195
zend_jit_restart(void)5196 ZEND_EXT_API void zend_jit_restart(void)
5197 {
5198 if (dasm_buf) {
5199 zend_jit_unprotect();
5200
5201 #if ZEND_JIT_TARGET_ARM64
5202 memset(dasm_labels_veneers, 0, sizeof(void*) * ZEND_MM_ALIGNED_SIZE_EX(zend_lb_MAX, DASM_ALIGNMENT));
5203 #endif
5204
5205 /* restore JIT buffer pos */
5206 dasm_ptr[0] = dasm_ptr[1];
5207
5208 zend_jit_trace_restart();
5209
5210 if (ZCSG(preload_script)) {
5211 zend_jit_restart_preloaded_script(ZCSG(preload_script));
5212 if (ZCSG(saved_scripts)) {
5213 zend_persistent_script **p = ZCSG(saved_scripts);
5214
5215 while (*p) {
5216 zend_jit_restart_preloaded_script(*p);
5217 p++;
5218 }
5219 }
5220 }
5221
5222 zend_jit_protect();
5223
5224 #ifdef HAVE_DISASM
5225 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
5226 zend_jit_disasm_shutdown();
5227 zend_jit_disasm_init();
5228 }
5229 #endif
5230 }
5231 }
5232
5233 #endif /* HAVE_JIT */
5234
