1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 #include "main/php.h"
20 #include "main/SAPI.h"
21 #include "php_version.h"
22 #include <ZendAccelerator.h>
23 #include "zend_shared_alloc.h"
24 #include "Zend/zend_execute.h"
25 #include "Zend/zend_vm.h"
26 #include "Zend/zend_exceptions.h"
27 #include "Zend/zend_constants.h"
28 #include "Zend/zend_closures.h"
29 #include "Zend/zend_ini.h"
30 #include "Zend/zend_observer.h"
31 #include "zend_smart_str.h"
32 #include "jit/zend_jit.h"
33
34 #ifdef HAVE_JIT
35
36 #include "Optimizer/zend_func_info.h"
37 #include "Optimizer/zend_ssa.h"
38 #include "Optimizer/zend_inference.h"
39 #include "Optimizer/zend_call_graph.h"
40 #include "Optimizer/zend_dump.h"
41
42 #include "jit/zend_jit_x86.h"
43 #include "jit/zend_jit_internal.h"
44
45 #ifdef ZTS
46 int jit_globals_id;
47 #else
48 zend_jit_globals jit_globals;
49 #endif
50
51 //#define CONTEXT_THREADED_JIT
52 #define ZEND_JIT_USE_RC_INFERENCE
53
54 #ifdef ZEND_JIT_USE_RC_INFERENCE
55 # define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE
56 # define RC_MAY_BE_1(info) (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0)
57 # define RC_MAY_BE_N(info) (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0)
58 #else
59 # define ZEND_SSA_RC_INFERENCE_FLAG 0
60 # define RC_MAY_BE_1(info) 1
61 # define RC_MAY_BE_N(info) 1
62 #endif
63
64 #define JIT_PREFIX "JIT$"
65 #define JIT_STUB_PREFIX "JIT$$"
66 #define TRACE_PREFIX "TRACE-"
67
68 #define DASM_M_GROW(ctx, t, p, sz, need) \
69 do { \
70 size_t _sz = (sz), _need = (need); \
71 if (_sz < _need) { \
72 if (_sz < 16) _sz = 16; \
73 while (_sz < _need) _sz += _sz; \
74 (p) = (t *)erealloc((p), _sz); \
75 (sz) = _sz; \
76 } \
77 } while(0)
78
79 #define DASM_M_FREE(ctx, p, sz) efree(p)
80
81 #if ZEND_DEBUG
82 # define DASM_CHECKS 1
83 #endif
84
85 #include "dynasm/dasm_proto.h"
86
87 typedef struct _zend_jit_stub {
88 const char *name;
89 int (*stub)(dasm_State **Dst);
90 } zend_jit_stub;
91
92 #define JIT_STUB(name) \
93 {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub}
94
95 zend_ulong zend_jit_profile_counter = 0;
96 int zend_jit_profile_counter_rid = -1;
97
98 int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
99
100 const zend_op *zend_jit_halt_op = NULL;
101 static int zend_jit_vm_kind = 0;
102
103 static void *dasm_buf = NULL;
104 static void *dasm_end = NULL;
105 static void **dasm_ptr = NULL;
106
107 static size_t dasm_size = 0;
108
109 static zend_long jit_bisect_pos = 0;
110
111 static const void *zend_jit_runtime_jit_handler = NULL;
112 static const void *zend_jit_profile_jit_handler = NULL;
113 static const void *zend_jit_func_hot_counter_handler = NULL;
114 static const void *zend_jit_loop_hot_counter_handler = NULL;
115 static const void *zend_jit_func_trace_counter_handler = NULL;
116 static const void *zend_jit_ret_trace_counter_handler = NULL;
117 static const void *zend_jit_loop_trace_counter_handler = NULL;
118
119 static int ZEND_FASTCALL zend_runtime_jit(void);
120
121 static int zend_jit_trace_op_len(const zend_op *opline);
122 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
123 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags);
124 static const void *zend_jit_trace_get_exit_addr(uint32_t n);
125 static void zend_jit_trace_add_code(const void *start, uint32_t size);
126
dominates(const zend_basic_block * blocks,int a,int b)127 static zend_bool dominates(const zend_basic_block *blocks, int a, int b) {
128 while (blocks[b].level > blocks[a].level) {
129 b = blocks[b].idom;
130 }
131 return a == b;
132 }
133
zend_ssa_is_last_use(const zend_op_array * op_array,const zend_ssa * ssa,int var,int use)134 static zend_bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
135 {
136 int next_use;
137
138 if (ssa->vars[var].phi_use_chain) {
139 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
140 do {
141 if (!ssa->vars[phi->ssa_var].no_val) {
142 return 0;
143 }
144 phi = zend_ssa_next_use_phi(ssa, var, phi);
145 } while (phi);
146 }
147
148 if (ssa->cfg.blocks[ssa->cfg.map[use]].loop_header > 0
149 || (ssa->cfg.blocks[ssa->cfg.map[use]].flags & ZEND_BB_LOOP_HEADER)) {
150 int b = ssa->cfg.map[use];
151 int prev_use = ssa->vars[var].use_chain;
152
153 while (prev_use >= 0 && prev_use != use) {
154 if (b != ssa->cfg.map[prev_use]
155 && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use])
156 && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) {
157 return 0;
158 }
159 prev_use = zend_ssa_next_use(ssa->ops, var, prev_use);
160 }
161 }
162
163 next_use = zend_ssa_next_use(ssa->ops, var, use);
164 if (next_use < 0) {
165 return 1;
166 } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) {
167 return 1;
168 }
169 return 0;
170 }
171
zend_ival_is_last_use(const zend_lifetime_interval * ival,int use)172 static zend_bool zend_ival_is_last_use(const zend_lifetime_interval *ival, int use)
173 {
174 if (ival->flags & ZREG_LAST_USE) {
175 const zend_life_range *range = &ival->range;
176
177 while (range->next) {
178 range = range->next;
179 }
180 return range->end == use;
181 }
182 return 0;
183 }
184
zend_is_commutative(zend_uchar opcode)185 static zend_bool zend_is_commutative(zend_uchar opcode)
186 {
187 return
188 opcode == ZEND_ADD ||
189 opcode == ZEND_MUL ||
190 opcode == ZEND_BW_OR ||
191 opcode == ZEND_BW_AND ||
192 opcode == ZEND_BW_XOR;
193 }
194
zend_long_is_power_of_two(zend_long x)195 static zend_bool zend_long_is_power_of_two(zend_long x)
196 {
197 return (x > 0) && !(x & (x - 1));
198 }
199
zend_long_floor_log2(zend_long x)200 static zend_always_inline uint32_t zend_long_floor_log2(zend_long x)
201 {
202 ZEND_ASSERT(zend_long_is_power_of_two(x));
203 return zend_ulong_ntz(x);
204 }
205
206 #define OP_RANGE(ssa_op, opN) \
207 (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
208 ssa->var_info && \
209 (ssa_op)->opN##_use >= 0 && \
210 ssa->var_info[(ssa_op)->opN##_use].has_range) ? \
211 &ssa->var_info[(ssa_op)->opN##_use].range : NULL)
212
213 #define OP1_RANGE() OP_RANGE(ssa_op, op1)
214 #define OP2_RANGE() OP_RANGE(ssa_op, op2)
215 #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)
216
217 #include "dynasm/dasm_x86.h"
218 #include "jit/zend_jit_helpers.c"
219 #include "jit/zend_jit_disasm_x86.c"
220 #ifndef _WIN32
221 #include "jit/zend_jit_gdb.c"
222 #include "jit/zend_jit_perf_dump.c"
223 #endif
224 #ifdef HAVE_OPROFILE
225 # include "jit/zend_jit_oprofile.c"
226 #endif
227 #include "jit/zend_jit_vtune.c"
228
229 #include "jit/zend_jit_x86.c"
230
231 #if _WIN32
232 # include <Windows.h>
233 #else
234 # include <sys/mman.h>
235 # if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
236 # define MAP_ANONYMOUS MAP_ANON
237 # endif
238 #endif
239
zend_jit_status(zval * ret)240 ZEND_EXT_API void zend_jit_status(zval *ret)
241 {
242 zval stats;
243 array_init(&stats);
244 add_assoc_bool(&stats, "enabled", JIT_G(enabled));
245 add_assoc_bool(&stats, "on", JIT_G(on));
246 add_assoc_long(&stats, "kind", JIT_G(trigger));
247 add_assoc_long(&stats, "opt_level", JIT_G(opt_level));
248 add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags));
249 if (dasm_buf) {
250 add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
251 add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
252 } else {
253 add_assoc_long(&stats, "buffer_size", 0);
254 add_assoc_long(&stats, "buffer_free", 0);
255 }
256 add_assoc_zval(ret, "jit", &stats);
257 }
258
zend_jit_func_name(const zend_op_array * op_array)259 static zend_string *zend_jit_func_name(const zend_op_array *op_array)
260 {
261 smart_str buf = {0};
262
263 if (op_array->function_name) {
264 if (op_array->scope) {
265 smart_str_appends(&buf, JIT_PREFIX);
266 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
267 smart_str_appends(&buf, "::");
268 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
269 smart_str_0(&buf);
270 return buf.s;
271 } else {
272 smart_str_appends(&buf, JIT_PREFIX);
273 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
274 smart_str_0(&buf);
275 return buf.s;
276 }
277 } else if (op_array->filename) {
278 smart_str_appends(&buf, JIT_PREFIX);
279 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
280 smart_str_0(&buf);
281 return buf.s;
282 } else {
283 return NULL;
284 }
285 }
286
287 #if ZEND_DEBUG
handle_dasm_error(int ret)288 static void handle_dasm_error(int ret) {
289 switch (ret & 0xff000000u) {
290 case DASM_S_NOMEM:
291 fprintf(stderr, "DASM_S_NOMEM\n");
292 break;
293 case DASM_S_PHASE:
294 fprintf(stderr, "DASM_S_PHASE\n");
295 break;
296 case DASM_S_MATCH_SEC:
297 fprintf(stderr, "DASM_S_MATCH_SEC\n");
298 break;
299 case DASM_S_RANGE_I:
300 fprintf(stderr, "DASM_S_RANGE_I\n");
301 break;
302 case DASM_S_RANGE_SEC:
303 fprintf(stderr, "DASM_S_RANGE_SEC\n");
304 break;
305 case DASM_S_RANGE_LG:
306 fprintf(stderr, "DASM_S_RANGE_LG\n");
307 break;
308 case DASM_S_RANGE_PC:
309 fprintf(stderr, "DASM_S_RANGE_PC %d\n", ret & 0xffffffu);
310 break;
311 case DASM_S_RANGE_VREG:
312 fprintf(stderr, "DASM_S_RANGE_VREG\n");
313 break;
314 case DASM_S_UNDEF_L:
315 fprintf(stderr, "DASM_S_UNDEF_L\n");
316 break;
317 case DASM_S_UNDEF_PC:
318 fprintf(stderr, "DASM_S_UNDEF_PC\n");
319 break;
320 }
321 ZEND_UNREACHABLE();
322 }
323 #endif
324
dasm_link_and_encode(dasm_State ** dasm_state,const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline,zend_lifetime_interval ** ra,const char * name,uint32_t trace_num)325 static void *dasm_link_and_encode(dasm_State **dasm_state,
326 const zend_op_array *op_array,
327 zend_ssa *ssa,
328 const zend_op *rt_opline,
329 zend_lifetime_interval **ra,
330 const char *name,
331 uint32_t trace_num)
332 {
333 size_t size;
334 int ret;
335 void *entry;
336 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
337 zend_string *str = NULL;
338 #endif
339
340 if (rt_opline && ssa && ssa->cfg.map) {
341 /* Create additional entry point, to switch from interpreter to JIT-ed
342 * code at run-time.
343 */
344 int b = ssa->cfg.map[rt_opline - op_array->opcodes];
345
346 //#ifdef CONTEXT_THREADED_JIT
347 // if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY))) {
348 //#else
349 if (!(ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY))) {
350 //#endif
351 zend_jit_label(dasm_state, ssa->cfg.blocks_count + b);
352 zend_jit_prologue(dasm_state);
353 if (ra) {
354 int i;
355 zend_lifetime_interval *ival;
356 zend_life_range *range;
357 uint32_t pos = rt_opline - op_array->opcodes;
358
359 for (i = 0; i < ssa->vars_count; i++) {
360 ival = ra[i];
361
362 if (ival && ival->reg != ZREG_NONE) {
363 range = &ival->range;
364
365 if (pos >= range->start && pos <= range->end) {
366 if (!zend_jit_load_var(dasm_state, ssa->var_info[i].type, ssa->vars[i].var, ival->reg)) {
367 return NULL;
368 }
369 break;
370 }
371 range = range->next;
372 }
373 }
374 }
375 zend_jit_jmp(dasm_state, b);
376 }
377 }
378
379 ret = dasm_link(dasm_state, &size);
380 if (ret != DASM_S_OK) {
381 #if ZEND_DEBUG
382 handle_dasm_error(ret);
383 #endif
384 return NULL;
385 }
386
387 if ((void*)((char*)*dasm_ptr + size) > dasm_end) {
388 *dasm_ptr = dasm_end; //prevent further try
389 // TODO: jit_buffer_size overflow ???
390 return NULL;
391 }
392
393 ret = dasm_encode(dasm_state, *dasm_ptr);
394 if (ret != DASM_S_OK) {
395 #if ZEND_DEBUG
396 handle_dasm_error(ret);
397 #endif
398 return NULL;
399 }
400
401 entry = *dasm_ptr;
402 *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT));
403
404 if (trace_num) {
405 zend_jit_trace_add_code(entry, size);
406 }
407
408 if (op_array && ssa) {
409 int b;
410
411 for (b = 0; b < ssa->cfg.blocks_count; b++) {
412 //#ifdef CONTEXT_THREADED_JIT
413 // if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
414 //#else
415 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) {
416 //#endif
417 zend_op *opline = op_array->opcodes + ssa->cfg.blocks[b].start;
418 int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
419
420 if (offset >= 0) {
421 opline->handler = (void*)(((char*)entry) + offset);
422 }
423 }
424 }
425 if (rt_opline && ssa && ssa->cfg.map) {
426 int b = ssa->cfg.map[rt_opline - op_array->opcodes];
427 zend_op *opline = (zend_op*)rt_opline;
428 int offset = dasm_getpclabel(dasm_state, ssa->cfg.blocks_count + b);
429
430 if (offset >= 0) {
431 opline->handler = (void*)(((char*)entry) + offset);
432 }
433 }
434 }
435
436 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
437 if (!name) {
438 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_OPROFILE|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_VTUNE|ZEND_JIT_DEBUG_PERF_DUMP)) {
439 str = zend_jit_func_name(op_array);
440 if (str) {
441 name = ZSTR_VAL(str);
442 }
443 }
444 #ifdef HAVE_DISASM
445 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
446 zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
447 zend_jit_disasm(
448 name,
449 (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
450 op_array,
451 &ssa->cfg,
452 entry,
453 size);
454 }
455 } else {
456 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_ASM)) {
457 zend_jit_disasm_add_symbol(name, (uintptr_t)entry, size);
458 if ((JIT_G(debug) & (trace_num ? ZEND_JIT_DEBUG_ASM : ZEND_JIT_DEBUG_ASM_STUBS)) != 0) {
459 zend_jit_disasm(
460 name,
461 (op_array && op_array->filename) ? ZSTR_VAL(op_array->filename) : NULL,
462 op_array,
463 ssa ? &ssa->cfg : NULL,
464 entry,
465 size);
466 }
467 }
468 # endif
469 }
470 #endif
471
472 #ifdef HAVE_GDB
473 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
474 if (name) {
475 zend_jit_gdb_register(
476 name,
477 op_array,
478 entry,
479 size);
480 }
481 }
482 #endif
483
484 #ifdef HAVE_OPROFILE
485 if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
486 zend_jit_oprofile_register(
487 name,
488 entry,
489 size);
490 }
491 #endif
492
493 #ifdef HAVE_PERFTOOLS
494 if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
495 if (name) {
496 zend_jit_perf_map_register(
497 name,
498 entry,
499 size);
500 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
501 zend_jit_perf_jitdump_register(
502 name,
503 entry,
504 size);
505 }
506 }
507 }
508 #endif
509
510 #ifdef HAVE_VTUNE
511 if (JIT_G(debug) & ZEND_JIT_DEBUG_VTUNE) {
512 if (name) {
513 zend_jit_vtune_register(
514 name,
515 entry,
516 size);
517 }
518 }
519 #endif
520
521 #if defined(HAVE_DISASM) || defined(HAVE_GDB) || defined(HAVE_OPROFILE) || defined(HAVE_PERFTOOLS) || defined(HAVE_VTUNE)
522 if (str) {
523 zend_string_release(str);
524 }
525 #endif
526
527 return entry;
528 }
529
zend_may_overflow(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa)530 static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
531 {
532 int res;
533 zend_long op1_min, op1_max, op2_min, op2_max;
534
535 if (!ssa->ops || !ssa->var_info) {
536 return 1;
537 }
538 switch (opline->opcode) {
539 case ZEND_PRE_INC:
540 case ZEND_POST_INC:
541 res = ssa_op->op1_def;
542 if (res < 0
543 || !ssa->var_info[res].has_range
544 || ssa->var_info[res].range.overflow) {
545 if (!OP1_HAS_RANGE()) {
546 return 1;
547 }
548 op1_max = OP1_MAX_RANGE();
549 if (op1_max == ZEND_LONG_MAX) {
550 return 1;
551 }
552 }
553 return 0;
554 case ZEND_PRE_DEC:
555 case ZEND_POST_DEC:
556 res = ssa_op->op1_def;
557 if (res < 0
558 || !ssa->var_info[res].has_range
559 || ssa->var_info[res].range.underflow) {
560 if (!OP1_HAS_RANGE()) {
561 return 1;
562 }
563 op1_min = OP1_MIN_RANGE();
564 if (op1_min == ZEND_LONG_MIN) {
565 return 1;
566 }
567 }
568 return 0;
569 case ZEND_ADD:
570 res = ssa_op->result_def;
571 if (res < 0
572 || !ssa->var_info[res].has_range
573 || ssa->var_info[res].range.underflow) {
574 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
575 return 1;
576 }
577 op1_min = OP1_MIN_RANGE();
578 op2_min = OP2_MIN_RANGE();
579 if (zend_add_will_overflow(op1_min, op2_min)) {
580 return 1;
581 }
582 }
583 if (res < 0
584 || !ssa->var_info[res].has_range
585 || ssa->var_info[res].range.overflow) {
586 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
587 return 1;
588 }
589 op1_max = OP1_MAX_RANGE();
590 op2_max = OP2_MAX_RANGE();
591 if (zend_add_will_overflow(op1_max, op2_max)) {
592 return 1;
593 }
594 }
595 return 0;
596 case ZEND_SUB:
597 res = ssa_op->result_def;
598 if (res < 0
599 || !ssa->var_info[res].has_range
600 || ssa->var_info[res].range.underflow) {
601 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
602 return 1;
603 }
604 op1_min = OP1_MIN_RANGE();
605 op2_max = OP2_MAX_RANGE();
606 if (zend_sub_will_overflow(op1_min, op2_max)) {
607 return 1;
608 }
609 }
610 if (res < 0
611 || !ssa->var_info[res].has_range
612 || ssa->var_info[res].range.overflow) {
613 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
614 return 1;
615 }
616 op1_max = OP1_MAX_RANGE();
617 op2_min = OP2_MIN_RANGE();
618 if (zend_sub_will_overflow(op1_max, op2_min)) {
619 return 1;
620 }
621 }
622 return 0;
623 case ZEND_MUL:
624 res = ssa_op->result_def;
625 return (res < 0 ||
626 !ssa->var_info[res].has_range ||
627 ssa->var_info[res].range.underflow ||
628 ssa->var_info[res].range.overflow);
629 case ZEND_ASSIGN_OP:
630 if (opline->extended_value == ZEND_ADD) {
631 res = ssa_op->op1_def;
632 if (res < 0
633 || !ssa->var_info[res].has_range
634 || ssa->var_info[res].range.underflow) {
635 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
636 return 1;
637 }
638 op1_min = OP1_MIN_RANGE();
639 op2_min = OP2_MIN_RANGE();
640 if (zend_add_will_overflow(op1_min, op2_min)) {
641 return 1;
642 }
643 }
644 if (res < 0
645 || !ssa->var_info[res].has_range
646 || ssa->var_info[res].range.overflow) {
647 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
648 return 1;
649 }
650 op1_max = OP1_MAX_RANGE();
651 op2_max = OP2_MAX_RANGE();
652 if (zend_add_will_overflow(op1_max, op2_max)) {
653 return 1;
654 }
655 }
656 return 0;
657 } else if (opline->extended_value == ZEND_SUB) {
658 res = ssa_op->op1_def;
659 if (res < 0
660 || !ssa->var_info[res].has_range
661 || ssa->var_info[res].range.underflow) {
662 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
663 return 1;
664 }
665 op1_min = OP1_MIN_RANGE();
666 op2_max = OP2_MAX_RANGE();
667 if (zend_sub_will_overflow(op1_min, op2_max)) {
668 return 1;
669 }
670 }
671 if (res < 0
672 || !ssa->var_info[res].has_range
673 || ssa->var_info[res].range.overflow) {
674 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
675 return 1;
676 }
677 op1_max = OP1_MAX_RANGE();
678 op2_min = OP2_MIN_RANGE();
679 if (zend_sub_will_overflow(op1_max, op2_min)) {
680 return 1;
681 }
682 }
683 return 0;
684 } else if (opline->extended_value == ZEND_MUL) {
685 res = ssa_op->op1_def;
686 return (res < 0 ||
687 !ssa->var_info[res].has_range ||
688 ssa->var_info[res].range.underflow ||
689 ssa->var_info[res].range.overflow);
690 }
691 default:
692 return 1;
693 }
694 }
695
zend_jit_build_cfg(const zend_op_array * op_array,zend_cfg * cfg)696 static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg)
697 {
698 uint32_t flags;
699
700 flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY;
701
702 if (zend_build_cfg(&CG(arena), op_array, flags, cfg) != SUCCESS) {
703 return FAILURE;
704 }
705
706 /* Don't JIT huge functions. Apart from likely being detrimental due to the amount of
707 * generated code, some of our analysis is recursive and will stack overflow with many
708 * blocks. */
709 if (cfg->blocks_count > 100000) {
710 return FAILURE;
711 }
712
713 if (zend_cfg_build_predecessors(&CG(arena), cfg) != SUCCESS) {
714 return FAILURE;
715 }
716
717 /* Compute Dominators Tree */
718 if (zend_cfg_compute_dominators_tree(op_array, cfg) != SUCCESS) {
719 return FAILURE;
720 }
721
722 /* Identify reducible and irreducible loops */
723 if (zend_cfg_identify_loops(op_array, cfg) != SUCCESS) {
724 return FAILURE;
725 }
726
727 return SUCCESS;
728 }
729
zend_jit_op_array_analyze1(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa)730 static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
731 {
732 if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) {
733 return FAILURE;
734 }
735
736 #if 0
737 /* TODO: debugger and profiler supports? */
738 if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) {
739 return FAILURE;
740 }
741 #endif
742
743 /* TODO: move this to zend_cfg.c ? */
744 if (!op_array->function_name) {
745 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
746 }
747
748 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
749 && ssa->cfg.blocks
750 && op_array->last_try_catch == 0
751 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
752 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
753 if (zend_build_ssa(&CG(arena), script, op_array, ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) {
754 return FAILURE;
755 }
756
757 if (zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa) != SUCCESS) {
758 return FAILURE;
759 }
760
761 if (zend_ssa_find_false_dependencies(op_array, ssa) != SUCCESS) {
762 return FAILURE;
763 }
764
765 if (zend_ssa_find_sccs(op_array, ssa) != SUCCESS){
766 return FAILURE;
767 }
768 }
769
770 return SUCCESS;
771 }
772
zend_jit_op_array_analyze2(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa,uint32_t optimization_level)773 static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t optimization_level)
774 {
775 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
776 && ssa->cfg.blocks
777 && op_array->last_try_catch == 0
778 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
779 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
780 if (zend_ssa_inference(&CG(arena), op_array, script, ssa,
781 optimization_level & ~ZEND_OPTIMIZER_NARROW_TO_DOUBLE) != SUCCESS) {
782 return FAILURE;
783 }
784 }
785
786 return SUCCESS;
787 }
788
zend_jit_add_range(zend_lifetime_interval ** intervals,int var,uint32_t from,uint32_t to)789 static int zend_jit_add_range(zend_lifetime_interval **intervals, int var, uint32_t from, uint32_t to)
790 {
791 zend_lifetime_interval *ival = intervals[var];
792
793 if (!ival) {
794 ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
795 if (!ival) {
796 return FAILURE;
797 }
798 ival->ssa_var = var;
799 ival->reg = ZREG_NONE;
800 ival->flags = 0;
801 ival->range.start = from;
802 ival->range.end = to;
803 ival->range.next = NULL;
804 ival->hint = NULL;
805 ival->used_as_hint = NULL;
806 intervals[var] = ival;
807 } else if (ival->range.start > to + 1) {
808 zend_life_range *range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
809
810 if (!range) {
811 return FAILURE;
812 }
813 range->start = ival->range.start;
814 range->end = ival->range.end;
815 range->next = ival->range.next;
816 ival->range.start = from;
817 ival->range.end = to;
818 ival->range.next = range;
819 } else if (ival->range.start == to + 1) {
820 ival->range.start = from;
821 } else {
822 zend_life_range *range = &ival->range;
823 zend_life_range *last = NULL;
824
825 do {
826 if (range->start > to + 1) {
827 break;
828 } else if (range->end + 1 >= from) {
829 if (range->start > from) {
830 range->start = from;
831 }
832 last = range;
833 range = range->next;
834 while (range) {
835 if (range->start > to + 1) {
836 break;
837 }
838 last->end = range->end;
839 range = range->next;
840 last->next = range;
841 }
842 if (to > last->end) {
843 last->end = to;
844 }
845 return SUCCESS;
846 }
847 last = range;
848 range = range->next;
849 } while (range);
850
851 range = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
852 if (!range) {
853 return FAILURE;
854 }
855 range->start = from;
856 range->end = to;
857 range->next = last->next;
858 last->next = range;
859 }
860
861 return SUCCESS;
862 }
863
zend_jit_begin_range(zend_lifetime_interval ** intervals,int var,uint32_t block_start,uint32_t from)864 static int zend_jit_begin_range(zend_lifetime_interval **intervals, int var, uint32_t block_start, uint32_t from)
865 {
866 if (block_start != from && intervals[var]) {
867 zend_life_range *range = &intervals[var]->range;
868
869 do {
870 if (from >= range->start && from <= range->end) {
871 if (range->start == block_start) {
872 range->start = from;
873 } else {
874 zend_life_range *r = zend_arena_alloc(&CG(arena), sizeof(zend_life_range));
875 if (!r) {
876 return FAILURE;
877 }
878 r->start = from;
879 r->end = range->end;
880 r->next = range->next;
881 range->end = block_start - 1;
882 range->next = r;
883 }
884 return SUCCESS;
885 }
886 range = range->next;
887 } while (range);
888 }
889
890 // dead store
891 return zend_jit_add_range(intervals, var, from, from);
892 }
893
zend_jit_insert_interval(zend_lifetime_interval ** list,zend_lifetime_interval * ival)894 static void zend_jit_insert_interval(zend_lifetime_interval **list, zend_lifetime_interval *ival)
895 {
896 while (1) {
897 if (*list == NULL) {
898 *list = ival;
899 ival->list_next = NULL;
900 return;
901 } else if (ival->range.start < (*list)->range.start) {
902 ival->list_next = *list;
903 *list = ival;
904 return;
905 }
906 list = &(*list)->list_next;
907 }
908 }
909
zend_jit_split_interval(zend_lifetime_interval * current,uint32_t pos,zend_lifetime_interval ** list,zend_lifetime_interval ** free)910 static int zend_jit_split_interval(zend_lifetime_interval *current, uint32_t pos, zend_lifetime_interval **list, zend_lifetime_interval **free)
911 {
912 zend_lifetime_interval *ival;
913 zend_life_range *range = ¤t->range;
914 zend_life_range *prev = NULL;
915
916 if (*free) {
917 ival = *free;
918 *free = ival->list_next;
919 } else {
920 ival = zend_arena_alloc(&CG(arena), sizeof(zend_lifetime_interval));
921
922 if (!ival) {
923 return FAILURE;
924 }
925 }
926
927 current->flags |= ZREG_STORE;
928
929 ival->ssa_var = current->ssa_var;
930 ival->reg = ZREG_NONE;
931 ival->flags |= ZREG_SPLIT | ZREG_LOAD;
932 ival->flags &= ~ZREG_STORE;
933 ival->hint = NULL;
934
935 do {
936 if (pos >= range->start && pos <= range->end) {
937 break;
938 }
939 prev = range;
940 range = range->next;
941 } while(range);
942
943 ZEND_ASSERT(range != NULL);
944
945 ival->range.start = pos;
946 ival->range.end = range->end;
947 ival->range.next = range->next;
948
949 if (pos == range->start) {
950 ZEND_ASSERT(prev != NULL);
951 prev->next = NULL;
952 } else {
953 range->end = pos - 1;
954 }
955
956 zend_jit_insert_interval(list, ival);
957
958 return SUCCESS;
959 }
960
zend_jit_sort_intervals(zend_lifetime_interval ** intervals,int count)961 static zend_lifetime_interval *zend_jit_sort_intervals(zend_lifetime_interval **intervals, int count)
962 {
963 zend_lifetime_interval *list, *last;
964 int i;
965
966 list = NULL;
967 i = 0;
968 while (i < count) {
969 list = intervals[i];
970 i++;
971 if (list) {
972 last = list;
973 last->list_next = NULL;
974 break;
975 }
976 }
977
978 while (i < count) {
979 zend_lifetime_interval *ival = intervals[i];
980
981 i++;
982 if (ival) {
983 if ((ival->range.start > last->range.start) ||
984 (ival->range.start == last->range.start &&
985 ((!ival->hint && last->hint && last->hint != ival) ||
986 ival->range.end > last->range.end))) {
987 last->list_next = ival;
988 last = ival;
989 ival->list_next = NULL;
990 } else {
991 zend_lifetime_interval **p = &list;
992
993 while (1) {
994 if (*p == NULL) {
995 *p = last = ival;
996 ival->list_next = NULL;
997 break;
998 } else if ((ival->range.start < (*p)->range.start) ||
999 (ival->range.start == (*p)->range.start &&
1000 ((ival->hint && !(*p)->hint && ival->hint != *p) ||
1001 ival->range.end < (*p)->range.end))) {
1002 ival->list_next = *p;
1003 *p = ival;
1004 break;
1005 }
1006 p = &(*p)->list_next;
1007 }
1008 }
1009 }
1010 }
1011
1012 return list;
1013 }
1014
zend_jit_print_regset(zend_regset regset)1015 static ZEND_ATTRIBUTE_UNUSED void zend_jit_print_regset(zend_regset regset)
1016 {
1017 zend_reg reg;
1018 int first = 1;
1019
1020 ZEND_REGSET_FOREACH(regset, reg) {
1021 if (first) {
1022 first = 0;
1023 fprintf(stderr, "%s", zend_reg_name[reg]);
1024 } else {
1025 fprintf(stderr, ", %s", zend_reg_name[reg]);
1026 }
1027 } ZEND_REGSET_FOREACH_END();
1028 }
1029
zend_jit_compute_block_order_int(zend_ssa * ssa,int n,int * block_order)1030 static int *zend_jit_compute_block_order_int(zend_ssa *ssa, int n, int *block_order)
1031 {
1032 zend_basic_block *b = ssa->cfg.blocks + n;
1033
1034 tail_call:
1035 *block_order = n;
1036 block_order++;
1037
1038 n = b->children;
1039 while (n >= 0) {
1040 b = ssa->cfg.blocks + n;
1041 if (b->next_child < 0) {
1042 goto tail_call;
1043 }
1044 block_order = zend_jit_compute_block_order_int(ssa, n, block_order);
1045 n = b->next_child;
1046 }
1047
1048 return block_order;
1049 }
1050
zend_jit_compute_block_order(zend_ssa * ssa,int * block_order)1051 static int zend_jit_compute_block_order(zend_ssa *ssa, int *block_order)
1052 {
1053 int *end = zend_jit_compute_block_order_int(ssa, 0, block_order);
1054
1055 return end - block_order;
1056 }
1057
zend_jit_in_loop(zend_ssa * ssa,int header,zend_basic_block * b)1058 static zend_bool zend_jit_in_loop(zend_ssa *ssa, int header, zend_basic_block *b)
1059 {
1060 while (b->loop_header >= 0) {
1061 if (b->loop_header == header) {
1062 return 1;
1063 }
1064 b = ssa->cfg.blocks + b->loop_header;
1065 }
1066 return 0;
1067 }
1068
zend_jit_compute_loop_body(zend_ssa * ssa,int header,int n,zend_bitset loop_body)1069 static void zend_jit_compute_loop_body(zend_ssa *ssa, int header, int n, zend_bitset loop_body)
1070 {
1071 zend_basic_block *b = ssa->cfg.blocks + n;
1072 uint32_t i;
1073
1074 tail_call:
1075 if (b->len) {
1076 for (i = b->start; i < b->start + b->len; i++) {
1077 zend_bitset_incl(loop_body, i);
1078 }
1079 }
1080
1081 n = b->children;
1082 while (n >= 0) {
1083 b = ssa->cfg.blocks + n;
1084 if (zend_jit_in_loop(ssa, header, b)) {
1085 if (b->next_child < 0) {
1086 goto tail_call;
1087 }
1088 zend_jit_compute_loop_body(ssa, header, n, loop_body);
1089 }
1090 n = b->next_child;
1091 }
1092 }
1093
zend_jit_add_hint(zend_lifetime_interval ** intervals,int dst,int src)1094 static void zend_jit_add_hint(zend_lifetime_interval **intervals, int dst, int src)
1095 {
1096 if (intervals[dst]->range.start < intervals[src]->range.start) {
1097 int tmp = src;
1098 src = dst;
1099 dst = tmp;
1100 }
1101 while (dst != src && intervals[dst]->hint) {
1102 if (intervals[dst]->hint->range.start < intervals[src]->range.start) {
1103 int tmp = src;
1104 src = intervals[dst]->hint->ssa_var;
1105 dst = tmp;
1106 } else {
1107 dst = intervals[dst]->hint->ssa_var;
1108 }
1109 }
1110 if (dst != src) {
1111 intervals[dst]->hint = intervals[src];
1112 }
1113 }
1114
1115 /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
1116 Michael Franz, CGO'10 (2010), Figure 4. */
zend_jit_compute_liveness(const zend_op_array * op_array,zend_ssa * ssa,zend_bitset candidates,zend_lifetime_interval ** list)1117 static int zend_jit_compute_liveness(const zend_op_array *op_array, zend_ssa *ssa, zend_bitset candidates, zend_lifetime_interval **list)
1118 {
1119 int set_size, i, j, k, l;
1120 uint32_t n;
1121 zend_bitset live, live_in, pi_vars, loop_body;
1122 int *block_order;
1123 zend_ssa_phi *phi;
1124 zend_lifetime_interval **intervals;
1125 size_t mem_size;
1126 ALLOCA_FLAG(use_heap);
1127
1128 set_size = zend_bitset_len(ssa->vars_count);
1129 mem_size =
1130 ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)) +
1131 ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE) +
1132 ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
1133 ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE) +
1134 ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE) +
1135 ZEND_MM_ALIGNED_SIZE(ssa->cfg.blocks_count * sizeof(int));
1136 intervals = do_alloca(mem_size, use_heap);
1137 if (!intervals) {
1138 *list = NULL;
1139 return FAILURE;
1140 }
1141
1142 live_in = (zend_bitset)((char*)intervals + ZEND_MM_ALIGNED_SIZE(ssa->vars_count * sizeof(zend_lifetime_interval*)));
1143 live = (zend_bitset)((char*)live_in + ZEND_MM_ALIGNED_SIZE((set_size * ssa->cfg.blocks_count) * ZEND_BITSET_ELM_SIZE));
1144 pi_vars = (zend_bitset)((char*)live + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
1145 loop_body = (zend_bitset)((char*)pi_vars + ZEND_MM_ALIGNED_SIZE(set_size * ZEND_BITSET_ELM_SIZE));
1146 block_order = (int*)((char*)loop_body + ZEND_MM_ALIGNED_SIZE(zend_bitset_len(op_array->last) * ZEND_BITSET_ELM_SIZE));
1147
1148 memset(intervals, 0, ssa->vars_count * sizeof(zend_lifetime_interval*));
1149 zend_bitset_clear(live_in, set_size * ssa->cfg.blocks_count);
1150
1151 /* TODO: Provide a linear block order where all dominators of a block
1152 * are before this block, and where all blocks belonging to the same loop
1153 * are contiguous ???
1154 */
1155 for (l = zend_jit_compute_block_order(ssa, block_order) - 1; l >= 0; l--) {
1156 zend_basic_block *b;
1157
1158 i = block_order[l];
1159 b = ssa->cfg.blocks + i;
1160
1161 /* live = UNION of successor.liveIn for each successor of b */
1162 /* live.add(phi.inputOf(b)) for each phi of successors of b */
1163 zend_bitset_clear(live, set_size);
1164 for (j = 0; j < b->successors_count; j++) {
1165 int succ = b->successors[j];
1166
1167 zend_bitset_union(live, live_in + set_size * succ, set_size);
1168 zend_bitset_clear(pi_vars, set_size);
1169 for (phi = ssa->blocks[succ].phis; phi; phi = phi->next) {
1170 if (ssa->vars[phi->ssa_var].no_val) {
1171 /* skip */
1172 } else if (phi->pi >= 0) {
1173 if (phi->pi == i && phi->sources[0] >= 0) {
1174 if (zend_bitset_in(candidates, phi->sources[0])) {
1175 zend_bitset_incl(live, phi->sources[0]);
1176 }
1177 zend_bitset_incl(pi_vars, phi->var);
1178 }
1179 } else if (!zend_bitset_in(pi_vars, phi->var)) {
1180 for (k = 0; k < ssa->cfg.blocks[succ].predecessors_count; k++) {
1181 if (ssa->cfg.predecessors[ssa->cfg.blocks[succ].predecessor_offset + k] == i) {
1182 if (phi->sources[k] >= 0 && zend_bitset_in(candidates, phi->sources[k])) {
1183 zend_bitset_incl(live, phi->sources[k]);
1184 }
1185 break;
1186 }
1187 }
1188 }
1189 }
1190 }
1191
1192 /* addRange(var, b.from, b.to) for each var in live */
1193 ZEND_BITSET_FOREACH(live, set_size, j) {
1194 if (zend_bitset_in(candidates, j)) {
1195 if (zend_jit_add_range(intervals, j, b->start, b->start + b->len - 1) != SUCCESS) {
1196 goto failure;
1197 }
1198 }
1199 } ZEND_BITSET_FOREACH_END();
1200
1201 /* for each operation op of b in reverse order */
1202 for (n = b->start + b->len; n > b->start;) {
1203 zend_ssa_op *op;
1204 const zend_op *opline;
1205 uint32_t num;
1206
1207 n--;
1208 op = ssa->ops + n;
1209 opline = op_array->opcodes + n;
1210
1211 if (UNEXPECTED(opline->opcode == ZEND_OP_DATA)) {
1212 num = n - 1;
1213 } else {
1214 num = n;
1215 }
1216
1217 /* for each output operand opd of op do */
1218 /* setFrom(opd, op) */
1219 /* live.remove(opd) */
1220 if (op->op1_def >= 0 && zend_bitset_in(candidates, op->op1_def)) {
1221 if (zend_jit_begin_range(intervals, op->op1_def, b->start, num) != SUCCESS) {
1222 goto failure;
1223 }
1224 zend_bitset_excl(live, op->op1_def);
1225 }
1226 if (op->op2_def >= 0 && zend_bitset_in(candidates, op->op2_def)) {
1227 if (zend_jit_begin_range(intervals, op->op2_def, b->start, num) != SUCCESS) {
1228 goto failure;
1229 }
1230 zend_bitset_excl(live, op->op2_def);
1231 }
1232 if (op->result_def >= 0 && zend_bitset_in(candidates, op->result_def)) {
1233 if (zend_jit_begin_range(intervals, op->result_def, b->start, num) != SUCCESS) {
1234 goto failure;
1235 }
1236 zend_bitset_excl(live, op->result_def);
1237 }
1238
1239 /* for each input operand opd of op do */
1240 /* live.add(opd) */
1241 /* addRange(opd, b.from, op) */
1242 if (op->op1_use >= 0
1243 && zend_bitset_in(candidates, op->op1_use)
1244 && !zend_ssa_is_no_val_use(opline, op, op->op1_use)) {
1245 zend_bitset_incl(live, op->op1_use);
1246 if (zend_jit_add_range(intervals, op->op1_use, b->start, num) != SUCCESS) {
1247 goto failure;
1248 }
1249 }
1250 if (op->op2_use >= 0
1251 && zend_bitset_in(candidates, op->op2_use)
1252 && !zend_ssa_is_no_val_use(opline, op, op->op2_use)) {
1253 zend_bitset_incl(live, op->op2_use);
1254 if (zend_jit_add_range(intervals, op->op2_use, b->start, num) != SUCCESS) {
1255 goto failure;
1256 }
1257 }
1258 if (op->result_use >= 0
1259 && zend_bitset_in(candidates, op->result_use)
1260 && !zend_ssa_is_no_val_use(opline, op, op->result_use)) {
1261 zend_bitset_incl(live, op->result_use);
1262 if (zend_jit_add_range(intervals, op->result_use, b->start, num) != SUCCESS) {
1263 goto failure;
1264 }
1265 }
1266 }
1267
1268 /* live.remove(phi.output) for each phi of b */
1269 for (phi = ssa->blocks[i].phis; phi; phi = phi->next) {
1270 zend_bitset_excl(live, phi->ssa_var);
1271 }
1272
1273 /* b.liveIn = live */
1274 zend_bitset_copy(live_in + set_size * i, live, set_size);
1275 }
1276
1277 for (i = ssa->cfg.blocks_count - 1; i >= 0; i--) {
1278 zend_basic_block *b = ssa->cfg.blocks + i;
1279
1280 /* if b is loop header */
1281 if ((b->flags & ZEND_BB_LOOP_HEADER)) {
1282 live = live_in + set_size * i;
1283
1284 if (!zend_bitset_empty(live, set_size)) {
1285 uint32_t set_size2 = zend_bitset_len(op_array->last);
1286
1287 zend_bitset_clear(loop_body, set_size2);
1288 zend_jit_compute_loop_body(ssa, i, i, loop_body);
1289 while (!zend_bitset_empty(loop_body, set_size2)) {
1290 uint32_t from = zend_bitset_first(loop_body, set_size2);
1291 uint32_t to = from;
1292
1293 do {
1294 zend_bitset_excl(loop_body, to);
1295 to++;
1296 } while (zend_bitset_in(loop_body, to));
1297 to--;
1298
1299 ZEND_BITSET_FOREACH(live, set_size, j) {
1300 if (zend_jit_add_range(intervals, j, from, to) != SUCCESS) {
1301 goto failure;
1302 }
1303 } ZEND_BITSET_FOREACH_END();
1304 }
1305 }
1306 }
1307
1308 }
1309
1310 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
1311 /* Register hinting (a cheap way for register coalescing) */
1312 for (i = 0; i < ssa->vars_count; i++) {
1313 if (intervals[i]) {
1314 int src;
1315
1316 if (ssa->vars[i].definition_phi) {
1317 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
1318
1319 if (phi->pi >= 0) {
1320 src = phi->sources[0];
1321 if (intervals[src]) {
1322 zend_jit_add_hint(intervals, i, src);
1323 }
1324 } else {
1325 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1326 src = phi->sources[k];
1327 if (src >= 0) {
1328 if (ssa->vars[src].definition_phi
1329 && ssa->vars[src].definition_phi->pi >= 0
1330 && phi->block == ssa->vars[src].definition_phi->block) {
1331 /* Skip zero-length interval for Pi variable */
1332 src = ssa->vars[src].definition_phi->sources[0];
1333 }
1334 if (intervals[src]) {
1335 zend_jit_add_hint(intervals, i, src);
1336 }
1337 }
1338 }
1339 }
1340 }
1341 }
1342 }
1343 for (i = 0; i < ssa->vars_count; i++) {
1344 if (intervals[i] && !intervals[i]->hint) {
1345
1346 if (ssa->vars[i].definition >= 0) {
1347 uint32_t line = ssa->vars[i].definition;
1348 const zend_op *opline = op_array->opcodes + line;
1349
1350 switch (opline->opcode) {
1351 case ZEND_QM_ASSIGN:
1352 case ZEND_POST_INC:
1353 case ZEND_POST_DEC:
1354 if (ssa->ops[line].op1_use >= 0 &&
1355 intervals[ssa->ops[line].op1_use] &&
1356 (i == ssa->ops[line].op1_def ||
1357 (i == ssa->ops[line].result_def &&
1358 (ssa->ops[line].op1_def < 0 ||
1359 !intervals[ssa->ops[line].op1_def])))) {
1360 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1361 }
1362 break;
1363 case ZEND_SEND_VAR:
1364 case ZEND_PRE_INC:
1365 case ZEND_PRE_DEC:
1366 if (i == ssa->ops[line].op1_def &&
1367 ssa->ops[line].op1_use >= 0 &&
1368 intervals[ssa->ops[line].op1_use]) {
1369 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1370 }
1371 break;
1372 case ZEND_ASSIGN:
1373 if (ssa->ops[line].op2_use >= 0 &&
1374 intervals[ssa->ops[line].op2_use] &&
1375 (i == ssa->ops[line].op2_def ||
1376 (i == ssa->ops[line].op1_def &&
1377 (ssa->ops[line].op2_def < 0 ||
1378 !intervals[ssa->ops[line].op2_def])) ||
1379 (i == ssa->ops[line].result_def &&
1380 (ssa->ops[line].op2_def < 0 ||
1381 !intervals[ssa->ops[line].op2_def]) &&
1382 (ssa->ops[line].op1_def < 0 ||
1383 !intervals[ssa->ops[line].op1_def])))) {
1384 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
1385 }
1386 break;
1387 case ZEND_SUB:
1388 case ZEND_ADD:
1389 case ZEND_MUL:
1390 case ZEND_BW_OR:
1391 case ZEND_BW_AND:
1392 case ZEND_BW_XOR:
1393 if (i == ssa->ops[line].result_def) {
1394 if (ssa->ops[line].op1_use >= 0 &&
1395 intervals[ssa->ops[line].op1_use] &&
1396 ssa->ops[line].op1_use_chain < 0 &&
1397 !ssa->vars[ssa->ops[line].op1_use].phi_use_chain &&
1398 (ssa->var_info[i].type & MAY_BE_ANY) ==
1399 (ssa->var_info[ssa->ops[line].op1_use].type & MAY_BE_ANY)) {
1400 zend_jit_add_hint(intervals, i, ssa->ops[line].op1_use);
1401 } else if (opline->opcode != ZEND_SUB &&
1402 ssa->ops[line].op2_use >= 0 &&
1403 intervals[ssa->ops[line].op2_use] &&
1404 ssa->ops[line].op2_use_chain < 0 &&
1405 !ssa->vars[ssa->ops[line].op2_use].phi_use_chain &&
1406 (ssa->var_info[i].type & MAY_BE_ANY) ==
1407 (ssa->var_info[ssa->ops[line].op2_use].type & MAY_BE_ANY)) {
1408 zend_jit_add_hint(intervals, i, ssa->ops[line].op2_use);
1409 }
1410 }
1411 break;
1412 }
1413 }
1414 }
1415 }
1416 }
1417
1418 *list = zend_jit_sort_intervals(intervals, ssa->vars_count);
1419
1420 if (*list) {
1421 zend_lifetime_interval *ival = *list;
1422 while (ival) {
1423 if (ival->hint) {
1424 ival->hint->used_as_hint = ival;
1425 }
1426 ival = ival->list_next;
1427 }
1428 }
1429
1430 free_alloca(intervals, use_heap);
1431 return SUCCESS;
1432
1433 failure:
1434 *list = NULL;
1435 free_alloca(intervals, use_heap);
1436 return FAILURE;
1437 }
1438
zend_interval_end(zend_lifetime_interval * ival)1439 static uint32_t zend_interval_end(zend_lifetime_interval *ival)
1440 {
1441 zend_life_range *range = &ival->range;
1442
1443 while (range->next) {
1444 range = range->next;
1445 }
1446 return range->end;
1447 }
1448
zend_interval_covers(zend_lifetime_interval * ival,uint32_t position)1449 static zend_bool zend_interval_covers(zend_lifetime_interval *ival, uint32_t position)
1450 {
1451 zend_life_range *range = &ival->range;
1452
1453 do {
1454 if (position >= range->start && position <= range->end) {
1455 return 1;
1456 }
1457 range = range->next;
1458 } while (range);
1459
1460 return 0;
1461 }
1462
zend_interval_intersection(zend_lifetime_interval * ival1,zend_lifetime_interval * ival2)1463 static uint32_t zend_interval_intersection(zend_lifetime_interval *ival1, zend_lifetime_interval *ival2)
1464 {
1465 zend_life_range *r1 = &ival1->range;
1466 zend_life_range *r2 = &ival2->range;
1467
1468 do {
1469 if (r1->start <= r2->end) {
1470 if (r2->start <= r1->end) {
1471 return MAX(r1->start, r2->start);
1472 } else {
1473 r2 = r2->next;
1474 }
1475 } else {
1476 r1 = r1->next;
1477 }
1478 } while (r1 && r2);
1479
1480 return 0xffffffff;
1481 }
1482
1483 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
1484 Christian Wimmer VEE'05 (2005), Figure 4. Allocation without spilling */
zend_jit_try_allocate_free_reg(const zend_op_array * op_array,const zend_op ** ssa_opcodes,zend_ssa * ssa,zend_lifetime_interval * current,zend_regset available,zend_regset * hints,zend_lifetime_interval * active,zend_lifetime_interval * inactive,zend_lifetime_interval ** list,zend_lifetime_interval ** free)1485 static int zend_jit_try_allocate_free_reg(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *current, zend_regset available, zend_regset *hints, zend_lifetime_interval *active, zend_lifetime_interval *inactive, zend_lifetime_interval **list, zend_lifetime_interval **free)
1486 {
1487 zend_lifetime_interval *it;
1488 uint32_t freeUntilPos[ZREG_NUM];
1489 uint32_t pos, pos2;
1490 zend_reg i, reg, reg2;
1491 zend_reg hint = ZREG_NONE;
1492 zend_regset low_priority_regs;
1493 zend_life_range *range;
1494
1495 if ((ssa->var_info[current->ssa_var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) {
1496 available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_FP);
1497 } else {
1498 available = ZEND_REGSET_INTERSECTION(available, ZEND_REGSET_GP);
1499 }
1500
1501 /* TODO: Allow usage of preserved registers ???
1502 * Their values have to be stored in prologue and restored in epilogue
1503 */
1504 available = ZEND_REGSET_DIFFERENCE(available, ZEND_REGSET_PRESERVED);
1505
1506 /* Set freeUntilPos of all physical registers to maxInt */
1507 for (i = 0; i < ZREG_NUM; i++) {
1508 freeUntilPos[i] = 0xffffffff;
1509 }
1510
1511 /* for each interval it in active do */
1512 /* freeUntilPos[it.reg] = 0 */
1513 it = active;
1514 if (ssa->vars[current->ssa_var].definition == current->range.start) {
1515 while (it) {
1516 if (current->range.start != zend_interval_end(it)) {
1517 freeUntilPos[it->reg] = 0;
1518 } else if (zend_jit_may_reuse_reg(
1519 ssa_opcodes ? ssa_opcodes[current->range.start] : op_array->opcodes + current->range.start,
1520 ssa->ops + current->range.start, ssa, current->ssa_var, it->ssa_var)) {
1521 if (!ZEND_REGSET_IN(*hints, it->reg) &&
1522 /* TODO: Avoid most often scratch registers. Find a better way ??? */
1523 (!current->used_as_hint ||
1524 (it->reg != ZREG_R0 && it->reg != ZREG_R1 && it->reg != ZREG_XMM0 && it->reg != ZREG_XMM1))) {
1525 hint = it->reg;
1526 }
1527 } else {
1528 freeUntilPos[it->reg] = 0;
1529 }
1530 it = it->list_next;
1531 }
1532 } else {
1533 while (it) {
1534 freeUntilPos[it->reg] = 0;
1535 it = it->list_next;
1536 }
1537 }
1538 if (current->hint) {
1539 hint = current->hint->reg;
1540 if (hint != ZREG_NONE && current->hint->used_as_hint == current) {
1541 ZEND_REGSET_EXCL(*hints, hint);
1542 }
1543 }
1544
1545 if (hint == ZREG_NONE && ZEND_REGSET_IS_EMPTY(available)) {
1546 return 0;
1547 }
1548
1549 /* See "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
1550 Michael Franz, CGO'10 (2010), Figure 6. */
1551 if (current->flags & ZREG_SPLIT) {
1552 /* for each interval it in inactive intersecting with current do */
1553 /* freeUntilPos[it.reg] = next intersection of it with current */
1554 it = inactive;
1555 while (it) {
1556 uint32_t next = zend_interval_intersection(current, it);
1557
1558 //ZEND_ASSERT(next != 0xffffffff && !current->split);
1559 if (next < freeUntilPos[it->reg]) {
1560 freeUntilPos[it->reg] = next;
1561 }
1562 it = it->list_next;
1563 }
1564 }
1565
1566 /* Handle Scratch Registers */
1567 /* TODO: Optimize ??? */
1568 range = ¤t->range;
1569 do {
1570 uint32_t line = range->start;
1571 uint32_t last_use_line = (uint32_t)-1;
1572 zend_regset regset;
1573 zend_reg reg;
1574
1575 if ((current->flags & ZREG_LAST_USE) && !range->next) {
1576 last_use_line = range->end;
1577 }
1578 if (ssa->ops[line].op1_def == current->ssa_var ||
1579 ssa->ops[line].op2_def == current->ssa_var ||
1580 ssa->ops[line].result_def == current->ssa_var) {
1581 regset = zend_jit_get_def_scratch_regset(
1582 ssa_opcodes ? ssa_opcodes[line] : op_array->opcodes + line,
1583 ssa->ops + line,
1584 op_array, ssa, current->ssa_var, line == last_use_line);
1585 ZEND_REGSET_FOREACH(regset, reg) {
1586 if (line < freeUntilPos[reg]) {
1587 freeUntilPos[reg] = line;
1588 }
1589 } ZEND_REGSET_FOREACH_END();
1590 line++;
1591 }
1592 while (line <= range->end) {
1593 regset = zend_jit_get_scratch_regset(
1594 ssa_opcodes ? ssa_opcodes[line] : op_array->opcodes + line,
1595 ssa->ops + line,
1596 op_array, ssa, current->ssa_var, line == last_use_line);
1597 ZEND_REGSET_FOREACH(regset, reg) {
1598 if (line < freeUntilPos[reg]) {
1599 freeUntilPos[reg] = line;
1600 }
1601 } ZEND_REGSET_FOREACH_END();
1602 line++;
1603 }
1604 range = range->next;
1605 } while (range);
1606
1607 #if 0
1608 /* Coalescing */
1609 if (ssa->vars[current->ssa_var].definition == current->start) {
1610 zend_op *opline = op_array->opcodes + current->start;
1611 int hint = -1;
1612
1613 switch (opline->opcode) {
1614 case ZEND_ASSIGN:
1615 hint = ssa->ops[current->start].op2_use;
1616 case ZEND_QM_ASSIGN:
1617 hint = ssa->ops[current->start].op1_use;
1618 break;
1619 case ZEND_ADD:
1620 case ZEND_SUB:
1621 case ZEND_MUL:
1622 hint = ssa->ops[current->start].op1_use;
1623 break;
1624 case ZEND_ASSIGN_OP:
1625 if (opline->extended_value == ZEND_ADD
1626 || opline->extended_value == ZEND_SUB
1627 || opline->extended_value == ZEND_MUL) {
1628 hint = ssa->ops[current->start].op1_use;
1629 }
1630 break;
1631 }
1632 if (hint >= 0) {
1633 }
1634 }
1635 #endif
1636
1637 if (hint != ZREG_NONE && freeUntilPos[hint] > zend_interval_end(current)) {
1638 current->reg = hint;
1639 if (current->used_as_hint) {
1640 ZEND_REGSET_INCL(*hints, hint);
1641 }
1642 return 1;
1643 }
1644
1645 if (ZEND_REGSET_IS_EMPTY(available)) {
1646 return 0;
1647 }
1648
1649 pos = 0; reg = ZREG_NONE;
1650 pos2 = 0; reg2 = ZREG_NONE;
1651 low_priority_regs = *hints;
1652 if (current->used_as_hint) {
1653 /* TODO: Avoid most often scratch registers. Find a better way ??? */
1654 ZEND_REGSET_INCL(low_priority_regs, ZREG_R0);
1655 ZEND_REGSET_INCL(low_priority_regs, ZREG_R1);
1656 ZEND_REGSET_INCL(low_priority_regs, ZREG_XMM0);
1657 ZEND_REGSET_INCL(low_priority_regs, ZREG_XMM1);
1658 }
1659
1660 ZEND_REGSET_FOREACH(available, i) {
1661 if (ZEND_REGSET_IN(low_priority_regs, i)) {
1662 if (freeUntilPos[i] > pos2) {
1663 reg2 = i;
1664 pos2 = freeUntilPos[i];
1665 }
1666 } else if (freeUntilPos[i] > pos) {
1667 reg = i;
1668 pos = freeUntilPos[i];
1669 }
1670 } ZEND_REGSET_FOREACH_END();
1671
1672 if (reg == ZREG_NONE) {
1673 if (reg2 != ZREG_NONE) {
1674 reg = reg2;
1675 pos = pos2;
1676 reg2 = ZREG_NONE;
1677 }
1678 }
1679
1680 if (reg == ZREG_NONE) {
1681 /* no register available without spilling */
1682 return 0;
1683 } else if (zend_interval_end(current) < pos) {
1684 /* register available for the whole interval */
1685 current->reg = reg;
1686 if (current->used_as_hint) {
1687 ZEND_REGSET_INCL(*hints, reg);
1688 }
1689 return 1;
1690 #if 0
1691 // TODO: allow low priority register usage
1692 } else if (reg2 != ZREG_NONE && zend_interval_end(current) < pos2) {
1693 /* register available for the whole interval */
1694 current->reg = reg2;
1695 if (current->used_as_hint) {
1696 ZEND_REGSET_INCL(*hints, reg2);
1697 }
1698 return 1;
1699 #endif
1700 } else {
1701 /* TODO: enable interval splitting ??? */
1702 /* register available for the first part of the interval */
1703 if (1 || zend_jit_split_interval(current, pos, list, free) != SUCCESS) {
1704 return 0;
1705 }
1706 current->reg = reg;
1707 if (current->used_as_hint) {
1708 ZEND_REGSET_INCL(*hints, reg);
1709 }
1710 return 1;
1711 }
1712 }
1713
1714 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
1715 Christian Wimmer VEE'05 (2005), Figure 5. Allocation with spilling.
1716 and "Linear Scan Register Allocation on SSA Form", Christian Wimmer and
1717 Michael Franz, CGO'10 (2010), Figure 6. */
zend_jit_allocate_blocked_reg(void)1718 static int zend_jit_allocate_blocked_reg(void)
1719 {
1720 /* TODO: ??? */
1721 return 0;
1722 }
1723
1724 /* See "Optimized Interval Splitting in a Linear Scan Register Allocator",
1725 Christian Wimmer VEE'10 (2005), Figure 2. */
zend_jit_linear_scan(const zend_op_array * op_array,const zend_op ** ssa_opcodes,zend_ssa * ssa,zend_lifetime_interval * list)1726 static zend_lifetime_interval* zend_jit_linear_scan(const zend_op_array *op_array, const zend_op **ssa_opcodes, zend_ssa *ssa, zend_lifetime_interval *list)
1727 {
1728 zend_lifetime_interval *unhandled, *active, *inactive, *handled, *free;
1729 zend_lifetime_interval *current, **p, *q;
1730 uint32_t position;
1731 zend_regset available = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP);
1732 zend_regset hints = ZEND_REGSET_EMPTY;
1733
1734 unhandled = list;
1735 /* active = inactive = handled = free = {} */
1736 active = inactive = handled = free = NULL;
1737 while (unhandled != NULL) {
1738 current = unhandled;
1739 unhandled = unhandled->list_next;
1740 position = current->range.start;
1741
1742 p = &active;
1743 while (*p) {
1744 uint32_t end = zend_interval_end(*p);
1745
1746 q = *p;
1747 if (end < position) {
1748 /* move ival from active to handled */
1749 ZEND_REGSET_INCL(available, q->reg);
1750 *p = q->list_next;
1751 q->list_next = handled;
1752 handled = q;
1753 } else if (!zend_interval_covers(q, position)) {
1754 /* move ival from active to inactive */
1755 ZEND_REGSET_INCL(available, q->reg);
1756 *p = q->list_next;
1757 q->list_next = inactive;
1758 inactive = q;
1759 } else {
1760 p = &q->list_next;
1761 }
1762 }
1763
1764 p = &inactive;
1765 while (*p) {
1766 uint32_t end = zend_interval_end(*p);
1767
1768 q = *p;
1769 if (end < position) {
1770 /* move ival from inactive to handled */
1771 *p = q->list_next;
1772 q->list_next = handled;
1773 handled = q;
1774 } else if (zend_interval_covers(q, position)) {
1775 /* move ival from inactive to active */
1776 ZEND_REGSET_EXCL(available, q->reg);
1777 *p = q->list_next;
1778 q->list_next = active;
1779 active = q;
1780 } else {
1781 p = &q->list_next;
1782 }
1783 }
1784
1785 if (zend_jit_try_allocate_free_reg(op_array, ssa_opcodes, ssa, current, available, &hints, active, inactive, &unhandled, &free) ||
1786 zend_jit_allocate_blocked_reg()) {
1787 ZEND_REGSET_EXCL(available, current->reg);
1788 current->list_next = active;
1789 active = current;
1790 } else {
1791 current->list_next = free;
1792 free = current;
1793 }
1794 }
1795
1796 /* move active to handled */
1797 while (active) {
1798 current = active;
1799 active = active->list_next;
1800 current->list_next = handled;
1801 handled = current;
1802 }
1803
1804 /* move inactive to handled */
1805 while (inactive) {
1806 current = inactive;
1807 inactive = inactive->list_next;
1808 current->list_next = handled;
1809 handled = current;
1810 }
1811
1812 return handled;
1813 }
1814
zend_jit_dump_lifetime_interval(const zend_op_array * op_array,const zend_ssa * ssa,const zend_lifetime_interval * ival)1815 static void zend_jit_dump_lifetime_interval(const zend_op_array *op_array, const zend_ssa *ssa, const zend_lifetime_interval *ival)
1816 {
1817 zend_life_range *range;
1818 int var_num = ssa->vars[ival->ssa_var].var;
1819
1820 fprintf(stderr, "#%d.", ival->ssa_var);
1821 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
1822 fprintf(stderr, ": %u-%u", ival->range.start, ival->range.end);
1823 range = ival->range.next;
1824 while (range) {
1825 fprintf(stderr, ", %u-%u", range->start, range->end);
1826 range = range->next;
1827 }
1828 if (ival->reg != ZREG_NONE) {
1829 fprintf(stderr, " (%s)", zend_reg_name[ival->reg]);
1830 }
1831 if (ival->flags & ZREG_LAST_USE) {
1832 fprintf(stderr, " last_use");
1833 }
1834 if (ival->flags & ZREG_LOAD) {
1835 fprintf(stderr, " load");
1836 }
1837 if (ival->flags & ZREG_STORE) {
1838 fprintf(stderr, " store");
1839 }
1840 if (ival->hint) {
1841 fprintf(stderr, " hint");
1842 if (ival->hint->ssa_var >= 0) {
1843 var_num = ssa->vars[ival->hint->ssa_var].var;
1844 fprintf(stderr, "=#%d.", ival->hint->ssa_var);
1845 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
1846 }
1847 if (ival->hint->reg != ZREG_NONE) {
1848 fprintf(stderr, " (%s)", zend_reg_name[ival->hint->reg]);
1849 }
1850 }
1851 fprintf(stderr, "\n");
1852 }
1853
zend_jit_allocate_registers(const zend_op_array * op_array,zend_ssa * ssa)1854 static zend_lifetime_interval** zend_jit_allocate_registers(const zend_op_array *op_array, zend_ssa *ssa)
1855 {
1856 void *checkpoint;
1857 int set_size, candidates_count, i;
1858 zend_bitset candidates = NULL;
1859 zend_lifetime_interval *list, *ival;
1860 zend_lifetime_interval **intervals;
1861 ALLOCA_FLAG(use_heap);
1862
1863 if (!ssa->var_info) {
1864 return NULL;
1865 }
1866
1867 /* Identify SSA variables suitable for register allocation */
1868 set_size = zend_bitset_len(ssa->vars_count);
1869 candidates = ZEND_BITSET_ALLOCA(set_size, use_heap);
1870 if (!candidates) {
1871 return NULL;
1872 }
1873 candidates_count = 0;
1874 zend_bitset_clear(candidates, set_size);
1875 for (i = 0; i < ssa->vars_count; i++) {
1876 if (zend_jit_may_be_in_reg(op_array, ssa, i)) {
1877 zend_bitset_incl(candidates, i);
1878 candidates_count++;
1879 }
1880 }
1881 if (!candidates_count) {
1882 free_alloca(candidates, use_heap);
1883 return NULL;
1884 }
1885
1886 checkpoint = zend_arena_checkpoint(CG(arena));
1887
1888 /* Find life-time intervals */
1889 if (zend_jit_compute_liveness(op_array, ssa, candidates, &list) != SUCCESS) {
1890 goto failure;
1891 }
1892
1893 if (list) {
1894 /* Set ZREG_LAST_USE flags */
1895 ival = list;
1896 while (ival) {
1897 zend_life_range *range = &ival->range;
1898
1899 while (range->next) {
1900 range = range->next;
1901 }
1902 if (zend_ssa_is_last_use(op_array, ssa, ival->ssa_var, range->end)) {
1903 ival->flags |= ZREG_LAST_USE;
1904 }
1905 ival = ival->list_next;
1906 }
1907 }
1908
1909 if (list) {
1910 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
1911 fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
1912 ival = list;
1913 while (ival) {
1914 zend_jit_dump_lifetime_interval(op_array, ssa, ival);
1915 ival = ival->list_next;
1916 }
1917 fprintf(stderr, "\n");
1918 }
1919
1920 /* Linear Scan Register Allocation */
1921 list = zend_jit_linear_scan(op_array, NULL, ssa, list);
1922
1923 if (list) {
1924 intervals = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_lifetime_interval*));
1925 if (!intervals) {
1926 goto failure;
1927 }
1928
1929 ival = list;
1930 while (ival != NULL) {
1931 zend_lifetime_interval *next = ival->list_next;
1932
1933 ival->list_next = intervals[ival->ssa_var];
1934 intervals[ival->ssa_var] = ival;
1935 ival = next;
1936 }
1937
1938 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
1939 /* Naive SSA resolution */
1940 for (i = 0; i < ssa->vars_count; i++) {
1941 if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
1942 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
1943 int k, src;
1944
1945 if (phi->pi >= 0) {
1946 if (!ssa->vars[i].phi_use_chain
1947 || ssa->vars[i].phi_use_chain->block != phi->block) {
1948 src = phi->sources[0];
1949 if (intervals[i]) {
1950 if (!intervals[src]) {
1951 intervals[i]->flags |= ZREG_LOAD;
1952 } else if (intervals[i]->reg != intervals[src]->reg) {
1953 intervals[i]->flags |= ZREG_LOAD;
1954 intervals[src]->flags |= ZREG_STORE;
1955 }
1956 } else if (intervals[src]) {
1957 intervals[src]->flags |= ZREG_STORE;
1958 }
1959 }
1960 } else {
1961 int need_move = 0;
1962
1963 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1964 src = phi->sources[k];
1965 if (src >= 0) {
1966 if (ssa->vars[src].definition_phi
1967 && ssa->vars[src].definition_phi->pi >= 0
1968 && phi->block == ssa->vars[src].definition_phi->block) {
1969 /* Skip zero-length interval for Pi variable */
1970 src = ssa->vars[src].definition_phi->sources[0];
1971 }
1972 if (intervals[i]) {
1973 if (!intervals[src]) {
1974 need_move = 1;
1975 } else if (intervals[i]->reg != intervals[src]->reg) {
1976 need_move = 1;
1977 }
1978 } else if (intervals[src]) {
1979 need_move = 1;
1980 }
1981 }
1982 }
1983 if (need_move) {
1984 if (intervals[i]) {
1985 intervals[i]->flags |= ZREG_LOAD;
1986 }
1987 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1988 src = phi->sources[k];
1989 if (src >= 0) {
1990 if (ssa->vars[src].definition_phi
1991 && ssa->vars[src].definition_phi->pi >= 0
1992 && phi->block == ssa->vars[src].definition_phi->block) {
1993 /* Skip zero-length interval for Pi variable */
1994 src = ssa->vars[src].definition_phi->sources[0];
1995 }
1996 if (intervals[src]) {
1997 intervals[src]->flags |= ZREG_STORE;
1998 }
1999 }
2000 }
2001 }
2002 }
2003 }
2004 }
2005 /* Remove useless register allocation */
2006 for (i = 0; i < ssa->vars_count; i++) {
2007 if (intervals[i] &&
2008 ((intervals[i]->flags & ZREG_LOAD) ||
2009 ((intervals[i]->flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) &&
2010 ssa->vars[i].use_chain < 0) {
2011 zend_bool may_remove = 1;
2012 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
2013
2014 while (phi) {
2015 if (intervals[phi->ssa_var] &&
2016 !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) {
2017 may_remove = 0;
2018 break;
2019 }
2020 phi = zend_ssa_next_use_phi(ssa, i, phi);
2021 }
2022 if (may_remove) {
2023 intervals[i] = NULL;
2024 }
2025 }
2026 }
2027 /* Remove intervals used once */
2028 for (i = 0; i < ssa->vars_count; i++) {
2029 if (intervals[i] &&
2030 (intervals[i]->flags & ZREG_LOAD) &&
2031 (intervals[i]->flags & ZREG_STORE) &&
2032 (ssa->vars[i].use_chain < 0 ||
2033 zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) {
2034 zend_bool may_remove = 1;
2035 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
2036
2037 while (phi) {
2038 if (intervals[phi->ssa_var] &&
2039 !(intervals[phi->ssa_var]->flags & ZREG_LOAD)) {
2040 may_remove = 0;
2041 break;
2042 }
2043 phi = zend_ssa_next_use_phi(ssa, i, phi);
2044 }
2045 if (may_remove) {
2046 intervals[i] = NULL;
2047 }
2048 }
2049 }
2050 }
2051
2052 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
2053 fprintf(stderr, "Allocated Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
2054 for (i = 0; i < ssa->vars_count; i++) {
2055 ival = intervals[i];
2056 while (ival) {
2057 zend_jit_dump_lifetime_interval(op_array, ssa, ival);
2058 ival = ival->list_next;
2059 }
2060 }
2061 fprintf(stderr, "\n");
2062 }
2063
2064 free_alloca(candidates, use_heap);
2065 return intervals;
2066 }
2067 }
2068
2069 failure:
2070 zend_arena_release(&CG(arena), checkpoint);
2071 free_alloca(candidates, use_heap);
2072 return NULL;
2073 }
2074
2075
zend_jit_supported_binary_op(zend_uchar op,uint32_t op1_info,uint32_t op2_info)2076 static bool zend_jit_supported_binary_op(zend_uchar op, uint32_t op1_info, uint32_t op2_info)
2077 {
2078 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2079 return false;
2080 }
2081 switch (op) {
2082 case ZEND_POW:
2083 case ZEND_DIV:
2084 // TODO: check for division by zero ???
2085 return false;
2086 case ZEND_ADD:
2087 case ZEND_SUB:
2088 case ZEND_MUL:
2089 return (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
2090 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
2091 case ZEND_BW_OR:
2092 case ZEND_BW_AND:
2093 case ZEND_BW_XOR:
2094 case ZEND_SL:
2095 case ZEND_SR:
2096 case ZEND_MOD:
2097 return (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG);
2098 case ZEND_CONCAT:
2099 return (op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING);
2100 EMPTY_SWITCH_DEFAULT_CASE()
2101 }
2102 }
2103
zend_jit(const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline)2104 static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline)
2105 {
2106 int b, i, end;
2107 zend_op *opline;
2108 dasm_State* dasm_state = NULL;
2109 void *handler;
2110 int call_level = 0;
2111 void *checkpoint = NULL;
2112 zend_lifetime_interval **ra = NULL;
2113 zend_bool is_terminated = 1; /* previous basic block is terminated by jump */
2114 zend_bool recv_emitted = 0; /* emitted at least one RECV opcode */
2115 zend_uchar smart_branch_opcode;
2116 uint32_t target_label, target_label2;
2117 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info;
2118 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
2119 zend_class_entry *ce;
2120 zend_bool ce_is_instanceof;
2121
2122 if (JIT_G(bisect_limit)) {
2123 jit_bisect_pos++;
2124 if (jit_bisect_pos >= JIT_G(bisect_limit)) {
2125 if (jit_bisect_pos == JIT_G(bisect_limit)) {
2126 fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n",
2127 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
2128 op_array->scope ? "::" : "",
2129 op_array->function_name ? ZSTR_VAL(op_array->function_name) : "{main}",
2130 ZSTR_VAL(op_array->filename), op_array->line_start);
2131 }
2132 return FAILURE;
2133 }
2134 }
2135
2136 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2137 checkpoint = zend_arena_checkpoint(CG(arena));
2138 ra = zend_jit_allocate_registers(op_array, ssa);
2139 }
2140
2141 /* mark hidden branch targets */
2142 for (b = 0; b < ssa->cfg.blocks_count; b++) {
2143 if (ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE &&
2144 ssa->cfg.blocks[b].len > 1) {
2145
2146 opline = op_array->opcodes + ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
2147 if (opline->opcode == ZEND_DO_FCALL &&
2148 (opline-1)->opcode == ZEND_NEW) {
2149 ssa->cfg.blocks[ssa->cfg.blocks[b].successors[0]].flags |= ZEND_BB_TARGET;
2150 }
2151 }
2152 }
2153
2154 dasm_init(&dasm_state, DASM_MAXSECTION);
2155 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
2156 dasm_setup(&dasm_state, dasm_actions);
2157
2158 dasm_growpc(&dasm_state, ssa->cfg.blocks_count * 2 + 1);
2159
2160 zend_jit_align_func(&dasm_state);
2161 for (b = 0; b < ssa->cfg.blocks_count; b++) {
2162 if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) {
2163 continue;
2164 }
2165 //#ifndef CONTEXT_THREADED_JIT
2166 if (ssa->cfg.blocks[b].flags & ZEND_BB_ENTRY) {
2167 if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
2168 /* pass */
2169 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2170 ssa->cfg.blocks[b].len == 1 &&
2171 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT) &&
2172 op_array->opcodes[ssa->cfg.blocks[b].start].opcode != ZEND_JMP) {
2173 /* don't generate code for BB with single opcode */
2174 continue;
2175 }
2176 if (ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW) {
2177 if (!is_terminated) {
2178 zend_jit_jmp(&dasm_state, b);
2179 }
2180 }
2181 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2182 zend_jit_prologue(&dasm_state);
2183 } else
2184 //#endif
2185 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
2186 opline = op_array->opcodes + ssa->cfg.blocks[b].start;
2187 if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) {
2188 if (opline->opcode == ZEND_RECV_INIT) {
2189 if (opline == op_array->opcodes ||
2190 (opline-1)->opcode != ZEND_RECV_INIT) {
2191 if (recv_emitted) {
2192 zend_jit_jmp(&dasm_state, b);
2193 }
2194 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2195 for (i = 1; (opline+i)->opcode == ZEND_RECV_INIT; i++) {
2196 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b + i);
2197 }
2198 zend_jit_prologue(&dasm_state);
2199 }
2200 recv_emitted = 1;
2201 } else if (opline->opcode == ZEND_RECV) {
2202 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2203 /* skip */
2204 continue;
2205 } else if (recv_emitted) {
2206 zend_jit_jmp(&dasm_state, b);
2207 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2208 zend_jit_prologue(&dasm_state);
2209 } else {
2210 zend_arg_info *arg_info;
2211
2212 if (opline->op1.num <= op_array->num_args) {
2213 arg_info = &op_array->arg_info[opline->op1.num - 1];
2214 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
2215 arg_info = &op_array->arg_info[op_array->num_args];
2216 } else {
2217 /* skip */
2218 continue;
2219 }
2220 if (!ZEND_TYPE_IS_SET(arg_info->type)) {
2221 /* skip */
2222 continue;
2223 }
2224 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2225 zend_jit_prologue(&dasm_state);
2226 recv_emitted = 1;
2227 }
2228 } else {
2229 if (recv_emitted) {
2230 zend_jit_jmp(&dasm_state, b);
2231 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2232 ssa->cfg.blocks[b].len == 1 &&
2233 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
2234 /* don't generate code for BB with single opcode */
2235 dasm_free(&dasm_state);
2236
2237 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2238 zend_arena_release(&CG(arena), checkpoint);
2239 }
2240 return SUCCESS;
2241 }
2242 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2243 zend_jit_prologue(&dasm_state);
2244 recv_emitted = 1;
2245 }
2246 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
2247 ssa->cfg.blocks[b].len == 1 &&
2248 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
2249 /* don't generate code for BB with single opcode */
2250 dasm_free(&dasm_state);
2251
2252 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2253 zend_arena_release(&CG(arena), checkpoint);
2254 }
2255 return SUCCESS;
2256 } else {
2257 zend_jit_label(&dasm_state, ssa->cfg.blocks_count + b);
2258 zend_jit_prologue(&dasm_state);
2259 }
2260 }
2261
2262 is_terminated = 0;
2263
2264 zend_jit_label(&dasm_state, b);
2265 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
2266 if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
2267 && ssa->cfg.blocks[b].start != 0
2268 && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
2269 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_LONG
2270 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_STRING
2271 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_MATCH)) {
2272 zend_jit_reset_last_valid_opline();
2273 if (!zend_jit_set_ip(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start)) {
2274 goto jit_failure;
2275 }
2276 } else {
2277 zend_jit_set_last_valid_opline(op_array->opcodes + ssa->cfg.blocks[b].start);
2278 }
2279 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
2280 zend_jit_reset_last_valid_opline();
2281 } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY|ZEND_BB_ENTRY)) {
2282 zend_jit_set_last_valid_opline(op_array->opcodes + ssa->cfg.blocks[b].start);
2283 }
2284 if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) {
2285 if (!zend_jit_check_timeout(&dasm_state, op_array->opcodes + ssa->cfg.blocks[b].start, NULL)) {
2286 goto jit_failure;
2287 }
2288 }
2289 if (!ssa->cfg.blocks[b].len) {
2290 continue;
2291 }
2292 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
2293 zend_ssa_phi *phi = ssa->blocks[b].phis;
2294
2295 while (phi) {
2296 zend_lifetime_interval *ival = ra[phi->ssa_var];
2297
2298 if (ival) {
2299 if (ival->flags & ZREG_LOAD) {
2300 ZEND_ASSERT(ival->reg != ZREG_NONE);
2301
2302 if (!zend_jit_load_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg)) {
2303 goto jit_failure;
2304 }
2305 } else if (ival->flags & ZREG_STORE) {
2306 ZEND_ASSERT(ival->reg != ZREG_NONE);
2307
2308 if (!zend_jit_store_var(&dasm_state, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, ival->reg, 1)) {
2309 goto jit_failure;
2310 }
2311 }
2312 }
2313 phi = phi->next;
2314 }
2315 }
2316 end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
2317 for (i = ssa->cfg.blocks[b].start; i <= end; i++) {
2318 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[i] : NULL;
2319 opline = op_array->opcodes + i;
2320 switch (opline->opcode) {
2321 case ZEND_INIT_FCALL:
2322 case ZEND_INIT_FCALL_BY_NAME:
2323 case ZEND_INIT_NS_FCALL_BY_NAME:
2324 case ZEND_INIT_METHOD_CALL:
2325 case ZEND_INIT_DYNAMIC_CALL:
2326 case ZEND_INIT_STATIC_METHOD_CALL:
2327 case ZEND_INIT_USER_CALL:
2328 case ZEND_NEW:
2329 call_level++;
2330 }
2331
2332 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
2333 switch (opline->opcode) {
2334 case ZEND_PRE_INC:
2335 case ZEND_PRE_DEC:
2336 case ZEND_POST_INC:
2337 case ZEND_POST_DEC:
2338 if (opline->op1_type != IS_CV) {
2339 break;
2340 }
2341 op1_info = OP1_INFO();
2342 if (!(op1_info & MAY_BE_LONG)) {
2343 break;
2344 }
2345 if (opline->result_type != IS_UNUSED) {
2346 res_use_info = -1;
2347
2348 if (opline->result_type == IS_CV
2349 && ssa->vars
2350 && ssa_op->result_use >= 0
2351 && !ssa->vars[ssa_op->result_use].no_val) {
2352 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
2353
2354 if (Z_MODE(res_use_addr) != IS_REG
2355 || Z_LOAD(res_use_addr)
2356 || Z_STORE(res_use_addr)) {
2357 res_use_info = RES_USE_INFO();
2358 }
2359 }
2360 res_info = RES_INFO();
2361 res_addr = RES_REG_ADDR();
2362 } else {
2363 res_use_info = -1;
2364 res_info = -1;
2365 res_addr = 0;
2366 }
2367 op1_def_info = OP1_DEF_INFO();
2368 if (!zend_jit_inc_dec(&dasm_state, opline,
2369 op1_info, OP1_REG_ADDR(),
2370 op1_def_info, OP1_DEF_REG_ADDR(),
2371 res_use_info, res_info,
2372 res_addr,
2373 (op1_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
2374 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2375 goto jit_failure;
2376 }
2377 goto done;
2378 case ZEND_BW_OR:
2379 case ZEND_BW_AND:
2380 case ZEND_BW_XOR:
2381 case ZEND_SL:
2382 case ZEND_SR:
2383 case ZEND_MOD:
2384 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2385 break;
2386 }
2387 op1_info = OP1_INFO();
2388 op2_info = OP2_INFO();
2389 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2390 break;
2391 }
2392 if (!(op1_info & MAY_BE_LONG)
2393 || !(op2_info & MAY_BE_LONG)) {
2394 break;
2395 }
2396 res_addr = RES_REG_ADDR();
2397 if (Z_MODE(res_addr) != IS_REG
2398 && opline->result_type == IS_TMP_VAR
2399 && (i + 1) <= end
2400 && (opline+1)->opcode == ZEND_SEND_VAL
2401 && (opline+1)->op1_type == IS_TMP_VAR
2402 && (opline+1)->op2_type != IS_CONST
2403 && (opline+1)->op1.var == opline->result.var) {
2404 i++;
2405 res_use_info = -1;
2406 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
2407 if (!zend_jit_reuse_ip(&dasm_state)) {
2408 goto jit_failure;
2409 }
2410 } else {
2411 res_use_info = -1;
2412
2413 if (opline->result_type == IS_CV
2414 && ssa->vars
2415 && ssa_op->result_use >= 0
2416 && !ssa->vars[ssa_op->result_use].no_val) {
2417 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
2418
2419 if (Z_MODE(res_use_addr) != IS_REG
2420 || Z_LOAD(res_use_addr)
2421 || Z_STORE(res_use_addr)) {
2422 res_use_info = RES_USE_INFO();
2423 }
2424 }
2425 }
2426 if (!zend_jit_long_math(&dasm_state, opline,
2427 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
2428 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
2429 res_use_info, RES_INFO(), res_addr,
2430 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2431 goto jit_failure;
2432 }
2433 goto done;
2434 case ZEND_ADD:
2435 case ZEND_SUB:
2436 case ZEND_MUL:
2437 // case ZEND_DIV: // TODO: check for division by zero ???
2438 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2439 break;
2440 }
2441 op1_info = OP1_INFO();
2442 op2_info = OP2_INFO();
2443 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2444 break;
2445 }
2446 if (opline->opcode == ZEND_ADD &&
2447 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
2448 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
2449 /* pass */
2450 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
2451 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
2452 break;
2453 }
2454 res_addr = RES_REG_ADDR();
2455 if (Z_MODE(res_addr) != IS_REG
2456 && opline->result_type == IS_TMP_VAR
2457 && (i + 1) <= end
2458 && (opline+1)->opcode == ZEND_SEND_VAL
2459 && (opline+1)->op1_type == IS_TMP_VAR
2460 && (opline+1)->op2_type != IS_CONST
2461 && (opline+1)->op1.var == opline->result.var) {
2462 i++;
2463 res_use_info = -1;
2464 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
2465 if (!zend_jit_reuse_ip(&dasm_state)) {
2466 goto jit_failure;
2467 }
2468 } else {
2469 res_use_info = -1;
2470
2471 if (opline->result_type == IS_CV
2472 && ssa->vars
2473 && ssa_op->result_use >= 0
2474 && !ssa->vars[ssa_op->result_use].no_val) {
2475 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
2476
2477 if (Z_MODE(res_use_addr) != IS_REG
2478 || Z_LOAD(res_use_addr)
2479 || Z_STORE(res_use_addr)) {
2480 res_use_info = RES_USE_INFO();
2481 }
2482 }
2483 }
2484 res_info = RES_INFO();
2485 if (opline->opcode == ZEND_ADD &&
2486 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
2487 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
2488 if (!zend_jit_add_arrays(&dasm_state, opline, op1_info, op2_info, res_addr)) {
2489 goto jit_failure;
2490 }
2491 } else {
2492 if (!zend_jit_math(&dasm_state, opline,
2493 op1_info, OP1_REG_ADDR(),
2494 op2_info, OP2_REG_ADDR(),
2495 res_use_info, res_info, res_addr,
2496 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
2497 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2498 goto jit_failure;
2499 }
2500 }
2501 goto done;
2502 case ZEND_CONCAT:
2503 case ZEND_FAST_CONCAT:
2504 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2505 break;
2506 }
2507 op1_info = OP1_INFO();
2508 op2_info = OP2_INFO();
2509 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
2510 break;
2511 }
2512 if (!(op1_info & MAY_BE_STRING) ||
2513 !(op2_info & MAY_BE_STRING)) {
2514 break;
2515 }
2516 res_addr = RES_REG_ADDR();
2517 if (Z_MODE(res_addr) != IS_REG
2518 && opline->result_type == IS_TMP_VAR
2519 && (i + 1) <= end
2520 && (opline+1)->opcode == ZEND_SEND_VAL
2521 && (opline+1)->op1_type == IS_TMP_VAR
2522 && (opline+1)->op2_type != IS_CONST
2523 && (opline+1)->op1.var == opline->result.var) {
2524 i++;
2525 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
2526 if (!zend_jit_reuse_ip(&dasm_state)) {
2527 goto jit_failure;
2528 }
2529 }
2530 if (!zend_jit_concat(&dasm_state, opline,
2531 op1_info, op2_info, res_addr,
2532 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2533 goto jit_failure;
2534 }
2535 goto done;
2536 case ZEND_ASSIGN_OP:
2537 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
2538 break;
2539 }
2540 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2541 break;
2542 }
2543 op1_info = OP1_INFO();
2544 op2_info = OP2_INFO();
2545 if (!zend_jit_supported_binary_op(
2546 opline->extended_value, op1_info, op2_info)) {
2547 break;
2548 }
2549 op1_def_info = OP1_DEF_INFO();
2550 if (!zend_jit_assign_op(&dasm_state, opline,
2551 op1_info, op1_def_info, OP1_RANGE(),
2552 op2_info, OP2_RANGE(),
2553 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
2554 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2555 goto jit_failure;
2556 }
2557 goto done;
2558 case ZEND_ASSIGN_DIM_OP:
2559 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
2560 break;
2561 }
2562 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2563 break;
2564 }
2565 if (!zend_jit_supported_binary_op(
2566 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
2567 break;
2568 }
2569 if (!zend_jit_assign_dim_op(&dasm_state, opline,
2570 OP1_INFO(), OP1_DEF_INFO(), OP1_REG_ADDR(), OP2_INFO(),
2571 OP1_DATA_INFO(), OP1_DATA_RANGE(),
2572 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2573 goto jit_failure;
2574 }
2575 goto done;
2576 case ZEND_ASSIGN_DIM:
2577 if (opline->op1_type != IS_CV) {
2578 break;
2579 }
2580 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2581 break;
2582 }
2583 if (!zend_jit_assign_dim(&dasm_state, opline,
2584 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), OP1_DATA_INFO(),
2585 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2586 goto jit_failure;
2587 }
2588 goto done;
2589 case ZEND_PRE_INC_OBJ:
2590 case ZEND_PRE_DEC_OBJ:
2591 case ZEND_POST_INC_OBJ:
2592 case ZEND_POST_DEC_OBJ:
2593 if (opline->op2_type != IS_CONST
2594 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2595 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2596 break;
2597 }
2598 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2599 break;
2600 }
2601 ce = NULL;
2602 ce_is_instanceof = 0;
2603 if (opline->op1_type == IS_UNUSED) {
2604 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
2605 ce = op_array->scope;
2606 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
2607 op1_addr = 0;
2608 } else {
2609 op1_info = OP1_INFO();
2610 if (!(op1_info & MAY_BE_OBJECT)) {
2611 break;
2612 }
2613 op1_addr = OP1_REG_ADDR();
2614 if (ssa->var_info && ssa->ops) {
2615 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
2616 if (ssa_op->op1_use >= 0) {
2617 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
2618 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
2619 ce = op1_ssa->ce;
2620 ce_is_instanceof = op1_ssa->is_instanceof;
2621 }
2622 }
2623 }
2624 }
2625 if (!zend_jit_incdec_obj(&dasm_state, opline, op_array, ssa, ssa_op,
2626 op1_info, op1_addr,
2627 0, ce, ce_is_instanceof, 0, NULL,
2628 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2629 goto jit_failure;
2630 }
2631 goto done;
2632 case ZEND_ASSIGN_OBJ_OP:
2633 if (opline->result_type != IS_UNUSED) {
2634 break;
2635 }
2636 if (opline->op2_type != IS_CONST
2637 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2638 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2639 break;
2640 }
2641 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2642 break;
2643 }
2644 if (!zend_jit_supported_binary_op(
2645 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
2646 break;
2647 }
2648 ce = NULL;
2649 ce_is_instanceof = 0;
2650 if (opline->op1_type == IS_UNUSED) {
2651 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
2652 ce = op_array->scope;
2653 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
2654 op1_addr = 0;
2655 } else {
2656 op1_info = OP1_INFO();
2657 if (!(op1_info & MAY_BE_OBJECT)) {
2658 break;
2659 }
2660 op1_addr = OP1_REG_ADDR();
2661 if (ssa->var_info && ssa->ops) {
2662 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
2663 if (ssa_op->op1_use >= 0) {
2664 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
2665 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
2666 ce = op1_ssa->ce;
2667 ce_is_instanceof = op1_ssa->is_instanceof;
2668 }
2669 }
2670 }
2671 }
2672 if (!zend_jit_assign_obj_op(&dasm_state, opline, op_array, ssa, ssa_op,
2673 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_RANGE(),
2674 0, ce, ce_is_instanceof, 0, NULL,
2675 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2676 goto jit_failure;
2677 }
2678 goto done;
2679 case ZEND_ASSIGN_OBJ:
2680 if (opline->op2_type != IS_CONST
2681 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2682 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2683 break;
2684 }
2685 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2686 break;
2687 }
2688 ce = NULL;
2689 ce_is_instanceof = 0;
2690 if (opline->op1_type == IS_UNUSED) {
2691 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
2692 ce = op_array->scope;
2693 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
2694 op1_addr = 0;
2695 } else {
2696 op1_info = OP1_INFO();
2697 if (!(op1_info & MAY_BE_OBJECT)) {
2698 break;
2699 }
2700 op1_addr = OP1_REG_ADDR();
2701 if (ssa->var_info && ssa->ops) {
2702 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
2703 if (ssa_op->op1_use >= 0) {
2704 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
2705 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
2706 ce = op1_ssa->ce;
2707 ce_is_instanceof = op1_ssa->is_instanceof;
2708 }
2709 }
2710 }
2711 }
2712 if (!zend_jit_assign_obj(&dasm_state, opline, op_array, ssa, ssa_op,
2713 op1_info, op1_addr, OP1_DATA_INFO(),
2714 0, ce, ce_is_instanceof, 0, NULL,
2715 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2716 goto jit_failure;
2717 }
2718 goto done;
2719 case ZEND_ASSIGN:
2720 if (opline->op1_type != IS_CV) {
2721 break;
2722 }
2723 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2724 break;
2725 }
2726 op2_addr = OP2_REG_ADDR();
2727 if (ra
2728 && ssa->ops[opline - op_array->opcodes].op2_def >= 0
2729 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) {
2730 op2_def_addr = OP2_DEF_REG_ADDR();
2731 } else {
2732 op2_def_addr = op2_addr;
2733 }
2734 op1_info = OP1_INFO();
2735 if (ra && ssa->vars[ssa_op->op1_use].no_val) {
2736 op1_info |= MAY_BE_UNDEF; // requres type assignment
2737 }
2738 if (opline->result_type == IS_UNUSED) {
2739 res_addr = 0;
2740 res_info = -1;
2741 } else {
2742 res_addr = RES_REG_ADDR();
2743 res_info = RES_INFO();
2744 if (Z_MODE(res_addr) != IS_REG
2745 && opline->result_type == IS_TMP_VAR
2746 && (i + 1) <= end
2747 && (opline+1)->opcode == ZEND_SEND_VAL
2748 && (opline+1)->op1_type == IS_TMP_VAR
2749 && (opline+1)->op2_type != IS_CONST
2750 && (opline+1)->op1.var == opline->result.var
2751 && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) {
2752 i++;
2753 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
2754 if (!zend_jit_reuse_ip(&dasm_state)) {
2755 goto jit_failure;
2756 }
2757 }
2758 }
2759 if (!zend_jit_assign(&dasm_state, opline,
2760 op1_info, OP1_REG_ADDR(),
2761 OP1_DEF_INFO(), OP1_DEF_REG_ADDR(),
2762 OP2_INFO(), op2_addr, op2_def_addr,
2763 res_info, res_addr,
2764 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2765 goto jit_failure;
2766 }
2767 goto done;
2768 case ZEND_QM_ASSIGN:
2769 op1_addr = OP1_REG_ADDR();
2770 if (ra
2771 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
2772 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
2773 op1_def_addr = OP1_DEF_REG_ADDR();
2774 } else {
2775 op1_def_addr = op1_addr;
2776 }
2777 if (!zend_jit_qm_assign(&dasm_state, opline,
2778 OP1_INFO(), op1_addr, op1_def_addr,
2779 -1, RES_INFO(), RES_REG_ADDR())) {
2780 goto jit_failure;
2781 }
2782 goto done;
2783 case ZEND_INIT_FCALL:
2784 case ZEND_INIT_FCALL_BY_NAME:
2785 case ZEND_INIT_NS_FCALL_BY_NAME:
2786 if (!zend_jit_init_fcall(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level, NULL, 1)) {
2787 goto jit_failure;
2788 }
2789 goto done;
2790 case ZEND_SEND_VAL:
2791 case ZEND_SEND_VAL_EX:
2792 if (opline->op2_type == IS_CONST) {
2793 /* Named parameters not supported in JIT (yet) */
2794 break;
2795 }
2796 if (opline->opcode == ZEND_SEND_VAL_EX
2797 && opline->op2.num > MAX_ARG_FLAG_NUM) {
2798 break;
2799 }
2800 if (!zend_jit_send_val(&dasm_state, opline,
2801 OP1_INFO(), OP1_REG_ADDR())) {
2802 goto jit_failure;
2803 }
2804 goto done;
2805 case ZEND_SEND_REF:
2806 if (opline->op2_type == IS_CONST) {
2807 /* Named parameters not supported in JIT (yet) */
2808 break;
2809 }
2810 if (!zend_jit_send_ref(&dasm_state, opline, op_array,
2811 OP1_INFO(), 0)) {
2812 goto jit_failure;
2813 }
2814 goto done;
2815 case ZEND_SEND_VAR:
2816 case ZEND_SEND_VAR_EX:
2817 case ZEND_SEND_VAR_NO_REF:
2818 case ZEND_SEND_VAR_NO_REF_EX:
2819 case ZEND_SEND_FUNC_ARG:
2820 if (opline->op2_type == IS_CONST) {
2821 /* Named parameters not supported in JIT (yet) */
2822 break;
2823 }
2824 if ((opline->opcode == ZEND_SEND_VAR_EX
2825 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
2826 && opline->op2.num > MAX_ARG_FLAG_NUM) {
2827 break;
2828 }
2829 op1_addr = OP1_REG_ADDR();
2830 if (ra
2831 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
2832 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
2833 op1_def_addr = OP1_DEF_REG_ADDR();
2834 } else {
2835 op1_def_addr = op1_addr;
2836 }
2837 if (!zend_jit_send_var(&dasm_state, opline, op_array,
2838 OP1_INFO(), op1_addr, op1_def_addr)) {
2839 goto jit_failure;
2840 }
2841 goto done;
2842 case ZEND_CHECK_FUNC_ARG:
2843 if (opline->op2_type == IS_CONST) {
2844 /* Named parameters not supported in JIT (yet) */
2845 break;
2846 }
2847 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
2848 break;
2849 }
2850 if (!zend_jit_check_func_arg(&dasm_state, opline)) {
2851 goto jit_failure;
2852 }
2853 goto done;
2854 case ZEND_CHECK_UNDEF_ARGS:
2855 if (!zend_jit_check_undef_args(&dasm_state, opline)) {
2856 goto jit_failure;
2857 }
2858 goto done;
2859 case ZEND_DO_UCALL:
2860 is_terminated = 1;
2861 /* break missing intentionally */
2862 case ZEND_DO_ICALL:
2863 case ZEND_DO_FCALL_BY_NAME:
2864 case ZEND_DO_FCALL:
2865 if (!zend_jit_do_fcall(&dasm_state, opline, op_array, ssa, call_level, b + 1, NULL)) {
2866 goto jit_failure;
2867 }
2868 goto done;
2869 case ZEND_IS_EQUAL:
2870 case ZEND_IS_NOT_EQUAL:
2871 case ZEND_IS_SMALLER:
2872 case ZEND_IS_SMALLER_OR_EQUAL:
2873 case ZEND_CASE: {
2874 res_addr = RES_REG_ADDR();
2875 if ((opline->result_type & IS_TMP_VAR)
2876 && (i + 1) <= end
2877 && ((opline+1)->opcode == ZEND_JMPZ
2878 || (opline+1)->opcode == ZEND_JMPNZ
2879 || (opline+1)->opcode == ZEND_JMPZ_EX
2880 || (opline+1)->opcode == ZEND_JMPNZ_EX
2881 || (opline+1)->opcode == ZEND_JMPZNZ)
2882 && (opline+1)->op1_type == IS_TMP_VAR
2883 && (opline+1)->op1.var == opline->result.var) {
2884 i++;
2885 smart_branch_opcode = (opline+1)->opcode;
2886 target_label = ssa->cfg.blocks[b].successors[0];
2887 target_label2 = ssa->cfg.blocks[b].successors[1];
2888 /* For EX variant write into the result of EX opcode. */
2889 if ((opline+1)->opcode == ZEND_JMPZ_EX
2890 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
2891 res_addr = OP_REG_ADDR(opline + 1, result_type, result, result_def);
2892 }
2893 } else {
2894 smart_branch_opcode = 0;
2895 target_label = target_label2 = (uint32_t)-1;
2896 }
2897 if (!zend_jit_cmp(&dasm_state, opline,
2898 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
2899 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
2900 res_addr,
2901 zend_may_throw(opline, ssa_op, op_array, ssa),
2902 smart_branch_opcode, target_label, target_label2,
2903 NULL, 0)) {
2904 goto jit_failure;
2905 }
2906 goto done;
2907 }
2908 case ZEND_IS_IDENTICAL:
2909 case ZEND_IS_NOT_IDENTICAL:
2910 case ZEND_CASE_STRICT:
2911 if ((opline->result_type & IS_TMP_VAR)
2912 && (i + 1) <= end
2913 && ((opline+1)->opcode == ZEND_JMPZ
2914 || (opline+1)->opcode == ZEND_JMPNZ
2915 || (opline+1)->opcode == ZEND_JMPZNZ)
2916 && (opline+1)->op1_type == IS_TMP_VAR
2917 && (opline+1)->op1.var == opline->result.var) {
2918 i++;
2919 smart_branch_opcode = (opline+1)->opcode;
2920 target_label = ssa->cfg.blocks[b].successors[0];
2921 target_label2 = ssa->cfg.blocks[b].successors[1];
2922 } else {
2923 smart_branch_opcode = 0;
2924 target_label = target_label2 = (uint32_t)-1;
2925 }
2926 if (!zend_jit_identical(&dasm_state, opline,
2927 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
2928 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
2929 RES_REG_ADDR(),
2930 zend_may_throw(opline, ssa_op, op_array, ssa),
2931 smart_branch_opcode, target_label, target_label2,
2932 NULL, 0)) {
2933 goto jit_failure;
2934 }
2935 goto done;
2936 case ZEND_DEFINED:
2937 if ((opline->result_type & IS_TMP_VAR)
2938 && (i + 1) <= end
2939 && ((opline+1)->opcode == ZEND_JMPZ
2940 || (opline+1)->opcode == ZEND_JMPNZ
2941 || (opline+1)->opcode == ZEND_JMPZNZ)
2942 && (opline+1)->op1_type == IS_TMP_VAR
2943 && (opline+1)->op1.var == opline->result.var) {
2944 i++;
2945 smart_branch_opcode = (opline+1)->opcode;
2946 target_label = ssa->cfg.blocks[b].successors[0];
2947 target_label2 = ssa->cfg.blocks[b].successors[1];
2948 } else {
2949 smart_branch_opcode = 0;
2950 target_label = target_label2 = (uint32_t)-1;
2951 }
2952 if (!zend_jit_defined(&dasm_state, opline, smart_branch_opcode, target_label, target_label2, NULL)) {
2953 goto jit_failure;
2954 }
2955 goto done;
2956 case ZEND_TYPE_CHECK:
2957 if (opline->extended_value == MAY_BE_RESOURCE) {
2958 // TODO: support for is_resource() ???
2959 break;
2960 }
2961 if ((opline->result_type & IS_TMP_VAR)
2962 && (i + 1) <= end
2963 && ((opline+1)->opcode == ZEND_JMPZ
2964 || (opline+1)->opcode == ZEND_JMPNZ
2965 || (opline+1)->opcode == ZEND_JMPZNZ)
2966 && (opline+1)->op1_type == IS_TMP_VAR
2967 && (opline+1)->op1.var == opline->result.var) {
2968 i++;
2969 smart_branch_opcode = (opline+1)->opcode;
2970 target_label = ssa->cfg.blocks[b].successors[0];
2971 target_label2 = ssa->cfg.blocks[b].successors[1];
2972 } else {
2973 smart_branch_opcode = 0;
2974 target_label = target_label2 = (uint32_t)-1;
2975 }
2976 if (!zend_jit_type_check(&dasm_state, opline, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) {
2977 goto jit_failure;
2978 }
2979 goto done;
2980 case ZEND_RETURN:
2981 op1_info = OP1_INFO();
2982 if ((PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info))
2983 || op_array->type == ZEND_EVAL_CODE
2984 // TODO: support for top-level code
2985 || !op_array->function_name
2986 // TODO: support for IS_UNDEF ???
2987 || (op1_info & MAY_BE_UNDEF)) {
2988 if (!zend_jit_tail_handler(&dasm_state, opline)) {
2989 goto jit_failure;
2990 }
2991 } else {
2992 int j;
2993 zend_bool left_frame = 0;
2994
2995 if (!zend_jit_return(&dasm_state, opline, op_array,
2996 op1_info, OP1_REG_ADDR())) {
2997 goto jit_failure;
2998 }
2999 if (jit_return_label >= 0) {
3000 if (!zend_jit_jmp(&dasm_state, jit_return_label)) {
3001 goto jit_failure;
3002 }
3003 goto done;
3004 }
3005 jit_return_label = ssa->cfg.blocks_count * 2;
3006 if (!zend_jit_label(&dasm_state, jit_return_label)) {
3007 goto jit_failure;
3008 }
3009 for (j = 0 ; j < op_array->last_var; j++) {
3010 uint32_t info = zend_ssa_cv_info(op_array, ssa, j);
3011
3012 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
3013 if (!left_frame) {
3014 left_frame = 1;
3015 if (!zend_jit_leave_frame(&dasm_state)) {
3016 goto jit_failure;
3017 }
3018 }
3019 if (!zend_jit_free_cv(&dasm_state, info, j)) {
3020 goto jit_failure;
3021 }
3022 }
3023 }
3024 if (!zend_jit_leave_func(&dasm_state, op_array, opline, op1_info, left_frame,
3025 NULL, NULL, (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, 1)) {
3026 goto jit_failure;
3027 }
3028 }
3029 goto done;
3030 case ZEND_BOOL:
3031 case ZEND_BOOL_NOT:
3032 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
3033 OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(),
3034 -1, -1,
3035 zend_may_throw(opline, ssa_op, op_array, ssa),
3036 opline->opcode, NULL)) {
3037 goto jit_failure;
3038 }
3039 goto done;
3040 case ZEND_JMPZ:
3041 case ZEND_JMPNZ:
3042 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
3043 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
3044 /* smart branch */
3045 if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
3046 goto jit_failure;
3047 }
3048 goto done;
3049 }
3050 /* break missing intentionally */
3051 case ZEND_JMPZNZ:
3052 case ZEND_JMPZ_EX:
3053 case ZEND_JMPNZ_EX:
3054 if (opline->result_type == IS_UNDEF) {
3055 res_addr = 0;
3056 } else {
3057 res_addr = RES_REG_ADDR();
3058 }
3059 if (!zend_jit_bool_jmpznz(&dasm_state, opline,
3060 OP1_INFO(), OP1_REG_ADDR(), res_addr,
3061 ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1],
3062 zend_may_throw(opline, ssa_op, op_array, ssa),
3063 opline->opcode, NULL)) {
3064 goto jit_failure;
3065 }
3066 goto done;
3067 case ZEND_ISSET_ISEMPTY_CV:
3068 if ((opline->extended_value & ZEND_ISEMPTY)) {
3069 // TODO: support for empty() ???
3070 break;
3071 }
3072 if ((opline->result_type & IS_TMP_VAR)
3073 && (i + 1) <= end
3074 && ((opline+1)->opcode == ZEND_JMPZ
3075 || (opline+1)->opcode == ZEND_JMPNZ
3076 || (opline+1)->opcode == ZEND_JMPZNZ)
3077 && (opline+1)->op1_type == IS_TMP_VAR
3078 && (opline+1)->op1.var == opline->result.var) {
3079 i++;
3080 smart_branch_opcode = (opline+1)->opcode;
3081 target_label = ssa->cfg.blocks[b].successors[0];
3082 target_label2 = ssa->cfg.blocks[b].successors[1];
3083 } else {
3084 smart_branch_opcode = 0;
3085 target_label = target_label2 = (uint32_t)-1;
3086 }
3087 if (!zend_jit_isset_isempty_cv(&dasm_state, opline,
3088 OP1_INFO(), OP1_REG_ADDR(),
3089 smart_branch_opcode, target_label, target_label2,
3090 NULL)) {
3091 goto jit_failure;
3092 }
3093 goto done;
3094 case ZEND_IN_ARRAY:
3095 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
3096 break;
3097 }
3098 op1_info = OP1_INFO();
3099 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
3100 break;
3101 }
3102 if ((opline->result_type & IS_TMP_VAR)
3103 && (i + 1) <= end
3104 && ((opline+1)->opcode == ZEND_JMPZ
3105 || (opline+1)->opcode == ZEND_JMPNZ
3106 || (opline+1)->opcode == ZEND_JMPZNZ)
3107 && (opline+1)->op1_type == IS_TMP_VAR
3108 && (opline+1)->op1.var == opline->result.var) {
3109 i++;
3110 smart_branch_opcode = (opline+1)->opcode;
3111 target_label = ssa->cfg.blocks[b].successors[0];
3112 target_label2 = ssa->cfg.blocks[b].successors[1];
3113 } else {
3114 smart_branch_opcode = 0;
3115 target_label = target_label2 = (uint32_t)-1;
3116 }
3117 if (!zend_jit_in_array(&dasm_state, opline,
3118 op1_info, OP1_REG_ADDR(),
3119 smart_branch_opcode, target_label, target_label2,
3120 NULL)) {
3121 goto jit_failure;
3122 }
3123 goto done;
3124 case ZEND_FETCH_DIM_R:
3125 case ZEND_FETCH_DIM_IS:
3126 case ZEND_FETCH_LIST_R:
3127 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3128 break;
3129 }
3130 if (!zend_jit_fetch_dim_read(&dasm_state, opline, ssa, ssa_op,
3131 OP1_INFO(), OP1_REG_ADDR(), 0,
3132 OP2_INFO(), RES_INFO(), RES_REG_ADDR(),
3133 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3134 goto jit_failure;
3135 }
3136 goto done;
3137 case ZEND_FETCH_DIM_W:
3138 case ZEND_FETCH_DIM_RW:
3139 // case ZEND_FETCH_DIM_UNSET:
3140 case ZEND_FETCH_LIST_W:
3141 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3142 break;
3143 }
3144 if (opline->op1_type != IS_CV) {
3145 break;
3146 }
3147 if (!zend_jit_fetch_dim(&dasm_state, opline,
3148 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), RES_REG_ADDR(),
3149 zend_may_throw_ex(opline, ssa_op, op_array, ssa, OP1_INFO(),
3150 OP2_INFO()))) {
3151 goto jit_failure;
3152 }
3153 goto done;
3154 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
3155 if ((opline->extended_value & ZEND_ISEMPTY)) {
3156 // TODO: support for empty() ???
3157 break;
3158 }
3159 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
3160 break;
3161 }
3162 if ((opline->result_type & IS_TMP_VAR)
3163 && (i + 1) <= end
3164 && ((opline+1)->opcode == ZEND_JMPZ
3165 || (opline+1)->opcode == ZEND_JMPNZ
3166 || (opline+1)->opcode == ZEND_JMPZNZ)
3167 && (opline+1)->op1_type == IS_TMP_VAR
3168 && (opline+1)->op1.var == opline->result.var) {
3169 i++;
3170 smart_branch_opcode = (opline+1)->opcode;
3171 target_label = ssa->cfg.blocks[b].successors[0];
3172 target_label2 = ssa->cfg.blocks[b].successors[1];
3173 } else {
3174 smart_branch_opcode = 0;
3175 target_label = target_label2 = (uint32_t)-1;
3176 }
3177 if (!zend_jit_isset_isempty_dim(&dasm_state, opline,
3178 OP1_INFO(), OP1_REG_ADDR(), 0,
3179 OP2_INFO(),
3180 zend_may_throw(opline, ssa_op, op_array, ssa),
3181 smart_branch_opcode, target_label, target_label2,
3182 NULL)) {
3183 goto jit_failure;
3184 }
3185 goto done;
3186 case ZEND_FETCH_OBJ_R:
3187 case ZEND_FETCH_OBJ_IS:
3188 case ZEND_FETCH_OBJ_W:
3189 if (opline->op2_type != IS_CONST
3190 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
3191 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
3192 break;
3193 }
3194 ce = NULL;
3195 ce_is_instanceof = 0;
3196 if (opline->op1_type == IS_UNUSED) {
3197 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3198 op1_addr = 0;
3199 ce = op_array->scope;
3200 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3201 } else {
3202 op1_info = OP1_INFO();
3203 if (!(op1_info & MAY_BE_OBJECT)) {
3204 break;
3205 }
3206 op1_addr = OP1_REG_ADDR();
3207 if (ssa->var_info && ssa->ops) {
3208 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3209 if (ssa_op->op1_use >= 0) {
3210 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3211 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3212 ce = op1_ssa->ce;
3213 ce_is_instanceof = op1_ssa->is_instanceof;
3214 }
3215 }
3216 }
3217 }
3218 if (!zend_jit_fetch_obj(&dasm_state, opline, op_array, ssa, ssa_op,
3219 op1_info, op1_addr, 0, ce, ce_is_instanceof, 0, 0, NULL,
3220 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3221 goto jit_failure;
3222 }
3223 goto done;
3224 case ZEND_BIND_GLOBAL:
3225 if (!ssa->ops || !ssa->var_info) {
3226 op1_info = MAY_BE_ANY|MAY_BE_REF;
3227 } else {
3228 op1_info = OP1_INFO();
3229 }
3230 if (!zend_jit_bind_global(&dasm_state, opline, op1_info)) {
3231 goto jit_failure;
3232 }
3233 goto done;
3234 case ZEND_RECV:
3235 if (!zend_jit_recv(&dasm_state, opline, op_array)) {
3236 goto jit_failure;
3237 }
3238 goto done;
3239 case ZEND_RECV_INIT:
3240 if (!zend_jit_recv_init(&dasm_state, opline, op_array,
3241 (opline + 1)->opcode != ZEND_RECV_INIT,
3242 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3243 goto jit_failure;
3244 }
3245 goto done;
3246 case ZEND_FREE:
3247 case ZEND_FE_FREE:
3248 if (!zend_jit_free(&dasm_state, opline, OP1_INFO(),
3249 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3250 goto jit_failure;
3251 }
3252 goto done;
3253 case ZEND_ECHO:
3254 op1_info = OP1_INFO();
3255 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3256 break;
3257 }
3258 if (!zend_jit_echo(&dasm_state, opline, op1_info)) {
3259 goto jit_failure;
3260 }
3261 goto done;
3262 case ZEND_STRLEN:
3263 op1_info = OP1_INFO();
3264 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
3265 break;
3266 }
3267 if (!zend_jit_strlen(&dasm_state, opline, op1_info, OP1_REG_ADDR())) {
3268 goto jit_failure;
3269 }
3270 goto done;
3271 case ZEND_FETCH_THIS:
3272 if (!zend_jit_fetch_this(&dasm_state, opline, op_array, 0)) {
3273 goto jit_failure;
3274 }
3275 goto done;
3276 case ZEND_SWITCH_LONG:
3277 case ZEND_SWITCH_STRING:
3278 case ZEND_MATCH:
3279 if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL, NULL)) {
3280 goto jit_failure;
3281 }
3282 goto done;
3283 case ZEND_VERIFY_RETURN_TYPE:
3284 if (opline->op1_type == IS_UNUSED) {
3285 /* Always throws */
3286 break;
3287 }
3288 if (opline->op1_type == IS_CONST) {
3289 /* TODO Different instruction format, has return value */
3290 break;
3291 }
3292 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
3293 /* Not worth bothering with */
3294 break;
3295 }
3296 if (OP1_INFO() & MAY_BE_REF) {
3297 /* TODO May need reference unwrapping. */
3298 break;
3299 }
3300 if (!zend_jit_verify_return_type(&dasm_state, opline, op_array, OP1_INFO())) {
3301 goto jit_failure;
3302 }
3303 goto done;
3304 case ZEND_FE_RESET_R:
3305 op1_info = OP1_INFO();
3306 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
3307 break;
3308 }
3309 if (!zend_jit_fe_reset(&dasm_state, opline, op1_info)) {
3310 goto jit_failure;
3311 }
3312 goto done;
3313 case ZEND_FE_FETCH_R:
3314 op1_info = OP1_INFO();
3315 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
3316 break;
3317 }
3318 if (!zend_jit_fe_fetch(&dasm_state, opline, op1_info, OP2_INFO(),
3319 ssa->cfg.blocks[b].successors[0], opline->opcode, NULL)) {
3320 goto jit_failure;
3321 }
3322 goto done;
3323 case ZEND_FETCH_CONSTANT:
3324 if (!zend_jit_fetch_constant(&dasm_state, opline, op_array, ssa, ssa_op)) {
3325 goto jit_failure;
3326 }
3327 goto done;
3328 case ZEND_INIT_METHOD_CALL:
3329 if (opline->op2_type != IS_CONST
3330 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
3331 break;
3332 }
3333 ce = NULL;
3334 ce_is_instanceof = 0;
3335 if (opline->op1_type == IS_UNUSED) {
3336 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
3337 op1_addr = 0;
3338 ce = op_array->scope;
3339 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
3340 } else {
3341 op1_info = OP1_INFO();
3342 if (!(op1_info & MAY_BE_OBJECT)) {
3343 break;
3344 }
3345 op1_addr = OP1_REG_ADDR();
3346 if (ssa->var_info && ssa->ops) {
3347 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
3348 if (ssa_op->op1_use >= 0) {
3349 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
3350 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
3351 ce = op1_ssa->ce;
3352 ce_is_instanceof = op1_ssa->is_instanceof;
3353 }
3354 }
3355 }
3356 }
3357 if (!zend_jit_init_method_call(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level,
3358 op1_info, op1_addr, ce, ce_is_instanceof, 0, NULL,
3359 NULL, 1, 0)) {
3360 goto jit_failure;
3361 }
3362 goto done;
3363 default:
3364 break;
3365 }
3366 }
3367
3368 switch (opline->opcode) {
3369 case ZEND_RECV_INIT:
3370 case ZEND_BIND_GLOBAL:
3371 if (opline == op_array->opcodes ||
3372 opline->opcode != op_array->opcodes[i-1].opcode) {
3373 /* repeatable opcodes */
3374 if (!zend_jit_handler(&dasm_state, opline,
3375 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3376 goto jit_failure;
3377 }
3378 }
3379 zend_jit_set_last_valid_opline(opline+1);
3380 break;
3381 case ZEND_NOP:
3382 case ZEND_OP_DATA:
3383 case ZEND_SWITCH_LONG:
3384 case ZEND_SWITCH_STRING:
3385 case ZEND_MATCH:
3386 break;
3387 case ZEND_JMP:
3388 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
3389 const zend_op *target = OP_JMP_ADDR(opline, opline->op1);
3390
3391 if (!zend_jit_set_ip(&dasm_state, target)) {
3392 goto jit_failure;
3393 }
3394 }
3395 if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) {
3396 goto jit_failure;
3397 }
3398 is_terminated = 1;
3399 break;
3400 case ZEND_CATCH:
3401 case ZEND_FAST_CALL:
3402 case ZEND_FAST_RET:
3403 case ZEND_GENERATOR_CREATE:
3404 case ZEND_GENERATOR_RETURN:
3405 case ZEND_RETURN_BY_REF:
3406 case ZEND_RETURN:
3407 case ZEND_EXIT:
3408 case ZEND_MATCH_ERROR:
3409 /* switch through trampoline */
3410 case ZEND_YIELD:
3411 case ZEND_YIELD_FROM:
3412 if (!zend_jit_tail_handler(&dasm_state, opline)) {
3413 goto jit_failure;
3414 }
3415 is_terminated = 1;
3416 break;
3417 /* stackless execution */
3418 case ZEND_INCLUDE_OR_EVAL:
3419 case ZEND_DO_FCALL:
3420 case ZEND_DO_UCALL:
3421 case ZEND_DO_FCALL_BY_NAME:
3422 if (!zend_jit_call(&dasm_state, opline, b + 1)) {
3423 goto jit_failure;
3424 }
3425 is_terminated = 1;
3426 break;
3427 case ZEND_JMPZNZ:
3428 if (!zend_jit_handler(&dasm_state, opline,
3429 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
3430 !zend_jit_cond_jmp(&dasm_state, OP_JMP_ADDR(opline, opline->op2), ssa->cfg.blocks[b].successors[1]) ||
3431 !zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b].successors[0])) {
3432 goto jit_failure;
3433 }
3434 is_terminated = 1;
3435 break;
3436 case ZEND_JMPZ:
3437 case ZEND_JMPNZ:
3438 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
3439 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
3440 /* smart branch */
3441 if (!zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
3442 goto jit_failure;
3443 }
3444 goto done;
3445 }
3446 /* break missing intentionally */
3447 case ZEND_JMPZ_EX:
3448 case ZEND_JMPNZ_EX:
3449 case ZEND_JMP_SET:
3450 case ZEND_COALESCE:
3451 case ZEND_JMP_NULL:
3452 case ZEND_FE_RESET_R:
3453 case ZEND_FE_RESET_RW:
3454 case ZEND_ASSERT_CHECK:
3455 case ZEND_FE_FETCH_R:
3456 case ZEND_FE_FETCH_RW:
3457 if (!zend_jit_handler(&dasm_state, opline,
3458 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
3459 !zend_jit_cond_jmp(&dasm_state, opline + 1, ssa->cfg.blocks[b].successors[0])) {
3460 goto jit_failure;
3461 }
3462 break;
3463 case ZEND_NEW:
3464 if (!zend_jit_handler(&dasm_state, opline, 1)) {
3465 return 0;
3466 }
3467 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
3468 zend_class_entry *ce = NULL;
3469
3470 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
3471 if (ssa->ops && ssa->var_info) {
3472 zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
3473 if (res_ssa->ce && !res_ssa->is_instanceof) {
3474 ce = res_ssa->ce;
3475 }
3476 }
3477 } else {
3478 if (opline->op1_type == IS_CONST) {
3479 zval *zv = RT_CONSTANT(opline, opline->op1);
3480 if (Z_TYPE_P(zv) == IS_STRING) {
3481 zval *lc = zv + 1;
3482 ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc));
3483 }
3484 }
3485 }
3486
3487 i++;
3488
3489 if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) {
3490 const zend_op *next_opline = opline + 1;
3491
3492 zend_jit_cond_jmp(&dasm_state, next_opline, ssa->cfg.blocks[b].successors[0]);
3493 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
3494 zend_jit_call(&dasm_state, next_opline, b + 1);
3495 is_terminated = 1;
3496 } else {
3497 zend_jit_do_fcall(&dasm_state, next_opline, op_array, ssa, call_level, b + 1, NULL);
3498 }
3499 }
3500 }
3501 break;
3502 default:
3503 if (!zend_jit_handler(&dasm_state, opline,
3504 zend_may_throw(opline, ssa_op, op_array, ssa))) {
3505 goto jit_failure;
3506 }
3507 if (i == end
3508 && (opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
3509 /* smart branch split across basic blocks */
3510 if (!zend_jit_cond_jmp(&dasm_state, opline + 2, ssa->cfg.blocks[b+1].successors[0])) {
3511 goto jit_failure;
3512 }
3513 if (!zend_jit_jmp(&dasm_state, ssa->cfg.blocks[b+1].successors[1])) {
3514 goto jit_failure;
3515 }
3516 is_terminated = 1;
3517 }
3518 }
3519 done:
3520 switch (opline->opcode) {
3521 case ZEND_DO_FCALL:
3522 case ZEND_DO_ICALL:
3523 case ZEND_DO_UCALL:
3524 case ZEND_DO_FCALL_BY_NAME:
3525 call_level--;
3526 }
3527 }
3528 }
3529
3530 handler = dasm_link_and_encode(&dasm_state, op_array, ssa, rt_opline, ra, NULL, 0);
3531 if (!handler) {
3532 goto jit_failure;
3533 }
3534 dasm_free(&dasm_state);
3535
3536 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
3537 zend_arena_release(&CG(arena), checkpoint);
3538 }
3539 return SUCCESS;
3540
3541 jit_failure:
3542 if (dasm_state) {
3543 dasm_free(&dasm_state);
3544 }
3545 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
3546 zend_arena_release(&CG(arena), checkpoint);
3547 }
3548 return FAILURE;
3549 }
3550
zend_jit_collect_calls(zend_op_array * op_array,zend_script * script)3551 static int zend_jit_collect_calls(zend_op_array *op_array, zend_script *script)
3552 {
3553 zend_func_info *func_info;
3554
3555 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
3556 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
3557 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
3558 func_info = ZEND_FUNC_INFO(op_array);
3559 } else {
3560 func_info = zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info));
3561 ZEND_SET_FUNC_INFO(op_array, func_info);
3562 }
3563 return zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, op_array, func_info);
3564 }
3565
zend_jit_cleanup_func_info(zend_op_array * op_array)3566 static void zend_jit_cleanup_func_info(zend_op_array *op_array)
3567 {
3568 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
3569 zend_call_info *caller_info, *callee_info;
3570
3571 if (func_info) {
3572 caller_info = func_info->caller_info;
3573 callee_info = func_info->callee_info;
3574
3575 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
3576 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
3577 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
3578 func_info->num = 0;
3579 func_info->flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
3580 | ZEND_FUNC_JIT_ON_PROF_REQUEST
3581 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
3582 | ZEND_FUNC_JIT_ON_HOT_TRACE;
3583 memset(&func_info->ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
3584 } else {
3585 ZEND_SET_FUNC_INFO(op_array, NULL);
3586 }
3587
3588 while (caller_info) {
3589 if (caller_info->caller_op_array) {
3590 zend_jit_cleanup_func_info(caller_info->caller_op_array);
3591 }
3592 caller_info = caller_info->next_caller;
3593 }
3594 while (callee_info) {
3595 if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) {
3596 zend_jit_cleanup_func_info(&callee_info->callee_func->op_array);
3597 }
3598 callee_info = callee_info->next_callee;
3599 }
3600 }
3601 }
3602
zend_real_jit_func(zend_op_array * op_array,zend_script * script,const zend_op * rt_opline)3603 static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline)
3604 {
3605 zend_ssa ssa;
3606 void *checkpoint;
3607 zend_func_info *func_info;
3608
3609 if (*dasm_ptr == dasm_end) {
3610 return FAILURE;
3611 }
3612
3613 checkpoint = zend_arena_checkpoint(CG(arena));
3614
3615 /* Build SSA */
3616 memset(&ssa, 0, sizeof(zend_ssa));
3617
3618 if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) {
3619 goto jit_failure;
3620 }
3621
3622 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
3623 if (zend_jit_collect_calls(op_array, script) != SUCCESS) {
3624 goto jit_failure;
3625 }
3626 func_info = ZEND_FUNC_INFO(op_array);
3627 func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array);
3628 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
3629 zend_init_func_return_info(op_array, script, &func_info->return_info);
3630 }
3631 }
3632
3633 if (zend_jit_op_array_analyze2(op_array, script, &ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
3634 goto jit_failure;
3635 }
3636
3637 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
3638 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa);
3639 }
3640
3641 if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) {
3642 goto jit_failure;
3643 }
3644
3645 zend_jit_cleanup_func_info(op_array);
3646 zend_arena_release(&CG(arena), checkpoint);
3647 return SUCCESS;
3648
3649 jit_failure:
3650 zend_jit_cleanup_func_info(op_array);
3651 zend_arena_release(&CG(arena), checkpoint);
3652 return FAILURE;
3653 }
3654
3655 /* Run-time JIT handler */
zend_runtime_jit(void)3656 static int ZEND_FASTCALL zend_runtime_jit(void)
3657 {
3658 zend_execute_data *execute_data = EG(current_execute_data);
3659 zend_op_array *op_array = &EX(func)->op_array;
3660 zend_op *opline = op_array->opcodes;
3661 zend_jit_op_array_extension *jit_extension;
3662
3663 zend_shared_alloc_lock();
3664
3665 if (ZEND_FUNC_INFO(op_array)) {
3666 SHM_UNPROTECT();
3667 zend_jit_unprotect();
3668
3669 /* restore original opcode handlers */
3670 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3671 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3672 opline++;
3673 }
3674 }
3675 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
3676 opline->handler = jit_extension->orig_handler;
3677
3678 /* perform real JIT for this function */
3679 zend_real_jit_func(op_array, NULL, NULL);
3680
3681 zend_jit_protect();
3682 SHM_PROTECT();
3683 }
3684
3685 zend_shared_alloc_unlock();
3686
3687 /* JIT-ed code is going to be called by VM */
3688 return 0;
3689 }
3690
zend_jit_check_funcs(HashTable * function_table,zend_bool is_method)3691 void zend_jit_check_funcs(HashTable *function_table, zend_bool is_method) {
3692 zend_op *opline;
3693 zend_function *func;
3694 zend_op_array *op_array;
3695 uintptr_t counter;
3696 zend_jit_op_array_extension *jit_extension;
3697
3698 ZEND_HASH_REVERSE_FOREACH_PTR(function_table, func) {
3699 if (func->type == ZEND_INTERNAL_FUNCTION) {
3700 break;
3701 }
3702 op_array = &func->op_array;
3703 opline = op_array->opcodes;
3704 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3705 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3706 opline++;
3707 }
3708 }
3709 if (opline->handler == zend_jit_profile_jit_handler) {
3710 if (!RUN_TIME_CACHE(op_array)) {
3711 continue;
3712 }
3713 counter = (uintptr_t)ZEND_COUNTER_INFO(op_array);
3714 ZEND_COUNTER_INFO(op_array) = 0;
3715 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
3716 opline->handler = jit_extension->orig_handler;
3717 if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) {
3718 zend_real_jit_func(op_array, NULL, NULL);
3719 }
3720 }
3721 } ZEND_HASH_FOREACH_END();
3722 }
3723
zend_jit_hot_func(zend_execute_data * execute_data,const zend_op * opline)3724 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
3725 {
3726 zend_op_array *op_array = &EX(func)->op_array;
3727 zend_jit_op_array_hot_extension *jit_extension;
3728 uint32_t i;
3729
3730 zend_shared_alloc_lock();
3731 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
3732
3733 if (jit_extension) {
3734 SHM_UNPROTECT();
3735 zend_jit_unprotect();
3736
3737 for (i = 0; i < op_array->last; i++) {
3738 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
3739 }
3740
3741 /* perform real JIT for this function */
3742 zend_real_jit_func(op_array, NULL, opline);
3743
3744 zend_jit_protect();
3745 SHM_PROTECT();
3746 }
3747
3748 zend_shared_alloc_unlock();
3749
3750 /* JIT-ed code is going to be called by VM */
3751 }
3752
zend_jit_setup_hot_counters_ex(zend_op_array * op_array,zend_cfg * cfg)3753 static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg)
3754 {
3755 if (JIT_G(hot_func)) {
3756 zend_op *opline = op_array->opcodes;
3757
3758 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3759 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3760 opline++;
3761 }
3762 }
3763
3764 opline->handler = (const void*)zend_jit_func_hot_counter_handler;
3765 }
3766
3767 if (JIT_G(hot_loop)) {
3768 uint32_t i;
3769
3770 for (i = 0; i < cfg->blocks_count; i++) {
3771 if ((cfg->blocks[i].flags & ZEND_BB_REACHABLE) &&
3772 (cfg->blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
3773 op_array->opcodes[cfg->blocks[i].start].handler =
3774 (const void*)zend_jit_loop_hot_counter_handler;
3775 }
3776 }
3777 }
3778 }
3779
zend_jit_restart_hot_counters(zend_op_array * op_array)3780 static int zend_jit_restart_hot_counters(zend_op_array *op_array)
3781 {
3782 zend_jit_op_array_hot_extension *jit_extension;
3783 zend_cfg cfg;
3784 uint32_t i;
3785
3786 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
3787 for (i = 0; i < op_array->last; i++) {
3788 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
3789 }
3790
3791 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
3792 return FAILURE;
3793 }
3794
3795 zend_jit_setup_hot_counters_ex(op_array, &cfg);
3796
3797 return SUCCESS;
3798 }
3799
zend_jit_setup_hot_counters(zend_op_array * op_array)3800 static int zend_jit_setup_hot_counters(zend_op_array *op_array)
3801 {
3802 zend_jit_op_array_hot_extension *jit_extension;
3803 zend_cfg cfg;
3804 uint32_t i;
3805
3806 ZEND_ASSERT(zend_jit_func_hot_counter_handler != NULL);
3807 ZEND_ASSERT(zend_jit_loop_hot_counter_handler != NULL);
3808
3809 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
3810 return FAILURE;
3811 }
3812
3813 jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*));
3814 if (!jit_extension) {
3815 return FAILURE;
3816 }
3817 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3818 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS;
3819 jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
3820 for (i = 0; i < op_array->last; i++) {
3821 jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
3822 }
3823 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3824
3825 zend_jit_setup_hot_counters_ex(op_array, &cfg);
3826
3827 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3828
3829 return SUCCESS;
3830 }
3831
3832 #include "jit/zend_jit_trace.c"
3833
zend_jit_op_array(zend_op_array * op_array,zend_script * script)3834 ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
3835 {
3836 if (dasm_ptr == NULL) {
3837 return FAILURE;
3838 }
3839
3840 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) {
3841 zend_jit_op_array_extension *jit_extension;
3842 zend_op *opline = op_array->opcodes;
3843
3844 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
3845 ZEND_SET_FUNC_INFO(op_array, NULL);
3846 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
3847 return SUCCESS;
3848 }
3849
3850 /* Set run-time JIT handler */
3851 ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL);
3852 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3853 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3854 opline++;
3855 }
3856 }
3857 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
3858 if (!jit_extension) {
3859 return FAILURE;
3860 }
3861 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3862 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC;
3863 jit_extension->orig_handler = (void*)opline->handler;
3864 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3865 opline->handler = (const void*)zend_jit_runtime_jit_handler;
3866 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3867
3868 return SUCCESS;
3869 } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) {
3870 zend_jit_op_array_extension *jit_extension;
3871 zend_op *opline = op_array->opcodes;
3872
3873 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
3874 ZEND_SET_FUNC_INFO(op_array, NULL);
3875 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
3876 return SUCCESS;
3877 }
3878
3879 ZEND_ASSERT(zend_jit_profile_jit_handler != NULL);
3880 if (op_array->function_name) {
3881 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3882 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3883 opline++;
3884 }
3885 }
3886 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
3887 if (!jit_extension) {
3888 return FAILURE;
3889 }
3890 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3891 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST;
3892 jit_extension->orig_handler = (void*)opline->handler;
3893 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3894 opline->handler = (const void*)zend_jit_profile_jit_handler;
3895 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3896 }
3897
3898 return SUCCESS;
3899 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
3900 return zend_jit_setup_hot_counters(op_array);
3901 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3902 return zend_jit_setup_hot_trace_counters(op_array);
3903 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3904 return zend_real_jit_func(op_array, script, NULL);
3905 } else {
3906 ZEND_UNREACHABLE();
3907 }
3908 }
3909
zend_jit_script(zend_script * script)3910 ZEND_EXT_API int zend_jit_script(zend_script *script)
3911 {
3912 void *checkpoint;
3913 zend_call_graph call_graph;
3914 zend_func_info *info;
3915 int i;
3916
3917 if (dasm_ptr == NULL || *dasm_ptr == dasm_end) {
3918 return FAILURE;
3919 }
3920
3921 checkpoint = zend_arena_checkpoint(CG(arena));
3922
3923 call_graph.op_arrays_count = 0;
3924 if (zend_build_call_graph(&CG(arena), script, &call_graph) != SUCCESS) {
3925 goto jit_failure;
3926 }
3927
3928 zend_analyze_call_graph(&CG(arena), script, &call_graph);
3929
3930 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
3931 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
3932 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS ||
3933 JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3934 for (i = 0; i < call_graph.op_arrays_count; i++) {
3935 if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
3936 goto jit_failure;
3937 }
3938 }
3939 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3940 for (i = 0; i < call_graph.op_arrays_count; i++) {
3941 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3942 if (info) {
3943 if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
3944 goto jit_failure;
3945 }
3946 info->flags = info->ssa.cfg.flags;
3947 }
3948 }
3949
3950 for (i = 0; i < call_graph.op_arrays_count; i++) {
3951 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3952 if (info) {
3953 info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]);
3954 if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
3955 zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info);
3956 }
3957 }
3958 }
3959
3960 for (i = 0; i < call_graph.op_arrays_count; i++) {
3961 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3962 if (info) {
3963 if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
3964 goto jit_failure;
3965 }
3966 info->flags = info->ssa.cfg.flags;
3967 }
3968 }
3969
3970 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
3971 for (i = 0; i < call_graph.op_arrays_count; i++) {
3972 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3973 if (info) {
3974 zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa);
3975 }
3976 }
3977 }
3978
3979 for (i = 0; i < call_graph.op_arrays_count; i++) {
3980 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3981 if (info) {
3982 if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) {
3983 goto jit_failure;
3984 }
3985 }
3986 }
3987
3988 for (i = 0; i < call_graph.op_arrays_count; i++) {
3989 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
3990 }
3991 } else {
3992 ZEND_UNREACHABLE();
3993 }
3994
3995 zend_arena_release(&CG(arena), checkpoint);
3996
3997 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC
3998 || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST
3999 || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS
4000 || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4001 zend_class_entry *ce;
4002 zend_op_array *op_array;
4003
4004 ZEND_HASH_FOREACH_PTR(&script->class_table, ce) {
4005 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
4006 if (!ZEND_FUNC_INFO(op_array)) {
4007 void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes);
4008
4009 if (jit_extension) {
4010 ZEND_SET_FUNC_INFO(op_array, jit_extension);
4011 }
4012 }
4013 } ZEND_HASH_FOREACH_END();
4014 } ZEND_HASH_FOREACH_END();
4015 }
4016
4017 return SUCCESS;
4018
4019 jit_failure:
4020 if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
4021 for (i = 0; i < call_graph.op_arrays_count; i++) {
4022 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
4023 }
4024 }
4025 zend_arena_release(&CG(arena), checkpoint);
4026 return FAILURE;
4027 }
4028
zend_jit_unprotect(void)4029 ZEND_EXT_API void zend_jit_unprotect(void)
4030 {
4031 #ifdef HAVE_MPROTECT
4032 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4033 int opts = PROT_READ | PROT_WRITE;
4034 #ifdef ZTS
4035 /* Another thread may be executing JITed code. */
4036 opts |= PROT_EXEC;
4037 #endif
4038 if (mprotect(dasm_buf, dasm_size, opts) != 0) {
4039 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4040 }
4041 }
4042 #elif _WIN32
4043 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4044 DWORD old, new;
4045 #ifdef ZTS
4046 new = PAGE_EXECUTE_READWRITE;
4047 #else
4048 new = PAGE_READWRITE;
4049 #endif
4050 if (!VirtualProtect(dasm_buf, dasm_size, new, &old)) {
4051 DWORD err = GetLastError();
4052 char *msg = php_win32_error_to_msg(err);
4053 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4054 php_win32_error_msg_free(msg);
4055 }
4056 }
4057 #endif
4058 }
4059
zend_jit_protect(void)4060 ZEND_EXT_API void zend_jit_protect(void)
4061 {
4062 #ifdef HAVE_MPROTECT
4063 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4064 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
4065 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4066 }
4067 }
4068 #elif _WIN32
4069 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
4070 DWORD old;
4071
4072 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
4073 DWORD err = GetLastError();
4074 char *msg = php_win32_error_to_msg(err);
4075 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4076 php_win32_error_msg_free(msg);
4077 }
4078 }
4079 #endif
4080 }
4081
zend_jit_init_handlers(void)4082 static void zend_jit_init_handlers(void)
4083 {
4084 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
4085 zend_jit_runtime_jit_handler = dasm_labels[zend_lbhybrid_runtime_jit];
4086 zend_jit_profile_jit_handler = dasm_labels[zend_lbhybrid_profile_jit];
4087 zend_jit_func_hot_counter_handler = dasm_labels[zend_lbhybrid_func_hot_counter];
4088 zend_jit_loop_hot_counter_handler = dasm_labels[zend_lbhybrid_loop_hot_counter];
4089 zend_jit_func_trace_counter_handler = dasm_labels[zend_lbhybrid_func_trace_counter];
4090 zend_jit_ret_trace_counter_handler = dasm_labels[zend_lbhybrid_ret_trace_counter];
4091 zend_jit_loop_trace_counter_handler = dasm_labels[zend_lbhybrid_loop_trace_counter];
4092 } else {
4093 zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
4094 zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
4095 zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper;
4096 zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper;
4097 zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper;
4098 zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper;
4099 zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper;
4100 }
4101 }
4102
zend_jit_make_stubs(void)4103 static int zend_jit_make_stubs(void)
4104 {
4105 dasm_State* dasm_state = NULL;
4106 uint32_t i;
4107
4108 dasm_init(&dasm_state, DASM_MAXSECTION);
4109 dasm_setupglobal(&dasm_state, dasm_labels, zend_lb_MAX);
4110
4111 for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) {
4112 dasm_setup(&dasm_state, dasm_actions);
4113 if (!zend_jit_stubs[i].stub(&dasm_state)) {
4114 return 0;
4115 }
4116 if (!dasm_link_and_encode(&dasm_state, NULL, NULL, NULL, NULL, zend_jit_stubs[i].name, 0)) {
4117 return 0;
4118 }
4119 }
4120
4121 zend_jit_init_handlers();
4122
4123 dasm_free(&dasm_state);
4124 return 1;
4125 }
4126
zend_jit_globals_ctor(zend_jit_globals * jit_globals)4127 static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
4128 {
4129 memset(jit_globals, 0, sizeof(zend_jit_globals));
4130 zend_jit_trace_init_caches();
4131 }
4132
zend_jit_parse_config_num(zend_long jit)4133 static int zend_jit_parse_config_num(zend_long jit)
4134 {
4135 if (jit == 0) {
4136 JIT_G(on) = 0;
4137 return SUCCESS;
4138 }
4139
4140 if (jit < 0) return FAILURE;
4141
4142 if (jit % 10 == 0 || jit % 10 > 5) return FAILURE;
4143 JIT_G(opt_level) = jit % 10;
4144
4145 jit /= 10;
4146 if (jit % 10 > 5) return FAILURE;
4147 JIT_G(trigger) = jit % 10;
4148
4149 jit /= 10;
4150 if (jit % 10 > 2) return FAILURE;
4151 JIT_G(opt_flags) = jit % 10;
4152
4153 jit /= 10;
4154 if (jit % 10 > 1) return FAILURE;
4155 JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0);
4156
4157 if (jit / 10 != 0) return FAILURE;
4158
4159 JIT_G(on) = 1;
4160
4161 return SUCCESS;
4162 }
4163
zend_jit_config(zend_string * jit,int stage)4164 ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage)
4165 {
4166 if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) {
4167 if (stage == ZEND_INI_STAGE_RUNTIME) {
4168 zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)");
4169 }
4170 return FAILURE;
4171 }
4172
4173 if (ZSTR_LEN(jit) == 0
4174 || zend_string_equals_literal_ci(jit, "disable")) {
4175 JIT_G(enabled) = 0;
4176 JIT_G(on) = 0;
4177 return SUCCESS;
4178 } else if (zend_string_equals_literal_ci(jit, "0")
4179 || zend_string_equals_literal_ci(jit, "off")
4180 || zend_string_equals_literal_ci(jit, "no")
4181 || zend_string_equals_literal_ci(jit, "false")) {
4182 JIT_G(enabled) = 1;
4183 JIT_G(on) = 0;
4184 return SUCCESS;
4185 } else if (zend_string_equals_literal_ci(jit, "1")
4186 || zend_string_equals_literal_ci(jit, "on")
4187 || zend_string_equals_literal_ci(jit, "yes")
4188 || zend_string_equals_literal_ci(jit, "true")
4189 || zend_string_equals_literal_ci(jit, "tracing")) {
4190 JIT_G(enabled) = 1;
4191 JIT_G(on) = 1;
4192 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS;
4193 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
4194 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
4195 return SUCCESS;
4196 } else if (zend_string_equals_literal_ci(jit, "function")) {
4197 JIT_G(enabled) = 1;
4198 JIT_G(on) = 1;
4199 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT;
4200 JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD;
4201 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
4202 return SUCCESS;
4203 } else {
4204 char *end;
4205 zend_long num = ZEND_STRTOL(ZSTR_VAL(jit), &end, 10);
4206 if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) {
4207 goto failure;
4208 }
4209 JIT_G(enabled) = 1;
4210 return SUCCESS;
4211 }
4212
4213 failure:
4214 zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number");
4215 JIT_G(enabled) = 0;
4216 JIT_G(on) = 0;
4217 return FAILURE;
4218 }
4219
zend_jit_debug_config(zend_long old_val,zend_long new_val,int stage)4220 ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage)
4221 {
4222 if (stage != ZEND_INI_STAGE_STARTUP) {
4223 if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) {
4224 if (stage == ZEND_INI_STAGE_RUNTIME) {
4225 zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup");
4226 }
4227 return FAILURE;
4228 }
4229 #ifdef HAVE_DISASM
4230 if (new_val & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
4231 if (JIT_G(enabled) && !JIT_G(symbols) && !zend_jit_disasm_init()) {
4232 // TODO: error reporting and cleanup ???
4233 return FAILURE;
4234 }
4235 // TODO: symbols for JIT-ed code compiled before are missing ???
4236 }
4237 #endif
4238 }
4239 return SUCCESS;
4240 }
4241
zend_jit_init(void)4242 ZEND_EXT_API void zend_jit_init(void)
4243 {
4244 #ifdef ZTS
4245 jit_globals_id = ts_allocate_id(&jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, NULL);
4246 #else
4247 zend_jit_globals_ctor(&jit_globals);
4248 #endif
4249 }
4250
zend_jit_check_support(void)4251 ZEND_EXT_API int zend_jit_check_support(void)
4252 {
4253 int i;
4254
4255 zend_jit_vm_kind = zend_vm_kind();
4256 if (zend_jit_vm_kind != ZEND_VM_KIND_CALL &&
4257 zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
4258 zend_error(E_WARNING, "JIT is compatible only with CALL and HYBRID VM. JIT disabled.");
4259 JIT_G(enabled) = 0;
4260 JIT_G(on) = 0;
4261 return FAILURE;
4262 }
4263
4264 if (zend_execute_ex != execute_ex) {
4265 if (strcmp(sapi_module.name, "phpdbg") != 0) {
4266 zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled.");
4267 }
4268 JIT_G(enabled) = 0;
4269 JIT_G(on) = 0;
4270 return FAILURE;
4271 }
4272
4273 for (i = 0; i <= 256; i++) {
4274 switch (i) {
4275 /* JIT has no effect on these opcodes */
4276 case ZEND_BEGIN_SILENCE:
4277 case ZEND_END_SILENCE:
4278 case ZEND_EXIT:
4279 break;
4280 default:
4281 if (zend_get_user_opcode_handler(i) != NULL) {
4282 zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled.");
4283 JIT_G(enabled) = 0;
4284 JIT_G(on) = 0;
4285 return FAILURE;
4286 }
4287 }
4288 }
4289
4290 return SUCCESS;
4291 }
4292
zend_jit_startup(void * buf,size_t size,zend_bool reattached)4293 ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, zend_bool reattached)
4294 {
4295 int ret;
4296
4297 zend_jit_halt_op = zend_get_halt_op();
4298
4299 if (zend_jit_setup() != SUCCESS) {
4300 // TODO: error reporting and cleanup ???
4301 return FAILURE;
4302 }
4303
4304 zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(ACCELERATOR_PRODUCT_NAME);
4305
4306 #ifdef HAVE_GDB
4307 zend_jit_gdb_init();
4308 #endif
4309
4310 #ifdef HAVE_OPROFILE
4311 if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
4312 if (!zend_jit_oprofile_startup()) {
4313 // TODO: error reporting and cleanup ???
4314 return FAILURE;
4315 }
4316 }
4317 #endif
4318
4319 dasm_buf = buf;
4320 dasm_size = size;
4321
4322 #ifdef HAVE_MPROTECT
4323 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
4324 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
4325 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4326 }
4327 } else {
4328 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
4329 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
4330 }
4331 }
4332 #elif _WIN32
4333 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
4334 DWORD old;
4335
4336 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
4337 DWORD err = GetLastError();
4338 char *msg = php_win32_error_to_msg(err);
4339 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4340 php_win32_error_msg_free(msg);
4341 }
4342 } else {
4343 DWORD old;
4344
4345 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
4346 DWORD err = GetLastError();
4347 char *msg = php_win32_error_to_msg(err);
4348 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
4349 php_win32_error_msg_free(msg);
4350 }
4351 }
4352 #endif
4353
4354 dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr) * 2);
4355 if (!reattached) {
4356 zend_jit_unprotect();
4357 *dasm_ptr = dasm_buf;
4358 #if _WIN32
4359 /* reserve space for global labels */
4360 *dasm_ptr = (void**)*dasm_ptr + zend_lb_MAX;
4361 #endif
4362 zend_jit_protect();
4363 }
4364
4365 #ifdef HAVE_DISASM
4366 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
4367 if (!zend_jit_disasm_init()) {
4368 // TODO: error reporting and cleanup ???
4369 return FAILURE;
4370 }
4371 }
4372 #endif
4373
4374 #ifdef HAVE_PERFTOOLS
4375 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
4376 zend_jit_perf_jitdump_open();
4377 }
4378 #endif
4379
4380 if (!reattached) {
4381 zend_jit_unprotect();
4382 ret = zend_jit_make_stubs();
4383 #if _WIN32
4384 /* save global labels */
4385 memcpy(dasm_buf, dasm_labels, sizeof(void*) * zend_lb_MAX);
4386 #endif
4387 zend_jit_protect();
4388 if (!ret) {
4389 // TODO: error reporting and cleanup ???
4390 return FAILURE;
4391 }
4392 } else {
4393 #if _WIN32
4394 /* restore global labels */
4395 memcpy(dasm_labels, dasm_buf, sizeof(void*) * zend_lb_MAX);
4396 zend_jit_init_handlers();
4397 #endif
4398 }
4399
4400 if (zend_jit_trace_startup(reattached) != SUCCESS) {
4401 return FAILURE;
4402 }
4403
4404 /* save JIT buffer pos */
4405 zend_jit_unprotect();
4406 dasm_ptr[1] = dasm_ptr[0];
4407 zend_jit_protect();
4408
4409 return SUCCESS;
4410 }
4411
zend_jit_shutdown(void)4412 ZEND_EXT_API void zend_jit_shutdown(void)
4413 {
4414 if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE) {
4415 fprintf(stderr, "\nJIT memory usage: %td\n", (char*)*dasm_ptr - (char*)dasm_buf);
4416 }
4417
4418 #ifdef HAVE_OPROFILE
4419 if (JIT_G(debug) & ZEND_JIT_DEBUG_OPROFILE) {
4420 zend_jit_oprofile_shutdown();
4421 }
4422 #endif
4423
4424 #ifdef HAVE_GDB
4425 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
4426 zend_jit_gdb_unregister();
4427 }
4428 #endif
4429
4430 #ifdef HAVE_DISASM
4431 zend_jit_disasm_shutdown();
4432 #endif
4433
4434 #ifdef HAVE_PERFTOOLS
4435 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
4436 zend_jit_perf_jitdump_close();
4437 }
4438 #endif
4439 if (JIT_G(exit_counters)) {
4440 free(JIT_G(exit_counters));
4441 }
4442 }
4443
zend_jit_reset_counters(void)4444 static void zend_jit_reset_counters(void)
4445 {
4446 int i;
4447
4448 for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
4449 zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT;
4450 }
4451 }
4452
zend_jit_activate(void)4453 ZEND_EXT_API void zend_jit_activate(void)
4454 {
4455 zend_jit_profile_counter = 0;
4456 if (JIT_G(on)) {
4457 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
4458 zend_jit_reset_counters();
4459 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
4460 zend_jit_reset_counters();
4461 zend_jit_trace_reset_caches();
4462 }
4463 }
4464 }
4465
zend_jit_deactivate(void)4466 ZEND_EXT_API void zend_jit_deactivate(void)
4467 {
4468 if (zend_jit_profile_counter) {
4469 zend_class_entry *ce;
4470
4471 zend_shared_alloc_lock();
4472 SHM_UNPROTECT();
4473 zend_jit_unprotect();
4474
4475 zend_jit_check_funcs(EG(function_table), 0);
4476 ZEND_HASH_REVERSE_FOREACH_PTR(EG(class_table), ce) {
4477 if (ce->type == ZEND_INTERNAL_CLASS) {
4478 break;
4479 }
4480 zend_jit_check_funcs(&ce->function_table, 1);
4481 } ZEND_HASH_FOREACH_END();
4482
4483 zend_jit_protect();
4484 SHM_PROTECT();
4485 zend_shared_alloc_unlock();
4486
4487 zend_jit_profile_counter = 0;
4488 }
4489 }
4490
zend_jit_restart_preloaded_op_array(zend_op_array * op_array)4491 static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array)
4492 {
4493 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
4494
4495 if (!func_info) {
4496 return;
4497 }
4498
4499 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
4500 zend_jit_restart_hot_trace_counters(op_array);
4501 } else if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_COUNTERS) {
4502 zend_jit_restart_hot_counters(op_array);
4503 #if 0
4504 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ???
4505 } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) {
4506 zend_op *opline = op_array->opcodes;
4507 zend_jit_op_array_extension *jit_extension =
4508 (zend_jit_op_array_extension*)func_info;
4509
4510 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
4511 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
4512 opline++;
4513 }
4514 }
4515 if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) {
4516 opline->handler = (const void*)zend_jit_runtime_jit_handler;
4517 } else {
4518 opline->handler = (const void*)zend_jit_profile_jit_handler;
4519 }
4520 #endif
4521 }
4522 }
4523
zend_jit_restart_preloaded_script(zend_persistent_script * script)4524 static void zend_jit_restart_preloaded_script(zend_persistent_script *script)
4525 {
4526 zend_class_entry *ce;
4527 zend_op_array *op_array;
4528
4529 zend_jit_restart_preloaded_op_array(&script->script.main_op_array);
4530
4531 ZEND_HASH_FOREACH_PTR(&script->script.function_table, op_array) {
4532 zend_jit_restart_preloaded_op_array(op_array);
4533 } ZEND_HASH_FOREACH_END();
4534
4535 ZEND_HASH_FOREACH_PTR(&script->script.class_table, ce) {
4536 ZEND_HASH_FOREACH_PTR(&ce->function_table, op_array) {
4537 if (op_array->type == ZEND_USER_FUNCTION) {
4538 zend_jit_restart_preloaded_op_array(op_array);
4539 }
4540 } ZEND_HASH_FOREACH_END();
4541 } ZEND_HASH_FOREACH_END();
4542 }
4543
zend_jit_restart(void)4544 ZEND_EXT_API void zend_jit_restart(void)
4545 {
4546 if (dasm_buf) {
4547 zend_jit_unprotect();
4548
4549 /* restore JIT buffer pos */
4550 dasm_ptr[0] = dasm_ptr[1];
4551
4552 zend_jit_trace_restart();
4553
4554 if (ZCSG(preload_script)) {
4555 zend_jit_restart_preloaded_script(ZCSG(preload_script));
4556 if (ZCSG(saved_scripts)) {
4557 zend_persistent_script **p = ZCSG(saved_scripts);
4558
4559 while (*p) {
4560 zend_jit_restart_preloaded_script(*p);
4561 p++;
4562 }
4563 }
4564 }
4565
4566 zend_jit_protect();
4567
4568 #ifdef HAVE_DISASM
4569 if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
4570 zend_jit_disasm_shutdown();
4571 zend_jit_disasm_init();
4572 }
4573 #endif
4574 }
4575 }
4576
4577 #endif /* HAVE_JIT */
4578
