1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 #include "main/php.h"
20 #include "main/SAPI.h"
21 #include "php_version.h"
22 #include <ZendAccelerator.h>
23 #include "zend_shared_alloc.h"
24 #include "Zend/zend_execute.h"
25 #include "Zend/zend_vm.h"
26 #include "Zend/zend_exceptions.h"
27 #include "Zend/zend_constants.h"
28 #include "Zend/zend_closures.h"
29 #include "Zend/zend_ini.h"
30 #include "Zend/zend_observer.h"
31 #include "zend_smart_str.h"
32 #include "jit/zend_jit.h"
33
34 #ifdef HAVE_JIT
35
36 #include "Optimizer/zend_func_info.h"
37 #include "Optimizer/zend_ssa.h"
38 #include "Optimizer/zend_inference.h"
39 #include "Optimizer/zend_call_graph.h"
40 #include "Optimizer/zend_dump.h"
41 #include "Optimizer/zend_worklist.h"
42
43 #include "jit/zend_jit_internal.h"
44
45 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
46 #include <pthread.h>
47 #endif
48
49 #ifdef ZTS
50 int jit_globals_id;
51 #else
52 zend_jit_globals jit_globals;
53 #endif
54
55 //#define CONTEXT_THREADED_JIT
56 #define ZEND_JIT_USE_RC_INFERENCE
57
58 #ifdef ZEND_JIT_USE_RC_INFERENCE
59 # define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE
60 # define RC_MAY_BE_1(info) (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0)
61 # define RC_MAY_BE_N(info) (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0)
62 #else
63 # define ZEND_SSA_RC_INFERENCE_FLAG 0
64 # define RC_MAY_BE_1(info) 1
65 # define RC_MAY_BE_N(info) 1
66 #endif
67
68 #define JIT_PREFIX "JIT$"
69 #define JIT_STUB_PREFIX "JIT$$"
70 #define TRACE_PREFIX "TRACE-"
71
72 bool zend_jit_startup_ok = false;
73
74 zend_ulong zend_jit_profile_counter = 0;
75 int zend_jit_profile_counter_rid = -1;
76
77 int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
78
79 const zend_op *zend_jit_halt_op = NULL;
80 static int zend_jit_vm_kind = 0;
81 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
82 static int zend_write_protect = 1;
83 #endif
84
85 static void *dasm_buf = NULL;
86 static void *dasm_end = NULL;
87 static void **dasm_ptr = NULL;
88
89 static size_t dasm_size = 0;
90
91 static zend_long jit_bisect_pos = 0;
92
93 static const void *zend_jit_runtime_jit_handler = NULL;
94 static const void *zend_jit_profile_jit_handler = NULL;
95 static const void *zend_jit_func_hot_counter_handler = NULL;
96 static const void *zend_jit_loop_hot_counter_handler = NULL;
97 static const void *zend_jit_func_trace_counter_handler = NULL;
98 static const void *zend_jit_ret_trace_counter_handler = NULL;
99 static const void *zend_jit_loop_trace_counter_handler = NULL;
100
101 static int ZEND_FASTCALL zend_runtime_jit(void);
102
103 static int zend_jit_trace_op_len(const zend_op *opline);
104 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
105 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags);
106 static const void *zend_jit_trace_get_exit_addr(uint32_t n);
107 static void zend_jit_trace_add_code(const void *start, uint32_t size);
108 static zend_string *zend_jit_func_name(const zend_op_array *op_array);
109
110 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info);
111 static bool zend_jit_supported_binary_op(uint8_t op, uint32_t op1_info, uint32_t op2_info);
112
dominates(const zend_basic_block * blocks,int a,int b)113 static bool dominates(const zend_basic_block *blocks, int a, int b) {
114 while (blocks[b].level > blocks[a].level) {
115 b = blocks[b].idom;
116 }
117 return a == b;
118 }
119
zend_ssa_is_last_use(const zend_op_array * op_array,const zend_ssa * ssa,int var,int use)120 static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
121 {
122 int next_use;
123
124 if (ssa->vars[var].phi_use_chain) {
125 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
126 do {
127 if (!ssa->vars[phi->ssa_var].no_val) {
128 return 0;
129 }
130 phi = zend_ssa_next_use_phi(ssa, var, phi);
131 } while (phi);
132 }
133
134 if (ssa->cfg.blocks[ssa->cfg.map[use]].loop_header > 0
135 || (ssa->cfg.blocks[ssa->cfg.map[use]].flags & ZEND_BB_LOOP_HEADER)) {
136 int b = ssa->cfg.map[use];
137 int prev_use = ssa->vars[var].use_chain;
138 int def_block;
139
140 if (ssa->vars[var].definition >= 0) {
141 def_block =ssa->cfg.map[ssa->vars[var].definition];
142 } else {
143 ZEND_ASSERT(ssa->vars[var].definition_phi);
144 def_block = ssa->vars[var].definition_phi->block;
145 }
146 if (dominates(ssa->cfg.blocks, def_block,
147 (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) ? b : ssa->cfg.blocks[b].loop_header)) {
148 return 0;
149 }
150
151 while (prev_use >= 0 && prev_use != use) {
152 if (b != ssa->cfg.map[prev_use]
153 && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use])
154 && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) {
155 return 0;
156 }
157 prev_use = zend_ssa_next_use(ssa->ops, var, prev_use);
158 }
159 }
160
161 next_use = zend_ssa_next_use(ssa->ops, var, use);
162 if (next_use < 0) {
163 return 1;
164 } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) {
165 return 1;
166 }
167 return 0;
168 }
169
zend_jit_is_constant_cmp_long_long(const zend_op * opline,zend_ssa_range * op1_range,zend_jit_addr op1_addr,zend_ssa_range * op2_range,zend_jit_addr op2_addr,bool * result)170 static int zend_jit_is_constant_cmp_long_long(const zend_op *opline,
171 zend_ssa_range *op1_range,
172 zend_jit_addr op1_addr,
173 zend_ssa_range *op2_range,
174 zend_jit_addr op2_addr,
175 bool *result)
176 {
177 zend_long op1_min;
178 zend_long op1_max;
179 zend_long op2_min;
180 zend_long op2_max;
181
182 if (op1_range) {
183 op1_min = op1_range->min;
184 op1_max = op1_range->max;
185 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
186 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG);
187 op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr));
188 } else {
189 return 0;
190 }
191
192 if (op2_range) {
193 op2_min = op2_range->min;
194 op2_max = op2_range->max;
195 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
196 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG);
197 op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr));
198 } else {
199 return 0;
200 }
201
202 switch (opline->opcode) {
203 case ZEND_IS_EQUAL:
204 case ZEND_IS_IDENTICAL:
205 case ZEND_CASE:
206 case ZEND_CASE_STRICT:
207 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
208 *result = 1;
209 return 1;
210 } else if (op1_max < op2_min || op1_min > op2_max) {
211 *result = 0;
212 return 1;
213 }
214 return 0;
215 case ZEND_IS_NOT_EQUAL:
216 case ZEND_IS_NOT_IDENTICAL:
217 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
218 *result = 0;
219 return 1;
220 } else if (op1_max < op2_min || op1_min > op2_max) {
221 *result = 1;
222 return 1;
223 }
224 return 0;
225 case ZEND_IS_SMALLER:
226 if (op1_max < op2_min) {
227 *result = 1;
228 return 1;
229 } else if (op1_min >= op2_max) {
230 *result = 0;
231 return 1;
232 }
233 return 0;
234 case ZEND_IS_SMALLER_OR_EQUAL:
235 if (op1_max <= op2_min) {
236 *result = 1;
237 return 1;
238 } else if (op1_min > op2_max) {
239 *result = 0;
240 return 1;
241 }
242 return 0;
243 default:
244 ZEND_UNREACHABLE();
245 }
246 return 0;
247 }
248
249 #define ADVANCE_SSA_OP(ssa_op, offset) \
250 do { \
251 if (ssa_op) ssa_op += offset; \
252 } while (0)
253
zend_jit_needs_call_chain(zend_call_info * call_info,uint32_t b,const zend_op_array * op_array,zend_ssa * ssa,const zend_ssa_op * ssa_op,const zend_op * opline,int call_level,zend_jit_trace_rec * trace)254 static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace)
255 {
256 int skip;
257
258 if (trace) {
259 zend_jit_trace_rec *p = trace;
260
261 ADVANCE_SSA_OP(ssa_op, 1);
262 while (1) {
263 if (p->op == ZEND_JIT_TRACE_VM) {
264 switch (p->opline->opcode) {
265 case ZEND_SEND_ARRAY:
266 case ZEND_SEND_USER:
267 case ZEND_SEND_UNPACK:
268 case ZEND_INIT_FCALL:
269 case ZEND_INIT_METHOD_CALL:
270 case ZEND_INIT_STATIC_METHOD_CALL:
271 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
272 case ZEND_INIT_FCALL_BY_NAME:
273 case ZEND_INIT_NS_FCALL_BY_NAME:
274 case ZEND_INIT_DYNAMIC_CALL:
275 case ZEND_NEW:
276 case ZEND_INIT_USER_CALL:
277 case ZEND_FAST_CALL:
278 case ZEND_JMP:
279 case ZEND_JMPZ:
280 case ZEND_JMPNZ:
281 case ZEND_JMPZ_EX:
282 case ZEND_JMPNZ_EX:
283 case ZEND_FE_RESET_R:
284 case ZEND_FE_RESET_RW:
285 case ZEND_JMP_SET:
286 case ZEND_COALESCE:
287 case ZEND_JMP_NULL:
288 case ZEND_ASSERT_CHECK:
289 case ZEND_CATCH:
290 case ZEND_DECLARE_ANON_CLASS:
291 case ZEND_FE_FETCH_R:
292 case ZEND_FE_FETCH_RW:
293 case ZEND_BIND_INIT_STATIC_OR_JMP:
294 case ZEND_JMP_FRAMELESS:
295 return 1;
296 case ZEND_DO_ICALL:
297 case ZEND_DO_UCALL:
298 case ZEND_DO_FCALL_BY_NAME:
299 case ZEND_DO_FCALL:
300 case ZEND_CALLABLE_CONVERT:
301 return 0;
302 case ZEND_SEND_VAL:
303 case ZEND_SEND_VAR:
304 case ZEND_SEND_VAL_EX:
305 case ZEND_SEND_VAR_EX:
306 case ZEND_SEND_FUNC_ARG:
307 case ZEND_SEND_REF:
308 case ZEND_SEND_VAR_NO_REF:
309 case ZEND_SEND_VAR_NO_REF_EX:
310 /* skip */
311 break;
312 default:
313 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
314 return 1;
315 }
316 }
317 ADVANCE_SSA_OP(ssa_op, zend_jit_trace_op_len(opline));
318 } else if (p->op == ZEND_JIT_TRACE_ENTER ||
319 p->op == ZEND_JIT_TRACE_BACK ||
320 p->op == ZEND_JIT_TRACE_END) {
321 return 1;
322 }
323 p++;
324 }
325 }
326
327 if (!call_info) {
328 const zend_op *end = op_array->opcodes + op_array->last;
329
330 opline++;
331 ADVANCE_SSA_OP(ssa_op, 1);
332 skip = (call_level == 1);
333 while (opline != end) {
334 if (!skip) {
335 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
336 return 1;
337 }
338 }
339 switch (opline->opcode) {
340 case ZEND_SEND_VAL:
341 case ZEND_SEND_VAR:
342 case ZEND_SEND_VAL_EX:
343 case ZEND_SEND_VAR_EX:
344 case ZEND_SEND_FUNC_ARG:
345 case ZEND_SEND_REF:
346 case ZEND_SEND_VAR_NO_REF:
347 case ZEND_SEND_VAR_NO_REF_EX:
348 skip = 0;
349 break;
350 case ZEND_SEND_ARRAY:
351 case ZEND_SEND_USER:
352 case ZEND_SEND_UNPACK:
353 case ZEND_INIT_FCALL:
354 case ZEND_INIT_METHOD_CALL:
355 case ZEND_INIT_STATIC_METHOD_CALL:
356 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
357 case ZEND_INIT_FCALL_BY_NAME:
358 case ZEND_INIT_NS_FCALL_BY_NAME:
359 case ZEND_INIT_DYNAMIC_CALL:
360 case ZEND_NEW:
361 case ZEND_INIT_USER_CALL:
362 case ZEND_FAST_CALL:
363 case ZEND_JMP:
364 case ZEND_JMPZ:
365 case ZEND_JMPNZ:
366 case ZEND_JMPZ_EX:
367 case ZEND_JMPNZ_EX:
368 case ZEND_FE_RESET_R:
369 case ZEND_FE_RESET_RW:
370 case ZEND_JMP_SET:
371 case ZEND_COALESCE:
372 case ZEND_JMP_NULL:
373 case ZEND_ASSERT_CHECK:
374 case ZEND_CATCH:
375 case ZEND_DECLARE_ANON_CLASS:
376 case ZEND_FE_FETCH_R:
377 case ZEND_FE_FETCH_RW:
378 case ZEND_BIND_INIT_STATIC_OR_JMP:
379 case ZEND_JMP_FRAMELESS:
380 return 1;
381 case ZEND_DO_ICALL:
382 case ZEND_DO_UCALL:
383 case ZEND_DO_FCALL_BY_NAME:
384 case ZEND_DO_FCALL:
385 case ZEND_CALLABLE_CONVERT:
386 end = opline;
387 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
388 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
389 return 1;
390 }
391 return 0;
392 }
393 opline++;
394 ADVANCE_SSA_OP(ssa_op, 1);
395 }
396
397 return 1;
398 } else {
399 const zend_op *end = call_info->caller_call_opline;
400
401 /* end may be null if an opcode like EXIT is part of the argument list. */
402 if (!end || end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
403 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
404 return 1;
405 }
406
407 opline++;
408 ADVANCE_SSA_OP(ssa_op, 1);
409 skip = (call_level == 1);
410 while (opline != end) {
411 if (skip) {
412 switch (opline->opcode) {
413 case ZEND_SEND_VAL:
414 case ZEND_SEND_VAR:
415 case ZEND_SEND_VAL_EX:
416 case ZEND_SEND_VAR_EX:
417 case ZEND_SEND_FUNC_ARG:
418 case ZEND_SEND_REF:
419 case ZEND_SEND_VAR_NO_REF:
420 case ZEND_SEND_VAR_NO_REF_EX:
421 skip = 0;
422 break;
423 case ZEND_SEND_ARRAY:
424 case ZEND_SEND_USER:
425 case ZEND_SEND_UNPACK:
426 return 1;
427 }
428 } else {
429 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
430 return 1;
431 }
432 }
433 opline++;
434 ADVANCE_SSA_OP(ssa_op, 1);
435 }
436
437 return 0;
438 }
439 }
440
skip_valid_arguments(const zend_op_array * op_array,zend_ssa * ssa,const zend_call_info * call_info)441 static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
442 {
443 uint32_t num_args = 0;
444 zend_function *func = call_info->callee_func;
445
446 /* It's okay to handle prototypes here, because they can only increase the accepted arguments.
447 * Anything legal for the parent method is also legal for the parent method. */
448 while (num_args < call_info->num_args) {
449 zend_arg_info *arg_info = func->op_array.arg_info + num_args;
450
451 if (ZEND_TYPE_IS_SET(arg_info->type)) {
452 if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) {
453 zend_op *opline = call_info->arg_info[num_args].opline;
454 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[opline - op_array->opcodes] : NULL;
455 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
456 if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) {
457 break;
458 }
459 } else {
460 break;
461 }
462 }
463 num_args++;
464 }
465 return num_args;
466 }
467
zend_ssa_cv_info(const zend_op_array * op_array,zend_ssa * ssa,uint32_t var)468 static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var)
469 {
470 uint32_t j, info;
471
472 if (ssa->vars && ssa->var_info) {
473 info = ssa->var_info[var].type;
474 for (j = op_array->last_var; j < ssa->vars_count; j++) {
475 if (ssa->vars[j].var == var) {
476 info |= ssa->var_info[j].type;
477 }
478 }
479 } else {
480 info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF |
481 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
482 }
483
484 #ifdef ZEND_JIT_USE_RC_INFERENCE
485 /* Refcount may be increased by RETURN opcode */
486 if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
487 for (j = 0; j < ssa->cfg.blocks_count; j++) {
488 if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
489 ssa->cfg.blocks[j].len > 0) {
490 const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
491
492 if (opline->opcode == ZEND_RETURN) {
493 if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) {
494 info |= MAY_BE_RCN;
495 break;
496 }
497 }
498 }
499 }
500 }
501 #endif
502
503 return info;
504 }
505
zend_jit_may_avoid_refcounting(const zend_op * opline,uint32_t op1_info)506 static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_info)
507 {
508 switch (opline->opcode) {
509 case ZEND_FETCH_OBJ_FUNC_ARG:
510 if (!JIT_G(current_frame) ||
511 !JIT_G(current_frame)->call->func ||
512 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
513 return 0;
514 }
515 /* break missing intentionally */
516 case ZEND_FETCH_OBJ_R:
517 case ZEND_FETCH_OBJ_IS:
518 if ((op1_info & MAY_BE_OBJECT)
519 && opline->op2_type == IS_CONST
520 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
521 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
522 return 1;
523 }
524 break;
525 case ZEND_FETCH_DIM_FUNC_ARG:
526 if (!JIT_G(current_frame) ||
527 !JIT_G(current_frame)->call->func ||
528 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
529 return 0;
530 }
531 /* break missing intentionally */
532 case ZEND_FETCH_DIM_R:
533 case ZEND_FETCH_DIM_IS:
534 return 1;
535 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
536 if (!(opline->extended_value & ZEND_ISEMPTY)) {
537 return 1;
538 }
539 break;
540 }
541 return 0;
542 }
543
zend_jit_is_persistent_constant(zval * key,uint32_t flags)544 static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags)
545 {
546 zval *zv;
547 zend_constant *c = NULL;
548
549 /* null/true/false are resolved during compilation, so don't check for them here. */
550 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
551 if (zv) {
552 c = (zend_constant*)Z_PTR_P(zv);
553 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
554 key++;
555 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
556 if (zv) {
557 c = (zend_constant*)Z_PTR_P(zv);
558 }
559 }
560 return c && (ZEND_CONSTANT_FLAGS(c) & CONST_PERSISTENT);
561 }
562
zend_get_known_class(const zend_op_array * op_array,const zend_op * opline,uint8_t op_type,znode_op op)563 static zend_class_entry* zend_get_known_class(const zend_op_array *op_array, const zend_op *opline, uint8_t op_type, znode_op op)
564 {
565 zend_class_entry *ce = NULL;
566
567 if (op_type == IS_CONST) {
568 zval *zv = RT_CONSTANT(opline, op);
569 zend_string *class_name;
570
571 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
572 class_name = Z_STR_P(zv);
573 ce = zend_lookup_class_ex(class_name, NULL, ZEND_FETCH_CLASS_NO_AUTOLOAD);
574 if (ce && (ce->type == ZEND_INTERNAL_CLASS || ce->info.user.filename != op_array->filename)) {
575 ce = NULL;
576 }
577 } else {
578 ZEND_ASSERT(op_type == IS_UNUSED);
579 if ((op.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_SELF) {
580 ce = op_array->scope;
581 } else {
582 ZEND_ASSERT((op.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_PARENT);
583 ce = op_array->scope;
584 if (ce) {
585 if (ce->parent) {
586 ce = ce->parent;
587 if (ce->type == ZEND_INTERNAL_CLASS || ce->info.user.filename != op_array->filename) {
588 ce = NULL;
589 }
590 } else {
591 ce = NULL;
592 }
593 }
594 }
595 }
596
597 return ce;
598 }
599
zend_get_known_property_info(const zend_op_array * op_array,zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)600 static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
601 {
602 zend_property_info *info = NULL;
603
604 if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) ||
605 !ce ||
606 !(ce->ce_flags & ZEND_ACC_LINKED) ||
607 (ce->ce_flags & ZEND_ACC_TRAIT) ||
608 ce->create_object) {
609 return NULL;
610 }
611
612 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
613 if (ce->info.user.filename != filename) {
614 /* class declaration might be changed independently */
615 return NULL;
616 }
617
618 if (ce->parent) {
619 zend_class_entry *parent = ce->parent;
620
621 do {
622 if (parent->type == ZEND_INTERNAL_CLASS) {
623 break;
624 } else if (parent->info.user.filename != filename) {
625 /* some of parents class declarations might be changed independently */
626 /* TODO: this check may be not enough, because even
627 * in the same it's possible to conditionally define
628 * few classes with the same name, and "parent" may
629 * change from request to request.
630 */
631 return NULL;
632 }
633 parent = parent->parent;
634 } while (parent);
635 }
636 }
637
638 // TODO: Treat property hooks more precisely.
639 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
640 if (info == NULL ||
641 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
642 (info->flags & ZEND_ACC_STATIC) ||
643 info->hooks) {
644 return NULL;
645 }
646
647 if (info->flags & ZEND_ACC_PUBLIC) {
648 return info;
649 } else if (on_this) {
650 if (ce == info->ce) {
651 if (ce == op_array->scope) {
652 return info;
653 } else {
654 return NULL;
655 }
656 } else if ((info->flags & ZEND_ACC_PROTECTED)
657 && instanceof_function_slow(ce, info->ce)) {
658 return info;
659 }
660 }
661
662 return NULL;
663 }
664
zend_may_be_dynamic_property(zend_class_entry * ce,zend_string * member,bool on_this,const zend_op_array * op_array)665 static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, const zend_op_array *op_array)
666 {
667 zend_property_info *info;
668
669 if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT) || (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) {
670 return 1;
671 }
672
673 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
674 if (ce->info.user.filename != op_array->filename) {
675 /* class declaration might be changed independently */
676 return 1;
677 }
678 }
679
680 // TODO: Treat property hooks more precisely.
681 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
682 if (info == NULL ||
683 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
684 (info->flags & ZEND_ACC_STATIC) ||
685 info->hooks) {
686 return 1;
687 }
688
689 if (!(info->flags & ZEND_ACC_PUBLIC) &&
690 (!on_this || info->ce != ce)) {
691 return 1;
692 }
693
694 return 0;
695 }
696
zend_jit_class_may_be_modified(const zend_class_entry * ce,const zend_op_array * called_from)697 static bool zend_jit_class_may_be_modified(const zend_class_entry *ce, const zend_op_array *called_from)
698 {
699 uint32_t i;
700
701 if (ce->type == ZEND_INTERNAL_CLASS) {
702 #ifdef _WIN32
703 /* ASLR */
704 return 1;
705 #else
706 return 0;
707 #endif
708 } else if (ce->type == ZEND_USER_CLASS) {
709 if (ce->ce_flags & ZEND_ACC_PRELOADED) {
710 return 0;
711 }
712 if (ce->info.user.filename == called_from->filename) {
713 if (ce->parent
714 && (!(ce->ce_flags & ZEND_ACC_LINKED)
715 || zend_jit_class_may_be_modified(ce->parent, called_from))) {
716 return 1;
717 }
718 if (ce->num_interfaces) {
719 if (!(ce->ce_flags & ZEND_ACC_LINKED)) {
720 return 1;
721 }
722 for (i = 0; i < ce->num_interfaces; i++) {
723 if (zend_jit_class_may_be_modified(ce->interfaces[i], called_from)) {
724 return 1;
725 }
726 }
727 }
728 if (ce->num_traits) {
729 if (!(ce->ce_flags & ZEND_ACC_LINKED)) {
730 return 1;
731 }
732 for (i=0; i < ce->num_traits; i++) {
733 zend_class_entry *trait = zend_fetch_class_by_name(ce->trait_names[i].name,
734 ce->trait_names[i].lc_name,
735 ZEND_FETCH_CLASS_TRAIT | ZEND_FETCH_CLASS_NO_AUTOLOAD | ZEND_FETCH_CLASS_SILENT);
736 if (!trait || zend_jit_class_may_be_modified(trait, called_from)) {
737 return 1;
738 }
739 }
740 }
741 return 0;
742 }
743 }
744 return 1;
745 }
746
zend_jit_may_be_modified(const zend_function * func,const zend_op_array * called_from)747 static bool zend_jit_may_be_modified(const zend_function *func, const zend_op_array *called_from)
748 {
749 if (func->type == ZEND_INTERNAL_FUNCTION) {
750 #ifdef _WIN32
751 /* ASLR */
752 return 1;
753 #else
754 return 0;
755 #endif
756 } else if (func->type == ZEND_USER_FUNCTION) {
757 if (func->common.fn_flags & ZEND_ACC_PRELOADED) {
758 return 0;
759 }
760 if (func->op_array.filename == called_from->filename
761 && (!func->op_array.scope
762 || !zend_jit_class_may_be_modified(func->op_array.scope, called_from))) {
763 return 0;
764 }
765 }
766 return 1;
767 }
768
769 #define OP_RANGE(ssa_op, opN) \
770 (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
771 ssa->var_info && \
772 (ssa_op)->opN##_use >= 0 && \
773 ssa->var_info[(ssa_op)->opN##_use].has_range) ? \
774 &ssa->var_info[(ssa_op)->opN##_use].range : NULL)
775
776 #define OP1_RANGE() OP_RANGE(ssa_op, op1)
777 #define OP2_RANGE() OP_RANGE(ssa_op, op2)
778 #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)
779
780 #include "jit/zend_jit_helpers.c"
781 #include "Zend/zend_cpuinfo.h"
782
783 #ifdef HAVE_GCC_GLOBAL_REGS
784 # define GCC_GLOBAL_REGS 1
785 #else
786 # define GCC_GLOBAL_REGS 0
787 #endif
788
789 /* By default avoid JITing inline handlers if it does not seem profitable due to lack of
790 * type information. Disabling this option allows testing some JIT handlers in the
791 * presence of try/catch blocks, which prevent SSA construction. */
792 #ifndef PROFITABILITY_CHECKS
793 # define PROFITABILITY_CHECKS 1
794 #endif
795
796 #define BP_JIT_IS 6 /* Used for ISSET_ISEMPTY_DIM_OBJ. see BP_VAR_*defines in Zend/zend_compile.h */
797
798 /* The generated code may contain tautological comparisons, ignore them. */
799 #if defined(__clang__)
800 # pragma clang diagnostic push
801 # pragma clang diagnostic ignored "-Wtautological-compare"
802 # pragma clang diagnostic ignored "-Wstring-compare"
803 #endif
804
805 #include "jit/zend_jit_ir.c"
806
807 #if defined(__clang__)
808 # pragma clang diagnostic pop
809 #endif
810
811 #ifdef _WIN32
812 # include <Windows.h>
813 #else
814 # include <sys/mman.h>
815 # if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
816 # define MAP_ANONYMOUS MAP_ANON
817 # endif
818 #endif
819
zend_jit_status(zval * ret)820 ZEND_EXT_API void zend_jit_status(zval *ret)
821 {
822 zval stats;
823 array_init(&stats);
824 add_assoc_bool(&stats, "enabled", JIT_G(enabled));
825 add_assoc_bool(&stats, "on", JIT_G(on));
826 add_assoc_long(&stats, "kind", JIT_G(trigger));
827 add_assoc_long(&stats, "opt_level", JIT_G(opt_level));
828 add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags));
829 if (dasm_buf) {
830 add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
831 add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
832 } else {
833 add_assoc_long(&stats, "buffer_size", 0);
834 add_assoc_long(&stats, "buffer_free", 0);
835 }
836 add_assoc_zval(ret, "jit", &stats);
837 }
838
zend_jit_func_name(const zend_op_array * op_array)839 static zend_string *zend_jit_func_name(const zend_op_array *op_array)
840 {
841 smart_str buf = {0};
842
843 if (op_array->function_name) {
844 smart_str_appends(&buf, JIT_PREFIX);
845 if (op_array->scope) {
846 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
847 smart_str_appends(&buf, "::");
848 }
849 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
850 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
851 smart_str_appends(&buf, ":");
852 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
853 smart_str_appends(&buf, ":");
854 smart_str_append_long(&buf, op_array->line_start);
855 }
856 smart_str_0(&buf);
857 return buf.s;
858 } else if (op_array->filename) {
859 smart_str_appends(&buf, JIT_PREFIX);
860 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
861 smart_str_0(&buf);
862 return buf.s;
863 } else {
864 return NULL;
865 }
866 }
867
zend_may_overflow(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa)868 static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
869 {
870 int res;
871 zend_long op1_min, op1_max, op2_min, op2_max;
872
873 if (!ssa->ops || !ssa->var_info) {
874 return 1;
875 }
876 switch (opline->opcode) {
877 case ZEND_PRE_INC:
878 case ZEND_POST_INC:
879 res = ssa_op->op1_def;
880 if (res < 0
881 || !ssa->var_info[res].has_range
882 || ssa->var_info[res].range.overflow) {
883 if (!OP1_HAS_RANGE()) {
884 return 1;
885 }
886 op1_max = OP1_MAX_RANGE();
887 if (op1_max == ZEND_LONG_MAX) {
888 return 1;
889 }
890 }
891 return 0;
892 case ZEND_PRE_DEC:
893 case ZEND_POST_DEC:
894 res = ssa_op->op1_def;
895 if (res < 0
896 || !ssa->var_info[res].has_range
897 || ssa->var_info[res].range.underflow) {
898 if (!OP1_HAS_RANGE()) {
899 return 1;
900 }
901 op1_min = OP1_MIN_RANGE();
902 if (op1_min == ZEND_LONG_MIN) {
903 return 1;
904 }
905 }
906 return 0;
907 case ZEND_ADD:
908 res = ssa_op->result_def;
909 if (res < 0
910 || !ssa->var_info[res].has_range
911 || ssa->var_info[res].range.underflow) {
912 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
913 return 1;
914 }
915 op1_min = OP1_MIN_RANGE();
916 op2_min = OP2_MIN_RANGE();
917 if (zend_add_will_overflow(op1_min, op2_min)) {
918 return 1;
919 }
920 }
921 if (res < 0
922 || !ssa->var_info[res].has_range
923 || ssa->var_info[res].range.overflow) {
924 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
925 return 1;
926 }
927 op1_max = OP1_MAX_RANGE();
928 op2_max = OP2_MAX_RANGE();
929 if (zend_add_will_overflow(op1_max, op2_max)) {
930 return 1;
931 }
932 }
933 return 0;
934 case ZEND_SUB:
935 res = ssa_op->result_def;
936 if (res < 0
937 || !ssa->var_info[res].has_range
938 || ssa->var_info[res].range.underflow) {
939 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
940 return 1;
941 }
942 op1_min = OP1_MIN_RANGE();
943 op2_max = OP2_MAX_RANGE();
944 if (zend_sub_will_overflow(op1_min, op2_max)) {
945 return 1;
946 }
947 }
948 if (res < 0
949 || !ssa->var_info[res].has_range
950 || ssa->var_info[res].range.overflow) {
951 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
952 return 1;
953 }
954 op1_max = OP1_MAX_RANGE();
955 op2_min = OP2_MIN_RANGE();
956 if (zend_sub_will_overflow(op1_max, op2_min)) {
957 return 1;
958 }
959 }
960 return 0;
961 case ZEND_MUL:
962 res = ssa_op->result_def;
963 return (res < 0 ||
964 !ssa->var_info[res].has_range ||
965 ssa->var_info[res].range.underflow ||
966 ssa->var_info[res].range.overflow);
967 case ZEND_ASSIGN_OP:
968 if (opline->extended_value == ZEND_ADD) {
969 res = ssa_op->op1_def;
970 if (res < 0
971 || !ssa->var_info[res].has_range
972 || ssa->var_info[res].range.underflow) {
973 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
974 return 1;
975 }
976 op1_min = OP1_MIN_RANGE();
977 op2_min = OP2_MIN_RANGE();
978 if (zend_add_will_overflow(op1_min, op2_min)) {
979 return 1;
980 }
981 }
982 if (res < 0
983 || !ssa->var_info[res].has_range
984 || ssa->var_info[res].range.overflow) {
985 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
986 return 1;
987 }
988 op1_max = OP1_MAX_RANGE();
989 op2_max = OP2_MAX_RANGE();
990 if (zend_add_will_overflow(op1_max, op2_max)) {
991 return 1;
992 }
993 }
994 return 0;
995 } else if (opline->extended_value == ZEND_SUB) {
996 res = ssa_op->op1_def;
997 if (res < 0
998 || !ssa->var_info[res].has_range
999 || ssa->var_info[res].range.underflow) {
1000 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1001 return 1;
1002 }
1003 op1_min = OP1_MIN_RANGE();
1004 op2_max = OP2_MAX_RANGE();
1005 if (zend_sub_will_overflow(op1_min, op2_max)) {
1006 return 1;
1007 }
1008 }
1009 if (res < 0
1010 || !ssa->var_info[res].has_range
1011 || ssa->var_info[res].range.overflow) {
1012 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
1013 return 1;
1014 }
1015 op1_max = OP1_MAX_RANGE();
1016 op2_min = OP2_MIN_RANGE();
1017 if (zend_sub_will_overflow(op1_max, op2_min)) {
1018 return 1;
1019 }
1020 }
1021 return 0;
1022 } else if (opline->extended_value == ZEND_MUL) {
1023 res = ssa_op->op1_def;
1024 return (res < 0 ||
1025 !ssa->var_info[res].has_range ||
1026 ssa->var_info[res].range.underflow ||
1027 ssa->var_info[res].range.overflow);
1028 }
1029 ZEND_FALLTHROUGH;
1030 default:
1031 return 1;
1032 }
1033 }
1034
zend_jit_build_cfg(const zend_op_array * op_array,zend_cfg * cfg)1035 static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg)
1036 {
1037 uint32_t flags;
1038
1039 flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY;
1040
1041 zend_build_cfg(&CG(arena), op_array, flags, cfg);
1042
1043 /* Don't JIT huge functions. Apart from likely being detrimental due to the amount of
1044 * generated code, some of our analysis is recursive and will stack overflow with many
1045 * blocks. */
1046 if (cfg->blocks_count > 100000) {
1047 return FAILURE;
1048 }
1049
1050 zend_cfg_build_predecessors(&CG(arena), cfg);
1051
1052 /* Compute Dominators Tree */
1053 zend_cfg_compute_dominators_tree(op_array, cfg);
1054
1055 /* Identify reducible and irreducible loops */
1056 zend_cfg_identify_loops(op_array, cfg);
1057
1058 return SUCCESS;
1059 }
1060
zend_jit_op_array_analyze1(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa)1061 static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
1062 {
1063 if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) {
1064 return FAILURE;
1065 }
1066
1067 #if 0
1068 /* TODO: debugger and profiler supports? */
1069 if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) {
1070 return FAILURE;
1071 }
1072 #endif
1073
1074 /* TODO: move this to zend_cfg.c ? */
1075 if (!op_array->function_name) {
1076 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
1077 }
1078
1079 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
1080 && ssa->cfg.blocks
1081 && op_array->last_try_catch == 0
1082 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
1083 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
1084 if (zend_build_ssa(&CG(arena), script, op_array, ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) {
1085 return FAILURE;
1086 }
1087
1088 zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa);
1089
1090 zend_ssa_find_false_dependencies(op_array, ssa);
1091
1092 zend_ssa_find_sccs(op_array, ssa);
1093 }
1094
1095 return SUCCESS;
1096 }
1097
zend_jit_op_array_analyze2(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa,uint32_t optimization_level)1098 static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t optimization_level)
1099 {
1100 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
1101 && ssa->cfg.blocks
1102 && op_array->last_try_catch == 0
1103 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
1104 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
1105 if (zend_ssa_inference(&CG(arena), op_array, script, ssa,
1106 optimization_level & ~ZEND_OPTIMIZER_NARROW_TO_DOUBLE) != SUCCESS) {
1107 return FAILURE;
1108 }
1109 }
1110
1111 return SUCCESS;
1112 }
1113
zend_jit_allocate_registers(zend_jit_ctx * ctx,const zend_op_array * op_array,zend_ssa * ssa)1114 static void zend_jit_allocate_registers(zend_jit_ctx *ctx, const zend_op_array *op_array, zend_ssa *ssa)
1115 {
1116 void *checkpoint;
1117 int candidates_count, i;
1118 zend_jit_reg_var *ra;
1119
1120 checkpoint = zend_arena_checkpoint(CG(arena));
1121 ra = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_jit_reg_var));
1122 candidates_count = 0;
1123 for (i = 0; i < ssa->vars_count; i++) {
1124 if (zend_jit_may_be_in_reg(op_array, ssa, i)) {
1125 ra[i].ref = IR_NULL;
1126 candidates_count++;
1127 }
1128 }
1129 if (!candidates_count) {
1130 zend_arena_release(&CG(arena), checkpoint);
1131 return;
1132 }
1133
1134 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
1135 /* Naive SSA resolution */
1136 for (i = 0; i < ssa->vars_count; i++) {
1137 if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
1138 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
1139 int k, src;
1140
1141 if (phi->pi >= 0) {
1142 src = phi->sources[0];
1143 if (ra[i].ref) {
1144 if (!ra[src].ref) {
1145 ra[i].flags |= ZREG_LOAD;
1146 } else {
1147 ra[i].flags |= ZREG_PI;
1148 }
1149 } else if (ra[src].ref) {
1150 ra[src].flags |= ZREG_STORE;
1151 }
1152 } else {
1153 int need_move = 0;
1154
1155 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1156 src = phi->sources[k];
1157 if (src >= 0) {
1158 if (ssa->vars[src].definition_phi
1159 && ssa->vars[src].definition_phi->pi >= 0
1160 && phi->block == ssa->vars[src].definition_phi->block) {
1161 /* Skip zero-length interval for Pi variable */
1162 src = ssa->vars[src].definition_phi->sources[0];
1163 }
1164 if (ra[i].ref) {
1165 if (!ra[src].ref) {
1166 need_move = 1;
1167 }
1168 } else if (ra[src].ref) {
1169 need_move = 1;
1170 }
1171 }
1172 }
1173 if (need_move) {
1174 if (ra[i].ref) {
1175 ra[i].flags |= ZREG_LOAD;
1176 }
1177 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1178 src = phi->sources[k];
1179 if (src >= 0) {
1180 if (ssa->vars[src].definition_phi
1181 && ssa->vars[src].definition_phi->pi >= 0
1182 && phi->block == ssa->vars[src].definition_phi->block) {
1183 /* Skip zero-length interval for Pi variable */
1184 src = ssa->vars[src].definition_phi->sources[0];
1185 }
1186 if (ra[src].ref) {
1187 ra[src].flags |= ZREG_STORE;
1188 }
1189 }
1190 }
1191 } else {
1192 ra[i].flags |= ZREG_PHI;
1193 }
1194 }
1195 }
1196 }
1197
1198 /* Remove useless register allocation */
1199 for (i = 0; i < ssa->vars_count; i++) {
1200 if (ra[i].ref &&
1201 ((ra[i].flags & ZREG_LOAD) ||
1202 ((ra[i].flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) &&
1203 ssa->vars[i].use_chain < 0) {
1204 bool may_remove = 1;
1205 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
1206
1207 while (phi) {
1208 if (ra[phi->ssa_var].ref &&
1209 !(ra[phi->ssa_var].flags & ZREG_LOAD)) {
1210 may_remove = 0;
1211 break;
1212 }
1213 phi = zend_ssa_next_use_phi(ssa, i, phi);
1214 }
1215 if (may_remove) {
1216 ra[i].ref = IR_UNUSED;
1217 }
1218 }
1219 }
1220
1221 /* Remove intervals used once */
1222 for (i = 0; i < ssa->vars_count; i++) {
1223 if (ra[i].ref) {
1224 if (!(ra[i].flags & (ZREG_LOAD|ZREG_STORE))) {
1225 uint32_t var_num = ssa->vars[i].var;
1226 uint32_t op_num = ssa->vars[i].definition;
1227
1228 /* Check if a tempoary variable may be freed by exception handler */
1229 if (op_array->last_live_range
1230 && var_num >= op_array->last_var
1231 && ssa->vars[i].definition >= 0
1232 && ssa->ops[op_num].result_def == i) {
1233 const zend_live_range *range = op_array->live_range;
1234 int j;
1235
1236 op_num++;
1237 if (op_array->opcodes[op_num].opcode == ZEND_OP_DATA) {
1238 op_num++;
1239 }
1240 for (j = 0; j < op_array->last_live_range; range++, j++) {
1241 if (range->start > op_num) {
1242 /* further blocks will not be relevant... */
1243 break;
1244 } else if (op_num < range->end && var_num == (range->var & ~ZEND_LIVE_MASK)) {
1245 /* check if opcodes in range may throw */
1246 do {
1247 if (zend_may_throw(op_array->opcodes + op_num, ssa->ops + op_num, op_array, ssa)) {
1248 ra[i].flags |= ZREG_STORE;
1249 break;
1250 }
1251 op_num++;
1252 if (op_array->opcodes[op_num].opcode == ZEND_OP_DATA) {
1253 op_num++;
1254 }
1255 } while (op_num < range->end);
1256 break;
1257 }
1258 }
1259 }
1260 }
1261 if ((ra[i].flags & ZREG_LOAD)
1262 && (ra[i].flags & ZREG_STORE)
1263 && (ssa->vars[i].use_chain < 0
1264 || zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) {
1265 bool may_remove = 1;
1266 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
1267
1268 while (phi) {
1269 if (ra[phi->ssa_var].ref &&
1270 !(ra[phi->ssa_var].flags & ZREG_LOAD)) {
1271 may_remove = 0;
1272 break;
1273 }
1274 phi = zend_ssa_next_use_phi(ssa, i, phi);
1275 }
1276 if (may_remove) {
1277 ra[i].ref = IR_UNUSED;
1278 }
1279 }
1280 }
1281 }
1282 }
1283
1284 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
1285 fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
1286 for (i = 0; i < ssa->vars_count; i++) {
1287 if (ra[i].ref) {
1288 fprintf(stderr, "#%d.", i);
1289 uint32_t var_num = ssa->vars[i].var;
1290 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
1291 if (ra[i].flags & ZREG_LOAD) {
1292 fprintf(stderr, " load");
1293 }
1294 if (ra[i].flags & ZREG_STORE) {
1295 fprintf(stderr, " store");
1296 }
1297 fprintf(stderr, "\n");
1298 }
1299 }
1300 fprintf(stderr, "\n");
1301 }
1302
1303 ctx->ra = ra;
1304 }
1305
zend_jit_compute_post_order(zend_cfg * cfg,int start,int * post_order)1306 static int zend_jit_compute_post_order(zend_cfg *cfg, int start, int *post_order)
1307 {
1308 int count = 0;
1309 int b, n, *p;
1310 zend_basic_block *bb;
1311 zend_worklist worklist;
1312 ALLOCA_FLAG(use_heap)
1313
1314 ZEND_WORKLIST_ALLOCA(&worklist, cfg->blocks_count, use_heap);
1315 zend_worklist_push(&worklist, start);
1316
1317 while (zend_worklist_len(&worklist) != 0) {
1318 next:
1319 b = zend_worklist_peek(&worklist);
1320 bb = &cfg->blocks[b];
1321 n = bb->successors_count;
1322 if (n > 0) {
1323 p = bb->successors;
1324 do {
1325 if (cfg->blocks[*p].flags & (ZEND_BB_CATCH|ZEND_BB_FINALLY|ZEND_BB_FINALLY_END)) {
1326 /* skip */
1327 } else if (zend_worklist_push(&worklist, *p)) {
1328 goto next;
1329 }
1330 p++;
1331 n--;
1332 } while (n > 0);
1333 }
1334 zend_worklist_pop(&worklist);
1335 post_order[count++] = b;
1336 }
1337 ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
1338 return count;
1339 }
1340
zend_jit_next_is_send_result(const zend_op * opline)1341 static bool zend_jit_next_is_send_result(const zend_op *opline)
1342 {
1343 if (opline->result_type == IS_TMP_VAR
1344 && (opline+1)->opcode == ZEND_SEND_VAL
1345 && (opline+1)->op1_type == IS_TMP_VAR
1346 && (opline+1)->op2_type != IS_CONST
1347 && (opline+1)->op1.var == opline->result.var) {
1348 return 1;
1349 }
1350 return 0;
1351 }
1352
zend_jit_supported_binary_op(uint8_t op,uint32_t op1_info,uint32_t op2_info)1353 static bool zend_jit_supported_binary_op(uint8_t op, uint32_t op1_info, uint32_t op2_info)
1354 {
1355 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1356 return false;
1357 }
1358 switch (op) {
1359 case ZEND_POW:
1360 case ZEND_DIV:
1361 // TODO: check for division by zero ???
1362 return false;
1363 case ZEND_ADD:
1364 case ZEND_SUB:
1365 case ZEND_MUL:
1366 return (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
1367 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
1368 case ZEND_BW_OR:
1369 case ZEND_BW_AND:
1370 case ZEND_BW_XOR:
1371 case ZEND_SL:
1372 case ZEND_SR:
1373 case ZEND_MOD:
1374 return (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG);
1375 case ZEND_CONCAT:
1376 return (op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING);
1377 EMPTY_SWITCH_DEFAULT_CASE()
1378 }
1379 }
1380
zend_jit(const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline)1381 static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline)
1382 {
1383 int b, i, end;
1384 zend_op *opline;
1385 zend_jit_ctx ctx;
1386 zend_jit_ctx *jit = &ctx;
1387 zend_jit_reg_var *ra = NULL;
1388 void *handler;
1389 int call_level = 0;
1390 void *checkpoint = NULL;
1391 bool recv_emitted = 0; /* emitted at least one RECV opcode */
1392 uint8_t smart_branch_opcode;
1393 uint32_t target_label, target_label2;
1394 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_mem_info;
1395 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
1396 zend_class_entry *ce;
1397 bool ce_is_instanceof;
1398 bool on_this;
1399
1400 ZEND_ASSERT(!(op_array->fn_flags & ZEND_ACC_CLOSURE) || !(op_array->scope));
1401
1402 if (JIT_G(bisect_limit)) {
1403 jit_bisect_pos++;
1404 if (jit_bisect_pos >= JIT_G(bisect_limit)) {
1405 if (jit_bisect_pos == JIT_G(bisect_limit)) {
1406 fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n",
1407 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
1408 op_array->scope ? "::" : "",
1409 op_array->function_name ? ZSTR_VAL(op_array->function_name) : "{main}",
1410 ZSTR_VAL(op_array->filename), op_array->line_start);
1411 }
1412 return FAILURE;
1413 }
1414 }
1415
1416 if (ssa->cfg.flags & ZEND_FUNC_IRREDUCIBLE) {
1417 /* We can't order blocks properly */
1418 return FAILURE;
1419 }
1420
1421 if (rt_opline) {
1422 /* Set BB_ENTRY flag to limit register usage across the OSR ENTRY point */
1423 ssa->cfg.blocks[ssa->cfg.map[rt_opline - op_array->opcodes]].flags |= ZEND_BB_ENTRY;
1424 }
1425
1426 zend_jit_start(&ctx, op_array, ssa);
1427 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1428 checkpoint = zend_arena_checkpoint(CG(arena));
1429 zend_jit_allocate_registers(&ctx, op_array, ssa);
1430 ra = ctx.ra;
1431 }
1432
1433 /* Process blocks in Reverse Post Order */
1434 int *sorted_blocks = alloca(sizeof(int) * ssa->cfg.blocks_count);
1435 int n = zend_jit_compute_post_order(&ssa->cfg, 0, sorted_blocks);
1436
1437 while (n > 0) {
1438 b = sorted_blocks[--n];
1439 if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) {
1440 continue;
1441 }
1442
1443 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
1444 opline = op_array->opcodes + ssa->cfg.blocks[b].start;
1445 if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) {
1446 if (opline->opcode == ZEND_RECV_INIT) {
1447 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1448 if (opline != op_array->opcodes && (opline-1)->opcode != ZEND_RECV_INIT) {
1449 zend_jit_recv_entry(&ctx, b);
1450 }
1451 } else {
1452 if (opline != op_array->opcodes && recv_emitted) {
1453 zend_jit_recv_entry(&ctx, b);
1454 }
1455 }
1456 recv_emitted = 1;
1457 } else if (opline->opcode == ZEND_RECV) {
1458 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
1459 /* skip */
1460 zend_jit_bb_start(&ctx, b);
1461 zend_jit_bb_end(&ctx, b);
1462 continue;
1463 } else if (recv_emitted) {
1464 zend_jit_recv_entry(&ctx, b);
1465 } else {
1466 recv_emitted = 1;
1467 }
1468 } else {
1469 if (recv_emitted) {
1470 zend_jit_recv_entry(&ctx, b);
1471 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
1472 ssa->cfg.blocks[b].len == 1 &&
1473 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
1474 /* don't generate code for BB with single opcode */
1475 zend_jit_free_ctx(&ctx);
1476
1477 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1478 zend_arena_release(&CG(arena), checkpoint);
1479 }
1480 return SUCCESS;
1481 }
1482 }
1483 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
1484 ssa->cfg.blocks[b].len == 1 &&
1485 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
1486 /* don't generate code for BB with single opcode */
1487 zend_jit_free_ctx(&ctx);
1488
1489 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1490 zend_arena_release(&CG(arena), checkpoint);
1491 }
1492 return SUCCESS;
1493 }
1494 }
1495
1496 zend_jit_bb_start(&ctx, b);
1497
1498 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ctx.ra) {
1499 zend_ssa_phi *phi = ssa->blocks[b].phis;
1500
1501 /* First try to insert IR Phi */
1502 while (phi) {
1503 zend_jit_reg_var *ival = &ctx.ra[phi->ssa_var];
1504
1505 if (ival->ref) {
1506 if (ival->flags & ZREG_PI) {
1507 zend_jit_gen_pi(jit, phi);
1508 } else if (ival->flags & ZREG_PHI) {
1509 zend_jit_gen_phi(jit, phi);
1510 }
1511 }
1512 phi = phi->next;
1513 }
1514 }
1515
1516 if (rt_opline
1517 && (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) == 0
1518 && rt_opline == op_array->opcodes + ssa->cfg.blocks[b].start) {
1519 zend_jit_osr_entry(&ctx, b); /* OSR (On-Stack-Replacement) Entry-Point */
1520 }
1521
1522 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1523 if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
1524 && ssa->cfg.blocks[b].start != 0
1525 && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
1526 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_LONG
1527 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_STRING
1528 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_MATCH)) {
1529 zend_jit_reset_last_valid_opline(&ctx);
1530 } else {
1531 zend_jit_set_last_valid_opline(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start);
1532 }
1533 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
1534 zend_jit_reset_last_valid_opline(&ctx);
1535 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY) {
1536 zend_jit_reset_last_valid_opline(&ctx);
1537 } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) {
1538 zend_jit_set_last_valid_opline(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start);
1539 }
1540 if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) {
1541 zend_jit_check_timeout(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start, NULL);
1542 }
1543 if (!ssa->cfg.blocks[b].len) {
1544 zend_jit_bb_end(&ctx, b);
1545 continue;
1546 }
1547 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
1548 zend_ssa_phi *phi = ssa->blocks[b].phis;
1549
1550 while (phi) {
1551 zend_jit_reg_var *ival = &ra[phi->ssa_var];
1552
1553 if (ival->ref) {
1554 if (ival->flags & ZREG_LOAD) {
1555 ZEND_ASSERT(ival->ref == IR_NULL);
1556
1557 if (!zend_jit_load_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var)) {
1558 goto jit_failure;
1559 }
1560 } else if (ival->flags & ZREG_STORE) {
1561 ZEND_ASSERT(ival->ref != IR_NULL);
1562
1563 if (!zend_jit_store_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var, 1)) {
1564 goto jit_failure;
1565 }
1566 }
1567 }
1568 phi = phi->next;
1569 }
1570 }
1571 end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
1572 for (i = ssa->cfg.blocks[b].start; i <= end; i++) {
1573 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[i] : NULL;
1574 opline = op_array->opcodes + i;
1575 switch (opline->opcode) {
1576 case ZEND_INIT_FCALL:
1577 case ZEND_INIT_FCALL_BY_NAME:
1578 case ZEND_INIT_NS_FCALL_BY_NAME:
1579 case ZEND_INIT_METHOD_CALL:
1580 case ZEND_INIT_DYNAMIC_CALL:
1581 case ZEND_INIT_STATIC_METHOD_CALL:
1582 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
1583 case ZEND_INIT_USER_CALL:
1584 case ZEND_NEW:
1585 call_level++;
1586 }
1587
1588 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
1589 switch (opline->opcode) {
1590 case ZEND_PRE_INC:
1591 case ZEND_PRE_DEC:
1592 case ZEND_POST_INC:
1593 case ZEND_POST_DEC:
1594 if (opline->op1_type != IS_CV) {
1595 break;
1596 }
1597 op1_info = OP1_INFO();
1598 if (!(op1_info & MAY_BE_LONG)) {
1599 break;
1600 }
1601 if (opline->result_type != IS_UNUSED) {
1602 res_use_info = -1;
1603
1604 if (opline->result_type == IS_CV
1605 && ssa->vars
1606 && ssa_op->result_use >= 0
1607 && !ssa->vars[ssa_op->result_use].no_val) {
1608 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1609
1610 if (Z_MODE(res_use_addr) != IS_REG
1611 || Z_LOAD(res_use_addr)
1612 || Z_STORE(res_use_addr)) {
1613 res_use_info = RES_USE_INFO();
1614 }
1615 }
1616 res_info = RES_INFO();
1617 res_addr = RES_REG_ADDR();
1618 } else {
1619 res_use_info = -1;
1620 res_info = -1;
1621 res_addr = 0;
1622 }
1623 op1_def_info = OP1_DEF_INFO();
1624 if (!zend_jit_inc_dec(&ctx, opline,
1625 op1_info, OP1_REG_ADDR(),
1626 op1_def_info, OP1_DEF_REG_ADDR(),
1627 res_use_info, res_info,
1628 res_addr,
1629 (op1_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1630 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1631 goto jit_failure;
1632 }
1633 goto done;
1634 case ZEND_BW_OR:
1635 case ZEND_BW_AND:
1636 case ZEND_BW_XOR:
1637 case ZEND_SL:
1638 case ZEND_SR:
1639 case ZEND_MOD:
1640 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1641 break;
1642 }
1643 op1_info = OP1_INFO();
1644 op2_info = OP2_INFO();
1645 if (!(op1_info & MAY_BE_LONG)
1646 || !(op2_info & MAY_BE_LONG)) {
1647 break;
1648 }
1649 res_addr = RES_REG_ADDR();
1650 if (Z_MODE(res_addr) != IS_REG
1651 && (i + 1) <= end
1652 && zend_jit_next_is_send_result(opline)) {
1653 i++;
1654 res_use_info = -1;
1655 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1656 if (!zend_jit_reuse_ip(&ctx)) {
1657 goto jit_failure;
1658 }
1659 } else {
1660 res_use_info = -1;
1661
1662 if (opline->result_type == IS_CV
1663 && ssa->vars
1664 && ssa_op->result_use >= 0
1665 && !ssa->vars[ssa_op->result_use].no_val) {
1666 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1667
1668 if (Z_MODE(res_use_addr) != IS_REG
1669 || Z_LOAD(res_use_addr)
1670 || Z_STORE(res_use_addr)) {
1671 res_use_info = RES_USE_INFO();
1672 }
1673 }
1674 }
1675 if (!zend_jit_long_math(&ctx, opline,
1676 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
1677 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
1678 res_use_info, RES_INFO(), res_addr,
1679 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1680 goto jit_failure;
1681 }
1682 goto done;
1683 case ZEND_ADD:
1684 case ZEND_SUB:
1685 case ZEND_MUL:
1686 // case ZEND_DIV: // TODO: check for division by zero ???
1687 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1688 break;
1689 }
1690 op1_info = OP1_INFO();
1691 op2_info = OP2_INFO();
1692 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1693 break;
1694 }
1695 if (opline->opcode == ZEND_ADD &&
1696 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
1697 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
1698 /* pass */
1699 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
1700 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1701 break;
1702 }
1703 res_addr = RES_REG_ADDR();
1704 if (Z_MODE(res_addr) != IS_REG
1705 && (i + 1) <= end
1706 && zend_jit_next_is_send_result(opline)) {
1707 i++;
1708 res_use_info = -1;
1709 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1710 if (!zend_jit_reuse_ip(&ctx)) {
1711 goto jit_failure;
1712 }
1713 } else {
1714 res_use_info = -1;
1715
1716 if (opline->result_type == IS_CV
1717 && ssa->vars
1718 && ssa_op->result_use >= 0
1719 && !ssa->vars[ssa_op->result_use].no_val) {
1720 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1721
1722 if (Z_MODE(res_use_addr) != IS_REG
1723 || Z_LOAD(res_use_addr)
1724 || Z_STORE(res_use_addr)) {
1725 res_use_info = RES_USE_INFO();
1726 }
1727 }
1728 }
1729 res_info = RES_INFO();
1730 if (opline->opcode == ZEND_ADD &&
1731 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
1732 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
1733 if (!zend_jit_add_arrays(&ctx, opline, op1_info, OP1_REG_ADDR(), op2_info, OP2_REG_ADDR(), res_addr)) {
1734 goto jit_failure;
1735 }
1736 } else {
1737 if (!zend_jit_math(&ctx, opline,
1738 op1_info, OP1_REG_ADDR(),
1739 op2_info, OP2_REG_ADDR(),
1740 res_use_info, res_info, res_addr,
1741 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1742 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1743 goto jit_failure;
1744 }
1745 }
1746 goto done;
1747 case ZEND_CONCAT:
1748 case ZEND_FAST_CONCAT:
1749 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1750 break;
1751 }
1752 op1_info = OP1_INFO();
1753 op2_info = OP2_INFO();
1754 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1755 break;
1756 }
1757 if (!(op1_info & MAY_BE_STRING) ||
1758 !(op2_info & MAY_BE_STRING)) {
1759 break;
1760 }
1761 res_addr = RES_REG_ADDR();
1762 if ((i + 1) <= end
1763 && zend_jit_next_is_send_result(opline)) {
1764 i++;
1765 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1766 if (!zend_jit_reuse_ip(&ctx)) {
1767 goto jit_failure;
1768 }
1769 }
1770 if (!zend_jit_concat(&ctx, opline,
1771 op1_info, op2_info, res_addr,
1772 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1773 goto jit_failure;
1774 }
1775 goto done;
1776 case ZEND_ASSIGN_OP:
1777 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1778 break;
1779 }
1780 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1781 break;
1782 }
1783 op1_info = OP1_INFO();
1784 op2_info = OP2_INFO();
1785 if (!zend_jit_supported_binary_op(
1786 opline->extended_value, op1_info, op2_info)) {
1787 break;
1788 }
1789 op1_addr = OP1_REG_ADDR();
1790 op1_mem_info = -1;
1791 if (Z_MODE(op1_addr) != IS_REG
1792 || Z_LOAD(op1_addr)
1793 || Z_STORE(op1_addr)) {
1794 op1_mem_info = op1_info;
1795 }
1796 op1_def_info = OP1_DEF_INFO();
1797 if (!zend_jit_assign_op(&ctx, opline,
1798 op1_info, op1_addr, OP1_RANGE(),
1799 op1_def_info, OP1_DEF_REG_ADDR(), op1_mem_info,
1800 op2_info, OP2_REG_ADDR(), OP2_RANGE(),
1801 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1802 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1803 goto jit_failure;
1804 }
1805 goto done;
1806 case ZEND_ASSIGN_DIM_OP:
1807 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1808 break;
1809 }
1810 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1811 break;
1812 }
1813 if (!zend_jit_supported_binary_op(
1814 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
1815 break;
1816 }
1817 if (!zend_jit_assign_dim_op(&ctx, opline,
1818 OP1_INFO(), OP1_DEF_INFO(), OP1_REG_ADDR(), 0,
1819 OP2_INFO(), (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
1820 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
1821 OP1_DATA_INFO(), OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(), IS_UNKNOWN,
1822 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1823 goto jit_failure;
1824 }
1825 goto done;
1826 case ZEND_ASSIGN_DIM:
1827 if (opline->op1_type != IS_CV) {
1828 break;
1829 }
1830 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1831 break;
1832 }
1833 if (!zend_jit_assign_dim(&ctx, opline,
1834 OP1_INFO(), OP1_REG_ADDR(), 0,
1835 OP2_INFO(), (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
1836 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
1837 OP1_DATA_INFO(), OP1_DATA_REG_ADDR(),
1838 (ctx.ra && (ssa_op+1)->op1_def >= 0) ? OP1_DATA_DEF_REG_ADDR() : 0,
1839 (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
1840 IS_UNKNOWN,
1841 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1842 goto jit_failure;
1843 }
1844 goto done;
1845 case ZEND_PRE_INC_OBJ:
1846 case ZEND_PRE_DEC_OBJ:
1847 case ZEND_POST_INC_OBJ:
1848 case ZEND_POST_DEC_OBJ:
1849 if (opline->op2_type != IS_CONST
1850 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1851 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1852 break;
1853 }
1854 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1855 break;
1856 }
1857 ce = NULL;
1858 ce_is_instanceof = 0;
1859 on_this = 0;
1860 if (opline->op1_type == IS_UNUSED) {
1861 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1862 ce = op_array->scope;
1863 /* scope is NULL for closures. */
1864 if (ce) {
1865 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
1866 }
1867 op1_addr = 0;
1868 on_this = 1;
1869 } else {
1870 op1_info = OP1_INFO();
1871 if (!(op1_info & MAY_BE_OBJECT)) {
1872 break;
1873 }
1874 op1_addr = OP1_REG_ADDR();
1875 if (ssa->var_info && ssa->ops) {
1876 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1877 if (ssa_op->op1_use >= 0) {
1878 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1879 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1880 ce = op1_ssa->ce;
1881 ce_is_instanceof = op1_ssa->is_instanceof;
1882 }
1883 }
1884 }
1885 }
1886 if (!zend_jit_incdec_obj(&ctx, opline, op_array, ssa, ssa_op,
1887 op1_info, op1_addr,
1888 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
1889 goto jit_failure;
1890 }
1891 goto done;
1892 case ZEND_ASSIGN_OBJ_OP:
1893 if (opline->result_type != IS_UNUSED) {
1894 break;
1895 }
1896 if (opline->op2_type != IS_CONST
1897 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1898 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1899 break;
1900 }
1901 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1902 break;
1903 }
1904 if (!zend_jit_supported_binary_op(
1905 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
1906 break;
1907 }
1908 ce = NULL;
1909 ce_is_instanceof = 0;
1910 on_this = 0;
1911 if (opline->op1_type == IS_UNUSED) {
1912 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1913 ce = op_array->scope;
1914 /* scope is NULL for closures. */
1915 if (ce) {
1916 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
1917 }
1918 op1_addr = 0;
1919 on_this = 1;
1920 } else {
1921 op1_info = OP1_INFO();
1922 if (!(op1_info & MAY_BE_OBJECT)) {
1923 break;
1924 }
1925 op1_addr = OP1_REG_ADDR();
1926 if (ssa->var_info && ssa->ops) {
1927 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1928 if (ssa_op->op1_use >= 0) {
1929 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1930 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1931 ce = op1_ssa->ce;
1932 ce_is_instanceof = op1_ssa->is_instanceof;
1933 }
1934 }
1935 }
1936 }
1937 if (!zend_jit_assign_obj_op(&ctx, opline, op_array, ssa, ssa_op,
1938 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(),
1939 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
1940 goto jit_failure;
1941 }
1942 goto done;
1943 case ZEND_ASSIGN_OBJ:
1944 if (opline->op2_type != IS_CONST
1945 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1946 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1947 break;
1948 }
1949 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1950 break;
1951 }
1952 ce = NULL;
1953 ce_is_instanceof = 0;
1954 on_this = 0;
1955 if (opline->op1_type == IS_UNUSED) {
1956 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1957 ce = op_array->scope;
1958 /* scope is NULL for closures. */
1959 if (ce) {
1960 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
1961 }
1962 op1_addr = 0;
1963 on_this = 1;
1964 } else {
1965 op1_info = OP1_INFO();
1966 if (!(op1_info & MAY_BE_OBJECT)) {
1967 break;
1968 }
1969 op1_addr = OP1_REG_ADDR();
1970 if (ssa->var_info && ssa->ops) {
1971 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1972 if (ssa_op->op1_use >= 0) {
1973 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1974 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1975 ce = op1_ssa->ce;
1976 ce_is_instanceof = op1_ssa->is_instanceof;
1977 }
1978 }
1979 }
1980 }
1981 if (!zend_jit_assign_obj(&ctx, opline, op_array, ssa, ssa_op,
1982 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_REG_ADDR(), OP1_DATA_DEF_REG_ADDR(),
1983 (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
1984 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN,
1985 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1986 goto jit_failure;
1987 }
1988 goto done;
1989 case ZEND_ASSIGN:
1990 if (opline->op1_type != IS_CV) {
1991 break;
1992 }
1993 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1994 break;
1995 }
1996 op2_addr = OP2_REG_ADDR();
1997 if (ra
1998 && ssa->ops[opline - op_array->opcodes].op2_def >= 0
1999 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) {
2000 op2_def_addr = OP2_DEF_REG_ADDR();
2001 } else {
2002 op2_def_addr = op2_addr;
2003 }
2004 op1_info = OP1_INFO();
2005 if (ra && ssa->vars[ssa_op->op1_use].no_val) {
2006 op1_info |= MAY_BE_UNDEF; // requres type assignment
2007 }
2008 if (opline->result_type == IS_UNUSED) {
2009 res_addr = 0;
2010 res_info = -1;
2011 } else {
2012 res_addr = RES_REG_ADDR();
2013 res_info = RES_INFO();
2014 if (Z_MODE(res_addr) != IS_REG
2015 && (i + 1) <= end
2016 && zend_jit_next_is_send_result(opline)
2017 && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) {
2018 i++;
2019 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
2020 if (!zend_jit_reuse_ip(&ctx)) {
2021 goto jit_failure;
2022 }
2023 }
2024 }
2025 if (!zend_jit_assign(&ctx, opline,
2026 op1_info, OP1_REG_ADDR(),
2027 OP1_DEF_INFO(), OP1_DEF_REG_ADDR(),
2028 OP2_INFO(), op2_addr, op2_def_addr,
2029 res_info, res_addr,
2030 0,
2031 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2032 goto jit_failure;
2033 }
2034 goto done;
2035 case ZEND_QM_ASSIGN:
2036 op1_addr = OP1_REG_ADDR();
2037 if (ra
2038 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
2039 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
2040 op1_def_addr = OP1_DEF_REG_ADDR();
2041 } else {
2042 op1_def_addr = op1_addr;
2043 }
2044 if (!zend_jit_qm_assign(&ctx, opline,
2045 OP1_INFO(), op1_addr, op1_def_addr,
2046 -1, RES_INFO(), RES_REG_ADDR())) {
2047 goto jit_failure;
2048 }
2049 goto done;
2050 case ZEND_INIT_FCALL:
2051 case ZEND_INIT_FCALL_BY_NAME:
2052 case ZEND_INIT_NS_FCALL_BY_NAME:
2053 if (!zend_jit_init_fcall(&ctx, opline, b, op_array, ssa, ssa_op, call_level, NULL, 0)) {
2054 goto jit_failure;
2055 }
2056 goto done;
2057 case ZEND_SEND_VAL:
2058 case ZEND_SEND_VAL_EX:
2059 if (opline->op2_type == IS_CONST) {
2060 /* Named parameters not supported in JIT (yet) */
2061 break;
2062 }
2063 if (opline->opcode == ZEND_SEND_VAL_EX
2064 && opline->op2.num > MAX_ARG_FLAG_NUM) {
2065 break;
2066 }
2067 if (!zend_jit_send_val(&ctx, opline,
2068 OP1_INFO(), OP1_REG_ADDR())) {
2069 goto jit_failure;
2070 }
2071 goto done;
2072 case ZEND_SEND_REF:
2073 if (opline->op2_type == IS_CONST) {
2074 /* Named parameters not supported in JIT (yet) */
2075 break;
2076 }
2077 if (!zend_jit_send_ref(&ctx, opline, op_array,
2078 OP1_INFO(), 0)) {
2079 goto jit_failure;
2080 }
2081 goto done;
2082 case ZEND_SEND_VAR:
2083 case ZEND_SEND_VAR_EX:
2084 case ZEND_SEND_VAR_NO_REF:
2085 case ZEND_SEND_VAR_NO_REF_EX:
2086 case ZEND_SEND_FUNC_ARG:
2087 if (opline->op2_type == IS_CONST) {
2088 /* Named parameters not supported in JIT (yet) */
2089 break;
2090 }
2091 if ((opline->opcode == ZEND_SEND_VAR_EX
2092 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
2093 && opline->op2.num > MAX_ARG_FLAG_NUM) {
2094 break;
2095 }
2096 op1_addr = OP1_REG_ADDR();
2097 if (ra
2098 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
2099 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
2100 op1_def_addr = OP1_DEF_REG_ADDR();
2101 } else {
2102 op1_def_addr = op1_addr;
2103 }
2104 if (!zend_jit_send_var(&ctx, opline, op_array,
2105 OP1_INFO(), op1_addr, op1_def_addr)) {
2106 goto jit_failure;
2107 }
2108 goto done;
2109 case ZEND_CHECK_FUNC_ARG:
2110 if (opline->op2_type == IS_CONST) {
2111 /* Named parameters not supported in JIT (yet) */
2112 break;
2113 }
2114 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
2115 break;
2116 }
2117 if (!zend_jit_check_func_arg(&ctx, opline)) {
2118 goto jit_failure;
2119 }
2120 goto done;
2121 case ZEND_CHECK_UNDEF_ARGS:
2122 if (!zend_jit_check_undef_args(&ctx, opline)) {
2123 goto jit_failure;
2124 }
2125 goto done;
2126 case ZEND_DO_UCALL:
2127 ZEND_FALLTHROUGH;
2128 case ZEND_DO_ICALL:
2129 case ZEND_DO_FCALL_BY_NAME:
2130 case ZEND_DO_FCALL:
2131 if (!zend_jit_do_fcall(&ctx, opline, op_array, ssa, call_level, b + 1, NULL)) {
2132 goto jit_failure;
2133 }
2134 goto done;
2135 case ZEND_IS_EQUAL:
2136 case ZEND_IS_NOT_EQUAL:
2137 case ZEND_IS_SMALLER:
2138 case ZEND_IS_SMALLER_OR_EQUAL:
2139 case ZEND_CASE: {
2140 res_addr = RES_REG_ADDR();
2141 if ((opline->result_type & IS_TMP_VAR)
2142 && (i + 1) <= end
2143 && ((opline+1)->opcode == ZEND_JMPZ
2144 || (opline+1)->opcode == ZEND_JMPNZ
2145 || (opline+1)->opcode == ZEND_JMPZ_EX
2146 || (opline+1)->opcode == ZEND_JMPNZ_EX)
2147 && (opline+1)->op1_type == IS_TMP_VAR
2148 && (opline+1)->op1.var == opline->result.var) {
2149 i++;
2150 smart_branch_opcode = (opline+1)->opcode;
2151 target_label = ssa->cfg.blocks[b].successors[0];
2152 target_label2 = ssa->cfg.blocks[b].successors[1];
2153 /* For EX variant write into the result of EX opcode. */
2154 if ((opline+1)->opcode == ZEND_JMPZ_EX
2155 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
2156 res_addr = OP_REG_ADDR(opline + 1, ssa_op + 1, result_type, result, result_def);
2157 }
2158 } else {
2159 smart_branch_opcode = 0;
2160 target_label = target_label2 = (uint32_t)-1;
2161 }
2162 if (!zend_jit_cmp(&ctx, opline,
2163 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
2164 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
2165 res_addr,
2166 zend_may_throw(opline, ssa_op, op_array, ssa),
2167 smart_branch_opcode, target_label, target_label2,
2168 NULL, 0)) {
2169 goto jit_failure;
2170 }
2171 goto done;
2172 }
2173 case ZEND_IS_IDENTICAL:
2174 case ZEND_IS_NOT_IDENTICAL:
2175 case ZEND_CASE_STRICT:
2176 res_addr = RES_REG_ADDR();
2177 if ((opline->result_type & IS_TMP_VAR)
2178 && (i + 1) <= end
2179 && ((opline+1)->opcode == ZEND_JMPZ
2180 || (opline+1)->opcode == ZEND_JMPZ_EX
2181 || (opline+1)->opcode == ZEND_JMPNZ_EX
2182 || (opline+1)->opcode == ZEND_JMPNZ)
2183 && (opline+1)->op1_type == IS_TMP_VAR
2184 && (opline+1)->op1.var == opline->result.var) {
2185 i++;
2186 smart_branch_opcode = (opline+1)->opcode;
2187 target_label = ssa->cfg.blocks[b].successors[0];
2188 target_label2 = ssa->cfg.blocks[b].successors[1];
2189 /* For EX variant write into the result of EX opcode. */
2190 if ((opline+1)->opcode == ZEND_JMPZ_EX
2191 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
2192 res_addr = OP_REG_ADDR(opline + 1, ssa_op + 1, result_type, result, result_def);
2193 }
2194 } else {
2195 smart_branch_opcode = 0;
2196 target_label = target_label2 = (uint32_t)-1;
2197 }
2198 if (!zend_jit_identical(&ctx, opline,
2199 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
2200 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
2201 res_addr,
2202 zend_may_throw(opline, ssa_op, op_array, ssa),
2203 smart_branch_opcode, target_label, target_label2,
2204 NULL, 0)) {
2205 goto jit_failure;
2206 }
2207 goto done;
2208 case ZEND_DEFINED:
2209 if ((opline->result_type & IS_TMP_VAR)
2210 && (i + 1) <= end
2211 && ((opline+1)->opcode == ZEND_JMPZ
2212 || (opline+1)->opcode == ZEND_JMPNZ)
2213 && (opline+1)->op1_type == IS_TMP_VAR
2214 && (opline+1)->op1.var == opline->result.var) {
2215 i++;
2216 smart_branch_opcode = (opline+1)->opcode;
2217 target_label = ssa->cfg.blocks[b].successors[0];
2218 target_label2 = ssa->cfg.blocks[b].successors[1];
2219 } else {
2220 smart_branch_opcode = 0;
2221 target_label = target_label2 = (uint32_t)-1;
2222 }
2223 if (!zend_jit_defined(&ctx, opline, smart_branch_opcode, target_label, target_label2, NULL)) {
2224 goto jit_failure;
2225 }
2226 goto done;
2227 case ZEND_TYPE_CHECK:
2228 if (opline->extended_value == MAY_BE_RESOURCE) {
2229 // TODO: support for is_resource() ???
2230 break;
2231 }
2232 if ((opline->result_type & IS_TMP_VAR)
2233 && (i + 1) <= end
2234 && ((opline+1)->opcode == ZEND_JMPZ
2235 || (opline+1)->opcode == ZEND_JMPNZ)
2236 && (opline+1)->op1_type == IS_TMP_VAR
2237 && (opline+1)->op1.var == opline->result.var) {
2238 i++;
2239 smart_branch_opcode = (opline+1)->opcode;
2240 target_label = ssa->cfg.blocks[b].successors[0];
2241 target_label2 = ssa->cfg.blocks[b].successors[1];
2242 } else {
2243 smart_branch_opcode = 0;
2244 target_label = target_label2 = (uint32_t)-1;
2245 }
2246 if (!zend_jit_type_check(&ctx, opline, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) {
2247 goto jit_failure;
2248 }
2249 goto done;
2250 case ZEND_RETURN:
2251 op1_info = OP1_INFO();
2252 if ((PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info))
2253 || op_array->type == ZEND_EVAL_CODE
2254 // TODO: support for top-level code
2255 || !op_array->function_name
2256 // TODO: support for IS_UNDEF ???
2257 || (op1_info & MAY_BE_UNDEF)) {
2258 if (!zend_jit_tail_handler(&ctx, opline)) {
2259 goto jit_failure;
2260 }
2261 } else {
2262 if (!zend_jit_return(&ctx, opline, op_array,
2263 op1_info, OP1_REG_ADDR())) {
2264 goto jit_failure;
2265 }
2266 }
2267 goto done;
2268 case ZEND_BOOL:
2269 case ZEND_BOOL_NOT:
2270 if (!zend_jit_bool_jmpznz(&ctx, opline,
2271 OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(),
2272 -1, -1,
2273 zend_may_throw(opline, ssa_op, op_array, ssa),
2274 opline->opcode, NULL)) {
2275 goto jit_failure;
2276 }
2277 goto done;
2278 case ZEND_JMPZ:
2279 case ZEND_JMPNZ:
2280 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
2281 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2282 /* smart branch */
2283 if (!zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2284 goto jit_failure;
2285 }
2286 goto done;
2287 }
2288 ZEND_FALLTHROUGH;
2289 case ZEND_JMPZ_EX:
2290 case ZEND_JMPNZ_EX:
2291 if (opline->result_type == IS_UNDEF) {
2292 res_addr = 0;
2293 } else {
2294 res_addr = RES_REG_ADDR();
2295 }
2296 if (!zend_jit_bool_jmpznz(&ctx, opline,
2297 OP1_INFO(), OP1_REG_ADDR(), res_addr,
2298 ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1],
2299 zend_may_throw(opline, ssa_op, op_array, ssa),
2300 opline->opcode, NULL)) {
2301 goto jit_failure;
2302 }
2303 goto done;
2304 case ZEND_ISSET_ISEMPTY_CV:
2305 if ((opline->extended_value & ZEND_ISEMPTY)) {
2306 // TODO: support for empty() ???
2307 break;
2308 }
2309 if ((opline->result_type & IS_TMP_VAR)
2310 && (i + 1) <= end
2311 && ((opline+1)->opcode == ZEND_JMPZ
2312 || (opline+1)->opcode == ZEND_JMPNZ)
2313 && (opline+1)->op1_type == IS_TMP_VAR
2314 && (opline+1)->op1.var == opline->result.var) {
2315 i++;
2316 smart_branch_opcode = (opline+1)->opcode;
2317 target_label = ssa->cfg.blocks[b].successors[0];
2318 target_label2 = ssa->cfg.blocks[b].successors[1];
2319 } else {
2320 smart_branch_opcode = 0;
2321 target_label = target_label2 = (uint32_t)-1;
2322 }
2323 if (!zend_jit_isset_isempty_cv(&ctx, opline,
2324 OP1_INFO(), OP1_REG_ADDR(),
2325 smart_branch_opcode, target_label, target_label2,
2326 NULL)) {
2327 goto jit_failure;
2328 }
2329 goto done;
2330 case ZEND_IN_ARRAY:
2331 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
2332 break;
2333 }
2334 op1_info = OP1_INFO();
2335 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
2336 break;
2337 }
2338 if ((opline->result_type & IS_TMP_VAR)
2339 && (i + 1) <= end
2340 && ((opline+1)->opcode == ZEND_JMPZ
2341 || (opline+1)->opcode == ZEND_JMPNZ)
2342 && (opline+1)->op1_type == IS_TMP_VAR
2343 && (opline+1)->op1.var == opline->result.var) {
2344 i++;
2345 smart_branch_opcode = (opline+1)->opcode;
2346 target_label = ssa->cfg.blocks[b].successors[0];
2347 target_label2 = ssa->cfg.blocks[b].successors[1];
2348 } else {
2349 smart_branch_opcode = 0;
2350 target_label = target_label2 = (uint32_t)-1;
2351 }
2352 if (!zend_jit_in_array(&ctx, opline,
2353 op1_info, OP1_REG_ADDR(),
2354 smart_branch_opcode, target_label, target_label2,
2355 NULL)) {
2356 goto jit_failure;
2357 }
2358 goto done;
2359 case ZEND_FETCH_DIM_R:
2360 case ZEND_FETCH_DIM_IS:
2361 case ZEND_FETCH_LIST_R:
2362 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2363 break;
2364 }
2365 if (!zend_jit_fetch_dim_read(&ctx, opline, ssa, ssa_op,
2366 OP1_INFO(), OP1_REG_ADDR(), 0,
2367 OP2_INFO(), OP2_REG_ADDR(), OP2_RANGE(),
2368 RES_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
2369 goto jit_failure;
2370 }
2371 goto done;
2372 case ZEND_FETCH_DIM_W:
2373 case ZEND_FETCH_DIM_RW:
2374 // case ZEND_FETCH_DIM_UNSET:
2375 case ZEND_FETCH_LIST_W:
2376 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2377 break;
2378 }
2379 if (opline->op1_type != IS_CV) {
2380 break;
2381 }
2382 if (!zend_jit_fetch_dim(&ctx, opline,
2383 OP1_INFO(), OP1_REG_ADDR(),
2384 OP2_INFO(), (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
2385 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : 0,
2386 RES_REG_ADDR(), IS_UNKNOWN)) {
2387 goto jit_failure;
2388 }
2389 goto done;
2390 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
2391 if ((opline->extended_value & ZEND_ISEMPTY)) {
2392 // TODO: support for empty() ???
2393 break;
2394 }
2395 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2396 break;
2397 }
2398 if ((opline->result_type & IS_TMP_VAR)
2399 && (i + 1) <= end
2400 && ((opline+1)->opcode == ZEND_JMPZ
2401 || (opline+1)->opcode == ZEND_JMPNZ)
2402 && (opline+1)->op1_type == IS_TMP_VAR
2403 && (opline+1)->op1.var == opline->result.var) {
2404 i++;
2405 smart_branch_opcode = (opline+1)->opcode;
2406 target_label = ssa->cfg.blocks[b].successors[0];
2407 target_label2 = ssa->cfg.blocks[b].successors[1];
2408 } else {
2409 smart_branch_opcode = 0;
2410 target_label = target_label2 = (uint32_t)-1;
2411 }
2412 if (!zend_jit_isset_isempty_dim(&ctx, opline,
2413 OP1_INFO(), OP1_REG_ADDR(), 0,
2414 OP2_INFO(), OP2_REG_ADDR(), OP2_RANGE(), IS_UNKNOWN,
2415 zend_may_throw(opline, ssa_op, op_array, ssa),
2416 smart_branch_opcode, target_label, target_label2,
2417 NULL)) {
2418 goto jit_failure;
2419 }
2420 goto done;
2421 case ZEND_FETCH_OBJ_R:
2422 case ZEND_FETCH_OBJ_IS:
2423 case ZEND_FETCH_OBJ_W:
2424 if (opline->op2_type != IS_CONST
2425 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2426 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2427 break;
2428 }
2429 ce = NULL;
2430 ce_is_instanceof = 0;
2431 on_this = 0;
2432 if (opline->op1_type == IS_UNUSED) {
2433 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
2434 op1_addr = 0;
2435 ce = op_array->scope;
2436 /* scope is NULL for closures. */
2437 if (ce) {
2438 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
2439 }
2440 on_this = 1;
2441 } else {
2442 op1_info = OP1_INFO();
2443 if (!(op1_info & MAY_BE_OBJECT)) {
2444 break;
2445 }
2446 op1_addr = OP1_REG_ADDR();
2447 if (ssa->var_info && ssa->ops) {
2448 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
2449 if (ssa_op->op1_use >= 0) {
2450 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
2451 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
2452 ce = op1_ssa->ce;
2453 ce_is_instanceof = op1_ssa->is_instanceof;
2454 }
2455 }
2456 }
2457 }
2458 if (!zend_jit_fetch_obj(&ctx, opline, op_array, ssa, ssa_op,
2459 op1_info, op1_addr, 0, ce, ce_is_instanceof, on_this, 0, 0, NULL,
2460 RES_REG_ADDR(), IS_UNKNOWN,
2461 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2462 goto jit_failure;
2463 }
2464 goto done;
2465 case ZEND_FETCH_STATIC_PROP_R:
2466 case ZEND_FETCH_STATIC_PROP_IS:
2467 case ZEND_FETCH_STATIC_PROP_W:
2468 case ZEND_FETCH_STATIC_PROP_RW:
2469 case ZEND_FETCH_STATIC_PROP_UNSET:
2470 if (!(opline->op1_type == IS_CONST
2471 && (opline->op2_type == IS_CONST
2472 || (opline->op2_type == IS_UNUSED
2473 && ((opline->op2.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_SELF
2474 || (opline->op2.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_PARENT))))) {
2475 break;
2476 }
2477 if (!zend_jit_fetch_static_prop(&ctx, opline, op_array)) {
2478 goto jit_failure;
2479 }
2480 goto done;
2481 case ZEND_BIND_GLOBAL:
2482 if (!ssa->ops || !ssa->var_info) {
2483 op1_info = MAY_BE_ANY|MAY_BE_REF;
2484 } else {
2485 op1_info = OP1_INFO();
2486 }
2487 if (!zend_jit_bind_global(&ctx, opline, op1_info)) {
2488 goto jit_failure;
2489 }
2490 goto done;
2491 case ZEND_RECV:
2492 if (!zend_jit_recv(&ctx, opline, op_array)) {
2493 goto jit_failure;
2494 }
2495 goto done;
2496 case ZEND_RECV_INIT:
2497 if (!zend_jit_recv_init(&ctx, opline, op_array,
2498 (opline + 1)->opcode != ZEND_RECV_INIT,
2499 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2500 goto jit_failure;
2501 }
2502 goto done;
2503 case ZEND_FREE:
2504 case ZEND_FE_FREE:
2505 if (!zend_jit_free(&ctx, opline, OP1_INFO(),
2506 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2507 goto jit_failure;
2508 }
2509 goto done;
2510 case ZEND_ECHO:
2511 op1_info = OP1_INFO();
2512 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2513 break;
2514 }
2515 if (!zend_jit_echo(&ctx, opline, op1_info)) {
2516 goto jit_failure;
2517 }
2518 goto done;
2519 case ZEND_STRLEN:
2520 op1_info = OP1_INFO();
2521 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2522 break;
2523 }
2524 if (!zend_jit_strlen(&ctx, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR())) {
2525 goto jit_failure;
2526 }
2527 goto done;
2528 case ZEND_COUNT:
2529 op1_info = OP1_INFO();
2530 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
2531 break;
2532 }
2533 if (!zend_jit_count(&ctx, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
2534 goto jit_failure;
2535 }
2536 goto done;
2537 case ZEND_FETCH_THIS:
2538 if (!zend_jit_fetch_this(&ctx, opline, op_array, 0)) {
2539 goto jit_failure;
2540 }
2541 goto done;
2542 case ZEND_SWITCH_LONG:
2543 case ZEND_SWITCH_STRING:
2544 case ZEND_MATCH:
2545 if (!zend_jit_switch(&ctx, opline, op_array, ssa, NULL, NULL)) {
2546 goto jit_failure;
2547 }
2548 goto done;
2549 case ZEND_VERIFY_RETURN_TYPE:
2550 if (opline->op1_type == IS_UNUSED) {
2551 /* Always throws */
2552 break;
2553 }
2554 if (opline->op1_type == IS_CONST) {
2555 /* TODO Different instruction format, has return value */
2556 break;
2557 }
2558 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
2559 /* Not worth bothering with */
2560 break;
2561 }
2562 if (OP1_INFO() & MAY_BE_REF) {
2563 /* TODO May need reference unwrapping. */
2564 break;
2565 }
2566 if (!zend_jit_verify_return_type(&ctx, opline, op_array, OP1_INFO())) {
2567 goto jit_failure;
2568 }
2569 goto done;
2570 case ZEND_FE_RESET_R:
2571 op1_info = OP1_INFO();
2572 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
2573 break;
2574 }
2575 if (!zend_jit_fe_reset(&ctx, opline, op1_info)) {
2576 goto jit_failure;
2577 }
2578 goto done;
2579 case ZEND_FE_FETCH_R:
2580 op1_info = OP1_INFO();
2581 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
2582 break;
2583 }
2584 if (!zend_jit_fe_fetch(&ctx, opline, op1_info, OP2_INFO(),
2585 ssa->cfg.blocks[b].successors[0], opline->opcode, NULL)) {
2586 goto jit_failure;
2587 }
2588 goto done;
2589 case ZEND_FETCH_CONSTANT:
2590 if (!zend_jit_fetch_constant(&ctx, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
2591 goto jit_failure;
2592 }
2593 goto done;
2594 case ZEND_INIT_METHOD_CALL:
2595 if (opline->op2_type != IS_CONST
2596 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2597 break;
2598 }
2599 ce = NULL;
2600 ce_is_instanceof = 0;
2601 on_this = 0;
2602 if (opline->op1_type == IS_UNUSED) {
2603 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
2604 op1_addr = 0;
2605 ce = op_array->scope;
2606 /* scope is NULL for closures. */
2607 if (ce) {
2608 ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
2609 }
2610 on_this = 1;
2611 } else {
2612 op1_info = OP1_INFO();
2613 if (!(op1_info & MAY_BE_OBJECT)) {
2614 break;
2615 }
2616 op1_addr = OP1_REG_ADDR();
2617 if (ssa->var_info && ssa->ops) {
2618 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
2619 if (ssa_op->op1_use >= 0) {
2620 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
2621 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
2622 ce = op1_ssa->ce;
2623 ce_is_instanceof = op1_ssa->is_instanceof;
2624 }
2625 }
2626 }
2627 }
2628 if (!zend_jit_init_method_call(&ctx, opline, b, op_array, ssa, ssa_op, call_level,
2629 op1_info, op1_addr, ce, ce_is_instanceof, on_this, 0, NULL,
2630 NULL, 0,
2631 -1, -1,
2632 0)) {
2633 goto jit_failure;
2634 }
2635 goto done;
2636 case ZEND_INIT_STATIC_METHOD_CALL:
2637 if (!(opline->op2_type == IS_CONST
2638 && (opline->op1_type == IS_CONST
2639 || (opline->op1_type == IS_UNUSED
2640 && ((opline->op1.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_SELF
2641 || (opline->op1.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_PARENT))))) {
2642 break;
2643 }
2644 if (!zend_jit_init_static_method_call(&ctx, opline, b, op_array, ssa, ssa_op, call_level,
2645 NULL, 0)) {
2646 goto jit_failure;
2647 }
2648 goto done;
2649 case ZEND_ROPE_INIT:
2650 case ZEND_ROPE_ADD:
2651 case ZEND_ROPE_END:
2652 op2_info = OP2_INFO();
2653 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2654 break;
2655 }
2656 if (!zend_jit_rope(&ctx, opline, op2_info)) {
2657 goto jit_failure;
2658 }
2659 goto done;
2660 default:
2661 break;
2662 }
2663 }
2664
2665 switch (opline->opcode) {
2666 case ZEND_RECV_INIT:
2667 case ZEND_BIND_GLOBAL:
2668 if (opline == op_array->opcodes ||
2669 opline->opcode != op_array->opcodes[i-1].opcode) {
2670 /* repeatable opcodes */
2671 if (!zend_jit_handler(&ctx, opline,
2672 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2673 goto jit_failure;
2674 }
2675 }
2676 zend_jit_set_last_valid_opline(&ctx, opline+1);
2677 break;
2678 case ZEND_NOP:
2679 case ZEND_OP_DATA:
2680 case ZEND_SWITCH_LONG:
2681 case ZEND_SWITCH_STRING:
2682 break;
2683 case ZEND_MATCH:
2684 /* We have to exit to the VM because the MATCH handler performs an N-way jump for
2685 * which we can't generate simple (opcache.jit=1201) JIT code. */
2686 if (!zend_jit_tail_handler(&ctx, opline)) {
2687 goto jit_failure;
2688 }
2689 break;
2690 case ZEND_JMP:
2691 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
2692 const zend_op *target = OP_JMP_ADDR(opline, opline->op1);
2693
2694 if (!zend_jit_set_ip(&ctx, target)) {
2695 goto jit_failure;
2696 }
2697 }
2698 break;
2699 case ZEND_CATCH:
2700 case ZEND_FAST_CALL:
2701 case ZEND_FAST_RET:
2702 case ZEND_GENERATOR_CREATE:
2703 case ZEND_GENERATOR_RETURN:
2704 case ZEND_RETURN_BY_REF:
2705 case ZEND_RETURN:
2706 case ZEND_MATCH_ERROR:
2707 /* switch through trampoline */
2708 case ZEND_YIELD:
2709 case ZEND_YIELD_FROM:
2710 case ZEND_THROW:
2711 case ZEND_VERIFY_NEVER_TYPE:
2712 if (!zend_jit_tail_handler(&ctx, opline)) {
2713 goto jit_failure;
2714 }
2715 /* THROW and EXIT may be used in the middle of BB */
2716 /* don't generate code for the rest of BB */
2717 i = end;
2718 break;
2719 /* stackless execution */
2720 case ZEND_INCLUDE_OR_EVAL:
2721 case ZEND_DO_FCALL:
2722 case ZEND_DO_UCALL:
2723 case ZEND_DO_FCALL_BY_NAME:
2724 if (!zend_jit_call(&ctx, opline, b + 1)) {
2725 goto jit_failure;
2726 }
2727 break;
2728 case ZEND_JMPZ:
2729 case ZEND_JMPNZ:
2730 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
2731 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2732 /* smart branch */
2733 if (!zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2734 goto jit_failure;
2735 }
2736 goto done;
2737 }
2738 ZEND_FALLTHROUGH;
2739 case ZEND_JMPZ_EX:
2740 case ZEND_JMPNZ_EX:
2741 case ZEND_JMP_SET:
2742 case ZEND_COALESCE:
2743 case ZEND_JMP_NULL:
2744 case ZEND_FE_RESET_R:
2745 case ZEND_FE_RESET_RW:
2746 case ZEND_ASSERT_CHECK:
2747 case ZEND_FE_FETCH_R:
2748 case ZEND_FE_FETCH_RW:
2749 case ZEND_BIND_INIT_STATIC_OR_JMP:
2750 if (!zend_jit_handler(&ctx, opline,
2751 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
2752 !zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2753 goto jit_failure;
2754 }
2755 break;
2756 case ZEND_JMP_FRAMELESS:
2757 if (!zend_jit_jmp_frameless(&ctx, opline, /* exit_addr */ NULL, /* guard */ 0)) {
2758 goto jit_failure;
2759 }
2760 break;
2761 case ZEND_NEW:
2762 if (!zend_jit_handler(&ctx, opline, 1)) {
2763 return 0;
2764 }
2765 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
2766 zend_class_entry *ce = NULL;
2767
2768 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
2769 if (ssa->ops && ssa->var_info) {
2770 zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
2771 if (res_ssa->ce && !res_ssa->is_instanceof) {
2772 ce = res_ssa->ce;
2773 }
2774 }
2775 } else {
2776 if (opline->op1_type == IS_CONST) {
2777 zval *zv = RT_CONSTANT(opline, opline->op1);
2778 if (Z_TYPE_P(zv) == IS_STRING) {
2779 zval *lc = zv + 1;
2780 ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc));
2781 }
2782 }
2783 }
2784
2785 i++;
2786
2787 if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) {
2788 const zend_op *next_opline = opline + 1;
2789
2790 ZEND_ASSERT(b + 1 == ssa->cfg.blocks[b].successors[0]);
2791 zend_jit_constructor(&ctx, next_opline, op_array, ssa, call_level, b + 1);
2792 }
2793
2794 /* We skip over the DO_FCALL, so decrement call_level ourselves. */
2795 call_level--;
2796 }
2797 break;
2798 case ZEND_FRAMELESS_ICALL_0:
2799 jit_frameless_icall0(jit, opline);
2800 goto done;
2801 case ZEND_FRAMELESS_ICALL_1:
2802 op1_info = OP1_INFO();
2803 jit_frameless_icall1(jit, opline, op1_info);
2804 goto done;
2805 case ZEND_FRAMELESS_ICALL_2:
2806 op1_info = OP1_INFO();
2807 op2_info = OP2_INFO();
2808 jit_frameless_icall2(jit, opline, op1_info, op2_info);
2809 goto done;
2810 case ZEND_FRAMELESS_ICALL_3:
2811 op1_info = OP1_INFO();
2812 op2_info = OP2_INFO();
2813 jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO());
2814 goto done;
2815 default:
2816 if (!zend_jit_handler(&ctx, opline,
2817 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2818 goto jit_failure;
2819 }
2820 if (i == end
2821 && (opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2822 /* smart branch split across basic blocks */
2823 if (!zend_jit_set_cond(&ctx, opline + 2, opline->result.var)) {
2824 goto jit_failure;
2825 }
2826 }
2827 }
2828 done:
2829 switch (opline->opcode) {
2830 case ZEND_DO_FCALL:
2831 case ZEND_DO_ICALL:
2832 case ZEND_DO_UCALL:
2833 case ZEND_DO_FCALL_BY_NAME:
2834 case ZEND_CALLABLE_CONVERT:
2835 call_level--;
2836 }
2837 }
2838 zend_jit_bb_end(&ctx, b);
2839 }
2840
2841 if (jit->return_inputs) {
2842 zend_jit_common_return(jit);
2843
2844 bool left_frame = 0;
2845 if (op_array->last_var > 100) {
2846 /* To many CVs to unroll */
2847 if (!zend_jit_free_cvs(&ctx)) {
2848 goto jit_failure;
2849 }
2850 left_frame = 1;
2851 }
2852 if (!left_frame) {
2853 int j;
2854
2855 for (j = 0 ; j < op_array->last_var; j++) {
2856 uint32_t info = zend_ssa_cv_info(op_array, ssa, j);
2857
2858 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
2859 if (!left_frame) {
2860 left_frame = 1;
2861 if (!zend_jit_leave_frame(&ctx)) {
2862 goto jit_failure;
2863 }
2864 }
2865 if (!zend_jit_free_cv(&ctx, info, j)) {
2866 goto jit_failure;
2867 }
2868 }
2869 }
2870 }
2871 if (!zend_jit_leave_func(&ctx, op_array, NULL, MAY_BE_ANY, left_frame,
2872 NULL, NULL, (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, 1)) {
2873 goto jit_failure;
2874 }
2875 }
2876
2877 handler = zend_jit_finish(&ctx);
2878 if (!handler) {
2879 goto jit_failure;
2880 }
2881 zend_jit_free_ctx(&ctx);
2882
2883 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2884 zend_arena_release(&CG(arena), checkpoint);
2885 }
2886 return SUCCESS;
2887
2888 jit_failure:
2889 zend_jit_free_ctx(&ctx);
2890 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2891 zend_arena_release(&CG(arena), checkpoint);
2892 }
2893 return FAILURE;
2894 }
2895
zend_jit_collect_calls(zend_op_array * op_array,zend_script * script)2896 static void zend_jit_collect_calls(zend_op_array *op_array, zend_script *script)
2897 {
2898 zend_func_info *func_info;
2899
2900 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
2901 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
2902 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
2903 func_info = ZEND_FUNC_INFO(op_array);
2904 } else {
2905 func_info = zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info));
2906 ZEND_SET_FUNC_INFO(op_array, func_info);
2907 }
2908 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, op_array, func_info);
2909 }
2910
zend_jit_cleanup_func_info(zend_op_array * op_array)2911 static void zend_jit_cleanup_func_info(zend_op_array *op_array)
2912 {
2913 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
2914 zend_call_info *caller_info, *callee_info;
2915
2916 if (func_info) {
2917 caller_info = func_info->caller_info;
2918 callee_info = func_info->callee_info;
2919
2920 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
2921 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
2922 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
2923 func_info->num = 0;
2924 func_info->flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
2925 | ZEND_FUNC_JIT_ON_PROF_REQUEST
2926 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
2927 | ZEND_FUNC_JIT_ON_HOT_TRACE;
2928 memset(&func_info->ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
2929 } else {
2930 ZEND_SET_FUNC_INFO(op_array, NULL);
2931 }
2932
2933 while (caller_info) {
2934 if (caller_info->caller_op_array) {
2935 zend_jit_cleanup_func_info(caller_info->caller_op_array);
2936 }
2937 caller_info = caller_info->next_caller;
2938 }
2939 while (callee_info) {
2940 if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) {
2941 zend_jit_cleanup_func_info(&callee_info->callee_func->op_array);
2942 }
2943 callee_info = callee_info->next_callee;
2944 }
2945 }
2946 }
2947
zend_real_jit_func(zend_op_array * op_array,zend_script * script,const zend_op * rt_opline,uint8_t trigger)2948 static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline, uint8_t trigger)
2949 {
2950 zend_ssa ssa;
2951 void *checkpoint;
2952 zend_func_info *func_info;
2953 uint8_t orig_trigger;
2954
2955 if (*dasm_ptr == dasm_end) {
2956 return FAILURE;
2957 }
2958
2959 orig_trigger = JIT_G(trigger);
2960 JIT_G(trigger) = trigger;
2961 checkpoint = zend_arena_checkpoint(CG(arena));
2962
2963 /* Build SSA */
2964 memset(&ssa, 0, sizeof(zend_ssa));
2965
2966 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
2967 if (trigger == ZEND_JIT_ON_FIRST_EXEC) {
2968 zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
2969 op_array = (zend_op_array*) jit_extension->op_array;
2970 } else if (trigger == ZEND_JIT_ON_HOT_COUNTERS) {
2971 zend_jit_op_array_hot_extension *jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
2972 op_array = (zend_op_array*) jit_extension->op_array;
2973 } else {
2974 ZEND_ASSERT(!op_array->scope);
2975 }
2976 }
2977
2978 if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) {
2979 goto jit_failure;
2980 }
2981
2982 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
2983 zend_jit_collect_calls(op_array, script);
2984 func_info = ZEND_FUNC_INFO(op_array);
2985 func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array);
2986 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
2987 zend_init_func_return_info(op_array, script, &func_info->return_info);
2988 }
2989 }
2990
2991 if (zend_jit_op_array_analyze2(op_array, script, &ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
2992 goto jit_failure;
2993 }
2994
2995 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
2996 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa);
2997 }
2998
2999 if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) {
3000 goto jit_failure;
3001 }
3002
3003 zend_jit_cleanup_func_info(op_array);
3004 zend_arena_release(&CG(arena), checkpoint);
3005 JIT_G(trigger) = orig_trigger;
3006 return SUCCESS;
3007
3008 jit_failure:
3009 zend_jit_cleanup_func_info(op_array);
3010 zend_arena_release(&CG(arena), checkpoint);
3011 JIT_G(trigger) = orig_trigger;
3012 return FAILURE;
3013 }
3014
3015 /* Run-time JIT handler */
zend_runtime_jit(void)3016 static int ZEND_FASTCALL zend_runtime_jit(void)
3017 {
3018 zend_execute_data *execute_data = EG(current_execute_data);
3019 zend_op_array *op_array = &EX(func)->op_array;
3020 zend_op *opline = op_array->opcodes;
3021 zend_jit_op_array_extension *jit_extension;
3022 bool do_bailout = 0;
3023
3024 zend_shared_alloc_lock();
3025
3026 if (ZEND_FUNC_INFO(op_array)) {
3027
3028 SHM_UNPROTECT();
3029 zend_jit_unprotect();
3030
3031 zend_try {
3032 /* restore original opcode handlers */
3033 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3034 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3035 opline++;
3036 }
3037 }
3038 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
3039 opline->handler = jit_extension->orig_handler;
3040
3041 /* perform real JIT for this function */
3042 zend_real_jit_func(op_array, NULL, NULL, ZEND_JIT_ON_FIRST_EXEC);
3043 } zend_catch {
3044 do_bailout = true;
3045 } zend_end_try();
3046
3047 zend_jit_protect();
3048 SHM_PROTECT();
3049 }
3050
3051 zend_shared_alloc_unlock();
3052
3053 if (do_bailout) {
3054 zend_bailout();
3055 }
3056
3057 /* JIT-ed code is going to be called by VM */
3058 return 0;
3059 }
3060
zend_jit_check_funcs(HashTable * function_table,bool is_method)3061 void zend_jit_check_funcs(HashTable *function_table, bool is_method) {
3062 zend_op *opline;
3063 zend_function *func;
3064 zend_op_array *op_array;
3065 uintptr_t counter;
3066 zend_jit_op_array_extension *jit_extension;
3067
3068 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(function_table, func) {
3069 if (func->type == ZEND_INTERNAL_FUNCTION) {
3070 break;
3071 }
3072 op_array = &func->op_array;
3073 opline = op_array->opcodes;
3074 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3075 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3076 opline++;
3077 }
3078 }
3079 if (opline->handler == zend_jit_profile_jit_handler) {
3080 if (!RUN_TIME_CACHE(op_array)) {
3081 continue;
3082 }
3083 counter = (uintptr_t)ZEND_COUNTER_INFO(op_array);
3084 ZEND_COUNTER_INFO(op_array) = 0;
3085 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
3086 opline->handler = jit_extension->orig_handler;
3087 if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) {
3088 zend_real_jit_func(op_array, NULL, NULL, ZEND_JIT_ON_PROF_REQUEST);
3089 }
3090 }
3091 } ZEND_HASH_FOREACH_END();
3092 }
3093
zend_jit_hot_func(zend_execute_data * execute_data,const zend_op * opline)3094 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
3095 {
3096 zend_op_array *op_array = &EX(func)->op_array;
3097 zend_jit_op_array_hot_extension *jit_extension;
3098 uint32_t i;
3099 bool do_bailout = 0;
3100
3101 zend_shared_alloc_lock();
3102 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
3103
3104 if (jit_extension) {
3105 SHM_UNPROTECT();
3106 zend_jit_unprotect();
3107
3108 zend_try {
3109 for (i = 0; i < op_array->last; i++) {
3110 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
3111 }
3112
3113 /* perform real JIT for this function */
3114 zend_real_jit_func(op_array, NULL, opline, ZEND_JIT_ON_HOT_COUNTERS);
3115 } zend_catch {
3116 do_bailout = 1;
3117 } zend_end_try();
3118
3119 zend_jit_protect();
3120 SHM_PROTECT();
3121 }
3122
3123 zend_shared_alloc_unlock();
3124
3125 if (do_bailout) {
3126 zend_bailout();
3127 }
3128 /* JIT-ed code is going to be called by VM */
3129 }
3130
zend_jit_setup_hot_counters_ex(zend_op_array * op_array,zend_cfg * cfg)3131 static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg)
3132 {
3133 if (JIT_G(hot_func)) {
3134 zend_op *opline = op_array->opcodes;
3135
3136 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3137 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3138 opline++;
3139 }
3140 }
3141
3142 opline->handler = (const void*)zend_jit_func_hot_counter_handler;
3143 }
3144
3145 if (JIT_G(hot_loop)) {
3146 uint32_t i;
3147
3148 for (i = 0; i < cfg->blocks_count; i++) {
3149 if ((cfg->blocks[i].flags & ZEND_BB_REACHABLE) &&
3150 (cfg->blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
3151 op_array->opcodes[cfg->blocks[i].start].handler =
3152 (const void*)zend_jit_loop_hot_counter_handler;
3153 }
3154 }
3155 }
3156 }
3157
zend_jit_restart_hot_counters(zend_op_array * op_array)3158 static int zend_jit_restart_hot_counters(zend_op_array *op_array)
3159 {
3160 zend_jit_op_array_hot_extension *jit_extension;
3161 zend_cfg cfg;
3162 uint32_t i;
3163
3164 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
3165 for (i = 0; i < op_array->last; i++) {
3166 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
3167 }
3168
3169 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
3170 return FAILURE;
3171 }
3172
3173 zend_jit_setup_hot_counters_ex(op_array, &cfg);
3174
3175 return SUCCESS;
3176 }
3177
zend_jit_setup_hot_counters(zend_op_array * op_array)3178 static int zend_jit_setup_hot_counters(zend_op_array *op_array)
3179 {
3180 zend_jit_op_array_hot_extension *jit_extension;
3181 zend_cfg cfg;
3182 uint32_t i;
3183
3184 ZEND_ASSERT(!JIT_G(hot_func) || zend_jit_func_hot_counter_handler != NULL);
3185 ZEND_ASSERT(!JIT_G(hot_loop) || zend_jit_loop_hot_counter_handler != NULL);
3186
3187 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
3188 return FAILURE;
3189 }
3190
3191 jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*));
3192 if (!jit_extension) {
3193 return FAILURE;
3194 }
3195 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3196 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS;
3197 jit_extension->op_array = op_array;
3198 jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
3199 for (i = 0; i < op_array->last; i++) {
3200 jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
3201 }
3202 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3203
3204 zend_jit_setup_hot_counters_ex(op_array, &cfg);
3205
3206 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3207
3208 return SUCCESS;
3209 }
3210
3211 #include "jit/zend_jit_trace.c"
3212
zend_jit_op_array(zend_op_array * op_array,zend_script * script)3213 int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
3214 {
3215 if (dasm_ptr == NULL) {
3216 return FAILURE;
3217 }
3218
3219 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) {
3220 zend_jit_op_array_extension *jit_extension;
3221 zend_op *opline = op_array->opcodes;
3222
3223 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
3224 ZEND_SET_FUNC_INFO(op_array, NULL);
3225 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
3226 return SUCCESS;
3227 }
3228
3229 /* Set run-time JIT handler */
3230 ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL);
3231 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3232 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3233 opline++;
3234 }
3235 }
3236 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
3237 if (!jit_extension) {
3238 return FAILURE;
3239 }
3240 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3241 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC;
3242 jit_extension->op_array = op_array;
3243 jit_extension->orig_handler = (void*)opline->handler;
3244 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3245 opline->handler = (const void*)zend_jit_runtime_jit_handler;
3246 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3247
3248 return SUCCESS;
3249 } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) {
3250 zend_jit_op_array_extension *jit_extension;
3251 zend_op *opline = op_array->opcodes;
3252
3253 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
3254 ZEND_SET_FUNC_INFO(op_array, NULL);
3255 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
3256 return SUCCESS;
3257 }
3258
3259 ZEND_ASSERT(zend_jit_profile_jit_handler != NULL);
3260 if (op_array->function_name) {
3261 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3262 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3263 opline++;
3264 }
3265 }
3266 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
3267 if (!jit_extension) {
3268 return FAILURE;
3269 }
3270 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3271 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST;
3272 jit_extension->op_array = op_array;
3273 jit_extension->orig_handler = (void*)opline->handler;
3274 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3275 opline->handler = (const void*)zend_jit_profile_jit_handler;
3276 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3277 }
3278
3279 return SUCCESS;
3280 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
3281 return zend_jit_setup_hot_counters(op_array);
3282 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3283 return zend_jit_setup_hot_trace_counters(op_array);
3284 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3285 return zend_real_jit_func(op_array, script, NULL, ZEND_JIT_ON_SCRIPT_LOAD);
3286 } else {
3287 ZEND_UNREACHABLE();
3288 }
3289 return FAILURE;
3290 }
3291
zend_jit_script(zend_script * script)3292 int zend_jit_script(zend_script *script)
3293 {
3294 void *checkpoint;
3295 zend_call_graph call_graph;
3296 zend_func_info *info;
3297 int i;
3298
3299 if (dasm_ptr == NULL || *dasm_ptr == dasm_end) {
3300 return FAILURE;
3301 }
3302
3303 checkpoint = zend_arena_checkpoint(CG(arena));
3304
3305 call_graph.op_arrays_count = 0;
3306 zend_build_call_graph(&CG(arena), script, &call_graph);
3307
3308 zend_analyze_call_graph(&CG(arena), script, &call_graph);
3309
3310 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
3311 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
3312 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS ||
3313 JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3314 for (i = 0; i < call_graph.op_arrays_count; i++) {
3315 if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
3316 goto jit_failure;
3317 }
3318 }
3319 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3320 for (i = 0; i < call_graph.op_arrays_count; i++) {
3321 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3322 if (info) {
3323 if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
3324 goto jit_failure;
3325 }
3326 info->ssa.cfg.flags |= info->flags;
3327 info->flags = info->ssa.cfg.flags;
3328 }
3329 }
3330
3331 for (i = 0; i < call_graph.op_arrays_count; i++) {
3332 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3333 if (info) {
3334 info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]);
3335 if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
3336 zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info);
3337 }
3338 }
3339 }
3340
3341 for (i = 0; i < call_graph.op_arrays_count; i++) {
3342 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3343 if (info) {
3344 if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
3345 goto jit_failure;
3346 }
3347 info->flags = info->ssa.cfg.flags;
3348 }
3349 }
3350
3351 for (i = 0; i < call_graph.op_arrays_count; i++) {
3352 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3353 if (info) {
3354 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
3355 zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa);
3356 }
3357 if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) {
3358 goto jit_failure;
3359 }
3360 }
3361 }
3362
3363 for (i = 0; i < call_graph.op_arrays_count; i++) {
3364 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
3365 }
3366 } else {
3367 ZEND_UNREACHABLE();
3368 }
3369
3370 zend_arena_release(&CG(arena), checkpoint);
3371
3372 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC
3373 || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST
3374 || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS
3375 || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3376 zend_class_entry *ce;
3377 zend_op_array *op_array;
3378
3379 ZEND_HASH_MAP_FOREACH_PTR(&script->class_table, ce) {
3380 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
3381 if (!ZEND_FUNC_INFO(op_array)) {
3382 void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes);
3383
3384 if (jit_extension) {
3385 ZEND_SET_FUNC_INFO(op_array, jit_extension);
3386 }
3387 }
3388 } ZEND_HASH_FOREACH_END();
3389 } ZEND_HASH_FOREACH_END();
3390 }
3391
3392 return SUCCESS;
3393
3394 jit_failure:
3395 if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3396 for (i = 0; i < call_graph.op_arrays_count; i++) {
3397 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
3398 }
3399 }
3400 zend_arena_release(&CG(arena), checkpoint);
3401 return FAILURE;
3402 }
3403
zend_jit_unprotect(void)3404 void zend_jit_unprotect(void)
3405 {
3406 #ifdef HAVE_MPROTECT
3407 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3408 int opts = PROT_READ | PROT_WRITE;
3409 #ifdef ZTS
3410 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3411 if (zend_write_protect) {
3412 pthread_jit_write_protect_np(0);
3413 }
3414 #endif
3415 opts |= PROT_EXEC;
3416 #endif
3417 if (mprotect(dasm_buf, dasm_size, opts) != 0) {
3418 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3419 }
3420 }
3421 #elif defined(_WIN32)
3422 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3423 DWORD old, new;
3424 #ifdef ZTS
3425 new = PAGE_EXECUTE_READWRITE;
3426 #else
3427 new = PAGE_READWRITE;
3428 #endif
3429 if (!VirtualProtect(dasm_buf, dasm_size, new, &old)) {
3430 DWORD err = GetLastError();
3431 char *msg = php_win32_error_to_msg(err);
3432 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3433 php_win32_error_msg_free(msg);
3434 }
3435 }
3436 #endif
3437 }
3438
zend_jit_protect(void)3439 void zend_jit_protect(void)
3440 {
3441 #ifdef HAVE_MPROTECT
3442 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3443 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3444 if (zend_write_protect) {
3445 pthread_jit_write_protect_np(1);
3446 }
3447 #endif
3448 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
3449 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3450 }
3451 }
3452 #elif defined(_WIN32)
3453 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3454 DWORD old;
3455
3456 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
3457 DWORD err = GetLastError();
3458 char *msg = php_win32_error_to_msg(err);
3459 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3460 php_win32_error_msg_free(msg);
3461 }
3462 }
3463 #endif
3464 }
3465
zend_jit_init_handlers(void)3466 static void zend_jit_init_handlers(void)
3467 {
3468 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
3469 zend_jit_runtime_jit_handler = zend_jit_stub_handlers[jit_stub_hybrid_runtime_jit];
3470 zend_jit_profile_jit_handler = zend_jit_stub_handlers[jit_stub_hybrid_profile_jit];
3471 zend_jit_func_hot_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_func_hot_counter];
3472 zend_jit_loop_hot_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_loop_hot_counter];
3473 zend_jit_func_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_func_trace_counter];
3474 zend_jit_ret_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_ret_trace_counter];
3475 zend_jit_loop_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_loop_trace_counter];
3476 } else {
3477 zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
3478 zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
3479 zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper;
3480 zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper;
3481 zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper;
3482 zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper;
3483 zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper;
3484 }
3485 }
3486
zend_jit_globals_ctor(zend_jit_globals * jit_globals)3487 static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
3488 {
3489 memset(jit_globals, 0, sizeof(zend_jit_globals));
3490 zend_jit_trace_init_caches();
3491 }
3492
3493 #ifdef ZTS
zend_jit_globals_dtor(zend_jit_globals * jit_globals)3494 static void zend_jit_globals_dtor(zend_jit_globals *jit_globals)
3495 {
3496 zend_jit_trace_free_caches(jit_globals);
3497 }
3498 #endif
3499
zend_jit_parse_config_num(zend_long jit)3500 static int zend_jit_parse_config_num(zend_long jit)
3501 {
3502 if (jit == 0) {
3503 JIT_G(on) = 0;
3504 return SUCCESS;
3505 }
3506
3507 if (jit < 0) return FAILURE;
3508
3509 if (jit % 10 == 0 || jit % 10 > 5) return FAILURE;
3510 JIT_G(opt_level) = jit % 10;
3511
3512 jit /= 10;
3513 if (jit % 10 > 5 || jit % 10 == 4) return FAILURE;
3514 JIT_G(trigger) = jit % 10;
3515
3516 jit /= 10;
3517 if (jit % 10 > 2) return FAILURE;
3518 JIT_G(opt_flags) = jit % 10;
3519
3520 jit /= 10;
3521 if (jit % 10 > 1) return FAILURE;
3522 JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0);
3523
3524 if (jit / 10 != 0) return FAILURE;
3525
3526 JIT_G(on) = 1;
3527
3528 return SUCCESS;
3529 }
3530
zend_jit_config(zend_string * jit,int stage)3531 int zend_jit_config(zend_string *jit, int stage)
3532 {
3533 if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) {
3534 if (stage == ZEND_INI_STAGE_RUNTIME) {
3535 zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)");
3536 }
3537 return FAILURE;
3538 }
3539
3540 if (zend_string_equals_literal_ci(jit, "disable")) {
3541 JIT_G(enabled) = 0;
3542 JIT_G(on) = 0;
3543 return SUCCESS;
3544 } else if (ZSTR_LEN(jit) == 0
3545 || zend_string_equals_literal_ci(jit, "0")
3546 || zend_string_equals_literal_ci(jit, "off")
3547 || zend_string_equals_literal_ci(jit, "no")
3548 || zend_string_equals_literal_ci(jit, "false")) {
3549 JIT_G(enabled) = 1;
3550 JIT_G(on) = 0;
3551 return SUCCESS;
3552 } else if (zend_string_equals_literal_ci(jit, "1")
3553 || zend_string_equals_literal_ci(jit, "on")
3554 || zend_string_equals_literal_ci(jit, "yes")
3555 || zend_string_equals_literal_ci(jit, "true")
3556 || zend_string_equals_literal_ci(jit, "tracing")) {
3557 JIT_G(enabled) = 1;
3558 JIT_G(on) = 1;
3559 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS;
3560 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
3561 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
3562 return SUCCESS;
3563 } else if (zend_string_equals_ci(jit, ZSTR_KNOWN(ZEND_STR_FUNCTION))) {
3564 JIT_G(enabled) = 1;
3565 JIT_G(on) = 1;
3566 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT;
3567 JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD;
3568 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
3569 return SUCCESS;
3570 } else {
3571 char *end;
3572 zend_long num = ZEND_STRTOL(ZSTR_VAL(jit), &end, 10);
3573 if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) {
3574 goto failure;
3575 }
3576 JIT_G(enabled) = 1;
3577 return SUCCESS;
3578 }
3579
3580 failure:
3581 zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number");
3582 JIT_G(enabled) = 0;
3583 JIT_G(on) = 0;
3584 return FAILURE;
3585 }
3586
zend_jit_debug_config(zend_long old_val,zend_long new_val,int stage)3587 int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage)
3588 {
3589 if (stage != ZEND_INI_STAGE_STARTUP) {
3590 if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) {
3591 if (stage == ZEND_INI_STAGE_RUNTIME) {
3592 zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup");
3593 }
3594 return FAILURE;
3595 }
3596 }
3597 return SUCCESS;
3598 }
3599
zend_jit_init(void)3600 void zend_jit_init(void)
3601 {
3602 #ifdef ZTS
3603 jit_globals_id = ts_allocate_id(&jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, (ts_allocate_dtor) zend_jit_globals_dtor);
3604 #else
3605 zend_jit_globals_ctor(&jit_globals);
3606 #endif
3607 }
3608
zend_jit_check_support(void)3609 int zend_jit_check_support(void)
3610 {
3611 int i;
3612
3613 zend_jit_vm_kind = zend_vm_kind();
3614 if (zend_jit_vm_kind != ZEND_VM_KIND_CALL &&
3615 zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
3616 zend_error(E_WARNING, "JIT is compatible only with CALL and HYBRID VM. JIT disabled.");
3617 JIT_G(enabled) = 0;
3618 JIT_G(on) = 0;
3619 return FAILURE;
3620 }
3621
3622 if (zend_execute_ex != execute_ex) {
3623 if (zend_dtrace_enabled) {
3624 zend_error(E_WARNING, "JIT is incompatible with DTrace. JIT disabled.");
3625 } else if (strcmp(sapi_module.name, "phpdbg") != 0) {
3626 zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled.");
3627 }
3628 JIT_G(enabled) = 0;
3629 JIT_G(on) = 0;
3630 return FAILURE;
3631 }
3632
3633 for (i = 0; i <= 256; i++) {
3634 switch (i) {
3635 /* JIT has no effect on these opcodes */
3636 case ZEND_BEGIN_SILENCE:
3637 case ZEND_END_SILENCE:
3638 break;
3639 default:
3640 if (zend_get_user_opcode_handler(i) != NULL) {
3641 zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled.");
3642 JIT_G(enabled) = 0;
3643 JIT_G(on) = 0;
3644 return FAILURE;
3645 }
3646 }
3647 }
3648
3649 #if defined(IR_TARGET_AARCH64)
3650 if (JIT_G(buffer_size) > 128*1024*1024) {
3651 zend_error(E_WARNING, "JIT on AArch64 doesn't support opcache.jit_buffer_size above 128M.");
3652 JIT_G(enabled) = 0;
3653 JIT_G(on) = 0;
3654 return FAILURE;
3655 }
3656 #elif defined(IR_TARGET_X64)
3657 if (JIT_G(buffer_size) > 2 * Z_L(1024*1024*1024)) {
3658 zend_error(E_WARNING, "JIT on x86_64 doesn't support opcache.jit_buffer_size above 2G.");
3659 JIT_G(enabled) = 0;
3660 JIT_G(on) = 0;
3661 return FAILURE;
3662 }
3663 #endif
3664
3665 return SUCCESS;
3666 }
3667
zend_jit_startup(void * buf,size_t size,bool reattached)3668 void zend_jit_startup(void *buf, size_t size, bool reattached)
3669 {
3670 zend_jit_halt_op = zend_get_halt_op();
3671 zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(ACCELERATOR_PRODUCT_NAME);
3672
3673 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3674 zend_write_protect = pthread_jit_write_protect_supported_np();
3675 #endif
3676
3677 dasm_buf = buf;
3678 dasm_size = size;
3679 dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr) * 2);
3680
3681 #ifdef HAVE_MPROTECT
3682 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3683 if (zend_write_protect) {
3684 pthread_jit_write_protect_np(1);
3685 }
3686 #endif
3687 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
3688 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
3689 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3690 }
3691 } else {
3692 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
3693 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3694 }
3695 }
3696 #elif defined(_WIN32)
3697 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
3698 DWORD old;
3699
3700 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
3701 DWORD err = GetLastError();
3702 char *msg = php_win32_error_to_msg(err);
3703 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3704 php_win32_error_msg_free(msg);
3705 }
3706 } else {
3707 DWORD old;
3708
3709 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
3710 DWORD err = GetLastError();
3711 char *msg = php_win32_error_to_msg(err);
3712 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3713 php_win32_error_msg_free(msg);
3714 }
3715 }
3716 #endif
3717
3718 if (!reattached) {
3719 zend_jit_unprotect();
3720 *dasm_ptr = dasm_buf;
3721 #if defined(_WIN32)
3722 zend_jit_stub_handlers = dasm_buf;
3723 *dasm_ptr = (void**)*dasm_ptr + sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
3724 #elif defined(IR_TARGET_AARCH64)
3725 zend_jit_stub_handlers = dasm_buf;
3726 *dasm_ptr = (void**)*dasm_ptr + (sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])) * 2;
3727 memset(zend_jit_stub_handlers, 0, (sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])) * 2 * sizeof(void*));
3728 #endif
3729 *dasm_ptr = (void*)ZEND_MM_ALIGNED_SIZE_EX(((size_t)(*dasm_ptr)), 16);
3730 zend_jit_protect();
3731 } else {
3732 #if defined(_WIN32) || defined(IR_TARGET_AARCH64)
3733 zend_jit_stub_handlers = dasm_buf;
3734 zend_jit_init_handlers();
3735 #endif
3736 }
3737
3738 zend_jit_unprotect();
3739 zend_jit_setup();
3740 zend_jit_protect();
3741 zend_jit_init_handlers();
3742
3743 zend_jit_trace_startup(reattached);
3744
3745 zend_jit_unprotect();
3746 /* save JIT buffer pos */
3747 dasm_ptr[1] = dasm_ptr[0];
3748 zend_jit_protect();
3749 }
3750
zend_jit_shutdown(void)3751 void zend_jit_shutdown(void)
3752 {
3753 if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE && dasm_ptr != NULL) {
3754 fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf));
3755 }
3756
3757 zend_jit_shutdown_ir();
3758
3759 #ifdef ZTS
3760 ts_free_id(jit_globals_id);
3761 #else
3762 zend_jit_trace_free_caches(&jit_globals);
3763 #endif
3764 }
3765
zend_jit_reset_counters(void)3766 static void zend_jit_reset_counters(void)
3767 {
3768 int i;
3769
3770 for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
3771 zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT;
3772 }
3773 }
3774
zend_jit_activate(void)3775 void zend_jit_activate(void)
3776 {
3777 #ifdef ZTS
3778 if (!zend_jit_startup_ok) {
3779 JIT_G(enabled) = 0;
3780 JIT_G(on) = 0;
3781 return;
3782 }
3783 #endif
3784 zend_jit_profile_counter = 0;
3785 if (JIT_G(on)) {
3786 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
3787 zend_jit_reset_counters();
3788 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3789 zend_jit_reset_counters();
3790 zend_jit_trace_reset_caches();
3791 }
3792 }
3793 }
3794
zend_jit_deactivate(void)3795 void zend_jit_deactivate(void)
3796 {
3797 if (zend_jit_profile_counter && !CG(unclean_shutdown)) {
3798 zend_class_entry *ce;
3799
3800 zend_shared_alloc_lock();
3801 SHM_UNPROTECT();
3802 zend_jit_unprotect();
3803
3804 zend_jit_check_funcs(EG(function_table), 0);
3805 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(EG(class_table), ce) {
3806 if (ce->type == ZEND_INTERNAL_CLASS) {
3807 break;
3808 }
3809 zend_jit_check_funcs(&ce->function_table, 1);
3810 } ZEND_HASH_FOREACH_END();
3811
3812 zend_jit_protect();
3813 SHM_PROTECT();
3814 zend_shared_alloc_unlock();
3815 }
3816
3817 zend_jit_profile_counter = 0;
3818 }
3819
zend_jit_restart_preloaded_op_array(zend_op_array * op_array)3820 static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array)
3821 {
3822 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
3823
3824 if (!func_info) {
3825 return;
3826 }
3827
3828 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
3829 zend_jit_restart_hot_trace_counters(op_array);
3830 } else if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_COUNTERS) {
3831 zend_jit_restart_hot_counters(op_array);
3832 #if 0
3833 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ???
3834 } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) {
3835 zend_op *opline = op_array->opcodes;
3836 zend_jit_op_array_extension *jit_extension =
3837 (zend_jit_op_array_extension*)func_info;
3838
3839 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3840 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3841 opline++;
3842 }
3843 }
3844 if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) {
3845 opline->handler = (const void*)zend_jit_runtime_jit_handler;
3846 } else {
3847 opline->handler = (const void*)zend_jit_profile_jit_handler;
3848 }
3849 #endif
3850 }
3851 if (op_array->num_dynamic_func_defs) {
3852 for (uint32_t i = 0; i < op_array->num_dynamic_func_defs; i++) {
3853 zend_jit_restart_preloaded_op_array(op_array->dynamic_func_defs[i]);
3854 }
3855 }
3856 }
3857
zend_jit_restart_preloaded_script(zend_persistent_script * script)3858 static void zend_jit_restart_preloaded_script(zend_persistent_script *script)
3859 {
3860 zend_class_entry *ce;
3861 zend_op_array *op_array;
3862
3863 zend_jit_restart_preloaded_op_array(&script->script.main_op_array);
3864
3865 ZEND_HASH_MAP_FOREACH_PTR(&script->script.function_table, op_array) {
3866 zend_jit_restart_preloaded_op_array(op_array);
3867 } ZEND_HASH_FOREACH_END();
3868
3869 ZEND_HASH_MAP_FOREACH_PTR(&script->script.class_table, ce) {
3870 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
3871 if (op_array->type == ZEND_USER_FUNCTION) {
3872 zend_jit_restart_preloaded_op_array(op_array);
3873 }
3874 } ZEND_HASH_FOREACH_END();
3875 } ZEND_HASH_FOREACH_END();
3876 }
3877
zend_jit_restart(void)3878 void zend_jit_restart(void)
3879 {
3880 if (dasm_buf) {
3881 zend_jit_unprotect();
3882
3883 /* restore JIT buffer pos */
3884 dasm_ptr[0] = dasm_ptr[1];
3885
3886 zend_jit_trace_restart();
3887
3888 if (ZCSG(preload_script)) {
3889 zend_jit_restart_preloaded_script(ZCSG(preload_script));
3890 if (ZCSG(saved_scripts)) {
3891 zend_persistent_script **p = ZCSG(saved_scripts);
3892
3893 while (*p) {
3894 zend_jit_restart_preloaded_script(*p);
3895 p++;
3896 }
3897 }
3898 }
3899
3900 zend_jit_protect();
3901 }
3902 }
3903
3904 #endif /* HAVE_JIT */
3905
