1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 #include "main/php.h"
20 #include "main/SAPI.h"
21 #include "php_version.h"
22 #include <ZendAccelerator.h>
23 #include "zend_shared_alloc.h"
24 #include "Zend/zend_execute.h"
25 #include "Zend/zend_vm.h"
26 #include "Zend/zend_exceptions.h"
27 #include "Zend/zend_constants.h"
28 #include "Zend/zend_closures.h"
29 #include "Zend/zend_ini.h"
30 #include "Zend/zend_observer.h"
31 #include "zend_smart_str.h"
32 #include "jit/zend_jit.h"
33
34 #ifdef HAVE_JIT
35
36 #include "Optimizer/zend_func_info.h"
37 #include "Optimizer/zend_ssa.h"
38 #include "Optimizer/zend_inference.h"
39 #include "Optimizer/zend_call_graph.h"
40 #include "Optimizer/zend_dump.h"
41 #include "Optimizer/zend_worklist.h"
42
43 #include "jit/zend_jit_internal.h"
44
45 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
46 #include <pthread.h>
47 #endif
48
49 #ifdef ZTS
50 int jit_globals_id;
51 #else
52 zend_jit_globals jit_globals;
53 #endif
54
55 //#define CONTEXT_THREADED_JIT
56 #define ZEND_JIT_USE_RC_INFERENCE
57
58 #ifdef ZEND_JIT_USE_RC_INFERENCE
59 # define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE
60 # define RC_MAY_BE_1(info) (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0)
61 # define RC_MAY_BE_N(info) (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0)
62 #else
63 # define ZEND_SSA_RC_INFERENCE_FLAG 0
64 # define RC_MAY_BE_1(info) 1
65 # define RC_MAY_BE_N(info) 1
66 #endif
67
68 #define JIT_PREFIX "JIT$"
69 #define JIT_STUB_PREFIX "JIT$$"
70 #define TRACE_PREFIX "TRACE-"
71
72 zend_ulong zend_jit_profile_counter = 0;
73 int zend_jit_profile_counter_rid = -1;
74
75 int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
76
77 const zend_op *zend_jit_halt_op = NULL;
78 static int zend_jit_vm_kind = 0;
79 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
80 static int zend_write_protect = 1;
81 #endif
82
83 static void *dasm_buf = NULL;
84 static void *dasm_end = NULL;
85 static void **dasm_ptr = NULL;
86
87 static size_t dasm_size = 0;
88
89 static zend_long jit_bisect_pos = 0;
90
91 static const void *zend_jit_runtime_jit_handler = NULL;
92 static const void *zend_jit_profile_jit_handler = NULL;
93 static const void *zend_jit_func_hot_counter_handler = NULL;
94 static const void *zend_jit_loop_hot_counter_handler = NULL;
95 static const void *zend_jit_func_trace_counter_handler = NULL;
96 static const void *zend_jit_ret_trace_counter_handler = NULL;
97 static const void *zend_jit_loop_trace_counter_handler = NULL;
98
99 static int ZEND_FASTCALL zend_runtime_jit(void);
100
101 static int zend_jit_trace_op_len(const zend_op *opline);
102 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
103 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags);
104 static const void *zend_jit_trace_get_exit_addr(uint32_t n);
105 static void zend_jit_trace_add_code(const void *start, uint32_t size);
106 static zend_string *zend_jit_func_name(const zend_op_array *op_array);
107
108 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info);
109
dominates(const zend_basic_block * blocks,int a,int b)110 static bool dominates(const zend_basic_block *blocks, int a, int b) {
111 while (blocks[b].level > blocks[a].level) {
112 b = blocks[b].idom;
113 }
114 return a == b;
115 }
116
zend_ssa_is_last_use(const zend_op_array * op_array,const zend_ssa * ssa,int var,int use)117 static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
118 {
119 int next_use;
120
121 if (ssa->vars[var].phi_use_chain) {
122 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
123 do {
124 if (!ssa->vars[phi->ssa_var].no_val) {
125 return 0;
126 }
127 phi = zend_ssa_next_use_phi(ssa, var, phi);
128 } while (phi);
129 }
130
131 if (ssa->cfg.blocks[ssa->cfg.map[use]].loop_header > 0
132 || (ssa->cfg.blocks[ssa->cfg.map[use]].flags & ZEND_BB_LOOP_HEADER)) {
133 int b = ssa->cfg.map[use];
134 int prev_use = ssa->vars[var].use_chain;
135 int def_block;
136
137 if (ssa->vars[var].definition >= 0) {
138 def_block =ssa->cfg.map[ssa->vars[var].definition];
139 } else {
140 ZEND_ASSERT(ssa->vars[var].definition_phi);
141 def_block = ssa->vars[var].definition_phi->block;
142 }
143 if (dominates(ssa->cfg.blocks, def_block,
144 (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) ? b : ssa->cfg.blocks[b].loop_header)) {
145 return 0;
146 }
147
148 while (prev_use >= 0 && prev_use != use) {
149 if (b != ssa->cfg.map[prev_use]
150 && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use])
151 && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) {
152 return 0;
153 }
154 prev_use = zend_ssa_next_use(ssa->ops, var, prev_use);
155 }
156 }
157
158 next_use = zend_ssa_next_use(ssa->ops, var, use);
159 if (next_use < 0) {
160 return 1;
161 } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) {
162 return 1;
163 }
164 return 0;
165 }
166
zend_jit_is_constant_cmp_long_long(const zend_op * opline,zend_ssa_range * op1_range,zend_jit_addr op1_addr,zend_ssa_range * op2_range,zend_jit_addr op2_addr,bool * result)167 static int zend_jit_is_constant_cmp_long_long(const zend_op *opline,
168 zend_ssa_range *op1_range,
169 zend_jit_addr op1_addr,
170 zend_ssa_range *op2_range,
171 zend_jit_addr op2_addr,
172 bool *result)
173 {
174 zend_long op1_min;
175 zend_long op1_max;
176 zend_long op2_min;
177 zend_long op2_max;
178
179 if (op1_range) {
180 op1_min = op1_range->min;
181 op1_max = op1_range->max;
182 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
183 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG);
184 op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr));
185 } else {
186 return 0;
187 }
188
189 if (op2_range) {
190 op2_min = op2_range->min;
191 op2_max = op2_range->max;
192 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
193 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG);
194 op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr));
195 } else {
196 return 0;
197 }
198
199 switch (opline->opcode) {
200 case ZEND_IS_EQUAL:
201 case ZEND_IS_IDENTICAL:
202 case ZEND_CASE:
203 case ZEND_CASE_STRICT:
204 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
205 *result = 1;
206 return 1;
207 } else if (op1_max < op2_min || op1_min > op2_max) {
208 *result = 0;
209 return 1;
210 }
211 return 0;
212 case ZEND_IS_NOT_EQUAL:
213 case ZEND_IS_NOT_IDENTICAL:
214 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
215 *result = 0;
216 return 1;
217 } else if (op1_max < op2_min || op1_min > op2_max) {
218 *result = 1;
219 return 1;
220 }
221 return 0;
222 case ZEND_IS_SMALLER:
223 if (op1_max < op2_min) {
224 *result = 1;
225 return 1;
226 } else if (op1_min >= op2_max) {
227 *result = 0;
228 return 1;
229 }
230 return 0;
231 case ZEND_IS_SMALLER_OR_EQUAL:
232 if (op1_max <= op2_min) {
233 *result = 1;
234 return 1;
235 } else if (op1_min > op2_max) {
236 *result = 0;
237 return 1;
238 }
239 return 0;
240 default:
241 ZEND_UNREACHABLE();
242 }
243 return 0;
244 }
245
zend_jit_needs_call_chain(zend_call_info * call_info,uint32_t b,const zend_op_array * op_array,zend_ssa * ssa,const zend_ssa_op * ssa_op,const zend_op * opline,int call_level,zend_jit_trace_rec * trace)246 static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace)
247 {
248 int skip;
249
250 if (trace) {
251 zend_jit_trace_rec *p = trace;
252
253 ssa_op++;
254 while (1) {
255 if (p->op == ZEND_JIT_TRACE_VM) {
256 switch (p->opline->opcode) {
257 case ZEND_SEND_ARRAY:
258 case ZEND_SEND_USER:
259 case ZEND_SEND_UNPACK:
260 case ZEND_INIT_FCALL:
261 case ZEND_INIT_METHOD_CALL:
262 case ZEND_INIT_STATIC_METHOD_CALL:
263 case ZEND_INIT_FCALL_BY_NAME:
264 case ZEND_INIT_NS_FCALL_BY_NAME:
265 case ZEND_INIT_DYNAMIC_CALL:
266 case ZEND_NEW:
267 case ZEND_INIT_USER_CALL:
268 case ZEND_FAST_CALL:
269 case ZEND_JMP:
270 case ZEND_JMPZ:
271 case ZEND_JMPNZ:
272 case ZEND_JMPZ_EX:
273 case ZEND_JMPNZ_EX:
274 case ZEND_FE_RESET_R:
275 case ZEND_FE_RESET_RW:
276 case ZEND_JMP_SET:
277 case ZEND_COALESCE:
278 case ZEND_JMP_NULL:
279 case ZEND_ASSERT_CHECK:
280 case ZEND_CATCH:
281 case ZEND_DECLARE_ANON_CLASS:
282 case ZEND_FE_FETCH_R:
283 case ZEND_FE_FETCH_RW:
284 case ZEND_BIND_INIT_STATIC_OR_JMP:
285 case ZEND_JMP_FRAMELESS:
286 return 1;
287 case ZEND_DO_ICALL:
288 case ZEND_DO_UCALL:
289 case ZEND_DO_FCALL_BY_NAME:
290 case ZEND_DO_FCALL:
291 case ZEND_CALLABLE_CONVERT:
292 return 0;
293 case ZEND_SEND_VAL:
294 case ZEND_SEND_VAR:
295 case ZEND_SEND_VAL_EX:
296 case ZEND_SEND_VAR_EX:
297 case ZEND_SEND_FUNC_ARG:
298 case ZEND_SEND_REF:
299 case ZEND_SEND_VAR_NO_REF:
300 case ZEND_SEND_VAR_NO_REF_EX:
301 /* skip */
302 break;
303 default:
304 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
305 return 1;
306 }
307 }
308 ssa_op += zend_jit_trace_op_len(opline);
309 } else if (p->op == ZEND_JIT_TRACE_ENTER ||
310 p->op == ZEND_JIT_TRACE_BACK ||
311 p->op == ZEND_JIT_TRACE_END) {
312 return 1;
313 }
314 p++;
315 }
316 }
317
318 if (!call_info) {
319 const zend_op *end = op_array->opcodes + op_array->last;
320
321 opline++;
322 ssa_op++;
323 skip = (call_level == 1);
324 while (opline != end) {
325 if (!skip) {
326 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
327 return 1;
328 }
329 }
330 switch (opline->opcode) {
331 case ZEND_SEND_VAL:
332 case ZEND_SEND_VAR:
333 case ZEND_SEND_VAL_EX:
334 case ZEND_SEND_VAR_EX:
335 case ZEND_SEND_FUNC_ARG:
336 case ZEND_SEND_REF:
337 case ZEND_SEND_VAR_NO_REF:
338 case ZEND_SEND_VAR_NO_REF_EX:
339 skip = 0;
340 break;
341 case ZEND_SEND_ARRAY:
342 case ZEND_SEND_USER:
343 case ZEND_SEND_UNPACK:
344 case ZEND_INIT_FCALL:
345 case ZEND_INIT_METHOD_CALL:
346 case ZEND_INIT_STATIC_METHOD_CALL:
347 case ZEND_INIT_FCALL_BY_NAME:
348 case ZEND_INIT_NS_FCALL_BY_NAME:
349 case ZEND_INIT_DYNAMIC_CALL:
350 case ZEND_NEW:
351 case ZEND_INIT_USER_CALL:
352 case ZEND_FAST_CALL:
353 case ZEND_JMP:
354 case ZEND_JMPZ:
355 case ZEND_JMPNZ:
356 case ZEND_JMPZ_EX:
357 case ZEND_JMPNZ_EX:
358 case ZEND_FE_RESET_R:
359 case ZEND_FE_RESET_RW:
360 case ZEND_JMP_SET:
361 case ZEND_COALESCE:
362 case ZEND_JMP_NULL:
363 case ZEND_ASSERT_CHECK:
364 case ZEND_CATCH:
365 case ZEND_DECLARE_ANON_CLASS:
366 case ZEND_FE_FETCH_R:
367 case ZEND_FE_FETCH_RW:
368 case ZEND_BIND_INIT_STATIC_OR_JMP:
369 case ZEND_JMP_FRAMELESS:
370 return 1;
371 case ZEND_DO_ICALL:
372 case ZEND_DO_UCALL:
373 case ZEND_DO_FCALL_BY_NAME:
374 case ZEND_DO_FCALL:
375 case ZEND_CALLABLE_CONVERT:
376 end = opline;
377 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
378 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
379 return 1;
380 }
381 return 0;
382 }
383 opline++;
384 ssa_op++;
385 }
386
387 return 1;
388 } else {
389 const zend_op *end = call_info->caller_call_opline;
390
391 /* end may be null if an opcode like EXIT is part of the argument list. */
392 if (!end || end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
393 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
394 return 1;
395 }
396
397 opline++;
398 ssa_op++;
399 skip = (call_level == 1);
400 while (opline != end) {
401 if (skip) {
402 switch (opline->opcode) {
403 case ZEND_SEND_VAL:
404 case ZEND_SEND_VAR:
405 case ZEND_SEND_VAL_EX:
406 case ZEND_SEND_VAR_EX:
407 case ZEND_SEND_FUNC_ARG:
408 case ZEND_SEND_REF:
409 case ZEND_SEND_VAR_NO_REF:
410 case ZEND_SEND_VAR_NO_REF_EX:
411 skip = 0;
412 break;
413 case ZEND_SEND_ARRAY:
414 case ZEND_SEND_USER:
415 case ZEND_SEND_UNPACK:
416 return 1;
417 }
418 } else {
419 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
420 return 1;
421 }
422 }
423 opline++;
424 ssa_op++;
425 }
426
427 return 0;
428 }
429 }
430
skip_valid_arguments(const zend_op_array * op_array,zend_ssa * ssa,const zend_call_info * call_info)431 static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
432 {
433 uint32_t num_args = 0;
434 zend_function *func = call_info->callee_func;
435
436 /* It's okay to handle prototypes here, because they can only increase the accepted arguments.
437 * Anything legal for the parent method is also legal for the parent method. */
438 while (num_args < call_info->num_args) {
439 zend_arg_info *arg_info = func->op_array.arg_info + num_args;
440
441 if (ZEND_TYPE_IS_SET(arg_info->type)) {
442 if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) {
443 zend_op *opline = call_info->arg_info[num_args].opline;
444 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
445 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
446 if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) {
447 break;
448 }
449 } else {
450 break;
451 }
452 }
453 num_args++;
454 }
455 return num_args;
456 }
457
zend_ssa_cv_info(const zend_op_array * op_array,zend_ssa * ssa,uint32_t var)458 static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var)
459 {
460 uint32_t j, info;
461
462 if (ssa->vars && ssa->var_info) {
463 info = ssa->var_info[var].type;
464 for (j = op_array->last_var; j < ssa->vars_count; j++) {
465 if (ssa->vars[j].var == var) {
466 info |= ssa->var_info[j].type;
467 }
468 }
469 } else {
470 info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF |
471 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
472 }
473
474 #ifdef ZEND_JIT_USE_RC_INFERENCE
475 /* Refcount may be increased by RETURN opcode */
476 if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
477 for (j = 0; j < ssa->cfg.blocks_count; j++) {
478 if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
479 ssa->cfg.blocks[j].len > 0) {
480 const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
481
482 if (opline->opcode == ZEND_RETURN) {
483 if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) {
484 info |= MAY_BE_RCN;
485 break;
486 }
487 }
488 }
489 }
490 }
491 #endif
492
493 return info;
494 }
495
zend_jit_may_avoid_refcounting(const zend_op * opline,uint32_t op1_info)496 static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_info)
497 {
498 switch (opline->opcode) {
499 case ZEND_FETCH_OBJ_FUNC_ARG:
500 if (!JIT_G(current_frame) ||
501 !JIT_G(current_frame)->call->func ||
502 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
503 return 0;
504 }
505 /* break missing intentionally */
506 case ZEND_FETCH_OBJ_R:
507 case ZEND_FETCH_OBJ_IS:
508 if ((op1_info & MAY_BE_OBJECT)
509 && opline->op2_type == IS_CONST
510 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
511 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
512 return 1;
513 }
514 break;
515 case ZEND_FETCH_DIM_FUNC_ARG:
516 if (!JIT_G(current_frame) ||
517 !JIT_G(current_frame)->call->func ||
518 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
519 return 0;
520 }
521 /* break missing intentionally */
522 case ZEND_FETCH_DIM_R:
523 case ZEND_FETCH_DIM_IS:
524 return 1;
525 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
526 if (!(opline->extended_value & ZEND_ISEMPTY)) {
527 return 1;
528 }
529 break;
530 }
531 return 0;
532 }
533
zend_jit_is_persistent_constant(zval * key,uint32_t flags)534 static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags)
535 {
536 zval *zv;
537 zend_constant *c = NULL;
538
539 /* null/true/false are resolved during compilation, so don't check for them here. */
540 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
541 if (zv) {
542 c = (zend_constant*)Z_PTR_P(zv);
543 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
544 key++;
545 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
546 if (zv) {
547 c = (zend_constant*)Z_PTR_P(zv);
548 }
549 }
550 return c && (ZEND_CONSTANT_FLAGS(c) & CONST_PERSISTENT);
551 }
552
zend_get_known_property_info(const zend_op_array * op_array,zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)553 static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
554 {
555 zend_property_info *info = NULL;
556
557 if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) ||
558 !ce ||
559 !(ce->ce_flags & ZEND_ACC_LINKED) ||
560 (ce->ce_flags & ZEND_ACC_TRAIT) ||
561 ce->create_object) {
562 return NULL;
563 }
564
565 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
566 if (ce->info.user.filename != filename) {
567 /* class declaration might be changed independently */
568 return NULL;
569 }
570
571 if (ce->parent) {
572 zend_class_entry *parent = ce->parent;
573
574 do {
575 if (parent->type == ZEND_INTERNAL_CLASS) {
576 break;
577 } else if (parent->info.user.filename != filename) {
578 /* some of parents class declarations might be changed independently */
579 /* TODO: this check may be not enough, because even
580 * in the same it's possible to conditionally define
581 * few classes with the same name, and "parent" may
582 * change from request to request.
583 */
584 return NULL;
585 }
586 parent = parent->parent;
587 } while (parent);
588 }
589 }
590
591 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
592 if (info == NULL ||
593 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
594 (info->flags & ZEND_ACC_STATIC)) {
595 return NULL;
596 }
597
598 if (info->flags & ZEND_ACC_PUBLIC) {
599 return info;
600 } else if (on_this) {
601 if (ce == info->ce) {
602 if (ce == op_array->scope) {
603 return info;
604 } else {
605 return NULL;
606 }
607 } else if ((info->flags & ZEND_ACC_PROTECTED)
608 && instanceof_function_slow(ce, info->ce)) {
609 return info;
610 }
611 }
612
613 return NULL;
614 }
615
zend_may_be_dynamic_property(zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)616 static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
617 {
618 zend_property_info *info;
619
620 if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) {
621 return 1;
622 }
623
624 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
625 if (ce->info.user.filename != filename) {
626 /* class declaration might be changed independently */
627 return 1;
628 }
629 }
630
631 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
632 if (info == NULL ||
633 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
634 (info->flags & ZEND_ACC_STATIC)) {
635 return 1;
636 }
637
638 if (!(info->flags & ZEND_ACC_PUBLIC) &&
639 (!on_this || info->ce != ce)) {
640 return 1;
641 }
642
643 return 0;
644 }
645
646 #define OP_RANGE(ssa_op, opN) \
647 (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
648 ssa->var_info && \
649 (ssa_op)->opN##_use >= 0 && \
650 ssa->var_info[(ssa_op)->opN##_use].has_range) ? \
651 &ssa->var_info[(ssa_op)->opN##_use].range : NULL)
652
653 #define OP1_RANGE() OP_RANGE(ssa_op, op1)
654 #define OP2_RANGE() OP_RANGE(ssa_op, op2)
655 #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)
656
657 #include "jit/zend_jit_helpers.c"
658 #include "Zend/zend_cpuinfo.h"
659
660 #ifdef HAVE_GCC_GLOBAL_REGS
661 # define GCC_GLOBAL_REGS 1
662 #else
663 # define GCC_GLOBAL_REGS 0
664 #endif
665
666 /* By default avoid JITing inline handlers if it does not seem profitable due to lack of
667 * type information. Disabling this option allows testing some JIT handlers in the
668 * presence of try/catch blocks, which prevent SSA construction. */
669 #ifndef PROFITABILITY_CHECKS
670 # define PROFITABILITY_CHECKS 1
671 #endif
672
673 #define BP_JIT_IS 6 /* Used for ISSET_ISEMPTY_DIM_OBJ. see BP_VAR_*defines in Zend/zend_compile.h */
674
675 /* The generated code may contain tautological comparisons, ignore them. */
676 #if defined(__clang__)
677 # pragma clang diagnostic push
678 # pragma clang diagnostic ignored "-Wtautological-compare"
679 # pragma clang diagnostic ignored "-Wstring-compare"
680 #endif
681
682 #include "jit/zend_jit_ir.c"
683
684 #if defined(__clang__)
685 # pragma clang diagnostic pop
686 #endif
687
688 #if _WIN32
689 # include <Windows.h>
690 #else
691 # include <sys/mman.h>
692 # if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
693 # define MAP_ANONYMOUS MAP_ANON
694 # endif
695 #endif
696
zend_jit_status(zval * ret)697 ZEND_EXT_API void zend_jit_status(zval *ret)
698 {
699 zval stats;
700 array_init(&stats);
701 add_assoc_bool(&stats, "enabled", JIT_G(enabled));
702 add_assoc_bool(&stats, "on", JIT_G(on));
703 add_assoc_long(&stats, "kind", JIT_G(trigger));
704 add_assoc_long(&stats, "opt_level", JIT_G(opt_level));
705 add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags));
706 if (dasm_buf) {
707 add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
708 add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
709 } else {
710 add_assoc_long(&stats, "buffer_size", 0);
711 add_assoc_long(&stats, "buffer_free", 0);
712 }
713 add_assoc_zval(ret, "jit", &stats);
714 }
715
zend_jit_func_name(const zend_op_array * op_array)716 static zend_string *zend_jit_func_name(const zend_op_array *op_array)
717 {
718 smart_str buf = {0};
719
720 if (op_array->function_name) {
721 smart_str_appends(&buf, JIT_PREFIX);
722 if (op_array->scope) {
723 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
724 smart_str_appends(&buf, "::");
725 }
726 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
727 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
728 smart_str_appends(&buf, ":");
729 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
730 smart_str_appends(&buf, ":");
731 smart_str_append_long(&buf, op_array->line_start);
732 }
733 smart_str_0(&buf);
734 return buf.s;
735 } else if (op_array->filename) {
736 smart_str_appends(&buf, JIT_PREFIX);
737 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
738 smart_str_0(&buf);
739 return buf.s;
740 } else {
741 return NULL;
742 }
743 }
744
zend_may_overflow(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa)745 static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
746 {
747 int res;
748 zend_long op1_min, op1_max, op2_min, op2_max;
749
750 if (!ssa->ops || !ssa->var_info) {
751 return 1;
752 }
753 switch (opline->opcode) {
754 case ZEND_PRE_INC:
755 case ZEND_POST_INC:
756 res = ssa_op->op1_def;
757 if (res < 0
758 || !ssa->var_info[res].has_range
759 || ssa->var_info[res].range.overflow) {
760 if (!OP1_HAS_RANGE()) {
761 return 1;
762 }
763 op1_max = OP1_MAX_RANGE();
764 if (op1_max == ZEND_LONG_MAX) {
765 return 1;
766 }
767 }
768 return 0;
769 case ZEND_PRE_DEC:
770 case ZEND_POST_DEC:
771 res = ssa_op->op1_def;
772 if (res < 0
773 || !ssa->var_info[res].has_range
774 || ssa->var_info[res].range.underflow) {
775 if (!OP1_HAS_RANGE()) {
776 return 1;
777 }
778 op1_min = OP1_MIN_RANGE();
779 if (op1_min == ZEND_LONG_MIN) {
780 return 1;
781 }
782 }
783 return 0;
784 case ZEND_ADD:
785 res = ssa_op->result_def;
786 if (res < 0
787 || !ssa->var_info[res].has_range
788 || ssa->var_info[res].range.underflow) {
789 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
790 return 1;
791 }
792 op1_min = OP1_MIN_RANGE();
793 op2_min = OP2_MIN_RANGE();
794 if (zend_add_will_overflow(op1_min, op2_min)) {
795 return 1;
796 }
797 }
798 if (res < 0
799 || !ssa->var_info[res].has_range
800 || ssa->var_info[res].range.overflow) {
801 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
802 return 1;
803 }
804 op1_max = OP1_MAX_RANGE();
805 op2_max = OP2_MAX_RANGE();
806 if (zend_add_will_overflow(op1_max, op2_max)) {
807 return 1;
808 }
809 }
810 return 0;
811 case ZEND_SUB:
812 res = ssa_op->result_def;
813 if (res < 0
814 || !ssa->var_info[res].has_range
815 || ssa->var_info[res].range.underflow) {
816 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
817 return 1;
818 }
819 op1_min = OP1_MIN_RANGE();
820 op2_max = OP2_MAX_RANGE();
821 if (zend_sub_will_overflow(op1_min, op2_max)) {
822 return 1;
823 }
824 }
825 if (res < 0
826 || !ssa->var_info[res].has_range
827 || ssa->var_info[res].range.overflow) {
828 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
829 return 1;
830 }
831 op1_max = OP1_MAX_RANGE();
832 op2_min = OP2_MIN_RANGE();
833 if (zend_sub_will_overflow(op1_max, op2_min)) {
834 return 1;
835 }
836 }
837 return 0;
838 case ZEND_MUL:
839 res = ssa_op->result_def;
840 return (res < 0 ||
841 !ssa->var_info[res].has_range ||
842 ssa->var_info[res].range.underflow ||
843 ssa->var_info[res].range.overflow);
844 case ZEND_ASSIGN_OP:
845 if (opline->extended_value == ZEND_ADD) {
846 res = ssa_op->op1_def;
847 if (res < 0
848 || !ssa->var_info[res].has_range
849 || ssa->var_info[res].range.underflow) {
850 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
851 return 1;
852 }
853 op1_min = OP1_MIN_RANGE();
854 op2_min = OP2_MIN_RANGE();
855 if (zend_add_will_overflow(op1_min, op2_min)) {
856 return 1;
857 }
858 }
859 if (res < 0
860 || !ssa->var_info[res].has_range
861 || ssa->var_info[res].range.overflow) {
862 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
863 return 1;
864 }
865 op1_max = OP1_MAX_RANGE();
866 op2_max = OP2_MAX_RANGE();
867 if (zend_add_will_overflow(op1_max, op2_max)) {
868 return 1;
869 }
870 }
871 return 0;
872 } else if (opline->extended_value == ZEND_SUB) {
873 res = ssa_op->op1_def;
874 if (res < 0
875 || !ssa->var_info[res].has_range
876 || ssa->var_info[res].range.underflow) {
877 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
878 return 1;
879 }
880 op1_min = OP1_MIN_RANGE();
881 op2_max = OP2_MAX_RANGE();
882 if (zend_sub_will_overflow(op1_min, op2_max)) {
883 return 1;
884 }
885 }
886 if (res < 0
887 || !ssa->var_info[res].has_range
888 || ssa->var_info[res].range.overflow) {
889 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
890 return 1;
891 }
892 op1_max = OP1_MAX_RANGE();
893 op2_min = OP2_MIN_RANGE();
894 if (zend_sub_will_overflow(op1_max, op2_min)) {
895 return 1;
896 }
897 }
898 return 0;
899 } else if (opline->extended_value == ZEND_MUL) {
900 res = ssa_op->op1_def;
901 return (res < 0 ||
902 !ssa->var_info[res].has_range ||
903 ssa->var_info[res].range.underflow ||
904 ssa->var_info[res].range.overflow);
905 }
906 ZEND_FALLTHROUGH;
907 default:
908 return 1;
909 }
910 }
911
zend_jit_build_cfg(const zend_op_array * op_array,zend_cfg * cfg)912 static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg)
913 {
914 uint32_t flags;
915
916 flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY;
917
918 zend_build_cfg(&CG(arena), op_array, flags, cfg);
919
920 /* Don't JIT huge functions. Apart from likely being detrimental due to the amount of
921 * generated code, some of our analysis is recursive and will stack overflow with many
922 * blocks. */
923 if (cfg->blocks_count > 100000) {
924 return FAILURE;
925 }
926
927 zend_cfg_build_predecessors(&CG(arena), cfg);
928
929 /* Compute Dominators Tree */
930 zend_cfg_compute_dominators_tree(op_array, cfg);
931
932 /* Identify reducible and irreducible loops */
933 zend_cfg_identify_loops(op_array, cfg);
934
935 return SUCCESS;
936 }
937
zend_jit_op_array_analyze1(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa)938 static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
939 {
940 if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) {
941 return FAILURE;
942 }
943
944 #if 0
945 /* TODO: debugger and profiler supports? */
946 if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) {
947 return FAILURE;
948 }
949 #endif
950
951 /* TODO: move this to zend_cfg.c ? */
952 if (!op_array->function_name) {
953 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
954 }
955
956 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
957 && ssa->cfg.blocks
958 && op_array->last_try_catch == 0
959 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
960 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
961 if (zend_build_ssa(&CG(arena), script, op_array, ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) {
962 return FAILURE;
963 }
964
965 zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa);
966
967 zend_ssa_find_false_dependencies(op_array, ssa);
968
969 zend_ssa_find_sccs(op_array, ssa);
970 }
971
972 return SUCCESS;
973 }
974
zend_jit_op_array_analyze2(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa,uint32_t optimization_level)975 static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t optimization_level)
976 {
977 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
978 && ssa->cfg.blocks
979 && op_array->last_try_catch == 0
980 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
981 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
982 if (zend_ssa_inference(&CG(arena), op_array, script, ssa,
983 optimization_level & ~ZEND_OPTIMIZER_NARROW_TO_DOUBLE) != SUCCESS) {
984 return FAILURE;
985 }
986 }
987
988 return SUCCESS;
989 }
990
zend_jit_allocate_registers(zend_jit_ctx * ctx,const zend_op_array * op_array,zend_ssa * ssa)991 static void zend_jit_allocate_registers(zend_jit_ctx *ctx, const zend_op_array *op_array, zend_ssa *ssa)
992 {
993 void *checkpoint;
994 int candidates_count, i;
995 zend_jit_reg_var *ra;
996
997 checkpoint = zend_arena_checkpoint(CG(arena));
998 ra = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_jit_reg_var));
999 candidates_count = 0;
1000 for (i = 0; i < ssa->vars_count; i++) {
1001 if (zend_jit_may_be_in_reg(op_array, ssa, i)) {
1002 ra[i].ref = IR_NULL;
1003 candidates_count++;
1004 }
1005 }
1006 if (!candidates_count) {
1007 zend_arena_release(&CG(arena), checkpoint);
1008 return;
1009 }
1010
1011 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
1012 /* Naive SSA resolution */
1013 for (i = 0; i < ssa->vars_count; i++) {
1014 if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
1015 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
1016 int k, src;
1017
1018 if (phi->pi >= 0) {
1019 src = phi->sources[0];
1020 if (ra[i].ref) {
1021 if (!ra[src].ref) {
1022 ra[i].flags |= ZREG_LOAD;
1023 } else {
1024 ra[i].flags |= ZREG_PI;
1025 }
1026 } else if (ra[src].ref) {
1027 ra[src].flags |= ZREG_STORE;
1028 }
1029 } else {
1030 int need_move = 0;
1031
1032 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1033 src = phi->sources[k];
1034 if (src >= 0) {
1035 if (ssa->vars[src].definition_phi
1036 && ssa->vars[src].definition_phi->pi >= 0
1037 && phi->block == ssa->vars[src].definition_phi->block) {
1038 /* Skip zero-length interval for Pi variable */
1039 src = ssa->vars[src].definition_phi->sources[0];
1040 }
1041 if (ra[i].ref) {
1042 if (!ra[src].ref) {
1043 need_move = 1;
1044 }
1045 } else if (ra[src].ref) {
1046 need_move = 1;
1047 }
1048 }
1049 }
1050 if (need_move) {
1051 if (ra[i].ref) {
1052 ra[i].flags |= ZREG_LOAD;
1053 }
1054 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1055 src = phi->sources[k];
1056 if (src >= 0) {
1057 if (ssa->vars[src].definition_phi
1058 && ssa->vars[src].definition_phi->pi >= 0
1059 && phi->block == ssa->vars[src].definition_phi->block) {
1060 /* Skip zero-length interval for Pi variable */
1061 src = ssa->vars[src].definition_phi->sources[0];
1062 }
1063 if (ra[src].ref) {
1064 ra[src].flags |= ZREG_STORE;
1065 }
1066 }
1067 }
1068 } else {
1069 ra[i].flags |= ZREG_PHI;
1070 }
1071 }
1072 }
1073 }
1074
1075 /* Remove useless register allocation */
1076 for (i = 0; i < ssa->vars_count; i++) {
1077 if (ra[i].ref &&
1078 ((ra[i].flags & ZREG_LOAD) ||
1079 ((ra[i].flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) &&
1080 ssa->vars[i].use_chain < 0) {
1081 bool may_remove = 1;
1082 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
1083
1084 while (phi) {
1085 if (ra[phi->ssa_var].ref &&
1086 !(ra[phi->ssa_var].flags & ZREG_LOAD)) {
1087 may_remove = 0;
1088 break;
1089 }
1090 phi = zend_ssa_next_use_phi(ssa, i, phi);
1091 }
1092 if (may_remove) {
1093 ra[i].ref = IR_UNUSED;
1094 }
1095 }
1096 }
1097
1098 /* Remove intervals used once */
1099 for (i = 0; i < ssa->vars_count; i++) {
1100 if (ra[i].ref) {
1101 if (!(ra[i].flags & (ZREG_LOAD|ZREG_STORE))) {
1102 uint32_t var_num = ssa->vars[i].var;
1103 uint32_t op_num = ssa->vars[i].definition;
1104
1105 /* Check if a tempoary variable may be freed by exception handler */
1106 if (op_array->last_live_range
1107 && var_num >= op_array->last_var
1108 && ssa->vars[i].definition >= 0
1109 && ssa->ops[op_num].result_def == i) {
1110 const zend_live_range *range = op_array->live_range;
1111 int j;
1112
1113 op_num++;
1114 if (op_array->opcodes[op_num].opcode == ZEND_OP_DATA) {
1115 op_num++;
1116 }
1117 for (j = 0; j < op_array->last_live_range; range++, j++) {
1118 if (range->start > op_num) {
1119 /* further blocks will not be relevant... */
1120 break;
1121 } else if (op_num < range->end && var_num == (range->var & ~ZEND_LIVE_MASK)) {
1122 /* check if opcodes in range may throw */
1123 do {
1124 if (zend_may_throw(op_array->opcodes + op_num, ssa->ops + op_num, op_array, ssa)) {
1125 ra[i].flags |= ZREG_STORE;
1126 break;
1127 }
1128 op_num++;
1129 if (op_array->opcodes[op_num].opcode == ZEND_OP_DATA) {
1130 op_num++;
1131 }
1132 } while (op_num < range->end);
1133 break;
1134 }
1135 }
1136 }
1137 }
1138 if ((ra[i].flags & ZREG_LOAD)
1139 && (ra[i].flags & ZREG_STORE)
1140 && (ssa->vars[i].use_chain < 0
1141 || zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) {
1142 bool may_remove = 1;
1143 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
1144
1145 while (phi) {
1146 if (ra[phi->ssa_var].ref &&
1147 !(ra[phi->ssa_var].flags & ZREG_LOAD)) {
1148 may_remove = 0;
1149 break;
1150 }
1151 phi = zend_ssa_next_use_phi(ssa, i, phi);
1152 }
1153 if (may_remove) {
1154 ra[i].ref = IR_UNUSED;
1155 }
1156 }
1157 }
1158 }
1159 }
1160
1161 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
1162 fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
1163 for (i = 0; i < ssa->vars_count; i++) {
1164 if (ra[i].ref) {
1165 fprintf(stderr, "#%d.", i);
1166 uint32_t var_num = ssa->vars[i].var;
1167 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
1168 if (ra[i].flags & ZREG_LOAD) {
1169 fprintf(stderr, " load");
1170 }
1171 if (ra[i].flags & ZREG_STORE) {
1172 fprintf(stderr, " store");
1173 }
1174 fprintf(stderr, "\n");
1175 }
1176 }
1177 fprintf(stderr, "\n");
1178 }
1179
1180 ctx->ra = ra;
1181 }
1182
zend_jit_compute_post_order(zend_cfg * cfg,int start,int * post_order)1183 static int zend_jit_compute_post_order(zend_cfg *cfg, int start, int *post_order)
1184 {
1185 int count = 0;
1186 int b, n, *p;
1187 zend_basic_block *bb;
1188 zend_worklist worklist;
1189 ALLOCA_FLAG(use_heap)
1190
1191 ZEND_WORKLIST_ALLOCA(&worklist, cfg->blocks_count, use_heap);
1192 zend_worklist_push(&worklist, start);
1193
1194 while (zend_worklist_len(&worklist) != 0) {
1195 next:
1196 b = zend_worklist_peek(&worklist);
1197 bb = &cfg->blocks[b];
1198 n = bb->successors_count;
1199 if (n > 0) {
1200 p = bb->successors;
1201 do {
1202 if (cfg->blocks[*p].flags & (ZEND_BB_CATCH|ZEND_BB_FINALLY|ZEND_BB_FINALLY_END)) {
1203 /* skip */
1204 } else if (zend_worklist_push(&worklist, *p)) {
1205 goto next;
1206 }
1207 p++;
1208 n--;
1209 } while (n > 0);
1210 }
1211 zend_worklist_pop(&worklist);
1212 post_order[count++] = b;
1213 }
1214 ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
1215 return count;
1216 }
1217
zend_jit_next_is_send_result(const zend_op * opline)1218 static bool zend_jit_next_is_send_result(const zend_op *opline)
1219 {
1220 if (opline->result_type == IS_TMP_VAR
1221 && (opline+1)->opcode == ZEND_SEND_VAL
1222 && (opline+1)->op1_type == IS_TMP_VAR
1223 && (opline+1)->op2_type != IS_CONST
1224 && (opline+1)->op1.var == opline->result.var) {
1225 return 1;
1226 }
1227 return 0;
1228 }
1229
zend_jit_supported_binary_op(uint8_t op,uint32_t op1_info,uint32_t op2_info)1230 static bool zend_jit_supported_binary_op(uint8_t op, uint32_t op1_info, uint32_t op2_info)
1231 {
1232 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1233 return false;
1234 }
1235 switch (op) {
1236 case ZEND_POW:
1237 case ZEND_DIV:
1238 // TODO: check for division by zero ???
1239 return false;
1240 case ZEND_ADD:
1241 case ZEND_SUB:
1242 case ZEND_MUL:
1243 return (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
1244 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
1245 case ZEND_BW_OR:
1246 case ZEND_BW_AND:
1247 case ZEND_BW_XOR:
1248 case ZEND_SL:
1249 case ZEND_SR:
1250 case ZEND_MOD:
1251 return (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG);
1252 case ZEND_CONCAT:
1253 return (op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING);
1254 EMPTY_SWITCH_DEFAULT_CASE()
1255 }
1256 }
1257
zend_jit(const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline)1258 static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline)
1259 {
1260 int b, i, end;
1261 zend_op *opline;
1262 zend_jit_ctx ctx;
1263 zend_jit_ctx *jit = &ctx;
1264 zend_jit_reg_var *ra = NULL;
1265 void *handler;
1266 int call_level = 0;
1267 void *checkpoint = NULL;
1268 bool recv_emitted = 0; /* emitted at least one RECV opcode */
1269 uint8_t smart_branch_opcode;
1270 uint32_t target_label, target_label2;
1271 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info;
1272 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
1273 zend_class_entry *ce;
1274 bool ce_is_instanceof;
1275 bool on_this;
1276
1277 if (JIT_G(bisect_limit)) {
1278 jit_bisect_pos++;
1279 if (jit_bisect_pos >= JIT_G(bisect_limit)) {
1280 if (jit_bisect_pos == JIT_G(bisect_limit)) {
1281 fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n",
1282 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
1283 op_array->scope ? "::" : "",
1284 op_array->function_name ? ZSTR_VAL(op_array->function_name) : "{main}",
1285 ZSTR_VAL(op_array->filename), op_array->line_start);
1286 }
1287 return FAILURE;
1288 }
1289 }
1290
1291 if (ssa->cfg.flags & ZEND_FUNC_IRREDUCIBLE) {
1292 /* We can't order blocks properly */
1293 return FAILURE;
1294 }
1295
1296 if (rt_opline) {
1297 /* Set BB_ENTRY flag to limit register usage across the OSR ENTRY point */
1298 ssa->cfg.blocks[ssa->cfg.map[rt_opline - op_array->opcodes]].flags |= ZEND_BB_ENTRY;
1299 }
1300
1301 zend_jit_start(&ctx, op_array, ssa);
1302 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1303 checkpoint = zend_arena_checkpoint(CG(arena));
1304 zend_jit_allocate_registers(&ctx, op_array, ssa);
1305 ra = ctx.ra;
1306 }
1307
1308 /* Process blocks in Reverse Post Order */
1309 int *sorted_blocks = alloca(sizeof(int) * ssa->cfg.blocks_count);
1310 int n = zend_jit_compute_post_order(&ssa->cfg, 0, sorted_blocks);
1311
1312 while (n > 0) {
1313 b = sorted_blocks[--n];
1314 if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) {
1315 continue;
1316 }
1317
1318 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
1319 opline = op_array->opcodes + ssa->cfg.blocks[b].start;
1320 if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) {
1321 if (opline->opcode == ZEND_RECV_INIT) {
1322 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1323 if (opline != op_array->opcodes && (opline-1)->opcode != ZEND_RECV_INIT) {
1324 zend_jit_recv_entry(&ctx, b);
1325 }
1326 } else {
1327 if (opline != op_array->opcodes && recv_emitted) {
1328 zend_jit_recv_entry(&ctx, b);
1329 }
1330 }
1331 recv_emitted = 1;
1332 } else if (opline->opcode == ZEND_RECV) {
1333 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
1334 /* skip */
1335 zend_jit_bb_start(&ctx, b);
1336 zend_jit_bb_end(&ctx, b);
1337 continue;
1338 } else if (recv_emitted) {
1339 zend_jit_recv_entry(&ctx, b);
1340 } else {
1341 recv_emitted = 1;
1342 }
1343 } else {
1344 if (recv_emitted) {
1345 zend_jit_recv_entry(&ctx, b);
1346 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
1347 ssa->cfg.blocks[b].len == 1 &&
1348 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
1349 /* don't generate code for BB with single opcode */
1350 zend_jit_free_ctx(&ctx);
1351
1352 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1353 zend_arena_release(&CG(arena), checkpoint);
1354 }
1355 return SUCCESS;
1356 }
1357 }
1358 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
1359 ssa->cfg.blocks[b].len == 1 &&
1360 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
1361 /* don't generate code for BB with single opcode */
1362 zend_jit_free_ctx(&ctx);
1363
1364 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1365 zend_arena_release(&CG(arena), checkpoint);
1366 }
1367 return SUCCESS;
1368 }
1369 }
1370
1371 zend_jit_bb_start(&ctx, b);
1372
1373 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ctx.ra) {
1374 zend_ssa_phi *phi = ssa->blocks[b].phis;
1375
1376 /* First try to insert IR Phi */
1377 while (phi) {
1378 zend_jit_reg_var *ival = &ctx.ra[phi->ssa_var];
1379
1380 if (ival->ref) {
1381 if (ival->flags & ZREG_PI) {
1382 zend_jit_gen_pi(jit, phi);
1383 } else if (ival->flags & ZREG_PHI) {
1384 zend_jit_gen_phi(jit, phi);
1385 }
1386 }
1387 phi = phi->next;
1388 }
1389 }
1390
1391 if (rt_opline
1392 && (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) == 0
1393 && rt_opline == op_array->opcodes + ssa->cfg.blocks[b].start) {
1394 zend_jit_osr_entry(&ctx, b); /* OSR (On-Stack-Replacement) Entry-Point */
1395 }
1396
1397 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1398 if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
1399 && ssa->cfg.blocks[b].start != 0
1400 && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
1401 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_LONG
1402 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_STRING
1403 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_MATCH)) {
1404 zend_jit_reset_last_valid_opline(&ctx);
1405 } else {
1406 zend_jit_set_last_valid_opline(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start);
1407 }
1408 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
1409 zend_jit_reset_last_valid_opline(&ctx);
1410 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY) {
1411 zend_jit_reset_last_valid_opline(&ctx);
1412 } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) {
1413 zend_jit_set_last_valid_opline(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start);
1414 }
1415 if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) {
1416 if (!zend_jit_check_timeout(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start, NULL)) {
1417 goto jit_failure;
1418 }
1419 }
1420 if (!ssa->cfg.blocks[b].len) {
1421 zend_jit_bb_end(&ctx, b);
1422 continue;
1423 }
1424 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
1425 zend_ssa_phi *phi = ssa->blocks[b].phis;
1426
1427 while (phi) {
1428 zend_jit_reg_var *ival = &ra[phi->ssa_var];
1429
1430 if (ival->ref) {
1431 if (ival->flags & ZREG_LOAD) {
1432 ZEND_ASSERT(ival->ref == IR_NULL);
1433
1434 if (!zend_jit_load_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var)) {
1435 goto jit_failure;
1436 }
1437 } else if (ival->flags & ZREG_STORE) {
1438 ZEND_ASSERT(ival->ref != IR_NULL);
1439
1440 if (!zend_jit_store_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var, 1)) {
1441 goto jit_failure;
1442 }
1443 }
1444 }
1445 phi = phi->next;
1446 }
1447 }
1448 end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
1449 for (i = ssa->cfg.blocks[b].start; i <= end; i++) {
1450 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[i] : NULL;
1451 opline = op_array->opcodes + i;
1452 switch (opline->opcode) {
1453 case ZEND_INIT_FCALL:
1454 case ZEND_INIT_FCALL_BY_NAME:
1455 case ZEND_INIT_NS_FCALL_BY_NAME:
1456 case ZEND_INIT_METHOD_CALL:
1457 case ZEND_INIT_DYNAMIC_CALL:
1458 case ZEND_INIT_STATIC_METHOD_CALL:
1459 case ZEND_INIT_USER_CALL:
1460 case ZEND_NEW:
1461 call_level++;
1462 }
1463
1464 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
1465 switch (opline->opcode) {
1466 case ZEND_PRE_INC:
1467 case ZEND_PRE_DEC:
1468 case ZEND_POST_INC:
1469 case ZEND_POST_DEC:
1470 if (opline->op1_type != IS_CV) {
1471 break;
1472 }
1473 op1_info = OP1_INFO();
1474 if (!(op1_info & MAY_BE_LONG)) {
1475 break;
1476 }
1477 if (opline->result_type != IS_UNUSED) {
1478 res_use_info = -1;
1479
1480 if (opline->result_type == IS_CV
1481 && ssa->vars
1482 && ssa_op->result_use >= 0
1483 && !ssa->vars[ssa_op->result_use].no_val) {
1484 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1485
1486 if (Z_MODE(res_use_addr) != IS_REG
1487 || Z_LOAD(res_use_addr)
1488 || Z_STORE(res_use_addr)) {
1489 res_use_info = RES_USE_INFO();
1490 }
1491 }
1492 res_info = RES_INFO();
1493 res_addr = RES_REG_ADDR();
1494 } else {
1495 res_use_info = -1;
1496 res_info = -1;
1497 res_addr = 0;
1498 }
1499 op1_def_info = OP1_DEF_INFO();
1500 if (!zend_jit_inc_dec(&ctx, opline,
1501 op1_info, OP1_REG_ADDR(),
1502 op1_def_info, OP1_DEF_REG_ADDR(),
1503 res_use_info, res_info,
1504 res_addr,
1505 (op1_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1506 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1507 goto jit_failure;
1508 }
1509 goto done;
1510 case ZEND_BW_OR:
1511 case ZEND_BW_AND:
1512 case ZEND_BW_XOR:
1513 case ZEND_SL:
1514 case ZEND_SR:
1515 case ZEND_MOD:
1516 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1517 break;
1518 }
1519 op1_info = OP1_INFO();
1520 op2_info = OP2_INFO();
1521 if (!(op1_info & MAY_BE_LONG)
1522 || !(op2_info & MAY_BE_LONG)) {
1523 break;
1524 }
1525 res_addr = RES_REG_ADDR();
1526 if (Z_MODE(res_addr) != IS_REG
1527 && (i + 1) <= end
1528 && zend_jit_next_is_send_result(opline)) {
1529 i++;
1530 res_use_info = -1;
1531 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1532 if (!zend_jit_reuse_ip(&ctx)) {
1533 goto jit_failure;
1534 }
1535 } else {
1536 res_use_info = -1;
1537
1538 if (opline->result_type == IS_CV
1539 && ssa->vars
1540 && ssa_op->result_use >= 0
1541 && !ssa->vars[ssa_op->result_use].no_val) {
1542 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1543
1544 if (Z_MODE(res_use_addr) != IS_REG
1545 || Z_LOAD(res_use_addr)
1546 || Z_STORE(res_use_addr)) {
1547 res_use_info = RES_USE_INFO();
1548 }
1549 }
1550 }
1551 if (!zend_jit_long_math(&ctx, opline,
1552 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
1553 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
1554 res_use_info, RES_INFO(), res_addr,
1555 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1556 goto jit_failure;
1557 }
1558 goto done;
1559 case ZEND_ADD:
1560 case ZEND_SUB:
1561 case ZEND_MUL:
1562 // case ZEND_DIV: // TODO: check for division by zero ???
1563 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1564 break;
1565 }
1566 op1_info = OP1_INFO();
1567 op2_info = OP2_INFO();
1568 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1569 break;
1570 }
1571 if (opline->opcode == ZEND_ADD &&
1572 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
1573 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
1574 /* pass */
1575 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
1576 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1577 break;
1578 }
1579 res_addr = RES_REG_ADDR();
1580 if (Z_MODE(res_addr) != IS_REG
1581 && (i + 1) <= end
1582 && zend_jit_next_is_send_result(opline)) {
1583 i++;
1584 res_use_info = -1;
1585 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1586 if (!zend_jit_reuse_ip(&ctx)) {
1587 goto jit_failure;
1588 }
1589 } else {
1590 res_use_info = -1;
1591
1592 if (opline->result_type == IS_CV
1593 && ssa->vars
1594 && ssa_op->result_use >= 0
1595 && !ssa->vars[ssa_op->result_use].no_val) {
1596 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1597
1598 if (Z_MODE(res_use_addr) != IS_REG
1599 || Z_LOAD(res_use_addr)
1600 || Z_STORE(res_use_addr)) {
1601 res_use_info = RES_USE_INFO();
1602 }
1603 }
1604 }
1605 res_info = RES_INFO();
1606 if (opline->opcode == ZEND_ADD &&
1607 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
1608 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
1609 if (!zend_jit_add_arrays(&ctx, opline, op1_info, OP1_REG_ADDR(), op2_info, OP2_REG_ADDR(), res_addr)) {
1610 goto jit_failure;
1611 }
1612 } else {
1613 if (!zend_jit_math(&ctx, opline,
1614 op1_info, OP1_REG_ADDR(),
1615 op2_info, OP2_REG_ADDR(),
1616 res_use_info, res_info, res_addr,
1617 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1618 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1619 goto jit_failure;
1620 }
1621 }
1622 goto done;
1623 case ZEND_CONCAT:
1624 case ZEND_FAST_CONCAT:
1625 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1626 break;
1627 }
1628 op1_info = OP1_INFO();
1629 op2_info = OP2_INFO();
1630 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1631 break;
1632 }
1633 if (!(op1_info & MAY_BE_STRING) ||
1634 !(op2_info & MAY_BE_STRING)) {
1635 break;
1636 }
1637 res_addr = RES_REG_ADDR();
1638 if ((i + 1) <= end
1639 && zend_jit_next_is_send_result(opline)) {
1640 i++;
1641 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1642 if (!zend_jit_reuse_ip(&ctx)) {
1643 goto jit_failure;
1644 }
1645 }
1646 if (!zend_jit_concat(&ctx, opline,
1647 op1_info, op2_info, res_addr,
1648 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1649 goto jit_failure;
1650 }
1651 goto done;
1652 case ZEND_ASSIGN_OP:
1653 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1654 break;
1655 }
1656 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1657 break;
1658 }
1659 op1_info = OP1_INFO();
1660 op2_info = OP2_INFO();
1661 if (!zend_jit_supported_binary_op(
1662 opline->extended_value, op1_info, op2_info)) {
1663 break;
1664 }
1665 op1_def_info = OP1_DEF_INFO();
1666 if (!zend_jit_assign_op(&ctx, opline,
1667 op1_info, op1_def_info, OP1_RANGE(),
1668 op2_info, OP2_RANGE(),
1669 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1670 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1671 goto jit_failure;
1672 }
1673 goto done;
1674 case ZEND_ASSIGN_DIM_OP:
1675 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1676 break;
1677 }
1678 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1679 break;
1680 }
1681 if (!zend_jit_supported_binary_op(
1682 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
1683 break;
1684 }
1685 if (!zend_jit_assign_dim_op(&ctx, opline,
1686 OP1_INFO(), OP1_DEF_INFO(), OP1_REG_ADDR(), OP2_INFO(),
1687 OP1_DATA_INFO(), OP1_DATA_RANGE(), IS_UNKNOWN,
1688 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1689 goto jit_failure;
1690 }
1691 goto done;
1692 case ZEND_ASSIGN_DIM:
1693 if (opline->op1_type != IS_CV) {
1694 break;
1695 }
1696 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1697 break;
1698 }
1699 if (!zend_jit_assign_dim(&ctx, opline,
1700 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), OP1_DATA_INFO(), IS_UNKNOWN,
1701 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1702 goto jit_failure;
1703 }
1704 goto done;
1705 case ZEND_PRE_INC_OBJ:
1706 case ZEND_PRE_DEC_OBJ:
1707 case ZEND_POST_INC_OBJ:
1708 case ZEND_POST_DEC_OBJ:
1709 if (opline->op2_type != IS_CONST
1710 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1711 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1712 break;
1713 }
1714 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1715 break;
1716 }
1717 ce = NULL;
1718 ce_is_instanceof = 0;
1719 on_this = 0;
1720 if (opline->op1_type == IS_UNUSED) {
1721 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1722 ce = op_array->scope;
1723 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
1724 op1_addr = 0;
1725 on_this = 1;
1726 } else {
1727 op1_info = OP1_INFO();
1728 if (!(op1_info & MAY_BE_OBJECT)) {
1729 break;
1730 }
1731 op1_addr = OP1_REG_ADDR();
1732 if (ssa->var_info && ssa->ops) {
1733 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1734 if (ssa_op->op1_use >= 0) {
1735 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1736 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1737 ce = op1_ssa->ce;
1738 ce_is_instanceof = op1_ssa->is_instanceof;
1739 }
1740 }
1741 }
1742 }
1743 if (!zend_jit_incdec_obj(&ctx, opline, op_array, ssa, ssa_op,
1744 op1_info, op1_addr,
1745 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
1746 goto jit_failure;
1747 }
1748 goto done;
1749 case ZEND_ASSIGN_OBJ_OP:
1750 if (opline->result_type != IS_UNUSED) {
1751 break;
1752 }
1753 if (opline->op2_type != IS_CONST
1754 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1755 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1756 break;
1757 }
1758 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1759 break;
1760 }
1761 if (!zend_jit_supported_binary_op(
1762 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
1763 break;
1764 }
1765 ce = NULL;
1766 ce_is_instanceof = 0;
1767 on_this = 0;
1768 if (opline->op1_type == IS_UNUSED) {
1769 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1770 ce = op_array->scope;
1771 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
1772 op1_addr = 0;
1773 on_this = 1;
1774 } else {
1775 op1_info = OP1_INFO();
1776 if (!(op1_info & MAY_BE_OBJECT)) {
1777 break;
1778 }
1779 op1_addr = OP1_REG_ADDR();
1780 if (ssa->var_info && ssa->ops) {
1781 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1782 if (ssa_op->op1_use >= 0) {
1783 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1784 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1785 ce = op1_ssa->ce;
1786 ce_is_instanceof = op1_ssa->is_instanceof;
1787 }
1788 }
1789 }
1790 }
1791 if (!zend_jit_assign_obj_op(&ctx, opline, op_array, ssa, ssa_op,
1792 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_RANGE(),
1793 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
1794 goto jit_failure;
1795 }
1796 goto done;
1797 case ZEND_ASSIGN_OBJ:
1798 if (opline->op2_type != IS_CONST
1799 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1800 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1801 break;
1802 }
1803 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1804 break;
1805 }
1806 ce = NULL;
1807 ce_is_instanceof = 0;
1808 on_this = 0;
1809 if (opline->op1_type == IS_UNUSED) {
1810 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1811 ce = op_array->scope;
1812 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
1813 op1_addr = 0;
1814 on_this = 1;
1815 } else {
1816 op1_info = OP1_INFO();
1817 if (!(op1_info & MAY_BE_OBJECT)) {
1818 break;
1819 }
1820 op1_addr = OP1_REG_ADDR();
1821 if (ssa->var_info && ssa->ops) {
1822 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1823 if (ssa_op->op1_use >= 0) {
1824 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1825 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1826 ce = op1_ssa->ce;
1827 ce_is_instanceof = op1_ssa->is_instanceof;
1828 }
1829 }
1830 }
1831 }
1832 if (!zend_jit_assign_obj(&ctx, opline, op_array, ssa, ssa_op,
1833 op1_info, op1_addr, OP1_DATA_INFO(),
1834 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN,
1835 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1836 goto jit_failure;
1837 }
1838 goto done;
1839 case ZEND_ASSIGN:
1840 if (opline->op1_type != IS_CV) {
1841 break;
1842 }
1843 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1844 break;
1845 }
1846 op2_addr = OP2_REG_ADDR();
1847 if (ra
1848 && ssa->ops[opline - op_array->opcodes].op2_def >= 0
1849 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) {
1850 op2_def_addr = OP2_DEF_REG_ADDR();
1851 } else {
1852 op2_def_addr = op2_addr;
1853 }
1854 op1_info = OP1_INFO();
1855 if (ra && ssa->vars[ssa_op->op1_use].no_val) {
1856 op1_info |= MAY_BE_UNDEF; // requres type assignment
1857 }
1858 if (opline->result_type == IS_UNUSED) {
1859 res_addr = 0;
1860 res_info = -1;
1861 } else {
1862 res_addr = RES_REG_ADDR();
1863 res_info = RES_INFO();
1864 if (Z_MODE(res_addr) != IS_REG
1865 && (i + 1) <= end
1866 && zend_jit_next_is_send_result(opline)
1867 && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) {
1868 i++;
1869 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1870 if (!zend_jit_reuse_ip(&ctx)) {
1871 goto jit_failure;
1872 }
1873 }
1874 }
1875 if (!zend_jit_assign(&ctx, opline,
1876 op1_info, OP1_REG_ADDR(),
1877 OP1_DEF_INFO(), OP1_DEF_REG_ADDR(),
1878 OP2_INFO(), op2_addr, op2_def_addr,
1879 res_info, res_addr,
1880 0,
1881 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1882 goto jit_failure;
1883 }
1884 goto done;
1885 case ZEND_QM_ASSIGN:
1886 op1_addr = OP1_REG_ADDR();
1887 if (ra
1888 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
1889 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
1890 op1_def_addr = OP1_DEF_REG_ADDR();
1891 } else {
1892 op1_def_addr = op1_addr;
1893 }
1894 if (!zend_jit_qm_assign(&ctx, opline,
1895 OP1_INFO(), op1_addr, op1_def_addr,
1896 -1, RES_INFO(), RES_REG_ADDR())) {
1897 goto jit_failure;
1898 }
1899 goto done;
1900 case ZEND_INIT_FCALL:
1901 case ZEND_INIT_FCALL_BY_NAME:
1902 case ZEND_INIT_NS_FCALL_BY_NAME:
1903 if (!zend_jit_init_fcall(&ctx, opline, b, op_array, ssa, ssa_op, call_level, NULL, 0)) {
1904 goto jit_failure;
1905 }
1906 goto done;
1907 case ZEND_SEND_VAL:
1908 case ZEND_SEND_VAL_EX:
1909 if (opline->op2_type == IS_CONST) {
1910 /* Named parameters not supported in JIT (yet) */
1911 break;
1912 }
1913 if (opline->opcode == ZEND_SEND_VAL_EX
1914 && opline->op2.num > MAX_ARG_FLAG_NUM) {
1915 break;
1916 }
1917 if (!zend_jit_send_val(&ctx, opline,
1918 OP1_INFO(), OP1_REG_ADDR())) {
1919 goto jit_failure;
1920 }
1921 goto done;
1922 case ZEND_SEND_REF:
1923 if (opline->op2_type == IS_CONST) {
1924 /* Named parameters not supported in JIT (yet) */
1925 break;
1926 }
1927 if (!zend_jit_send_ref(&ctx, opline, op_array,
1928 OP1_INFO(), 0)) {
1929 goto jit_failure;
1930 }
1931 goto done;
1932 case ZEND_SEND_VAR:
1933 case ZEND_SEND_VAR_EX:
1934 case ZEND_SEND_VAR_NO_REF:
1935 case ZEND_SEND_VAR_NO_REF_EX:
1936 case ZEND_SEND_FUNC_ARG:
1937 if (opline->op2_type == IS_CONST) {
1938 /* Named parameters not supported in JIT (yet) */
1939 break;
1940 }
1941 if ((opline->opcode == ZEND_SEND_VAR_EX
1942 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
1943 && opline->op2.num > MAX_ARG_FLAG_NUM) {
1944 break;
1945 }
1946 op1_addr = OP1_REG_ADDR();
1947 if (ra
1948 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
1949 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
1950 op1_def_addr = OP1_DEF_REG_ADDR();
1951 } else {
1952 op1_def_addr = op1_addr;
1953 }
1954 if (!zend_jit_send_var(&ctx, opline, op_array,
1955 OP1_INFO(), op1_addr, op1_def_addr)) {
1956 goto jit_failure;
1957 }
1958 goto done;
1959 case ZEND_CHECK_FUNC_ARG:
1960 if (opline->op2_type == IS_CONST) {
1961 /* Named parameters not supported in JIT (yet) */
1962 break;
1963 }
1964 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
1965 break;
1966 }
1967 if (!zend_jit_check_func_arg(&ctx, opline)) {
1968 goto jit_failure;
1969 }
1970 goto done;
1971 case ZEND_CHECK_UNDEF_ARGS:
1972 if (!zend_jit_check_undef_args(&ctx, opline)) {
1973 goto jit_failure;
1974 }
1975 goto done;
1976 case ZEND_DO_UCALL:
1977 ZEND_FALLTHROUGH;
1978 case ZEND_DO_ICALL:
1979 case ZEND_DO_FCALL_BY_NAME:
1980 case ZEND_DO_FCALL:
1981 if (!zend_jit_do_fcall(&ctx, opline, op_array, ssa, call_level, b + 1, NULL)) {
1982 goto jit_failure;
1983 }
1984 goto done;
1985 case ZEND_IS_EQUAL:
1986 case ZEND_IS_NOT_EQUAL:
1987 case ZEND_IS_SMALLER:
1988 case ZEND_IS_SMALLER_OR_EQUAL:
1989 case ZEND_CASE: {
1990 res_addr = RES_REG_ADDR();
1991 if ((opline->result_type & IS_TMP_VAR)
1992 && (i + 1) <= end
1993 && ((opline+1)->opcode == ZEND_JMPZ
1994 || (opline+1)->opcode == ZEND_JMPNZ
1995 || (opline+1)->opcode == ZEND_JMPZ_EX
1996 || (opline+1)->opcode == ZEND_JMPNZ_EX)
1997 && (opline+1)->op1_type == IS_TMP_VAR
1998 && (opline+1)->op1.var == opline->result.var) {
1999 i++;
2000 smart_branch_opcode = (opline+1)->opcode;
2001 target_label = ssa->cfg.blocks[b].successors[0];
2002 target_label2 = ssa->cfg.blocks[b].successors[1];
2003 /* For EX variant write into the result of EX opcode. */
2004 if ((opline+1)->opcode == ZEND_JMPZ_EX
2005 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
2006 res_addr = OP_REG_ADDR(opline + 1, result_type, result, result_def);
2007 }
2008 } else {
2009 smart_branch_opcode = 0;
2010 target_label = target_label2 = (uint32_t)-1;
2011 }
2012 if (!zend_jit_cmp(&ctx, opline,
2013 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
2014 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
2015 res_addr,
2016 zend_may_throw(opline, ssa_op, op_array, ssa),
2017 smart_branch_opcode, target_label, target_label2,
2018 NULL, 0)) {
2019 goto jit_failure;
2020 }
2021 goto done;
2022 }
2023 case ZEND_IS_IDENTICAL:
2024 case ZEND_IS_NOT_IDENTICAL:
2025 case ZEND_CASE_STRICT:
2026 res_addr = RES_REG_ADDR();
2027 if ((opline->result_type & IS_TMP_VAR)
2028 && (i + 1) <= end
2029 && ((opline+1)->opcode == ZEND_JMPZ
2030 || (opline+1)->opcode == ZEND_JMPZ_EX
2031 || (opline+1)->opcode == ZEND_JMPNZ_EX
2032 || (opline+1)->opcode == ZEND_JMPNZ)
2033 && (opline+1)->op1_type == IS_TMP_VAR
2034 && (opline+1)->op1.var == opline->result.var) {
2035 i++;
2036 smart_branch_opcode = (opline+1)->opcode;
2037 target_label = ssa->cfg.blocks[b].successors[0];
2038 target_label2 = ssa->cfg.blocks[b].successors[1];
2039 /* For EX variant write into the result of EX opcode. */
2040 if ((opline+1)->opcode == ZEND_JMPZ_EX
2041 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
2042 res_addr = OP_REG_ADDR(opline + 1, result_type, result, result_def);
2043 }
2044 } else {
2045 smart_branch_opcode = 0;
2046 target_label = target_label2 = (uint32_t)-1;
2047 }
2048 if (!zend_jit_identical(&ctx, opline,
2049 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
2050 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
2051 res_addr,
2052 zend_may_throw(opline, ssa_op, op_array, ssa),
2053 smart_branch_opcode, target_label, target_label2,
2054 NULL, 0)) {
2055 goto jit_failure;
2056 }
2057 goto done;
2058 case ZEND_DEFINED:
2059 if ((opline->result_type & IS_TMP_VAR)
2060 && (i + 1) <= end
2061 && ((opline+1)->opcode == ZEND_JMPZ
2062 || (opline+1)->opcode == ZEND_JMPNZ)
2063 && (opline+1)->op1_type == IS_TMP_VAR
2064 && (opline+1)->op1.var == opline->result.var) {
2065 i++;
2066 smart_branch_opcode = (opline+1)->opcode;
2067 target_label = ssa->cfg.blocks[b].successors[0];
2068 target_label2 = ssa->cfg.blocks[b].successors[1];
2069 } else {
2070 smart_branch_opcode = 0;
2071 target_label = target_label2 = (uint32_t)-1;
2072 }
2073 if (!zend_jit_defined(&ctx, opline, smart_branch_opcode, target_label, target_label2, NULL)) {
2074 goto jit_failure;
2075 }
2076 goto done;
2077 case ZEND_TYPE_CHECK:
2078 if (opline->extended_value == MAY_BE_RESOURCE) {
2079 // TODO: support for is_resource() ???
2080 break;
2081 }
2082 if ((opline->result_type & IS_TMP_VAR)
2083 && (i + 1) <= end
2084 && ((opline+1)->opcode == ZEND_JMPZ
2085 || (opline+1)->opcode == ZEND_JMPNZ)
2086 && (opline+1)->op1_type == IS_TMP_VAR
2087 && (opline+1)->op1.var == opline->result.var) {
2088 i++;
2089 smart_branch_opcode = (opline+1)->opcode;
2090 target_label = ssa->cfg.blocks[b].successors[0];
2091 target_label2 = ssa->cfg.blocks[b].successors[1];
2092 } else {
2093 smart_branch_opcode = 0;
2094 target_label = target_label2 = (uint32_t)-1;
2095 }
2096 if (!zend_jit_type_check(&ctx, opline, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) {
2097 goto jit_failure;
2098 }
2099 goto done;
2100 case ZEND_RETURN:
2101 op1_info = OP1_INFO();
2102 if ((PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info))
2103 || op_array->type == ZEND_EVAL_CODE
2104 // TODO: support for top-level code
2105 || !op_array->function_name
2106 // TODO: support for IS_UNDEF ???
2107 || (op1_info & MAY_BE_UNDEF)) {
2108 if (!zend_jit_tail_handler(&ctx, opline)) {
2109 goto jit_failure;
2110 }
2111 } else {
2112 if (!zend_jit_return(&ctx, opline, op_array,
2113 op1_info, OP1_REG_ADDR())) {
2114 goto jit_failure;
2115 }
2116 }
2117 goto done;
2118 case ZEND_BOOL:
2119 case ZEND_BOOL_NOT:
2120 if (!zend_jit_bool_jmpznz(&ctx, opline,
2121 OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(),
2122 -1, -1,
2123 zend_may_throw(opline, ssa_op, op_array, ssa),
2124 opline->opcode, NULL)) {
2125 goto jit_failure;
2126 }
2127 goto done;
2128 case ZEND_JMPZ:
2129 case ZEND_JMPNZ:
2130 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
2131 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2132 /* smart branch */
2133 if (!zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2134 goto jit_failure;
2135 }
2136 goto done;
2137 }
2138 ZEND_FALLTHROUGH;
2139 case ZEND_JMPZ_EX:
2140 case ZEND_JMPNZ_EX:
2141 if (opline->result_type == IS_UNDEF) {
2142 res_addr = 0;
2143 } else {
2144 res_addr = RES_REG_ADDR();
2145 }
2146 if (!zend_jit_bool_jmpznz(&ctx, opline,
2147 OP1_INFO(), OP1_REG_ADDR(), res_addr,
2148 ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1],
2149 zend_may_throw(opline, ssa_op, op_array, ssa),
2150 opline->opcode, NULL)) {
2151 goto jit_failure;
2152 }
2153 goto done;
2154 case ZEND_ISSET_ISEMPTY_CV:
2155 if ((opline->extended_value & ZEND_ISEMPTY)) {
2156 // TODO: support for empty() ???
2157 break;
2158 }
2159 if ((opline->result_type & IS_TMP_VAR)
2160 && (i + 1) <= end
2161 && ((opline+1)->opcode == ZEND_JMPZ
2162 || (opline+1)->opcode == ZEND_JMPNZ)
2163 && (opline+1)->op1_type == IS_TMP_VAR
2164 && (opline+1)->op1.var == opline->result.var) {
2165 i++;
2166 smart_branch_opcode = (opline+1)->opcode;
2167 target_label = ssa->cfg.blocks[b].successors[0];
2168 target_label2 = ssa->cfg.blocks[b].successors[1];
2169 } else {
2170 smart_branch_opcode = 0;
2171 target_label = target_label2 = (uint32_t)-1;
2172 }
2173 if (!zend_jit_isset_isempty_cv(&ctx, opline,
2174 OP1_INFO(), OP1_REG_ADDR(),
2175 smart_branch_opcode, target_label, target_label2,
2176 NULL)) {
2177 goto jit_failure;
2178 }
2179 goto done;
2180 case ZEND_IN_ARRAY:
2181 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
2182 break;
2183 }
2184 op1_info = OP1_INFO();
2185 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
2186 break;
2187 }
2188 if ((opline->result_type & IS_TMP_VAR)
2189 && (i + 1) <= end
2190 && ((opline+1)->opcode == ZEND_JMPZ
2191 || (opline+1)->opcode == ZEND_JMPNZ)
2192 && (opline+1)->op1_type == IS_TMP_VAR
2193 && (opline+1)->op1.var == opline->result.var) {
2194 i++;
2195 smart_branch_opcode = (opline+1)->opcode;
2196 target_label = ssa->cfg.blocks[b].successors[0];
2197 target_label2 = ssa->cfg.blocks[b].successors[1];
2198 } else {
2199 smart_branch_opcode = 0;
2200 target_label = target_label2 = (uint32_t)-1;
2201 }
2202 if (!zend_jit_in_array(&ctx, opline,
2203 op1_info, OP1_REG_ADDR(),
2204 smart_branch_opcode, target_label, target_label2,
2205 NULL)) {
2206 goto jit_failure;
2207 }
2208 goto done;
2209 case ZEND_FETCH_DIM_R:
2210 case ZEND_FETCH_DIM_IS:
2211 case ZEND_FETCH_LIST_R:
2212 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2213 break;
2214 }
2215 if (!zend_jit_fetch_dim_read(&ctx, opline, ssa, ssa_op,
2216 OP1_INFO(), OP1_REG_ADDR(), 0,
2217 OP2_INFO(), RES_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
2218 goto jit_failure;
2219 }
2220 goto done;
2221 case ZEND_FETCH_DIM_W:
2222 case ZEND_FETCH_DIM_RW:
2223 // case ZEND_FETCH_DIM_UNSET:
2224 case ZEND_FETCH_LIST_W:
2225 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2226 break;
2227 }
2228 if (opline->op1_type != IS_CV) {
2229 break;
2230 }
2231 if (!zend_jit_fetch_dim(&ctx, opline,
2232 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
2233 goto jit_failure;
2234 }
2235 goto done;
2236 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
2237 if ((opline->extended_value & ZEND_ISEMPTY)) {
2238 // TODO: support for empty() ???
2239 break;
2240 }
2241 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2242 break;
2243 }
2244 if ((opline->result_type & IS_TMP_VAR)
2245 && (i + 1) <= end
2246 && ((opline+1)->opcode == ZEND_JMPZ
2247 || (opline+1)->opcode == ZEND_JMPNZ)
2248 && (opline+1)->op1_type == IS_TMP_VAR
2249 && (opline+1)->op1.var == opline->result.var) {
2250 i++;
2251 smart_branch_opcode = (opline+1)->opcode;
2252 target_label = ssa->cfg.blocks[b].successors[0];
2253 target_label2 = ssa->cfg.blocks[b].successors[1];
2254 } else {
2255 smart_branch_opcode = 0;
2256 target_label = target_label2 = (uint32_t)-1;
2257 }
2258 if (!zend_jit_isset_isempty_dim(&ctx, opline,
2259 OP1_INFO(), OP1_REG_ADDR(), 0,
2260 OP2_INFO(), IS_UNKNOWN,
2261 zend_may_throw(opline, ssa_op, op_array, ssa),
2262 smart_branch_opcode, target_label, target_label2,
2263 NULL)) {
2264 goto jit_failure;
2265 }
2266 goto done;
2267 case ZEND_FETCH_OBJ_R:
2268 case ZEND_FETCH_OBJ_IS:
2269 case ZEND_FETCH_OBJ_W:
2270 if (opline->op2_type != IS_CONST
2271 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2272 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2273 break;
2274 }
2275 ce = NULL;
2276 ce_is_instanceof = 0;
2277 on_this = 0;
2278 if (opline->op1_type == IS_UNUSED) {
2279 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
2280 op1_addr = 0;
2281 ce = op_array->scope;
2282 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
2283 on_this = 1;
2284 } else {
2285 op1_info = OP1_INFO();
2286 if (!(op1_info & MAY_BE_OBJECT)) {
2287 break;
2288 }
2289 op1_addr = OP1_REG_ADDR();
2290 if (ssa->var_info && ssa->ops) {
2291 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
2292 if (ssa_op->op1_use >= 0) {
2293 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
2294 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
2295 ce = op1_ssa->ce;
2296 ce_is_instanceof = op1_ssa->is_instanceof;
2297 }
2298 }
2299 }
2300 }
2301 if (!zend_jit_fetch_obj(&ctx, opline, op_array, ssa, ssa_op,
2302 op1_info, op1_addr, 0, ce, ce_is_instanceof, on_this, 0, 0, NULL,
2303 IS_UNKNOWN,
2304 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2305 goto jit_failure;
2306 }
2307 goto done;
2308 case ZEND_BIND_GLOBAL:
2309 if (!ssa->ops || !ssa->var_info) {
2310 op1_info = MAY_BE_ANY|MAY_BE_REF;
2311 } else {
2312 op1_info = OP1_INFO();
2313 }
2314 if (!zend_jit_bind_global(&ctx, opline, op1_info)) {
2315 goto jit_failure;
2316 }
2317 goto done;
2318 case ZEND_RECV:
2319 if (!zend_jit_recv(&ctx, opline, op_array)) {
2320 goto jit_failure;
2321 }
2322 goto done;
2323 case ZEND_RECV_INIT:
2324 if (!zend_jit_recv_init(&ctx, opline, op_array,
2325 (opline + 1)->opcode != ZEND_RECV_INIT,
2326 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2327 goto jit_failure;
2328 }
2329 goto done;
2330 case ZEND_FREE:
2331 case ZEND_FE_FREE:
2332 if (!zend_jit_free(&ctx, opline, OP1_INFO(),
2333 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2334 goto jit_failure;
2335 }
2336 goto done;
2337 case ZEND_ECHO:
2338 op1_info = OP1_INFO();
2339 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2340 break;
2341 }
2342 if (!zend_jit_echo(&ctx, opline, op1_info)) {
2343 goto jit_failure;
2344 }
2345 goto done;
2346 case ZEND_STRLEN:
2347 op1_info = OP1_INFO();
2348 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2349 break;
2350 }
2351 if (!zend_jit_strlen(&ctx, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR())) {
2352 goto jit_failure;
2353 }
2354 goto done;
2355 case ZEND_COUNT:
2356 op1_info = OP1_INFO();
2357 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
2358 break;
2359 }
2360 if (!zend_jit_count(&ctx, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
2361 goto jit_failure;
2362 }
2363 goto done;
2364 case ZEND_FETCH_THIS:
2365 if (!zend_jit_fetch_this(&ctx, opline, op_array, 0)) {
2366 goto jit_failure;
2367 }
2368 goto done;
2369 case ZEND_SWITCH_LONG:
2370 case ZEND_SWITCH_STRING:
2371 case ZEND_MATCH:
2372 if (!zend_jit_switch(&ctx, opline, op_array, ssa, NULL, NULL)) {
2373 goto jit_failure;
2374 }
2375 goto done;
2376 case ZEND_VERIFY_RETURN_TYPE:
2377 if (opline->op1_type == IS_UNUSED) {
2378 /* Always throws */
2379 break;
2380 }
2381 if (opline->op1_type == IS_CONST) {
2382 /* TODO Different instruction format, has return value */
2383 break;
2384 }
2385 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
2386 /* Not worth bothering with */
2387 break;
2388 }
2389 if (OP1_INFO() & MAY_BE_REF) {
2390 /* TODO May need reference unwrapping. */
2391 break;
2392 }
2393 if (!zend_jit_verify_return_type(&ctx, opline, op_array, OP1_INFO())) {
2394 goto jit_failure;
2395 }
2396 goto done;
2397 case ZEND_FE_RESET_R:
2398 op1_info = OP1_INFO();
2399 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
2400 break;
2401 }
2402 if (!zend_jit_fe_reset(&ctx, opline, op1_info)) {
2403 goto jit_failure;
2404 }
2405 goto done;
2406 case ZEND_FE_FETCH_R:
2407 op1_info = OP1_INFO();
2408 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
2409 break;
2410 }
2411 if (!zend_jit_fe_fetch(&ctx, opline, op1_info, OP2_INFO(),
2412 ssa->cfg.blocks[b].successors[0], opline->opcode, NULL)) {
2413 goto jit_failure;
2414 }
2415 goto done;
2416 case ZEND_FETCH_CONSTANT:
2417 if (!zend_jit_fetch_constant(&ctx, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
2418 goto jit_failure;
2419 }
2420 goto done;
2421 case ZEND_INIT_METHOD_CALL:
2422 if (opline->op2_type != IS_CONST
2423 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2424 break;
2425 }
2426 ce = NULL;
2427 ce_is_instanceof = 0;
2428 on_this = 0;
2429 if (opline->op1_type == IS_UNUSED) {
2430 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
2431 op1_addr = 0;
2432 ce = op_array->scope;
2433 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
2434 on_this = 1;
2435 } else {
2436 op1_info = OP1_INFO();
2437 if (!(op1_info & MAY_BE_OBJECT)) {
2438 break;
2439 }
2440 op1_addr = OP1_REG_ADDR();
2441 if (ssa->var_info && ssa->ops) {
2442 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
2443 if (ssa_op->op1_use >= 0) {
2444 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
2445 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
2446 ce = op1_ssa->ce;
2447 ce_is_instanceof = op1_ssa->is_instanceof;
2448 }
2449 }
2450 }
2451 }
2452 if (!zend_jit_init_method_call(&ctx, opline, b, op_array, ssa, ssa_op, call_level,
2453 op1_info, op1_addr, ce, ce_is_instanceof, on_this, 0, NULL,
2454 NULL, 0,
2455 -1, -1,
2456 0)) {
2457 goto jit_failure;
2458 }
2459 goto done;
2460 case ZEND_ROPE_INIT:
2461 case ZEND_ROPE_ADD:
2462 case ZEND_ROPE_END:
2463 op2_info = OP2_INFO();
2464 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2465 break;
2466 }
2467 if (!zend_jit_rope(&ctx, opline, op2_info)) {
2468 goto jit_failure;
2469 }
2470 goto done;
2471 default:
2472 break;
2473 }
2474 }
2475
2476 switch (opline->opcode) {
2477 case ZEND_RECV_INIT:
2478 case ZEND_BIND_GLOBAL:
2479 if (opline == op_array->opcodes ||
2480 opline->opcode != op_array->opcodes[i-1].opcode) {
2481 /* repeatable opcodes */
2482 if (!zend_jit_handler(&ctx, opline,
2483 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2484 goto jit_failure;
2485 }
2486 }
2487 zend_jit_set_last_valid_opline(&ctx, opline+1);
2488 break;
2489 case ZEND_NOP:
2490 case ZEND_OP_DATA:
2491 case ZEND_SWITCH_LONG:
2492 case ZEND_SWITCH_STRING:
2493 case ZEND_MATCH:
2494 break;
2495 case ZEND_JMP:
2496 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
2497 const zend_op *target = OP_JMP_ADDR(opline, opline->op1);
2498
2499 if (!zend_jit_set_ip(&ctx, target)) {
2500 goto jit_failure;
2501 }
2502 }
2503 break;
2504 case ZEND_CATCH:
2505 case ZEND_FAST_CALL:
2506 case ZEND_FAST_RET:
2507 case ZEND_GENERATOR_CREATE:
2508 case ZEND_GENERATOR_RETURN:
2509 case ZEND_RETURN_BY_REF:
2510 case ZEND_RETURN:
2511 case ZEND_EXIT:
2512 case ZEND_MATCH_ERROR:
2513 /* switch through trampoline */
2514 case ZEND_YIELD:
2515 case ZEND_YIELD_FROM:
2516 case ZEND_THROW:
2517 case ZEND_VERIFY_NEVER_TYPE:
2518 if (!zend_jit_tail_handler(&ctx, opline)) {
2519 goto jit_failure;
2520 }
2521 /* THROW and EXIT may be used in the middle of BB */
2522 /* don't generate code for the rest of BB */
2523 i = end;
2524 break;
2525 /* stackless execution */
2526 case ZEND_INCLUDE_OR_EVAL:
2527 case ZEND_DO_FCALL:
2528 case ZEND_DO_UCALL:
2529 case ZEND_DO_FCALL_BY_NAME:
2530 if (!zend_jit_call(&ctx, opline, b + 1)) {
2531 goto jit_failure;
2532 }
2533 break;
2534 case ZEND_JMPZ:
2535 case ZEND_JMPNZ:
2536 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
2537 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2538 /* smart branch */
2539 if (!zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2540 goto jit_failure;
2541 }
2542 goto done;
2543 }
2544 ZEND_FALLTHROUGH;
2545 case ZEND_JMPZ_EX:
2546 case ZEND_JMPNZ_EX:
2547 case ZEND_JMP_SET:
2548 case ZEND_COALESCE:
2549 case ZEND_JMP_NULL:
2550 case ZEND_FE_RESET_R:
2551 case ZEND_FE_RESET_RW:
2552 case ZEND_ASSERT_CHECK:
2553 case ZEND_FE_FETCH_R:
2554 case ZEND_FE_FETCH_RW:
2555 case ZEND_BIND_INIT_STATIC_OR_JMP:
2556 if (!zend_jit_handler(&ctx, opline,
2557 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
2558 !zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2559 goto jit_failure;
2560 }
2561 break;
2562 case ZEND_JMP_FRAMELESS:
2563 if (!zend_jit_jmp_frameless(&ctx, opline, /* exit_addr */ NULL, /* guard */ 0)) {
2564 goto jit_failure;
2565 }
2566 break;
2567 case ZEND_NEW:
2568 if (!zend_jit_handler(&ctx, opline, 1)) {
2569 return 0;
2570 }
2571 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
2572 zend_class_entry *ce = NULL;
2573
2574 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
2575 if (ssa->ops && ssa->var_info) {
2576 zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
2577 if (res_ssa->ce && !res_ssa->is_instanceof) {
2578 ce = res_ssa->ce;
2579 }
2580 }
2581 } else {
2582 if (opline->op1_type == IS_CONST) {
2583 zval *zv = RT_CONSTANT(opline, opline->op1);
2584 if (Z_TYPE_P(zv) == IS_STRING) {
2585 zval *lc = zv + 1;
2586 ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc));
2587 }
2588 }
2589 }
2590
2591 i++;
2592
2593 if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) {
2594 const zend_op *next_opline = opline + 1;
2595
2596 ZEND_ASSERT(b + 1 == ssa->cfg.blocks[b].successors[0]);
2597 zend_jit_constructor(&ctx, next_opline, op_array, ssa, call_level, b + 1);
2598 }
2599
2600 /* We skip over the DO_FCALL, so decrement call_level ourselves. */
2601 call_level--;
2602 }
2603 break;
2604 case ZEND_FRAMELESS_ICALL_0:
2605 jit_frameless_icall0(jit, opline);
2606 goto done;
2607 case ZEND_FRAMELESS_ICALL_1:
2608 op1_info = OP1_INFO();
2609 jit_frameless_icall1(jit, opline, op1_info);
2610 goto done;
2611 case ZEND_FRAMELESS_ICALL_2:
2612 op1_info = OP1_INFO();
2613 op2_info = OP2_INFO();
2614 jit_frameless_icall2(jit, opline, op1_info, op2_info);
2615 goto done;
2616 case ZEND_FRAMELESS_ICALL_3:
2617 op1_info = OP1_INFO();
2618 op2_info = OP2_INFO();
2619 jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO());
2620 goto done;
2621 default:
2622 if (!zend_jit_handler(&ctx, opline,
2623 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2624 goto jit_failure;
2625 }
2626 if (i == end
2627 && (opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2628 /* smart branch split across basic blocks */
2629 if (!zend_jit_set_cond(&ctx, opline + 2, opline->result.var)) {
2630 goto jit_failure;
2631 }
2632 }
2633 }
2634 done:
2635 switch (opline->opcode) {
2636 case ZEND_DO_FCALL:
2637 case ZEND_DO_ICALL:
2638 case ZEND_DO_UCALL:
2639 case ZEND_DO_FCALL_BY_NAME:
2640 case ZEND_CALLABLE_CONVERT:
2641 call_level--;
2642 }
2643 }
2644 zend_jit_bb_end(&ctx, b);
2645 }
2646
2647 if (jit->return_inputs) {
2648 zend_jit_common_return(jit);
2649
2650 bool left_frame = 0;
2651 if (op_array->last_var > 100) {
2652 /* To many CVs to unroll */
2653 if (!zend_jit_free_cvs(&ctx)) {
2654 goto jit_failure;
2655 }
2656 left_frame = 1;
2657 }
2658 if (!left_frame) {
2659 int j;
2660
2661 for (j = 0 ; j < op_array->last_var; j++) {
2662 uint32_t info = zend_ssa_cv_info(op_array, ssa, j);
2663
2664 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
2665 if (!left_frame) {
2666 left_frame = 1;
2667 if (!zend_jit_leave_frame(&ctx)) {
2668 goto jit_failure;
2669 }
2670 }
2671 if (!zend_jit_free_cv(&ctx, info, j)) {
2672 goto jit_failure;
2673 }
2674 }
2675 }
2676 }
2677 if (!zend_jit_leave_func(&ctx, op_array, NULL, MAY_BE_ANY, left_frame,
2678 NULL, NULL, (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, 1)) {
2679 goto jit_failure;
2680 }
2681 }
2682
2683 handler = zend_jit_finish(&ctx);
2684 if (!handler) {
2685 goto jit_failure;
2686 }
2687 zend_jit_free_ctx(&ctx);
2688
2689 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2690 zend_arena_release(&CG(arena), checkpoint);
2691 }
2692 return SUCCESS;
2693
2694 jit_failure:
2695 zend_jit_free_ctx(&ctx);
2696 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2697 zend_arena_release(&CG(arena), checkpoint);
2698 }
2699 return FAILURE;
2700 }
2701
zend_jit_collect_calls(zend_op_array * op_array,zend_script * script)2702 static void zend_jit_collect_calls(zend_op_array *op_array, zend_script *script)
2703 {
2704 zend_func_info *func_info;
2705
2706 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
2707 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
2708 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
2709 func_info = ZEND_FUNC_INFO(op_array);
2710 } else {
2711 func_info = zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info));
2712 ZEND_SET_FUNC_INFO(op_array, func_info);
2713 }
2714 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, op_array, func_info);
2715 }
2716
zend_jit_cleanup_func_info(zend_op_array * op_array)2717 static void zend_jit_cleanup_func_info(zend_op_array *op_array)
2718 {
2719 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
2720 zend_call_info *caller_info, *callee_info;
2721
2722 if (func_info) {
2723 caller_info = func_info->caller_info;
2724 callee_info = func_info->callee_info;
2725
2726 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
2727 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
2728 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
2729 func_info->num = 0;
2730 func_info->flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
2731 | ZEND_FUNC_JIT_ON_PROF_REQUEST
2732 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
2733 | ZEND_FUNC_JIT_ON_HOT_TRACE;
2734 memset(&func_info->ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
2735 } else {
2736 ZEND_SET_FUNC_INFO(op_array, NULL);
2737 }
2738
2739 while (caller_info) {
2740 if (caller_info->caller_op_array) {
2741 zend_jit_cleanup_func_info(caller_info->caller_op_array);
2742 }
2743 caller_info = caller_info->next_caller;
2744 }
2745 while (callee_info) {
2746 if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) {
2747 zend_jit_cleanup_func_info(&callee_info->callee_func->op_array);
2748 }
2749 callee_info = callee_info->next_callee;
2750 }
2751 }
2752 }
2753
zend_real_jit_func(zend_op_array * op_array,zend_script * script,const zend_op * rt_opline)2754 static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline)
2755 {
2756 zend_ssa ssa;
2757 void *checkpoint;
2758 zend_func_info *func_info;
2759
2760 if (*dasm_ptr == dasm_end) {
2761 return FAILURE;
2762 }
2763
2764 checkpoint = zend_arena_checkpoint(CG(arena));
2765
2766 /* Build SSA */
2767 memset(&ssa, 0, sizeof(zend_ssa));
2768
2769 if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) {
2770 goto jit_failure;
2771 }
2772
2773 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
2774 zend_jit_collect_calls(op_array, script);
2775 func_info = ZEND_FUNC_INFO(op_array);
2776 func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array);
2777 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
2778 zend_init_func_return_info(op_array, script, &func_info->return_info);
2779 }
2780 }
2781
2782 if (zend_jit_op_array_analyze2(op_array, script, &ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
2783 goto jit_failure;
2784 }
2785
2786 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
2787 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa);
2788 }
2789
2790 if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) {
2791 goto jit_failure;
2792 }
2793
2794 zend_jit_cleanup_func_info(op_array);
2795 zend_arena_release(&CG(arena), checkpoint);
2796 return SUCCESS;
2797
2798 jit_failure:
2799 zend_jit_cleanup_func_info(op_array);
2800 zend_arena_release(&CG(arena), checkpoint);
2801 return FAILURE;
2802 }
2803
2804 /* Run-time JIT handler */
zend_runtime_jit(void)2805 static int ZEND_FASTCALL zend_runtime_jit(void)
2806 {
2807 zend_execute_data *execute_data = EG(current_execute_data);
2808 zend_op_array *op_array = &EX(func)->op_array;
2809 zend_op *opline = op_array->opcodes;
2810 zend_jit_op_array_extension *jit_extension;
2811 bool do_bailout = 0;
2812
2813 zend_shared_alloc_lock();
2814
2815 if (ZEND_FUNC_INFO(op_array)) {
2816
2817 SHM_UNPROTECT();
2818 zend_jit_unprotect();
2819
2820 zend_try {
2821 /* restore original opcode handlers */
2822 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2823 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
2824 opline++;
2825 }
2826 }
2827 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
2828 opline->handler = jit_extension->orig_handler;
2829
2830 /* perform real JIT for this function */
2831 zend_real_jit_func(op_array, NULL, NULL);
2832 } zend_catch {
2833 do_bailout = true;
2834 } zend_end_try();
2835
2836 zend_jit_protect();
2837 SHM_PROTECT();
2838 }
2839
2840 zend_shared_alloc_unlock();
2841
2842 if (do_bailout) {
2843 zend_bailout();
2844 }
2845
2846 /* JIT-ed code is going to be called by VM */
2847 return 0;
2848 }
2849
zend_jit_check_funcs(HashTable * function_table,bool is_method)2850 void zend_jit_check_funcs(HashTable *function_table, bool is_method) {
2851 zend_op *opline;
2852 zend_function *func;
2853 zend_op_array *op_array;
2854 uintptr_t counter;
2855 zend_jit_op_array_extension *jit_extension;
2856
2857 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(function_table, func) {
2858 if (func->type == ZEND_INTERNAL_FUNCTION) {
2859 break;
2860 }
2861 op_array = &func->op_array;
2862 opline = op_array->opcodes;
2863 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2864 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
2865 opline++;
2866 }
2867 }
2868 if (opline->handler == zend_jit_profile_jit_handler) {
2869 if (!RUN_TIME_CACHE(op_array)) {
2870 continue;
2871 }
2872 counter = (uintptr_t)ZEND_COUNTER_INFO(op_array);
2873 ZEND_COUNTER_INFO(op_array) = 0;
2874 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
2875 opline->handler = jit_extension->orig_handler;
2876 if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) {
2877 zend_real_jit_func(op_array, NULL, NULL);
2878 }
2879 }
2880 } ZEND_HASH_FOREACH_END();
2881 }
2882
zend_jit_hot_func(zend_execute_data * execute_data,const zend_op * opline)2883 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
2884 {
2885 zend_op_array *op_array = &EX(func)->op_array;
2886 zend_jit_op_array_hot_extension *jit_extension;
2887 uint32_t i;
2888 bool do_bailout = 0;
2889
2890 zend_shared_alloc_lock();
2891 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
2892
2893 if (jit_extension) {
2894 SHM_UNPROTECT();
2895 zend_jit_unprotect();
2896
2897 zend_try {
2898 for (i = 0; i < op_array->last; i++) {
2899 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
2900 }
2901
2902 /* perform real JIT for this function */
2903 zend_real_jit_func(op_array, NULL, opline);
2904 } zend_catch {
2905 do_bailout = 1;
2906 } zend_end_try();
2907
2908 zend_jit_protect();
2909 SHM_PROTECT();
2910 }
2911
2912 zend_shared_alloc_unlock();
2913
2914 if (do_bailout) {
2915 zend_bailout();
2916 }
2917 /* JIT-ed code is going to be called by VM */
2918 }
2919
zend_jit_setup_hot_counters_ex(zend_op_array * op_array,zend_cfg * cfg)2920 static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg)
2921 {
2922 if (JIT_G(hot_func)) {
2923 zend_op *opline = op_array->opcodes;
2924
2925 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2926 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
2927 opline++;
2928 }
2929 }
2930
2931 opline->handler = (const void*)zend_jit_func_hot_counter_handler;
2932 }
2933
2934 if (JIT_G(hot_loop)) {
2935 uint32_t i;
2936
2937 for (i = 0; i < cfg->blocks_count; i++) {
2938 if ((cfg->blocks[i].flags & ZEND_BB_REACHABLE) &&
2939 (cfg->blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
2940 op_array->opcodes[cfg->blocks[i].start].handler =
2941 (const void*)zend_jit_loop_hot_counter_handler;
2942 }
2943 }
2944 }
2945 }
2946
zend_jit_restart_hot_counters(zend_op_array * op_array)2947 static int zend_jit_restart_hot_counters(zend_op_array *op_array)
2948 {
2949 zend_jit_op_array_hot_extension *jit_extension;
2950 zend_cfg cfg;
2951 uint32_t i;
2952
2953 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
2954 for (i = 0; i < op_array->last; i++) {
2955 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
2956 }
2957
2958 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
2959 return FAILURE;
2960 }
2961
2962 zend_jit_setup_hot_counters_ex(op_array, &cfg);
2963
2964 return SUCCESS;
2965 }
2966
zend_jit_setup_hot_counters(zend_op_array * op_array)2967 static int zend_jit_setup_hot_counters(zend_op_array *op_array)
2968 {
2969 zend_jit_op_array_hot_extension *jit_extension;
2970 zend_cfg cfg;
2971 uint32_t i;
2972
2973 ZEND_ASSERT(!JIT_G(hot_func) || zend_jit_func_hot_counter_handler != NULL);
2974 ZEND_ASSERT(!JIT_G(hot_loop) || zend_jit_loop_hot_counter_handler != NULL);
2975
2976 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
2977 return FAILURE;
2978 }
2979
2980 jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*));
2981 if (!jit_extension) {
2982 return FAILURE;
2983 }
2984 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
2985 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS;
2986 jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
2987 for (i = 0; i < op_array->last; i++) {
2988 jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
2989 }
2990 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
2991
2992 zend_jit_setup_hot_counters_ex(op_array, &cfg);
2993
2994 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
2995
2996 return SUCCESS;
2997 }
2998
2999 #include "jit/zend_jit_trace.c"
3000
zend_jit_op_array(zend_op_array * op_array,zend_script * script)3001 ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
3002 {
3003 if (dasm_ptr == NULL) {
3004 return FAILURE;
3005 }
3006
3007 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) {
3008 zend_jit_op_array_extension *jit_extension;
3009 zend_op *opline = op_array->opcodes;
3010
3011 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
3012 ZEND_SET_FUNC_INFO(op_array, NULL);
3013 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
3014 return SUCCESS;
3015 }
3016
3017 /* Set run-time JIT handler */
3018 ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL);
3019 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3020 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3021 opline++;
3022 }
3023 }
3024 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
3025 if (!jit_extension) {
3026 return FAILURE;
3027 }
3028 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3029 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC;
3030 jit_extension->orig_handler = (void*)opline->handler;
3031 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3032 opline->handler = (const void*)zend_jit_runtime_jit_handler;
3033 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3034
3035 return SUCCESS;
3036 } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) {
3037 zend_jit_op_array_extension *jit_extension;
3038 zend_op *opline = op_array->opcodes;
3039
3040 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
3041 ZEND_SET_FUNC_INFO(op_array, NULL);
3042 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
3043 return SUCCESS;
3044 }
3045
3046 ZEND_ASSERT(zend_jit_profile_jit_handler != NULL);
3047 if (op_array->function_name) {
3048 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3049 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3050 opline++;
3051 }
3052 }
3053 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
3054 if (!jit_extension) {
3055 return FAILURE;
3056 }
3057 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3058 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST;
3059 jit_extension->orig_handler = (void*)opline->handler;
3060 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3061 opline->handler = (const void*)zend_jit_profile_jit_handler;
3062 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3063 }
3064
3065 return SUCCESS;
3066 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
3067 return zend_jit_setup_hot_counters(op_array);
3068 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3069 return zend_jit_setup_hot_trace_counters(op_array);
3070 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3071 return zend_real_jit_func(op_array, script, NULL);
3072 } else {
3073 ZEND_UNREACHABLE();
3074 }
3075 return FAILURE;
3076 }
3077
zend_jit_script(zend_script * script)3078 ZEND_EXT_API int zend_jit_script(zend_script *script)
3079 {
3080 void *checkpoint;
3081 zend_call_graph call_graph;
3082 zend_func_info *info;
3083 int i;
3084
3085 if (dasm_ptr == NULL || *dasm_ptr == dasm_end) {
3086 return FAILURE;
3087 }
3088
3089 checkpoint = zend_arena_checkpoint(CG(arena));
3090
3091 call_graph.op_arrays_count = 0;
3092 zend_build_call_graph(&CG(arena), script, &call_graph);
3093
3094 zend_analyze_call_graph(&CG(arena), script, &call_graph);
3095
3096 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
3097 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
3098 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS ||
3099 JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3100 for (i = 0; i < call_graph.op_arrays_count; i++) {
3101 if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
3102 goto jit_failure;
3103 }
3104 }
3105 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3106 for (i = 0; i < call_graph.op_arrays_count; i++) {
3107 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3108 if (info) {
3109 if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
3110 goto jit_failure;
3111 }
3112 info->ssa.cfg.flags |= info->flags;
3113 info->flags = info->ssa.cfg.flags;
3114 }
3115 }
3116
3117 for (i = 0; i < call_graph.op_arrays_count; i++) {
3118 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3119 if (info) {
3120 info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]);
3121 if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
3122 zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info);
3123 }
3124 }
3125 }
3126
3127 for (i = 0; i < call_graph.op_arrays_count; i++) {
3128 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3129 if (info) {
3130 if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
3131 goto jit_failure;
3132 }
3133 info->flags = info->ssa.cfg.flags;
3134 }
3135 }
3136
3137 for (i = 0; i < call_graph.op_arrays_count; i++) {
3138 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3139 if (info) {
3140 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
3141 zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa);
3142 }
3143 if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) {
3144 goto jit_failure;
3145 }
3146 }
3147 }
3148
3149 for (i = 0; i < call_graph.op_arrays_count; i++) {
3150 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
3151 }
3152 } else {
3153 ZEND_UNREACHABLE();
3154 }
3155
3156 zend_arena_release(&CG(arena), checkpoint);
3157
3158 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC
3159 || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST
3160 || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS
3161 || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3162 zend_class_entry *ce;
3163 zend_op_array *op_array;
3164
3165 ZEND_HASH_MAP_FOREACH_PTR(&script->class_table, ce) {
3166 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
3167 if (!ZEND_FUNC_INFO(op_array)) {
3168 void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes);
3169
3170 if (jit_extension) {
3171 ZEND_SET_FUNC_INFO(op_array, jit_extension);
3172 }
3173 }
3174 } ZEND_HASH_FOREACH_END();
3175 } ZEND_HASH_FOREACH_END();
3176 }
3177
3178 return SUCCESS;
3179
3180 jit_failure:
3181 if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3182 for (i = 0; i < call_graph.op_arrays_count; i++) {
3183 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
3184 }
3185 }
3186 zend_arena_release(&CG(arena), checkpoint);
3187 return FAILURE;
3188 }
3189
zend_jit_unprotect(void)3190 ZEND_EXT_API void zend_jit_unprotect(void)
3191 {
3192 #ifdef HAVE_MPROTECT
3193 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3194 int opts = PROT_READ | PROT_WRITE;
3195 #ifdef ZTS
3196 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3197 if (zend_write_protect) {
3198 pthread_jit_write_protect_np(0);
3199 }
3200 #endif
3201 opts |= PROT_EXEC;
3202 #endif
3203 if (mprotect(dasm_buf, dasm_size, opts) != 0) {
3204 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3205 }
3206 }
3207 #elif _WIN32
3208 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3209 DWORD old, new;
3210 #ifdef ZTS
3211 new = PAGE_EXECUTE_READWRITE;
3212 #else
3213 new = PAGE_READWRITE;
3214 #endif
3215 if (!VirtualProtect(dasm_buf, dasm_size, new, &old)) {
3216 DWORD err = GetLastError();
3217 char *msg = php_win32_error_to_msg(err);
3218 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3219 php_win32_error_msg_free(msg);
3220 }
3221 }
3222 #endif
3223 }
3224
zend_jit_protect(void)3225 ZEND_EXT_API void zend_jit_protect(void)
3226 {
3227 #ifdef HAVE_MPROTECT
3228 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3229 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3230 if (zend_write_protect) {
3231 pthread_jit_write_protect_np(1);
3232 }
3233 #endif
3234 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
3235 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3236 }
3237 }
3238 #elif _WIN32
3239 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3240 DWORD old;
3241
3242 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
3243 DWORD err = GetLastError();
3244 char *msg = php_win32_error_to_msg(err);
3245 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3246 php_win32_error_msg_free(msg);
3247 }
3248 }
3249 #endif
3250 }
3251
zend_jit_init_handlers(void)3252 static void zend_jit_init_handlers(void)
3253 {
3254 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
3255 zend_jit_runtime_jit_handler = zend_jit_stub_handlers[jit_stub_hybrid_runtime_jit];
3256 zend_jit_profile_jit_handler = zend_jit_stub_handlers[jit_stub_hybrid_profile_jit];
3257 zend_jit_func_hot_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_func_hot_counter];
3258 zend_jit_loop_hot_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_loop_hot_counter];
3259 zend_jit_func_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_func_trace_counter];
3260 zend_jit_ret_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_ret_trace_counter];
3261 zend_jit_loop_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_loop_trace_counter];
3262 } else {
3263 zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
3264 zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
3265 zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper;
3266 zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper;
3267 zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper;
3268 zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper;
3269 zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper;
3270 }
3271 }
3272
zend_jit_globals_ctor(zend_jit_globals * jit_globals)3273 static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
3274 {
3275 memset(jit_globals, 0, sizeof(zend_jit_globals));
3276 zend_jit_trace_init_caches();
3277 }
3278
3279 #ifdef ZTS
zend_jit_globals_dtor(zend_jit_globals * jit_globals)3280 static void zend_jit_globals_dtor(zend_jit_globals *jit_globals)
3281 {
3282 zend_jit_trace_free_caches(jit_globals);
3283 }
3284 #endif
3285
zend_jit_parse_config_num(zend_long jit)3286 static int zend_jit_parse_config_num(zend_long jit)
3287 {
3288 if (jit == 0) {
3289 JIT_G(on) = 0;
3290 return SUCCESS;
3291 }
3292
3293 if (jit < 0) return FAILURE;
3294
3295 if (jit % 10 == 0 || jit % 10 > 5) return FAILURE;
3296 JIT_G(opt_level) = jit % 10;
3297
3298 jit /= 10;
3299 if (jit % 10 > 5) return FAILURE;
3300 JIT_G(trigger) = jit % 10;
3301
3302 jit /= 10;
3303 if (jit % 10 > 2) return FAILURE;
3304 JIT_G(opt_flags) = jit % 10;
3305
3306 jit /= 10;
3307 if (jit % 10 > 1) return FAILURE;
3308 JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0);
3309
3310 if (jit / 10 != 0) return FAILURE;
3311
3312 JIT_G(on) = 1;
3313
3314 return SUCCESS;
3315 }
3316
zend_jit_config(zend_string * jit,int stage)3317 ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage)
3318 {
3319 if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) {
3320 if (stage == ZEND_INI_STAGE_RUNTIME) {
3321 zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)");
3322 }
3323 return FAILURE;
3324 }
3325
3326 if (ZSTR_LEN(jit) == 0
3327 || zend_string_equals_literal_ci(jit, "disable")) {
3328 JIT_G(enabled) = 0;
3329 JIT_G(on) = 0;
3330 return SUCCESS;
3331 } else if (zend_string_equals_literal_ci(jit, "0")
3332 || zend_string_equals_literal_ci(jit, "off")
3333 || zend_string_equals_literal_ci(jit, "no")
3334 || zend_string_equals_literal_ci(jit, "false")) {
3335 JIT_G(enabled) = 1;
3336 JIT_G(on) = 0;
3337 return SUCCESS;
3338 } else if (zend_string_equals_literal_ci(jit, "1")
3339 || zend_string_equals_literal_ci(jit, "on")
3340 || zend_string_equals_literal_ci(jit, "yes")
3341 || zend_string_equals_literal_ci(jit, "true")
3342 || zend_string_equals_literal_ci(jit, "tracing")) {
3343 JIT_G(enabled) = 1;
3344 JIT_G(on) = 1;
3345 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS;
3346 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
3347 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
3348 return SUCCESS;
3349 } else if (zend_string_equals_ci(jit, ZSTR_KNOWN(ZEND_STR_FUNCTION))) {
3350 JIT_G(enabled) = 1;
3351 JIT_G(on) = 1;
3352 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT;
3353 JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD;
3354 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
3355 return SUCCESS;
3356 } else {
3357 char *end;
3358 zend_long num = ZEND_STRTOL(ZSTR_VAL(jit), &end, 10);
3359 if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) {
3360 goto failure;
3361 }
3362 JIT_G(enabled) = 1;
3363 return SUCCESS;
3364 }
3365
3366 failure:
3367 zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number");
3368 JIT_G(enabled) = 0;
3369 JIT_G(on) = 0;
3370 return FAILURE;
3371 }
3372
zend_jit_debug_config(zend_long old_val,zend_long new_val,int stage)3373 ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage)
3374 {
3375 if (stage != ZEND_INI_STAGE_STARTUP) {
3376 if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) {
3377 if (stage == ZEND_INI_STAGE_RUNTIME) {
3378 zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup");
3379 }
3380 return FAILURE;
3381 }
3382 }
3383 return SUCCESS;
3384 }
3385
zend_jit_init(void)3386 ZEND_EXT_API void zend_jit_init(void)
3387 {
3388 #ifdef ZTS
3389 jit_globals_id = ts_allocate_id(&jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, (ts_allocate_dtor) zend_jit_globals_dtor);
3390 #else
3391 zend_jit_globals_ctor(&jit_globals);
3392 #endif
3393 }
3394
zend_jit_check_support(void)3395 ZEND_EXT_API int zend_jit_check_support(void)
3396 {
3397 int i;
3398
3399 zend_jit_vm_kind = zend_vm_kind();
3400 if (zend_jit_vm_kind != ZEND_VM_KIND_CALL &&
3401 zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
3402 zend_error(E_WARNING, "JIT is compatible only with CALL and HYBRID VM. JIT disabled.");
3403 JIT_G(enabled) = 0;
3404 JIT_G(on) = 0;
3405 return FAILURE;
3406 }
3407
3408 if (zend_execute_ex != execute_ex) {
3409 if (strcmp(sapi_module.name, "phpdbg") != 0) {
3410 zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled.");
3411 }
3412 JIT_G(enabled) = 0;
3413 JIT_G(on) = 0;
3414 return FAILURE;
3415 }
3416
3417 for (i = 0; i <= 256; i++) {
3418 switch (i) {
3419 /* JIT has no effect on these opcodes */
3420 case ZEND_BEGIN_SILENCE:
3421 case ZEND_END_SILENCE:
3422 case ZEND_EXIT:
3423 break;
3424 default:
3425 if (zend_get_user_opcode_handler(i) != NULL) {
3426 zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled.");
3427 JIT_G(enabled) = 0;
3428 JIT_G(on) = 0;
3429 return FAILURE;
3430 }
3431 }
3432 }
3433
3434 #if defined(IR_TARGET_AARCH64)
3435 if (JIT_G(buffer_size) > 128*1024*1024) {
3436 zend_error(E_WARNING, "JIT on AArch64 doesn't support opcache.jit_buffer_size above 128M.");
3437 JIT_G(enabled) = 0;
3438 JIT_G(on) = 0;
3439 return FAILURE;
3440 }
3441 #elif defined(IR_TARGET_X64)
3442 if (JIT_G(buffer_size) > 2 * Z_L(1024*1024*1024)) {
3443 zend_error(E_WARNING, "JIT on x86_64 doesn't support opcache.jit_buffer_size above 2G.");
3444 JIT_G(enabled) = 0;
3445 JIT_G(on) = 0;
3446 return FAILURE;
3447 }
3448 #endif
3449
3450 return SUCCESS;
3451 }
3452
zend_jit_startup(void * buf,size_t size,bool reattached)3453 ZEND_EXT_API void zend_jit_startup(void *buf, size_t size, bool reattached)
3454 {
3455 zend_jit_halt_op = zend_get_halt_op();
3456 zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(ACCELERATOR_PRODUCT_NAME);
3457
3458 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3459 zend_write_protect = pthread_jit_write_protect_supported_np();
3460 #endif
3461
3462 dasm_buf = buf;
3463 dasm_size = size;
3464 dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr) * 2);
3465
3466 #ifdef HAVE_MPROTECT
3467 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3468 if (zend_write_protect) {
3469 pthread_jit_write_protect_np(1);
3470 }
3471 #endif
3472 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
3473 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
3474 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3475 }
3476 } else {
3477 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
3478 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3479 }
3480 }
3481 #elif _WIN32
3482 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
3483 DWORD old;
3484
3485 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
3486 DWORD err = GetLastError();
3487 char *msg = php_win32_error_to_msg(err);
3488 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3489 php_win32_error_msg_free(msg);
3490 }
3491 } else {
3492 DWORD old;
3493
3494 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
3495 DWORD err = GetLastError();
3496 char *msg = php_win32_error_to_msg(err);
3497 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3498 php_win32_error_msg_free(msg);
3499 }
3500 }
3501 #endif
3502
3503 if (!reattached) {
3504 zend_jit_unprotect();
3505 *dasm_ptr = dasm_buf;
3506 #if defined(_WIN32)
3507 zend_jit_stub_handlers = dasm_buf;
3508 *dasm_ptr = (void**)*dasm_ptr + sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
3509 #elif defined(IR_TARGET_AARCH64)
3510 zend_jit_stub_handlers = dasm_buf;
3511 *dasm_ptr = (void**)*dasm_ptr + (sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])) * 2;
3512 memset(zend_jit_stub_handlers, 0, (sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])) * 2 * sizeof(void*));
3513 #endif
3514 *dasm_ptr = (void*)ZEND_MM_ALIGNED_SIZE_EX(((size_t)(*dasm_ptr)), 16);
3515 zend_jit_protect();
3516 } else {
3517 #if defined(_WIN32) || defined(IR_TARGET_AARCH64)
3518 zend_jit_stub_handlers = dasm_buf;
3519 zend_jit_init_handlers();
3520 #endif
3521 }
3522
3523 zend_jit_unprotect();
3524 zend_jit_setup();
3525 zend_jit_protect();
3526 zend_jit_init_handlers();
3527
3528 zend_jit_trace_startup(reattached);
3529
3530 zend_jit_unprotect();
3531 /* save JIT buffer pos */
3532 dasm_ptr[1] = dasm_ptr[0];
3533 zend_jit_protect();
3534 }
3535
zend_jit_shutdown(void)3536 ZEND_EXT_API void zend_jit_shutdown(void)
3537 {
3538 if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE && dasm_ptr != NULL) {
3539 fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf));
3540 }
3541
3542 zend_jit_shutdown_ir();
3543
3544 #ifdef ZTS
3545 ts_free_id(jit_globals_id);
3546 #else
3547 zend_jit_trace_free_caches(&jit_globals);
3548 #endif
3549 }
3550
zend_jit_reset_counters(void)3551 static void zend_jit_reset_counters(void)
3552 {
3553 int i;
3554
3555 for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
3556 zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT;
3557 }
3558 }
3559
zend_jit_activate(void)3560 ZEND_EXT_API void zend_jit_activate(void)
3561 {
3562 zend_jit_profile_counter = 0;
3563 if (JIT_G(on)) {
3564 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
3565 zend_jit_reset_counters();
3566 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3567 zend_jit_reset_counters();
3568 zend_jit_trace_reset_caches();
3569 }
3570 }
3571 }
3572
zend_jit_deactivate(void)3573 ZEND_EXT_API void zend_jit_deactivate(void)
3574 {
3575 if (zend_jit_profile_counter && !CG(unclean_shutdown)) {
3576 zend_class_entry *ce;
3577
3578 zend_shared_alloc_lock();
3579 SHM_UNPROTECT();
3580 zend_jit_unprotect();
3581
3582 zend_jit_check_funcs(EG(function_table), 0);
3583 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(EG(class_table), ce) {
3584 if (ce->type == ZEND_INTERNAL_CLASS) {
3585 break;
3586 }
3587 zend_jit_check_funcs(&ce->function_table, 1);
3588 } ZEND_HASH_FOREACH_END();
3589
3590 zend_jit_protect();
3591 SHM_PROTECT();
3592 zend_shared_alloc_unlock();
3593 }
3594
3595 zend_jit_profile_counter = 0;
3596 }
3597
zend_jit_restart_preloaded_op_array(zend_op_array * op_array)3598 static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array)
3599 {
3600 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
3601
3602 if (!func_info) {
3603 return;
3604 }
3605
3606 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
3607 zend_jit_restart_hot_trace_counters(op_array);
3608 } else if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_COUNTERS) {
3609 zend_jit_restart_hot_counters(op_array);
3610 #if 0
3611 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ???
3612 } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) {
3613 zend_op *opline = op_array->opcodes;
3614 zend_jit_op_array_extension *jit_extension =
3615 (zend_jit_op_array_extension*)func_info;
3616
3617 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3618 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3619 opline++;
3620 }
3621 }
3622 if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) {
3623 opline->handler = (const void*)zend_jit_runtime_jit_handler;
3624 } else {
3625 opline->handler = (const void*)zend_jit_profile_jit_handler;
3626 }
3627 #endif
3628 }
3629 if (op_array->num_dynamic_func_defs) {
3630 for (uint32_t i = 0; i < op_array->num_dynamic_func_defs; i++) {
3631 zend_jit_restart_preloaded_op_array(op_array->dynamic_func_defs[i]);
3632 }
3633 }
3634 }
3635
zend_jit_restart_preloaded_script(zend_persistent_script * script)3636 static void zend_jit_restart_preloaded_script(zend_persistent_script *script)
3637 {
3638 zend_class_entry *ce;
3639 zend_op_array *op_array;
3640
3641 zend_jit_restart_preloaded_op_array(&script->script.main_op_array);
3642
3643 ZEND_HASH_MAP_FOREACH_PTR(&script->script.function_table, op_array) {
3644 zend_jit_restart_preloaded_op_array(op_array);
3645 } ZEND_HASH_FOREACH_END();
3646
3647 ZEND_HASH_MAP_FOREACH_PTR(&script->script.class_table, ce) {
3648 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
3649 if (op_array->type == ZEND_USER_FUNCTION) {
3650 zend_jit_restart_preloaded_op_array(op_array);
3651 }
3652 } ZEND_HASH_FOREACH_END();
3653 } ZEND_HASH_FOREACH_END();
3654 }
3655
zend_jit_restart(void)3656 ZEND_EXT_API void zend_jit_restart(void)
3657 {
3658 if (dasm_buf) {
3659 zend_jit_unprotect();
3660
3661 /* restore JIT buffer pos */
3662 dasm_ptr[0] = dasm_ptr[1];
3663
3664 zend_jit_trace_restart();
3665
3666 if (ZCSG(preload_script)) {
3667 zend_jit_restart_preloaded_script(ZCSG(preload_script));
3668 if (ZCSG(saved_scripts)) {
3669 zend_persistent_script **p = ZCSG(saved_scripts);
3670
3671 while (*p) {
3672 zend_jit_restart_preloaded_script(*p);
3673 p++;
3674 }
3675 }
3676 }
3677
3678 zend_jit_protect();
3679 }
3680 }
3681
3682 #endif /* HAVE_JIT */
3683