1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 #include "main/php.h"
20 #include "main/SAPI.h"
21 #include "php_version.h"
22 #include <ZendAccelerator.h>
23 #include "zend_shared_alloc.h"
24 #include "Zend/zend_execute.h"
25 #include "Zend/zend_vm.h"
26 #include "Zend/zend_exceptions.h"
27 #include "Zend/zend_constants.h"
28 #include "Zend/zend_closures.h"
29 #include "Zend/zend_ini.h"
30 #include "Zend/zend_observer.h"
31 #include "zend_smart_str.h"
32 #include "jit/zend_jit.h"
33
34 #ifdef HAVE_JIT
35
36 #include "Optimizer/zend_func_info.h"
37 #include "Optimizer/zend_ssa.h"
38 #include "Optimizer/zend_inference.h"
39 #include "Optimizer/zend_call_graph.h"
40 #include "Optimizer/zend_dump.h"
41 #include "Optimizer/zend_worklist.h"
42
43 #include "jit/zend_jit_internal.h"
44
45 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
46 #include <pthread.h>
47 #endif
48
49 #ifdef ZTS
50 int jit_globals_id;
51 #else
52 zend_jit_globals jit_globals;
53 #endif
54
55 //#define CONTEXT_THREADED_JIT
56 #define ZEND_JIT_USE_RC_INFERENCE
57
58 #ifdef ZEND_JIT_USE_RC_INFERENCE
59 # define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE
60 # define RC_MAY_BE_1(info) (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0)
61 # define RC_MAY_BE_N(info) (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0)
62 #else
63 # define ZEND_SSA_RC_INFERENCE_FLAG 0
64 # define RC_MAY_BE_1(info) 1
65 # define RC_MAY_BE_N(info) 1
66 #endif
67
68 #define JIT_PREFIX "JIT$"
69 #define JIT_STUB_PREFIX "JIT$$"
70 #define TRACE_PREFIX "TRACE-"
71
72 zend_ulong zend_jit_profile_counter = 0;
73 int zend_jit_profile_counter_rid = -1;
74
75 int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
76
77 const zend_op *zend_jit_halt_op = NULL;
78 static int zend_jit_vm_kind = 0;
79 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
80 static int zend_write_protect = 1;
81 #endif
82
83 static void *dasm_buf = NULL;
84 static void *dasm_end = NULL;
85 static void **dasm_ptr = NULL;
86
87 static size_t dasm_size = 0;
88
89 static zend_long jit_bisect_pos = 0;
90
91 static const void *zend_jit_runtime_jit_handler = NULL;
92 static const void *zend_jit_profile_jit_handler = NULL;
93 static const void *zend_jit_func_hot_counter_handler = NULL;
94 static const void *zend_jit_loop_hot_counter_handler = NULL;
95 static const void *zend_jit_func_trace_counter_handler = NULL;
96 static const void *zend_jit_ret_trace_counter_handler = NULL;
97 static const void *zend_jit_loop_trace_counter_handler = NULL;
98
99 static int ZEND_FASTCALL zend_runtime_jit(void);
100
101 static int zend_jit_trace_op_len(const zend_op *opline);
102 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
103 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags);
104 static const void *zend_jit_trace_get_exit_addr(uint32_t n);
105 static void zend_jit_trace_add_code(const void *start, uint32_t size);
106 static zend_string *zend_jit_func_name(const zend_op_array *op_array);
107
108 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info);
109 static bool zend_jit_supported_binary_op(uint8_t op, uint32_t op1_info, uint32_t op2_info);
110
dominates(const zend_basic_block * blocks,int a,int b)111 static bool dominates(const zend_basic_block *blocks, int a, int b) {
112 while (blocks[b].level > blocks[a].level) {
113 b = blocks[b].idom;
114 }
115 return a == b;
116 }
117
zend_ssa_is_last_use(const zend_op_array * op_array,const zend_ssa * ssa,int var,int use)118 static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
119 {
120 int next_use;
121
122 if (ssa->vars[var].phi_use_chain) {
123 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
124 do {
125 if (!ssa->vars[phi->ssa_var].no_val) {
126 return 0;
127 }
128 phi = zend_ssa_next_use_phi(ssa, var, phi);
129 } while (phi);
130 }
131
132 if (ssa->cfg.blocks[ssa->cfg.map[use]].loop_header > 0
133 || (ssa->cfg.blocks[ssa->cfg.map[use]].flags & ZEND_BB_LOOP_HEADER)) {
134 int b = ssa->cfg.map[use];
135 int prev_use = ssa->vars[var].use_chain;
136 int def_block;
137
138 if (ssa->vars[var].definition >= 0) {
139 def_block =ssa->cfg.map[ssa->vars[var].definition];
140 } else {
141 ZEND_ASSERT(ssa->vars[var].definition_phi);
142 def_block = ssa->vars[var].definition_phi->block;
143 }
144 if (dominates(ssa->cfg.blocks, def_block,
145 (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) ? b : ssa->cfg.blocks[b].loop_header)) {
146 return 0;
147 }
148
149 while (prev_use >= 0 && prev_use != use) {
150 if (b != ssa->cfg.map[prev_use]
151 && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use])
152 && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) {
153 return 0;
154 }
155 prev_use = zend_ssa_next_use(ssa->ops, var, prev_use);
156 }
157 }
158
159 next_use = zend_ssa_next_use(ssa->ops, var, use);
160 if (next_use < 0) {
161 return 1;
162 } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) {
163 return 1;
164 }
165 return 0;
166 }
167
zend_jit_is_constant_cmp_long_long(const zend_op * opline,zend_ssa_range * op1_range,zend_jit_addr op1_addr,zend_ssa_range * op2_range,zend_jit_addr op2_addr,bool * result)168 static int zend_jit_is_constant_cmp_long_long(const zend_op *opline,
169 zend_ssa_range *op1_range,
170 zend_jit_addr op1_addr,
171 zend_ssa_range *op2_range,
172 zend_jit_addr op2_addr,
173 bool *result)
174 {
175 zend_long op1_min;
176 zend_long op1_max;
177 zend_long op2_min;
178 zend_long op2_max;
179
180 if (op1_range) {
181 op1_min = op1_range->min;
182 op1_max = op1_range->max;
183 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
184 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG);
185 op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr));
186 } else {
187 return 0;
188 }
189
190 if (op2_range) {
191 op2_min = op2_range->min;
192 op2_max = op2_range->max;
193 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
194 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG);
195 op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr));
196 } else {
197 return 0;
198 }
199
200 switch (opline->opcode) {
201 case ZEND_IS_EQUAL:
202 case ZEND_IS_IDENTICAL:
203 case ZEND_CASE:
204 case ZEND_CASE_STRICT:
205 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
206 *result = 1;
207 return 1;
208 } else if (op1_max < op2_min || op1_min > op2_max) {
209 *result = 0;
210 return 1;
211 }
212 return 0;
213 case ZEND_IS_NOT_EQUAL:
214 case ZEND_IS_NOT_IDENTICAL:
215 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
216 *result = 0;
217 return 1;
218 } else if (op1_max < op2_min || op1_min > op2_max) {
219 *result = 1;
220 return 1;
221 }
222 return 0;
223 case ZEND_IS_SMALLER:
224 if (op1_max < op2_min) {
225 *result = 1;
226 return 1;
227 } else if (op1_min >= op2_max) {
228 *result = 0;
229 return 1;
230 }
231 return 0;
232 case ZEND_IS_SMALLER_OR_EQUAL:
233 if (op1_max <= op2_min) {
234 *result = 1;
235 return 1;
236 } else if (op1_min > op2_max) {
237 *result = 0;
238 return 1;
239 }
240 return 0;
241 default:
242 ZEND_UNREACHABLE();
243 }
244 return 0;
245 }
246
247 #define ADVANCE_SSA_OP(ssa_op, offset) \
248 do { \
249 if (ssa_op) ssa_op += offset; \
250 } while (0)
251
zend_jit_needs_call_chain(zend_call_info * call_info,uint32_t b,const zend_op_array * op_array,zend_ssa * ssa,const zend_ssa_op * ssa_op,const zend_op * opline,int call_level,zend_jit_trace_rec * trace)252 static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace)
253 {
254 int skip;
255
256 if (trace) {
257 zend_jit_trace_rec *p = trace;
258
259 ADVANCE_SSA_OP(ssa_op, 1);
260 while (1) {
261 if (p->op == ZEND_JIT_TRACE_VM) {
262 switch (p->opline->opcode) {
263 case ZEND_SEND_ARRAY:
264 case ZEND_SEND_USER:
265 case ZEND_SEND_UNPACK:
266 case ZEND_INIT_FCALL:
267 case ZEND_INIT_METHOD_CALL:
268 case ZEND_INIT_STATIC_METHOD_CALL:
269 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
270 case ZEND_INIT_FCALL_BY_NAME:
271 case ZEND_INIT_NS_FCALL_BY_NAME:
272 case ZEND_INIT_DYNAMIC_CALL:
273 case ZEND_NEW:
274 case ZEND_INIT_USER_CALL:
275 case ZEND_FAST_CALL:
276 case ZEND_JMP:
277 case ZEND_JMPZ:
278 case ZEND_JMPNZ:
279 case ZEND_JMPZ_EX:
280 case ZEND_JMPNZ_EX:
281 case ZEND_FE_RESET_R:
282 case ZEND_FE_RESET_RW:
283 case ZEND_JMP_SET:
284 case ZEND_COALESCE:
285 case ZEND_JMP_NULL:
286 case ZEND_ASSERT_CHECK:
287 case ZEND_CATCH:
288 case ZEND_DECLARE_ANON_CLASS:
289 case ZEND_FE_FETCH_R:
290 case ZEND_FE_FETCH_RW:
291 case ZEND_BIND_INIT_STATIC_OR_JMP:
292 case ZEND_JMP_FRAMELESS:
293 return 1;
294 case ZEND_DO_ICALL:
295 case ZEND_DO_UCALL:
296 case ZEND_DO_FCALL_BY_NAME:
297 case ZEND_DO_FCALL:
298 case ZEND_CALLABLE_CONVERT:
299 return 0;
300 case ZEND_SEND_VAL:
301 case ZEND_SEND_VAR:
302 case ZEND_SEND_VAL_EX:
303 case ZEND_SEND_VAR_EX:
304 case ZEND_SEND_FUNC_ARG:
305 case ZEND_SEND_REF:
306 case ZEND_SEND_VAR_NO_REF:
307 case ZEND_SEND_VAR_NO_REF_EX:
308 /* skip */
309 break;
310 default:
311 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
312 return 1;
313 }
314 }
315 ADVANCE_SSA_OP(ssa_op, zend_jit_trace_op_len(opline));
316 } else if (p->op == ZEND_JIT_TRACE_ENTER ||
317 p->op == ZEND_JIT_TRACE_BACK ||
318 p->op == ZEND_JIT_TRACE_END) {
319 return 1;
320 }
321 p++;
322 }
323 }
324
325 if (!call_info) {
326 const zend_op *end = op_array->opcodes + op_array->last;
327
328 opline++;
329 ADVANCE_SSA_OP(ssa_op, 1);
330 skip = (call_level == 1);
331 while (opline != end) {
332 if (!skip) {
333 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
334 return 1;
335 }
336 }
337 switch (opline->opcode) {
338 case ZEND_SEND_VAL:
339 case ZEND_SEND_VAR:
340 case ZEND_SEND_VAL_EX:
341 case ZEND_SEND_VAR_EX:
342 case ZEND_SEND_FUNC_ARG:
343 case ZEND_SEND_REF:
344 case ZEND_SEND_VAR_NO_REF:
345 case ZEND_SEND_VAR_NO_REF_EX:
346 skip = 0;
347 break;
348 case ZEND_SEND_ARRAY:
349 case ZEND_SEND_USER:
350 case ZEND_SEND_UNPACK:
351 case ZEND_INIT_FCALL:
352 case ZEND_INIT_METHOD_CALL:
353 case ZEND_INIT_STATIC_METHOD_CALL:
354 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
355 case ZEND_INIT_FCALL_BY_NAME:
356 case ZEND_INIT_NS_FCALL_BY_NAME:
357 case ZEND_INIT_DYNAMIC_CALL:
358 case ZEND_NEW:
359 case ZEND_INIT_USER_CALL:
360 case ZEND_FAST_CALL:
361 case ZEND_JMP:
362 case ZEND_JMPZ:
363 case ZEND_JMPNZ:
364 case ZEND_JMPZ_EX:
365 case ZEND_JMPNZ_EX:
366 case ZEND_FE_RESET_R:
367 case ZEND_FE_RESET_RW:
368 case ZEND_JMP_SET:
369 case ZEND_COALESCE:
370 case ZEND_JMP_NULL:
371 case ZEND_ASSERT_CHECK:
372 case ZEND_CATCH:
373 case ZEND_DECLARE_ANON_CLASS:
374 case ZEND_FE_FETCH_R:
375 case ZEND_FE_FETCH_RW:
376 case ZEND_BIND_INIT_STATIC_OR_JMP:
377 case ZEND_JMP_FRAMELESS:
378 return 1;
379 case ZEND_DO_ICALL:
380 case ZEND_DO_UCALL:
381 case ZEND_DO_FCALL_BY_NAME:
382 case ZEND_DO_FCALL:
383 case ZEND_CALLABLE_CONVERT:
384 end = opline;
385 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
386 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
387 return 1;
388 }
389 return 0;
390 }
391 opline++;
392 ADVANCE_SSA_OP(ssa_op, 1);
393 }
394
395 return 1;
396 } else {
397 const zend_op *end = call_info->caller_call_opline;
398
399 /* end may be null if an opcode like EXIT is part of the argument list. */
400 if (!end || end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
401 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
402 return 1;
403 }
404
405 opline++;
406 ADVANCE_SSA_OP(ssa_op, 1);
407 skip = (call_level == 1);
408 while (opline != end) {
409 if (skip) {
410 switch (opline->opcode) {
411 case ZEND_SEND_VAL:
412 case ZEND_SEND_VAR:
413 case ZEND_SEND_VAL_EX:
414 case ZEND_SEND_VAR_EX:
415 case ZEND_SEND_FUNC_ARG:
416 case ZEND_SEND_REF:
417 case ZEND_SEND_VAR_NO_REF:
418 case ZEND_SEND_VAR_NO_REF_EX:
419 skip = 0;
420 break;
421 case ZEND_SEND_ARRAY:
422 case ZEND_SEND_USER:
423 case ZEND_SEND_UNPACK:
424 return 1;
425 }
426 } else {
427 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
428 return 1;
429 }
430 }
431 opline++;
432 ADVANCE_SSA_OP(ssa_op, 1);
433 }
434
435 return 0;
436 }
437 }
438
skip_valid_arguments(const zend_op_array * op_array,zend_ssa * ssa,const zend_call_info * call_info)439 static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
440 {
441 uint32_t num_args = 0;
442 zend_function *func = call_info->callee_func;
443
444 /* It's okay to handle prototypes here, because they can only increase the accepted arguments.
445 * Anything legal for the parent method is also legal for the parent method. */
446 while (num_args < call_info->num_args) {
447 zend_arg_info *arg_info = func->op_array.arg_info + num_args;
448
449 if (ZEND_TYPE_IS_SET(arg_info->type)) {
450 if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) {
451 zend_op *opline = call_info->arg_info[num_args].opline;
452 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[opline - op_array->opcodes] : NULL;
453 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
454 if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) {
455 break;
456 }
457 } else {
458 break;
459 }
460 }
461 num_args++;
462 }
463 return num_args;
464 }
465
zend_ssa_cv_info(const zend_op_array * op_array,zend_ssa * ssa,uint32_t var)466 static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var)
467 {
468 uint32_t j, info;
469
470 if (ssa->vars && ssa->var_info) {
471 info = ssa->var_info[var].type;
472 for (j = op_array->last_var; j < ssa->vars_count; j++) {
473 if (ssa->vars[j].var == var) {
474 info |= ssa->var_info[j].type;
475 }
476 }
477 } else {
478 info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF |
479 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
480 }
481
482 #ifdef ZEND_JIT_USE_RC_INFERENCE
483 /* Refcount may be increased by RETURN opcode */
484 if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
485 for (j = 0; j < ssa->cfg.blocks_count; j++) {
486 if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
487 ssa->cfg.blocks[j].len > 0) {
488 const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
489
490 if (opline->opcode == ZEND_RETURN) {
491 if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) {
492 info |= MAY_BE_RCN;
493 break;
494 }
495 }
496 }
497 }
498 }
499 #endif
500
501 return info;
502 }
503
zend_jit_may_avoid_refcounting(const zend_op * opline,uint32_t op1_info)504 static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_info)
505 {
506 switch (opline->opcode) {
507 case ZEND_FETCH_OBJ_FUNC_ARG:
508 if (!JIT_G(current_frame) ||
509 !JIT_G(current_frame)->call->func ||
510 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
511 return 0;
512 }
513 /* break missing intentionally */
514 case ZEND_FETCH_OBJ_R:
515 case ZEND_FETCH_OBJ_IS:
516 if ((op1_info & MAY_BE_OBJECT)
517 && opline->op2_type == IS_CONST
518 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
519 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
520 return 1;
521 }
522 break;
523 case ZEND_FETCH_DIM_FUNC_ARG:
524 if (!JIT_G(current_frame) ||
525 !JIT_G(current_frame)->call->func ||
526 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
527 return 0;
528 }
529 /* break missing intentionally */
530 case ZEND_FETCH_DIM_R:
531 case ZEND_FETCH_DIM_IS:
532 return 1;
533 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
534 if (!(opline->extended_value & ZEND_ISEMPTY)) {
535 return 1;
536 }
537 break;
538 }
539 return 0;
540 }
541
zend_jit_is_persistent_constant(zval * key,uint32_t flags)542 static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags)
543 {
544 zval *zv;
545 zend_constant *c = NULL;
546
547 /* null/true/false are resolved during compilation, so don't check for them here. */
548 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
549 if (zv) {
550 c = (zend_constant*)Z_PTR_P(zv);
551 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
552 key++;
553 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
554 if (zv) {
555 c = (zend_constant*)Z_PTR_P(zv);
556 }
557 }
558 return c && (ZEND_CONSTANT_FLAGS(c) & CONST_PERSISTENT);
559 }
560
zend_get_known_property_info(const zend_op_array * op_array,zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)561 static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
562 {
563 zend_property_info *info = NULL;
564
565 if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) ||
566 !ce ||
567 !(ce->ce_flags & ZEND_ACC_LINKED) ||
568 (ce->ce_flags & ZEND_ACC_TRAIT) ||
569 ce->create_object) {
570 return NULL;
571 }
572
573 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
574 if (ce->info.user.filename != filename) {
575 /* class declaration might be changed independently */
576 return NULL;
577 }
578
579 if (ce->parent) {
580 zend_class_entry *parent = ce->parent;
581
582 do {
583 if (parent->type == ZEND_INTERNAL_CLASS) {
584 break;
585 } else if (parent->info.user.filename != filename) {
586 /* some of parents class declarations might be changed independently */
587 /* TODO: this check may be not enough, because even
588 * in the same it's possible to conditionally define
589 * few classes with the same name, and "parent" may
590 * change from request to request.
591 */
592 return NULL;
593 }
594 parent = parent->parent;
595 } while (parent);
596 }
597 }
598
599 // TODO: Treat property hooks more precisely.
600 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
601 if (info == NULL ||
602 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
603 (info->flags & ZEND_ACC_STATIC) ||
604 info->hooks) {
605 return NULL;
606 }
607
608 if (info->flags & ZEND_ACC_PUBLIC) {
609 return info;
610 } else if (on_this) {
611 if (ce == info->ce) {
612 if (ce == op_array->scope) {
613 return info;
614 } else {
615 return NULL;
616 }
617 } else if ((info->flags & ZEND_ACC_PROTECTED)
618 && instanceof_function_slow(ce, info->ce)) {
619 return info;
620 }
621 }
622
623 return NULL;
624 }
625
zend_may_be_dynamic_property(zend_class_entry * ce,zend_string * member,bool on_this,const zend_op_array * op_array)626 static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, const zend_op_array *op_array)
627 {
628 zend_property_info *info;
629
630 if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT) || (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) {
631 return 1;
632 }
633
634 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
635 if (ce->info.user.filename != op_array->filename) {
636 /* class declaration might be changed independently */
637 return 1;
638 }
639 }
640
641 // TODO: Treat property hooks more precisely.
642 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
643 if (info == NULL ||
644 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
645 (info->flags & ZEND_ACC_STATIC) ||
646 info->hooks) {
647 return 1;
648 }
649
650 if (!(info->flags & ZEND_ACC_PUBLIC) &&
651 (!on_this || info->ce != ce)) {
652 return 1;
653 }
654
655 return 0;
656 }
657
658 #define OP_RANGE(ssa_op, opN) \
659 (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
660 ssa->var_info && \
661 (ssa_op)->opN##_use >= 0 && \
662 ssa->var_info[(ssa_op)->opN##_use].has_range) ? \
663 &ssa->var_info[(ssa_op)->opN##_use].range : NULL)
664
665 #define OP1_RANGE() OP_RANGE(ssa_op, op1)
666 #define OP2_RANGE() OP_RANGE(ssa_op, op2)
667 #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)
668
669 #include "jit/zend_jit_helpers.c"
670 #include "Zend/zend_cpuinfo.h"
671
672 #ifdef HAVE_GCC_GLOBAL_REGS
673 # define GCC_GLOBAL_REGS 1
674 #else
675 # define GCC_GLOBAL_REGS 0
676 #endif
677
678 /* By default avoid JITing inline handlers if it does not seem profitable due to lack of
679 * type information. Disabling this option allows testing some JIT handlers in the
680 * presence of try/catch blocks, which prevent SSA construction. */
681 #ifndef PROFITABILITY_CHECKS
682 # define PROFITABILITY_CHECKS 1
683 #endif
684
685 #define BP_JIT_IS 6 /* Used for ISSET_ISEMPTY_DIM_OBJ. see BP_VAR_*defines in Zend/zend_compile.h */
686
687 /* The generated code may contain tautological comparisons, ignore them. */
688 #if defined(__clang__)
689 # pragma clang diagnostic push
690 # pragma clang diagnostic ignored "-Wtautological-compare"
691 # pragma clang diagnostic ignored "-Wstring-compare"
692 #endif
693
694 #include "jit/zend_jit_ir.c"
695
696 #if defined(__clang__)
697 # pragma clang diagnostic pop
698 #endif
699
700 #ifdef _WIN32
701 # include <Windows.h>
702 #else
703 # include <sys/mman.h>
704 # if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
705 # define MAP_ANONYMOUS MAP_ANON
706 # endif
707 #endif
708
zend_jit_status(zval * ret)709 ZEND_EXT_API void zend_jit_status(zval *ret)
710 {
711 zval stats;
712 array_init(&stats);
713 add_assoc_bool(&stats, "enabled", JIT_G(enabled));
714 add_assoc_bool(&stats, "on", JIT_G(on));
715 add_assoc_long(&stats, "kind", JIT_G(trigger));
716 add_assoc_long(&stats, "opt_level", JIT_G(opt_level));
717 add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags));
718 if (dasm_buf) {
719 add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
720 add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
721 } else {
722 add_assoc_long(&stats, "buffer_size", 0);
723 add_assoc_long(&stats, "buffer_free", 0);
724 }
725 add_assoc_zval(ret, "jit", &stats);
726 }
727
zend_jit_func_name(const zend_op_array * op_array)728 static zend_string *zend_jit_func_name(const zend_op_array *op_array)
729 {
730 smart_str buf = {0};
731
732 if (op_array->function_name) {
733 smart_str_appends(&buf, JIT_PREFIX);
734 if (op_array->scope) {
735 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
736 smart_str_appends(&buf, "::");
737 }
738 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
739 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
740 smart_str_appends(&buf, ":");
741 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
742 smart_str_appends(&buf, ":");
743 smart_str_append_long(&buf, op_array->line_start);
744 }
745 smart_str_0(&buf);
746 return buf.s;
747 } else if (op_array->filename) {
748 smart_str_appends(&buf, JIT_PREFIX);
749 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
750 smart_str_0(&buf);
751 return buf.s;
752 } else {
753 return NULL;
754 }
755 }
756
zend_may_overflow(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa)757 static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
758 {
759 int res;
760 zend_long op1_min, op1_max, op2_min, op2_max;
761
762 if (!ssa->ops || !ssa->var_info) {
763 return 1;
764 }
765 switch (opline->opcode) {
766 case ZEND_PRE_INC:
767 case ZEND_POST_INC:
768 res = ssa_op->op1_def;
769 if (res < 0
770 || !ssa->var_info[res].has_range
771 || ssa->var_info[res].range.overflow) {
772 if (!OP1_HAS_RANGE()) {
773 return 1;
774 }
775 op1_max = OP1_MAX_RANGE();
776 if (op1_max == ZEND_LONG_MAX) {
777 return 1;
778 }
779 }
780 return 0;
781 case ZEND_PRE_DEC:
782 case ZEND_POST_DEC:
783 res = ssa_op->op1_def;
784 if (res < 0
785 || !ssa->var_info[res].has_range
786 || ssa->var_info[res].range.underflow) {
787 if (!OP1_HAS_RANGE()) {
788 return 1;
789 }
790 op1_min = OP1_MIN_RANGE();
791 if (op1_min == ZEND_LONG_MIN) {
792 return 1;
793 }
794 }
795 return 0;
796 case ZEND_ADD:
797 res = ssa_op->result_def;
798 if (res < 0
799 || !ssa->var_info[res].has_range
800 || ssa->var_info[res].range.underflow) {
801 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
802 return 1;
803 }
804 op1_min = OP1_MIN_RANGE();
805 op2_min = OP2_MIN_RANGE();
806 if (zend_add_will_overflow(op1_min, op2_min)) {
807 return 1;
808 }
809 }
810 if (res < 0
811 || !ssa->var_info[res].has_range
812 || ssa->var_info[res].range.overflow) {
813 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
814 return 1;
815 }
816 op1_max = OP1_MAX_RANGE();
817 op2_max = OP2_MAX_RANGE();
818 if (zend_add_will_overflow(op1_max, op2_max)) {
819 return 1;
820 }
821 }
822 return 0;
823 case ZEND_SUB:
824 res = ssa_op->result_def;
825 if (res < 0
826 || !ssa->var_info[res].has_range
827 || ssa->var_info[res].range.underflow) {
828 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
829 return 1;
830 }
831 op1_min = OP1_MIN_RANGE();
832 op2_max = OP2_MAX_RANGE();
833 if (zend_sub_will_overflow(op1_min, op2_max)) {
834 return 1;
835 }
836 }
837 if (res < 0
838 || !ssa->var_info[res].has_range
839 || ssa->var_info[res].range.overflow) {
840 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
841 return 1;
842 }
843 op1_max = OP1_MAX_RANGE();
844 op2_min = OP2_MIN_RANGE();
845 if (zend_sub_will_overflow(op1_max, op2_min)) {
846 return 1;
847 }
848 }
849 return 0;
850 case ZEND_MUL:
851 res = ssa_op->result_def;
852 return (res < 0 ||
853 !ssa->var_info[res].has_range ||
854 ssa->var_info[res].range.underflow ||
855 ssa->var_info[res].range.overflow);
856 case ZEND_ASSIGN_OP:
857 if (opline->extended_value == ZEND_ADD) {
858 res = ssa_op->op1_def;
859 if (res < 0
860 || !ssa->var_info[res].has_range
861 || ssa->var_info[res].range.underflow) {
862 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
863 return 1;
864 }
865 op1_min = OP1_MIN_RANGE();
866 op2_min = OP2_MIN_RANGE();
867 if (zend_add_will_overflow(op1_min, op2_min)) {
868 return 1;
869 }
870 }
871 if (res < 0
872 || !ssa->var_info[res].has_range
873 || ssa->var_info[res].range.overflow) {
874 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
875 return 1;
876 }
877 op1_max = OP1_MAX_RANGE();
878 op2_max = OP2_MAX_RANGE();
879 if (zend_add_will_overflow(op1_max, op2_max)) {
880 return 1;
881 }
882 }
883 return 0;
884 } else if (opline->extended_value == ZEND_SUB) {
885 res = ssa_op->op1_def;
886 if (res < 0
887 || !ssa->var_info[res].has_range
888 || ssa->var_info[res].range.underflow) {
889 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
890 return 1;
891 }
892 op1_min = OP1_MIN_RANGE();
893 op2_max = OP2_MAX_RANGE();
894 if (zend_sub_will_overflow(op1_min, op2_max)) {
895 return 1;
896 }
897 }
898 if (res < 0
899 || !ssa->var_info[res].has_range
900 || ssa->var_info[res].range.overflow) {
901 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
902 return 1;
903 }
904 op1_max = OP1_MAX_RANGE();
905 op2_min = OP2_MIN_RANGE();
906 if (zend_sub_will_overflow(op1_max, op2_min)) {
907 return 1;
908 }
909 }
910 return 0;
911 } else if (opline->extended_value == ZEND_MUL) {
912 res = ssa_op->op1_def;
913 return (res < 0 ||
914 !ssa->var_info[res].has_range ||
915 ssa->var_info[res].range.underflow ||
916 ssa->var_info[res].range.overflow);
917 }
918 ZEND_FALLTHROUGH;
919 default:
920 return 1;
921 }
922 }
923
zend_jit_build_cfg(const zend_op_array * op_array,zend_cfg * cfg)924 static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg)
925 {
926 uint32_t flags;
927
928 flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY;
929
930 zend_build_cfg(&CG(arena), op_array, flags, cfg);
931
932 /* Don't JIT huge functions. Apart from likely being detrimental due to the amount of
933 * generated code, some of our analysis is recursive and will stack overflow with many
934 * blocks. */
935 if (cfg->blocks_count > 100000) {
936 return FAILURE;
937 }
938
939 zend_cfg_build_predecessors(&CG(arena), cfg);
940
941 /* Compute Dominators Tree */
942 zend_cfg_compute_dominators_tree(op_array, cfg);
943
944 /* Identify reducible and irreducible loops */
945 zend_cfg_identify_loops(op_array, cfg);
946
947 return SUCCESS;
948 }
949
zend_jit_op_array_analyze1(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa)950 static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
951 {
952 if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) {
953 return FAILURE;
954 }
955
956 #if 0
957 /* TODO: debugger and profiler supports? */
958 if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) {
959 return FAILURE;
960 }
961 #endif
962
963 /* TODO: move this to zend_cfg.c ? */
964 if (!op_array->function_name) {
965 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
966 }
967
968 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
969 && ssa->cfg.blocks
970 && op_array->last_try_catch == 0
971 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
972 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
973 if (zend_build_ssa(&CG(arena), script, op_array, ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) {
974 return FAILURE;
975 }
976
977 zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa);
978
979 zend_ssa_find_false_dependencies(op_array, ssa);
980
981 zend_ssa_find_sccs(op_array, ssa);
982 }
983
984 return SUCCESS;
985 }
986
zend_jit_op_array_analyze2(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa,uint32_t optimization_level)987 static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t optimization_level)
988 {
989 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
990 && ssa->cfg.blocks
991 && op_array->last_try_catch == 0
992 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
993 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
994 if (zend_ssa_inference(&CG(arena), op_array, script, ssa,
995 optimization_level & ~ZEND_OPTIMIZER_NARROW_TO_DOUBLE) != SUCCESS) {
996 return FAILURE;
997 }
998 }
999
1000 return SUCCESS;
1001 }
1002
zend_jit_allocate_registers(zend_jit_ctx * ctx,const zend_op_array * op_array,zend_ssa * ssa)1003 static void zend_jit_allocate_registers(zend_jit_ctx *ctx, const zend_op_array *op_array, zend_ssa *ssa)
1004 {
1005 void *checkpoint;
1006 int candidates_count, i;
1007 zend_jit_reg_var *ra;
1008
1009 checkpoint = zend_arena_checkpoint(CG(arena));
1010 ra = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_jit_reg_var));
1011 candidates_count = 0;
1012 for (i = 0; i < ssa->vars_count; i++) {
1013 if (zend_jit_may_be_in_reg(op_array, ssa, i)) {
1014 ra[i].ref = IR_NULL;
1015 candidates_count++;
1016 }
1017 }
1018 if (!candidates_count) {
1019 zend_arena_release(&CG(arena), checkpoint);
1020 return;
1021 }
1022
1023 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
1024 /* Naive SSA resolution */
1025 for (i = 0; i < ssa->vars_count; i++) {
1026 if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
1027 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
1028 int k, src;
1029
1030 if (phi->pi >= 0) {
1031 src = phi->sources[0];
1032 if (ra[i].ref) {
1033 if (!ra[src].ref) {
1034 ra[i].flags |= ZREG_LOAD;
1035 } else {
1036 ra[i].flags |= ZREG_PI;
1037 }
1038 } else if (ra[src].ref) {
1039 ra[src].flags |= ZREG_STORE;
1040 }
1041 } else {
1042 int need_move = 0;
1043
1044 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1045 src = phi->sources[k];
1046 if (src >= 0) {
1047 if (ssa->vars[src].definition_phi
1048 && ssa->vars[src].definition_phi->pi >= 0
1049 && phi->block == ssa->vars[src].definition_phi->block) {
1050 /* Skip zero-length interval for Pi variable */
1051 src = ssa->vars[src].definition_phi->sources[0];
1052 }
1053 if (ra[i].ref) {
1054 if (!ra[src].ref) {
1055 need_move = 1;
1056 }
1057 } else if (ra[src].ref) {
1058 need_move = 1;
1059 }
1060 }
1061 }
1062 if (need_move) {
1063 if (ra[i].ref) {
1064 ra[i].flags |= ZREG_LOAD;
1065 }
1066 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1067 src = phi->sources[k];
1068 if (src >= 0) {
1069 if (ssa->vars[src].definition_phi
1070 && ssa->vars[src].definition_phi->pi >= 0
1071 && phi->block == ssa->vars[src].definition_phi->block) {
1072 /* Skip zero-length interval for Pi variable */
1073 src = ssa->vars[src].definition_phi->sources[0];
1074 }
1075 if (ra[src].ref) {
1076 ra[src].flags |= ZREG_STORE;
1077 }
1078 }
1079 }
1080 } else {
1081 ra[i].flags |= ZREG_PHI;
1082 }
1083 }
1084 }
1085 }
1086
1087 /* Remove useless register allocation */
1088 for (i = 0; i < ssa->vars_count; i++) {
1089 if (ra[i].ref &&
1090 ((ra[i].flags & ZREG_LOAD) ||
1091 ((ra[i].flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) &&
1092 ssa->vars[i].use_chain < 0) {
1093 bool may_remove = 1;
1094 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
1095
1096 while (phi) {
1097 if (ra[phi->ssa_var].ref &&
1098 !(ra[phi->ssa_var].flags & ZREG_LOAD)) {
1099 may_remove = 0;
1100 break;
1101 }
1102 phi = zend_ssa_next_use_phi(ssa, i, phi);
1103 }
1104 if (may_remove) {
1105 ra[i].ref = IR_UNUSED;
1106 }
1107 }
1108 }
1109
1110 /* Remove intervals used once */
1111 for (i = 0; i < ssa->vars_count; i++) {
1112 if (ra[i].ref) {
1113 if (!(ra[i].flags & (ZREG_LOAD|ZREG_STORE))) {
1114 uint32_t var_num = ssa->vars[i].var;
1115 uint32_t op_num = ssa->vars[i].definition;
1116
1117 /* Check if a tempoary variable may be freed by exception handler */
1118 if (op_array->last_live_range
1119 && var_num >= op_array->last_var
1120 && ssa->vars[i].definition >= 0
1121 && ssa->ops[op_num].result_def == i) {
1122 const zend_live_range *range = op_array->live_range;
1123 int j;
1124
1125 op_num++;
1126 if (op_array->opcodes[op_num].opcode == ZEND_OP_DATA) {
1127 op_num++;
1128 }
1129 for (j = 0; j < op_array->last_live_range; range++, j++) {
1130 if (range->start > op_num) {
1131 /* further blocks will not be relevant... */
1132 break;
1133 } else if (op_num < range->end && var_num == (range->var & ~ZEND_LIVE_MASK)) {
1134 /* check if opcodes in range may throw */
1135 do {
1136 if (zend_may_throw(op_array->opcodes + op_num, ssa->ops + op_num, op_array, ssa)) {
1137 ra[i].flags |= ZREG_STORE;
1138 break;
1139 }
1140 op_num++;
1141 if (op_array->opcodes[op_num].opcode == ZEND_OP_DATA) {
1142 op_num++;
1143 }
1144 } while (op_num < range->end);
1145 break;
1146 }
1147 }
1148 }
1149 }
1150 if ((ra[i].flags & ZREG_LOAD)
1151 && (ra[i].flags & ZREG_STORE)
1152 && (ssa->vars[i].use_chain < 0
1153 || zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) {
1154 bool may_remove = 1;
1155 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
1156
1157 while (phi) {
1158 if (ra[phi->ssa_var].ref &&
1159 !(ra[phi->ssa_var].flags & ZREG_LOAD)) {
1160 may_remove = 0;
1161 break;
1162 }
1163 phi = zend_ssa_next_use_phi(ssa, i, phi);
1164 }
1165 if (may_remove) {
1166 ra[i].ref = IR_UNUSED;
1167 }
1168 }
1169 }
1170 }
1171 }
1172
1173 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
1174 fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
1175 for (i = 0; i < ssa->vars_count; i++) {
1176 if (ra[i].ref) {
1177 fprintf(stderr, "#%d.", i);
1178 uint32_t var_num = ssa->vars[i].var;
1179 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
1180 if (ra[i].flags & ZREG_LOAD) {
1181 fprintf(stderr, " load");
1182 }
1183 if (ra[i].flags & ZREG_STORE) {
1184 fprintf(stderr, " store");
1185 }
1186 fprintf(stderr, "\n");
1187 }
1188 }
1189 fprintf(stderr, "\n");
1190 }
1191
1192 ctx->ra = ra;
1193 }
1194
zend_jit_compute_post_order(zend_cfg * cfg,int start,int * post_order)1195 static int zend_jit_compute_post_order(zend_cfg *cfg, int start, int *post_order)
1196 {
1197 int count = 0;
1198 int b, n, *p;
1199 zend_basic_block *bb;
1200 zend_worklist worklist;
1201 ALLOCA_FLAG(use_heap)
1202
1203 ZEND_WORKLIST_ALLOCA(&worklist, cfg->blocks_count, use_heap);
1204 zend_worklist_push(&worklist, start);
1205
1206 while (zend_worklist_len(&worklist) != 0) {
1207 next:
1208 b = zend_worklist_peek(&worklist);
1209 bb = &cfg->blocks[b];
1210 n = bb->successors_count;
1211 if (n > 0) {
1212 p = bb->successors;
1213 do {
1214 if (cfg->blocks[*p].flags & (ZEND_BB_CATCH|ZEND_BB_FINALLY|ZEND_BB_FINALLY_END)) {
1215 /* skip */
1216 } else if (zend_worklist_push(&worklist, *p)) {
1217 goto next;
1218 }
1219 p++;
1220 n--;
1221 } while (n > 0);
1222 }
1223 zend_worklist_pop(&worklist);
1224 post_order[count++] = b;
1225 }
1226 ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
1227 return count;
1228 }
1229
zend_jit_next_is_send_result(const zend_op * opline)1230 static bool zend_jit_next_is_send_result(const zend_op *opline)
1231 {
1232 if (opline->result_type == IS_TMP_VAR
1233 && (opline+1)->opcode == ZEND_SEND_VAL
1234 && (opline+1)->op1_type == IS_TMP_VAR
1235 && (opline+1)->op2_type != IS_CONST
1236 && (opline+1)->op1.var == opline->result.var) {
1237 return 1;
1238 }
1239 return 0;
1240 }
1241
zend_jit_supported_binary_op(uint8_t op,uint32_t op1_info,uint32_t op2_info)1242 static bool zend_jit_supported_binary_op(uint8_t op, uint32_t op1_info, uint32_t op2_info)
1243 {
1244 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1245 return false;
1246 }
1247 switch (op) {
1248 case ZEND_POW:
1249 case ZEND_DIV:
1250 // TODO: check for division by zero ???
1251 return false;
1252 case ZEND_ADD:
1253 case ZEND_SUB:
1254 case ZEND_MUL:
1255 return (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
1256 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
1257 case ZEND_BW_OR:
1258 case ZEND_BW_AND:
1259 case ZEND_BW_XOR:
1260 case ZEND_SL:
1261 case ZEND_SR:
1262 case ZEND_MOD:
1263 return (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG);
1264 case ZEND_CONCAT:
1265 return (op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING);
1266 EMPTY_SWITCH_DEFAULT_CASE()
1267 }
1268 }
1269
zend_jit(const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline)1270 static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline)
1271 {
1272 int b, i, end;
1273 zend_op *opline;
1274 zend_jit_ctx ctx;
1275 zend_jit_ctx *jit = &ctx;
1276 zend_jit_reg_var *ra = NULL;
1277 void *handler;
1278 int call_level = 0;
1279 void *checkpoint = NULL;
1280 bool recv_emitted = 0; /* emitted at least one RECV opcode */
1281 uint8_t smart_branch_opcode;
1282 uint32_t target_label, target_label2;
1283 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_mem_info;
1284 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
1285 zend_class_entry *ce;
1286 bool ce_is_instanceof;
1287 bool on_this;
1288
1289 if (JIT_G(bisect_limit)) {
1290 jit_bisect_pos++;
1291 if (jit_bisect_pos >= JIT_G(bisect_limit)) {
1292 if (jit_bisect_pos == JIT_G(bisect_limit)) {
1293 fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n",
1294 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
1295 op_array->scope ? "::" : "",
1296 op_array->function_name ? ZSTR_VAL(op_array->function_name) : "{main}",
1297 ZSTR_VAL(op_array->filename), op_array->line_start);
1298 }
1299 return FAILURE;
1300 }
1301 }
1302
1303 if (ssa->cfg.flags & ZEND_FUNC_IRREDUCIBLE) {
1304 /* We can't order blocks properly */
1305 return FAILURE;
1306 }
1307
1308 if (rt_opline) {
1309 /* Set BB_ENTRY flag to limit register usage across the OSR ENTRY point */
1310 ssa->cfg.blocks[ssa->cfg.map[rt_opline - op_array->opcodes]].flags |= ZEND_BB_ENTRY;
1311 }
1312
1313 zend_jit_start(&ctx, op_array, ssa);
1314 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1315 checkpoint = zend_arena_checkpoint(CG(arena));
1316 zend_jit_allocate_registers(&ctx, op_array, ssa);
1317 ra = ctx.ra;
1318 }
1319
1320 /* Process blocks in Reverse Post Order */
1321 int *sorted_blocks = alloca(sizeof(int) * ssa->cfg.blocks_count);
1322 int n = zend_jit_compute_post_order(&ssa->cfg, 0, sorted_blocks);
1323
1324 while (n > 0) {
1325 b = sorted_blocks[--n];
1326 if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) {
1327 continue;
1328 }
1329
1330 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
1331 opline = op_array->opcodes + ssa->cfg.blocks[b].start;
1332 if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) {
1333 if (opline->opcode == ZEND_RECV_INIT) {
1334 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1335 if (opline != op_array->opcodes && (opline-1)->opcode != ZEND_RECV_INIT) {
1336 zend_jit_recv_entry(&ctx, b);
1337 }
1338 } else {
1339 if (opline != op_array->opcodes && recv_emitted) {
1340 zend_jit_recv_entry(&ctx, b);
1341 }
1342 }
1343 recv_emitted = 1;
1344 } else if (opline->opcode == ZEND_RECV) {
1345 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
1346 /* skip */
1347 zend_jit_bb_start(&ctx, b);
1348 zend_jit_bb_end(&ctx, b);
1349 continue;
1350 } else if (recv_emitted) {
1351 zend_jit_recv_entry(&ctx, b);
1352 } else {
1353 recv_emitted = 1;
1354 }
1355 } else {
1356 if (recv_emitted) {
1357 zend_jit_recv_entry(&ctx, b);
1358 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
1359 ssa->cfg.blocks[b].len == 1 &&
1360 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
1361 /* don't generate code for BB with single opcode */
1362 zend_jit_free_ctx(&ctx);
1363
1364 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1365 zend_arena_release(&CG(arena), checkpoint);
1366 }
1367 return SUCCESS;
1368 }
1369 }
1370 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
1371 ssa->cfg.blocks[b].len == 1 &&
1372 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
1373 /* don't generate code for BB with single opcode */
1374 zend_jit_free_ctx(&ctx);
1375
1376 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1377 zend_arena_release(&CG(arena), checkpoint);
1378 }
1379 return SUCCESS;
1380 }
1381 }
1382
1383 zend_jit_bb_start(&ctx, b);
1384
1385 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ctx.ra) {
1386 zend_ssa_phi *phi = ssa->blocks[b].phis;
1387
1388 /* First try to insert IR Phi */
1389 while (phi) {
1390 zend_jit_reg_var *ival = &ctx.ra[phi->ssa_var];
1391
1392 if (ival->ref) {
1393 if (ival->flags & ZREG_PI) {
1394 zend_jit_gen_pi(jit, phi);
1395 } else if (ival->flags & ZREG_PHI) {
1396 zend_jit_gen_phi(jit, phi);
1397 }
1398 }
1399 phi = phi->next;
1400 }
1401 }
1402
1403 if (rt_opline
1404 && (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) == 0
1405 && rt_opline == op_array->opcodes + ssa->cfg.blocks[b].start) {
1406 zend_jit_osr_entry(&ctx, b); /* OSR (On-Stack-Replacement) Entry-Point */
1407 }
1408
1409 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1410 if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
1411 && ssa->cfg.blocks[b].start != 0
1412 && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
1413 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_LONG
1414 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_STRING
1415 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_MATCH)) {
1416 zend_jit_reset_last_valid_opline(&ctx);
1417 } else {
1418 zend_jit_set_last_valid_opline(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start);
1419 }
1420 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
1421 zend_jit_reset_last_valid_opline(&ctx);
1422 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY) {
1423 zend_jit_reset_last_valid_opline(&ctx);
1424 } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) {
1425 zend_jit_set_last_valid_opline(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start);
1426 }
1427 if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) {
1428 zend_jit_check_timeout(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start, NULL);
1429 }
1430 if (!ssa->cfg.blocks[b].len) {
1431 zend_jit_bb_end(&ctx, b);
1432 continue;
1433 }
1434 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
1435 zend_ssa_phi *phi = ssa->blocks[b].phis;
1436
1437 while (phi) {
1438 zend_jit_reg_var *ival = &ra[phi->ssa_var];
1439
1440 if (ival->ref) {
1441 if (ival->flags & ZREG_LOAD) {
1442 ZEND_ASSERT(ival->ref == IR_NULL);
1443
1444 if (!zend_jit_load_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var)) {
1445 goto jit_failure;
1446 }
1447 } else if (ival->flags & ZREG_STORE) {
1448 ZEND_ASSERT(ival->ref != IR_NULL);
1449
1450 if (!zend_jit_store_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var, 1)) {
1451 goto jit_failure;
1452 }
1453 }
1454 }
1455 phi = phi->next;
1456 }
1457 }
1458 end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
1459 for (i = ssa->cfg.blocks[b].start; i <= end; i++) {
1460 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[i] : NULL;
1461 opline = op_array->opcodes + i;
1462 switch (opline->opcode) {
1463 case ZEND_INIT_FCALL:
1464 case ZEND_INIT_FCALL_BY_NAME:
1465 case ZEND_INIT_NS_FCALL_BY_NAME:
1466 case ZEND_INIT_METHOD_CALL:
1467 case ZEND_INIT_DYNAMIC_CALL:
1468 case ZEND_INIT_STATIC_METHOD_CALL:
1469 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
1470 case ZEND_INIT_USER_CALL:
1471 case ZEND_NEW:
1472 call_level++;
1473 }
1474
1475 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
1476 switch (opline->opcode) {
1477 case ZEND_PRE_INC:
1478 case ZEND_PRE_DEC:
1479 case ZEND_POST_INC:
1480 case ZEND_POST_DEC:
1481 if (opline->op1_type != IS_CV) {
1482 break;
1483 }
1484 op1_info = OP1_INFO();
1485 if (!(op1_info & MAY_BE_LONG)) {
1486 break;
1487 }
1488 if (opline->result_type != IS_UNUSED) {
1489 res_use_info = -1;
1490
1491 if (opline->result_type == IS_CV
1492 && ssa->vars
1493 && ssa_op->result_use >= 0
1494 && !ssa->vars[ssa_op->result_use].no_val) {
1495 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1496
1497 if (Z_MODE(res_use_addr) != IS_REG
1498 || Z_LOAD(res_use_addr)
1499 || Z_STORE(res_use_addr)) {
1500 res_use_info = RES_USE_INFO();
1501 }
1502 }
1503 res_info = RES_INFO();
1504 res_addr = RES_REG_ADDR();
1505 } else {
1506 res_use_info = -1;
1507 res_info = -1;
1508 res_addr = 0;
1509 }
1510 op1_def_info = OP1_DEF_INFO();
1511 if (!zend_jit_inc_dec(&ctx, opline,
1512 op1_info, OP1_REG_ADDR(),
1513 op1_def_info, OP1_DEF_REG_ADDR(),
1514 res_use_info, res_info,
1515 res_addr,
1516 (op1_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1517 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1518 goto jit_failure;
1519 }
1520 goto done;
1521 case ZEND_BW_OR:
1522 case ZEND_BW_AND:
1523 case ZEND_BW_XOR:
1524 case ZEND_SL:
1525 case ZEND_SR:
1526 case ZEND_MOD:
1527 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1528 break;
1529 }
1530 op1_info = OP1_INFO();
1531 op2_info = OP2_INFO();
1532 if (!(op1_info & MAY_BE_LONG)
1533 || !(op2_info & MAY_BE_LONG)) {
1534 break;
1535 }
1536 res_addr = RES_REG_ADDR();
1537 if (Z_MODE(res_addr) != IS_REG
1538 && (i + 1) <= end
1539 && zend_jit_next_is_send_result(opline)) {
1540 i++;
1541 res_use_info = -1;
1542 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1543 if (!zend_jit_reuse_ip(&ctx)) {
1544 goto jit_failure;
1545 }
1546 } else {
1547 res_use_info = -1;
1548
1549 if (opline->result_type == IS_CV
1550 && ssa->vars
1551 && ssa_op->result_use >= 0
1552 && !ssa->vars[ssa_op->result_use].no_val) {
1553 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1554
1555 if (Z_MODE(res_use_addr) != IS_REG
1556 || Z_LOAD(res_use_addr)
1557 || Z_STORE(res_use_addr)) {
1558 res_use_info = RES_USE_INFO();
1559 }
1560 }
1561 }
1562 if (!zend_jit_long_math(&ctx, opline,
1563 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
1564 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
1565 res_use_info, RES_INFO(), res_addr,
1566 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1567 goto jit_failure;
1568 }
1569 goto done;
1570 case ZEND_ADD:
1571 case ZEND_SUB:
1572 case ZEND_MUL:
1573 // case ZEND_DIV: // TODO: check for division by zero ???
1574 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1575 break;
1576 }
1577 op1_info = OP1_INFO();
1578 op2_info = OP2_INFO();
1579 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1580 break;
1581 }
1582 if (opline->opcode == ZEND_ADD &&
1583 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
1584 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
1585 /* pass */
1586 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
1587 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1588 break;
1589 }
1590 res_addr = RES_REG_ADDR();
1591 if (Z_MODE(res_addr) != IS_REG
1592 && (i + 1) <= end
1593 && zend_jit_next_is_send_result(opline)) {
1594 i++;
1595 res_use_info = -1;
1596 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1597 if (!zend_jit_reuse_ip(&ctx)) {
1598 goto jit_failure;
1599 }
1600 } else {
1601 res_use_info = -1;
1602
1603 if (opline->result_type == IS_CV
1604 && ssa->vars
1605 && ssa_op->result_use >= 0
1606 && !ssa->vars[ssa_op->result_use].no_val) {
1607 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1608
1609 if (Z_MODE(res_use_addr) != IS_REG
1610 || Z_LOAD(res_use_addr)
1611 || Z_STORE(res_use_addr)) {
1612 res_use_info = RES_USE_INFO();
1613 }
1614 }
1615 }
1616 res_info = RES_INFO();
1617 if (opline->opcode == ZEND_ADD &&
1618 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
1619 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
1620 if (!zend_jit_add_arrays(&ctx, opline, op1_info, OP1_REG_ADDR(), op2_info, OP2_REG_ADDR(), res_addr)) {
1621 goto jit_failure;
1622 }
1623 } else {
1624 if (!zend_jit_math(&ctx, opline,
1625 op1_info, OP1_REG_ADDR(),
1626 op2_info, OP2_REG_ADDR(),
1627 res_use_info, res_info, res_addr,
1628 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1629 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1630 goto jit_failure;
1631 }
1632 }
1633 goto done;
1634 case ZEND_CONCAT:
1635 case ZEND_FAST_CONCAT:
1636 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1637 break;
1638 }
1639 op1_info = OP1_INFO();
1640 op2_info = OP2_INFO();
1641 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1642 break;
1643 }
1644 if (!(op1_info & MAY_BE_STRING) ||
1645 !(op2_info & MAY_BE_STRING)) {
1646 break;
1647 }
1648 res_addr = RES_REG_ADDR();
1649 if ((i + 1) <= end
1650 && zend_jit_next_is_send_result(opline)) {
1651 i++;
1652 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1653 if (!zend_jit_reuse_ip(&ctx)) {
1654 goto jit_failure;
1655 }
1656 }
1657 if (!zend_jit_concat(&ctx, opline,
1658 op1_info, op2_info, res_addr,
1659 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1660 goto jit_failure;
1661 }
1662 goto done;
1663 case ZEND_ASSIGN_OP:
1664 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1665 break;
1666 }
1667 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1668 break;
1669 }
1670 op1_info = OP1_INFO();
1671 op2_info = OP2_INFO();
1672 if (!zend_jit_supported_binary_op(
1673 opline->extended_value, op1_info, op2_info)) {
1674 break;
1675 }
1676 op1_addr = OP1_REG_ADDR();
1677 op1_mem_info = -1;
1678 if (Z_MODE(op1_addr) != IS_REG
1679 || Z_LOAD(op1_addr)
1680 || Z_STORE(op1_addr)) {
1681 op1_mem_info = op1_info;
1682 }
1683 op1_def_info = OP1_DEF_INFO();
1684 if (!zend_jit_assign_op(&ctx, opline,
1685 op1_info, op1_addr, OP1_RANGE(),
1686 op1_def_info, OP1_DEF_REG_ADDR(), op1_mem_info,
1687 op2_info, OP2_REG_ADDR(), OP2_RANGE(),
1688 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1689 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1690 goto jit_failure;
1691 }
1692 goto done;
1693 case ZEND_ASSIGN_DIM_OP:
1694 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1695 break;
1696 }
1697 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1698 break;
1699 }
1700 if (!zend_jit_supported_binary_op(
1701 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
1702 break;
1703 }
1704 if (!zend_jit_assign_dim_op(&ctx, opline,
1705 OP1_INFO(), OP1_DEF_INFO(), OP1_REG_ADDR(),
1706 OP2_INFO(), (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
1707 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
1708 OP1_DATA_INFO(), OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(), IS_UNKNOWN,
1709 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1710 goto jit_failure;
1711 }
1712 goto done;
1713 case ZEND_ASSIGN_DIM:
1714 if (opline->op1_type != IS_CV) {
1715 break;
1716 }
1717 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1718 break;
1719 }
1720 if (!zend_jit_assign_dim(&ctx, opline,
1721 OP1_INFO(), OP1_REG_ADDR(),
1722 OP2_INFO(), (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
1723 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
1724 OP1_DATA_INFO(), OP1_DATA_REG_ADDR(),
1725 (ctx.ra && (ssa_op+1)->op1_def >= 0) ? OP1_DATA_DEF_REG_ADDR() : 0,
1726 (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
1727 IS_UNKNOWN,
1728 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1729 goto jit_failure;
1730 }
1731 goto done;
1732 case ZEND_PRE_INC_OBJ:
1733 case ZEND_PRE_DEC_OBJ:
1734 case ZEND_POST_INC_OBJ:
1735 case ZEND_POST_DEC_OBJ:
1736 if (opline->op2_type != IS_CONST
1737 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1738 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1739 break;
1740 }
1741 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1742 break;
1743 }
1744 ce = NULL;
1745 ce_is_instanceof = 0;
1746 on_this = 0;
1747 if (opline->op1_type == IS_UNUSED) {
1748 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1749 ce = op_array->scope;
1750 /* scope is NULL for closures. */
1751 if (ce) {
1752 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
1753 }
1754 op1_addr = 0;
1755 on_this = 1;
1756 } else {
1757 op1_info = OP1_INFO();
1758 if (!(op1_info & MAY_BE_OBJECT)) {
1759 break;
1760 }
1761 op1_addr = OP1_REG_ADDR();
1762 if (ssa->var_info && ssa->ops) {
1763 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1764 if (ssa_op->op1_use >= 0) {
1765 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1766 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1767 ce = op1_ssa->ce;
1768 ce_is_instanceof = op1_ssa->is_instanceof;
1769 }
1770 }
1771 }
1772 }
1773 if (!zend_jit_incdec_obj(&ctx, opline, op_array, ssa, ssa_op,
1774 op1_info, op1_addr,
1775 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
1776 goto jit_failure;
1777 }
1778 goto done;
1779 case ZEND_ASSIGN_OBJ_OP:
1780 if (opline->result_type != IS_UNUSED) {
1781 break;
1782 }
1783 if (opline->op2_type != IS_CONST
1784 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1785 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1786 break;
1787 }
1788 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1789 break;
1790 }
1791 if (!zend_jit_supported_binary_op(
1792 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
1793 break;
1794 }
1795 ce = NULL;
1796 ce_is_instanceof = 0;
1797 on_this = 0;
1798 if (opline->op1_type == IS_UNUSED) {
1799 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1800 ce = op_array->scope;
1801 /* scope is NULL for closures. */
1802 if (ce) {
1803 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
1804 }
1805 op1_addr = 0;
1806 on_this = 1;
1807 } else {
1808 op1_info = OP1_INFO();
1809 if (!(op1_info & MAY_BE_OBJECT)) {
1810 break;
1811 }
1812 op1_addr = OP1_REG_ADDR();
1813 if (ssa->var_info && ssa->ops) {
1814 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1815 if (ssa_op->op1_use >= 0) {
1816 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1817 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1818 ce = op1_ssa->ce;
1819 ce_is_instanceof = op1_ssa->is_instanceof;
1820 }
1821 }
1822 }
1823 }
1824 if (!zend_jit_assign_obj_op(&ctx, opline, op_array, ssa, ssa_op,
1825 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(),
1826 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
1827 goto jit_failure;
1828 }
1829 goto done;
1830 case ZEND_ASSIGN_OBJ:
1831 if (opline->op2_type != IS_CONST
1832 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1833 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1834 break;
1835 }
1836 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1837 break;
1838 }
1839 ce = NULL;
1840 ce_is_instanceof = 0;
1841 on_this = 0;
1842 if (opline->op1_type == IS_UNUSED) {
1843 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1844 ce = op_array->scope;
1845 /* scope is NULL for closures. */
1846 if (ce) {
1847 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
1848 }
1849 op1_addr = 0;
1850 on_this = 1;
1851 } else {
1852 op1_info = OP1_INFO();
1853 if (!(op1_info & MAY_BE_OBJECT)) {
1854 break;
1855 }
1856 op1_addr = OP1_REG_ADDR();
1857 if (ssa->var_info && ssa->ops) {
1858 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1859 if (ssa_op->op1_use >= 0) {
1860 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1861 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1862 ce = op1_ssa->ce;
1863 ce_is_instanceof = op1_ssa->is_instanceof;
1864 }
1865 }
1866 }
1867 }
1868 if (!zend_jit_assign_obj(&ctx, opline, op_array, ssa, ssa_op,
1869 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_REG_ADDR(), OP1_DATA_DEF_REG_ADDR(),
1870 (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
1871 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN,
1872 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1873 goto jit_failure;
1874 }
1875 goto done;
1876 case ZEND_ASSIGN:
1877 if (opline->op1_type != IS_CV) {
1878 break;
1879 }
1880 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1881 break;
1882 }
1883 op2_addr = OP2_REG_ADDR();
1884 if (ra
1885 && ssa->ops[opline - op_array->opcodes].op2_def >= 0
1886 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) {
1887 op2_def_addr = OP2_DEF_REG_ADDR();
1888 } else {
1889 op2_def_addr = op2_addr;
1890 }
1891 op1_info = OP1_INFO();
1892 if (ra && ssa->vars[ssa_op->op1_use].no_val) {
1893 op1_info |= MAY_BE_UNDEF; // requres type assignment
1894 }
1895 if (opline->result_type == IS_UNUSED) {
1896 res_addr = 0;
1897 res_info = -1;
1898 } else {
1899 res_addr = RES_REG_ADDR();
1900 res_info = RES_INFO();
1901 if (Z_MODE(res_addr) != IS_REG
1902 && (i + 1) <= end
1903 && zend_jit_next_is_send_result(opline)
1904 && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) {
1905 i++;
1906 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1907 if (!zend_jit_reuse_ip(&ctx)) {
1908 goto jit_failure;
1909 }
1910 }
1911 }
1912 if (!zend_jit_assign(&ctx, opline,
1913 op1_info, OP1_REG_ADDR(),
1914 OP1_DEF_INFO(), OP1_DEF_REG_ADDR(),
1915 OP2_INFO(), op2_addr, op2_def_addr,
1916 res_info, res_addr,
1917 0,
1918 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1919 goto jit_failure;
1920 }
1921 goto done;
1922 case ZEND_QM_ASSIGN:
1923 op1_addr = OP1_REG_ADDR();
1924 if (ra
1925 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
1926 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
1927 op1_def_addr = OP1_DEF_REG_ADDR();
1928 } else {
1929 op1_def_addr = op1_addr;
1930 }
1931 if (!zend_jit_qm_assign(&ctx, opline,
1932 OP1_INFO(), op1_addr, op1_def_addr,
1933 -1, RES_INFO(), RES_REG_ADDR())) {
1934 goto jit_failure;
1935 }
1936 goto done;
1937 case ZEND_INIT_FCALL:
1938 case ZEND_INIT_FCALL_BY_NAME:
1939 case ZEND_INIT_NS_FCALL_BY_NAME:
1940 if (!zend_jit_init_fcall(&ctx, opline, b, op_array, ssa, ssa_op, call_level, NULL, 0)) {
1941 goto jit_failure;
1942 }
1943 goto done;
1944 case ZEND_SEND_VAL:
1945 case ZEND_SEND_VAL_EX:
1946 if (opline->op2_type == IS_CONST) {
1947 /* Named parameters not supported in JIT (yet) */
1948 break;
1949 }
1950 if (opline->opcode == ZEND_SEND_VAL_EX
1951 && opline->op2.num > MAX_ARG_FLAG_NUM) {
1952 break;
1953 }
1954 if (!zend_jit_send_val(&ctx, opline,
1955 OP1_INFO(), OP1_REG_ADDR())) {
1956 goto jit_failure;
1957 }
1958 goto done;
1959 case ZEND_SEND_REF:
1960 if (opline->op2_type == IS_CONST) {
1961 /* Named parameters not supported in JIT (yet) */
1962 break;
1963 }
1964 if (!zend_jit_send_ref(&ctx, opline, op_array,
1965 OP1_INFO(), 0)) {
1966 goto jit_failure;
1967 }
1968 goto done;
1969 case ZEND_SEND_VAR:
1970 case ZEND_SEND_VAR_EX:
1971 case ZEND_SEND_VAR_NO_REF:
1972 case ZEND_SEND_VAR_NO_REF_EX:
1973 case ZEND_SEND_FUNC_ARG:
1974 if (opline->op2_type == IS_CONST) {
1975 /* Named parameters not supported in JIT (yet) */
1976 break;
1977 }
1978 if ((opline->opcode == ZEND_SEND_VAR_EX
1979 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
1980 && opline->op2.num > MAX_ARG_FLAG_NUM) {
1981 break;
1982 }
1983 op1_addr = OP1_REG_ADDR();
1984 if (ra
1985 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
1986 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
1987 op1_def_addr = OP1_DEF_REG_ADDR();
1988 } else {
1989 op1_def_addr = op1_addr;
1990 }
1991 if (!zend_jit_send_var(&ctx, opline, op_array,
1992 OP1_INFO(), op1_addr, op1_def_addr)) {
1993 goto jit_failure;
1994 }
1995 goto done;
1996 case ZEND_CHECK_FUNC_ARG:
1997 if (opline->op2_type == IS_CONST) {
1998 /* Named parameters not supported in JIT (yet) */
1999 break;
2000 }
2001 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
2002 break;
2003 }
2004 if (!zend_jit_check_func_arg(&ctx, opline)) {
2005 goto jit_failure;
2006 }
2007 goto done;
2008 case ZEND_CHECK_UNDEF_ARGS:
2009 if (!zend_jit_check_undef_args(&ctx, opline)) {
2010 goto jit_failure;
2011 }
2012 goto done;
2013 case ZEND_DO_UCALL:
2014 ZEND_FALLTHROUGH;
2015 case ZEND_DO_ICALL:
2016 case ZEND_DO_FCALL_BY_NAME:
2017 case ZEND_DO_FCALL:
2018 if (!zend_jit_do_fcall(&ctx, opline, op_array, ssa, call_level, b + 1, NULL)) {
2019 goto jit_failure;
2020 }
2021 goto done;
2022 case ZEND_IS_EQUAL:
2023 case ZEND_IS_NOT_EQUAL:
2024 case ZEND_IS_SMALLER:
2025 case ZEND_IS_SMALLER_OR_EQUAL:
2026 case ZEND_CASE: {
2027 res_addr = RES_REG_ADDR();
2028 if ((opline->result_type & IS_TMP_VAR)
2029 && (i + 1) <= end
2030 && ((opline+1)->opcode == ZEND_JMPZ
2031 || (opline+1)->opcode == ZEND_JMPNZ
2032 || (opline+1)->opcode == ZEND_JMPZ_EX
2033 || (opline+1)->opcode == ZEND_JMPNZ_EX)
2034 && (opline+1)->op1_type == IS_TMP_VAR
2035 && (opline+1)->op1.var == opline->result.var) {
2036 i++;
2037 smart_branch_opcode = (opline+1)->opcode;
2038 target_label = ssa->cfg.blocks[b].successors[0];
2039 target_label2 = ssa->cfg.blocks[b].successors[1];
2040 /* For EX variant write into the result of EX opcode. */
2041 if ((opline+1)->opcode == ZEND_JMPZ_EX
2042 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
2043 res_addr = OP_REG_ADDR(opline + 1, ssa_op + 1, result_type, result, result_def);
2044 }
2045 } else {
2046 smart_branch_opcode = 0;
2047 target_label = target_label2 = (uint32_t)-1;
2048 }
2049 if (!zend_jit_cmp(&ctx, opline,
2050 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
2051 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
2052 res_addr,
2053 zend_may_throw(opline, ssa_op, op_array, ssa),
2054 smart_branch_opcode, target_label, target_label2,
2055 NULL, 0)) {
2056 goto jit_failure;
2057 }
2058 goto done;
2059 }
2060 case ZEND_IS_IDENTICAL:
2061 case ZEND_IS_NOT_IDENTICAL:
2062 case ZEND_CASE_STRICT:
2063 res_addr = RES_REG_ADDR();
2064 if ((opline->result_type & IS_TMP_VAR)
2065 && (i + 1) <= end
2066 && ((opline+1)->opcode == ZEND_JMPZ
2067 || (opline+1)->opcode == ZEND_JMPZ_EX
2068 || (opline+1)->opcode == ZEND_JMPNZ_EX
2069 || (opline+1)->opcode == ZEND_JMPNZ)
2070 && (opline+1)->op1_type == IS_TMP_VAR
2071 && (opline+1)->op1.var == opline->result.var) {
2072 i++;
2073 smart_branch_opcode = (opline+1)->opcode;
2074 target_label = ssa->cfg.blocks[b].successors[0];
2075 target_label2 = ssa->cfg.blocks[b].successors[1];
2076 /* For EX variant write into the result of EX opcode. */
2077 if ((opline+1)->opcode == ZEND_JMPZ_EX
2078 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
2079 res_addr = OP_REG_ADDR(opline + 1, ssa_op + 1, result_type, result, result_def);
2080 }
2081 } else {
2082 smart_branch_opcode = 0;
2083 target_label = target_label2 = (uint32_t)-1;
2084 }
2085 if (!zend_jit_identical(&ctx, opline,
2086 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
2087 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
2088 res_addr,
2089 zend_may_throw(opline, ssa_op, op_array, ssa),
2090 smart_branch_opcode, target_label, target_label2,
2091 NULL, 0)) {
2092 goto jit_failure;
2093 }
2094 goto done;
2095 case ZEND_DEFINED:
2096 if ((opline->result_type & IS_TMP_VAR)
2097 && (i + 1) <= end
2098 && ((opline+1)->opcode == ZEND_JMPZ
2099 || (opline+1)->opcode == ZEND_JMPNZ)
2100 && (opline+1)->op1_type == IS_TMP_VAR
2101 && (opline+1)->op1.var == opline->result.var) {
2102 i++;
2103 smart_branch_opcode = (opline+1)->opcode;
2104 target_label = ssa->cfg.blocks[b].successors[0];
2105 target_label2 = ssa->cfg.blocks[b].successors[1];
2106 } else {
2107 smart_branch_opcode = 0;
2108 target_label = target_label2 = (uint32_t)-1;
2109 }
2110 if (!zend_jit_defined(&ctx, opline, smart_branch_opcode, target_label, target_label2, NULL)) {
2111 goto jit_failure;
2112 }
2113 goto done;
2114 case ZEND_TYPE_CHECK:
2115 if (opline->extended_value == MAY_BE_RESOURCE) {
2116 // TODO: support for is_resource() ???
2117 break;
2118 }
2119 if ((opline->result_type & IS_TMP_VAR)
2120 && (i + 1) <= end
2121 && ((opline+1)->opcode == ZEND_JMPZ
2122 || (opline+1)->opcode == ZEND_JMPNZ)
2123 && (opline+1)->op1_type == IS_TMP_VAR
2124 && (opline+1)->op1.var == opline->result.var) {
2125 i++;
2126 smart_branch_opcode = (opline+1)->opcode;
2127 target_label = ssa->cfg.blocks[b].successors[0];
2128 target_label2 = ssa->cfg.blocks[b].successors[1];
2129 } else {
2130 smart_branch_opcode = 0;
2131 target_label = target_label2 = (uint32_t)-1;
2132 }
2133 if (!zend_jit_type_check(&ctx, opline, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) {
2134 goto jit_failure;
2135 }
2136 goto done;
2137 case ZEND_RETURN:
2138 op1_info = OP1_INFO();
2139 if ((PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info))
2140 || op_array->type == ZEND_EVAL_CODE
2141 // TODO: support for top-level code
2142 || !op_array->function_name
2143 // TODO: support for IS_UNDEF ???
2144 || (op1_info & MAY_BE_UNDEF)) {
2145 if (!zend_jit_tail_handler(&ctx, opline)) {
2146 goto jit_failure;
2147 }
2148 } else {
2149 if (!zend_jit_return(&ctx, opline, op_array,
2150 op1_info, OP1_REG_ADDR())) {
2151 goto jit_failure;
2152 }
2153 }
2154 goto done;
2155 case ZEND_BOOL:
2156 case ZEND_BOOL_NOT:
2157 if (!zend_jit_bool_jmpznz(&ctx, opline,
2158 OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(),
2159 -1, -1,
2160 zend_may_throw(opline, ssa_op, op_array, ssa),
2161 opline->opcode, NULL)) {
2162 goto jit_failure;
2163 }
2164 goto done;
2165 case ZEND_JMPZ:
2166 case ZEND_JMPNZ:
2167 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
2168 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2169 /* smart branch */
2170 if (!zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2171 goto jit_failure;
2172 }
2173 goto done;
2174 }
2175 ZEND_FALLTHROUGH;
2176 case ZEND_JMPZ_EX:
2177 case ZEND_JMPNZ_EX:
2178 if (opline->result_type == IS_UNDEF) {
2179 res_addr = 0;
2180 } else {
2181 res_addr = RES_REG_ADDR();
2182 }
2183 if (!zend_jit_bool_jmpznz(&ctx, opline,
2184 OP1_INFO(), OP1_REG_ADDR(), res_addr,
2185 ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1],
2186 zend_may_throw(opline, ssa_op, op_array, ssa),
2187 opline->opcode, NULL)) {
2188 goto jit_failure;
2189 }
2190 goto done;
2191 case ZEND_ISSET_ISEMPTY_CV:
2192 if ((opline->extended_value & ZEND_ISEMPTY)) {
2193 // TODO: support for empty() ???
2194 break;
2195 }
2196 if ((opline->result_type & IS_TMP_VAR)
2197 && (i + 1) <= end
2198 && ((opline+1)->opcode == ZEND_JMPZ
2199 || (opline+1)->opcode == ZEND_JMPNZ)
2200 && (opline+1)->op1_type == IS_TMP_VAR
2201 && (opline+1)->op1.var == opline->result.var) {
2202 i++;
2203 smart_branch_opcode = (opline+1)->opcode;
2204 target_label = ssa->cfg.blocks[b].successors[0];
2205 target_label2 = ssa->cfg.blocks[b].successors[1];
2206 } else {
2207 smart_branch_opcode = 0;
2208 target_label = target_label2 = (uint32_t)-1;
2209 }
2210 if (!zend_jit_isset_isempty_cv(&ctx, opline,
2211 OP1_INFO(), OP1_REG_ADDR(),
2212 smart_branch_opcode, target_label, target_label2,
2213 NULL)) {
2214 goto jit_failure;
2215 }
2216 goto done;
2217 case ZEND_IN_ARRAY:
2218 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
2219 break;
2220 }
2221 op1_info = OP1_INFO();
2222 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
2223 break;
2224 }
2225 if ((opline->result_type & IS_TMP_VAR)
2226 && (i + 1) <= end
2227 && ((opline+1)->opcode == ZEND_JMPZ
2228 || (opline+1)->opcode == ZEND_JMPNZ)
2229 && (opline+1)->op1_type == IS_TMP_VAR
2230 && (opline+1)->op1.var == opline->result.var) {
2231 i++;
2232 smart_branch_opcode = (opline+1)->opcode;
2233 target_label = ssa->cfg.blocks[b].successors[0];
2234 target_label2 = ssa->cfg.blocks[b].successors[1];
2235 } else {
2236 smart_branch_opcode = 0;
2237 target_label = target_label2 = (uint32_t)-1;
2238 }
2239 if (!zend_jit_in_array(&ctx, opline,
2240 op1_info, OP1_REG_ADDR(),
2241 smart_branch_opcode, target_label, target_label2,
2242 NULL)) {
2243 goto jit_failure;
2244 }
2245 goto done;
2246 case ZEND_FETCH_DIM_R:
2247 case ZEND_FETCH_DIM_IS:
2248 case ZEND_FETCH_LIST_R:
2249 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2250 break;
2251 }
2252 if (!zend_jit_fetch_dim_read(&ctx, opline, ssa, ssa_op,
2253 OP1_INFO(), OP1_REG_ADDR(), 0,
2254 OP2_INFO(), OP2_REG_ADDR(), OP2_RANGE(),
2255 RES_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
2256 goto jit_failure;
2257 }
2258 goto done;
2259 case ZEND_FETCH_DIM_W:
2260 case ZEND_FETCH_DIM_RW:
2261 // case ZEND_FETCH_DIM_UNSET:
2262 case ZEND_FETCH_LIST_W:
2263 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2264 break;
2265 }
2266 if (opline->op1_type != IS_CV) {
2267 break;
2268 }
2269 if (!zend_jit_fetch_dim(&ctx, opline,
2270 OP1_INFO(), OP1_REG_ADDR(),
2271 OP2_INFO(), (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
2272 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : 0,
2273 RES_REG_ADDR(), IS_UNKNOWN)) {
2274 goto jit_failure;
2275 }
2276 goto done;
2277 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
2278 if ((opline->extended_value & ZEND_ISEMPTY)) {
2279 // TODO: support for empty() ???
2280 break;
2281 }
2282 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2283 break;
2284 }
2285 if ((opline->result_type & IS_TMP_VAR)
2286 && (i + 1) <= end
2287 && ((opline+1)->opcode == ZEND_JMPZ
2288 || (opline+1)->opcode == ZEND_JMPNZ)
2289 && (opline+1)->op1_type == IS_TMP_VAR
2290 && (opline+1)->op1.var == opline->result.var) {
2291 i++;
2292 smart_branch_opcode = (opline+1)->opcode;
2293 target_label = ssa->cfg.blocks[b].successors[0];
2294 target_label2 = ssa->cfg.blocks[b].successors[1];
2295 } else {
2296 smart_branch_opcode = 0;
2297 target_label = target_label2 = (uint32_t)-1;
2298 }
2299 if (!zend_jit_isset_isempty_dim(&ctx, opline,
2300 OP1_INFO(), OP1_REG_ADDR(), 0,
2301 OP2_INFO(), OP2_REG_ADDR(), OP2_RANGE(), IS_UNKNOWN,
2302 zend_may_throw(opline, ssa_op, op_array, ssa),
2303 smart_branch_opcode, target_label, target_label2,
2304 NULL)) {
2305 goto jit_failure;
2306 }
2307 goto done;
2308 case ZEND_FETCH_OBJ_R:
2309 case ZEND_FETCH_OBJ_IS:
2310 case ZEND_FETCH_OBJ_W:
2311 if (opline->op2_type != IS_CONST
2312 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2313 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2314 break;
2315 }
2316 ce = NULL;
2317 ce_is_instanceof = 0;
2318 on_this = 0;
2319 if (opline->op1_type == IS_UNUSED) {
2320 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
2321 op1_addr = 0;
2322 ce = op_array->scope;
2323 /* scope is NULL for closures. */
2324 if (ce) {
2325 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
2326 }
2327 on_this = 1;
2328 } else {
2329 op1_info = OP1_INFO();
2330 if (!(op1_info & MAY_BE_OBJECT)) {
2331 break;
2332 }
2333 op1_addr = OP1_REG_ADDR();
2334 if (ssa->var_info && ssa->ops) {
2335 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
2336 if (ssa_op->op1_use >= 0) {
2337 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
2338 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
2339 ce = op1_ssa->ce;
2340 ce_is_instanceof = op1_ssa->is_instanceof;
2341 }
2342 }
2343 }
2344 }
2345 if (!zend_jit_fetch_obj(&ctx, opline, op_array, ssa, ssa_op,
2346 op1_info, op1_addr, 0, ce, ce_is_instanceof, on_this, 0, 0, NULL,
2347 RES_REG_ADDR(), IS_UNKNOWN,
2348 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2349 goto jit_failure;
2350 }
2351 goto done;
2352 case ZEND_BIND_GLOBAL:
2353 if (!ssa->ops || !ssa->var_info) {
2354 op1_info = MAY_BE_ANY|MAY_BE_REF;
2355 } else {
2356 op1_info = OP1_INFO();
2357 }
2358 if (!zend_jit_bind_global(&ctx, opline, op1_info)) {
2359 goto jit_failure;
2360 }
2361 goto done;
2362 case ZEND_RECV:
2363 if (!zend_jit_recv(&ctx, opline, op_array)) {
2364 goto jit_failure;
2365 }
2366 goto done;
2367 case ZEND_RECV_INIT:
2368 if (!zend_jit_recv_init(&ctx, opline, op_array,
2369 (opline + 1)->opcode != ZEND_RECV_INIT,
2370 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2371 goto jit_failure;
2372 }
2373 goto done;
2374 case ZEND_FREE:
2375 case ZEND_FE_FREE:
2376 if (!zend_jit_free(&ctx, opline, OP1_INFO(),
2377 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2378 goto jit_failure;
2379 }
2380 goto done;
2381 case ZEND_ECHO:
2382 op1_info = OP1_INFO();
2383 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2384 break;
2385 }
2386 if (!zend_jit_echo(&ctx, opline, op1_info)) {
2387 goto jit_failure;
2388 }
2389 goto done;
2390 case ZEND_STRLEN:
2391 op1_info = OP1_INFO();
2392 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2393 break;
2394 }
2395 if (!zend_jit_strlen(&ctx, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR())) {
2396 goto jit_failure;
2397 }
2398 goto done;
2399 case ZEND_COUNT:
2400 op1_info = OP1_INFO();
2401 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
2402 break;
2403 }
2404 if (!zend_jit_count(&ctx, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
2405 goto jit_failure;
2406 }
2407 goto done;
2408 case ZEND_FETCH_THIS:
2409 if (!zend_jit_fetch_this(&ctx, opline, op_array, 0)) {
2410 goto jit_failure;
2411 }
2412 goto done;
2413 case ZEND_SWITCH_LONG:
2414 case ZEND_SWITCH_STRING:
2415 case ZEND_MATCH:
2416 if (!zend_jit_switch(&ctx, opline, op_array, ssa, NULL, NULL)) {
2417 goto jit_failure;
2418 }
2419 goto done;
2420 case ZEND_VERIFY_RETURN_TYPE:
2421 if (opline->op1_type == IS_UNUSED) {
2422 /* Always throws */
2423 break;
2424 }
2425 if (opline->op1_type == IS_CONST) {
2426 /* TODO Different instruction format, has return value */
2427 break;
2428 }
2429 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
2430 /* Not worth bothering with */
2431 break;
2432 }
2433 if (OP1_INFO() & MAY_BE_REF) {
2434 /* TODO May need reference unwrapping. */
2435 break;
2436 }
2437 if (!zend_jit_verify_return_type(&ctx, opline, op_array, OP1_INFO())) {
2438 goto jit_failure;
2439 }
2440 goto done;
2441 case ZEND_FE_RESET_R:
2442 op1_info = OP1_INFO();
2443 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
2444 break;
2445 }
2446 if (!zend_jit_fe_reset(&ctx, opline, op1_info)) {
2447 goto jit_failure;
2448 }
2449 goto done;
2450 case ZEND_FE_FETCH_R:
2451 op1_info = OP1_INFO();
2452 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
2453 break;
2454 }
2455 if (!zend_jit_fe_fetch(&ctx, opline, op1_info, OP2_INFO(),
2456 ssa->cfg.blocks[b].successors[0], opline->opcode, NULL)) {
2457 goto jit_failure;
2458 }
2459 goto done;
2460 case ZEND_FETCH_CONSTANT:
2461 if (!zend_jit_fetch_constant(&ctx, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
2462 goto jit_failure;
2463 }
2464 goto done;
2465 case ZEND_INIT_METHOD_CALL:
2466 if (opline->op2_type != IS_CONST
2467 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2468 break;
2469 }
2470 ce = NULL;
2471 ce_is_instanceof = 0;
2472 on_this = 0;
2473 if (opline->op1_type == IS_UNUSED) {
2474 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
2475 op1_addr = 0;
2476 ce = op_array->scope;
2477 /* scope is NULL for closures. */
2478 if (ce) {
2479 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
2480 }
2481 on_this = 1;
2482 } else {
2483 op1_info = OP1_INFO();
2484 if (!(op1_info & MAY_BE_OBJECT)) {
2485 break;
2486 }
2487 op1_addr = OP1_REG_ADDR();
2488 if (ssa->var_info && ssa->ops) {
2489 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
2490 if (ssa_op->op1_use >= 0) {
2491 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
2492 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
2493 ce = op1_ssa->ce;
2494 ce_is_instanceof = op1_ssa->is_instanceof;
2495 }
2496 }
2497 }
2498 }
2499 if (!zend_jit_init_method_call(&ctx, opline, b, op_array, ssa, ssa_op, call_level,
2500 op1_info, op1_addr, ce, ce_is_instanceof, on_this, 0, NULL,
2501 NULL, 0,
2502 -1, -1,
2503 0)) {
2504 goto jit_failure;
2505 }
2506 goto done;
2507 case ZEND_ROPE_INIT:
2508 case ZEND_ROPE_ADD:
2509 case ZEND_ROPE_END:
2510 op2_info = OP2_INFO();
2511 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2512 break;
2513 }
2514 if (!zend_jit_rope(&ctx, opline, op2_info)) {
2515 goto jit_failure;
2516 }
2517 goto done;
2518 default:
2519 break;
2520 }
2521 }
2522
2523 switch (opline->opcode) {
2524 case ZEND_RECV_INIT:
2525 case ZEND_BIND_GLOBAL:
2526 if (opline == op_array->opcodes ||
2527 opline->opcode != op_array->opcodes[i-1].opcode) {
2528 /* repeatable opcodes */
2529 if (!zend_jit_handler(&ctx, opline,
2530 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2531 goto jit_failure;
2532 }
2533 }
2534 zend_jit_set_last_valid_opline(&ctx, opline+1);
2535 break;
2536 case ZEND_NOP:
2537 case ZEND_OP_DATA:
2538 case ZEND_SWITCH_LONG:
2539 case ZEND_SWITCH_STRING:
2540 break;
2541 case ZEND_MATCH:
2542 /* We have to exit to the VM because the MATCH handler performs an N-way jump for
2543 * which we can't generate simple (opcache.jit=1201) JIT code. */
2544 if (!zend_jit_tail_handler(&ctx, opline)) {
2545 goto jit_failure;
2546 }
2547 break;
2548 case ZEND_JMP:
2549 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
2550 const zend_op *target = OP_JMP_ADDR(opline, opline->op1);
2551
2552 if (!zend_jit_set_ip(&ctx, target)) {
2553 goto jit_failure;
2554 }
2555 }
2556 break;
2557 case ZEND_CATCH:
2558 case ZEND_FAST_CALL:
2559 case ZEND_FAST_RET:
2560 case ZEND_GENERATOR_CREATE:
2561 case ZEND_GENERATOR_RETURN:
2562 case ZEND_RETURN_BY_REF:
2563 case ZEND_RETURN:
2564 case ZEND_MATCH_ERROR:
2565 /* switch through trampoline */
2566 case ZEND_YIELD:
2567 case ZEND_YIELD_FROM:
2568 case ZEND_THROW:
2569 case ZEND_VERIFY_NEVER_TYPE:
2570 if (!zend_jit_tail_handler(&ctx, opline)) {
2571 goto jit_failure;
2572 }
2573 /* THROW and EXIT may be used in the middle of BB */
2574 /* don't generate code for the rest of BB */
2575 i = end;
2576 break;
2577 /* stackless execution */
2578 case ZEND_INCLUDE_OR_EVAL:
2579 case ZEND_DO_FCALL:
2580 case ZEND_DO_UCALL:
2581 case ZEND_DO_FCALL_BY_NAME:
2582 if (!zend_jit_call(&ctx, opline, b + 1)) {
2583 goto jit_failure;
2584 }
2585 break;
2586 case ZEND_JMPZ:
2587 case ZEND_JMPNZ:
2588 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
2589 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2590 /* smart branch */
2591 if (!zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2592 goto jit_failure;
2593 }
2594 goto done;
2595 }
2596 ZEND_FALLTHROUGH;
2597 case ZEND_JMPZ_EX:
2598 case ZEND_JMPNZ_EX:
2599 case ZEND_JMP_SET:
2600 case ZEND_COALESCE:
2601 case ZEND_JMP_NULL:
2602 case ZEND_FE_RESET_R:
2603 case ZEND_FE_RESET_RW:
2604 case ZEND_ASSERT_CHECK:
2605 case ZEND_FE_FETCH_R:
2606 case ZEND_FE_FETCH_RW:
2607 case ZEND_BIND_INIT_STATIC_OR_JMP:
2608 if (!zend_jit_handler(&ctx, opline,
2609 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
2610 !zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2611 goto jit_failure;
2612 }
2613 break;
2614 case ZEND_JMP_FRAMELESS:
2615 if (!zend_jit_jmp_frameless(&ctx, opline, /* exit_addr */ NULL, /* guard */ 0)) {
2616 goto jit_failure;
2617 }
2618 break;
2619 case ZEND_NEW:
2620 if (!zend_jit_handler(&ctx, opline, 1)) {
2621 return 0;
2622 }
2623 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
2624 zend_class_entry *ce = NULL;
2625
2626 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
2627 if (ssa->ops && ssa->var_info) {
2628 zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
2629 if (res_ssa->ce && !res_ssa->is_instanceof) {
2630 ce = res_ssa->ce;
2631 }
2632 }
2633 } else {
2634 if (opline->op1_type == IS_CONST) {
2635 zval *zv = RT_CONSTANT(opline, opline->op1);
2636 if (Z_TYPE_P(zv) == IS_STRING) {
2637 zval *lc = zv + 1;
2638 ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc));
2639 }
2640 }
2641 }
2642
2643 i++;
2644
2645 if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) {
2646 const zend_op *next_opline = opline + 1;
2647
2648 ZEND_ASSERT(b + 1 == ssa->cfg.blocks[b].successors[0]);
2649 zend_jit_constructor(&ctx, next_opline, op_array, ssa, call_level, b + 1);
2650 }
2651
2652 /* We skip over the DO_FCALL, so decrement call_level ourselves. */
2653 call_level--;
2654 }
2655 break;
2656 case ZEND_FRAMELESS_ICALL_0:
2657 jit_frameless_icall0(jit, opline);
2658 goto done;
2659 case ZEND_FRAMELESS_ICALL_1:
2660 op1_info = OP1_INFO();
2661 jit_frameless_icall1(jit, opline, op1_info);
2662 goto done;
2663 case ZEND_FRAMELESS_ICALL_2:
2664 op1_info = OP1_INFO();
2665 op2_info = OP2_INFO();
2666 jit_frameless_icall2(jit, opline, op1_info, op2_info);
2667 goto done;
2668 case ZEND_FRAMELESS_ICALL_3:
2669 op1_info = OP1_INFO();
2670 op2_info = OP2_INFO();
2671 jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO());
2672 goto done;
2673 default:
2674 if (!zend_jit_handler(&ctx, opline,
2675 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2676 goto jit_failure;
2677 }
2678 if (i == end
2679 && (opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2680 /* smart branch split across basic blocks */
2681 if (!zend_jit_set_cond(&ctx, opline + 2, opline->result.var)) {
2682 goto jit_failure;
2683 }
2684 }
2685 }
2686 done:
2687 switch (opline->opcode) {
2688 case ZEND_DO_FCALL:
2689 case ZEND_DO_ICALL:
2690 case ZEND_DO_UCALL:
2691 case ZEND_DO_FCALL_BY_NAME:
2692 case ZEND_CALLABLE_CONVERT:
2693 call_level--;
2694 }
2695 }
2696 zend_jit_bb_end(&ctx, b);
2697 }
2698
2699 if (jit->return_inputs) {
2700 zend_jit_common_return(jit);
2701
2702 bool left_frame = 0;
2703 if (op_array->last_var > 100) {
2704 /* To many CVs to unroll */
2705 if (!zend_jit_free_cvs(&ctx)) {
2706 goto jit_failure;
2707 }
2708 left_frame = 1;
2709 }
2710 if (!left_frame) {
2711 int j;
2712
2713 for (j = 0 ; j < op_array->last_var; j++) {
2714 uint32_t info = zend_ssa_cv_info(op_array, ssa, j);
2715
2716 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
2717 if (!left_frame) {
2718 left_frame = 1;
2719 if (!zend_jit_leave_frame(&ctx)) {
2720 goto jit_failure;
2721 }
2722 }
2723 if (!zend_jit_free_cv(&ctx, info, j)) {
2724 goto jit_failure;
2725 }
2726 }
2727 }
2728 }
2729 if (!zend_jit_leave_func(&ctx, op_array, NULL, MAY_BE_ANY, left_frame,
2730 NULL, NULL, (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, 1)) {
2731 goto jit_failure;
2732 }
2733 }
2734
2735 handler = zend_jit_finish(&ctx);
2736 if (!handler) {
2737 goto jit_failure;
2738 }
2739 zend_jit_free_ctx(&ctx);
2740
2741 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2742 zend_arena_release(&CG(arena), checkpoint);
2743 }
2744 return SUCCESS;
2745
2746 jit_failure:
2747 zend_jit_free_ctx(&ctx);
2748 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2749 zend_arena_release(&CG(arena), checkpoint);
2750 }
2751 return FAILURE;
2752 }
2753
zend_jit_collect_calls(zend_op_array * op_array,zend_script * script)2754 static void zend_jit_collect_calls(zend_op_array *op_array, zend_script *script)
2755 {
2756 zend_func_info *func_info;
2757
2758 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
2759 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
2760 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
2761 func_info = ZEND_FUNC_INFO(op_array);
2762 } else {
2763 func_info = zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info));
2764 ZEND_SET_FUNC_INFO(op_array, func_info);
2765 }
2766 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, op_array, func_info);
2767 }
2768
zend_jit_cleanup_func_info(zend_op_array * op_array)2769 static void zend_jit_cleanup_func_info(zend_op_array *op_array)
2770 {
2771 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
2772 zend_call_info *caller_info, *callee_info;
2773
2774 if (func_info) {
2775 caller_info = func_info->caller_info;
2776 callee_info = func_info->callee_info;
2777
2778 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
2779 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
2780 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
2781 func_info->num = 0;
2782 func_info->flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
2783 | ZEND_FUNC_JIT_ON_PROF_REQUEST
2784 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
2785 | ZEND_FUNC_JIT_ON_HOT_TRACE;
2786 memset(&func_info->ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
2787 } else {
2788 ZEND_SET_FUNC_INFO(op_array, NULL);
2789 }
2790
2791 while (caller_info) {
2792 if (caller_info->caller_op_array) {
2793 zend_jit_cleanup_func_info(caller_info->caller_op_array);
2794 }
2795 caller_info = caller_info->next_caller;
2796 }
2797 while (callee_info) {
2798 if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) {
2799 zend_jit_cleanup_func_info(&callee_info->callee_func->op_array);
2800 }
2801 callee_info = callee_info->next_callee;
2802 }
2803 }
2804 }
2805
zend_real_jit_func(zend_op_array * op_array,zend_script * script,const zend_op * rt_opline)2806 static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline)
2807 {
2808 zend_ssa ssa;
2809 void *checkpoint;
2810 zend_func_info *func_info;
2811
2812 if (*dasm_ptr == dasm_end) {
2813 return FAILURE;
2814 }
2815
2816 checkpoint = zend_arena_checkpoint(CG(arena));
2817
2818 /* Build SSA */
2819 memset(&ssa, 0, sizeof(zend_ssa));
2820
2821 if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) {
2822 goto jit_failure;
2823 }
2824
2825 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
2826 zend_jit_collect_calls(op_array, script);
2827 func_info = ZEND_FUNC_INFO(op_array);
2828 func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array);
2829 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
2830 zend_init_func_return_info(op_array, script, &func_info->return_info);
2831 }
2832 }
2833
2834 if (zend_jit_op_array_analyze2(op_array, script, &ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
2835 goto jit_failure;
2836 }
2837
2838 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
2839 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa);
2840 }
2841
2842 if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) {
2843 goto jit_failure;
2844 }
2845
2846 zend_jit_cleanup_func_info(op_array);
2847 zend_arena_release(&CG(arena), checkpoint);
2848 return SUCCESS;
2849
2850 jit_failure:
2851 zend_jit_cleanup_func_info(op_array);
2852 zend_arena_release(&CG(arena), checkpoint);
2853 return FAILURE;
2854 }
2855
2856 /* Run-time JIT handler */
zend_runtime_jit(void)2857 static int ZEND_FASTCALL zend_runtime_jit(void)
2858 {
2859 zend_execute_data *execute_data = EG(current_execute_data);
2860 zend_op_array *op_array = &EX(func)->op_array;
2861 zend_op *opline = op_array->opcodes;
2862 zend_jit_op_array_extension *jit_extension;
2863 bool do_bailout = 0;
2864
2865 zend_shared_alloc_lock();
2866
2867 if (ZEND_FUNC_INFO(op_array)) {
2868
2869 SHM_UNPROTECT();
2870 zend_jit_unprotect();
2871
2872 zend_try {
2873 /* restore original opcode handlers */
2874 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2875 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
2876 opline++;
2877 }
2878 }
2879 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
2880 opline->handler = jit_extension->orig_handler;
2881
2882 /* perform real JIT for this function */
2883 zend_real_jit_func(op_array, NULL, NULL);
2884 } zend_catch {
2885 do_bailout = true;
2886 } zend_end_try();
2887
2888 zend_jit_protect();
2889 SHM_PROTECT();
2890 }
2891
2892 zend_shared_alloc_unlock();
2893
2894 if (do_bailout) {
2895 zend_bailout();
2896 }
2897
2898 /* JIT-ed code is going to be called by VM */
2899 return 0;
2900 }
2901
zend_jit_check_funcs(HashTable * function_table,bool is_method)2902 void zend_jit_check_funcs(HashTable *function_table, bool is_method) {
2903 zend_op *opline;
2904 zend_function *func;
2905 zend_op_array *op_array;
2906 uintptr_t counter;
2907 zend_jit_op_array_extension *jit_extension;
2908
2909 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(function_table, func) {
2910 if (func->type == ZEND_INTERNAL_FUNCTION) {
2911 break;
2912 }
2913 op_array = &func->op_array;
2914 opline = op_array->opcodes;
2915 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2916 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
2917 opline++;
2918 }
2919 }
2920 if (opline->handler == zend_jit_profile_jit_handler) {
2921 if (!RUN_TIME_CACHE(op_array)) {
2922 continue;
2923 }
2924 counter = (uintptr_t)ZEND_COUNTER_INFO(op_array);
2925 ZEND_COUNTER_INFO(op_array) = 0;
2926 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
2927 opline->handler = jit_extension->orig_handler;
2928 if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) {
2929 zend_real_jit_func(op_array, NULL, NULL);
2930 }
2931 }
2932 } ZEND_HASH_FOREACH_END();
2933 }
2934
zend_jit_hot_func(zend_execute_data * execute_data,const zend_op * opline)2935 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
2936 {
2937 zend_op_array *op_array = &EX(func)->op_array;
2938 zend_jit_op_array_hot_extension *jit_extension;
2939 uint32_t i;
2940 bool do_bailout = 0;
2941
2942 zend_shared_alloc_lock();
2943 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
2944
2945 if (jit_extension) {
2946 SHM_UNPROTECT();
2947 zend_jit_unprotect();
2948
2949 zend_try {
2950 for (i = 0; i < op_array->last; i++) {
2951 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
2952 }
2953
2954 /* perform real JIT for this function */
2955 zend_real_jit_func(op_array, NULL, opline);
2956 } zend_catch {
2957 do_bailout = 1;
2958 } zend_end_try();
2959
2960 zend_jit_protect();
2961 SHM_PROTECT();
2962 }
2963
2964 zend_shared_alloc_unlock();
2965
2966 if (do_bailout) {
2967 zend_bailout();
2968 }
2969 /* JIT-ed code is going to be called by VM */
2970 }
2971
zend_jit_setup_hot_counters_ex(zend_op_array * op_array,zend_cfg * cfg)2972 static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg)
2973 {
2974 if (JIT_G(hot_func)) {
2975 zend_op *opline = op_array->opcodes;
2976
2977 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2978 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
2979 opline++;
2980 }
2981 }
2982
2983 opline->handler = (const void*)zend_jit_func_hot_counter_handler;
2984 }
2985
2986 if (JIT_G(hot_loop)) {
2987 uint32_t i;
2988
2989 for (i = 0; i < cfg->blocks_count; i++) {
2990 if ((cfg->blocks[i].flags & ZEND_BB_REACHABLE) &&
2991 (cfg->blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
2992 op_array->opcodes[cfg->blocks[i].start].handler =
2993 (const void*)zend_jit_loop_hot_counter_handler;
2994 }
2995 }
2996 }
2997 }
2998
zend_jit_restart_hot_counters(zend_op_array * op_array)2999 static int zend_jit_restart_hot_counters(zend_op_array *op_array)
3000 {
3001 zend_jit_op_array_hot_extension *jit_extension;
3002 zend_cfg cfg;
3003 uint32_t i;
3004
3005 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
3006 for (i = 0; i < op_array->last; i++) {
3007 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
3008 }
3009
3010 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
3011 return FAILURE;
3012 }
3013
3014 zend_jit_setup_hot_counters_ex(op_array, &cfg);
3015
3016 return SUCCESS;
3017 }
3018
zend_jit_setup_hot_counters(zend_op_array * op_array)3019 static int zend_jit_setup_hot_counters(zend_op_array *op_array)
3020 {
3021 zend_jit_op_array_hot_extension *jit_extension;
3022 zend_cfg cfg;
3023 uint32_t i;
3024
3025 ZEND_ASSERT(!JIT_G(hot_func) || zend_jit_func_hot_counter_handler != NULL);
3026 ZEND_ASSERT(!JIT_G(hot_loop) || zend_jit_loop_hot_counter_handler != NULL);
3027
3028 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
3029 return FAILURE;
3030 }
3031
3032 jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*));
3033 if (!jit_extension) {
3034 return FAILURE;
3035 }
3036 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3037 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS;
3038 jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
3039 for (i = 0; i < op_array->last; i++) {
3040 jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
3041 }
3042 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3043
3044 zend_jit_setup_hot_counters_ex(op_array, &cfg);
3045
3046 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3047
3048 return SUCCESS;
3049 }
3050
3051 #include "jit/zend_jit_trace.c"
3052
zend_jit_op_array(zend_op_array * op_array,zend_script * script)3053 int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
3054 {
3055 if (dasm_ptr == NULL) {
3056 return FAILURE;
3057 }
3058
3059 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) {
3060 zend_jit_op_array_extension *jit_extension;
3061 zend_op *opline = op_array->opcodes;
3062
3063 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
3064 ZEND_SET_FUNC_INFO(op_array, NULL);
3065 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
3066 return SUCCESS;
3067 }
3068
3069 /* Set run-time JIT handler */
3070 ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL);
3071 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3072 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3073 opline++;
3074 }
3075 }
3076 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
3077 if (!jit_extension) {
3078 return FAILURE;
3079 }
3080 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3081 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC;
3082 jit_extension->orig_handler = (void*)opline->handler;
3083 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3084 opline->handler = (const void*)zend_jit_runtime_jit_handler;
3085 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3086
3087 return SUCCESS;
3088 } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) {
3089 zend_jit_op_array_extension *jit_extension;
3090 zend_op *opline = op_array->opcodes;
3091
3092 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
3093 ZEND_SET_FUNC_INFO(op_array, NULL);
3094 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
3095 return SUCCESS;
3096 }
3097
3098 ZEND_ASSERT(zend_jit_profile_jit_handler != NULL);
3099 if (op_array->function_name) {
3100 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3101 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3102 opline++;
3103 }
3104 }
3105 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
3106 if (!jit_extension) {
3107 return FAILURE;
3108 }
3109 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3110 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST;
3111 jit_extension->orig_handler = (void*)opline->handler;
3112 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3113 opline->handler = (const void*)zend_jit_profile_jit_handler;
3114 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3115 }
3116
3117 return SUCCESS;
3118 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
3119 return zend_jit_setup_hot_counters(op_array);
3120 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3121 return zend_jit_setup_hot_trace_counters(op_array);
3122 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3123 return zend_real_jit_func(op_array, script, NULL);
3124 } else {
3125 ZEND_UNREACHABLE();
3126 }
3127 return FAILURE;
3128 }
3129
zend_jit_script(zend_script * script)3130 int zend_jit_script(zend_script *script)
3131 {
3132 void *checkpoint;
3133 zend_call_graph call_graph;
3134 zend_func_info *info;
3135 int i;
3136
3137 if (dasm_ptr == NULL || *dasm_ptr == dasm_end) {
3138 return FAILURE;
3139 }
3140
3141 checkpoint = zend_arena_checkpoint(CG(arena));
3142
3143 call_graph.op_arrays_count = 0;
3144 zend_build_call_graph(&CG(arena), script, &call_graph);
3145
3146 zend_analyze_call_graph(&CG(arena), script, &call_graph);
3147
3148 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
3149 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
3150 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS ||
3151 JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3152 for (i = 0; i < call_graph.op_arrays_count; i++) {
3153 if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
3154 goto jit_failure;
3155 }
3156 }
3157 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3158 for (i = 0; i < call_graph.op_arrays_count; i++) {
3159 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3160 if (info) {
3161 if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
3162 goto jit_failure;
3163 }
3164 info->ssa.cfg.flags |= info->flags;
3165 info->flags = info->ssa.cfg.flags;
3166 }
3167 }
3168
3169 for (i = 0; i < call_graph.op_arrays_count; i++) {
3170 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3171 if (info) {
3172 info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]);
3173 if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
3174 zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info);
3175 }
3176 }
3177 }
3178
3179 for (i = 0; i < call_graph.op_arrays_count; i++) {
3180 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3181 if (info) {
3182 if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
3183 goto jit_failure;
3184 }
3185 info->flags = info->ssa.cfg.flags;
3186 }
3187 }
3188
3189 for (i = 0; i < call_graph.op_arrays_count; i++) {
3190 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3191 if (info) {
3192 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
3193 zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa);
3194 }
3195 if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) {
3196 goto jit_failure;
3197 }
3198 }
3199 }
3200
3201 for (i = 0; i < call_graph.op_arrays_count; i++) {
3202 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
3203 }
3204 } else {
3205 ZEND_UNREACHABLE();
3206 }
3207
3208 zend_arena_release(&CG(arena), checkpoint);
3209
3210 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC
3211 || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST
3212 || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS
3213 || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3214 zend_class_entry *ce;
3215 zend_op_array *op_array;
3216
3217 ZEND_HASH_MAP_FOREACH_PTR(&script->class_table, ce) {
3218 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
3219 if (!ZEND_FUNC_INFO(op_array)) {
3220 void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes);
3221
3222 if (jit_extension) {
3223 ZEND_SET_FUNC_INFO(op_array, jit_extension);
3224 }
3225 }
3226 } ZEND_HASH_FOREACH_END();
3227 } ZEND_HASH_FOREACH_END();
3228 }
3229
3230 return SUCCESS;
3231
3232 jit_failure:
3233 if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3234 for (i = 0; i < call_graph.op_arrays_count; i++) {
3235 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
3236 }
3237 }
3238 zend_arena_release(&CG(arena), checkpoint);
3239 return FAILURE;
3240 }
3241
zend_jit_unprotect(void)3242 void zend_jit_unprotect(void)
3243 {
3244 #ifdef HAVE_MPROTECT
3245 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3246 int opts = PROT_READ | PROT_WRITE;
3247 #ifdef ZTS
3248 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3249 if (zend_write_protect) {
3250 pthread_jit_write_protect_np(0);
3251 }
3252 #endif
3253 opts |= PROT_EXEC;
3254 #endif
3255 if (mprotect(dasm_buf, dasm_size, opts) != 0) {
3256 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3257 }
3258 }
3259 #elif defined(_WIN32)
3260 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3261 DWORD old, new;
3262 #ifdef ZTS
3263 new = PAGE_EXECUTE_READWRITE;
3264 #else
3265 new = PAGE_READWRITE;
3266 #endif
3267 if (!VirtualProtect(dasm_buf, dasm_size, new, &old)) {
3268 DWORD err = GetLastError();
3269 char *msg = php_win32_error_to_msg(err);
3270 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3271 php_win32_error_msg_free(msg);
3272 }
3273 }
3274 #endif
3275 }
3276
zend_jit_protect(void)3277 void zend_jit_protect(void)
3278 {
3279 #ifdef HAVE_MPROTECT
3280 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3281 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3282 if (zend_write_protect) {
3283 pthread_jit_write_protect_np(1);
3284 }
3285 #endif
3286 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
3287 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3288 }
3289 }
3290 #elif defined(_WIN32)
3291 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3292 DWORD old;
3293
3294 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
3295 DWORD err = GetLastError();
3296 char *msg = php_win32_error_to_msg(err);
3297 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3298 php_win32_error_msg_free(msg);
3299 }
3300 }
3301 #endif
3302 }
3303
zend_jit_init_handlers(void)3304 static void zend_jit_init_handlers(void)
3305 {
3306 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
3307 zend_jit_runtime_jit_handler = zend_jit_stub_handlers[jit_stub_hybrid_runtime_jit];
3308 zend_jit_profile_jit_handler = zend_jit_stub_handlers[jit_stub_hybrid_profile_jit];
3309 zend_jit_func_hot_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_func_hot_counter];
3310 zend_jit_loop_hot_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_loop_hot_counter];
3311 zend_jit_func_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_func_trace_counter];
3312 zend_jit_ret_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_ret_trace_counter];
3313 zend_jit_loop_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_loop_trace_counter];
3314 } else {
3315 zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
3316 zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
3317 zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper;
3318 zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper;
3319 zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper;
3320 zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper;
3321 zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper;
3322 }
3323 }
3324
zend_jit_globals_ctor(zend_jit_globals * jit_globals)3325 static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
3326 {
3327 memset(jit_globals, 0, sizeof(zend_jit_globals));
3328 zend_jit_trace_init_caches();
3329 }
3330
3331 #ifdef ZTS
zend_jit_globals_dtor(zend_jit_globals * jit_globals)3332 static void zend_jit_globals_dtor(zend_jit_globals *jit_globals)
3333 {
3334 zend_jit_trace_free_caches(jit_globals);
3335 }
3336 #endif
3337
zend_jit_parse_config_num(zend_long jit)3338 static int zend_jit_parse_config_num(zend_long jit)
3339 {
3340 if (jit == 0) {
3341 JIT_G(on) = 0;
3342 return SUCCESS;
3343 }
3344
3345 if (jit < 0) return FAILURE;
3346
3347 if (jit % 10 == 0 || jit % 10 > 5) return FAILURE;
3348 JIT_G(opt_level) = jit % 10;
3349
3350 jit /= 10;
3351 if (jit % 10 > 5 || jit % 10 == 4) return FAILURE;
3352 JIT_G(trigger) = jit % 10;
3353
3354 jit /= 10;
3355 if (jit % 10 > 2) return FAILURE;
3356 JIT_G(opt_flags) = jit % 10;
3357
3358 jit /= 10;
3359 if (jit % 10 > 1) return FAILURE;
3360 JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0);
3361
3362 if (jit / 10 != 0) return FAILURE;
3363
3364 JIT_G(on) = 1;
3365
3366 return SUCCESS;
3367 }
3368
zend_jit_config(zend_string * jit,int stage)3369 int zend_jit_config(zend_string *jit, int stage)
3370 {
3371 if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) {
3372 if (stage == ZEND_INI_STAGE_RUNTIME) {
3373 zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)");
3374 }
3375 return FAILURE;
3376 }
3377
3378 if (zend_string_equals_literal_ci(jit, "disable")) {
3379 JIT_G(enabled) = 0;
3380 JIT_G(on) = 0;
3381 return SUCCESS;
3382 } else if (ZSTR_LEN(jit) == 0
3383 || zend_string_equals_literal_ci(jit, "0")
3384 || zend_string_equals_literal_ci(jit, "off")
3385 || zend_string_equals_literal_ci(jit, "no")
3386 || zend_string_equals_literal_ci(jit, "false")) {
3387 JIT_G(enabled) = 1;
3388 JIT_G(on) = 0;
3389 return SUCCESS;
3390 } else if (zend_string_equals_literal_ci(jit, "1")
3391 || zend_string_equals_literal_ci(jit, "on")
3392 || zend_string_equals_literal_ci(jit, "yes")
3393 || zend_string_equals_literal_ci(jit, "true")
3394 || zend_string_equals_literal_ci(jit, "tracing")) {
3395 JIT_G(enabled) = 1;
3396 JIT_G(on) = 1;
3397 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS;
3398 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
3399 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
3400 return SUCCESS;
3401 } else if (zend_string_equals_ci(jit, ZSTR_KNOWN(ZEND_STR_FUNCTION))) {
3402 JIT_G(enabled) = 1;
3403 JIT_G(on) = 1;
3404 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT;
3405 JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD;
3406 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
3407 return SUCCESS;
3408 } else {
3409 char *end;
3410 zend_long num = ZEND_STRTOL(ZSTR_VAL(jit), &end, 10);
3411 if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) {
3412 goto failure;
3413 }
3414 JIT_G(enabled) = 1;
3415 return SUCCESS;
3416 }
3417
3418 failure:
3419 zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number");
3420 JIT_G(enabled) = 0;
3421 JIT_G(on) = 0;
3422 return FAILURE;
3423 }
3424
zend_jit_debug_config(zend_long old_val,zend_long new_val,int stage)3425 int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage)
3426 {
3427 if (stage != ZEND_INI_STAGE_STARTUP) {
3428 if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) {
3429 if (stage == ZEND_INI_STAGE_RUNTIME) {
3430 zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup");
3431 }
3432 return FAILURE;
3433 }
3434 }
3435 return SUCCESS;
3436 }
3437
zend_jit_init(void)3438 void zend_jit_init(void)
3439 {
3440 #ifdef ZTS
3441 jit_globals_id = ts_allocate_id(&jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, (ts_allocate_dtor) zend_jit_globals_dtor);
3442 #else
3443 zend_jit_globals_ctor(&jit_globals);
3444 #endif
3445 }
3446
zend_jit_check_support(void)3447 int zend_jit_check_support(void)
3448 {
3449 int i;
3450
3451 zend_jit_vm_kind = zend_vm_kind();
3452 if (zend_jit_vm_kind != ZEND_VM_KIND_CALL &&
3453 zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
3454 zend_error(E_WARNING, "JIT is compatible only with CALL and HYBRID VM. JIT disabled.");
3455 JIT_G(enabled) = 0;
3456 JIT_G(on) = 0;
3457 return FAILURE;
3458 }
3459
3460 if (zend_execute_ex != execute_ex) {
3461 if (zend_dtrace_enabled) {
3462 zend_error(E_WARNING, "JIT is incompatible with DTrace. JIT disabled.");
3463 } else if (strcmp(sapi_module.name, "phpdbg") != 0) {
3464 zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled.");
3465 }
3466 JIT_G(enabled) = 0;
3467 JIT_G(on) = 0;
3468 return FAILURE;
3469 }
3470
3471 for (i = 0; i <= 256; i++) {
3472 switch (i) {
3473 /* JIT has no effect on these opcodes */
3474 case ZEND_BEGIN_SILENCE:
3475 case ZEND_END_SILENCE:
3476 break;
3477 default:
3478 if (zend_get_user_opcode_handler(i) != NULL) {
3479 zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled.");
3480 JIT_G(enabled) = 0;
3481 JIT_G(on) = 0;
3482 return FAILURE;
3483 }
3484 }
3485 }
3486
3487 #if defined(IR_TARGET_AARCH64)
3488 if (JIT_G(buffer_size) > 128*1024*1024) {
3489 zend_error(E_WARNING, "JIT on AArch64 doesn't support opcache.jit_buffer_size above 128M.");
3490 JIT_G(enabled) = 0;
3491 JIT_G(on) = 0;
3492 return FAILURE;
3493 }
3494 #elif defined(IR_TARGET_X64)
3495 if (JIT_G(buffer_size) > 2 * Z_L(1024*1024*1024)) {
3496 zend_error(E_WARNING, "JIT on x86_64 doesn't support opcache.jit_buffer_size above 2G.");
3497 JIT_G(enabled) = 0;
3498 JIT_G(on) = 0;
3499 return FAILURE;
3500 }
3501 #endif
3502
3503 return SUCCESS;
3504 }
3505
zend_jit_startup(void * buf,size_t size,bool reattached)3506 void zend_jit_startup(void *buf, size_t size, bool reattached)
3507 {
3508 zend_jit_halt_op = zend_get_halt_op();
3509 zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(ACCELERATOR_PRODUCT_NAME);
3510
3511 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3512 zend_write_protect = pthread_jit_write_protect_supported_np();
3513 #endif
3514
3515 dasm_buf = buf;
3516 dasm_size = size;
3517 dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr) * 2);
3518
3519 #ifdef HAVE_MPROTECT
3520 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3521 if (zend_write_protect) {
3522 pthread_jit_write_protect_np(1);
3523 }
3524 #endif
3525 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
3526 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
3527 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3528 }
3529 } else {
3530 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
3531 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3532 }
3533 }
3534 #elif defined(_WIN32)
3535 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
3536 DWORD old;
3537
3538 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
3539 DWORD err = GetLastError();
3540 char *msg = php_win32_error_to_msg(err);
3541 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3542 php_win32_error_msg_free(msg);
3543 }
3544 } else {
3545 DWORD old;
3546
3547 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
3548 DWORD err = GetLastError();
3549 char *msg = php_win32_error_to_msg(err);
3550 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3551 php_win32_error_msg_free(msg);
3552 }
3553 }
3554 #endif
3555
3556 if (!reattached) {
3557 zend_jit_unprotect();
3558 *dasm_ptr = dasm_buf;
3559 #if defined(_WIN32)
3560 zend_jit_stub_handlers = dasm_buf;
3561 *dasm_ptr = (void**)*dasm_ptr + sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
3562 #elif defined(IR_TARGET_AARCH64)
3563 zend_jit_stub_handlers = dasm_buf;
3564 *dasm_ptr = (void**)*dasm_ptr + (sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])) * 2;
3565 memset(zend_jit_stub_handlers, 0, (sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])) * 2 * sizeof(void*));
3566 #endif
3567 *dasm_ptr = (void*)ZEND_MM_ALIGNED_SIZE_EX(((size_t)(*dasm_ptr)), 16);
3568 zend_jit_protect();
3569 } else {
3570 #if defined(_WIN32) || defined(IR_TARGET_AARCH64)
3571 zend_jit_stub_handlers = dasm_buf;
3572 zend_jit_init_handlers();
3573 #endif
3574 }
3575
3576 zend_jit_unprotect();
3577 zend_jit_setup();
3578 zend_jit_protect();
3579 zend_jit_init_handlers();
3580
3581 zend_jit_trace_startup(reattached);
3582
3583 zend_jit_unprotect();
3584 /* save JIT buffer pos */
3585 dasm_ptr[1] = dasm_ptr[0];
3586 zend_jit_protect();
3587 }
3588
zend_jit_shutdown(void)3589 void zend_jit_shutdown(void)
3590 {
3591 if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE && dasm_ptr != NULL) {
3592 fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf));
3593 }
3594
3595 zend_jit_shutdown_ir();
3596
3597 #ifdef ZTS
3598 ts_free_id(jit_globals_id);
3599 #else
3600 zend_jit_trace_free_caches(&jit_globals);
3601 #endif
3602 }
3603
zend_jit_reset_counters(void)3604 static void zend_jit_reset_counters(void)
3605 {
3606 int i;
3607
3608 for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
3609 zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT;
3610 }
3611 }
3612
zend_jit_activate(void)3613 void zend_jit_activate(void)
3614 {
3615 zend_jit_profile_counter = 0;
3616 if (JIT_G(on)) {
3617 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
3618 zend_jit_reset_counters();
3619 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3620 zend_jit_reset_counters();
3621 zend_jit_trace_reset_caches();
3622 }
3623 }
3624 }
3625
zend_jit_deactivate(void)3626 void zend_jit_deactivate(void)
3627 {
3628 if (zend_jit_profile_counter && !CG(unclean_shutdown)) {
3629 zend_class_entry *ce;
3630
3631 zend_shared_alloc_lock();
3632 SHM_UNPROTECT();
3633 zend_jit_unprotect();
3634
3635 zend_jit_check_funcs(EG(function_table), 0);
3636 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(EG(class_table), ce) {
3637 if (ce->type == ZEND_INTERNAL_CLASS) {
3638 break;
3639 }
3640 zend_jit_check_funcs(&ce->function_table, 1);
3641 } ZEND_HASH_FOREACH_END();
3642
3643 zend_jit_protect();
3644 SHM_PROTECT();
3645 zend_shared_alloc_unlock();
3646 }
3647
3648 zend_jit_profile_counter = 0;
3649 }
3650
zend_jit_restart_preloaded_op_array(zend_op_array * op_array)3651 static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array)
3652 {
3653 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
3654
3655 if (!func_info) {
3656 return;
3657 }
3658
3659 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
3660 zend_jit_restart_hot_trace_counters(op_array);
3661 } else if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_COUNTERS) {
3662 zend_jit_restart_hot_counters(op_array);
3663 #if 0
3664 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ???
3665 } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) {
3666 zend_op *opline = op_array->opcodes;
3667 zend_jit_op_array_extension *jit_extension =
3668 (zend_jit_op_array_extension*)func_info;
3669
3670 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3671 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3672 opline++;
3673 }
3674 }
3675 if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) {
3676 opline->handler = (const void*)zend_jit_runtime_jit_handler;
3677 } else {
3678 opline->handler = (const void*)zend_jit_profile_jit_handler;
3679 }
3680 #endif
3681 }
3682 if (op_array->num_dynamic_func_defs) {
3683 for (uint32_t i = 0; i < op_array->num_dynamic_func_defs; i++) {
3684 zend_jit_restart_preloaded_op_array(op_array->dynamic_func_defs[i]);
3685 }
3686 }
3687 }
3688
zend_jit_restart_preloaded_script(zend_persistent_script * script)3689 static void zend_jit_restart_preloaded_script(zend_persistent_script *script)
3690 {
3691 zend_class_entry *ce;
3692 zend_op_array *op_array;
3693
3694 zend_jit_restart_preloaded_op_array(&script->script.main_op_array);
3695
3696 ZEND_HASH_MAP_FOREACH_PTR(&script->script.function_table, op_array) {
3697 zend_jit_restart_preloaded_op_array(op_array);
3698 } ZEND_HASH_FOREACH_END();
3699
3700 ZEND_HASH_MAP_FOREACH_PTR(&script->script.class_table, ce) {
3701 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
3702 if (op_array->type == ZEND_USER_FUNCTION) {
3703 zend_jit_restart_preloaded_op_array(op_array);
3704 }
3705 } ZEND_HASH_FOREACH_END();
3706 } ZEND_HASH_FOREACH_END();
3707 }
3708
zend_jit_restart(void)3709 void zend_jit_restart(void)
3710 {
3711 if (dasm_buf) {
3712 zend_jit_unprotect();
3713
3714 /* restore JIT buffer pos */
3715 dasm_ptr[0] = dasm_ptr[1];
3716
3717 zend_jit_trace_restart();
3718
3719 if (ZCSG(preload_script)) {
3720 zend_jit_restart_preloaded_script(ZCSG(preload_script));
3721 if (ZCSG(saved_scripts)) {
3722 zend_persistent_script **p = ZCSG(saved_scripts);
3723
3724 while (*p) {
3725 zend_jit_restart_preloaded_script(*p);
3726 p++;
3727 }
3728 }
3729 }
3730
3731 zend_jit_protect();
3732 }
3733 }
3734
3735 #endif /* HAVE_JIT */
3736
