1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 #include "main/php.h"
20 #include "main/SAPI.h"
21 #include "php_version.h"
22 #include <ZendAccelerator.h>
23 #include "zend_shared_alloc.h"
24 #include "Zend/zend_execute.h"
25 #include "Zend/zend_vm.h"
26 #include "Zend/zend_exceptions.h"
27 #include "Zend/zend_constants.h"
28 #include "Zend/zend_closures.h"
29 #include "Zend/zend_ini.h"
30 #include "Zend/zend_observer.h"
31 #include "zend_smart_str.h"
32 #include "jit/zend_jit.h"
33
34 #ifdef HAVE_JIT
35
36 #include "Optimizer/zend_func_info.h"
37 #include "Optimizer/zend_ssa.h"
38 #include "Optimizer/zend_inference.h"
39 #include "Optimizer/zend_call_graph.h"
40 #include "Optimizer/zend_dump.h"
41 #include "Optimizer/zend_worklist.h"
42
43 #include "jit/zend_jit_internal.h"
44
45 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
46 #include <pthread.h>
47 #endif
48
49 #ifdef ZTS
50 int jit_globals_id;
51 #else
52 zend_jit_globals jit_globals;
53 #endif
54
55 //#define CONTEXT_THREADED_JIT
56 #define ZEND_JIT_USE_RC_INFERENCE
57
58 #ifdef ZEND_JIT_USE_RC_INFERENCE
59 # define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE
60 # define RC_MAY_BE_1(info) (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0)
61 # define RC_MAY_BE_N(info) (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0)
62 #else
63 # define ZEND_SSA_RC_INFERENCE_FLAG 0
64 # define RC_MAY_BE_1(info) 1
65 # define RC_MAY_BE_N(info) 1
66 #endif
67
68 #define JIT_PREFIX "JIT$"
69 #define JIT_STUB_PREFIX "JIT$$"
70 #define TRACE_PREFIX "TRACE-"
71
72 zend_ulong zend_jit_profile_counter = 0;
73 int zend_jit_profile_counter_rid = -1;
74
75 int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
76
77 const zend_op *zend_jit_halt_op = NULL;
78 static int zend_jit_vm_kind = 0;
79 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
80 static int zend_write_protect = 1;
81 #endif
82
83 static void *dasm_buf = NULL;
84 static void *dasm_end = NULL;
85 static void **dasm_ptr = NULL;
86
87 static size_t dasm_size = 0;
88
89 static zend_long jit_bisect_pos = 0;
90
91 static const void *zend_jit_runtime_jit_handler = NULL;
92 static const void *zend_jit_profile_jit_handler = NULL;
93 static const void *zend_jit_func_hot_counter_handler = NULL;
94 static const void *zend_jit_loop_hot_counter_handler = NULL;
95 static const void *zend_jit_func_trace_counter_handler = NULL;
96 static const void *zend_jit_ret_trace_counter_handler = NULL;
97 static const void *zend_jit_loop_trace_counter_handler = NULL;
98
99 static int ZEND_FASTCALL zend_runtime_jit(void);
100
101 static int zend_jit_trace_op_len(const zend_op *opline);
102 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
103 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags);
104 static const void *zend_jit_trace_get_exit_addr(uint32_t n);
105 static void zend_jit_trace_add_code(const void *start, uint32_t size);
106 static zend_string *zend_jit_func_name(const zend_op_array *op_array);
107
108 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info);
109 static bool zend_jit_supported_binary_op(uint8_t op, uint32_t op1_info, uint32_t op2_info);
110
dominates(const zend_basic_block * blocks,int a,int b)111 static bool dominates(const zend_basic_block *blocks, int a, int b) {
112 while (blocks[b].level > blocks[a].level) {
113 b = blocks[b].idom;
114 }
115 return a == b;
116 }
117
zend_ssa_is_last_use(const zend_op_array * op_array,const zend_ssa * ssa,int var,int use)118 static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
119 {
120 int next_use;
121
122 if (ssa->vars[var].phi_use_chain) {
123 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
124 do {
125 if (!ssa->vars[phi->ssa_var].no_val) {
126 return 0;
127 }
128 phi = zend_ssa_next_use_phi(ssa, var, phi);
129 } while (phi);
130 }
131
132 if (ssa->cfg.blocks[ssa->cfg.map[use]].loop_header > 0
133 || (ssa->cfg.blocks[ssa->cfg.map[use]].flags & ZEND_BB_LOOP_HEADER)) {
134 int b = ssa->cfg.map[use];
135 int prev_use = ssa->vars[var].use_chain;
136 int def_block;
137
138 if (ssa->vars[var].definition >= 0) {
139 def_block =ssa->cfg.map[ssa->vars[var].definition];
140 } else {
141 ZEND_ASSERT(ssa->vars[var].definition_phi);
142 def_block = ssa->vars[var].definition_phi->block;
143 }
144 if (dominates(ssa->cfg.blocks, def_block,
145 (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) ? b : ssa->cfg.blocks[b].loop_header)) {
146 return 0;
147 }
148
149 while (prev_use >= 0 && prev_use != use) {
150 if (b != ssa->cfg.map[prev_use]
151 && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use])
152 && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) {
153 return 0;
154 }
155 prev_use = zend_ssa_next_use(ssa->ops, var, prev_use);
156 }
157 }
158
159 next_use = zend_ssa_next_use(ssa->ops, var, use);
160 if (next_use < 0) {
161 return 1;
162 } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) {
163 return 1;
164 }
165 return 0;
166 }
167
zend_jit_is_constant_cmp_long_long(const zend_op * opline,zend_ssa_range * op1_range,zend_jit_addr op1_addr,zend_ssa_range * op2_range,zend_jit_addr op2_addr,bool * result)168 static int zend_jit_is_constant_cmp_long_long(const zend_op *opline,
169 zend_ssa_range *op1_range,
170 zend_jit_addr op1_addr,
171 zend_ssa_range *op2_range,
172 zend_jit_addr op2_addr,
173 bool *result)
174 {
175 zend_long op1_min;
176 zend_long op1_max;
177 zend_long op2_min;
178 zend_long op2_max;
179
180 if (op1_range) {
181 op1_min = op1_range->min;
182 op1_max = op1_range->max;
183 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
184 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG);
185 op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr));
186 } else {
187 return 0;
188 }
189
190 if (op2_range) {
191 op2_min = op2_range->min;
192 op2_max = op2_range->max;
193 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
194 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG);
195 op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr));
196 } else {
197 return 0;
198 }
199
200 switch (opline->opcode) {
201 case ZEND_IS_EQUAL:
202 case ZEND_IS_IDENTICAL:
203 case ZEND_CASE:
204 case ZEND_CASE_STRICT:
205 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
206 *result = 1;
207 return 1;
208 } else if (op1_max < op2_min || op1_min > op2_max) {
209 *result = 0;
210 return 1;
211 }
212 return 0;
213 case ZEND_IS_NOT_EQUAL:
214 case ZEND_IS_NOT_IDENTICAL:
215 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
216 *result = 0;
217 return 1;
218 } else if (op1_max < op2_min || op1_min > op2_max) {
219 *result = 1;
220 return 1;
221 }
222 return 0;
223 case ZEND_IS_SMALLER:
224 if (op1_max < op2_min) {
225 *result = 1;
226 return 1;
227 } else if (op1_min >= op2_max) {
228 *result = 0;
229 return 1;
230 }
231 return 0;
232 case ZEND_IS_SMALLER_OR_EQUAL:
233 if (op1_max <= op2_min) {
234 *result = 1;
235 return 1;
236 } else if (op1_min > op2_max) {
237 *result = 0;
238 return 1;
239 }
240 return 0;
241 default:
242 ZEND_UNREACHABLE();
243 }
244 return 0;
245 }
246
247 #define ADVANCE_SSA_OP(ssa_op, offset) \
248 do { \
249 if (ssa_op) ssa_op += offset; \
250 } while (0)
251
zend_jit_needs_call_chain(zend_call_info * call_info,uint32_t b,const zend_op_array * op_array,zend_ssa * ssa,const zend_ssa_op * ssa_op,const zend_op * opline,int call_level,zend_jit_trace_rec * trace)252 static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace)
253 {
254 int skip;
255
256 if (trace) {
257 zend_jit_trace_rec *p = trace;
258
259 ADVANCE_SSA_OP(ssa_op, 1);
260 while (1) {
261 if (p->op == ZEND_JIT_TRACE_VM) {
262 switch (p->opline->opcode) {
263 case ZEND_SEND_ARRAY:
264 case ZEND_SEND_USER:
265 case ZEND_SEND_UNPACK:
266 case ZEND_INIT_FCALL:
267 case ZEND_INIT_METHOD_CALL:
268 case ZEND_INIT_STATIC_METHOD_CALL:
269 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
270 case ZEND_INIT_FCALL_BY_NAME:
271 case ZEND_INIT_NS_FCALL_BY_NAME:
272 case ZEND_INIT_DYNAMIC_CALL:
273 case ZEND_NEW:
274 case ZEND_INIT_USER_CALL:
275 case ZEND_FAST_CALL:
276 case ZEND_JMP:
277 case ZEND_JMPZ:
278 case ZEND_JMPNZ:
279 case ZEND_JMPZ_EX:
280 case ZEND_JMPNZ_EX:
281 case ZEND_FE_RESET_R:
282 case ZEND_FE_RESET_RW:
283 case ZEND_JMP_SET:
284 case ZEND_COALESCE:
285 case ZEND_JMP_NULL:
286 case ZEND_ASSERT_CHECK:
287 case ZEND_CATCH:
288 case ZEND_DECLARE_ANON_CLASS:
289 case ZEND_FE_FETCH_R:
290 case ZEND_FE_FETCH_RW:
291 case ZEND_BIND_INIT_STATIC_OR_JMP:
292 case ZEND_JMP_FRAMELESS:
293 return 1;
294 case ZEND_DO_ICALL:
295 case ZEND_DO_UCALL:
296 case ZEND_DO_FCALL_BY_NAME:
297 case ZEND_DO_FCALL:
298 case ZEND_CALLABLE_CONVERT:
299 return 0;
300 case ZEND_SEND_VAL:
301 case ZEND_SEND_VAR:
302 case ZEND_SEND_VAL_EX:
303 case ZEND_SEND_VAR_EX:
304 case ZEND_SEND_FUNC_ARG:
305 case ZEND_SEND_REF:
306 case ZEND_SEND_VAR_NO_REF:
307 case ZEND_SEND_VAR_NO_REF_EX:
308 /* skip */
309 break;
310 default:
311 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
312 return 1;
313 }
314 }
315 ADVANCE_SSA_OP(ssa_op, zend_jit_trace_op_len(opline));
316 } else if (p->op == ZEND_JIT_TRACE_ENTER ||
317 p->op == ZEND_JIT_TRACE_BACK ||
318 p->op == ZEND_JIT_TRACE_END) {
319 return 1;
320 }
321 p++;
322 }
323 }
324
325 if (!call_info) {
326 const zend_op *end = op_array->opcodes + op_array->last;
327
328 opline++;
329 ADVANCE_SSA_OP(ssa_op, 1);
330 skip = (call_level == 1);
331 while (opline != end) {
332 if (!skip) {
333 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
334 return 1;
335 }
336 }
337 switch (opline->opcode) {
338 case ZEND_SEND_VAL:
339 case ZEND_SEND_VAR:
340 case ZEND_SEND_VAL_EX:
341 case ZEND_SEND_VAR_EX:
342 case ZEND_SEND_FUNC_ARG:
343 case ZEND_SEND_REF:
344 case ZEND_SEND_VAR_NO_REF:
345 case ZEND_SEND_VAR_NO_REF_EX:
346 skip = 0;
347 break;
348 case ZEND_SEND_ARRAY:
349 case ZEND_SEND_USER:
350 case ZEND_SEND_UNPACK:
351 case ZEND_INIT_FCALL:
352 case ZEND_INIT_METHOD_CALL:
353 case ZEND_INIT_STATIC_METHOD_CALL:
354 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
355 case ZEND_INIT_FCALL_BY_NAME:
356 case ZEND_INIT_NS_FCALL_BY_NAME:
357 case ZEND_INIT_DYNAMIC_CALL:
358 case ZEND_NEW:
359 case ZEND_INIT_USER_CALL:
360 case ZEND_FAST_CALL:
361 case ZEND_JMP:
362 case ZEND_JMPZ:
363 case ZEND_JMPNZ:
364 case ZEND_JMPZ_EX:
365 case ZEND_JMPNZ_EX:
366 case ZEND_FE_RESET_R:
367 case ZEND_FE_RESET_RW:
368 case ZEND_JMP_SET:
369 case ZEND_COALESCE:
370 case ZEND_JMP_NULL:
371 case ZEND_ASSERT_CHECK:
372 case ZEND_CATCH:
373 case ZEND_DECLARE_ANON_CLASS:
374 case ZEND_FE_FETCH_R:
375 case ZEND_FE_FETCH_RW:
376 case ZEND_BIND_INIT_STATIC_OR_JMP:
377 case ZEND_JMP_FRAMELESS:
378 return 1;
379 case ZEND_DO_ICALL:
380 case ZEND_DO_UCALL:
381 case ZEND_DO_FCALL_BY_NAME:
382 case ZEND_DO_FCALL:
383 case ZEND_CALLABLE_CONVERT:
384 end = opline;
385 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
386 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
387 return 1;
388 }
389 return 0;
390 }
391 opline++;
392 ADVANCE_SSA_OP(ssa_op, 1);
393 }
394
395 return 1;
396 } else {
397 const zend_op *end = call_info->caller_call_opline;
398
399 /* end may be null if an opcode like EXIT is part of the argument list. */
400 if (!end || end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
401 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
402 return 1;
403 }
404
405 opline++;
406 ADVANCE_SSA_OP(ssa_op, 1);
407 skip = (call_level == 1);
408 while (opline != end) {
409 if (skip) {
410 switch (opline->opcode) {
411 case ZEND_SEND_VAL:
412 case ZEND_SEND_VAR:
413 case ZEND_SEND_VAL_EX:
414 case ZEND_SEND_VAR_EX:
415 case ZEND_SEND_FUNC_ARG:
416 case ZEND_SEND_REF:
417 case ZEND_SEND_VAR_NO_REF:
418 case ZEND_SEND_VAR_NO_REF_EX:
419 skip = 0;
420 break;
421 case ZEND_SEND_ARRAY:
422 case ZEND_SEND_USER:
423 case ZEND_SEND_UNPACK:
424 return 1;
425 }
426 } else {
427 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
428 return 1;
429 }
430 }
431 opline++;
432 ADVANCE_SSA_OP(ssa_op, 1);
433 }
434
435 return 0;
436 }
437 }
438
skip_valid_arguments(const zend_op_array * op_array,zend_ssa * ssa,const zend_call_info * call_info)439 static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
440 {
441 uint32_t num_args = 0;
442 zend_function *func = call_info->callee_func;
443
444 /* It's okay to handle prototypes here, because they can only increase the accepted arguments.
445 * Anything legal for the parent method is also legal for the parent method. */
446 while (num_args < call_info->num_args) {
447 zend_arg_info *arg_info = func->op_array.arg_info + num_args;
448
449 if (ZEND_TYPE_IS_SET(arg_info->type)) {
450 if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) {
451 zend_op *opline = call_info->arg_info[num_args].opline;
452 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[opline - op_array->opcodes] : NULL;
453 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
454 if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) {
455 break;
456 }
457 } else {
458 break;
459 }
460 }
461 num_args++;
462 }
463 return num_args;
464 }
465
zend_ssa_cv_info(const zend_op_array * op_array,zend_ssa * ssa,uint32_t var)466 static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var)
467 {
468 uint32_t j, info;
469
470 if (ssa->vars && ssa->var_info) {
471 info = ssa->var_info[var].type;
472 for (j = op_array->last_var; j < ssa->vars_count; j++) {
473 if (ssa->vars[j].var == var) {
474 info |= ssa->var_info[j].type;
475 }
476 }
477 } else {
478 info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF |
479 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
480 }
481
482 #ifdef ZEND_JIT_USE_RC_INFERENCE
483 /* Refcount may be increased by RETURN opcode */
484 if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
485 for (j = 0; j < ssa->cfg.blocks_count; j++) {
486 if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
487 ssa->cfg.blocks[j].len > 0) {
488 const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
489
490 if (opline->opcode == ZEND_RETURN) {
491 if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) {
492 info |= MAY_BE_RCN;
493 break;
494 }
495 }
496 }
497 }
498 }
499 #endif
500
501 return info;
502 }
503
zend_jit_may_avoid_refcounting(const zend_op * opline,uint32_t op1_info)504 static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_info)
505 {
506 switch (opline->opcode) {
507 case ZEND_FETCH_OBJ_FUNC_ARG:
508 if (!JIT_G(current_frame) ||
509 !JIT_G(current_frame)->call->func ||
510 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
511 return 0;
512 }
513 /* break missing intentionally */
514 case ZEND_FETCH_OBJ_R:
515 case ZEND_FETCH_OBJ_IS:
516 if ((op1_info & MAY_BE_OBJECT)
517 && opline->op2_type == IS_CONST
518 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
519 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
520 return 1;
521 }
522 break;
523 case ZEND_FETCH_DIM_FUNC_ARG:
524 if (!JIT_G(current_frame) ||
525 !JIT_G(current_frame)->call->func ||
526 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
527 return 0;
528 }
529 /* break missing intentionally */
530 case ZEND_FETCH_DIM_R:
531 case ZEND_FETCH_DIM_IS:
532 return 1;
533 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
534 if (!(opline->extended_value & ZEND_ISEMPTY)) {
535 return 1;
536 }
537 break;
538 }
539 return 0;
540 }
541
zend_jit_is_persistent_constant(zval * key,uint32_t flags)542 static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags)
543 {
544 zval *zv;
545 zend_constant *c = NULL;
546
547 /* null/true/false are resolved during compilation, so don't check for them here. */
548 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
549 if (zv) {
550 c = (zend_constant*)Z_PTR_P(zv);
551 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
552 key++;
553 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
554 if (zv) {
555 c = (zend_constant*)Z_PTR_P(zv);
556 }
557 }
558 return c && (ZEND_CONSTANT_FLAGS(c) & CONST_PERSISTENT);
559 }
560
zend_get_known_property_info(const zend_op_array * op_array,zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)561 static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
562 {
563 zend_property_info *info = NULL;
564
565 if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) ||
566 !ce ||
567 !(ce->ce_flags & ZEND_ACC_LINKED) ||
568 (ce->ce_flags & ZEND_ACC_TRAIT) ||
569 ce->create_object) {
570 return NULL;
571 }
572
573 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
574 if (ce->info.user.filename != filename) {
575 /* class declaration might be changed independently */
576 return NULL;
577 }
578
579 if (ce->parent) {
580 zend_class_entry *parent = ce->parent;
581
582 do {
583 if (parent->type == ZEND_INTERNAL_CLASS) {
584 break;
585 } else if (parent->info.user.filename != filename) {
586 /* some of parents class declarations might be changed independently */
587 /* TODO: this check may be not enough, because even
588 * in the same it's possible to conditionally define
589 * few classes with the same name, and "parent" may
590 * change from request to request.
591 */
592 return NULL;
593 }
594 parent = parent->parent;
595 } while (parent);
596 }
597 }
598
599 // TODO: Treat property hooks more precisely.
600 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
601 if (info == NULL ||
602 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
603 (info->flags & ZEND_ACC_STATIC) ||
604 info->hooks) {
605 return NULL;
606 }
607
608 if (info->flags & ZEND_ACC_PUBLIC) {
609 return info;
610 } else if (on_this) {
611 if (ce == info->ce) {
612 if (ce == op_array->scope) {
613 return info;
614 } else {
615 return NULL;
616 }
617 } else if ((info->flags & ZEND_ACC_PROTECTED)
618 && instanceof_function_slow(ce, info->ce)) {
619 return info;
620 }
621 }
622
623 return NULL;
624 }
625
zend_may_be_dynamic_property(zend_class_entry * ce,zend_string * member,bool on_this,const zend_op_array * op_array)626 static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, const zend_op_array *op_array)
627 {
628 zend_property_info *info;
629
630 if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT) || (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) {
631 return 1;
632 }
633
634 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
635 if (ce->info.user.filename != op_array->filename) {
636 /* class declaration might be changed independently */
637 return 1;
638 }
639 }
640
641 // TODO: Treat property hooks more precisely.
642 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
643 if (info == NULL ||
644 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
645 (info->flags & ZEND_ACC_STATIC) ||
646 info->hooks) {
647 return 1;
648 }
649
650 if (!(info->flags & ZEND_ACC_PUBLIC) &&
651 (!on_this || info->ce != ce)) {
652 return 1;
653 }
654
655 return 0;
656 }
657
658 #define OP_RANGE(ssa_op, opN) \
659 (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
660 ssa->var_info && \
661 (ssa_op)->opN##_use >= 0 && \
662 ssa->var_info[(ssa_op)->opN##_use].has_range) ? \
663 &ssa->var_info[(ssa_op)->opN##_use].range : NULL)
664
665 #define OP1_RANGE() OP_RANGE(ssa_op, op1)
666 #define OP2_RANGE() OP_RANGE(ssa_op, op2)
667 #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)
668
669 #include "jit/zend_jit_helpers.c"
670 #include "Zend/zend_cpuinfo.h"
671
672 #ifdef HAVE_GCC_GLOBAL_REGS
673 # define GCC_GLOBAL_REGS 1
674 #else
675 # define GCC_GLOBAL_REGS 0
676 #endif
677
678 /* By default avoid JITing inline handlers if it does not seem profitable due to lack of
679 * type information. Disabling this option allows testing some JIT handlers in the
680 * presence of try/catch blocks, which prevent SSA construction. */
681 #ifndef PROFITABILITY_CHECKS
682 # define PROFITABILITY_CHECKS 1
683 #endif
684
685 #define BP_JIT_IS 6 /* Used for ISSET_ISEMPTY_DIM_OBJ. see BP_VAR_*defines in Zend/zend_compile.h */
686
687 /* The generated code may contain tautological comparisons, ignore them. */
688 #if defined(__clang__)
689 # pragma clang diagnostic push
690 # pragma clang diagnostic ignored "-Wtautological-compare"
691 # pragma clang diagnostic ignored "-Wstring-compare"
692 #endif
693
694 #include "jit/zend_jit_ir.c"
695
696 #if defined(__clang__)
697 # pragma clang diagnostic pop
698 #endif
699
700 #ifdef _WIN32
701 # include <Windows.h>
702 #else
703 # include <sys/mman.h>
704 # if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
705 # define MAP_ANONYMOUS MAP_ANON
706 # endif
707 #endif
708
zend_jit_status(zval * ret)709 ZEND_EXT_API void zend_jit_status(zval *ret)
710 {
711 zval stats;
712 array_init(&stats);
713 add_assoc_bool(&stats, "enabled", JIT_G(enabled));
714 add_assoc_bool(&stats, "on", JIT_G(on));
715 add_assoc_long(&stats, "kind", JIT_G(trigger));
716 add_assoc_long(&stats, "opt_level", JIT_G(opt_level));
717 add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags));
718 if (dasm_buf) {
719 add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
720 add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
721 } else {
722 add_assoc_long(&stats, "buffer_size", 0);
723 add_assoc_long(&stats, "buffer_free", 0);
724 }
725 add_assoc_zval(ret, "jit", &stats);
726 }
727
zend_jit_func_name(const zend_op_array * op_array)728 static zend_string *zend_jit_func_name(const zend_op_array *op_array)
729 {
730 smart_str buf = {0};
731
732 if (op_array->function_name) {
733 smart_str_appends(&buf, JIT_PREFIX);
734 if (op_array->scope) {
735 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
736 smart_str_appends(&buf, "::");
737 }
738 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
739 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
740 smart_str_appends(&buf, ":");
741 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
742 smart_str_appends(&buf, ":");
743 smart_str_append_long(&buf, op_array->line_start);
744 }
745 smart_str_0(&buf);
746 return buf.s;
747 } else if (op_array->filename) {
748 smart_str_appends(&buf, JIT_PREFIX);
749 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
750 smart_str_0(&buf);
751 return buf.s;
752 } else {
753 return NULL;
754 }
755 }
756
zend_may_overflow(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa)757 static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
758 {
759 int res;
760 zend_long op1_min, op1_max, op2_min, op2_max;
761
762 if (!ssa->ops || !ssa->var_info) {
763 return 1;
764 }
765 switch (opline->opcode) {
766 case ZEND_PRE_INC:
767 case ZEND_POST_INC:
768 res = ssa_op->op1_def;
769 if (res < 0
770 || !ssa->var_info[res].has_range
771 || ssa->var_info[res].range.overflow) {
772 if (!OP1_HAS_RANGE()) {
773 return 1;
774 }
775 op1_max = OP1_MAX_RANGE();
776 if (op1_max == ZEND_LONG_MAX) {
777 return 1;
778 }
779 }
780 return 0;
781 case ZEND_PRE_DEC:
782 case ZEND_POST_DEC:
783 res = ssa_op->op1_def;
784 if (res < 0
785 || !ssa->var_info[res].has_range
786 || ssa->var_info[res].range.underflow) {
787 if (!OP1_HAS_RANGE()) {
788 return 1;
789 }
790 op1_min = OP1_MIN_RANGE();
791 if (op1_min == ZEND_LONG_MIN) {
792 return 1;
793 }
794 }
795 return 0;
796 case ZEND_ADD:
797 res = ssa_op->result_def;
798 if (res < 0
799 || !ssa->var_info[res].has_range
800 || ssa->var_info[res].range.underflow) {
801 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
802 return 1;
803 }
804 op1_min = OP1_MIN_RANGE();
805 op2_min = OP2_MIN_RANGE();
806 if (zend_add_will_overflow(op1_min, op2_min)) {
807 return 1;
808 }
809 }
810 if (res < 0
811 || !ssa->var_info[res].has_range
812 || ssa->var_info[res].range.overflow) {
813 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
814 return 1;
815 }
816 op1_max = OP1_MAX_RANGE();
817 op2_max = OP2_MAX_RANGE();
818 if (zend_add_will_overflow(op1_max, op2_max)) {
819 return 1;
820 }
821 }
822 return 0;
823 case ZEND_SUB:
824 res = ssa_op->result_def;
825 if (res < 0
826 || !ssa->var_info[res].has_range
827 || ssa->var_info[res].range.underflow) {
828 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
829 return 1;
830 }
831 op1_min = OP1_MIN_RANGE();
832 op2_max = OP2_MAX_RANGE();
833 if (zend_sub_will_overflow(op1_min, op2_max)) {
834 return 1;
835 }
836 }
837 if (res < 0
838 || !ssa->var_info[res].has_range
839 || ssa->var_info[res].range.overflow) {
840 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
841 return 1;
842 }
843 op1_max = OP1_MAX_RANGE();
844 op2_min = OP2_MIN_RANGE();
845 if (zend_sub_will_overflow(op1_max, op2_min)) {
846 return 1;
847 }
848 }
849 return 0;
850 case ZEND_MUL:
851 res = ssa_op->result_def;
852 return (res < 0 ||
853 !ssa->var_info[res].has_range ||
854 ssa->var_info[res].range.underflow ||
855 ssa->var_info[res].range.overflow);
856 case ZEND_ASSIGN_OP:
857 if (opline->extended_value == ZEND_ADD) {
858 res = ssa_op->op1_def;
859 if (res < 0
860 || !ssa->var_info[res].has_range
861 || ssa->var_info[res].range.underflow) {
862 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
863 return 1;
864 }
865 op1_min = OP1_MIN_RANGE();
866 op2_min = OP2_MIN_RANGE();
867 if (zend_add_will_overflow(op1_min, op2_min)) {
868 return 1;
869 }
870 }
871 if (res < 0
872 || !ssa->var_info[res].has_range
873 || ssa->var_info[res].range.overflow) {
874 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
875 return 1;
876 }
877 op1_max = OP1_MAX_RANGE();
878 op2_max = OP2_MAX_RANGE();
879 if (zend_add_will_overflow(op1_max, op2_max)) {
880 return 1;
881 }
882 }
883 return 0;
884 } else if (opline->extended_value == ZEND_SUB) {
885 res = ssa_op->op1_def;
886 if (res < 0
887 || !ssa->var_info[res].has_range
888 || ssa->var_info[res].range.underflow) {
889 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
890 return 1;
891 }
892 op1_min = OP1_MIN_RANGE();
893 op2_max = OP2_MAX_RANGE();
894 if (zend_sub_will_overflow(op1_min, op2_max)) {
895 return 1;
896 }
897 }
898 if (res < 0
899 || !ssa->var_info[res].has_range
900 || ssa->var_info[res].range.overflow) {
901 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
902 return 1;
903 }
904 op1_max = OP1_MAX_RANGE();
905 op2_min = OP2_MIN_RANGE();
906 if (zend_sub_will_overflow(op1_max, op2_min)) {
907 return 1;
908 }
909 }
910 return 0;
911 } else if (opline->extended_value == ZEND_MUL) {
912 res = ssa_op->op1_def;
913 return (res < 0 ||
914 !ssa->var_info[res].has_range ||
915 ssa->var_info[res].range.underflow ||
916 ssa->var_info[res].range.overflow);
917 }
918 ZEND_FALLTHROUGH;
919 default:
920 return 1;
921 }
922 }
923
zend_jit_build_cfg(const zend_op_array * op_array,zend_cfg * cfg)924 static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg)
925 {
926 uint32_t flags;
927
928 flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY;
929
930 zend_build_cfg(&CG(arena), op_array, flags, cfg);
931
932 /* Don't JIT huge functions. Apart from likely being detrimental due to the amount of
933 * generated code, some of our analysis is recursive and will stack overflow with many
934 * blocks. */
935 if (cfg->blocks_count > 100000) {
936 return FAILURE;
937 }
938
939 zend_cfg_build_predecessors(&CG(arena), cfg);
940
941 /* Compute Dominators Tree */
942 zend_cfg_compute_dominators_tree(op_array, cfg);
943
944 /* Identify reducible and irreducible loops */
945 zend_cfg_identify_loops(op_array, cfg);
946
947 return SUCCESS;
948 }
949
zend_jit_op_array_analyze1(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa)950 static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
951 {
952 if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) {
953 return FAILURE;
954 }
955
956 #if 0
957 /* TODO: debugger and profiler supports? */
958 if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) {
959 return FAILURE;
960 }
961 #endif
962
963 /* TODO: move this to zend_cfg.c ? */
964 if (!op_array->function_name) {
965 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
966 }
967
968 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
969 && ssa->cfg.blocks
970 && op_array->last_try_catch == 0
971 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
972 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
973 if (zend_build_ssa(&CG(arena), script, op_array, ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) {
974 return FAILURE;
975 }
976
977 zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa);
978
979 zend_ssa_find_false_dependencies(op_array, ssa);
980
981 zend_ssa_find_sccs(op_array, ssa);
982 }
983
984 return SUCCESS;
985 }
986
zend_jit_op_array_analyze2(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa,uint32_t optimization_level)987 static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t optimization_level)
988 {
989 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
990 && ssa->cfg.blocks
991 && op_array->last_try_catch == 0
992 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
993 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
994 if (zend_ssa_inference(&CG(arena), op_array, script, ssa,
995 optimization_level & ~ZEND_OPTIMIZER_NARROW_TO_DOUBLE) != SUCCESS) {
996 return FAILURE;
997 }
998 }
999
1000 return SUCCESS;
1001 }
1002
zend_jit_allocate_registers(zend_jit_ctx * ctx,const zend_op_array * op_array,zend_ssa * ssa)1003 static void zend_jit_allocate_registers(zend_jit_ctx *ctx, const zend_op_array *op_array, zend_ssa *ssa)
1004 {
1005 void *checkpoint;
1006 int candidates_count, i;
1007 zend_jit_reg_var *ra;
1008
1009 checkpoint = zend_arena_checkpoint(CG(arena));
1010 ra = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_jit_reg_var));
1011 candidates_count = 0;
1012 for (i = 0; i < ssa->vars_count; i++) {
1013 if (zend_jit_may_be_in_reg(op_array, ssa, i)) {
1014 ra[i].ref = IR_NULL;
1015 candidates_count++;
1016 }
1017 }
1018 if (!candidates_count) {
1019 zend_arena_release(&CG(arena), checkpoint);
1020 return;
1021 }
1022
1023 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
1024 /* Naive SSA resolution */
1025 for (i = 0; i < ssa->vars_count; i++) {
1026 if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
1027 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
1028 int k, src;
1029
1030 if (phi->pi >= 0) {
1031 src = phi->sources[0];
1032 if (ra[i].ref) {
1033 if (!ra[src].ref) {
1034 ra[i].flags |= ZREG_LOAD;
1035 } else {
1036 ra[i].flags |= ZREG_PI;
1037 }
1038 } else if (ra[src].ref) {
1039 ra[src].flags |= ZREG_STORE;
1040 }
1041 } else {
1042 int need_move = 0;
1043
1044 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1045 src = phi->sources[k];
1046 if (src >= 0) {
1047 if (ssa->vars[src].definition_phi
1048 && ssa->vars[src].definition_phi->pi >= 0
1049 && phi->block == ssa->vars[src].definition_phi->block) {
1050 /* Skip zero-length interval for Pi variable */
1051 src = ssa->vars[src].definition_phi->sources[0];
1052 }
1053 if (ra[i].ref) {
1054 if (!ra[src].ref) {
1055 need_move = 1;
1056 }
1057 } else if (ra[src].ref) {
1058 need_move = 1;
1059 }
1060 }
1061 }
1062 if (need_move) {
1063 if (ra[i].ref) {
1064 ra[i].flags |= ZREG_LOAD;
1065 }
1066 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1067 src = phi->sources[k];
1068 if (src >= 0) {
1069 if (ssa->vars[src].definition_phi
1070 && ssa->vars[src].definition_phi->pi >= 0
1071 && phi->block == ssa->vars[src].definition_phi->block) {
1072 /* Skip zero-length interval for Pi variable */
1073 src = ssa->vars[src].definition_phi->sources[0];
1074 }
1075 if (ra[src].ref) {
1076 ra[src].flags |= ZREG_STORE;
1077 }
1078 }
1079 }
1080 } else {
1081 ra[i].flags |= ZREG_PHI;
1082 }
1083 }
1084 }
1085 }
1086
1087 /* Remove useless register allocation */
1088 for (i = 0; i < ssa->vars_count; i++) {
1089 if (ra[i].ref &&
1090 ((ra[i].flags & ZREG_LOAD) ||
1091 ((ra[i].flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) &&
1092 ssa->vars[i].use_chain < 0) {
1093 bool may_remove = 1;
1094 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
1095
1096 while (phi) {
1097 if (ra[phi->ssa_var].ref &&
1098 !(ra[phi->ssa_var].flags & ZREG_LOAD)) {
1099 may_remove = 0;
1100 break;
1101 }
1102 phi = zend_ssa_next_use_phi(ssa, i, phi);
1103 }
1104 if (may_remove) {
1105 ra[i].ref = IR_UNUSED;
1106 }
1107 }
1108 }
1109
1110 /* Remove intervals used once */
1111 for (i = 0; i < ssa->vars_count; i++) {
1112 if (ra[i].ref) {
1113 if (!(ra[i].flags & (ZREG_LOAD|ZREG_STORE))) {
1114 uint32_t var_num = ssa->vars[i].var;
1115 uint32_t op_num = ssa->vars[i].definition;
1116
1117 /* Check if a tempoary variable may be freed by exception handler */
1118 if (op_array->last_live_range
1119 && var_num >= op_array->last_var
1120 && ssa->vars[i].definition >= 0
1121 && ssa->ops[op_num].result_def == i) {
1122 const zend_live_range *range = op_array->live_range;
1123 int j;
1124
1125 op_num++;
1126 if (op_array->opcodes[op_num].opcode == ZEND_OP_DATA) {
1127 op_num++;
1128 }
1129 for (j = 0; j < op_array->last_live_range; range++, j++) {
1130 if (range->start > op_num) {
1131 /* further blocks will not be relevant... */
1132 break;
1133 } else if (op_num < range->end && var_num == (range->var & ~ZEND_LIVE_MASK)) {
1134 /* check if opcodes in range may throw */
1135 do {
1136 if (zend_may_throw(op_array->opcodes + op_num, ssa->ops + op_num, op_array, ssa)) {
1137 ra[i].flags |= ZREG_STORE;
1138 break;
1139 }
1140 op_num++;
1141 if (op_array->opcodes[op_num].opcode == ZEND_OP_DATA) {
1142 op_num++;
1143 }
1144 } while (op_num < range->end);
1145 break;
1146 }
1147 }
1148 }
1149 }
1150 if ((ra[i].flags & ZREG_LOAD)
1151 && (ra[i].flags & ZREG_STORE)
1152 && (ssa->vars[i].use_chain < 0
1153 || zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) {
1154 bool may_remove = 1;
1155 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
1156
1157 while (phi) {
1158 if (ra[phi->ssa_var].ref &&
1159 !(ra[phi->ssa_var].flags & ZREG_LOAD)) {
1160 may_remove = 0;
1161 break;
1162 }
1163 phi = zend_ssa_next_use_phi(ssa, i, phi);
1164 }
1165 if (may_remove) {
1166 ra[i].ref = IR_UNUSED;
1167 }
1168 }
1169 }
1170 }
1171 }
1172
1173 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
1174 fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
1175 for (i = 0; i < ssa->vars_count; i++) {
1176 if (ra[i].ref) {
1177 fprintf(stderr, "#%d.", i);
1178 uint32_t var_num = ssa->vars[i].var;
1179 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
1180 if (ra[i].flags & ZREG_LOAD) {
1181 fprintf(stderr, " load");
1182 }
1183 if (ra[i].flags & ZREG_STORE) {
1184 fprintf(stderr, " store");
1185 }
1186 fprintf(stderr, "\n");
1187 }
1188 }
1189 fprintf(stderr, "\n");
1190 }
1191
1192 ctx->ra = ra;
1193 }
1194
zend_jit_compute_post_order(zend_cfg * cfg,int start,int * post_order)1195 static int zend_jit_compute_post_order(zend_cfg *cfg, int start, int *post_order)
1196 {
1197 int count = 0;
1198 int b, n, *p;
1199 zend_basic_block *bb;
1200 zend_worklist worklist;
1201 ALLOCA_FLAG(use_heap)
1202
1203 ZEND_WORKLIST_ALLOCA(&worklist, cfg->blocks_count, use_heap);
1204 zend_worklist_push(&worklist, start);
1205
1206 while (zend_worklist_len(&worklist) != 0) {
1207 next:
1208 b = zend_worklist_peek(&worklist);
1209 bb = &cfg->blocks[b];
1210 n = bb->successors_count;
1211 if (n > 0) {
1212 p = bb->successors;
1213 do {
1214 if (cfg->blocks[*p].flags & (ZEND_BB_CATCH|ZEND_BB_FINALLY|ZEND_BB_FINALLY_END)) {
1215 /* skip */
1216 } else if (zend_worklist_push(&worklist, *p)) {
1217 goto next;
1218 }
1219 p++;
1220 n--;
1221 } while (n > 0);
1222 }
1223 zend_worklist_pop(&worklist);
1224 post_order[count++] = b;
1225 }
1226 ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
1227 return count;
1228 }
1229
zend_jit_next_is_send_result(const zend_op * opline)1230 static bool zend_jit_next_is_send_result(const zend_op *opline)
1231 {
1232 if (opline->result_type == IS_TMP_VAR
1233 && (opline+1)->opcode == ZEND_SEND_VAL
1234 && (opline+1)->op1_type == IS_TMP_VAR
1235 && (opline+1)->op2_type != IS_CONST
1236 && (opline+1)->op1.var == opline->result.var) {
1237 return 1;
1238 }
1239 return 0;
1240 }
1241
zend_jit_supported_binary_op(uint8_t op,uint32_t op1_info,uint32_t op2_info)1242 static bool zend_jit_supported_binary_op(uint8_t op, uint32_t op1_info, uint32_t op2_info)
1243 {
1244 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1245 return false;
1246 }
1247 switch (op) {
1248 case ZEND_POW:
1249 case ZEND_DIV:
1250 // TODO: check for division by zero ???
1251 return false;
1252 case ZEND_ADD:
1253 case ZEND_SUB:
1254 case ZEND_MUL:
1255 return (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
1256 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
1257 case ZEND_BW_OR:
1258 case ZEND_BW_AND:
1259 case ZEND_BW_XOR:
1260 case ZEND_SL:
1261 case ZEND_SR:
1262 case ZEND_MOD:
1263 return (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG);
1264 case ZEND_CONCAT:
1265 return (op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING);
1266 EMPTY_SWITCH_DEFAULT_CASE()
1267 }
1268 }
1269
zend_jit(const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline)1270 static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline)
1271 {
1272 int b, i, end;
1273 zend_op *opline;
1274 zend_jit_ctx ctx;
1275 zend_jit_ctx *jit = &ctx;
1276 zend_jit_reg_var *ra = NULL;
1277 void *handler;
1278 int call_level = 0;
1279 void *checkpoint = NULL;
1280 bool recv_emitted = 0; /* emitted at least one RECV opcode */
1281 uint8_t smart_branch_opcode;
1282 uint32_t target_label, target_label2;
1283 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_mem_info;
1284 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
1285 zend_class_entry *ce;
1286 bool ce_is_instanceof;
1287 bool on_this;
1288
1289 ZEND_ASSERT(!(op_array->fn_flags & ZEND_ACC_CLOSURE) || !(op_array->scope));
1290
1291 if (JIT_G(bisect_limit)) {
1292 jit_bisect_pos++;
1293 if (jit_bisect_pos >= JIT_G(bisect_limit)) {
1294 if (jit_bisect_pos == JIT_G(bisect_limit)) {
1295 fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n",
1296 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
1297 op_array->scope ? "::" : "",
1298 op_array->function_name ? ZSTR_VAL(op_array->function_name) : "{main}",
1299 ZSTR_VAL(op_array->filename), op_array->line_start);
1300 }
1301 return FAILURE;
1302 }
1303 }
1304
1305 if (ssa->cfg.flags & ZEND_FUNC_IRREDUCIBLE) {
1306 /* We can't order blocks properly */
1307 return FAILURE;
1308 }
1309
1310 if (rt_opline) {
1311 /* Set BB_ENTRY flag to limit register usage across the OSR ENTRY point */
1312 ssa->cfg.blocks[ssa->cfg.map[rt_opline - op_array->opcodes]].flags |= ZEND_BB_ENTRY;
1313 }
1314
1315 zend_jit_start(&ctx, op_array, ssa);
1316 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1317 checkpoint = zend_arena_checkpoint(CG(arena));
1318 zend_jit_allocate_registers(&ctx, op_array, ssa);
1319 ra = ctx.ra;
1320 }
1321
1322 /* Process blocks in Reverse Post Order */
1323 int *sorted_blocks = alloca(sizeof(int) * ssa->cfg.blocks_count);
1324 int n = zend_jit_compute_post_order(&ssa->cfg, 0, sorted_blocks);
1325
1326 while (n > 0) {
1327 b = sorted_blocks[--n];
1328 if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) {
1329 continue;
1330 }
1331
1332 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
1333 opline = op_array->opcodes + ssa->cfg.blocks[b].start;
1334 if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) {
1335 if (opline->opcode == ZEND_RECV_INIT) {
1336 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1337 if (opline != op_array->opcodes && (opline-1)->opcode != ZEND_RECV_INIT) {
1338 zend_jit_recv_entry(&ctx, b);
1339 }
1340 } else {
1341 if (opline != op_array->opcodes && recv_emitted) {
1342 zend_jit_recv_entry(&ctx, b);
1343 }
1344 }
1345 recv_emitted = 1;
1346 } else if (opline->opcode == ZEND_RECV) {
1347 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
1348 /* skip */
1349 zend_jit_bb_start(&ctx, b);
1350 zend_jit_bb_end(&ctx, b);
1351 continue;
1352 } else if (recv_emitted) {
1353 zend_jit_recv_entry(&ctx, b);
1354 } else {
1355 recv_emitted = 1;
1356 }
1357 } else {
1358 if (recv_emitted) {
1359 zend_jit_recv_entry(&ctx, b);
1360 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
1361 ssa->cfg.blocks[b].len == 1 &&
1362 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
1363 /* don't generate code for BB with single opcode */
1364 zend_jit_free_ctx(&ctx);
1365
1366 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1367 zend_arena_release(&CG(arena), checkpoint);
1368 }
1369 return SUCCESS;
1370 }
1371 }
1372 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
1373 ssa->cfg.blocks[b].len == 1 &&
1374 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
1375 /* don't generate code for BB with single opcode */
1376 zend_jit_free_ctx(&ctx);
1377
1378 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1379 zend_arena_release(&CG(arena), checkpoint);
1380 }
1381 return SUCCESS;
1382 }
1383 }
1384
1385 zend_jit_bb_start(&ctx, b);
1386
1387 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ctx.ra) {
1388 zend_ssa_phi *phi = ssa->blocks[b].phis;
1389
1390 /* First try to insert IR Phi */
1391 while (phi) {
1392 zend_jit_reg_var *ival = &ctx.ra[phi->ssa_var];
1393
1394 if (ival->ref) {
1395 if (ival->flags & ZREG_PI) {
1396 zend_jit_gen_pi(jit, phi);
1397 } else if (ival->flags & ZREG_PHI) {
1398 zend_jit_gen_phi(jit, phi);
1399 }
1400 }
1401 phi = phi->next;
1402 }
1403 }
1404
1405 if (rt_opline
1406 && (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) == 0
1407 && rt_opline == op_array->opcodes + ssa->cfg.blocks[b].start) {
1408 zend_jit_osr_entry(&ctx, b); /* OSR (On-Stack-Replacement) Entry-Point */
1409 }
1410
1411 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1412 if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
1413 && ssa->cfg.blocks[b].start != 0
1414 && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
1415 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_LONG
1416 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_STRING
1417 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_MATCH)) {
1418 zend_jit_reset_last_valid_opline(&ctx);
1419 } else {
1420 zend_jit_set_last_valid_opline(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start);
1421 }
1422 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
1423 zend_jit_reset_last_valid_opline(&ctx);
1424 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY) {
1425 zend_jit_reset_last_valid_opline(&ctx);
1426 } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) {
1427 zend_jit_set_last_valid_opline(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start);
1428 }
1429 if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) {
1430 zend_jit_check_timeout(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start, NULL);
1431 }
1432 if (!ssa->cfg.blocks[b].len) {
1433 zend_jit_bb_end(&ctx, b);
1434 continue;
1435 }
1436 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
1437 zend_ssa_phi *phi = ssa->blocks[b].phis;
1438
1439 while (phi) {
1440 zend_jit_reg_var *ival = &ra[phi->ssa_var];
1441
1442 if (ival->ref) {
1443 if (ival->flags & ZREG_LOAD) {
1444 ZEND_ASSERT(ival->ref == IR_NULL);
1445
1446 if (!zend_jit_load_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var)) {
1447 goto jit_failure;
1448 }
1449 } else if (ival->flags & ZREG_STORE) {
1450 ZEND_ASSERT(ival->ref != IR_NULL);
1451
1452 if (!zend_jit_store_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var, 1)) {
1453 goto jit_failure;
1454 }
1455 }
1456 }
1457 phi = phi->next;
1458 }
1459 }
1460 end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
1461 for (i = ssa->cfg.blocks[b].start; i <= end; i++) {
1462 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[i] : NULL;
1463 opline = op_array->opcodes + i;
1464 switch (opline->opcode) {
1465 case ZEND_INIT_FCALL:
1466 case ZEND_INIT_FCALL_BY_NAME:
1467 case ZEND_INIT_NS_FCALL_BY_NAME:
1468 case ZEND_INIT_METHOD_CALL:
1469 case ZEND_INIT_DYNAMIC_CALL:
1470 case ZEND_INIT_STATIC_METHOD_CALL:
1471 case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
1472 case ZEND_INIT_USER_CALL:
1473 case ZEND_NEW:
1474 call_level++;
1475 }
1476
1477 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
1478 switch (opline->opcode) {
1479 case ZEND_PRE_INC:
1480 case ZEND_PRE_DEC:
1481 case ZEND_POST_INC:
1482 case ZEND_POST_DEC:
1483 if (opline->op1_type != IS_CV) {
1484 break;
1485 }
1486 op1_info = OP1_INFO();
1487 if (!(op1_info & MAY_BE_LONG)) {
1488 break;
1489 }
1490 if (opline->result_type != IS_UNUSED) {
1491 res_use_info = -1;
1492
1493 if (opline->result_type == IS_CV
1494 && ssa->vars
1495 && ssa_op->result_use >= 0
1496 && !ssa->vars[ssa_op->result_use].no_val) {
1497 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1498
1499 if (Z_MODE(res_use_addr) != IS_REG
1500 || Z_LOAD(res_use_addr)
1501 || Z_STORE(res_use_addr)) {
1502 res_use_info = RES_USE_INFO();
1503 }
1504 }
1505 res_info = RES_INFO();
1506 res_addr = RES_REG_ADDR();
1507 } else {
1508 res_use_info = -1;
1509 res_info = -1;
1510 res_addr = 0;
1511 }
1512 op1_def_info = OP1_DEF_INFO();
1513 if (!zend_jit_inc_dec(&ctx, opline,
1514 op1_info, OP1_REG_ADDR(),
1515 op1_def_info, OP1_DEF_REG_ADDR(),
1516 res_use_info, res_info,
1517 res_addr,
1518 (op1_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1519 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1520 goto jit_failure;
1521 }
1522 goto done;
1523 case ZEND_BW_OR:
1524 case ZEND_BW_AND:
1525 case ZEND_BW_XOR:
1526 case ZEND_SL:
1527 case ZEND_SR:
1528 case ZEND_MOD:
1529 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1530 break;
1531 }
1532 op1_info = OP1_INFO();
1533 op2_info = OP2_INFO();
1534 if (!(op1_info & MAY_BE_LONG)
1535 || !(op2_info & MAY_BE_LONG)) {
1536 break;
1537 }
1538 res_addr = RES_REG_ADDR();
1539 if (Z_MODE(res_addr) != IS_REG
1540 && (i + 1) <= end
1541 && zend_jit_next_is_send_result(opline)) {
1542 i++;
1543 res_use_info = -1;
1544 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1545 if (!zend_jit_reuse_ip(&ctx)) {
1546 goto jit_failure;
1547 }
1548 } else {
1549 res_use_info = -1;
1550
1551 if (opline->result_type == IS_CV
1552 && ssa->vars
1553 && ssa_op->result_use >= 0
1554 && !ssa->vars[ssa_op->result_use].no_val) {
1555 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1556
1557 if (Z_MODE(res_use_addr) != IS_REG
1558 || Z_LOAD(res_use_addr)
1559 || Z_STORE(res_use_addr)) {
1560 res_use_info = RES_USE_INFO();
1561 }
1562 }
1563 }
1564 if (!zend_jit_long_math(&ctx, opline,
1565 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
1566 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
1567 res_use_info, RES_INFO(), res_addr,
1568 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1569 goto jit_failure;
1570 }
1571 goto done;
1572 case ZEND_ADD:
1573 case ZEND_SUB:
1574 case ZEND_MUL:
1575 // case ZEND_DIV: // TODO: check for division by zero ???
1576 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1577 break;
1578 }
1579 op1_info = OP1_INFO();
1580 op2_info = OP2_INFO();
1581 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1582 break;
1583 }
1584 if (opline->opcode == ZEND_ADD &&
1585 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
1586 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
1587 /* pass */
1588 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
1589 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1590 break;
1591 }
1592 res_addr = RES_REG_ADDR();
1593 if (Z_MODE(res_addr) != IS_REG
1594 && (i + 1) <= end
1595 && zend_jit_next_is_send_result(opline)) {
1596 i++;
1597 res_use_info = -1;
1598 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1599 if (!zend_jit_reuse_ip(&ctx)) {
1600 goto jit_failure;
1601 }
1602 } else {
1603 res_use_info = -1;
1604
1605 if (opline->result_type == IS_CV
1606 && ssa->vars
1607 && ssa_op->result_use >= 0
1608 && !ssa->vars[ssa_op->result_use].no_val) {
1609 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1610
1611 if (Z_MODE(res_use_addr) != IS_REG
1612 || Z_LOAD(res_use_addr)
1613 || Z_STORE(res_use_addr)) {
1614 res_use_info = RES_USE_INFO();
1615 }
1616 }
1617 }
1618 res_info = RES_INFO();
1619 if (opline->opcode == ZEND_ADD &&
1620 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
1621 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
1622 if (!zend_jit_add_arrays(&ctx, opline, op1_info, OP1_REG_ADDR(), op2_info, OP2_REG_ADDR(), res_addr)) {
1623 goto jit_failure;
1624 }
1625 } else {
1626 if (!zend_jit_math(&ctx, opline,
1627 op1_info, OP1_REG_ADDR(),
1628 op2_info, OP2_REG_ADDR(),
1629 res_use_info, res_info, res_addr,
1630 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1631 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1632 goto jit_failure;
1633 }
1634 }
1635 goto done;
1636 case ZEND_CONCAT:
1637 case ZEND_FAST_CONCAT:
1638 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1639 break;
1640 }
1641 op1_info = OP1_INFO();
1642 op2_info = OP2_INFO();
1643 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1644 break;
1645 }
1646 if (!(op1_info & MAY_BE_STRING) ||
1647 !(op2_info & MAY_BE_STRING)) {
1648 break;
1649 }
1650 res_addr = RES_REG_ADDR();
1651 if ((i + 1) <= end
1652 && zend_jit_next_is_send_result(opline)) {
1653 i++;
1654 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1655 if (!zend_jit_reuse_ip(&ctx)) {
1656 goto jit_failure;
1657 }
1658 }
1659 if (!zend_jit_concat(&ctx, opline,
1660 op1_info, op2_info, res_addr,
1661 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1662 goto jit_failure;
1663 }
1664 goto done;
1665 case ZEND_ASSIGN_OP:
1666 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1667 break;
1668 }
1669 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1670 break;
1671 }
1672 op1_info = OP1_INFO();
1673 op2_info = OP2_INFO();
1674 if (!zend_jit_supported_binary_op(
1675 opline->extended_value, op1_info, op2_info)) {
1676 break;
1677 }
1678 op1_addr = OP1_REG_ADDR();
1679 op1_mem_info = -1;
1680 if (Z_MODE(op1_addr) != IS_REG
1681 || Z_LOAD(op1_addr)
1682 || Z_STORE(op1_addr)) {
1683 op1_mem_info = op1_info;
1684 }
1685 op1_def_info = OP1_DEF_INFO();
1686 if (!zend_jit_assign_op(&ctx, opline,
1687 op1_info, op1_addr, OP1_RANGE(),
1688 op1_def_info, OP1_DEF_REG_ADDR(), op1_mem_info,
1689 op2_info, OP2_REG_ADDR(), OP2_RANGE(),
1690 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1691 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1692 goto jit_failure;
1693 }
1694 goto done;
1695 case ZEND_ASSIGN_DIM_OP:
1696 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1697 break;
1698 }
1699 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1700 break;
1701 }
1702 if (!zend_jit_supported_binary_op(
1703 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
1704 break;
1705 }
1706 if (!zend_jit_assign_dim_op(&ctx, opline,
1707 OP1_INFO(), OP1_DEF_INFO(), OP1_REG_ADDR(),
1708 OP2_INFO(), (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
1709 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
1710 OP1_DATA_INFO(), OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(), IS_UNKNOWN,
1711 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1712 goto jit_failure;
1713 }
1714 goto done;
1715 case ZEND_ASSIGN_DIM:
1716 if (opline->op1_type != IS_CV) {
1717 break;
1718 }
1719 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1720 break;
1721 }
1722 if (!zend_jit_assign_dim(&ctx, opline,
1723 OP1_INFO(), OP1_REG_ADDR(),
1724 OP2_INFO(), (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
1725 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
1726 OP1_DATA_INFO(), OP1_DATA_REG_ADDR(),
1727 (ctx.ra && (ssa_op+1)->op1_def >= 0) ? OP1_DATA_DEF_REG_ADDR() : 0,
1728 (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
1729 IS_UNKNOWN,
1730 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1731 goto jit_failure;
1732 }
1733 goto done;
1734 case ZEND_PRE_INC_OBJ:
1735 case ZEND_PRE_DEC_OBJ:
1736 case ZEND_POST_INC_OBJ:
1737 case ZEND_POST_DEC_OBJ:
1738 if (opline->op2_type != IS_CONST
1739 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1740 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1741 break;
1742 }
1743 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1744 break;
1745 }
1746 ce = NULL;
1747 ce_is_instanceof = 0;
1748 on_this = 0;
1749 if (opline->op1_type == IS_UNUSED) {
1750 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1751 ce = op_array->scope;
1752 /* scope is NULL for closures. */
1753 if (ce) {
1754 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
1755 }
1756 op1_addr = 0;
1757 on_this = 1;
1758 } else {
1759 op1_info = OP1_INFO();
1760 if (!(op1_info & MAY_BE_OBJECT)) {
1761 break;
1762 }
1763 op1_addr = OP1_REG_ADDR();
1764 if (ssa->var_info && ssa->ops) {
1765 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1766 if (ssa_op->op1_use >= 0) {
1767 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1768 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1769 ce = op1_ssa->ce;
1770 ce_is_instanceof = op1_ssa->is_instanceof;
1771 }
1772 }
1773 }
1774 }
1775 if (!zend_jit_incdec_obj(&ctx, opline, op_array, ssa, ssa_op,
1776 op1_info, op1_addr,
1777 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
1778 goto jit_failure;
1779 }
1780 goto done;
1781 case ZEND_ASSIGN_OBJ_OP:
1782 if (opline->result_type != IS_UNUSED) {
1783 break;
1784 }
1785 if (opline->op2_type != IS_CONST
1786 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1787 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1788 break;
1789 }
1790 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1791 break;
1792 }
1793 if (!zend_jit_supported_binary_op(
1794 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
1795 break;
1796 }
1797 ce = NULL;
1798 ce_is_instanceof = 0;
1799 on_this = 0;
1800 if (opline->op1_type == IS_UNUSED) {
1801 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1802 ce = op_array->scope;
1803 /* scope is NULL for closures. */
1804 if (ce) {
1805 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
1806 }
1807 op1_addr = 0;
1808 on_this = 1;
1809 } else {
1810 op1_info = OP1_INFO();
1811 if (!(op1_info & MAY_BE_OBJECT)) {
1812 break;
1813 }
1814 op1_addr = OP1_REG_ADDR();
1815 if (ssa->var_info && ssa->ops) {
1816 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1817 if (ssa_op->op1_use >= 0) {
1818 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1819 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1820 ce = op1_ssa->ce;
1821 ce_is_instanceof = op1_ssa->is_instanceof;
1822 }
1823 }
1824 }
1825 }
1826 if (!zend_jit_assign_obj_op(&ctx, opline, op_array, ssa, ssa_op,
1827 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(),
1828 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
1829 goto jit_failure;
1830 }
1831 goto done;
1832 case ZEND_ASSIGN_OBJ:
1833 if (opline->op2_type != IS_CONST
1834 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1835 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1836 break;
1837 }
1838 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1839 break;
1840 }
1841 ce = NULL;
1842 ce_is_instanceof = 0;
1843 on_this = 0;
1844 if (opline->op1_type == IS_UNUSED) {
1845 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1846 ce = op_array->scope;
1847 /* scope is NULL for closures. */
1848 if (ce) {
1849 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
1850 }
1851 op1_addr = 0;
1852 on_this = 1;
1853 } else {
1854 op1_info = OP1_INFO();
1855 if (!(op1_info & MAY_BE_OBJECT)) {
1856 break;
1857 }
1858 op1_addr = OP1_REG_ADDR();
1859 if (ssa->var_info && ssa->ops) {
1860 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1861 if (ssa_op->op1_use >= 0) {
1862 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1863 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1864 ce = op1_ssa->ce;
1865 ce_is_instanceof = op1_ssa->is_instanceof;
1866 }
1867 }
1868 }
1869 }
1870 if (!zend_jit_assign_obj(&ctx, opline, op_array, ssa, ssa_op,
1871 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_REG_ADDR(), OP1_DATA_DEF_REG_ADDR(),
1872 (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
1873 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN,
1874 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1875 goto jit_failure;
1876 }
1877 goto done;
1878 case ZEND_ASSIGN:
1879 if (opline->op1_type != IS_CV) {
1880 break;
1881 }
1882 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1883 break;
1884 }
1885 op2_addr = OP2_REG_ADDR();
1886 if (ra
1887 && ssa->ops[opline - op_array->opcodes].op2_def >= 0
1888 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) {
1889 op2_def_addr = OP2_DEF_REG_ADDR();
1890 } else {
1891 op2_def_addr = op2_addr;
1892 }
1893 op1_info = OP1_INFO();
1894 if (ra && ssa->vars[ssa_op->op1_use].no_val) {
1895 op1_info |= MAY_BE_UNDEF; // requres type assignment
1896 }
1897 if (opline->result_type == IS_UNUSED) {
1898 res_addr = 0;
1899 res_info = -1;
1900 } else {
1901 res_addr = RES_REG_ADDR();
1902 res_info = RES_INFO();
1903 if (Z_MODE(res_addr) != IS_REG
1904 && (i + 1) <= end
1905 && zend_jit_next_is_send_result(opline)
1906 && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) {
1907 i++;
1908 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1909 if (!zend_jit_reuse_ip(&ctx)) {
1910 goto jit_failure;
1911 }
1912 }
1913 }
1914 if (!zend_jit_assign(&ctx, opline,
1915 op1_info, OP1_REG_ADDR(),
1916 OP1_DEF_INFO(), OP1_DEF_REG_ADDR(),
1917 OP2_INFO(), op2_addr, op2_def_addr,
1918 res_info, res_addr,
1919 0,
1920 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1921 goto jit_failure;
1922 }
1923 goto done;
1924 case ZEND_QM_ASSIGN:
1925 op1_addr = OP1_REG_ADDR();
1926 if (ra
1927 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
1928 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
1929 op1_def_addr = OP1_DEF_REG_ADDR();
1930 } else {
1931 op1_def_addr = op1_addr;
1932 }
1933 if (!zend_jit_qm_assign(&ctx, opline,
1934 OP1_INFO(), op1_addr, op1_def_addr,
1935 -1, RES_INFO(), RES_REG_ADDR())) {
1936 goto jit_failure;
1937 }
1938 goto done;
1939 case ZEND_INIT_FCALL:
1940 case ZEND_INIT_FCALL_BY_NAME:
1941 case ZEND_INIT_NS_FCALL_BY_NAME:
1942 if (!zend_jit_init_fcall(&ctx, opline, b, op_array, ssa, ssa_op, call_level, NULL, 0)) {
1943 goto jit_failure;
1944 }
1945 goto done;
1946 case ZEND_SEND_VAL:
1947 case ZEND_SEND_VAL_EX:
1948 if (opline->op2_type == IS_CONST) {
1949 /* Named parameters not supported in JIT (yet) */
1950 break;
1951 }
1952 if (opline->opcode == ZEND_SEND_VAL_EX
1953 && opline->op2.num > MAX_ARG_FLAG_NUM) {
1954 break;
1955 }
1956 if (!zend_jit_send_val(&ctx, opline,
1957 OP1_INFO(), OP1_REG_ADDR())) {
1958 goto jit_failure;
1959 }
1960 goto done;
1961 case ZEND_SEND_REF:
1962 if (opline->op2_type == IS_CONST) {
1963 /* Named parameters not supported in JIT (yet) */
1964 break;
1965 }
1966 if (!zend_jit_send_ref(&ctx, opline, op_array,
1967 OP1_INFO(), 0)) {
1968 goto jit_failure;
1969 }
1970 goto done;
1971 case ZEND_SEND_VAR:
1972 case ZEND_SEND_VAR_EX:
1973 case ZEND_SEND_VAR_NO_REF:
1974 case ZEND_SEND_VAR_NO_REF_EX:
1975 case ZEND_SEND_FUNC_ARG:
1976 if (opline->op2_type == IS_CONST) {
1977 /* Named parameters not supported in JIT (yet) */
1978 break;
1979 }
1980 if ((opline->opcode == ZEND_SEND_VAR_EX
1981 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
1982 && opline->op2.num > MAX_ARG_FLAG_NUM) {
1983 break;
1984 }
1985 op1_addr = OP1_REG_ADDR();
1986 if (ra
1987 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
1988 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
1989 op1_def_addr = OP1_DEF_REG_ADDR();
1990 } else {
1991 op1_def_addr = op1_addr;
1992 }
1993 if (!zend_jit_send_var(&ctx, opline, op_array,
1994 OP1_INFO(), op1_addr, op1_def_addr)) {
1995 goto jit_failure;
1996 }
1997 goto done;
1998 case ZEND_CHECK_FUNC_ARG:
1999 if (opline->op2_type == IS_CONST) {
2000 /* Named parameters not supported in JIT (yet) */
2001 break;
2002 }
2003 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
2004 break;
2005 }
2006 if (!zend_jit_check_func_arg(&ctx, opline)) {
2007 goto jit_failure;
2008 }
2009 goto done;
2010 case ZEND_CHECK_UNDEF_ARGS:
2011 if (!zend_jit_check_undef_args(&ctx, opline)) {
2012 goto jit_failure;
2013 }
2014 goto done;
2015 case ZEND_DO_UCALL:
2016 ZEND_FALLTHROUGH;
2017 case ZEND_DO_ICALL:
2018 case ZEND_DO_FCALL_BY_NAME:
2019 case ZEND_DO_FCALL:
2020 if (!zend_jit_do_fcall(&ctx, opline, op_array, ssa, call_level, b + 1, NULL)) {
2021 goto jit_failure;
2022 }
2023 goto done;
2024 case ZEND_IS_EQUAL:
2025 case ZEND_IS_NOT_EQUAL:
2026 case ZEND_IS_SMALLER:
2027 case ZEND_IS_SMALLER_OR_EQUAL:
2028 case ZEND_CASE: {
2029 res_addr = RES_REG_ADDR();
2030 if ((opline->result_type & IS_TMP_VAR)
2031 && (i + 1) <= end
2032 && ((opline+1)->opcode == ZEND_JMPZ
2033 || (opline+1)->opcode == ZEND_JMPNZ
2034 || (opline+1)->opcode == ZEND_JMPZ_EX
2035 || (opline+1)->opcode == ZEND_JMPNZ_EX)
2036 && (opline+1)->op1_type == IS_TMP_VAR
2037 && (opline+1)->op1.var == opline->result.var) {
2038 i++;
2039 smart_branch_opcode = (opline+1)->opcode;
2040 target_label = ssa->cfg.blocks[b].successors[0];
2041 target_label2 = ssa->cfg.blocks[b].successors[1];
2042 /* For EX variant write into the result of EX opcode. */
2043 if ((opline+1)->opcode == ZEND_JMPZ_EX
2044 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
2045 res_addr = OP_REG_ADDR(opline + 1, ssa_op + 1, result_type, result, result_def);
2046 }
2047 } else {
2048 smart_branch_opcode = 0;
2049 target_label = target_label2 = (uint32_t)-1;
2050 }
2051 if (!zend_jit_cmp(&ctx, opline,
2052 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
2053 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
2054 res_addr,
2055 zend_may_throw(opline, ssa_op, op_array, ssa),
2056 smart_branch_opcode, target_label, target_label2,
2057 NULL, 0)) {
2058 goto jit_failure;
2059 }
2060 goto done;
2061 }
2062 case ZEND_IS_IDENTICAL:
2063 case ZEND_IS_NOT_IDENTICAL:
2064 case ZEND_CASE_STRICT:
2065 res_addr = RES_REG_ADDR();
2066 if ((opline->result_type & IS_TMP_VAR)
2067 && (i + 1) <= end
2068 && ((opline+1)->opcode == ZEND_JMPZ
2069 || (opline+1)->opcode == ZEND_JMPZ_EX
2070 || (opline+1)->opcode == ZEND_JMPNZ_EX
2071 || (opline+1)->opcode == ZEND_JMPNZ)
2072 && (opline+1)->op1_type == IS_TMP_VAR
2073 && (opline+1)->op1.var == opline->result.var) {
2074 i++;
2075 smart_branch_opcode = (opline+1)->opcode;
2076 target_label = ssa->cfg.blocks[b].successors[0];
2077 target_label2 = ssa->cfg.blocks[b].successors[1];
2078 /* For EX variant write into the result of EX opcode. */
2079 if ((opline+1)->opcode == ZEND_JMPZ_EX
2080 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
2081 res_addr = OP_REG_ADDR(opline + 1, ssa_op + 1, result_type, result, result_def);
2082 }
2083 } else {
2084 smart_branch_opcode = 0;
2085 target_label = target_label2 = (uint32_t)-1;
2086 }
2087 if (!zend_jit_identical(&ctx, opline,
2088 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
2089 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
2090 res_addr,
2091 zend_may_throw(opline, ssa_op, op_array, ssa),
2092 smart_branch_opcode, target_label, target_label2,
2093 NULL, 0)) {
2094 goto jit_failure;
2095 }
2096 goto done;
2097 case ZEND_DEFINED:
2098 if ((opline->result_type & IS_TMP_VAR)
2099 && (i + 1) <= end
2100 && ((opline+1)->opcode == ZEND_JMPZ
2101 || (opline+1)->opcode == ZEND_JMPNZ)
2102 && (opline+1)->op1_type == IS_TMP_VAR
2103 && (opline+1)->op1.var == opline->result.var) {
2104 i++;
2105 smart_branch_opcode = (opline+1)->opcode;
2106 target_label = ssa->cfg.blocks[b].successors[0];
2107 target_label2 = ssa->cfg.blocks[b].successors[1];
2108 } else {
2109 smart_branch_opcode = 0;
2110 target_label = target_label2 = (uint32_t)-1;
2111 }
2112 if (!zend_jit_defined(&ctx, opline, smart_branch_opcode, target_label, target_label2, NULL)) {
2113 goto jit_failure;
2114 }
2115 goto done;
2116 case ZEND_TYPE_CHECK:
2117 if (opline->extended_value == MAY_BE_RESOURCE) {
2118 // TODO: support for is_resource() ???
2119 break;
2120 }
2121 if ((opline->result_type & IS_TMP_VAR)
2122 && (i + 1) <= end
2123 && ((opline+1)->opcode == ZEND_JMPZ
2124 || (opline+1)->opcode == ZEND_JMPNZ)
2125 && (opline+1)->op1_type == IS_TMP_VAR
2126 && (opline+1)->op1.var == opline->result.var) {
2127 i++;
2128 smart_branch_opcode = (opline+1)->opcode;
2129 target_label = ssa->cfg.blocks[b].successors[0];
2130 target_label2 = ssa->cfg.blocks[b].successors[1];
2131 } else {
2132 smart_branch_opcode = 0;
2133 target_label = target_label2 = (uint32_t)-1;
2134 }
2135 if (!zend_jit_type_check(&ctx, opline, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) {
2136 goto jit_failure;
2137 }
2138 goto done;
2139 case ZEND_RETURN:
2140 op1_info = OP1_INFO();
2141 if ((PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info))
2142 || op_array->type == ZEND_EVAL_CODE
2143 // TODO: support for top-level code
2144 || !op_array->function_name
2145 // TODO: support for IS_UNDEF ???
2146 || (op1_info & MAY_BE_UNDEF)) {
2147 if (!zend_jit_tail_handler(&ctx, opline)) {
2148 goto jit_failure;
2149 }
2150 } else {
2151 if (!zend_jit_return(&ctx, opline, op_array,
2152 op1_info, OP1_REG_ADDR())) {
2153 goto jit_failure;
2154 }
2155 }
2156 goto done;
2157 case ZEND_BOOL:
2158 case ZEND_BOOL_NOT:
2159 if (!zend_jit_bool_jmpznz(&ctx, opline,
2160 OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(),
2161 -1, -1,
2162 zend_may_throw(opline, ssa_op, op_array, ssa),
2163 opline->opcode, NULL)) {
2164 goto jit_failure;
2165 }
2166 goto done;
2167 case ZEND_JMPZ:
2168 case ZEND_JMPNZ:
2169 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
2170 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2171 /* smart branch */
2172 if (!zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2173 goto jit_failure;
2174 }
2175 goto done;
2176 }
2177 ZEND_FALLTHROUGH;
2178 case ZEND_JMPZ_EX:
2179 case ZEND_JMPNZ_EX:
2180 if (opline->result_type == IS_UNDEF) {
2181 res_addr = 0;
2182 } else {
2183 res_addr = RES_REG_ADDR();
2184 }
2185 if (!zend_jit_bool_jmpznz(&ctx, opline,
2186 OP1_INFO(), OP1_REG_ADDR(), res_addr,
2187 ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1],
2188 zend_may_throw(opline, ssa_op, op_array, ssa),
2189 opline->opcode, NULL)) {
2190 goto jit_failure;
2191 }
2192 goto done;
2193 case ZEND_ISSET_ISEMPTY_CV:
2194 if ((opline->extended_value & ZEND_ISEMPTY)) {
2195 // TODO: support for empty() ???
2196 break;
2197 }
2198 if ((opline->result_type & IS_TMP_VAR)
2199 && (i + 1) <= end
2200 && ((opline+1)->opcode == ZEND_JMPZ
2201 || (opline+1)->opcode == ZEND_JMPNZ)
2202 && (opline+1)->op1_type == IS_TMP_VAR
2203 && (opline+1)->op1.var == opline->result.var) {
2204 i++;
2205 smart_branch_opcode = (opline+1)->opcode;
2206 target_label = ssa->cfg.blocks[b].successors[0];
2207 target_label2 = ssa->cfg.blocks[b].successors[1];
2208 } else {
2209 smart_branch_opcode = 0;
2210 target_label = target_label2 = (uint32_t)-1;
2211 }
2212 if (!zend_jit_isset_isempty_cv(&ctx, opline,
2213 OP1_INFO(), OP1_REG_ADDR(),
2214 smart_branch_opcode, target_label, target_label2,
2215 NULL)) {
2216 goto jit_failure;
2217 }
2218 goto done;
2219 case ZEND_IN_ARRAY:
2220 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
2221 break;
2222 }
2223 op1_info = OP1_INFO();
2224 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
2225 break;
2226 }
2227 if ((opline->result_type & IS_TMP_VAR)
2228 && (i + 1) <= end
2229 && ((opline+1)->opcode == ZEND_JMPZ
2230 || (opline+1)->opcode == ZEND_JMPNZ)
2231 && (opline+1)->op1_type == IS_TMP_VAR
2232 && (opline+1)->op1.var == opline->result.var) {
2233 i++;
2234 smart_branch_opcode = (opline+1)->opcode;
2235 target_label = ssa->cfg.blocks[b].successors[0];
2236 target_label2 = ssa->cfg.blocks[b].successors[1];
2237 } else {
2238 smart_branch_opcode = 0;
2239 target_label = target_label2 = (uint32_t)-1;
2240 }
2241 if (!zend_jit_in_array(&ctx, opline,
2242 op1_info, OP1_REG_ADDR(),
2243 smart_branch_opcode, target_label, target_label2,
2244 NULL)) {
2245 goto jit_failure;
2246 }
2247 goto done;
2248 case ZEND_FETCH_DIM_R:
2249 case ZEND_FETCH_DIM_IS:
2250 case ZEND_FETCH_LIST_R:
2251 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2252 break;
2253 }
2254 if (!zend_jit_fetch_dim_read(&ctx, opline, ssa, ssa_op,
2255 OP1_INFO(), OP1_REG_ADDR(), 0,
2256 OP2_INFO(), OP2_REG_ADDR(), OP2_RANGE(),
2257 RES_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
2258 goto jit_failure;
2259 }
2260 goto done;
2261 case ZEND_FETCH_DIM_W:
2262 case ZEND_FETCH_DIM_RW:
2263 // case ZEND_FETCH_DIM_UNSET:
2264 case ZEND_FETCH_LIST_W:
2265 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2266 break;
2267 }
2268 if (opline->op1_type != IS_CV) {
2269 break;
2270 }
2271 if (!zend_jit_fetch_dim(&ctx, opline,
2272 OP1_INFO(), OP1_REG_ADDR(),
2273 OP2_INFO(), (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
2274 (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : 0,
2275 RES_REG_ADDR(), IS_UNKNOWN)) {
2276 goto jit_failure;
2277 }
2278 goto done;
2279 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
2280 if ((opline->extended_value & ZEND_ISEMPTY)) {
2281 // TODO: support for empty() ???
2282 break;
2283 }
2284 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2285 break;
2286 }
2287 if ((opline->result_type & IS_TMP_VAR)
2288 && (i + 1) <= end
2289 && ((opline+1)->opcode == ZEND_JMPZ
2290 || (opline+1)->opcode == ZEND_JMPNZ)
2291 && (opline+1)->op1_type == IS_TMP_VAR
2292 && (opline+1)->op1.var == opline->result.var) {
2293 i++;
2294 smart_branch_opcode = (opline+1)->opcode;
2295 target_label = ssa->cfg.blocks[b].successors[0];
2296 target_label2 = ssa->cfg.blocks[b].successors[1];
2297 } else {
2298 smart_branch_opcode = 0;
2299 target_label = target_label2 = (uint32_t)-1;
2300 }
2301 if (!zend_jit_isset_isempty_dim(&ctx, opline,
2302 OP1_INFO(), OP1_REG_ADDR(), 0,
2303 OP2_INFO(), OP2_REG_ADDR(), OP2_RANGE(), IS_UNKNOWN,
2304 zend_may_throw(opline, ssa_op, op_array, ssa),
2305 smart_branch_opcode, target_label, target_label2,
2306 NULL)) {
2307 goto jit_failure;
2308 }
2309 goto done;
2310 case ZEND_FETCH_OBJ_R:
2311 case ZEND_FETCH_OBJ_IS:
2312 case ZEND_FETCH_OBJ_W:
2313 if (opline->op2_type != IS_CONST
2314 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2315 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2316 break;
2317 }
2318 ce = NULL;
2319 ce_is_instanceof = 0;
2320 on_this = 0;
2321 if (opline->op1_type == IS_UNUSED) {
2322 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
2323 op1_addr = 0;
2324 ce = op_array->scope;
2325 /* scope is NULL for closures. */
2326 if (ce) {
2327 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
2328 }
2329 on_this = 1;
2330 } else {
2331 op1_info = OP1_INFO();
2332 if (!(op1_info & MAY_BE_OBJECT)) {
2333 break;
2334 }
2335 op1_addr = OP1_REG_ADDR();
2336 if (ssa->var_info && ssa->ops) {
2337 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
2338 if (ssa_op->op1_use >= 0) {
2339 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
2340 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
2341 ce = op1_ssa->ce;
2342 ce_is_instanceof = op1_ssa->is_instanceof;
2343 }
2344 }
2345 }
2346 }
2347 if (!zend_jit_fetch_obj(&ctx, opline, op_array, ssa, ssa_op,
2348 op1_info, op1_addr, 0, ce, ce_is_instanceof, on_this, 0, 0, NULL,
2349 RES_REG_ADDR(), IS_UNKNOWN,
2350 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2351 goto jit_failure;
2352 }
2353 goto done;
2354 case ZEND_BIND_GLOBAL:
2355 if (!ssa->ops || !ssa->var_info) {
2356 op1_info = MAY_BE_ANY|MAY_BE_REF;
2357 } else {
2358 op1_info = OP1_INFO();
2359 }
2360 if (!zend_jit_bind_global(&ctx, opline, op1_info)) {
2361 goto jit_failure;
2362 }
2363 goto done;
2364 case ZEND_RECV:
2365 if (!zend_jit_recv(&ctx, opline, op_array)) {
2366 goto jit_failure;
2367 }
2368 goto done;
2369 case ZEND_RECV_INIT:
2370 if (!zend_jit_recv_init(&ctx, opline, op_array,
2371 (opline + 1)->opcode != ZEND_RECV_INIT,
2372 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2373 goto jit_failure;
2374 }
2375 goto done;
2376 case ZEND_FREE:
2377 case ZEND_FE_FREE:
2378 if (!zend_jit_free(&ctx, opline, OP1_INFO(),
2379 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2380 goto jit_failure;
2381 }
2382 goto done;
2383 case ZEND_ECHO:
2384 op1_info = OP1_INFO();
2385 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2386 break;
2387 }
2388 if (!zend_jit_echo(&ctx, opline, op1_info)) {
2389 goto jit_failure;
2390 }
2391 goto done;
2392 case ZEND_STRLEN:
2393 op1_info = OP1_INFO();
2394 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2395 break;
2396 }
2397 if (!zend_jit_strlen(&ctx, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR())) {
2398 goto jit_failure;
2399 }
2400 goto done;
2401 case ZEND_COUNT:
2402 op1_info = OP1_INFO();
2403 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
2404 break;
2405 }
2406 if (!zend_jit_count(&ctx, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
2407 goto jit_failure;
2408 }
2409 goto done;
2410 case ZEND_FETCH_THIS:
2411 if (!zend_jit_fetch_this(&ctx, opline, op_array, 0)) {
2412 goto jit_failure;
2413 }
2414 goto done;
2415 case ZEND_SWITCH_LONG:
2416 case ZEND_SWITCH_STRING:
2417 case ZEND_MATCH:
2418 if (!zend_jit_switch(&ctx, opline, op_array, ssa, NULL, NULL)) {
2419 goto jit_failure;
2420 }
2421 goto done;
2422 case ZEND_VERIFY_RETURN_TYPE:
2423 if (opline->op1_type == IS_UNUSED) {
2424 /* Always throws */
2425 break;
2426 }
2427 if (opline->op1_type == IS_CONST) {
2428 /* TODO Different instruction format, has return value */
2429 break;
2430 }
2431 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
2432 /* Not worth bothering with */
2433 break;
2434 }
2435 if (OP1_INFO() & MAY_BE_REF) {
2436 /* TODO May need reference unwrapping. */
2437 break;
2438 }
2439 if (!zend_jit_verify_return_type(&ctx, opline, op_array, OP1_INFO())) {
2440 goto jit_failure;
2441 }
2442 goto done;
2443 case ZEND_FE_RESET_R:
2444 op1_info = OP1_INFO();
2445 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
2446 break;
2447 }
2448 if (!zend_jit_fe_reset(&ctx, opline, op1_info)) {
2449 goto jit_failure;
2450 }
2451 goto done;
2452 case ZEND_FE_FETCH_R:
2453 op1_info = OP1_INFO();
2454 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
2455 break;
2456 }
2457 if (!zend_jit_fe_fetch(&ctx, opline, op1_info, OP2_INFO(),
2458 ssa->cfg.blocks[b].successors[0], opline->opcode, NULL)) {
2459 goto jit_failure;
2460 }
2461 goto done;
2462 case ZEND_FETCH_CONSTANT:
2463 if (!zend_jit_fetch_constant(&ctx, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
2464 goto jit_failure;
2465 }
2466 goto done;
2467 case ZEND_INIT_METHOD_CALL:
2468 if (opline->op2_type != IS_CONST
2469 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2470 break;
2471 }
2472 ce = NULL;
2473 ce_is_instanceof = 0;
2474 on_this = 0;
2475 if (opline->op1_type == IS_UNUSED) {
2476 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
2477 op1_addr = 0;
2478 ce = op_array->scope;
2479 /* scope is NULL for closures. */
2480 if (ce) {
2481 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
2482 }
2483 on_this = 1;
2484 } else {
2485 op1_info = OP1_INFO();
2486 if (!(op1_info & MAY_BE_OBJECT)) {
2487 break;
2488 }
2489 op1_addr = OP1_REG_ADDR();
2490 if (ssa->var_info && ssa->ops) {
2491 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
2492 if (ssa_op->op1_use >= 0) {
2493 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
2494 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
2495 ce = op1_ssa->ce;
2496 ce_is_instanceof = op1_ssa->is_instanceof;
2497 }
2498 }
2499 }
2500 }
2501 if (!zend_jit_init_method_call(&ctx, opline, b, op_array, ssa, ssa_op, call_level,
2502 op1_info, op1_addr, ce, ce_is_instanceof, on_this, 0, NULL,
2503 NULL, 0,
2504 -1, -1,
2505 0)) {
2506 goto jit_failure;
2507 }
2508 goto done;
2509 case ZEND_ROPE_INIT:
2510 case ZEND_ROPE_ADD:
2511 case ZEND_ROPE_END:
2512 op2_info = OP2_INFO();
2513 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2514 break;
2515 }
2516 if (!zend_jit_rope(&ctx, opline, op2_info)) {
2517 goto jit_failure;
2518 }
2519 goto done;
2520 default:
2521 break;
2522 }
2523 }
2524
2525 switch (opline->opcode) {
2526 case ZEND_RECV_INIT:
2527 case ZEND_BIND_GLOBAL:
2528 if (opline == op_array->opcodes ||
2529 opline->opcode != op_array->opcodes[i-1].opcode) {
2530 /* repeatable opcodes */
2531 if (!zend_jit_handler(&ctx, opline,
2532 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2533 goto jit_failure;
2534 }
2535 }
2536 zend_jit_set_last_valid_opline(&ctx, opline+1);
2537 break;
2538 case ZEND_NOP:
2539 case ZEND_OP_DATA:
2540 case ZEND_SWITCH_LONG:
2541 case ZEND_SWITCH_STRING:
2542 break;
2543 case ZEND_MATCH:
2544 /* We have to exit to the VM because the MATCH handler performs an N-way jump for
2545 * which we can't generate simple (opcache.jit=1201) JIT code. */
2546 if (!zend_jit_tail_handler(&ctx, opline)) {
2547 goto jit_failure;
2548 }
2549 break;
2550 case ZEND_JMP:
2551 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
2552 const zend_op *target = OP_JMP_ADDR(opline, opline->op1);
2553
2554 if (!zend_jit_set_ip(&ctx, target)) {
2555 goto jit_failure;
2556 }
2557 }
2558 break;
2559 case ZEND_CATCH:
2560 case ZEND_FAST_CALL:
2561 case ZEND_FAST_RET:
2562 case ZEND_GENERATOR_CREATE:
2563 case ZEND_GENERATOR_RETURN:
2564 case ZEND_RETURN_BY_REF:
2565 case ZEND_RETURN:
2566 case ZEND_MATCH_ERROR:
2567 /* switch through trampoline */
2568 case ZEND_YIELD:
2569 case ZEND_YIELD_FROM:
2570 case ZEND_THROW:
2571 case ZEND_VERIFY_NEVER_TYPE:
2572 if (!zend_jit_tail_handler(&ctx, opline)) {
2573 goto jit_failure;
2574 }
2575 /* THROW and EXIT may be used in the middle of BB */
2576 /* don't generate code for the rest of BB */
2577 i = end;
2578 break;
2579 /* stackless execution */
2580 case ZEND_INCLUDE_OR_EVAL:
2581 case ZEND_DO_FCALL:
2582 case ZEND_DO_UCALL:
2583 case ZEND_DO_FCALL_BY_NAME:
2584 if (!zend_jit_call(&ctx, opline, b + 1)) {
2585 goto jit_failure;
2586 }
2587 break;
2588 case ZEND_JMPZ:
2589 case ZEND_JMPNZ:
2590 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
2591 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2592 /* smart branch */
2593 if (!zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2594 goto jit_failure;
2595 }
2596 goto done;
2597 }
2598 ZEND_FALLTHROUGH;
2599 case ZEND_JMPZ_EX:
2600 case ZEND_JMPNZ_EX:
2601 case ZEND_JMP_SET:
2602 case ZEND_COALESCE:
2603 case ZEND_JMP_NULL:
2604 case ZEND_FE_RESET_R:
2605 case ZEND_FE_RESET_RW:
2606 case ZEND_ASSERT_CHECK:
2607 case ZEND_FE_FETCH_R:
2608 case ZEND_FE_FETCH_RW:
2609 case ZEND_BIND_INIT_STATIC_OR_JMP:
2610 if (!zend_jit_handler(&ctx, opline,
2611 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
2612 !zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2613 goto jit_failure;
2614 }
2615 break;
2616 case ZEND_JMP_FRAMELESS:
2617 if (!zend_jit_jmp_frameless(&ctx, opline, /* exit_addr */ NULL, /* guard */ 0)) {
2618 goto jit_failure;
2619 }
2620 break;
2621 case ZEND_NEW:
2622 if (!zend_jit_handler(&ctx, opline, 1)) {
2623 return 0;
2624 }
2625 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
2626 zend_class_entry *ce = NULL;
2627
2628 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
2629 if (ssa->ops && ssa->var_info) {
2630 zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
2631 if (res_ssa->ce && !res_ssa->is_instanceof) {
2632 ce = res_ssa->ce;
2633 }
2634 }
2635 } else {
2636 if (opline->op1_type == IS_CONST) {
2637 zval *zv = RT_CONSTANT(opline, opline->op1);
2638 if (Z_TYPE_P(zv) == IS_STRING) {
2639 zval *lc = zv + 1;
2640 ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc));
2641 }
2642 }
2643 }
2644
2645 i++;
2646
2647 if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) {
2648 const zend_op *next_opline = opline + 1;
2649
2650 ZEND_ASSERT(b + 1 == ssa->cfg.blocks[b].successors[0]);
2651 zend_jit_constructor(&ctx, next_opline, op_array, ssa, call_level, b + 1);
2652 }
2653
2654 /* We skip over the DO_FCALL, so decrement call_level ourselves. */
2655 call_level--;
2656 }
2657 break;
2658 case ZEND_FRAMELESS_ICALL_0:
2659 jit_frameless_icall0(jit, opline);
2660 goto done;
2661 case ZEND_FRAMELESS_ICALL_1:
2662 op1_info = OP1_INFO();
2663 jit_frameless_icall1(jit, opline, op1_info);
2664 goto done;
2665 case ZEND_FRAMELESS_ICALL_2:
2666 op1_info = OP1_INFO();
2667 op2_info = OP2_INFO();
2668 jit_frameless_icall2(jit, opline, op1_info, op2_info);
2669 goto done;
2670 case ZEND_FRAMELESS_ICALL_3:
2671 op1_info = OP1_INFO();
2672 op2_info = OP2_INFO();
2673 jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO());
2674 goto done;
2675 default:
2676 if (!zend_jit_handler(&ctx, opline,
2677 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2678 goto jit_failure;
2679 }
2680 if (i == end
2681 && (opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2682 /* smart branch split across basic blocks */
2683 if (!zend_jit_set_cond(&ctx, opline + 2, opline->result.var)) {
2684 goto jit_failure;
2685 }
2686 }
2687 }
2688 done:
2689 switch (opline->opcode) {
2690 case ZEND_DO_FCALL:
2691 case ZEND_DO_ICALL:
2692 case ZEND_DO_UCALL:
2693 case ZEND_DO_FCALL_BY_NAME:
2694 case ZEND_CALLABLE_CONVERT:
2695 call_level--;
2696 }
2697 }
2698 zend_jit_bb_end(&ctx, b);
2699 }
2700
2701 if (jit->return_inputs) {
2702 zend_jit_common_return(jit);
2703
2704 bool left_frame = 0;
2705 if (op_array->last_var > 100) {
2706 /* To many CVs to unroll */
2707 if (!zend_jit_free_cvs(&ctx)) {
2708 goto jit_failure;
2709 }
2710 left_frame = 1;
2711 }
2712 if (!left_frame) {
2713 int j;
2714
2715 for (j = 0 ; j < op_array->last_var; j++) {
2716 uint32_t info = zend_ssa_cv_info(op_array, ssa, j);
2717
2718 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
2719 if (!left_frame) {
2720 left_frame = 1;
2721 if (!zend_jit_leave_frame(&ctx)) {
2722 goto jit_failure;
2723 }
2724 }
2725 if (!zend_jit_free_cv(&ctx, info, j)) {
2726 goto jit_failure;
2727 }
2728 }
2729 }
2730 }
2731 if (!zend_jit_leave_func(&ctx, op_array, NULL, MAY_BE_ANY, left_frame,
2732 NULL, NULL, (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, 1)) {
2733 goto jit_failure;
2734 }
2735 }
2736
2737 handler = zend_jit_finish(&ctx);
2738 if (!handler) {
2739 goto jit_failure;
2740 }
2741 zend_jit_free_ctx(&ctx);
2742
2743 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2744 zend_arena_release(&CG(arena), checkpoint);
2745 }
2746 return SUCCESS;
2747
2748 jit_failure:
2749 zend_jit_free_ctx(&ctx);
2750 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2751 zend_arena_release(&CG(arena), checkpoint);
2752 }
2753 return FAILURE;
2754 }
2755
zend_jit_collect_calls(zend_op_array * op_array,zend_script * script)2756 static void zend_jit_collect_calls(zend_op_array *op_array, zend_script *script)
2757 {
2758 zend_func_info *func_info;
2759
2760 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
2761 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
2762 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
2763 func_info = ZEND_FUNC_INFO(op_array);
2764 } else {
2765 func_info = zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info));
2766 ZEND_SET_FUNC_INFO(op_array, func_info);
2767 }
2768 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, op_array, func_info);
2769 }
2770
zend_jit_cleanup_func_info(zend_op_array * op_array)2771 static void zend_jit_cleanup_func_info(zend_op_array *op_array)
2772 {
2773 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
2774 zend_call_info *caller_info, *callee_info;
2775
2776 if (func_info) {
2777 caller_info = func_info->caller_info;
2778 callee_info = func_info->callee_info;
2779
2780 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
2781 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
2782 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
2783 func_info->num = 0;
2784 func_info->flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
2785 | ZEND_FUNC_JIT_ON_PROF_REQUEST
2786 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
2787 | ZEND_FUNC_JIT_ON_HOT_TRACE;
2788 memset(&func_info->ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
2789 } else {
2790 ZEND_SET_FUNC_INFO(op_array, NULL);
2791 }
2792
2793 while (caller_info) {
2794 if (caller_info->caller_op_array) {
2795 zend_jit_cleanup_func_info(caller_info->caller_op_array);
2796 }
2797 caller_info = caller_info->next_caller;
2798 }
2799 while (callee_info) {
2800 if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) {
2801 zend_jit_cleanup_func_info(&callee_info->callee_func->op_array);
2802 }
2803 callee_info = callee_info->next_callee;
2804 }
2805 }
2806 }
2807
zend_real_jit_func(zend_op_array * op_array,zend_script * script,const zend_op * rt_opline,uint8_t trigger)2808 static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline, uint8_t trigger)
2809 {
2810 zend_ssa ssa;
2811 void *checkpoint;
2812 zend_func_info *func_info;
2813 uint8_t orig_trigger;
2814
2815 if (*dasm_ptr == dasm_end) {
2816 return FAILURE;
2817 }
2818
2819 orig_trigger = JIT_G(trigger);
2820 JIT_G(trigger) = trigger;
2821 checkpoint = zend_arena_checkpoint(CG(arena));
2822
2823 /* Build SSA */
2824 memset(&ssa, 0, sizeof(zend_ssa));
2825
2826 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
2827 if (trigger == ZEND_JIT_ON_FIRST_EXEC) {
2828 zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
2829 op_array = (zend_op_array*) jit_extension->op_array;
2830 } else if (trigger == ZEND_JIT_ON_HOT_COUNTERS) {
2831 zend_jit_op_array_hot_extension *jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
2832 op_array = (zend_op_array*) jit_extension->op_array;
2833 } else {
2834 ZEND_ASSERT(!op_array->scope);
2835 }
2836 }
2837
2838 if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) {
2839 goto jit_failure;
2840 }
2841
2842 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
2843 zend_jit_collect_calls(op_array, script);
2844 func_info = ZEND_FUNC_INFO(op_array);
2845 func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array);
2846 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
2847 zend_init_func_return_info(op_array, script, &func_info->return_info);
2848 }
2849 }
2850
2851 if (zend_jit_op_array_analyze2(op_array, script, &ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
2852 goto jit_failure;
2853 }
2854
2855 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
2856 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa);
2857 }
2858
2859 if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) {
2860 goto jit_failure;
2861 }
2862
2863 zend_jit_cleanup_func_info(op_array);
2864 zend_arena_release(&CG(arena), checkpoint);
2865 JIT_G(trigger) = orig_trigger;
2866 return SUCCESS;
2867
2868 jit_failure:
2869 zend_jit_cleanup_func_info(op_array);
2870 zend_arena_release(&CG(arena), checkpoint);
2871 JIT_G(trigger) = orig_trigger;
2872 return FAILURE;
2873 }
2874
2875 /* Run-time JIT handler */
zend_runtime_jit(void)2876 static int ZEND_FASTCALL zend_runtime_jit(void)
2877 {
2878 zend_execute_data *execute_data = EG(current_execute_data);
2879 zend_op_array *op_array = &EX(func)->op_array;
2880 zend_op *opline = op_array->opcodes;
2881 zend_jit_op_array_extension *jit_extension;
2882 bool do_bailout = 0;
2883
2884 zend_shared_alloc_lock();
2885
2886 if (ZEND_FUNC_INFO(op_array)) {
2887
2888 SHM_UNPROTECT();
2889 zend_jit_unprotect();
2890
2891 zend_try {
2892 /* restore original opcode handlers */
2893 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2894 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
2895 opline++;
2896 }
2897 }
2898 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
2899 opline->handler = jit_extension->orig_handler;
2900
2901 /* perform real JIT for this function */
2902 zend_real_jit_func(op_array, NULL, NULL, ZEND_JIT_ON_FIRST_EXEC);
2903 } zend_catch {
2904 do_bailout = true;
2905 } zend_end_try();
2906
2907 zend_jit_protect();
2908 SHM_PROTECT();
2909 }
2910
2911 zend_shared_alloc_unlock();
2912
2913 if (do_bailout) {
2914 zend_bailout();
2915 }
2916
2917 /* JIT-ed code is going to be called by VM */
2918 return 0;
2919 }
2920
zend_jit_check_funcs(HashTable * function_table,bool is_method)2921 void zend_jit_check_funcs(HashTable *function_table, bool is_method) {
2922 zend_op *opline;
2923 zend_function *func;
2924 zend_op_array *op_array;
2925 uintptr_t counter;
2926 zend_jit_op_array_extension *jit_extension;
2927
2928 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(function_table, func) {
2929 if (func->type == ZEND_INTERNAL_FUNCTION) {
2930 break;
2931 }
2932 op_array = &func->op_array;
2933 opline = op_array->opcodes;
2934 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2935 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
2936 opline++;
2937 }
2938 }
2939 if (opline->handler == zend_jit_profile_jit_handler) {
2940 if (!RUN_TIME_CACHE(op_array)) {
2941 continue;
2942 }
2943 counter = (uintptr_t)ZEND_COUNTER_INFO(op_array);
2944 ZEND_COUNTER_INFO(op_array) = 0;
2945 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
2946 opline->handler = jit_extension->orig_handler;
2947 if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) {
2948 zend_real_jit_func(op_array, NULL, NULL, ZEND_JIT_ON_PROF_REQUEST);
2949 }
2950 }
2951 } ZEND_HASH_FOREACH_END();
2952 }
2953
zend_jit_hot_func(zend_execute_data * execute_data,const zend_op * opline)2954 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
2955 {
2956 zend_op_array *op_array = &EX(func)->op_array;
2957 zend_jit_op_array_hot_extension *jit_extension;
2958 uint32_t i;
2959 bool do_bailout = 0;
2960
2961 zend_shared_alloc_lock();
2962 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
2963
2964 if (jit_extension) {
2965 SHM_UNPROTECT();
2966 zend_jit_unprotect();
2967
2968 zend_try {
2969 for (i = 0; i < op_array->last; i++) {
2970 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
2971 }
2972
2973 /* perform real JIT for this function */
2974 zend_real_jit_func(op_array, NULL, opline, ZEND_JIT_ON_HOT_COUNTERS);
2975 } zend_catch {
2976 do_bailout = 1;
2977 } zend_end_try();
2978
2979 zend_jit_protect();
2980 SHM_PROTECT();
2981 }
2982
2983 zend_shared_alloc_unlock();
2984
2985 if (do_bailout) {
2986 zend_bailout();
2987 }
2988 /* JIT-ed code is going to be called by VM */
2989 }
2990
zend_jit_setup_hot_counters_ex(zend_op_array * op_array,zend_cfg * cfg)2991 static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg)
2992 {
2993 if (JIT_G(hot_func)) {
2994 zend_op *opline = op_array->opcodes;
2995
2996 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2997 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
2998 opline++;
2999 }
3000 }
3001
3002 opline->handler = (const void*)zend_jit_func_hot_counter_handler;
3003 }
3004
3005 if (JIT_G(hot_loop)) {
3006 uint32_t i;
3007
3008 for (i = 0; i < cfg->blocks_count; i++) {
3009 if ((cfg->blocks[i].flags & ZEND_BB_REACHABLE) &&
3010 (cfg->blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
3011 op_array->opcodes[cfg->blocks[i].start].handler =
3012 (const void*)zend_jit_loop_hot_counter_handler;
3013 }
3014 }
3015 }
3016 }
3017
zend_jit_restart_hot_counters(zend_op_array * op_array)3018 static int zend_jit_restart_hot_counters(zend_op_array *op_array)
3019 {
3020 zend_jit_op_array_hot_extension *jit_extension;
3021 zend_cfg cfg;
3022 uint32_t i;
3023
3024 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
3025 for (i = 0; i < op_array->last; i++) {
3026 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
3027 }
3028
3029 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
3030 return FAILURE;
3031 }
3032
3033 zend_jit_setup_hot_counters_ex(op_array, &cfg);
3034
3035 return SUCCESS;
3036 }
3037
zend_jit_setup_hot_counters(zend_op_array * op_array)3038 static int zend_jit_setup_hot_counters(zend_op_array *op_array)
3039 {
3040 zend_jit_op_array_hot_extension *jit_extension;
3041 zend_cfg cfg;
3042 uint32_t i;
3043
3044 ZEND_ASSERT(!JIT_G(hot_func) || zend_jit_func_hot_counter_handler != NULL);
3045 ZEND_ASSERT(!JIT_G(hot_loop) || zend_jit_loop_hot_counter_handler != NULL);
3046
3047 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
3048 return FAILURE;
3049 }
3050
3051 jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*));
3052 if (!jit_extension) {
3053 return FAILURE;
3054 }
3055 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3056 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS;
3057 jit_extension->op_array = op_array;
3058 jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
3059 for (i = 0; i < op_array->last; i++) {
3060 jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
3061 }
3062 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3063
3064 zend_jit_setup_hot_counters_ex(op_array, &cfg);
3065
3066 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3067
3068 return SUCCESS;
3069 }
3070
3071 #include "jit/zend_jit_trace.c"
3072
zend_jit_op_array(zend_op_array * op_array,zend_script * script)3073 int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
3074 {
3075 if (dasm_ptr == NULL) {
3076 return FAILURE;
3077 }
3078
3079 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) {
3080 zend_jit_op_array_extension *jit_extension;
3081 zend_op *opline = op_array->opcodes;
3082
3083 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
3084 ZEND_SET_FUNC_INFO(op_array, NULL);
3085 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
3086 return SUCCESS;
3087 }
3088
3089 /* Set run-time JIT handler */
3090 ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL);
3091 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3092 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3093 opline++;
3094 }
3095 }
3096 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
3097 if (!jit_extension) {
3098 return FAILURE;
3099 }
3100 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3101 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC;
3102 jit_extension->op_array = op_array;
3103 jit_extension->orig_handler = (void*)opline->handler;
3104 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3105 opline->handler = (const void*)zend_jit_runtime_jit_handler;
3106 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3107
3108 return SUCCESS;
3109 } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) {
3110 zend_jit_op_array_extension *jit_extension;
3111 zend_op *opline = op_array->opcodes;
3112
3113 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
3114 ZEND_SET_FUNC_INFO(op_array, NULL);
3115 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
3116 return SUCCESS;
3117 }
3118
3119 ZEND_ASSERT(zend_jit_profile_jit_handler != NULL);
3120 if (op_array->function_name) {
3121 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3122 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3123 opline++;
3124 }
3125 }
3126 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
3127 if (!jit_extension) {
3128 return FAILURE;
3129 }
3130 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3131 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST;
3132 jit_extension->op_array = op_array;
3133 jit_extension->orig_handler = (void*)opline->handler;
3134 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3135 opline->handler = (const void*)zend_jit_profile_jit_handler;
3136 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3137 }
3138
3139 return SUCCESS;
3140 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
3141 return zend_jit_setup_hot_counters(op_array);
3142 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3143 return zend_jit_setup_hot_trace_counters(op_array);
3144 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3145 return zend_real_jit_func(op_array, script, NULL, ZEND_JIT_ON_SCRIPT_LOAD);
3146 } else {
3147 ZEND_UNREACHABLE();
3148 }
3149 return FAILURE;
3150 }
3151
zend_jit_script(zend_script * script)3152 int zend_jit_script(zend_script *script)
3153 {
3154 void *checkpoint;
3155 zend_call_graph call_graph;
3156 zend_func_info *info;
3157 int i;
3158
3159 if (dasm_ptr == NULL || *dasm_ptr == dasm_end) {
3160 return FAILURE;
3161 }
3162
3163 checkpoint = zend_arena_checkpoint(CG(arena));
3164
3165 call_graph.op_arrays_count = 0;
3166 zend_build_call_graph(&CG(arena), script, &call_graph);
3167
3168 zend_analyze_call_graph(&CG(arena), script, &call_graph);
3169
3170 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
3171 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
3172 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS ||
3173 JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3174 for (i = 0; i < call_graph.op_arrays_count; i++) {
3175 if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
3176 goto jit_failure;
3177 }
3178 }
3179 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3180 for (i = 0; i < call_graph.op_arrays_count; i++) {
3181 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3182 if (info) {
3183 if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
3184 goto jit_failure;
3185 }
3186 info->ssa.cfg.flags |= info->flags;
3187 info->flags = info->ssa.cfg.flags;
3188 }
3189 }
3190
3191 for (i = 0; i < call_graph.op_arrays_count; i++) {
3192 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3193 if (info) {
3194 info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]);
3195 if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
3196 zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info);
3197 }
3198 }
3199 }
3200
3201 for (i = 0; i < call_graph.op_arrays_count; i++) {
3202 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3203 if (info) {
3204 if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
3205 goto jit_failure;
3206 }
3207 info->flags = info->ssa.cfg.flags;
3208 }
3209 }
3210
3211 for (i = 0; i < call_graph.op_arrays_count; i++) {
3212 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3213 if (info) {
3214 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
3215 zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa);
3216 }
3217 if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) {
3218 goto jit_failure;
3219 }
3220 }
3221 }
3222
3223 for (i = 0; i < call_graph.op_arrays_count; i++) {
3224 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
3225 }
3226 } else {
3227 ZEND_UNREACHABLE();
3228 }
3229
3230 zend_arena_release(&CG(arena), checkpoint);
3231
3232 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC
3233 || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST
3234 || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS
3235 || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3236 zend_class_entry *ce;
3237 zend_op_array *op_array;
3238
3239 ZEND_HASH_MAP_FOREACH_PTR(&script->class_table, ce) {
3240 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
3241 if (!ZEND_FUNC_INFO(op_array)) {
3242 void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes);
3243
3244 if (jit_extension) {
3245 ZEND_SET_FUNC_INFO(op_array, jit_extension);
3246 }
3247 }
3248 } ZEND_HASH_FOREACH_END();
3249 } ZEND_HASH_FOREACH_END();
3250 }
3251
3252 return SUCCESS;
3253
3254 jit_failure:
3255 if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3256 for (i = 0; i < call_graph.op_arrays_count; i++) {
3257 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
3258 }
3259 }
3260 zend_arena_release(&CG(arena), checkpoint);
3261 return FAILURE;
3262 }
3263
zend_jit_unprotect(void)3264 void zend_jit_unprotect(void)
3265 {
3266 #ifdef HAVE_MPROTECT
3267 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3268 int opts = PROT_READ | PROT_WRITE;
3269 #ifdef ZTS
3270 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3271 if (zend_write_protect) {
3272 pthread_jit_write_protect_np(0);
3273 }
3274 #endif
3275 opts |= PROT_EXEC;
3276 #endif
3277 if (mprotect(dasm_buf, dasm_size, opts) != 0) {
3278 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3279 }
3280 }
3281 #elif defined(_WIN32)
3282 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3283 DWORD old, new;
3284 #ifdef ZTS
3285 new = PAGE_EXECUTE_READWRITE;
3286 #else
3287 new = PAGE_READWRITE;
3288 #endif
3289 if (!VirtualProtect(dasm_buf, dasm_size, new, &old)) {
3290 DWORD err = GetLastError();
3291 char *msg = php_win32_error_to_msg(err);
3292 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3293 php_win32_error_msg_free(msg);
3294 }
3295 }
3296 #endif
3297 }
3298
zend_jit_protect(void)3299 void zend_jit_protect(void)
3300 {
3301 #ifdef HAVE_MPROTECT
3302 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3303 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3304 if (zend_write_protect) {
3305 pthread_jit_write_protect_np(1);
3306 }
3307 #endif
3308 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
3309 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3310 }
3311 }
3312 #elif defined(_WIN32)
3313 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3314 DWORD old;
3315
3316 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
3317 DWORD err = GetLastError();
3318 char *msg = php_win32_error_to_msg(err);
3319 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3320 php_win32_error_msg_free(msg);
3321 }
3322 }
3323 #endif
3324 }
3325
zend_jit_init_handlers(void)3326 static void zend_jit_init_handlers(void)
3327 {
3328 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
3329 zend_jit_runtime_jit_handler = zend_jit_stub_handlers[jit_stub_hybrid_runtime_jit];
3330 zend_jit_profile_jit_handler = zend_jit_stub_handlers[jit_stub_hybrid_profile_jit];
3331 zend_jit_func_hot_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_func_hot_counter];
3332 zend_jit_loop_hot_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_loop_hot_counter];
3333 zend_jit_func_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_func_trace_counter];
3334 zend_jit_ret_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_ret_trace_counter];
3335 zend_jit_loop_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_loop_trace_counter];
3336 } else {
3337 zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
3338 zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
3339 zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper;
3340 zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper;
3341 zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper;
3342 zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper;
3343 zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper;
3344 }
3345 }
3346
zend_jit_globals_ctor(zend_jit_globals * jit_globals)3347 static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
3348 {
3349 memset(jit_globals, 0, sizeof(zend_jit_globals));
3350 zend_jit_trace_init_caches();
3351 }
3352
3353 #ifdef ZTS
zend_jit_globals_dtor(zend_jit_globals * jit_globals)3354 static void zend_jit_globals_dtor(zend_jit_globals *jit_globals)
3355 {
3356 zend_jit_trace_free_caches(jit_globals);
3357 }
3358 #endif
3359
zend_jit_parse_config_num(zend_long jit)3360 static int zend_jit_parse_config_num(zend_long jit)
3361 {
3362 if (jit == 0) {
3363 JIT_G(on) = 0;
3364 return SUCCESS;
3365 }
3366
3367 if (jit < 0) return FAILURE;
3368
3369 if (jit % 10 == 0 || jit % 10 > 5) return FAILURE;
3370 JIT_G(opt_level) = jit % 10;
3371
3372 jit /= 10;
3373 if (jit % 10 > 5 || jit % 10 == 4) return FAILURE;
3374 JIT_G(trigger) = jit % 10;
3375
3376 jit /= 10;
3377 if (jit % 10 > 2) return FAILURE;
3378 JIT_G(opt_flags) = jit % 10;
3379
3380 jit /= 10;
3381 if (jit % 10 > 1) return FAILURE;
3382 JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0);
3383
3384 if (jit / 10 != 0) return FAILURE;
3385
3386 JIT_G(on) = 1;
3387
3388 return SUCCESS;
3389 }
3390
zend_jit_config(zend_string * jit,int stage)3391 int zend_jit_config(zend_string *jit, int stage)
3392 {
3393 if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) {
3394 if (stage == ZEND_INI_STAGE_RUNTIME) {
3395 zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)");
3396 }
3397 return FAILURE;
3398 }
3399
3400 if (zend_string_equals_literal_ci(jit, "disable")) {
3401 JIT_G(enabled) = 0;
3402 JIT_G(on) = 0;
3403 return SUCCESS;
3404 } else if (ZSTR_LEN(jit) == 0
3405 || zend_string_equals_literal_ci(jit, "0")
3406 || zend_string_equals_literal_ci(jit, "off")
3407 || zend_string_equals_literal_ci(jit, "no")
3408 || zend_string_equals_literal_ci(jit, "false")) {
3409 JIT_G(enabled) = 1;
3410 JIT_G(on) = 0;
3411 return SUCCESS;
3412 } else if (zend_string_equals_literal_ci(jit, "1")
3413 || zend_string_equals_literal_ci(jit, "on")
3414 || zend_string_equals_literal_ci(jit, "yes")
3415 || zend_string_equals_literal_ci(jit, "true")
3416 || zend_string_equals_literal_ci(jit, "tracing")) {
3417 JIT_G(enabled) = 1;
3418 JIT_G(on) = 1;
3419 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS;
3420 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
3421 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
3422 return SUCCESS;
3423 } else if (zend_string_equals_ci(jit, ZSTR_KNOWN(ZEND_STR_FUNCTION))) {
3424 JIT_G(enabled) = 1;
3425 JIT_G(on) = 1;
3426 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT;
3427 JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD;
3428 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
3429 return SUCCESS;
3430 } else {
3431 char *end;
3432 zend_long num = ZEND_STRTOL(ZSTR_VAL(jit), &end, 10);
3433 if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) {
3434 goto failure;
3435 }
3436 JIT_G(enabled) = 1;
3437 return SUCCESS;
3438 }
3439
3440 failure:
3441 zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number");
3442 JIT_G(enabled) = 0;
3443 JIT_G(on) = 0;
3444 return FAILURE;
3445 }
3446
zend_jit_debug_config(zend_long old_val,zend_long new_val,int stage)3447 int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage)
3448 {
3449 if (stage != ZEND_INI_STAGE_STARTUP) {
3450 if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) {
3451 if (stage == ZEND_INI_STAGE_RUNTIME) {
3452 zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup");
3453 }
3454 return FAILURE;
3455 }
3456 }
3457 return SUCCESS;
3458 }
3459
zend_jit_init(void)3460 void zend_jit_init(void)
3461 {
3462 #ifdef ZTS
3463 jit_globals_id = ts_allocate_id(&jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, (ts_allocate_dtor) zend_jit_globals_dtor);
3464 #else
3465 zend_jit_globals_ctor(&jit_globals);
3466 #endif
3467 }
3468
zend_jit_check_support(void)3469 int zend_jit_check_support(void)
3470 {
3471 int i;
3472
3473 zend_jit_vm_kind = zend_vm_kind();
3474 if (zend_jit_vm_kind != ZEND_VM_KIND_CALL &&
3475 zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
3476 zend_error(E_WARNING, "JIT is compatible only with CALL and HYBRID VM. JIT disabled.");
3477 JIT_G(enabled) = 0;
3478 JIT_G(on) = 0;
3479 return FAILURE;
3480 }
3481
3482 if (zend_execute_ex != execute_ex) {
3483 if (zend_dtrace_enabled) {
3484 zend_error(E_WARNING, "JIT is incompatible with DTrace. JIT disabled.");
3485 } else if (strcmp(sapi_module.name, "phpdbg") != 0) {
3486 zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled.");
3487 }
3488 JIT_G(enabled) = 0;
3489 JIT_G(on) = 0;
3490 return FAILURE;
3491 }
3492
3493 for (i = 0; i <= 256; i++) {
3494 switch (i) {
3495 /* JIT has no effect on these opcodes */
3496 case ZEND_BEGIN_SILENCE:
3497 case ZEND_END_SILENCE:
3498 break;
3499 default:
3500 if (zend_get_user_opcode_handler(i) != NULL) {
3501 zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled.");
3502 JIT_G(enabled) = 0;
3503 JIT_G(on) = 0;
3504 return FAILURE;
3505 }
3506 }
3507 }
3508
3509 #if defined(IR_TARGET_AARCH64)
3510 if (JIT_G(buffer_size) > 128*1024*1024) {
3511 zend_error(E_WARNING, "JIT on AArch64 doesn't support opcache.jit_buffer_size above 128M.");
3512 JIT_G(enabled) = 0;
3513 JIT_G(on) = 0;
3514 return FAILURE;
3515 }
3516 #elif defined(IR_TARGET_X64)
3517 if (JIT_G(buffer_size) > 2 * Z_L(1024*1024*1024)) {
3518 zend_error(E_WARNING, "JIT on x86_64 doesn't support opcache.jit_buffer_size above 2G.");
3519 JIT_G(enabled) = 0;
3520 JIT_G(on) = 0;
3521 return FAILURE;
3522 }
3523 #endif
3524
3525 return SUCCESS;
3526 }
3527
zend_jit_startup(void * buf,size_t size,bool reattached)3528 void zend_jit_startup(void *buf, size_t size, bool reattached)
3529 {
3530 zend_jit_halt_op = zend_get_halt_op();
3531 zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(ACCELERATOR_PRODUCT_NAME);
3532
3533 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3534 zend_write_protect = pthread_jit_write_protect_supported_np();
3535 #endif
3536
3537 dasm_buf = buf;
3538 dasm_size = size;
3539 dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr) * 2);
3540
3541 #ifdef HAVE_MPROTECT
3542 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3543 if (zend_write_protect) {
3544 pthread_jit_write_protect_np(1);
3545 }
3546 #endif
3547 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
3548 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
3549 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3550 }
3551 } else {
3552 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
3553 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3554 }
3555 }
3556 #elif defined(_WIN32)
3557 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
3558 DWORD old;
3559
3560 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
3561 DWORD err = GetLastError();
3562 char *msg = php_win32_error_to_msg(err);
3563 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3564 php_win32_error_msg_free(msg);
3565 }
3566 } else {
3567 DWORD old;
3568
3569 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
3570 DWORD err = GetLastError();
3571 char *msg = php_win32_error_to_msg(err);
3572 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3573 php_win32_error_msg_free(msg);
3574 }
3575 }
3576 #endif
3577
3578 if (!reattached) {
3579 zend_jit_unprotect();
3580 *dasm_ptr = dasm_buf;
3581 #if defined(_WIN32)
3582 zend_jit_stub_handlers = dasm_buf;
3583 *dasm_ptr = (void**)*dasm_ptr + sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
3584 #elif defined(IR_TARGET_AARCH64)
3585 zend_jit_stub_handlers = dasm_buf;
3586 *dasm_ptr = (void**)*dasm_ptr + (sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])) * 2;
3587 memset(zend_jit_stub_handlers, 0, (sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])) * 2 * sizeof(void*));
3588 #endif
3589 *dasm_ptr = (void*)ZEND_MM_ALIGNED_SIZE_EX(((size_t)(*dasm_ptr)), 16);
3590 zend_jit_protect();
3591 } else {
3592 #if defined(_WIN32) || defined(IR_TARGET_AARCH64)
3593 zend_jit_stub_handlers = dasm_buf;
3594 zend_jit_init_handlers();
3595 #endif
3596 }
3597
3598 zend_jit_unprotect();
3599 zend_jit_setup();
3600 zend_jit_protect();
3601 zend_jit_init_handlers();
3602
3603 zend_jit_trace_startup(reattached);
3604
3605 zend_jit_unprotect();
3606 /* save JIT buffer pos */
3607 dasm_ptr[1] = dasm_ptr[0];
3608 zend_jit_protect();
3609 }
3610
zend_jit_shutdown(void)3611 void zend_jit_shutdown(void)
3612 {
3613 if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE && dasm_ptr != NULL) {
3614 fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf));
3615 }
3616
3617 zend_jit_shutdown_ir();
3618
3619 #ifdef ZTS
3620 ts_free_id(jit_globals_id);
3621 #else
3622 zend_jit_trace_free_caches(&jit_globals);
3623 #endif
3624 }
3625
zend_jit_reset_counters(void)3626 static void zend_jit_reset_counters(void)
3627 {
3628 int i;
3629
3630 for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
3631 zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT;
3632 }
3633 }
3634
zend_jit_activate(void)3635 void zend_jit_activate(void)
3636 {
3637 zend_jit_profile_counter = 0;
3638 if (JIT_G(on)) {
3639 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
3640 zend_jit_reset_counters();
3641 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3642 zend_jit_reset_counters();
3643 zend_jit_trace_reset_caches();
3644 }
3645 }
3646 }
3647
zend_jit_deactivate(void)3648 void zend_jit_deactivate(void)
3649 {
3650 if (zend_jit_profile_counter && !CG(unclean_shutdown)) {
3651 zend_class_entry *ce;
3652
3653 zend_shared_alloc_lock();
3654 SHM_UNPROTECT();
3655 zend_jit_unprotect();
3656
3657 zend_jit_check_funcs(EG(function_table), 0);
3658 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(EG(class_table), ce) {
3659 if (ce->type == ZEND_INTERNAL_CLASS) {
3660 break;
3661 }
3662 zend_jit_check_funcs(&ce->function_table, 1);
3663 } ZEND_HASH_FOREACH_END();
3664
3665 zend_jit_protect();
3666 SHM_PROTECT();
3667 zend_shared_alloc_unlock();
3668 }
3669
3670 zend_jit_profile_counter = 0;
3671 }
3672
zend_jit_restart_preloaded_op_array(zend_op_array * op_array)3673 static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array)
3674 {
3675 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
3676
3677 if (!func_info) {
3678 return;
3679 }
3680
3681 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
3682 zend_jit_restart_hot_trace_counters(op_array);
3683 } else if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_COUNTERS) {
3684 zend_jit_restart_hot_counters(op_array);
3685 #if 0
3686 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ???
3687 } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) {
3688 zend_op *opline = op_array->opcodes;
3689 zend_jit_op_array_extension *jit_extension =
3690 (zend_jit_op_array_extension*)func_info;
3691
3692 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3693 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3694 opline++;
3695 }
3696 }
3697 if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) {
3698 opline->handler = (const void*)zend_jit_runtime_jit_handler;
3699 } else {
3700 opline->handler = (const void*)zend_jit_profile_jit_handler;
3701 }
3702 #endif
3703 }
3704 if (op_array->num_dynamic_func_defs) {
3705 for (uint32_t i = 0; i < op_array->num_dynamic_func_defs; i++) {
3706 zend_jit_restart_preloaded_op_array(op_array->dynamic_func_defs[i]);
3707 }
3708 }
3709 }
3710
zend_jit_restart_preloaded_script(zend_persistent_script * script)3711 static void zend_jit_restart_preloaded_script(zend_persistent_script *script)
3712 {
3713 zend_class_entry *ce;
3714 zend_op_array *op_array;
3715
3716 zend_jit_restart_preloaded_op_array(&script->script.main_op_array);
3717
3718 ZEND_HASH_MAP_FOREACH_PTR(&script->script.function_table, op_array) {
3719 zend_jit_restart_preloaded_op_array(op_array);
3720 } ZEND_HASH_FOREACH_END();
3721
3722 ZEND_HASH_MAP_FOREACH_PTR(&script->script.class_table, ce) {
3723 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
3724 if (op_array->type == ZEND_USER_FUNCTION) {
3725 zend_jit_restart_preloaded_op_array(op_array);
3726 }
3727 } ZEND_HASH_FOREACH_END();
3728 } ZEND_HASH_FOREACH_END();
3729 }
3730
zend_jit_restart(void)3731 void zend_jit_restart(void)
3732 {
3733 if (dasm_buf) {
3734 zend_jit_unprotect();
3735
3736 /* restore JIT buffer pos */
3737 dasm_ptr[0] = dasm_ptr[1];
3738
3739 zend_jit_trace_restart();
3740
3741 if (ZCSG(preload_script)) {
3742 zend_jit_restart_preloaded_script(ZCSG(preload_script));
3743 if (ZCSG(saved_scripts)) {
3744 zend_persistent_script **p = ZCSG(saved_scripts);
3745
3746 while (*p) {
3747 zend_jit_restart_preloaded_script(*p);
3748 p++;
3749 }
3750 }
3751 }
3752
3753 zend_jit_protect();
3754 }
3755 }
3756
3757 #endif /* HAVE_JIT */
3758
