1 /*
2 +----------------------------------------------------------------------+
3 | Zend JIT |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Dmitry Stogov <dmitry@php.net> |
16 +----------------------------------------------------------------------+
17 */
18
19 #include "main/php.h"
20 #include "main/SAPI.h"
21 #include "php_version.h"
22 #include <ZendAccelerator.h>
23 #include "zend_shared_alloc.h"
24 #include "Zend/zend_execute.h"
25 #include "Zend/zend_vm.h"
26 #include "Zend/zend_exceptions.h"
27 #include "Zend/zend_constants.h"
28 #include "Zend/zend_closures.h"
29 #include "Zend/zend_ini.h"
30 #include "Zend/zend_observer.h"
31 #include "zend_smart_str.h"
32 #include "jit/zend_jit.h"
33
34 #ifdef HAVE_JIT
35
36 #include "Optimizer/zend_func_info.h"
37 #include "Optimizer/zend_ssa.h"
38 #include "Optimizer/zend_inference.h"
39 #include "Optimizer/zend_call_graph.h"
40 #include "Optimizer/zend_dump.h"
41 #include "Optimizer/zend_worklist.h"
42
43 #include "jit/zend_jit_internal.h"
44
45 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
46 #include <pthread.h>
47 #endif
48
49 #ifdef ZTS
50 int jit_globals_id;
51 #else
52 zend_jit_globals jit_globals;
53 #endif
54
55 //#define CONTEXT_THREADED_JIT
56 #define ZEND_JIT_USE_RC_INFERENCE
57
58 #ifdef ZEND_JIT_USE_RC_INFERENCE
59 # define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE
60 # define RC_MAY_BE_1(info) (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0)
61 # define RC_MAY_BE_N(info) (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0)
62 #else
63 # define ZEND_SSA_RC_INFERENCE_FLAG 0
64 # define RC_MAY_BE_1(info) 1
65 # define RC_MAY_BE_N(info) 1
66 #endif
67
68 #define JIT_PREFIX "JIT$"
69 #define JIT_STUB_PREFIX "JIT$$"
70 #define TRACE_PREFIX "TRACE-"
71
72 zend_ulong zend_jit_profile_counter = 0;
73 int zend_jit_profile_counter_rid = -1;
74
75 int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];
76
77 const zend_op *zend_jit_halt_op = NULL;
78 static int zend_jit_vm_kind = 0;
79 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
80 static int zend_write_protect = 1;
81 #endif
82
83 static void *dasm_buf = NULL;
84 static void *dasm_end = NULL;
85 static void **dasm_ptr = NULL;
86
87 static size_t dasm_size = 0;
88
89 static zend_long jit_bisect_pos = 0;
90
91 static const void *zend_jit_runtime_jit_handler = NULL;
92 static const void *zend_jit_profile_jit_handler = NULL;
93 static const void *zend_jit_func_hot_counter_handler = NULL;
94 static const void *zend_jit_loop_hot_counter_handler = NULL;
95 static const void *zend_jit_func_trace_counter_handler = NULL;
96 static const void *zend_jit_ret_trace_counter_handler = NULL;
97 static const void *zend_jit_loop_trace_counter_handler = NULL;
98
99 static int ZEND_FASTCALL zend_runtime_jit(void);
100
101 static int zend_jit_trace_op_len(const zend_op *opline);
102 static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
103 static uint32_t zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags);
104 static const void *zend_jit_trace_get_exit_addr(uint32_t n);
105 static void zend_jit_trace_add_code(const void *start, uint32_t size);
106 static zend_string *zend_jit_func_name(const zend_op_array *op_array);
107
108 static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info);
109
dominates(const zend_basic_block * blocks,int a,int b)110 static bool dominates(const zend_basic_block *blocks, int a, int b) {
111 while (blocks[b].level > blocks[a].level) {
112 b = blocks[b].idom;
113 }
114 return a == b;
115 }
116
zend_ssa_is_last_use(const zend_op_array * op_array,const zend_ssa * ssa,int var,int use)117 static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
118 {
119 int next_use;
120
121 if (ssa->vars[var].phi_use_chain) {
122 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
123 do {
124 if (!ssa->vars[phi->ssa_var].no_val) {
125 return 0;
126 }
127 phi = zend_ssa_next_use_phi(ssa, var, phi);
128 } while (phi);
129 }
130
131 if (ssa->cfg.blocks[ssa->cfg.map[use]].loop_header > 0
132 || (ssa->cfg.blocks[ssa->cfg.map[use]].flags & ZEND_BB_LOOP_HEADER)) {
133 int b = ssa->cfg.map[use];
134 int prev_use = ssa->vars[var].use_chain;
135 int def_block;
136
137 if (ssa->vars[var].definition >= 0) {
138 def_block =ssa->cfg.map[ssa->vars[var].definition];
139 } else {
140 ZEND_ASSERT(ssa->vars[var].definition_phi);
141 def_block = ssa->vars[var].definition_phi->block;
142 }
143 if (dominates(ssa->cfg.blocks, def_block,
144 (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) ? b : ssa->cfg.blocks[b].loop_header)) {
145 return 0;
146 }
147
148 while (prev_use >= 0 && prev_use != use) {
149 if (b != ssa->cfg.map[prev_use]
150 && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use])
151 && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) {
152 return 0;
153 }
154 prev_use = zend_ssa_next_use(ssa->ops, var, prev_use);
155 }
156 }
157
158 next_use = zend_ssa_next_use(ssa->ops, var, use);
159 if (next_use < 0) {
160 return 1;
161 } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) {
162 return 1;
163 }
164 return 0;
165 }
166
zend_jit_is_constant_cmp_long_long(const zend_op * opline,zend_ssa_range * op1_range,zend_jit_addr op1_addr,zend_ssa_range * op2_range,zend_jit_addr op2_addr,bool * result)167 static int zend_jit_is_constant_cmp_long_long(const zend_op *opline,
168 zend_ssa_range *op1_range,
169 zend_jit_addr op1_addr,
170 zend_ssa_range *op2_range,
171 zend_jit_addr op2_addr,
172 bool *result)
173 {
174 zend_long op1_min;
175 zend_long op1_max;
176 zend_long op2_min;
177 zend_long op2_max;
178
179 if (op1_range) {
180 op1_min = op1_range->min;
181 op1_max = op1_range->max;
182 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
183 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG);
184 op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr));
185 } else {
186 return 0;
187 }
188
189 if (op2_range) {
190 op2_min = op2_range->min;
191 op2_max = op2_range->max;
192 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
193 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG);
194 op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr));
195 } else {
196 return 0;
197 }
198
199 switch (opline->opcode) {
200 case ZEND_IS_EQUAL:
201 case ZEND_IS_IDENTICAL:
202 case ZEND_CASE:
203 case ZEND_CASE_STRICT:
204 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
205 *result = 1;
206 return 1;
207 } else if (op1_max < op2_min || op1_min > op2_max) {
208 *result = 0;
209 return 1;
210 }
211 return 0;
212 case ZEND_IS_NOT_EQUAL:
213 case ZEND_IS_NOT_IDENTICAL:
214 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
215 *result = 0;
216 return 1;
217 } else if (op1_max < op2_min || op1_min > op2_max) {
218 *result = 1;
219 return 1;
220 }
221 return 0;
222 case ZEND_IS_SMALLER:
223 if (op1_max < op2_min) {
224 *result = 1;
225 return 1;
226 } else if (op1_min >= op2_max) {
227 *result = 0;
228 return 1;
229 }
230 return 0;
231 case ZEND_IS_SMALLER_OR_EQUAL:
232 if (op1_max <= op2_min) {
233 *result = 1;
234 return 1;
235 } else if (op1_min > op2_max) {
236 *result = 0;
237 return 1;
238 }
239 return 0;
240 default:
241 ZEND_UNREACHABLE();
242 }
243 return 0;
244 }
245
246 #define ADVANCE_SSA_OP(ssa_op, offset) \
247 do { \
248 if (ssa_op) ssa_op += offset; \
249 } while (0)
250
zend_jit_needs_call_chain(zend_call_info * call_info,uint32_t b,const zend_op_array * op_array,zend_ssa * ssa,const zend_ssa_op * ssa_op,const zend_op * opline,int call_level,zend_jit_trace_rec * trace)251 static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace)
252 {
253 int skip;
254
255 if (trace) {
256 zend_jit_trace_rec *p = trace;
257
258 ADVANCE_SSA_OP(ssa_op, 1);
259 while (1) {
260 if (p->op == ZEND_JIT_TRACE_VM) {
261 switch (p->opline->opcode) {
262 case ZEND_SEND_ARRAY:
263 case ZEND_SEND_USER:
264 case ZEND_SEND_UNPACK:
265 case ZEND_INIT_FCALL:
266 case ZEND_INIT_METHOD_CALL:
267 case ZEND_INIT_STATIC_METHOD_CALL:
268 case ZEND_INIT_FCALL_BY_NAME:
269 case ZEND_INIT_NS_FCALL_BY_NAME:
270 case ZEND_INIT_DYNAMIC_CALL:
271 case ZEND_NEW:
272 case ZEND_INIT_USER_CALL:
273 case ZEND_FAST_CALL:
274 case ZEND_JMP:
275 case ZEND_JMPZ:
276 case ZEND_JMPNZ:
277 case ZEND_JMPZ_EX:
278 case ZEND_JMPNZ_EX:
279 case ZEND_FE_RESET_R:
280 case ZEND_FE_RESET_RW:
281 case ZEND_JMP_SET:
282 case ZEND_COALESCE:
283 case ZEND_JMP_NULL:
284 case ZEND_ASSERT_CHECK:
285 case ZEND_CATCH:
286 case ZEND_DECLARE_ANON_CLASS:
287 case ZEND_FE_FETCH_R:
288 case ZEND_FE_FETCH_RW:
289 case ZEND_BIND_INIT_STATIC_OR_JMP:
290 case ZEND_JMP_FRAMELESS:
291 return 1;
292 case ZEND_DO_ICALL:
293 case ZEND_DO_UCALL:
294 case ZEND_DO_FCALL_BY_NAME:
295 case ZEND_DO_FCALL:
296 case ZEND_CALLABLE_CONVERT:
297 return 0;
298 case ZEND_SEND_VAL:
299 case ZEND_SEND_VAR:
300 case ZEND_SEND_VAL_EX:
301 case ZEND_SEND_VAR_EX:
302 case ZEND_SEND_FUNC_ARG:
303 case ZEND_SEND_REF:
304 case ZEND_SEND_VAR_NO_REF:
305 case ZEND_SEND_VAR_NO_REF_EX:
306 /* skip */
307 break;
308 default:
309 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
310 return 1;
311 }
312 }
313 ADVANCE_SSA_OP(ssa_op, zend_jit_trace_op_len(opline));
314 } else if (p->op == ZEND_JIT_TRACE_ENTER ||
315 p->op == ZEND_JIT_TRACE_BACK ||
316 p->op == ZEND_JIT_TRACE_END) {
317 return 1;
318 }
319 p++;
320 }
321 }
322
323 if (!call_info) {
324 const zend_op *end = op_array->opcodes + op_array->last;
325
326 opline++;
327 ADVANCE_SSA_OP(ssa_op, 1);
328 skip = (call_level == 1);
329 while (opline != end) {
330 if (!skip) {
331 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
332 return 1;
333 }
334 }
335 switch (opline->opcode) {
336 case ZEND_SEND_VAL:
337 case ZEND_SEND_VAR:
338 case ZEND_SEND_VAL_EX:
339 case ZEND_SEND_VAR_EX:
340 case ZEND_SEND_FUNC_ARG:
341 case ZEND_SEND_REF:
342 case ZEND_SEND_VAR_NO_REF:
343 case ZEND_SEND_VAR_NO_REF_EX:
344 skip = 0;
345 break;
346 case ZEND_SEND_ARRAY:
347 case ZEND_SEND_USER:
348 case ZEND_SEND_UNPACK:
349 case ZEND_INIT_FCALL:
350 case ZEND_INIT_METHOD_CALL:
351 case ZEND_INIT_STATIC_METHOD_CALL:
352 case ZEND_INIT_FCALL_BY_NAME:
353 case ZEND_INIT_NS_FCALL_BY_NAME:
354 case ZEND_INIT_DYNAMIC_CALL:
355 case ZEND_NEW:
356 case ZEND_INIT_USER_CALL:
357 case ZEND_FAST_CALL:
358 case ZEND_JMP:
359 case ZEND_JMPZ:
360 case ZEND_JMPNZ:
361 case ZEND_JMPZ_EX:
362 case ZEND_JMPNZ_EX:
363 case ZEND_FE_RESET_R:
364 case ZEND_FE_RESET_RW:
365 case ZEND_JMP_SET:
366 case ZEND_COALESCE:
367 case ZEND_JMP_NULL:
368 case ZEND_ASSERT_CHECK:
369 case ZEND_CATCH:
370 case ZEND_DECLARE_ANON_CLASS:
371 case ZEND_FE_FETCH_R:
372 case ZEND_FE_FETCH_RW:
373 case ZEND_BIND_INIT_STATIC_OR_JMP:
374 case ZEND_JMP_FRAMELESS:
375 return 1;
376 case ZEND_DO_ICALL:
377 case ZEND_DO_UCALL:
378 case ZEND_DO_FCALL_BY_NAME:
379 case ZEND_DO_FCALL:
380 case ZEND_CALLABLE_CONVERT:
381 end = opline;
382 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
383 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
384 return 1;
385 }
386 return 0;
387 }
388 opline++;
389 ADVANCE_SSA_OP(ssa_op, 1);
390 }
391
392 return 1;
393 } else {
394 const zend_op *end = call_info->caller_call_opline;
395
396 /* end may be null if an opcode like EXIT is part of the argument list. */
397 if (!end || end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
398 /* INIT_FCALL and DO_FCALL in different BasicBlocks */
399 return 1;
400 }
401
402 opline++;
403 ADVANCE_SSA_OP(ssa_op, 1);
404 skip = (call_level == 1);
405 while (opline != end) {
406 if (skip) {
407 switch (opline->opcode) {
408 case ZEND_SEND_VAL:
409 case ZEND_SEND_VAR:
410 case ZEND_SEND_VAL_EX:
411 case ZEND_SEND_VAR_EX:
412 case ZEND_SEND_FUNC_ARG:
413 case ZEND_SEND_REF:
414 case ZEND_SEND_VAR_NO_REF:
415 case ZEND_SEND_VAR_NO_REF_EX:
416 skip = 0;
417 break;
418 case ZEND_SEND_ARRAY:
419 case ZEND_SEND_USER:
420 case ZEND_SEND_UNPACK:
421 return 1;
422 }
423 } else {
424 if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
425 return 1;
426 }
427 }
428 opline++;
429 ADVANCE_SSA_OP(ssa_op, 1);
430 }
431
432 return 0;
433 }
434 }
435
skip_valid_arguments(const zend_op_array * op_array,zend_ssa * ssa,const zend_call_info * call_info)436 static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
437 {
438 uint32_t num_args = 0;
439 zend_function *func = call_info->callee_func;
440
441 /* It's okay to handle prototypes here, because they can only increase the accepted arguments.
442 * Anything legal for the parent method is also legal for the parent method. */
443 while (num_args < call_info->num_args) {
444 zend_arg_info *arg_info = func->op_array.arg_info + num_args;
445
446 if (ZEND_TYPE_IS_SET(arg_info->type)) {
447 if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) {
448 zend_op *opline = call_info->arg_info[num_args].opline;
449 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[opline - op_array->opcodes] : NULL;
450 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
451 if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) {
452 break;
453 }
454 } else {
455 break;
456 }
457 }
458 num_args++;
459 }
460 return num_args;
461 }
462
zend_ssa_cv_info(const zend_op_array * op_array,zend_ssa * ssa,uint32_t var)463 static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var)
464 {
465 uint32_t j, info;
466
467 if (ssa->vars && ssa->var_info) {
468 info = ssa->var_info[var].type;
469 for (j = op_array->last_var; j < ssa->vars_count; j++) {
470 if (ssa->vars[j].var == var) {
471 info |= ssa->var_info[j].type;
472 }
473 }
474 } else {
475 info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF |
476 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
477 }
478
479 #ifdef ZEND_JIT_USE_RC_INFERENCE
480 /* Refcount may be increased by RETURN opcode */
481 if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
482 for (j = 0; j < ssa->cfg.blocks_count; j++) {
483 if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
484 ssa->cfg.blocks[j].len > 0) {
485 const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;
486
487 if (opline->opcode == ZEND_RETURN) {
488 if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) {
489 info |= MAY_BE_RCN;
490 break;
491 }
492 }
493 }
494 }
495 }
496 #endif
497
498 return info;
499 }
500
zend_jit_may_avoid_refcounting(const zend_op * opline,uint32_t op1_info)501 static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_info)
502 {
503 switch (opline->opcode) {
504 case ZEND_FETCH_OBJ_FUNC_ARG:
505 if (!JIT_G(current_frame) ||
506 !JIT_G(current_frame)->call->func ||
507 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
508 return 0;
509 }
510 /* break missing intentionally */
511 case ZEND_FETCH_OBJ_R:
512 case ZEND_FETCH_OBJ_IS:
513 if ((op1_info & MAY_BE_OBJECT)
514 && opline->op2_type == IS_CONST
515 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
516 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
517 return 1;
518 }
519 break;
520 case ZEND_FETCH_DIM_FUNC_ARG:
521 if (!JIT_G(current_frame) ||
522 !JIT_G(current_frame)->call->func ||
523 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
524 return 0;
525 }
526 /* break missing intentionally */
527 case ZEND_FETCH_DIM_R:
528 case ZEND_FETCH_DIM_IS:
529 return 1;
530 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
531 if (!(opline->extended_value & ZEND_ISEMPTY)) {
532 return 1;
533 }
534 break;
535 }
536 return 0;
537 }
538
zend_jit_is_persistent_constant(zval * key,uint32_t flags)539 static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags)
540 {
541 zval *zv;
542 zend_constant *c = NULL;
543
544 /* null/true/false are resolved during compilation, so don't check for them here. */
545 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
546 if (zv) {
547 c = (zend_constant*)Z_PTR_P(zv);
548 } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
549 key++;
550 zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
551 if (zv) {
552 c = (zend_constant*)Z_PTR_P(zv);
553 }
554 }
555 return c && (ZEND_CONSTANT_FLAGS(c) & CONST_PERSISTENT);
556 }
557
zend_get_known_property_info(const zend_op_array * op_array,zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)558 static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
559 {
560 zend_property_info *info = NULL;
561
562 if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) ||
563 !ce ||
564 !(ce->ce_flags & ZEND_ACC_LINKED) ||
565 (ce->ce_flags & ZEND_ACC_TRAIT) ||
566 ce->create_object) {
567 return NULL;
568 }
569
570 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
571 if (ce->info.user.filename != filename) {
572 /* class declaration might be changed independently */
573 return NULL;
574 }
575
576 if (ce->parent) {
577 zend_class_entry *parent = ce->parent;
578
579 do {
580 if (parent->type == ZEND_INTERNAL_CLASS) {
581 break;
582 } else if (parent->info.user.filename != filename) {
583 /* some of parents class declarations might be changed independently */
584 /* TODO: this check may be not enough, because even
585 * in the same it's possible to conditionally define
586 * few classes with the same name, and "parent" may
587 * change from request to request.
588 */
589 return NULL;
590 }
591 parent = parent->parent;
592 } while (parent);
593 }
594 }
595
596 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
597 if (info == NULL ||
598 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
599 (info->flags & ZEND_ACC_STATIC)) {
600 return NULL;
601 }
602
603 if (info->flags & ZEND_ACC_PUBLIC) {
604 return info;
605 } else if (on_this) {
606 if (ce == info->ce) {
607 if (ce == op_array->scope) {
608 return info;
609 } else {
610 return NULL;
611 }
612 } else if ((info->flags & ZEND_ACC_PROTECTED)
613 && instanceof_function_slow(ce, info->ce)) {
614 return info;
615 }
616 }
617
618 return NULL;
619 }
620
zend_may_be_dynamic_property(zend_class_entry * ce,zend_string * member,bool on_this,zend_string * filename)621 static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
622 {
623 zend_property_info *info;
624
625 if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) {
626 return 1;
627 }
628
629 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
630 if (ce->info.user.filename != filename) {
631 /* class declaration might be changed independently */
632 return 1;
633 }
634 }
635
636 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
637 if (info == NULL ||
638 !IS_VALID_PROPERTY_OFFSET(info->offset) ||
639 (info->flags & ZEND_ACC_STATIC)) {
640 return 1;
641 }
642
643 if (!(info->flags & ZEND_ACC_PUBLIC) &&
644 (!on_this || info->ce != ce)) {
645 return 1;
646 }
647
648 return 0;
649 }
650
651 #define OP_RANGE(ssa_op, opN) \
652 (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
653 ssa->var_info && \
654 (ssa_op)->opN##_use >= 0 && \
655 ssa->var_info[(ssa_op)->opN##_use].has_range) ? \
656 &ssa->var_info[(ssa_op)->opN##_use].range : NULL)
657
658 #define OP1_RANGE() OP_RANGE(ssa_op, op1)
659 #define OP2_RANGE() OP_RANGE(ssa_op, op2)
660 #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)
661
662 #include "jit/zend_jit_helpers.c"
663 #include "Zend/zend_cpuinfo.h"
664
665 #ifdef HAVE_GCC_GLOBAL_REGS
666 # define GCC_GLOBAL_REGS 1
667 #else
668 # define GCC_GLOBAL_REGS 0
669 #endif
670
671 /* By default avoid JITing inline handlers if it does not seem profitable due to lack of
672 * type information. Disabling this option allows testing some JIT handlers in the
673 * presence of try/catch blocks, which prevent SSA construction. */
674 #ifndef PROFITABILITY_CHECKS
675 # define PROFITABILITY_CHECKS 1
676 #endif
677
678 #define BP_JIT_IS 6 /* Used for ISSET_ISEMPTY_DIM_OBJ. see BP_VAR_*defines in Zend/zend_compile.h */
679
680 /* The generated code may contain tautological comparisons, ignore them. */
681 #if defined(__clang__)
682 # pragma clang diagnostic push
683 # pragma clang diagnostic ignored "-Wtautological-compare"
684 # pragma clang diagnostic ignored "-Wstring-compare"
685 #endif
686
687 #include "jit/zend_jit_ir.c"
688
689 #if defined(__clang__)
690 # pragma clang diagnostic pop
691 #endif
692
693 #if _WIN32
694 # include <Windows.h>
695 #else
696 # include <sys/mman.h>
697 # if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
698 # define MAP_ANONYMOUS MAP_ANON
699 # endif
700 #endif
701
zend_jit_status(zval * ret)702 ZEND_EXT_API void zend_jit_status(zval *ret)
703 {
704 zval stats;
705 array_init(&stats);
706 add_assoc_bool(&stats, "enabled", JIT_G(enabled));
707 add_assoc_bool(&stats, "on", JIT_G(on));
708 add_assoc_long(&stats, "kind", JIT_G(trigger));
709 add_assoc_long(&stats, "opt_level", JIT_G(opt_level));
710 add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags));
711 if (dasm_buf) {
712 add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
713 add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
714 } else {
715 add_assoc_long(&stats, "buffer_size", 0);
716 add_assoc_long(&stats, "buffer_free", 0);
717 }
718 add_assoc_zval(ret, "jit", &stats);
719 }
720
zend_jit_func_name(const zend_op_array * op_array)721 static zend_string *zend_jit_func_name(const zend_op_array *op_array)
722 {
723 smart_str buf = {0};
724
725 if (op_array->function_name) {
726 smart_str_appends(&buf, JIT_PREFIX);
727 if (op_array->scope) {
728 smart_str_appendl(&buf, ZSTR_VAL(op_array->scope->name), ZSTR_LEN(op_array->scope->name));
729 smart_str_appends(&buf, "::");
730 }
731 smart_str_appendl(&buf, ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name));
732 if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
733 smart_str_appends(&buf, ":");
734 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
735 smart_str_appends(&buf, ":");
736 smart_str_append_long(&buf, op_array->line_start);
737 }
738 smart_str_0(&buf);
739 return buf.s;
740 } else if (op_array->filename) {
741 smart_str_appends(&buf, JIT_PREFIX);
742 smart_str_appendl(&buf, ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename));
743 smart_str_0(&buf);
744 return buf.s;
745 } else {
746 return NULL;
747 }
748 }
749
zend_may_overflow(const zend_op * opline,const zend_ssa_op * ssa_op,const zend_op_array * op_array,zend_ssa * ssa)750 static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
751 {
752 int res;
753 zend_long op1_min, op1_max, op2_min, op2_max;
754
755 if (!ssa->ops || !ssa->var_info) {
756 return 1;
757 }
758 switch (opline->opcode) {
759 case ZEND_PRE_INC:
760 case ZEND_POST_INC:
761 res = ssa_op->op1_def;
762 if (res < 0
763 || !ssa->var_info[res].has_range
764 || ssa->var_info[res].range.overflow) {
765 if (!OP1_HAS_RANGE()) {
766 return 1;
767 }
768 op1_max = OP1_MAX_RANGE();
769 if (op1_max == ZEND_LONG_MAX) {
770 return 1;
771 }
772 }
773 return 0;
774 case ZEND_PRE_DEC:
775 case ZEND_POST_DEC:
776 res = ssa_op->op1_def;
777 if (res < 0
778 || !ssa->var_info[res].has_range
779 || ssa->var_info[res].range.underflow) {
780 if (!OP1_HAS_RANGE()) {
781 return 1;
782 }
783 op1_min = OP1_MIN_RANGE();
784 if (op1_min == ZEND_LONG_MIN) {
785 return 1;
786 }
787 }
788 return 0;
789 case ZEND_ADD:
790 res = ssa_op->result_def;
791 if (res < 0
792 || !ssa->var_info[res].has_range
793 || ssa->var_info[res].range.underflow) {
794 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
795 return 1;
796 }
797 op1_min = OP1_MIN_RANGE();
798 op2_min = OP2_MIN_RANGE();
799 if (zend_add_will_overflow(op1_min, op2_min)) {
800 return 1;
801 }
802 }
803 if (res < 0
804 || !ssa->var_info[res].has_range
805 || ssa->var_info[res].range.overflow) {
806 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
807 return 1;
808 }
809 op1_max = OP1_MAX_RANGE();
810 op2_max = OP2_MAX_RANGE();
811 if (zend_add_will_overflow(op1_max, op2_max)) {
812 return 1;
813 }
814 }
815 return 0;
816 case ZEND_SUB:
817 res = ssa_op->result_def;
818 if (res < 0
819 || !ssa->var_info[res].has_range
820 || ssa->var_info[res].range.underflow) {
821 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
822 return 1;
823 }
824 op1_min = OP1_MIN_RANGE();
825 op2_max = OP2_MAX_RANGE();
826 if (zend_sub_will_overflow(op1_min, op2_max)) {
827 return 1;
828 }
829 }
830 if (res < 0
831 || !ssa->var_info[res].has_range
832 || ssa->var_info[res].range.overflow) {
833 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
834 return 1;
835 }
836 op1_max = OP1_MAX_RANGE();
837 op2_min = OP2_MIN_RANGE();
838 if (zend_sub_will_overflow(op1_max, op2_min)) {
839 return 1;
840 }
841 }
842 return 0;
843 case ZEND_MUL:
844 res = ssa_op->result_def;
845 return (res < 0 ||
846 !ssa->var_info[res].has_range ||
847 ssa->var_info[res].range.underflow ||
848 ssa->var_info[res].range.overflow);
849 case ZEND_ASSIGN_OP:
850 if (opline->extended_value == ZEND_ADD) {
851 res = ssa_op->op1_def;
852 if (res < 0
853 || !ssa->var_info[res].has_range
854 || ssa->var_info[res].range.underflow) {
855 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
856 return 1;
857 }
858 op1_min = OP1_MIN_RANGE();
859 op2_min = OP2_MIN_RANGE();
860 if (zend_add_will_overflow(op1_min, op2_min)) {
861 return 1;
862 }
863 }
864 if (res < 0
865 || !ssa->var_info[res].has_range
866 || ssa->var_info[res].range.overflow) {
867 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
868 return 1;
869 }
870 op1_max = OP1_MAX_RANGE();
871 op2_max = OP2_MAX_RANGE();
872 if (zend_add_will_overflow(op1_max, op2_max)) {
873 return 1;
874 }
875 }
876 return 0;
877 } else if (opline->extended_value == ZEND_SUB) {
878 res = ssa_op->op1_def;
879 if (res < 0
880 || !ssa->var_info[res].has_range
881 || ssa->var_info[res].range.underflow) {
882 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
883 return 1;
884 }
885 op1_min = OP1_MIN_RANGE();
886 op2_max = OP2_MAX_RANGE();
887 if (zend_sub_will_overflow(op1_min, op2_max)) {
888 return 1;
889 }
890 }
891 if (res < 0
892 || !ssa->var_info[res].has_range
893 || ssa->var_info[res].range.overflow) {
894 if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
895 return 1;
896 }
897 op1_max = OP1_MAX_RANGE();
898 op2_min = OP2_MIN_RANGE();
899 if (zend_sub_will_overflow(op1_max, op2_min)) {
900 return 1;
901 }
902 }
903 return 0;
904 } else if (opline->extended_value == ZEND_MUL) {
905 res = ssa_op->op1_def;
906 return (res < 0 ||
907 !ssa->var_info[res].has_range ||
908 ssa->var_info[res].range.underflow ||
909 ssa->var_info[res].range.overflow);
910 }
911 ZEND_FALLTHROUGH;
912 default:
913 return 1;
914 }
915 }
916
zend_jit_build_cfg(const zend_op_array * op_array,zend_cfg * cfg)917 static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg)
918 {
919 uint32_t flags;
920
921 flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY;
922
923 zend_build_cfg(&CG(arena), op_array, flags, cfg);
924
925 /* Don't JIT huge functions. Apart from likely being detrimental due to the amount of
926 * generated code, some of our analysis is recursive and will stack overflow with many
927 * blocks. */
928 if (cfg->blocks_count > 100000) {
929 return FAILURE;
930 }
931
932 zend_cfg_build_predecessors(&CG(arena), cfg);
933
934 /* Compute Dominators Tree */
935 zend_cfg_compute_dominators_tree(op_array, cfg);
936
937 /* Identify reducible and irreducible loops */
938 zend_cfg_identify_loops(op_array, cfg);
939
940 return SUCCESS;
941 }
942
zend_jit_op_array_analyze1(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa)943 static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
944 {
945 if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) {
946 return FAILURE;
947 }
948
949 #if 0
950 /* TODO: debugger and profiler supports? */
951 if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) {
952 return FAILURE;
953 }
954 #endif
955
956 /* TODO: move this to zend_cfg.c ? */
957 if (!op_array->function_name) {
958 ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
959 }
960
961 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
962 && ssa->cfg.blocks
963 && op_array->last_try_catch == 0
964 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
965 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
966 if (zend_build_ssa(&CG(arena), script, op_array, ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) {
967 return FAILURE;
968 }
969
970 zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa);
971
972 zend_ssa_find_false_dependencies(op_array, ssa);
973
974 zend_ssa_find_sccs(op_array, ssa);
975 }
976
977 return SUCCESS;
978 }
979
zend_jit_op_array_analyze2(const zend_op_array * op_array,zend_script * script,zend_ssa * ssa,uint32_t optimization_level)980 static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t optimization_level)
981 {
982 if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
983 && ssa->cfg.blocks
984 && op_array->last_try_catch == 0
985 && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
986 && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
987 if (zend_ssa_inference(&CG(arena), op_array, script, ssa,
988 optimization_level & ~ZEND_OPTIMIZER_NARROW_TO_DOUBLE) != SUCCESS) {
989 return FAILURE;
990 }
991 }
992
993 return SUCCESS;
994 }
995
zend_jit_allocate_registers(zend_jit_ctx * ctx,const zend_op_array * op_array,zend_ssa * ssa)996 static void zend_jit_allocate_registers(zend_jit_ctx *ctx, const zend_op_array *op_array, zend_ssa *ssa)
997 {
998 void *checkpoint;
999 int candidates_count, i;
1000 zend_jit_reg_var *ra;
1001
1002 checkpoint = zend_arena_checkpoint(CG(arena));
1003 ra = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_jit_reg_var));
1004 candidates_count = 0;
1005 for (i = 0; i < ssa->vars_count; i++) {
1006 if (zend_jit_may_be_in_reg(op_array, ssa, i)) {
1007 ra[i].ref = IR_NULL;
1008 candidates_count++;
1009 }
1010 }
1011 if (!candidates_count) {
1012 zend_arena_release(&CG(arena), checkpoint);
1013 return;
1014 }
1015
1016 if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
1017 /* Naive SSA resolution */
1018 for (i = 0; i < ssa->vars_count; i++) {
1019 if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
1020 zend_ssa_phi *phi = ssa->vars[i].definition_phi;
1021 int k, src;
1022
1023 if (phi->pi >= 0) {
1024 src = phi->sources[0];
1025 if (ra[i].ref) {
1026 if (!ra[src].ref) {
1027 ra[i].flags |= ZREG_LOAD;
1028 } else {
1029 ra[i].flags |= ZREG_PI;
1030 }
1031 } else if (ra[src].ref) {
1032 ra[src].flags |= ZREG_STORE;
1033 }
1034 } else {
1035 int need_move = 0;
1036
1037 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1038 src = phi->sources[k];
1039 if (src >= 0) {
1040 if (ssa->vars[src].definition_phi
1041 && ssa->vars[src].definition_phi->pi >= 0
1042 && phi->block == ssa->vars[src].definition_phi->block) {
1043 /* Skip zero-length interval for Pi variable */
1044 src = ssa->vars[src].definition_phi->sources[0];
1045 }
1046 if (ra[i].ref) {
1047 if (!ra[src].ref) {
1048 need_move = 1;
1049 }
1050 } else if (ra[src].ref) {
1051 need_move = 1;
1052 }
1053 }
1054 }
1055 if (need_move) {
1056 if (ra[i].ref) {
1057 ra[i].flags |= ZREG_LOAD;
1058 }
1059 for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
1060 src = phi->sources[k];
1061 if (src >= 0) {
1062 if (ssa->vars[src].definition_phi
1063 && ssa->vars[src].definition_phi->pi >= 0
1064 && phi->block == ssa->vars[src].definition_phi->block) {
1065 /* Skip zero-length interval for Pi variable */
1066 src = ssa->vars[src].definition_phi->sources[0];
1067 }
1068 if (ra[src].ref) {
1069 ra[src].flags |= ZREG_STORE;
1070 }
1071 }
1072 }
1073 } else {
1074 ra[i].flags |= ZREG_PHI;
1075 }
1076 }
1077 }
1078 }
1079
1080 /* Remove useless register allocation */
1081 for (i = 0; i < ssa->vars_count; i++) {
1082 if (ra[i].ref &&
1083 ((ra[i].flags & ZREG_LOAD) ||
1084 ((ra[i].flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) &&
1085 ssa->vars[i].use_chain < 0) {
1086 bool may_remove = 1;
1087 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
1088
1089 while (phi) {
1090 if (ra[phi->ssa_var].ref &&
1091 !(ra[phi->ssa_var].flags & ZREG_LOAD)) {
1092 may_remove = 0;
1093 break;
1094 }
1095 phi = zend_ssa_next_use_phi(ssa, i, phi);
1096 }
1097 if (may_remove) {
1098 ra[i].ref = IR_UNUSED;
1099 }
1100 }
1101 }
1102
1103 /* Remove intervals used once */
1104 for (i = 0; i < ssa->vars_count; i++) {
1105 if (ra[i].ref) {
1106 if (!(ra[i].flags & (ZREG_LOAD|ZREG_STORE))) {
1107 uint32_t var_num = ssa->vars[i].var;
1108 uint32_t op_num = ssa->vars[i].definition;
1109
1110 /* Check if a tempoary variable may be freed by exception handler */
1111 if (op_array->last_live_range
1112 && var_num >= op_array->last_var
1113 && ssa->vars[i].definition >= 0
1114 && ssa->ops[op_num].result_def == i) {
1115 const zend_live_range *range = op_array->live_range;
1116 int j;
1117
1118 op_num++;
1119 if (op_array->opcodes[op_num].opcode == ZEND_OP_DATA) {
1120 op_num++;
1121 }
1122 for (j = 0; j < op_array->last_live_range; range++, j++) {
1123 if (range->start > op_num) {
1124 /* further blocks will not be relevant... */
1125 break;
1126 } else if (op_num < range->end && var_num == (range->var & ~ZEND_LIVE_MASK)) {
1127 /* check if opcodes in range may throw */
1128 do {
1129 if (zend_may_throw(op_array->opcodes + op_num, ssa->ops + op_num, op_array, ssa)) {
1130 ra[i].flags |= ZREG_STORE;
1131 break;
1132 }
1133 op_num++;
1134 if (op_array->opcodes[op_num].opcode == ZEND_OP_DATA) {
1135 op_num++;
1136 }
1137 } while (op_num < range->end);
1138 break;
1139 }
1140 }
1141 }
1142 }
1143 if ((ra[i].flags & ZREG_LOAD)
1144 && (ra[i].flags & ZREG_STORE)
1145 && (ssa->vars[i].use_chain < 0
1146 || zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) {
1147 bool may_remove = 1;
1148 zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;
1149
1150 while (phi) {
1151 if (ra[phi->ssa_var].ref &&
1152 !(ra[phi->ssa_var].flags & ZREG_LOAD)) {
1153 may_remove = 0;
1154 break;
1155 }
1156 phi = zend_ssa_next_use_phi(ssa, i, phi);
1157 }
1158 if (may_remove) {
1159 ra[i].ref = IR_UNUSED;
1160 }
1161 }
1162 }
1163 }
1164 }
1165
1166 if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
1167 fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
1168 for (i = 0; i < ssa->vars_count; i++) {
1169 if (ra[i].ref) {
1170 fprintf(stderr, "#%d.", i);
1171 uint32_t var_num = ssa->vars[i].var;
1172 zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
1173 if (ra[i].flags & ZREG_LOAD) {
1174 fprintf(stderr, " load");
1175 }
1176 if (ra[i].flags & ZREG_STORE) {
1177 fprintf(stderr, " store");
1178 }
1179 fprintf(stderr, "\n");
1180 }
1181 }
1182 fprintf(stderr, "\n");
1183 }
1184
1185 ctx->ra = ra;
1186 }
1187
zend_jit_compute_post_order(zend_cfg * cfg,int start,int * post_order)1188 static int zend_jit_compute_post_order(zend_cfg *cfg, int start, int *post_order)
1189 {
1190 int count = 0;
1191 int b, n, *p;
1192 zend_basic_block *bb;
1193 zend_worklist worklist;
1194 ALLOCA_FLAG(use_heap)
1195
1196 ZEND_WORKLIST_ALLOCA(&worklist, cfg->blocks_count, use_heap);
1197 zend_worklist_push(&worklist, start);
1198
1199 while (zend_worklist_len(&worklist) != 0) {
1200 next:
1201 b = zend_worklist_peek(&worklist);
1202 bb = &cfg->blocks[b];
1203 n = bb->successors_count;
1204 if (n > 0) {
1205 p = bb->successors;
1206 do {
1207 if (cfg->blocks[*p].flags & (ZEND_BB_CATCH|ZEND_BB_FINALLY|ZEND_BB_FINALLY_END)) {
1208 /* skip */
1209 } else if (zend_worklist_push(&worklist, *p)) {
1210 goto next;
1211 }
1212 p++;
1213 n--;
1214 } while (n > 0);
1215 }
1216 zend_worklist_pop(&worklist);
1217 post_order[count++] = b;
1218 }
1219 ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
1220 return count;
1221 }
1222
zend_jit_next_is_send_result(const zend_op * opline)1223 static bool zend_jit_next_is_send_result(const zend_op *opline)
1224 {
1225 if (opline->result_type == IS_TMP_VAR
1226 && (opline+1)->opcode == ZEND_SEND_VAL
1227 && (opline+1)->op1_type == IS_TMP_VAR
1228 && (opline+1)->op2_type != IS_CONST
1229 && (opline+1)->op1.var == opline->result.var) {
1230 return 1;
1231 }
1232 return 0;
1233 }
1234
zend_jit_supported_binary_op(uint8_t op,uint32_t op1_info,uint32_t op2_info)1235 static bool zend_jit_supported_binary_op(uint8_t op, uint32_t op1_info, uint32_t op2_info)
1236 {
1237 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1238 return false;
1239 }
1240 switch (op) {
1241 case ZEND_POW:
1242 case ZEND_DIV:
1243 // TODO: check for division by zero ???
1244 return false;
1245 case ZEND_ADD:
1246 case ZEND_SUB:
1247 case ZEND_MUL:
1248 return (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
1249 && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
1250 case ZEND_BW_OR:
1251 case ZEND_BW_AND:
1252 case ZEND_BW_XOR:
1253 case ZEND_SL:
1254 case ZEND_SR:
1255 case ZEND_MOD:
1256 return (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG);
1257 case ZEND_CONCAT:
1258 return (op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING);
1259 EMPTY_SWITCH_DEFAULT_CASE()
1260 }
1261 }
1262
zend_jit(const zend_op_array * op_array,zend_ssa * ssa,const zend_op * rt_opline)1263 static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline)
1264 {
1265 int b, i, end;
1266 zend_op *opline;
1267 zend_jit_ctx ctx;
1268 zend_jit_ctx *jit = &ctx;
1269 zend_jit_reg_var *ra = NULL;
1270 void *handler;
1271 int call_level = 0;
1272 void *checkpoint = NULL;
1273 bool recv_emitted = 0; /* emitted at least one RECV opcode */
1274 uint8_t smart_branch_opcode;
1275 uint32_t target_label, target_label2;
1276 uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info;
1277 zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
1278 zend_class_entry *ce;
1279 bool ce_is_instanceof;
1280 bool on_this;
1281
1282 if (JIT_G(bisect_limit)) {
1283 jit_bisect_pos++;
1284 if (jit_bisect_pos >= JIT_G(bisect_limit)) {
1285 if (jit_bisect_pos == JIT_G(bisect_limit)) {
1286 fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n",
1287 op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
1288 op_array->scope ? "::" : "",
1289 op_array->function_name ? ZSTR_VAL(op_array->function_name) : "{main}",
1290 ZSTR_VAL(op_array->filename), op_array->line_start);
1291 }
1292 return FAILURE;
1293 }
1294 }
1295
1296 if (ssa->cfg.flags & ZEND_FUNC_IRREDUCIBLE) {
1297 /* We can't order blocks properly */
1298 return FAILURE;
1299 }
1300
1301 if (rt_opline) {
1302 /* Set BB_ENTRY flag to limit register usage across the OSR ENTRY point */
1303 ssa->cfg.blocks[ssa->cfg.map[rt_opline - op_array->opcodes]].flags |= ZEND_BB_ENTRY;
1304 }
1305
1306 zend_jit_start(&ctx, op_array, ssa);
1307 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1308 checkpoint = zend_arena_checkpoint(CG(arena));
1309 zend_jit_allocate_registers(&ctx, op_array, ssa);
1310 ra = ctx.ra;
1311 }
1312
1313 /* Process blocks in Reverse Post Order */
1314 int *sorted_blocks = alloca(sizeof(int) * ssa->cfg.blocks_count);
1315 int n = zend_jit_compute_post_order(&ssa->cfg, 0, sorted_blocks);
1316
1317 while (n > 0) {
1318 b = sorted_blocks[--n];
1319 if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) {
1320 continue;
1321 }
1322
1323 if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
1324 opline = op_array->opcodes + ssa->cfg.blocks[b].start;
1325 if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) {
1326 if (opline->opcode == ZEND_RECV_INIT) {
1327 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1328 if (opline != op_array->opcodes && (opline-1)->opcode != ZEND_RECV_INIT) {
1329 zend_jit_recv_entry(&ctx, b);
1330 }
1331 } else {
1332 if (opline != op_array->opcodes && recv_emitted) {
1333 zend_jit_recv_entry(&ctx, b);
1334 }
1335 }
1336 recv_emitted = 1;
1337 } else if (opline->opcode == ZEND_RECV) {
1338 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
1339 /* skip */
1340 zend_jit_bb_start(&ctx, b);
1341 zend_jit_bb_end(&ctx, b);
1342 continue;
1343 } else if (recv_emitted) {
1344 zend_jit_recv_entry(&ctx, b);
1345 } else {
1346 recv_emitted = 1;
1347 }
1348 } else {
1349 if (recv_emitted) {
1350 zend_jit_recv_entry(&ctx, b);
1351 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
1352 ssa->cfg.blocks[b].len == 1 &&
1353 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
1354 /* don't generate code for BB with single opcode */
1355 zend_jit_free_ctx(&ctx);
1356
1357 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1358 zend_arena_release(&CG(arena), checkpoint);
1359 }
1360 return SUCCESS;
1361 }
1362 }
1363 } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
1364 ssa->cfg.blocks[b].len == 1 &&
1365 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
1366 /* don't generate code for BB with single opcode */
1367 zend_jit_free_ctx(&ctx);
1368
1369 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
1370 zend_arena_release(&CG(arena), checkpoint);
1371 }
1372 return SUCCESS;
1373 }
1374 }
1375
1376 zend_jit_bb_start(&ctx, b);
1377
1378 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ctx.ra) {
1379 zend_ssa_phi *phi = ssa->blocks[b].phis;
1380
1381 /* First try to insert IR Phi */
1382 while (phi) {
1383 zend_jit_reg_var *ival = &ctx.ra[phi->ssa_var];
1384
1385 if (ival->ref) {
1386 if (ival->flags & ZREG_PI) {
1387 zend_jit_gen_pi(jit, phi);
1388 } else if (ival->flags & ZREG_PHI) {
1389 zend_jit_gen_phi(jit, phi);
1390 }
1391 }
1392 phi = phi->next;
1393 }
1394 }
1395
1396 if (rt_opline
1397 && (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) == 0
1398 && rt_opline == op_array->opcodes + ssa->cfg.blocks[b].start) {
1399 zend_jit_osr_entry(&ctx, b); /* OSR (On-Stack-Replacement) Entry-Point */
1400 }
1401
1402 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
1403 if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
1404 && ssa->cfg.blocks[b].start != 0
1405 && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
1406 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_LONG
1407 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_STRING
1408 || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_MATCH)) {
1409 zend_jit_reset_last_valid_opline(&ctx);
1410 } else {
1411 zend_jit_set_last_valid_opline(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start);
1412 }
1413 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
1414 zend_jit_reset_last_valid_opline(&ctx);
1415 } else if (ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY) {
1416 zend_jit_reset_last_valid_opline(&ctx);
1417 } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) {
1418 zend_jit_set_last_valid_opline(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start);
1419 }
1420 if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) {
1421 if (!zend_jit_check_timeout(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start, NULL)) {
1422 goto jit_failure;
1423 }
1424 }
1425 if (!ssa->cfg.blocks[b].len) {
1426 zend_jit_bb_end(&ctx, b);
1427 continue;
1428 }
1429 if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
1430 zend_ssa_phi *phi = ssa->blocks[b].phis;
1431
1432 while (phi) {
1433 zend_jit_reg_var *ival = &ra[phi->ssa_var];
1434
1435 if (ival->ref) {
1436 if (ival->flags & ZREG_LOAD) {
1437 ZEND_ASSERT(ival->ref == IR_NULL);
1438
1439 if (!zend_jit_load_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var)) {
1440 goto jit_failure;
1441 }
1442 } else if (ival->flags & ZREG_STORE) {
1443 ZEND_ASSERT(ival->ref != IR_NULL);
1444
1445 if (!zend_jit_store_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var, 1)) {
1446 goto jit_failure;
1447 }
1448 }
1449 }
1450 phi = phi->next;
1451 }
1452 }
1453 end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
1454 for (i = ssa->cfg.blocks[b].start; i <= end; i++) {
1455 zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[i] : NULL;
1456 opline = op_array->opcodes + i;
1457 switch (opline->opcode) {
1458 case ZEND_INIT_FCALL:
1459 case ZEND_INIT_FCALL_BY_NAME:
1460 case ZEND_INIT_NS_FCALL_BY_NAME:
1461 case ZEND_INIT_METHOD_CALL:
1462 case ZEND_INIT_DYNAMIC_CALL:
1463 case ZEND_INIT_STATIC_METHOD_CALL:
1464 case ZEND_INIT_USER_CALL:
1465 case ZEND_NEW:
1466 call_level++;
1467 }
1468
1469 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
1470 switch (opline->opcode) {
1471 case ZEND_PRE_INC:
1472 case ZEND_PRE_DEC:
1473 case ZEND_POST_INC:
1474 case ZEND_POST_DEC:
1475 if (opline->op1_type != IS_CV) {
1476 break;
1477 }
1478 op1_info = OP1_INFO();
1479 if (!(op1_info & MAY_BE_LONG)) {
1480 break;
1481 }
1482 if (opline->result_type != IS_UNUSED) {
1483 res_use_info = -1;
1484
1485 if (opline->result_type == IS_CV
1486 && ssa->vars
1487 && ssa_op->result_use >= 0
1488 && !ssa->vars[ssa_op->result_use].no_val) {
1489 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1490
1491 if (Z_MODE(res_use_addr) != IS_REG
1492 || Z_LOAD(res_use_addr)
1493 || Z_STORE(res_use_addr)) {
1494 res_use_info = RES_USE_INFO();
1495 }
1496 }
1497 res_info = RES_INFO();
1498 res_addr = RES_REG_ADDR();
1499 } else {
1500 res_use_info = -1;
1501 res_info = -1;
1502 res_addr = 0;
1503 }
1504 op1_def_info = OP1_DEF_INFO();
1505 if (!zend_jit_inc_dec(&ctx, opline,
1506 op1_info, OP1_REG_ADDR(),
1507 op1_def_info, OP1_DEF_REG_ADDR(),
1508 res_use_info, res_info,
1509 res_addr,
1510 (op1_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1511 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1512 goto jit_failure;
1513 }
1514 goto done;
1515 case ZEND_BW_OR:
1516 case ZEND_BW_AND:
1517 case ZEND_BW_XOR:
1518 case ZEND_SL:
1519 case ZEND_SR:
1520 case ZEND_MOD:
1521 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1522 break;
1523 }
1524 op1_info = OP1_INFO();
1525 op2_info = OP2_INFO();
1526 if (!(op1_info & MAY_BE_LONG)
1527 || !(op2_info & MAY_BE_LONG)) {
1528 break;
1529 }
1530 res_addr = RES_REG_ADDR();
1531 if (Z_MODE(res_addr) != IS_REG
1532 && (i + 1) <= end
1533 && zend_jit_next_is_send_result(opline)) {
1534 i++;
1535 res_use_info = -1;
1536 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1537 if (!zend_jit_reuse_ip(&ctx)) {
1538 goto jit_failure;
1539 }
1540 } else {
1541 res_use_info = -1;
1542
1543 if (opline->result_type == IS_CV
1544 && ssa->vars
1545 && ssa_op->result_use >= 0
1546 && !ssa->vars[ssa_op->result_use].no_val) {
1547 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1548
1549 if (Z_MODE(res_use_addr) != IS_REG
1550 || Z_LOAD(res_use_addr)
1551 || Z_STORE(res_use_addr)) {
1552 res_use_info = RES_USE_INFO();
1553 }
1554 }
1555 }
1556 if (!zend_jit_long_math(&ctx, opline,
1557 op1_info, OP1_RANGE(), OP1_REG_ADDR(),
1558 op2_info, OP2_RANGE(), OP2_REG_ADDR(),
1559 res_use_info, RES_INFO(), res_addr,
1560 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1561 goto jit_failure;
1562 }
1563 goto done;
1564 case ZEND_ADD:
1565 case ZEND_SUB:
1566 case ZEND_MUL:
1567 // case ZEND_DIV: // TODO: check for division by zero ???
1568 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1569 break;
1570 }
1571 op1_info = OP1_INFO();
1572 op2_info = OP2_INFO();
1573 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1574 break;
1575 }
1576 if (opline->opcode == ZEND_ADD &&
1577 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
1578 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
1579 /* pass */
1580 } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
1581 !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
1582 break;
1583 }
1584 res_addr = RES_REG_ADDR();
1585 if (Z_MODE(res_addr) != IS_REG
1586 && (i + 1) <= end
1587 && zend_jit_next_is_send_result(opline)) {
1588 i++;
1589 res_use_info = -1;
1590 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1591 if (!zend_jit_reuse_ip(&ctx)) {
1592 goto jit_failure;
1593 }
1594 } else {
1595 res_use_info = -1;
1596
1597 if (opline->result_type == IS_CV
1598 && ssa->vars
1599 && ssa_op->result_use >= 0
1600 && !ssa->vars[ssa_op->result_use].no_val) {
1601 zend_jit_addr res_use_addr = RES_USE_REG_ADDR();
1602
1603 if (Z_MODE(res_use_addr) != IS_REG
1604 || Z_LOAD(res_use_addr)
1605 || Z_STORE(res_use_addr)) {
1606 res_use_info = RES_USE_INFO();
1607 }
1608 }
1609 }
1610 res_info = RES_INFO();
1611 if (opline->opcode == ZEND_ADD &&
1612 (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
1613 (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
1614 if (!zend_jit_add_arrays(&ctx, opline, op1_info, OP1_REG_ADDR(), op2_info, OP2_REG_ADDR(), res_addr)) {
1615 goto jit_failure;
1616 }
1617 } else {
1618 if (!zend_jit_math(&ctx, opline,
1619 op1_info, OP1_REG_ADDR(),
1620 op2_info, OP2_REG_ADDR(),
1621 res_use_info, res_info, res_addr,
1622 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1623 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1624 goto jit_failure;
1625 }
1626 }
1627 goto done;
1628 case ZEND_CONCAT:
1629 case ZEND_FAST_CONCAT:
1630 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1631 break;
1632 }
1633 op1_info = OP1_INFO();
1634 op2_info = OP2_INFO();
1635 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
1636 break;
1637 }
1638 if (!(op1_info & MAY_BE_STRING) ||
1639 !(op2_info & MAY_BE_STRING)) {
1640 break;
1641 }
1642 res_addr = RES_REG_ADDR();
1643 if ((i + 1) <= end
1644 && zend_jit_next_is_send_result(opline)) {
1645 i++;
1646 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1647 if (!zend_jit_reuse_ip(&ctx)) {
1648 goto jit_failure;
1649 }
1650 }
1651 if (!zend_jit_concat(&ctx, opline,
1652 op1_info, op2_info, res_addr,
1653 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1654 goto jit_failure;
1655 }
1656 goto done;
1657 case ZEND_ASSIGN_OP:
1658 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1659 break;
1660 }
1661 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1662 break;
1663 }
1664 op1_info = OP1_INFO();
1665 op2_info = OP2_INFO();
1666 if (!zend_jit_supported_binary_op(
1667 opline->extended_value, op1_info, op2_info)) {
1668 break;
1669 }
1670 op1_def_info = OP1_DEF_INFO();
1671 if (!zend_jit_assign_op(&ctx, opline,
1672 op1_info, op1_def_info, OP1_RANGE(),
1673 op2_info, OP2_RANGE(),
1674 (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
1675 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1676 goto jit_failure;
1677 }
1678 goto done;
1679 case ZEND_ASSIGN_DIM_OP:
1680 if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
1681 break;
1682 }
1683 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1684 break;
1685 }
1686 if (!zend_jit_supported_binary_op(
1687 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
1688 break;
1689 }
1690 if (!zend_jit_assign_dim_op(&ctx, opline,
1691 OP1_INFO(), OP1_DEF_INFO(), OP1_REG_ADDR(), OP2_INFO(),
1692 OP1_DATA_INFO(), OP1_DATA_RANGE(), IS_UNKNOWN,
1693 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1694 goto jit_failure;
1695 }
1696 goto done;
1697 case ZEND_ASSIGN_DIM:
1698 if (opline->op1_type != IS_CV) {
1699 break;
1700 }
1701 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1702 break;
1703 }
1704 if (!zend_jit_assign_dim(&ctx, opline,
1705 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), OP1_DATA_INFO(), IS_UNKNOWN,
1706 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1707 goto jit_failure;
1708 }
1709 goto done;
1710 case ZEND_PRE_INC_OBJ:
1711 case ZEND_PRE_DEC_OBJ:
1712 case ZEND_POST_INC_OBJ:
1713 case ZEND_POST_DEC_OBJ:
1714 if (opline->op2_type != IS_CONST
1715 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1716 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1717 break;
1718 }
1719 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1720 break;
1721 }
1722 ce = NULL;
1723 ce_is_instanceof = 0;
1724 on_this = 0;
1725 if (opline->op1_type == IS_UNUSED) {
1726 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1727 ce = op_array->scope;
1728 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
1729 op1_addr = 0;
1730 on_this = 1;
1731 } else {
1732 op1_info = OP1_INFO();
1733 if (!(op1_info & MAY_BE_OBJECT)) {
1734 break;
1735 }
1736 op1_addr = OP1_REG_ADDR();
1737 if (ssa->var_info && ssa->ops) {
1738 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1739 if (ssa_op->op1_use >= 0) {
1740 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1741 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1742 ce = op1_ssa->ce;
1743 ce_is_instanceof = op1_ssa->is_instanceof;
1744 }
1745 }
1746 }
1747 }
1748 if (!zend_jit_incdec_obj(&ctx, opline, op_array, ssa, ssa_op,
1749 op1_info, op1_addr,
1750 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
1751 goto jit_failure;
1752 }
1753 goto done;
1754 case ZEND_ASSIGN_OBJ_OP:
1755 if (opline->result_type != IS_UNUSED) {
1756 break;
1757 }
1758 if (opline->op2_type != IS_CONST
1759 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1760 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1761 break;
1762 }
1763 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1764 break;
1765 }
1766 if (!zend_jit_supported_binary_op(
1767 opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
1768 break;
1769 }
1770 ce = NULL;
1771 ce_is_instanceof = 0;
1772 on_this = 0;
1773 if (opline->op1_type == IS_UNUSED) {
1774 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1775 ce = op_array->scope;
1776 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
1777 op1_addr = 0;
1778 on_this = 1;
1779 } else {
1780 op1_info = OP1_INFO();
1781 if (!(op1_info & MAY_BE_OBJECT)) {
1782 break;
1783 }
1784 op1_addr = OP1_REG_ADDR();
1785 if (ssa->var_info && ssa->ops) {
1786 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1787 if (ssa_op->op1_use >= 0) {
1788 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1789 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1790 ce = op1_ssa->ce;
1791 ce_is_instanceof = op1_ssa->is_instanceof;
1792 }
1793 }
1794 }
1795 }
1796 if (!zend_jit_assign_obj_op(&ctx, opline, op_array, ssa, ssa_op,
1797 op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_RANGE(),
1798 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
1799 goto jit_failure;
1800 }
1801 goto done;
1802 case ZEND_ASSIGN_OBJ:
1803 if (opline->op2_type != IS_CONST
1804 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
1805 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
1806 break;
1807 }
1808 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1809 break;
1810 }
1811 ce = NULL;
1812 ce_is_instanceof = 0;
1813 on_this = 0;
1814 if (opline->op1_type == IS_UNUSED) {
1815 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
1816 ce = op_array->scope;
1817 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
1818 op1_addr = 0;
1819 on_this = 1;
1820 } else {
1821 op1_info = OP1_INFO();
1822 if (!(op1_info & MAY_BE_OBJECT)) {
1823 break;
1824 }
1825 op1_addr = OP1_REG_ADDR();
1826 if (ssa->var_info && ssa->ops) {
1827 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
1828 if (ssa_op->op1_use >= 0) {
1829 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
1830 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
1831 ce = op1_ssa->ce;
1832 ce_is_instanceof = op1_ssa->is_instanceof;
1833 }
1834 }
1835 }
1836 }
1837 if (!zend_jit_assign_obj(&ctx, opline, op_array, ssa, ssa_op,
1838 op1_info, op1_addr, OP1_DATA_INFO(),
1839 0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN,
1840 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1841 goto jit_failure;
1842 }
1843 goto done;
1844 case ZEND_ASSIGN:
1845 if (opline->op1_type != IS_CV) {
1846 break;
1847 }
1848 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
1849 break;
1850 }
1851 op2_addr = OP2_REG_ADDR();
1852 if (ra
1853 && ssa->ops[opline - op_array->opcodes].op2_def >= 0
1854 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) {
1855 op2_def_addr = OP2_DEF_REG_ADDR();
1856 } else {
1857 op2_def_addr = op2_addr;
1858 }
1859 op1_info = OP1_INFO();
1860 if (ra && ssa->vars[ssa_op->op1_use].no_val) {
1861 op1_info |= MAY_BE_UNDEF; // requres type assignment
1862 }
1863 if (opline->result_type == IS_UNUSED) {
1864 res_addr = 0;
1865 res_info = -1;
1866 } else {
1867 res_addr = RES_REG_ADDR();
1868 res_info = RES_INFO();
1869 if (Z_MODE(res_addr) != IS_REG
1870 && (i + 1) <= end
1871 && zend_jit_next_is_send_result(opline)
1872 && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) {
1873 i++;
1874 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
1875 if (!zend_jit_reuse_ip(&ctx)) {
1876 goto jit_failure;
1877 }
1878 }
1879 }
1880 if (!zend_jit_assign(&ctx, opline,
1881 op1_info, OP1_REG_ADDR(),
1882 OP1_DEF_INFO(), OP1_DEF_REG_ADDR(),
1883 OP2_INFO(), op2_addr, op2_def_addr,
1884 res_info, res_addr,
1885 0,
1886 zend_may_throw(opline, ssa_op, op_array, ssa))) {
1887 goto jit_failure;
1888 }
1889 goto done;
1890 case ZEND_QM_ASSIGN:
1891 op1_addr = OP1_REG_ADDR();
1892 if (ra
1893 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
1894 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
1895 op1_def_addr = OP1_DEF_REG_ADDR();
1896 } else {
1897 op1_def_addr = op1_addr;
1898 }
1899 if (!zend_jit_qm_assign(&ctx, opline,
1900 OP1_INFO(), op1_addr, op1_def_addr,
1901 -1, RES_INFO(), RES_REG_ADDR())) {
1902 goto jit_failure;
1903 }
1904 goto done;
1905 case ZEND_INIT_FCALL:
1906 case ZEND_INIT_FCALL_BY_NAME:
1907 case ZEND_INIT_NS_FCALL_BY_NAME:
1908 if (!zend_jit_init_fcall(&ctx, opline, b, op_array, ssa, ssa_op, call_level, NULL, 0)) {
1909 goto jit_failure;
1910 }
1911 goto done;
1912 case ZEND_SEND_VAL:
1913 case ZEND_SEND_VAL_EX:
1914 if (opline->op2_type == IS_CONST) {
1915 /* Named parameters not supported in JIT (yet) */
1916 break;
1917 }
1918 if (opline->opcode == ZEND_SEND_VAL_EX
1919 && opline->op2.num > MAX_ARG_FLAG_NUM) {
1920 break;
1921 }
1922 if (!zend_jit_send_val(&ctx, opline,
1923 OP1_INFO(), OP1_REG_ADDR())) {
1924 goto jit_failure;
1925 }
1926 goto done;
1927 case ZEND_SEND_REF:
1928 if (opline->op2_type == IS_CONST) {
1929 /* Named parameters not supported in JIT (yet) */
1930 break;
1931 }
1932 if (!zend_jit_send_ref(&ctx, opline, op_array,
1933 OP1_INFO(), 0)) {
1934 goto jit_failure;
1935 }
1936 goto done;
1937 case ZEND_SEND_VAR:
1938 case ZEND_SEND_VAR_EX:
1939 case ZEND_SEND_VAR_NO_REF:
1940 case ZEND_SEND_VAR_NO_REF_EX:
1941 case ZEND_SEND_FUNC_ARG:
1942 if (opline->op2_type == IS_CONST) {
1943 /* Named parameters not supported in JIT (yet) */
1944 break;
1945 }
1946 if ((opline->opcode == ZEND_SEND_VAR_EX
1947 || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
1948 && opline->op2.num > MAX_ARG_FLAG_NUM) {
1949 break;
1950 }
1951 op1_addr = OP1_REG_ADDR();
1952 if (ra
1953 && ssa->ops[opline - op_array->opcodes].op1_def >= 0
1954 && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
1955 op1_def_addr = OP1_DEF_REG_ADDR();
1956 } else {
1957 op1_def_addr = op1_addr;
1958 }
1959 if (!zend_jit_send_var(&ctx, opline, op_array,
1960 OP1_INFO(), op1_addr, op1_def_addr)) {
1961 goto jit_failure;
1962 }
1963 goto done;
1964 case ZEND_CHECK_FUNC_ARG:
1965 if (opline->op2_type == IS_CONST) {
1966 /* Named parameters not supported in JIT (yet) */
1967 break;
1968 }
1969 if (opline->op2.num > MAX_ARG_FLAG_NUM) {
1970 break;
1971 }
1972 if (!zend_jit_check_func_arg(&ctx, opline)) {
1973 goto jit_failure;
1974 }
1975 goto done;
1976 case ZEND_CHECK_UNDEF_ARGS:
1977 if (!zend_jit_check_undef_args(&ctx, opline)) {
1978 goto jit_failure;
1979 }
1980 goto done;
1981 case ZEND_DO_UCALL:
1982 ZEND_FALLTHROUGH;
1983 case ZEND_DO_ICALL:
1984 case ZEND_DO_FCALL_BY_NAME:
1985 case ZEND_DO_FCALL:
1986 if (!zend_jit_do_fcall(&ctx, opline, op_array, ssa, call_level, b + 1, NULL)) {
1987 goto jit_failure;
1988 }
1989 goto done;
1990 case ZEND_IS_EQUAL:
1991 case ZEND_IS_NOT_EQUAL:
1992 case ZEND_IS_SMALLER:
1993 case ZEND_IS_SMALLER_OR_EQUAL:
1994 case ZEND_CASE: {
1995 res_addr = RES_REG_ADDR();
1996 if ((opline->result_type & IS_TMP_VAR)
1997 && (i + 1) <= end
1998 && ((opline+1)->opcode == ZEND_JMPZ
1999 || (opline+1)->opcode == ZEND_JMPNZ
2000 || (opline+1)->opcode == ZEND_JMPZ_EX
2001 || (opline+1)->opcode == ZEND_JMPNZ_EX)
2002 && (opline+1)->op1_type == IS_TMP_VAR
2003 && (opline+1)->op1.var == opline->result.var) {
2004 i++;
2005 smart_branch_opcode = (opline+1)->opcode;
2006 target_label = ssa->cfg.blocks[b].successors[0];
2007 target_label2 = ssa->cfg.blocks[b].successors[1];
2008 /* For EX variant write into the result of EX opcode. */
2009 if ((opline+1)->opcode == ZEND_JMPZ_EX
2010 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
2011 res_addr = OP_REG_ADDR(opline + 1, result_type, result, result_def);
2012 }
2013 } else {
2014 smart_branch_opcode = 0;
2015 target_label = target_label2 = (uint32_t)-1;
2016 }
2017 if (!zend_jit_cmp(&ctx, opline,
2018 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
2019 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
2020 res_addr,
2021 zend_may_throw(opline, ssa_op, op_array, ssa),
2022 smart_branch_opcode, target_label, target_label2,
2023 NULL, 0)) {
2024 goto jit_failure;
2025 }
2026 goto done;
2027 }
2028 case ZEND_IS_IDENTICAL:
2029 case ZEND_IS_NOT_IDENTICAL:
2030 case ZEND_CASE_STRICT:
2031 res_addr = RES_REG_ADDR();
2032 if ((opline->result_type & IS_TMP_VAR)
2033 && (i + 1) <= end
2034 && ((opline+1)->opcode == ZEND_JMPZ
2035 || (opline+1)->opcode == ZEND_JMPZ_EX
2036 || (opline+1)->opcode == ZEND_JMPNZ_EX
2037 || (opline+1)->opcode == ZEND_JMPNZ)
2038 && (opline+1)->op1_type == IS_TMP_VAR
2039 && (opline+1)->op1.var == opline->result.var) {
2040 i++;
2041 smart_branch_opcode = (opline+1)->opcode;
2042 target_label = ssa->cfg.blocks[b].successors[0];
2043 target_label2 = ssa->cfg.blocks[b].successors[1];
2044 /* For EX variant write into the result of EX opcode. */
2045 if ((opline+1)->opcode == ZEND_JMPZ_EX
2046 || (opline+1)->opcode == ZEND_JMPNZ_EX) {
2047 res_addr = OP_REG_ADDR(opline + 1, result_type, result, result_def);
2048 }
2049 } else {
2050 smart_branch_opcode = 0;
2051 target_label = target_label2 = (uint32_t)-1;
2052 }
2053 if (!zend_jit_identical(&ctx, opline,
2054 OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
2055 OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
2056 res_addr,
2057 zend_may_throw(opline, ssa_op, op_array, ssa),
2058 smart_branch_opcode, target_label, target_label2,
2059 NULL, 0)) {
2060 goto jit_failure;
2061 }
2062 goto done;
2063 case ZEND_DEFINED:
2064 if ((opline->result_type & IS_TMP_VAR)
2065 && (i + 1) <= end
2066 && ((opline+1)->opcode == ZEND_JMPZ
2067 || (opline+1)->opcode == ZEND_JMPNZ)
2068 && (opline+1)->op1_type == IS_TMP_VAR
2069 && (opline+1)->op1.var == opline->result.var) {
2070 i++;
2071 smart_branch_opcode = (opline+1)->opcode;
2072 target_label = ssa->cfg.blocks[b].successors[0];
2073 target_label2 = ssa->cfg.blocks[b].successors[1];
2074 } else {
2075 smart_branch_opcode = 0;
2076 target_label = target_label2 = (uint32_t)-1;
2077 }
2078 if (!zend_jit_defined(&ctx, opline, smart_branch_opcode, target_label, target_label2, NULL)) {
2079 goto jit_failure;
2080 }
2081 goto done;
2082 case ZEND_TYPE_CHECK:
2083 if (opline->extended_value == MAY_BE_RESOURCE) {
2084 // TODO: support for is_resource() ???
2085 break;
2086 }
2087 if ((opline->result_type & IS_TMP_VAR)
2088 && (i + 1) <= end
2089 && ((opline+1)->opcode == ZEND_JMPZ
2090 || (opline+1)->opcode == ZEND_JMPNZ)
2091 && (opline+1)->op1_type == IS_TMP_VAR
2092 && (opline+1)->op1.var == opline->result.var) {
2093 i++;
2094 smart_branch_opcode = (opline+1)->opcode;
2095 target_label = ssa->cfg.blocks[b].successors[0];
2096 target_label2 = ssa->cfg.blocks[b].successors[1];
2097 } else {
2098 smart_branch_opcode = 0;
2099 target_label = target_label2 = (uint32_t)-1;
2100 }
2101 if (!zend_jit_type_check(&ctx, opline, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) {
2102 goto jit_failure;
2103 }
2104 goto done;
2105 case ZEND_RETURN:
2106 op1_info = OP1_INFO();
2107 if ((PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info))
2108 || op_array->type == ZEND_EVAL_CODE
2109 // TODO: support for top-level code
2110 || !op_array->function_name
2111 // TODO: support for IS_UNDEF ???
2112 || (op1_info & MAY_BE_UNDEF)) {
2113 if (!zend_jit_tail_handler(&ctx, opline)) {
2114 goto jit_failure;
2115 }
2116 } else {
2117 if (!zend_jit_return(&ctx, opline, op_array,
2118 op1_info, OP1_REG_ADDR())) {
2119 goto jit_failure;
2120 }
2121 }
2122 goto done;
2123 case ZEND_BOOL:
2124 case ZEND_BOOL_NOT:
2125 if (!zend_jit_bool_jmpznz(&ctx, opline,
2126 OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(),
2127 -1, -1,
2128 zend_may_throw(opline, ssa_op, op_array, ssa),
2129 opline->opcode, NULL)) {
2130 goto jit_failure;
2131 }
2132 goto done;
2133 case ZEND_JMPZ:
2134 case ZEND_JMPNZ:
2135 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
2136 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2137 /* smart branch */
2138 if (!zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2139 goto jit_failure;
2140 }
2141 goto done;
2142 }
2143 ZEND_FALLTHROUGH;
2144 case ZEND_JMPZ_EX:
2145 case ZEND_JMPNZ_EX:
2146 if (opline->result_type == IS_UNDEF) {
2147 res_addr = 0;
2148 } else {
2149 res_addr = RES_REG_ADDR();
2150 }
2151 if (!zend_jit_bool_jmpznz(&ctx, opline,
2152 OP1_INFO(), OP1_REG_ADDR(), res_addr,
2153 ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1],
2154 zend_may_throw(opline, ssa_op, op_array, ssa),
2155 opline->opcode, NULL)) {
2156 goto jit_failure;
2157 }
2158 goto done;
2159 case ZEND_ISSET_ISEMPTY_CV:
2160 if ((opline->extended_value & ZEND_ISEMPTY)) {
2161 // TODO: support for empty() ???
2162 break;
2163 }
2164 if ((opline->result_type & IS_TMP_VAR)
2165 && (i + 1) <= end
2166 && ((opline+1)->opcode == ZEND_JMPZ
2167 || (opline+1)->opcode == ZEND_JMPNZ)
2168 && (opline+1)->op1_type == IS_TMP_VAR
2169 && (opline+1)->op1.var == opline->result.var) {
2170 i++;
2171 smart_branch_opcode = (opline+1)->opcode;
2172 target_label = ssa->cfg.blocks[b].successors[0];
2173 target_label2 = ssa->cfg.blocks[b].successors[1];
2174 } else {
2175 smart_branch_opcode = 0;
2176 target_label = target_label2 = (uint32_t)-1;
2177 }
2178 if (!zend_jit_isset_isempty_cv(&ctx, opline,
2179 OP1_INFO(), OP1_REG_ADDR(),
2180 smart_branch_opcode, target_label, target_label2,
2181 NULL)) {
2182 goto jit_failure;
2183 }
2184 goto done;
2185 case ZEND_IN_ARRAY:
2186 if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
2187 break;
2188 }
2189 op1_info = OP1_INFO();
2190 if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
2191 break;
2192 }
2193 if ((opline->result_type & IS_TMP_VAR)
2194 && (i + 1) <= end
2195 && ((opline+1)->opcode == ZEND_JMPZ
2196 || (opline+1)->opcode == ZEND_JMPNZ)
2197 && (opline+1)->op1_type == IS_TMP_VAR
2198 && (opline+1)->op1.var == opline->result.var) {
2199 i++;
2200 smart_branch_opcode = (opline+1)->opcode;
2201 target_label = ssa->cfg.blocks[b].successors[0];
2202 target_label2 = ssa->cfg.blocks[b].successors[1];
2203 } else {
2204 smart_branch_opcode = 0;
2205 target_label = target_label2 = (uint32_t)-1;
2206 }
2207 if (!zend_jit_in_array(&ctx, opline,
2208 op1_info, OP1_REG_ADDR(),
2209 smart_branch_opcode, target_label, target_label2,
2210 NULL)) {
2211 goto jit_failure;
2212 }
2213 goto done;
2214 case ZEND_FETCH_DIM_R:
2215 case ZEND_FETCH_DIM_IS:
2216 case ZEND_FETCH_LIST_R:
2217 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2218 break;
2219 }
2220 if (!zend_jit_fetch_dim_read(&ctx, opline, ssa, ssa_op,
2221 OP1_INFO(), OP1_REG_ADDR(), 0,
2222 OP2_INFO(), RES_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
2223 goto jit_failure;
2224 }
2225 goto done;
2226 case ZEND_FETCH_DIM_W:
2227 case ZEND_FETCH_DIM_RW:
2228 // case ZEND_FETCH_DIM_UNSET:
2229 case ZEND_FETCH_LIST_W:
2230 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2231 break;
2232 }
2233 if (opline->op1_type != IS_CV) {
2234 break;
2235 }
2236 if (!zend_jit_fetch_dim(&ctx, opline,
2237 OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
2238 goto jit_failure;
2239 }
2240 goto done;
2241 case ZEND_ISSET_ISEMPTY_DIM_OBJ:
2242 if ((opline->extended_value & ZEND_ISEMPTY)) {
2243 // TODO: support for empty() ???
2244 break;
2245 }
2246 if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
2247 break;
2248 }
2249 if ((opline->result_type & IS_TMP_VAR)
2250 && (i + 1) <= end
2251 && ((opline+1)->opcode == ZEND_JMPZ
2252 || (opline+1)->opcode == ZEND_JMPNZ)
2253 && (opline+1)->op1_type == IS_TMP_VAR
2254 && (opline+1)->op1.var == opline->result.var) {
2255 i++;
2256 smart_branch_opcode = (opline+1)->opcode;
2257 target_label = ssa->cfg.blocks[b].successors[0];
2258 target_label2 = ssa->cfg.blocks[b].successors[1];
2259 } else {
2260 smart_branch_opcode = 0;
2261 target_label = target_label2 = (uint32_t)-1;
2262 }
2263 if (!zend_jit_isset_isempty_dim(&ctx, opline,
2264 OP1_INFO(), OP1_REG_ADDR(), 0,
2265 OP2_INFO(), IS_UNKNOWN,
2266 zend_may_throw(opline, ssa_op, op_array, ssa),
2267 smart_branch_opcode, target_label, target_label2,
2268 NULL)) {
2269 goto jit_failure;
2270 }
2271 goto done;
2272 case ZEND_FETCH_OBJ_R:
2273 case ZEND_FETCH_OBJ_IS:
2274 case ZEND_FETCH_OBJ_W:
2275 if (opline->op2_type != IS_CONST
2276 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
2277 || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
2278 break;
2279 }
2280 ce = NULL;
2281 ce_is_instanceof = 0;
2282 on_this = 0;
2283 if (opline->op1_type == IS_UNUSED) {
2284 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
2285 op1_addr = 0;
2286 ce = op_array->scope;
2287 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
2288 on_this = 1;
2289 } else {
2290 op1_info = OP1_INFO();
2291 if (!(op1_info & MAY_BE_OBJECT)) {
2292 break;
2293 }
2294 op1_addr = OP1_REG_ADDR();
2295 if (ssa->var_info && ssa->ops) {
2296 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
2297 if (ssa_op->op1_use >= 0) {
2298 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
2299 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
2300 ce = op1_ssa->ce;
2301 ce_is_instanceof = op1_ssa->is_instanceof;
2302 }
2303 }
2304 }
2305 }
2306 if (!zend_jit_fetch_obj(&ctx, opline, op_array, ssa, ssa_op,
2307 op1_info, op1_addr, 0, ce, ce_is_instanceof, on_this, 0, 0, NULL,
2308 IS_UNKNOWN,
2309 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2310 goto jit_failure;
2311 }
2312 goto done;
2313 case ZEND_BIND_GLOBAL:
2314 if (!ssa->ops || !ssa->var_info) {
2315 op1_info = MAY_BE_ANY|MAY_BE_REF;
2316 } else {
2317 op1_info = OP1_INFO();
2318 }
2319 if (!zend_jit_bind_global(&ctx, opline, op1_info)) {
2320 goto jit_failure;
2321 }
2322 goto done;
2323 case ZEND_RECV:
2324 if (!zend_jit_recv(&ctx, opline, op_array)) {
2325 goto jit_failure;
2326 }
2327 goto done;
2328 case ZEND_RECV_INIT:
2329 if (!zend_jit_recv_init(&ctx, opline, op_array,
2330 (opline + 1)->opcode != ZEND_RECV_INIT,
2331 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2332 goto jit_failure;
2333 }
2334 goto done;
2335 case ZEND_FREE:
2336 case ZEND_FE_FREE:
2337 if (!zend_jit_free(&ctx, opline, OP1_INFO(),
2338 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2339 goto jit_failure;
2340 }
2341 goto done;
2342 case ZEND_ECHO:
2343 op1_info = OP1_INFO();
2344 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2345 break;
2346 }
2347 if (!zend_jit_echo(&ctx, opline, op1_info)) {
2348 goto jit_failure;
2349 }
2350 goto done;
2351 case ZEND_STRLEN:
2352 op1_info = OP1_INFO();
2353 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2354 break;
2355 }
2356 if (!zend_jit_strlen(&ctx, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR())) {
2357 goto jit_failure;
2358 }
2359 goto done;
2360 case ZEND_COUNT:
2361 op1_info = OP1_INFO();
2362 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
2363 break;
2364 }
2365 if (!zend_jit_count(&ctx, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
2366 goto jit_failure;
2367 }
2368 goto done;
2369 case ZEND_FETCH_THIS:
2370 if (!zend_jit_fetch_this(&ctx, opline, op_array, 0)) {
2371 goto jit_failure;
2372 }
2373 goto done;
2374 case ZEND_SWITCH_LONG:
2375 case ZEND_SWITCH_STRING:
2376 case ZEND_MATCH:
2377 if (!zend_jit_switch(&ctx, opline, op_array, ssa, NULL, NULL)) {
2378 goto jit_failure;
2379 }
2380 goto done;
2381 case ZEND_VERIFY_RETURN_TYPE:
2382 if (opline->op1_type == IS_UNUSED) {
2383 /* Always throws */
2384 break;
2385 }
2386 if (opline->op1_type == IS_CONST) {
2387 /* TODO Different instruction format, has return value */
2388 break;
2389 }
2390 if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
2391 /* Not worth bothering with */
2392 break;
2393 }
2394 if (OP1_INFO() & MAY_BE_REF) {
2395 /* TODO May need reference unwrapping. */
2396 break;
2397 }
2398 if (!zend_jit_verify_return_type(&ctx, opline, op_array, OP1_INFO())) {
2399 goto jit_failure;
2400 }
2401 goto done;
2402 case ZEND_FE_RESET_R:
2403 op1_info = OP1_INFO();
2404 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
2405 break;
2406 }
2407 if (!zend_jit_fe_reset(&ctx, opline, op1_info)) {
2408 goto jit_failure;
2409 }
2410 goto done;
2411 case ZEND_FE_FETCH_R:
2412 op1_info = OP1_INFO();
2413 if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
2414 break;
2415 }
2416 if (!zend_jit_fe_fetch(&ctx, opline, op1_info, OP2_INFO(),
2417 ssa->cfg.blocks[b].successors[0], opline->opcode, NULL)) {
2418 goto jit_failure;
2419 }
2420 goto done;
2421 case ZEND_FETCH_CONSTANT:
2422 if (!zend_jit_fetch_constant(&ctx, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
2423 goto jit_failure;
2424 }
2425 goto done;
2426 case ZEND_INIT_METHOD_CALL:
2427 if (opline->op2_type != IS_CONST
2428 || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
2429 break;
2430 }
2431 ce = NULL;
2432 ce_is_instanceof = 0;
2433 on_this = 0;
2434 if (opline->op1_type == IS_UNUSED) {
2435 op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
2436 op1_addr = 0;
2437 ce = op_array->scope;
2438 ce_is_instanceof = (ce->ce_flags & ZEND_ACC_FINAL) != 0;
2439 on_this = 1;
2440 } else {
2441 op1_info = OP1_INFO();
2442 if (!(op1_info & MAY_BE_OBJECT)) {
2443 break;
2444 }
2445 op1_addr = OP1_REG_ADDR();
2446 if (ssa->var_info && ssa->ops) {
2447 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
2448 if (ssa_op->op1_use >= 0) {
2449 zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
2450 if (op1_ssa->ce && !op1_ssa->ce->create_object) {
2451 ce = op1_ssa->ce;
2452 ce_is_instanceof = op1_ssa->is_instanceof;
2453 }
2454 }
2455 }
2456 }
2457 if (!zend_jit_init_method_call(&ctx, opline, b, op_array, ssa, ssa_op, call_level,
2458 op1_info, op1_addr, ce, ce_is_instanceof, on_this, 0, NULL,
2459 NULL, 0,
2460 -1, -1,
2461 0)) {
2462 goto jit_failure;
2463 }
2464 goto done;
2465 case ZEND_ROPE_INIT:
2466 case ZEND_ROPE_ADD:
2467 case ZEND_ROPE_END:
2468 op2_info = OP2_INFO();
2469 if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
2470 break;
2471 }
2472 if (!zend_jit_rope(&ctx, opline, op2_info)) {
2473 goto jit_failure;
2474 }
2475 goto done;
2476 default:
2477 break;
2478 }
2479 }
2480
2481 switch (opline->opcode) {
2482 case ZEND_RECV_INIT:
2483 case ZEND_BIND_GLOBAL:
2484 if (opline == op_array->opcodes ||
2485 opline->opcode != op_array->opcodes[i-1].opcode) {
2486 /* repeatable opcodes */
2487 if (!zend_jit_handler(&ctx, opline,
2488 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2489 goto jit_failure;
2490 }
2491 }
2492 zend_jit_set_last_valid_opline(&ctx, opline+1);
2493 break;
2494 case ZEND_NOP:
2495 case ZEND_OP_DATA:
2496 case ZEND_SWITCH_LONG:
2497 case ZEND_SWITCH_STRING:
2498 case ZEND_MATCH:
2499 break;
2500 case ZEND_JMP:
2501 if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
2502 const zend_op *target = OP_JMP_ADDR(opline, opline->op1);
2503
2504 if (!zend_jit_set_ip(&ctx, target)) {
2505 goto jit_failure;
2506 }
2507 }
2508 break;
2509 case ZEND_CATCH:
2510 case ZEND_FAST_CALL:
2511 case ZEND_FAST_RET:
2512 case ZEND_GENERATOR_CREATE:
2513 case ZEND_GENERATOR_RETURN:
2514 case ZEND_RETURN_BY_REF:
2515 case ZEND_RETURN:
2516 case ZEND_EXIT:
2517 case ZEND_MATCH_ERROR:
2518 /* switch through trampoline */
2519 case ZEND_YIELD:
2520 case ZEND_YIELD_FROM:
2521 case ZEND_THROW:
2522 case ZEND_VERIFY_NEVER_TYPE:
2523 if (!zend_jit_tail_handler(&ctx, opline)) {
2524 goto jit_failure;
2525 }
2526 /* THROW and EXIT may be used in the middle of BB */
2527 /* don't generate code for the rest of BB */
2528 i = end;
2529 break;
2530 /* stackless execution */
2531 case ZEND_INCLUDE_OR_EVAL:
2532 case ZEND_DO_FCALL:
2533 case ZEND_DO_UCALL:
2534 case ZEND_DO_FCALL_BY_NAME:
2535 if (!zend_jit_call(&ctx, opline, b + 1)) {
2536 goto jit_failure;
2537 }
2538 break;
2539 case ZEND_JMPZ:
2540 case ZEND_JMPNZ:
2541 if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
2542 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2543 /* smart branch */
2544 if (!zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2545 goto jit_failure;
2546 }
2547 goto done;
2548 }
2549 ZEND_FALLTHROUGH;
2550 case ZEND_JMPZ_EX:
2551 case ZEND_JMPNZ_EX:
2552 case ZEND_JMP_SET:
2553 case ZEND_COALESCE:
2554 case ZEND_JMP_NULL:
2555 case ZEND_FE_RESET_R:
2556 case ZEND_FE_RESET_RW:
2557 case ZEND_ASSERT_CHECK:
2558 case ZEND_FE_FETCH_R:
2559 case ZEND_FE_FETCH_RW:
2560 case ZEND_BIND_INIT_STATIC_OR_JMP:
2561 if (!zend_jit_handler(&ctx, opline,
2562 zend_may_throw(opline, ssa_op, op_array, ssa)) ||
2563 !zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
2564 goto jit_failure;
2565 }
2566 break;
2567 case ZEND_JMP_FRAMELESS:
2568 if (!zend_jit_jmp_frameless(&ctx, opline, /* exit_addr */ NULL, /* guard */ 0)) {
2569 goto jit_failure;
2570 }
2571 break;
2572 case ZEND_NEW:
2573 if (!zend_jit_handler(&ctx, opline, 1)) {
2574 return 0;
2575 }
2576 if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
2577 zend_class_entry *ce = NULL;
2578
2579 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
2580 if (ssa->ops && ssa->var_info) {
2581 zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
2582 if (res_ssa->ce && !res_ssa->is_instanceof) {
2583 ce = res_ssa->ce;
2584 }
2585 }
2586 } else {
2587 if (opline->op1_type == IS_CONST) {
2588 zval *zv = RT_CONSTANT(opline, opline->op1);
2589 if (Z_TYPE_P(zv) == IS_STRING) {
2590 zval *lc = zv + 1;
2591 ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc));
2592 }
2593 }
2594 }
2595
2596 i++;
2597
2598 if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) {
2599 const zend_op *next_opline = opline + 1;
2600
2601 ZEND_ASSERT(b + 1 == ssa->cfg.blocks[b].successors[0]);
2602 zend_jit_constructor(&ctx, next_opline, op_array, ssa, call_level, b + 1);
2603 }
2604
2605 /* We skip over the DO_FCALL, so decrement call_level ourselves. */
2606 call_level--;
2607 }
2608 break;
2609 case ZEND_FRAMELESS_ICALL_0:
2610 jit_frameless_icall0(jit, opline);
2611 goto done;
2612 case ZEND_FRAMELESS_ICALL_1:
2613 op1_info = OP1_INFO();
2614 jit_frameless_icall1(jit, opline, op1_info);
2615 goto done;
2616 case ZEND_FRAMELESS_ICALL_2:
2617 op1_info = OP1_INFO();
2618 op2_info = OP2_INFO();
2619 jit_frameless_icall2(jit, opline, op1_info, op2_info);
2620 goto done;
2621 case ZEND_FRAMELESS_ICALL_3:
2622 op1_info = OP1_INFO();
2623 op2_info = OP2_INFO();
2624 jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO());
2625 goto done;
2626 default:
2627 if (!zend_jit_handler(&ctx, opline,
2628 zend_may_throw(opline, ssa_op, op_array, ssa))) {
2629 goto jit_failure;
2630 }
2631 if (i == end
2632 && (opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
2633 /* smart branch split across basic blocks */
2634 if (!zend_jit_set_cond(&ctx, opline + 2, opline->result.var)) {
2635 goto jit_failure;
2636 }
2637 }
2638 }
2639 done:
2640 switch (opline->opcode) {
2641 case ZEND_DO_FCALL:
2642 case ZEND_DO_ICALL:
2643 case ZEND_DO_UCALL:
2644 case ZEND_DO_FCALL_BY_NAME:
2645 case ZEND_CALLABLE_CONVERT:
2646 call_level--;
2647 }
2648 }
2649 zend_jit_bb_end(&ctx, b);
2650 }
2651
2652 if (jit->return_inputs) {
2653 zend_jit_common_return(jit);
2654
2655 bool left_frame = 0;
2656 if (op_array->last_var > 100) {
2657 /* To many CVs to unroll */
2658 if (!zend_jit_free_cvs(&ctx)) {
2659 goto jit_failure;
2660 }
2661 left_frame = 1;
2662 }
2663 if (!left_frame) {
2664 int j;
2665
2666 for (j = 0 ; j < op_array->last_var; j++) {
2667 uint32_t info = zend_ssa_cv_info(op_array, ssa, j);
2668
2669 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
2670 if (!left_frame) {
2671 left_frame = 1;
2672 if (!zend_jit_leave_frame(&ctx)) {
2673 goto jit_failure;
2674 }
2675 }
2676 if (!zend_jit_free_cv(&ctx, info, j)) {
2677 goto jit_failure;
2678 }
2679 }
2680 }
2681 }
2682 if (!zend_jit_leave_func(&ctx, op_array, NULL, MAY_BE_ANY, left_frame,
2683 NULL, NULL, (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, 1)) {
2684 goto jit_failure;
2685 }
2686 }
2687
2688 handler = zend_jit_finish(&ctx);
2689 if (!handler) {
2690 goto jit_failure;
2691 }
2692 zend_jit_free_ctx(&ctx);
2693
2694 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2695 zend_arena_release(&CG(arena), checkpoint);
2696 }
2697 return SUCCESS;
2698
2699 jit_failure:
2700 zend_jit_free_ctx(&ctx);
2701 if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
2702 zend_arena_release(&CG(arena), checkpoint);
2703 }
2704 return FAILURE;
2705 }
2706
zend_jit_collect_calls(zend_op_array * op_array,zend_script * script)2707 static void zend_jit_collect_calls(zend_op_array *op_array, zend_script *script)
2708 {
2709 zend_func_info *func_info;
2710
2711 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
2712 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
2713 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
2714 func_info = ZEND_FUNC_INFO(op_array);
2715 } else {
2716 func_info = zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info));
2717 ZEND_SET_FUNC_INFO(op_array, func_info);
2718 }
2719 zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, op_array, func_info);
2720 }
2721
zend_jit_cleanup_func_info(zend_op_array * op_array)2722 static void zend_jit_cleanup_func_info(zend_op_array *op_array)
2723 {
2724 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
2725 zend_call_info *caller_info, *callee_info;
2726
2727 if (func_info) {
2728 caller_info = func_info->caller_info;
2729 callee_info = func_info->callee_info;
2730
2731 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
2732 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
2733 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
2734 func_info->num = 0;
2735 func_info->flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
2736 | ZEND_FUNC_JIT_ON_PROF_REQUEST
2737 | ZEND_FUNC_JIT_ON_HOT_COUNTERS
2738 | ZEND_FUNC_JIT_ON_HOT_TRACE;
2739 memset(&func_info->ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
2740 } else {
2741 ZEND_SET_FUNC_INFO(op_array, NULL);
2742 }
2743
2744 while (caller_info) {
2745 if (caller_info->caller_op_array) {
2746 zend_jit_cleanup_func_info(caller_info->caller_op_array);
2747 }
2748 caller_info = caller_info->next_caller;
2749 }
2750 while (callee_info) {
2751 if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) {
2752 zend_jit_cleanup_func_info(&callee_info->callee_func->op_array);
2753 }
2754 callee_info = callee_info->next_callee;
2755 }
2756 }
2757 }
2758
zend_real_jit_func(zend_op_array * op_array,zend_script * script,const zend_op * rt_opline)2759 static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline)
2760 {
2761 zend_ssa ssa;
2762 void *checkpoint;
2763 zend_func_info *func_info;
2764
2765 if (*dasm_ptr == dasm_end) {
2766 return FAILURE;
2767 }
2768
2769 checkpoint = zend_arena_checkpoint(CG(arena));
2770
2771 /* Build SSA */
2772 memset(&ssa, 0, sizeof(zend_ssa));
2773
2774 if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) {
2775 goto jit_failure;
2776 }
2777
2778 if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
2779 zend_jit_collect_calls(op_array, script);
2780 func_info = ZEND_FUNC_INFO(op_array);
2781 func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array);
2782 if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
2783 zend_init_func_return_info(op_array, script, &func_info->return_info);
2784 }
2785 }
2786
2787 if (zend_jit_op_array_analyze2(op_array, script, &ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
2788 goto jit_failure;
2789 }
2790
2791 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
2792 zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa);
2793 }
2794
2795 if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) {
2796 goto jit_failure;
2797 }
2798
2799 zend_jit_cleanup_func_info(op_array);
2800 zend_arena_release(&CG(arena), checkpoint);
2801 return SUCCESS;
2802
2803 jit_failure:
2804 zend_jit_cleanup_func_info(op_array);
2805 zend_arena_release(&CG(arena), checkpoint);
2806 return FAILURE;
2807 }
2808
2809 /* Run-time JIT handler */
zend_runtime_jit(void)2810 static int ZEND_FASTCALL zend_runtime_jit(void)
2811 {
2812 zend_execute_data *execute_data = EG(current_execute_data);
2813 zend_op_array *op_array = &EX(func)->op_array;
2814 zend_op *opline = op_array->opcodes;
2815 zend_jit_op_array_extension *jit_extension;
2816 bool do_bailout = 0;
2817
2818 zend_shared_alloc_lock();
2819
2820 if (ZEND_FUNC_INFO(op_array)) {
2821
2822 SHM_UNPROTECT();
2823 zend_jit_unprotect();
2824
2825 zend_try {
2826 /* restore original opcode handlers */
2827 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2828 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
2829 opline++;
2830 }
2831 }
2832 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
2833 opline->handler = jit_extension->orig_handler;
2834
2835 /* perform real JIT for this function */
2836 zend_real_jit_func(op_array, NULL, NULL);
2837 } zend_catch {
2838 do_bailout = true;
2839 } zend_end_try();
2840
2841 zend_jit_protect();
2842 SHM_PROTECT();
2843 }
2844
2845 zend_shared_alloc_unlock();
2846
2847 if (do_bailout) {
2848 zend_bailout();
2849 }
2850
2851 /* JIT-ed code is going to be called by VM */
2852 return 0;
2853 }
2854
zend_jit_check_funcs(HashTable * function_table,bool is_method)2855 void zend_jit_check_funcs(HashTable *function_table, bool is_method) {
2856 zend_op *opline;
2857 zend_function *func;
2858 zend_op_array *op_array;
2859 uintptr_t counter;
2860 zend_jit_op_array_extension *jit_extension;
2861
2862 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(function_table, func) {
2863 if (func->type == ZEND_INTERNAL_FUNCTION) {
2864 break;
2865 }
2866 op_array = &func->op_array;
2867 opline = op_array->opcodes;
2868 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2869 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
2870 opline++;
2871 }
2872 }
2873 if (opline->handler == zend_jit_profile_jit_handler) {
2874 if (!RUN_TIME_CACHE(op_array)) {
2875 continue;
2876 }
2877 counter = (uintptr_t)ZEND_COUNTER_INFO(op_array);
2878 ZEND_COUNTER_INFO(op_array) = 0;
2879 jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
2880 opline->handler = jit_extension->orig_handler;
2881 if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) {
2882 zend_real_jit_func(op_array, NULL, NULL);
2883 }
2884 }
2885 } ZEND_HASH_FOREACH_END();
2886 }
2887
zend_jit_hot_func(zend_execute_data * execute_data,const zend_op * opline)2888 void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
2889 {
2890 zend_op_array *op_array = &EX(func)->op_array;
2891 zend_jit_op_array_hot_extension *jit_extension;
2892 uint32_t i;
2893 bool do_bailout = 0;
2894
2895 zend_shared_alloc_lock();
2896 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
2897
2898 if (jit_extension) {
2899 SHM_UNPROTECT();
2900 zend_jit_unprotect();
2901
2902 zend_try {
2903 for (i = 0; i < op_array->last; i++) {
2904 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
2905 }
2906
2907 /* perform real JIT for this function */
2908 zend_real_jit_func(op_array, NULL, opline);
2909 } zend_catch {
2910 do_bailout = 1;
2911 } zend_end_try();
2912
2913 zend_jit_protect();
2914 SHM_PROTECT();
2915 }
2916
2917 zend_shared_alloc_unlock();
2918
2919 if (do_bailout) {
2920 zend_bailout();
2921 }
2922 /* JIT-ed code is going to be called by VM */
2923 }
2924
zend_jit_setup_hot_counters_ex(zend_op_array * op_array,zend_cfg * cfg)2925 static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg)
2926 {
2927 if (JIT_G(hot_func)) {
2928 zend_op *opline = op_array->opcodes;
2929
2930 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
2931 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
2932 opline++;
2933 }
2934 }
2935
2936 opline->handler = (const void*)zend_jit_func_hot_counter_handler;
2937 }
2938
2939 if (JIT_G(hot_loop)) {
2940 uint32_t i;
2941
2942 for (i = 0; i < cfg->blocks_count; i++) {
2943 if ((cfg->blocks[i].flags & ZEND_BB_REACHABLE) &&
2944 (cfg->blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
2945 op_array->opcodes[cfg->blocks[i].start].handler =
2946 (const void*)zend_jit_loop_hot_counter_handler;
2947 }
2948 }
2949 }
2950 }
2951
zend_jit_restart_hot_counters(zend_op_array * op_array)2952 static int zend_jit_restart_hot_counters(zend_op_array *op_array)
2953 {
2954 zend_jit_op_array_hot_extension *jit_extension;
2955 zend_cfg cfg;
2956 uint32_t i;
2957
2958 jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
2959 for (i = 0; i < op_array->last; i++) {
2960 op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
2961 }
2962
2963 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
2964 return FAILURE;
2965 }
2966
2967 zend_jit_setup_hot_counters_ex(op_array, &cfg);
2968
2969 return SUCCESS;
2970 }
2971
zend_jit_setup_hot_counters(zend_op_array * op_array)2972 static int zend_jit_setup_hot_counters(zend_op_array *op_array)
2973 {
2974 zend_jit_op_array_hot_extension *jit_extension;
2975 zend_cfg cfg;
2976 uint32_t i;
2977
2978 ZEND_ASSERT(!JIT_G(hot_func) || zend_jit_func_hot_counter_handler != NULL);
2979 ZEND_ASSERT(!JIT_G(hot_loop) || zend_jit_loop_hot_counter_handler != NULL);
2980
2981 if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
2982 return FAILURE;
2983 }
2984
2985 jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*));
2986 if (!jit_extension) {
2987 return FAILURE;
2988 }
2989 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
2990 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS;
2991 jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
2992 for (i = 0; i < op_array->last; i++) {
2993 jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
2994 }
2995 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
2996
2997 zend_jit_setup_hot_counters_ex(op_array, &cfg);
2998
2999 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3000
3001 return SUCCESS;
3002 }
3003
3004 #include "jit/zend_jit_trace.c"
3005
zend_jit_op_array(zend_op_array * op_array,zend_script * script)3006 ZEND_EXT_API int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
3007 {
3008 if (dasm_ptr == NULL) {
3009 return FAILURE;
3010 }
3011
3012 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) {
3013 zend_jit_op_array_extension *jit_extension;
3014 zend_op *opline = op_array->opcodes;
3015
3016 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
3017 ZEND_SET_FUNC_INFO(op_array, NULL);
3018 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
3019 return SUCCESS;
3020 }
3021
3022 /* Set run-time JIT handler */
3023 ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL);
3024 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3025 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3026 opline++;
3027 }
3028 }
3029 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
3030 if (!jit_extension) {
3031 return FAILURE;
3032 }
3033 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3034 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC;
3035 jit_extension->orig_handler = (void*)opline->handler;
3036 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3037 opline->handler = (const void*)zend_jit_runtime_jit_handler;
3038 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3039
3040 return SUCCESS;
3041 } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) {
3042 zend_jit_op_array_extension *jit_extension;
3043 zend_op *opline = op_array->opcodes;
3044
3045 if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
3046 ZEND_SET_FUNC_INFO(op_array, NULL);
3047 zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
3048 return SUCCESS;
3049 }
3050
3051 ZEND_ASSERT(zend_jit_profile_jit_handler != NULL);
3052 if (op_array->function_name) {
3053 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3054 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3055 opline++;
3056 }
3057 }
3058 jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
3059 if (!jit_extension) {
3060 return FAILURE;
3061 }
3062 memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
3063 jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST;
3064 jit_extension->orig_handler = (void*)opline->handler;
3065 ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
3066 opline->handler = (const void*)zend_jit_profile_jit_handler;
3067 zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
3068 }
3069
3070 return SUCCESS;
3071 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
3072 return zend_jit_setup_hot_counters(op_array);
3073 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3074 return zend_jit_setup_hot_trace_counters(op_array);
3075 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3076 return zend_real_jit_func(op_array, script, NULL);
3077 } else {
3078 ZEND_UNREACHABLE();
3079 }
3080 return FAILURE;
3081 }
3082
zend_jit_script(zend_script * script)3083 ZEND_EXT_API int zend_jit_script(zend_script *script)
3084 {
3085 void *checkpoint;
3086 zend_call_graph call_graph;
3087 zend_func_info *info;
3088 int i;
3089
3090 if (dasm_ptr == NULL || *dasm_ptr == dasm_end) {
3091 return FAILURE;
3092 }
3093
3094 checkpoint = zend_arena_checkpoint(CG(arena));
3095
3096 call_graph.op_arrays_count = 0;
3097 zend_build_call_graph(&CG(arena), script, &call_graph);
3098
3099 zend_analyze_call_graph(&CG(arena), script, &call_graph);
3100
3101 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
3102 JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
3103 JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS ||
3104 JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3105 for (i = 0; i < call_graph.op_arrays_count; i++) {
3106 if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
3107 goto jit_failure;
3108 }
3109 }
3110 } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3111 for (i = 0; i < call_graph.op_arrays_count; i++) {
3112 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3113 if (info) {
3114 if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
3115 goto jit_failure;
3116 }
3117 info->ssa.cfg.flags |= info->flags;
3118 info->flags = info->ssa.cfg.flags;
3119 }
3120 }
3121
3122 for (i = 0; i < call_graph.op_arrays_count; i++) {
3123 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3124 if (info) {
3125 info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]);
3126 if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
3127 zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info);
3128 }
3129 }
3130 }
3131
3132 for (i = 0; i < call_graph.op_arrays_count; i++) {
3133 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3134 if (info) {
3135 if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
3136 goto jit_failure;
3137 }
3138 info->flags = info->ssa.cfg.flags;
3139 }
3140 }
3141
3142 for (i = 0; i < call_graph.op_arrays_count; i++) {
3143 info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
3144 if (info) {
3145 if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
3146 zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa);
3147 }
3148 if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) {
3149 goto jit_failure;
3150 }
3151 }
3152 }
3153
3154 for (i = 0; i < call_graph.op_arrays_count; i++) {
3155 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
3156 }
3157 } else {
3158 ZEND_UNREACHABLE();
3159 }
3160
3161 zend_arena_release(&CG(arena), checkpoint);
3162
3163 if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC
3164 || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST
3165 || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS
3166 || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3167 zend_class_entry *ce;
3168 zend_op_array *op_array;
3169
3170 ZEND_HASH_MAP_FOREACH_PTR(&script->class_table, ce) {
3171 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
3172 if (!ZEND_FUNC_INFO(op_array)) {
3173 void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes);
3174
3175 if (jit_extension) {
3176 ZEND_SET_FUNC_INFO(op_array, jit_extension);
3177 }
3178 }
3179 } ZEND_HASH_FOREACH_END();
3180 } ZEND_HASH_FOREACH_END();
3181 }
3182
3183 return SUCCESS;
3184
3185 jit_failure:
3186 if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
3187 for (i = 0; i < call_graph.op_arrays_count; i++) {
3188 ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
3189 }
3190 }
3191 zend_arena_release(&CG(arena), checkpoint);
3192 return FAILURE;
3193 }
3194
zend_jit_unprotect(void)3195 ZEND_EXT_API void zend_jit_unprotect(void)
3196 {
3197 #ifdef HAVE_MPROTECT
3198 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3199 int opts = PROT_READ | PROT_WRITE;
3200 #ifdef ZTS
3201 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3202 if (zend_write_protect) {
3203 pthread_jit_write_protect_np(0);
3204 }
3205 #endif
3206 opts |= PROT_EXEC;
3207 #endif
3208 if (mprotect(dasm_buf, dasm_size, opts) != 0) {
3209 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3210 }
3211 }
3212 #elif _WIN32
3213 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3214 DWORD old, new;
3215 #ifdef ZTS
3216 new = PAGE_EXECUTE_READWRITE;
3217 #else
3218 new = PAGE_READWRITE;
3219 #endif
3220 if (!VirtualProtect(dasm_buf, dasm_size, new, &old)) {
3221 DWORD err = GetLastError();
3222 char *msg = php_win32_error_to_msg(err);
3223 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3224 php_win32_error_msg_free(msg);
3225 }
3226 }
3227 #endif
3228 }
3229
zend_jit_protect(void)3230 ZEND_EXT_API void zend_jit_protect(void)
3231 {
3232 #ifdef HAVE_MPROTECT
3233 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3234 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3235 if (zend_write_protect) {
3236 pthread_jit_write_protect_np(1);
3237 }
3238 #endif
3239 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
3240 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3241 }
3242 }
3243 #elif _WIN32
3244 if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
3245 DWORD old;
3246
3247 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
3248 DWORD err = GetLastError();
3249 char *msg = php_win32_error_to_msg(err);
3250 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3251 php_win32_error_msg_free(msg);
3252 }
3253 }
3254 #endif
3255 }
3256
zend_jit_init_handlers(void)3257 static void zend_jit_init_handlers(void)
3258 {
3259 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
3260 zend_jit_runtime_jit_handler = zend_jit_stub_handlers[jit_stub_hybrid_runtime_jit];
3261 zend_jit_profile_jit_handler = zend_jit_stub_handlers[jit_stub_hybrid_profile_jit];
3262 zend_jit_func_hot_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_func_hot_counter];
3263 zend_jit_loop_hot_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_loop_hot_counter];
3264 zend_jit_func_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_func_trace_counter];
3265 zend_jit_ret_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_ret_trace_counter];
3266 zend_jit_loop_trace_counter_handler = zend_jit_stub_handlers[jit_stub_hybrid_loop_trace_counter];
3267 } else {
3268 zend_jit_runtime_jit_handler = (const void*)zend_runtime_jit;
3269 zend_jit_profile_jit_handler = (const void*)zend_jit_profile_helper;
3270 zend_jit_func_hot_counter_handler = (const void*)zend_jit_func_counter_helper;
3271 zend_jit_loop_hot_counter_handler = (const void*)zend_jit_loop_counter_helper;
3272 zend_jit_func_trace_counter_handler = (const void*)zend_jit_func_trace_helper;
3273 zend_jit_ret_trace_counter_handler = (const void*)zend_jit_ret_trace_helper;
3274 zend_jit_loop_trace_counter_handler = (const void*)zend_jit_loop_trace_helper;
3275 }
3276 }
3277
zend_jit_globals_ctor(zend_jit_globals * jit_globals)3278 static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
3279 {
3280 memset(jit_globals, 0, sizeof(zend_jit_globals));
3281 zend_jit_trace_init_caches();
3282 }
3283
3284 #ifdef ZTS
zend_jit_globals_dtor(zend_jit_globals * jit_globals)3285 static void zend_jit_globals_dtor(zend_jit_globals *jit_globals)
3286 {
3287 zend_jit_trace_free_caches(jit_globals);
3288 }
3289 #endif
3290
zend_jit_parse_config_num(zend_long jit)3291 static int zend_jit_parse_config_num(zend_long jit)
3292 {
3293 if (jit == 0) {
3294 JIT_G(on) = 0;
3295 return SUCCESS;
3296 }
3297
3298 if (jit < 0) return FAILURE;
3299
3300 if (jit % 10 == 0 || jit % 10 > 5) return FAILURE;
3301 JIT_G(opt_level) = jit % 10;
3302
3303 jit /= 10;
3304 if (jit % 10 > 5) return FAILURE;
3305 JIT_G(trigger) = jit % 10;
3306
3307 jit /= 10;
3308 if (jit % 10 > 2) return FAILURE;
3309 JIT_G(opt_flags) = jit % 10;
3310
3311 jit /= 10;
3312 if (jit % 10 > 1) return FAILURE;
3313 JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0);
3314
3315 if (jit / 10 != 0) return FAILURE;
3316
3317 JIT_G(on) = 1;
3318
3319 return SUCCESS;
3320 }
3321
zend_jit_config(zend_string * jit,int stage)3322 ZEND_EXT_API int zend_jit_config(zend_string *jit, int stage)
3323 {
3324 if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) {
3325 if (stage == ZEND_INI_STAGE_RUNTIME) {
3326 zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)");
3327 }
3328 return FAILURE;
3329 }
3330
3331 if (ZSTR_LEN(jit) == 0
3332 || zend_string_equals_literal_ci(jit, "disable")) {
3333 JIT_G(enabled) = 0;
3334 JIT_G(on) = 0;
3335 return SUCCESS;
3336 } else if (zend_string_equals_literal_ci(jit, "0")
3337 || zend_string_equals_literal_ci(jit, "off")
3338 || zend_string_equals_literal_ci(jit, "no")
3339 || zend_string_equals_literal_ci(jit, "false")) {
3340 JIT_G(enabled) = 1;
3341 JIT_G(on) = 0;
3342 return SUCCESS;
3343 } else if (zend_string_equals_literal_ci(jit, "1")
3344 || zend_string_equals_literal_ci(jit, "on")
3345 || zend_string_equals_literal_ci(jit, "yes")
3346 || zend_string_equals_literal_ci(jit, "true")
3347 || zend_string_equals_literal_ci(jit, "tracing")) {
3348 JIT_G(enabled) = 1;
3349 JIT_G(on) = 1;
3350 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS;
3351 JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
3352 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
3353 return SUCCESS;
3354 } else if (zend_string_equals_ci(jit, ZSTR_KNOWN(ZEND_STR_FUNCTION))) {
3355 JIT_G(enabled) = 1;
3356 JIT_G(on) = 1;
3357 JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT;
3358 JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD;
3359 JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
3360 return SUCCESS;
3361 } else {
3362 char *end;
3363 zend_long num = ZEND_STRTOL(ZSTR_VAL(jit), &end, 10);
3364 if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) {
3365 goto failure;
3366 }
3367 JIT_G(enabled) = 1;
3368 return SUCCESS;
3369 }
3370
3371 failure:
3372 zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number");
3373 JIT_G(enabled) = 0;
3374 JIT_G(on) = 0;
3375 return FAILURE;
3376 }
3377
zend_jit_debug_config(zend_long old_val,zend_long new_val,int stage)3378 ZEND_EXT_API int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage)
3379 {
3380 if (stage != ZEND_INI_STAGE_STARTUP) {
3381 if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) {
3382 if (stage == ZEND_INI_STAGE_RUNTIME) {
3383 zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup");
3384 }
3385 return FAILURE;
3386 }
3387 }
3388 return SUCCESS;
3389 }
3390
zend_jit_init(void)3391 ZEND_EXT_API void zend_jit_init(void)
3392 {
3393 #ifdef ZTS
3394 jit_globals_id = ts_allocate_id(&jit_globals_id, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, (ts_allocate_dtor) zend_jit_globals_dtor);
3395 #else
3396 zend_jit_globals_ctor(&jit_globals);
3397 #endif
3398 }
3399
zend_jit_check_support(void)3400 ZEND_EXT_API int zend_jit_check_support(void)
3401 {
3402 int i;
3403
3404 zend_jit_vm_kind = zend_vm_kind();
3405 if (zend_jit_vm_kind != ZEND_VM_KIND_CALL &&
3406 zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
3407 zend_error(E_WARNING, "JIT is compatible only with CALL and HYBRID VM. JIT disabled.");
3408 JIT_G(enabled) = 0;
3409 JIT_G(on) = 0;
3410 return FAILURE;
3411 }
3412
3413 if (zend_execute_ex != execute_ex) {
3414 if (strcmp(sapi_module.name, "phpdbg") != 0) {
3415 zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled.");
3416 }
3417 JIT_G(enabled) = 0;
3418 JIT_G(on) = 0;
3419 return FAILURE;
3420 }
3421
3422 for (i = 0; i <= 256; i++) {
3423 switch (i) {
3424 /* JIT has no effect on these opcodes */
3425 case ZEND_BEGIN_SILENCE:
3426 case ZEND_END_SILENCE:
3427 case ZEND_EXIT:
3428 break;
3429 default:
3430 if (zend_get_user_opcode_handler(i) != NULL) {
3431 zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled.");
3432 JIT_G(enabled) = 0;
3433 JIT_G(on) = 0;
3434 return FAILURE;
3435 }
3436 }
3437 }
3438
3439 #if defined(IR_TARGET_AARCH64)
3440 if (JIT_G(buffer_size) > 128*1024*1024) {
3441 zend_error(E_WARNING, "JIT on AArch64 doesn't support opcache.jit_buffer_size above 128M.");
3442 JIT_G(enabled) = 0;
3443 JIT_G(on) = 0;
3444 return FAILURE;
3445 }
3446 #elif defined(IR_TARGET_X64)
3447 if (JIT_G(buffer_size) > 2 * Z_L(1024*1024*1024)) {
3448 zend_error(E_WARNING, "JIT on x86_64 doesn't support opcache.jit_buffer_size above 2G.");
3449 JIT_G(enabled) = 0;
3450 JIT_G(on) = 0;
3451 return FAILURE;
3452 }
3453 #endif
3454
3455 return SUCCESS;
3456 }
3457
zend_jit_startup(void * buf,size_t size,bool reattached)3458 ZEND_EXT_API void zend_jit_startup(void *buf, size_t size, bool reattached)
3459 {
3460 zend_jit_halt_op = zend_get_halt_op();
3461 zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(ACCELERATOR_PRODUCT_NAME);
3462
3463 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3464 zend_write_protect = pthread_jit_write_protect_supported_np();
3465 #endif
3466
3467 dasm_buf = buf;
3468 dasm_size = size;
3469 dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr) * 2);
3470
3471 #ifdef HAVE_MPROTECT
3472 #ifdef HAVE_PTHREAD_JIT_WRITE_PROTECT_NP
3473 if (zend_write_protect) {
3474 pthread_jit_write_protect_np(1);
3475 }
3476 #endif
3477 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
3478 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
3479 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3480 }
3481 } else {
3482 if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
3483 fprintf(stderr, "mprotect() failed [%d] %s\n", errno, strerror(errno));
3484 }
3485 }
3486 #elif _WIN32
3487 if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
3488 DWORD old;
3489
3490 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
3491 DWORD err = GetLastError();
3492 char *msg = php_win32_error_to_msg(err);
3493 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3494 php_win32_error_msg_free(msg);
3495 }
3496 } else {
3497 DWORD old;
3498
3499 if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
3500 DWORD err = GetLastError();
3501 char *msg = php_win32_error_to_msg(err);
3502 fprintf(stderr, "VirtualProtect() failed [%u] %s\n", err, msg);
3503 php_win32_error_msg_free(msg);
3504 }
3505 }
3506 #endif
3507
3508 if (!reattached) {
3509 zend_jit_unprotect();
3510 *dasm_ptr = dasm_buf;
3511 #if defined(_WIN32)
3512 zend_jit_stub_handlers = dasm_buf;
3513 *dasm_ptr = (void**)*dasm_ptr + sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
3514 #elif defined(IR_TARGET_AARCH64)
3515 zend_jit_stub_handlers = dasm_buf;
3516 *dasm_ptr = (void**)*dasm_ptr + (sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])) * 2;
3517 memset(zend_jit_stub_handlers, 0, (sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])) * 2 * sizeof(void*));
3518 #endif
3519 *dasm_ptr = (void*)ZEND_MM_ALIGNED_SIZE_EX(((size_t)(*dasm_ptr)), 16);
3520 zend_jit_protect();
3521 } else {
3522 #if defined(_WIN32) || defined(IR_TARGET_AARCH64)
3523 zend_jit_stub_handlers = dasm_buf;
3524 zend_jit_init_handlers();
3525 #endif
3526 }
3527
3528 zend_jit_unprotect();
3529 zend_jit_setup();
3530 zend_jit_protect();
3531 zend_jit_init_handlers();
3532
3533 zend_jit_trace_startup(reattached);
3534
3535 zend_jit_unprotect();
3536 /* save JIT buffer pos */
3537 dasm_ptr[1] = dasm_ptr[0];
3538 zend_jit_protect();
3539 }
3540
zend_jit_shutdown(void)3541 ZEND_EXT_API void zend_jit_shutdown(void)
3542 {
3543 if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE && dasm_ptr != NULL) {
3544 fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf));
3545 }
3546
3547 zend_jit_shutdown_ir();
3548
3549 #ifdef ZTS
3550 ts_free_id(jit_globals_id);
3551 #else
3552 zend_jit_trace_free_caches(&jit_globals);
3553 #endif
3554 }
3555
zend_jit_reset_counters(void)3556 static void zend_jit_reset_counters(void)
3557 {
3558 int i;
3559
3560 for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
3561 zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT;
3562 }
3563 }
3564
zend_jit_activate(void)3565 ZEND_EXT_API void zend_jit_activate(void)
3566 {
3567 zend_jit_profile_counter = 0;
3568 if (JIT_G(on)) {
3569 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
3570 zend_jit_reset_counters();
3571 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
3572 zend_jit_reset_counters();
3573 zend_jit_trace_reset_caches();
3574 }
3575 }
3576 }
3577
zend_jit_deactivate(void)3578 ZEND_EXT_API void zend_jit_deactivate(void)
3579 {
3580 if (zend_jit_profile_counter && !CG(unclean_shutdown)) {
3581 zend_class_entry *ce;
3582
3583 zend_shared_alloc_lock();
3584 SHM_UNPROTECT();
3585 zend_jit_unprotect();
3586
3587 zend_jit_check_funcs(EG(function_table), 0);
3588 ZEND_HASH_MAP_REVERSE_FOREACH_PTR(EG(class_table), ce) {
3589 if (ce->type == ZEND_INTERNAL_CLASS) {
3590 break;
3591 }
3592 zend_jit_check_funcs(&ce->function_table, 1);
3593 } ZEND_HASH_FOREACH_END();
3594
3595 zend_jit_protect();
3596 SHM_PROTECT();
3597 zend_shared_alloc_unlock();
3598 }
3599
3600 zend_jit_profile_counter = 0;
3601 }
3602
zend_jit_restart_preloaded_op_array(zend_op_array * op_array)3603 static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array)
3604 {
3605 zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
3606
3607 if (!func_info) {
3608 return;
3609 }
3610
3611 if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
3612 zend_jit_restart_hot_trace_counters(op_array);
3613 } else if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_COUNTERS) {
3614 zend_jit_restart_hot_counters(op_array);
3615 #if 0
3616 // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ???
3617 } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) {
3618 zend_op *opline = op_array->opcodes;
3619 zend_jit_op_array_extension *jit_extension =
3620 (zend_jit_op_array_extension*)func_info;
3621
3622 if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
3623 while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
3624 opline++;
3625 }
3626 }
3627 if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) {
3628 opline->handler = (const void*)zend_jit_runtime_jit_handler;
3629 } else {
3630 opline->handler = (const void*)zend_jit_profile_jit_handler;
3631 }
3632 #endif
3633 }
3634 if (op_array->num_dynamic_func_defs) {
3635 for (uint32_t i = 0; i < op_array->num_dynamic_func_defs; i++) {
3636 zend_jit_restart_preloaded_op_array(op_array->dynamic_func_defs[i]);
3637 }
3638 }
3639 }
3640
zend_jit_restart_preloaded_script(zend_persistent_script * script)3641 static void zend_jit_restart_preloaded_script(zend_persistent_script *script)
3642 {
3643 zend_class_entry *ce;
3644 zend_op_array *op_array;
3645
3646 zend_jit_restart_preloaded_op_array(&script->script.main_op_array);
3647
3648 ZEND_HASH_MAP_FOREACH_PTR(&script->script.function_table, op_array) {
3649 zend_jit_restart_preloaded_op_array(op_array);
3650 } ZEND_HASH_FOREACH_END();
3651
3652 ZEND_HASH_MAP_FOREACH_PTR(&script->script.class_table, ce) {
3653 ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
3654 if (op_array->type == ZEND_USER_FUNCTION) {
3655 zend_jit_restart_preloaded_op_array(op_array);
3656 }
3657 } ZEND_HASH_FOREACH_END();
3658 } ZEND_HASH_FOREACH_END();
3659 }
3660
zend_jit_restart(void)3661 ZEND_EXT_API void zend_jit_restart(void)
3662 {
3663 if (dasm_buf) {
3664 zend_jit_unprotect();
3665
3666 /* restore JIT buffer pos */
3667 dasm_ptr[0] = dasm_ptr[1];
3668
3669 zend_jit_trace_restart();
3670
3671 if (ZCSG(preload_script)) {
3672 zend_jit_restart_preloaded_script(ZCSG(preload_script));
3673 if (ZCSG(saved_scripts)) {
3674 zend_persistent_script **p = ZCSG(saved_scripts);
3675
3676 while (*p) {
3677 zend_jit_restart_preloaded_script(*p);
3678 p++;
3679 }
3680 }
3681 }
3682
3683 zend_jit_protect();
3684 }
3685 }
3686
3687 #endif /* HAVE_JIT */
3688
