1/* 2 * +----------------------------------------------------------------------+ 3 * | Zend JIT | 4 * +----------------------------------------------------------------------+ 5 * | Copyright (c) The PHP Group | 6 * +----------------------------------------------------------------------+ 7 * | This source file is subject to version 3.01 of the PHP license, | 8 * | that is bundled with this package in the file LICENSE, and is | 9 * | available through the world-wide-web at the following url: | 10 * | https://www.php.net/license/3_01.txt | 11 * | If you did not receive a copy of the PHP license and are unable to | 12 * | obtain it through the world-wide-web, please send a note to | 13 * | license@php.net so we can mail you a copy immediately. | 14 * +----------------------------------------------------------------------+ 15 * | Authors: Dmitry Stogov <dmitry@php.net> | 16 * | Xinchen Hui <laruence@php.net> | 17 * | Hao Sun <hao.sun@arm.com> | 18 * +----------------------------------------------------------------------+ 19 */ 20 21|.arch arm64 22 23|.define FP, x27 24|.define IP, x28 25|.define IPl, w28 26|.define RX, x28 // the same as VM IP reused as a general purpose reg 27|.define LR, x30 28|.define CARG1, x0 29|.define CARG2, x1 30|.define CARG3, x2 31|.define CARG4, x3 32|.define CARG5, x4 33|.define CARG6, x5 34|.define CARG1w, w0 35|.define CARG2w, w1 36|.define CARG3w, w2 37|.define CARG4w, w3 38|.define CARG5w, w4 39|.define CARG6w, w5 40|.define RETVALx, x0 41|.define RETVALw, w0 42|.define FCARG1x, x0 43|.define FCARG1w, w0 44|.define FCARG2x, x1 45|.define FCARG2w, w1 46|.define SPAD, 0x20 // padding for CPU stack alignment 47|.define NR_SPAD, 0x30 // padding for CPU stack alignment 48|.define T3, [sp, #0x28] // Used to store old value of IP (CALL VM only) 49|.define T2, [sp, #0x20] // Used to store old value of FP (CALL VM only) 50|.define T1, [sp, #0x10] 51 52// We use REG0/1/2 and FPR0/1 to replace r0/1/2 and xmm0/1 in the x86 implementation. 53// Scratch registers 54|.define REG0, x8 55|.define REG0w, w8 56|.define REG1, x9 57|.define REG1w, w9 58|.define REG2, x10 59|.define REG2w, w10 60|.define FPR0, d0 61|.define FPR1, d1 62 63|.define ZREG_REG0, ZREG_X8 64|.define ZREG_REG1, ZREG_X9 65|.define ZREG_REG2, ZREG_X10 66|.define ZREG_FPR0, ZREG_V0 67|.define ZREG_FPR1, ZREG_V1 68 69// Temporaries, not preserved across calls 70|.define TMP1, x15 71|.define TMP1w, w15 72|.define TMP2, x16 73|.define TMP2w, w16 74|.define TMP3, x17 // TODO: remember about hard-coded: mrs TMP3, tpidr_el0 75|.define TMP3w, w17 76|.define FPTMP, d16 77 78|.define ZREG_TMP1, ZREG_X15 79|.define ZREG_TMP2, ZREG_X16 80|.define ZREG_TMP3, ZREG_X17 81|.define ZREG_FPTMP, ZREG_V16 82 83|.define HYBRID_SPAD, 32 // padding for stack alignment 84 85#define TMP_ZVAL_OFFSET 16 86#define DASM_ALIGNMENT 16 87 88const char* zend_reg_name[] = { 89 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", 90 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", 91 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", 92 "x24", "x25", "x26", "x27", "x28", "x29", "x30", "sp", 93 "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", 94 "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15", 95 "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23", 96 "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31" 97}; 98 99#define ZREG_FCARG1 ZREG_X0 100#define ZREG_FCARG2 ZREG_X1 101 102|.type EX, zend_execute_data, FP 103|.type OP, zend_op 104|.type ZVAL, zval 105|.actionlist dasm_actions 106|.globals zend_lb 107|.section code, cold_code, jmp_table 108 109static void* dasm_labels[zend_lb_MAX]; 110 111#if ZTS 112static size_t tsrm_ls_cache_tcb_offset = 0; 113# ifdef __APPLE__ 114struct TLVDescriptor { 115 void* (*thunk)(struct TLVDescriptor*); 116 uint64_t key; 117 uint64_t offset; 118}; 119typedef struct TLVDescriptor TLVDescriptor; 120# endif 121#endif 122 123#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) 124 125/* Encoding of immediate. */ 126#define MAX_IMM12 0xfff // maximum value for imm12 127#define MAX_IMM16 0xffff // maximum value for imm16 128#define MOVZ_IMM MAX_IMM16 // movz insn 129#define LDR_STR_PIMM64 (MAX_IMM12*8) // ldr/str insn for 64-bit register: pimm is imm12 * 8 130#define LDR_STR_PIMM32 (MAX_IMM12*4) // ldr/str insn for 32-bit register: pimm is imm12 * 4 131#define LDRB_STRB_PIMM MAX_IMM12 // ldrb/strb insn 132 133#define B_IMM (1<<27) // signed imm26 * 4 134#define ADR_IMM (1<<20) // signed imm21 135#define ADRP_IMM (1LL<<32) // signed imm21 * 4096 136 137static bool arm64_may_use_b(const void *addr) 138{ 139 if (addr >= dasm_buf && addr < dasm_end) { 140 return (((char*)dasm_end - (char*)dasm_buf) < B_IMM); 141 } else if (addr >= dasm_end) { 142 return (((char*)addr - (char*)dasm_buf) < B_IMM); 143 } else if (addr < dasm_buf) { 144 return (((char*)dasm_end - (char*)addr) < B_IMM); 145 } 146 return 0; 147} 148 149static bool arm64_may_use_adr(const void *addr) 150{ 151 if (addr >= dasm_buf && addr < dasm_end) { 152 return (((char*)dasm_end - (char*)dasm_buf) < ADR_IMM); 153 } else if (addr >= dasm_end) { 154 return (((char*)addr - (char*)dasm_buf) < ADR_IMM); 155 } else if (addr < dasm_buf) { 156 return (((char*)dasm_end - (char*)addr) < ADR_IMM); 157 } 158 return 0; 159} 160 161static bool arm64_may_use_adrp(const void *addr) 162{ 163 if (addr >= dasm_buf && addr < dasm_end) { 164 return (((char*)dasm_end - (char*)dasm_buf) < ADRP_IMM); 165 } else if (addr >= dasm_end) { 166 return (((char*)addr - (char*)dasm_buf) < ADRP_IMM); 167 } else if (addr < dasm_buf) { 168 return (((char*)dasm_end - (char*)addr) < ADRP_IMM); 169 } 170 return 0; 171} 172 173/* Determine whether "val" falls into two allowed ranges: 174 * Range 1: [0, 0xfff] 175 * Range 2: LSL #12 to Range 1 176 * Used to guard the immediate encoding for add/adds/sub/subs/cmp/cmn instructions. */ 177static bool arm64_may_encode_imm12(const int64_t val) 178{ 179 return (val >= 0 && (val <= MAX_IMM12 || !(val & 0xffffffffff000fff))); 180} 181 182/* Determine whether an immediate value can be encoded as the immediate operand of logical instructions. */ 183static bool logical_immediate_p(uint64_t value, uint32_t reg_size) 184{ 185 /* fast path: power of two */ 186 if (value > 0 && !(value & (value - 1))) { 187 return true; 188 } 189 190 if (reg_size == 32) { 191 if (dasm_imm13((uint32_t)value, (uint32_t)value) != -1) { 192 return true; 193 } 194 } else if (reg_size == 64) { 195 if (dasm_imm13((uint32_t)value, (uint32_t)(value >> 32)) != -1) { 196 return true; 197 } 198 } else { 199 ZEND_UNREACHABLE(); 200 } 201 202 return false; 203} 204 205/* Not Implemented Yet */ 206|.macro NIY 207|| //ZEND_ASSERT(0); 208| brk #0 209|.endmacro 210 211|.macro NIY_STUB 212|| //ZEND_ASSERT(0); 213| brk #0 214|.endmacro 215 216|.macro ADD_HYBRID_SPAD 217||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 218| add sp, sp, # HYBRID_SPAD 219||#endif 220|.endmacro 221 222|.macro SUB_HYBRID_SPAD 223||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 224| sub sp, sp, # HYBRID_SPAD 225||#endif 226|.endmacro 227 228/* Move address into register. TODO: Support 52-bit address */ 229|.macro LOAD_ADDR, reg, addr 230| // 48-bit virtual address 231|| if (((uintptr_t)(addr)) == 0) { 232| mov reg, xzr 233|| } else if (((uintptr_t)(addr)) <= MOVZ_IMM) { 234| movz reg, #((uint64_t)(addr)) 235|| } else if (arm64_may_use_adr((void*)(addr))) { 236| adr reg, &addr 237|| } else if (arm64_may_use_adrp((void*)(addr))) { 238| adrp reg, &(((uintptr_t)(addr))) 239|| if (((uintptr_t)(addr)) & 0xfff) { 240| add reg, reg, #(((uintptr_t)(addr)) & 0xfff) 241|| } 242|| } else if ((uintptr_t)(addr) & 0xffff) { 243| movz reg, #((uintptr_t)(addr) & 0xffff) 244|| if (((uintptr_t)(addr) >> 16) & 0xffff) { 245| movk reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 246|| } 247|| if (((uintptr_t)(addr) >> 32) & 0xffff) { 248| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 249|| } 250|| } else if (((uintptr_t)(addr) >> 16) & 0xffff) { 251| movz reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 252|| if (((uintptr_t)(addr) >> 32) & 0xffff) { 253| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 254|| } 255|| } else { 256| movz reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 257|| } 258|.endmacro 259 260/* Move 32-bit immediate value into register. */ 261|.macro LOAD_32BIT_VAL, reg, val 262|| if (((uint32_t)(val)) <= MOVZ_IMM) { 263| movz reg, #((uint32_t)(val)) 264|| } else if (((uint32_t)(val) & 0xffff)) { 265| movz reg, #((uint32_t)(val) & 0xffff) 266|| if ((((uint32_t)(val) >> 16) & 0xffff)) { 267| movk reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 268|| } 269|| } else { 270| movz reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 271|| } 272|.endmacro 273 274/* Move 64-bit immediate value into register. */ 275|.macro LOAD_64BIT_VAL, reg, val 276|| if (((uint64_t)(val)) == 0) { 277| mov reg, xzr 278|| } else if (((uint64_t)(val)) <= MOVZ_IMM) { 279| movz reg, #((uint64_t)(val)) 280|| } else if (~((uint64_t)(val)) <= MOVZ_IMM) { 281| movn reg, #(~((uint64_t)(val))) 282|| } else if ((uint64_t)(val) & 0xffff) { 283| movz reg, #((uint64_t)(val) & 0xffff) 284|| if (((uint64_t)(val) >> 16) & 0xffff) { 285| movk reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 286|| } 287|| if (((uint64_t)(val) >> 32) & 0xffff) { 288| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 289|| } 290|| if ((((uint64_t)(val) >> 48) & 0xffff)) { 291| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 292|| } 293|| } else if (((uint64_t)(val) >> 16) & 0xffff) { 294| movz reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 295|| if (((uint64_t)(val) >> 32) & 0xffff) { 296| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 297|| } 298|| if ((((uint64_t)(val) >> 48) & 0xffff)) { 299| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 300|| } 301|| } else if (((uint64_t)(val) >> 32) & 0xffff) { 302| movz reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 303|| if ((((uint64_t)(val) >> 48) & 0xffff)) { 304| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 305|| } 306|| } else { 307| movz reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 308|| } 309|.endmacro 310 311/* Extract the low 8 bits from 'src_reg' into 'dst_reg'. 312 * Note: 0xff can be encoded as imm for 'and' instruction. */ 313|.macro GET_LOW_8BITS, dst_reg, src_reg 314| and dst_reg, src_reg, #0xff 315|.endmacro 316 317/* Bitwise operation with immediate. 'bw_ins' can be and/orr/eor/ands. 318 * 32-bit and 64-bit registers are distinguished. */ 319|.macro BW_OP_32_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg 320|| if (val == 0) { 321| bw_ins dst_reg, src_reg1, wzr 322|| } else if (logical_immediate_p((uint32_t)val, 32)) { 323| bw_ins dst_reg, src_reg1, #val 324|| } else { 325| LOAD_32BIT_VAL tmp_reg, val 326| bw_ins dst_reg, src_reg1, tmp_reg 327|| } 328|.endmacro 329 330|.macro BW_OP_64_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg 331|| if (val == 0) { 332| bw_ins dst_reg, src_reg1, xzr 333|| } else if (logical_immediate_p(val, 64)) { 334| bw_ins dst_reg, src_reg1, #val 335|| } else { 336| LOAD_64BIT_VAL tmp_reg, val 337| bw_ins dst_reg, src_reg1, tmp_reg 338|| } 339|.endmacro 340 341/* Test bits 'tst' with immediate. 32-bit and 64-bit registers are distinguished. */ 342|.macro TST_32_WITH_CONST, reg, val, tmp_reg 343|| if (val == 0) { 344| tst reg, wzr 345|| } else if (logical_immediate_p((uint32_t)val, 32)) { 346| tst reg, #val 347|| } else { 348| LOAD_32BIT_VAL tmp_reg, val 349| tst reg, tmp_reg 350|| } 351|.endmacro 352 353|.macro TST_64_WITH_CONST, reg, val, tmp_reg 354|| if (val == 0) { 355| tst reg, xzr 356|| } else if (logical_immediate_p(val, 64)) { 357| tst reg, #val 358|| } else { 359| LOAD_64BIT_VAL tmp_reg, val 360| tst reg, tmp_reg 361|| } 362|.endmacro 363 364/* Test bits between 64-bit register with constant 1. */ 365|.macro TST_64_WITH_ONE, reg 366| tst reg, #1 367|.endmacro 368 369/* Compare a register value with immediate. 32-bit and 64-bit registers are distinguished. 370 * Note: Comparing 64-bit register with 32-bit immediate is handled in CMP_64_WITH_CONST_32. */ 371|.macro CMP_32_WITH_CONST, reg, val, tmp_reg 372|| if (val == 0) { 373| cmp reg, wzr 374|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 375| cmp reg, #val 376|| } else if (arm64_may_encode_imm12((int64_t)(-val))) { 377| cmn reg, #-val 378|| } else { 379| LOAD_32BIT_VAL tmp_reg, val 380| cmp reg, tmp_reg 381|| } 382|.endmacro 383 384|.macro CMP_64_WITH_CONST_32, reg, val, tmp_reg 385|| if (val == 0) { 386| cmp reg, xzr 387|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 388| cmp reg, #val 389|| } else if (arm64_may_encode_imm12((int64_t)(-val))) { 390| cmn reg, #-val 391|| } else { 392| LOAD_32BIT_VAL tmp_reg, val 393| cmp reg, tmp_reg 394|| } 395|.endmacro 396 397|.macro CMP_64_WITH_CONST, reg, val, tmp_reg 398|| if (val == 0) { 399| cmp reg, xzr 400|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 401| cmp reg, #val 402|| } else if (arm64_may_encode_imm12((int64_t)(-val))) { 403| cmn reg, #-val 404|| } else { 405| LOAD_64BIT_VAL tmp_reg, val 406| cmp reg, tmp_reg 407|| } 408|.endmacro 409 410/* Add/sub a register value with immediate. 'add_sub_ins' can be add/sub/adds/subs. 411 * 32-bit and 64-bit registers are distinguished. 412 * Note: Case of 64-bit register with 32-bit immediate is handled in ADD_SUB_64_WITH_CONST_32. */ 413|.macro ADD_SUB_32_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg 414|| if (val == 0) { 415| add_sub_ins dst_reg, src_reg1, wzr 416|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 417| add_sub_ins dst_reg, src_reg1, #val 418|| } else { 419| LOAD_32BIT_VAL tmp_reg, val 420| add_sub_ins dst_reg, src_reg1, tmp_reg 421|| } 422|.endmacro 423 424|.macro ADD_SUB_64_WITH_CONST_32, add_sub_ins, dst_reg, src_reg1, val, tmp_reg 425|| if (val == 0) { 426| add_sub_ins dst_reg, src_reg1, xzr 427|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 428| add_sub_ins dst_reg, src_reg1, #val 429|| } else { 430| LOAD_32BIT_VAL tmp_reg, val 431| add_sub_ins dst_reg, src_reg1, tmp_reg 432|| } 433|.endmacro 434 435|.macro ADD_SUB_64_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg 436|| if (val == 0) { 437| add_sub_ins dst_reg, src_reg1, xzr 438|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 439| add_sub_ins dst_reg, src_reg1, #val 440|| } else { 441| LOAD_64BIT_VAL tmp_reg, val 442| add_sub_ins dst_reg, src_reg1, tmp_reg 443|| } 444|.endmacro 445 446/* Memory access(load/store) with 32-bit 'offset'. 447 * Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register. 448 * 8-bit, 32-bit and 64-bit registers to be transferred are distinguished. 449 * Note: 'reg' can be used as 'tmp_reg' if 1) 'reg' is one GPR, AND 2) 'reg' != 'base_reg', AND 3) ins is 'ldr'. */ 450|.macro MEM_ACCESS_64_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg 451|| if (((uintptr_t)(offset)) > LDR_STR_PIMM64) { 452| LOAD_32BIT_VAL tmp_reg, offset 453| ldr_str_ins reg, [base_reg, tmp_reg] 454|| } else { 455| ldr_str_ins reg, [base_reg, #(offset)] 456|| } 457|.endmacro 458 459|.macro MEM_ACCESS_32_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg 460|| if (((uintptr_t)(offset)) > LDR_STR_PIMM32) { 461| LOAD_32BIT_VAL tmp_reg, offset 462| ldr_str_ins reg, [base_reg, tmp_reg] 463|| } else { 464| ldr_str_ins reg, [base_reg, #(offset)] 465|| } 466|.endmacro 467 468|.macro MEM_ACCESS_8_WITH_UOFFSET, ldrb_strb_ins, reg, base_reg, offset, tmp_reg 469|| if (((uintptr_t)(offset)) > LDRB_STRB_PIMM) { 470| LOAD_32BIT_VAL tmp_reg, offset 471| ldrb_strb_ins reg, [base_reg, tmp_reg] 472|| } else { 473| ldrb_strb_ins reg, [base_reg, #(offset)] 474|| } 475|.endmacro 476 477/* Memory access(load/store) with 64-bit 'offset'. 478 * Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register. */ 479|.macro MEM_ACCESS_64_WITH_UOFFSET_64, ldr_str_ins, reg, base_reg, offset, tmp_reg 480|| if (((uintptr_t)(offset)) > LDR_STR_PIMM64) { 481| LOAD_64BIT_VAL tmp_reg, offset 482| ldr_str_ins reg, [base_reg, tmp_reg] 483|| } else { 484| ldr_str_ins reg, [base_reg, #(offset)] 485|| } 486|.endmacro 487 488/* ZTS: get thread local variable "_tsrm_ls_cache" */ 489|.macro LOAD_TSRM_CACHE, reg 490||#ifdef __APPLE__ 491| .long 0xd53bd071 // TODO: hard-coded: mrs TMP3, tpidrro_el0 492| and TMP3, TMP3, #0xfffffffffffffff8 493| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, TMP3, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->key << 3), TMP1 494| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, reg, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->offset), TMP1 495||#else 496| .long 0xd53bd051 // TODO: hard-coded: mrs TMP3, tpidr_el0 497|| ZEND_ASSERT(tsrm_ls_cache_tcb_offset <= LDR_STR_PIMM64); 498| ldr reg, [TMP3, #tsrm_ls_cache_tcb_offset] 499||#endif 500|.endmacro 501 502|.macro LOAD_ADDR_ZTS, reg, struct, field 503| .if ZTS 504| LOAD_TSRM_CACHE TMP3 505| ADD_SUB_64_WITH_CONST_32 add, reg, TMP3, (struct.._offset + offsetof(zend_..struct, field)), reg 506| .else 507| LOAD_ADDR reg, &struct.field 508| .endif 509|.endmacro 510 511/* Store address 'addr' into memory 'mem'. */ 512|.macro ADDR_STORE, mem, addr, tmp_reg 513| LOAD_ADDR tmp_reg, addr 514| str tmp_reg, mem 515|.endmacro 516 517/* Store a register value 'reg' into memory 'addr'. 518 * For ZTS mode, base register with unsigned offset variant is used, 519 * and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */ 520|.macro MEM_STORE, str_ins, reg, addr, tmp_reg 521|| if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) { 522| adr tmp_reg, &addr 523| str_ins reg, [tmp_reg] 524|| } else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) { 525| adrp tmp_reg, &(((uintptr_t)(addr))) 526| str_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)] 527|| } else { 528| LOAD_ADDR tmp_reg, addr 529| str_ins reg, [tmp_reg] 530|| } 531|.endmacro 532 533|.macro MEM_STORE_64_ZTS, str_ins, reg, struct, field, tmp_reg 534| .if ZTS 535| LOAD_TSRM_CACHE TMP3 536| MEM_ACCESS_64_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 537| .else 538| MEM_STORE str_ins, reg, &struct.field, tmp_reg 539| .endif 540|.endmacro 541 542|.macro MEM_STORE_32_ZTS, str_ins, reg, struct, field, tmp_reg 543| .if ZTS 544| LOAD_TSRM_CACHE TMP3 545| MEM_ACCESS_32_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 546| .else 547| MEM_STORE str_ins, reg, &struct.field, tmp_reg 548| .endif 549|.endmacro 550 551|.macro MEM_STORE_8_ZTS, strb_ins, reg, struct, field, tmp_reg 552| .if ZTS 553| LOAD_TSRM_CACHE TMP3 554| MEM_ACCESS_8_WITH_UOFFSET strb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 555| .else 556| MEM_STORE strb_ins, reg, &struct.field, tmp_reg 557| .endif 558|.endmacro 559 560/* Load value from memory 'addr' and write it into register 'reg'. 561 * For ZTS mode, base register with unsigned offset variant is used, 562 * and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */ 563|.macro MEM_LOAD, ldr_ins, reg, addr, tmp_reg 564|| if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) { 565| adr tmp_reg, &addr 566| ldr_ins reg, [tmp_reg] 567|| } else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) { 568| adrp tmp_reg, &(((uintptr_t)(addr))) 569| ldr_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)] 570|| } else { 571| LOAD_ADDR tmp_reg, addr 572| ldr_ins reg, [tmp_reg] 573|| } 574|.endmacro 575 576|.macro MEM_LOAD_64_ZTS, ldr_ins, reg, struct, field, tmp_reg 577| .if ZTS 578| LOAD_TSRM_CACHE TMP3 579| MEM_ACCESS_64_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 580| .else 581| MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg 582| .endif 583|.endmacro 584 585|.macro MEM_LOAD_32_ZTS, ldr_ins, reg, struct, field, tmp_reg 586| .if ZTS 587| LOAD_TSRM_CACHE TMP3 588| MEM_ACCESS_32_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 589| .else 590| MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg 591| .endif 592|.endmacro 593 594|.macro MEM_LOAD_8_ZTS, ldrb_ins, reg, struct, field, tmp_reg 595| .if ZTS 596| LOAD_TSRM_CACHE TMP3 597| MEM_ACCESS_8_WITH_UOFFSET ldrb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 598| .else 599| MEM_LOAD ldrb_ins, reg, &struct.field, tmp_reg 600| .endif 601|.endmacro 602 603/* Conduct arithmetic operation between the value in memory 'addr' and register value in 'reg', 604 * and the computation result is stored back in 'reg'. 'op_ins' can be add/sub. */ 605|.macro MEM_LOAD_OP, op_ins, ldr_ins, reg, addr, tmp_reg1, tmp_reg2 606| MEM_LOAD ldr_ins, tmp_reg1, addr, tmp_reg2 607| op_ins reg, reg, tmp_reg1 608|.endmacro 609 610|.macro MEM_LOAD_OP_ZTS, op_ins, ldr_ins, reg, struct, field, tmp_reg1, tmp_reg2 611| .if ZTS 612| LOAD_TSRM_CACHE TMP3 613| MEM_ACCESS_64_WITH_UOFFSET ldr_ins, tmp_reg2, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg1 614| op_ins reg, reg, tmp_reg2 615| .else 616| MEM_LOAD_OP op_ins, ldr_ins, reg, &struct.field, tmp_reg1, tmp_reg2 617| .endif 618|.endmacro 619 620/* Conduct arithmetic operation between the value in memory 'addr' and operand 'op', and the computation 621 * result is stored back to memory 'addr'. Operand 'op' can be either a register value or an immediate value. 622 * Currently, only add instruction is used as 'op_ins'. 623 * Note: It should be guaranteed that the immediate value can be encoded for 'op_ins'. */ 624|.macro MEM_UPDATE, op_ins, ldr_ins, str_ins, op, addr, tmp_reg1, tmp_reg2 625| LOAD_ADDR tmp_reg2, addr 626| ldr_ins, tmp_reg1, [tmp_reg2] 627| op_ins tmp_reg1, tmp_reg1, op 628| str_ins tmp_reg1, [tmp_reg2] 629|.endmacro 630 631|.macro MEM_UPDATE_ZTS, op_ins, ldr_ins, str_ins, op, struct, field, tmp_reg1, tmp_reg2 632| .if ZTS 633| LOAD_TSRM_CACHE TMP3 634|| if (((uintptr_t)(struct.._offset+offsetof(zend_..struct, field))) > LDR_STR_PIMM64) { 635| LOAD_32BIT_VAL tmp_reg1, (struct.._offset+offsetof(zend_..struct, field)) 636| ldr_ins tmp_reg2, [TMP3, tmp_reg1] 637| op_ins tmp_reg2, tmp_reg2, op 638| str_ins tmp_reg2, [TMP3, tmp_reg1] 639|| } else { 640| ldr_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] 641| op_ins tmp_reg2, tmp_reg2, op 642| str_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] 643|| } 644| .else 645| MEM_UPDATE op_ins, ldr_ins, str_ins, op, &struct.field, tmp_reg1, tmp_reg2 646| .endif 647|.endmacro 648 649|.macro EXT_CALL, func, tmp_reg 650|| if (arm64_may_use_b(func)) { 651| bl &func 652|| } else { 653| LOAD_ADDR tmp_reg, func 654| blr tmp_reg 655|| } 656|.endmacro 657 658|.macro EXT_JMP, func, tmp_reg 659|| if (arm64_may_use_b(func)) { 660| b &func 661|| } else { 662| LOAD_ADDR tmp_reg, func 663| br tmp_reg 664|| } 665|.endmacro 666 667|.macro SAVE_IP 668|| if (GCC_GLOBAL_REGS) { 669| str IP, EX->opline 670|| } 671|.endmacro 672 673|.macro LOAD_IP 674|| if (GCC_GLOBAL_REGS) { 675| ldr IP, EX->opline 676|| } 677|.endmacro 678 679|.macro LOAD_IP_ADDR, addr 680|| if (GCC_GLOBAL_REGS) { 681| LOAD_ADDR IP, addr 682|| } else { 683| ADDR_STORE EX->opline, addr, RX 684|| } 685|.endmacro 686 687|.macro LOAD_IP_ADDR_ZTS, struct, field, tmp_reg 688| .if ZTS 689|| if (GCC_GLOBAL_REGS) { 690| LOAD_TSRM_CACHE IP 691| MEM_ACCESS_64_WITH_UOFFSET ldr, IP, IP, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 692|| } else { 693| LOAD_TSRM_CACHE RX 694| ADD_SUB_64_WITH_CONST_32 add, RX, RX, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 695| str RX, EX->opline 696|| } 697| .else 698| LOAD_IP_ADDR &struct.field 699| .endif 700|.endmacro 701 702|.macro GET_IP, reg 703|| if (GCC_GLOBAL_REGS) { 704| mov reg, IP 705|| } else { 706| ldr reg, EX->opline 707|| } 708|.endmacro 709 710/* Update IP with register 'reg'. Note: shift variant is handled by ADD_IP_SHIFT. */ 711|.macro ADD_IP, reg, tmp_reg 712|| if (GCC_GLOBAL_REGS) { 713| add IP, IP, reg 714|| } else { 715| ldr tmp_reg, EX->opline 716| add tmp_reg, tmp_reg, reg 717| str tmp_reg, EX->opline 718|| } 719|.endmacro 720 721|.macro ADD_IP_SHIFT, reg, shift, tmp_reg 722|| if (GCC_GLOBAL_REGS) { 723| add IP, IP, reg, shift 724|| } else { 725| ldr tmp_reg, EX->opline 726| add tmp_reg, tmp_reg, reg, shift 727| str tmp_reg, EX->opline 728|| } 729|.endmacro 730 731/* Update IP with 32-bit immediate 'val'. */ 732|.macro ADD_IP_WITH_CONST, val, tmp_reg 733|| ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val))); 734|| if (GCC_GLOBAL_REGS) { 735| add IP, IP, #val 736|| } else { 737| ldr tmp_reg, EX->opline 738| add tmp_reg, tmp_reg, #val 739| str tmp_reg, EX->opline 740|| } 741|.endmacro 742 743|.macro JMP_IP, tmp_reg 744|| if (GCC_GLOBAL_REGS) { 745| ldr tmp_reg, [IP] 746| br tmp_reg 747|| } else { 748| ldr tmp_reg, EX:CARG1->opline 749| br tmp_reg 750|| } 751|.endmacro 752 753|.macro CMP_IP, addr, tmp_reg1, tmp_reg2 754| LOAD_ADDR tmp_reg1, addr 755|| if (GCC_GLOBAL_REGS) { 756| cmp IP, tmp_reg1 757|| } else { 758| ldr tmp_reg2, EX->opline 759| cmp tmp_reg2, tmp_reg1 760|| } 761|.endmacro 762 763|.macro LOAD_ZVAL_ADDR, reg, addr 764|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 765| LOAD_ADDR reg, Z_ZV(addr) 766|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 767|| if (Z_OFFSET(addr)) { 768| ADD_SUB_64_WITH_CONST_32 add, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), reg 769|| } else { 770|| if (Z_REG(addr) == ZREG_RSP) { 771| mov reg, sp 772|| } else { 773| mov reg, Rx(Z_REG(addr)) 774|| } 775|| } 776|| } else { 777|| ZEND_UNREACHABLE(); 778|| } 779|.endmacro 780 781|.macro GET_Z_TYPE_INFO, reg, zv 782| ldr reg, [zv, #offsetof(zval,u1.type_info)] 783|.endmacro 784 785|.macro SET_Z_TYPE_INFO, zv, type, tmp_reg 786| LOAD_32BIT_VAL tmp_reg, type 787| str tmp_reg, [zv, #offsetof(zval,u1.type_info)] 788|.endmacro 789 790|.macro GET_ZVAL_TYPE, reg, addr, tmp_reg 791|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 792| MEM_ACCESS_8_WITH_UOFFSET ldrb, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.v.type), tmp_reg 793|.endmacro 794 795|.macro GET_ZVAL_TYPE_INFO, reg, addr, tmp_reg 796|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 797| MEM_ACCESS_32_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg 798|.endmacro 799 800|.macro SET_ZVAL_TYPE_INFO, addr, type, tmp_reg1, tmp_reg2 801|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 802| LOAD_32BIT_VAL tmp_reg1, type 803| MEM_ACCESS_32_WITH_UOFFSET str, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg2 804|.endmacro 805 806|.macro SET_ZVAL_TYPE_INFO_FROM_REG, addr, reg, tmp_reg 807|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 808| MEM_ACCESS_32_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg 809|.endmacro 810 811|.macro GET_Z_PTR, reg, zv 812| ldr reg, [zv] 813|.endmacro 814 815|.macro GET_ZVAL_PTR, reg, addr, tmp_reg 816|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 817| MEM_ACCESS_64_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 818|.endmacro 819 820|.macro SET_ZVAL_PTR, addr, reg, tmp_reg 821|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 822| MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 823|.endmacro 824 825|.macro UNDEF_OPLINE_RESULT, tmp_reg 826| ldr REG0, EX->opline 827| ldr REG0w, OP:REG0->result.var 828| add REG0, FP, REG0 829| SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg 830|.endmacro 831 832|.macro UNDEF_OPLINE_RESULT_IF_USED, tmp_reg1, tmp_reg2 833| ldrb tmp_reg1, OP:RX->result_type 834| TST_32_WITH_CONST tmp_reg1, (IS_TMP_VAR|IS_VAR), tmp_reg2 835| beq >1 836| ldr REG0w, OP:RX->result.var 837| add REG0, FP, REG0 838| SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg1 839|1: 840|.endmacro 841 842/* Floating-point comparison between register 'reg' and value from memory 'addr'. 843 * Note: the equivalent macros in JIT/x86 are SSE_AVX_OP and SSE_OP. */ 844|.macro DOUBLE_CMP, reg, addr, tmp_reg, fp_tmp_reg 845|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 846| MEM_LOAD ldr, Rd(fp_tmp_reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg) 847| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) 848|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 849| MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) 850| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) 851|| } else if (Z_MODE(addr) == IS_REG) { 852| fcmp Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) 853|| } else { 854|| ZEND_UNREACHABLE(); 855|| } 856|.endmacro 857 858/* Convert LONG value 'val' into DOUBLE type, and move it into FP register 'reg'. 859 * Note: the equivalent macro in JIT/x86 is SSE_GET_LONG. */ 860|.macro DOUBLE_GET_LONG, reg, val, tmp_reg 861|| if (val == 0) { 862| fmov Rd(reg-ZREG_V0), xzr // TODO: "movi d0, #0" is not recognized by DynASM/arm64 863|| } else { 864| LOAD_64BIT_VAL Rx(tmp_reg), val 865| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg) 866|| } 867|.endmacro 868 869/* Convert LONG value from memory 'addr' into DOUBLE type, and move it into FP register 'reg'. 870 * Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_LVAL. */ 871|.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg1, tmp_reg2 872|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 873| DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg1 874|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 875| MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg2) 876| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg1) 877|| } else if (Z_MODE(addr) == IS_REG) { 878| scvtf Rd(reg-ZREG_V0), Rx(Z_REG(addr)) 879|| } else { 880|| ZEND_UNREACHABLE(); 881|| } 882|.endmacro 883 884/* Floating-point arithmetic operation between two FP registers. 885 * Note: the equivalent macro in JIT/x86 is AVX_MATH_REG. */ 886|.macro DOUBLE_MATH_REG, opcode, dst_reg, op1_reg, op2_reg 887|| switch (opcode) { 888|| case ZEND_ADD: 889| fadd Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 890|| break; 891|| case ZEND_SUB: 892| fsub Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 893|| break; 894|| case ZEND_MUL: 895| fmul Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 896|| break; 897|| case ZEND_DIV: 898| fdiv Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 899|| break; 900|| } 901|.endmacro 902 903/* Conduct binary operation between register 'reg' and value from memory 'addr', 904 * and the computation result is stored in 'reg'. 905 * For LONG_ADD_SUB, 'add_sub_ins' can be adds/subs. For LONG_BW_OP, 'bw_ins' can be and/orr/eor. 906 * For LONG_CMP, 'cmp' instruction is used by default and only flag registers are affected. 907 * Note: the equivalent macro in JIT/x86 is LONG_OP. */ 908|.macro LONG_ADD_SUB, add_sub_ins, reg, addr, tmp_reg 909|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 910| ADD_SUB_64_WITH_CONST add_sub_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg 911|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 912| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 913| add_sub_ins Rx(reg), Rx(reg), tmp_reg 914|| } else if (Z_MODE(addr) == IS_REG) { 915| add_sub_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) 916|| } else { 917|| ZEND_UNREACHABLE(); 918|| } 919|.endmacro 920 921|.macro LONG_BW_OP, bw_ins, reg, addr, tmp_reg 922|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 923| BW_OP_64_WITH_CONST bw_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg 924|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 925| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 926| bw_ins Rx(reg), Rx(reg), tmp_reg 927|| } else if (Z_MODE(addr) == IS_REG) { 928| bw_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) 929|| } else { 930|| ZEND_UNREACHABLE(); 931|| } 932|.endmacro 933 934|.macro LONG_CMP, reg, addr, tmp_reg 935|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 936| CMP_64_WITH_CONST Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg 937|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 938| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 939| cmp Rx(reg), tmp_reg 940|| } else if (Z_MODE(addr) == IS_REG) { 941| cmp Rx(reg), Rx(Z_REG(addr)) 942|| } else { 943|| ZEND_UNREACHABLE(); 944|| } 945|.endmacro 946 947/* Conduct add/sub between value from memory 'addr' and an immediate value 'val', and 948 * the computation result is stored back into 'addr'. 949 * Note: it should be guaranteed that 'val' can be encoded into add/sub instruction. */ 950|.macro LONG_ADD_SUB_WITH_IMM, add_sub_ins, addr, val, tmp_reg1, tmp_reg2 951|| ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val))); 952|| if (Z_MODE(addr) == IS_MEM_ZVAL) { 953|| if (((uint32_t)(Z_OFFSET(addr))) > LDR_STR_PIMM64) { 954| LOAD_32BIT_VAL tmp_reg2, Z_OFFSET(addr) 955| ldr tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2] 956| add_sub_ins tmp_reg1, tmp_reg1, #val 957| str tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2] 958|| } else { 959| ldr tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)] 960| add_sub_ins tmp_reg1, tmp_reg1, #val 961| str tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)] 962|| } 963|| } else if (Z_MODE(addr) == IS_REG) { 964| add_sub_ins Rx(Z_REG(addr)), Rx(Z_REG(addr)), #val 965|| } else { 966|| ZEND_UNREACHABLE(); 967|| } 968|.endmacro 969 970/* Compare value from memory 'addr' with immediate value 'val'. 971 * Note: the equivalent macro in JIT/x86 is LONG_OP_WITH_CONST. */ 972|.macro LONG_CMP_WITH_CONST, addr, val, tmp_reg1, tmp_reg2 973|| if (Z_MODE(addr) == IS_MEM_ZVAL) { 974| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 975| CMP_64_WITH_CONST tmp_reg1, val, tmp_reg2 976|| } else if (Z_MODE(addr) == IS_REG) { 977| CMP_64_WITH_CONST Rx(Z_REG(addr)), val, tmp_reg1 978|| } else { 979|| ZEND_UNREACHABLE(); 980|| } 981|.endmacro 982 983|.macro GET_ZVAL_LVAL, reg, addr, tmp_reg 984|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 985|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { 986| mov Rx(reg), xzr 987|| } else { 988| LOAD_64BIT_VAL Rx(reg), Z_LVAL_P(Z_ZV(addr)) 989|| } 990|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 991| MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(reg), Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 992|| } else if (Z_MODE(addr) == IS_REG) { 993|| if (reg != Z_REG(addr)) { 994| mov Rx(reg), Rx(Z_REG(addr)) 995|| } 996|| } else { 997|| ZEND_UNREACHABLE(); 998|| } 999|.endmacro 1000 1001|.macro LONG_MATH, opcode, reg, addr, tmp_reg1 1002|| switch (opcode) { 1003|| case ZEND_ADD: 1004| LONG_ADD_SUB adds, reg, addr, tmp_reg1 1005|| break; 1006|| case ZEND_SUB: 1007| LONG_ADD_SUB subs, reg, addr, tmp_reg1 1008|| break; 1009|| case ZEND_BW_OR: 1010| LONG_BW_OP orr, reg, addr, tmp_reg1 1011|| break; 1012|| case ZEND_BW_AND: 1013| LONG_BW_OP and, reg, addr, tmp_reg1 1014|| break; 1015|| case ZEND_BW_XOR: 1016| LONG_BW_OP eor, reg, addr, tmp_reg1 1017|| break; 1018|| default: 1019|| ZEND_UNREACHABLE(); 1020|| } 1021|.endmacro 1022 1023|.macro LONG_MATH_REG, opcode, dst_reg, src_reg1, src_reg2 1024|| switch (opcode) { 1025|| case ZEND_ADD: 1026| adds dst_reg, src_reg1, src_reg2 1027|| break; 1028|| case ZEND_SUB: 1029| subs dst_reg, src_reg1, src_reg2 1030|| break; 1031|| case ZEND_BW_OR: 1032| orr dst_reg, src_reg1, src_reg2 1033|| break; 1034|| case ZEND_BW_AND: 1035| and dst_reg, src_reg1, src_reg2 1036|| break; 1037|| case ZEND_BW_XOR: 1038| eor dst_reg, src_reg1, src_reg2 1039|| break; 1040|| default: 1041|| ZEND_UNREACHABLE(); 1042|| } 1043|.endmacro 1044 1045/* Store LONG value into memory 'addr'. 1046 * This LONG value can be an immediate value i.e. 'val' in macro SET_ZVAL_LVAL, or 1047 * a register value i.e. 'reg' in macro SET_ZVAL_LVAL_FROM_REG. */ 1048|.macro SET_ZVAL_LVAL_FROM_REG, addr, reg, tmp_reg 1049|| if (Z_MODE(addr) == IS_REG) { 1050| mov Rx(Z_REG(addr)), reg 1051|| } else { 1052|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1053| MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 1054|| } 1055|.endmacro 1056 1057|.macro SET_ZVAL_LVAL, addr, val, tmp_reg1, tmp_reg2 1058|| if (val == 0) { 1059| SET_ZVAL_LVAL_FROM_REG addr, xzr, tmp_reg2 1060|| } else { 1061| LOAD_64BIT_VAL tmp_reg1, val 1062| SET_ZVAL_LVAL_FROM_REG addr, tmp_reg1, tmp_reg2 1063|| } 1064|.endmacro 1065 1066/* Store DOUBLE value from FP register 'reg' into memory 'addr'. 1067 * Note: the equivalent macro in JIT/x86 is SSE_SET_ZVAL_DVAL. */ 1068|.macro SET_ZVAL_DVAL, addr, reg, tmp_reg 1069|| if (Z_MODE(addr) == IS_REG) { 1070|| if (reg != Z_REG(addr)) { 1071| fmov Rd(Z_REG(addr)-ZREG_V0), Rd(reg-ZREG_V0) 1072|| } 1073|| } else { 1074|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1075| MEM_ACCESS_64_WITH_UOFFSET str, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) 1076|| } 1077|.endmacro 1078 1079/* Load DOUBLE value from memory 'addr' into FP register 'reg'. 1080 * Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_DVAL. */ 1081|.macro GET_ZVAL_DVAL, reg, addr, tmp_reg 1082|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { 1083|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 1084| MEM_LOAD ldr, Rd(reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg) 1085|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 1086| MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) 1087|| } else if (Z_MODE(addr) == IS_REG) { 1088| fmov Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) 1089|| } else { 1090|| ZEND_UNREACHABLE(); 1091|| } 1092|| } 1093|.endmacro 1094 1095|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg 1096|| if (Z_TYPE_P(zv) > IS_TRUE) { 1097|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 1098|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; 1099| MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1) 1100| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 1101|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 1102|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; 1103| DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), tmp_reg1 1104| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 1105|| } else { 1106| // In x64, if the range of this LONG value can be represented via INT type, only move the low 32 bits into dst_addr. 1107| // Note that imm32 is signed extended to 64 bits during mov. 1108| // In aarch64, we choose to handle both cases in the same way. Even though 4 mov's are used for 64-bit value and 2 mov's are 1109| // needed for 32-bit value, an extra ext insn is needed for 32-bit vlaue. 1110| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1111|| } 1112|| } 1113|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1114|| if (dst_def_info == MAY_BE_DOUBLE) { 1115|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1116| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) 1117|| } 1118|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1119| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2) 1120|| } 1121|| } 1122|.endmacro 1123 1124|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg 1125|| if (Z_TYPE_P(zv) > IS_TRUE) { 1126|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 1127|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? 1128|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : fp_tmp_reg); 1129| MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1) 1130| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 1131| SET_ZVAL_DVAL res_addr, dst_reg, tmp_reg2 1132|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 1133|| if (Z_MODE(dst_addr) == IS_REG) { 1134| DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), tmp_reg1 1135| SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg2 1136|| } else if (Z_MODE(res_addr) == IS_REG) { 1137| DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), tmp_reg1 1138| SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg2 1139|| } else { 1140| DOUBLE_GET_LONG fp_tmp_reg, Z_LVAL_P(zv), tmp_reg1 1141| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg2 1142| SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg2 1143|| } 1144|| } else { 1145|| if (Z_MODE(dst_addr) == IS_REG) { 1146| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1147| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg1) 1148|| } else if (Z_MODE(res_addr) == IS_REG) { 1149| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1150| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg1) 1151|| } else { 1152| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1153| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1154|| } 1155|| } 1156|| } 1157|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1158|| if (dst_def_info == MAY_BE_DOUBLE) { 1159|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1160| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) 1161|| } 1162|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1163| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2) 1164|| } 1165|| } 1166|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1167|| if (dst_def_info == MAY_BE_DOUBLE) { 1168| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) 1169|| } else { 1170| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2) 1171|| } 1172|| } 1173|.endmacro 1174 1175// the same as above, but "src" may overlap with "reg1" 1176// Useful info would be stored into reg1 and reg2, and they might be used afterward. 1177|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg1, tmp_reg2, fp_tmp_reg 1178| ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg1, fp_tmp_reg 1179|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1180|| !(src_info & MAY_BE_GUARD) && 1181|| has_concrete_type(src_info & MAY_BE_ANY)) { 1182|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1183|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) { 1184|| uint32_t type = concrete_type(src_info); 1185| SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg1), Rx(tmp_reg2) 1186|| } 1187|| } 1188|| } else { 1189| GET_ZVAL_TYPE_INFO Rw(reg1), src_addr, Rx(tmp_reg1) 1190| SET_ZVAL_TYPE_INFO_FROM_REG dst_addr, Rw(reg1), Rx(tmp_reg1) 1191|| } 1192|.endmacro 1193 1194|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg, fp_tmp_reg 1195|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1196|| if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) { 1197|| if (Z_MODE(src_addr) == IS_REG) { 1198|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1199| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) 1200|| } 1201|| } else if (Z_MODE(dst_addr) == IS_REG) { 1202| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) 1203|| } else { 1204| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) 1205| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg) 1206|| } 1207|| } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 1208|| if (Z_MODE(src_addr) == IS_REG) { 1209| SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg 1210|| } else if (Z_MODE(dst_addr) == IS_REG) { 1211| GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg 1212|| } else { 1213| GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg 1214| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg 1215|| } 1216|| // Combine the following two branches. 1217|| // } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) { 1218|| } else { 1219| GET_ZVAL_PTR Rx(reg2), src_addr, Rx(tmp_reg) 1220| SET_ZVAL_PTR dst_addr, Rx(reg2), Rx(tmp_reg) 1221|| } 1222|| } 1223|.endmacro 1224 1225|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, reg1, reg2, tmp_reg, tmp_reg2, fp_tmp_reg 1226|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1227|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) { 1228|| if (Z_MODE(src_addr) == IS_REG) { 1229|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1230| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) 1231|| } 1232|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) { 1233| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) 1234|| } 1235|| } else if (Z_MODE(dst_addr) == IS_REG) { 1236| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) 1237|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) { 1238| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg) 1239|| } 1240|| } else if (Z_MODE(res_addr) == IS_REG) { 1241| GET_ZVAL_LVAL Z_REG(res_addr), src_addr, Rx(tmp_reg) 1242| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg) 1243|| } else { 1244| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) 1245| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg) 1246| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(reg2), Rx(tmp_reg) 1247|| } 1248|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 1249|| if (Z_MODE(src_addr) == IS_REG) { 1250| SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg 1251| SET_ZVAL_DVAL res_addr, Z_REG(src_addr), tmp_reg 1252|| } else if (Z_MODE(dst_addr) == IS_REG) { 1253| GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg 1254| SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg 1255|| } else if (Z_MODE(res_addr) == IS_REG) { 1256| GET_ZVAL_DVAL Z_REG(res_addr), src_addr, tmp_reg 1257| SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg 1258|| } else { 1259| GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg 1260| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg 1261| SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg 1262|| } 1263|| } else { 1264| GET_ZVAL_PTR Rx(reg2), src_addr, Rx(tmp_reg) 1265| SET_ZVAL_PTR dst_addr, Rx(reg2), Rx(tmp_reg) 1266| SET_ZVAL_PTR res_addr, Rx(reg2), Rx(tmp_reg) 1267|| } 1268|| } 1269|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1270|| has_concrete_type(src_info & MAY_BE_ANY)) { 1271|| uint32_t type = concrete_type(src_info); 1272|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1273|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { 1274| SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg), Rx(tmp_reg2) 1275|| } 1276|| } 1277|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1278| SET_ZVAL_TYPE_INFO res_addr, type, Rw(tmp_reg), Rx(tmp_reg2) 1279|| } 1280|| } else { 1281| GET_ZVAL_TYPE_INFO Rw(reg1), src_addr, Rx(tmp_reg) 1282| SET_ZVAL_TYPE_INFO_FROM_REG dst_addr, Rw(reg1), Rx(tmp_reg) 1283| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, Rw(reg1), Rx(tmp_reg) 1284|| } 1285|.endmacro 1286 1287|.macro IF_UNDEF, type_reg, label 1288| cbz type_reg, label 1289|.endmacro 1290 1291|.macro IF_TYPE, type, val, label 1292|| if (val == 0) { 1293| cbz type, label 1294|| } else { 1295| cmp type, #val 1296| beq label 1297|| } 1298|.endmacro 1299 1300|.macro IF_NOT_TYPE, type, val, label 1301|| if (val == 0) { 1302| cbnz type, label 1303|| } else { 1304| cmp type, #val 1305| bne label 1306|| } 1307|.endmacro 1308 1309|.macro IF_Z_TYPE, zv, val, label, tmp_reg 1310| ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)] 1311| IF_TYPE tmp_reg, val, label 1312|.endmacro 1313 1314|.macro IF_NOT_Z_TYPE, zv, val, label, tmp_reg 1315| ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)] 1316| IF_NOT_TYPE tmp_reg, val, label 1317|.endmacro 1318 1319|.macro CMP_ZVAL_TYPE, addr, val, tmp_reg 1320|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1321| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) 1322| cmp Rw(tmp_reg), #val 1323|.endmacro 1324 1325|.macro IF_ZVAL_TYPE, addr, val, label, tmp_reg 1326|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1327| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) 1328| IF_TYPE Rw(tmp_reg), val, label 1329|.endmacro 1330 1331|.macro IF_NOT_ZVAL_TYPE, addr, val, label, tmp_reg 1332|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1333| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) 1334| IF_NOT_TYPE Rw(tmp_reg), val, label 1335|.endmacro 1336 1337|.macro IF_FLAGS, type_flags, mask, label, tmp_reg 1338| TST_32_WITH_CONST type_flags, mask, tmp_reg 1339| bne label 1340|.endmacro 1341 1342|.macro IF_NOT_FLAGS, type_flags, mask, label, tmp_reg 1343| TST_32_WITH_CONST type_flags, mask, tmp_reg 1344| beq label 1345|.endmacro 1346 1347|.macro IF_REFCOUNTED, type_flags, label, tmp_reg 1348| TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg 1349| bne label 1350|.endmacro 1351 1352|.macro IF_NOT_REFCOUNTED, type_flags, label, tmp_reg 1353| TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg 1354| beq label 1355|.endmacro 1356 1357|.macro IF_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 1358|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1359| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2) 1360| IF_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2) 1361|.endmacro 1362 1363|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 1364|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1365| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2) 1366| IF_NOT_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2) 1367|.endmacro 1368 1369|.macro IF_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2 1370| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2 1371|.endmacro 1372 1373|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2 1374| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2 1375|.endmacro 1376 1377|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label, tmp_reg1, tmp_reg2 1378| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label, tmp_reg1, tmp_reg2 1379|.endmacro 1380 1381|.macro GC_ADDREF, zv, tmp_reg 1382| ldr tmp_reg, [zv] 1383| add tmp_reg, tmp_reg, #1 1384| str tmp_reg, [zv] 1385|.endmacro 1386 1387|.macro GC_ADDREF_2, zv, tmp_reg 1388| ldr tmp_reg, [zv] 1389| add tmp_reg, tmp_reg, #2 1390| str tmp_reg, [zv] 1391|.endmacro 1392 1393|.macro GC_DELREF, zv, tmp_reg 1394| ldr tmp_reg, [zv] 1395| subs tmp_reg, tmp_reg, #1 1396| str tmp_reg, [zv] 1397|.endmacro 1398 1399|.macro IF_GC_MAY_NOT_LEAK, ptr, label, tmp_reg1, tmp_reg2 1400| ldr tmp_reg1, [ptr, #4] 1401| TST_32_WITH_CONST tmp_reg1, (GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)), tmp_reg2 1402| bne label 1403|.endmacro 1404 1405|.macro ADDREF_CONST, zv, tmp_reg1, tmp_reg2 1406| LOAD_ADDR tmp_reg1, Z_PTR_P(zv) 1407| ldr tmp_reg2, [tmp_reg1] 1408| add tmp_reg2, tmp_reg2, #1 1409| str tmp_reg2, [tmp_reg1] 1410|.endmacro 1411 1412|.macro ADDREF_CONST_2, zv, tmp_reg1, tmp_reg2 1413| LOAD_ADDR tmp_reg1, Z_PTR_P(zv) 1414| ldr tmp_reg2, [tmp_reg1] 1415| add tmp_reg2, tmp_reg2, #2 1416| str tmp_reg2, [tmp_reg1] 1417|.endmacro 1418 1419|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg, tmp_reg 1420|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1421|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1422| IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg 1423|| } 1424| GC_ADDREF value_ptr_reg, tmp_reg 1425|1: 1426|| } 1427|.endmacro 1428 1429|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg, tmp_reg 1430|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1431|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1432| IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg 1433|| } 1434| ldr tmp_reg, [value_ptr_reg] 1435| add tmp_reg, tmp_reg, #2 1436| str tmp_reg, [value_ptr_reg] 1437|1: 1438|| } 1439|.endmacro 1440 1441|.macro ZVAL_DEREF, reg, info, tmp_reg 1442|| if (info & MAY_BE_REF) { 1443| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1, tmp_reg 1444| GET_Z_PTR reg, reg 1445| add reg, reg, #offsetof(zend_reference, val) 1446|1: 1447|| } 1448|.endmacro 1449 1450|.macro SET_EX_OPLINE, op, tmp_reg 1451|| if (op == last_valid_opline) { 1452|| zend_jit_use_last_valid_opline(); 1453| SAVE_IP 1454|| } else { 1455| ADDR_STORE EX->opline, op, tmp_reg 1456|| if (!GCC_GLOBAL_REGS) { 1457|| zend_jit_reset_last_valid_opline(); 1458|| } 1459|| } 1460|.endmacro 1461 1462// arg1 "zval" should be in FCARG1x 1463|.macro ZVAL_DTOR_FUNC, var_info, opline, tmp_reg 1464|| do { 1465|| if (!((var_info) & MAY_BE_GUARD) 1466|| && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1467|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); 1468|| if (type == IS_STRING && !ZEND_DEBUG) { 1469| EXT_CALL _efree, tmp_reg 1470|| break; 1471|| } else if (type == IS_ARRAY) { 1472|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { 1473|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { 1474| SET_EX_OPLINE opline, tmp_reg 1475|| } 1476| EXT_CALL zend_array_destroy, tmp_reg 1477|| } else { 1478| EXT_CALL zend_jit_array_free, tmp_reg 1479|| } 1480|| break; 1481|| } else if (type == IS_OBJECT) { 1482|| if (opline) { 1483| SET_EX_OPLINE opline, REG0 1484|| } 1485| EXT_CALL zend_objects_store_del, tmp_reg 1486|| break; 1487|| } 1488|| } 1489|| if (opline) { 1490| SET_EX_OPLINE opline, tmp_reg 1491|| } 1492| EXT_CALL rc_dtor_func, tmp_reg 1493|| } while(0); 1494|.endmacro 1495 1496|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline, tmp_reg1, tmp_reg2 1497|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) { 1498|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1499| // if (Z_REFCOUNTED_P(cv)) { 1500|| if (cold) { 1501| IF_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 1502|.cold_code 1503|1: 1504|| } else { 1505| IF_NOT_ZVAL_REFCOUNTED addr, >4, tmp_reg1, tmp_reg2 1506|| } 1507|| } 1508| // if (!Z_DELREF_P(cv)) { 1509| GET_ZVAL_PTR FCARG1x, addr, Rx(tmp_reg2) 1510| GC_DELREF FCARG1x, Rw(tmp_reg1) 1511|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) { 1512|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) { 1513|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1514| bne >3 1515|| } else { 1516| bne >4 1517|| } 1518|| } 1519| // zval_dtor_func(r); 1520| ZVAL_DTOR_FUNC op_info, opline, Rx(tmp_reg1) 1521|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1522| b >4 1523|| } 1524|3: 1525|| } 1526|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1527|| if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) { 1528|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 1529| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1, tmp_reg1 1530| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4, tmp_reg1, tmp_reg2 1531| GET_ZVAL_PTR FCARG1x, ref_addr, Rx(tmp_reg2) 1532|1: 1533|| } 1534| IF_GC_MAY_NOT_LEAK FCARG1x, >4, Rw(tmp_reg1), Rw(tmp_reg2) 1535| // gc_possible_root(Z_COUNTED_P(z)) 1536| EXT_CALL gc_possible_root, Rx(tmp_reg1) 1537|| } 1538|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { 1539| b >4 1540|.code 1541|| } 1542|4: 1543|| } 1544|.endmacro 1545 1546|.macro FREE_OP, op_type, op, op_info, cold, opline, tmp_reg1, tmp_reg2 1547|| if (op_type & (IS_VAR|IS_TMP_VAR)) { 1548|| zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 1549| ZVAL_PTR_DTOR addr, op_info, 0, cold, opline, tmp_reg1, tmp_reg2 1550|| } 1551|.endmacro 1552 1553|.macro SEPARATE_ARRAY, addr, op_info, cold, tmp_reg1, tmp_reg2 1554|| if (RC_MAY_BE_N(op_info)) { 1555|| if (Z_REG(addr) != ZREG_FP) { 1556| GET_ZVAL_LVAL ZREG_REG0, addr, Rx(tmp_reg1) 1557|| if (RC_MAY_BE_1(op_info)) { 1558| // if (GC_REFCOUNT() > 1) 1559| ldr Rw(tmp_reg1), [REG0] 1560| cmp Rw(tmp_reg1), #1 1561| bls >2 1562|| } 1563|| if (Z_REG(addr) != ZREG_FCARG1 || Z_OFFSET(addr) != 0) { 1564| LOAD_ZVAL_ADDR FCARG1x, addr 1565|| } 1566| EXT_CALL zend_jit_zval_array_dup, REG0 1567| mov REG0, RETVALx 1568|2: 1569| mov FCARG1x, REG0 1570|| } else { 1571| GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1) 1572|| if (RC_MAY_BE_1(op_info)) { 1573| // if (GC_REFCOUNT() > 1) 1574| ldr Rw(tmp_reg1), [FCARG1x] 1575| cmp Rw(tmp_reg1), #1 1576|| if (cold) { 1577| bhi >1 1578|.cold_code 1579|1: 1580|| } else { 1581| bls >2 1582|| } 1583|| } 1584| IF_NOT_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 1585| GC_DELREF FCARG1x, Rw(tmp_reg1) 1586|1: 1587| EXT_CALL zend_array_dup, REG0 1588| mov REG0, RETVALx 1589| SET_ZVAL_PTR addr, REG0, Rx(tmp_reg1) 1590| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX, Rw(tmp_reg1), Rx(tmp_reg2) 1591| mov FCARG1x, REG0 1592|| if (RC_MAY_BE_1(op_info)) { 1593|| if (cold) { 1594| b >2 1595|.code 1596|| } 1597|| } 1598|2: 1599|| } 1600|| } else { 1601| GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1) 1602|| } 1603|.endmacro 1604 1605/* argument is passed in FCARG1x */ 1606|.macro EFREE_REFERENCE 1607||#if ZEND_DEBUG 1608| mov FCARG2x, xzr // filename 1609| mov CARG3w, wzr // lineno 1610| mov CARG4, xzr 1611| mov CARG5, xzr 1612| EXT_CALL _efree, REG0 1613||#else 1614||#ifdef HAVE_BUILTIN_CONSTANT_P 1615| EXT_CALL _efree_32, REG0 1616||#else 1617| EXT_CALL _efree, REG0 1618||#endif 1619||#endif 1620|.endmacro 1621 1622|.macro EMALLOC, size, op_array, opline 1623||#if ZEND_DEBUG 1624|| const char *filename = op_array->filename ? op_array->filename->val : NULL; 1625| mov FCARG1x, #size 1626| LOAD_ADDR FCARG2x, filename 1627| LOAD_32BIT_VAL CARG3w, opline->lineno 1628| mov CARG4, xzr 1629| mov CARG5, xzr 1630| EXT_CALL _emalloc, REG0 1631| mov REG0, RETVALx 1632||#else 1633||#ifdef HAVE_BUILTIN_CONSTANT_P 1634|| if (size > 24 && size <= 32) { 1635| EXT_CALL _emalloc_32, REG0 1636| mov REG0, RETVALx 1637|| } else { 1638| mov FCARG1x, #size 1639| EXT_CALL _emalloc, REG0 1640| mov REG0, RETVALx 1641|| } 1642||#else 1643| mov FCARG1x, #size 1644| EXT_CALL _emalloc, REG0 1645| mov REG0, RETVALx 1646||#endif 1647||#endif 1648|.endmacro 1649 1650|.macro OBJ_RELEASE, reg, exit_label, tmp_reg1, tmp_reg2 1651| GC_DELREF Rx(reg), Rw(tmp_reg1) 1652| bne >1 1653| // zend_objects_store_del(obj); 1654|| if (reg != ZREG_FCARG1) { 1655| mov FCARG1x, Rx(reg) 1656|| } 1657| EXT_CALL zend_objects_store_del, Rx(tmp_reg1) 1658| b exit_label 1659|1: 1660| IF_GC_MAY_NOT_LEAK Rx(reg), >1, Rw(tmp_reg1), Rw(tmp_reg2) 1661| // gc_possible_root(obj) 1662|| if (reg != ZREG_FCARG1) { 1663| mov FCARG1x, Rx(reg) 1664|| } 1665| EXT_CALL gc_possible_root, Rx(tmp_reg1) 1666|1: 1667|.endmacro 1668 1669|.macro UNDEFINED_OFFSET, opline 1670|| if (opline == last_valid_opline) { 1671|| zend_jit_use_last_valid_opline(); 1672| bl ->undefined_offset_ex 1673|| } else { 1674| SET_EX_OPLINE opline, REG0 1675| bl ->undefined_offset 1676|| } 1677|.endmacro 1678 1679|.macro UNDEFINED_INDEX, opline 1680|| if (opline == last_valid_opline) { 1681|| zend_jit_use_last_valid_opline(); 1682| bl ->undefined_index_ex 1683|| } else { 1684| SET_EX_OPLINE opline, REG0 1685| bl ->undefined_index 1686|| } 1687|.endmacro 1688 1689|.macro CANNOT_ADD_ELEMENT, opline 1690|| if (opline == last_valid_opline) { 1691|| zend_jit_use_last_valid_opline(); 1692| bl ->cannot_add_element_ex 1693|| } else { 1694| SET_EX_OPLINE opline, REG0 1695| bl ->cannot_add_element 1696|| } 1697|.endmacro 1698 1699static bool reuse_ip = 0; 1700static bool delayed_call_chain = 0; 1701static uint32_t delayed_call_level = 0; 1702static const zend_op *last_valid_opline = NULL; 1703static bool use_last_vald_opline = 0; 1704static bool track_last_valid_opline = 0; 1705static int jit_return_label = -1; 1706static uint32_t current_trace_num = 0; 1707static uint32_t allowed_opt_flags = 0; 1708 1709static void zend_jit_track_last_valid_opline(void) 1710{ 1711 use_last_vald_opline = 0; 1712 track_last_valid_opline = 1; 1713} 1714 1715static void zend_jit_use_last_valid_opline(void) 1716{ 1717 if (track_last_valid_opline) { 1718 use_last_vald_opline = 1; 1719 track_last_valid_opline = 0; 1720 } 1721} 1722 1723static bool zend_jit_trace_uses_initial_ip(void) 1724{ 1725 return use_last_vald_opline; 1726} 1727 1728static void zend_jit_set_last_valid_opline(const zend_op *target_opline) 1729{ 1730 if (!reuse_ip) { 1731 track_last_valid_opline = 0; 1732 last_valid_opline = target_opline; 1733 } 1734} 1735 1736static void zend_jit_reset_last_valid_opline(void) 1737{ 1738 track_last_valid_opline = 0; 1739 last_valid_opline = NULL; 1740} 1741 1742static void zend_jit_start_reuse_ip(void) 1743{ 1744 zend_jit_reset_last_valid_opline(); 1745 reuse_ip = 1; 1746} 1747 1748static int zend_jit_reuse_ip(dasm_State **Dst) 1749{ 1750 if (!reuse_ip) { 1751 zend_jit_start_reuse_ip(); 1752 | // call = EX(call); 1753 | ldr RX, EX->call 1754 } 1755 return 1; 1756} 1757 1758static void zend_jit_stop_reuse_ip(void) 1759{ 1760 reuse_ip = 0; 1761} 1762 1763static int zend_jit_interrupt_handler_stub(dasm_State **Dst) 1764{ 1765 |->interrupt_handler: 1766 | SAVE_IP 1767 | //EG(vm_interrupt) = 0; 1768 | MEM_STORE_8_ZTS strb, wzr, executor_globals, vm_interrupt, TMP1 1769 | //if (EG(timed_out)) { 1770 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, timed_out, TMP1 1771 | cbz TMP1w, >1 1772 | //zend_timeout(); 1773 | EXT_CALL zend_timeout, TMP1 1774 |1: 1775 | //} else if (zend_interrupt_function) { 1776 if (zend_interrupt_function) { 1777 | //zend_interrupt_function(execute_data); 1778 | mov CARG1, FP 1779 | EXT_CALL zend_interrupt_function, TMP1 1780 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 1781 | cbz REG0, >1 1782 | EXT_CALL zend_jit_exception_in_interrupt_handler_helper, TMP1 1783 |1: 1784 | //ZEND_VM_ENTER(); 1785 | //execute_data = EG(current_execute_data); 1786 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 1787 | LOAD_IP 1788 } 1789 | //ZEND_VM_CONTINUE() 1790 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1791 | ADD_HYBRID_SPAD 1792 | JMP_IP TMP1 1793 } else if (GCC_GLOBAL_REGS) { 1794 | ldp x29, x30, [sp], # SPAD // stack alignment 1795 | JMP_IP TMP1 1796 } else { 1797 | ldp FP, RX, T2 // retore FP and IP 1798 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1799 | mov RETVALx, #1 // ZEND_VM_ENTER 1800 | ret 1801 } 1802 1803 return 1; 1804} 1805 1806static int zend_jit_exception_handler_stub(dasm_State **Dst) 1807{ 1808 |->exception_handler: 1809 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1810 const void *handler = zend_get_opcode_handler_func(EG(exception_op)); 1811 1812 | ADD_HYBRID_SPAD 1813 | EXT_CALL handler, REG0 1814 | JMP_IP TMP1 1815 } else { 1816 const void *handler = EG(exception_op)->handler; 1817 1818 if (GCC_GLOBAL_REGS) { 1819 | ldp x29, x30, [sp], # SPAD // stack alignment 1820 | EXT_JMP handler, REG0 1821 } else { 1822 | mov FCARG1x, FP 1823 | EXT_CALL handler, REG0 1824 | ldp FP, RX, T2 // retore FP and IP 1825 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1826 | tbnz RETVALw, #31, >1 1827 | mov RETVALw, #1 // ZEND_VM_ENTER 1828 |1: 1829 | ret 1830 } 1831 } 1832 1833 return 1; 1834} 1835 1836static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) 1837{ 1838 |->exception_handler_undef: 1839 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, opline_before_exception, REG0 1840 | ldrb TMP1w, OP:REG0->result_type 1841 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1842 | beq >1 1843 | ldr REG0w, OP:REG0->result.var 1844 | add REG0, REG0, FP 1845 | SET_Z_TYPE_INFO REG0, IS_UNDEF, TMP1w 1846 |1: 1847 | b ->exception_handler 1848 1849 return 1; 1850} 1851 1852static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst) 1853{ 1854 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 1855 1856 |->exception_handler_free_op1_op2: 1857 | UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w 1858 | ldrb TMP1w, OP:RX->op1_type 1859 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1860 | beq >9 1861 | ldr REG0w, OP:RX->op1.var 1862 | add REG0, REG0, FP 1863 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 1864 |9: 1865 | ldrb TMP1w, OP:RX->op2_type 1866 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1867 | beq >9 1868 | ldr REG0w, OP:RX->op2.var 1869 | add REG0, REG0, FP 1870 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 1871 |9: 1872 | b ->exception_handler 1873 return 1; 1874} 1875 1876static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst) 1877{ 1878 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 1879 1880 |->exception_handler_free_op2: 1881 | MEM_LOAD_64_ZTS ldr, RX, executor_globals, opline_before_exception, REG0 1882 | UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w 1883 | ldrb TMP1w, OP:RX->op2_type 1884 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1885 | beq >9 1886 | ldr REG0w, OP:RX->op2.var 1887 | add REG0, REG0, FP 1888 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 1889 |9: 1890 | b ->exception_handler 1891 return 1; 1892} 1893 1894static int zend_jit_leave_function_stub(dasm_State **Dst) 1895{ 1896 |->leave_function_handler: 1897 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1898 | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w 1899 | bne >1 1900 | EXT_CALL zend_jit_leave_nested_func_helper, REG0 1901 | ADD_HYBRID_SPAD 1902 | JMP_IP TMP1 1903 |1: 1904 | EXT_CALL zend_jit_leave_top_func_helper, REG0 1905 | ADD_HYBRID_SPAD 1906 | JMP_IP TMP1 1907 } else { 1908 if (GCC_GLOBAL_REGS) { 1909 | ldp x29, x30, [sp], # SPAD // stack alignment 1910 } else { 1911 | mov FCARG2x, FP 1912 | ldp FP, RX, T2 // retore FP and IP 1913 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1914 } 1915 | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w 1916 | bne >1 1917 | EXT_JMP zend_jit_leave_nested_func_helper, REG0 1918 |1: 1919 | EXT_JMP zend_jit_leave_top_func_helper, REG0 1920 } 1921 1922 return 1; 1923} 1924 1925static int zend_jit_leave_throw_stub(dasm_State **Dst) 1926{ 1927 |->leave_throw_handler: 1928 | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { 1929 if (GCC_GLOBAL_REGS) { 1930 | ldrb TMP1w, OP:IP->opcode 1931 | cmp TMP1w, #ZEND_HANDLE_EXCEPTION 1932 | beq >5 1933 | // EG(opline_before_exception) = opline; 1934 | MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2 1935 |5: 1936 | // opline = EG(exception_op); 1937 | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 1938 | // HANDLE_EXCEPTION() 1939 | b ->exception_handler 1940 } else { 1941 | GET_IP TMP1 1942 | ldrb TMP2w, OP:TMP1->opcode 1943 | cmp TMP2w, #ZEND_HANDLE_EXCEPTION 1944 | beq >5 1945 | // EG(opline_before_exception) = opline; 1946 | MEM_STORE_64_ZTS str, TMP1, executor_globals, opline_before_exception, TMP2 1947 |5: 1948 | // opline = EG(exception_op); 1949 | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 1950 | ldp FP, RX, T2 // retore FP and IP 1951 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1952 | mov RETVALx, #2 // ZEND_VM_LEAVE 1953 | ret 1954 } 1955 1956 return 1; 1957} 1958 1959static int zend_jit_icall_throw_stub(dasm_State **Dst) 1960{ 1961 |->icall_throw_handler: 1962 | // zend_rethrow_exception(zend_execute_data *execute_data) 1963 | ldr IP, EX->opline 1964 | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { 1965 | ldrb TMP1w, OP:IP->opcode 1966 | cmp TMP1w, #ZEND_HANDLE_EXCEPTION 1967 | beq >1 1968 | // EG(opline_before_exception) = opline; 1969 | MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2 1970 |1: 1971 | // opline = EG(exception_op); 1972 | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 1973 || if (GCC_GLOBAL_REGS) { 1974 | str IP, EX->opline 1975 || } 1976 | // HANDLE_EXCEPTION() 1977 | b ->exception_handler 1978 1979 return 1; 1980} 1981 1982static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) 1983{ 1984 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 1985 1986 |->throw_cannot_pass_by_ref: 1987 | ldr REG0, EX->opline 1988 | ldr REG1w, OP:REG0->result.var 1989 | add REG1, REG1, RX 1990 | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w 1991 | // last EX(call) frame may be delayed 1992 | ldr TMP1, EX->call 1993 | cmp RX, TMP1 1994 | beq >1 1995 | ldr REG1, EX->call 1996 | str REG1, EX:RX->prev_execute_data 1997 | str RX, EX->call 1998 |1: 1999 | mov RX, REG0 2000 | ldr FCARG1w, OP:REG0->op2.num 2001 | EXT_CALL zend_cannot_pass_by_reference, REG0 2002 | ldrb TMP1w, OP:RX->op1_type 2003 | cmp TMP1w, #IS_TMP_VAR 2004 | bne >9 2005 | ldr REG0w, OP:RX->op1.var 2006 | add REG0, REG0, FP 2007 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 2008 |9: 2009 | b ->exception_handler 2010 2011 return 1; 2012} 2013 2014static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) 2015{ 2016 |->undefined_offset_ex: 2017 | SAVE_IP 2018 | b ->undefined_offset 2019 2020 return 1; 2021} 2022 2023static int zend_jit_undefined_offset_stub(dasm_State **Dst) 2024{ 2025 |->undefined_offset: 2026#ifdef __APPLE__ 2027 | stp x29, x30, [sp, # -16]! 2028 | mov x29, sp 2029#endif 2030 | //sub r4, 8 2031 | ldr REG0, EX->opline 2032 | ldr REG1w, OP:REG0->result.var 2033 | add REG1, REG1, FP 2034 | SET_Z_TYPE_INFO REG1, IS_NULL, TMP1w 2035 | ldrb REG1w, OP:REG0->op2_type 2036 | cmp REG1w, #IS_CONST 2037 | bne >2 2038 | ldrsw REG1, OP:REG0->op2.constant 2039 | add REG0, REG0, REG1 2040 | b >3 2041 |2: 2042 | ldr REG0w, OP:REG0->op2.var 2043 | add REG0, REG0, FP 2044 |3: 2045 | mov CARG1, #E_WARNING 2046 | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT 2047 | ldr CARG3, [REG0] 2048#ifdef __APPLE__ 2049 | str CARG3, [sp, #-16]! 2050 | EXT_CALL zend_error, REG0 2051 | add sp, sp, #16 2052 | ldp x29, x30, [sp], #16 2053 | ret 2054#else 2055 | EXT_JMP zend_error, REG0 // tail call 2056 | //add r4, 8 // stack alignment 2057 | //ret 2058#endif 2059 2060 return 1; 2061} 2062 2063static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) 2064{ 2065 |->undefined_index_ex: 2066 | SAVE_IP 2067 | b ->undefined_index 2068 2069 return 1; 2070} 2071 2072static int zend_jit_undefined_index_stub(dasm_State **Dst) 2073{ 2074 |->undefined_index: 2075#ifdef __APPLE__ 2076 | stp x29, x30, [sp, # -16]! 2077 | mov x29, sp 2078#endif 2079 | //sub r4, 8 2080 | ldr REG0, EX->opline 2081 | ldr REG1w, OP:REG0->result.var 2082 | add REG1, REG1, FP 2083 | SET_Z_TYPE_INFO REG1, IS_NULL, TMP1w 2084 | ldrb REG1w, OP:REG0->op2_type 2085 | cmp REG1w, #IS_CONST 2086 | bne >2 2087 | ldrsw REG1, OP:REG0->op2.constant 2088 | add REG0, REG0, REG1 2089 | b >3 2090 |2: 2091 | ldr REG0w, OP:REG0->op2.var 2092 | add REG0, REG0, FP 2093 |3: 2094 | mov CARG1, #E_WARNING 2095 | LOAD_ADDR CARG2, "Undefined array key \"%s\"" 2096 | ldr CARG3, [REG0] 2097 | add CARG3, CARG3, #offsetof(zend_string, val) 2098#ifdef __APPLE__ 2099 | str CARG3, [sp, #-16]! 2100 | EXT_CALL zend_error, REG0 2101 | add sp, sp, #16 2102 | ldp x29, x30, [sp], #16 2103 | ret 2104#else 2105 | EXT_JMP zend_error, REG0 // tail call 2106 | //add r4, 8 2107 | //ret 2108#endif 2109 2110 return 1; 2111} 2112 2113static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) 2114{ 2115 |->cannot_add_element_ex: 2116 | SAVE_IP 2117 | b ->cannot_add_element 2118 2119 return 1; 2120} 2121 2122static int zend_jit_cannot_add_element_stub(dasm_State **Dst) 2123{ 2124 |->cannot_add_element: 2125 | // sub r4, 8 2126 | ldr REG0, EX->opline 2127 | ldrb TMP1w, OP:REG0->result_type 2128 | cmp TMP1w, #IS_UNUSED 2129 | beq >1 2130 | ldr REG0w, OP:REG0->result.var 2131 | add REG0, REG0, FP 2132 | SET_Z_TYPE_INFO REG0, IS_NULL, TMP1w 2133 |1: 2134 | mov CARG1, xzr 2135 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2136 | EXT_JMP zend_throw_error, REG0 // tail call 2137 | // add r4, 8 2138 | //ret 2139 2140 return 1; 2141} 2142 2143static int zend_jit_undefined_function_stub(dasm_State **Dst) 2144{ 2145 |->undefined_function: 2146 | ldr REG0, EX->opline 2147 | mov CARG1, xzr 2148 | LOAD_ADDR CARG2, "Call to undefined function %s()" 2149 | ldrsw CARG3, [REG0, #offsetof(zend_op, op2.constant)] 2150 | ldr CARG3, [REG0, CARG3] 2151 | add CARG3, CARG3, #offsetof(zend_string, val) 2152#ifdef __APPLE__ 2153 | str CARG3, [sp, #-16]! 2154#endif 2155 | EXT_CALL zend_throw_error, REG0 2156#ifdef __APPLE__ 2157 | add sp, sp, #16 2158#endif 2159 | b ->exception_handler 2160 return 1; 2161} 2162 2163static int zend_jit_negative_shift_stub(dasm_State **Dst) 2164{ 2165 |->negative_shift: 2166 | ldr RX, EX->opline 2167 | LOAD_ADDR CARG1, zend_ce_arithmetic_error 2168 | LOAD_ADDR CARG2, "Bit shift by negative number" 2169 | EXT_CALL zend_throw_error, REG0 2170 | b ->exception_handler_free_op1_op2 2171 return 1; 2172} 2173 2174static int zend_jit_mod_by_zero_stub(dasm_State **Dst) 2175{ 2176 |->mod_by_zero: 2177 | ldr RX, EX->opline 2178 | LOAD_ADDR CARG1, zend_ce_division_by_zero_error 2179 | LOAD_ADDR CARG2, "Modulo by zero" 2180 | EXT_CALL zend_throw_error, REG0 2181 | b ->exception_handler_free_op1_op2 2182 return 1; 2183} 2184 2185static int zend_jit_invalid_this_stub(dasm_State **Dst) 2186{ 2187 |->invalid_this: 2188 | UNDEF_OPLINE_RESULT TMP1w 2189 | mov CARG1, xzr 2190 | LOAD_ADDR CARG2, "Using $this when not in object context" 2191 | EXT_CALL zend_throw_error, REG0 2192 | b ->exception_handler 2193 2194 return 1; 2195} 2196 2197static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) 2198{ 2199 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2200 return 1; 2201 } 2202 2203 |->hybrid_runtime_jit: 2204 | EXT_CALL zend_runtime_jit, REG0 2205 | JMP_IP TMP1 2206 return 1; 2207} 2208 2209static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) 2210{ 2211 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2212 return 1; 2213 } 2214 2215 |->hybrid_profile_jit: 2216 | // ++zend_jit_profile_counter; 2217 | LOAD_ADDR REG0, &zend_jit_profile_counter 2218 | ldr TMP1, [REG0] 2219 | add TMP1, TMP1, #1 2220 | str TMP1, [REG0] 2221 | // op_array = (zend_op_array*)EX(func); 2222 | ldr REG0, EX->func 2223 | // run_time_cache = EX(run_time_cache); 2224 | ldr REG2, EX->run_time_cache 2225 | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); 2226 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2227 | // ++ZEND_COUNTER_INFO(op_array) 2228 || if ((zend_jit_profile_counter_rid * sizeof(void*)) > LDR_STR_PIMM64) { 2229 | LOAD_32BIT_VAL TMP1, (zend_jit_profile_counter_rid * sizeof(void*)) 2230 | ldr TMP2, [REG2, TMP1] 2231 | add TMP2, TMP2, #1 2232 | str TMP2, [REG2, TMP1] 2233 || } else { 2234 | ldr TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] 2235 | add TMP2, TMP2, #1 2236 | str TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] 2237 || } 2238 | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() 2239 | ldr TMP1, [REG0, #offsetof(zend_jit_op_array_extension, orig_handler)] 2240 | br TMP1 2241 return 1; 2242} 2243 2244static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) 2245{ 2246 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2247 return 1; 2248 } 2249 2250 |->hybrid_hot_code: 2251 || ZEND_ASSERT(ZEND_JIT_COUNTER_INIT <= MOVZ_IMM); 2252 | movz TMP1w, #ZEND_JIT_COUNTER_INIT 2253 | strh TMP1w, [REG2] 2254 | mov FCARG1x, FP 2255 | GET_IP FCARG2x 2256 | EXT_CALL zend_jit_hot_func, REG0 2257 | JMP_IP TMP1 2258 return 1; 2259} 2260 2261/* 2262 * This code is based Mike Pall's "Hashed profile counters" idea, implemented 2263 * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual 2264 * property disclosure and research opportunities" email 2265 * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html 2266 * 2267 * In addition we use a variation of Knuth's multiplicative hash function 2268 * described at https://code.i-harness.com/en/q/a21ce 2269 * 2270 * uint64_t hash(uint64_t x) { 2271 * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; 2272 * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; 2273 * x = x ^ (x >> 31); 2274 * return x; 2275 * } 2276 * 2277 * uint_32_t hash(uint32_t x) { 2278 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2279 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2280 * x = (x >> 16) ^ x; 2281 * return x; 2282 * } 2283 * 2284 */ 2285static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) 2286{ 2287 | ldr REG0, EX->func 2288 | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2289 | ldr REG2, [REG1, #offsetof(zend_jit_op_array_hot_extension, counter)] 2290 | ldrh TMP2w, [REG2] 2291 | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w 2292 | strh TMP2w, [REG2] 2293 | ble ->hybrid_hot_code 2294 | GET_IP REG2 2295 | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] 2296 | sub REG2, REG2, TMP1 2297 | // divide by sizeof(zend_op) 2298 || ZEND_ASSERT(sizeof(zend_op) == 32); 2299 | add TMP1, REG1, REG2, asr #2 2300 | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2301 | br TMP1 2302 return 1; 2303} 2304 2305static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) 2306{ 2307 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2308 return 1; 2309 } 2310 2311 |->hybrid_func_hot_counter: 2312 2313 return zend_jit_hybrid_hot_counter_stub(Dst, 2314 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2315} 2316 2317static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) 2318{ 2319 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2320 return 1; 2321 } 2322 2323 |->hybrid_loop_hot_counter: 2324 2325 return zend_jit_hybrid_hot_counter_stub(Dst, 2326 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2327} 2328 2329static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) 2330{ 2331 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2332 return 1; 2333 } 2334 2335 // On entry from counter stub: 2336 // REG2 -> zend_op_trace_info.counter 2337 2338 |->hybrid_hot_trace: 2339 | mov TMP1w, #ZEND_JIT_COUNTER_INIT 2340 | strh TMP1w, [REG2] 2341 | mov FCARG1x, FP 2342 | GET_IP FCARG2x 2343 | EXT_CALL zend_jit_trace_hot_root, REG0 2344 | tbnz RETVALw, #31, >1 // Result is < 0 on failure. 2345 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 2346 | LOAD_IP 2347 | JMP_IP TMP1 2348 |1: 2349 | EXT_JMP zend_jit_halt_op->handler, REG0 2350 2351 return 1; 2352} 2353 2354static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) 2355{ 2356 | ldr REG0, EX->func 2357 | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2358 | ldr REG1, [REG1, #offsetof(zend_jit_op_array_trace_extension, offset)] 2359 | add TMP1, REG1, IP 2360 | ldr REG2, [TMP1, #offsetof(zend_op_trace_info, counter)] 2361 | ldrh TMP2w, [REG2] 2362 | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w 2363 | strh TMP2w, [REG2] 2364 | ble ->hybrid_hot_trace 2365 // Note: "REG1 + IP" is re-calculated as TMP1 is used as temporary register by the prior 2366 // ADD_SUB_32_WITH_CONST. Will optimize in the future if more temporary registers are available. 2367 | add TMP1, REG1, IP 2368 | ldr TMP2, [TMP1, #offsetof(zend_op_trace_info, orig_handler)] 2369 | br TMP2 2370 2371 return 1; 2372} 2373 2374static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) 2375{ 2376 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2377 return 1; 2378 } 2379 2380 |->hybrid_func_trace_counter: 2381 2382 return zend_jit_hybrid_trace_counter_stub(Dst, 2383 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2384} 2385 2386static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) 2387{ 2388 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { 2389 return 1; 2390 } 2391 2392 |->hybrid_ret_trace_counter: 2393 2394 return zend_jit_hybrid_trace_counter_stub(Dst, 2395 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); 2396} 2397 2398static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) 2399{ 2400 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2401 return 1; 2402 } 2403 2404 |->hybrid_loop_trace_counter: 2405 2406 return zend_jit_hybrid_trace_counter_stub(Dst, 2407 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2408} 2409 2410static int zend_jit_trace_halt_stub(dasm_State **Dst) 2411{ 2412 |->trace_halt: 2413 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2414 | ADD_HYBRID_SPAD 2415 | EXT_JMP zend_jit_halt_op->handler, REG0 2416 } else if (GCC_GLOBAL_REGS) { 2417 | ldp x29, x30, [sp], # SPAD // stack alignment 2418 | mov IP, xzr // PC must be zero 2419 | ret 2420 } else { 2421 | ldp FP, RX, T2 // retore FP and IP 2422 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2423 | movn RETVALx, #0 // ZEND_VM_RETURN (-1) 2424 | ret 2425 } 2426 return 1; 2427} 2428 2429static int zend_jit_trace_exit_stub(dasm_State **Dst) 2430{ 2431 |->trace_exit: 2432 | 2433 | // Save CPU registers(32 GP regs + 32 FP regs) on stack in the order of d31 to x0 2434 | 2435 | stp d30, d31, [sp, #-16]! 2436 | stp d28, d29, [sp, #-16]! 2437 | stp d26, d27, [sp, #-16]! 2438 | stp d24, d25, [sp, #-16]! 2439 | stp d22, d23, [sp, #-16]! 2440 | stp d20, d21, [sp, #-16]! 2441 | stp d18, d19, [sp, #-16]! 2442 | stp d16, d17, [sp, #-16]! 2443 | //stp d14, d15, [sp, #-16]! // we don't use preserved registers yet 2444 | //stp d12, d13, [sp, #-16]! 2445 | //stp d10, d11, [sp, #-16]! 2446 | //stp d8, d9, [sp, #-16]! 2447 | stp d6, d7, [sp, #(-16*5)]! 2448 | stp d4, d5, [sp, #-16]! 2449 | stp d2, d3, [sp, #-16]! 2450 | stp d0, d1, [sp, #-16]! 2451 | 2452 | //str x30, [sp, #-16]! // we don't use callee-saved registers yet (x31 can be omitted) 2453 | stp x28, x29, [sp, #(-16*2)]! // we have to store RX (x28) 2454 | //stp x26, x27, [sp, #-16]! // we don't use callee-saved registers yet 2455 | //stp x24, x25, [sp, #-16]! 2456 | //stp x22, x23, [sp, #-16]! 2457 | //stp x20, x21, [sp, #-16]! 2458 | //stp x18, x19, [sp, #-16]! 2459 | //stp x16, x17, [sp, #-16]! // we don't need temporary registers 2460 | stp x14, x15, [sp, #-(16*7)]! 2461 | stp x12, x13, [sp, #-16]! 2462 | stp x10, x11, [sp, #-16]! 2463 | stp x8, x9, [sp, #-16]! 2464 | stp x6, x7, [sp, #-16]! 2465 | stp x4, x5, [sp, #-16]! 2466 | stp x2, x3, [sp, #-16]! 2467 | stp x0, x1, [sp, #-16]! 2468 | 2469 | mov FCARG1w, TMP1w // exit_num 2470 | mov FCARG2x, sp 2471 | 2472 | // EX(opline) = opline 2473 | SAVE_IP 2474 | // zend_jit_trace_exit(trace_num, exit_num) 2475 | EXT_CALL zend_jit_trace_exit, REG0 2476 | 2477 | add sp, sp, #(32 * 16) // restore sp 2478 | 2479 2480 | tst RETVALw, RETVALw 2481 | bne >1 // not zero 2482 2483 | // execute_data = EG(current_execute_data) 2484 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 2485 | // opline = EX(opline) 2486 | LOAD_IP 2487 2488 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2489 | ADD_HYBRID_SPAD 2490 | JMP_IP TMP1 2491 } else if (GCC_GLOBAL_REGS) { 2492 | ldp x29, x30, [sp], # SPAD // stack alignment 2493 | JMP_IP TMP1 2494 } else { 2495 | ldp FP, RX, T2 // retore FP and IP 2496 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2497 | mov RETVALx, #1 // ZEND_VM_ENTER 2498 | ret 2499 } 2500 2501 |1: 2502 | blt ->trace_halt 2503 2504 | // execute_data = EG(current_execute_data) 2505 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 2506 | // opline = EX(opline) 2507 | LOAD_IP 2508 2509 | // check for interrupt (try to avoid this ???) 2510 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 2511 | cbnz TMP1w, ->interrupt_handler 2512 2513 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2514 | ADD_HYBRID_SPAD 2515 | ldr REG0, EX->func 2516 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2517 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 2518 | ldr REG0, [IP, REG0] 2519 | br REG0 2520 } else if (GCC_GLOBAL_REGS) { 2521 | ldp x29, x30, [sp], # SPAD // stack alignment 2522 | ldr REG0, EX->func 2523 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2524 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 2525 | ldr REG0, [IP, REG0] 2526 | br REG0 2527 } else { 2528 | ldr IP, EX->opline 2529 | mov FCARG1x, FP 2530 | ldr REG0, EX->func 2531 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2532 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 2533 | ldr REG0, [IP, REG0] 2534 | blr REG0 2535 | 2536 | tst RETVALw, RETVALw 2537 | blt ->trace_halt 2538 | 2539 | ldp FP, RX, T2 // retore FP and IP 2540 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2541 | mov RETVALx, #1 // ZEND_VM_ENTER 2542 | ret 2543 } 2544 2545 return 1; 2546} 2547 2548static int zend_jit_trace_escape_stub(dasm_State **Dst) 2549{ 2550 |->trace_escape: 2551 | 2552 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2553 | ADD_HYBRID_SPAD 2554 | JMP_IP, TMP1 2555 } else if (GCC_GLOBAL_REGS) { 2556 | ldp x29, x30, [sp], # SPAD // stack alignment 2557 | JMP_IP, TMP1 2558 } else { 2559 | ldp FP, RX, T2 // retore FP and IP 2560 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2561 | mov RETVALx, #1 // ZEND_VM_ENTER 2562 | ret 2563 } 2564 2565 return 1; 2566} 2567 2568/* Keep 32 exit points in a single code block */ 2569#define ZEND_JIT_EXIT_POINTS_SPACING 4 // bl = bytes 2570#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points 2571 2572static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) 2573{ 2574 uint32_t i; 2575 2576 | bl >2 2577 |1: 2578 for (i = 1; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { 2579 | bl >2 2580 } 2581 |2: 2582 | adr TMP1, <1 2583 | sub TMP1, lr, TMP1 2584 | lsr TMP1, TMP1, #2 2585 if (n) { 2586 | ADD_SUB_32_WITH_CONST add, TMP1w, TMP1w, n, TMP2w 2587 } 2588 | b ->trace_exit // pass exit_num in TMP1w 2589 2590 return 1; 2591} 2592 2593#ifdef CONTEXT_THREADED_JIT 2594static int zend_jit_context_threaded_call_stub(dasm_State **Dst) 2595{ 2596 |->context_threaded_call: 2597 | NIY_STUB // TODO 2598 return 1; 2599} 2600#endif 2601 2602static int zend_jit_assign_const_stub(dasm_State **Dst) 2603{ 2604 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2605 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2606 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2607 2608 |->assign_const: 2609 | stp x29, x30, [sp, #-32]! 2610 | mov x29, sp 2611 if (!zend_jit_assign_to_variable( 2612 Dst, NULL, 2613 var_addr, var_addr, -1, -1, 2614 IS_CONST, val_addr, val_info, 2615 0, 0)) { 2616 return 0; 2617 } 2618 | ldp x29, x30, [sp], #32 2619 | ret 2620 return 1; 2621} 2622 2623static int zend_jit_assign_tmp_stub(dasm_State **Dst) 2624{ 2625 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2626 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2627 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2628 2629 |->assign_tmp: 2630 | stp x29, x30, [sp, #-32]! 2631 | mov x29, sp 2632 if (!zend_jit_assign_to_variable( 2633 Dst, NULL, 2634 var_addr, var_addr, -1, -1, 2635 IS_TMP_VAR, val_addr, val_info, 2636 0, 0)) { 2637 return 0; 2638 } 2639 | ldp x29, x30, [sp], #32 2640 | ret 2641 return 1; 2642} 2643 2644static int zend_jit_assign_var_stub(dasm_State **Dst) 2645{ 2646 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2647 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2648 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; 2649 2650 |->assign_var: 2651 | stp x29, x30, [sp, #-32]! 2652 | mov x29, sp 2653 if (!zend_jit_assign_to_variable( 2654 Dst, NULL, 2655 var_addr, var_addr, -1, -1, 2656 IS_VAR, val_addr, val_info, 2657 0, 0)) { 2658 return 0; 2659 } 2660 | ldp x29, x30, [sp], #32 2661 | ret 2662 return 1; 2663} 2664 2665static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) 2666{ 2667 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2668 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2669 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; 2670 2671 |->assign_cv_noref: 2672 | stp x29, x30, [sp, #-32]! 2673 | mov x29, sp 2674 if (!zend_jit_assign_to_variable( 2675 Dst, NULL, 2676 var_addr, var_addr, -1, -1, 2677 IS_CV, val_addr, val_info, 2678 0, 0)) { 2679 return 0; 2680 } 2681 | ldp x29, x30, [sp], #32 2682 | ret 2683 return 1; 2684} 2685 2686static int zend_jit_assign_cv_stub(dasm_State **Dst) 2687{ 2688 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2689 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2690 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; 2691 2692 |->assign_cv: 2693 | stp x29, x30, [sp, #-32]! 2694 | mov x29, sp 2695 if (!zend_jit_assign_to_variable( 2696 Dst, NULL, 2697 var_addr, var_addr, -1, -1, 2698 IS_CV, val_addr, val_info, 2699 0, 0)) { 2700 return 0; 2701 } 2702 | ldp x29, x30, [sp], #32 2703 | ret 2704 return 1; 2705} 2706 2707static const zend_jit_stub zend_jit_stubs[] = { 2708 JIT_STUB(interrupt_handler, SP_ADJ_JIT, SP_ADJ_VM), 2709 JIT_STUB(exception_handler, SP_ADJ_JIT, SP_ADJ_VM), 2710 JIT_STUB(exception_handler_undef, SP_ADJ_JIT, SP_ADJ_VM), 2711 JIT_STUB(exception_handler_free_op1_op2, SP_ADJ_JIT, SP_ADJ_VM), 2712 JIT_STUB(exception_handler_free_op2, SP_ADJ_JIT, SP_ADJ_VM), 2713 JIT_STUB(leave_function, SP_ADJ_JIT, SP_ADJ_VM), 2714 JIT_STUB(leave_throw, SP_ADJ_JIT, SP_ADJ_VM), 2715 JIT_STUB(icall_throw, SP_ADJ_JIT, SP_ADJ_VM), 2716 JIT_STUB(throw_cannot_pass_by_ref, SP_ADJ_JIT, SP_ADJ_VM), 2717 JIT_STUB(undefined_offset, SP_ADJ_JIT, SP_ADJ_VM), 2718 JIT_STUB(undefined_index, SP_ADJ_JIT, SP_ADJ_VM), 2719 JIT_STUB(cannot_add_element, SP_ADJ_JIT, SP_ADJ_VM), 2720 JIT_STUB(undefined_offset_ex, SP_ADJ_JIT, SP_ADJ_VM), 2721 JIT_STUB(undefined_index_ex, SP_ADJ_JIT, SP_ADJ_VM), 2722 JIT_STUB(cannot_add_element_ex, SP_ADJ_JIT, SP_ADJ_VM), 2723 JIT_STUB(undefined_function, SP_ADJ_JIT, SP_ADJ_VM), 2724 JIT_STUB(negative_shift, SP_ADJ_JIT, SP_ADJ_VM), 2725 JIT_STUB(mod_by_zero, SP_ADJ_JIT, SP_ADJ_VM), 2726 JIT_STUB(invalid_this, SP_ADJ_JIT, SP_ADJ_VM), 2727 JIT_STUB(trace_halt, SP_ADJ_JIT, SP_ADJ_VM), 2728 JIT_STUB(trace_exit, SP_ADJ_JIT, SP_ADJ_VM), 2729 JIT_STUB(trace_escape, SP_ADJ_JIT, SP_ADJ_VM), 2730 JIT_STUB(hybrid_runtime_jit, SP_ADJ_VM, SP_ADJ_NONE), 2731 JIT_STUB(hybrid_profile_jit, SP_ADJ_VM, SP_ADJ_NONE), 2732 JIT_STUB(hybrid_hot_code, SP_ADJ_VM, SP_ADJ_NONE), 2733 JIT_STUB(hybrid_func_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2734 JIT_STUB(hybrid_loop_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2735 JIT_STUB(hybrid_hot_trace, SP_ADJ_VM, SP_ADJ_NONE), 2736 JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2737 JIT_STUB(hybrid_ret_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2738 JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2739 JIT_STUB(assign_const, SP_ADJ_RET, SP_ADJ_ASSIGN), 2740 JIT_STUB(assign_tmp, SP_ADJ_RET, SP_ADJ_ASSIGN), 2741 JIT_STUB(assign_var, SP_ADJ_RET, SP_ADJ_ASSIGN), 2742 JIT_STUB(assign_cv_noref, SP_ADJ_RET, SP_ADJ_ASSIGN), 2743 JIT_STUB(assign_cv, SP_ADJ_RET, SP_ADJ_ASSIGN), 2744#ifdef CONTEXT_THREADED_JIT 2745 JIT_STUB(context_threaded_call, SP_ADJ_NONE, SP_ADJ_NONE), 2746#endif 2747}; 2748 2749#ifdef HAVE_GDB 2750# if 0 2751typedef struct _Unwind_Context _Unwind_Context; 2752typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *); 2753extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *); 2754extern uintptr_t _Unwind_GetCFA(_Unwind_Context *); 2755 2756typedef struct _zend_jit_unwind_arg { 2757 int cnt; 2758 uintptr_t cfa[3]; 2759} zend_jit_unwind_arg; 2760 2761static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a) 2762{ 2763 zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a; 2764 arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx); 2765 arg->cnt++; 2766 if (arg->cnt == 3) { 2767 return 5; // _URC_END_OF_STACK 2768 } 2769 return 0; // _URC_NO_REASON; 2770} 2771 2772static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) 2773{ 2774 zend_jit_unwind_arg arg; 2775 2776 memset(&arg, 0, sizeof(arg)); 2777 _Unwind_Backtrace(zend_jit_unwind_cb, &arg); 2778 if (arg.cnt == 3) { 2779 sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1]; 2780 } 2781} 2782# else 2783static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) 2784{ 2785 uintptr_t ret; 2786 2787 __asm__ ( 2788 "ldr %0, [x29]\n\t" 2789 "sub %0 ,%0, x29" 2790 : "=r"(ret)); 2791 2792 sp_adj[SP_ADJ_VM] = ret; 2793} 2794# endif 2795 2796extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data); 2797 2798static zend_never_inline void zend_jit_set_sp_adj_vm(void) 2799{ 2800 void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *); 2801 2802 orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data; 2803 zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data; 2804 execute_ex(NULL); // set sp_adj[SP_ADJ_VM] 2805 zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data; 2806} 2807#endif 2808 2809static int zend_jit_setup(void) 2810{ 2811 allowed_opt_flags = 0; 2812 2813#if ZTS 2814 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2815 ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0); 2816#endif 2817 2818 memset(sp_adj, 0, sizeof(sp_adj)); 2819#ifdef HAVE_GDB 2820 sp_adj[SP_ADJ_RET] = 0; 2821 sp_adj[SP_ADJ_ASSIGN] = 32; 2822 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2823 zend_jit_set_sp_adj_vm(); // set sp_adj[SP_ADJ_VM] 2824#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 2825 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD 2826#else 2827 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM]; 2828#endif 2829 } else if (GCC_GLOBAL_REGS) { 2830 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD; // sub r4, SPAD 2831 } else { 2832 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD; // sub r4, NR_SPAD 2833 } 2834#endif 2835 2836 return SUCCESS; 2837} 2838 2839static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) 2840{ 2841 | brk #0 2842 return 1; 2843} 2844 2845static int zend_jit_align_func(dasm_State **Dst) 2846{ 2847 reuse_ip = 0; 2848 delayed_call_chain = 0; 2849 last_valid_opline = NULL; 2850 use_last_vald_opline = 0; 2851 track_last_valid_opline = 0; 2852 jit_return_label = -1; 2853 |.align 16 2854 return 1; 2855} 2856 2857static int zend_jit_prologue(dasm_State **Dst) 2858{ 2859 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2860 | SUB_HYBRID_SPAD 2861 } else if (GCC_GLOBAL_REGS) { 2862 | stp x29, x30, [sp, # -SPAD]! // stack alignment 2863 |// mov x29, sp 2864 } else { 2865 | stp x29, x30, [sp, # -NR_SPAD]! // stack alignment 2866 |// mov x29, sp 2867 | stp FP, RX, T2 // save FP and IP 2868 | mov FP, FCARG1x 2869 } 2870 return 1; 2871} 2872 2873static int zend_jit_label(dasm_State **Dst, unsigned int label) 2874{ 2875 |=>label: 2876 return 1; 2877} 2878 2879static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) 2880{ 2881 | // call->prev_execute_data = EX(call); 2882 if (call_level == 1) { 2883 | str xzr, EX:RX->prev_execute_data 2884 } else { 2885 | ldr REG0, EX->call 2886 | str REG0, EX:RX->prev_execute_data 2887 } 2888 | // EX(call) = call; 2889 | str RX, EX->call 2890 2891 delayed_call_chain = 0; 2892 2893 return 1; 2894} 2895 2896static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) 2897{ 2898 if (last_valid_opline == opline) { 2899 zend_jit_use_last_valid_opline(); 2900 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 2901 zend_jit_use_last_valid_opline(); 2902 | LOAD_64BIT_VAL TMP1, (opline - last_valid_opline) * sizeof(zend_op) 2903 | ADD_IP TMP1, TMP2 2904 } else { 2905 | LOAD_IP_ADDR opline 2906 } 2907 zend_jit_set_last_valid_opline(opline); 2908 2909 return 1; 2910} 2911 2912static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg) 2913{ 2914 return zend_jit_set_ip(Dst, opline); 2915} 2916 2917static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) 2918{ 2919 if (delayed_call_chain) { 2920 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 2921 return 0; 2922 } 2923 } 2924 if (!zend_jit_set_ip(Dst, opline)) { 2925 return 0; 2926 } 2927 reuse_ip = 0; 2928 return 1; 2929} 2930 2931static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) 2932{ 2933 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 2934 if (exit_addr) { 2935 | cbnz TMP1w, &exit_addr 2936 } else if (last_valid_opline == opline) { 2937 || zend_jit_use_last_valid_opline(); 2938 | cbnz TMP1w, ->interrupt_handler 2939 } else { 2940 | cbnz TMP1w, >1 2941 |.cold_code 2942 |1: 2943 | LOAD_IP_ADDR opline 2944 | b ->interrupt_handler 2945 |.code 2946 } 2947 return 1; 2948} 2949 2950static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) 2951{ 2952 if (timeout_exit_addr) { 2953 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 2954 | cbz TMP1w, =>loop_label 2955 | b &timeout_exit_addr 2956 } else { 2957 | b =>loop_label 2958 } 2959 return 1; 2960} 2961 2962static int zend_jit_check_exception(dasm_State **Dst) 2963{ 2964 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 2965 | cbnz TMP2, ->exception_handler 2966 return 1; 2967} 2968 2969static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) 2970{ 2971 if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { 2972 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 2973 | cbnz TMP2, ->exception_handler_undef 2974 return 1; 2975 } 2976 return zend_jit_check_exception(Dst); 2977} 2978 2979static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) 2980{ 2981 2982 current_trace_num = trace_num; 2983 2984 | // EG(jit_trace_num) = trace_num; 2985 | LOAD_32BIT_VAL TMP1w, trace_num 2986 | MEM_STORE_32_ZTS str, TMP1w, executor_globals, jit_trace_num, TMP2 2987 2988 return 1; 2989} 2990 2991static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t) 2992{ 2993 uint32_t i; 2994 const void *exit_addr; 2995 2996 /* Emit veneers table for exit points (B instruction for each exit number) */ 2997 |.cold_code 2998 for (i = 0; i < t->exit_count; i++) { 2999 exit_addr = zend_jit_trace_get_exit_addr(i); 3000 if (!exit_addr) { 3001 return 0; 3002 } 3003 | b &exit_addr 3004 } 3005 |=>1: // end of the code 3006 |.code 3007 return 1; 3008} 3009 3010static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) 3011{ 3012 int ret = 0; 3013 uint8_t *p, *end; 3014 const void *veneer = NULL; 3015 ptrdiff_t delta; 3016 3017 if (jmp_table_size) { 3018 const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*))); 3019 3020 do { 3021 if (*jmp_slot == from_addr) { 3022 *jmp_slot = to_addr; 3023 ret++; 3024 } 3025 jmp_slot++; 3026 } while (--jmp_table_size); 3027 } 3028 3029 end = (uint8_t*)code; 3030 p = end + size; 3031 while (p > end) { 3032 uint32_t *ins_ptr; 3033 uint32_t ins; 3034 3035 p -= 4; 3036 ins_ptr = (uint32_t*)p; 3037 ins = *ins_ptr; 3038 if ((ins & 0xfc000000u) == 0x14000000u) { 3039 // B (imm26:0..25) 3040 delta = (uint32_t*)from_addr - ins_ptr; 3041 if (((ins ^ (uint32_t)delta) & 0x01ffffffu) == 0) { 3042 delta = (uint32_t*)to_addr - ins_ptr; 3043 if (((delta + 0x02000000) >> 26) != 0) { 3044 abort(); // brnach target out of range 3045 } 3046 *ins_ptr = (ins & 0xfc000000u) | ((uint32_t)delta & 0x03ffffffu); 3047 ret++; 3048 if (!veneer) { 3049 veneer = p; 3050 } 3051 } 3052 } else if ((ins & 0xff000000u) == 0x54000000u || 3053 (ins & 0x7e000000u) == 0x34000000u) { 3054 // B.cond, CBZ, CBNZ (imm19:5..23) 3055 delta = (uint32_t*)from_addr - ins_ptr; 3056 if (((ins ^ ((uint32_t)delta << 5)) & 0x00ffffe0u) == 0) { 3057 delta = (uint32_t*)to_addr - ins_ptr; 3058 if (((delta + 0x40000) >> 19) != 0) { 3059 if (veneer) { 3060 delta = (uint32_t*)veneer - ins_ptr; 3061 if (((delta + 0x40000) >> 19) != 0) { 3062 abort(); // brnach target out of range 3063 } 3064 } else { 3065 abort(); // brnach target out of range 3066 } 3067 } 3068 *ins_ptr = (ins & 0xff00001fu) | (((uint32_t)delta & 0x7ffffu) << 5); 3069 ret++; 3070 } 3071 } else if ((ins & 0x7e000000u) == 0x36000000u) { 3072 // TBZ, TBNZ (imm14:5..18) 3073 delta = (uint32_t*)from_addr - ins_ptr; 3074 if (((ins ^ ((uint32_t)delta << 5)) & 0x0007ffe0u) == 0) { 3075 delta = (uint32_t*)to_addr - ins_ptr; 3076 if (((delta + 0x2000) >> 14) != 0) { 3077 if (veneer) { 3078 delta = (uint32_t*)veneer - ins_ptr; 3079 if (((delta + 0x2000) >> 14) != 0) { 3080 abort(); // brnach target out of range 3081 } 3082 } else { 3083 abort(); // brnach target out of range 3084 } 3085 } 3086 *ins_ptr = (ins & 0xfff8001fu) | (((uint32_t)delta & 0x3fffu) << 5); 3087 ret++; 3088 } 3089 } 3090 } 3091 3092 JIT_CACHE_FLUSH(code, (char*)code + size); 3093 3094#ifdef HAVE_VALGRIND 3095 VALGRIND_DISCARD_TRANSLATIONS(code, size); 3096#endif 3097 3098 return ret; 3099} 3100 3101static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) 3102{ 3103 return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); 3104} 3105 3106static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) 3107{ 3108 const void *link_addr; 3109 size_t prologue_size; 3110 3111 /* Skip prologue. */ 3112 // TODO: don't hardcode this ??? 3113 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3114#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 3115 prologue_size = 0; 3116#else 3117 // sub sp, sp, #0x20 3118 prologue_size = 4; 3119#endif 3120 } else if (GCC_GLOBAL_REGS) { 3121 // stp x29, x30, [sp, # -SPAD]! 3122 prologue_size = 4; 3123 } else { 3124 // stp x29, x30, [sp, # -NR_SPAD]! // stack alignment 3125 // stp FP, RX, T2 3126 // mov FP, FCARG1x 3127 prologue_size = 12; 3128 } 3129 link_addr = (const void*)((const char*)t->code_start + prologue_size); 3130 3131 if (timeout_exit_addr) { 3132 /* Check timeout for links to LOOP */ 3133 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 3134 | cbz TMP1w, &link_addr 3135 | b &timeout_exit_addr 3136 } else { 3137 | b &link_addr 3138 } 3139 return 1; 3140} 3141 3142static int zend_jit_trace_return(dasm_State **Dst, bool original_handler, const zend_op *opline) 3143{ 3144 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3145 | ADD_HYBRID_SPAD 3146 if (!original_handler) { 3147 | JMP_IP TMP1 3148 } else { 3149 | ldr REG0, EX->func 3150 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 3151 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 3152 | ldr REG0, [IP, REG0] 3153 | br REG0 3154 } 3155 } else if (GCC_GLOBAL_REGS) { 3156 | ldp x29, x30, [sp], # SPAD // stack alignment 3157 if (!original_handler) { 3158 | JMP_IP TMP1 3159 } else { 3160 | ldr REG0, EX->func 3161 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 3162 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 3163 | ldr REG0, [IP, REG0] 3164 | br REG0 3165 } 3166 } else { 3167 if (original_handler) { 3168 | mov FCARG1x, FP 3169 | ldr REG0, EX->func 3170 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 3171 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 3172 | ldr REG0, [IP, REG0] 3173 | blr REG0 3174 } 3175 | ldp FP, RX, T2 // retore FP and IP 3176 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 3177 if (!original_handler || !opline || 3178 (opline->opcode != ZEND_RETURN 3179 && opline->opcode != ZEND_RETURN_BY_REF 3180 && opline->opcode != ZEND_GENERATOR_RETURN 3181 && opline->opcode != ZEND_GENERATOR_CREATE 3182 && opline->opcode != ZEND_YIELD 3183 && opline->opcode != ZEND_YIELD_FROM)) { 3184 | mov RETVALx, #2 // ZEND_VM_LEAVE 3185 } 3186 | ret 3187 } 3188 return 1; 3189} 3190 3191static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) 3192{ 3193 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3194 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3195 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3196 3197 if (!exit_addr) { 3198 return 0; 3199 } 3200 3201 | IF_NOT_ZVAL_TYPE var_addr, type, &exit_addr, ZREG_TMP1 3202 3203 return 1; 3204} 3205 3206static int zend_jit_scalar_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var) 3207{ 3208 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3209 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3210 3211 if (!exit_addr) { 3212 return 0; 3213 } 3214 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, var+offsetof(zval, u1.v.type), TMP1 3215 | cmp TMP1w, #IS_STRING 3216 | bhs &exit_addr 3217 3218 return 1; 3219} 3220static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) 3221{ 3222 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 3223 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3224 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3225 3226 if (!exit_addr) { 3227 return 0; 3228 } 3229 3230 | GET_ZVAL_LVAL ZREG_FCARG1, var_addr, TMP1 3231 if (op_info & MAY_BE_ARRAY_PACKED) { 3232 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 3233 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 3234 | beq &exit_addr 3235 } else { 3236 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 3237 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 3238 | bne &exit_addr 3239 } 3240 3241 return 1; 3242} 3243 3244static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) 3245{ 3246 zend_jit_op_array_trace_extension *jit_extension = 3247 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); 3248 size_t offset = jit_extension->offset; 3249 const void *handler = 3250 (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; 3251 3252 if (!zend_jit_set_valid_ip(Dst, opline)) { 3253 return 0; 3254 } 3255 if (!GCC_GLOBAL_REGS) { 3256 | mov FCARG1x, FP 3257 } 3258 | EXT_CALL handler, REG0 3259 if (may_throw 3260 && opline->opcode != ZEND_RETURN 3261 && opline->opcode != ZEND_RETURN_BY_REF) { 3262 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 3263 | cbnz REG0, ->exception_handler 3264 } 3265 3266 while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { 3267 trace++; 3268 } 3269 3270 if (!GCC_GLOBAL_REGS 3271 && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { 3272 if (opline->opcode == ZEND_RETURN || 3273 opline->opcode == ZEND_RETURN_BY_REF || 3274 opline->opcode == ZEND_DO_UCALL || 3275 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3276 opline->opcode == ZEND_DO_FCALL || 3277 opline->opcode == ZEND_GENERATOR_CREATE) { 3278 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 3279 } 3280 } 3281 3282 if (zend_jit_trace_may_exit(op_array, opline)) { 3283 if (opline->opcode == ZEND_RETURN || 3284 opline->opcode == ZEND_RETURN_BY_REF || 3285 opline->opcode == ZEND_GENERATOR_CREATE) { 3286 3287 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3288#if 0 3289 /* this check should be handled by the following OPLINE guard or jmp [IP] */ 3290 | LOAD_ADDR TMP1, zend_jit_halt_op 3291 | cmp IP, TMP1 3292 | beq ->trace_halt 3293#endif 3294 } else if (GCC_GLOBAL_REGS) { 3295 | cbz IP, ->trace_halt 3296 } else { 3297 | tst RETVALw, RETVALw 3298 | blt ->trace_halt 3299 } 3300 } else if (opline->opcode == ZEND_EXIT || 3301 opline->opcode == ZEND_GENERATOR_RETURN || 3302 opline->opcode == ZEND_YIELD || 3303 opline->opcode == ZEND_YIELD_FROM) { 3304 | b ->trace_halt 3305 } 3306 if (trace->op != ZEND_JIT_TRACE_END || 3307 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && 3308 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 3309 3310 const zend_op *next_opline = trace->opline; 3311 const zend_op *exit_opline = NULL; 3312 uint32_t exit_point; 3313 const void *exit_addr; 3314 uint32_t old_info = 0; 3315 uint32_t old_res_info = 0; 3316 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 3317 3318 if (zend_is_smart_branch(opline)) { 3319 bool exit_if_true = 0; 3320 exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); 3321 } else { 3322 switch (opline->opcode) { 3323 case ZEND_JMPZ: 3324 case ZEND_JMPNZ: 3325 case ZEND_JMPZ_EX: 3326 case ZEND_JMPNZ_EX: 3327 case ZEND_JMP_SET: 3328 case ZEND_COALESCE: 3329 case ZEND_JMP_NULL: 3330 case ZEND_FE_RESET_R: 3331 case ZEND_FE_RESET_RW: 3332 exit_opline = (trace->opline == opline + 1) ? 3333 OP_JMP_ADDR(opline, opline->op2) : 3334 opline + 1; 3335 break; 3336 case ZEND_JMPZNZ: 3337 exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ? 3338 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3339 OP_JMP_ADDR(opline, opline->op2); 3340 break; 3341 case ZEND_FE_FETCH_R: 3342 case ZEND_FE_FETCH_RW: 3343 exit_opline = (trace->opline == opline + 1) ? 3344 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3345 opline + 1; 3346 break; 3347 3348 } 3349 } 3350 3351 switch (opline->opcode) { 3352 case ZEND_FE_FETCH_R: 3353 case ZEND_FE_FETCH_RW: 3354 if (opline->op2_type != IS_UNUSED) { 3355 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); 3356 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); 3357 } 3358 break; 3359 } 3360 3361 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3362 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3363 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 3364 } 3365 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); 3366 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3367 3368 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3369 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3370 } 3371 switch (opline->opcode) { 3372 case ZEND_FE_FETCH_R: 3373 case ZEND_FE_FETCH_RW: 3374 if (opline->op2_type != IS_UNUSED) { 3375 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); 3376 } 3377 break; 3378 } 3379 3380 if (!exit_addr) { 3381 return 0; 3382 } 3383 | CMP_IP next_opline, TMP1, TMP2 3384 | bne &exit_addr 3385 } 3386 } 3387 3388 zend_jit_set_last_valid_opline(trace->opline); 3389 3390 return 1; 3391} 3392 3393static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) 3394{ 3395 const void *handler; 3396 3397 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3398 handler = zend_get_opcode_handler_func(opline); 3399 } else { 3400 handler = opline->handler; 3401 } 3402 3403 if (!zend_jit_set_valid_ip(Dst, opline)) { 3404 return 0; 3405 } 3406 if (!GCC_GLOBAL_REGS) { 3407 | mov FCARG1x, FP 3408 } 3409 | EXT_CALL handler, REG0 3410 if (may_throw) { 3411 zend_jit_check_exception(Dst); 3412 } 3413 3414 /* Skip the following OP_DATA */ 3415 switch (opline->opcode) { 3416 case ZEND_ASSIGN_DIM: 3417 case ZEND_ASSIGN_OBJ: 3418 case ZEND_ASSIGN_STATIC_PROP: 3419 case ZEND_ASSIGN_DIM_OP: 3420 case ZEND_ASSIGN_OBJ_OP: 3421 case ZEND_ASSIGN_STATIC_PROP_OP: 3422 case ZEND_ASSIGN_STATIC_PROP_REF: 3423 case ZEND_ASSIGN_OBJ_REF: 3424 zend_jit_set_last_valid_opline(opline + 2); 3425 break; 3426 default: 3427 zend_jit_set_last_valid_opline(opline + 1); 3428 break; 3429 } 3430 3431 return 1; 3432} 3433 3434static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) 3435{ 3436 if (!zend_jit_set_valid_ip(Dst, opline)) { 3437 return 0; 3438 } 3439 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3440 if (opline->opcode == ZEND_DO_UCALL || 3441 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3442 opline->opcode == ZEND_DO_FCALL || 3443 opline->opcode == ZEND_RETURN) { 3444 3445 /* Use inlined HYBRID VM handler */ 3446 const void *handler = opline->handler; 3447 3448 | ADD_HYBRID_SPAD 3449 | EXT_JMP handler, REG0 3450 } else { 3451 const void *handler = zend_get_opcode_handler_func(opline); 3452 3453 | EXT_CALL handler, REG0 3454 | ADD_HYBRID_SPAD 3455 | JMP_IP TMP1 3456 } 3457 } else { 3458 const void *handler = opline->handler; 3459 3460 if (GCC_GLOBAL_REGS) { 3461 | ldp x29, x30, [sp], # SPAD // stack alignment 3462 } else { 3463 | mov FCARG1x, FP 3464 | ldp FP, RX, T2 // retore FP and IP 3465 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 3466 } 3467 | EXT_JMP handler, REG0 3468 } 3469 zend_jit_reset_last_valid_opline(); 3470 return 1; 3471} 3472 3473static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) 3474{ 3475 uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); 3476 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3477 3478 if (!exit_addr) { 3479 return 0; 3480 } 3481 | CMP_IP opline, TMP1, TMP2 3482 | bne &exit_addr 3483 3484 zend_jit_set_last_valid_opline(opline); 3485 3486 return 1; 3487} 3488 3489static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) 3490{ 3491 | b =>target_label 3492 return 1; 3493} 3494 3495static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) 3496{ 3497 | CMP_IP next_opline, TMP1, TMP2 3498 | bne =>target_label 3499 3500 zend_jit_set_last_valid_opline(next_opline); 3501 3502 return 1; 3503} 3504 3505#ifdef CONTEXT_THREADED_JIT 3506static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3507{ 3508 | NIY // TODO 3509 return 1; 3510} 3511#endif 3512 3513static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3514{ 3515#ifdef CONTEXT_THREADED_JIT 3516 return zend_jit_context_threaded_call(Dst, opline, next_block); 3517#else 3518 return zend_jit_tail_handler(Dst, opline); 3519#endif 3520} 3521 3522static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type) 3523{ 3524 ZEND_ASSERT(Z_MODE(src) == IS_REG); 3525 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); 3526 3527 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3528 | SET_ZVAL_LVAL_FROM_REG dst, Rx(Z_REG(src)), TMP1 3529 if (set_type && 3530 (Z_REG(dst) != ZREG_FP || 3531 !JIT_G(current_frame) || 3532 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) { 3533 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 3534 } 3535 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3536 | SET_ZVAL_DVAL dst, Z_REG(src), ZREG_TMP1 3537 if (set_type && 3538 (Z_REG(dst) != ZREG_FP || 3539 !JIT_G(current_frame) || 3540 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) { 3541 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 3542 } 3543 } else { 3544 ZEND_UNREACHABLE(); 3545 } 3546 return 1; 3547} 3548 3549static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3550{ 3551 ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); 3552 ZEND_ASSERT(Z_MODE(dst) == IS_REG); 3553 3554 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3555 | GET_ZVAL_LVAL Z_REG(dst), src, TMP1 3556 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3557 | GET_ZVAL_DVAL Z_REG(dst), src, ZREG_TMP1 3558 } else { 3559 ZEND_UNREACHABLE(); 3560 } 3561 return 1; 3562} 3563 3564static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type) 3565{ 3566 zend_jit_addr src = ZEND_ADDR_REG(reg); 3567 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3568 3569 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3570} 3571 3572static int zend_jit_store_var_type(dasm_State **Dst, int var, uint32_t type) 3573{ 3574 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3575 3576 | SET_ZVAL_TYPE_INFO dst, type, TMP1w, TMP2 3577 return 1; 3578} 3579 3580static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) 3581{ 3582 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3583 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3584 return zend_jit_spill_store(Dst, src, dst, info, 1); 3585 } 3586 return 1; 3587} 3588 3589static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) 3590{ 3591 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3592 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3593 bool set_type = 1; 3594 3595 if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == 3596 (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { 3597 if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { 3598 set_type = 0; 3599 } 3600 } 3601 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3602 } 3603 return 1; 3604} 3605 3606static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) 3607{ 3608 zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3609 zend_jit_addr dst = ZEND_ADDR_REG(reg); 3610 3611 return zend_jit_load_reg(Dst, src, dst, info); 3612} 3613 3614static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, zend_uchar op_type, zend_jit_addr addr, znode_op op) 3615{ 3616 if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) { 3617 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 3618 | SET_ZVAL_TYPE_INFO dst, IS_UNDEF, TMP1w, TMP2 3619 } 3620 return 1; 3621} 3622 3623static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3624{ 3625 if (!zend_jit_same_addr(src, dst)) { 3626 if (Z_MODE(src) == IS_REG) { 3627 if (Z_MODE(dst) == IS_REG) { 3628 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3629 | mov Rx(Z_REG(dst)), Rx(Z_REG(src)) 3630 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3631 | fmov Rd(Z_REG(dst)-ZREG_V0), Rd(Z_REG(src)-ZREG_V0) 3632 } else { 3633 ZEND_UNREACHABLE(); 3634 } 3635 if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) { 3636 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3637 3638 if (!zend_jit_spill_store(Dst, dst, var_addr, info, 3639 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3640 JIT_G(current_frame) == NULL || 3641 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3642 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3643 )) { 3644 return 0; 3645 } 3646 } 3647 } else if (Z_MODE(dst) == IS_MEM_ZVAL) { 3648 if (!Z_LOAD(src) && !Z_STORE(src)) { 3649 if (!zend_jit_spill_store(Dst, src, dst, info, 3650 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3651 JIT_G(current_frame) == NULL || 3652 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3653 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3654 )) { 3655 return 0; 3656 } 3657 } 3658 } else { 3659 ZEND_UNREACHABLE(); 3660 } 3661 } else if (Z_MODE(src) == IS_MEM_ZVAL) { 3662 if (Z_MODE(dst) == IS_REG) { 3663 if (!zend_jit_load_reg(Dst, src, dst, info)) { 3664 return 0; 3665 } 3666 } else { 3667 ZEND_UNREACHABLE(); 3668 } 3669 } else { 3670 ZEND_UNREACHABLE(); 3671 } 3672 } else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) { 3673 dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3674 if (!zend_jit_spill_store(Dst, src, dst, info, 3675 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3676 JIT_G(current_frame) == NULL || 3677 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3678 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3679 )) { 3680 return 0; 3681 } 3682 } 3683 return 1; 3684} 3685 3686static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) 3687{ 3688 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 3689 3690 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 3691 3692 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { 3693 if (!zend_jit_save_call_chain(Dst, -1)) { 3694 return 0; 3695 } 3696 } 3697 3698 ZEND_ASSERT(opline); 3699 3700 if ((opline-1)->opcode != ZEND_FETCH_CONSTANT 3701 && (opline-1)->opcode != ZEND_FETCH_LIST_R 3702 && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR)) 3703 && !(flags & ZEND_JIT_EXIT_FREE_OP1)) { 3704 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var); 3705 3706 | IF_NOT_ZVAL_REFCOUNTED val_addr, >2, ZREG_TMP1, ZREG_TMP2 3707 | GET_ZVAL_PTR TMP1, val_addr, TMP2 3708 | GC_ADDREF TMP1, TMP2w 3709 |2: 3710 } 3711 3712 | LOAD_IP_ADDR (opline - 1) 3713 | b ->trace_escape 3714 |1: 3715 3716 return 1; 3717} 3718 3719static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) 3720{ 3721 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3722 3723 if (reg == ZREG_LONG_MIN_MINUS_1) { 3724 uint64_t val = 0xc3e0000000000000; 3725 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3726 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 3727 } else if (reg == ZREG_LONG_MIN) { 3728 uint64_t val = 0x8000000000000000; 3729 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3730 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 3731 } else if (reg == ZREG_LONG_MAX) { 3732 uint64_t val = 0x7fffffffffffffff; 3733 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3734 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 3735 } else if (reg == ZREG_LONG_MAX_PLUS_1) { 3736 uint64_t val = 0x43e0000000000000; 3737 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3738 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 3739 } else if (reg == ZREG_NULL) { 3740 | SET_ZVAL_TYPE_INFO dst, IS_NULL, TMP1w, TMP2 3741 } else if (reg == ZREG_ZVAL_TRY_ADDREF) { 3742 | IF_NOT_ZVAL_REFCOUNTED dst, >1, ZREG_TMP1, ZREG_TMP2 3743 | GET_ZVAL_PTR TMP1, dst, TMP2 3744 | GC_ADDREF TMP1, TMP2w 3745 |1: 3746 } else if (reg == ZREG_ZVAL_COPY_GPR0) { 3747 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 3748 3749 | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3750 | TRY_ADDREF -1, REG1w, REG2, TMP1w 3751 } else { 3752 ZEND_UNREACHABLE(); 3753 } 3754 return 1; 3755} 3756 3757static int zend_jit_free_trampoline(dasm_State **Dst) 3758{ 3759 | // if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) 3760 | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] 3761 | TST_32_WITH_CONST TMP1w, ZEND_ACC_CALL_VIA_TRAMPOLINE, TMP2w 3762 | beq >1 3763 | mov FCARG1x, REG0 3764 | EXT_CALL zend_jit_free_trampoline_helper, REG0 3765 |1: 3766 return 1; 3767} 3768 3769static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 3770{ 3771 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { 3772 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 3773 } 3774 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 3775 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3776 } 3777 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { 3778 return 0; 3779 } 3780 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3781 | LONG_ADD_SUB_WITH_IMM adds, op1_def_addr, Z_L(1), TMP1, TMP2 3782 } else { 3783 | LONG_ADD_SUB_WITH_IMM subs, op1_def_addr, Z_L(1), TMP1, TMP2 3784 } 3785 3786 if (may_overflow && 3787 (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || 3788 ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { 3789 int32_t exit_point; 3790 const void *exit_addr; 3791 zend_jit_trace_stack *stack; 3792 uint32_t old_op1_info, old_res_info = 0; 3793 3794 stack = JIT_G(current_frame)->stack; 3795 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 3796 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); 3797 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3798 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); 3799 } else { 3800 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); 3801 } 3802 if (opline->result_type != IS_UNUSED) { 3803 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3804 if (opline->opcode == ZEND_PRE_INC) { 3805 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 3806 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); 3807 } else if (opline->opcode == ZEND_PRE_DEC) { 3808 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 3809 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); 3810 } else if (opline->opcode == ZEND_POST_INC) { 3811 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 3812 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); 3813 } else if (opline->opcode == ZEND_POST_DEC) { 3814 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 3815 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); 3816 } 3817 } 3818 3819 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 3820 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3821 if (!exit_addr) { 3822 return 0; 3823 } 3824 | bvs &exit_addr 3825 3826 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3827 opline->result_type != IS_UNUSED) { 3828 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3829 } 3830 3831 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 3832 if (opline->result_type != IS_UNUSED) { 3833 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3834 } 3835 } else if (may_overflow) { 3836 | bvs >1 3837 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3838 opline->result_type != IS_UNUSED) { 3839 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3840 } 3841 |.cold_code 3842 |1: 3843 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3844 uint64_t val = 0x43e0000000000000; 3845 if (Z_MODE(op1_def_addr) == IS_REG) { 3846 | LOAD_64BIT_VAL TMP1, val 3847 | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 3848 } else { 3849 | SET_ZVAL_LVAL op1_def_addr, val, TMP2, TMP1 3850 } 3851 } else { 3852 uint64_t val = 0xc3e0000000000000; 3853 if (Z_MODE(op1_def_addr) == IS_REG) { 3854 | LOAD_64BIT_VAL TMP1, val 3855 | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 3856 } else { 3857 | SET_ZVAL_LVAL op1_def_addr, val, TMP2, TMP1 3858 } 3859 } 3860 if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { 3861 | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE, TMP1w, TMP2 3862 } 3863 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3864 opline->result_type != IS_UNUSED) { 3865 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3866 } 3867 | b >3 3868 |.code 3869 } else { 3870 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3871 opline->result_type != IS_UNUSED) { 3872 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3873 } 3874 } 3875 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 3876 |.cold_code 3877 |2: 3878 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 3879 | SET_EX_OPLINE opline, REG0 3880 if (op1_info & MAY_BE_UNDEF) { 3881 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2, ZREG_TMP1 3882 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 3883 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 3884 | EXT_CALL zend_jit_undefined_op_helper, REG0 3885 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 3886 op1_info |= MAY_BE_NULL; 3887 } 3888 |2: 3889 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 3890 3891 | // ZVAL_DEREF(var_ptr); 3892 if (op1_info & MAY_BE_REF) { 3893 | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >2, TMP1w 3894 | GET_Z_PTR FCARG1x, FCARG1x 3895 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 3896 | cbz TMP1, >1 3897 if (RETURN_VALUE_USED(opline)) { 3898 | LOAD_ZVAL_ADDR FCARG2x, res_addr 3899 } else { 3900 | mov FCARG2x, xzr 3901 } 3902 if (opline->opcode == ZEND_PRE_INC) { 3903 | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 3904 } else if (opline->opcode == ZEND_PRE_DEC) { 3905 | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 3906 } else if (opline->opcode == ZEND_POST_INC) { 3907 | EXT_CALL zend_jit_post_inc_typed_ref, REG0 3908 } else if (opline->opcode == ZEND_POST_DEC) { 3909 | EXT_CALL zend_jit_post_dec_typed_ref, REG0 3910 } else { 3911 ZEND_UNREACHABLE(); 3912 } 3913 zend_jit_check_exception(Dst); 3914 | b >3 3915 |1: 3916 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) 3917 |2: 3918 } 3919 3920 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 3921 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 3922 3923 | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3924 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 3925 } 3926 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3927 if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { 3928 | LOAD_ZVAL_ADDR FCARG2x, res_addr 3929 | EXT_CALL zend_jit_pre_inc, REG0 3930 } else { 3931 | EXT_CALL increment_function, REG0 3932 } 3933 } else { 3934 if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { 3935 | LOAD_ZVAL_ADDR FCARG2x, res_addr 3936 | EXT_CALL zend_jit_pre_dec, REG0 3937 } else { 3938 | EXT_CALL decrement_function, REG0 3939 } 3940 } 3941 if (may_throw) { 3942 zend_jit_check_exception(Dst); 3943 } 3944 } else { 3945 zend_reg tmp_reg; 3946 3947 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 3948 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3949 } 3950 if (Z_MODE(op1_def_addr) == IS_REG) { 3951 tmp_reg = Z_REG(op1_def_addr); 3952 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 3953 tmp_reg = Z_REG(op1_addr); 3954 } else { 3955 tmp_reg = ZREG_FPR0; 3956 } 3957 | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 3958 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3959 uint64_t val = 0x3ff0000000000000; // 1.0 3960 | LOAD_64BIT_VAL TMP1, val 3961 | fmov FPTMP, TMP1 3962 | fadd Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP 3963 } else { 3964 uint64_t val = 0x3ff0000000000000; // 1.0 3965 | LOAD_64BIT_VAL TMP1, val 3966 | fmov FPTMP, TMP1 3967 | fsub Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP 3968 } 3969 | SET_ZVAL_DVAL op1_def_addr, tmp_reg, ZREG_TMP1 3970 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3971 opline->result_type != IS_UNUSED) { 3972 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3973 | TRY_ADDREF op1_def_info, REG0w, REG1, TMP1w 3974 } 3975 } 3976 | b >3 3977 |.code 3978 } 3979 |3: 3980 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { 3981 return 0; 3982 } 3983 if (opline->result_type != IS_UNUSED) { 3984 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 3985 return 0; 3986 } 3987 } 3988 return 1; 3989} 3990 3991static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) 3992{ 3993 if ((opline+1)->opcode == ZEND_OP_DATA 3994 && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) 3995 && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { 3996 return 1; 3997 } 3998 return 3999 ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4000 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || 4001 ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4002 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || 4003 ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4004 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); 4005} 4006 4007static int zend_jit_math_long_long(dasm_State **Dst, 4008 const zend_op *opline, 4009 zend_uchar opcode, 4010 zend_jit_addr op1_addr, 4011 zend_jit_addr op2_addr, 4012 zend_jit_addr res_addr, 4013 uint32_t res_info, 4014 uint32_t res_use_info, 4015 int may_overflow) 4016{ 4017 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4018 zend_reg result_reg; 4019 zend_reg tmp_reg = ZREG_REG0; 4020 bool use_ovf_flag = 1; 4021 4022 if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { 4023 if (may_overflow && (res_info & MAY_BE_GUARD) 4024 && JIT_G(current_frame) 4025 && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { 4026 result_reg = ZREG_REG0; 4027 } else { 4028 result_reg = Z_REG(res_addr); 4029 } 4030 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { 4031 result_reg = Z_REG(op1_addr); 4032 } else if (Z_REG(res_addr) != ZREG_REG0) { 4033 result_reg = ZREG_REG0; 4034 } else { 4035 /* ASSIGN_DIM_OP */ 4036 result_reg = ZREG_FCARG1; 4037 tmp_reg = ZREG_FCARG1; 4038 } 4039 4040 if (opcode == ZEND_MUL && 4041 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4042 Z_LVAL_P(Z_ZV(op2_addr)) == 2) { 4043 if (Z_MODE(op1_addr) == IS_REG) { 4044 | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) 4045 } else { 4046 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4047 | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) 4048 } 4049 } else if (opcode == ZEND_MUL && 4050 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4051 !may_overflow && 4052 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { 4053 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4054 | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4055 | lsl Rx(result_reg), Rx(result_reg), TMP1 4056 } else if (opcode == ZEND_MUL && 4057 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4058 Z_LVAL_P(Z_ZV(op1_addr)) == 2) { 4059 if (Z_MODE(op2_addr) == IS_REG) { 4060 | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) 4061 } else { 4062 | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 4063 | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) 4064 } 4065 } else if (opcode == ZEND_MUL && 4066 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4067 !may_overflow && 4068 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { 4069 | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 4070 | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) 4071 | lsl Rx(result_reg), Rx(result_reg), TMP1 4072 } else if (opcode == ZEND_DIV && 4073 (Z_MODE(op2_addr) == IS_CONST_ZVAL && 4074 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { 4075 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4076 | asr Rx(result_reg), Rx(result_reg), #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4077#if 0 4078 /* x86 specific optimizations through LEA instraction are not supported on ARM */ 4079 } else if (opcode == ZEND_ADD && 4080 !may_overflow && 4081 Z_MODE(op1_addr) == IS_REG && 4082 Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4083 | NIY // TODO: test 4084 } else if (opcode == ZEND_ADD && 4085 !may_overflow && 4086 Z_MODE(op2_addr) == IS_REG && 4087 Z_MODE(op1_addr) == IS_CONST_ZVAL) { 4088 | NIY // TODO: test 4089 } else if (opcode == ZEND_SUB && 4090 !may_overflow && 4091 Z_MODE(op1_addr) == IS_REG && 4092 Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4093 | NIY // TODO: test 4094#endif 4095 } else if (opcode == ZEND_MUL) { 4096 | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 4097 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4098 | mul Rx(result_reg), TMP1, TMP2 4099 if(may_overflow) { 4100 /* Use 'smulh' to get the upper 64 bits fo the 128-bit result. 4101 * For signed multiplication, the top 65 bits of the result will contain 4102 * either all zeros or all ones if no overflow occurred. 4103 * Flag: bne -> overflow. beq -> no overflow. 4104 */ 4105 use_ovf_flag = 0; 4106 | smulh TMP1, TMP1, TMP2 4107 | cmp TMP1, Rx(result_reg), asr #63 4108 } 4109 } else { 4110 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4111 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4112 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4113 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4114 /* +/- 0 */ 4115 may_overflow = 0; 4116 } else if (same_ops && opcode != ZEND_DIV) { 4117 | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) 4118 } else { 4119 | LONG_MATH opcode, result_reg, op2_addr, TMP1 4120 } 4121 } 4122 if (may_overflow) { 4123 if (res_info & MAY_BE_GUARD) { 4124 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 4125 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4126 if (!exit_addr) { 4127 return 0; 4128 } 4129 if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { 4130 if (use_ovf_flag) { 4131 | bvs &exit_addr 4132 } else { 4133 | bne &exit_addr 4134 } 4135 if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { 4136 | mov Rx(Z_REG(res_addr)), Rx(result_reg) 4137 } 4138 } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 4139 if (use_ovf_flag) { 4140 | bvc &exit_addr 4141 } else { 4142 | beq &exit_addr 4143 } 4144 } else { 4145 ZEND_UNREACHABLE(); 4146 } 4147 } else { 4148 if (res_info & MAY_BE_LONG) { 4149 if (use_ovf_flag) { 4150 | bvs >1 4151 } else { 4152 | bne >1 4153 } 4154 } else { 4155 if (use_ovf_flag) { 4156 | bvc >1 4157 } else { 4158 | beq >1 4159 } 4160 } 4161 } 4162 } 4163 4164 if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { 4165 | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 4166 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4167 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4168 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 4169 } 4170 } 4171 } 4172 4173 if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { 4174 zend_reg tmp_reg1 = ZREG_FPR0; 4175 zend_reg tmp_reg2 = ZREG_FPR1; 4176 4177 if (res_info & MAY_BE_LONG) { 4178 |.cold_code 4179 |1: 4180 } 4181 4182 do { 4183 if ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || 4184 (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)) { 4185 if (opcode == ZEND_ADD) { 4186 uint64_t val = 0x43e0000000000000; 4187 if (Z_MODE(res_addr) == IS_REG) { 4188 | LOAD_64BIT_VAL TMP1, val 4189 | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 4190 } else { 4191 | SET_ZVAL_LVAL res_addr, val, TMP2, TMP1 4192 } 4193 break; 4194 } else if (opcode == ZEND_SUB) { 4195 uint64_t val = 0xc3e0000000000000; 4196 if (Z_MODE(res_addr) == IS_REG) { 4197 | LOAD_64BIT_VAL TMP1, val 4198 | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 4199 } else { 4200 | SET_ZVAL_LVAL res_addr, val, TMP2, TMP1 4201 } 4202 break; 4203 } 4204 } 4205 4206 | DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg, ZREG_TMP1 4207 | DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg, ZREG_TMP1 4208 | DOUBLE_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 4209 | SET_ZVAL_DVAL res_addr, tmp_reg1, ZREG_TMP1 4210 } while (0); 4211 4212 if (Z_MODE(res_addr) == IS_MEM_ZVAL 4213 && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4214 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4215 } 4216 if (res_info & MAY_BE_LONG) { 4217 | b >2 4218 |.code 4219 } 4220 |2: 4221 } 4222 4223 return 1; 4224} 4225 4226static int zend_jit_math_long_double(dasm_State **Dst, 4227 zend_uchar opcode, 4228 zend_jit_addr op1_addr, 4229 zend_jit_addr op2_addr, 4230 zend_jit_addr res_addr, 4231 uint32_t res_use_info) 4232{ 4233 zend_reg result_reg = 4234 (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_FPR0; 4235 zend_reg op2_reg; 4236 4237 | DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, ZREG_TMP1, ZREG_TMP2 4238 4239 if (Z_MODE(op2_addr) == IS_REG) { 4240 op2_reg = Z_REG(op2_addr); 4241 } else { 4242 op2_reg = ZREG_FPTMP; 4243 | GET_ZVAL_DVAL op2_reg, op2_addr, ZREG_TMP1 4244 } 4245 4246 | DOUBLE_MATH_REG opcode, result_reg, result_reg, op2_reg 4247 4248 | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 4249 4250 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4251 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4252 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4253 } 4254 } 4255 4256 return 1; 4257} 4258 4259static int zend_jit_math_double_long(dasm_State **Dst, 4260 zend_uchar opcode, 4261 zend_jit_addr op1_addr, 4262 zend_jit_addr op2_addr, 4263 zend_jit_addr res_addr, 4264 uint32_t res_use_info) 4265{ 4266 zend_reg result_reg, op1_reg, op2_reg; 4267 4268 if (zend_is_commutative(opcode) 4269 && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { 4270 if (Z_MODE(res_addr) == IS_REG) { 4271 result_reg = Z_REG(res_addr); 4272 } else { 4273 result_reg = ZREG_FPR0; 4274 } 4275 | DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 4276 if (Z_MODE(op1_addr) == IS_REG) { 4277 op1_reg = Z_REG(op1_addr); 4278 } else { 4279 op1_reg = ZREG_FPTMP; 4280 | GET_ZVAL_DVAL op1_reg, op1_addr, ZREG_TMP1 4281 } 4282 | DOUBLE_MATH_REG opcode, result_reg, result_reg, op1_reg 4283 } else { 4284 if (Z_MODE(res_addr) == IS_REG) { 4285 result_reg = Z_REG(res_addr); 4286 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4287 result_reg = Z_REG(op1_addr); 4288 } else { 4289 result_reg = ZREG_FPR0; 4290 } 4291 4292 if (Z_MODE(op1_addr) == IS_REG) { 4293 op1_reg = Z_REG(op1_addr); 4294 } else { 4295 | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 4296 op1_reg = result_reg; 4297 } 4298 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4299 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4300 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4301 /* +/- 0 */ 4302 } else { 4303 op2_reg = ZREG_FPTMP; 4304 | DOUBLE_GET_ZVAL_LVAL op2_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 4305 | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg 4306 } 4307 } 4308 4309 | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 4310 4311 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4312 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4313 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4314 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4315 } 4316 } 4317 } 4318 4319 return 1; 4320} 4321 4322static int zend_jit_math_double_double(dasm_State **Dst, 4323 zend_uchar opcode, 4324 zend_jit_addr op1_addr, 4325 zend_jit_addr op2_addr, 4326 zend_jit_addr res_addr, 4327 uint32_t res_use_info) 4328{ 4329 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4330 zend_reg result_reg, op1_reg, op2_reg; 4331 zend_jit_addr val_addr; 4332 4333 if (Z_MODE(res_addr) == IS_REG) { 4334 result_reg = Z_REG(res_addr); 4335 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4336 result_reg = Z_REG(op1_addr); 4337 } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { 4338 result_reg = Z_REG(op2_addr); 4339 } else { 4340 result_reg = ZREG_FPR0; 4341 } 4342 4343 if (Z_MODE(op1_addr) == IS_REG) { 4344 op1_reg = Z_REG(op1_addr); 4345 val_addr = op2_addr; 4346 } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4347 op1_reg = Z_REG(op2_addr); 4348 val_addr = op1_addr; 4349 } else { 4350 | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 4351 op1_reg = result_reg; 4352 val_addr = op2_addr; 4353 } 4354 4355 if ((opcode == ZEND_MUL) && 4356 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4357 | DOUBLE_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg 4358 } else { 4359 if (same_ops) { 4360 op2_reg = op1_reg; 4361 } else if (Z_MODE(val_addr) == IS_REG) { 4362 op2_reg = Z_REG(val_addr); 4363 } else { 4364 op2_reg = ZREG_FPTMP; 4365 | GET_ZVAL_DVAL op2_reg, val_addr, ZREG_TMP1 4366 } 4367 | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg 4368 } 4369 4370 | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 4371 4372 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4373 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4374 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4375 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4376 } 4377 } 4378 } 4379 return 1; 4380} 4381 4382static int zend_jit_math_helper(dasm_State **Dst, 4383 const zend_op *opline, 4384 zend_uchar opcode, 4385 zend_uchar op1_type, 4386 znode_op op1, 4387 zend_jit_addr op1_addr, 4388 uint32_t op1_info, 4389 zend_uchar op2_type, 4390 znode_op op2, 4391 zend_jit_addr op2_addr, 4392 uint32_t op2_info, 4393 uint32_t res_var, 4394 zend_jit_addr res_addr, 4395 uint32_t res_info, 4396 uint32_t res_use_info, 4397 int may_overflow, 4398 int may_throw) 4399/* Labels: 1,2,3,4,5,6 */ 4400{ 4401 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4402 4403 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4404 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { 4405 if (op1_info & MAY_BE_DOUBLE) { 4406 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 4407 } else { 4408 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 4409 } 4410 } 4411 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { 4412 if (op2_info & MAY_BE_DOUBLE) { 4413 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1, ZREG_TMP1 4414 |.cold_code 4415 |1: 4416 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4417 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4418 } 4419 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4420 return 0; 4421 } 4422 | b >5 4423 |.code 4424 } else { 4425 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4426 } 4427 } 4428 if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { 4429 return 0; 4430 } 4431 if (op1_info & MAY_BE_DOUBLE) { 4432 |.cold_code 4433 |3: 4434 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4435 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 4436 } 4437 if (op2_info & MAY_BE_DOUBLE) { 4438 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4439 if (!same_ops) { 4440 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1, ZREG_TMP1 4441 } else { 4442 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4443 } 4444 } 4445 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4446 return 0; 4447 } 4448 | b >5 4449 } 4450 if (!same_ops) { 4451 |1: 4452 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4453 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4454 } 4455 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4456 return 0; 4457 } 4458 | b >5 4459 } 4460 |.code 4461 } 4462 } else if ((op1_info & MAY_BE_DOUBLE) && 4463 !(op1_info & MAY_BE_LONG) && 4464 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4465 (res_info & MAY_BE_DOUBLE)) { 4466 if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4467 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 4468 } 4469 if (op2_info & MAY_BE_DOUBLE) { 4470 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4471 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4472 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1, ZREG_TMP1 4473 } else { 4474 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4475 } 4476 } 4477 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4478 return 0; 4479 } 4480 } 4481 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4482 if (op2_info & MAY_BE_DOUBLE) { 4483 |.cold_code 4484 } 4485 |1: 4486 if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4487 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4488 } 4489 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4490 return 0; 4491 } 4492 if (op2_info & MAY_BE_DOUBLE) { 4493 | b >5 4494 |.code 4495 } 4496 } 4497 } else if ((op2_info & MAY_BE_DOUBLE) && 4498 !(op2_info & MAY_BE_LONG) && 4499 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4500 (res_info & MAY_BE_DOUBLE)) { 4501 if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4502 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4503 } 4504 if (op1_info & MAY_BE_DOUBLE) { 4505 if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4506 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4507 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1, ZREG_TMP1 4508 } else { 4509 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 4510 } 4511 } 4512 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4513 return 0; 4514 } 4515 } 4516 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4517 if (op1_info & MAY_BE_DOUBLE) { 4518 |.cold_code 4519 } 4520 |1: 4521 if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4522 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 4523 } 4524 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4525 return 0; 4526 } 4527 if (op1_info & MAY_BE_DOUBLE) { 4528 | b >5 4529 |.code 4530 } 4531 } 4532 } 4533 4534 |5: 4535 4536 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 4537 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 4538 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4539 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4540 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4541 |.cold_code 4542 } 4543 |6: 4544 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4545 if (Z_MODE(res_addr) == IS_REG) { 4546 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4547 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4548 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4549 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4550 } 4551 if (Z_MODE(op1_addr) == IS_REG) { 4552 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4553 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4554 return 0; 4555 } 4556 op1_addr = real_addr; 4557 } 4558 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4559 } else { 4560 if (Z_MODE(op1_addr) == IS_REG) { 4561 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4562 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4563 return 0; 4564 } 4565 op1_addr = real_addr; 4566 } 4567 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4568 if (Z_MODE(res_addr) == IS_REG) { 4569 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4570 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4571 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4572 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4573 } 4574 } 4575 if (Z_MODE(op2_addr) == IS_REG) { 4576 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 4577 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 4578 return 0; 4579 } 4580 op2_addr = real_addr; 4581 } 4582 | LOAD_ZVAL_ADDR CARG3, op2_addr 4583 | SET_EX_OPLINE opline, REG0 4584 if (opcode == ZEND_ADD) { 4585 | EXT_CALL add_function, REG0 4586 } else if (opcode == ZEND_SUB) { 4587 | EXT_CALL sub_function, REG0 4588 } else if (opcode == ZEND_MUL) { 4589 | EXT_CALL mul_function, REG0 4590 } else if (opcode == ZEND_DIV) { 4591 | EXT_CALL div_function, REG0 4592 } else { 4593 ZEND_UNREACHABLE(); 4594 } 4595 | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4596 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4597 if (may_throw) { 4598 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 4599 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 4600 | cbnz TMP2, ->exception_handler_free_op2 4601 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 4602 zend_jit_check_exception_undef_result(Dst, opline); 4603 } else { 4604 zend_jit_check_exception(Dst); 4605 } 4606 } 4607 if (Z_MODE(res_addr) == IS_REG) { 4608 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4609 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 4610 return 0; 4611 } 4612 } 4613 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4614 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4615 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4616 | b <5 4617 |.code 4618 } 4619 } 4620 4621 return 1; 4622} 4623 4624static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 4625{ 4626 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 4627 ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4628 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); 4629 4630 if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { 4631 return 0; 4632 } 4633 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4634 return 0; 4635 } 4636 return 1; 4637} 4638 4639static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr) 4640{ 4641 if (Z_MODE(op2_addr) != IS_MEM_ZVAL || Z_REG(op2_addr) != ZREG_FCARG1) { 4642 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 4643 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 4644 } else if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4645 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 4646 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 4647 } else { 4648 | GET_ZVAL_LVAL ZREG_REG0, op2_addr, TMP1 4649 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 4650 | mov FCARG2x, REG0 4651 } 4652 | EXT_CALL zend_jit_add_arrays_helper, REG0 4653 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 4654 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX, TMP1w, TMP2 4655 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 4656 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 4657 return 1; 4658} 4659 4660static int zend_jit_long_math_helper(dasm_State **Dst, 4661 const zend_op *opline, 4662 zend_uchar opcode, 4663 zend_uchar op1_type, 4664 znode_op op1, 4665 zend_jit_addr op1_addr, 4666 uint32_t op1_info, 4667 zend_ssa_range *op1_range, 4668 zend_uchar op2_type, 4669 znode_op op2, 4670 zend_jit_addr op2_addr, 4671 uint32_t op2_info, 4672 zend_ssa_range *op2_range, 4673 uint32_t res_var, 4674 zend_jit_addr res_addr, 4675 uint32_t res_info, 4676 uint32_t res_use_info, 4677 int may_throw) 4678/* Labels: 6 */ 4679{ 4680 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4681 zend_reg result_reg; 4682 4683 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 4684 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 4685 } 4686 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 4687 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4688 } 4689 4690 if (Z_MODE(res_addr) == IS_REG) { 4691 if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) 4692 && opline->op2_type != IS_CONST) { 4693 result_reg = ZREG_REG0; 4694 } else { 4695 result_reg = Z_REG(res_addr); 4696 } 4697 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4698 result_reg = Z_REG(op1_addr); 4699 } else if (Z_REG(res_addr) != ZREG_REG0) { 4700 result_reg = ZREG_REG0; 4701 } else { 4702 /* ASSIGN_DIM_OP */ 4703 result_reg = ZREG_FCARG1; 4704 } 4705 4706 if (opcode == ZEND_SL) { 4707 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4708 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 4709 4710 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 4711 if (EXPECTED(op2_lval > 0)) { 4712 | mov Rx(result_reg), xzr 4713 } else { 4714 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4715 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4716 | SET_EX_OPLINE opline, REG0 4717 | b ->negative_shift 4718 } 4719 } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { 4720 | add Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) 4721 } else { 4722 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4723 | lsl Rx(result_reg), Rx(result_reg), #op2_lval 4724 } 4725 } else { 4726 zend_reg op2_reg; 4727 4728 if (Z_MODE(op2_addr) == IS_REG) { 4729 op2_reg = Z_REG(op2_addr); 4730 } else { 4731 op2_reg = ZREG_TMP2; 4732 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4733 } 4734 if (!op2_range || 4735 op2_range->min < 0 || 4736 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 4737 4738 | cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8) 4739 | bhs >1 4740 |.cold_code 4741 |1: 4742 | mov Rx(result_reg), xzr 4743 | cmp Rx(op2_reg), xzr 4744 | bgt >1 4745 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4746 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4747 | SET_EX_OPLINE opline, REG0 4748 | b ->negative_shift 4749 |.code 4750 } 4751 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4752 | lsl Rx(result_reg), Rx(result_reg), Rx(op2_reg) 4753 |1: 4754 } 4755 } else if (opcode == ZEND_SR) { 4756 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4757 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4758 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 4759 4760 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 4761 if (EXPECTED(op2_lval > 0)) { 4762 | asr Rx(result_reg), Rx(result_reg), #((SIZEOF_ZEND_LONG * 8) - 1) 4763 } else { 4764 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4765 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4766 | SET_EX_OPLINE opline, REG0 4767 | b ->negative_shift 4768 } 4769 } else { 4770 | asr Rx(result_reg), Rx(result_reg), #op2_lval 4771 } 4772 } else { 4773 zend_reg op2_reg; 4774 4775 if (Z_MODE(op2_addr) == IS_REG) { 4776 op2_reg = Z_REG(op2_addr); 4777 } else { 4778 op2_reg = ZREG_TMP2; 4779 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4780 } 4781 if (!op2_range || 4782 op2_range->min < 0 || 4783 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 4784 | cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8) 4785 | bhs >1 4786 |.cold_code 4787 |1: 4788 | cmp Rx(op2_reg), xzr 4789 | mov Rx(op2_reg), #((SIZEOF_ZEND_LONG * 8) - 1) 4790 | bgt >1 4791 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4792 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4793 | SET_EX_OPLINE opline, REG0 4794 | b ->negative_shift 4795 |.code 4796 } 4797 |1: 4798 | asr Rx(result_reg), Rx(result_reg), Rx(op2_reg) 4799 } 4800 } else if (opcode == ZEND_MOD) { 4801 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4802 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 4803 4804 if (op2_lval == 0) { 4805 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4806 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4807 | SET_EX_OPLINE opline, REG0 4808 | b ->mod_by_zero 4809 } else if (op2_lval == -1) { 4810 | mov Rx(result_reg), xzr 4811 } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) { 4812 zval tmp; 4813 zend_jit_addr tmp_addr; 4814 4815 /* Optimisation for mod of power of 2 */ 4816 ZVAL_LONG(&tmp, op2_lval - 1); 4817 tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp); 4818 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4819 | LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, TMP1 4820 } else { 4821 | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 4822 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4823 | sdiv Rx(result_reg), TMP1, TMP2 4824 | msub Rx(result_reg), Rx(result_reg), TMP2, TMP1 4825 } 4826 } else { 4827 zend_reg op2_reg; 4828 4829 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 4830 | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP2, Rx(Z_REG(op2_addr)), Z_OFFSET(op2_addr), TMP2 4831 op2_reg = ZREG_TMP2; 4832 } else { 4833 ZEND_ASSERT(Z_MODE(op2_addr) == IS_REG); 4834 op2_reg = Z_REG(op2_addr); 4835 } 4836 4837 if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { 4838 | cbz Rx(op2_reg), >1 4839 |.cold_code 4840 |1: 4841 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4842 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4843 | SET_EX_OPLINE opline, REG0 4844 | b ->mod_by_zero 4845 |.code 4846 } 4847 4848 /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ 4849 if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { 4850 | cmn Rx(op2_reg), #1 4851 | beq >1 4852 |.cold_code 4853 |1: 4854 | SET_ZVAL_LVAL_FROM_REG res_addr, xzr, TMP1 4855 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4856 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4857 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4858 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 4859 } 4860 } 4861 } 4862 | b >5 4863 |.code 4864 } 4865 4866 | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 4867 | sdiv Rx(result_reg), TMP1, Rx(op2_reg) 4868 | msub Rx(result_reg), Rx(result_reg), Rx(op2_reg), TMP1 4869 } 4870 } else if (same_ops) { 4871 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4872 | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) 4873 } else { 4874 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4875 | LONG_MATH opcode, result_reg, op2_addr, TMP1 4876 } 4877 4878 if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { 4879 | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 4880 } 4881 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4882 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4883 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4884 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 4885 } 4886 } 4887 } 4888 4889 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || 4890 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 4891 if ((op1_info & MAY_BE_LONG) && 4892 (op2_info & MAY_BE_LONG)) { 4893 |.cold_code 4894 } 4895 |6: 4896 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4897 if (Z_MODE(res_addr) == IS_REG) { 4898 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4899 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4900 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4901 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4902 } 4903 if (Z_MODE(op1_addr) == IS_REG) { 4904 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4905 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4906 return 0; 4907 } 4908 op1_addr = real_addr; 4909 } 4910 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4911 } else { 4912 if (Z_MODE(op1_addr) == IS_REG) { 4913 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4914 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4915 return 0; 4916 } 4917 op1_addr = real_addr; 4918 } 4919 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4920 if (Z_MODE(res_addr) == IS_REG) { 4921 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4922 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4923 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4924 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4925 } 4926 } 4927 if (Z_MODE(op2_addr) == IS_REG) { 4928 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 4929 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 4930 return 0; 4931 } 4932 op2_addr = real_addr; 4933 } 4934 | LOAD_ZVAL_ADDR CARG3, op2_addr 4935 | SET_EX_OPLINE opline, REG0 4936 if (opcode == ZEND_BW_OR) { 4937 | EXT_CALL bitwise_or_function, REG0 4938 } else if (opcode == ZEND_BW_AND) { 4939 | EXT_CALL bitwise_and_function, REG0 4940 } else if (opcode == ZEND_BW_XOR) { 4941 | EXT_CALL bitwise_xor_function, REG0 4942 } else if (opcode == ZEND_SL) { 4943 | EXT_CALL shift_left_function, REG0 4944 } else if (opcode == ZEND_SR) { 4945 | EXT_CALL shift_right_function, REG0 4946 } else if (opcode == ZEND_MOD) { 4947 | EXT_CALL mod_function, REG0 4948 } else { 4949 ZEND_UNREACHABLE(); 4950 } 4951 if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) { 4952 /* compound assignment may decrement "op2" refcount */ 4953 op2_info |= MAY_BE_RC1; 4954 } 4955 | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4956 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4957 if (may_throw) { 4958 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 4959 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 4960 | cbnz TMP2, ->exception_handler_free_op2 4961 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 4962 zend_jit_check_exception_undef_result(Dst, opline); 4963 } else { 4964 zend_jit_check_exception(Dst); 4965 } 4966 } 4967 if (Z_MODE(res_addr) == IS_REG) { 4968 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4969 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 4970 return 0; 4971 } 4972 } 4973 if ((op1_info & MAY_BE_LONG) && 4974 (op2_info & MAY_BE_LONG)) { 4975 | b >5 4976 |.code 4977 } 4978 } 4979 |5: 4980 4981 return 1; 4982} 4983 4984static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 4985{ 4986 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 4987 ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); 4988 4989 if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, 4990 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 4991 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 4992 opline->result.var, res_addr, res_info, res_use_info, may_throw)) { 4993 return 0; 4994 } 4995 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4996 return 0; 4997 } 4998 return 1; 4999} 5000 5001static int zend_jit_concat_helper(dasm_State **Dst, 5002 const zend_op *opline, 5003 zend_uchar op1_type, 5004 znode_op op1, 5005 zend_jit_addr op1_addr, 5006 uint32_t op1_info, 5007 zend_uchar op2_type, 5008 znode_op op2, 5009 zend_jit_addr op2_addr, 5010 uint32_t op2_info, 5011 zend_jit_addr res_addr, 5012 int may_throw) 5013{ 5014 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5015 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5016 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 5017 } 5018 if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5019 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6, ZREG_TMP1 5020 } 5021 if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { 5022 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5023 | LOAD_ZVAL_ADDR FCARG1x, res_addr 5024 } 5025 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 5026 | EXT_CALL zend_jit_fast_assign_concat_helper, REG0 5027 /* concatination with itself may reduce refcount */ 5028 op2_info |= MAY_BE_RC1; 5029 } else { 5030 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5031 | LOAD_ZVAL_ADDR FCARG1x, res_addr 5032 } 5033 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 5034 | LOAD_ZVAL_ADDR CARG3, op2_addr 5035 if (op1_type == IS_CV || op1_type == IS_CONST) { 5036 | EXT_CALL zend_jit_fast_concat_helper, REG0 5037 } else { 5038 | EXT_CALL zend_jit_fast_concat_tmp_helper, REG0 5039 } 5040 } 5041 /* concatination with empty string may increase refcount */ 5042 op2_info |= MAY_BE_RCN; 5043 | FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 5044 |5: 5045 } 5046 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || 5047 (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { 5048 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5049 |.cold_code 5050 |6: 5051 } 5052 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 5053 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5054 | LOAD_ZVAL_ADDR FCARG1x, res_addr 5055 } 5056 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 5057 } else { 5058 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 5059 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5060 | LOAD_ZVAL_ADDR FCARG1x, res_addr 5061 } 5062 } 5063 | LOAD_ZVAL_ADDR CARG3, op2_addr 5064 | SET_EX_OPLINE opline, REG0 5065 | EXT_CALL concat_function, REG0 5066 /* concatination with empty string may increase refcount */ 5067 op1_info |= MAY_BE_RCN; 5068 op2_info |= MAY_BE_RCN; 5069 | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 5070 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 5071 if (may_throw) { 5072 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5073 zend_jit_check_exception_undef_result(Dst, opline); 5074 } else { 5075 zend_jit_check_exception(Dst); 5076 } 5077 } 5078 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5079 | b <5 5080 |.code 5081 } 5082 } 5083 5084 return 1; 5085} 5086 5087static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) 5088{ 5089 zend_jit_addr op1_addr, op2_addr; 5090 5091 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5092 ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); 5093 5094 op1_addr = OP1_ADDR(); 5095 op2_addr = OP2_ADDR(); 5096 5097 return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); 5098} 5099 5100static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) 5101/* Labels: 1,2,3,4,5 */ 5102{ 5103 zend_jit_addr op2_addr = OP2_ADDR(); 5104 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 5105 5106 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 5107 && type == BP_VAR_R 5108 && !exit_addr) { 5109 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5110 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5111 if (!exit_addr) { 5112 return 0; 5113 } 5114 } 5115 5116 if (op2_info & MAY_BE_LONG) { 5117 bool op2_loaded = 0; 5118 bool packed_loaded = 0; 5119 bool bad_packed_key = 0; 5120 5121 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { 5122 | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) 5123 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 5124 } 5125 if (op1_info & MAY_BE_PACKED_GUARD) { 5126 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 5127 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5128 5129 if (!exit_addr) { 5130 return 0; 5131 } 5132 if (op1_info & MAY_BE_ARRAY_PACKED) { 5133 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 5134 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 5135 | beq &exit_addr 5136 } else { 5137 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 5138 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 5139 | bne &exit_addr 5140 } 5141 } 5142 if (type == BP_VAR_W) { 5143 | // hval = Z_LVAL_P(dim); 5144 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5145 op2_loaded = 1; 5146 } 5147 if (op1_info & MAY_BE_ARRAY_PACKED) { 5148 zend_long val = -1; 5149 5150 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5151 val = Z_LVAL_P(Z_ZV(op2_addr)); 5152 if (val >= 0 && val < HT_MAX_SIZE) { 5153 packed_loaded = 1; 5154 } else { 5155 bad_packed_key = 1; 5156 } 5157 } else { 5158 if (!op2_loaded) { 5159 | // hval = Z_LVAL_P(dim); 5160 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5161 op2_loaded = 1; 5162 } 5163 packed_loaded = 1; 5164 } 5165 5166 if (dim_type == IS_UNDEF && type == BP_VAR_W) { 5167 /* don't generate "fast" code for packed array */ 5168 packed_loaded = 0; 5169 } 5170 5171 if (packed_loaded) { 5172 | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); 5173 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5174 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 5175 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 5176 | beq >4 // HASH_FIND 5177 } 5178 | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) 5179 5180 | ldr REG0w, [FCARG1x, #offsetof(zend_array, nNumUsed)] 5181 if (val == 0) { 5182 | cmp REG0, xzr 5183 } else if (val > 0 && !op2_loaded) { 5184 | CMP_64_WITH_CONST REG0, val, TMP1 5185 } else { 5186 | cmp REG0, FCARG2x 5187 } 5188 5189 if (type == BP_JIT_IS) { 5190 if (not_found_exit_addr) { 5191 | bls ¬_found_exit_addr 5192 } else { 5193 | bls >9 // NOT_FOUND 5194 } 5195 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5196 | bls &exit_addr 5197 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5198 | bls ¬_found_exit_addr 5199 } else if (type == BP_VAR_RW && not_found_exit_addr) { 5200 | bls ¬_found_exit_addr 5201 } else if (type == BP_VAR_IS && found_exit_addr) { 5202 | bls >7 // NOT_FOUND 5203 } else { 5204 | bls >2 // NOT_FOUND 5205 } 5206 | // _ret = &_ht->arData[_h].val; 5207 if (val >= 0) { 5208 | ldr REG0, [FCARG1x, #offsetof(zend_array, arData)] 5209 if (val != 0) { 5210 | ADD_SUB_64_WITH_CONST add, REG0, REG0, (val * sizeof(Bucket)), TMP1 5211 } 5212 } else { 5213 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] 5214 | add REG0, TMP1, FCARG2x, lsl #5 5215 } 5216 } 5217 } 5218 switch (type) { 5219 case BP_JIT_IS: 5220 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5221 if (packed_loaded) { 5222 | b >5 5223 } 5224 |4: 5225 if (!op2_loaded) { 5226 | // hval = Z_LVAL_P(dim); 5227 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5228 } 5229 if (packed_loaded) { 5230 | EXT_CALL _zend_hash_index_find, REG0 5231 } else { 5232 | EXT_CALL zend_hash_index_find, REG0 5233 } 5234 | mov REG0, RETVALx 5235 if (not_found_exit_addr) { 5236 | cbz REG0, ¬_found_exit_addr 5237 } else { 5238 | cbz REG0, >9 // NOT_FOUND 5239 } 5240 if (op2_info & MAY_BE_STRING) { 5241 | b >5 5242 } 5243 } else if (packed_loaded) { 5244 if (op2_info & MAY_BE_STRING) { 5245 | b >5 5246 } 5247 } else if (not_found_exit_addr) { 5248 | b ¬_found_exit_addr 5249 } else { 5250 | b >9 // NOT_FOUND 5251 } 5252 break; 5253 case BP_VAR_R: 5254 case BP_VAR_IS: 5255 case BP_VAR_UNSET: 5256 if (packed_loaded) { 5257 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5258 | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w 5259 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5260 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 5261 if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5262 | IF_Z_TYPE REG0, IS_UNDEF, &exit_addr, TMP1w 5263 } 5264 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5265 | IF_Z_TYPE REG0, IS_UNDEF, ¬_found_exit_addr, TMP1w 5266 } else if (type == BP_VAR_IS && found_exit_addr) { 5267 | IF_Z_TYPE REG0, IS_UNDEF, >7, TMP1w // NOT_FOUND 5268 } else { 5269 | IF_Z_TYPE REG0, IS_UNDEF, >2, TMP1w // NOT_FOUND 5270 } 5271 } 5272 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_NUMERIC_HASH))) { 5273 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5274 | b &exit_addr 5275 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5276 | b ¬_found_exit_addr 5277 } else if (type == BP_VAR_IS && found_exit_addr) { 5278 | b >7 // NOT_FOUND 5279 } else { 5280 | b >2 // NOT_FOUND 5281 } 5282 } 5283 if (!packed_loaded || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5284 |4: 5285 if (!op2_loaded) { 5286 | // hval = Z_LVAL_P(dim); 5287 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5288 } 5289 if (packed_loaded) { 5290 | EXT_CALL _zend_hash_index_find, REG0 5291 } else { 5292 | EXT_CALL zend_hash_index_find, REG0 5293 } 5294 | mov REG0, RETVALx 5295 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5296 | cbz REG0, &exit_addr 5297 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5298 | cbz REG0, ¬_found_exit_addr 5299 } else if (type == BP_VAR_IS && found_exit_addr) { 5300 | cbz REG0, >7 // NOT_FOUND 5301 } else { 5302 | cbz REG0, >2 // NOT_FOUND 5303 } 5304 } 5305 |.cold_code 5306 |2: 5307 switch (type) { 5308 case BP_VAR_R: 5309 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5310 | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); 5311 | // retval = &EG(uninitialized_zval); 5312 | UNDEFINED_OFFSET opline 5313 | b >9 5314 } 5315 break; 5316 case BP_VAR_IS: 5317 case BP_VAR_UNSET: 5318 if (!not_found_exit_addr && !found_exit_addr) { 5319 | // retval = &EG(uninitialized_zval); 5320 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 5321 | b >9 5322 } 5323 break; 5324 default: 5325 ZEND_UNREACHABLE(); 5326 } 5327 |.code 5328 break; 5329 case BP_VAR_RW: 5330 if (packed_loaded && !not_found_exit_addr) { 5331 | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w 5332 } 5333 if (!packed_loaded || 5334 !not_found_exit_addr || 5335 (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5336 if (packed_loaded && not_found_exit_addr) { 5337 |.cold_code 5338 } 5339 |2: 5340 |4: 5341 if (!op2_loaded) { 5342 | // hval = Z_LVAL_P(dim); 5343 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5344 } 5345 if (packed_loaded) { 5346 | EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, REG0 5347 } else { 5348 | EXT_CALL zend_jit_hash_index_lookup_rw, REG0 5349 } 5350 | mov REG0, RETVALx 5351 if (not_found_exit_addr) { 5352 if (packed_loaded) { 5353 | cbnz REG0, >8 5354 | b ¬_found_exit_addr 5355 |.code 5356 } else { 5357 | cbz REG0, ¬_found_exit_addr 5358 } 5359 } else { 5360 | cbz REG0, >9 5361 } 5362 } 5363 break; 5364 case BP_VAR_W: 5365 if (packed_loaded) { 5366 | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w 5367 } 5368 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) { 5369 |2: 5370 |4: 5371 if (!op2_loaded) { 5372 | // hval = Z_LVAL_P(dim); 5373 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5374 } 5375 | EXT_CALL zend_hash_index_lookup, REG0 5376 | mov REG0, RETVALx 5377 } 5378 break; 5379 default: 5380 ZEND_UNREACHABLE(); 5381 } 5382 5383 if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { 5384 | b >8 5385 } 5386 } 5387 5388 if (op2_info & MAY_BE_STRING) { 5389 |3: 5390 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5391 | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) 5392 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3, ZREG_TMP1 5393 } 5394 | // offset_key = Z_STR_P(dim); 5395 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5396 | // retval = zend_hash_find(ht, offset_key); 5397 switch (type) { 5398 case BP_JIT_IS: 5399 if (opline->op2_type != IS_CONST) { 5400 | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] 5401 | cmp TMP1w, #((uint8_t) ('9')) 5402 | ble >1 5403 |.cold_code 5404 |1: 5405 | EXT_CALL zend_jit_symtable_find, REG0 5406 | b >1 5407 |.code 5408 | EXT_CALL zend_hash_find, REG0 5409 |1: 5410 } else { 5411 | EXT_CALL zend_hash_find_known_hash, REG0 5412 } 5413 | mov REG0, RETVALx 5414 if (not_found_exit_addr) { 5415 | cbz REG0, ¬_found_exit_addr 5416 } else { 5417 | cbz REG0, >9 // NOT_FOUND 5418 } 5419 break; 5420 case BP_VAR_R: 5421 case BP_VAR_IS: 5422 case BP_VAR_UNSET: 5423 if (opline->op2_type != IS_CONST) { 5424 | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] 5425 | cmp TMP1w, #((uint8_t) ('9')) 5426 | ble >1 5427 |.cold_code 5428 |1: 5429 | EXT_CALL zend_jit_symtable_find, REG0 5430 | b >1 5431 |.code 5432 | EXT_CALL zend_hash_find, REG0 5433 |1: 5434 } else { 5435 | EXT_CALL zend_hash_find_known_hash, REG0 5436 } 5437 | mov REG0, RETVALx 5438 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5439 | cbz REG0, &exit_addr 5440 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5441 | cbz REG0, ¬_found_exit_addr 5442 } else if (type == BP_VAR_IS && found_exit_addr) { 5443 | cbz REG0, >7 5444 } else { 5445 | cbz REG0, >2 // NOT_FOUND 5446 |.cold_code 5447 |2: 5448 switch (type) { 5449 case BP_VAR_R: 5450 // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); 5451 | UNDEFINED_INDEX opline 5452 | b >9 5453 break; 5454 case BP_VAR_IS: 5455 case BP_VAR_UNSET: 5456 | // retval = &EG(uninitialized_zval); 5457 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 5458 | b >9 5459 break; 5460 default: 5461 ZEND_UNREACHABLE(); 5462 } 5463 |.code 5464 } 5465 break; 5466 case BP_VAR_RW: 5467 if (opline->op2_type != IS_CONST) { 5468 | EXT_CALL zend_jit_symtable_lookup_rw, REG0 5469 } else { 5470 | EXT_CALL zend_jit_hash_lookup_rw, REG0 5471 } 5472 | mov REG0, RETVALx 5473 if (not_found_exit_addr) { 5474 | cbz REG0, ¬_found_exit_addr 5475 } else { 5476 | cbz REG0, >9 5477 } 5478 break; 5479 case BP_VAR_W: 5480 if (opline->op2_type != IS_CONST) { 5481 | EXT_CALL zend_jit_symtable_lookup_w, REG0 5482 } else { 5483 | EXT_CALL zend_hash_lookup, REG0 5484 } 5485 | mov REG0, RETVALx 5486 break; 5487 default: 5488 ZEND_UNREACHABLE(); 5489 } 5490 } 5491 5492 if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { 5493 |5: 5494 if (op1_info & MAY_BE_ARRAY_OF_REF) { 5495 | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w 5496 } 5497 | ldrb TMP1w, [REG0,#offsetof(zval, u1.v.type)] 5498 | cmp TMP1w, #IS_NULL 5499 if (not_found_exit_addr) { 5500 | ble ¬_found_exit_addr 5501 } else if (found_exit_addr) { 5502 | bgt &found_exit_addr 5503 } else { 5504 | ble >9 // NOT FOUND 5505 } 5506 } 5507 5508 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5509 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5510 |.cold_code 5511 |3: 5512 } 5513 if (type != BP_VAR_RW) { 5514 | SET_EX_OPLINE opline, REG0 5515 } 5516 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 5517 switch (type) { 5518 case BP_VAR_R: 5519 | LOAD_ZVAL_ADDR CARG3, res_addr 5520 | EXT_CALL zend_jit_fetch_dim_r_helper, REG0 5521 | mov REG0, RETVALx 5522 | b >9 5523 break; 5524 case BP_JIT_IS: 5525 | EXT_CALL zend_jit_fetch_dim_isset_helper, REG0 5526 | mov REG0, RETVALx 5527 if (not_found_exit_addr) { 5528 | cbz REG0, ¬_found_exit_addr 5529 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5530 | b >8 5531 } 5532 } else if (found_exit_addr) { 5533 | cbnz REG0, &found_exit_addr 5534 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5535 | b >9 5536 } 5537 } else { 5538 | cbnz REG0, >8 5539 | b >9 5540 } 5541 break; 5542 case BP_VAR_IS: 5543 case BP_VAR_UNSET: 5544 | LOAD_ZVAL_ADDR CARG3, res_addr 5545 | EXT_CALL zend_jit_fetch_dim_is_helper, REG0 5546 | mov REG0, RETVALx 5547 | b >9 5548 break; 5549 case BP_VAR_RW: 5550 | EXT_CALL zend_jit_fetch_dim_rw_helper, REG0 5551 | mov REG0, RETVALx 5552 | cbnz REG0, >8 5553 | b >9 5554 break; 5555 case BP_VAR_W: 5556 | EXT_CALL zend_jit_fetch_dim_w_helper, REG0 5557 | mov REG0, RETVALx 5558 | cbnz REG0, >8 5559 | b >9 5560 break; 5561 default: 5562 ZEND_UNREACHABLE(); 5563 } 5564 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5565 |.code 5566 } 5567 } 5568 5569 return 1; 5570} 5571 5572static int zend_jit_simple_assign(dasm_State **Dst, 5573 const zend_op *opline, 5574 zend_jit_addr var_addr, 5575 uint32_t var_info, 5576 uint32_t var_def_info, 5577 zend_uchar val_type, 5578 zend_jit_addr val_addr, 5579 uint32_t val_info, 5580 zend_jit_addr res_addr, 5581 int in_cold, 5582 int save_r1, 5583 bool check_exception) 5584/* Labels: 1,2,3 */ 5585{ 5586 zend_reg tmp_reg; 5587 5588 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_REG0) { 5589 tmp_reg = ZREG_REG0; 5590 } else { 5591 /* ASSIGN_DIM */ 5592 tmp_reg = ZREG_FCARG1; 5593 } 5594 5595 if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 5596 zval *zv = Z_ZV(val_addr); 5597 5598 if (!res_addr) { 5599 | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 5600 } else { 5601 | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 5602 } 5603 if (Z_REFCOUNTED_P(zv)) { 5604 if (!res_addr) { 5605 | ADDREF_CONST zv, TMP1, TMP2 5606 } else { 5607 | ADDREF_CONST_2 zv, TMP1, TMP2 5608 } 5609 } 5610 } else { 5611 if (val_info & MAY_BE_UNDEF) { 5612 if (in_cold) { 5613 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2, ZREG_TMP1 5614 } else { 5615 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 5616 |.cold_code 5617 |1: 5618 } 5619 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 5620 if (save_r1) { 5621 | str FCARG1x, T1 // save 5622 } 5623 | SET_ZVAL_TYPE_INFO var_addr, IS_NULL, TMP1w, TMP2 5624 if (res_addr) { 5625 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 5626 } 5627 if (opline) { 5628 | SET_EX_OPLINE opline, Rx(tmp_reg) 5629 } 5630 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); 5631 | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) 5632 | EXT_CALL zend_jit_undefined_op_helper, REG0 5633 if (check_exception) { 5634 | cbz RETVALx, ->exception_handler_undef 5635 } 5636 if (save_r1) { 5637 | ldr FCARG1x, T1 // restore 5638 } 5639 | b >3 5640 if (in_cold) { 5641 |2: 5642 } else { 5643 |.code 5644 } 5645 } 5646 if (val_info & MAY_BE_REF) { 5647 if (val_type == IS_CV) { 5648 ZEND_ASSERT(Z_REG(var_addr) != ZREG_REG2); 5649 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_REG2 || Z_OFFSET(val_addr) != 0) { 5650 | LOAD_ZVAL_ADDR REG2, val_addr 5651 } 5652 | ZVAL_DEREF REG2, val_info, TMP1w 5653 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); 5654 } else { 5655 zend_jit_addr ref_addr; 5656 5657 if (in_cold) { 5658 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 5659 } else { 5660 | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 5661 |.cold_code 5662 |1: 5663 } 5664 if (Z_REG(val_addr) == ZREG_REG2) { 5665 | str REG2, T1 // save 5666 } 5667 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 5668 | GET_ZVAL_PTR REG2, val_addr, TMP1 5669 | GC_DELREF REG2, TMP1w 5670 | // ZVAL_COPY_VALUE(return_value, &ref->val); 5671 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, offsetof(zend_reference, val)); 5672 if (!res_addr) { 5673 | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5674 } else { 5675 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5676 } 5677 | beq >2 // GC_DELREF() reached zero 5678 | IF_NOT_REFCOUNTED REG2w, >3, TMP1w 5679 if (!res_addr) { 5680 | GC_ADDREF Rx(tmp_reg), TMP1w 5681 } else { 5682 | GC_ADDREF_2 Rx(tmp_reg), TMP1w 5683 } 5684 | b >3 5685 |2: 5686 if (res_addr) { 5687 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w 5688 | GC_ADDREF Rx(tmp_reg), TMP1w 5689 |2: 5690 } 5691 if (Z_REG(val_addr) == ZREG_REG2) { 5692 | ldr REG2, T1 // restore 5693 } 5694 if (save_r1) { 5695 | str FCARG1x, T1 // save 5696 } 5697 | GET_ZVAL_PTR FCARG1x, val_addr, TMP1 5698 | EFREE_REFERENCE 5699 if (save_r1) { 5700 | ldr FCARG1x, T1 // restore 5701 } 5702 | b >3 5703 if (in_cold) { 5704 |1: 5705 } else { 5706 |.code 5707 } 5708 } 5709 } 5710 5711 if (!res_addr) { 5712 | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5713 } else { 5714 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5715 } 5716 5717 if (val_type == IS_CV) { 5718 if (!res_addr) { 5719 | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w 5720 } else { 5721 | TRY_ADDREF_2 val_info, REG2w, Rx(tmp_reg), TMP1w 5722 } 5723 } else { 5724 if (res_addr) { 5725 | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w 5726 } 5727 } 5728 |3: 5729 } 5730 return 1; 5731} 5732 5733static int zend_jit_assign_to_typed_ref(dasm_State **Dst, 5734 const zend_op *opline, 5735 zend_uchar val_type, 5736 zend_jit_addr val_addr, 5737 zend_jit_addr res_addr, 5738 bool check_exception) 5739{ 5740 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 5741 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 5742 | cbnz TMP1, >2 5743 |.cold_code 5744 |2: 5745 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 5746 | LOAD_ZVAL_ADDR FCARG2x, val_addr 5747 } 5748 if (opline) { 5749 | SET_EX_OPLINE opline, REG0 5750 } 5751 if (val_type == IS_CONST) { 5752 | EXT_CALL zend_jit_assign_const_to_typed_ref, REG0 5753 } else if (val_type == IS_TMP_VAR) { 5754 | EXT_CALL zend_jit_assign_tmp_to_typed_ref, REG0 5755 } else if (val_type == IS_VAR) { 5756 | EXT_CALL zend_jit_assign_var_to_typed_ref, REG0 5757 } else if (val_type == IS_CV) { 5758 | EXT_CALL zend_jit_assign_cv_to_typed_ref, REG0 5759 } else { 5760 ZEND_UNREACHABLE(); 5761 } 5762 if (res_addr) { 5763 zend_jit_addr ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_X0, 0); // RETVAL 5764 5765 | ZVAL_COPY_VALUE res_addr, -1, ret_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5766 | TRY_ADDREF -1, REG1w, REG2, TMP1w 5767 } 5768 if (check_exception) { 5769 | // if (UNEXPECTED(EG(exception) != NULL)) { 5770 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 5771 | cbz REG0, >8 // END OF zend_jit_assign_to_variable() 5772 | b ->exception_handler 5773 } else { 5774 | b >8 5775 } 5776 |.code 5777 5778 return 1; 5779} 5780 5781static int zend_jit_assign_to_variable_call(dasm_State **Dst, 5782 const zend_op *opline, 5783 zend_jit_addr __var_use_addr, 5784 zend_jit_addr var_addr, 5785 uint32_t __var_info, 5786 uint32_t __var_def_info, 5787 zend_uchar val_type, 5788 zend_jit_addr val_addr, 5789 uint32_t val_info, 5790 zend_jit_addr __res_addr, 5791 bool __check_exception) 5792{ 5793 if (val_info & MAY_BE_UNDEF) { 5794 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 5795 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5796 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5797 5798 if (!exit_addr) { 5799 return 0; 5800 } 5801 5802 | IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr, ZREG_TMP1 5803 } else { 5804 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 5805 |.cold_code 5806 |1: 5807 ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP); 5808 if (Z_REG(var_addr) != ZREG_FP) { 5809 | str Rx(Z_REG(var_addr)), T1 // save 5810 } 5811 | SET_EX_OPLINE opline, REG0 5812 | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) 5813 | EXT_CALL zend_jit_undefined_op_helper, REG0 5814 if (Z_REG(var_addr) != ZREG_FP) { 5815 | ldr Rx(Z_REG(var_addr)), T1 // restore 5816 } 5817 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 5818 | LOAD_ZVAL_ADDR FCARG1x, var_addr 5819 } 5820 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 5821 | bl ->assign_const 5822 | b >9 5823 |.code 5824 |1: 5825 } 5826 } 5827 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 5828 | LOAD_ZVAL_ADDR FCARG1x, var_addr 5829 } 5830 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 5831 | LOAD_ZVAL_ADDR FCARG2x, val_addr 5832 } 5833 if (opline) { 5834 | SET_EX_OPLINE opline, REG0 5835 } 5836 if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 5837 | bl ->assign_tmp 5838 } else if (val_type == IS_CONST) { 5839 | bl ->assign_const 5840 } else if (val_type == IS_TMP_VAR) { 5841 | bl ->assign_tmp 5842 } else if (val_type == IS_VAR) { 5843 if (!(val_info & MAY_BE_REF)) { 5844 | bl ->assign_tmp 5845 } else { 5846 | bl ->assign_var 5847 } 5848 } else if (val_type == IS_CV) { 5849 if (!(val_info & MAY_BE_REF)) { 5850 | bl ->assign_cv_noref 5851 } else { 5852 | bl ->assign_cv 5853 } 5854 if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5855 |9: 5856 } 5857 } else { 5858 ZEND_UNREACHABLE(); 5859 } 5860 5861 return 1; 5862} 5863 5864static int zend_jit_assign_to_variable(dasm_State **Dst, 5865 const zend_op *opline, 5866 zend_jit_addr var_use_addr, 5867 zend_jit_addr var_addr, 5868 uint32_t var_info, 5869 uint32_t var_def_info, 5870 zend_uchar val_type, 5871 zend_jit_addr val_addr, 5872 uint32_t val_info, 5873 zend_jit_addr res_addr, 5874 bool check_exception) 5875/* Labels: 1,2,3,4,5,8 */ 5876{ 5877 int done = 0; 5878 zend_reg ref_reg, tmp_reg; 5879 5880 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_REG0) { 5881 ref_reg = ZREG_FCARG1; 5882 tmp_reg = ZREG_REG0; 5883 } else { 5884 /* ASSIGN_DIM */ 5885 ref_reg = ZREG_REG0; 5886 tmp_reg = ZREG_FCARG1; 5887 } 5888 5889 if (var_info & MAY_BE_REF) { 5890 if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { 5891 | LOAD_ZVAL_ADDR Rx(ref_reg), var_use_addr 5892 var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); 5893 } 5894 | // if (Z_ISREF_P(variable_ptr)) { 5895 | IF_NOT_Z_TYPE Rx(ref_reg), IS_REFERENCE, >3, TMP1w 5896 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 5897 | GET_Z_PTR FCARG1x, Rx(ref_reg) 5898 if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) { 5899 return 0; 5900 } 5901 | add Rx(ref_reg), FCARG1x, #offsetof(zend_reference, val) 5902 |3: 5903 } 5904 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 5905 if (RC_MAY_BE_1(var_info)) { 5906 int in_cold = 0; 5907 5908 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 5909 | IF_ZVAL_REFCOUNTED var_use_addr, >1, ZREG_TMP1, ZREG_TMP2 5910 |.cold_code 5911 |1: 5912 in_cold = 1; 5913 } 5914 if (Z_REG(var_use_addr) == ZREG_FCARG1 || Z_REG(var_use_addr) == ZREG_REG0) { 5915 bool keep_gc = 0; 5916 5917 | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 5918#if 0 5919 // TODO: This optiization doesn't work on ARM 5920 if (tmp_reg == ZREG_FCARG1) { 5921 if (Z_MODE(val_addr) == IS_REG) { 5922 keep_gc = 1; 5923 } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { 5924 keep_gc = 1; 5925 } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 5926 zval *zv = Z_ZV(val_addr); 5927 if (Z_TYPE_P(zv) == IS_DOUBLE) { 5928 if (Z_DVAL_P(zv) == 0) { 5929 keep_gc = 1; 5930 } 5931 } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 5932 keep_gc = 1; 5933 } 5934 } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { 5935 if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 5936 keep_gc = 1; 5937 } 5938 } 5939 } 5940#endif 5941 if (!keep_gc) { 5942 | str Rx(tmp_reg), T1 // save 5943 } 5944 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0, 0)) { 5945 return 0; 5946 } 5947 if (!keep_gc) { 5948 | ldr FCARG1x, T1 // restore 5949 } 5950 } else { 5951 | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 5952 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1, 0)) { 5953 return 0; 5954 } 5955 } 5956 | GC_DELREF FCARG1x, TMP1w 5957 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 5958 | bne >4 5959 } else { 5960 | bne >8 5961 } 5962 | ZVAL_DTOR_FUNC var_info, opline, TMP1 5963 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { 5964 if (check_exception && !(val_info & MAY_BE_UNDEF)) { 5965 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 5966 | cbz REG0, >8 5967 | b ->exception_handler 5968 } else { 5969 | b >8 5970 } 5971 } 5972 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 5973 |4: 5974 | IF_GC_MAY_NOT_LEAK FCARG1x, >8, TMP1w, TMP2w 5975 | EXT_CALL gc_possible_root, REG0 5976 if (in_cold) { 5977 | b >8 5978 } 5979 } 5980 if (check_exception && (val_info & MAY_BE_UNDEF)) { 5981 |8: 5982 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 5983 | cbz REG0, >8 5984 | b ->exception_handler 5985 } 5986 if (in_cold) { 5987 |.code 5988 } else { 5989 done = 1; 5990 } 5991 } else /* if (RC_MAY_BE_N(var_info)) */ { 5992 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 5993 | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5, ZREG_TMP1, ZREG_TMP2 5994 } 5995 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { 5996 if (Z_REG(var_use_addr) != ZREG_FP) { 5997 | str Rx(Z_REG(var_use_addr)), T1 // save 5998 } 5999 | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 6000 | GC_DELREF FCARG1x, TMP1w 6001 | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w 6002 | EXT_CALL gc_possible_root, TMP1 6003 if (Z_REG(var_use_addr) != ZREG_FP) { 6004 | ldr Rx(Z_REG(var_use_addr)), T1 // restore 6005 } 6006 } else { 6007 | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 6008 | GC_DELREF Rx(tmp_reg), TMP1w 6009 } 6010 |5: 6011 } 6012 } 6013 6014 if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0, check_exception)) { 6015 return 0; 6016 } 6017 6018 |8: 6019 6020 return 1; 6021} 6022 6023static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw) 6024{ 6025 zend_jit_addr op2_addr, op3_addr, res_addr; 6026 6027 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6028 op3_addr = OP1_DATA_ADDR(); 6029 if (opline->result_type == IS_UNUSED) { 6030 res_addr = 0; 6031 } else { 6032 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 6033 } 6034 6035 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { 6036 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6037 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6038 6039 if (!exit_addr) { 6040 return 0; 6041 } 6042 6043 | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr, ZREG_TMP1 6044 6045 val_info &= ~MAY_BE_UNDEF; 6046 } 6047 6048 if (op1_info & MAY_BE_REF) { 6049 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6050 | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w 6051 | GET_Z_PTR FCARG2x, FCARG1x 6052 | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] 6053 | cmp TMP1w, #IS_ARRAY 6054 | bne >2 6055 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) 6056 | b >3 6057 |.cold_code 6058 |2: 6059 | SET_EX_OPLINE opline, REG0 6060 | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 6061 | mov FCARG1x, RETVALx 6062 | cbnz FCARG1x, >1 6063 | b ->exception_handler_undef 6064 |.code 6065 |1: 6066 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6067 } 6068 6069 if (op1_info & MAY_BE_ARRAY) { 6070 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6071 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 6072 } 6073 |3: 6074 | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 6075 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6076 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6077 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 6078 | bgt >7 6079 } 6080 | // ZVAL_ARR(container, zend_new_array(8)); 6081 if (Z_REG(op1_addr) != ZREG_FP) { 6082 | str Rx(Z_REG(op1_addr)), T1 // save 6083 } 6084 | EXT_CALL _zend_new_array_0, REG0 6085 | mov REG0, RETVALx 6086 if (Z_REG(op1_addr) != ZREG_FP) { 6087 | ldr Rx(Z_REG(op1_addr)), T1 // restore 6088 } 6089 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 6090 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 6091 | mov FCARG1x, REG0 6092 } 6093 6094 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6095 |6: 6096 if (opline->op2_type == IS_UNUSED) { 6097 uint32_t var_info = MAY_BE_NULL; 6098 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 6099 6100 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6101 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 6102 | EXT_CALL zend_hash_next_index_insert, REG0 6103 | // if (UNEXPECTED(!var_ptr)) { 6104 | mov REG0, RETVALx 6105 | cbz REG0, >1 6106 |.cold_code 6107 |1: 6108 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6109 | CANNOT_ADD_ELEMENT opline 6110 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6111 | b >9 6112 |.code 6113 6114 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0, 0)) { 6115 return 0; 6116 } 6117 } else { 6118 uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6119 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 6120 6121 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 6122 return 0; 6123 } 6124 6125 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6126 var_info |= MAY_BE_REF; 6127 } 6128 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6129 var_info |= MAY_BE_RC1; 6130 } 6131 6132 |8: 6133 | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); 6134 if (opline->op1_type == IS_VAR) { 6135 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6136 if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6137 return 0; 6138 } 6139 } else { 6140 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6141 return 0; 6142 } 6143 } 6144 } 6145 } 6146 6147 if (((op1_info & MAY_BE_ARRAY) && 6148 (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) || 6149 (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY)))) { 6150 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6151 |.cold_code 6152 |7: 6153 } 6154 6155 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) && 6156 (op1_info & MAY_BE_ARRAY)) { 6157 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6158 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 6159 | bgt >2 6160 } 6161 | // ZVAL_ARR(container, zend_new_array(8)); 6162 if (Z_REG(op1_addr) != ZREG_FP) { 6163 | str Rx(Z_REG(op1_addr)), T1 // save 6164 } 6165 | EXT_CALL _zend_new_array_0, REG0 6166 | mov REG0, RETVALx 6167 if (Z_REG(op1_addr) != ZREG_FP) { 6168 | ldr Rx(Z_REG(op1_addr)), T1 // restore 6169 } 6170 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 6171 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 6172 | mov FCARG1x, REG0 6173 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); 6174 | b <6 6175 |2: 6176 } 6177 6178 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6179 | SET_EX_OPLINE opline, REG0 6180 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6181 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6182 } 6183 if (opline->op2_type == IS_UNUSED) { 6184 | mov FCARG2x, xzr 6185 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6186 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6187 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 6188 } else { 6189 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 6190 } 6191 if (opline->result_type == IS_UNUSED) { 6192 | mov CARG4, xzr 6193 } else { 6194 | LOAD_ZVAL_ADDR CARG4, res_addr 6195 } 6196 | LOAD_ZVAL_ADDR CARG3, op3_addr 6197 | EXT_CALL zend_jit_assign_dim_helper, REG0 6198 6199#ifdef ZEND_JIT_USE_RC_INFERENCE 6200 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { 6201 /* ASSIGN_DIM may increase refcount of the value */ 6202 val_info |= MAY_BE_RCN; 6203 } 6204#endif 6205 6206 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 6207 } 6208 6209 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6210 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6211 | b >9 // END 6212 } 6213 |.code 6214 } 6215 } 6216 6217#ifdef ZEND_JIT_USE_RC_INFERENCE 6218 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 6219 /* ASSIGN_DIM may increase refcount of the key */ 6220 op2_info |= MAY_BE_RCN; 6221 } 6222#endif 6223 6224 |9: 6225 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6226 6227 if (may_throw) { 6228 zend_jit_check_exception(Dst); 6229 } 6230 6231 return 1; 6232} 6233 6234static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) 6235{ 6236 zend_jit_addr op2_addr, op3_addr, var_addr; 6237 const void *not_found_exit_addr = NULL; 6238 uint32_t var_info = MAY_BE_NULL; 6239 6240 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6241 6242 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6243 op3_addr = OP1_DATA_ADDR(); 6244 6245 | SET_EX_OPLINE opline, REG0 6246 if (op1_info & MAY_BE_REF) { 6247 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6248 | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w 6249 | GET_Z_PTR FCARG2x, FCARG1x 6250 | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] 6251 | cmp TMP1w, #IS_ARRAY 6252 | bne >2 6253 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) 6254 | b >3 6255 |.cold_code 6256 |2: 6257 | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 6258 | mov FCARG1x, RETVALx 6259 | cbnz RETVALx, >1 6260 | b ->exception_handler_undef 6261 |.code 6262 |1: 6263 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6264 } 6265 6266 if (op1_info & MAY_BE_ARRAY) { 6267 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6268 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 6269 } 6270 |3: 6271 | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 6272 } 6273 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6274 if (op1_info & MAY_BE_ARRAY) { 6275 |.cold_code 6276 |7: 6277 } 6278 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6279 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 6280 | bgt >7 6281 } 6282 if (Z_REG(op1_addr) != ZREG_FP) { 6283 | str Rx(Z_REG(op1_addr)), T1 // save 6284 } 6285 if (op1_info & MAY_BE_UNDEF) { 6286 if (op1_info & MAY_BE_NULL) { 6287 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 6288 } 6289 | LOAD_32BIT_VAL FCARG1x, opline->op1.var 6290 | EXT_CALL zend_jit_undefined_op_helper, REG0 6291 |1: 6292 } 6293 | // ZVAL_ARR(container, zend_new_array(8)); 6294 | EXT_CALL _zend_new_array_0, REG0 6295 | mov REG0, RETVALx 6296 if (Z_REG(op1_addr) != ZREG_FP) { 6297 | ldr Rx(Z_REG(op1_addr)), T1 // restore 6298 } 6299 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 6300 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 6301 | mov FCARG1x, REG0 6302 if (op1_info & MAY_BE_ARRAY) { 6303 | b >1 6304 |.code 6305 |1: 6306 } 6307 } 6308 6309 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6310 uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); 6311 6312 |6: 6313 if (opline->op2_type == IS_UNUSED) { 6314 var_info = MAY_BE_NULL; 6315 6316 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6317 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 6318 | EXT_CALL zend_hash_next_index_insert, REG0 6319 | mov REG0, RETVALx 6320 | // if (UNEXPECTED(!var_ptr)) { 6321 | cbz REG0, >1 6322 |.cold_code 6323 |1: 6324 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6325 | CANNOT_ADD_ELEMENT opline 6326 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6327 | b >9 6328 |.code 6329 } else { 6330 var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6331 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6332 var_info |= MAY_BE_REF; 6333 } 6334 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6335 var_info |= MAY_BE_RC1; 6336 } 6337 6338 if (dim_type != IS_UNKNOWN 6339 && dim_type != IS_UNDEF 6340 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY 6341 && (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) 6342 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 6343 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 6344 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6345 if (!not_found_exit_addr) { 6346 return 0; 6347 } 6348 } 6349 6350 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, not_found_exit_addr, NULL)) { 6351 return 0; 6352 } 6353 6354 |8: 6355 if (not_found_exit_addr && dim_type != IS_REFERENCE) { 6356 | IF_NOT_Z_TYPE, REG0, dim_type, ¬_found_exit_addr, TMP1w 6357 var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 6358 } 6359 if (var_info & MAY_BE_REF) { 6360 binary_op_type binary_op = get_binary_op(opline->extended_value); 6361 | IF_NOT_Z_TYPE, REG0, IS_REFERENCE, >1, TMP1w 6362 | GET_Z_PTR FCARG1x, REG0 6363 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 6364 | cbnz TMP1, >2 6365 | add REG0, FCARG1x, #offsetof(zend_reference, val) 6366 |.cold_code 6367 |2: 6368 | LOAD_ZVAL_ADDR FCARG2x, op3_addr 6369 | LOAD_ADDR CARG3, binary_op 6370 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 6371 && (op1_data_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6372 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 6373 } else { 6374 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 6375 } 6376 | b >9 6377 |.code 6378 |1: 6379 } 6380 } 6381 6382 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 6383 switch (opline->extended_value) { 6384 case ZEND_ADD: 6385 case ZEND_SUB: 6386 case ZEND_MUL: 6387 case ZEND_DIV: 6388 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, 6389 1 /* may overflow */, may_throw)) { 6390 return 0; 6391 } 6392 break; 6393 case ZEND_BW_OR: 6394 case ZEND_BW_AND: 6395 case ZEND_BW_XOR: 6396 case ZEND_SL: 6397 case ZEND_SR: 6398 case ZEND_MOD: 6399 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6400 IS_CV, opline->op1, var_addr, var_info, NULL, 6401 (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 6402 op1_data_range, 6403 0, var_addr, var_def_info, var_info, may_throw)) { 6404 return 0; 6405 } 6406 break; 6407 case ZEND_CONCAT: 6408 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, 6409 may_throw)) { 6410 return 0; 6411 } 6412 break; 6413 default: 6414 ZEND_UNREACHABLE(); 6415 } 6416 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6417 } 6418 6419 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6420 binary_op_type binary_op; 6421 6422 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6423 |.cold_code 6424 |7: 6425 } 6426 6427 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6428 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6429 } 6430 if (opline->op2_type == IS_UNUSED) { 6431 | mov FCARG2x, xzr 6432 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6433 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6434 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 6435 } else { 6436 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 6437 } 6438 binary_op = get_binary_op(opline->extended_value); 6439 | LOAD_ZVAL_ADDR CARG3, op3_addr 6440 | LOAD_ADDR CARG4, binary_op 6441 | EXT_CALL zend_jit_assign_dim_op_helper, REG0 6442 6443 |9: 6444 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 6445 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 6446 if (may_throw) { 6447 zend_jit_check_exception(Dst); 6448 } 6449 6450 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6451 | b >9 // END 6452 |.code 6453 |9: 6454 } 6455 } else if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) 6456 && (!not_found_exit_addr || (var_info & MAY_BE_REF))) { 6457 |.cold_code 6458 |9: 6459 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6460 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6461 if (may_throw) { 6462 zend_jit_check_exception(Dst); 6463 } 6464 | b >9 6465 |.code 6466 |9: 6467 } 6468 6469 return 1; 6470} 6471 6472static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) 6473{ 6474 zend_jit_addr op1_addr, op2_addr; 6475 6476 ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); 6477 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 6478 6479 op1_addr = OP1_ADDR(); 6480 op2_addr = OP2_ADDR(); 6481 6482 if (op1_info & MAY_BE_REF) { 6483 binary_op_type binary_op = get_binary_op(opline->extended_value); 6484 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6485 | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >1, TMP1w 6486 | GET_Z_PTR FCARG1x, FCARG1x 6487 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 6488 | cbnz TMP1, >2 6489 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) 6490 |.cold_code 6491 |2: 6492 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 6493 | LOAD_ADDR CARG3, binary_op 6494 | SET_EX_OPLINE opline, REG0 6495 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 6496 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6497 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 6498 } else { 6499 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 6500 } 6501 zend_jit_check_exception(Dst); 6502 | b >9 6503 |.code 6504 |1: 6505 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6506 } 6507 6508 int result; 6509 switch (opline->extended_value) { 6510 case ZEND_ADD: 6511 case ZEND_SUB: 6512 case ZEND_MUL: 6513 case ZEND_DIV: 6514 result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); 6515 break; 6516 case ZEND_BW_OR: 6517 case ZEND_BW_AND: 6518 case ZEND_BW_XOR: 6519 case ZEND_SL: 6520 case ZEND_SR: 6521 case ZEND_MOD: 6522 result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6523 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 6524 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 6525 opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); 6526 break; 6527 case ZEND_CONCAT: 6528 result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); 6529 break; 6530 default: 6531 ZEND_UNREACHABLE(); 6532 } 6533 |9: 6534 return result; 6535} 6536 6537static int zend_jit_cmp_long_long(dasm_State **Dst, 6538 const zend_op *opline, 6539 zend_ssa_range *op1_range, 6540 zend_jit_addr op1_addr, 6541 zend_ssa_range *op2_range, 6542 zend_jit_addr op2_addr, 6543 zend_jit_addr res_addr, 6544 zend_uchar smart_branch_opcode, 6545 uint32_t target_label, 6546 uint32_t target_label2, 6547 const void *exit_addr, 6548 bool skip_comparison) 6549{ 6550 bool swap = 0; 6551 bool result; 6552 6553 if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { 6554 if (!smart_branch_opcode || 6555 smart_branch_opcode == ZEND_JMPZ_EX || 6556 smart_branch_opcode == ZEND_JMPNZ_EX) { 6557 | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE), TMP1w, TMP2 6558 } 6559 if (smart_branch_opcode && !exit_addr) { 6560 if (smart_branch_opcode == ZEND_JMPZ || 6561 smart_branch_opcode == ZEND_JMPZ_EX) { 6562 if (!result) { 6563 | b => target_label 6564 } 6565 } else if (smart_branch_opcode == ZEND_JMPNZ || 6566 smart_branch_opcode == ZEND_JMPNZ_EX) { 6567 if (result) { 6568 | b => target_label 6569 } 6570 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 6571 if (!result) { 6572 | b => target_label 6573 } else { 6574 | b => target_label2 6575 } 6576 } else { 6577 ZEND_UNREACHABLE(); 6578 } 6579 } 6580 return 1; 6581 } 6582 6583 if (skip_comparison) { 6584 if (Z_MODE(op1_addr) != IS_REG && 6585 (Z_MODE(op2_addr) == IS_REG || 6586 (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { 6587 swap = 1; 6588 } 6589 } else if (Z_MODE(op1_addr) == IS_REG) { 6590 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 6591 | cmp Rx(Z_REG(op1_addr)), xzr 6592 } else { 6593 | LONG_CMP Z_REG(op1_addr), op2_addr, TMP1 6594 } 6595 } else if (Z_MODE(op2_addr) == IS_REG) { 6596 if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { 6597 | cmp Rx(Z_REG(op2_addr)), xzr 6598 } else { 6599 | LONG_CMP Z_REG(op2_addr), op1_addr, TMP1 6600 } 6601 swap = 1; 6602 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 6603 | LONG_CMP_WITH_CONST op2_addr, Z_LVAL_P(Z_ZV(op1_addr)), TMP1, TMP2 6604 swap = 1; 6605 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { 6606 | LONG_CMP_WITH_CONST op1_addr, Z_LVAL_P(Z_ZV(op2_addr)), TMP1, TMP2 6607 } else { 6608 | GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1 6609 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 6610 | cmp Rx(ZREG_REG0), xzr 6611 } else { 6612 | LONG_CMP ZREG_REG0, op2_addr, TMP1 6613 } 6614 } 6615 6616 if (smart_branch_opcode) { 6617 if (smart_branch_opcode == ZEND_JMPZ_EX || 6618 smart_branch_opcode == ZEND_JMPNZ_EX) { 6619 6620 switch (opline->opcode) { 6621 case ZEND_IS_EQUAL: 6622 case ZEND_IS_IDENTICAL: 6623 case ZEND_CASE: 6624 case ZEND_CASE_STRICT: 6625 | cset REG0w, eq 6626 break; 6627 case ZEND_IS_NOT_EQUAL: 6628 case ZEND_IS_NOT_IDENTICAL: 6629 | cset REG0w, ne 6630 break; 6631 case ZEND_IS_SMALLER: 6632 if (swap) { 6633 | cset REG0w, gt 6634 } else { 6635 | cset REG0w, lt 6636 } 6637 break; 6638 case ZEND_IS_SMALLER_OR_EQUAL: 6639 if (swap) { 6640 | cset REG0w, ge 6641 } else { 6642 | cset REG0w, le 6643 } 6644 break; 6645 default: 6646 ZEND_UNREACHABLE(); 6647 } 6648 | add REG0w, REG0w, #2 6649 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 6650 } 6651 if (smart_branch_opcode == ZEND_JMPZ || 6652 smart_branch_opcode == ZEND_JMPZ_EX) { 6653 switch (opline->opcode) { 6654 case ZEND_IS_EQUAL: 6655 case ZEND_IS_IDENTICAL: 6656 case ZEND_CASE: 6657 case ZEND_CASE_STRICT: 6658 if (exit_addr) { 6659 | bne &exit_addr 6660 } else { 6661 | bne => target_label 6662 } 6663 break; 6664 case ZEND_IS_NOT_EQUAL: 6665 if (exit_addr) { 6666 | beq &exit_addr 6667 } else { 6668 | beq => target_label 6669 } 6670 break; 6671 case ZEND_IS_NOT_IDENTICAL: 6672 if (exit_addr) { 6673 | bne &exit_addr 6674 } else { 6675 | beq => target_label 6676 } 6677 break; 6678 case ZEND_IS_SMALLER: 6679 if (swap) { 6680 if (exit_addr) { 6681 | ble &exit_addr 6682 } else { 6683 | ble => target_label 6684 } 6685 } else { 6686 if (exit_addr) { 6687 | bge &exit_addr 6688 } else { 6689 | bge => target_label 6690 } 6691 } 6692 break; 6693 case ZEND_IS_SMALLER_OR_EQUAL: 6694 if (swap) { 6695 if (exit_addr) { 6696 | blt &exit_addr 6697 } else { 6698 | blt => target_label 6699 } 6700 } else { 6701 if (exit_addr) { 6702 | bgt &exit_addr 6703 } else { 6704 | bgt => target_label 6705 } 6706 } 6707 break; 6708 default: 6709 ZEND_UNREACHABLE(); 6710 } 6711 } else if (smart_branch_opcode == ZEND_JMPNZ || 6712 smart_branch_opcode == ZEND_JMPNZ_EX) { 6713 switch (opline->opcode) { 6714 case ZEND_IS_EQUAL: 6715 case ZEND_IS_IDENTICAL: 6716 case ZEND_CASE: 6717 case ZEND_CASE_STRICT: 6718 if (exit_addr) { 6719 | beq &exit_addr 6720 } else { 6721 | beq => target_label 6722 } 6723 break; 6724 case ZEND_IS_NOT_EQUAL: 6725 if (exit_addr) { 6726 | bne &exit_addr 6727 } else { 6728 | bne => target_label 6729 } 6730 break; 6731 case ZEND_IS_NOT_IDENTICAL: 6732 if (exit_addr) { 6733 | beq &exit_addr 6734 } else { 6735 | bne => target_label 6736 } 6737 break; 6738 case ZEND_IS_SMALLER: 6739 if (swap) { 6740 if (exit_addr) { 6741 | bgt &exit_addr 6742 } else { 6743 | bgt => target_label 6744 } 6745 } else { 6746 if (exit_addr) { 6747 | blt &exit_addr 6748 } else { 6749 | blt => target_label 6750 } 6751 } 6752 break; 6753 case ZEND_IS_SMALLER_OR_EQUAL: 6754 if (swap) { 6755 if (exit_addr) { 6756 | bge &exit_addr 6757 } else { 6758 | bge => target_label 6759 } 6760 } else { 6761 if (exit_addr) { 6762 | ble &exit_addr 6763 } else { 6764 | ble => target_label 6765 } 6766 } 6767 break; 6768 default: 6769 ZEND_UNREACHABLE(); 6770 } 6771 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 6772 switch (opline->opcode) { 6773 case ZEND_IS_EQUAL: 6774 case ZEND_IS_IDENTICAL: 6775 case ZEND_CASE: 6776 case ZEND_CASE_STRICT: 6777 | bne => target_label 6778 break; 6779 case ZEND_IS_NOT_EQUAL: 6780 case ZEND_IS_NOT_IDENTICAL: 6781 | beq => target_label 6782 break; 6783 case ZEND_IS_SMALLER: 6784 if (swap) { 6785 | ble => target_label 6786 } else { 6787 | bge => target_label 6788 } 6789 break; 6790 case ZEND_IS_SMALLER_OR_EQUAL: 6791 if (swap) { 6792 | blt => target_label 6793 } else { 6794 | bgt => target_label 6795 } 6796 break; 6797 default: 6798 ZEND_UNREACHABLE(); 6799 } 6800 | b => target_label2 6801 } else { 6802 ZEND_UNREACHABLE(); 6803 } 6804 } else { 6805 switch (opline->opcode) { 6806 case ZEND_IS_EQUAL: 6807 case ZEND_IS_IDENTICAL: 6808 case ZEND_CASE: 6809 case ZEND_CASE_STRICT: 6810 | cset REG0w, eq 6811 break; 6812 case ZEND_IS_NOT_EQUAL: 6813 case ZEND_IS_NOT_IDENTICAL: 6814 | cset REG0w, ne 6815 break; 6816 case ZEND_IS_SMALLER: 6817 if (swap) { 6818 | cset REG0w, gt 6819 } else { 6820 | cset REG0w, lt 6821 } 6822 break; 6823 case ZEND_IS_SMALLER_OR_EQUAL: 6824 if (swap) { 6825 | cset REG0w, ge 6826 } else { 6827 | cset REG0w, le 6828 } 6829 break; 6830 default: 6831 ZEND_UNREACHABLE(); 6832 } 6833 | add REG0w, REG0w, #2 6834 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 6835 } 6836 6837 return 1; 6838} 6839 6840static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 6841{ 6842 if (smart_branch_opcode) { 6843 if (smart_branch_opcode == ZEND_JMPZ) { 6844 switch (opline->opcode) { 6845 case ZEND_IS_EQUAL: 6846 case ZEND_IS_IDENTICAL: 6847 case ZEND_CASE: 6848 case ZEND_CASE_STRICT: 6849 if (exit_addr) { 6850 | bne &exit_addr 6851 } else { 6852 | bne => target_label 6853 } 6854 break; 6855 case ZEND_IS_NOT_EQUAL: 6856 | bvs >1 6857 if (exit_addr) { 6858 | beq &exit_addr 6859 } else { 6860 | beq => target_label 6861 } 6862 |1: 6863 break; 6864 case ZEND_IS_NOT_IDENTICAL: 6865 if (exit_addr) { 6866 | bvs &exit_addr 6867 | bne &exit_addr 6868 } else { 6869 | bvs >1 6870 | beq => target_label 6871 |1: 6872 } 6873 break; 6874 case ZEND_IS_SMALLER: 6875 if (swap) { 6876 if (exit_addr) { 6877 | bvs &exit_addr 6878 | bls &exit_addr 6879 } else { 6880 | bvs => target_label 6881 | bls => target_label 6882 } 6883 } else { 6884 if (exit_addr) { 6885 | bhs &exit_addr 6886 } else { 6887 | bhs => target_label 6888 } 6889 } 6890 break; 6891 case ZEND_IS_SMALLER_OR_EQUAL: 6892 if (swap) { 6893 if (exit_addr) { 6894 | bvs &exit_addr 6895 | blo &exit_addr 6896 } else { 6897 | bvs => target_label 6898 | blo => target_label 6899 } 6900 } else { 6901 if (exit_addr) { 6902 | bhi &exit_addr 6903 } else { 6904 | bhi => target_label 6905 } 6906 } 6907 break; 6908 default: 6909 ZEND_UNREACHABLE(); 6910 } 6911 } else if (smart_branch_opcode == ZEND_JMPNZ) { 6912 switch (opline->opcode) { 6913 case ZEND_IS_EQUAL: 6914 case ZEND_IS_IDENTICAL: 6915 case ZEND_CASE: 6916 case ZEND_CASE_STRICT: 6917 | bvs >1 6918 if (exit_addr) { 6919 | beq &exit_addr 6920 } else { 6921 | beq => target_label 6922 } 6923 |1: 6924 break; 6925 case ZEND_IS_NOT_EQUAL: 6926 if (exit_addr) { 6927 | bne &exit_addr 6928 } else { 6929 | bne => target_label 6930 } 6931 break; 6932 case ZEND_IS_NOT_IDENTICAL: 6933 if (exit_addr) { 6934 | bvs >1 6935 | beq &exit_addr 6936 |1: 6937 } else { 6938 | bne => target_label 6939 } 6940 break; 6941 case ZEND_IS_SMALLER: 6942 if (swap) { 6943 | bvs >1 // Always False if involving NaN 6944 if (exit_addr) { 6945 | bhi &exit_addr 6946 } else { 6947 | bhi => target_label 6948 } 6949 |1: 6950 } else { 6951 | bvs >1 6952 if (exit_addr) { 6953 | blo &exit_addr 6954 } else { 6955 | blo => target_label 6956 } 6957 |1: 6958 } 6959 break; 6960 case ZEND_IS_SMALLER_OR_EQUAL: 6961 if (swap) { 6962 | bvs >1 // Always False if involving NaN 6963 if (exit_addr) { 6964 | bhs &exit_addr 6965 } else { 6966 | bhs => target_label 6967 } 6968 |1: 6969 } else { 6970 | bvs >1 6971 if (exit_addr) { 6972 | bls &exit_addr 6973 } else { 6974 | bls => target_label 6975 } 6976 |1: 6977 } 6978 break; 6979 default: 6980 ZEND_UNREACHABLE(); 6981 } 6982 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 6983 switch (opline->opcode) { 6984 case ZEND_IS_EQUAL: 6985 case ZEND_IS_IDENTICAL: 6986 case ZEND_CASE: 6987 case ZEND_CASE_STRICT: 6988 | bne => target_label 6989 break; 6990 case ZEND_IS_NOT_EQUAL: 6991 case ZEND_IS_NOT_IDENTICAL: 6992 | bvs => target_label2 6993 | beq => target_label 6994 break; 6995 case ZEND_IS_SMALLER: 6996 if (swap) { 6997 | bvs => target_label 6998 | bls => target_label 6999 } else { 7000 | bhs => target_label 7001 } 7002 break; 7003 case ZEND_IS_SMALLER_OR_EQUAL: 7004 if (swap) { 7005 | bvs => target_label 7006 | blo => target_label 7007 } else { 7008 | bhi => target_label 7009 } 7010 break; 7011 default: 7012 ZEND_UNREACHABLE(); 7013 } 7014 | b => target_label2 7015 } else if (smart_branch_opcode == ZEND_JMPZ_EX) { 7016 switch (opline->opcode) { 7017 case ZEND_IS_EQUAL: 7018 case ZEND_IS_IDENTICAL: 7019 case ZEND_CASE: 7020 case ZEND_CASE_STRICT: 7021 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7022 | bne => target_label 7023 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7024 break; 7025 case ZEND_IS_NOT_EQUAL: 7026 case ZEND_IS_NOT_IDENTICAL: 7027 | bvs >1 7028 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7029 | beq => target_label 7030 |1: 7031 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7032 break; 7033 case ZEND_IS_SMALLER: 7034 if (swap) { 7035 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7036 | bvs => target_label 7037 | bls => target_label 7038 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7039 } else { 7040 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7041 | bhs => target_label 7042 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7043 } 7044 break; 7045 case ZEND_IS_SMALLER_OR_EQUAL: 7046 if (swap) { 7047 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7048 | bvs => target_label 7049 | blo => target_label 7050 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7051 } else { 7052 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7053 | bhi => target_label 7054 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7055 } 7056 break; 7057 default: 7058 ZEND_UNREACHABLE(); 7059 } 7060 } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { 7061 switch (opline->opcode) { 7062 case ZEND_IS_EQUAL: 7063 case ZEND_IS_IDENTICAL: 7064 case ZEND_CASE: 7065 case ZEND_CASE_STRICT: 7066 | bvs >1 7067 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7068 | beq => target_label 7069 |1: 7070 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7071 break; 7072 case ZEND_IS_NOT_EQUAL: 7073 case ZEND_IS_NOT_IDENTICAL: 7074 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7075 | bvs => target_label 7076 | bne => target_label 7077 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7078 break; 7079 case ZEND_IS_SMALLER: 7080 if (swap) { 7081 | cset REG0w, hi 7082 | add REG0w, REG0w, #2 7083 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7084 | bvs >1 // Always False if involving NaN 7085 | bhi => target_label 7086 |1: 7087 } else { 7088 | bvs >1 7089 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7090 | blo => target_label 7091 |1: 7092 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7093 } 7094 break; 7095 case ZEND_IS_SMALLER_OR_EQUAL: 7096 if (swap) { 7097 | cset REG0w, hs 7098 | add REG0w, REG0w, #2 7099 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7100 | bvs >1 // Always False if involving NaN 7101 | bhs => target_label 7102 |1: 7103 } else { 7104 | bvs >1 7105 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7106 | bls => target_label 7107 |1: 7108 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7109 } 7110 break; 7111 default: 7112 ZEND_UNREACHABLE(); 7113 } 7114 } else { 7115 ZEND_UNREACHABLE(); 7116 } 7117 } else { 7118 switch (opline->opcode) { 7119 case ZEND_IS_EQUAL: 7120 case ZEND_IS_IDENTICAL: 7121 case ZEND_CASE: 7122 case ZEND_CASE_STRICT: 7123 | bvs >1 7124 | mov REG0, #IS_TRUE 7125 | beq >2 7126 |1: 7127 | mov REG0, #IS_FALSE 7128 |2: 7129 break; 7130 case ZEND_IS_NOT_EQUAL: 7131 case ZEND_IS_NOT_IDENTICAL: 7132 | bvs >1 7133 | mov REG0, #IS_FALSE 7134 | beq >2 7135 |1: 7136 | mov REG0, #IS_TRUE 7137 |2: 7138 break; 7139 case ZEND_IS_SMALLER: 7140 | bvs >1 7141 | mov REG0, #IS_TRUE 7142 || if (swap) { 7143 | bhi >2 7144 || } else { 7145 | blo >2 7146 || } 7147 |1: 7148 | mov REG0, #IS_FALSE 7149 |2: 7150 break; 7151 case ZEND_IS_SMALLER_OR_EQUAL: 7152 | bvs >1 7153 | mov REG0, #IS_TRUE 7154 || if (swap) { 7155 | bhs >2 7156 || } else { 7157 | bls >2 7158 || } 7159 |1: 7160 | mov REG0, #IS_FALSE 7161 |2: 7162 break; 7163 default: 7164 ZEND_UNREACHABLE(); 7165 } 7166 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7167 } 7168 7169 return 1; 7170} 7171 7172static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7173{ 7174 zend_reg tmp_reg = ZREG_FPR0; 7175 7176 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_REG0, ZREG_TMP1 7177 | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP 7178 7179 return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); 7180} 7181 7182static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7183{ 7184 zend_reg tmp_reg = ZREG_FPR0; 7185 7186 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_REG0, ZREG_TMP1 7187 | DOUBLE_CMP tmp_reg, op1_addr, ZREG_TMP1, ZREG_FPTMP 7188 7189 return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); 7190} 7191 7192static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7193{ 7194 bool swap = 0; 7195 7196 if (Z_MODE(op1_addr) == IS_REG) { 7197 | DOUBLE_CMP Z_REG(op1_addr), op2_addr, ZREG_TMP1, ZREG_FPTMP 7198 } else if (Z_MODE(op2_addr) == IS_REG) { 7199 | DOUBLE_CMP Z_REG(op2_addr), op1_addr, ZREG_TMP1, ZREG_FPTMP 7200 swap = 1; 7201 } else { 7202 zend_reg tmp_reg = ZREG_FPR0; 7203 7204 | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 7205 | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP 7206 } 7207 7208 return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); 7209} 7210 7211static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7212{ 7213 | tst RETVALw, RETVALw 7214 if (smart_branch_opcode) { 7215 if (smart_branch_opcode == ZEND_JMPZ_EX || 7216 smart_branch_opcode == ZEND_JMPNZ_EX) { 7217 switch (opline->opcode) { 7218 case ZEND_IS_EQUAL: 7219 case ZEND_CASE: 7220 | cset REG0w, eq 7221 break; 7222 case ZEND_IS_NOT_EQUAL: 7223 | cset REG0w, ne 7224 break; 7225 case ZEND_IS_SMALLER: 7226 | cset REG0w, lt 7227 break; 7228 case ZEND_IS_SMALLER_OR_EQUAL: 7229 | cset REG0w, le 7230 break; 7231 default: 7232 ZEND_UNREACHABLE(); 7233 } 7234 | add REG0w, REG0w, #2 7235 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7236 } 7237 if (smart_branch_opcode == ZEND_JMPZ || 7238 smart_branch_opcode == ZEND_JMPZ_EX) { 7239 switch (opline->opcode) { 7240 case ZEND_IS_EQUAL: 7241 case ZEND_CASE: 7242 if (exit_addr) { 7243 | bne &exit_addr 7244 } else { 7245 | bne => target_label 7246 } 7247 break; 7248 case ZEND_IS_NOT_EQUAL: 7249 if (exit_addr) { 7250 | beq &exit_addr 7251 } else { 7252 | beq => target_label 7253 } 7254 break; 7255 case ZEND_IS_SMALLER: 7256 if (exit_addr) { 7257 | bge &exit_addr 7258 } else { 7259 | bge => target_label 7260 } 7261 break; 7262 case ZEND_IS_SMALLER_OR_EQUAL: 7263 if (exit_addr) { 7264 | bgt &exit_addr 7265 } else { 7266 | bgt => target_label 7267 } 7268 break; 7269 default: 7270 ZEND_UNREACHABLE(); 7271 } 7272 } else if (smart_branch_opcode == ZEND_JMPNZ || 7273 smart_branch_opcode == ZEND_JMPNZ_EX) { 7274 switch (opline->opcode) { 7275 case ZEND_IS_EQUAL: 7276 case ZEND_CASE: 7277 if (exit_addr) { 7278 | beq &exit_addr 7279 } else { 7280 | beq => target_label 7281 } 7282 break; 7283 case ZEND_IS_NOT_EQUAL: 7284 if (exit_addr) { 7285 | bne &exit_addr 7286 } else { 7287 | bne => target_label 7288 } 7289 break; 7290 case ZEND_IS_SMALLER: 7291 if (exit_addr) { 7292 | blt &exit_addr 7293 } else { 7294 | blt => target_label 7295 } 7296 break; 7297 case ZEND_IS_SMALLER_OR_EQUAL: 7298 if (exit_addr) { 7299 | ble &exit_addr 7300 } else { 7301 | ble => target_label 7302 } 7303 break; 7304 default: 7305 ZEND_UNREACHABLE(); 7306 } 7307 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7308 switch (opline->opcode) { 7309 case ZEND_IS_EQUAL: 7310 case ZEND_CASE: 7311 | bne => target_label 7312 break; 7313 case ZEND_IS_NOT_EQUAL: 7314 | beq => target_label 7315 break; 7316 case ZEND_IS_SMALLER: 7317 | bge => target_label 7318 break; 7319 case ZEND_IS_SMALLER_OR_EQUAL: 7320 | bgt => target_label 7321 break; 7322 default: 7323 ZEND_UNREACHABLE(); 7324 } 7325 | b => target_label2 7326 } else { 7327 ZEND_UNREACHABLE(); 7328 } 7329 } else { 7330 switch (opline->opcode) { 7331 case ZEND_IS_EQUAL: 7332 case ZEND_CASE: 7333 | cset REG0w, eq 7334 break; 7335 case ZEND_IS_NOT_EQUAL: 7336 | cset REG0w, ne 7337 break; 7338 case ZEND_IS_SMALLER: 7339 | cset REG0w, lt 7340 break; 7341 case ZEND_IS_SMALLER_OR_EQUAL: 7342 | cset REG0w, le 7343 break; 7344 default: 7345 ZEND_UNREACHABLE(); 7346 } 7347 | add REG0w, REG0w, #2 7348 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7349 } 7350 7351 return 1; 7352} 7353 7354static int zend_jit_cmp(dasm_State **Dst, 7355 const zend_op *opline, 7356 uint32_t op1_info, 7357 zend_ssa_range *op1_range, 7358 zend_jit_addr op1_addr, 7359 uint32_t op2_info, 7360 zend_ssa_range *op2_range, 7361 zend_jit_addr op2_addr, 7362 zend_jit_addr res_addr, 7363 int may_throw, 7364 zend_uchar smart_branch_opcode, 7365 uint32_t target_label, 7366 uint32_t target_label2, 7367 const void *exit_addr, 7368 bool skip_comparison) 7369{ 7370 bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); 7371 bool has_slow; 7372 7373 has_slow = 7374 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7375 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7376 ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7377 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); 7378 7379 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 7380 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 7381 if (op1_info & MAY_BE_DOUBLE) { 7382 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4, ZREG_TMP1 7383 } else { 7384 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 7385 } 7386 } 7387 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 7388 if (op2_info & MAY_BE_DOUBLE) { 7389 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 7390 |.cold_code 7391 |3: 7392 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7393 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7394 } 7395 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7396 return 0; 7397 } 7398 | b >6 7399 |.code 7400 } else { 7401 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 7402 } 7403 } 7404 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 7405 return 0; 7406 } 7407 if (op1_info & MAY_BE_DOUBLE) { 7408 |.cold_code 7409 |4: 7410 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7411 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 7412 } 7413 if (op2_info & MAY_BE_DOUBLE) { 7414 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7415 if (!same_ops) { 7416 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5, ZREG_TMP1 7417 } else { 7418 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7419 } 7420 } 7421 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7422 return 0; 7423 } 7424 | b >6 7425 } 7426 if (!same_ops) { 7427 |5: 7428 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7429 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 7430 } 7431 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7432 return 0; 7433 } 7434 | b >6 7435 } 7436 |.code 7437 } 7438 } else if ((op1_info & MAY_BE_DOUBLE) && 7439 !(op1_info & MAY_BE_LONG) && 7440 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7441 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7442 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 7443 } 7444 if (op2_info & MAY_BE_DOUBLE) { 7445 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7446 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7447 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3, ZREG_TMP1 7448 } else { 7449 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7450 } 7451 } 7452 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7453 return 0; 7454 } 7455 } 7456 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7457 if (op2_info & MAY_BE_DOUBLE) { 7458 |.cold_code 7459 } 7460 |3: 7461 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7462 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 7463 } 7464 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7465 return 0; 7466 } 7467 if (op2_info & MAY_BE_DOUBLE) { 7468 | b >6 7469 |.code 7470 } 7471 } 7472 } else if ((op2_info & MAY_BE_DOUBLE) && 7473 !(op2_info & MAY_BE_LONG) && 7474 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7475 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7476 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7477 } 7478 if (op1_info & MAY_BE_DOUBLE) { 7479 if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7480 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7481 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3, ZREG_TMP1 7482 } else { 7483 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 7484 } 7485 } 7486 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7487 return 0; 7488 } 7489 } 7490 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7491 if (op1_info & MAY_BE_DOUBLE) { 7492 |.cold_code 7493 } 7494 |3: 7495 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7496 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 7497 } 7498 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7499 return 0; 7500 } 7501 if (op1_info & MAY_BE_DOUBLE) { 7502 | b >6 7503 |.code 7504 } 7505 } 7506 } 7507 7508 if (has_slow || 7509 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7510 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 7511 if (has_slow) { 7512 |.cold_code 7513 |9: 7514 } 7515 | SET_EX_OPLINE opline, REG0 7516 if (Z_MODE(op1_addr) == IS_REG) { 7517 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 7518 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 7519 return 0; 7520 } 7521 op1_addr = real_addr; 7522 } 7523 if (Z_MODE(op2_addr) == IS_REG) { 7524 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 7525 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 7526 return 0; 7527 } 7528 op2_addr = real_addr; 7529 } 7530 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7531 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { 7532 | IF_NOT_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w 7533 | LOAD_32BIT_VAL FCARG1x, opline->op1.var 7534 | EXT_CALL zend_jit_undefined_op_helper, REG0 7535 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 7536 |1: 7537 } 7538 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { 7539 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 7540 | str FCARG1x, T1 // save 7541 | LOAD_32BIT_VAL FCARG1x, opline->op2.var 7542 | EXT_CALL zend_jit_undefined_op_helper, REG0 7543 | ldr FCARG1x, T1 // restore 7544 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 7545 | b >2 7546 |1: 7547 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7548 |2: 7549 } else { 7550 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7551 } 7552 | EXT_CALL zend_compare, REG0 7553 if ((opline->opcode != ZEND_CASE && 7554 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7555 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 7556 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7557 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 7558 | str RETVALw, T1 // save 7559 if (opline->opcode != ZEND_CASE) { 7560 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 7561 } 7562 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 7563 | ldr RETVALw, T1 // restore 7564 } 7565 if (may_throw) { 7566 zend_jit_check_exception_undef_result(Dst, opline); 7567 } 7568 if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7569 return 0; 7570 } 7571 if (has_slow) { 7572 | b >6 7573 |.code 7574 } 7575 } 7576 7577 |6: 7578 7579 return 1; 7580} 7581 7582static int zend_jit_identical(dasm_State **Dst, 7583 const zend_op *opline, 7584 uint32_t op1_info, 7585 zend_ssa_range *op1_range, 7586 zend_jit_addr op1_addr, 7587 uint32_t op2_info, 7588 zend_ssa_range *op2_range, 7589 zend_jit_addr op2_addr, 7590 zend_jit_addr res_addr, 7591 int may_throw, 7592 zend_uchar smart_branch_opcode, 7593 uint32_t target_label, 7594 uint32_t target_label2, 7595 const void *exit_addr, 7596 bool skip_comparison) 7597{ 7598 uint32_t identical_label = (uint32_t)-1; 7599 uint32_t not_identical_label = (uint32_t)-1; 7600 7601 if (smart_branch_opcode && !exit_addr) { 7602 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7603 if (smart_branch_opcode == ZEND_JMPZ) { 7604 not_identical_label = target_label; 7605 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7606 identical_label = target_label; 7607 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7608 not_identical_label = target_label; 7609 identical_label = target_label2; 7610 } else { 7611 ZEND_UNREACHABLE(); 7612 } 7613 } else { 7614 if (smart_branch_opcode == ZEND_JMPZ) { 7615 identical_label = target_label; 7616 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7617 not_identical_label = target_label; 7618 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7619 identical_label = target_label; 7620 not_identical_label = target_label2; 7621 } else { 7622 ZEND_UNREACHABLE(); 7623 } 7624 } 7625 } 7626 7627 if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && 7628 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { 7629 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 7630 return 0; 7631 } 7632 return 1; 7633 } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && 7634 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { 7635 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7636 return 0; 7637 } 7638 return 1; 7639 } 7640 7641 if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { 7642 op1_info |= MAY_BE_NULL; 7643 op2_info |= MAY_BE_NULL; 7644 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7645 | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w 7646 |.cold_code 7647 |1: 7648 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7649 | SET_EX_OPLINE opline, REG0 7650 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 7651 | EXT_CALL zend_jit_undefined_op_helper, REG0 7652 if (may_throw) { 7653 zend_jit_check_exception_undef_result(Dst, opline); 7654 } 7655 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 7656 | b >1 7657 |.code 7658 |1: 7659 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7660 | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w 7661 |.cold_code 7662 |1: 7663 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7664 | SET_EX_OPLINE opline, REG0 7665 | str FCARG1x, T1 // save 7666 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 7667 | EXT_CALL zend_jit_undefined_op_helper, REG0 7668 if (may_throw) { 7669 zend_jit_check_exception_undef_result(Dst, opline); 7670 } 7671 | ldr FCARG1x, T1 // restore 7672 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 7673 | b >1 7674 |.code 7675 |1: 7676 } else if (op1_info & MAY_BE_UNDEF) { 7677 op1_info |= MAY_BE_NULL; 7678 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7679 | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w 7680 |.cold_code 7681 |1: 7682 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7683 | SET_EX_OPLINE opline, REG0 7684 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 7685 | EXT_CALL zend_jit_undefined_op_helper, REG0 7686 if (may_throw) { 7687 zend_jit_check_exception_undef_result(Dst, opline); 7688 } 7689 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 7690 | b >1 7691 |.code 7692 |1: 7693 if (opline->op2_type != IS_CONST) { 7694 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7695 } 7696 } else if (op2_info & MAY_BE_UNDEF) { 7697 op2_info |= MAY_BE_NULL; 7698 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7699 | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w 7700 |.cold_code 7701 |1: 7702 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7703 | SET_EX_OPLINE opline, REG0 7704 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 7705 | EXT_CALL zend_jit_undefined_op_helper, REG0 7706 if (may_throw) { 7707 zend_jit_check_exception_undef_result(Dst, opline); 7708 } 7709 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 7710 | b >1 7711 |.code 7712 |1: 7713 if (opline->op1_type != IS_CONST) { 7714 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7715 } 7716 } else if ((op1_info & op2_info & MAY_BE_ANY) != 0) { 7717 if (opline->op1_type != IS_CONST) { 7718 if (Z_MODE(op1_addr) == IS_REG) { 7719 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 7720 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 7721 return 0; 7722 } 7723 op1_addr = real_addr; 7724 } 7725 } 7726 if (opline->op2_type != IS_CONST) { 7727 if (Z_MODE(op2_addr) == IS_REG) { 7728 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 7729 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 7730 return 0; 7731 } 7732 op2_addr = real_addr; 7733 } 7734 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7735 } 7736 if (opline->op1_type != IS_CONST) { 7737 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7738 } 7739 } 7740 7741 if ((op1_info & op2_info & MAY_BE_ANY) == 0) { 7742 if ((opline->opcode != ZEND_CASE_STRICT && 7743 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7744 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 7745 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7746 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 7747 if (opline->opcode != ZEND_CASE_STRICT) { 7748 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7749 } 7750 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7751 } 7752 if (smart_branch_opcode) { 7753 if (may_throw) { 7754 zend_jit_check_exception_undef_result(Dst, opline); 7755 } 7756 if (exit_addr) { 7757 if (smart_branch_opcode == ZEND_JMPZ) { 7758 | b &exit_addr 7759 } 7760 } else if (not_identical_label != (uint32_t)-1) { 7761 | b =>not_identical_label 7762 } 7763 } else { 7764 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 7765 if (may_throw) { 7766 zend_jit_check_exception(Dst); 7767 } 7768 } 7769 return 1; 7770 } 7771 7772 if (opline->op1_type & (IS_CV|IS_VAR)) { 7773 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 7774 } 7775 if (opline->op2_type & (IS_CV|IS_VAR)) { 7776 | ZVAL_DEREF FCARG2x, op2_info, TMP1w 7777 } 7778 7779 if (has_concrete_type(op1_info) 7780 && has_concrete_type(op2_info) 7781 && concrete_type(op1_info) == concrete_type(op2_info) 7782 && concrete_type(op1_info) <= IS_TRUE) { 7783 if (smart_branch_opcode) { 7784 if (exit_addr) { 7785 if (smart_branch_opcode == ZEND_JMPNZ) { 7786 | b &exit_addr 7787 } 7788 } else if (identical_label != (uint32_t)-1) { 7789 | b =>identical_label 7790 } 7791 } else { 7792 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 7793 } 7794 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { 7795 if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { 7796 if (smart_branch_opcode) { 7797 if (exit_addr) { 7798 if (smart_branch_opcode == ZEND_JMPNZ) { 7799 | b &exit_addr 7800 } 7801 } else if (identical_label != (uint32_t)-1) { 7802 | b =>identical_label 7803 } 7804 } else { 7805 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 7806 } 7807 } else { 7808 if (smart_branch_opcode) { 7809 if (exit_addr) { 7810 if (smart_branch_opcode == ZEND_JMPZ) { 7811 | b &exit_addr 7812 } 7813 } else if (not_identical_label != (uint32_t)-1) { 7814 | b =>not_identical_label 7815 } 7816 } else { 7817 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 7818 } 7819 } 7820 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { 7821 zval *val = Z_ZV(op1_addr); 7822 7823 | ldrb TMP1w, [FCARG2x, #offsetof(zval, u1.v.type)] 7824 | cmp TMP1w, #Z_TYPE_P(val) 7825 if (smart_branch_opcode) { 7826 if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { 7827 | bne >8 7828 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7829 if (may_throw) { 7830 zend_jit_check_exception_undef_result(Dst, opline); 7831 } 7832 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7833 | b &exit_addr 7834 } else if (identical_label != (uint32_t)-1) { 7835 | b =>identical_label 7836 } else { 7837 | b >9 7838 } 7839 |8: 7840 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7841 | beq &exit_addr 7842 } else if (identical_label != (uint32_t)-1) { 7843 | beq =>identical_label 7844 } else { 7845 | beq >9 7846 } 7847 } else { 7848 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7849 | cset REG0w, eq 7850 } else { 7851 | cset REG0w, ne 7852 } 7853 | add REG0w, REG0w, #2 7854 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7855 } 7856 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7857 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 7858 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7859 if (may_throw) { 7860 zend_jit_check_exception_undef_result(Dst, opline); 7861 } 7862 } 7863 if (exit_addr) { 7864 if (smart_branch_opcode == ZEND_JMPZ) { 7865 | b &exit_addr 7866 } 7867 } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { 7868 | b =>not_identical_label 7869 } 7870 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { 7871 zval *val = Z_ZV(op2_addr); 7872 7873 | ldrb TMP1w, [FCARG1x, #offsetof(zval, u1.v.type)] 7874 | cmp TMP1w, #Z_TYPE_P(val) 7875 if (smart_branch_opcode) { 7876 if (opline->opcode != ZEND_CASE_STRICT 7877 && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { 7878 | bne >8 7879 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7880 if (may_throw) { 7881 zend_jit_check_exception_undef_result(Dst, opline); 7882 } 7883 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7884 | b &exit_addr 7885 } else if (identical_label != (uint32_t)-1) { 7886 | b =>identical_label 7887 } else { 7888 | b >9 7889 } 7890 |8: 7891 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7892 | beq &exit_addr 7893 } else if (identical_label != (uint32_t)-1) { 7894 | beq =>identical_label 7895 } else { 7896 | beq >9 7897 } 7898 } else { 7899 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7900 | cset REG0w, eq 7901 } else { 7902 | cset REG0w, ne 7903 } 7904 | add REG0w, REG0w, #2 7905 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7906 } 7907 if (opline->opcode != ZEND_CASE_STRICT 7908 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7909 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 7910 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7911 if (may_throw) { 7912 zend_jit_check_exception_undef_result(Dst, opline); 7913 } 7914 } 7915 if (smart_branch_opcode) { 7916 if (exit_addr) { 7917 if (smart_branch_opcode == ZEND_JMPZ) { 7918 | b &exit_addr 7919 } 7920 } else if (not_identical_label != (uint32_t)-1) { 7921 | b =>not_identical_label 7922 } 7923 } 7924 } else { 7925 if (opline->op1_type == IS_CONST) { 7926 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7927 } 7928 if (opline->op2_type == IS_CONST) { 7929 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7930 } 7931 | EXT_CALL zend_is_identical, REG0 7932 if ((opline->opcode != ZEND_CASE_STRICT && 7933 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7934 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 7935 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7936 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 7937 | str RETVALw, T1 // save 7938 if (opline->opcode != ZEND_CASE_STRICT) { 7939 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7940 } 7941 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7942 if (may_throw) { 7943 zend_jit_check_exception_undef_result(Dst, opline); 7944 } 7945 | ldr RETVALw, T1 // restore 7946 } 7947 if (smart_branch_opcode) { 7948 if (exit_addr) { 7949 if (smart_branch_opcode == ZEND_JMPNZ) { 7950 | cbnz RETVALw, &exit_addr 7951 } else { 7952 | cbz RETVALw, &exit_addr 7953 } 7954 } else if (not_identical_label != (uint32_t)-1) { 7955 | cbz RETVALw, =>not_identical_label 7956 if (identical_label != (uint32_t)-1) { 7957 | b =>identical_label 7958 } 7959 } else if (identical_label != (uint32_t)-1) { 7960 | cbnz RETVALw, =>identical_label 7961 } 7962 } else { 7963 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7964 | add RETVALw, RETVALw, #2 7965 } else { 7966 | neg RETVALw, RETVALw 7967 | add RETVALw, RETVALw, #3 7968 } 7969 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, RETVALw, TMP1 7970 } 7971 } 7972 7973 |9: 7974 if (may_throw) { 7975 zend_jit_check_exception(Dst); 7976 } 7977 return 1; 7978} 7979 7980static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr) 7981{ 7982 uint32_t true_label = -1; 7983 uint32_t false_label = -1; 7984 bool set_bool = 0; 7985 bool set_bool_not = 0; 7986 bool set_delayed = 0; 7987 bool jmp_done = 0; 7988 7989 if (branch_opcode == ZEND_BOOL) { 7990 set_bool = 1; 7991 } else if (branch_opcode == ZEND_BOOL_NOT) { 7992 set_bool = 1; 7993 set_bool_not = 1; 7994 } else if (branch_opcode == ZEND_JMPZ) { 7995 false_label = target_label; 7996 } else if (branch_opcode == ZEND_JMPNZ) { 7997 true_label = target_label; 7998 } else if (branch_opcode == ZEND_JMPZNZ) { 7999 true_label = target_label2; 8000 false_label = target_label; 8001 } else if (branch_opcode == ZEND_JMPZ_EX) { 8002 set_bool = 1; 8003 false_label = target_label; 8004 } else if (branch_opcode == ZEND_JMPNZ_EX) { 8005 set_bool = 1; 8006 true_label = target_label; 8007 } else { 8008 ZEND_UNREACHABLE(); 8009 } 8010 8011 if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { 8012 if (zend_is_true(Z_ZV(op1_addr))) { 8013 /* Always TRUE */ 8014 if (set_bool) { 8015 if (set_bool_not) { 8016 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8017 } else { 8018 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8019 } 8020 } 8021 if (true_label != (uint32_t)-1) { 8022 | b =>true_label 8023 } 8024 } else { 8025 /* Always FALSE */ 8026 if (set_bool) { 8027 if (set_bool_not) { 8028 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8029 } else { 8030 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8031 } 8032 } 8033 if (false_label != (uint32_t)-1) { 8034 | b =>false_label 8035 } 8036 } 8037 return 1; 8038 } 8039 8040 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) { 8041 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8042 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 8043 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 8044 } 8045 8046 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { 8047 if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { 8048 /* Always TRUE */ 8049 if (set_bool) { 8050 if (set_bool_not) { 8051 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8052 } else { 8053 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8054 } 8055 } 8056 if (true_label != (uint32_t)-1) { 8057 | b =>true_label 8058 } 8059 } else { 8060 if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { 8061 /* Always FALSE */ 8062 if (set_bool) { 8063 if (set_bool_not) { 8064 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8065 } else { 8066 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8067 } 8068 } 8069 } else { 8070 | CMP_ZVAL_TYPE op1_addr, IS_TRUE, ZREG_TMP1 8071 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 8072 if ((op1_info & MAY_BE_LONG) && 8073 !(op1_info & MAY_BE_UNDEF) && 8074 !set_bool) { 8075 if (exit_addr) { 8076 if (branch_opcode == ZEND_JMPNZ) { 8077 | blt >9 8078 } else { 8079 | blt &exit_addr 8080 } 8081 } else if (false_label != (uint32_t)-1) { 8082 | blt =>false_label 8083 } else { 8084 | blt >9 8085 } 8086 jmp_done = 1; 8087 } else { 8088 | bgt >2 8089 } 8090 } 8091 if (!(op1_info & MAY_BE_TRUE)) { 8092 /* It's FALSE */ 8093 if (set_bool) { 8094 if (set_bool_not) { 8095 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8096 } else { 8097 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8098 } 8099 } 8100 } else { 8101 if (exit_addr) { 8102 if (set_bool) { 8103 | bne >1 8104 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8105 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8106 | b &exit_addr 8107 } else { 8108 | b >9 8109 } 8110 |1: 8111 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8112 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8113 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8114 | bne &exit_addr 8115 } 8116 } 8117 } else { 8118 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8119 | beq &exit_addr 8120 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8121 | bne &exit_addr 8122 } else { 8123 | beq >9 8124 } 8125 } 8126 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8127 if (set_bool) { 8128 | bne >1 8129 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8130 if (true_label != (uint32_t)-1) { 8131 | b =>true_label 8132 } else { 8133 | b >9 8134 } 8135 |1: 8136 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8137 } else { 8138 if (true_label != (uint32_t)-1) { 8139 | beq =>true_label 8140 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8141 | bne =>false_label 8142 jmp_done = 1; 8143 } else { 8144 | beq >9 8145 } 8146 } 8147 } else if (set_bool) { 8148 | cset REG0w, eq 8149 if (set_bool_not) { 8150 | neg REG0w, REG0w 8151 | add REG0w, REG0w, #3 8152 } else { 8153 | add REG0w, REG0w, #2 8154 } 8155 if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { 8156 set_delayed = 1; 8157 } else { 8158 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8159 } 8160 } 8161 } 8162 } 8163 8164 /* It's FALSE, but may be UNDEF */ 8165 if (op1_info & MAY_BE_UNDEF) { 8166 if (op1_info & MAY_BE_ANY) { 8167 if (set_delayed) { 8168 | CMP_ZVAL_TYPE op1_addr, IS_UNDEF, ZREG_TMP1 8169 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8170 | beq >1 8171 } else { 8172 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 8173 } 8174 |.cold_code 8175 |1: 8176 } 8177 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 8178 | SET_EX_OPLINE opline, REG0 8179 | EXT_CALL zend_jit_undefined_op_helper, REG0 8180 8181 if (may_throw) { 8182 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 8183 return 0; 8184 } 8185 } 8186 8187 if (exit_addr) { 8188 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8189 | b &exit_addr 8190 } 8191 } else if (false_label != (uint32_t)-1) { 8192 | b =>false_label 8193 } 8194 if (op1_info & MAY_BE_ANY) { 8195 if (exit_addr) { 8196 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8197 | b >9 8198 } 8199 } else if (false_label == (uint32_t)-1) { 8200 | b >9 8201 } 8202 |.code 8203 } 8204 } 8205 8206 if (!jmp_done) { 8207 if (exit_addr) { 8208 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8209 if (op1_info & MAY_BE_LONG) { 8210 | b >9 8211 } 8212 } else if (op1_info & MAY_BE_LONG) { 8213 | b &exit_addr 8214 } 8215 } else if (false_label != (uint32_t)-1) { 8216 | b =>false_label 8217 } else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 8218 | b >9 8219 } 8220 } 8221 } 8222 } 8223 8224 if (op1_info & MAY_BE_LONG) { 8225 |2: 8226 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8227 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 8228 } 8229 if (Z_MODE(op1_addr) == IS_REG) { 8230 | tst Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) 8231 } else { 8232 | LONG_CMP_WITH_CONST op1_addr, Z_L(0), TMP1, TMP2 8233 } 8234 if (set_bool) { 8235 | cset REG0w, ne 8236 if (set_bool_not) { 8237 | neg REG0w, REG0w 8238 | add REG0w, REG0w, #3 8239 } else { 8240 | add REG0w, REG0w, #2 8241 } 8242 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8243 } 8244 if (exit_addr) { 8245 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8246 | bne &exit_addr 8247 } else { 8248 | beq &exit_addr 8249 } 8250 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8251 if (true_label != (uint32_t)-1) { 8252 | bne =>true_label 8253 if (false_label != (uint32_t)-1) { 8254 | b =>false_label 8255 } 8256 } else { 8257 | beq =>false_label 8258 } 8259 } 8260 } 8261 8262 if ((op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) == MAY_BE_DOUBLE) { 8263 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8264 |.cold_code 8265 } 8266 |2: 8267 | fmov FPR0, xzr // TODO: "movi d0, #0" is not recognized by DynASM/arm64 8268 | DOUBLE_CMP ZREG_FPR0, op1_addr, ZREG_TMP1, ZREG_FPTMP 8269 8270 if (set_bool) { 8271 if (exit_addr) { 8272 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8273 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8274 | bvs &exit_addr 8275 | bne &exit_addr 8276 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8277 } else { 8278 | bvs >1 8279 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8280 | beq &exit_addr 8281 |1: 8282 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8283 } 8284 } else if (false_label != (uint32_t)-1) { // JMPZ_EX 8285 | bvs >1 8286 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8287 | beq => false_label 8288 |1: 8289 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8290 } else if (true_label != (uint32_t)-1) { // JMPNZ_EX 8291 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8292 | bvs => true_label 8293 | bne => true_label 8294 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8295 } else if (set_bool_not) { // BOOL_NOT 8296 | mov REG0w, #IS_FALSE 8297 | bvs >1 8298 | bne >1 8299 | mov REG0w, #IS_TRUE 8300 |1: 8301 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8302 } else { // BOOL 8303 | mov REG0w, #IS_TRUE 8304 | bvs >1 8305 | bne >1 8306 | mov REG0w, #IS_FALSE 8307 |1: 8308 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8309 } 8310 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8311 | b >9 8312 |.code 8313 } 8314 } else { 8315 if (exit_addr) { 8316 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8317 | bvs &exit_addr 8318 | bne &exit_addr 8319 |1: 8320 } else { 8321 | bvs >1 8322 | beq &exit_addr 8323 |1: 8324 } 8325 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8326 | b >9 8327 } 8328 } else { 8329 ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); 8330 if (false_label != (uint32_t)-1 ) { 8331 | bvs >1 8332 | beq => false_label 8333 |1: 8334 if (true_label != (uint32_t)-1) { 8335 | b =>true_label 8336 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8337 | b >9 8338 } 8339 } else { 8340 | bvs => true_label 8341 | bne => true_label 8342 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8343 | b >9 8344 } 8345 } 8346 } 8347 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8348 |.code 8349 } 8350 } 8351 } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8352 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8353 |.cold_code 8354 |2: 8355 } 8356 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8357 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8358 } 8359 | SET_EX_OPLINE opline, REG0 8360 | EXT_CALL zend_is_true, REG0 8361 | mov REG0, RETVALx 8362 8363 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8364 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8365 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8366 8367 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8368 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3, ZREG_TMP1, ZREG_TMP2 8369 } 8370 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 8371 | GC_DELREF FCARG1x, TMP1w 8372 | bne >3 8373 // In x86, r0 is used in macro ZVAL_DTOR_FUNC as temporary register, hence, r0 should be saved/restored 8374 // before/after this macro. In AArch64, TMP1 is used, but we still have to store REG0, 8375 // because it's clobbered by function call. 8376 | str REG0, T1 // save 8377 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 8378 | ldr REG0, T1 // restore 8379 |3: 8380 } 8381 if (may_throw) { 8382 | MEM_LOAD_64_ZTS ldr, REG1, executor_globals, exception, TMP1 8383 | cbnz REG1, ->exception_handler_undef 8384 } 8385 8386 if (set_bool) { 8387 if (set_bool_not) { 8388 | neg REG0w, REG0w 8389 | add REG0w, REG0w, #3 8390 } else { 8391 | add REG0w, REG0w, #2 8392 } 8393 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8394 if (exit_addr) { 8395 | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 8396 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8397 | bne &exit_addr 8398 } else { 8399 | beq &exit_addr 8400 } 8401 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8402 | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 8403 if (true_label != (uint32_t)-1) { 8404 | bne =>true_label 8405 if (false_label != (uint32_t)-1) { 8406 | b =>false_label 8407 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8408 | b >9 8409 } 8410 } else { 8411 | beq =>false_label 8412 } 8413 } 8414 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8415 | b >9 8416 |.code 8417 } 8418 } else { 8419 if (exit_addr) { 8420 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8421 | cbnz REG0w, &exit_addr 8422 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8423 | b >9 8424 } 8425 } else { 8426 | cbz REG0w, &exit_addr 8427 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8428 | b >9 8429 } 8430 } 8431 } else if (true_label != (uint32_t)-1) { 8432 | cbnz REG0w, =>true_label 8433 if (false_label != (uint32_t)-1) { 8434 | b =>false_label 8435 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8436 | b >9 8437 } 8438 } else { 8439 | cbz REG0w, =>false_label 8440 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8441 | b >9 8442 } 8443 } 8444 8445 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8446 |.code 8447 } 8448 } 8449 } 8450 8451 |9: 8452 8453 return 1; 8454} 8455 8456static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) 8457{ 8458 if (op1_addr != op1_def_addr) { 8459 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 8460 return 0; 8461 } 8462 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 8463 op1_addr = op1_def_addr; 8464 } 8465 } 8466 8467 if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0, 1)) { 8468 return 0; 8469 } 8470 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8471 return 0; 8472 } 8473 if (op1_info & MAY_BE_UNDEF) { 8474 zend_jit_check_exception(Dst); 8475 } 8476 return 1; 8477} 8478 8479static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 8480{ 8481 ZEND_ASSERT(opline->op1_type == IS_CV); 8482 8483 if (op2_addr != op2_def_addr) { 8484 if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { 8485 return 0; 8486 } 8487 if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { 8488 op2_addr = op2_def_addr; 8489 } 8490 } 8491 8492 if (Z_MODE(op1_addr) != IS_REG 8493 && Z_MODE(op1_use_addr) == IS_REG 8494 && !Z_LOAD(op1_use_addr) 8495 && !Z_STORE(op1_use_addr)) { 8496 /* Force type update */ 8497 op1_info |= MAY_BE_UNDEF; 8498 } 8499 if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, 8500 may_throw)) { 8501 return 0; 8502 } 8503 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { 8504 return 0; 8505 } 8506 if (opline->result_type != IS_UNUSED) { 8507 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8508 return 0; 8509 } 8510 } 8511 8512 return 1; 8513} 8514 8515/* copy of hidden zend_closure */ 8516typedef struct _zend_closure { 8517 zend_object std; 8518 zend_function func; 8519 zval this_ptr; 8520 zend_class_entry *called_scope; 8521 zif_handler orig_internal_handler; 8522} zend_closure; 8523 8524static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) 8525{ 8526 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8527 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8528 8529 if (!exit_addr) { 8530 return 0; 8531 } 8532 8533 | // Check Stack Overflow 8534 | MEM_LOAD_64_ZTS ldr, REG1, executor_globals, vm_stack_end, TMP1 8535 | MEM_LOAD_OP_ZTS sub, ldr, REG1, executor_globals, vm_stack_top, TMP1, TMP2 8536 | CMP_64_WITH_CONST_32 REG1, used_stack, TMP1 8537 | blo &exit_addr 8538 8539 return 1; 8540} 8541 8542static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack) 8543{ 8544 uint32_t used_stack; 8545 bool stack_check = 1; 8546 8547 // REG0 -> zend_function 8548 // FCARG1 -> used_stack 8549 8550 if (func) { 8551 used_stack = zend_vm_calc_used_stack(opline->extended_value, func); 8552 if ((int)used_stack <= checked_stack) { 8553 stack_check = 0; 8554 } 8555 } else { 8556 used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval); 8557 8558 | // if (EXPECTED(ZEND_USER_CODE(func->type))) { 8559 if (!is_closure) { 8560 | LOAD_32BIT_VAL FCARG1w, used_stack 8561 | // Check whether REG0 is an internal function. 8562 | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] 8563 | TST_32_WITH_CONST TMP1w, 1, TMP2w 8564 | bne >1 8565 } else { 8566 | LOAD_32BIT_VAL FCARG1w, used_stack 8567 } 8568 | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); 8569 | LOAD_32BIT_VAL REG2w, opline->extended_value 8570 if (!is_closure) { 8571 | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.num_args)] 8572 | cmp REG2w, TMP1w 8573 | csel REG2w, REG2w, TMP1w, le 8574 | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.last_var)] 8575 | sub REG2w, REG2w, TMP1w 8576 | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.T)] 8577 | sub REG2w, REG2w, TMP1w 8578 } else { 8579 | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.num_args)] 8580 | cmp REG2w, TMP1w 8581 | csel REG2w, REG2w, TMP1w, le 8582 | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.last_var)] 8583 | sub REG2w, REG2w, TMP1w 8584 | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.T)] 8585 | sub REG2w, REG2w, TMP1w 8586 } 8587 | sxtw REG2, REG2w 8588 | sub FCARG1x, FCARG1x, REG2, lsl #4 8589 |1: 8590 } 8591 8592 zend_jit_start_reuse_ip(); 8593 8594 | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { 8595 | MEM_LOAD_64_ZTS ldr, RX, executor_globals, vm_stack_top, TMP1 8596 8597 if (stack_check) { 8598 | // Check Stack Overflow 8599 | MEM_LOAD_64_ZTS ldr, REG2, executor_globals, vm_stack_end, TMP1 8600 | sub REG2, REG2, RX 8601 if (func) { 8602 | CMP_64_WITH_CONST_32 REG2, used_stack, TMP1 8603 } else { 8604 | cmp REG2, FCARG1x 8605 } 8606 8607 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8608 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8609 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8610 8611 if (!exit_addr) { 8612 return 0; 8613 } 8614 8615 | blo &exit_addr 8616 } else { 8617 | blo >1 8618 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); 8619 |.cold_code 8620 |1: 8621 if (func) { 8622 | LOAD_32BIT_VAL FCARG1w, used_stack 8623 } 8624 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 8625 | SET_EX_OPLINE opline, REG0 8626 | EXT_CALL zend_jit_int_extend_stack_helper, REG0 8627 } else { 8628 if (!is_closure) { 8629 | mov FCARG2x, REG0 8630 } else { 8631 | add FCARG2x, REG0, #offsetof(zend_closure, func) 8632 } 8633 | SET_EX_OPLINE opline, REG0 8634 | EXT_CALL zend_jit_extend_stack_helper, REG0 8635 } 8636 | mov RX, RETVALx 8637 | b >1 8638 |.code 8639 } 8640 } 8641 8642 if (func) { 8643 || if (arm64_may_encode_imm12((int64_t)used_stack)) { 8644 | MEM_UPDATE_ZTS add, ldr, str, #used_stack, executor_globals, vm_stack_top, REG2, TMP1 8645 || } else { 8646 | LOAD_32BIT_VAL TMP1w, used_stack 8647 | MEM_UPDATE_ZTS add, ldr, str, TMP1, executor_globals, vm_stack_top, REG2, TMP2 8648 || } 8649 } else { 8650 | MEM_UPDATE_ZTS add, ldr, str, FCARG1x, executor_globals, vm_stack_top, REG2, TMP1 8651 } 8652 | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); 8653 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { 8654 | // ZEND_SET_CALL_INFO(call, 0, call_info); 8655 | LOAD_32BIT_VAL TMP1w, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) 8656 | str TMP1w, EX:RX->This.u1.type_info 8657 } 8658 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 8659 | // call->func = func; 8660 |1: 8661 | ADDR_STORE EX:RX->func, func, REG1 8662 } else { 8663 if (!is_closure) { 8664 | // call->func = func; 8665 | str REG0, EX:RX->func 8666 } else { 8667 | // call->func = &closure->func; 8668 | add REG1, REG0, #offsetof(zend_closure, func) 8669 | str REG1, EX:RX->func 8670 } 8671 |1: 8672 } 8673 if (opline->opcode == ZEND_INIT_METHOD_CALL) { 8674 | // Z_PTR(call->This) = obj; 8675 | ldr REG1, T1 8676 | str REG1, EX:RX->This.value.ptr 8677 if (opline->op1_type == IS_UNUSED || delayed_fetch_this) { 8678 | // call->call_info |= ZEND_CALL_HAS_THIS; 8679 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8680 | LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS 8681 | str TMP1w, EX:RX->This.u1.type_info 8682 } else { 8683 | ldr TMP1w, EX:RX->This.u1.type_info 8684 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_HAS_THIS, TMP2w 8685 | str TMP1w, EX:RX->This.u1.type_info 8686 } 8687 } else { 8688 if (opline->op1_type == IS_CV) { 8689 | // GC_ADDREF(obj); 8690 | GC_ADDREF REG1, TMP1w 8691 } 8692 | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; 8693 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8694 | LOAD_32BIT_VAL TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 8695 | str TMP1w, EX:RX->This.u1.type_info 8696 } else { 8697 | ldr TMP1w, EX:RX->This.u1.type_info 8698 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS), TMP2w 8699 | str TMP1w, EX:RX->This.u1.type_info 8700 } 8701 } 8702 } else if (!is_closure) { 8703 | // Z_CE(call->This) = called_scope; 8704 | str xzr, EX:RX->This.value.ptr 8705 } else { 8706 if (opline->op2_type == IS_CV) { 8707 | // GC_ADDREF(closure); 8708 | GC_ADDREF REG0, TMP1w 8709 } 8710 | // object_or_called_scope = closure->called_scope; 8711 | ldr REG1, [REG0, #offsetof(zend_closure, called_scope)] 8712 | str REG1, EX:RX->This.value.ptr 8713 | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | 8714 | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); 8715 | ldr REG2w, [REG0, #offsetof(zend_closure, func.common.fn_flags)] 8716 | BW_OP_32_WITH_CONST and, REG2w, REG2w, ZEND_ACC_FAKE_CLOSURE, TMP1w 8717 | BW_OP_32_WITH_CONST orr, REG2w, REG2w, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE), TMP1w 8718 | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { 8719 | ldrb TMP1w, [REG0, #offsetof(zend_closure, this_ptr.u1.v.type)] 8720 | cmp TMP1w, #IS_UNDEF 8721 | beq >1 8722 | // call_info |= ZEND_CALL_HAS_THIS; 8723 | BW_OP_32_WITH_CONST orr, REG2w, REG2w, ZEND_CALL_HAS_THIS, TMP1w 8724 | // object_or_called_scope = Z_OBJ(closure->this_ptr); 8725 | ldr REG1, [REG0, #offsetof(zend_closure, this_ptr.value.ptr)] 8726 |1: 8727 | // ZEND_SET_CALL_INFO(call, 0, call_info); 8728 | ldr TMP1w, EX:RX->This.u1.type_info 8729 | orr TMP1w, TMP1w, REG2w 8730 | str TMP1w, EX:RX->This.u1.type_info 8731 | // Z_PTR(call->This) = object_or_called_scope; 8732 | str REG1, EX:RX->This.value.ptr 8733 | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.run_time_cache__ptr)] 8734 | cbnz TMP1, >1 8735 | add FCARG1x, REG0, #offsetof(zend_closure, func) 8736 | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 8737 |1: 8738 } 8739 | // ZEND_CALL_NUM_ARGS(call) = num_args; 8740 | LOAD_32BIT_VAL TMP1w, opline->extended_value 8741 | str TMP1w, EX:RX->This.u2.num_args 8742 8743 return 1; 8744} 8745 8746static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) 8747{ 8748 int32_t exit_point; 8749 const void *exit_addr; 8750 8751 if (func->type == ZEND_INTERNAL_FUNCTION) { 8752#ifdef ZEND_WIN32 8753 // TODO: ASLR may cause different addresses in different workers ??? 8754 return 0; 8755#endif 8756 } else if (func->type == ZEND_USER_FUNCTION) { 8757 if (!zend_accel_in_shm(func->op_array.opcodes)) { 8758 /* op_array and op_array->opcodes are not persistent. We can't link. */ 8759 return 0; 8760 } 8761 } else { 8762 ZEND_UNREACHABLE(); 8763 return 0; 8764 } 8765 8766 exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); 8767 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8768 if (!exit_addr) { 8769 return 0; 8770 } 8771 8772 | // call = EX(call); 8773 | ldr REG1, EX->call 8774 while (level > 0) { 8775 | ldr REG1, EX:REG1->prev_execute_data 8776 level--; 8777 } 8778 8779 if (func->type == ZEND_USER_FUNCTION && 8780 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 8781 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 8782 !func->common.function_name)) { 8783 const zend_op *opcodes = func->op_array.opcodes; 8784 8785 | ldr REG1, EX:REG1->func 8786 | LOAD_ADDR REG2, ((ptrdiff_t)opcodes) 8787 | ldr TMP1, [REG1, #offsetof(zend_op_array, opcodes)] 8788 | cmp TMP1, REG2 8789 | bne &exit_addr 8790 } else { 8791 | LOAD_ADDR REG2, ((ptrdiff_t)func) 8792 | ldr TMP1, EX:REG1->func 8793 | cmp TMP1, REG2 8794 | bne &exit_addr 8795 } 8796 8797 return 1; 8798} 8799 8800static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack) 8801{ 8802 zend_func_info *info = ZEND_FUNC_INFO(op_array); 8803 zend_call_info *call_info = NULL; 8804 zend_function *func = NULL; 8805 8806 if (delayed_call_chain) { 8807 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 8808 return 0; 8809 } 8810 } 8811 8812 if (info) { 8813 call_info = info->callee_info; 8814 while (call_info && call_info->caller_init_opline != opline) { 8815 call_info = call_info->next_callee; 8816 } 8817 if (call_info && call_info->callee_func && !call_info->is_prototype) { 8818 func = call_info->callee_func; 8819 } 8820 } 8821 8822 if (!func 8823 && trace 8824 && trace->op == ZEND_JIT_TRACE_INIT_CALL) { 8825 func = (zend_function*)trace->func; 8826 } 8827 8828 if (opline->opcode == ZEND_INIT_FCALL 8829 && func 8830 && func->type == ZEND_INTERNAL_FUNCTION) { 8831 /* load constant address later */ 8832 } else if (func && op_array == &func->op_array) { 8833 /* recursive call */ 8834 | ldr REG0, EX->func 8835 } else { 8836 | // if (CACHED_PTR(opline->result.num)) 8837 | ldr REG2, EX->run_time_cache 8838 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG2, opline->result.num, TMP1 8839 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 8840 && func 8841 && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) 8842 && opline->opcode != ZEND_INIT_FCALL) { 8843 /* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */ 8844 | LOAD_ADDR REG1, ((ptrdiff_t)func) 8845 | cmp REG0, REG1 8846 | bne >1 8847 } else { 8848 | cbz REG0, >1 8849 } 8850 |.cold_code 8851 |1: 8852 if (opline->opcode == ZEND_INIT_FCALL 8853 && func 8854 && func->type == ZEND_USER_FUNCTION 8855 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { 8856 | LOAD_ADDR FCARG1x, func 8857 | MEM_ACCESS_64_WITH_UOFFSET str, FCARG1x, REG2, opline->result.num, TMP1 8858 | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 8859 | mov REG0, RETVALx 8860 | b >3 8861 } else { 8862 zval *zv = RT_CONSTANT(opline, opline->op2); 8863 8864 if (opline->opcode == ZEND_INIT_FCALL) { 8865 | LOAD_ADDR FCARG1x, Z_STR_P(zv); 8866 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 8867 | EXT_CALL zend_jit_find_func_helper, REG0 8868 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { 8869 | LOAD_ADDR FCARG1x, Z_STR_P(zv + 1); 8870 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 8871 | EXT_CALL zend_jit_find_func_helper, REG0 8872 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { 8873 | LOAD_ADDR FCARG1x, zv; 8874 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 8875 | EXT_CALL zend_jit_find_ns_func_helper, REG0 8876 } else { 8877 ZEND_UNREACHABLE(); 8878 } 8879 | // Get the return value of function zend_jit_find_func_helper/zend_jit_find_ns_func_helper 8880 | mov REG0, RETVALx 8881 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8882 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 8883 func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0); 8884 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8885 8886 if (!exit_addr) { 8887 return 0; 8888 } 8889 8890 if (!func || opline->opcode == ZEND_INIT_FCALL) { 8891 | cbnz REG0, >3 8892 } else if (func->type == ZEND_USER_FUNCTION 8893 && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { 8894 const zend_op *opcodes = func->op_array.opcodes; 8895 8896 | LOAD_ADDR REG1, ((ptrdiff_t)opcodes) 8897 | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] 8898 | cmp TMP1, REG1 8899 | beq >3 8900 } else { 8901 | LOAD_ADDR REG1, ((ptrdiff_t)func) 8902 | cmp REG0, REG1 8903 | beq >3 8904 } 8905 | b &exit_addr 8906 } else { 8907 | cbnz REG0, >3 8908 | // SAVE_OPLINE(); 8909 | SET_EX_OPLINE opline, REG0 8910 | b ->undefined_function 8911 } 8912 } 8913 |.code 8914 |3: 8915 } 8916 8917 if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, checked_stack)) { 8918 return 0; 8919 } 8920 8921 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 8922 if (!zend_jit_save_call_chain(Dst, call_level)) { 8923 return 0; 8924 } 8925 } else { 8926 delayed_call_chain = 1; 8927 delayed_call_level = call_level; 8928 } 8929 8930 return 1; 8931} 8932 8933static int zend_jit_init_method_call(dasm_State **Dst, 8934 const zend_op *opline, 8935 uint32_t b, 8936 const zend_op_array *op_array, 8937 zend_ssa *ssa, 8938 const zend_ssa_op *ssa_op, 8939 int call_level, 8940 uint32_t op1_info, 8941 zend_jit_addr op1_addr, 8942 zend_class_entry *ce, 8943 bool ce_is_instanceof, 8944 bool on_this, 8945 bool delayed_fetch_this, 8946 zend_class_entry *trace_ce, 8947 zend_jit_trace_rec *trace, 8948 int checked_stack, 8949 bool polymorphic_side_trace) 8950{ 8951 zend_func_info *info = ZEND_FUNC_INFO(op_array); 8952 zend_call_info *call_info = NULL; 8953 zend_function *func = NULL; 8954 zval *function_name; 8955 8956 ZEND_ASSERT(opline->op2_type == IS_CONST); 8957 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 8958 8959 function_name = RT_CONSTANT(opline, opline->op2); 8960 8961 if (info) { 8962 call_info = info->callee_info; 8963 while (call_info && call_info->caller_init_opline != opline) { 8964 call_info = call_info->next_callee; 8965 } 8966 if (call_info && call_info->callee_func && !call_info->is_prototype) { 8967 func = call_info->callee_func; 8968 } 8969 } 8970 8971 if (polymorphic_side_trace) { 8972 /* function is passed in r0 from parent_trace */ 8973 } else { 8974 if (on_this) { 8975 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 8976 8977 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 8978 } else { 8979 if (op1_info & MAY_BE_REF) { 8980 if (opline->op1_type == IS_CV) { 8981 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8982 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8983 } 8984 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 8985 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 8986 } else { 8987 /* Hack: Convert reference to regular value to simplify JIT code */ 8988 ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); 8989 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 8990 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8991 | EXT_CALL zend_jit_unref_helper, REG0 8992 |1: 8993 } 8994 } 8995 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 8996 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8997 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8998 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8999 9000 if (!exit_addr) { 9001 return 0; 9002 } 9003 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 9004 } else { 9005 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 9006 |.cold_code 9007 |1: 9008 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 9009 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 9010 } 9011 | SET_EX_OPLINE opline, REG0 9012 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9013 | EXT_CALL zend_jit_invalid_method_call_tmp, REG0 9014 } else { 9015 | EXT_CALL zend_jit_invalid_method_call, REG0 9016 } 9017 | b ->exception_handler 9018 |.code 9019 } 9020 } 9021 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 9022 } 9023 9024 if (delayed_call_chain) { 9025 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9026 return 0; 9027 } 9028 } 9029 9030 | str FCARG1x, T1 // save 9031 9032 if (func) { 9033 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9034 | ldr REG0, EX->run_time_cache 9035 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 9036 | cbz REG0, >1 9037 } else { 9038 | // if (CACHED_PTR(opline->result.num) == obj->ce)) { 9039 | ldr REG0, EX->run_time_cache 9040 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->result.num, TMP1 9041 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 9042 | cmp REG2, TMP1 9043 | bne >1 9044 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9045 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 9046 } 9047 9048 |.cold_code 9049 |1: 9050 | LOAD_ADDR FCARG2x, function_name 9051 if (TMP_ZVAL_OFFSET == 0) { 9052 | mov CARG3, sp 9053 } else { 9054 | add CARG3, sp, #TMP_ZVAL_OFFSET 9055 } 9056 | SET_EX_OPLINE opline, REG0 9057 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9058 | EXT_CALL zend_jit_find_method_tmp_helper, REG0 9059 } else { 9060 | EXT_CALL zend_jit_find_method_helper, REG0 9061 } 9062 | mov REG0, RETVALx 9063 | cbnz REG0, >2 9064 | b ->exception_handler 9065 |.code 9066 |2: 9067 } 9068 9069 if ((!func || zend_jit_may_be_modified(func, op_array)) 9070 && trace 9071 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9072 && trace->func 9073 ) { 9074 int32_t exit_point; 9075 const void *exit_addr; 9076 9077 exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); 9078 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9079 if (!exit_addr) { 9080 return 0; 9081 } 9082 9083 func = (zend_function*)trace->func; 9084 9085 if (func->type == ZEND_USER_FUNCTION && 9086 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 9087 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 9088 !func->common.function_name)) { 9089 const zend_op *opcodes = func->op_array.opcodes; 9090 9091 | LOAD_ADDR TMP1, opcodes 9092 | ldr TMP2, [REG0, #offsetof(zend_op_array, opcodes)] 9093 | cmp TMP2, TMP1 9094 | bne &exit_addr 9095 } else { 9096 | LOAD_ADDR TMP1, func 9097 | cmp REG0, TMP1 9098 | bne &exit_addr 9099 } 9100 } 9101 9102 if (!func) { 9103 | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { 9104 | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] 9105 | TST_32_WITH_CONST TMP1w, ZEND_ACC_STATIC, TMP2w 9106 | bne >1 9107 |.cold_code 9108 |1: 9109 } 9110 9111 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { 9112 | ldr FCARG1x, T1 // restore 9113 | mov FCARG2x, REG0 9114 | LOAD_32BIT_VAL CARG3w, opline->extended_value 9115 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9116 | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, REG0 9117 } else { 9118 | EXT_CALL zend_jit_push_static_metod_call_frame, REG0 9119 } 9120 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) { 9121 | cbz RETVALx, ->exception_handler 9122 } 9123 | mov RX, RETVALx 9124 } 9125 9126 if (!func) { 9127 | b >9 9128 |.code 9129 } 9130 9131 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { 9132 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, delayed_fetch_this, checked_stack)) { 9133 return 0; 9134 } 9135 } 9136 9137 if (!func) { 9138 |9: 9139 } 9140 zend_jit_start_reuse_ip(); 9141 9142 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9143 if (!zend_jit_save_call_chain(Dst, call_level)) { 9144 return 0; 9145 } 9146 } else { 9147 delayed_call_chain = 1; 9148 delayed_call_level = call_level; 9149 } 9150 9151 return 1; 9152} 9153 9154static int zend_jit_init_closure_call(dasm_State **Dst, 9155 const zend_op *opline, 9156 uint32_t b, 9157 const zend_op_array *op_array, 9158 zend_ssa *ssa, 9159 const zend_ssa_op *ssa_op, 9160 int call_level, 9161 zend_jit_trace_rec *trace, 9162 int checked_stack) 9163{ 9164 zend_function *func = NULL; 9165 zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 9166 9167 | GET_ZVAL_PTR REG0, op2_addr, TMP1 9168 9169 if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure 9170 && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { 9171 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9172 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9173 9174 if (!exit_addr) { 9175 return 0; 9176 } 9177 9178 | LOAD_ADDR FCARG1x, ((ptrdiff_t)zend_ce_closure) 9179 | ldr, TMP1, [REG0, #offsetof(zend_object, ce)] 9180 | cmp TMP1, FCARG1x 9181 | bne &exit_addr 9182 if (ssa->var_info && ssa_op->op2_use >= 0) { 9183 ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; 9184 ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; 9185 ssa->var_info[ssa_op->op2_use].is_instanceof = 0; 9186 } 9187 } 9188 9189 if (trace 9190 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9191 && trace->func 9192 && trace->func->type == ZEND_USER_FUNCTION) { 9193 const zend_op *opcodes; 9194 int32_t exit_point; 9195 const void *exit_addr; 9196 9197 func = (zend_function*)trace->func; 9198 opcodes = func->op_array.opcodes; 9199 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); 9200 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9201 if (!exit_addr) { 9202 return 0; 9203 } 9204 9205 | LOAD_ADDR FCARG1x, ((ptrdiff_t)opcodes) 9206 | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.opcodes)] 9207 | cmp TMP1, FCARG1x 9208 | bne &exit_addr 9209 } 9210 9211 if (delayed_call_chain) { 9212 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9213 return 0; 9214 } 9215 } 9216 9217 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, checked_stack)) { 9218 return 0; 9219 } 9220 9221 if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9222 if (!zend_jit_save_call_chain(Dst, call_level)) { 9223 return 0; 9224 } 9225 } else { 9226 delayed_call_chain = 1; 9227 delayed_call_level = call_level; 9228 } 9229 9230 if (trace 9231 && trace->op == ZEND_JIT_TRACE_END 9232 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 9233 if (!zend_jit_set_valid_ip(Dst, opline + 1)) { 9234 return 0; 9235 } 9236 } 9237 9238 return 1; 9239} 9240 9241static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) 9242{ 9243 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9244 zend_call_info *call_info = NULL; 9245 const zend_function *func = NULL; 9246 uint32_t i; 9247 zend_jit_addr res_addr; 9248 uint32_t call_num_args = 0; 9249 bool unknown_num_args = 0; 9250 const void *exit_addr = NULL; 9251 const zend_op *prev_opline; 9252 9253 if (RETURN_VALUE_USED(opline)) { 9254 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 9255 } else { 9256 /* CPU stack allocated temporary zval */ 9257 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RSP, TMP_ZVAL_OFFSET); 9258 } 9259 9260 prev_opline = opline - 1; 9261 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { 9262 prev_opline--; 9263 } 9264 if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || 9265 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 9266 unknown_num_args = 1; 9267 } 9268 9269 if (info) { 9270 call_info = info->callee_info; 9271 while (call_info && call_info->caller_call_opline != opline) { 9272 call_info = call_info->next_callee; 9273 } 9274 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9275 func = call_info->callee_func; 9276 } 9277 if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) 9278 && JIT_G(current_frame) 9279 && JIT_G(current_frame)->call 9280 && !JIT_G(current_frame)->call->func) { 9281 call_info = NULL; func = NULL; /* megamorphic call from trait */ 9282 } 9283 } 9284 if (!func) { 9285 /* resolve function at run time */ 9286 } else if (func->type == ZEND_USER_FUNCTION) { 9287 ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); 9288 call_num_args = call_info->num_args; 9289 } else if (func->type == ZEND_INTERNAL_FUNCTION) { 9290 ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); 9291 call_num_args = call_info->num_args; 9292 } else { 9293 ZEND_UNREACHABLE(); 9294 } 9295 9296 if (trace && !func) { 9297 if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { 9298 ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); 9299#ifndef ZEND_WIN32 9300 // TODO: ASLR may cause different addresses in different workers ??? 9301 func = trace->func; 9302 if (JIT_G(current_frame) && 9303 JIT_G(current_frame)->call && 9304 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9305 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9306 } else { 9307 unknown_num_args = 1; 9308 } 9309#endif 9310 } else if (trace->op == ZEND_JIT_TRACE_ENTER) { 9311 ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); 9312 if (zend_accel_in_shm(trace->func->op_array.opcodes)) { 9313 func = trace->func; 9314 if (JIT_G(current_frame) && 9315 JIT_G(current_frame)->call && 9316 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9317 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9318 } else { 9319 unknown_num_args = 1; 9320 } 9321 } 9322 } 9323 } 9324 9325 bool may_have_extra_named_params = 9326 opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && 9327 (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); 9328 9329 if (!reuse_ip) { 9330 zend_jit_start_reuse_ip(); 9331 | // call = EX(call); 9332 | ldr RX, EX->call 9333 } 9334 zend_jit_stop_reuse_ip(); 9335 9336 | // fbc = call->func; 9337 | // mov r2, EX:RX->func ??? 9338 | // SAVE_OPLINE(); 9339 | SET_EX_OPLINE opline, REG0 9340 9341 if (opline->opcode == ZEND_DO_FCALL) { 9342 if (!func) { 9343 if (trace) { 9344 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9345 9346 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9347 if (!exit_addr) { 9348 return 0; 9349 } 9350 | ldr REG0, EX:RX->func 9351 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9352 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9353 | bne &exit_addr 9354 } 9355 } 9356 } 9357 9358 if (!delayed_call_chain) { 9359 if (call_level == 1) { 9360 | str xzr, EX->call 9361 } else { 9362 | //EX(call) = call->prev_execute_data; 9363 | ldr REG0, EX:RX->prev_execute_data 9364 | str REG0, EX->call 9365 } 9366 } 9367 delayed_call_chain = 0; 9368 9369 | //call->prev_execute_data = execute_data; 9370 | str EX, EX:RX->prev_execute_data 9371 9372 if (!func) { 9373 | ldr REG0, EX:RX->func 9374 } 9375 9376 if (opline->opcode == ZEND_DO_FCALL) { 9377 if (!func) { 9378 if (!trace) { 9379 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9380 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9381 | bne >1 9382 |.cold_code 9383 |1: 9384 if (!GCC_GLOBAL_REGS) { 9385 | mov FCARG1x, RX 9386 } 9387 | EXT_CALL zend_jit_deprecated_helper, REG0 9388 | GET_LOW_8BITS RETVALw, RETVALw 9389 | ldr REG0, EX:RX->func // reload 9390 | cbnz RETVALw, >1 // Result is 0 on exception 9391 | b ->exception_handler 9392 |.code 9393 |1: 9394 } 9395 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 9396 if (!GCC_GLOBAL_REGS) { 9397 | mov FCARG1x, RX 9398 } 9399 | EXT_CALL zend_jit_deprecated_helper, REG0 9400 | cbz RETVALw, ->exception_handler 9401 } 9402 } 9403 9404 if (!func 9405 && opline->opcode != ZEND_DO_UCALL 9406 && opline->opcode != ZEND_DO_ICALL) { 9407 | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] 9408 | cmp TMP1w, #ZEND_USER_FUNCTION 9409 | bne >8 9410 } 9411 9412 if ((!func || func->type == ZEND_USER_FUNCTION) 9413 && opline->opcode != ZEND_DO_ICALL) { 9414 | // EX(call) = NULL; 9415 | str xzr, EX:RX->call 9416 9417 if (RETURN_VALUE_USED(opline)) { 9418 | // EX(return_value) = EX_VAR(opline->result.var); 9419 | LOAD_ZVAL_ADDR REG2, res_addr 9420 | str REG2, EX:RX->return_value 9421 } else { 9422 | // EX(return_value) = 0; 9423 | str xzr, EX:RX->return_value 9424 } 9425 9426 //EX_LOAD_RUN_TIME_CACHE(op_array); 9427 if (!func || func->op_array.cache_size) { 9428 if (func && op_array == &func->op_array) { 9429 /* recursive call */ 9430 if (trace || func->op_array.cache_size > sizeof(void*)) { 9431 | ldr REG2, EX->run_time_cache 9432 | str REG2, EX:RX->run_time_cache 9433 } 9434 } else { 9435// Always defined as ZEND_MAP_PTR_KIND_PTR_OR_OFFSET. See Zend/zend_map_ptr.h. 9436#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR 9437 if (func) { 9438 | ldr REG0, EX:RX->func 9439 } 9440 | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] 9441 | ldr REG2, [REG2] 9442#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET 9443 if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { 9444 if (ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { 9445 | MEM_LOAD_64_ZTS ldr, REG2, compiler_globals, map_ptr_base, TMP1 9446 | ADD_SUB_64_WITH_CONST add, REG2, REG2, (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache), TMP1 9447 | ldr REG2, [REG2] 9448 } else if ((func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9449 && (!func->op_array.scope || (func->op_array.scope->ce_flags & ZEND_ACC_LINKED))) { 9450 if (func) { 9451 | ldr REG0, EX:RX->func 9452 } 9453 | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] 9454 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 9455 | ldr REG2, [REG2] 9456 } else { 9457 /* the called op_array may be not persisted yet */ 9458 if (func) { 9459 | ldr REG0, EX:RX->func 9460 } 9461 | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] 9462 | TST_64_WITH_ONE REG2 9463 | beq >1 9464 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 9465 |1: 9466 | ldr REG2, [REG2] 9467 } 9468 } else { 9469 if (func) { 9470 | ldr REG0, EX:RX->func 9471 } 9472 | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] 9473 | TST_64_WITH_ONE REG2 9474 | beq >1 9475 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 9476 |1: 9477 | ldr REG2, [REG2] 9478 } 9479#else 9480# error "Unknown ZEND_MAP_PTR_KIND" 9481#endif 9482 | str REG2, EX:RX->run_time_cache 9483 } 9484 } 9485 9486 | // EG(current_execute_data) = execute_data; 9487 | MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1 9488 | mov FP, RX 9489 9490 | // opline = op_array->opcodes; 9491 if (func && !unknown_num_args) { 9492 | ADD_SUB_64_WITH_CONST_32 add, TMP1, RX, (EX_NUM_TO_VAR(call_num_args) + offsetof(zval, u1.type_info)), TMP1 // induction variable 9493 for (i = call_num_args; i < func->op_array.last_var; i++) { 9494 | // ZVAL_UNDEF(EX_VAR(n)) 9495 | str wzr, [TMP1], #16 9496 } 9497 9498 if (call_num_args <= func->op_array.num_args) { 9499 if (!trace || (trace->op == ZEND_JIT_TRACE_END 9500 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 9501 uint32_t num_args; 9502 9503 if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { 9504 if (trace) { 9505 num_args = 0; 9506 } else if (call_info) { 9507 num_args = skip_valid_arguments(op_array, ssa, call_info); 9508 } else { 9509 num_args = call_num_args; 9510 } 9511 } else { 9512 num_args = call_num_args; 9513 } 9514 if (zend_accel_in_shm(func->op_array.opcodes)) { 9515 | LOAD_IP_ADDR (func->op_array.opcodes + num_args) 9516 } else { 9517 | ldr REG0, EX->func 9518 || ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(num_args * sizeof(zend_op)))); 9519 if (GCC_GLOBAL_REGS) { 9520 | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] 9521 if (num_args) { 9522 | add IP, IP, #(num_args * sizeof(zend_op)) 9523 } 9524 } else { 9525 | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] 9526 if (num_args) { 9527 | add FCARG1x, FCARG1x, #(num_args * sizeof(zend_op)) 9528 } 9529 | str FCARG1x, EX->opline 9530 } 9531 } 9532 9533 if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array 9534 && num_args >= op_array->required_num_args) { 9535 /* recursive call */ 9536 if (ZEND_OBSERVER_ENABLED) { 9537 | SAVE_IP 9538 | mov FCARG1x, FP 9539 | EXT_CALL zend_observer_fcall_begin, REG0 9540 } 9541#ifdef CONTEXT_THREADED_JIT 9542 | NIY // TODO 9543#else 9544 | b =>num_args 9545#endif 9546 return 1; 9547 } 9548 } 9549 } else { 9550 if (!trace || (trace->op == ZEND_JIT_TRACE_END 9551 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 9552 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 9553 | LOAD_IP_ADDR (func->op_array.opcodes) 9554 } else if (GCC_GLOBAL_REGS) { 9555 | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] 9556 } else { 9557 | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] 9558 | str FCARG1x, EX->opline 9559 } 9560 } 9561 if (!GCC_GLOBAL_REGS) { 9562 | mov FCARG1x, FP 9563 } 9564 | EXT_CALL zend_jit_copy_extra_args_helper, REG0 9565 } 9566 } else { 9567 | // opline = op_array->opcodes 9568 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 9569 | LOAD_IP_ADDR (func->op_array.opcodes) 9570 } else if (GCC_GLOBAL_REGS) { 9571 | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] 9572 } else { 9573 | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] 9574 | str FCARG1x, EX->opline 9575 } 9576 if (func) { 9577 | // num_args = EX_NUM_ARGS(); 9578 | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] 9579 | // if (UNEXPECTED(num_args > first_extra_arg)) 9580 | CMP_32_WITH_CONST REG1w, (func->op_array.num_args), TMP1w 9581 } else { 9582 | // first_extra_arg = op_array->num_args; 9583 | ldr REG2w, [REG0, #offsetof(zend_op_array, num_args)] 9584 | // num_args = EX_NUM_ARGS(); 9585 | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] 9586 | // if (UNEXPECTED(num_args > first_extra_arg)) 9587 | cmp REG1w, REG2w 9588 } 9589 | bgt >1 9590 |.cold_code 9591 |1: 9592 if (!GCC_GLOBAL_REGS) { 9593 | mov FCARG1x, FP 9594 } 9595 | EXT_CALL zend_jit_copy_extra_args_helper, REG0 9596 if (!func) { 9597 | ldr REG0, EX->func // reload 9598 } 9599 | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] // reload 9600 | b >1 9601 |.code 9602 if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { 9603 if (!func) { 9604 | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) 9605 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9606 | TST_32_WITH_CONST TMP1w, ZEND_ACC_HAS_TYPE_HINTS, TMP2w 9607 | bne >1 9608 } 9609 | // opline += num_args; 9610 || ZEND_ASSERT(sizeof(zend_op) == 32); 9611 | mov REG2w, REG1w 9612 | ADD_IP_SHIFT REG2, lsl #5, TMP1 9613 } 9614 |1: 9615 | // if (EXPECTED((int)num_args < op_array->last_var)) { 9616 if (func) { 9617 | LOAD_32BIT_VAL REG2w, func->op_array.last_var 9618 } else { 9619 | ldr REG2w, [REG0, #offsetof(zend_op_array, last_var)] 9620 } 9621 | subs REG2w, REG2w, REG1w 9622 | ble >3 9623 | // zval *var = EX_VAR_NUM(num_args); 9624 | add REG1, FP, REG1, lsl #4 9625 || ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(ZEND_CALL_FRAME_SLOT * sizeof(zval)))); 9626 | add REG1, REG1, #(ZEND_CALL_FRAME_SLOT * sizeof(zval)) 9627 |2: 9628 | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w 9629 | add REG1, REG1, #16 9630 | subs REG2w, REG2w, #1 9631 | bne <2 9632 |3: 9633 } 9634 9635 if (ZEND_OBSERVER_ENABLED) { 9636 | SAVE_IP 9637 | mov FCARG1x, FP 9638 | EXT_CALL zend_observer_fcall_begin, REG0 9639 } 9640 9641 if (trace) { 9642 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 9643 | b >9 9644 } 9645 } else { 9646#ifdef CONTEXT_THREADED_JIT 9647 | NIY // TODO: CONTEXT_THREADED_JIT is always undefined. 9648#else 9649 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 9650 | ADD_HYBRID_SPAD 9651 | JMP_IP TMP1 9652 } else if (GCC_GLOBAL_REGS) { 9653 | ldp x29, x30, [sp], # SPAD // stack alignment 9654 | JMP_IP TMP1 9655 } else { 9656 | ldp FP, RX, T2 // retore FP and IP 9657 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 9658 | mov RETVALx, #1 // ZEND_VM_ENTER 9659 | ret 9660 } 9661 } 9662#endif 9663 } 9664 9665 if ((!func || func->type == ZEND_INTERNAL_FUNCTION) 9666 && (opline->opcode != ZEND_DO_UCALL)) { 9667 if (!func && (opline->opcode != ZEND_DO_ICALL)) { 9668 |8: 9669 } 9670 if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { 9671 if (!func) { 9672 if (trace) { 9673 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9674 9675 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9676 if (!exit_addr) { 9677 return 0; 9678 } 9679 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9680 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9681 | bne &exit_addr 9682 } else { 9683 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9684 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9685 | bne >1 9686 |.cold_code 9687 |1: 9688 if (!GCC_GLOBAL_REGS) { 9689 | mov FCARG1x, RX 9690 } 9691 | EXT_CALL zend_jit_deprecated_helper, REG0 9692 | GET_LOW_8BITS RETVALw, RETVALw 9693 | ldr REG0, EX:RX->func // reload 9694 | cbnz RETVALw, >1 // Result is 0 on exception 9695 | b ->exception_handler 9696 |.code 9697 |1: 9698 } 9699 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 9700 if (!GCC_GLOBAL_REGS) { 9701 | mov FCARG1x, RX 9702 } 9703 | EXT_CALL zend_jit_deprecated_helper, REG0 9704 | cbz RETVALw, ->exception_handler 9705 | ldr REG0, EX:RX->func // reload 9706 } 9707 } 9708 9709 | // ZVAL_NULL(EX_VAR(opline->result.var)); 9710 | LOAD_ZVAL_ADDR FCARG2x, res_addr 9711 | SET_Z_TYPE_INFO FCARG2x, IS_NULL, TMP1w 9712 9713 | // EG(current_execute_data) = execute_data; 9714 | MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1 9715 9716 zend_jit_reset_last_valid_opline(); 9717 9718 | // (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret); 9719 | mov FCARG1x, RX 9720 if (zend_execute_internal) { 9721 | EXT_CALL zend_execute_internal, REG0 9722 } else { 9723 if (func) { 9724 | EXT_CALL func->internal_function.handler, REG0 9725 } else { 9726 | ldr TMP1, [REG0, #offsetof(zend_internal_function, handler)] 9727 | blr TMP1 9728 } 9729 } 9730 9731 | // EG(current_execute_data) = execute_data; 9732 | MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0 9733 9734 | // zend_vm_stack_free_args(call); 9735 if (func && !unknown_num_args) { 9736 for (i = 0; i < call_num_args; i++ ) { 9737 if (zend_jit_needs_arg_dtor(func, i, call_info)) { 9738 uint32_t offset = EX_NUM_TO_VAR(i); 9739 zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset); 9740 | ZVAL_PTR_DTOR arg_addr, (MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN), 0, 1, opline, ZREG_TMP1, ZREG_TMP2 9741 } 9742 } 9743 } else { 9744 | mov FCARG1x, RX 9745 | EXT_CALL zend_jit_vm_stack_free_args_helper, REG0 9746 } 9747 if (may_have_extra_named_params) { 9748 | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] 9749 | TST_32_WITH_CONST TMP1w, (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24), TMP2w 9750 | bne >1 9751 |.cold_code 9752 |1: 9753 | ldr FCARG1x, [RX, #offsetof(zend_execute_data, extra_named_params)] 9754 | EXT_CALL zend_free_extra_named_params, REG0 9755 | b >2 9756 |.code 9757 |2: 9758 } 9759 9760 |8: 9761 if (opline->opcode == ZEND_DO_FCALL) { 9762 // TODO: optimize ??? 9763 | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) 9764 | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] 9765 | TST_32_WITH_CONST TMP1w, (ZEND_CALL_RELEASE_THIS >> 16), TMP2w 9766 | bne >1 9767 |.cold_code 9768 |1: 9769 | add TMP1, RX, #offsetof(zend_execute_data, This) 9770 | GET_Z_PTR FCARG1x, TMP1 9771 | // OBJ_RELEASE(object); 9772 | OBJ_RELEASE ZREG_FCARG1, >2, ZREG_TMP1, ZREG_TMP2 9773 | b >2 9774 |.code 9775 |2: 9776 } 9777 9778 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 9779 !JIT_G(current_frame) || 9780 !JIT_G(current_frame)->call || 9781 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || 9782 prev_opline->opcode == ZEND_SEND_UNPACK || 9783 prev_opline->opcode == ZEND_SEND_ARRAY || 9784 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 9785 9786 | // zend_vm_stack_free_call_frame(call); 9787 | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] 9788 | TST_32_WITH_CONST TMP1w, ((ZEND_CALL_ALLOCATED >> 16) & 0xff), TMP2w 9789 | bne >1 9790 |.cold_code 9791 |1: 9792 | mov FCARG1x, RX 9793 | EXT_CALL zend_jit_free_call_frame, REG0 9794 | b >1 9795 |.code 9796 } 9797 | MEM_STORE_64_ZTS str, RX, executor_globals, vm_stack_top, REG0 9798 |1: 9799 9800 if (!RETURN_VALUE_USED(opline)) { 9801 zend_class_entry *ce; 9802 bool ce_is_instanceof; 9803 uint32_t func_info = call_info ? 9804 zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : 9805 (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); 9806 9807 /* If an exception is thrown, the return_value may stay at the 9808 * original value of null. */ 9809 func_info |= MAY_BE_NULL; 9810 9811 if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 9812 | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline, ZREG_TMP1, ZREG_TMP2 9813 } 9814 } 9815 9816 | // if (UNEXPECTED(EG(exception) != NULL)) { 9817 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 9818 | cbnz REG0, ->icall_throw_handler 9819 9820 // TODO: Can we avoid checking for interrupts after each call ??? 9821 if (trace && last_valid_opline != opline) { 9822 int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); 9823 9824 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9825 if (!exit_addr) { 9826 return 0; 9827 } 9828 } else { 9829 exit_addr = NULL; 9830 } 9831 if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { 9832 return 0; 9833 } 9834 9835 if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { 9836 | LOAD_IP_ADDR (opline + 1) 9837 } else if (trace 9838 && trace->op == ZEND_JIT_TRACE_END 9839 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 9840 | LOAD_IP_ADDR (opline + 1) 9841 } 9842 } 9843 9844 if (!func) { 9845 |9: 9846 } 9847 9848 return 1; 9849} 9850 9851static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) 9852{ 9853 uint32_t arg_num = opline->op2.num; 9854 zend_jit_addr arg_addr; 9855 9856 ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); 9857 9858 if (!zend_jit_reuse_ip(Dst)) { 9859 return 0; 9860 } 9861 9862 if (opline->opcode == ZEND_SEND_VAL_EX) { 9863 uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); 9864 9865 ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); 9866 9867 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9868 && JIT_G(current_frame) 9869 && JIT_G(current_frame)->call 9870 && JIT_G(current_frame)->call->func) { 9871 if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 9872 /* Don't generate code that always throws exception */ 9873 return 0; 9874 } 9875 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9876 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9877 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9878 if (!exit_addr) { 9879 return 0; 9880 } 9881 | ldr REG0, EX:RX->func 9882 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9883 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9884 | bne &exit_addr 9885 } else { 9886 | ldr REG0, EX:RX->func 9887 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9888 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9889 | bne >1 9890 |.cold_code 9891 |1: 9892 if (Z_MODE(op1_addr) == IS_REG) { 9893 /* set type to avoid zval_ptr_dtor() on uninitialized value */ 9894 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 9895 | SET_ZVAL_TYPE_INFO addr, IS_UNDEF, TMP1w, TMP2 9896 } 9897 | SET_EX_OPLINE opline, REG0 9898 | b ->throw_cannot_pass_by_ref 9899 |.code 9900 } 9901 } 9902 9903 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 9904 9905 if (opline->op1_type == IS_CONST) { 9906 zval *zv = RT_CONSTANT(opline, opline->op1); 9907 9908 | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 9909 if (Z_REFCOUNTED_P(zv)) { 9910 | ADDREF_CONST zv, REG0, TMP1 9911 } 9912 } else { 9913 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9914 } 9915 9916 return 1; 9917} 9918 9919static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) 9920{ 9921 | ldr FCARG1x, EX->call 9922 | ldrb TMP1w, [FCARG1x, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] 9923 | TST_32_WITH_CONST TMP1w, (ZEND_CALL_MAY_HAVE_UNDEF >> 24), TMP2w 9924 | bne >1 9925 |.cold_code 9926 |1: 9927 | SET_EX_OPLINE opline, REG0 9928 | EXT_CALL zend_handle_undef_args, REG0 9929 | cbz RETVALw, >2 9930 | b ->exception_handler 9931 |.code 9932 |2: 9933 9934 return 1; 9935} 9936 9937static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) 9938{ 9939 zend_jit_addr op1_addr, arg_addr, ref_addr; 9940 9941 op1_addr = OP1_ADDR(); 9942 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 9943 9944 if (!zend_jit_reuse_ip(Dst)) { 9945 return 0; 9946 } 9947 9948 if (opline->op1_type == IS_VAR) { 9949 if (op1_info & MAY_BE_INDIRECT) { 9950 | LOAD_ZVAL_ADDR REG0, op1_addr 9951 | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { 9952 | IF_NOT_Z_TYPE REG0, IS_INDIRECT, >1, TMP1w 9953 | // ret = Z_INDIRECT_P(ret); 9954 | GET_Z_PTR REG0, REG0 9955 |1: 9956 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 9957 } 9958 } else if (opline->op1_type == IS_CV) { 9959 if (op1_info & MAY_BE_UNDEF) { 9960 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 9961 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 9962 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 9963 | b >2 9964 |1: 9965 } 9966 op1_info &= ~MAY_BE_UNDEF; 9967 op1_info |= MAY_BE_NULL; 9968 } 9969 } else { 9970 ZEND_UNREACHABLE(); 9971 } 9972 9973 if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { 9974 if (op1_info & MAY_BE_REF) { 9975 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2, ZREG_TMP1 9976 | GET_ZVAL_PTR REG1, op1_addr, TMP1 9977 | GC_ADDREF REG1, TMP1w 9978 | SET_ZVAL_PTR arg_addr, REG1, TMP1 9979 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 9980 | b >6 9981 } 9982 |2: 9983 | // ZVAL_NEW_REF(arg, varptr); 9984 if (opline->op1_type == IS_VAR) { 9985 if (Z_REG(op1_addr) != ZREG_REG0 || Z_OFFSET(op1_addr) != 0) { 9986 | LOAD_ZVAL_ADDR REG0, op1_addr 9987 } 9988 | str REG0, T1 // save 9989 } 9990 | EMALLOC sizeof(zend_reference), op_array, opline // Allocate space in REG0 9991 | mov TMP1w, #2 9992 | str TMP1w, [REG0] 9993 || ZEND_ASSERT(GC_REFERENCE <= MOVZ_IMM); 9994 | movz TMP1w, #GC_REFERENCE 9995 | str TMP1w, [REG0, #offsetof(zend_reference, gc.u.type_info)] 9996 | str xzr, [REG0, #offsetof(zend_reference, sources.ptr)] 9997 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); 9998 if (opline->op1_type == IS_VAR) { 9999 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); 10000 10001 | ldr REG1, T1 // restore 10002 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10003 | SET_ZVAL_PTR val_addr, REG0, TMP1 10004 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX, TMP1w, TMP2 10005 } else { 10006 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10007 | SET_ZVAL_PTR op1_addr, REG0, TMP1 10008 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 10009 } 10010 | SET_ZVAL_PTR arg_addr, REG0, TMP1 10011 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 10012 } 10013 10014 |6: 10015 | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline, ZREG_TMP1, ZREG_TMP2 10016 |7: 10017 10018 return 1; 10019} 10020 10021static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) 10022{ 10023 uint32_t arg_num = opline->op2.num; 10024 zend_jit_addr arg_addr; 10025 10026 ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && 10027 opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || 10028 arg_num <= MAX_ARG_FLAG_NUM); 10029 10030 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10031 10032 if (!zend_jit_reuse_ip(Dst)) { 10033 return 0; 10034 } 10035 10036 if (opline->opcode == ZEND_SEND_VAR_EX) { 10037 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10038 && JIT_G(current_frame) 10039 && JIT_G(current_frame)->call 10040 && JIT_G(current_frame)->call->func) { 10041 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10042 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10043 return 0; 10044 } 10045 return 1; 10046 } 10047 } else { 10048 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10049 10050 | ldr REG0, EX:RX->func 10051 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 10052 | TST_32_WITH_CONST TMP1w, mask, TMP2w 10053 | bne >1 10054 |.cold_code 10055 |1: 10056 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10057 return 0; 10058 } 10059 | b >7 10060 |.code 10061 } 10062 } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { 10063 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10064 && JIT_G(current_frame) 10065 && JIT_G(current_frame)->call 10066 && JIT_G(current_frame)->call->func) { 10067 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10068 10069 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10070 10071 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10072 if (!(op1_info & MAY_BE_REF)) { 10073 /* Don't generate code that always throws exception */ 10074 return 0; 10075 } else { 10076 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10077 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10078 if (!exit_addr) { 10079 return 0; 10080 } 10081 | GET_LOW_8BITS TMP1w, REG1w 10082 | cmp TMP1w, #IS_REFERENCE 10083 | bne &exit_addr 10084 } 10085 } 10086 return 1; 10087 } 10088 } else { 10089 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10090 10091 | ldr REG0, EX:RX->func 10092 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 10093 | TST_32_WITH_CONST TMP1w, mask, TMP2w 10094 | bne >1 10095 |.cold_code 10096 |1: 10097 10098 mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); 10099 10100 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10101 if (op1_info & MAY_BE_REF) { 10102 | GET_LOW_8BITS TMP1w, REG1w 10103 | cmp TMP1w, #IS_REFERENCE 10104 | beq >7 10105 } 10106 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 10107 | TST_32_WITH_CONST TMP1w, mask, TMP2w 10108 | bne >7 10109 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10110 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10111 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10112 if (!exit_addr) { 10113 return 0; 10114 } 10115 | b &exit_addr 10116 } else { 10117 | SET_EX_OPLINE opline, REG0 10118 | LOAD_ZVAL_ADDR FCARG1x, arg_addr 10119 | EXT_CALL zend_jit_only_vars_by_reference, REG0 10120 if (!zend_jit_check_exception(Dst)) { 10121 return 0; 10122 } 10123 | b >7 10124 } 10125 10126 |.code 10127 } 10128 } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { 10129 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10130 && JIT_G(current_frame) 10131 && JIT_G(current_frame)->call 10132 && JIT_G(current_frame)->call->func) { 10133 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10134 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10135 return 0; 10136 } 10137 return 1; 10138 } 10139 } else { 10140 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10141 | TST_32_WITH_CONST TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10142 | bne >1 10143 |.cold_code 10144 |1: 10145 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10146 return 0; 10147 } 10148 | b >7 10149 |.code 10150 } 10151 } 10152 10153 if (op1_info & MAY_BE_UNDEF) { 10154 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10155 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 10156 |.cold_code 10157 |1: 10158 } 10159 10160 | SET_EX_OPLINE opline, REG0 10161 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 10162 | EXT_CALL zend_jit_undefined_op_helper, REG0 10163 | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL, TMP1w, TMP2 10164 | cbz RETVALx, ->exception_handler 10165 10166 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10167 | b >7 10168 |.code 10169 } else { 10170 |7: 10171 return 1; 10172 } 10173 } 10174 10175 if (opline->opcode == ZEND_SEND_VAR_NO_REF) { 10176 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10177 if (op1_info & MAY_BE_REF) { 10178 | GET_LOW_8BITS TMP1w, REG1w 10179 | cmp TMP1w, #IS_REFERENCE 10180 | beq >7 10181 } 10182 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10183 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10184 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10185 if (!exit_addr) { 10186 return 0; 10187 } 10188 | b &exit_addr 10189 } else { 10190 | SET_EX_OPLINE opline, REG0 10191 | LOAD_ZVAL_ADDR FCARG1x, arg_addr 10192 | EXT_CALL zend_jit_only_vars_by_reference, REG0 10193 if (!zend_jit_check_exception(Dst)) { 10194 return 0; 10195 } 10196 } 10197 } else { 10198 if (op1_info & MAY_BE_REF) { 10199 if (opline->op1_type == IS_CV) { 10200 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 10201 10202 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 10203 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 10204 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10205 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 10206 } else { 10207 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 8); 10208 10209 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 10210 |.cold_code 10211 |1: 10212 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 10213 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10214 | // ZVAL_COPY_VALUE(return_value, &ref->value); 10215 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10216 | GC_DELREF FCARG1x, TMP1w 10217 | beq >1 10218 | IF_NOT_REFCOUNTED REG0w, >2, TMP1w 10219 | GC_ADDREF REG2, TMP1w 10220 | b >2 10221 |1: 10222 | EFREE_REFERENCE 10223 | b >2 10224 |.code 10225 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10226 |2: 10227 } 10228 } else { 10229 if (op1_addr != op1_def_addr) { 10230 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 10231 return 0; 10232 } 10233 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 10234 op1_addr= op1_def_addr; 10235 } 10236 } 10237 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10238 if (opline->op1_type == IS_CV) { 10239 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 10240 } 10241 } 10242 } 10243 |7: 10244 10245 return 1; 10246} 10247 10248static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) 10249{ 10250 uint32_t arg_num = opline->op2.num; 10251 10252 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10253 && JIT_G(current_frame) 10254 && JIT_G(current_frame)->call 10255 && JIT_G(current_frame)->call->func) { 10256 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10257 if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { 10258 TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); 10259 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10260 || if (reuse_ip) { 10261 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10262 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10263 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10264 || } else { 10265 | ldr REG0, EX->call 10266 | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10267 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10268 | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10269 || } 10270 } 10271 } else { 10272 if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 10273 TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); 10274 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10275 || if (reuse_ip) { 10276 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10277 | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w 10278 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10279 || } else { 10280 | ldr REG0, EX->call 10281 | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10282 | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w 10283 | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10284 || } 10285 } 10286 } 10287 } else { 10288 // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { 10289 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10290 10291 if (!zend_jit_reuse_ip(Dst)) { 10292 return 0; 10293 } 10294 10295 | ldr REG0, EX:RX->func 10296 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 10297 | TST_32_WITH_CONST TMP1w, mask, TMP2w 10298 | bne >1 10299 |.cold_code 10300 |1: 10301 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10302 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10303 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10304 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10305 | b >1 10306 |.code 10307 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10308 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10309 | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~(ZEND_CALL_SEND_ARG_BY_REF)), TMP2w 10310 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10311 |1: 10312 } 10313 10314 return 1; 10315} 10316 10317static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) 10318{ 10319 if (smart_branch_opcode) { 10320 if (smart_branch_opcode == ZEND_JMPZ) { 10321 if (jmp) { 10322 | b >7 10323 } 10324 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10325 | b =>target_label 10326 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10327 | b =>target_label2 10328 } else { 10329 ZEND_UNREACHABLE(); 10330 } 10331 } else { 10332 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10333 10334 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 10335 if (jmp) { 10336 | b >7 10337 } 10338 } 10339 10340 return 1; 10341} 10342 10343static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) 10344{ 10345 if (smart_branch_opcode) { 10346 if (smart_branch_opcode == ZEND_JMPZ) { 10347 | b =>target_label 10348 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10349 if (jmp) { 10350 | b >7 10351 } 10352 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10353 | b =>target_label 10354 } else { 10355 ZEND_UNREACHABLE(); 10356 } 10357 } else { 10358 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10359 10360 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 10361 if (jmp) { 10362 | b >7 10363 } 10364 } 10365 10366 return 1; 10367} 10368 10369static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 10370{ 10371 uint32_t defined_label = (uint32_t)-1; 10372 uint32_t undefined_label = (uint32_t)-1; 10373 zval *zv = RT_CONSTANT(opline, opline->op1); 10374 zend_jit_addr res_addr = 0; 10375 10376 if (smart_branch_opcode && !exit_addr) { 10377 if (smart_branch_opcode == ZEND_JMPZ) { 10378 undefined_label = target_label; 10379 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10380 defined_label = target_label; 10381 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10382 undefined_label = target_label; 10383 defined_label = target_label2; 10384 } else { 10385 ZEND_UNREACHABLE(); 10386 } 10387 } 10388 10389 | // if (CACHED_PTR(opline->extended_value)) { 10390 | ldr REG0, EX->run_time_cache 10391 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, opline->extended_value, TMP1 10392 | cbz REG0, >1 10393 | TST_64_WITH_ONE REG0 10394 | bne >4 10395 |.cold_code 10396 |4: 10397 | MEM_LOAD_64_ZTS ldr, FCARG1x, executor_globals, zend_constants, FCARG1x 10398 | ldr TMP1w, [FCARG1x, #offsetof(HashTable, nNumOfElements)] 10399 | cmp TMP1, REG0, lsr #1 10400 10401 if (smart_branch_opcode) { 10402 if (exit_addr) { 10403 if (smart_branch_opcode == ZEND_JMPZ) { 10404 | beq &exit_addr 10405 } else { 10406 | beq >3 10407 } 10408 } else if (undefined_label != (uint32_t)-1) { 10409 | beq =>undefined_label 10410 } else { 10411 | beq >3 10412 } 10413 } else { 10414 | beq >2 10415 } 10416 |1: 10417 | SET_EX_OPLINE opline, REG0 10418 | LOAD_ADDR FCARG1x, zv 10419 | EXT_CALL zend_jit_check_constant, REG0 10420 if (exit_addr) { 10421 if (smart_branch_opcode == ZEND_JMPNZ) { 10422 | cbz RETVALx, >3 10423 } else { 10424 | cbnz RETVALx, >3 10425 } 10426 | b &exit_addr 10427 } else if (smart_branch_opcode) { 10428 if (undefined_label != (uint32_t)-1) { 10429 | cbz RETVALx, =>undefined_label 10430 } else { 10431 | cbz RETVALx, >3 10432 } 10433 if (defined_label != (uint32_t)-1) { 10434 | b =>defined_label 10435 } else { 10436 | b >3 10437 } 10438 } else { 10439 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10440 | cbnz RETVALx, >1 10441 |2: 10442 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 10443 | b >3 10444 } 10445 |.code 10446 if (smart_branch_opcode) { 10447 if (exit_addr) { 10448 if (smart_branch_opcode == ZEND_JMPNZ) { 10449 | b &exit_addr 10450 } 10451 } else if (defined_label != (uint32_t)-1) { 10452 | b =>defined_label 10453 } 10454 } else { 10455 |1: 10456 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 10457 } 10458 |3: 10459 10460 return 1; 10461} 10462 10463static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 10464{ 10465 uint32_t mask; 10466 zend_jit_addr op1_addr = OP1_ADDR(); 10467 10468 // TODO: support for is_resource() ??? 10469 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); 10470 10471 if (op1_info & MAY_BE_UNDEF) { 10472 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10473 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 10474 |.cold_code 10475 |1: 10476 } 10477 | SET_EX_OPLINE opline, REG0 10478 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 10479 | EXT_CALL zend_jit_undefined_op_helper, REG0 10480 zend_jit_check_exception_undef_result(Dst, opline); 10481 if (opline->extended_value & MAY_BE_NULL) { 10482 if (exit_addr) { 10483 if (smart_branch_opcode == ZEND_JMPNZ) { 10484 | b &exit_addr 10485 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 10486 | b >7 10487 } 10488 } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { 10489 return 0; 10490 } 10491 } else { 10492 if (exit_addr) { 10493 if (smart_branch_opcode == ZEND_JMPZ) { 10494 | b &exit_addr 10495 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 10496 | b >7 10497 } 10498 } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { 10499 return 0; 10500 } 10501 } 10502 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10503 |.code 10504 } 10505 } 10506 10507 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10508 mask = opline->extended_value; 10509 if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { 10510 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10511 if (exit_addr) { 10512 if (smart_branch_opcode == ZEND_JMPNZ) { 10513 | b &exit_addr 10514 } 10515 } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { 10516 return 0; 10517 } 10518 } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { 10519 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10520 if (exit_addr) { 10521 if (smart_branch_opcode == ZEND_JMPZ) { 10522 | b &exit_addr 10523 } 10524 } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { 10525 return 0; 10526 } 10527 } else { 10528 bool invert = 0; 10529 zend_uchar type; 10530 10531 switch (mask) { 10532 case MAY_BE_NULL: type = IS_NULL; break; 10533 case MAY_BE_FALSE: type = IS_FALSE; break; 10534 case MAY_BE_TRUE: type = IS_TRUE; break; 10535 case MAY_BE_LONG: type = IS_LONG; break; 10536 case MAY_BE_DOUBLE: type = IS_DOUBLE; break; 10537 case MAY_BE_STRING: type = IS_STRING; break; 10538 case MAY_BE_ARRAY: type = IS_ARRAY; break; 10539 case MAY_BE_OBJECT: type = IS_OBJECT; break; 10540 case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break; 10541 case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break; 10542 case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break; 10543 case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break; 10544 case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break; 10545 case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break; 10546 case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break; 10547 case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break; 10548 case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break; 10549 default: 10550 type = 0; 10551 } 10552 10553 if (op1_info & MAY_BE_REF) { 10554 | LOAD_ZVAL_ADDR REG0, op1_addr 10555 | ZVAL_DEREF REG0, op1_info, TMP1w 10556 } 10557 if (type == 0) { 10558 if (smart_branch_opcode && 10559 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 10560 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10561 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10562 | // if (Z_REFCOUNTED_P(cv)) { 10563 | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 10564 |.cold_code 10565 |1: 10566 } 10567 | // if (!Z_DELREF_P(cv)) { 10568 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10569 | GC_DELREF FCARG1x, TMP1w 10570 if (RC_MAY_BE_1(op1_info)) { 10571 if (RC_MAY_BE_N(op1_info)) { 10572 | bne >3 10573 } 10574 if (op1_info & MAY_BE_REF) { 10575 | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] 10576 } else { 10577 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10578 } 10579 | str REG0w, T1 // save 10580 | // zval_dtor_func(r); 10581 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 10582 | ldr REG1w, T1 // restore 10583 | b >2 10584 } 10585 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10586 if (!RC_MAY_BE_1(op1_info)) { 10587 | b >3 10588 } 10589 |.code 10590 } 10591 |3: 10592 if (op1_info & MAY_BE_REF) { 10593 | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] 10594 } else { 10595 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10596 } 10597 |2: 10598 } else { 10599 if (op1_info & MAY_BE_REF) { 10600 | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] 10601 } else { 10602 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10603 } 10604 } 10605 | mov REG0w, #1 10606 | lsl REG0w, REG0w, REG1w 10607 | TST_32_WITH_CONST REG0w, mask, TMP1w 10608 if (exit_addr) { 10609 if (smart_branch_opcode == ZEND_JMPNZ) { 10610 | bne &exit_addr 10611 } else { 10612 | beq &exit_addr 10613 } 10614 } else if (smart_branch_opcode) { 10615 if (smart_branch_opcode == ZEND_JMPZ) { 10616 | beq =>target_label 10617 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10618 | bne =>target_label 10619 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10620 | beq =>target_label 10621 | b =>target_label2 10622 } else { 10623 ZEND_UNREACHABLE(); 10624 } 10625 } else { 10626 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10627 10628 | cset REG0w, ne 10629 | add REG0w, REG0w, #2 10630 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 10631 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10632 } 10633 } else { 10634 if (smart_branch_opcode && 10635 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 10636 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10637 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10638 | // if (Z_REFCOUNTED_P(cv)) { 10639 | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 10640 |.cold_code 10641 |1: 10642 } 10643 | // if (!Z_DELREF_P(cv)) { 10644 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10645 | GC_DELREF FCARG1x, TMP1w 10646 if (RC_MAY_BE_1(op1_info)) { 10647 if (RC_MAY_BE_N(op1_info)) { 10648 | bne >3 10649 } 10650 if (op1_info & MAY_BE_REF) { 10651 | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] 10652 } else { 10653 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10654 } 10655 | str REG0w, T1 // save 10656 | // zval_dtor_func(r); 10657 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 10658 | ldr REG1w, T1 // restore 10659 | b >2 10660 } 10661 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10662 if (!RC_MAY_BE_1(op1_info)) { 10663 | b >3 10664 } 10665 |.code 10666 } 10667 |3: 10668 if (op1_info & MAY_BE_REF) { 10669 | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] 10670 } else { 10671 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10672 } 10673 |2: 10674 // Note: 'type' is of uchar type and holds a positive value, 10675 // hence it's safe to directly encode it as the imm field of 'cmp' instruction. 10676 | cmp REG1w, #type 10677 } else { 10678 if (op1_info & MAY_BE_REF) { 10679 | ldrb TMP1w, [REG0, #offsetof(zval,u1.v.type)] 10680 | cmp TMP1w, #type 10681 } else { 10682 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10683 | cmp TMP1w, #type 10684 } 10685 } 10686 if (exit_addr) { 10687 if (invert) { 10688 if (smart_branch_opcode == ZEND_JMPNZ) { 10689 | bne &exit_addr 10690 } else { 10691 | beq &exit_addr 10692 } 10693 } else { 10694 if (smart_branch_opcode == ZEND_JMPNZ) { 10695 | beq &exit_addr 10696 } else { 10697 | bne &exit_addr 10698 } 10699 } 10700 } else if (smart_branch_opcode) { 10701 if (invert) { 10702 if (smart_branch_opcode == ZEND_JMPZ) { 10703 | beq =>target_label 10704 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10705 | bne =>target_label 10706 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10707 | beq =>target_label 10708 | b =>target_label2 10709 } else { 10710 ZEND_UNREACHABLE(); 10711 } 10712 } else { 10713 if (smart_branch_opcode == ZEND_JMPZ) { 10714 | bne =>target_label 10715 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10716 | beq =>target_label 10717 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 10718 | bne =>target_label 10719 | b =>target_label2 10720 } else { 10721 ZEND_UNREACHABLE(); 10722 } 10723 } 10724 } else { 10725 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10726 10727 if (invert) { 10728 | cset REG0w, ne 10729 } else { 10730 | cset REG0w, eq 10731 } 10732 | add REG0w, REG0w, #2 10733 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 10734 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10735 } 10736 } 10737 } 10738 } 10739 10740 |7: 10741 10742 return 1; 10743} 10744 10745static int zend_jit_leave_frame(dasm_State **Dst) 10746{ 10747 | // EG(current_execute_data) = EX(prev_execute_data); 10748 | ldr REG0, EX->prev_execute_data 10749 | MEM_STORE_64_ZTS str, REG0, executor_globals, current_execute_data, REG2 10750 return 1; 10751} 10752 10753static int zend_jit_free_cvs(dasm_State **Dst) 10754{ 10755 | // EG(current_execute_data) = EX(prev_execute_data); 10756 | ldr FCARG1x, EX->prev_execute_data 10757 | MEM_STORE_64_ZTS str, FCARG1x, executor_globals, current_execute_data, REG0 10758 | // zend_free_compiled_variables(execute_data); 10759 | mov FCARG1x, FP 10760 | EXT_CALL zend_free_compiled_variables, REG0 10761 return 1; 10762} 10763 10764static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) 10765{ 10766 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10767 uint32_t offset = EX_NUM_TO_VAR(var); 10768 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset); 10769 | ZVAL_PTR_DTOR addr, info, 1, 1, NULL, ZREG_TMP1, ZREG_TMP2 10770 } 10771 return 1; 10772} 10773 10774static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) 10775{ 10776 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10777 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset); 10778 | ZVAL_PTR_DTOR addr, info, 0, 1, opline, ZREG_TMP1, ZREG_TMP2 10779 } 10780 return 1; 10781} 10782 10783static int zend_jit_leave_func(dasm_State **Dst, 10784 const zend_op_array *op_array, 10785 const zend_op *opline, 10786 uint32_t op1_info, 10787 bool left_frame, 10788 zend_jit_trace_rec *trace, 10789 zend_jit_trace_info *trace_info, 10790 int indirect_var_access, 10791 int may_throw) 10792{ 10793 bool may_be_top_frame = 10794 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10795 !JIT_G(current_frame) || 10796 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); 10797 bool may_need_call_helper = 10798 indirect_var_access || /* may have symbol table */ 10799 !op_array->function_name || /* may have symbol table */ 10800 may_be_top_frame || 10801 (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ 10802 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10803 !JIT_G(current_frame) || 10804 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ 10805 (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ 10806 bool may_need_release_this = 10807 !(op_array->fn_flags & ZEND_ACC_CLOSURE) && 10808 op_array->scope && 10809 !(op_array->fn_flags & ZEND_ACC_STATIC) && 10810 (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10811 !JIT_G(current_frame) || 10812 !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame))); 10813 10814 if (may_need_call_helper || may_need_release_this) { 10815 | ldr FCARG1w, [FP, #offsetof(zend_execute_data, This.u1.type_info)] 10816 } 10817 if (may_need_call_helper) { 10818 if (!left_frame) { 10819 left_frame = 1; 10820 if (!zend_jit_leave_frame(Dst)) { 10821 return 0; 10822 } 10823 } 10824 /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ 10825 10826 | TST_32_WITH_CONST FCARG1w, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE), TMP1w 10827 if (trace && trace->op != ZEND_JIT_TRACE_END) { 10828 | bne >1 10829 |.cold_code 10830 |1: 10831 if (!GCC_GLOBAL_REGS) { 10832 | mov FCARG1x, FP 10833 } 10834 | EXT_CALL zend_jit_leave_func_helper, REG0 10835 10836 if (may_be_top_frame) { 10837 // TODO: try to avoid this check ??? 10838 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10839#if 0 10840 /* this check should be handled by the following OPLINE guard */ 10841 | LOAD_ADDR TMP1, zend_jit_halt_op 10842 | cmp IP, TMP1 10843 | beq ->trace_halt 10844#endif 10845 } else if (GCC_GLOBAL_REGS) { 10846 | cbz IP, ->trace_halt 10847 } else { 10848 | tst RETVALw, RETVALw 10849 | blt ->trace_halt 10850 } 10851 } 10852 10853 if (!GCC_GLOBAL_REGS) { 10854 | // execute_data = EG(current_execute_data) 10855 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 10856 } 10857 | b >8 10858 |.code 10859 } else { 10860 | bne ->leave_function_handler 10861 } 10862 } 10863 10864 if (op_array->fn_flags & ZEND_ACC_CLOSURE) { 10865 if (!left_frame) { 10866 left_frame = 1; 10867 if (!zend_jit_leave_frame(Dst)) { 10868 return 0; 10869 } 10870 } 10871 | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); 10872 | ldr FCARG1x, EX->func 10873 | sub FCARG1x, FCARG1x, #sizeof(zend_object) 10874 | OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2 10875 |4: 10876 } else if (may_need_release_this) { 10877 if (!left_frame) { 10878 left_frame = 1; 10879 if (!zend_jit_leave_frame(Dst)) { 10880 return 0; 10881 } 10882 } 10883 | // if (call_info & ZEND_CALL_RELEASE_THIS) 10884 | TST_32_WITH_CONST FCARG1w, ZEND_CALL_RELEASE_THIS, TMP1w 10885 | beq >4 10886 | // zend_object *object = Z_OBJ(execute_data->This); 10887 | ldr FCARG1x, EX->This.value.obj 10888 | // OBJ_RELEASE(object); 10889 | OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2 10890 |4: 10891 // TODO: avoid EG(excption) check for $this->foo() calls 10892 may_throw = 1; 10893 } 10894 10895 | // EG(vm_stack_top) = (zval*)execute_data; 10896 | MEM_STORE_64_ZTS str, FP, executor_globals, vm_stack_top, REG0 10897 | // execute_data = EX(prev_execute_data); 10898 | ldr FP, EX->prev_execute_data 10899 10900 if (!left_frame) { 10901 | // EG(current_execute_data) = execute_data; 10902 | MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0 10903 } 10904 10905 |9: 10906 if (trace) { 10907 if (trace->op != ZEND_JIT_TRACE_END 10908 && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 10909 zend_jit_reset_last_valid_opline(); 10910 } else { 10911 | LOAD_IP 10912 | ADD_IP_WITH_CONST sizeof(zend_op), TMP1 10913 } 10914 10915 |8: 10916 10917 if (trace->op == ZEND_JIT_TRACE_BACK 10918 && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 10919 const zend_op *next_opline = trace->opline; 10920 10921 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 10922 && (op1_info & MAY_BE_RC1) 10923 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { 10924 /* exception might be thrown during destruction of unused return value */ 10925 | // if (EG(exception)) 10926 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 10927 | cbnz REG0, ->leave_throw_handler 10928 } 10929 do { 10930 trace++; 10931 } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); 10932 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 10933 next_opline = trace->opline; 10934 ZEND_ASSERT(next_opline != NULL); 10935 10936 if (trace->op == ZEND_JIT_TRACE_END 10937 && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { 10938 trace_info->flags |= ZEND_JIT_TRACE_LOOP; 10939 | CMP_IP next_opline, TMP1, TMP2 10940 | beq =>0 // LOOP 10941#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 10942 | JMP_IP TMP1 10943#else 10944 | b ->trace_escape 10945#endif 10946 } else { 10947 | CMP_IP next_opline, TMP1, TMP2 10948 | bne ->trace_escape 10949 } 10950 10951 zend_jit_set_last_valid_opline(trace->opline); 10952 10953 return 1; 10954 } else if (may_throw || 10955 (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 10956 && (op1_info & MAY_BE_RC1) 10957 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) 10958 && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { 10959 | // if (EG(exception)) 10960 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 10961 | cbnz REG0, ->leave_throw_handler 10962 } 10963 10964 return 1; 10965 } else { 10966 | // if (EG(exception)) 10967 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 10968 | LOAD_IP 10969 | cbnz REG0, ->leave_throw_handler 10970 | // opline = EX(opline) + 1 10971 | ADD_IP_WITH_CONST sizeof(zend_op), TMP1 10972 } 10973 10974 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10975 | ADD_HYBRID_SPAD 10976#ifdef CONTEXT_THREADED_JIT 10977 | NIY // TODO: CONTEXT_THREADED_JIT is always undefined 10978#else 10979 | JMP_IP TMP1 10980#endif 10981 } else if (GCC_GLOBAL_REGS) { 10982 | ldp x29, x30, [sp], # SPAD // stack alignment 10983#ifdef CONTEXT_THREADED_JIT 10984 | NIY // TODO 10985#else 10986 | JMP_IP TMP1 10987#endif 10988 } else { 10989#ifdef CONTEXT_THREADED_JIT 10990 ZEND_UNREACHABLE(); 10991 // TODO: context threading can't work without GLOBAL REGS because we have to change 10992 // the value of execute_data in execute_ex() 10993 | NIY // TODO 10994#else 10995 | ldp FP, RX, T2 // retore FP and IP 10996 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 10997 | mov RETVALx, #2 // ZEND_VM_LEAVE ???? 10998 | ret 10999#endif 11000 } 11001 11002 return 1; 11003} 11004 11005static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) 11006{ 11007 zend_jit_addr ret_addr; 11008 int8_t return_value_used; 11009 11010 ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); 11011 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); 11012 11013 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { 11014 if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { 11015 return_value_used = 1; 11016 } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { 11017 return_value_used = 0; 11018 } else { 11019 return_value_used = -1; 11020 } 11021 } else { 11022 return_value_used = -1; 11023 } 11024 11025 if (ZEND_OBSERVER_ENABLED) { 11026 if (Z_MODE(op1_addr) == IS_REG) { 11027 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 11028 11029 if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { 11030 return 0; 11031 } 11032 op1_addr = dst; 11033 } 11034 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 11035 | mov FCARG1x, FP 11036 | SET_EX_OPLINE opline, REG0 11037 | EXT_CALL zend_observer_fcall_end, REG0 11038 } 11039 11040 // if (!EX(return_value)) 11041 if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_REG1) { 11042 if (return_value_used != 0) { 11043 | ldr REG2, EX->return_value 11044 } 11045 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); 11046 } else { 11047 if (return_value_used != 0) { 11048 | ldr REG1, EX->return_value 11049 } 11050 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); 11051 } 11052 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11053 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11054 if (return_value_used == -1) { 11055 | cbz Rx(Z_REG(ret_addr)), >1 11056 |.cold_code 11057 |1: 11058 } 11059 if (return_value_used != 1) { 11060 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11061 if (jit_return_label >= 0) { 11062 | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label, ZREG_TMP1, ZREG_TMP2 11063 } else { 11064 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9, ZREG_TMP1, ZREG_TMP2 11065 } 11066 } 11067 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 11068 | GC_DELREF FCARG1x, TMP1w 11069 if (RC_MAY_BE_1(op1_info)) { 11070 if (RC_MAY_BE_N(op1_info)) { 11071 if (jit_return_label >= 0) { 11072 | bne =>jit_return_label 11073 } else { 11074 | bne >9 11075 } 11076 } 11077 | //SAVE_OPLINE() 11078 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 11079 | //????ldr REG1, EX->return_value // reload ??? 11080 } 11081 if (return_value_used == -1) { 11082 if (jit_return_label >= 0) { 11083 | b =>jit_return_label 11084 } else { 11085 | b >9 11086 } 11087 |.code 11088 } 11089 } 11090 } else if (return_value_used == -1) { 11091 if (jit_return_label >= 0) { 11092 | cbz Rx(Z_REG(ret_addr)), =>jit_return_label 11093 } else { 11094 | cbz Rx(Z_REG(ret_addr)), >9 11095 } 11096 } 11097 11098 if (return_value_used == 0) { 11099 |9: 11100 return 1; 11101 } 11102 11103 if (opline->op1_type == IS_CONST) { 11104 zval *zv = RT_CONSTANT(opline, opline->op1); 11105 | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 11106 if (Z_REFCOUNTED_P(zv)) { 11107 | ADDREF_CONST zv, REG0, TMP1 11108 } 11109 } else if (opline->op1_type == IS_TMP_VAR) { 11110 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11111 } else if (opline->op1_type == IS_CV) { 11112 if (op1_info & MAY_BE_REF) { 11113 | LOAD_ZVAL_ADDR REG0, op1_addr 11114 | ZVAL_DEREF REG0, op1_info, TMP1w 11115 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 11116 } 11117 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11118 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 11119 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11120 (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || 11121 !op_array->function_name) { 11122 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 11123 } else if (return_value_used != 1) { 11124 | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); 11125 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 11126 } 11127 } 11128 } else { 11129 if (op1_info & MAY_BE_REF) { 11130 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); 11131 11132 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 11133 |.cold_code 11134 |1: 11135 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 11136 | GET_ZVAL_PTR REG0, op1_addr, TMP1 11137 | // ZVAL_COPY_VALUE(return_value, &ref->value); 11138 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11139 | GC_DELREF REG0, TMP1w 11140 | beq >2 11141 | // if (IS_REFCOUNTED()) 11142 if (jit_return_label >= 0) { 11143 | IF_NOT_REFCOUNTED REG2w, =>jit_return_label, TMP1w 11144 } else { 11145 | IF_NOT_REFCOUNTED REG2w, >9, TMP1w 11146 } 11147 | // ADDREF 11148 | GET_ZVAL_PTR REG2, ret_addr, TMP1 // reload 11149 | GC_ADDREF REG2, TMP1w 11150 if (jit_return_label >= 0) { 11151 | b =>jit_return_label 11152 } else { 11153 | b >9 11154 } 11155 |2: 11156 | mov FCARG1x, REG0 11157 | EFREE_REFERENCE 11158 if (jit_return_label >= 0) { 11159 | b =>jit_return_label 11160 } else { 11161 | b >9 11162 } 11163 |.code 11164 } 11165 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11166 } 11167 11168 |9: 11169 return 1; 11170} 11171 11172static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) 11173{ 11174 ZEND_ASSERT(type_reg == ZREG_REG2); 11175 11176 | GET_ZVAL_PTR REG1, val_addr, TMP1 11177 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w 11178 | GET_LOW_8BITS TMP2w, REG2w 11179 | IF_NOT_TYPE TMP2w, IS_REFERENCE, >1 11180 | add REG1, REG1, #offsetof(zend_reference, val) 11181 | GET_Z_TYPE_INFO REG2w, REG1 11182 | GET_Z_PTR REG1, REG1 11183 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w 11184 |1: 11185 | GC_ADDREF REG1, TMP2w 11186 |2: 11187 | SET_ZVAL_PTR res_addr, REG1, TMP1 11188 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 11189 11190 return 1; 11191} 11192 11193static int zend_jit_fetch_dim_read(dasm_State **Dst, 11194 const zend_op *opline, 11195 zend_ssa *ssa, 11196 const zend_ssa_op *ssa_op, 11197 uint32_t op1_info, 11198 zend_jit_addr op1_addr, 11199 bool op1_avoid_refcounting, 11200 uint32_t op2_info, 11201 uint32_t res_info, 11202 zend_jit_addr res_addr, 11203 uint8_t dim_type) 11204{ 11205 zend_jit_addr orig_op1_addr, op2_addr; 11206 const void *exit_addr = NULL; 11207 const void *not_found_exit_addr = NULL; 11208 const void *res_exit_addr = NULL; 11209 bool result_avoid_refcounting = 0; 11210 uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; 11211 int may_throw = 0; 11212 11213 orig_op1_addr = OP1_ADDR(); 11214 op2_addr = OP2_ADDR(); 11215 11216 if (opline->opcode != ZEND_FETCH_DIM_IS 11217 && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11218 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 11219 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11220 if (!exit_addr) { 11221 return 0; 11222 } 11223 } 11224 11225 if ((res_info & MAY_BE_GUARD) 11226 && JIT_G(current_frame) 11227 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { 11228 uint32_t flags = 0; 11229 uint32_t old_op1_info = 0; 11230 uint32_t old_info; 11231 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 11232 int32_t exit_point; 11233 11234 if (opline->opcode != ZEND_FETCH_LIST_R 11235 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11236 && !op1_avoid_refcounting) { 11237 flags |= ZEND_JIT_EXIT_FREE_OP1; 11238 } 11239 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) 11240 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11241 flags |= ZEND_JIT_EXIT_FREE_OP2; 11242 } 11243 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 11244 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 11245 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 11246 && (ssa_op+1)->op1_use == ssa_op->result_def 11247 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) 11248 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 11249 result_avoid_refcounting = 1; 11250 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 11251 } 11252 11253 if (op1_avoid_refcounting) { 11254 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 11255 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 11256 } 11257 11258 if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { 11259 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11260 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 11261 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 11262 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11263 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11264 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11265 if (!res_exit_addr) { 11266 return 0; 11267 } 11268 res_info &= ~MAY_BE_GUARD; 11269 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 11270 } 11271 11272 if (opline->opcode == ZEND_FETCH_DIM_IS 11273 && !(res_info & MAY_BE_NULL)) { 11274 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11275 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); 11276 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); 11277 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11278 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11279 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11280 if (!not_found_exit_addr) { 11281 return 0; 11282 } 11283 } 11284 11285 if (op1_avoid_refcounting) { 11286 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 11287 } 11288 } 11289 11290 if (op1_info & MAY_BE_REF) { 11291 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11292 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 11293 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11294 } 11295 11296 if (op1_info & MAY_BE_ARRAY) { 11297 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11298 if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { 11299 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr, ZREG_TMP1 11300 } else { 11301 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 11302 } 11303 } 11304 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 11305 if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) || 11306 (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) { 11307 may_throw = 1; 11308 } 11309 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, dim_type, res_exit_addr, not_found_exit_addr, exit_addr)) { 11310 return 0; 11311 } 11312 } 11313 11314 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 11315 if (op1_info & MAY_BE_ARRAY) { 11316 |.cold_code 11317 |7: 11318 } 11319 11320 if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { 11321 may_throw = 1; 11322 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { 11323 if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { 11324 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr, ZREG_TMP1 11325 } else { 11326 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 11327 } 11328 } 11329 | SET_EX_OPLINE opline, REG0 11330 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 11331 if (opline->opcode != ZEND_FETCH_DIM_IS) { 11332 if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { 11333 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 11334 | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, REG0 11335 } else { 11336 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11337 | EXT_CALL zend_jit_fetch_dim_str_r_helper, REG0 11338 } 11339 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 11340 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 11341 } else { 11342 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11343 | LOAD_ZVAL_ADDR CARG3, res_addr 11344 | EXT_CALL zend_jit_fetch_dim_str_is_helper, REG0 11345 } 11346 if ((op1_info & MAY_BE_ARRAY) || 11347 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { 11348 | b >9 // END 11349 } 11350 |6: 11351 } 11352 11353 if (op1_info & MAY_BE_OBJECT) { 11354 may_throw = 1; 11355 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { 11356 if (exit_addr) { 11357 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 11358 } else { 11359 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6, ZREG_TMP1 11360 } 11361 } 11362 | SET_EX_OPLINE opline, REG0 11363 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11364 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11365 } 11366 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11367 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11368 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 11369 } else { 11370 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11371 } 11372 | LOAD_ZVAL_ADDR CARG3, res_addr 11373 if (opline->opcode != ZEND_FETCH_DIM_IS) { 11374 | EXT_CALL zend_jit_fetch_dim_obj_r_helper, REG0 11375 } else { 11376 | EXT_CALL zend_jit_fetch_dim_obj_is_helper, REG0 11377 } 11378 if ((op1_info & MAY_BE_ARRAY) || 11379 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 11380 | b >9 // END 11381 } 11382 |6: 11383 } 11384 11385 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) 11386 && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 11387 if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { 11388 | SET_EX_OPLINE opline, REG0 11389 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { 11390 may_throw = 1; 11391 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 11392 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 11393 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 11394 | EXT_CALL zend_jit_undefined_op_helper, REG0 11395 |1: 11396 } 11397 11398 if (op2_info & MAY_BE_UNDEF) { 11399 may_throw = 1; 11400 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 11401 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 11402 | EXT_CALL zend_jit_undefined_op_helper, REG0 11403 |1: 11404 } 11405 } 11406 11407 if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { 11408 may_throw = 1; 11409 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { 11410 | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr 11411 } else { 11412 | SET_EX_OPLINE opline, REG0 11413 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || 11414 Z_REG(op1_addr) != ZREG_FCARG1 || 11415 Z_OFFSET(op1_addr) != 0) { 11416 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11417 } 11418 } 11419 | EXT_CALL zend_jit_invalid_array_access, REG0 11420 } 11421 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 11422 if (op1_info & MAY_BE_ARRAY) { 11423 | b >9 // END 11424 } 11425 } 11426 11427 if (op1_info & MAY_BE_ARRAY) { 11428 |.code 11429 } 11430 } 11431 11432 if (op1_info & MAY_BE_ARRAY) { 11433 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 11434 11435 |8: 11436 if (res_exit_addr) { 11437 uint32_t type = concrete_type(res_info); 11438 if ((op1_info & MAY_BE_ARRAY_OF_REF) 11439 && dim_type != IS_UNKNOWN 11440 && dim_type != IS_REFERENCE) { 11441 if (type < IS_STRING) { 11442 | IF_NOT_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1 11443 |.cold_code 11444 |1: 11445 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, &res_exit_addr, ZREG_TMP1 11446 | GET_Z_PTR REG0, REG0 11447 | add REG0, REG0, #offsetof(zend_reference, val) 11448 | IF_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1 11449 | b &res_exit_addr 11450 |.code 11451 |1: 11452 } else { 11453 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 11454 | GET_LOW_8BITS TMP1w, REG2w 11455 | IF_NOT_TYPE TMP1w, type, >1 11456 |.cold_code 11457 |1: 11458 | IF_NOT_TYPE TMP1w, IS_REFERENCE, &res_exit_addr 11459 | GET_Z_PTR REG0, REG0 11460 | add REG0, REG0, #offsetof(zend_reference, val) 11461 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 11462 | GET_LOW_8BITS TMP1w, REG2w 11463 | IF_TYPE TMP1w, type, >1 11464 | b &res_exit_addr 11465 |.code 11466 |1: 11467 } 11468 } else { 11469 if (op1_info & MAY_BE_ARRAY_OF_REF) { 11470 | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w 11471 } 11472 if (type < IS_STRING) { 11473 | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr, ZREG_TMP1 11474 } else { 11475 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 11476 | GET_LOW_8BITS TMP1w, REG2w 11477 | IF_NOT_TYPE TMP1w, type, &res_exit_addr 11478 } 11479 } 11480 | // ZVAL_COPY 11481 |7: 11482 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 11483 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 11484 if (type < IS_STRING) { 11485 if (Z_REG(res_addr) != ZREG_FP || 11486 JIT_G(current_frame) == NULL || 11487 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 11488 | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 11489 } 11490 } else { 11491 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 11492 if (!result_avoid_refcounting) { 11493 | TRY_ADDREF res_info, REG2w, REG1, TMP1w 11494 } 11495 } 11496 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 11497 return 0; 11498 } 11499 } else if (op1_info & MAY_BE_ARRAY_OF_REF) { 11500 | // ZVAL_COPY_DEREF 11501 | GET_ZVAL_TYPE_INFO Rw(ZREG_REG2), val_addr, TMP1 11502 if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_REG2)) { 11503 return 0; 11504 } 11505 } else { 11506 | // ZVAL_COPY 11507 | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11508 | TRY_ADDREF res_info, REG1w, REG2, TMP1w 11509 } 11510 } 11511 |9: // END 11512 11513#ifdef ZEND_JIT_USE_RC_INFERENCE 11514 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 11515 /* Magic offsetGet() may increase refcount of the key */ 11516 op2_info |= MAY_BE_RCN; 11517 } 11518#endif 11519 11520 if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) { 11521 if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) { 11522 may_throw = 1; 11523 } 11524 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11525 } 11526 if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { 11527 if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) { 11528 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 11529 may_throw = 1; 11530 } 11531 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11532 } 11533 } 11534 11535 if (may_throw) { 11536 if (!zend_jit_check_exception(Dst)) { 11537 return 0; 11538 } 11539 } 11540 11541 return 1; 11542} 11543 11544static int zend_jit_fetch_dim(dasm_State **Dst, 11545 const zend_op *opline, 11546 uint32_t op1_info, 11547 zend_jit_addr op1_addr, 11548 uint32_t op2_info, 11549 zend_jit_addr res_addr, 11550 uint8_t dim_type) 11551{ 11552 zend_jit_addr op2_addr; 11553 int may_throw = 0; 11554 11555 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 11556 11557 if (opline->opcode == ZEND_FETCH_DIM_RW) { 11558 | SET_EX_OPLINE opline, REG0 11559 } 11560 if (op1_info & MAY_BE_REF) { 11561 may_throw = 1; 11562 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11563 | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w 11564 | GET_Z_PTR FCARG2x, FCARG1x 11565 | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] 11566 | cmp TMP1w, #IS_ARRAY 11567 | bne >2 11568 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) 11569 | b >3 11570 |.cold_code 11571 |2: 11572 | SET_EX_OPLINE opline, REG0 11573 if (opline->opcode != ZEND_FETCH_DIM_RW) { 11574 | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 11575 } 11576 | mov FCARG1x, RETVALx 11577 | cbnz FCARG1x, >1 11578 | b ->exception_handler_undef 11579 |.code 11580 |1: 11581 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11582 } 11583 11584 if (op1_info & MAY_BE_ARRAY) { 11585 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11586 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 11587 } 11588 |3: 11589 | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 11590 } 11591 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 11592 if (op1_info & MAY_BE_ARRAY) { 11593 |.cold_code 11594 |7: 11595 } 11596 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 11597 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 11598 | bgt >7 11599 } 11600 if (Z_REG(op1_addr) != ZREG_FP) { 11601 | str Rx(Z_REG(op1_addr)), T1 // save 11602 } 11603 if ((op1_info & MAY_BE_UNDEF) 11604 && opline->opcode == ZEND_FETCH_DIM_RW) { 11605 may_throw = 1; 11606 if (op1_info & MAY_BE_NULL) { 11607 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 11608 } 11609 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 11610 | EXT_CALL zend_jit_undefined_op_helper, REG0 11611 |1: 11612 } 11613 | // ZVAL_ARR(container, zend_new_array(8)); 11614 | EXT_CALL _zend_new_array_0, REG0 11615 | mov REG0, RETVALx 11616 if (Z_REG(op1_addr) != ZREG_FP) { 11617 | ldr Rx(Z_REG(op1_addr)), T1 // restore 11618 } 11619 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 11620 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 11621 | mov FCARG1x, REG0 11622 if (op1_info & MAY_BE_ARRAY) { 11623 | b >1 11624 |.code 11625 |1: 11626 } 11627 } 11628 11629 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 11630 |6: 11631 if (opline->op2_type == IS_UNUSED) { 11632 may_throw = 1; 11633 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 11634 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 11635 | EXT_CALL zend_hash_next_index_insert, REG0 11636 | // if (UNEXPECTED(!var_ptr)) { 11637 | cbz RETVALx, >1 11638 |.cold_code 11639 |1: 11640 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 11641 | CANNOT_ADD_ELEMENT opline 11642 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 11643 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 11644 | b >8 11645 |.code 11646 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 11647 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 11648 } else { 11649 uint32_t type; 11650 11651 switch (opline->opcode) { 11652 case ZEND_FETCH_DIM_W: 11653 case ZEND_FETCH_LIST_W: 11654 type = BP_VAR_W; 11655 break; 11656 case ZEND_FETCH_DIM_RW: 11657 may_throw = 1; 11658 type = BP_VAR_RW; 11659 break; 11660 case ZEND_FETCH_DIM_UNSET: 11661 type = BP_VAR_UNSET; 11662 break; 11663 default: 11664 ZEND_UNREACHABLE(); 11665 } 11666 11667 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 11668 may_throw = 1; 11669 } 11670 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 11671 return 0; 11672 } 11673 11674 |8: 11675 | SET_ZVAL_PTR res_addr, REG0, TMP1 11676 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 11677 11678 if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 11679 |.cold_code 11680 |9: 11681 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 11682 | b >8 11683 |.code 11684 } 11685 } 11686 } 11687 11688 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 11689 may_throw = 1; 11690 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 11691 |.cold_code 11692 |7: 11693 } 11694 11695 if (opline->opcode != ZEND_FETCH_DIM_RW) { 11696 | SET_EX_OPLINE opline, REG0 11697 } 11698 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11699 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11700 } 11701 if (opline->op2_type == IS_UNUSED) { 11702 | mov FCARG2x, xzr 11703 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11704 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11705 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 11706 } else { 11707 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11708 } 11709 | LOAD_ZVAL_ADDR CARG3, res_addr 11710 switch (opline->opcode) { 11711 case ZEND_FETCH_DIM_W: 11712 case ZEND_FETCH_LIST_W: 11713 | EXT_CALL zend_jit_fetch_dim_obj_w_helper, REG0 11714 break; 11715 case ZEND_FETCH_DIM_RW: 11716 | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, REG0 11717 break; 11718// case ZEND_FETCH_DIM_UNSET: 11719// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, REG0 11720// break; 11721 default: 11722 ZEND_UNREACHABLE(); 11723 } 11724 11725 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 11726 | b >8 // END 11727 |.code 11728 } 11729 } 11730 11731#ifdef ZEND_JIT_USE_RC_INFERENCE 11732 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 11733 /* ASSIGN_DIM may increase refcount of the key */ 11734 op2_info |= MAY_BE_RCN; 11735 } 11736#endif 11737 11738 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 11739 && (op2_info & MAY_HAVE_DTOR) 11740 && (op2_info & MAY_BE_RC1)) { 11741 may_throw = 1; 11742 } 11743 |8: 11744 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11745 11746 if (may_throw) { 11747 if (!zend_jit_check_exception(Dst)) { 11748 return 0; 11749 } 11750 } 11751 return 1; 11752} 11753 11754static int zend_jit_isset_isempty_dim(dasm_State **Dst, 11755 const zend_op *opline, 11756 uint32_t op1_info, 11757 zend_jit_addr op1_addr, 11758 bool op1_avoid_refcounting, 11759 uint32_t op2_info, 11760 uint8_t dim_type, 11761 int may_throw, 11762 zend_uchar smart_branch_opcode, 11763 uint32_t target_label, 11764 uint32_t target_label2, 11765 const void *exit_addr) 11766{ 11767 zend_jit_addr op2_addr, res_addr; 11768 11769 // TODO: support for empty() ??? 11770 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 11771 11772 op2_addr = OP2_ADDR(); 11773 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11774 11775 if (op1_info & MAY_BE_REF) { 11776 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11777 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 11778 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11779 } 11780 11781 if (op1_info & MAY_BE_ARRAY) { 11782 const void *found_exit_addr = NULL; 11783 const void *not_found_exit_addr = NULL; 11784 11785 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11786 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 11787 } 11788 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 11789 if (exit_addr 11790 && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) 11791 && !may_throw 11792 && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) 11793 && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { 11794 if (smart_branch_opcode == ZEND_JMPNZ) { 11795 found_exit_addr = exit_addr; 11796 } else { 11797 not_found_exit_addr = exit_addr; 11798 } 11799 } 11800 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, dim_type, found_exit_addr, not_found_exit_addr, NULL)) { 11801 return 0; 11802 } 11803 11804 if (found_exit_addr) { 11805 |9: 11806 return 1; 11807 } else if (not_found_exit_addr) { 11808 |8: 11809 return 1; 11810 } 11811 } 11812 11813 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 11814 if (op1_info & MAY_BE_ARRAY) { 11815 |.cold_code 11816 |7: 11817 } 11818 11819 if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { 11820 | SET_EX_OPLINE opline, REG0 11821 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11822 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11823 } 11824 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11825 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11826 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 11827 } else { 11828 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11829 } 11830 | EXT_CALL zend_jit_isset_dim_helper, REG0 11831 | cbz RETVALw, >9 11832 if (op1_info & MAY_BE_ARRAY) { 11833 | b >8 11834 |.code 11835 } 11836 } else { 11837 if (op2_info & MAY_BE_UNDEF) { 11838 if (op2_info & MAY_BE_ANY) { 11839 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 11840 } 11841 | SET_EX_OPLINE opline, REG0 11842 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 11843 | EXT_CALL zend_jit_undefined_op_helper, REG0 11844 |1: 11845 } 11846 if (op1_info & MAY_BE_ARRAY) { 11847 | b >9 11848 |.code 11849 } 11850 } 11851 } 11852 11853#ifdef ZEND_JIT_USE_RC_INFERENCE 11854 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 11855 /* Magic offsetExists() may increase refcount of the key */ 11856 op2_info |= MAY_BE_RCN; 11857 } 11858#endif 11859 11860 if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { 11861 |8: 11862 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11863 if (!op1_avoid_refcounting) { 11864 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11865 } 11866 if (may_throw) { 11867 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 11868 return 0; 11869 } 11870 } 11871 if (!(opline->extended_value & ZEND_ISEMPTY)) { 11872 if (exit_addr) { 11873 if (smart_branch_opcode == ZEND_JMPNZ) { 11874 | b &exit_addr 11875 } else { 11876 | b >8 11877 } 11878 } else if (smart_branch_opcode) { 11879 if (smart_branch_opcode == ZEND_JMPZ) { 11880 | b =>target_label2 11881 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11882 | b =>target_label 11883 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11884 | b =>target_label2 11885 } else { 11886 ZEND_UNREACHABLE(); 11887 } 11888 } else { 11889 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 11890 | b >8 11891 } 11892 } else { 11893 | NIY // TODO: support for empty() 11894 } 11895 } 11896 11897 |9: // not found 11898 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11899 if (!op1_avoid_refcounting) { 11900 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11901 } 11902 if (may_throw) { 11903 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 11904 return 0; 11905 } 11906 } 11907 if (!(opline->extended_value & ZEND_ISEMPTY)) { 11908 if (exit_addr) { 11909 if (smart_branch_opcode == ZEND_JMPZ) { 11910 | b &exit_addr 11911 } 11912 } else if (smart_branch_opcode) { 11913 if (smart_branch_opcode == ZEND_JMPZ) { 11914 | b =>target_label 11915 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11916 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11917 | b =>target_label 11918 } else { 11919 ZEND_UNREACHABLE(); 11920 } 11921 } else { 11922 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 11923 } 11924 } else { 11925 | NIY // TODO: support for empty() 11926 } 11927 11928 |8: 11929 11930 return 1; 11931} 11932 11933static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 11934{ 11935 zend_jit_addr op1_addr = OP1_ADDR(); 11936 zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); 11937 11938 | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; 11939 | ldr FCARG2x, EX->run_time_cache 11940 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG2x, opline->extended_value, TMP1 11941 | sub REG0, REG0, #1 11942 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) 11943 | MEM_LOAD_32_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1 11944 | cmp REG0, REG1, lsl #5 11945 | bhs >9 11946 | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); 11947 | MEM_LOAD_64_ZTS ldr, TMP1, executor_globals, symbol_table.arData, REG1 11948 | add REG0, REG0, TMP1 11949 | IF_NOT_Z_TYPE REG0, IS_REFERENCE, >9, TMP1w 11950 | // (EXPECTED(p->key == varname)) 11951 | ldr TMP1, [REG0, #offsetof(Bucket, key)] 11952 | LOAD_ADDR TMP2, varname 11953 | cmp TMP1, TMP2 11954 | bne >9 11955 | GET_Z_PTR REG0, REG0 11956 | GC_ADDREF REG0, TMP1w 11957 |1: 11958 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11959 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11960 | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) 11961 | IF_ZVAL_REFCOUNTED op1_addr, >2, ZREG_TMP1, ZREG_TMP2 11962 |.cold_code 11963 |2: 11964 } 11965 | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); 11966 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 11967 | // ZVAL_REF(variable_ptr, ref) 11968 | SET_ZVAL_PTR op1_addr, REG0, TMP1 11969 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 11970 | // if (GC_DELREF(garbage) == 0) 11971 | GC_DELREF FCARG1x, TMP1w 11972 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 11973 | bne >3 11974 } else { 11975 | bne >5 11976 } 11977 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 11978 | b >5 11979 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 11980 |3: 11981 | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) 11982 | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w 11983 | EXT_CALL gc_possible_root, REG0 11984 | b >5 11985 } 11986 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11987 |.code 11988 } 11989 } 11990 11991 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11992 | // ZVAL_REF(variable_ptr, ref) 11993 | SET_ZVAL_PTR op1_addr, REG0, TMP1 11994 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 11995 } 11996 |5: 11997 //END of handler 11998 11999 |.cold_code 12000 |9: 12001 | LOAD_ADDR FCARG1x, (ptrdiff_t)varname 12002 if (opline->extended_value) { 12003 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, FCARG2x, opline->extended_value, TMP1 12004 } 12005 | EXT_CALL zend_jit_fetch_global_helper, REG0 12006 | mov REG0, RETVALx 12007 | b <1 12008 |.code 12009 12010 return 1; 12011} 12012 12013static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception) 12014{ 12015 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12016 bool in_cold = 0; 12017 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 12018 zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1 : ZREG_REG0; 12019 12020 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12021 && JIT_G(current_frame) 12022 && JIT_G(current_frame)->prev) { 12023 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 12024 uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)); 12025 12026 if (type != IS_UNKNOWN && (type_mask & (1u << type))) { 12027 return 1; 12028 } 12029 } 12030 12031 if (ZEND_ARG_SEND_MODE(arg_info)) { 12032 if (opline->opcode == ZEND_RECV_INIT) { 12033 | LOAD_ZVAL_ADDR Rx(tmp_reg), res_addr 12034 | ZVAL_DEREF Rx(tmp_reg), MAY_BE_REF, TMP1w 12035 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); 12036 } else { 12037 | GET_ZVAL_PTR Rx(tmp_reg), res_addr, TMP1 12038 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); 12039 } 12040 } 12041 12042 if (type_mask != 0) { 12043 if (is_power_of_two(type_mask)) { 12044 uint32_t type_code = concrete_type(type_mask); 12045 | IF_NOT_ZVAL_TYPE res_addr, type_code, >1, ZREG_TMP1 12046 } else { 12047 | mov REG2w, #1 12048 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, Rx(Z_REG(res_addr)), Z_OFFSET(res_addr)+offsetof(zval, u1.v.type), TMP1 12049 | lsl REG2w, REG2w, REG1w 12050 | TST_32_WITH_CONST REG2w, type_mask, TMP1w 12051 | beq >1 12052 } 12053 12054 |.cold_code 12055 |1: 12056 12057 in_cold = 1; 12058 } 12059 12060 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 12061 | LOAD_ZVAL_ADDR FCARG1x, res_addr 12062 } 12063 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12064 | SET_EX_OPLINE opline, REG0 12065 } else { 12066 | ADDR_STORE EX->opline, opline, REG0 12067 } 12068 | LOAD_ADDR FCARG2x, (ptrdiff_t)arg_info 12069 | EXT_CALL zend_jit_verify_arg_slow, REG0 12070 12071 if (check_exception) { 12072 | GET_LOW_8BITS REG0w, RETVALw 12073 if (in_cold) { 12074 | cbnz REG0w, >1 12075 | b ->exception_handler 12076 |.code 12077 |1: 12078 } else { 12079 | cbz REG0w, ->exception_handler 12080 } 12081 } else if (in_cold) { 12082 | b >1 12083 |.code 12084 |1: 12085 } 12086 12087 return 1; 12088} 12089 12090static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) 12091{ 12092 uint32_t arg_num = opline->op1.num; 12093 zend_arg_info *arg_info = NULL; 12094 12095 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12096 if (EXPECTED(arg_num <= op_array->num_args)) { 12097 arg_info = &op_array->arg_info[arg_num-1]; 12098 } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { 12099 arg_info = &op_array->arg_info[op_array->num_args]; 12100 } 12101 if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { 12102 arg_info = NULL; 12103 } 12104 } 12105 12106 if (arg_info || (opline+1)->opcode != ZEND_RECV) { 12107 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12108 if (!JIT_G(current_frame) || 12109 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 || 12110 arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 12111 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12112 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12113 12114 if (!exit_addr) { 12115 return 0; 12116 } 12117 | ldr TMP1w, EX->This.u2.num_args 12118 | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w 12119 | blo &exit_addr 12120 } 12121 } else { 12122 | ldr TMP1w, EX->This.u2.num_args 12123 | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w 12124 | blo >1 12125 |.cold_code 12126 |1: 12127 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12128 | SET_EX_OPLINE opline, REG0 12129 } else { 12130 | ADDR_STORE EX->opline, opline, REG0 12131 } 12132 | mov FCARG1x, FP 12133 | EXT_CALL zend_missing_arg_error, REG0 12134 | b ->exception_handler 12135 |.code 12136 } 12137 } 12138 12139 if (arg_info) { 12140 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { 12141 return 0; 12142 } 12143 } 12144 12145 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12146 if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { 12147 | LOAD_IP_ADDR (opline + 1) 12148 zend_jit_set_last_valid_opline(opline + 1); 12149 } 12150 } 12151 12152 return 1; 12153} 12154 12155static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw) 12156{ 12157 uint32_t arg_num = opline->op1.num; 12158 zval *zv = RT_CONSTANT(opline, opline->op2); 12159 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12160 12161 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12162 && JIT_G(current_frame) 12163 && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) { 12164 if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 12165 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 12166 if (Z_REFCOUNTED_P(zv)) { 12167 | ADDREF_CONST zv, REG0, TMP1 12168 } 12169 } 12170 } else { 12171 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 12172 (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 12173 | ldr TMP1w, EX->This.u2.num_args 12174 | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w 12175 | bhs >5 12176 } 12177 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 12178 if (Z_REFCOUNTED_P(zv)) { 12179 | ADDREF_CONST zv, REG0, TMP1 12180 } 12181 } 12182 12183 if (Z_CONSTANT_P(zv)) { 12184 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12185 | SET_EX_OPLINE opline, REG0 12186 } else { 12187 | ADDR_STORE EX->opline, opline, REG0 12188 } 12189 | LOAD_ZVAL_ADDR FCARG1x, res_addr 12190 | ldr REG0, EX->func 12191 | ldr FCARG2x, [REG0, #offsetof(zend_op_array, scope)] 12192 | EXT_CALL zval_update_constant_ex, REG0 12193 | cbnz RETVALw, >1 12194 |.cold_code 12195 |1: 12196 | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline, ZREG_TMP1, ZREG_TMP2 12197 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 12198 | b ->exception_handler 12199 |.code 12200 } 12201 12202 |5: 12203 12204 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12205 do { 12206 zend_arg_info *arg_info; 12207 12208 if (arg_num <= op_array->num_args) { 12209 arg_info = &op_array->arg_info[arg_num-1]; 12210 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { 12211 arg_info = &op_array->arg_info[op_array->num_args]; 12212 } else { 12213 break; 12214 } 12215 if (!ZEND_TYPE_IS_SET(arg_info->type)) { 12216 break; 12217 } 12218 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { 12219 return 0; 12220 } 12221 } while (0); 12222 } 12223 12224 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12225 if (is_last) { 12226 | LOAD_IP_ADDR (opline + 1) 12227 zend_jit_set_last_valid_opline(opline + 1); 12228 } 12229 } 12230 return 1; 12231} 12232 12233static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) 12234{ 12235 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 12236 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12237 12238 if (!exit_addr) { 12239 return 0; 12240 } 12241 12242 | LOAD_ADDR TMP1, ((ptrdiff_t)ce) 12243 | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] 12244 | cmp TMP2, TMP1 12245 | bne &exit_addr 12246 12247 return 1; 12248} 12249 12250static int zend_jit_fetch_obj(dasm_State **Dst, 12251 const zend_op *opline, 12252 const zend_op_array *op_array, 12253 zend_ssa *ssa, 12254 const zend_ssa_op *ssa_op, 12255 uint32_t op1_info, 12256 zend_jit_addr op1_addr, 12257 bool op1_indirect, 12258 zend_class_entry *ce, 12259 bool ce_is_instanceof, 12260 bool on_this, 12261 bool delayed_fetch_this, 12262 bool op1_avoid_refcounting, 12263 zend_class_entry *trace_ce, 12264 uint8_t prop_type, 12265 int may_throw) 12266{ 12267 zval *member; 12268 zend_property_info *prop_info; 12269 bool may_be_dynamic = 1; 12270 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12271 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 12272 zend_jit_addr prop_addr; 12273 uint32_t res_info = RES_INFO(); 12274 bool type_loaded = 0; 12275 12276 ZEND_ASSERT(opline->op2_type == IS_CONST); 12277 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 12278 12279 member = RT_CONSTANT(opline, opline->op2); 12280 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 12281 prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); 12282 12283 if (on_this) { 12284 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 12285 } else { 12286 if (opline->op1_type == IS_VAR 12287 && opline->opcode == ZEND_FETCH_OBJ_W 12288 && (op1_info & MAY_BE_INDIRECT) 12289 && Z_REG(op1_addr) == ZREG_FP) { 12290 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12291 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 12292 | GET_Z_PTR FCARG1x, FCARG1x 12293 |1: 12294 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12295 } 12296 if (op1_info & MAY_BE_REF) { 12297 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12298 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12299 } 12300 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 12301 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12302 } 12303 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 12304 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12305 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12306 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12307 12308 if (!exit_addr) { 12309 return 0; 12310 } 12311 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 12312 } else { 12313 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, ZREG_TMP1 12314 } 12315 } 12316 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 12317 } 12318 12319 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12320 prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename); 12321 if (prop_info) { 12322 ce = trace_ce; 12323 ce_is_instanceof = 0; 12324 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 12325 if (on_this && JIT_G(current_frame) 12326 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 12327 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 12328 } else if (zend_jit_class_guard(Dst, opline, ce)) { 12329 if (on_this && JIT_G(current_frame)) { 12330 JIT_G(current_frame)->ce = ce; 12331 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 12332 } 12333 } else { 12334 return 0; 12335 } 12336 if (ssa->var_info && ssa_op->op1_use >= 0) { 12337 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 12338 ssa->var_info[ssa_op->op1_use].ce = ce; 12339 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 12340 } 12341 } 12342 } 12343 } 12344 12345 if (!prop_info) { 12346 | ldr REG0, EX->run_time_cache 12347 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS), TMP1 12348 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 12349 | cmp REG2, TMP1 12350 | bne >5 12351 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)), TMP1 12352 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); 12353 if (may_be_dynamic) { 12354 | tst REG0, REG0 12355 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12356 | blt >5 12357 } else { 12358 | blt >8 // dynamic property 12359 } 12360 } 12361 | add TMP1, FCARG1x, REG0 12362 | ldr REG2w, [TMP1, #offsetof(zval,u1.type_info)] 12363 | IF_UNDEF REG2w, >5 12364 | mov FCARG1x, TMP1 12365 type_loaded = 1; 12366 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12367 if (opline->opcode == ZEND_FETCH_OBJ_W 12368 && (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) { 12369 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 12370 12371 | ldr REG0, EX->run_time_cache 12372 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2), TMP1 12373 | cbnz FCARG2x, >1 12374 |.cold_code 12375 |1: 12376 | ldr TMP1w, [FCARG2x, #offsetof(zend_property_info, flags)] 12377 | tst TMP1w, #ZEND_ACC_READONLY 12378 if (flags) { 12379 | beq >3 12380 } else { 12381 | beq >4 12382 } 12383 | IF_NOT_TYPE REG2w, IS_OBJECT_EX, >2 12384 | GET_Z_PTR REG2, FCARG1x 12385 | GC_ADDREF REG2, TMP1w 12386 | SET_ZVAL_PTR res_addr, REG2, TMP1 12387 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2 12388 | b >9 12389 |2: 12390 | mov FCARG1x, FCARG2x 12391 | SET_EX_OPLINE opline, REG0 12392 | EXT_CALL zend_readonly_property_modification_error, REG0 12393 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 12394 | b >9 12395 |3: 12396 if (flags == ZEND_FETCH_DIM_WRITE) { 12397 | SET_EX_OPLINE opline, REG0 12398 | EXT_CALL zend_jit_check_array_promotion, REG0 12399 | b >9 12400 } else if (flags == ZEND_FETCH_REF) { 12401 | LOAD_ZVAL_ADDR CARG3, res_addr 12402 | EXT_CALL zend_jit_create_typed_ref, REG0 12403 | b >9 12404 } else { 12405 ZEND_ASSERT(flags == 0); 12406 } 12407 |.code 12408 |4: 12409 } 12410 } else { 12411 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 12412 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12413 if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { 12414 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 12415 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12416 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12417 12418 if (!exit_addr) { 12419 return 0; 12420 } 12421 type_loaded = 1; 12422 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12423 | IF_UNDEF REG2w, &exit_addr 12424 } 12425 } else { 12426 type_loaded = 1; 12427 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12428 | IF_UNDEF REG2w, >5 12429 } 12430 if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) { 12431 if (!type_loaded) { 12432 type_loaded = 1; 12433 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12434 } 12435 | IF_NOT_TYPE REG2w, IS_OBJECT_EX, >4 12436 | GET_ZVAL_PTR REG2, prop_addr, TMP1 12437 | GC_ADDREF REG2, TMP1w 12438 | SET_ZVAL_PTR res_addr, REG2, TMP1 12439 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2 12440 | b >9 12441 |.cold_code 12442 |4: 12443 | LOAD_ADDR FCARG1x, prop_info 12444 | SET_EX_OPLINE opline, REG0 12445 | EXT_CALL zend_readonly_property_modification_error, REG0 12446 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 12447 | b >9 12448 |.code 12449 } 12450 if (opline->opcode == ZEND_FETCH_OBJ_W 12451 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) 12452 && ZEND_TYPE_IS_SET(prop_info->type)) { 12453 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 12454 12455 if (flags == ZEND_FETCH_DIM_WRITE) { 12456 if ((ZEND_TYPE_FULL_MASK(prop_info->type) & (MAY_BE_ITERABLE|MAY_BE_ARRAY)) == 0) { 12457 if (!type_loaded) { 12458 type_loaded = 1; 12459 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12460 } 12461 | cmp REG2w, #IS_FALSE 12462 | ble >1 12463 |.cold_code 12464 |1: 12465 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12466 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12467 } 12468 | LOAD_ADDR FCARG2x, prop_info 12469 | SET_EX_OPLINE opline, REG0 12470 | EXT_CALL zend_jit_check_array_promotion, REG0 12471 | b >9 12472 |.code 12473 } 12474 } else if (flags == ZEND_FETCH_REF) { 12475 if (!type_loaded) { 12476 type_loaded = 1; 12477 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12478 } 12479 | GET_LOW_8BITS TMP1w, REG2w 12480 | IF_TYPE TMP1w, IS_REFERENCE, >1 12481 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 12482 | LOAD_ADDR FCARG2x, prop_info 12483 } else { 12484 int prop_info_offset = 12485 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 12486 12487 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 12488 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 12489 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 12490 } 12491 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12492 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12493 } 12494 | LOAD_ZVAL_ADDR CARG3, res_addr 12495 | EXT_CALL zend_jit_create_typed_ref, REG0 12496 | b >9 12497 |1: 12498 } else { 12499 ZEND_UNREACHABLE(); 12500 } 12501 } 12502 } 12503 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12504 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12505 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12506 } 12507 | SET_ZVAL_PTR res_addr, FCARG1x, TMP1 12508 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 12509 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { 12510 ssa->var_info[ssa_op->result_def].indirect_reference = 1; 12511 } 12512 } else { 12513 bool result_avoid_refcounting = 0; 12514 12515 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { 12516 uint32_t flags = 0; 12517 uint32_t old_info; 12518 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 12519 int32_t exit_point; 12520 const void *exit_addr; 12521 uint32_t type; 12522 zend_jit_addr val_addr = prop_addr; 12523 12524 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 12525 && !delayed_fetch_this 12526 && !op1_avoid_refcounting) { 12527 flags = ZEND_JIT_EXIT_FREE_OP1; 12528 } 12529 12530 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 12531 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 12532 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 12533 && (ssa_op+1)->op1_use == ssa_op->result_def 12534 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 12535 result_avoid_refcounting = 1; 12536 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 12537 } 12538 12539 type = concrete_type(res_info); 12540 12541 if (prop_type != IS_UNKNOWN 12542 && prop_type != IS_UNDEF 12543 && prop_type != IS_REFERENCE 12544 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) { 12545 exit_point = zend_jit_trace_get_exit_point(opline, 0); 12546 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12547 if (!exit_addr) { 12548 return 0; 12549 } 12550 } else { 12551 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 12552 | LOAD_ZVAL_ADDR REG0, prop_addr 12553 if (op1_avoid_refcounting) { 12554 SET_STACK_REG(JIT_G(current_frame)->stack, 12555 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 12556 } 12557 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 12558 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 12559 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 12560 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 12561 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 12562 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12563 if (!exit_addr) { 12564 return 0; 12565 } 12566 12567 if (!type_loaded) { 12568 type_loaded = 1; 12569 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12570 } 12571 | // ZVAL_DEREF() 12572 | GET_LOW_8BITS TMP1w, REG2w 12573 | IF_NOT_TYPE TMP1w, IS_REFERENCE, >1 12574 | GET_Z_PTR REG0, REG0 12575 | add REG0, REG0, #offsetof(zend_reference, val) 12576 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 12577 } 12578 res_info &= ~MAY_BE_GUARD; 12579 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 12580 if (type < IS_STRING) { 12581 |1: 12582 if (type_loaded) { 12583 | IF_NOT_TYPE REG2w, type, &exit_addr 12584 } else { 12585 | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr, ZREG_TMP1 12586 } 12587 } else { 12588 if (!type_loaded) { 12589 type_loaded = 1; 12590 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 12591 } 12592 |1: 12593 | GET_LOW_8BITS TMP1w, REG2w 12594 | IF_NOT_TYPE TMP1w, type, &exit_addr 12595 } 12596 | // ZVAL_COPY 12597 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 12598 if (type < IS_STRING) { 12599 if (Z_REG(res_addr) != ZREG_FP || 12600 JIT_G(current_frame) == NULL || 12601 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 12602 | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 12603 } 12604 } else { 12605 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 12606 if (!result_avoid_refcounting) { 12607 | TRY_ADDREF res_info, REG2w, REG1, TMP1w 12608 } 12609 } 12610 } else { 12611 if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_REG2)) { 12612 return 0; 12613 } 12614 } 12615 } 12616 12617 if (op1_avoid_refcounting) { 12618 SET_STACK_REG(JIT_G(current_frame)->stack, 12619 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 12620 } 12621 12622 |.cold_code 12623 12624 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { 12625 |5: 12626 | SET_EX_OPLINE opline, REG0 12627 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12628 | EXT_CALL zend_jit_fetch_obj_w_slow, REG0 12629 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { 12630 | EXT_CALL zend_jit_fetch_obj_r_slow, REG0 12631 } else { 12632 | EXT_CALL zend_jit_fetch_obj_is_slow, REG0 12633 } 12634 | b >9 12635 } 12636 12637 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12638 |7: 12639 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 12640 | SET_EX_OPLINE opline, REG0 12641 if (opline->opcode != ZEND_FETCH_OBJ_W 12642 && (op1_info & MAY_BE_UNDEF)) { 12643 zend_jit_addr orig_op1_addr = OP1_ADDR(); 12644 12645 if (op1_info & MAY_BE_ANY) { 12646 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 12647 } 12648 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 12649 | EXT_CALL zend_jit_undefined_op_helper, REG0 12650 |1: 12651 | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr 12652 } else if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12653 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12654 } 12655 | LOAD_ADDR FCARG2x, Z_STRVAL_P(member) 12656 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12657 | EXT_CALL zend_jit_invalid_property_write, REG0 12658 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 12659 } else { 12660 | EXT_CALL zend_jit_invalid_property_read, REG0 12661 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 12662 } 12663 | b >9 12664 } else { 12665 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 12666 | b >9 12667 } 12668 } 12669 12670 if (!prop_info 12671 && may_be_dynamic 12672 && opline->opcode != ZEND_FETCH_OBJ_W) { 12673 |8: 12674 | mov FCARG2x, REG0 12675 | SET_EX_OPLINE opline, REG0 12676 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 12677 | EXT_CALL zend_jit_fetch_obj_r_dynamic, REG0 12678 } else { 12679 | EXT_CALL zend_jit_fetch_obj_is_dynamic, REG0 12680 } 12681 | b >9 12682 } 12683 12684 |.code; 12685 |9: // END 12686 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 12687 if (opline->op1_type == IS_VAR 12688 && opline->opcode == ZEND_FETCH_OBJ_W 12689 && (op1_info & MAY_BE_RC1)) { 12690 zend_jit_addr orig_op1_addr = OP1_ADDR(); 12691 12692 | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1, ZREG_TMP1, ZREG_TMP2 12693 | GET_ZVAL_PTR FCARG1x, orig_op1_addr, TMP1 12694 | GC_DELREF FCARG1x, TMP1w 12695 | bne >1 12696 | SET_EX_OPLINE opline, REG0 12697 | EXT_CALL zend_jit_extract_helper, REG0 12698 |1: 12699 } else if (!op1_avoid_refcounting) { 12700 if (on_this) { 12701 op1_info &= ~MAY_BE_RC1; 12702 } 12703 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 12704 } 12705 } 12706 12707 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12708 && prop_info 12709 && (opline->opcode != ZEND_FETCH_OBJ_W || 12710 !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) || 12711 !ZEND_TYPE_IS_SET(prop_info->type)) 12712 && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) { 12713 may_throw = 0; 12714 } 12715 12716 if (may_throw) { 12717 if (!zend_jit_check_exception(Dst)) { 12718 return 0; 12719 } 12720 } 12721 12722 return 1; 12723} 12724 12725static int zend_jit_incdec_obj(dasm_State **Dst, 12726 const zend_op *opline, 12727 const zend_op_array *op_array, 12728 zend_ssa *ssa, 12729 const zend_ssa_op *ssa_op, 12730 uint32_t op1_info, 12731 zend_jit_addr op1_addr, 12732 bool op1_indirect, 12733 zend_class_entry *ce, 12734 bool ce_is_instanceof, 12735 bool on_this, 12736 bool delayed_fetch_this, 12737 zend_class_entry *trace_ce, 12738 uint8_t prop_type) 12739{ 12740 zval *member; 12741 zend_string *name; 12742 zend_property_info *prop_info; 12743 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 12744 zend_jit_addr res_addr = 0; 12745 zend_jit_addr prop_addr; 12746 bool needs_slow_path = 0; 12747 bool use_prop_guard = 0; 12748 bool may_throw = 0; 12749 uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0; 12750 12751 ZEND_ASSERT(opline->op2_type == IS_CONST); 12752 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 12753 12754 if (opline->result_type != IS_UNUSED) { 12755 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12756 } 12757 12758 member = RT_CONSTANT(opline, opline->op2); 12759 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 12760 name = Z_STR_P(member); 12761 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 12762 12763 if (on_this) { 12764 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 12765 } else { 12766 if (opline->op1_type == IS_VAR 12767 && (op1_info & MAY_BE_INDIRECT) 12768 && Z_REG(op1_addr) == ZREG_FP) { 12769 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12770 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 12771 | GET_Z_PTR FCARG1x, FCARG1x 12772 |1: 12773 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12774 } 12775 if (op1_info & MAY_BE_REF) { 12776 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12777 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12778 } 12779 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 12780 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12781 } 12782 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 12783 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12784 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12785 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12786 12787 if (!exit_addr) { 12788 return 0; 12789 } 12790 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 12791 } else { 12792 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 12793 |.cold_code 12794 |1: 12795 | SET_EX_OPLINE opline, REG0 12796 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12797 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12798 } 12799 | LOAD_ADDR FCARG2x, ZSTR_VAL(name) 12800 | EXT_CALL zend_jit_invalid_property_incdec, REG0 12801 | b ->exception_handler 12802 |.code 12803 } 12804 } 12805 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 12806 } 12807 12808 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12809 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 12810 if (prop_info) { 12811 ce = trace_ce; 12812 ce_is_instanceof = 0; 12813 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 12814 if (on_this && JIT_G(current_frame) 12815 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 12816 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 12817 } else if (zend_jit_class_guard(Dst, opline, ce)) { 12818 if (on_this && JIT_G(current_frame)) { 12819 JIT_G(current_frame)->ce = ce; 12820 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 12821 } 12822 } else { 12823 return 0; 12824 } 12825 if (ssa->var_info && ssa_op->op1_use >= 0) { 12826 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 12827 ssa->var_info[ssa_op->op1_use].ce = ce; 12828 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 12829 } 12830 if (ssa->var_info && ssa_op->op1_def >= 0) { 12831 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 12832 ssa->var_info[ssa_op->op1_def].ce = ce; 12833 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 12834 } 12835 } 12836 } 12837 } 12838 12839 use_prop_guard = (prop_type != IS_UNKNOWN 12840 && prop_type != IS_UNDEF 12841 && prop_type != IS_REFERENCE 12842 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 12843 12844 if (!prop_info) { 12845 needs_slow_path = 1; 12846 12847 | ldr REG0, EX->run_time_cache 12848 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 12849 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 12850 | cmp REG2, TMP1 12851 | bne >7 12852 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 12853 | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 12854 | cbnz TMP1, >7 12855 } 12856 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 12857 | tst REG0, REG0 12858 | blt >7 12859 | add TMP1, FCARG1x, REG0 12860 if (!use_prop_guard) { 12861 | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] 12862 | IF_TYPE TMP2w , IS_UNDEF, >7 12863 } 12864 | mov FCARG1x, TMP1 12865 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12866 } else { 12867 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 12868 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 12869 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12870 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12871 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12872 12873 if (!exit_addr) { 12874 return 0; 12875 } 12876 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 12877 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr 12878 } else { 12879 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 12880 | IF_TYPE TMP2w, IS_UNDEF, >7 12881 needs_slow_path = 1; 12882 } 12883 } 12884 if (ZEND_TYPE_IS_SET(prop_info->type)) { 12885 may_throw = 1; 12886 | SET_EX_OPLINE opline, REG0 12887 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 12888 | LOAD_ADDR FCARG2x, prop_info 12889 } else { 12890 int prop_info_offset = 12891 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 12892 12893 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 12894 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 12895 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 12896 } 12897 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12898 if (opline->result_type == IS_UNUSED) { 12899 switch (opline->opcode) { 12900 case ZEND_PRE_INC_OBJ: 12901 case ZEND_POST_INC_OBJ: 12902 | EXT_CALL zend_jit_inc_typed_prop, REG0 12903 break; 12904 case ZEND_PRE_DEC_OBJ: 12905 case ZEND_POST_DEC_OBJ: 12906 | EXT_CALL zend_jit_dec_typed_prop, REG0 12907 break; 12908 default: 12909 ZEND_UNREACHABLE(); 12910 } 12911 } else { 12912 | LOAD_ZVAL_ADDR CARG3, res_addr 12913 switch (opline->opcode) { 12914 case ZEND_PRE_INC_OBJ: 12915 | EXT_CALL zend_jit_pre_inc_typed_prop, REG0 12916 break; 12917 case ZEND_PRE_DEC_OBJ: 12918 | EXT_CALL zend_jit_pre_dec_typed_prop, REG0 12919 break; 12920 case ZEND_POST_INC_OBJ: 12921 | EXT_CALL zend_jit_post_inc_typed_prop, REG0 12922 break; 12923 case ZEND_POST_DEC_OBJ: 12924 | EXT_CALL zend_jit_post_dec_typed_prop, REG0 12925 break; 12926 default: 12927 ZEND_UNREACHABLE(); 12928 } 12929 } 12930 } 12931 } 12932 12933 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 12934 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 12935 zend_jit_addr var_addr = prop_addr; 12936 12937 if (use_prop_guard) { 12938 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 12939 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12940 if (!exit_addr) { 12941 return 0; 12942 } 12943 12944 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1 12945 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 12946 } 12947 12948 if (var_info & MAY_BE_REF) { 12949 may_throw = 1; 12950 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12951 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12952 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12953 } 12954 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 12955 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 12956 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 12957 | cbnz TMP1, >1 12958 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) 12959 |.cold_code 12960 |1: 12961 if (opline) { 12962 | SET_EX_OPLINE opline, REG0 12963 } 12964 if (opline->result_type == IS_UNUSED) { 12965 | mov FCARG2x, xzr 12966 } else { 12967 | LOAD_ZVAL_ADDR FCARG2x, res_addr 12968 } 12969 switch (opline->opcode) { 12970 case ZEND_PRE_INC_OBJ: 12971 | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 12972 break; 12973 case ZEND_PRE_DEC_OBJ: 12974 | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 12975 break; 12976 case ZEND_POST_INC_OBJ: 12977 | EXT_CALL zend_jit_post_inc_typed_ref, REG0 12978 break; 12979 case ZEND_POST_DEC_OBJ: 12980 | EXT_CALL zend_jit_post_dec_typed_ref, REG0 12981 break; 12982 default: 12983 ZEND_UNREACHABLE(); 12984 } 12985 | b >9 12986 |.code 12987 |2: 12988 } 12989 12990 if (var_info & MAY_BE_LONG) { 12991 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 12992 | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2, ZREG_TMP1 12993 } 12994 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 12995 if (opline->result_type != IS_UNUSED) { 12996 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 12997 } 12998 } 12999 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13000 | LONG_ADD_SUB_WITH_IMM adds, var_addr, Z_L(1), TMP1, TMP2 13001 } else { 13002 | LONG_ADD_SUB_WITH_IMM subs, var_addr, Z_L(1), TMP1, TMP2 13003 } 13004 | bvs >3 13005 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { 13006 if (opline->result_type != IS_UNUSED) { 13007 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 13008 } 13009 } 13010 |.cold_code 13011 } 13012 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 13013 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13014 may_throw = 1; 13015 } 13016 if (var_info & MAY_BE_LONG) { 13017 |2: 13018 } 13019 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 13020 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13021 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 13022 } 13023 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 13024 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 13025 | TRY_ADDREF MAY_BE_ANY, REG0w, REG2, TMP1w 13026 } 13027 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13028 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13029 | LOAD_ZVAL_ADDR FCARG2x, res_addr 13030 | EXT_CALL zend_jit_pre_inc, REG0 13031 } else { 13032 | EXT_CALL increment_function, REG0 13033 } 13034 } else { 13035 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13036 | LOAD_ZVAL_ADDR FCARG2x, res_addr 13037 | EXT_CALL zend_jit_pre_dec, REG0 13038 } else { 13039 | EXT_CALL decrement_function, REG0 13040 } 13041 } 13042 if (var_info & MAY_BE_LONG) { 13043 | b >4 13044 } 13045 } 13046 13047 if (var_info & MAY_BE_LONG) { 13048 |3: 13049 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13050 uint64_t val = 0x43e0000000000000; 13051 | LOAD_64BIT_VAL TMP2, val 13052 | SET_ZVAL_LVAL_FROM_REG var_addr, TMP2, TMP1 13053 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2 13054 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13055 | SET_ZVAL_LVAL_FROM_REG res_addr, TMP2, TMP1 13056 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 13057 } 13058 } else { 13059 uint64_t val = 0xc3e0000000000000; 13060 | LOAD_64BIT_VAL TMP2, val 13061 | SET_ZVAL_LVAL_FROM_REG var_addr, TMP2, TMP1 13062 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2 13063 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13064 | SET_ZVAL_LVAL_FROM_REG res_addr, TMP2, TMP1 13065 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 13066 } 13067 } 13068 if (opline->result_type != IS_UNUSED 13069 && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) 13070 && prop_info 13071 && !ZEND_TYPE_IS_SET(prop_info->type) 13072 && (res_info & MAY_BE_GUARD) 13073 && (res_info & MAY_BE_LONG)) { 13074 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 13075 uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 13076 int32_t exit_point; 13077 const void *exit_addr; 13078 13079 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 13080 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 13081 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13082 if (!exit_addr) { 13083 return 0; 13084 } 13085 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 13086 ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD; 13087 | b &exit_addr 13088 |.code 13089 } else { 13090 | b >4 13091 |.code 13092 |4: 13093 } 13094 } 13095 } 13096 13097 if (needs_slow_path) { 13098 may_throw = 1; 13099 |.cold_code 13100 |7: 13101 | SET_EX_OPLINE opline, REG0 13102 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13103 | LOAD_ADDR FCARG2x, name 13104 | ldr CARG3, EX->run_time_cache 13105 | ADD_SUB_64_WITH_CONST_32 add, CARG3, CARG3, opline->extended_value, TMP1 13106 if (opline->result_type == IS_UNUSED) { 13107 | mov CARG4, xzr 13108 } else { 13109 | LOAD_ZVAL_ADDR CARG4, res_addr 13110 } 13111 13112 switch (opline->opcode) { 13113 case ZEND_PRE_INC_OBJ: 13114 | EXT_CALL zend_jit_pre_inc_obj_helper, REG0 13115 break; 13116 case ZEND_PRE_DEC_OBJ: 13117 | EXT_CALL zend_jit_pre_dec_obj_helper, REG0 13118 break; 13119 case ZEND_POST_INC_OBJ: 13120 | EXT_CALL zend_jit_post_inc_obj_helper, REG0 13121 break; 13122 case ZEND_POST_DEC_OBJ: 13123 | EXT_CALL zend_jit_post_dec_obj_helper, REG0 13124 break; 13125 default: 13126 ZEND_UNREACHABLE(); 13127 } 13128 13129 | b >9 13130 |.code 13131 } 13132 13133 |9: 13134 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13135 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 13136 may_throw = 1; 13137 } 13138 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 13139 } 13140 13141 if (may_throw) { 13142 if (!zend_jit_check_exception(Dst)) { 13143 return 0; 13144 } 13145 } 13146 13147 return 1; 13148} 13149 13150static int zend_jit_assign_obj_op(dasm_State **Dst, 13151 const zend_op *opline, 13152 const zend_op_array *op_array, 13153 zend_ssa *ssa, 13154 const zend_ssa_op *ssa_op, 13155 uint32_t op1_info, 13156 zend_jit_addr op1_addr, 13157 uint32_t val_info, 13158 zend_ssa_range *val_range, 13159 bool op1_indirect, 13160 zend_class_entry *ce, 13161 bool ce_is_instanceof, 13162 bool on_this, 13163 bool delayed_fetch_this, 13164 zend_class_entry *trace_ce, 13165 uint8_t prop_type) 13166{ 13167 zval *member; 13168 zend_string *name; 13169 zend_property_info *prop_info; 13170 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13171 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13172 zend_jit_addr prop_addr; 13173 bool needs_slow_path = 0; 13174 bool use_prop_guard = 0; 13175 bool may_throw = 0; 13176 binary_op_type binary_op = get_binary_op(opline->extended_value); 13177 13178 ZEND_ASSERT(opline->op2_type == IS_CONST); 13179 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13180 ZEND_ASSERT(opline->result_type == IS_UNUSED); 13181 13182 member = RT_CONSTANT(opline, opline->op2); 13183 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13184 name = Z_STR_P(member); 13185 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13186 13187 if (on_this) { 13188 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 13189 } else { 13190 if (opline->op1_type == IS_VAR 13191 && (op1_info & MAY_BE_INDIRECT) 13192 && Z_REG(op1_addr) == ZREG_FP) { 13193 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13194 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 13195 | GET_Z_PTR FCARG1x, FCARG1x 13196 |1: 13197 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13198 } 13199 if (op1_info & MAY_BE_REF) { 13200 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13201 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13202 } 13203 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 13204 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13205 } 13206 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13207 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13208 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13209 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13210 13211 if (!exit_addr) { 13212 return 0; 13213 } 13214 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 13215 } else { 13216 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 13217 |.cold_code 13218 |1: 13219 | SET_EX_OPLINE opline, REG0 13220 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13221 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13222 } 13223 | LOAD_ADDR FCARG2x, ZSTR_VAL(name) 13224 if (op1_info & MAY_BE_UNDEF) { 13225 | EXT_CALL zend_jit_invalid_property_assign_op, REG0 13226 } else { 13227 | EXT_CALL zend_jit_invalid_property_assign, REG0 13228 } 13229 may_throw = 1; 13230 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13231 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13232 | b >8 13233 } else { 13234 | b >9 13235 } 13236 |.code 13237 } 13238 } 13239 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 13240 } 13241 13242 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13243 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13244 if (prop_info) { 13245 ce = trace_ce; 13246 ce_is_instanceof = 0; 13247 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13248 if (on_this && JIT_G(current_frame) 13249 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13250 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13251 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13252 if (on_this && JIT_G(current_frame)) { 13253 JIT_G(current_frame)->ce = ce; 13254 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13255 } 13256 } else { 13257 return 0; 13258 } 13259 if (ssa->var_info && ssa_op->op1_use >= 0) { 13260 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13261 ssa->var_info[ssa_op->op1_use].ce = ce; 13262 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13263 } 13264 if (ssa->var_info && ssa_op->op1_def >= 0) { 13265 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13266 ssa->var_info[ssa_op->op1_def].ce = ce; 13267 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13268 } 13269 } 13270 } 13271 } 13272 13273 use_prop_guard = (prop_type != IS_UNKNOWN 13274 && prop_type != IS_UNDEF 13275 && prop_type != IS_REFERENCE 13276 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 13277 13278 if (!prop_info) { 13279 needs_slow_path = 1; 13280 13281 | ldr REG0, EX->run_time_cache 13282 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, ((opline+1)->extended_value), TMP1 13283 | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] 13284 | cmp REG2, TMP2 13285 | bne >7 13286 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 13287 | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, ((opline+1)->extended_value + sizeof(void*) * 2), TMP1 13288 | cbnz TMP1, >7 13289 } 13290 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline+1)->extended_value + sizeof(void*)), TMP1 13291 | tst REG0, REG0 13292 | blt >7 13293 | add TMP1, FCARG1x, REG0 13294 if (!use_prop_guard) { 13295 | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] 13296 | IF_TYPE TMP2w, IS_UNDEF, >7 13297 } 13298 | mov FCARG1x, TMP1 13299 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13300 } else { 13301 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13302 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 13303 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13304 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13305 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13306 13307 if (!exit_addr) { 13308 return 0; 13309 } 13310 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13311 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr 13312 } else { 13313 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13314 | IF_TYPE TMP2w, IS_UNDEF, >7 13315 needs_slow_path = 1; 13316 } 13317 } 13318 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13319 uint32_t info = val_info; 13320 13321 may_throw = 1; 13322 13323 if (opline) { 13324 | SET_EX_OPLINE opline, REG0 13325 } 13326 13327 | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1, ZREG_TMP1 13328 |.cold_code 13329 |1: 13330 | GET_ZVAL_PTR FCARG1x, prop_addr, TMP1 13331 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 13332 | LOAD_ZVAL_ADDR FCARG2x, val_addr 13333 } 13334 | LOAD_ADDR CARG3, binary_op 13335 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 13336 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13337 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 13338 } else { 13339 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 13340 } 13341 | b >9 13342 |.code 13343 13344 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13345 13346 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13347 | LOAD_ADDR FCARG2x, prop_info 13348 } else { 13349 int prop_info_offset = 13350 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13351 13352 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 13353 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 13354 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 13355 } 13356 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 13357 | LOAD_ZVAL_ADDR CARG3, val_addr 13358 | LOAD_ADDR CARG4, binary_op 13359 13360 | EXT_CALL zend_jit_assign_op_to_typed_prop, REG0 13361 13362 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13363 info |= MAY_BE_RC1|MAY_BE_RCN; 13364 } 13365 13366 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2 13367 } 13368 } 13369 13370 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13371 zend_jit_addr var_addr = prop_addr; 13372 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13373 uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13374 13375 if (use_prop_guard) { 13376 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 13377 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13378 if (!exit_addr) { 13379 return 0; 13380 } 13381 13382 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1 13383 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 13384 } 13385 13386 if (var_info & MAY_BE_REF) { 13387 may_throw = 1; 13388 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 13389 | LOAD_ZVAL_ADDR REG0, prop_addr 13390 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 13391 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 13392 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 13393 | cbnz TMP1, >1 13394 | add REG0, FCARG1x, #offsetof(zend_reference, val) 13395 |.cold_code 13396 |1: 13397 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 13398 | LOAD_ZVAL_ADDR FCARG2x, val_addr 13399 } 13400 if (opline) { 13401 | SET_EX_OPLINE opline, REG0 13402 } 13403 | LOAD_ADDR CARG3, binary_op 13404 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 13405 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13406 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 13407 } else { 13408 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 13409 } 13410 | b >9 13411 |.code 13412 |2: 13413 } 13414 13415 switch (opline->extended_value) { 13416 case ZEND_ADD: 13417 case ZEND_SUB: 13418 case ZEND_MUL: 13419 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13420 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13421 if (opline->extended_value != ZEND_ADD || 13422 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 13423 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 13424 may_throw = 1; 13425 } 13426 } 13427 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, 13428 1 /* may overflow */, 0)) { 13429 return 0; 13430 } 13431 break; 13432 case ZEND_BW_OR: 13433 case ZEND_BW_AND: 13434 case ZEND_BW_XOR: 13435 may_throw = 1; 13436 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13437 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13438 if ((var_info & MAY_BE_ANY) != MAY_BE_STRING || 13439 (val_info & MAY_BE_ANY) != MAY_BE_STRING) { 13440 may_throw = 1; 13441 } 13442 } 13443 goto long_math; 13444 case ZEND_SL: 13445 case ZEND_SR: 13446 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13447 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13448 may_throw = 1; 13449 } 13450 if ((opline+1)->op1_type != IS_CONST || 13451 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 13452 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) { 13453 may_throw = 1; 13454 } 13455 goto long_math; 13456 case ZEND_MOD: 13457 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13458 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13459 if (opline->extended_value != ZEND_ADD || 13460 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 13461 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 13462 may_throw = 1; 13463 } 13464 } 13465 if ((opline+1)->op1_type != IS_CONST || 13466 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 13467 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) { 13468 may_throw = 1; 13469 } 13470long_math: 13471 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 13472 IS_CV, opline->op1, var_addr, var_info, NULL, 13473 (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 13474 val_range, 13475 0, var_addr, var_def_info, var_info, 0)) { 13476 return 0; 13477 } 13478 break; 13479 case ZEND_CONCAT: 13480 may_throw = 1; 13481 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, 13482 0)) { 13483 return 0; 13484 } 13485 break; 13486 default: 13487 ZEND_UNREACHABLE(); 13488 } 13489 } 13490 13491 if (needs_slow_path) { 13492 may_throw = 1; 13493 |.cold_code 13494 |7: 13495 | SET_EX_OPLINE opline, REG0 13496 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13497 | LOAD_ADDR FCARG2x, name 13498 | LOAD_ZVAL_ADDR CARG3, val_addr 13499 | ldr CARG4, EX->run_time_cache 13500 | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, (opline+1)->extended_value, TMP1 13501 | LOAD_ADDR CARG5, binary_op 13502 | EXT_CALL zend_jit_assign_obj_op_helper, REG0 13503 13504 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13505 val_info |= MAY_BE_RC1|MAY_BE_RCN; 13506 } 13507 13508 |8: 13509 | // FREE_OP_DATA(); 13510 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13511 | b >9 13512 |.code 13513 } 13514 13515 |9: 13516 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13517 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 13518 may_throw = 1; 13519 } 13520 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 13521 } 13522 13523 if (may_throw) { 13524 if (!zend_jit_check_exception(Dst)) { 13525 return 0; 13526 } 13527 } 13528 13529 return 1; 13530} 13531 13532static int zend_jit_assign_obj(dasm_State **Dst, 13533 const zend_op *opline, 13534 const zend_op_array *op_array, 13535 zend_ssa *ssa, 13536 const zend_ssa_op *ssa_op, 13537 uint32_t op1_info, 13538 zend_jit_addr op1_addr, 13539 uint32_t val_info, 13540 bool op1_indirect, 13541 zend_class_entry *ce, 13542 bool ce_is_instanceof, 13543 bool on_this, 13544 bool delayed_fetch_this, 13545 zend_class_entry *trace_ce, 13546 uint8_t prop_type, 13547 int may_throw) 13548{ 13549 zval *member; 13550 zend_string *name; 13551 zend_property_info *prop_info; 13552 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13553 zend_jit_addr res_addr = 0; 13554 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13555 zend_jit_addr prop_addr; 13556 bool needs_slow_path = 0; 13557 bool needs_val_dtor = 0; 13558 13559 if (RETURN_VALUE_USED(opline)) { 13560 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 13561 } 13562 13563 ZEND_ASSERT(opline->op2_type == IS_CONST); 13564 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13565 13566 member = RT_CONSTANT(opline, opline->op2); 13567 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13568 name = Z_STR_P(member); 13569 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13570 13571 if (on_this) { 13572 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 13573 } else { 13574 if (opline->op1_type == IS_VAR 13575 && (op1_info & MAY_BE_INDIRECT) 13576 && Z_REG(op1_addr) == ZREG_FP) { 13577 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13578 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 13579 | GET_Z_PTR FCARG1x, FCARG1x 13580 |1: 13581 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13582 } 13583 if (op1_info & MAY_BE_REF) { 13584 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13585 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13586 } 13587 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 13588 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13589 } 13590 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13591 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13592 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13593 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13594 13595 if (!exit_addr) { 13596 return 0; 13597 } 13598 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 13599 } else { 13600 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 13601 |.cold_code 13602 |1: 13603 | SET_EX_OPLINE opline, REG0 13604 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13605 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13606 } 13607 | LOAD_ADDR FCARG2x, ZSTR_VAL(name) 13608 | EXT_CALL zend_jit_invalid_property_assign, REG0 13609 if (RETURN_VALUE_USED(opline)) { 13610 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 13611 } 13612 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13613 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13614 needs_val_dtor = 1; 13615 | b >7 13616 } else { 13617 | b >9 13618 } 13619 |.code 13620 } 13621 } 13622 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 13623 } 13624 13625 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13626 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13627 if (prop_info) { 13628 ce = trace_ce; 13629 ce_is_instanceof = 0; 13630 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13631 if (on_this && JIT_G(current_frame) 13632 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13633 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13634 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13635 if (on_this && JIT_G(current_frame)) { 13636 JIT_G(current_frame)->ce = ce; 13637 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13638 } 13639 } else { 13640 return 0; 13641 } 13642 if (ssa->var_info && ssa_op->op1_use >= 0) { 13643 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13644 ssa->var_info[ssa_op->op1_use].ce = ce; 13645 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13646 } 13647 if (ssa->var_info && ssa_op->op1_def >= 0) { 13648 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13649 ssa->var_info[ssa_op->op1_def].ce = ce; 13650 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13651 } 13652 } 13653 } 13654 } 13655 13656 if (!prop_info) { 13657 needs_slow_path = 1; 13658 13659 | ldr REG0, EX->run_time_cache 13660 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 13661 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 13662 | cmp REG2, TMP1 13663 | bne >5 13664 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 13665 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 13666 } 13667 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 13668 | tst REG0, REG0 13669 | blt >5 13670 | add TMP2, FCARG1x, REG0 13671 | ldrb TMP1w, [TMP2, #offsetof(zval,u1.v.type)] 13672 | IF_TYPE TMP1w, IS_UNDEF, >5 13673 | mov FCARG1x, TMP2 13674 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13675 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 13676 | cbnz FCARG2x, >1 13677 |.cold_code 13678 |1: 13679 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13680 | SET_EX_OPLINE opline, REG0 13681 | LOAD_ZVAL_ADDR CARG3, val_addr 13682 if (RETURN_VALUE_USED(opline)) { 13683 | LOAD_ZVAL_ADDR CARG4, res_addr 13684 } else { 13685 | mov CARG4, xzr 13686 } 13687 13688 | EXT_CALL zend_jit_assign_to_typed_prop, REG0 13689 13690 if ((opline+1)->op1_type == IS_CONST) { 13691 | // TODO: ??? 13692 | // if (Z_TYPE_P(value) == orig_type) { 13693 | // CACHE_PTR_EX(cache_slot + 2, NULL); 13694 } 13695 13696 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13697 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13698 | b >7 13699 } else { 13700 | b >9 13701 } 13702 |.code 13703 } 13704 } else { 13705 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13706 if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) { 13707 // Undefined property with magic __get()/__set() 13708 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13709 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13710 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13711 13712 if (!exit_addr) { 13713 return 0; 13714 } 13715 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13716 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr 13717 } else { 13718 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13719 | IF_TYPE TMP2w, IS_UNDEF, >5 13720 needs_slow_path = 1; 13721 } 13722 } 13723 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13724 uint32_t info = val_info; 13725 13726 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13727 | SET_EX_OPLINE opline, REG0 13728 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13729 | LOAD_ADDR FCARG2x, prop_info 13730 } else { 13731 int prop_info_offset = 13732 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13733 13734 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 13735 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 13736 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 13737 } 13738 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 13739 | LOAD_ZVAL_ADDR CARG3, val_addr 13740 if (RETURN_VALUE_USED(opline)) { 13741 | LOAD_ZVAL_ADDR CARG4, res_addr 13742 } else { 13743 | mov CARG4, xzr 13744 } 13745 13746 | EXT_CALL zend_jit_assign_to_typed_prop, REG0 13747 13748 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13749 info |= MAY_BE_RC1|MAY_BE_RCN; 13750 } 13751 13752 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2 13753 } 13754 } 13755 13756 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13757 // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); 13758 if (opline->result_type == IS_UNUSED) { 13759 if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 13760 return 0; 13761 } 13762 } else { 13763 if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 13764 return 0; 13765 } 13766 } 13767 } 13768 13769 if (needs_slow_path) { 13770 |.cold_code 13771 |5: 13772 | SET_EX_OPLINE opline, REG0 13773 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13774 | LOAD_ADDR FCARG2x, name 13775 13776 | LOAD_ZVAL_ADDR CARG3, val_addr 13777 | ldr CARG4, EX->run_time_cache 13778 | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, opline->extended_value, TMP1 13779 if (RETURN_VALUE_USED(opline)) { 13780 | LOAD_ZVAL_ADDR CARG5, res_addr 13781 } else { 13782 | mov CARG5, xzr 13783 } 13784 13785 | EXT_CALL zend_jit_assign_obj_helper, REG0 13786 13787 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13788 val_info |= MAY_BE_RC1|MAY_BE_RCN; 13789 } 13790 13791 |7: 13792 | // FREE_OP_DATA(); 13793 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13794 | b >9 13795 |.code 13796 } else if (needs_val_dtor) { 13797 |.cold_code 13798 |7: 13799 | // FREE_OP_DATA(); 13800 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13801 | b >9 13802 |.code 13803 } 13804 13805 |9: 13806 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13807 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 13808 } 13809 13810 if (may_throw) { 13811 if (!zend_jit_check_exception(Dst)) { 13812 return 0; 13813 } 13814 } 13815 13816 return 1; 13817} 13818 13819static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) 13820{ 13821 zend_jit_addr op1_addr = OP1_ADDR(); 13822 13823 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 13824 if (may_throw) { 13825 | SET_EX_OPLINE opline, REG0 13826 } 13827 if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { 13828 if (op1_info & MAY_BE_ARRAY) { 13829 | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 13830 } 13831 | MEM_ACCESS_32_WITH_UOFFSET ldr, FCARG1w, FP, (opline->op1.var + offsetof(zval, u2.fe_iter_idx)), TMP1 13832 | mvn TMP1w, wzr // TODO: DynAsm fails loading #-1 13833 | cmp FCARG1w, TMP1w 13834 | beq >7 13835 | EXT_CALL zend_hash_iterator_del, REG0 13836 |7: 13837 } 13838 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 13839 if (may_throw) { 13840 if (!zend_jit_check_exception(Dst)) { 13841 return 0; 13842 } 13843 } 13844 } 13845 return 1; 13846} 13847 13848static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 13849{ 13850 if (opline->op1_type == IS_CONST) { 13851 zval *zv; 13852 size_t len; 13853 13854 zv = RT_CONSTANT(opline, opline->op1); 13855 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 13856 len = Z_STRLEN_P(zv); 13857 13858 if (len > 0) { 13859 const char *str = Z_STRVAL_P(zv); 13860 13861 | SET_EX_OPLINE opline, REG0 13862 | LOAD_ADDR CARG1, str 13863 | LOAD_64BIT_VAL CARG2, len 13864 | EXT_CALL zend_write, REG0 13865 if (!zend_jit_check_exception(Dst)) { 13866 return 0; 13867 } 13868 } 13869 } else { 13870 zend_jit_addr op1_addr = OP1_ADDR(); 13871 13872 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 13873 13874 | SET_EX_OPLINE opline, REG0 13875 | GET_ZVAL_PTR REG0, op1_addr, TMP1 13876 | add CARG1, REG0, #offsetof(zend_string, val) 13877 | ldr CARG2, [REG0, #offsetof(zend_string, len)] 13878 | EXT_CALL zend_write, REG0 13879 if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { 13880 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 13881 } 13882 if (!zend_jit_check_exception(Dst)) { 13883 return 0; 13884 } 13885 } 13886 return 1; 13887} 13888 13889static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) 13890{ 13891 if (opline->op1_type == IS_CONST) { 13892 zval *zv; 13893 size_t len; 13894 13895 zv = RT_CONSTANT(opline, opline->op1); 13896 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 13897 len = Z_STRLEN_P(zv); 13898 13899 | SET_ZVAL_LVAL res_addr, len, TMP1, TMP2 13900 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 13901 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13902 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13903 return 0; 13904 } 13905 } else { 13906 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 13907 13908 if (Z_MODE(res_addr) == IS_REG) { 13909 | GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1 13910 | ldr Rx(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(zend_string, len)] 13911 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13912 return 0; 13913 } 13914 } else { 13915 | GET_ZVAL_PTR REG0, op1_addr, TMP1 13916 | ldr REG0, [REG0, #offsetof(zend_string, len)] 13917 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 13918 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13919 } 13920 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13921 } 13922 return 1; 13923} 13924 13925static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) 13926{ 13927 if (opline->op1_type == IS_CONST) { 13928 zval *zv; 13929 zend_long count; 13930 13931 zv = RT_CONSTANT(opline, opline->op1); 13932 ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); 13933 count = zend_hash_num_elements(Z_ARRVAL_P(zv)); 13934 13935 | SET_ZVAL_LVAL res_addr, count, TMP1, TMP2 13936 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 13937 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13938 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13939 return 0; 13940 } 13941 } else { 13942 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY); 13943 // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+. 13944 13945 if (Z_MODE(res_addr) == IS_REG) { 13946 | GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1 13947 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 13948 | ldr Rw(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(HashTable, nNumOfElements)] 13949 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13950 return 0; 13951 } 13952 } else { 13953 | GET_ZVAL_PTR REG0, op1_addr, TMP1 13954 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 13955 | ldr REG0w, [REG0, #offsetof(HashTable, nNumOfElements)] 13956 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 13957 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13958 } 13959 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13960 } 13961 13962 if (may_throw) { 13963 return zend_jit_check_exception(Dst); 13964 } 13965 return 1; 13966} 13967 13968static int zend_jit_load_this(dasm_State **Dst, uint32_t var) 13969{ 13970 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 13971 13972 | ldr FCARG1x, EX->This.value.ptr 13973 | SET_ZVAL_PTR var_addr, FCARG1x, TMP1 13974 | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX, TMP1w, TMP2 13975 | GC_ADDREF FCARG1x, TMP1w 13976 return 1; 13977} 13978 13979static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only) 13980{ 13981 if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) { 13982 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13983 if (!JIT_G(current_frame) || 13984 !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { 13985 13986 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13987 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13988 13989 if (!exit_addr) { 13990 return 0; 13991 } 13992 13993 | ldrb TMP1w, EX->This.u1.v.type 13994 | cmp TMP1w, #IS_OBJECT 13995 | bne &exit_addr 13996 13997 if (JIT_G(current_frame)) { 13998 TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); 13999 } 14000 } 14001 } else { 14002 14003 | ldrb TMP1w, EX->This.u1.v.type 14004 | cmp TMP1w, #IS_OBJECT 14005 | bne >1 14006 |.cold_code 14007 |1: 14008 | SET_EX_OPLINE opline, REG0 14009 | b ->invalid_this 14010 |.code 14011 } 14012 } 14013 14014 if (!check_only) { 14015 if (!zend_jit_load_this(Dst, opline->result.var)) { 14016 return 0; 14017 } 14018 } 14019 return 1; 14020} 14021 14022static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) 14023{ 14024 uint32_t count; 14025 Bucket *p; 14026 const zend_op *target; 14027 int b; 14028 int32_t exit_point; 14029 const void *exit_addr; 14030 14031 if (default_label) { 14032 | cbz REG0, &default_label 14033 } else if (next_opline) { 14034 | cbz REG0, >3 14035 } else { 14036 | cbz REG0, =>default_b 14037 } 14038 | LOAD_ADDR FCARG1x, jumptable 14039 | ldr TMP1, [FCARG1x, #offsetof(HashTable, arData)] 14040 | sub REG0, REG0, TMP1 14041 | mov FCARG1x, #(sizeof(Bucket) / sizeof(void*)) 14042 | sdiv REG0, REG0, FCARG1x 14043 | adr FCARG1x, >4 14044 | ldr TMP1, [FCARG1x, REG0] 14045 | br TMP1 14046 14047 |.jmp_table 14048 |.align 8 14049 |4: 14050 if (trace_info) { 14051 trace_info->jmp_table_size += zend_hash_num_elements(jumptable); 14052 } 14053 14054 count = jumptable->nNumUsed; 14055 p = jumptable->arData; 14056 do { 14057 if (Z_TYPE(p->val) == IS_UNDEF) { 14058 if (default_label) { 14059 | .addr &default_label 14060 } else if (next_opline) { 14061 | .addr >3 14062 } else { 14063 | .addr =>default_b 14064 } 14065 } else { 14066 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 14067 if (!next_opline) { 14068 b = ssa->cfg.map[target - op_array->opcodes]; 14069 | .addr =>b 14070 } else if (next_opline == target) { 14071 | .addr >3 14072 } else { 14073 exit_point = zend_jit_trace_get_exit_point(target, 0); 14074 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14075 if (!exit_addr) { 14076 return 0; 14077 } 14078 | .addr &exit_addr 14079 } 14080 } 14081 p++; 14082 count--; 14083 } while (count); 14084 |.code 14085 14086 return 1; 14087} 14088 14089static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) 14090{ 14091 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 14092 const zend_op *next_opline = NULL; 14093 14094 if (trace) { 14095 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 14096 ZEND_ASSERT(trace->opline != NULL); 14097 next_opline = trace->opline; 14098 } 14099 14100 if (opline->op1_type == IS_CONST) { 14101 zval *zv = RT_CONSTANT(opline, opline->op1); 14102 zval *jump_zv = NULL; 14103 int b; 14104 14105 if (opline->opcode == ZEND_SWITCH_LONG) { 14106 if (Z_TYPE_P(zv) == IS_LONG) { 14107 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14108 } 14109 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14110 if (Z_TYPE_P(zv) == IS_STRING) { 14111 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 14112 } 14113 } else if (opline->opcode == ZEND_MATCH) { 14114 if (Z_TYPE_P(zv) == IS_LONG) { 14115 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14116 } else if (Z_TYPE_P(zv) == IS_STRING) { 14117 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 14118 } 14119 } else { 14120 ZEND_UNREACHABLE(); 14121 } 14122 if (next_opline) { 14123 const zend_op *target; 14124 14125 if (jump_zv != NULL) { 14126 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); 14127 } else { 14128 target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14129 } 14130 ZEND_ASSERT(target == next_opline); 14131 } else { 14132 if (jump_zv != NULL) { 14133 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; 14134 } else { 14135 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; 14136 } 14137 | b =>b 14138 } 14139 } else { 14140 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; 14141 uint32_t op1_info = OP1_INFO(); 14142 zend_jit_addr op1_addr = OP1_ADDR(); 14143 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14144 const zend_op *target; 14145 int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; 14146 int b; 14147 int32_t exit_point; 14148 const void *fallback_label = NULL; 14149 const void *default_label = NULL; 14150 const void *exit_addr; 14151 14152 if (next_opline) { 14153 if (next_opline != opline + 1) { 14154 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 14155 fallback_label = zend_jit_trace_get_exit_addr(exit_point); 14156 if (!fallback_label) { 14157 return 0; 14158 } 14159 } 14160 if (next_opline != default_opline) { 14161 exit_point = zend_jit_trace_get_exit_point(default_opline, 0); 14162 default_label = zend_jit_trace_get_exit_addr(exit_point); 14163 if (!default_label) { 14164 return 0; 14165 } 14166 } 14167 } 14168 14169 if (opline->opcode == ZEND_SWITCH_LONG) { 14170 if (op1_info & MAY_BE_LONG) { 14171 if (op1_info & MAY_BE_REF) { 14172 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1, ZREG_TMP1 14173 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 14174 |.cold_code 14175 |1: 14176 | // ZVAL_DEREF(op) 14177 if (fallback_label) { 14178 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 14179 } else { 14180 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 14181 } 14182 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14183 if (fallback_label) { 14184 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14185 | IF_NOT_Z_TYPE TMP1, IS_LONG, &fallback_label, TMP2w 14186 } else { 14187 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14188 | IF_NOT_Z_TYPE TMP1, IS_LONG, >3, TMP2w 14189 } 14190 | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.lval)] 14191 | b >2 14192 |.code 14193 |2: 14194 } else { 14195 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14196 if (fallback_label) { 14197 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label, ZREG_TMP1 14198 } else { 14199 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 14200 } 14201 } 14202 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 14203 } 14204 if (HT_IS_PACKED(jumptable)) { 14205 uint32_t count = jumptable->nNumUsed; 14206 Bucket *p = jumptable->arData; 14207 14208 | CMP_64_WITH_CONST_32 FCARG2x, jumptable->nNumUsed, TMP1 14209 if (default_label) { 14210 | bhs &default_label 14211 } else if (next_opline) { 14212 | bhs >3 14213 } else { 14214 | bhs =>default_b 14215 } 14216 | adr REG0, >4 14217 | ldr TMP1, [REG0, FCARG2x, lsl #3] 14218 | br TMP1 14219 14220 |.jmp_table 14221 |.align 8 14222 |4: 14223 if (trace_info) { 14224 trace_info->jmp_table_size += count; 14225 } 14226 p = jumptable->arData; 14227 do { 14228 if (Z_TYPE(p->val) == IS_UNDEF) { 14229 if (default_label) { 14230 | .addr &default_label 14231 } else if (next_opline) { 14232 | .addr >3 14233 } else { 14234 | .addr =>default_b 14235 } 14236 } else { 14237 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 14238 if (!next_opline) { 14239 b = ssa->cfg.map[target - op_array->opcodes]; 14240 | .addr =>b 14241 } else if (next_opline == target) { 14242 | .addr >3 14243 } else { 14244 exit_point = zend_jit_trace_get_exit_point(target, 0); 14245 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14246 if (!exit_addr) { 14247 return 0; 14248 } 14249 | .addr &exit_addr 14250 } 14251 } 14252 p++; 14253 count--; 14254 } while (count); 14255 |.code 14256 |3: 14257 } else { 14258 | LOAD_ADDR FCARG1x, jumptable 14259 | EXT_CALL zend_hash_index_find, REG0 14260 | mov REG0, RETVALx 14261 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14262 return 0; 14263 } 14264 |3: 14265 } 14266 } 14267 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14268 if (op1_info & MAY_BE_STRING) { 14269 if (op1_info & MAY_BE_REF) { 14270 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1, ZREG_TMP1 14271 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14272 |.cold_code 14273 |1: 14274 | // ZVAL_DEREF(op) 14275 if (fallback_label) { 14276 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 14277 } else { 14278 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 14279 } 14280 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14281 if (fallback_label) { 14282 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14283 | IF_NOT_Z_TYPE TMP1, IS_STRING, &fallback_label, TMP2w 14284 } else { 14285 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14286 | IF_NOT_Z_TYPE TMP1, IS_STRING, >3, TMP2w 14287 } 14288 | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.ptr)] 14289 | b >2 14290 |.code 14291 |2: 14292 } else { 14293 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { 14294 if (fallback_label) { 14295 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label, ZREG_TMP1 14296 } else { 14297 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 14298 } 14299 } 14300 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14301 } 14302 | LOAD_ADDR FCARG1x, jumptable 14303 | EXT_CALL zend_hash_find, REG0 14304 | mov REG0, RETVALx 14305 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14306 return 0; 14307 } 14308 |3: 14309 } 14310 } else if (opline->opcode == ZEND_MATCH) { 14311 if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { 14312 if (op1_info & MAY_BE_REF) { 14313 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 14314 | ZVAL_DEREF FCARG2x, op1_info, TMP1w 14315 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 14316 } 14317 | LOAD_ADDR FCARG1x, jumptable 14318 if (op1_info & MAY_BE_LONG) { 14319 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14320 if (op1_info & MAY_BE_STRING) { 14321 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5, ZREG_TMP1 14322 } else if (op1_info & MAY_BE_UNDEF) { 14323 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 14324 } else if (default_label) { 14325 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label, ZREG_TMP1 14326 } else if (next_opline) { 14327 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 14328 } else { 14329 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b, ZREG_TMP1 14330 } 14331 } 14332 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 14333 | EXT_CALL zend_hash_index_find, REG0 14334 | mov REG0, RETVALx 14335 if (op1_info & MAY_BE_STRING) { 14336 | b >2 14337 } 14338 } 14339 if (op1_info & MAY_BE_STRING) { 14340 |5: 14341 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { 14342 if (op1_info & MAY_BE_UNDEF) { 14343 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 14344 } else if (default_label) { 14345 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label, ZREG_TMP1 14346 } else if (next_opline) { 14347 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 14348 } else { 14349 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b, ZREG_TMP1 14350 } 14351 } 14352 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14353 | EXT_CALL zend_hash_find, REG0 14354 | mov REG0, RETVALx 14355 } 14356 |2: 14357 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14358 return 0; 14359 } 14360 } 14361 if (op1_info & MAY_BE_UNDEF) { 14362 |6: 14363 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { 14364 if (default_label) { 14365 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label, ZREG_TMP1 14366 } else if (next_opline) { 14367 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3, ZREG_TMP1 14368 } else { 14369 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b, ZREG_TMP1 14370 } 14371 } 14372 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 14373 | SET_EX_OPLINE opline, REG0 14374 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 14375 | EXT_CALL zend_jit_undefined_op_helper, REG0 14376 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 14377 return 0; 14378 } 14379 } 14380 if (default_label) { 14381 | b &default_label 14382 } else if (next_opline) { 14383 | b >3 14384 } else { 14385 | b =>default_b 14386 } 14387 |3: 14388 } else { 14389 ZEND_UNREACHABLE(); 14390 } 14391 } 14392 return 1; 14393} 14394 14395static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) 14396{ 14397 zend_arg_info *arg_info = &op_array->arg_info[-1]; 14398 ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); 14399 zend_jit_addr op1_addr = OP1_ADDR(); 14400 bool needs_slow_check = 1; 14401 bool slow_check_in_cold = 1; 14402 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 14403 14404 if (type_mask == 0) { 14405 slow_check_in_cold = 0; 14406 } else { 14407 if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { 14408 slow_check_in_cold = 0; 14409 } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { 14410 needs_slow_check = 0; 14411 } else if (is_power_of_two(type_mask)) { 14412 uint32_t type_code = concrete_type(type_mask); 14413 | IF_NOT_ZVAL_TYPE op1_addr, type_code, >6, ZREG_TMP1 14414 } else { 14415 | mov REG2w, #1 14416 | GET_ZVAL_TYPE REG1w, op1_addr, TMP1 14417 | lsl REG2w, REG2w, REG1w 14418 | TST_32_WITH_CONST REG2w, type_mask, TMP1w 14419 | beq >6 14420 } 14421 } 14422 if (needs_slow_check) { 14423 if (slow_check_in_cold) { 14424 |.cold_code 14425 |6: 14426 } 14427 | SET_EX_OPLINE opline, REG1 14428 if (op1_info & MAY_BE_UNDEF) { 14429 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7, ZREG_TMP1 14430 | LOAD_32BIT_VAL FCARG1x, opline->op1.var 14431 | EXT_CALL zend_jit_undefined_op_helper, REG0 14432 | cbz RETVALx, ->exception_handler 14433 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 14434 | b >8 14435 } 14436 |7: 14437 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 14438 |8: 14439 | ldr FCARG2x, EX->func 14440 | LOAD_ADDR CARG3, (ptrdiff_t)arg_info 14441 | ldr REG0, EX->run_time_cache 14442 | ADD_SUB_64_WITH_CONST_32 add, CARG4, REG0, opline->op2.num, TMP1 14443 | EXT_CALL zend_jit_verify_return_slow, REG0 14444 if (!zend_jit_check_exception(Dst)) { 14445 return 0; 14446 } 14447 if (slow_check_in_cold) { 14448 | b >9 14449 |.code 14450 } 14451 } 14452 |9: 14453 return 1; 14454} 14455 14456static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 14457{ 14458 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14459 14460 // TODO: support for empty() ??? 14461 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 14462 14463 if (op1_info & MAY_BE_REF) { 14464 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14465 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 14466 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14467 } 14468 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 14469 |1: 14470 } 14471 14472 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { 14473 if (exit_addr) { 14474 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); 14475 } else if (smart_branch_opcode) { 14476 if (smart_branch_opcode == ZEND_JMPNZ) { 14477 | b =>target_label 14478 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 14479 | b =>target_label2 14480 } 14481 } else { 14482 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 14483 } 14484 } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { 14485 if (exit_addr) { 14486 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); 14487 } else if (smart_branch_opcode) { 14488 if (smart_branch_opcode != ZEND_JMPNZ) { 14489 | b =>target_label 14490 } 14491 } else { 14492 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 14493 } 14494 } else { 14495 ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); 14496 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, Rx(Z_REG(op1_addr)), (Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)), TMP1 14497 | cmp TMP1w, #IS_NULL 14498 if (exit_addr) { 14499 if (smart_branch_opcode == ZEND_JMPNZ) { 14500 | bgt &exit_addr 14501 } else { 14502 | ble &exit_addr 14503 } 14504 } else if (smart_branch_opcode) { 14505 if (smart_branch_opcode == ZEND_JMPZ) { 14506 | ble =>target_label 14507 } else if (smart_branch_opcode == ZEND_JMPNZ) { 14508 | bgt =>target_label 14509 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 14510 | ble =>target_label 14511 | b =>target_label2 14512 } else { 14513 ZEND_UNREACHABLE(); 14514 } 14515 } else { 14516 | cset REG0w, gt 14517 | add REG0w, REG0w, #IS_FALSE 14518 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 14519 } 14520 } 14521 14522 return 1; 14523} 14524 14525static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 14526{ 14527 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14528 14529 if (opline->op1_type == IS_CONST) { 14530 zval *zv = RT_CONSTANT(opline, opline->op1); 14531 14532 | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 14533 if (Z_REFCOUNTED_P(zv)) { 14534 | ADDREF_CONST zv, REG0, TMP1 14535 } 14536 } else { 14537 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 14538 14539 | // ZVAL_COPY(res, value); 14540 | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 14541 if (opline->op1_type == IS_CV) { 14542 | TRY_ADDREF op1_info, REG0w, FCARG1x, TMP1w 14543 } 14544 } 14545 | // Z_FE_POS_P(res) = 0; 14546 | MEM_ACCESS_32_WITH_UOFFSET str, wzr, FP, (opline->result.var + offsetof(zval, u2.fe_pos)), TMP1 14547 14548 return 1; 14549} 14550 14551static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr) 14552{ 14553 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 14554 14555 if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) { 14556 /* empty array */ 14557 if (exit_addr) { 14558 if (exit_opcode == ZEND_JMP) { 14559 | b &exit_addr 14560 } 14561 } else { 14562 | b =>target_label 14563 } 14564 return 1; 14565 } 14566 14567 | // array = EX_VAR(opline->op1.var); 14568 | // fe_ht = Z_ARRVAL_P(array); 14569 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 14570 | // pos = Z_FE_POS_P(array); 14571 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 14572 | // p = fe_ht->arData + pos; 14573 || ZEND_ASSERT(sizeof(Bucket) == 32); 14574 | mov FCARG2w, REG0w 14575 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] 14576 | add FCARG2x, TMP1, FCARG2x, lsl #5 14577 |1: 14578 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 14579 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, nNumUsed)] 14580 | cmp TMP1w, REG0w 14581 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 14582 | // ZEND_VM_CONTINUE(); 14583 if (exit_addr) { 14584 if (exit_opcode == ZEND_JMP) { 14585 | bls &exit_addr 14586 } else { 14587 | bls >3 14588 } 14589 } else { 14590 | bls =>target_label 14591 } 14592 | // pos++; 14593 | add REG0w, REG0w, #1 14594 | // value_type = Z_TYPE_INFO_P(value); 14595 | // if (EXPECTED(value_type != IS_UNDEF)) { 14596 if (!exit_addr || exit_opcode == ZEND_JMP) { 14597 | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >3, TMP1w 14598 } else { 14599 | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr, TMP1w 14600 } 14601 | // p++; 14602 | add FCARG2x, FCARG2x, #sizeof(Bucket) 14603 | b <1 14604 |3: 14605 14606 if (!exit_addr || exit_opcode == ZEND_JMP) { 14607 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 14608 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 14609 uint32_t val_info; 14610 14611 | // Z_FE_POS_P(array) = pos + 1; 14612 | MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 14613 14614 if (RETURN_VALUE_USED(opline)) { 14615 zend_jit_addr res_addr = RES_ADDR(); 14616 14617 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) 14618 && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { 14619 | // if (!p->key) { 14620 | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] 14621 | cbz REG0, >2 14622 } 14623 if (op1_info & MAY_BE_ARRAY_KEY_STRING) { 14624 | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); 14625 | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] 14626 | SET_ZVAL_PTR res_addr, REG0, TMP1 14627 | ldr TMP1w, [REG0, #offsetof(zend_refcounted, gc.u.type_info)] 14628 | TST_32_WITH_CONST TMP1w, IS_STR_INTERNED, TMP2w 14629 | beq >1 14630 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 14631 | b >3 14632 |1: 14633 | GC_ADDREF REG0, TMP1w 14634 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2 14635 14636 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 14637 | b >3 14638 |2: 14639 } 14640 } 14641 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 14642 | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); 14643 | ldr REG0, [FCARG2x, #offsetof(Bucket, h)] 14644 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 14645 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 14646 } 14647 |3: 14648 } 14649 14650 val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); 14651 if (val_info & MAY_BE_ARRAY) { 14652 val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 14653 } 14654 if (op1_info & MAY_BE_ARRAY_OF_REF) { 14655 val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | 14656 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 14657 } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14658 val_info |= MAY_BE_RC1 | MAY_BE_RCN; 14659 } 14660 14661 if (opline->op2_type == IS_CV) { 14662 | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); 14663 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { 14664 return 0; 14665 } 14666 } else { 14667 | // ZVAL_COPY(res, value); 14668 | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 14669 | TRY_ADDREF val_info, REG0w, FCARG1x, TMP1w 14670 } 14671 } 14672 14673 return 1; 14674} 14675 14676static int zend_jit_fetch_constant(dasm_State **Dst, 14677 const zend_op *opline, 14678 const zend_op_array *op_array, 14679 zend_ssa *ssa, 14680 const zend_ssa_op *ssa_op, 14681 zend_jit_addr res_addr) 14682{ 14683 zval *zv = RT_CONSTANT(opline, opline->op2) + 1; 14684 zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 14685 uint32_t res_info = RES_INFO(); 14686 14687 | // c = CACHED_PTR(opline->extended_value); 14688 | ldr FCARG1x, EX->run_time_cache 14689 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG1x, opline->extended_value, TMP1 14690 | // if (c != NULL) 14691 | cbz REG0, >9 14692 if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) { 14693 | // if (!IS_SPECIAL_CACHE_VAL(c)) 14694 || ZEND_ASSERT(CACHE_SPECIAL == 1); 14695 | TST_64_WITH_ONE REG0 14696 | bne >9 14697 } 14698 |8: 14699 14700 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { 14701 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 14702 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 14703 int32_t exit_point; 14704 const void *exit_addr = NULL; 14705 14706 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 14707 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 14708 exit_point = zend_jit_trace_get_exit_point(opline+1, 0); 14709 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 14710 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14711 if (!exit_addr) { 14712 return 0; 14713 } 14714 res_info &= ~MAY_BE_GUARD; 14715 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 14716 14717 uint32_t type = concrete_type(res_info); 14718 14719 if (type < IS_STRING) { 14720 | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr, ZREG_TMP1 14721 } else { 14722 | GET_ZVAL_TYPE_INFO REG2w, const_addr, TMP1 14723 | IF_NOT_TYPE REG2w, type, &exit_addr 14724 } 14725 | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 14726 if (type < IS_STRING) { 14727 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 14728 | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 14729 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 14730 return 0; 14731 } 14732 } else { 14733 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 14734 | TRY_ADDREF res_info, REG2w, REG1, TMP1w 14735 } 14736 } else { 14737 | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 14738 | TRY_ADDREF MAY_BE_ANY, REG0w, REG1, TMP1w 14739 } 14740 14741 |.cold_code 14742 |9: 14743 | // SAVE_OPLINE(); 14744 | SET_EX_OPLINE opline, REG0 14745 | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); 14746 | LOAD_ADDR FCARG1x, zv 14747 | LOAD_32BIT_VAL FCARG2w, opline->op1.num 14748 | EXT_CALL zend_jit_get_constant, REG0 14749 | mov REG0, RETVALx 14750 | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); 14751 | cbnz REG0, <8 14752 | b ->exception_handler 14753 |.code 14754 return 1; 14755} 14756 14757static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 14758{ 14759 HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 14760 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14761 14762 ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); 14763 ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); 14764 14765 | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); 14766 | LOAD_ADDR FCARG1x, ht 14767 if (opline->op1_type != IS_CONST) { 14768 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14769 | EXT_CALL zend_hash_find, REG0 14770 } else { 14771 zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); 14772 | LOAD_ADDR FCARG2x, str 14773 | EXT_CALL zend_hash_find_known_hash, REG0 14774 } 14775 if (exit_addr) { 14776 if (smart_branch_opcode == ZEND_JMPZ) { 14777 | cbz RETVALx, &exit_addr 14778 } else { 14779 | cbnz RETVALx, &exit_addr 14780 } 14781 } else if (smart_branch_opcode) { 14782 if (smart_branch_opcode == ZEND_JMPZ) { 14783 | cbz RETVALx, =>target_label 14784 } else if (smart_branch_opcode == ZEND_JMPNZ) { 14785 | cbnz RETVALx, =>target_label 14786 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 14787 | cbz RETVALx, =>target_label 14788 | b =>target_label2 14789 } else { 14790 ZEND_UNREACHABLE(); 14791 } 14792 } else { 14793 | tst RETVALx, RETVALx 14794 | cset REG0w, ne 14795 | add REG0w, REG0w, #IS_FALSE 14796 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 14797 } 14798 14799 return 1; 14800} 14801 14802static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info) 14803{ 14804 uint32_t offset; 14805 14806 offset = (opline->opcode == ZEND_ROPE_INIT) ? 14807 opline->result.var : 14808 opline->op1.var + opline->extended_value * sizeof(zend_string*); 14809 14810 if (opline->op2_type == IS_CONST) { 14811 zval *zv = RT_CONSTANT(opline, opline->op2); 14812 zend_string *str; 14813 14814 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14815 str = Z_STR_P(zv); 14816 | LOAD_ADDR REG0, str 14817 | MEM_ACCESS_64_WITH_UOFFSET str, REG0, FP, offset, TMP1 14818 } else { 14819 zend_jit_addr op2_addr = OP2_ADDR(); 14820 14821 ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14822 14823 | GET_ZVAL_PTR REG1, op2_addr, TMP1 14824 | MEM_ACCESS_64_WITH_UOFFSET str, REG1, FP, offset, TMP1 14825 if (opline->op2_type == IS_CV) { 14826 | GET_ZVAL_TYPE_INFO REG0w, op2_addr, TMP1 14827 | TRY_ADDREF op2_info, REG0w, REG1, TMP1w 14828 } 14829 } 14830 14831 if (opline->opcode == ZEND_ROPE_END) { 14832 zend_jit_addr res_addr = RES_ADDR(); 14833 14834 | ADD_SUB_64_WITH_CONST add, FCARG1x, FP, opline->op1.var, TMP1 14835 | LOAD_32BIT_VAL FCARG2w, opline->extended_value 14836 | EXT_CALL zend_jit_rope_end, TMP1 14837 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 14838 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2 14839 } 14840 14841 return 1; 14842} 14843 14844static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) 14845{ 14846 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14847 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14848 14849 if (!exit_addr) { 14850 return 0; 14851 } 14852 | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 14853 14854 return 1; 14855} 14856 14857static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard) 14858{ 14859 zend_jit_addr var_addr = *var_addr_ptr; 14860 uint32_t var_info = *var_info_ptr; 14861 const void *exit_addr = NULL; 14862 14863 if (add_ref_guard || add_type_guard) { 14864 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14865 14866 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14867 if (!exit_addr) { 14868 return 0; 14869 } 14870 } 14871 14872 if (add_ref_guard) { 14873 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 14874 } 14875 if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { 14876 /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ 14877 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 14878 | LOAD_ZVAL_ADDR FCARG1x, var_addr 14879 } 14880 | EXT_CALL zend_jit_unref_helper, REG0 14881 } else { 14882 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 14883 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 14884 *var_addr_ptr = var_addr; 14885 } 14886 14887 if (var_type != IS_UNKNOWN) { 14888 var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); 14889 } 14890 if (add_type_guard 14891 && var_type != IS_UNKNOWN 14892 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 14893 | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr, ZREG_TMP1 14894 14895 ZEND_ASSERT(var_info & (1 << var_type)); 14896 if (var_type < IS_STRING) { 14897 var_info = (1 << var_type); 14898 } else if (var_type != IS_ARRAY) { 14899 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 14900 } else { 14901 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 14902 } 14903 14904 *var_info_ptr = var_info; 14905 } else { 14906 var_info &= ~MAY_BE_REF; 14907 *var_info_ptr = var_info; 14908 } 14909 *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */ 14910 14911 return 1; 14912} 14913 14914static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard) 14915{ 14916 zend_jit_addr var_addr = *var_addr_ptr; 14917 uint32_t var_info = *var_info_ptr; 14918 int32_t exit_point; 14919 const void *exit_addr; 14920 14921 if (add_indirect_guard) { 14922 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14923 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14924 14925 if (!exit_addr) { 14926 return 0; 14927 } 14928 | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr, ZREG_TMP1 14929 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 14930 } else { 14931 /* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */ 14932 if (opline->op1_type != IS_VAR || 14933 (opline-1)->result_type != IS_VAR || 14934 (opline-1)->result.var != opline->op1.var || 14935 (opline-1)->op1_type == IS_VAR || 14936 (opline-1)->op2_type == IS_VAR || 14937 (opline-1)->op2_type == IS_TMP_VAR) { 14938 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 14939 } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { 14940 | mov FCARG1x, REG0 14941 } 14942 } 14943 *var_info_ptr &= ~MAY_BE_INDIRECT; 14944 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14945 *var_addr_ptr = var_addr; 14946 14947 if (var_type != IS_UNKNOWN) { 14948 var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); 14949 } 14950 if (!(var_type & IS_TRACE_REFERENCE) 14951 && var_type != IS_UNKNOWN 14952 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 14953 exit_point = zend_jit_trace_get_exit_point(opline, 0); 14954 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14955 14956 if (!exit_addr) { 14957 return 0; 14958 } 14959 14960 | IF_NOT_Z_TYPE FCARG1x, var_type, &exit_addr, TMP1w 14961 14962 //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); 14963 ZEND_ASSERT(var_info & (1 << var_type)); 14964 if (var_type < IS_STRING) { 14965 var_info = (1 << var_type); 14966 } else if (var_type != IS_ARRAY) { 14967 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 14968 } else { 14969 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 14970 } 14971 14972 *var_info_ptr = var_info; 14973 } 14974 14975 return 1; 14976} 14977 14978static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) 14979{ 14980 if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { 14981 return 0; 14982 } 14983 14984 switch (opline->opcode) { 14985 case ZEND_QM_ASSIGN: 14986 case ZEND_SEND_VAR: 14987 case ZEND_ASSIGN: 14988 case ZEND_PRE_INC: 14989 case ZEND_PRE_DEC: 14990 case ZEND_POST_INC: 14991 case ZEND_POST_DEC: 14992 return 1; 14993 case ZEND_ADD: 14994 case ZEND_SUB: 14995 case ZEND_MUL: 14996 case ZEND_BW_OR: 14997 case ZEND_BW_AND: 14998 case ZEND_BW_XOR: 14999 case ZEND_SL: 15000 case ZEND_SR: 15001 if (def_var == ssa_op->result_def && 15002 use_var == ssa_op->op1_use) { 15003 return 1; 15004 } 15005 break; 15006 default: 15007 break; 15008 } 15009 return 0; 15010} 15011 15012static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) 15013{ 15014 uint32_t op1_info, op2_info; 15015 15016 switch (opline->opcode) { 15017 case ZEND_SEND_VAR: 15018 case ZEND_SEND_VAL: 15019 case ZEND_SEND_VAL_EX: 15020 return (opline->op2_type != IS_CONST); 15021 case ZEND_QM_ASSIGN: 15022 case ZEND_IS_SMALLER: 15023 case ZEND_IS_SMALLER_OR_EQUAL: 15024 case ZEND_IS_EQUAL: 15025 case ZEND_IS_NOT_EQUAL: 15026 case ZEND_IS_IDENTICAL: 15027 case ZEND_IS_NOT_IDENTICAL: 15028 case ZEND_CASE: 15029 return 1; 15030 case ZEND_RETURN: 15031 return (op_array->type != ZEND_EVAL_CODE && op_array->function_name); 15032 case ZEND_ASSIGN: 15033 op1_info = OP1_INFO(); 15034 op2_info = OP2_INFO(); 15035 return 15036 opline->op1_type == IS_CV && 15037 !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) && 15038 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); 15039 case ZEND_ADD: 15040 case ZEND_SUB: 15041 case ZEND_MUL: 15042 op1_info = OP1_INFO(); 15043 op2_info = OP2_INFO(); 15044 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); 15045 case ZEND_BW_OR: 15046 case ZEND_BW_AND: 15047 case ZEND_BW_XOR: 15048 case ZEND_SL: 15049 case ZEND_SR: 15050 case ZEND_MOD: 15051 op1_info = OP1_INFO(); 15052 op2_info = OP2_INFO(); 15053 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); 15054 case ZEND_PRE_INC: 15055 case ZEND_PRE_DEC: 15056 case ZEND_POST_INC: 15057 case ZEND_POST_DEC: 15058 op1_info = OP1_INFO(); 15059 op2_info = OP1_DEF_INFO(); 15060 return opline->op1_type == IS_CV 15061 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)) 15062 && (op2_info & MAY_BE_LONG); 15063 case ZEND_STRLEN: 15064 op1_info = OP1_INFO(); 15065 return (opline->op1_type & (IS_CV|IS_CONST)) 15066 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING; 15067 case ZEND_COUNT: 15068 op1_info = OP1_INFO(); 15069 return (opline->op1_type & (IS_CV|IS_CONST)) 15070 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY; 15071 case ZEND_JMPZ: 15072 case ZEND_JMPNZ: 15073 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 15074 if (!ssa->cfg.map) { 15075 return 0; 15076 } 15077 if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start && 15078 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { 15079 return 0; 15080 } 15081 } 15082 ZEND_FALLTHROUGH; 15083 case ZEND_BOOL: 15084 case ZEND_BOOL_NOT: 15085 case ZEND_JMPZNZ: 15086 case ZEND_JMPZ_EX: 15087 case ZEND_JMPNZ_EX: 15088 return 1; 15089 case ZEND_FETCH_CONSTANT: 15090 return 1; 15091 case ZEND_FETCH_DIM_R: 15092 op1_info = OP1_INFO(); 15093 op2_info = OP2_INFO(); 15094 if (trace 15095 && trace->op1_type != IS_UNKNOWN 15096 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { 15097 op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); 15098 } 15099 return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && 15100 (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && 15101 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || 15102 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && 15103 (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); 15104 } 15105 return 0; 15106} 15107 15108static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) 15109{ 15110 if (ssa->vars[var].no_val) { 15111 /* we don't need the value */ 15112 return 0; 15113 } 15114 15115 if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { 15116 /* Disable global register allocation, 15117 * register allocation for SSA variables connected through Phi functions 15118 */ 15119 if (ssa->vars[var].definition_phi) { 15120 return 0; 15121 } 15122 if (ssa->vars[var].phi_use_chain) { 15123 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; 15124 do { 15125 if (!ssa->vars[phi->ssa_var].no_val) { 15126 return 0; 15127 } 15128 phi = zend_ssa_next_use_phi(ssa, var, phi); 15129 } while (phi); 15130 } 15131 } 15132 15133 if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && 15134 ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { 15135 /* bad type */ 15136 return 0; 15137 } 15138 15139 return 1; 15140} 15141 15142static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) 15143{ 15144 if (!zend_jit_var_supports_reg(ssa, var)) { 15145 return 0; 15146 } 15147 15148 if (ssa->vars[var].definition >= 0) { 15149 uint32_t def = ssa->vars[var].definition; 15150 if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { 15151 return 0; 15152 } 15153 } 15154 15155 if (ssa->vars[var].use_chain >= 0) { 15156 int use = ssa->vars[var].use_chain; 15157 15158 do { 15159 if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && 15160 !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { 15161 return 0; 15162 } 15163 use = zend_ssa_next_use(ssa->ops, var, use); 15164 } while (use >= 0); 15165 } 15166 15167 return 1; 15168} 15169 15170static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 15171{ 15172 uint32_t op1_info, op2_info; 15173 15174 switch (opline->opcode) { 15175 case ZEND_FETCH_DIM_R: 15176 op1_info = OP1_INFO(); 15177 op2_info = OP2_INFO(); 15178 if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && 15179 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || 15180 ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && 15181 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { 15182 return ZEND_REGSET(ZREG_FCARG1); 15183 } 15184 break; 15185 default: 15186 break; 15187 } 15188 15189 return ZEND_REGSET_EMPTY; 15190} 15191 15192static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 15193{ 15194 uint32_t op1_info, op2_info, res_info; 15195 zend_regset regset = ZEND_REGSET_SCRATCH; 15196 15197 switch (opline->opcode) { 15198 case ZEND_NOP: 15199 case ZEND_OP_DATA: 15200 case ZEND_JMP: 15201 case ZEND_RETURN: 15202 regset = ZEND_REGSET_EMPTY; 15203 break; 15204 case ZEND_QM_ASSIGN: 15205 if (ssa_op->op1_def == current_var || 15206 ssa_op->result_def == current_var) { 15207 regset = ZEND_REGSET_EMPTY; 15208 break; 15209 } 15210 /* break missing intentionally */ 15211 case ZEND_SEND_VAL: 15212 case ZEND_SEND_VAL_EX: 15213 if (opline->op2_type == IS_CONST) { 15214 break; 15215 } 15216 if (ssa_op->op1_use == current_var) { 15217 regset = ZEND_REGSET(ZREG_REG0); 15218 break; 15219 } 15220 op1_info = OP1_INFO(); 15221 if (!(op1_info & MAY_BE_UNDEF)) { 15222 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15223 regset = ZEND_REGSET(ZREG_FPR0); 15224 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15225 regset = ZEND_REGSET(ZREG_REG0); 15226 } else { 15227 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); 15228 } 15229 } 15230 break; 15231 case ZEND_SEND_VAR: 15232 if (opline->op2_type == IS_CONST) { 15233 break; 15234 } 15235 if (ssa_op->op1_use == current_var || 15236 ssa_op->op1_def == current_var) { 15237 regset = ZEND_REGSET_EMPTY; 15238 break; 15239 } 15240 op1_info = OP1_INFO(); 15241 if (!(op1_info & MAY_BE_UNDEF)) { 15242 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15243 regset = ZEND_REGSET(ZREG_FPR0); 15244 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15245 } else { 15246 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); 15247 if (op1_info & MAY_BE_REF) { 15248 ZEND_REGSET_INCL(regset, ZREG_REG1); 15249 } 15250 } 15251 } 15252 break; 15253 case ZEND_ASSIGN: 15254 if (ssa_op->op2_use == current_var || 15255 ssa_op->op2_def == current_var || 15256 ssa_op->op1_def == current_var || 15257 ssa_op->result_def == current_var) { 15258 regset = ZEND_REGSET_EMPTY; 15259 break; 15260 } 15261 op1_info = OP1_INFO(); 15262 op2_info = OP2_INFO(); 15263 if (opline->op1_type == IS_CV 15264 && !(op2_info & MAY_BE_UNDEF) 15265 && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 15266 if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15267 regset = ZEND_REGSET(ZREG_FPR0); 15268 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15269 regset = ZEND_REGSET(ZREG_REG0); 15270 } else { 15271 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); 15272 } 15273 } 15274 break; 15275 case ZEND_PRE_INC: 15276 case ZEND_PRE_DEC: 15277 case ZEND_POST_INC: 15278 case ZEND_POST_DEC: 15279 if (ssa_op->op1_use == current_var || 15280 ssa_op->op1_def == current_var || 15281 ssa_op->result_def == current_var) { 15282 regset = ZEND_REGSET_EMPTY; 15283 break; 15284 } 15285 op1_info = OP1_INFO(); 15286 if (opline->op1_type == IS_CV 15287 && (op1_info & MAY_BE_LONG) 15288 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15289 regset = ZEND_REGSET_EMPTY; 15290 if (op1_info & MAY_BE_DOUBLE) { 15291 regset = ZEND_REGSET(ZREG_FPR0); 15292 } 15293 if (opline->result_type != IS_UNUSED && (op1_info & MAY_BE_LONG)) { 15294 ZEND_REGSET_INCL(regset, ZREG_REG1); 15295 } 15296 } 15297 break; 15298 case ZEND_ADD: 15299 case ZEND_SUB: 15300 case ZEND_MUL: 15301 op1_info = OP1_INFO(); 15302 op2_info = OP2_INFO(); 15303 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 15304 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15305 15306 regset = ZEND_REGSET_EMPTY; 15307 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 15308 if (ssa_op->result_def != current_var && 15309 (ssa_op->op1_use != current_var || !last_use)) { 15310 ZEND_REGSET_INCL(regset, ZREG_REG0); 15311 } 15312 res_info = RES_INFO(); 15313 if (res_info & MAY_BE_DOUBLE) { 15314 ZEND_REGSET_INCL(regset, ZREG_REG0); 15315 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15316 ZEND_REGSET_INCL(regset, ZREG_FPR1); 15317 } else if (res_info & MAY_BE_GUARD) { 15318 ZEND_REGSET_INCL(regset, ZREG_REG0); 15319 } 15320 } 15321 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 15322 if (ssa_op->result_def != current_var) { 15323 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15324 } 15325 } 15326 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 15327 if (zend_is_commutative(opline->opcode)) { 15328 if (ssa_op->result_def != current_var) { 15329 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15330 } 15331 } else { 15332 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15333 if (ssa_op->result_def != current_var && 15334 (ssa_op->op1_use != current_var || !last_use)) { 15335 ZEND_REGSET_INCL(regset, ZREG_FPR1); 15336 } 15337 } 15338 } 15339 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 15340 if (ssa_op->result_def != current_var && 15341 (ssa_op->op1_use != current_var || !last_use) && 15342 (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { 15343 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15344 } 15345 } 15346 } 15347 break; 15348 case ZEND_BW_OR: 15349 case ZEND_BW_AND: 15350 case ZEND_BW_XOR: 15351 case ZEND_SL: 15352 case ZEND_SR: 15353 case ZEND_MOD: 15354 op1_info = OP1_INFO(); 15355 op2_info = OP2_INFO(); 15356 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 15357 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 15358 regset = ZEND_REGSET_EMPTY; 15359 if (ssa_op->result_def != current_var && 15360 (ssa_op->op1_use != current_var || !last_use)) { 15361 ZEND_REGSET_INCL(regset, ZREG_REG0); 15362 } 15363 } 15364 break; 15365 case ZEND_IS_SMALLER: 15366 case ZEND_IS_SMALLER_OR_EQUAL: 15367 case ZEND_IS_EQUAL: 15368 case ZEND_IS_NOT_EQUAL: 15369 case ZEND_IS_IDENTICAL: 15370 case ZEND_IS_NOT_IDENTICAL: 15371 case ZEND_CASE: 15372 op1_info = OP1_INFO(); 15373 op2_info = OP2_INFO(); 15374 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 15375 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15376 regset = ZEND_REGSET_EMPTY; 15377 if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { 15378 ZEND_REGSET_INCL(regset, ZREG_REG0); 15379 } 15380 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && 15381 opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { 15382 if (ssa_op->op1_use != current_var && 15383 ssa_op->op2_use != current_var) { 15384 ZEND_REGSET_INCL(regset, ZREG_REG0); 15385 } 15386 } 15387 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 15388 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15389 } 15390 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 15391 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15392 } 15393 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 15394 if (ssa_op->op1_use != current_var && 15395 ssa_op->op2_use != current_var) { 15396 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15397 } 15398 } 15399 } 15400 break; 15401 case ZEND_BOOL: 15402 case ZEND_BOOL_NOT: 15403 case ZEND_JMPZ: 15404 case ZEND_JMPNZ: 15405 case ZEND_JMPZNZ: 15406 case ZEND_JMPZ_EX: 15407 case ZEND_JMPNZ_EX: 15408 op1_info = OP1_INFO(); 15409 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15410 regset = ZEND_REGSET_EMPTY; 15411 if (op1_info & MAY_BE_DOUBLE) { 15412 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15413 } 15414 if (opline->opcode == ZEND_BOOL || 15415 opline->opcode == ZEND_BOOL_NOT || 15416 opline->opcode == ZEND_JMPZ_EX || 15417 opline->opcode == ZEND_JMPNZ_EX) { 15418 ZEND_REGSET_INCL(regset, ZREG_REG0); 15419 } 15420 } 15421 break; 15422 case ZEND_DO_UCALL: 15423 case ZEND_DO_FCALL: 15424 case ZEND_DO_FCALL_BY_NAME: 15425 case ZEND_INCLUDE_OR_EVAL: 15426 case ZEND_GENERATOR_CREATE: 15427 case ZEND_YIELD: 15428 case ZEND_YIELD_FROM: 15429 regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); 15430 break; 15431 default: 15432 break; 15433 } 15434 15435 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 15436 if (ssa_op == ssa->ops 15437 && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL 15438 && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { 15439 ZEND_REGSET_INCL(regset, ZREG_REG0); 15440 ZEND_REGSET_INCL(regset, ZREG_REG1); 15441 } 15442 } 15443 15444 return regset; 15445} 15446 15447static size_t dasm_venners_size = 0; 15448void **dasm_labels_veneers = NULL; 15449 15450static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset) 15451{ 15452 void *veneer; 15453 ptrdiff_t na; 15454 int n, m; 15455 15456 /* try to reuse veneers for global labels */ 15457 if ((ins >> 16) == DASM_REL_LG 15458 && *(b-1) < 0 15459 && dasm_labels_veneers[-*(b-1)]) { 15460 15461 veneer = dasm_labels_veneers[-*(b-1)]; 15462 na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; 15463 n = (int)na; 15464 15465 /* check if we can jump to veneer */ 15466 if ((ptrdiff_t)n != na) { 15467 /* pass */ 15468 } else if (!(ins & 0xf800)) { /* B, BL */ 15469 if ((n & 3) == 0 && ((n+0x08000000) >> 28) == 0) { 15470 return n; 15471 } 15472 } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ 15473 if ((n & 3) == 0 && ((n+0x00100000) >> 21) == 0) { 15474 return n; 15475 } 15476 } else if ((ins & 0x3000) == 0x2000) { /* ADR */ 15477 /* pass */ 15478 } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ 15479 /* pass */ 15480 } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ 15481 if ((n & 3) == 0 && ((n+0x00008000) >> 16) == 0) { 15482 return n; 15483 } 15484 } 15485 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 15486 && (ins >> 16) == DASM_REL_A) { 15487 ptrdiff_t addr = (((ptrdiff_t)(*(b-1))) << 32) | (unsigned int)(*(b-2)); 15488 15489 if ((void*)addr >= dasm_buf && (void*)addr < dasm_end) { 15490 uint32_t exit_point = zend_jit_trace_find_exit_point((void*)addr); 15491 zend_jit_trace_info *t = zend_jit_get_current_trace_info(); 15492 15493 if (exit_point != (uint32_t)-1) { 15494 /* Use exit points table */ 15495 15496 ZEND_ASSERT(exit_point < t->exit_count); 15497 15498 veneer = (char*)buffer + dasm_getpclabel(&Dst, 1) - (t->exit_count - exit_point) * 4; 15499 na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; 15500 n = (int)na; 15501 15502 /* check if we can jump to veneer */ 15503 if ((ptrdiff_t)n != na) { 15504 ZEND_ASSERT(0); 15505 return 0; 15506 } else if (!(ins & 0xf800)) { /* B, BL */ 15507 if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) { 15508 ZEND_ASSERT(0); 15509 return 0; 15510 } 15511 } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ 15512 if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) { 15513 ZEND_ASSERT(0); 15514 return 0; 15515 } 15516 } else if ((ins & 0x3000) == 0x2000) { /* ADR */ 15517 ZEND_ASSERT(0); 15518 return 0; 15519 } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ 15520 ZEND_ASSERT(0); 15521 return 0; 15522 } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ 15523 if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) { 15524 ZEND_ASSERT(0); 15525 return 0; 15526 } 15527 } else { 15528 ZEND_ASSERT(0); 15529 return 0; 15530 } 15531 return n; 15532 } 15533 } 15534 } 15535 15536 veneer = (char*)buffer + (Dst->codesize + dasm_venners_size); 15537 15538 if (veneer > dasm_end) { 15539 return 0; /* jit_buffer_size overflow */ 15540 } 15541 15542 na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; 15543 n = (int)na; 15544 15545 /* check if we can jump to veneer */ 15546 if ((ptrdiff_t)n != na) { 15547 ZEND_ASSERT(0); 15548 return 0; 15549 } else if (!(ins & 0xf800)) { /* B, BL */ 15550 if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) { 15551 ZEND_ASSERT(0); 15552 return 0; 15553 } 15554 } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ 15555 if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) { 15556 ZEND_ASSERT(0); 15557 return 0; 15558 } 15559 } else if ((ins & 0x3000) == 0x2000) { /* ADR */ 15560 ZEND_ASSERT(0); 15561 return 0; 15562 } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ 15563 ZEND_ASSERT(0); 15564 return 0; 15565 } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ 15566 if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) { 15567 ZEND_ASSERT(0); 15568 return 0; 15569 } 15570 } else if ((ins & 0x8000)) { /* absolute */ 15571 ZEND_ASSERT(0); 15572 return 0; 15573 } else { 15574 ZEND_ASSERT(0); 15575 return 0; 15576 } 15577 15578 // TODO: support for long veneers (above 128MB) ??? 15579 15580 /* check if we can use B to jump from veneer */ 15581 na = (ptrdiff_t)cp + offset - (ptrdiff_t)veneer - 4; 15582 m = (int)na; 15583 if ((ptrdiff_t)m != na) { 15584 ZEND_ASSERT(0); 15585 return 0; 15586 } else if ((m & 3) != 0 || ((m+0x08000000) >> 28) != 0) { 15587 ZEND_ASSERT(0); 15588 return 0; 15589 } 15590 15591 /* generate B instruction */ 15592 *(uint32_t*)veneer = 0x14000000 | ((m >> 2) & 0x03ffffff); 15593 dasm_venners_size += 4; 15594 15595 if ((ins >> 16) == DASM_REL_LG 15596 && *(b-1) < 0) { 15597 /* reuse this veneer for the future jumps to global label */ 15598 dasm_labels_veneers[-*(b-1)] = veneer; 15599 /* Dst->globals[*(b-1)] = veneer; */ 15600 15601#ifdef HAVE_DISASM 15602 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { 15603 const char *name = zend_jit_disasm_find_symbol((ptrdiff_t)cp + offset - 4, (int64_t *)(&offset)); 15604 15605 if (name && !offset) { 15606 if (strstr(name, "@veneer") == NULL) { 15607 char *new_name; 15608 15609 zend_spprintf(&new_name, 0, "%s@veneer", name); 15610 zend_jit_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4); 15611 efree(new_name); 15612 } else { 15613 zend_jit_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4); 15614 } 15615 } 15616 } 15617#endif 15618 } 15619 15620 return n; 15621} 15622 15623/* 15624 * Local variables: 15625 * tab-width: 4 15626 * c-basic-offset: 4 15627 * indent-tabs-mode: t 15628 * End: 15629 */ 15630
