1/* 2 * +----------------------------------------------------------------------+ 3 * | Zend JIT | 4 * +----------------------------------------------------------------------+ 5 * | Copyright (c) The PHP Group | 6 * +----------------------------------------------------------------------+ 7 * | This source file is subject to version 3.01 of the PHP license, | 8 * | that is bundled with this package in the file LICENSE, and is | 9 * | available through the world-wide-web at the following url: | 10 * | https://www.php.net/license/3_01.txt | 11 * | If you did not receive a copy of the PHP license and are unable to | 12 * | obtain it through the world-wide-web, please send a note to | 13 * | license@php.net so we can mail you a copy immediately. | 14 * +----------------------------------------------------------------------+ 15 * | Authors: Dmitry Stogov <dmitry@php.net> | 16 * | Xinchen Hui <laruence@php.net> | 17 * | Hao Sun <hao.sun@arm.com> | 18 * +----------------------------------------------------------------------+ 19 */ 20 21|.arch arm64 22 23|.define FP, x27 24|.define IP, x28 25|.define IPl, w28 26|.define RX, x28 // the same as VM IP reused as a general purpose reg 27|.define LR, x30 28|.define CARG1, x0 29|.define CARG2, x1 30|.define CARG3, x2 31|.define CARG4, x3 32|.define CARG5, x4 33|.define CARG6, x5 34|.define CARG1w, w0 35|.define CARG2w, w1 36|.define CARG3w, w2 37|.define CARG4w, w3 38|.define CARG5w, w4 39|.define CARG6w, w5 40|.define RETVALx, x0 41|.define RETVALw, w0 42|.define FCARG1x, x0 43|.define FCARG1w, w0 44|.define FCARG2x, x1 45|.define FCARG2w, w1 46|.define SPAD, 0x20 // padding for CPU stack alignment 47|.define NR_SPAD, 0x30 // padding for CPU stack alignment 48|.define T3, [sp, #0x28] // Used to store old value of IP (CALL VM only) 49|.define T2, [sp, #0x20] // Used to store old value of FP (CALL VM only) 50|.define T1, [sp, #0x10] 51 52// We use REG0/1/2 and FPR0/1 to replace r0/1/2 and xmm0/1 in the x86 implementation. 53// Scratch registers 54|.define REG0, x8 55|.define REG0w, w8 56|.define REG1, x9 57|.define REG1w, w9 58|.define REG2, x10 59|.define REG2w, w10 60|.define FPR0, d0 61|.define FPR1, d1 62 63|.define ZREG_REG0, ZREG_X8 64|.define ZREG_REG1, ZREG_X9 65|.define ZREG_REG2, ZREG_X10 66|.define ZREG_FPR0, ZREG_V0 67|.define ZREG_FPR1, ZREG_V1 68 69// Temporaries, not preserved across calls 70|.define TMP1, x15 71|.define TMP1w, w15 72|.define TMP2, x16 73|.define TMP2w, w16 74|.define TMP3, x17 // TODO: remember about hard-coded: mrs TMP3, tpidr_el0 75|.define TMP3w, w17 76|.define FPTMP, d16 77 78|.define ZREG_TMP1, ZREG_X15 79|.define ZREG_TMP2, ZREG_X16 80|.define ZREG_TMP3, ZREG_X17 81|.define ZREG_FPTMP, ZREG_V16 82 83|.define HYBRID_SPAD, 32 // padding for stack alignment 84 85#define TMP_ZVAL_OFFSET 16 86#define DASM_ALIGNMENT 16 87 88const char* zend_reg_name[] = { 89 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", 90 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", 91 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", 92 "x24", "x25", "x26", "x27", "x28", "x29", "x30", "sp", 93 "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", 94 "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15", 95 "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23", 96 "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31" 97}; 98 99#define ZREG_FCARG1 ZREG_X0 100#define ZREG_FCARG2 ZREG_X1 101 102|.type EX, zend_execute_data, FP 103|.type OP, zend_op 104|.type ZVAL, zval 105|.actionlist dasm_actions 106|.globals zend_lb 107|.section code, cold_code, jmp_table 108 109static void* dasm_labels[zend_lb_MAX]; 110 111#if ZTS 112static size_t tsrm_ls_cache_tcb_offset = 0; 113static size_t tsrm_tls_index = 0; 114static size_t tsrm_tls_offset = 0; 115# ifdef __APPLE__ 116struct TLVDescriptor { 117 void* (*thunk)(struct TLVDescriptor*); 118 uint64_t key; 119 uint64_t offset; 120}; 121typedef struct TLVDescriptor TLVDescriptor; 122# elif defined(__FreeBSD__) 123/* https://github.com/freebsd/freebsd-src/blob/c52ca7dd09066648b1cc40f758289404d68ab886/libexec/rtld-elf/aarch64/reloc.c#L180-L184 */ 124typedef struct TLSDescriptor { 125 void* thunk; 126 int index; 127 size_t offset; 128} TLSDescriptor; 129# endif 130#endif 131 132#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) 133 134/* Encoding of immediate. */ 135#define MAX_IMM12 0xfff // maximum value for imm12 136#define MAX_IMM16 0xffff // maximum value for imm16 137#define MOVZ_IMM MAX_IMM16 // movz insn 138#define LDR_STR_PIMM64 (MAX_IMM12*8) // ldr/str insn for 64-bit register: pimm is imm12 * 8 139#define LDR_STR_PIMM32 (MAX_IMM12*4) // ldr/str insn for 32-bit register: pimm is imm12 * 4 140#define LDRB_STRB_PIMM MAX_IMM12 // ldrb/strb insn 141 142#define B_IMM (1<<27) // signed imm26 * 4 143#define ADR_IMM (1<<20) // signed imm21 144#define ADRP_IMM (1LL<<32) // signed imm21 * 4096 145 146static bool arm64_may_use_b(const void *addr) 147{ 148 if (addr >= dasm_buf && addr < dasm_end) { 149 return (((char*)dasm_end - (char*)dasm_buf) < B_IMM); 150 } else if (addr >= dasm_end) { 151 return (((char*)addr - (char*)dasm_buf) < B_IMM); 152 } else if (addr < dasm_buf) { 153 return (((char*)dasm_end - (char*)addr) < B_IMM); 154 } 155 return 0; 156} 157 158static bool arm64_may_use_adr(const void *addr) 159{ 160 if (addr >= dasm_buf && addr < dasm_end) { 161 return (((char*)dasm_end - (char*)dasm_buf) < ADR_IMM); 162 } else if (addr >= dasm_end) { 163 return (((char*)addr - (char*)dasm_buf) < ADR_IMM); 164 } else if (addr < dasm_buf) { 165 return (((char*)dasm_end - (char*)addr) < ADR_IMM); 166 } 167 return 0; 168} 169 170static bool arm64_may_use_adrp(const void *addr) 171{ 172 if (addr >= dasm_buf && addr < dasm_end) { 173 return (((char*)dasm_end - (char*)dasm_buf) < ADRP_IMM); 174 } else if (addr >= dasm_end) { 175 return (((char*)addr - (char*)dasm_buf) < ADRP_IMM); 176 } else if (addr < dasm_buf) { 177 return (((char*)dasm_end - (char*)addr) < ADRP_IMM); 178 } 179 return 0; 180} 181 182/* Determine whether "val" falls into two allowed ranges: 183 * Range 1: [0, 0xfff] 184 * Range 2: LSL #12 to Range 1 185 * Used to guard the immediate encoding for add/adds/sub/subs/cmp/cmn instructions. */ 186static bool arm64_may_encode_imm12(const int64_t val) 187{ 188 return (val >= 0 && (val <= MAX_IMM12 || !(val & 0xffffffffff000fff))); 189} 190 191/* Determine whether an immediate value can be encoded as the immediate operand of logical instructions. */ 192static bool logical_immediate_p(uint64_t value, uint32_t reg_size) 193{ 194 /* fast path: power of two */ 195 if (value > 0 && !(value & (value - 1))) { 196 return true; 197 } 198 199 if (reg_size == 32) { 200 if (dasm_imm13((uint32_t)value, (uint32_t)value) != -1) { 201 return true; 202 } 203 } else if (reg_size == 64) { 204 if (dasm_imm13((uint32_t)value, (uint32_t)(value >> 32)) != -1) { 205 return true; 206 } 207 } else { 208 ZEND_UNREACHABLE(); 209 } 210 211 return false; 212} 213 214/* Not Implemented Yet */ 215|.macro NIY 216|| //ZEND_ASSERT(0); 217| brk #0 218|.endmacro 219 220|.macro NIY_STUB 221|| //ZEND_ASSERT(0); 222| brk #0 223|.endmacro 224 225|.macro ADD_HYBRID_SPAD 226||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 227| add sp, sp, # HYBRID_SPAD 228||#endif 229|.endmacro 230 231|.macro SUB_HYBRID_SPAD 232||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 233| sub sp, sp, # HYBRID_SPAD 234||#endif 235|.endmacro 236 237/* Move address into register. TODO: Support 52-bit address */ 238|.macro LOAD_ADDR, reg, addr 239| // 48-bit virtual address 240|| if (((uintptr_t)(addr)) == 0) { 241| mov reg, xzr 242|| } else if (((uintptr_t)(addr)) <= MOVZ_IMM) { 243| movz reg, #((uint64_t)(addr)) 244|| } else if (arm64_may_use_adr((void*)(addr))) { 245| adr reg, &addr 246|| } else if (arm64_may_use_adrp((void*)(addr))) { 247| adrp reg, &(((uintptr_t)(addr))) 248|| if (((uintptr_t)(addr)) & 0xfff) { 249| add reg, reg, #(((uintptr_t)(addr)) & 0xfff) 250|| } 251|| } else if ((uintptr_t)(addr) & 0xffff) { 252| movz reg, #((uintptr_t)(addr) & 0xffff) 253|| if (((uintptr_t)(addr) >> 16) & 0xffff) { 254| movk reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 255|| } 256|| if (((uintptr_t)(addr) >> 32) & 0xffff) { 257| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 258|| } 259|| } else if (((uintptr_t)(addr) >> 16) & 0xffff) { 260| movz reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 261|| if (((uintptr_t)(addr) >> 32) & 0xffff) { 262| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 263|| } 264|| } else { 265| movz reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 266|| } 267|.endmacro 268 269/* Move 32-bit immediate value into register. */ 270|.macro LOAD_32BIT_VAL, reg, val 271|| if (((uint32_t)(val)) <= MOVZ_IMM) { 272| movz reg, #((uint32_t)(val)) 273|| } else if (((uint32_t)(val) & 0xffff)) { 274| movz reg, #((uint32_t)(val) & 0xffff) 275|| if ((((uint32_t)(val) >> 16) & 0xffff)) { 276| movk reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 277|| } 278|| } else { 279| movz reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 280|| } 281|.endmacro 282 283/* Move 64-bit immediate value into register. */ 284|.macro LOAD_64BIT_VAL, reg, val 285|| if (((uint64_t)(val)) == 0) { 286| mov reg, xzr 287|| } else if (((uint64_t)(val)) <= MOVZ_IMM) { 288| movz reg, #((uint64_t)(val)) 289|| } else if (~((uint64_t)(val)) <= MOVZ_IMM) { 290| movn reg, #(~((uint64_t)(val))) 291|| } else if ((uint64_t)(val) & 0xffff) { 292| movz reg, #((uint64_t)(val) & 0xffff) 293|| if (((uint64_t)(val) >> 16) & 0xffff) { 294| movk reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 295|| } 296|| if (((uint64_t)(val) >> 32) & 0xffff) { 297| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 298|| } 299|| if ((((uint64_t)(val) >> 48) & 0xffff)) { 300| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 301|| } 302|| } else if (((uint64_t)(val) >> 16) & 0xffff) { 303| movz reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 304|| if (((uint64_t)(val) >> 32) & 0xffff) { 305| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 306|| } 307|| if ((((uint64_t)(val) >> 48) & 0xffff)) { 308| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 309|| } 310|| } else if (((uint64_t)(val) >> 32) & 0xffff) { 311| movz reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 312|| if ((((uint64_t)(val) >> 48) & 0xffff)) { 313| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 314|| } 315|| } else { 316| movz reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 317|| } 318|.endmacro 319 320/* Extract the low 8 bits from 'src_reg' into 'dst_reg'. 321 * Note: 0xff can be encoded as imm for 'and' instruction. */ 322|.macro GET_LOW_8BITS, dst_reg, src_reg 323| and dst_reg, src_reg, #0xff 324|.endmacro 325 326/* Bitwise operation with immediate. 'bw_ins' can be and/orr/eor/ands. 327 * 32-bit and 64-bit registers are distinguished. */ 328|.macro BW_OP_32_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg 329|| if (val == 0) { 330| bw_ins dst_reg, src_reg1, wzr 331|| } else if (logical_immediate_p((uint32_t)val, 32)) { 332| bw_ins dst_reg, src_reg1, #val 333|| } else { 334| LOAD_32BIT_VAL tmp_reg, val 335| bw_ins dst_reg, src_reg1, tmp_reg 336|| } 337|.endmacro 338 339|.macro BW_OP_64_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg 340|| if (val == 0) { 341| bw_ins dst_reg, src_reg1, xzr 342|| } else if (logical_immediate_p(val, 64)) { 343| bw_ins dst_reg, src_reg1, #val 344|| } else { 345| LOAD_64BIT_VAL tmp_reg, val 346| bw_ins dst_reg, src_reg1, tmp_reg 347|| } 348|.endmacro 349 350/* Test bits 'tst' with immediate. 32-bit and 64-bit registers are distinguished. */ 351|.macro TST_32_WITH_CONST, reg, val, tmp_reg 352|| if (val == 0) { 353| tst reg, wzr 354|| } else if (logical_immediate_p((uint32_t)val, 32)) { 355| tst reg, #val 356|| } else { 357| LOAD_32BIT_VAL tmp_reg, val 358| tst reg, tmp_reg 359|| } 360|.endmacro 361 362|.macro TST_64_WITH_CONST, reg, val, tmp_reg 363|| if (val == 0) { 364| tst reg, xzr 365|| } else if (logical_immediate_p(val, 64)) { 366| tst reg, #val 367|| } else { 368| LOAD_64BIT_VAL tmp_reg, val 369| tst reg, tmp_reg 370|| } 371|.endmacro 372 373/* Test bits between 64-bit register with constant 1. */ 374|.macro TST_64_WITH_ONE, reg 375| tst reg, #1 376|.endmacro 377 378/* Compare a register value with immediate. 32-bit and 64-bit registers are distinguished. 379 * Note: Comparing 64-bit register with 32-bit immediate is handled in CMP_64_WITH_CONST_32. */ 380|.macro CMP_32_WITH_CONST, reg, val, tmp_reg 381|| if (val == 0) { 382| cmp reg, wzr 383|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 384| cmp reg, #val 385|| } else if (arm64_may_encode_imm12((int64_t)(-val))) { 386| cmn reg, #-val 387|| } else { 388| LOAD_32BIT_VAL tmp_reg, val 389| cmp reg, tmp_reg 390|| } 391|.endmacro 392 393|.macro CMP_64_WITH_CONST_32, reg, val, tmp_reg 394|| if (val == 0) { 395| cmp reg, xzr 396|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 397| cmp reg, #val 398|| } else if (arm64_may_encode_imm12((int64_t)(-val))) { 399| cmn reg, #-val 400|| } else { 401| LOAD_32BIT_VAL tmp_reg, val 402| cmp reg, tmp_reg 403|| } 404|.endmacro 405 406|.macro CMP_64_WITH_CONST, reg, val, tmp_reg 407|| if (val == 0) { 408| cmp reg, xzr 409|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 410| cmp reg, #val 411|| } else if (arm64_may_encode_imm12((int64_t)(-val))) { 412| cmn reg, #-val 413|| } else { 414| LOAD_64BIT_VAL tmp_reg, val 415| cmp reg, tmp_reg 416|| } 417|.endmacro 418 419/* Add/sub a register value with immediate. 'add_sub_ins' can be add/sub/adds/subs. 420 * 32-bit and 64-bit registers are distinguished. 421 * Note: Case of 64-bit register with 32-bit immediate is handled in ADD_SUB_64_WITH_CONST_32. */ 422|.macro ADD_SUB_32_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg 423|| if (val == 0) { 424| add_sub_ins dst_reg, src_reg1, wzr 425|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 426| add_sub_ins dst_reg, src_reg1, #val 427|| } else { 428| LOAD_32BIT_VAL tmp_reg, val 429| add_sub_ins dst_reg, src_reg1, tmp_reg 430|| } 431|.endmacro 432 433|.macro ADD_SUB_64_WITH_CONST_32, add_sub_ins, dst_reg, src_reg1, val, tmp_reg 434|| if (val == 0) { 435| add_sub_ins dst_reg, src_reg1, xzr 436|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 437| add_sub_ins dst_reg, src_reg1, #val 438|| } else { 439| LOAD_32BIT_VAL tmp_reg, val 440| add_sub_ins dst_reg, src_reg1, tmp_reg 441|| } 442|.endmacro 443 444|.macro ADD_SUB_64_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg 445|| if (val == 0) { 446| add_sub_ins dst_reg, src_reg1, xzr 447|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 448| add_sub_ins dst_reg, src_reg1, #val 449|| } else { 450| LOAD_64BIT_VAL tmp_reg, val 451| add_sub_ins dst_reg, src_reg1, tmp_reg 452|| } 453|.endmacro 454 455/* Memory access(load/store) with 32-bit 'offset'. 456 * Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register. 457 * 8-bit, 32-bit and 64-bit registers to be transferred are distinguished. 458 * Note: 'reg' can be used as 'tmp_reg' if 1) 'reg' is one GPR, AND 2) 'reg' != 'base_reg', AND 3) ins is 'ldr'. */ 459|.macro MEM_ACCESS_64_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg 460|| if (((uintptr_t)(offset)) > LDR_STR_PIMM64) { 461| LOAD_32BIT_VAL tmp_reg, offset 462| ldr_str_ins reg, [base_reg, tmp_reg] 463|| } else { 464| ldr_str_ins reg, [base_reg, #(offset)] 465|| } 466|.endmacro 467 468|.macro MEM_ACCESS_32_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg 469|| if (((uintptr_t)(offset)) > LDR_STR_PIMM32) { 470| LOAD_32BIT_VAL tmp_reg, offset 471| ldr_str_ins reg, [base_reg, tmp_reg] 472|| } else { 473| ldr_str_ins reg, [base_reg, #(offset)] 474|| } 475|.endmacro 476 477|.macro MEM_ACCESS_8_WITH_UOFFSET, ldrb_strb_ins, reg, base_reg, offset, tmp_reg 478|| if (((uintptr_t)(offset)) > LDRB_STRB_PIMM) { 479| LOAD_32BIT_VAL tmp_reg, offset 480| ldrb_strb_ins reg, [base_reg, tmp_reg] 481|| } else { 482| ldrb_strb_ins reg, [base_reg, #(offset)] 483|| } 484|.endmacro 485 486/* Memory access(load/store) with 64-bit 'offset'. 487 * Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register. */ 488|.macro MEM_ACCESS_64_WITH_UOFFSET_64, ldr_str_ins, reg, base_reg, offset, tmp_reg 489|| if (((uintptr_t)(offset)) > LDR_STR_PIMM64) { 490| LOAD_64BIT_VAL tmp_reg, offset 491| ldr_str_ins reg, [base_reg, tmp_reg] 492|| } else { 493| ldr_str_ins reg, [base_reg, #(offset)] 494|| } 495|.endmacro 496 497/* ZTS: get thread local variable "_tsrm_ls_cache" */ 498|.macro LOAD_TSRM_CACHE, reg 499||#ifdef __APPLE__ 500| .long 0xd53bd071 // TODO: hard-coded: mrs TMP3, tpidrro_el0 501| and TMP3, TMP3, #0xfffffffffffffff8 502| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, TMP3, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->key << 3), TMP1 503| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, reg, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->offset), TMP1 504||#else 505| .long 0xd53bd051 // TODO: hard-coded: mrs TMP3, tpidr_el0 506|| if (tsrm_ls_cache_tcb_offset == 0) { 507| ldr TMP3, [TMP3, #0] 508| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, TMP3, TMP3, tsrm_tls_index, TMP1 509| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, reg, TMP3, tsrm_tls_offset, TMP1 510|| } else { 511|| ZEND_ASSERT(tsrm_ls_cache_tcb_offset <= LDR_STR_PIMM64); 512| ldr reg, [TMP3, #tsrm_ls_cache_tcb_offset] 513|| } 514||#endif 515|.endmacro 516 517|.macro LOAD_ADDR_ZTS, reg, struct, field 518| .if ZTS 519| LOAD_TSRM_CACHE TMP3 520| ADD_SUB_64_WITH_CONST_32 add, reg, TMP3, (struct.._offset + offsetof(zend_..struct, field)), reg 521| .else 522| LOAD_ADDR reg, &struct.field 523| .endif 524|.endmacro 525 526/* Store address 'addr' into memory 'mem'. */ 527|.macro ADDR_STORE, mem, addr, tmp_reg 528| LOAD_ADDR tmp_reg, addr 529| str tmp_reg, mem 530|.endmacro 531 532/* Store a register value 'reg' into memory 'addr'. 533 * For ZTS mode, base register with unsigned offset variant is used, 534 * and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */ 535|.macro MEM_STORE, str_ins, reg, addr, tmp_reg 536|| if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) { 537| adr tmp_reg, &addr 538| str_ins reg, [tmp_reg] 539|| } else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) { 540| adrp tmp_reg, &(((uintptr_t)(addr))) 541| str_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)] 542|| } else { 543| LOAD_ADDR tmp_reg, addr 544| str_ins reg, [tmp_reg] 545|| } 546|.endmacro 547 548|.macro MEM_STORE_64_ZTS, str_ins, reg, struct, field, tmp_reg 549| .if ZTS 550| LOAD_TSRM_CACHE TMP3 551| MEM_ACCESS_64_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 552| .else 553| MEM_STORE str_ins, reg, &struct.field, tmp_reg 554| .endif 555|.endmacro 556 557|.macro MEM_STORE_32_ZTS, str_ins, reg, struct, field, tmp_reg 558| .if ZTS 559| LOAD_TSRM_CACHE TMP3 560| MEM_ACCESS_32_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 561| .else 562| MEM_STORE str_ins, reg, &struct.field, tmp_reg 563| .endif 564|.endmacro 565 566|.macro MEM_STORE_8_ZTS, strb_ins, reg, struct, field, tmp_reg 567| .if ZTS 568| LOAD_TSRM_CACHE TMP3 569| MEM_ACCESS_8_WITH_UOFFSET strb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 570| .else 571| MEM_STORE strb_ins, reg, &struct.field, tmp_reg 572| .endif 573|.endmacro 574 575/* Load value from memory 'addr' and write it into register 'reg'. 576 * For ZTS mode, base register with unsigned offset variant is used, 577 * and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */ 578|.macro MEM_LOAD, ldr_ins, reg, addr, tmp_reg 579|| if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) { 580| adr tmp_reg, &addr 581| ldr_ins reg, [tmp_reg] 582|| } else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) { 583| adrp tmp_reg, &(((uintptr_t)(addr))) 584| ldr_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)] 585|| } else { 586| LOAD_ADDR tmp_reg, addr 587| ldr_ins reg, [tmp_reg] 588|| } 589|.endmacro 590 591|.macro MEM_LOAD_64_ZTS, ldr_ins, reg, struct, field, tmp_reg 592| .if ZTS 593| LOAD_TSRM_CACHE TMP3 594| MEM_ACCESS_64_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 595| .else 596| MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg 597| .endif 598|.endmacro 599 600|.macro MEM_LOAD_32_ZTS, ldr_ins, reg, struct, field, tmp_reg 601| .if ZTS 602| LOAD_TSRM_CACHE TMP3 603| MEM_ACCESS_32_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 604| .else 605| MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg 606| .endif 607|.endmacro 608 609|.macro MEM_LOAD_8_ZTS, ldrb_ins, reg, struct, field, tmp_reg 610| .if ZTS 611| LOAD_TSRM_CACHE TMP3 612| MEM_ACCESS_8_WITH_UOFFSET ldrb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 613| .else 614| MEM_LOAD ldrb_ins, reg, &struct.field, tmp_reg 615| .endif 616|.endmacro 617 618/* Conduct arithmetic operation between the value in memory 'addr' and register value in 'reg', 619 * and the computation result is stored back in 'reg'. 'op_ins' can be add/sub. */ 620|.macro MEM_LOAD_OP, op_ins, ldr_ins, reg, addr, tmp_reg1, tmp_reg2 621| MEM_LOAD ldr_ins, tmp_reg1, addr, tmp_reg2 622| op_ins reg, reg, tmp_reg1 623|.endmacro 624 625|.macro MEM_LOAD_OP_ZTS, op_ins, ldr_ins, reg, struct, field, tmp_reg1, tmp_reg2 626| .if ZTS 627| LOAD_TSRM_CACHE TMP3 628| MEM_ACCESS_64_WITH_UOFFSET ldr_ins, tmp_reg2, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg1 629| op_ins reg, reg, tmp_reg2 630| .else 631| MEM_LOAD_OP op_ins, ldr_ins, reg, &struct.field, tmp_reg1, tmp_reg2 632| .endif 633|.endmacro 634 635/* Conduct arithmetic operation between the value in memory 'addr' and operand 'op', and the computation 636 * result is stored back to memory 'addr'. Operand 'op' can be either a register value or an immediate value. 637 * Currently, only add instruction is used as 'op_ins'. 638 * Note: It should be guaranteed that the immediate value can be encoded for 'op_ins'. */ 639|.macro MEM_UPDATE, op_ins, ldr_ins, str_ins, op, addr, tmp_reg1, tmp_reg2 640| LOAD_ADDR tmp_reg2, addr 641| ldr_ins, tmp_reg1, [tmp_reg2] 642| op_ins tmp_reg1, tmp_reg1, op 643| str_ins tmp_reg1, [tmp_reg2] 644|.endmacro 645 646|.macro MEM_UPDATE_ZTS, op_ins, ldr_ins, str_ins, op, struct, field, tmp_reg1, tmp_reg2 647| .if ZTS 648| LOAD_TSRM_CACHE TMP3 649|| if (((uintptr_t)(struct.._offset+offsetof(zend_..struct, field))) > LDR_STR_PIMM64) { 650| LOAD_32BIT_VAL tmp_reg1, (struct.._offset+offsetof(zend_..struct, field)) 651| ldr_ins tmp_reg2, [TMP3, tmp_reg1] 652| op_ins tmp_reg2, tmp_reg2, op 653| str_ins tmp_reg2, [TMP3, tmp_reg1] 654|| } else { 655| ldr_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] 656| op_ins tmp_reg2, tmp_reg2, op 657| str_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] 658|| } 659| .else 660| MEM_UPDATE op_ins, ldr_ins, str_ins, op, &struct.field, tmp_reg1, tmp_reg2 661| .endif 662|.endmacro 663 664|.macro EXT_CALL, func, tmp_reg 665|| if (arm64_may_use_b(func)) { 666| bl &func 667|| } else { 668| LOAD_ADDR tmp_reg, func 669| blr tmp_reg 670|| } 671|.endmacro 672 673|.macro EXT_JMP, func, tmp_reg 674|| if (arm64_may_use_b(func)) { 675| b &func 676|| } else { 677| LOAD_ADDR tmp_reg, func 678| br tmp_reg 679|| } 680|.endmacro 681 682|.macro SAVE_IP 683|| if (GCC_GLOBAL_REGS) { 684| str IP, EX->opline 685|| } 686|.endmacro 687 688|.macro LOAD_IP 689|| if (GCC_GLOBAL_REGS) { 690| ldr IP, EX->opline 691|| } 692|.endmacro 693 694|.macro LOAD_IP_ADDR, addr 695|| if (GCC_GLOBAL_REGS) { 696| LOAD_ADDR IP, addr 697|| } else { 698| ADDR_STORE EX->opline, addr, RX 699|| } 700|.endmacro 701 702|.macro LOAD_IP_ADDR_ZTS, struct, field, tmp_reg 703| .if ZTS 704|| if (GCC_GLOBAL_REGS) { 705| LOAD_TSRM_CACHE IP 706| ADD_SUB_64_WITH_CONST_32 add, IP, IP, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 707|| } else { 708| LOAD_TSRM_CACHE RX 709| ADD_SUB_64_WITH_CONST_32 add, RX, RX, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 710| str RX, EX->opline 711|| } 712| .else 713| LOAD_IP_ADDR &struct.field 714| .endif 715|.endmacro 716 717|.macro GET_IP, reg 718|| if (GCC_GLOBAL_REGS) { 719| mov reg, IP 720|| } else { 721| ldr reg, EX->opline 722|| } 723|.endmacro 724 725/* Update IP with register 'reg'. Note: shift variant is handled by ADD_IP_SHIFT. */ 726|.macro ADD_IP, reg, tmp_reg 727|| if (GCC_GLOBAL_REGS) { 728| add IP, IP, reg 729|| } else { 730| ldr tmp_reg, EX->opline 731| add tmp_reg, tmp_reg, reg 732| str tmp_reg, EX->opline 733|| } 734|.endmacro 735 736|.macro ADD_IP_SHIFT, reg, shift, tmp_reg 737|| if (GCC_GLOBAL_REGS) { 738| add IP, IP, reg, shift 739|| } else { 740| ldr tmp_reg, EX->opline 741| add tmp_reg, tmp_reg, reg, shift 742| str tmp_reg, EX->opline 743|| } 744|.endmacro 745 746/* Update IP with 32-bit immediate 'val'. */ 747|.macro ADD_IP_WITH_CONST, val, tmp_reg 748|| ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val))); 749|| if (GCC_GLOBAL_REGS) { 750| add IP, IP, #val 751|| } else { 752| ldr tmp_reg, EX->opline 753| add tmp_reg, tmp_reg, #val 754| str tmp_reg, EX->opline 755|| } 756|.endmacro 757 758|.macro JMP_IP, tmp_reg 759|| if (GCC_GLOBAL_REGS) { 760| ldr tmp_reg, [IP] 761| br tmp_reg 762|| } else { 763| ldr tmp_reg, EX:CARG1->opline 764| br tmp_reg 765|| } 766|.endmacro 767 768|.macro CMP_IP, addr, tmp_reg1, tmp_reg2 769| LOAD_ADDR tmp_reg1, addr 770|| if (GCC_GLOBAL_REGS) { 771| cmp IP, tmp_reg1 772|| } else { 773| ldr tmp_reg2, EX->opline 774| cmp tmp_reg2, tmp_reg1 775|| } 776|.endmacro 777 778|.macro LOAD_ZVAL_ADDR, reg, addr 779|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 780| LOAD_ADDR reg, Z_ZV(addr) 781|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 782|| if (Z_OFFSET(addr)) { 783| ADD_SUB_64_WITH_CONST_32 add, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), reg 784|| } else { 785|| if (Z_REG(addr) == ZREG_RSP) { 786| mov reg, sp 787|| } else { 788| mov reg, Rx(Z_REG(addr)) 789|| } 790|| } 791|| } else { 792|| ZEND_UNREACHABLE(); 793|| } 794|.endmacro 795 796|.macro GET_Z_TYPE_INFO, reg, zv 797| ldr reg, [zv, #offsetof(zval,u1.type_info)] 798|.endmacro 799 800|.macro SET_Z_TYPE_INFO, zv, type, tmp_reg 801| LOAD_32BIT_VAL tmp_reg, type 802| str tmp_reg, [zv, #offsetof(zval,u1.type_info)] 803|.endmacro 804 805|.macro GET_ZVAL_TYPE, reg, addr, tmp_reg 806|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 807| MEM_ACCESS_8_WITH_UOFFSET ldrb, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.v.type), tmp_reg 808|.endmacro 809 810|.macro GET_ZVAL_TYPE_INFO, reg, addr, tmp_reg 811|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 812| MEM_ACCESS_32_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg 813|.endmacro 814 815|.macro SET_ZVAL_TYPE_INFO, addr, type, tmp_reg1, tmp_reg2 816|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 817| LOAD_32BIT_VAL tmp_reg1, type 818| MEM_ACCESS_32_WITH_UOFFSET str, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg2 819|.endmacro 820 821|.macro SET_ZVAL_TYPE_INFO_FROM_REG, addr, reg, tmp_reg 822|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 823| MEM_ACCESS_32_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg 824|.endmacro 825 826|.macro GET_Z_PTR, reg, zv 827| ldr reg, [zv] 828|.endmacro 829 830|.macro GET_ZVAL_PTR, reg, addr, tmp_reg 831|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 832| MEM_ACCESS_64_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 833|.endmacro 834 835|.macro SET_ZVAL_PTR, addr, reg, tmp_reg 836|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 837| MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 838|.endmacro 839 840|.macro UNDEF_OPLINE_RESULT, tmp_reg 841| ldr REG0, EX->opline 842| ldr REG0w, OP:REG0->result.var 843| add REG0, FP, REG0 844| SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg 845|.endmacro 846 847|.macro UNDEF_OPLINE_RESULT_IF_USED, tmp_reg1, tmp_reg2 848| ldrb tmp_reg1, OP:RX->result_type 849| TST_32_WITH_CONST tmp_reg1, (IS_TMP_VAR|IS_VAR), tmp_reg2 850| beq >1 851| ldr REG0w, OP:RX->result.var 852| add REG0, FP, REG0 853| SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg1 854|1: 855|.endmacro 856 857/* Floating-point comparison between register 'reg' and value from memory 'addr'. 858 * Note: the equivalent macros in JIT/x86 are SSE_AVX_OP and SSE_OP. */ 859|.macro DOUBLE_CMP, reg, addr, tmp_reg, fp_tmp_reg 860|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 861| MEM_LOAD ldr, Rd(fp_tmp_reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg) 862| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) 863|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 864| MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) 865| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) 866|| } else if (Z_MODE(addr) == IS_REG) { 867| fcmp Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) 868|| } else { 869|| ZEND_UNREACHABLE(); 870|| } 871|.endmacro 872 873/* Convert LONG value 'val' into DOUBLE type, and move it into FP register 'reg'. 874 * Note: the equivalent macro in JIT/x86 is SSE_GET_LONG. */ 875|.macro DOUBLE_GET_LONG, reg, val, tmp_reg 876|| if (val == 0) { 877| fmov Rd(reg-ZREG_V0), xzr // TODO: "movi d0, #0" is not recognized by DynASM/arm64 878|| } else { 879| LOAD_64BIT_VAL Rx(tmp_reg), val 880| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg) 881|| } 882|.endmacro 883 884/* Convert LONG value from memory 'addr' into DOUBLE type, and move it into FP register 'reg'. 885 * Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_LVAL. */ 886|.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg1, tmp_reg2 887|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 888| DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg1 889|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 890| MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg2) 891| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg1) 892|| } else if (Z_MODE(addr) == IS_REG) { 893| scvtf Rd(reg-ZREG_V0), Rx(Z_REG(addr)) 894|| } else { 895|| ZEND_UNREACHABLE(); 896|| } 897|.endmacro 898 899/* Floating-point arithmetic operation between two FP registers. 900 * Note: the equivalent macro in JIT/x86 is AVX_MATH_REG. */ 901|.macro DOUBLE_MATH_REG, opcode, dst_reg, op1_reg, op2_reg 902|| switch (opcode) { 903|| case ZEND_ADD: 904| fadd Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 905|| break; 906|| case ZEND_SUB: 907| fsub Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 908|| break; 909|| case ZEND_MUL: 910| fmul Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 911|| break; 912|| case ZEND_DIV: 913| fdiv Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 914|| break; 915|| } 916|.endmacro 917 918/* Conduct binary operation between register 'reg' and value from memory 'addr', 919 * and the computation result is stored in 'reg'. 920 * For LONG_ADD_SUB, 'add_sub_ins' can be adds/subs. For LONG_BW_OP, 'bw_ins' can be and/orr/eor. 921 * For LONG_CMP, 'cmp' instruction is used by default and only flag registers are affected. 922 * Note: the equivalent macro in JIT/x86 is LONG_OP. */ 923|.macro LONG_ADD_SUB, add_sub_ins, reg, addr, tmp_reg 924|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 925| ADD_SUB_64_WITH_CONST add_sub_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg 926|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 927| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 928| add_sub_ins Rx(reg), Rx(reg), tmp_reg 929|| } else if (Z_MODE(addr) == IS_REG) { 930| add_sub_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) 931|| } else { 932|| ZEND_UNREACHABLE(); 933|| } 934|.endmacro 935 936|.macro LONG_BW_OP, bw_ins, reg, addr, tmp_reg 937|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 938| BW_OP_64_WITH_CONST bw_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg 939|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 940| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 941| bw_ins Rx(reg), Rx(reg), tmp_reg 942|| } else if (Z_MODE(addr) == IS_REG) { 943| bw_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) 944|| } else { 945|| ZEND_UNREACHABLE(); 946|| } 947|.endmacro 948 949|.macro LONG_CMP, reg, addr, tmp_reg 950|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 951| CMP_64_WITH_CONST Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg 952|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 953| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 954| cmp Rx(reg), tmp_reg 955|| } else if (Z_MODE(addr) == IS_REG) { 956| cmp Rx(reg), Rx(Z_REG(addr)) 957|| } else { 958|| ZEND_UNREACHABLE(); 959|| } 960|.endmacro 961 962/* Conduct add/sub between value from memory 'addr' and an immediate value 'val', and 963 * the computation result is stored back into 'addr'. 964 * Note: it should be guaranteed that 'val' can be encoded into add/sub instruction. */ 965|.macro LONG_ADD_SUB_WITH_IMM, add_sub_ins, addr, val, tmp_reg1, tmp_reg2 966|| ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val))); 967|| if (Z_MODE(addr) == IS_MEM_ZVAL) { 968|| if (((uint32_t)(Z_OFFSET(addr))) > LDR_STR_PIMM64) { 969| LOAD_32BIT_VAL tmp_reg2, Z_OFFSET(addr) 970| ldr tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2] 971| add_sub_ins tmp_reg1, tmp_reg1, #val 972| str tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2] 973|| } else { 974| ldr tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)] 975| add_sub_ins tmp_reg1, tmp_reg1, #val 976| str tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)] 977|| } 978|| } else if (Z_MODE(addr) == IS_REG) { 979| add_sub_ins Rx(Z_REG(addr)), Rx(Z_REG(addr)), #val 980|| } else { 981|| ZEND_UNREACHABLE(); 982|| } 983|.endmacro 984 985/* Compare value from memory 'addr' with immediate value 'val'. 986 * Note: the equivalent macro in JIT/x86 is LONG_OP_WITH_CONST. */ 987|.macro LONG_CMP_WITH_CONST, addr, val, tmp_reg1, tmp_reg2 988|| if (Z_MODE(addr) == IS_MEM_ZVAL) { 989| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 990| CMP_64_WITH_CONST tmp_reg1, val, tmp_reg2 991|| } else if (Z_MODE(addr) == IS_REG) { 992| CMP_64_WITH_CONST Rx(Z_REG(addr)), val, tmp_reg1 993|| } else { 994|| ZEND_UNREACHABLE(); 995|| } 996|.endmacro 997 998|.macro GET_ZVAL_LVAL, reg, addr, tmp_reg 999|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 1000|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { 1001| mov Rx(reg), xzr 1002|| } else { 1003| LOAD_64BIT_VAL Rx(reg), Z_LVAL_P(Z_ZV(addr)) 1004|| } 1005|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 1006| MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(reg), Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 1007|| } else if (Z_MODE(addr) == IS_REG) { 1008|| if (reg != Z_REG(addr)) { 1009| mov Rx(reg), Rx(Z_REG(addr)) 1010|| } 1011|| } else { 1012|| ZEND_UNREACHABLE(); 1013|| } 1014|.endmacro 1015 1016|.macro LONG_MATH, opcode, reg, addr, tmp_reg1 1017|| switch (opcode) { 1018|| case ZEND_ADD: 1019| LONG_ADD_SUB adds, reg, addr, tmp_reg1 1020|| break; 1021|| case ZEND_SUB: 1022| LONG_ADD_SUB subs, reg, addr, tmp_reg1 1023|| break; 1024|| case ZEND_BW_OR: 1025| LONG_BW_OP orr, reg, addr, tmp_reg1 1026|| break; 1027|| case ZEND_BW_AND: 1028| LONG_BW_OP and, reg, addr, tmp_reg1 1029|| break; 1030|| case ZEND_BW_XOR: 1031| LONG_BW_OP eor, reg, addr, tmp_reg1 1032|| break; 1033|| default: 1034|| ZEND_UNREACHABLE(); 1035|| } 1036|.endmacro 1037 1038|.macro LONG_MATH_REG, opcode, dst_reg, src_reg1, src_reg2 1039|| switch (opcode) { 1040|| case ZEND_ADD: 1041| adds dst_reg, src_reg1, src_reg2 1042|| break; 1043|| case ZEND_SUB: 1044| subs dst_reg, src_reg1, src_reg2 1045|| break; 1046|| case ZEND_BW_OR: 1047| orr dst_reg, src_reg1, src_reg2 1048|| break; 1049|| case ZEND_BW_AND: 1050| and dst_reg, src_reg1, src_reg2 1051|| break; 1052|| case ZEND_BW_XOR: 1053| eor dst_reg, src_reg1, src_reg2 1054|| break; 1055|| default: 1056|| ZEND_UNREACHABLE(); 1057|| } 1058|.endmacro 1059 1060/* Store LONG value into memory 'addr'. 1061 * This LONG value can be an immediate value i.e. 'val' in macro SET_ZVAL_LVAL, or 1062 * a register value i.e. 'reg' in macro SET_ZVAL_LVAL_FROM_REG. */ 1063|.macro SET_ZVAL_LVAL_FROM_REG, addr, reg, tmp_reg 1064|| if (Z_MODE(addr) == IS_REG) { 1065| mov Rx(Z_REG(addr)), reg 1066|| } else { 1067|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1068| MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 1069|| } 1070|.endmacro 1071 1072|.macro SET_ZVAL_LVAL, addr, val, tmp_reg1, tmp_reg2 1073|| if (val == 0) { 1074| SET_ZVAL_LVAL_FROM_REG addr, xzr, tmp_reg2 1075|| } else { 1076| LOAD_64BIT_VAL tmp_reg1, val 1077| SET_ZVAL_LVAL_FROM_REG addr, tmp_reg1, tmp_reg2 1078|| } 1079|.endmacro 1080 1081/* Store DOUBLE value from FP register 'reg' into memory 'addr'. 1082 * Note: the equivalent macro in JIT/x86 is SSE_SET_ZVAL_DVAL. */ 1083|.macro SET_ZVAL_DVAL, addr, reg, tmp_reg 1084|| if (Z_MODE(addr) == IS_REG) { 1085|| if (reg != Z_REG(addr)) { 1086| fmov Rd(Z_REG(addr)-ZREG_V0), Rd(reg-ZREG_V0) 1087|| } 1088|| } else { 1089|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1090| MEM_ACCESS_64_WITH_UOFFSET str, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) 1091|| } 1092|.endmacro 1093 1094/* Load DOUBLE value from memory 'addr' into FP register 'reg'. 1095 * Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_DVAL. */ 1096|.macro GET_ZVAL_DVAL, reg, addr, tmp_reg 1097|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { 1098|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 1099| MEM_LOAD ldr, Rd(reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg) 1100|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 1101| MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) 1102|| } else if (Z_MODE(addr) == IS_REG) { 1103| fmov Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) 1104|| } else { 1105|| ZEND_UNREACHABLE(); 1106|| } 1107|| } 1108|.endmacro 1109 1110|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg 1111|| if (Z_TYPE_P(zv) > IS_TRUE) { 1112|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 1113|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; 1114| MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1) 1115| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 1116|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 1117|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; 1118| DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), tmp_reg1 1119| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 1120|| } else { 1121| // In x64, if the range of this LONG value can be represented via INT type, only move the low 32 bits into dst_addr. 1122| // Note that imm32 is signed extended to 64 bits during mov. 1123| // In aarch64, we choose to handle both cases in the same way. Even though 4 mov's are used for 64-bit value and 2 mov's are 1124| // needed for 32-bit value, an extra ext insn is needed for 32-bit value. 1125| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1126|| } 1127|| } 1128|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1129|| if (dst_def_info == MAY_BE_DOUBLE) { 1130|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1131| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) 1132|| } 1133|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1134| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2) 1135|| } 1136|| } 1137|.endmacro 1138 1139|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg 1140|| if (Z_TYPE_P(zv) > IS_TRUE) { 1141|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 1142|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? 1143|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : fp_tmp_reg); 1144| MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1) 1145| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 1146| SET_ZVAL_DVAL res_addr, dst_reg, tmp_reg2 1147|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 1148|| if (Z_MODE(dst_addr) == IS_REG) { 1149| DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), tmp_reg1 1150| SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg2 1151|| } else if (Z_MODE(res_addr) == IS_REG) { 1152| DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), tmp_reg1 1153| SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg2 1154|| } else { 1155| DOUBLE_GET_LONG fp_tmp_reg, Z_LVAL_P(zv), tmp_reg1 1156| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg2 1157| SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg2 1158|| } 1159|| } else { 1160|| if (Z_MODE(dst_addr) == IS_REG) { 1161| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1162| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg1) 1163|| } else if (Z_MODE(res_addr) == IS_REG) { 1164| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1165| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg1) 1166|| } else { 1167| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1168| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1169|| } 1170|| } 1171|| } 1172|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1173|| if (dst_def_info == MAY_BE_DOUBLE) { 1174|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1175| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) 1176|| } 1177|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1178| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2) 1179|| } 1180|| } 1181|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1182|| if (dst_def_info == MAY_BE_DOUBLE) { 1183| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) 1184|| } else { 1185| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2) 1186|| } 1187|| } 1188|.endmacro 1189 1190// the same as above, but "src" may overlap with "reg1" 1191// Useful info would be stored into reg1 and reg2, and they might be used afterward. 1192|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg1, tmp_reg2, fp_tmp_reg 1193| ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg1, fp_tmp_reg 1194|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1195|| !(src_info & MAY_BE_GUARD) && 1196|| has_concrete_type(src_info & MAY_BE_ANY)) { 1197|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1198|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) { 1199|| uint32_t type = concrete_type(src_info); 1200| SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg1), Rx(tmp_reg2) 1201|| } 1202|| } 1203|| } else { 1204| GET_ZVAL_TYPE_INFO Rw(reg1), src_addr, Rx(tmp_reg1) 1205| SET_ZVAL_TYPE_INFO_FROM_REG dst_addr, Rw(reg1), Rx(tmp_reg1) 1206|| } 1207|.endmacro 1208 1209|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg, fp_tmp_reg 1210|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1211|| if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) { 1212|| if (Z_MODE(src_addr) == IS_REG) { 1213|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1214| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) 1215|| } 1216|| } else if (Z_MODE(dst_addr) == IS_REG) { 1217| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) 1218|| } else { 1219| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) 1220| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg) 1221|| } 1222|| } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 1223|| if (Z_MODE(src_addr) == IS_REG) { 1224| SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg 1225|| } else if (Z_MODE(dst_addr) == IS_REG) { 1226| GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg 1227|| } else { 1228| GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg 1229| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg 1230|| } 1231|| // Combine the following two branches. 1232|| // } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) { 1233|| } else { 1234| GET_ZVAL_PTR Rx(reg2), src_addr, Rx(tmp_reg) 1235| SET_ZVAL_PTR dst_addr, Rx(reg2), Rx(tmp_reg) 1236|| } 1237|| } 1238|.endmacro 1239 1240|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, reg1, reg2, tmp_reg, tmp_reg2, fp_tmp_reg 1241|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1242|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) { 1243|| if (Z_MODE(src_addr) == IS_REG) { 1244|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1245| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) 1246|| } 1247|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) { 1248| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) 1249|| } 1250|| } else if (Z_MODE(dst_addr) == IS_REG) { 1251| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) 1252|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) { 1253| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg) 1254|| } 1255|| } else if (Z_MODE(res_addr) == IS_REG) { 1256| GET_ZVAL_LVAL Z_REG(res_addr), src_addr, Rx(tmp_reg) 1257| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg) 1258|| } else { 1259| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) 1260| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg) 1261| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(reg2), Rx(tmp_reg) 1262|| } 1263|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 1264|| if (Z_MODE(src_addr) == IS_REG) { 1265| SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg 1266| SET_ZVAL_DVAL res_addr, Z_REG(src_addr), tmp_reg 1267|| } else if (Z_MODE(dst_addr) == IS_REG) { 1268| GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg 1269| SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg 1270|| } else if (Z_MODE(res_addr) == IS_REG) { 1271| GET_ZVAL_DVAL Z_REG(res_addr), src_addr, tmp_reg 1272| SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg 1273|| } else { 1274| GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg 1275| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg 1276| SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg 1277|| } 1278|| } else { 1279| GET_ZVAL_PTR Rx(reg2), src_addr, Rx(tmp_reg) 1280| SET_ZVAL_PTR dst_addr, Rx(reg2), Rx(tmp_reg) 1281| SET_ZVAL_PTR res_addr, Rx(reg2), Rx(tmp_reg) 1282|| } 1283|| } 1284|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1285|| has_concrete_type(src_info & MAY_BE_ANY)) { 1286|| uint32_t type = concrete_type(src_info); 1287|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1288|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { 1289| SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg), Rx(tmp_reg2) 1290|| } 1291|| } 1292|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1293| SET_ZVAL_TYPE_INFO res_addr, type, Rw(tmp_reg), Rx(tmp_reg2) 1294|| } 1295|| } else { 1296| GET_ZVAL_TYPE_INFO Rw(reg1), src_addr, Rx(tmp_reg) 1297| SET_ZVAL_TYPE_INFO_FROM_REG dst_addr, Rw(reg1), Rx(tmp_reg) 1298| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, Rw(reg1), Rx(tmp_reg) 1299|| } 1300|.endmacro 1301 1302|.macro IF_UNDEF, type_reg, label 1303| cbz type_reg, label 1304|.endmacro 1305 1306|.macro IF_TYPE, type, val, label 1307|| if (val == 0) { 1308| cbz type, label 1309|| } else { 1310| cmp type, #val 1311| beq label 1312|| } 1313|.endmacro 1314 1315|.macro IF_NOT_TYPE, type, val, label 1316|| if (val == 0) { 1317| cbnz type, label 1318|| } else { 1319| cmp type, #val 1320| bne label 1321|| } 1322|.endmacro 1323 1324|.macro IF_Z_TYPE, zv, val, label, tmp_reg 1325| ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)] 1326| IF_TYPE tmp_reg, val, label 1327|.endmacro 1328 1329|.macro IF_NOT_Z_TYPE, zv, val, label, tmp_reg 1330| ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)] 1331| IF_NOT_TYPE tmp_reg, val, label 1332|.endmacro 1333 1334|.macro CMP_ZVAL_TYPE, addr, val, tmp_reg 1335|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1336| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) 1337| cmp Rw(tmp_reg), #val 1338|.endmacro 1339 1340|.macro IF_ZVAL_TYPE, addr, val, label, tmp_reg 1341|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1342| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) 1343| IF_TYPE Rw(tmp_reg), val, label 1344|.endmacro 1345 1346|.macro IF_NOT_ZVAL_TYPE, addr, val, label, tmp_reg 1347|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1348| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) 1349| IF_NOT_TYPE Rw(tmp_reg), val, label 1350|.endmacro 1351 1352|.macro IF_FLAGS, type_flags, mask, label, tmp_reg 1353| TST_32_WITH_CONST type_flags, mask, tmp_reg 1354| bne label 1355|.endmacro 1356 1357|.macro IF_NOT_FLAGS, type_flags, mask, label, tmp_reg 1358| TST_32_WITH_CONST type_flags, mask, tmp_reg 1359| beq label 1360|.endmacro 1361 1362|.macro IF_REFCOUNTED, type_flags, label, tmp_reg 1363| TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg 1364| bne label 1365|.endmacro 1366 1367|.macro IF_NOT_REFCOUNTED, type_flags, label, tmp_reg 1368| TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg 1369| beq label 1370|.endmacro 1371 1372|.macro IF_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 1373|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1374| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2) 1375| IF_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2) 1376|.endmacro 1377 1378|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 1379|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1380| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2) 1381| IF_NOT_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2) 1382|.endmacro 1383 1384|.macro IF_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2 1385| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2 1386|.endmacro 1387 1388|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2 1389| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2 1390|.endmacro 1391 1392|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label, tmp_reg1, tmp_reg2 1393| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label, tmp_reg1, tmp_reg2 1394|.endmacro 1395 1396|.macro GC_ADDREF, zv, tmp_reg 1397| ldr tmp_reg, [zv] 1398| add tmp_reg, tmp_reg, #1 1399| str tmp_reg, [zv] 1400|.endmacro 1401 1402|.macro GC_ADDREF_2, zv, tmp_reg 1403| ldr tmp_reg, [zv] 1404| add tmp_reg, tmp_reg, #2 1405| str tmp_reg, [zv] 1406|.endmacro 1407 1408|.macro GC_DELREF, zv, tmp_reg 1409| ldr tmp_reg, [zv] 1410| subs tmp_reg, tmp_reg, #1 1411| str tmp_reg, [zv] 1412|.endmacro 1413 1414|.macro IF_GC_MAY_NOT_LEAK, ptr, label, tmp_reg1, tmp_reg2 1415| ldr tmp_reg1, [ptr, #4] 1416| TST_32_WITH_CONST tmp_reg1, (GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)), tmp_reg2 1417| bne label 1418|.endmacro 1419 1420|.macro ADDREF_CONST, zv, tmp_reg1, tmp_reg2 1421| LOAD_ADDR tmp_reg1, Z_PTR_P(zv) 1422| ldr tmp_reg2, [tmp_reg1] 1423| add tmp_reg2, tmp_reg2, #1 1424| str tmp_reg2, [tmp_reg1] 1425|.endmacro 1426 1427|.macro ADDREF_CONST_2, zv, tmp_reg1, tmp_reg2 1428| LOAD_ADDR tmp_reg1, Z_PTR_P(zv) 1429| ldr tmp_reg2, [tmp_reg1] 1430| add tmp_reg2, tmp_reg2, #2 1431| str tmp_reg2, [tmp_reg1] 1432|.endmacro 1433 1434|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg, tmp_reg 1435|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1436|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1437| IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg 1438|| } 1439| GC_ADDREF value_ptr_reg, tmp_reg 1440|1: 1441|| } 1442|.endmacro 1443 1444|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg, tmp_reg 1445|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1446|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1447| IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg 1448|| } 1449| ldr tmp_reg, [value_ptr_reg] 1450| add tmp_reg, tmp_reg, #2 1451| str tmp_reg, [value_ptr_reg] 1452|1: 1453|| } 1454|.endmacro 1455 1456|.macro ZVAL_DEREF, reg, info, tmp_reg 1457|| if (info & MAY_BE_REF) { 1458| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1, tmp_reg 1459| GET_Z_PTR reg, reg 1460| add reg, reg, #offsetof(zend_reference, val) 1461|1: 1462|| } 1463|.endmacro 1464 1465|.macro SET_EX_OPLINE, op, tmp_reg 1466|| if (op == last_valid_opline) { 1467|| zend_jit_use_last_valid_opline(); 1468| SAVE_IP 1469|| } else { 1470| ADDR_STORE EX->opline, op, tmp_reg 1471|| if (!GCC_GLOBAL_REGS) { 1472|| zend_jit_reset_last_valid_opline(); 1473|| } 1474|| } 1475|.endmacro 1476 1477// arg1 "zval" should be in FCARG1x 1478|.macro ZVAL_DTOR_FUNC, var_info, opline, tmp_reg 1479|| do { 1480|| if (!((var_info) & MAY_BE_GUARD) 1481|| && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1482|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); 1483|| if (type == IS_STRING && !ZEND_DEBUG) { 1484| EXT_CALL _efree, tmp_reg 1485|| break; 1486|| } else if (type == IS_ARRAY) { 1487|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { 1488|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { 1489| SET_EX_OPLINE opline, tmp_reg 1490|| } 1491| EXT_CALL zend_array_destroy, tmp_reg 1492|| } else { 1493| EXT_CALL zend_jit_array_free, tmp_reg 1494|| } 1495|| break; 1496|| } else if (type == IS_OBJECT) { 1497|| if (opline) { 1498| SET_EX_OPLINE opline, REG0 1499|| } 1500| EXT_CALL zend_objects_store_del, tmp_reg 1501|| break; 1502|| } 1503|| } 1504|| if (opline) { 1505| SET_EX_OPLINE opline, tmp_reg 1506|| } 1507| EXT_CALL rc_dtor_func, tmp_reg 1508|| } while(0); 1509|.endmacro 1510 1511|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline, tmp_reg1, tmp_reg2 1512|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) { 1513|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1514| // if (Z_REFCOUNTED_P(cv)) { 1515|| if (cold) { 1516| IF_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 1517|.cold_code 1518|1: 1519|| } else { 1520| IF_NOT_ZVAL_REFCOUNTED addr, >4, tmp_reg1, tmp_reg2 1521|| } 1522|| } 1523| // if (!Z_DELREF_P(cv)) { 1524| GET_ZVAL_PTR FCARG1x, addr, Rx(tmp_reg2) 1525| GC_DELREF FCARG1x, Rw(tmp_reg1) 1526|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) { 1527|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) { 1528|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1529| bne >3 1530|| } else { 1531| bne >4 1532|| } 1533|| } 1534| // zval_dtor_func(r); 1535| ZVAL_DTOR_FUNC op_info, opline, Rx(tmp_reg1) 1536|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1537| b >4 1538|| } 1539|3: 1540|| } 1541|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1542|| if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) { 1543|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 1544| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1, tmp_reg1 1545| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4, tmp_reg1, tmp_reg2 1546| GET_ZVAL_PTR FCARG1x, ref_addr, Rx(tmp_reg2) 1547|1: 1548|| } 1549| IF_GC_MAY_NOT_LEAK FCARG1x, >4, Rw(tmp_reg1), Rw(tmp_reg2) 1550| // gc_possible_root(Z_COUNTED_P(z)) 1551| EXT_CALL gc_possible_root, Rx(tmp_reg1) 1552|| } 1553|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { 1554| b >4 1555|.code 1556|| } 1557|4: 1558|| } 1559|.endmacro 1560 1561|.macro FREE_OP, op_type, op, op_info, cold, opline, tmp_reg1, tmp_reg2 1562|| if (op_type & (IS_VAR|IS_TMP_VAR)) { 1563|| zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 1564| ZVAL_PTR_DTOR addr, op_info, 0, cold, opline, tmp_reg1, tmp_reg2 1565|| } 1566|.endmacro 1567 1568|.macro SEPARATE_ARRAY, addr, op_info, cold, tmp_reg1, tmp_reg2 1569|| if (RC_MAY_BE_N(op_info)) { 1570|| if (Z_REG(addr) != ZREG_FP) { 1571| GET_ZVAL_LVAL ZREG_REG0, addr, Rx(tmp_reg1) 1572|| if (RC_MAY_BE_1(op_info)) { 1573| // if (GC_REFCOUNT() > 1) 1574| ldr Rw(tmp_reg1), [REG0] 1575| cmp Rw(tmp_reg1), #1 1576| bls >2 1577|| } 1578|| if (Z_REG(addr) != ZREG_FCARG1 || Z_OFFSET(addr) != 0) { 1579| LOAD_ZVAL_ADDR FCARG1x, addr 1580|| } 1581| EXT_CALL zend_jit_zval_array_dup, REG0 1582| mov REG0, RETVALx 1583|2: 1584| mov FCARG1x, REG0 1585|| } else { 1586| GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1) 1587|| if (RC_MAY_BE_1(op_info)) { 1588| // if (GC_REFCOUNT() > 1) 1589| ldr Rw(tmp_reg1), [FCARG1x] 1590| cmp Rw(tmp_reg1), #1 1591|| if (cold) { 1592| bhi >1 1593|.cold_code 1594|1: 1595|| } else { 1596| bls >2 1597|| } 1598|| } 1599| IF_NOT_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 1600| GC_DELREF FCARG1x, Rw(tmp_reg1) 1601|1: 1602| EXT_CALL zend_array_dup, REG0 1603| mov REG0, RETVALx 1604| SET_ZVAL_PTR addr, REG0, Rx(tmp_reg1) 1605| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX, Rw(tmp_reg1), Rx(tmp_reg2) 1606| mov FCARG1x, REG0 1607|| if (RC_MAY_BE_1(op_info)) { 1608|| if (cold) { 1609| b >2 1610|.code 1611|| } 1612|| } 1613|2: 1614|| } 1615|| } else { 1616| GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1) 1617|| } 1618|.endmacro 1619 1620/* argument is passed in FCARG1x */ 1621|.macro EFREE_REFERENCE 1622||#if ZEND_DEBUG 1623| mov FCARG2x, xzr // filename 1624| mov CARG3w, wzr // lineno 1625| mov CARG4, xzr 1626| mov CARG5, xzr 1627| EXT_CALL _efree, REG0 1628||#else 1629||#ifdef HAVE_BUILTIN_CONSTANT_P 1630| EXT_CALL _efree_32, REG0 1631||#else 1632| EXT_CALL _efree, REG0 1633||#endif 1634||#endif 1635|.endmacro 1636 1637|.macro EMALLOC, size, op_array, opline 1638||#if ZEND_DEBUG 1639|| const char *filename = op_array->filename ? op_array->filename->val : NULL; 1640| mov FCARG1x, #size 1641| LOAD_ADDR FCARG2x, filename 1642| LOAD_32BIT_VAL CARG3w, opline->lineno 1643| mov CARG4, xzr 1644| mov CARG5, xzr 1645| EXT_CALL _emalloc, REG0 1646| mov REG0, RETVALx 1647||#else 1648||#ifdef HAVE_BUILTIN_CONSTANT_P 1649|| if (size > 24 && size <= 32) { 1650| EXT_CALL _emalloc_32, REG0 1651| mov REG0, RETVALx 1652|| } else { 1653| mov FCARG1x, #size 1654| EXT_CALL _emalloc, REG0 1655| mov REG0, RETVALx 1656|| } 1657||#else 1658| mov FCARG1x, #size 1659| EXT_CALL _emalloc, REG0 1660| mov REG0, RETVALx 1661||#endif 1662||#endif 1663|.endmacro 1664 1665|.macro OBJ_RELEASE, reg, exit_label, tmp_reg1, tmp_reg2 1666| GC_DELREF Rx(reg), Rw(tmp_reg1) 1667| bne >1 1668| // zend_objects_store_del(obj); 1669|| if (reg != ZREG_FCARG1) { 1670| mov FCARG1x, Rx(reg) 1671|| } 1672| EXT_CALL zend_objects_store_del, Rx(tmp_reg1) 1673| b exit_label 1674|1: 1675| IF_GC_MAY_NOT_LEAK Rx(reg), >1, Rw(tmp_reg1), Rw(tmp_reg2) 1676| // gc_possible_root(obj) 1677|| if (reg != ZREG_FCARG1) { 1678| mov FCARG1x, Rx(reg) 1679|| } 1680| EXT_CALL gc_possible_root, Rx(tmp_reg1) 1681|1: 1682|.endmacro 1683 1684|.macro UNDEFINED_OFFSET, opline 1685|| if (opline == last_valid_opline) { 1686|| zend_jit_use_last_valid_opline(); 1687| bl ->undefined_offset_ex 1688|| } else { 1689| SET_EX_OPLINE opline, REG0 1690| bl ->undefined_offset 1691|| } 1692|.endmacro 1693 1694|.macro UNDEFINED_INDEX, opline 1695|| if (opline == last_valid_opline) { 1696|| zend_jit_use_last_valid_opline(); 1697| bl ->undefined_index_ex 1698|| } else { 1699| SET_EX_OPLINE opline, REG0 1700| bl ->undefined_index 1701|| } 1702|.endmacro 1703 1704|.macro CANNOT_ADD_ELEMENT, opline 1705|| if (opline == last_valid_opline) { 1706|| zend_jit_use_last_valid_opline(); 1707| bl ->cannot_add_element_ex 1708|| } else { 1709| SET_EX_OPLINE opline, REG0 1710| bl ->cannot_add_element 1711|| } 1712|.endmacro 1713 1714static bool reuse_ip = 0; 1715static bool delayed_call_chain = 0; 1716static uint32_t delayed_call_level = 0; 1717static const zend_op *last_valid_opline = NULL; 1718static bool use_last_vald_opline = 0; 1719static bool track_last_valid_opline = 0; 1720static int jit_return_label = -1; 1721static uint32_t current_trace_num = 0; 1722static uint32_t allowed_opt_flags = 0; 1723 1724static void zend_jit_track_last_valid_opline(void) 1725{ 1726 use_last_vald_opline = 0; 1727 track_last_valid_opline = 1; 1728} 1729 1730static void zend_jit_use_last_valid_opline(void) 1731{ 1732 if (track_last_valid_opline) { 1733 use_last_vald_opline = 1; 1734 track_last_valid_opline = 0; 1735 } 1736} 1737 1738static bool zend_jit_trace_uses_initial_ip(void) 1739{ 1740 return use_last_vald_opline; 1741} 1742 1743static void zend_jit_set_last_valid_opline(const zend_op *target_opline) 1744{ 1745 if (!reuse_ip) { 1746 track_last_valid_opline = 0; 1747 last_valid_opline = target_opline; 1748 } 1749} 1750 1751static void zend_jit_reset_last_valid_opline(void) 1752{ 1753 track_last_valid_opline = 0; 1754 last_valid_opline = NULL; 1755} 1756 1757static void zend_jit_start_reuse_ip(void) 1758{ 1759 zend_jit_reset_last_valid_opline(); 1760 reuse_ip = 1; 1761} 1762 1763static int zend_jit_reuse_ip(dasm_State **Dst) 1764{ 1765 if (!reuse_ip) { 1766 zend_jit_start_reuse_ip(); 1767 | // call = EX(call); 1768 | ldr RX, EX->call 1769 } 1770 return 1; 1771} 1772 1773static void zend_jit_stop_reuse_ip(void) 1774{ 1775 reuse_ip = 0; 1776} 1777 1778static int zend_jit_interrupt_handler_stub(dasm_State **Dst) 1779{ 1780 |->interrupt_handler: 1781 | SAVE_IP 1782 | //EG(vm_interrupt) = 0; 1783 | MEM_STORE_8_ZTS strb, wzr, executor_globals, vm_interrupt, TMP1 1784 | //if (EG(timed_out)) { 1785 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, timed_out, TMP1 1786 | cbz TMP1w, >1 1787 | //zend_timeout(); 1788 | EXT_CALL zend_timeout, TMP1 1789 |1: 1790 | //} else if (zend_interrupt_function) { 1791 if (zend_interrupt_function) { 1792 | //zend_interrupt_function(execute_data); 1793 | mov CARG1, FP 1794 | EXT_CALL zend_interrupt_function, TMP1 1795 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 1796 | cbz REG0, >1 1797 | EXT_CALL zend_jit_exception_in_interrupt_handler_helper, TMP1 1798 |1: 1799 | //ZEND_VM_ENTER(); 1800 | //execute_data = EG(current_execute_data); 1801 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 1802 | LOAD_IP 1803 } 1804 | //ZEND_VM_CONTINUE() 1805 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1806 | ADD_HYBRID_SPAD 1807 | JMP_IP TMP1 1808 } else if (GCC_GLOBAL_REGS) { 1809 | ldp x29, x30, [sp], # SPAD // stack alignment 1810 | JMP_IP TMP1 1811 } else { 1812 | ldp FP, RX, T2 // restore FP and IP 1813 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1814 | mov RETVALx, #1 // ZEND_VM_ENTER 1815 | ret 1816 } 1817 1818 return 1; 1819} 1820 1821static int zend_jit_exception_handler_stub(dasm_State **Dst) 1822{ 1823 |->exception_handler: 1824 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1825 const void *handler = zend_get_opcode_handler_func(EG(exception_op)); 1826 1827 | ADD_HYBRID_SPAD 1828 | EXT_CALL handler, REG0 1829 | JMP_IP TMP1 1830 } else { 1831 const void *handler = EG(exception_op)->handler; 1832 1833 if (GCC_GLOBAL_REGS) { 1834 | ldp x29, x30, [sp], # SPAD // stack alignment 1835 | EXT_JMP handler, REG0 1836 } else { 1837 | mov FCARG1x, FP 1838 | EXT_CALL handler, REG0 1839 | ldp FP, RX, T2 // restore FP and IP 1840 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1841 | tbnz RETVALw, #31, >1 1842 | mov RETVALw, #1 // ZEND_VM_ENTER 1843 |1: 1844 | ret 1845 } 1846 } 1847 1848 return 1; 1849} 1850 1851static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) 1852{ 1853 |->exception_handler_undef: 1854 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, opline_before_exception, REG0 1855 | ldrb TMP1w, OP:REG0->result_type 1856 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1857 | beq >1 1858 | ldr REG0w, OP:REG0->result.var 1859 | add REG0, REG0, FP 1860 | SET_Z_TYPE_INFO REG0, IS_UNDEF, TMP1w 1861 |1: 1862 | b ->exception_handler 1863 1864 return 1; 1865} 1866 1867static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst) 1868{ 1869 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 1870 1871 |->exception_handler_free_op1_op2: 1872 | UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w 1873 | ldrb TMP1w, OP:RX->op1_type 1874 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1875 | beq >9 1876 | ldr REG0w, OP:RX->op1.var 1877 | add REG0, REG0, FP 1878 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 1879 |9: 1880 | ldrb TMP1w, OP:RX->op2_type 1881 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1882 | beq >9 1883 | ldr REG0w, OP:RX->op2.var 1884 | add REG0, REG0, FP 1885 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 1886 |9: 1887 | b ->exception_handler 1888 return 1; 1889} 1890 1891static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst) 1892{ 1893 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 1894 1895 |->exception_handler_free_op2: 1896 | MEM_LOAD_64_ZTS ldr, RX, executor_globals, opline_before_exception, REG0 1897 | UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w 1898 | ldrb TMP1w, OP:RX->op2_type 1899 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1900 | beq >9 1901 | ldr REG0w, OP:RX->op2.var 1902 | add REG0, REG0, FP 1903 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 1904 |9: 1905 | b ->exception_handler 1906 return 1; 1907} 1908 1909static int zend_jit_leave_function_stub(dasm_State **Dst) 1910{ 1911 |->leave_function_handler: 1912 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1913 | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w 1914 | bne >1 1915 | EXT_CALL zend_jit_leave_nested_func_helper, REG0 1916 | ADD_HYBRID_SPAD 1917 | JMP_IP TMP1 1918 |1: 1919 | EXT_CALL zend_jit_leave_top_func_helper, REG0 1920 | ADD_HYBRID_SPAD 1921 | JMP_IP TMP1 1922 } else { 1923 if (GCC_GLOBAL_REGS) { 1924 | ldp x29, x30, [sp], # SPAD // stack alignment 1925 } else { 1926 | mov FCARG2x, FP 1927 | ldp FP, RX, T2 // restore FP and IP 1928 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1929 } 1930 | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w 1931 | bne >1 1932 | EXT_JMP zend_jit_leave_nested_func_helper, REG0 1933 |1: 1934 | EXT_JMP zend_jit_leave_top_func_helper, REG0 1935 } 1936 1937 return 1; 1938} 1939 1940static int zend_jit_leave_throw_stub(dasm_State **Dst) 1941{ 1942 |->leave_throw_handler: 1943 | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { 1944 if (GCC_GLOBAL_REGS) { 1945 | ldrb TMP1w, OP:IP->opcode 1946 | cmp TMP1w, #ZEND_HANDLE_EXCEPTION 1947 | beq >5 1948 | // EG(opline_before_exception) = opline; 1949 | MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2 1950 |5: 1951 | // opline = EG(exception_op); 1952 | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 1953 | str IP, EX->opline 1954 | // HANDLE_EXCEPTION() 1955 | b ->exception_handler 1956 } else { 1957 | GET_IP TMP1 1958 | ldrb TMP2w, OP:TMP1->opcode 1959 | cmp TMP2w, #ZEND_HANDLE_EXCEPTION 1960 | beq >5 1961 | // EG(opline_before_exception) = opline; 1962 | MEM_STORE_64_ZTS str, TMP1, executor_globals, opline_before_exception, TMP2 1963 |5: 1964 | // opline = EG(exception_op); 1965 | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 1966 | ldp FP, RX, T2 // restore FP and IP 1967 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1968 | mov RETVALx, #2 // ZEND_VM_LEAVE 1969 | ret 1970 } 1971 1972 return 1; 1973} 1974 1975static int zend_jit_icall_throw_stub(dasm_State **Dst) 1976{ 1977 |->icall_throw_handler: 1978 | // zend_rethrow_exception(zend_execute_data *execute_data) 1979 | ldr IP, EX->opline 1980 | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { 1981 | ldrb TMP1w, OP:IP->opcode 1982 | cmp TMP1w, #ZEND_HANDLE_EXCEPTION 1983 | beq >1 1984 | // EG(opline_before_exception) = opline; 1985 | MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2 1986 |1: 1987 | // opline = EG(exception_op); 1988 | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 1989 || if (GCC_GLOBAL_REGS) { 1990 | str IP, EX->opline 1991 || } 1992 | // HANDLE_EXCEPTION() 1993 | b ->exception_handler 1994 1995 return 1; 1996} 1997 1998static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) 1999{ 2000 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 2001 2002 |->throw_cannot_pass_by_ref: 2003 | ldr REG0, EX->opline 2004 | ldr REG1w, OP:REG0->result.var 2005 | add REG1, REG1, RX 2006 | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w 2007 | // last EX(call) frame may be delayed 2008 | ldr TMP1, EX->call 2009 | cmp RX, TMP1 2010 | beq >1 2011 | ldr REG1, EX->call 2012 | str REG1, EX:RX->prev_execute_data 2013 | str RX, EX->call 2014 |1: 2015 | mov RX, REG0 2016 | ldr FCARG1w, OP:REG0->op2.num 2017 | EXT_CALL zend_cannot_pass_by_reference, REG0 2018 | ldrb TMP1w, OP:RX->op1_type 2019 | cmp TMP1w, #IS_TMP_VAR 2020 | bne >9 2021 | ldr REG0w, OP:RX->op1.var 2022 | add REG0, REG0, FP 2023 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 2024 |9: 2025 | b ->exception_handler 2026 2027 return 1; 2028} 2029 2030static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) 2031{ 2032 |->undefined_offset_ex: 2033 | SAVE_IP 2034 | b ->undefined_offset 2035 2036 return 1; 2037} 2038 2039static int zend_jit_undefined_offset_stub(dasm_State **Dst) 2040{ 2041 |->undefined_offset: 2042 || if (!GCC_GLOBAL_REGS) { 2043 | mov FCARG1x, FP 2044 || } 2045 | EXT_JMP zend_jit_undefined_long_key, REG0 2046 2047 return 1; 2048} 2049 2050static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) 2051{ 2052 |->undefined_index_ex: 2053 | SAVE_IP 2054 | b ->undefined_index 2055 2056 return 1; 2057} 2058 2059static int zend_jit_undefined_index_stub(dasm_State **Dst) 2060{ 2061 |->undefined_index: 2062 || if (!GCC_GLOBAL_REGS) { 2063 | mov FCARG1x, FP 2064 || } 2065 | EXT_JMP zend_jit_undefined_string_key, REG0 2066 2067 return 1; 2068} 2069 2070static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) 2071{ 2072 |->cannot_add_element_ex: 2073 | SAVE_IP 2074 | b ->cannot_add_element 2075 2076 return 1; 2077} 2078 2079static int zend_jit_cannot_add_element_stub(dasm_State **Dst) 2080{ 2081 |->cannot_add_element: 2082 | // sub r4, 8 2083 | ldr REG0, EX->opline 2084 | ldrb TMP1w, OP:REG0->result_type 2085 | cmp TMP1w, #IS_UNUSED 2086 | beq >1 2087 | ldr REG0w, OP:REG0->result.var 2088 | add REG0, REG0, FP 2089 | SET_Z_TYPE_INFO REG0, IS_NULL, TMP1w 2090 |1: 2091 | mov CARG1, xzr 2092 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2093 | EXT_JMP zend_throw_error, REG0 // tail call 2094 | // add r4, 8 2095 | //ret 2096 2097 return 1; 2098} 2099 2100static int zend_jit_undefined_function_stub(dasm_State **Dst) 2101{ 2102 |->undefined_function: 2103 | ldr REG0, EX->opline 2104 | mov CARG1, xzr 2105 | LOAD_ADDR CARG2, "Call to undefined function %s()" 2106 | ldrsw CARG3, [REG0, #offsetof(zend_op, op2.constant)] 2107 | ldr CARG3, [REG0, CARG3] 2108 | add CARG3, CARG3, #offsetof(zend_string, val) 2109#ifdef __APPLE__ 2110 | str CARG3, [sp, #-16]! 2111#endif 2112 | EXT_CALL zend_throw_error, REG0 2113#ifdef __APPLE__ 2114 | add sp, sp, #16 2115#endif 2116 | b ->exception_handler 2117 return 1; 2118} 2119 2120static int zend_jit_negative_shift_stub(dasm_State **Dst) 2121{ 2122 |->negative_shift: 2123 | ldr RX, EX->opline 2124 | LOAD_ADDR CARG1, zend_ce_arithmetic_error 2125 | LOAD_ADDR CARG2, "Bit shift by negative number" 2126 | EXT_CALL zend_throw_error, REG0 2127 | b ->exception_handler_free_op1_op2 2128 return 1; 2129} 2130 2131static int zend_jit_mod_by_zero_stub(dasm_State **Dst) 2132{ 2133 |->mod_by_zero: 2134 | ldr RX, EX->opline 2135 | LOAD_ADDR CARG1, zend_ce_division_by_zero_error 2136 | LOAD_ADDR CARG2, "Modulo by zero" 2137 | EXT_CALL zend_throw_error, REG0 2138 | b ->exception_handler_free_op1_op2 2139 return 1; 2140} 2141 2142static int zend_jit_invalid_this_stub(dasm_State **Dst) 2143{ 2144 |->invalid_this: 2145 | UNDEF_OPLINE_RESULT TMP1w 2146 | mov CARG1, xzr 2147 | LOAD_ADDR CARG2, "Using $this when not in object context" 2148 | EXT_CALL zend_throw_error, REG0 2149 | b ->exception_handler 2150 2151 return 1; 2152} 2153 2154static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) 2155{ 2156 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2157 return 1; 2158 } 2159 2160 |->hybrid_runtime_jit: 2161 | EXT_CALL zend_runtime_jit, REG0 2162 | JMP_IP TMP1 2163 return 1; 2164} 2165 2166static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) 2167{ 2168 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2169 return 1; 2170 } 2171 2172 |->hybrid_profile_jit: 2173 | // ++zend_jit_profile_counter; 2174 | LOAD_ADDR REG0, &zend_jit_profile_counter 2175 | ldr TMP1, [REG0] 2176 | add TMP1, TMP1, #1 2177 | str TMP1, [REG0] 2178 | // op_array = (zend_op_array*)EX(func); 2179 | ldr REG0, EX->func 2180 | // run_time_cache = EX(run_time_cache); 2181 | ldr REG2, EX->run_time_cache 2182 | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); 2183 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2184 | // ++ZEND_COUNTER_INFO(op_array) 2185 || if ((zend_jit_profile_counter_rid * sizeof(void*)) > LDR_STR_PIMM64) { 2186 | LOAD_32BIT_VAL TMP1, (zend_jit_profile_counter_rid * sizeof(void*)) 2187 | ldr TMP2, [REG2, TMP1] 2188 | add TMP2, TMP2, #1 2189 | str TMP2, [REG2, TMP1] 2190 || } else { 2191 | ldr TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] 2192 | add TMP2, TMP2, #1 2193 | str TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] 2194 || } 2195 | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() 2196 | ldr TMP1, [REG0, #offsetof(zend_jit_op_array_extension, orig_handler)] 2197 | br TMP1 2198 return 1; 2199} 2200 2201static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) 2202{ 2203 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2204 return 1; 2205 } 2206 2207 |->hybrid_hot_code: 2208 || ZEND_ASSERT(ZEND_JIT_COUNTER_INIT <= MOVZ_IMM); 2209 | movz TMP1w, #ZEND_JIT_COUNTER_INIT 2210 | strh TMP1w, [REG2] 2211 | mov FCARG1x, FP 2212 | GET_IP FCARG2x 2213 | EXT_CALL zend_jit_hot_func, REG0 2214 | JMP_IP TMP1 2215 return 1; 2216} 2217 2218/* 2219 * This code is based Mike Pall's "Hashed profile counters" idea, implemented 2220 * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual 2221 * property disclosure and research opportunities" email 2222 * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html 2223 * 2224 * In addition we use a variation of Knuth's multiplicative hash function 2225 * described at https://code.i-harness.com/en/q/a21ce 2226 * 2227 * uint64_t hash(uint64_t x) { 2228 * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; 2229 * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; 2230 * x = x ^ (x >> 31); 2231 * return x; 2232 * } 2233 * 2234 * uint_32_t hash(uint32_t x) { 2235 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2236 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2237 * x = (x >> 16) ^ x; 2238 * return x; 2239 * } 2240 * 2241 */ 2242static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) 2243{ 2244 | ldr REG0, EX->func 2245 | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2246 | ldr REG2, [REG1, #offsetof(zend_jit_op_array_hot_extension, counter)] 2247 | ldrh TMP2w, [REG2] 2248 | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w 2249 | strh TMP2w, [REG2] 2250 | ble ->hybrid_hot_code 2251 | GET_IP REG2 2252 | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] 2253 | sub REG2, REG2, TMP1 2254 | // divide by sizeof(zend_op) 2255 || ZEND_ASSERT(sizeof(zend_op) == 32); 2256 | add TMP1, REG1, REG2, asr #2 2257 | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2258 | br TMP1 2259 return 1; 2260} 2261 2262static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) 2263{ 2264 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2265 return 1; 2266 } 2267 2268 |->hybrid_func_hot_counter: 2269 2270 return zend_jit_hybrid_hot_counter_stub(Dst, 2271 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2272} 2273 2274static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) 2275{ 2276 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2277 return 1; 2278 } 2279 2280 |->hybrid_loop_hot_counter: 2281 2282 return zend_jit_hybrid_hot_counter_stub(Dst, 2283 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2284} 2285 2286static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) 2287{ 2288 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2289 return 1; 2290 } 2291 2292 // On entry from counter stub: 2293 // REG2 -> zend_op_trace_info.counter 2294 2295 |->hybrid_hot_trace: 2296 | mov TMP1w, #ZEND_JIT_COUNTER_INIT 2297 | strh TMP1w, [REG2] 2298 | mov FCARG1x, FP 2299 | GET_IP FCARG2x 2300 | EXT_CALL zend_jit_trace_hot_root, REG0 2301 | tbnz RETVALw, #31, >1 // Result is < 0 on failure. 2302 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 2303 | LOAD_IP 2304 | JMP_IP TMP1 2305 |1: 2306 | EXT_JMP zend_jit_halt_op->handler, REG0 2307 2308 return 1; 2309} 2310 2311static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) 2312{ 2313 | ldr REG0, EX->func 2314 | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2315 | ldr REG1, [REG1, #offsetof(zend_jit_op_array_trace_extension, offset)] 2316 | add TMP1, REG1, IP 2317 | ldr REG2, [TMP1, #offsetof(zend_op_trace_info, counter)] 2318 | ldrh TMP2w, [REG2] 2319 | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w 2320 | strh TMP2w, [REG2] 2321 | ble ->hybrid_hot_trace 2322 // Note: "REG1 + IP" is re-calculated as TMP1 is used as temporary register by the prior 2323 // ADD_SUB_32_WITH_CONST. Will optimize in the future if more temporary registers are available. 2324 | add TMP1, REG1, IP 2325 | ldr TMP2, [TMP1, #offsetof(zend_op_trace_info, orig_handler)] 2326 | br TMP2 2327 2328 return 1; 2329} 2330 2331static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) 2332{ 2333 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2334 return 1; 2335 } 2336 2337 |->hybrid_func_trace_counter: 2338 2339 return zend_jit_hybrid_trace_counter_stub(Dst, 2340 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2341} 2342 2343static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) 2344{ 2345 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { 2346 return 1; 2347 } 2348 2349 |->hybrid_ret_trace_counter: 2350 2351 return zend_jit_hybrid_trace_counter_stub(Dst, 2352 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); 2353} 2354 2355static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) 2356{ 2357 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2358 return 1; 2359 } 2360 2361 |->hybrid_loop_trace_counter: 2362 2363 return zend_jit_hybrid_trace_counter_stub(Dst, 2364 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2365} 2366 2367static int zend_jit_trace_halt_stub(dasm_State **Dst) 2368{ 2369 |->trace_halt: 2370 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2371 | ADD_HYBRID_SPAD 2372 | EXT_JMP zend_jit_halt_op->handler, REG0 2373 } else if (GCC_GLOBAL_REGS) { 2374 | ldp x29, x30, [sp], # SPAD // stack alignment 2375 | mov IP, xzr // PC must be zero 2376 | ret 2377 } else { 2378 | ldp FP, RX, T2 // restore FP and IP 2379 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2380 | movn RETVALx, #0 // ZEND_VM_RETURN (-1) 2381 | ret 2382 } 2383 return 1; 2384} 2385 2386static int zend_jit_trace_exit_stub(dasm_State **Dst) 2387{ 2388 |->trace_exit: 2389 | 2390 | // Save CPU registers(32 GP regs + 32 FP regs) on stack in the order of d31 to x0 2391 | 2392 | stp d30, d31, [sp, #-16]! 2393 | stp d28, d29, [sp, #-16]! 2394 | stp d26, d27, [sp, #-16]! 2395 | stp d24, d25, [sp, #-16]! 2396 | stp d22, d23, [sp, #-16]! 2397 | stp d20, d21, [sp, #-16]! 2398 | stp d18, d19, [sp, #-16]! 2399 | stp d16, d17, [sp, #-16]! 2400 | //stp d14, d15, [sp, #-16]! // we don't use preserved registers yet 2401 | //stp d12, d13, [sp, #-16]! 2402 | //stp d10, d11, [sp, #-16]! 2403 | //stp d8, d9, [sp, #-16]! 2404 | stp d6, d7, [sp, #(-16*5)]! 2405 | stp d4, d5, [sp, #-16]! 2406 | stp d2, d3, [sp, #-16]! 2407 | stp d0, d1, [sp, #-16]! 2408 | 2409 | //str x30, [sp, #-16]! // we don't use callee-saved registers yet (x31 can be omitted) 2410 | stp x28, x29, [sp, #(-16*2)]! // we have to store RX (x28) 2411 | //stp x26, x27, [sp, #-16]! // we don't use callee-saved registers yet 2412 | //stp x24, x25, [sp, #-16]! 2413 | //stp x22, x23, [sp, #-16]! 2414 | //stp x20, x21, [sp, #-16]! 2415 | //stp x18, x19, [sp, #-16]! 2416 | //stp x16, x17, [sp, #-16]! // we don't need temporary registers 2417 | stp x14, x15, [sp, #-(16*7)]! 2418 | stp x12, x13, [sp, #-16]! 2419 | stp x10, x11, [sp, #-16]! 2420 | stp x8, x9, [sp, #-16]! 2421 | stp x6, x7, [sp, #-16]! 2422 | stp x4, x5, [sp, #-16]! 2423 | stp x2, x3, [sp, #-16]! 2424 | stp x0, x1, [sp, #-16]! 2425 | 2426 | mov FCARG1w, TMP1w // exit_num 2427 | mov FCARG2x, sp 2428 | 2429 | // EX(opline) = opline 2430 | SAVE_IP 2431 | // zend_jit_trace_exit(trace_num, exit_num) 2432 | EXT_CALL zend_jit_trace_exit, REG0 2433 | 2434 | add sp, sp, #(32 * 16) // restore sp 2435 | 2436 2437 | tst RETVALw, RETVALw 2438 | bne >1 // not zero 2439 2440 | // execute_data = EG(current_execute_data) 2441 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 2442 | // opline = EX(opline) 2443 | LOAD_IP 2444 2445 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2446 | ADD_HYBRID_SPAD 2447 | JMP_IP TMP1 2448 } else if (GCC_GLOBAL_REGS) { 2449 | ldp x29, x30, [sp], # SPAD // stack alignment 2450 | JMP_IP TMP1 2451 } else { 2452 | ldp FP, RX, T2 // restore FP and IP 2453 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2454 | mov RETVALx, #1 // ZEND_VM_ENTER 2455 | ret 2456 } 2457 2458 |1: 2459 | blt ->trace_halt 2460 2461 | // execute_data = EG(current_execute_data) 2462 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 2463 | // opline = EX(opline) 2464 | LOAD_IP 2465 2466 | // check for interrupt (try to avoid this ???) 2467 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 2468 | cbnz TMP1w, ->interrupt_handler 2469 2470 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2471 | ADD_HYBRID_SPAD 2472 | ldr REG0, EX->func 2473 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2474 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 2475 | ldr REG0, [IP, REG0] 2476 | br REG0 2477 } else if (GCC_GLOBAL_REGS) { 2478 | ldp x29, x30, [sp], # SPAD // stack alignment 2479 | ldr REG0, EX->func 2480 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2481 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 2482 | ldr REG0, [IP, REG0] 2483 | br REG0 2484 } else { 2485 | ldr IP, EX->opline 2486 | mov FCARG1x, FP 2487 | ldr REG0, EX->func 2488 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2489 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 2490 | ldr REG0, [IP, REG0] 2491 | blr REG0 2492 | 2493 | tst RETVALw, RETVALw 2494 | blt ->trace_halt 2495 | 2496 | ldp FP, RX, T2 // restore FP and IP 2497 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2498 | mov RETVALx, #1 // ZEND_VM_ENTER 2499 | ret 2500 } 2501 2502 return 1; 2503} 2504 2505static int zend_jit_trace_escape_stub(dasm_State **Dst) 2506{ 2507 |->trace_escape: 2508 | 2509 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2510 | ADD_HYBRID_SPAD 2511 | JMP_IP, TMP1 2512 } else if (GCC_GLOBAL_REGS) { 2513 | ldp x29, x30, [sp], # SPAD // stack alignment 2514 | JMP_IP, TMP1 2515 } else { 2516 | ldp FP, RX, T2 // restore FP and IP 2517 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2518 | mov RETVALx, #1 // ZEND_VM_ENTER 2519 | ret 2520 } 2521 2522 return 1; 2523} 2524 2525/* Keep 32 exit points in a single code block */ 2526#define ZEND_JIT_EXIT_POINTS_SPACING 4 // bl = bytes 2527#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points 2528 2529static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) 2530{ 2531 uint32_t i; 2532 2533 | bl >2 2534 |1: 2535 for (i = 1; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { 2536 | bl >2 2537 } 2538 |2: 2539 | adr TMP1, <1 2540 | sub TMP1, lr, TMP1 2541 | lsr TMP1, TMP1, #2 2542 if (n) { 2543 | ADD_SUB_32_WITH_CONST add, TMP1w, TMP1w, n, TMP2w 2544 } 2545 | b ->trace_exit // pass exit_num in TMP1w 2546 2547 return 1; 2548} 2549 2550#ifdef CONTEXT_THREADED_JIT 2551static int zend_jit_context_threaded_call_stub(dasm_State **Dst) 2552{ 2553 |->context_threaded_call: 2554 | NIY_STUB // TODO 2555 return 1; 2556} 2557#endif 2558 2559static int zend_jit_assign_const_stub(dasm_State **Dst) 2560{ 2561 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2562 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2563 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2564 2565 |->assign_const: 2566 | stp x29, x30, [sp, #-32]! 2567 | mov x29, sp 2568 if (!zend_jit_assign_to_variable( 2569 Dst, NULL, 2570 var_addr, var_addr, -1, -1, 2571 IS_CONST, val_addr, val_info, 2572 0, 0)) { 2573 return 0; 2574 } 2575 | ldp x29, x30, [sp], #32 2576 | ret 2577 return 1; 2578} 2579 2580static int zend_jit_assign_tmp_stub(dasm_State **Dst) 2581{ 2582 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2583 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2584 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2585 2586 |->assign_tmp: 2587 | stp x29, x30, [sp, #-32]! 2588 | mov x29, sp 2589 if (!zend_jit_assign_to_variable( 2590 Dst, NULL, 2591 var_addr, var_addr, -1, -1, 2592 IS_TMP_VAR, val_addr, val_info, 2593 0, 0)) { 2594 return 0; 2595 } 2596 | ldp x29, x30, [sp], #32 2597 | ret 2598 return 1; 2599} 2600 2601static int zend_jit_assign_var_stub(dasm_State **Dst) 2602{ 2603 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2604 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2605 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; 2606 2607 |->assign_var: 2608 | stp x29, x30, [sp, #-32]! 2609 | mov x29, sp 2610 if (!zend_jit_assign_to_variable( 2611 Dst, NULL, 2612 var_addr, var_addr, -1, -1, 2613 IS_VAR, val_addr, val_info, 2614 0, 0)) { 2615 return 0; 2616 } 2617 | ldp x29, x30, [sp], #32 2618 | ret 2619 return 1; 2620} 2621 2622static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) 2623{ 2624 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2625 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2626 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; 2627 2628 |->assign_cv_noref: 2629 | stp x29, x30, [sp, #-32]! 2630 | mov x29, sp 2631 if (!zend_jit_assign_to_variable( 2632 Dst, NULL, 2633 var_addr, var_addr, -1, -1, 2634 IS_CV, val_addr, val_info, 2635 0, 0)) { 2636 return 0; 2637 } 2638 | ldp x29, x30, [sp], #32 2639 | ret 2640 return 1; 2641} 2642 2643static int zend_jit_assign_cv_stub(dasm_State **Dst) 2644{ 2645 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2646 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2647 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; 2648 2649 |->assign_cv: 2650 | stp x29, x30, [sp, #-32]! 2651 | mov x29, sp 2652 if (!zend_jit_assign_to_variable( 2653 Dst, NULL, 2654 var_addr, var_addr, -1, -1, 2655 IS_CV, val_addr, val_info, 2656 0, 0)) { 2657 return 0; 2658 } 2659 | ldp x29, x30, [sp], #32 2660 | ret 2661 return 1; 2662} 2663 2664static const zend_jit_stub zend_jit_stubs[] = { 2665 JIT_STUB(interrupt_handler, SP_ADJ_JIT, SP_ADJ_VM), 2666 JIT_STUB(exception_handler, SP_ADJ_JIT, SP_ADJ_VM), 2667 JIT_STUB(exception_handler_undef, SP_ADJ_JIT, SP_ADJ_VM), 2668 JIT_STUB(exception_handler_free_op1_op2, SP_ADJ_JIT, SP_ADJ_VM), 2669 JIT_STUB(exception_handler_free_op2, SP_ADJ_JIT, SP_ADJ_VM), 2670 JIT_STUB(leave_function, SP_ADJ_JIT, SP_ADJ_VM), 2671 JIT_STUB(leave_throw, SP_ADJ_JIT, SP_ADJ_VM), 2672 JIT_STUB(icall_throw, SP_ADJ_JIT, SP_ADJ_VM), 2673 JIT_STUB(throw_cannot_pass_by_ref, SP_ADJ_JIT, SP_ADJ_VM), 2674 JIT_STUB(undefined_offset, SP_ADJ_JIT, SP_ADJ_VM), 2675 JIT_STUB(undefined_index, SP_ADJ_JIT, SP_ADJ_VM), 2676 JIT_STUB(cannot_add_element, SP_ADJ_JIT, SP_ADJ_VM), 2677 JIT_STUB(undefined_offset_ex, SP_ADJ_JIT, SP_ADJ_VM), 2678 JIT_STUB(undefined_index_ex, SP_ADJ_JIT, SP_ADJ_VM), 2679 JIT_STUB(cannot_add_element_ex, SP_ADJ_JIT, SP_ADJ_VM), 2680 JIT_STUB(undefined_function, SP_ADJ_JIT, SP_ADJ_VM), 2681 JIT_STUB(negative_shift, SP_ADJ_JIT, SP_ADJ_VM), 2682 JIT_STUB(mod_by_zero, SP_ADJ_JIT, SP_ADJ_VM), 2683 JIT_STUB(invalid_this, SP_ADJ_JIT, SP_ADJ_VM), 2684 JIT_STUB(trace_halt, SP_ADJ_JIT, SP_ADJ_VM), 2685 JIT_STUB(trace_exit, SP_ADJ_JIT, SP_ADJ_VM), 2686 JIT_STUB(trace_escape, SP_ADJ_JIT, SP_ADJ_VM), 2687 JIT_STUB(hybrid_runtime_jit, SP_ADJ_VM, SP_ADJ_NONE), 2688 JIT_STUB(hybrid_profile_jit, SP_ADJ_VM, SP_ADJ_NONE), 2689 JIT_STUB(hybrid_hot_code, SP_ADJ_VM, SP_ADJ_NONE), 2690 JIT_STUB(hybrid_func_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2691 JIT_STUB(hybrid_loop_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2692 JIT_STUB(hybrid_hot_trace, SP_ADJ_VM, SP_ADJ_NONE), 2693 JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2694 JIT_STUB(hybrid_ret_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2695 JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2696 JIT_STUB(assign_const, SP_ADJ_RET, SP_ADJ_ASSIGN), 2697 JIT_STUB(assign_tmp, SP_ADJ_RET, SP_ADJ_ASSIGN), 2698 JIT_STUB(assign_var, SP_ADJ_RET, SP_ADJ_ASSIGN), 2699 JIT_STUB(assign_cv_noref, SP_ADJ_RET, SP_ADJ_ASSIGN), 2700 JIT_STUB(assign_cv, SP_ADJ_RET, SP_ADJ_ASSIGN), 2701#ifdef CONTEXT_THREADED_JIT 2702 JIT_STUB(context_threaded_call, SP_ADJ_NONE, SP_ADJ_NONE), 2703#endif 2704}; 2705 2706#ifdef HAVE_GDB 2707# if 0 2708typedef struct _Unwind_Context _Unwind_Context; 2709typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *); 2710extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *); 2711extern uintptr_t _Unwind_GetCFA(_Unwind_Context *); 2712 2713typedef struct _zend_jit_unwind_arg { 2714 int cnt; 2715 uintptr_t cfa[3]; 2716} zend_jit_unwind_arg; 2717 2718static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a) 2719{ 2720 zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a; 2721 arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx); 2722 arg->cnt++; 2723 if (arg->cnt == 3) { 2724 return 5; // _URC_END_OF_STACK 2725 } 2726 return 0; // _URC_NO_REASON; 2727} 2728 2729static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) 2730{ 2731 zend_jit_unwind_arg arg; 2732 2733 memset(&arg, 0, sizeof(arg)); 2734 _Unwind_Backtrace(zend_jit_unwind_cb, &arg); 2735 if (arg.cnt == 3) { 2736 sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1]; 2737 } 2738} 2739# else 2740static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) 2741{ 2742 uintptr_t ret; 2743 2744 __asm__ ( 2745 "ldr %0, [x29]\n\t" 2746 "sub %0 ,%0, x29" 2747 : "=r"(ret)); 2748 2749 sp_adj[SP_ADJ_VM] = ret; 2750} 2751# endif 2752 2753extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data); 2754 2755static zend_never_inline void zend_jit_set_sp_adj_vm(void) 2756{ 2757 void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *); 2758 2759 orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data; 2760 zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data; 2761 execute_ex(NULL); // set sp_adj[SP_ADJ_VM] 2762 zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data; 2763} 2764#endif 2765 2766static int zend_jit_setup(void) 2767{ 2768 allowed_opt_flags = 0; 2769 2770#if ZTS 2771 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2772# if defined(__FreeBSD__) 2773 if (tsrm_ls_cache_tcb_offset == 0) { 2774 TLSDescriptor **where; 2775 2776 __asm__( 2777 "adrp %0, :tlsdesc:_tsrm_ls_cache\n" 2778 "add %0, %0, :tlsdesc_lo12:_tsrm_ls_cache\n" 2779 : "=r" (where)); 2780 /* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst 2781 * section "Relocations for thread-local storage". 2782 * The first entry holds a pointer to the variable's TLS descriptor resolver function and the second entry holds 2783 * a platform-specific offset or pointer. */ 2784 TLSDescriptor *tlsdesc = where[1]; 2785 2786 tsrm_tls_offset = tlsdesc->offset; 2787 /* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/22ca6db50f4e6bd75a141f57cf953d8de6531a06/lib/libc/gen/tls.c#L88) */ 2788 tsrm_tls_index = (tlsdesc->index + 1) * 8; 2789 } 2790# else 2791 ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0); 2792# endif 2793#endif 2794 2795 memset(sp_adj, 0, sizeof(sp_adj)); 2796#ifdef HAVE_GDB 2797 sp_adj[SP_ADJ_RET] = 0; 2798 sp_adj[SP_ADJ_ASSIGN] = 32; 2799 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2800 zend_jit_set_sp_adj_vm(); // set sp_adj[SP_ADJ_VM] 2801#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 2802 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD 2803#else 2804 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM]; 2805#endif 2806 } else if (GCC_GLOBAL_REGS) { 2807 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD; // sub r4, SPAD 2808 } else { 2809 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD; // sub r4, NR_SPAD 2810 } 2811#endif 2812 2813 return SUCCESS; 2814} 2815 2816static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) 2817{ 2818 | brk #0 2819 return 1; 2820} 2821 2822static int zend_jit_align_func(dasm_State **Dst) 2823{ 2824 reuse_ip = 0; 2825 delayed_call_chain = 0; 2826 last_valid_opline = NULL; 2827 use_last_vald_opline = 0; 2828 track_last_valid_opline = 0; 2829 jit_return_label = -1; 2830 |.align 16 2831 return 1; 2832} 2833 2834static int zend_jit_prologue(dasm_State **Dst) 2835{ 2836 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2837 | SUB_HYBRID_SPAD 2838 } else if (GCC_GLOBAL_REGS) { 2839 | stp x29, x30, [sp, # -SPAD]! // stack alignment 2840 |// mov x29, sp 2841 } else { 2842 | stp x29, x30, [sp, # -NR_SPAD]! // stack alignment 2843 |// mov x29, sp 2844 | stp FP, RX, T2 // save FP and IP 2845 | mov FP, FCARG1x 2846 } 2847 return 1; 2848} 2849 2850static int zend_jit_label(dasm_State **Dst, unsigned int label) 2851{ 2852 |=>label: 2853 return 1; 2854} 2855 2856static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) 2857{ 2858 | // call->prev_execute_data = EX(call); 2859 if (call_level == 1) { 2860 | str xzr, EX:RX->prev_execute_data 2861 } else { 2862 | ldr REG0, EX->call 2863 | str REG0, EX:RX->prev_execute_data 2864 } 2865 | // EX(call) = call; 2866 | str RX, EX->call 2867 2868 delayed_call_chain = 0; 2869 2870 return 1; 2871} 2872 2873static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) 2874{ 2875 if (last_valid_opline == opline) { 2876 zend_jit_use_last_valid_opline(); 2877 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 2878 zend_jit_use_last_valid_opline(); 2879 | LOAD_64BIT_VAL TMP1, (opline - last_valid_opline) * sizeof(zend_op) 2880 | ADD_IP TMP1, TMP2 2881 } else { 2882 | LOAD_IP_ADDR opline 2883 } 2884 zend_jit_set_last_valid_opline(opline); 2885 2886 return 1; 2887} 2888 2889static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg) 2890{ 2891 return zend_jit_set_ip(Dst, opline); 2892} 2893 2894static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) 2895{ 2896 if (delayed_call_chain) { 2897 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 2898 return 0; 2899 } 2900 } 2901 if (!zend_jit_set_ip(Dst, opline)) { 2902 return 0; 2903 } 2904 reuse_ip = 0; 2905 return 1; 2906} 2907 2908static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) 2909{ 2910 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 2911 if (exit_addr) { 2912 | cbnz TMP1w, &exit_addr 2913 } else if (last_valid_opline == opline) { 2914 || zend_jit_use_last_valid_opline(); 2915 | cbnz TMP1w, ->interrupt_handler 2916 } else { 2917 | cbnz TMP1w, >1 2918 |.cold_code 2919 |1: 2920 | LOAD_IP_ADDR opline 2921 | b ->interrupt_handler 2922 |.code 2923 } 2924 return 1; 2925} 2926 2927static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) 2928{ 2929 if (timeout_exit_addr) { 2930 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 2931 | cbz TMP1w, =>loop_label 2932 | b &timeout_exit_addr 2933 } else { 2934 | b =>loop_label 2935 } 2936 return 1; 2937} 2938 2939static int zend_jit_check_exception(dasm_State **Dst) 2940{ 2941 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 2942 | cbnz TMP2, ->exception_handler 2943 return 1; 2944} 2945 2946static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) 2947{ 2948 if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { 2949 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 2950 | cbnz TMP2, ->exception_handler_undef 2951 return 1; 2952 } 2953 return zend_jit_check_exception(Dst); 2954} 2955 2956static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) 2957{ 2958 2959 current_trace_num = trace_num; 2960 2961 | // EG(jit_trace_num) = trace_num; 2962 | LOAD_32BIT_VAL TMP1w, trace_num 2963 | MEM_STORE_32_ZTS str, TMP1w, executor_globals, jit_trace_num, TMP2 2964 2965 return 1; 2966} 2967 2968static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t) 2969{ 2970 uint32_t i; 2971 const void *exit_addr; 2972 2973 /* Emit veneers table for exit points (B instruction for each exit number) */ 2974 |.cold_code 2975 for (i = 0; i < t->exit_count; i++) { 2976 exit_addr = zend_jit_trace_get_exit_addr(i); 2977 if (!exit_addr) { 2978 return 0; 2979 } 2980 | b &exit_addr 2981 } 2982 |=>1: // end of the code 2983 |.code 2984 return 1; 2985} 2986 2987static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) 2988{ 2989 int ret = 0; 2990 uint8_t *p, *end; 2991 const void *veneer = NULL; 2992 ptrdiff_t delta; 2993 2994 if (jmp_table_size) { 2995 const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*))); 2996 2997 do { 2998 if (*jmp_slot == from_addr) { 2999 *jmp_slot = to_addr; 3000 ret++; 3001 } 3002 jmp_slot++; 3003 } while (--jmp_table_size); 3004 } 3005 3006 end = (uint8_t*)code; 3007 p = end + size; 3008 while (p > end) { 3009 uint32_t *ins_ptr; 3010 uint32_t ins; 3011 3012 p -= 4; 3013 ins_ptr = (uint32_t*)p; 3014 ins = *ins_ptr; 3015 if ((ins & 0xfc000000u) == 0x14000000u) { 3016 // B (imm26:0..25) 3017 delta = (uint32_t*)from_addr - ins_ptr; 3018 if (((ins ^ (uint32_t)delta) & 0x01ffffffu) == 0) { 3019 delta = (uint32_t*)to_addr - ins_ptr; 3020 if (((delta + 0x02000000) >> 26) != 0) { 3021 abort(); // branch target out of range 3022 } 3023 *ins_ptr = (ins & 0xfc000000u) | ((uint32_t)delta & 0x03ffffffu); 3024 ret++; 3025 if (!veneer) { 3026 veneer = p; 3027 } 3028 } 3029 } else if ((ins & 0xff000000u) == 0x54000000u || 3030 (ins & 0x7e000000u) == 0x34000000u) { 3031 // B.cond, CBZ, CBNZ (imm19:5..23) 3032 delta = (uint32_t*)from_addr - ins_ptr; 3033 if (((ins ^ ((uint32_t)delta << 5)) & 0x00ffffe0u) == 0) { 3034 delta = (uint32_t*)to_addr - ins_ptr; 3035 if (((delta + 0x40000) >> 19) != 0) { 3036 if (veneer) { 3037 delta = (uint32_t*)veneer - ins_ptr; 3038 if (((delta + 0x40000) >> 19) != 0) { 3039 abort(); // branch target out of range 3040 } 3041 } else { 3042 abort(); // branch target out of range 3043 } 3044 } 3045 *ins_ptr = (ins & 0xff00001fu) | (((uint32_t)delta & 0x7ffffu) << 5); 3046 ret++; 3047 } 3048 } else if ((ins & 0x7e000000u) == 0x36000000u) { 3049 // TBZ, TBNZ (imm14:5..18) 3050 delta = (uint32_t*)from_addr - ins_ptr; 3051 if (((ins ^ ((uint32_t)delta << 5)) & 0x0007ffe0u) == 0) { 3052 delta = (uint32_t*)to_addr - ins_ptr; 3053 if (((delta + 0x2000) >> 14) != 0) { 3054 if (veneer) { 3055 delta = (uint32_t*)veneer - ins_ptr; 3056 if (((delta + 0x2000) >> 14) != 0) { 3057 abort(); // branch target out of range 3058 } 3059 } else { 3060 abort(); // branch target out of range 3061 } 3062 } 3063 *ins_ptr = (ins & 0xfff8001fu) | (((uint32_t)delta & 0x3fffu) << 5); 3064 ret++; 3065 } 3066 } 3067 } 3068 3069 JIT_CACHE_FLUSH(code, (char*)code + size); 3070 3071#ifdef HAVE_VALGRIND 3072 VALGRIND_DISCARD_TRANSLATIONS(code, size); 3073#endif 3074 3075 return ret; 3076} 3077 3078static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) 3079{ 3080 return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); 3081} 3082 3083static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) 3084{ 3085 const void *link_addr; 3086 size_t prologue_size; 3087 3088 /* Skip prologue. */ 3089 // TODO: don't hardcode this ??? 3090 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3091#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 3092 prologue_size = 0; 3093#else 3094 // sub sp, sp, #0x20 3095 prologue_size = 4; 3096#endif 3097 } else if (GCC_GLOBAL_REGS) { 3098 // stp x29, x30, [sp, # -SPAD]! 3099 prologue_size = 4; 3100 } else { 3101 // stp x29, x30, [sp, # -NR_SPAD]! // stack alignment 3102 // stp FP, RX, T2 3103 // mov FP, FCARG1x 3104 prologue_size = 12; 3105 } 3106 link_addr = (const void*)((const char*)t->code_start + prologue_size); 3107 3108 if (timeout_exit_addr) { 3109 /* Check timeout for links to LOOP */ 3110 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 3111 | cbz TMP1w, &link_addr 3112 | b &timeout_exit_addr 3113 } else { 3114 | b &link_addr 3115 } 3116 return 1; 3117} 3118 3119static int zend_jit_trace_return(dasm_State **Dst, bool original_handler, const zend_op *opline) 3120{ 3121 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3122 | ADD_HYBRID_SPAD 3123 if (!original_handler) { 3124 | JMP_IP TMP1 3125 } else { 3126 | ldr REG0, EX->func 3127 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 3128 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 3129 | ldr REG0, [IP, REG0] 3130 | br REG0 3131 } 3132 } else if (GCC_GLOBAL_REGS) { 3133 | ldp x29, x30, [sp], # SPAD // stack alignment 3134 if (!original_handler) { 3135 | JMP_IP TMP1 3136 } else { 3137 | ldr REG0, EX->func 3138 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 3139 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 3140 | ldr REG0, [IP, REG0] 3141 | br REG0 3142 } 3143 } else { 3144 if (original_handler) { 3145 | mov FCARG1x, FP 3146 | ldr REG0, EX->func 3147 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 3148 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 3149 | ldr REG0, [IP, REG0] 3150 | blr REG0 3151 } 3152 | ldp FP, RX, T2 // restore FP and IP 3153 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 3154 if (!original_handler || !opline || 3155 (opline->opcode != ZEND_RETURN 3156 && opline->opcode != ZEND_RETURN_BY_REF 3157 && opline->opcode != ZEND_GENERATOR_RETURN 3158 && opline->opcode != ZEND_GENERATOR_CREATE 3159 && opline->opcode != ZEND_YIELD 3160 && opline->opcode != ZEND_YIELD_FROM)) { 3161 | mov RETVALx, #2 // ZEND_VM_LEAVE 3162 } 3163 | ret 3164 } 3165 return 1; 3166} 3167 3168static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) 3169{ 3170 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3171 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3172 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3173 3174 if (!exit_addr) { 3175 return 0; 3176 } 3177 3178 | IF_NOT_ZVAL_TYPE var_addr, type, &exit_addr, ZREG_TMP1 3179 3180 return 1; 3181} 3182 3183static int zend_jit_scalar_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var) 3184{ 3185 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3186 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3187 3188 if (!exit_addr) { 3189 return 0; 3190 } 3191 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, var+offsetof(zval, u1.v.type), TMP1 3192 | cmp TMP1w, #IS_STRING 3193 | bhs &exit_addr 3194 3195 return 1; 3196} 3197static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) 3198{ 3199 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 3200 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3201 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3202 3203 if (!exit_addr) { 3204 return 0; 3205 } 3206 3207 | GET_ZVAL_LVAL ZREG_FCARG1, var_addr, TMP1 3208 if (op_info & MAY_BE_ARRAY_PACKED) { 3209 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 3210 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 3211 | beq &exit_addr 3212 } else { 3213 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 3214 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 3215 | bne &exit_addr 3216 } 3217 3218 return 1; 3219} 3220 3221static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) 3222{ 3223 zend_jit_op_array_trace_extension *jit_extension = 3224 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); 3225 size_t offset = jit_extension->offset; 3226 const void *handler = 3227 (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; 3228 3229 if (!zend_jit_set_valid_ip(Dst, opline)) { 3230 return 0; 3231 } 3232 if (!GCC_GLOBAL_REGS) { 3233 | mov FCARG1x, FP 3234 } 3235 | EXT_CALL handler, REG0 3236 if (may_throw 3237 && opline->opcode != ZEND_RETURN 3238 && opline->opcode != ZEND_RETURN_BY_REF) { 3239 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 3240 | cbnz REG0, ->exception_handler 3241 } 3242 3243 while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { 3244 trace++; 3245 } 3246 3247 if (!GCC_GLOBAL_REGS 3248 && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { 3249 if (opline->opcode == ZEND_RETURN || 3250 opline->opcode == ZEND_RETURN_BY_REF || 3251 opline->opcode == ZEND_DO_UCALL || 3252 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3253 opline->opcode == ZEND_DO_FCALL || 3254 opline->opcode == ZEND_GENERATOR_CREATE) { 3255 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 3256 } 3257 } 3258 3259 if (zend_jit_trace_may_exit(op_array, opline)) { 3260 if (opline->opcode == ZEND_RETURN || 3261 opline->opcode == ZEND_RETURN_BY_REF || 3262 opline->opcode == ZEND_GENERATOR_CREATE) { 3263 3264 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3265#if 0 3266 /* this check should be handled by the following OPLINE guard or jmp [IP] */ 3267 | LOAD_ADDR TMP1, zend_jit_halt_op 3268 | cmp IP, TMP1 3269 | beq ->trace_halt 3270#endif 3271 } else if (GCC_GLOBAL_REGS) { 3272 | cbz IP, ->trace_halt 3273 } else { 3274 | tst RETVALw, RETVALw 3275 | blt ->trace_halt 3276 } 3277 } else if (opline->opcode == ZEND_EXIT || 3278 opline->opcode == ZEND_GENERATOR_RETURN || 3279 opline->opcode == ZEND_YIELD || 3280 opline->opcode == ZEND_YIELD_FROM) { 3281 | b ->trace_halt 3282 } 3283 if (trace->op != ZEND_JIT_TRACE_END || 3284 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && 3285 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 3286 3287 const zend_op *next_opline = trace->opline; 3288 const zend_op *exit_opline = NULL; 3289 uint32_t exit_point; 3290 const void *exit_addr; 3291 uint32_t old_info = 0; 3292 uint32_t old_res_info = 0; 3293 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 3294 3295 if (zend_is_smart_branch(opline)) { 3296 bool exit_if_true = 0; 3297 exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); 3298 } else { 3299 switch (opline->opcode) { 3300 case ZEND_JMPZ: 3301 case ZEND_JMPNZ: 3302 case ZEND_JMPZ_EX: 3303 case ZEND_JMPNZ_EX: 3304 case ZEND_JMP_SET: 3305 case ZEND_COALESCE: 3306 case ZEND_JMP_NULL: 3307 case ZEND_FE_RESET_R: 3308 case ZEND_FE_RESET_RW: 3309 exit_opline = (trace->opline == opline + 1) ? 3310 OP_JMP_ADDR(opline, opline->op2) : 3311 opline + 1; 3312 break; 3313 case ZEND_FE_FETCH_R: 3314 case ZEND_FE_FETCH_RW: 3315 exit_opline = (trace->opline == opline + 1) ? 3316 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3317 opline + 1; 3318 break; 3319 3320 } 3321 } 3322 3323 switch (opline->opcode) { 3324 case ZEND_FE_FETCH_R: 3325 case ZEND_FE_FETCH_RW: 3326 if (opline->op2_type != IS_UNUSED) { 3327 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); 3328 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); 3329 } 3330 break; 3331 } 3332 3333 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3334 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3335 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 3336 } 3337 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); 3338 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3339 3340 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3341 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3342 } 3343 switch (opline->opcode) { 3344 case ZEND_FE_FETCH_R: 3345 case ZEND_FE_FETCH_RW: 3346 if (opline->op2_type != IS_UNUSED) { 3347 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); 3348 } 3349 break; 3350 } 3351 3352 if (!exit_addr) { 3353 return 0; 3354 } 3355 | CMP_IP next_opline, TMP1, TMP2 3356 | bne &exit_addr 3357 } 3358 } 3359 3360 zend_jit_set_last_valid_opline(trace->opline); 3361 3362 return 1; 3363} 3364 3365static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) 3366{ 3367 const void *handler; 3368 3369 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3370 handler = zend_get_opcode_handler_func(opline); 3371 } else { 3372 handler = opline->handler; 3373 } 3374 3375 if (!zend_jit_set_valid_ip(Dst, opline)) { 3376 return 0; 3377 } 3378 if (!GCC_GLOBAL_REGS) { 3379 | mov FCARG1x, FP 3380 } 3381 | EXT_CALL handler, REG0 3382 if (may_throw) { 3383 zend_jit_check_exception(Dst); 3384 } 3385 3386 /* Skip the following OP_DATA */ 3387 switch (opline->opcode) { 3388 case ZEND_ASSIGN_DIM: 3389 case ZEND_ASSIGN_OBJ: 3390 case ZEND_ASSIGN_STATIC_PROP: 3391 case ZEND_ASSIGN_DIM_OP: 3392 case ZEND_ASSIGN_OBJ_OP: 3393 case ZEND_ASSIGN_STATIC_PROP_OP: 3394 case ZEND_ASSIGN_STATIC_PROP_REF: 3395 case ZEND_ASSIGN_OBJ_REF: 3396 zend_jit_set_last_valid_opline(opline + 2); 3397 break; 3398 default: 3399 zend_jit_set_last_valid_opline(opline + 1); 3400 break; 3401 } 3402 3403 return 1; 3404} 3405 3406static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) 3407{ 3408 if (!zend_jit_set_valid_ip(Dst, opline)) { 3409 return 0; 3410 } 3411 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3412 if (opline->opcode == ZEND_DO_UCALL || 3413 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3414 opline->opcode == ZEND_DO_FCALL || 3415 opline->opcode == ZEND_RETURN) { 3416 3417 /* Use inlined HYBRID VM handler */ 3418 const void *handler = opline->handler; 3419 3420 | ADD_HYBRID_SPAD 3421 | EXT_JMP handler, REG0 3422 } else { 3423 const void *handler = zend_get_opcode_handler_func(opline); 3424 3425 | EXT_CALL handler, REG0 3426 | ADD_HYBRID_SPAD 3427 | JMP_IP TMP1 3428 } 3429 } else { 3430 const void *handler = opline->handler; 3431 3432 if (GCC_GLOBAL_REGS) { 3433 | ldp x29, x30, [sp], # SPAD // stack alignment 3434 } else { 3435 | mov FCARG1x, FP 3436 | ldp FP, RX, T2 // restore FP and IP 3437 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 3438 } 3439 | EXT_JMP handler, REG0 3440 } 3441 zend_jit_reset_last_valid_opline(); 3442 return 1; 3443} 3444 3445static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) 3446{ 3447 uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); 3448 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3449 3450 if (!exit_addr) { 3451 return 0; 3452 } 3453 | CMP_IP opline, TMP1, TMP2 3454 | bne &exit_addr 3455 3456 zend_jit_set_last_valid_opline(opline); 3457 3458 return 1; 3459} 3460 3461static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) 3462{ 3463 | b =>target_label 3464 return 1; 3465} 3466 3467static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) 3468{ 3469 | CMP_IP next_opline, TMP1, TMP2 3470 | bne =>target_label 3471 3472 zend_jit_set_last_valid_opline(next_opline); 3473 3474 return 1; 3475} 3476 3477#ifdef CONTEXT_THREADED_JIT 3478static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3479{ 3480 | NIY // TODO 3481 return 1; 3482} 3483#endif 3484 3485static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3486{ 3487#ifdef CONTEXT_THREADED_JIT 3488 return zend_jit_context_threaded_call(Dst, opline, next_block); 3489#else 3490 return zend_jit_tail_handler(Dst, opline); 3491#endif 3492} 3493 3494static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type) 3495{ 3496 ZEND_ASSERT(Z_MODE(src) == IS_REG); 3497 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); 3498 3499 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3500 | SET_ZVAL_LVAL_FROM_REG dst, Rx(Z_REG(src)), TMP1 3501 if (set_type && 3502 (Z_REG(dst) != ZREG_FP || 3503 !JIT_G(current_frame) || 3504 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) { 3505 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 3506 } 3507 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3508 | SET_ZVAL_DVAL dst, Z_REG(src), ZREG_TMP1 3509 if (set_type && 3510 (Z_REG(dst) != ZREG_FP || 3511 !JIT_G(current_frame) || 3512 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) { 3513 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 3514 } 3515 } else { 3516 ZEND_UNREACHABLE(); 3517 } 3518 return 1; 3519} 3520 3521static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3522{ 3523 ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); 3524 ZEND_ASSERT(Z_MODE(dst) == IS_REG); 3525 3526 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3527 | GET_ZVAL_LVAL Z_REG(dst), src, TMP1 3528 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3529 | GET_ZVAL_DVAL Z_REG(dst), src, ZREG_TMP1 3530 } else { 3531 ZEND_UNREACHABLE(); 3532 } 3533 return 1; 3534} 3535 3536static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type) 3537{ 3538 zend_jit_addr src = ZEND_ADDR_REG(reg); 3539 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3540 3541 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3542} 3543 3544static int zend_jit_store_var_type(dasm_State **Dst, int var, uint32_t type) 3545{ 3546 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3547 3548 | SET_ZVAL_TYPE_INFO dst, type, TMP1w, TMP2 3549 return 1; 3550} 3551 3552static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) 3553{ 3554 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3555 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3556 return zend_jit_spill_store(Dst, src, dst, info, 1); 3557 } 3558 return 1; 3559} 3560 3561static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) 3562{ 3563 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3564 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3565 bool set_type = 1; 3566 3567 if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == 3568 (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { 3569 if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { 3570 set_type = 0; 3571 } 3572 } 3573 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3574 } 3575 return 1; 3576} 3577 3578static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) 3579{ 3580 zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3581 zend_jit_addr dst = ZEND_ADDR_REG(reg); 3582 3583 return zend_jit_load_reg(Dst, src, dst, info); 3584} 3585 3586static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, zend_uchar op_type, zend_jit_addr addr, znode_op op) 3587{ 3588 if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) { 3589 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 3590 | SET_ZVAL_TYPE_INFO dst, IS_UNDEF, TMP1w, TMP2 3591 } 3592 return 1; 3593} 3594 3595static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3596{ 3597 if (!zend_jit_same_addr(src, dst)) { 3598 if (Z_MODE(src) == IS_REG) { 3599 if (Z_MODE(dst) == IS_REG) { 3600 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3601 | mov Rx(Z_REG(dst)), Rx(Z_REG(src)) 3602 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3603 | fmov Rd(Z_REG(dst)-ZREG_V0), Rd(Z_REG(src)-ZREG_V0) 3604 } else { 3605 ZEND_UNREACHABLE(); 3606 } 3607 if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) { 3608 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3609 3610 if (!zend_jit_spill_store(Dst, dst, var_addr, info, 3611 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3612 JIT_G(current_frame) == NULL || 3613 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3614 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3615 )) { 3616 return 0; 3617 } 3618 } 3619 } else if (Z_MODE(dst) == IS_MEM_ZVAL) { 3620 if (!Z_LOAD(src) && !Z_STORE(src)) { 3621 if (!zend_jit_spill_store(Dst, src, dst, info, 3622 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3623 JIT_G(current_frame) == NULL || 3624 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3625 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3626 )) { 3627 return 0; 3628 } 3629 } 3630 } else { 3631 ZEND_UNREACHABLE(); 3632 } 3633 } else if (Z_MODE(src) == IS_MEM_ZVAL) { 3634 if (Z_MODE(dst) == IS_REG) { 3635 if (!zend_jit_load_reg(Dst, src, dst, info)) { 3636 return 0; 3637 } 3638 } else { 3639 ZEND_UNREACHABLE(); 3640 } 3641 } else { 3642 ZEND_UNREACHABLE(); 3643 } 3644 } else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) { 3645 dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3646 if (!zend_jit_spill_store(Dst, src, dst, info, 3647 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3648 JIT_G(current_frame) == NULL || 3649 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3650 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3651 )) { 3652 return 0; 3653 } 3654 } 3655 return 1; 3656} 3657 3658static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) 3659{ 3660 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 3661 3662 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 3663 3664 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { 3665 if (!zend_jit_save_call_chain(Dst, -1)) { 3666 return 0; 3667 } 3668 } 3669 3670 ZEND_ASSERT(opline); 3671 3672 if ((opline-1)->opcode != ZEND_FETCH_CONSTANT 3673 && (opline-1)->opcode != ZEND_FETCH_LIST_R 3674 && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR)) 3675 && !(flags & ZEND_JIT_EXIT_FREE_OP1)) { 3676 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var); 3677 3678 | IF_NOT_ZVAL_REFCOUNTED val_addr, >2, ZREG_TMP1, ZREG_TMP2 3679 | GET_ZVAL_PTR TMP1, val_addr, TMP2 3680 | GC_ADDREF TMP1, TMP2w 3681 |2: 3682 } 3683 3684 | LOAD_IP_ADDR (opline - 1) 3685 | b ->trace_escape 3686 |1: 3687 3688 return 1; 3689} 3690 3691static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) 3692{ 3693 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3694 3695 if (reg == ZREG_LONG_MIN_MINUS_1) { 3696 uint64_t val = 0xc3e0000000000000; 3697 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3698 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 3699 } else if (reg == ZREG_LONG_MIN) { 3700 uint64_t val = 0x8000000000000000; 3701 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3702 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 3703 } else if (reg == ZREG_LONG_MAX) { 3704 uint64_t val = 0x7fffffffffffffff; 3705 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3706 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 3707 } else if (reg == ZREG_LONG_MAX_PLUS_1) { 3708 uint64_t val = 0x43e0000000000000; 3709 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3710 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 3711 } else if (reg == ZREG_NULL) { 3712 | SET_ZVAL_TYPE_INFO dst, IS_NULL, TMP1w, TMP2 3713 } else if (reg == ZREG_ZVAL_TRY_ADDREF) { 3714 | IF_NOT_ZVAL_REFCOUNTED dst, >1, ZREG_TMP1, ZREG_TMP2 3715 | GET_ZVAL_PTR TMP1, dst, TMP2 3716 | GC_ADDREF TMP1, TMP2w 3717 |1: 3718 } else if (reg == ZREG_ZVAL_COPY_GPR0) { 3719 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 3720 3721 | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3722 | TRY_ADDREF -1, REG1w, REG2, TMP1w 3723 } else { 3724 ZEND_UNREACHABLE(); 3725 } 3726 return 1; 3727} 3728 3729static int zend_jit_free_trampoline(dasm_State **Dst) 3730{ 3731 | // if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) 3732 | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] 3733 | TST_32_WITH_CONST TMP1w, ZEND_ACC_CALL_VIA_TRAMPOLINE, TMP2w 3734 | beq >1 3735 | mov FCARG1x, REG0 3736 | EXT_CALL zend_jit_free_trampoline_helper, REG0 3737 |1: 3738 return 1; 3739} 3740 3741static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 3742{ 3743 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { 3744 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 3745 } 3746 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 3747 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3748 } 3749 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { 3750 return 0; 3751 } 3752 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3753 | LONG_ADD_SUB_WITH_IMM adds, op1_def_addr, Z_L(1), TMP1, TMP2 3754 } else { 3755 | LONG_ADD_SUB_WITH_IMM subs, op1_def_addr, Z_L(1), TMP1, TMP2 3756 } 3757 3758 if (may_overflow && 3759 (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || 3760 ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { 3761 int32_t exit_point; 3762 const void *exit_addr; 3763 zend_jit_trace_stack *stack; 3764 uint32_t old_op1_info, old_res_info = 0; 3765 3766 stack = JIT_G(current_frame)->stack; 3767 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 3768 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); 3769 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3770 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); 3771 } else { 3772 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); 3773 } 3774 if (opline->result_type != IS_UNUSED) { 3775 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3776 if (opline->opcode == ZEND_PRE_INC) { 3777 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 3778 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); 3779 } else if (opline->opcode == ZEND_PRE_DEC) { 3780 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 3781 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); 3782 } else if (opline->opcode == ZEND_POST_INC) { 3783 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 3784 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); 3785 } else if (opline->opcode == ZEND_POST_DEC) { 3786 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 3787 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); 3788 } 3789 } 3790 3791 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 3792 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3793 if (!exit_addr) { 3794 return 0; 3795 } 3796 | bvs &exit_addr 3797 3798 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3799 opline->result_type != IS_UNUSED) { 3800 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3801 } 3802 3803 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 3804 if (opline->result_type != IS_UNUSED) { 3805 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3806 } 3807 } else if (may_overflow) { 3808 | bvs >1 3809 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3810 opline->result_type != IS_UNUSED) { 3811 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3812 } 3813 |.cold_code 3814 |1: 3815 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3816 uint64_t val = 0x43e0000000000000; 3817 if (Z_MODE(op1_def_addr) == IS_REG) { 3818 | LOAD_64BIT_VAL TMP1, val 3819 | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 3820 } else { 3821 | SET_ZVAL_LVAL op1_def_addr, val, TMP2, TMP1 3822 } 3823 } else { 3824 uint64_t val = 0xc3e0000000000000; 3825 if (Z_MODE(op1_def_addr) == IS_REG) { 3826 | LOAD_64BIT_VAL TMP1, val 3827 | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 3828 } else { 3829 | SET_ZVAL_LVAL op1_def_addr, val, TMP2, TMP1 3830 } 3831 } 3832 if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { 3833 | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE, TMP1w, TMP2 3834 } 3835 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3836 opline->result_type != IS_UNUSED) { 3837 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3838 } 3839 | b >3 3840 |.code 3841 } else { 3842 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3843 opline->result_type != IS_UNUSED) { 3844 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3845 } 3846 } 3847 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 3848 |.cold_code 3849 |2: 3850 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 3851 | SET_EX_OPLINE opline, REG0 3852 if (op1_info & MAY_BE_UNDEF) { 3853 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2, ZREG_TMP1 3854 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 3855 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 3856 | EXT_CALL zend_jit_undefined_op_helper, REG0 3857 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 3858 op1_info |= MAY_BE_NULL; 3859 } 3860 |2: 3861 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 3862 3863 | // ZVAL_DEREF(var_ptr); 3864 if (op1_info & MAY_BE_REF) { 3865 | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >2, TMP1w 3866 | GET_Z_PTR FCARG1x, FCARG1x 3867 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 3868 | cbz TMP1, >1 3869 if (RETURN_VALUE_USED(opline)) { 3870 | LOAD_ZVAL_ADDR FCARG2x, res_addr 3871 } else { 3872 | mov FCARG2x, xzr 3873 } 3874 if (opline->opcode == ZEND_PRE_INC) { 3875 | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 3876 } else if (opline->opcode == ZEND_PRE_DEC) { 3877 | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 3878 } else if (opline->opcode == ZEND_POST_INC) { 3879 | EXT_CALL zend_jit_post_inc_typed_ref, REG0 3880 } else if (opline->opcode == ZEND_POST_DEC) { 3881 | EXT_CALL zend_jit_post_dec_typed_ref, REG0 3882 } else { 3883 ZEND_UNREACHABLE(); 3884 } 3885 zend_jit_check_exception(Dst); 3886 | b >3 3887 |1: 3888 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) 3889 |2: 3890 } 3891 3892 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 3893 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 3894 3895 | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3896 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 3897 } 3898 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3899 if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { 3900 | LOAD_ZVAL_ADDR FCARG2x, res_addr 3901 | EXT_CALL zend_jit_pre_inc, REG0 3902 } else { 3903 | EXT_CALL increment_function, REG0 3904 } 3905 } else { 3906 if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { 3907 | LOAD_ZVAL_ADDR FCARG2x, res_addr 3908 | EXT_CALL zend_jit_pre_dec, REG0 3909 } else { 3910 | EXT_CALL decrement_function, REG0 3911 } 3912 } 3913 if (may_throw) { 3914 zend_jit_check_exception(Dst); 3915 } 3916 } else { 3917 zend_reg tmp_reg; 3918 3919 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 3920 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3921 } 3922 if (Z_MODE(op1_def_addr) == IS_REG) { 3923 tmp_reg = Z_REG(op1_def_addr); 3924 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 3925 tmp_reg = Z_REG(op1_addr); 3926 } else { 3927 tmp_reg = ZREG_FPR0; 3928 } 3929 | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 3930 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3931 uint64_t val = 0x3ff0000000000000; // 1.0 3932 | LOAD_64BIT_VAL TMP1, val 3933 | fmov FPTMP, TMP1 3934 | fadd Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP 3935 } else { 3936 uint64_t val = 0x3ff0000000000000; // 1.0 3937 | LOAD_64BIT_VAL TMP1, val 3938 | fmov FPTMP, TMP1 3939 | fsub Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP 3940 } 3941 | SET_ZVAL_DVAL op1_def_addr, tmp_reg, ZREG_TMP1 3942 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3943 opline->result_type != IS_UNUSED) { 3944 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3945 | TRY_ADDREF op1_def_info, REG0w, REG1, TMP1w 3946 } 3947 } 3948 | b >3 3949 |.code 3950 } 3951 |3: 3952 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { 3953 return 0; 3954 } 3955 if (opline->result_type != IS_UNUSED) { 3956 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 3957 return 0; 3958 } 3959 } 3960 return 1; 3961} 3962 3963static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) 3964{ 3965 if ((opline+1)->opcode == ZEND_OP_DATA 3966 && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) 3967 && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { 3968 return 1; 3969 } 3970 return 3971 ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 3972 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || 3973 ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 3974 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || 3975 ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 3976 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); 3977} 3978 3979static int zend_jit_math_long_long(dasm_State **Dst, 3980 const zend_op *opline, 3981 zend_uchar opcode, 3982 zend_jit_addr op1_addr, 3983 zend_jit_addr op2_addr, 3984 zend_jit_addr res_addr, 3985 uint32_t res_info, 3986 uint32_t res_use_info, 3987 int may_overflow) 3988{ 3989 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 3990 zend_reg result_reg; 3991 zend_reg tmp_reg = ZREG_REG0; 3992 bool use_ovf_flag = 1; 3993 3994 if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { 3995 if (may_overflow && (res_info & MAY_BE_GUARD) 3996 && JIT_G(current_frame) 3997 && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { 3998 result_reg = ZREG_REG0; 3999 } else { 4000 result_reg = Z_REG(res_addr); 4001 } 4002 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { 4003 result_reg = Z_REG(op1_addr); 4004 } else if (Z_REG(res_addr) != ZREG_REG0) { 4005 result_reg = ZREG_REG0; 4006 } else { 4007 /* ASSIGN_DIM_OP */ 4008 result_reg = ZREG_FCARG1; 4009 tmp_reg = ZREG_FCARG1; 4010 } 4011 4012 if (opcode == ZEND_MUL && 4013 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4014 Z_LVAL_P(Z_ZV(op2_addr)) == 2) { 4015 if (Z_MODE(op1_addr) == IS_REG) { 4016 | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) 4017 } else { 4018 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4019 | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) 4020 } 4021 } else if (opcode == ZEND_MUL && 4022 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4023 !may_overflow && 4024 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { 4025 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4026 | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4027 | lsl Rx(result_reg), Rx(result_reg), TMP1 4028 } else if (opcode == ZEND_MUL && 4029 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4030 Z_LVAL_P(Z_ZV(op1_addr)) == 2) { 4031 if (Z_MODE(op2_addr) == IS_REG) { 4032 | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) 4033 } else { 4034 | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 4035 | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) 4036 } 4037 } else if (opcode == ZEND_MUL && 4038 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4039 !may_overflow && 4040 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { 4041 | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 4042 | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) 4043 | lsl Rx(result_reg), Rx(result_reg), TMP1 4044 } else if (opcode == ZEND_DIV && 4045 (Z_MODE(op2_addr) == IS_CONST_ZVAL && 4046 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { 4047 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4048 | asr Rx(result_reg), Rx(result_reg), #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4049#if 0 4050 /* x86 specific optimizations through LEA instraction are not supported on ARM */ 4051 } else if (opcode == ZEND_ADD && 4052 !may_overflow && 4053 Z_MODE(op1_addr) == IS_REG && 4054 Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4055 | NIY // TODO: test 4056 } else if (opcode == ZEND_ADD && 4057 !may_overflow && 4058 Z_MODE(op2_addr) == IS_REG && 4059 Z_MODE(op1_addr) == IS_CONST_ZVAL) { 4060 | NIY // TODO: test 4061 } else if (opcode == ZEND_SUB && 4062 !may_overflow && 4063 Z_MODE(op1_addr) == IS_REG && 4064 Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4065 | NIY // TODO: test 4066#endif 4067 } else if (opcode == ZEND_MUL) { 4068 | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 4069 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4070 | mul Rx(result_reg), TMP1, TMP2 4071 if(may_overflow) { 4072 /* Use 'smulh' to get the upper 64 bits fo the 128-bit result. 4073 * For signed multiplication, the top 65 bits of the result will contain 4074 * either all zeros or all ones if no overflow occurred. 4075 * Flag: bne -> overflow. beq -> no overflow. 4076 */ 4077 use_ovf_flag = 0; 4078 | smulh TMP1, TMP1, TMP2 4079 | cmp TMP1, Rx(result_reg), asr #63 4080 } 4081 } else { 4082 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4083 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4084 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4085 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4086 /* +/- 0 */ 4087 may_overflow = 0; 4088 } else if (same_ops && opcode != ZEND_DIV) { 4089 | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) 4090 } else { 4091 | LONG_MATH opcode, result_reg, op2_addr, TMP1 4092 } 4093 } 4094 if (may_overflow) { 4095 if (res_info & MAY_BE_GUARD) { 4096 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 4097 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4098 if (!exit_addr) { 4099 return 0; 4100 } 4101 if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { 4102 if (use_ovf_flag) { 4103 | bvs &exit_addr 4104 } else { 4105 | bne &exit_addr 4106 } 4107 if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { 4108 | mov Rx(Z_REG(res_addr)), Rx(result_reg) 4109 } 4110 } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 4111 if (use_ovf_flag) { 4112 | bvc &exit_addr 4113 } else { 4114 | beq &exit_addr 4115 } 4116 } else { 4117 ZEND_UNREACHABLE(); 4118 } 4119 } else { 4120 if (res_info & MAY_BE_LONG) { 4121 if (use_ovf_flag) { 4122 | bvs >1 4123 } else { 4124 | bne >1 4125 } 4126 } else { 4127 if (use_ovf_flag) { 4128 | bvc >1 4129 } else { 4130 | beq >1 4131 } 4132 } 4133 } 4134 } 4135 4136 if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { 4137 | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 4138 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4139 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4140 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 4141 } 4142 } 4143 } 4144 4145 if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { 4146 zend_reg tmp_reg1 = ZREG_FPR0; 4147 zend_reg tmp_reg2 = ZREG_FPR1; 4148 4149 if (res_info & MAY_BE_LONG) { 4150 |.cold_code 4151 |1: 4152 } 4153 4154 do { 4155 if ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || 4156 (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)) { 4157 if (opcode == ZEND_ADD) { 4158 uint64_t val = 0x43e0000000000000; 4159 if (Z_MODE(res_addr) == IS_REG) { 4160 | LOAD_64BIT_VAL TMP1, val 4161 | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 4162 } else { 4163 | SET_ZVAL_LVAL res_addr, val, TMP2, TMP1 4164 } 4165 break; 4166 } else if (opcode == ZEND_SUB) { 4167 uint64_t val = 0xc3e0000000000000; 4168 if (Z_MODE(res_addr) == IS_REG) { 4169 | LOAD_64BIT_VAL TMP1, val 4170 | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 4171 } else { 4172 | SET_ZVAL_LVAL res_addr, val, TMP2, TMP1 4173 } 4174 break; 4175 } 4176 } 4177 4178 | DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg, ZREG_TMP1 4179 | DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg, ZREG_TMP1 4180 | DOUBLE_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 4181 | SET_ZVAL_DVAL res_addr, tmp_reg1, ZREG_TMP1 4182 } while (0); 4183 4184 if (Z_MODE(res_addr) == IS_MEM_ZVAL 4185 && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4186 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4187 } 4188 if (res_info & MAY_BE_LONG) { 4189 | b >2 4190 |.code 4191 } 4192 |2: 4193 } 4194 4195 return 1; 4196} 4197 4198static int zend_jit_math_long_double(dasm_State **Dst, 4199 zend_uchar opcode, 4200 zend_jit_addr op1_addr, 4201 zend_jit_addr op2_addr, 4202 zend_jit_addr res_addr, 4203 uint32_t res_use_info) 4204{ 4205 zend_reg result_reg = 4206 (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_FPR0; 4207 zend_reg op2_reg; 4208 4209 | DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, ZREG_TMP1, ZREG_TMP2 4210 4211 if (Z_MODE(op2_addr) == IS_REG) { 4212 op2_reg = Z_REG(op2_addr); 4213 } else { 4214 op2_reg = ZREG_FPTMP; 4215 | GET_ZVAL_DVAL op2_reg, op2_addr, ZREG_TMP1 4216 } 4217 4218 | DOUBLE_MATH_REG opcode, result_reg, result_reg, op2_reg 4219 4220 | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 4221 4222 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4223 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4224 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4225 } 4226 } 4227 4228 return 1; 4229} 4230 4231static int zend_jit_math_double_long(dasm_State **Dst, 4232 zend_uchar opcode, 4233 zend_jit_addr op1_addr, 4234 zend_jit_addr op2_addr, 4235 zend_jit_addr res_addr, 4236 uint32_t res_use_info) 4237{ 4238 zend_reg result_reg, op1_reg, op2_reg; 4239 4240 if (zend_is_commutative(opcode) 4241 && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { 4242 if (Z_MODE(res_addr) == IS_REG) { 4243 result_reg = Z_REG(res_addr); 4244 } else { 4245 result_reg = ZREG_FPR0; 4246 } 4247 | DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 4248 if (Z_MODE(op1_addr) == IS_REG) { 4249 op1_reg = Z_REG(op1_addr); 4250 } else { 4251 op1_reg = ZREG_FPTMP; 4252 | GET_ZVAL_DVAL op1_reg, op1_addr, ZREG_TMP1 4253 } 4254 | DOUBLE_MATH_REG opcode, result_reg, result_reg, op1_reg 4255 } else { 4256 if (Z_MODE(res_addr) == IS_REG) { 4257 result_reg = Z_REG(res_addr); 4258 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4259 result_reg = Z_REG(op1_addr); 4260 } else { 4261 result_reg = ZREG_FPR0; 4262 } 4263 4264 if (Z_MODE(op1_addr) == IS_REG) { 4265 op1_reg = Z_REG(op1_addr); 4266 } else { 4267 | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 4268 op1_reg = result_reg; 4269 } 4270 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4271 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4272 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4273 /* +/- 0 */ 4274 } else { 4275 op2_reg = ZREG_FPTMP; 4276 | DOUBLE_GET_ZVAL_LVAL op2_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 4277 | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg 4278 } 4279 } 4280 4281 | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 4282 4283 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4284 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4285 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4286 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4287 } 4288 } 4289 } 4290 4291 return 1; 4292} 4293 4294static int zend_jit_math_double_double(dasm_State **Dst, 4295 zend_uchar opcode, 4296 zend_jit_addr op1_addr, 4297 zend_jit_addr op2_addr, 4298 zend_jit_addr res_addr, 4299 uint32_t res_use_info) 4300{ 4301 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4302 zend_reg result_reg, op1_reg, op2_reg; 4303 zend_jit_addr val_addr; 4304 4305 if (Z_MODE(res_addr) == IS_REG) { 4306 result_reg = Z_REG(res_addr); 4307 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4308 result_reg = Z_REG(op1_addr); 4309 } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { 4310 result_reg = Z_REG(op2_addr); 4311 } else { 4312 result_reg = ZREG_FPR0; 4313 } 4314 4315 if (Z_MODE(op1_addr) == IS_REG) { 4316 op1_reg = Z_REG(op1_addr); 4317 val_addr = op2_addr; 4318 } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4319 op1_reg = Z_REG(op2_addr); 4320 val_addr = op1_addr; 4321 } else { 4322 | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 4323 op1_reg = result_reg; 4324 val_addr = op2_addr; 4325 } 4326 4327 if ((opcode == ZEND_MUL) && 4328 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4329 | DOUBLE_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg 4330 } else { 4331 if (same_ops) { 4332 op2_reg = op1_reg; 4333 } else if (Z_MODE(val_addr) == IS_REG) { 4334 op2_reg = Z_REG(val_addr); 4335 } else { 4336 op2_reg = ZREG_FPTMP; 4337 | GET_ZVAL_DVAL op2_reg, val_addr, ZREG_TMP1 4338 } 4339 | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg 4340 } 4341 4342 | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 4343 4344 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4345 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4346 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4347 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4348 } 4349 } 4350 } 4351 return 1; 4352} 4353 4354static int zend_jit_math_helper(dasm_State **Dst, 4355 const zend_op *opline, 4356 zend_uchar opcode, 4357 zend_uchar op1_type, 4358 znode_op op1, 4359 zend_jit_addr op1_addr, 4360 uint32_t op1_info, 4361 zend_uchar op2_type, 4362 znode_op op2, 4363 zend_jit_addr op2_addr, 4364 uint32_t op2_info, 4365 uint32_t res_var, 4366 zend_jit_addr res_addr, 4367 uint32_t res_info, 4368 uint32_t res_use_info, 4369 int may_overflow, 4370 int may_throw) 4371/* Labels: 1,2,3,4,5,6 */ 4372{ 4373 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4374 4375 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4376 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { 4377 if (op1_info & MAY_BE_DOUBLE) { 4378 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 4379 } else { 4380 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 4381 } 4382 } 4383 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { 4384 if (op2_info & MAY_BE_DOUBLE) { 4385 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1, ZREG_TMP1 4386 |.cold_code 4387 |1: 4388 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4389 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4390 } 4391 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4392 return 0; 4393 } 4394 | b >5 4395 |.code 4396 } else { 4397 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4398 } 4399 } 4400 if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { 4401 return 0; 4402 } 4403 if (op1_info & MAY_BE_DOUBLE) { 4404 |.cold_code 4405 |3: 4406 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4407 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 4408 } 4409 if (op2_info & MAY_BE_DOUBLE) { 4410 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4411 if (!same_ops) { 4412 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1, ZREG_TMP1 4413 } else { 4414 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4415 } 4416 } 4417 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4418 return 0; 4419 } 4420 | b >5 4421 } 4422 if (!same_ops) { 4423 |1: 4424 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4425 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4426 } 4427 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4428 return 0; 4429 } 4430 | b >5 4431 } 4432 |.code 4433 } 4434 } else if ((op1_info & MAY_BE_DOUBLE) && 4435 !(op1_info & MAY_BE_LONG) && 4436 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4437 (res_info & MAY_BE_DOUBLE)) { 4438 if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4439 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 4440 } 4441 if (op2_info & MAY_BE_DOUBLE) { 4442 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4443 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4444 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1, ZREG_TMP1 4445 } else { 4446 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4447 } 4448 } 4449 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4450 return 0; 4451 } 4452 } 4453 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4454 if (op2_info & MAY_BE_DOUBLE) { 4455 |.cold_code 4456 } 4457 |1: 4458 if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4459 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4460 } 4461 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4462 return 0; 4463 } 4464 if (op2_info & MAY_BE_DOUBLE) { 4465 | b >5 4466 |.code 4467 } 4468 } 4469 } else if ((op2_info & MAY_BE_DOUBLE) && 4470 !(op2_info & MAY_BE_LONG) && 4471 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4472 (res_info & MAY_BE_DOUBLE)) { 4473 if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4474 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4475 } 4476 if (op1_info & MAY_BE_DOUBLE) { 4477 if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4478 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4479 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1, ZREG_TMP1 4480 } else { 4481 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 4482 } 4483 } 4484 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4485 return 0; 4486 } 4487 } 4488 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4489 if (op1_info & MAY_BE_DOUBLE) { 4490 |.cold_code 4491 } 4492 |1: 4493 if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4494 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 4495 } 4496 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4497 return 0; 4498 } 4499 if (op1_info & MAY_BE_DOUBLE) { 4500 | b >5 4501 |.code 4502 } 4503 } 4504 } 4505 4506 |5: 4507 4508 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 4509 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 4510 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4511 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4512 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4513 |.cold_code 4514 } 4515 |6: 4516 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4517 if (Z_MODE(res_addr) == IS_REG) { 4518 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4519 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4520 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4521 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4522 } 4523 if (Z_MODE(op1_addr) == IS_REG) { 4524 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4525 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4526 return 0; 4527 } 4528 op1_addr = real_addr; 4529 } 4530 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4531 } else { 4532 if (Z_MODE(op1_addr) == IS_REG) { 4533 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4534 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4535 return 0; 4536 } 4537 op1_addr = real_addr; 4538 } 4539 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4540 if (Z_MODE(res_addr) == IS_REG) { 4541 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4542 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4543 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4544 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4545 } 4546 } 4547 if (Z_MODE(op2_addr) == IS_REG) { 4548 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 4549 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 4550 return 0; 4551 } 4552 op2_addr = real_addr; 4553 } 4554 | LOAD_ZVAL_ADDR CARG3, op2_addr 4555 | SET_EX_OPLINE opline, REG0 4556 if (opcode == ZEND_ADD) { 4557 | EXT_CALL add_function, REG0 4558 } else if (opcode == ZEND_SUB) { 4559 | EXT_CALL sub_function, REG0 4560 } else if (opcode == ZEND_MUL) { 4561 | EXT_CALL mul_function, REG0 4562 } else if (opcode == ZEND_DIV) { 4563 | EXT_CALL div_function, REG0 4564 } else { 4565 ZEND_UNREACHABLE(); 4566 } 4567 | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4568 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4569 if (may_throw) { 4570 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 4571 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 4572 | cbnz TMP2, ->exception_handler_free_op2 4573 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 4574 zend_jit_check_exception_undef_result(Dst, opline); 4575 } else { 4576 zend_jit_check_exception(Dst); 4577 } 4578 } 4579 if (Z_MODE(res_addr) == IS_REG) { 4580 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4581 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 4582 return 0; 4583 } 4584 } 4585 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4586 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4587 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4588 | b <5 4589 |.code 4590 } 4591 } 4592 4593 return 1; 4594} 4595 4596static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 4597{ 4598 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 4599 ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4600 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); 4601 4602 if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { 4603 return 0; 4604 } 4605 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4606 return 0; 4607 } 4608 return 1; 4609} 4610 4611static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr) 4612{ 4613 if (Z_MODE(op2_addr) != IS_MEM_ZVAL || Z_REG(op2_addr) != ZREG_FCARG1) { 4614 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 4615 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 4616 } else if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4617 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 4618 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 4619 } else { 4620 | GET_ZVAL_LVAL ZREG_REG0, op2_addr, TMP1 4621 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 4622 | mov FCARG2x, REG0 4623 } 4624 | EXT_CALL zend_jit_add_arrays_helper, REG0 4625 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 4626 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX, TMP1w, TMP2 4627 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 4628 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 4629 return 1; 4630} 4631 4632static int zend_jit_long_math_helper(dasm_State **Dst, 4633 const zend_op *opline, 4634 zend_uchar opcode, 4635 zend_uchar op1_type, 4636 znode_op op1, 4637 zend_jit_addr op1_addr, 4638 uint32_t op1_info, 4639 zend_ssa_range *op1_range, 4640 zend_uchar op2_type, 4641 znode_op op2, 4642 zend_jit_addr op2_addr, 4643 uint32_t op2_info, 4644 zend_ssa_range *op2_range, 4645 uint32_t res_var, 4646 zend_jit_addr res_addr, 4647 uint32_t res_info, 4648 uint32_t res_use_info, 4649 int may_throw) 4650/* Labels: 6 */ 4651{ 4652 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4653 zend_reg result_reg; 4654 4655 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 4656 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 4657 } 4658 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 4659 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4660 } 4661 4662 if (Z_MODE(res_addr) == IS_REG) { 4663 if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) 4664 && opline->op2_type != IS_CONST) { 4665 result_reg = ZREG_REG0; 4666 } else { 4667 result_reg = Z_REG(res_addr); 4668 } 4669 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4670 result_reg = Z_REG(op1_addr); 4671 } else if (Z_REG(res_addr) != ZREG_REG0) { 4672 result_reg = ZREG_REG0; 4673 } else { 4674 /* ASSIGN_DIM_OP */ 4675 result_reg = ZREG_FCARG1; 4676 } 4677 4678 if (opcode == ZEND_SL) { 4679 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4680 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 4681 4682 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 4683 if (EXPECTED(op2_lval > 0)) { 4684 | mov Rx(result_reg), xzr 4685 } else { 4686 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4687 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4688 | SET_EX_OPLINE opline, REG0 4689 | b ->negative_shift 4690 } 4691 } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { 4692 | add Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) 4693 } else { 4694 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4695 | lsl Rx(result_reg), Rx(result_reg), #op2_lval 4696 } 4697 } else { 4698 zend_reg op2_reg; 4699 4700 if (Z_MODE(op2_addr) == IS_REG) { 4701 op2_reg = Z_REG(op2_addr); 4702 } else { 4703 op2_reg = ZREG_TMP2; 4704 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4705 } 4706 if (!op2_range || 4707 op2_range->min < 0 || 4708 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 4709 4710 | cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8) 4711 | bhs >1 4712 |.cold_code 4713 |1: 4714 | mov Rx(result_reg), xzr 4715 | cmp Rx(op2_reg), xzr 4716 | bgt >1 4717 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4718 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4719 | SET_EX_OPLINE opline, REG0 4720 | b ->negative_shift 4721 |.code 4722 } 4723 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4724 | lsl Rx(result_reg), Rx(result_reg), Rx(op2_reg) 4725 |1: 4726 } 4727 } else if (opcode == ZEND_SR) { 4728 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4729 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4730 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 4731 4732 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 4733 if (EXPECTED(op2_lval > 0)) { 4734 | asr Rx(result_reg), Rx(result_reg), #((SIZEOF_ZEND_LONG * 8) - 1) 4735 } else { 4736 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4737 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4738 | SET_EX_OPLINE opline, REG0 4739 | b ->negative_shift 4740 } 4741 } else { 4742 | asr Rx(result_reg), Rx(result_reg), #op2_lval 4743 } 4744 } else { 4745 zend_reg op2_reg; 4746 4747 if (Z_MODE(op2_addr) == IS_REG) { 4748 op2_reg = Z_REG(op2_addr); 4749 } else { 4750 op2_reg = ZREG_TMP2; 4751 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4752 } 4753 if (!op2_range || 4754 op2_range->min < 0 || 4755 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 4756 | cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8) 4757 | bhs >1 4758 |.cold_code 4759 |1: 4760 | cmp Rx(op2_reg), xzr 4761 | mov Rx(op2_reg), #((SIZEOF_ZEND_LONG * 8) - 1) 4762 | bgt >1 4763 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4764 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4765 | SET_EX_OPLINE opline, REG0 4766 | b ->negative_shift 4767 |.code 4768 } 4769 |1: 4770 | asr Rx(result_reg), Rx(result_reg), Rx(op2_reg) 4771 } 4772 } else if (opcode == ZEND_MOD) { 4773 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4774 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 4775 4776 if (op2_lval == 0) { 4777 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4778 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4779 | SET_EX_OPLINE opline, REG0 4780 | b ->mod_by_zero 4781 } else if (op2_lval == -1) { 4782 | mov Rx(result_reg), xzr 4783 } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) { 4784 zval tmp; 4785 zend_jit_addr tmp_addr; 4786 4787 /* Optimisation for mod of power of 2 */ 4788 ZVAL_LONG(&tmp, op2_lval - 1); 4789 tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp); 4790 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4791 | LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, TMP1 4792 } else { 4793 | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 4794 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4795 | sdiv Rx(result_reg), TMP1, TMP2 4796 | msub Rx(result_reg), Rx(result_reg), TMP2, TMP1 4797 } 4798 } else { 4799 zend_reg op2_reg; 4800 4801 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 4802 | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP2, Rx(Z_REG(op2_addr)), Z_OFFSET(op2_addr), TMP2 4803 op2_reg = ZREG_TMP2; 4804 } else { 4805 ZEND_ASSERT(Z_MODE(op2_addr) == IS_REG); 4806 op2_reg = Z_REG(op2_addr); 4807 } 4808 4809 if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { 4810 | cbz Rx(op2_reg), >1 4811 |.cold_code 4812 |1: 4813 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4814 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4815 | SET_EX_OPLINE opline, REG0 4816 | b ->mod_by_zero 4817 |.code 4818 } 4819 4820 /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ 4821 if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { 4822 | cmn Rx(op2_reg), #1 4823 | beq >1 4824 |.cold_code 4825 |1: 4826 | SET_ZVAL_LVAL_FROM_REG res_addr, xzr, TMP1 4827 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4828 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4829 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4830 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 4831 } 4832 } 4833 } 4834 | b >5 4835 |.code 4836 } 4837 4838 | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 4839 | sdiv Rx(result_reg), TMP1, Rx(op2_reg) 4840 | msub Rx(result_reg), Rx(result_reg), Rx(op2_reg), TMP1 4841 } 4842 } else if (same_ops) { 4843 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4844 | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) 4845 } else { 4846 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4847 | LONG_MATH opcode, result_reg, op2_addr, TMP1 4848 } 4849 4850 if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { 4851 | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 4852 } 4853 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4854 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4855 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4856 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 4857 } 4858 } 4859 } 4860 4861 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || 4862 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 4863 if ((op1_info & MAY_BE_LONG) && 4864 (op2_info & MAY_BE_LONG)) { 4865 |.cold_code 4866 } 4867 |6: 4868 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4869 if (Z_MODE(res_addr) == IS_REG) { 4870 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4871 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4872 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4873 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4874 } 4875 if (Z_MODE(op1_addr) == IS_REG) { 4876 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4877 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4878 return 0; 4879 } 4880 op1_addr = real_addr; 4881 } 4882 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4883 } else { 4884 if (Z_MODE(op1_addr) == IS_REG) { 4885 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4886 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4887 return 0; 4888 } 4889 op1_addr = real_addr; 4890 } 4891 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4892 if (Z_MODE(res_addr) == IS_REG) { 4893 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4894 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4895 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4896 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4897 } 4898 } 4899 if (Z_MODE(op2_addr) == IS_REG) { 4900 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 4901 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 4902 return 0; 4903 } 4904 op2_addr = real_addr; 4905 } 4906 | LOAD_ZVAL_ADDR CARG3, op2_addr 4907 | SET_EX_OPLINE opline, REG0 4908 if (opcode == ZEND_BW_OR) { 4909 | EXT_CALL bitwise_or_function, REG0 4910 } else if (opcode == ZEND_BW_AND) { 4911 | EXT_CALL bitwise_and_function, REG0 4912 } else if (opcode == ZEND_BW_XOR) { 4913 | EXT_CALL bitwise_xor_function, REG0 4914 } else if (opcode == ZEND_SL) { 4915 | EXT_CALL shift_left_function, REG0 4916 } else if (opcode == ZEND_SR) { 4917 | EXT_CALL shift_right_function, REG0 4918 } else if (opcode == ZEND_MOD) { 4919 | EXT_CALL mod_function, REG0 4920 } else { 4921 ZEND_UNREACHABLE(); 4922 } 4923 if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) { 4924 /* compound assignment may decrement "op2" refcount */ 4925 op2_info |= MAY_BE_RC1; 4926 } 4927 | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4928 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4929 if (may_throw) { 4930 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 4931 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 4932 | cbnz TMP2, ->exception_handler_free_op2 4933 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 4934 zend_jit_check_exception_undef_result(Dst, opline); 4935 } else { 4936 zend_jit_check_exception(Dst); 4937 } 4938 } 4939 if (Z_MODE(res_addr) == IS_REG) { 4940 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4941 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 4942 return 0; 4943 } 4944 } 4945 if ((op1_info & MAY_BE_LONG) && 4946 (op2_info & MAY_BE_LONG)) { 4947 | b >5 4948 |.code 4949 } 4950 } 4951 |5: 4952 4953 return 1; 4954} 4955 4956static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 4957{ 4958 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 4959 ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); 4960 4961 if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, 4962 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 4963 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 4964 opline->result.var, res_addr, res_info, res_use_info, may_throw)) { 4965 return 0; 4966 } 4967 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4968 return 0; 4969 } 4970 return 1; 4971} 4972 4973static int zend_jit_concat_helper(dasm_State **Dst, 4974 const zend_op *opline, 4975 zend_uchar op1_type, 4976 znode_op op1, 4977 zend_jit_addr op1_addr, 4978 uint32_t op1_info, 4979 zend_uchar op2_type, 4980 znode_op op2, 4981 zend_jit_addr op2_addr, 4982 uint32_t op2_info, 4983 zend_jit_addr res_addr, 4984 int may_throw) 4985{ 4986 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 4987 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 4988 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 4989 } 4990 if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 4991 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6, ZREG_TMP1 4992 } 4993 if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { 4994 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4995 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4996 } 4997 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 4998 | EXT_CALL zend_jit_fast_assign_concat_helper, REG0 4999 /* concatenation with itself may reduce refcount */ 5000 op2_info |= MAY_BE_RC1; 5001 } else { 5002 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5003 | LOAD_ZVAL_ADDR FCARG1x, res_addr 5004 } 5005 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 5006 | LOAD_ZVAL_ADDR CARG3, op2_addr 5007 if (op1_type == IS_CV || op1_type == IS_CONST) { 5008 | EXT_CALL zend_jit_fast_concat_helper, REG0 5009 } else { 5010 | EXT_CALL zend_jit_fast_concat_tmp_helper, REG0 5011 } 5012 } 5013 /* concatenation with empty string may increase refcount */ 5014 op2_info |= MAY_BE_RCN; 5015 | FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 5016 |5: 5017 } 5018 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || 5019 (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { 5020 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5021 |.cold_code 5022 |6: 5023 } 5024 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 5025 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5026 | LOAD_ZVAL_ADDR FCARG1x, res_addr 5027 } 5028 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 5029 } else { 5030 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 5031 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5032 | LOAD_ZVAL_ADDR FCARG1x, res_addr 5033 } 5034 } 5035 | LOAD_ZVAL_ADDR CARG3, op2_addr 5036 | SET_EX_OPLINE opline, REG0 5037 | EXT_CALL concat_function, REG0 5038 /* concatenation with empty string may increase refcount */ 5039 op1_info |= MAY_BE_RCN; 5040 op2_info |= MAY_BE_RCN; 5041 | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 5042 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 5043 if (may_throw) { 5044 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5045 zend_jit_check_exception_undef_result(Dst, opline); 5046 } else { 5047 zend_jit_check_exception(Dst); 5048 } 5049 } 5050 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5051 | b <5 5052 |.code 5053 } 5054 } 5055 5056 return 1; 5057} 5058 5059static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) 5060{ 5061 zend_jit_addr op1_addr, op2_addr; 5062 5063 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5064 ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); 5065 5066 op1_addr = OP1_ADDR(); 5067 op2_addr = OP2_ADDR(); 5068 5069 return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); 5070} 5071 5072static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) 5073/* Labels: 1,2,3,4,5 */ 5074{ 5075 zend_jit_addr op2_addr = OP2_ADDR(); 5076 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 5077 5078 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 5079 && type == BP_VAR_R 5080 && !exit_addr) { 5081 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5082 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5083 if (!exit_addr) { 5084 return 0; 5085 } 5086 } 5087 5088 if (op2_info & MAY_BE_LONG) { 5089 bool op2_loaded = 0; 5090 bool packed_loaded = 0; 5091 bool bad_packed_key = 0; 5092 5093 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { 5094 | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) 5095 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 5096 } 5097 if (op1_info & MAY_BE_PACKED_GUARD) { 5098 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 5099 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5100 5101 if (!exit_addr) { 5102 return 0; 5103 } 5104 if (op1_info & MAY_BE_ARRAY_PACKED) { 5105 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 5106 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 5107 | beq &exit_addr 5108 } else { 5109 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 5110 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 5111 | bne &exit_addr 5112 } 5113 } 5114 if (type == BP_VAR_W) { 5115 | // hval = Z_LVAL_P(dim); 5116 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5117 op2_loaded = 1; 5118 } 5119 if (op1_info & MAY_BE_ARRAY_PACKED) { 5120 zend_long val = -1; 5121 5122 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5123 val = Z_LVAL_P(Z_ZV(op2_addr)); 5124 if (val >= 0 && val < HT_MAX_SIZE) { 5125 packed_loaded = 1; 5126 } else { 5127 bad_packed_key = 1; 5128 } 5129 } else { 5130 if (!op2_loaded) { 5131 | // hval = Z_LVAL_P(dim); 5132 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5133 op2_loaded = 1; 5134 } 5135 packed_loaded = 1; 5136 } 5137 5138 if (dim_type == IS_UNDEF && type == BP_VAR_W) { 5139 /* don't generate "fast" code for packed array */ 5140 packed_loaded = 0; 5141 } 5142 5143 if (packed_loaded) { 5144 | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); 5145 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5146 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 5147 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 5148 | beq >4 // HASH_FIND 5149 } 5150 | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) 5151 5152 | ldr REG0w, [FCARG1x, #offsetof(zend_array, nNumUsed)] 5153 if (val == 0) { 5154 | cmp REG0, xzr 5155 } else if (val > 0 && !op2_loaded) { 5156 | CMP_64_WITH_CONST REG0, val, TMP1 5157 } else { 5158 | cmp REG0, FCARG2x 5159 } 5160 5161 if (type == BP_JIT_IS) { 5162 if (not_found_exit_addr) { 5163 | bls ¬_found_exit_addr 5164 } else { 5165 | bls >9 // NOT_FOUND 5166 } 5167 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5168 | bls &exit_addr 5169 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5170 | bls ¬_found_exit_addr 5171 } else if (type == BP_VAR_RW && not_found_exit_addr) { 5172 | bls ¬_found_exit_addr 5173 } else if (type == BP_VAR_IS && found_exit_addr) { 5174 | bls >7 // NOT_FOUND 5175 } else { 5176 | bls >2 // NOT_FOUND 5177 } 5178 | // _ret = &_ht->arPacked[_h].val; 5179 if (val >= 0) { 5180 | ldr REG0, [FCARG1x, #offsetof(zend_array, arPacked)] 5181 if (val != 0) { 5182 | ADD_SUB_64_WITH_CONST add, REG0, REG0, (val * sizeof(zval)), TMP1 5183 } 5184 } else { 5185 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arPacked)] 5186 | add REG0, TMP1, FCARG2x, lsl #4 5187 } 5188 } 5189 } 5190 switch (type) { 5191 case BP_JIT_IS: 5192 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5193 if (packed_loaded) { 5194 | b >5 5195 } 5196 |4: 5197 if (!op2_loaded) { 5198 | // hval = Z_LVAL_P(dim); 5199 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5200 } 5201 if (packed_loaded) { 5202 | EXT_CALL _zend_hash_index_find, REG0 5203 } else { 5204 | EXT_CALL zend_hash_index_find, REG0 5205 } 5206 | mov REG0, RETVALx 5207 if (not_found_exit_addr) { 5208 | cbz REG0, ¬_found_exit_addr 5209 } else { 5210 | cbz REG0, >9 // NOT_FOUND 5211 } 5212 if (op2_info & MAY_BE_STRING) { 5213 | b >5 5214 } 5215 } else if (packed_loaded) { 5216 if (op2_info & MAY_BE_STRING) { 5217 | b >5 5218 } 5219 } else if (not_found_exit_addr) { 5220 | b ¬_found_exit_addr 5221 } else { 5222 | b >9 // NOT_FOUND 5223 } 5224 break; 5225 case BP_VAR_R: 5226 case BP_VAR_IS: 5227 case BP_VAR_UNSET: 5228 if (packed_loaded) { 5229 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5230 | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w 5231 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5232 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 5233 if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5234 | IF_Z_TYPE REG0, IS_UNDEF, &exit_addr, TMP1w 5235 } 5236 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5237 | IF_Z_TYPE REG0, IS_UNDEF, ¬_found_exit_addr, TMP1w 5238 } else if (type == BP_VAR_IS && found_exit_addr) { 5239 | IF_Z_TYPE REG0, IS_UNDEF, >7, TMP1w // NOT_FOUND 5240 } else { 5241 | IF_Z_TYPE REG0, IS_UNDEF, >2, TMP1w // NOT_FOUND 5242 } 5243 } 5244 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_NUMERIC_HASH))) { 5245 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5246 | b &exit_addr 5247 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5248 | b ¬_found_exit_addr 5249 } else if (type == BP_VAR_IS && found_exit_addr) { 5250 | b >7 // NOT_FOUND 5251 } else { 5252 | b >2 // NOT_FOUND 5253 } 5254 } 5255 if (!packed_loaded || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5256 |4: 5257 if (!op2_loaded) { 5258 | // hval = Z_LVAL_P(dim); 5259 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5260 } 5261 if (packed_loaded) { 5262 | EXT_CALL _zend_hash_index_find, REG0 5263 } else { 5264 | EXT_CALL zend_hash_index_find, REG0 5265 } 5266 | mov REG0, RETVALx 5267 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5268 | cbz REG0, &exit_addr 5269 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5270 | cbz REG0, ¬_found_exit_addr 5271 } else if (type == BP_VAR_IS && found_exit_addr) { 5272 | cbz REG0, >7 // NOT_FOUND 5273 } else { 5274 | cbz REG0, >2 // NOT_FOUND 5275 } 5276 } 5277 |.cold_code 5278 |2: 5279 switch (type) { 5280 case BP_VAR_R: 5281 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5282 | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); 5283 | // retval = &EG(uninitialized_zval); 5284 | UNDEFINED_OFFSET opline 5285 | b >9 5286 } 5287 break; 5288 case BP_VAR_IS: 5289 case BP_VAR_UNSET: 5290 if (!not_found_exit_addr && !found_exit_addr) { 5291 | // retval = &EG(uninitialized_zval); 5292 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 5293 | b >9 5294 } 5295 break; 5296 default: 5297 ZEND_UNREACHABLE(); 5298 } 5299 |.code 5300 break; 5301 case BP_VAR_RW: 5302 if (packed_loaded && !not_found_exit_addr) { 5303 | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w 5304 } 5305 if (!packed_loaded || 5306 !not_found_exit_addr || 5307 (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5308 if (packed_loaded && not_found_exit_addr) { 5309 |.cold_code 5310 } 5311 |2: 5312 |4: 5313 if (!op2_loaded) { 5314 | // hval = Z_LVAL_P(dim); 5315 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5316 } 5317 if (packed_loaded) { 5318 | EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, REG0 5319 } else { 5320 | EXT_CALL zend_jit_hash_index_lookup_rw, REG0 5321 } 5322 | mov REG0, RETVALx 5323 if (not_found_exit_addr) { 5324 if (packed_loaded) { 5325 | cbnz REG0, >8 5326 | b ¬_found_exit_addr 5327 |.code 5328 } else { 5329 | cbz REG0, ¬_found_exit_addr 5330 } 5331 } else { 5332 | cbz REG0, >9 5333 } 5334 } 5335 break; 5336 case BP_VAR_W: 5337 if (packed_loaded) { 5338 | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w 5339 } 5340 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) { 5341 |2: 5342 |4: 5343 if (!op2_loaded) { 5344 | // hval = Z_LVAL_P(dim); 5345 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5346 } 5347 | EXT_CALL zend_hash_index_lookup, REG0 5348 | mov REG0, RETVALx 5349 } 5350 break; 5351 default: 5352 ZEND_UNREACHABLE(); 5353 } 5354 5355 if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { 5356 | b >8 5357 } 5358 } 5359 5360 if (op2_info & MAY_BE_STRING) { 5361 |3: 5362 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5363 | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) 5364 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3, ZREG_TMP1 5365 } 5366 | // offset_key = Z_STR_P(dim); 5367 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5368 | // retval = zend_hash_find(ht, offset_key); 5369 switch (type) { 5370 case BP_JIT_IS: 5371 if (opline->op2_type != IS_CONST) { 5372 | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] 5373 | cmp TMP1w, #((uint8_t) ('9')) 5374 | ble >1 5375 |.cold_code 5376 |1: 5377 | EXT_CALL zend_jit_symtable_find, REG0 5378 | b >1 5379 |.code 5380 | EXT_CALL zend_hash_find, REG0 5381 |1: 5382 } else { 5383 | EXT_CALL zend_hash_find_known_hash, REG0 5384 } 5385 | mov REG0, RETVALx 5386 if (not_found_exit_addr) { 5387 | cbz REG0, ¬_found_exit_addr 5388 } else { 5389 | cbz REG0, >9 // NOT_FOUND 5390 } 5391 break; 5392 case BP_VAR_R: 5393 case BP_VAR_IS: 5394 case BP_VAR_UNSET: 5395 if (opline->op2_type != IS_CONST) { 5396 | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] 5397 | cmp TMP1w, #((uint8_t) ('9')) 5398 | ble >1 5399 |.cold_code 5400 |1: 5401 | EXT_CALL zend_jit_symtable_find, REG0 5402 | b >1 5403 |.code 5404 | EXT_CALL zend_hash_find, REG0 5405 |1: 5406 } else { 5407 | EXT_CALL zend_hash_find_known_hash, REG0 5408 } 5409 | mov REG0, RETVALx 5410 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5411 | cbz REG0, &exit_addr 5412 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5413 | cbz REG0, ¬_found_exit_addr 5414 } else if (type == BP_VAR_IS && found_exit_addr) { 5415 | cbz REG0, >7 5416 } else { 5417 | cbz REG0, >2 // NOT_FOUND 5418 |.cold_code 5419 |2: 5420 switch (type) { 5421 case BP_VAR_R: 5422 // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); 5423 | UNDEFINED_INDEX opline 5424 | b >9 5425 break; 5426 case BP_VAR_IS: 5427 case BP_VAR_UNSET: 5428 | // retval = &EG(uninitialized_zval); 5429 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 5430 | b >9 5431 break; 5432 default: 5433 ZEND_UNREACHABLE(); 5434 } 5435 |.code 5436 } 5437 break; 5438 case BP_VAR_RW: 5439 if (opline->op2_type != IS_CONST) { 5440 | EXT_CALL zend_jit_symtable_lookup_rw, REG0 5441 } else { 5442 | EXT_CALL zend_jit_hash_lookup_rw, REG0 5443 } 5444 | mov REG0, RETVALx 5445 if (not_found_exit_addr) { 5446 | cbz REG0, ¬_found_exit_addr 5447 } else { 5448 | cbz REG0, >9 5449 } 5450 break; 5451 case BP_VAR_W: 5452 if (opline->op2_type != IS_CONST) { 5453 | EXT_CALL zend_jit_symtable_lookup_w, REG0 5454 } else { 5455 | EXT_CALL zend_hash_lookup, REG0 5456 } 5457 | mov REG0, RETVALx 5458 break; 5459 default: 5460 ZEND_UNREACHABLE(); 5461 } 5462 } 5463 5464 if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { 5465 |5: 5466 if (op1_info & MAY_BE_ARRAY_OF_REF) { 5467 | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w 5468 } 5469 | ldrb TMP1w, [REG0,#offsetof(zval, u1.v.type)] 5470 | cmp TMP1w, #IS_NULL 5471 if (not_found_exit_addr) { 5472 | ble ¬_found_exit_addr 5473 } else if (found_exit_addr) { 5474 | bgt &found_exit_addr 5475 } else { 5476 | ble >9 // NOT FOUND 5477 } 5478 } 5479 5480 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5481 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5482 |.cold_code 5483 |3: 5484 } 5485 if (type != BP_VAR_RW) { 5486 | SET_EX_OPLINE opline, REG0 5487 } 5488 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 5489 switch (type) { 5490 case BP_VAR_R: 5491 | LOAD_ZVAL_ADDR CARG3, res_addr 5492 | EXT_CALL zend_jit_fetch_dim_r_helper, REG0 5493 | mov REG0, RETVALx 5494 | b >9 5495 break; 5496 case BP_JIT_IS: 5497 | EXT_CALL zend_jit_fetch_dim_isset_helper, REG0 5498 | mov REG0, RETVALx 5499 if (not_found_exit_addr) { 5500 | cbz REG0, ¬_found_exit_addr 5501 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5502 | b >8 5503 } 5504 } else if (found_exit_addr) { 5505 | cbnz REG0, &found_exit_addr 5506 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5507 | b >9 5508 } 5509 } else { 5510 | cbnz REG0, >8 5511 | b >9 5512 } 5513 break; 5514 case BP_VAR_IS: 5515 case BP_VAR_UNSET: 5516 | LOAD_ZVAL_ADDR CARG3, res_addr 5517 | EXT_CALL zend_jit_fetch_dim_is_helper, REG0 5518 | mov REG0, RETVALx 5519 | b >9 5520 break; 5521 case BP_VAR_RW: 5522 | EXT_CALL zend_jit_fetch_dim_rw_helper, REG0 5523 | mov REG0, RETVALx 5524 | cbnz REG0, >8 5525 | b >9 5526 break; 5527 case BP_VAR_W: 5528 | EXT_CALL zend_jit_fetch_dim_w_helper, REG0 5529 | mov REG0, RETVALx 5530 | cbnz REG0, >8 5531 | b >9 5532 break; 5533 default: 5534 ZEND_UNREACHABLE(); 5535 } 5536 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5537 |.code 5538 } 5539 } 5540 5541 return 1; 5542} 5543 5544static int zend_jit_simple_assign(dasm_State **Dst, 5545 const zend_op *opline, 5546 zend_jit_addr var_addr, 5547 uint32_t var_info, 5548 uint32_t var_def_info, 5549 zend_uchar val_type, 5550 zend_jit_addr val_addr, 5551 uint32_t val_info, 5552 zend_jit_addr res_addr, 5553 int in_cold, 5554 int save_r1, 5555 bool check_exception) 5556/* Labels: 1,2,3 */ 5557{ 5558 zend_reg tmp_reg; 5559 5560 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_REG0) { 5561 tmp_reg = ZREG_REG0; 5562 } else { 5563 /* ASSIGN_DIM */ 5564 tmp_reg = ZREG_FCARG1; 5565 } 5566 5567 if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 5568 zval *zv = Z_ZV(val_addr); 5569 5570 if (!res_addr) { 5571 | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 5572 } else { 5573 | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 5574 } 5575 if (Z_REFCOUNTED_P(zv)) { 5576 if (!res_addr) { 5577 | ADDREF_CONST zv, TMP1, TMP2 5578 } else { 5579 | ADDREF_CONST_2 zv, TMP1, TMP2 5580 } 5581 } 5582 } else { 5583 if (val_info & MAY_BE_UNDEF) { 5584 if (in_cold) { 5585 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2, ZREG_TMP1 5586 } else { 5587 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 5588 |.cold_code 5589 |1: 5590 } 5591 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 5592 if (save_r1) { 5593 | str FCARG1x, T1 // save 5594 } 5595 | SET_ZVAL_TYPE_INFO var_addr, IS_NULL, TMP1w, TMP2 5596 if (res_addr) { 5597 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 5598 } 5599 if (opline) { 5600 | SET_EX_OPLINE opline, Rx(tmp_reg) 5601 } 5602 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); 5603 | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) 5604 | EXT_CALL zend_jit_undefined_op_helper, REG0 5605 if (check_exception) { 5606 | cbz RETVALx, ->exception_handler_undef 5607 } 5608 if (save_r1) { 5609 | ldr FCARG1x, T1 // restore 5610 } 5611 | b >3 5612 if (in_cold) { 5613 |2: 5614 } else { 5615 |.code 5616 } 5617 } 5618 if (val_info & MAY_BE_REF) { 5619 if (val_type == IS_CV) { 5620 ZEND_ASSERT(Z_REG(var_addr) != ZREG_REG2); 5621 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_REG2 || Z_OFFSET(val_addr) != 0) { 5622 | LOAD_ZVAL_ADDR REG2, val_addr 5623 } 5624 | ZVAL_DEREF REG2, val_info, TMP1w 5625 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); 5626 } else { 5627 zend_jit_addr ref_addr; 5628 5629 if (in_cold) { 5630 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 5631 } else { 5632 | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 5633 |.cold_code 5634 |1: 5635 } 5636 if (Z_REG(val_addr) == ZREG_REG2) { 5637 | str REG2, T1 // save 5638 } 5639 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 5640 | GET_ZVAL_PTR REG2, val_addr, TMP1 5641 | GC_DELREF REG2, TMP1w 5642 | // ZVAL_COPY_VALUE(return_value, &ref->val); 5643 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, offsetof(zend_reference, val)); 5644 if (!res_addr) { 5645 | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5646 } else { 5647 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5648 } 5649 | beq >2 // GC_DELREF() reached zero 5650 | IF_NOT_REFCOUNTED REG2w, >3, TMP1w 5651 if (!res_addr) { 5652 | GC_ADDREF Rx(tmp_reg), TMP1w 5653 } else { 5654 | GC_ADDREF_2 Rx(tmp_reg), TMP1w 5655 } 5656 | b >3 5657 |2: 5658 if (res_addr) { 5659 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w 5660 | GC_ADDREF Rx(tmp_reg), TMP1w 5661 |2: 5662 } 5663 if (Z_REG(val_addr) == ZREG_REG2) { 5664 | ldr REG2, T1 // restore 5665 } 5666 if (save_r1) { 5667 | str FCARG1x, T1 // save 5668 } 5669 | GET_ZVAL_PTR FCARG1x, val_addr, TMP1 5670 | EFREE_REFERENCE 5671 if (save_r1) { 5672 | ldr FCARG1x, T1 // restore 5673 } 5674 | b >3 5675 if (in_cold) { 5676 |1: 5677 } else { 5678 |.code 5679 } 5680 } 5681 } 5682 5683 if (!res_addr) { 5684 | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5685 } else { 5686 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5687 } 5688 5689 if (val_type == IS_CV) { 5690 if (!res_addr) { 5691 | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w 5692 } else { 5693 | TRY_ADDREF_2 val_info, REG2w, Rx(tmp_reg), TMP1w 5694 } 5695 } else { 5696 if (res_addr) { 5697 | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w 5698 } 5699 } 5700 |3: 5701 } 5702 return 1; 5703} 5704 5705static int zend_jit_assign_to_typed_ref(dasm_State **Dst, 5706 const zend_op *opline, 5707 zend_uchar val_type, 5708 zend_jit_addr val_addr, 5709 zend_jit_addr res_addr, 5710 bool check_exception) 5711{ 5712 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 5713 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 5714 | cbnz TMP1, >2 5715 |.cold_code 5716 |2: 5717 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 5718 | LOAD_ZVAL_ADDR FCARG2x, val_addr 5719 } 5720 if (opline) { 5721 | SET_EX_OPLINE opline, REG0 5722 } 5723 if (val_type == IS_CONST) { 5724 | EXT_CALL zend_jit_assign_const_to_typed_ref, REG0 5725 } else if (val_type == IS_TMP_VAR) { 5726 | EXT_CALL zend_jit_assign_tmp_to_typed_ref, REG0 5727 } else if (val_type == IS_VAR) { 5728 | EXT_CALL zend_jit_assign_var_to_typed_ref, REG0 5729 } else if (val_type == IS_CV) { 5730 | EXT_CALL zend_jit_assign_cv_to_typed_ref, REG0 5731 } else { 5732 ZEND_UNREACHABLE(); 5733 } 5734 if (res_addr) { 5735 zend_jit_addr ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_X0, 0); // RETVAL 5736 5737 | ZVAL_COPY_VALUE res_addr, -1, ret_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5738 | TRY_ADDREF -1, REG1w, REG2, TMP1w 5739 } 5740 if (check_exception) { 5741 | // if (UNEXPECTED(EG(exception) != NULL)) { 5742 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 5743 | cbz REG0, >8 // END OF zend_jit_assign_to_variable() 5744 | b ->exception_handler 5745 } else { 5746 | b >8 5747 } 5748 |.code 5749 5750 return 1; 5751} 5752 5753static int zend_jit_assign_to_variable_call(dasm_State **Dst, 5754 const zend_op *opline, 5755 zend_jit_addr __var_use_addr, 5756 zend_jit_addr var_addr, 5757 uint32_t __var_info, 5758 uint32_t __var_def_info, 5759 zend_uchar val_type, 5760 zend_jit_addr val_addr, 5761 uint32_t val_info, 5762 zend_jit_addr __res_addr, 5763 bool __check_exception) 5764{ 5765 if (val_info & MAY_BE_UNDEF) { 5766 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 5767 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5768 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5769 5770 if (!exit_addr) { 5771 return 0; 5772 } 5773 5774 | IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr, ZREG_TMP1 5775 } else { 5776 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 5777 |.cold_code 5778 |1: 5779 ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP); 5780 if (Z_REG(var_addr) != ZREG_FP) { 5781 | str Rx(Z_REG(var_addr)), T1 // save 5782 } 5783 | SET_EX_OPLINE opline, REG0 5784 | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) 5785 | EXT_CALL zend_jit_undefined_op_helper, REG0 5786 if (Z_REG(var_addr) != ZREG_FP) { 5787 | ldr Rx(Z_REG(var_addr)), T1 // restore 5788 } 5789 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 5790 | LOAD_ZVAL_ADDR FCARG1x, var_addr 5791 } 5792 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 5793 | bl ->assign_const 5794 | b >9 5795 |.code 5796 |1: 5797 } 5798 } 5799 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 5800 | LOAD_ZVAL_ADDR FCARG1x, var_addr 5801 } 5802 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 5803 | LOAD_ZVAL_ADDR FCARG2x, val_addr 5804 } 5805 if (opline) { 5806 | SET_EX_OPLINE opline, REG0 5807 } 5808 if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 5809 | bl ->assign_tmp 5810 } else if (val_type == IS_CONST) { 5811 | bl ->assign_const 5812 } else if (val_type == IS_TMP_VAR) { 5813 | bl ->assign_tmp 5814 } else if (val_type == IS_VAR) { 5815 if (!(val_info & MAY_BE_REF)) { 5816 | bl ->assign_tmp 5817 } else { 5818 | bl ->assign_var 5819 } 5820 } else if (val_type == IS_CV) { 5821 if (!(val_info & MAY_BE_REF)) { 5822 | bl ->assign_cv_noref 5823 } else { 5824 | bl ->assign_cv 5825 } 5826 if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5827 |9: 5828 } 5829 } else { 5830 ZEND_UNREACHABLE(); 5831 } 5832 5833 return 1; 5834} 5835 5836static int zend_jit_assign_to_variable(dasm_State **Dst, 5837 const zend_op *opline, 5838 zend_jit_addr var_use_addr, 5839 zend_jit_addr var_addr, 5840 uint32_t var_info, 5841 uint32_t var_def_info, 5842 zend_uchar val_type, 5843 zend_jit_addr val_addr, 5844 uint32_t val_info, 5845 zend_jit_addr res_addr, 5846 bool check_exception) 5847/* Labels: 1,2,3,4,5,8 */ 5848{ 5849 int done = 0; 5850 zend_reg ref_reg, tmp_reg; 5851 5852 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_REG0) { 5853 ref_reg = ZREG_FCARG1; 5854 tmp_reg = ZREG_REG0; 5855 } else { 5856 /* ASSIGN_DIM */ 5857 ref_reg = ZREG_REG0; 5858 tmp_reg = ZREG_FCARG1; 5859 } 5860 5861 if (var_info & MAY_BE_REF) { 5862 if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { 5863 | LOAD_ZVAL_ADDR Rx(ref_reg), var_use_addr 5864 var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); 5865 } 5866 | // if (Z_ISREF_P(variable_ptr)) { 5867 | IF_NOT_Z_TYPE Rx(ref_reg), IS_REFERENCE, >3, TMP1w 5868 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 5869 | GET_Z_PTR FCARG1x, Rx(ref_reg) 5870 if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) { 5871 return 0; 5872 } 5873 | add Rx(ref_reg), FCARG1x, #offsetof(zend_reference, val) 5874 |3: 5875 } 5876 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 5877 if (RC_MAY_BE_1(var_info)) { 5878 int in_cold = 0; 5879 5880 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 5881 | IF_ZVAL_REFCOUNTED var_use_addr, >1, ZREG_TMP1, ZREG_TMP2 5882 |.cold_code 5883 |1: 5884 in_cold = 1; 5885 } 5886 if (Z_REG(var_use_addr) == ZREG_FCARG1 || Z_REG(var_use_addr) == ZREG_REG0) { 5887 bool keep_gc = 0; 5888 5889 | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 5890#if 0 5891 // TODO: This optiization doesn't work on ARM 5892 if (tmp_reg == ZREG_FCARG1) { 5893 if (Z_MODE(val_addr) == IS_REG) { 5894 keep_gc = 1; 5895 } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { 5896 keep_gc = 1; 5897 } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 5898 zval *zv = Z_ZV(val_addr); 5899 if (Z_TYPE_P(zv) == IS_DOUBLE) { 5900 if (Z_DVAL_P(zv) == 0) { 5901 keep_gc = 1; 5902 } 5903 } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 5904 keep_gc = 1; 5905 } 5906 } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { 5907 if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 5908 keep_gc = 1; 5909 } 5910 } 5911 } 5912#endif 5913 if (!keep_gc) { 5914 | str Rx(tmp_reg), T1 // save 5915 } 5916 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0, 0)) { 5917 return 0; 5918 } 5919 if (!keep_gc) { 5920 | ldr FCARG1x, T1 // restore 5921 } 5922 } else { 5923 | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 5924 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1, 0)) { 5925 return 0; 5926 } 5927 } 5928 | GC_DELREF FCARG1x, TMP1w 5929 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 5930 | bne >4 5931 } else { 5932 | bne >8 5933 } 5934 | ZVAL_DTOR_FUNC var_info, opline, TMP1 5935 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { 5936 if (check_exception && !(val_info & MAY_BE_UNDEF)) { 5937 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 5938 | cbz REG0, >8 5939 | b ->exception_handler 5940 } else { 5941 | b >8 5942 } 5943 } 5944 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 5945 |4: 5946 | IF_GC_MAY_NOT_LEAK FCARG1x, >8, TMP1w, TMP2w 5947 | EXT_CALL gc_possible_root, REG0 5948 if (in_cold) { 5949 | b >8 5950 } 5951 } 5952 if (check_exception && (val_info & MAY_BE_UNDEF)) { 5953 |8: 5954 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 5955 | cbz REG0, >8 5956 | b ->exception_handler 5957 } 5958 if (in_cold) { 5959 |.code 5960 } else { 5961 done = 1; 5962 } 5963 } else /* if (RC_MAY_BE_N(var_info)) */ { 5964 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 5965 | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5, ZREG_TMP1, ZREG_TMP2 5966 } 5967 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { 5968 if (Z_REG(var_use_addr) != ZREG_FP) { 5969 | str Rx(Z_REG(var_use_addr)), T1 // save 5970 } 5971 | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 5972 | GC_DELREF FCARG1x, TMP1w 5973 | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w 5974 | EXT_CALL gc_possible_root, TMP1 5975 if (Z_REG(var_use_addr) != ZREG_FP) { 5976 | ldr Rx(Z_REG(var_use_addr)), T1 // restore 5977 } 5978 } else { 5979 | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 5980 | GC_DELREF Rx(tmp_reg), TMP1w 5981 } 5982 |5: 5983 } 5984 } 5985 5986 if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0, check_exception)) { 5987 return 0; 5988 } 5989 5990 |8: 5991 5992 return 1; 5993} 5994 5995static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw) 5996{ 5997 zend_jit_addr op2_addr, op3_addr, res_addr; 5998 5999 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6000 op3_addr = OP1_DATA_ADDR(); 6001 if (opline->result_type == IS_UNUSED) { 6002 res_addr = 0; 6003 } else { 6004 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 6005 } 6006 6007 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { 6008 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6009 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6010 6011 if (!exit_addr) { 6012 return 0; 6013 } 6014 6015 | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr, ZREG_TMP1 6016 6017 val_info &= ~MAY_BE_UNDEF; 6018 } 6019 6020 if (op1_info & MAY_BE_REF) { 6021 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6022 | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w 6023 | GET_Z_PTR FCARG2x, FCARG1x 6024 | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] 6025 | cmp TMP1w, #IS_ARRAY 6026 | bne >2 6027 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) 6028 | b >3 6029 |.cold_code 6030 |2: 6031 | SET_EX_OPLINE opline, REG0 6032 | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 6033 | mov FCARG1x, RETVALx 6034 | cbnz FCARG1x, >1 6035 | b ->exception_handler_undef 6036 |.code 6037 |1: 6038 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6039 } 6040 6041 if (op1_info & MAY_BE_ARRAY) { 6042 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6043 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 6044 } 6045 |3: 6046 | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 6047 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6048 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6049 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 6050 | bgt >7 6051 } 6052 | // ZVAL_ARR(container, zend_new_array(8)); 6053 if (Z_REG(op1_addr) != ZREG_FP) { 6054 | str Rx(Z_REG(op1_addr)), T1 // save 6055 } 6056 | EXT_CALL _zend_new_array_0, REG0 6057 | mov REG0, RETVALx 6058 if (Z_REG(op1_addr) != ZREG_FP) { 6059 | ldr Rx(Z_REG(op1_addr)), T1 // restore 6060 } 6061 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 6062 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 6063 | mov FCARG1x, REG0 6064 } 6065 6066 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6067 |6: 6068 if (opline->op2_type == IS_UNUSED) { 6069 uint32_t var_info = MAY_BE_NULL; 6070 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 6071 6072 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6073 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 6074 | EXT_CALL zend_hash_next_index_insert, REG0 6075 | // if (UNEXPECTED(!var_ptr)) { 6076 | mov REG0, RETVALx 6077 | cbz REG0, >1 6078 |.cold_code 6079 |1: 6080 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6081 | CANNOT_ADD_ELEMENT opline 6082 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6083 | b >9 6084 |.code 6085 6086 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0, 0)) { 6087 return 0; 6088 } 6089 } else { 6090 uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6091 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 6092 6093 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 6094 return 0; 6095 } 6096 6097 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6098 var_info |= MAY_BE_REF; 6099 } 6100 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6101 var_info |= MAY_BE_RC1; 6102 } 6103 6104 |8: 6105 | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); 6106 if (opline->op1_type == IS_VAR) { 6107 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6108 if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6109 return 0; 6110 } 6111 } else { 6112 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6113 return 0; 6114 } 6115 } 6116 } 6117 } 6118 6119 if (((op1_info & MAY_BE_ARRAY) && 6120 (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) || 6121 (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY)))) { 6122 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6123 |.cold_code 6124 |7: 6125 } 6126 6127 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) && 6128 (op1_info & MAY_BE_ARRAY)) { 6129 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6130 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 6131 | bgt >2 6132 } 6133 | // ZVAL_ARR(container, zend_new_array(8)); 6134 if (Z_REG(op1_addr) != ZREG_FP) { 6135 | str Rx(Z_REG(op1_addr)), T1 // save 6136 } 6137 | EXT_CALL _zend_new_array_0, REG0 6138 | mov REG0, RETVALx 6139 if (Z_REG(op1_addr) != ZREG_FP) { 6140 | ldr Rx(Z_REG(op1_addr)), T1 // restore 6141 } 6142 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 6143 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 6144 | mov FCARG1x, REG0 6145 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); 6146 | b <6 6147 |2: 6148 } 6149 6150 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6151 | SET_EX_OPLINE opline, REG0 6152 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6153 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6154 } 6155 if (opline->op2_type == IS_UNUSED) { 6156 | mov FCARG2x, xzr 6157 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6158 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6159 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 6160 } else { 6161 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 6162 } 6163 if (opline->result_type == IS_UNUSED) { 6164 | mov CARG4, xzr 6165 } else { 6166 | LOAD_ZVAL_ADDR CARG4, res_addr 6167 } 6168 | LOAD_ZVAL_ADDR CARG3, op3_addr 6169 | EXT_CALL zend_jit_assign_dim_helper, REG0 6170 6171#ifdef ZEND_JIT_USE_RC_INFERENCE 6172 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { 6173 /* ASSIGN_DIM may increase refcount of the value */ 6174 val_info |= MAY_BE_RCN; 6175 } 6176#endif 6177 6178 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 6179 } 6180 6181 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6182 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6183 | b >9 // END 6184 } 6185 |.code 6186 } 6187 } 6188 6189#ifdef ZEND_JIT_USE_RC_INFERENCE 6190 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 6191 /* ASSIGN_DIM may increase refcount of the key */ 6192 op2_info |= MAY_BE_RCN; 6193 } 6194#endif 6195 6196 |9: 6197 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6198 6199 if (may_throw) { 6200 zend_jit_check_exception(Dst); 6201 } 6202 6203 return 1; 6204} 6205 6206static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) 6207{ 6208 zend_jit_addr op2_addr, op3_addr, var_addr; 6209 const void *not_found_exit_addr = NULL; 6210 uint32_t var_info = MAY_BE_NULL; 6211 6212 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6213 6214 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6215 op3_addr = OP1_DATA_ADDR(); 6216 6217 | SET_EX_OPLINE opline, REG0 6218 if (op1_info & MAY_BE_REF) { 6219 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6220 | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w 6221 | GET_Z_PTR FCARG2x, FCARG1x 6222 | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] 6223 | cmp TMP1w, #IS_ARRAY 6224 | bne >2 6225 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) 6226 | b >3 6227 |.cold_code 6228 |2: 6229 | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 6230 | mov FCARG1x, RETVALx 6231 | cbnz RETVALx, >1 6232 | b ->exception_handler_undef 6233 |.code 6234 |1: 6235 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6236 } 6237 6238 if (op1_info & MAY_BE_ARRAY) { 6239 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6240 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 6241 } 6242 |3: 6243 | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 6244 } 6245 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6246 if (op1_info & MAY_BE_ARRAY) { 6247 |.cold_code 6248 |7: 6249 } 6250 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6251 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 6252 | bgt >7 6253 } 6254 if (Z_REG(op1_addr) != ZREG_FP) { 6255 | str Rx(Z_REG(op1_addr)), T1 // save 6256 } 6257 if (op1_info & MAY_BE_UNDEF) { 6258 if (op1_info & MAY_BE_NULL) { 6259 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 6260 } 6261 | LOAD_32BIT_VAL FCARG1x, opline->op1.var 6262 | EXT_CALL zend_jit_undefined_op_helper, REG0 6263 |1: 6264 } 6265 | // ZVAL_ARR(container, zend_new_array(8)); 6266 | EXT_CALL _zend_new_array_0, REG0 6267 | mov REG0, RETVALx 6268 if (Z_REG(op1_addr) != ZREG_FP) { 6269 | ldr Rx(Z_REG(op1_addr)), T1 // restore 6270 } 6271 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 6272 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 6273 | mov FCARG1x, REG0 6274 if (op1_info & MAY_BE_ARRAY) { 6275 | b >1 6276 |.code 6277 |1: 6278 } 6279 } 6280 6281 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6282 uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); 6283 6284 |6: 6285 if (opline->op2_type == IS_UNUSED) { 6286 var_info = MAY_BE_NULL; 6287 6288 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6289 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 6290 | EXT_CALL zend_hash_next_index_insert, REG0 6291 | mov REG0, RETVALx 6292 | // if (UNEXPECTED(!var_ptr)) { 6293 | cbz REG0, >1 6294 |.cold_code 6295 |1: 6296 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6297 | CANNOT_ADD_ELEMENT opline 6298 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6299 | b >9 6300 |.code 6301 } else { 6302 var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6303 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6304 var_info |= MAY_BE_REF; 6305 } 6306 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6307 var_info |= MAY_BE_RC1; 6308 } 6309 6310 if (dim_type != IS_UNKNOWN 6311 && dim_type != IS_UNDEF 6312 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY 6313 && (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) 6314 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 6315 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 6316 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6317 if (!not_found_exit_addr) { 6318 return 0; 6319 } 6320 } 6321 6322 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, not_found_exit_addr, NULL)) { 6323 return 0; 6324 } 6325 6326 |8: 6327 if (not_found_exit_addr && dim_type != IS_REFERENCE) { 6328 | IF_NOT_Z_TYPE, REG0, dim_type, ¬_found_exit_addr, TMP1w 6329 var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 6330 } 6331 if (var_info & MAY_BE_REF) { 6332 binary_op_type binary_op = get_binary_op(opline->extended_value); 6333 | IF_NOT_Z_TYPE, REG0, IS_REFERENCE, >1, TMP1w 6334 | GET_Z_PTR FCARG1x, REG0 6335 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 6336 | cbnz TMP1, >2 6337 | add REG0, FCARG1x, #offsetof(zend_reference, val) 6338 |.cold_code 6339 |2: 6340 | LOAD_ZVAL_ADDR FCARG2x, op3_addr 6341 | LOAD_ADDR CARG3, binary_op 6342 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 6343 && (op1_data_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6344 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 6345 } else { 6346 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 6347 } 6348 | b >9 6349 |.code 6350 |1: 6351 } 6352 } 6353 6354 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 6355 switch (opline->extended_value) { 6356 case ZEND_ADD: 6357 case ZEND_SUB: 6358 case ZEND_MUL: 6359 case ZEND_DIV: 6360 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, 6361 1 /* may overflow */, may_throw)) { 6362 return 0; 6363 } 6364 break; 6365 case ZEND_BW_OR: 6366 case ZEND_BW_AND: 6367 case ZEND_BW_XOR: 6368 case ZEND_SL: 6369 case ZEND_SR: 6370 case ZEND_MOD: 6371 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6372 IS_CV, opline->op1, var_addr, var_info, NULL, 6373 (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 6374 op1_data_range, 6375 0, var_addr, var_def_info, var_info, may_throw)) { 6376 return 0; 6377 } 6378 break; 6379 case ZEND_CONCAT: 6380 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, 6381 may_throw)) { 6382 return 0; 6383 } 6384 break; 6385 default: 6386 ZEND_UNREACHABLE(); 6387 } 6388 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6389 } 6390 6391 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6392 binary_op_type binary_op; 6393 6394 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6395 |.cold_code 6396 |7: 6397 } 6398 6399 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6400 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6401 } 6402 if (opline->op2_type == IS_UNUSED) { 6403 | mov FCARG2x, xzr 6404 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6405 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6406 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 6407 } else { 6408 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 6409 } 6410 binary_op = get_binary_op(opline->extended_value); 6411 | LOAD_ZVAL_ADDR CARG3, op3_addr 6412 | LOAD_ADDR CARG4, binary_op 6413 | EXT_CALL zend_jit_assign_dim_op_helper, REG0 6414 6415 |9: 6416 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 6417 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 6418 if (may_throw) { 6419 zend_jit_check_exception(Dst); 6420 } 6421 6422 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6423 | b >9 // END 6424 |.code 6425 |9: 6426 } 6427 } else if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) 6428 && (!not_found_exit_addr || (var_info & MAY_BE_REF))) { 6429 |.cold_code 6430 |9: 6431 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6432 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6433 if (may_throw) { 6434 zend_jit_check_exception(Dst); 6435 } 6436 | b >9 6437 |.code 6438 |9: 6439 } 6440 6441 return 1; 6442} 6443 6444static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) 6445{ 6446 zend_jit_addr op1_addr, op2_addr; 6447 6448 ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); 6449 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 6450 6451 op1_addr = OP1_ADDR(); 6452 op2_addr = OP2_ADDR(); 6453 6454 if (op1_info & MAY_BE_REF) { 6455 binary_op_type binary_op = get_binary_op(opline->extended_value); 6456 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6457 | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >1, TMP1w 6458 | GET_Z_PTR FCARG1x, FCARG1x 6459 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 6460 | cbnz TMP1, >2 6461 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) 6462 |.cold_code 6463 |2: 6464 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 6465 | LOAD_ADDR CARG3, binary_op 6466 | SET_EX_OPLINE opline, REG0 6467 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 6468 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6469 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 6470 } else { 6471 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 6472 } 6473 zend_jit_check_exception(Dst); 6474 | b >9 6475 |.code 6476 |1: 6477 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6478 } 6479 6480 int result; 6481 switch (opline->extended_value) { 6482 case ZEND_ADD: 6483 case ZEND_SUB: 6484 case ZEND_MUL: 6485 case ZEND_DIV: 6486 result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); 6487 break; 6488 case ZEND_BW_OR: 6489 case ZEND_BW_AND: 6490 case ZEND_BW_XOR: 6491 case ZEND_SL: 6492 case ZEND_SR: 6493 case ZEND_MOD: 6494 result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6495 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 6496 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 6497 opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); 6498 break; 6499 case ZEND_CONCAT: 6500 result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); 6501 break; 6502 default: 6503 ZEND_UNREACHABLE(); 6504 } 6505 |9: 6506 return result; 6507} 6508 6509static int zend_jit_cmp_long_long(dasm_State **Dst, 6510 const zend_op *opline, 6511 zend_ssa_range *op1_range, 6512 zend_jit_addr op1_addr, 6513 zend_ssa_range *op2_range, 6514 zend_jit_addr op2_addr, 6515 zend_jit_addr res_addr, 6516 zend_uchar smart_branch_opcode, 6517 uint32_t target_label, 6518 uint32_t target_label2, 6519 const void *exit_addr, 6520 bool skip_comparison) 6521{ 6522 bool swap = 0; 6523 bool result; 6524 6525 if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { 6526 if (!smart_branch_opcode || 6527 smart_branch_opcode == ZEND_JMPZ_EX || 6528 smart_branch_opcode == ZEND_JMPNZ_EX) { 6529 | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE), TMP1w, TMP2 6530 } 6531 if (smart_branch_opcode && !exit_addr) { 6532 if (smart_branch_opcode == ZEND_JMPZ || 6533 smart_branch_opcode == ZEND_JMPZ_EX) { 6534 if (!result) { 6535 | b => target_label 6536 } 6537 } else if (smart_branch_opcode == ZEND_JMPNZ || 6538 smart_branch_opcode == ZEND_JMPNZ_EX) { 6539 if (result) { 6540 | b => target_label 6541 } 6542 } else { 6543 ZEND_UNREACHABLE(); 6544 } 6545 } 6546 return 1; 6547 } 6548 6549 if (skip_comparison) { 6550 if (Z_MODE(op1_addr) != IS_REG && 6551 (Z_MODE(op2_addr) == IS_REG || 6552 (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { 6553 swap = 1; 6554 } 6555 } else if (Z_MODE(op1_addr) == IS_REG) { 6556 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 6557 | cmp Rx(Z_REG(op1_addr)), xzr 6558 } else { 6559 | LONG_CMP Z_REG(op1_addr), op2_addr, TMP1 6560 } 6561 } else if (Z_MODE(op2_addr) == IS_REG) { 6562 if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { 6563 | cmp Rx(Z_REG(op2_addr)), xzr 6564 } else { 6565 | LONG_CMP Z_REG(op2_addr), op1_addr, TMP1 6566 } 6567 swap = 1; 6568 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 6569 | LONG_CMP_WITH_CONST op2_addr, Z_LVAL_P(Z_ZV(op1_addr)), TMP1, TMP2 6570 swap = 1; 6571 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { 6572 | LONG_CMP_WITH_CONST op1_addr, Z_LVAL_P(Z_ZV(op2_addr)), TMP1, TMP2 6573 } else { 6574 | GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1 6575 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 6576 | cmp Rx(ZREG_REG0), xzr 6577 } else { 6578 | LONG_CMP ZREG_REG0, op2_addr, TMP1 6579 } 6580 } 6581 6582 if (smart_branch_opcode) { 6583 if (smart_branch_opcode == ZEND_JMPZ_EX || 6584 smart_branch_opcode == ZEND_JMPNZ_EX) { 6585 6586 switch (opline->opcode) { 6587 case ZEND_IS_EQUAL: 6588 case ZEND_IS_IDENTICAL: 6589 case ZEND_CASE: 6590 case ZEND_CASE_STRICT: 6591 | cset REG0w, eq 6592 break; 6593 case ZEND_IS_NOT_EQUAL: 6594 case ZEND_IS_NOT_IDENTICAL: 6595 | cset REG0w, ne 6596 break; 6597 case ZEND_IS_SMALLER: 6598 if (swap) { 6599 | cset REG0w, gt 6600 } else { 6601 | cset REG0w, lt 6602 } 6603 break; 6604 case ZEND_IS_SMALLER_OR_EQUAL: 6605 if (swap) { 6606 | cset REG0w, ge 6607 } else { 6608 | cset REG0w, le 6609 } 6610 break; 6611 default: 6612 ZEND_UNREACHABLE(); 6613 } 6614 | add REG0w, REG0w, #2 6615 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 6616 } 6617 if (smart_branch_opcode == ZEND_JMPZ || 6618 smart_branch_opcode == ZEND_JMPZ_EX) { 6619 switch (opline->opcode) { 6620 case ZEND_IS_EQUAL: 6621 case ZEND_IS_IDENTICAL: 6622 case ZEND_CASE: 6623 case ZEND_CASE_STRICT: 6624 if (exit_addr) { 6625 | bne &exit_addr 6626 } else { 6627 | bne => target_label 6628 } 6629 break; 6630 case ZEND_IS_NOT_EQUAL: 6631 if (exit_addr) { 6632 | beq &exit_addr 6633 } else { 6634 | beq => target_label 6635 } 6636 break; 6637 case ZEND_IS_NOT_IDENTICAL: 6638 if (exit_addr) { 6639 | bne &exit_addr 6640 } else { 6641 | beq => target_label 6642 } 6643 break; 6644 case ZEND_IS_SMALLER: 6645 if (swap) { 6646 if (exit_addr) { 6647 | ble &exit_addr 6648 } else { 6649 | ble => target_label 6650 } 6651 } else { 6652 if (exit_addr) { 6653 | bge &exit_addr 6654 } else { 6655 | bge => target_label 6656 } 6657 } 6658 break; 6659 case ZEND_IS_SMALLER_OR_EQUAL: 6660 if (swap) { 6661 if (exit_addr) { 6662 | blt &exit_addr 6663 } else { 6664 | blt => target_label 6665 } 6666 } else { 6667 if (exit_addr) { 6668 | bgt &exit_addr 6669 } else { 6670 | bgt => target_label 6671 } 6672 } 6673 break; 6674 default: 6675 ZEND_UNREACHABLE(); 6676 } 6677 } else if (smart_branch_opcode == ZEND_JMPNZ || 6678 smart_branch_opcode == ZEND_JMPNZ_EX) { 6679 switch (opline->opcode) { 6680 case ZEND_IS_EQUAL: 6681 case ZEND_IS_IDENTICAL: 6682 case ZEND_CASE: 6683 case ZEND_CASE_STRICT: 6684 if (exit_addr) { 6685 | beq &exit_addr 6686 } else { 6687 | beq => target_label 6688 } 6689 break; 6690 case ZEND_IS_NOT_EQUAL: 6691 if (exit_addr) { 6692 | bne &exit_addr 6693 } else { 6694 | bne => target_label 6695 } 6696 break; 6697 case ZEND_IS_NOT_IDENTICAL: 6698 if (exit_addr) { 6699 | beq &exit_addr 6700 } else { 6701 | bne => target_label 6702 } 6703 break; 6704 case ZEND_IS_SMALLER: 6705 if (swap) { 6706 if (exit_addr) { 6707 | bgt &exit_addr 6708 } else { 6709 | bgt => target_label 6710 } 6711 } else { 6712 if (exit_addr) { 6713 | blt &exit_addr 6714 } else { 6715 | blt => target_label 6716 } 6717 } 6718 break; 6719 case ZEND_IS_SMALLER_OR_EQUAL: 6720 if (swap) { 6721 if (exit_addr) { 6722 | bge &exit_addr 6723 } else { 6724 | bge => target_label 6725 } 6726 } else { 6727 if (exit_addr) { 6728 | ble &exit_addr 6729 } else { 6730 | ble => target_label 6731 } 6732 } 6733 break; 6734 default: 6735 ZEND_UNREACHABLE(); 6736 } 6737 } else { 6738 ZEND_UNREACHABLE(); 6739 } 6740 } else { 6741 switch (opline->opcode) { 6742 case ZEND_IS_EQUAL: 6743 case ZEND_IS_IDENTICAL: 6744 case ZEND_CASE: 6745 case ZEND_CASE_STRICT: 6746 | cset REG0w, eq 6747 break; 6748 case ZEND_IS_NOT_EQUAL: 6749 case ZEND_IS_NOT_IDENTICAL: 6750 | cset REG0w, ne 6751 break; 6752 case ZEND_IS_SMALLER: 6753 if (swap) { 6754 | cset REG0w, gt 6755 } else { 6756 | cset REG0w, lt 6757 } 6758 break; 6759 case ZEND_IS_SMALLER_OR_EQUAL: 6760 if (swap) { 6761 | cset REG0w, ge 6762 } else { 6763 | cset REG0w, le 6764 } 6765 break; 6766 default: 6767 ZEND_UNREACHABLE(); 6768 } 6769 | add REG0w, REG0w, #2 6770 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 6771 } 6772 6773 return 1; 6774} 6775 6776static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 6777{ 6778 if (smart_branch_opcode) { 6779 if (smart_branch_opcode == ZEND_JMPZ) { 6780 switch (opline->opcode) { 6781 case ZEND_IS_EQUAL: 6782 case ZEND_IS_IDENTICAL: 6783 case ZEND_CASE: 6784 case ZEND_CASE_STRICT: 6785 if (exit_addr) { 6786 | bne &exit_addr 6787 } else { 6788 | bne => target_label 6789 } 6790 break; 6791 case ZEND_IS_NOT_EQUAL: 6792 | bvs >1 6793 if (exit_addr) { 6794 | beq &exit_addr 6795 } else { 6796 | beq => target_label 6797 } 6798 |1: 6799 break; 6800 case ZEND_IS_NOT_IDENTICAL: 6801 if (exit_addr) { 6802 | bvs &exit_addr 6803 | bne &exit_addr 6804 } else { 6805 | bvs >1 6806 | beq => target_label 6807 |1: 6808 } 6809 break; 6810 case ZEND_IS_SMALLER: 6811 if (swap) { 6812 if (exit_addr) { 6813 | bvs &exit_addr 6814 | bls &exit_addr 6815 } else { 6816 | bvs => target_label 6817 | bls => target_label 6818 } 6819 } else { 6820 if (exit_addr) { 6821 | bhs &exit_addr 6822 } else { 6823 | bhs => target_label 6824 } 6825 } 6826 break; 6827 case ZEND_IS_SMALLER_OR_EQUAL: 6828 if (swap) { 6829 if (exit_addr) { 6830 | bvs &exit_addr 6831 | blo &exit_addr 6832 } else { 6833 | bvs => target_label 6834 | blo => target_label 6835 } 6836 } else { 6837 if (exit_addr) { 6838 | bhi &exit_addr 6839 } else { 6840 | bhi => target_label 6841 } 6842 } 6843 break; 6844 default: 6845 ZEND_UNREACHABLE(); 6846 } 6847 } else if (smart_branch_opcode == ZEND_JMPNZ) { 6848 switch (opline->opcode) { 6849 case ZEND_IS_EQUAL: 6850 case ZEND_IS_IDENTICAL: 6851 case ZEND_CASE: 6852 case ZEND_CASE_STRICT: 6853 | bvs >1 6854 if (exit_addr) { 6855 | beq &exit_addr 6856 } else { 6857 | beq => target_label 6858 } 6859 |1: 6860 break; 6861 case ZEND_IS_NOT_EQUAL: 6862 if (exit_addr) { 6863 | bne &exit_addr 6864 } else { 6865 | bne => target_label 6866 } 6867 break; 6868 case ZEND_IS_NOT_IDENTICAL: 6869 if (exit_addr) { 6870 | bvs >1 6871 | beq &exit_addr 6872 |1: 6873 } else { 6874 | bne => target_label 6875 } 6876 break; 6877 case ZEND_IS_SMALLER: 6878 if (swap) { 6879 | bvs >1 // Always False if involving NaN 6880 if (exit_addr) { 6881 | bhi &exit_addr 6882 } else { 6883 | bhi => target_label 6884 } 6885 |1: 6886 } else { 6887 | bvs >1 6888 if (exit_addr) { 6889 | blo &exit_addr 6890 } else { 6891 | blo => target_label 6892 } 6893 |1: 6894 } 6895 break; 6896 case ZEND_IS_SMALLER_OR_EQUAL: 6897 if (swap) { 6898 | bvs >1 // Always False if involving NaN 6899 if (exit_addr) { 6900 | bhs &exit_addr 6901 } else { 6902 | bhs => target_label 6903 } 6904 |1: 6905 } else { 6906 | bvs >1 6907 if (exit_addr) { 6908 | bls &exit_addr 6909 } else { 6910 | bls => target_label 6911 } 6912 |1: 6913 } 6914 break; 6915 default: 6916 ZEND_UNREACHABLE(); 6917 } 6918 } else if (smart_branch_opcode == ZEND_JMPZ_EX) { 6919 switch (opline->opcode) { 6920 case ZEND_IS_EQUAL: 6921 case ZEND_IS_IDENTICAL: 6922 case ZEND_CASE: 6923 case ZEND_CASE_STRICT: 6924 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6925 | bne => target_label 6926 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6927 break; 6928 case ZEND_IS_NOT_EQUAL: 6929 case ZEND_IS_NOT_IDENTICAL: 6930 | bvs >1 6931 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6932 | beq => target_label 6933 |1: 6934 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6935 break; 6936 case ZEND_IS_SMALLER: 6937 if (swap) { 6938 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6939 | bvs => target_label 6940 | bls => target_label 6941 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6942 } else { 6943 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6944 | bhs => target_label 6945 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6946 } 6947 break; 6948 case ZEND_IS_SMALLER_OR_EQUAL: 6949 if (swap) { 6950 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6951 | bvs => target_label 6952 | blo => target_label 6953 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6954 } else { 6955 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6956 | bhi => target_label 6957 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6958 } 6959 break; 6960 default: 6961 ZEND_UNREACHABLE(); 6962 } 6963 } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { 6964 switch (opline->opcode) { 6965 case ZEND_IS_EQUAL: 6966 case ZEND_IS_IDENTICAL: 6967 case ZEND_CASE: 6968 case ZEND_CASE_STRICT: 6969 | bvs >1 6970 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6971 | beq => target_label 6972 |1: 6973 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6974 break; 6975 case ZEND_IS_NOT_EQUAL: 6976 case ZEND_IS_NOT_IDENTICAL: 6977 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6978 | bvs => target_label 6979 | bne => target_label 6980 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6981 break; 6982 case ZEND_IS_SMALLER: 6983 if (swap) { 6984 | cset REG0w, hi 6985 | add REG0w, REG0w, #2 6986 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 6987 | bvs >1 // Always False if involving NaN 6988 | bhi => target_label 6989 |1: 6990 } else { 6991 | bvs >1 6992 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6993 | blo => target_label 6994 |1: 6995 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6996 } 6997 break; 6998 case ZEND_IS_SMALLER_OR_EQUAL: 6999 if (swap) { 7000 | cset REG0w, hs 7001 | add REG0w, REG0w, #2 7002 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7003 | bvs >1 // Always False if involving NaN 7004 | bhs => target_label 7005 |1: 7006 } else { 7007 | bvs >1 7008 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7009 | bls => target_label 7010 |1: 7011 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7012 } 7013 break; 7014 default: 7015 ZEND_UNREACHABLE(); 7016 } 7017 } else { 7018 ZEND_UNREACHABLE(); 7019 } 7020 } else { 7021 switch (opline->opcode) { 7022 case ZEND_IS_EQUAL: 7023 case ZEND_IS_IDENTICAL: 7024 case ZEND_CASE: 7025 case ZEND_CASE_STRICT: 7026 | bvs >1 7027 | mov REG0, #IS_TRUE 7028 | beq >2 7029 |1: 7030 | mov REG0, #IS_FALSE 7031 |2: 7032 break; 7033 case ZEND_IS_NOT_EQUAL: 7034 case ZEND_IS_NOT_IDENTICAL: 7035 | bvs >1 7036 | mov REG0, #IS_FALSE 7037 | beq >2 7038 |1: 7039 | mov REG0, #IS_TRUE 7040 |2: 7041 break; 7042 case ZEND_IS_SMALLER: 7043 | bvs >1 7044 | mov REG0, #IS_TRUE 7045 || if (swap) { 7046 | bhi >2 7047 || } else { 7048 | blo >2 7049 || } 7050 |1: 7051 | mov REG0, #IS_FALSE 7052 |2: 7053 break; 7054 case ZEND_IS_SMALLER_OR_EQUAL: 7055 | bvs >1 7056 | mov REG0, #IS_TRUE 7057 || if (swap) { 7058 | bhs >2 7059 || } else { 7060 | bls >2 7061 || } 7062 |1: 7063 | mov REG0, #IS_FALSE 7064 |2: 7065 break; 7066 default: 7067 ZEND_UNREACHABLE(); 7068 } 7069 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7070 } 7071 7072 return 1; 7073} 7074 7075static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7076{ 7077 zend_reg tmp_reg = ZREG_FPR0; 7078 7079 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_REG0, ZREG_TMP1 7080 | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP 7081 7082 return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); 7083} 7084 7085static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7086{ 7087 zend_reg tmp_reg = ZREG_FPR0; 7088 7089 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_REG0, ZREG_TMP1 7090 | DOUBLE_CMP tmp_reg, op1_addr, ZREG_TMP1, ZREG_FPTMP 7091 7092 return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); 7093} 7094 7095static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7096{ 7097 bool swap = 0; 7098 7099 if (Z_MODE(op1_addr) == IS_REG) { 7100 | DOUBLE_CMP Z_REG(op1_addr), op2_addr, ZREG_TMP1, ZREG_FPTMP 7101 } else if (Z_MODE(op2_addr) == IS_REG) { 7102 | DOUBLE_CMP Z_REG(op2_addr), op1_addr, ZREG_TMP1, ZREG_FPTMP 7103 swap = 1; 7104 } else { 7105 zend_reg tmp_reg = ZREG_FPR0; 7106 7107 | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 7108 | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP 7109 } 7110 7111 return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); 7112} 7113 7114static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7115{ 7116 | tst RETVALw, RETVALw 7117 if (smart_branch_opcode) { 7118 if (smart_branch_opcode == ZEND_JMPZ_EX || 7119 smart_branch_opcode == ZEND_JMPNZ_EX) { 7120 switch (opline->opcode) { 7121 case ZEND_IS_EQUAL: 7122 case ZEND_CASE: 7123 | cset REG0w, eq 7124 break; 7125 case ZEND_IS_NOT_EQUAL: 7126 | cset REG0w, ne 7127 break; 7128 case ZEND_IS_SMALLER: 7129 | cset REG0w, lt 7130 break; 7131 case ZEND_IS_SMALLER_OR_EQUAL: 7132 | cset REG0w, le 7133 break; 7134 default: 7135 ZEND_UNREACHABLE(); 7136 } 7137 | add REG0w, REG0w, #2 7138 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7139 } 7140 if (smart_branch_opcode == ZEND_JMPZ || 7141 smart_branch_opcode == ZEND_JMPZ_EX) { 7142 switch (opline->opcode) { 7143 case ZEND_IS_EQUAL: 7144 case ZEND_CASE: 7145 if (exit_addr) { 7146 | bne &exit_addr 7147 } else { 7148 | bne => target_label 7149 } 7150 break; 7151 case ZEND_IS_NOT_EQUAL: 7152 if (exit_addr) { 7153 | beq &exit_addr 7154 } else { 7155 | beq => target_label 7156 } 7157 break; 7158 case ZEND_IS_SMALLER: 7159 if (exit_addr) { 7160 | bge &exit_addr 7161 } else { 7162 | bge => target_label 7163 } 7164 break; 7165 case ZEND_IS_SMALLER_OR_EQUAL: 7166 if (exit_addr) { 7167 | bgt &exit_addr 7168 } else { 7169 | bgt => target_label 7170 } 7171 break; 7172 default: 7173 ZEND_UNREACHABLE(); 7174 } 7175 } else if (smart_branch_opcode == ZEND_JMPNZ || 7176 smart_branch_opcode == ZEND_JMPNZ_EX) { 7177 switch (opline->opcode) { 7178 case ZEND_IS_EQUAL: 7179 case ZEND_CASE: 7180 if (exit_addr) { 7181 | beq &exit_addr 7182 } else { 7183 | beq => target_label 7184 } 7185 break; 7186 case ZEND_IS_NOT_EQUAL: 7187 if (exit_addr) { 7188 | bne &exit_addr 7189 } else { 7190 | bne => target_label 7191 } 7192 break; 7193 case ZEND_IS_SMALLER: 7194 if (exit_addr) { 7195 | blt &exit_addr 7196 } else { 7197 | blt => target_label 7198 } 7199 break; 7200 case ZEND_IS_SMALLER_OR_EQUAL: 7201 if (exit_addr) { 7202 | ble &exit_addr 7203 } else { 7204 | ble => target_label 7205 } 7206 break; 7207 default: 7208 ZEND_UNREACHABLE(); 7209 } 7210 } else { 7211 ZEND_UNREACHABLE(); 7212 } 7213 } else { 7214 switch (opline->opcode) { 7215 case ZEND_IS_EQUAL: 7216 case ZEND_CASE: 7217 | cset REG0w, eq 7218 break; 7219 case ZEND_IS_NOT_EQUAL: 7220 | cset REG0w, ne 7221 break; 7222 case ZEND_IS_SMALLER: 7223 | cset REG0w, lt 7224 break; 7225 case ZEND_IS_SMALLER_OR_EQUAL: 7226 | cset REG0w, le 7227 break; 7228 default: 7229 ZEND_UNREACHABLE(); 7230 } 7231 | add REG0w, REG0w, #2 7232 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7233 } 7234 7235 return 1; 7236} 7237 7238static int zend_jit_cmp(dasm_State **Dst, 7239 const zend_op *opline, 7240 uint32_t op1_info, 7241 zend_ssa_range *op1_range, 7242 zend_jit_addr op1_addr, 7243 uint32_t op2_info, 7244 zend_ssa_range *op2_range, 7245 zend_jit_addr op2_addr, 7246 zend_jit_addr res_addr, 7247 int may_throw, 7248 zend_uchar smart_branch_opcode, 7249 uint32_t target_label, 7250 uint32_t target_label2, 7251 const void *exit_addr, 7252 bool skip_comparison) 7253{ 7254 bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); 7255 bool has_slow; 7256 7257 has_slow = 7258 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7259 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7260 ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7261 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); 7262 7263 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 7264 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 7265 if (op1_info & MAY_BE_DOUBLE) { 7266 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4, ZREG_TMP1 7267 } else { 7268 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 7269 } 7270 } 7271 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 7272 if (op2_info & MAY_BE_DOUBLE) { 7273 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 7274 |.cold_code 7275 |3: 7276 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7277 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7278 } 7279 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7280 return 0; 7281 } 7282 | b >6 7283 |.code 7284 } else { 7285 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 7286 } 7287 } 7288 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 7289 return 0; 7290 } 7291 if (op1_info & MAY_BE_DOUBLE) { 7292 |.cold_code 7293 |4: 7294 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7295 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 7296 } 7297 if (op2_info & MAY_BE_DOUBLE) { 7298 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7299 if (!same_ops) { 7300 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5, ZREG_TMP1 7301 } else { 7302 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7303 } 7304 } 7305 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7306 return 0; 7307 } 7308 | b >6 7309 } 7310 if (!same_ops) { 7311 |5: 7312 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7313 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 7314 } 7315 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7316 return 0; 7317 } 7318 | b >6 7319 } 7320 |.code 7321 } 7322 } else if ((op1_info & MAY_BE_DOUBLE) && 7323 !(op1_info & MAY_BE_LONG) && 7324 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7325 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7326 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 7327 } 7328 if (op2_info & MAY_BE_DOUBLE) { 7329 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7330 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7331 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3, ZREG_TMP1 7332 } else { 7333 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7334 } 7335 } 7336 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7337 return 0; 7338 } 7339 } 7340 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7341 if (op2_info & MAY_BE_DOUBLE) { 7342 |.cold_code 7343 } 7344 |3: 7345 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7346 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 7347 } 7348 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7349 return 0; 7350 } 7351 if (op2_info & MAY_BE_DOUBLE) { 7352 | b >6 7353 |.code 7354 } 7355 } 7356 } else if ((op2_info & MAY_BE_DOUBLE) && 7357 !(op2_info & MAY_BE_LONG) && 7358 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7359 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7360 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7361 } 7362 if (op1_info & MAY_BE_DOUBLE) { 7363 if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7364 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7365 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3, ZREG_TMP1 7366 } else { 7367 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 7368 } 7369 } 7370 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7371 return 0; 7372 } 7373 } 7374 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7375 if (op1_info & MAY_BE_DOUBLE) { 7376 |.cold_code 7377 } 7378 |3: 7379 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7380 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 7381 } 7382 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7383 return 0; 7384 } 7385 if (op1_info & MAY_BE_DOUBLE) { 7386 | b >6 7387 |.code 7388 } 7389 } 7390 } 7391 7392 if (has_slow || 7393 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7394 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 7395 if (has_slow) { 7396 |.cold_code 7397 |9: 7398 } 7399 | SET_EX_OPLINE opline, REG0 7400 if (Z_MODE(op1_addr) == IS_REG) { 7401 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 7402 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 7403 return 0; 7404 } 7405 op1_addr = real_addr; 7406 } 7407 if (Z_MODE(op2_addr) == IS_REG) { 7408 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 7409 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 7410 return 0; 7411 } 7412 op2_addr = real_addr; 7413 } 7414 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7415 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { 7416 | IF_NOT_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w 7417 | LOAD_32BIT_VAL FCARG1x, opline->op1.var 7418 | EXT_CALL zend_jit_undefined_op_helper, REG0 7419 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 7420 |1: 7421 } 7422 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { 7423 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 7424 | str FCARG1x, T1 // save 7425 | LOAD_32BIT_VAL FCARG1x, opline->op2.var 7426 | EXT_CALL zend_jit_undefined_op_helper, REG0 7427 | ldr FCARG1x, T1 // restore 7428 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 7429 | b >2 7430 |1: 7431 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7432 |2: 7433 } else { 7434 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7435 } 7436 | EXT_CALL zend_compare, REG0 7437 if ((opline->opcode != ZEND_CASE && 7438 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7439 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 7440 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7441 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 7442 | str RETVALw, T1 // save 7443 if (opline->opcode != ZEND_CASE) { 7444 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 7445 } 7446 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 7447 | ldr RETVALw, T1 // restore 7448 } 7449 if (may_throw) { 7450 zend_jit_check_exception_undef_result(Dst, opline); 7451 } 7452 if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7453 return 0; 7454 } 7455 if (has_slow) { 7456 | b >6 7457 |.code 7458 } 7459 } 7460 7461 |6: 7462 7463 return 1; 7464} 7465 7466static int zend_jit_identical(dasm_State **Dst, 7467 const zend_op *opline, 7468 uint32_t op1_info, 7469 zend_ssa_range *op1_range, 7470 zend_jit_addr op1_addr, 7471 uint32_t op2_info, 7472 zend_ssa_range *op2_range, 7473 zend_jit_addr op2_addr, 7474 zend_jit_addr res_addr, 7475 int may_throw, 7476 zend_uchar smart_branch_opcode, 7477 uint32_t target_label, 7478 uint32_t target_label2, 7479 const void *exit_addr, 7480 bool skip_comparison) 7481{ 7482 uint32_t identical_label = (uint32_t)-1; 7483 uint32_t not_identical_label = (uint32_t)-1; 7484 7485 if (smart_branch_opcode && !exit_addr) { 7486 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7487 if (smart_branch_opcode == ZEND_JMPZ) { 7488 not_identical_label = target_label; 7489 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7490 identical_label = target_label; 7491 } else { 7492 ZEND_UNREACHABLE(); 7493 } 7494 } else { 7495 if (smart_branch_opcode == ZEND_JMPZ) { 7496 identical_label = target_label; 7497 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7498 not_identical_label = target_label; 7499 } else { 7500 ZEND_UNREACHABLE(); 7501 } 7502 } 7503 } 7504 7505 if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && 7506 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { 7507 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 7508 return 0; 7509 } 7510 return 1; 7511 } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && 7512 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { 7513 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7514 return 0; 7515 } 7516 return 1; 7517 } 7518 7519 if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { 7520 op1_info |= MAY_BE_NULL; 7521 op2_info |= MAY_BE_NULL; 7522 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7523 | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w 7524 |.cold_code 7525 |1: 7526 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7527 | SET_EX_OPLINE opline, REG0 7528 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 7529 | EXT_CALL zend_jit_undefined_op_helper, REG0 7530 if (may_throw) { 7531 zend_jit_check_exception_undef_result(Dst, opline); 7532 } 7533 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 7534 | b >1 7535 |.code 7536 |1: 7537 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7538 | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w 7539 |.cold_code 7540 |1: 7541 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7542 | SET_EX_OPLINE opline, REG0 7543 | str FCARG1x, T1 // save 7544 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 7545 | EXT_CALL zend_jit_undefined_op_helper, REG0 7546 if (may_throw) { 7547 zend_jit_check_exception_undef_result(Dst, opline); 7548 } 7549 | ldr FCARG1x, T1 // restore 7550 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 7551 | b >1 7552 |.code 7553 |1: 7554 } else if (op1_info & MAY_BE_UNDEF) { 7555 op1_info |= MAY_BE_NULL; 7556 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7557 | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w 7558 |.cold_code 7559 |1: 7560 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7561 | SET_EX_OPLINE opline, REG0 7562 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 7563 | EXT_CALL zend_jit_undefined_op_helper, REG0 7564 if (may_throw) { 7565 zend_jit_check_exception_undef_result(Dst, opline); 7566 } 7567 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 7568 | b >1 7569 |.code 7570 |1: 7571 if (opline->op2_type != IS_CONST) { 7572 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7573 } 7574 } else if (op2_info & MAY_BE_UNDEF) { 7575 op2_info |= MAY_BE_NULL; 7576 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7577 | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w 7578 |.cold_code 7579 |1: 7580 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7581 | SET_EX_OPLINE opline, REG0 7582 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 7583 | EXT_CALL zend_jit_undefined_op_helper, REG0 7584 if (may_throw) { 7585 zend_jit_check_exception_undef_result(Dst, opline); 7586 } 7587 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 7588 | b >1 7589 |.code 7590 |1: 7591 if (opline->op1_type != IS_CONST) { 7592 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7593 } 7594 } else if ((op1_info & op2_info & MAY_BE_ANY) != 0) { 7595 if (opline->op1_type != IS_CONST) { 7596 if (Z_MODE(op1_addr) == IS_REG) { 7597 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 7598 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 7599 return 0; 7600 } 7601 op1_addr = real_addr; 7602 } 7603 } 7604 if (opline->op2_type != IS_CONST) { 7605 if (Z_MODE(op2_addr) == IS_REG) { 7606 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 7607 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 7608 return 0; 7609 } 7610 op2_addr = real_addr; 7611 } 7612 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7613 } 7614 if (opline->op1_type != IS_CONST) { 7615 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7616 } 7617 } 7618 7619 if ((op1_info & op2_info & MAY_BE_ANY) == 0) { 7620 if ((opline->opcode != ZEND_CASE_STRICT && 7621 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7622 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 7623 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7624 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 7625 if (opline->opcode != ZEND_CASE_STRICT) { 7626 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7627 } 7628 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7629 } 7630 if (smart_branch_opcode) { 7631 if (may_throw) { 7632 zend_jit_check_exception_undef_result(Dst, opline); 7633 } 7634 if (exit_addr) { 7635 if (smart_branch_opcode == ZEND_JMPZ) { 7636 | b &exit_addr 7637 } 7638 } else if (not_identical_label != (uint32_t)-1) { 7639 | b =>not_identical_label 7640 } 7641 } else { 7642 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 7643 if (may_throw) { 7644 zend_jit_check_exception(Dst); 7645 } 7646 } 7647 return 1; 7648 } 7649 7650 if (opline->op1_type & (IS_CV|IS_VAR)) { 7651 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 7652 } 7653 if (opline->op2_type & (IS_CV|IS_VAR)) { 7654 | ZVAL_DEREF FCARG2x, op2_info, TMP1w 7655 } 7656 7657 if (has_concrete_type(op1_info) 7658 && has_concrete_type(op2_info) 7659 && concrete_type(op1_info) == concrete_type(op2_info) 7660 && concrete_type(op1_info) <= IS_TRUE) { 7661 if (smart_branch_opcode) { 7662 if (exit_addr) { 7663 if (smart_branch_opcode == ZEND_JMPNZ) { 7664 | b &exit_addr 7665 } 7666 } else if (identical_label != (uint32_t)-1) { 7667 | b =>identical_label 7668 } 7669 } else { 7670 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 7671 } 7672 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { 7673 if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { 7674 if (smart_branch_opcode) { 7675 if (exit_addr) { 7676 if (smart_branch_opcode == ZEND_JMPNZ) { 7677 | b &exit_addr 7678 } 7679 } else if (identical_label != (uint32_t)-1) { 7680 | b =>identical_label 7681 } 7682 } else { 7683 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 7684 } 7685 } else { 7686 if (smart_branch_opcode) { 7687 if (exit_addr) { 7688 if (smart_branch_opcode == ZEND_JMPZ) { 7689 | b &exit_addr 7690 } 7691 } else if (not_identical_label != (uint32_t)-1) { 7692 | b =>not_identical_label 7693 } 7694 } else { 7695 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 7696 } 7697 } 7698 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { 7699 zval *val = Z_ZV(op1_addr); 7700 7701 | ldrb TMP1w, [FCARG2x, #offsetof(zval, u1.v.type)] 7702 | cmp TMP1w, #Z_TYPE_P(val) 7703 if (smart_branch_opcode) { 7704 if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { 7705 | bne >8 7706 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7707 if (may_throw) { 7708 zend_jit_check_exception_undef_result(Dst, opline); 7709 } 7710 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7711 | b &exit_addr 7712 } else if (identical_label != (uint32_t)-1) { 7713 | b =>identical_label 7714 } else { 7715 | b >9 7716 } 7717 |8: 7718 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7719 | beq &exit_addr 7720 } else if (identical_label != (uint32_t)-1) { 7721 | beq =>identical_label 7722 } else { 7723 | beq >9 7724 } 7725 } else { 7726 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7727 | cset REG0w, eq 7728 } else { 7729 | cset REG0w, ne 7730 } 7731 | add REG0w, REG0w, #2 7732 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7733 } 7734 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7735 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 7736 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7737 if (may_throw) { 7738 zend_jit_check_exception_undef_result(Dst, opline); 7739 } 7740 } 7741 if (exit_addr) { 7742 if (smart_branch_opcode == ZEND_JMPZ) { 7743 | b &exit_addr 7744 } 7745 } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { 7746 | b =>not_identical_label 7747 } 7748 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { 7749 zval *val = Z_ZV(op2_addr); 7750 7751 | ldrb TMP1w, [FCARG1x, #offsetof(zval, u1.v.type)] 7752 | cmp TMP1w, #Z_TYPE_P(val) 7753 if (smart_branch_opcode) { 7754 if (opline->opcode != ZEND_CASE_STRICT 7755 && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { 7756 | bne >8 7757 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7758 if (may_throw) { 7759 zend_jit_check_exception_undef_result(Dst, opline); 7760 } 7761 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7762 | b &exit_addr 7763 } else if (identical_label != (uint32_t)-1) { 7764 | b =>identical_label 7765 } else { 7766 | b >9 7767 } 7768 |8: 7769 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7770 | beq &exit_addr 7771 } else if (identical_label != (uint32_t)-1) { 7772 | beq =>identical_label 7773 } else { 7774 | beq >9 7775 } 7776 } else { 7777 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7778 | cset REG0w, eq 7779 } else { 7780 | cset REG0w, ne 7781 } 7782 | add REG0w, REG0w, #2 7783 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7784 } 7785 if (opline->opcode != ZEND_CASE_STRICT 7786 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7787 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 7788 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7789 if (may_throw) { 7790 zend_jit_check_exception_undef_result(Dst, opline); 7791 } 7792 } 7793 if (smart_branch_opcode) { 7794 if (exit_addr) { 7795 if (smart_branch_opcode == ZEND_JMPZ) { 7796 | b &exit_addr 7797 } 7798 } else if (not_identical_label != (uint32_t)-1) { 7799 | b =>not_identical_label 7800 } 7801 } 7802 } else { 7803 if (opline->op1_type == IS_CONST) { 7804 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7805 } 7806 if (opline->op2_type == IS_CONST) { 7807 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7808 } 7809 | EXT_CALL zend_is_identical, REG0 7810 if ((opline->opcode != ZEND_CASE_STRICT && 7811 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7812 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 7813 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7814 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 7815 | str RETVALw, T1 // save 7816 if (opline->opcode != ZEND_CASE_STRICT) { 7817 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7818 } 7819 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7820 if (may_throw) { 7821 zend_jit_check_exception_undef_result(Dst, opline); 7822 } 7823 | ldr RETVALw, T1 // restore 7824 } 7825 if (smart_branch_opcode) { 7826 if (exit_addr) { 7827 if (smart_branch_opcode == ZEND_JMPNZ) { 7828 | cbnz RETVALw, &exit_addr 7829 } else { 7830 | cbz RETVALw, &exit_addr 7831 } 7832 } else if (not_identical_label != (uint32_t)-1) { 7833 | cbz RETVALw, =>not_identical_label 7834 if (identical_label != (uint32_t)-1) { 7835 | b =>identical_label 7836 } 7837 } else if (identical_label != (uint32_t)-1) { 7838 | cbnz RETVALw, =>identical_label 7839 } 7840 } else { 7841 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7842 | add RETVALw, RETVALw, #2 7843 } else { 7844 | neg RETVALw, RETVALw 7845 | add RETVALw, RETVALw, #3 7846 } 7847 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, RETVALw, TMP1 7848 } 7849 } 7850 7851 |9: 7852 if (may_throw) { 7853 zend_jit_check_exception(Dst); 7854 } 7855 return 1; 7856} 7857 7858static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr) 7859{ 7860 uint32_t true_label = -1; 7861 uint32_t false_label = -1; 7862 bool set_bool = 0; 7863 bool set_bool_not = 0; 7864 bool set_delayed = 0; 7865 bool jmp_done = 0; 7866 7867 if (branch_opcode == ZEND_BOOL) { 7868 set_bool = 1; 7869 } else if (branch_opcode == ZEND_BOOL_NOT) { 7870 set_bool = 1; 7871 set_bool_not = 1; 7872 } else if (branch_opcode == ZEND_JMPZ) { 7873 false_label = target_label; 7874 } else if (branch_opcode == ZEND_JMPNZ) { 7875 true_label = target_label; 7876 } else if (branch_opcode == ZEND_JMPZ_EX) { 7877 set_bool = 1; 7878 false_label = target_label; 7879 } else if (branch_opcode == ZEND_JMPNZ_EX) { 7880 set_bool = 1; 7881 true_label = target_label; 7882 } else { 7883 ZEND_UNREACHABLE(); 7884 } 7885 7886 if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { 7887 if (zend_is_true(Z_ZV(op1_addr))) { 7888 /* Always TRUE */ 7889 if (set_bool) { 7890 if (set_bool_not) { 7891 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7892 } else { 7893 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7894 } 7895 } 7896 if (true_label != (uint32_t)-1) { 7897 | b =>true_label 7898 } 7899 } else { 7900 /* Always FALSE */ 7901 if (set_bool) { 7902 if (set_bool_not) { 7903 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7904 } else { 7905 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7906 } 7907 } 7908 if (false_label != (uint32_t)-1) { 7909 | b =>false_label 7910 } 7911 } 7912 return 1; 7913 } 7914 7915 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) { 7916 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7917 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 7918 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 7919 } 7920 7921 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { 7922 if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { 7923 /* Always TRUE */ 7924 if (set_bool) { 7925 if (set_bool_not) { 7926 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7927 } else { 7928 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7929 } 7930 } 7931 if (true_label != (uint32_t)-1) { 7932 | b =>true_label 7933 } 7934 } else { 7935 if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { 7936 /* Always FALSE */ 7937 if (set_bool) { 7938 if (set_bool_not) { 7939 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7940 } else { 7941 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7942 } 7943 } 7944 } else { 7945 | CMP_ZVAL_TYPE op1_addr, IS_TRUE, ZREG_TMP1 7946 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 7947 if ((op1_info & MAY_BE_LONG) && 7948 !(op1_info & MAY_BE_UNDEF) && 7949 !set_bool) { 7950 if (exit_addr) { 7951 if (branch_opcode == ZEND_JMPNZ) { 7952 | blt >9 7953 } else { 7954 | blt &exit_addr 7955 } 7956 } else if (false_label != (uint32_t)-1) { 7957 | blt =>false_label 7958 } else { 7959 | blt >9 7960 } 7961 jmp_done = 1; 7962 } else { 7963 | bgt >2 7964 } 7965 } 7966 if (!(op1_info & MAY_BE_TRUE)) { 7967 /* It's FALSE */ 7968 if (set_bool) { 7969 if (set_bool_not) { 7970 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7971 } else { 7972 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7973 } 7974 } 7975 } else { 7976 if (exit_addr) { 7977 if (set_bool) { 7978 | bne >1 7979 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7980 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 7981 | b &exit_addr 7982 } else { 7983 | b >9 7984 } 7985 |1: 7986 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7987 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 7988 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 7989 | bne &exit_addr 7990 } 7991 } 7992 } else { 7993 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 7994 | beq &exit_addr 7995 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 7996 | bne &exit_addr 7997 } else { 7998 | beq >9 7999 } 8000 } 8001 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8002 if (set_bool) { 8003 | bne >1 8004 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8005 if (true_label != (uint32_t)-1) { 8006 | b =>true_label 8007 } else { 8008 | b >9 8009 } 8010 |1: 8011 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8012 } else { 8013 if (true_label != (uint32_t)-1) { 8014 | beq =>true_label 8015 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8016 | bne =>false_label 8017 jmp_done = 1; 8018 } else { 8019 | beq >9 8020 } 8021 } 8022 } else if (set_bool) { 8023 | cset REG0w, eq 8024 if (set_bool_not) { 8025 | neg REG0w, REG0w 8026 | add REG0w, REG0w, #3 8027 } else { 8028 | add REG0w, REG0w, #2 8029 } 8030 if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { 8031 set_delayed = 1; 8032 } else { 8033 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8034 } 8035 } 8036 } 8037 } 8038 8039 /* It's FALSE, but may be UNDEF */ 8040 if (op1_info & MAY_BE_UNDEF) { 8041 if (op1_info & MAY_BE_ANY) { 8042 if (set_delayed) { 8043 | CMP_ZVAL_TYPE op1_addr, IS_UNDEF, ZREG_TMP1 8044 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8045 | beq >1 8046 } else { 8047 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 8048 } 8049 |.cold_code 8050 |1: 8051 } 8052 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 8053 | SET_EX_OPLINE opline, REG0 8054 | EXT_CALL zend_jit_undefined_op_helper, REG0 8055 8056 if (may_throw) { 8057 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 8058 return 0; 8059 } 8060 } 8061 8062 if (exit_addr) { 8063 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8064 | b &exit_addr 8065 } 8066 } else if (false_label != (uint32_t)-1) { 8067 | b =>false_label 8068 } 8069 if (op1_info & MAY_BE_ANY) { 8070 if (exit_addr) { 8071 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8072 | b >9 8073 } 8074 } else if (false_label == (uint32_t)-1) { 8075 | b >9 8076 } 8077 |.code 8078 } 8079 } 8080 8081 if (!jmp_done) { 8082 if (exit_addr) { 8083 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8084 if (op1_info & MAY_BE_LONG) { 8085 | b >9 8086 } 8087 } else if (op1_info & MAY_BE_LONG) { 8088 | b &exit_addr 8089 } 8090 } else if (false_label != (uint32_t)-1) { 8091 | b =>false_label 8092 } else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 8093 | b >9 8094 } 8095 } 8096 } 8097 } 8098 8099 if (op1_info & MAY_BE_LONG) { 8100 |2: 8101 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8102 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 8103 } 8104 if (Z_MODE(op1_addr) == IS_REG) { 8105 | tst Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) 8106 } else { 8107 | LONG_CMP_WITH_CONST op1_addr, Z_L(0), TMP1, TMP2 8108 } 8109 if (set_bool) { 8110 | cset REG0w, ne 8111 if (set_bool_not) { 8112 | neg REG0w, REG0w 8113 | add REG0w, REG0w, #3 8114 } else { 8115 | add REG0w, REG0w, #2 8116 } 8117 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8118 } 8119 if (exit_addr) { 8120 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8121 | bne &exit_addr 8122 } else { 8123 | beq &exit_addr 8124 } 8125 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8126 if (true_label != (uint32_t)-1) { 8127 | bne =>true_label 8128 if (false_label != (uint32_t)-1) { 8129 | b =>false_label 8130 } 8131 } else { 8132 | beq =>false_label 8133 } 8134 } 8135 } 8136 8137 if ((op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) == MAY_BE_DOUBLE) { 8138 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8139 |.cold_code 8140 } 8141 |2: 8142 | fmov FPR0, xzr // TODO: "movi d0, #0" is not recognized by DynASM/arm64 8143 | DOUBLE_CMP ZREG_FPR0, op1_addr, ZREG_TMP1, ZREG_FPTMP 8144 8145 if (set_bool) { 8146 if (exit_addr) { 8147 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8148 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8149 | bvs &exit_addr 8150 | bne &exit_addr 8151 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8152 } else { 8153 | bvs >1 8154 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8155 | beq &exit_addr 8156 |1: 8157 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8158 } 8159 } else if (false_label != (uint32_t)-1) { // JMPZ_EX 8160 | bvs >1 8161 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8162 | beq => false_label 8163 |1: 8164 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8165 } else if (true_label != (uint32_t)-1) { // JMPNZ_EX 8166 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8167 | bvs => true_label 8168 | bne => true_label 8169 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8170 } else if (set_bool_not) { // BOOL_NOT 8171 | mov REG0w, #IS_FALSE 8172 | bvs >1 8173 | bne >1 8174 | mov REG0w, #IS_TRUE 8175 |1: 8176 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8177 } else { // BOOL 8178 | mov REG0w, #IS_TRUE 8179 | bvs >1 8180 | bne >1 8181 | mov REG0w, #IS_FALSE 8182 |1: 8183 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8184 } 8185 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8186 | b >9 8187 |.code 8188 } 8189 } else { 8190 if (exit_addr) { 8191 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8192 | bvs &exit_addr 8193 | bne &exit_addr 8194 |1: 8195 } else { 8196 | bvs >1 8197 | beq &exit_addr 8198 |1: 8199 } 8200 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8201 | b >9 8202 } 8203 } else { 8204 ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); 8205 if (false_label != (uint32_t)-1 ) { 8206 | bvs >1 8207 | beq => false_label 8208 |1: 8209 if (true_label != (uint32_t)-1) { 8210 | b =>true_label 8211 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8212 | b >9 8213 } 8214 } else { 8215 | bvs => true_label 8216 | bne => true_label 8217 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8218 | b >9 8219 } 8220 } 8221 } 8222 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8223 |.code 8224 } 8225 } 8226 } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8227 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8228 |.cold_code 8229 |2: 8230 } 8231 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8232 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8233 } 8234 | SET_EX_OPLINE opline, REG0 8235 | EXT_CALL zend_is_true, REG0 8236 | mov REG0, RETVALx 8237 8238 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8239 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8240 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8241 8242 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8243 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3, ZREG_TMP1, ZREG_TMP2 8244 } 8245 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 8246 | GC_DELREF FCARG1x, TMP1w 8247 | bne >3 8248 // In x86, r0 is used in macro ZVAL_DTOR_FUNC as temporary register, hence, r0 should be saved/restored 8249 // before/after this macro. In AArch64, TMP1 is used, but we still have to store REG0, 8250 // because it's clobbered by function call. 8251 | str REG0, T1 // save 8252 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 8253 | ldr REG0, T1 // restore 8254 |3: 8255 } 8256 if (may_throw) { 8257 | MEM_LOAD_64_ZTS ldr, REG1, executor_globals, exception, TMP1 8258 | cbnz REG1, ->exception_handler_undef 8259 } 8260 8261 if (set_bool) { 8262 if (set_bool_not) { 8263 | neg REG0w, REG0w 8264 | add REG0w, REG0w, #3 8265 } else { 8266 | add REG0w, REG0w, #2 8267 } 8268 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8269 if (exit_addr) { 8270 | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 8271 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8272 | bne &exit_addr 8273 } else { 8274 | beq &exit_addr 8275 } 8276 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8277 | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 8278 if (true_label != (uint32_t)-1) { 8279 | bne =>true_label 8280 if (false_label != (uint32_t)-1) { 8281 | b =>false_label 8282 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8283 | b >9 8284 } 8285 } else { 8286 | beq =>false_label 8287 } 8288 } 8289 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8290 | b >9 8291 |.code 8292 } 8293 } else { 8294 if (exit_addr) { 8295 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8296 | cbnz REG0w, &exit_addr 8297 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8298 | b >9 8299 } 8300 } else { 8301 | cbz REG0w, &exit_addr 8302 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8303 | b >9 8304 } 8305 } 8306 } else if (true_label != (uint32_t)-1) { 8307 | cbnz REG0w, =>true_label 8308 if (false_label != (uint32_t)-1) { 8309 | b =>false_label 8310 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8311 | b >9 8312 } 8313 } else { 8314 | cbz REG0w, =>false_label 8315 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8316 | b >9 8317 } 8318 } 8319 8320 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8321 |.code 8322 } 8323 } 8324 } 8325 8326 |9: 8327 8328 return 1; 8329} 8330 8331static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) 8332{ 8333 if (op1_addr != op1_def_addr) { 8334 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 8335 return 0; 8336 } 8337 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 8338 op1_addr = op1_def_addr; 8339 } 8340 } 8341 8342 if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0, 1)) { 8343 return 0; 8344 } 8345 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8346 return 0; 8347 } 8348 if (op1_info & MAY_BE_UNDEF) { 8349 zend_jit_check_exception(Dst); 8350 } 8351 return 1; 8352} 8353 8354static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 8355{ 8356 ZEND_ASSERT(opline->op1_type == IS_CV); 8357 8358 if (op2_addr != op2_def_addr) { 8359 if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { 8360 return 0; 8361 } 8362 if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { 8363 op2_addr = op2_def_addr; 8364 } 8365 } 8366 8367 if (Z_MODE(op1_addr) != IS_REG 8368 && Z_MODE(op1_use_addr) == IS_REG 8369 && !Z_LOAD(op1_use_addr) 8370 && !Z_STORE(op1_use_addr)) { 8371 /* Force type update */ 8372 op1_info |= MAY_BE_UNDEF; 8373 } 8374 if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, 8375 may_throw)) { 8376 return 0; 8377 } 8378 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { 8379 return 0; 8380 } 8381 if (opline->result_type != IS_UNUSED) { 8382 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8383 return 0; 8384 } 8385 } 8386 8387 return 1; 8388} 8389 8390/* copy of hidden zend_closure */ 8391typedef struct _zend_closure { 8392 zend_object std; 8393 zend_function func; 8394 zval this_ptr; 8395 zend_class_entry *called_scope; 8396 zif_handler orig_internal_handler; 8397} zend_closure; 8398 8399static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) 8400{ 8401 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8402 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8403 8404 if (!exit_addr) { 8405 return 0; 8406 } 8407 8408 | // Check Stack Overflow 8409 | MEM_LOAD_64_ZTS ldr, REG1, executor_globals, vm_stack_end, TMP1 8410 | MEM_LOAD_OP_ZTS sub, ldr, REG1, executor_globals, vm_stack_top, TMP1, TMP2 8411 | CMP_64_WITH_CONST_32 REG1, used_stack, TMP1 8412 | blo &exit_addr 8413 8414 return 1; 8415} 8416 8417static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack) 8418{ 8419 uint32_t used_stack; 8420 bool stack_check = 1; 8421 8422 // REG0 -> zend_function 8423 // FCARG1 -> used_stack 8424 8425 if (func) { 8426 used_stack = zend_vm_calc_used_stack(opline->extended_value, func); 8427 if ((int)used_stack <= checked_stack) { 8428 stack_check = 0; 8429 } 8430 } else { 8431 used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value + ZEND_OBSERVER_ENABLED) * sizeof(zval); 8432 8433 | // if (EXPECTED(ZEND_USER_CODE(func->type))) { 8434 if (!is_closure) { 8435 | LOAD_32BIT_VAL FCARG1w, used_stack 8436 | // Check whether REG0 is an internal function. 8437 | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] 8438 | TST_32_WITH_CONST TMP1w, 1, TMP2w 8439 | bne >1 8440 } else { 8441 | LOAD_32BIT_VAL FCARG1w, used_stack 8442 } 8443 | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); 8444 | LOAD_32BIT_VAL REG2w, opline->extended_value 8445 if (!is_closure) { 8446 | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.num_args)] 8447 | cmp REG2w, TMP1w 8448 | csel REG2w, REG2w, TMP1w, le 8449 | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.last_var)] 8450 | sub REG2w, REG2w, TMP1w 8451 | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.T)] 8452 | sub REG2w, REG2w, TMP1w 8453 } else { 8454 | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.num_args)] 8455 | cmp REG2w, TMP1w 8456 | csel REG2w, REG2w, TMP1w, le 8457 | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.last_var)] 8458 | sub REG2w, REG2w, TMP1w 8459 | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.T)] 8460 | sub REG2w, REG2w, TMP1w 8461 } 8462 | sxtw REG2, REG2w 8463 | sub FCARG1x, FCARG1x, REG2, lsl #4 8464 |1: 8465 } 8466 8467 zend_jit_start_reuse_ip(); 8468 8469 | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { 8470 | MEM_LOAD_64_ZTS ldr, RX, executor_globals, vm_stack_top, TMP1 8471 8472 if (stack_check) { 8473 | // Check Stack Overflow 8474 | MEM_LOAD_64_ZTS ldr, REG2, executor_globals, vm_stack_end, TMP1 8475 | sub REG2, REG2, RX 8476 if (func) { 8477 | CMP_64_WITH_CONST_32 REG2, used_stack, TMP1 8478 } else { 8479 | cmp REG2, FCARG1x 8480 } 8481 8482 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8483 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8484 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8485 8486 if (!exit_addr) { 8487 return 0; 8488 } 8489 8490 | blo &exit_addr 8491 } else { 8492 | blo >1 8493 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); 8494 |.cold_code 8495 |1: 8496 if (func) { 8497 | LOAD_32BIT_VAL FCARG1w, used_stack 8498 } 8499 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 8500 | SET_EX_OPLINE opline, REG0 8501 | EXT_CALL zend_jit_int_extend_stack_helper, REG0 8502 } else { 8503 if (!is_closure) { 8504 | mov FCARG2x, REG0 8505 } else { 8506 | add FCARG2x, REG0, #offsetof(zend_closure, func) 8507 } 8508 | SET_EX_OPLINE opline, REG0 8509 | EXT_CALL zend_jit_extend_stack_helper, REG0 8510 } 8511 | mov RX, RETVALx 8512 | b >1 8513 |.code 8514 } 8515 } 8516 8517 if (func) { 8518 || if (arm64_may_encode_imm12((int64_t)used_stack)) { 8519 | MEM_UPDATE_ZTS add, ldr, str, #used_stack, executor_globals, vm_stack_top, REG2, TMP1 8520 || } else { 8521 | LOAD_32BIT_VAL TMP1w, used_stack 8522 | MEM_UPDATE_ZTS add, ldr, str, TMP1, executor_globals, vm_stack_top, REG2, TMP2 8523 || } 8524 } else { 8525 | MEM_UPDATE_ZTS add, ldr, str, FCARG1x, executor_globals, vm_stack_top, REG2, TMP1 8526 } 8527 | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); 8528 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { 8529 | // ZEND_SET_CALL_INFO(call, 0, call_info); 8530 | LOAD_32BIT_VAL TMP1w, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) 8531 | str TMP1w, EX:RX->This.u1.type_info 8532 } 8533 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 8534 | // call->func = func; 8535 |1: 8536 | ADDR_STORE EX:RX->func, func, REG1 8537 } else { 8538 if (!is_closure) { 8539 | // call->func = func; 8540 | str REG0, EX:RX->func 8541 } else { 8542 | // call->func = &closure->func; 8543 | add REG1, REG0, #offsetof(zend_closure, func) 8544 | str REG1, EX:RX->func 8545 } 8546 |1: 8547 } 8548 if (opline->opcode == ZEND_INIT_METHOD_CALL) { 8549 | // Z_PTR(call->This) = obj; 8550 | ldr REG1, T1 8551 | str REG1, EX:RX->This.value.ptr 8552 if (opline->op1_type == IS_UNUSED || delayed_fetch_this) { 8553 | // call->call_info |= ZEND_CALL_HAS_THIS; 8554 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8555 | LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS 8556 | str TMP1w, EX:RX->This.u1.type_info 8557 } else { 8558 | ldr TMP1w, EX:RX->This.u1.type_info 8559 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_HAS_THIS, TMP2w 8560 | str TMP1w, EX:RX->This.u1.type_info 8561 } 8562 } else { 8563 if (opline->op1_type == IS_CV) { 8564 | // GC_ADDREF(obj); 8565 | GC_ADDREF REG1, TMP1w 8566 } 8567 | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; 8568 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8569 | LOAD_32BIT_VAL TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 8570 | str TMP1w, EX:RX->This.u1.type_info 8571 } else { 8572 | ldr TMP1w, EX:RX->This.u1.type_info 8573 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS), TMP2w 8574 | str TMP1w, EX:RX->This.u1.type_info 8575 } 8576 } 8577 } else if (!is_closure) { 8578 | // Z_CE(call->This) = called_scope; 8579 | str xzr, EX:RX->This.value.ptr 8580 } else { 8581 if (opline->op2_type == IS_CV) { 8582 | // GC_ADDREF(closure); 8583 | GC_ADDREF REG0, TMP1w 8584 } 8585 | // object_or_called_scope = closure->called_scope; 8586 | ldr REG1, [REG0, #offsetof(zend_closure, called_scope)] 8587 | str REG1, EX:RX->This.value.ptr 8588 | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | 8589 | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); 8590 | ldr REG2w, [REG0, #offsetof(zend_closure, func.common.fn_flags)] 8591 | BW_OP_32_WITH_CONST and, REG2w, REG2w, ZEND_ACC_FAKE_CLOSURE, TMP1w 8592 | BW_OP_32_WITH_CONST orr, REG2w, REG2w, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE), TMP1w 8593 | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { 8594 | ldrb TMP1w, [REG0, #offsetof(zend_closure, this_ptr.u1.v.type)] 8595 | cmp TMP1w, #IS_UNDEF 8596 | beq >1 8597 | // call_info |= ZEND_CALL_HAS_THIS; 8598 | BW_OP_32_WITH_CONST orr, REG2w, REG2w, ZEND_CALL_HAS_THIS, TMP1w 8599 | // object_or_called_scope = Z_OBJ(closure->this_ptr); 8600 | ldr REG1, [REG0, #offsetof(zend_closure, this_ptr.value.ptr)] 8601 |1: 8602 | // ZEND_SET_CALL_INFO(call, 0, call_info); 8603 | ldr TMP1w, EX:RX->This.u1.type_info 8604 | orr TMP1w, TMP1w, REG2w 8605 | str TMP1w, EX:RX->This.u1.type_info 8606 | // Z_PTR(call->This) = object_or_called_scope; 8607 | str REG1, EX:RX->This.value.ptr 8608 | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.run_time_cache__ptr)] 8609 | cbnz TMP1, >1 8610 | add FCARG1x, REG0, #offsetof(zend_closure, func) 8611 | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 8612 |1: 8613 } 8614 | // ZEND_CALL_NUM_ARGS(call) = num_args; 8615 | LOAD_32BIT_VAL TMP1w, opline->extended_value 8616 | str TMP1w, EX:RX->This.u2.num_args 8617 8618 return 1; 8619} 8620 8621static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) 8622{ 8623 int32_t exit_point; 8624 const void *exit_addr; 8625 8626 if (func->type == ZEND_INTERNAL_FUNCTION) { 8627#ifdef ZEND_WIN32 8628 // TODO: ASLR may cause different addresses in different workers ??? 8629 return 0; 8630#endif 8631 } else if (func->type == ZEND_USER_FUNCTION) { 8632 if (!zend_accel_in_shm(func->op_array.opcodes)) { 8633 /* op_array and op_array->opcodes are not persistent. We can't link. */ 8634 return 0; 8635 } 8636 } else { 8637 ZEND_UNREACHABLE(); 8638 return 0; 8639 } 8640 8641 exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); 8642 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8643 if (!exit_addr) { 8644 return 0; 8645 } 8646 8647 | // call = EX(call); 8648 | ldr REG1, EX->call 8649 while (level > 0) { 8650 | ldr REG1, EX:REG1->prev_execute_data 8651 level--; 8652 } 8653 8654 if (func->type == ZEND_USER_FUNCTION && 8655 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 8656 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 8657 !func->common.function_name)) { 8658 const zend_op *opcodes = func->op_array.opcodes; 8659 8660 | ldr REG1, EX:REG1->func 8661 | LOAD_ADDR REG2, ((ptrdiff_t)opcodes) 8662 | ldr TMP1, [REG1, #offsetof(zend_op_array, opcodes)] 8663 | cmp TMP1, REG2 8664 | bne &exit_addr 8665 } else { 8666 | LOAD_ADDR REG2, ((ptrdiff_t)func) 8667 | ldr TMP1, EX:REG1->func 8668 | cmp TMP1, REG2 8669 | bne &exit_addr 8670 } 8671 8672 return 1; 8673} 8674 8675static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack) 8676{ 8677 zend_func_info *info = ZEND_FUNC_INFO(op_array); 8678 zend_call_info *call_info = NULL; 8679 zend_function *func = NULL; 8680 8681 if (delayed_call_chain) { 8682 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 8683 return 0; 8684 } 8685 } 8686 8687 if (info) { 8688 call_info = info->callee_info; 8689 while (call_info && call_info->caller_init_opline != opline) { 8690 call_info = call_info->next_callee; 8691 } 8692 if (call_info && call_info->callee_func && !call_info->is_prototype) { 8693 func = call_info->callee_func; 8694 } 8695 } 8696 8697 if (!func 8698 && trace 8699 && trace->op == ZEND_JIT_TRACE_INIT_CALL) { 8700 func = (zend_function*)trace->func; 8701 } 8702 8703 if (opline->opcode == ZEND_INIT_FCALL 8704 && func 8705 && func->type == ZEND_INTERNAL_FUNCTION) { 8706 /* load constant address later */ 8707 } else if (func && op_array == &func->op_array) { 8708 /* recursive call */ 8709 | ldr REG0, EX->func 8710 } else { 8711 | // if (CACHED_PTR(opline->result.num)) 8712 | ldr REG2, EX->run_time_cache 8713 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG2, opline->result.num, TMP1 8714 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 8715 && func 8716 && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) 8717 && opline->opcode != ZEND_INIT_FCALL) { 8718 /* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */ 8719 | LOAD_ADDR REG1, ((ptrdiff_t)func) 8720 | cmp REG0, REG1 8721 | bne >1 8722 } else { 8723 | cbz REG0, >1 8724 } 8725 |.cold_code 8726 |1: 8727 if (opline->opcode == ZEND_INIT_FCALL 8728 && func 8729 && func->type == ZEND_USER_FUNCTION 8730 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { 8731 | LOAD_ADDR FCARG1x, func 8732 | MEM_ACCESS_64_WITH_UOFFSET str, FCARG1x, REG2, opline->result.num, TMP1 8733 | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 8734 | mov REG0, RETVALx 8735 | b >3 8736 } else { 8737 zval *zv = RT_CONSTANT(opline, opline->op2); 8738 8739 if (opline->opcode == ZEND_INIT_FCALL) { 8740 | LOAD_ADDR FCARG1x, Z_STR_P(zv); 8741 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 8742 | EXT_CALL zend_jit_find_func_helper, REG0 8743 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { 8744 | LOAD_ADDR FCARG1x, Z_STR_P(zv + 1); 8745 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 8746 | EXT_CALL zend_jit_find_func_helper, REG0 8747 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { 8748 | LOAD_ADDR FCARG1x, zv; 8749 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 8750 | EXT_CALL zend_jit_find_ns_func_helper, REG0 8751 } else { 8752 ZEND_UNREACHABLE(); 8753 } 8754 | // Get the return value of function zend_jit_find_func_helper/zend_jit_find_ns_func_helper 8755 | mov REG0, RETVALx 8756 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8757 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 8758 func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0); 8759 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8760 8761 if (!exit_addr) { 8762 return 0; 8763 } 8764 8765 if (!func || opline->opcode == ZEND_INIT_FCALL) { 8766 | cbnz REG0, >3 8767 } else if (func->type == ZEND_USER_FUNCTION 8768 && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { 8769 const zend_op *opcodes = func->op_array.opcodes; 8770 8771 | LOAD_ADDR REG1, ((ptrdiff_t)opcodes) 8772 | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] 8773 | cmp TMP1, REG1 8774 | beq >3 8775 } else { 8776 | LOAD_ADDR REG1, ((ptrdiff_t)func) 8777 | cmp REG0, REG1 8778 | beq >3 8779 } 8780 | b &exit_addr 8781 } else { 8782 | cbnz REG0, >3 8783 | // SAVE_OPLINE(); 8784 | SET_EX_OPLINE opline, REG0 8785 | b ->undefined_function 8786 } 8787 } 8788 |.code 8789 |3: 8790 } 8791 8792 if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, checked_stack)) { 8793 return 0; 8794 } 8795 8796 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 8797 if (!zend_jit_save_call_chain(Dst, call_level)) { 8798 return 0; 8799 } 8800 } else { 8801 delayed_call_chain = 1; 8802 delayed_call_level = call_level; 8803 } 8804 8805 return 1; 8806} 8807 8808static int zend_jit_init_method_call(dasm_State **Dst, 8809 const zend_op *opline, 8810 uint32_t b, 8811 const zend_op_array *op_array, 8812 zend_ssa *ssa, 8813 const zend_ssa_op *ssa_op, 8814 int call_level, 8815 uint32_t op1_info, 8816 zend_jit_addr op1_addr, 8817 zend_class_entry *ce, 8818 bool ce_is_instanceof, 8819 bool on_this, 8820 bool delayed_fetch_this, 8821 zend_class_entry *trace_ce, 8822 zend_jit_trace_rec *trace, 8823 int checked_stack, 8824 bool polymorphic_side_trace) 8825{ 8826 zend_func_info *info = ZEND_FUNC_INFO(op_array); 8827 zend_call_info *call_info = NULL; 8828 zend_function *func = NULL; 8829 zval *function_name; 8830 8831 ZEND_ASSERT(opline->op2_type == IS_CONST); 8832 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 8833 8834 function_name = RT_CONSTANT(opline, opline->op2); 8835 8836 if (info) { 8837 call_info = info->callee_info; 8838 while (call_info && call_info->caller_init_opline != opline) { 8839 call_info = call_info->next_callee; 8840 } 8841 if (call_info && call_info->callee_func && !call_info->is_prototype) { 8842 func = call_info->callee_func; 8843 } 8844 } 8845 8846 if (polymorphic_side_trace) { 8847 /* function is passed in r0 from parent_trace */ 8848 } else { 8849 if (on_this) { 8850 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 8851 8852 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 8853 } else { 8854 if (op1_info & MAY_BE_REF) { 8855 if (opline->op1_type == IS_CV) { 8856 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8857 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8858 } 8859 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 8860 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 8861 } else { 8862 /* Hack: Convert reference to regular value to simplify JIT code */ 8863 ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); 8864 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 8865 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8866 | EXT_CALL zend_jit_unref_helper, REG0 8867 |1: 8868 } 8869 } 8870 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 8871 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8872 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8873 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8874 8875 if (!exit_addr) { 8876 return 0; 8877 } 8878 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 8879 } else { 8880 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 8881 |.cold_code 8882 |1: 8883 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8884 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8885 } 8886 | SET_EX_OPLINE opline, REG0 8887 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 8888 | EXT_CALL zend_jit_invalid_method_call_tmp, REG0 8889 } else { 8890 | EXT_CALL zend_jit_invalid_method_call, REG0 8891 } 8892 | b ->exception_handler 8893 |.code 8894 } 8895 } 8896 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 8897 } 8898 8899 if (delayed_call_chain) { 8900 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 8901 return 0; 8902 } 8903 } 8904 8905 | str FCARG1x, T1 // save 8906 8907 if (func) { 8908 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 8909 | ldr REG0, EX->run_time_cache 8910 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 8911 | cbz REG0, >1 8912 } else { 8913 | // if (CACHED_PTR(opline->result.num) == obj->ce)) { 8914 | ldr REG0, EX->run_time_cache 8915 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->result.num, TMP1 8916 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 8917 | cmp REG2, TMP1 8918 | bne >1 8919 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 8920 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 8921 } 8922 8923 |.cold_code 8924 |1: 8925 | LOAD_ADDR FCARG2x, function_name 8926 if (TMP_ZVAL_OFFSET == 0) { 8927 | mov CARG3, sp 8928 } else { 8929 | add CARG3, sp, #TMP_ZVAL_OFFSET 8930 } 8931 | SET_EX_OPLINE opline, REG0 8932 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 8933 | EXT_CALL zend_jit_find_method_tmp_helper, REG0 8934 } else { 8935 | EXT_CALL zend_jit_find_method_helper, REG0 8936 } 8937 | mov REG0, RETVALx 8938 | cbnz REG0, >2 8939 | b ->exception_handler 8940 |.code 8941 |2: 8942 } 8943 8944 if ((!func || zend_jit_may_be_modified(func, op_array)) 8945 && trace 8946 && trace->op == ZEND_JIT_TRACE_INIT_CALL 8947 && trace->func 8948 ) { 8949 int32_t exit_point; 8950 const void *exit_addr; 8951 8952 exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); 8953 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8954 if (!exit_addr) { 8955 return 0; 8956 } 8957 8958 func = (zend_function*)trace->func; 8959 8960 if (func->type == ZEND_USER_FUNCTION && 8961 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 8962 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 8963 !func->common.function_name)) { 8964 const zend_op *opcodes = func->op_array.opcodes; 8965 8966 | LOAD_ADDR TMP1, opcodes 8967 | ldr TMP2, [REG0, #offsetof(zend_op_array, opcodes)] 8968 | cmp TMP2, TMP1 8969 | bne &exit_addr 8970 } else { 8971 | LOAD_ADDR TMP1, func 8972 | cmp REG0, TMP1 8973 | bne &exit_addr 8974 } 8975 } 8976 8977 if (!func) { 8978 | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { 8979 | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] 8980 | TST_32_WITH_CONST TMP1w, ZEND_ACC_STATIC, TMP2w 8981 | bne >1 8982 |.cold_code 8983 |1: 8984 } 8985 8986 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { 8987 | ldr FCARG1x, T1 // restore 8988 | mov FCARG2x, REG0 8989 | LOAD_32BIT_VAL CARG3w, opline->extended_value 8990 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 8991 | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, REG0 8992 } else { 8993 | EXT_CALL zend_jit_push_static_metod_call_frame, REG0 8994 } 8995 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) { 8996 | cbz RETVALx, ->exception_handler 8997 } 8998 | mov RX, RETVALx 8999 } 9000 9001 if (!func) { 9002 | b >9 9003 |.code 9004 } 9005 9006 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { 9007 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, delayed_fetch_this, checked_stack)) { 9008 return 0; 9009 } 9010 } 9011 9012 if (!func) { 9013 |9: 9014 } 9015 zend_jit_start_reuse_ip(); 9016 9017 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9018 if (!zend_jit_save_call_chain(Dst, call_level)) { 9019 return 0; 9020 } 9021 } else { 9022 delayed_call_chain = 1; 9023 delayed_call_level = call_level; 9024 } 9025 9026 return 1; 9027} 9028 9029static int zend_jit_init_closure_call(dasm_State **Dst, 9030 const zend_op *opline, 9031 uint32_t b, 9032 const zend_op_array *op_array, 9033 zend_ssa *ssa, 9034 const zend_ssa_op *ssa_op, 9035 int call_level, 9036 zend_jit_trace_rec *trace, 9037 int checked_stack) 9038{ 9039 zend_function *func = NULL; 9040 zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 9041 9042 | GET_ZVAL_PTR REG0, op2_addr, TMP1 9043 9044 if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure 9045 && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { 9046 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9047 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9048 9049 if (!exit_addr) { 9050 return 0; 9051 } 9052 9053 | LOAD_ADDR FCARG1x, ((ptrdiff_t)zend_ce_closure) 9054 | ldr, TMP1, [REG0, #offsetof(zend_object, ce)] 9055 | cmp TMP1, FCARG1x 9056 | bne &exit_addr 9057 if (ssa->var_info && ssa_op->op2_use >= 0) { 9058 ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; 9059 ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; 9060 ssa->var_info[ssa_op->op2_use].is_instanceof = 0; 9061 } 9062 } 9063 9064 if (trace 9065 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9066 && trace->func 9067 && trace->func->type == ZEND_USER_FUNCTION) { 9068 const zend_op *opcodes; 9069 int32_t exit_point; 9070 const void *exit_addr; 9071 9072 func = (zend_function*)trace->func; 9073 opcodes = func->op_array.opcodes; 9074 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); 9075 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9076 if (!exit_addr) { 9077 return 0; 9078 } 9079 9080 | LOAD_ADDR FCARG1x, ((ptrdiff_t)opcodes) 9081 | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.opcodes)] 9082 | cmp TMP1, FCARG1x 9083 | bne &exit_addr 9084 } 9085 9086 if (delayed_call_chain) { 9087 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9088 return 0; 9089 } 9090 } 9091 9092 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, checked_stack)) { 9093 return 0; 9094 } 9095 9096 if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9097 if (!zend_jit_save_call_chain(Dst, call_level)) { 9098 return 0; 9099 } 9100 } else { 9101 delayed_call_chain = 1; 9102 delayed_call_level = call_level; 9103 } 9104 9105 if (trace 9106 && trace->op == ZEND_JIT_TRACE_END 9107 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 9108 if (!zend_jit_set_valid_ip(Dst, opline + 1)) { 9109 return 0; 9110 } 9111 } 9112 9113 return 1; 9114} 9115 9116static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) 9117{ 9118 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9119 zend_call_info *call_info = NULL; 9120 const zend_function *func = NULL; 9121 uint32_t i; 9122 zend_jit_addr res_addr; 9123 uint32_t call_num_args = 0; 9124 bool unknown_num_args = 0; 9125 const void *exit_addr = NULL; 9126 const zend_op *prev_opline; 9127 9128 if (RETURN_VALUE_USED(opline)) { 9129 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 9130 } else { 9131 /* CPU stack allocated temporary zval */ 9132 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RSP, TMP_ZVAL_OFFSET); 9133 } 9134 9135 prev_opline = opline - 1; 9136 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { 9137 prev_opline--; 9138 } 9139 if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || 9140 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 9141 unknown_num_args = 1; 9142 } 9143 9144 if (info) { 9145 call_info = info->callee_info; 9146 while (call_info && call_info->caller_call_opline != opline) { 9147 call_info = call_info->next_callee; 9148 } 9149 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9150 func = call_info->callee_func; 9151 } 9152 if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) 9153 && JIT_G(current_frame) 9154 && JIT_G(current_frame)->call 9155 && !JIT_G(current_frame)->call->func) { 9156 call_info = NULL; func = NULL; /* megamorphic call from trait */ 9157 } 9158 } 9159 if (!func) { 9160 /* resolve function at run time */ 9161 } else if (func->type == ZEND_USER_FUNCTION) { 9162 ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); 9163 call_num_args = call_info->num_args; 9164 } else if (func->type == ZEND_INTERNAL_FUNCTION) { 9165 ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); 9166 call_num_args = call_info->num_args; 9167 } else { 9168 ZEND_UNREACHABLE(); 9169 } 9170 9171 if (trace && !func) { 9172 if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { 9173 ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); 9174#ifndef ZEND_WIN32 9175 // TODO: ASLR may cause different addresses in different workers ??? 9176 func = trace->func; 9177 if (JIT_G(current_frame) && 9178 JIT_G(current_frame)->call && 9179 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9180 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9181 } else { 9182 unknown_num_args = 1; 9183 } 9184#endif 9185 } else if (trace->op == ZEND_JIT_TRACE_ENTER) { 9186 ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); 9187 if (zend_accel_in_shm(trace->func->op_array.opcodes)) { 9188 func = trace->func; 9189 if (JIT_G(current_frame) && 9190 JIT_G(current_frame)->call && 9191 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9192 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9193 } else { 9194 unknown_num_args = 1; 9195 } 9196 } 9197 } 9198 } 9199 9200 bool may_have_extra_named_params = 9201 opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && 9202 (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); 9203 9204 if (!reuse_ip) { 9205 zend_jit_start_reuse_ip(); 9206 | // call = EX(call); 9207 | ldr RX, EX->call 9208 } 9209 zend_jit_stop_reuse_ip(); 9210 9211 | // fbc = call->func; 9212 | // mov r2, EX:RX->func ??? 9213 | // SAVE_OPLINE(); 9214 | SET_EX_OPLINE opline, REG0 9215 9216 if (opline->opcode == ZEND_DO_FCALL) { 9217 if (!func) { 9218 if (trace) { 9219 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9220 9221 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9222 if (!exit_addr) { 9223 return 0; 9224 } 9225 | ldr REG0, EX:RX->func 9226 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9227 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9228 | bne &exit_addr 9229 } 9230 } 9231 } 9232 9233 if (!delayed_call_chain) { 9234 if (call_level == 1) { 9235 | str xzr, EX->call 9236 } else { 9237 | //EX(call) = call->prev_execute_data; 9238 | ldr REG0, EX:RX->prev_execute_data 9239 | str REG0, EX->call 9240 } 9241 } 9242 delayed_call_chain = 0; 9243 9244 | //call->prev_execute_data = execute_data; 9245 | str EX, EX:RX->prev_execute_data 9246 9247 if (!func) { 9248 | ldr REG0, EX:RX->func 9249 } 9250 9251 if (opline->opcode == ZEND_DO_FCALL) { 9252 if (!func) { 9253 if (!trace) { 9254 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9255 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9256 | bne >1 9257 |.cold_code 9258 |1: 9259 if (!GCC_GLOBAL_REGS) { 9260 | mov FCARG1x, RX 9261 } 9262 | EXT_CALL zend_jit_deprecated_helper, REG0 9263 | GET_LOW_8BITS RETVALw, RETVALw 9264 | ldr REG0, EX:RX->func // reload 9265 | cbnz RETVALw, >1 // Result is 0 on exception 9266 | b ->exception_handler 9267 |.code 9268 |1: 9269 } 9270 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 9271 if (!GCC_GLOBAL_REGS) { 9272 | mov FCARG1x, RX 9273 } 9274 | EXT_CALL zend_jit_deprecated_helper, REG0 9275 | cbz RETVALw, ->exception_handler 9276 } 9277 } 9278 9279 if (!func 9280 && opline->opcode != ZEND_DO_UCALL 9281 && opline->opcode != ZEND_DO_ICALL) { 9282 | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] 9283 | cmp TMP1w, #ZEND_USER_FUNCTION 9284 | bne >8 9285 } 9286 9287 if ((!func || func->type == ZEND_USER_FUNCTION) 9288 && opline->opcode != ZEND_DO_ICALL) { 9289 | // EX(call) = NULL; 9290 | str xzr, EX:RX->call 9291 9292 if (RETURN_VALUE_USED(opline)) { 9293 | // EX(return_value) = EX_VAR(opline->result.var); 9294 | LOAD_ZVAL_ADDR REG2, res_addr 9295 | str REG2, EX:RX->return_value 9296 } else { 9297 | // EX(return_value) = 0; 9298 | str xzr, EX:RX->return_value 9299 } 9300 9301 //EX_LOAD_RUN_TIME_CACHE(op_array); 9302 if (!func || func->op_array.cache_size) { 9303 if (func && op_array == &func->op_array) { 9304 /* recursive call */ 9305 if (trace || func->op_array.cache_size > sizeof(void*)) { 9306 | ldr REG2, EX->run_time_cache 9307 | str REG2, EX:RX->run_time_cache 9308 } 9309 } else { 9310 if (func 9311 && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE) 9312 && ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { 9313 | MEM_LOAD_64_ZTS ldr, REG2, compiler_globals, map_ptr_base, TMP1 9314 | ADD_SUB_64_WITH_CONST add, REG2, REG2, (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache), TMP1 9315 | ldr REG2, [REG2] 9316 } else if ((func && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) || 9317 (JIT_G(current_frame) && 9318 JIT_G(current_frame)->call && 9319 TRACE_FRAME_IS_CLOSURE_CALL(JIT_G(current_frame)->call))) { 9320 /* Closures always use direct pointers */ 9321 | ldr REG0, EX:RX->func 9322 | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] 9323 } else { 9324 if (func) { 9325 | ldr REG0, EX:RX->func 9326 } 9327 | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] 9328 | TST_64_WITH_ONE REG2 9329 | beq >1 9330 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 9331 | ldr REG2, [REG2] 9332 |1: 9333 } 9334 | str REG2, EX:RX->run_time_cache 9335 } 9336 } 9337 9338 | // EG(current_execute_data) = execute_data; 9339 | MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1 9340 | mov FP, RX 9341 9342 | // opline = op_array->opcodes; 9343 if (func && !unknown_num_args) { 9344 | ADD_SUB_64_WITH_CONST_32 add, TMP1, RX, (EX_NUM_TO_VAR(call_num_args) + offsetof(zval, u1.type_info)), TMP1 // induction variable 9345 for (i = call_num_args; i < func->op_array.last_var; i++) { 9346 | // ZVAL_UNDEF(EX_VAR(n)) 9347 | str wzr, [TMP1], #16 9348 } 9349 9350 if (call_num_args <= func->op_array.num_args) { 9351 if (!trace || (trace->op == ZEND_JIT_TRACE_END 9352 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 9353 uint32_t num_args; 9354 9355 if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { 9356 if (trace) { 9357 num_args = 0; 9358 } else if (call_info) { 9359 num_args = skip_valid_arguments(op_array, ssa, call_info); 9360 } else { 9361 num_args = call_num_args; 9362 } 9363 } else { 9364 num_args = call_num_args; 9365 } 9366 if (zend_accel_in_shm(func->op_array.opcodes)) { 9367 | LOAD_IP_ADDR (func->op_array.opcodes + num_args) 9368 } else { 9369 | ldr REG0, EX->func 9370 || ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(num_args * sizeof(zend_op)))); 9371 if (GCC_GLOBAL_REGS) { 9372 | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] 9373 if (num_args) { 9374 | add IP, IP, #(num_args * sizeof(zend_op)) 9375 } 9376 } else { 9377 | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] 9378 if (num_args) { 9379 | add FCARG1x, FCARG1x, #(num_args * sizeof(zend_op)) 9380 } 9381 | str FCARG1x, EX->opline 9382 } 9383 } 9384 9385 if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array 9386 && num_args >= op_array->required_num_args) { 9387 /* recursive call */ 9388 if (ZEND_OBSERVER_ENABLED) { 9389 | SAVE_IP 9390 | mov FCARG1x, FP 9391 | EXT_CALL zend_observer_fcall_begin, REG0 9392 } 9393#ifdef CONTEXT_THREADED_JIT 9394 | NIY // TODO 9395#else 9396 | b =>num_args 9397#endif 9398 return 1; 9399 } 9400 } 9401 } else { 9402 if (!trace || (trace->op == ZEND_JIT_TRACE_END 9403 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 9404 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 9405 | LOAD_IP_ADDR (func->op_array.opcodes) 9406 } else if (GCC_GLOBAL_REGS) { 9407 | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] 9408 } else { 9409 | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] 9410 | str FCARG1x, EX->opline 9411 } 9412 } 9413 if (!GCC_GLOBAL_REGS) { 9414 | mov FCARG1x, FP 9415 } 9416 | EXT_CALL zend_jit_copy_extra_args_helper, REG0 9417 } 9418 } else { 9419 | // opline = op_array->opcodes 9420 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 9421 | LOAD_IP_ADDR (func->op_array.opcodes) 9422 } else if (GCC_GLOBAL_REGS) { 9423 | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] 9424 } else { 9425 | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] 9426 | str FCARG1x, EX->opline 9427 } 9428 if (func) { 9429 | // num_args = EX_NUM_ARGS(); 9430 | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] 9431 | // if (UNEXPECTED(num_args > first_extra_arg)) 9432 | CMP_32_WITH_CONST REG1w, (func->op_array.num_args), TMP1w 9433 } else { 9434 | // first_extra_arg = op_array->num_args; 9435 | ldr REG2w, [REG0, #offsetof(zend_op_array, num_args)] 9436 | // num_args = EX_NUM_ARGS(); 9437 | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] 9438 | // if (UNEXPECTED(num_args > first_extra_arg)) 9439 | cmp REG1w, REG2w 9440 } 9441 | bgt >1 9442 |.cold_code 9443 |1: 9444 if (!GCC_GLOBAL_REGS) { 9445 | mov FCARG1x, FP 9446 } 9447 | EXT_CALL zend_jit_copy_extra_args_helper, REG0 9448 if (!func) { 9449 | ldr REG0, EX->func // reload 9450 } 9451 | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] // reload 9452 | b >1 9453 |.code 9454 if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { 9455 if (!func) { 9456 | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) 9457 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9458 | TST_32_WITH_CONST TMP1w, ZEND_ACC_HAS_TYPE_HINTS, TMP2w 9459 | bne >1 9460 } 9461 | // opline += num_args; 9462 || ZEND_ASSERT(sizeof(zend_op) == 32); 9463 | mov REG2w, REG1w 9464 | ADD_IP_SHIFT REG2, lsl #5, TMP1 9465 } 9466 |1: 9467 | // if (EXPECTED((int)num_args < op_array->last_var)) { 9468 if (func) { 9469 | LOAD_32BIT_VAL REG2w, func->op_array.last_var 9470 } else { 9471 | ldr REG2w, [REG0, #offsetof(zend_op_array, last_var)] 9472 } 9473 | subs REG2w, REG2w, REG1w 9474 | ble >3 9475 | // zval *var = EX_VAR_NUM(num_args); 9476 | add REG1, FP, REG1, lsl #4 9477 || ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(ZEND_CALL_FRAME_SLOT * sizeof(zval)))); 9478 | add REG1, REG1, #(ZEND_CALL_FRAME_SLOT * sizeof(zval)) 9479 |2: 9480 | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w 9481 | add REG1, REG1, #16 9482 | subs REG2w, REG2w, #1 9483 | bne <2 9484 |3: 9485 } 9486 9487 if (ZEND_OBSERVER_ENABLED) { 9488 if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 9489 ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END); 9490 | SET_EX_OPLINE trace[1].opline, REG0 9491 } else { 9492 | SAVE_IP 9493 } 9494 | mov FCARG1x, FP 9495 | EXT_CALL zend_observer_fcall_begin, REG0 9496 } 9497 9498 if (trace) { 9499 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 9500 | b >9 9501 } 9502 } else { 9503#ifdef CONTEXT_THREADED_JIT 9504 | NIY // TODO: CONTEXT_THREADED_JIT is always undefined. 9505#else 9506 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 9507 | ADD_HYBRID_SPAD 9508 | JMP_IP TMP1 9509 } else if (GCC_GLOBAL_REGS) { 9510 | ldp x29, x30, [sp], # SPAD // stack alignment 9511 | JMP_IP TMP1 9512 } else { 9513 | ldp FP, RX, T2 // restore FP and IP 9514 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 9515 | mov RETVALx, #1 // ZEND_VM_ENTER 9516 | ret 9517 } 9518 } 9519#endif 9520 } 9521 9522 if ((!func || func->type == ZEND_INTERNAL_FUNCTION) 9523 && (opline->opcode != ZEND_DO_UCALL)) { 9524 if (!func && (opline->opcode != ZEND_DO_ICALL)) { 9525 |8: 9526 } 9527 if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { 9528 if (!func) { 9529 if (trace) { 9530 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9531 9532 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9533 if (!exit_addr) { 9534 return 0; 9535 } 9536 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9537 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9538 | bne &exit_addr 9539 } else { 9540 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9541 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9542 | bne >1 9543 |.cold_code 9544 |1: 9545 if (!GCC_GLOBAL_REGS) { 9546 | mov FCARG1x, RX 9547 } 9548 | EXT_CALL zend_jit_deprecated_helper, REG0 9549 | GET_LOW_8BITS RETVALw, RETVALw 9550 | ldr REG0, EX:RX->func // reload 9551 | cbnz RETVALw, >1 // Result is 0 on exception 9552 | b ->exception_handler 9553 |.code 9554 |1: 9555 } 9556 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 9557 if (!GCC_GLOBAL_REGS) { 9558 | mov FCARG1x, RX 9559 } 9560 | EXT_CALL zend_jit_deprecated_helper, REG0 9561 | cbz RETVALw, ->exception_handler 9562 | ldr REG0, EX:RX->func // reload 9563 } 9564 } 9565 9566 | // EG(current_execute_data) = execute_data; 9567 | MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1 9568 9569 if (ZEND_OBSERVER_ENABLED) { 9570 | mov FCARG1x, RX 9571 | EXT_CALL zend_observer_fcall_begin, REG0 9572 | ldr REG0, EX:RX->func // reload 9573 } 9574 9575 | // ZVAL_NULL(EX_VAR(opline->result.var)); 9576 | LOAD_ZVAL_ADDR FCARG2x, res_addr 9577 | SET_Z_TYPE_INFO FCARG2x, IS_NULL, TMP1w 9578 9579 zend_jit_reset_last_valid_opline(); 9580 9581 | // (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret); 9582 | mov FCARG1x, RX 9583 if (zend_execute_internal) { 9584 | EXT_CALL zend_execute_internal, REG0 9585 } else { 9586 if (func) { 9587 | EXT_CALL func->internal_function.handler, REG0 9588 } else { 9589 | ldr TMP1, [REG0, #offsetof(zend_internal_function, handler)] 9590 | blr TMP1 9591 } 9592 } 9593 9594 if (ZEND_OBSERVER_ENABLED) { 9595 | LOAD_ZVAL_ADDR FCARG2x, res_addr 9596 | mov FCARG1x, RX 9597 | EXT_CALL zend_observer_fcall_end, REG0 9598 } 9599 9600 | // EG(current_execute_data) = execute_data; 9601 | MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0 9602 9603 | // zend_vm_stack_free_args(call); 9604 if (func && !unknown_num_args) { 9605 for (i = 0; i < call_num_args; i++ ) { 9606 if (zend_jit_needs_arg_dtor(func, i, call_info)) { 9607 uint32_t offset = EX_NUM_TO_VAR(i); 9608 zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset); 9609 | ZVAL_PTR_DTOR arg_addr, (MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN), 0, 1, opline, ZREG_TMP1, ZREG_TMP2 9610 } 9611 } 9612 } else { 9613 | mov FCARG1x, RX 9614 | EXT_CALL zend_jit_vm_stack_free_args_helper, REG0 9615 } 9616 if (may_have_extra_named_params) { 9617 | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] 9618 | TST_32_WITH_CONST TMP1w, (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24), TMP2w 9619 | bne >1 9620 |.cold_code 9621 |1: 9622 | ldr FCARG1x, [RX, #offsetof(zend_execute_data, extra_named_params)] 9623 | EXT_CALL zend_free_extra_named_params, REG0 9624 | b >2 9625 |.code 9626 |2: 9627 } 9628 9629 |8: 9630 if (opline->opcode == ZEND_DO_FCALL) { 9631 // TODO: optimize ??? 9632 | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) 9633 | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] 9634 | TST_32_WITH_CONST TMP1w, (ZEND_CALL_RELEASE_THIS >> 16), TMP2w 9635 | bne >1 9636 |.cold_code 9637 |1: 9638 | add TMP1, RX, #offsetof(zend_execute_data, This) 9639 | GET_Z_PTR FCARG1x, TMP1 9640 | // OBJ_RELEASE(object); 9641 | OBJ_RELEASE ZREG_FCARG1, >2, ZREG_TMP1, ZREG_TMP2 9642 | b >2 9643 |.code 9644 |2: 9645 } 9646 9647 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 9648 !JIT_G(current_frame) || 9649 !JIT_G(current_frame)->call || 9650 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || 9651 prev_opline->opcode == ZEND_SEND_UNPACK || 9652 prev_opline->opcode == ZEND_SEND_ARRAY || 9653 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 9654 9655 | // zend_vm_stack_free_call_frame(call); 9656 | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] 9657 | TST_32_WITH_CONST TMP1w, ((ZEND_CALL_ALLOCATED >> 16) & 0xff), TMP2w 9658 | bne >1 9659 |.cold_code 9660 |1: 9661 | mov FCARG1x, RX 9662 | EXT_CALL zend_jit_free_call_frame, REG0 9663 | b >1 9664 |.code 9665 } 9666 | MEM_STORE_64_ZTS str, RX, executor_globals, vm_stack_top, REG0 9667 |1: 9668 9669 if (!RETURN_VALUE_USED(opline)) { 9670 zend_class_entry *ce; 9671 bool ce_is_instanceof; 9672 uint32_t func_info = call_info ? 9673 zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : 9674 (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); 9675 9676 /* If an exception is thrown, the return_value may stay at the 9677 * original value of null. */ 9678 func_info |= MAY_BE_NULL; 9679 9680 if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 9681 | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline, ZREG_TMP1, ZREG_TMP2 9682 } 9683 } 9684 9685 | // if (UNEXPECTED(EG(exception) != NULL)) { 9686 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 9687 | cbnz REG0, ->icall_throw_handler 9688 9689 // TODO: Can we avoid checking for interrupts after each call ??? 9690 if (trace && last_valid_opline != opline) { 9691 int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); 9692 9693 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9694 if (!exit_addr) { 9695 return 0; 9696 } 9697 } else { 9698 exit_addr = NULL; 9699 } 9700 if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { 9701 return 0; 9702 } 9703 9704 if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { 9705 | LOAD_IP_ADDR (opline + 1) 9706 } else if (trace 9707 && trace->op == ZEND_JIT_TRACE_END 9708 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 9709 | LOAD_IP_ADDR (opline + 1) 9710 } 9711 } 9712 9713 if (!func) { 9714 |9: 9715 } 9716 9717 return 1; 9718} 9719 9720static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) 9721{ 9722 uint32_t arg_num = opline->op2.num; 9723 zend_jit_addr arg_addr; 9724 9725 ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); 9726 9727 if (!zend_jit_reuse_ip(Dst)) { 9728 return 0; 9729 } 9730 9731 if (opline->opcode == ZEND_SEND_VAL_EX) { 9732 uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); 9733 9734 ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); 9735 9736 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9737 && JIT_G(current_frame) 9738 && JIT_G(current_frame)->call 9739 && JIT_G(current_frame)->call->func) { 9740 if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 9741 /* Don't generate code that always throws exception */ 9742 return 0; 9743 } 9744 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9745 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9746 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9747 if (!exit_addr) { 9748 return 0; 9749 } 9750 | ldr REG0, EX:RX->func 9751 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9752 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9753 | bne &exit_addr 9754 } else { 9755 | ldr REG0, EX:RX->func 9756 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9757 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9758 | bne >1 9759 |.cold_code 9760 |1: 9761 if (Z_MODE(op1_addr) == IS_REG) { 9762 /* set type to avoid zval_ptr_dtor() on uninitialized value */ 9763 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 9764 | SET_ZVAL_TYPE_INFO addr, IS_UNDEF, TMP1w, TMP2 9765 } 9766 | SET_EX_OPLINE opline, REG0 9767 | b ->throw_cannot_pass_by_ref 9768 |.code 9769 } 9770 } 9771 9772 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 9773 9774 if (opline->op1_type == IS_CONST) { 9775 zval *zv = RT_CONSTANT(opline, opline->op1); 9776 9777 | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 9778 if (Z_REFCOUNTED_P(zv)) { 9779 | ADDREF_CONST zv, REG0, TMP1 9780 } 9781 } else { 9782 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9783 } 9784 9785 return 1; 9786} 9787 9788static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) 9789{ 9790 | ldr FCARG1x, EX->call 9791 | ldrb TMP1w, [FCARG1x, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] 9792 | TST_32_WITH_CONST TMP1w, (ZEND_CALL_MAY_HAVE_UNDEF >> 24), TMP2w 9793 | bne >1 9794 |.cold_code 9795 |1: 9796 | SET_EX_OPLINE opline, REG0 9797 | EXT_CALL zend_handle_undef_args, REG0 9798 | cbz RETVALw, >2 9799 | b ->exception_handler 9800 |.code 9801 |2: 9802 9803 return 1; 9804} 9805 9806static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) 9807{ 9808 zend_jit_addr op1_addr, arg_addr, ref_addr; 9809 9810 op1_addr = OP1_ADDR(); 9811 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 9812 9813 if (!zend_jit_reuse_ip(Dst)) { 9814 return 0; 9815 } 9816 9817 if (opline->op1_type == IS_VAR) { 9818 if (op1_info & MAY_BE_INDIRECT) { 9819 | LOAD_ZVAL_ADDR REG0, op1_addr 9820 | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { 9821 | IF_NOT_Z_TYPE REG0, IS_INDIRECT, >1, TMP1w 9822 | // ret = Z_INDIRECT_P(ret); 9823 | GET_Z_PTR REG0, REG0 9824 |1: 9825 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 9826 } 9827 } else if (opline->op1_type == IS_CV) { 9828 if (op1_info & MAY_BE_UNDEF) { 9829 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 9830 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 9831 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 9832 | b >2 9833 |1: 9834 } 9835 op1_info &= ~MAY_BE_UNDEF; 9836 op1_info |= MAY_BE_NULL; 9837 } 9838 } else { 9839 ZEND_UNREACHABLE(); 9840 } 9841 9842 if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { 9843 if (op1_info & MAY_BE_REF) { 9844 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2, ZREG_TMP1 9845 | GET_ZVAL_PTR REG1, op1_addr, TMP1 9846 | GC_ADDREF REG1, TMP1w 9847 | SET_ZVAL_PTR arg_addr, REG1, TMP1 9848 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 9849 | b >6 9850 } 9851 |2: 9852 | // ZVAL_NEW_REF(arg, varptr); 9853 if (opline->op1_type == IS_VAR) { 9854 if (Z_REG(op1_addr) != ZREG_REG0 || Z_OFFSET(op1_addr) != 0) { 9855 | LOAD_ZVAL_ADDR REG0, op1_addr 9856 } 9857 | str REG0, T1 // save 9858 } 9859 | EMALLOC sizeof(zend_reference), op_array, opline // Allocate space in REG0 9860 | mov TMP1w, #2 9861 | str TMP1w, [REG0] 9862 || ZEND_ASSERT(GC_REFERENCE <= MOVZ_IMM); 9863 | movz TMP1w, #GC_REFERENCE 9864 | str TMP1w, [REG0, #offsetof(zend_reference, gc.u.type_info)] 9865 | str xzr, [REG0, #offsetof(zend_reference, sources.ptr)] 9866 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); 9867 if (opline->op1_type == IS_VAR) { 9868 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); 9869 9870 | ldr REG1, T1 // restore 9871 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9872 | SET_ZVAL_PTR val_addr, REG0, TMP1 9873 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX, TMP1w, TMP2 9874 } else { 9875 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9876 | SET_ZVAL_PTR op1_addr, REG0, TMP1 9877 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 9878 } 9879 | SET_ZVAL_PTR arg_addr, REG0, TMP1 9880 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 9881 } 9882 9883 |6: 9884 | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline, ZREG_TMP1, ZREG_TMP2 9885 |7: 9886 9887 return 1; 9888} 9889 9890static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) 9891{ 9892 uint32_t arg_num = opline->op2.num; 9893 zend_jit_addr arg_addr; 9894 9895 ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && 9896 opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || 9897 arg_num <= MAX_ARG_FLAG_NUM); 9898 9899 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 9900 9901 if (!zend_jit_reuse_ip(Dst)) { 9902 return 0; 9903 } 9904 9905 if (opline->opcode == ZEND_SEND_VAR_EX) { 9906 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9907 && JIT_G(current_frame) 9908 && JIT_G(current_frame)->call 9909 && JIT_G(current_frame)->call->func) { 9910 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 9911 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 9912 return 0; 9913 } 9914 return 1; 9915 } 9916 } else { 9917 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 9918 9919 | ldr REG0, EX:RX->func 9920 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9921 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9922 | bne >1 9923 |.cold_code 9924 |1: 9925 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 9926 return 0; 9927 } 9928 | b >7 9929 |.code 9930 } 9931 } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { 9932 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9933 && JIT_G(current_frame) 9934 && JIT_G(current_frame)->call 9935 && JIT_G(current_frame)->call->func) { 9936 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 9937 9938 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9939 9940 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 9941 if (!(op1_info & MAY_BE_REF)) { 9942 /* Don't generate code that always throws exception */ 9943 return 0; 9944 } else { 9945 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9946 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9947 if (!exit_addr) { 9948 return 0; 9949 } 9950 | GET_LOW_8BITS TMP1w, REG1w 9951 | cmp TMP1w, #IS_REFERENCE 9952 | bne &exit_addr 9953 } 9954 } 9955 return 1; 9956 } 9957 } else { 9958 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 9959 9960 | ldr REG0, EX:RX->func 9961 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9962 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9963 | bne >1 9964 |.cold_code 9965 |1: 9966 9967 mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); 9968 9969 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9970 if (op1_info & MAY_BE_REF) { 9971 | GET_LOW_8BITS TMP1w, REG1w 9972 | cmp TMP1w, #IS_REFERENCE 9973 | beq >7 9974 } 9975 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9976 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9977 | bne >7 9978 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9979 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9980 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9981 if (!exit_addr) { 9982 return 0; 9983 } 9984 | b &exit_addr 9985 } else { 9986 | SET_EX_OPLINE opline, REG0 9987 | LOAD_ZVAL_ADDR FCARG1x, arg_addr 9988 | EXT_CALL zend_jit_only_vars_by_reference, REG0 9989 if (!zend_jit_check_exception(Dst)) { 9990 return 0; 9991 } 9992 | b >7 9993 } 9994 9995 |.code 9996 } 9997 } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { 9998 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9999 && JIT_G(current_frame) 10000 && JIT_G(current_frame)->call 10001 && JIT_G(current_frame)->call->func) { 10002 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10003 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10004 return 0; 10005 } 10006 return 1; 10007 } 10008 } else { 10009 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10010 | TST_32_WITH_CONST TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10011 | bne >1 10012 |.cold_code 10013 |1: 10014 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10015 return 0; 10016 } 10017 | b >7 10018 |.code 10019 } 10020 } 10021 10022 if (op1_info & MAY_BE_UNDEF) { 10023 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10024 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 10025 |.cold_code 10026 |1: 10027 } 10028 10029 | SET_EX_OPLINE opline, REG0 10030 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 10031 | EXT_CALL zend_jit_undefined_op_helper, REG0 10032 | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL, TMP1w, TMP2 10033 | cbz RETVALx, ->exception_handler 10034 10035 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10036 | b >7 10037 |.code 10038 } else { 10039 |7: 10040 return 1; 10041 } 10042 } 10043 10044 if (opline->opcode == ZEND_SEND_VAR_NO_REF) { 10045 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10046 if (op1_info & MAY_BE_REF) { 10047 | GET_LOW_8BITS TMP1w, REG1w 10048 | cmp TMP1w, #IS_REFERENCE 10049 | beq >7 10050 } 10051 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10052 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10053 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10054 if (!exit_addr) { 10055 return 0; 10056 } 10057 | b &exit_addr 10058 } else { 10059 | SET_EX_OPLINE opline, REG0 10060 | LOAD_ZVAL_ADDR FCARG1x, arg_addr 10061 | EXT_CALL zend_jit_only_vars_by_reference, REG0 10062 if (!zend_jit_check_exception(Dst)) { 10063 return 0; 10064 } 10065 } 10066 } else { 10067 if (op1_info & MAY_BE_REF) { 10068 if (opline->op1_type == IS_CV) { 10069 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 10070 10071 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 10072 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 10073 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10074 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 10075 } else { 10076 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 8); 10077 10078 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 10079 |.cold_code 10080 |1: 10081 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 10082 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10083 | // ZVAL_COPY_VALUE(return_value, &ref->value); 10084 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10085 | GC_DELREF FCARG1x, TMP1w 10086 | beq >1 10087 | IF_NOT_REFCOUNTED REG0w, >2, TMP1w 10088 | GC_ADDREF REG2, TMP1w 10089 | b >2 10090 |1: 10091 | EFREE_REFERENCE 10092 | b >2 10093 |.code 10094 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10095 |2: 10096 } 10097 } else { 10098 if (op1_addr != op1_def_addr) { 10099 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 10100 return 0; 10101 } 10102 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 10103 op1_addr= op1_def_addr; 10104 } 10105 } 10106 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10107 if (opline->op1_type == IS_CV) { 10108 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 10109 } 10110 } 10111 } 10112 |7: 10113 10114 return 1; 10115} 10116 10117static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) 10118{ 10119 uint32_t arg_num = opline->op2.num; 10120 10121 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10122 && JIT_G(current_frame) 10123 && JIT_G(current_frame)->call 10124 && JIT_G(current_frame)->call->func) { 10125 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10126 if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { 10127 TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); 10128 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10129 || if (reuse_ip) { 10130 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10131 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10132 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10133 || } else { 10134 | ldr REG0, EX->call 10135 | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10136 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10137 | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10138 || } 10139 } 10140 } else { 10141 if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 10142 TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); 10143 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10144 || if (reuse_ip) { 10145 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10146 | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w 10147 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10148 || } else { 10149 | ldr REG0, EX->call 10150 | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10151 | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w 10152 | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10153 || } 10154 } 10155 } 10156 } else { 10157 // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { 10158 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10159 10160 if (!zend_jit_reuse_ip(Dst)) { 10161 return 0; 10162 } 10163 10164 | ldr REG0, EX:RX->func 10165 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 10166 | TST_32_WITH_CONST TMP1w, mask, TMP2w 10167 | bne >1 10168 |.cold_code 10169 |1: 10170 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10171 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10172 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10173 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10174 | b >1 10175 |.code 10176 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10177 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10178 | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~(ZEND_CALL_SEND_ARG_BY_REF)), TMP2w 10179 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10180 |1: 10181 } 10182 10183 return 1; 10184} 10185 10186static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) 10187{ 10188 if (smart_branch_opcode) { 10189 if (smart_branch_opcode == ZEND_JMPZ) { 10190 if (jmp) { 10191 | b >7 10192 } 10193 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10194 | b =>target_label 10195 } else { 10196 ZEND_UNREACHABLE(); 10197 } 10198 } else { 10199 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10200 10201 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 10202 if (jmp) { 10203 | b >7 10204 } 10205 } 10206 10207 return 1; 10208} 10209 10210static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) 10211{ 10212 if (smart_branch_opcode) { 10213 if (smart_branch_opcode == ZEND_JMPZ) { 10214 | b =>target_label 10215 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10216 if (jmp) { 10217 | b >7 10218 } 10219 } else { 10220 ZEND_UNREACHABLE(); 10221 } 10222 } else { 10223 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10224 10225 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 10226 if (jmp) { 10227 | b >7 10228 } 10229 } 10230 10231 return 1; 10232} 10233 10234static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 10235{ 10236 uint32_t defined_label = (uint32_t)-1; 10237 uint32_t undefined_label = (uint32_t)-1; 10238 zval *zv = RT_CONSTANT(opline, opline->op1); 10239 zend_jit_addr res_addr = 0; 10240 10241 if (smart_branch_opcode && !exit_addr) { 10242 if (smart_branch_opcode == ZEND_JMPZ) { 10243 undefined_label = target_label; 10244 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10245 defined_label = target_label; 10246 } else { 10247 ZEND_UNREACHABLE(); 10248 } 10249 } 10250 10251 | // if (CACHED_PTR(opline->extended_value)) { 10252 | ldr REG0, EX->run_time_cache 10253 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, opline->extended_value, TMP1 10254 | cbz REG0, >1 10255 | TST_64_WITH_ONE REG0 10256 | bne >4 10257 |.cold_code 10258 |4: 10259 | MEM_LOAD_64_ZTS ldr, FCARG1x, executor_globals, zend_constants, FCARG1x 10260 | ldr TMP1w, [FCARG1x, #offsetof(HashTable, nNumOfElements)] 10261 | cmp TMP1, REG0, lsr #1 10262 10263 if (smart_branch_opcode) { 10264 if (exit_addr) { 10265 if (smart_branch_opcode == ZEND_JMPZ) { 10266 | beq &exit_addr 10267 } else { 10268 | beq >3 10269 } 10270 } else if (undefined_label != (uint32_t)-1) { 10271 | beq =>undefined_label 10272 } else { 10273 | beq >3 10274 } 10275 } else { 10276 | beq >2 10277 } 10278 |1: 10279 | SET_EX_OPLINE opline, REG0 10280 | LOAD_ADDR FCARG1x, zv 10281 | EXT_CALL zend_jit_check_constant, REG0 10282 if (exit_addr) { 10283 if (smart_branch_opcode == ZEND_JMPNZ) { 10284 | cbz RETVALx, >3 10285 } else { 10286 | cbnz RETVALx, >3 10287 } 10288 | b &exit_addr 10289 } else if (smart_branch_opcode) { 10290 if (undefined_label != (uint32_t)-1) { 10291 | cbz RETVALx, =>undefined_label 10292 } else { 10293 | cbz RETVALx, >3 10294 } 10295 if (defined_label != (uint32_t)-1) { 10296 | b =>defined_label 10297 } else { 10298 | b >3 10299 } 10300 } else { 10301 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10302 | cbnz RETVALx, >1 10303 |2: 10304 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 10305 | b >3 10306 } 10307 |.code 10308 if (smart_branch_opcode) { 10309 if (exit_addr) { 10310 if (smart_branch_opcode == ZEND_JMPNZ) { 10311 | b &exit_addr 10312 } 10313 } else if (defined_label != (uint32_t)-1) { 10314 | b =>defined_label 10315 } 10316 } else { 10317 |1: 10318 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 10319 } 10320 |3: 10321 10322 return 1; 10323} 10324 10325static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 10326{ 10327 uint32_t mask; 10328 zend_jit_addr op1_addr = OP1_ADDR(); 10329 10330 // TODO: support for is_resource() ??? 10331 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); 10332 10333 if (op1_info & MAY_BE_UNDEF) { 10334 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10335 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 10336 |.cold_code 10337 |1: 10338 } 10339 | SET_EX_OPLINE opline, REG0 10340 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 10341 | EXT_CALL zend_jit_undefined_op_helper, REG0 10342 zend_jit_check_exception_undef_result(Dst, opline); 10343 if (opline->extended_value & MAY_BE_NULL) { 10344 if (exit_addr) { 10345 if (smart_branch_opcode == ZEND_JMPNZ) { 10346 | b &exit_addr 10347 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 10348 | b >7 10349 } 10350 } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { 10351 return 0; 10352 } 10353 } else { 10354 if (exit_addr) { 10355 if (smart_branch_opcode == ZEND_JMPZ) { 10356 | b &exit_addr 10357 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 10358 | b >7 10359 } 10360 } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { 10361 return 0; 10362 } 10363 } 10364 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10365 |.code 10366 } 10367 } 10368 10369 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10370 mask = opline->extended_value; 10371 if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { 10372 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10373 if (exit_addr) { 10374 if (smart_branch_opcode == ZEND_JMPNZ) { 10375 | b &exit_addr 10376 } 10377 } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { 10378 return 0; 10379 } 10380 } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { 10381 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10382 if (exit_addr) { 10383 if (smart_branch_opcode == ZEND_JMPZ) { 10384 | b &exit_addr 10385 } 10386 } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { 10387 return 0; 10388 } 10389 } else { 10390 bool invert = 0; 10391 zend_uchar type; 10392 10393 switch (mask) { 10394 case MAY_BE_NULL: type = IS_NULL; break; 10395 case MAY_BE_FALSE: type = IS_FALSE; break; 10396 case MAY_BE_TRUE: type = IS_TRUE; break; 10397 case MAY_BE_LONG: type = IS_LONG; break; 10398 case MAY_BE_DOUBLE: type = IS_DOUBLE; break; 10399 case MAY_BE_STRING: type = IS_STRING; break; 10400 case MAY_BE_ARRAY: type = IS_ARRAY; break; 10401 case MAY_BE_OBJECT: type = IS_OBJECT; break; 10402 case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break; 10403 case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break; 10404 case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break; 10405 case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break; 10406 case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break; 10407 case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break; 10408 case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break; 10409 case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break; 10410 case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break; 10411 default: 10412 type = 0; 10413 } 10414 10415 if (op1_info & MAY_BE_REF) { 10416 | LOAD_ZVAL_ADDR REG0, op1_addr 10417 | ZVAL_DEREF REG0, op1_info, TMP1w 10418 } 10419 if (type == 0) { 10420 if (smart_branch_opcode && 10421 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 10422 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10423 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10424 | // if (Z_REFCOUNTED_P(cv)) { 10425 | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 10426 |.cold_code 10427 |1: 10428 } 10429 | // if (!Z_DELREF_P(cv)) { 10430 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10431 | GC_DELREF FCARG1x, TMP1w 10432 if (RC_MAY_BE_1(op1_info)) { 10433 if (RC_MAY_BE_N(op1_info)) { 10434 | bne >3 10435 } 10436 if (op1_info & MAY_BE_REF) { 10437 | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] 10438 } else { 10439 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10440 } 10441 | str REG0w, T1 // save 10442 | // zval_dtor_func(r); 10443 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 10444 | ldr REG1w, T1 // restore 10445 | b >2 10446 } 10447 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10448 if (!RC_MAY_BE_1(op1_info)) { 10449 | b >3 10450 } 10451 |.code 10452 } 10453 |3: 10454 if (op1_info & MAY_BE_REF) { 10455 | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] 10456 } else { 10457 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10458 } 10459 |2: 10460 } else { 10461 if (op1_info & MAY_BE_REF) { 10462 | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] 10463 } else { 10464 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10465 } 10466 } 10467 | mov REG0w, #1 10468 | lsl REG0w, REG0w, REG1w 10469 | TST_32_WITH_CONST REG0w, mask, TMP1w 10470 if (exit_addr) { 10471 if (smart_branch_opcode == ZEND_JMPNZ) { 10472 | bne &exit_addr 10473 } else { 10474 | beq &exit_addr 10475 } 10476 } else if (smart_branch_opcode) { 10477 if (smart_branch_opcode == ZEND_JMPZ) { 10478 | beq =>target_label 10479 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10480 | bne =>target_label 10481 } else { 10482 ZEND_UNREACHABLE(); 10483 } 10484 } else { 10485 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10486 10487 | cset REG0w, ne 10488 | add REG0w, REG0w, #2 10489 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 10490 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10491 } 10492 } else { 10493 if (smart_branch_opcode && 10494 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 10495 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10496 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10497 | // if (Z_REFCOUNTED_P(cv)) { 10498 | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 10499 |.cold_code 10500 |1: 10501 } 10502 | // if (!Z_DELREF_P(cv)) { 10503 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10504 | GC_DELREF FCARG1x, TMP1w 10505 if (RC_MAY_BE_1(op1_info)) { 10506 if (RC_MAY_BE_N(op1_info)) { 10507 | bne >3 10508 } 10509 if (op1_info & MAY_BE_REF) { 10510 | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] 10511 } else { 10512 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10513 } 10514 | str REG0w, T1 // save 10515 | // zval_dtor_func(r); 10516 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 10517 | ldr REG1w, T1 // restore 10518 | b >2 10519 } 10520 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10521 if (!RC_MAY_BE_1(op1_info)) { 10522 | b >3 10523 } 10524 |.code 10525 } 10526 |3: 10527 if (op1_info & MAY_BE_REF) { 10528 | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] 10529 } else { 10530 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10531 } 10532 |2: 10533 // Note: 'type' is of uchar type and holds a positive value, 10534 // hence it's safe to directly encode it as the imm field of 'cmp' instruction. 10535 | cmp REG1w, #type 10536 } else { 10537 if (op1_info & MAY_BE_REF) { 10538 | ldrb TMP1w, [REG0, #offsetof(zval,u1.v.type)] 10539 | cmp TMP1w, #type 10540 } else { 10541 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10542 | cmp TMP1w, #type 10543 } 10544 } 10545 if (exit_addr) { 10546 if (invert) { 10547 if (smart_branch_opcode == ZEND_JMPNZ) { 10548 | bne &exit_addr 10549 } else { 10550 | beq &exit_addr 10551 } 10552 } else { 10553 if (smart_branch_opcode == ZEND_JMPNZ) { 10554 | beq &exit_addr 10555 } else { 10556 | bne &exit_addr 10557 } 10558 } 10559 } else if (smart_branch_opcode) { 10560 if (invert) { 10561 if (smart_branch_opcode == ZEND_JMPZ) { 10562 | beq =>target_label 10563 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10564 | bne =>target_label 10565 } else { 10566 ZEND_UNREACHABLE(); 10567 } 10568 } else { 10569 if (smart_branch_opcode == ZEND_JMPZ) { 10570 | bne =>target_label 10571 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10572 | beq =>target_label 10573 } else { 10574 ZEND_UNREACHABLE(); 10575 } 10576 } 10577 } else { 10578 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10579 10580 if (invert) { 10581 | cset REG0w, ne 10582 } else { 10583 | cset REG0w, eq 10584 } 10585 | add REG0w, REG0w, #2 10586 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 10587 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10588 } 10589 } 10590 } 10591 } 10592 10593 |7: 10594 10595 return 1; 10596} 10597 10598static int zend_jit_leave_frame(dasm_State **Dst) 10599{ 10600 | // EG(current_execute_data) = EX(prev_execute_data); 10601 | ldr REG0, EX->prev_execute_data 10602 | MEM_STORE_64_ZTS str, REG0, executor_globals, current_execute_data, REG2 10603 return 1; 10604} 10605 10606static int zend_jit_free_cvs(dasm_State **Dst) 10607{ 10608 | // EG(current_execute_data) = EX(prev_execute_data); 10609 | ldr FCARG1x, EX->prev_execute_data 10610 | MEM_STORE_64_ZTS str, FCARG1x, executor_globals, current_execute_data, REG0 10611 | // zend_free_compiled_variables(execute_data); 10612 | mov FCARG1x, FP 10613 | EXT_CALL zend_free_compiled_variables, REG0 10614 return 1; 10615} 10616 10617static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) 10618{ 10619 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10620 uint32_t offset = EX_NUM_TO_VAR(var); 10621 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset); 10622 | ZVAL_PTR_DTOR addr, info, 1, 1, NULL, ZREG_TMP1, ZREG_TMP2 10623 } 10624 return 1; 10625} 10626 10627static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) 10628{ 10629 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10630 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset); 10631 | ZVAL_PTR_DTOR addr, info, 0, 1, opline, ZREG_TMP1, ZREG_TMP2 10632 } 10633 return 1; 10634} 10635 10636static int zend_jit_leave_func(dasm_State **Dst, 10637 const zend_op_array *op_array, 10638 const zend_op *opline, 10639 uint32_t op1_info, 10640 bool left_frame, 10641 zend_jit_trace_rec *trace, 10642 zend_jit_trace_info *trace_info, 10643 int indirect_var_access, 10644 int may_throw) 10645{ 10646 bool may_be_top_frame = 10647 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10648 !JIT_G(current_frame) || 10649 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); 10650 bool may_need_call_helper = 10651 indirect_var_access || /* may have symbol table */ 10652 !op_array->function_name || /* may have symbol table */ 10653 may_be_top_frame || 10654 (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ 10655 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10656 !JIT_G(current_frame) || 10657 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ 10658 (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ 10659 bool may_need_release_this = 10660 !(op_array->fn_flags & ZEND_ACC_CLOSURE) && 10661 op_array->scope && 10662 !(op_array->fn_flags & ZEND_ACC_STATIC) && 10663 (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10664 !JIT_G(current_frame) || 10665 !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame))); 10666 10667 if (may_need_call_helper || may_need_release_this) { 10668 | ldr FCARG1w, [FP, #offsetof(zend_execute_data, This.u1.type_info)] 10669 } 10670 if (may_need_call_helper) { 10671 if (!left_frame) { 10672 left_frame = 1; 10673 if (!zend_jit_leave_frame(Dst)) { 10674 return 0; 10675 } 10676 } 10677 /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ 10678 10679 | TST_32_WITH_CONST FCARG1w, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE), TMP1w 10680 if (trace && trace->op != ZEND_JIT_TRACE_END) { 10681 | bne >1 10682 |.cold_code 10683 |1: 10684 if (!GCC_GLOBAL_REGS) { 10685 | mov FCARG1x, FP 10686 } 10687 | EXT_CALL zend_jit_leave_func_helper, REG0 10688 10689 if (may_be_top_frame) { 10690 // TODO: try to avoid this check ??? 10691 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10692#if 0 10693 /* this check should be handled by the following OPLINE guard */ 10694 | LOAD_ADDR TMP1, zend_jit_halt_op 10695 | cmp IP, TMP1 10696 | beq ->trace_halt 10697#endif 10698 } else if (GCC_GLOBAL_REGS) { 10699 | cbz IP, ->trace_halt 10700 } else { 10701 | tst RETVALw, RETVALw 10702 | blt ->trace_halt 10703 } 10704 } 10705 10706 if (!GCC_GLOBAL_REGS) { 10707 | // execute_data = EG(current_execute_data) 10708 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 10709 } 10710 | b >8 10711 |.code 10712 } else { 10713 | bne ->leave_function_handler 10714 } 10715 } 10716 10717 if (op_array->fn_flags & ZEND_ACC_CLOSURE) { 10718 if (!left_frame) { 10719 left_frame = 1; 10720 if (!zend_jit_leave_frame(Dst)) { 10721 return 0; 10722 } 10723 } 10724 | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); 10725 | ldr FCARG1x, EX->func 10726 | sub FCARG1x, FCARG1x, #sizeof(zend_object) 10727 | OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2 10728 |4: 10729 } else if (may_need_release_this) { 10730 if (!left_frame) { 10731 left_frame = 1; 10732 if (!zend_jit_leave_frame(Dst)) { 10733 return 0; 10734 } 10735 } 10736 if (!JIT_G(current_frame) || !TRACE_FRAME_ALWAYS_RELEASE_THIS(JIT_G(current_frame))) { 10737 | // if (call_info & ZEND_CALL_RELEASE_THIS) 10738 | TST_32_WITH_CONST FCARG1w, ZEND_CALL_RELEASE_THIS, TMP1w 10739 | beq >4 10740 } 10741 | // zend_object *object = Z_OBJ(execute_data->This); 10742 | ldr FCARG1x, EX->This.value.obj 10743 | // OBJ_RELEASE(object); 10744 | OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2 10745 |4: 10746 // TODO: avoid EG(excption) check for $this->foo() calls 10747 may_throw = 1; 10748 } 10749 10750 | // EG(vm_stack_top) = (zval*)execute_data; 10751 | MEM_STORE_64_ZTS str, FP, executor_globals, vm_stack_top, REG0 10752 | // execute_data = EX(prev_execute_data); 10753 | ldr FP, EX->prev_execute_data 10754 10755 if (!left_frame) { 10756 | // EG(current_execute_data) = execute_data; 10757 | MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0 10758 } 10759 10760 |9: 10761 if (trace) { 10762 if (trace->op != ZEND_JIT_TRACE_END 10763 && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 10764 zend_jit_reset_last_valid_opline(); 10765 } else { 10766 | LOAD_IP 10767 | ADD_IP_WITH_CONST sizeof(zend_op), TMP1 10768 } 10769 10770 |8: 10771 10772 if (trace->op == ZEND_JIT_TRACE_BACK 10773 && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 10774 const zend_op *next_opline = trace->opline; 10775 10776 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 10777 && (op1_info & MAY_BE_RC1) 10778 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { 10779 /* exception might be thrown during destruction of unused return value */ 10780 | // if (EG(exception)) 10781 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 10782 | cbnz REG0, ->leave_throw_handler 10783 } 10784 do { 10785 trace++; 10786 } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); 10787 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 10788 next_opline = trace->opline; 10789 ZEND_ASSERT(next_opline != NULL); 10790 10791 if (trace->op == ZEND_JIT_TRACE_END 10792 && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { 10793 trace_info->flags |= ZEND_JIT_TRACE_LOOP; 10794 | CMP_IP next_opline, TMP1, TMP2 10795 | beq =>0 // LOOP 10796#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 10797 | JMP_IP TMP1 10798#else 10799 | b ->trace_escape 10800#endif 10801 } else { 10802 | CMP_IP next_opline, TMP1, TMP2 10803 | bne ->trace_escape 10804 } 10805 10806 zend_jit_set_last_valid_opline(trace->opline); 10807 10808 return 1; 10809 } else if (may_throw || 10810 (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 10811 && (op1_info & MAY_BE_RC1) 10812 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) 10813 && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { 10814 | // if (EG(exception)) 10815 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 10816 | cbnz REG0, ->leave_throw_handler 10817 } 10818 10819 return 1; 10820 } else { 10821 | // if (EG(exception)) 10822 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 10823 | LOAD_IP 10824 | cbnz REG0, ->leave_throw_handler 10825 | // opline = EX(opline) + 1 10826 | ADD_IP_WITH_CONST sizeof(zend_op), TMP1 10827 } 10828 10829 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10830 | ADD_HYBRID_SPAD 10831#ifdef CONTEXT_THREADED_JIT 10832 | NIY // TODO: CONTEXT_THREADED_JIT is always undefined 10833#else 10834 | JMP_IP TMP1 10835#endif 10836 } else if (GCC_GLOBAL_REGS) { 10837 | ldp x29, x30, [sp], # SPAD // stack alignment 10838#ifdef CONTEXT_THREADED_JIT 10839 | NIY // TODO 10840#else 10841 | JMP_IP TMP1 10842#endif 10843 } else { 10844#ifdef CONTEXT_THREADED_JIT 10845 ZEND_UNREACHABLE(); 10846 // TODO: context threading can't work without GLOBAL REGS because we have to change 10847 // the value of execute_data in execute_ex() 10848 | NIY // TODO 10849#else 10850 | ldp FP, RX, T2 // restore FP and IP 10851 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 10852 | mov RETVALx, #2 // ZEND_VM_LEAVE ???? 10853 | ret 10854#endif 10855 } 10856 10857 return 1; 10858} 10859 10860static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) 10861{ 10862 zend_jit_addr ret_addr; 10863 int8_t return_value_used; 10864 10865 ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); 10866 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); 10867 10868 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { 10869 if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { 10870 return_value_used = 1; 10871 } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { 10872 return_value_used = 0; 10873 } else { 10874 return_value_used = -1; 10875 } 10876 } else { 10877 return_value_used = -1; 10878 } 10879 10880 if (ZEND_OBSERVER_ENABLED) { 10881 if (Z_MODE(op1_addr) == IS_REG) { 10882 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 10883 10884 if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { 10885 return 0; 10886 } 10887 op1_addr = dst; 10888 } 10889 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 10890 | mov FCARG1x, FP 10891 | SET_EX_OPLINE opline, REG0 10892 | EXT_CALL zend_observer_fcall_end, REG0 10893 } 10894 10895 // if (!EX(return_value)) 10896 if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_REG1) { 10897 if (return_value_used != 0) { 10898 | ldr REG2, EX->return_value 10899 } 10900 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); 10901 } else { 10902 if (return_value_used != 0) { 10903 | ldr REG1, EX->return_value 10904 } 10905 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); 10906 } 10907 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 10908 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10909 if (return_value_used == -1) { 10910 | cbz Rx(Z_REG(ret_addr)), >1 10911 |.cold_code 10912 |1: 10913 } 10914 if (return_value_used != 1) { 10915 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10916 if (jit_return_label >= 0) { 10917 | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label, ZREG_TMP1, ZREG_TMP2 10918 } else { 10919 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9, ZREG_TMP1, ZREG_TMP2 10920 } 10921 } 10922 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10923 | GC_DELREF FCARG1x, TMP1w 10924 if (RC_MAY_BE_1(op1_info)) { 10925 if (RC_MAY_BE_N(op1_info)) { 10926 if (jit_return_label >= 0) { 10927 | bne =>jit_return_label 10928 } else { 10929 | bne >9 10930 } 10931 } 10932 | //SAVE_OPLINE() 10933 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 10934 | //????ldr REG1, EX->return_value // reload ??? 10935 } 10936 if (return_value_used == -1) { 10937 if (jit_return_label >= 0) { 10938 | b =>jit_return_label 10939 } else { 10940 | b >9 10941 } 10942 |.code 10943 } 10944 } 10945 } else if (return_value_used == -1) { 10946 if (jit_return_label >= 0) { 10947 | cbz Rx(Z_REG(ret_addr)), =>jit_return_label 10948 } else { 10949 | cbz Rx(Z_REG(ret_addr)), >9 10950 } 10951 } 10952 10953 if (return_value_used == 0) { 10954 |9: 10955 return 1; 10956 } 10957 10958 if (opline->op1_type == IS_CONST) { 10959 zval *zv = RT_CONSTANT(opline, opline->op1); 10960 | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 10961 if (Z_REFCOUNTED_P(zv)) { 10962 | ADDREF_CONST zv, REG0, TMP1 10963 } 10964 } else if (opline->op1_type == IS_TMP_VAR) { 10965 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10966 } else if (opline->op1_type == IS_CV) { 10967 if (op1_info & MAY_BE_REF) { 10968 | LOAD_ZVAL_ADDR REG0, op1_addr 10969 | ZVAL_DEREF REG0, op1_info, TMP1w 10970 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 10971 } 10972 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10973 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 10974 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10975 (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || 10976 !op_array->function_name) { 10977 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 10978 } else if (return_value_used != 1) { 10979 | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); 10980 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 10981 } 10982 } 10983 } else { 10984 if (op1_info & MAY_BE_REF) { 10985 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); 10986 10987 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 10988 |.cold_code 10989 |1: 10990 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 10991 | GET_ZVAL_PTR REG0, op1_addr, TMP1 10992 | // ZVAL_COPY_VALUE(return_value, &ref->value); 10993 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10994 | GC_DELREF REG0, TMP1w 10995 | beq >2 10996 | // if (IS_REFCOUNTED()) 10997 if (jit_return_label >= 0) { 10998 | IF_NOT_REFCOUNTED REG2w, =>jit_return_label, TMP1w 10999 } else { 11000 | IF_NOT_REFCOUNTED REG2w, >9, TMP1w 11001 } 11002 | // ADDREF 11003 | GET_ZVAL_PTR REG2, ret_addr, TMP1 // reload 11004 | GC_ADDREF REG2, TMP1w 11005 if (jit_return_label >= 0) { 11006 | b =>jit_return_label 11007 } else { 11008 | b >9 11009 } 11010 |2: 11011 | mov FCARG1x, REG0 11012 | EFREE_REFERENCE 11013 if (jit_return_label >= 0) { 11014 | b =>jit_return_label 11015 } else { 11016 | b >9 11017 } 11018 |.code 11019 } 11020 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11021 } 11022 11023 |9: 11024 return 1; 11025} 11026 11027static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) 11028{ 11029 ZEND_ASSERT(type_reg == ZREG_REG2); 11030 11031 | GET_ZVAL_PTR REG1, val_addr, TMP1 11032 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w 11033 | GET_LOW_8BITS TMP2w, REG2w 11034 | IF_NOT_TYPE TMP2w, IS_REFERENCE, >1 11035 | add REG1, REG1, #offsetof(zend_reference, val) 11036 | GET_Z_TYPE_INFO REG2w, REG1 11037 | GET_Z_PTR REG1, REG1 11038 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w 11039 |1: 11040 | GC_ADDREF REG1, TMP2w 11041 |2: 11042 | SET_ZVAL_PTR res_addr, REG1, TMP1 11043 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 11044 11045 return 1; 11046} 11047 11048static int zend_jit_fetch_dim_read(dasm_State **Dst, 11049 const zend_op *opline, 11050 zend_ssa *ssa, 11051 const zend_ssa_op *ssa_op, 11052 uint32_t op1_info, 11053 zend_jit_addr op1_addr, 11054 bool op1_avoid_refcounting, 11055 uint32_t op2_info, 11056 uint32_t res_info, 11057 zend_jit_addr res_addr, 11058 uint8_t dim_type) 11059{ 11060 zend_jit_addr orig_op1_addr, op2_addr; 11061 const void *exit_addr = NULL; 11062 const void *not_found_exit_addr = NULL; 11063 const void *res_exit_addr = NULL; 11064 bool result_avoid_refcounting = 0; 11065 uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; 11066 int may_throw = 0; 11067 11068 orig_op1_addr = OP1_ADDR(); 11069 op2_addr = OP2_ADDR(); 11070 11071 if (opline->opcode != ZEND_FETCH_DIM_IS 11072 && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11073 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 11074 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11075 if (!exit_addr) { 11076 return 0; 11077 } 11078 } 11079 11080 if ((res_info & MAY_BE_GUARD) 11081 && JIT_G(current_frame) 11082 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { 11083 uint32_t flags = 0; 11084 uint32_t old_op1_info = 0; 11085 uint32_t old_info; 11086 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 11087 int32_t exit_point; 11088 11089 if (opline->opcode != ZEND_FETCH_LIST_R 11090 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11091 && !op1_avoid_refcounting) { 11092 flags |= ZEND_JIT_EXIT_FREE_OP1; 11093 } 11094 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) 11095 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11096 flags |= ZEND_JIT_EXIT_FREE_OP2; 11097 } 11098 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 11099 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 11100 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 11101 && (ssa_op+1)->op1_use == ssa_op->result_def 11102 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) 11103 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 11104 result_avoid_refcounting = 1; 11105 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 11106 } 11107 11108 if (op1_avoid_refcounting) { 11109 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 11110 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 11111 } 11112 11113 if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { 11114 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11115 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 11116 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 11117 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11118 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11119 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11120 if (!res_exit_addr) { 11121 return 0; 11122 } 11123 res_info &= ~MAY_BE_GUARD; 11124 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 11125 } 11126 11127 if (opline->opcode == ZEND_FETCH_DIM_IS 11128 && !(res_info & MAY_BE_NULL)) { 11129 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11130 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); 11131 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); 11132 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11133 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11134 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11135 if (!not_found_exit_addr) { 11136 return 0; 11137 } 11138 } 11139 11140 if (op1_avoid_refcounting) { 11141 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 11142 } 11143 } 11144 11145 if (op1_info & MAY_BE_REF) { 11146 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11147 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 11148 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11149 } 11150 11151 if (op1_info & MAY_BE_ARRAY) { 11152 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11153 if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { 11154 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr, ZREG_TMP1 11155 } else { 11156 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 11157 } 11158 } 11159 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 11160 if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) || 11161 (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) { 11162 may_throw = 1; 11163 } 11164 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, dim_type, res_exit_addr, not_found_exit_addr, exit_addr)) { 11165 return 0; 11166 } 11167 } 11168 11169 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 11170 if (op1_info & MAY_BE_ARRAY) { 11171 |.cold_code 11172 |7: 11173 } 11174 11175 if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { 11176 may_throw = 1; 11177 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { 11178 if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { 11179 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr, ZREG_TMP1 11180 } else { 11181 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 11182 } 11183 } 11184 | SET_EX_OPLINE opline, REG0 11185 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 11186 if (opline->opcode != ZEND_FETCH_DIM_IS) { 11187 if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { 11188 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 11189 | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, REG0 11190 } else { 11191 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11192 | EXT_CALL zend_jit_fetch_dim_str_r_helper, REG0 11193 } 11194 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 11195 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 11196 } else { 11197 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11198 | LOAD_ZVAL_ADDR CARG3, res_addr 11199 | EXT_CALL zend_jit_fetch_dim_str_is_helper, REG0 11200 } 11201 if ((op1_info & MAY_BE_ARRAY) || 11202 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { 11203 | b >9 // END 11204 } 11205 |6: 11206 } 11207 11208 if (op1_info & MAY_BE_OBJECT) { 11209 may_throw = 1; 11210 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { 11211 if (exit_addr) { 11212 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 11213 } else { 11214 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6, ZREG_TMP1 11215 } 11216 } 11217 | SET_EX_OPLINE opline, REG0 11218 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11219 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11220 } 11221 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11222 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11223 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 11224 } else { 11225 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11226 } 11227 | LOAD_ZVAL_ADDR CARG3, res_addr 11228 if (opline->opcode != ZEND_FETCH_DIM_IS) { 11229 | EXT_CALL zend_jit_fetch_dim_obj_r_helper, REG0 11230 } else { 11231 | EXT_CALL zend_jit_fetch_dim_obj_is_helper, REG0 11232 } 11233 if ((op1_info & MAY_BE_ARRAY) || 11234 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 11235 | b >9 // END 11236 } 11237 |6: 11238 } 11239 11240 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) 11241 && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 11242 if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { 11243 | SET_EX_OPLINE opline, REG0 11244 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { 11245 may_throw = 1; 11246 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 11247 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 11248 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 11249 | EXT_CALL zend_jit_undefined_op_helper, REG0 11250 |1: 11251 } 11252 11253 if (op2_info & MAY_BE_UNDEF) { 11254 may_throw = 1; 11255 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 11256 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 11257 | EXT_CALL zend_jit_undefined_op_helper, REG0 11258 |1: 11259 } 11260 } 11261 11262 if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { 11263 may_throw = 1; 11264 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { 11265 | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr 11266 } else { 11267 | SET_EX_OPLINE opline, REG0 11268 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || 11269 Z_REG(op1_addr) != ZREG_FCARG1 || 11270 Z_OFFSET(op1_addr) != 0) { 11271 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11272 } 11273 } 11274 | EXT_CALL zend_jit_invalid_array_access, REG0 11275 } 11276 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 11277 if (op1_info & MAY_BE_ARRAY) { 11278 | b >9 // END 11279 } 11280 } 11281 11282 if (op1_info & MAY_BE_ARRAY) { 11283 |.code 11284 } 11285 } 11286 11287 if (op1_info & MAY_BE_ARRAY) { 11288 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 11289 11290 |8: 11291 if (res_exit_addr) { 11292 uint32_t type = concrete_type(res_info); 11293 if ((op1_info & MAY_BE_ARRAY_OF_REF) 11294 && dim_type != IS_UNKNOWN 11295 && dim_type != IS_REFERENCE) { 11296 if (type < IS_STRING) { 11297 | IF_NOT_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1 11298 |.cold_code 11299 |1: 11300 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, &res_exit_addr, ZREG_TMP1 11301 | GET_Z_PTR REG0, REG0 11302 | add REG0, REG0, #offsetof(zend_reference, val) 11303 | IF_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1 11304 | b &res_exit_addr 11305 |.code 11306 |1: 11307 } else { 11308 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 11309 | GET_LOW_8BITS TMP1w, REG2w 11310 | IF_NOT_TYPE TMP1w, type, >1 11311 |.cold_code 11312 |1: 11313 | IF_NOT_TYPE TMP1w, IS_REFERENCE, &res_exit_addr 11314 | GET_Z_PTR REG0, REG0 11315 | add REG0, REG0, #offsetof(zend_reference, val) 11316 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 11317 | GET_LOW_8BITS TMP1w, REG2w 11318 | IF_TYPE TMP1w, type, >1 11319 | b &res_exit_addr 11320 |.code 11321 |1: 11322 } 11323 } else { 11324 if (op1_info & MAY_BE_ARRAY_OF_REF) { 11325 | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w 11326 } 11327 if (type < IS_STRING) { 11328 | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr, ZREG_TMP1 11329 } else { 11330 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 11331 | GET_LOW_8BITS TMP1w, REG2w 11332 | IF_NOT_TYPE TMP1w, type, &res_exit_addr 11333 } 11334 } 11335 | // ZVAL_COPY 11336 |7: 11337 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 11338 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 11339 if (type < IS_STRING) { 11340 if (Z_REG(res_addr) != ZREG_FP || 11341 JIT_G(current_frame) == NULL || 11342 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 11343 | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 11344 } 11345 } else { 11346 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 11347 if (!result_avoid_refcounting) { 11348 | TRY_ADDREF res_info, REG2w, REG1, TMP1w 11349 } 11350 } 11351 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 11352 return 0; 11353 } 11354 } else if (op1_info & MAY_BE_ARRAY_OF_REF) { 11355 | // ZVAL_COPY_DEREF 11356 | GET_ZVAL_TYPE_INFO Rw(ZREG_REG2), val_addr, TMP1 11357 if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_REG2)) { 11358 return 0; 11359 } 11360 } else { 11361 | // ZVAL_COPY 11362 | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11363 | TRY_ADDREF res_info, REG1w, REG2, TMP1w 11364 } 11365 } 11366 |9: // END 11367 11368#ifdef ZEND_JIT_USE_RC_INFERENCE 11369 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 11370 /* Magic offsetGet() may increase refcount of the key */ 11371 op2_info |= MAY_BE_RCN; 11372 } 11373#endif 11374 11375 if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) { 11376 if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) { 11377 may_throw = 1; 11378 } 11379 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11380 } 11381 if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { 11382 if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) { 11383 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 11384 may_throw = 1; 11385 } 11386 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11387 } 11388 } 11389 11390 if (may_throw) { 11391 if (!zend_jit_check_exception(Dst)) { 11392 return 0; 11393 } 11394 } 11395 11396 return 1; 11397} 11398 11399static int zend_jit_fetch_dim(dasm_State **Dst, 11400 const zend_op *opline, 11401 uint32_t op1_info, 11402 zend_jit_addr op1_addr, 11403 uint32_t op2_info, 11404 zend_jit_addr res_addr, 11405 uint8_t dim_type) 11406{ 11407 zend_jit_addr op2_addr; 11408 int may_throw = 0; 11409 11410 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 11411 11412 if (opline->opcode == ZEND_FETCH_DIM_RW) { 11413 | SET_EX_OPLINE opline, REG0 11414 } 11415 if (op1_info & MAY_BE_REF) { 11416 may_throw = 1; 11417 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11418 | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w 11419 | GET_Z_PTR FCARG2x, FCARG1x 11420 | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] 11421 | cmp TMP1w, #IS_ARRAY 11422 | bne >2 11423 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) 11424 | b >3 11425 |.cold_code 11426 |2: 11427 | SET_EX_OPLINE opline, REG0 11428 if (opline->opcode != ZEND_FETCH_DIM_RW) { 11429 | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 11430 } 11431 | mov FCARG1x, RETVALx 11432 | cbnz FCARG1x, >1 11433 | b ->exception_handler_undef 11434 |.code 11435 |1: 11436 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11437 } 11438 11439 if (op1_info & MAY_BE_ARRAY) { 11440 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11441 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 11442 } 11443 |3: 11444 | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 11445 } 11446 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 11447 if (op1_info & MAY_BE_ARRAY) { 11448 |.cold_code 11449 |7: 11450 } 11451 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 11452 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 11453 | bgt >7 11454 } 11455 if (Z_REG(op1_addr) != ZREG_FP) { 11456 | str Rx(Z_REG(op1_addr)), T1 // save 11457 } 11458 if ((op1_info & MAY_BE_UNDEF) 11459 && opline->opcode == ZEND_FETCH_DIM_RW) { 11460 may_throw = 1; 11461 if (op1_info & MAY_BE_NULL) { 11462 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 11463 } 11464 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 11465 | EXT_CALL zend_jit_undefined_op_helper, REG0 11466 |1: 11467 } 11468 | // ZVAL_ARR(container, zend_new_array(8)); 11469 | EXT_CALL _zend_new_array_0, REG0 11470 | mov REG0, RETVALx 11471 if (Z_REG(op1_addr) != ZREG_FP) { 11472 | ldr Rx(Z_REG(op1_addr)), T1 // restore 11473 } 11474 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 11475 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 11476 | mov FCARG1x, REG0 11477 if (op1_info & MAY_BE_ARRAY) { 11478 | b >1 11479 |.code 11480 |1: 11481 } 11482 } 11483 11484 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 11485 |6: 11486 if (opline->op2_type == IS_UNUSED) { 11487 may_throw = 1; 11488 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 11489 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 11490 | EXT_CALL zend_hash_next_index_insert, REG0 11491 | // if (UNEXPECTED(!var_ptr)) { 11492 | cbz RETVALx, >1 11493 |.cold_code 11494 |1: 11495 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 11496 | CANNOT_ADD_ELEMENT opline 11497 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 11498 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 11499 | b >8 11500 |.code 11501 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 11502 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 11503 } else { 11504 uint32_t type; 11505 11506 switch (opline->opcode) { 11507 case ZEND_FETCH_DIM_W: 11508 case ZEND_FETCH_LIST_W: 11509 type = BP_VAR_W; 11510 break; 11511 case ZEND_FETCH_DIM_RW: 11512 may_throw = 1; 11513 type = BP_VAR_RW; 11514 break; 11515 case ZEND_FETCH_DIM_UNSET: 11516 type = BP_VAR_UNSET; 11517 break; 11518 default: 11519 ZEND_UNREACHABLE(); 11520 } 11521 11522 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 11523 may_throw = 1; 11524 } 11525 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 11526 return 0; 11527 } 11528 11529 |8: 11530 | SET_ZVAL_PTR res_addr, REG0, TMP1 11531 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 11532 11533 if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 11534 |.cold_code 11535 |9: 11536 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 11537 | b >8 11538 |.code 11539 } 11540 } 11541 } 11542 11543 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 11544 may_throw = 1; 11545 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 11546 |.cold_code 11547 |7: 11548 } 11549 11550 if (opline->opcode != ZEND_FETCH_DIM_RW) { 11551 | SET_EX_OPLINE opline, REG0 11552 } 11553 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11554 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11555 } 11556 if (opline->op2_type == IS_UNUSED) { 11557 | mov FCARG2x, xzr 11558 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11559 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11560 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 11561 } else { 11562 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11563 } 11564 | LOAD_ZVAL_ADDR CARG3, res_addr 11565 switch (opline->opcode) { 11566 case ZEND_FETCH_DIM_W: 11567 case ZEND_FETCH_LIST_W: 11568 | EXT_CALL zend_jit_fetch_dim_obj_w_helper, REG0 11569 break; 11570 case ZEND_FETCH_DIM_RW: 11571 | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, REG0 11572 break; 11573// case ZEND_FETCH_DIM_UNSET: 11574// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, REG0 11575// break; 11576 default: 11577 ZEND_UNREACHABLE(); 11578 } 11579 11580 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 11581 | b >8 // END 11582 |.code 11583 } 11584 } 11585 11586#ifdef ZEND_JIT_USE_RC_INFERENCE 11587 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 11588 /* ASSIGN_DIM may increase refcount of the key */ 11589 op2_info |= MAY_BE_RCN; 11590 } 11591#endif 11592 11593 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 11594 && (op2_info & MAY_HAVE_DTOR) 11595 && (op2_info & MAY_BE_RC1)) { 11596 may_throw = 1; 11597 } 11598 |8: 11599 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11600 11601 if (may_throw) { 11602 if (!zend_jit_check_exception(Dst)) { 11603 return 0; 11604 } 11605 } 11606 return 1; 11607} 11608 11609static int zend_jit_isset_isempty_dim(dasm_State **Dst, 11610 const zend_op *opline, 11611 uint32_t op1_info, 11612 zend_jit_addr op1_addr, 11613 bool op1_avoid_refcounting, 11614 uint32_t op2_info, 11615 uint8_t dim_type, 11616 int may_throw, 11617 zend_uchar smart_branch_opcode, 11618 uint32_t target_label, 11619 uint32_t target_label2, 11620 const void *exit_addr) 11621{ 11622 zend_jit_addr op2_addr, res_addr; 11623 11624 // TODO: support for empty() ??? 11625 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 11626 11627 op2_addr = OP2_ADDR(); 11628 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11629 11630 if (op1_info & MAY_BE_REF) { 11631 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11632 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 11633 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11634 } 11635 11636 if (op1_info & MAY_BE_ARRAY) { 11637 const void *found_exit_addr = NULL; 11638 const void *not_found_exit_addr = NULL; 11639 11640 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11641 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 11642 } 11643 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 11644 if (exit_addr 11645 && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) 11646 && !may_throw 11647 && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) 11648 && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { 11649 if (smart_branch_opcode == ZEND_JMPNZ) { 11650 found_exit_addr = exit_addr; 11651 } else { 11652 not_found_exit_addr = exit_addr; 11653 } 11654 } 11655 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, dim_type, found_exit_addr, not_found_exit_addr, NULL)) { 11656 return 0; 11657 } 11658 11659 if (found_exit_addr) { 11660 |9: 11661 return 1; 11662 } else if (not_found_exit_addr) { 11663 |8: 11664 return 1; 11665 } 11666 } 11667 11668 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 11669 if (op1_info & MAY_BE_ARRAY) { 11670 |.cold_code 11671 |7: 11672 } 11673 11674 if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { 11675 | SET_EX_OPLINE opline, REG0 11676 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11677 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11678 } 11679 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11680 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11681 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 11682 } else { 11683 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11684 } 11685 | EXT_CALL zend_jit_isset_dim_helper, REG0 11686 | cbz RETVALw, >9 11687 if (op1_info & MAY_BE_ARRAY) { 11688 | b >8 11689 |.code 11690 } 11691 } else { 11692 if (op2_info & MAY_BE_UNDEF) { 11693 if (op2_info & MAY_BE_ANY) { 11694 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 11695 } 11696 | SET_EX_OPLINE opline, REG0 11697 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 11698 | EXT_CALL zend_jit_undefined_op_helper, REG0 11699 |1: 11700 } 11701 if (op1_info & MAY_BE_ARRAY) { 11702 | b >9 11703 |.code 11704 } 11705 } 11706 } 11707 11708#ifdef ZEND_JIT_USE_RC_INFERENCE 11709 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 11710 /* Magic offsetExists() may increase refcount of the key */ 11711 op2_info |= MAY_BE_RCN; 11712 } 11713#endif 11714 11715 if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { 11716 |8: 11717 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11718 if (!op1_avoid_refcounting) { 11719 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11720 } 11721 if (may_throw) { 11722 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 11723 return 0; 11724 } 11725 } 11726 if (!(opline->extended_value & ZEND_ISEMPTY)) { 11727 if (exit_addr) { 11728 if (smart_branch_opcode == ZEND_JMPNZ) { 11729 | b &exit_addr 11730 } else { 11731 | b >8 11732 } 11733 } else if (smart_branch_opcode) { 11734 if (smart_branch_opcode == ZEND_JMPZ) { 11735 | b =>target_label2 11736 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11737 | b =>target_label 11738 } else { 11739 ZEND_UNREACHABLE(); 11740 } 11741 } else { 11742 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 11743 | b >8 11744 } 11745 } else { 11746 | NIY // TODO: support for empty() 11747 } 11748 } 11749 11750 |9: // not found 11751 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11752 if (!op1_avoid_refcounting) { 11753 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11754 } 11755 if (may_throw) { 11756 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 11757 return 0; 11758 } 11759 } 11760 if (!(opline->extended_value & ZEND_ISEMPTY)) { 11761 if (exit_addr) { 11762 if (smart_branch_opcode == ZEND_JMPZ) { 11763 | b &exit_addr 11764 } 11765 } else if (smart_branch_opcode) { 11766 if (smart_branch_opcode == ZEND_JMPZ) { 11767 | b =>target_label 11768 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11769 } else { 11770 ZEND_UNREACHABLE(); 11771 } 11772 } else { 11773 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 11774 } 11775 } else { 11776 | NIY // TODO: support for empty() 11777 } 11778 11779 |8: 11780 11781 return 1; 11782} 11783 11784static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 11785{ 11786 zend_jit_addr op1_addr = OP1_ADDR(); 11787 zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); 11788 11789 | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; 11790 | ldr FCARG2x, EX->run_time_cache 11791 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG2x, opline->extended_value, TMP1 11792 | sub REG0, REG0, #1 11793 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) 11794 | MEM_LOAD_32_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1 11795 | cmp REG0, REG1, lsl #5 11796 | bhs >9 11797 | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); 11798 | MEM_LOAD_64_ZTS ldr, TMP1, executor_globals, symbol_table.arData, REG1 11799 | add REG0, REG0, TMP1 11800 | IF_NOT_Z_TYPE REG0, IS_REFERENCE, >9, TMP1w 11801 | // (EXPECTED(p->key == varname)) 11802 | ldr TMP1, [REG0, #offsetof(Bucket, key)] 11803 | LOAD_ADDR TMP2, varname 11804 | cmp TMP1, TMP2 11805 | bne >9 11806 | GET_Z_PTR REG0, REG0 11807 | GC_ADDREF REG0, TMP1w 11808 |1: 11809 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11810 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11811 | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) 11812 | IF_ZVAL_REFCOUNTED op1_addr, >2, ZREG_TMP1, ZREG_TMP2 11813 |.cold_code 11814 |2: 11815 } 11816 | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); 11817 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 11818 | // ZVAL_REF(variable_ptr, ref) 11819 | SET_ZVAL_PTR op1_addr, REG0, TMP1 11820 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 11821 | // if (GC_DELREF(garbage) == 0) 11822 | GC_DELREF FCARG1x, TMP1w 11823 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 11824 | bne >3 11825 } else { 11826 | bne >5 11827 } 11828 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 11829 | b >5 11830 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 11831 |3: 11832 | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) 11833 | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w 11834 | EXT_CALL gc_possible_root, REG0 11835 | b >5 11836 } 11837 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11838 |.code 11839 } 11840 } 11841 11842 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11843 | // ZVAL_REF(variable_ptr, ref) 11844 | SET_ZVAL_PTR op1_addr, REG0, TMP1 11845 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 11846 } 11847 |5: 11848 //END of handler 11849 11850 |.cold_code 11851 |9: 11852 | LOAD_ADDR FCARG1x, (ptrdiff_t)varname 11853 if (opline->extended_value) { 11854 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, FCARG2x, opline->extended_value, TMP1 11855 } 11856 | EXT_CALL zend_jit_fetch_global_helper, REG0 11857 | mov REG0, RETVALx 11858 | b <1 11859 |.code 11860 11861 return 1; 11862} 11863 11864static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception) 11865{ 11866 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11867 bool in_cold = 0; 11868 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 11869 zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1 : ZREG_REG0; 11870 11871 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 11872 && JIT_G(current_frame) 11873 && JIT_G(current_frame)->prev) { 11874 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 11875 uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)); 11876 11877 if (type != IS_UNKNOWN && (type_mask & (1u << type))) { 11878 return 1; 11879 } 11880 } 11881 11882 if (ZEND_ARG_SEND_MODE(arg_info)) { 11883 if (opline->opcode == ZEND_RECV_INIT) { 11884 | LOAD_ZVAL_ADDR Rx(tmp_reg), res_addr 11885 | ZVAL_DEREF Rx(tmp_reg), MAY_BE_REF, TMP1w 11886 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); 11887 } else { 11888 | GET_ZVAL_PTR Rx(tmp_reg), res_addr, TMP1 11889 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); 11890 } 11891 } 11892 11893 if (type_mask != 0) { 11894 if (is_power_of_two(type_mask)) { 11895 uint32_t type_code = concrete_type(type_mask); 11896 | IF_NOT_ZVAL_TYPE res_addr, type_code, >1, ZREG_TMP1 11897 } else { 11898 | mov REG2w, #1 11899 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, Rx(Z_REG(res_addr)), Z_OFFSET(res_addr)+offsetof(zval, u1.v.type), TMP1 11900 | lsl REG2w, REG2w, REG1w 11901 | TST_32_WITH_CONST REG2w, type_mask, TMP1w 11902 | beq >1 11903 } 11904 11905 |.cold_code 11906 |1: 11907 11908 in_cold = 1; 11909 } 11910 11911 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 11912 | LOAD_ZVAL_ADDR FCARG1x, res_addr 11913 } 11914 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11915 | SET_EX_OPLINE opline, REG0 11916 } else { 11917 | ADDR_STORE EX->opline, opline, REG0 11918 } 11919 | LOAD_ADDR FCARG2x, (ptrdiff_t)arg_info 11920 | EXT_CALL zend_jit_verify_arg_slow, REG0 11921 11922 if (check_exception) { 11923 | GET_LOW_8BITS REG0w, RETVALw 11924 if (in_cold) { 11925 | cbnz REG0w, >1 11926 | b ->exception_handler 11927 |.code 11928 |1: 11929 } else { 11930 | cbz REG0w, ->exception_handler 11931 } 11932 } else if (in_cold) { 11933 | b >1 11934 |.code 11935 |1: 11936 } 11937 11938 return 1; 11939} 11940 11941static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) 11942{ 11943 uint32_t arg_num = opline->op1.num; 11944 zend_arg_info *arg_info = NULL; 11945 11946 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 11947 if (EXPECTED(arg_num <= op_array->num_args)) { 11948 arg_info = &op_array->arg_info[arg_num-1]; 11949 } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { 11950 arg_info = &op_array->arg_info[op_array->num_args]; 11951 } 11952 if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { 11953 arg_info = NULL; 11954 } 11955 } 11956 11957 if (arg_info || (opline+1)->opcode != ZEND_RECV) { 11958 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11959 if (!JIT_G(current_frame) || 11960 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 || 11961 arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 11962 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 11963 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11964 11965 if (!exit_addr) { 11966 return 0; 11967 } 11968 | ldr TMP1w, EX->This.u2.num_args 11969 | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w 11970 | blo &exit_addr 11971 } 11972 } else { 11973 | ldr TMP1w, EX->This.u2.num_args 11974 | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w 11975 | blo >1 11976 |.cold_code 11977 |1: 11978 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11979 | SET_EX_OPLINE opline, REG0 11980 } else { 11981 | ADDR_STORE EX->opline, opline, REG0 11982 } 11983 | mov FCARG1x, FP 11984 | EXT_CALL zend_missing_arg_error, REG0 11985 | b ->exception_handler 11986 |.code 11987 } 11988 } 11989 11990 if (arg_info) { 11991 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { 11992 return 0; 11993 } 11994 } 11995 11996 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 11997 if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { 11998 | LOAD_IP_ADDR (opline + 1) 11999 zend_jit_set_last_valid_opline(opline + 1); 12000 } 12001 } 12002 12003 return 1; 12004} 12005 12006static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw) 12007{ 12008 uint32_t arg_num = opline->op1.num; 12009 zval *zv = RT_CONSTANT(opline, opline->op2); 12010 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12011 12012 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12013 && JIT_G(current_frame) 12014 && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) { 12015 if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 12016 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 12017 if (Z_REFCOUNTED_P(zv)) { 12018 | ADDREF_CONST zv, REG0, TMP1 12019 } 12020 } 12021 } else { 12022 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 12023 (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 12024 | ldr TMP1w, EX->This.u2.num_args 12025 | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w 12026 | bhs >5 12027 } 12028 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 12029 if (Z_REFCOUNTED_P(zv)) { 12030 | ADDREF_CONST zv, REG0, TMP1 12031 } 12032 } 12033 12034 if (Z_CONSTANT_P(zv)) { 12035 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12036 | SET_EX_OPLINE opline, REG0 12037 } else { 12038 | ADDR_STORE EX->opline, opline, REG0 12039 } 12040 | LOAD_ZVAL_ADDR FCARG1x, res_addr 12041 | ldr REG0, EX->func 12042 | ldr FCARG2x, [REG0, #offsetof(zend_op_array, scope)] 12043 | EXT_CALL zval_update_constant_ex, REG0 12044 | cbnz RETVALw, >1 12045 |.cold_code 12046 |1: 12047 | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline, ZREG_TMP1, ZREG_TMP2 12048 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 12049 | b ->exception_handler 12050 |.code 12051 } 12052 12053 |5: 12054 12055 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12056 do { 12057 zend_arg_info *arg_info; 12058 12059 if (arg_num <= op_array->num_args) { 12060 arg_info = &op_array->arg_info[arg_num-1]; 12061 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { 12062 arg_info = &op_array->arg_info[op_array->num_args]; 12063 } else { 12064 break; 12065 } 12066 if (!ZEND_TYPE_IS_SET(arg_info->type)) { 12067 break; 12068 } 12069 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { 12070 return 0; 12071 } 12072 } while (0); 12073 } 12074 12075 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12076 if (is_last) { 12077 | LOAD_IP_ADDR (opline + 1) 12078 zend_jit_set_last_valid_opline(opline + 1); 12079 } 12080 } 12081 return 1; 12082} 12083 12084static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) 12085{ 12086 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 12087 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12088 12089 if (!exit_addr) { 12090 return 0; 12091 } 12092 12093 | LOAD_ADDR TMP1, ((ptrdiff_t)ce) 12094 | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] 12095 | cmp TMP2, TMP1 12096 | bne &exit_addr 12097 12098 return 1; 12099} 12100 12101static int zend_jit_fetch_obj(dasm_State **Dst, 12102 const zend_op *opline, 12103 const zend_op_array *op_array, 12104 zend_ssa *ssa, 12105 const zend_ssa_op *ssa_op, 12106 uint32_t op1_info, 12107 zend_jit_addr op1_addr, 12108 bool op1_indirect, 12109 zend_class_entry *ce, 12110 bool ce_is_instanceof, 12111 bool on_this, 12112 bool delayed_fetch_this, 12113 bool op1_avoid_refcounting, 12114 zend_class_entry *trace_ce, 12115 uint8_t prop_type, 12116 int may_throw) 12117{ 12118 zval *member; 12119 zend_property_info *prop_info; 12120 bool may_be_dynamic = 1; 12121 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12122 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 12123 zend_jit_addr prop_addr; 12124 uint32_t res_info = RES_INFO(); 12125 bool type_loaded = 0; 12126 12127 ZEND_ASSERT(opline->op2_type == IS_CONST); 12128 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 12129 12130 member = RT_CONSTANT(opline, opline->op2); 12131 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 12132 prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); 12133 12134 if (on_this) { 12135 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 12136 } else { 12137 if (opline->op1_type == IS_VAR 12138 && opline->opcode == ZEND_FETCH_OBJ_W 12139 && (op1_info & MAY_BE_INDIRECT) 12140 && Z_REG(op1_addr) == ZREG_FP) { 12141 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12142 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 12143 | GET_Z_PTR FCARG1x, FCARG1x 12144 |1: 12145 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12146 } 12147 if (op1_info & MAY_BE_REF) { 12148 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12149 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12150 } 12151 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 12152 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12153 } 12154 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 12155 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12156 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12157 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12158 12159 if (!exit_addr) { 12160 return 0; 12161 } 12162 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 12163 } else { 12164 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, ZREG_TMP1 12165 } 12166 } 12167 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 12168 } 12169 12170 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12171 prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename); 12172 if (prop_info) { 12173 ce = trace_ce; 12174 ce_is_instanceof = 0; 12175 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 12176 if (on_this && JIT_G(current_frame) 12177 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 12178 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 12179 } else if (zend_jit_class_guard(Dst, opline, ce)) { 12180 if (on_this && JIT_G(current_frame)) { 12181 JIT_G(current_frame)->ce = ce; 12182 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 12183 } 12184 } else { 12185 return 0; 12186 } 12187 if (ssa->var_info && ssa_op->op1_use >= 0) { 12188 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 12189 ssa->var_info[ssa_op->op1_use].ce = ce; 12190 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 12191 } 12192 } 12193 } 12194 } 12195 12196 if (!prop_info) { 12197 | ldr REG0, EX->run_time_cache 12198 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS), TMP1 12199 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 12200 | cmp REG2, TMP1 12201 | bne >5 12202 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)), TMP1 12203 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array); 12204 if (may_be_dynamic) { 12205 | tst REG0, REG0 12206 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12207 | blt >5 12208 } else { 12209 | blt >8 // dynamic property 12210 } 12211 } 12212 | add TMP1, FCARG1x, REG0 12213 | ldr REG2w, [TMP1, #offsetof(zval,u1.type_info)] 12214 | IF_UNDEF REG2w, >5 12215 | mov FCARG1x, TMP1 12216 type_loaded = 1; 12217 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12218 if (opline->opcode == ZEND_FETCH_OBJ_W 12219 && (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) { 12220 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 12221 12222 | ldr REG0, EX->run_time_cache 12223 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2), TMP1 12224 | cbnz FCARG2x, >1 12225 |.cold_code 12226 |1: 12227 | ldr TMP1w, [FCARG2x, #offsetof(zend_property_info, flags)] 12228 | tst TMP1w, #ZEND_ACC_READONLY 12229 if (flags) { 12230 | beq >3 12231 } else { 12232 | beq >4 12233 } 12234 | IF_NOT_TYPE REG2w, IS_OBJECT_EX, >2 12235 | GET_Z_PTR REG2, FCARG1x 12236 | GC_ADDREF REG2, TMP1w 12237 | SET_ZVAL_PTR res_addr, REG2, TMP1 12238 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2 12239 | b >9 12240 |2: 12241 | mov FCARG1x, FCARG2x 12242 | SET_EX_OPLINE opline, REG0 12243 | EXT_CALL zend_readonly_property_modification_error, REG0 12244 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 12245 | b >9 12246 |3: 12247 if (flags == ZEND_FETCH_DIM_WRITE) { 12248 | SET_EX_OPLINE opline, REG0 12249 | EXT_CALL zend_jit_check_array_promotion, REG0 12250 | b >9 12251 } else if (flags == ZEND_FETCH_REF) { 12252 | LOAD_ZVAL_ADDR CARG3, res_addr 12253 | EXT_CALL zend_jit_create_typed_ref, REG0 12254 | b >9 12255 } else { 12256 ZEND_ASSERT(flags == 0); 12257 } 12258 |.code 12259 |4: 12260 } 12261 } else { 12262 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 12263 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12264 if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { 12265 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 12266 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12267 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12268 12269 if (!exit_addr) { 12270 return 0; 12271 } 12272 type_loaded = 1; 12273 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12274 | IF_UNDEF REG2w, &exit_addr 12275 } 12276 } else { 12277 type_loaded = 1; 12278 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12279 | IF_UNDEF REG2w, >5 12280 } 12281 if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) { 12282 if (!type_loaded) { 12283 type_loaded = 1; 12284 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12285 } 12286 | IF_NOT_TYPE REG2w, IS_OBJECT_EX, >4 12287 | GET_ZVAL_PTR REG2, prop_addr, TMP1 12288 | GC_ADDREF REG2, TMP1w 12289 | SET_ZVAL_PTR res_addr, REG2, TMP1 12290 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2 12291 | b >9 12292 |.cold_code 12293 |4: 12294 | LOAD_ADDR FCARG1x, prop_info 12295 | SET_EX_OPLINE opline, REG0 12296 | EXT_CALL zend_readonly_property_modification_error, REG0 12297 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 12298 | b >9 12299 |.code 12300 } 12301 if (opline->opcode == ZEND_FETCH_OBJ_W 12302 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) 12303 && ZEND_TYPE_IS_SET(prop_info->type)) { 12304 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 12305 12306 if (flags == ZEND_FETCH_DIM_WRITE) { 12307 if ((ZEND_TYPE_FULL_MASK(prop_info->type) & MAY_BE_ARRAY) == 0) { 12308 if (!type_loaded) { 12309 type_loaded = 1; 12310 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12311 } 12312 | cmp REG2w, #IS_FALSE 12313 | ble >1 12314 |.cold_code 12315 |1: 12316 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12317 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12318 } 12319 | LOAD_ADDR FCARG2x, prop_info 12320 | SET_EX_OPLINE opline, REG0 12321 | EXT_CALL zend_jit_check_array_promotion, REG0 12322 | b >9 12323 |.code 12324 } 12325 } else if (flags == ZEND_FETCH_REF) { 12326 if (!type_loaded) { 12327 type_loaded = 1; 12328 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12329 } 12330 | GET_LOW_8BITS TMP1w, REG2w 12331 | IF_TYPE TMP1w, IS_REFERENCE, >1 12332 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 12333 | LOAD_ADDR FCARG2x, prop_info 12334 } else { 12335 int prop_info_offset = 12336 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 12337 12338 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 12339 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 12340 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 12341 } 12342 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12343 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12344 } 12345 | LOAD_ZVAL_ADDR CARG3, res_addr 12346 | EXT_CALL zend_jit_create_typed_ref, REG0 12347 | b >9 12348 |1: 12349 } else { 12350 ZEND_UNREACHABLE(); 12351 } 12352 } 12353 } 12354 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12355 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12356 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12357 } 12358 | SET_ZVAL_PTR res_addr, FCARG1x, TMP1 12359 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 12360 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { 12361 ssa->var_info[ssa_op->result_def].indirect_reference = 1; 12362 } 12363 } else { 12364 bool result_avoid_refcounting = 0; 12365 12366 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { 12367 uint32_t flags = 0; 12368 uint32_t old_info; 12369 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 12370 int32_t exit_point; 12371 const void *exit_addr; 12372 uint32_t type; 12373 zend_jit_addr val_addr = prop_addr; 12374 12375 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 12376 && !delayed_fetch_this 12377 && !op1_avoid_refcounting) { 12378 flags = ZEND_JIT_EXIT_FREE_OP1; 12379 } 12380 12381 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 12382 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 12383 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 12384 && (ssa_op+1)->op1_use == ssa_op->result_def 12385 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 12386 result_avoid_refcounting = 1; 12387 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 12388 } 12389 12390 type = concrete_type(res_info); 12391 12392 if (prop_type != IS_UNKNOWN 12393 && prop_type != IS_UNDEF 12394 && prop_type != IS_REFERENCE 12395 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) { 12396 exit_point = zend_jit_trace_get_exit_point(opline, 0); 12397 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12398 if (!exit_addr) { 12399 return 0; 12400 } 12401 } else { 12402 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 12403 | LOAD_ZVAL_ADDR REG0, prop_addr 12404 if (op1_avoid_refcounting) { 12405 SET_STACK_REG(JIT_G(current_frame)->stack, 12406 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 12407 } 12408 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 12409 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 12410 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 12411 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 12412 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 12413 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12414 if (!exit_addr) { 12415 return 0; 12416 } 12417 12418 if (!type_loaded) { 12419 type_loaded = 1; 12420 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12421 } 12422 | // ZVAL_DEREF() 12423 | GET_LOW_8BITS TMP1w, REG2w 12424 | IF_NOT_TYPE TMP1w, IS_REFERENCE, >1 12425 | GET_Z_PTR REG0, REG0 12426 | add REG0, REG0, #offsetof(zend_reference, val) 12427 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 12428 } 12429 res_info &= ~MAY_BE_GUARD; 12430 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 12431 if (type < IS_STRING) { 12432 |1: 12433 if (type_loaded) { 12434 | IF_NOT_TYPE REG2w, type, &exit_addr 12435 } else { 12436 | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr, ZREG_TMP1 12437 } 12438 } else { 12439 if (!type_loaded) { 12440 type_loaded = 1; 12441 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 12442 } 12443 |1: 12444 | GET_LOW_8BITS TMP1w, REG2w 12445 | IF_NOT_TYPE TMP1w, type, &exit_addr 12446 } 12447 | // ZVAL_COPY 12448 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 12449 if (type < IS_STRING) { 12450 if (Z_REG(res_addr) != ZREG_FP || 12451 JIT_G(current_frame) == NULL || 12452 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 12453 | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 12454 } 12455 } else { 12456 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 12457 if (!result_avoid_refcounting) { 12458 | TRY_ADDREF res_info, REG2w, REG1, TMP1w 12459 } 12460 } 12461 } else { 12462 if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_REG2)) { 12463 return 0; 12464 } 12465 } 12466 } 12467 12468 if (op1_avoid_refcounting) { 12469 SET_STACK_REG(JIT_G(current_frame)->stack, 12470 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 12471 } 12472 12473 |.cold_code 12474 12475 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { 12476 |5: 12477 | SET_EX_OPLINE opline, REG0 12478 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12479 | EXT_CALL zend_jit_fetch_obj_w_slow, REG0 12480 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { 12481 | EXT_CALL zend_jit_fetch_obj_r_slow, REG0 12482 } else { 12483 | EXT_CALL zend_jit_fetch_obj_is_slow, REG0 12484 } 12485 | b >9 12486 } 12487 12488 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12489 |7: 12490 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 12491 | SET_EX_OPLINE opline, REG0 12492 if (opline->opcode != ZEND_FETCH_OBJ_W 12493 && (op1_info & MAY_BE_UNDEF)) { 12494 zend_jit_addr orig_op1_addr = OP1_ADDR(); 12495 12496 if (op1_info & MAY_BE_ANY) { 12497 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 12498 } 12499 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 12500 | EXT_CALL zend_jit_undefined_op_helper, REG0 12501 |1: 12502 | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr 12503 } else if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12504 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12505 } 12506 | LOAD_ADDR FCARG2x, Z_STRVAL_P(member) 12507 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12508 | EXT_CALL zend_jit_invalid_property_write, REG0 12509 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 12510 } else { 12511 | EXT_CALL zend_jit_invalid_property_read, REG0 12512 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 12513 } 12514 | b >9 12515 } else { 12516 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 12517 | b >9 12518 } 12519 } 12520 12521 if (!prop_info 12522 && may_be_dynamic 12523 && opline->opcode != ZEND_FETCH_OBJ_W) { 12524 |8: 12525 | mov FCARG2x, REG0 12526 | SET_EX_OPLINE opline, REG0 12527 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 12528 | EXT_CALL zend_jit_fetch_obj_r_dynamic, REG0 12529 } else { 12530 | EXT_CALL zend_jit_fetch_obj_is_dynamic, REG0 12531 } 12532 | b >9 12533 } 12534 12535 |.code; 12536 |9: // END 12537 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 12538 if (opline->op1_type == IS_VAR 12539 && opline->opcode == ZEND_FETCH_OBJ_W 12540 && (op1_info & MAY_BE_RC1)) { 12541 zend_jit_addr orig_op1_addr = OP1_ADDR(); 12542 12543 | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1, ZREG_TMP1, ZREG_TMP2 12544 | GET_ZVAL_PTR FCARG1x, orig_op1_addr, TMP1 12545 | GC_DELREF FCARG1x, TMP1w 12546 | bne >1 12547 | SET_EX_OPLINE opline, REG0 12548 | EXT_CALL zend_jit_extract_helper, REG0 12549 |1: 12550 } else if (!op1_avoid_refcounting) { 12551 if (on_this) { 12552 op1_info &= ~MAY_BE_RC1; 12553 } 12554 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 12555 } 12556 } 12557 12558 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12559 && prop_info 12560 && (opline->opcode != ZEND_FETCH_OBJ_W || 12561 !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) || 12562 !ZEND_TYPE_IS_SET(prop_info->type)) 12563 && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) { 12564 may_throw = 0; 12565 } 12566 12567 if (may_throw) { 12568 if (!zend_jit_check_exception(Dst)) { 12569 return 0; 12570 } 12571 } 12572 12573 return 1; 12574} 12575 12576static int zend_jit_incdec_obj(dasm_State **Dst, 12577 const zend_op *opline, 12578 const zend_op_array *op_array, 12579 zend_ssa *ssa, 12580 const zend_ssa_op *ssa_op, 12581 uint32_t op1_info, 12582 zend_jit_addr op1_addr, 12583 bool op1_indirect, 12584 zend_class_entry *ce, 12585 bool ce_is_instanceof, 12586 bool on_this, 12587 bool delayed_fetch_this, 12588 zend_class_entry *trace_ce, 12589 uint8_t prop_type) 12590{ 12591 zval *member; 12592 zend_string *name; 12593 zend_property_info *prop_info; 12594 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 12595 zend_jit_addr res_addr = 0; 12596 zend_jit_addr prop_addr; 12597 bool needs_slow_path = 0; 12598 bool use_prop_guard = 0; 12599 bool may_throw = 0; 12600 uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0; 12601 12602 ZEND_ASSERT(opline->op2_type == IS_CONST); 12603 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 12604 12605 if (opline->result_type != IS_UNUSED) { 12606 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12607 } 12608 12609 member = RT_CONSTANT(opline, opline->op2); 12610 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 12611 name = Z_STR_P(member); 12612 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 12613 12614 if (on_this) { 12615 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 12616 } else { 12617 if (opline->op1_type == IS_VAR 12618 && (op1_info & MAY_BE_INDIRECT) 12619 && Z_REG(op1_addr) == ZREG_FP) { 12620 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12621 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 12622 | GET_Z_PTR FCARG1x, FCARG1x 12623 |1: 12624 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12625 } 12626 if (op1_info & MAY_BE_REF) { 12627 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12628 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12629 } 12630 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 12631 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12632 } 12633 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 12634 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12635 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12636 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12637 12638 if (!exit_addr) { 12639 return 0; 12640 } 12641 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 12642 } else { 12643 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 12644 |.cold_code 12645 |1: 12646 | SET_EX_OPLINE opline, REG0 12647 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12648 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12649 } 12650 | LOAD_ADDR FCARG2x, ZSTR_VAL(name) 12651 | EXT_CALL zend_jit_invalid_property_incdec, REG0 12652 | b ->exception_handler 12653 |.code 12654 } 12655 } 12656 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 12657 } 12658 12659 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12660 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 12661 if (prop_info) { 12662 ce = trace_ce; 12663 ce_is_instanceof = 0; 12664 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 12665 if (on_this && JIT_G(current_frame) 12666 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 12667 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 12668 } else if (zend_jit_class_guard(Dst, opline, ce)) { 12669 if (on_this && JIT_G(current_frame)) { 12670 JIT_G(current_frame)->ce = ce; 12671 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 12672 } 12673 } else { 12674 return 0; 12675 } 12676 if (ssa->var_info && ssa_op->op1_use >= 0) { 12677 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 12678 ssa->var_info[ssa_op->op1_use].ce = ce; 12679 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 12680 } 12681 if (ssa->var_info && ssa_op->op1_def >= 0) { 12682 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 12683 ssa->var_info[ssa_op->op1_def].ce = ce; 12684 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 12685 } 12686 } 12687 } 12688 } 12689 12690 use_prop_guard = (prop_type != IS_UNKNOWN 12691 && prop_type != IS_UNDEF 12692 && prop_type != IS_REFERENCE 12693 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 12694 12695 if (!prop_info) { 12696 needs_slow_path = 1; 12697 12698 | ldr REG0, EX->run_time_cache 12699 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 12700 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 12701 | cmp REG2, TMP1 12702 | bne >7 12703 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 12704 | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 12705 | cbnz TMP1, >7 12706 } 12707 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 12708 | tst REG0, REG0 12709 | blt >7 12710 | add TMP1, FCARG1x, REG0 12711 if (!use_prop_guard) { 12712 | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] 12713 | IF_TYPE TMP2w , IS_UNDEF, >7 12714 } 12715 | mov FCARG1x, TMP1 12716 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12717 } else { 12718 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 12719 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 12720 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12721 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12722 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12723 12724 if (!exit_addr) { 12725 return 0; 12726 } 12727 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 12728 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr 12729 } else { 12730 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 12731 | IF_TYPE TMP2w, IS_UNDEF, >7 12732 needs_slow_path = 1; 12733 } 12734 } 12735 if (ZEND_TYPE_IS_SET(prop_info->type)) { 12736 may_throw = 1; 12737 | SET_EX_OPLINE opline, REG0 12738 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 12739 | LOAD_ADDR FCARG2x, prop_info 12740 } else { 12741 int prop_info_offset = 12742 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 12743 12744 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 12745 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 12746 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 12747 } 12748 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12749 if (opline->result_type == IS_UNUSED) { 12750 switch (opline->opcode) { 12751 case ZEND_PRE_INC_OBJ: 12752 case ZEND_POST_INC_OBJ: 12753 | EXT_CALL zend_jit_inc_typed_prop, REG0 12754 break; 12755 case ZEND_PRE_DEC_OBJ: 12756 case ZEND_POST_DEC_OBJ: 12757 | EXT_CALL zend_jit_dec_typed_prop, REG0 12758 break; 12759 default: 12760 ZEND_UNREACHABLE(); 12761 } 12762 } else { 12763 | LOAD_ZVAL_ADDR CARG3, res_addr 12764 switch (opline->opcode) { 12765 case ZEND_PRE_INC_OBJ: 12766 | EXT_CALL zend_jit_pre_inc_typed_prop, REG0 12767 break; 12768 case ZEND_PRE_DEC_OBJ: 12769 | EXT_CALL zend_jit_pre_dec_typed_prop, REG0 12770 break; 12771 case ZEND_POST_INC_OBJ: 12772 | EXT_CALL zend_jit_post_inc_typed_prop, REG0 12773 break; 12774 case ZEND_POST_DEC_OBJ: 12775 | EXT_CALL zend_jit_post_dec_typed_prop, REG0 12776 break; 12777 default: 12778 ZEND_UNREACHABLE(); 12779 } 12780 } 12781 } 12782 } 12783 12784 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 12785 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 12786 zend_jit_addr var_addr = prop_addr; 12787 12788 if (use_prop_guard) { 12789 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 12790 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12791 if (!exit_addr) { 12792 return 0; 12793 } 12794 12795 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1 12796 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 12797 } 12798 12799 if (var_info & MAY_BE_REF) { 12800 may_throw = 1; 12801 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12802 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12803 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12804 } 12805 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 12806 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 12807 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 12808 | cbnz TMP1, >1 12809 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) 12810 |.cold_code 12811 |1: 12812 if (opline) { 12813 | SET_EX_OPLINE opline, REG0 12814 } 12815 if (opline->result_type == IS_UNUSED) { 12816 | mov FCARG2x, xzr 12817 } else { 12818 | LOAD_ZVAL_ADDR FCARG2x, res_addr 12819 } 12820 switch (opline->opcode) { 12821 case ZEND_PRE_INC_OBJ: 12822 | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 12823 break; 12824 case ZEND_PRE_DEC_OBJ: 12825 | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 12826 break; 12827 case ZEND_POST_INC_OBJ: 12828 | EXT_CALL zend_jit_post_inc_typed_ref, REG0 12829 break; 12830 case ZEND_POST_DEC_OBJ: 12831 | EXT_CALL zend_jit_post_dec_typed_ref, REG0 12832 break; 12833 default: 12834 ZEND_UNREACHABLE(); 12835 } 12836 | b >9 12837 |.code 12838 |2: 12839 } 12840 12841 if (var_info & MAY_BE_LONG) { 12842 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 12843 | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2, ZREG_TMP1 12844 } 12845 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 12846 if (opline->result_type != IS_UNUSED) { 12847 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 12848 } 12849 } 12850 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 12851 | LONG_ADD_SUB_WITH_IMM adds, var_addr, Z_L(1), TMP1, TMP2 12852 } else { 12853 | LONG_ADD_SUB_WITH_IMM subs, var_addr, Z_L(1), TMP1, TMP2 12854 } 12855 | bvs >3 12856 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { 12857 if (opline->result_type != IS_UNUSED) { 12858 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 12859 } 12860 } 12861 |.cold_code 12862 } 12863 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 12864 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 12865 may_throw = 1; 12866 } 12867 if (var_info & MAY_BE_LONG) { 12868 |2: 12869 } 12870 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 12871 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12872 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12873 } 12874 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 12875 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 12876 | TRY_ADDREF MAY_BE_ANY, REG0w, REG2, TMP1w 12877 } 12878 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 12879 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 12880 | LOAD_ZVAL_ADDR FCARG2x, res_addr 12881 | EXT_CALL zend_jit_pre_inc, REG0 12882 } else { 12883 | EXT_CALL increment_function, REG0 12884 } 12885 } else { 12886 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 12887 | LOAD_ZVAL_ADDR FCARG2x, res_addr 12888 | EXT_CALL zend_jit_pre_dec, REG0 12889 } else { 12890 | EXT_CALL decrement_function, REG0 12891 } 12892 } 12893 if (var_info & MAY_BE_LONG) { 12894 | b >4 12895 } 12896 } 12897 12898 if (var_info & MAY_BE_LONG) { 12899 |3: 12900 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 12901 uint64_t val = 0x43e0000000000000; 12902 | LOAD_64BIT_VAL TMP2, val 12903 | SET_ZVAL_LVAL_FROM_REG var_addr, TMP2, TMP1 12904 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2 12905 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 12906 | SET_ZVAL_LVAL_FROM_REG res_addr, TMP2, TMP1 12907 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 12908 } 12909 } else { 12910 uint64_t val = 0xc3e0000000000000; 12911 | LOAD_64BIT_VAL TMP2, val 12912 | SET_ZVAL_LVAL_FROM_REG var_addr, TMP2, TMP1 12913 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2 12914 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 12915 | SET_ZVAL_LVAL_FROM_REG res_addr, TMP2, TMP1 12916 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 12917 } 12918 } 12919 if (opline->result_type != IS_UNUSED 12920 && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) 12921 && prop_info 12922 && !ZEND_TYPE_IS_SET(prop_info->type) 12923 && (res_info & MAY_BE_GUARD) 12924 && (res_info & MAY_BE_LONG)) { 12925 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 12926 uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 12927 int32_t exit_point; 12928 const void *exit_addr; 12929 12930 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 12931 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 12932 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12933 if (!exit_addr) { 12934 return 0; 12935 } 12936 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 12937 ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD; 12938 | b &exit_addr 12939 |.code 12940 } else { 12941 | b >4 12942 |.code 12943 |4: 12944 } 12945 } 12946 } 12947 12948 if (needs_slow_path) { 12949 may_throw = 1; 12950 |.cold_code 12951 |7: 12952 | SET_EX_OPLINE opline, REG0 12953 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 12954 | LOAD_ADDR FCARG2x, name 12955 | ldr CARG3, EX->run_time_cache 12956 | ADD_SUB_64_WITH_CONST_32 add, CARG3, CARG3, opline->extended_value, TMP1 12957 if (opline->result_type == IS_UNUSED) { 12958 | mov CARG4, xzr 12959 } else { 12960 | LOAD_ZVAL_ADDR CARG4, res_addr 12961 } 12962 12963 switch (opline->opcode) { 12964 case ZEND_PRE_INC_OBJ: 12965 | EXT_CALL zend_jit_pre_inc_obj_helper, REG0 12966 break; 12967 case ZEND_PRE_DEC_OBJ: 12968 | EXT_CALL zend_jit_pre_dec_obj_helper, REG0 12969 break; 12970 case ZEND_POST_INC_OBJ: 12971 | EXT_CALL zend_jit_post_inc_obj_helper, REG0 12972 break; 12973 case ZEND_POST_DEC_OBJ: 12974 | EXT_CALL zend_jit_post_dec_obj_helper, REG0 12975 break; 12976 default: 12977 ZEND_UNREACHABLE(); 12978 } 12979 12980 | b >9 12981 |.code 12982 } 12983 12984 |9: 12985 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 12986 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 12987 may_throw = 1; 12988 } 12989 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 12990 } 12991 12992 if (may_throw) { 12993 if (!zend_jit_check_exception(Dst)) { 12994 return 0; 12995 } 12996 } 12997 12998 return 1; 12999} 13000 13001static int zend_jit_assign_obj_op(dasm_State **Dst, 13002 const zend_op *opline, 13003 const zend_op_array *op_array, 13004 zend_ssa *ssa, 13005 const zend_ssa_op *ssa_op, 13006 uint32_t op1_info, 13007 zend_jit_addr op1_addr, 13008 uint32_t val_info, 13009 zend_ssa_range *val_range, 13010 bool op1_indirect, 13011 zend_class_entry *ce, 13012 bool ce_is_instanceof, 13013 bool on_this, 13014 bool delayed_fetch_this, 13015 zend_class_entry *trace_ce, 13016 uint8_t prop_type) 13017{ 13018 zval *member; 13019 zend_string *name; 13020 zend_property_info *prop_info; 13021 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13022 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13023 zend_jit_addr prop_addr; 13024 bool needs_slow_path = 0; 13025 bool use_prop_guard = 0; 13026 bool may_throw = 0; 13027 binary_op_type binary_op = get_binary_op(opline->extended_value); 13028 13029 ZEND_ASSERT(opline->op2_type == IS_CONST); 13030 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13031 ZEND_ASSERT(opline->result_type == IS_UNUSED); 13032 13033 member = RT_CONSTANT(opline, opline->op2); 13034 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13035 name = Z_STR_P(member); 13036 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13037 13038 if (on_this) { 13039 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 13040 } else { 13041 if (opline->op1_type == IS_VAR 13042 && (op1_info & MAY_BE_INDIRECT) 13043 && Z_REG(op1_addr) == ZREG_FP) { 13044 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13045 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 13046 | GET_Z_PTR FCARG1x, FCARG1x 13047 |1: 13048 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13049 } 13050 if (op1_info & MAY_BE_REF) { 13051 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13052 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13053 } 13054 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 13055 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13056 } 13057 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13058 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13059 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13060 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13061 13062 if (!exit_addr) { 13063 return 0; 13064 } 13065 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 13066 } else { 13067 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 13068 |.cold_code 13069 |1: 13070 | SET_EX_OPLINE opline, REG0 13071 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13072 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13073 } 13074 | LOAD_ADDR FCARG2x, ZSTR_VAL(name) 13075 if (op1_info & MAY_BE_UNDEF) { 13076 | EXT_CALL zend_jit_invalid_property_assign_op, REG0 13077 } else { 13078 | EXT_CALL zend_jit_invalid_property_assign, REG0 13079 } 13080 may_throw = 1; 13081 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13082 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13083 | b >8 13084 } else { 13085 | b >9 13086 } 13087 |.code 13088 } 13089 } 13090 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 13091 } 13092 13093 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13094 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13095 if (prop_info) { 13096 ce = trace_ce; 13097 ce_is_instanceof = 0; 13098 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13099 if (on_this && JIT_G(current_frame) 13100 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13101 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13102 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13103 if (on_this && JIT_G(current_frame)) { 13104 JIT_G(current_frame)->ce = ce; 13105 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13106 } 13107 } else { 13108 return 0; 13109 } 13110 if (ssa->var_info && ssa_op->op1_use >= 0) { 13111 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13112 ssa->var_info[ssa_op->op1_use].ce = ce; 13113 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13114 } 13115 if (ssa->var_info && ssa_op->op1_def >= 0) { 13116 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13117 ssa->var_info[ssa_op->op1_def].ce = ce; 13118 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13119 } 13120 } 13121 } 13122 } 13123 13124 use_prop_guard = (prop_type != IS_UNKNOWN 13125 && prop_type != IS_UNDEF 13126 && prop_type != IS_REFERENCE 13127 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 13128 13129 if (!prop_info) { 13130 needs_slow_path = 1; 13131 13132 | ldr REG0, EX->run_time_cache 13133 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, ((opline+1)->extended_value), TMP1 13134 | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] 13135 | cmp REG2, TMP2 13136 | bne >7 13137 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 13138 | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, ((opline+1)->extended_value + sizeof(void*) * 2), TMP1 13139 | cbnz TMP1, >7 13140 } 13141 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline+1)->extended_value + sizeof(void*)), TMP1 13142 | tst REG0, REG0 13143 | blt >7 13144 | add TMP1, FCARG1x, REG0 13145 if (!use_prop_guard) { 13146 | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] 13147 | IF_TYPE TMP2w, IS_UNDEF, >7 13148 } 13149 | mov FCARG1x, TMP1 13150 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13151 } else { 13152 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13153 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 13154 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13155 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13156 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13157 13158 if (!exit_addr) { 13159 return 0; 13160 } 13161 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13162 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr 13163 } else { 13164 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13165 | IF_TYPE TMP2w, IS_UNDEF, >7 13166 needs_slow_path = 1; 13167 } 13168 } 13169 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13170 uint32_t info = val_info; 13171 13172 may_throw = 1; 13173 13174 if (opline) { 13175 | SET_EX_OPLINE opline, REG0 13176 } 13177 13178 | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1, ZREG_TMP1 13179 |.cold_code 13180 |1: 13181 | GET_ZVAL_PTR FCARG1x, prop_addr, TMP1 13182 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 13183 | LOAD_ZVAL_ADDR FCARG2x, val_addr 13184 } 13185 | LOAD_ADDR CARG3, binary_op 13186 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 13187 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13188 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 13189 } else { 13190 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 13191 } 13192 | b >9 13193 |.code 13194 13195 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13196 13197 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13198 | LOAD_ADDR FCARG2x, prop_info 13199 } else { 13200 int prop_info_offset = 13201 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13202 13203 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 13204 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 13205 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 13206 } 13207 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 13208 | LOAD_ZVAL_ADDR CARG3, val_addr 13209 | LOAD_ADDR CARG4, binary_op 13210 13211 | EXT_CALL zend_jit_assign_op_to_typed_prop, REG0 13212 13213 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13214 info |= MAY_BE_RC1|MAY_BE_RCN; 13215 } 13216 13217 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2 13218 } 13219 } 13220 13221 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13222 zend_jit_addr var_addr = prop_addr; 13223 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13224 uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13225 13226 if (use_prop_guard) { 13227 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 13228 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13229 if (!exit_addr) { 13230 return 0; 13231 } 13232 13233 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1 13234 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 13235 } 13236 13237 if (var_info & MAY_BE_REF) { 13238 may_throw = 1; 13239 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 13240 | LOAD_ZVAL_ADDR REG0, prop_addr 13241 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 13242 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 13243 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 13244 | cbnz TMP1, >1 13245 | add REG0, FCARG1x, #offsetof(zend_reference, val) 13246 |.cold_code 13247 |1: 13248 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 13249 | LOAD_ZVAL_ADDR FCARG2x, val_addr 13250 } 13251 if (opline) { 13252 | SET_EX_OPLINE opline, REG0 13253 } 13254 | LOAD_ADDR CARG3, binary_op 13255 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 13256 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13257 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 13258 } else { 13259 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 13260 } 13261 | b >9 13262 |.code 13263 |2: 13264 } 13265 13266 switch (opline->extended_value) { 13267 case ZEND_ADD: 13268 case ZEND_SUB: 13269 case ZEND_MUL: 13270 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13271 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13272 if (opline->extended_value != ZEND_ADD || 13273 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 13274 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 13275 may_throw = 1; 13276 } 13277 } 13278 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, 13279 1 /* may overflow */, 0)) { 13280 return 0; 13281 } 13282 break; 13283 case ZEND_BW_OR: 13284 case ZEND_BW_AND: 13285 case ZEND_BW_XOR: 13286 may_throw = 1; 13287 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13288 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13289 if ((var_info & MAY_BE_ANY) != MAY_BE_STRING || 13290 (val_info & MAY_BE_ANY) != MAY_BE_STRING) { 13291 may_throw = 1; 13292 } 13293 } 13294 goto long_math; 13295 case ZEND_SL: 13296 case ZEND_SR: 13297 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13298 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13299 may_throw = 1; 13300 } 13301 if ((opline+1)->op1_type != IS_CONST || 13302 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 13303 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) { 13304 may_throw = 1; 13305 } 13306 goto long_math; 13307 case ZEND_MOD: 13308 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13309 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13310 if (opline->extended_value != ZEND_ADD || 13311 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 13312 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 13313 may_throw = 1; 13314 } 13315 } 13316 if ((opline+1)->op1_type != IS_CONST || 13317 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 13318 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) { 13319 may_throw = 1; 13320 } 13321long_math: 13322 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 13323 IS_CV, opline->op1, var_addr, var_info, NULL, 13324 (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 13325 val_range, 13326 0, var_addr, var_def_info, var_info, 0)) { 13327 return 0; 13328 } 13329 break; 13330 case ZEND_CONCAT: 13331 may_throw = 1; 13332 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, 13333 0)) { 13334 return 0; 13335 } 13336 break; 13337 default: 13338 ZEND_UNREACHABLE(); 13339 } 13340 } 13341 13342 if (needs_slow_path) { 13343 may_throw = 1; 13344 |.cold_code 13345 |7: 13346 | SET_EX_OPLINE opline, REG0 13347 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13348 | LOAD_ADDR FCARG2x, name 13349 | LOAD_ZVAL_ADDR CARG3, val_addr 13350 | ldr CARG4, EX->run_time_cache 13351 | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, (opline+1)->extended_value, TMP1 13352 | LOAD_ADDR CARG5, binary_op 13353 | EXT_CALL zend_jit_assign_obj_op_helper, REG0 13354 13355 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13356 val_info |= MAY_BE_RC1|MAY_BE_RCN; 13357 } 13358 13359 |8: 13360 | // FREE_OP_DATA(); 13361 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13362 | b >9 13363 |.code 13364 } 13365 13366 |9: 13367 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13368 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 13369 may_throw = 1; 13370 } 13371 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 13372 } 13373 13374 if (may_throw) { 13375 if (!zend_jit_check_exception(Dst)) { 13376 return 0; 13377 } 13378 } 13379 13380 return 1; 13381} 13382 13383static int zend_jit_assign_obj(dasm_State **Dst, 13384 const zend_op *opline, 13385 const zend_op_array *op_array, 13386 zend_ssa *ssa, 13387 const zend_ssa_op *ssa_op, 13388 uint32_t op1_info, 13389 zend_jit_addr op1_addr, 13390 uint32_t val_info, 13391 bool op1_indirect, 13392 zend_class_entry *ce, 13393 bool ce_is_instanceof, 13394 bool on_this, 13395 bool delayed_fetch_this, 13396 zend_class_entry *trace_ce, 13397 uint8_t prop_type, 13398 int may_throw) 13399{ 13400 zval *member; 13401 zend_string *name; 13402 zend_property_info *prop_info; 13403 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13404 zend_jit_addr res_addr = 0; 13405 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13406 zend_jit_addr prop_addr; 13407 bool needs_slow_path = 0; 13408 bool needs_val_dtor = 0; 13409 13410 if (RETURN_VALUE_USED(opline)) { 13411 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 13412 } 13413 13414 ZEND_ASSERT(opline->op2_type == IS_CONST); 13415 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13416 13417 member = RT_CONSTANT(opline, opline->op2); 13418 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13419 name = Z_STR_P(member); 13420 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13421 13422 if (on_this) { 13423 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 13424 } else { 13425 if (opline->op1_type == IS_VAR 13426 && (op1_info & MAY_BE_INDIRECT) 13427 && Z_REG(op1_addr) == ZREG_FP) { 13428 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13429 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 13430 | GET_Z_PTR FCARG1x, FCARG1x 13431 |1: 13432 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13433 } 13434 if (op1_info & MAY_BE_REF) { 13435 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13436 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13437 } 13438 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 13439 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13440 } 13441 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13442 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13443 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13444 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13445 13446 if (!exit_addr) { 13447 return 0; 13448 } 13449 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 13450 } else { 13451 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 13452 |.cold_code 13453 |1: 13454 | SET_EX_OPLINE opline, REG0 13455 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13456 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13457 } 13458 | LOAD_ADDR FCARG2x, ZSTR_VAL(name) 13459 | EXT_CALL zend_jit_invalid_property_assign, REG0 13460 if (RETURN_VALUE_USED(opline)) { 13461 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 13462 } 13463 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13464 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13465 needs_val_dtor = 1; 13466 | b >7 13467 } else { 13468 | b >9 13469 } 13470 |.code 13471 } 13472 } 13473 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 13474 } 13475 13476 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13477 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13478 if (prop_info) { 13479 ce = trace_ce; 13480 ce_is_instanceof = 0; 13481 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13482 if (on_this && JIT_G(current_frame) 13483 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13484 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13485 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13486 if (on_this && JIT_G(current_frame)) { 13487 JIT_G(current_frame)->ce = ce; 13488 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13489 } 13490 } else { 13491 return 0; 13492 } 13493 if (ssa->var_info && ssa_op->op1_use >= 0) { 13494 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13495 ssa->var_info[ssa_op->op1_use].ce = ce; 13496 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13497 } 13498 if (ssa->var_info && ssa_op->op1_def >= 0) { 13499 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13500 ssa->var_info[ssa_op->op1_def].ce = ce; 13501 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13502 } 13503 } 13504 } 13505 } 13506 13507 if (!prop_info) { 13508 needs_slow_path = 1; 13509 13510 | ldr REG0, EX->run_time_cache 13511 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 13512 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 13513 | cmp REG2, TMP1 13514 | bne >5 13515 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 13516 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 13517 } 13518 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 13519 | tst REG0, REG0 13520 | blt >5 13521 | add TMP2, FCARG1x, REG0 13522 | ldrb TMP1w, [TMP2, #offsetof(zval,u1.v.type)] 13523 | IF_TYPE TMP1w, IS_UNDEF, >5 13524 | mov FCARG1x, TMP2 13525 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13526 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 13527 | cbnz FCARG2x, >1 13528 |.cold_code 13529 |1: 13530 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13531 | SET_EX_OPLINE opline, REG0 13532 | LOAD_ZVAL_ADDR CARG3, val_addr 13533 if (RETURN_VALUE_USED(opline)) { 13534 | LOAD_ZVAL_ADDR CARG4, res_addr 13535 } else { 13536 | mov CARG4, xzr 13537 } 13538 13539 | EXT_CALL zend_jit_assign_to_typed_prop, REG0 13540 13541 if ((opline+1)->op1_type == IS_CONST) { 13542 | // TODO: ??? 13543 | // if (Z_TYPE_P(value) == orig_type) { 13544 | // CACHE_PTR_EX(cache_slot + 2, NULL); 13545 } 13546 13547 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13548 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13549 | b >7 13550 } else { 13551 | b >9 13552 } 13553 |.code 13554 } 13555 } else { 13556 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13557 if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) { 13558 // Undefined property with magic __get()/__set() 13559 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13560 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13561 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13562 13563 if (!exit_addr) { 13564 return 0; 13565 } 13566 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13567 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr 13568 } else { 13569 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13570 | IF_TYPE TMP2w, IS_UNDEF, >5 13571 needs_slow_path = 1; 13572 } 13573 } 13574 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13575 uint32_t info = val_info; 13576 13577 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13578 | SET_EX_OPLINE opline, REG0 13579 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13580 | LOAD_ADDR FCARG2x, prop_info 13581 } else { 13582 int prop_info_offset = 13583 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13584 13585 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 13586 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 13587 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 13588 } 13589 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 13590 | LOAD_ZVAL_ADDR CARG3, val_addr 13591 if (RETURN_VALUE_USED(opline)) { 13592 | LOAD_ZVAL_ADDR CARG4, res_addr 13593 } else { 13594 | mov CARG4, xzr 13595 } 13596 13597 | EXT_CALL zend_jit_assign_to_typed_prop, REG0 13598 13599 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13600 info |= MAY_BE_RC1|MAY_BE_RCN; 13601 } 13602 13603 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2 13604 } 13605 } 13606 13607 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13608 // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); 13609 if (opline->result_type == IS_UNUSED) { 13610 if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 13611 return 0; 13612 } 13613 } else { 13614 if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 13615 return 0; 13616 } 13617 } 13618 } 13619 13620 if (needs_slow_path) { 13621 |.cold_code 13622 |5: 13623 | SET_EX_OPLINE opline, REG0 13624 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13625 | LOAD_ADDR FCARG2x, name 13626 13627 | LOAD_ZVAL_ADDR CARG3, val_addr 13628 | ldr CARG4, EX->run_time_cache 13629 | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, opline->extended_value, TMP1 13630 if (RETURN_VALUE_USED(opline)) { 13631 | LOAD_ZVAL_ADDR CARG5, res_addr 13632 } else { 13633 | mov CARG5, xzr 13634 } 13635 13636 | EXT_CALL zend_jit_assign_obj_helper, REG0 13637 13638 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13639 val_info |= MAY_BE_RC1|MAY_BE_RCN; 13640 } 13641 13642 |7: 13643 | // FREE_OP_DATA(); 13644 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13645 | b >9 13646 |.code 13647 } else if (needs_val_dtor) { 13648 |.cold_code 13649 |7: 13650 | // FREE_OP_DATA(); 13651 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13652 | b >9 13653 |.code 13654 } 13655 13656 |9: 13657 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13658 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 13659 } 13660 13661 if (may_throw) { 13662 if (!zend_jit_check_exception(Dst)) { 13663 return 0; 13664 } 13665 } 13666 13667 return 1; 13668} 13669 13670static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) 13671{ 13672 zend_jit_addr op1_addr = OP1_ADDR(); 13673 13674 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 13675 if (may_throw) { 13676 | SET_EX_OPLINE opline, REG0 13677 } 13678 if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { 13679 if (op1_info & MAY_BE_ARRAY) { 13680 | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 13681 } 13682 | MEM_ACCESS_32_WITH_UOFFSET ldr, FCARG1w, FP, (opline->op1.var + offsetof(zval, u2.fe_iter_idx)), TMP1 13683 | mvn TMP1w, wzr // TODO: DynAsm fails loading #-1 13684 | cmp FCARG1w, TMP1w 13685 | beq >7 13686 | EXT_CALL zend_hash_iterator_del, REG0 13687 |7: 13688 } 13689 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 13690 if (may_throw) { 13691 if (!zend_jit_check_exception(Dst)) { 13692 return 0; 13693 } 13694 } 13695 } 13696 return 1; 13697} 13698 13699static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 13700{ 13701 if (opline->op1_type == IS_CONST) { 13702 zval *zv; 13703 size_t len; 13704 13705 zv = RT_CONSTANT(opline, opline->op1); 13706 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 13707 len = Z_STRLEN_P(zv); 13708 13709 if (len > 0) { 13710 const char *str = Z_STRVAL_P(zv); 13711 13712 | SET_EX_OPLINE opline, REG0 13713 | LOAD_ADDR CARG1, str 13714 | LOAD_64BIT_VAL CARG2, len 13715 | EXT_CALL zend_write, REG0 13716 if (!zend_jit_check_exception(Dst)) { 13717 return 0; 13718 } 13719 } 13720 } else { 13721 zend_jit_addr op1_addr = OP1_ADDR(); 13722 13723 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 13724 13725 | SET_EX_OPLINE opline, REG0 13726 | GET_ZVAL_PTR REG0, op1_addr, TMP1 13727 | add CARG1, REG0, #offsetof(zend_string, val) 13728 | ldr CARG2, [REG0, #offsetof(zend_string, len)] 13729 | EXT_CALL zend_write, REG0 13730 if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { 13731 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 13732 } 13733 if (!zend_jit_check_exception(Dst)) { 13734 return 0; 13735 } 13736 } 13737 return 1; 13738} 13739 13740static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) 13741{ 13742 if (opline->op1_type == IS_CONST) { 13743 zval *zv; 13744 size_t len; 13745 13746 zv = RT_CONSTANT(opline, opline->op1); 13747 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 13748 len = Z_STRLEN_P(zv); 13749 13750 | SET_ZVAL_LVAL res_addr, len, TMP1, TMP2 13751 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 13752 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13753 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13754 return 0; 13755 } 13756 } else { 13757 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 13758 13759 if (Z_MODE(res_addr) == IS_REG) { 13760 | GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1 13761 | ldr Rx(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(zend_string, len)] 13762 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13763 return 0; 13764 } 13765 } else { 13766 | GET_ZVAL_PTR REG0, op1_addr, TMP1 13767 | ldr REG0, [REG0, #offsetof(zend_string, len)] 13768 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 13769 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13770 } 13771 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13772 } 13773 return 1; 13774} 13775 13776static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) 13777{ 13778 if (opline->op1_type == IS_CONST) { 13779 zval *zv; 13780 zend_long count; 13781 13782 zv = RT_CONSTANT(opline, opline->op1); 13783 ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); 13784 count = zend_hash_num_elements(Z_ARRVAL_P(zv)); 13785 13786 | SET_ZVAL_LVAL res_addr, count, TMP1, TMP2 13787 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 13788 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13789 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13790 return 0; 13791 } 13792 } else { 13793 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY); 13794 // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+. 13795 13796 if (Z_MODE(res_addr) == IS_REG) { 13797 | GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1 13798 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 13799 | ldr Rw(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(HashTable, nNumOfElements)] 13800 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13801 return 0; 13802 } 13803 } else { 13804 | GET_ZVAL_PTR REG0, op1_addr, TMP1 13805 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 13806 | ldr REG0w, [REG0, #offsetof(HashTable, nNumOfElements)] 13807 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 13808 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13809 } 13810 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13811 } 13812 13813 if (may_throw) { 13814 return zend_jit_check_exception(Dst); 13815 } 13816 return 1; 13817} 13818 13819static int zend_jit_load_this(dasm_State **Dst, uint32_t var) 13820{ 13821 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 13822 13823 | ldr FCARG1x, EX->This.value.ptr 13824 | SET_ZVAL_PTR var_addr, FCARG1x, TMP1 13825 | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX, TMP1w, TMP2 13826 | GC_ADDREF FCARG1x, TMP1w 13827 return 1; 13828} 13829 13830static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only) 13831{ 13832 if (!op_array->scope || 13833 (op_array->fn_flags & ZEND_ACC_STATIC) || 13834 ((op_array->fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_IMMUTABLE)) == ZEND_ACC_CLOSURE)) { 13835 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13836 if (!JIT_G(current_frame) || 13837 !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { 13838 13839 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13840 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13841 13842 if (!exit_addr) { 13843 return 0; 13844 } 13845 13846 | ldrb TMP1w, EX->This.u1.v.type 13847 | cmp TMP1w, #IS_OBJECT 13848 | bne &exit_addr 13849 13850 if (JIT_G(current_frame)) { 13851 TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); 13852 } 13853 } 13854 } else { 13855 13856 | ldrb TMP1w, EX->This.u1.v.type 13857 | cmp TMP1w, #IS_OBJECT 13858 | bne >1 13859 |.cold_code 13860 |1: 13861 | SET_EX_OPLINE opline, REG0 13862 | b ->invalid_this 13863 |.code 13864 } 13865 } 13866 13867 if (!check_only) { 13868 if (!zend_jit_load_this(Dst, opline->result.var)) { 13869 return 0; 13870 } 13871 } 13872 return 1; 13873} 13874 13875static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) 13876{ 13877 uint32_t count; 13878 Bucket *p; 13879 const zend_op *target; 13880 int b; 13881 int32_t exit_point; 13882 const void *exit_addr; 13883 13884 if (default_label) { 13885 | cbz REG0, &default_label 13886 } else if (next_opline) { 13887 | cbz REG0, >3 13888 } else { 13889 | cbz REG0, =>default_b 13890 } 13891 | LOAD_ADDR FCARG1x, jumptable 13892 | ldr TMP1, [FCARG1x, #offsetof(HashTable, arData)] 13893 | sub REG0, REG0, TMP1 13894 if (HT_IS_PACKED(jumptable)) { 13895 | mov FCARG1x, #(sizeof(zval) / sizeof(void*)) 13896 } else { 13897 | mov FCARG1x, #(sizeof(Bucket) / sizeof(void*)) 13898 } 13899 | sdiv REG0, REG0, FCARG1x 13900 | adr FCARG1x, >4 13901 | ldr TMP1, [FCARG1x, REG0] 13902 | br TMP1 13903 13904 |.jmp_table 13905 |.align 8 13906 |4: 13907 if (trace_info) { 13908 trace_info->jmp_table_size += zend_hash_num_elements(jumptable); 13909 } 13910 13911 count = jumptable->nNumUsed; 13912 p = jumptable->arData; 13913 do { 13914 if (Z_TYPE(p->val) == IS_UNDEF) { 13915 if (default_label) { 13916 | .addr &default_label 13917 } else if (next_opline) { 13918 | .addr >3 13919 } else { 13920 | .addr =>default_b 13921 } 13922 } else { 13923 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 13924 if (!next_opline) { 13925 b = ssa->cfg.map[target - op_array->opcodes]; 13926 | .addr =>b 13927 } else if (next_opline == target) { 13928 | .addr >3 13929 } else { 13930 exit_point = zend_jit_trace_get_exit_point(target, 0); 13931 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13932 if (!exit_addr) { 13933 return 0; 13934 } 13935 | .addr &exit_addr 13936 } 13937 } 13938 if (HT_IS_PACKED(jumptable)) { 13939 p = (Bucket*)(((zval*)p)+1); 13940 } else { 13941 p++; 13942 } 13943 count--; 13944 } while (count); 13945 |.code 13946 13947 return 1; 13948} 13949 13950static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) 13951{ 13952 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 13953 const zend_op *next_opline = NULL; 13954 13955 if (trace) { 13956 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 13957 ZEND_ASSERT(trace->opline != NULL); 13958 next_opline = trace->opline; 13959 } 13960 13961 if (opline->op1_type == IS_CONST) { 13962 zval *zv = RT_CONSTANT(opline, opline->op1); 13963 zval *jump_zv = NULL; 13964 int b; 13965 13966 if (opline->opcode == ZEND_SWITCH_LONG) { 13967 if (Z_TYPE_P(zv) == IS_LONG) { 13968 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 13969 } 13970 } else if (opline->opcode == ZEND_SWITCH_STRING) { 13971 if (Z_TYPE_P(zv) == IS_STRING) { 13972 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 13973 } 13974 } else if (opline->opcode == ZEND_MATCH) { 13975 if (Z_TYPE_P(zv) == IS_LONG) { 13976 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 13977 } else if (Z_TYPE_P(zv) == IS_STRING) { 13978 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 13979 } 13980 } else { 13981 ZEND_UNREACHABLE(); 13982 } 13983 if (next_opline) { 13984 const zend_op *target; 13985 13986 if (jump_zv != NULL) { 13987 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); 13988 } else { 13989 target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 13990 } 13991 ZEND_ASSERT(target == next_opline); 13992 } else { 13993 if (jump_zv != NULL) { 13994 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; 13995 } else { 13996 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; 13997 } 13998 | b =>b 13999 } 14000 } else { 14001 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; 14002 uint32_t op1_info = OP1_INFO(); 14003 zend_jit_addr op1_addr = OP1_ADDR(); 14004 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14005 const zend_op *target; 14006 int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; 14007 int b; 14008 int32_t exit_point; 14009 const void *fallback_label = NULL; 14010 const void *default_label = NULL; 14011 const void *exit_addr; 14012 14013 if (next_opline) { 14014 if (next_opline != opline + 1) { 14015 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 14016 fallback_label = zend_jit_trace_get_exit_addr(exit_point); 14017 if (!fallback_label) { 14018 return 0; 14019 } 14020 } 14021 if (next_opline != default_opline) { 14022 exit_point = zend_jit_trace_get_exit_point(default_opline, 0); 14023 default_label = zend_jit_trace_get_exit_addr(exit_point); 14024 if (!default_label) { 14025 return 0; 14026 } 14027 } 14028 } 14029 14030 if (opline->opcode == ZEND_SWITCH_LONG) { 14031 if (op1_info & MAY_BE_LONG) { 14032 if (op1_info & MAY_BE_REF) { 14033 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1, ZREG_TMP1 14034 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 14035 |.cold_code 14036 |1: 14037 | // ZVAL_DEREF(op) 14038 if (fallback_label) { 14039 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 14040 } else { 14041 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 14042 } 14043 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14044 if (fallback_label) { 14045 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14046 | IF_NOT_Z_TYPE TMP1, IS_LONG, &fallback_label, TMP2w 14047 } else { 14048 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14049 | IF_NOT_Z_TYPE TMP1, IS_LONG, >3, TMP2w 14050 } 14051 | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.lval)] 14052 | b >2 14053 |.code 14054 |2: 14055 } else { 14056 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14057 if (fallback_label) { 14058 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label, ZREG_TMP1 14059 } else { 14060 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 14061 } 14062 } 14063 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 14064 } 14065 if (HT_IS_PACKED(jumptable)) { 14066 uint32_t count = jumptable->nNumUsed; 14067 zval *zv = jumptable->arPacked; 14068 14069 | CMP_64_WITH_CONST_32 FCARG2x, jumptable->nNumUsed, TMP1 14070 if (default_label) { 14071 | bhs &default_label 14072 } else if (next_opline) { 14073 | bhs >3 14074 } else { 14075 | bhs =>default_b 14076 } 14077 | adr REG0, >4 14078 | ldr TMP1, [REG0, FCARG2x, lsl #3] 14079 | br TMP1 14080 14081 |.jmp_table 14082 |.align 8 14083 |4: 14084 if (trace_info) { 14085 trace_info->jmp_table_size += count; 14086 } 14087 do { 14088 if (Z_TYPE_P(zv) == IS_UNDEF) { 14089 if (default_label) { 14090 | .addr &default_label 14091 } else if (next_opline) { 14092 | .addr >3 14093 } else { 14094 | .addr =>default_b 14095 } 14096 } else { 14097 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); 14098 if (!next_opline) { 14099 b = ssa->cfg.map[target - op_array->opcodes]; 14100 | .addr =>b 14101 } else if (next_opline == target) { 14102 | .addr >3 14103 } else { 14104 exit_point = zend_jit_trace_get_exit_point(target, 0); 14105 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14106 if (!exit_addr) { 14107 return 0; 14108 } 14109 | .addr &exit_addr 14110 } 14111 } 14112 zv++; 14113 count--; 14114 } while (count); 14115 |.code 14116 |3: 14117 } else { 14118 | LOAD_ADDR FCARG1x, jumptable 14119 | EXT_CALL zend_hash_index_find, REG0 14120 | mov REG0, RETVALx 14121 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14122 return 0; 14123 } 14124 |3: 14125 } 14126 } 14127 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14128 if (op1_info & MAY_BE_STRING) { 14129 if (op1_info & MAY_BE_REF) { 14130 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1, ZREG_TMP1 14131 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14132 |.cold_code 14133 |1: 14134 | // ZVAL_DEREF(op) 14135 if (fallback_label) { 14136 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 14137 } else { 14138 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 14139 } 14140 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14141 if (fallback_label) { 14142 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14143 | IF_NOT_Z_TYPE TMP1, IS_STRING, &fallback_label, TMP2w 14144 } else { 14145 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14146 | IF_NOT_Z_TYPE TMP1, IS_STRING, >3, TMP2w 14147 } 14148 | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.ptr)] 14149 | b >2 14150 |.code 14151 |2: 14152 } else { 14153 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { 14154 if (fallback_label) { 14155 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label, ZREG_TMP1 14156 } else { 14157 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 14158 } 14159 } 14160 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14161 } 14162 | LOAD_ADDR FCARG1x, jumptable 14163 | EXT_CALL zend_hash_find, REG0 14164 | mov REG0, RETVALx 14165 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14166 return 0; 14167 } 14168 |3: 14169 } 14170 } else if (opline->opcode == ZEND_MATCH) { 14171 if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { 14172 if (op1_info & MAY_BE_REF) { 14173 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 14174 | ZVAL_DEREF FCARG2x, op1_info, TMP1w 14175 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 14176 } 14177 | LOAD_ADDR FCARG1x, jumptable 14178 if (op1_info & MAY_BE_LONG) { 14179 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14180 if (op1_info & MAY_BE_STRING) { 14181 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5, ZREG_TMP1 14182 } else if (op1_info & MAY_BE_UNDEF) { 14183 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 14184 } else if (default_label) { 14185 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label, ZREG_TMP1 14186 } else if (next_opline) { 14187 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 14188 } else { 14189 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b, ZREG_TMP1 14190 } 14191 } 14192 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 14193 | EXT_CALL zend_hash_index_find, REG0 14194 | mov REG0, RETVALx 14195 if (op1_info & MAY_BE_STRING) { 14196 | b >2 14197 } 14198 } 14199 if (op1_info & MAY_BE_STRING) { 14200 |5: 14201 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { 14202 if (op1_info & MAY_BE_UNDEF) { 14203 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 14204 } else if (default_label) { 14205 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label, ZREG_TMP1 14206 } else if (next_opline) { 14207 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 14208 } else { 14209 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b, ZREG_TMP1 14210 } 14211 } 14212 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14213 | EXT_CALL zend_hash_find, REG0 14214 | mov REG0, RETVALx 14215 } 14216 |2: 14217 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14218 return 0; 14219 } 14220 } 14221 if (op1_info & MAY_BE_UNDEF) { 14222 |6: 14223 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { 14224 if (default_label) { 14225 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label, ZREG_TMP1 14226 } else if (next_opline) { 14227 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3, ZREG_TMP1 14228 } else { 14229 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b, ZREG_TMP1 14230 } 14231 } 14232 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 14233 | SET_EX_OPLINE opline, REG0 14234 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 14235 | EXT_CALL zend_jit_undefined_op_helper, REG0 14236 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 14237 return 0; 14238 } 14239 } 14240 if (default_label) { 14241 | b &default_label 14242 } else if (next_opline) { 14243 | b >3 14244 } else { 14245 | b =>default_b 14246 } 14247 |3: 14248 } else { 14249 ZEND_UNREACHABLE(); 14250 } 14251 } 14252 return 1; 14253} 14254 14255static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) 14256{ 14257 zend_arg_info *arg_info = &op_array->arg_info[-1]; 14258 ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); 14259 zend_jit_addr op1_addr = OP1_ADDR(); 14260 bool needs_slow_check = 1; 14261 bool slow_check_in_cold = 1; 14262 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 14263 14264 if (type_mask == 0) { 14265 slow_check_in_cold = 0; 14266 } else { 14267 if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { 14268 slow_check_in_cold = 0; 14269 } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { 14270 needs_slow_check = 0; 14271 } else if (is_power_of_two(type_mask)) { 14272 uint32_t type_code = concrete_type(type_mask); 14273 | IF_NOT_ZVAL_TYPE op1_addr, type_code, >6, ZREG_TMP1 14274 } else { 14275 | mov REG2w, #1 14276 | GET_ZVAL_TYPE REG1w, op1_addr, TMP1 14277 | lsl REG2w, REG2w, REG1w 14278 | TST_32_WITH_CONST REG2w, type_mask, TMP1w 14279 | beq >6 14280 } 14281 } 14282 if (needs_slow_check) { 14283 if (slow_check_in_cold) { 14284 |.cold_code 14285 |6: 14286 } 14287 | SET_EX_OPLINE opline, REG1 14288 if (op1_info & MAY_BE_UNDEF) { 14289 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7, ZREG_TMP1 14290 | LOAD_32BIT_VAL FCARG1x, opline->op1.var 14291 | EXT_CALL zend_jit_undefined_op_helper, REG0 14292 | cbz RETVALx, ->exception_handler 14293 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 14294 | b >8 14295 } 14296 |7: 14297 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 14298 |8: 14299 | ldr FCARG2x, EX->func 14300 | LOAD_ADDR CARG3, (ptrdiff_t)arg_info 14301 | ldr REG0, EX->run_time_cache 14302 | ADD_SUB_64_WITH_CONST_32 add, CARG4, REG0, opline->op2.num, TMP1 14303 | EXT_CALL zend_jit_verify_return_slow, REG0 14304 if (!zend_jit_check_exception(Dst)) { 14305 return 0; 14306 } 14307 if (slow_check_in_cold) { 14308 | b >9 14309 |.code 14310 } 14311 } 14312 |9: 14313 return 1; 14314} 14315 14316static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 14317{ 14318 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14319 14320 // TODO: support for empty() ??? 14321 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 14322 14323 if (op1_info & MAY_BE_REF) { 14324 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14325 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 14326 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14327 } 14328 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 14329 |1: 14330 } 14331 14332 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { 14333 if (exit_addr) { 14334 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); 14335 } else if (smart_branch_opcode) { 14336 if (smart_branch_opcode == ZEND_JMPNZ) { 14337 | b =>target_label 14338 } 14339 } else { 14340 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 14341 } 14342 } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { 14343 if (exit_addr) { 14344 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); 14345 } else if (smart_branch_opcode) { 14346 if (smart_branch_opcode != ZEND_JMPNZ) { 14347 | b =>target_label 14348 } 14349 } else { 14350 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 14351 } 14352 } else { 14353 ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); 14354 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, Rx(Z_REG(op1_addr)), (Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)), TMP1 14355 | cmp TMP1w, #IS_NULL 14356 if (exit_addr) { 14357 if (smart_branch_opcode == ZEND_JMPNZ) { 14358 | bgt &exit_addr 14359 } else { 14360 | ble &exit_addr 14361 } 14362 } else if (smart_branch_opcode) { 14363 if (smart_branch_opcode == ZEND_JMPZ) { 14364 | ble =>target_label 14365 } else if (smart_branch_opcode == ZEND_JMPNZ) { 14366 | bgt =>target_label 14367 } else { 14368 ZEND_UNREACHABLE(); 14369 } 14370 } else { 14371 | cset REG0w, gt 14372 | add REG0w, REG0w, #IS_FALSE 14373 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 14374 } 14375 } 14376 14377 return 1; 14378} 14379 14380static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 14381{ 14382 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14383 14384 if (opline->op1_type == IS_CONST) { 14385 zval *zv = RT_CONSTANT(opline, opline->op1); 14386 14387 | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 14388 if (Z_REFCOUNTED_P(zv)) { 14389 | ADDREF_CONST zv, REG0, TMP1 14390 } 14391 } else { 14392 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 14393 14394 | // ZVAL_COPY(res, value); 14395 | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 14396 if (opline->op1_type == IS_CV) { 14397 | TRY_ADDREF op1_info, REG0w, FCARG1x, TMP1w 14398 } 14399 } 14400 | // Z_FE_POS_P(res) = 0; 14401 | MEM_ACCESS_32_WITH_UOFFSET str, wzr, FP, (opline->result.var + offsetof(zval, u2.fe_pos)), TMP1 14402 14403 return 1; 14404} 14405 14406static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr) 14407{ 14408 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 14409 14410 if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) { 14411 /* empty array */ 14412 if (exit_addr) { 14413 if (exit_opcode == ZEND_JMP) { 14414 | b &exit_addr 14415 } 14416 } else { 14417 | b =>target_label 14418 } 14419 return 1; 14420 } 14421 14422 | // array = EX_VAR(opline->op1.var); 14423 | // fe_ht = Z_ARRVAL_P(array); 14424 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 14425 14426 if (op1_info & MAY_BE_PACKED_GUARD) { 14427 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 14428 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14429 14430 if (!exit_addr) { 14431 return 0; 14432 } 14433 if (op1_info & MAY_BE_ARRAY_PACKED) { 14434 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 14435 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 14436 | beq &exit_addr 14437 } else { 14438 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 14439 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 14440 | bne &exit_addr 14441 } 14442 } 14443 14444 | // pos = Z_FE_POS_P(array); 14445 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 14446 14447 if (MAY_BE_HASH(op1_info)) { 14448 if (MAY_BE_PACKED(op1_info)) { 14449 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 14450 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 14451 | bne >2 14452 } 14453 | // p = fe_ht->arData + pos; 14454 || ZEND_ASSERT(sizeof(Bucket) == 32); 14455 | mov FCARG2w, REG0w 14456 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] 14457 | add FCARG2x, TMP1, FCARG2x, lsl #5 14458 |1: 14459 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 14460 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, nNumUsed)] 14461 | cmp TMP1w, REG0w 14462 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 14463 | // ZEND_VM_CONTINUE(); 14464 if (exit_addr) { 14465 if (exit_opcode == ZEND_JMP) { 14466 | bls &exit_addr 14467 } else { 14468 | bls >3 14469 } 14470 } else { 14471 | bls =>target_label 14472 } 14473 | // pos++; 14474 | add REG0w, REG0w, #1 14475 | // value_type = Z_TYPE_INFO_P(value); 14476 | // if (EXPECTED(value_type != IS_UNDEF)) { 14477 if (!exit_addr || exit_opcode == ZEND_JMP) { 14478 | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >3, TMP1w 14479 } else { 14480 | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr, TMP1w 14481 } 14482 | // p++; 14483 | add FCARG2x, FCARG2x, #sizeof(Bucket) 14484 | b <1 14485 if (MAY_BE_PACKED(op1_info)) { 14486 |2: 14487 } 14488 } 14489 if (MAY_BE_PACKED(op1_info)) { 14490 | // p = fe_ht->arPacked + pos; 14491 || ZEND_ASSERT(sizeof(zval) == 16); 14492 | mov FCARG2w, REG0w 14493 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arPacked)] 14494 | add FCARG2x, TMP1, FCARG2x, lsl #4 14495 |1: 14496 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 14497 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, nNumUsed)] 14498 | cmp TMP1w, REG0w 14499 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 14500 | // ZEND_VM_CONTINUE(); 14501 if (exit_addr) { 14502 if (exit_opcode == ZEND_JMP) { 14503 | bls &exit_addr 14504 } else { 14505 | bls >4 14506 } 14507 } else { 14508 | bls =>target_label 14509 } 14510 | // pos++; 14511 | add REG0w, REG0w, #1 14512 | // value_type = Z_TYPE_INFO_P(value); 14513 | // if (EXPECTED(value_type != IS_UNDEF)) { 14514 if (!exit_addr || exit_opcode == ZEND_JMP) { 14515 | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >4, TMP1w 14516 } else { 14517 | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr, TMP1w 14518 } 14519 | // p++; 14520 | add FCARG2x, FCARG2x, #sizeof(zval) 14521 | b <1 14522 } 14523 14524 if (!exit_addr || exit_opcode == ZEND_JMP) { 14525 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 14526 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 14527 uint32_t val_info; 14528 14529 if (RETURN_VALUE_USED(opline)) { 14530 zend_jit_addr res_addr = RES_ADDR(); 14531 14532 if (MAY_BE_HASH(op1_info)) { 14533 |3: 14534 | // Z_FE_POS_P(array) = pos + 1; 14535 | MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 14536 14537 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) 14538 && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { 14539 | // if (!p->key) { 14540 | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] 14541 | cbz REG0, >2 14542 } 14543 if (op1_info & MAY_BE_ARRAY_KEY_STRING) { 14544 | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); 14545 | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] 14546 | SET_ZVAL_PTR res_addr, REG0, TMP1 14547 | ldr TMP1w, [REG0, #offsetof(zend_refcounted, gc.u.type_info)] 14548 | TST_32_WITH_CONST TMP1w, IS_STR_INTERNED, TMP2w 14549 | beq >1 14550 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 14551 | b >3 14552 |1: 14553 | GC_ADDREF REG0, TMP1w 14554 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2 14555 14556 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) || MAY_BE_PACKED(op1_info)) { 14557 | b >3 14558 |2: 14559 } 14560 } 14561 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 14562 | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); 14563 | ldr REG0, [FCARG2x, #offsetof(Bucket, h)] 14564 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 14565 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 14566 if (MAY_BE_PACKED(op1_info)) { 14567 | b >3 14568 } 14569 } 14570 } 14571 if (MAY_BE_PACKED(op1_info)) { 14572 |4: 14573 | // Z_FE_POS_P(array) = pos + 1; 14574 | MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 14575 | sub REG0w, REG0w, #1 14576 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 14577 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 14578 } 14579 |3: 14580 } else { 14581 |3: 14582 |4: 14583 | // Z_FE_POS_P(array) = pos + 1; 14584 | MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 14585 } 14586 14587 val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); 14588 if (val_info & MAY_BE_ARRAY) { 14589 val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 14590 } 14591 if (op1_info & MAY_BE_ARRAY_OF_REF) { 14592 val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | 14593 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 14594 } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14595 val_info |= MAY_BE_RC1 | MAY_BE_RCN; 14596 } 14597 14598 if (opline->op2_type == IS_CV) { 14599 | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); 14600 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { 14601 return 0; 14602 } 14603 } else { 14604 | // ZVAL_COPY(res, value); 14605 | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 14606 | TRY_ADDREF val_info, REG0w, FCARG1x, TMP1w 14607 } 14608 } else { 14609 |3: 14610 |4: 14611 } 14612 14613 return 1; 14614} 14615 14616static int zend_jit_fetch_constant(dasm_State **Dst, 14617 const zend_op *opline, 14618 const zend_op_array *op_array, 14619 zend_ssa *ssa, 14620 const zend_ssa_op *ssa_op, 14621 zend_jit_addr res_addr) 14622{ 14623 zval *zv = RT_CONSTANT(opline, opline->op2) + 1; 14624 zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 14625 uint32_t res_info = RES_INFO(); 14626 14627 | // c = CACHED_PTR(opline->extended_value); 14628 | ldr FCARG1x, EX->run_time_cache 14629 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG1x, opline->extended_value, TMP1 14630 | // if (c != NULL) 14631 | cbz REG0, >9 14632 if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) { 14633 | // if (!IS_SPECIAL_CACHE_VAL(c)) 14634 || ZEND_ASSERT(CACHE_SPECIAL == 1); 14635 | TST_64_WITH_ONE REG0 14636 | bne >9 14637 } 14638 |8: 14639 14640 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { 14641 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 14642 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 14643 int32_t exit_point; 14644 const void *exit_addr = NULL; 14645 14646 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 14647 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 14648 exit_point = zend_jit_trace_get_exit_point(opline+1, 0); 14649 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 14650 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14651 if (!exit_addr) { 14652 return 0; 14653 } 14654 res_info &= ~MAY_BE_GUARD; 14655 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 14656 14657 uint32_t type = concrete_type(res_info); 14658 14659 if (type < IS_STRING) { 14660 | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr, ZREG_TMP1 14661 } else { 14662 | GET_ZVAL_TYPE_INFO REG2w, const_addr, TMP1 14663 | IF_NOT_TYPE REG2w, type, &exit_addr 14664 } 14665 | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 14666 if (type < IS_STRING) { 14667 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 14668 | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 14669 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 14670 return 0; 14671 } 14672 } else { 14673 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 14674 | TRY_ADDREF res_info, REG2w, REG1, TMP1w 14675 } 14676 } else { 14677 | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 14678 | TRY_ADDREF MAY_BE_ANY, REG0w, REG1, TMP1w 14679 } 14680 14681 |.cold_code 14682 |9: 14683 | // SAVE_OPLINE(); 14684 | SET_EX_OPLINE opline, REG0 14685 | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); 14686 | LOAD_ADDR FCARG1x, zv 14687 | LOAD_32BIT_VAL FCARG2w, opline->op1.num 14688 | EXT_CALL zend_jit_get_constant, REG0 14689 | mov REG0, RETVALx 14690 | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); 14691 | cbnz REG0, <8 14692 | b ->exception_handler 14693 |.code 14694 return 1; 14695} 14696 14697static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 14698{ 14699 HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 14700 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14701 14702 ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); 14703 ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); 14704 14705 | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); 14706 | LOAD_ADDR FCARG1x, ht 14707 if (opline->op1_type != IS_CONST) { 14708 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14709 | EXT_CALL zend_hash_find, REG0 14710 } else { 14711 zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); 14712 | LOAD_ADDR FCARG2x, str 14713 | EXT_CALL zend_hash_find_known_hash, REG0 14714 } 14715 if (exit_addr) { 14716 if (smart_branch_opcode == ZEND_JMPZ) { 14717 | cbz RETVALx, &exit_addr 14718 } else { 14719 | cbnz RETVALx, &exit_addr 14720 } 14721 } else if (smart_branch_opcode) { 14722 if (smart_branch_opcode == ZEND_JMPZ) { 14723 | cbz RETVALx, =>target_label 14724 } else if (smart_branch_opcode == ZEND_JMPNZ) { 14725 | cbnz RETVALx, =>target_label 14726 } else { 14727 ZEND_UNREACHABLE(); 14728 } 14729 } else { 14730 | tst RETVALx, RETVALx 14731 | cset REG0w, ne 14732 | add REG0w, REG0w, #IS_FALSE 14733 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 14734 } 14735 14736 return 1; 14737} 14738 14739static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info) 14740{ 14741 uint32_t offset; 14742 14743 offset = (opline->opcode == ZEND_ROPE_INIT) ? 14744 opline->result.var : 14745 opline->op1.var + opline->extended_value * sizeof(zend_string*); 14746 14747 if (opline->op2_type == IS_CONST) { 14748 zval *zv = RT_CONSTANT(opline, opline->op2); 14749 zend_string *str; 14750 14751 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14752 str = Z_STR_P(zv); 14753 | LOAD_ADDR REG0, str 14754 | MEM_ACCESS_64_WITH_UOFFSET str, REG0, FP, offset, TMP1 14755 } else { 14756 zend_jit_addr op2_addr = OP2_ADDR(); 14757 14758 ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14759 14760 | GET_ZVAL_PTR REG1, op2_addr, TMP1 14761 | MEM_ACCESS_64_WITH_UOFFSET str, REG1, FP, offset, TMP1 14762 if (opline->op2_type == IS_CV) { 14763 | GET_ZVAL_TYPE_INFO REG0w, op2_addr, TMP1 14764 | TRY_ADDREF op2_info, REG0w, REG1, TMP1w 14765 } 14766 } 14767 14768 if (opline->opcode == ZEND_ROPE_END) { 14769 zend_jit_addr res_addr = RES_ADDR(); 14770 14771 | ADD_SUB_64_WITH_CONST add, FCARG1x, FP, opline->op1.var, TMP1 14772 | LOAD_32BIT_VAL FCARG2w, opline->extended_value 14773 | EXT_CALL zend_jit_rope_end, TMP1 14774 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 14775 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2 14776 } 14777 14778 return 1; 14779} 14780 14781static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) 14782{ 14783 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14784 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14785 14786 if (!exit_addr) { 14787 return 0; 14788 } 14789 | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 14790 14791 return 1; 14792} 14793 14794static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard) 14795{ 14796 zend_jit_addr var_addr = *var_addr_ptr; 14797 uint32_t var_info = *var_info_ptr; 14798 const void *exit_addr = NULL; 14799 14800 if (add_ref_guard || add_type_guard) { 14801 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14802 14803 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14804 if (!exit_addr) { 14805 return 0; 14806 } 14807 } 14808 14809 if (add_ref_guard) { 14810 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 14811 } 14812 if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { 14813 /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ 14814 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 14815 | LOAD_ZVAL_ADDR FCARG1x, var_addr 14816 } 14817 | EXT_CALL zend_jit_unref_helper, REG0 14818 } else { 14819 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 14820 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 14821 *var_addr_ptr = var_addr; 14822 } 14823 14824 if (var_type != IS_UNKNOWN) { 14825 var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); 14826 } 14827 if (add_type_guard 14828 && var_type != IS_UNKNOWN 14829 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 14830 | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr, ZREG_TMP1 14831 14832 ZEND_ASSERT(var_info & (1 << var_type)); 14833 if (var_type < IS_STRING) { 14834 var_info = (1 << var_type); 14835 } else if (var_type != IS_ARRAY) { 14836 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 14837 } else { 14838 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 14839 } 14840 14841 *var_info_ptr = var_info; 14842 } else { 14843 var_info &= ~MAY_BE_REF; 14844 *var_info_ptr = var_info; 14845 } 14846 *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */ 14847 14848 return 1; 14849} 14850 14851static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard) 14852{ 14853 zend_jit_addr var_addr = *var_addr_ptr; 14854 uint32_t var_info = *var_info_ptr; 14855 int32_t exit_point; 14856 const void *exit_addr; 14857 14858 if (add_indirect_guard) { 14859 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14860 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14861 14862 if (!exit_addr) { 14863 return 0; 14864 } 14865 | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr, ZREG_TMP1 14866 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 14867 } else { 14868 /* May be already loaded into FCARG1a or RAX by previous FETCH_OBJ_W/DIM_W */ 14869 if (opline->op1_type != IS_VAR || 14870 (opline-1)->result_type != IS_VAR || 14871 (opline-1)->result.var != opline->op1.var || 14872 (opline-1)->op1_type == IS_VAR || 14873 (opline-1)->op2_type == IS_VAR || 14874 (opline-1)->op2_type == IS_TMP_VAR) { 14875 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 14876 } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { 14877 | mov FCARG1x, REG0 14878 } 14879 } 14880 *var_info_ptr &= ~MAY_BE_INDIRECT; 14881 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14882 *var_addr_ptr = var_addr; 14883 14884 if (var_type != IS_UNKNOWN) { 14885 var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); 14886 } 14887 if (!(var_type & IS_TRACE_REFERENCE) 14888 && var_type != IS_UNKNOWN 14889 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 14890 exit_point = zend_jit_trace_get_exit_point(opline, 0); 14891 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14892 14893 if (!exit_addr) { 14894 return 0; 14895 } 14896 14897 | IF_NOT_Z_TYPE FCARG1x, var_type, &exit_addr, TMP1w 14898 14899 //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); 14900 ZEND_ASSERT(var_info & (1 << var_type)); 14901 if (var_type < IS_STRING) { 14902 var_info = (1 << var_type); 14903 } else if (var_type != IS_ARRAY) { 14904 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 14905 } else { 14906 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 14907 } 14908 14909 *var_info_ptr = var_info; 14910 } 14911 14912 return 1; 14913} 14914 14915static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) 14916{ 14917 if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { 14918 return 0; 14919 } 14920 14921 switch (opline->opcode) { 14922 case ZEND_QM_ASSIGN: 14923 case ZEND_SEND_VAR: 14924 case ZEND_ASSIGN: 14925 case ZEND_PRE_INC: 14926 case ZEND_PRE_DEC: 14927 case ZEND_POST_INC: 14928 case ZEND_POST_DEC: 14929 return 1; 14930 case ZEND_ADD: 14931 case ZEND_SUB: 14932 case ZEND_MUL: 14933 case ZEND_BW_OR: 14934 case ZEND_BW_AND: 14935 case ZEND_BW_XOR: 14936 case ZEND_SL: 14937 case ZEND_SR: 14938 if (def_var == ssa_op->result_def && 14939 use_var == ssa_op->op1_use) { 14940 return 1; 14941 } 14942 break; 14943 default: 14944 break; 14945 } 14946 return 0; 14947} 14948 14949static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) 14950{ 14951 uint32_t op1_info, op2_info; 14952 14953 switch (opline->opcode) { 14954 case ZEND_SEND_VAR: 14955 case ZEND_SEND_VAL: 14956 case ZEND_SEND_VAL_EX: 14957 return (opline->op2_type != IS_CONST); 14958 case ZEND_QM_ASSIGN: 14959 case ZEND_IS_SMALLER: 14960 case ZEND_IS_SMALLER_OR_EQUAL: 14961 case ZEND_IS_EQUAL: 14962 case ZEND_IS_NOT_EQUAL: 14963 case ZEND_IS_IDENTICAL: 14964 case ZEND_IS_NOT_IDENTICAL: 14965 case ZEND_CASE: 14966 return 1; 14967 case ZEND_RETURN: 14968 return (op_array->type != ZEND_EVAL_CODE && op_array->function_name); 14969 case ZEND_ASSIGN: 14970 op1_info = OP1_INFO(); 14971 op2_info = OP2_INFO(); 14972 return 14973 opline->op1_type == IS_CV && 14974 !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) && 14975 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); 14976 case ZEND_ADD: 14977 case ZEND_SUB: 14978 case ZEND_MUL: 14979 op1_info = OP1_INFO(); 14980 op2_info = OP2_INFO(); 14981 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); 14982 case ZEND_BW_OR: 14983 case ZEND_BW_AND: 14984 case ZEND_BW_XOR: 14985 case ZEND_SL: 14986 case ZEND_SR: 14987 case ZEND_MOD: 14988 op1_info = OP1_INFO(); 14989 op2_info = OP2_INFO(); 14990 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); 14991 case ZEND_PRE_INC: 14992 case ZEND_PRE_DEC: 14993 case ZEND_POST_INC: 14994 case ZEND_POST_DEC: 14995 op1_info = OP1_INFO(); 14996 op2_info = OP1_DEF_INFO(); 14997 return opline->op1_type == IS_CV 14998 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)) 14999 && (op2_info & MAY_BE_LONG); 15000 case ZEND_STRLEN: 15001 op1_info = OP1_INFO(); 15002 return (opline->op1_type & (IS_CV|IS_CONST)) 15003 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING; 15004 case ZEND_COUNT: 15005 op1_info = OP1_INFO(); 15006 return (opline->op1_type & (IS_CV|IS_CONST)) 15007 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY; 15008 case ZEND_JMPZ: 15009 case ZEND_JMPNZ: 15010 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 15011 if (!ssa->cfg.map) { 15012 return 0; 15013 } 15014 if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start && 15015 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { 15016 return 0; 15017 } 15018 } 15019 ZEND_FALLTHROUGH; 15020 case ZEND_BOOL: 15021 case ZEND_BOOL_NOT: 15022 case ZEND_JMPZ_EX: 15023 case ZEND_JMPNZ_EX: 15024 return 1; 15025 case ZEND_FETCH_CONSTANT: 15026 return 1; 15027 case ZEND_FETCH_DIM_R: 15028 op1_info = OP1_INFO(); 15029 op2_info = OP2_INFO(); 15030 if (trace 15031 && trace->op1_type != IS_UNKNOWN 15032 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { 15033 op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); 15034 } 15035 return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && 15036 (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && 15037 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || 15038 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && 15039 (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); 15040 } 15041 return 0; 15042} 15043 15044static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) 15045{ 15046 if (ssa->vars[var].no_val) { 15047 /* we don't need the value */ 15048 return 0; 15049 } 15050 15051 if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { 15052 /* Disable global register allocation, 15053 * register allocation for SSA variables connected through Phi functions 15054 */ 15055 if (ssa->vars[var].definition_phi) { 15056 return 0; 15057 } 15058 if (ssa->vars[var].phi_use_chain) { 15059 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; 15060 do { 15061 if (!ssa->vars[phi->ssa_var].no_val) { 15062 return 0; 15063 } 15064 phi = zend_ssa_next_use_phi(ssa, var, phi); 15065 } while (phi); 15066 } 15067 } 15068 15069 if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && 15070 ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { 15071 /* bad type */ 15072 return 0; 15073 } 15074 15075 return 1; 15076} 15077 15078static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) 15079{ 15080 if (!zend_jit_var_supports_reg(ssa, var)) { 15081 return 0; 15082 } 15083 15084 if (ssa->vars[var].definition >= 0) { 15085 uint32_t def = ssa->vars[var].definition; 15086 if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { 15087 return 0; 15088 } 15089 } 15090 15091 if (ssa->vars[var].use_chain >= 0) { 15092 int use = ssa->vars[var].use_chain; 15093 15094 do { 15095 if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && 15096 !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { 15097 return 0; 15098 } 15099 use = zend_ssa_next_use(ssa->ops, var, use); 15100 } while (use >= 0); 15101 } 15102 15103 return 1; 15104} 15105 15106static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 15107{ 15108 uint32_t op1_info, op2_info; 15109 15110 switch (opline->opcode) { 15111 case ZEND_FETCH_DIM_R: 15112 op1_info = OP1_INFO(); 15113 op2_info = OP2_INFO(); 15114 if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && 15115 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || 15116 ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && 15117 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { 15118 return ZEND_REGSET(ZREG_FCARG1); 15119 } 15120 break; 15121 default: 15122 break; 15123 } 15124 15125 return ZEND_REGSET_EMPTY; 15126} 15127 15128static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 15129{ 15130 uint32_t op1_info, op2_info, res_info; 15131 zend_regset regset = ZEND_REGSET_SCRATCH; 15132 15133 switch (opline->opcode) { 15134 case ZEND_NOP: 15135 case ZEND_OP_DATA: 15136 case ZEND_JMP: 15137 case ZEND_RETURN: 15138 regset = ZEND_REGSET_EMPTY; 15139 break; 15140 case ZEND_QM_ASSIGN: 15141 if (ssa_op->op1_def == current_var || 15142 ssa_op->result_def == current_var) { 15143 regset = ZEND_REGSET_EMPTY; 15144 break; 15145 } 15146 /* break missing intentionally */ 15147 case ZEND_SEND_VAL: 15148 case ZEND_SEND_VAL_EX: 15149 if (opline->op2_type == IS_CONST) { 15150 break; 15151 } 15152 if (ssa_op->op1_use == current_var) { 15153 regset = ZEND_REGSET(ZREG_REG0); 15154 break; 15155 } 15156 op1_info = OP1_INFO(); 15157 if (!(op1_info & MAY_BE_UNDEF)) { 15158 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15159 regset = ZEND_REGSET(ZREG_FPR0); 15160 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15161 regset = ZEND_REGSET(ZREG_REG0); 15162 } else { 15163 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); 15164 } 15165 } 15166 break; 15167 case ZEND_SEND_VAR: 15168 if (opline->op2_type == IS_CONST) { 15169 break; 15170 } 15171 if (ssa_op->op1_use == current_var || 15172 ssa_op->op1_def == current_var) { 15173 regset = ZEND_REGSET_EMPTY; 15174 break; 15175 } 15176 op1_info = OP1_INFO(); 15177 if (!(op1_info & MAY_BE_UNDEF)) { 15178 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15179 regset = ZEND_REGSET(ZREG_FPR0); 15180 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15181 } else { 15182 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); 15183 if (op1_info & MAY_BE_REF) { 15184 ZEND_REGSET_INCL(regset, ZREG_REG1); 15185 } 15186 } 15187 } 15188 break; 15189 case ZEND_ASSIGN: 15190 if (ssa_op->op2_use == current_var || 15191 ssa_op->op2_def == current_var || 15192 ssa_op->op1_def == current_var || 15193 ssa_op->result_def == current_var) { 15194 regset = ZEND_REGSET_EMPTY; 15195 break; 15196 } 15197 op1_info = OP1_INFO(); 15198 op2_info = OP2_INFO(); 15199 if (opline->op1_type == IS_CV 15200 && !(op2_info & MAY_BE_UNDEF) 15201 && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 15202 if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15203 regset = ZEND_REGSET(ZREG_FPR0); 15204 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15205 regset = ZEND_REGSET(ZREG_REG0); 15206 } else { 15207 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); 15208 } 15209 } 15210 break; 15211 case ZEND_PRE_INC: 15212 case ZEND_PRE_DEC: 15213 case ZEND_POST_INC: 15214 case ZEND_POST_DEC: 15215 if (ssa_op->op1_use == current_var || 15216 ssa_op->op1_def == current_var || 15217 ssa_op->result_def == current_var) { 15218 regset = ZEND_REGSET_EMPTY; 15219 break; 15220 } 15221 op1_info = OP1_INFO(); 15222 if (opline->op1_type == IS_CV 15223 && (op1_info & MAY_BE_LONG) 15224 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15225 regset = ZEND_REGSET_EMPTY; 15226 if (op1_info & MAY_BE_DOUBLE) { 15227 regset = ZEND_REGSET(ZREG_FPR0); 15228 } 15229 if (opline->result_type != IS_UNUSED && (op1_info & MAY_BE_LONG)) { 15230 ZEND_REGSET_INCL(regset, ZREG_REG1); 15231 } 15232 } 15233 break; 15234 case ZEND_ADD: 15235 case ZEND_SUB: 15236 case ZEND_MUL: 15237 op1_info = OP1_INFO(); 15238 op2_info = OP2_INFO(); 15239 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 15240 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15241 15242 regset = ZEND_REGSET_EMPTY; 15243 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 15244 if (ssa_op->result_def != current_var && 15245 (ssa_op->op1_use != current_var || !last_use)) { 15246 ZEND_REGSET_INCL(regset, ZREG_REG0); 15247 } 15248 res_info = RES_INFO(); 15249 if (res_info & MAY_BE_DOUBLE) { 15250 ZEND_REGSET_INCL(regset, ZREG_REG0); 15251 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15252 ZEND_REGSET_INCL(regset, ZREG_FPR1); 15253 } else if (res_info & MAY_BE_GUARD) { 15254 ZEND_REGSET_INCL(regset, ZREG_REG0); 15255 } 15256 } 15257 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 15258 if (ssa_op->result_def != current_var) { 15259 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15260 } 15261 } 15262 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 15263 if (zend_is_commutative(opline->opcode)) { 15264 if (ssa_op->result_def != current_var) { 15265 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15266 } 15267 } else { 15268 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15269 if (ssa_op->result_def != current_var && 15270 (ssa_op->op1_use != current_var || !last_use)) { 15271 ZEND_REGSET_INCL(regset, ZREG_FPR1); 15272 } 15273 } 15274 } 15275 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 15276 if (ssa_op->result_def != current_var && 15277 (ssa_op->op1_use != current_var || !last_use) && 15278 (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { 15279 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15280 } 15281 } 15282 } 15283 break; 15284 case ZEND_BW_OR: 15285 case ZEND_BW_AND: 15286 case ZEND_BW_XOR: 15287 case ZEND_SL: 15288 case ZEND_SR: 15289 case ZEND_MOD: 15290 op1_info = OP1_INFO(); 15291 op2_info = OP2_INFO(); 15292 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 15293 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 15294 regset = ZEND_REGSET_EMPTY; 15295 if (ssa_op->result_def != current_var && 15296 (ssa_op->op1_use != current_var || !last_use)) { 15297 ZEND_REGSET_INCL(regset, ZREG_REG0); 15298 } 15299 } 15300 break; 15301 case ZEND_IS_SMALLER: 15302 case ZEND_IS_SMALLER_OR_EQUAL: 15303 case ZEND_IS_EQUAL: 15304 case ZEND_IS_NOT_EQUAL: 15305 case ZEND_IS_IDENTICAL: 15306 case ZEND_IS_NOT_IDENTICAL: 15307 case ZEND_CASE: 15308 op1_info = OP1_INFO(); 15309 op2_info = OP2_INFO(); 15310 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 15311 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15312 regset = ZEND_REGSET_EMPTY; 15313 if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { 15314 ZEND_REGSET_INCL(regset, ZREG_REG0); 15315 } 15316 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && 15317 opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { 15318 if (ssa_op->op1_use != current_var && 15319 ssa_op->op2_use != current_var) { 15320 ZEND_REGSET_INCL(regset, ZREG_REG0); 15321 } 15322 } 15323 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 15324 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15325 } 15326 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 15327 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15328 } 15329 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 15330 if (ssa_op->op1_use != current_var && 15331 ssa_op->op2_use != current_var) { 15332 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15333 } 15334 } 15335 } 15336 break; 15337 case ZEND_BOOL: 15338 case ZEND_BOOL_NOT: 15339 case ZEND_JMPZ: 15340 case ZEND_JMPNZ: 15341 case ZEND_JMPZ_EX: 15342 case ZEND_JMPNZ_EX: 15343 op1_info = OP1_INFO(); 15344 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15345 regset = ZEND_REGSET_EMPTY; 15346 if (op1_info & MAY_BE_DOUBLE) { 15347 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15348 } 15349 if (opline->opcode == ZEND_BOOL || 15350 opline->opcode == ZEND_BOOL_NOT || 15351 opline->opcode == ZEND_JMPZ_EX || 15352 opline->opcode == ZEND_JMPNZ_EX) { 15353 ZEND_REGSET_INCL(regset, ZREG_REG0); 15354 } 15355 } 15356 break; 15357 case ZEND_DO_UCALL: 15358 case ZEND_DO_FCALL: 15359 case ZEND_DO_FCALL_BY_NAME: 15360 case ZEND_INCLUDE_OR_EVAL: 15361 case ZEND_GENERATOR_CREATE: 15362 case ZEND_YIELD: 15363 case ZEND_YIELD_FROM: 15364 regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); 15365 break; 15366 default: 15367 break; 15368 } 15369 15370 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 15371 if (ssa_op == ssa->ops 15372 && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL 15373 && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { 15374 ZEND_REGSET_INCL(regset, ZREG_REG0); 15375 ZEND_REGSET_INCL(regset, ZREG_REG1); 15376 } 15377 } 15378 15379 return regset; 15380} 15381 15382static size_t dasm_venners_size = 0; 15383void **dasm_labels_veneers = NULL; 15384 15385static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset) 15386{ 15387 void *veneer; 15388 ptrdiff_t na; 15389 int n, m; 15390 15391 /* try to reuse veneers for global labels */ 15392 if ((ins >> 16) == DASM_REL_LG 15393 && *(b-1) < 0 15394 && dasm_labels_veneers[-*(b-1)]) { 15395 15396 veneer = dasm_labels_veneers[-*(b-1)]; 15397 na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; 15398 n = (int)na; 15399 15400 /* check if we can jump to veneer */ 15401 if ((ptrdiff_t)n != na) { 15402 /* pass */ 15403 } else if (!(ins & 0xf800)) { /* B, BL */ 15404 if ((n & 3) == 0 && ((n+0x08000000) >> 28) == 0) { 15405 return n; 15406 } 15407 } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ 15408 if ((n & 3) == 0 && ((n+0x00100000) >> 21) == 0) { 15409 return n; 15410 } 15411 } else if ((ins & 0x3000) == 0x2000) { /* ADR */ 15412 /* pass */ 15413 } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ 15414 /* pass */ 15415 } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ 15416 if ((n & 3) == 0 && ((n+0x00008000) >> 16) == 0) { 15417 return n; 15418 } 15419 } 15420 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 15421 && (ins >> 16) == DASM_REL_A) { 15422 ptrdiff_t addr = (((ptrdiff_t)(*(b-1))) << 32) | (unsigned int)(*(b-2)); 15423 15424 if ((void*)addr >= dasm_buf && (void*)addr < dasm_end) { 15425 uint32_t exit_point = zend_jit_trace_find_exit_point((void*)addr); 15426 zend_jit_trace_info *t = zend_jit_get_current_trace_info(); 15427 15428 if (exit_point != (uint32_t)-1) { 15429 /* Use exit points table */ 15430 15431 ZEND_ASSERT(exit_point < t->exit_count); 15432 15433 veneer = (char*)buffer + dasm_getpclabel(&Dst, 1) - (t->exit_count - exit_point) * 4; 15434 na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; 15435 n = (int)na; 15436 15437 /* check if we can jump to veneer */ 15438 if ((ptrdiff_t)n != na) { 15439 ZEND_ASSERT(0); 15440 return 0; 15441 } else if (!(ins & 0xf800)) { /* B, BL */ 15442 if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) { 15443 ZEND_ASSERT(0); 15444 return 0; 15445 } 15446 } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ 15447 if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) { 15448 ZEND_ASSERT(0); 15449 return 0; 15450 } 15451 } else if ((ins & 0x3000) == 0x2000) { /* ADR */ 15452 ZEND_ASSERT(0); 15453 return 0; 15454 } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ 15455 ZEND_ASSERT(0); 15456 return 0; 15457 } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ 15458 if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) { 15459 ZEND_ASSERT(0); 15460 return 0; 15461 } 15462 } else { 15463 ZEND_ASSERT(0); 15464 return 0; 15465 } 15466 return n; 15467 } 15468 } 15469 } 15470 15471 veneer = (char*)buffer + (Dst->codesize + dasm_venners_size); 15472 15473 if (veneer > dasm_end) { 15474 return 0; /* jit_buffer_size overflow */ 15475 } 15476 15477 na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; 15478 n = (int)na; 15479 15480 /* check if we can jump to veneer */ 15481 if ((ptrdiff_t)n != na) { 15482 ZEND_ASSERT(0); 15483 return 0; 15484 } else if (!(ins & 0xf800)) { /* B, BL */ 15485 if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) { 15486 ZEND_ASSERT(0); 15487 return 0; 15488 } 15489 } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ 15490 if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) { 15491 ZEND_ASSERT(0); 15492 return 0; 15493 } 15494 } else if ((ins & 0x3000) == 0x2000) { /* ADR */ 15495 ZEND_ASSERT(0); 15496 return 0; 15497 } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ 15498 ZEND_ASSERT(0); 15499 return 0; 15500 } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ 15501 if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) { 15502 ZEND_ASSERT(0); 15503 return 0; 15504 } 15505 } else if ((ins & 0x8000)) { /* absolute */ 15506 ZEND_ASSERT(0); 15507 return 0; 15508 } else { 15509 ZEND_ASSERT(0); 15510 return 0; 15511 } 15512 15513 // TODO: support for long veneers (above 128MB) ??? 15514 15515 /* check if we can use B to jump from veneer */ 15516 na = (ptrdiff_t)cp + offset - (ptrdiff_t)veneer - 4; 15517 m = (int)na; 15518 if ((ptrdiff_t)m != na) { 15519 ZEND_ASSERT(0); 15520 return 0; 15521 } else if ((m & 3) != 0 || ((m+0x08000000) >> 28) != 0) { 15522 ZEND_ASSERT(0); 15523 return 0; 15524 } 15525 15526 /* generate B instruction */ 15527 *(uint32_t*)veneer = 0x14000000 | ((m >> 2) & 0x03ffffff); 15528 dasm_venners_size += 4; 15529 15530 if ((ins >> 16) == DASM_REL_LG 15531 && *(b-1) < 0) { 15532 /* reuse this veneer for the future jumps to global label */ 15533 dasm_labels_veneers[-*(b-1)] = veneer; 15534 /* Dst->globals[*(b-1)] = veneer; */ 15535 15536#ifdef HAVE_DISASM 15537 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { 15538 const char *name = zend_jit_disasm_find_symbol((ptrdiff_t)cp + offset - 4, (int64_t *)(&offset)); 15539 15540 if (name && !offset) { 15541 if (strstr(name, "@veneer") == NULL) { 15542 char *new_name; 15543 15544 zend_spprintf(&new_name, 0, "%s@veneer", name); 15545 zend_jit_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4); 15546 efree(new_name); 15547 } else { 15548 zend_jit_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4); 15549 } 15550 } 15551 } 15552#endif 15553 } 15554 15555 return n; 15556} 15557 15558/* 15559 * Local variables: 15560 * tab-width: 4 15561 * c-basic-offset: 4 15562 * indent-tabs-mode: t 15563 * End: 15564 */ 15565
