1/* 2 * +----------------------------------------------------------------------+ 3 * | Zend JIT | 4 * +----------------------------------------------------------------------+ 5 * | Copyright (c) The PHP Group | 6 * +----------------------------------------------------------------------+ 7 * | This source file is subject to version 3.01 of the PHP license, | 8 * | that is bundled with this package in the file LICENSE, and is | 9 * | available through the world-wide-web at the following url: | 10 * | https://www.php.net/license/3_01.txt | 11 * | If you did not receive a copy of the PHP license and are unable to | 12 * | obtain it through the world-wide-web, please send a note to | 13 * | license@php.net so we can mail you a copy immediately. | 14 * +----------------------------------------------------------------------+ 15 * | Authors: Dmitry Stogov <dmitry@php.net> | 16 * | Xinchen Hui <laruence@php.net> | 17 * | Hao Sun <hao.sun@arm.com> | 18 * +----------------------------------------------------------------------+ 19 */ 20 21|.arch arm64 22 23|.define FP, x27 24|.define IP, x28 25|.define IPl, w28 26|.define RX, x28 // the same as VM IP reused as a general purpose reg 27|.define LR, x30 28|.define CARG1, x0 29|.define CARG2, x1 30|.define CARG3, x2 31|.define CARG4, x3 32|.define CARG5, x4 33|.define CARG6, x5 34|.define CARG1w, w0 35|.define CARG2w, w1 36|.define CARG3w, w2 37|.define CARG4w, w3 38|.define CARG5w, w4 39|.define CARG6w, w5 40|.define RETVALx, x0 41|.define RETVALw, w0 42|.define FCARG1x, x0 43|.define FCARG1w, w0 44|.define FCARG2x, x1 45|.define FCARG2w, w1 46|.define SPAD, 0x20 // padding for CPU stack alignment 47|.define NR_SPAD, 0x30 // padding for CPU stack alignment 48|.define T3, [sp, #0x28] // Used to store old value of IP (CALL VM only) 49|.define T2, [sp, #0x20] // Used to store old value of FP (CALL VM only) 50|.define T1, [sp, #0x10] 51 52// We use REG0/1/2 and FPR0/1 to replace r0/1/2 and xmm0/1 in the x86 implementation. 53// Scratch registers 54|.define REG0, x8 55|.define REG0w, w8 56|.define REG1, x9 57|.define REG1w, w9 58|.define REG2, x10 59|.define REG2w, w10 60|.define FPR0, d0 61|.define FPR1, d1 62 63|.define ZREG_REG0, ZREG_X8 64|.define ZREG_REG1, ZREG_X9 65|.define ZREG_REG2, ZREG_X10 66|.define ZREG_FPR0, ZREG_V0 67|.define ZREG_FPR1, ZREG_V1 68 69// Temporaries, not preserved across calls 70|.define TMP1, x15 71|.define TMP1w, w15 72|.define TMP2, x16 73|.define TMP2w, w16 74|.define TMP3, x17 // TODO: remember about hard-coded: mrs TMP3, tpidr_el0 75|.define TMP3w, w17 76|.define FPTMP, d16 77 78|.define ZREG_TMP1, ZREG_X15 79|.define ZREG_TMP2, ZREG_X16 80|.define ZREG_TMP3, ZREG_X17 81|.define ZREG_FPTMP, ZREG_V16 82 83|.define HYBRID_SPAD, 32 // padding for stack alignment 84 85#define TMP_ZVAL_OFFSET 16 86#define DASM_ALIGNMENT 16 87 88const char* zend_reg_name[] = { 89 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", 90 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", 91 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", 92 "x24", "x25", "x26", "x27", "x28", "x29", "x30", "sp", 93 "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", 94 "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15", 95 "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23", 96 "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31" 97}; 98 99#define ZREG_FCARG1 ZREG_X0 100#define ZREG_FCARG2 ZREG_X1 101 102|.type EX, zend_execute_data, FP 103|.type OP, zend_op 104|.type ZVAL, zval 105|.actionlist dasm_actions 106|.globals zend_lb 107|.section code, cold_code, jmp_table 108 109static void* dasm_labels[zend_lb_MAX]; 110 111#if ZTS 112static size_t tsrm_ls_cache_tcb_offset = 0; 113static size_t tsrm_tls_index = 0; 114static size_t tsrm_tls_offset = 0; 115# ifdef __APPLE__ 116struct TLVDescriptor { 117 void* (*thunk)(struct TLVDescriptor*); 118 uint64_t key; 119 uint64_t offset; 120}; 121typedef struct TLVDescriptor TLVDescriptor; 122# elif defined(__FreeBSD__) 123/* https://github.com/freebsd/freebsd-src/blob/c52ca7dd09066648b1cc40f758289404d68ab886/libexec/rtld-elf/aarch64/reloc.c#L180-L184 */ 124typedef struct TLSDescriptor { 125 void* thunk; 126 int index; 127 size_t offset; 128} TLSDescriptor; 129# endif 130#endif 131 132#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) 133 134/* Encoding of immediate. */ 135#define MAX_IMM12 0xfff // maximum value for imm12 136#define MAX_IMM16 0xffff // maximum value for imm16 137#define MOVZ_IMM MAX_IMM16 // movz insn 138#define LDR_STR_PIMM64 (MAX_IMM12*8) // ldr/str insn for 64-bit register: pimm is imm12 * 8 139#define LDR_STR_PIMM32 (MAX_IMM12*4) // ldr/str insn for 32-bit register: pimm is imm12 * 4 140#define LDRB_STRB_PIMM MAX_IMM12 // ldrb/strb insn 141 142#define B_IMM (1<<27) // signed imm26 * 4 143#define ADR_IMM (1<<20) // signed imm21 144#define ADRP_IMM (1LL<<32) // signed imm21 * 4096 145 146static bool arm64_may_use_b(const void *addr) 147{ 148 if (addr >= dasm_buf && addr < dasm_end) { 149 return (((char*)dasm_end - (char*)dasm_buf) < B_IMM); 150 } else if (addr >= dasm_end) { 151 return (((char*)addr - (char*)dasm_buf) < B_IMM); 152 } else if (addr < dasm_buf) { 153 return (((char*)dasm_end - (char*)addr) < B_IMM); 154 } 155 return 0; 156} 157 158static bool arm64_may_use_adr(const void *addr) 159{ 160 if (addr >= dasm_buf && addr < dasm_end) { 161 return (((char*)dasm_end - (char*)dasm_buf) < ADR_IMM); 162 } else if (addr >= dasm_end) { 163 return (((char*)addr - (char*)dasm_buf) < ADR_IMM); 164 } else if (addr < dasm_buf) { 165 return (((char*)dasm_end - (char*)addr) < ADR_IMM); 166 } 167 return 0; 168} 169 170static bool arm64_may_use_adrp(const void *addr) 171{ 172 if (addr >= dasm_buf && addr < dasm_end) { 173 return (((char*)dasm_end - (char*)dasm_buf) < ADRP_IMM); 174 } else if (addr >= dasm_end) { 175 return (((char*)addr - (char*)dasm_buf) < ADRP_IMM); 176 } else if (addr < dasm_buf) { 177 return (((char*)dasm_end - (char*)addr) < ADRP_IMM); 178 } 179 return 0; 180} 181 182/* Determine whether "val" falls into two allowed ranges: 183 * Range 1: [0, 0xfff] 184 * Range 2: LSL #12 to Range 1 185 * Used to guard the immediate encoding for add/adds/sub/subs/cmp/cmn instructions. */ 186static bool arm64_may_encode_imm12(const int64_t val) 187{ 188 return (val >= 0 && (val <= MAX_IMM12 || !(val & 0xffffffffff000fff))); 189} 190 191/* Determine whether an immediate value can be encoded as the immediate operand of logical instructions. */ 192static bool logical_immediate_p(uint64_t value, uint32_t reg_size) 193{ 194 /* fast path: power of two */ 195 if (value > 0 && !(value & (value - 1))) { 196 return true; 197 } 198 199 if (reg_size == 32) { 200 if (dasm_imm13((uint32_t)value, (uint32_t)value) != -1) { 201 return true; 202 } 203 } else if (reg_size == 64) { 204 if (dasm_imm13((uint32_t)value, (uint32_t)(value >> 32)) != -1) { 205 return true; 206 } 207 } else { 208 ZEND_UNREACHABLE(); 209 } 210 211 return false; 212} 213 214/* Not Implemented Yet */ 215|.macro NIY 216|| //ZEND_ASSERT(0); 217| brk #0 218|.endmacro 219 220|.macro NIY_STUB 221|| //ZEND_ASSERT(0); 222| brk #0 223|.endmacro 224 225|.macro ADD_HYBRID_SPAD 226||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 227| add sp, sp, # HYBRID_SPAD 228||#endif 229|.endmacro 230 231|.macro SUB_HYBRID_SPAD 232||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 233| sub sp, sp, # HYBRID_SPAD 234||#endif 235|.endmacro 236 237/* Move address into register. TODO: Support 52-bit address */ 238|.macro LOAD_ADDR, reg, addr 239| // 48-bit virtual address 240|| if (((uintptr_t)(addr)) == 0) { 241| mov reg, xzr 242|| } else if (((uintptr_t)(addr)) <= MOVZ_IMM) { 243| movz reg, #((uint64_t)(addr)) 244|| } else if (arm64_may_use_adr((void*)(addr))) { 245| adr reg, &addr 246|| } else if (arm64_may_use_adrp((void*)(addr))) { 247| adrp reg, &(((uintptr_t)(addr))) 248|| if (((uintptr_t)(addr)) & 0xfff) { 249| add reg, reg, #(((uintptr_t)(addr)) & 0xfff) 250|| } 251|| } else if ((uintptr_t)(addr) & 0xffff) { 252| movz reg, #((uintptr_t)(addr) & 0xffff) 253|| if (((uintptr_t)(addr) >> 16) & 0xffff) { 254| movk reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 255|| } 256|| if (((uintptr_t)(addr) >> 32) & 0xffff) { 257| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 258|| } 259|| } else if (((uintptr_t)(addr) >> 16) & 0xffff) { 260| movz reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 261|| if (((uintptr_t)(addr) >> 32) & 0xffff) { 262| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 263|| } 264|| } else { 265| movz reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 266|| } 267|.endmacro 268 269/* Move 32-bit immediate value into register. */ 270|.macro LOAD_32BIT_VAL, reg, val 271|| if (((uint32_t)(val)) <= MOVZ_IMM) { 272| movz reg, #((uint32_t)(val)) 273|| } else if (((uint32_t)(val) & 0xffff)) { 274| movz reg, #((uint32_t)(val) & 0xffff) 275|| if ((((uint32_t)(val) >> 16) & 0xffff)) { 276| movk reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 277|| } 278|| } else { 279| movz reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 280|| } 281|.endmacro 282 283/* Move 64-bit immediate value into register. */ 284|.macro LOAD_64BIT_VAL, reg, val 285|| if (((uint64_t)(val)) == 0) { 286| mov reg, xzr 287|| } else if (((uint64_t)(val)) <= MOVZ_IMM) { 288| movz reg, #((uint64_t)(val)) 289|| } else if (~((uint64_t)(val)) <= MOVZ_IMM) { 290| movn reg, #(~((uint64_t)(val))) 291|| } else if ((uint64_t)(val) & 0xffff) { 292| movz reg, #((uint64_t)(val) & 0xffff) 293|| if (((uint64_t)(val) >> 16) & 0xffff) { 294| movk reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 295|| } 296|| if (((uint64_t)(val) >> 32) & 0xffff) { 297| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 298|| } 299|| if ((((uint64_t)(val) >> 48) & 0xffff)) { 300| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 301|| } 302|| } else if (((uint64_t)(val) >> 16) & 0xffff) { 303| movz reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 304|| if (((uint64_t)(val) >> 32) & 0xffff) { 305| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 306|| } 307|| if ((((uint64_t)(val) >> 48) & 0xffff)) { 308| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 309|| } 310|| } else if (((uint64_t)(val) >> 32) & 0xffff) { 311| movz reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 312|| if ((((uint64_t)(val) >> 48) & 0xffff)) { 313| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 314|| } 315|| } else { 316| movz reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 317|| } 318|.endmacro 319 320/* Extract the low 8 bits from 'src_reg' into 'dst_reg'. 321 * Note: 0xff can be encoded as imm for 'and' instruction. */ 322|.macro GET_LOW_8BITS, dst_reg, src_reg 323| and dst_reg, src_reg, #0xff 324|.endmacro 325 326/* Bitwise operation with immediate. 'bw_ins' can be and/orr/eor/ands. 327 * 32-bit and 64-bit registers are distinguished. */ 328|.macro BW_OP_32_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg 329|| if (val == 0) { 330| bw_ins dst_reg, src_reg1, wzr 331|| } else if (logical_immediate_p((uint32_t)val, 32)) { 332| bw_ins dst_reg, src_reg1, #val 333|| } else { 334| LOAD_32BIT_VAL tmp_reg, val 335| bw_ins dst_reg, src_reg1, tmp_reg 336|| } 337|.endmacro 338 339|.macro BW_OP_64_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg 340|| if (val == 0) { 341| bw_ins dst_reg, src_reg1, xzr 342|| } else if (logical_immediate_p(val, 64)) { 343| bw_ins dst_reg, src_reg1, #val 344|| } else { 345| LOAD_64BIT_VAL tmp_reg, val 346| bw_ins dst_reg, src_reg1, tmp_reg 347|| } 348|.endmacro 349 350/* Test bits 'tst' with immediate. 32-bit and 64-bit registers are distinguished. */ 351|.macro TST_32_WITH_CONST, reg, val, tmp_reg 352|| if (val == 0) { 353| tst reg, wzr 354|| } else if (logical_immediate_p((uint32_t)val, 32)) { 355| tst reg, #val 356|| } else { 357| LOAD_32BIT_VAL tmp_reg, val 358| tst reg, tmp_reg 359|| } 360|.endmacro 361 362|.macro TST_64_WITH_CONST, reg, val, tmp_reg 363|| if (val == 0) { 364| tst reg, xzr 365|| } else if (logical_immediate_p(val, 64)) { 366| tst reg, #val 367|| } else { 368| LOAD_64BIT_VAL tmp_reg, val 369| tst reg, tmp_reg 370|| } 371|.endmacro 372 373/* Test bits between 64-bit register with constant 1. */ 374|.macro TST_64_WITH_ONE, reg 375| tst reg, #1 376|.endmacro 377 378/* Compare a register value with immediate. 32-bit and 64-bit registers are distinguished. 379 * Note: Comparing 64-bit register with 32-bit immediate is handled in CMP_64_WITH_CONST_32. */ 380|.macro CMP_32_WITH_CONST, reg, val, tmp_reg 381|| if (val == 0) { 382| cmp reg, wzr 383|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 384| cmp reg, #val 385|| } else if (arm64_may_encode_imm12((int64_t)(-val))) { 386| cmn reg, #-val 387|| } else { 388| LOAD_32BIT_VAL tmp_reg, val 389| cmp reg, tmp_reg 390|| } 391|.endmacro 392 393|.macro CMP_64_WITH_CONST_32, reg, val, tmp_reg 394|| if (val == 0) { 395| cmp reg, xzr 396|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 397| cmp reg, #val 398|| } else if (arm64_may_encode_imm12((int64_t)(-val))) { 399| cmn reg, #-val 400|| } else { 401| LOAD_32BIT_VAL tmp_reg, val 402| cmp reg, tmp_reg 403|| } 404|.endmacro 405 406|.macro CMP_64_WITH_CONST, reg, val, tmp_reg 407|| if (val == 0) { 408| cmp reg, xzr 409|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 410| cmp reg, #val 411|| } else if (arm64_may_encode_imm12((int64_t)(-val))) { 412| cmn reg, #-val 413|| } else { 414| LOAD_64BIT_VAL tmp_reg, val 415| cmp reg, tmp_reg 416|| } 417|.endmacro 418 419/* Add/sub a register value with immediate. 'add_sub_ins' can be add/sub/adds/subs. 420 * 32-bit and 64-bit registers are distinguished. 421 * Note: Case of 64-bit register with 32-bit immediate is handled in ADD_SUB_64_WITH_CONST_32. */ 422|.macro ADD_SUB_32_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg 423|| if (val == 0) { 424| add_sub_ins dst_reg, src_reg1, wzr 425|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 426| add_sub_ins dst_reg, src_reg1, #val 427|| } else { 428| LOAD_32BIT_VAL tmp_reg, val 429| add_sub_ins dst_reg, src_reg1, tmp_reg 430|| } 431|.endmacro 432 433|.macro ADD_SUB_64_WITH_CONST_32, add_sub_ins, dst_reg, src_reg1, val, tmp_reg 434|| if (val == 0) { 435| add_sub_ins dst_reg, src_reg1, xzr 436|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 437| add_sub_ins dst_reg, src_reg1, #val 438|| } else { 439| LOAD_32BIT_VAL tmp_reg, val 440| add_sub_ins dst_reg, src_reg1, tmp_reg 441|| } 442|.endmacro 443 444|.macro ADD_SUB_64_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg 445|| if (val == 0) { 446| add_sub_ins dst_reg, src_reg1, xzr 447|| } else if (arm64_may_encode_imm12((int64_t)(val))) { 448| add_sub_ins dst_reg, src_reg1, #val 449|| } else { 450| LOAD_64BIT_VAL tmp_reg, val 451| add_sub_ins dst_reg, src_reg1, tmp_reg 452|| } 453|.endmacro 454 455/* Memory access(load/store) with 32-bit 'offset'. 456 * Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register. 457 * 8-bit, 32-bit and 64-bit registers to be transferred are distinguished. 458 * Note: 'reg' can be used as 'tmp_reg' if 1) 'reg' is one GPR, AND 2) 'reg' != 'base_reg', AND 3) ins is 'ldr'. */ 459|.macro MEM_ACCESS_64_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg 460|| if (((uintptr_t)(offset)) > LDR_STR_PIMM64) { 461| LOAD_32BIT_VAL tmp_reg, offset 462| ldr_str_ins reg, [base_reg, tmp_reg] 463|| } else { 464| ldr_str_ins reg, [base_reg, #(offset)] 465|| } 466|.endmacro 467 468|.macro MEM_ACCESS_32_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg 469|| if (((uintptr_t)(offset)) > LDR_STR_PIMM32) { 470| LOAD_32BIT_VAL tmp_reg, offset 471| ldr_str_ins reg, [base_reg, tmp_reg] 472|| } else { 473| ldr_str_ins reg, [base_reg, #(offset)] 474|| } 475|.endmacro 476 477|.macro MEM_ACCESS_8_WITH_UOFFSET, ldrb_strb_ins, reg, base_reg, offset, tmp_reg 478|| if (((uintptr_t)(offset)) > LDRB_STRB_PIMM) { 479| LOAD_32BIT_VAL tmp_reg, offset 480| ldrb_strb_ins reg, [base_reg, tmp_reg] 481|| } else { 482| ldrb_strb_ins reg, [base_reg, #(offset)] 483|| } 484|.endmacro 485 486/* Memory access(load/store) with 64-bit 'offset'. 487 * Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register. */ 488|.macro MEM_ACCESS_64_WITH_UOFFSET_64, ldr_str_ins, reg, base_reg, offset, tmp_reg 489|| if (((uintptr_t)(offset)) > LDR_STR_PIMM64) { 490| LOAD_64BIT_VAL tmp_reg, offset 491| ldr_str_ins reg, [base_reg, tmp_reg] 492|| } else { 493| ldr_str_ins reg, [base_reg, #(offset)] 494|| } 495|.endmacro 496 497/* ZTS: get thread local variable "_tsrm_ls_cache" */ 498|.macro LOAD_TSRM_CACHE, reg 499||#ifdef __APPLE__ 500| .long 0xd53bd071 // TODO: hard-coded: mrs TMP3, tpidrro_el0 501| and TMP3, TMP3, #0xfffffffffffffff8 502| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, TMP3, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->key << 3), TMP1 503| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, reg, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->offset), TMP1 504||#else 505| .long 0xd53bd051 // TODO: hard-coded: mrs TMP3, tpidr_el0 506|| if (tsrm_ls_cache_tcb_offset == 0) { 507| ldr TMP3, [TMP3, #0] 508| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, TMP3, TMP3, tsrm_tls_index, TMP1 509| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, reg, TMP3, tsrm_tls_offset, TMP1 510|| } else { 511|| ZEND_ASSERT(tsrm_ls_cache_tcb_offset <= LDR_STR_PIMM64); 512| ldr reg, [TMP3, #tsrm_ls_cache_tcb_offset] 513|| } 514||#endif 515|.endmacro 516 517|.macro LOAD_ADDR_ZTS, reg, struct, field 518| .if ZTS 519| LOAD_TSRM_CACHE TMP3 520| ADD_SUB_64_WITH_CONST_32 add, reg, TMP3, (struct.._offset + offsetof(zend_..struct, field)), reg 521| .else 522| LOAD_ADDR reg, &struct.field 523| .endif 524|.endmacro 525 526/* Store address 'addr' into memory 'mem'. */ 527|.macro ADDR_STORE, mem, addr, tmp_reg 528| LOAD_ADDR tmp_reg, addr 529| str tmp_reg, mem 530|.endmacro 531 532/* Store a register value 'reg' into memory 'addr'. 533 * For ZTS mode, base register with unsigned offset variant is used, 534 * and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */ 535|.macro MEM_STORE, str_ins, reg, addr, tmp_reg 536|| if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) { 537| adr tmp_reg, &addr 538| str_ins reg, [tmp_reg] 539|| } else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) { 540| adrp tmp_reg, &(((uintptr_t)(addr))) 541| str_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)] 542|| } else { 543| LOAD_ADDR tmp_reg, addr 544| str_ins reg, [tmp_reg] 545|| } 546|.endmacro 547 548|.macro MEM_STORE_64_ZTS, str_ins, reg, struct, field, tmp_reg 549| .if ZTS 550| LOAD_TSRM_CACHE TMP3 551| MEM_ACCESS_64_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 552| .else 553| MEM_STORE str_ins, reg, &struct.field, tmp_reg 554| .endif 555|.endmacro 556 557|.macro MEM_STORE_32_ZTS, str_ins, reg, struct, field, tmp_reg 558| .if ZTS 559| LOAD_TSRM_CACHE TMP3 560| MEM_ACCESS_32_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 561| .else 562| MEM_STORE str_ins, reg, &struct.field, tmp_reg 563| .endif 564|.endmacro 565 566|.macro MEM_STORE_8_ZTS, strb_ins, reg, struct, field, tmp_reg 567| .if ZTS 568| LOAD_TSRM_CACHE TMP3 569| MEM_ACCESS_8_WITH_UOFFSET strb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 570| .else 571| MEM_STORE strb_ins, reg, &struct.field, tmp_reg 572| .endif 573|.endmacro 574 575/* Load value from memory 'addr' and write it into register 'reg'. 576 * For ZTS mode, base register with unsigned offset variant is used, 577 * and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */ 578|.macro MEM_LOAD, ldr_ins, reg, addr, tmp_reg 579|| if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) { 580| adr tmp_reg, &addr 581| ldr_ins reg, [tmp_reg] 582|| } else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) { 583| adrp tmp_reg, &(((uintptr_t)(addr))) 584| ldr_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)] 585|| } else { 586| LOAD_ADDR tmp_reg, addr 587| ldr_ins reg, [tmp_reg] 588|| } 589|.endmacro 590 591|.macro MEM_LOAD_64_ZTS, ldr_ins, reg, struct, field, tmp_reg 592| .if ZTS 593| LOAD_TSRM_CACHE TMP3 594| MEM_ACCESS_64_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 595| .else 596| MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg 597| .endif 598|.endmacro 599 600|.macro MEM_LOAD_32_ZTS, ldr_ins, reg, struct, field, tmp_reg 601| .if ZTS 602| LOAD_TSRM_CACHE TMP3 603| MEM_ACCESS_32_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 604| .else 605| MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg 606| .endif 607|.endmacro 608 609|.macro MEM_LOAD_8_ZTS, ldrb_ins, reg, struct, field, tmp_reg 610| .if ZTS 611| LOAD_TSRM_CACHE TMP3 612| MEM_ACCESS_8_WITH_UOFFSET ldrb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 613| .else 614| MEM_LOAD ldrb_ins, reg, &struct.field, tmp_reg 615| .endif 616|.endmacro 617 618/* Conduct arithmetic operation between the value in memory 'addr' and register value in 'reg', 619 * and the computation result is stored back in 'reg'. 'op_ins' can be add/sub. */ 620|.macro MEM_LOAD_OP, op_ins, ldr_ins, reg, addr, tmp_reg1, tmp_reg2 621| MEM_LOAD ldr_ins, tmp_reg1, addr, tmp_reg2 622| op_ins reg, reg, tmp_reg1 623|.endmacro 624 625|.macro MEM_LOAD_OP_ZTS, op_ins, ldr_ins, reg, struct, field, tmp_reg1, tmp_reg2 626| .if ZTS 627| LOAD_TSRM_CACHE TMP3 628| MEM_ACCESS_64_WITH_UOFFSET ldr_ins, tmp_reg2, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg1 629| op_ins reg, reg, tmp_reg2 630| .else 631| MEM_LOAD_OP op_ins, ldr_ins, reg, &struct.field, tmp_reg1, tmp_reg2 632| .endif 633|.endmacro 634 635/* Conduct arithmetic operation between the value in memory 'addr' and operand 'op', and the computation 636 * result is stored back to memory 'addr'. Operand 'op' can be either a register value or an immediate value. 637 * Currently, only add instruction is used as 'op_ins'. 638 * Note: It should be guaranteed that the immediate value can be encoded for 'op_ins'. */ 639|.macro MEM_UPDATE, op_ins, ldr_ins, str_ins, op, addr, tmp_reg1, tmp_reg2 640| LOAD_ADDR tmp_reg2, addr 641| ldr_ins, tmp_reg1, [tmp_reg2] 642| op_ins tmp_reg1, tmp_reg1, op 643| str_ins tmp_reg1, [tmp_reg2] 644|.endmacro 645 646|.macro MEM_UPDATE_ZTS, op_ins, ldr_ins, str_ins, op, struct, field, tmp_reg1, tmp_reg2 647| .if ZTS 648| LOAD_TSRM_CACHE TMP3 649|| if (((uintptr_t)(struct.._offset+offsetof(zend_..struct, field))) > LDR_STR_PIMM64) { 650| LOAD_32BIT_VAL tmp_reg1, (struct.._offset+offsetof(zend_..struct, field)) 651| ldr_ins tmp_reg2, [TMP3, tmp_reg1] 652| op_ins tmp_reg2, tmp_reg2, op 653| str_ins tmp_reg2, [TMP3, tmp_reg1] 654|| } else { 655| ldr_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] 656| op_ins tmp_reg2, tmp_reg2, op 657| str_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] 658|| } 659| .else 660| MEM_UPDATE op_ins, ldr_ins, str_ins, op, &struct.field, tmp_reg1, tmp_reg2 661| .endif 662|.endmacro 663 664|.macro EXT_CALL, func, tmp_reg 665|| if (arm64_may_use_b(func)) { 666| bl &func 667|| } else { 668| LOAD_ADDR tmp_reg, func 669| blr tmp_reg 670|| } 671|.endmacro 672 673|.macro EXT_JMP, func, tmp_reg 674|| if (arm64_may_use_b(func)) { 675| b &func 676|| } else { 677| LOAD_ADDR tmp_reg, func 678| br tmp_reg 679|| } 680|.endmacro 681 682|.macro SAVE_IP 683|| if (GCC_GLOBAL_REGS) { 684| str IP, EX->opline 685|| } 686|.endmacro 687 688|.macro LOAD_IP 689|| if (GCC_GLOBAL_REGS) { 690| ldr IP, EX->opline 691|| } 692|.endmacro 693 694|.macro LOAD_IP_ADDR, addr 695|| if (GCC_GLOBAL_REGS) { 696| LOAD_ADDR IP, addr 697|| } else { 698| ADDR_STORE EX->opline, addr, RX 699|| } 700|.endmacro 701 702|.macro LOAD_IP_ADDR_ZTS, struct, field, tmp_reg 703| .if ZTS 704|| if (GCC_GLOBAL_REGS) { 705| LOAD_TSRM_CACHE IP 706| ADD_SUB_64_WITH_CONST_32 add, IP, IP, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 707|| } else { 708| LOAD_TSRM_CACHE RX 709| ADD_SUB_64_WITH_CONST_32 add, RX, RX, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg 710| str RX, EX->opline 711|| } 712| .else 713| LOAD_IP_ADDR &struct.field 714| .endif 715|.endmacro 716 717|.macro GET_IP, reg 718|| if (GCC_GLOBAL_REGS) { 719| mov reg, IP 720|| } else { 721| ldr reg, EX->opline 722|| } 723|.endmacro 724 725/* Update IP with register 'reg'. Note: shift variant is handled by ADD_IP_SHIFT. */ 726|.macro ADD_IP, reg, tmp_reg 727|| if (GCC_GLOBAL_REGS) { 728| add IP, IP, reg 729|| } else { 730| ldr tmp_reg, EX->opline 731| add tmp_reg, tmp_reg, reg 732| str tmp_reg, EX->opline 733|| } 734|.endmacro 735 736|.macro ADD_IP_SHIFT, reg, shift, tmp_reg 737|| if (GCC_GLOBAL_REGS) { 738| add IP, IP, reg, shift 739|| } else { 740| ldr tmp_reg, EX->opline 741| add tmp_reg, tmp_reg, reg, shift 742| str tmp_reg, EX->opline 743|| } 744|.endmacro 745 746/* Update IP with 32-bit immediate 'val'. */ 747|.macro ADD_IP_WITH_CONST, val, tmp_reg 748|| ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val))); 749|| if (GCC_GLOBAL_REGS) { 750| add IP, IP, #val 751|| } else { 752| ldr tmp_reg, EX->opline 753| add tmp_reg, tmp_reg, #val 754| str tmp_reg, EX->opline 755|| } 756|.endmacro 757 758|.macro JMP_IP, tmp_reg 759|| if (GCC_GLOBAL_REGS) { 760| ldr tmp_reg, [IP] 761| br tmp_reg 762|| } else { 763| ldr tmp_reg, EX:CARG1->opline 764| br tmp_reg 765|| } 766|.endmacro 767 768|.macro CMP_IP, addr, tmp_reg1, tmp_reg2 769| LOAD_ADDR tmp_reg1, addr 770|| if (GCC_GLOBAL_REGS) { 771| cmp IP, tmp_reg1 772|| } else { 773| ldr tmp_reg2, EX->opline 774| cmp tmp_reg2, tmp_reg1 775|| } 776|.endmacro 777 778|.macro LOAD_ZVAL_ADDR, reg, addr 779|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 780| LOAD_ADDR reg, Z_ZV(addr) 781|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 782|| if (Z_OFFSET(addr)) { 783| ADD_SUB_64_WITH_CONST_32 add, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), reg 784|| } else { 785|| if (Z_REG(addr) == ZREG_RSP) { 786| mov reg, sp 787|| } else { 788| mov reg, Rx(Z_REG(addr)) 789|| } 790|| } 791|| } else { 792|| ZEND_UNREACHABLE(); 793|| } 794|.endmacro 795 796|.macro GET_Z_TYPE_INFO, reg, zv 797| ldr reg, [zv, #offsetof(zval,u1.type_info)] 798|.endmacro 799 800|.macro SET_Z_TYPE_INFO, zv, type, tmp_reg 801| LOAD_32BIT_VAL tmp_reg, type 802| str tmp_reg, [zv, #offsetof(zval,u1.type_info)] 803|.endmacro 804 805|.macro GET_ZVAL_TYPE, reg, addr, tmp_reg 806|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 807| MEM_ACCESS_8_WITH_UOFFSET ldrb, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.v.type), tmp_reg 808|.endmacro 809 810|.macro GET_ZVAL_TYPE_INFO, reg, addr, tmp_reg 811|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 812| MEM_ACCESS_32_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg 813|.endmacro 814 815|.macro SET_ZVAL_TYPE_INFO, addr, type, tmp_reg1, tmp_reg2 816|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 817| LOAD_32BIT_VAL tmp_reg1, type 818| MEM_ACCESS_32_WITH_UOFFSET str, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg2 819|.endmacro 820 821|.macro SET_ZVAL_TYPE_INFO_FROM_REG, addr, reg, tmp_reg 822|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 823| MEM_ACCESS_32_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg 824|.endmacro 825 826|.macro GET_Z_PTR, reg, zv 827| ldr reg, [zv] 828|.endmacro 829 830|.macro GET_ZVAL_PTR, reg, addr, tmp_reg 831|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 832| MEM_ACCESS_64_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 833|.endmacro 834 835|.macro SET_ZVAL_PTR, addr, reg, tmp_reg 836|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 837| MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 838|.endmacro 839 840|.macro UNDEF_OPLINE_RESULT, tmp_reg 841| ldr REG0, EX->opline 842| ldr REG0w, OP:REG0->result.var 843| add REG0, FP, REG0 844| SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg 845|.endmacro 846 847|.macro UNDEF_OPLINE_RESULT_IF_USED, tmp_reg1, tmp_reg2 848| ldrb tmp_reg1, OP:RX->result_type 849| TST_32_WITH_CONST tmp_reg1, (IS_TMP_VAR|IS_VAR), tmp_reg2 850| beq >1 851| ldr REG0w, OP:RX->result.var 852| add REG0, FP, REG0 853| SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg1 854|1: 855|.endmacro 856 857/* Floating-point comparison between register 'reg' and value from memory 'addr'. 858 * Note: the equivalent macros in JIT/x86 are SSE_AVX_OP and SSE_OP. */ 859|.macro DOUBLE_CMP, reg, addr, tmp_reg, fp_tmp_reg 860|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 861| MEM_LOAD ldr, Rd(fp_tmp_reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg) 862| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) 863|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 864| MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) 865| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) 866|| } else if (Z_MODE(addr) == IS_REG) { 867| fcmp Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) 868|| } else { 869|| ZEND_UNREACHABLE(); 870|| } 871|.endmacro 872 873/* Convert LONG value 'val' into DOUBLE type, and move it into FP register 'reg'. 874 * Note: the equivalent macro in JIT/x86 is SSE_GET_LONG. */ 875|.macro DOUBLE_GET_LONG, reg, val, tmp_reg 876|| if (val == 0) { 877| fmov Rd(reg-ZREG_V0), xzr // TODO: "movi d0, #0" is not recognized by DynASM/arm64 878|| } else { 879| LOAD_64BIT_VAL Rx(tmp_reg), val 880| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg) 881|| } 882|.endmacro 883 884/* Convert LONG value from memory 'addr' into DOUBLE type, and move it into FP register 'reg'. 885 * Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_LVAL. */ 886|.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg1, tmp_reg2 887|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 888| DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg1 889|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 890| MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg2) 891| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg1) 892|| } else if (Z_MODE(addr) == IS_REG) { 893| scvtf Rd(reg-ZREG_V0), Rx(Z_REG(addr)) 894|| } else { 895|| ZEND_UNREACHABLE(); 896|| } 897|.endmacro 898 899/* Floating-point arithmetic operation between two FP registers. 900 * Note: the equivalent macro in JIT/x86 is AVX_MATH_REG. */ 901|.macro DOUBLE_MATH_REG, opcode, dst_reg, op1_reg, op2_reg 902|| switch (opcode) { 903|| case ZEND_ADD: 904| fadd Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 905|| break; 906|| case ZEND_SUB: 907| fsub Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 908|| break; 909|| case ZEND_MUL: 910| fmul Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 911|| break; 912|| case ZEND_DIV: 913| fdiv Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) 914|| break; 915|| } 916|.endmacro 917 918/* Conduct binary operation between register 'reg' and value from memory 'addr', 919 * and the computation result is stored in 'reg'. 920 * For LONG_ADD_SUB, 'add_sub_ins' can be adds/subs. For LONG_BW_OP, 'bw_ins' can be and/orr/eor. 921 * For LONG_CMP, 'cmp' instruction is used by default and only flag registers are affected. 922 * Note: the equivalent macro in JIT/x86 is LONG_OP. */ 923|.macro LONG_ADD_SUB, add_sub_ins, reg, addr, tmp_reg 924|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 925| ADD_SUB_64_WITH_CONST add_sub_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg 926|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 927| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 928| add_sub_ins Rx(reg), Rx(reg), tmp_reg 929|| } else if (Z_MODE(addr) == IS_REG) { 930| add_sub_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) 931|| } else { 932|| ZEND_UNREACHABLE(); 933|| } 934|.endmacro 935 936|.macro LONG_BW_OP, bw_ins, reg, addr, tmp_reg 937|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 938| BW_OP_64_WITH_CONST bw_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg 939|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 940| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 941| bw_ins Rx(reg), Rx(reg), tmp_reg 942|| } else if (Z_MODE(addr) == IS_REG) { 943| bw_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) 944|| } else { 945|| ZEND_UNREACHABLE(); 946|| } 947|.endmacro 948 949|.macro LONG_CMP, reg, addr, tmp_reg 950|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 951| CMP_64_WITH_CONST Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg 952|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 953| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 954| cmp Rx(reg), tmp_reg 955|| } else if (Z_MODE(addr) == IS_REG) { 956| cmp Rx(reg), Rx(Z_REG(addr)) 957|| } else { 958|| ZEND_UNREACHABLE(); 959|| } 960|.endmacro 961 962/* Conduct add/sub between value from memory 'addr' and an immediate value 'val', and 963 * the computation result is stored back into 'addr'. 964 * Note: it should be guaranteed that 'val' can be encoded into add/sub instruction. */ 965|.macro LONG_ADD_SUB_WITH_IMM, add_sub_ins, addr, val, tmp_reg1, tmp_reg2 966|| ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val))); 967|| if (Z_MODE(addr) == IS_MEM_ZVAL) { 968|| if (((uint32_t)(Z_OFFSET(addr))) > LDR_STR_PIMM64) { 969| LOAD_32BIT_VAL tmp_reg2, Z_OFFSET(addr) 970| ldr tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2] 971| add_sub_ins tmp_reg1, tmp_reg1, #val 972| str tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2] 973|| } else { 974| ldr tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)] 975| add_sub_ins tmp_reg1, tmp_reg1, #val 976| str tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)] 977|| } 978|| } else if (Z_MODE(addr) == IS_REG) { 979| add_sub_ins Rx(Z_REG(addr)), Rx(Z_REG(addr)), #val 980|| } else { 981|| ZEND_UNREACHABLE(); 982|| } 983|.endmacro 984 985/* Compare value from memory 'addr' with immediate value 'val'. 986 * Note: the equivalent macro in JIT/x86 is LONG_OP_WITH_CONST. */ 987|.macro LONG_CMP_WITH_CONST, addr, val, tmp_reg1, tmp_reg2 988|| if (Z_MODE(addr) == IS_MEM_ZVAL) { 989| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 990| CMP_64_WITH_CONST tmp_reg1, val, tmp_reg2 991|| } else if (Z_MODE(addr) == IS_REG) { 992| CMP_64_WITH_CONST Rx(Z_REG(addr)), val, tmp_reg1 993|| } else { 994|| ZEND_UNREACHABLE(); 995|| } 996|.endmacro 997 998|.macro GET_ZVAL_LVAL, reg, addr, tmp_reg 999|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 1000|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { 1001| mov Rx(reg), xzr 1002|| } else { 1003| LOAD_64BIT_VAL Rx(reg), Z_LVAL_P(Z_ZV(addr)) 1004|| } 1005|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 1006| MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(reg), Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 1007|| } else if (Z_MODE(addr) == IS_REG) { 1008|| if (reg != Z_REG(addr)) { 1009| mov Rx(reg), Rx(Z_REG(addr)) 1010|| } 1011|| } else { 1012|| ZEND_UNREACHABLE(); 1013|| } 1014|.endmacro 1015 1016|.macro LONG_MATH, opcode, reg, addr, tmp_reg1 1017|| switch (opcode) { 1018|| case ZEND_ADD: 1019| LONG_ADD_SUB adds, reg, addr, tmp_reg1 1020|| break; 1021|| case ZEND_SUB: 1022| LONG_ADD_SUB subs, reg, addr, tmp_reg1 1023|| break; 1024|| case ZEND_BW_OR: 1025| LONG_BW_OP orr, reg, addr, tmp_reg1 1026|| break; 1027|| case ZEND_BW_AND: 1028| LONG_BW_OP and, reg, addr, tmp_reg1 1029|| break; 1030|| case ZEND_BW_XOR: 1031| LONG_BW_OP eor, reg, addr, tmp_reg1 1032|| break; 1033|| default: 1034|| ZEND_UNREACHABLE(); 1035|| } 1036|.endmacro 1037 1038|.macro LONG_MATH_REG, opcode, dst_reg, src_reg1, src_reg2 1039|| switch (opcode) { 1040|| case ZEND_ADD: 1041| adds dst_reg, src_reg1, src_reg2 1042|| break; 1043|| case ZEND_SUB: 1044| subs dst_reg, src_reg1, src_reg2 1045|| break; 1046|| case ZEND_BW_OR: 1047| orr dst_reg, src_reg1, src_reg2 1048|| break; 1049|| case ZEND_BW_AND: 1050| and dst_reg, src_reg1, src_reg2 1051|| break; 1052|| case ZEND_BW_XOR: 1053| eor dst_reg, src_reg1, src_reg2 1054|| break; 1055|| default: 1056|| ZEND_UNREACHABLE(); 1057|| } 1058|.endmacro 1059 1060/* Store LONG value into memory 'addr'. 1061 * This LONG value can be an immediate value i.e. 'val' in macro SET_ZVAL_LVAL, or 1062 * a register value i.e. 'reg' in macro SET_ZVAL_LVAL_FROM_REG. */ 1063|.macro SET_ZVAL_LVAL_FROM_REG, addr, reg, tmp_reg 1064|| if (Z_MODE(addr) == IS_REG) { 1065| mov Rx(Z_REG(addr)), reg 1066|| } else { 1067|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1068| MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg 1069|| } 1070|.endmacro 1071 1072|.macro SET_ZVAL_LVAL, addr, val, tmp_reg1, tmp_reg2 1073|| if (val == 0) { 1074| SET_ZVAL_LVAL_FROM_REG addr, xzr, tmp_reg2 1075|| } else { 1076| LOAD_64BIT_VAL tmp_reg1, val 1077| SET_ZVAL_LVAL_FROM_REG addr, tmp_reg1, tmp_reg2 1078|| } 1079|.endmacro 1080 1081/* Store DOUBLE value from FP register 'reg' into memory 'addr'. 1082 * Note: the equivalent macro in JIT/x86 is SSE_SET_ZVAL_DVAL. */ 1083|.macro SET_ZVAL_DVAL, addr, reg, tmp_reg 1084|| if (Z_MODE(addr) == IS_REG) { 1085|| if (reg != Z_REG(addr)) { 1086| fmov Rd(Z_REG(addr)-ZREG_V0), Rd(reg-ZREG_V0) 1087|| } 1088|| } else { 1089|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1090| MEM_ACCESS_64_WITH_UOFFSET str, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) 1091|| } 1092|.endmacro 1093 1094/* Load DOUBLE value from memory 'addr' into FP register 'reg'. 1095 * Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_DVAL. */ 1096|.macro GET_ZVAL_DVAL, reg, addr, tmp_reg 1097|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { 1098|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 1099| MEM_LOAD ldr, Rd(reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg) 1100|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 1101| MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) 1102|| } else if (Z_MODE(addr) == IS_REG) { 1103| fmov Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) 1104|| } else { 1105|| ZEND_UNREACHABLE(); 1106|| } 1107|| } 1108|.endmacro 1109 1110|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg 1111|| if (Z_TYPE_P(zv) > IS_TRUE) { 1112|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 1113|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; 1114| MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1) 1115| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 1116|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 1117|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; 1118| DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), tmp_reg1 1119| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 1120|| } else { 1121| // In x64, if the range of this LONG value can be represented via INT type, only move the low 32 bits into dst_addr. 1122| // Note that imm32 is signed extended to 64 bits during mov. 1123| // In aarch64, we choose to handle both cases in the same way. Even though 4 mov's are used for 64-bit value and 2 mov's are 1124| // needed for 32-bit value, an extra ext insn is needed for 32-bit value. 1125| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1126|| } 1127|| } 1128|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1129|| if (dst_def_info == MAY_BE_DOUBLE) { 1130|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1131| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) 1132|| } 1133|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1134| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2) 1135|| } 1136|| } 1137|.endmacro 1138 1139|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg 1140|| if (Z_TYPE_P(zv) > IS_TRUE) { 1141|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 1142|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? 1143|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : fp_tmp_reg); 1144| MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1) 1145| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 1146| SET_ZVAL_DVAL res_addr, dst_reg, tmp_reg2 1147|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 1148|| if (Z_MODE(dst_addr) == IS_REG) { 1149| DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), tmp_reg1 1150| SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg2 1151|| } else if (Z_MODE(res_addr) == IS_REG) { 1152| DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), tmp_reg1 1153| SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg2 1154|| } else { 1155| DOUBLE_GET_LONG fp_tmp_reg, Z_LVAL_P(zv), tmp_reg1 1156| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg2 1157| SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg2 1158|| } 1159|| } else { 1160|| if (Z_MODE(dst_addr) == IS_REG) { 1161| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1162| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg1) 1163|| } else if (Z_MODE(res_addr) == IS_REG) { 1164| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1165| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg1) 1166|| } else { 1167| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1168| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) 1169|| } 1170|| } 1171|| } 1172|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1173|| if (dst_def_info == MAY_BE_DOUBLE) { 1174|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1175| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) 1176|| } 1177|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1178| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2) 1179|| } 1180|| } 1181|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1182|| if (dst_def_info == MAY_BE_DOUBLE) { 1183| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) 1184|| } else { 1185| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2) 1186|| } 1187|| } 1188|.endmacro 1189 1190// the same as above, but "src" may overlap with "reg1" 1191// Useful info would be stored into reg1 and reg2, and they might be used afterward. 1192|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg1, tmp_reg2, fp_tmp_reg 1193| ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg1, fp_tmp_reg 1194|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1195|| !(src_info & MAY_BE_GUARD) && 1196|| has_concrete_type(src_info & MAY_BE_ANY)) { 1197|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1198|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) { 1199|| uint32_t type = concrete_type(src_info); 1200| SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg1), Rx(tmp_reg2) 1201|| } 1202|| } 1203|| } else { 1204| GET_ZVAL_TYPE_INFO Rw(reg1), src_addr, Rx(tmp_reg1) 1205| SET_ZVAL_TYPE_INFO_FROM_REG dst_addr, Rw(reg1), Rx(tmp_reg1) 1206|| } 1207|.endmacro 1208 1209|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg, fp_tmp_reg 1210|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1211|| if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) { 1212|| if (Z_MODE(src_addr) == IS_REG) { 1213|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1214| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) 1215|| } 1216|| } else if (Z_MODE(dst_addr) == IS_REG) { 1217| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) 1218|| } else { 1219| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) 1220| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg) 1221|| } 1222|| } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 1223|| if (Z_MODE(src_addr) == IS_REG) { 1224| SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg 1225|| } else if (Z_MODE(dst_addr) == IS_REG) { 1226| GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg 1227|| } else { 1228| GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg 1229| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg 1230|| } 1231|| // Combine the following two branches. 1232|| // } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) { 1233|| } else { 1234| GET_ZVAL_PTR Rx(reg2), src_addr, Rx(tmp_reg) 1235| SET_ZVAL_PTR dst_addr, Rx(reg2), Rx(tmp_reg) 1236|| } 1237|| } 1238|.endmacro 1239 1240|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, reg1, reg2, tmp_reg, tmp_reg2, fp_tmp_reg 1241|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1242|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) { 1243|| if (Z_MODE(src_addr) == IS_REG) { 1244|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1245| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) 1246|| } 1247|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) { 1248| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) 1249|| } 1250|| } else if (Z_MODE(dst_addr) == IS_REG) { 1251| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) 1252|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) { 1253| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg) 1254|| } 1255|| } else if (Z_MODE(res_addr) == IS_REG) { 1256| GET_ZVAL_LVAL Z_REG(res_addr), src_addr, Rx(tmp_reg) 1257| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg) 1258|| } else { 1259| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) 1260| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg) 1261| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(reg2), Rx(tmp_reg) 1262|| } 1263|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 1264|| if (Z_MODE(src_addr) == IS_REG) { 1265| SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg 1266| SET_ZVAL_DVAL res_addr, Z_REG(src_addr), tmp_reg 1267|| } else if (Z_MODE(dst_addr) == IS_REG) { 1268| GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg 1269| SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg 1270|| } else if (Z_MODE(res_addr) == IS_REG) { 1271| GET_ZVAL_DVAL Z_REG(res_addr), src_addr, tmp_reg 1272| SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg 1273|| } else { 1274| GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg 1275| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg 1276| SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg 1277|| } 1278|| } else { 1279| GET_ZVAL_PTR Rx(reg2), src_addr, Rx(tmp_reg) 1280| SET_ZVAL_PTR dst_addr, Rx(reg2), Rx(tmp_reg) 1281| SET_ZVAL_PTR res_addr, Rx(reg2), Rx(tmp_reg) 1282|| } 1283|| } 1284|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1285|| has_concrete_type(src_info & MAY_BE_ANY)) { 1286|| uint32_t type = concrete_type(src_info); 1287|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1288|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { 1289| SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg), Rx(tmp_reg2) 1290|| } 1291|| } 1292|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1293| SET_ZVAL_TYPE_INFO res_addr, type, Rw(tmp_reg), Rx(tmp_reg2) 1294|| } 1295|| } else { 1296| GET_ZVAL_TYPE_INFO Rw(reg1), src_addr, Rx(tmp_reg) 1297| SET_ZVAL_TYPE_INFO_FROM_REG dst_addr, Rw(reg1), Rx(tmp_reg) 1298| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, Rw(reg1), Rx(tmp_reg) 1299|| } 1300|.endmacro 1301 1302|.macro IF_UNDEF, type_reg, label 1303| cbz type_reg, label 1304|.endmacro 1305 1306|.macro IF_TYPE, type, val, label 1307|| if (val == 0) { 1308| cbz type, label 1309|| } else { 1310| cmp type, #val 1311| beq label 1312|| } 1313|.endmacro 1314 1315|.macro IF_NOT_TYPE, type, val, label 1316|| if (val == 0) { 1317| cbnz type, label 1318|| } else { 1319| cmp type, #val 1320| bne label 1321|| } 1322|.endmacro 1323 1324|.macro IF_Z_TYPE, zv, val, label, tmp_reg 1325| ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)] 1326| IF_TYPE tmp_reg, val, label 1327|.endmacro 1328 1329|.macro IF_NOT_Z_TYPE, zv, val, label, tmp_reg 1330| ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)] 1331| IF_NOT_TYPE tmp_reg, val, label 1332|.endmacro 1333 1334|.macro CMP_ZVAL_TYPE, addr, val, tmp_reg 1335|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1336| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) 1337| cmp Rw(tmp_reg), #val 1338|.endmacro 1339 1340|.macro IF_ZVAL_TYPE, addr, val, label, tmp_reg 1341|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1342| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) 1343| IF_TYPE Rw(tmp_reg), val, label 1344|.endmacro 1345 1346|.macro IF_NOT_ZVAL_TYPE, addr, val, label, tmp_reg 1347|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1348| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) 1349| IF_NOT_TYPE Rw(tmp_reg), val, label 1350|.endmacro 1351 1352|.macro IF_FLAGS, type_flags, mask, label, tmp_reg 1353| TST_32_WITH_CONST type_flags, mask, tmp_reg 1354| bne label 1355|.endmacro 1356 1357|.macro IF_NOT_FLAGS, type_flags, mask, label, tmp_reg 1358| TST_32_WITH_CONST type_flags, mask, tmp_reg 1359| beq label 1360|.endmacro 1361 1362|.macro IF_REFCOUNTED, type_flags, label, tmp_reg 1363| TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg 1364| bne label 1365|.endmacro 1366 1367|.macro IF_NOT_REFCOUNTED, type_flags, label, tmp_reg 1368| TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg 1369| beq label 1370|.endmacro 1371 1372|.macro IF_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 1373|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1374| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2) 1375| IF_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2) 1376|.endmacro 1377 1378|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 1379|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1380| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2) 1381| IF_NOT_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2) 1382|.endmacro 1383 1384|.macro IF_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2 1385| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2 1386|.endmacro 1387 1388|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2 1389| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2 1390|.endmacro 1391 1392|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label, tmp_reg1, tmp_reg2 1393| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label, tmp_reg1, tmp_reg2 1394|.endmacro 1395 1396|.macro GC_ADDREF, zv, tmp_reg 1397| ldr tmp_reg, [zv] 1398| add tmp_reg, tmp_reg, #1 1399| str tmp_reg, [zv] 1400|.endmacro 1401 1402|.macro GC_ADDREF_2, zv, tmp_reg 1403| ldr tmp_reg, [zv] 1404| add tmp_reg, tmp_reg, #2 1405| str tmp_reg, [zv] 1406|.endmacro 1407 1408|.macro GC_DELREF, zv, tmp_reg 1409| ldr tmp_reg, [zv] 1410| subs tmp_reg, tmp_reg, #1 1411| str tmp_reg, [zv] 1412|.endmacro 1413 1414|.macro IF_GC_MAY_NOT_LEAK, ptr, label, tmp_reg1, tmp_reg2 1415| ldr tmp_reg1, [ptr, #4] 1416| TST_32_WITH_CONST tmp_reg1, (GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)), tmp_reg2 1417| bne label 1418|.endmacro 1419 1420|.macro ADDREF_CONST, zv, tmp_reg1, tmp_reg2 1421| LOAD_ADDR tmp_reg1, Z_PTR_P(zv) 1422| ldr tmp_reg2, [tmp_reg1] 1423| add tmp_reg2, tmp_reg2, #1 1424| str tmp_reg2, [tmp_reg1] 1425|.endmacro 1426 1427|.macro ADDREF_CONST_2, zv, tmp_reg1, tmp_reg2 1428| LOAD_ADDR tmp_reg1, Z_PTR_P(zv) 1429| ldr tmp_reg2, [tmp_reg1] 1430| add tmp_reg2, tmp_reg2, #2 1431| str tmp_reg2, [tmp_reg1] 1432|.endmacro 1433 1434|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg, tmp_reg 1435|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1436|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1437| IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg 1438|| } 1439| GC_ADDREF value_ptr_reg, tmp_reg 1440|1: 1441|| } 1442|.endmacro 1443 1444|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg, tmp_reg 1445|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1446|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1447| IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg 1448|| } 1449| ldr tmp_reg, [value_ptr_reg] 1450| add tmp_reg, tmp_reg, #2 1451| str tmp_reg, [value_ptr_reg] 1452|1: 1453|| } 1454|.endmacro 1455 1456|.macro ZVAL_DEREF, reg, info, tmp_reg 1457|| if (info & MAY_BE_REF) { 1458| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1, tmp_reg 1459| GET_Z_PTR reg, reg 1460| add reg, reg, #offsetof(zend_reference, val) 1461|1: 1462|| } 1463|.endmacro 1464 1465|.macro SET_EX_OPLINE, op, tmp_reg 1466|| if (op == last_valid_opline) { 1467|| zend_jit_use_last_valid_opline(); 1468| SAVE_IP 1469|| } else { 1470| ADDR_STORE EX->opline, op, tmp_reg 1471|| if (!GCC_GLOBAL_REGS) { 1472|| zend_jit_reset_last_valid_opline(); 1473|| } 1474|| } 1475|.endmacro 1476 1477// arg1 "zval" should be in FCARG1x 1478|.macro ZVAL_DTOR_FUNC, var_info, opline, tmp_reg 1479|| do { 1480|| if (!((var_info) & MAY_BE_GUARD) 1481|| && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1482|| uint8_t type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); 1483|| if (type == IS_STRING && !ZEND_DEBUG) { 1484| EXT_CALL _efree, tmp_reg 1485|| break; 1486|| } else if (type == IS_ARRAY) { 1487|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { 1488|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { 1489| SET_EX_OPLINE opline, tmp_reg 1490|| } 1491| EXT_CALL zend_array_destroy, tmp_reg 1492|| } else { 1493| EXT_CALL zend_jit_array_free, tmp_reg 1494|| } 1495|| break; 1496|| } else if (type == IS_OBJECT) { 1497|| if (opline) { 1498| SET_EX_OPLINE opline, REG0 1499|| } 1500| EXT_CALL zend_objects_store_del, tmp_reg 1501|| break; 1502|| } 1503|| } 1504|| if (opline) { 1505| SET_EX_OPLINE opline, tmp_reg 1506|| } 1507| EXT_CALL rc_dtor_func, tmp_reg 1508|| } while(0); 1509|.endmacro 1510 1511|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline, tmp_reg1, tmp_reg2 1512|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) { 1513|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1514| // if (Z_REFCOUNTED_P(cv)) { 1515|| if (cold) { 1516| IF_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 1517|.cold_code 1518|1: 1519|| } else { 1520| IF_NOT_ZVAL_REFCOUNTED addr, >4, tmp_reg1, tmp_reg2 1521|| } 1522|| } 1523| // if (!Z_DELREF_P(cv)) { 1524| GET_ZVAL_PTR FCARG1x, addr, Rx(tmp_reg2) 1525| GC_DELREF FCARG1x, Rw(tmp_reg1) 1526|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) { 1527|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) { 1528|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1529| bne >3 1530|| } else { 1531| bne >4 1532|| } 1533|| } 1534| // zval_dtor_func(r); 1535| ZVAL_DTOR_FUNC op_info, opline, Rx(tmp_reg1) 1536|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1537| b >4 1538|| } 1539|3: 1540|| } 1541|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1542|| if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) { 1543|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 1544| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1, tmp_reg1 1545| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4, tmp_reg1, tmp_reg2 1546| GET_ZVAL_PTR FCARG1x, ref_addr, Rx(tmp_reg2) 1547|1: 1548|| } 1549| IF_GC_MAY_NOT_LEAK FCARG1x, >4, Rw(tmp_reg1), Rw(tmp_reg2) 1550| // gc_possible_root(Z_COUNTED_P(z)) 1551| EXT_CALL gc_possible_root, Rx(tmp_reg1) 1552|| } 1553|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { 1554| b >4 1555|.code 1556|| } 1557|4: 1558|| } 1559|.endmacro 1560 1561|.macro FREE_OP, op_type, op, op_info, cold, opline, tmp_reg1, tmp_reg2 1562|| if (op_type & (IS_VAR|IS_TMP_VAR)) { 1563|| zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 1564| ZVAL_PTR_DTOR addr, op_info, 0, cold, opline, tmp_reg1, tmp_reg2 1565|| } 1566|.endmacro 1567 1568|.macro SEPARATE_ARRAY, addr, op_info, cold, tmp_reg1, tmp_reg2 1569|| if (RC_MAY_BE_N(op_info)) { 1570|| if (Z_REG(addr) != ZREG_FP) { 1571| GET_ZVAL_LVAL ZREG_REG0, addr, Rx(tmp_reg1) 1572|| if (RC_MAY_BE_1(op_info)) { 1573| // if (GC_REFCOUNT() > 1) 1574| ldr Rw(tmp_reg1), [REG0] 1575| cmp Rw(tmp_reg1), #1 1576| bls >2 1577|| } 1578|| if (Z_REG(addr) != ZREG_FCARG1 || Z_OFFSET(addr) != 0) { 1579| LOAD_ZVAL_ADDR FCARG1x, addr 1580|| } 1581| EXT_CALL zend_jit_zval_array_dup, REG0 1582| mov REG0, RETVALx 1583|2: 1584| mov FCARG1x, REG0 1585|| } else { 1586| GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1) 1587|| if (RC_MAY_BE_1(op_info)) { 1588| // if (GC_REFCOUNT() > 1) 1589| ldr Rw(tmp_reg1), [FCARG1x] 1590| cmp Rw(tmp_reg1), #1 1591|| if (cold) { 1592| bhi >1 1593|.cold_code 1594|1: 1595|| } else { 1596| bls >2 1597|| } 1598|| } 1599| IF_NOT_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 1600| GC_DELREF FCARG1x, Rw(tmp_reg1) 1601|1: 1602| EXT_CALL zend_array_dup, REG0 1603| mov REG0, RETVALx 1604| SET_ZVAL_PTR addr, REG0, Rx(tmp_reg1) 1605| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX, Rw(tmp_reg1), Rx(tmp_reg2) 1606| mov FCARG1x, REG0 1607|| if (RC_MAY_BE_1(op_info)) { 1608|| if (cold) { 1609| b >2 1610|.code 1611|| } 1612|| } 1613|2: 1614|| } 1615|| } else { 1616| GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1) 1617|| } 1618|.endmacro 1619 1620/* argument is passed in FCARG1x */ 1621|.macro EFREE_REFERENCE 1622||#if ZEND_DEBUG 1623| mov FCARG2x, xzr // filename 1624| mov CARG3w, wzr // lineno 1625| mov CARG4, xzr 1626| mov CARG5, xzr 1627| EXT_CALL _efree, REG0 1628||#else 1629||#ifdef HAVE_BUILTIN_CONSTANT_P 1630| EXT_CALL _efree_32, REG0 1631||#else 1632| EXT_CALL _efree, REG0 1633||#endif 1634||#endif 1635|.endmacro 1636 1637|.macro EMALLOC, size, op_array, opline 1638||#if ZEND_DEBUG 1639|| const char *filename = op_array->filename ? op_array->filename->val : NULL; 1640| mov FCARG1x, #size 1641| LOAD_ADDR FCARG2x, filename 1642| LOAD_32BIT_VAL CARG3w, opline->lineno 1643| mov CARG4, xzr 1644| mov CARG5, xzr 1645| EXT_CALL _emalloc, REG0 1646| mov REG0, RETVALx 1647||#else 1648||#ifdef HAVE_BUILTIN_CONSTANT_P 1649|| if (size > 24 && size <= 32) { 1650| EXT_CALL _emalloc_32, REG0 1651| mov REG0, RETVALx 1652|| } else { 1653| mov FCARG1x, #size 1654| EXT_CALL _emalloc, REG0 1655| mov REG0, RETVALx 1656|| } 1657||#else 1658| mov FCARG1x, #size 1659| EXT_CALL _emalloc, REG0 1660| mov REG0, RETVALx 1661||#endif 1662||#endif 1663|.endmacro 1664 1665|.macro OBJ_RELEASE, reg, exit_label, tmp_reg1, tmp_reg2 1666| GC_DELREF Rx(reg), Rw(tmp_reg1) 1667| bne >1 1668| // zend_objects_store_del(obj); 1669|| if (reg != ZREG_FCARG1) { 1670| mov FCARG1x, Rx(reg) 1671|| } 1672| EXT_CALL zend_objects_store_del, Rx(tmp_reg1) 1673| b exit_label 1674|1: 1675| IF_GC_MAY_NOT_LEAK Rx(reg), >1, Rw(tmp_reg1), Rw(tmp_reg2) 1676| // gc_possible_root(obj) 1677|| if (reg != ZREG_FCARG1) { 1678| mov FCARG1x, Rx(reg) 1679|| } 1680| EXT_CALL gc_possible_root, Rx(tmp_reg1) 1681|1: 1682|.endmacro 1683 1684|.macro UNDEFINED_OFFSET, opline 1685|| if (opline == last_valid_opline) { 1686|| zend_jit_use_last_valid_opline(); 1687| bl ->undefined_offset_ex 1688|| } else { 1689| SET_EX_OPLINE opline, REG0 1690| bl ->undefined_offset 1691|| } 1692|.endmacro 1693 1694|.macro UNDEFINED_INDEX, opline 1695|| if (opline == last_valid_opline) { 1696|| zend_jit_use_last_valid_opline(); 1697| bl ->undefined_index_ex 1698|| } else { 1699| SET_EX_OPLINE opline, REG0 1700| bl ->undefined_index 1701|| } 1702|.endmacro 1703 1704|.macro CANNOT_ADD_ELEMENT, opline 1705|| if (opline == last_valid_opline) { 1706|| zend_jit_use_last_valid_opline(); 1707| bl ->cannot_add_element_ex 1708|| } else { 1709| SET_EX_OPLINE opline, REG0 1710| bl ->cannot_add_element 1711|| } 1712|.endmacro 1713 1714static bool reuse_ip = 0; 1715static bool delayed_call_chain = 0; 1716static uint32_t delayed_call_level = 0; 1717static const zend_op *last_valid_opline = NULL; 1718static bool use_last_vald_opline = 0; 1719static bool track_last_valid_opline = 0; 1720static int jit_return_label = -1; 1721static uint32_t current_trace_num = 0; 1722static uint32_t allowed_opt_flags = 0; 1723 1724static void zend_jit_track_last_valid_opline(void) 1725{ 1726 use_last_vald_opline = 0; 1727 track_last_valid_opline = 1; 1728} 1729 1730static void zend_jit_use_last_valid_opline(void) 1731{ 1732 if (track_last_valid_opline) { 1733 use_last_vald_opline = 1; 1734 track_last_valid_opline = 0; 1735 } 1736} 1737 1738static bool zend_jit_trace_uses_initial_ip(void) 1739{ 1740 return use_last_vald_opline; 1741} 1742 1743static void zend_jit_set_last_valid_opline(const zend_op *target_opline) 1744{ 1745 if (!reuse_ip) { 1746 track_last_valid_opline = 0; 1747 last_valid_opline = target_opline; 1748 } 1749} 1750 1751static void zend_jit_reset_last_valid_opline(void) 1752{ 1753 track_last_valid_opline = 0; 1754 last_valid_opline = NULL; 1755} 1756 1757static void zend_jit_start_reuse_ip(void) 1758{ 1759 zend_jit_reset_last_valid_opline(); 1760 reuse_ip = 1; 1761} 1762 1763static int zend_jit_reuse_ip(dasm_State **Dst) 1764{ 1765 if (!reuse_ip) { 1766 zend_jit_start_reuse_ip(); 1767 | // call = EX(call); 1768 | ldr RX, EX->call 1769 } 1770 return 1; 1771} 1772 1773static void zend_jit_stop_reuse_ip(void) 1774{ 1775 reuse_ip = 0; 1776} 1777 1778static int zend_jit_interrupt_handler_stub(dasm_State **Dst) 1779{ 1780 |->interrupt_handler: 1781 | SAVE_IP 1782 | //EG(vm_interrupt) = 0; 1783 | MEM_STORE_8_ZTS strb, wzr, executor_globals, vm_interrupt, TMP1 1784 | //if (EG(timed_out)) { 1785 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, timed_out, TMP1 1786 | cbz TMP1w, >1 1787 | //zend_timeout(); 1788 | EXT_CALL zend_timeout, TMP1 1789 |1: 1790 | //} else if (zend_interrupt_function) { 1791 if (zend_interrupt_function) { 1792 | //zend_interrupt_function(execute_data); 1793 | mov CARG1, FP 1794 | EXT_CALL zend_interrupt_function, TMP1 1795 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 1796 | cbz REG0, >1 1797 | EXT_CALL zend_jit_exception_in_interrupt_handler_helper, TMP1 1798 |1: 1799 | //ZEND_VM_ENTER(); 1800 | //execute_data = EG(current_execute_data); 1801 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 1802 | LOAD_IP 1803 } 1804 | //ZEND_VM_CONTINUE() 1805 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1806 | ADD_HYBRID_SPAD 1807 | JMP_IP TMP1 1808 } else if (GCC_GLOBAL_REGS) { 1809 | ldp x29, x30, [sp], # SPAD // stack alignment 1810 | JMP_IP TMP1 1811 } else { 1812 | ldp FP, RX, T2 // restore FP and IP 1813 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1814 | mov RETVALx, #1 // ZEND_VM_ENTER 1815 | ret 1816 } 1817 1818 return 1; 1819} 1820 1821static int zend_jit_exception_handler_stub(dasm_State **Dst) 1822{ 1823 |->exception_handler: 1824 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1825 const void *handler = zend_get_opcode_handler_func(EG(exception_op)); 1826 1827 | ADD_HYBRID_SPAD 1828 | EXT_CALL handler, REG0 1829 | JMP_IP TMP1 1830 } else { 1831 const void *handler = EG(exception_op)->handler; 1832 1833 if (GCC_GLOBAL_REGS) { 1834 | ldp x29, x30, [sp], # SPAD // stack alignment 1835 | EXT_JMP handler, REG0 1836 } else { 1837 | mov FCARG1x, FP 1838 | EXT_CALL handler, REG0 1839 | ldp FP, RX, T2 // restore FP and IP 1840 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1841 | tbnz RETVALw, #31, >1 1842 | mov RETVALw, #1 // ZEND_VM_ENTER 1843 |1: 1844 | ret 1845 } 1846 } 1847 1848 return 1; 1849} 1850 1851static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) 1852{ 1853 |->exception_handler_undef: 1854 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, opline_before_exception, REG0 1855 | ldrb TMP1w, OP:REG0->result_type 1856 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1857 | beq >1 1858 | ldr REG0w, OP:REG0->result.var 1859 | add REG0, REG0, FP 1860 | SET_Z_TYPE_INFO REG0, IS_UNDEF, TMP1w 1861 |1: 1862 | b ->exception_handler 1863 1864 return 1; 1865} 1866 1867static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst) 1868{ 1869 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 1870 1871 |->exception_handler_free_op1_op2: 1872 | UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w 1873 | ldrb TMP1w, OP:RX->op1_type 1874 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1875 | beq >9 1876 | ldr REG0w, OP:RX->op1.var 1877 | add REG0, REG0, FP 1878 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 1879 |9: 1880 | ldrb TMP1w, OP:RX->op2_type 1881 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1882 | beq >9 1883 | ldr REG0w, OP:RX->op2.var 1884 | add REG0, REG0, FP 1885 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 1886 |9: 1887 | b ->exception_handler 1888 return 1; 1889} 1890 1891static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst) 1892{ 1893 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 1894 1895 |->exception_handler_free_op2: 1896 | MEM_LOAD_64_ZTS ldr, RX, executor_globals, opline_before_exception, REG0 1897 | UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w 1898 | ldrb TMP1w, OP:RX->op2_type 1899 | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w 1900 | beq >9 1901 | ldr REG0w, OP:RX->op2.var 1902 | add REG0, REG0, FP 1903 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 1904 |9: 1905 | b ->exception_handler 1906 return 1; 1907} 1908 1909static int zend_jit_leave_function_stub(dasm_State **Dst) 1910{ 1911 |->leave_function_handler: 1912 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1913 | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w 1914 | bne >1 1915 | EXT_CALL zend_jit_leave_nested_func_helper, REG0 1916 | ADD_HYBRID_SPAD 1917 | JMP_IP TMP1 1918 |1: 1919 | EXT_CALL zend_jit_leave_top_func_helper, REG0 1920 | ADD_HYBRID_SPAD 1921 | JMP_IP TMP1 1922 } else { 1923 if (GCC_GLOBAL_REGS) { 1924 | ldp x29, x30, [sp], # SPAD // stack alignment 1925 } else { 1926 | mov FCARG2x, FP 1927 | ldp FP, RX, T2 // restore FP and IP 1928 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1929 } 1930 | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w 1931 | bne >1 1932 | EXT_JMP zend_jit_leave_nested_func_helper, REG0 1933 |1: 1934 | EXT_JMP zend_jit_leave_top_func_helper, REG0 1935 } 1936 1937 return 1; 1938} 1939 1940static int zend_jit_leave_throw_stub(dasm_State **Dst) 1941{ 1942 |->leave_throw_handler: 1943 | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { 1944 if (GCC_GLOBAL_REGS) { 1945 | ldrb TMP1w, OP:IP->opcode 1946 | cmp TMP1w, #ZEND_HANDLE_EXCEPTION 1947 | beq >5 1948 | // EG(opline_before_exception) = opline; 1949 | MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2 1950 |5: 1951 | // opline = EG(exception_op); 1952 | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 1953 | str IP, EX->opline 1954 | // HANDLE_EXCEPTION() 1955 | b ->exception_handler 1956 } else { 1957 | GET_IP TMP1 1958 | ldrb TMP2w, OP:TMP1->opcode 1959 | cmp TMP2w, #ZEND_HANDLE_EXCEPTION 1960 | beq >5 1961 | // EG(opline_before_exception) = opline; 1962 | MEM_STORE_64_ZTS str, TMP1, executor_globals, opline_before_exception, TMP2 1963 |5: 1964 | // opline = EG(exception_op); 1965 | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 1966 | ldp FP, RX, T2 // restore FP and IP 1967 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 1968 | mov RETVALx, #2 // ZEND_VM_LEAVE 1969 | ret 1970 } 1971 1972 return 1; 1973} 1974 1975static int zend_jit_icall_throw_stub(dasm_State **Dst) 1976{ 1977 |->icall_throw_handler: 1978 | // zend_rethrow_exception(zend_execute_data *execute_data) 1979 | ldr IP, EX->opline 1980 | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { 1981 | ldrb TMP1w, OP:IP->opcode 1982 | cmp TMP1w, #ZEND_HANDLE_EXCEPTION 1983 | beq >1 1984 | // EG(opline_before_exception) = opline; 1985 | MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2 1986 |1: 1987 | // opline = EG(exception_op); 1988 | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 1989 || if (GCC_GLOBAL_REGS) { 1990 | str IP, EX->opline 1991 || } 1992 | // HANDLE_EXCEPTION() 1993 | b ->exception_handler 1994 1995 return 1; 1996} 1997 1998static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) 1999{ 2000 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 2001 2002 |->throw_cannot_pass_by_ref: 2003 | ldr REG0, EX->opline 2004 | ldr REG1w, OP:REG0->result.var 2005 | add REG1, REG1, RX 2006 | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w 2007 | // last EX(call) frame may be delayed 2008 | ldr TMP1, EX->call 2009 | cmp RX, TMP1 2010 | beq >1 2011 | ldr REG1, EX->call 2012 | str REG1, EX:RX->prev_execute_data 2013 | str RX, EX->call 2014 |1: 2015 | mov RX, REG0 2016 | ldr FCARG1w, OP:REG0->op2.num 2017 | EXT_CALL zend_cannot_pass_by_reference, REG0 2018 | ldrb TMP1w, OP:RX->op1_type 2019 | cmp TMP1w, #IS_TMP_VAR 2020 | bne >9 2021 | ldr REG0w, OP:RX->op1.var 2022 | add REG0, REG0, FP 2023 | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 2024 |9: 2025 | b ->exception_handler 2026 2027 return 1; 2028} 2029 2030static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) 2031{ 2032 |->undefined_offset_ex: 2033 | SAVE_IP 2034 | b ->undefined_offset 2035 2036 return 1; 2037} 2038 2039static int zend_jit_undefined_offset_stub(dasm_State **Dst) 2040{ 2041 |->undefined_offset: 2042 || if (!GCC_GLOBAL_REGS) { 2043 | mov FCARG1x, FP 2044 || } 2045 | EXT_JMP zend_jit_undefined_long_key, REG0 2046 2047 return 1; 2048} 2049 2050static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) 2051{ 2052 |->undefined_index_ex: 2053 | SAVE_IP 2054 | b ->undefined_index 2055 2056 return 1; 2057} 2058 2059static int zend_jit_undefined_index_stub(dasm_State **Dst) 2060{ 2061 |->undefined_index: 2062 || if (!GCC_GLOBAL_REGS) { 2063 | mov FCARG1x, FP 2064 || } 2065 | EXT_JMP zend_jit_undefined_string_key, REG0 2066 2067 return 1; 2068} 2069 2070static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) 2071{ 2072 |->cannot_add_element_ex: 2073 | SAVE_IP 2074 | b ->cannot_add_element 2075 2076 return 1; 2077} 2078 2079static int zend_jit_cannot_add_element_stub(dasm_State **Dst) 2080{ 2081 |->cannot_add_element: 2082 | // sub r4, 8 2083 | ldr REG0, EX->opline 2084 | ldrb TMP1w, OP:REG0->result_type 2085 | cmp TMP1w, #IS_UNUSED 2086 | beq >1 2087 | ldr REG0w, OP:REG0->result.var 2088 | add REG0, REG0, FP 2089 | SET_Z_TYPE_INFO REG0, IS_NULL, TMP1w 2090 |1: 2091 | mov CARG1, xzr 2092 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2093 | EXT_JMP zend_throw_error, REG0 // tail call 2094 | // add r4, 8 2095 | //ret 2096 2097 return 1; 2098} 2099 2100static int zend_jit_undefined_function_stub(dasm_State **Dst) 2101{ 2102 |->undefined_function: 2103 | ldr REG0, EX->opline 2104 | mov CARG1, xzr 2105 | LOAD_ADDR CARG2, "Call to undefined function %s()" 2106 | ldrsw CARG3, [REG0, #offsetof(zend_op, op2.constant)] 2107 | ldr CARG3, [REG0, CARG3] 2108 | add CARG3, CARG3, #offsetof(zend_string, val) 2109#ifdef __APPLE__ 2110 | str CARG3, [sp, #-16]! 2111#endif 2112 | EXT_CALL zend_throw_error, REG0 2113#ifdef __APPLE__ 2114 | add sp, sp, #16 2115#endif 2116 | b ->exception_handler 2117 return 1; 2118} 2119 2120static int zend_jit_negative_shift_stub(dasm_State **Dst) 2121{ 2122 |->negative_shift: 2123 | ldr RX, EX->opline 2124 | LOAD_ADDR CARG1, zend_ce_arithmetic_error 2125 | LOAD_ADDR CARG2, "Bit shift by negative number" 2126 | EXT_CALL zend_throw_error, REG0 2127 | b ->exception_handler_free_op1_op2 2128 return 1; 2129} 2130 2131static int zend_jit_mod_by_zero_stub(dasm_State **Dst) 2132{ 2133 |->mod_by_zero: 2134 | ldr RX, EX->opline 2135 | LOAD_ADDR CARG1, zend_ce_division_by_zero_error 2136 | LOAD_ADDR CARG2, "Modulo by zero" 2137 | EXT_CALL zend_throw_error, REG0 2138 | b ->exception_handler_free_op1_op2 2139 return 1; 2140} 2141 2142static int zend_jit_invalid_this_stub(dasm_State **Dst) 2143{ 2144 |->invalid_this: 2145 | UNDEF_OPLINE_RESULT TMP1w 2146 | mov CARG1, xzr 2147 | LOAD_ADDR CARG2, "Using $this when not in object context" 2148 | EXT_CALL zend_throw_error, REG0 2149 | b ->exception_handler 2150 2151 return 1; 2152} 2153 2154static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) 2155{ 2156 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2157 return 1; 2158 } 2159 2160 |->hybrid_runtime_jit: 2161 | EXT_CALL zend_runtime_jit, REG0 2162 | JMP_IP TMP1 2163 return 1; 2164} 2165 2166static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) 2167{ 2168 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2169 return 1; 2170 } 2171 2172 |->hybrid_profile_jit: 2173 | // ++zend_jit_profile_counter; 2174 | LOAD_ADDR REG0, &zend_jit_profile_counter 2175 | ldr TMP1, [REG0] 2176 | add TMP1, TMP1, #1 2177 | str TMP1, [REG0] 2178 | // op_array = (zend_op_array*)EX(func); 2179 | ldr REG0, EX->func 2180 | // run_time_cache = EX(run_time_cache); 2181 | ldr REG2, EX->run_time_cache 2182 | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); 2183 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2184 | // ++ZEND_COUNTER_INFO(op_array) 2185 || if ((zend_jit_profile_counter_rid * sizeof(void*)) > LDR_STR_PIMM64) { 2186 | LOAD_32BIT_VAL TMP1, (zend_jit_profile_counter_rid * sizeof(void*)) 2187 | ldr TMP2, [REG2, TMP1] 2188 | add TMP2, TMP2, #1 2189 | str TMP2, [REG2, TMP1] 2190 || } else { 2191 | ldr TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] 2192 | add TMP2, TMP2, #1 2193 | str TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] 2194 || } 2195 | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() 2196 | ldr TMP1, [REG0, #offsetof(zend_jit_op_array_extension, orig_handler)] 2197 | br TMP1 2198 return 1; 2199} 2200 2201static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) 2202{ 2203 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2204 return 1; 2205 } 2206 2207 |->hybrid_hot_code: 2208 || ZEND_ASSERT(ZEND_JIT_COUNTER_INIT <= MOVZ_IMM); 2209 | movz TMP1w, #ZEND_JIT_COUNTER_INIT 2210 | strh TMP1w, [REG2] 2211 | mov FCARG1x, FP 2212 | GET_IP FCARG2x 2213 | EXT_CALL zend_jit_hot_func, REG0 2214 | JMP_IP TMP1 2215 return 1; 2216} 2217 2218/* 2219 * This code is based Mike Pall's "Hashed profile counters" idea, implemented 2220 * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual 2221 * property disclosure and research opportunities" email 2222 * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html 2223 * 2224 * In addition we use a variation of Knuth's multiplicative hash function 2225 * described at https://code.i-harness.com/en/q/a21ce 2226 * 2227 * uint64_t hash(uint64_t x) { 2228 * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; 2229 * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; 2230 * x = x ^ (x >> 31); 2231 * return x; 2232 * } 2233 * 2234 * uint_32_t hash(uint32_t x) { 2235 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2236 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2237 * x = (x >> 16) ^ x; 2238 * return x; 2239 * } 2240 * 2241 */ 2242static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) 2243{ 2244 | ldr REG0, EX->func 2245 | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2246 | ldr REG2, [REG1, #offsetof(zend_jit_op_array_hot_extension, counter)] 2247 | ldrh TMP2w, [REG2] 2248 | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w 2249 | strh TMP2w, [REG2] 2250 | ble ->hybrid_hot_code 2251 | GET_IP REG2 2252 | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] 2253 | sub REG2, REG2, TMP1 2254 | // divide by sizeof(zend_op) 2255 || ZEND_ASSERT(sizeof(zend_op) == 32); 2256 | add TMP1, REG1, REG2, asr #2 2257 | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2258 | br TMP1 2259 return 1; 2260} 2261 2262static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) 2263{ 2264 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2265 return 1; 2266 } 2267 2268 |->hybrid_func_hot_counter: 2269 2270 return zend_jit_hybrid_hot_counter_stub(Dst, 2271 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2272} 2273 2274static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) 2275{ 2276 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2277 return 1; 2278 } 2279 2280 |->hybrid_loop_hot_counter: 2281 2282 return zend_jit_hybrid_hot_counter_stub(Dst, 2283 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2284} 2285 2286static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) 2287{ 2288 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2289 return 1; 2290 } 2291 2292 // On entry from counter stub: 2293 // REG2 -> zend_op_trace_info.counter 2294 2295 |->hybrid_hot_trace: 2296 | mov TMP1w, #ZEND_JIT_COUNTER_INIT 2297 | strh TMP1w, [REG2] 2298 | mov FCARG1x, FP 2299 | GET_IP FCARG2x 2300 | EXT_CALL zend_jit_trace_hot_root, REG0 2301 | tbnz RETVALw, #31, >1 // Result is < 0 on failure. 2302 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 2303 | LOAD_IP 2304 | JMP_IP TMP1 2305 |1: 2306 | EXT_JMP zend_jit_halt_op->handler, REG0 2307 2308 return 1; 2309} 2310 2311static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) 2312{ 2313 | ldr REG0, EX->func 2314 | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2315 | ldr REG1, [REG1, #offsetof(zend_jit_op_array_trace_extension, offset)] 2316 | add TMP1, REG1, IP 2317 | ldr REG2, [TMP1, #offsetof(zend_op_trace_info, counter)] 2318 | ldrh TMP2w, [REG2] 2319 | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w 2320 | strh TMP2w, [REG2] 2321 | ble ->hybrid_hot_trace 2322 // Note: "REG1 + IP" is re-calculated as TMP1 is used as temporary register by the prior 2323 // ADD_SUB_32_WITH_CONST. Will optimize in the future if more temporary registers are available. 2324 | add TMP1, REG1, IP 2325 | ldr TMP2, [TMP1, #offsetof(zend_op_trace_info, orig_handler)] 2326 | br TMP2 2327 2328 return 1; 2329} 2330 2331static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) 2332{ 2333 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2334 return 1; 2335 } 2336 2337 |->hybrid_func_trace_counter: 2338 2339 return zend_jit_hybrid_trace_counter_stub(Dst, 2340 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2341} 2342 2343static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) 2344{ 2345 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { 2346 return 1; 2347 } 2348 2349 |->hybrid_ret_trace_counter: 2350 2351 return zend_jit_hybrid_trace_counter_stub(Dst, 2352 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); 2353} 2354 2355static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) 2356{ 2357 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2358 return 1; 2359 } 2360 2361 |->hybrid_loop_trace_counter: 2362 2363 return zend_jit_hybrid_trace_counter_stub(Dst, 2364 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2365} 2366 2367static int zend_jit_trace_halt_stub(dasm_State **Dst) 2368{ 2369 |->trace_halt: 2370 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2371 | ADD_HYBRID_SPAD 2372 | EXT_JMP zend_jit_halt_op->handler, REG0 2373 } else if (GCC_GLOBAL_REGS) { 2374 | ldp x29, x30, [sp], # SPAD // stack alignment 2375 | mov IP, xzr // PC must be zero 2376 | ret 2377 } else { 2378 | ldp FP, RX, T2 // restore FP and IP 2379 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2380 | movn RETVALx, #0 // ZEND_VM_RETURN (-1) 2381 | ret 2382 } 2383 return 1; 2384} 2385 2386static int zend_jit_trace_exit_stub(dasm_State **Dst) 2387{ 2388 |->trace_exit: 2389 | 2390 | // Save CPU registers(32 GP regs + 32 FP regs) on stack in the order of d31 to x0 2391 | 2392 | stp d30, d31, [sp, #-16]! 2393 | stp d28, d29, [sp, #-16]! 2394 | stp d26, d27, [sp, #-16]! 2395 | stp d24, d25, [sp, #-16]! 2396 | stp d22, d23, [sp, #-16]! 2397 | stp d20, d21, [sp, #-16]! 2398 | stp d18, d19, [sp, #-16]! 2399 | stp d16, d17, [sp, #-16]! 2400 | //stp d14, d15, [sp, #-16]! // we don't use preserved registers yet 2401 | //stp d12, d13, [sp, #-16]! 2402 | //stp d10, d11, [sp, #-16]! 2403 | //stp d8, d9, [sp, #-16]! 2404 | stp d6, d7, [sp, #(-16*5)]! 2405 | stp d4, d5, [sp, #-16]! 2406 | stp d2, d3, [sp, #-16]! 2407 | stp d0, d1, [sp, #-16]! 2408 | 2409 | //str x30, [sp, #-16]! // we don't use callee-saved registers yet (x31 can be omitted) 2410 | stp x28, x29, [sp, #(-16*2)]! // we have to store RX (x28) 2411 | //stp x26, x27, [sp, #-16]! // we don't use callee-saved registers yet 2412 | //stp x24, x25, [sp, #-16]! 2413 | //stp x22, x23, [sp, #-16]! 2414 | //stp x20, x21, [sp, #-16]! 2415 | //stp x18, x19, [sp, #-16]! 2416 | //stp x16, x17, [sp, #-16]! // we don't need temporary registers 2417 | stp x14, x15, [sp, #-(16*7)]! 2418 | stp x12, x13, [sp, #-16]! 2419 | stp x10, x11, [sp, #-16]! 2420 | stp x8, x9, [sp, #-16]! 2421 | stp x6, x7, [sp, #-16]! 2422 | stp x4, x5, [sp, #-16]! 2423 | stp x2, x3, [sp, #-16]! 2424 | stp x0, x1, [sp, #-16]! 2425 | 2426 | mov FCARG1w, TMP1w // exit_num 2427 | mov FCARG2x, sp 2428 | 2429 | // EX(opline) = opline 2430 | SAVE_IP 2431 | // zend_jit_trace_exit(trace_num, exit_num) 2432 | EXT_CALL zend_jit_trace_exit, REG0 2433 | 2434 | add sp, sp, #(32 * 16) // restore sp 2435 | 2436 2437 | tst RETVALw, RETVALw 2438 | bne >1 // not zero 2439 2440 | // execute_data = EG(current_execute_data) 2441 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 2442 | // opline = EX(opline) 2443 | LOAD_IP 2444 2445 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2446 | ADD_HYBRID_SPAD 2447 | JMP_IP TMP1 2448 } else if (GCC_GLOBAL_REGS) { 2449 | ldp x29, x30, [sp], # SPAD // stack alignment 2450 | JMP_IP TMP1 2451 } else { 2452 | ldp FP, RX, T2 // restore FP and IP 2453 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2454 | mov RETVALx, #1 // ZEND_VM_ENTER 2455 | ret 2456 } 2457 2458 |1: 2459 | blt ->trace_halt 2460 2461 | // execute_data = EG(current_execute_data) 2462 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 2463 | // opline = EX(opline) 2464 | LOAD_IP 2465 2466 | // check for interrupt (try to avoid this ???) 2467 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 2468 | cbnz TMP1w, ->interrupt_handler 2469 2470 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2471 | ADD_HYBRID_SPAD 2472 | ldr REG0, EX->func 2473 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2474 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 2475 | ldr REG0, [IP, REG0] 2476 | br REG0 2477 } else if (GCC_GLOBAL_REGS) { 2478 | ldp x29, x30, [sp], # SPAD // stack alignment 2479 | ldr REG0, EX->func 2480 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2481 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 2482 | ldr REG0, [IP, REG0] 2483 | br REG0 2484 } else { 2485 | ldr IP, EX->opline 2486 | mov FCARG1x, FP 2487 | ldr REG0, EX->func 2488 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 2489 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 2490 | ldr REG0, [IP, REG0] 2491 | blr REG0 2492 | 2493 | tst RETVALw, RETVALw 2494 | blt ->trace_halt 2495 | 2496 | ldp FP, RX, T2 // restore FP and IP 2497 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2498 | mov RETVALx, #1 // ZEND_VM_ENTER 2499 | ret 2500 } 2501 2502 return 1; 2503} 2504 2505static int zend_jit_trace_escape_stub(dasm_State **Dst) 2506{ 2507 |->trace_escape: 2508 | 2509 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2510 | ADD_HYBRID_SPAD 2511 | JMP_IP, TMP1 2512 } else if (GCC_GLOBAL_REGS) { 2513 | ldp x29, x30, [sp], # SPAD // stack alignment 2514 | JMP_IP, TMP1 2515 } else { 2516 | ldp FP, RX, T2 // restore FP and IP 2517 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 2518 | mov RETVALx, #1 // ZEND_VM_ENTER 2519 | ret 2520 } 2521 2522 return 1; 2523} 2524 2525/* Keep 32 exit points in a single code block */ 2526#define ZEND_JIT_EXIT_POINTS_SPACING 4 // bl = bytes 2527#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points 2528 2529static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) 2530{ 2531 uint32_t i; 2532 2533 | bl >2 2534 |1: 2535 for (i = 1; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { 2536 | bl >2 2537 } 2538 |2: 2539 | adr TMP1, <1 2540 | sub TMP1, lr, TMP1 2541 | lsr TMP1, TMP1, #2 2542 if (n) { 2543 | ADD_SUB_32_WITH_CONST add, TMP1w, TMP1w, n, TMP2w 2544 } 2545 | b ->trace_exit // pass exit_num in TMP1w 2546 2547 return 1; 2548} 2549 2550#ifdef CONTEXT_THREADED_JIT 2551static int zend_jit_context_threaded_call_stub(dasm_State **Dst) 2552{ 2553 |->context_threaded_call: 2554 | NIY_STUB // TODO 2555 return 1; 2556} 2557#endif 2558 2559static int zend_jit_assign_const_stub(dasm_State **Dst) 2560{ 2561 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2562 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2563 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2564 2565 |->assign_const: 2566 | stp x29, x30, [sp, #-32]! 2567 | mov x29, sp 2568 if (!zend_jit_assign_to_variable( 2569 Dst, NULL, 2570 var_addr, var_addr, -1, -1, 2571 IS_CONST, val_addr, val_info, 2572 0, 0)) { 2573 return 0; 2574 } 2575 | ldp x29, x30, [sp], #32 2576 | ret 2577 return 1; 2578} 2579 2580static int zend_jit_assign_tmp_stub(dasm_State **Dst) 2581{ 2582 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2583 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2584 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2585 2586 |->assign_tmp: 2587 | stp x29, x30, [sp, #-32]! 2588 | mov x29, sp 2589 if (!zend_jit_assign_to_variable( 2590 Dst, NULL, 2591 var_addr, var_addr, -1, -1, 2592 IS_TMP_VAR, val_addr, val_info, 2593 0, 0)) { 2594 return 0; 2595 } 2596 | ldp x29, x30, [sp], #32 2597 | ret 2598 return 1; 2599} 2600 2601static int zend_jit_assign_var_stub(dasm_State **Dst) 2602{ 2603 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2604 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2605 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; 2606 2607 |->assign_var: 2608 | stp x29, x30, [sp, #-32]! 2609 | mov x29, sp 2610 if (!zend_jit_assign_to_variable( 2611 Dst, NULL, 2612 var_addr, var_addr, -1, -1, 2613 IS_VAR, val_addr, val_info, 2614 0, 0)) { 2615 return 0; 2616 } 2617 | ldp x29, x30, [sp], #32 2618 | ret 2619 return 1; 2620} 2621 2622static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) 2623{ 2624 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2625 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2626 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; 2627 2628 |->assign_cv_noref: 2629 | stp x29, x30, [sp, #-32]! 2630 | mov x29, sp 2631 if (!zend_jit_assign_to_variable( 2632 Dst, NULL, 2633 var_addr, var_addr, -1, -1, 2634 IS_CV, val_addr, val_info, 2635 0, 0)) { 2636 return 0; 2637 } 2638 | ldp x29, x30, [sp], #32 2639 | ret 2640 return 1; 2641} 2642 2643static int zend_jit_assign_cv_stub(dasm_State **Dst) 2644{ 2645 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2646 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2647 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; 2648 2649 |->assign_cv: 2650 | stp x29, x30, [sp, #-32]! 2651 | mov x29, sp 2652 if (!zend_jit_assign_to_variable( 2653 Dst, NULL, 2654 var_addr, var_addr, -1, -1, 2655 IS_CV, val_addr, val_info, 2656 0, 0)) { 2657 return 0; 2658 } 2659 | ldp x29, x30, [sp], #32 2660 | ret 2661 return 1; 2662} 2663 2664static const zend_jit_stub zend_jit_stubs[] = { 2665 JIT_STUB(interrupt_handler, SP_ADJ_JIT, SP_ADJ_VM), 2666 JIT_STUB(exception_handler, SP_ADJ_JIT, SP_ADJ_VM), 2667 JIT_STUB(exception_handler_undef, SP_ADJ_JIT, SP_ADJ_VM), 2668 JIT_STUB(exception_handler_free_op1_op2, SP_ADJ_JIT, SP_ADJ_VM), 2669 JIT_STUB(exception_handler_free_op2, SP_ADJ_JIT, SP_ADJ_VM), 2670 JIT_STUB(leave_function, SP_ADJ_JIT, SP_ADJ_VM), 2671 JIT_STUB(leave_throw, SP_ADJ_JIT, SP_ADJ_VM), 2672 JIT_STUB(icall_throw, SP_ADJ_JIT, SP_ADJ_VM), 2673 JIT_STUB(throw_cannot_pass_by_ref, SP_ADJ_JIT, SP_ADJ_VM), 2674 JIT_STUB(undefined_offset, SP_ADJ_JIT, SP_ADJ_VM), 2675 JIT_STUB(undefined_index, SP_ADJ_JIT, SP_ADJ_VM), 2676 JIT_STUB(cannot_add_element, SP_ADJ_JIT, SP_ADJ_VM), 2677 JIT_STUB(undefined_offset_ex, SP_ADJ_JIT, SP_ADJ_VM), 2678 JIT_STUB(undefined_index_ex, SP_ADJ_JIT, SP_ADJ_VM), 2679 JIT_STUB(cannot_add_element_ex, SP_ADJ_JIT, SP_ADJ_VM), 2680 JIT_STUB(undefined_function, SP_ADJ_JIT, SP_ADJ_VM), 2681 JIT_STUB(negative_shift, SP_ADJ_JIT, SP_ADJ_VM), 2682 JIT_STUB(mod_by_zero, SP_ADJ_JIT, SP_ADJ_VM), 2683 JIT_STUB(invalid_this, SP_ADJ_JIT, SP_ADJ_VM), 2684 JIT_STUB(trace_halt, SP_ADJ_JIT, SP_ADJ_VM), 2685 JIT_STUB(trace_exit, SP_ADJ_JIT, SP_ADJ_VM), 2686 JIT_STUB(trace_escape, SP_ADJ_JIT, SP_ADJ_VM), 2687 JIT_STUB(hybrid_runtime_jit, SP_ADJ_VM, SP_ADJ_NONE), 2688 JIT_STUB(hybrid_profile_jit, SP_ADJ_VM, SP_ADJ_NONE), 2689 JIT_STUB(hybrid_hot_code, SP_ADJ_VM, SP_ADJ_NONE), 2690 JIT_STUB(hybrid_func_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2691 JIT_STUB(hybrid_loop_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2692 JIT_STUB(hybrid_hot_trace, SP_ADJ_VM, SP_ADJ_NONE), 2693 JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2694 JIT_STUB(hybrid_ret_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2695 JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2696 JIT_STUB(assign_const, SP_ADJ_RET, SP_ADJ_ASSIGN), 2697 JIT_STUB(assign_tmp, SP_ADJ_RET, SP_ADJ_ASSIGN), 2698 JIT_STUB(assign_var, SP_ADJ_RET, SP_ADJ_ASSIGN), 2699 JIT_STUB(assign_cv_noref, SP_ADJ_RET, SP_ADJ_ASSIGN), 2700 JIT_STUB(assign_cv, SP_ADJ_RET, SP_ADJ_ASSIGN), 2701#ifdef CONTEXT_THREADED_JIT 2702 JIT_STUB(context_threaded_call, SP_ADJ_NONE, SP_ADJ_NONE), 2703#endif 2704}; 2705 2706#ifdef HAVE_GDB 2707# if 0 2708typedef struct _Unwind_Context _Unwind_Context; 2709typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *); 2710extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *); 2711extern uintptr_t _Unwind_GetCFA(_Unwind_Context *); 2712 2713typedef struct _zend_jit_unwind_arg { 2714 int cnt; 2715 uintptr_t cfa[3]; 2716} zend_jit_unwind_arg; 2717 2718static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a) 2719{ 2720 zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a; 2721 arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx); 2722 arg->cnt++; 2723 if (arg->cnt == 3) { 2724 return 5; // _URC_END_OF_STACK 2725 } 2726 return 0; // _URC_NO_REASON; 2727} 2728 2729static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) 2730{ 2731 zend_jit_unwind_arg arg; 2732 2733 memset(&arg, 0, sizeof(arg)); 2734 _Unwind_Backtrace(zend_jit_unwind_cb, &arg); 2735 if (arg.cnt == 3) { 2736 sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1]; 2737 } 2738} 2739# else 2740static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) 2741{ 2742 uintptr_t ret; 2743 2744 __asm__ ( 2745 "ldr %0, [x29]\n\t" 2746 "sub %0 ,%0, x29" 2747 : "=r"(ret)); 2748 2749 sp_adj[SP_ADJ_VM] = ret; 2750} 2751# endif 2752 2753extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data); 2754 2755static zend_never_inline void zend_jit_set_sp_adj_vm(void) 2756{ 2757 void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *); 2758 2759 orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data; 2760 zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data; 2761 execute_ex(NULL); // set sp_adj[SP_ADJ_VM] 2762 zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data; 2763} 2764#endif 2765 2766static int zend_jit_setup(void) 2767{ 2768 allowed_opt_flags = 0; 2769 2770#if ZTS 2771 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2772# if defined(__FreeBSD__) 2773 if (tsrm_ls_cache_tcb_offset == 0) { 2774 TLSDescriptor **where; 2775 2776 __asm__( 2777 "adrp %0, :tlsdesc:_tsrm_ls_cache\n" 2778 "add %0, %0, :tlsdesc_lo12:_tsrm_ls_cache\n" 2779 : "=r" (where)); 2780 /* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst 2781 * section "Relocations for thread-local storage". 2782 * The first entry holds a pointer to the variable's TLS descriptor resolver function and the second entry holds 2783 * a platform-specific offset or pointer. */ 2784 TLSDescriptor *tlsdesc = where[1]; 2785 2786 tsrm_tls_offset = tlsdesc->offset; 2787 /* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/22ca6db50f4e6bd75a141f57cf953d8de6531a06/lib/libc/gen/tls.c#L88) */ 2788 tsrm_tls_index = (tlsdesc->index + 1) * 8; 2789 } 2790# else 2791 ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0); 2792# endif 2793#endif 2794 2795 memset(sp_adj, 0, sizeof(sp_adj)); 2796#ifdef HAVE_GDB 2797 sp_adj[SP_ADJ_RET] = 0; 2798 sp_adj[SP_ADJ_ASSIGN] = 32; 2799 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2800 zend_jit_set_sp_adj_vm(); // set sp_adj[SP_ADJ_VM] 2801#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 2802 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD 2803#else 2804 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM]; 2805#endif 2806 } else if (GCC_GLOBAL_REGS) { 2807 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD; // sub r4, SPAD 2808 } else { 2809 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD; // sub r4, NR_SPAD 2810 } 2811#endif 2812 2813 return SUCCESS; 2814} 2815 2816static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) 2817{ 2818 | brk #0 2819 return 1; 2820} 2821 2822static int zend_jit_align_func(dasm_State **Dst) 2823{ 2824 reuse_ip = 0; 2825 delayed_call_chain = 0; 2826 last_valid_opline = NULL; 2827 use_last_vald_opline = 0; 2828 track_last_valid_opline = 0; 2829 jit_return_label = -1; 2830 |.align 16 2831 return 1; 2832} 2833 2834static int zend_jit_align_stub(dasm_State **Dst) 2835{ 2836 |.align 16 2837 return 1; 2838} 2839 2840static int zend_jit_prologue(dasm_State **Dst) 2841{ 2842 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2843 | SUB_HYBRID_SPAD 2844 } else if (GCC_GLOBAL_REGS) { 2845 | stp x29, x30, [sp, # -SPAD]! // stack alignment 2846 |// mov x29, sp 2847 } else { 2848 | stp x29, x30, [sp, # -NR_SPAD]! // stack alignment 2849 |// mov x29, sp 2850 | stp FP, RX, T2 // save FP and IP 2851 | mov FP, FCARG1x 2852 } 2853 return 1; 2854} 2855 2856static int zend_jit_label(dasm_State **Dst, unsigned int label) 2857{ 2858 |=>label: 2859 return 1; 2860} 2861 2862static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) 2863{ 2864 | // call->prev_execute_data = EX(call); 2865 if (call_level == 1) { 2866 | str xzr, EX:RX->prev_execute_data 2867 } else { 2868 | ldr REG0, EX->call 2869 | str REG0, EX:RX->prev_execute_data 2870 } 2871 | // EX(call) = call; 2872 | str RX, EX->call 2873 2874 delayed_call_chain = 0; 2875 2876 return 1; 2877} 2878 2879static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) 2880{ 2881 if (last_valid_opline == opline) { 2882 zend_jit_use_last_valid_opline(); 2883 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 2884 zend_jit_use_last_valid_opline(); 2885 | LOAD_64BIT_VAL TMP1, (opline - last_valid_opline) * sizeof(zend_op) 2886 | ADD_IP TMP1, TMP2 2887 } else { 2888 | LOAD_IP_ADDR opline 2889 } 2890 zend_jit_set_last_valid_opline(opline); 2891 2892 return 1; 2893} 2894 2895static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg) 2896{ 2897 return zend_jit_set_ip(Dst, opline); 2898} 2899 2900static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) 2901{ 2902 if (delayed_call_chain) { 2903 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 2904 return 0; 2905 } 2906 } 2907 if (!zend_jit_set_ip(Dst, opline)) { 2908 return 0; 2909 } 2910 reuse_ip = 0; 2911 return 1; 2912} 2913 2914static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) 2915{ 2916 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 2917 if (exit_addr) { 2918 | cbnz TMP1w, &exit_addr 2919 } else if (last_valid_opline == opline) { 2920 || zend_jit_use_last_valid_opline(); 2921 | cbnz TMP1w, ->interrupt_handler 2922 } else { 2923 | cbnz TMP1w, >1 2924 |.cold_code 2925 |1: 2926 | LOAD_IP_ADDR opline 2927 | b ->interrupt_handler 2928 |.code 2929 } 2930 return 1; 2931} 2932 2933static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) 2934{ 2935 if (timeout_exit_addr) { 2936 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 2937 | cbz TMP1w, =>loop_label 2938 | b &timeout_exit_addr 2939 } else { 2940 | b =>loop_label 2941 } 2942 return 1; 2943} 2944 2945static int zend_jit_check_exception(dasm_State **Dst) 2946{ 2947 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 2948 | cbnz TMP2, ->exception_handler 2949 return 1; 2950} 2951 2952static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) 2953{ 2954 if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { 2955 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 2956 | cbnz TMP2, ->exception_handler_undef 2957 return 1; 2958 } 2959 return zend_jit_check_exception(Dst); 2960} 2961 2962static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) 2963{ 2964 2965 current_trace_num = trace_num; 2966 2967 | // EG(jit_trace_num) = trace_num; 2968 | LOAD_32BIT_VAL TMP1w, trace_num 2969 | MEM_STORE_32_ZTS str, TMP1w, executor_globals, jit_trace_num, TMP2 2970 2971 return 1; 2972} 2973 2974static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t) 2975{ 2976 uint32_t i; 2977 const void *exit_addr; 2978 2979 /* Emit veneers table for exit points (B instruction for each exit number) */ 2980 |.cold_code 2981 for (i = 0; i < t->exit_count; i++) { 2982 exit_addr = zend_jit_trace_get_exit_addr(i); 2983 if (!exit_addr) { 2984 return 0; 2985 } 2986 | b &exit_addr 2987 } 2988 |=>1: // end of the code 2989 |.code 2990 return 1; 2991} 2992 2993static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) 2994{ 2995 int ret = 0; 2996 uint8_t *p, *end; 2997 const void *veneer = NULL; 2998 ptrdiff_t delta; 2999 3000 if (jmp_table_size) { 3001 const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*))); 3002 3003 do { 3004 if (*jmp_slot == from_addr) { 3005 *jmp_slot = to_addr; 3006 ret++; 3007 } 3008 jmp_slot++; 3009 } while (--jmp_table_size); 3010 } 3011 3012 end = (uint8_t*)code; 3013 p = end + size; 3014 while (p > end) { 3015 uint32_t *ins_ptr; 3016 uint32_t ins; 3017 3018 p -= 4; 3019 ins_ptr = (uint32_t*)p; 3020 ins = *ins_ptr; 3021 if ((ins & 0xfc000000u) == 0x14000000u) { 3022 // B (imm26:0..25) 3023 delta = (uint32_t*)from_addr - ins_ptr; 3024 if (((ins ^ (uint32_t)delta) & 0x01ffffffu) == 0) { 3025 delta = (uint32_t*)to_addr - ins_ptr; 3026 if (((delta + 0x02000000) >> 26) != 0) { 3027 abort(); // branch target out of range 3028 } 3029 *ins_ptr = (ins & 0xfc000000u) | ((uint32_t)delta & 0x03ffffffu); 3030 ret++; 3031 if (!veneer) { 3032 veneer = p; 3033 } 3034 } 3035 } else if ((ins & 0xff000000u) == 0x54000000u || 3036 (ins & 0x7e000000u) == 0x34000000u) { 3037 // B.cond, CBZ, CBNZ (imm19:5..23) 3038 delta = (uint32_t*)from_addr - ins_ptr; 3039 if (((ins ^ ((uint32_t)delta << 5)) & 0x00ffffe0u) == 0) { 3040 delta = (uint32_t*)to_addr - ins_ptr; 3041 if (((delta + 0x40000) >> 19) != 0) { 3042 if (veneer) { 3043 delta = (uint32_t*)veneer - ins_ptr; 3044 if (((delta + 0x40000) >> 19) != 0) { 3045 abort(); // branch target out of range 3046 } 3047 } else { 3048 abort(); // branch target out of range 3049 } 3050 } 3051 *ins_ptr = (ins & 0xff00001fu) | (((uint32_t)delta & 0x7ffffu) << 5); 3052 ret++; 3053 } 3054 } else if ((ins & 0x7e000000u) == 0x36000000u) { 3055 // TBZ, TBNZ (imm14:5..18) 3056 delta = (uint32_t*)from_addr - ins_ptr; 3057 if (((ins ^ ((uint32_t)delta << 5)) & 0x0007ffe0u) == 0) { 3058 delta = (uint32_t*)to_addr - ins_ptr; 3059 if (((delta + 0x2000) >> 14) != 0) { 3060 if (veneer) { 3061 delta = (uint32_t*)veneer - ins_ptr; 3062 if (((delta + 0x2000) >> 14) != 0) { 3063 abort(); // branch target out of range 3064 } 3065 } else { 3066 abort(); // branch target out of range 3067 } 3068 } 3069 *ins_ptr = (ins & 0xfff8001fu) | (((uint32_t)delta & 0x3fffu) << 5); 3070 ret++; 3071 } 3072 } 3073 } 3074 3075 JIT_CACHE_FLUSH(code, (char*)code + size); 3076 3077#ifdef HAVE_VALGRIND 3078 VALGRIND_DISCARD_TRANSLATIONS(code, size); 3079#endif 3080 3081 return ret; 3082} 3083 3084static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) 3085{ 3086 return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); 3087} 3088 3089static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) 3090{ 3091 const void *link_addr; 3092 size_t prologue_size; 3093 3094 /* Skip prologue. */ 3095 // TODO: don't hardcode this ??? 3096 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3097#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 3098 prologue_size = 0; 3099#else 3100 // sub sp, sp, #0x20 3101 prologue_size = 4; 3102#endif 3103 } else if (GCC_GLOBAL_REGS) { 3104 // stp x29, x30, [sp, # -SPAD]! 3105 prologue_size = 4; 3106 } else { 3107 // stp x29, x30, [sp, # -NR_SPAD]! // stack alignment 3108 // stp FP, RX, T2 3109 // mov FP, FCARG1x 3110 prologue_size = 12; 3111 } 3112 link_addr = (const void*)((const char*)t->code_start + prologue_size); 3113 3114 if (timeout_exit_addr) { 3115 /* Check timeout for links to LOOP */ 3116 | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 3117 | cbz TMP1w, &link_addr 3118 | b &timeout_exit_addr 3119 } else { 3120 | b &link_addr 3121 } 3122 return 1; 3123} 3124 3125static int zend_jit_trace_return(dasm_State **Dst, bool original_handler, const zend_op *opline) 3126{ 3127 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3128 | ADD_HYBRID_SPAD 3129 if (!original_handler) { 3130 | JMP_IP TMP1 3131 } else { 3132 | ldr REG0, EX->func 3133 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 3134 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 3135 | ldr REG0, [IP, REG0] 3136 | br REG0 3137 } 3138 } else if (GCC_GLOBAL_REGS) { 3139 | ldp x29, x30, [sp], # SPAD // stack alignment 3140 if (!original_handler) { 3141 | JMP_IP TMP1 3142 } else { 3143 | ldr REG0, EX->func 3144 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 3145 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 3146 | ldr REG0, [IP, REG0] 3147 | br REG0 3148 } 3149 } else { 3150 if (original_handler) { 3151 | mov FCARG1x, FP 3152 | ldr REG0, EX->func 3153 | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] 3154 | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] 3155 | ldr REG0, [IP, REG0] 3156 | blr REG0 3157 } 3158 | ldp FP, RX, T2 // restore FP and IP 3159 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 3160 if (!original_handler || !opline || 3161 (opline->opcode != ZEND_RETURN 3162 && opline->opcode != ZEND_RETURN_BY_REF 3163 && opline->opcode != ZEND_GENERATOR_RETURN 3164 && opline->opcode != ZEND_GENERATOR_CREATE 3165 && opline->opcode != ZEND_YIELD 3166 && opline->opcode != ZEND_YIELD_FROM)) { 3167 | mov RETVALx, #2 // ZEND_VM_LEAVE 3168 } 3169 | ret 3170 } 3171 return 1; 3172} 3173 3174static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) 3175{ 3176 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3177 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3178 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3179 3180 if (!exit_addr) { 3181 return 0; 3182 } 3183 3184 | IF_NOT_ZVAL_TYPE var_addr, type, &exit_addr, ZREG_TMP1 3185 3186 return 1; 3187} 3188 3189static int zend_jit_scalar_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var) 3190{ 3191 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3192 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3193 3194 if (!exit_addr) { 3195 return 0; 3196 } 3197 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, var+offsetof(zval, u1.v.type), TMP1 3198 | cmp TMP1w, #IS_STRING 3199 | bhs &exit_addr 3200 3201 return 1; 3202} 3203static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) 3204{ 3205 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 3206 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3207 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3208 3209 if (!exit_addr) { 3210 return 0; 3211 } 3212 3213 | GET_ZVAL_LVAL ZREG_FCARG1, var_addr, TMP1 3214 if (op_info & MAY_BE_ARRAY_PACKED) { 3215 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 3216 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 3217 | beq &exit_addr 3218 } else { 3219 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 3220 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 3221 | bne &exit_addr 3222 } 3223 3224 return 1; 3225} 3226 3227static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) 3228{ 3229 zend_jit_op_array_trace_extension *jit_extension = 3230 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); 3231 size_t offset = jit_extension->offset; 3232 const void *handler = 3233 (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; 3234 3235 if (!zend_jit_set_valid_ip(Dst, opline)) { 3236 return 0; 3237 } 3238 if (!GCC_GLOBAL_REGS) { 3239 | mov FCARG1x, FP 3240 } 3241 | EXT_CALL handler, REG0 3242 if (may_throw 3243 && opline->opcode != ZEND_RETURN 3244 && opline->opcode != ZEND_RETURN_BY_REF) { 3245 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 3246 | cbnz REG0, ->exception_handler 3247 } 3248 3249 while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { 3250 trace++; 3251 } 3252 3253 if (!GCC_GLOBAL_REGS 3254 && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { 3255 if (opline->opcode == ZEND_RETURN || 3256 opline->opcode == ZEND_RETURN_BY_REF || 3257 opline->opcode == ZEND_DO_UCALL || 3258 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3259 opline->opcode == ZEND_DO_FCALL || 3260 opline->opcode == ZEND_GENERATOR_CREATE) { 3261 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 3262 } 3263 } 3264 3265 if (zend_jit_trace_may_exit(op_array, opline)) { 3266 if (opline->opcode == ZEND_RETURN || 3267 opline->opcode == ZEND_RETURN_BY_REF || 3268 opline->opcode == ZEND_GENERATOR_CREATE) { 3269 3270 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3271#if 0 3272 /* this check should be handled by the following OPLINE guard or jmp [IP] */ 3273 | LOAD_ADDR TMP1, zend_jit_halt_op 3274 | cmp IP, TMP1 3275 | beq ->trace_halt 3276#endif 3277 } else if (GCC_GLOBAL_REGS) { 3278 | cbz IP, ->trace_halt 3279 } else { 3280 | tst RETVALw, RETVALw 3281 | blt ->trace_halt 3282 } 3283 } else if (opline->opcode == ZEND_EXIT || 3284 opline->opcode == ZEND_GENERATOR_RETURN || 3285 opline->opcode == ZEND_YIELD || 3286 opline->opcode == ZEND_YIELD_FROM) { 3287 | b ->trace_halt 3288 } 3289 if (trace->op != ZEND_JIT_TRACE_END || 3290 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && 3291 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 3292 3293 const zend_op *next_opline = trace->opline; 3294 const zend_op *exit_opline = NULL; 3295 uint32_t exit_point; 3296 const void *exit_addr; 3297 uint32_t old_info = 0; 3298 uint32_t old_res_info = 0; 3299 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 3300 3301 if (zend_is_smart_branch(opline)) { 3302 bool exit_if_true = 0; 3303 exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); 3304 } else { 3305 switch (opline->opcode) { 3306 case ZEND_JMPZ: 3307 case ZEND_JMPNZ: 3308 case ZEND_JMPZ_EX: 3309 case ZEND_JMPNZ_EX: 3310 case ZEND_JMP_SET: 3311 case ZEND_COALESCE: 3312 case ZEND_JMP_NULL: 3313 case ZEND_FE_RESET_R: 3314 case ZEND_FE_RESET_RW: 3315 exit_opline = (trace->opline == opline + 1) ? 3316 OP_JMP_ADDR(opline, opline->op2) : 3317 opline + 1; 3318 break; 3319 case ZEND_FE_FETCH_R: 3320 case ZEND_FE_FETCH_RW: 3321 exit_opline = (trace->opline == opline + 1) ? 3322 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3323 opline + 1; 3324 break; 3325 3326 } 3327 } 3328 3329 switch (opline->opcode) { 3330 case ZEND_FE_FETCH_R: 3331 case ZEND_FE_FETCH_RW: 3332 if (opline->op2_type != IS_UNUSED) { 3333 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); 3334 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); 3335 } 3336 break; 3337 case ZEND_BIND_INIT_STATIC_OR_JMP: 3338 if (opline->op1_type == IS_CV) { 3339 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 3340 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1); 3341 } 3342 break; 3343 } 3344 3345 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3346 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3347 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 3348 } 3349 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); 3350 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3351 3352 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3353 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3354 } 3355 switch (opline->opcode) { 3356 case ZEND_FE_FETCH_R: 3357 case ZEND_FE_FETCH_RW: 3358 if (opline->op2_type != IS_UNUSED) { 3359 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); 3360 } 3361 break; 3362 case ZEND_BIND_INIT_STATIC_OR_JMP: 3363 if (opline->op1_type == IS_CV) { 3364 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info); 3365 } 3366 break; 3367 } 3368 3369 if (!exit_addr) { 3370 return 0; 3371 } 3372 | CMP_IP next_opline, TMP1, TMP2 3373 | bne &exit_addr 3374 } 3375 } 3376 3377 zend_jit_set_last_valid_opline(trace->opline); 3378 3379 return 1; 3380} 3381 3382static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) 3383{ 3384 const void *handler; 3385 3386 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3387 handler = zend_get_opcode_handler_func(opline); 3388 } else { 3389 handler = opline->handler; 3390 } 3391 3392 if (!zend_jit_set_valid_ip(Dst, opline)) { 3393 return 0; 3394 } 3395 if (!GCC_GLOBAL_REGS) { 3396 | mov FCARG1x, FP 3397 } 3398 | EXT_CALL handler, REG0 3399 if (may_throw) { 3400 zend_jit_check_exception(Dst); 3401 } 3402 3403 /* Skip the following OP_DATA */ 3404 switch (opline->opcode) { 3405 case ZEND_ASSIGN_DIM: 3406 case ZEND_ASSIGN_OBJ: 3407 case ZEND_ASSIGN_STATIC_PROP: 3408 case ZEND_ASSIGN_DIM_OP: 3409 case ZEND_ASSIGN_OBJ_OP: 3410 case ZEND_ASSIGN_STATIC_PROP_OP: 3411 case ZEND_ASSIGN_STATIC_PROP_REF: 3412 case ZEND_ASSIGN_OBJ_REF: 3413 zend_jit_set_last_valid_opline(opline + 2); 3414 break; 3415 default: 3416 zend_jit_set_last_valid_opline(opline + 1); 3417 break; 3418 } 3419 3420 return 1; 3421} 3422 3423static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) 3424{ 3425 if (!zend_jit_set_valid_ip(Dst, opline)) { 3426 return 0; 3427 } 3428 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3429 if (opline->opcode == ZEND_DO_UCALL || 3430 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3431 opline->opcode == ZEND_DO_FCALL || 3432 opline->opcode == ZEND_RETURN) { 3433 3434 /* Use inlined HYBRID VM handler */ 3435 const void *handler = opline->handler; 3436 3437 | ADD_HYBRID_SPAD 3438 | EXT_JMP handler, REG0 3439 } else { 3440 const void *handler = zend_get_opcode_handler_func(opline); 3441 3442 | EXT_CALL handler, REG0 3443 | ADD_HYBRID_SPAD 3444 | JMP_IP TMP1 3445 } 3446 } else { 3447 const void *handler = opline->handler; 3448 3449 if (GCC_GLOBAL_REGS) { 3450 | ldp x29, x30, [sp], # SPAD // stack alignment 3451 } else { 3452 | mov FCARG1x, FP 3453 | ldp FP, RX, T2 // restore FP and IP 3454 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 3455 } 3456 | EXT_JMP handler, REG0 3457 } 3458 zend_jit_reset_last_valid_opline(); 3459 return 1; 3460} 3461 3462static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) 3463{ 3464 uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); 3465 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3466 3467 if (!exit_addr) { 3468 return 0; 3469 } 3470 | CMP_IP opline, TMP1, TMP2 3471 | bne &exit_addr 3472 3473 zend_jit_set_last_valid_opline(opline); 3474 3475 return 1; 3476} 3477 3478static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) 3479{ 3480 | b =>target_label 3481 return 1; 3482} 3483 3484static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) 3485{ 3486 | CMP_IP next_opline, TMP1, TMP2 3487 | bne =>target_label 3488 3489 zend_jit_set_last_valid_opline(next_opline); 3490 3491 return 1; 3492} 3493 3494#ifdef CONTEXT_THREADED_JIT 3495static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3496{ 3497 | NIY // TODO 3498 return 1; 3499} 3500#endif 3501 3502static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3503{ 3504#ifdef CONTEXT_THREADED_JIT 3505 return zend_jit_context_threaded_call(Dst, opline, next_block); 3506#else 3507 return zend_jit_tail_handler(Dst, opline); 3508#endif 3509} 3510 3511static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type) 3512{ 3513 ZEND_ASSERT(Z_MODE(src) == IS_REG); 3514 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); 3515 3516 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3517 | SET_ZVAL_LVAL_FROM_REG dst, Rx(Z_REG(src)), TMP1 3518 if (set_type && 3519 (Z_REG(dst) != ZREG_FP || 3520 !JIT_G(current_frame) || 3521 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) { 3522 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 3523 } 3524 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3525 | SET_ZVAL_DVAL dst, Z_REG(src), ZREG_TMP1 3526 if (set_type && 3527 (Z_REG(dst) != ZREG_FP || 3528 !JIT_G(current_frame) || 3529 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) { 3530 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 3531 } 3532 } else { 3533 ZEND_UNREACHABLE(); 3534 } 3535 return 1; 3536} 3537 3538static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3539{ 3540 ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); 3541 ZEND_ASSERT(Z_MODE(dst) == IS_REG); 3542 3543 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3544 | GET_ZVAL_LVAL Z_REG(dst), src, TMP1 3545 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3546 | GET_ZVAL_DVAL Z_REG(dst), src, ZREG_TMP1 3547 } else { 3548 ZEND_UNREACHABLE(); 3549 } 3550 return 1; 3551} 3552 3553static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type) 3554{ 3555 zend_jit_addr src = ZEND_ADDR_REG(reg); 3556 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3557 3558 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3559} 3560 3561static int zend_jit_store_var_type(dasm_State **Dst, int var, uint32_t type) 3562{ 3563 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3564 3565 | SET_ZVAL_TYPE_INFO dst, type, TMP1w, TMP2 3566 return 1; 3567} 3568 3569static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) 3570{ 3571 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3572 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3573 return zend_jit_spill_store(Dst, src, dst, info, 1); 3574 } 3575 return 1; 3576} 3577 3578static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) 3579{ 3580 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3581 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3582 bool set_type = 1; 3583 3584 if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == 3585 (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { 3586 if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { 3587 set_type = 0; 3588 } 3589 } 3590 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3591 } 3592 return 1; 3593} 3594 3595static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) 3596{ 3597 zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3598 zend_jit_addr dst = ZEND_ADDR_REG(reg); 3599 3600 return zend_jit_load_reg(Dst, src, dst, info); 3601} 3602 3603static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, uint8_t op_type, zend_jit_addr addr, znode_op op) 3604{ 3605 if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) { 3606 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 3607 | SET_ZVAL_TYPE_INFO dst, IS_UNDEF, TMP1w, TMP2 3608 } 3609 return 1; 3610} 3611 3612static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3613{ 3614 if (!zend_jit_same_addr(src, dst)) { 3615 if (Z_MODE(src) == IS_REG) { 3616 if (Z_MODE(dst) == IS_REG) { 3617 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3618 | mov Rx(Z_REG(dst)), Rx(Z_REG(src)) 3619 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3620 | fmov Rd(Z_REG(dst)-ZREG_V0), Rd(Z_REG(src)-ZREG_V0) 3621 } else { 3622 ZEND_UNREACHABLE(); 3623 } 3624 if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) { 3625 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3626 3627 if (!zend_jit_spill_store(Dst, dst, var_addr, info, 3628 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3629 JIT_G(current_frame) == NULL || 3630 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3631 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3632 )) { 3633 return 0; 3634 } 3635 } 3636 } else if (Z_MODE(dst) == IS_MEM_ZVAL) { 3637 if (!Z_LOAD(src) && !Z_STORE(src)) { 3638 if (!zend_jit_spill_store(Dst, src, dst, info, 3639 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3640 JIT_G(current_frame) == NULL || 3641 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3642 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3643 )) { 3644 return 0; 3645 } 3646 } 3647 } else { 3648 ZEND_UNREACHABLE(); 3649 } 3650 } else if (Z_MODE(src) == IS_MEM_ZVAL) { 3651 if (Z_MODE(dst) == IS_REG) { 3652 if (!zend_jit_load_reg(Dst, src, dst, info)) { 3653 return 0; 3654 } 3655 } else { 3656 ZEND_UNREACHABLE(); 3657 } 3658 } else { 3659 ZEND_UNREACHABLE(); 3660 } 3661 } else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) { 3662 dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3663 if (!zend_jit_spill_store(Dst, src, dst, info, 3664 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3665 JIT_G(current_frame) == NULL || 3666 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3667 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3668 )) { 3669 return 0; 3670 } 3671 } 3672 return 1; 3673} 3674 3675static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) 3676{ 3677 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 3678 3679 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 3680 3681 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { 3682 if (!zend_jit_save_call_chain(Dst, -1)) { 3683 return 0; 3684 } 3685 } 3686 3687 ZEND_ASSERT(opline); 3688 3689 if ((opline-1)->opcode != ZEND_FETCH_CONSTANT 3690 && (opline-1)->opcode != ZEND_FETCH_LIST_R 3691 && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR)) 3692 && !(flags & ZEND_JIT_EXIT_FREE_OP1)) { 3693 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var); 3694 3695 | IF_NOT_ZVAL_REFCOUNTED val_addr, >2, ZREG_TMP1, ZREG_TMP2 3696 | GET_ZVAL_PTR TMP1, val_addr, TMP2 3697 | GC_ADDREF TMP1, TMP2w 3698 |2: 3699 } 3700 3701 | LOAD_IP_ADDR (opline - 1) 3702 | b ->trace_escape 3703 |1: 3704 3705 return 1; 3706} 3707 3708static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) 3709{ 3710 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3711 3712 if (reg == ZREG_LONG_MIN_MINUS_1) { 3713 uint64_t val = 0xc3e0000000000000; 3714 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3715 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 3716 } else if (reg == ZREG_LONG_MIN) { 3717 uint64_t val = 0x8000000000000000; 3718 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3719 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 3720 } else if (reg == ZREG_LONG_MAX) { 3721 uint64_t val = 0x7fffffffffffffff; 3722 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3723 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 3724 } else if (reg == ZREG_LONG_MAX_PLUS_1) { 3725 uint64_t val = 0x43e0000000000000; 3726 | SET_ZVAL_LVAL dst, val, TMP1, TMP2 3727 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 3728 } else if (reg == ZREG_NULL) { 3729 | SET_ZVAL_TYPE_INFO dst, IS_NULL, TMP1w, TMP2 3730 } else if (reg == ZREG_ZVAL_TRY_ADDREF) { 3731 | IF_NOT_ZVAL_REFCOUNTED dst, >1, ZREG_TMP1, ZREG_TMP2 3732 | GET_ZVAL_PTR TMP1, dst, TMP2 3733 | GC_ADDREF TMP1, TMP2w 3734 |1: 3735 } else if (reg == ZREG_ZVAL_COPY_GPR0) { 3736 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 3737 3738 | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3739 | TRY_ADDREF -1, REG1w, REG2, TMP1w 3740 } else { 3741 ZEND_UNREACHABLE(); 3742 } 3743 return 1; 3744} 3745 3746static int zend_jit_free_trampoline(dasm_State **Dst) 3747{ 3748 | // if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) 3749 | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] 3750 | TST_32_WITH_CONST TMP1w, ZEND_ACC_CALL_VIA_TRAMPOLINE, TMP2w 3751 | beq >1 3752 | mov FCARG1x, REG0 3753 | EXT_CALL zend_jit_free_trampoline_helper, REG0 3754 |1: 3755 return 1; 3756} 3757 3758static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 3759{ 3760 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { 3761 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 3762 } 3763 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 3764 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3765 } 3766 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { 3767 return 0; 3768 } 3769 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3770 | LONG_ADD_SUB_WITH_IMM adds, op1_def_addr, Z_L(1), TMP1, TMP2 3771 } else { 3772 | LONG_ADD_SUB_WITH_IMM subs, op1_def_addr, Z_L(1), TMP1, TMP2 3773 } 3774 3775 if (may_overflow && 3776 (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || 3777 ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { 3778 int32_t exit_point; 3779 const void *exit_addr; 3780 zend_jit_trace_stack *stack; 3781 uint32_t old_op1_info, old_res_info = 0; 3782 3783 stack = JIT_G(current_frame)->stack; 3784 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 3785 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); 3786 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3787 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); 3788 } else { 3789 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); 3790 } 3791 if (opline->result_type != IS_UNUSED) { 3792 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3793 if (opline->opcode == ZEND_PRE_INC) { 3794 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 3795 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); 3796 } else if (opline->opcode == ZEND_PRE_DEC) { 3797 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 3798 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); 3799 } else if (opline->opcode == ZEND_POST_INC) { 3800 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 3801 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); 3802 } else if (opline->opcode == ZEND_POST_DEC) { 3803 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 3804 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); 3805 } 3806 } 3807 3808 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 3809 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3810 if (!exit_addr) { 3811 return 0; 3812 } 3813 | bvs &exit_addr 3814 3815 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3816 opline->result_type != IS_UNUSED) { 3817 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3818 } 3819 3820 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 3821 if (opline->result_type != IS_UNUSED) { 3822 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3823 } 3824 } else if (may_overflow) { 3825 | bvs >1 3826 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3827 opline->result_type != IS_UNUSED) { 3828 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3829 } 3830 |.cold_code 3831 |1: 3832 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3833 uint64_t val = 0x43e0000000000000; 3834 if (Z_MODE(op1_def_addr) == IS_REG) { 3835 | LOAD_64BIT_VAL TMP1, val 3836 | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 3837 } else { 3838 | SET_ZVAL_LVAL op1_def_addr, val, TMP2, TMP1 3839 } 3840 } else { 3841 uint64_t val = 0xc3e0000000000000; 3842 if (Z_MODE(op1_def_addr) == IS_REG) { 3843 | LOAD_64BIT_VAL TMP1, val 3844 | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 3845 } else { 3846 | SET_ZVAL_LVAL op1_def_addr, val, TMP2, TMP1 3847 } 3848 } 3849 if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { 3850 | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE, TMP1w, TMP2 3851 } 3852 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3853 opline->result_type != IS_UNUSED) { 3854 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3855 } 3856 | b >3 3857 |.code 3858 } else { 3859 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3860 opline->result_type != IS_UNUSED) { 3861 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3862 } 3863 } 3864 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 3865 |.cold_code 3866 |2: 3867 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 3868 | SET_EX_OPLINE opline, REG0 3869 if (op1_info & MAY_BE_UNDEF) { 3870 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2, ZREG_TMP1 3871 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 3872 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 3873 | EXT_CALL zend_jit_undefined_op_helper, REG0 3874 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 3875 op1_info |= MAY_BE_NULL; 3876 } 3877 |2: 3878 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 3879 3880 | // ZVAL_DEREF(var_ptr); 3881 if (op1_info & MAY_BE_REF) { 3882 | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >2, TMP1w 3883 | GET_Z_PTR FCARG1x, FCARG1x 3884 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 3885 | cbz TMP1, >1 3886 if (RETURN_VALUE_USED(opline)) { 3887 | LOAD_ZVAL_ADDR FCARG2x, res_addr 3888 } else { 3889 | mov FCARG2x, xzr 3890 } 3891 if (opline->opcode == ZEND_PRE_INC) { 3892 | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 3893 } else if (opline->opcode == ZEND_PRE_DEC) { 3894 | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 3895 } else if (opline->opcode == ZEND_POST_INC) { 3896 | EXT_CALL zend_jit_post_inc_typed_ref, REG0 3897 } else if (opline->opcode == ZEND_POST_DEC) { 3898 | EXT_CALL zend_jit_post_dec_typed_ref, REG0 3899 } else { 3900 ZEND_UNREACHABLE(); 3901 } 3902 zend_jit_check_exception(Dst); 3903 | b >3 3904 |1: 3905 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) 3906 |2: 3907 } 3908 3909 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 3910 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 3911 3912 | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3913 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 3914 } 3915 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3916 if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { 3917 | LOAD_ZVAL_ADDR FCARG2x, res_addr 3918 | EXT_CALL zend_jit_pre_inc, REG0 3919 } else { 3920 | EXT_CALL increment_function, REG0 3921 } 3922 } else { 3923 if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { 3924 | LOAD_ZVAL_ADDR FCARG2x, res_addr 3925 | EXT_CALL zend_jit_pre_dec, REG0 3926 } else { 3927 | EXT_CALL decrement_function, REG0 3928 } 3929 } 3930 if (may_throw) { 3931 zend_jit_check_exception(Dst); 3932 } 3933 } else { 3934 zend_reg tmp_reg; 3935 3936 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 3937 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3938 } 3939 if (Z_MODE(op1_def_addr) == IS_REG) { 3940 tmp_reg = Z_REG(op1_def_addr); 3941 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 3942 tmp_reg = Z_REG(op1_addr); 3943 } else { 3944 tmp_reg = ZREG_FPR0; 3945 } 3946 | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 3947 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 3948 uint64_t val = 0x3ff0000000000000; // 1.0 3949 | LOAD_64BIT_VAL TMP1, val 3950 | fmov FPTMP, TMP1 3951 | fadd Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP 3952 } else { 3953 uint64_t val = 0x3ff0000000000000; // 1.0 3954 | LOAD_64BIT_VAL TMP1, val 3955 | fmov FPTMP, TMP1 3956 | fsub Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP 3957 } 3958 | SET_ZVAL_DVAL op1_def_addr, tmp_reg, ZREG_TMP1 3959 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 3960 opline->result_type != IS_UNUSED) { 3961 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 3962 | TRY_ADDREF op1_def_info, REG0w, REG1, TMP1w 3963 } 3964 } 3965 | b >3 3966 |.code 3967 } 3968 |3: 3969 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { 3970 return 0; 3971 } 3972 if (opline->result_type != IS_UNUSED) { 3973 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 3974 return 0; 3975 } 3976 } 3977 return 1; 3978} 3979 3980static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) 3981{ 3982 if ((opline+1)->opcode == ZEND_OP_DATA 3983 && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) 3984 && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { 3985 return 1; 3986 } 3987 return 3988 ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 3989 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || 3990 ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 3991 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || 3992 ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 3993 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); 3994} 3995 3996static int zend_jit_math_long_long(dasm_State **Dst, 3997 const zend_op *opline, 3998 uint8_t opcode, 3999 zend_jit_addr op1_addr, 4000 zend_jit_addr op2_addr, 4001 zend_jit_addr res_addr, 4002 uint32_t res_info, 4003 uint32_t res_use_info, 4004 int may_overflow) 4005{ 4006 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4007 zend_reg result_reg; 4008 zend_reg tmp_reg = ZREG_REG0; 4009 bool use_ovf_flag = 1; 4010 4011 if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { 4012 if (may_overflow && (res_info & MAY_BE_GUARD) 4013 && JIT_G(current_frame) 4014 && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { 4015 result_reg = ZREG_REG0; 4016 } else { 4017 result_reg = Z_REG(res_addr); 4018 } 4019 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { 4020 result_reg = Z_REG(op1_addr); 4021 } else if (Z_REG(res_addr) != ZREG_REG0) { 4022 result_reg = ZREG_REG0; 4023 } else { 4024 /* ASSIGN_DIM_OP */ 4025 result_reg = ZREG_FCARG1; 4026 tmp_reg = ZREG_FCARG1; 4027 } 4028 4029 if (opcode == ZEND_MUL && 4030 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4031 Z_LVAL_P(Z_ZV(op2_addr)) == 2) { 4032 if (Z_MODE(op1_addr) == IS_REG) { 4033 | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) 4034 } else { 4035 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4036 | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) 4037 } 4038 } else if (opcode == ZEND_MUL && 4039 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4040 !may_overflow && 4041 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { 4042 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4043 | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4044 | lsl Rx(result_reg), Rx(result_reg), TMP1 4045 } else if (opcode == ZEND_MUL && 4046 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4047 Z_LVAL_P(Z_ZV(op1_addr)) == 2) { 4048 if (Z_MODE(op2_addr) == IS_REG) { 4049 | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) 4050 } else { 4051 | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 4052 | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) 4053 } 4054 } else if (opcode == ZEND_MUL && 4055 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4056 !may_overflow && 4057 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { 4058 | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 4059 | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) 4060 | lsl Rx(result_reg), Rx(result_reg), TMP1 4061 } else if (opcode == ZEND_DIV && 4062 (Z_MODE(op2_addr) == IS_CONST_ZVAL && 4063 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { 4064 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4065 | asr Rx(result_reg), Rx(result_reg), #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4066#if 0 4067 /* x86 specific optimizations through LEA instraction are not supported on ARM */ 4068 } else if (opcode == ZEND_ADD && 4069 !may_overflow && 4070 Z_MODE(op1_addr) == IS_REG && 4071 Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4072 | NIY // TODO: test 4073 } else if (opcode == ZEND_ADD && 4074 !may_overflow && 4075 Z_MODE(op2_addr) == IS_REG && 4076 Z_MODE(op1_addr) == IS_CONST_ZVAL) { 4077 | NIY // TODO: test 4078 } else if (opcode == ZEND_SUB && 4079 !may_overflow && 4080 Z_MODE(op1_addr) == IS_REG && 4081 Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4082 | NIY // TODO: test 4083#endif 4084 } else if (opcode == ZEND_MUL) { 4085 | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 4086 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4087 | mul Rx(result_reg), TMP1, TMP2 4088 if(may_overflow) { 4089 /* Use 'smulh' to get the upper 64 bits fo the 128-bit result. 4090 * For signed multiplication, the top 65 bits of the result will contain 4091 * either all zeros or all ones if no overflow occurred. 4092 * Flag: bne -> overflow. beq -> no overflow. 4093 */ 4094 use_ovf_flag = 0; 4095 | smulh TMP1, TMP1, TMP2 4096 | cmp TMP1, Rx(result_reg), asr #63 4097 } 4098 } else { 4099 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4100 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4101 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4102 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4103 /* +/- 0 */ 4104 may_overflow = 0; 4105 } else if (same_ops && opcode != ZEND_DIV) { 4106 | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) 4107 } else { 4108 | LONG_MATH opcode, result_reg, op2_addr, TMP1 4109 } 4110 } 4111 if (may_overflow) { 4112 if (res_info & MAY_BE_GUARD) { 4113 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 4114 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4115 if (!exit_addr) { 4116 return 0; 4117 } 4118 if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { 4119 if (use_ovf_flag) { 4120 | bvs &exit_addr 4121 } else { 4122 | bne &exit_addr 4123 } 4124 if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { 4125 | mov Rx(Z_REG(res_addr)), Rx(result_reg) 4126 } 4127 } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 4128 if (use_ovf_flag) { 4129 | bvc &exit_addr 4130 } else { 4131 | beq &exit_addr 4132 } 4133 } else { 4134 ZEND_UNREACHABLE(); 4135 } 4136 } else { 4137 if (res_info & MAY_BE_LONG) { 4138 if (use_ovf_flag) { 4139 | bvs >1 4140 } else { 4141 | bne >1 4142 } 4143 } else { 4144 if (use_ovf_flag) { 4145 | bvc >1 4146 } else { 4147 | beq >1 4148 } 4149 } 4150 } 4151 } 4152 4153 if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { 4154 | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 4155 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4156 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4157 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 4158 } 4159 } 4160 } 4161 4162 if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { 4163 zend_reg tmp_reg1 = ZREG_FPR0; 4164 zend_reg tmp_reg2 = ZREG_FPR1; 4165 4166 if (res_info & MAY_BE_LONG) { 4167 |.cold_code 4168 |1: 4169 } 4170 4171 do { 4172 if ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || 4173 (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)) { 4174 if (opcode == ZEND_ADD) { 4175 uint64_t val = 0x43e0000000000000; 4176 if (Z_MODE(res_addr) == IS_REG) { 4177 | LOAD_64BIT_VAL TMP1, val 4178 | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 4179 } else { 4180 | SET_ZVAL_LVAL res_addr, val, TMP2, TMP1 4181 } 4182 break; 4183 } else if (opcode == ZEND_SUB) { 4184 uint64_t val = 0xc3e0000000000000; 4185 if (Z_MODE(res_addr) == IS_REG) { 4186 | LOAD_64BIT_VAL TMP1, val 4187 | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 4188 } else { 4189 | SET_ZVAL_LVAL res_addr, val, TMP2, TMP1 4190 } 4191 break; 4192 } 4193 } 4194 4195 | DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg, ZREG_TMP1 4196 | DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg, ZREG_TMP1 4197 | DOUBLE_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 4198 | SET_ZVAL_DVAL res_addr, tmp_reg1, ZREG_TMP1 4199 } while (0); 4200 4201 if (Z_MODE(res_addr) == IS_MEM_ZVAL 4202 && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4203 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4204 } 4205 if (res_info & MAY_BE_LONG) { 4206 | b >2 4207 |.code 4208 } 4209 |2: 4210 } 4211 4212 return 1; 4213} 4214 4215static int zend_jit_math_long_double(dasm_State **Dst, 4216 uint8_t opcode, 4217 zend_jit_addr op1_addr, 4218 zend_jit_addr op2_addr, 4219 zend_jit_addr res_addr, 4220 uint32_t res_use_info) 4221{ 4222 zend_reg result_reg = 4223 (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_FPR0; 4224 zend_reg op2_reg; 4225 4226 | DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, ZREG_TMP1, ZREG_TMP2 4227 4228 if (Z_MODE(op2_addr) == IS_REG) { 4229 op2_reg = Z_REG(op2_addr); 4230 } else { 4231 op2_reg = ZREG_FPTMP; 4232 | GET_ZVAL_DVAL op2_reg, op2_addr, ZREG_TMP1 4233 } 4234 4235 | DOUBLE_MATH_REG opcode, result_reg, result_reg, op2_reg 4236 4237 | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 4238 4239 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4240 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4241 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4242 } 4243 } 4244 4245 return 1; 4246} 4247 4248static int zend_jit_math_double_long(dasm_State **Dst, 4249 uint8_t opcode, 4250 zend_jit_addr op1_addr, 4251 zend_jit_addr op2_addr, 4252 zend_jit_addr res_addr, 4253 uint32_t res_use_info) 4254{ 4255 zend_reg result_reg, op1_reg, op2_reg; 4256 4257 if (zend_is_commutative(opcode) 4258 && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { 4259 if (Z_MODE(res_addr) == IS_REG) { 4260 result_reg = Z_REG(res_addr); 4261 } else { 4262 result_reg = ZREG_FPR0; 4263 } 4264 | DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 4265 if (Z_MODE(op1_addr) == IS_REG) { 4266 op1_reg = Z_REG(op1_addr); 4267 } else { 4268 op1_reg = ZREG_FPTMP; 4269 | GET_ZVAL_DVAL op1_reg, op1_addr, ZREG_TMP1 4270 } 4271 | DOUBLE_MATH_REG opcode, result_reg, result_reg, op1_reg 4272 } else { 4273 if (Z_MODE(res_addr) == IS_REG) { 4274 result_reg = Z_REG(res_addr); 4275 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4276 result_reg = Z_REG(op1_addr); 4277 } else { 4278 result_reg = ZREG_FPR0; 4279 } 4280 4281 if (Z_MODE(op1_addr) == IS_REG) { 4282 op1_reg = Z_REG(op1_addr); 4283 } else { 4284 | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 4285 op1_reg = result_reg; 4286 } 4287 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4288 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4289 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4290 /* +/- 0 */ 4291 } else { 4292 op2_reg = ZREG_FPTMP; 4293 | DOUBLE_GET_ZVAL_LVAL op2_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 4294 | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg 4295 } 4296 } 4297 4298 | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 4299 4300 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4301 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4302 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4303 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4304 } 4305 } 4306 } 4307 4308 return 1; 4309} 4310 4311static int zend_jit_math_double_double(dasm_State **Dst, 4312 uint8_t opcode, 4313 zend_jit_addr op1_addr, 4314 zend_jit_addr op2_addr, 4315 zend_jit_addr res_addr, 4316 uint32_t res_use_info) 4317{ 4318 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4319 zend_reg result_reg, op1_reg, op2_reg; 4320 zend_jit_addr val_addr; 4321 4322 if (Z_MODE(res_addr) == IS_REG) { 4323 result_reg = Z_REG(res_addr); 4324 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4325 result_reg = Z_REG(op1_addr); 4326 } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { 4327 result_reg = Z_REG(op2_addr); 4328 } else { 4329 result_reg = ZREG_FPR0; 4330 } 4331 4332 if (Z_MODE(op1_addr) == IS_REG) { 4333 op1_reg = Z_REG(op1_addr); 4334 val_addr = op2_addr; 4335 } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4336 op1_reg = Z_REG(op2_addr); 4337 val_addr = op1_addr; 4338 } else { 4339 | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 4340 op1_reg = result_reg; 4341 val_addr = op2_addr; 4342 } 4343 4344 if ((opcode == ZEND_MUL) && 4345 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4346 | DOUBLE_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg 4347 } else { 4348 if (same_ops) { 4349 op2_reg = op1_reg; 4350 } else if (Z_MODE(val_addr) == IS_REG) { 4351 op2_reg = Z_REG(val_addr); 4352 } else { 4353 op2_reg = ZREG_FPTMP; 4354 | GET_ZVAL_DVAL op2_reg, val_addr, ZREG_TMP1 4355 } 4356 | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg 4357 } 4358 4359 | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 4360 4361 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4362 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4363 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4364 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 4365 } 4366 } 4367 } 4368 return 1; 4369} 4370 4371static int zend_jit_math_helper(dasm_State **Dst, 4372 const zend_op *opline, 4373 uint8_t opcode, 4374 uint8_t op1_type, 4375 znode_op op1, 4376 zend_jit_addr op1_addr, 4377 uint32_t op1_info, 4378 uint8_t op2_type, 4379 znode_op op2, 4380 zend_jit_addr op2_addr, 4381 uint32_t op2_info, 4382 uint32_t res_var, 4383 zend_jit_addr res_addr, 4384 uint32_t res_info, 4385 uint32_t res_use_info, 4386 int may_overflow, 4387 int may_throw) 4388/* Labels: 1,2,3,4,5,6 */ 4389{ 4390 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4391 4392 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4393 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { 4394 if (op1_info & MAY_BE_DOUBLE) { 4395 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 4396 } else { 4397 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 4398 } 4399 } 4400 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { 4401 if (op2_info & MAY_BE_DOUBLE) { 4402 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1, ZREG_TMP1 4403 |.cold_code 4404 |1: 4405 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4406 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4407 } 4408 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4409 return 0; 4410 } 4411 | b >5 4412 |.code 4413 } else { 4414 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4415 } 4416 } 4417 if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { 4418 return 0; 4419 } 4420 if (op1_info & MAY_BE_DOUBLE) { 4421 |.cold_code 4422 |3: 4423 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4424 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 4425 } 4426 if (op2_info & MAY_BE_DOUBLE) { 4427 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4428 if (!same_ops) { 4429 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1, ZREG_TMP1 4430 } else { 4431 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4432 } 4433 } 4434 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4435 return 0; 4436 } 4437 | b >5 4438 } 4439 if (!same_ops) { 4440 |1: 4441 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4442 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4443 } 4444 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4445 return 0; 4446 } 4447 | b >5 4448 } 4449 |.code 4450 } 4451 } else if ((op1_info & MAY_BE_DOUBLE) && 4452 !(op1_info & MAY_BE_LONG) && 4453 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4454 (res_info & MAY_BE_DOUBLE)) { 4455 if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4456 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 4457 } 4458 if (op2_info & MAY_BE_DOUBLE) { 4459 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4460 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4461 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1, ZREG_TMP1 4462 } else { 4463 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4464 } 4465 } 4466 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4467 return 0; 4468 } 4469 } 4470 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4471 if (op2_info & MAY_BE_DOUBLE) { 4472 |.cold_code 4473 } 4474 |1: 4475 if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4476 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4477 } 4478 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4479 return 0; 4480 } 4481 if (op2_info & MAY_BE_DOUBLE) { 4482 | b >5 4483 |.code 4484 } 4485 } 4486 } else if ((op2_info & MAY_BE_DOUBLE) && 4487 !(op2_info & MAY_BE_LONG) && 4488 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4489 (res_info & MAY_BE_DOUBLE)) { 4490 if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4491 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 4492 } 4493 if (op1_info & MAY_BE_DOUBLE) { 4494 if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4495 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4496 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1, ZREG_TMP1 4497 } else { 4498 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 4499 } 4500 } 4501 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4502 return 0; 4503 } 4504 } 4505 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4506 if (op1_info & MAY_BE_DOUBLE) { 4507 |.cold_code 4508 } 4509 |1: 4510 if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4511 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 4512 } 4513 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4514 return 0; 4515 } 4516 if (op1_info & MAY_BE_DOUBLE) { 4517 | b >5 4518 |.code 4519 } 4520 } 4521 } 4522 4523 |5: 4524 4525 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 4526 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 4527 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4528 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4529 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4530 |.cold_code 4531 } 4532 |6: 4533 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4534 if (Z_MODE(res_addr) == IS_REG) { 4535 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4536 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4537 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4538 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4539 } 4540 if (Z_MODE(op1_addr) == IS_REG) { 4541 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4542 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4543 return 0; 4544 } 4545 op1_addr = real_addr; 4546 } 4547 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4548 } else { 4549 if (Z_MODE(op1_addr) == IS_REG) { 4550 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4551 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4552 return 0; 4553 } 4554 op1_addr = real_addr; 4555 } 4556 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4557 if (Z_MODE(res_addr) == IS_REG) { 4558 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4559 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4560 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4561 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4562 } 4563 } 4564 if (Z_MODE(op2_addr) == IS_REG) { 4565 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 4566 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 4567 return 0; 4568 } 4569 op2_addr = real_addr; 4570 } 4571 | LOAD_ZVAL_ADDR CARG3, op2_addr 4572 | SET_EX_OPLINE opline, REG0 4573 if (opcode == ZEND_ADD) { 4574 | EXT_CALL add_function, REG0 4575 } else if (opcode == ZEND_SUB) { 4576 | EXT_CALL sub_function, REG0 4577 } else if (opcode == ZEND_MUL) { 4578 | EXT_CALL mul_function, REG0 4579 } else if (opcode == ZEND_DIV) { 4580 | EXT_CALL div_function, REG0 4581 } else { 4582 ZEND_UNREACHABLE(); 4583 } 4584 | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4585 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4586 if (may_throw) { 4587 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 4588 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 4589 | cbnz TMP2, ->exception_handler_free_op2 4590 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 4591 zend_jit_check_exception_undef_result(Dst, opline); 4592 } else { 4593 zend_jit_check_exception(Dst); 4594 } 4595 } 4596 if (Z_MODE(res_addr) == IS_REG) { 4597 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4598 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 4599 return 0; 4600 } 4601 } 4602 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4603 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4604 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4605 | b <5 4606 |.code 4607 } 4608 } 4609 4610 return 1; 4611} 4612 4613static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 4614{ 4615 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 4616 ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4617 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); 4618 4619 if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { 4620 return 0; 4621 } 4622 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4623 return 0; 4624 } 4625 return 1; 4626} 4627 4628static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr) 4629{ 4630 if (Z_MODE(op2_addr) != IS_MEM_ZVAL || Z_REG(op2_addr) != ZREG_FCARG1) { 4631 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 4632 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 4633 } else if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4634 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 4635 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 4636 } else { 4637 | GET_ZVAL_LVAL ZREG_REG0, op2_addr, TMP1 4638 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 4639 | mov FCARG2x, REG0 4640 } 4641 | EXT_CALL zend_jit_add_arrays_helper, REG0 4642 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 4643 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX, TMP1w, TMP2 4644 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 4645 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 4646 return 1; 4647} 4648 4649static int zend_jit_long_math_helper(dasm_State **Dst, 4650 const zend_op *opline, 4651 uint8_t opcode, 4652 uint8_t op1_type, 4653 znode_op op1, 4654 zend_jit_addr op1_addr, 4655 uint32_t op1_info, 4656 zend_ssa_range *op1_range, 4657 uint8_t op2_type, 4658 znode_op op2, 4659 zend_jit_addr op2_addr, 4660 uint32_t op2_info, 4661 zend_ssa_range *op2_range, 4662 uint32_t res_var, 4663 zend_jit_addr res_addr, 4664 uint32_t res_info, 4665 uint32_t res_use_info, 4666 int may_throw) 4667/* Labels: 6 */ 4668{ 4669 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4670 zend_reg result_reg; 4671 4672 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 4673 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 4674 } 4675 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 4676 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 4677 } 4678 4679 if (Z_MODE(res_addr) == IS_REG) { 4680 if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) 4681 && opline->op2_type != IS_CONST) { 4682 result_reg = ZREG_REG0; 4683 } else { 4684 result_reg = Z_REG(res_addr); 4685 } 4686 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4687 result_reg = Z_REG(op1_addr); 4688 } else if (Z_REG(res_addr) != ZREG_REG0) { 4689 result_reg = ZREG_REG0; 4690 } else { 4691 /* ASSIGN_DIM_OP */ 4692 result_reg = ZREG_FCARG1; 4693 } 4694 4695 if (opcode == ZEND_SL) { 4696 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4697 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 4698 4699 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 4700 if (EXPECTED(op2_lval > 0)) { 4701 | mov Rx(result_reg), xzr 4702 } else { 4703 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4704 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4705 | SET_EX_OPLINE opline, REG0 4706 | b ->negative_shift 4707 } 4708 } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { 4709 | add Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) 4710 } else { 4711 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4712 | lsl Rx(result_reg), Rx(result_reg), #op2_lval 4713 } 4714 } else { 4715 zend_reg op2_reg; 4716 4717 if (Z_MODE(op2_addr) == IS_REG) { 4718 op2_reg = Z_REG(op2_addr); 4719 } else { 4720 op2_reg = ZREG_TMP2; 4721 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4722 } 4723 if (!op2_range || 4724 op2_range->min < 0 || 4725 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 4726 4727 | cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8) 4728 | bhs >1 4729 |.cold_code 4730 |1: 4731 | mov Rx(result_reg), xzr 4732 | cmp Rx(op2_reg), xzr 4733 | bgt >1 4734 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4735 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4736 | SET_EX_OPLINE opline, REG0 4737 | b ->negative_shift 4738 |.code 4739 } 4740 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4741 | lsl Rx(result_reg), Rx(result_reg), Rx(op2_reg) 4742 |1: 4743 } 4744 } else if (opcode == ZEND_SR) { 4745 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4746 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4747 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 4748 4749 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 4750 if (EXPECTED(op2_lval > 0)) { 4751 | asr Rx(result_reg), Rx(result_reg), #((SIZEOF_ZEND_LONG * 8) - 1) 4752 } else { 4753 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4754 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4755 | SET_EX_OPLINE opline, REG0 4756 | b ->negative_shift 4757 } 4758 } else { 4759 | asr Rx(result_reg), Rx(result_reg), #op2_lval 4760 } 4761 } else { 4762 zend_reg op2_reg; 4763 4764 if (Z_MODE(op2_addr) == IS_REG) { 4765 op2_reg = Z_REG(op2_addr); 4766 } else { 4767 op2_reg = ZREG_TMP2; 4768 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4769 } 4770 if (!op2_range || 4771 op2_range->min < 0 || 4772 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 4773 | cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8) 4774 | bhs >1 4775 |.cold_code 4776 |1: 4777 | cmp Rx(op2_reg), xzr 4778 | mov Rx(op2_reg), #((SIZEOF_ZEND_LONG * 8) - 1) 4779 | bgt >1 4780 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4781 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4782 | SET_EX_OPLINE opline, REG0 4783 | b ->negative_shift 4784 |.code 4785 } 4786 |1: 4787 | asr Rx(result_reg), Rx(result_reg), Rx(op2_reg) 4788 } 4789 } else if (opcode == ZEND_MOD) { 4790 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 4791 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 4792 4793 if (op2_lval == 0) { 4794 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4795 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4796 | SET_EX_OPLINE opline, REG0 4797 | b ->mod_by_zero 4798 } else if (op2_lval == -1) { 4799 | mov Rx(result_reg), xzr 4800 } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) { 4801 zval tmp; 4802 zend_jit_addr tmp_addr; 4803 4804 /* Optimisation for mod of power of 2 */ 4805 ZVAL_LONG(&tmp, op2_lval - 1); 4806 tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp); 4807 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4808 | LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, TMP1 4809 } else { 4810 | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 4811 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 4812 | sdiv Rx(result_reg), TMP1, TMP2 4813 | msub Rx(result_reg), Rx(result_reg), TMP2, TMP1 4814 } 4815 } else { 4816 zend_reg op2_reg; 4817 4818 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 4819 | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP2, Rx(Z_REG(op2_addr)), Z_OFFSET(op2_addr), TMP2 4820 op2_reg = ZREG_TMP2; 4821 } else { 4822 ZEND_ASSERT(Z_MODE(op2_addr) == IS_REG); 4823 op2_reg = Z_REG(op2_addr); 4824 } 4825 4826 if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { 4827 | cbz Rx(op2_reg), >1 4828 |.cold_code 4829 |1: 4830 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 4831 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 4832 | SET_EX_OPLINE opline, REG0 4833 | b ->mod_by_zero 4834 |.code 4835 } 4836 4837 /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ 4838 if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { 4839 | cmn Rx(op2_reg), #1 4840 | beq >1 4841 |.cold_code 4842 |1: 4843 | SET_ZVAL_LVAL_FROM_REG res_addr, xzr, TMP1 4844 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4845 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4846 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4847 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 4848 } 4849 } 4850 } 4851 | b >5 4852 |.code 4853 } 4854 4855 | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 4856 | sdiv Rx(result_reg), TMP1, Rx(op2_reg) 4857 | msub Rx(result_reg), Rx(result_reg), Rx(op2_reg), TMP1 4858 } 4859 } else if (same_ops) { 4860 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4861 | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) 4862 } else { 4863 | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 4864 | LONG_MATH opcode, result_reg, op2_addr, TMP1 4865 } 4866 4867 if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { 4868 | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 4869 } 4870 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4871 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4872 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4873 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 4874 } 4875 } 4876 } 4877 4878 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || 4879 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 4880 if ((op1_info & MAY_BE_LONG) && 4881 (op2_info & MAY_BE_LONG)) { 4882 |.cold_code 4883 } 4884 |6: 4885 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4886 if (Z_MODE(res_addr) == IS_REG) { 4887 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4888 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4889 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4890 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4891 } 4892 if (Z_MODE(op1_addr) == IS_REG) { 4893 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4894 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4895 return 0; 4896 } 4897 op1_addr = real_addr; 4898 } 4899 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4900 } else { 4901 if (Z_MODE(op1_addr) == IS_REG) { 4902 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4903 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4904 return 0; 4905 } 4906 op1_addr = real_addr; 4907 } 4908 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 4909 if (Z_MODE(res_addr) == IS_REG) { 4910 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4911 | LOAD_ZVAL_ADDR FCARG1x, real_addr 4912 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4913 | LOAD_ZVAL_ADDR FCARG1x, res_addr 4914 } 4915 } 4916 if (Z_MODE(op2_addr) == IS_REG) { 4917 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 4918 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 4919 return 0; 4920 } 4921 op2_addr = real_addr; 4922 } 4923 | LOAD_ZVAL_ADDR CARG3, op2_addr 4924 | SET_EX_OPLINE opline, REG0 4925 if (opcode == ZEND_BW_OR) { 4926 | EXT_CALL bitwise_or_function, REG0 4927 } else if (opcode == ZEND_BW_AND) { 4928 | EXT_CALL bitwise_and_function, REG0 4929 } else if (opcode == ZEND_BW_XOR) { 4930 | EXT_CALL bitwise_xor_function, REG0 4931 } else if (opcode == ZEND_SL) { 4932 | EXT_CALL shift_left_function, REG0 4933 } else if (opcode == ZEND_SR) { 4934 | EXT_CALL shift_right_function, REG0 4935 } else if (opcode == ZEND_MOD) { 4936 | EXT_CALL mod_function, REG0 4937 } else { 4938 ZEND_UNREACHABLE(); 4939 } 4940 if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) { 4941 /* compound assignment may decrement "op2" refcount */ 4942 op2_info |= MAY_BE_RC1; 4943 } 4944 | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4945 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 4946 if (may_throw) { 4947 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 4948 | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 4949 | cbnz TMP2, ->exception_handler_free_op2 4950 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 4951 zend_jit_check_exception_undef_result(Dst, opline); 4952 } else { 4953 zend_jit_check_exception(Dst); 4954 } 4955 } 4956 if (Z_MODE(res_addr) == IS_REG) { 4957 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4958 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 4959 return 0; 4960 } 4961 } 4962 if ((op1_info & MAY_BE_LONG) && 4963 (op2_info & MAY_BE_LONG)) { 4964 | b >5 4965 |.code 4966 } 4967 } 4968 |5: 4969 4970 return 1; 4971} 4972 4973static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 4974{ 4975 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 4976 ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); 4977 4978 if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, 4979 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 4980 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 4981 opline->result.var, res_addr, res_info, res_use_info, may_throw)) { 4982 return 0; 4983 } 4984 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4985 return 0; 4986 } 4987 return 1; 4988} 4989 4990static int zend_jit_concat_helper(dasm_State **Dst, 4991 const zend_op *opline, 4992 uint8_t op1_type, 4993 znode_op op1, 4994 zend_jit_addr op1_addr, 4995 uint32_t op1_info, 4996 uint8_t op2_type, 4997 znode_op op2, 4998 zend_jit_addr op2_addr, 4999 uint32_t op2_info, 5000 zend_jit_addr res_addr, 5001 int may_throw) 5002{ 5003 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5004 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5005 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 5006 } 5007 if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5008 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6, ZREG_TMP1 5009 } 5010 if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { 5011 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5012 | LOAD_ZVAL_ADDR FCARG1x, res_addr 5013 } 5014 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 5015 | EXT_CALL zend_jit_fast_assign_concat_helper, REG0 5016 /* concatenation with itself may reduce refcount */ 5017 op2_info |= MAY_BE_RC1; 5018 } else { 5019 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5020 | LOAD_ZVAL_ADDR FCARG1x, res_addr 5021 } 5022 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 5023 | LOAD_ZVAL_ADDR CARG3, op2_addr 5024 if (op1_type == IS_CV || op1_type == IS_CONST) { 5025 | EXT_CALL zend_jit_fast_concat_helper, REG0 5026 } else { 5027 | EXT_CALL zend_jit_fast_concat_tmp_helper, REG0 5028 } 5029 } 5030 /* concatenation with empty string may increase refcount */ 5031 op2_info |= MAY_BE_RCN; 5032 | FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 5033 |5: 5034 } 5035 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || 5036 (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { 5037 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5038 |.cold_code 5039 |6: 5040 } 5041 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 5042 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5043 | LOAD_ZVAL_ADDR FCARG1x, res_addr 5044 } 5045 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 5046 } else { 5047 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 5048 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5049 | LOAD_ZVAL_ADDR FCARG1x, res_addr 5050 } 5051 } 5052 | LOAD_ZVAL_ADDR CARG3, op2_addr 5053 | SET_EX_OPLINE opline, REG0 5054 | EXT_CALL concat_function, REG0 5055 /* concatenation with empty string may increase refcount */ 5056 op1_info |= MAY_BE_RCN; 5057 op2_info |= MAY_BE_RCN; 5058 | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 5059 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 5060 if (may_throw) { 5061 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5062 zend_jit_check_exception_undef_result(Dst, opline); 5063 } else { 5064 zend_jit_check_exception(Dst); 5065 } 5066 } 5067 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5068 | b <5 5069 |.code 5070 } 5071 } 5072 5073 return 1; 5074} 5075 5076static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) 5077{ 5078 zend_jit_addr op1_addr, op2_addr; 5079 5080 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5081 ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); 5082 5083 op1_addr = OP1_ADDR(); 5084 op2_addr = OP2_ADDR(); 5085 5086 return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); 5087} 5088 5089static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) 5090/* Labels: 1,2,3,4,5 */ 5091{ 5092 zend_jit_addr op2_addr = OP2_ADDR(); 5093 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 5094 5095 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 5096 && type == BP_VAR_R 5097 && !exit_addr) { 5098 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5099 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5100 if (!exit_addr) { 5101 return 0; 5102 } 5103 } 5104 5105 if (op2_info & MAY_BE_LONG) { 5106 bool op2_loaded = 0; 5107 bool packed_loaded = 0; 5108 bool bad_packed_key = 0; 5109 5110 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { 5111 | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) 5112 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 5113 } 5114 if (op1_info & MAY_BE_PACKED_GUARD) { 5115 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 5116 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5117 5118 if (!exit_addr) { 5119 return 0; 5120 } 5121 if (op1_info & MAY_BE_ARRAY_PACKED) { 5122 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 5123 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 5124 | beq &exit_addr 5125 } else { 5126 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 5127 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 5128 | bne &exit_addr 5129 } 5130 } 5131 if (type == BP_VAR_W) { 5132 | // hval = Z_LVAL_P(dim); 5133 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5134 op2_loaded = 1; 5135 } 5136 if (op1_info & MAY_BE_ARRAY_PACKED) { 5137 zend_long val = -1; 5138 5139 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5140 val = Z_LVAL_P(Z_ZV(op2_addr)); 5141 if (val >= 0 && val < HT_MAX_SIZE) { 5142 packed_loaded = 1; 5143 } else { 5144 bad_packed_key = 1; 5145 } 5146 } else { 5147 if (!op2_loaded) { 5148 | // hval = Z_LVAL_P(dim); 5149 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5150 op2_loaded = 1; 5151 } 5152 packed_loaded = 1; 5153 } 5154 5155 if (dim_type == IS_UNDEF && type == BP_VAR_W) { 5156 /* don't generate "fast" code for packed array */ 5157 packed_loaded = 0; 5158 } 5159 5160 if (packed_loaded) { 5161 | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); 5162 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5163 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 5164 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 5165 | beq >4 // HASH_FIND 5166 } 5167 | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) 5168 5169 | ldr REG0w, [FCARG1x, #offsetof(zend_array, nNumUsed)] 5170 if (val == 0) { 5171 | cmp REG0, xzr 5172 } else if (val > 0 && !op2_loaded) { 5173 | CMP_64_WITH_CONST REG0, val, TMP1 5174 } else { 5175 | cmp REG0, FCARG2x 5176 } 5177 5178 if (type == BP_JIT_IS) { 5179 if (not_found_exit_addr) { 5180 | bls ¬_found_exit_addr 5181 } else { 5182 | bls >9 // NOT_FOUND 5183 } 5184 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5185 | bls &exit_addr 5186 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5187 | bls ¬_found_exit_addr 5188 } else if (type == BP_VAR_RW && not_found_exit_addr) { 5189 | bls ¬_found_exit_addr 5190 } else if (type == BP_VAR_IS && found_exit_addr) { 5191 | bls >7 // NOT_FOUND 5192 } else { 5193 | bls >2 // NOT_FOUND 5194 } 5195 | // _ret = &_ht->arPacked[_h].val; 5196 if (val >= 0) { 5197 | ldr REG0, [FCARG1x, #offsetof(zend_array, arPacked)] 5198 if (val != 0) { 5199 | ADD_SUB_64_WITH_CONST add, REG0, REG0, (val * sizeof(zval)), TMP1 5200 } 5201 } else { 5202 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arPacked)] 5203 | add REG0, TMP1, FCARG2x, lsl #4 5204 } 5205 } 5206 } 5207 switch (type) { 5208 case BP_JIT_IS: 5209 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5210 if (packed_loaded) { 5211 | b >5 5212 } 5213 |4: 5214 if (!op2_loaded) { 5215 | // hval = Z_LVAL_P(dim); 5216 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5217 } 5218 if (packed_loaded) { 5219 | EXT_CALL _zend_hash_index_find, REG0 5220 } else { 5221 | EXT_CALL zend_hash_index_find, REG0 5222 } 5223 | mov REG0, RETVALx 5224 if (not_found_exit_addr) { 5225 | cbz REG0, ¬_found_exit_addr 5226 } else { 5227 | cbz REG0, >9 // NOT_FOUND 5228 } 5229 if (op2_info & MAY_BE_STRING) { 5230 | b >5 5231 } 5232 } else if (packed_loaded) { 5233 if (op2_info & MAY_BE_STRING) { 5234 | b >5 5235 } 5236 } else if (not_found_exit_addr) { 5237 | b ¬_found_exit_addr 5238 } else { 5239 | b >9 // NOT_FOUND 5240 } 5241 break; 5242 case BP_VAR_R: 5243 case BP_VAR_IS: 5244 case BP_VAR_UNSET: 5245 if (packed_loaded) { 5246 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5247 | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w 5248 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5249 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 5250 if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5251 | IF_Z_TYPE REG0, IS_UNDEF, &exit_addr, TMP1w 5252 } 5253 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5254 | IF_Z_TYPE REG0, IS_UNDEF, ¬_found_exit_addr, TMP1w 5255 } else if (type == BP_VAR_IS && found_exit_addr) { 5256 | IF_Z_TYPE REG0, IS_UNDEF, >7, TMP1w // NOT_FOUND 5257 } else { 5258 | IF_Z_TYPE REG0, IS_UNDEF, >2, TMP1w // NOT_FOUND 5259 } 5260 } 5261 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_NUMERIC_HASH))) { 5262 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5263 | b &exit_addr 5264 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5265 | b ¬_found_exit_addr 5266 } else if (type == BP_VAR_IS && found_exit_addr) { 5267 | b >7 // NOT_FOUND 5268 } else { 5269 | b >2 // NOT_FOUND 5270 } 5271 } 5272 if (!packed_loaded || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5273 |4: 5274 if (!op2_loaded) { 5275 | // hval = Z_LVAL_P(dim); 5276 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5277 } 5278 if (packed_loaded) { 5279 | EXT_CALL _zend_hash_index_find, REG0 5280 } else { 5281 | EXT_CALL zend_hash_index_find, REG0 5282 } 5283 | mov REG0, RETVALx 5284 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5285 | cbz REG0, &exit_addr 5286 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5287 | cbz REG0, ¬_found_exit_addr 5288 } else if (type == BP_VAR_IS && found_exit_addr) { 5289 | cbz REG0, >7 // NOT_FOUND 5290 } else { 5291 | cbz REG0, >2 // NOT_FOUND 5292 } 5293 } 5294 |.cold_code 5295 |2: 5296 switch (type) { 5297 case BP_VAR_R: 5298 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5299 | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); 5300 | // retval = &EG(uninitialized_zval); 5301 | UNDEFINED_OFFSET opline 5302 | b >9 5303 } 5304 break; 5305 case BP_VAR_IS: 5306 case BP_VAR_UNSET: 5307 if (!not_found_exit_addr && !found_exit_addr) { 5308 | // retval = &EG(uninitialized_zval); 5309 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 5310 | b >9 5311 } 5312 break; 5313 default: 5314 ZEND_UNREACHABLE(); 5315 } 5316 |.code 5317 break; 5318 case BP_VAR_RW: 5319 if (packed_loaded && !not_found_exit_addr) { 5320 | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w 5321 } 5322 if (!packed_loaded || 5323 !not_found_exit_addr || 5324 (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5325 if (packed_loaded && not_found_exit_addr) { 5326 |.cold_code 5327 } 5328 |2: 5329 |4: 5330 if (!op2_loaded) { 5331 | // hval = Z_LVAL_P(dim); 5332 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5333 } 5334 if (packed_loaded) { 5335 | EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, REG0 5336 } else { 5337 | EXT_CALL zend_jit_hash_index_lookup_rw, REG0 5338 } 5339 | mov REG0, RETVALx 5340 if (not_found_exit_addr) { 5341 if (packed_loaded) { 5342 | cbnz REG0, >8 5343 | b ¬_found_exit_addr 5344 |.code 5345 } else { 5346 | cbz REG0, ¬_found_exit_addr 5347 } 5348 } else { 5349 | cbz REG0, >9 5350 } 5351 } 5352 break; 5353 case BP_VAR_W: 5354 if (packed_loaded) { 5355 | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w 5356 } 5357 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) { 5358 |2: 5359 |4: 5360 if (!op2_loaded) { 5361 | // hval = Z_LVAL_P(dim); 5362 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5363 } 5364 | EXT_CALL zend_hash_index_lookup, REG0 5365 | mov REG0, RETVALx 5366 } 5367 break; 5368 default: 5369 ZEND_UNREACHABLE(); 5370 } 5371 5372 if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { 5373 | b >8 5374 } 5375 } 5376 5377 if (op2_info & MAY_BE_STRING) { 5378 |3: 5379 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5380 | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) 5381 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3, ZREG_TMP1 5382 } 5383 | // offset_key = Z_STR_P(dim); 5384 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 5385 | // retval = zend_hash_find(ht, offset_key); 5386 switch (type) { 5387 case BP_JIT_IS: 5388 if (opline->op2_type != IS_CONST) { 5389 | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] 5390 | cmp TMP1w, #((uint8_t) ('9')) 5391 | ble >1 5392 |.cold_code 5393 |1: 5394 | EXT_CALL zend_jit_symtable_find, REG0 5395 | b >1 5396 |.code 5397 | EXT_CALL zend_hash_find, REG0 5398 |1: 5399 } else { 5400 | EXT_CALL zend_hash_find_known_hash, REG0 5401 } 5402 | mov REG0, RETVALx 5403 if (not_found_exit_addr) { 5404 | cbz REG0, ¬_found_exit_addr 5405 } else { 5406 | cbz REG0, >9 // NOT_FOUND 5407 } 5408 break; 5409 case BP_VAR_R: 5410 case BP_VAR_IS: 5411 case BP_VAR_UNSET: 5412 if (opline->op2_type != IS_CONST) { 5413 | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] 5414 | cmp TMP1w, #((uint8_t) ('9')) 5415 | ble >1 5416 |.cold_code 5417 |1: 5418 | EXT_CALL zend_jit_symtable_find, REG0 5419 | b >1 5420 |.code 5421 | EXT_CALL zend_hash_find, REG0 5422 |1: 5423 } else { 5424 | EXT_CALL zend_hash_find_known_hash, REG0 5425 } 5426 | mov REG0, RETVALx 5427 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5428 | cbz REG0, &exit_addr 5429 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5430 | cbz REG0, ¬_found_exit_addr 5431 } else if (type == BP_VAR_IS && found_exit_addr) { 5432 | cbz REG0, >7 5433 } else { 5434 | cbz REG0, >2 // NOT_FOUND 5435 |.cold_code 5436 |2: 5437 switch (type) { 5438 case BP_VAR_R: 5439 // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); 5440 | UNDEFINED_INDEX opline 5441 | b >9 5442 break; 5443 case BP_VAR_IS: 5444 case BP_VAR_UNSET: 5445 | // retval = &EG(uninitialized_zval); 5446 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 5447 | b >9 5448 break; 5449 default: 5450 ZEND_UNREACHABLE(); 5451 } 5452 |.code 5453 } 5454 break; 5455 case BP_VAR_RW: 5456 if (opline->op2_type != IS_CONST) { 5457 | EXT_CALL zend_jit_symtable_lookup_rw, REG0 5458 } else { 5459 | EXT_CALL zend_jit_hash_lookup_rw, REG0 5460 } 5461 | mov REG0, RETVALx 5462 if (not_found_exit_addr) { 5463 | cbz REG0, ¬_found_exit_addr 5464 } else { 5465 | cbz REG0, >9 5466 } 5467 break; 5468 case BP_VAR_W: 5469 if (opline->op2_type != IS_CONST) { 5470 | EXT_CALL zend_jit_symtable_lookup_w, REG0 5471 } else { 5472 | EXT_CALL zend_hash_lookup, REG0 5473 } 5474 | mov REG0, RETVALx 5475 break; 5476 default: 5477 ZEND_UNREACHABLE(); 5478 } 5479 } 5480 5481 if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { 5482 |5: 5483 if (op1_info & MAY_BE_ARRAY_OF_REF) { 5484 | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w 5485 } 5486 | ldrb TMP1w, [REG0,#offsetof(zval, u1.v.type)] 5487 | cmp TMP1w, #IS_NULL 5488 if (not_found_exit_addr) { 5489 | ble ¬_found_exit_addr 5490 } else if (found_exit_addr) { 5491 | bgt &found_exit_addr 5492 } else { 5493 | ble >9 // NOT FOUND 5494 } 5495 } 5496 5497 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5498 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5499 |.cold_code 5500 |3: 5501 } 5502 if (type != BP_VAR_RW) { 5503 | SET_EX_OPLINE opline, REG0 5504 } 5505 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 5506 switch (type) { 5507 case BP_VAR_R: 5508 | LOAD_ZVAL_ADDR CARG3, res_addr 5509 | EXT_CALL zend_jit_fetch_dim_r_helper, REG0 5510 | mov REG0, RETVALx 5511 | b >9 5512 break; 5513 case BP_JIT_IS: 5514 | EXT_CALL zend_jit_fetch_dim_isset_helper, REG0 5515 | mov REG0, RETVALx 5516 if (not_found_exit_addr) { 5517 | cbz REG0, ¬_found_exit_addr 5518 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5519 | b >8 5520 } 5521 } else if (found_exit_addr) { 5522 | cbnz REG0, &found_exit_addr 5523 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5524 | b >9 5525 } 5526 } else { 5527 | cbnz REG0, >8 5528 | b >9 5529 } 5530 break; 5531 case BP_VAR_IS: 5532 case BP_VAR_UNSET: 5533 | LOAD_ZVAL_ADDR CARG3, res_addr 5534 | EXT_CALL zend_jit_fetch_dim_is_helper, REG0 5535 | mov REG0, RETVALx 5536 | b >9 5537 break; 5538 case BP_VAR_RW: 5539 | EXT_CALL zend_jit_fetch_dim_rw_helper, REG0 5540 | mov REG0, RETVALx 5541 | cbnz REG0, >8 5542 | b >9 5543 break; 5544 case BP_VAR_W: 5545 | EXT_CALL zend_jit_fetch_dim_w_helper, REG0 5546 | mov REG0, RETVALx 5547 | cbnz REG0, >8 5548 | b >9 5549 break; 5550 default: 5551 ZEND_UNREACHABLE(); 5552 } 5553 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5554 |.code 5555 } 5556 } 5557 5558 return 1; 5559} 5560 5561static int zend_jit_simple_assign(dasm_State **Dst, 5562 const zend_op *opline, 5563 zend_jit_addr var_addr, 5564 uint32_t var_info, 5565 uint32_t var_def_info, 5566 uint8_t val_type, 5567 zend_jit_addr val_addr, 5568 uint32_t val_info, 5569 zend_jit_addr res_addr, 5570 int in_cold, 5571 int save_r1, 5572 bool check_exception) 5573/* Labels: 1,2,3 */ 5574{ 5575 zend_reg tmp_reg; 5576 5577 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_REG0) { 5578 tmp_reg = ZREG_REG0; 5579 } else { 5580 /* ASSIGN_DIM */ 5581 tmp_reg = ZREG_FCARG1; 5582 } 5583 5584 if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 5585 zval *zv = Z_ZV(val_addr); 5586 5587 if (!res_addr) { 5588 | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 5589 } else { 5590 | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 5591 } 5592 if (Z_REFCOUNTED_P(zv)) { 5593 if (!res_addr) { 5594 | ADDREF_CONST zv, TMP1, TMP2 5595 } else { 5596 | ADDREF_CONST_2 zv, TMP1, TMP2 5597 } 5598 } 5599 } else { 5600 if (val_info & MAY_BE_UNDEF) { 5601 if (in_cold) { 5602 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2, ZREG_TMP1 5603 } else { 5604 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 5605 |.cold_code 5606 |1: 5607 } 5608 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 5609 if (save_r1) { 5610 | str FCARG1x, T1 // save 5611 } 5612 | SET_ZVAL_TYPE_INFO var_addr, IS_NULL, TMP1w, TMP2 5613 if (res_addr) { 5614 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 5615 } 5616 if (opline) { 5617 | SET_EX_OPLINE opline, Rx(tmp_reg) 5618 } 5619 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); 5620 | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) 5621 | EXT_CALL zend_jit_undefined_op_helper, REG0 5622 if (check_exception) { 5623 | cbz RETVALx, ->exception_handler_undef 5624 } 5625 if (save_r1) { 5626 | ldr FCARG1x, T1 // restore 5627 } 5628 | b >3 5629 if (in_cold) { 5630 |2: 5631 } else { 5632 |.code 5633 } 5634 } 5635 if (val_info & MAY_BE_REF) { 5636 if (val_type == IS_CV) { 5637 ZEND_ASSERT(Z_REG(var_addr) != ZREG_REG2); 5638 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_REG2 || Z_OFFSET(val_addr) != 0) { 5639 | LOAD_ZVAL_ADDR REG2, val_addr 5640 } 5641 | ZVAL_DEREF REG2, val_info, TMP1w 5642 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); 5643 } else { 5644 zend_jit_addr ref_addr; 5645 5646 if (in_cold) { 5647 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 5648 } else { 5649 | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 5650 |.cold_code 5651 |1: 5652 } 5653 if (Z_REG(val_addr) == ZREG_REG2) { 5654 | str REG2, T1 // save 5655 } 5656 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 5657 | GET_ZVAL_PTR REG2, val_addr, TMP1 5658 | GC_DELREF REG2, TMP1w 5659 | // ZVAL_COPY_VALUE(return_value, &ref->val); 5660 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, offsetof(zend_reference, val)); 5661 if (!res_addr) { 5662 | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5663 } else { 5664 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5665 } 5666 | beq >2 // GC_DELREF() reached zero 5667 | IF_NOT_REFCOUNTED REG2w, >3, TMP1w 5668 if (!res_addr) { 5669 | GC_ADDREF Rx(tmp_reg), TMP1w 5670 } else { 5671 | GC_ADDREF_2 Rx(tmp_reg), TMP1w 5672 } 5673 | b >3 5674 |2: 5675 if (res_addr) { 5676 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w 5677 | GC_ADDREF Rx(tmp_reg), TMP1w 5678 |2: 5679 } 5680 if (Z_REG(val_addr) == ZREG_REG2) { 5681 | ldr REG2, T1 // restore 5682 } 5683 if (save_r1) { 5684 | str FCARG1x, T1 // save 5685 } 5686 | GET_ZVAL_PTR FCARG1x, val_addr, TMP1 5687 | EFREE_REFERENCE 5688 if (save_r1) { 5689 | ldr FCARG1x, T1 // restore 5690 } 5691 | b >3 5692 if (in_cold) { 5693 |1: 5694 } else { 5695 |.code 5696 } 5697 } 5698 } 5699 5700 if (!res_addr) { 5701 | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5702 } else { 5703 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 5704 } 5705 5706 if (val_type == IS_CV) { 5707 if (!res_addr) { 5708 | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w 5709 } else { 5710 | TRY_ADDREF_2 val_info, REG2w, Rx(tmp_reg), TMP1w 5711 } 5712 } else { 5713 if (res_addr) { 5714 | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w 5715 } 5716 } 5717 |3: 5718 } 5719 return 1; 5720} 5721 5722static int zend_jit_assign_to_typed_ref(dasm_State **Dst, 5723 const zend_op *opline, 5724 uint8_t val_type, 5725 zend_jit_addr val_addr, 5726 zend_jit_addr res_addr, 5727 bool check_exception) 5728{ 5729 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 5730 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 5731 | cbnz TMP1, >2 5732 |.cold_code 5733 |2: 5734 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 5735 | LOAD_ZVAL_ADDR FCARG2x, val_addr 5736 } 5737 if (opline) { 5738 | SET_EX_OPLINE opline, REG0 5739 } 5740 if (!res_addr) { 5741 if (val_type == IS_CONST) { 5742 | EXT_CALL zend_jit_assign_const_to_typed_ref, REG0 5743 } else if (val_type == IS_TMP_VAR) { 5744 | EXT_CALL zend_jit_assign_tmp_to_typed_ref, REG0 5745 } else if (val_type == IS_VAR) { 5746 | EXT_CALL zend_jit_assign_var_to_typed_ref, REG0 5747 } else if (val_type == IS_CV) { 5748 | EXT_CALL zend_jit_assign_cv_to_typed_ref, REG0 5749 } else { 5750 ZEND_UNREACHABLE(); 5751 } 5752 } else { 5753 | LOAD_ZVAL_ADDR CARG3, res_addr 5754 if (val_type == IS_CONST) { 5755 | EXT_CALL zend_jit_assign_const_to_typed_ref2, REG0 5756 } else if (val_type == IS_TMP_VAR) { 5757 | EXT_CALL zend_jit_assign_tmp_to_typed_ref2, REG0 5758 } else if (val_type == IS_VAR) { 5759 | EXT_CALL zend_jit_assign_var_to_typed_ref2, REG0 5760 } else if (val_type == IS_CV) { 5761 | EXT_CALL zend_jit_assign_cv_to_typed_ref2, REG0 5762 } else { 5763 ZEND_UNREACHABLE(); 5764 } 5765 } 5766 if (check_exception) { 5767 | // if (UNEXPECTED(EG(exception) != NULL)) { 5768 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 5769 | cbz REG0, >8 // END OF zend_jit_assign_to_variable() 5770 | b ->exception_handler 5771 } else { 5772 | b >8 5773 } 5774 |.code 5775 5776 return 1; 5777} 5778 5779static int zend_jit_assign_to_variable_call(dasm_State **Dst, 5780 const zend_op *opline, 5781 zend_jit_addr __var_use_addr, 5782 zend_jit_addr var_addr, 5783 uint32_t __var_info, 5784 uint32_t __var_def_info, 5785 uint8_t val_type, 5786 zend_jit_addr val_addr, 5787 uint32_t val_info, 5788 zend_jit_addr __res_addr, 5789 bool __check_exception) 5790{ 5791 if (val_info & MAY_BE_UNDEF) { 5792 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 5793 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5794 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5795 5796 if (!exit_addr) { 5797 return 0; 5798 } 5799 5800 | IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr, ZREG_TMP1 5801 } else { 5802 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 5803 |.cold_code 5804 |1: 5805 ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP); 5806 if (Z_REG(var_addr) != ZREG_FP) { 5807 | str Rx(Z_REG(var_addr)), T1 // save 5808 } 5809 | SET_EX_OPLINE opline, REG0 5810 | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) 5811 | EXT_CALL zend_jit_undefined_op_helper, REG0 5812 if (Z_REG(var_addr) != ZREG_FP) { 5813 | ldr Rx(Z_REG(var_addr)), T1 // restore 5814 } 5815 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 5816 | LOAD_ZVAL_ADDR FCARG1x, var_addr 5817 } 5818 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 5819 | bl ->assign_const 5820 | b >9 5821 |.code 5822 |1: 5823 } 5824 } 5825 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 5826 | LOAD_ZVAL_ADDR FCARG1x, var_addr 5827 } 5828 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 5829 | LOAD_ZVAL_ADDR FCARG2x, val_addr 5830 } 5831 if (opline) { 5832 | SET_EX_OPLINE opline, REG0 5833 } 5834 if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 5835 | bl ->assign_tmp 5836 } else if (val_type == IS_CONST) { 5837 | bl ->assign_const 5838 } else if (val_type == IS_TMP_VAR) { 5839 | bl ->assign_tmp 5840 } else if (val_type == IS_VAR) { 5841 if (!(val_info & MAY_BE_REF)) { 5842 | bl ->assign_tmp 5843 } else { 5844 | bl ->assign_var 5845 } 5846 } else if (val_type == IS_CV) { 5847 if (!(val_info & MAY_BE_REF)) { 5848 | bl ->assign_cv_noref 5849 } else { 5850 | bl ->assign_cv 5851 } 5852 if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5853 |9: 5854 } 5855 } else { 5856 ZEND_UNREACHABLE(); 5857 } 5858 5859 return 1; 5860} 5861 5862static int zend_jit_assign_to_variable(dasm_State **Dst, 5863 const zend_op *opline, 5864 zend_jit_addr var_use_addr, 5865 zend_jit_addr var_addr, 5866 uint32_t var_info, 5867 uint32_t var_def_info, 5868 uint8_t val_type, 5869 zend_jit_addr val_addr, 5870 uint32_t val_info, 5871 zend_jit_addr res_addr, 5872 bool check_exception) 5873/* Labels: 1,2,3,4,5,8 */ 5874{ 5875 int done = 0; 5876 zend_reg ref_reg, tmp_reg; 5877 5878 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_REG0) { 5879 ref_reg = ZREG_FCARG1; 5880 tmp_reg = ZREG_REG0; 5881 } else { 5882 /* ASSIGN_DIM */ 5883 ref_reg = ZREG_REG0; 5884 tmp_reg = ZREG_FCARG1; 5885 } 5886 5887 if (var_info & MAY_BE_REF) { 5888 if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { 5889 | LOAD_ZVAL_ADDR Rx(ref_reg), var_use_addr 5890 var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); 5891 } 5892 | // if (Z_ISREF_P(variable_ptr)) { 5893 | IF_NOT_Z_TYPE Rx(ref_reg), IS_REFERENCE, >3, TMP1w 5894 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 5895 | GET_Z_PTR FCARG1x, Rx(ref_reg) 5896 if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) { 5897 return 0; 5898 } 5899 | add Rx(ref_reg), FCARG1x, #offsetof(zend_reference, val) 5900 |3: 5901 } 5902 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 5903 if (RC_MAY_BE_1(var_info)) { 5904 int in_cold = 0; 5905 5906 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 5907 | IF_ZVAL_REFCOUNTED var_use_addr, >1, ZREG_TMP1, ZREG_TMP2 5908 |.cold_code 5909 |1: 5910 in_cold = 1; 5911 } 5912 if (Z_REG(var_use_addr) == ZREG_FCARG1 || Z_REG(var_use_addr) == ZREG_REG0) { 5913 bool keep_gc = 0; 5914 5915 | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 5916#if 0 5917 // TODO: This optiization doesn't work on ARM 5918 if (tmp_reg == ZREG_FCARG1) { 5919 if (Z_MODE(val_addr) == IS_REG) { 5920 keep_gc = 1; 5921 } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { 5922 keep_gc = 1; 5923 } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 5924 zval *zv = Z_ZV(val_addr); 5925 if (Z_TYPE_P(zv) == IS_DOUBLE) { 5926 if (Z_DVAL_P(zv) == 0) { 5927 keep_gc = 1; 5928 } 5929 } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 5930 keep_gc = 1; 5931 } 5932 } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { 5933 if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 5934 keep_gc = 1; 5935 } 5936 } 5937 } 5938#endif 5939 if (!keep_gc) { 5940 | str Rx(tmp_reg), T1 // save 5941 } 5942 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0, 0)) { 5943 return 0; 5944 } 5945 if (!keep_gc) { 5946 | ldr FCARG1x, T1 // restore 5947 } 5948 } else { 5949 | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 5950 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1, 0)) { 5951 return 0; 5952 } 5953 } 5954 | GC_DELREF FCARG1x, TMP1w 5955 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 5956 | bne >4 5957 } else { 5958 | bne >8 5959 } 5960 | ZVAL_DTOR_FUNC var_info, opline, TMP1 5961 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { 5962 if (check_exception && !(val_info & MAY_BE_UNDEF)) { 5963 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 5964 | cbz REG0, >8 5965 | b ->exception_handler 5966 } else { 5967 | b >8 5968 } 5969 } 5970 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 5971 |4: 5972 | IF_GC_MAY_NOT_LEAK FCARG1x, >8, TMP1w, TMP2w 5973 | EXT_CALL gc_possible_root, REG0 5974 if (in_cold) { 5975 | b >8 5976 } 5977 } 5978 if (check_exception && (val_info & MAY_BE_UNDEF)) { 5979 |8: 5980 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 5981 | cbz REG0, >8 5982 | b ->exception_handler 5983 } 5984 if (in_cold) { 5985 |.code 5986 } else { 5987 done = 1; 5988 } 5989 } else /* if (RC_MAY_BE_N(var_info)) */ { 5990 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 5991 | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5, ZREG_TMP1, ZREG_TMP2 5992 } 5993 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { 5994 if (Z_REG(var_use_addr) != ZREG_FP) { 5995 | str Rx(Z_REG(var_use_addr)), T1 // save 5996 } 5997 | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 5998 | GC_DELREF FCARG1x, TMP1w 5999 | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w 6000 | EXT_CALL gc_possible_root, TMP1 6001 if (Z_REG(var_use_addr) != ZREG_FP) { 6002 | ldr Rx(Z_REG(var_use_addr)), T1 // restore 6003 } 6004 } else { 6005 | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 6006 | GC_DELREF Rx(tmp_reg), TMP1w 6007 } 6008 |5: 6009 } 6010 } 6011 6012 if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0, check_exception)) { 6013 return 0; 6014 } 6015 6016 |8: 6017 6018 return 1; 6019} 6020 6021static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw) 6022{ 6023 zend_jit_addr op2_addr, op3_addr, res_addr; 6024 6025 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6026 op3_addr = OP1_DATA_ADDR(); 6027 if (opline->result_type == IS_UNUSED) { 6028 res_addr = 0; 6029 } else { 6030 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 6031 } 6032 6033 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { 6034 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6035 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6036 6037 if (!exit_addr) { 6038 return 0; 6039 } 6040 6041 | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr, ZREG_TMP1 6042 6043 val_info &= ~MAY_BE_UNDEF; 6044 } 6045 6046 if (op1_info & MAY_BE_REF) { 6047 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6048 | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w 6049 | GET_Z_PTR FCARG2x, FCARG1x 6050 | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] 6051 | cmp TMP1w, #IS_ARRAY 6052 | bne >2 6053 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) 6054 | b >3 6055 |.cold_code 6056 |2: 6057 | SET_EX_OPLINE opline, REG0 6058 | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 6059 | mov FCARG1x, RETVALx 6060 | cbnz FCARG1x, >1 6061 | b ->exception_handler_undef 6062 |.code 6063 |1: 6064 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6065 } 6066 6067 if (op1_info & MAY_BE_ARRAY) { 6068 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6069 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 6070 } 6071 |3: 6072 | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 6073 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6074 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6075 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 6076 | bgt >7 6077 } 6078 | // ZVAL_ARR(container, zend_new_array(8)); 6079 if (Z_REG(op1_addr) != ZREG_FP) { 6080 | str Rx(Z_REG(op1_addr)), T1 // save 6081 } 6082 | EXT_CALL _zend_new_array_0, REG0 6083 | mov REG0, RETVALx 6084 if (Z_REG(op1_addr) != ZREG_FP) { 6085 | ldr Rx(Z_REG(op1_addr)), T1 // restore 6086 } 6087 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 6088 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 6089 | mov FCARG1x, REG0 6090 } 6091 6092 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6093 |6: 6094 if (opline->op2_type == IS_UNUSED) { 6095 uint32_t var_info = MAY_BE_NULL; 6096 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 6097 6098 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6099 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 6100 | EXT_CALL zend_hash_next_index_insert, REG0 6101 | // if (UNEXPECTED(!var_ptr)) { 6102 | mov REG0, RETVALx 6103 | cbz REG0, >1 6104 |.cold_code 6105 |1: 6106 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6107 | CANNOT_ADD_ELEMENT opline 6108 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6109 | b >9 6110 |.code 6111 6112 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0, 0)) { 6113 return 0; 6114 } 6115 } else { 6116 uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6117 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 6118 6119 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 6120 return 0; 6121 } 6122 6123 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6124 var_info |= MAY_BE_REF; 6125 } 6126 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6127 var_info |= MAY_BE_RC1; 6128 } 6129 6130 |8: 6131 | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); 6132 if (opline->op1_type == IS_VAR) { 6133 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6134 if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6135 return 0; 6136 } 6137 } else { 6138 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6139 return 0; 6140 } 6141 } 6142 } 6143 } 6144 6145 if (((op1_info & MAY_BE_ARRAY) && 6146 (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) || 6147 (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY)))) { 6148 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6149 |.cold_code 6150 |7: 6151 } 6152 6153 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) && 6154 (op1_info & MAY_BE_ARRAY)) { 6155 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6156 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 6157 | bgt >2 6158 } 6159 | // ZVAL_ARR(container, zend_new_array(8)); 6160 if (Z_REG(op1_addr) != ZREG_FP) { 6161 | str Rx(Z_REG(op1_addr)), T1 // save 6162 } 6163 | EXT_CALL _zend_new_array_0, REG0 6164 | mov REG0, RETVALx 6165 if (Z_REG(op1_addr) != ZREG_FP) { 6166 | ldr Rx(Z_REG(op1_addr)), T1 // restore 6167 } 6168 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 6169 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 6170 | mov FCARG1x, REG0 6171 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); 6172 | b <6 6173 |2: 6174 } 6175 6176 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6177 | SET_EX_OPLINE opline, REG0 6178 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6179 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6180 } 6181 if (opline->op2_type == IS_UNUSED) { 6182 | mov FCARG2x, xzr 6183 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6184 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6185 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 6186 } else { 6187 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 6188 } 6189 if (opline->result_type == IS_UNUSED) { 6190 | mov CARG4, xzr 6191 } else { 6192 | LOAD_ZVAL_ADDR CARG4, res_addr 6193 } 6194 | LOAD_ZVAL_ADDR CARG3, op3_addr 6195 | EXT_CALL zend_jit_assign_dim_helper, REG0 6196 6197#ifdef ZEND_JIT_USE_RC_INFERENCE 6198 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { 6199 /* ASSIGN_DIM may increase refcount of the value */ 6200 val_info |= MAY_BE_RCN; 6201 } 6202#endif 6203 6204 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 6205 } 6206 6207 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6208 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6209 | b >9 // END 6210 } 6211 |.code 6212 } 6213 } 6214 6215#ifdef ZEND_JIT_USE_RC_INFERENCE 6216 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 6217 /* ASSIGN_DIM may increase refcount of the key */ 6218 op2_info |= MAY_BE_RCN; 6219 } 6220#endif 6221 6222 |9: 6223 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6224 6225 if (may_throw) { 6226 zend_jit_check_exception(Dst); 6227 } 6228 6229 return 1; 6230} 6231 6232static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) 6233{ 6234 zend_jit_addr op2_addr, op3_addr, var_addr; 6235 const void *not_found_exit_addr = NULL; 6236 uint32_t var_info = MAY_BE_NULL; 6237 6238 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6239 6240 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6241 op3_addr = OP1_DATA_ADDR(); 6242 6243 | SET_EX_OPLINE opline, REG0 6244 if (op1_info & MAY_BE_REF) { 6245 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6246 | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w 6247 | GET_Z_PTR FCARG2x, FCARG1x 6248 | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] 6249 | cmp TMP1w, #IS_ARRAY 6250 | bne >2 6251 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) 6252 | b >3 6253 |.cold_code 6254 |2: 6255 | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 6256 | mov FCARG1x, RETVALx 6257 | cbnz RETVALx, >1 6258 | b ->exception_handler_undef 6259 |.code 6260 |1: 6261 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6262 } 6263 6264 if (op1_info & MAY_BE_ARRAY) { 6265 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6266 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 6267 } 6268 |3: 6269 | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 6270 } 6271 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6272 if (op1_info & MAY_BE_ARRAY) { 6273 |.cold_code 6274 |7: 6275 } 6276 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6277 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 6278 | bgt >7 6279 } 6280 if (Z_REG(op1_addr) != ZREG_FP) { 6281 | str Rx(Z_REG(op1_addr)), T1 // save 6282 } 6283 if (op1_info & MAY_BE_UNDEF) { 6284 if (op1_info & MAY_BE_NULL) { 6285 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 6286 } 6287 | LOAD_32BIT_VAL FCARG1x, opline->op1.var 6288 | EXT_CALL zend_jit_undefined_op_helper, REG0 6289 |1: 6290 } 6291 | // ZVAL_ARR(container, zend_new_array(8)); 6292 | EXT_CALL _zend_new_array_0, REG0 6293 | mov REG0, RETVALx 6294 if (Z_REG(op1_addr) != ZREG_FP) { 6295 | ldr Rx(Z_REG(op1_addr)), T1 // restore 6296 } 6297 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 6298 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 6299 | mov FCARG1x, REG0 6300 if (op1_info & MAY_BE_ARRAY) { 6301 | b >1 6302 |.code 6303 |1: 6304 } 6305 } 6306 6307 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6308 uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); 6309 6310 |6: 6311 if (opline->op2_type == IS_UNUSED) { 6312 var_info = MAY_BE_NULL; 6313 6314 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6315 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 6316 | EXT_CALL zend_hash_next_index_insert, REG0 6317 | mov REG0, RETVALx 6318 | // if (UNEXPECTED(!var_ptr)) { 6319 | cbz REG0, >1 6320 |.cold_code 6321 |1: 6322 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6323 | CANNOT_ADD_ELEMENT opline 6324 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6325 | b >9 6326 |.code 6327 } else { 6328 var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6329 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6330 var_info |= MAY_BE_REF; 6331 } 6332 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6333 var_info |= MAY_BE_RC1; 6334 } 6335 6336 if (dim_type != IS_UNKNOWN 6337 && dim_type != IS_UNDEF 6338 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY 6339 && (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) 6340 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 6341 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 6342 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6343 if (!not_found_exit_addr) { 6344 return 0; 6345 } 6346 } 6347 6348 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, not_found_exit_addr, NULL)) { 6349 return 0; 6350 } 6351 6352 |8: 6353 if (not_found_exit_addr && dim_type != IS_REFERENCE) { 6354 | IF_NOT_Z_TYPE, REG0, dim_type, ¬_found_exit_addr, TMP1w 6355 var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 6356 } 6357 if (var_info & MAY_BE_REF) { 6358 binary_op_type binary_op = get_binary_op(opline->extended_value); 6359 | IF_NOT_Z_TYPE, REG0, IS_REFERENCE, >1, TMP1w 6360 | GET_Z_PTR FCARG1x, REG0 6361 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 6362 | cbnz TMP1, >2 6363 | add REG0, FCARG1x, #offsetof(zend_reference, val) 6364 |.cold_code 6365 |2: 6366 | LOAD_ZVAL_ADDR FCARG2x, op3_addr 6367 | LOAD_ADDR CARG3, binary_op 6368 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 6369 && (op1_data_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6370 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 6371 } else { 6372 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 6373 } 6374 | b >9 6375 |.code 6376 |1: 6377 } 6378 } 6379 6380 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 6381 switch (opline->extended_value) { 6382 case ZEND_ADD: 6383 case ZEND_SUB: 6384 case ZEND_MUL: 6385 case ZEND_DIV: 6386 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, 6387 1 /* may overflow */, may_throw)) { 6388 return 0; 6389 } 6390 break; 6391 case ZEND_BW_OR: 6392 case ZEND_BW_AND: 6393 case ZEND_BW_XOR: 6394 case ZEND_SL: 6395 case ZEND_SR: 6396 case ZEND_MOD: 6397 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6398 IS_CV, opline->op1, var_addr, var_info, NULL, 6399 (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 6400 op1_data_range, 6401 0, var_addr, var_def_info, var_info, may_throw)) { 6402 return 0; 6403 } 6404 break; 6405 case ZEND_CONCAT: 6406 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, 6407 may_throw)) { 6408 return 0; 6409 } 6410 break; 6411 default: 6412 ZEND_UNREACHABLE(); 6413 } 6414 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6415 } 6416 6417 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6418 binary_op_type binary_op; 6419 6420 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6421 |.cold_code 6422 |7: 6423 } 6424 6425 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6426 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6427 } 6428 if (opline->op2_type == IS_UNUSED) { 6429 | mov FCARG2x, xzr 6430 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6431 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6432 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 6433 } else { 6434 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 6435 } 6436 binary_op = get_binary_op(opline->extended_value); 6437 | LOAD_ZVAL_ADDR CARG3, op3_addr 6438 | LOAD_ADDR CARG4, binary_op 6439 | EXT_CALL zend_jit_assign_dim_op_helper, REG0 6440 6441 |9: 6442 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 6443 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 6444 if (may_throw) { 6445 zend_jit_check_exception(Dst); 6446 } 6447 6448 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6449 | b >9 // END 6450 |.code 6451 |9: 6452 } 6453 } else if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) 6454 && (!not_found_exit_addr || (var_info & MAY_BE_REF))) { 6455 |.cold_code 6456 |9: 6457 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6458 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 6459 if (may_throw) { 6460 zend_jit_check_exception(Dst); 6461 } 6462 | b >9 6463 |.code 6464 |9: 6465 } 6466 6467 return 1; 6468} 6469 6470static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) 6471{ 6472 zend_jit_addr op1_addr, op2_addr; 6473 6474 ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); 6475 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 6476 6477 op1_addr = OP1_ADDR(); 6478 op2_addr = OP2_ADDR(); 6479 6480 if (op1_info & MAY_BE_REF) { 6481 binary_op_type binary_op = get_binary_op(opline->extended_value); 6482 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 6483 | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >1, TMP1w 6484 | GET_Z_PTR FCARG1x, FCARG1x 6485 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 6486 | cbnz TMP1, >2 6487 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) 6488 |.cold_code 6489 |2: 6490 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 6491 | LOAD_ADDR CARG3, binary_op 6492 | SET_EX_OPLINE opline, REG0 6493 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 6494 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6495 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 6496 } else { 6497 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 6498 } 6499 zend_jit_check_exception(Dst); 6500 | b >9 6501 |.code 6502 |1: 6503 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6504 } 6505 6506 int result; 6507 switch (opline->extended_value) { 6508 case ZEND_ADD: 6509 case ZEND_SUB: 6510 case ZEND_MUL: 6511 case ZEND_DIV: 6512 result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); 6513 break; 6514 case ZEND_BW_OR: 6515 case ZEND_BW_AND: 6516 case ZEND_BW_XOR: 6517 case ZEND_SL: 6518 case ZEND_SR: 6519 case ZEND_MOD: 6520 result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6521 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 6522 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 6523 opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); 6524 break; 6525 case ZEND_CONCAT: 6526 result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); 6527 break; 6528 default: 6529 ZEND_UNREACHABLE(); 6530 } 6531 |9: 6532 return result; 6533} 6534 6535static int zend_jit_cmp_long_long(dasm_State **Dst, 6536 const zend_op *opline, 6537 zend_ssa_range *op1_range, 6538 zend_jit_addr op1_addr, 6539 zend_ssa_range *op2_range, 6540 zend_jit_addr op2_addr, 6541 zend_jit_addr res_addr, 6542 uint8_t smart_branch_opcode, 6543 uint32_t target_label, 6544 uint32_t target_label2, 6545 const void *exit_addr, 6546 bool skip_comparison) 6547{ 6548 bool swap = 0; 6549 bool result; 6550 6551 if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { 6552 if (!smart_branch_opcode || 6553 smart_branch_opcode == ZEND_JMPZ_EX || 6554 smart_branch_opcode == ZEND_JMPNZ_EX) { 6555 | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE), TMP1w, TMP2 6556 } 6557 if (smart_branch_opcode && !exit_addr) { 6558 if (smart_branch_opcode == ZEND_JMPZ || 6559 smart_branch_opcode == ZEND_JMPZ_EX) { 6560 if (!result) { 6561 | b => target_label 6562 } 6563 } else if (smart_branch_opcode == ZEND_JMPNZ || 6564 smart_branch_opcode == ZEND_JMPNZ_EX) { 6565 if (result) { 6566 | b => target_label 6567 } 6568 } else { 6569 ZEND_UNREACHABLE(); 6570 } 6571 } 6572 return 1; 6573 } 6574 6575 if (skip_comparison) { 6576 if (Z_MODE(op1_addr) != IS_REG && 6577 (Z_MODE(op2_addr) == IS_REG || 6578 (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { 6579 swap = 1; 6580 } 6581 } else if (Z_MODE(op1_addr) == IS_REG) { 6582 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 6583 | cmp Rx(Z_REG(op1_addr)), xzr 6584 } else { 6585 | LONG_CMP Z_REG(op1_addr), op2_addr, TMP1 6586 } 6587 } else if (Z_MODE(op2_addr) == IS_REG) { 6588 if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { 6589 | cmp Rx(Z_REG(op2_addr)), xzr 6590 } else { 6591 | LONG_CMP Z_REG(op2_addr), op1_addr, TMP1 6592 } 6593 swap = 1; 6594 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 6595 | LONG_CMP_WITH_CONST op2_addr, Z_LVAL_P(Z_ZV(op1_addr)), TMP1, TMP2 6596 swap = 1; 6597 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { 6598 | LONG_CMP_WITH_CONST op1_addr, Z_LVAL_P(Z_ZV(op2_addr)), TMP1, TMP2 6599 } else { 6600 | GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1 6601 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 6602 | cmp Rx(ZREG_REG0), xzr 6603 } else { 6604 | LONG_CMP ZREG_REG0, op2_addr, TMP1 6605 } 6606 } 6607 6608 if (smart_branch_opcode) { 6609 if (smart_branch_opcode == ZEND_JMPZ_EX || 6610 smart_branch_opcode == ZEND_JMPNZ_EX) { 6611 6612 switch (opline->opcode) { 6613 case ZEND_IS_EQUAL: 6614 case ZEND_IS_IDENTICAL: 6615 case ZEND_CASE: 6616 case ZEND_CASE_STRICT: 6617 | cset REG0w, eq 6618 break; 6619 case ZEND_IS_NOT_EQUAL: 6620 case ZEND_IS_NOT_IDENTICAL: 6621 | cset REG0w, ne 6622 break; 6623 case ZEND_IS_SMALLER: 6624 if (swap) { 6625 | cset REG0w, gt 6626 } else { 6627 | cset REG0w, lt 6628 } 6629 break; 6630 case ZEND_IS_SMALLER_OR_EQUAL: 6631 if (swap) { 6632 | cset REG0w, ge 6633 } else { 6634 | cset REG0w, le 6635 } 6636 break; 6637 default: 6638 ZEND_UNREACHABLE(); 6639 } 6640 | add REG0w, REG0w, #2 6641 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 6642 } 6643 if (smart_branch_opcode == ZEND_JMPZ || 6644 smart_branch_opcode == ZEND_JMPZ_EX) { 6645 switch (opline->opcode) { 6646 case ZEND_IS_EQUAL: 6647 case ZEND_IS_IDENTICAL: 6648 case ZEND_CASE: 6649 case ZEND_CASE_STRICT: 6650 if (exit_addr) { 6651 | bne &exit_addr 6652 } else { 6653 | bne => target_label 6654 } 6655 break; 6656 case ZEND_IS_NOT_EQUAL: 6657 if (exit_addr) { 6658 | beq &exit_addr 6659 } else { 6660 | beq => target_label 6661 } 6662 break; 6663 case ZEND_IS_NOT_IDENTICAL: 6664 if (exit_addr) { 6665 | bne &exit_addr 6666 } else { 6667 | beq => target_label 6668 } 6669 break; 6670 case ZEND_IS_SMALLER: 6671 if (swap) { 6672 if (exit_addr) { 6673 | ble &exit_addr 6674 } else { 6675 | ble => target_label 6676 } 6677 } else { 6678 if (exit_addr) { 6679 | bge &exit_addr 6680 } else { 6681 | bge => target_label 6682 } 6683 } 6684 break; 6685 case ZEND_IS_SMALLER_OR_EQUAL: 6686 if (swap) { 6687 if (exit_addr) { 6688 | blt &exit_addr 6689 } else { 6690 | blt => target_label 6691 } 6692 } else { 6693 if (exit_addr) { 6694 | bgt &exit_addr 6695 } else { 6696 | bgt => target_label 6697 } 6698 } 6699 break; 6700 default: 6701 ZEND_UNREACHABLE(); 6702 } 6703 } else if (smart_branch_opcode == ZEND_JMPNZ || 6704 smart_branch_opcode == ZEND_JMPNZ_EX) { 6705 switch (opline->opcode) { 6706 case ZEND_IS_EQUAL: 6707 case ZEND_IS_IDENTICAL: 6708 case ZEND_CASE: 6709 case ZEND_CASE_STRICT: 6710 if (exit_addr) { 6711 | beq &exit_addr 6712 } else { 6713 | beq => target_label 6714 } 6715 break; 6716 case ZEND_IS_NOT_EQUAL: 6717 if (exit_addr) { 6718 | bne &exit_addr 6719 } else { 6720 | bne => target_label 6721 } 6722 break; 6723 case ZEND_IS_NOT_IDENTICAL: 6724 if (exit_addr) { 6725 | beq &exit_addr 6726 } else { 6727 | bne => target_label 6728 } 6729 break; 6730 case ZEND_IS_SMALLER: 6731 if (swap) { 6732 if (exit_addr) { 6733 | bgt &exit_addr 6734 } else { 6735 | bgt => target_label 6736 } 6737 } else { 6738 if (exit_addr) { 6739 | blt &exit_addr 6740 } else { 6741 | blt => target_label 6742 } 6743 } 6744 break; 6745 case ZEND_IS_SMALLER_OR_EQUAL: 6746 if (swap) { 6747 if (exit_addr) { 6748 | bge &exit_addr 6749 } else { 6750 | bge => target_label 6751 } 6752 } else { 6753 if (exit_addr) { 6754 | ble &exit_addr 6755 } else { 6756 | ble => target_label 6757 } 6758 } 6759 break; 6760 default: 6761 ZEND_UNREACHABLE(); 6762 } 6763 } else { 6764 ZEND_UNREACHABLE(); 6765 } 6766 } else { 6767 switch (opline->opcode) { 6768 case ZEND_IS_EQUAL: 6769 case ZEND_IS_IDENTICAL: 6770 case ZEND_CASE: 6771 case ZEND_CASE_STRICT: 6772 | cset REG0w, eq 6773 break; 6774 case ZEND_IS_NOT_EQUAL: 6775 case ZEND_IS_NOT_IDENTICAL: 6776 | cset REG0w, ne 6777 break; 6778 case ZEND_IS_SMALLER: 6779 if (swap) { 6780 | cset REG0w, gt 6781 } else { 6782 | cset REG0w, lt 6783 } 6784 break; 6785 case ZEND_IS_SMALLER_OR_EQUAL: 6786 if (swap) { 6787 | cset REG0w, ge 6788 } else { 6789 | cset REG0w, le 6790 } 6791 break; 6792 default: 6793 ZEND_UNREACHABLE(); 6794 } 6795 | add REG0w, REG0w, #2 6796 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 6797 } 6798 6799 return 1; 6800} 6801 6802static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 6803{ 6804 if (smart_branch_opcode) { 6805 if (smart_branch_opcode == ZEND_JMPZ) { 6806 switch (opline->opcode) { 6807 case ZEND_IS_EQUAL: 6808 case ZEND_IS_IDENTICAL: 6809 case ZEND_CASE: 6810 case ZEND_CASE_STRICT: 6811 if (exit_addr) { 6812 | bne &exit_addr 6813 } else { 6814 | bne => target_label 6815 } 6816 break; 6817 case ZEND_IS_NOT_EQUAL: 6818 | bvs >1 6819 if (exit_addr) { 6820 | beq &exit_addr 6821 } else { 6822 | beq => target_label 6823 } 6824 |1: 6825 break; 6826 case ZEND_IS_NOT_IDENTICAL: 6827 if (exit_addr) { 6828 | bvs &exit_addr 6829 | bne &exit_addr 6830 } else { 6831 | bvs >1 6832 | beq => target_label 6833 |1: 6834 } 6835 break; 6836 case ZEND_IS_SMALLER: 6837 if (swap) { 6838 if (exit_addr) { 6839 | bvs &exit_addr 6840 | bls &exit_addr 6841 } else { 6842 | bvs => target_label 6843 | bls => target_label 6844 } 6845 } else { 6846 if (exit_addr) { 6847 | bhs &exit_addr 6848 } else { 6849 | bhs => target_label 6850 } 6851 } 6852 break; 6853 case ZEND_IS_SMALLER_OR_EQUAL: 6854 if (swap) { 6855 if (exit_addr) { 6856 | bvs &exit_addr 6857 | blo &exit_addr 6858 } else { 6859 | bvs => target_label 6860 | blo => target_label 6861 } 6862 } else { 6863 if (exit_addr) { 6864 | bhi &exit_addr 6865 } else { 6866 | bhi => target_label 6867 } 6868 } 6869 break; 6870 default: 6871 ZEND_UNREACHABLE(); 6872 } 6873 } else if (smart_branch_opcode == ZEND_JMPNZ) { 6874 switch (opline->opcode) { 6875 case ZEND_IS_EQUAL: 6876 case ZEND_IS_IDENTICAL: 6877 case ZEND_CASE: 6878 case ZEND_CASE_STRICT: 6879 | bvs >1 6880 if (exit_addr) { 6881 | beq &exit_addr 6882 } else { 6883 | beq => target_label 6884 } 6885 |1: 6886 break; 6887 case ZEND_IS_NOT_EQUAL: 6888 if (exit_addr) { 6889 | bne &exit_addr 6890 } else { 6891 | bne => target_label 6892 } 6893 break; 6894 case ZEND_IS_NOT_IDENTICAL: 6895 if (exit_addr) { 6896 | bvs >1 6897 | beq &exit_addr 6898 |1: 6899 } else { 6900 | bne => target_label 6901 } 6902 break; 6903 case ZEND_IS_SMALLER: 6904 if (swap) { 6905 | bvs >1 // Always False if involving NaN 6906 if (exit_addr) { 6907 | bhi &exit_addr 6908 } else { 6909 | bhi => target_label 6910 } 6911 |1: 6912 } else { 6913 | bvs >1 6914 if (exit_addr) { 6915 | blo &exit_addr 6916 } else { 6917 | blo => target_label 6918 } 6919 |1: 6920 } 6921 break; 6922 case ZEND_IS_SMALLER_OR_EQUAL: 6923 if (swap) { 6924 | bvs >1 // Always False if involving NaN 6925 if (exit_addr) { 6926 | bhs &exit_addr 6927 } else { 6928 | bhs => target_label 6929 } 6930 |1: 6931 } else { 6932 | bvs >1 6933 if (exit_addr) { 6934 | bls &exit_addr 6935 } else { 6936 | bls => target_label 6937 } 6938 |1: 6939 } 6940 break; 6941 default: 6942 ZEND_UNREACHABLE(); 6943 } 6944 } else if (smart_branch_opcode == ZEND_JMPZ_EX) { 6945 switch (opline->opcode) { 6946 case ZEND_IS_EQUAL: 6947 case ZEND_IS_IDENTICAL: 6948 case ZEND_CASE: 6949 case ZEND_CASE_STRICT: 6950 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6951 | bne => target_label 6952 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6953 break; 6954 case ZEND_IS_NOT_EQUAL: 6955 case ZEND_IS_NOT_IDENTICAL: 6956 | bvs >1 6957 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6958 | beq => target_label 6959 |1: 6960 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6961 break; 6962 case ZEND_IS_SMALLER: 6963 if (swap) { 6964 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6965 | bvs => target_label 6966 | bls => target_label 6967 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6968 } else { 6969 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6970 | bhs => target_label 6971 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6972 } 6973 break; 6974 case ZEND_IS_SMALLER_OR_EQUAL: 6975 if (swap) { 6976 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6977 | bvs => target_label 6978 | blo => target_label 6979 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6980 } else { 6981 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 6982 | bhi => target_label 6983 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6984 } 6985 break; 6986 default: 6987 ZEND_UNREACHABLE(); 6988 } 6989 } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { 6990 switch (opline->opcode) { 6991 case ZEND_IS_EQUAL: 6992 case ZEND_IS_IDENTICAL: 6993 case ZEND_CASE: 6994 case ZEND_CASE_STRICT: 6995 | bvs >1 6996 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 6997 | beq => target_label 6998 |1: 6999 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7000 break; 7001 case ZEND_IS_NOT_EQUAL: 7002 case ZEND_IS_NOT_IDENTICAL: 7003 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7004 | bvs => target_label 7005 | bne => target_label 7006 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7007 break; 7008 case ZEND_IS_SMALLER: 7009 if (swap) { 7010 | cset REG0w, hi 7011 | add REG0w, REG0w, #2 7012 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7013 | bvs >1 // Always False if involving NaN 7014 | bhi => target_label 7015 |1: 7016 } else { 7017 | bvs >1 7018 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7019 | blo => target_label 7020 |1: 7021 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7022 } 7023 break; 7024 case ZEND_IS_SMALLER_OR_EQUAL: 7025 if (swap) { 7026 | cset REG0w, hs 7027 | add REG0w, REG0w, #2 7028 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7029 | bvs >1 // Always False if involving NaN 7030 | bhs => target_label 7031 |1: 7032 } else { 7033 | bvs >1 7034 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7035 | bls => target_label 7036 |1: 7037 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7038 } 7039 break; 7040 default: 7041 ZEND_UNREACHABLE(); 7042 } 7043 } else { 7044 ZEND_UNREACHABLE(); 7045 } 7046 } else { 7047 switch (opline->opcode) { 7048 case ZEND_IS_EQUAL: 7049 case ZEND_IS_IDENTICAL: 7050 case ZEND_CASE: 7051 case ZEND_CASE_STRICT: 7052 | bvs >1 7053 | mov REG0, #IS_TRUE 7054 | beq >2 7055 |1: 7056 | mov REG0, #IS_FALSE 7057 |2: 7058 break; 7059 case ZEND_IS_NOT_EQUAL: 7060 case ZEND_IS_NOT_IDENTICAL: 7061 | bvs >1 7062 | mov REG0, #IS_FALSE 7063 | beq >2 7064 |1: 7065 | mov REG0, #IS_TRUE 7066 |2: 7067 break; 7068 case ZEND_IS_SMALLER: 7069 | bvs >1 7070 | mov REG0, #IS_TRUE 7071 || if (swap) { 7072 | bhi >2 7073 || } else { 7074 | blo >2 7075 || } 7076 |1: 7077 | mov REG0, #IS_FALSE 7078 |2: 7079 break; 7080 case ZEND_IS_SMALLER_OR_EQUAL: 7081 | bvs >1 7082 | mov REG0, #IS_TRUE 7083 || if (swap) { 7084 | bhs >2 7085 || } else { 7086 | bls >2 7087 || } 7088 |1: 7089 | mov REG0, #IS_FALSE 7090 |2: 7091 break; 7092 default: 7093 ZEND_UNREACHABLE(); 7094 } 7095 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7096 } 7097 7098 return 1; 7099} 7100 7101static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7102{ 7103 zend_reg tmp_reg = ZREG_FPR0; 7104 7105 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_REG0, ZREG_TMP1 7106 | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP 7107 7108 return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); 7109} 7110 7111static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7112{ 7113 zend_reg tmp_reg = ZREG_FPR0; 7114 7115 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_REG0, ZREG_TMP1 7116 | DOUBLE_CMP tmp_reg, op1_addr, ZREG_TMP1, ZREG_FPTMP 7117 7118 return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); 7119} 7120 7121static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7122{ 7123 bool swap = 0; 7124 7125 if (Z_MODE(op1_addr) == IS_REG) { 7126 | DOUBLE_CMP Z_REG(op1_addr), op2_addr, ZREG_TMP1, ZREG_FPTMP 7127 } else if (Z_MODE(op2_addr) == IS_REG) { 7128 | DOUBLE_CMP Z_REG(op2_addr), op1_addr, ZREG_TMP1, ZREG_FPTMP 7129 swap = 1; 7130 } else { 7131 zend_reg tmp_reg = ZREG_FPR0; 7132 7133 | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 7134 | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP 7135 } 7136 7137 return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); 7138} 7139 7140static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7141{ 7142 | tst RETVALw, RETVALw 7143 if (smart_branch_opcode) { 7144 if (smart_branch_opcode == ZEND_JMPZ_EX || 7145 smart_branch_opcode == ZEND_JMPNZ_EX) { 7146 switch (opline->opcode) { 7147 case ZEND_IS_EQUAL: 7148 case ZEND_CASE: 7149 | cset REG0w, eq 7150 break; 7151 case ZEND_IS_NOT_EQUAL: 7152 | cset REG0w, ne 7153 break; 7154 case ZEND_IS_SMALLER: 7155 | cset REG0w, lt 7156 break; 7157 case ZEND_IS_SMALLER_OR_EQUAL: 7158 | cset REG0w, le 7159 break; 7160 default: 7161 ZEND_UNREACHABLE(); 7162 } 7163 | add REG0w, REG0w, #2 7164 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7165 } 7166 if (smart_branch_opcode == ZEND_JMPZ || 7167 smart_branch_opcode == ZEND_JMPZ_EX) { 7168 switch (opline->opcode) { 7169 case ZEND_IS_EQUAL: 7170 case ZEND_CASE: 7171 if (exit_addr) { 7172 | bne &exit_addr 7173 } else { 7174 | bne => target_label 7175 } 7176 break; 7177 case ZEND_IS_NOT_EQUAL: 7178 if (exit_addr) { 7179 | beq &exit_addr 7180 } else { 7181 | beq => target_label 7182 } 7183 break; 7184 case ZEND_IS_SMALLER: 7185 if (exit_addr) { 7186 | bge &exit_addr 7187 } else { 7188 | bge => target_label 7189 } 7190 break; 7191 case ZEND_IS_SMALLER_OR_EQUAL: 7192 if (exit_addr) { 7193 | bgt &exit_addr 7194 } else { 7195 | bgt => target_label 7196 } 7197 break; 7198 default: 7199 ZEND_UNREACHABLE(); 7200 } 7201 } else if (smart_branch_opcode == ZEND_JMPNZ || 7202 smart_branch_opcode == ZEND_JMPNZ_EX) { 7203 switch (opline->opcode) { 7204 case ZEND_IS_EQUAL: 7205 case ZEND_CASE: 7206 if (exit_addr) { 7207 | beq &exit_addr 7208 } else { 7209 | beq => target_label 7210 } 7211 break; 7212 case ZEND_IS_NOT_EQUAL: 7213 if (exit_addr) { 7214 | bne &exit_addr 7215 } else { 7216 | bne => target_label 7217 } 7218 break; 7219 case ZEND_IS_SMALLER: 7220 if (exit_addr) { 7221 | blt &exit_addr 7222 } else { 7223 | blt => target_label 7224 } 7225 break; 7226 case ZEND_IS_SMALLER_OR_EQUAL: 7227 if (exit_addr) { 7228 | ble &exit_addr 7229 } else { 7230 | ble => target_label 7231 } 7232 break; 7233 default: 7234 ZEND_UNREACHABLE(); 7235 } 7236 } else { 7237 ZEND_UNREACHABLE(); 7238 } 7239 } else { 7240 switch (opline->opcode) { 7241 case ZEND_IS_EQUAL: 7242 case ZEND_CASE: 7243 | cset REG0w, eq 7244 break; 7245 case ZEND_IS_NOT_EQUAL: 7246 | cset REG0w, ne 7247 break; 7248 case ZEND_IS_SMALLER: 7249 | cset REG0w, lt 7250 break; 7251 case ZEND_IS_SMALLER_OR_EQUAL: 7252 | cset REG0w, le 7253 break; 7254 default: 7255 ZEND_UNREACHABLE(); 7256 } 7257 | add REG0w, REG0w, #2 7258 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7259 } 7260 7261 return 1; 7262} 7263 7264static int zend_jit_cmp(dasm_State **Dst, 7265 const zend_op *opline, 7266 uint32_t op1_info, 7267 zend_ssa_range *op1_range, 7268 zend_jit_addr op1_addr, 7269 uint32_t op2_info, 7270 zend_ssa_range *op2_range, 7271 zend_jit_addr op2_addr, 7272 zend_jit_addr res_addr, 7273 int may_throw, 7274 uint8_t smart_branch_opcode, 7275 uint32_t target_label, 7276 uint32_t target_label2, 7277 const void *exit_addr, 7278 bool skip_comparison) 7279{ 7280 bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); 7281 bool has_slow; 7282 7283 has_slow = 7284 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7285 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7286 ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7287 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); 7288 7289 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 7290 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 7291 if (op1_info & MAY_BE_DOUBLE) { 7292 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4, ZREG_TMP1 7293 } else { 7294 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 7295 } 7296 } 7297 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 7298 if (op2_info & MAY_BE_DOUBLE) { 7299 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 7300 |.cold_code 7301 |3: 7302 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7303 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7304 } 7305 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7306 return 0; 7307 } 7308 | b >6 7309 |.code 7310 } else { 7311 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 7312 } 7313 } 7314 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 7315 return 0; 7316 } 7317 if (op1_info & MAY_BE_DOUBLE) { 7318 |.cold_code 7319 |4: 7320 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7321 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 7322 } 7323 if (op2_info & MAY_BE_DOUBLE) { 7324 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7325 if (!same_ops) { 7326 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5, ZREG_TMP1 7327 } else { 7328 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7329 } 7330 } 7331 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7332 return 0; 7333 } 7334 | b >6 7335 } 7336 if (!same_ops) { 7337 |5: 7338 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7339 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 7340 } 7341 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7342 return 0; 7343 } 7344 | b >6 7345 } 7346 |.code 7347 } 7348 } else if ((op1_info & MAY_BE_DOUBLE) && 7349 !(op1_info & MAY_BE_LONG) && 7350 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7351 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7352 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 7353 } 7354 if (op2_info & MAY_BE_DOUBLE) { 7355 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7356 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7357 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3, ZREG_TMP1 7358 } else { 7359 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7360 } 7361 } 7362 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7363 return 0; 7364 } 7365 } 7366 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7367 if (op2_info & MAY_BE_DOUBLE) { 7368 |.cold_code 7369 } 7370 |3: 7371 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7372 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 7373 } 7374 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7375 return 0; 7376 } 7377 if (op2_info & MAY_BE_DOUBLE) { 7378 | b >6 7379 |.code 7380 } 7381 } 7382 } else if ((op2_info & MAY_BE_DOUBLE) && 7383 !(op2_info & MAY_BE_LONG) && 7384 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7385 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7386 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 7387 } 7388 if (op1_info & MAY_BE_DOUBLE) { 7389 if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7390 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7391 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3, ZREG_TMP1 7392 } else { 7393 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 7394 } 7395 } 7396 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7397 return 0; 7398 } 7399 } 7400 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7401 if (op1_info & MAY_BE_DOUBLE) { 7402 |.cold_code 7403 } 7404 |3: 7405 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7406 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 7407 } 7408 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7409 return 0; 7410 } 7411 if (op1_info & MAY_BE_DOUBLE) { 7412 | b >6 7413 |.code 7414 } 7415 } 7416 } 7417 7418 if (has_slow || 7419 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7420 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 7421 if (has_slow) { 7422 |.cold_code 7423 |9: 7424 } 7425 | SET_EX_OPLINE opline, REG0 7426 if (Z_MODE(op1_addr) == IS_REG) { 7427 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 7428 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 7429 return 0; 7430 } 7431 op1_addr = real_addr; 7432 } 7433 if (Z_MODE(op2_addr) == IS_REG) { 7434 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 7435 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 7436 return 0; 7437 } 7438 op2_addr = real_addr; 7439 } 7440 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7441 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { 7442 | IF_NOT_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w 7443 | LOAD_32BIT_VAL FCARG1x, opline->op1.var 7444 | EXT_CALL zend_jit_undefined_op_helper, REG0 7445 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 7446 |1: 7447 } 7448 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { 7449 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 7450 | str FCARG1x, T1 // save 7451 | LOAD_32BIT_VAL FCARG1x, opline->op2.var 7452 | EXT_CALL zend_jit_undefined_op_helper, REG0 7453 | ldr FCARG1x, T1 // restore 7454 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 7455 | b >2 7456 |1: 7457 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7458 |2: 7459 } else { 7460 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7461 } 7462 | EXT_CALL zend_compare, REG0 7463 if ((opline->opcode != ZEND_CASE && 7464 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7465 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 7466 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7467 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 7468 | str RETVALw, T1 // save 7469 if (opline->opcode != ZEND_CASE) { 7470 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 7471 } 7472 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 7473 | ldr RETVALw, T1 // restore 7474 } 7475 if (may_throw) { 7476 zend_jit_check_exception_undef_result(Dst, opline); 7477 } 7478 if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7479 return 0; 7480 } 7481 if (has_slow) { 7482 | b >6 7483 |.code 7484 } 7485 } 7486 7487 |6: 7488 7489 return 1; 7490} 7491 7492static int zend_jit_identical(dasm_State **Dst, 7493 const zend_op *opline, 7494 uint32_t op1_info, 7495 zend_ssa_range *op1_range, 7496 zend_jit_addr op1_addr, 7497 uint32_t op2_info, 7498 zend_ssa_range *op2_range, 7499 zend_jit_addr op2_addr, 7500 zend_jit_addr res_addr, 7501 int may_throw, 7502 uint8_t smart_branch_opcode, 7503 uint32_t target_label, 7504 uint32_t target_label2, 7505 const void *exit_addr, 7506 bool skip_comparison) 7507{ 7508 uint32_t identical_label = (uint32_t)-1; 7509 uint32_t not_identical_label = (uint32_t)-1; 7510 7511 if (smart_branch_opcode && !exit_addr) { 7512 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7513 if (smart_branch_opcode == ZEND_JMPZ) { 7514 not_identical_label = target_label; 7515 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7516 identical_label = target_label; 7517 } else { 7518 ZEND_UNREACHABLE(); 7519 } 7520 } else { 7521 if (smart_branch_opcode == ZEND_JMPZ) { 7522 identical_label = target_label; 7523 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7524 not_identical_label = target_label; 7525 } else { 7526 ZEND_UNREACHABLE(); 7527 } 7528 } 7529 } 7530 7531 if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && 7532 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { 7533 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 7534 return 0; 7535 } 7536 return 1; 7537 } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && 7538 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { 7539 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7540 return 0; 7541 } 7542 return 1; 7543 } 7544 7545 if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { 7546 op1_info |= MAY_BE_NULL; 7547 op2_info |= MAY_BE_NULL; 7548 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7549 | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w 7550 |.cold_code 7551 |1: 7552 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7553 | SET_EX_OPLINE opline, REG0 7554 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 7555 | EXT_CALL zend_jit_undefined_op_helper, REG0 7556 if (may_throw) { 7557 zend_jit_check_exception_undef_result(Dst, opline); 7558 } 7559 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 7560 | b >1 7561 |.code 7562 |1: 7563 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7564 | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w 7565 |.cold_code 7566 |1: 7567 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7568 | SET_EX_OPLINE opline, REG0 7569 | str FCARG1x, T1 // save 7570 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 7571 | EXT_CALL zend_jit_undefined_op_helper, REG0 7572 if (may_throw) { 7573 zend_jit_check_exception_undef_result(Dst, opline); 7574 } 7575 | ldr FCARG1x, T1 // restore 7576 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 7577 | b >1 7578 |.code 7579 |1: 7580 } else if (op1_info & MAY_BE_UNDEF) { 7581 op1_info |= MAY_BE_NULL; 7582 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7583 | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w 7584 |.cold_code 7585 |1: 7586 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7587 | SET_EX_OPLINE opline, REG0 7588 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 7589 | EXT_CALL zend_jit_undefined_op_helper, REG0 7590 if (may_throw) { 7591 zend_jit_check_exception_undef_result(Dst, opline); 7592 } 7593 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 7594 | b >1 7595 |.code 7596 |1: 7597 if (opline->op2_type != IS_CONST) { 7598 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7599 } 7600 } else if (op2_info & MAY_BE_UNDEF) { 7601 op2_info |= MAY_BE_NULL; 7602 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7603 | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w 7604 |.cold_code 7605 |1: 7606 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 7607 | SET_EX_OPLINE opline, REG0 7608 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 7609 | EXT_CALL zend_jit_undefined_op_helper, REG0 7610 if (may_throw) { 7611 zend_jit_check_exception_undef_result(Dst, opline); 7612 } 7613 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 7614 | b >1 7615 |.code 7616 |1: 7617 if (opline->op1_type != IS_CONST) { 7618 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7619 } 7620 } else if ((op1_info & op2_info & MAY_BE_ANY) != 0) { 7621 if (opline->op1_type != IS_CONST) { 7622 if (Z_MODE(op1_addr) == IS_REG) { 7623 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 7624 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 7625 return 0; 7626 } 7627 op1_addr = real_addr; 7628 } 7629 } 7630 if (opline->op2_type != IS_CONST) { 7631 if (Z_MODE(op2_addr) == IS_REG) { 7632 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 7633 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 7634 return 0; 7635 } 7636 op2_addr = real_addr; 7637 } 7638 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7639 } 7640 if (opline->op1_type != IS_CONST) { 7641 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7642 } 7643 } 7644 7645 if ((op1_info & op2_info & MAY_BE_ANY) == 0) { 7646 if ((opline->opcode != ZEND_CASE_STRICT && 7647 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7648 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 7649 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7650 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 7651 if (opline->opcode != ZEND_CASE_STRICT) { 7652 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7653 } 7654 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7655 } 7656 if (smart_branch_opcode) { 7657 if (may_throw) { 7658 zend_jit_check_exception_undef_result(Dst, opline); 7659 } 7660 if (exit_addr) { 7661 if (smart_branch_opcode == ZEND_JMPZ) { 7662 | b &exit_addr 7663 } 7664 } else if (not_identical_label != (uint32_t)-1) { 7665 | b =>not_identical_label 7666 } 7667 } else { 7668 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 7669 if (may_throw) { 7670 zend_jit_check_exception(Dst); 7671 } 7672 } 7673 return 1; 7674 } 7675 7676 if (opline->op1_type & (IS_CV|IS_VAR)) { 7677 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 7678 } 7679 if (opline->op2_type & (IS_CV|IS_VAR)) { 7680 | ZVAL_DEREF FCARG2x, op2_info, TMP1w 7681 } 7682 7683 if (has_concrete_type(op1_info) 7684 && has_concrete_type(op2_info) 7685 && concrete_type(op1_info) == concrete_type(op2_info) 7686 && concrete_type(op1_info) <= IS_TRUE) { 7687 if (smart_branch_opcode) { 7688 if (exit_addr) { 7689 if (smart_branch_opcode == ZEND_JMPNZ) { 7690 | b &exit_addr 7691 } 7692 } else if (identical_label != (uint32_t)-1) { 7693 | b =>identical_label 7694 } 7695 } else { 7696 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 7697 } 7698 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { 7699 if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { 7700 if (smart_branch_opcode) { 7701 if (exit_addr) { 7702 if (smart_branch_opcode == ZEND_JMPNZ) { 7703 | b &exit_addr 7704 } 7705 } else if (identical_label != (uint32_t)-1) { 7706 | b =>identical_label 7707 } 7708 } else { 7709 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 7710 } 7711 } else { 7712 if (smart_branch_opcode) { 7713 if (exit_addr) { 7714 if (smart_branch_opcode == ZEND_JMPZ) { 7715 | b &exit_addr 7716 } 7717 } else if (not_identical_label != (uint32_t)-1) { 7718 | b =>not_identical_label 7719 } 7720 } else { 7721 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 7722 } 7723 } 7724 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { 7725 zval *val = Z_ZV(op1_addr); 7726 7727 | ldrb TMP1w, [FCARG2x, #offsetof(zval, u1.v.type)] 7728 | cmp TMP1w, #Z_TYPE_P(val) 7729 if (smart_branch_opcode) { 7730 if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { 7731 | bne >8 7732 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7733 if (may_throw) { 7734 zend_jit_check_exception_undef_result(Dst, opline); 7735 } 7736 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7737 | b &exit_addr 7738 } else if (identical_label != (uint32_t)-1) { 7739 | b =>identical_label 7740 } else { 7741 | b >9 7742 } 7743 |8: 7744 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7745 | beq &exit_addr 7746 } else if (identical_label != (uint32_t)-1) { 7747 | beq =>identical_label 7748 } else { 7749 | beq >9 7750 } 7751 } else { 7752 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7753 | cset REG0w, eq 7754 } else { 7755 | cset REG0w, ne 7756 } 7757 | add REG0w, REG0w, #2 7758 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7759 } 7760 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7761 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 7762 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7763 if (may_throw) { 7764 zend_jit_check_exception_undef_result(Dst, opline); 7765 } 7766 } 7767 if (exit_addr) { 7768 if (smart_branch_opcode == ZEND_JMPZ) { 7769 | b &exit_addr 7770 } 7771 } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { 7772 | b =>not_identical_label 7773 } 7774 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { 7775 zval *val = Z_ZV(op2_addr); 7776 7777 | ldrb TMP1w, [FCARG1x, #offsetof(zval, u1.v.type)] 7778 | cmp TMP1w, #Z_TYPE_P(val) 7779 if (smart_branch_opcode) { 7780 if (opline->opcode != ZEND_CASE_STRICT 7781 && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { 7782 | bne >8 7783 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7784 if (may_throw) { 7785 zend_jit_check_exception_undef_result(Dst, opline); 7786 } 7787 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7788 | b &exit_addr 7789 } else if (identical_label != (uint32_t)-1) { 7790 | b =>identical_label 7791 } else { 7792 | b >9 7793 } 7794 |8: 7795 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 7796 | beq &exit_addr 7797 } else if (identical_label != (uint32_t)-1) { 7798 | beq =>identical_label 7799 } else { 7800 | beq >9 7801 } 7802 } else { 7803 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7804 | cset REG0w, eq 7805 } else { 7806 | cset REG0w, ne 7807 } 7808 | add REG0w, REG0w, #2 7809 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 7810 } 7811 if (opline->opcode != ZEND_CASE_STRICT 7812 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7813 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 7814 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7815 if (may_throw) { 7816 zend_jit_check_exception_undef_result(Dst, opline); 7817 } 7818 } 7819 if (smart_branch_opcode) { 7820 if (exit_addr) { 7821 if (smart_branch_opcode == ZEND_JMPZ) { 7822 | b &exit_addr 7823 } 7824 } else if (not_identical_label != (uint32_t)-1) { 7825 | b =>not_identical_label 7826 } 7827 } 7828 } else { 7829 if (opline->op1_type == IS_CONST) { 7830 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7831 } 7832 if (opline->op2_type == IS_CONST) { 7833 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 7834 } 7835 | EXT_CALL zend_is_identical, REG0 7836 if ((opline->opcode != ZEND_CASE_STRICT && 7837 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 7838 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 7839 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 7840 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 7841 | str RETVALw, T1 // save 7842 if (opline->opcode != ZEND_CASE_STRICT) { 7843 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7844 } 7845 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 7846 if (may_throw) { 7847 zend_jit_check_exception_undef_result(Dst, opline); 7848 } 7849 | ldr RETVALw, T1 // restore 7850 } 7851 if (smart_branch_opcode) { 7852 if (exit_addr) { 7853 if (smart_branch_opcode == ZEND_JMPNZ) { 7854 | cbnz RETVALw, &exit_addr 7855 } else { 7856 | cbz RETVALw, &exit_addr 7857 } 7858 } else if (not_identical_label != (uint32_t)-1) { 7859 | cbz RETVALw, =>not_identical_label 7860 if (identical_label != (uint32_t)-1) { 7861 | b =>identical_label 7862 } 7863 } else if (identical_label != (uint32_t)-1) { 7864 | cbnz RETVALw, =>identical_label 7865 } 7866 } else { 7867 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 7868 | add RETVALw, RETVALw, #2 7869 } else { 7870 | neg RETVALw, RETVALw 7871 | add RETVALw, RETVALw, #3 7872 } 7873 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, RETVALw, TMP1 7874 } 7875 } 7876 7877 |9: 7878 if (may_throw) { 7879 zend_jit_check_exception(Dst); 7880 } 7881 return 1; 7882} 7883 7884static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, uint8_t branch_opcode, const void *exit_addr) 7885{ 7886 uint32_t true_label = -1; 7887 uint32_t false_label = -1; 7888 bool set_bool = 0; 7889 bool set_bool_not = 0; 7890 bool set_delayed = 0; 7891 bool jmp_done = 0; 7892 7893 if (branch_opcode == ZEND_BOOL) { 7894 set_bool = 1; 7895 } else if (branch_opcode == ZEND_BOOL_NOT) { 7896 set_bool = 1; 7897 set_bool_not = 1; 7898 } else if (branch_opcode == ZEND_JMPZ) { 7899 false_label = target_label; 7900 } else if (branch_opcode == ZEND_JMPNZ) { 7901 true_label = target_label; 7902 } else if (branch_opcode == ZEND_JMPZ_EX) { 7903 set_bool = 1; 7904 false_label = target_label; 7905 } else if (branch_opcode == ZEND_JMPNZ_EX) { 7906 set_bool = 1; 7907 true_label = target_label; 7908 } else { 7909 ZEND_UNREACHABLE(); 7910 } 7911 7912 if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { 7913 if (zend_is_true(Z_ZV(op1_addr))) { 7914 /* Always TRUE */ 7915 if (set_bool) { 7916 if (set_bool_not) { 7917 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7918 } else { 7919 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7920 } 7921 } 7922 if (true_label != (uint32_t)-1) { 7923 | b =>true_label 7924 } 7925 } else { 7926 /* Always FALSE */ 7927 if (set_bool) { 7928 if (set_bool_not) { 7929 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7930 } else { 7931 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7932 } 7933 } 7934 if (false_label != (uint32_t)-1) { 7935 | b =>false_label 7936 } 7937 } 7938 return 1; 7939 } 7940 7941 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) { 7942 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 7943 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 7944 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 7945 } 7946 7947 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { 7948 if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { 7949 /* Always TRUE */ 7950 if (set_bool) { 7951 if (set_bool_not) { 7952 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7953 } else { 7954 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7955 } 7956 } 7957 if (true_label != (uint32_t)-1) { 7958 | b =>true_label 7959 } 7960 } else { 7961 if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { 7962 /* Always FALSE */ 7963 if (set_bool) { 7964 if (set_bool_not) { 7965 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7966 } else { 7967 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7968 } 7969 } 7970 } else { 7971 | CMP_ZVAL_TYPE op1_addr, IS_TRUE, ZREG_TMP1 7972 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 7973 if ((op1_info & MAY_BE_LONG) && 7974 !(op1_info & MAY_BE_UNDEF) && 7975 !set_bool) { 7976 if (exit_addr) { 7977 if (branch_opcode == ZEND_JMPNZ) { 7978 | blt >9 7979 } else { 7980 | blt &exit_addr 7981 } 7982 } else if (false_label != (uint32_t)-1) { 7983 | blt =>false_label 7984 } else { 7985 | blt >9 7986 } 7987 jmp_done = 1; 7988 } else { 7989 | bgt >2 7990 } 7991 } 7992 if (!(op1_info & MAY_BE_TRUE)) { 7993 /* It's FALSE */ 7994 if (set_bool) { 7995 if (set_bool_not) { 7996 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 7997 } else { 7998 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 7999 } 8000 } 8001 } else { 8002 if (exit_addr) { 8003 if (set_bool) { 8004 | bne >1 8005 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8006 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8007 | b &exit_addr 8008 } else { 8009 | b >9 8010 } 8011 |1: 8012 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8013 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8014 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8015 | bne &exit_addr 8016 } 8017 } 8018 } else { 8019 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8020 | beq &exit_addr 8021 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8022 | bne &exit_addr 8023 } else { 8024 | beq >9 8025 } 8026 } 8027 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8028 if (set_bool) { 8029 | bne >1 8030 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8031 if (true_label != (uint32_t)-1) { 8032 | b =>true_label 8033 } else { 8034 | b >9 8035 } 8036 |1: 8037 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8038 } else { 8039 if (true_label != (uint32_t)-1) { 8040 | beq =>true_label 8041 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8042 | bne =>false_label 8043 jmp_done = 1; 8044 } else { 8045 | beq >9 8046 } 8047 } 8048 } else if (set_bool) { 8049 | cset REG0w, eq 8050 if (set_bool_not) { 8051 | neg REG0w, REG0w 8052 | add REG0w, REG0w, #3 8053 } else { 8054 | add REG0w, REG0w, #2 8055 } 8056 if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { 8057 set_delayed = 1; 8058 } else { 8059 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8060 } 8061 } 8062 } 8063 } 8064 8065 /* It's FALSE, but may be UNDEF */ 8066 if (op1_info & MAY_BE_UNDEF) { 8067 if (op1_info & MAY_BE_ANY) { 8068 if (set_delayed) { 8069 | CMP_ZVAL_TYPE op1_addr, IS_UNDEF, ZREG_TMP1 8070 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8071 | beq >1 8072 } else { 8073 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 8074 } 8075 |.cold_code 8076 |1: 8077 } 8078 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 8079 | SET_EX_OPLINE opline, REG0 8080 | EXT_CALL zend_jit_undefined_op_helper, REG0 8081 8082 if (may_throw) { 8083 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 8084 return 0; 8085 } 8086 } 8087 8088 if (exit_addr) { 8089 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8090 | b &exit_addr 8091 } 8092 } else if (false_label != (uint32_t)-1) { 8093 | b =>false_label 8094 } 8095 if (op1_info & MAY_BE_ANY) { 8096 if (exit_addr) { 8097 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8098 | b >9 8099 } 8100 } else if (false_label == (uint32_t)-1) { 8101 | b >9 8102 } 8103 |.code 8104 } 8105 } 8106 8107 if (!jmp_done) { 8108 if (exit_addr) { 8109 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8110 if (op1_info & MAY_BE_LONG) { 8111 | b >9 8112 } 8113 } else if (op1_info & MAY_BE_LONG) { 8114 | b &exit_addr 8115 } 8116 } else if (false_label != (uint32_t)-1) { 8117 | b =>false_label 8118 } else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 8119 | b >9 8120 } 8121 } 8122 } 8123 } 8124 8125 if (op1_info & MAY_BE_LONG) { 8126 |2: 8127 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8128 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 8129 } 8130 if (Z_MODE(op1_addr) == IS_REG) { 8131 | tst Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) 8132 } else { 8133 | LONG_CMP_WITH_CONST op1_addr, Z_L(0), TMP1, TMP2 8134 } 8135 if (set_bool) { 8136 | cset REG0w, ne 8137 if (set_bool_not) { 8138 | neg REG0w, REG0w 8139 | add REG0w, REG0w, #3 8140 } else { 8141 | add REG0w, REG0w, #2 8142 } 8143 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8144 } 8145 if (exit_addr) { 8146 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8147 | bne &exit_addr 8148 } else { 8149 | beq &exit_addr 8150 } 8151 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8152 if (true_label != (uint32_t)-1) { 8153 | bne =>true_label 8154 if (false_label != (uint32_t)-1) { 8155 | b =>false_label 8156 } 8157 } else { 8158 | beq =>false_label 8159 } 8160 } 8161 } 8162 8163 if ((op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) == MAY_BE_DOUBLE) { 8164 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8165 |.cold_code 8166 } 8167 |2: 8168 | fmov FPR0, xzr // TODO: "movi d0, #0" is not recognized by DynASM/arm64 8169 | DOUBLE_CMP ZREG_FPR0, op1_addr, ZREG_TMP1, ZREG_FPTMP 8170 8171 if (set_bool) { 8172 if (exit_addr) { 8173 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8174 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8175 | bvs &exit_addr 8176 | bne &exit_addr 8177 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8178 } else { 8179 | bvs >1 8180 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8181 | beq &exit_addr 8182 |1: 8183 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8184 } 8185 } else if (false_label != (uint32_t)-1) { // JMPZ_EX 8186 | bvs >1 8187 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8188 | beq => false_label 8189 |1: 8190 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8191 } else if (true_label != (uint32_t)-1) { // JMPNZ_EX 8192 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 8193 | bvs => true_label 8194 | bne => true_label 8195 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 8196 } else if (set_bool_not) { // BOOL_NOT 8197 | mov REG0w, #IS_FALSE 8198 | bvs >1 8199 | bne >1 8200 | mov REG0w, #IS_TRUE 8201 |1: 8202 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8203 } else { // BOOL 8204 | mov REG0w, #IS_TRUE 8205 | bvs >1 8206 | bne >1 8207 | mov REG0w, #IS_FALSE 8208 |1: 8209 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8210 } 8211 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8212 | b >9 8213 |.code 8214 } 8215 } else { 8216 if (exit_addr) { 8217 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8218 | bvs &exit_addr 8219 | bne &exit_addr 8220 |1: 8221 } else { 8222 | bvs >1 8223 | beq &exit_addr 8224 |1: 8225 } 8226 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8227 | b >9 8228 } 8229 } else { 8230 ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); 8231 if (false_label != (uint32_t)-1 ) { 8232 | bvs >1 8233 | beq => false_label 8234 |1: 8235 if (true_label != (uint32_t)-1) { 8236 | b =>true_label 8237 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8238 | b >9 8239 } 8240 } else { 8241 | bvs => true_label 8242 | bne => true_label 8243 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8244 | b >9 8245 } 8246 } 8247 } 8248 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8249 |.code 8250 } 8251 } 8252 } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8253 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8254 |.cold_code 8255 |2: 8256 } 8257 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8258 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8259 } 8260 | SET_EX_OPLINE opline, REG0 8261 | EXT_CALL zend_is_true, REG0 8262 | mov REG0, RETVALx 8263 8264 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8265 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8266 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8267 8268 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8269 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3, ZREG_TMP1, ZREG_TMP2 8270 } 8271 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 8272 | GC_DELREF FCARG1x, TMP1w 8273 | bne >3 8274 // In x86, r0 is used in macro ZVAL_DTOR_FUNC as temporary register, hence, r0 should be saved/restored 8275 // before/after this macro. In AArch64, TMP1 is used, but we still have to store REG0, 8276 // because it's clobbered by function call. 8277 | str REG0, T1 // save 8278 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 8279 | ldr REG0, T1 // restore 8280 |3: 8281 } 8282 if (may_throw) { 8283 | MEM_LOAD_64_ZTS ldr, REG1, executor_globals, exception, TMP1 8284 | cbnz REG1, ->exception_handler_undef 8285 } 8286 8287 if (set_bool) { 8288 if (set_bool_not) { 8289 | neg REG0w, REG0w 8290 | add REG0w, REG0w, #3 8291 } else { 8292 | add REG0w, REG0w, #2 8293 } 8294 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 8295 if (exit_addr) { 8296 | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 8297 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8298 | bne &exit_addr 8299 } else { 8300 | beq &exit_addr 8301 } 8302 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8303 | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 8304 if (true_label != (uint32_t)-1) { 8305 | bne =>true_label 8306 if (false_label != (uint32_t)-1) { 8307 | b =>false_label 8308 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8309 | b >9 8310 } 8311 } else { 8312 | beq =>false_label 8313 } 8314 } 8315 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8316 | b >9 8317 |.code 8318 } 8319 } else { 8320 if (exit_addr) { 8321 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8322 | cbnz REG0w, &exit_addr 8323 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8324 | b >9 8325 } 8326 } else { 8327 | cbz REG0w, &exit_addr 8328 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8329 | b >9 8330 } 8331 } 8332 } else if (true_label != (uint32_t)-1) { 8333 | cbnz REG0w, =>true_label 8334 if (false_label != (uint32_t)-1) { 8335 | b =>false_label 8336 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8337 | b >9 8338 } 8339 } else { 8340 | cbz REG0w, =>false_label 8341 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8342 | b >9 8343 } 8344 } 8345 8346 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8347 |.code 8348 } 8349 } 8350 } 8351 8352 |9: 8353 8354 return 1; 8355} 8356 8357static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) 8358{ 8359 if (op1_addr != op1_def_addr) { 8360 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 8361 return 0; 8362 } 8363 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 8364 op1_addr = op1_def_addr; 8365 } 8366 } 8367 8368 if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0, 1)) { 8369 return 0; 8370 } 8371 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8372 return 0; 8373 } 8374 if (op1_info & MAY_BE_UNDEF) { 8375 zend_jit_check_exception(Dst); 8376 } 8377 return 1; 8378} 8379 8380static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 8381{ 8382 ZEND_ASSERT(opline->op1_type == IS_CV); 8383 8384 if (op2_addr != op2_def_addr) { 8385 if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { 8386 return 0; 8387 } 8388 if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { 8389 op2_addr = op2_def_addr; 8390 } 8391 } 8392 8393 if (Z_MODE(op1_addr) != IS_REG 8394 && Z_MODE(op1_use_addr) == IS_REG 8395 && !Z_LOAD(op1_use_addr) 8396 && !Z_STORE(op1_use_addr)) { 8397 /* Force type update */ 8398 op1_info |= MAY_BE_UNDEF; 8399 } 8400 if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, 8401 may_throw)) { 8402 return 0; 8403 } 8404 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { 8405 return 0; 8406 } 8407 if (opline->result_type != IS_UNUSED) { 8408 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8409 return 0; 8410 } 8411 } 8412 8413 return 1; 8414} 8415 8416/* copy of hidden zend_closure */ 8417typedef struct _zend_closure { 8418 zend_object std; 8419 zend_function func; 8420 zval this_ptr; 8421 zend_class_entry *called_scope; 8422 zif_handler orig_internal_handler; 8423} zend_closure; 8424 8425static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) 8426{ 8427 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8428 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8429 8430 if (!exit_addr) { 8431 return 0; 8432 } 8433 8434 | // Check Stack Overflow 8435 | MEM_LOAD_64_ZTS ldr, REG1, executor_globals, vm_stack_end, TMP1 8436 | MEM_LOAD_OP_ZTS sub, ldr, REG1, executor_globals, vm_stack_top, TMP1, TMP2 8437 | CMP_64_WITH_CONST_32 REG1, used_stack, TMP1 8438 | blo &exit_addr 8439 8440 return 1; 8441} 8442 8443static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack) 8444{ 8445 uint32_t used_stack; 8446 bool stack_check = 1; 8447 8448 // REG0 -> zend_function 8449 // FCARG1 -> used_stack 8450 8451 if (func) { 8452 used_stack = zend_vm_calc_used_stack(opline->extended_value, func); 8453 if ((int)used_stack <= checked_stack) { 8454 stack_check = 0; 8455 } 8456 } else { 8457 used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value + ZEND_OBSERVER_ENABLED) * sizeof(zval); 8458 8459 | // if (EXPECTED(ZEND_USER_CODE(func->type))) { 8460 if (!is_closure) { 8461 | LOAD_32BIT_VAL FCARG1w, used_stack 8462 | // Check whether REG0 is an internal function. 8463 | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] 8464 | TST_32_WITH_CONST TMP1w, 1, TMP2w 8465 | bne >1 8466 } else { 8467 | LOAD_32BIT_VAL FCARG1w, used_stack 8468 } 8469 | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); 8470 | LOAD_32BIT_VAL REG2w, opline->extended_value 8471 if (!is_closure) { 8472 | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.num_args)] 8473 | cmp REG2w, TMP1w 8474 | csel REG2w, REG2w, TMP1w, le 8475 | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.last_var)] 8476 | sub REG2w, REG2w, TMP1w 8477 | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.T)] 8478 | sub REG2w, REG2w, TMP1w 8479 } else { 8480 | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.num_args)] 8481 | cmp REG2w, TMP1w 8482 | csel REG2w, REG2w, TMP1w, le 8483 | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.last_var)] 8484 | sub REG2w, REG2w, TMP1w 8485 | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.T)] 8486 | sub REG2w, REG2w, TMP1w 8487 } 8488 | sxtw REG2, REG2w 8489 | sub FCARG1x, FCARG1x, REG2, lsl #4 8490 |1: 8491 } 8492 8493 zend_jit_start_reuse_ip(); 8494 8495 | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { 8496 | MEM_LOAD_64_ZTS ldr, RX, executor_globals, vm_stack_top, TMP1 8497 8498 if (stack_check) { 8499 | // Check Stack Overflow 8500 | MEM_LOAD_64_ZTS ldr, REG2, executor_globals, vm_stack_end, TMP1 8501 | sub REG2, REG2, RX 8502 if (func) { 8503 | CMP_64_WITH_CONST_32 REG2, used_stack, TMP1 8504 } else { 8505 | cmp REG2, FCARG1x 8506 } 8507 8508 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8509 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8510 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8511 8512 if (!exit_addr) { 8513 return 0; 8514 } 8515 8516 | blo &exit_addr 8517 } else { 8518 | blo >1 8519 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); 8520 |.cold_code 8521 |1: 8522 if (func) { 8523 | LOAD_32BIT_VAL FCARG1w, used_stack 8524 } 8525 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 8526 | SET_EX_OPLINE opline, REG0 8527 | EXT_CALL zend_jit_int_extend_stack_helper, REG0 8528 } else { 8529 if (!is_closure) { 8530 | mov FCARG2x, REG0 8531 } else { 8532 | add FCARG2x, REG0, #offsetof(zend_closure, func) 8533 } 8534 | SET_EX_OPLINE opline, REG0 8535 | EXT_CALL zend_jit_extend_stack_helper, REG0 8536 } 8537 | mov RX, RETVALx 8538 | b >1 8539 |.code 8540 } 8541 } 8542 8543 if (func) { 8544 || if (arm64_may_encode_imm12((int64_t)used_stack)) { 8545 | MEM_UPDATE_ZTS add, ldr, str, #used_stack, executor_globals, vm_stack_top, REG2, TMP1 8546 || } else { 8547 | LOAD_32BIT_VAL TMP1w, used_stack 8548 | MEM_UPDATE_ZTS add, ldr, str, TMP1, executor_globals, vm_stack_top, REG2, TMP2 8549 || } 8550 } else { 8551 | MEM_UPDATE_ZTS add, ldr, str, FCARG1x, executor_globals, vm_stack_top, REG2, TMP1 8552 } 8553 | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); 8554 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { 8555 | // ZEND_SET_CALL_INFO(call, 0, call_info); 8556 | LOAD_32BIT_VAL TMP1w, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) 8557 | str TMP1w, EX:RX->This.u1.type_info 8558 } 8559 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 8560 | // call->func = func; 8561 |1: 8562 | ADDR_STORE EX:RX->func, func, REG1 8563 } else { 8564 if (!is_closure) { 8565 | // call->func = func; 8566 | str REG0, EX:RX->func 8567 } else { 8568 | // call->func = &closure->func; 8569 | add REG1, REG0, #offsetof(zend_closure, func) 8570 | str REG1, EX:RX->func 8571 } 8572 |1: 8573 } 8574 if (opline->opcode == ZEND_INIT_METHOD_CALL) { 8575 | // Z_PTR(call->This) = obj; 8576 | ldr REG1, T1 8577 | str REG1, EX:RX->This.value.ptr 8578 if (opline->op1_type == IS_UNUSED || delayed_fetch_this) { 8579 | // call->call_info |= ZEND_CALL_HAS_THIS; 8580 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8581 | LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS 8582 | str TMP1w, EX:RX->This.u1.type_info 8583 } else { 8584 | ldr TMP1w, EX:RX->This.u1.type_info 8585 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_HAS_THIS, TMP2w 8586 | str TMP1w, EX:RX->This.u1.type_info 8587 } 8588 } else { 8589 if (opline->op1_type == IS_CV) { 8590 | // GC_ADDREF(obj); 8591 | GC_ADDREF REG1, TMP1w 8592 } 8593 | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; 8594 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8595 | LOAD_32BIT_VAL TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 8596 | str TMP1w, EX:RX->This.u1.type_info 8597 } else { 8598 | ldr TMP1w, EX:RX->This.u1.type_info 8599 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS), TMP2w 8600 | str TMP1w, EX:RX->This.u1.type_info 8601 } 8602 } 8603 } else if (!is_closure) { 8604 | // Z_CE(call->This) = called_scope; 8605 | str xzr, EX:RX->This.value.ptr 8606 } else { 8607 if (opline->op2_type == IS_CV) { 8608 | // GC_ADDREF(closure); 8609 | GC_ADDREF REG0, TMP1w 8610 } 8611 | // object_or_called_scope = closure->called_scope; 8612 | ldr REG1, [REG0, #offsetof(zend_closure, called_scope)] 8613 | str REG1, EX:RX->This.value.ptr 8614 | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | 8615 | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); 8616 | ldr REG2w, [REG0, #offsetof(zend_closure, func.common.fn_flags)] 8617 | BW_OP_32_WITH_CONST and, REG2w, REG2w, ZEND_ACC_FAKE_CLOSURE, TMP1w 8618 | BW_OP_32_WITH_CONST orr, REG2w, REG2w, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE), TMP1w 8619 | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { 8620 | ldrb TMP1w, [REG0, #offsetof(zend_closure, this_ptr.u1.v.type)] 8621 | cmp TMP1w, #IS_UNDEF 8622 | beq >1 8623 | // call_info |= ZEND_CALL_HAS_THIS; 8624 | BW_OP_32_WITH_CONST orr, REG2w, REG2w, ZEND_CALL_HAS_THIS, TMP1w 8625 | // object_or_called_scope = Z_OBJ(closure->this_ptr); 8626 | ldr REG1, [REG0, #offsetof(zend_closure, this_ptr.value.ptr)] 8627 |1: 8628 | // ZEND_SET_CALL_INFO(call, 0, call_info); 8629 | ldr TMP1w, EX:RX->This.u1.type_info 8630 | orr TMP1w, TMP1w, REG2w 8631 | str TMP1w, EX:RX->This.u1.type_info 8632 | // Z_PTR(call->This) = object_or_called_scope; 8633 | str REG1, EX:RX->This.value.ptr 8634 | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.run_time_cache__ptr)] 8635 | cbnz TMP1, >1 8636 | add FCARG1x, REG0, #offsetof(zend_closure, func) 8637 | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 8638 |1: 8639 } 8640 | // ZEND_CALL_NUM_ARGS(call) = num_args; 8641 | LOAD_32BIT_VAL TMP1w, opline->extended_value 8642 | str TMP1w, EX:RX->This.u2.num_args 8643 8644 return 1; 8645} 8646 8647static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) 8648{ 8649 int32_t exit_point; 8650 const void *exit_addr; 8651 8652 if (func->type == ZEND_INTERNAL_FUNCTION) { 8653#ifdef ZEND_WIN32 8654 // TODO: ASLR may cause different addresses in different workers ??? 8655 return 0; 8656#endif 8657 } else if (func->type == ZEND_USER_FUNCTION) { 8658 if (!zend_accel_in_shm(func->op_array.opcodes)) { 8659 /* op_array and op_array->opcodes are not persistent. We can't link. */ 8660 return 0; 8661 } 8662 } else { 8663 ZEND_UNREACHABLE(); 8664 return 0; 8665 } 8666 8667 exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); 8668 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8669 if (!exit_addr) { 8670 return 0; 8671 } 8672 8673 | // call = EX(call); 8674 | ldr REG1, EX->call 8675 while (level > 0) { 8676 | ldr REG1, EX:REG1->prev_execute_data 8677 level--; 8678 } 8679 8680 if (func->type == ZEND_USER_FUNCTION && 8681 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 8682 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 8683 !func->common.function_name)) { 8684 const zend_op *opcodes = func->op_array.opcodes; 8685 8686 | ldr REG1, EX:REG1->func 8687 | LOAD_ADDR REG2, ((ptrdiff_t)opcodes) 8688 | ldr TMP1, [REG1, #offsetof(zend_op_array, opcodes)] 8689 | cmp TMP1, REG2 8690 | bne &exit_addr 8691 } else { 8692 | LOAD_ADDR REG2, ((ptrdiff_t)func) 8693 | ldr TMP1, EX:REG1->func 8694 | cmp TMP1, REG2 8695 | bne &exit_addr 8696 } 8697 8698 return 1; 8699} 8700 8701static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack) 8702{ 8703 zend_func_info *info = ZEND_FUNC_INFO(op_array); 8704 zend_call_info *call_info = NULL; 8705 zend_function *func = NULL; 8706 8707 if (delayed_call_chain) { 8708 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 8709 return 0; 8710 } 8711 } 8712 8713 if (info) { 8714 call_info = info->callee_info; 8715 while (call_info && call_info->caller_init_opline != opline) { 8716 call_info = call_info->next_callee; 8717 } 8718 if (call_info && call_info->callee_func && !call_info->is_prototype) { 8719 func = call_info->callee_func; 8720 } 8721 } 8722 8723 if (!func 8724 && trace 8725 && trace->op == ZEND_JIT_TRACE_INIT_CALL) { 8726 func = (zend_function*)trace->func; 8727 } 8728 8729 if (opline->opcode == ZEND_INIT_FCALL 8730 && func 8731 && func->type == ZEND_INTERNAL_FUNCTION) { 8732 /* load constant address later */ 8733 } else if (func && op_array == &func->op_array) { 8734 /* recursive call */ 8735 | ldr REG0, EX->func 8736 } else { 8737 | // if (CACHED_PTR(opline->result.num)) 8738 | ldr REG2, EX->run_time_cache 8739 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG2, opline->result.num, TMP1 8740 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 8741 && func 8742 && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) 8743 && opline->opcode != ZEND_INIT_FCALL) { 8744 /* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */ 8745 | LOAD_ADDR REG1, ((ptrdiff_t)func) 8746 | cmp REG0, REG1 8747 | bne >1 8748 } else { 8749 | cbz REG0, >1 8750 } 8751 |.cold_code 8752 |1: 8753 if (opline->opcode == ZEND_INIT_FCALL 8754 && func 8755 && func->type == ZEND_USER_FUNCTION 8756 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { 8757 | LOAD_ADDR FCARG1x, func 8758 | MEM_ACCESS_64_WITH_UOFFSET str, FCARG1x, REG2, opline->result.num, TMP1 8759 | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 8760 | mov REG0, RETVALx 8761 | b >3 8762 } else { 8763 zval *zv = RT_CONSTANT(opline, opline->op2); 8764 8765 if (opline->opcode == ZEND_INIT_FCALL) { 8766 | LOAD_ADDR FCARG1x, Z_STR_P(zv); 8767 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 8768 | EXT_CALL zend_jit_find_func_helper, REG0 8769 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { 8770 | LOAD_ADDR FCARG1x, Z_STR_P(zv + 1); 8771 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 8772 | EXT_CALL zend_jit_find_func_helper, REG0 8773 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { 8774 | LOAD_ADDR FCARG1x, zv; 8775 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 8776 | EXT_CALL zend_jit_find_ns_func_helper, REG0 8777 } else { 8778 ZEND_UNREACHABLE(); 8779 } 8780 | // Get the return value of function zend_jit_find_func_helper/zend_jit_find_ns_func_helper 8781 | mov REG0, RETVALx 8782 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8783 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 8784 func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0); 8785 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8786 8787 if (!exit_addr) { 8788 return 0; 8789 } 8790 8791 if (!func || opline->opcode == ZEND_INIT_FCALL) { 8792 | cbnz REG0, >3 8793 } else if (func->type == ZEND_USER_FUNCTION 8794 && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { 8795 const zend_op *opcodes = func->op_array.opcodes; 8796 8797 | LOAD_ADDR REG1, ((ptrdiff_t)opcodes) 8798 | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] 8799 | cmp TMP1, REG1 8800 | beq >3 8801 } else { 8802 | LOAD_ADDR REG1, ((ptrdiff_t)func) 8803 | cmp REG0, REG1 8804 | beq >3 8805 } 8806 | b &exit_addr 8807 } else { 8808 | cbnz REG0, >3 8809 | // SAVE_OPLINE(); 8810 | SET_EX_OPLINE opline, REG0 8811 | b ->undefined_function 8812 } 8813 } 8814 |.code 8815 |3: 8816 } 8817 8818 if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, checked_stack)) { 8819 return 0; 8820 } 8821 8822 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 8823 if (!zend_jit_save_call_chain(Dst, call_level)) { 8824 return 0; 8825 } 8826 } else { 8827 delayed_call_chain = 1; 8828 delayed_call_level = call_level; 8829 } 8830 8831 return 1; 8832} 8833 8834static int zend_jit_init_method_call(dasm_State **Dst, 8835 const zend_op *opline, 8836 uint32_t b, 8837 const zend_op_array *op_array, 8838 zend_ssa *ssa, 8839 const zend_ssa_op *ssa_op, 8840 int call_level, 8841 uint32_t op1_info, 8842 zend_jit_addr op1_addr, 8843 zend_class_entry *ce, 8844 bool ce_is_instanceof, 8845 bool on_this, 8846 bool delayed_fetch_this, 8847 zend_class_entry *trace_ce, 8848 zend_jit_trace_rec *trace, 8849 int checked_stack, 8850 bool polymorphic_side_trace) 8851{ 8852 zend_func_info *info = ZEND_FUNC_INFO(op_array); 8853 zend_call_info *call_info = NULL; 8854 zend_function *func = NULL; 8855 zval *function_name; 8856 8857 ZEND_ASSERT(opline->op2_type == IS_CONST); 8858 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 8859 8860 function_name = RT_CONSTANT(opline, opline->op2); 8861 8862 if (info) { 8863 call_info = info->callee_info; 8864 while (call_info && call_info->caller_init_opline != opline) { 8865 call_info = call_info->next_callee; 8866 } 8867 if (call_info && call_info->callee_func && !call_info->is_prototype) { 8868 func = call_info->callee_func; 8869 } 8870 } 8871 8872 if (polymorphic_side_trace) { 8873 /* function is passed in r0 from parent_trace */ 8874 } else { 8875 if (on_this) { 8876 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 8877 8878 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 8879 } else { 8880 if (op1_info & MAY_BE_REF) { 8881 if (opline->op1_type == IS_CV) { 8882 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8883 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8884 } 8885 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 8886 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 8887 } else { 8888 /* Hack: Convert reference to regular value to simplify JIT code */ 8889 ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); 8890 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 8891 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8892 | EXT_CALL zend_jit_unref_helper, REG0 8893 |1: 8894 } 8895 } 8896 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 8897 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 8898 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 8899 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8900 8901 if (!exit_addr) { 8902 return 0; 8903 } 8904 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 8905 } else { 8906 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 8907 |.cold_code 8908 |1: 8909 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8910 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 8911 } 8912 | SET_EX_OPLINE opline, REG0 8913 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 8914 | EXT_CALL zend_jit_invalid_method_call_tmp, REG0 8915 } else { 8916 | EXT_CALL zend_jit_invalid_method_call, REG0 8917 } 8918 | b ->exception_handler 8919 |.code 8920 } 8921 } 8922 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 8923 } 8924 8925 if (delayed_call_chain) { 8926 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 8927 return 0; 8928 } 8929 } 8930 8931 | str FCARG1x, T1 // save 8932 8933 if (func) { 8934 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 8935 | ldr REG0, EX->run_time_cache 8936 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 8937 | cbz REG0, >1 8938 } else { 8939 | // if (CACHED_PTR(opline->result.num) == obj->ce)) { 8940 | ldr REG0, EX->run_time_cache 8941 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->result.num, TMP1 8942 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 8943 | cmp REG2, TMP1 8944 | bne >1 8945 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 8946 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 8947 } 8948 8949 |.cold_code 8950 |1: 8951 | LOAD_ADDR FCARG2x, function_name 8952 if (TMP_ZVAL_OFFSET == 0) { 8953 | mov CARG3, sp 8954 } else { 8955 | add CARG3, sp, #TMP_ZVAL_OFFSET 8956 } 8957 | SET_EX_OPLINE opline, REG0 8958 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 8959 | EXT_CALL zend_jit_find_method_tmp_helper, REG0 8960 } else { 8961 | EXT_CALL zend_jit_find_method_helper, REG0 8962 } 8963 | mov REG0, RETVALx 8964 | cbnz REG0, >2 8965 | b ->exception_handler 8966 |.code 8967 |2: 8968 } 8969 8970 if ((!func || zend_jit_may_be_modified(func, op_array)) 8971 && trace 8972 && trace->op == ZEND_JIT_TRACE_INIT_CALL 8973 && trace->func 8974 ) { 8975 int32_t exit_point; 8976 const void *exit_addr; 8977 8978 exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); 8979 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 8980 if (!exit_addr) { 8981 return 0; 8982 } 8983 8984 func = (zend_function*)trace->func; 8985 8986 if (func->type == ZEND_USER_FUNCTION && 8987 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 8988 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 8989 !func->common.function_name)) { 8990 const zend_op *opcodes = func->op_array.opcodes; 8991 8992 | LOAD_ADDR TMP1, opcodes 8993 | ldr TMP2, [REG0, #offsetof(zend_op_array, opcodes)] 8994 | cmp TMP2, TMP1 8995 | bne &exit_addr 8996 } else { 8997 | LOAD_ADDR TMP1, func 8998 | cmp REG0, TMP1 8999 | bne &exit_addr 9000 } 9001 } 9002 9003 if (!func) { 9004 | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { 9005 | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] 9006 | TST_32_WITH_CONST TMP1w, ZEND_ACC_STATIC, TMP2w 9007 | bne >1 9008 |.cold_code 9009 |1: 9010 } 9011 9012 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { 9013 | ldr FCARG1x, T1 // restore 9014 | mov FCARG2x, REG0 9015 | LOAD_32BIT_VAL CARG3w, opline->extended_value 9016 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9017 | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, REG0 9018 } else { 9019 | EXT_CALL zend_jit_push_static_metod_call_frame, REG0 9020 } 9021 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) { 9022 | cbz RETVALx, ->exception_handler 9023 } 9024 | mov RX, RETVALx 9025 } 9026 9027 if (!func) { 9028 | b >9 9029 |.code 9030 } 9031 9032 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { 9033 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, delayed_fetch_this, checked_stack)) { 9034 return 0; 9035 } 9036 } 9037 9038 if (!func) { 9039 |9: 9040 } 9041 zend_jit_start_reuse_ip(); 9042 9043 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9044 if (!zend_jit_save_call_chain(Dst, call_level)) { 9045 return 0; 9046 } 9047 } else { 9048 delayed_call_chain = 1; 9049 delayed_call_level = call_level; 9050 } 9051 9052 return 1; 9053} 9054 9055static int zend_jit_init_closure_call(dasm_State **Dst, 9056 const zend_op *opline, 9057 uint32_t b, 9058 const zend_op_array *op_array, 9059 zend_ssa *ssa, 9060 const zend_ssa_op *ssa_op, 9061 int call_level, 9062 zend_jit_trace_rec *trace, 9063 int checked_stack) 9064{ 9065 zend_function *func = NULL; 9066 zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 9067 9068 | GET_ZVAL_PTR REG0, op2_addr, TMP1 9069 9070 if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure 9071 && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { 9072 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9073 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9074 9075 if (!exit_addr) { 9076 return 0; 9077 } 9078 9079 | LOAD_ADDR FCARG1x, ((ptrdiff_t)zend_ce_closure) 9080 | ldr, TMP1, [REG0, #offsetof(zend_object, ce)] 9081 | cmp TMP1, FCARG1x 9082 | bne &exit_addr 9083 if (ssa->var_info && ssa_op->op2_use >= 0) { 9084 ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; 9085 ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; 9086 ssa->var_info[ssa_op->op2_use].is_instanceof = 0; 9087 } 9088 } 9089 9090 if (trace 9091 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9092 && trace->func 9093 && trace->func->type == ZEND_USER_FUNCTION) { 9094 const zend_op *opcodes; 9095 int32_t exit_point; 9096 const void *exit_addr; 9097 9098 func = (zend_function*)trace->func; 9099 opcodes = func->op_array.opcodes; 9100 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); 9101 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9102 if (!exit_addr) { 9103 return 0; 9104 } 9105 9106 | LOAD_ADDR FCARG1x, ((ptrdiff_t)opcodes) 9107 | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.opcodes)] 9108 | cmp TMP1, FCARG1x 9109 | bne &exit_addr 9110 } 9111 9112 if (delayed_call_chain) { 9113 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9114 return 0; 9115 } 9116 } 9117 9118 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, checked_stack)) { 9119 return 0; 9120 } 9121 9122 if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9123 if (!zend_jit_save_call_chain(Dst, call_level)) { 9124 return 0; 9125 } 9126 } else { 9127 delayed_call_chain = 1; 9128 delayed_call_level = call_level; 9129 } 9130 9131 if (trace 9132 && trace->op == ZEND_JIT_TRACE_END 9133 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 9134 if (!zend_jit_set_valid_ip(Dst, opline + 1)) { 9135 return 0; 9136 } 9137 } 9138 9139 return 1; 9140} 9141 9142static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) 9143{ 9144 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9145 zend_call_info *call_info = NULL; 9146 const zend_function *func = NULL; 9147 uint32_t i; 9148 zend_jit_addr res_addr; 9149 uint32_t call_num_args = 0; 9150 bool unknown_num_args = 0; 9151 const void *exit_addr = NULL; 9152 const zend_op *prev_opline; 9153 9154 if (RETURN_VALUE_USED(opline)) { 9155 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 9156 } else { 9157 /* CPU stack allocated temporary zval */ 9158 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RSP, TMP_ZVAL_OFFSET); 9159 } 9160 9161 prev_opline = opline - 1; 9162 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { 9163 prev_opline--; 9164 } 9165 if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || 9166 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 9167 unknown_num_args = 1; 9168 } 9169 9170 if (info) { 9171 call_info = info->callee_info; 9172 while (call_info && call_info->caller_call_opline != opline) { 9173 call_info = call_info->next_callee; 9174 } 9175 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9176 func = call_info->callee_func; 9177 } 9178 if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) 9179 && JIT_G(current_frame) 9180 && JIT_G(current_frame)->call 9181 && !JIT_G(current_frame)->call->func) { 9182 call_info = NULL; func = NULL; /* megamorphic call from trait */ 9183 } 9184 } 9185 if (!func) { 9186 /* resolve function at run time */ 9187 } else if (func->type == ZEND_USER_FUNCTION) { 9188 ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); 9189 call_num_args = call_info->num_args; 9190 } else if (func->type == ZEND_INTERNAL_FUNCTION) { 9191 ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); 9192 call_num_args = call_info->num_args; 9193 } else { 9194 ZEND_UNREACHABLE(); 9195 } 9196 9197 if (trace && !func) { 9198 if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { 9199 ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); 9200#ifndef ZEND_WIN32 9201 // TODO: ASLR may cause different addresses in different workers ??? 9202 func = trace->func; 9203 if (JIT_G(current_frame) && 9204 JIT_G(current_frame)->call && 9205 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9206 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9207 } else { 9208 unknown_num_args = 1; 9209 } 9210#endif 9211 } else if (trace->op == ZEND_JIT_TRACE_ENTER) { 9212 ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); 9213 if (zend_accel_in_shm(trace->func->op_array.opcodes)) { 9214 func = trace->func; 9215 if (JIT_G(current_frame) && 9216 JIT_G(current_frame)->call && 9217 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9218 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9219 } else { 9220 unknown_num_args = 1; 9221 } 9222 } 9223 } 9224 } 9225 9226 bool may_have_extra_named_params = 9227 opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && 9228 (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); 9229 9230 if (!reuse_ip) { 9231 zend_jit_start_reuse_ip(); 9232 | // call = EX(call); 9233 | ldr RX, EX->call 9234 } 9235 zend_jit_stop_reuse_ip(); 9236 9237 | // fbc = call->func; 9238 | // mov r2, EX:RX->func ??? 9239 | // SAVE_OPLINE(); 9240 | SET_EX_OPLINE opline, REG0 9241 9242 if (opline->opcode == ZEND_DO_FCALL) { 9243 if (!func) { 9244 if (trace) { 9245 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9246 9247 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9248 if (!exit_addr) { 9249 return 0; 9250 } 9251 | ldr REG0, EX:RX->func 9252 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9253 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9254 | bne &exit_addr 9255 } 9256 } 9257 } 9258 9259 if (!delayed_call_chain) { 9260 if (call_level == 1) { 9261 | str xzr, EX->call 9262 } else { 9263 | //EX(call) = call->prev_execute_data; 9264 | ldr REG0, EX:RX->prev_execute_data 9265 | str REG0, EX->call 9266 } 9267 } 9268 delayed_call_chain = 0; 9269 9270 | //call->prev_execute_data = execute_data; 9271 | str EX, EX:RX->prev_execute_data 9272 9273 if (!func) { 9274 | ldr REG0, EX:RX->func 9275 } 9276 9277 if (opline->opcode == ZEND_DO_FCALL) { 9278 if (!func) { 9279 if (!trace) { 9280 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9281 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9282 | bne >1 9283 |.cold_code 9284 |1: 9285 if (!GCC_GLOBAL_REGS) { 9286 | mov FCARG1x, RX 9287 } 9288 | EXT_CALL zend_jit_deprecated_helper, REG0 9289 | GET_LOW_8BITS RETVALw, RETVALw 9290 | ldr REG0, EX:RX->func // reload 9291 | cbnz RETVALw, >1 // Result is 0 on exception 9292 | b ->exception_handler 9293 |.code 9294 |1: 9295 } 9296 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 9297 if (!GCC_GLOBAL_REGS) { 9298 | mov FCARG1x, RX 9299 } 9300 | EXT_CALL zend_jit_deprecated_helper, REG0 9301 | cbz RETVALw, ->exception_handler 9302 } 9303 } 9304 9305 if (!func 9306 && opline->opcode != ZEND_DO_UCALL 9307 && opline->opcode != ZEND_DO_ICALL) { 9308 | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] 9309 | cmp TMP1w, #ZEND_USER_FUNCTION 9310 | bne >8 9311 } 9312 9313 if ((!func || func->type == ZEND_USER_FUNCTION) 9314 && opline->opcode != ZEND_DO_ICALL) { 9315 | // EX(call) = NULL; 9316 | str xzr, EX:RX->call 9317 9318 if (RETURN_VALUE_USED(opline)) { 9319 | // EX(return_value) = EX_VAR(opline->result.var); 9320 | LOAD_ZVAL_ADDR REG2, res_addr 9321 | str REG2, EX:RX->return_value 9322 } else { 9323 | // EX(return_value) = 0; 9324 | str xzr, EX:RX->return_value 9325 } 9326 9327 //EX_LOAD_RUN_TIME_CACHE(op_array); 9328 if (!func || func->op_array.cache_size) { 9329 if (func && op_array == &func->op_array) { 9330 /* recursive call */ 9331 if (trace || func->op_array.cache_size > sizeof(void*)) { 9332 | ldr REG2, EX->run_time_cache 9333 | str REG2, EX:RX->run_time_cache 9334 } 9335 } else { 9336 if (func 9337 && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE) 9338 && ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { 9339 | MEM_LOAD_64_ZTS ldr, REG2, compiler_globals, map_ptr_base, TMP1 9340 | ADD_SUB_64_WITH_CONST add, REG2, REG2, (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache), TMP1 9341 | ldr REG2, [REG2] 9342 } else if ((func && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) || 9343 (JIT_G(current_frame) && 9344 JIT_G(current_frame)->call && 9345 TRACE_FRAME_IS_CLOSURE_CALL(JIT_G(current_frame)->call))) { 9346 /* Closures always use direct pointers */ 9347 | ldr REG0, EX:RX->func 9348 | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] 9349 } else { 9350 if (func) { 9351 | ldr REG0, EX:RX->func 9352 } 9353 | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] 9354 | TST_64_WITH_ONE REG2 9355 | beq >1 9356 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 9357 | ldr REG2, [REG2] 9358 |1: 9359 } 9360 | str REG2, EX:RX->run_time_cache 9361 } 9362 } 9363 9364 | // EG(current_execute_data) = execute_data; 9365 | MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1 9366 | mov FP, RX 9367 9368 | // opline = op_array->opcodes; 9369 if (func && !unknown_num_args) { 9370 | ADD_SUB_64_WITH_CONST_32 add, TMP1, RX, (EX_NUM_TO_VAR(call_num_args) + offsetof(zval, u1.type_info)), TMP1 // induction variable 9371 for (i = call_num_args; i < func->op_array.last_var; i++) { 9372 | // ZVAL_UNDEF(EX_VAR(n)) 9373 | str wzr, [TMP1], #16 9374 } 9375 9376 if (call_num_args <= func->op_array.num_args) { 9377 if (!trace || (trace->op == ZEND_JIT_TRACE_END 9378 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 9379 uint32_t num_args; 9380 9381 if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { 9382 if (trace) { 9383 num_args = 0; 9384 } else if (call_info) { 9385 num_args = skip_valid_arguments(op_array, ssa, call_info); 9386 } else { 9387 num_args = call_num_args; 9388 } 9389 } else { 9390 num_args = call_num_args; 9391 } 9392 if (zend_accel_in_shm(func->op_array.opcodes)) { 9393 | LOAD_IP_ADDR (func->op_array.opcodes + num_args) 9394 } else { 9395 | ldr REG0, EX->func 9396 || ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(num_args * sizeof(zend_op)))); 9397 if (GCC_GLOBAL_REGS) { 9398 | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] 9399 if (num_args) { 9400 | add IP, IP, #(num_args * sizeof(zend_op)) 9401 } 9402 } else { 9403 | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] 9404 if (num_args) { 9405 | add FCARG1x, FCARG1x, #(num_args * sizeof(zend_op)) 9406 } 9407 | str FCARG1x, EX->opline 9408 } 9409 } 9410 9411 if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array 9412 && num_args >= op_array->required_num_args) { 9413 /* recursive call */ 9414 if (ZEND_OBSERVER_ENABLED) { 9415 | SAVE_IP 9416 | mov FCARG1x, FP 9417 | EXT_CALL zend_observer_fcall_begin, REG0 9418 } 9419#ifdef CONTEXT_THREADED_JIT 9420 | NIY // TODO 9421#else 9422 | b =>num_args 9423#endif 9424 return 1; 9425 } 9426 } 9427 } else { 9428 if (!trace || (trace->op == ZEND_JIT_TRACE_END 9429 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 9430 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 9431 | LOAD_IP_ADDR (func->op_array.opcodes) 9432 } else if (GCC_GLOBAL_REGS) { 9433 | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] 9434 } else { 9435 | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] 9436 | str FCARG1x, EX->opline 9437 } 9438 } 9439 if (!GCC_GLOBAL_REGS) { 9440 | mov FCARG1x, FP 9441 } 9442 | EXT_CALL zend_jit_copy_extra_args_helper, REG0 9443 } 9444 } else { 9445 | // opline = op_array->opcodes 9446 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 9447 | LOAD_IP_ADDR (func->op_array.opcodes) 9448 } else if (GCC_GLOBAL_REGS) { 9449 | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] 9450 } else { 9451 | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] 9452 | str FCARG1x, EX->opline 9453 } 9454 if (func) { 9455 | // num_args = EX_NUM_ARGS(); 9456 | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] 9457 | // if (UNEXPECTED(num_args > first_extra_arg)) 9458 | CMP_32_WITH_CONST REG1w, (func->op_array.num_args), TMP1w 9459 } else { 9460 | // first_extra_arg = op_array->num_args; 9461 | ldr REG2w, [REG0, #offsetof(zend_op_array, num_args)] 9462 | // num_args = EX_NUM_ARGS(); 9463 | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] 9464 | // if (UNEXPECTED(num_args > first_extra_arg)) 9465 | cmp REG1w, REG2w 9466 } 9467 | bgt >1 9468 |.cold_code 9469 |1: 9470 if (!GCC_GLOBAL_REGS) { 9471 | mov FCARG1x, FP 9472 } 9473 | EXT_CALL zend_jit_copy_extra_args_helper, REG0 9474 if (!func) { 9475 | ldr REG0, EX->func // reload 9476 } 9477 | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] // reload 9478 | b >1 9479 |.code 9480 if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { 9481 if (!func) { 9482 | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) 9483 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9484 | TST_32_WITH_CONST TMP1w, ZEND_ACC_HAS_TYPE_HINTS, TMP2w 9485 | bne >1 9486 } 9487 | // opline += num_args; 9488 || ZEND_ASSERT(sizeof(zend_op) == 32); 9489 | mov REG2w, REG1w 9490 | ADD_IP_SHIFT REG2, lsl #5, TMP1 9491 } 9492 |1: 9493 | // if (EXPECTED((int)num_args < op_array->last_var)) { 9494 if (func) { 9495 | LOAD_32BIT_VAL REG2w, func->op_array.last_var 9496 } else { 9497 | ldr REG2w, [REG0, #offsetof(zend_op_array, last_var)] 9498 } 9499 | subs REG2w, REG2w, REG1w 9500 | ble >3 9501 | // zval *var = EX_VAR_NUM(num_args); 9502 | add REG1, FP, REG1, lsl #4 9503 || ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(ZEND_CALL_FRAME_SLOT * sizeof(zval)))); 9504 | add REG1, REG1, #(ZEND_CALL_FRAME_SLOT * sizeof(zval)) 9505 |2: 9506 | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w 9507 | add REG1, REG1, #16 9508 | subs REG2w, REG2w, #1 9509 | bne <2 9510 |3: 9511 } 9512 9513 if (ZEND_OBSERVER_ENABLED) { 9514 if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 9515 ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END); 9516 | SET_EX_OPLINE trace[1].opline, REG0 9517 } else { 9518 | SAVE_IP 9519 } 9520 | mov FCARG1x, FP 9521 | EXT_CALL zend_observer_fcall_begin, REG0 9522 } 9523 9524 if (trace) { 9525 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 9526 | b >9 9527 } 9528 } else { 9529#ifdef CONTEXT_THREADED_JIT 9530 | NIY // TODO: CONTEXT_THREADED_JIT is always undefined. 9531#else 9532 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 9533 | ADD_HYBRID_SPAD 9534 | JMP_IP TMP1 9535 } else if (GCC_GLOBAL_REGS) { 9536 | ldp x29, x30, [sp], # SPAD // stack alignment 9537 | JMP_IP TMP1 9538 } else { 9539 | ldp FP, RX, T2 // restore FP and IP 9540 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 9541 | mov RETVALx, #1 // ZEND_VM_ENTER 9542 | ret 9543 } 9544 } 9545#endif 9546 } 9547 9548 if ((!func || func->type == ZEND_INTERNAL_FUNCTION) 9549 && (opline->opcode != ZEND_DO_UCALL)) { 9550 if (!func && (opline->opcode != ZEND_DO_ICALL)) { 9551 |8: 9552 } 9553 if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { 9554 if (!func) { 9555 if (trace) { 9556 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9557 9558 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9559 if (!exit_addr) { 9560 return 0; 9561 } 9562 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9563 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9564 | bne &exit_addr 9565 } else { 9566 | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] 9567 | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w 9568 | bne >1 9569 |.cold_code 9570 |1: 9571 if (!GCC_GLOBAL_REGS) { 9572 | mov FCARG1x, RX 9573 } 9574 | EXT_CALL zend_jit_deprecated_helper, REG0 9575 | GET_LOW_8BITS RETVALw, RETVALw 9576 | ldr REG0, EX:RX->func // reload 9577 | cbnz RETVALw, >1 // Result is 0 on exception 9578 | b ->exception_handler 9579 |.code 9580 |1: 9581 } 9582 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 9583 if (!GCC_GLOBAL_REGS) { 9584 | mov FCARG1x, RX 9585 } 9586 | EXT_CALL zend_jit_deprecated_helper, REG0 9587 | cbz RETVALw, ->exception_handler 9588 | ldr REG0, EX:RX->func // reload 9589 } 9590 } 9591 9592 | // EG(current_execute_data) = execute_data; 9593 | MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1 9594 9595 if (ZEND_OBSERVER_ENABLED) { 9596 | mov FCARG1x, RX 9597 | EXT_CALL zend_observer_fcall_begin, REG0 9598 | ldr REG0, EX:RX->func // reload 9599 } 9600 9601 | // ZVAL_NULL(EX_VAR(opline->result.var)); 9602 | LOAD_ZVAL_ADDR FCARG2x, res_addr 9603 | SET_Z_TYPE_INFO FCARG2x, IS_NULL, TMP1w 9604 9605 zend_jit_reset_last_valid_opline(); 9606 9607 | // (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret); 9608 | mov FCARG1x, RX 9609 if (zend_execute_internal) { 9610 | EXT_CALL zend_execute_internal, REG0 9611 } else { 9612 if (func) { 9613 | EXT_CALL func->internal_function.handler, REG0 9614 } else { 9615 | ldr TMP1, [REG0, #offsetof(zend_internal_function, handler)] 9616 | blr TMP1 9617 } 9618 } 9619 9620 if (ZEND_OBSERVER_ENABLED) { 9621 | LOAD_ZVAL_ADDR FCARG2x, res_addr 9622 | mov FCARG1x, RX 9623 | EXT_CALL zend_observer_fcall_end, REG0 9624 } 9625 9626 | // EG(current_execute_data) = execute_data; 9627 | MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0 9628 9629 | // zend_vm_stack_free_args(call); 9630 if (func && !unknown_num_args) { 9631 for (i = 0; i < call_num_args; i++ ) { 9632 if (zend_jit_needs_arg_dtor(func, i, call_info)) { 9633 uint32_t offset = EX_NUM_TO_VAR(i); 9634 zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset); 9635 | ZVAL_PTR_DTOR arg_addr, (MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN), 0, 1, opline, ZREG_TMP1, ZREG_TMP2 9636 } 9637 } 9638 } else { 9639 | mov FCARG1x, RX 9640 | EXT_CALL zend_jit_vm_stack_free_args_helper, REG0 9641 } 9642 if (may_have_extra_named_params) { 9643 | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] 9644 | TST_32_WITH_CONST TMP1w, (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24), TMP2w 9645 | bne >1 9646 |.cold_code 9647 |1: 9648 | ldr FCARG1x, [RX, #offsetof(zend_execute_data, extra_named_params)] 9649 | EXT_CALL zend_free_extra_named_params, REG0 9650 | b >2 9651 |.code 9652 |2: 9653 } 9654 9655 |8: 9656 if (opline->opcode == ZEND_DO_FCALL) { 9657 // TODO: optimize ??? 9658 | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) 9659 | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] 9660 | TST_32_WITH_CONST TMP1w, (ZEND_CALL_RELEASE_THIS >> 16), TMP2w 9661 | bne >1 9662 |.cold_code 9663 |1: 9664 | add TMP1, RX, #offsetof(zend_execute_data, This) 9665 | GET_Z_PTR FCARG1x, TMP1 9666 | // OBJ_RELEASE(object); 9667 | OBJ_RELEASE ZREG_FCARG1, >2, ZREG_TMP1, ZREG_TMP2 9668 | b >2 9669 |.code 9670 |2: 9671 } 9672 9673 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 9674 !JIT_G(current_frame) || 9675 !JIT_G(current_frame)->call || 9676 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || 9677 prev_opline->opcode == ZEND_SEND_UNPACK || 9678 prev_opline->opcode == ZEND_SEND_ARRAY || 9679 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 9680 9681 | // zend_vm_stack_free_call_frame(call); 9682 | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] 9683 | TST_32_WITH_CONST TMP1w, ((ZEND_CALL_ALLOCATED >> 16) & 0xff), TMP2w 9684 | bne >1 9685 |.cold_code 9686 |1: 9687 | mov FCARG1x, RX 9688 | EXT_CALL zend_jit_free_call_frame, REG0 9689 | b >1 9690 |.code 9691 } 9692 | MEM_STORE_64_ZTS str, RX, executor_globals, vm_stack_top, REG0 9693 |1: 9694 9695 if (!RETURN_VALUE_USED(opline)) { 9696 zend_class_entry *ce; 9697 bool ce_is_instanceof; 9698 uint32_t func_info = call_info ? 9699 zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : 9700 (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); 9701 9702 /* If an exception is thrown, the return_value may stay at the 9703 * original value of null. */ 9704 func_info |= MAY_BE_NULL; 9705 9706 if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 9707 | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline, ZREG_TMP1, ZREG_TMP2 9708 } 9709 } 9710 9711 | // if (UNEXPECTED(EG(exception) != NULL)) { 9712 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 9713 | cbnz REG0, ->icall_throw_handler 9714 9715 // TODO: Can we avoid checking for interrupts after each call ??? 9716 if (trace && last_valid_opline != opline) { 9717 int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); 9718 9719 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9720 if (!exit_addr) { 9721 return 0; 9722 } 9723 } else { 9724 exit_addr = NULL; 9725 } 9726 if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { 9727 return 0; 9728 } 9729 9730 if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { 9731 | LOAD_IP_ADDR (opline + 1) 9732 } else if (trace 9733 && trace->op == ZEND_JIT_TRACE_END 9734 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 9735 | LOAD_IP_ADDR (opline + 1) 9736 } 9737 } 9738 9739 if (!func) { 9740 |9: 9741 } 9742 9743 return 1; 9744} 9745 9746static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) 9747{ 9748 uint32_t arg_num = opline->op2.num; 9749 zend_jit_addr arg_addr; 9750 9751 ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); 9752 9753 if (!zend_jit_reuse_ip(Dst)) { 9754 return 0; 9755 } 9756 9757 if (opline->opcode == ZEND_SEND_VAL_EX) { 9758 uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); 9759 9760 ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); 9761 9762 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9763 && JIT_G(current_frame) 9764 && JIT_G(current_frame)->call 9765 && JIT_G(current_frame)->call->func) { 9766 if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 9767 /* Don't generate code that always throws exception */ 9768 return 0; 9769 } 9770 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9771 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9772 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9773 if (!exit_addr) { 9774 return 0; 9775 } 9776 | ldr REG0, EX:RX->func 9777 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9778 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9779 | bne &exit_addr 9780 } else { 9781 | ldr REG0, EX:RX->func 9782 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9783 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9784 | bne >1 9785 |.cold_code 9786 |1: 9787 if (Z_MODE(op1_addr) == IS_REG) { 9788 /* set type to avoid zval_ptr_dtor() on uninitialized value */ 9789 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 9790 | SET_ZVAL_TYPE_INFO addr, IS_UNDEF, TMP1w, TMP2 9791 } 9792 | SET_EX_OPLINE opline, REG0 9793 | b ->throw_cannot_pass_by_ref 9794 |.code 9795 } 9796 } 9797 9798 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 9799 9800 if (opline->op1_type == IS_CONST) { 9801 zval *zv = RT_CONSTANT(opline, opline->op1); 9802 9803 | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 9804 if (Z_REFCOUNTED_P(zv)) { 9805 | ADDREF_CONST zv, REG0, TMP1 9806 } 9807 } else { 9808 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9809 } 9810 9811 return 1; 9812} 9813 9814static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) 9815{ 9816 | ldr FCARG1x, EX->call 9817 | ldrb TMP1w, [FCARG1x, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] 9818 | TST_32_WITH_CONST TMP1w, (ZEND_CALL_MAY_HAVE_UNDEF >> 24), TMP2w 9819 | bne >1 9820 |.cold_code 9821 |1: 9822 | SET_EX_OPLINE opline, REG0 9823 | EXT_CALL zend_handle_undef_args, REG0 9824 | cbz RETVALw, >2 9825 | b ->exception_handler 9826 |.code 9827 |2: 9828 9829 return 1; 9830} 9831 9832static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) 9833{ 9834 zend_jit_addr op1_addr, arg_addr, ref_addr; 9835 9836 op1_addr = OP1_ADDR(); 9837 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 9838 9839 if (!zend_jit_reuse_ip(Dst)) { 9840 return 0; 9841 } 9842 9843 if (opline->op1_type == IS_VAR) { 9844 if (op1_info & MAY_BE_INDIRECT) { 9845 | LOAD_ZVAL_ADDR REG0, op1_addr 9846 | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { 9847 | IF_NOT_Z_TYPE REG0, IS_INDIRECT, >1, TMP1w 9848 | // ret = Z_INDIRECT_P(ret); 9849 | GET_Z_PTR REG0, REG0 9850 |1: 9851 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 9852 } 9853 } else if (opline->op1_type == IS_CV) { 9854 if (op1_info & MAY_BE_UNDEF) { 9855 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 9856 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 9857 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 9858 | b >2 9859 |1: 9860 } 9861 op1_info &= ~MAY_BE_UNDEF; 9862 op1_info |= MAY_BE_NULL; 9863 } 9864 } else { 9865 ZEND_UNREACHABLE(); 9866 } 9867 9868 if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { 9869 if (op1_info & MAY_BE_REF) { 9870 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2, ZREG_TMP1 9871 | GET_ZVAL_PTR REG1, op1_addr, TMP1 9872 | GC_ADDREF REG1, TMP1w 9873 | SET_ZVAL_PTR arg_addr, REG1, TMP1 9874 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 9875 | b >6 9876 } 9877 |2: 9878 | // ZVAL_NEW_REF(arg, varptr); 9879 if (opline->op1_type == IS_VAR) { 9880 if (Z_REG(op1_addr) != ZREG_REG0 || Z_OFFSET(op1_addr) != 0) { 9881 | LOAD_ZVAL_ADDR REG0, op1_addr 9882 } 9883 | str REG0, T1 // save 9884 } 9885 | EMALLOC sizeof(zend_reference), op_array, opline // Allocate space in REG0 9886 | mov TMP1w, #2 9887 | str TMP1w, [REG0] 9888 || ZEND_ASSERT(GC_REFERENCE <= MOVZ_IMM); 9889 | movz TMP1w, #GC_REFERENCE 9890 | str TMP1w, [REG0, #offsetof(zend_reference, gc.u.type_info)] 9891 | str xzr, [REG0, #offsetof(zend_reference, sources.ptr)] 9892 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); 9893 if (opline->op1_type == IS_VAR) { 9894 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); 9895 9896 | ldr REG1, T1 // restore 9897 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9898 | SET_ZVAL_PTR val_addr, REG0, TMP1 9899 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX, TMP1w, TMP2 9900 } else { 9901 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9902 | SET_ZVAL_PTR op1_addr, REG0, TMP1 9903 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 9904 } 9905 | SET_ZVAL_PTR arg_addr, REG0, TMP1 9906 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 9907 } 9908 9909 |6: 9910 | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline, ZREG_TMP1, ZREG_TMP2 9911 |7: 9912 9913 return 1; 9914} 9915 9916static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) 9917{ 9918 uint32_t arg_num = opline->op2.num; 9919 zend_jit_addr arg_addr; 9920 9921 ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && 9922 opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || 9923 arg_num <= MAX_ARG_FLAG_NUM); 9924 9925 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 9926 9927 if (!zend_jit_reuse_ip(Dst)) { 9928 return 0; 9929 } 9930 9931 if (opline->opcode == ZEND_SEND_VAR_EX) { 9932 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9933 && JIT_G(current_frame) 9934 && JIT_G(current_frame)->call 9935 && JIT_G(current_frame)->call->func) { 9936 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 9937 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 9938 return 0; 9939 } 9940 return 1; 9941 } 9942 } else { 9943 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 9944 9945 | ldr REG0, EX:RX->func 9946 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9947 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9948 | bne >1 9949 |.cold_code 9950 |1: 9951 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 9952 return 0; 9953 } 9954 | b >7 9955 |.code 9956 } 9957 } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { 9958 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9959 && JIT_G(current_frame) 9960 && JIT_G(current_frame)->call 9961 && JIT_G(current_frame)->call->func) { 9962 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 9963 9964 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9965 9966 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 9967 if (!(op1_info & MAY_BE_REF)) { 9968 /* Don't generate code that always throws exception */ 9969 return 0; 9970 } else { 9971 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9972 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9973 if (!exit_addr) { 9974 return 0; 9975 } 9976 | GET_LOW_8BITS TMP1w, REG1w 9977 | cmp TMP1w, #IS_REFERENCE 9978 | bne &exit_addr 9979 } 9980 } 9981 return 1; 9982 } 9983 } else { 9984 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 9985 9986 | ldr REG0, EX:RX->func 9987 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 9988 | TST_32_WITH_CONST TMP1w, mask, TMP2w 9989 | bne >1 9990 |.cold_code 9991 |1: 9992 9993 mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); 9994 9995 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 9996 if (op1_info & MAY_BE_REF) { 9997 | GET_LOW_8BITS TMP1w, REG1w 9998 | cmp TMP1w, #IS_REFERENCE 9999 | beq >7 10000 } 10001 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 10002 | TST_32_WITH_CONST TMP1w, mask, TMP2w 10003 | bne >7 10004 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10005 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10006 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10007 if (!exit_addr) { 10008 return 0; 10009 } 10010 | b &exit_addr 10011 } else { 10012 | SET_EX_OPLINE opline, REG0 10013 | LOAD_ZVAL_ADDR FCARG1x, arg_addr 10014 | EXT_CALL zend_jit_only_vars_by_reference, REG0 10015 if (!zend_jit_check_exception(Dst)) { 10016 return 0; 10017 } 10018 | b >7 10019 } 10020 10021 |.code 10022 } 10023 } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { 10024 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10025 && JIT_G(current_frame) 10026 && JIT_G(current_frame)->call 10027 && JIT_G(current_frame)->call->func) { 10028 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10029 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10030 return 0; 10031 } 10032 return 1; 10033 } 10034 } else { 10035 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10036 | TST_32_WITH_CONST TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10037 | bne >1 10038 |.cold_code 10039 |1: 10040 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10041 return 0; 10042 } 10043 | b >7 10044 |.code 10045 } 10046 } 10047 10048 if (op1_info & MAY_BE_UNDEF) { 10049 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10050 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 10051 |.cold_code 10052 |1: 10053 } 10054 10055 | SET_EX_OPLINE opline, REG0 10056 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 10057 | EXT_CALL zend_jit_undefined_op_helper, REG0 10058 | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL, TMP1w, TMP2 10059 | cbz RETVALx, ->exception_handler 10060 10061 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10062 | b >7 10063 |.code 10064 } else { 10065 |7: 10066 return 1; 10067 } 10068 } 10069 10070 if (opline->opcode == ZEND_SEND_VAR_NO_REF) { 10071 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10072 if (op1_info & MAY_BE_REF) { 10073 | GET_LOW_8BITS TMP1w, REG1w 10074 | cmp TMP1w, #IS_REFERENCE 10075 | beq >7 10076 } 10077 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10078 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10079 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10080 if (!exit_addr) { 10081 return 0; 10082 } 10083 | b &exit_addr 10084 } else { 10085 | SET_EX_OPLINE opline, REG0 10086 | LOAD_ZVAL_ADDR FCARG1x, arg_addr 10087 | EXT_CALL zend_jit_only_vars_by_reference, REG0 10088 if (!zend_jit_check_exception(Dst)) { 10089 return 0; 10090 } 10091 } 10092 } else { 10093 if (op1_info & MAY_BE_REF) { 10094 if (opline->op1_type == IS_CV) { 10095 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 10096 10097 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 10098 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 10099 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10100 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 10101 } else { 10102 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 8); 10103 10104 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 10105 |.cold_code 10106 |1: 10107 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 10108 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10109 | // ZVAL_COPY_VALUE(return_value, &ref->value); 10110 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10111 | GC_DELREF FCARG1x, TMP1w 10112 | beq >1 10113 | IF_NOT_REFCOUNTED REG0w, >2, TMP1w 10114 | GC_ADDREF REG2, TMP1w 10115 | b >2 10116 |1: 10117 | EFREE_REFERENCE 10118 | b >2 10119 |.code 10120 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10121 |2: 10122 } 10123 } else { 10124 if (op1_addr != op1_def_addr) { 10125 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 10126 return 0; 10127 } 10128 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 10129 op1_addr= op1_def_addr; 10130 } 10131 } 10132 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10133 if (opline->op1_type == IS_CV) { 10134 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 10135 } 10136 } 10137 } 10138 |7: 10139 10140 return 1; 10141} 10142 10143static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) 10144{ 10145 uint32_t arg_num = opline->op2.num; 10146 10147 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10148 && JIT_G(current_frame) 10149 && JIT_G(current_frame)->call 10150 && JIT_G(current_frame)->call->func) { 10151 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10152 if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { 10153 TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); 10154 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10155 || if (reuse_ip) { 10156 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10157 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10158 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10159 || } else { 10160 | ldr REG0, EX->call 10161 | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10162 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10163 | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10164 || } 10165 } 10166 } else { 10167 if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 10168 TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); 10169 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10170 || if (reuse_ip) { 10171 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10172 | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w 10173 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10174 || } else { 10175 | ldr REG0, EX->call 10176 | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10177 | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w 10178 | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] 10179 || } 10180 } 10181 } 10182 } else { 10183 // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { 10184 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10185 10186 if (!zend_jit_reuse_ip(Dst)) { 10187 return 0; 10188 } 10189 10190 | ldr REG0, EX:RX->func 10191 | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] 10192 | TST_32_WITH_CONST TMP1w, mask, TMP2w 10193 | bne >1 10194 |.cold_code 10195 |1: 10196 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10197 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10198 | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w 10199 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10200 | b >1 10201 |.code 10202 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10203 | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10204 | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~(ZEND_CALL_SEND_ARG_BY_REF)), TMP2w 10205 | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] 10206 |1: 10207 } 10208 10209 return 1; 10210} 10211 10212static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2) 10213{ 10214 if (smart_branch_opcode) { 10215 if (smart_branch_opcode == ZEND_JMPZ) { 10216 if (jmp) { 10217 | b >7 10218 } 10219 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10220 | b =>target_label 10221 } else { 10222 ZEND_UNREACHABLE(); 10223 } 10224 } else { 10225 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10226 10227 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 10228 if (jmp) { 10229 | b >7 10230 } 10231 } 10232 10233 return 1; 10234} 10235 10236static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, uint8_t smart_branch_opcode, uint32_t target_label) 10237{ 10238 if (smart_branch_opcode) { 10239 if (smart_branch_opcode == ZEND_JMPZ) { 10240 | b =>target_label 10241 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10242 if (jmp) { 10243 | b >7 10244 } 10245 } else { 10246 ZEND_UNREACHABLE(); 10247 } 10248 } else { 10249 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10250 10251 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 10252 if (jmp) { 10253 | b >7 10254 } 10255 } 10256 10257 return 1; 10258} 10259 10260static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 10261{ 10262 uint32_t defined_label = (uint32_t)-1; 10263 uint32_t undefined_label = (uint32_t)-1; 10264 zval *zv = RT_CONSTANT(opline, opline->op1); 10265 zend_jit_addr res_addr = 0; 10266 10267 if (smart_branch_opcode && !exit_addr) { 10268 if (smart_branch_opcode == ZEND_JMPZ) { 10269 undefined_label = target_label; 10270 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10271 defined_label = target_label; 10272 } else { 10273 ZEND_UNREACHABLE(); 10274 } 10275 } 10276 10277 | // if (CACHED_PTR(opline->extended_value)) { 10278 | ldr REG0, EX->run_time_cache 10279 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, opline->extended_value, TMP1 10280 | cbz REG0, >1 10281 | TST_64_WITH_ONE REG0 10282 | bne >4 10283 |.cold_code 10284 |4: 10285 | MEM_LOAD_64_ZTS ldr, FCARG1x, executor_globals, zend_constants, FCARG1x 10286 | ldr TMP1w, [FCARG1x, #offsetof(HashTable, nNumOfElements)] 10287 | cmp TMP1, REG0, lsr #1 10288 10289 if (smart_branch_opcode) { 10290 if (exit_addr) { 10291 if (smart_branch_opcode == ZEND_JMPZ) { 10292 | beq &exit_addr 10293 } else { 10294 | beq >3 10295 } 10296 } else if (undefined_label != (uint32_t)-1) { 10297 | beq =>undefined_label 10298 } else { 10299 | beq >3 10300 } 10301 } else { 10302 | beq >2 10303 } 10304 |1: 10305 | SET_EX_OPLINE opline, REG0 10306 | LOAD_ADDR FCARG1x, zv 10307 | EXT_CALL zend_jit_check_constant, REG0 10308 if (exit_addr) { 10309 if (smart_branch_opcode == ZEND_JMPNZ) { 10310 | cbz RETVALx, >3 10311 } else { 10312 | cbnz RETVALx, >3 10313 } 10314 | b &exit_addr 10315 } else if (smart_branch_opcode) { 10316 if (undefined_label != (uint32_t)-1) { 10317 | cbz RETVALx, =>undefined_label 10318 } else { 10319 | cbz RETVALx, >3 10320 } 10321 if (defined_label != (uint32_t)-1) { 10322 | b =>defined_label 10323 } else { 10324 | b >3 10325 } 10326 } else { 10327 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10328 | cbnz RETVALx, >1 10329 |2: 10330 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 10331 | b >3 10332 } 10333 |.code 10334 if (smart_branch_opcode) { 10335 if (exit_addr) { 10336 if (smart_branch_opcode == ZEND_JMPNZ) { 10337 | b &exit_addr 10338 } 10339 } else if (defined_label != (uint32_t)-1) { 10340 | b =>defined_label 10341 } 10342 } else { 10343 |1: 10344 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 10345 } 10346 |3: 10347 10348 return 1; 10349} 10350 10351static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 10352{ 10353 uint32_t mask; 10354 zend_jit_addr op1_addr = OP1_ADDR(); 10355 10356 // TODO: support for is_resource() ??? 10357 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); 10358 10359 if (op1_info & MAY_BE_UNDEF) { 10360 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10361 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 10362 |.cold_code 10363 |1: 10364 } 10365 | SET_EX_OPLINE opline, REG0 10366 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 10367 | EXT_CALL zend_jit_undefined_op_helper, REG0 10368 zend_jit_check_exception_undef_result(Dst, opline); 10369 if (opline->extended_value & MAY_BE_NULL) { 10370 if (exit_addr) { 10371 if (smart_branch_opcode == ZEND_JMPNZ) { 10372 | b &exit_addr 10373 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 10374 | b >7 10375 } 10376 } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { 10377 return 0; 10378 } 10379 } else { 10380 if (exit_addr) { 10381 if (smart_branch_opcode == ZEND_JMPZ) { 10382 | b &exit_addr 10383 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 10384 | b >7 10385 } 10386 } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { 10387 return 0; 10388 } 10389 } 10390 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10391 |.code 10392 } 10393 } 10394 10395 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10396 mask = opline->extended_value; 10397 if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { 10398 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10399 if (exit_addr) { 10400 if (smart_branch_opcode == ZEND_JMPNZ) { 10401 | b &exit_addr 10402 } 10403 } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { 10404 return 0; 10405 } 10406 } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { 10407 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10408 if (exit_addr) { 10409 if (smart_branch_opcode == ZEND_JMPZ) { 10410 | b &exit_addr 10411 } 10412 } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { 10413 return 0; 10414 } 10415 } else { 10416 bool invert = 0; 10417 uint8_t type; 10418 10419 switch (mask) { 10420 case MAY_BE_NULL: type = IS_NULL; break; 10421 case MAY_BE_FALSE: type = IS_FALSE; break; 10422 case MAY_BE_TRUE: type = IS_TRUE; break; 10423 case MAY_BE_LONG: type = IS_LONG; break; 10424 case MAY_BE_DOUBLE: type = IS_DOUBLE; break; 10425 case MAY_BE_STRING: type = IS_STRING; break; 10426 case MAY_BE_ARRAY: type = IS_ARRAY; break; 10427 case MAY_BE_OBJECT: type = IS_OBJECT; break; 10428 case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break; 10429 case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break; 10430 case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break; 10431 case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break; 10432 case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break; 10433 case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break; 10434 case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break; 10435 case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break; 10436 case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break; 10437 default: 10438 type = 0; 10439 } 10440 10441 if (op1_info & MAY_BE_REF) { 10442 | LOAD_ZVAL_ADDR REG0, op1_addr 10443 | ZVAL_DEREF REG0, op1_info, TMP1w 10444 } 10445 if (type == 0) { 10446 if (smart_branch_opcode && 10447 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 10448 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10449 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10450 | // if (Z_REFCOUNTED_P(cv)) { 10451 | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 10452 |.cold_code 10453 |1: 10454 } 10455 | // if (!Z_DELREF_P(cv)) { 10456 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10457 | GC_DELREF FCARG1x, TMP1w 10458 if (RC_MAY_BE_1(op1_info)) { 10459 if (RC_MAY_BE_N(op1_info)) { 10460 | bne >3 10461 } 10462 if (op1_info & MAY_BE_REF) { 10463 | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] 10464 } else { 10465 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10466 } 10467 | str REG0w, T1 // save 10468 | // zval_dtor_func(r); 10469 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 10470 | ldr REG1w, T1 // restore 10471 | b >2 10472 } 10473 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10474 if (!RC_MAY_BE_1(op1_info)) { 10475 | b >3 10476 } 10477 |.code 10478 } 10479 |3: 10480 if (op1_info & MAY_BE_REF) { 10481 | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] 10482 } else { 10483 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10484 } 10485 |2: 10486 } else { 10487 if (op1_info & MAY_BE_REF) { 10488 | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] 10489 } else { 10490 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10491 } 10492 } 10493 | mov REG0w, #1 10494 | lsl REG0w, REG0w, REG1w 10495 | TST_32_WITH_CONST REG0w, mask, TMP1w 10496 if (exit_addr) { 10497 if (smart_branch_opcode == ZEND_JMPNZ) { 10498 | bne &exit_addr 10499 } else { 10500 | beq &exit_addr 10501 } 10502 } else if (smart_branch_opcode) { 10503 if (smart_branch_opcode == ZEND_JMPZ) { 10504 | beq =>target_label 10505 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10506 | bne =>target_label 10507 } else { 10508 ZEND_UNREACHABLE(); 10509 } 10510 } else { 10511 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10512 10513 | cset REG0w, ne 10514 | add REG0w, REG0w, #2 10515 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 10516 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10517 } 10518 } else { 10519 if (smart_branch_opcode && 10520 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 10521 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10522 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10523 | // if (Z_REFCOUNTED_P(cv)) { 10524 | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 10525 |.cold_code 10526 |1: 10527 } 10528 | // if (!Z_DELREF_P(cv)) { 10529 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10530 | GC_DELREF FCARG1x, TMP1w 10531 if (RC_MAY_BE_1(op1_info)) { 10532 if (RC_MAY_BE_N(op1_info)) { 10533 | bne >3 10534 } 10535 if (op1_info & MAY_BE_REF) { 10536 | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] 10537 } else { 10538 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10539 } 10540 | str REG0w, T1 // save 10541 | // zval_dtor_func(r); 10542 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 10543 | ldr REG1w, T1 // restore 10544 | b >2 10545 } 10546 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10547 if (!RC_MAY_BE_1(op1_info)) { 10548 | b >3 10549 } 10550 |.code 10551 } 10552 |3: 10553 if (op1_info & MAY_BE_REF) { 10554 | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] 10555 } else { 10556 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10557 } 10558 |2: 10559 // Note: 'type' is of uchar type and holds a positive value, 10560 // hence it's safe to directly encode it as the imm field of 'cmp' instruction. 10561 | cmp REG1w, #type 10562 } else { 10563 if (op1_info & MAY_BE_REF) { 10564 | ldrb TMP1w, [REG0, #offsetof(zval,u1.v.type)] 10565 | cmp TMP1w, #type 10566 } else { 10567 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 10568 | cmp TMP1w, #type 10569 } 10570 } 10571 if (exit_addr) { 10572 if (invert) { 10573 if (smart_branch_opcode == ZEND_JMPNZ) { 10574 | bne &exit_addr 10575 } else { 10576 | beq &exit_addr 10577 } 10578 } else { 10579 if (smart_branch_opcode == ZEND_JMPNZ) { 10580 | beq &exit_addr 10581 } else { 10582 | bne &exit_addr 10583 } 10584 } 10585 } else if (smart_branch_opcode) { 10586 if (invert) { 10587 if (smart_branch_opcode == ZEND_JMPZ) { 10588 | beq =>target_label 10589 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10590 | bne =>target_label 10591 } else { 10592 ZEND_UNREACHABLE(); 10593 } 10594 } else { 10595 if (smart_branch_opcode == ZEND_JMPZ) { 10596 | bne =>target_label 10597 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10598 | beq =>target_label 10599 } else { 10600 ZEND_UNREACHABLE(); 10601 } 10602 } 10603 } else { 10604 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10605 10606 if (invert) { 10607 | cset REG0w, ne 10608 } else { 10609 | cset REG0w, eq 10610 } 10611 | add REG0w, REG0w, #2 10612 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 10613 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 10614 } 10615 } 10616 } 10617 } 10618 10619 |7: 10620 10621 return 1; 10622} 10623 10624static int zend_jit_leave_frame(dasm_State **Dst) 10625{ 10626 | // EG(current_execute_data) = EX(prev_execute_data); 10627 | ldr REG0, EX->prev_execute_data 10628 | MEM_STORE_64_ZTS str, REG0, executor_globals, current_execute_data, REG2 10629 return 1; 10630} 10631 10632static int zend_jit_free_cvs(dasm_State **Dst) 10633{ 10634 | // EG(current_execute_data) = EX(prev_execute_data); 10635 | ldr FCARG1x, EX->prev_execute_data 10636 | MEM_STORE_64_ZTS str, FCARG1x, executor_globals, current_execute_data, REG0 10637 | // zend_free_compiled_variables(execute_data); 10638 | mov FCARG1x, FP 10639 | EXT_CALL zend_free_compiled_variables, REG0 10640 return 1; 10641} 10642 10643static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) 10644{ 10645 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10646 uint32_t offset = EX_NUM_TO_VAR(var); 10647 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset); 10648 | ZVAL_PTR_DTOR addr, info, 1, 1, NULL, ZREG_TMP1, ZREG_TMP2 10649 } 10650 return 1; 10651} 10652 10653static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) 10654{ 10655 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10656 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset); 10657 | ZVAL_PTR_DTOR addr, info, 0, 1, opline, ZREG_TMP1, ZREG_TMP2 10658 } 10659 return 1; 10660} 10661 10662static int zend_jit_leave_func(dasm_State **Dst, 10663 const zend_op_array *op_array, 10664 const zend_op *opline, 10665 uint32_t op1_info, 10666 bool left_frame, 10667 zend_jit_trace_rec *trace, 10668 zend_jit_trace_info *trace_info, 10669 int indirect_var_access, 10670 int may_throw) 10671{ 10672 bool may_be_top_frame = 10673 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10674 !JIT_G(current_frame) || 10675 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); 10676 bool may_need_call_helper = 10677 indirect_var_access || /* may have symbol table */ 10678 !op_array->function_name || /* may have symbol table */ 10679 may_be_top_frame || 10680 (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ 10681 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10682 !JIT_G(current_frame) || 10683 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ 10684 (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ 10685 bool may_need_release_this = 10686 !(op_array->fn_flags & ZEND_ACC_CLOSURE) && 10687 op_array->scope && 10688 !(op_array->fn_flags & ZEND_ACC_STATIC) && 10689 (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10690 !JIT_G(current_frame) || 10691 !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame))); 10692 10693 if (may_need_call_helper || may_need_release_this) { 10694 | ldr FCARG1w, [FP, #offsetof(zend_execute_data, This.u1.type_info)] 10695 } 10696 if (may_need_call_helper) { 10697 if (!left_frame) { 10698 left_frame = 1; 10699 if (!zend_jit_leave_frame(Dst)) { 10700 return 0; 10701 } 10702 } 10703 /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ 10704 10705 | TST_32_WITH_CONST FCARG1w, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE), TMP1w 10706 if (trace && trace->op != ZEND_JIT_TRACE_END) { 10707 | bne >1 10708 |.cold_code 10709 |1: 10710 if (!GCC_GLOBAL_REGS) { 10711 | mov FCARG1x, FP 10712 } 10713 | EXT_CALL zend_jit_leave_func_helper, REG0 10714 10715 if (may_be_top_frame) { 10716 // TODO: try to avoid this check ??? 10717 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10718#if 0 10719 /* this check should be handled by the following OPLINE guard */ 10720 | LOAD_ADDR TMP1, zend_jit_halt_op 10721 | cmp IP, TMP1 10722 | beq ->trace_halt 10723#endif 10724 } else if (GCC_GLOBAL_REGS) { 10725 | cbz IP, ->trace_halt 10726 } else { 10727 | tst RETVALw, RETVALw 10728 | blt ->trace_halt 10729 } 10730 } 10731 10732 if (!GCC_GLOBAL_REGS) { 10733 | // execute_data = EG(current_execute_data) 10734 | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 10735 } 10736 | b >8 10737 |.code 10738 } else { 10739 | bne ->leave_function_handler 10740 } 10741 } 10742 10743 if (op_array->fn_flags & ZEND_ACC_CLOSURE) { 10744 if (!left_frame) { 10745 left_frame = 1; 10746 if (!zend_jit_leave_frame(Dst)) { 10747 return 0; 10748 } 10749 } 10750 | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); 10751 | ldr FCARG1x, EX->func 10752 | sub FCARG1x, FCARG1x, #sizeof(zend_object) 10753 | OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2 10754 |4: 10755 } else if (may_need_release_this) { 10756 if (!left_frame) { 10757 left_frame = 1; 10758 if (!zend_jit_leave_frame(Dst)) { 10759 return 0; 10760 } 10761 } 10762 if (!JIT_G(current_frame) || !TRACE_FRAME_ALWAYS_RELEASE_THIS(JIT_G(current_frame))) { 10763 | // if (call_info & ZEND_CALL_RELEASE_THIS) 10764 | TST_32_WITH_CONST FCARG1w, ZEND_CALL_RELEASE_THIS, TMP1w 10765 | beq >4 10766 } 10767 | // zend_object *object = Z_OBJ(execute_data->This); 10768 | ldr FCARG1x, EX->This.value.obj 10769 | // OBJ_RELEASE(object); 10770 | OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2 10771 |4: 10772 // TODO: avoid EG(excption) check for $this->foo() calls 10773 may_throw = 1; 10774 } 10775 10776 | // EG(vm_stack_top) = (zval*)execute_data; 10777 | MEM_STORE_64_ZTS str, FP, executor_globals, vm_stack_top, REG0 10778 | // execute_data = EX(prev_execute_data); 10779 | ldr FP, EX->prev_execute_data 10780 10781 if (!left_frame) { 10782 | // EG(current_execute_data) = execute_data; 10783 | MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0 10784 } 10785 10786 |9: 10787 if (trace) { 10788 if (trace->op != ZEND_JIT_TRACE_END 10789 && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 10790 zend_jit_reset_last_valid_opline(); 10791 } else { 10792 | LOAD_IP 10793 | ADD_IP_WITH_CONST sizeof(zend_op), TMP1 10794 } 10795 10796 |8: 10797 10798 if (trace->op == ZEND_JIT_TRACE_BACK 10799 && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 10800 const zend_op *next_opline = trace->opline; 10801 10802 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 10803 && (op1_info & MAY_BE_RC1) 10804 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { 10805 /* exception might be thrown during destruction of unused return value */ 10806 | // if (EG(exception)) 10807 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 10808 | cbnz REG0, ->leave_throw_handler 10809 } 10810 do { 10811 trace++; 10812 } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); 10813 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 10814 next_opline = trace->opline; 10815 ZEND_ASSERT(next_opline != NULL); 10816 10817 if (trace->op == ZEND_JIT_TRACE_END 10818 && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { 10819 trace_info->flags |= ZEND_JIT_TRACE_LOOP; 10820 | CMP_IP next_opline, TMP1, TMP2 10821 | beq =>0 // LOOP 10822#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 10823 | JMP_IP TMP1 10824#else 10825 | b ->trace_escape 10826#endif 10827 } else { 10828 | CMP_IP next_opline, TMP1, TMP2 10829 | bne ->trace_escape 10830 } 10831 10832 zend_jit_set_last_valid_opline(trace->opline); 10833 10834 return 1; 10835 } else if (may_throw || 10836 (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 10837 && (op1_info & MAY_BE_RC1) 10838 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) 10839 && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { 10840 | // if (EG(exception)) 10841 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 10842 | cbnz REG0, ->leave_throw_handler 10843 } 10844 10845 return 1; 10846 } else { 10847 | // if (EG(exception)) 10848 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 10849 | LOAD_IP 10850 | cbnz REG0, ->leave_throw_handler 10851 | // opline = EX(opline) + 1 10852 | ADD_IP_WITH_CONST sizeof(zend_op), TMP1 10853 } 10854 10855 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10856 | ADD_HYBRID_SPAD 10857#ifdef CONTEXT_THREADED_JIT 10858 | NIY // TODO: CONTEXT_THREADED_JIT is always undefined 10859#else 10860 | JMP_IP TMP1 10861#endif 10862 } else if (GCC_GLOBAL_REGS) { 10863 | ldp x29, x30, [sp], # SPAD // stack alignment 10864#ifdef CONTEXT_THREADED_JIT 10865 | NIY // TODO 10866#else 10867 | JMP_IP TMP1 10868#endif 10869 } else { 10870#ifdef CONTEXT_THREADED_JIT 10871 ZEND_UNREACHABLE(); 10872 // TODO: context threading can't work without GLOBAL REGS because we have to change 10873 // the value of execute_data in execute_ex() 10874 | NIY // TODO 10875#else 10876 | ldp FP, RX, T2 // restore FP and IP 10877 | ldp x29, x30, [sp], # NR_SPAD // stack alignment 10878 | mov RETVALx, #2 // ZEND_VM_LEAVE ???? 10879 | ret 10880#endif 10881 } 10882 10883 return 1; 10884} 10885 10886static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) 10887{ 10888 zend_jit_addr ret_addr; 10889 int8_t return_value_used; 10890 10891 ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); 10892 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); 10893 10894 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { 10895 if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { 10896 return_value_used = 1; 10897 } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { 10898 return_value_used = 0; 10899 } else { 10900 return_value_used = -1; 10901 } 10902 } else { 10903 return_value_used = -1; 10904 } 10905 10906 if (ZEND_OBSERVER_ENABLED) { 10907 if (Z_MODE(op1_addr) == IS_REG) { 10908 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 10909 10910 if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { 10911 return 0; 10912 } 10913 op1_addr = dst; 10914 } 10915 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 10916 | mov FCARG1x, FP 10917 | SET_EX_OPLINE opline, REG0 10918 | EXT_CALL zend_observer_fcall_end, REG0 10919 } 10920 10921 // if (!EX(return_value)) 10922 if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_REG1) { 10923 if (return_value_used != 0) { 10924 | ldr REG2, EX->return_value 10925 } 10926 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); 10927 } else { 10928 if (return_value_used != 0) { 10929 | ldr REG1, EX->return_value 10930 } 10931 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); 10932 } 10933 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 10934 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10935 if (return_value_used == -1) { 10936 | cbz Rx(Z_REG(ret_addr)), >1 10937 |.cold_code 10938 |1: 10939 } 10940 if (return_value_used != 1) { 10941 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 10942 if (jit_return_label >= 0) { 10943 | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label, ZREG_TMP1, ZREG_TMP2 10944 } else { 10945 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9, ZREG_TMP1, ZREG_TMP2 10946 } 10947 } 10948 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 10949 | GC_DELREF FCARG1x, TMP1w 10950 if (RC_MAY_BE_1(op1_info)) { 10951 if (RC_MAY_BE_N(op1_info)) { 10952 if (jit_return_label >= 0) { 10953 | bne =>jit_return_label 10954 } else { 10955 | bne >9 10956 } 10957 } 10958 | //SAVE_OPLINE() 10959 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 10960 | //????ldr REG1, EX->return_value // reload ??? 10961 } 10962 if (return_value_used == -1) { 10963 if (jit_return_label >= 0) { 10964 | b =>jit_return_label 10965 } else { 10966 | b >9 10967 } 10968 |.code 10969 } 10970 } 10971 } else if (return_value_used == -1) { 10972 if (jit_return_label >= 0) { 10973 | cbz Rx(Z_REG(ret_addr)), =>jit_return_label 10974 } else { 10975 | cbz Rx(Z_REG(ret_addr)), >9 10976 } 10977 } 10978 10979 if (return_value_used == 0) { 10980 |9: 10981 return 1; 10982 } 10983 10984 if (opline->op1_type == IS_CONST) { 10985 zval *zv = RT_CONSTANT(opline, opline->op1); 10986 | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 10987 if (Z_REFCOUNTED_P(zv)) { 10988 | ADDREF_CONST zv, REG0, TMP1 10989 } 10990 } else if (opline->op1_type == IS_TMP_VAR) { 10991 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10992 } else if (opline->op1_type == IS_CV) { 10993 if (op1_info & MAY_BE_REF) { 10994 | LOAD_ZVAL_ADDR REG0, op1_addr 10995 | ZVAL_DEREF REG0, op1_info, TMP1w 10996 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 10997 } 10998 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 10999 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 11000 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11001 (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || 11002 !op_array->function_name) { 11003 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w 11004 } else if (return_value_used != 1) { 11005 | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); 11006 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 11007 } 11008 } 11009 } else { 11010 if (op1_info & MAY_BE_REF) { 11011 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); 11012 11013 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 11014 |.cold_code 11015 |1: 11016 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 11017 | GET_ZVAL_PTR REG0, op1_addr, TMP1 11018 | // ZVAL_COPY_VALUE(return_value, &ref->value); 11019 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11020 | GC_DELREF REG0, TMP1w 11021 | beq >2 11022 | // if (IS_REFCOUNTED()) 11023 if (jit_return_label >= 0) { 11024 | IF_NOT_REFCOUNTED REG2w, =>jit_return_label, TMP1w 11025 } else { 11026 | IF_NOT_REFCOUNTED REG2w, >9, TMP1w 11027 } 11028 | // ADDREF 11029 | GET_ZVAL_PTR REG2, ret_addr, TMP1 // reload 11030 | GC_ADDREF REG2, TMP1w 11031 if (jit_return_label >= 0) { 11032 | b =>jit_return_label 11033 } else { 11034 | b >9 11035 } 11036 |2: 11037 | mov FCARG1x, REG0 11038 | EFREE_REFERENCE 11039 if (jit_return_label >= 0) { 11040 | b =>jit_return_label 11041 } else { 11042 | b >9 11043 } 11044 |.code 11045 } 11046 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11047 } 11048 11049 |9: 11050 return 1; 11051} 11052 11053static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) 11054{ 11055 ZEND_ASSERT(type_reg == ZREG_REG2); 11056 11057 | GET_ZVAL_PTR REG1, val_addr, TMP1 11058 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w 11059 | GET_LOW_8BITS TMP2w, REG2w 11060 | IF_NOT_TYPE TMP2w, IS_REFERENCE, >1 11061 | add REG1, REG1, #offsetof(zend_reference, val) 11062 | GET_Z_TYPE_INFO REG2w, REG1 11063 | GET_Z_PTR REG1, REG1 11064 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w 11065 |1: 11066 | GC_ADDREF REG1, TMP2w 11067 |2: 11068 | SET_ZVAL_PTR res_addr, REG1, TMP1 11069 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 11070 11071 return 1; 11072} 11073 11074static int zend_jit_fetch_dim_read(dasm_State **Dst, 11075 const zend_op *opline, 11076 zend_ssa *ssa, 11077 const zend_ssa_op *ssa_op, 11078 uint32_t op1_info, 11079 zend_jit_addr op1_addr, 11080 bool op1_avoid_refcounting, 11081 uint32_t op2_info, 11082 uint32_t res_info, 11083 zend_jit_addr res_addr, 11084 uint8_t dim_type) 11085{ 11086 zend_jit_addr orig_op1_addr, op2_addr; 11087 const void *exit_addr = NULL; 11088 const void *not_found_exit_addr = NULL; 11089 const void *res_exit_addr = NULL; 11090 bool result_avoid_refcounting = 0; 11091 uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; 11092 int may_throw = 0; 11093 11094 orig_op1_addr = OP1_ADDR(); 11095 op2_addr = OP2_ADDR(); 11096 11097 if (opline->opcode != ZEND_FETCH_DIM_IS 11098 && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11099 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 11100 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11101 if (!exit_addr) { 11102 return 0; 11103 } 11104 } 11105 11106 if ((res_info & MAY_BE_GUARD) 11107 && JIT_G(current_frame) 11108 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { 11109 uint32_t flags = 0; 11110 uint32_t old_op1_info = 0; 11111 uint32_t old_info; 11112 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 11113 int32_t exit_point; 11114 11115 if (opline->opcode != ZEND_FETCH_LIST_R 11116 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11117 && !op1_avoid_refcounting) { 11118 flags |= ZEND_JIT_EXIT_FREE_OP1; 11119 } 11120 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) 11121 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11122 flags |= ZEND_JIT_EXIT_FREE_OP2; 11123 } 11124 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 11125 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 11126 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 11127 && (ssa_op+1)->op1_use == ssa_op->result_def 11128 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) 11129 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 11130 result_avoid_refcounting = 1; 11131 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 11132 } 11133 11134 if (op1_avoid_refcounting) { 11135 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 11136 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 11137 } 11138 11139 if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { 11140 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11141 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 11142 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 11143 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11144 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11145 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11146 if (!res_exit_addr) { 11147 return 0; 11148 } 11149 res_info &= ~MAY_BE_GUARD; 11150 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 11151 } 11152 11153 if (opline->opcode == ZEND_FETCH_DIM_IS 11154 && !(res_info & MAY_BE_NULL)) { 11155 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11156 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); 11157 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); 11158 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11159 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11160 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11161 if (!not_found_exit_addr) { 11162 return 0; 11163 } 11164 } 11165 11166 if (op1_avoid_refcounting) { 11167 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 11168 } 11169 } 11170 11171 if (op1_info & MAY_BE_REF) { 11172 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11173 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 11174 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11175 } 11176 11177 if (op1_info & MAY_BE_ARRAY) { 11178 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11179 if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { 11180 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr, ZREG_TMP1 11181 } else { 11182 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 11183 } 11184 } 11185 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 11186 if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) || 11187 (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) { 11188 may_throw = 1; 11189 } 11190 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, dim_type, res_exit_addr, not_found_exit_addr, exit_addr)) { 11191 return 0; 11192 } 11193 } 11194 11195 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 11196 if (op1_info & MAY_BE_ARRAY) { 11197 |.cold_code 11198 |7: 11199 } 11200 11201 if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { 11202 may_throw = 1; 11203 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { 11204 if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { 11205 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr, ZREG_TMP1 11206 } else { 11207 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 11208 } 11209 } 11210 | SET_EX_OPLINE opline, REG0 11211 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 11212 if (opline->opcode != ZEND_FETCH_DIM_IS) { 11213 if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { 11214 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 11215 | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, REG0 11216 } else { 11217 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11218 | EXT_CALL zend_jit_fetch_dim_str_r_helper, REG0 11219 } 11220 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 11221 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 11222 } else { 11223 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11224 | LOAD_ZVAL_ADDR CARG3, res_addr 11225 | EXT_CALL zend_jit_fetch_dim_str_is_helper, REG0 11226 } 11227 if ((op1_info & MAY_BE_ARRAY) || 11228 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { 11229 | b >9 // END 11230 } 11231 |6: 11232 } 11233 11234 if (op1_info & MAY_BE_OBJECT) { 11235 may_throw = 1; 11236 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { 11237 if (exit_addr) { 11238 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 11239 } else { 11240 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6, ZREG_TMP1 11241 } 11242 } 11243 | SET_EX_OPLINE opline, REG0 11244 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11245 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11246 } 11247 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11248 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11249 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 11250 } else { 11251 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11252 } 11253 | LOAD_ZVAL_ADDR CARG3, res_addr 11254 if (opline->opcode != ZEND_FETCH_DIM_IS) { 11255 | EXT_CALL zend_jit_fetch_dim_obj_r_helper, REG0 11256 } else { 11257 | EXT_CALL zend_jit_fetch_dim_obj_is_helper, REG0 11258 } 11259 if ((op1_info & MAY_BE_ARRAY) || 11260 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 11261 | b >9 // END 11262 } 11263 |6: 11264 } 11265 11266 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) 11267 && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 11268 if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { 11269 | SET_EX_OPLINE opline, REG0 11270 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { 11271 may_throw = 1; 11272 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 11273 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 11274 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 11275 | EXT_CALL zend_jit_undefined_op_helper, REG0 11276 |1: 11277 } 11278 11279 if (op2_info & MAY_BE_UNDEF) { 11280 may_throw = 1; 11281 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 11282 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 11283 | EXT_CALL zend_jit_undefined_op_helper, REG0 11284 |1: 11285 } 11286 } 11287 11288 if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { 11289 may_throw = 1; 11290 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { 11291 | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr 11292 } else { 11293 | SET_EX_OPLINE opline, REG0 11294 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || 11295 Z_REG(op1_addr) != ZREG_FCARG1 || 11296 Z_OFFSET(op1_addr) != 0) { 11297 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11298 } 11299 } 11300 | EXT_CALL zend_jit_invalid_array_access, REG0 11301 } 11302 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 11303 if (op1_info & MAY_BE_ARRAY) { 11304 | b >9 // END 11305 } 11306 } 11307 11308 if (op1_info & MAY_BE_ARRAY) { 11309 |.code 11310 } 11311 } 11312 11313 if (op1_info & MAY_BE_ARRAY) { 11314 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 11315 11316 |8: 11317 if (res_exit_addr) { 11318 uint32_t type = concrete_type(res_info); 11319 if ((op1_info & MAY_BE_ARRAY_OF_REF) 11320 && dim_type != IS_UNKNOWN 11321 && dim_type != IS_REFERENCE) { 11322 if (type < IS_STRING) { 11323 | IF_NOT_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1 11324 |.cold_code 11325 |1: 11326 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, &res_exit_addr, ZREG_TMP1 11327 | GET_Z_PTR REG0, REG0 11328 | add REG0, REG0, #offsetof(zend_reference, val) 11329 | IF_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1 11330 | b &res_exit_addr 11331 |.code 11332 |1: 11333 } else { 11334 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 11335 | GET_LOW_8BITS TMP1w, REG2w 11336 | IF_NOT_TYPE TMP1w, type, >1 11337 |.cold_code 11338 |1: 11339 | IF_NOT_TYPE TMP1w, IS_REFERENCE, &res_exit_addr 11340 | GET_Z_PTR REG0, REG0 11341 | add REG0, REG0, #offsetof(zend_reference, val) 11342 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 11343 | GET_LOW_8BITS TMP1w, REG2w 11344 | IF_TYPE TMP1w, type, >1 11345 | b &res_exit_addr 11346 |.code 11347 |1: 11348 } 11349 } else { 11350 if (op1_info & MAY_BE_ARRAY_OF_REF) { 11351 | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w 11352 } 11353 if (type < IS_STRING) { 11354 | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr, ZREG_TMP1 11355 } else { 11356 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 11357 | GET_LOW_8BITS TMP1w, REG2w 11358 | IF_NOT_TYPE TMP1w, type, &res_exit_addr 11359 } 11360 } 11361 | // ZVAL_COPY 11362 |7: 11363 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 11364 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 11365 if (type < IS_STRING) { 11366 if (Z_REG(res_addr) != ZREG_FP || 11367 JIT_G(current_frame) == NULL || 11368 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 11369 | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 11370 } 11371 } else { 11372 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 11373 if (!result_avoid_refcounting) { 11374 | TRY_ADDREF res_info, REG2w, REG1, TMP1w 11375 } 11376 } 11377 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 11378 return 0; 11379 } 11380 } else if (op1_info & MAY_BE_ARRAY_OF_REF) { 11381 | // ZVAL_COPY_DEREF 11382 | GET_ZVAL_TYPE_INFO Rw(ZREG_REG2), val_addr, TMP1 11383 if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_REG2)) { 11384 return 0; 11385 } 11386 } else { 11387 | // ZVAL_COPY 11388 | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 11389 | TRY_ADDREF res_info, REG1w, REG2, TMP1w 11390 } 11391 } 11392 |9: // END 11393 11394#ifdef ZEND_JIT_USE_RC_INFERENCE 11395 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 11396 /* Magic offsetGet() may increase refcount of the key */ 11397 op2_info |= MAY_BE_RCN; 11398 } 11399#endif 11400 11401 if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) { 11402 if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) { 11403 may_throw = 1; 11404 } 11405 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11406 } 11407 if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { 11408 if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) { 11409 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 11410 may_throw = 1; 11411 } 11412 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11413 } 11414 } 11415 11416 if (may_throw) { 11417 if (!zend_jit_check_exception(Dst)) { 11418 return 0; 11419 } 11420 } 11421 11422 return 1; 11423} 11424 11425static int zend_jit_fetch_dim(dasm_State **Dst, 11426 const zend_op *opline, 11427 uint32_t op1_info, 11428 zend_jit_addr op1_addr, 11429 uint32_t op2_info, 11430 zend_jit_addr res_addr, 11431 uint8_t dim_type) 11432{ 11433 zend_jit_addr op2_addr; 11434 int may_throw = 0; 11435 11436 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 11437 11438 if (opline->opcode == ZEND_FETCH_DIM_RW) { 11439 | SET_EX_OPLINE opline, REG0 11440 } 11441 if (op1_info & MAY_BE_REF) { 11442 may_throw = 1; 11443 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11444 | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w 11445 | GET_Z_PTR FCARG2x, FCARG1x 11446 | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] 11447 | cmp TMP1w, #IS_ARRAY 11448 | bne >2 11449 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) 11450 | b >3 11451 |.cold_code 11452 |2: 11453 | SET_EX_OPLINE opline, REG0 11454 if (opline->opcode != ZEND_FETCH_DIM_RW) { 11455 | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 11456 } 11457 | mov FCARG1x, RETVALx 11458 | cbnz FCARG1x, >1 11459 | b ->exception_handler_undef 11460 |.code 11461 |1: 11462 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11463 } 11464 11465 if (op1_info & MAY_BE_ARRAY) { 11466 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11467 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 11468 } 11469 |3: 11470 | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 11471 } 11472 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 11473 if (op1_info & MAY_BE_ARRAY) { 11474 |.cold_code 11475 |7: 11476 } 11477 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 11478 | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 11479 | bgt >7 11480 } 11481 if (Z_REG(op1_addr) != ZREG_FP) { 11482 | str Rx(Z_REG(op1_addr)), T1 // save 11483 } 11484 if ((op1_info & MAY_BE_UNDEF) 11485 && opline->opcode == ZEND_FETCH_DIM_RW) { 11486 may_throw = 1; 11487 if (op1_info & MAY_BE_NULL) { 11488 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 11489 } 11490 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 11491 | EXT_CALL zend_jit_undefined_op_helper, REG0 11492 |1: 11493 } 11494 | // ZVAL_ARR(container, zend_new_array(8)); 11495 | EXT_CALL _zend_new_array_0, REG0 11496 | mov REG0, RETVALx 11497 if (Z_REG(op1_addr) != ZREG_FP) { 11498 | ldr Rx(Z_REG(op1_addr)), T1 // restore 11499 } 11500 | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 11501 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 11502 | mov FCARG1x, REG0 11503 if (op1_info & MAY_BE_ARRAY) { 11504 | b >1 11505 |.code 11506 |1: 11507 } 11508 } 11509 11510 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 11511 |6: 11512 if (opline->op2_type == IS_UNUSED) { 11513 may_throw = 1; 11514 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 11515 | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval 11516 | EXT_CALL zend_hash_next_index_insert, REG0 11517 | // if (UNEXPECTED(!var_ptr)) { 11518 | cbz RETVALx, >1 11519 |.cold_code 11520 |1: 11521 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 11522 | CANNOT_ADD_ELEMENT opline 11523 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 11524 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 11525 | b >8 11526 |.code 11527 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 11528 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 11529 } else { 11530 uint32_t type; 11531 11532 switch (opline->opcode) { 11533 case ZEND_FETCH_DIM_W: 11534 case ZEND_FETCH_LIST_W: 11535 type = BP_VAR_W; 11536 break; 11537 case ZEND_FETCH_DIM_RW: 11538 may_throw = 1; 11539 type = BP_VAR_RW; 11540 break; 11541 case ZEND_FETCH_DIM_UNSET: 11542 type = BP_VAR_UNSET; 11543 break; 11544 default: 11545 ZEND_UNREACHABLE(); 11546 } 11547 11548 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 11549 may_throw = 1; 11550 } 11551 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 11552 return 0; 11553 } 11554 11555 |8: 11556 | SET_ZVAL_PTR res_addr, REG0, TMP1 11557 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 11558 11559 if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 11560 |.cold_code 11561 |9: 11562 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 11563 | b >8 11564 |.code 11565 } 11566 } 11567 } 11568 11569 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 11570 may_throw = 1; 11571 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 11572 |.cold_code 11573 |7: 11574 } 11575 11576 if (opline->opcode != ZEND_FETCH_DIM_RW) { 11577 | SET_EX_OPLINE opline, REG0 11578 } 11579 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11580 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11581 } 11582 if (opline->op2_type == IS_UNUSED) { 11583 | mov FCARG2x, xzr 11584 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11585 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11586 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 11587 } else { 11588 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11589 } 11590 | LOAD_ZVAL_ADDR CARG3, res_addr 11591 switch (opline->opcode) { 11592 case ZEND_FETCH_DIM_W: 11593 case ZEND_FETCH_LIST_W: 11594 | EXT_CALL zend_jit_fetch_dim_obj_w_helper, REG0 11595 break; 11596 case ZEND_FETCH_DIM_RW: 11597 | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, REG0 11598 break; 11599// case ZEND_FETCH_DIM_UNSET: 11600// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, REG0 11601// break; 11602 default: 11603 ZEND_UNREACHABLE(); 11604 } 11605 11606 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 11607 | b >8 // END 11608 |.code 11609 } 11610 } 11611 11612#ifdef ZEND_JIT_USE_RC_INFERENCE 11613 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 11614 /* ASSIGN_DIM may increase refcount of the key */ 11615 op2_info |= MAY_BE_RCN; 11616 } 11617#endif 11618 11619 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 11620 && (op2_info & MAY_HAVE_DTOR) 11621 && (op2_info & MAY_BE_RC1)) { 11622 may_throw = 1; 11623 } 11624 |8: 11625 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11626 11627 if (may_throw) { 11628 if (!zend_jit_check_exception(Dst)) { 11629 return 0; 11630 } 11631 } 11632 return 1; 11633} 11634 11635static int zend_jit_isset_isempty_dim(dasm_State **Dst, 11636 const zend_op *opline, 11637 uint32_t op1_info, 11638 zend_jit_addr op1_addr, 11639 bool op1_avoid_refcounting, 11640 uint32_t op2_info, 11641 uint8_t dim_type, 11642 int may_throw, 11643 uint8_t smart_branch_opcode, 11644 uint32_t target_label, 11645 uint32_t target_label2, 11646 const void *exit_addr) 11647{ 11648 zend_jit_addr op2_addr, res_addr; 11649 11650 // TODO: support for empty() ??? 11651 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 11652 11653 op2_addr = OP2_ADDR(); 11654 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11655 11656 if (op1_info & MAY_BE_REF) { 11657 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11658 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 11659 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11660 } 11661 11662 if (op1_info & MAY_BE_ARRAY) { 11663 const void *found_exit_addr = NULL; 11664 const void *not_found_exit_addr = NULL; 11665 11666 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11667 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 11668 } 11669 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 11670 if (exit_addr 11671 && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) 11672 && !may_throw 11673 && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) 11674 && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { 11675 if (smart_branch_opcode == ZEND_JMPNZ) { 11676 found_exit_addr = exit_addr; 11677 } else { 11678 not_found_exit_addr = exit_addr; 11679 } 11680 } 11681 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, dim_type, found_exit_addr, not_found_exit_addr, NULL)) { 11682 return 0; 11683 } 11684 11685 if (found_exit_addr) { 11686 |9: 11687 return 1; 11688 } else if (not_found_exit_addr) { 11689 |8: 11690 return 1; 11691 } 11692 } 11693 11694 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 11695 if (op1_info & MAY_BE_ARRAY) { 11696 |.cold_code 11697 |7: 11698 } 11699 11700 if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { 11701 | SET_EX_OPLINE opline, REG0 11702 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11703 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 11704 } 11705 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11706 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11707 | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) 11708 } else { 11709 | LOAD_ZVAL_ADDR FCARG2x, op2_addr 11710 } 11711 | EXT_CALL zend_jit_isset_dim_helper, REG0 11712 | cbz RETVALw, >9 11713 if (op1_info & MAY_BE_ARRAY) { 11714 | b >8 11715 |.code 11716 } 11717 } else { 11718 if (op2_info & MAY_BE_UNDEF) { 11719 if (op2_info & MAY_BE_ANY) { 11720 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 11721 } 11722 | SET_EX_OPLINE opline, REG0 11723 | LOAD_32BIT_VAL FCARG1w, opline->op2.var 11724 | EXT_CALL zend_jit_undefined_op_helper, REG0 11725 |1: 11726 } 11727 if (op1_info & MAY_BE_ARRAY) { 11728 | b >9 11729 |.code 11730 } 11731 } 11732 } 11733 11734#ifdef ZEND_JIT_USE_RC_INFERENCE 11735 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 11736 /* Magic offsetExists() may increase refcount of the key */ 11737 op2_info |= MAY_BE_RCN; 11738 } 11739#endif 11740 11741 if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { 11742 |8: 11743 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11744 if (!op1_avoid_refcounting) { 11745 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11746 } 11747 if (may_throw) { 11748 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 11749 return 0; 11750 } 11751 } 11752 if (!(opline->extended_value & ZEND_ISEMPTY)) { 11753 if (exit_addr) { 11754 if (smart_branch_opcode == ZEND_JMPNZ) { 11755 | b &exit_addr 11756 } else { 11757 | b >8 11758 } 11759 } else if (smart_branch_opcode) { 11760 if (smart_branch_opcode == ZEND_JMPZ) { 11761 | b =>target_label2 11762 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11763 | b =>target_label 11764 } else { 11765 ZEND_UNREACHABLE(); 11766 } 11767 } else { 11768 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 11769 | b >8 11770 } 11771 } else { 11772 | NIY // TODO: support for empty() 11773 } 11774 } 11775 11776 |9: // not found 11777 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11778 if (!op1_avoid_refcounting) { 11779 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 11780 } 11781 if (may_throw) { 11782 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 11783 return 0; 11784 } 11785 } 11786 if (!(opline->extended_value & ZEND_ISEMPTY)) { 11787 if (exit_addr) { 11788 if (smart_branch_opcode == ZEND_JMPZ) { 11789 | b &exit_addr 11790 } 11791 } else if (smart_branch_opcode) { 11792 if (smart_branch_opcode == ZEND_JMPZ) { 11793 | b =>target_label 11794 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11795 } else { 11796 ZEND_UNREACHABLE(); 11797 } 11798 } else { 11799 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 11800 } 11801 } else { 11802 | NIY // TODO: support for empty() 11803 } 11804 11805 |8: 11806 11807 return 1; 11808} 11809 11810static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 11811{ 11812 zend_jit_addr op1_addr = OP1_ADDR(); 11813 zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); 11814 11815 | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; 11816 | ldr FCARG2x, EX->run_time_cache 11817 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG2x, opline->extended_value, TMP1 11818 | sub REG0, REG0, #1 11819 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) 11820 | MEM_LOAD_32_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1 11821 | cmp REG0, REG1, lsl #5 11822 | bhs >9 11823 | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); 11824 | MEM_LOAD_64_ZTS ldr, TMP1, executor_globals, symbol_table.arData, REG1 11825 | add REG0, REG0, TMP1 11826 | IF_NOT_Z_TYPE REG0, IS_REFERENCE, >9, TMP1w 11827 | // (EXPECTED(p->key == varname)) 11828 | ldr TMP1, [REG0, #offsetof(Bucket, key)] 11829 | LOAD_ADDR TMP2, varname 11830 | cmp TMP1, TMP2 11831 | bne >9 11832 | GET_Z_PTR REG0, REG0 11833 | GC_ADDREF REG0, TMP1w 11834 |1: 11835 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11836 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11837 | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) 11838 | IF_ZVAL_REFCOUNTED op1_addr, >2, ZREG_TMP1, ZREG_TMP2 11839 |.cold_code 11840 |2: 11841 } 11842 | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); 11843 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 11844 | // ZVAL_REF(variable_ptr, ref) 11845 | SET_ZVAL_PTR op1_addr, REG0, TMP1 11846 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 11847 | // if (GC_DELREF(garbage) == 0) 11848 | GC_DELREF FCARG1x, TMP1w 11849 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 11850 | bne >3 11851 } else { 11852 | bne >5 11853 } 11854 | ZVAL_DTOR_FUNC op1_info, opline, TMP1 11855 | b >5 11856 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 11857 |3: 11858 | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) 11859 | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w 11860 | EXT_CALL gc_possible_root, REG0 11861 | b >5 11862 } 11863 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11864 |.code 11865 } 11866 } 11867 11868 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11869 | // ZVAL_REF(variable_ptr, ref) 11870 | SET_ZVAL_PTR op1_addr, REG0, TMP1 11871 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 11872 } 11873 |5: 11874 //END of handler 11875 11876 |.cold_code 11877 |9: 11878 | LOAD_ADDR FCARG1x, (ptrdiff_t)varname 11879 if (opline->extended_value) { 11880 | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, FCARG2x, opline->extended_value, TMP1 11881 } 11882 | EXT_CALL zend_jit_fetch_global_helper, REG0 11883 | mov REG0, RETVALx 11884 | b <1 11885 |.code 11886 11887 return 1; 11888} 11889 11890static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception) 11891{ 11892 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11893 bool in_cold = 0; 11894 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 11895 zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1 : ZREG_REG0; 11896 11897 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 11898 && JIT_G(current_frame) 11899 && JIT_G(current_frame)->prev) { 11900 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 11901 uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)); 11902 11903 if (type != IS_UNKNOWN && (type_mask & (1u << type))) { 11904 return 1; 11905 } 11906 } 11907 11908 if (ZEND_ARG_SEND_MODE(arg_info)) { 11909 if (opline->opcode == ZEND_RECV_INIT) { 11910 | LOAD_ZVAL_ADDR Rx(tmp_reg), res_addr 11911 | ZVAL_DEREF Rx(tmp_reg), MAY_BE_REF, TMP1w 11912 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); 11913 } else { 11914 | GET_ZVAL_PTR Rx(tmp_reg), res_addr, TMP1 11915 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); 11916 } 11917 } 11918 11919 if (type_mask != 0) { 11920 if (is_power_of_two(type_mask)) { 11921 uint32_t type_code = concrete_type(type_mask); 11922 | IF_NOT_ZVAL_TYPE res_addr, type_code, >1, ZREG_TMP1 11923 } else { 11924 | mov REG2w, #1 11925 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, Rx(Z_REG(res_addr)), Z_OFFSET(res_addr)+offsetof(zval, u1.v.type), TMP1 11926 | lsl REG2w, REG2w, REG1w 11927 | TST_32_WITH_CONST REG2w, type_mask, TMP1w 11928 | beq >1 11929 } 11930 11931 |.cold_code 11932 |1: 11933 11934 in_cold = 1; 11935 } 11936 11937 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 11938 | LOAD_ZVAL_ADDR FCARG1x, res_addr 11939 } 11940 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11941 | SET_EX_OPLINE opline, REG0 11942 } else { 11943 | ADDR_STORE EX->opline, opline, REG0 11944 } 11945 | LOAD_ADDR FCARG2x, (ptrdiff_t)arg_info 11946 | EXT_CALL zend_jit_verify_arg_slow, REG0 11947 11948 if (check_exception) { 11949 | GET_LOW_8BITS REG0w, RETVALw 11950 if (in_cold) { 11951 | cbnz REG0w, >1 11952 | b ->exception_handler 11953 |.code 11954 |1: 11955 } else { 11956 | cbz REG0w, ->exception_handler 11957 } 11958 } else if (in_cold) { 11959 | b >1 11960 |.code 11961 |1: 11962 } 11963 11964 return 1; 11965} 11966 11967static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) 11968{ 11969 uint32_t arg_num = opline->op1.num; 11970 zend_arg_info *arg_info = NULL; 11971 11972 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 11973 if (EXPECTED(arg_num <= op_array->num_args)) { 11974 arg_info = &op_array->arg_info[arg_num-1]; 11975 } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { 11976 arg_info = &op_array->arg_info[op_array->num_args]; 11977 } 11978 if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { 11979 arg_info = NULL; 11980 } 11981 } 11982 11983 if (arg_info || (opline+1)->opcode != ZEND_RECV) { 11984 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11985 if (!JIT_G(current_frame) || 11986 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 || 11987 arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 11988 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 11989 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11990 11991 if (!exit_addr) { 11992 return 0; 11993 } 11994 | ldr TMP1w, EX->This.u2.num_args 11995 | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w 11996 | blo &exit_addr 11997 } 11998 } else { 11999 | ldr TMP1w, EX->This.u2.num_args 12000 | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w 12001 | blo >1 12002 |.cold_code 12003 |1: 12004 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12005 | SET_EX_OPLINE opline, REG0 12006 } else { 12007 | ADDR_STORE EX->opline, opline, REG0 12008 } 12009 | mov FCARG1x, FP 12010 | EXT_CALL zend_missing_arg_error, REG0 12011 | b ->exception_handler 12012 |.code 12013 } 12014 } 12015 12016 if (arg_info) { 12017 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { 12018 return 0; 12019 } 12020 } 12021 12022 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12023 if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { 12024 | LOAD_IP_ADDR (opline + 1) 12025 zend_jit_set_last_valid_opline(opline + 1); 12026 } 12027 } 12028 12029 return 1; 12030} 12031 12032static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw) 12033{ 12034 uint32_t arg_num = opline->op1.num; 12035 zval *zv = RT_CONSTANT(opline, opline->op2); 12036 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12037 12038 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12039 && JIT_G(current_frame) 12040 && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) { 12041 if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 12042 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 12043 if (Z_REFCOUNTED_P(zv)) { 12044 | ADDREF_CONST zv, REG0, TMP1 12045 } 12046 } 12047 } else { 12048 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 12049 (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 12050 | ldr TMP1w, EX->This.u2.num_args 12051 | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w 12052 | bhs >5 12053 } 12054 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 12055 if (Z_REFCOUNTED_P(zv)) { 12056 | ADDREF_CONST zv, REG0, TMP1 12057 } 12058 } 12059 12060 if (Z_CONSTANT_P(zv)) { 12061 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12062 | SET_EX_OPLINE opline, REG0 12063 } else { 12064 | ADDR_STORE EX->opline, opline, REG0 12065 } 12066 | LOAD_ZVAL_ADDR FCARG1x, res_addr 12067 | ldr REG0, EX->func 12068 | ldr FCARG2x, [REG0, #offsetof(zend_op_array, scope)] 12069 | EXT_CALL zval_update_constant_ex, REG0 12070 | cbnz RETVALw, >1 12071 |.cold_code 12072 |1: 12073 | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline, ZREG_TMP1, ZREG_TMP2 12074 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 12075 | b ->exception_handler 12076 |.code 12077 } 12078 12079 |5: 12080 12081 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12082 do { 12083 zend_arg_info *arg_info; 12084 12085 if (arg_num <= op_array->num_args) { 12086 arg_info = &op_array->arg_info[arg_num-1]; 12087 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { 12088 arg_info = &op_array->arg_info[op_array->num_args]; 12089 } else { 12090 break; 12091 } 12092 if (!ZEND_TYPE_IS_SET(arg_info->type)) { 12093 break; 12094 } 12095 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { 12096 return 0; 12097 } 12098 } while (0); 12099 } 12100 12101 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12102 if (is_last) { 12103 | LOAD_IP_ADDR (opline + 1) 12104 zend_jit_set_last_valid_opline(opline + 1); 12105 } 12106 } 12107 return 1; 12108} 12109 12110static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) 12111{ 12112 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 12113 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12114 12115 if (!exit_addr) { 12116 return 0; 12117 } 12118 12119 | LOAD_ADDR TMP1, ((ptrdiff_t)ce) 12120 | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] 12121 | cmp TMP2, TMP1 12122 | bne &exit_addr 12123 12124 return 1; 12125} 12126 12127static int zend_jit_fetch_obj(dasm_State **Dst, 12128 const zend_op *opline, 12129 const zend_op_array *op_array, 12130 zend_ssa *ssa, 12131 const zend_ssa_op *ssa_op, 12132 uint32_t op1_info, 12133 zend_jit_addr op1_addr, 12134 bool op1_indirect, 12135 zend_class_entry *ce, 12136 bool ce_is_instanceof, 12137 bool on_this, 12138 bool delayed_fetch_this, 12139 bool op1_avoid_refcounting, 12140 zend_class_entry *trace_ce, 12141 uint8_t prop_type, 12142 int may_throw) 12143{ 12144 zval *member; 12145 zend_property_info *prop_info; 12146 bool may_be_dynamic = 1; 12147 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12148 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 12149 zend_jit_addr prop_addr; 12150 uint32_t res_info = RES_INFO(); 12151 bool type_loaded = 0; 12152 12153 ZEND_ASSERT(opline->op2_type == IS_CONST); 12154 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 12155 12156 member = RT_CONSTANT(opline, opline->op2); 12157 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 12158 prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); 12159 12160 if (on_this) { 12161 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 12162 } else { 12163 if (opline->op1_type == IS_VAR 12164 && opline->opcode == ZEND_FETCH_OBJ_W 12165 && (op1_info & MAY_BE_INDIRECT) 12166 && Z_REG(op1_addr) == ZREG_FP) { 12167 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12168 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 12169 | GET_Z_PTR FCARG1x, FCARG1x 12170 |1: 12171 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12172 } 12173 if (op1_info & MAY_BE_REF) { 12174 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12175 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12176 } 12177 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 12178 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12179 } 12180 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 12181 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12182 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12183 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12184 12185 if (!exit_addr) { 12186 return 0; 12187 } 12188 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 12189 } else { 12190 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, ZREG_TMP1 12191 } 12192 } 12193 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 12194 } 12195 12196 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12197 prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename); 12198 if (prop_info) { 12199 ce = trace_ce; 12200 ce_is_instanceof = 0; 12201 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 12202 if (on_this && JIT_G(current_frame) 12203 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 12204 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 12205 } else if (zend_jit_class_guard(Dst, opline, ce)) { 12206 if (on_this && JIT_G(current_frame)) { 12207 JIT_G(current_frame)->ce = ce; 12208 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 12209 } 12210 } else { 12211 return 0; 12212 } 12213 if (ssa->var_info && ssa_op->op1_use >= 0) { 12214 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 12215 ssa->var_info[ssa_op->op1_use].ce = ce; 12216 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 12217 } 12218 } 12219 } 12220 } 12221 12222 if (!prop_info) { 12223 | ldr REG0, EX->run_time_cache 12224 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS), TMP1 12225 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 12226 | cmp REG2, TMP1 12227 | bne >5 12228 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)), TMP1 12229 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); 12230 if (may_be_dynamic) { 12231 | tst REG0, REG0 12232 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12233 | blt >5 12234 } else { 12235 | blt >8 // dynamic property 12236 } 12237 } 12238 | add TMP1, FCARG1x, REG0 12239 | ldr REG2w, [TMP1, #offsetof(zval,u1.type_info)] 12240 | IF_UNDEF REG2w, >5 12241 | mov FCARG1x, TMP1 12242 type_loaded = 1; 12243 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12244 if (opline->opcode == ZEND_FETCH_OBJ_W 12245 && (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) { 12246 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 12247 12248 | ldr REG0, EX->run_time_cache 12249 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2), TMP1 12250 | cbnz FCARG2x, >1 12251 |.cold_code 12252 |1: 12253 | ldr TMP1w, [FCARG2x, #offsetof(zend_property_info, flags)] 12254 | tst TMP1w, #ZEND_ACC_READONLY 12255 if (flags) { 12256 | beq >3 12257 } else { 12258 | beq >4 12259 } 12260 | IF_NOT_TYPE REG2w, IS_OBJECT_EX, >2 12261 | GET_Z_PTR REG2, FCARG1x 12262 | GC_ADDREF REG2, TMP1w 12263 | SET_ZVAL_PTR res_addr, REG2, TMP1 12264 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2 12265 | b >9 12266 |2: 12267 | ldr REG0w, [FCARG1x, #offsetof(zval, u2.extra)] 12268 | TST_32_WITH_CONST REG0w, IS_PROP_REINITABLE, TMP1w 12269 | beq >6 12270 | and REG0w, REG0w, #(~IS_PROP_REINITABLE) 12271 | str REG0w, [FCARG1x, #offsetof(zval, u2.extra)] 12272 if (flags) { 12273 | b >3 12274 } else { 12275 | b >4 12276 } 12277 |6: 12278 | mov FCARG1x, FCARG2x 12279 | SET_EX_OPLINE opline, REG0 12280 | EXT_CALL zend_readonly_property_modification_error, REG0 12281 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 12282 | b >9 12283 |3: 12284 if (flags == ZEND_FETCH_DIM_WRITE) { 12285 | SET_EX_OPLINE opline, REG0 12286 | EXT_CALL zend_jit_check_array_promotion, REG0 12287 | b >9 12288 } else if (flags == ZEND_FETCH_REF) { 12289 | LOAD_ZVAL_ADDR CARG3, res_addr 12290 | EXT_CALL zend_jit_create_typed_ref, REG0 12291 | b >9 12292 } else { 12293 ZEND_ASSERT(flags == 0); 12294 } 12295 |.code 12296 |4: 12297 } 12298 } else { 12299 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 12300 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12301 if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { 12302 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 12303 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12304 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12305 12306 if (!exit_addr) { 12307 return 0; 12308 } 12309 type_loaded = 1; 12310 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12311 | IF_UNDEF REG2w, &exit_addr 12312 } 12313 } else { 12314 type_loaded = 1; 12315 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12316 | IF_UNDEF REG2w, >5 12317 } 12318 if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) { 12319 if (!type_loaded) { 12320 type_loaded = 1; 12321 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12322 } 12323 | IF_NOT_TYPE REG2w, IS_OBJECT_EX, >4 12324 | GET_ZVAL_PTR REG2, prop_addr, TMP1 12325 | GC_ADDREF REG2, TMP1w 12326 | SET_ZVAL_PTR res_addr, REG2, TMP1 12327 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2 12328 | b >9 12329 |.cold_code 12330 |4: 12331 | ldr REG0w, [FCARG1x, #(prop_info->offset + offsetof(zval, u2.extra))] 12332 | TST_32_WITH_CONST REG0w, IS_PROP_REINITABLE, TMP1w 12333 | beq >6 12334 | and REG0w, REG0w, #(~IS_PROP_REINITABLE) 12335 | str REG0w, [FCARG1x, #(prop_info->offset + offsetof(zval, u2.extra))] 12336 | b >4 12337 |6: 12338 | LOAD_ADDR FCARG1x, prop_info 12339 | SET_EX_OPLINE opline, REG0 12340 | EXT_CALL zend_readonly_property_modification_error, REG0 12341 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 12342 | b >9 12343 |.code 12344 |4: 12345 } 12346 if (opline->opcode == ZEND_FETCH_OBJ_W 12347 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) 12348 && ZEND_TYPE_IS_SET(prop_info->type)) { 12349 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 12350 12351 if (flags == ZEND_FETCH_DIM_WRITE) { 12352 if ((ZEND_TYPE_FULL_MASK(prop_info->type) & MAY_BE_ARRAY) == 0) { 12353 if (!type_loaded) { 12354 type_loaded = 1; 12355 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12356 } 12357 | cmp REG2w, #IS_FALSE 12358 | ble >1 12359 |.cold_code 12360 |1: 12361 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12362 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12363 } 12364 | LOAD_ADDR FCARG2x, prop_info 12365 | SET_EX_OPLINE opline, REG0 12366 | EXT_CALL zend_jit_check_array_promotion, REG0 12367 | b >9 12368 |.code 12369 } 12370 } else if (flags == ZEND_FETCH_REF) { 12371 if (!type_loaded) { 12372 type_loaded = 1; 12373 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12374 } 12375 | GET_LOW_8BITS TMP1w, REG2w 12376 | IF_TYPE TMP1w, IS_REFERENCE, >1 12377 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 12378 | LOAD_ADDR FCARG2x, prop_info 12379 } else { 12380 int prop_info_offset = 12381 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 12382 12383 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 12384 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 12385 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 12386 } 12387 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12388 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12389 } 12390 | LOAD_ZVAL_ADDR CARG3, res_addr 12391 | EXT_CALL zend_jit_create_typed_ref, REG0 12392 | b >9 12393 |1: 12394 } else { 12395 ZEND_UNREACHABLE(); 12396 } 12397 } 12398 } 12399 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12400 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12401 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12402 } 12403 | SET_ZVAL_PTR res_addr, FCARG1x, TMP1 12404 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 12405 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { 12406 ssa->var_info[ssa_op->result_def].indirect_reference = 1; 12407 } 12408 } else { 12409 bool result_avoid_refcounting = 0; 12410 12411 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { 12412 uint32_t flags = 0; 12413 uint32_t old_info; 12414 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 12415 int32_t exit_point; 12416 const void *exit_addr; 12417 uint32_t type; 12418 zend_jit_addr val_addr = prop_addr; 12419 12420 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 12421 && !delayed_fetch_this 12422 && !op1_avoid_refcounting) { 12423 flags = ZEND_JIT_EXIT_FREE_OP1; 12424 } 12425 12426 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 12427 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 12428 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 12429 && (ssa_op+1)->op1_use == ssa_op->result_def 12430 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 12431 result_avoid_refcounting = 1; 12432 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 12433 } 12434 12435 type = concrete_type(res_info); 12436 12437 if (prop_type != IS_UNKNOWN 12438 && prop_type != IS_UNDEF 12439 && prop_type != IS_REFERENCE 12440 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) { 12441 exit_point = zend_jit_trace_get_exit_point(opline, 0); 12442 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12443 if (!exit_addr) { 12444 return 0; 12445 } 12446 } else { 12447 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 12448 | LOAD_ZVAL_ADDR REG0, prop_addr 12449 if (op1_avoid_refcounting) { 12450 SET_STACK_REG(JIT_G(current_frame)->stack, 12451 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 12452 } 12453 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 12454 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 12455 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 12456 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 12457 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 12458 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12459 if (!exit_addr) { 12460 return 0; 12461 } 12462 12463 if (!type_loaded) { 12464 type_loaded = 1; 12465 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 12466 } 12467 | // ZVAL_DEREF() 12468 | GET_LOW_8BITS TMP1w, REG2w 12469 | IF_NOT_TYPE TMP1w, IS_REFERENCE, >1 12470 | GET_Z_PTR REG0, REG0 12471 | add REG0, REG0, #offsetof(zend_reference, val) 12472 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 12473 } 12474 res_info &= ~MAY_BE_GUARD; 12475 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 12476 if (type < IS_STRING) { 12477 |1: 12478 if (type_loaded) { 12479 | IF_NOT_TYPE REG2w, type, &exit_addr 12480 } else { 12481 | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr, ZREG_TMP1 12482 } 12483 } else { 12484 if (!type_loaded) { 12485 type_loaded = 1; 12486 | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 12487 } 12488 |1: 12489 | GET_LOW_8BITS TMP1w, REG2w 12490 | IF_NOT_TYPE TMP1w, type, &exit_addr 12491 } 12492 | // ZVAL_COPY 12493 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 12494 if (type < IS_STRING) { 12495 if (Z_REG(res_addr) != ZREG_FP || 12496 JIT_G(current_frame) == NULL || 12497 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 12498 | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 12499 } 12500 } else { 12501 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 12502 if (!result_avoid_refcounting) { 12503 | TRY_ADDREF res_info, REG2w, REG1, TMP1w 12504 } 12505 } 12506 } else { 12507 if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_REG2)) { 12508 return 0; 12509 } 12510 } 12511 } 12512 12513 if (op1_avoid_refcounting) { 12514 SET_STACK_REG(JIT_G(current_frame)->stack, 12515 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 12516 } 12517 12518 |.cold_code 12519 12520 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { 12521 |5: 12522 | SET_EX_OPLINE opline, REG0 12523 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12524 | EXT_CALL zend_jit_fetch_obj_w_slow, REG0 12525 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { 12526 | EXT_CALL zend_jit_fetch_obj_r_slow, REG0 12527 } else { 12528 | EXT_CALL zend_jit_fetch_obj_is_slow, REG0 12529 } 12530 | b >9 12531 } 12532 12533 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12534 |7: 12535 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 12536 | SET_EX_OPLINE opline, REG0 12537 if (opline->opcode != ZEND_FETCH_OBJ_W 12538 && (op1_info & MAY_BE_UNDEF)) { 12539 zend_jit_addr orig_op1_addr = OP1_ADDR(); 12540 12541 if (op1_info & MAY_BE_ANY) { 12542 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 12543 } 12544 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 12545 | EXT_CALL zend_jit_undefined_op_helper, REG0 12546 |1: 12547 | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr 12548 } else if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12549 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12550 } 12551 | LOAD_ADDR FCARG2x, Z_STRVAL_P(member) 12552 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12553 | EXT_CALL zend_jit_invalid_property_write, REG0 12554 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 12555 } else { 12556 | EXT_CALL zend_jit_invalid_property_read, REG0 12557 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 12558 } 12559 | b >9 12560 } else { 12561 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 12562 | b >9 12563 } 12564 } 12565 12566 if (!prop_info 12567 && may_be_dynamic 12568 && opline->opcode != ZEND_FETCH_OBJ_W) { 12569 |8: 12570 | mov FCARG2x, REG0 12571 | SET_EX_OPLINE opline, REG0 12572 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 12573 | EXT_CALL zend_jit_fetch_obj_r_dynamic, REG0 12574 } else { 12575 | EXT_CALL zend_jit_fetch_obj_is_dynamic, REG0 12576 } 12577 | b >9 12578 } 12579 12580 |.code; 12581 |9: // END 12582 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 12583 if (opline->op1_type == IS_VAR 12584 && opline->opcode == ZEND_FETCH_OBJ_W 12585 && (op1_info & MAY_BE_RC1)) { 12586 zend_jit_addr orig_op1_addr = OP1_ADDR(); 12587 12588 | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1, ZREG_TMP1, ZREG_TMP2 12589 | GET_ZVAL_PTR FCARG1x, orig_op1_addr, TMP1 12590 | GC_DELREF FCARG1x, TMP1w 12591 | bne >1 12592 | SET_EX_OPLINE opline, REG0 12593 | EXT_CALL zend_jit_extract_helper, REG0 12594 |1: 12595 } else if (!op1_avoid_refcounting) { 12596 if (on_this) { 12597 op1_info &= ~MAY_BE_RC1; 12598 } 12599 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 12600 } 12601 } 12602 12603 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12604 && prop_info 12605 && (opline->opcode != ZEND_FETCH_OBJ_W || 12606 !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) || 12607 !ZEND_TYPE_IS_SET(prop_info->type)) 12608 && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) { 12609 may_throw = 0; 12610 } 12611 12612 if (may_throw) { 12613 if (!zend_jit_check_exception(Dst)) { 12614 return 0; 12615 } 12616 } 12617 12618 return 1; 12619} 12620 12621static int zend_jit_incdec_obj(dasm_State **Dst, 12622 const zend_op *opline, 12623 const zend_op_array *op_array, 12624 zend_ssa *ssa, 12625 const zend_ssa_op *ssa_op, 12626 uint32_t op1_info, 12627 zend_jit_addr op1_addr, 12628 bool op1_indirect, 12629 zend_class_entry *ce, 12630 bool ce_is_instanceof, 12631 bool on_this, 12632 bool delayed_fetch_this, 12633 zend_class_entry *trace_ce, 12634 uint8_t prop_type) 12635{ 12636 zval *member; 12637 zend_string *name; 12638 zend_property_info *prop_info; 12639 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 12640 zend_jit_addr res_addr = 0; 12641 zend_jit_addr prop_addr; 12642 bool needs_slow_path = 0; 12643 bool use_prop_guard = 0; 12644 bool may_throw = 0; 12645 uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0; 12646 12647 ZEND_ASSERT(opline->op2_type == IS_CONST); 12648 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 12649 12650 if (opline->result_type != IS_UNUSED) { 12651 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12652 } 12653 12654 member = RT_CONSTANT(opline, opline->op2); 12655 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 12656 name = Z_STR_P(member); 12657 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 12658 12659 if (on_this) { 12660 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 12661 } else { 12662 if (opline->op1_type == IS_VAR 12663 && (op1_info & MAY_BE_INDIRECT) 12664 && Z_REG(op1_addr) == ZREG_FP) { 12665 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12666 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 12667 | GET_Z_PTR FCARG1x, FCARG1x 12668 |1: 12669 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12670 } 12671 if (op1_info & MAY_BE_REF) { 12672 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12673 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12674 } 12675 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 12676 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12677 } 12678 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 12679 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12680 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12681 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12682 12683 if (!exit_addr) { 12684 return 0; 12685 } 12686 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 12687 } else { 12688 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 12689 |.cold_code 12690 |1: 12691 | SET_EX_OPLINE opline, REG0 12692 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12693 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 12694 } 12695 | LOAD_ADDR FCARG2x, ZSTR_VAL(name) 12696 | EXT_CALL zend_jit_invalid_property_incdec, REG0 12697 | b ->exception_handler 12698 |.code 12699 } 12700 } 12701 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 12702 } 12703 12704 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12705 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 12706 if (prop_info) { 12707 ce = trace_ce; 12708 ce_is_instanceof = 0; 12709 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 12710 if (on_this && JIT_G(current_frame) 12711 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 12712 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 12713 } else if (zend_jit_class_guard(Dst, opline, ce)) { 12714 if (on_this && JIT_G(current_frame)) { 12715 JIT_G(current_frame)->ce = ce; 12716 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 12717 } 12718 } else { 12719 return 0; 12720 } 12721 if (ssa->var_info && ssa_op->op1_use >= 0) { 12722 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 12723 ssa->var_info[ssa_op->op1_use].ce = ce; 12724 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 12725 } 12726 if (ssa->var_info && ssa_op->op1_def >= 0) { 12727 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 12728 ssa->var_info[ssa_op->op1_def].ce = ce; 12729 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 12730 } 12731 } 12732 } 12733 } 12734 12735 use_prop_guard = (prop_type != IS_UNKNOWN 12736 && prop_type != IS_UNDEF 12737 && prop_type != IS_REFERENCE 12738 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 12739 12740 if (!prop_info) { 12741 needs_slow_path = 1; 12742 12743 | ldr REG0, EX->run_time_cache 12744 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 12745 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 12746 | cmp REG2, TMP1 12747 | bne >7 12748 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 12749 | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 12750 | cbnz TMP1, >7 12751 } 12752 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 12753 | tst REG0, REG0 12754 | blt >7 12755 | add TMP1, FCARG1x, REG0 12756 if (!use_prop_guard) { 12757 | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] 12758 | IF_TYPE TMP2w , IS_UNDEF, >7 12759 } 12760 | mov FCARG1x, TMP1 12761 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12762 } else { 12763 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 12764 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 12765 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12766 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12767 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12768 12769 if (!exit_addr) { 12770 return 0; 12771 } 12772 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 12773 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr 12774 } else { 12775 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 12776 | IF_TYPE TMP2w, IS_UNDEF, >7 12777 needs_slow_path = 1; 12778 } 12779 } 12780 if (ZEND_TYPE_IS_SET(prop_info->type)) { 12781 may_throw = 1; 12782 | SET_EX_OPLINE opline, REG0 12783 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 12784 | LOAD_ADDR FCARG2x, prop_info 12785 } else { 12786 int prop_info_offset = 12787 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 12788 12789 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 12790 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 12791 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 12792 } 12793 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12794 if (opline->result_type == IS_UNUSED) { 12795 switch (opline->opcode) { 12796 case ZEND_PRE_INC_OBJ: 12797 case ZEND_POST_INC_OBJ: 12798 | EXT_CALL zend_jit_inc_typed_prop, REG0 12799 break; 12800 case ZEND_PRE_DEC_OBJ: 12801 case ZEND_POST_DEC_OBJ: 12802 | EXT_CALL zend_jit_dec_typed_prop, REG0 12803 break; 12804 default: 12805 ZEND_UNREACHABLE(); 12806 } 12807 } else { 12808 | LOAD_ZVAL_ADDR CARG3, res_addr 12809 switch (opline->opcode) { 12810 case ZEND_PRE_INC_OBJ: 12811 | EXT_CALL zend_jit_pre_inc_typed_prop, REG0 12812 break; 12813 case ZEND_PRE_DEC_OBJ: 12814 | EXT_CALL zend_jit_pre_dec_typed_prop, REG0 12815 break; 12816 case ZEND_POST_INC_OBJ: 12817 | EXT_CALL zend_jit_post_inc_typed_prop, REG0 12818 break; 12819 case ZEND_POST_DEC_OBJ: 12820 | EXT_CALL zend_jit_post_dec_typed_prop, REG0 12821 break; 12822 default: 12823 ZEND_UNREACHABLE(); 12824 } 12825 } 12826 } 12827 } 12828 12829 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 12830 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 12831 zend_jit_addr var_addr = prop_addr; 12832 12833 if (use_prop_guard) { 12834 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 12835 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12836 if (!exit_addr) { 12837 return 0; 12838 } 12839 12840 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1 12841 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 12842 } 12843 12844 if (var_info & MAY_BE_REF) { 12845 may_throw = 1; 12846 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12847 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 12848 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12849 } 12850 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 12851 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 12852 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 12853 | cbnz TMP1, >1 12854 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) 12855 |.cold_code 12856 |1: 12857 if (opline) { 12858 | SET_EX_OPLINE opline, REG0 12859 } 12860 if (opline->result_type == IS_UNUSED) { 12861 | mov FCARG2x, xzr 12862 } else { 12863 | LOAD_ZVAL_ADDR FCARG2x, res_addr 12864 } 12865 switch (opline->opcode) { 12866 case ZEND_PRE_INC_OBJ: 12867 | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 12868 break; 12869 case ZEND_PRE_DEC_OBJ: 12870 | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 12871 break; 12872 case ZEND_POST_INC_OBJ: 12873 | EXT_CALL zend_jit_post_inc_typed_ref, REG0 12874 break; 12875 case ZEND_POST_DEC_OBJ: 12876 | EXT_CALL zend_jit_post_dec_typed_ref, REG0 12877 break; 12878 default: 12879 ZEND_UNREACHABLE(); 12880 } 12881 | b >9 12882 |.code 12883 |2: 12884 } 12885 12886 if (var_info & MAY_BE_LONG) { 12887 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 12888 | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2, ZREG_TMP1 12889 } 12890 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 12891 if (opline->result_type != IS_UNUSED) { 12892 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 12893 } 12894 } 12895 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 12896 | LONG_ADD_SUB_WITH_IMM adds, var_addr, Z_L(1), TMP1, TMP2 12897 } else { 12898 | LONG_ADD_SUB_WITH_IMM subs, var_addr, Z_L(1), TMP1, TMP2 12899 } 12900 | bvs >3 12901 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { 12902 if (opline->result_type != IS_UNUSED) { 12903 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 12904 } 12905 } 12906 |.cold_code 12907 } 12908 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 12909 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 12910 may_throw = 1; 12911 } 12912 if (var_info & MAY_BE_LONG) { 12913 |2: 12914 } 12915 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 12916 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12917 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 12918 } 12919 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 12920 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 12921 | TRY_ADDREF MAY_BE_ANY, REG0w, REG2, TMP1w 12922 } 12923 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 12924 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 12925 | LOAD_ZVAL_ADDR FCARG2x, res_addr 12926 | EXT_CALL zend_jit_pre_inc, REG0 12927 } else { 12928 | EXT_CALL increment_function, REG0 12929 } 12930 } else { 12931 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 12932 | LOAD_ZVAL_ADDR FCARG2x, res_addr 12933 | EXT_CALL zend_jit_pre_dec, REG0 12934 } else { 12935 | EXT_CALL decrement_function, REG0 12936 } 12937 } 12938 if (var_info & MAY_BE_LONG) { 12939 | b >4 12940 } 12941 } 12942 12943 if (var_info & MAY_BE_LONG) { 12944 |3: 12945 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 12946 uint64_t val = 0x43e0000000000000; 12947 | LOAD_64BIT_VAL TMP2, val 12948 | SET_ZVAL_LVAL_FROM_REG var_addr, TMP2, TMP1 12949 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2 12950 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 12951 | SET_ZVAL_LVAL_FROM_REG res_addr, TMP2, TMP1 12952 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 12953 } 12954 } else { 12955 uint64_t val = 0xc3e0000000000000; 12956 | LOAD_64BIT_VAL TMP2, val 12957 | SET_ZVAL_LVAL_FROM_REG var_addr, TMP2, TMP1 12958 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2 12959 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 12960 | SET_ZVAL_LVAL_FROM_REG res_addr, TMP2, TMP1 12961 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 12962 } 12963 } 12964 if (opline->result_type != IS_UNUSED 12965 && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) 12966 && prop_info 12967 && !ZEND_TYPE_IS_SET(prop_info->type) 12968 && (res_info & MAY_BE_GUARD) 12969 && (res_info & MAY_BE_LONG)) { 12970 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 12971 uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 12972 int32_t exit_point; 12973 const void *exit_addr; 12974 12975 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 12976 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 12977 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12978 if (!exit_addr) { 12979 return 0; 12980 } 12981 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 12982 ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD; 12983 | b &exit_addr 12984 |.code 12985 } else { 12986 | b >4 12987 |.code 12988 |4: 12989 } 12990 } 12991 } 12992 12993 if (needs_slow_path) { 12994 may_throw = 1; 12995 |.cold_code 12996 |7: 12997 | SET_EX_OPLINE opline, REG0 12998 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 12999 | LOAD_ADDR FCARG2x, name 13000 | ldr CARG3, EX->run_time_cache 13001 | ADD_SUB_64_WITH_CONST_32 add, CARG3, CARG3, opline->extended_value, TMP1 13002 if (opline->result_type == IS_UNUSED) { 13003 | mov CARG4, xzr 13004 } else { 13005 | LOAD_ZVAL_ADDR CARG4, res_addr 13006 } 13007 13008 switch (opline->opcode) { 13009 case ZEND_PRE_INC_OBJ: 13010 | EXT_CALL zend_jit_pre_inc_obj_helper, REG0 13011 break; 13012 case ZEND_PRE_DEC_OBJ: 13013 | EXT_CALL zend_jit_pre_dec_obj_helper, REG0 13014 break; 13015 case ZEND_POST_INC_OBJ: 13016 | EXT_CALL zend_jit_post_inc_obj_helper, REG0 13017 break; 13018 case ZEND_POST_DEC_OBJ: 13019 | EXT_CALL zend_jit_post_dec_obj_helper, REG0 13020 break; 13021 default: 13022 ZEND_UNREACHABLE(); 13023 } 13024 13025 | b >9 13026 |.code 13027 } 13028 13029 |9: 13030 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13031 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 13032 may_throw = 1; 13033 } 13034 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 13035 } 13036 13037 if (may_throw) { 13038 if (!zend_jit_check_exception(Dst)) { 13039 return 0; 13040 } 13041 } 13042 13043 return 1; 13044} 13045 13046static int zend_jit_assign_obj_op(dasm_State **Dst, 13047 const zend_op *opline, 13048 const zend_op_array *op_array, 13049 zend_ssa *ssa, 13050 const zend_ssa_op *ssa_op, 13051 uint32_t op1_info, 13052 zend_jit_addr op1_addr, 13053 uint32_t val_info, 13054 zend_ssa_range *val_range, 13055 bool op1_indirect, 13056 zend_class_entry *ce, 13057 bool ce_is_instanceof, 13058 bool on_this, 13059 bool delayed_fetch_this, 13060 zend_class_entry *trace_ce, 13061 uint8_t prop_type) 13062{ 13063 zval *member; 13064 zend_string *name; 13065 zend_property_info *prop_info; 13066 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13067 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13068 zend_jit_addr prop_addr; 13069 bool needs_slow_path = 0; 13070 bool use_prop_guard = 0; 13071 bool may_throw = 0; 13072 binary_op_type binary_op = get_binary_op(opline->extended_value); 13073 13074 ZEND_ASSERT(opline->op2_type == IS_CONST); 13075 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13076 ZEND_ASSERT(opline->result_type == IS_UNUSED); 13077 13078 member = RT_CONSTANT(opline, opline->op2); 13079 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13080 name = Z_STR_P(member); 13081 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13082 13083 if (on_this) { 13084 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 13085 } else { 13086 if (opline->op1_type == IS_VAR 13087 && (op1_info & MAY_BE_INDIRECT) 13088 && Z_REG(op1_addr) == ZREG_FP) { 13089 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13090 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 13091 | GET_Z_PTR FCARG1x, FCARG1x 13092 |1: 13093 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13094 } 13095 if (op1_info & MAY_BE_REF) { 13096 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13097 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13098 } 13099 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 13100 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13101 } 13102 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13103 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13104 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13105 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13106 13107 if (!exit_addr) { 13108 return 0; 13109 } 13110 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 13111 } else { 13112 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 13113 |.cold_code 13114 |1: 13115 | SET_EX_OPLINE opline, REG0 13116 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13117 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13118 } 13119 | LOAD_ADDR FCARG2x, ZSTR_VAL(name) 13120 if (op1_info & MAY_BE_UNDEF) { 13121 | EXT_CALL zend_jit_invalid_property_assign_op, REG0 13122 } else { 13123 | EXT_CALL zend_jit_invalid_property_assign, REG0 13124 } 13125 may_throw = 1; 13126 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13127 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13128 | b >8 13129 } else { 13130 | b >9 13131 } 13132 |.code 13133 } 13134 } 13135 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 13136 } 13137 13138 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13139 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13140 if (prop_info) { 13141 ce = trace_ce; 13142 ce_is_instanceof = 0; 13143 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13144 if (on_this && JIT_G(current_frame) 13145 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13146 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13147 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13148 if (on_this && JIT_G(current_frame)) { 13149 JIT_G(current_frame)->ce = ce; 13150 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13151 } 13152 } else { 13153 return 0; 13154 } 13155 if (ssa->var_info && ssa_op->op1_use >= 0) { 13156 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13157 ssa->var_info[ssa_op->op1_use].ce = ce; 13158 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13159 } 13160 if (ssa->var_info && ssa_op->op1_def >= 0) { 13161 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13162 ssa->var_info[ssa_op->op1_def].ce = ce; 13163 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13164 } 13165 } 13166 } 13167 } 13168 13169 use_prop_guard = (prop_type != IS_UNKNOWN 13170 && prop_type != IS_UNDEF 13171 && prop_type != IS_REFERENCE 13172 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 13173 13174 if (!prop_info) { 13175 needs_slow_path = 1; 13176 13177 | ldr REG0, EX->run_time_cache 13178 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, ((opline+1)->extended_value), TMP1 13179 | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] 13180 | cmp REG2, TMP2 13181 | bne >7 13182 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 13183 | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, ((opline+1)->extended_value + sizeof(void*) * 2), TMP1 13184 | cbnz TMP1, >7 13185 } 13186 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline+1)->extended_value + sizeof(void*)), TMP1 13187 | tst REG0, REG0 13188 | blt >7 13189 | add TMP1, FCARG1x, REG0 13190 if (!use_prop_guard) { 13191 | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] 13192 | IF_TYPE TMP2w, IS_UNDEF, >7 13193 } 13194 | mov FCARG1x, TMP1 13195 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13196 } else { 13197 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13198 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 13199 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13200 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13201 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13202 13203 if (!exit_addr) { 13204 return 0; 13205 } 13206 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13207 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr 13208 } else { 13209 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13210 | IF_TYPE TMP2w, IS_UNDEF, >7 13211 needs_slow_path = 1; 13212 } 13213 } 13214 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13215 uint32_t info = val_info; 13216 13217 may_throw = 1; 13218 13219 if (opline) { 13220 | SET_EX_OPLINE opline, REG0 13221 } 13222 13223 | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1, ZREG_TMP1 13224 |.cold_code 13225 |1: 13226 | GET_ZVAL_PTR FCARG1x, prop_addr, TMP1 13227 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 13228 | LOAD_ZVAL_ADDR FCARG2x, val_addr 13229 } 13230 | LOAD_ADDR CARG3, binary_op 13231 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 13232 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13233 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 13234 } else { 13235 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 13236 } 13237 | b >9 13238 |.code 13239 13240 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13241 13242 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13243 | LOAD_ADDR FCARG2x, prop_info 13244 } else { 13245 int prop_info_offset = 13246 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13247 13248 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 13249 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 13250 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 13251 } 13252 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 13253 | LOAD_ZVAL_ADDR CARG3, val_addr 13254 | LOAD_ADDR CARG4, binary_op 13255 13256 | EXT_CALL zend_jit_assign_op_to_typed_prop, REG0 13257 13258 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13259 info |= MAY_BE_RC1|MAY_BE_RCN; 13260 } 13261 13262 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2 13263 } 13264 } 13265 13266 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13267 zend_jit_addr var_addr = prop_addr; 13268 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13269 uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13270 13271 if (use_prop_guard) { 13272 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 13273 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13274 if (!exit_addr) { 13275 return 0; 13276 } 13277 13278 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1 13279 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 13280 } 13281 13282 if (var_info & MAY_BE_REF) { 13283 may_throw = 1; 13284 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 13285 | LOAD_ZVAL_ADDR REG0, prop_addr 13286 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 13287 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 13288 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] 13289 | cbnz TMP1, >1 13290 | add REG0, FCARG1x, #offsetof(zend_reference, val) 13291 |.cold_code 13292 |1: 13293 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 13294 | LOAD_ZVAL_ADDR FCARG2x, val_addr 13295 } 13296 if (opline) { 13297 | SET_EX_OPLINE opline, REG0 13298 } 13299 | LOAD_ADDR CARG3, binary_op 13300 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 13301 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13302 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 13303 } else { 13304 | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 13305 } 13306 | b >9 13307 |.code 13308 |2: 13309 } 13310 13311 switch (opline->extended_value) { 13312 case ZEND_ADD: 13313 case ZEND_SUB: 13314 case ZEND_MUL: 13315 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13316 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13317 if (opline->extended_value != ZEND_ADD || 13318 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 13319 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 13320 may_throw = 1; 13321 } 13322 } 13323 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, 13324 1 /* may overflow */, 0)) { 13325 return 0; 13326 } 13327 break; 13328 case ZEND_BW_OR: 13329 case ZEND_BW_AND: 13330 case ZEND_BW_XOR: 13331 may_throw = 1; 13332 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13333 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13334 if ((var_info & MAY_BE_ANY) != MAY_BE_STRING || 13335 (val_info & MAY_BE_ANY) != MAY_BE_STRING) { 13336 may_throw = 1; 13337 } 13338 } 13339 goto long_math; 13340 case ZEND_SL: 13341 case ZEND_SR: 13342 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13343 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13344 may_throw = 1; 13345 } 13346 if ((opline+1)->op1_type != IS_CONST || 13347 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 13348 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) { 13349 may_throw = 1; 13350 } 13351 goto long_math; 13352 case ZEND_MOD: 13353 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 13354 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13355 if (opline->extended_value != ZEND_ADD || 13356 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 13357 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 13358 may_throw = 1; 13359 } 13360 } 13361 if ((opline+1)->op1_type != IS_CONST || 13362 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 13363 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) { 13364 may_throw = 1; 13365 } 13366long_math: 13367 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 13368 IS_CV, opline->op1, var_addr, var_info, NULL, 13369 (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 13370 val_range, 13371 0, var_addr, var_def_info, var_info, 0)) { 13372 return 0; 13373 } 13374 break; 13375 case ZEND_CONCAT: 13376 may_throw = 1; 13377 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, 13378 0)) { 13379 return 0; 13380 } 13381 break; 13382 default: 13383 ZEND_UNREACHABLE(); 13384 } 13385 } 13386 13387 if (needs_slow_path) { 13388 may_throw = 1; 13389 |.cold_code 13390 |7: 13391 | SET_EX_OPLINE opline, REG0 13392 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13393 | LOAD_ADDR FCARG2x, name 13394 | LOAD_ZVAL_ADDR CARG3, val_addr 13395 | ldr CARG4, EX->run_time_cache 13396 | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, (opline+1)->extended_value, TMP1 13397 | LOAD_ADDR CARG5, binary_op 13398 | EXT_CALL zend_jit_assign_obj_op_helper, REG0 13399 13400 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13401 val_info |= MAY_BE_RC1|MAY_BE_RCN; 13402 } 13403 13404 |8: 13405 | // FREE_OP_DATA(); 13406 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13407 | b >9 13408 |.code 13409 } 13410 13411 |9: 13412 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13413 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 13414 may_throw = 1; 13415 } 13416 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 13417 } 13418 13419 if (may_throw) { 13420 if (!zend_jit_check_exception(Dst)) { 13421 return 0; 13422 } 13423 } 13424 13425 return 1; 13426} 13427 13428static int zend_jit_assign_obj(dasm_State **Dst, 13429 const zend_op *opline, 13430 const zend_op_array *op_array, 13431 zend_ssa *ssa, 13432 const zend_ssa_op *ssa_op, 13433 uint32_t op1_info, 13434 zend_jit_addr op1_addr, 13435 uint32_t val_info, 13436 bool op1_indirect, 13437 zend_class_entry *ce, 13438 bool ce_is_instanceof, 13439 bool on_this, 13440 bool delayed_fetch_this, 13441 zend_class_entry *trace_ce, 13442 uint8_t prop_type, 13443 int may_throw) 13444{ 13445 zval *member; 13446 zend_string *name; 13447 zend_property_info *prop_info; 13448 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13449 zend_jit_addr res_addr = 0; 13450 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13451 zend_jit_addr prop_addr; 13452 bool needs_slow_path = 0; 13453 bool needs_val_dtor = 0; 13454 13455 if (RETURN_VALUE_USED(opline)) { 13456 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 13457 } 13458 13459 ZEND_ASSERT(opline->op2_type == IS_CONST); 13460 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13461 13462 member = RT_CONSTANT(opline, opline->op2); 13463 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13464 name = Z_STR_P(member); 13465 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13466 13467 if (on_this) { 13468 | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 13469 } else { 13470 if (opline->op1_type == IS_VAR 13471 && (op1_info & MAY_BE_INDIRECT) 13472 && Z_REG(op1_addr) == ZREG_FP) { 13473 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13474 | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w 13475 | GET_Z_PTR FCARG1x, FCARG1x 13476 |1: 13477 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13478 } 13479 if (op1_info & MAY_BE_REF) { 13480 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13481 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13482 } 13483 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 13484 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13485 } 13486 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13487 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13488 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13489 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13490 13491 if (!exit_addr) { 13492 return 0; 13493 } 13494 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 13495 } else { 13496 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 13497 |.cold_code 13498 |1: 13499 | SET_EX_OPLINE opline, REG0 13500 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13501 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 13502 } 13503 | LOAD_ADDR FCARG2x, ZSTR_VAL(name) 13504 | EXT_CALL zend_jit_invalid_property_assign, REG0 13505 if (RETURN_VALUE_USED(opline)) { 13506 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 13507 } 13508 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13509 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13510 needs_val_dtor = 1; 13511 | b >7 13512 } else { 13513 | b >9 13514 } 13515 |.code 13516 } 13517 } 13518 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 13519 } 13520 13521 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13522 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13523 if (prop_info) { 13524 ce = trace_ce; 13525 ce_is_instanceof = 0; 13526 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13527 if (on_this && JIT_G(current_frame) 13528 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13529 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13530 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13531 if (on_this && JIT_G(current_frame)) { 13532 JIT_G(current_frame)->ce = ce; 13533 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13534 } 13535 } else { 13536 return 0; 13537 } 13538 if (ssa->var_info && ssa_op->op1_use >= 0) { 13539 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13540 ssa->var_info[ssa_op->op1_use].ce = ce; 13541 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13542 } 13543 if (ssa->var_info && ssa_op->op1_def >= 0) { 13544 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13545 ssa->var_info[ssa_op->op1_def].ce = ce; 13546 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13547 } 13548 } 13549 } 13550 } 13551 13552 if (!prop_info) { 13553 needs_slow_path = 1; 13554 13555 | ldr REG0, EX->run_time_cache 13556 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 13557 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] 13558 | cmp REG2, TMP1 13559 | bne >5 13560 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 13561 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 13562 } 13563 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 13564 | tst REG0, REG0 13565 | blt >5 13566 | add TMP2, FCARG1x, REG0 13567 | ldrb TMP1w, [TMP2, #offsetof(zval,u1.v.type)] 13568 | IF_TYPE TMP1w, IS_UNDEF, >5 13569 | mov FCARG1x, TMP2 13570 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13571 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 13572 | cbnz FCARG2x, >1 13573 |.cold_code 13574 |1: 13575 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13576 | SET_EX_OPLINE opline, REG0 13577 | LOAD_ZVAL_ADDR CARG3, val_addr 13578 if (RETURN_VALUE_USED(opline)) { 13579 | LOAD_ZVAL_ADDR CARG4, res_addr 13580 } else { 13581 | mov CARG4, xzr 13582 } 13583 13584 | EXT_CALL zend_jit_assign_to_typed_prop, REG0 13585 13586 if ((opline+1)->op1_type == IS_CONST) { 13587 | // TODO: ??? 13588 | // if (Z_TYPE_P(value) == orig_type) { 13589 | // CACHE_PTR_EX(cache_slot + 2, NULL); 13590 } 13591 13592 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13593 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13594 | b >7 13595 } else { 13596 | b >9 13597 } 13598 |.code 13599 } 13600 } else { 13601 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13602 if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) { 13603 // Undefined property with magic __get()/__set() 13604 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13605 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13606 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13607 13608 if (!exit_addr) { 13609 return 0; 13610 } 13611 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13612 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr 13613 } else { 13614 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 13615 | IF_TYPE TMP2w, IS_UNDEF, >5 13616 needs_slow_path = 1; 13617 } 13618 } 13619 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13620 uint32_t info = val_info; 13621 13622 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 13623 | SET_EX_OPLINE opline, REG0 13624 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13625 | LOAD_ADDR FCARG2x, prop_info 13626 } else { 13627 int prop_info_offset = 13628 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13629 13630 | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] 13631 | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] 13632 | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 13633 } 13634 | LOAD_ZVAL_ADDR FCARG1x, prop_addr 13635 | LOAD_ZVAL_ADDR CARG3, val_addr 13636 if (RETURN_VALUE_USED(opline)) { 13637 | LOAD_ZVAL_ADDR CARG4, res_addr 13638 } else { 13639 | mov CARG4, xzr 13640 } 13641 13642 | EXT_CALL zend_jit_assign_to_typed_prop, REG0 13643 13644 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13645 info |= MAY_BE_RC1|MAY_BE_RCN; 13646 } 13647 13648 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2 13649 } 13650 } 13651 13652 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13653 // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); 13654 if (opline->result_type == IS_UNUSED) { 13655 if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 13656 return 0; 13657 } 13658 } else { 13659 if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 13660 return 0; 13661 } 13662 } 13663 } 13664 13665 if (needs_slow_path) { 13666 |.cold_code 13667 |5: 13668 | SET_EX_OPLINE opline, REG0 13669 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13670 | LOAD_ADDR FCARG2x, name 13671 13672 | LOAD_ZVAL_ADDR CARG3, val_addr 13673 | ldr CARG4, EX->run_time_cache 13674 | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, opline->extended_value, TMP1 13675 if (RETURN_VALUE_USED(opline)) { 13676 | LOAD_ZVAL_ADDR CARG5, res_addr 13677 } else { 13678 | mov CARG5, xzr 13679 } 13680 13681 | EXT_CALL zend_jit_assign_obj_helper, REG0 13682 13683 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13684 val_info |= MAY_BE_RC1|MAY_BE_RCN; 13685 } 13686 13687 |7: 13688 | // FREE_OP_DATA(); 13689 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13690 | b >9 13691 |.code 13692 } else if (needs_val_dtor) { 13693 |.cold_code 13694 |7: 13695 | // FREE_OP_DATA(); 13696 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13697 | b >9 13698 |.code 13699 } 13700 13701 |9: 13702 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13703 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 13704 } 13705 13706 if (may_throw) { 13707 if (!zend_jit_check_exception(Dst)) { 13708 return 0; 13709 } 13710 } 13711 13712 return 1; 13713} 13714 13715static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) 13716{ 13717 zend_jit_addr op1_addr = OP1_ADDR(); 13718 13719 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 13720 if (may_throw) { 13721 | SET_EX_OPLINE opline, REG0 13722 } 13723 if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { 13724 if (op1_info & MAY_BE_ARRAY) { 13725 | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 13726 } 13727 | MEM_ACCESS_32_WITH_UOFFSET ldr, FCARG1w, FP, (opline->op1.var + offsetof(zval, u2.fe_iter_idx)), TMP1 13728 | mvn TMP1w, wzr // TODO: DynAsm fails loading #-1 13729 | cmp FCARG1w, TMP1w 13730 | beq >7 13731 | EXT_CALL zend_hash_iterator_del, REG0 13732 |7: 13733 } 13734 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 13735 if (may_throw) { 13736 if (!zend_jit_check_exception(Dst)) { 13737 return 0; 13738 } 13739 } 13740 } 13741 return 1; 13742} 13743 13744static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 13745{ 13746 if (opline->op1_type == IS_CONST) { 13747 zval *zv; 13748 size_t len; 13749 13750 zv = RT_CONSTANT(opline, opline->op1); 13751 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 13752 len = Z_STRLEN_P(zv); 13753 13754 if (len > 0) { 13755 const char *str = Z_STRVAL_P(zv); 13756 13757 | SET_EX_OPLINE opline, REG0 13758 | LOAD_ADDR CARG1, str 13759 | LOAD_64BIT_VAL CARG2, len 13760 | EXT_CALL zend_write, REG0 13761 if (!zend_jit_check_exception(Dst)) { 13762 return 0; 13763 } 13764 } 13765 } else { 13766 zend_jit_addr op1_addr = OP1_ADDR(); 13767 13768 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 13769 13770 | SET_EX_OPLINE opline, REG0 13771 | GET_ZVAL_PTR REG0, op1_addr, TMP1 13772 | add CARG1, REG0, #offsetof(zend_string, val) 13773 | ldr CARG2, [REG0, #offsetof(zend_string, len)] 13774 | EXT_CALL zend_write, REG0 13775 if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { 13776 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 13777 } 13778 if (!zend_jit_check_exception(Dst)) { 13779 return 0; 13780 } 13781 } 13782 return 1; 13783} 13784 13785static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) 13786{ 13787 if (opline->op1_type == IS_CONST) { 13788 zval *zv; 13789 size_t len; 13790 13791 zv = RT_CONSTANT(opline, opline->op1); 13792 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 13793 len = Z_STRLEN_P(zv); 13794 13795 | SET_ZVAL_LVAL res_addr, len, TMP1, TMP2 13796 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 13797 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13798 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13799 return 0; 13800 } 13801 } else { 13802 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 13803 13804 if (Z_MODE(res_addr) == IS_REG) { 13805 | GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1 13806 | ldr Rx(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(zend_string, len)] 13807 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13808 return 0; 13809 } 13810 } else { 13811 | GET_ZVAL_PTR REG0, op1_addr, TMP1 13812 | ldr REG0, [REG0, #offsetof(zend_string, len)] 13813 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 13814 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13815 } 13816 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13817 } 13818 return 1; 13819} 13820 13821static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) 13822{ 13823 if (opline->op1_type == IS_CONST) { 13824 zval *zv; 13825 zend_long count; 13826 13827 zv = RT_CONSTANT(opline, opline->op1); 13828 ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); 13829 count = zend_hash_num_elements(Z_ARRVAL_P(zv)); 13830 13831 | SET_ZVAL_LVAL res_addr, count, TMP1, TMP2 13832 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 13833 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13834 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13835 return 0; 13836 } 13837 } else { 13838 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY); 13839 // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+. 13840 13841 if (Z_MODE(res_addr) == IS_REG) { 13842 | GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1 13843 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 13844 | ldr Rw(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(HashTable, nNumOfElements)] 13845 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 13846 return 0; 13847 } 13848 } else { 13849 | GET_ZVAL_PTR REG0, op1_addr, TMP1 13850 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 13851 | ldr REG0w, [REG0, #offsetof(HashTable, nNumOfElements)] 13852 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 13853 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 13854 } 13855 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 13856 } 13857 13858 if (may_throw) { 13859 return zend_jit_check_exception(Dst); 13860 } 13861 return 1; 13862} 13863 13864static int zend_jit_load_this(dasm_State **Dst, uint32_t var) 13865{ 13866 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 13867 13868 | ldr FCARG1x, EX->This.value.ptr 13869 | SET_ZVAL_PTR var_addr, FCARG1x, TMP1 13870 | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX, TMP1w, TMP2 13871 | GC_ADDREF FCARG1x, TMP1w 13872 return 1; 13873} 13874 13875static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only) 13876{ 13877 if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) { 13878 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13879 if (!JIT_G(current_frame) || 13880 !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { 13881 13882 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13883 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13884 13885 if (!exit_addr) { 13886 return 0; 13887 } 13888 13889 | ldrb TMP1w, EX->This.u1.v.type 13890 | cmp TMP1w, #IS_OBJECT 13891 | bne &exit_addr 13892 13893 if (JIT_G(current_frame)) { 13894 TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); 13895 } 13896 } 13897 } else { 13898 13899 | ldrb TMP1w, EX->This.u1.v.type 13900 | cmp TMP1w, #IS_OBJECT 13901 | bne >1 13902 |.cold_code 13903 |1: 13904 | SET_EX_OPLINE opline, REG0 13905 | b ->invalid_this 13906 |.code 13907 } 13908 } 13909 13910 if (!check_only) { 13911 if (!zend_jit_load_this(Dst, opline->result.var)) { 13912 return 0; 13913 } 13914 } 13915 return 1; 13916} 13917 13918static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) 13919{ 13920 uint32_t count; 13921 Bucket *p; 13922 const zend_op *target; 13923 int b; 13924 int32_t exit_point; 13925 const void *exit_addr; 13926 13927 if (default_label) { 13928 | cbz REG0, &default_label 13929 } else if (next_opline) { 13930 | cbz REG0, >3 13931 } else { 13932 | cbz REG0, =>default_b 13933 } 13934 | LOAD_ADDR FCARG1x, jumptable 13935 | ldr TMP1, [FCARG1x, #offsetof(HashTable, arData)] 13936 | sub REG0, REG0, TMP1 13937 if (HT_IS_PACKED(jumptable)) { 13938 | mov FCARG1x, #(sizeof(zval) / sizeof(void*)) 13939 } else { 13940 | mov FCARG1x, #(sizeof(Bucket) / sizeof(void*)) 13941 } 13942 | sdiv REG0, REG0, FCARG1x 13943 | adr FCARG1x, >4 13944 | ldr TMP1, [FCARG1x, REG0] 13945 | br TMP1 13946 13947 |.jmp_table 13948 |.align 8 13949 |4: 13950 if (trace_info) { 13951 trace_info->jmp_table_size += zend_hash_num_elements(jumptable); 13952 } 13953 13954 count = jumptable->nNumUsed; 13955 p = jumptable->arData; 13956 do { 13957 if (Z_TYPE(p->val) == IS_UNDEF) { 13958 if (default_label) { 13959 | .addr &default_label 13960 } else if (next_opline) { 13961 | .addr >3 13962 } else { 13963 | .addr =>default_b 13964 } 13965 } else { 13966 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 13967 if (!next_opline) { 13968 b = ssa->cfg.map[target - op_array->opcodes]; 13969 | .addr =>b 13970 } else if (next_opline == target) { 13971 | .addr >3 13972 } else { 13973 exit_point = zend_jit_trace_get_exit_point(target, 0); 13974 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13975 if (!exit_addr) { 13976 return 0; 13977 } 13978 | .addr &exit_addr 13979 } 13980 } 13981 if (HT_IS_PACKED(jumptable)) { 13982 p = (Bucket*)(((zval*)p)+1); 13983 } else { 13984 p++; 13985 } 13986 count--; 13987 } while (count); 13988 |.code 13989 13990 return 1; 13991} 13992 13993static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) 13994{ 13995 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 13996 const zend_op *next_opline = NULL; 13997 13998 if (trace) { 13999 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 14000 ZEND_ASSERT(trace->opline != NULL); 14001 next_opline = trace->opline; 14002 } 14003 14004 if (opline->op1_type == IS_CONST) { 14005 zval *zv = RT_CONSTANT(opline, opline->op1); 14006 zval *jump_zv = NULL; 14007 int b; 14008 14009 if (opline->opcode == ZEND_SWITCH_LONG) { 14010 if (Z_TYPE_P(zv) == IS_LONG) { 14011 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14012 } 14013 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14014 if (Z_TYPE_P(zv) == IS_STRING) { 14015 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 14016 } 14017 } else if (opline->opcode == ZEND_MATCH) { 14018 if (Z_TYPE_P(zv) == IS_LONG) { 14019 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14020 } else if (Z_TYPE_P(zv) == IS_STRING) { 14021 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 14022 } 14023 } else { 14024 ZEND_UNREACHABLE(); 14025 } 14026 if (next_opline) { 14027 const zend_op *target; 14028 14029 if (jump_zv != NULL) { 14030 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); 14031 } else { 14032 target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14033 } 14034 ZEND_ASSERT(target == next_opline); 14035 } else { 14036 if (jump_zv != NULL) { 14037 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; 14038 } else { 14039 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; 14040 } 14041 | b =>b 14042 } 14043 } else { 14044 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; 14045 uint32_t op1_info = OP1_INFO(); 14046 zend_jit_addr op1_addr = OP1_ADDR(); 14047 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14048 const zend_op *target; 14049 int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; 14050 int b; 14051 int32_t exit_point; 14052 const void *fallback_label = NULL; 14053 const void *default_label = NULL; 14054 const void *exit_addr; 14055 14056 if (next_opline) { 14057 if (next_opline != opline + 1) { 14058 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 14059 fallback_label = zend_jit_trace_get_exit_addr(exit_point); 14060 if (!fallback_label) { 14061 return 0; 14062 } 14063 } 14064 if (next_opline != default_opline) { 14065 exit_point = zend_jit_trace_get_exit_point(default_opline, 0); 14066 default_label = zend_jit_trace_get_exit_addr(exit_point); 14067 if (!default_label) { 14068 return 0; 14069 } 14070 } 14071 } 14072 14073 if (opline->opcode == ZEND_SWITCH_LONG) { 14074 if (op1_info & MAY_BE_LONG) { 14075 if (op1_info & MAY_BE_REF) { 14076 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1, ZREG_TMP1 14077 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 14078 |.cold_code 14079 |1: 14080 | // ZVAL_DEREF(op) 14081 if (fallback_label) { 14082 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 14083 } else { 14084 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 14085 } 14086 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14087 if (fallback_label) { 14088 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14089 | IF_NOT_Z_TYPE TMP1, IS_LONG, &fallback_label, TMP2w 14090 } else { 14091 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14092 | IF_NOT_Z_TYPE TMP1, IS_LONG, >3, TMP2w 14093 } 14094 | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.lval)] 14095 | b >2 14096 |.code 14097 |2: 14098 } else { 14099 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14100 if (fallback_label) { 14101 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label, ZREG_TMP1 14102 } else { 14103 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 14104 } 14105 } 14106 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 14107 } 14108 if (HT_IS_PACKED(jumptable)) { 14109 uint32_t count = jumptable->nNumUsed; 14110 zval *zv = jumptable->arPacked; 14111 14112 | CMP_64_WITH_CONST_32 FCARG2x, jumptable->nNumUsed, TMP1 14113 if (default_label) { 14114 | bhs &default_label 14115 } else if (next_opline) { 14116 | bhs >3 14117 } else { 14118 | bhs =>default_b 14119 } 14120 | adr REG0, >4 14121 | ldr TMP1, [REG0, FCARG2x, lsl #3] 14122 | br TMP1 14123 14124 |.jmp_table 14125 |.align 8 14126 |4: 14127 if (trace_info) { 14128 trace_info->jmp_table_size += count; 14129 } 14130 do { 14131 if (Z_TYPE_P(zv) == IS_UNDEF) { 14132 if (default_label) { 14133 | .addr &default_label 14134 } else if (next_opline) { 14135 | .addr >3 14136 } else { 14137 | .addr =>default_b 14138 } 14139 } else { 14140 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); 14141 if (!next_opline) { 14142 b = ssa->cfg.map[target - op_array->opcodes]; 14143 | .addr =>b 14144 } else if (next_opline == target) { 14145 | .addr >3 14146 } else { 14147 exit_point = zend_jit_trace_get_exit_point(target, 0); 14148 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14149 if (!exit_addr) { 14150 return 0; 14151 } 14152 | .addr &exit_addr 14153 } 14154 } 14155 zv++; 14156 count--; 14157 } while (count); 14158 |.code 14159 |3: 14160 } else { 14161 | LOAD_ADDR FCARG1x, jumptable 14162 | EXT_CALL zend_hash_index_find, REG0 14163 | mov REG0, RETVALx 14164 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14165 return 0; 14166 } 14167 |3: 14168 } 14169 } 14170 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14171 if (op1_info & MAY_BE_STRING) { 14172 if (op1_info & MAY_BE_REF) { 14173 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1, ZREG_TMP1 14174 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14175 |.cold_code 14176 |1: 14177 | // ZVAL_DEREF(op) 14178 if (fallback_label) { 14179 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 14180 } else { 14181 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 14182 } 14183 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14184 if (fallback_label) { 14185 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14186 | IF_NOT_Z_TYPE TMP1, IS_STRING, &fallback_label, TMP2w 14187 } else { 14188 | add TMP1, FCARG2x, #offsetof(zend_reference, val) 14189 | IF_NOT_Z_TYPE TMP1, IS_STRING, >3, TMP2w 14190 } 14191 | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.ptr)] 14192 | b >2 14193 |.code 14194 |2: 14195 } else { 14196 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { 14197 if (fallback_label) { 14198 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label, ZREG_TMP1 14199 } else { 14200 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 14201 } 14202 } 14203 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14204 } 14205 | LOAD_ADDR FCARG1x, jumptable 14206 | EXT_CALL zend_hash_find, REG0 14207 | mov REG0, RETVALx 14208 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14209 return 0; 14210 } 14211 |3: 14212 } 14213 } else if (opline->opcode == ZEND_MATCH) { 14214 if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { 14215 if (op1_info & MAY_BE_REF) { 14216 | LOAD_ZVAL_ADDR FCARG2x, op1_addr 14217 | ZVAL_DEREF FCARG2x, op1_info, TMP1w 14218 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 14219 } 14220 | LOAD_ADDR FCARG1x, jumptable 14221 if (op1_info & MAY_BE_LONG) { 14222 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14223 if (op1_info & MAY_BE_STRING) { 14224 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5, ZREG_TMP1 14225 } else if (op1_info & MAY_BE_UNDEF) { 14226 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 14227 } else if (default_label) { 14228 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label, ZREG_TMP1 14229 } else if (next_opline) { 14230 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 14231 } else { 14232 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b, ZREG_TMP1 14233 } 14234 } 14235 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 14236 | EXT_CALL zend_hash_index_find, REG0 14237 | mov REG0, RETVALx 14238 if (op1_info & MAY_BE_STRING) { 14239 | b >2 14240 } 14241 } 14242 if (op1_info & MAY_BE_STRING) { 14243 |5: 14244 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { 14245 if (op1_info & MAY_BE_UNDEF) { 14246 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 14247 } else if (default_label) { 14248 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label, ZREG_TMP1 14249 } else if (next_opline) { 14250 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 14251 } else { 14252 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b, ZREG_TMP1 14253 } 14254 } 14255 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14256 | EXT_CALL zend_hash_find, REG0 14257 | mov REG0, RETVALx 14258 } 14259 |2: 14260 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14261 return 0; 14262 } 14263 } 14264 if (op1_info & MAY_BE_UNDEF) { 14265 |6: 14266 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { 14267 if (default_label) { 14268 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label, ZREG_TMP1 14269 } else if (next_opline) { 14270 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3, ZREG_TMP1 14271 } else { 14272 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b, ZREG_TMP1 14273 } 14274 } 14275 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 14276 | SET_EX_OPLINE opline, REG0 14277 | LOAD_32BIT_VAL FCARG1w, opline->op1.var 14278 | EXT_CALL zend_jit_undefined_op_helper, REG0 14279 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 14280 return 0; 14281 } 14282 } 14283 if (default_label) { 14284 | b &default_label 14285 } else if (next_opline) { 14286 | b >3 14287 } else { 14288 | b =>default_b 14289 } 14290 |3: 14291 } else { 14292 ZEND_UNREACHABLE(); 14293 } 14294 } 14295 return 1; 14296} 14297 14298static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) 14299{ 14300 zend_arg_info *arg_info = &op_array->arg_info[-1]; 14301 ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); 14302 zend_jit_addr op1_addr = OP1_ADDR(); 14303 bool needs_slow_check = 1; 14304 bool slow_check_in_cold = 1; 14305 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 14306 14307 if (type_mask == 0) { 14308 slow_check_in_cold = 0; 14309 } else { 14310 if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { 14311 slow_check_in_cold = 0; 14312 } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { 14313 needs_slow_check = 0; 14314 } else if (is_power_of_two(type_mask)) { 14315 uint32_t type_code = concrete_type(type_mask); 14316 | IF_NOT_ZVAL_TYPE op1_addr, type_code, >6, ZREG_TMP1 14317 } else { 14318 | mov REG2w, #1 14319 | GET_ZVAL_TYPE REG1w, op1_addr, TMP1 14320 | lsl REG2w, REG2w, REG1w 14321 | TST_32_WITH_CONST REG2w, type_mask, TMP1w 14322 | beq >6 14323 } 14324 } 14325 if (needs_slow_check) { 14326 if (slow_check_in_cold) { 14327 |.cold_code 14328 |6: 14329 } 14330 | SET_EX_OPLINE opline, REG1 14331 if (op1_info & MAY_BE_UNDEF) { 14332 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7, ZREG_TMP1 14333 | LOAD_32BIT_VAL FCARG1x, opline->op1.var 14334 | EXT_CALL zend_jit_undefined_op_helper, REG0 14335 | cbz RETVALx, ->exception_handler 14336 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval 14337 | b >8 14338 } 14339 |7: 14340 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 14341 |8: 14342 | ldr FCARG2x, EX->func 14343 | LOAD_ADDR CARG3, (ptrdiff_t)arg_info 14344 | ldr REG0, EX->run_time_cache 14345 | ADD_SUB_64_WITH_CONST_32 add, CARG4, REG0, opline->op2.num, TMP1 14346 | EXT_CALL zend_jit_verify_return_slow, REG0 14347 if (!zend_jit_check_exception(Dst)) { 14348 return 0; 14349 } 14350 if (slow_check_in_cold) { 14351 | b >9 14352 |.code 14353 } 14354 } 14355 |9: 14356 return 1; 14357} 14358 14359static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 14360{ 14361 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14362 14363 // TODO: support for empty() ??? 14364 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 14365 14366 if (op1_info & MAY_BE_REF) { 14367 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14368 | LOAD_ZVAL_ADDR FCARG1x, op1_addr 14369 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14370 } 14371 | ZVAL_DEREF FCARG1x, op1_info, TMP1w 14372 |1: 14373 } 14374 14375 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { 14376 if (exit_addr) { 14377 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); 14378 } else if (smart_branch_opcode) { 14379 if (smart_branch_opcode == ZEND_JMPNZ) { 14380 | b =>target_label 14381 } 14382 } else { 14383 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 14384 } 14385 } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { 14386 if (exit_addr) { 14387 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); 14388 } else if (smart_branch_opcode) { 14389 if (smart_branch_opcode != ZEND_JMPNZ) { 14390 | b =>target_label 14391 } 14392 } else { 14393 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 14394 } 14395 } else { 14396 ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); 14397 | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, Rx(Z_REG(op1_addr)), (Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)), TMP1 14398 | cmp TMP1w, #IS_NULL 14399 if (exit_addr) { 14400 if (smart_branch_opcode == ZEND_JMPNZ) { 14401 | bgt &exit_addr 14402 } else { 14403 | ble &exit_addr 14404 } 14405 } else if (smart_branch_opcode) { 14406 if (smart_branch_opcode == ZEND_JMPZ) { 14407 | ble =>target_label 14408 } else if (smart_branch_opcode == ZEND_JMPNZ) { 14409 | bgt =>target_label 14410 } else { 14411 ZEND_UNREACHABLE(); 14412 } 14413 } else { 14414 | cset REG0w, gt 14415 | add REG0w, REG0w, #IS_FALSE 14416 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 14417 } 14418 } 14419 14420 return 1; 14421} 14422 14423static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 14424{ 14425 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14426 14427 if (opline->op1_type == IS_CONST) { 14428 zval *zv = RT_CONSTANT(opline, opline->op1); 14429 14430 | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 14431 if (Z_REFCOUNTED_P(zv)) { 14432 | ADDREF_CONST zv, REG0, TMP1 14433 } 14434 } else { 14435 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 14436 14437 | // ZVAL_COPY(res, value); 14438 | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 14439 if (opline->op1_type == IS_CV) { 14440 | TRY_ADDREF op1_info, REG0w, FCARG1x, TMP1w 14441 } 14442 } 14443 | // Z_FE_POS_P(res) = 0; 14444 | MEM_ACCESS_32_WITH_UOFFSET str, wzr, FP, (opline->result.var + offsetof(zval, u2.fe_pos)), TMP1 14445 14446 return 1; 14447} 14448 14449static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, uint8_t exit_opcode, const void *exit_addr) 14450{ 14451 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 14452 14453 if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) { 14454 /* empty array */ 14455 if (exit_addr) { 14456 if (exit_opcode == ZEND_JMP) { 14457 | b &exit_addr 14458 } 14459 } else { 14460 | b =>target_label 14461 } 14462 return 1; 14463 } 14464 14465 | // array = EX_VAR(opline->op1.var); 14466 | // fe_ht = Z_ARRVAL_P(array); 14467 | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 14468 14469 if (op1_info & MAY_BE_PACKED_GUARD) { 14470 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 14471 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14472 14473 if (!exit_addr) { 14474 return 0; 14475 } 14476 if (op1_info & MAY_BE_ARRAY_PACKED) { 14477 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 14478 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 14479 | beq &exit_addr 14480 } else { 14481 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 14482 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 14483 | bne &exit_addr 14484 } 14485 } 14486 14487 | // pos = Z_FE_POS_P(array); 14488 | MEM_ACCESS_32_WITH_UOFFSET ldr, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 14489 14490 if (MAY_BE_HASH(op1_info)) { 14491 if (MAY_BE_PACKED(op1_info)) { 14492 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] 14493 | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w 14494 | bne >2 14495 } 14496 | // p = fe_ht->arData + pos; 14497 || ZEND_ASSERT(sizeof(Bucket) == 32); 14498 | mov FCARG2w, REG0w 14499 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] 14500 | add FCARG2x, TMP1, FCARG2x, lsl #5 14501 |1: 14502 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 14503 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, nNumUsed)] 14504 | cmp TMP1w, REG0w 14505 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 14506 | // ZEND_VM_CONTINUE(); 14507 if (exit_addr) { 14508 if (exit_opcode == ZEND_JMP) { 14509 | bls &exit_addr 14510 } else { 14511 | bls >3 14512 } 14513 } else { 14514 | bls =>target_label 14515 } 14516 | // pos++; 14517 | add REG0w, REG0w, #1 14518 | // value_type = Z_TYPE_INFO_P(value); 14519 | // if (EXPECTED(value_type != IS_UNDEF)) { 14520 if (!exit_addr || exit_opcode == ZEND_JMP) { 14521 | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >3, TMP1w 14522 } else { 14523 | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr, TMP1w 14524 } 14525 | // p++; 14526 | add FCARG2x, FCARG2x, #sizeof(Bucket) 14527 | b <1 14528 if (MAY_BE_PACKED(op1_info)) { 14529 |2: 14530 } 14531 } 14532 if (MAY_BE_PACKED(op1_info)) { 14533 | // p = fe_ht->arPacked + pos; 14534 || ZEND_ASSERT(sizeof(zval) == 16); 14535 | mov FCARG2w, REG0w 14536 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arPacked)] 14537 | add FCARG2x, TMP1, FCARG2x, lsl #4 14538 |1: 14539 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 14540 | ldr TMP1w, [FCARG1x, #offsetof(zend_array, nNumUsed)] 14541 | cmp TMP1w, REG0w 14542 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 14543 | // ZEND_VM_CONTINUE(); 14544 if (exit_addr) { 14545 if (exit_opcode == ZEND_JMP) { 14546 | bls &exit_addr 14547 } else { 14548 | bls >4 14549 } 14550 } else { 14551 | bls =>target_label 14552 } 14553 | // pos++; 14554 | add REG0w, REG0w, #1 14555 | // value_type = Z_TYPE_INFO_P(value); 14556 | // if (EXPECTED(value_type != IS_UNDEF)) { 14557 if (!exit_addr || exit_opcode == ZEND_JMP) { 14558 | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >4, TMP1w 14559 } else { 14560 | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr, TMP1w 14561 } 14562 | // p++; 14563 | add FCARG2x, FCARG2x, #sizeof(zval) 14564 | b <1 14565 } 14566 14567 if (!exit_addr || exit_opcode == ZEND_JMP) { 14568 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 14569 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 14570 uint32_t val_info; 14571 14572 if (RETURN_VALUE_USED(opline)) { 14573 zend_jit_addr res_addr = RES_ADDR(); 14574 14575 if (MAY_BE_HASH(op1_info)) { 14576 |3: 14577 | // Z_FE_POS_P(array) = pos + 1; 14578 | MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 14579 14580 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) 14581 && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { 14582 | // if (!p->key) { 14583 | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] 14584 | cbz REG0, >2 14585 } 14586 if (op1_info & MAY_BE_ARRAY_KEY_STRING) { 14587 | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); 14588 | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] 14589 | SET_ZVAL_PTR res_addr, REG0, TMP1 14590 | ldr TMP1w, [REG0, #offsetof(zend_refcounted, gc.u.type_info)] 14591 | TST_32_WITH_CONST TMP1w, IS_STR_INTERNED, TMP2w 14592 | beq >1 14593 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 14594 | b >3 14595 |1: 14596 | GC_ADDREF REG0, TMP1w 14597 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2 14598 14599 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) || MAY_BE_PACKED(op1_info)) { 14600 | b >3 14601 |2: 14602 } 14603 } 14604 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 14605 | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); 14606 | ldr REG0, [FCARG2x, #offsetof(Bucket, h)] 14607 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 14608 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 14609 if (MAY_BE_PACKED(op1_info)) { 14610 | b >3 14611 } 14612 } 14613 } 14614 if (MAY_BE_PACKED(op1_info)) { 14615 |4: 14616 | // Z_FE_POS_P(array) = pos + 1; 14617 | MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 14618 | sub REG0w, REG0w, #1 14619 | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 14620 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 14621 } 14622 |3: 14623 } else { 14624 |3: 14625 |4: 14626 | // Z_FE_POS_P(array) = pos + 1; 14627 | MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 14628 } 14629 14630 val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); 14631 if (val_info & MAY_BE_ARRAY) { 14632 val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 14633 } 14634 if (op1_info & MAY_BE_ARRAY_OF_REF) { 14635 val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | 14636 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 14637 } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14638 val_info |= MAY_BE_RC1 | MAY_BE_RCN; 14639 } 14640 14641 if (opline->op2_type == IS_CV) { 14642 | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); 14643 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { 14644 return 0; 14645 } 14646 } else { 14647 | // ZVAL_COPY(res, value); 14648 | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 14649 | TRY_ADDREF val_info, REG0w, FCARG1x, TMP1w 14650 } 14651 } else { 14652 |3: 14653 |4: 14654 } 14655 14656 return 1; 14657} 14658 14659static int zend_jit_fetch_constant(dasm_State **Dst, 14660 const zend_op *opline, 14661 const zend_op_array *op_array, 14662 zend_ssa *ssa, 14663 const zend_ssa_op *ssa_op, 14664 zend_jit_addr res_addr) 14665{ 14666 zval *zv = RT_CONSTANT(opline, opline->op2) + 1; 14667 zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); 14668 uint32_t res_info = RES_INFO(); 14669 14670 | // c = CACHED_PTR(opline->extended_value); 14671 | ldr FCARG1x, EX->run_time_cache 14672 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG1x, opline->extended_value, TMP1 14673 | // if (c != NULL) 14674 | cbz REG0, >9 14675 if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) { 14676 | // if (!IS_SPECIAL_CACHE_VAL(c)) 14677 || ZEND_ASSERT(CACHE_SPECIAL == 1); 14678 | TST_64_WITH_ONE REG0 14679 | bne >9 14680 } 14681 |8: 14682 14683 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { 14684 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 14685 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 14686 int32_t exit_point; 14687 const void *exit_addr = NULL; 14688 14689 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 14690 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 14691 exit_point = zend_jit_trace_get_exit_point(opline+1, 0); 14692 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 14693 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14694 if (!exit_addr) { 14695 return 0; 14696 } 14697 res_info &= ~MAY_BE_GUARD; 14698 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 14699 14700 uint32_t type = concrete_type(res_info); 14701 14702 if (type < IS_STRING) { 14703 | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr, ZREG_TMP1 14704 } else { 14705 | GET_ZVAL_TYPE_INFO REG2w, const_addr, TMP1 14706 | IF_NOT_TYPE REG2w, type, &exit_addr 14707 } 14708 | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 14709 if (type < IS_STRING) { 14710 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 14711 | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 14712 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 14713 return 0; 14714 } 14715 } else { 14716 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 14717 | TRY_ADDREF res_info, REG2w, REG1, TMP1w 14718 } 14719 } else { 14720 | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 14721 | TRY_ADDREF MAY_BE_ANY, REG0w, REG1, TMP1w 14722 } 14723 14724 |.cold_code 14725 |9: 14726 | // SAVE_OPLINE(); 14727 | SET_EX_OPLINE opline, REG0 14728 | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); 14729 | LOAD_ADDR FCARG1x, zv 14730 | LOAD_32BIT_VAL FCARG2w, opline->op1.num 14731 | EXT_CALL zend_jit_get_constant, REG0 14732 | mov REG0, RETVALx 14733 | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); 14734 | cbnz REG0, <8 14735 | b ->exception_handler 14736 |.code 14737 return 1; 14738} 14739 14740static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 14741{ 14742 HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 14743 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14744 14745 ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); 14746 ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); 14747 14748 | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); 14749 | LOAD_ADDR FCARG1x, ht 14750 if (opline->op1_type != IS_CONST) { 14751 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 14752 | EXT_CALL zend_hash_find, REG0 14753 } else { 14754 zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); 14755 | LOAD_ADDR FCARG2x, str 14756 | EXT_CALL zend_hash_find_known_hash, REG0 14757 } 14758 if (exit_addr) { 14759 if (smart_branch_opcode == ZEND_JMPZ) { 14760 | cbz RETVALx, &exit_addr 14761 } else { 14762 | cbnz RETVALx, &exit_addr 14763 } 14764 } else if (smart_branch_opcode) { 14765 if (smart_branch_opcode == ZEND_JMPZ) { 14766 | cbz RETVALx, =>target_label 14767 } else if (smart_branch_opcode == ZEND_JMPNZ) { 14768 | cbnz RETVALx, =>target_label 14769 } else { 14770 ZEND_UNREACHABLE(); 14771 } 14772 } else { 14773 | tst RETVALx, RETVALx 14774 | cset REG0w, ne 14775 | add REG0w, REG0w, #IS_FALSE 14776 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 14777 } 14778 14779 return 1; 14780} 14781 14782static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info) 14783{ 14784 uint32_t offset; 14785 14786 offset = (opline->opcode == ZEND_ROPE_INIT) ? 14787 opline->result.var : 14788 opline->op1.var + opline->extended_value * sizeof(zend_string*); 14789 14790 if (opline->op2_type == IS_CONST) { 14791 zval *zv = RT_CONSTANT(opline, opline->op2); 14792 zend_string *str; 14793 14794 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14795 str = Z_STR_P(zv); 14796 | LOAD_ADDR REG0, str 14797 | MEM_ACCESS_64_WITH_UOFFSET str, REG0, FP, offset, TMP1 14798 } else { 14799 zend_jit_addr op2_addr = OP2_ADDR(); 14800 14801 ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14802 14803 | GET_ZVAL_PTR REG1, op2_addr, TMP1 14804 | MEM_ACCESS_64_WITH_UOFFSET str, REG1, FP, offset, TMP1 14805 if (opline->op2_type == IS_CV) { 14806 | GET_ZVAL_TYPE_INFO REG0w, op2_addr, TMP1 14807 | TRY_ADDREF op2_info, REG0w, REG1, TMP1w 14808 } 14809 } 14810 14811 if (opline->opcode == ZEND_ROPE_END) { 14812 zend_jit_addr res_addr = RES_ADDR(); 14813 14814 | ADD_SUB_64_WITH_CONST add, FCARG1x, FP, opline->op1.var, TMP1 14815 | LOAD_32BIT_VAL FCARG2w, opline->extended_value 14816 | EXT_CALL zend_jit_rope_end, TMP1 14817 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 14818 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2 14819 } 14820 14821 return 1; 14822} 14823 14824static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) 14825{ 14826 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14827 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14828 14829 if (!exit_addr) { 14830 return 0; 14831 } 14832 | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 14833 14834 return 1; 14835} 14836 14837static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard) 14838{ 14839 zend_jit_addr var_addr = *var_addr_ptr; 14840 uint32_t var_info = *var_info_ptr; 14841 const void *exit_addr = NULL; 14842 14843 if (add_ref_guard || add_type_guard) { 14844 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14845 14846 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14847 if (!exit_addr) { 14848 return 0; 14849 } 14850 } 14851 14852 if (add_ref_guard) { 14853 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 14854 } 14855 if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { 14856 /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ 14857 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 14858 | LOAD_ZVAL_ADDR FCARG1x, var_addr 14859 } 14860 | EXT_CALL zend_jit_unref_helper, REG0 14861 } else { 14862 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 14863 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 14864 *var_addr_ptr = var_addr; 14865 } 14866 14867 if (var_type != IS_UNKNOWN) { 14868 var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); 14869 } 14870 if (add_type_guard 14871 && var_type != IS_UNKNOWN 14872 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 14873 | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr, ZREG_TMP1 14874 14875 ZEND_ASSERT(var_info & (1 << var_type)); 14876 if (var_type < IS_STRING) { 14877 var_info = (1 << var_type); 14878 } else if (var_type != IS_ARRAY) { 14879 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 14880 } else { 14881 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 14882 } 14883 14884 *var_info_ptr = var_info; 14885 } else { 14886 var_info &= ~MAY_BE_REF; 14887 *var_info_ptr = var_info; 14888 } 14889 *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */ 14890 14891 return 1; 14892} 14893 14894static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard) 14895{ 14896 zend_jit_addr var_addr = *var_addr_ptr; 14897 uint32_t var_info = *var_info_ptr; 14898 int32_t exit_point; 14899 const void *exit_addr; 14900 14901 if (add_indirect_guard) { 14902 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14903 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14904 14905 if (!exit_addr) { 14906 return 0; 14907 } 14908 | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr, ZREG_TMP1 14909 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 14910 } else { 14911 /* May be already loaded into FCARG1a or RAX by previous FETCH_OBJ_W/DIM_W */ 14912 if (opline->op1_type != IS_VAR || 14913 (opline-1)->result_type != IS_VAR || 14914 (opline-1)->result.var != opline->op1.var || 14915 (opline-1)->op1_type == IS_VAR || 14916 (opline-1)->op2_type == IS_VAR || 14917 (opline-1)->op2_type == IS_TMP_VAR) { 14918 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 14919 } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { 14920 | mov FCARG1x, REG0 14921 } 14922 } 14923 *var_info_ptr &= ~MAY_BE_INDIRECT; 14924 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14925 *var_addr_ptr = var_addr; 14926 14927 if (var_type != IS_UNKNOWN) { 14928 var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); 14929 } 14930 if (!(var_type & IS_TRACE_REFERENCE) 14931 && var_type != IS_UNKNOWN 14932 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 14933 exit_point = zend_jit_trace_get_exit_point(opline, 0); 14934 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14935 14936 if (!exit_addr) { 14937 return 0; 14938 } 14939 14940 | IF_NOT_Z_TYPE FCARG1x, var_type, &exit_addr, TMP1w 14941 14942 //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); 14943 ZEND_ASSERT(var_info & (1 << var_type)); 14944 if (var_type < IS_STRING) { 14945 var_info = (1 << var_type); 14946 } else if (var_type != IS_ARRAY) { 14947 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 14948 } else { 14949 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 14950 } 14951 14952 *var_info_ptr = var_info; 14953 } 14954 14955 return 1; 14956} 14957 14958static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) 14959{ 14960 if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { 14961 return 0; 14962 } 14963 14964 switch (opline->opcode) { 14965 case ZEND_QM_ASSIGN: 14966 case ZEND_SEND_VAR: 14967 case ZEND_ASSIGN: 14968 case ZEND_PRE_INC: 14969 case ZEND_PRE_DEC: 14970 case ZEND_POST_INC: 14971 case ZEND_POST_DEC: 14972 return 1; 14973 case ZEND_ADD: 14974 case ZEND_SUB: 14975 case ZEND_MUL: 14976 case ZEND_BW_OR: 14977 case ZEND_BW_AND: 14978 case ZEND_BW_XOR: 14979 case ZEND_SL: 14980 case ZEND_SR: 14981 if (def_var == ssa_op->result_def && 14982 use_var == ssa_op->op1_use) { 14983 return 1; 14984 } 14985 break; 14986 default: 14987 break; 14988 } 14989 return 0; 14990} 14991 14992static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) 14993{ 14994 uint32_t op1_info, op2_info; 14995 14996 switch (opline->opcode) { 14997 case ZEND_SEND_VAR: 14998 case ZEND_SEND_VAL: 14999 case ZEND_SEND_VAL_EX: 15000 return (opline->op2_type != IS_CONST); 15001 case ZEND_QM_ASSIGN: 15002 case ZEND_IS_SMALLER: 15003 case ZEND_IS_SMALLER_OR_EQUAL: 15004 case ZEND_IS_EQUAL: 15005 case ZEND_IS_NOT_EQUAL: 15006 case ZEND_IS_IDENTICAL: 15007 case ZEND_IS_NOT_IDENTICAL: 15008 case ZEND_CASE: 15009 return 1; 15010 case ZEND_RETURN: 15011 return (op_array->type != ZEND_EVAL_CODE && op_array->function_name); 15012 case ZEND_ASSIGN: 15013 op1_info = OP1_INFO(); 15014 op2_info = OP2_INFO(); 15015 return 15016 opline->op1_type == IS_CV && 15017 !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) && 15018 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); 15019 case ZEND_ADD: 15020 case ZEND_SUB: 15021 case ZEND_MUL: 15022 op1_info = OP1_INFO(); 15023 op2_info = OP2_INFO(); 15024 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); 15025 case ZEND_BW_OR: 15026 case ZEND_BW_AND: 15027 case ZEND_BW_XOR: 15028 case ZEND_SL: 15029 case ZEND_SR: 15030 case ZEND_MOD: 15031 op1_info = OP1_INFO(); 15032 op2_info = OP2_INFO(); 15033 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); 15034 case ZEND_PRE_INC: 15035 case ZEND_PRE_DEC: 15036 case ZEND_POST_INC: 15037 case ZEND_POST_DEC: 15038 op1_info = OP1_INFO(); 15039 op2_info = OP1_DEF_INFO(); 15040 return opline->op1_type == IS_CV 15041 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)) 15042 && (op2_info & MAY_BE_LONG); 15043 case ZEND_STRLEN: 15044 op1_info = OP1_INFO(); 15045 return (opline->op1_type & (IS_CV|IS_CONST)) 15046 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING; 15047 case ZEND_COUNT: 15048 op1_info = OP1_INFO(); 15049 return (opline->op1_type & (IS_CV|IS_CONST)) 15050 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY; 15051 case ZEND_JMPZ: 15052 case ZEND_JMPNZ: 15053 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 15054 if (!ssa->cfg.map) { 15055 return 0; 15056 } 15057 if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start && 15058 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { 15059 return 0; 15060 } 15061 } 15062 ZEND_FALLTHROUGH; 15063 case ZEND_BOOL: 15064 case ZEND_BOOL_NOT: 15065 case ZEND_JMPZ_EX: 15066 case ZEND_JMPNZ_EX: 15067 return 1; 15068 case ZEND_FETCH_CONSTANT: 15069 return 1; 15070 case ZEND_FETCH_DIM_R: 15071 op1_info = OP1_INFO(); 15072 op2_info = OP2_INFO(); 15073 if (trace 15074 && trace->op1_type != IS_UNKNOWN 15075 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { 15076 op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); 15077 } 15078 return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && 15079 (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && 15080 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || 15081 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && 15082 (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); 15083 } 15084 return 0; 15085} 15086 15087static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) 15088{ 15089 if (ssa->vars[var].no_val) { 15090 /* we don't need the value */ 15091 return 0; 15092 } 15093 15094 if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { 15095 /* Disable global register allocation, 15096 * register allocation for SSA variables connected through Phi functions 15097 */ 15098 if (ssa->vars[var].definition_phi) { 15099 return 0; 15100 } 15101 if (ssa->vars[var].phi_use_chain) { 15102 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; 15103 do { 15104 if (!ssa->vars[phi->ssa_var].no_val) { 15105 return 0; 15106 } 15107 phi = zend_ssa_next_use_phi(ssa, var, phi); 15108 } while (phi); 15109 } 15110 } 15111 15112 if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && 15113 ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { 15114 /* bad type */ 15115 return 0; 15116 } 15117 15118 return 1; 15119} 15120 15121static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) 15122{ 15123 if (!zend_jit_var_supports_reg(ssa, var)) { 15124 return 0; 15125 } 15126 15127 if (ssa->vars[var].definition >= 0) { 15128 uint32_t def = ssa->vars[var].definition; 15129 if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { 15130 return 0; 15131 } 15132 } 15133 15134 if (ssa->vars[var].use_chain >= 0) { 15135 int use = ssa->vars[var].use_chain; 15136 15137 do { 15138 if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && 15139 !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { 15140 return 0; 15141 } 15142 use = zend_ssa_next_use(ssa->ops, var, use); 15143 } while (use >= 0); 15144 } 15145 15146 return 1; 15147} 15148 15149static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 15150{ 15151 uint32_t op1_info, op2_info; 15152 15153 switch (opline->opcode) { 15154 case ZEND_FETCH_DIM_R: 15155 op1_info = OP1_INFO(); 15156 op2_info = OP2_INFO(); 15157 if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && 15158 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || 15159 ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && 15160 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { 15161 return ZEND_REGSET(ZREG_FCARG1); 15162 } 15163 break; 15164 default: 15165 break; 15166 } 15167 15168 return ZEND_REGSET_EMPTY; 15169} 15170 15171static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 15172{ 15173 uint32_t op1_info, op2_info, res_info; 15174 zend_regset regset = ZEND_REGSET_SCRATCH; 15175 15176 switch (opline->opcode) { 15177 case ZEND_NOP: 15178 case ZEND_OP_DATA: 15179 case ZEND_JMP: 15180 case ZEND_RETURN: 15181 regset = ZEND_REGSET_EMPTY; 15182 break; 15183 case ZEND_QM_ASSIGN: 15184 if (ssa_op->op1_def == current_var || 15185 ssa_op->result_def == current_var) { 15186 regset = ZEND_REGSET_EMPTY; 15187 break; 15188 } 15189 /* break missing intentionally */ 15190 case ZEND_SEND_VAL: 15191 case ZEND_SEND_VAL_EX: 15192 if (opline->op2_type == IS_CONST) { 15193 break; 15194 } 15195 if (ssa_op->op1_use == current_var) { 15196 regset = ZEND_REGSET(ZREG_REG0); 15197 break; 15198 } 15199 op1_info = OP1_INFO(); 15200 if (!(op1_info & MAY_BE_UNDEF)) { 15201 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15202 regset = ZEND_REGSET(ZREG_FPR0); 15203 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15204 regset = ZEND_REGSET(ZREG_REG0); 15205 } else { 15206 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); 15207 } 15208 } 15209 break; 15210 case ZEND_SEND_VAR: 15211 if (opline->op2_type == IS_CONST) { 15212 break; 15213 } 15214 if (ssa_op->op1_use == current_var || 15215 ssa_op->op1_def == current_var) { 15216 regset = ZEND_REGSET_EMPTY; 15217 break; 15218 } 15219 op1_info = OP1_INFO(); 15220 if (!(op1_info & MAY_BE_UNDEF)) { 15221 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15222 regset = ZEND_REGSET(ZREG_FPR0); 15223 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15224 } else { 15225 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); 15226 if (op1_info & MAY_BE_REF) { 15227 ZEND_REGSET_INCL(regset, ZREG_REG1); 15228 } 15229 } 15230 } 15231 break; 15232 case ZEND_ASSIGN: 15233 if (ssa_op->op2_use == current_var || 15234 ssa_op->op2_def == current_var || 15235 ssa_op->op1_def == current_var || 15236 ssa_op->result_def == current_var) { 15237 regset = ZEND_REGSET_EMPTY; 15238 break; 15239 } 15240 op1_info = OP1_INFO(); 15241 op2_info = OP2_INFO(); 15242 if (opline->op1_type == IS_CV 15243 && !(op2_info & MAY_BE_UNDEF) 15244 && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 15245 if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15246 regset = ZEND_REGSET(ZREG_FPR0); 15247 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15248 regset = ZEND_REGSET(ZREG_REG0); 15249 } else { 15250 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); 15251 } 15252 } 15253 break; 15254 case ZEND_PRE_INC: 15255 case ZEND_PRE_DEC: 15256 case ZEND_POST_INC: 15257 case ZEND_POST_DEC: 15258 if (ssa_op->op1_use == current_var || 15259 ssa_op->op1_def == current_var || 15260 ssa_op->result_def == current_var) { 15261 regset = ZEND_REGSET_EMPTY; 15262 break; 15263 } 15264 op1_info = OP1_INFO(); 15265 if (opline->op1_type == IS_CV 15266 && (op1_info & MAY_BE_LONG) 15267 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15268 regset = ZEND_REGSET_EMPTY; 15269 if (op1_info & MAY_BE_DOUBLE) { 15270 regset = ZEND_REGSET(ZREG_FPR0); 15271 } 15272 if (opline->result_type != IS_UNUSED && (op1_info & MAY_BE_LONG)) { 15273 ZEND_REGSET_INCL(regset, ZREG_REG1); 15274 } 15275 } 15276 break; 15277 case ZEND_ADD: 15278 case ZEND_SUB: 15279 case ZEND_MUL: 15280 op1_info = OP1_INFO(); 15281 op2_info = OP2_INFO(); 15282 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 15283 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15284 15285 regset = ZEND_REGSET_EMPTY; 15286 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 15287 if (ssa_op->result_def != current_var && 15288 (ssa_op->op1_use != current_var || !last_use)) { 15289 ZEND_REGSET_INCL(regset, ZREG_REG0); 15290 } 15291 res_info = RES_INFO(); 15292 if (res_info & MAY_BE_DOUBLE) { 15293 ZEND_REGSET_INCL(regset, ZREG_REG0); 15294 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15295 ZEND_REGSET_INCL(regset, ZREG_FPR1); 15296 } else if (res_info & MAY_BE_GUARD) { 15297 ZEND_REGSET_INCL(regset, ZREG_REG0); 15298 } 15299 } 15300 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 15301 if (ssa_op->result_def != current_var) { 15302 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15303 } 15304 } 15305 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 15306 if (zend_is_commutative(opline->opcode)) { 15307 if (ssa_op->result_def != current_var) { 15308 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15309 } 15310 } else { 15311 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15312 if (ssa_op->result_def != current_var && 15313 (ssa_op->op1_use != current_var || !last_use)) { 15314 ZEND_REGSET_INCL(regset, ZREG_FPR1); 15315 } 15316 } 15317 } 15318 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 15319 if (ssa_op->result_def != current_var && 15320 (ssa_op->op1_use != current_var || !last_use) && 15321 (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { 15322 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15323 } 15324 } 15325 } 15326 break; 15327 case ZEND_BW_OR: 15328 case ZEND_BW_AND: 15329 case ZEND_BW_XOR: 15330 case ZEND_SL: 15331 case ZEND_SR: 15332 case ZEND_MOD: 15333 op1_info = OP1_INFO(); 15334 op2_info = OP2_INFO(); 15335 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 15336 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 15337 regset = ZEND_REGSET_EMPTY; 15338 if (ssa_op->result_def != current_var && 15339 (ssa_op->op1_use != current_var || !last_use)) { 15340 ZEND_REGSET_INCL(regset, ZREG_REG0); 15341 } 15342 } 15343 break; 15344 case ZEND_IS_SMALLER: 15345 case ZEND_IS_SMALLER_OR_EQUAL: 15346 case ZEND_IS_EQUAL: 15347 case ZEND_IS_NOT_EQUAL: 15348 case ZEND_IS_IDENTICAL: 15349 case ZEND_IS_NOT_IDENTICAL: 15350 case ZEND_CASE: 15351 op1_info = OP1_INFO(); 15352 op2_info = OP2_INFO(); 15353 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 15354 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15355 regset = ZEND_REGSET_EMPTY; 15356 if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { 15357 ZEND_REGSET_INCL(regset, ZREG_REG0); 15358 } 15359 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && 15360 opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { 15361 if (ssa_op->op1_use != current_var && 15362 ssa_op->op2_use != current_var) { 15363 ZEND_REGSET_INCL(regset, ZREG_REG0); 15364 } 15365 } 15366 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 15367 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15368 } 15369 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 15370 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15371 } 15372 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 15373 if (ssa_op->op1_use != current_var && 15374 ssa_op->op2_use != current_var) { 15375 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15376 } 15377 } 15378 } 15379 break; 15380 case ZEND_BOOL: 15381 case ZEND_BOOL_NOT: 15382 case ZEND_JMPZ: 15383 case ZEND_JMPNZ: 15384 case ZEND_JMPZ_EX: 15385 case ZEND_JMPNZ_EX: 15386 op1_info = OP1_INFO(); 15387 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15388 regset = ZEND_REGSET_EMPTY; 15389 if (op1_info & MAY_BE_DOUBLE) { 15390 ZEND_REGSET_INCL(regset, ZREG_FPR0); 15391 } 15392 if (opline->opcode == ZEND_BOOL || 15393 opline->opcode == ZEND_BOOL_NOT || 15394 opline->opcode == ZEND_JMPZ_EX || 15395 opline->opcode == ZEND_JMPNZ_EX) { 15396 ZEND_REGSET_INCL(regset, ZREG_REG0); 15397 } 15398 } 15399 break; 15400 case ZEND_DO_UCALL: 15401 case ZEND_DO_FCALL: 15402 case ZEND_DO_FCALL_BY_NAME: 15403 case ZEND_INCLUDE_OR_EVAL: 15404 case ZEND_GENERATOR_CREATE: 15405 case ZEND_YIELD: 15406 case ZEND_YIELD_FROM: 15407 regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); 15408 break; 15409 default: 15410 break; 15411 } 15412 15413 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 15414 if (ssa_op == ssa->ops 15415 && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL 15416 && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { 15417 ZEND_REGSET_INCL(regset, ZREG_REG0); 15418 ZEND_REGSET_INCL(regset, ZREG_REG1); 15419 } 15420 } 15421 15422 return regset; 15423} 15424 15425static size_t dasm_venners_size = 0; 15426void **dasm_labels_veneers = NULL; 15427 15428static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset) 15429{ 15430 void *veneer; 15431 ptrdiff_t na; 15432 int n, m; 15433 15434 /* try to reuse veneers for global labels */ 15435 if ((ins >> 16) == DASM_REL_LG 15436 && *(b-1) < 0 15437 && dasm_labels_veneers[-*(b-1)]) { 15438 15439 veneer = dasm_labels_veneers[-*(b-1)]; 15440 na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; 15441 n = (int)na; 15442 15443 /* check if we can jump to veneer */ 15444 if ((ptrdiff_t)n != na) { 15445 /* pass */ 15446 } else if (!(ins & 0xf800)) { /* B, BL */ 15447 if ((n & 3) == 0 && ((n+0x08000000) >> 28) == 0) { 15448 return n; 15449 } 15450 } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ 15451 if ((n & 3) == 0 && ((n+0x00100000) >> 21) == 0) { 15452 return n; 15453 } 15454 } else if ((ins & 0x3000) == 0x2000) { /* ADR */ 15455 /* pass */ 15456 } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ 15457 /* pass */ 15458 } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ 15459 if ((n & 3) == 0 && ((n+0x00008000) >> 16) == 0) { 15460 return n; 15461 } 15462 } 15463 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 15464 && (ins >> 16) == DASM_REL_A) { 15465 ptrdiff_t addr = (((ptrdiff_t)(*(b-1))) << 32) | (unsigned int)(*(b-2)); 15466 15467 if ((void*)addr >= dasm_buf && (void*)addr < dasm_end) { 15468 uint32_t exit_point = zend_jit_trace_find_exit_point((void*)addr); 15469 zend_jit_trace_info *t = zend_jit_get_current_trace_info(); 15470 15471 if (exit_point != (uint32_t)-1) { 15472 /* Use exit points table */ 15473 15474 ZEND_ASSERT(exit_point < t->exit_count); 15475 15476 veneer = (char*)buffer + dasm_getpclabel(&Dst, 1) - (t->exit_count - exit_point) * 4; 15477 na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; 15478 n = (int)na; 15479 15480 /* check if we can jump to veneer */ 15481 if ((ptrdiff_t)n != na) { 15482 ZEND_ASSERT(0); 15483 return 0; 15484 } else if (!(ins & 0xf800)) { /* B, BL */ 15485 if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) { 15486 ZEND_ASSERT(0); 15487 return 0; 15488 } 15489 } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ 15490 if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) { 15491 ZEND_ASSERT(0); 15492 return 0; 15493 } 15494 } else if ((ins & 0x3000) == 0x2000) { /* ADR */ 15495 ZEND_ASSERT(0); 15496 return 0; 15497 } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ 15498 ZEND_ASSERT(0); 15499 return 0; 15500 } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ 15501 if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) { 15502 ZEND_ASSERT(0); 15503 return 0; 15504 } 15505 } else { 15506 ZEND_ASSERT(0); 15507 return 0; 15508 } 15509 return n; 15510 } 15511 } 15512 } 15513 15514 veneer = (char*)buffer + (Dst->codesize + dasm_venners_size); 15515 15516 if (veneer > dasm_end) { 15517 return 0; /* jit_buffer_size overflow */ 15518 } 15519 15520 na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; 15521 n = (int)na; 15522 15523 /* check if we can jump to veneer */ 15524 if ((ptrdiff_t)n != na) { 15525 ZEND_ASSERT(0); 15526 return 0; 15527 } else if (!(ins & 0xf800)) { /* B, BL */ 15528 if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) { 15529 ZEND_ASSERT(0); 15530 return 0; 15531 } 15532 } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ 15533 if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) { 15534 ZEND_ASSERT(0); 15535 return 0; 15536 } 15537 } else if ((ins & 0x3000) == 0x2000) { /* ADR */ 15538 ZEND_ASSERT(0); 15539 return 0; 15540 } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ 15541 ZEND_ASSERT(0); 15542 return 0; 15543 } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ 15544 if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) { 15545 ZEND_ASSERT(0); 15546 return 0; 15547 } 15548 } else if ((ins & 0x8000)) { /* absolute */ 15549 ZEND_ASSERT(0); 15550 return 0; 15551 } else { 15552 ZEND_ASSERT(0); 15553 return 0; 15554 } 15555 15556 // TODO: support for long veneers (above 128MB) ??? 15557 15558 /* check if we can use B to jump from veneer */ 15559 na = (ptrdiff_t)cp + offset - (ptrdiff_t)veneer - 4; 15560 m = (int)na; 15561 if ((ptrdiff_t)m != na) { 15562 ZEND_ASSERT(0); 15563 return 0; 15564 } else if ((m & 3) != 0 || ((m+0x08000000) >> 28) != 0) { 15565 ZEND_ASSERT(0); 15566 return 0; 15567 } 15568 15569 /* generate B instruction */ 15570 *(uint32_t*)veneer = 0x14000000 | ((m >> 2) & 0x03ffffff); 15571 dasm_venners_size += 4; 15572 15573 if ((ins >> 16) == DASM_REL_LG 15574 && *(b-1) < 0) { 15575 /* reuse this veneer for the future jumps to global label */ 15576 dasm_labels_veneers[-*(b-1)] = veneer; 15577 /* Dst->globals[*(b-1)] = veneer; */ 15578 15579#ifdef HAVE_DISASM 15580 if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { 15581 const char *name = zend_jit_disasm_find_symbol((ptrdiff_t)cp + offset - 4, (int64_t *)(&offset)); 15582 15583 if (name && !offset) { 15584 if (strstr(name, "@veneer") == NULL) { 15585 char *new_name; 15586 15587 zend_spprintf(&new_name, 0, "%s@veneer", name); 15588 zend_jit_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4); 15589 efree(new_name); 15590 } else { 15591 zend_jit_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4); 15592 } 15593 } 15594 } 15595#endif 15596 } 15597 15598 return n; 15599} 15600 15601/* 15602 * Local variables: 15603 * tab-width: 4 15604 * c-basic-offset: 4 15605 * indent-tabs-mode: t 15606 * End: 15607 */ 15608
