/* * +----------------------------------------------------------------------+ * | Zend JIT | * +----------------------------------------------------------------------+ * | Copyright (c) The PHP Group | * +----------------------------------------------------------------------+ * | This source file is subject to version 3.01 of the PHP license, | * | that is bundled with this package in the file LICENSE, and is | * | available through the world-wide-web at the following url: | * | https://www.php.net/license/3_01.txt | * | If you did not receive a copy of the PHP license and are unable to | * | obtain it through the world-wide-web, please send a note to | * | license@php.net so we can mail you a copy immediately. | * +----------------------------------------------------------------------+ * | Authors: Dmitry Stogov | * | Xinchen Hui | * | Hao Sun | * +----------------------------------------------------------------------+ */ |.arch arm64 |.define FP, x27 |.define IP, x28 |.define IPl, w28 |.define RX, x28 // the same as VM IP reused as a general purpose reg |.define LR, x30 |.define CARG1, x0 |.define CARG2, x1 |.define CARG3, x2 |.define CARG4, x3 |.define CARG5, x4 |.define CARG6, x5 |.define CARG1w, w0 |.define CARG2w, w1 |.define CARG3w, w2 |.define CARG4w, w3 |.define CARG5w, w4 |.define CARG6w, w5 |.define RETVALx, x0 |.define RETVALw, w0 |.define FCARG1x, x0 |.define FCARG1w, w0 |.define FCARG2x, x1 |.define FCARG2w, w1 |.define SPAD, 0x20 // padding for CPU stack alignment |.define NR_SPAD, 0x30 // padding for CPU stack alignment |.define T3, [sp, #0x28] // Used to store old value of IP (CALL VM only) |.define T2, [sp, #0x20] // Used to store old value of FP (CALL VM only) |.define T1, [sp, #0x10] // We use REG0/1/2 and FPR0/1 to replace r0/1/2 and xmm0/1 in the x86 implementation. // Scratch registers |.define REG0, x8 |.define REG0w, w8 |.define REG1, x9 |.define REG1w, w9 |.define REG2, x10 |.define REG2w, w10 |.define FPR0, d0 |.define FPR1, d1 |.define ZREG_REG0, ZREG_X8 |.define ZREG_REG1, ZREG_X9 |.define ZREG_REG2, ZREG_X10 |.define ZREG_FPR0, ZREG_V0 |.define ZREG_FPR1, ZREG_V1 // Temporaries, not preserved across calls |.define TMP1, x15 |.define TMP1w, w15 |.define TMP2, x16 |.define TMP2w, w16 |.define TMP3, x17 // TODO: remember about hard-coded: mrs TMP3, tpidr_el0 |.define TMP3w, w17 |.define FPTMP, d16 |.define ZREG_TMP1, ZREG_X15 |.define ZREG_TMP2, ZREG_X16 |.define ZREG_TMP3, ZREG_X17 |.define ZREG_FPTMP, ZREG_V16 |.define HYBRID_SPAD, 32 // padding for stack alignment #define TMP_ZVAL_OFFSET 16 #define DASM_ALIGNMENT 16 const char* zend_reg_name[] = { "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "x29", "x30", "sp", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15", "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23", "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31" }; #define ZREG_FCARG1 ZREG_X0 #define ZREG_FCARG2 ZREG_X1 |.type EX, zend_execute_data, FP |.type OP, zend_op |.type ZVAL, zval |.actionlist dasm_actions |.globals zend_lb |.section code, cold_code, jmp_table static void* dasm_labels[zend_lb_MAX]; #if ZTS static size_t tsrm_ls_cache_tcb_offset = 0; # ifdef __APPLE__ struct TLVDescriptor { void* (*thunk)(struct TLVDescriptor*); uint64_t key; uint64_t offset; }; typedef struct TLVDescriptor TLVDescriptor; # endif #endif #define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) /* Encoding of immediate. */ #define MAX_IMM12 0xfff // maximum value for imm12 #define MAX_IMM16 0xffff // maximum value for imm16 #define MOVZ_IMM MAX_IMM16 // movz insn #define LDR_STR_PIMM64 (MAX_IMM12*8) // ldr/str insn for 64-bit register: pimm is imm12 * 8 #define LDR_STR_PIMM32 (MAX_IMM12*4) // ldr/str insn for 32-bit register: pimm is imm12 * 4 #define LDRB_STRB_PIMM MAX_IMM12 // ldrb/strb insn #define B_IMM (1<<27) // signed imm26 * 4 #define ADR_IMM (1<<20) // signed imm21 #define ADRP_IMM (1LL<<32) // signed imm21 * 4096 static bool arm64_may_use_b(const void *addr) { if (addr >= dasm_buf && addr < dasm_end) { return (((char*)dasm_end - (char*)dasm_buf) < B_IMM); } else if (addr >= dasm_end) { return (((char*)addr - (char*)dasm_buf) < B_IMM); } else if (addr < dasm_buf) { return (((char*)dasm_end - (char*)addr) < B_IMM); } return 0; } static bool arm64_may_use_adr(const void *addr) { if (addr >= dasm_buf && addr < dasm_end) { return (((char*)dasm_end - (char*)dasm_buf) < ADR_IMM); } else if (addr >= dasm_end) { return (((char*)addr - (char*)dasm_buf) < ADR_IMM); } else if (addr < dasm_buf) { return (((char*)dasm_end - (char*)addr) < ADR_IMM); } return 0; } static bool arm64_may_use_adrp(const void *addr) { if (addr >= dasm_buf && addr < dasm_end) { return (((char*)dasm_end - (char*)dasm_buf) < ADRP_IMM); } else if (addr >= dasm_end) { return (((char*)addr - (char*)dasm_buf) < ADRP_IMM); } else if (addr < dasm_buf) { return (((char*)dasm_end - (char*)addr) < ADRP_IMM); } return 0; } /* Determine whether "val" falls into two allowed ranges: * Range 1: [0, 0xfff] * Range 2: LSL #12 to Range 1 * Used to guard the immediate encoding for add/adds/sub/subs/cmp/cmn instructions. */ static bool arm64_may_encode_imm12(const int64_t val) { return (val >= 0 && (val <= MAX_IMM12 || !(val & 0xffffffffff000fff))); } /* Determine whether an immediate value can be encoded as the immediate operand of logical instructions. */ static bool logical_immediate_p(uint64_t value, uint32_t reg_size) { /* fast path: power of two */ if (value > 0 && !(value & (value - 1))) { return true; } if (reg_size == 32) { if (dasm_imm13((uint32_t)value, (uint32_t)value) != -1) { return true; } } else if (reg_size == 64) { if (dasm_imm13((uint32_t)value, (uint32_t)(value >> 32)) != -1) { return true; } } else { ZEND_UNREACHABLE(); } return false; } /* Not Implemented Yet */ |.macro NIY || //ZEND_ASSERT(0); | brk #0 |.endmacro |.macro NIY_STUB || //ZEND_ASSERT(0); | brk #0 |.endmacro |.macro ADD_HYBRID_SPAD ||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | add sp, sp, # HYBRID_SPAD ||#endif |.endmacro |.macro SUB_HYBRID_SPAD ||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | sub sp, sp, # HYBRID_SPAD ||#endif |.endmacro /* Move address into register. TODO: Support 52-bit address */ |.macro LOAD_ADDR, reg, addr | // 48-bit virtual address || if (((uintptr_t)(addr)) == 0) { | mov reg, xzr || } else if (((uintptr_t)(addr)) <= MOVZ_IMM) { | movz reg, #((uint64_t)(addr)) || } else if (arm64_may_use_adr((void*)(addr))) { | adr reg, &addr || } else if (arm64_may_use_adrp((void*)(addr))) { | adrp reg, &(((uintptr_t)(addr))) || if (((uintptr_t)(addr)) & 0xfff) { | add reg, reg, #(((uintptr_t)(addr)) & 0xfff) || } || } else if ((uintptr_t)(addr) & 0xffff) { | movz reg, #((uintptr_t)(addr) & 0xffff) || if (((uintptr_t)(addr) >> 16) & 0xffff) { | movk reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 || } || if (((uintptr_t)(addr) >> 32) & 0xffff) { | movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 || } || } else if (((uintptr_t)(addr) >> 16) & 0xffff) { | movz reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 || if (((uintptr_t)(addr) >> 32) & 0xffff) { | movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 || } || } else { | movz reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 || } |.endmacro /* Move 32-bit immediate value into register. */ |.macro LOAD_32BIT_VAL, reg, val || if (((uint32_t)(val)) <= MOVZ_IMM) { | movz reg, #((uint32_t)(val)) || } else if (((uint32_t)(val) & 0xffff)) { | movz reg, #((uint32_t)(val) & 0xffff) || if ((((uint32_t)(val) >> 16) & 0xffff)) { | movk reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 || } || } else { | movz reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 || } |.endmacro /* Move 64-bit immediate value into register. */ |.macro LOAD_64BIT_VAL, reg, val || if (((uint64_t)(val)) == 0) { | mov reg, xzr || } else if (((uint64_t)(val)) <= MOVZ_IMM) { | movz reg, #((uint64_t)(val)) || } else if (~((uint64_t)(val)) <= MOVZ_IMM) { | movn reg, #(~((uint64_t)(val))) || } else if ((uint64_t)(val) & 0xffff) { | movz reg, #((uint64_t)(val) & 0xffff) || if (((uint64_t)(val) >> 16) & 0xffff) { | movk reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 || } || if (((uint64_t)(val) >> 32) & 0xffff) { | movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 || } || if ((((uint64_t)(val) >> 48) & 0xffff)) { | movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 || } || } else if (((uint64_t)(val) >> 16) & 0xffff) { | movz reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 || if (((uint64_t)(val) >> 32) & 0xffff) { | movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 || } || if ((((uint64_t)(val) >> 48) & 0xffff)) { | movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 || } || } else if (((uint64_t)(val) >> 32) & 0xffff) { | movz reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 || if ((((uint64_t)(val) >> 48) & 0xffff)) { | movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 || } || } else { | movz reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 || } |.endmacro /* Extract the low 8 bits from 'src_reg' into 'dst_reg'. * Note: 0xff can be encoded as imm for 'and' instruction. */ |.macro GET_LOW_8BITS, dst_reg, src_reg | and dst_reg, src_reg, #0xff |.endmacro /* Bitwise operation with immediate. 'bw_ins' can be and/orr/eor/ands. * 32-bit and 64-bit registers are distinguished. */ |.macro BW_OP_32_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg || if (val == 0) { | bw_ins dst_reg, src_reg1, wzr || } else if (logical_immediate_p((uint32_t)val, 32)) { | bw_ins dst_reg, src_reg1, #val || } else { | LOAD_32BIT_VAL tmp_reg, val | bw_ins dst_reg, src_reg1, tmp_reg || } |.endmacro |.macro BW_OP_64_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg || if (val == 0) { | bw_ins dst_reg, src_reg1, xzr || } else if (logical_immediate_p(val, 64)) { | bw_ins dst_reg, src_reg1, #val || } else { | LOAD_64BIT_VAL tmp_reg, val | bw_ins dst_reg, src_reg1, tmp_reg || } |.endmacro /* Test bits 'tst' with immediate. 32-bit and 64-bit registers are distinguished. */ |.macro TST_32_WITH_CONST, reg, val, tmp_reg || if (val == 0) { | tst reg, wzr || } else if (logical_immediate_p((uint32_t)val, 32)) { | tst reg, #val || } else { | LOAD_32BIT_VAL tmp_reg, val | tst reg, tmp_reg || } |.endmacro |.macro TST_64_WITH_CONST, reg, val, tmp_reg || if (val == 0) { | tst reg, xzr || } else if (logical_immediate_p(val, 64)) { | tst reg, #val || } else { | LOAD_64BIT_VAL tmp_reg, val | tst reg, tmp_reg || } |.endmacro /* Test bits between 64-bit register with constant 1. */ |.macro TST_64_WITH_ONE, reg | tst reg, #1 |.endmacro /* Compare a register value with immediate. 32-bit and 64-bit registers are distinguished. * Note: Comparing 64-bit register with 32-bit immediate is handled in CMP_64_WITH_CONST_32. */ |.macro CMP_32_WITH_CONST, reg, val, tmp_reg || if (val == 0) { | cmp reg, wzr || } else if (arm64_may_encode_imm12((int64_t)(val))) { | cmp reg, #val || } else if (arm64_may_encode_imm12((int64_t)(-val))) { | cmn reg, #-val || } else { | LOAD_32BIT_VAL tmp_reg, val | cmp reg, tmp_reg || } |.endmacro |.macro CMP_64_WITH_CONST_32, reg, val, tmp_reg || if (val == 0) { | cmp reg, xzr || } else if (arm64_may_encode_imm12((int64_t)(val))) { | cmp reg, #val || } else if (arm64_may_encode_imm12((int64_t)(-val))) { | cmn reg, #-val || } else { | LOAD_32BIT_VAL tmp_reg, val | cmp reg, tmp_reg || } |.endmacro |.macro CMP_64_WITH_CONST, reg, val, tmp_reg || if (val == 0) { | cmp reg, xzr || } else if (arm64_may_encode_imm12((int64_t)(val))) { | cmp reg, #val || } else if (arm64_may_encode_imm12((int64_t)(-val))) { | cmn reg, #-val || } else { | LOAD_64BIT_VAL tmp_reg, val | cmp reg, tmp_reg || } |.endmacro /* Add/sub a register value with immediate. 'add_sub_ins' can be add/sub/adds/subs. * 32-bit and 64-bit registers are distinguished. * Note: Case of 64-bit register with 32-bit immediate is handled in ADD_SUB_64_WITH_CONST_32. */ |.macro ADD_SUB_32_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg || if (val == 0) { | add_sub_ins dst_reg, src_reg1, wzr || } else if (arm64_may_encode_imm12((int64_t)(val))) { | add_sub_ins dst_reg, src_reg1, #val || } else { | LOAD_32BIT_VAL tmp_reg, val | add_sub_ins dst_reg, src_reg1, tmp_reg || } |.endmacro |.macro ADD_SUB_64_WITH_CONST_32, add_sub_ins, dst_reg, src_reg1, val, tmp_reg || if (val == 0) { | add_sub_ins dst_reg, src_reg1, xzr || } else if (arm64_may_encode_imm12((int64_t)(val))) { | add_sub_ins dst_reg, src_reg1, #val || } else { | LOAD_32BIT_VAL tmp_reg, val | add_sub_ins dst_reg, src_reg1, tmp_reg || } |.endmacro |.macro ADD_SUB_64_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg || if (val == 0) { | add_sub_ins dst_reg, src_reg1, xzr || } else if (arm64_may_encode_imm12((int64_t)(val))) { | add_sub_ins dst_reg, src_reg1, #val || } else { | LOAD_64BIT_VAL tmp_reg, val | add_sub_ins dst_reg, src_reg1, tmp_reg || } |.endmacro /* Memory access(load/store) with 32-bit 'offset'. * Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register. * 8-bit, 32-bit and 64-bit registers to be transferred are distinguished. * Note: 'reg' can be used as 'tmp_reg' if 1) 'reg' is one GPR, AND 2) 'reg' != 'base_reg', AND 3) ins is 'ldr'. */ |.macro MEM_ACCESS_64_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg || if (((uintptr_t)(offset)) > LDR_STR_PIMM64) { | LOAD_32BIT_VAL tmp_reg, offset | ldr_str_ins reg, [base_reg, tmp_reg] || } else { | ldr_str_ins reg, [base_reg, #(offset)] || } |.endmacro |.macro MEM_ACCESS_32_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg || if (((uintptr_t)(offset)) > LDR_STR_PIMM32) { | LOAD_32BIT_VAL tmp_reg, offset | ldr_str_ins reg, [base_reg, tmp_reg] || } else { | ldr_str_ins reg, [base_reg, #(offset)] || } |.endmacro |.macro MEM_ACCESS_8_WITH_UOFFSET, ldrb_strb_ins, reg, base_reg, offset, tmp_reg || if (((uintptr_t)(offset)) > LDRB_STRB_PIMM) { | LOAD_32BIT_VAL tmp_reg, offset | ldrb_strb_ins reg, [base_reg, tmp_reg] || } else { | ldrb_strb_ins reg, [base_reg, #(offset)] || } |.endmacro /* Memory access(load/store) with 64-bit 'offset'. * Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register. */ |.macro MEM_ACCESS_64_WITH_UOFFSET_64, ldr_str_ins, reg, base_reg, offset, tmp_reg || if (((uintptr_t)(offset)) > LDR_STR_PIMM64) { | LOAD_64BIT_VAL tmp_reg, offset | ldr_str_ins reg, [base_reg, tmp_reg] || } else { | ldr_str_ins reg, [base_reg, #(offset)] || } |.endmacro /* ZTS: get thread local variable "_tsrm_ls_cache" */ |.macro LOAD_TSRM_CACHE, reg ||#ifdef __APPLE__ | .long 0xd53bd071 // TODO: hard-coded: mrs TMP3, tpidrro_el0 | and TMP3, TMP3, #0xfffffffffffffff8 | MEM_ACCESS_64_WITH_UOFFSET_64 ldr, TMP3, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->key << 3), TMP1 | MEM_ACCESS_64_WITH_UOFFSET_64 ldr, reg, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->offset), TMP1 ||#else | .long 0xd53bd051 // TODO: hard-coded: mrs TMP3, tpidr_el0 || ZEND_ASSERT(tsrm_ls_cache_tcb_offset <= LDR_STR_PIMM64); | ldr reg, [TMP3, #tsrm_ls_cache_tcb_offset] ||#endif |.endmacro |.macro LOAD_ADDR_ZTS, reg, struct, field | .if ZTS | LOAD_TSRM_CACHE TMP3 | ADD_SUB_64_WITH_CONST_32 add, reg, TMP3, (struct.._offset + offsetof(zend_..struct, field)), reg | .else | LOAD_ADDR reg, &struct.field | .endif |.endmacro /* Store address 'addr' into memory 'mem'. */ |.macro ADDR_STORE, mem, addr, tmp_reg | LOAD_ADDR tmp_reg, addr | str tmp_reg, mem |.endmacro /* Store a register value 'reg' into memory 'addr'. * For ZTS mode, base register with unsigned offset variant is used, * and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */ |.macro MEM_STORE, str_ins, reg, addr, tmp_reg || if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) { | adr tmp_reg, &addr | str_ins reg, [tmp_reg] || } else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) { | adrp tmp_reg, &(((uintptr_t)(addr))) | str_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)] || } else { | LOAD_ADDR tmp_reg, addr | str_ins reg, [tmp_reg] || } |.endmacro |.macro MEM_STORE_64_ZTS, str_ins, reg, struct, field, tmp_reg | .if ZTS | LOAD_TSRM_CACHE TMP3 | MEM_ACCESS_64_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg | .else | MEM_STORE str_ins, reg, &struct.field, tmp_reg | .endif |.endmacro |.macro MEM_STORE_32_ZTS, str_ins, reg, struct, field, tmp_reg | .if ZTS | LOAD_TSRM_CACHE TMP3 | MEM_ACCESS_32_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg | .else | MEM_STORE str_ins, reg, &struct.field, tmp_reg | .endif |.endmacro |.macro MEM_STORE_8_ZTS, strb_ins, reg, struct, field, tmp_reg | .if ZTS | LOAD_TSRM_CACHE TMP3 | MEM_ACCESS_8_WITH_UOFFSET strb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg | .else | MEM_STORE strb_ins, reg, &struct.field, tmp_reg | .endif |.endmacro /* Load value from memory 'addr' and write it into register 'reg'. * For ZTS mode, base register with unsigned offset variant is used, * and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */ |.macro MEM_LOAD, ldr_ins, reg, addr, tmp_reg || if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) { | adr tmp_reg, &addr | ldr_ins reg, [tmp_reg] || } else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) { | adrp tmp_reg, &(((uintptr_t)(addr))) | ldr_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)] || } else { | LOAD_ADDR tmp_reg, addr | ldr_ins reg, [tmp_reg] || } |.endmacro |.macro MEM_LOAD_64_ZTS, ldr_ins, reg, struct, field, tmp_reg | .if ZTS | LOAD_TSRM_CACHE TMP3 | MEM_ACCESS_64_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg | .else | MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg | .endif |.endmacro |.macro MEM_LOAD_32_ZTS, ldr_ins, reg, struct, field, tmp_reg | .if ZTS | LOAD_TSRM_CACHE TMP3 | MEM_ACCESS_32_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg | .else | MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg | .endif |.endmacro |.macro MEM_LOAD_8_ZTS, ldrb_ins, reg, struct, field, tmp_reg | .if ZTS | LOAD_TSRM_CACHE TMP3 | MEM_ACCESS_8_WITH_UOFFSET ldrb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg | .else | MEM_LOAD ldrb_ins, reg, &struct.field, tmp_reg | .endif |.endmacro /* Conduct arithmetic operation between the value in memory 'addr' and register value in 'reg', * and the computation result is stored back in 'reg'. 'op_ins' can be add/sub. */ |.macro MEM_LOAD_OP, op_ins, ldr_ins, reg, addr, tmp_reg1, tmp_reg2 | MEM_LOAD ldr_ins, tmp_reg1, addr, tmp_reg2 | op_ins reg, reg, tmp_reg1 |.endmacro |.macro MEM_LOAD_OP_ZTS, op_ins, ldr_ins, reg, struct, field, tmp_reg1, tmp_reg2 | .if ZTS | LOAD_TSRM_CACHE TMP3 | MEM_ACCESS_64_WITH_UOFFSET ldr_ins, tmp_reg2, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg1 | op_ins reg, reg, tmp_reg2 | .else | MEM_LOAD_OP op_ins, ldr_ins, reg, &struct.field, tmp_reg1, tmp_reg2 | .endif |.endmacro /* Conduct arithmetic operation between the value in memory 'addr' and operand 'op', and the computation * result is stored back to memory 'addr'. Operand 'op' can be either a register value or an immediate value. * Currently, only add instruction is used as 'op_ins'. * Note: It should be guaranteed that the immediate value can be encoded for 'op_ins'. */ |.macro MEM_UPDATE, op_ins, ldr_ins, str_ins, op, addr, tmp_reg1, tmp_reg2 | LOAD_ADDR tmp_reg2, addr | ldr_ins, tmp_reg1, [tmp_reg2] | op_ins tmp_reg1, tmp_reg1, op | str_ins tmp_reg1, [tmp_reg2] |.endmacro |.macro MEM_UPDATE_ZTS, op_ins, ldr_ins, str_ins, op, struct, field, tmp_reg1, tmp_reg2 | .if ZTS | LOAD_TSRM_CACHE TMP3 || if (((uintptr_t)(struct.._offset+offsetof(zend_..struct, field))) > LDR_STR_PIMM64) { | LOAD_32BIT_VAL tmp_reg1, (struct.._offset+offsetof(zend_..struct, field)) | ldr_ins tmp_reg2, [TMP3, tmp_reg1] | op_ins tmp_reg2, tmp_reg2, op | str_ins tmp_reg2, [TMP3, tmp_reg1] || } else { | ldr_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] | op_ins tmp_reg2, tmp_reg2, op | str_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] || } | .else | MEM_UPDATE op_ins, ldr_ins, str_ins, op, &struct.field, tmp_reg1, tmp_reg2 | .endif |.endmacro |.macro EXT_CALL, func, tmp_reg || if (arm64_may_use_b(func)) { | bl &func || } else { | LOAD_ADDR tmp_reg, func | blr tmp_reg || } |.endmacro |.macro EXT_JMP, func, tmp_reg || if (arm64_may_use_b(func)) { | b &func || } else { | LOAD_ADDR tmp_reg, func | br tmp_reg || } |.endmacro |.macro SAVE_IP || if (GCC_GLOBAL_REGS) { | str IP, EX->opline || } |.endmacro |.macro LOAD_IP || if (GCC_GLOBAL_REGS) { | ldr IP, EX->opline || } |.endmacro |.macro LOAD_IP_ADDR, addr || if (GCC_GLOBAL_REGS) { | LOAD_ADDR IP, addr || } else { | ADDR_STORE EX->opline, addr, RX || } |.endmacro |.macro LOAD_IP_ADDR_ZTS, struct, field, tmp_reg | .if ZTS || if (GCC_GLOBAL_REGS) { | LOAD_TSRM_CACHE IP | MEM_ACCESS_64_WITH_UOFFSET ldr, IP, IP, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg || } else { | LOAD_TSRM_CACHE RX | ADD_SUB_64_WITH_CONST_32 add, RX, RX, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg | str RX, EX->opline || } | .else | LOAD_IP_ADDR &struct.field | .endif |.endmacro |.macro GET_IP, reg || if (GCC_GLOBAL_REGS) { | mov reg, IP || } else { | ldr reg, EX->opline || } |.endmacro /* Update IP with register 'reg'. Note: shift variant is handled by ADD_IP_SHIFT. */ |.macro ADD_IP, reg, tmp_reg || if (GCC_GLOBAL_REGS) { | add IP, IP, reg || } else { | ldr tmp_reg, EX->opline | add tmp_reg, tmp_reg, reg | str tmp_reg, EX->opline || } |.endmacro |.macro ADD_IP_SHIFT, reg, shift, tmp_reg || if (GCC_GLOBAL_REGS) { | add IP, IP, reg, shift || } else { | ldr tmp_reg, EX->opline | add tmp_reg, tmp_reg, reg, shift | str tmp_reg, EX->opline || } |.endmacro /* Update IP with 32-bit immediate 'val'. */ |.macro ADD_IP_WITH_CONST, val, tmp_reg || ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val))); || if (GCC_GLOBAL_REGS) { | add IP, IP, #val || } else { | ldr tmp_reg, EX->opline | add tmp_reg, tmp_reg, #val | str tmp_reg, EX->opline || } |.endmacro |.macro JMP_IP, tmp_reg || if (GCC_GLOBAL_REGS) { | ldr tmp_reg, [IP] | br tmp_reg || } else { | ldr tmp_reg, EX:CARG1->opline | br tmp_reg || } |.endmacro |.macro CMP_IP, addr, tmp_reg1, tmp_reg2 | LOAD_ADDR tmp_reg1, addr || if (GCC_GLOBAL_REGS) { | cmp IP, tmp_reg1 || } else { | ldr tmp_reg2, EX->opline | cmp tmp_reg2, tmp_reg1 || } |.endmacro |.macro LOAD_ZVAL_ADDR, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { | LOAD_ADDR reg, Z_ZV(addr) || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { || if (Z_OFFSET(addr)) { | ADD_SUB_64_WITH_CONST_32 add, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), reg || } else { || if (Z_REG(addr) == ZREG_RSP) { | mov reg, sp || } else { | mov reg, Rx(Z_REG(addr)) || } || } || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro GET_Z_TYPE_INFO, reg, zv | ldr reg, [zv, #offsetof(zval,u1.type_info)] |.endmacro |.macro SET_Z_TYPE_INFO, zv, type, tmp_reg | LOAD_32BIT_VAL tmp_reg, type | str tmp_reg, [zv, #offsetof(zval,u1.type_info)] |.endmacro |.macro GET_ZVAL_TYPE, reg, addr, tmp_reg || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | MEM_ACCESS_8_WITH_UOFFSET ldrb, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.v.type), tmp_reg |.endmacro |.macro GET_ZVAL_TYPE_INFO, reg, addr, tmp_reg || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | MEM_ACCESS_32_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg |.endmacro |.macro SET_ZVAL_TYPE_INFO, addr, type, tmp_reg1, tmp_reg2 || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | LOAD_32BIT_VAL tmp_reg1, type | MEM_ACCESS_32_WITH_UOFFSET str, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg2 |.endmacro |.macro SET_ZVAL_TYPE_INFO_FROM_REG, addr, reg, tmp_reg || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | MEM_ACCESS_32_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg |.endmacro |.macro GET_Z_PTR, reg, zv | ldr reg, [zv] |.endmacro |.macro GET_ZVAL_PTR, reg, addr, tmp_reg || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | MEM_ACCESS_64_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg |.endmacro |.macro SET_ZVAL_PTR, addr, reg, tmp_reg || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg |.endmacro |.macro UNDEF_OPLINE_RESULT, tmp_reg | ldr REG0, EX->opline | ldr REG0w, OP:REG0->result.var | add REG0, FP, REG0 | SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg |.endmacro |.macro UNDEF_OPLINE_RESULT_IF_USED, tmp_reg1, tmp_reg2 | ldrb tmp_reg1, OP:RX->result_type | TST_32_WITH_CONST tmp_reg1, (IS_TMP_VAR|IS_VAR), tmp_reg2 | beq >1 | ldr REG0w, OP:RX->result.var | add REG0, FP, REG0 | SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg1 |1: |.endmacro /* Floating-point comparison between register 'reg' and value from memory 'addr'. * Note: the equivalent macros in JIT/x86 are SSE_AVX_OP and SSE_OP. */ |.macro DOUBLE_CMP, reg, addr, tmp_reg, fp_tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { | MEM_LOAD ldr, Rd(fp_tmp_reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg) | fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) | fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) || } else if (Z_MODE(addr) == IS_REG) { | fcmp Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) || } else { || ZEND_UNREACHABLE(); || } |.endmacro /* Convert LONG value 'val' into DOUBLE type, and move it into FP register 'reg'. * Note: the equivalent macro in JIT/x86 is SSE_GET_LONG. */ |.macro DOUBLE_GET_LONG, reg, val, tmp_reg || if (val == 0) { | fmov Rd(reg-ZREG_V0), xzr // TODO: "movi d0, #0" is not recognized by DynASM/arm64 || } else { | LOAD_64BIT_VAL Rx(tmp_reg), val | scvtf Rd(reg-ZREG_V0), Rx(tmp_reg) || } |.endmacro /* Convert LONG value from memory 'addr' into DOUBLE type, and move it into FP register 'reg'. * Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_LVAL. */ |.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg1, tmp_reg2 || if (Z_MODE(addr) == IS_CONST_ZVAL) { | DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg1 || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg2) | scvtf Rd(reg-ZREG_V0), Rx(tmp_reg1) || } else if (Z_MODE(addr) == IS_REG) { | scvtf Rd(reg-ZREG_V0), Rx(Z_REG(addr)) || } else { || ZEND_UNREACHABLE(); || } |.endmacro /* Floating-point arithmetic operation between two FP registers. * Note: the equivalent macro in JIT/x86 is AVX_MATH_REG. */ |.macro DOUBLE_MATH_REG, opcode, dst_reg, op1_reg, op2_reg || switch (opcode) { || case ZEND_ADD: | fadd Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) || break; || case ZEND_SUB: | fsub Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) || break; || case ZEND_MUL: | fmul Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) || break; || case ZEND_DIV: | fdiv Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) || break; || } |.endmacro /* Conduct binary operation between register 'reg' and value from memory 'addr', * and the computation result is stored in 'reg'. * For LONG_ADD_SUB, 'add_sub_ins' can be adds/subs. For LONG_BW_OP, 'bw_ins' can be and/orr/eor. * For LONG_CMP, 'cmp' instruction is used by default and only flag registers are affected. * Note: the equivalent macro in JIT/x86 is LONG_OP. */ |.macro LONG_ADD_SUB, add_sub_ins, reg, addr, tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { | ADD_SUB_64_WITH_CONST add_sub_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg | add_sub_ins Rx(reg), Rx(reg), tmp_reg || } else if (Z_MODE(addr) == IS_REG) { | add_sub_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro LONG_BW_OP, bw_ins, reg, addr, tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { | BW_OP_64_WITH_CONST bw_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg | bw_ins Rx(reg), Rx(reg), tmp_reg || } else if (Z_MODE(addr) == IS_REG) { | bw_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro LONG_CMP, reg, addr, tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { | CMP_64_WITH_CONST Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg | cmp Rx(reg), tmp_reg || } else if (Z_MODE(addr) == IS_REG) { | cmp Rx(reg), Rx(Z_REG(addr)) || } else { || ZEND_UNREACHABLE(); || } |.endmacro /* Conduct add/sub between value from memory 'addr' and an immediate value 'val', and * the computation result is stored back into 'addr'. * Note: it should be guaranteed that 'val' can be encoded into add/sub instruction. */ |.macro LONG_ADD_SUB_WITH_IMM, add_sub_ins, addr, val, tmp_reg1, tmp_reg2 || ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val))); || if (Z_MODE(addr) == IS_MEM_ZVAL) { || if (((uint32_t)(Z_OFFSET(addr))) > LDR_STR_PIMM64) { | LOAD_32BIT_VAL tmp_reg2, Z_OFFSET(addr) | ldr tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2] | add_sub_ins tmp_reg1, tmp_reg1, #val | str tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2] || } else { | ldr tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)] | add_sub_ins tmp_reg1, tmp_reg1, #val | str tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)] || } || } else if (Z_MODE(addr) == IS_REG) { | add_sub_ins Rx(Z_REG(addr)), Rx(Z_REG(addr)), #val || } else { || ZEND_UNREACHABLE(); || } |.endmacro /* Compare value from memory 'addr' with immediate value 'val'. * Note: the equivalent macro in JIT/x86 is LONG_OP_WITH_CONST. */ |.macro LONG_CMP_WITH_CONST, addr, val, tmp_reg1, tmp_reg2 || if (Z_MODE(addr) == IS_MEM_ZVAL) { | MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 | CMP_64_WITH_CONST tmp_reg1, val, tmp_reg2 || } else if (Z_MODE(addr) == IS_REG) { | CMP_64_WITH_CONST Rx(Z_REG(addr)), val, tmp_reg1 || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro GET_ZVAL_LVAL, reg, addr, tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { || if (Z_LVAL_P(Z_ZV(addr)) == 0) { | mov Rx(reg), xzr || } else { | LOAD_64BIT_VAL Rx(reg), Z_LVAL_P(Z_ZV(addr)) || } || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(reg), Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg || } else if (Z_MODE(addr) == IS_REG) { || if (reg != Z_REG(addr)) { | mov Rx(reg), Rx(Z_REG(addr)) || } || } else { || ZEND_UNREACHABLE(); || } |.endmacro |.macro LONG_MATH, opcode, reg, addr, tmp_reg1 || switch (opcode) { || case ZEND_ADD: | LONG_ADD_SUB adds, reg, addr, tmp_reg1 || break; || case ZEND_SUB: | LONG_ADD_SUB subs, reg, addr, tmp_reg1 || break; || case ZEND_BW_OR: | LONG_BW_OP orr, reg, addr, tmp_reg1 || break; || case ZEND_BW_AND: | LONG_BW_OP and, reg, addr, tmp_reg1 || break; || case ZEND_BW_XOR: | LONG_BW_OP eor, reg, addr, tmp_reg1 || break; || default: || ZEND_UNREACHABLE(); || } |.endmacro |.macro LONG_MATH_REG, opcode, dst_reg, src_reg1, src_reg2 || switch (opcode) { || case ZEND_ADD: | adds dst_reg, src_reg1, src_reg2 || break; || case ZEND_SUB: | subs dst_reg, src_reg1, src_reg2 || break; || case ZEND_BW_OR: | orr dst_reg, src_reg1, src_reg2 || break; || case ZEND_BW_AND: | and dst_reg, src_reg1, src_reg2 || break; || case ZEND_BW_XOR: | eor dst_reg, src_reg1, src_reg2 || break; || default: || ZEND_UNREACHABLE(); || } |.endmacro /* Store LONG value into memory 'addr'. * This LONG value can be an immediate value i.e. 'val' in macro SET_ZVAL_LVAL, or * a register value i.e. 'reg' in macro SET_ZVAL_LVAL_FROM_REG. */ |.macro SET_ZVAL_LVAL_FROM_REG, addr, reg, tmp_reg || if (Z_MODE(addr) == IS_REG) { | mov Rx(Z_REG(addr)), reg || } else { || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg || } |.endmacro |.macro SET_ZVAL_LVAL, addr, val, tmp_reg1, tmp_reg2 || if (val == 0) { | SET_ZVAL_LVAL_FROM_REG addr, xzr, tmp_reg2 || } else { | LOAD_64BIT_VAL tmp_reg1, val | SET_ZVAL_LVAL_FROM_REG addr, tmp_reg1, tmp_reg2 || } |.endmacro /* Store DOUBLE value from FP register 'reg' into memory 'addr'. * Note: the equivalent macro in JIT/x86 is SSE_SET_ZVAL_DVAL. */ |.macro SET_ZVAL_DVAL, addr, reg, tmp_reg || if (Z_MODE(addr) == IS_REG) { || if (reg != Z_REG(addr)) { | fmov Rd(Z_REG(addr)-ZREG_V0), Rd(reg-ZREG_V0) || } || } else { || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | MEM_ACCESS_64_WITH_UOFFSET str, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) || } |.endmacro /* Load DOUBLE value from memory 'addr' into FP register 'reg'. * Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_DVAL. */ |.macro GET_ZVAL_DVAL, reg, addr, tmp_reg || if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { || if (Z_MODE(addr) == IS_CONST_ZVAL) { | MEM_LOAD ldr, Rd(reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg) || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) || } else if (Z_MODE(addr) == IS_REG) { | fmov Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) || } else { || ZEND_UNREACHABLE(); || } || } |.endmacro |.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg || if (Z_TYPE_P(zv) > IS_TRUE) { || if (Z_TYPE_P(zv) == IS_DOUBLE) { || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; | MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1) | SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 || } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; | DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), tmp_reg1 | SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 || } else { | // In x64, if the range of this LONG value can be represented via INT type, only move the low 32 bits into dst_addr. | // Note that imm32 is signed extended to 64 bits during mov. | // In aarch64, we choose to handle both cases in the same way. Even though 4 mov's are used for 64-bit value and 2 mov's are | // needed for 32-bit value, an extra ext insn is needed for 32-bit vlaue. | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) || } || } || if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { || if (dst_def_info == MAY_BE_DOUBLE) { || if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) || } || } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1< IS_TRUE) { || if (Z_TYPE_P(zv) == IS_DOUBLE) { || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? || Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : fp_tmp_reg); | MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1) | SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 | SET_ZVAL_DVAL res_addr, dst_reg, tmp_reg2 || } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { || if (Z_MODE(dst_addr) == IS_REG) { | DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), tmp_reg1 | SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg2 || } else if (Z_MODE(res_addr) == IS_REG) { | DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), tmp_reg1 | SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg2 || } else { | DOUBLE_GET_LONG fp_tmp_reg, Z_LVAL_P(zv), tmp_reg1 | SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg2 | SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg2 || } || } else { || if (Z_MODE(dst_addr) == IS_REG) { | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg1) || } else if (Z_MODE(res_addr) == IS_REG) { | SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) | SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg1) || } else { | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) | SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) || } || } || } || if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { || if (dst_def_info == MAY_BE_DOUBLE) { || if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) || } || } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<1, tmp_reg || } | GC_ADDREF value_ptr_reg, tmp_reg |1: || } |.endmacro |.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg, tmp_reg || if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { || if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg || } | ldr tmp_reg, [value_ptr_reg] | add tmp_reg, tmp_reg, #2 | str tmp_reg, [value_ptr_reg] |1: || } |.endmacro |.macro ZVAL_DEREF, reg, info, tmp_reg || if (info & MAY_BE_REF) { | IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1, tmp_reg | GET_Z_PTR reg, reg | add reg, reg, #offsetof(zend_reference, val) |1: || } |.endmacro |.macro SET_EX_OPLINE, op, tmp_reg || if (op == last_valid_opline) { || zend_jit_use_last_valid_opline(); | SAVE_IP || } else { | ADDR_STORE EX->opline, op, tmp_reg || if (!GCC_GLOBAL_REGS) { || zend_jit_reset_last_valid_opline(); || } || } |.endmacro // arg1 "zval" should be in FCARG1x |.macro ZVAL_DTOR_FUNC, var_info, opline, tmp_reg || do { || if (!((var_info) & MAY_BE_GUARD) || && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { || zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); || if (type == IS_STRING && !ZEND_DEBUG) { | EXT_CALL _efree, tmp_reg || break; || } else if (type == IS_ARRAY) { || if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { || if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { | SET_EX_OPLINE opline, tmp_reg || } | EXT_CALL zend_array_destroy, tmp_reg || } else { | EXT_CALL zend_jit_array_free, tmp_reg || } || break; || } else if (type == IS_OBJECT) { || if (opline) { | SET_EX_OPLINE opline, REG0 || } | EXT_CALL zend_objects_store_del, tmp_reg || break; || } || } || if (opline) { | SET_EX_OPLINE opline, tmp_reg || } | EXT_CALL rc_dtor_func, tmp_reg || } while(0); |.endmacro |.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline, tmp_reg1, tmp_reg2 || if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) { || if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { || if (cold) { | IF_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 |.cold_code |1: || } else { | IF_NOT_ZVAL_REFCOUNTED addr, >4, tmp_reg1, tmp_reg2 || } || } | // if (!Z_DELREF_P(cv)) { | GET_ZVAL_PTR FCARG1x, addr, Rx(tmp_reg2) | GC_DELREF FCARG1x, Rw(tmp_reg1) || if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) { || if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) { || if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { | bne >3 || } else { | bne >4 || } || } | // zval_dtor_func(r); | ZVAL_DTOR_FUNC op_info, opline, Rx(tmp_reg1) || if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { | b >4 || } |3: || } || if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { || if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) { || zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); | IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1, tmp_reg1 | IF_NOT_ZVAL_COLLECTABLE ref_addr, >4, tmp_reg1, tmp_reg2 | GET_ZVAL_PTR FCARG1x, ref_addr, Rx(tmp_reg2) |1: || } | IF_GC_MAY_NOT_LEAK FCARG1x, >4, Rw(tmp_reg1), Rw(tmp_reg2) | // gc_possible_root(Z_COUNTED_P(z)) | EXT_CALL gc_possible_root, Rx(tmp_reg1) || } || if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { | b >4 |.code || } |4: || } |.endmacro |.macro FREE_OP, op_type, op, op_info, cold, opline, tmp_reg1, tmp_reg2 || if (op_type & (IS_VAR|IS_TMP_VAR)) { || zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); | ZVAL_PTR_DTOR addr, op_info, 0, cold, opline, tmp_reg1, tmp_reg2 || } |.endmacro |.macro SEPARATE_ARRAY, addr, op_info, cold, tmp_reg1, tmp_reg2 || if (RC_MAY_BE_N(op_info)) { || if (Z_REG(addr) != ZREG_FP) { | GET_ZVAL_LVAL ZREG_REG0, addr, Rx(tmp_reg1) || if (RC_MAY_BE_1(op_info)) { | // if (GC_REFCOUNT() > 1) | ldr Rw(tmp_reg1), [REG0] | cmp Rw(tmp_reg1), #1 | bls >2 || } || if (Z_REG(addr) != ZREG_FCARG1 || Z_OFFSET(addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, addr || } | EXT_CALL zend_jit_zval_array_dup, REG0 | mov REG0, RETVALx |2: | mov FCARG1x, REG0 || } else { | GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1) || if (RC_MAY_BE_1(op_info)) { | // if (GC_REFCOUNT() > 1) | ldr Rw(tmp_reg1), [FCARG1x] | cmp Rw(tmp_reg1), #1 || if (cold) { | bhi >1 |.cold_code |1: || } else { | bls >2 || } || } | IF_NOT_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 | GC_DELREF FCARG1x, Rw(tmp_reg1) |1: | EXT_CALL zend_array_dup, REG0 | mov REG0, RETVALx | SET_ZVAL_PTR addr, REG0, Rx(tmp_reg1) | SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX, Rw(tmp_reg1), Rx(tmp_reg2) | mov FCARG1x, REG0 || if (RC_MAY_BE_1(op_info)) { || if (cold) { | b >2 |.code || } || } |2: || } || } else { | GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1) || } |.endmacro /* argument is passed in FCARG1x */ |.macro EFREE_REFERENCE ||#if ZEND_DEBUG | mov FCARG2x, xzr // filename | mov CARG3w, wzr // lineno | mov CARG4, xzr | mov CARG5, xzr | EXT_CALL _efree, REG0 ||#else ||#ifdef HAVE_BUILTIN_CONSTANT_P | EXT_CALL _efree_32, REG0 ||#else | EXT_CALL _efree, REG0 ||#endif ||#endif |.endmacro |.macro EMALLOC, size, op_array, opline ||#if ZEND_DEBUG || const char *filename = op_array->filename ? op_array->filename->val : NULL; | mov FCARG1x, #size | LOAD_ADDR FCARG2x, filename | LOAD_32BIT_VAL CARG3w, opline->lineno | mov CARG4, xzr | mov CARG5, xzr | EXT_CALL _emalloc, REG0 | mov REG0, RETVALx ||#else ||#ifdef HAVE_BUILTIN_CONSTANT_P || if (size > 24 && size <= 32) { | EXT_CALL _emalloc_32, REG0 | mov REG0, RETVALx || } else { | mov FCARG1x, #size | EXT_CALL _emalloc, REG0 | mov REG0, RETVALx || } ||#else | mov FCARG1x, #size | EXT_CALL _emalloc, REG0 | mov REG0, RETVALx ||#endif ||#endif |.endmacro |.macro OBJ_RELEASE, reg, exit_label, tmp_reg1, tmp_reg2 | GC_DELREF Rx(reg), Rw(tmp_reg1) | bne >1 | // zend_objects_store_del(obj); || if (reg != ZREG_FCARG1) { | mov FCARG1x, Rx(reg) || } | EXT_CALL zend_objects_store_del, Rx(tmp_reg1) | b exit_label |1: | IF_GC_MAY_NOT_LEAK Rx(reg), >1, Rw(tmp_reg1), Rw(tmp_reg2) | // gc_possible_root(obj) || if (reg != ZREG_FCARG1) { | mov FCARG1x, Rx(reg) || } | EXT_CALL gc_possible_root, Rx(tmp_reg1) |1: |.endmacro |.macro UNDEFINED_OFFSET, opline || if (opline == last_valid_opline) { || zend_jit_use_last_valid_opline(); | bl ->undefined_offset_ex || } else { | SET_EX_OPLINE opline, REG0 | bl ->undefined_offset || } |.endmacro |.macro UNDEFINED_INDEX, opline || if (opline == last_valid_opline) { || zend_jit_use_last_valid_opline(); | bl ->undefined_index_ex || } else { | SET_EX_OPLINE opline, REG0 | bl ->undefined_index || } |.endmacro |.macro CANNOT_ADD_ELEMENT, opline || if (opline == last_valid_opline) { || zend_jit_use_last_valid_opline(); | bl ->cannot_add_element_ex || } else { | SET_EX_OPLINE opline, REG0 | bl ->cannot_add_element || } |.endmacro static bool reuse_ip = 0; static bool delayed_call_chain = 0; static uint32_t delayed_call_level = 0; static const zend_op *last_valid_opline = NULL; static bool use_last_vald_opline = 0; static bool track_last_valid_opline = 0; static int jit_return_label = -1; static uint32_t current_trace_num = 0; static uint32_t allowed_opt_flags = 0; static void zend_jit_track_last_valid_opline(void) { use_last_vald_opline = 0; track_last_valid_opline = 1; } static void zend_jit_use_last_valid_opline(void) { if (track_last_valid_opline) { use_last_vald_opline = 1; track_last_valid_opline = 0; } } static bool zend_jit_trace_uses_initial_ip(void) { return use_last_vald_opline; } static void zend_jit_set_last_valid_opline(const zend_op *target_opline) { if (!reuse_ip) { track_last_valid_opline = 0; last_valid_opline = target_opline; } } static void zend_jit_reset_last_valid_opline(void) { track_last_valid_opline = 0; last_valid_opline = NULL; } static void zend_jit_start_reuse_ip(void) { zend_jit_reset_last_valid_opline(); reuse_ip = 1; } static int zend_jit_reuse_ip(dasm_State **Dst) { if (!reuse_ip) { zend_jit_start_reuse_ip(); | // call = EX(call); | ldr RX, EX->call } return 1; } static void zend_jit_stop_reuse_ip(void) { reuse_ip = 0; } static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { |->interrupt_handler: | SAVE_IP | //EG(vm_interrupt) = 0; | MEM_STORE_8_ZTS strb, wzr, executor_globals, vm_interrupt, TMP1 | //if (EG(timed_out)) { | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, timed_out, TMP1 | cbz TMP1w, >1 | //zend_timeout(); | EXT_CALL zend_timeout, TMP1 |1: | //} else if (zend_interrupt_function) { if (zend_interrupt_function) { | //zend_interrupt_function(execute_data); | mov CARG1, FP | EXT_CALL zend_interrupt_function, TMP1 | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 | cbz REG0, >1 | EXT_CALL zend_jit_exception_in_interrupt_handler_helper, TMP1 |1: | //ZEND_VM_ENTER(); | //execute_data = EG(current_execute_data); | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 | LOAD_IP } | //ZEND_VM_CONTINUE() if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD | JMP_IP TMP1 } else if (GCC_GLOBAL_REGS) { | ldp x29, x30, [sp], # SPAD // stack alignment | JMP_IP TMP1 } else { | ldp FP, RX, T2 // retore FP and IP | ldp x29, x30, [sp], # NR_SPAD // stack alignment | mov RETVALx, #1 // ZEND_VM_ENTER | ret } return 1; } static int zend_jit_exception_handler_stub(dasm_State **Dst) { |->exception_handler: if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { const void *handler = zend_get_opcode_handler_func(EG(exception_op)); | ADD_HYBRID_SPAD | EXT_CALL handler, REG0 | JMP_IP TMP1 } else { const void *handler = EG(exception_op)->handler; if (GCC_GLOBAL_REGS) { | ldp x29, x30, [sp], # SPAD // stack alignment | EXT_JMP handler, REG0 } else { | mov FCARG1x, FP | EXT_CALL handler, REG0 | ldp FP, RX, T2 // retore FP and IP | ldp x29, x30, [sp], # NR_SPAD // stack alignment | tbnz RETVALw, #31, >1 | mov RETVALw, #1 // ZEND_VM_ENTER |1: | ret } } return 1; } static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) { |->exception_handler_undef: | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, opline_before_exception, REG0 | ldrb TMP1w, OP:REG0->result_type | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w | beq >1 | ldr REG0w, OP:REG0->result.var | add REG0, REG0, FP | SET_Z_TYPE_INFO REG0, IS_UNDEF, TMP1w |1: | b ->exception_handler return 1; } static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst) { zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); |->exception_handler_free_op1_op2: | UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w | ldrb TMP1w, OP:RX->op1_type | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w | beq >9 | ldr REG0w, OP:RX->op1.var | add REG0, REG0, FP | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 |9: | ldrb TMP1w, OP:RX->op2_type | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w | beq >9 | ldr REG0w, OP:RX->op2.var | add REG0, REG0, FP | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 |9: | b ->exception_handler return 1; } static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst) { zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); |->exception_handler_free_op2: | MEM_LOAD_64_ZTS ldr, RX, executor_globals, opline_before_exception, REG0 | UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w | ldrb TMP1w, OP:RX->op2_type | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w | beq >9 | ldr REG0w, OP:RX->op2.var | add REG0, REG0, FP | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 |9: | b ->exception_handler return 1; } static int zend_jit_leave_function_stub(dasm_State **Dst) { |->leave_function_handler: if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w | bne >1 | EXT_CALL zend_jit_leave_nested_func_helper, REG0 | ADD_HYBRID_SPAD | JMP_IP TMP1 |1: | EXT_CALL zend_jit_leave_top_func_helper, REG0 | ADD_HYBRID_SPAD | JMP_IP TMP1 } else { if (GCC_GLOBAL_REGS) { | ldp x29, x30, [sp], # SPAD // stack alignment } else { | mov FCARG2x, FP | ldp FP, RX, T2 // retore FP and IP | ldp x29, x30, [sp], # NR_SPAD // stack alignment } | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w | bne >1 | EXT_JMP zend_jit_leave_nested_func_helper, REG0 |1: | EXT_JMP zend_jit_leave_top_func_helper, REG0 } return 1; } static int zend_jit_leave_throw_stub(dasm_State **Dst) { |->leave_throw_handler: | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { if (GCC_GLOBAL_REGS) { | ldrb TMP1w, OP:IP->opcode | cmp TMP1w, #ZEND_HANDLE_EXCEPTION | beq >5 | // EG(opline_before_exception) = opline; | MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2 |5: | // opline = EG(exception_op); | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 | // HANDLE_EXCEPTION() | b ->exception_handler } else { | GET_IP TMP1 | ldrb TMP2w, OP:TMP1->opcode | cmp TMP2w, #ZEND_HANDLE_EXCEPTION | beq >5 | // EG(opline_before_exception) = opline; | MEM_STORE_64_ZTS str, TMP1, executor_globals, opline_before_exception, TMP2 |5: | // opline = EG(exception_op); | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 | ldp FP, RX, T2 // retore FP and IP | ldp x29, x30, [sp], # NR_SPAD // stack alignment | mov RETVALx, #2 // ZEND_VM_LEAVE | ret } return 1; } static int zend_jit_icall_throw_stub(dasm_State **Dst) { |->icall_throw_handler: | // zend_rethrow_exception(zend_execute_data *execute_data) | ldr IP, EX->opline | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { | ldrb TMP1w, OP:IP->opcode | cmp TMP1w, #ZEND_HANDLE_EXCEPTION | beq >1 | // EG(opline_before_exception) = opline; | MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2 |1: | // opline = EG(exception_op); | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 || if (GCC_GLOBAL_REGS) { | str IP, EX->opline || } | // HANDLE_EXCEPTION() | b ->exception_handler return 1; } static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) { zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); |->throw_cannot_pass_by_ref: | ldr REG0, EX->opline | ldr REG1w, OP:REG0->result.var | add REG1, REG1, RX | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w | // last EX(call) frame may be delayed | ldr TMP1, EX->call | cmp RX, TMP1 | beq >1 | ldr REG1, EX->call | str REG1, EX:RX->prev_execute_data | str RX, EX->call |1: | mov RX, REG0 | ldr FCARG1w, OP:REG0->op2.num | EXT_CALL zend_cannot_pass_by_reference, REG0 | ldrb TMP1w, OP:RX->op1_type | cmp TMP1w, #IS_TMP_VAR | bne >9 | ldr REG0w, OP:RX->op1.var | add REG0, REG0, FP | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 |9: | b ->exception_handler return 1; } static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) { |->undefined_offset_ex: | SAVE_IP | b ->undefined_offset return 1; } static int zend_jit_undefined_offset_stub(dasm_State **Dst) { |->undefined_offset: #ifdef __APPLE__ | stp x29, x30, [sp, # -16]! | mov x29, sp #endif | //sub r4, 8 | ldr REG0, EX->opline | ldr REG1w, OP:REG0->result.var | add REG1, REG1, FP | SET_Z_TYPE_INFO REG1, IS_NULL, TMP1w | ldrb REG1w, OP:REG0->op2_type | cmp REG1w, #IS_CONST | bne >2 | ldrsw REG1, OP:REG0->op2.constant | add REG0, REG0, REG1 | b >3 |2: | ldr REG0w, OP:REG0->op2.var | add REG0, REG0, FP |3: | mov CARG1, #E_WARNING | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT | ldr CARG3, [REG0] #ifdef __APPLE__ | str CARG3, [sp, #-16]! | EXT_CALL zend_error, REG0 | add sp, sp, #16 | ldp x29, x30, [sp], #16 | ret #else | EXT_JMP zend_error, REG0 // tail call | //add r4, 8 // stack alignment | //ret #endif return 1; } static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) { |->undefined_index_ex: | SAVE_IP | b ->undefined_index return 1; } static int zend_jit_undefined_index_stub(dasm_State **Dst) { |->undefined_index: #ifdef __APPLE__ | stp x29, x30, [sp, # -16]! | mov x29, sp #endif | //sub r4, 8 | ldr REG0, EX->opline | ldr REG1w, OP:REG0->result.var | add REG1, REG1, FP | SET_Z_TYPE_INFO REG1, IS_NULL, TMP1w | ldrb REG1w, OP:REG0->op2_type | cmp REG1w, #IS_CONST | bne >2 | ldrsw REG1, OP:REG0->op2.constant | add REG0, REG0, REG1 | b >3 |2: | ldr REG0w, OP:REG0->op2.var | add REG0, REG0, FP |3: | mov CARG1, #E_WARNING | LOAD_ADDR CARG2, "Undefined array key \"%s\"" | ldr CARG3, [REG0] | add CARG3, CARG3, #offsetof(zend_string, val) #ifdef __APPLE__ | str CARG3, [sp, #-16]! | EXT_CALL zend_error, REG0 | add sp, sp, #16 | ldp x29, x30, [sp], #16 | ret #else | EXT_JMP zend_error, REG0 // tail call | //add r4, 8 | //ret #endif return 1; } static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) { |->cannot_add_element_ex: | SAVE_IP | b ->cannot_add_element return 1; } static int zend_jit_cannot_add_element_stub(dasm_State **Dst) { |->cannot_add_element: | // sub r4, 8 | ldr REG0, EX->opline | ldrb TMP1w, OP:REG0->result_type | cmp TMP1w, #IS_UNUSED | beq >1 | ldr REG0w, OP:REG0->result.var | add REG0, REG0, FP | SET_Z_TYPE_INFO REG0, IS_NULL, TMP1w |1: | mov CARG1, xzr | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" | EXT_JMP zend_throw_error, REG0 // tail call | // add r4, 8 | //ret return 1; } static int zend_jit_undefined_function_stub(dasm_State **Dst) { |->undefined_function: | ldr REG0, EX->opline | mov CARG1, xzr | LOAD_ADDR CARG2, "Call to undefined function %s()" | ldrsw CARG3, [REG0, #offsetof(zend_op, op2.constant)] | ldr CARG3, [REG0, CARG3] | add CARG3, CARG3, #offsetof(zend_string, val) #ifdef __APPLE__ | str CARG3, [sp, #-16]! #endif | EXT_CALL zend_throw_error, REG0 #ifdef __APPLE__ | add sp, sp, #16 #endif | b ->exception_handler return 1; } static int zend_jit_negative_shift_stub(dasm_State **Dst) { |->negative_shift: | ldr RX, EX->opline | LOAD_ADDR CARG1, zend_ce_arithmetic_error | LOAD_ADDR CARG2, "Bit shift by negative number" | EXT_CALL zend_throw_error, REG0 | b ->exception_handler_free_op1_op2 return 1; } static int zend_jit_mod_by_zero_stub(dasm_State **Dst) { |->mod_by_zero: | ldr RX, EX->opline | LOAD_ADDR CARG1, zend_ce_division_by_zero_error | LOAD_ADDR CARG2, "Modulo by zero" | EXT_CALL zend_throw_error, REG0 | b ->exception_handler_free_op1_op2 return 1; } static int zend_jit_invalid_this_stub(dasm_State **Dst) { |->invalid_this: | UNDEF_OPLINE_RESULT TMP1w | mov CARG1, xzr | LOAD_ADDR CARG2, "Using $this when not in object context" | EXT_CALL zend_throw_error, REG0 | b ->exception_handler return 1; } static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { return 1; } |->hybrid_runtime_jit: | EXT_CALL zend_runtime_jit, REG0 | JMP_IP TMP1 return 1; } static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { return 1; } |->hybrid_profile_jit: | // ++zend_jit_profile_counter; | LOAD_ADDR REG0, &zend_jit_profile_counter | ldr TMP1, [REG0] | add TMP1, TMP1, #1 | str TMP1, [REG0] | // op_array = (zend_op_array*)EX(func); | ldr REG0, EX->func | // run_time_cache = EX(run_time_cache); | ldr REG2, EX->run_time_cache | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | // ++ZEND_COUNTER_INFO(op_array) || if ((zend_jit_profile_counter_rid * sizeof(void*)) > LDR_STR_PIMM64) { | LOAD_32BIT_VAL TMP1, (zend_jit_profile_counter_rid * sizeof(void*)) | ldr TMP2, [REG2, TMP1] | add TMP2, TMP2, #1 | str TMP2, [REG2, TMP1] || } else { | ldr TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] | add TMP2, TMP2, #1 | str TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] || } | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() | ldr TMP1, [REG0, #offsetof(zend_jit_op_array_extension, orig_handler)] | br TMP1 return 1; } static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { return 1; } |->hybrid_hot_code: || ZEND_ASSERT(ZEND_JIT_COUNTER_INIT <= MOVZ_IMM); | movz TMP1w, #ZEND_JIT_COUNTER_INIT | strh TMP1w, [REG2] | mov FCARG1x, FP | GET_IP FCARG2x | EXT_CALL zend_jit_hot_func, REG0 | JMP_IP TMP1 return 1; } /* * This code is based Mike Pall's "Hashed profile counters" idea, implemented * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual * property disclosure and research opportunities" email * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html * * In addition we use a variation of Knuth's multiplicative hash function * described at https://code.i-harness.com/en/q/a21ce * * uint64_t hash(uint64_t x) { * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; * x = x ^ (x >> 31); * return x; * } * * uint_32_t hash(uint32_t x) { * x = ((x >> 16) ^ x) * 0x45d9f3b; * x = ((x >> 16) ^ x) * 0x45d9f3b; * x = (x >> 16) ^ x; * return x; * } * */ static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) { | ldr REG0, EX->func | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG2, [REG1, #offsetof(zend_jit_op_array_hot_extension, counter)] | ldrh TMP2w, [REG2] | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w | strh TMP2w, [REG2] | ble ->hybrid_hot_code | GET_IP REG2 | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] | sub REG2, REG2, TMP1 | // divide by sizeof(zend_op) || ZEND_ASSERT(sizeof(zend_op) == 32); | add TMP1, REG1, REG2, asr #2 | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_hot_extension, orig_handlers)] | br TMP1 return 1; } static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { return 1; } |->hybrid_func_hot_counter: return zend_jit_hybrid_hot_counter_stub(Dst, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); } static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { return 1; } |->hybrid_loop_hot_counter: return zend_jit_hybrid_hot_counter_stub(Dst, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); } static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { return 1; } // On entry from counter stub: // REG2 -> zend_op_trace_info.counter |->hybrid_hot_trace: | mov TMP1w, #ZEND_JIT_COUNTER_INIT | strh TMP1w, [REG2] | mov FCARG1x, FP | GET_IP FCARG2x | EXT_CALL zend_jit_trace_hot_root, REG0 | tbnz RETVALw, #31, >1 // Result is < 0 on failure. | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 | LOAD_IP | JMP_IP TMP1 |1: | EXT_JMP zend_jit_halt_op->handler, REG0 return 1; } static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) { | ldr REG0, EX->func | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG1, [REG1, #offsetof(zend_jit_op_array_trace_extension, offset)] | add TMP1, REG1, IP | ldr REG2, [TMP1, #offsetof(zend_op_trace_info, counter)] | ldrh TMP2w, [REG2] | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w | strh TMP2w, [REG2] | ble ->hybrid_hot_trace // Note: "REG1 + IP" is re-calculated as TMP1 is used as temporary register by the prior // ADD_SUB_32_WITH_CONST. Will optimize in the future if more temporary registers are available. | add TMP1, REG1, IP | ldr TMP2, [TMP1, #offsetof(zend_op_trace_info, orig_handler)] | br TMP2 return 1; } static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { return 1; } |->hybrid_func_trace_counter: return zend_jit_hybrid_trace_counter_stub(Dst, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); } static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { return 1; } |->hybrid_ret_trace_counter: return zend_jit_hybrid_trace_counter_stub(Dst, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); } static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { return 1; } |->hybrid_loop_trace_counter: return zend_jit_hybrid_trace_counter_stub(Dst, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); } static int zend_jit_trace_halt_stub(dasm_State **Dst) { |->trace_halt: if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD | EXT_JMP zend_jit_halt_op->handler, REG0 } else if (GCC_GLOBAL_REGS) { | ldp x29, x30, [sp], # SPAD // stack alignment | mov IP, xzr // PC must be zero | ret } else { | ldp FP, RX, T2 // retore FP and IP | ldp x29, x30, [sp], # NR_SPAD // stack alignment | movn RETVALx, #0 // ZEND_VM_RETURN (-1) | ret } return 1; } static int zend_jit_trace_exit_stub(dasm_State **Dst) { |->trace_exit: | | // Save CPU registers(32 GP regs + 32 FP regs) on stack in the order of d31 to x0 | | stp d30, d31, [sp, #-16]! | stp d28, d29, [sp, #-16]! | stp d26, d27, [sp, #-16]! | stp d24, d25, [sp, #-16]! | stp d22, d23, [sp, #-16]! | stp d20, d21, [sp, #-16]! | stp d18, d19, [sp, #-16]! | stp d16, d17, [sp, #-16]! | //stp d14, d15, [sp, #-16]! // we don't use preserved registers yet | //stp d12, d13, [sp, #-16]! | //stp d10, d11, [sp, #-16]! | //stp d8, d9, [sp, #-16]! | stp d6, d7, [sp, #(-16*5)]! | stp d4, d5, [sp, #-16]! | stp d2, d3, [sp, #-16]! | stp d0, d1, [sp, #-16]! | | //str x30, [sp, #-16]! // we don't use callee-saved registers yet (x31 can be omitted) | stp x28, x29, [sp, #(-16*2)]! // we have to store RX (x28) | //stp x26, x27, [sp, #-16]! // we don't use callee-saved registers yet | //stp x24, x25, [sp, #-16]! | //stp x22, x23, [sp, #-16]! | //stp x20, x21, [sp, #-16]! | //stp x18, x19, [sp, #-16]! | //stp x16, x17, [sp, #-16]! // we don't need temporary registers | stp x14, x15, [sp, #-(16*7)]! | stp x12, x13, [sp, #-16]! | stp x10, x11, [sp, #-16]! | stp x8, x9, [sp, #-16]! | stp x6, x7, [sp, #-16]! | stp x4, x5, [sp, #-16]! | stp x2, x3, [sp, #-16]! | stp x0, x1, [sp, #-16]! | | mov FCARG1w, TMP1w // exit_num | mov FCARG2x, sp | | // EX(opline) = opline | SAVE_IP | // zend_jit_trace_exit(trace_num, exit_num) | EXT_CALL zend_jit_trace_exit, REG0 | | add sp, sp, #(32 * 16) // restore sp | | tst RETVALw, RETVALw | bne >1 // not zero | // execute_data = EG(current_execute_data) | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 | // opline = EX(opline) | LOAD_IP if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD | JMP_IP TMP1 } else if (GCC_GLOBAL_REGS) { | ldp x29, x30, [sp], # SPAD // stack alignment | JMP_IP TMP1 } else { | ldp FP, RX, T2 // retore FP and IP | ldp x29, x30, [sp], # NR_SPAD // stack alignment | mov RETVALx, #1 // ZEND_VM_ENTER | ret } |1: | blt ->trace_halt | // execute_data = EG(current_execute_data) | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0 | // opline = EX(opline) | LOAD_IP | // check for interrupt (try to avoid this ???) | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 | cbnz TMP1w, ->interrupt_handler if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] | ldr REG0, [IP, REG0] | br REG0 } else if (GCC_GLOBAL_REGS) { | ldp x29, x30, [sp], # SPAD // stack alignment | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] | ldr REG0, [IP, REG0] | br REG0 } else { | ldr IP, EX->opline | mov FCARG1x, FP | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] | ldr REG0, [IP, REG0] | blr REG0 | | tst RETVALw, RETVALw | blt ->trace_halt | | ldp FP, RX, T2 // retore FP and IP | ldp x29, x30, [sp], # NR_SPAD // stack alignment | mov RETVALx, #1 // ZEND_VM_ENTER | ret } return 1; } static int zend_jit_trace_escape_stub(dasm_State **Dst) { |->trace_escape: | if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD | JMP_IP, TMP1 } else if (GCC_GLOBAL_REGS) { | ldp x29, x30, [sp], # SPAD // stack alignment | JMP_IP, TMP1 } else { | ldp FP, RX, T2 // retore FP and IP | ldp x29, x30, [sp], # NR_SPAD // stack alignment | mov RETVALx, #1 // ZEND_VM_ENTER | ret } return 1; } /* Keep 32 exit points in a single code block */ #define ZEND_JIT_EXIT_POINTS_SPACING 4 // bl = bytes #define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) { uint32_t i; | bl >2 |1: for (i = 1; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { | bl >2 } |2: | adr TMP1, <1 | sub TMP1, lr, TMP1 | lsr TMP1, TMP1, #2 if (n) { | ADD_SUB_32_WITH_CONST add, TMP1w, TMP1w, n, TMP2w } | b ->trace_exit // pass exit_num in TMP1w return 1; } #ifdef CONTEXT_THREADED_JIT static int zend_jit_context_threaded_call_stub(dasm_State **Dst) { |->context_threaded_call: | NIY_STUB // TODO return 1; } #endif static int zend_jit_assign_const_stub(dasm_State **Dst) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; |->assign_const: | stp x29, x30, [sp, #-32]! | mov x29, sp if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, IS_CONST, val_addr, val_info, 0, 0)) { return 0; } | ldp x29, x30, [sp], #32 | ret return 1; } static int zend_jit_assign_tmp_stub(dasm_State **Dst) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; |->assign_tmp: | stp x29, x30, [sp, #-32]! | mov x29, sp if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, IS_TMP_VAR, val_addr, val_info, 0, 0)) { return 0; } | ldp x29, x30, [sp], #32 | ret return 1; } static int zend_jit_assign_var_stub(dasm_State **Dst) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; |->assign_var: | stp x29, x30, [sp, #-32]! | mov x29, sp if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, IS_VAR, val_addr, val_info, 0, 0)) { return 0; } | ldp x29, x30, [sp], #32 | ret return 1; } static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; |->assign_cv_noref: | stp x29, x30, [sp, #-32]! | mov x29, sp if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, IS_CV, val_addr, val_info, 0, 0)) { return 0; } | ldp x29, x30, [sp], #32 | ret return 1; } static int zend_jit_assign_cv_stub(dasm_State **Dst) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; |->assign_cv: | stp x29, x30, [sp, #-32]! | mov x29, sp if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, IS_CV, val_addr, val_info, 0, 0)) { return 0; } | ldp x29, x30, [sp], #32 | ret return 1; } static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(interrupt_handler, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(exception_handler, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(exception_handler_undef, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(exception_handler_free_op1_op2, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(exception_handler_free_op2, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(leave_function, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(leave_throw, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(icall_throw, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(throw_cannot_pass_by_ref, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(undefined_offset, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(undefined_index, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(cannot_add_element, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(undefined_offset_ex, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(undefined_index_ex, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(cannot_add_element_ex, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(undefined_function, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(negative_shift, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(mod_by_zero, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(invalid_this, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(trace_halt, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(trace_exit, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(trace_escape, SP_ADJ_JIT, SP_ADJ_VM), JIT_STUB(hybrid_runtime_jit, SP_ADJ_VM, SP_ADJ_NONE), JIT_STUB(hybrid_profile_jit, SP_ADJ_VM, SP_ADJ_NONE), JIT_STUB(hybrid_hot_code, SP_ADJ_VM, SP_ADJ_NONE), JIT_STUB(hybrid_func_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), JIT_STUB(hybrid_loop_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), JIT_STUB(hybrid_hot_trace, SP_ADJ_VM, SP_ADJ_NONE), JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), JIT_STUB(hybrid_ret_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), JIT_STUB(assign_const, SP_ADJ_RET, SP_ADJ_ASSIGN), JIT_STUB(assign_tmp, SP_ADJ_RET, SP_ADJ_ASSIGN), JIT_STUB(assign_var, SP_ADJ_RET, SP_ADJ_ASSIGN), JIT_STUB(assign_cv_noref, SP_ADJ_RET, SP_ADJ_ASSIGN), JIT_STUB(assign_cv, SP_ADJ_RET, SP_ADJ_ASSIGN), #ifdef CONTEXT_THREADED_JIT JIT_STUB(context_threaded_call, SP_ADJ_NONE, SP_ADJ_NONE), #endif }; #ifdef HAVE_GDB # if 0 typedef struct _Unwind_Context _Unwind_Context; typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *); extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *); extern uintptr_t _Unwind_GetCFA(_Unwind_Context *); typedef struct _zend_jit_unwind_arg { int cnt; uintptr_t cfa[3]; } zend_jit_unwind_arg; static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a) { zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a; arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx); arg->cnt++; if (arg->cnt == 3) { return 5; // _URC_END_OF_STACK } return 0; // _URC_NO_REASON; } static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) { zend_jit_unwind_arg arg; memset(&arg, 0, sizeof(arg)); _Unwind_Backtrace(zend_jit_unwind_cb, &arg); if (arg.cnt == 3) { sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1]; } } # else static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) { uintptr_t ret; __asm__ ( "ldr %0, [x29]\n\t" "sub %0 ,%0, x29" : "=r"(ret)); sp_adj[SP_ADJ_VM] = ret; } # endif extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data); static zend_never_inline void zend_jit_set_sp_adj_vm(void) { void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *); orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data; zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data; execute_ex(NULL); // set sp_adj[SP_ADJ_VM] zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data; } #endif static int zend_jit_setup(void) { allowed_opt_flags = 0; #if ZTS tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0); #endif memset(sp_adj, 0, sizeof(sp_adj)); #ifdef HAVE_GDB sp_adj[SP_ADJ_RET] = 0; sp_adj[SP_ADJ_ASSIGN] = 32; if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { zend_jit_set_sp_adj_vm(); // set sp_adj[SP_ADJ_VM] #ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD #else || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM]; #endif } else if (GCC_GLOBAL_REGS) { || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD; // sub r4, SPAD } else { || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD; // sub r4, NR_SPAD } #endif return SUCCESS; } static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) { | brk #0 return 1; } static int zend_jit_align_func(dasm_State **Dst) { reuse_ip = 0; delayed_call_chain = 0; last_valid_opline = NULL; use_last_vald_opline = 0; track_last_valid_opline = 0; jit_return_label = -1; |.align 16 return 1; } static int zend_jit_prologue(dasm_State **Dst) { if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | SUB_HYBRID_SPAD } else if (GCC_GLOBAL_REGS) { | stp x29, x30, [sp, # -SPAD]! // stack alignment |// mov x29, sp } else { | stp x29, x30, [sp, # -NR_SPAD]! // stack alignment |// mov x29, sp | stp FP, RX, T2 // save FP and IP | mov FP, FCARG1x } return 1; } static int zend_jit_label(dasm_State **Dst, unsigned int label) { |=>label: return 1; } static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) { | // call->prev_execute_data = EX(call); if (call_level == 1) { | str xzr, EX:RX->prev_execute_data } else { | ldr REG0, EX->call | str REG0, EX:RX->prev_execute_data } | // EX(call) = call; | str RX, EX->call delayed_call_chain = 0; return 1; } static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) { if (last_valid_opline == opline) { zend_jit_use_last_valid_opline(); } else if (GCC_GLOBAL_REGS && last_valid_opline) { zend_jit_use_last_valid_opline(); | LOAD_64BIT_VAL TMP1, (opline - last_valid_opline) * sizeof(zend_op) | ADD_IP TMP1, TMP2 } else { | LOAD_IP_ADDR opline } zend_jit_set_last_valid_opline(opline); return 1; } static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg) { return zend_jit_set_ip(Dst, opline); } static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) { if (delayed_call_chain) { if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { return 0; } } if (!zend_jit_set_ip(Dst, opline)) { return 0; } reuse_ip = 0; return 1; } static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) { | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 if (exit_addr) { | cbnz TMP1w, &exit_addr } else if (last_valid_opline == opline) { || zend_jit_use_last_valid_opline(); | cbnz TMP1w, ->interrupt_handler } else { | cbnz TMP1w, >1 |.cold_code |1: | LOAD_IP_ADDR opline | b ->interrupt_handler |.code } return 1; } static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) { if (timeout_exit_addr) { | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 | cbz TMP1w, =>loop_label | b &timeout_exit_addr } else { | b =>loop_label } return 1; } static int zend_jit_check_exception(dasm_State **Dst) { | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 | cbnz TMP2, ->exception_handler return 1; } static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) { if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 | cbnz TMP2, ->exception_handler_undef return 1; } return zend_jit_check_exception(Dst); } static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) { current_trace_num = trace_num; | // EG(jit_trace_num) = trace_num; | LOAD_32BIT_VAL TMP1w, trace_num | MEM_STORE_32_ZTS str, TMP1w, executor_globals, jit_trace_num, TMP2 return 1; } static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t) { uint32_t i; const void *exit_addr; /* Emit veneers table for exit points (B instruction for each exit number) */ |.cold_code for (i = 0; i < t->exit_count; i++) { exit_addr = zend_jit_trace_get_exit_addr(i); if (!exit_addr) { return 0; } | b &exit_addr } |=>1: // end of the code |.code return 1; } static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) { int ret = 0; uint8_t *p, *end; const void *veneer = NULL; ptrdiff_t delta; if (jmp_table_size) { const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*))); do { if (*jmp_slot == from_addr) { *jmp_slot = to_addr; ret++; } jmp_slot++; } while (--jmp_table_size); } end = (uint8_t*)code; p = end + size; while (p > end) { uint32_t *ins_ptr; uint32_t ins; p -= 4; ins_ptr = (uint32_t*)p; ins = *ins_ptr; if ((ins & 0xfc000000u) == 0x14000000u) { // B (imm26:0..25) delta = (uint32_t*)from_addr - ins_ptr; if (((ins ^ (uint32_t)delta) & 0x01ffffffu) == 0) { delta = (uint32_t*)to_addr - ins_ptr; if (((delta + 0x02000000) >> 26) != 0) { abort(); // brnach target out of range } *ins_ptr = (ins & 0xfc000000u) | ((uint32_t)delta & 0x03ffffffu); ret++; if (!veneer) { veneer = p; } } } else if ((ins & 0xff000000u) == 0x54000000u || (ins & 0x7e000000u) == 0x34000000u) { // B.cond, CBZ, CBNZ (imm19:5..23) delta = (uint32_t*)from_addr - ins_ptr; if (((ins ^ ((uint32_t)delta << 5)) & 0x00ffffe0u) == 0) { delta = (uint32_t*)to_addr - ins_ptr; if (((delta + 0x40000) >> 19) != 0) { if (veneer) { delta = (uint32_t*)veneer - ins_ptr; if (((delta + 0x40000) >> 19) != 0) { abort(); // brnach target out of range } } else { abort(); // brnach target out of range } } *ins_ptr = (ins & 0xff00001fu) | (((uint32_t)delta & 0x7ffffu) << 5); ret++; } } else if ((ins & 0x7e000000u) == 0x36000000u) { // TBZ, TBNZ (imm14:5..18) delta = (uint32_t*)from_addr - ins_ptr; if (((ins ^ ((uint32_t)delta << 5)) & 0x0007ffe0u) == 0) { delta = (uint32_t*)to_addr - ins_ptr; if (((delta + 0x2000) >> 14) != 0) { if (veneer) { delta = (uint32_t*)veneer - ins_ptr; if (((delta + 0x2000) >> 14) != 0) { abort(); // brnach target out of range } } else { abort(); // brnach target out of range } } *ins_ptr = (ins & 0xfff8001fu) | (((uint32_t)delta & 0x3fffu) << 5); ret++; } } } JIT_CACHE_FLUSH(code, (char*)code + size); #ifdef HAVE_VALGRIND VALGRIND_DISCARD_TRANSLATIONS(code, size); #endif return ret; } static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) { return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); } static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) { const void *link_addr; size_t prologue_size; /* Skip prologue. */ // TODO: don't hardcode this ??? if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { #ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE prologue_size = 0; #else // sub sp, sp, #0x20 prologue_size = 4; #endif } else if (GCC_GLOBAL_REGS) { // stp x29, x30, [sp, # -SPAD]! prologue_size = 4; } else { // stp x29, x30, [sp, # -NR_SPAD]! // stack alignment // stp FP, RX, T2 // mov FP, FCARG1x prologue_size = 12; } link_addr = (const void*)((const char*)t->code_start + prologue_size); if (timeout_exit_addr) { /* Check timeout for links to LOOP */ | MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 | cbz TMP1w, &link_addr | b &timeout_exit_addr } else { | b &link_addr } return 1; } static int zend_jit_trace_return(dasm_State **Dst, bool original_handler, const zend_op *opline) { if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD if (!original_handler) { | JMP_IP TMP1 } else { | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] | ldr REG0, [IP, REG0] | br REG0 } } else if (GCC_GLOBAL_REGS) { | ldp x29, x30, [sp], # SPAD // stack alignment if (!original_handler) { | JMP_IP TMP1 } else { | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] | ldr REG0, [IP, REG0] | br REG0 } } else { if (original_handler) { | mov FCARG1x, FP | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] | ldr REG0, [IP, REG0] | blr REG0 } | ldp FP, RX, T2 // retore FP and IP | ldp x29, x30, [sp], # NR_SPAD // stack alignment if (!original_handler || !opline || (opline->opcode != ZEND_RETURN && opline->opcode != ZEND_RETURN_BY_REF && opline->opcode != ZEND_GENERATOR_RETURN && opline->opcode != ZEND_GENERATOR_CREATE && opline->opcode != ZEND_YIELD && opline->opcode != ZEND_YIELD_FROM)) { | mov RETVALx, #2 // ZEND_VM_LEAVE } | ret } return 1; } static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE var_addr, type, &exit_addr, ZREG_TMP1 return 1; } static int zend_jit_scalar_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, var+offsetof(zval, u1.v.type), TMP1 | cmp TMP1w, #IS_STRING | bhs &exit_addr return 1; } static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); if (!exit_addr) { return 0; } | GET_ZVAL_LVAL ZREG_FCARG1, var_addr, TMP1 if (op_info & MAY_BE_ARRAY_PACKED) { | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w | beq &exit_addr } else { | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w | bne &exit_addr } return 1; } static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) { zend_jit_op_array_trace_extension *jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); size_t offset = jit_extension->offset; const void *handler = (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } if (!GCC_GLOBAL_REGS) { | mov FCARG1x, FP } | EXT_CALL handler, REG0 if (may_throw && opline->opcode != ZEND_RETURN && opline->opcode != ZEND_RETURN_BY_REF) { | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 | cbnz REG0, ->exception_handler } while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { trace++; } if (!GCC_GLOBAL_REGS && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { if (opline->opcode == ZEND_RETURN || opline->opcode == ZEND_RETURN_BY_REF || opline->opcode == ZEND_DO_UCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME || opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_GENERATOR_CREATE) { | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 } } if (zend_jit_trace_may_exit(op_array, opline)) { if (opline->opcode == ZEND_RETURN || opline->opcode == ZEND_RETURN_BY_REF || opline->opcode == ZEND_GENERATOR_CREATE) { if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { #if 0 /* this check should be handled by the following OPLINE guard or jmp [IP] */ | LOAD_ADDR TMP1, zend_jit_halt_op | cmp IP, TMP1 | beq ->trace_halt #endif } else if (GCC_GLOBAL_REGS) { | cbz IP, ->trace_halt } else { | tst RETVALw, RETVALw | blt ->trace_halt } } else if (opline->opcode == ZEND_EXIT || opline->opcode == ZEND_GENERATOR_RETURN || opline->opcode == ZEND_YIELD || opline->opcode == ZEND_YIELD_FROM) { | b ->trace_halt } if (trace->op != ZEND_JIT_TRACE_END || (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { const zend_op *next_opline = trace->opline; const zend_op *exit_opline = NULL; uint32_t exit_point; const void *exit_addr; uint32_t old_info = 0; uint32_t old_res_info = 0; zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; if (zend_is_smart_branch(opline)) { bool exit_if_true = 0; exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); } else { switch (opline->opcode) { case ZEND_JMPZ: case ZEND_JMPNZ: case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: case ZEND_JMP_SET: case ZEND_COALESCE: case ZEND_JMP_NULL: case ZEND_FE_RESET_R: case ZEND_FE_RESET_RW: exit_opline = (trace->opline == opline + 1) ? OP_JMP_ADDR(opline, opline->op2) : opline + 1; break; case ZEND_JMPZNZ: exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ? ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : OP_JMP_ADDR(opline, opline->op2); break; case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: exit_opline = (trace->opline == opline + 1) ? ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : opline + 1; break; } } switch (opline->opcode) { case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: if (opline->op2_type != IS_UNUSED) { old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); } break; } if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); } exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); } switch (opline->opcode) { case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: if (opline->op2_type != IS_UNUSED) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); } break; } if (!exit_addr) { return 0; } | CMP_IP next_opline, TMP1, TMP2 | bne &exit_addr } } zend_jit_set_last_valid_opline(trace->opline); return 1; } static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) { const void *handler; if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { handler = zend_get_opcode_handler_func(opline); } else { handler = opline->handler; } if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } if (!GCC_GLOBAL_REGS) { | mov FCARG1x, FP } | EXT_CALL handler, REG0 if (may_throw) { zend_jit_check_exception(Dst); } /* Skip the following OP_DATA */ switch (opline->opcode) { case ZEND_ASSIGN_DIM: case ZEND_ASSIGN_OBJ: case ZEND_ASSIGN_STATIC_PROP: case ZEND_ASSIGN_DIM_OP: case ZEND_ASSIGN_OBJ_OP: case ZEND_ASSIGN_STATIC_PROP_OP: case ZEND_ASSIGN_STATIC_PROP_REF: case ZEND_ASSIGN_OBJ_REF: zend_jit_set_last_valid_opline(opline + 2); break; default: zend_jit_set_last_valid_opline(opline + 1); break; } return 1; } static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) { if (!zend_jit_set_valid_ip(Dst, opline)) { return 0; } if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { if (opline->opcode == ZEND_DO_UCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME || opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_RETURN) { /* Use inlined HYBRID VM handler */ const void *handler = opline->handler; | ADD_HYBRID_SPAD | EXT_JMP handler, REG0 } else { const void *handler = zend_get_opcode_handler_func(opline); | EXT_CALL handler, REG0 | ADD_HYBRID_SPAD | JMP_IP TMP1 } } else { const void *handler = opline->handler; if (GCC_GLOBAL_REGS) { | ldp x29, x30, [sp], # SPAD // stack alignment } else { | mov FCARG1x, FP | ldp FP, RX, T2 // retore FP and IP | ldp x29, x30, [sp], # NR_SPAD // stack alignment } | EXT_JMP handler, REG0 } zend_jit_reset_last_valid_opline(); return 1; } static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) { uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | CMP_IP opline, TMP1, TMP2 | bne &exit_addr zend_jit_set_last_valid_opline(opline); return 1; } static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) { | b =>target_label return 1; } static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) { | CMP_IP next_opline, TMP1, TMP2 | bne =>target_label zend_jit_set_last_valid_opline(next_opline); return 1; } #ifdef CONTEXT_THREADED_JIT static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) { | NIY // TODO return 1; } #endif static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) { #ifdef CONTEXT_THREADED_JIT return zend_jit_context_threaded_call(Dst, opline, next_block); #else return zend_jit_tail_handler(Dst, opline); #endif } static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type) { ZEND_ASSERT(Z_MODE(src) == IS_REG); ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); if ((info & MAY_BE_ANY) == MAY_BE_LONG) { | SET_ZVAL_LVAL_FROM_REG dst, Rx(Z_REG(src)), TMP1 if (set_type && (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame) || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) { | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 } } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | SET_ZVAL_DVAL dst, Z_REG(src), ZREG_TMP1 if (set_type && (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame) || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) { | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 } } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) { ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); ZEND_ASSERT(Z_MODE(dst) == IS_REG); if ((info & MAY_BE_ANY) == MAY_BE_LONG) { | GET_ZVAL_LVAL Z_REG(dst), src, TMP1 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | GET_ZVAL_DVAL Z_REG(dst), src, ZREG_TMP1 } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type) { zend_jit_addr src = ZEND_ADDR_REG(reg); zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); return zend_jit_spill_store(Dst, src, dst, info, set_type); } static int zend_jit_store_var_type(dasm_State **Dst, int var, uint32_t type) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); | SET_ZVAL_TYPE_INFO dst, type, TMP1w, TMP2 return 1; } static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) { if (Z_MODE(src) == IS_REG && Z_STORE(src)) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); return zend_jit_spill_store(Dst, src, dst, info, 1); } return 1; } static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) { if (Z_MODE(src) == IS_REG && Z_STORE(src)) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); bool set_type = 1; if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { set_type = 0; } } return zend_jit_spill_store(Dst, src, dst, info, set_type); } return 1; } static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) { zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); zend_jit_addr dst = ZEND_ADDR_REG(reg); return zend_jit_load_reg(Dst, src, dst, info); } static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, zend_uchar op_type, zend_jit_addr addr, znode_op op) { if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); | SET_ZVAL_TYPE_INFO dst, IS_UNDEF, TMP1w, TMP2 } return 1; } static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) { if (!zend_jit_same_addr(src, dst)) { if (Z_MODE(src) == IS_REG) { if (Z_MODE(dst) == IS_REG) { if ((info & MAY_BE_ANY) == MAY_BE_LONG) { | mov Rx(Z_REG(dst)), Rx(Z_REG(src)) } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | fmov Rd(Z_REG(dst)-ZREG_V0), Rd(Z_REG(src)-ZREG_V0) } else { ZEND_UNREACHABLE(); } if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); if (!zend_jit_spill_store(Dst, dst, var_addr, info, JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || JIT_G(current_frame) == NULL || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) )) { return 0; } } } else if (Z_MODE(dst) == IS_MEM_ZVAL) { if (!Z_LOAD(src) && !Z_STORE(src)) { if (!zend_jit_spill_store(Dst, src, dst, info, JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || JIT_G(current_frame) == NULL || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) )) { return 0; } } } else { ZEND_UNREACHABLE(); } } else if (Z_MODE(src) == IS_MEM_ZVAL) { if (Z_MODE(dst) == IS_REG) { if (!zend_jit_load_reg(Dst, src, dst, info)) { return 0; } } else { ZEND_UNREACHABLE(); } } else { ZEND_UNREACHABLE(); } } else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) { dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); if (!zend_jit_spill_store(Dst, src, dst, info, JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || JIT_G(current_frame) == NULL || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) )) { return 0; } } return 1; } static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { if (!zend_jit_save_call_chain(Dst, -1)) { return 0; } } ZEND_ASSERT(opline); if ((opline-1)->opcode != ZEND_FETCH_CONSTANT && (opline-1)->opcode != ZEND_FETCH_LIST_R && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR)) && !(flags & ZEND_JIT_EXIT_FREE_OP1)) { val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var); | IF_NOT_ZVAL_REFCOUNTED val_addr, >2, ZREG_TMP1, ZREG_TMP2 | GET_ZVAL_PTR TMP1, val_addr, TMP2 | GC_ADDREF TMP1, TMP2w |2: } | LOAD_IP_ADDR (opline - 1) | b ->trace_escape |1: return 1; } static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); if (reg == ZREG_LONG_MIN_MINUS_1) { uint64_t val = 0xc3e0000000000000; | SET_ZVAL_LVAL dst, val, TMP1, TMP2 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 } else if (reg == ZREG_LONG_MIN) { uint64_t val = 0x8000000000000000; | SET_ZVAL_LVAL dst, val, TMP1, TMP2 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 } else if (reg == ZREG_LONG_MAX) { uint64_t val = 0x7fffffffffffffff; | SET_ZVAL_LVAL dst, val, TMP1, TMP2 | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 } else if (reg == ZREG_LONG_MAX_PLUS_1) { uint64_t val = 0x43e0000000000000; | SET_ZVAL_LVAL dst, val, TMP1, TMP2 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 } else if (reg == ZREG_NULL) { | SET_ZVAL_TYPE_INFO dst, IS_NULL, TMP1w, TMP2 } else if (reg == ZREG_ZVAL_TRY_ADDREF) { | IF_NOT_ZVAL_REFCOUNTED dst, >1, ZREG_TMP1, ZREG_TMP2 | GET_ZVAL_PTR TMP1, dst, TMP2 | GC_ADDREF TMP1, TMP2w |1: } else if (reg == ZREG_ZVAL_COPY_GPR0) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | TRY_ADDREF -1, REG1w, REG2, TMP1w } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_free_trampoline(dasm_State **Dst) { | // if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] | TST_32_WITH_CONST TMP1w, ZEND_ACC_CALL_VIA_TRAMPOLINE, TMP2w | beq >1 | mov FCARG1x, REG0 | EXT_CALL zend_jit_free_trampoline_helper, REG0 |1: return 1; } static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 } if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { return 0; } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { | LONG_ADD_SUB_WITH_IMM adds, op1_def_addr, Z_L(1), TMP1, TMP2 } else { | LONG_ADD_SUB_WITH_IMM subs, op1_def_addr, Z_L(1), TMP1, TMP2 } if (may_overflow && (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { int32_t exit_point; const void *exit_addr; zend_jit_trace_stack *stack; uint32_t old_op1_info, old_res_info = 0; stack = JIT_G(current_frame)->stack; old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); } else { SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); } if (opline->result_type != IS_UNUSED) { old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); if (opline->opcode == ZEND_PRE_INC) { SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); } else if (opline->opcode == ZEND_PRE_DEC) { SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); } else if (opline->opcode == ZEND_POST_INC) { SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); } else if (opline->opcode == ZEND_POST_DEC) { SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); } } exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | bvs &exit_addr if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); if (opline->result_type != IS_UNUSED) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); } } else if (may_overflow) { | bvs >1 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } |.cold_code |1: if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { uint64_t val = 0x43e0000000000000; if (Z_MODE(op1_def_addr) == IS_REG) { | LOAD_64BIT_VAL TMP1, val | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 } else { | SET_ZVAL_LVAL op1_def_addr, val, TMP2, TMP1 } } else { uint64_t val = 0xc3e0000000000000; if (Z_MODE(op1_def_addr) == IS_REG) { | LOAD_64BIT_VAL TMP1, val | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 } else { | SET_ZVAL_LVAL op1_def_addr, val, TMP2, TMP1 } } if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE, TMP1w, TMP2 } if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } | b >3 |.code } else { if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { |.cold_code |2: if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | SET_EX_OPLINE opline, REG0 if (op1_info & MAY_BE_UNDEF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2, ZREG_TMP1 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | LOAD_32BIT_VAL FCARG1w, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 op1_info |= MAY_BE_NULL; } |2: | LOAD_ZVAL_ADDR FCARG1x, op1_addr | // ZVAL_DEREF(var_ptr); if (op1_info & MAY_BE_REF) { | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >2, TMP1w | GET_Z_PTR FCARG1x, FCARG1x | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] | cbz TMP1, >1 if (RETURN_VALUE_USED(opline)) { | LOAD_ZVAL_ADDR FCARG2x, res_addr } else { | mov FCARG2x, xzr } if (opline->opcode == ZEND_PRE_INC) { | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 } else if (opline->opcode == ZEND_PRE_DEC) { | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 } else if (opline->opcode == ZEND_POST_INC) { | EXT_CALL zend_jit_post_inc_typed_ref, REG0 } else if (opline->opcode == ZEND_POST_DEC) { | EXT_CALL zend_jit_post_dec_typed_ref, REG0 } else { ZEND_UNREACHABLE(); } zend_jit_check_exception(Dst); | b >3 |1: | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) |2: } if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { | LOAD_ZVAL_ADDR FCARG2x, res_addr | EXT_CALL zend_jit_pre_inc, REG0 } else { | EXT_CALL increment_function, REG0 } } else { if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { | LOAD_ZVAL_ADDR FCARG2x, res_addr | EXT_CALL zend_jit_pre_dec, REG0 } else { | EXT_CALL decrement_function, REG0 } } if (may_throw) { zend_jit_check_exception(Dst); } } else { zend_reg tmp_reg; if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } if (Z_MODE(op1_def_addr) == IS_REG) { tmp_reg = Z_REG(op1_def_addr); } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { tmp_reg = Z_REG(op1_addr); } else { tmp_reg = ZREG_FPR0; } | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { uint64_t val = 0x3ff0000000000000; // 1.0 | LOAD_64BIT_VAL TMP1, val | fmov FPTMP, TMP1 | fadd Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP } else { uint64_t val = 0x3ff0000000000000; // 1.0 | LOAD_64BIT_VAL TMP1, val | fmov FPTMP, TMP1 | fsub Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP } | SET_ZVAL_DVAL op1_def_addr, tmp_reg, ZREG_TMP1 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | TRY_ADDREF op1_def_info, REG0w, REG1, TMP1w } } | b >3 |.code } |3: if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { return 0; } if (opline->result_type != IS_UNUSED) { if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { return 0; } } return 1; } static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) { if ((opline+1)->opcode == ZEND_OP_DATA && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { return 1; } return ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); } static int zend_jit_math_long_long(dasm_State **Dst, const zend_op *opline, zend_uchar opcode, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint32_t res_info, uint32_t res_use_info, int may_overflow) { bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); zend_reg result_reg; zend_reg tmp_reg = ZREG_REG0; bool use_ovf_flag = 1; if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { if (may_overflow && (res_info & MAY_BE_GUARD) && JIT_G(current_frame) && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { result_reg = ZREG_REG0; } else { result_reg = Z_REG(res_addr); } } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { result_reg = Z_REG(op1_addr); } else if (Z_REG(res_addr) != ZREG_REG0) { result_reg = ZREG_REG0; } else { /* ASSIGN_DIM_OP */ result_reg = ZREG_FCARG1; tmp_reg = ZREG_FCARG1; } if (opcode == ZEND_MUL && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 2) { if (Z_MODE(op1_addr) == IS_REG) { | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) } else { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) } } else if (opcode == ZEND_MUL && Z_MODE(op2_addr) == IS_CONST_ZVAL && !may_overflow && zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) | lsl Rx(result_reg), Rx(result_reg), TMP1 } else if (opcode == ZEND_MUL && Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 2) { if (Z_MODE(op2_addr) == IS_REG) { | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) } else { | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) } } else if (opcode == ZEND_MUL && Z_MODE(op1_addr) == IS_CONST_ZVAL && !may_overflow && zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) | lsl Rx(result_reg), Rx(result_reg), TMP1 } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | asr Rx(result_reg), Rx(result_reg), #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) #if 0 /* x86 specific optimizations through LEA instraction are not supported on ARM */ } else if (opcode == ZEND_ADD && !may_overflow && Z_MODE(op1_addr) == IS_REG && Z_MODE(op2_addr) == IS_CONST_ZVAL) { | NIY // TODO: test } else if (opcode == ZEND_ADD && !may_overflow && Z_MODE(op2_addr) == IS_REG && Z_MODE(op1_addr) == IS_CONST_ZVAL) { | NIY // TODO: test } else if (opcode == ZEND_SUB && !may_overflow && Z_MODE(op1_addr) == IS_REG && Z_MODE(op2_addr) == IS_CONST_ZVAL) { | NIY // TODO: test #endif } else if (opcode == ZEND_MUL) { | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 | mul Rx(result_reg), TMP1, TMP2 if(may_overflow) { /* Use 'smulh' to get the upper 64 bits fo the 128-bit result. * For signed multiplication, the top 65 bits of the result will contain * either all zeros or all ones if no overflow occurred. * Flag: bne -> overflow. beq -> no overflow. */ use_ovf_flag = 0; | smulh TMP1, TMP1, TMP2 | cmp TMP1, Rx(result_reg), asr #63 } } else { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { /* +/- 0 */ may_overflow = 0; } else if (same_ops && opcode != ZEND_DIV) { | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) } else { | LONG_MATH opcode, result_reg, op2_addr, TMP1 } } if (may_overflow) { if (res_info & MAY_BE_GUARD) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { if (use_ovf_flag) { | bvs &exit_addr } else { | bne &exit_addr } if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { | mov Rx(Z_REG(res_addr)), Rx(result_reg) } } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { if (use_ovf_flag) { | bvc &exit_addr } else { | beq &exit_addr } } else { ZEND_UNREACHABLE(); } } else { if (res_info & MAY_BE_LONG) { if (use_ovf_flag) { | bvs >1 } else { | bne >1 } } else { if (use_ovf_flag) { | bvc >1 } else { | beq >1 } } } } if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 } } } if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { zend_reg tmp_reg1 = ZREG_FPR0; zend_reg tmp_reg2 = ZREG_FPR1; if (res_info & MAY_BE_LONG) { |.cold_code |1: } do { if ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)) { if (opcode == ZEND_ADD) { uint64_t val = 0x43e0000000000000; if (Z_MODE(res_addr) == IS_REG) { | LOAD_64BIT_VAL TMP1, val | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 } else { | SET_ZVAL_LVAL res_addr, val, TMP2, TMP1 } break; } else if (opcode == ZEND_SUB) { uint64_t val = 0xc3e0000000000000; if (Z_MODE(res_addr) == IS_REG) { | LOAD_64BIT_VAL TMP1, val | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 } else { | SET_ZVAL_LVAL res_addr, val, TMP2, TMP1 } break; } } | DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg, ZREG_TMP1 | DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg, ZREG_TMP1 | DOUBLE_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 | SET_ZVAL_DVAL res_addr, tmp_reg1, ZREG_TMP1 } while (0); if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 } if (res_info & MAY_BE_LONG) { | b >2 |.code } |2: } return 1; } static int zend_jit_math_long_double(dasm_State **Dst, zend_uchar opcode, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint32_t res_use_info) { zend_reg result_reg = (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_FPR0; zend_reg op2_reg; | DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, ZREG_TMP1, ZREG_TMP2 if (Z_MODE(op2_addr) == IS_REG) { op2_reg = Z_REG(op2_addr); } else { op2_reg = ZREG_FPTMP; | GET_ZVAL_DVAL op2_reg, op2_addr, ZREG_TMP1 } | DOUBLE_MATH_REG opcode, result_reg, result_reg, op2_reg | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 } } return 1; } static int zend_jit_math_double_long(dasm_State **Dst, zend_uchar opcode, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint32_t res_use_info) { zend_reg result_reg, op1_reg, op2_reg; if (zend_is_commutative(opcode) && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { if (Z_MODE(res_addr) == IS_REG) { result_reg = Z_REG(res_addr); } else { result_reg = ZREG_FPR0; } | DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 if (Z_MODE(op1_addr) == IS_REG) { op1_reg = Z_REG(op1_addr); } else { op1_reg = ZREG_FPTMP; | GET_ZVAL_DVAL op1_reg, op1_addr, ZREG_TMP1 } | DOUBLE_MATH_REG opcode, result_reg, result_reg, op1_reg } else { if (Z_MODE(res_addr) == IS_REG) { result_reg = Z_REG(res_addr); } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { result_reg = Z_REG(op1_addr); } else { result_reg = ZREG_FPR0; } if (Z_MODE(op1_addr) == IS_REG) { op1_reg = Z_REG(op1_addr); } else { | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 op1_reg = result_reg; } if ((opcode == ZEND_ADD || opcode == ZEND_SUB) && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { /* +/- 0 */ } else { op2_reg = ZREG_FPTMP; | DOUBLE_GET_ZVAL_LVAL op2_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg } } | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 } } } return 1; } static int zend_jit_math_double_double(dasm_State **Dst, zend_uchar opcode, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint32_t res_use_info) { bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); zend_reg result_reg, op1_reg, op2_reg; zend_jit_addr val_addr; if (Z_MODE(res_addr) == IS_REG) { result_reg = Z_REG(res_addr); } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { result_reg = Z_REG(op1_addr); } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { result_reg = Z_REG(op2_addr); } else { result_reg = ZREG_FPR0; } if (Z_MODE(op1_addr) == IS_REG) { op1_reg = Z_REG(op1_addr); val_addr = op2_addr; } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { op1_reg = Z_REG(op2_addr); val_addr = op1_addr; } else { | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 op1_reg = result_reg; val_addr = op2_addr; } if ((opcode == ZEND_MUL) && Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { | DOUBLE_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg } else { if (same_ops) { op2_reg = op1_reg; } else if (Z_MODE(val_addr) == IS_REG) { op2_reg = Z_REG(val_addr); } else { op2_reg = ZREG_FPTMP; | GET_ZVAL_DVAL op2_reg, val_addr, ZREG_TMP1 } | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg } | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 } } } return 1; } static int zend_jit_math_helper(dasm_State **Dst, const zend_op *opline, zend_uchar opcode, zend_uchar op1_type, znode_op op1, zend_jit_addr op1_addr, uint32_t op1_info, zend_uchar op2_type, znode_op op2, zend_jit_addr op2_addr, uint32_t op2_info, uint32_t res_var, zend_jit_addr res_addr, uint32_t res_info, uint32_t res_use_info, int may_overflow, int may_throw) /* Labels: 1,2,3,4,5,6 */ { bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { if (op2_info & MAY_BE_DOUBLE) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1, ZREG_TMP1 |.cold_code |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 } if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } | b >5 |.code } else { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 } } if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { |.cold_code |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops) { | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6, ZREG_TMP1 } } if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } | b >5 } if (!same_ops) { |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 } if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } | b >5 } |.code } } else if ((op1_info & MAY_BE_DOUBLE) && !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (res_info & MAY_BE_DOUBLE)) { if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op2_info & MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 } } if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } } if (!same_ops && (op2_info & MAY_BE_LONG)) { if (op2_info & MAY_BE_DOUBLE) { |.cold_code } |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 } if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } if (op2_info & MAY_BE_DOUBLE) { | b >5 |.code } } } else if ((op2_info & MAY_BE_DOUBLE) && !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (res_info & MAY_BE_DOUBLE)) { if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op1_info & MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 } } if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } } if (!same_ops && (op1_info & MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { |.cold_code } |1: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 } if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { | b >5 |.code } } } |5: if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { |.cold_code } |6: if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); | LOAD_ZVAL_ADDR FCARG1x, real_addr } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, res_addr } if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { return 0; } op1_addr = real_addr; } | LOAD_ZVAL_ADDR FCARG2x, op1_addr } else { if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { return 0; } op1_addr = real_addr; } | LOAD_ZVAL_ADDR FCARG2x, op1_addr if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); | LOAD_ZVAL_ADDR FCARG1x, real_addr } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, res_addr } } if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { return 0; } op2_addr = real_addr; } | LOAD_ZVAL_ADDR CARG3, op2_addr | SET_EX_OPLINE opline, REG0 if (opcode == ZEND_ADD) { | EXT_CALL add_function, REG0 } else if (opcode == ZEND_SUB) { | EXT_CALL sub_function, REG0 } else if (opcode == ZEND_MUL) { | EXT_CALL mul_function, REG0 } else if (opcode == ZEND_DIV) { | EXT_CALL div_function, REG0 } else { ZEND_UNREACHABLE(); } | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 if (may_throw) { if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 | cbnz TMP2, ->exception_handler_free_op2 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { zend_jit_check_exception_undef_result(Dst, opline); } else { zend_jit_check_exception(Dst); } } if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { return 0; } } if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { | b <5 |.code } } return 1; } static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) { ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { return 0; } if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { return 0; } return 1; } static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr) { if (Z_MODE(op2_addr) != IS_MEM_ZVAL || Z_REG(op2_addr) != ZREG_FCARG1) { | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 } else if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 } else { | GET_ZVAL_LVAL ZREG_REG0, op2_addr, TMP1 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 | mov FCARG2x, REG0 } | EXT_CALL zend_jit_add_arrays_helper, REG0 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX, TMP1w, TMP2 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 return 1; } static int zend_jit_long_math_helper(dasm_State **Dst, const zend_op *opline, zend_uchar opcode, zend_uchar op1_type, znode_op op1, zend_jit_addr op1_addr, uint32_t op1_info, zend_ssa_range *op1_range, zend_uchar op2_type, znode_op op2, zend_jit_addr op2_addr, uint32_t op2_info, zend_ssa_range *op2_range, uint32_t res_var, zend_jit_addr res_addr, uint32_t res_info, uint32_t res_use_info, int may_throw) /* Labels: 6 */ { bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); zend_reg result_reg; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 } if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 } if (Z_MODE(res_addr) == IS_REG) { if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) && opline->op2_type != IS_CONST) { result_reg = ZREG_REG0; } else { result_reg = Z_REG(res_addr); } } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { result_reg = Z_REG(op1_addr); } else if (Z_REG(res_addr) != ZREG_REG0) { result_reg = ZREG_REG0; } else { /* ASSIGN_DIM_OP */ result_reg = ZREG_FCARG1; } if (opcode == ZEND_SL) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { if (EXPECTED(op2_lval > 0)) { | mov Rx(result_reg), xzr } else { zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); | SET_EX_OPLINE opline, REG0 | b ->negative_shift } } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { | add Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) } else { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | lsl Rx(result_reg), Rx(result_reg), #op2_lval } } else { zend_reg op2_reg; if (Z_MODE(op2_addr) == IS_REG) { op2_reg = Z_REG(op2_addr); } else { op2_reg = ZREG_TMP2; | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 } if (!op2_range || op2_range->min < 0 || op2_range->max >= SIZEOF_ZEND_LONG * 8) { | cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8) | bhs >1 |.cold_code |1: | mov Rx(result_reg), xzr | cmp Rx(op2_reg), xzr | bgt >1 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); | SET_EX_OPLINE opline, REG0 | b ->negative_shift |.code } | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | lsl Rx(result_reg), Rx(result_reg), Rx(op2_reg) |1: } } else if (opcode == ZEND_SR) { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { if (EXPECTED(op2_lval > 0)) { | asr Rx(result_reg), Rx(result_reg), #((SIZEOF_ZEND_LONG * 8) - 1) } else { zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); | SET_EX_OPLINE opline, REG0 | b ->negative_shift } } else { | asr Rx(result_reg), Rx(result_reg), #op2_lval } } else { zend_reg op2_reg; if (Z_MODE(op2_addr) == IS_REG) { op2_reg = Z_REG(op2_addr); } else { op2_reg = ZREG_TMP2; | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 } if (!op2_range || op2_range->min < 0 || op2_range->max >= SIZEOF_ZEND_LONG * 8) { | cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8) | bhs >1 |.cold_code |1: | cmp Rx(op2_reg), xzr | mov Rx(op2_reg), #((SIZEOF_ZEND_LONG * 8) - 1) | bgt >1 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); | SET_EX_OPLINE opline, REG0 | b ->negative_shift |.code } |1: | asr Rx(result_reg), Rx(result_reg), Rx(op2_reg) } } else if (opcode == ZEND_MOD) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); if (op2_lval == 0) { zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); | SET_EX_OPLINE opline, REG0 | b ->mod_by_zero } else if (op2_lval == -1) { | mov Rx(result_reg), xzr } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) { zval tmp; zend_jit_addr tmp_addr; /* Optimisation for mod of power of 2 */ ZVAL_LONG(&tmp, op2_lval - 1); tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp); | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, TMP1 } else { | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 | sdiv Rx(result_reg), TMP1, TMP2 | msub Rx(result_reg), Rx(result_reg), TMP2, TMP1 } } else { zend_reg op2_reg; if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP2, Rx(Z_REG(op2_addr)), Z_OFFSET(op2_addr), TMP2 op2_reg = ZREG_TMP2; } else { ZEND_ASSERT(Z_MODE(op2_addr) == IS_REG); op2_reg = Z_REG(op2_addr); } if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { | cbz Rx(op2_reg), >1 |.cold_code |1: zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); | SET_EX_OPLINE opline, REG0 | b ->mod_by_zero |.code } /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { | cmn Rx(op2_reg), #1 | beq >1 |.cold_code |1: | SET_ZVAL_LVAL_FROM_REG res_addr, xzr, TMP1 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 } } } | b >5 |.code } | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 | sdiv Rx(result_reg), TMP1, Rx(op2_reg) | msub Rx(result_reg), Rx(result_reg), Rx(op2_reg), TMP1 } } else if (same_ops) { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) } else { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | LONG_MATH opcode, result_reg, op2_addr, TMP1 } if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 } if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 } } } if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { |.cold_code } |6: if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); | LOAD_ZVAL_ADDR FCARG1x, real_addr } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, res_addr } if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { return 0; } op1_addr = real_addr; } | LOAD_ZVAL_ADDR FCARG2x, op1_addr } else { if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { return 0; } op1_addr = real_addr; } | LOAD_ZVAL_ADDR FCARG2x, op1_addr if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); | LOAD_ZVAL_ADDR FCARG1x, real_addr } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, res_addr } } if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { return 0; } op2_addr = real_addr; } | LOAD_ZVAL_ADDR CARG3, op2_addr | SET_EX_OPLINE opline, REG0 if (opcode == ZEND_BW_OR) { | EXT_CALL bitwise_or_function, REG0 } else if (opcode == ZEND_BW_AND) { | EXT_CALL bitwise_and_function, REG0 } else if (opcode == ZEND_BW_XOR) { | EXT_CALL bitwise_xor_function, REG0 } else if (opcode == ZEND_SL) { | EXT_CALL shift_left_function, REG0 } else if (opcode == ZEND_SR) { | EXT_CALL shift_right_function, REG0 } else if (opcode == ZEND_MOD) { | EXT_CALL mod_function, REG0 } else { ZEND_UNREACHABLE(); } if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) { /* compound assignment may decrement "op2" refcount */ op2_info |= MAY_BE_RC1; } | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 if (may_throw) { if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { | MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1 | cbnz TMP2, ->exception_handler_free_op2 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { zend_jit_check_exception_undef_result(Dst, opline); } else { zend_jit_check_exception(Dst); } } if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { return 0; } } if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { | b >5 |.code } } |5: return 1; } static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) { ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, opline->result.var, res_addr, res_info, res_use_info, may_throw)) { return 0; } if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { return 0; } return 1; } static int zend_jit_concat_helper(dasm_State **Dst, const zend_op *opline, zend_uchar op1_type, znode_op op1, zend_jit_addr op1_addr, uint32_t op1_info, zend_uchar op2_type, znode_op op2, zend_jit_addr op2_addr, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) { if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 } if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6, ZREG_TMP1 } if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, res_addr } | LOAD_ZVAL_ADDR FCARG2x, op2_addr | EXT_CALL zend_jit_fast_assign_concat_helper, REG0 /* concatination with itself may reduce refcount */ op2_info |= MAY_BE_RC1; } else { if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, res_addr } | LOAD_ZVAL_ADDR FCARG2x, op1_addr | LOAD_ZVAL_ADDR CARG3, op2_addr if (op1_type == IS_CV || op1_type == IS_CONST) { | EXT_CALL zend_jit_fast_concat_helper, REG0 } else { | EXT_CALL zend_jit_fast_concat_tmp_helper, REG0 } } /* concatination with empty string may increase refcount */ op2_info |= MAY_BE_RCN; | FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 |5: } if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { |.cold_code |6: } if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, res_addr } | LOAD_ZVAL_ADDR FCARG2x, op1_addr } else { | LOAD_ZVAL_ADDR FCARG2x, op1_addr if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, res_addr } } | LOAD_ZVAL_ADDR CARG3, op2_addr | SET_EX_OPLINE opline, REG0 | EXT_CALL concat_function, REG0 /* concatination with empty string may increase refcount */ op1_info |= MAY_BE_RCN; op2_info |= MAY_BE_RCN; | FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 | FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 if (may_throw) { if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { zend_jit_check_exception_undef_result(Dst, opline); } else { zend_jit_check_exception(Dst); } } if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { | b <5 |.code } } return 1; } static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) { zend_jit_addr op1_addr, op2_addr; ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); op1_addr = OP1_ADDR(); op2_addr = OP2_ADDR(); return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); } static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) /* Labels: 1,2,3,4,5 */ { zend_jit_addr op2_addr = OP2_ADDR(); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R && !exit_addr) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } if (op2_info & MAY_BE_LONG) { bool op2_loaded = 0; bool packed_loaded = 0; bool bad_packed_key = 0; if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 } if (op1_info & MAY_BE_PACKED_GUARD) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } if (op1_info & MAY_BE_ARRAY_PACKED) { | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w | beq &exit_addr } else { | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w | bne &exit_addr } } if (type == BP_VAR_W) { | // hval = Z_LVAL_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 op2_loaded = 1; } if (op1_info & MAY_BE_ARRAY_PACKED) { zend_long val = -1; if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { val = Z_LVAL_P(Z_ZV(op2_addr)); if (val >= 0 && val < HT_MAX_SIZE) { packed_loaded = 1; } else { bad_packed_key = 1; } } else { if (!op2_loaded) { | // hval = Z_LVAL_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 op2_loaded = 1; } packed_loaded = 1; } if (dim_type == IS_UNDEF && type == BP_VAR_W) { /* don't generate "fast" code for packed array */ packed_loaded = 0; } if (packed_loaded) { | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w | beq >4 // HASH_FIND } | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) | ldr REG0w, [FCARG1x, #offsetof(zend_array, nNumUsed)] if (val == 0) { | cmp REG0, xzr } else if (val > 0 && !op2_loaded) { | CMP_64_WITH_CONST REG0, val, TMP1 } else { | cmp REG0, FCARG2x } if (type == BP_JIT_IS) { if (not_found_exit_addr) { | bls ¬_found_exit_addr } else { | bls >9 // NOT_FOUND } } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { | bls &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { | bls ¬_found_exit_addr } else if (type == BP_VAR_RW && not_found_exit_addr) { | bls ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { | bls >7 // NOT_FOUND } else { | bls >2 // NOT_FOUND } | // _ret = &_ht->arData[_h].val; if (val >= 0) { | ldr REG0, [FCARG1x, #offsetof(zend_array, arData)] if (val != 0) { | ADD_SUB_64_WITH_CONST add, REG0, REG0, (val * sizeof(Bucket)), TMP1 } } else { | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] | add REG0, TMP1, FCARG2x, lsl #5 } } } switch (type) { case BP_JIT_IS: if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { if (packed_loaded) { | b >5 } |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 } if (packed_loaded) { | EXT_CALL _zend_hash_index_find, REG0 } else { | EXT_CALL zend_hash_index_find, REG0 } | mov REG0, RETVALx if (not_found_exit_addr) { | cbz REG0, ¬_found_exit_addr } else { | cbz REG0, >9 // NOT_FOUND } if (op2_info & MAY_BE_STRING) { | b >5 } } else if (packed_loaded) { if (op2_info & MAY_BE_STRING) { | b >5 } } else if (not_found_exit_addr) { | b ¬_found_exit_addr } else { | b >9 // NOT_FOUND } break; case BP_VAR_R: case BP_VAR_IS: case BP_VAR_UNSET: if (packed_loaded) { if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { /* perform IS_UNDEF check only after result type guard (during deoptimization) */ if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { | IF_Z_TYPE REG0, IS_UNDEF, &exit_addr, TMP1w } } else if (type == BP_VAR_IS && not_found_exit_addr) { | IF_Z_TYPE REG0, IS_UNDEF, ¬_found_exit_addr, TMP1w } else if (type == BP_VAR_IS && found_exit_addr) { | IF_Z_TYPE REG0, IS_UNDEF, >7, TMP1w // NOT_FOUND } else { | IF_Z_TYPE REG0, IS_UNDEF, >2, TMP1w // NOT_FOUND } } if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_NUMERIC_HASH))) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { | b &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { | b ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { | b >7 // NOT_FOUND } else { | b >2 // NOT_FOUND } } if (!packed_loaded || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 } if (packed_loaded) { | EXT_CALL _zend_hash_index_find, REG0 } else { | EXT_CALL zend_hash_index_find, REG0 } | mov REG0, RETVALx if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { | cbz REG0, &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { | cbz REG0, ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { | cbz REG0, >7 // NOT_FOUND } else { | cbz REG0, >2 // NOT_FOUND } } |.cold_code |2: switch (type) { case BP_VAR_R: if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); | // retval = &EG(uninitialized_zval); | UNDEFINED_OFFSET opline | b >9 } break; case BP_VAR_IS: case BP_VAR_UNSET: if (!not_found_exit_addr && !found_exit_addr) { | // retval = &EG(uninitialized_zval); | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 | b >9 } break; default: ZEND_UNREACHABLE(); } |.code break; case BP_VAR_RW: if (packed_loaded && !not_found_exit_addr) { | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w } if (!packed_loaded || !not_found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { if (packed_loaded && not_found_exit_addr) { |.cold_code } |2: |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 } if (packed_loaded) { | EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, REG0 } else { | EXT_CALL zend_jit_hash_index_lookup_rw, REG0 } | mov REG0, RETVALx if (not_found_exit_addr) { if (packed_loaded) { | cbnz REG0, >8 | b ¬_found_exit_addr |.code } else { | cbz REG0, ¬_found_exit_addr } } else { | cbz REG0, >9 } } break; case BP_VAR_W: if (packed_loaded) { | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w } if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) { |2: |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 } | EXT_CALL zend_hash_index_lookup, REG0 | mov REG0, RETVALx } break; default: ZEND_UNREACHABLE(); } if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { | b >8 } } if (op2_info & MAY_BE_STRING) { |3: if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3, ZREG_TMP1 } | // offset_key = Z_STR_P(dim); | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 | // retval = zend_hash_find(ht, offset_key); switch (type) { case BP_JIT_IS: if (opline->op2_type != IS_CONST) { | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] | cmp TMP1w, #((uint8_t) ('9')) | ble >1 |.cold_code |1: | EXT_CALL zend_jit_symtable_find, REG0 | b >1 |.code | EXT_CALL zend_hash_find, REG0 |1: } else { | EXT_CALL zend_hash_find_known_hash, REG0 } | mov REG0, RETVALx if (not_found_exit_addr) { | cbz REG0, ¬_found_exit_addr } else { | cbz REG0, >9 // NOT_FOUND } break; case BP_VAR_R: case BP_VAR_IS: case BP_VAR_UNSET: if (opline->op2_type != IS_CONST) { | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] | cmp TMP1w, #((uint8_t) ('9')) | ble >1 |.cold_code |1: | EXT_CALL zend_jit_symtable_find, REG0 | b >1 |.code | EXT_CALL zend_hash_find, REG0 |1: } else { | EXT_CALL zend_hash_find_known_hash, REG0 } | mov REG0, RETVALx if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { | cbz REG0, &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { | cbz REG0, ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { | cbz REG0, >7 } else { | cbz REG0, >2 // NOT_FOUND |.cold_code |2: switch (type) { case BP_VAR_R: // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); | UNDEFINED_INDEX opline | b >9 break; case BP_VAR_IS: case BP_VAR_UNSET: | // retval = &EG(uninitialized_zval); | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 | b >9 break; default: ZEND_UNREACHABLE(); } |.code } break; case BP_VAR_RW: if (opline->op2_type != IS_CONST) { | EXT_CALL zend_jit_symtable_lookup_rw, REG0 } else { | EXT_CALL zend_jit_hash_lookup_rw, REG0 } | mov REG0, RETVALx if (not_found_exit_addr) { | cbz REG0, ¬_found_exit_addr } else { | cbz REG0, >9 } break; case BP_VAR_W: if (opline->op2_type != IS_CONST) { | EXT_CALL zend_jit_symtable_lookup_w, REG0 } else { | EXT_CALL zend_hash_lookup, REG0 } | mov REG0, RETVALx break; default: ZEND_UNREACHABLE(); } } if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { |5: if (op1_info & MAY_BE_ARRAY_OF_REF) { | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w } | ldrb TMP1w, [REG0,#offsetof(zval, u1.v.type)] | cmp TMP1w, #IS_NULL if (not_found_exit_addr) { | ble ¬_found_exit_addr } else if (found_exit_addr) { | bgt &found_exit_addr } else { | ble >9 // NOT FOUND } } if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { |.cold_code |3: } if (type != BP_VAR_RW) { | SET_EX_OPLINE opline, REG0 } | LOAD_ZVAL_ADDR FCARG2x, op2_addr switch (type) { case BP_VAR_R: | LOAD_ZVAL_ADDR CARG3, res_addr | EXT_CALL zend_jit_fetch_dim_r_helper, REG0 | mov REG0, RETVALx | b >9 break; case BP_JIT_IS: | EXT_CALL zend_jit_fetch_dim_isset_helper, REG0 | mov REG0, RETVALx if (not_found_exit_addr) { | cbz REG0, ¬_found_exit_addr if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { | b >8 } } else if (found_exit_addr) { | cbnz REG0, &found_exit_addr if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { | b >9 } } else { | cbnz REG0, >8 | b >9 } break; case BP_VAR_IS: case BP_VAR_UNSET: | LOAD_ZVAL_ADDR CARG3, res_addr | EXT_CALL zend_jit_fetch_dim_is_helper, REG0 | mov REG0, RETVALx | b >9 break; case BP_VAR_RW: | EXT_CALL zend_jit_fetch_dim_rw_helper, REG0 | mov REG0, RETVALx | cbnz REG0, >8 | b >9 break; case BP_VAR_W: | EXT_CALL zend_jit_fetch_dim_w_helper, REG0 | mov REG0, RETVALx | cbnz REG0, >8 | b >9 break; default: ZEND_UNREACHABLE(); } if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { |.code } } return 1; } static int zend_jit_simple_assign(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr, uint32_t var_info, uint32_t var_def_info, zend_uchar val_type, zend_jit_addr val_addr, uint32_t val_info, zend_jit_addr res_addr, int in_cold, int save_r1, bool check_exception) /* Labels: 1,2,3 */ { zend_reg tmp_reg; if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_REG0) { tmp_reg = ZREG_REG0; } else { /* ASSIGN_DIM */ tmp_reg = ZREG_FCARG1; } if (Z_MODE(val_addr) == IS_CONST_ZVAL) { zval *zv = Z_ZV(val_addr); if (!res_addr) { | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 } else { | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 } if (Z_REFCOUNTED_P(zv)) { if (!res_addr) { | ADDREF_CONST zv, TMP1, TMP2 } else { | ADDREF_CONST_2 zv, TMP1, TMP2 } } } else { if (val_info & MAY_BE_UNDEF) { if (in_cold) { | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2, ZREG_TMP1 } else { | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 |.cold_code |1: } | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); if (save_r1) { | str FCARG1x, T1 // save } | SET_ZVAL_TYPE_INFO var_addr, IS_NULL, TMP1w, TMP2 if (res_addr) { | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 } if (opline) { | SET_EX_OPLINE opline, Rx(tmp_reg) } ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) | EXT_CALL zend_jit_undefined_op_helper, REG0 if (check_exception) { | cbz RETVALx, ->exception_handler_undef } if (save_r1) { | ldr FCARG1x, T1 // restore } | b >3 if (in_cold) { |2: } else { |.code } } if (val_info & MAY_BE_REF) { if (val_type == IS_CV) { ZEND_ASSERT(Z_REG(var_addr) != ZREG_REG2); if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_REG2 || Z_OFFSET(val_addr) != 0) { | LOAD_ZVAL_ADDR REG2, val_addr } | ZVAL_DEREF REG2, val_info, TMP1w val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); } else { zend_jit_addr ref_addr; if (in_cold) { | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 } else { | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 |.cold_code |1: } if (Z_REG(val_addr) == ZREG_REG2) { | str REG2, T1 // save } | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); | GET_ZVAL_PTR REG2, val_addr, TMP1 | GC_DELREF REG2, TMP1w | // ZVAL_COPY_VALUE(return_value, &ref->val); ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, offsetof(zend_reference, val)); if (!res_addr) { | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } else { | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } | beq >2 // GC_DELREF() reached zero | IF_NOT_REFCOUNTED REG2w, >3, TMP1w if (!res_addr) { | GC_ADDREF Rx(tmp_reg), TMP1w } else { | GC_ADDREF_2 Rx(tmp_reg), TMP1w } | b >3 |2: if (res_addr) { | IF_NOT_REFCOUNTED REG2w, >2, TMP1w | GC_ADDREF Rx(tmp_reg), TMP1w |2: } if (Z_REG(val_addr) == ZREG_REG2) { | ldr REG2, T1 // restore } if (save_r1) { | str FCARG1x, T1 // save } | GET_ZVAL_PTR FCARG1x, val_addr, TMP1 | EFREE_REFERENCE if (save_r1) { | ldr FCARG1x, T1 // restore } | b >3 if (in_cold) { |1: } else { |.code } } } if (!res_addr) { | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } else { | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } if (val_type == IS_CV) { if (!res_addr) { | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w } else { | TRY_ADDREF_2 val_info, REG2w, Rx(tmp_reg), TMP1w } } else { if (res_addr) { | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w } } |3: } return 1; } static int zend_jit_assign_to_typed_ref(dasm_State **Dst, const zend_op *opline, zend_uchar val_type, zend_jit_addr val_addr, zend_jit_addr res_addr, bool check_exception) { | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] | cbnz TMP1, >2 |.cold_code |2: if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { | LOAD_ZVAL_ADDR FCARG2x, val_addr } if (opline) { | SET_EX_OPLINE opline, REG0 } if (val_type == IS_CONST) { | EXT_CALL zend_jit_assign_const_to_typed_ref, REG0 } else if (val_type == IS_TMP_VAR) { | EXT_CALL zend_jit_assign_tmp_to_typed_ref, REG0 } else if (val_type == IS_VAR) { | EXT_CALL zend_jit_assign_var_to_typed_ref, REG0 } else if (val_type == IS_CV) { | EXT_CALL zend_jit_assign_cv_to_typed_ref, REG0 } else { ZEND_UNREACHABLE(); } if (res_addr) { zend_jit_addr ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_X0, 0); // RETVAL | ZVAL_COPY_VALUE res_addr, -1, ret_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | TRY_ADDREF -1, REG1w, REG2, TMP1w } if (check_exception) { | // if (UNEXPECTED(EG(exception) != NULL)) { | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 | cbz REG0, >8 // END OF zend_jit_assign_to_variable() | b ->exception_handler } else { | b >8 } |.code return 1; } static int zend_jit_assign_to_variable_call(dasm_State **Dst, const zend_op *opline, zend_jit_addr __var_use_addr, zend_jit_addr var_addr, uint32_t __var_info, uint32_t __var_def_info, zend_uchar val_type, zend_jit_addr val_addr, uint32_t val_info, zend_jit_addr __res_addr, bool __check_exception) { if (val_info & MAY_BE_UNDEF) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr, ZREG_TMP1 } else { | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 |.cold_code |1: ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP); if (Z_REG(var_addr) != ZREG_FP) { | str Rx(Z_REG(var_addr)), T1 // save } | SET_EX_OPLINE opline, REG0 | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) | EXT_CALL zend_jit_undefined_op_helper, REG0 if (Z_REG(var_addr) != ZREG_FP) { | ldr Rx(Z_REG(var_addr)), T1 // restore } if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, var_addr } | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval | bl ->assign_const | b >9 |.code |1: } } if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, var_addr } if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { | LOAD_ZVAL_ADDR FCARG2x, val_addr } if (opline) { | SET_EX_OPLINE opline, REG0 } if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { | bl ->assign_tmp } else if (val_type == IS_CONST) { | bl ->assign_const } else if (val_type == IS_TMP_VAR) { | bl ->assign_tmp } else if (val_type == IS_VAR) { if (!(val_info & MAY_BE_REF)) { | bl ->assign_tmp } else { | bl ->assign_var } } else if (val_type == IS_CV) { if (!(val_info & MAY_BE_REF)) { | bl ->assign_cv_noref } else { | bl ->assign_cv } if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { |9: } } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_assign_to_variable(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_use_addr, zend_jit_addr var_addr, uint32_t var_info, uint32_t var_def_info, zend_uchar val_type, zend_jit_addr val_addr, uint32_t val_info, zend_jit_addr res_addr, bool check_exception) /* Labels: 1,2,3,4,5,8 */ { int done = 0; zend_reg ref_reg, tmp_reg; if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_REG0) { ref_reg = ZREG_FCARG1; tmp_reg = ZREG_REG0; } else { /* ASSIGN_DIM */ ref_reg = ZREG_REG0; tmp_reg = ZREG_FCARG1; } if (var_info & MAY_BE_REF) { if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { | LOAD_ZVAL_ADDR Rx(ref_reg), var_use_addr var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); } | // if (Z_ISREF_P(variable_ptr)) { | IF_NOT_Z_TYPE Rx(ref_reg), IS_REFERENCE, >3, TMP1w | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { | GET_Z_PTR FCARG1x, Rx(ref_reg) if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) { return 0; } | add Rx(ref_reg), FCARG1x, #offsetof(zend_reference, val) |3: } if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { if (RC_MAY_BE_1(var_info)) { int in_cold = 0; if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | IF_ZVAL_REFCOUNTED var_use_addr, >1, ZREG_TMP1, ZREG_TMP2 |.cold_code |1: in_cold = 1; } if (Z_REG(var_use_addr) == ZREG_FCARG1 || Z_REG(var_use_addr) == ZREG_REG0) { bool keep_gc = 0; | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 #if 0 // TODO: This optiization doesn't work on ARM if (tmp_reg == ZREG_FCARG1) { if (Z_MODE(val_addr) == IS_REG) { keep_gc = 1; } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { keep_gc = 1; } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { zval *zv = Z_ZV(val_addr); if (Z_TYPE_P(zv) == IS_DOUBLE) { if (Z_DVAL_P(zv) == 0) { keep_gc = 1; } } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { keep_gc = 1; } } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { keep_gc = 1; } } } #endif if (!keep_gc) { | str Rx(tmp_reg), T1 // save } if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0, 0)) { return 0; } if (!keep_gc) { | ldr FCARG1x, T1 // restore } } else { | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1, 0)) { return 0; } } | GC_DELREF FCARG1x, TMP1w if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { | bne >4 } else { | bne >8 } | ZVAL_DTOR_FUNC var_info, opline, TMP1 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { if (check_exception && !(val_info & MAY_BE_UNDEF)) { | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 | cbz REG0, >8 | b ->exception_handler } else { | b >8 } } if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { |4: | IF_GC_MAY_NOT_LEAK FCARG1x, >8, TMP1w, TMP2w | EXT_CALL gc_possible_root, REG0 if (in_cold) { | b >8 } } if (check_exception && (val_info & MAY_BE_UNDEF)) { |8: | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 | cbz REG0, >8 | b ->exception_handler } if (in_cold) { |.code } else { done = 1; } } else /* if (RC_MAY_BE_N(var_info)) */ { if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5, ZREG_TMP1, ZREG_TMP2 } if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { if (Z_REG(var_use_addr) != ZREG_FP) { | str Rx(Z_REG(var_use_addr)), T1 // save } | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 | GC_DELREF FCARG1x, TMP1w | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w | EXT_CALL gc_possible_root, TMP1 if (Z_REG(var_use_addr) != ZREG_FP) { | ldr Rx(Z_REG(var_use_addr)), T1 // restore } } else { | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 | GC_DELREF Rx(tmp_reg), TMP1w } |5: } } if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0, check_exception)) { return 0; } |8: return 1; } static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw) { zend_jit_addr op2_addr, op3_addr, res_addr; op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; op3_addr = OP1_DATA_ADDR(); if (opline->result_type == IS_UNUSED) { res_addr = 0; } else { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr, ZREG_TMP1 val_info &= ~MAY_BE_UNDEF; } if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w | GET_Z_PTR FCARG2x, FCARG1x | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] | cmp TMP1w, #IS_ARRAY | bne >2 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) | b >3 |.cold_code |2: | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 | mov FCARG1x, RETVALx | cbnz FCARG1x, >1 | b ->exception_handler_undef |.code |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } |3: | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 | bgt >7 } | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { | str Rx(Z_REG(op1_addr)), T1 // save } | EXT_CALL _zend_new_array_0, REG0 | mov REG0, RETVALx if (Z_REG(op1_addr) != ZREG_FP) { | ldr Rx(Z_REG(op1_addr)), T1 // restore } | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 | mov FCARG1x, REG0 } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { |6: if (opline->op2_type == IS_UNUSED) { uint32_t var_info = MAY_BE_NULL; zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval | EXT_CALL zend_hash_next_index_insert, REG0 | // if (UNEXPECTED(!var_ptr)) { | mov REG0, RETVALx | cbz REG0, >1 |.cold_code |1: | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); | CANNOT_ADD_ELEMENT opline | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); | b >9 |.code if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0, 0)) { return 0; } } else { uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { return 0; } if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { var_info |= MAY_BE_REF; } if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { var_info |= MAY_BE_RC1; } |8: | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); if (opline->op1_type == IS_VAR) { ZEND_ASSERT(opline->result_type == IS_UNUSED); if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { return 0; } } else { if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { return 0; } } } } if (((op1_info & MAY_BE_ARRAY) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) || (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY)))) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { |.cold_code |7: } if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) && (op1_info & MAY_BE_ARRAY)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 | bgt >2 } | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { | str Rx(Z_REG(op1_addr)), T1 // save } | EXT_CALL _zend_new_array_0, REG0 | mov REG0, RETVALx if (Z_REG(op1_addr) != ZREG_FP) { | ldr Rx(Z_REG(op1_addr)), T1 // restore } | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 | mov FCARG1x, REG0 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); | b <6 |2: } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { | SET_EX_OPLINE opline, REG0 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } if (opline->op2_type == IS_UNUSED) { | mov FCARG2x, xzr } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) } else { | LOAD_ZVAL_ADDR FCARG2x, op2_addr } if (opline->result_type == IS_UNUSED) { | mov CARG4, xzr } else { | LOAD_ZVAL_ADDR CARG4, res_addr } | LOAD_ZVAL_ADDR CARG3, op3_addr | EXT_CALL zend_jit_assign_dim_helper, REG0 #ifdef ZEND_JIT_USE_RC_INFERENCE if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { /* ASSIGN_DIM may increase refcount of the value */ val_info |= MAY_BE_RCN; } #endif | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { | b >9 // END } |.code } } #ifdef ZEND_JIT_USE_RC_INFERENCE if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { /* ASSIGN_DIM may increase refcount of the key */ op2_info |= MAY_BE_RCN; } #endif |9: | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { zend_jit_check_exception(Dst); } return 1; } static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) { zend_jit_addr op2_addr, op3_addr, var_addr; const void *not_found_exit_addr = NULL; uint32_t var_info = MAY_BE_NULL; ZEND_ASSERT(opline->result_type == IS_UNUSED); op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; op3_addr = OP1_DATA_ADDR(); | SET_EX_OPLINE opline, REG0 if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w | GET_Z_PTR FCARG2x, FCARG1x | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] | cmp TMP1w, #IS_ARRAY | bne >2 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) | b >3 |.cold_code |2: | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 | mov FCARG1x, RETVALx | cbnz RETVALx, >1 | b ->exception_handler_undef |.code |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } |3: | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { if (op1_info & MAY_BE_ARRAY) { |.cold_code |7: } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 | bgt >7 } if (Z_REG(op1_addr) != ZREG_FP) { | str Rx(Z_REG(op1_addr)), T1 // save } if (op1_info & MAY_BE_UNDEF) { if (op1_info & MAY_BE_NULL) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 } | LOAD_32BIT_VAL FCARG1x, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 |1: } | // ZVAL_ARR(container, zend_new_array(8)); | EXT_CALL _zend_new_array_0, REG0 | mov REG0, RETVALx if (Z_REG(op1_addr) != ZREG_FP) { | ldr Rx(Z_REG(op1_addr)), T1 // restore } | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 | mov FCARG1x, REG0 if (op1_info & MAY_BE_ARRAY) { | b >1 |.code |1: } } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); |6: if (opline->op2_type == IS_UNUSED) { var_info = MAY_BE_NULL; | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval | EXT_CALL zend_hash_next_index_insert, REG0 | mov REG0, RETVALx | // if (UNEXPECTED(!var_ptr)) { | cbz REG0, >1 |.cold_code |1: | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); | CANNOT_ADD_ELEMENT opline | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); | b >9 |.code } else { var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { var_info |= MAY_BE_REF; } if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { var_info |= MAY_BE_RC1; } if (dim_type != IS_UNKNOWN && dim_type != IS_UNDEF && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY && (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!not_found_exit_addr) { return 0; } } if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, not_found_exit_addr, NULL)) { return 0; } |8: if (not_found_exit_addr && dim_type != IS_REFERENCE) { | IF_NOT_Z_TYPE, REG0, dim_type, ¬_found_exit_addr, TMP1w var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); } if (var_info & MAY_BE_REF) { binary_op_type binary_op = get_binary_op(opline->extended_value); | IF_NOT_Z_TYPE, REG0, IS_REFERENCE, >1, TMP1w | GET_Z_PTR FCARG1x, REG0 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] | cbnz TMP1, >2 | add REG0, FCARG1x, #offsetof(zend_reference, val) |.cold_code |2: | LOAD_ZVAL_ADDR FCARG2x, op3_addr | LOAD_ADDR CARG3, binary_op if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (op1_data_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 } else { | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 } | b >9 |.code |1: } } var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); switch (opline->extended_value) { case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: case ZEND_DIV: if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, 1 /* may overflow */, may_throw)) { return 0; } break; case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: case ZEND_SL: case ZEND_SR: case ZEND_MOD: if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, NULL, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, op1_data_range, 0, var_addr, var_def_info, var_info, may_throw)) { return 0; } break; case ZEND_CONCAT: if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, may_throw)) { return 0; } break; default: ZEND_UNREACHABLE(); } | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { binary_op_type binary_op; if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { |.cold_code |7: } if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } if (opline->op2_type == IS_UNUSED) { | mov FCARG2x, xzr } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) } else { | LOAD_ZVAL_ADDR FCARG2x, op2_addr } binary_op = get_binary_op(opline->extended_value); | LOAD_ZVAL_ADDR CARG3, op3_addr | LOAD_ADDR CARG4, binary_op | EXT_CALL zend_jit_assign_dim_op_helper, REG0 |9: | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 if (may_throw) { zend_jit_check_exception(Dst); } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { | b >9 // END |.code |9: } } else if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) && (!not_found_exit_addr || (var_info & MAY_BE_REF))) { |.cold_code |9: | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, opline, ZREG_TMP1, ZREG_TMP2 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { zend_jit_check_exception(Dst); } | b >9 |.code |9: } return 1; } static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) { zend_jit_addr op1_addr, op2_addr; ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); op1_addr = OP1_ADDR(); op2_addr = OP2_ADDR(); if (op1_info & MAY_BE_REF) { binary_op_type binary_op = get_binary_op(opline->extended_value); | LOAD_ZVAL_ADDR FCARG1x, op1_addr | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >1, TMP1w | GET_Z_PTR FCARG1x, FCARG1x | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] | cbnz TMP1, >2 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) |.cold_code |2: | LOAD_ZVAL_ADDR FCARG2x, op2_addr | LOAD_ADDR CARG3, binary_op | SET_EX_OPLINE opline, REG0 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 } else { | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 } zend_jit_check_exception(Dst); | b >9 |.code |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } int result; switch (opline->extended_value) { case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: case ZEND_DIV: result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); break; case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: case ZEND_SL: case ZEND_SR: case ZEND_MOD: result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); break; case ZEND_CONCAT: result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); break; default: ZEND_UNREACHABLE(); } |9: return result; } static int zend_jit_cmp_long_long(dasm_State **Dst, const zend_op *opline, zend_ssa_range *op1_range, zend_jit_addr op1_addr, zend_ssa_range *op2_range, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr, bool skip_comparison) { bool swap = 0; bool result; if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE), TMP1w, TMP2 } if (smart_branch_opcode && !exit_addr) { if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { if (!result) { | b => target_label } } else if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) { if (result) { | b => target_label } } else if (smart_branch_opcode == ZEND_JMPZNZ) { if (!result) { | b => target_label } else { | b => target_label2 } } else { ZEND_UNREACHABLE(); } } return 1; } if (skip_comparison) { if (Z_MODE(op1_addr) != IS_REG && (Z_MODE(op2_addr) == IS_REG || (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { swap = 1; } } else if (Z_MODE(op1_addr) == IS_REG) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { | cmp Rx(Z_REG(op1_addr)), xzr } else { | LONG_CMP Z_REG(op1_addr), op2_addr, TMP1 } } else if (Z_MODE(op2_addr) == IS_REG) { if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { | cmp Rx(Z_REG(op2_addr)), xzr } else { | LONG_CMP Z_REG(op2_addr), op1_addr, TMP1 } swap = 1; } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { | LONG_CMP_WITH_CONST op2_addr, Z_LVAL_P(Z_ZV(op1_addr)), TMP1, TMP2 swap = 1; } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { | LONG_CMP_WITH_CONST op1_addr, Z_LVAL_P(Z_ZV(op2_addr)), TMP1, TMP2 } else { | GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { | cmp Rx(ZREG_REG0), xzr } else { | LONG_CMP ZREG_REG0, op2_addr, TMP1 } } if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | cset REG0w, eq break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | cset REG0w, ne break; case ZEND_IS_SMALLER: if (swap) { | cset REG0w, gt } else { | cset REG0w, lt } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | cset REG0w, ge } else { | cset REG0w, le } break; default: ZEND_UNREACHABLE(); } | add REG0w, REG0w, #2 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: if (exit_addr) { | bne &exit_addr } else { | bne => target_label } break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { | beq &exit_addr } else { | beq => target_label } break; case ZEND_IS_NOT_IDENTICAL: if (exit_addr) { | bne &exit_addr } else { | beq => target_label } break; case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { | ble &exit_addr } else { | ble => target_label } } else { if (exit_addr) { | bge &exit_addr } else { | bge => target_label } } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { | blt &exit_addr } else { | blt => target_label } } else { if (exit_addr) { | bgt &exit_addr } else { | bgt => target_label } } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: if (exit_addr) { | beq &exit_addr } else { | beq => target_label } break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { | bne &exit_addr } else { | bne => target_label } break; case ZEND_IS_NOT_IDENTICAL: if (exit_addr) { | beq &exit_addr } else { | bne => target_label } break; case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { | bgt &exit_addr } else { | bgt => target_label } } else { if (exit_addr) { | blt &exit_addr } else { | blt => target_label } } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { | bge &exit_addr } else { | bge => target_label } } else { if (exit_addr) { | ble &exit_addr } else { | ble => target_label } } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPZNZ) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | bne => target_label break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | beq => target_label break; case ZEND_IS_SMALLER: if (swap) { | ble => target_label } else { | bge => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | blt => target_label } else { | bgt => target_label } break; default: ZEND_UNREACHABLE(); } | b => target_label2 } else { ZEND_UNREACHABLE(); } } else { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | cset REG0w, eq break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | cset REG0w, ne break; case ZEND_IS_SMALLER: if (swap) { | cset REG0w, gt } else { | cset REG0w, lt } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | cset REG0w, ge } else { | cset REG0w, le } break; default: ZEND_UNREACHABLE(); } | add REG0w, REG0w, #2 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } return 1; } static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: if (exit_addr) { | bne &exit_addr } else { | bne => target_label } break; case ZEND_IS_NOT_EQUAL: | bvs >1 if (exit_addr) { | beq &exit_addr } else { | beq => target_label } |1: break; case ZEND_IS_NOT_IDENTICAL: if (exit_addr) { | bvs &exit_addr | bne &exit_addr } else { | bvs >1 | beq => target_label |1: } break; case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { | bvs &exit_addr | bls &exit_addr } else { | bvs => target_label | bls => target_label } } else { if (exit_addr) { | bhs &exit_addr } else { | bhs => target_label } } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { | bvs &exit_addr | blo &exit_addr } else { | bvs => target_label | blo => target_label } } else { if (exit_addr) { | bhi &exit_addr } else { | bhi => target_label } } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPNZ) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | bvs >1 if (exit_addr) { | beq &exit_addr } else { | beq => target_label } |1: break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { | bne &exit_addr } else { | bne => target_label } break; case ZEND_IS_NOT_IDENTICAL: if (exit_addr) { | bvs >1 | beq &exit_addr |1: } else { | bne => target_label } break; case ZEND_IS_SMALLER: if (swap) { | bvs >1 // Always False if involving NaN if (exit_addr) { | bhi &exit_addr } else { | bhi => target_label } |1: } else { | bvs >1 if (exit_addr) { | blo &exit_addr } else { | blo => target_label } |1: } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | bvs >1 // Always False if involving NaN if (exit_addr) { | bhs &exit_addr } else { | bhs => target_label } |1: } else { | bvs >1 if (exit_addr) { | bls &exit_addr } else { | bls => target_label } |1: } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPZNZ) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | bne => target_label break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | bvs => target_label2 | beq => target_label break; case ZEND_IS_SMALLER: if (swap) { | bvs => target_label | bls => target_label } else { | bhs => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | bvs => target_label | blo => target_label } else { | bhi => target_label } break; default: ZEND_UNREACHABLE(); } | b => target_label2 } else if (smart_branch_opcode == ZEND_JMPZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 | bne => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | bvs >1 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 | beq => target_label |1: | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 break; case ZEND_IS_SMALLER: if (swap) { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 | bvs => target_label | bls => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 | bhs => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 | bvs => target_label | blo => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 | bhi => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | bvs >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 | beq => target_label |1: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 | bvs => target_label | bne => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 break; case ZEND_IS_SMALLER: if (swap) { | cset REG0w, hi | add REG0w, REG0w, #2 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 | bvs >1 // Always False if involving NaN | bhi => target_label |1: } else { | bvs >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 | blo => target_label |1: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | cset REG0w, hs | add REG0w, REG0w, #2 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 | bvs >1 // Always False if involving NaN | bhs => target_label |1: } else { | bvs >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 | bls => target_label |1: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } break; default: ZEND_UNREACHABLE(); } } else { ZEND_UNREACHABLE(); } } else { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: | bvs >1 | mov REG0, #IS_TRUE | beq >2 |1: | mov REG0, #IS_FALSE |2: break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: | bvs >1 | mov REG0, #IS_FALSE | beq >2 |1: | mov REG0, #IS_TRUE |2: break; case ZEND_IS_SMALLER: | bvs >1 | mov REG0, #IS_TRUE || if (swap) { | bhi >2 || } else { | blo >2 || } |1: | mov REG0, #IS_FALSE |2: break; case ZEND_IS_SMALLER_OR_EQUAL: | bvs >1 | mov REG0, #IS_TRUE || if (swap) { | bhs >2 || } else { | bls >2 || } |1: | mov REG0, #IS_FALSE |2: break; default: ZEND_UNREACHABLE(); } | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } return 1; } static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_reg tmp_reg = ZREG_FPR0; | DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_REG0, ZREG_TMP1 | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); } static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_reg tmp_reg = ZREG_FPR0; | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_REG0, ZREG_TMP1 | DOUBLE_CMP tmp_reg, op1_addr, ZREG_TMP1, ZREG_FPTMP return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); } static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { bool swap = 0; if (Z_MODE(op1_addr) == IS_REG) { | DOUBLE_CMP Z_REG(op1_addr), op2_addr, ZREG_TMP1, ZREG_FPTMP } else if (Z_MODE(op2_addr) == IS_REG) { | DOUBLE_CMP Z_REG(op2_addr), op1_addr, ZREG_TMP1, ZREG_FPTMP swap = 1; } else { zend_reg tmp_reg = ZREG_FPR0; | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP } return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); } static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { | tst RETVALw, RETVALw if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: | cset REG0w, eq break; case ZEND_IS_NOT_EQUAL: | cset REG0w, ne break; case ZEND_IS_SMALLER: | cset REG0w, lt break; case ZEND_IS_SMALLER_OR_EQUAL: | cset REG0w, le break; default: ZEND_UNREACHABLE(); } | add REG0w, REG0w, #2 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: if (exit_addr) { | bne &exit_addr } else { | bne => target_label } break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { | beq &exit_addr } else { | beq => target_label } break; case ZEND_IS_SMALLER: if (exit_addr) { | bge &exit_addr } else { | bge => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (exit_addr) { | bgt &exit_addr } else { | bgt => target_label } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: if (exit_addr) { | beq &exit_addr } else { | beq => target_label } break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { | bne &exit_addr } else { | bne => target_label } break; case ZEND_IS_SMALLER: if (exit_addr) { | blt &exit_addr } else { | blt => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (exit_addr) { | ble &exit_addr } else { | ble => target_label } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPZNZ) { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: | bne => target_label break; case ZEND_IS_NOT_EQUAL: | beq => target_label break; case ZEND_IS_SMALLER: | bge => target_label break; case ZEND_IS_SMALLER_OR_EQUAL: | bgt => target_label break; default: ZEND_UNREACHABLE(); } | b => target_label2 } else { ZEND_UNREACHABLE(); } } else { switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_CASE: | cset REG0w, eq break; case ZEND_IS_NOT_EQUAL: | cset REG0w, ne break; case ZEND_IS_SMALLER: | cset REG0w, lt break; case ZEND_IS_SMALLER_OR_EQUAL: | cset REG0w, le break; default: ZEND_UNREACHABLE(); } | add REG0w, REG0w, #2 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } return 1; } static int zend_jit_cmp(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr, bool skip_comparison) { bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); bool has_slow; has_slow = (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 } } if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { if (op2_info & MAY_BE_DOUBLE) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 |.cold_code |3: if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 } if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } | b >6 |.code } else { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 } } if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { |.cold_code |4: if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { if (!same_ops) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 } } if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } | b >6 } if (!same_ops) { |5: if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 } if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } | b >6 } |.code } } else if ((op1_info & MAY_BE_DOUBLE) && !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { if (!same_ops && (op2_info & MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 } } if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } } if (!same_ops && (op2_info & MAY_BE_LONG)) { if (op2_info & MAY_BE_DOUBLE) { |.cold_code } |3: if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 } if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } if (op2_info & MAY_BE_DOUBLE) { | b >6 |.code } } } else if ((op2_info & MAY_BE_DOUBLE) && !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { if (!same_ops && (op1_info & MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 } } if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } } if (!same_ops && (op1_info & MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { |.cold_code } |3: if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 } if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } if (op1_info & MAY_BE_DOUBLE) { | b >6 |.code } } } if (has_slow || (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { if (has_slow) { |.cold_code |9: } | SET_EX_OPLINE opline, REG0 if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { return 0; } op1_addr = real_addr; } if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { return 0; } op2_addr = real_addr; } | LOAD_ZVAL_ADDR FCARG1x, op1_addr if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { | IF_NOT_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w | LOAD_32BIT_VAL FCARG1x, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval |1: } if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 | str FCARG1x, T1 // save | LOAD_32BIT_VAL FCARG1x, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, REG0 | ldr FCARG1x, T1 // restore | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval | b >2 |1: | LOAD_ZVAL_ADDR FCARG2x, op2_addr |2: } else { | LOAD_ZVAL_ADDR FCARG2x, op2_addr } | EXT_CALL zend_compare, REG0 if ((opline->opcode != ZEND_CASE && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { | str RETVALw, T1 // save if (opline->opcode != ZEND_CASE) { | FREE_OP opline->op1_type, opline->op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 } | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2 | ldr RETVALw, T1 // restore } if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } if (has_slow) { | b >6 |.code } } |6: return 1; } static int zend_jit_identical(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr, bool skip_comparison) { uint32_t identical_label = (uint32_t)-1; uint32_t not_identical_label = (uint32_t)-1; if (smart_branch_opcode && !exit_addr) { if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { if (smart_branch_opcode == ZEND_JMPZ) { not_identical_label = target_label; } else if (smart_branch_opcode == ZEND_JMPNZ) { identical_label = target_label; } else if (smart_branch_opcode == ZEND_JMPZNZ) { not_identical_label = target_label; identical_label = target_label2; } else { ZEND_UNREACHABLE(); } } else { if (smart_branch_opcode == ZEND_JMPZ) { identical_label = target_label; } else if (smart_branch_opcode == ZEND_JMPNZ) { not_identical_label = target_label; } else if (smart_branch_opcode == ZEND_JMPZNZ) { identical_label = target_label; not_identical_label = target_label2; } else { ZEND_UNREACHABLE(); } } } if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { return 0; } return 1; } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } return 1; } if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { op1_info |= MAY_BE_NULL; op2_info |= MAY_BE_NULL; | LOAD_ZVAL_ADDR FCARG1x, op1_addr | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w |.cold_code |1: | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | SET_EX_OPLINE opline, REG0 | LOAD_32BIT_VAL FCARG1w, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval | b >1 |.code |1: | LOAD_ZVAL_ADDR FCARG2x, op2_addr | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w |.cold_code |1: | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | SET_EX_OPLINE opline, REG0 | str FCARG1x, T1 // save | LOAD_32BIT_VAL FCARG1w, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, REG0 if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } | ldr FCARG1x, T1 // restore | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval | b >1 |.code |1: } else if (op1_info & MAY_BE_UNDEF) { op1_info |= MAY_BE_NULL; | LOAD_ZVAL_ADDR FCARG1x, op1_addr | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w |.cold_code |1: | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | SET_EX_OPLINE opline, REG0 | LOAD_32BIT_VAL FCARG1w, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval | b >1 |.code |1: if (opline->op2_type != IS_CONST) { | LOAD_ZVAL_ADDR FCARG2x, op2_addr } } else if (op2_info & MAY_BE_UNDEF) { op2_info |= MAY_BE_NULL; | LOAD_ZVAL_ADDR FCARG2x, op2_addr | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w |.cold_code |1: | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | SET_EX_OPLINE opline, REG0 | LOAD_32BIT_VAL FCARG1w, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, REG0 if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval | b >1 |.code |1: if (opline->op1_type != IS_CONST) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } } else if ((op1_info & op2_info & MAY_BE_ANY) != 0) { if (opline->op1_type != IS_CONST) { if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { return 0; } op1_addr = real_addr; } } if (opline->op2_type != IS_CONST) { if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { return 0; } op2_addr = real_addr; } | LOAD_ZVAL_ADDR FCARG2x, op2_addr } if (opline->op1_type != IS_CONST) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } } if ((op1_info & op2_info & MAY_BE_ANY) == 0) { if ((opline->opcode != ZEND_CASE_STRICT && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { if (opline->opcode != ZEND_CASE_STRICT) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } if (smart_branch_opcode) { if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | b &exit_addr } } else if (not_identical_label != (uint32_t)-1) { | b =>not_identical_label } } else { | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 if (may_throw) { zend_jit_check_exception(Dst); } } return 1; } if (opline->op1_type & (IS_CV|IS_VAR)) { | ZVAL_DEREF FCARG1x, op1_info, TMP1w } if (opline->op2_type & (IS_CV|IS_VAR)) { | ZVAL_DEREF FCARG2x, op2_info, TMP1w } if (has_concrete_type(op1_info) && has_concrete_type(op2_info) && concrete_type(op1_info) == concrete_type(op2_info) && concrete_type(op1_info) <= IS_TRUE) { if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | b &exit_addr } } else if (identical_label != (uint32_t)-1) { | b =>identical_label } } else { | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 } } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | b &exit_addr } } else if (identical_label != (uint32_t)-1) { | b =>identical_label } } else { | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 } } else { if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | b &exit_addr } } else if (not_identical_label != (uint32_t)-1) { | b =>not_identical_label } } else { | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 } } } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { zval *val = Z_ZV(op1_addr); | ldrb TMP1w, [FCARG2x, #offsetof(zval, u1.v.type)] | cmp TMP1w, #Z_TYPE_P(val) if (smart_branch_opcode) { if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { | bne >8 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { | b &exit_addr } else if (identical_label != (uint32_t)-1) { | b =>identical_label } else { | b >9 } |8: } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { | beq &exit_addr } else if (identical_label != (uint32_t)-1) { | beq =>identical_label } else { | beq >9 } } else { if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { | cset REG0w, eq } else { | cset REG0w, ne } | add REG0w, REG0w, #2 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | b &exit_addr } } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { | b =>not_identical_label } } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { zval *val = Z_ZV(op2_addr); | ldrb TMP1w, [FCARG1x, #offsetof(zval, u1.v.type)] | cmp TMP1w, #Z_TYPE_P(val) if (smart_branch_opcode) { if (opline->opcode != ZEND_CASE_STRICT && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { | bne >8 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { | b &exit_addr } else if (identical_label != (uint32_t)-1) { | b =>identical_label } else { | b >9 } |8: } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { | beq &exit_addr } else if (identical_label != (uint32_t)-1) { | beq =>identical_label } else { | beq >9 } } else { if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { | cset REG0w, eq } else { | cset REG0w, ne } | add REG0w, REG0w, #2 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } if (opline->opcode != ZEND_CASE_STRICT && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } } if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | b &exit_addr } } else if (not_identical_label != (uint32_t)-1) { | b =>not_identical_label } } } else { if (opline->op1_type == IS_CONST) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } if (opline->op2_type == IS_CONST) { | LOAD_ZVAL_ADDR FCARG2x, op2_addr } | EXT_CALL zend_is_identical, REG0 if ((opline->opcode != ZEND_CASE_STRICT && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { | str RETVALw, T1 // save if (opline->opcode != ZEND_CASE_STRICT) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } | ldr RETVALw, T1 // restore } if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | cbnz RETVALw, &exit_addr } else { | cbz RETVALw, &exit_addr } } else if (not_identical_label != (uint32_t)-1) { | cbz RETVALw, =>not_identical_label if (identical_label != (uint32_t)-1) { | b =>identical_label } } else if (identical_label != (uint32_t)-1) { | cbnz RETVALw, =>identical_label } } else { if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { | add RETVALw, RETVALw, #2 } else { | neg RETVALw, RETVALw | add RETVALw, RETVALw, #3 } | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, RETVALw, TMP1 } } |9: if (may_throw) { zend_jit_check_exception(Dst); } return 1; } static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr) { uint32_t true_label = -1; uint32_t false_label = -1; bool set_bool = 0; bool set_bool_not = 0; bool set_delayed = 0; bool jmp_done = 0; if (branch_opcode == ZEND_BOOL) { set_bool = 1; } else if (branch_opcode == ZEND_BOOL_NOT) { set_bool = 1; set_bool_not = 1; } else if (branch_opcode == ZEND_JMPZ) { false_label = target_label; } else if (branch_opcode == ZEND_JMPNZ) { true_label = target_label; } else if (branch_opcode == ZEND_JMPZNZ) { true_label = target_label2; false_label = target_label; } else if (branch_opcode == ZEND_JMPZ_EX) { set_bool = 1; false_label = target_label; } else if (branch_opcode == ZEND_JMPNZ_EX) { set_bool = 1; true_label = target_label; } else { ZEND_UNREACHABLE(); } if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { if (zend_is_true(Z_ZV(op1_addr))) { /* Always TRUE */ if (set_bool) { if (set_bool_not) { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } else { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } } if (true_label != (uint32_t)-1) { | b =>true_label } } else { /* Always FALSE */ if (set_bool) { if (set_bool_not) { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } } if (false_label != (uint32_t)-1) { | b =>false_label } } return 1; } if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr | ZVAL_DEREF FCARG1x, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { /* Always TRUE */ if (set_bool) { if (set_bool_not) { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } else { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } } if (true_label != (uint32_t)-1) { | b =>true_label } } else { if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { /* Always FALSE */ if (set_bool) { if (set_bool_not) { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } } } else { | CMP_ZVAL_TYPE op1_addr, IS_TRUE, ZREG_TMP1 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { if ((op1_info & MAY_BE_LONG) && !(op1_info & MAY_BE_UNDEF) && !set_bool) { if (exit_addr) { if (branch_opcode == ZEND_JMPNZ) { | blt >9 } else { | blt &exit_addr } } else if (false_label != (uint32_t)-1) { | blt =>false_label } else { | blt >9 } jmp_done = 1; } else { | bgt >2 } } if (!(op1_info & MAY_BE_TRUE)) { /* It's FALSE */ if (set_bool) { if (set_bool_not) { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } } } else { if (exit_addr) { if (set_bool) { | bne >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | b &exit_addr } else { | b >9 } |1: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { | bne &exit_addr } } } else { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | beq &exit_addr } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { | bne &exit_addr } else { | beq >9 } } } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { if (set_bool) { | bne >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 if (true_label != (uint32_t)-1) { | b =>true_label } else { | b >9 } |1: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } else { if (true_label != (uint32_t)-1) { | beq =>true_label } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { | bne =>false_label jmp_done = 1; } else { | beq >9 } } } else if (set_bool) { | cset REG0w, eq if (set_bool_not) { | neg REG0w, REG0w | add REG0w, REG0w, #3 } else { | add REG0w, REG0w, #2 } if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { set_delayed = 1; } else { | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } } } } /* It's FALSE, but may be UNDEF */ if (op1_info & MAY_BE_UNDEF) { if (op1_info & MAY_BE_ANY) { if (set_delayed) { | CMP_ZVAL_TYPE op1_addr, IS_UNDEF, ZREG_TMP1 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 | beq >1 } else { | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 } |.cold_code |1: } | LOAD_32BIT_VAL FCARG1w, opline->op1.var | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_jit_undefined_op_helper, REG0 if (may_throw) { if (!zend_jit_check_exception_undef_result(Dst, opline)) { return 0; } } if (exit_addr) { if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { | b &exit_addr } } else if (false_label != (uint32_t)-1) { | b =>false_label } if (op1_info & MAY_BE_ANY) { if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | b >9 } } else if (false_label == (uint32_t)-1) { | b >9 } |.code } } if (!jmp_done) { if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { if (op1_info & MAY_BE_LONG) { | b >9 } } else if (op1_info & MAY_BE_LONG) { | b &exit_addr } } else if (false_label != (uint32_t)-1) { | b =>false_label } else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | b >9 } } } } if (op1_info & MAY_BE_LONG) { |2: if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 } if (Z_MODE(op1_addr) == IS_REG) { | tst Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) } else { | LONG_CMP_WITH_CONST op1_addr, Z_L(0), TMP1, TMP2 } if (set_bool) { | cset REG0w, ne if (set_bool_not) { | neg REG0w, REG0w | add REG0w, REG0w, #3 } else { | add REG0w, REG0w, #2 } | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | bne &exit_addr } else { | beq &exit_addr } } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { if (true_label != (uint32_t)-1) { | bne =>true_label if (false_label != (uint32_t)-1) { | b =>false_label } } else { | beq =>false_label } } } if ((op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) == MAY_BE_DOUBLE) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { |.cold_code } |2: | fmov FPR0, xzr // TODO: "movi d0, #0" is not recognized by DynASM/arm64 | DOUBLE_CMP ZREG_FPR0, op1_addr, ZREG_TMP1, ZREG_FPTMP if (set_bool) { if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 | bvs &exit_addr | bne &exit_addr | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } else { | bvs >1 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 | beq &exit_addr |1: | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } } else if (false_label != (uint32_t)-1) { // JMPZ_EX | bvs >1 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 | beq => false_label |1: | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } else if (true_label != (uint32_t)-1) { // JMPNZ_EX | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 | bvs => true_label | bne => true_label | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } else if (set_bool_not) { // BOOL_NOT | mov REG0w, #IS_FALSE | bvs >1 | bne >1 | mov REG0w, #IS_TRUE |1: | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } else { // BOOL | mov REG0w, #IS_TRUE | bvs >1 | bne >1 | mov REG0w, #IS_FALSE |1: | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 |.code } } else { if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | bvs &exit_addr | bne &exit_addr |1: } else { | bvs >1 | beq &exit_addr |1: } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } } else { ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); if (false_label != (uint32_t)-1 ) { | bvs >1 | beq => false_label |1: if (true_label != (uint32_t)-1) { | b =>true_label } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } } else { | bvs => true_label | bne => true_label if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } } } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { |.code } } } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { |.cold_code |2: } if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_is_true, REG0 | mov REG0, RETVALx if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3, ZREG_TMP1, ZREG_TMP2 } | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 | GC_DELREF FCARG1x, TMP1w | bne >3 // In x86, r0 is used in macro ZVAL_DTOR_FUNC as temporary register, hence, r0 should be saved/restored // before/after this macro. In AArch64, TMP1 is used, but we still have to store REG0, // because it's clobbered by function call. | str REG0, T1 // save | ZVAL_DTOR_FUNC op1_info, opline, TMP1 | ldr REG0, T1 // restore |3: } if (may_throw) { | MEM_LOAD_64_ZTS ldr, REG1, executor_globals, exception, TMP1 | cbnz REG1, ->exception_handler_undef } if (set_bool) { if (set_bool_not) { | neg REG0w, REG0w | add REG0w, REG0w, #3 } else { | add REG0w, REG0w, #2 } | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 if (exit_addr) { | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | bne &exit_addr } else { | beq &exit_addr } } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 if (true_label != (uint32_t)-1) { | bne =>true_label if (false_label != (uint32_t)-1) { | b =>false_label } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } } else { | beq =>false_label } } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 |.code } } else { if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | cbnz REG0w, &exit_addr if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } } else { | cbz REG0w, &exit_addr if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } } } else if (true_label != (uint32_t)-1) { | cbnz REG0w, =>true_label if (false_label != (uint32_t)-1) { | b =>false_label } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } } else { | cbz REG0w, =>false_label if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { |.code } } } |9: return 1; } static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) { if (op1_addr != op1_def_addr) { if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { return 0; } if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { op1_addr = op1_def_addr; } } if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0, 1)) { return 0; } if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { return 0; } if (op1_info & MAY_BE_UNDEF) { zend_jit_check_exception(Dst); } return 1; } static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) { ZEND_ASSERT(opline->op1_type == IS_CV); if (op2_addr != op2_def_addr) { if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { return 0; } if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { op2_addr = op2_def_addr; } } if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op1_use_addr) == IS_REG && !Z_LOAD(op1_use_addr) && !Z_STORE(op1_use_addr)) { /* Force type update */ op1_info |= MAY_BE_UNDEF; } if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, may_throw)) { return 0; } if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { return 0; } if (opline->result_type != IS_UNUSED) { if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { return 0; } } return 1; } /* copy of hidden zend_closure */ typedef struct _zend_closure { zend_object std; zend_function func; zval this_ptr; zend_class_entry *called_scope; zif_handler orig_internal_handler; } zend_closure; static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | // Check Stack Overflow | MEM_LOAD_64_ZTS ldr, REG1, executor_globals, vm_stack_end, TMP1 | MEM_LOAD_OP_ZTS sub, ldr, REG1, executor_globals, vm_stack_top, TMP1, TMP2 | CMP_64_WITH_CONST_32 REG1, used_stack, TMP1 | blo &exit_addr return 1; } static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack) { uint32_t used_stack; bool stack_check = 1; // REG0 -> zend_function // FCARG1 -> used_stack if (func) { used_stack = zend_vm_calc_used_stack(opline->extended_value, func); if ((int)used_stack <= checked_stack) { stack_check = 0; } } else { used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval); | // if (EXPECTED(ZEND_USER_CODE(func->type))) { if (!is_closure) { | LOAD_32BIT_VAL FCARG1w, used_stack | // Check whether REG0 is an internal function. | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] | TST_32_WITH_CONST TMP1w, 1, TMP2w | bne >1 } else { | LOAD_32BIT_VAL FCARG1w, used_stack } | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); | LOAD_32BIT_VAL REG2w, opline->extended_value if (!is_closure) { | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.num_args)] | cmp REG2w, TMP1w | csel REG2w, REG2w, TMP1w, le | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.last_var)] | sub REG2w, REG2w, TMP1w | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.T)] | sub REG2w, REG2w, TMP1w } else { | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.num_args)] | cmp REG2w, TMP1w | csel REG2w, REG2w, TMP1w, le | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.last_var)] | sub REG2w, REG2w, TMP1w | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.T)] | sub REG2w, REG2w, TMP1w } | sxtw REG2, REG2w | sub FCARG1x, FCARG1x, REG2, lsl #4 |1: } zend_jit_start_reuse_ip(); | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { | MEM_LOAD_64_ZTS ldr, RX, executor_globals, vm_stack_top, TMP1 if (stack_check) { | // Check Stack Overflow | MEM_LOAD_64_ZTS ldr, REG2, executor_globals, vm_stack_end, TMP1 | sub REG2, REG2, RX if (func) { | CMP_64_WITH_CONST_32 REG2, used_stack, TMP1 } else { | cmp REG2, FCARG1x } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | blo &exit_addr } else { | blo >1 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); |.cold_code |1: if (func) { | LOAD_32BIT_VAL FCARG1w, used_stack } if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_jit_int_extend_stack_helper, REG0 } else { if (!is_closure) { | mov FCARG2x, REG0 } else { | add FCARG2x, REG0, #offsetof(zend_closure, func) } | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_jit_extend_stack_helper, REG0 } | mov RX, RETVALx | b >1 |.code } } if (func) { || if (arm64_may_encode_imm12((int64_t)used_stack)) { | MEM_UPDATE_ZTS add, ldr, str, #used_stack, executor_globals, vm_stack_top, REG2, TMP1 || } else { | LOAD_32BIT_VAL TMP1w, used_stack | MEM_UPDATE_ZTS add, ldr, str, TMP1, executor_globals, vm_stack_top, REG2, TMP2 || } } else { | MEM_UPDATE_ZTS add, ldr, str, FCARG1x, executor_globals, vm_stack_top, REG2, TMP1 } | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { | // ZEND_SET_CALL_INFO(call, 0, call_info); | LOAD_32BIT_VAL TMP1w, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) | str TMP1w, EX:RX->This.u1.type_info } if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { | // call->func = func; |1: | ADDR_STORE EX:RX->func, func, REG1 } else { if (!is_closure) { | // call->func = func; | str REG0, EX:RX->func } else { | // call->func = &closure->func; | add REG1, REG0, #offsetof(zend_closure, func) | str REG1, EX:RX->func } |1: } if (opline->opcode == ZEND_INIT_METHOD_CALL) { | // Z_PTR(call->This) = obj; | ldr REG1, T1 | str REG1, EX:RX->This.value.ptr if (opline->op1_type == IS_UNUSED || delayed_fetch_this) { | // call->call_info |= ZEND_CALL_HAS_THIS; if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS | str TMP1w, EX:RX->This.u1.type_info } else { | ldr TMP1w, EX:RX->This.u1.type_info | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_HAS_THIS, TMP2w | str TMP1w, EX:RX->This.u1.type_info } } else { if (opline->op1_type == IS_CV) { | // GC_ADDREF(obj); | GC_ADDREF REG1, TMP1w } | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | LOAD_32BIT_VAL TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) | str TMP1w, EX:RX->This.u1.type_info } else { | ldr TMP1w, EX:RX->This.u1.type_info | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS), TMP2w | str TMP1w, EX:RX->This.u1.type_info } } } else if (!is_closure) { | // Z_CE(call->This) = called_scope; | str xzr, EX:RX->This.value.ptr } else { if (opline->op2_type == IS_CV) { | // GC_ADDREF(closure); | GC_ADDREF REG0, TMP1w } | // object_or_called_scope = closure->called_scope; | ldr REG1, [REG0, #offsetof(zend_closure, called_scope)] | str REG1, EX:RX->This.value.ptr | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); | ldr REG2w, [REG0, #offsetof(zend_closure, func.common.fn_flags)] | BW_OP_32_WITH_CONST and, REG2w, REG2w, ZEND_ACC_FAKE_CLOSURE, TMP1w | BW_OP_32_WITH_CONST orr, REG2w, REG2w, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE), TMP1w | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { | ldrb TMP1w, [REG0, #offsetof(zend_closure, this_ptr.u1.v.type)] | cmp TMP1w, #IS_UNDEF | beq >1 | // call_info |= ZEND_CALL_HAS_THIS; | BW_OP_32_WITH_CONST orr, REG2w, REG2w, ZEND_CALL_HAS_THIS, TMP1w | // object_or_called_scope = Z_OBJ(closure->this_ptr); | ldr REG1, [REG0, #offsetof(zend_closure, this_ptr.value.ptr)] |1: | // ZEND_SET_CALL_INFO(call, 0, call_info); | ldr TMP1w, EX:RX->This.u1.type_info | orr TMP1w, TMP1w, REG2w | str TMP1w, EX:RX->This.u1.type_info | // Z_PTR(call->This) = object_or_called_scope; | str REG1, EX:RX->This.value.ptr | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.run_time_cache__ptr)] | cbnz TMP1, >1 | add FCARG1x, REG0, #offsetof(zend_closure, func) | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 |1: } | // ZEND_CALL_NUM_ARGS(call) = num_args; | LOAD_32BIT_VAL TMP1w, opline->extended_value | str TMP1w, EX:RX->This.u2.num_args return 1; } static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) { int32_t exit_point; const void *exit_addr; if (func->type == ZEND_INTERNAL_FUNCTION) { #ifdef ZEND_WIN32 // TODO: ASLR may cause different addresses in different workers ??? return 0; #endif } else if (func->type == ZEND_USER_FUNCTION) { if (!zend_accel_in_shm(func->op_array.opcodes)) { /* op_array and op_array->opcodes are not persistent. We can't link. */ return 0; } } else { ZEND_UNREACHABLE(); return 0; } exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | // call = EX(call); | ldr REG1, EX->call while (level > 0) { | ldr REG1, EX:REG1->prev_execute_data level--; } if (func->type == ZEND_USER_FUNCTION && (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || (func->common.fn_flags & ZEND_ACC_CLOSURE) || !func->common.function_name)) { const zend_op *opcodes = func->op_array.opcodes; | ldr REG1, EX:REG1->func | LOAD_ADDR REG2, ((ptrdiff_t)opcodes) | ldr TMP1, [REG1, #offsetof(zend_op_array, opcodes)] | cmp TMP1, REG2 | bne &exit_addr } else { | LOAD_ADDR REG2, ((ptrdiff_t)func) | ldr TMP1, EX:REG1->func | cmp TMP1, REG2 | bne &exit_addr } return 1; } static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; zend_function *func = NULL; if (delayed_call_chain) { if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { return 0; } } if (info) { call_info = info->callee_info; while (call_info && call_info->caller_init_opline != opline) { call_info = call_info->next_callee; } if (call_info && call_info->callee_func && !call_info->is_prototype) { func = call_info->callee_func; } } if (!func && trace && trace->op == ZEND_JIT_TRACE_INIT_CALL) { func = (zend_function*)trace->func; } if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { /* load constant address later */ } else if (func && op_array == &func->op_array) { /* recursive call */ | ldr REG0, EX->func } else { | // if (CACHED_PTR(opline->result.num)) | ldr REG2, EX->run_time_cache | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG2, opline->result.num, TMP1 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) && opline->opcode != ZEND_INIT_FCALL) { /* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */ | LOAD_ADDR REG1, ((ptrdiff_t)func) | cmp REG0, REG1 | bne >1 } else { | cbz REG0, >1 } |.cold_code |1: if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_USER_FUNCTION && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { | LOAD_ADDR FCARG1x, func | MEM_ACCESS_64_WITH_UOFFSET str, FCARG1x, REG2, opline->result.num, TMP1 | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 | mov REG0, RETVALx | b >3 } else { zval *zv = RT_CONSTANT(opline, opline->op2); if (opline->opcode == ZEND_INIT_FCALL) { | LOAD_ADDR FCARG1x, Z_STR_P(zv); | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 | EXT_CALL zend_jit_find_func_helper, REG0 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { | LOAD_ADDR FCARG1x, Z_STR_P(zv + 1); | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 | EXT_CALL zend_jit_find_func_helper, REG0 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { | LOAD_ADDR FCARG1x, zv; | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1 | EXT_CALL zend_jit_find_ns_func_helper, REG0 } else { ZEND_UNREACHABLE(); } | // Get the return value of function zend_jit_find_func_helper/zend_jit_find_ns_func_helper | mov REG0, RETVALx if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } if (!func || opline->opcode == ZEND_INIT_FCALL) { | cbnz REG0, >3 } else if (func->type == ZEND_USER_FUNCTION && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { const zend_op *opcodes = func->op_array.opcodes; | LOAD_ADDR REG1, ((ptrdiff_t)opcodes) | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] | cmp TMP1, REG1 | beq >3 } else { | LOAD_ADDR REG1, ((ptrdiff_t)func) | cmp REG0, REG1 | beq >3 } | b &exit_addr } else { | cbnz REG0, >3 | // SAVE_OPLINE(); | SET_EX_OPLINE opline, REG0 | b ->undefined_function } } |.code |3: } if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, checked_stack)) { return 0; } if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { if (!zend_jit_save_call_chain(Dst, call_level)) { return 0; } } else { delayed_call_chain = 1; delayed_call_level = call_level; } return 1; } static int zend_jit_init_method_call(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, uint32_t op1_info, zend_jit_addr op1_addr, zend_class_entry *ce, bool ce_is_instanceof, bool on_this, bool delayed_fetch_this, zend_class_entry *trace_ce, zend_jit_trace_rec *trace, int checked_stack, bool polymorphic_side_trace) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; zend_function *func = NULL; zval *function_name; ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); function_name = RT_CONSTANT(opline, opline->op2); if (info) { call_info = info->callee_info; while (call_info && call_info->caller_init_opline != opline) { call_info = call_info->next_callee; } if (call_info && call_info->callee_func && !call_info->is_prototype) { func = call_info->callee_func; } } if (polymorphic_side_trace) { /* function is passed in r0 from parent_trace */ } else { if (on_this) { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 } else { if (op1_info & MAY_BE_REF) { if (opline->op1_type == IS_CV) { if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } | ZVAL_DEREF FCARG1x, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } else { /* Hack: Convert reference to regular value to simplify JIT code */ ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 | LOAD_ZVAL_ADDR FCARG1x, op1_addr | EXT_CALL zend_jit_unref_helper, REG0 |1: } } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 |.cold_code |1: if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } | SET_EX_OPLINE opline, REG0 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { | EXT_CALL zend_jit_invalid_method_call_tmp, REG0 } else { | EXT_CALL zend_jit_invalid_method_call, REG0 } | b ->exception_handler |.code } } | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 } if (delayed_call_chain) { if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { return 0; } } | str FCARG1x, T1 // save if (func) { | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); | ldr REG0, EX->run_time_cache | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 | cbz REG0, >1 } else { | // if (CACHED_PTR(opline->result.num) == obj->ce)) { | ldr REG0, EX->run_time_cache | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->result.num, TMP1 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >1 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 } |.cold_code |1: | LOAD_ADDR FCARG2x, function_name if (TMP_ZVAL_OFFSET == 0) { | mov CARG3, sp } else { | add CARG3, sp, #TMP_ZVAL_OFFSET } | SET_EX_OPLINE opline, REG0 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { | EXT_CALL zend_jit_find_method_tmp_helper, REG0 } else { | EXT_CALL zend_jit_find_method_helper, REG0 } | mov REG0, RETVALx | cbnz REG0, >2 | b ->exception_handler |.code |2: } if ((!func || zend_jit_may_be_modified(func, op_array)) && trace && trace->op == ZEND_JIT_TRACE_INIT_CALL && trace->func ) { int32_t exit_point; const void *exit_addr; exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } func = (zend_function*)trace->func; if (func->type == ZEND_USER_FUNCTION && (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || (func->common.fn_flags & ZEND_ACC_CLOSURE) || !func->common.function_name)) { const zend_op *opcodes = func->op_array.opcodes; | LOAD_ADDR TMP1, opcodes | ldr TMP2, [REG0, #offsetof(zend_op_array, opcodes)] | cmp TMP2, TMP1 | bne &exit_addr } else { | LOAD_ADDR TMP1, func | cmp REG0, TMP1 | bne &exit_addr } } if (!func) { | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] | TST_32_WITH_CONST TMP1w, ZEND_ACC_STATIC, TMP2w | bne >1 |.cold_code |1: } if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { | ldr FCARG1x, T1 // restore | mov FCARG2x, REG0 | LOAD_32BIT_VAL CARG3w, opline->extended_value if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, REG0 } else { | EXT_CALL zend_jit_push_static_metod_call_frame, REG0 } if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) { | cbz RETVALx, ->exception_handler } | mov RX, RETVALx } if (!func) { | b >9 |.code } if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, delayed_fetch_this, checked_stack)) { return 0; } } if (!func) { |9: } zend_jit_start_reuse_ip(); if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { if (!zend_jit_save_call_chain(Dst, call_level)) { return 0; } } else { delayed_call_chain = 1; delayed_call_level = call_level; } return 1; } static int zend_jit_init_closure_call(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack) { zend_function *func = NULL; zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); | GET_ZVAL_PTR REG0, op2_addr, TMP1 if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | LOAD_ADDR FCARG1x, ((ptrdiff_t)zend_ce_closure) | ldr, TMP1, [REG0, #offsetof(zend_object, ce)] | cmp TMP1, FCARG1x | bne &exit_addr if (ssa->var_info && ssa_op->op2_use >= 0) { ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; ssa->var_info[ssa_op->op2_use].is_instanceof = 0; } } if (trace && trace->op == ZEND_JIT_TRACE_INIT_CALL && trace->func && trace->func->type == ZEND_USER_FUNCTION) { const zend_op *opcodes; int32_t exit_point; const void *exit_addr; func = (zend_function*)trace->func; opcodes = func->op_array.opcodes; exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | LOAD_ADDR FCARG1x, ((ptrdiff_t)opcodes) | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.opcodes)] | cmp TMP1, FCARG1x | bne &exit_addr } if (delayed_call_chain) { if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { return 0; } } if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, checked_stack)) { return 0; } if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { if (!zend_jit_save_call_chain(Dst, call_level)) { return 0; } } else { delayed_call_chain = 1; delayed_call_level = call_level; } if (trace && trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { if (!zend_jit_set_valid_ip(Dst, opline + 1)) { return 0; } } return 1; } static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; const zend_function *func = NULL; uint32_t i; zend_jit_addr res_addr; uint32_t call_num_args = 0; bool unknown_num_args = 0; const void *exit_addr = NULL; const zend_op *prev_opline; if (RETURN_VALUE_USED(opline)) { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); } else { /* CPU stack allocated temporary zval */ res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RSP, TMP_ZVAL_OFFSET); } prev_opline = opline - 1; while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { prev_opline--; } if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { unknown_num_args = 1; } if (info) { call_info = info->callee_info; while (call_info && call_info->caller_call_opline != opline) { call_info = call_info->next_callee; } if (call_info && call_info->callee_func && !call_info->is_prototype) { func = call_info->callee_func; } if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) && JIT_G(current_frame) && JIT_G(current_frame)->call && !JIT_G(current_frame)->call->func) { call_info = NULL; func = NULL; /* megamorphic call from trait */ } } if (!func) { /* resolve function at run time */ } else if (func->type == ZEND_USER_FUNCTION) { ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); call_num_args = call_info->num_args; } else if (func->type == ZEND_INTERNAL_FUNCTION) { ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); call_num_args = call_info->num_args; } else { ZEND_UNREACHABLE(); } if (trace && !func) { if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); #ifndef ZEND_WIN32 // TODO: ASLR may cause different addresses in different workers ??? func = trace->func; if (JIT_G(current_frame) && JIT_G(current_frame)->call && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); } else { unknown_num_args = 1; } #endif } else if (trace->op == ZEND_JIT_TRACE_ENTER) { ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); if (zend_accel_in_shm(trace->func->op_array.opcodes)) { func = trace->func; if (JIT_G(current_frame) && JIT_G(current_frame)->call && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); } else { unknown_num_args = 1; } } } } bool may_have_extra_named_params = opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); if (!reuse_ip) { zend_jit_start_reuse_ip(); | // call = EX(call); | ldr RX, EX->call } zend_jit_stop_reuse_ip(); | // fbc = call->func; | // mov r2, EX:RX->func ??? | // SAVE_OPLINE(); | SET_EX_OPLINE opline, REG0 if (opline->opcode == ZEND_DO_FCALL) { if (!func) { if (trace) { uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w | bne &exit_addr } } } if (!delayed_call_chain) { if (call_level == 1) { | str xzr, EX->call } else { | //EX(call) = call->prev_execute_data; | ldr REG0, EX:RX->prev_execute_data | str REG0, EX->call } } delayed_call_chain = 0; | //call->prev_execute_data = execute_data; | str EX, EX:RX->prev_execute_data if (!func) { | ldr REG0, EX:RX->func } if (opline->opcode == ZEND_DO_FCALL) { if (!func) { if (!trace) { | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w | bne >1 |.cold_code |1: if (!GCC_GLOBAL_REGS) { | mov FCARG1x, RX } | EXT_CALL zend_jit_deprecated_helper, REG0 | GET_LOW_8BITS RETVALw, RETVALw | ldr REG0, EX:RX->func // reload | cbnz RETVALw, >1 // Result is 0 on exception | b ->exception_handler |.code |1: } } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { if (!GCC_GLOBAL_REGS) { | mov FCARG1x, RX } | EXT_CALL zend_jit_deprecated_helper, REG0 | cbz RETVALw, ->exception_handler } } if (!func && opline->opcode != ZEND_DO_UCALL && opline->opcode != ZEND_DO_ICALL) { | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] | cmp TMP1w, #ZEND_USER_FUNCTION | bne >8 } if ((!func || func->type == ZEND_USER_FUNCTION) && opline->opcode != ZEND_DO_ICALL) { | // EX(call) = NULL; | str xzr, EX:RX->call if (RETURN_VALUE_USED(opline)) { | // EX(return_value) = EX_VAR(opline->result.var); | LOAD_ZVAL_ADDR REG2, res_addr | str REG2, EX:RX->return_value } else { | // EX(return_value) = 0; | str xzr, EX:RX->return_value } //EX_LOAD_RUN_TIME_CACHE(op_array); if (!func || func->op_array.cache_size) { if (func && op_array == &func->op_array) { /* recursive call */ if (trace || func->op_array.cache_size > sizeof(void*)) { | ldr REG2, EX->run_time_cache | str REG2, EX:RX->run_time_cache } } else { // Always defined as ZEND_MAP_PTR_KIND_PTR_OR_OFFSET. See Zend/zend_map_ptr.h. #if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR if (func) { | ldr REG0, EX:RX->func } | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] | ldr REG2, [REG2] #elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { if (ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { | MEM_LOAD_64_ZTS ldr, REG2, compiler_globals, map_ptr_base, TMP1 | ADD_SUB_64_WITH_CONST add, REG2, REG2, (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache), TMP1 | ldr REG2, [REG2] } else if ((func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) && (!func->op_array.scope || (func->op_array.scope->ce_flags & ZEND_ACC_LINKED))) { if (func) { | ldr REG0, EX:RX->func } | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 | ldr REG2, [REG2] } else { /* the called op_array may be not persisted yet */ if (func) { | ldr REG0, EX:RX->func } | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] | TST_64_WITH_ONE REG2 | beq >1 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 |1: | ldr REG2, [REG2] } } else { if (func) { | ldr REG0, EX:RX->func } | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] | TST_64_WITH_ONE REG2 | beq >1 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 |1: | ldr REG2, [REG2] } #else # error "Unknown ZEND_MAP_PTR_KIND" #endif | str REG2, EX:RX->run_time_cache } } | // EG(current_execute_data) = execute_data; | MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1 | mov FP, RX | // opline = op_array->opcodes; if (func && !unknown_num_args) { | ADD_SUB_64_WITH_CONST_32 add, TMP1, RX, (EX_NUM_TO_VAR(call_num_args) + offsetof(zval, u1.type_info)), TMP1 // induction variable for (i = call_num_args; i < func->op_array.last_var; i++) { | // ZVAL_UNDEF(EX_VAR(n)) | str wzr, [TMP1], #16 } if (call_num_args <= func->op_array.num_args) { if (!trace || (trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { uint32_t num_args; if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { if (trace) { num_args = 0; } else if (call_info) { num_args = skip_valid_arguments(op_array, ssa, call_info); } else { num_args = call_num_args; } } else { num_args = call_num_args; } if (zend_accel_in_shm(func->op_array.opcodes)) { | LOAD_IP_ADDR (func->op_array.opcodes + num_args) } else { | ldr REG0, EX->func || ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(num_args * sizeof(zend_op)))); if (GCC_GLOBAL_REGS) { | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] if (num_args) { | add IP, IP, #(num_args * sizeof(zend_op)) } } else { | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] if (num_args) { | add FCARG1x, FCARG1x, #(num_args * sizeof(zend_op)) } | str FCARG1x, EX->opline } } if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array && num_args >= op_array->required_num_args) { /* recursive call */ if (ZEND_OBSERVER_ENABLED) { | SAVE_IP | mov FCARG1x, FP | EXT_CALL zend_observer_fcall_begin, REG0 } #ifdef CONTEXT_THREADED_JIT | NIY // TODO #else | b =>num_args #endif return 1; } } } else { if (!trace || (trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { if (func && zend_accel_in_shm(func->op_array.opcodes)) { | LOAD_IP_ADDR (func->op_array.opcodes) } else if (GCC_GLOBAL_REGS) { | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] } else { | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] | str FCARG1x, EX->opline } } if (!GCC_GLOBAL_REGS) { | mov FCARG1x, FP } | EXT_CALL zend_jit_copy_extra_args_helper, REG0 } } else { | // opline = op_array->opcodes if (func && zend_accel_in_shm(func->op_array.opcodes)) { | LOAD_IP_ADDR (func->op_array.opcodes) } else if (GCC_GLOBAL_REGS) { | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] } else { | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] | str FCARG1x, EX->opline } if (func) { | // num_args = EX_NUM_ARGS(); | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] | // if (UNEXPECTED(num_args > first_extra_arg)) | CMP_32_WITH_CONST REG1w, (func->op_array.num_args), TMP1w } else { | // first_extra_arg = op_array->num_args; | ldr REG2w, [REG0, #offsetof(zend_op_array, num_args)] | // num_args = EX_NUM_ARGS(); | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] | // if (UNEXPECTED(num_args > first_extra_arg)) | cmp REG1w, REG2w } | bgt >1 |.cold_code |1: if (!GCC_GLOBAL_REGS) { | mov FCARG1x, FP } | EXT_CALL zend_jit_copy_extra_args_helper, REG0 if (!func) { | ldr REG0, EX->func // reload } | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] // reload | b >1 |.code if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { if (!func) { | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] | TST_32_WITH_CONST TMP1w, ZEND_ACC_HAS_TYPE_HINTS, TMP2w | bne >1 } | // opline += num_args; || ZEND_ASSERT(sizeof(zend_op) == 32); | mov REG2w, REG1w | ADD_IP_SHIFT REG2, lsl #5, TMP1 } |1: | // if (EXPECTED((int)num_args < op_array->last_var)) { if (func) { | LOAD_32BIT_VAL REG2w, func->op_array.last_var } else { | ldr REG2w, [REG0, #offsetof(zend_op_array, last_var)] } | subs REG2w, REG2w, REG1w | ble >3 | // zval *var = EX_VAR_NUM(num_args); | add REG1, FP, REG1, lsl #4 || ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(ZEND_CALL_FRAME_SLOT * sizeof(zval)))); | add REG1, REG1, #(ZEND_CALL_FRAME_SLOT * sizeof(zval)) |2: | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w | add REG1, REG1, #16 | subs REG2w, REG2w, #1 | bne <2 |3: } if (ZEND_OBSERVER_ENABLED) { | SAVE_IP | mov FCARG1x, FP | EXT_CALL zend_observer_fcall_begin, REG0 } if (trace) { if (!func && (opline->opcode != ZEND_DO_UCALL)) { | b >9 } } else { #ifdef CONTEXT_THREADED_JIT | NIY // TODO: CONTEXT_THREADED_JIT is always undefined. #else if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD | JMP_IP TMP1 } else if (GCC_GLOBAL_REGS) { | ldp x29, x30, [sp], # SPAD // stack alignment | JMP_IP TMP1 } else { | ldp FP, RX, T2 // retore FP and IP | ldp x29, x30, [sp], # NR_SPAD // stack alignment | mov RETVALx, #1 // ZEND_VM_ENTER | ret } } #endif } if ((!func || func->type == ZEND_INTERNAL_FUNCTION) && (opline->opcode != ZEND_DO_UCALL)) { if (!func && (opline->opcode != ZEND_DO_ICALL)) { |8: } if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { if (!func) { if (trace) { uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w | bne &exit_addr } else { | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w | bne >1 |.cold_code |1: if (!GCC_GLOBAL_REGS) { | mov FCARG1x, RX } | EXT_CALL zend_jit_deprecated_helper, REG0 | GET_LOW_8BITS RETVALw, RETVALw | ldr REG0, EX:RX->func // reload | cbnz RETVALw, >1 // Result is 0 on exception | b ->exception_handler |.code |1: } } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { if (!GCC_GLOBAL_REGS) { | mov FCARG1x, RX } | EXT_CALL zend_jit_deprecated_helper, REG0 | cbz RETVALw, ->exception_handler | ldr REG0, EX:RX->func // reload } } | // ZVAL_NULL(EX_VAR(opline->result.var)); | LOAD_ZVAL_ADDR FCARG2x, res_addr | SET_Z_TYPE_INFO FCARG2x, IS_NULL, TMP1w | // EG(current_execute_data) = execute_data; | MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1 zend_jit_reset_last_valid_opline(); | // (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret); | mov FCARG1x, RX if (zend_execute_internal) { | EXT_CALL zend_execute_internal, REG0 } else { if (func) { | EXT_CALL func->internal_function.handler, REG0 } else { | ldr TMP1, [REG0, #offsetof(zend_internal_function, handler)] | blr TMP1 } } | // EG(current_execute_data) = execute_data; | MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0 | // zend_vm_stack_free_args(call); if (func && !unknown_num_args) { for (i = 0; i < call_num_args; i++ ) { if (zend_jit_needs_arg_dtor(func, i, call_info)) { uint32_t offset = EX_NUM_TO_VAR(i); zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset); | ZVAL_PTR_DTOR arg_addr, (MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN), 0, 1, opline, ZREG_TMP1, ZREG_TMP2 } } } else { | mov FCARG1x, RX | EXT_CALL zend_jit_vm_stack_free_args_helper, REG0 } if (may_have_extra_named_params) { | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] | TST_32_WITH_CONST TMP1w, (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24), TMP2w | bne >1 |.cold_code |1: | ldr FCARG1x, [RX, #offsetof(zend_execute_data, extra_named_params)] | EXT_CALL zend_free_extra_named_params, REG0 | b >2 |.code |2: } |8: if (opline->opcode == ZEND_DO_FCALL) { // TODO: optimize ??? | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] | TST_32_WITH_CONST TMP1w, (ZEND_CALL_RELEASE_THIS >> 16), TMP2w | bne >1 |.cold_code |1: | add TMP1, RX, #offsetof(zend_execute_data, This) | GET_Z_PTR FCARG1x, TMP1 | // OBJ_RELEASE(object); | OBJ_RELEASE ZREG_FCARG1, >2, ZREG_TMP1, ZREG_TMP2 | b >2 |.code |2: } if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || !JIT_G(current_frame)->call || !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { | // zend_vm_stack_free_call_frame(call); | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] | TST_32_WITH_CONST TMP1w, ((ZEND_CALL_ALLOCATED >> 16) & 0xff), TMP2w | bne >1 |.cold_code |1: | mov FCARG1x, RX | EXT_CALL zend_jit_free_call_frame, REG0 | b >1 |.code } | MEM_STORE_64_ZTS str, RX, executor_globals, vm_stack_top, REG0 |1: if (!RETURN_VALUE_USED(opline)) { zend_class_entry *ce; bool ce_is_instanceof; uint32_t func_info = call_info ? zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); /* If an exception is thrown, the return_value may stay at the * original value of null. */ func_info |= MAY_BE_NULL; if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline, ZREG_TMP1, ZREG_TMP2 } } | // if (UNEXPECTED(EG(exception) != NULL)) { | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 | cbnz REG0, ->icall_throw_handler // TODO: Can we avoid checking for interrupts after each call ??? if (trace && last_valid_opline != opline) { int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } else { exit_addr = NULL; } if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { return 0; } if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { | LOAD_IP_ADDR (opline + 1) } else if (trace && trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { | LOAD_IP_ADDR (opline + 1) } } if (!func) { |9: } return 1; } static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) { uint32_t arg_num = opline->op2.num; zend_jit_addr arg_addr; ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); if (!zend_jit_reuse_ip(Dst)) { return 0; } if (opline->opcode == ZEND_SEND_VAL_EX) { uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { /* Don't generate code that always throws exception */ return 0; } } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] | TST_32_WITH_CONST TMP1w, mask, TMP2w | bne &exit_addr } else { | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] | TST_32_WITH_CONST TMP1w, mask, TMP2w | bne >1 |.cold_code |1: if (Z_MODE(op1_addr) == IS_REG) { /* set type to avoid zval_ptr_dtor() on uninitialized value */ zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); | SET_ZVAL_TYPE_INFO addr, IS_UNDEF, TMP1w, TMP2 } | SET_EX_OPLINE opline, REG0 | b ->throw_cannot_pass_by_ref |.code } } arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, REG0, TMP1 } } else { | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } return 1; } static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) { | ldr FCARG1x, EX->call | ldrb TMP1w, [FCARG1x, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] | TST_32_WITH_CONST TMP1w, (ZEND_CALL_MAY_HAVE_UNDEF >> 24), TMP2w | bne >1 |.cold_code |1: | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_handle_undef_args, REG0 | cbz RETVALw, >2 | b ->exception_handler |.code |2: return 1; } static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) { zend_jit_addr op1_addr, arg_addr, ref_addr; op1_addr = OP1_ADDR(); arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (!zend_jit_reuse_ip(Dst)) { return 0; } if (opline->op1_type == IS_VAR) { if (op1_info & MAY_BE_INDIRECT) { | LOAD_ZVAL_ADDR REG0, op1_addr | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { | IF_NOT_Z_TYPE REG0, IS_INDIRECT, >1, TMP1w | // ret = Z_INDIRECT_P(ret); | GET_Z_PTR REG0, REG0 |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); } } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 | b >2 |1: } op1_info &= ~MAY_BE_UNDEF; op1_info |= MAY_BE_NULL; } } else { ZEND_UNREACHABLE(); } if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { if (op1_info & MAY_BE_REF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2, ZREG_TMP1 | GET_ZVAL_PTR REG1, op1_addr, TMP1 | GC_ADDREF REG1, TMP1w | SET_ZVAL_PTR arg_addr, REG1, TMP1 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 | b >6 } |2: | // ZVAL_NEW_REF(arg, varptr); if (opline->op1_type == IS_VAR) { if (Z_REG(op1_addr) != ZREG_REG0 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR REG0, op1_addr } | str REG0, T1 // save } | EMALLOC sizeof(zend_reference), op_array, opline // Allocate space in REG0 | mov TMP1w, #2 | str TMP1w, [REG0] || ZEND_ASSERT(GC_REFERENCE <= MOVZ_IMM); | movz TMP1w, #GC_REFERENCE | str TMP1w, [REG0, #offsetof(zend_reference, gc.u.type_info)] | str xzr, [REG0, #offsetof(zend_reference, sources.ptr)] ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); if (opline->op1_type == IS_VAR) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); | ldr REG1, T1 // restore | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | SET_ZVAL_PTR val_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX, TMP1w, TMP2 } else { | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | SET_ZVAL_PTR op1_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 } | SET_ZVAL_PTR arg_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 } |6: | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline, ZREG_TMP1, ZREG_TMP2 |7: return 1; } static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) { uint32_t arg_num = opline->op2.num; zend_jit_addr arg_addr; ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || arg_num <= MAX_ARG_FLAG_NUM); arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (!zend_jit_reuse_ip(Dst)) { return 0; } if (opline->opcode == ZEND_SEND_VAR_EX) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { return 0; } return 1; } } else { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] | TST_32_WITH_CONST TMP1w, mask, TMP2w | bne >1 |.cold_code |1: if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { return 0; } | b >7 |.code } } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { if (!(op1_info & MAY_BE_REF)) { /* Don't generate code that always throws exception */ return 0; } else { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | GET_LOW_8BITS TMP1w, REG1w | cmp TMP1w, #IS_REFERENCE | bne &exit_addr } } return 1; } } else { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] | TST_32_WITH_CONST TMP1w, mask, TMP2w | bne >1 |.cold_code |1: mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (op1_info & MAY_BE_REF) { | GET_LOW_8BITS TMP1w, REG1w | cmp TMP1w, #IS_REFERENCE | beq >7 } | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] | TST_32_WITH_CONST TMP1w, mask, TMP2w | bne >7 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | b &exit_addr } else { | SET_EX_OPLINE opline, REG0 | LOAD_ZVAL_ADDR FCARG1x, arg_addr | EXT_CALL zend_jit_only_vars_by_reference, REG0 if (!zend_jit_check_exception(Dst)) { return 0; } | b >7 } |.code } } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { return 0; } return 1; } } else { | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] | TST_32_WITH_CONST TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w | bne >1 |.cold_code |1: if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { return 0; } | b >7 |.code } } if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 |.cold_code |1: } | SET_EX_OPLINE opline, REG0 | LOAD_32BIT_VAL FCARG1w, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL, TMP1w, TMP2 | cbz RETVALx, ->exception_handler if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { | b >7 |.code } else { |7: return 1; } } if (opline->opcode == ZEND_SEND_VAR_NO_REF) { | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (op1_info & MAY_BE_REF) { | GET_LOW_8BITS TMP1w, REG1w | cmp TMP1w, #IS_REFERENCE | beq >7 } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | b &exit_addr } else { | SET_EX_OPLINE opline, REG0 | LOAD_ZVAL_ADDR FCARG1x, arg_addr | EXT_CALL zend_jit_only_vars_by_reference, REG0 if (!zend_jit_check_exception(Dst)) { return 0; } } } else { if (op1_info & MAY_BE_REF) { if (opline->op1_type == IS_CV) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); | LOAD_ZVAL_ADDR FCARG1x, op1_addr | ZVAL_DEREF FCARG1x, op1_info, TMP1w | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | TRY_ADDREF op1_info, REG0w, REG2, TMP1w } else { zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 8); | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 |.cold_code |1: | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 | // ZVAL_COPY_VALUE(return_value, &ref->value); | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | GC_DELREF FCARG1x, TMP1w | beq >1 | IF_NOT_REFCOUNTED REG0w, >2, TMP1w | GC_ADDREF REG2, TMP1w | b >2 |1: | EFREE_REFERENCE | b >2 |.code | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 |2: } } else { if (op1_addr != op1_def_addr) { if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { return 0; } if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { op1_addr= op1_def_addr; } } | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (opline->op1_type == IS_CV) { | TRY_ADDREF op1_info, REG0w, REG2, TMP1w } } } |7: return 1; } static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) { uint32_t arg_num = opline->op2.num; if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); || if (reuse_ip) { | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] || } else { | ldr REG0, EX->call | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] || } } } else { if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); || if (reuse_ip) { | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] || } else { | ldr REG0, EX->call | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] || } } } } else { // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); if (!zend_jit_reuse_ip(Dst)) { return 0; } | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] | TST_32_WITH_CONST TMP1w, mask, TMP2w | bne >1 |.cold_code |1: | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] | b >1 |.code | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~(ZEND_CALL_SEND_ARG_BY_REF)), TMP2w | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] |1: } return 1; } static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) { if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { if (jmp) { | b >7 } } else if (smart_branch_opcode == ZEND_JMPNZ) { | b =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | b =>target_label2 } else { ZEND_UNREACHABLE(); } } else { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 if (jmp) { | b >7 } } return 1; } static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) { if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | b =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { if (jmp) { | b >7 } } else if (smart_branch_opcode == ZEND_JMPZNZ) { | b =>target_label } else { ZEND_UNREACHABLE(); } } else { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 if (jmp) { | b >7 } } return 1; } static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { uint32_t defined_label = (uint32_t)-1; uint32_t undefined_label = (uint32_t)-1; zval *zv = RT_CONSTANT(opline, opline->op1); zend_jit_addr res_addr = 0; if (smart_branch_opcode && !exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { undefined_label = target_label; } else if (smart_branch_opcode == ZEND_JMPNZ) { defined_label = target_label; } else if (smart_branch_opcode == ZEND_JMPZNZ) { undefined_label = target_label; defined_label = target_label2; } else { ZEND_UNREACHABLE(); } } | // if (CACHED_PTR(opline->extended_value)) { | ldr REG0, EX->run_time_cache | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, opline->extended_value, TMP1 | cbz REG0, >1 | TST_64_WITH_ONE REG0 | bne >4 |.cold_code |4: | MEM_LOAD_64_ZTS ldr, FCARG1x, executor_globals, zend_constants, FCARG1x | ldr TMP1w, [FCARG1x, #offsetof(HashTable, nNumOfElements)] | cmp TMP1, REG0, lsr #1 if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | beq &exit_addr } else { | beq >3 } } else if (undefined_label != (uint32_t)-1) { | beq =>undefined_label } else { | beq >3 } } else { | beq >2 } |1: | SET_EX_OPLINE opline, REG0 | LOAD_ADDR FCARG1x, zv | EXT_CALL zend_jit_check_constant, REG0 if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | cbz RETVALx, >3 } else { | cbnz RETVALx, >3 } | b &exit_addr } else if (smart_branch_opcode) { if (undefined_label != (uint32_t)-1) { | cbz RETVALx, =>undefined_label } else { | cbz RETVALx, >3 } if (defined_label != (uint32_t)-1) { | b =>defined_label } else { | b >3 } } else { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); | cbnz RETVALx, >1 |2: | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 | b >3 } |.code if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | b &exit_addr } } else if (defined_label != (uint32_t)-1) { | b =>defined_label } } else { |1: | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } |3: return 1; } static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { uint32_t mask; zend_jit_addr op1_addr = OP1_ADDR(); // TODO: support for is_resource() ??? ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 |.cold_code |1: } | SET_EX_OPLINE opline, REG0 | LOAD_32BIT_VAL FCARG1w, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 zend_jit_check_exception_undef_result(Dst, opline); if (opline->extended_value & MAY_BE_NULL) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | b &exit_addr } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { | b >7 } } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { return 0; } } else { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | b &exit_addr } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { | b >7 } } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { return 0; } } if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { |.code } } if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { mask = opline->extended_value; if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | b &exit_addr } } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { return 0; } } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | b &exit_addr } } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { return 0; } } else { bool invert = 0; zend_uchar type; switch (mask) { case MAY_BE_NULL: type = IS_NULL; break; case MAY_BE_FALSE: type = IS_FALSE; break; case MAY_BE_TRUE: type = IS_TRUE; break; case MAY_BE_LONG: type = IS_LONG; break; case MAY_BE_DOUBLE: type = IS_DOUBLE; break; case MAY_BE_STRING: type = IS_STRING; break; case MAY_BE_ARRAY: type = IS_ARRAY; break; case MAY_BE_OBJECT: type = IS_OBJECT; break; case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break; case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break; case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break; case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break; case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break; case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break; case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break; case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break; case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break; default: type = 0; } if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR REG0, op1_addr | ZVAL_DEREF REG0, op1_info, TMP1w } if (type == 0) { if (smart_branch_opcode && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 |.cold_code |1: } | // if (!Z_DELREF_P(cv)) { | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 | GC_DELREF FCARG1x, TMP1w if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { | bne >3 } if (op1_info & MAY_BE_REF) { | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] } else { | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 } | str REG0w, T1 // save | // zval_dtor_func(r); | ZVAL_DTOR_FUNC op1_info, opline, TMP1 | ldr REG1w, T1 // restore | b >2 } if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (!RC_MAY_BE_1(op1_info)) { | b >3 } |.code } |3: if (op1_info & MAY_BE_REF) { | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] } else { | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 } |2: } else { if (op1_info & MAY_BE_REF) { | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] } else { | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 } } | mov REG0w, #1 | lsl REG0w, REG0w, REG1w | TST_32_WITH_CONST REG0w, mask, TMP1w if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | bne &exit_addr } else { | beq &exit_addr } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | beq =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { | bne =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | beq =>target_label | b =>target_label2 } else { ZEND_UNREACHABLE(); } } else { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); | cset REG0w, ne | add REG0w, REG0w, #2 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } } else { if (smart_branch_opcode && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 |.cold_code |1: } | // if (!Z_DELREF_P(cv)) { | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 | GC_DELREF FCARG1x, TMP1w if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { | bne >3 } if (op1_info & MAY_BE_REF) { | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] } else { | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 } | str REG0w, T1 // save | // zval_dtor_func(r); | ZVAL_DTOR_FUNC op1_info, opline, TMP1 | ldr REG1w, T1 // restore | b >2 } if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (!RC_MAY_BE_1(op1_info)) { | b >3 } |.code } |3: if (op1_info & MAY_BE_REF) { | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] } else { | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 } |2: // Note: 'type' is of uchar type and holds a positive value, // hence it's safe to directly encode it as the imm field of 'cmp' instruction. | cmp REG1w, #type } else { if (op1_info & MAY_BE_REF) { | ldrb TMP1w, [REG0, #offsetof(zval,u1.v.type)] | cmp TMP1w, #type } else { | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 | cmp TMP1w, #type } } if (exit_addr) { if (invert) { if (smart_branch_opcode == ZEND_JMPNZ) { | bne &exit_addr } else { | beq &exit_addr } } else { if (smart_branch_opcode == ZEND_JMPNZ) { | beq &exit_addr } else { | bne &exit_addr } } } else if (smart_branch_opcode) { if (invert) { if (smart_branch_opcode == ZEND_JMPZ) { | beq =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { | bne =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | beq =>target_label | b =>target_label2 } else { ZEND_UNREACHABLE(); } } else { if (smart_branch_opcode == ZEND_JMPZ) { | bne =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { | beq =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | bne =>target_label | b =>target_label2 } else { ZEND_UNREACHABLE(); } } } else { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (invert) { | cset REG0w, ne } else { | cset REG0w, eq } | add REG0w, REG0w, #2 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } } } } |7: return 1; } static int zend_jit_leave_frame(dasm_State **Dst) { | // EG(current_execute_data) = EX(prev_execute_data); | ldr REG0, EX->prev_execute_data | MEM_STORE_64_ZTS str, REG0, executor_globals, current_execute_data, REG2 return 1; } static int zend_jit_free_cvs(dasm_State **Dst) { | // EG(current_execute_data) = EX(prev_execute_data); | ldr FCARG1x, EX->prev_execute_data | MEM_STORE_64_ZTS str, FCARG1x, executor_globals, current_execute_data, REG0 | // zend_free_compiled_variables(execute_data); | mov FCARG1x, FP | EXT_CALL zend_free_compiled_variables, REG0 return 1; } static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) { if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { uint32_t offset = EX_NUM_TO_VAR(var); zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset); | ZVAL_PTR_DTOR addr, info, 1, 1, NULL, ZREG_TMP1, ZREG_TMP2 } return 1; } static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) { if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset); | ZVAL_PTR_DTOR addr, info, 0, 1, opline, ZREG_TMP1, ZREG_TMP2 } return 1; } static int zend_jit_leave_func(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, uint32_t op1_info, bool left_frame, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info, int indirect_var_access, int may_throw) { bool may_be_top_frame = JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); bool may_need_call_helper = indirect_var_access || /* may have symbol table */ !op_array->function_name || /* may have symbol table */ may_be_top_frame || (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ bool may_need_release_this = !(op_array->fn_flags & ZEND_ACC_CLOSURE) && op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC) && (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame))); if (may_need_call_helper || may_need_release_this) { | ldr FCARG1w, [FP, #offsetof(zend_execute_data, This.u1.type_info)] } if (may_need_call_helper) { if (!left_frame) { left_frame = 1; if (!zend_jit_leave_frame(Dst)) { return 0; } } /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ | TST_32_WITH_CONST FCARG1w, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE), TMP1w if (trace && trace->op != ZEND_JIT_TRACE_END) { | bne >1 |.cold_code |1: if (!GCC_GLOBAL_REGS) { | mov FCARG1x, FP } | EXT_CALL zend_jit_leave_func_helper, REG0 if (may_be_top_frame) { // TODO: try to avoid this check ??? if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { #if 0 /* this check should be handled by the following OPLINE guard */ | LOAD_ADDR TMP1, zend_jit_halt_op | cmp IP, TMP1 | beq ->trace_halt #endif } else if (GCC_GLOBAL_REGS) { | cbz IP, ->trace_halt } else { | tst RETVALw, RETVALw | blt ->trace_halt } } if (!GCC_GLOBAL_REGS) { | // execute_data = EG(current_execute_data) | MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 } | b >8 |.code } else { | bne ->leave_function_handler } } if (op_array->fn_flags & ZEND_ACC_CLOSURE) { if (!left_frame) { left_frame = 1; if (!zend_jit_leave_frame(Dst)) { return 0; } } | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); | ldr FCARG1x, EX->func | sub FCARG1x, FCARG1x, #sizeof(zend_object) | OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2 |4: } else if (may_need_release_this) { if (!left_frame) { left_frame = 1; if (!zend_jit_leave_frame(Dst)) { return 0; } } | // if (call_info & ZEND_CALL_RELEASE_THIS) | TST_32_WITH_CONST FCARG1w, ZEND_CALL_RELEASE_THIS, TMP1w | beq >4 | // zend_object *object = Z_OBJ(execute_data->This); | ldr FCARG1x, EX->This.value.obj | // OBJ_RELEASE(object); | OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2 |4: // TODO: avoid EG(excption) check for $this->foo() calls may_throw = 1; } | // EG(vm_stack_top) = (zval*)execute_data; | MEM_STORE_64_ZTS str, FP, executor_globals, vm_stack_top, REG0 | // execute_data = EX(prev_execute_data); | ldr FP, EX->prev_execute_data if (!left_frame) { | // EG(current_execute_data) = execute_data; | MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0 } |9: if (trace) { if (trace->op != ZEND_JIT_TRACE_END && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { zend_jit_reset_last_valid_opline(); } else { | LOAD_IP | ADD_IP_WITH_CONST sizeof(zend_op), TMP1 } |8: if (trace->op == ZEND_JIT_TRACE_BACK && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { const zend_op *next_opline = trace->opline; if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & MAY_BE_RC1) && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { /* exception might be thrown during destruction of unused return value */ | // if (EG(exception)) | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 | cbnz REG0, ->leave_throw_handler } do { trace++; } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); next_opline = trace->opline; ZEND_ASSERT(next_opline != NULL); if (trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { trace_info->flags |= ZEND_JIT_TRACE_LOOP; | CMP_IP next_opline, TMP1, TMP2 | beq =>0 // LOOP #ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | JMP_IP TMP1 #else | b ->trace_escape #endif } else { | CMP_IP next_opline, TMP1, TMP2 | bne ->trace_escape } zend_jit_set_last_valid_opline(trace->opline); return 1; } else if (may_throw || (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & MAY_BE_RC1) && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { | // if (EG(exception)) | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 | cbnz REG0, ->leave_throw_handler } return 1; } else { | // if (EG(exception)) | MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1 | LOAD_IP | cbnz REG0, ->leave_throw_handler | // opline = EX(opline) + 1 | ADD_IP_WITH_CONST sizeof(zend_op), TMP1 } if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD #ifdef CONTEXT_THREADED_JIT | NIY // TODO: CONTEXT_THREADED_JIT is always undefined #else | JMP_IP TMP1 #endif } else if (GCC_GLOBAL_REGS) { | ldp x29, x30, [sp], # SPAD // stack alignment #ifdef CONTEXT_THREADED_JIT | NIY // TODO #else | JMP_IP TMP1 #endif } else { #ifdef CONTEXT_THREADED_JIT ZEND_UNREACHABLE(); // TODO: context threading can't work without GLOBAL REGS because we have to change // the value of execute_data in execute_ex() | NIY // TODO #else | ldp FP, RX, T2 // retore FP and IP | ldp x29, x30, [sp], # NR_SPAD // stack alignment | mov RETVALx, #2 // ZEND_VM_LEAVE ???? | ret #endif } return 1; } static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) { zend_jit_addr ret_addr; int8_t return_value_used; ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { return_value_used = 1; } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { return_value_used = 0; } else { return_value_used = -1; } } else { return_value_used = -1; } if (ZEND_OBSERVER_ENABLED) { if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { return 0; } op1_addr = dst; } | LOAD_ZVAL_ADDR FCARG2x, op1_addr | mov FCARG1x, FP | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_observer_fcall_end, REG0 } // if (!EX(return_value)) if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_REG1) { if (return_value_used != 0) { | ldr REG2, EX->return_value } ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); } else { if (return_value_used != 0) { | ldr REG1, EX->return_value } ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); } if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (return_value_used == -1) { | cbz Rx(Z_REG(ret_addr)), >1 |.cold_code |1: } if (return_value_used != 1) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (jit_return_label >= 0) { | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label, ZREG_TMP1, ZREG_TMP2 } else { | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9, ZREG_TMP1, ZREG_TMP2 } } | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 | GC_DELREF FCARG1x, TMP1w if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { if (jit_return_label >= 0) { | bne =>jit_return_label } else { | bne >9 } } | //SAVE_OPLINE() | ZVAL_DTOR_FUNC op1_info, opline, TMP1 | //????ldr REG1, EX->return_value // reload ??? } if (return_value_used == -1) { if (jit_return_label >= 0) { | b =>jit_return_label } else { | b >9 } |.code } } } else if (return_value_used == -1) { if (jit_return_label >= 0) { | cbz Rx(Z_REG(ret_addr)), =>jit_return_label } else { | cbz Rx(Z_REG(ret_addr)), >9 } } if (return_value_used == 0) { |9: return 1; } if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, REG0, TMP1 } } else if (opline->op1_type == IS_TMP_VAR) { | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR REG0, op1_addr | ZVAL_DEREF REG0, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); } | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || !op_array->function_name) { | TRY_ADDREF op1_info, REG0w, REG2, TMP1w } else if (return_value_used != 1) { | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 } } } else { if (op1_info & MAY_BE_REF) { zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 |.cold_code |1: | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); | GET_ZVAL_PTR REG0, op1_addr, TMP1 | // ZVAL_COPY_VALUE(return_value, &ref->value); | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | GC_DELREF REG0, TMP1w | beq >2 | // if (IS_REFCOUNTED()) if (jit_return_label >= 0) { | IF_NOT_REFCOUNTED REG2w, =>jit_return_label, TMP1w } else { | IF_NOT_REFCOUNTED REG2w, >9, TMP1w } | // ADDREF | GET_ZVAL_PTR REG2, ret_addr, TMP1 // reload | GC_ADDREF REG2, TMP1w if (jit_return_label >= 0) { | b =>jit_return_label } else { | b >9 } |2: | mov FCARG1x, REG0 | EFREE_REFERENCE if (jit_return_label >= 0) { | b =>jit_return_label } else { | b >9 } |.code } | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } |9: return 1; } static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) { ZEND_ASSERT(type_reg == ZREG_REG2); | GET_ZVAL_PTR REG1, val_addr, TMP1 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w | GET_LOW_8BITS TMP2w, REG2w | IF_NOT_TYPE TMP2w, IS_REFERENCE, >1 | add REG1, REG1, #offsetof(zend_reference, val) | GET_Z_TYPE_INFO REG2w, REG1 | GET_Z_PTR REG1, REG1 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w |1: | GC_ADDREF REG1, TMP2w |2: | SET_ZVAL_PTR res_addr, REG1, TMP1 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 return 1; } static int zend_jit_fetch_dim_read(dasm_State **Dst, const zend_op *opline, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, bool op1_avoid_refcounting, uint32_t op2_info, uint32_t res_info, zend_jit_addr res_addr, uint8_t dim_type) { zend_jit_addr orig_op1_addr, op2_addr; const void *exit_addr = NULL; const void *not_found_exit_addr = NULL; const void *res_exit_addr = NULL; bool result_avoid_refcounting = 0; uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; int may_throw = 0; orig_op1_addr = OP1_ADDR(); op2_addr = OP2_ADDR(); if (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { uint32_t flags = 0; uint32_t old_op1_info = 0; uint32_t old_info; zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; int32_t exit_point; if (opline->opcode != ZEND_FETCH_LIST_R && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !op1_avoid_refcounting) { flags |= ZEND_JIT_EXIT_FREE_OP1; } if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { flags |= ZEND_JIT_EXIT_FREE_OP2; } if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) && !(flags & ZEND_JIT_EXIT_FREE_OP1) && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) && (ssa_op+1)->op1_use == ssa_op->result_def && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) && zend_jit_may_avoid_refcounting(opline+1, res_info)) { result_avoid_refcounting = 1; ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; } if (op1_avoid_refcounting) { old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); } if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); exit_point = zend_jit_trace_get_exit_point(opline+1, flags); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!res_exit_addr) { return 0; } res_info &= ~MAY_BE_GUARD; ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; } if (opline->opcode == ZEND_FETCH_DIM_IS && !(res_info & MAY_BE_NULL)) { old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); exit_point = zend_jit_trace_get_exit_point(opline+1, flags); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!not_found_exit_addr) { return 0; } } if (op1_avoid_refcounting) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); } } if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr | ZVAL_DEREF FCARG1x, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } } | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) || (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) { may_throw = 1; } if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, dim_type, res_exit_addr, not_found_exit_addr, exit_addr)) { return 0; } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { if (op1_info & MAY_BE_ARRAY) { |.cold_code |7: } if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { may_throw = 1; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 } } | SET_EX_OPLINE opline, REG0 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 if (opline->opcode != ZEND_FETCH_DIM_IS) { if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1 | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, REG0 } else { | LOAD_ZVAL_ADDR FCARG2x, op2_addr | EXT_CALL zend_jit_fetch_dim_str_r_helper, REG0 } | SET_ZVAL_PTR res_addr, RETVALx, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 } else { | LOAD_ZVAL_ADDR FCARG2x, op2_addr | LOAD_ZVAL_ADDR CARG3, res_addr | EXT_CALL zend_jit_fetch_dim_str_is_helper, REG0 } if ((op1_info & MAY_BE_ARRAY) || (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { | b >9 // END } |6: } if (op1_info & MAY_BE_OBJECT) { may_throw = 1; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { if (exit_addr) { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6, ZREG_TMP1 } } | SET_EX_OPLINE opline, REG0 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) } else { | LOAD_ZVAL_ADDR FCARG2x, op2_addr } | LOAD_ZVAL_ADDR CARG3, res_addr if (opline->opcode != ZEND_FETCH_DIM_IS) { | EXT_CALL zend_jit_fetch_dim_obj_r_helper, REG0 } else { | EXT_CALL zend_jit_fetch_dim_obj_is_helper, REG0 } if ((op1_info & MAY_BE_ARRAY) || (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { | b >9 // END } |6: } if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { | SET_EX_OPLINE opline, REG0 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { may_throw = 1; | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | LOAD_32BIT_VAL FCARG1w, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 |1: } if (op2_info & MAY_BE_UNDEF) { may_throw = 1; | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 | LOAD_32BIT_VAL FCARG1w, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, REG0 |1: } } if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { may_throw = 1; if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr } else { | SET_EX_OPLINE opline, REG0 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } } | EXT_CALL zend_jit_invalid_array_access, REG0 } | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 if (op1_info & MAY_BE_ARRAY) { | b >9 // END } } if (op1_info & MAY_BE_ARRAY) { |.code } } if (op1_info & MAY_BE_ARRAY) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); |8: if (res_exit_addr) { uint32_t type = concrete_type(res_info); if ((op1_info & MAY_BE_ARRAY_OF_REF) && dim_type != IS_UNKNOWN && dim_type != IS_REFERENCE) { if (type < IS_STRING) { | IF_NOT_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1 |.cold_code |1: | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, &res_exit_addr, ZREG_TMP1 | GET_Z_PTR REG0, REG0 | add REG0, REG0, #offsetof(zend_reference, val) | IF_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1 | b &res_exit_addr |.code |1: } else { | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 | GET_LOW_8BITS TMP1w, REG2w | IF_NOT_TYPE TMP1w, type, >1 |.cold_code |1: | IF_NOT_TYPE TMP1w, IS_REFERENCE, &res_exit_addr | GET_Z_PTR REG0, REG0 | add REG0, REG0, #offsetof(zend_reference, val) | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 | GET_LOW_8BITS TMP1w, REG2w | IF_TYPE TMP1w, type, >1 | b &res_exit_addr |.code |1: } } else { if (op1_info & MAY_BE_ARRAY_OF_REF) { | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w } if (type < IS_STRING) { | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr, ZREG_TMP1 } else { | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 | GET_LOW_8BITS TMP1w, REG2w | IF_NOT_TYPE TMP1w, type, &res_exit_addr } } | // ZVAL_COPY |7: | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (type < IS_STRING) { if (Z_REG(res_addr) != ZREG_FP || JIT_G(current_frame) == NULL || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 } } else { | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 if (!result_avoid_refcounting) { | TRY_ADDREF res_info, REG2w, REG1, TMP1w } } } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { return 0; } } else if (op1_info & MAY_BE_ARRAY_OF_REF) { | // ZVAL_COPY_DEREF | GET_ZVAL_TYPE_INFO Rw(ZREG_REG2), val_addr, TMP1 if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_REG2)) { return 0; } } else { | // ZVAL_COPY | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | TRY_ADDREF res_info, REG1w, REG2, TMP1w } } |9: // END #ifdef ZEND_JIT_USE_RC_INFERENCE if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { /* Magic offsetGet() may increase refcount of the key */ op2_info |= MAY_BE_RCN; } #endif if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) { if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) { may_throw = 1; } | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 } if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) { if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { may_throw = 1; } | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 } } if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_fetch_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr res_addr, uint8_t dim_type) { zend_jit_addr op2_addr; int may_throw = 0; op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; if (opline->opcode == ZEND_FETCH_DIM_RW) { | SET_EX_OPLINE opline, REG0 } if (op1_info & MAY_BE_REF) { may_throw = 1; | LOAD_ZVAL_ADDR FCARG1x, op1_addr | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w | GET_Z_PTR FCARG2x, FCARG1x | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] | cmp TMP1w, #IS_ARRAY | bne >2 | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) | b >3 |.cold_code |2: | SET_EX_OPLINE opline, REG0 if (opline->opcode != ZEND_FETCH_DIM_RW) { | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 } | mov FCARG1x, RETVALx | cbnz FCARG1x, >1 | b ->exception_handler_undef |.code |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } |3: | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { if (op1_info & MAY_BE_ARRAY) { |.cold_code |7: } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { | CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1 | bgt >7 } if (Z_REG(op1_addr) != ZREG_FP) { | str Rx(Z_REG(op1_addr)), T1 // save } if ((op1_info & MAY_BE_UNDEF) && opline->opcode == ZEND_FETCH_DIM_RW) { may_throw = 1; if (op1_info & MAY_BE_NULL) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 } | LOAD_32BIT_VAL FCARG1w, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 |1: } | // ZVAL_ARR(container, zend_new_array(8)); | EXT_CALL _zend_new_array_0, REG0 | mov REG0, RETVALx if (Z_REG(op1_addr) != ZREG_FP) { | ldr Rx(Z_REG(op1_addr)), T1 // restore } | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 | mov FCARG1x, REG0 if (op1_info & MAY_BE_ARRAY) { | b >1 |.code |1: } } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { |6: if (opline->op2_type == IS_UNUSED) { may_throw = 1; | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval | EXT_CALL zend_hash_next_index_insert, REG0 | // if (UNEXPECTED(!var_ptr)) { | cbz RETVALx, >1 |.cold_code |1: | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); | CANNOT_ADD_ELEMENT opline | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); | b >8 |.code | SET_ZVAL_PTR res_addr, RETVALx, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 } else { uint32_t type; switch (opline->opcode) { case ZEND_FETCH_DIM_W: case ZEND_FETCH_LIST_W: type = BP_VAR_W; break; case ZEND_FETCH_DIM_RW: may_throw = 1; type = BP_VAR_RW; break; case ZEND_FETCH_DIM_UNSET: type = BP_VAR_UNSET; break; default: ZEND_UNREACHABLE(); } if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { may_throw = 1; } if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { return 0; } |8: | SET_ZVAL_PTR res_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { |.cold_code |9: | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 | b >8 |.code } } } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { may_throw = 1; if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { |.cold_code |7: } if (opline->opcode != ZEND_FETCH_DIM_RW) { | SET_EX_OPLINE opline, REG0 } if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } if (opline->op2_type == IS_UNUSED) { | mov FCARG2x, xzr } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) } else { | LOAD_ZVAL_ADDR FCARG2x, op2_addr } | LOAD_ZVAL_ADDR CARG3, res_addr switch (opline->opcode) { case ZEND_FETCH_DIM_W: case ZEND_FETCH_LIST_W: | EXT_CALL zend_jit_fetch_dim_obj_w_helper, REG0 break; case ZEND_FETCH_DIM_RW: | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, REG0 break; // case ZEND_FETCH_DIM_UNSET: // | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, REG0 // break; default: ZEND_UNREACHABLE(); } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { | b >8 // END |.code } } #ifdef ZEND_JIT_USE_RC_INFERENCE if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { /* ASSIGN_DIM may increase refcount of the key */ op2_info |= MAY_BE_RCN; } #endif if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) { may_throw = 1; } |8: | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_isset_isempty_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, bool op1_avoid_refcounting, uint32_t op2_info, uint8_t dim_type, int may_throw, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_jit_addr op2_addr, res_addr; // TODO: support for empty() ??? ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); op2_addr = OP2_ADDR(); res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr | ZVAL_DEREF FCARG1x, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & MAY_BE_ARRAY) { const void *found_exit_addr = NULL; const void *not_found_exit_addr = NULL; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 if (exit_addr && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) && !may_throw && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { if (smart_branch_opcode == ZEND_JMPNZ) { found_exit_addr = exit_addr; } else { not_found_exit_addr = exit_addr; } } if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, dim_type, found_exit_addr, not_found_exit_addr, NULL)) { return 0; } if (found_exit_addr) { |9: return 1; } else if (not_found_exit_addr) { |8: return 1; } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { if (op1_info & MAY_BE_ARRAY) { |.cold_code |7: } if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { | SET_EX_OPLINE opline, REG0 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) } else { | LOAD_ZVAL_ADDR FCARG2x, op2_addr } | EXT_CALL zend_jit_isset_dim_helper, REG0 | cbz RETVALw, >9 if (op1_info & MAY_BE_ARRAY) { | b >8 |.code } } else { if (op2_info & MAY_BE_UNDEF) { if (op2_info & MAY_BE_ANY) { | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 } | SET_EX_OPLINE opline, REG0 | LOAD_32BIT_VAL FCARG1w, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, REG0 |1: } if (op1_info & MAY_BE_ARRAY) { | b >9 |.code } } } #ifdef ZEND_JIT_USE_RC_INFERENCE if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { /* Magic offsetExists() may increase refcount of the key */ op2_info |= MAY_BE_RCN; } #endif if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { |8: | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 if (!op1_avoid_refcounting) { | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 } if (may_throw) { if (!zend_jit_check_exception_undef_result(Dst, opline)) { return 0; } } if (!(opline->extended_value & ZEND_ISEMPTY)) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | b &exit_addr } else { | b >8 } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | b =>target_label2 } else if (smart_branch_opcode == ZEND_JMPNZ) { | b =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | b =>target_label2 } else { ZEND_UNREACHABLE(); } } else { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 | b >8 } } else { | NIY // TODO: support for empty() } } |9: // not found | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 if (!op1_avoid_refcounting) { | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 } if (may_throw) { if (!zend_jit_check_exception_undef_result(Dst, opline)) { return 0; } } if (!(opline->extended_value & ZEND_ISEMPTY)) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | b &exit_addr } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | b =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { } else if (smart_branch_opcode == ZEND_JMPZNZ) { | b =>target_label } else { ZEND_UNREACHABLE(); } } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } } else { | NIY // TODO: support for empty() } |8: return 1; } static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) { zend_jit_addr op1_addr = OP1_ADDR(); zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; | ldr FCARG2x, EX->run_time_cache | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG2x, opline->extended_value, TMP1 | sub REG0, REG0, #1 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) | MEM_LOAD_32_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1 | cmp REG0, REG1, lsl #5 | bhs >9 | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); | MEM_LOAD_64_ZTS ldr, TMP1, executor_globals, symbol_table.arData, REG1 | add REG0, REG0, TMP1 | IF_NOT_Z_TYPE REG0, IS_REFERENCE, >9, TMP1w | // (EXPECTED(p->key == varname)) | ldr TMP1, [REG0, #offsetof(Bucket, key)] | LOAD_ADDR TMP2, varname | cmp TMP1, TMP2 | bne >9 | GET_Z_PTR REG0, REG0 | GC_ADDREF REG0, TMP1w |1: if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) | IF_ZVAL_REFCOUNTED op1_addr, >2, ZREG_TMP1, ZREG_TMP2 |.cold_code |2: } | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 | // ZVAL_REF(variable_ptr, ref) | SET_ZVAL_PTR op1_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 | // if (GC_DELREF(garbage) == 0) | GC_DELREF FCARG1x, TMP1w if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { | bne >3 } else { | bne >5 } | ZVAL_DTOR_FUNC op1_info, opline, TMP1 | b >5 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { |3: | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w | EXT_CALL gc_possible_root, REG0 | b >5 } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { |.code } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // ZVAL_REF(variable_ptr, ref) | SET_ZVAL_PTR op1_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 } |5: //END of handler |.cold_code |9: | LOAD_ADDR FCARG1x, (ptrdiff_t)varname if (opline->extended_value) { | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, FCARG2x, opline->extended_value, TMP1 } | EXT_CALL zend_jit_fetch_global_helper, REG0 | mov REG0, RETVALx | b <1 |.code return 1; } static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception) { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); bool in_cold = 0; uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1 : ZREG_REG0; if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->prev) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)); if (type != IS_UNKNOWN && (type_mask & (1u << type))) { return 1; } } if (ZEND_ARG_SEND_MODE(arg_info)) { if (opline->opcode == ZEND_RECV_INIT) { | LOAD_ZVAL_ADDR Rx(tmp_reg), res_addr | ZVAL_DEREF Rx(tmp_reg), MAY_BE_REF, TMP1w res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); } else { | GET_ZVAL_PTR Rx(tmp_reg), res_addr, TMP1 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); } } if (type_mask != 0) { if (is_power_of_two(type_mask)) { uint32_t type_code = concrete_type(type_mask); | IF_NOT_ZVAL_TYPE res_addr, type_code, >1, ZREG_TMP1 } else { | mov REG2w, #1 | MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, Rx(Z_REG(res_addr)), Z_OFFSET(res_addr)+offsetof(zval, u1.v.type), TMP1 | lsl REG2w, REG2w, REG1w | TST_32_WITH_CONST REG2w, type_mask, TMP1w | beq >1 } |.cold_code |1: in_cold = 1; } if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, res_addr } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | SET_EX_OPLINE opline, REG0 } else { | ADDR_STORE EX->opline, opline, REG0 } | LOAD_ADDR FCARG2x, (ptrdiff_t)arg_info | EXT_CALL zend_jit_verify_arg_slow, REG0 if (check_exception) { | GET_LOW_8BITS REG0w, RETVALw if (in_cold) { | cbnz REG0w, >1 | b ->exception_handler |.code |1: } else { | cbz REG0w, ->exception_handler } } else if (in_cold) { | b >1 |.code |1: } return 1; } static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) { uint32_t arg_num = opline->op1.num; zend_arg_info *arg_info = NULL; if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { if (EXPECTED(arg_num <= op_array->num_args)) { arg_info = &op_array->arg_info[arg_num-1]; } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { arg_info = &op_array->arg_info[op_array->num_args]; } if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { arg_info = NULL; } } if (arg_info || (opline+1)->opcode != ZEND_RECV) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (!JIT_G(current_frame) || TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 || arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | ldr TMP1w, EX->This.u2.num_args | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w | blo &exit_addr } } else { | ldr TMP1w, EX->This.u2.num_args | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w | blo >1 |.cold_code |1: if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | SET_EX_OPLINE opline, REG0 } else { | ADDR_STORE EX->opline, opline, REG0 } | mov FCARG1x, FP | EXT_CALL zend_missing_arg_error, REG0 | b ->exception_handler |.code } } if (arg_info) { if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { return 0; } } if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { | LOAD_IP_ADDR (opline + 1) zend_jit_set_last_valid_opline(opline + 1); } } return 1; } static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw) { uint32_t arg_num = opline->op1.num; zval *zv = RT_CONSTANT(opline, opline->op2); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) { if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, REG0, TMP1 } } } else { if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { | ldr TMP1w, EX->This.u2.num_args | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w | bhs >5 } | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, REG0, TMP1 } } if (Z_CONSTANT_P(zv)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | SET_EX_OPLINE opline, REG0 } else { | ADDR_STORE EX->opline, opline, REG0 } | LOAD_ZVAL_ADDR FCARG1x, res_addr | ldr REG0, EX->func | ldr FCARG2x, [REG0, #offsetof(zend_op_array, scope)] | EXT_CALL zval_update_constant_ex, REG0 | cbnz RETVALw, >1 |.cold_code |1: | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline, ZREG_TMP1, ZREG_TMP2 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 | b ->exception_handler |.code } |5: if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { do { zend_arg_info *arg_info; if (arg_num <= op_array->num_args) { arg_info = &op_array->arg_info[arg_num-1]; } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { arg_info = &op_array->arg_info[op_array->num_args]; } else { break; } if (!ZEND_TYPE_IS_SET(arg_info->type)) { break; } if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { return 0; } } while (0); } if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { if (is_last) { | LOAD_IP_ADDR (opline + 1) zend_jit_set_last_valid_opline(opline + 1); } } return 1; } static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | LOAD_ADDR TMP1, ((ptrdiff_t)ce) | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] | cmp TMP2, TMP1 | bne &exit_addr return 1; } static int zend_jit_fetch_obj(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, bool op1_indirect, zend_class_entry *ce, bool ce_is_instanceof, bool on_this, bool delayed_fetch_this, bool op1_avoid_refcounting, zend_class_entry *trace_ce, uint8_t prop_type, int may_throw) { zval *member; zend_property_info *prop_info; bool may_be_dynamic = 1; zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr prop_addr; uint32_t res_info = RES_INFO(); bool type_loaded = 0; ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 } else { if (opline->op1_type == IS_VAR && opline->opcode == ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w | GET_Z_PTR FCARG1x, FCARG1x |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & MAY_BE_REF) { if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } | ZVAL_DEREF FCARG1x, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, ZREG_TMP1 } } | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 } if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename); if (prop_info) { ce = trace_ce; ce_is_instanceof = 0; if (!(op1_info & MAY_BE_CLASS_GUARD)) { if (on_this && JIT_G(current_frame) && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { ZEND_ASSERT(JIT_G(current_frame)->ce == ce); } else if (zend_jit_class_guard(Dst, opline, ce)) { if (on_this && JIT_G(current_frame)) { JIT_G(current_frame)->ce = ce; TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); } } else { return 0; } if (ssa->var_info && ssa_op->op1_use >= 0) { ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_use].ce = ce; ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; } } } } if (!prop_info) { | ldr REG0, EX->run_time_cache | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS), TMP1 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >5 | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)), TMP1 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (may_be_dynamic) { | tst REG0, REG0 if (opline->opcode == ZEND_FETCH_OBJ_W) { | blt >5 } else { | blt >8 // dynamic property } } | add TMP1, FCARG1x, REG0 | ldr REG2w, [TMP1, #offsetof(zval,u1.type_info)] | IF_UNDEF REG2w, >5 | mov FCARG1x, TMP1 type_loaded = 1; prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); if (opline->opcode == ZEND_FETCH_OBJ_W && (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) { uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; | ldr REG0, EX->run_time_cache | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2), TMP1 | cbnz FCARG2x, >1 |.cold_code |1: | ldr TMP1w, [FCARG2x, #offsetof(zend_property_info, flags)] | tst TMP1w, #ZEND_ACC_READONLY if (flags) { | beq >3 } else { | beq >4 } | IF_NOT_TYPE REG2w, IS_OBJECT_EX, >2 | GET_Z_PTR REG2, FCARG1x | GC_ADDREF REG2, TMP1w | SET_ZVAL_PTR res_addr, REG2, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2 | b >9 |2: | mov FCARG1x, FCARG2x | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_readonly_property_modification_error, REG0 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 | b >9 |3: if (flags == ZEND_FETCH_DIM_WRITE) { | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_jit_check_array_promotion, REG0 | b >9 } else if (flags == ZEND_FETCH_REF) { | LOAD_ZVAL_ADDR CARG3, res_addr | EXT_CALL zend_jit_create_typed_ref, REG0 | b >9 } else { ZEND_ASSERT(flags == 0); } |.code |4: } } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { /* perform IS_UNDEF check only after result type guard (during deoptimization) */ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } type_loaded = 1; | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 | IF_UNDEF REG2w, &exit_addr } } else { type_loaded = 1; | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 | IF_UNDEF REG2w, >5 } if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) { if (!type_loaded) { type_loaded = 1; | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 } | IF_NOT_TYPE REG2w, IS_OBJECT_EX, >4 | GET_ZVAL_PTR REG2, prop_addr, TMP1 | GC_ADDREF REG2, TMP1w | SET_ZVAL_PTR res_addr, REG2, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2 | b >9 |.cold_code |4: | LOAD_ADDR FCARG1x, prop_info | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_readonly_property_modification_error, REG0 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 | b >9 |.code } if (opline->opcode == ZEND_FETCH_OBJ_W && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) && ZEND_TYPE_IS_SET(prop_info->type)) { uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; if (flags == ZEND_FETCH_DIM_WRITE) { if ((ZEND_TYPE_FULL_MASK(prop_info->type) & (MAY_BE_ITERABLE|MAY_BE_ARRAY)) == 0) { if (!type_loaded) { type_loaded = 1; | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 } | cmp REG2w, #IS_FALSE | ble >1 |.cold_code |1: if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, prop_addr } | LOAD_ADDR FCARG2x, prop_info | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_jit_check_array_promotion, REG0 | b >9 |.code } } else if (flags == ZEND_FETCH_REF) { if (!type_loaded) { type_loaded = 1; | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 } | GET_LOW_8BITS TMP1w, REG2w | IF_TYPE TMP1w, IS_REFERENCE, >1 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { | LOAD_ADDR FCARG2x, prop_info } else { int prop_info_offset = (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 } if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, prop_addr } | LOAD_ZVAL_ADDR CARG3, res_addr | EXT_CALL zend_jit_create_typed_ref, REG0 | b >9 |1: } else { ZEND_UNREACHABLE(); } } } if (opline->opcode == ZEND_FETCH_OBJ_W) { if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, prop_addr } | SET_ZVAL_PTR res_addr, FCARG1x, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { ssa->var_info[ssa_op->result_def].indirect_reference = 1; } } else { bool result_avoid_refcounting = 0; if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { uint32_t flags = 0; uint32_t old_info; zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; int32_t exit_point; const void *exit_addr; uint32_t type; zend_jit_addr val_addr = prop_addr; if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this && !op1_avoid_refcounting) { flags = ZEND_JIT_EXIT_FREE_OP1; } if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) && !(flags & ZEND_JIT_EXIT_FREE_OP1) && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) && (ssa_op+1)->op1_use == ssa_op->result_def && zend_jit_may_avoid_refcounting(opline+1, res_info)) { result_avoid_refcounting = 1; ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; } type = concrete_type(res_info); if (prop_type != IS_UNKNOWN && prop_type != IS_UNDEF && prop_type != IS_REFERENCE && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) { exit_point = zend_jit_trace_get_exit_point(opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } else { val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); | LOAD_ZVAL_ADDR REG0, prop_addr if (op1_avoid_refcounting) { SET_STACK_REG(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); } old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); exit_point = zend_jit_trace_get_exit_point(opline+1, flags); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } if (!type_loaded) { type_loaded = 1; | MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 } | // ZVAL_DEREF() | GET_LOW_8BITS TMP1w, REG2w | IF_NOT_TYPE TMP1w, IS_REFERENCE, >1 | GET_Z_PTR REG0, REG0 | add REG0, REG0, #offsetof(zend_reference, val) | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 } res_info &= ~MAY_BE_GUARD; ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; if (type < IS_STRING) { |1: if (type_loaded) { | IF_NOT_TYPE REG2w, type, &exit_addr } else { | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr, ZREG_TMP1 } } else { if (!type_loaded) { type_loaded = 1; | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 } |1: | GET_LOW_8BITS TMP1w, REG2w | IF_NOT_TYPE TMP1w, type, &exit_addr } | // ZVAL_COPY | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 if (type < IS_STRING) { if (Z_REG(res_addr) != ZREG_FP || JIT_G(current_frame) == NULL || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 } } else { | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 if (!result_avoid_refcounting) { | TRY_ADDREF res_info, REG2w, REG1, TMP1w } } } else { if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_REG2)) { return 0; } } } if (op1_avoid_refcounting) { SET_STACK_REG(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); } |.cold_code if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { |5: | SET_EX_OPLINE opline, REG0 if (opline->opcode == ZEND_FETCH_OBJ_W) { | EXT_CALL zend_jit_fetch_obj_w_slow, REG0 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { | EXT_CALL zend_jit_fetch_obj_r_slow, REG0 } else { | EXT_CALL zend_jit_fetch_obj_is_slow, REG0 } | b >9 } if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { |7: if (opline->opcode != ZEND_FETCH_OBJ_IS) { | SET_EX_OPLINE opline, REG0 if (opline->opcode != ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_UNDEF)) { zend_jit_addr orig_op1_addr = OP1_ADDR(); if (op1_info & MAY_BE_ANY) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 } | LOAD_32BIT_VAL FCARG1w, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 |1: | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr } else if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } | LOAD_ADDR FCARG2x, Z_STRVAL_P(member) if (opline->opcode == ZEND_FETCH_OBJ_W) { | EXT_CALL zend_jit_invalid_property_write, REG0 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 } else { | EXT_CALL zend_jit_invalid_property_read, REG0 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 } | b >9 } else { | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 | b >9 } } if (!prop_info && may_be_dynamic && opline->opcode != ZEND_FETCH_OBJ_W) { |8: | mov FCARG2x, REG0 | SET_EX_OPLINE opline, REG0 if (opline->opcode != ZEND_FETCH_OBJ_IS) { | EXT_CALL zend_jit_fetch_obj_r_dynamic, REG0 } else { | EXT_CALL zend_jit_fetch_obj_is_dynamic, REG0 } | b >9 } |.code; |9: // END if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { if (opline->op1_type == IS_VAR && opline->opcode == ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_RC1)) { zend_jit_addr orig_op1_addr = OP1_ADDR(); | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1, ZREG_TMP1, ZREG_TMP2 | GET_ZVAL_PTR FCARG1x, orig_op1_addr, TMP1 | GC_DELREF FCARG1x, TMP1w | bne >1 | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_jit_extract_helper, REG0 |1: } else if (!op1_avoid_refcounting) { if (on_this) { op1_info &= ~MAY_BE_RC1; } | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info && (opline->opcode != ZEND_FETCH_OBJ_W || !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) || !ZEND_TYPE_IS_SET(prop_info->type)) && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) { may_throw = 0; } if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_incdec_obj(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, bool op1_indirect, zend_class_entry *ce, bool ce_is_instanceof, bool on_this, bool delayed_fetch_this, zend_class_entry *trace_ce, uint8_t prop_type) { zval *member; zend_string *name; zend_property_info *prop_info; zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr res_addr = 0; zend_jit_addr prop_addr; bool needs_slow_path = 0; bool use_prop_guard = 0; bool may_throw = 0; uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0; ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); if (opline->result_type != IS_UNUSED) { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); } member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); name = Z_STR_P(member); prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 } else { if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w | GET_Z_PTR FCARG1x, FCARG1x |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & MAY_BE_REF) { if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } | ZVAL_DEREF FCARG1x, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 |.cold_code |1: | SET_EX_OPLINE opline, REG0 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } | LOAD_ADDR FCARG2x, ZSTR_VAL(name) | EXT_CALL zend_jit_invalid_property_incdec, REG0 | b ->exception_handler |.code } } | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 } if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); if (prop_info) { ce = trace_ce; ce_is_instanceof = 0; if (!(op1_info & MAY_BE_CLASS_GUARD)) { if (on_this && JIT_G(current_frame) && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { ZEND_ASSERT(JIT_G(current_frame)->ce == ce); } else if (zend_jit_class_guard(Dst, opline, ce)) { if (on_this && JIT_G(current_frame)) { JIT_G(current_frame)->ce = ce; TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); } } else { return 0; } if (ssa->var_info && ssa_op->op1_use >= 0) { ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_use].ce = ce; ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; } if (ssa->var_info && ssa_op->op1_def >= 0) { ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_def].ce = ce; ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; } } } } use_prop_guard = (prop_type != IS_UNKNOWN && prop_type != IS_UNDEF && prop_type != IS_REFERENCE && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); if (!prop_info) { needs_slow_path = 1; | ldr REG0, EX->run_time_cache | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >7 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 | cbnz TMP1, >7 } | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 | tst REG0, REG0 | blt >7 | add TMP1, FCARG1x, REG0 if (!use_prop_guard) { | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] | IF_TYPE TMP2w , IS_UNDEF, >7 } | mov FCARG1x, TMP1 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr } else { | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 | IF_TYPE TMP2w, IS_UNDEF, >7 needs_slow_path = 1; } } if (ZEND_TYPE_IS_SET(prop_info->type)) { may_throw = 1; | SET_EX_OPLINE opline, REG0 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { | LOAD_ADDR FCARG2x, prop_info } else { int prop_info_offset = (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 } | LOAD_ZVAL_ADDR FCARG1x, prop_addr if (opline->result_type == IS_UNUSED) { switch (opline->opcode) { case ZEND_PRE_INC_OBJ: case ZEND_POST_INC_OBJ: | EXT_CALL zend_jit_inc_typed_prop, REG0 break; case ZEND_PRE_DEC_OBJ: case ZEND_POST_DEC_OBJ: | EXT_CALL zend_jit_dec_typed_prop, REG0 break; default: ZEND_UNREACHABLE(); } } else { | LOAD_ZVAL_ADDR CARG3, res_addr switch (opline->opcode) { case ZEND_PRE_INC_OBJ: | EXT_CALL zend_jit_pre_inc_typed_prop, REG0 break; case ZEND_PRE_DEC_OBJ: | EXT_CALL zend_jit_pre_dec_typed_prop, REG0 break; case ZEND_POST_INC_OBJ: | EXT_CALL zend_jit_post_inc_typed_prop, REG0 break; case ZEND_POST_DEC_OBJ: | EXT_CALL zend_jit_post_dec_typed_prop, REG0 break; default: ZEND_UNREACHABLE(); } } } } if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; zend_jit_addr var_addr = prop_addr; if (use_prop_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); } if (var_info & MAY_BE_REF) { may_throw = 1; var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, prop_addr } | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] | cbnz TMP1, >1 | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) |.cold_code |1: if (opline) { | SET_EX_OPLINE opline, REG0 } if (opline->result_type == IS_UNUSED) { | mov FCARG2x, xzr } else { | LOAD_ZVAL_ADDR FCARG2x, res_addr } switch (opline->opcode) { case ZEND_PRE_INC_OBJ: | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 break; case ZEND_PRE_DEC_OBJ: | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 break; case ZEND_POST_INC_OBJ: | EXT_CALL zend_jit_post_inc_typed_ref, REG0 break; case ZEND_POST_DEC_OBJ: | EXT_CALL zend_jit_post_dec_typed_ref, REG0 break; default: ZEND_UNREACHABLE(); } | b >9 |.code |2: } if (var_info & MAY_BE_LONG) { if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2, ZREG_TMP1 } if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { if (opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } } if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { | LONG_ADD_SUB_WITH_IMM adds, var_addr, Z_L(1), TMP1, TMP2 } else { | LONG_ADD_SUB_WITH_IMM subs, var_addr, Z_L(1), TMP1, TMP2 } | bvs >3 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { if (opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } } |.cold_code } if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { may_throw = 1; } if (var_info & MAY_BE_LONG) { |2: } if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); | LOAD_ZVAL_ADDR FCARG1x, prop_addr } if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | TRY_ADDREF MAY_BE_ANY, REG0w, REG2, TMP1w } if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { | LOAD_ZVAL_ADDR FCARG2x, res_addr | EXT_CALL zend_jit_pre_inc, REG0 } else { | EXT_CALL increment_function, REG0 } } else { if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { | LOAD_ZVAL_ADDR FCARG2x, res_addr | EXT_CALL zend_jit_pre_dec, REG0 } else { | EXT_CALL decrement_function, REG0 } } if (var_info & MAY_BE_LONG) { | b >4 } } if (var_info & MAY_BE_LONG) { |3: if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { uint64_t val = 0x43e0000000000000; | LOAD_64BIT_VAL TMP2, val | SET_ZVAL_LVAL_FROM_REG var_addr, TMP2, TMP1 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { | SET_ZVAL_LVAL_FROM_REG res_addr, TMP2, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 } } else { uint64_t val = 0xc3e0000000000000; | LOAD_64BIT_VAL TMP2, val | SET_ZVAL_LVAL_FROM_REG var_addr, TMP2, TMP1 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { | SET_ZVAL_LVAL_FROM_REG res_addr, TMP2, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 } } if (opline->result_type != IS_UNUSED && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) && prop_info && !ZEND_TYPE_IS_SET(prop_info->type) && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG)) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); int32_t exit_point; const void *exit_addr; SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD; | b &exit_addr |.code } else { | b >4 |.code |4: } } } if (needs_slow_path) { may_throw = 1; |.cold_code |7: | SET_EX_OPLINE opline, REG0 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); | LOAD_ADDR FCARG2x, name | ldr CARG3, EX->run_time_cache | ADD_SUB_64_WITH_CONST_32 add, CARG3, CARG3, opline->extended_value, TMP1 if (opline->result_type == IS_UNUSED) { | mov CARG4, xzr } else { | LOAD_ZVAL_ADDR CARG4, res_addr } switch (opline->opcode) { case ZEND_PRE_INC_OBJ: | EXT_CALL zend_jit_pre_inc_obj_helper, REG0 break; case ZEND_PRE_DEC_OBJ: | EXT_CALL zend_jit_pre_dec_obj_helper, REG0 break; case ZEND_POST_INC_OBJ: | EXT_CALL zend_jit_post_inc_obj_helper, REG0 break; case ZEND_POST_DEC_OBJ: | EXT_CALL zend_jit_post_dec_obj_helper, REG0 break; default: ZEND_UNREACHABLE(); } | b >9 |.code } |9: if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { may_throw = 1; } | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_assign_obj_op(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t val_info, zend_ssa_range *val_range, bool op1_indirect, zend_class_entry *ce, bool ce_is_instanceof, bool on_this, bool delayed_fetch_this, zend_class_entry *trace_ce, uint8_t prop_type) { zval *member; zend_string *name; zend_property_info *prop_info; zend_jit_addr val_addr = OP1_DATA_ADDR(); zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr prop_addr; bool needs_slow_path = 0; bool use_prop_guard = 0; bool may_throw = 0; binary_op_type binary_op = get_binary_op(opline->extended_value); ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); ZEND_ASSERT(opline->result_type == IS_UNUSED); member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); name = Z_STR_P(member); prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 } else { if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w | GET_Z_PTR FCARG1x, FCARG1x |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & MAY_BE_REF) { if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } | ZVAL_DEREF FCARG1x, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 |.cold_code |1: | SET_EX_OPLINE opline, REG0 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } | LOAD_ADDR FCARG2x, ZSTR_VAL(name) if (op1_info & MAY_BE_UNDEF) { | EXT_CALL zend_jit_invalid_property_assign_op, REG0 } else { | EXT_CALL zend_jit_invalid_property_assign, REG0 } may_throw = 1; if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | b >8 } else { | b >9 } |.code } } | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 } if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); if (prop_info) { ce = trace_ce; ce_is_instanceof = 0; if (!(op1_info & MAY_BE_CLASS_GUARD)) { if (on_this && JIT_G(current_frame) && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { ZEND_ASSERT(JIT_G(current_frame)->ce == ce); } else if (zend_jit_class_guard(Dst, opline, ce)) { if (on_this && JIT_G(current_frame)) { JIT_G(current_frame)->ce = ce; TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); } } else { return 0; } if (ssa->var_info && ssa_op->op1_use >= 0) { ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_use].ce = ce; ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; } if (ssa->var_info && ssa_op->op1_def >= 0) { ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_def].ce = ce; ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; } } } } use_prop_guard = (prop_type != IS_UNKNOWN && prop_type != IS_UNDEF && prop_type != IS_REFERENCE && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); if (!prop_info) { needs_slow_path = 1; | ldr REG0, EX->run_time_cache | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, ((opline+1)->extended_value), TMP1 | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP2 | bne >7 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { | MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, ((opline+1)->extended_value + sizeof(void*) * 2), TMP1 | cbnz TMP1, >7 } | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline+1)->extended_value + sizeof(void*)), TMP1 | tst REG0, REG0 | blt >7 | add TMP1, FCARG1x, REG0 if (!use_prop_guard) { | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] | IF_TYPE TMP2w, IS_UNDEF, >7 } | mov FCARG1x, TMP1 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr } else { | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 | IF_TYPE TMP2w, IS_UNDEF, >7 needs_slow_path = 1; } } if (ZEND_TYPE_IS_SET(prop_info->type)) { uint32_t info = val_info; may_throw = 1; if (opline) { | SET_EX_OPLINE opline, REG0 } | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1, ZREG_TMP1 |.cold_code |1: | GET_ZVAL_PTR FCARG1x, prop_addr, TMP1 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { | LOAD_ZVAL_ADDR FCARG2x, val_addr } | LOAD_ADDR CARG3, binary_op if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 } else { | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 } | b >9 |.code | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { | LOAD_ADDR FCARG2x, prop_info } else { int prop_info_offset = (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 } | LOAD_ZVAL_ADDR FCARG1x, prop_addr | LOAD_ZVAL_ADDR CARG3, val_addr | LOAD_ADDR CARG4, binary_op | EXT_CALL zend_jit_assign_op_to_typed_prop, REG0 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { info |= MAY_BE_RC1|MAY_BE_RCN; } | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2 } } if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { zend_jit_addr var_addr = prop_addr; uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; if (use_prop_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); } if (var_info & MAY_BE_REF) { may_throw = 1; var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); | LOAD_ZVAL_ADDR REG0, prop_addr | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] | cbnz TMP1, >1 | add REG0, FCARG1x, #offsetof(zend_reference, val) |.cold_code |1: if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { | LOAD_ZVAL_ADDR FCARG2x, val_addr } if (opline) { | SET_EX_OPLINE opline, REG0 } | LOAD_ADDR CARG3, binary_op if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0 } else { | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 } | b >9 |.code |2: } switch (opline->extended_value) { case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (opline->extended_value != ZEND_ADD || (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { may_throw = 1; } } if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, 1 /* may overflow */, 0)) { return 0; } break; case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: may_throw = 1; if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if ((var_info & MAY_BE_ANY) != MAY_BE_STRING || (val_info & MAY_BE_ANY) != MAY_BE_STRING) { may_throw = 1; } } goto long_math; case ZEND_SL: case ZEND_SR: if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { may_throw = 1; } if ((opline+1)->op1_type != IS_CONST || Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) { may_throw = 1; } goto long_math; case ZEND_MOD: if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (opline->extended_value != ZEND_ADD || (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { may_throw = 1; } } if ((opline+1)->op1_type != IS_CONST || Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) { may_throw = 1; } long_math: if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, NULL, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, val_range, 0, var_addr, var_def_info, var_info, 0)) { return 0; } break; case ZEND_CONCAT: may_throw = 1; if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, 0)) { return 0; } break; default: ZEND_UNREACHABLE(); } } if (needs_slow_path) { may_throw = 1; |.cold_code |7: | SET_EX_OPLINE opline, REG0 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); | LOAD_ADDR FCARG2x, name | LOAD_ZVAL_ADDR CARG3, val_addr | ldr CARG4, EX->run_time_cache | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, (opline+1)->extended_value, TMP1 | LOAD_ADDR CARG5, binary_op | EXT_CALL zend_jit_assign_obj_op_helper, REG0 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { val_info |= MAY_BE_RC1|MAY_BE_RCN; } |8: | // FREE_OP_DATA(); | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 | b >9 |.code } |9: if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { may_throw = 1; } | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_assign_obj(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t val_info, bool op1_indirect, zend_class_entry *ce, bool ce_is_instanceof, bool on_this, bool delayed_fetch_this, zend_class_entry *trace_ce, uint8_t prop_type, int may_throw) { zval *member; zend_string *name; zend_property_info *prop_info; zend_jit_addr val_addr = OP1_DATA_ADDR(); zend_jit_addr res_addr = 0; zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr prop_addr; bool needs_slow_path = 0; bool needs_val_dtor = 0; if (RETURN_VALUE_USED(opline)) { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); } ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); name = Z_STR_P(member); prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 } else { if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w | GET_Z_PTR FCARG1x, FCARG1x |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & MAY_BE_REF) { if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } | ZVAL_DEREF FCARG1x, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 |.cold_code |1: | SET_EX_OPLINE opline, REG0 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } | LOAD_ADDR FCARG2x, ZSTR_VAL(name) | EXT_CALL zend_jit_invalid_property_assign, REG0 if (RETURN_VALUE_USED(opline)) { | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 } if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { needs_val_dtor = 1; | b >7 } else { | b >9 } |.code } } | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 } if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); if (prop_info) { ce = trace_ce; ce_is_instanceof = 0; if (!(op1_info & MAY_BE_CLASS_GUARD)) { if (on_this && JIT_G(current_frame) && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { ZEND_ASSERT(JIT_G(current_frame)->ce == ce); } else if (zend_jit_class_guard(Dst, opline, ce)) { if (on_this && JIT_G(current_frame)) { JIT_G(current_frame)->ce = ce; TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); } } else { return 0; } if (ssa->var_info && ssa_op->op1_use >= 0) { ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_use].ce = ce; ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; } if (ssa->var_info && ssa_op->op1_def >= 0) { ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_def].ce = ce; ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; } } } } if (!prop_info) { needs_slow_path = 1; | ldr REG0, EX->run_time_cache | MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >5 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 } | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 | tst REG0, REG0 | blt >5 | add TMP2, FCARG1x, REG0 | ldrb TMP1w, [TMP2, #offsetof(zval,u1.v.type)] | IF_TYPE TMP1w, IS_UNDEF, >5 | mov FCARG1x, TMP2 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { | cbnz FCARG2x, >1 |.cold_code |1: | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); | SET_EX_OPLINE opline, REG0 | LOAD_ZVAL_ADDR CARG3, val_addr if (RETURN_VALUE_USED(opline)) { | LOAD_ZVAL_ADDR CARG4, res_addr } else { | mov CARG4, xzr } | EXT_CALL zend_jit_assign_to_typed_prop, REG0 if ((opline+1)->op1_type == IS_CONST) { | // TODO: ??? | // if (Z_TYPE_P(value) == orig_type) { | // CACHE_PTR_EX(cache_slot + 2, NULL); } if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | b >7 } else { | b >9 } |.code } } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) { // Undefined property with magic __get()/__set() if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr } else { | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 | IF_TYPE TMP2w, IS_UNDEF, >5 needs_slow_path = 1; } } if (ZEND_TYPE_IS_SET(prop_info->type)) { uint32_t info = val_info; | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); | SET_EX_OPLINE opline, REG0 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { | LOAD_ADDR FCARG2x, prop_info } else { int prop_info_offset = (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] | MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 } | LOAD_ZVAL_ADDR FCARG1x, prop_addr | LOAD_ZVAL_ADDR CARG3, val_addr if (RETURN_VALUE_USED(opline)) { | LOAD_ZVAL_ADDR CARG4, res_addr } else { | mov CARG4, xzr } | EXT_CALL zend_jit_assign_to_typed_prop, REG0 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { info |= MAY_BE_RC1|MAY_BE_RCN; } | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2 } } if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); if (opline->result_type == IS_UNUSED) { if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { return 0; } } else { if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { return 0; } } } if (needs_slow_path) { |.cold_code |5: | SET_EX_OPLINE opline, REG0 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); | LOAD_ADDR FCARG2x, name | LOAD_ZVAL_ADDR CARG3, val_addr | ldr CARG4, EX->run_time_cache | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, opline->extended_value, TMP1 if (RETURN_VALUE_USED(opline)) { | LOAD_ZVAL_ADDR CARG5, res_addr } else { | mov CARG5, xzr } | EXT_CALL zend_jit_assign_obj_helper, REG0 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { val_info |= MAY_BE_RC1|MAY_BE_RCN; } |7: | // FREE_OP_DATA(); | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 | b >9 |.code } else if (needs_val_dtor) { |.cold_code |7: | // FREE_OP_DATA(); | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 | b >9 |.code } |9: if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) { zend_jit_addr op1_addr = OP1_ADDR(); if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (may_throw) { | SET_EX_OPLINE opline, REG0 } if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { if (op1_info & MAY_BE_ARRAY) { | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } | MEM_ACCESS_32_WITH_UOFFSET ldr, FCARG1w, FP, (opline->op1.var + offsetof(zval, u2.fe_iter_idx)), TMP1 | mvn TMP1w, wzr // TODO: DynAsm fails loading #-1 | cmp FCARG1w, TMP1w | beq >7 | EXT_CALL zend_hash_iterator_del, REG0 |7: } | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { if (!zend_jit_check_exception(Dst)) { return 0; } } } return 1; } static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) { if (opline->op1_type == IS_CONST) { zval *zv; size_t len; zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); len = Z_STRLEN_P(zv); if (len > 0) { const char *str = Z_STRVAL_P(zv); | SET_EX_OPLINE opline, REG0 | LOAD_ADDR CARG1, str | LOAD_64BIT_VAL CARG2, len | EXT_CALL zend_write, REG0 if (!zend_jit_check_exception(Dst)) { return 0; } } } else { zend_jit_addr op1_addr = OP1_ADDR(); ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); | SET_EX_OPLINE opline, REG0 | GET_ZVAL_PTR REG0, op1_addr, TMP1 | add CARG1, REG0, #offsetof(zend_string, val) | ldr CARG2, [REG0, #offsetof(zend_string, len)] | EXT_CALL zend_write, REG0 if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 } if (!zend_jit_check_exception(Dst)) { return 0; } } return 1; } static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) { if (opline->op1_type == IS_CONST) { zval *zv; size_t len; zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); len = Z_STRLEN_P(zv); | SET_ZVAL_LVAL res_addr, len, TMP1, TMP2 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { return 0; } } else { ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); if (Z_MODE(res_addr) == IS_REG) { | GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1 | ldr Rx(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(zend_string, len)] if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { return 0; } } else { | GET_ZVAL_PTR REG0, op1_addr, TMP1 | ldr REG0, [REG0, #offsetof(zend_string, len)] | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 } | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 } return 1; } static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) { if (opline->op1_type == IS_CONST) { zval *zv; zend_long count; zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); count = zend_hash_num_elements(Z_ARRVAL_P(zv)); | SET_ZVAL_LVAL res_addr, count, TMP1, TMP2 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { return 0; } } else { ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY); // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+. if (Z_MODE(res_addr) == IS_REG) { | GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1 // Sign-extend the 32-bit value to a potentially 64-bit zend_long | ldr Rw(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(HashTable, nNumOfElements)] if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { return 0; } } else { | GET_ZVAL_PTR REG0, op1_addr, TMP1 // Sign-extend the 32-bit value to a potentially 64-bit zend_long | ldr REG0w, [REG0, #offsetof(HashTable, nNumOfElements)] | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 } | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 } if (may_throw) { return zend_jit_check_exception(Dst); } return 1; } static int zend_jit_load_this(dasm_State **Dst, uint32_t var) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); | ldr FCARG1x, EX->This.value.ptr | SET_ZVAL_PTR var_addr, FCARG1x, TMP1 | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX, TMP1w, TMP2 | GC_ADDREF FCARG1x, TMP1w return 1; } static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only) { if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (!JIT_G(current_frame) || !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | ldrb TMP1w, EX->This.u1.v.type | cmp TMP1w, #IS_OBJECT | bne &exit_addr if (JIT_G(current_frame)) { TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); } } } else { | ldrb TMP1w, EX->This.u1.v.type | cmp TMP1w, #IS_OBJECT | bne >1 |.cold_code |1: | SET_EX_OPLINE opline, REG0 | b ->invalid_this |.code } } if (!check_only) { if (!zend_jit_load_this(Dst, opline->result.var)) { return 0; } } return 1; } static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) { uint32_t count; Bucket *p; const zend_op *target; int b; int32_t exit_point; const void *exit_addr; if (default_label) { | cbz REG0, &default_label } else if (next_opline) { | cbz REG0, >3 } else { | cbz REG0, =>default_b } | LOAD_ADDR FCARG1x, jumptable | ldr TMP1, [FCARG1x, #offsetof(HashTable, arData)] | sub REG0, REG0, TMP1 | mov FCARG1x, #(sizeof(Bucket) / sizeof(void*)) | sdiv REG0, REG0, FCARG1x | adr FCARG1x, >4 | ldr TMP1, [FCARG1x, REG0] | br TMP1 |.jmp_table |.align 8 |4: if (trace_info) { trace_info->jmp_table_size += zend_hash_num_elements(jumptable); } count = jumptable->nNumUsed; p = jumptable->arData; do { if (Z_TYPE(p->val) == IS_UNDEF) { if (default_label) { | .addr &default_label } else if (next_opline) { | .addr >3 } else { | .addr =>default_b } } else { target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); if (!next_opline) { b = ssa->cfg.map[target - op_array->opcodes]; | .addr =>b } else if (next_opline == target) { | .addr >3 } else { exit_point = zend_jit_trace_get_exit_point(target, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | .addr &exit_addr } } p++; count--; } while (count); |.code return 1; } static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) { HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); const zend_op *next_opline = NULL; if (trace) { ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); ZEND_ASSERT(trace->opline != NULL); next_opline = trace->opline; } if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); zval *jump_zv = NULL; int b; if (opline->opcode == ZEND_SWITCH_LONG) { if (Z_TYPE_P(zv) == IS_LONG) { jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); } } else if (opline->opcode == ZEND_SWITCH_STRING) { if (Z_TYPE_P(zv) == IS_STRING) { jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); } } else if (opline->opcode == ZEND_MATCH) { if (Z_TYPE_P(zv) == IS_LONG) { jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); } else if (Z_TYPE_P(zv) == IS_STRING) { jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); } } else { ZEND_UNREACHABLE(); } if (next_opline) { const zend_op *target; if (jump_zv != NULL) { target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); } else { target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); } ZEND_ASSERT(target == next_opline); } else { if (jump_zv != NULL) { b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; } else { b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; } | b =>b } } else { zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; uint32_t op1_info = OP1_INFO(); zend_jit_addr op1_addr = OP1_ADDR(); const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); const zend_op *target; int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; int b; int32_t exit_point; const void *fallback_label = NULL; const void *default_label = NULL; const void *exit_addr; if (next_opline) { if (next_opline != opline + 1) { exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); fallback_label = zend_jit_trace_get_exit_addr(exit_point); if (!fallback_label) { return 0; } } if (next_opline != default_opline) { exit_point = zend_jit_trace_get_exit_point(default_opline, 0); default_label = zend_jit_trace_get_exit_addr(exit_point); if (!default_label) { return 0; } } } if (opline->opcode == ZEND_SWITCH_LONG) { if (op1_info & MAY_BE_LONG) { if (op1_info & MAY_BE_REF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1, ZREG_TMP1 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 |.cold_code |1: | // ZVAL_DEREF(op) if (fallback_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 } | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 if (fallback_label) { | add TMP1, FCARG2x, #offsetof(zend_reference, val) | IF_NOT_Z_TYPE TMP1, IS_LONG, &fallback_label, TMP2w } else { | add TMP1, FCARG2x, #offsetof(zend_reference, val) | IF_NOT_Z_TYPE TMP1, IS_LONG, >3, TMP2w } | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.lval)] | b >2 |.code |2: } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { if (fallback_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 } } | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 } if (HT_IS_PACKED(jumptable)) { uint32_t count = jumptable->nNumUsed; Bucket *p = jumptable->arData; | CMP_64_WITH_CONST_32 FCARG2x, jumptable->nNumUsed, TMP1 if (default_label) { | bhs &default_label } else if (next_opline) { | bhs >3 } else { | bhs =>default_b } | adr REG0, >4 | ldr TMP1, [REG0, FCARG2x, lsl #3] | br TMP1 |.jmp_table |.align 8 |4: if (trace_info) { trace_info->jmp_table_size += count; } p = jumptable->arData; do { if (Z_TYPE(p->val) == IS_UNDEF) { if (default_label) { | .addr &default_label } else if (next_opline) { | .addr >3 } else { | .addr =>default_b } } else { target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); if (!next_opline) { b = ssa->cfg.map[target - op_array->opcodes]; | .addr =>b } else if (next_opline == target) { | .addr >3 } else { exit_point = zend_jit_trace_get_exit_point(target, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | .addr &exit_addr } } p++; count--; } while (count); |.code |3: } else { | LOAD_ADDR FCARG1x, jumptable | EXT_CALL zend_hash_index_find, REG0 | mov REG0, RETVALx if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { return 0; } |3: } } } else if (opline->opcode == ZEND_SWITCH_STRING) { if (op1_info & MAY_BE_STRING) { if (op1_info & MAY_BE_REF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1, ZREG_TMP1 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 |.cold_code |1: | // ZVAL_DEREF(op) if (fallback_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 } | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 if (fallback_label) { | add TMP1, FCARG2x, #offsetof(zend_reference, val) | IF_NOT_Z_TYPE TMP1, IS_STRING, &fallback_label, TMP2w } else { | add TMP1, FCARG2x, #offsetof(zend_reference, val) | IF_NOT_Z_TYPE TMP1, IS_STRING, >3, TMP2w } | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.ptr)] | b >2 |.code |2: } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { if (fallback_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 } } | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 } | LOAD_ADDR FCARG1x, jumptable | EXT_CALL zend_hash_find, REG0 | mov REG0, RETVALx if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { return 0; } |3: } } else if (opline->opcode == ZEND_MATCH) { if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { if (op1_info & MAY_BE_REF) { | LOAD_ZVAL_ADDR FCARG2x, op1_addr | ZVAL_DEREF FCARG2x, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); } | LOAD_ADDR FCARG1x, jumptable if (op1_info & MAY_BE_LONG) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { if (op1_info & MAY_BE_STRING) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5, ZREG_TMP1 } else if (op1_info & MAY_BE_UNDEF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 } else if (default_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label, ZREG_TMP1 } else if (next_opline) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b, ZREG_TMP1 } } | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1 | EXT_CALL zend_hash_index_find, REG0 | mov REG0, RETVALx if (op1_info & MAY_BE_STRING) { | b >2 } } if (op1_info & MAY_BE_STRING) { |5: if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { if (op1_info & MAY_BE_UNDEF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 } else if (default_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label, ZREG_TMP1 } else if (next_opline) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b, ZREG_TMP1 } } | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 | EXT_CALL zend_hash_find, REG0 | mov REG0, RETVALx } |2: if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { return 0; } } if (op1_info & MAY_BE_UNDEF) { |6: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { if (default_label) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label, ZREG_TMP1 } else if (next_opline) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3, ZREG_TMP1 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b, ZREG_TMP1 } } | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | SET_EX_OPLINE opline, REG0 | LOAD_32BIT_VAL FCARG1w, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 if (!zend_jit_check_exception_undef_result(Dst, opline)) { return 0; } } if (default_label) { | b &default_label } else if (next_opline) { | b >3 } else { | b =>default_b } |3: } else { ZEND_UNREACHABLE(); } } return 1; } static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) { zend_arg_info *arg_info = &op_array->arg_info[-1]; ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); zend_jit_addr op1_addr = OP1_ADDR(); bool needs_slow_check = 1; bool slow_check_in_cold = 1; uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; if (type_mask == 0) { slow_check_in_cold = 0; } else { if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { slow_check_in_cold = 0; } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { needs_slow_check = 0; } else if (is_power_of_two(type_mask)) { uint32_t type_code = concrete_type(type_mask); | IF_NOT_ZVAL_TYPE op1_addr, type_code, >6, ZREG_TMP1 } else { | mov REG2w, #1 | GET_ZVAL_TYPE REG1w, op1_addr, TMP1 | lsl REG2w, REG2w, REG1w | TST_32_WITH_CONST REG2w, type_mask, TMP1w | beq >6 } } if (needs_slow_check) { if (slow_check_in_cold) { |.cold_code |6: } | SET_EX_OPLINE opline, REG1 if (op1_info & MAY_BE_UNDEF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7, ZREG_TMP1 | LOAD_32BIT_VAL FCARG1x, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 | cbz RETVALx, ->exception_handler | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval | b >8 } |7: | LOAD_ZVAL_ADDR FCARG1x, op1_addr |8: | ldr FCARG2x, EX->func | LOAD_ADDR CARG3, (ptrdiff_t)arg_info | ldr REG0, EX->run_time_cache | ADD_SUB_64_WITH_CONST_32 add, CARG4, REG0, opline->op2.num, TMP1 | EXT_CALL zend_jit_verify_return_slow, REG0 if (!zend_jit_check_exception(Dst)) { return 0; } if (slow_check_in_cold) { | b >9 |.code } } |9: return 1; } static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); // TODO: support for empty() ??? ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); if (op1_info & MAY_BE_REF) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); } | ZVAL_DEREF FCARG1x, op1_info, TMP1w |1: } if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { if (exit_addr) { ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPNZ) { | b =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | b =>target_label2 } } else { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { if (exit_addr) { ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); } else if (smart_branch_opcode) { if (smart_branch_opcode != ZEND_JMPNZ) { | b =>target_label } } else { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } } else { ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); | MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, Rx(Z_REG(op1_addr)), (Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)), TMP1 | cmp TMP1w, #IS_NULL if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | bgt &exit_addr } else { | ble &exit_addr } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | ble =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { | bgt =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | ble =>target_label | b =>target_label2 } else { ZEND_UNREACHABLE(); } } else { | cset REG0w, gt | add REG0w, REG0w, #IS_FALSE | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } } return 1; } static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { | ADDREF_CONST zv, REG0, TMP1 } } else { zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); | // ZVAL_COPY(res, value); | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (opline->op1_type == IS_CV) { | TRY_ADDREF op1_info, REG0w, FCARG1x, TMP1w } } | // Z_FE_POS_P(res) = 0; | MEM_ACCESS_32_WITH_UOFFSET str, wzr, FP, (opline->result.var + offsetof(zval, u2.fe_pos)), TMP1 return 1; } static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr) { zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) { /* empty array */ if (exit_addr) { if (exit_opcode == ZEND_JMP) { | b &exit_addr } } else { | b =>target_label } return 1; } | // array = EX_VAR(opline->op1.var); | // fe_ht = Z_ARRVAL_P(array); | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 | // pos = Z_FE_POS_P(array); | MEM_ACCESS_32_WITH_UOFFSET ldr, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 | // p = fe_ht->arData + pos; || ZEND_ASSERT(sizeof(Bucket) == 32); | mov FCARG2w, REG0w | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] | add FCARG2x, TMP1, FCARG2x, lsl #5 |1: | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { | ldr TMP1w, [FCARG1x, #offsetof(zend_array, nNumUsed)] | cmp TMP1w, REG0w | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); | // ZEND_VM_CONTINUE(); if (exit_addr) { if (exit_opcode == ZEND_JMP) { | bls &exit_addr } else { | bls >3 } } else { | bls =>target_label } | // pos++; | add REG0w, REG0w, #1 | // value_type = Z_TYPE_INFO_P(value); | // if (EXPECTED(value_type != IS_UNDEF)) { if (!exit_addr || exit_opcode == ZEND_JMP) { | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >3, TMP1w } else { | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr, TMP1w } | // p++; | add FCARG2x, FCARG2x, #sizeof(Bucket) | b <1 |3: if (!exit_addr || exit_opcode == ZEND_JMP) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); uint32_t val_info; | // Z_FE_POS_P(array) = pos + 1; | MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 if (RETURN_VALUE_USED(opline)) { zend_jit_addr res_addr = RES_ADDR(); if ((op1_info & MAY_BE_ARRAY_KEY_LONG) && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { | // if (!p->key) { | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] | cbz REG0, >2 } if (op1_info & MAY_BE_ARRAY_KEY_STRING) { | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] | SET_ZVAL_PTR res_addr, REG0, TMP1 | ldr TMP1w, [REG0, #offsetof(zend_refcounted, gc.u.type_info)] | TST_32_WITH_CONST TMP1w, IS_STR_INTERNED, TMP2w | beq >1 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 | b >3 |1: | GC_ADDREF REG0, TMP1w | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { | b >3 |2: } } if (op1_info & MAY_BE_ARRAY_KEY_LONG) { | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); | ldr REG0, [FCARG2x, #offsetof(Bucket, h)] | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 } |3: } val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); if (val_info & MAY_BE_ARRAY) { val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } if (op1_info & MAY_BE_ARRAY_OF_REF) { val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { val_info |= MAY_BE_RC1 | MAY_BE_RCN; } if (opline->op2_type == IS_CV) { | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { return 0; } } else { | // ZVAL_COPY(res, value); | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | TRY_ADDREF val_info, REG0w, FCARG1x, TMP1w } } return 1; } static int zend_jit_fetch_constant(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, zend_jit_addr res_addr) { zval *zv = RT_CONSTANT(opline, opline->op2) + 1; zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); uint32_t res_info = RES_INFO(); | // c = CACHED_PTR(opline->extended_value); | ldr FCARG1x, EX->run_time_cache | MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG1x, opline->extended_value, TMP1 | // if (c != NULL) | cbz REG0, >9 if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) { | // if (!IS_SPECIAL_CACHE_VAL(c)) || ZEND_ASSERT(CACHE_SPECIAL == 1); | TST_64_WITH_ONE REG0 | bne >9 } |8: if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); int32_t exit_point; const void *exit_addr = NULL; SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); exit_point = zend_jit_trace_get_exit_point(opline+1, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } res_info &= ~MAY_BE_GUARD; ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; uint32_t type = concrete_type(res_info); if (type < IS_STRING) { | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr, ZREG_TMP1 } else { | GET_ZVAL_TYPE_INFO REG2w, const_addr, TMP1 | IF_NOT_TYPE REG2w, type, &exit_addr } | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 if (type < IS_STRING) { if (Z_MODE(res_addr) == IS_MEM_ZVAL) { | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { return 0; } } else { | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 | TRY_ADDREF res_info, REG2w, REG1, TMP1w } } else { | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | TRY_ADDREF MAY_BE_ANY, REG0w, REG1, TMP1w } |.cold_code |9: | // SAVE_OPLINE(); | SET_EX_OPLINE opline, REG0 | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); | LOAD_ADDR FCARG1x, zv | LOAD_32BIT_VAL FCARG2w, opline->op1.num | EXT_CALL zend_jit_get_constant, REG0 | mov REG0, RETVALx | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); | cbnz REG0, <8 | b ->exception_handler |.code return 1; } static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); | LOAD_ADDR FCARG1x, ht if (opline->op1_type != IS_CONST) { | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 | EXT_CALL zend_hash_find, REG0 } else { zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); | LOAD_ADDR FCARG2x, str | EXT_CALL zend_hash_find_known_hash, REG0 } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { | cbz RETVALx, &exit_addr } else { | cbnz RETVALx, &exit_addr } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { | cbz RETVALx, =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { | cbnz RETVALx, =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { | cbz RETVALx, =>target_label | b =>target_label2 } else { ZEND_UNREACHABLE(); } } else { | tst RETVALx, RETVALx | cset REG0w, ne | add REG0w, REG0w, #IS_FALSE | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } return 1; } static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info) { uint32_t offset; offset = (opline->opcode == ZEND_ROPE_INIT) ? opline->result.var : opline->op1.var + opline->extended_value * sizeof(zend_string*); if (opline->op2_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op2); zend_string *str; ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); str = Z_STR_P(zv); | LOAD_ADDR REG0, str | MEM_ACCESS_64_WITH_UOFFSET str, REG0, FP, offset, TMP1 } else { zend_jit_addr op2_addr = OP2_ADDR(); ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); | GET_ZVAL_PTR REG1, op2_addr, TMP1 | MEM_ACCESS_64_WITH_UOFFSET str, REG1, FP, offset, TMP1 if (opline->op2_type == IS_CV) { | GET_ZVAL_TYPE_INFO REG0w, op2_addr, TMP1 | TRY_ADDREF op2_info, REG0w, REG1, TMP1w } } if (opline->opcode == ZEND_ROPE_END) { zend_jit_addr res_addr = RES_ADDR(); | ADD_SUB_64_WITH_CONST add, FCARG1x, FP, opline->op1.var, TMP1 | LOAD_32BIT_VAL FCARG2w, opline->extended_value | EXT_CALL zend_jit_rope_end, TMP1 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2 } return 1; } static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 return 1; } static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard) { zend_jit_addr var_addr = *var_addr_ptr; uint32_t var_info = *var_info_ptr; const void *exit_addr = NULL; if (add_ref_guard || add_type_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } if (add_ref_guard) { | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 } if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, var_addr } | EXT_CALL zend_jit_unref_helper, REG0 } else { | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); *var_addr_ptr = var_addr; } if (var_type != IS_UNKNOWN) { var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); } if (add_type_guard && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr, ZREG_TMP1 ZEND_ASSERT(var_info & (1 << var_type)); if (var_type < IS_STRING) { var_info = (1 << var_type); } else if (var_type != IS_ARRAY) { var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); } else { var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); } *var_info_ptr = var_info; } else { var_info &= ~MAY_BE_REF; *var_info_ptr = var_info; } *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */ return 1; } static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard) { zend_jit_addr var_addr = *var_addr_ptr; uint32_t var_info = *var_info_ptr; int32_t exit_point; const void *exit_addr; if (add_indirect_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr, ZREG_TMP1 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 } else { /* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */ if (opline->op1_type != IS_VAR || (opline-1)->result_type != IS_VAR || (opline-1)->result.var != opline->op1.var || (opline-1)->op1_type == IS_VAR || (opline-1)->op2_type == IS_VAR || (opline-1)->op2_type == IS_TMP_VAR) { | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { | mov FCARG1x, REG0 } } *var_info_ptr &= ~MAY_BE_INDIRECT; var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); *var_addr_ptr = var_addr; if (var_type != IS_UNKNOWN) { var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); } if (!(var_type & IS_TRACE_REFERENCE) && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { exit_point = zend_jit_trace_get_exit_point(opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } | IF_NOT_Z_TYPE FCARG1x, var_type, &exit_addr, TMP1w //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); ZEND_ASSERT(var_info & (1 << var_type)); if (var_type < IS_STRING) { var_info = (1 << var_type); } else if (var_type != IS_ARRAY) { var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); } else { var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); } *var_info_ptr = var_info; } return 1; } static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) { if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { return 0; } switch (opline->opcode) { case ZEND_QM_ASSIGN: case ZEND_SEND_VAR: case ZEND_ASSIGN: case ZEND_PRE_INC: case ZEND_PRE_DEC: case ZEND_POST_INC: case ZEND_POST_DEC: return 1; case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: case ZEND_SL: case ZEND_SR: if (def_var == ssa_op->result_def && use_var == ssa_op->op1_use) { return 1; } break; default: break; } return 0; } static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) { uint32_t op1_info, op2_info; switch (opline->opcode) { case ZEND_SEND_VAR: case ZEND_SEND_VAL: case ZEND_SEND_VAL_EX: return (opline->op2_type != IS_CONST); case ZEND_QM_ASSIGN: case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: case ZEND_IS_EQUAL: case ZEND_IS_NOT_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_IS_NOT_IDENTICAL: case ZEND_CASE: return 1; case ZEND_RETURN: return (op_array->type != ZEND_EVAL_CODE && op_array->function_name); case ZEND_ASSIGN: op1_info = OP1_INFO(); op2_info = OP2_INFO(); return opline->op1_type == IS_CV && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: op1_info = OP1_INFO(); op2_info = OP2_INFO(); return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: case ZEND_SL: case ZEND_SR: case ZEND_MOD: op1_info = OP1_INFO(); op2_info = OP2_INFO(); return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); case ZEND_PRE_INC: case ZEND_PRE_DEC: case ZEND_POST_INC: case ZEND_POST_DEC: op1_info = OP1_INFO(); op2_info = OP1_DEF_INFO(); return opline->op1_type == IS_CV && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)) && (op2_info & MAY_BE_LONG); case ZEND_STRLEN: op1_info = OP1_INFO(); return (opline->op1_type & (IS_CV|IS_CONST)) && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING; case ZEND_COUNT: op1_info = OP1_INFO(); return (opline->op1_type & (IS_CV|IS_CONST)) && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY; case ZEND_JMPZ: case ZEND_JMPNZ: if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { if (!ssa->cfg.map) { return 0; } if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start && ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { return 0; } } ZEND_FALLTHROUGH; case ZEND_BOOL: case ZEND_BOOL_NOT: case ZEND_JMPZNZ: case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: return 1; case ZEND_FETCH_CONSTANT: return 1; case ZEND_FETCH_DIM_R: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (trace && trace->op1_type != IS_UNKNOWN && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); } return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); } return 0; } static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) { if (ssa->vars[var].no_val) { /* we don't need the value */ return 0; } if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { /* Disable global register allocation, * register allocation for SSA variables connected through Phi functions */ if (ssa->vars[var].definition_phi) { return 0; } if (ssa->vars[var].phi_use_chain) { zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; do { if (!ssa->vars[phi->ssa_var].no_val) { return 0; } phi = zend_ssa_next_use_phi(ssa, var, phi); } while (phi); } } if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { /* bad type */ return 0; } return 1; } static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) { if (!zend_jit_var_supports_reg(ssa, var)) { return 0; } if (ssa->vars[var].definition >= 0) { uint32_t def = ssa->vars[var].definition; if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { return 0; } } if (ssa->vars[var].use_chain >= 0) { int use = ssa->vars[var].use_chain; do { if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { return 0; } use = zend_ssa_next_use(ssa->ops, var, use); } while (use >= 0); } return 1; } static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) { uint32_t op1_info, op2_info; switch (opline->opcode) { case ZEND_FETCH_DIM_R: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { return ZEND_REGSET(ZREG_FCARG1); } break; default: break; } return ZEND_REGSET_EMPTY; } static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) { uint32_t op1_info, op2_info, res_info; zend_regset regset = ZEND_REGSET_SCRATCH; switch (opline->opcode) { case ZEND_NOP: case ZEND_OP_DATA: case ZEND_JMP: case ZEND_RETURN: regset = ZEND_REGSET_EMPTY; break; case ZEND_QM_ASSIGN: if (ssa_op->op1_def == current_var || ssa_op->result_def == current_var) { regset = ZEND_REGSET_EMPTY; break; } /* break missing intentionally */ case ZEND_SEND_VAL: case ZEND_SEND_VAL_EX: if (opline->op2_type == IS_CONST) { break; } if (ssa_op->op1_use == current_var) { regset = ZEND_REGSET(ZREG_REG0); break; } op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_UNDEF)) { if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_FPR0); } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { regset = ZEND_REGSET(ZREG_REG0); } else { regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); } } break; case ZEND_SEND_VAR: if (opline->op2_type == IS_CONST) { break; } if (ssa_op->op1_use == current_var || ssa_op->op1_def == current_var) { regset = ZEND_REGSET_EMPTY; break; } op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_UNDEF)) { if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_FPR0); } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { } else { regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); if (op1_info & MAY_BE_REF) { ZEND_REGSET_INCL(regset, ZREG_REG1); } } } break; case ZEND_ASSIGN: if (ssa_op->op2_use == current_var || ssa_op->op2_def == current_var || ssa_op->op1_def == current_var || ssa_op->result_def == current_var) { regset = ZEND_REGSET_EMPTY; break; } op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (opline->op1_type == IS_CV && !(op2_info & MAY_BE_UNDEF) && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_FPR0); } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { regset = ZEND_REGSET(ZREG_REG0); } else { regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); } } break; case ZEND_PRE_INC: case ZEND_PRE_DEC: case ZEND_POST_INC: case ZEND_POST_DEC: if (ssa_op->op1_use == current_var || ssa_op->op1_def == current_var || ssa_op->result_def == current_var) { regset = ZEND_REGSET_EMPTY; break; } op1_info = OP1_INFO(); if (opline->op1_type == IS_CV && (op1_info & MAY_BE_LONG) && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; if (op1_info & MAY_BE_DOUBLE) { regset = ZEND_REGSET(ZREG_FPR0); } if (opline->result_type != IS_UNUSED && (op1_info & MAY_BE_LONG)) { ZEND_REGSET_INCL(regset, ZREG_REG1); } } break; case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (ssa_op->result_def != current_var && (ssa_op->op1_use != current_var || !last_use)) { ZEND_REGSET_INCL(regset, ZREG_REG0); } res_info = RES_INFO(); if (res_info & MAY_BE_DOUBLE) { ZEND_REGSET_INCL(regset, ZREG_REG0); ZEND_REGSET_INCL(regset, ZREG_FPR0); ZEND_REGSET_INCL(regset, ZREG_FPR1); } else if (res_info & MAY_BE_GUARD) { ZEND_REGSET_INCL(regset, ZREG_REG0); } } if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { if (ssa_op->result_def != current_var) { ZEND_REGSET_INCL(regset, ZREG_FPR0); } } if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { if (zend_is_commutative(opline->opcode)) { if (ssa_op->result_def != current_var) { ZEND_REGSET_INCL(regset, ZREG_FPR0); } } else { ZEND_REGSET_INCL(regset, ZREG_FPR0); if (ssa_op->result_def != current_var && (ssa_op->op1_use != current_var || !last_use)) { ZEND_REGSET_INCL(regset, ZREG_FPR1); } } } if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { if (ssa_op->result_def != current_var && (ssa_op->op1_use != current_var || !last_use) && (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { ZEND_REGSET_INCL(regset, ZREG_FPR0); } } } break; case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: case ZEND_SL: case ZEND_SR: case ZEND_MOD: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { regset = ZEND_REGSET_EMPTY; if (ssa_op->result_def != current_var && (ssa_op->op1_use != current_var || !last_use)) { ZEND_REGSET_INCL(regset, ZREG_REG0); } } break; case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: case ZEND_IS_EQUAL: case ZEND_IS_NOT_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_IS_NOT_IDENTICAL: case ZEND_CASE: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { ZEND_REGSET_INCL(regset, ZREG_REG0); } if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { if (ssa_op->op1_use != current_var && ssa_op->op2_use != current_var) { ZEND_REGSET_INCL(regset, ZREG_REG0); } } if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { ZEND_REGSET_INCL(regset, ZREG_FPR0); } if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { ZEND_REGSET_INCL(regset, ZREG_FPR0); } if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { if (ssa_op->op1_use != current_var && ssa_op->op2_use != current_var) { ZEND_REGSET_INCL(regset, ZREG_FPR0); } } } break; case ZEND_BOOL: case ZEND_BOOL_NOT: case ZEND_JMPZ: case ZEND_JMPNZ: case ZEND_JMPZNZ: case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: op1_info = OP1_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; if (op1_info & MAY_BE_DOUBLE) { ZEND_REGSET_INCL(regset, ZREG_FPR0); } if (opline->opcode == ZEND_BOOL || opline->opcode == ZEND_BOOL_NOT || opline->opcode == ZEND_JMPZ_EX || opline->opcode == ZEND_JMPNZ_EX) { ZEND_REGSET_INCL(regset, ZREG_REG0); } } break; case ZEND_DO_UCALL: case ZEND_DO_FCALL: case ZEND_DO_FCALL_BY_NAME: case ZEND_INCLUDE_OR_EVAL: case ZEND_GENERATOR_CREATE: case ZEND_YIELD: case ZEND_YIELD_FROM: regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); break; default: break; } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (ssa_op == ssa->ops && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { ZEND_REGSET_INCL(regset, ZREG_REG0); ZEND_REGSET_INCL(regset, ZREG_REG1); } } return regset; } static size_t dasm_venners_size = 0; void **dasm_labels_veneers = NULL; static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset) { void *veneer; ptrdiff_t na; int n, m; /* try to reuse veneers for global labels */ if ((ins >> 16) == DASM_REL_LG && *(b-1) < 0 && dasm_labels_veneers[-*(b-1)]) { veneer = dasm_labels_veneers[-*(b-1)]; na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; n = (int)na; /* check if we can jump to veneer */ if ((ptrdiff_t)n != na) { /* pass */ } else if (!(ins & 0xf800)) { /* B, BL */ if ((n & 3) == 0 && ((n+0x08000000) >> 28) == 0) { return n; } } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ if ((n & 3) == 0 && ((n+0x00100000) >> 21) == 0) { return n; } } else if ((ins & 0x3000) == 0x2000) { /* ADR */ /* pass */ } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ /* pass */ } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ if ((n & 3) == 0 && ((n+0x00008000) >> 16) == 0) { return n; } } } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (ins >> 16) == DASM_REL_A) { ptrdiff_t addr = (((ptrdiff_t)(*(b-1))) << 32) | (unsigned int)(*(b-2)); if ((void*)addr >= dasm_buf && (void*)addr < dasm_end) { uint32_t exit_point = zend_jit_trace_find_exit_point((void*)addr); zend_jit_trace_info *t = zend_jit_get_current_trace_info(); if (exit_point != (uint32_t)-1) { /* Use exit points table */ ZEND_ASSERT(exit_point < t->exit_count); veneer = (char*)buffer + dasm_getpclabel(&Dst, 1) - (t->exit_count - exit_point) * 4; na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; n = (int)na; /* check if we can jump to veneer */ if ((ptrdiff_t)n != na) { ZEND_ASSERT(0); return 0; } else if (!(ins & 0xf800)) { /* B, BL */ if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) { ZEND_ASSERT(0); return 0; } } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) { ZEND_ASSERT(0); return 0; } } else if ((ins & 0x3000) == 0x2000) { /* ADR */ ZEND_ASSERT(0); return 0; } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ ZEND_ASSERT(0); return 0; } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) { ZEND_ASSERT(0); return 0; } } else { ZEND_ASSERT(0); return 0; } return n; } } } veneer = (char*)buffer + (Dst->codesize + dasm_venners_size); if (veneer > dasm_end) { return 0; /* jit_buffer_size overflow */ } na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; n = (int)na; /* check if we can jump to veneer */ if ((ptrdiff_t)n != na) { ZEND_ASSERT(0); return 0; } else if (!(ins & 0xf800)) { /* B, BL */ if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) { ZEND_ASSERT(0); return 0; } } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) { ZEND_ASSERT(0); return 0; } } else if ((ins & 0x3000) == 0x2000) { /* ADR */ ZEND_ASSERT(0); return 0; } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ ZEND_ASSERT(0); return 0; } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) { ZEND_ASSERT(0); return 0; } } else if ((ins & 0x8000)) { /* absolute */ ZEND_ASSERT(0); return 0; } else { ZEND_ASSERT(0); return 0; } // TODO: support for long veneers (above 128MB) ??? /* check if we can use B to jump from veneer */ na = (ptrdiff_t)cp + offset - (ptrdiff_t)veneer - 4; m = (int)na; if ((ptrdiff_t)m != na) { ZEND_ASSERT(0); return 0; } else if ((m & 3) != 0 || ((m+0x08000000) >> 28) != 0) { ZEND_ASSERT(0); return 0; } /* generate B instruction */ *(uint32_t*)veneer = 0x14000000 | ((m >> 2) & 0x03ffffff); dasm_venners_size += 4; if ((ins >> 16) == DASM_REL_LG && *(b-1) < 0) { /* reuse this veneer for the future jumps to global label */ dasm_labels_veneers[-*(b-1)] = veneer; /* Dst->globals[*(b-1)] = veneer; */ #ifdef HAVE_DISASM if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { const char *name = zend_jit_disasm_find_symbol((ptrdiff_t)cp + offset - 4, (int64_t *)(&offset)); if (name && !offset) { if (strstr(name, "@veneer") == NULL) { char *new_name; zend_spprintf(&new_name, 0, "%s@veneer", name); zend_jit_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4); efree(new_name); } else { zend_jit_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4); } } } #endif } return n; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * indent-tabs-mode: t * End: */