1/* 2 * +----------------------------------------------------------------------+ 3 * | Zend JIT | 4 * +----------------------------------------------------------------------+ 5 * | Copyright (c) The PHP Group | 6 * +----------------------------------------------------------------------+ 7 * | This source file is subject to version 3.01 of the PHP license, | 8 * | that is bundled with this package in the file LICENSE, and is | 9 * | available through the world-wide-web at the following url: | 10 * | https://www.php.net/license/3_01.txt | 11 * | If you did not receive a copy of the PHP license and are unable to | 12 * | obtain it through the world-wide-web, please send a note to | 13 * | license@php.net so we can mail you a copy immediately. | 14 * +----------------------------------------------------------------------+ 15 * | Authors: Dmitry Stogov <dmitry@php.net> | 16 * | Xinchen Hui <laruence@php.net> | 17 * +----------------------------------------------------------------------+ 18 */ 19 20|.if X64 21 |.arch x64 22|.else 23 |.arch x86 24|.endif 25 26|.if X64WIN 27 |.define FP, r14 28 |.define IP, r15 29 |.define IPl, r15d 30 |.define RX, r15 // the same as VM IP reused as a general purpose reg 31 |.define CARG1, rcx // x64/POSIX C call arguments. 32 |.define CARG2, rdx 33 |.define CARG3, r8 34 |.define CARG4, r9 35 |.define CARG1d, ecx 36 |.define CARG2d, edx 37 |.define CARG3d, r8d 38 |.define CARG4d, r9d 39 |.define FCARG1a, CARG1 // Simulate x86 fastcall. 40 |.define FCARG2a, CARG2 41 |.define FCARG1d, CARG1d 42 |.define FCARG2d, CARG2d 43 |.define SPAD, 0x58 // padding for CPU stack alignment 44 |.define NR_SPAD, 0x58 // padding for CPU stack alignment 45 |.define T3, [r4+0x50] // Used to store old value of IP 46 |.define T2, [r4+0x48] // Used to store old value of FP 47 |.define T1, [r4+0x40] 48 |.define A6, [r4+0x28] // preallocated slot for 6-th argument 49 |.define A5, [r4+0x20] // preallocated slot for 5-th argument 50|.elif X64 51 |.define FP, r14 52 |.define IP, r15 53 |.define IPl, r15d 54 |.define RX, r15 // the same as VM IP reused as a general purpose reg 55 |.define CARG1, rdi // x64/POSIX C call arguments. 56 |.define CARG2, rsi 57 |.define CARG3, rdx 58 |.define CARG4, rcx 59 |.define CARG5, r8 60 |.define CARG6, r9 61 |.define CARG1d, edi 62 |.define CARG2d, esi 63 |.define CARG3d, edx 64 |.define CARG4d, ecx 65 |.define CARG5d, r8d 66 |.define CARG6d, r9d 67 |.define FCARG1a, CARG1 // Simulate x86 fastcall. 68 |.define FCARG2a, CARG2 69 |.define FCARG1d, CARG1d 70 |.define FCARG2d, CARG2d 71 |.define SPAD, 0x18 // padding for CPU stack alignment 72 |.define NR_SPAD, 0x28 // padding for CPU stack alignment 73 |.define T3, [r4+0x20] // Used to store old value of IP (CALL VM only) 74 |.define T2, [r4+0x18] // Used to store old value of FP (CALL VM only) 75 |.define T1, [r4] 76|.else 77 |.define FP, esi 78 |.define IP, edi 79 |.define IPl, edi 80 |.define RX, edi // the same as VM IP reused as a general purpose reg 81 |.define FCARG1a, ecx // x86 fastcall arguments. 82 |.define FCARG2a, edx 83 |.define FCARG1d, ecx 84 |.define FCARG2d, edx 85 |.define SPAD, 0x1c // padding for CPU stack alignment 86 |.define NR_SPAD, 0x1c // padding for CPU stack alignment 87 |.define T3, [r4+0x18] // Used to store old value of IP (CALL VM only) 88 |.define T2, [r4+0x14] // Used to store old value of FP (CALL VM only) 89 |.define T1, [r4] 90 |.define A4, [r4+0xC] // preallocated slots for arguments of "cdecl" functions (intersect with T1) 91 |.define A3, [r4+0x8] 92 |.define A2, [r4+0x4] 93 |.define A1, [r4] 94|.endif 95 96|.define HYBRID_SPAD, 16 // padding for stack alignment 97 98#ifdef _WIN64 99# define TMP_ZVAL_OFFSET 0x20 100#else 101# define TMP_ZVAL_OFFSET 0 102#endif 103 104#define DASM_ALIGNMENT 16 105 106/* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to 107 * guarantee proper alignment of 128-bit SSE data allocated on stack. 108 * With broken alignment any execution of SSE code, including calls to 109 * memcpy() and others, may lead to crash. 110 */ 111 112const char* zend_reg_name[] = { 113#if defined(__x86_64__) || defined(_M_X64) 114 "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", 115 "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", 116 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", 117 "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15" 118#else 119 "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", 120 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" 121#endif 122}; 123 124/* Simulate x86 fastcall */ 125#ifdef _WIN64 126# define ZREG_FCARG1 ZREG_RCX 127# define ZREG_FCARG2 ZREG_RDX 128#elif defined(__x86_64__) 129# define ZREG_FCARG1 ZREG_RDI 130# define ZREG_FCARG2 ZREG_RSI 131#else 132# define ZREG_FCARG1 ZREG_RCX 133# define ZREG_FCARG2 ZREG_RDX 134#endif 135 136|.type EX, zend_execute_data, FP 137|.type OP, zend_op 138|.type ZVAL, zval 139|.actionlist dasm_actions 140|.globals zend_lb 141|.section code, cold_code, jmp_table 142 143static void* dasm_labels[zend_lb_MAX]; 144 145#if ZTS 146static size_t tsrm_ls_cache_tcb_offset = 0; 147static size_t tsrm_tls_index; 148static size_t tsrm_tls_offset; 149#endif 150 151#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) 152 153#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) 154 155/* Call range is before or after 2GB */ 156#define MAY_USE_32BIT_ADDR(addr) \ 157 (IS_SIGNED_32BIT((char*)(addr) - (char*)dasm_buf) && \ 158 IS_SIGNED_32BIT((char*)(addr) - (char*)dasm_end)) 159 160#define CAN_USE_AVX() (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) 161 162/* Not Implemented Yet */ 163|.macro NIY 164|| //ZEND_ASSERT(0); 165| int3 166|.endmacro 167 168|.macro NIY_STUB 169|| //ZEND_ASSERT(0); 170| int3 171|.endmacro 172 173|.macro ADD_HYBRID_SPAD 174||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 175| add r4, HYBRID_SPAD 176||#endif 177|.endmacro 178 179|.macro SUB_HYBRID_SPAD 180||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 181| sub r4, HYBRID_SPAD 182||#endif 183|.endmacro 184 185|.macro LOAD_ADDR, reg, addr 186| .if X64 187|| if (IS_SIGNED_32BIT(addr)) { 188| mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 <imm-32-bit> 189|| } else { 190| mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 <imm-64-bit> 191|| } 192| .else 193| mov reg, ((ptrdiff_t)addr) 194| .endif 195|.endmacro 196 197|.macro LOAD_TSRM_CACHE, reg 198| .if X64WIN 199| gs 200| mov reg, aword [0x58] 201| mov reg, aword [reg+tsrm_tls_index] 202| mov reg, aword [reg+tsrm_tls_offset] 203| .elif WIN 204| fs 205| mov reg, aword [0x2c] 206| mov reg, aword [reg+tsrm_tls_index] 207| mov reg, aword [reg+tsrm_tls_offset] 208| .elif X64APPLE 209| gs 210|| if (tsrm_ls_cache_tcb_offset) { 211| mov reg, aword [tsrm_ls_cache_tcb_offset] 212|| } else { 213| mov reg, aword [tsrm_tls_index] 214| mov reg, aword [reg+tsrm_tls_offset] 215|| } 216| .elif X64 217| fs 218|| if (tsrm_ls_cache_tcb_offset) { 219| mov reg, aword [tsrm_ls_cache_tcb_offset] 220|| } else { 221| mov reg, [0x8] 222| mov reg, aword [reg+tsrm_tls_index] 223| mov reg, aword [reg+tsrm_tls_offset] 224|| } 225| .else 226| gs 227|| if (tsrm_ls_cache_tcb_offset) { 228| mov reg, aword [tsrm_ls_cache_tcb_offset] 229|| } else { 230| mov reg, [0x4] 231| mov reg, aword [reg+tsrm_tls_index] 232| mov reg, aword [reg+tsrm_tls_offset] 233|| } 234| .endif 235|.endmacro 236 237|.macro LOAD_ADDR_ZTS, reg, struct, field 238| .if ZTS 239| LOAD_TSRM_CACHE reg 240| lea reg, aword [reg + (struct.._offset + offsetof(zend_..struct, field))] 241| .else 242| LOAD_ADDR reg, &struct.field 243| .endif 244|.endmacro 245 246|.macro PUSH_ADDR, addr, tmp_reg 247| .if X64 248|| if (IS_SIGNED_32BIT(addr)) { 249| push ((ptrdiff_t)addr) 250|| } else { 251| mov64 tmp_reg, ((ptrdiff_t)addr) 252| push tmp_reg 253|| } 254| .else 255| push ((ptrdiff_t)addr) 256| .endif 257|.endmacro 258 259|.macro ADDR_STORE, mem, addr, tmp_reg 260| .if X64 261|| if (IS_SIGNED_32BIT(addr)) { 262| mov mem, ((ptrdiff_t)addr) 263|| } else { 264| mov64 tmp_reg, ((ptrdiff_t)addr) 265| mov mem, tmp_reg 266|| } 267| .else 268| mov mem, ((ptrdiff_t)addr) 269| .endif 270|.endmacro 271 272|.macro ADDR_CMP, mem, addr, tmp_reg 273| .if X64 274|| if (IS_SIGNED_32BIT(addr)) { 275| cmp mem, ((ptrdiff_t)addr) 276|| } else { 277| mov64 tmp_reg, ((ptrdiff_t)addr) 278| cmp mem, tmp_reg 279|| } 280| .else 281| cmp mem, ((ptrdiff_t)addr) 282| .endif 283|.endmacro 284 285|.macro PUSH_ADDR_ZTS, struct, field, tmp_reg 286| .if ZTS 287| LOAD_TSRM_CACHE tmp_reg 288| lea tmp_reg, aword [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))] 289| push tmp_reg 290| .else 291| PUSH_ADDR &struct.field, tmp_reg 292| .endif 293|.endmacro 294 295|.macro _MEM_OP, mem_ins, prefix, addr, op2, tmp_reg 296| .if X64 297|| if (IS_SIGNED_32BIT(addr)) { 298| mem_ins prefix [addr], op2 299|| } else { 300| mov64 tmp_reg, ((ptrdiff_t)addr) 301| mem_ins prefix [tmp_reg], op2 302|| } 303| .else 304| mem_ins prefix [addr], op2 305| .endif 306|.endmacro 307 308|.macro MEM_LOAD_OP, mem_ins, reg, prefix, addr, tmp_reg 309| .if X64 310|| if (IS_SIGNED_32BIT(addr)) { 311| mem_ins reg, prefix [addr] 312|| } else { 313| mov64 tmp_reg, ((ptrdiff_t)addr) 314| mem_ins reg, prefix [tmp_reg] 315|| } 316| .else 317| mem_ins reg, prefix [addr] 318| .endif 319|.endmacro 320 321|.macro MEM_LOAD, op1, prefix, addr, tmp_reg 322| MEM_LOAD_OP mov, op1, prefix, addr, tmp_reg 323|.endmacro 324 325|.macro _MEM_OP_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg 326| .if ZTS 327| LOAD_TSRM_CACHE tmp_reg 328| mem_ins prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))], op2 329| .else 330| _MEM_OP mem_ins, prefix, &struct.field, op2, tmp_reg 331| .endif 332|.endmacro 333 334|.macro MEM_STORE_ZTS, prefix, struct, field, op2, tmp_reg 335| _MEM_OP_ZTS mov, prefix, struct, field, op2, tmp_reg 336|.endmacro 337 338|.macro MEM_CMP_ZTS, prefix, struct, field, op2, tmp_reg 339| _MEM_OP_ZTS cmp, prefix, struct, field, op2, tmp_reg 340|.endmacro 341 342|.macro MEM_UPDATE_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg 343| _MEM_OP_ZTS mem_ins, prefix, struct, field, op2, tmp_reg 344|.endmacro 345 346|.macro MEM_LOAD_OP_ZTS, mem_ins, reg, prefix, struct, field, tmp_reg 347| .if ZTS 348| LOAD_TSRM_CACHE tmp_reg 349| mem_ins reg, prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))] 350| .else 351| MEM_LOAD_OP mem_ins, reg, prefix, &struct.field, tmp_reg 352| .endif 353|.endmacro 354 355|.macro MEM_LOAD_ZTS, reg, prefix, struct, field, tmp_reg 356| MEM_LOAD_OP_ZTS mov, reg, prefix, struct, field, tmp_reg 357|.endmacro 358 359|.macro EXT_CALL, func, tmp_reg 360| .if X64 361|| if (MAY_USE_32BIT_ADDR(func)) { 362| call qword &func 363|| } else { 364| LOAD_ADDR tmp_reg, func 365| call tmp_reg 366|| } 367| .else 368| call dword &func 369| .endif 370|.endmacro 371 372|.macro EXT_JMP, func, tmp_reg 373| .if X64 374|| if (MAY_USE_32BIT_ADDR(func)) { 375| jmp qword &func 376|| } else { 377| LOAD_ADDR tmp_reg, func 378| jmp tmp_reg 379|| } 380| .else 381| jmp dword &func 382| .endif 383|.endmacro 384 385|.macro SAVE_IP 386|| if (GCC_GLOBAL_REGS) { 387| mov aword EX->opline, IP 388|| } 389|.endmacro 390 391|.macro LOAD_IP 392|| if (GCC_GLOBAL_REGS) { 393| mov IP, aword EX->opline 394|| } 395|.endmacro 396 397|.macro LOAD_IP_ADDR, addr 398|| if (GCC_GLOBAL_REGS) { 399| LOAD_ADDR IP, addr 400|| } else { 401| ADDR_STORE aword EX->opline, addr, RX 402|| } 403|.endmacro 404 405|.macro LOAD_IP_ADDR_ZTS, struct, field 406| .if ZTS 407|| if (GCC_GLOBAL_REGS) { 408| LOAD_TSRM_CACHE IP 409| lea IP, aword [IP + (struct.._offset + offsetof(zend_..struct, field))] 410|| } else { 411| LOAD_TSRM_CACHE RX 412| lea RX, aword [RX + (struct.._offset + offsetof(zend_..struct, field))] 413| mov aword EX->opline, RX 414|| } 415| .else 416| LOAD_IP_ADDR &struct.field 417| .endif 418|.endmacro 419 420|.macro GET_IP, reg 421|| if (GCC_GLOBAL_REGS) { 422| mov reg, IP 423|| } else { 424| mov reg, aword EX->opline 425|| } 426|.endmacro 427 428|.macro ADD_IP, val 429|| if (GCC_GLOBAL_REGS) { 430| add IP, val 431|| } else { 432| add aword EX->opline, val 433|| } 434|.endmacro 435 436|.macro JMP_IP 437|| if (GCC_GLOBAL_REGS) { 438| jmp aword [IP] 439|| } else { 440| mov r0, aword EX:FCARG1a->opline 441| jmp aword [r0] 442|| } 443|.endmacro 444 445/* In 64-bit build we compare only low 32-bits. 446 * x86_64 cmp instruction doesn't support immediate 64-bit operand, and full 447 * comparison would require an additional load of 64-bit address into register. 448 * This is not a problem at all, while JIT buffer size is less than 4GB. 449 */ 450|.macro CMP_IP, addr 451|| if (GCC_GLOBAL_REGS) { 452| cmp IPl, addr 453|| } else { 454| cmp dword EX->opline, addr 455|| } 456|.endmacro 457 458|.macro LOAD_ZVAL_ADDR, reg, addr 459|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 460| LOAD_ADDR reg, Z_ZV(addr) 461|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 462|| if (Z_OFFSET(addr)) { 463| lea reg, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 464|| } else { 465| mov reg, Ra(Z_REG(addr)) 466|| } 467|| } else { 468|| ZEND_UNREACHABLE(); 469|| } 470|.endmacro 471 472|.macro PUSH_ZVAL_ADDR, addr, tmp_reg 473|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 474| PUSH_ADDR Z_ZV(addr), tmp_reg 475|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 476|| if (Z_OFFSET(addr)) { 477| lea tmp_reg, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 478| push tmp_reg 479|| } else { 480| push Ra(Z_REG(addr)) 481|| } 482|| } else { 483|| ZEND_UNREACHABLE(); 484|| } 485|.endmacro 486 487|.macro GET_Z_TYPE_INFO, reg, zv 488| mov reg, dword [zv+offsetof(zval,u1.type_info)] 489|.endmacro 490 491|.macro SET_Z_TYPE_INFO, zv, type 492| mov dword [zv+offsetof(zval,u1.type_info)], type 493|.endmacro 494 495|.macro GET_ZVAL_TYPE, reg, addr 496|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 497| mov reg, byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.v.type)] 498|.endmacro 499 500|.macro GET_ZVAL_TYPE_INFO, reg, addr 501|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 502| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)] 503|.endmacro 504 505|.macro SET_ZVAL_TYPE_INFO, addr, type 506|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 507| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)], type 508|.endmacro 509 510|.macro GET_Z_PTR, reg, zv 511| mov reg, aword [zv] 512|.endmacro 513 514|.macro GET_Z_W2, reg, zv 515| mov reg, dword [zv+4] 516|.endmacro 517 518|.macro SET_Z_W2, zv, reg 519| mov dword [zv+4], reg 520|.endmacro 521 522|.macro GET_ZVAL_PTR, reg, addr 523|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 524| mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 525|.endmacro 526 527|.macro SET_ZVAL_PTR, addr, val 528|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 529| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], val 530|.endmacro 531 532|.macro GET_ZVAL_W2, reg, addr 533|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 534| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4] 535|.endmacro 536 537|.macro SET_ZVAL_W2, addr, val 538|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 539| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4], val 540|.endmacro 541 542|.macro UNDEF_OPLINE_RESULT 543| mov r0, EX->opline 544| mov eax, dword OP:r0->result.var 545| SET_Z_TYPE_INFO FP + r0, IS_UNDEF 546|.endmacro 547 548|.macro UNDEF_OPLINE_RESULT_IF_USED 549| test byte OP:RX->result_type, (IS_TMP_VAR|IS_VAR) 550| jz >1 551| mov eax, dword OP:RX->result.var 552| SET_Z_TYPE_INFO FP + r0, IS_UNDEF 553|1: 554|.endmacro 555 556|.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2 557|| if (CAN_USE_AVX()) { 558| avx_ins op1, op2 559|| } else { 560| sse_ins op1, op2 561|| } 562|.endmacro 563 564|.macro SSE_OP, sse_ins, reg, addr, tmp_reg 565|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 566| MEM_LOAD_OP sse_ins, xmm(reg-ZREG_XMM0), qword, Z_ZV(addr), tmp_reg 567|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 568| sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 569|| } else if (Z_MODE(addr) == IS_REG) { 570| sse_ins xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 571|| } else { 572|| ZEND_UNREACHABLE(); 573|| } 574|.endmacro 575 576|.macro DOUBLE_CMP, reg, addr 577|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 578| .if X64 579|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 580| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 581|| } else { 582| LOAD_ADDR r0, Z_ZV(addr) 583| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [r0] 584|| } 585| .else 586| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 587| .endif 588|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 589| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 590|| } else if (Z_MODE(addr) == IS_REG) { 591| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 592|| } else { 593|| ZEND_UNREACHABLE(); 594|| } 595|.endmacro 596 597|.macro DOUBLE_GET_LONG, reg, lval, tmp_reg 598|| if (lval == 0) { 599|| if (CAN_USE_AVX()) { 600| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 601|| } else { 602| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 603|| } 604|| } else { 605|.if X64 606|| if (!IS_SIGNED_32BIT(lval)) { 607| mov64 Ra(tmp_reg), lval 608|| } else { 609| mov Ra(tmp_reg), lval 610|| } 611|.else 612| mov Ra(tmp_reg), lval 613|.endif 614|| if (CAN_USE_AVX()) { 615| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 616| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(tmp_reg) 617|| } else { 618| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 619| cvtsi2sd, xmm(reg-ZREG_XMM0), Ra(tmp_reg) 620|| } 621|| } 622|.endmacro 623 624|.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg 625|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 626| DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg 627|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 628|| if (CAN_USE_AVX()) { 629| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 630| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 631|| } else { 632| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 633| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 634|| } 635|| } else if (Z_MODE(addr) == IS_REG) { 636|| if (CAN_USE_AVX()) { 637| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 638| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) 639|| } else { 640| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 641| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) 642|| } 643|| } else { 644|| ZEND_UNREACHABLE(); 645|| } 646|.endmacro 647 648|.macro DOUBLE_GET_ZVAL_DVAL, reg, addr 649|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { 650|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 651| .if X64 652|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 653| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 654|| } else { 655| LOAD_ADDR r0, Z_ZV(addr) 656| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [r0] 657|| } 658| .else 659| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 660| .endif 661|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 662| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 663|| } else if (Z_MODE(addr) == IS_REG) { 664| SSE_AVX_INS movaps, vmovaps, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 665|| } else { 666|| ZEND_UNREACHABLE(); 667|| } 668|| } 669|.endmacro 670 671|.macro SSE_MATH, opcode, reg, addr, tmp_reg 672|| switch (opcode) { 673|| case ZEND_ADD: 674| SSE_OP addsd, reg, addr, tmp_reg 675|| break; 676|| case ZEND_SUB: 677| SSE_OP subsd, reg, addr, tmp_reg 678|| break; 679|| case ZEND_MUL: 680| SSE_OP mulsd, reg, addr, tmp_reg 681|| break; 682|| case ZEND_DIV: 683| SSE_OP divsd, reg, addr, tmp_reg 684|| break; 685|| } 686|.endmacro 687 688|.macro SSE_MATH_REG, opcode, dst_reg, src_reg 689|| switch (opcode) { 690|| case ZEND_ADD: 691| addsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 692|| break; 693|| case ZEND_SUB: 694| subsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 695|| break; 696|| case ZEND_MUL: 697| mulsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 698|| break; 699|| case ZEND_DIV: 700| divsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 701|| break; 702|| } 703|.endmacro 704 705|.macro DOUBLE_SET_ZVAL_DVAL, addr, reg 706|| if (Z_MODE(addr) == IS_REG) { 707|| if (reg != Z_REG(addr)) { 708| SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0) 709|| } 710|| } else { 711|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 712| SSE_AVX_INS movsd, vmovsd, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0) 713|| } 714|.endmacro 715 716|.macro AVX_OP, avx_ins, reg, op1_reg, addr, tmp_reg 717|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 718| .if X64 719|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 720| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Z_ZV(addr)] 721|| } else { 722| mov64 tmp_reg, ((ptrdiff_t)Z_ZV(addr)) 723| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [tmp_reg] 724|| } 725| .else 726| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [addr] 727| .endif 728|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 729| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 730|| } else if (Z_MODE(addr) == IS_REG) { 731| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 732|| } else { 733|| ZEND_UNREACHABLE(); 734|| } 735|.endmacro 736 737|.macro AVX_MATH, opcode, reg, op1_reg, addr, tmp_reg 738|| switch (opcode) { 739|| case ZEND_ADD: 740| AVX_OP vaddsd, reg, op1_reg, addr, tmp_reg 741|| break; 742|| case ZEND_SUB: 743| AVX_OP vsubsd, reg, op1_reg, addr, tmp_reg 744|| break; 745|| case ZEND_MUL: 746| AVX_OP vmulsd, reg, op1_reg, addr, tmp_reg 747|| break; 748|| case ZEND_DIV: 749| AVX_OP vdivsd, reg, op1_reg, addr, tmp_reg 750|| break; 751|| } 752|.endmacro 753 754|.macro AVX_MATH_REG, opcode, dst_reg, op1_reg, src_reg 755|| switch (opcode) { 756|| case ZEND_ADD: 757| vaddsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 758|| break; 759|| case ZEND_SUB: 760| vsubsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 761|| break; 762|| case ZEND_MUL: 763| vmulsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 764|| break; 765|| case ZEND_DIV: 766| vdivsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 767|| break; 768|| } 769|.endmacro 770 771|.macro LONG_OP, long_ins, reg, addr, tmp_reg 772|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 773| .if X64 774|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { 775| mov64 tmp_reg, Z_LVAL_P(Z_ZV(addr)) 776| long_ins Ra(reg), tmp_reg 777|| } else { 778| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) 779|| } 780| .else 781| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) 782| .endif 783|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 784| long_ins Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 785|| } else if (Z_MODE(addr) == IS_REG) { 786| long_ins Ra(reg), Ra(Z_REG(addr)) 787|| } else { 788|| ZEND_UNREACHABLE(); 789|| } 790|.endmacro 791 792|.macro LONG_OP_WITH_32BIT_CONST, long_ins, op1_addr, lval 793|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { 794| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 795|| } else if (Z_MODE(op1_addr) == IS_REG) { 796| long_ins Ra(Z_REG(op1_addr)), lval 797|| } else { 798|| ZEND_UNREACHABLE(); 799|| } 800|.endmacro 801 802|.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval 803|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { 804| .if X64 805|| if (!IS_SIGNED_32BIT(lval)) { 806| mov64 r0, lval 807| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0 808|| } else { 809| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 810|| } 811| .else 812| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 813| .endif 814|| } else if (Z_MODE(op1_addr) == IS_REG) { 815| .if X64 816|| if (!IS_SIGNED_32BIT(lval)) { 817| mov64 r0, lval 818| long_ins Ra(Z_REG(op1_addr)), r0 819|| } else { 820| long_ins Ra(Z_REG(op1_addr)), lval 821|| } 822| .else 823| long_ins Ra(Z_REG(op1_addr)), lval 824| .endif 825|| } else { 826|| ZEND_UNREACHABLE(); 827|| } 828|.endmacro 829 830|.macro GET_ZVAL_LVAL, reg, addr 831|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 832|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { 833| xor Ra(reg), Ra(reg) 834|| } else { 835| .if X64 836|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { 837| mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr)) 838|| } else { 839| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) 840|| } 841| .else 842| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) 843| .endif 844|| } 845|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 846| mov Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 847|| } else if (Z_MODE(addr) == IS_REG) { 848|| if (reg != Z_REG(addr)) { 849| mov Ra(reg), Ra(Z_REG(addr)) 850|| } 851|| } else { 852|| ZEND_UNREACHABLE(); 853|| } 854|.endmacro 855 856|.macro LONG_MATH, opcode, reg, addr, tmp_reg 857|| switch (opcode) { 858|| case ZEND_ADD: 859| LONG_OP add, reg, addr, Ra(tmp_reg) 860|| break; 861|| case ZEND_SUB: 862| LONG_OP sub, reg, addr, Ra(tmp_reg) 863|| break; 864|| case ZEND_MUL: 865| LONG_OP imul, reg, addr, Ra(tmp_reg) 866|| break; 867|| case ZEND_BW_OR: 868| LONG_OP or, reg, addr, Ra(tmp_reg) 869|| break; 870|| case ZEND_BW_AND: 871| LONG_OP and, reg, addr, Ra(tmp_reg) 872|| break; 873|| case ZEND_BW_XOR: 874| LONG_OP xor, reg, addr, Ra(tmp_reg) 875|| break; 876|| default: 877|| ZEND_UNREACHABLE(); 878|| } 879|.endmacro 880 881|.macro LONG_MATH_REG, opcode, dst_reg, src_reg 882|| switch (opcode) { 883|| case ZEND_ADD: 884| add dst_reg, src_reg 885|| break; 886|| case ZEND_SUB: 887| sub dst_reg, src_reg 888|| break; 889|| case ZEND_MUL: 890| imul dst_reg, src_reg 891|| break; 892|| case ZEND_BW_OR: 893| or dst_reg, src_reg 894|| break; 895|| case ZEND_BW_AND: 896| and dst_reg, src_reg 897|| break; 898|| case ZEND_BW_XOR: 899| xor dst_reg, src_reg 900|| break; 901|| default: 902|| ZEND_UNREACHABLE(); 903|| } 904|.endmacro 905 906|.macro SET_ZVAL_LVAL, addr, lval 907|| if (Z_MODE(addr) == IS_REG) { 908| mov Ra(Z_REG(addr)), lval 909|| } else { 910|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 911| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval 912|| } 913|.endmacro 914 915|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg 916|| if (Z_TYPE_P(zv) > IS_TRUE) { 917|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 918|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; 919|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { 920|| if (CAN_USE_AVX()) { 921| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 922|| } else { 923| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 924|| } 925| .if X64 926|| } else if (!IS_SIGNED_32BIT(zv)) { 927| mov64 Ra(tmp_reg), ((uintptr_t)zv) 928| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] 929| .endif 930|| } else { 931| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] 932|| } 933| DOUBLE_SET_ZVAL_DVAL dst_addr, dst_reg 934|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 935|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; 936| DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), ZREG_R0 937| DOUBLE_SET_ZVAL_DVAL dst_addr, dst_reg 938|| } else if (Z_LVAL_P(zv) == 0 && Z_MODE(dst_addr) == IS_REG) { 939| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) 940|| } else { 941| .if X64 942|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 943|| if (Z_MODE(dst_addr) == IS_REG) { 944| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) 945|| } else { 946| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) 947| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) 948|| } 949|| } else { 950| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 951|| } 952| .else 953| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 954| .endif 955|| } 956|| } 957|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 958|| if (dst_def_info == MAY_BE_DOUBLE) { 959|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 960| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE 961|| } 962|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 963| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) 964|| } 965|| } 966|.endmacro 967 968|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg 969|| if (Z_TYPE_P(zv) > IS_TRUE) { 970|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 971|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? 972|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0); 973|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { 974|| if (CAN_USE_AVX()) { 975| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 976|| } else { 977| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 978|| } 979| .if X64 980|| } else if (!IS_SIGNED_32BIT(zv)) { 981| mov64 Ra(tmp_reg), ((uintptr_t)zv) 982| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] 983| .endif 984|| } else { 985| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] 986|| } 987| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 988| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 989|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 990|| if (Z_MODE(dst_addr) == IS_REG) { 991| DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), ZREG_R0 992| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) 993|| } else if (Z_MODE(res_addr) == IS_REG) { 994| DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), ZREG_R0 995| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) 996|| } else { 997| DOUBLE_GET_LONG ZREG_XMM0, Z_LVAL_P(zv), ZREG_R0 998| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 999| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 1000|| } 1001|| } else if (Z_LVAL_P(zv) == 0 && (Z_MODE(dst_addr) == IS_REG || Z_MODE(res_addr) == IS_REG)) { 1002|| if (Z_MODE(dst_addr) == IS_REG) { 1003| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) 1004| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1005|| } else { 1006| xor Ra(Z_REG(res_addr)), Ra(Z_REG(res_addr)) 1007| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1008|| } 1009|| } else { 1010| .if X64 1011|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1012|| if (Z_MODE(dst_addr) == IS_REG) { 1013| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) 1014| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1015|| } else if (Z_MODE(res_addr) == IS_REG) { 1016| mov64 Ra(Z_REG(res_addr)), ((uintptr_t)Z_LVAL_P(zv)) 1017| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1018|| } else { 1019| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) 1020| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) 1021| SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 1022|| } 1023|| } else if (Z_MODE(dst_addr) == IS_REG) { 1024| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1025| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1026|| } else if (Z_MODE(res_addr) == IS_REG) { 1027| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1028| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1029|| } else { 1030| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1031| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1032|| } 1033| .else 1034|| if (Z_MODE(dst_addr) == IS_REG) { 1035| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1036| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1037|| } else if (Z_MODE(res_addr) == IS_REG) { 1038| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1039| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1040|| } else { 1041| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1042| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1043|| } 1044| .endif 1045|| } 1046|| } 1047|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1048|| if (dst_def_info == MAY_BE_DOUBLE) { 1049|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1050| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE 1051|| } 1052|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1053| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) 1054|| } 1055|| } 1056|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1057|| if (dst_def_info == MAY_BE_DOUBLE) { 1058| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 1059|| } else { 1060| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv) 1061|| } 1062|| } 1063|.endmacro 1064 1065/* the same as above, but "src" may overlap with "tmp_reg1" */ 1066|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1067| ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1068|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1069|| !(src_info & MAY_BE_GUARD) && 1070|| has_concrete_type(src_info & MAY_BE_ANY)) { 1071|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1072|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) { 1073|| zend_uchar type = concrete_type(src_info); 1074| SET_ZVAL_TYPE_INFO dst_addr, type 1075|| } 1076|| } 1077|| } else { 1078| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr 1079| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) 1080|| } 1081|.endmacro 1082 1083|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1084|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1085|| if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) { 1086|| if (Z_MODE(src_addr) == IS_REG) { 1087|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1088| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) 1089|| } 1090|| } else if (Z_MODE(dst_addr) == IS_REG) { 1091| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr 1092|| } else { 1093| GET_ZVAL_LVAL tmp_reg2, src_addr 1094| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) 1095|| } 1096|| } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 1097|| if (Z_MODE(src_addr) == IS_REG) { 1098| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) 1099|| } else if (Z_MODE(dst_addr) == IS_REG) { 1100| DOUBLE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr 1101|| } else { 1102| DOUBLE_GET_ZVAL_DVAL ZREG_XMM0, src_addr 1103| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1104|| } 1105|| } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) { 1106| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1107| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1108|| } else { 1109| .if X64 1110| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1111| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1112| .else 1113|| if ((tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr))) { 1114| GET_ZVAL_W2 Ra(tmp_reg2), src_addr 1115| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) 1116| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1117| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1118|| } else { 1119| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1120| GET_ZVAL_W2 Ra(tmp_reg1), src_addr 1121| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1122| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) 1123|| } 1124| .endif 1125|| } 1126|| } 1127|.endmacro 1128 1129|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, tmp_reg1, tmp_reg2 1130|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1131|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) { 1132|| if (Z_MODE(src_addr) == IS_REG) { 1133|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1134| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) 1135|| } 1136|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) { 1137| SET_ZVAL_LVAL res_addr, Ra(Z_REG(src_addr)) 1138|| } 1139|| } else if (Z_MODE(dst_addr) == IS_REG) { 1140| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr 1141|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) { 1142| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1143|| } 1144|| } else if (Z_MODE(res_addr) == IS_REG) { 1145| GET_ZVAL_LVAL Z_REG(res_addr), src_addr 1146| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1147|| } else { 1148| GET_ZVAL_LVAL tmp_reg2, src_addr 1149| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) 1150| SET_ZVAL_LVAL res_addr, Ra(tmp_reg2) 1151|| } 1152|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 1153|| if (Z_MODE(src_addr) == IS_REG) { 1154| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) 1155| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(src_addr) 1156|| } else if (Z_MODE(dst_addr) == IS_REG) { 1157| DOUBLE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr 1158| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) 1159|| } else if (Z_MODE(res_addr) == IS_REG) { 1160| DOUBLE_GET_ZVAL_DVAL Z_REG(res_addr), src_addr 1161| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) 1162|| } else { 1163| DOUBLE_GET_ZVAL_DVAL ZREG_XMM0, src_addr 1164| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1165| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 1166|| } 1167|| } else if (!(src_info & MAY_BE_DOUBLE)) { 1168| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1169| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1170| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1171|| } else { 1172| .if X64 1173| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1174| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1175| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1176| .else 1177|| if (tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) { 1178| GET_ZVAL_W2 Ra(tmp_reg2), src_addr 1179| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) 1180| SET_ZVAL_W2 res_addr, Ra(tmp_reg2) 1181| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1182| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1183| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1184|| } else { 1185| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1186| GET_ZVAL_W2 Ra(tmp_reg1), src_addr 1187| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1188| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1189| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) 1190| SET_ZVAL_W2 res_addr, Ra(tmp_reg1) 1191|| } 1192| .endif 1193|| } 1194|| } 1195|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1196|| has_concrete_type(src_info & MAY_BE_ANY)) { 1197|| zend_uchar type = concrete_type(src_info); 1198|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1199|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { 1200| SET_ZVAL_TYPE_INFO dst_addr, type 1201|| } 1202|| } 1203|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1204| SET_ZVAL_TYPE_INFO res_addr, type 1205|| } 1206|| } else { 1207| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr 1208| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) 1209| SET_ZVAL_TYPE_INFO res_addr, Rd(tmp_reg1) 1210|| } 1211|.endmacro 1212 1213|.macro IF_UNDEF, type_reg, label 1214| test type_reg, type_reg 1215| je label 1216|.endmacro 1217 1218|.macro IF_TYPE, type, val, label 1219| cmp type, val 1220| je label 1221|.endmacro 1222 1223|.macro IF_NOT_TYPE, type, val, label 1224| cmp type, val 1225| jne label 1226|.endmacro 1227 1228|.macro IF_Z_TYPE, zv, val, label 1229| IF_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label 1230|.endmacro 1231 1232|.macro IF_NOT_Z_TYPE, zv, val, label 1233| IF_NOT_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label 1234|.endmacro 1235 1236|.macro CMP_ZVAL_TYPE, addr, val 1237|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1238| cmp byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val 1239|.endmacro 1240 1241|.macro IF_ZVAL_TYPE, addr, val, label 1242|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1243| IF_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label 1244|.endmacro 1245 1246|.macro IF_NOT_ZVAL_TYPE, addr, val, label 1247|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1248| IF_NOT_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label 1249|.endmacro 1250 1251|.macro IF_FLAGS, type_flags, mask, label 1252| test type_flags, mask 1253| jnz label 1254|.endmacro 1255 1256|.macro IF_NOT_FLAGS, type_flags, mask, label 1257| test type_flags, mask 1258| jz label 1259|.endmacro 1260 1261|.macro IF_REFCOUNTED, type_flags, label 1262| IF_FLAGS type_flags, IS_TYPE_REFCOUNTED, label 1263|.endmacro 1264 1265|.macro IF_NOT_REFCOUNTED, type_flags, label 1266| //IF_NOT_FLAGS type_flags, IS_TYPE_REFCOUNTED, label 1267| test type_flags, type_flags 1268| jz label 1269|.endmacro 1270 1271|.macro IF_ZVAL_FLAGS, addr, mask, label 1272|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1273| IF_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label 1274|.endmacro 1275 1276|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label 1277|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1278| IF_NOT_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label 1279|.endmacro 1280 1281|.macro IF_ZVAL_REFCOUNTED, addr, label 1282| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label 1283|.endmacro 1284 1285|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label 1286| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label 1287|.endmacro 1288 1289|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label 1290| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label 1291|.endmacro 1292 1293|.macro GC_ADDREF, zv 1294| add dword [zv], 1 1295|.endmacro 1296 1297|.macro GC_DELREF, zv 1298| sub dword [zv], 1 1299|.endmacro 1300 1301|.macro IF_GC_MAY_NOT_LEAK, ptr, label 1302| test dword [ptr+4],(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)) 1303| jne label 1304|.endmacro 1305 1306|.macro ADDREF_CONST, zv, tmp_reg 1307| .if X64 1308|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1309| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) 1310| add dword [tmp_reg], 1 1311|| } else { 1312| add dword [Z_LVAL_P(zv)], 1 1313|| } 1314| .else 1315| add dword [Z_LVAL_P(zv)], 1 1316| .endif 1317|.endmacro 1318 1319|.macro ADDREF_CONST_2, zv, tmp_reg 1320| .if X64 1321|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1322| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) 1323| add dword [tmp_reg], 2 1324|| } else { 1325| add dword [Z_LVAL_P(zv)], 2 1326|| } 1327| .else 1328| add dword [Z_LVAL_P(zv)], 2 1329| .endif 1330|.endmacro 1331 1332|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg 1333|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1334|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1335| IF_NOT_REFCOUNTED type_flags_reg, >1 1336|| } 1337| GC_ADDREF value_ptr_reg 1338|1: 1339|| } 1340|.endmacro 1341 1342|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg 1343|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1344|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1345| IF_NOT_REFCOUNTED type_flags_reg, >1 1346|| } 1347| add dword [value_ptr_reg], 2 1348|1: 1349|| } 1350|.endmacro 1351 1352|.macro ZVAL_DEREF, reg, info 1353|| if (info & MAY_BE_REF) { 1354| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1 1355| GET_Z_PTR reg, reg 1356| add reg, offsetof(zend_reference, val) 1357|1: 1358|| } 1359|.endmacro 1360 1361|.macro SET_EX_OPLINE, op, tmp_reg 1362|| if (op == last_valid_opline) { 1363|| zend_jit_use_last_valid_opline(); 1364| SAVE_IP 1365|| } else { 1366| ADDR_STORE aword EX->opline, op, tmp_reg 1367|| if (!GCC_GLOBAL_REGS) { 1368|| zend_jit_reset_last_valid_opline(); 1369|| } 1370|| } 1371|.endmacro 1372 1373// zval should be in FCARG1a 1374|.macro ZVAL_DTOR_FUNC, var_info, opline // arg1 must be in FCARG1a 1375|| do { 1376|| if (!((var_info) & MAY_BE_GUARD) 1377|| && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1378|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); 1379|| if (type == IS_STRING && !ZEND_DEBUG) { 1380| EXT_CALL _efree, r0 1381|| break; 1382|| } else if (type == IS_ARRAY) { 1383|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { 1384|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { 1385| SET_EX_OPLINE opline, r0 1386|| } 1387| EXT_CALL zend_array_destroy, r0 1388|| } else { 1389| EXT_CALL zend_jit_array_free, r0 1390|| } 1391|| break; 1392|| } else if (type == IS_OBJECT) { 1393|| if (opline) { 1394| SET_EX_OPLINE opline, r0 1395|| } 1396| EXT_CALL zend_objects_store_del, r0 1397|| break; 1398|| } 1399|| } 1400|| if (opline) { 1401| SET_EX_OPLINE opline, r0 1402|| } 1403| EXT_CALL rc_dtor_func, r0 1404|| } while(0); 1405|.endmacro 1406 1407|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline 1408|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) { 1409|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1410| // if (Z_REFCOUNTED_P(cv)) { 1411|| if (cold) { 1412| IF_ZVAL_REFCOUNTED addr, >1 1413|.cold_code 1414|1: 1415|| } else { 1416| IF_NOT_ZVAL_REFCOUNTED addr, >4 1417|| } 1418|| } 1419| // if (!Z_DELREF_P(cv)) { 1420| GET_ZVAL_PTR FCARG1a, addr 1421| GC_DELREF FCARG1a 1422|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) { 1423|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) { 1424|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1425| jnz >3 1426|| } else { 1427| jnz >4 1428|| } 1429|| } 1430| // zval_dtor_func(r); 1431| ZVAL_DTOR_FUNC op_info, opline 1432|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1433| jmp >4 1434|| } 1435|3: 1436|| } 1437|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1438|| if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) { 1439|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 1440| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1 1441| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4 1442| GET_ZVAL_PTR FCARG1a, ref_addr 1443|1: 1444|| } 1445| IF_GC_MAY_NOT_LEAK FCARG1a, >4 1446| // gc_possible_root(Z_COUNTED_P(z)) 1447| EXT_CALL gc_possible_root, r0 1448|| } 1449|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { 1450| jmp >4 1451|.code 1452|| } 1453|4: 1454|| } 1455|.endmacro 1456 1457|.macro FREE_OP, op_type, op, op_info, cold, opline 1458|| if (op_type & (IS_VAR|IS_TMP_VAR)) { 1459| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, opline 1460|| } 1461|.endmacro 1462 1463|.macro SEPARATE_ARRAY, addr, op_info, cold 1464|| if (RC_MAY_BE_N(op_info)) { 1465|| if (Z_REG(addr) != ZREG_FP) { 1466| GET_ZVAL_LVAL ZREG_R0, addr 1467|| if (RC_MAY_BE_1(op_info)) { 1468| cmp dword [r0], 1 // if (GC_REFCOUNT() > 1) 1469| jbe >2 1470|| } 1471|| if (Z_REG(addr) != ZREG_FCARG1 || Z_OFFSET(addr) != 0) { 1472| LOAD_ZVAL_ADDR FCARG1a, addr 1473|| } 1474| EXT_CALL zend_jit_zval_array_dup, r0 1475|2: 1476| mov FCARG1a, r0 1477|| } else { 1478| GET_ZVAL_LVAL ZREG_FCARG1, addr 1479|| if (RC_MAY_BE_1(op_info)) { 1480| cmp dword [FCARG1a], 1 // if (GC_REFCOUNT() > 1) 1481|| if (cold) { 1482| ja >1 1483|.cold_code 1484|1: 1485|| } else { 1486| jbe >2 1487|| } 1488|| } 1489| IF_NOT_ZVAL_REFCOUNTED addr, >1 1490| GC_DELREF FCARG1a 1491|1: 1492| EXT_CALL zend_array_dup, r0 1493| SET_ZVAL_PTR addr, r0 1494| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX 1495| mov FCARG1a, r0 1496|| if (RC_MAY_BE_1(op_info)) { 1497|| if (cold) { 1498| jmp >2 1499|.code 1500|| } 1501|| } 1502|2: 1503|| } 1504|| } else { 1505| GET_ZVAL_LVAL ZREG_FCARG1, addr 1506|| } 1507|.endmacro 1508 1509|.macro EFREE_REG_REFERENCE 1510||#if ZEND_DEBUG 1511| xor FCARG2a, FCARG2a // filename 1512| .if X64WIN 1513| xor CARG3d, CARG3d // lineno 1514| xor CARG4, CARG4 1515| mov aword A5, 0 1516| EXT_CALL _efree, r0 1517| .elif X64 1518| xor CARG3d, CARG3d // lineno 1519| xor CARG4, CARG4 1520| xor CARG5, CARG5 1521| EXT_CALL _efree, r0 1522| .else 1523| sub r4, 4 1524| push 0 1525| push 0 1526| push 0 // lineno 1527| EXT_CALL _efree, r0 1528| add r4, 4 1529| .endif 1530||#else 1531||#ifdef HAVE_BUILTIN_CONSTANT_P 1532| EXT_CALL _efree_32, r0 1533||#else 1534| EXT_CALL _efree, r0 1535||#endif 1536||#endif 1537|.endmacro 1538 1539|.macro EFREE_REFERENCE, ptr 1540| mov FCARG1a, ptr 1541| EFREE_REG_REFERENCE 1542|.endmacro 1543 1544|.macro EMALLOC, size, op_array, opline 1545||#if ZEND_DEBUG 1546|| const char *filename = op_array->filename ? op_array->filename->val : NULL; 1547| mov FCARG1a, size 1548| LOAD_ADDR FCARG2a, filename 1549| .if X64WIN 1550| mov CARG3d, opline->lineno 1551| xor CARG4, CARG4 1552| mov aword A5, 0 1553| EXT_CALL _emalloc, r0 1554| .elif X64 1555| mov CARG3d, opline->lineno 1556| xor CARG4, CARG4 1557| xor CARG5, CARG5 1558| EXT_CALL _emalloc, r0 1559| .else 1560| sub r4, 4 1561| push 0 1562| push 0 1563| push opline->lineno 1564| EXT_CALL _emalloc, r0 1565| add r4, 4 1566| .endif 1567||#else 1568||#ifdef HAVE_BUILTIN_CONSTANT_P 1569|| if (size > 24 && size <= 32) { 1570| EXT_CALL _emalloc_32, r0 1571|| } else { 1572| mov FCARG1a, size 1573| EXT_CALL _emalloc, r0 1574|| } 1575||#else 1576| mov FCARG1a, size 1577| EXT_CALL _emalloc, r0 1578||#endif 1579||#endif 1580|.endmacro 1581 1582|.macro OBJ_RELEASE, reg, exit_label 1583| GC_DELREF Ra(reg) 1584| jne >1 1585| // zend_objects_store_del(obj); 1586|| if (reg != ZREG_FCARG1) { 1587| mov FCARG1a, Ra(reg) 1588|| } 1589| EXT_CALL zend_objects_store_del, r0 1590| jmp exit_label 1591|1: 1592| IF_GC_MAY_NOT_LEAK Ra(reg), >1 1593| // gc_possible_root(obj) 1594|| if (reg != ZREG_FCARG1) { 1595| mov FCARG1a, Ra(reg) 1596|| } 1597| EXT_CALL gc_possible_root, r0 1598|1: 1599|.endmacro 1600 1601|.macro UNDEFINED_OFFSET, opline 1602|| if (opline == last_valid_opline) { 1603|| zend_jit_use_last_valid_opline(); 1604| call ->undefined_offset_ex 1605|| } else { 1606| SET_EX_OPLINE opline, r0 1607| call ->undefined_offset 1608|| } 1609|.endmacro 1610 1611|.macro UNDEFINED_INDEX, opline 1612|| if (opline == last_valid_opline) { 1613|| zend_jit_use_last_valid_opline(); 1614| call ->undefined_index_ex 1615|| } else { 1616| SET_EX_OPLINE opline, r0 1617| call ->undefined_index 1618|| } 1619|.endmacro 1620 1621|.macro CANNOT_ADD_ELEMENT, opline 1622|| if (opline == last_valid_opline) { 1623|| zend_jit_use_last_valid_opline(); 1624| call ->cannot_add_element_ex 1625|| } else { 1626| SET_EX_OPLINE opline, r0 1627| call ->cannot_add_element 1628|| } 1629|.endmacro 1630 1631|.macro ENDBR 1632||#if defined (__CET__) && (__CET__ & 1) != 0 1633| .if X64 1634| endbr64 1635| .else 1636| endbr32 1637| .endif 1638||#endif 1639|.endmacro 1640 1641#if defined (__CET__) && (__CET__ & 1) != 0 1642# define ENDBR_PADDING 4 1643#else 1644# define ENDBR_PADDING 0 1645#endif 1646 1647static bool reuse_ip = 0; 1648static bool delayed_call_chain = 0; 1649static uint32_t delayed_call_level = 0; 1650static const zend_op *last_valid_opline = NULL; 1651static bool use_last_vald_opline = 0; 1652static bool track_last_valid_opline = 0; 1653static int jit_return_label = -1; 1654static uint32_t current_trace_num = 0; 1655static uint32_t allowed_opt_flags = 0; 1656 1657static void zend_jit_track_last_valid_opline(void) 1658{ 1659 use_last_vald_opline = 0; 1660 track_last_valid_opline = 1; 1661} 1662 1663static void zend_jit_use_last_valid_opline(void) 1664{ 1665 if (track_last_valid_opline) { 1666 use_last_vald_opline = 1; 1667 track_last_valid_opline = 0; 1668 } 1669} 1670 1671static bool zend_jit_trace_uses_initial_ip(void) 1672{ 1673 return use_last_vald_opline; 1674} 1675 1676static void zend_jit_set_last_valid_opline(const zend_op *target_opline) 1677{ 1678 if (!reuse_ip) { 1679 track_last_valid_opline = 0; 1680 last_valid_opline = target_opline; 1681 } 1682} 1683 1684static void zend_jit_reset_last_valid_opline(void) 1685{ 1686 track_last_valid_opline = 0; 1687 last_valid_opline = NULL; 1688} 1689 1690static void zend_jit_start_reuse_ip(void) 1691{ 1692 zend_jit_reset_last_valid_opline(); 1693 reuse_ip = 1; 1694} 1695 1696static int zend_jit_reuse_ip(dasm_State **Dst) 1697{ 1698 if (!reuse_ip) { 1699 zend_jit_start_reuse_ip(); 1700 | // call = EX(call); 1701 | mov RX, EX->call 1702 } 1703 return 1; 1704} 1705 1706static void zend_jit_stop_reuse_ip(void) 1707{ 1708 reuse_ip = 0; 1709} 1710 1711static int zend_jit_interrupt_handler_stub(dasm_State **Dst) 1712{ 1713 |->interrupt_handler: 1714 | SAVE_IP 1715 | //EG(vm_interrupt) = 0; 1716 | MEM_STORE_ZTS byte, executor_globals, vm_interrupt, 0, r0 1717 | //if (EG(timed_out)) { 1718 | MEM_CMP_ZTS byte, executor_globals, timed_out, 0, r0 1719 | je >1 1720 | //zend_timeout(); 1721 | EXT_CALL zend_timeout, r0 1722 |1: 1723 | //} else if (zend_interrupt_function) { 1724 if (zend_interrupt_function) { 1725 | //zend_interrupt_function(execute_data); 1726 |.if X64 1727 | mov CARG1, FP 1728 | EXT_CALL zend_interrupt_function, r0 1729 |.else 1730 | mov aword A1, FP 1731 | EXT_CALL zend_interrupt_function, r0 1732 |.endif 1733 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 1734 | je >1 1735 | EXT_CALL zend_jit_exception_in_interrupt_handler_helper, r0 1736 |1: 1737 | //ZEND_VM_ENTER(); 1738 | //execute_data = EG(current_execute_data); 1739 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 1740 | LOAD_IP 1741 } 1742 | //ZEND_VM_CONTINUE() 1743 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1744 | ADD_HYBRID_SPAD 1745 | JMP_IP 1746 } else if (GCC_GLOBAL_REGS) { 1747 | add r4, SPAD // stack alignment 1748 | JMP_IP 1749 } else { 1750 | mov FP, aword T2 // restore FP 1751 | mov RX, aword T3 // restore IP 1752 | add r4, NR_SPAD // stack alignment 1753 | mov r0, 1 // ZEND_VM_ENTER 1754 | ret 1755 } 1756 1757 return 1; 1758} 1759 1760static int zend_jit_exception_handler_stub(dasm_State **Dst) 1761{ 1762 |->exception_handler: 1763 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1764 const void *handler = zend_get_opcode_handler_func(EG(exception_op)); 1765 1766 | ADD_HYBRID_SPAD 1767 | EXT_CALL handler, r0 1768 | JMP_IP 1769 } else { 1770 const void *handler = EG(exception_op)->handler; 1771 1772 if (GCC_GLOBAL_REGS) { 1773 | add r4, SPAD // stack alignment 1774 | EXT_JMP handler, r0 1775 } else { 1776 | mov FCARG1a, FP 1777 | EXT_CALL handler, r0 1778 | mov FP, aword T2 // restore FP 1779 | mov RX, aword T3 // restore IP 1780 | add r4, NR_SPAD // stack alignment 1781 | test eax, eax 1782 | jl >1 1783 | mov r0, 1 // ZEND_VM_ENTER 1784 |1: 1785 | ret 1786 } 1787 } 1788 1789 return 1; 1790} 1791 1792static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) 1793{ 1794 |->exception_handler_undef: 1795 | MEM_LOAD_ZTS r0, aword, executor_globals, opline_before_exception, r0 1796 | test byte OP:r0->result_type, (IS_TMP_VAR|IS_VAR) 1797 | jz >1 1798 | mov eax, dword OP:r0->result.var 1799 | SET_Z_TYPE_INFO FP + r0, IS_UNDEF 1800 |1: 1801 | jmp ->exception_handler 1802 1803 return 1; 1804} 1805 1806 1807static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst) 1808{ 1809 |->exception_handler_free_op1_op2: 1810 | UNDEF_OPLINE_RESULT_IF_USED 1811 | test byte OP:RX->op1_type, (IS_TMP_VAR|IS_VAR) 1812 | je >9 1813 | mov eax, dword OP:RX->op1.var 1814 | add r0, FP 1815 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1816 |9: 1817 | test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR) 1818 | je >9 1819 | mov eax, dword OP:RX->op2.var 1820 | add r0, FP 1821 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1822 |9: 1823 | jmp ->exception_handler 1824 return 1; 1825} 1826 1827static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst) 1828{ 1829 |->exception_handler_free_op2: 1830 | MEM_LOAD_ZTS RX, aword, executor_globals, opline_before_exception, r0 1831 | UNDEF_OPLINE_RESULT_IF_USED 1832 | test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR) 1833 | je >9 1834 | mov eax, dword OP:RX->op2.var 1835 | add r0, FP 1836 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1837 |9: 1838 | jmp ->exception_handler 1839 return 1; 1840} 1841 1842static int zend_jit_leave_function_stub(dasm_State **Dst) 1843{ 1844 |->leave_function_handler: 1845 | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)] 1846 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1847 | test FCARG1d, ZEND_CALL_TOP 1848 | jnz >1 1849 | EXT_CALL zend_jit_leave_nested_func_helper, r0 1850 | ADD_HYBRID_SPAD 1851 | JMP_IP 1852 |1: 1853 | EXT_CALL zend_jit_leave_top_func_helper, r0 1854 | ADD_HYBRID_SPAD 1855 | JMP_IP 1856 } else { 1857 if (GCC_GLOBAL_REGS) { 1858 | add r4, SPAD 1859 } else { 1860 | mov FCARG2a, FP 1861 | mov FP, aword T2 // restore FP 1862 | mov RX, aword T3 // restore IP 1863 | add r4, NR_SPAD 1864 } 1865 | test FCARG1d, ZEND_CALL_TOP 1866 | jnz >1 1867 | EXT_JMP zend_jit_leave_nested_func_helper, r0 1868 |1: 1869 | EXT_JMP zend_jit_leave_top_func_helper, r0 1870 } 1871 1872 return 1; 1873} 1874 1875static int zend_jit_leave_throw_stub(dasm_State **Dst) 1876{ 1877 |->leave_throw_handler: 1878 | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { 1879 if (GCC_GLOBAL_REGS) { 1880 | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION 1881 | je >5 1882 | // EG(opline_before_exception) = opline; 1883 | MEM_STORE_ZTS aword, executor_globals, opline_before_exception, IP, r0 1884 |5: 1885 | // opline = EG(exception_op); 1886 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1887 | mov aword EX->opline, IP 1888 | // HANDLE_EXCEPTION() 1889 | jmp ->exception_handler 1890 } else { 1891 | GET_IP FCARG1a 1892 | cmp byte OP:FCARG1a->opcode, ZEND_HANDLE_EXCEPTION 1893 | je >5 1894 | // EG(opline_before_exception) = opline; 1895 | MEM_STORE_ZTS aword, executor_globals, opline_before_exception, FCARG1a, r0 1896 |5: 1897 | // opline = EG(exception_op); 1898 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1899 | mov FP, aword T2 // restore FP 1900 | mov RX, aword T3 // restore IP 1901 | add r4, NR_SPAD // stack alignment 1902 | mov r0, 2 // ZEND_VM_LEAVE 1903 | ret 1904 } 1905 1906 return 1; 1907} 1908 1909static int zend_jit_icall_throw_stub(dasm_State **Dst) 1910{ 1911 |->icall_throw_handler: 1912 | // zend_rethrow_exception(zend_execute_data *execute_data) 1913 | mov IP, aword EX->opline 1914 | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { 1915 | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION 1916 | je >1 1917 | // EG(opline_before_exception) = opline; 1918 | MEM_STORE_ZTS aword, executor_globals, opline_before_exception, IP, r0 1919 |1: 1920 | // opline = EG(exception_op); 1921 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1922 || if (GCC_GLOBAL_REGS) { 1923 | mov aword EX->opline, IP 1924 || } 1925 | // HANDLE_EXCEPTION() 1926 | jmp ->exception_handler 1927 1928 return 1; 1929} 1930 1931static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) 1932{ 1933 |->throw_cannot_pass_by_ref: 1934 | mov r0, EX->opline 1935 | mov ecx, dword OP:r0->result.var 1936 | SET_Z_TYPE_INFO RX+r1, IS_UNDEF 1937 | // last EX(call) frame may be delayed 1938 | cmp RX, EX->call 1939 | je >1 1940 | mov r1, EX->call 1941 | mov EX:RX->prev_execute_data, r1 1942 | mov EX->call, RX 1943 |1: 1944 | mov RX, r0 1945 | mov FCARG1d, dword OP:r0->op2.num 1946 | EXT_CALL zend_cannot_pass_by_reference, r0 1947 | cmp byte OP:RX->op1_type, IS_TMP_VAR 1948 | jne >9 1949 | mov eax, dword OP:RX->op1.var 1950 | add r0, FP 1951 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1952 |9: 1953 | jmp ->exception_handler 1954 1955 return 1; 1956} 1957 1958static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) 1959{ 1960 |->undefined_offset_ex: 1961 | SAVE_IP 1962 | jmp ->undefined_offset 1963 1964 return 1; 1965} 1966 1967static int zend_jit_undefined_offset_stub(dasm_State **Dst) 1968{ 1969 |->undefined_offset: 1970 || if (!GCC_GLOBAL_REGS) { 1971 | mov FCARG1a, FP 1972 || } 1973 | EXT_JMP zend_jit_undefined_long_key, r0 1974 1975 return 1; 1976} 1977 1978static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) 1979{ 1980 |->undefined_index_ex: 1981 | SAVE_IP 1982 | jmp ->undefined_index 1983 1984 return 1; 1985} 1986 1987static int zend_jit_undefined_index_stub(dasm_State **Dst) 1988{ 1989 |->undefined_index: 1990 || if (!GCC_GLOBAL_REGS) { 1991 | mov FCARG1a, FP 1992 || } 1993 | EXT_JMP zend_jit_undefined_string_key, r0 1994 1995 return 1; 1996} 1997 1998static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) 1999{ 2000 |->cannot_add_element_ex: 2001 | SAVE_IP 2002 | jmp ->cannot_add_element 2003 2004 return 1; 2005} 2006 2007static int zend_jit_cannot_add_element_stub(dasm_State **Dst) 2008{ 2009 |->cannot_add_element: 2010 |.if X64WIN 2011 | sub r4, 0x28 2012 |.elif X64 2013 | sub r4, 8 2014 |.else 2015 | sub r4, 12 2016 |.endif 2017 | mov r0, EX->opline 2018 | cmp byte OP:r0->result_type, IS_UNUSED 2019 | jz >1 2020 | mov eax, dword OP:r0->result.var 2021 | SET_Z_TYPE_INFO FP + r0, IS_NULL 2022 |1: 2023 |.if X64WIN 2024 | xor CARG1, CARG1 2025 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2026 | EXT_CALL zend_throw_error, r0 2027 | add r4, 0x28 2028 |.elif X64 2029 | xor CARG1, CARG1 2030 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2031 | EXT_CALL zend_throw_error, r0 2032 | add r4, 8 2033 |.else 2034 | sub r4, 8 2035 | push "Cannot add element to the array as the next element is already occupied" 2036 | push 0 2037 | EXT_CALL zend_throw_error, r0 2038 | add r4, 28 2039 |.endif 2040 | ret 2041 2042 return 1; 2043} 2044 2045static int zend_jit_undefined_function_stub(dasm_State **Dst) 2046{ 2047 |->undefined_function: 2048 | mov r0, aword EX->opline 2049 |.if X64 2050 | xor CARG1, CARG1 2051 | LOAD_ADDR CARG2, "Call to undefined function %s()" 2052 | movsxd CARG3, dword [r0 + offsetof(zend_op, op2.constant)] 2053 | mov CARG3, aword [r0 + CARG3] 2054 | add CARG3, offsetof(zend_string, val) 2055 | EXT_CALL zend_throw_error, r0 2056 |.else 2057 | mov r0, aword [r0 + offsetof(zend_op, op2.zv)] 2058 | mov r0, aword [r0] 2059 | add r0, offsetof(zend_string, val) 2060 | mov aword A3, r0 2061 | mov aword A2, "Call to undefined function %s()" 2062 | mov aword A1, 0 2063 | EXT_CALL zend_throw_error, r0 2064 |.endif 2065 | jmp ->exception_handler 2066 return 1; 2067} 2068 2069static int zend_jit_negative_shift_stub(dasm_State **Dst) 2070{ 2071 |->negative_shift: 2072 | mov RX, EX->opline 2073 |.if X64 2074 |.if WIN 2075 | LOAD_ADDR CARG1, &zend_ce_arithmetic_error 2076 | mov CARG1, aword [CARG1] 2077 |.else 2078 | LOAD_ADDR CARG1, zend_ce_arithmetic_error 2079 |.endif 2080 | LOAD_ADDR CARG2, "Bit shift by negative number" 2081 | EXT_CALL zend_throw_error, r0 2082 |.else 2083 | sub r4, 8 2084 | push "Bit shift by negative number" 2085 |.if WIN 2086 | LOAD_ADDR r0, &zend_ce_arithmetic_error 2087 | push aword [r0] 2088 |.else 2089 | PUSH_ADDR zend_ce_arithmetic_error, r0 2090 |.endif 2091 | EXT_CALL zend_throw_error, r0 2092 | add r4, 16 2093 |.endif 2094 | jmp ->exception_handler_free_op1_op2 2095 return 1; 2096} 2097 2098static int zend_jit_mod_by_zero_stub(dasm_State **Dst) 2099{ 2100 |->mod_by_zero: 2101 | mov RX, EX->opline 2102 |.if X64 2103 |.if WIN 2104 | LOAD_ADDR CARG1, &zend_ce_division_by_zero_error 2105 | mov CARG1, aword [CARG1] 2106 |.else 2107 | LOAD_ADDR CARG1, zend_ce_division_by_zero_error 2108 |.endif 2109 | LOAD_ADDR CARG2, "Modulo by zero" 2110 | EXT_CALL zend_throw_error, r0 2111 |.else 2112 | sub r4, 8 2113 | push "Modulo by zero" 2114 |.if WIN 2115 | LOAD_ADDR r0, &zend_ce_division_by_zero_error 2116 | push aword [r0] 2117 |.else 2118 | PUSH_ADDR zend_ce_division_by_zero_error, r0 2119 |.endif 2120 | EXT_CALL zend_throw_error, r0 2121 | add r4, 16 2122 |.endif 2123 | jmp ->exception_handler_free_op1_op2 2124 return 1; 2125} 2126 2127static int zend_jit_invalid_this_stub(dasm_State **Dst) 2128{ 2129 |->invalid_this: 2130 | UNDEF_OPLINE_RESULT 2131 |.if X64 2132 | xor CARG1, CARG1 2133 | LOAD_ADDR CARG2, "Using $this when not in object context" 2134 | EXT_CALL zend_throw_error, r0 2135 |.else 2136 | sub r4, 8 2137 | push "Using $this when not in object context" 2138 | push 0 2139 | EXT_CALL zend_throw_error, r0 2140 | add r4, 16 2141 |.endif 2142 | jmp ->exception_handler 2143 return 1; 2144} 2145 2146static int zend_jit_double_one_stub(dasm_State **Dst) 2147{ 2148 |->one: 2149 |.dword 0, 0x3ff00000 2150 return 1; 2151} 2152 2153static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) 2154{ 2155 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2156 return 1; 2157 } 2158 2159 |->hybrid_runtime_jit: 2160 | EXT_CALL zend_runtime_jit, r0 2161 | JMP_IP 2162 return 1; 2163} 2164 2165static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) 2166{ 2167 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2168 return 1; 2169 } 2170 2171 |->hybrid_profile_jit: 2172 | // ++zend_jit_profile_counter; 2173 | .if X64 2174 | LOAD_ADDR r0, &zend_jit_profile_counter 2175 | inc aword [r0] 2176 | .else 2177 | inc aword [&zend_jit_profile_counter] 2178 | .endif 2179 | // op_array = (zend_op_array*)EX(func); 2180 | mov r0, EX->func 2181 | // run_time_cache = EX(run_time_cache); 2182 | mov r2, EX->run_time_cache 2183 | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); 2184 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2185 | // ++ZEND_COUNTER_INFO(op_array) 2186 | inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void*)] 2187 | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() 2188 | jmp aword [r0 + offsetof(zend_jit_op_array_extension, orig_handler)] 2189 return 1; 2190} 2191 2192static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) 2193{ 2194 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2195 return 1; 2196 } 2197 2198 |->hybrid_hot_code: 2199 | mov word [r2], ZEND_JIT_COUNTER_INIT 2200 | mov FCARG1a, FP 2201 | GET_IP FCARG2a 2202 | EXT_CALL zend_jit_hot_func, r0 2203 | JMP_IP 2204 return 1; 2205} 2206 2207/* 2208 * This code is based Mike Pall's "Hashed profile counters" idea, implemented 2209 * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual 2210 * property disclosure and research opportunities" email 2211 * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html 2212 * 2213 * In addition we use a variation of Knuth's multiplicative hash function 2214 * described at https://code.i-harness.com/en/q/a21ce 2215 * 2216 * uint64_t hash(uint64_t x) { 2217 * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; 2218 * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; 2219 * x = x ^ (x >> 31); 2220 * return x; 2221 * } 2222 * 2223 * uint_32_t hash(uint32_t x) { 2224 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2225 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2226 * x = (x >> 16) ^ x; 2227 * return x; 2228 * } 2229 * 2230 */ 2231static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) 2232{ 2233 | ENDBR 2234 | mov r0, EX->func 2235 | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2236 | mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)] 2237 | sub word [r2], cost 2238 | jle ->hybrid_hot_code 2239 | GET_IP r2 2240 | sub r2, aword [r0 + offsetof(zend_op_array, opcodes)] 2241 | // divide by sizeof(zend_op) 2242 | .if X64 2243 || ZEND_ASSERT(sizeof(zend_op) == 32); 2244 | sar r2, 2 2245 | .else 2246 || ZEND_ASSERT(sizeof(zend_op) == 28); 2247 | imul r2, 0xb6db6db7 2248 | .endif 2249 | .if X64 2250 | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2251 | .else 2252 | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2253 | .endif 2254 return 1; 2255} 2256 2257static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) 2258{ 2259 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2260 return 1; 2261 } 2262 2263 |->hybrid_func_hot_counter: 2264 2265 return zend_jit_hybrid_hot_counter_stub(Dst, 2266 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2267} 2268 2269static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) 2270{ 2271 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2272 return 1; 2273 } 2274 2275 |->hybrid_loop_hot_counter: 2276 2277 return zend_jit_hybrid_hot_counter_stub(Dst, 2278 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2279} 2280 2281static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) 2282{ 2283 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2284 return 1; 2285 } 2286 2287 |->hybrid_hot_trace: 2288 | mov word [r2], ZEND_JIT_COUNTER_INIT 2289 | mov FCARG1a, FP 2290 | GET_IP FCARG2a 2291 | EXT_CALL zend_jit_trace_hot_root, r0 2292 | test eax, eax // TODO : remove this check at least for HYBRID VM ??? 2293 | jl >1 2294 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 2295 | LOAD_IP 2296 | JMP_IP 2297 |1: 2298 | EXT_JMP zend_jit_halt_op->handler, r0 2299 return 1; 2300} 2301 2302static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) 2303{ 2304 | ENDBR 2305 | mov r0, EX->func 2306 | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2307 | mov r1, aword [r1 + offsetof(zend_jit_op_array_trace_extension, offset)] 2308 | mov r2, aword [IP + r1 + offsetof(zend_op_trace_info, counter)] 2309 | sub word [r2], cost 2310 | jle ->hybrid_hot_trace 2311 | jmp aword [IP + r1] 2312 return 1; 2313} 2314 2315static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) 2316{ 2317 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2318 return 1; 2319 } 2320 2321 |->hybrid_func_trace_counter: 2322 2323 return zend_jit_hybrid_trace_counter_stub(Dst, 2324 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2325} 2326 2327static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) 2328{ 2329 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { 2330 return 1; 2331 } 2332 2333 |->hybrid_ret_trace_counter: 2334 2335 return zend_jit_hybrid_trace_counter_stub(Dst, 2336 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); 2337} 2338 2339static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) 2340{ 2341 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2342 return 1; 2343 } 2344 2345 |->hybrid_loop_trace_counter: 2346 2347 return zend_jit_hybrid_trace_counter_stub(Dst, 2348 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2349} 2350 2351static int zend_jit_trace_halt_stub(dasm_State **Dst) 2352{ 2353 |->trace_halt: 2354 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2355 | ADD_HYBRID_SPAD 2356 | EXT_JMP zend_jit_halt_op->handler, r0 2357 } else if (GCC_GLOBAL_REGS) { 2358 | add r4, SPAD // stack alignment 2359 | xor IP, IP // PC must be zero 2360 | ret 2361 } else { 2362 | mov FP, aword T2 // restore FP 2363 | mov RX, aword T3 // restore IP 2364 | add r4, NR_SPAD // stack alignment 2365 | mov r0, -1 // ZEND_VM_RETURN 2366 | ret 2367 } 2368 return 1; 2369} 2370 2371static int zend_jit_trace_exit_stub(dasm_State **Dst) 2372{ 2373 |->trace_exit: 2374 | 2375 | // Save CPU registers 2376 |.if X64 2377 | sub r4, 16*8+16*8-8 /* CPU regs + SSE regs */ 2378 | mov aword [r4+15*8], r15 2379 | mov aword [r4+11*8], r11 2380 | mov aword [r4+10*8], r10 2381 | mov aword [r4+9*8], r9 2382 | mov aword [r4+8*8], r8 2383 | mov aword [r4+7*8], rdi 2384 | mov aword [r4+6*8], rsi 2385 | mov aword [r4+2*8], rdx 2386 | mov aword [r4+1*8], rcx 2387 | mov aword [r4+0*8], rax 2388 | mov FCARG1a, aword [r4+16*8+16*8-8] // exit_num = POP 2389 | mov FCARG2a, r4 2390 | movsd qword [r4+16*8+15*8], xmm15 2391 | movsd qword [r4+16*8+14*8], xmm14 2392 | movsd qword [r4+16*8+13*8], xmm13 2393 | movsd qword [r4+16*8+12*8], xmm12 2394 | movsd qword [r4+16*8+11*8], xmm11 2395 | movsd qword [r4+16*8+10*8], xmm10 2396 | movsd qword [r4+16*8+9*8], xmm9 2397 | movsd qword [r4+16*8+8*8], xmm8 2398 | movsd qword [r4+16*8+7*8], xmm7 2399 | movsd qword [r4+16*8+6*8], xmm6 2400 | movsd qword [r4+16*8+5*8], xmm5 2401 | movsd qword [r4+16*8+4*8], xmm4 2402 | movsd qword [r4+16*8+3*8], xmm3 2403 | movsd qword [r4+16*8+2*8], xmm2 2404 | movsd qword [r4+16*8+1*8], xmm1 2405 | movsd qword [r4+16*8+0*8], xmm0 2406 |.if X64WIN 2407 | sub r4, 32 /* shadow space */ 2408 |.endif 2409 |.else 2410 | sub r4, 8*4+8*8-4 /* CPU regs + SSE regs */ 2411 | mov aword [r4+7*4], edi 2412 | mov aword [r4+2*4], edx 2413 | mov aword [r4+1*4], ecx 2414 | mov aword [r4+0*4], eax 2415 | mov FCARG1a, aword [r4+8*4+8*8-4] // exit_num = POP 2416 | mov FCARG2a, r4 2417 | movsd qword [r4+8*4+7*8], xmm7 2418 | movsd qword [r4+8*4+6*8], xmm6 2419 | movsd qword [r4+8*4+5*8], xmm5 2420 | movsd qword [r4+8*4+4*8], xmm4 2421 | movsd qword [r4+8*4+3*8], xmm3 2422 | movsd qword [r4+8*4+2*8], xmm2 2423 | movsd qword [r4+8*4+1*8], xmm1 2424 | movsd qword [r4+8*4+0*8], xmm0 2425 |.endif 2426 | 2427 | // EX(opline) = opline 2428 | SAVE_IP 2429 | // zend_jit_trace_exit(trace_num, exit_num) 2430 | EXT_CALL zend_jit_trace_exit, r0 2431 |.if X64WIN 2432 | add r4, 16*8+16*8+32 /* CPU regs + SSE regs + shadow space */ 2433 |.elif X64 2434 | add r4, 16*8+16*8 /* CPU regs + SSE regs */ 2435 |.else 2436 | add r4, 8*4+8*8 /* CPU regs + SSE regs */ 2437 |.endif 2438 2439 | test eax, eax 2440 | jne >1 2441 2442 | // execute_data = EG(current_execute_data) 2443 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 2444 | // opline = EX(opline) 2445 | LOAD_IP 2446 2447 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2448 | ADD_HYBRID_SPAD 2449 | JMP_IP 2450 } else if (GCC_GLOBAL_REGS) { 2451 | add r4, SPAD // stack alignment 2452 | JMP_IP 2453 } else { 2454 | mov FP, aword T2 // restore FP 2455 | mov RX, aword T3 // restore IP 2456 | add r4, NR_SPAD // stack alignment 2457 | mov r0, 1 // ZEND_VM_ENTER 2458 | ret 2459 } 2460 2461 |1: 2462 | jl ->trace_halt 2463 2464 | // execute_data = EG(current_execute_data) 2465 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 2466 | // opline = EX(opline) 2467 | LOAD_IP 2468 2469 | // check for interrupt (try to avoid this ???) 2470 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 2471 | jne ->interrupt_handler 2472 2473 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2474 | ADD_HYBRID_SPAD 2475 | mov r0, EX->func 2476 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2477 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2478 | jmp aword [IP + r0] 2479 } else if (GCC_GLOBAL_REGS) { 2480 | add r4, SPAD // stack alignment 2481 | mov r0, EX->func 2482 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2483 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2484 | jmp aword [IP + r0] 2485 } else { 2486 | mov IP, aword EX->opline 2487 | mov FCARG1a, FP 2488 | mov r0, EX->func 2489 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2490 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2491 | call aword [IP + r0] 2492 | test eax, eax 2493 | jl ->trace_halt 2494 | mov FP, aword T2 // restore FP 2495 | mov RX, aword T3 // restore IP 2496 | add r4, NR_SPAD // stack alignment 2497 | mov r0, 1 // ZEND_VM_ENTER 2498 | ret 2499 } 2500 2501 return 1; 2502} 2503 2504static int zend_jit_trace_escape_stub(dasm_State **Dst) 2505{ 2506 |->trace_escape: 2507 | 2508 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2509 | ADD_HYBRID_SPAD 2510 | JMP_IP 2511 } else if (GCC_GLOBAL_REGS) { 2512 | add r4, SPAD // stack alignment 2513 | JMP_IP 2514 } else { 2515 | mov FP, aword T2 // restore FP 2516 | mov RX, aword T3 // restore IP 2517 | add r4, NR_SPAD // stack alignment 2518 | mov r0, 1 // ZEND_VM_ENTER 2519 | ret 2520 } 2521 2522 return 1; 2523} 2524 2525/* Keep 32 exit points in a single code block */ 2526#define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes 2527#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points 2528 2529static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) 2530{ 2531 uint32_t i; 2532 2533 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP - 1; i++) { 2534 | push byte i 2535 | .byte 0xeb, (4*(ZEND_JIT_EXIT_POINTS_PER_GROUP-i)-6) // jmp >1 2536 } 2537 | push byte i 2538 |// 1: 2539 | add aword [r4], n 2540 | jmp ->trace_exit 2541 2542 return 1; 2543} 2544 2545#ifdef CONTEXT_THREADED_JIT 2546static int zend_jit_context_threaded_call_stub(dasm_State **Dst) 2547{ 2548 |->context_threaded_call: 2549 | pop r0 2550 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2551 | ADD_HYBRID_SPAD 2552 | jmp aword [IP] 2553 } else if (GCC_GLOBAL_REGS) { 2554 | add r4, SPAD // stack alignment 2555 | jmp aword [IP] 2556 } else { 2557 ZEND_UNREACHABLE(); 2558 // TODO: context threading can't work without GLOBAL REGS because we have to change 2559 // the value of execute_data in execute_ex() 2560 | mov FCARG1a, FP 2561 | mov r0, aword [FP] 2562 | mov FP, aword T2 // restore FP 2563 | mov RX, aword T3 // restore IP 2564 | add r4, NR_SPAD // stack alignment 2565 | jmp aword [r0] 2566 } 2567 return 1; 2568} 2569#endif 2570 2571static int zend_jit_assign_const_stub(dasm_State **Dst) 2572{ 2573 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2574 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2575 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2576 2577 |->assign_const: 2578 |.if X64WIN 2579 | sub r4, 0x28 2580 |.elif X64 2581 | sub r4, 8 2582 |.else 2583 | sub r4, 12 2584 |.endif 2585 if (!zend_jit_assign_to_variable( 2586 Dst, NULL, 2587 var_addr, var_addr, -1, -1, 2588 IS_CONST, val_addr, val_info, 2589 0, 0)) { 2590 return 0; 2591 } 2592 |.if X64WIN 2593 | add r4, 0x28 2594 |.elif X64 2595 | add r4, 8 2596 |.else 2597 | add r4, 12 2598 |.endif 2599 | ret 2600 return 1; 2601} 2602 2603static int zend_jit_assign_tmp_stub(dasm_State **Dst) 2604{ 2605 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2606 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2607 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2608 2609 |->assign_tmp: 2610 |.if X64WIN 2611 | sub r4, 0x28 2612 |.elif X64 2613 | sub r4, 8 2614 |.else 2615 | sub r4, 12 2616 |.endif 2617 if (!zend_jit_assign_to_variable( 2618 Dst, NULL, 2619 var_addr, var_addr, -1, -1, 2620 IS_TMP_VAR, val_addr, val_info, 2621 0, 0)) { 2622 return 0; 2623 } 2624 |.if X64WIN 2625 | add r4, 0x28 2626 |.elif X64 2627 | add r4, 8 2628 |.else 2629 | add r4, 12 2630 |.endif 2631 | ret 2632 return 1; 2633} 2634 2635static int zend_jit_assign_var_stub(dasm_State **Dst) 2636{ 2637 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2638 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2639 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; 2640 2641 |->assign_var: 2642 |.if X64WIN 2643 | sub r4, 0x28 2644 |.elif X64 2645 | sub r4, 8 2646 |.else 2647 | sub r4, 12 2648 |.endif 2649 if (!zend_jit_assign_to_variable( 2650 Dst, NULL, 2651 var_addr, var_addr, -1, -1, 2652 IS_VAR, val_addr, val_info, 2653 0, 0)) { 2654 return 0; 2655 } 2656 |.if X64WIN 2657 | add r4, 0x28 2658 |.elif X64 2659 | add r4, 8 2660 |.else 2661 | add r4, 12 2662 |.endif 2663 | ret 2664 return 1; 2665} 2666 2667static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) 2668{ 2669 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2670 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2671 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; 2672 2673 |->assign_cv_noref: 2674 |.if X64WIN 2675 | sub r4, 0x28 2676 |.elif X64 2677 | sub r4, 8 2678 |.else 2679 | sub r4, 12 2680 |.endif 2681 if (!zend_jit_assign_to_variable( 2682 Dst, NULL, 2683 var_addr, var_addr, -1, -1, 2684 IS_CV, val_addr, val_info, 2685 0, 0)) { 2686 return 0; 2687 } 2688 |.if X64WIN 2689 | add r4, 0x28 2690 |.elif X64 2691 | add r4, 8 2692 |.else 2693 | add r4, 12 2694 |.endif 2695 | ret 2696 return 1; 2697} 2698 2699static int zend_jit_assign_cv_stub(dasm_State **Dst) 2700{ 2701 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2702 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2703 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; 2704 2705 |->assign_cv: 2706 |.if X64WIN 2707 | sub r4, 0x28 2708 |.elif X64 2709 | sub r4, 8 2710 |.else 2711 | sub r4, 12 2712 |.endif 2713 if (!zend_jit_assign_to_variable( 2714 Dst, NULL, 2715 var_addr, var_addr, -1, -1, 2716 IS_CV, val_addr, val_info, 2717 0, 0)) { 2718 return 0; 2719 } 2720 |.if X64WIN 2721 | add r4, 0x28 2722 |.elif X64 2723 | add r4, 8 2724 |.else 2725 | add r4, 12 2726 |.endif 2727 | ret 2728 return 1; 2729} 2730 2731static const zend_jit_stub zend_jit_stubs[] = { 2732 JIT_STUB(interrupt_handler, SP_ADJ_JIT, SP_ADJ_VM), 2733 JIT_STUB(exception_handler, SP_ADJ_JIT, SP_ADJ_VM), 2734 JIT_STUB(exception_handler_undef, SP_ADJ_JIT, SP_ADJ_VM), 2735 JIT_STUB(exception_handler_free_op1_op2, SP_ADJ_JIT, SP_ADJ_VM), 2736 JIT_STUB(exception_handler_free_op2, SP_ADJ_JIT, SP_ADJ_VM), 2737 JIT_STUB(leave_function, SP_ADJ_JIT, SP_ADJ_VM), 2738 JIT_STUB(leave_throw, SP_ADJ_JIT, SP_ADJ_VM), 2739 JIT_STUB(icall_throw, SP_ADJ_JIT, SP_ADJ_VM), 2740 JIT_STUB(throw_cannot_pass_by_ref, SP_ADJ_JIT, SP_ADJ_VM), 2741 JIT_STUB(undefined_offset, SP_ADJ_JIT, SP_ADJ_VM), 2742 JIT_STUB(undefined_index, SP_ADJ_JIT, SP_ADJ_VM), 2743 JIT_STUB(cannot_add_element, SP_ADJ_JIT, SP_ADJ_VM), 2744 JIT_STUB(undefined_offset_ex, SP_ADJ_JIT, SP_ADJ_VM), 2745 JIT_STUB(undefined_index_ex, SP_ADJ_JIT, SP_ADJ_VM), 2746 JIT_STUB(cannot_add_element_ex, SP_ADJ_JIT, SP_ADJ_VM), 2747 JIT_STUB(undefined_function, SP_ADJ_JIT, SP_ADJ_VM), 2748 JIT_STUB(negative_shift, SP_ADJ_JIT, SP_ADJ_VM), 2749 JIT_STUB(mod_by_zero, SP_ADJ_JIT, SP_ADJ_VM), 2750 JIT_STUB(invalid_this, SP_ADJ_JIT, SP_ADJ_VM), 2751 JIT_STUB(trace_halt, SP_ADJ_JIT, SP_ADJ_VM), 2752 JIT_STUB(trace_exit, SP_ADJ_JIT, SP_ADJ_VM), 2753 JIT_STUB(trace_escape, SP_ADJ_JIT, SP_ADJ_VM), 2754 JIT_STUB(hybrid_runtime_jit, SP_ADJ_VM, SP_ADJ_NONE), 2755 JIT_STUB(hybrid_profile_jit, SP_ADJ_VM, SP_ADJ_NONE), 2756 JIT_STUB(hybrid_hot_code, SP_ADJ_VM, SP_ADJ_NONE), 2757 JIT_STUB(hybrid_func_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2758 JIT_STUB(hybrid_loop_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2759 JIT_STUB(hybrid_hot_trace, SP_ADJ_VM, SP_ADJ_NONE), 2760 JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2761 JIT_STUB(hybrid_ret_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2762 JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2763 JIT_STUB(assign_const, SP_ADJ_RET, SP_ADJ_ASSIGN), 2764 JIT_STUB(assign_tmp, SP_ADJ_RET, SP_ADJ_ASSIGN), 2765 JIT_STUB(assign_var, SP_ADJ_RET, SP_ADJ_ASSIGN), 2766 JIT_STUB(assign_cv_noref, SP_ADJ_RET, SP_ADJ_ASSIGN), 2767 JIT_STUB(assign_cv, SP_ADJ_RET, SP_ADJ_ASSIGN), 2768 JIT_STUB(double_one, SP_ADJ_NONE, SP_ADJ_NONE), 2769#ifdef CONTEXT_THREADED_JIT 2770 JIT_STUB(context_threaded_call, SP_ADJ_RET, SP_ADJ_NONE), 2771#endif 2772}; 2773 2774#if ZTS && defined(ZEND_WIN32) 2775extern uint32_t _tls_index; 2776extern char *_tls_start; 2777extern char *_tls_end; 2778#endif 2779 2780#ifdef HAVE_GDB 2781typedef struct _Unwind_Context _Unwind_Context; 2782typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *); 2783extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *); 2784extern uintptr_t _Unwind_GetCFA(_Unwind_Context *); 2785 2786typedef struct _zend_jit_unwind_arg { 2787 int cnt; 2788 uintptr_t cfa[3]; 2789} zend_jit_unwind_arg; 2790 2791static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a) 2792{ 2793 zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a; 2794 arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx); 2795 arg->cnt++; 2796 if (arg->cnt == 3) { 2797 return 5; // _URC_END_OF_STACK 2798 } 2799 return 0; // _URC_NO_REASON; 2800} 2801 2802static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) 2803{ 2804 zend_jit_unwind_arg arg; 2805 2806 memset(&arg, 0, sizeof(arg)); 2807 _Unwind_Backtrace(zend_jit_unwind_cb, &arg); 2808 if (arg.cnt == 3) { 2809 sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1]; 2810 } 2811} 2812 2813extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data); 2814 2815static zend_never_inline void zend_jit_set_sp_adj_vm(void) 2816{ 2817 void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *); 2818 2819 orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data; 2820 zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data; 2821 execute_ex(NULL); // set sp_adj[SP_ADJ_VM] 2822 zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data; 2823} 2824#endif 2825 2826static int zend_jit_setup(void) 2827{ 2828 if (!zend_cpu_supports_sse2()) { 2829 zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); 2830 return FAILURE; 2831 } 2832 allowed_opt_flags = 0; 2833 if (zend_cpu_supports_avx()) { 2834 allowed_opt_flags |= ZEND_JIT_CPU_AVX; 2835 } 2836 2837#if ZTS 2838# ifdef _WIN64 2839 tsrm_tls_index = _tls_index * sizeof(void*); 2840 2841 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ 2842 /* Probably, it might be better solution */ 2843 do { 2844 void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index]; 2845 void *val = _tsrm_ls_cache; 2846 size_t offset = 0; 2847 size_t size = (char*)&_tls_end - (char*)&_tls_start; 2848 2849 while (offset < size) { 2850 if (*tls_mem == val) { 2851 tsrm_tls_offset = offset; 2852 break; 2853 } 2854 tls_mem++; 2855 offset += sizeof(void*); 2856 } 2857 if (offset >= size) { 2858 // TODO: error message ??? 2859 return FAILURE; 2860 } 2861 } while(0); 2862# elif ZEND_WIN32 2863 tsrm_tls_index = _tls_index * sizeof(void*); 2864 2865 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ 2866 /* Probably, it might be better solution */ 2867 do { 2868 void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index]; 2869 void *val = _tsrm_ls_cache; 2870 size_t offset = 0; 2871 size_t size = (char*)&_tls_end - (char*)&_tls_start; 2872 2873 while (offset < size) { 2874 if (*tls_mem == val) { 2875 tsrm_tls_offset = offset; 2876 break; 2877 } 2878 tls_mem++; 2879 offset += sizeof(void*); 2880 } 2881 if (offset >= size) { 2882 // TODO: error message ??? 2883 return FAILURE; 2884 } 2885 } while(0); 2886# elif defined(__APPLE__) && defined(__x86_64__) 2887 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2888 if (tsrm_ls_cache_tcb_offset == 0) { 2889 size_t *ti; 2890 __asm__( 2891 "leaq __tsrm_ls_cache(%%rip),%0" 2892 : "=r" (ti)); 2893 tsrm_tls_offset = ti[2]; 2894 tsrm_tls_index = ti[1] * 8; 2895 } 2896# elif defined(__GNUC__) && defined(__x86_64__) 2897 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2898 if (tsrm_ls_cache_tcb_offset == 0) { 2899#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__) 2900 size_t ret; 2901 2902 asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0" 2903 : "=r" (ret)); 2904 tsrm_ls_cache_tcb_offset = ret; 2905#elif defined(__MUSL__) 2906 size_t *ti; 2907 2908 __asm__( 2909 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" 2910 : "=a" (ti)); 2911 tsrm_tls_offset = ti[1]; 2912 tsrm_tls_index = ti[0] * 8; 2913#elif defined(__FreeBSD__) 2914 size_t *ti; 2915 2916 __asm__( 2917 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" 2918 : "=a" (ti)); 2919 tsrm_tls_offset = ti[1]; 2920 /* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/bf56e8b9c8639ac4447d223b83cdc128107cc3cd/libexec/rtld-elf/rtld.c#L5260) */ 2921 tsrm_tls_index = (ti[0] + 1) * 8; 2922#else 2923 size_t *ti; 2924 2925 __asm__( 2926 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" 2927 : "=a" (ti)); 2928 tsrm_tls_offset = ti[1]; 2929 tsrm_tls_index = ti[0] * 16; 2930#endif 2931 } 2932# elif defined(__GNUC__) && defined(__i386__) 2933 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2934 if (tsrm_ls_cache_tcb_offset == 0) { 2935#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__) 2936 size_t ret; 2937 2938 asm ("leal _tsrm_ls_cache@ntpoff,%0\n" 2939 : "=a" (ret)); 2940 tsrm_ls_cache_tcb_offset = ret; 2941#else 2942 size_t *ti, _ebx, _ecx, _edx; 2943 2944 __asm__( 2945 "call 1f\n" 2946 ".subsection 1\n" 2947 "1:\tmovl (%%esp), %%ebx\n\t" 2948 "ret\n" 2949 ".previous\n\t" 2950 "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" 2951 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t" 2952 "call ___tls_get_addr@plt\n\t" 2953 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n" 2954 : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx)); 2955 tsrm_tls_offset = ti[1]; 2956 tsrm_tls_index = ti[0] * 8; 2957#endif 2958 } 2959# endif 2960#endif 2961 2962 memset(sp_adj, 0, sizeof(sp_adj)); 2963#ifdef HAVE_GDB 2964 sp_adj[SP_ADJ_RET] = sizeof(void*); 2965 |.if X64WIN 2966 || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 0x28; // sub r4, 0x28 2967 |.elif X64 2968 || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 8; // sub r4, 8 2969 |.else 2970 || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 12; // sub r4, 12 2971 |.endif 2972 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2973 zend_jit_set_sp_adj_vm(); // set sp_adj[SP_ADJ_VM] 2974#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 2975 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD 2976#else 2977 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM]; 2978#endif 2979 } else if (GCC_GLOBAL_REGS) { 2980 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD; // sub r4, SPAD 2981 } else { 2982 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD; // sub r4, NR_SPAD 2983 } 2984#endif 2985 2986 return SUCCESS; 2987} 2988 2989static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) 2990{ 2991 | int3 2992 return 1; 2993} 2994 2995static int zend_jit_align_func(dasm_State **Dst) 2996{ 2997 reuse_ip = 0; 2998 delayed_call_chain = 0; 2999 last_valid_opline = NULL; 3000 use_last_vald_opline = 0; 3001 track_last_valid_opline = 0; 3002 jit_return_label = -1; 3003 |.align 16 3004 return 1; 3005} 3006 3007static int zend_jit_prologue(dasm_State **Dst) 3008{ 3009 | ENDBR 3010 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3011 | SUB_HYBRID_SPAD 3012 } else if (GCC_GLOBAL_REGS) { 3013 | sub r4, SPAD // stack alignment 3014 } else { 3015 | sub r4, NR_SPAD // stack alignment 3016 | mov aword T2, FP // save FP 3017 | mov aword T3, RX // save IP 3018 | mov FP, FCARG1a 3019 } 3020 return 1; 3021} 3022 3023static int zend_jit_label(dasm_State **Dst, unsigned int label) 3024{ 3025 |=>label: 3026 return 1; 3027} 3028 3029static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) 3030{ 3031 | // call->prev_execute_data = EX(call); 3032 if (call_level == 1) { 3033 | mov aword EX:RX->prev_execute_data, 0 3034 } else { 3035 | mov r0, EX->call 3036 | mov EX:RX->prev_execute_data, r0 3037 } 3038 | // EX(call) = call; 3039 | mov EX->call, RX 3040 3041 delayed_call_chain = 0; 3042 3043 return 1; 3044} 3045 3046static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) 3047{ 3048 if (last_valid_opline == opline) { 3049 zend_jit_use_last_valid_opline(); 3050 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 3051 zend_jit_use_last_valid_opline(); 3052 | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); 3053 } else { 3054 | LOAD_IP_ADDR opline 3055 } 3056 zend_jit_set_last_valid_opline(opline); 3057 3058 return 1; 3059} 3060 3061static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg) 3062{ 3063 if (last_valid_opline == opline) { 3064 zend_jit_use_last_valid_opline(); 3065 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 3066 zend_jit_use_last_valid_opline(); 3067 | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); 3068 } else if (!GCC_GLOBAL_REGS && set_ip_reg) { 3069 | LOAD_ADDR RX, opline 3070 | mov aword EX->opline, RX 3071 } else { 3072 | LOAD_IP_ADDR opline 3073 } 3074 zend_jit_set_last_valid_opline(opline); 3075 3076 return 1; 3077} 3078 3079static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) 3080{ 3081 if (delayed_call_chain) { 3082 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 3083 return 0; 3084 } 3085 } 3086 if (!zend_jit_set_ip(Dst, opline)) { 3087 return 0; 3088 } 3089 reuse_ip = 0; 3090 return 1; 3091} 3092 3093static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) 3094{ 3095#if 0 3096 if (!zend_jit_set_valid_ip(Dst, opline)) { 3097 return 0; 3098 } 3099 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3100 | jne ->interrupt_handler 3101#else 3102 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3103 if (exit_addr) { 3104 | jne &exit_addr 3105 } else if (last_valid_opline == opline) { 3106 || zend_jit_use_last_valid_opline(); 3107 | jne ->interrupt_handler 3108 } else { 3109 | jne >1 3110 |.cold_code 3111 |1: 3112 | LOAD_IP_ADDR opline 3113 | jmp ->interrupt_handler 3114 |.code 3115 } 3116#endif 3117 return 1; 3118} 3119 3120static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) 3121{ 3122 if (timeout_exit_addr) { 3123 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3124 | je =>loop_label 3125 | jmp &timeout_exit_addr 3126 } else { 3127 | jmp =>loop_label 3128 } 3129 return 1; 3130} 3131 3132static int zend_jit_check_exception(dasm_State **Dst) 3133{ 3134 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 3135 | jne ->exception_handler 3136 return 1; 3137} 3138 3139static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) 3140{ 3141 if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { 3142 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 3143 | jne ->exception_handler_undef 3144 return 1; 3145 } 3146 return zend_jit_check_exception(Dst); 3147} 3148 3149static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) 3150{ 3151 zend_regset regset = ZEND_REGSET_SCRATCH; 3152 3153#if ZTS 3154 if (1) { 3155#else 3156 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(jit_trace_num)))) { 3157#endif 3158 /* assignment to EG(jit_trace_num) shouldn't clober CPU register used by deoptimizer */ 3159 if (parent) { 3160 int i; 3161 int parent_vars_count = parent->exit_info[exit_num].stack_size; 3162 zend_jit_trace_stack *parent_stack = 3163 parent->stack_map + 3164 parent->exit_info[exit_num].stack_offset; 3165 3166 for (i = 0; i < parent_vars_count; i++) { 3167 if (STACK_REG(parent_stack, i) != ZREG_NONE) { 3168 if (STACK_REG(parent_stack, i) < ZREG_NUM) { 3169 ZEND_REGSET_EXCL(regset, STACK_REG(parent_stack, i)); 3170 } else if (STACK_REG(parent_stack, i) == ZREG_ZVAL_COPY_GPR0) { 3171 ZEND_REGSET_EXCL(regset, ZREG_R0); 3172 } 3173 } 3174 } 3175 } 3176 } 3177 3178 if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { 3179 ZEND_REGSET_EXCL(regset, ZREG_R0); 3180 } 3181 3182 current_trace_num = trace_num; 3183 3184 | // EG(jit_trace_num) = trace_num; 3185 if (regset == ZEND_REGSET_EMPTY) { 3186 | push r0 3187 | MEM_STORE_ZTS dword, executor_globals, jit_trace_num, trace_num, r0 3188 | pop r0 3189 } else { 3190 zend_reg tmp = ZEND_REGSET_FIRST(regset); 3191 3192 | MEM_STORE_ZTS dword, executor_globals, jit_trace_num, trace_num, Ra(tmp) 3193 (void)tmp; 3194 } 3195 3196 return 1; 3197} 3198 3199static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t) 3200{ 3201 |.cold_code 3202 |=>1: // end of the code 3203 |.code 3204 return 1; 3205} 3206 3207/* This taken from LuaJIT. Thanks to Mike Pall. */ 3208static uint32_t _asm_x86_inslen(const uint8_t* p) 3209{ 3210 static const uint8_t map_op1[256] = { 3211 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x20, 3212 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51, 3213 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, 3214 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, 3215#if defined(__x86_64__) || defined(_M_X64) 3216 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x14,0x14,0x14,0x14,0x14,0x14,0x14,0x14, 3217#else 3218 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, 3219#endif 3220 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, 3221 0x51,0x51,0x92,0x92,0x10,0x10,0x12,0x11,0x45,0x86,0x52,0x93,0x51,0x51,0x51,0x51, 3222 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 3223 0x93,0x86,0x93,0x93,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 3224 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x47,0x51,0x51,0x51,0x51,0x51, 3225#if defined(__x86_64__) || defined(_M_X64) 3226 0x59,0x59,0x59,0x59,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, 3227#else 3228 0x55,0x55,0x55,0x55,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, 3229#endif 3230 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x05,0x05,0x05,0x05,0x05,0x05,0x05,0x05, 3231 0x93,0x93,0x53,0x51,0x70,0x71,0x93,0x86,0x54,0x51,0x53,0x51,0x51,0x52,0x51,0x51, 3232 0x92,0x92,0x92,0x92,0x52,0x52,0x51,0x51,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 3233 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x45,0x45,0x47,0x52,0x51,0x51,0x51,0x51, 3234 0x10,0x51,0x10,0x10,0x51,0x51,0x63,0x66,0x51,0x51,0x51,0x51,0x51,0x51,0x92,0x92 3235 }; 3236 static const uint8_t map_op2[256] = { 3237 0x93,0x93,0x93,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x51,0x52,0x51,0x93,0x52,0x94, 3238 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3239 0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3240 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x34,0x51,0x35,0x51,0x51,0x51,0x51,0x51, 3241 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3242 0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3243 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3244 0x94,0x54,0x54,0x54,0x93,0x93,0x93,0x52,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3245 0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46, 3246 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3247 0x52,0x52,0x52,0x93,0x94,0x93,0x51,0x51,0x52,0x52,0x52,0x93,0x94,0x93,0x93,0x93, 3248 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x94,0x93,0x93,0x93,0x93,0x93, 3249 0x93,0x93,0x94,0x93,0x94,0x94,0x94,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 3250 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3251 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3252 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x52 3253 }; 3254 uint32_t result = 0; 3255 uint32_t prefixes = 0; 3256 uint32_t x = map_op1[*p]; 3257 3258 for (;;) { 3259 switch (x >> 4) { 3260 case 0: 3261 return result + x + (prefixes & 4); 3262 case 1: 3263 prefixes |= x; 3264 x = map_op1[*++p]; 3265 result++; 3266 break; 3267 case 2: 3268 x = map_op2[*++p]; 3269 break; 3270 case 3: 3271 p++; 3272 goto mrm; 3273 case 4: 3274 result -= (prefixes & 2); 3275 /* fallthrough */ 3276 case 5: 3277 return result + (x & 15); 3278 case 6: /* Group 3. */ 3279 if (p[1] & 0x38) { 3280 x = 2; 3281 } else if ((prefixes & 2) && (x == 0x66)) { 3282 x = 4; 3283 } 3284 goto mrm; 3285 case 7: /* VEX c4/c5. */ 3286#if !defined(__x86_64__) && !defined(_M_X64) 3287 if (p[1] < 0xc0) { 3288 x = 2; 3289 goto mrm; 3290 } 3291#endif 3292 if (x == 0x70) { 3293 x = *++p & 0x1f; 3294 result++; 3295 if (x >= 2) { 3296 p += 2; 3297 result += 2; 3298 goto mrm; 3299 } 3300 } 3301 p++; 3302 result++; 3303 x = map_op2[*++p]; 3304 break; 3305 case 8: 3306 result -= (prefixes & 2); 3307 /* fallthrough */ 3308 case 9: 3309mrm: 3310 /* ModR/M and possibly SIB. */ 3311 result += (x & 15); 3312 x = *++p; 3313 switch (x >> 6) { 3314 case 0: 3315 if ((x & 7) == 5) { 3316 return result + 4; 3317 } 3318 break; 3319 case 1: 3320 result++; 3321 break; 3322 case 2: 3323 result += 4; 3324 break; 3325 case 3: 3326 return result; 3327 } 3328 if ((x & 7) == 4) { 3329 result++; 3330 if (x < 0x40 && (p[1] & 7) == 5) { 3331 result += 4; 3332 } 3333 } 3334 return result; 3335 } 3336 } 3337} 3338 3339typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t); 3340typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t); 3341 3342static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) 3343{ 3344 int ret = 0; 3345 uint8_t *p, *end; 3346 3347 if (jmp_table_size) { 3348 const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*))); 3349 3350 do { 3351 if (*jmp_slot == from_addr) { 3352 *jmp_slot = to_addr; 3353 ret++; 3354 } 3355 jmp_slot++; 3356 } while (--jmp_table_size); 3357 } 3358 3359 p = (uint8_t*)code; 3360 end = p + size - 5; 3361 while (p < end) { 3362 if ((*(unaligned_uint16_t*)p & 0xf0ff) == 0x800f && p + *(unaligned_int32_t*)(p+2) == (uint8_t*)from_addr - 6) { 3363 *(unaligned_int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6)); 3364 ret++; 3365 } else if (*p == 0xe9 && p + *(unaligned_int32_t*)(p+1) == (uint8_t*)from_addr - 5) { 3366 *(unaligned_int32_t*)(p+1) = ((uint8_t*)to_addr - (p + 5)); 3367 ret++; 3368 } 3369 p += _asm_x86_inslen(p); 3370 } 3371#ifdef HAVE_VALGRIND 3372 VALGRIND_DISCARD_TRANSLATIONS(code, size); 3373#endif 3374 return ret; 3375} 3376 3377static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) 3378{ 3379 return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); 3380} 3381 3382static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) 3383{ 3384 const void *link_addr; 3385 size_t prologue_size; 3386 3387 /* Skip prologue. */ 3388 // TODO: don't hardcode this ??? 3389 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3390#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 3391 prologue_size = 0; 3392#elif defined(__x86_64__) || defined(_M_X64) 3393 // sub r4, HYBRID_SPAD 3394 prologue_size = 4; 3395#else 3396 // sub r4, HYBRID_SPAD 3397 prologue_size = 3; 3398#endif 3399 } else if (GCC_GLOBAL_REGS) { 3400 // sub r4, SPAD // stack alignment 3401#if defined(__x86_64__) || defined(_M_X64) 3402 prologue_size = 4; 3403#else 3404 prologue_size = 3; 3405#endif 3406 } else { 3407 // sub r4, NR_SPAD // stack alignment 3408 // mov aword T2, FP // save FP 3409 // mov aword T3, RX // save IP 3410 // mov FP, FCARG1a 3411#if defined(__x86_64__) || defined(_M_X64) 3412 prologue_size = 17; 3413#else 3414 prologue_size = 13; 3415#endif 3416 } 3417 link_addr = (const void*)((const char*)t->code_start + prologue_size + ENDBR_PADDING); 3418 3419 if (timeout_exit_addr) { 3420 /* Check timeout for links to LOOP */ 3421 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3422 | je &link_addr 3423 | jmp &timeout_exit_addr 3424 } else { 3425 | jmp &link_addr 3426 } 3427 return 1; 3428} 3429 3430static int zend_jit_trace_return(dasm_State **Dst, bool original_handler, const zend_op *opline) 3431{ 3432#if 0 3433 | jmp ->trace_escape 3434#else 3435 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3436 | ADD_HYBRID_SPAD 3437 if (!original_handler) { 3438 | JMP_IP 3439 } else { 3440 | mov r0, EX->func 3441 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3442 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3443 | jmp aword [IP + r0] 3444 } 3445 } else if (GCC_GLOBAL_REGS) { 3446 | add r4, SPAD // stack alignment 3447 if (!original_handler) { 3448 | JMP_IP 3449 } else { 3450 | mov r0, EX->func 3451 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3452 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3453 | jmp aword [IP + r0] 3454 } 3455 } else { 3456 if (original_handler) { 3457 | mov FCARG1a, FP 3458 | mov r0, EX->func 3459 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3460 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3461 | call aword [IP + r0] 3462 } 3463 | mov FP, aword T2 // restore FP 3464 | mov RX, aword T3 // restore IP 3465 | add r4, NR_SPAD // stack alignment 3466 if (!original_handler || !opline || 3467 (opline->opcode != ZEND_RETURN 3468 && opline->opcode != ZEND_RETURN_BY_REF 3469 && opline->opcode != ZEND_GENERATOR_RETURN 3470 && opline->opcode != ZEND_GENERATOR_CREATE 3471 && opline->opcode != ZEND_YIELD 3472 && opline->opcode != ZEND_YIELD_FROM)) { 3473 | mov r0, 2 // ZEND_VM_LEAVE 3474 } 3475 | ret 3476 } 3477#endif 3478 return 1; 3479} 3480 3481static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) 3482{ 3483 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3484 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3485 3486 if (!exit_addr) { 3487 return 0; 3488 } 3489 | IF_NOT_Z_TYPE FP + var, type, &exit_addr 3490 3491 return 1; 3492} 3493 3494static int zend_jit_scalar_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var) 3495{ 3496 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3497 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3498 3499 if (!exit_addr) { 3500 return 0; 3501 } 3502 | cmp byte [FP+var+offsetof(zval, u1.v.type)], IS_STRING 3503 | jae &exit_addr 3504 3505 return 1; 3506} 3507 3508static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) 3509{ 3510 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 3511 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3512 3513 if (!exit_addr) { 3514 return 0; 3515 } 3516 3517 | GET_ZVAL_LVAL ZREG_FCARG1, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var) 3518 if (op_info & MAY_BE_ARRAY_PACKED) { 3519 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 3520 | jz &exit_addr 3521 } else { 3522 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 3523 | jnz &exit_addr 3524 } 3525 3526 return 1; 3527} 3528 3529static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) 3530{ 3531 zend_jit_op_array_trace_extension *jit_extension = 3532 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); 3533 size_t offset = jit_extension->offset; 3534 const void *handler = 3535 (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; 3536 3537 if (!zend_jit_set_valid_ip(Dst, opline)) { 3538 return 0; 3539 } 3540 if (!GCC_GLOBAL_REGS) { 3541 | mov FCARG1a, FP 3542 } 3543 | EXT_CALL handler, r0 3544 if (may_throw 3545 && opline->opcode != ZEND_RETURN 3546 && opline->opcode != ZEND_RETURN_BY_REF) { 3547 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r1 3548 | jne ->exception_handler 3549 } 3550 3551 while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { 3552 trace++; 3553 } 3554 3555 if (!GCC_GLOBAL_REGS 3556 && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { 3557 if (opline->opcode == ZEND_RETURN || 3558 opline->opcode == ZEND_RETURN_BY_REF || 3559 opline->opcode == ZEND_DO_UCALL || 3560 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3561 opline->opcode == ZEND_DO_FCALL || 3562 opline->opcode == ZEND_GENERATOR_CREATE) { 3563 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r1 3564 } 3565 } 3566 3567 if (zend_jit_trace_may_exit(op_array, opline)) { 3568 if (opline->opcode == ZEND_RETURN || 3569 opline->opcode == ZEND_RETURN_BY_REF || 3570 opline->opcode == ZEND_GENERATOR_CREATE) { 3571 3572 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3573 if (trace->op != ZEND_JIT_TRACE_END || 3574 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && 3575 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 3576 /* this check may be handled by the following OPLINE guard or jmp [IP] */ 3577 | cmp IP, zend_jit_halt_op 3578 | je ->trace_halt 3579 } 3580 } else if (GCC_GLOBAL_REGS) { 3581 | test IP, IP 3582 | je ->trace_halt 3583 } else { 3584 | test eax, eax 3585 | jl ->trace_halt 3586 } 3587 } else if (opline->opcode == ZEND_EXIT || 3588 opline->opcode == ZEND_GENERATOR_RETURN || 3589 opline->opcode == ZEND_YIELD || 3590 opline->opcode == ZEND_YIELD_FROM) { 3591 | jmp ->trace_halt 3592 } 3593 if (trace->op != ZEND_JIT_TRACE_END || 3594 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && 3595 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 3596 3597 const zend_op *next_opline = trace->opline; 3598 const zend_op *exit_opline = NULL; 3599 uint32_t exit_point; 3600 const void *exit_addr; 3601 uint32_t old_info = 0; 3602 uint32_t old_res_info = 0; 3603 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 3604 3605 if (zend_is_smart_branch(opline)) { 3606 bool exit_if_true = 0; 3607 exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); 3608 } else { 3609 switch (opline->opcode) { 3610 case ZEND_JMPZ: 3611 case ZEND_JMPNZ: 3612 case ZEND_JMPZ_EX: 3613 case ZEND_JMPNZ_EX: 3614 case ZEND_JMP_SET: 3615 case ZEND_COALESCE: 3616 case ZEND_JMP_NULL: 3617 case ZEND_FE_RESET_R: 3618 case ZEND_FE_RESET_RW: 3619 exit_opline = (trace->opline == opline + 1) ? 3620 OP_JMP_ADDR(opline, opline->op2) : 3621 opline + 1; 3622 break; 3623 case ZEND_FE_FETCH_R: 3624 case ZEND_FE_FETCH_RW: 3625 exit_opline = (trace->opline == opline + 1) ? 3626 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3627 opline + 1; 3628 break; 3629 3630 } 3631 } 3632 3633 switch (opline->opcode) { 3634 case ZEND_FE_FETCH_R: 3635 case ZEND_FE_FETCH_RW: 3636 if (opline->op2_type != IS_UNUSED) { 3637 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); 3638 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); 3639 } 3640 break; 3641 } 3642 3643 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3644 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3645 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 3646 } 3647 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); 3648 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3649 3650 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3651 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3652 } 3653 switch (opline->opcode) { 3654 case ZEND_FE_FETCH_R: 3655 case ZEND_FE_FETCH_RW: 3656 if (opline->op2_type != IS_UNUSED) { 3657 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); 3658 } 3659 break; 3660 } 3661 3662 if (!exit_addr) { 3663 return 0; 3664 } 3665 | CMP_IP next_opline 3666 | jne &exit_addr 3667 } 3668 } 3669 3670 zend_jit_set_last_valid_opline(trace->opline); 3671 3672 return 1; 3673} 3674 3675static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) 3676{ 3677 const void *handler; 3678 3679 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3680 handler = zend_get_opcode_handler_func(opline); 3681 } else { 3682 handler = opline->handler; 3683 } 3684 3685 if (!zend_jit_set_valid_ip(Dst, opline)) { 3686 return 0; 3687 } 3688 if (!GCC_GLOBAL_REGS) { 3689 | mov FCARG1a, FP 3690 } 3691 | EXT_CALL handler, r0 3692 if (may_throw) { 3693 zend_jit_check_exception(Dst); 3694 } 3695 3696 /* Skip the following OP_DATA */ 3697 switch (opline->opcode) { 3698 case ZEND_ASSIGN_DIM: 3699 case ZEND_ASSIGN_OBJ: 3700 case ZEND_ASSIGN_STATIC_PROP: 3701 case ZEND_ASSIGN_DIM_OP: 3702 case ZEND_ASSIGN_OBJ_OP: 3703 case ZEND_ASSIGN_STATIC_PROP_OP: 3704 case ZEND_ASSIGN_STATIC_PROP_REF: 3705 case ZEND_ASSIGN_OBJ_REF: 3706 zend_jit_set_last_valid_opline(opline + 2); 3707 break; 3708 default: 3709 zend_jit_set_last_valid_opline(opline + 1); 3710 break; 3711 } 3712 3713 return 1; 3714} 3715 3716static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) 3717{ 3718 if (!zend_jit_set_valid_ip(Dst, opline)) { 3719 return 0; 3720 } 3721 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3722 if (opline->opcode == ZEND_DO_UCALL || 3723 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3724 opline->opcode == ZEND_DO_FCALL || 3725 opline->opcode == ZEND_RETURN) { 3726 3727 /* Use inlined HYBRID VM handler */ 3728 const void *handler = opline->handler; 3729 3730 | ADD_HYBRID_SPAD 3731 | EXT_JMP handler, r0 3732 } else { 3733 const void *handler = zend_get_opcode_handler_func(opline); 3734 3735 | EXT_CALL handler, r0 3736 | ADD_HYBRID_SPAD 3737 | JMP_IP 3738 } 3739 } else { 3740 const void *handler = opline->handler; 3741 3742 if (GCC_GLOBAL_REGS) { 3743 | add r4, SPAD // stack alignment 3744 } else { 3745 | mov FCARG1a, FP 3746 | mov FP, aword T2 // restore FP 3747 | mov RX, aword T3 // restore IP 3748 | add r4, NR_SPAD // stack alignment 3749 } 3750 | EXT_JMP handler, r0 3751 } 3752 zend_jit_reset_last_valid_opline(); 3753 return 1; 3754} 3755 3756static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) 3757{ 3758 uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); 3759 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3760 3761 if (!exit_addr) { 3762 return 0; 3763 } 3764 | CMP_IP opline 3765 | jne &exit_addr 3766 3767 zend_jit_set_last_valid_opline(opline); 3768 3769 return 1; 3770} 3771 3772static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) 3773{ 3774 | jmp =>target_label 3775 return 1; 3776} 3777 3778static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) 3779{ 3780 | CMP_IP next_opline 3781 | jne =>target_label 3782 3783 zend_jit_set_last_valid_opline(next_opline); 3784 3785 return 1; 3786} 3787 3788#ifdef CONTEXT_THREADED_JIT 3789static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3790{ 3791 if (!zend_jit_handler(Dst, opline, 1)) return 0; 3792 if (opline->opcode == ZEND_DO_UCALL) { 3793 | call ->context_threaded_call 3794 } else { 3795 const zend_op *next_opline = opline + 1; 3796 3797 | CMP_IP next_opline 3798 | je =>next_block 3799 | call ->context_threaded_call 3800 } 3801 return 1; 3802} 3803#endif 3804 3805static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3806{ 3807#ifdef CONTEXT_THREADED_JIT 3808 return zend_jit_context_threaded_call(Dst, opline, next_block); 3809#else 3810 return zend_jit_tail_handler(Dst, opline); 3811#endif 3812} 3813 3814static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type) 3815{ 3816 ZEND_ASSERT(Z_MODE(src) == IS_REG); 3817 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); 3818 3819 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3820 | SET_ZVAL_LVAL dst, Ra(Z_REG(src)) 3821 if (set_type && 3822 (Z_REG(dst) != ZREG_FP || 3823 !JIT_G(current_frame) || 3824 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) { 3825 | SET_ZVAL_TYPE_INFO dst, IS_LONG 3826 } 3827 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3828 | DOUBLE_SET_ZVAL_DVAL dst, Z_REG(src) 3829 if (set_type && 3830 (Z_REG(dst) != ZREG_FP || 3831 !JIT_G(current_frame) || 3832 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) { 3833 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 3834 } 3835 } else { 3836 ZEND_UNREACHABLE(); 3837 } 3838 return 1; 3839} 3840 3841static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3842{ 3843 ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); 3844 ZEND_ASSERT(Z_MODE(dst) == IS_REG); 3845 3846 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3847 | GET_ZVAL_LVAL Z_REG(dst), src 3848 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3849 | DOUBLE_GET_ZVAL_DVAL Z_REG(dst), src 3850 } else { 3851 ZEND_UNREACHABLE(); 3852 } 3853 return 1; 3854} 3855 3856static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type) 3857{ 3858 zend_jit_addr src = ZEND_ADDR_REG(reg); 3859 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3860 3861 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3862} 3863 3864static int zend_jit_store_var_type(dasm_State **Dst, int var, uint32_t type) 3865{ 3866 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3867 3868 | SET_ZVAL_TYPE_INFO dst, type 3869 return 1; 3870} 3871 3872static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) 3873{ 3874 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3875 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3876 return zend_jit_spill_store(Dst, src, dst, info, 1); 3877 } 3878 return 1; 3879} 3880 3881static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) 3882{ 3883 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3884 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3885 bool set_type = 1; 3886 3887 if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == 3888 (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { 3889 if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { 3890 set_type = 0; 3891 } 3892 } 3893 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3894 } 3895 return 1; 3896} 3897 3898static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) 3899{ 3900 zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3901 zend_jit_addr dst = ZEND_ADDR_REG(reg); 3902 3903 return zend_jit_load_reg(Dst, src, dst, info); 3904} 3905 3906static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, zend_uchar op_type, zend_jit_addr addr, znode_op op) 3907{ 3908 if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) { 3909 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 3910 | SET_ZVAL_TYPE_INFO dst, IS_UNDEF 3911 } 3912 return 1; 3913} 3914 3915static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3916{ 3917 if (!zend_jit_same_addr(src, dst)) { 3918 if (Z_MODE(src) == IS_REG) { 3919 if (Z_MODE(dst) == IS_REG) { 3920 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3921 | mov Ra(Z_REG(dst)), Ra(Z_REG(src)) 3922 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3923 | SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0) 3924 } else { 3925 ZEND_UNREACHABLE(); 3926 } 3927 if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) { 3928 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3929 3930 if (!zend_jit_spill_store(Dst, dst, var_addr, info, 3931 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3932 JIT_G(current_frame) == NULL || 3933 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3934 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3935 )) { 3936 return 0; 3937 } 3938 } 3939 } else if (Z_MODE(dst) == IS_MEM_ZVAL) { 3940 if (!Z_LOAD(src) && !Z_STORE(src)) { 3941 if (!zend_jit_spill_store(Dst, src, dst, info, 3942 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3943 JIT_G(current_frame) == NULL || 3944 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3945 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3946 )) { 3947 return 0; 3948 } 3949 } 3950 } else { 3951 ZEND_UNREACHABLE(); 3952 } 3953 } else if (Z_MODE(src) == IS_MEM_ZVAL) { 3954 if (Z_MODE(dst) == IS_REG) { 3955 if (!zend_jit_load_reg(Dst, src, dst, info)) { 3956 return 0; 3957 } 3958 } else { 3959 ZEND_UNREACHABLE(); 3960 } 3961 } else { 3962 ZEND_UNREACHABLE(); 3963 } 3964 } else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) { 3965 dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3966 if (!zend_jit_spill_store(Dst, src, dst, info, 3967 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3968 JIT_G(current_frame) == NULL || 3969 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3970 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3971 )) { 3972 return 0; 3973 } 3974 } 3975 return 1; 3976} 3977 3978static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) 3979{ 3980 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 3981 3982 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1 3983 3984 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { 3985 if (!zend_jit_save_call_chain(Dst, -1)) { 3986 return 0; 3987 } 3988 } 3989 3990 ZEND_ASSERT(opline); 3991 3992 if ((opline-1)->opcode != ZEND_FETCH_CONSTANT 3993 && (opline-1)->opcode != ZEND_FETCH_LIST_R 3994 && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR)) 3995 && !(flags & ZEND_JIT_EXIT_FREE_OP1)) { 3996 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var); 3997 3998 | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 3999 | GET_ZVAL_PTR r0, val_addr 4000 | GC_ADDREF r0 4001 |2: 4002 } 4003 4004 | LOAD_IP_ADDR (opline - 1) 4005 | jmp ->trace_escape 4006 |1: 4007 4008 return 1; 4009} 4010 4011static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) 4012{ 4013 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 4014 4015 if (reg == ZREG_LONG_MIN_MINUS_1) { 4016 |.if X64 4017 | SET_ZVAL_LVAL dst, 0x00000000 4018 | SET_ZVAL_W2 dst, 0xc3e00000 4019 |.else 4020 | SET_ZVAL_LVAL dst, 0x00200000 4021 | SET_ZVAL_W2 dst, 0xc1e00000 4022 |.endif 4023 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 4024 } else if (reg == ZREG_LONG_MIN) { 4025 |.if X64 4026 | SET_ZVAL_LVAL dst, 0x00000000 4027 | SET_ZVAL_W2 dst, 0x80000000 4028 |.else 4029 | SET_ZVAL_LVAL dst, ZEND_LONG_MIN 4030 |.endif 4031 | SET_ZVAL_TYPE_INFO dst, IS_LONG 4032 } else if (reg == ZREG_LONG_MAX) { 4033 |.if X64 4034 | SET_ZVAL_LVAL dst, 0xffffffff 4035 | SET_ZVAL_W2 dst, 0x7fffffff 4036 |.else 4037 | SET_ZVAL_LVAL dst, ZEND_LONG_MAX 4038 |.endif 4039 | SET_ZVAL_TYPE_INFO dst, IS_LONG 4040 } else if (reg == ZREG_LONG_MAX_PLUS_1) { 4041 |.if X64 4042 | SET_ZVAL_LVAL dst, 0 4043 | SET_ZVAL_W2 dst, 0x43e00000 4044 |.else 4045 | SET_ZVAL_LVAL dst, 0 4046 | SET_ZVAL_W2 dst, 0x41e00000 4047 |.endif 4048 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 4049 } else if (reg == ZREG_NULL) { 4050 | SET_ZVAL_TYPE_INFO dst, IS_NULL 4051 } else if (reg == ZREG_ZVAL_TRY_ADDREF) { 4052 | IF_NOT_ZVAL_REFCOUNTED dst, >1 4053 | GET_ZVAL_PTR r1, dst 4054 | GC_ADDREF r1 4055 |1: 4056 } else if (reg == ZREG_ZVAL_COPY_GPR0) { 4057 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 4058 4059 | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_R1, ZREG_R2 4060 | TRY_ADDREF -1, ch, r2 4061 } else { 4062 ZEND_UNREACHABLE(); 4063 } 4064 return 1; 4065} 4066 4067static int zend_jit_free_trampoline(dasm_State **Dst) 4068{ 4069 | /// if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) 4070 | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_CALL_VIA_TRAMPOLINE 4071 | jz >1 4072 | mov FCARG1a, r0 4073 | EXT_CALL zend_jit_free_trampoline_helper, r0 4074 |1: 4075 return 1; 4076} 4077 4078static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 4079{ 4080 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { 4081 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 4082 } 4083 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4084 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4085 } 4086 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { 4087 return 0; 4088 } 4089 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4090 | LONG_OP_WITH_32BIT_CONST add, op1_def_addr, Z_L(1) 4091 } else { 4092 | LONG_OP_WITH_32BIT_CONST sub, op1_def_addr, Z_L(1) 4093 } 4094 4095 if (may_overflow && 4096 (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || 4097 ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { 4098 int32_t exit_point; 4099 const void *exit_addr; 4100 zend_jit_trace_stack *stack; 4101 uint32_t old_op1_info, old_res_info = 0; 4102 4103 stack = JIT_G(current_frame)->stack; 4104 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 4105 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); 4106 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4107 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); 4108 } else { 4109 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); 4110 } 4111 if (opline->result_type != IS_UNUSED) { 4112 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 4113 if (opline->opcode == ZEND_PRE_INC) { 4114 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 4115 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); 4116 } else if (opline->opcode == ZEND_PRE_DEC) { 4117 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 4118 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); 4119 } else if (opline->opcode == ZEND_POST_INC) { 4120 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 4121 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); 4122 } else if (opline->opcode == ZEND_POST_DEC) { 4123 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 4124 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); 4125 } 4126 } 4127 4128 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 4129 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4130 if (!exit_addr) { 4131 return 0; 4132 } 4133 | jo &exit_addr 4134 4135 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4136 opline->result_type != IS_UNUSED) { 4137 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4138 } 4139 4140 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 4141 if (opline->result_type != IS_UNUSED) { 4142 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 4143 } 4144 } else if (may_overflow) { 4145 | jo >1 4146 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4147 opline->result_type != IS_UNUSED) { 4148 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4149 } 4150 |.cold_code 4151 |1: 4152 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4153 |.if X64 4154 | mov64 rax, 0x43e0000000000000 4155 | SET_ZVAL_LVAL op1_def_addr, rax 4156 |.else 4157 | SET_ZVAL_LVAL op1_def_addr, 0 4158 | SET_ZVAL_W2 op1_def_addr, 0x41e00000 4159 |.endif 4160 } else { 4161 |.if X64 4162 | mov64 rax, 0xc3e0000000000000 4163 | SET_ZVAL_LVAL op1_def_addr, rax 4164 |.else 4165 | SET_ZVAL_LVAL op1_def_addr, 0x00200000 4166 | SET_ZVAL_W2 op1_def_addr, 0xc1e00000 4167 |.endif 4168 } 4169 if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { 4170 | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE 4171 } 4172 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4173 opline->result_type != IS_UNUSED) { 4174 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1 4175 } 4176 | jmp >3 4177 |.code 4178 } else { 4179 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4180 opline->result_type != IS_UNUSED) { 4181 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4182 } 4183 } 4184 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 4185 |.cold_code 4186 |2: 4187 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4188 | SET_EX_OPLINE opline, r0 4189 if (op1_info & MAY_BE_UNDEF) { 4190 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2 4191 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 4192 | mov FCARG1d, opline->op1.var 4193 | EXT_CALL zend_jit_undefined_op_helper, r0 4194 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 4195 op1_info |= MAY_BE_NULL; 4196 } 4197 |2: 4198 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 4199 4200 | // ZVAL_DEREF(var_ptr); 4201 if (op1_info & MAY_BE_REF) { 4202 | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >2 4203 | GET_Z_PTR FCARG1a, FCARG1a 4204 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 4205 | jz >1 4206 if (RETURN_VALUE_USED(opline)) { 4207 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4208 } else { 4209 | xor FCARG2a, FCARG2a 4210 } 4211 if (opline->opcode == ZEND_PRE_INC) { 4212 | EXT_CALL zend_jit_pre_inc_typed_ref, r0 4213 } else if (opline->opcode == ZEND_PRE_DEC) { 4214 | EXT_CALL zend_jit_pre_dec_typed_ref, r0 4215 } else if (opline->opcode == ZEND_POST_INC) { 4216 | EXT_CALL zend_jit_post_inc_typed_ref, r0 4217 } else if (opline->opcode == ZEND_POST_DEC) { 4218 | EXT_CALL zend_jit_post_dec_typed_ref, r0 4219 } else { 4220 ZEND_UNREACHABLE(); 4221 } 4222 zend_jit_check_exception(Dst); 4223 | jmp >3 4224 |1: 4225 | lea FCARG1a, [FCARG1a + offsetof(zend_reference, val)] 4226 |2: 4227 } 4228 4229 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4230 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 4231 4232 | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2 4233 | TRY_ADDREF op1_info, ah, r2 4234 } 4235 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4236 if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { 4237 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4238 | EXT_CALL zend_jit_pre_inc, r0 4239 } else { 4240 | EXT_CALL increment_function, r0 4241 } 4242 } else { 4243 if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { 4244 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4245 | EXT_CALL zend_jit_pre_dec, r0 4246 } else { 4247 | EXT_CALL decrement_function, r0 4248 } 4249 } 4250 if (may_throw) { 4251 zend_jit_check_exception(Dst); 4252 } 4253 } else { 4254 zend_reg tmp_reg; 4255 4256 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4257 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2 4258 } 4259 if (Z_MODE(op1_def_addr) == IS_REG) { 4260 tmp_reg = Z_REG(op1_def_addr); 4261 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4262 tmp_reg = Z_REG(op1_addr); 4263 } else { 4264 tmp_reg = ZREG_XMM0; 4265 } 4266 | DOUBLE_GET_ZVAL_DVAL tmp_reg, op1_addr 4267 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4268 if (CAN_USE_AVX()) { 4269 | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] 4270 } else { 4271 | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] 4272 } 4273 } else { 4274 if (CAN_USE_AVX()) { 4275 | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] 4276 } else { 4277 | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] 4278 } 4279 } 4280 | DOUBLE_SET_ZVAL_DVAL op1_def_addr, tmp_reg 4281 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4282 opline->result_type != IS_UNUSED) { 4283 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1 4284 | TRY_ADDREF op1_def_info, ah, r1 4285 } 4286 } 4287 | jmp >3 4288 |.code 4289 } 4290 |3: 4291 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { 4292 return 0; 4293 } 4294 if (opline->result_type != IS_UNUSED) { 4295 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4296 return 0; 4297 } 4298 } 4299 return 1; 4300} 4301 4302static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) 4303{ 4304 if ((opline+1)->opcode == ZEND_OP_DATA 4305 && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) 4306 && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { 4307 return 1; 4308 } 4309 return 4310 ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4311 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || 4312 ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4313 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || 4314 ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4315 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); 4316} 4317 4318static int zend_jit_math_long_long(dasm_State **Dst, 4319 const zend_op *opline, 4320 zend_uchar opcode, 4321 zend_jit_addr op1_addr, 4322 zend_jit_addr op2_addr, 4323 zend_jit_addr res_addr, 4324 uint32_t res_info, 4325 uint32_t res_use_info, 4326 int may_overflow) 4327{ 4328 bool must_set_cflags = 0; 4329 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4330 zend_reg result_reg; 4331 zend_reg tmp_reg = ZREG_R0; 4332 4333 if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { 4334 if (may_overflow && (res_info & MAY_BE_GUARD) 4335 && JIT_G(current_frame) 4336 && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { 4337 result_reg = ZREG_R0; 4338 } else { 4339 result_reg = Z_REG(res_addr); 4340 } 4341 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { 4342 result_reg = Z_REG(op1_addr); 4343 } else if (Z_REG(res_addr) != ZREG_R0) { 4344 result_reg = ZREG_R0; 4345 } else { 4346 /* ASSIGN_DIM_OP */ 4347 result_reg = ZREG_FCARG1; 4348 tmp_reg = ZREG_FCARG1; 4349 } 4350 4351 if (may_overflow) { 4352 must_set_cflags = 1; 4353 } else { 4354 const zend_op *next_opline = opline + 1; 4355 4356 if (next_opline->opcode == ZEND_IS_EQUAL || 4357 next_opline->opcode == ZEND_IS_NOT_EQUAL || 4358 next_opline->opcode == ZEND_IS_SMALLER || 4359 next_opline->opcode == ZEND_IS_SMALLER_OR_EQUAL || 4360 next_opline->opcode == ZEND_CASE || 4361 next_opline->opcode == ZEND_IS_IDENTICAL || 4362 next_opline->opcode == ZEND_IS_NOT_IDENTICAL || 4363 next_opline->opcode == ZEND_CASE_STRICT) { 4364 if (next_opline->op1_type == IS_CONST 4365 && Z_TYPE_P(RT_CONSTANT(next_opline, next_opline->op1)) == IS_LONG 4366 && Z_LVAL_P(RT_CONSTANT(next_opline, next_opline->op1)) == 0 4367 && next_opline->op2_type == opline->result_type 4368 && next_opline->op2.var == opline->result.var) { 4369 must_set_cflags = 1; 4370 } else if (next_opline->op2_type == IS_CONST 4371 && Z_TYPE_P(RT_CONSTANT(next_opline, next_opline->op2)) == IS_LONG 4372 && Z_LVAL_P(RT_CONSTANT(next_opline, next_opline->op2)) == 0 4373 && next_opline->op2_type == opline->result_type 4374 && next_opline->op2.var == opline->result.var) { 4375 must_set_cflags = 1; 4376 } 4377 } 4378 } 4379 4380 if (opcode == ZEND_MUL && 4381 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4382 Z_LVAL_P(Z_ZV(op2_addr)) == 2) { 4383 if (Z_MODE(op1_addr) == IS_REG && !must_set_cflags) { 4384 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] 4385 } else { 4386 | GET_ZVAL_LVAL result_reg, op1_addr 4387 | add Ra(result_reg), Ra(result_reg) 4388 } 4389 } else if (opcode == ZEND_MUL && 4390 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4391 !must_set_cflags && 4392 Z_LVAL_P(Z_ZV(op2_addr)) > 0 && 4393 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { 4394 | GET_ZVAL_LVAL result_reg, op1_addr 4395 | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4396 } else if (opcode == ZEND_MUL && 4397 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4398 Z_LVAL_P(Z_ZV(op1_addr)) == 2) { 4399 if (Z_MODE(op2_addr) == IS_REG && !must_set_cflags) { 4400 | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Ra(Z_REG(op2_addr))] 4401 } else { 4402 | GET_ZVAL_LVAL result_reg, op2_addr 4403 | add Ra(result_reg), Ra(result_reg) 4404 } 4405 } else if (opcode == ZEND_MUL && 4406 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4407 !must_set_cflags && 4408 Z_LVAL_P(Z_ZV(op1_addr)) > 0 && 4409 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { 4410 | GET_ZVAL_LVAL result_reg, op2_addr 4411 | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) 4412 } else if (opcode == ZEND_DIV && 4413 (Z_MODE(op2_addr) == IS_CONST_ZVAL && 4414 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { 4415 | GET_ZVAL_LVAL result_reg, op1_addr 4416 | shr Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4417 } else if (opcode == ZEND_ADD && 4418 !must_set_cflags && 4419 Z_MODE(op1_addr) == IS_REG && 4420 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4421 IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr)))) { 4422 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Z_LVAL_P(Z_ZV(op2_addr))] 4423 } else if (opcode == ZEND_ADD && 4424 !must_set_cflags && 4425 Z_MODE(op2_addr) == IS_REG && 4426 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4427 IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr)))) { 4428 | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Z_LVAL_P(Z_ZV(op1_addr))] 4429 } else if (opcode == ZEND_SUB && 4430 !must_set_cflags && 4431 Z_MODE(op1_addr) == IS_REG && 4432 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4433 IS_SIGNED_32BIT(-Z_LVAL_P(Z_ZV(op2_addr)))) { 4434 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))-Z_LVAL_P(Z_ZV(op2_addr))] 4435 } else { 4436 | GET_ZVAL_LVAL result_reg, op1_addr 4437 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4438 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4439 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4440 /* +/- 0 */ 4441 may_overflow = 0; 4442 } else if (same_ops && opcode != ZEND_DIV) { 4443 | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) 4444 } else { 4445 zend_reg tmp_reg; 4446 4447 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4448 if (Z_REG(res_addr) != ZREG_R0 && result_reg != ZREG_R0) { 4449 tmp_reg = ZREG_R0; 4450 } else if (Z_REG(res_addr) != ZREG_R1 && result_reg != ZREG_R1) { 4451 tmp_reg = ZREG_R1; 4452 } else { 4453 tmp_reg = ZREG_R2; 4454 } 4455 } else if (result_reg != ZREG_R0) { 4456 tmp_reg = ZREG_R0; 4457 } else { 4458 tmp_reg = ZREG_R1; 4459 } 4460 | LONG_MATH opcode, result_reg, op2_addr, tmp_reg 4461 (void)tmp_reg; 4462 } 4463 } 4464 if (may_overflow) { 4465 if (res_info & MAY_BE_GUARD) { 4466 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 4467 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4468 if (!exit_addr) { 4469 return 0; 4470 } 4471 if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { 4472 | jo &exit_addr 4473 if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { 4474 | mov Ra(Z_REG(res_addr)), Ra(result_reg) 4475 } 4476 } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 4477 | jno &exit_addr 4478 } else { 4479 ZEND_UNREACHABLE(); 4480 } 4481 } else { 4482 if (res_info & MAY_BE_LONG) { 4483 | jo >1 4484 } else { 4485 | jno >1 4486 } 4487 } 4488 } 4489 4490 if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { 4491 | SET_ZVAL_LVAL res_addr, Ra(result_reg) 4492 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4493 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4494 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 4495 } 4496 } 4497 } 4498 4499 if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { 4500 zend_reg tmp_reg1 = ZREG_XMM0; 4501 zend_reg tmp_reg2 = ZREG_XMM1; 4502 4503 if (res_info & MAY_BE_LONG) { 4504 |.cold_code 4505 |1: 4506 } 4507 4508 do { 4509 if ((sizeof(void*) == 8 || Z_MODE(res_addr) != IS_REG) && 4510 ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || 4511 (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1))) { 4512 if (opcode == ZEND_ADD) { 4513 |.if X64 4514 | mov64 Ra(tmp_reg), 0x43e0000000000000 4515 if (Z_MODE(res_addr) == IS_REG) { 4516 | movd xmm(Z_REG(res_addr)-ZREG_XMM0), Ra(tmp_reg) 4517 } else { 4518 | SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 4519 } 4520 |.else 4521 | SET_ZVAL_LVAL res_addr, 0 4522 | SET_ZVAL_W2 res_addr, 0x41e00000 4523 |.endif 4524 break; 4525 } else if (opcode == ZEND_SUB) { 4526 |.if X64 4527 | mov64 Ra(tmp_reg), 0xc3e0000000000000 4528 if (Z_MODE(res_addr) == IS_REG) { 4529 | movd xmm(Z_REG(res_addr)-ZREG_XMM0), Ra(tmp_reg) 4530 } else { 4531 | SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 4532 } 4533 |.else 4534 | SET_ZVAL_LVAL res_addr, 0x00200000 4535 | SET_ZVAL_W2 res_addr, 0xc1e00000 4536 |.endif 4537 break; 4538 } 4539 } 4540 4541 | DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg 4542 | DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg 4543 if (CAN_USE_AVX()) { 4544 | AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 4545 } else { 4546 | SSE_MATH_REG opcode, tmp_reg1, tmp_reg2 4547 } 4548 | DOUBLE_SET_ZVAL_DVAL res_addr, tmp_reg1 4549 } while (0); 4550 4551 if (Z_MODE(res_addr) == IS_MEM_ZVAL 4552 && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4553 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4554 } 4555 if (res_info & MAY_BE_LONG) { 4556 | jmp >2 4557 |.code 4558 } 4559 |2: 4560 } 4561 4562 return 1; 4563} 4564 4565static int zend_jit_math_long_double(dasm_State **Dst, 4566 zend_uchar opcode, 4567 zend_jit_addr op1_addr, 4568 zend_jit_addr op2_addr, 4569 zend_jit_addr res_addr, 4570 uint32_t res_use_info) 4571{ 4572 zend_reg result_reg = 4573 (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; 4574 zend_reg tmp_reg; 4575 4576 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4577 /* ASSIGN_DIM_OP */ 4578 tmp_reg = ZREG_R1; 4579 } else { 4580 tmp_reg = ZREG_R0; 4581 } 4582 4583 | DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, tmp_reg 4584 4585 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4586 /* ASSIGN_DIM_OP */ 4587 if (CAN_USE_AVX()) { 4588 | AVX_MATH opcode, result_reg, result_reg, op2_addr, r1 4589 } else { 4590 | SSE_MATH opcode, result_reg, op2_addr, r1 4591 } 4592 } else { 4593 if (CAN_USE_AVX()) { 4594 | AVX_MATH opcode, result_reg, result_reg, op2_addr, r0 4595 } else { 4596 | SSE_MATH opcode, result_reg, op2_addr, r0 4597 } 4598 } 4599 | DOUBLE_SET_ZVAL_DVAL res_addr, result_reg 4600 4601 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4602 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4603 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4604 } 4605 } 4606 4607 return 1; 4608} 4609 4610static int zend_jit_math_double_long(dasm_State **Dst, 4611 zend_uchar opcode, 4612 zend_jit_addr op1_addr, 4613 zend_jit_addr op2_addr, 4614 zend_jit_addr res_addr, 4615 uint32_t res_use_info) 4616{ 4617 zend_reg result_reg, tmp_reg_gp; 4618 4619 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4620 /* ASSIGN_DIM_OP */ 4621 tmp_reg_gp = ZREG_R1; 4622 } else { 4623 tmp_reg_gp = ZREG_R0; 4624 } 4625 4626 if (zend_is_commutative(opcode) 4627 && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { 4628 if (Z_MODE(res_addr) == IS_REG) { 4629 result_reg = Z_REG(res_addr); 4630 } else { 4631 result_reg = ZREG_XMM0; 4632 } 4633 | DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, tmp_reg_gp 4634 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4635 /* ASSIGN_DIM_OP */ 4636 if (CAN_USE_AVX()) { 4637 | AVX_MATH opcode, result_reg, result_reg, op1_addr, r1 4638 } else { 4639 | SSE_MATH opcode, result_reg, op1_addr, r1 4640 } 4641 } else { 4642 if (CAN_USE_AVX()) { 4643 | AVX_MATH opcode, result_reg, result_reg, op1_addr, r0 4644 } else { 4645 | SSE_MATH opcode, result_reg, op1_addr, r0 4646 } 4647 } 4648 } else { 4649 zend_reg tmp_reg; 4650 4651 if (Z_MODE(res_addr) == IS_REG) { 4652 result_reg = Z_REG(res_addr); 4653 tmp_reg = (result_reg == ZREG_XMM0) ? ZREG_XMM1 : ZREG_XMM0; 4654 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4655 result_reg = Z_REG(op1_addr); 4656 tmp_reg = ZREG_XMM0; 4657 } else { 4658 result_reg = ZREG_XMM0; 4659 tmp_reg = ZREG_XMM1; 4660 } 4661 if (CAN_USE_AVX()) { 4662 zend_reg op1_reg; 4663 4664 if (Z_MODE(op1_addr) == IS_REG) { 4665 op1_reg = Z_REG(op1_addr); 4666 } else { 4667 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4668 op1_reg = result_reg; 4669 } 4670 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4671 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4672 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4673 /* +/- 0 */ 4674 } else { 4675 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp 4676 | AVX_MATH_REG opcode, result_reg, op1_reg, tmp_reg 4677 } 4678 } else { 4679 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4680 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4681 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4682 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4683 /* +/- 0 */ 4684 } else { 4685 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp 4686 | SSE_MATH_REG opcode, result_reg, tmp_reg 4687 } 4688 } 4689 } 4690 | DOUBLE_SET_ZVAL_DVAL res_addr, result_reg 4691 4692 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4693 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4694 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4695 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4696 } 4697 } 4698 } 4699 4700 return 1; 4701} 4702 4703static int zend_jit_math_double_double(dasm_State **Dst, 4704 zend_uchar opcode, 4705 zend_jit_addr op1_addr, 4706 zend_jit_addr op2_addr, 4707 zend_jit_addr res_addr, 4708 uint32_t res_use_info) 4709{ 4710 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4711 zend_reg result_reg; 4712 4713 if (Z_MODE(res_addr) == IS_REG) { 4714 result_reg = Z_REG(res_addr); 4715 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4716 result_reg = Z_REG(op1_addr); 4717 } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { 4718 result_reg = Z_REG(op2_addr); 4719 } else { 4720 result_reg = ZREG_XMM0; 4721 } 4722 4723 if (CAN_USE_AVX()) { 4724 zend_reg op1_reg; 4725 zend_jit_addr val_addr; 4726 4727 if (Z_MODE(op1_addr) == IS_REG) { 4728 op1_reg = Z_REG(op1_addr); 4729 val_addr = op2_addr; 4730 } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4731 op1_reg = Z_REG(op2_addr); 4732 val_addr = op1_addr; 4733 } else { 4734 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4735 op1_reg = result_reg; 4736 val_addr = op2_addr; 4737 } 4738 if ((opcode == ZEND_MUL) && 4739 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4740 | AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg 4741 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4742 /* ASSIGN_DIM_OP */ 4743 | AVX_MATH opcode, result_reg, op1_reg, val_addr, r1 4744 } else { 4745 | AVX_MATH opcode, result_reg, op1_reg, val_addr, r0 4746 } 4747 } else { 4748 zend_jit_addr val_addr; 4749 4750 if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4751 | DOUBLE_GET_ZVAL_DVAL result_reg, op2_addr 4752 val_addr = op1_addr; 4753 } else { 4754 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4755 val_addr = op2_addr; 4756 } 4757 if (same_ops) { 4758 | SSE_MATH_REG opcode, result_reg, result_reg 4759 } else if ((opcode == ZEND_MUL) && 4760 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4761 | SSE_MATH_REG ZEND_ADD, result_reg, result_reg 4762 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4763 /* ASSIGN_DIM_OP */ 4764 | SSE_MATH opcode, result_reg, val_addr, r1 4765 } else { 4766 | SSE_MATH opcode, result_reg, val_addr, r0 4767 } 4768 } 4769 | DOUBLE_SET_ZVAL_DVAL res_addr, result_reg 4770 4771 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4772 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4773 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4774 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4775 } 4776 } 4777 } 4778 4779 return 1; 4780} 4781 4782static int zend_jit_math_helper(dasm_State **Dst, 4783 const zend_op *opline, 4784 zend_uchar opcode, 4785 zend_uchar op1_type, 4786 znode_op op1, 4787 zend_jit_addr op1_addr, 4788 uint32_t op1_info, 4789 zend_uchar op2_type, 4790 znode_op op2, 4791 zend_jit_addr op2_addr, 4792 uint32_t op2_info, 4793 uint32_t res_var, 4794 zend_jit_addr res_addr, 4795 uint32_t res_info, 4796 uint32_t res_use_info, 4797 int may_overflow, 4798 int may_throw) 4799/* Labels: 1,2,3,4,5,6 */ 4800{ 4801 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4802 4803 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4804 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { 4805 if (op1_info & MAY_BE_DOUBLE) { 4806 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 4807 } else { 4808 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 4809 } 4810 } 4811 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { 4812 if (op2_info & MAY_BE_DOUBLE) { 4813 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1 4814 |.cold_code 4815 |1: 4816 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4817 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4818 } 4819 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4820 return 0; 4821 } 4822 | jmp >5 4823 |.code 4824 } else { 4825 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4826 } 4827 } 4828 if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { 4829 return 0; 4830 } 4831 if (op1_info & MAY_BE_DOUBLE) { 4832 |.cold_code 4833 |3: 4834 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4835 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4836 } 4837 if (op2_info & MAY_BE_DOUBLE) { 4838 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4839 if (!same_ops) { 4840 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1 4841 } else { 4842 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6 4843 } 4844 } 4845 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4846 return 0; 4847 } 4848 | jmp >5 4849 } 4850 if (!same_ops) { 4851 |1: 4852 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4853 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4854 } 4855 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4856 return 0; 4857 } 4858 | jmp >5 4859 } 4860 |.code 4861 } 4862 } else if ((op1_info & MAY_BE_DOUBLE) && 4863 !(op1_info & MAY_BE_LONG) && 4864 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4865 (res_info & MAY_BE_DOUBLE)) { 4866 if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4867 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4868 } 4869 if (op2_info & MAY_BE_DOUBLE) { 4870 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4871 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4872 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1 4873 } else { 4874 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4875 } 4876 } 4877 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4878 return 0; 4879 } 4880 } 4881 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4882 if (op2_info & MAY_BE_DOUBLE) { 4883 |.cold_code 4884 } 4885 |1: 4886 if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4887 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4888 } 4889 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4890 return 0; 4891 } 4892 if (op2_info & MAY_BE_DOUBLE) { 4893 | jmp >5 4894 |.code 4895 } 4896 } 4897 } else if ((op2_info & MAY_BE_DOUBLE) && 4898 !(op2_info & MAY_BE_LONG) && 4899 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4900 (res_info & MAY_BE_DOUBLE)) { 4901 if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4902 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4903 } 4904 if (op1_info & MAY_BE_DOUBLE) { 4905 if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4906 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4907 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1 4908 } else { 4909 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4910 } 4911 } 4912 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4913 return 0; 4914 } 4915 } 4916 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4917 if (op1_info & MAY_BE_DOUBLE) { 4918 |.cold_code 4919 } 4920 |1: 4921 if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4922 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 4923 } 4924 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4925 return 0; 4926 } 4927 if (op1_info & MAY_BE_DOUBLE) { 4928 | jmp >5 4929 |.code 4930 } 4931 } 4932 } 4933 4934 |5: 4935 4936 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 4937 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 4938 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4939 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4940 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4941 |.cold_code 4942 } 4943 |6: 4944 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4945 if (Z_MODE(res_addr) == IS_REG) { 4946 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4947 | LOAD_ZVAL_ADDR FCARG1a, real_addr 4948 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4949 | LOAD_ZVAL_ADDR FCARG1a, res_addr 4950 } 4951 if (Z_MODE(op1_addr) == IS_REG) { 4952 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4953 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4954 return 0; 4955 } 4956 op1_addr = real_addr; 4957 } 4958 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 4959 } else { 4960 if (Z_MODE(op1_addr) == IS_REG) { 4961 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4962 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4963 return 0; 4964 } 4965 op1_addr = real_addr; 4966 } 4967 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 4968 if (Z_MODE(res_addr) == IS_REG) { 4969 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4970 | LOAD_ZVAL_ADDR FCARG1a, real_addr 4971 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4972 | LOAD_ZVAL_ADDR FCARG1a, res_addr 4973 } 4974 } 4975 4976 if (Z_MODE(op2_addr) == IS_REG) { 4977 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 4978 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 4979 return 0; 4980 } 4981 op2_addr = real_addr; 4982 } 4983 |.if X64 4984 | LOAD_ZVAL_ADDR CARG3, op2_addr 4985 |.else 4986 | sub r4, 12 4987 | PUSH_ZVAL_ADDR op2_addr, r0 4988 |.endif 4989 | SET_EX_OPLINE opline, r0 4990 if (opcode == ZEND_ADD) { 4991 | EXT_CALL add_function, r0 4992 } else if (opcode == ZEND_SUB) { 4993 | EXT_CALL sub_function, r0 4994 } else if (opcode == ZEND_MUL) { 4995 | EXT_CALL mul_function, r0 4996 } else if (opcode == ZEND_DIV) { 4997 | EXT_CALL div_function, r0 4998 } else { 4999 ZEND_UNREACHABLE(); 5000 } 5001 |.if not(X64) 5002 | add r4, 12 5003 |.endif 5004 | FREE_OP op1_type, op1, op1_info, 0, NULL 5005 | FREE_OP op2_type, op2, op2_info, 0, NULL 5006 if (may_throw) { 5007 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 5008 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 5009 | jne ->exception_handler_free_op2 5010 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5011 zend_jit_check_exception_undef_result(Dst, opline); 5012 } else { 5013 zend_jit_check_exception(Dst); 5014 } 5015 } 5016 if (Z_MODE(res_addr) == IS_REG) { 5017 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5018 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 5019 return 0; 5020 } 5021 } 5022 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 5023 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 5024 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 5025 | jmp <5 5026 |.code 5027 } 5028 } 5029 5030 return 1; 5031} 5032 5033static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 5034{ 5035 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5036 ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 5037 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); 5038 5039 if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { 5040 return 0; 5041 } 5042 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 5043 return 0; 5044 } 5045 return 1; 5046} 5047 5048static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr) 5049{ 5050 if (Z_MODE(op2_addr) != IS_MEM_ZVAL || Z_REG(op2_addr) != ZREG_FCARG1) { 5051 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 5052 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5053 } else if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG2) { 5054 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5055 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 5056 } else { 5057 | GET_ZVAL_LVAL ZREG_R0, op2_addr 5058 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 5059 | mov FCARG2a, r0 5060 } 5061 | EXT_CALL zend_jit_add_arrays_helper, r0 5062 | SET_ZVAL_PTR res_addr, r0 5063 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX 5064 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 5065 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 5066 return 1; 5067} 5068 5069static int zend_jit_long_math_helper(dasm_State **Dst, 5070 const zend_op *opline, 5071 zend_uchar opcode, 5072 zend_uchar op1_type, 5073 znode_op op1, 5074 zend_jit_addr op1_addr, 5075 uint32_t op1_info, 5076 zend_ssa_range *op1_range, 5077 zend_uchar op2_type, 5078 znode_op op2, 5079 zend_jit_addr op2_addr, 5080 uint32_t op2_info, 5081 zend_ssa_range *op2_range, 5082 uint32_t res_var, 5083 zend_jit_addr res_addr, 5084 uint32_t res_info, 5085 uint32_t res_use_info, 5086 int may_throw) 5087/* Labels: 6 */ 5088{ 5089 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 5090 zend_reg result_reg; 5091 5092 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 5093 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 5094 } 5095 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 5096 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 5097 } 5098 5099 if (opcode == ZEND_MOD) { 5100 result_reg = ZREG_RAX; 5101 } else if (Z_MODE(res_addr) == IS_REG) { 5102 if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) 5103 && opline->op2_type != IS_CONST) { 5104 result_reg = ZREG_R0; 5105 } else { 5106 result_reg = Z_REG(res_addr); 5107 } 5108 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 5109 result_reg = Z_REG(op1_addr); 5110 } else if (Z_REG(res_addr) != ZREG_R0) { 5111 result_reg = ZREG_R0; 5112 } else { 5113 /* ASSIGN_DIM_OP */ 5114 if (ZREG_FCARG1 == ZREG_RCX 5115 && (opcode == ZEND_SL || opcode == ZEND_SR) 5116 && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 5117 result_reg = ZREG_R2; 5118 } else { 5119 result_reg = ZREG_FCARG1; 5120 } 5121 } 5122 5123 if (opcode == ZEND_SL) { 5124 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5125 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5126 5127 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 5128 if (EXPECTED(op2_lval > 0)) { 5129 | xor Ra(result_reg), Ra(result_reg) 5130 } else { 5131 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5132 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5133 | SET_EX_OPLINE opline, r0 5134 | jmp ->negative_shift 5135 } 5136 } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { 5137 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] 5138 } else { 5139 | GET_ZVAL_LVAL result_reg, op1_addr 5140 | shl Ra(result_reg), op2_lval 5141 } 5142 } else { 5143 if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { 5144 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5145 } 5146 if (!op2_range || 5147 op2_range->min < 0 || 5148 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 5149 | cmp r1, (SIZEOF_ZEND_LONG*8) 5150 | jae >1 5151 |.cold_code 5152 |1: 5153 | cmp r1, 0 5154 | mov Ra(result_reg), 0 5155 | jg >1 5156 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5157 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5158 | SET_EX_OPLINE opline, r0 5159 | jmp ->negative_shift 5160 |.code 5161 } 5162 | GET_ZVAL_LVAL result_reg, op1_addr 5163 | shl Ra(result_reg), cl 5164 |1: 5165 } 5166 } else if (opcode == ZEND_SR) { 5167 | GET_ZVAL_LVAL result_reg, op1_addr 5168 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5169 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5170 5171 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 5172 if (EXPECTED(op2_lval > 0)) { 5173 | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1 5174 } else { 5175 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5176 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5177 | SET_EX_OPLINE opline, r0 5178 | jmp ->negative_shift 5179 } 5180 } else { 5181 | sar Ra(result_reg), op2_lval 5182 } 5183 } else { 5184 if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { 5185 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5186 } 5187 if (!op2_range || 5188 op2_range->min < 0 || 5189 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 5190 | cmp r1, (SIZEOF_ZEND_LONG*8) 5191 | jae >1 5192 |.cold_code 5193 |1: 5194 | cmp r1, 0 5195 | mov r1, (SIZEOF_ZEND_LONG * 8) - 1 5196 | jg >1 5197 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5198 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5199 | SET_EX_OPLINE opline, r0 5200 | jmp ->negative_shift 5201 |.code 5202 } 5203 |1: 5204 | sar Ra(result_reg), cl 5205 } 5206 } else if (opcode == ZEND_MOD) { 5207 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5208 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5209 5210 if (op2_lval == 0) { 5211 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5212 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5213 | SET_EX_OPLINE opline, r0 5214 | jmp ->mod_by_zero 5215 } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) { 5216 zval tmp; 5217 zend_jit_addr tmp_addr; 5218 zend_reg tmp_reg; 5219 5220 /* Optimisation for mod of power of 2 */ 5221 ZVAL_LONG(&tmp, op2_lval - 1); 5222 tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp); 5223 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 5224 tmp_reg = ZREG_R1; 5225 } else if (result_reg != ZREG_R0) { 5226 tmp_reg = ZREG_R0; 5227 } else { 5228 tmp_reg = ZREG_R1; 5229 } 5230 | GET_ZVAL_LVAL result_reg, op1_addr 5231 | LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, tmp_reg 5232 (void)tmp_reg; 5233 } else { 5234 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5235 | mov aword T1, r0 // save 5236 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) { 5237 | mov aword T1, Ra(ZREG_RCX) // save 5238 } 5239 result_reg = ZREG_RDX; 5240 if (op2_lval == -1) { 5241 | xor Ra(result_reg), Ra(result_reg) 5242 } else { 5243 | GET_ZVAL_LVAL ZREG_RAX, op1_addr 5244 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5245 |.if X64 5246 | cqo 5247 |.else 5248 | cdq 5249 |.endif 5250 | idiv Ra(ZREG_RCX) 5251 } 5252 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5253 | mov r0, aword T1 // restore 5254 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) { 5255 | mov Ra(ZREG_RCX), aword T1 // restore 5256 } 5257 } 5258 } else { 5259 if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { 5260 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5261 | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], 0 5262 } else if (Z_MODE(op2_addr) == IS_REG) { 5263 | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) 5264 } 5265 | jz >1 5266 |.cold_code 5267 |1: 5268 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5269 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5270 | SET_EX_OPLINE opline, r0 5271 | jmp ->mod_by_zero 5272 |.code 5273 } 5274 5275 /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ 5276 if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { 5277 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5278 | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], -1 5279 } else if (Z_MODE(op2_addr) == IS_REG) { 5280 | cmp Ra(Z_REG(op2_addr)), -1 5281 } 5282 | jz >1 5283 |.cold_code 5284 |1: 5285 | SET_ZVAL_LVAL res_addr, 0 5286 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5287 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 5288 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 5289 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 5290 } 5291 } 5292 } 5293 | jmp >5 5294 |.code 5295 } 5296 5297 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5298 | mov aword T1, r0 // save 5299 } 5300 result_reg = ZREG_RDX; 5301 | GET_ZVAL_LVAL ZREG_RAX, op1_addr 5302 |.if X64 5303 | cqo 5304 |.else 5305 | cdq 5306 |.endif 5307 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5308 | idiv aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)] 5309 } else if (Z_MODE(op2_addr) == IS_REG) { 5310 | idiv Ra(Z_REG(op2_addr)) 5311 } 5312 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5313 | mov r0, aword T1 // restore 5314 } 5315 } 5316 } else if (same_ops) { 5317 | GET_ZVAL_LVAL result_reg, op1_addr 5318 | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) 5319 } else { 5320 zend_reg tmp_reg; 5321 5322 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5323 if (Z_REG(res_addr) != ZREG_R0 && result_reg != ZREG_R0) { 5324 tmp_reg = ZREG_R0; 5325 } else if (Z_REG(res_addr) != ZREG_R1 && result_reg != ZREG_R1) { 5326 tmp_reg = ZREG_R1; 5327 } else { 5328 tmp_reg = ZREG_R2; 5329 } 5330 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R1) { 5331 tmp_reg = ZREG_R0; 5332 } else if (result_reg != ZREG_R0) { 5333 tmp_reg = ZREG_R0; 5334 } else { 5335 tmp_reg = ZREG_R1; 5336 } 5337 | GET_ZVAL_LVAL result_reg, op1_addr 5338 | LONG_MATH opcode, result_reg, op2_addr, tmp_reg 5339 (void)tmp_reg; 5340 } 5341 5342 if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { 5343 | SET_ZVAL_LVAL res_addr, Ra(result_reg) 5344 } 5345 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5346 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 5347 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 5348 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 5349 } 5350 } 5351 } 5352 5353 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || 5354 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 5355 if ((op1_info & MAY_BE_LONG) && 5356 (op2_info & MAY_BE_LONG)) { 5357 |.cold_code 5358 } 5359 |6: 5360 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 5361 if (Z_MODE(res_addr) == IS_REG) { 5362 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5363 | LOAD_ZVAL_ADDR FCARG1a, real_addr 5364 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5365 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5366 } 5367 if (Z_MODE(op1_addr) == IS_REG) { 5368 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 5369 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 5370 return 0; 5371 } 5372 op1_addr = real_addr; 5373 } 5374 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5375 } else { 5376 if (Z_MODE(op1_addr) == IS_REG) { 5377 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 5378 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 5379 return 0; 5380 } 5381 op1_addr = real_addr; 5382 } 5383 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5384 if (Z_MODE(res_addr) == IS_REG) { 5385 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5386 | LOAD_ZVAL_ADDR FCARG1a, real_addr 5387 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5388 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5389 } 5390 } 5391 if (Z_MODE(op2_addr) == IS_REG) { 5392 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 5393 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 5394 return 0; 5395 } 5396 op2_addr = real_addr; 5397 } 5398 |.if X64 5399 | LOAD_ZVAL_ADDR CARG3, op2_addr 5400 |.else 5401 | sub r4, 12 5402 | PUSH_ZVAL_ADDR op2_addr, r0 5403 |.endif 5404 | SET_EX_OPLINE opline, r0 5405 if (opcode == ZEND_BW_OR) { 5406 | EXT_CALL bitwise_or_function, r0 5407 } else if (opcode == ZEND_BW_AND) { 5408 | EXT_CALL bitwise_and_function, r0 5409 } else if (opcode == ZEND_BW_XOR) { 5410 | EXT_CALL bitwise_xor_function, r0 5411 } else if (opcode == ZEND_SL) { 5412 | EXT_CALL shift_left_function, r0 5413 } else if (opcode == ZEND_SR) { 5414 | EXT_CALL shift_right_function, r0 5415 } else if (opcode == ZEND_MOD) { 5416 | EXT_CALL mod_function, r0 5417 } else { 5418 ZEND_UNREACHABLE(); 5419 } 5420 |.if not(X64) 5421 | add r4, 12 5422 |.endif 5423 if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) { 5424 /* compound assignment may decrement "op2" refcount */ 5425 op2_info |= MAY_BE_RC1; 5426 } 5427 | FREE_OP op1_type, op1, op1_info, 0, NULL 5428 | FREE_OP op2_type, op2, op2_info, 0, NULL 5429 if (may_throw) { 5430 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 5431 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 5432 | jne ->exception_handler_free_op2 5433 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5434 zend_jit_check_exception_undef_result(Dst, opline); 5435 } else { 5436 zend_jit_check_exception(Dst); 5437 } 5438 } 5439 if (Z_MODE(res_addr) == IS_REG) { 5440 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5441 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 5442 return 0; 5443 } 5444 } 5445 if ((op1_info & MAY_BE_LONG) && 5446 (op2_info & MAY_BE_LONG)) { 5447 | jmp >5 5448 |.code 5449 } 5450 } 5451 |5: 5452 5453 return 1; 5454} 5455 5456static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 5457{ 5458 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5459 ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); 5460 5461 if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, 5462 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 5463 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 5464 opline->result.var, res_addr, res_info, res_use_info, may_throw)) { 5465 return 0; 5466 } 5467 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 5468 return 0; 5469 } 5470 return 1; 5471} 5472 5473static int zend_jit_concat_helper(dasm_State **Dst, 5474 const zend_op *opline, 5475 zend_uchar op1_type, 5476 znode_op op1, 5477 zend_jit_addr op1_addr, 5478 uint32_t op1_info, 5479 zend_uchar op2_type, 5480 znode_op op2, 5481 zend_jit_addr op2_addr, 5482 uint32_t op2_info, 5483 zend_jit_addr res_addr, 5484 int may_throw) 5485{ 5486#if 1 5487 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5488 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5489 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 5490 } 5491 if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5492 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6 5493 } 5494 if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { 5495 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5496 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5497 } 5498 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 5499 | EXT_CALL zend_jit_fast_assign_concat_helper, r0 5500 /* concatenation with itself may reduce refcount */ 5501 op2_info |= MAY_BE_RC1; 5502 } else { 5503 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5504 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5505 } 5506 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5507 |.if X64 5508 | LOAD_ZVAL_ADDR CARG3, op2_addr 5509 |.else 5510 | sub r4, 12 5511 | PUSH_ZVAL_ADDR op2_addr, r0 5512 |.endif 5513 if (op1_type == IS_CV || op1_type == IS_CONST) { 5514 | EXT_CALL zend_jit_fast_concat_helper, r0 5515 } else { 5516 | EXT_CALL zend_jit_fast_concat_tmp_helper, r0 5517 } 5518 |.if not(X64) 5519 | add r4, 12 5520 |.endif 5521 } 5522 /* concatenation with empty string may increase refcount */ 5523 op2_info |= MAY_BE_RCN; 5524 | FREE_OP op2_type, op2, op2_info, 0, opline 5525 |5: 5526 } 5527 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || 5528 (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { 5529 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5530 |.cold_code 5531 |6: 5532 } 5533#endif 5534 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 5535 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5536 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5537 } 5538 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5539 } else { 5540 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5541 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5542 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5543 } 5544 } 5545 |.if X64 5546 | LOAD_ZVAL_ADDR CARG3, op2_addr 5547 |.else 5548 | sub r4, 12 5549 | PUSH_ZVAL_ADDR op2_addr, r0 5550 |.endif 5551 | SET_EX_OPLINE opline, r0 5552 | EXT_CALL concat_function, r0 5553 |.if not(X64) 5554 | add r4, 12 5555 |.endif 5556 /* concatenation with empty string may increase refcount */ 5557 op1_info |= MAY_BE_RCN; 5558 op2_info |= MAY_BE_RCN; 5559 | FREE_OP op1_type, op1, op1_info, 0, NULL 5560 | FREE_OP op2_type, op2, op2_info, 0, NULL 5561 if (may_throw) { 5562 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5563 zend_jit_check_exception_undef_result(Dst, opline); 5564 } else { 5565 zend_jit_check_exception(Dst); 5566 } 5567 } 5568#if 1 5569 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5570 | jmp <5 5571 |.code 5572 } 5573 } 5574#endif 5575 5576 return 1; 5577} 5578 5579static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) 5580{ 5581 zend_jit_addr op1_addr, op2_addr; 5582 5583 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5584 ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); 5585 5586 op1_addr = OP1_ADDR(); 5587 op2_addr = OP2_ADDR(); 5588 5589 return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); 5590} 5591 5592static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) 5593/* Labels: 1,2,3,4,5 */ 5594{ 5595 zend_jit_addr op2_addr = OP2_ADDR(); 5596 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 5597 5598 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 5599 && type == BP_VAR_R 5600 && !exit_addr) { 5601 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5602 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5603 if (!exit_addr) { 5604 return 0; 5605 } 5606 } 5607 5608 if (op2_info & MAY_BE_LONG) { 5609 bool op2_loaded = 0; 5610 bool packed_loaded = 0; 5611 bool bad_packed_key = 0; 5612 5613 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { 5614 | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) 5615 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 5616 } 5617 if (op1_info & MAY_BE_PACKED_GUARD) { 5618 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 5619 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5620 5621 if (!exit_addr) { 5622 return 0; 5623 } 5624 if (op1_info & MAY_BE_ARRAY_PACKED) { 5625 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5626 | jz &exit_addr 5627 } else { 5628 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5629 | jnz &exit_addr 5630 } 5631 } 5632 if (type == BP_VAR_W) { 5633 | // hval = Z_LVAL_P(dim); 5634 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5635 op2_loaded = 1; 5636 } 5637 if (op1_info & MAY_BE_ARRAY_PACKED) { 5638 zend_long val = -1; 5639 5640 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5641 val = Z_LVAL_P(Z_ZV(op2_addr)); 5642 if (val >= 0 && val < HT_MAX_SIZE) { 5643 packed_loaded = 1; 5644 } else { 5645 bad_packed_key = 1; 5646 } 5647 } else { 5648 if (!op2_loaded) { 5649 | // hval = Z_LVAL_P(dim); 5650 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5651 op2_loaded = 1; 5652 } 5653 packed_loaded = 1; 5654 } 5655 5656 if (dim_type == IS_UNDEF && type == BP_VAR_W && packed_loaded) { 5657 /* don't generate "fast" code for packed array */ 5658 packed_loaded = 0; 5659 } 5660 5661 if (packed_loaded) { 5662 | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); 5663 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5664 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5665 | jz >4 // HASH_FIND 5666 } 5667 | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) 5668 |.if X64 5669 | mov eax, dword [FCARG1a + offsetof(zend_array, nNumUsed)] 5670 if (val == 0) { 5671 | test r0, r0 5672 } else if (val > 0 && !op2_loaded) { 5673 | cmp r0, val 5674 } else { 5675 | cmp r0, FCARG2a 5676 } 5677 |.else 5678 if (val >= 0 && !op2_loaded) { 5679 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val 5680 } else { 5681 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a 5682 } 5683 |.endif 5684 if (type == BP_JIT_IS) { 5685 if (not_found_exit_addr) { 5686 | jbe ¬_found_exit_addr 5687 } else { 5688 | jbe >9 // NOT_FOUND 5689 } 5690 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5691 | jbe &exit_addr 5692 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5693 | jbe ¬_found_exit_addr 5694 } else if (type == BP_VAR_RW && not_found_exit_addr) { 5695 | jbe ¬_found_exit_addr 5696 } else if (type == BP_VAR_IS && found_exit_addr) { 5697 | jbe >7 // NOT_FOUND 5698 } else { 5699 | jbe >2 // NOT_FOUND 5700 } 5701 | // _ret = &_ht->arPacked[h]; 5702 if (val >= 0) { 5703 | mov r0, aword [FCARG1a + offsetof(zend_array, arPacked)] 5704 if (val != 0) { 5705 | add r0, val * sizeof(zval) 5706 } 5707 } else { 5708 |.if X64 5709 | mov r0, FCARG2a 5710 | shl r0, 4 5711 |.else 5712 | imul r0, FCARG2a, sizeof(zval) 5713 |.endif 5714 | add r0, aword [FCARG1a + offsetof(zend_array, arPacked)] 5715 } 5716 } 5717 } 5718 switch (type) { 5719 case BP_JIT_IS: 5720 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5721 if (packed_loaded) { 5722 | jmp >5 5723 } 5724 |4: 5725 if (!op2_loaded) { 5726 | // hval = Z_LVAL_P(dim); 5727 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5728 } 5729 if (packed_loaded) { 5730 | EXT_CALL _zend_hash_index_find, r0 5731 } else { 5732 | EXT_CALL zend_hash_index_find, r0 5733 } 5734 | test r0, r0 5735 if (not_found_exit_addr) { 5736 | jz ¬_found_exit_addr 5737 } else { 5738 | jz >9 // NOT_FOUND 5739 } 5740 if (op2_info & MAY_BE_STRING) { 5741 | jmp >5 5742 } 5743 } else if (packed_loaded) { 5744 if (op2_info & MAY_BE_STRING) { 5745 | jmp >5 5746 } 5747 } else if (not_found_exit_addr) { 5748 | jmp ¬_found_exit_addr 5749 } else { 5750 | jmp >9 // NOT_FOUND 5751 } 5752 break; 5753 case BP_VAR_R: 5754 case BP_VAR_IS: 5755 case BP_VAR_UNSET: 5756 if (packed_loaded) { 5757 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5758 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5759 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5760 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 5761 if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5762 | IF_Z_TYPE r0, IS_UNDEF, &exit_addr 5763 } 5764 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5765 | IF_Z_TYPE r0, IS_UNDEF, ¬_found_exit_addr 5766 } else if (type == BP_VAR_IS && found_exit_addr) { 5767 | IF_Z_TYPE r0, IS_UNDEF, >7 // NOT_FOUND 5768 } else { 5769 | IF_Z_TYPE r0, IS_UNDEF, >2 // NOT_FOUND 5770 } 5771 } 5772 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_NUMERIC_HASH))) { 5773 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5774 | jmp &exit_addr 5775 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5776 | jmp ¬_found_exit_addr 5777 } else if (type == BP_VAR_IS && found_exit_addr) { 5778 | jmp >7 // NOT_FOUND 5779 } else { 5780 | jmp >2 // NOT_FOUND 5781 } 5782 } 5783 if (!packed_loaded || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5784 |4: 5785 if (!op2_loaded) { 5786 | // hval = Z_LVAL_P(dim); 5787 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5788 } 5789 if (packed_loaded) { 5790 | EXT_CALL _zend_hash_index_find, r0 5791 } else { 5792 | EXT_CALL zend_hash_index_find, r0 5793 } 5794 | test r0, r0 5795 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5796 | jz &exit_addr 5797 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5798 | jz ¬_found_exit_addr 5799 } else if (type == BP_VAR_IS && found_exit_addr) { 5800 | jz >7 // NOT_FOUND 5801 } else { 5802 | jz >2 // NOT_FOUND 5803 } 5804 } 5805 |.cold_code 5806 |2: 5807 switch (type) { 5808 case BP_VAR_R: 5809 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5810 | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); 5811 | // retval = &EG(uninitialized_zval); 5812 | UNDEFINED_OFFSET opline 5813 | jmp >9 5814 } 5815 break; 5816 case BP_VAR_IS: 5817 case BP_VAR_UNSET: 5818 if (!not_found_exit_addr && !found_exit_addr) { 5819 | // retval = &EG(uninitialized_zval); 5820 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 5821 | jmp >9 5822 } 5823 break; 5824 default: 5825 ZEND_UNREACHABLE(); 5826 } 5827 |.code 5828 break; 5829 case BP_VAR_RW: 5830 if (packed_loaded && !not_found_exit_addr) { 5831 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5832 } 5833 if (!packed_loaded || 5834 !not_found_exit_addr || 5835 (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5836 if (packed_loaded && not_found_exit_addr) { 5837 |.cold_code 5838 } 5839 |2: 5840 |4: 5841 if (!op2_loaded) { 5842 | // hval = Z_LVAL_P(dim); 5843 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5844 } 5845 if (packed_loaded) { 5846 | EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, r0 5847 } else { 5848 | EXT_CALL zend_jit_hash_index_lookup_rw, r0 5849 } 5850 | test r0, r0 5851 if (not_found_exit_addr) { 5852 if (packed_loaded) { 5853 | jnz >8 5854 | jmp ¬_found_exit_addr 5855 |.code 5856 } else { 5857 | jz ¬_found_exit_addr 5858 } 5859 } else { 5860 | jz >9 5861 } 5862 } 5863 break; 5864 case BP_VAR_W: 5865 if (packed_loaded) { 5866 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5867 } 5868 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) { 5869 |2: 5870 |4: 5871 if (!op2_loaded) { 5872 | // hval = Z_LVAL_P(dim); 5873 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5874 } 5875 | EXT_CALL zend_hash_index_lookup, r0 5876 } 5877 break; 5878 default: 5879 ZEND_UNREACHABLE(); 5880 } 5881 5882 if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { 5883 | jmp >8 5884 } 5885 } 5886 5887 if (op2_info & MAY_BE_STRING) { 5888 |3: 5889 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5890 | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) 5891 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3 5892 } 5893 | // offset_key = Z_STR_P(dim); 5894 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5895 | // retval = zend_hash_find(ht, offset_key); 5896 switch (type) { 5897 case BP_JIT_IS: 5898 if (opline->op2_type != IS_CONST) { 5899 | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' 5900 | jle >1 5901 |.cold_code 5902 |1: 5903 | EXT_CALL zend_jit_symtable_find, r0 5904 | jmp >1 5905 |.code 5906 | EXT_CALL zend_hash_find, r0 5907 |1: 5908 } else { 5909 | EXT_CALL zend_hash_find_known_hash, r0 5910 } 5911 | test r0, r0 5912 if (not_found_exit_addr) { 5913 | jz ¬_found_exit_addr 5914 } else { 5915 | jz >9 // NOT_FOUND 5916 } 5917 break; 5918 case BP_VAR_R: 5919 case BP_VAR_IS: 5920 case BP_VAR_UNSET: 5921 if (opline->op2_type != IS_CONST) { 5922 | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' 5923 | jle >1 5924 |.cold_code 5925 |1: 5926 | EXT_CALL zend_jit_symtable_find, r0 5927 | jmp >1 5928 |.code 5929 | EXT_CALL zend_hash_find, r0 5930 |1: 5931 } else { 5932 | EXT_CALL zend_hash_find_known_hash, r0 5933 } 5934 | test r0, r0 5935 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5936 | jz &exit_addr 5937 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5938 | jz ¬_found_exit_addr 5939 } else if (type == BP_VAR_IS && found_exit_addr) { 5940 | jz >7 // NOT_FOUND 5941 } else { 5942 | jz >2 // NOT_FOUND 5943 |.cold_code 5944 |2: 5945 switch (type) { 5946 case BP_VAR_R: 5947 // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); 5948 | UNDEFINED_INDEX opline 5949 | jmp >9 5950 break; 5951 case BP_VAR_IS: 5952 case BP_VAR_UNSET: 5953 | // retval = &EG(uninitialized_zval); 5954 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 5955 | jmp >9 5956 break; 5957 default: 5958 ZEND_UNREACHABLE(); 5959 } 5960 |.code 5961 } 5962 break; 5963 case BP_VAR_RW: 5964 if (opline->op2_type != IS_CONST) { 5965 | EXT_CALL zend_jit_symtable_lookup_rw, r0 5966 } else { 5967 | EXT_CALL zend_jit_hash_lookup_rw, r0 5968 } 5969 | test r0, r0 5970 if (not_found_exit_addr) { 5971 | jz ¬_found_exit_addr 5972 } else { 5973 | jz >9 5974 } 5975 break; 5976 case BP_VAR_W: 5977 if (opline->op2_type != IS_CONST) { 5978 | EXT_CALL zend_jit_symtable_lookup_w, r0 5979 } else { 5980 | EXT_CALL zend_hash_lookup, r0 5981 } 5982 break; 5983 default: 5984 ZEND_UNREACHABLE(); 5985 } 5986 } 5987 5988 if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { 5989 |5: 5990 if (op1_info & MAY_BE_ARRAY_OF_REF) { 5991 | ZVAL_DEREF r0, MAY_BE_REF 5992 } 5993 | cmp byte [r0 + 8], IS_NULL 5994 if (not_found_exit_addr) { 5995 | jle ¬_found_exit_addr 5996 } else if (found_exit_addr) { 5997 | jg &found_exit_addr 5998 } else { 5999 | jle >9 // NOT FOUND 6000 } 6001 } 6002 6003 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 6004 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 6005 |.cold_code 6006 |3: 6007 } 6008 if (type != BP_VAR_RW) { 6009 | SET_EX_OPLINE opline, r0 6010 } 6011 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6012 switch (type) { 6013 case BP_VAR_R: 6014 |.if X64 6015 | LOAD_ZVAL_ADDR CARG3, res_addr 6016 |.else 6017 | sub r4, 12 6018 | PUSH_ZVAL_ADDR res_addr, r0 6019 |.endif 6020 | EXT_CALL zend_jit_fetch_dim_r_helper, r0 6021 |.if not(X64) 6022 | add r4, 12 6023 |.endif 6024 | jmp >9 6025 break; 6026 case BP_JIT_IS: 6027 | EXT_CALL zend_jit_fetch_dim_isset_helper, r0 6028 | test r0, r0 6029 if (not_found_exit_addr) { 6030 | je ¬_found_exit_addr 6031 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 6032 | jmp >8 6033 } 6034 } else if (found_exit_addr) { 6035 | jne &found_exit_addr 6036 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 6037 | jmp >9 6038 } 6039 } else { 6040 | jne >8 6041 | jmp >9 6042 } 6043 break; 6044 case BP_VAR_IS: 6045 case BP_VAR_UNSET: 6046 |.if X64 6047 | LOAD_ZVAL_ADDR CARG3, res_addr 6048 |.else 6049 | sub r4, 12 6050 | PUSH_ZVAL_ADDR res_addr, r0 6051 |.endif 6052 | EXT_CALL zend_jit_fetch_dim_is_helper, r0 6053 |.if not(X64) 6054 | add r4, 12 6055 |.endif 6056 | jmp >9 6057 break; 6058 case BP_VAR_RW: 6059 | EXT_CALL zend_jit_fetch_dim_rw_helper, r0 6060 | test r0, r0 6061 | jne >8 6062 | jmp >9 6063 break; 6064 case BP_VAR_W: 6065 | EXT_CALL zend_jit_fetch_dim_w_helper, r0 6066 | test r0, r0 6067 | jne >8 6068 | jmp >9 6069 break; 6070 default: 6071 ZEND_UNREACHABLE(); 6072 } 6073 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 6074 |.code 6075 } 6076 } 6077 6078 return 1; 6079} 6080 6081static int zend_jit_simple_assign(dasm_State **Dst, 6082 const zend_op *opline, 6083 zend_jit_addr var_addr, 6084 uint32_t var_info, 6085 uint32_t var_def_info, 6086 zend_uchar val_type, 6087 zend_jit_addr val_addr, 6088 uint32_t val_info, 6089 zend_jit_addr res_addr, 6090 int in_cold, 6091 int save_r1, 6092 bool check_exception) 6093/* Labels: 1,2,3 */ 6094{ 6095 zend_reg tmp_reg; 6096 6097 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_R0) { 6098 tmp_reg = ZREG_R0; 6099 } else { 6100 /* ASSIGN_DIM */ 6101 tmp_reg = ZREG_FCARG1; 6102 } 6103 6104 if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 6105 zval *zv = Z_ZV(val_addr); 6106 6107 if (!res_addr) { 6108 | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg 6109 } else { 6110 | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg 6111 } 6112 if (Z_REFCOUNTED_P(zv)) { 6113 if (!res_addr) { 6114 | ADDREF_CONST zv, Ra(tmp_reg) 6115 } else { 6116 | ADDREF_CONST_2 zv, Ra(tmp_reg) 6117 } 6118 } 6119 } else { 6120 if (val_info & MAY_BE_UNDEF) { 6121 if (in_cold) { 6122 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2 6123 } else { 6124 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 6125 |.cold_code 6126 |1: 6127 } 6128 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 6129 if (save_r1) { 6130 | mov aword T1, FCARG1a // save 6131 } 6132 | SET_ZVAL_TYPE_INFO var_addr, IS_NULL 6133 if (res_addr) { 6134 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 6135 } 6136 if (opline) { 6137 | SET_EX_OPLINE opline, Ra(tmp_reg) 6138 } 6139 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); 6140 | mov FCARG1d, Z_OFFSET(val_addr) 6141 | EXT_CALL zend_jit_undefined_op_helper, r0 6142 if (check_exception) { 6143 | test r0, r0 6144 | jz ->exception_handler_undef 6145 } 6146 if (save_r1) { 6147 | mov FCARG1a, aword T1 // restore 6148 } 6149 | jmp >3 6150 if (in_cold) { 6151 |2: 6152 } else { 6153 |.code 6154 } 6155 } 6156 if (val_info & MAY_BE_REF) { 6157 if (val_type == IS_CV) { 6158 ZEND_ASSERT(Z_REG(var_addr) != ZREG_R2); 6159 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_R2 || Z_OFFSET(val_addr) != 0) { 6160 | LOAD_ZVAL_ADDR r2, val_addr 6161 } 6162 | ZVAL_DEREF r2, val_info 6163 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); 6164 } else { 6165 zend_jit_addr ref_addr; 6166 zend_reg type_reg = tmp_reg; 6167 6168 if (in_cold) { 6169 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 6170 } else { 6171 | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1 6172 |.cold_code 6173 |1: 6174 } 6175 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 6176 | GET_ZVAL_PTR r2, val_addr 6177 | GC_DELREF r2 6178 | // ZVAL_COPY_VALUE(return_value, &ref->value); 6179 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8); 6180 if (!res_addr) { 6181 | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, type_reg, tmp_reg 6182 } else { 6183 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, type_reg, tmp_reg 6184 } 6185 | je >2 6186 if (tmp_reg == ZREG_R0) { 6187 | IF_NOT_REFCOUNTED ah, >3 6188 } else { 6189 | IF_NOT_FLAGS Rd(tmp_reg), (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), >3 6190 } 6191 | GET_ZVAL_PTR Ra(tmp_reg), var_addr 6192 6193 if (!res_addr) { 6194 | GC_ADDREF Ra(tmp_reg) 6195 } else { 6196 | add dword [Ra(tmp_reg)], 2 6197 } 6198 | jmp >3 6199 |2: 6200 if (res_addr) { 6201 if (tmp_reg == ZREG_R0) { 6202 | IF_NOT_REFCOUNTED ah, >2 6203 } else { 6204 | IF_NOT_FLAGS Rd(tmp_reg), (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), >2 6205 } 6206 | GET_ZVAL_PTR Ra(tmp_reg), var_addr 6207 | GC_ADDREF Ra(tmp_reg) 6208 |2: 6209 } 6210 if (save_r1) { 6211 | mov aword T1, FCARG1a // save 6212 } 6213 | EFREE_REFERENCE r2 6214 if (save_r1) { 6215 | mov FCARG1a, aword T1 // restore 6216 } 6217 | jmp >3 6218 if (in_cold) { 6219 |1: 6220 } else { 6221 |.code 6222 } 6223 } 6224 } 6225 6226 if (!res_addr) { 6227 | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, tmp_reg 6228 } else { 6229 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_R2, tmp_reg 6230 } 6231 6232 if (val_type == IS_CV) { 6233 if (!res_addr) { 6234 | TRY_ADDREF val_info, dh, Ra(tmp_reg) 6235 } else { 6236 | TRY_ADDREF_2 val_info, dh, Ra(tmp_reg) 6237 } 6238 } else { 6239 if (res_addr) { 6240 | TRY_ADDREF val_info, dh, Ra(tmp_reg) 6241 } 6242 } 6243 |3: 6244 } 6245 return 1; 6246} 6247 6248static int zend_jit_assign_to_typed_ref(dasm_State **Dst, 6249 const zend_op *opline, 6250 zend_uchar val_type, 6251 zend_jit_addr val_addr, 6252 zend_jit_addr res_addr, 6253 bool check_exception) 6254{ 6255 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 6256 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6257 | jnz >2 6258 |.cold_code 6259 |2: 6260 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 6261 | LOAD_ZVAL_ADDR FCARG2a, val_addr 6262 } 6263 if (opline) { 6264 | SET_EX_OPLINE opline, r0 6265 } 6266 if (val_type == IS_CONST) { 6267 | EXT_CALL zend_jit_assign_const_to_typed_ref, r0 6268 } else if (val_type == IS_TMP_VAR) { 6269 | EXT_CALL zend_jit_assign_tmp_to_typed_ref, r0 6270 } else if (val_type == IS_VAR) { 6271 | EXT_CALL zend_jit_assign_var_to_typed_ref, r0 6272 } else if (val_type == IS_CV) { 6273 | EXT_CALL zend_jit_assign_cv_to_typed_ref, r0 6274 } else { 6275 ZEND_UNREACHABLE(); 6276 } 6277 if (res_addr) { 6278 zend_jit_addr ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6279 6280 | ZVAL_COPY_VALUE res_addr, -1, ret_addr, -1, ZREG_R1, ZREG_R2 6281 | TRY_ADDREF -1, ch, r2 6282 } 6283 if (check_exception) { 6284 | // if (UNEXPECTED(EG(exception) != NULL)) { 6285 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 6286 | je >8 // END OF zend_jit_assign_to_variable() 6287 | jmp ->exception_handler 6288 } else { 6289 | jmp >8 6290 } 6291 |.code 6292 6293 return 1; 6294} 6295 6296static int zend_jit_assign_to_variable_call(dasm_State **Dst, 6297 const zend_op *opline, 6298 zend_jit_addr __var_use_addr, 6299 zend_jit_addr var_addr, 6300 uint32_t __var_info, 6301 uint32_t __var_def_info, 6302 zend_uchar val_type, 6303 zend_jit_addr val_addr, 6304 uint32_t val_info, 6305 zend_jit_addr __res_addr, 6306 bool __check_exception) 6307{ 6308 if (val_info & MAY_BE_UNDEF) { 6309 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 6310 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6311 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6312 6313 if (!exit_addr) { 6314 return 0; 6315 } 6316 6317 | IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr 6318 } else { 6319 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 6320 |.cold_code 6321 |1: 6322 ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP); 6323 if (Z_REG(var_addr) != ZREG_FP) { 6324 | mov aword T1, Ra(Z_REG(var_addr)) // save 6325 } 6326 | SET_EX_OPLINE opline, r0 6327 | mov FCARG1d, Z_OFFSET(val_addr) 6328 | EXT_CALL zend_jit_undefined_op_helper, r0 6329 if (Z_REG(var_addr) != ZREG_FP) { 6330 | mov Ra(Z_REG(var_addr)), aword T1 // restore 6331 } 6332 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 6333 | LOAD_ZVAL_ADDR FCARG1a, var_addr 6334 } 6335 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6336 | call ->assign_const 6337 | jmp >9 6338 |.code 6339 } 6340 } 6341 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 6342 | LOAD_ZVAL_ADDR FCARG1a, var_addr 6343 } 6344 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 6345 | LOAD_ZVAL_ADDR FCARG2a, val_addr 6346 } 6347 if (opline) { 6348 | SET_EX_OPLINE opline, r0 6349 } 6350 if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 6351 | call ->assign_tmp 6352 } else if (val_type == IS_CONST) { 6353 | call ->assign_const 6354 } else if (val_type == IS_TMP_VAR) { 6355 | call ->assign_tmp 6356 } else if (val_type == IS_VAR) { 6357 if (!(val_info & MAY_BE_REF)) { 6358 | call ->assign_tmp 6359 } else { 6360 | call ->assign_var 6361 } 6362 } else if (val_type == IS_CV) { 6363 if (!(val_info & MAY_BE_REF)) { 6364 | call ->assign_cv_noref 6365 } else { 6366 | call ->assign_cv 6367 } 6368 if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 6369 |9: 6370 } 6371 } else { 6372 ZEND_UNREACHABLE(); 6373 } 6374 6375 return 1; 6376} 6377 6378static int zend_jit_assign_to_variable(dasm_State **Dst, 6379 const zend_op *opline, 6380 zend_jit_addr var_use_addr, 6381 zend_jit_addr var_addr, 6382 uint32_t var_info, 6383 uint32_t var_def_info, 6384 zend_uchar val_type, 6385 zend_jit_addr val_addr, 6386 uint32_t val_info, 6387 zend_jit_addr res_addr, 6388 bool check_exception) 6389/* Labels: 1,2,3,4,5,8 */ 6390{ 6391 int done = 0; 6392 zend_reg ref_reg, tmp_reg; 6393 6394 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_R0) { 6395 ref_reg = ZREG_FCARG1; 6396 tmp_reg = ZREG_R0; 6397 } else { 6398 /* ASSIGN_DIM */ 6399 ref_reg = ZREG_R0; 6400 tmp_reg = ZREG_FCARG1; 6401 } 6402 6403 if (var_info & MAY_BE_REF) { 6404 if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { 6405 | LOAD_ZVAL_ADDR Ra(ref_reg), var_use_addr 6406 var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); 6407 } 6408 | // if (Z_ISREF_P(variable_ptr)) { 6409 | IF_NOT_Z_TYPE, Ra(ref_reg), IS_REFERENCE, >3 6410 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 6411 | GET_Z_PTR FCARG1a, Ra(ref_reg) 6412 if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) { 6413 return 0; 6414 } 6415 | lea Ra(ref_reg), [FCARG1a + offsetof(zend_reference, val)] 6416 |3: 6417 } 6418 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6419 if (RC_MAY_BE_1(var_info)) { 6420 int in_cold = 0; 6421 6422 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6423 | IF_ZVAL_REFCOUNTED var_use_addr, >1 6424 |.cold_code 6425 |1: 6426 in_cold = 1; 6427 } 6428 if (Z_REG(var_use_addr) == ZREG_FCARG1 || Z_REG(var_use_addr) == ZREG_R0) { 6429 bool keep_gc = 0; 6430 6431 | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr 6432 if (tmp_reg == ZREG_FCARG1) { 6433 if (Z_MODE(val_addr) == IS_REG) { 6434 keep_gc = 1; 6435 } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { 6436 keep_gc = 1; 6437 } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 6438 if (sizeof(void*) == 4) { 6439 keep_gc = 1; 6440 } else { 6441 zval *zv = Z_ZV(val_addr); 6442 6443 if (Z_TYPE_P(zv) == IS_DOUBLE) { 6444 if (Z_DVAL_P(zv) == 0 || IS_SIGNED_32BIT(zv)) { 6445 keep_gc = 1; 6446 } 6447 } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 6448 keep_gc = 1; 6449 } 6450 } 6451 } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { 6452 if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 6453 keep_gc = 1; 6454 } 6455 } 6456 } 6457 if (!keep_gc) { 6458 | mov aword T1, Ra(tmp_reg) // save 6459 } 6460 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0, 0)) { 6461 return 0; 6462 } 6463 if (!keep_gc) { 6464 | mov FCARG1a, aword T1 // restore 6465 } 6466 } else { 6467 | GET_ZVAL_PTR FCARG1a, var_use_addr 6468 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1, 0)) { 6469 return 0; 6470 } 6471 } 6472 | GC_DELREF FCARG1a 6473 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 6474 | jnz >4 6475 } else { 6476 | jnz >8 6477 } 6478 | ZVAL_DTOR_FUNC var_info, opline 6479 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { 6480 if (check_exception && !(val_info & MAY_BE_UNDEF)) { 6481 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 6482 | je >8 6483 | jmp ->exception_handler 6484 } else { 6485 | jmp >8 6486 } 6487 } 6488 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 6489 |4: 6490 | IF_GC_MAY_NOT_LEAK FCARG1a, >8 6491 | EXT_CALL gc_possible_root, r0 6492 if (in_cold) { 6493 | jmp >8 6494 } 6495 } 6496 if (check_exception && (val_info & MAY_BE_UNDEF)) { 6497 |8: 6498 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 6499 | je >8 6500 | jmp ->exception_handler 6501 } 6502 if (in_cold) { 6503 |.code 6504 } else { 6505 done = 1; 6506 } 6507 } else /* if (RC_MAY_BE_N(var_info)) */ { 6508 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6509 | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5 6510 } 6511 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { 6512 if (Z_REG(var_use_addr) != ZREG_FP) { 6513 | mov T1, Ra(Z_REG(var_use_addr)) // save 6514 } 6515 | GET_ZVAL_PTR FCARG1a, var_use_addr 6516 | GC_DELREF FCARG1a 6517 | IF_GC_MAY_NOT_LEAK FCARG1a, >5 6518 | EXT_CALL gc_possible_root, r0 6519 if (Z_REG(var_use_addr) != ZREG_FP) { 6520 | mov Ra(Z_REG(var_use_addr)), T1 // restore 6521 } 6522 } else { 6523 | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr 6524 | GC_DELREF Ra(tmp_reg) 6525 } 6526 |5: 6527 } 6528 } 6529 6530 if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0, check_exception)) { 6531 return 0; 6532 } 6533 6534 |8: 6535 6536 return 1; 6537} 6538 6539static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw) 6540{ 6541 zend_jit_addr op2_addr, op3_addr, res_addr; 6542 6543 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6544 op3_addr = OP1_DATA_ADDR(); 6545 if (opline->result_type == IS_UNUSED) { 6546 res_addr = 0; 6547 } else { 6548 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 6549 } 6550 6551 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { 6552 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6553 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6554 6555 if (!exit_addr) { 6556 return 0; 6557 } 6558 6559 | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr 6560 6561 val_info &= ~MAY_BE_UNDEF; 6562 } 6563 6564 if (op1_info & MAY_BE_REF) { 6565 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6566 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 6567 | GET_Z_PTR FCARG2a, FCARG1a 6568 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 6569 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 6570 | jmp >3 6571 |.cold_code 6572 |2: 6573 | SET_EX_OPLINE opline, r0 6574 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 6575 | test r0, r0 6576 | mov FCARG1a, r0 6577 | jne >1 6578 | jmp ->exception_handler_undef 6579 |.code 6580 |1: 6581 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6582 } 6583 6584 if (op1_info & MAY_BE_ARRAY) { 6585 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6586 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 6587 } 6588 |3: 6589 | SEPARATE_ARRAY op1_addr, op1_info, 1 6590 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6591 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6592 | CMP_ZVAL_TYPE op1_addr, IS_NULL 6593 | jg >7 6594 } 6595 | // ZVAL_ARR(container, zend_new_array(8)); 6596 if (Z_REG(op1_addr) != ZREG_FP) { 6597 | mov T1, Ra(Z_REG(op1_addr)) // save 6598 } 6599 | EXT_CALL _zend_new_array_0, r0 6600 if (Z_REG(op1_addr) != ZREG_FP) { 6601 | mov Ra(Z_REG(op1_addr)), T1 // restore 6602 } 6603 | SET_ZVAL_LVAL op1_addr, r0 6604 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6605 | mov FCARG1a, r0 6606 } 6607 6608 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6609 |6: 6610 if (opline->op2_type == IS_UNUSED) { 6611 uint32_t var_info = MAY_BE_NULL; 6612 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6613 6614 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6615 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6616 | EXT_CALL zend_hash_next_index_insert, r0 6617 | // if (UNEXPECTED(!var_ptr)) { 6618 | test r0, r0 6619 | jz >1 6620 |.cold_code 6621 |1: 6622 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6623 | CANNOT_ADD_ELEMENT opline 6624 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6625 | jmp >9 6626 |.code 6627 6628 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0, 0)) { 6629 return 0; 6630 } 6631 } else { 6632 uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6633 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6634 6635 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 6636 return 0; 6637 } 6638 6639 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6640 var_info |= MAY_BE_REF; 6641 } 6642 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6643 var_info |= MAY_BE_RC1; 6644 } 6645 6646 |8: 6647 | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); 6648 if (opline->op1_type == IS_VAR) { 6649 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6650 if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6651 return 0; 6652 } 6653 } else { 6654 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6655 return 0; 6656 } 6657 } 6658 } 6659 } 6660 6661 if (((op1_info & MAY_BE_ARRAY) && 6662 (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) || 6663 (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY)))) { 6664 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6665 |.cold_code 6666 |7: 6667 } 6668 6669 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) && 6670 (op1_info & MAY_BE_ARRAY)) { 6671 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6672 | CMP_ZVAL_TYPE op1_addr, IS_NULL 6673 | jg >2 6674 } 6675 | // ZVAL_ARR(container, zend_new_array(8)); 6676 if (Z_REG(op1_addr) != ZREG_FP) { 6677 | mov T1, Ra(Z_REG(op1_addr)) // save 6678 } 6679 | EXT_CALL _zend_new_array_0, r0 6680 if (Z_REG(op1_addr) != ZREG_FP) { 6681 | mov Ra(Z_REG(op1_addr)), T1 // restore 6682 } 6683 | SET_ZVAL_LVAL op1_addr, r0 6684 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6685 | mov FCARG1a, r0 6686 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); 6687 | jmp <6 6688 |2: 6689 } 6690 6691 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6692 | SET_EX_OPLINE opline, r0 6693 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6694 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6695 } 6696 if (opline->op2_type == IS_UNUSED) { 6697 | xor FCARG2a, FCARG2a 6698 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6699 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6700 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 6701 } else { 6702 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6703 } 6704 |.if not(X64) 6705 | sub r4, 8 6706 |.endif 6707 if (opline->result_type == IS_UNUSED) { 6708 |.if X64 6709 | xor CARG4, CARG4 6710 |.else 6711 | push 0 6712 |.endif 6713 } else { 6714 |.if X64 6715 | LOAD_ZVAL_ADDR CARG4, res_addr 6716 |.else 6717 | PUSH_ZVAL_ADDR res_addr, r0 6718 |.endif 6719 } 6720 |.if X64 6721 | LOAD_ZVAL_ADDR CARG3, op3_addr 6722 |.else 6723 | PUSH_ZVAL_ADDR op3_addr, r0 6724 |.endif 6725 | EXT_CALL zend_jit_assign_dim_helper, r0 6726 |.if not(X64) 6727 | add r4, 8 6728 |.endif 6729 6730#ifdef ZEND_JIT_USE_RC_INFERENCE 6731 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { 6732 /* ASSIGN_DIM may increase refcount of the value */ 6733 val_info |= MAY_BE_RCN; 6734 } 6735#endif 6736 6737 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, NULL 6738 } 6739 6740 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6741 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6742 | jmp >9 // END 6743 } 6744 |.code 6745 } 6746 } 6747 6748#ifdef ZEND_JIT_USE_RC_INFERENCE 6749 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 6750 /* ASSIGN_DIM may increase refcount of the key */ 6751 op2_info |= MAY_BE_RCN; 6752 } 6753#endif 6754 6755 |9: 6756 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 6757 6758 if (may_throw) { 6759 zend_jit_check_exception(Dst); 6760 } 6761 6762 return 1; 6763} 6764 6765static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) 6766{ 6767 zend_jit_addr op2_addr, op3_addr, var_addr; 6768 const void *not_found_exit_addr = NULL; 6769 uint32_t var_info = MAY_BE_NULL; 6770 6771 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6772 6773 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6774 op3_addr = OP1_DATA_ADDR(); 6775 6776 | SET_EX_OPLINE opline, r0 6777 if (op1_info & MAY_BE_REF) { 6778 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6779 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 6780 | GET_Z_PTR FCARG2a, FCARG1a 6781 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 6782 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 6783 | jmp >3 6784 |.cold_code 6785 |2: 6786 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 6787 | test r0, r0 6788 | mov FCARG1a, r0 6789 | jne >1 6790 | jmp ->exception_handler_undef 6791 |.code 6792 |1: 6793 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6794 } 6795 6796 if (op1_info & MAY_BE_ARRAY) { 6797 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6798 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 6799 } 6800 |3: 6801 | SEPARATE_ARRAY op1_addr, op1_info, 1 6802 } 6803 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6804 if (op1_info & MAY_BE_ARRAY) { 6805 |.cold_code 6806 |7: 6807 } 6808 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6809 | CMP_ZVAL_TYPE op1_addr, IS_NULL 6810 | jg >7 6811 } 6812 if (Z_REG(op1_addr) != ZREG_FP) { 6813 | mov T1, Ra(Z_REG(op1_addr)) // save 6814 } 6815 if (op1_info & MAY_BE_UNDEF) { 6816 if (op1_info & MAY_BE_NULL) { 6817 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 6818 } 6819 | mov FCARG1a, opline->op1.var 6820 | EXT_CALL zend_jit_undefined_op_helper, r0 6821 |1: 6822 } 6823 | // ZVAL_ARR(container, zend_new_array(8)); 6824 | EXT_CALL _zend_new_array_0, r0 6825 if (Z_REG(op1_addr) != ZREG_FP) { 6826 | mov Ra(Z_REG(op1_addr)), T1 // restore 6827 } 6828 | SET_ZVAL_LVAL op1_addr, r0 6829 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6830 | mov FCARG1a, r0 6831 if (op1_info & MAY_BE_ARRAY) { 6832 | jmp >1 6833 |.code 6834 |1: 6835 } 6836 } 6837 6838 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6839 uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); 6840 6841 |6: 6842 if (opline->op2_type == IS_UNUSED) { 6843 var_info = MAY_BE_NULL; 6844 6845 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6846 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6847 | EXT_CALL zend_hash_next_index_insert, r0 6848 | // if (UNEXPECTED(!var_ptr)) { 6849 | test r0, r0 6850 | jz >1 6851 |.cold_code 6852 |1: 6853 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6854 | CANNOT_ADD_ELEMENT opline 6855 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6856 | jmp >9 6857 |.code 6858 } else { 6859 var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6860 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6861 var_info |= MAY_BE_REF; 6862 } 6863 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6864 var_info |= MAY_BE_RC1; 6865 } 6866 6867 if (dim_type != IS_UNKNOWN 6868 && dim_type != IS_UNDEF 6869 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY 6870 && (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) 6871 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 6872 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 6873 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6874 if (!not_found_exit_addr) { 6875 return 0; 6876 } 6877 } 6878 6879 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, not_found_exit_addr, NULL)) { 6880 return 0; 6881 } 6882 6883 |8: 6884 if (not_found_exit_addr && dim_type != IS_REFERENCE) { 6885 | IF_NOT_Z_TYPE, r0, dim_type, ¬_found_exit_addr 6886 var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 6887 } 6888 if (var_info & MAY_BE_REF) { 6889 binary_op_type binary_op = get_binary_op(opline->extended_value); 6890 | IF_NOT_Z_TYPE, r0, IS_REFERENCE, >1 6891 | GET_Z_PTR FCARG1a, r0 6892 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6893 | jnz >2 6894 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] 6895 |.cold_code 6896 |2: 6897 | LOAD_ZVAL_ADDR FCARG2a, op3_addr 6898 |.if X64 6899 | LOAD_ADDR CARG3, binary_op 6900 |.else 6901 | sub r4, 12 6902 | PUSH_ADDR binary_op, r0 6903 |.endif 6904 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 6905 && (op1_data_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6906 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 6907 } else { 6908 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 6909 } 6910 |.if not(X64) 6911 | add r4, 12 6912 |.endif 6913 | jmp >9 6914 |.code 6915 |1: 6916 } 6917 } 6918 6919 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6920 switch (opline->extended_value) { 6921 case ZEND_ADD: 6922 case ZEND_SUB: 6923 case ZEND_MUL: 6924 case ZEND_DIV: 6925 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, 6926 1 /* may overflow */, may_throw)) { 6927 return 0; 6928 } 6929 break; 6930 case ZEND_BW_OR: 6931 case ZEND_BW_AND: 6932 case ZEND_BW_XOR: 6933 case ZEND_SL: 6934 case ZEND_SR: 6935 case ZEND_MOD: 6936 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6937 IS_CV, opline->op1, var_addr, var_info, NULL, 6938 (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 6939 op1_data_range, 6940 0, var_addr, var_def_info, var_info, may_throw)) { 6941 return 0; 6942 } 6943 break; 6944 case ZEND_CONCAT: 6945 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, 6946 may_throw)) { 6947 return 0; 6948 } 6949 break; 6950 default: 6951 ZEND_UNREACHABLE(); 6952 } 6953 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 6954 } 6955 6956 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6957 binary_op_type binary_op; 6958 6959 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6960 |.cold_code 6961 |7: 6962 } 6963 6964 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6965 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6966 } 6967 if (opline->op2_type == IS_UNUSED) { 6968 | xor FCARG2a, FCARG2a 6969 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6970 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6971 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 6972 } else { 6973 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6974 } 6975 binary_op = get_binary_op(opline->extended_value); 6976 |.if X64 6977 | LOAD_ZVAL_ADDR CARG3, op3_addr 6978 | LOAD_ADDR CARG4, binary_op 6979 |.else 6980 | sub r4, 8 6981 | PUSH_ADDR binary_op, r0 6982 | PUSH_ZVAL_ADDR op3_addr, r0 6983 |.endif 6984 | EXT_CALL zend_jit_assign_dim_op_helper, r0 6985 |.if not(X64) 6986 | add r4, 8 6987 |.endif 6988 6989 |9: 6990 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, NULL 6991 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL 6992 if (may_throw) { 6993 zend_jit_check_exception(Dst); 6994 } 6995 6996 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6997 | jmp >9 // END 6998 |.code 6999 |9: 7000 } 7001 } else if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) 7002 && (!not_found_exit_addr || (var_info & MAY_BE_REF))) { 7003 |.cold_code 7004 |9: 7005 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, opline 7006 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 7007 if (may_throw) { 7008 zend_jit_check_exception(Dst); 7009 } 7010 | jmp >9 7011 |.code 7012 |9: 7013 } 7014 7015 return 1; 7016} 7017 7018static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) 7019{ 7020 zend_jit_addr op1_addr, op2_addr; 7021 7022 ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); 7023 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 7024 7025 op1_addr = OP1_ADDR(); 7026 op2_addr = OP2_ADDR(); 7027 7028 if (op1_info & MAY_BE_REF) { 7029 binary_op_type binary_op = get_binary_op(opline->extended_value); 7030 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 7031 | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >1 7032 | GET_Z_PTR FCARG1a, FCARG1a 7033 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 7034 | jnz >2 7035 | add FCARG1a, offsetof(zend_reference, val) 7036 |.cold_code 7037 |2: 7038 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 7039 |.if X64 7040 | LOAD_ADDR CARG3, binary_op 7041 |.else 7042 | sub r4, 12 7043 | PUSH_ADDR binary_op, r0 7044 |.endif 7045 | SET_EX_OPLINE opline, r0 7046 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 7047 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 7048 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 7049 } else { 7050 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 7051 } 7052 |.if not(X64) 7053 | add r4, 12 7054 |.endif 7055 zend_jit_check_exception(Dst); 7056 | jmp >9 7057 |.code 7058 |1: 7059 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 7060 } 7061 7062 int result; 7063 switch (opline->extended_value) { 7064 case ZEND_ADD: 7065 case ZEND_SUB: 7066 case ZEND_MUL: 7067 case ZEND_DIV: 7068 result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); 7069 break; 7070 case ZEND_BW_OR: 7071 case ZEND_BW_AND: 7072 case ZEND_BW_XOR: 7073 case ZEND_SL: 7074 case ZEND_SR: 7075 case ZEND_MOD: 7076 result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, 7077 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 7078 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 7079 opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); 7080 break; 7081 case ZEND_CONCAT: 7082 result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); 7083 break; 7084 default: 7085 ZEND_UNREACHABLE(); 7086 } 7087 |9: 7088 return result; 7089} 7090 7091static int zend_jit_cmp_long_long(dasm_State **Dst, 7092 const zend_op *opline, 7093 zend_ssa_range *op1_range, 7094 zend_jit_addr op1_addr, 7095 zend_ssa_range *op2_range, 7096 zend_jit_addr op2_addr, 7097 zend_jit_addr res_addr, 7098 zend_uchar smart_branch_opcode, 7099 uint32_t target_label, 7100 uint32_t target_label2, 7101 const void *exit_addr, 7102 bool skip_comparison) 7103{ 7104 bool swap = 0; 7105 bool result; 7106 7107 if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { 7108 if (!smart_branch_opcode || 7109 smart_branch_opcode == ZEND_JMPZ_EX || 7110 smart_branch_opcode == ZEND_JMPNZ_EX) { 7111 | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE) 7112 } 7113 if (smart_branch_opcode && !exit_addr) { 7114 if (smart_branch_opcode == ZEND_JMPZ || 7115 smart_branch_opcode == ZEND_JMPZ_EX) { 7116 if (!result) { 7117 | jmp => target_label 7118 } 7119 } else if (smart_branch_opcode == ZEND_JMPNZ || 7120 smart_branch_opcode == ZEND_JMPNZ_EX) { 7121 if (result) { 7122 | jmp => target_label 7123 } 7124 } else { 7125 ZEND_UNREACHABLE(); 7126 } 7127 } 7128 return 1; 7129 } 7130 7131 if (skip_comparison) { 7132 if (Z_MODE(op1_addr) != IS_REG && 7133 (Z_MODE(op2_addr) == IS_REG || 7134 (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { 7135 swap = 1; 7136 } 7137 } else if (Z_MODE(op1_addr) == IS_REG) { 7138 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 7139 | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) 7140 } else { 7141 | LONG_OP cmp, Z_REG(op1_addr), op2_addr, r0 7142 } 7143 } else if (Z_MODE(op2_addr) == IS_REG) { 7144 if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { 7145 | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) 7146 } else { 7147 | LONG_OP cmp, Z_REG(op2_addr), op1_addr, r0 7148 } 7149 swap = 1; 7150 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 7151 | LONG_OP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) 7152 swap = 1; 7153 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { 7154 | LONG_OP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)) 7155 } else { 7156 | GET_ZVAL_LVAL ZREG_R0, op1_addr 7157 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 7158 | test r0, r0 7159 } else { 7160 | LONG_OP cmp, ZREG_R0, op2_addr, r0 7161 } 7162 } 7163 7164 if (smart_branch_opcode) { 7165 if (smart_branch_opcode == ZEND_JMPZ_EX || 7166 smart_branch_opcode == ZEND_JMPNZ_EX) { 7167 7168 switch (opline->opcode) { 7169 case ZEND_IS_EQUAL: 7170 case ZEND_IS_IDENTICAL: 7171 case ZEND_CASE: 7172 case ZEND_CASE_STRICT: 7173 | sete al 7174 break; 7175 case ZEND_IS_NOT_EQUAL: 7176 case ZEND_IS_NOT_IDENTICAL: 7177 | setne al 7178 break; 7179 case ZEND_IS_SMALLER: 7180 if (swap) { 7181 | setg al 7182 } else { 7183 | setl al 7184 } 7185 break; 7186 case ZEND_IS_SMALLER_OR_EQUAL: 7187 if (swap) { 7188 | setge al 7189 } else { 7190 | setle al 7191 } 7192 break; 7193 default: 7194 ZEND_UNREACHABLE(); 7195 } 7196 | movzx eax, al 7197 | lea eax, [eax + 2] 7198 | SET_ZVAL_TYPE_INFO res_addr, eax 7199 } 7200 if (smart_branch_opcode == ZEND_JMPZ || 7201 smart_branch_opcode == ZEND_JMPZ_EX) { 7202 switch (opline->opcode) { 7203 case ZEND_IS_EQUAL: 7204 case ZEND_IS_IDENTICAL: 7205 case ZEND_CASE: 7206 case ZEND_CASE_STRICT: 7207 if (exit_addr) { 7208 | jne &exit_addr 7209 } else { 7210 | jne => target_label 7211 } 7212 break; 7213 case ZEND_IS_NOT_EQUAL: 7214 if (exit_addr) { 7215 | je &exit_addr 7216 } else { 7217 | je => target_label 7218 } 7219 break; 7220 case ZEND_IS_NOT_IDENTICAL: 7221 if (exit_addr) { 7222 | jne &exit_addr 7223 } else { 7224 | je => target_label 7225 } 7226 break; 7227 case ZEND_IS_SMALLER: 7228 if (swap) { 7229 if (exit_addr) { 7230 | jle &exit_addr 7231 } else { 7232 | jle => target_label 7233 } 7234 } else { 7235 if (exit_addr) { 7236 | jge &exit_addr 7237 } else { 7238 | jge => target_label 7239 } 7240 } 7241 break; 7242 case ZEND_IS_SMALLER_OR_EQUAL: 7243 if (swap) { 7244 if (exit_addr) { 7245 | jl &exit_addr 7246 } else { 7247 | jl => target_label 7248 } 7249 } else { 7250 if (exit_addr) { 7251 | jg &exit_addr 7252 } else { 7253 | jg => target_label 7254 } 7255 } 7256 break; 7257 default: 7258 ZEND_UNREACHABLE(); 7259 } 7260 } else if (smart_branch_opcode == ZEND_JMPNZ || 7261 smart_branch_opcode == ZEND_JMPNZ_EX) { 7262 switch (opline->opcode) { 7263 case ZEND_IS_EQUAL: 7264 case ZEND_IS_IDENTICAL: 7265 case ZEND_CASE: 7266 case ZEND_CASE_STRICT: 7267 if (exit_addr) { 7268 | je &exit_addr 7269 } else { 7270 | je => target_label 7271 } 7272 break; 7273 case ZEND_IS_NOT_EQUAL: 7274 if (exit_addr) { 7275 | jne &exit_addr 7276 } else { 7277 | jne => target_label 7278 } 7279 break; 7280 case ZEND_IS_NOT_IDENTICAL: 7281 if (exit_addr) { 7282 | je &exit_addr 7283 } else { 7284 | jne => target_label 7285 } 7286 break; 7287 case ZEND_IS_SMALLER: 7288 if (swap) { 7289 if (exit_addr) { 7290 | jg &exit_addr 7291 } else { 7292 | jg => target_label 7293 } 7294 } else { 7295 if (exit_addr) { 7296 | jl &exit_addr 7297 } else { 7298 | jl => target_label 7299 } 7300 } 7301 break; 7302 case ZEND_IS_SMALLER_OR_EQUAL: 7303 if (swap) { 7304 if (exit_addr) { 7305 | jge &exit_addr 7306 } else { 7307 | jge => target_label 7308 } 7309 } else { 7310 if (exit_addr) { 7311 | jle &exit_addr 7312 } else { 7313 | jle => target_label 7314 } 7315 } 7316 break; 7317 default: 7318 ZEND_UNREACHABLE(); 7319 } 7320 } else { 7321 ZEND_UNREACHABLE(); 7322 } 7323 } else { 7324 switch (opline->opcode) { 7325 case ZEND_IS_EQUAL: 7326 case ZEND_IS_IDENTICAL: 7327 case ZEND_CASE: 7328 case ZEND_CASE_STRICT: 7329 | sete al 7330 break; 7331 case ZEND_IS_NOT_EQUAL: 7332 case ZEND_IS_NOT_IDENTICAL: 7333 | setne al 7334 break; 7335 case ZEND_IS_SMALLER: 7336 if (swap) { 7337 | setg al 7338 } else { 7339 | setl al 7340 } 7341 break; 7342 case ZEND_IS_SMALLER_OR_EQUAL: 7343 if (swap) { 7344 | setge al 7345 } else { 7346 | setle al 7347 } 7348 break; 7349 default: 7350 ZEND_UNREACHABLE(); 7351 } 7352 | movzx eax, al 7353 | add eax, 2 7354 | SET_ZVAL_TYPE_INFO res_addr, eax 7355 } 7356 7357 return 1; 7358} 7359 7360static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7361{ 7362 if (smart_branch_opcode) { 7363 if (smart_branch_opcode == ZEND_JMPZ) { 7364 switch (opline->opcode) { 7365 case ZEND_IS_EQUAL: 7366 case ZEND_IS_IDENTICAL: 7367 case ZEND_CASE: 7368 case ZEND_CASE_STRICT: 7369 if (exit_addr) { 7370 | jne &exit_addr 7371 | jp &exit_addr 7372 } else { 7373 | jne => target_label 7374 | jp => target_label 7375 } 7376 break; 7377 case ZEND_IS_NOT_EQUAL: 7378 | jp >1 7379 if (exit_addr) { 7380 | je &exit_addr 7381 } else { 7382 | je => target_label 7383 } 7384 |1: 7385 break; 7386 case ZEND_IS_NOT_IDENTICAL: 7387 if (exit_addr) { 7388 | jne &exit_addr 7389 | jp &exit_addr 7390 } else { 7391 | jp >1 7392 | je => target_label 7393 |1: 7394 } 7395 break; 7396 case ZEND_IS_SMALLER: 7397 if (swap) { 7398 if (exit_addr) { 7399 | jbe &exit_addr 7400 } else { 7401 | jbe => target_label 7402 } 7403 } else { 7404 if (exit_addr) { 7405 | jae &exit_addr 7406 | jp &exit_addr 7407 } else { 7408 | jae => target_label 7409 | jp => target_label 7410 } 7411 } 7412 break; 7413 case ZEND_IS_SMALLER_OR_EQUAL: 7414 if (swap) { 7415 if (exit_addr) { 7416 | jb &exit_addr 7417 } else { 7418 | jb => target_label 7419 } 7420 } else { 7421 if (exit_addr) { 7422 | ja &exit_addr 7423 | jp &exit_addr 7424 } else { 7425 | ja => target_label 7426 | jp => target_label 7427 } 7428 } 7429 break; 7430 default: 7431 ZEND_UNREACHABLE(); 7432 } 7433 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7434 switch (opline->opcode) { 7435 case ZEND_IS_EQUAL: 7436 case ZEND_IS_IDENTICAL: 7437 case ZEND_CASE: 7438 case ZEND_CASE_STRICT: 7439 | jp >1 7440 if (exit_addr) { 7441 | je &exit_addr 7442 } else { 7443 | je => target_label 7444 } 7445 |1: 7446 break; 7447 case ZEND_IS_NOT_EQUAL: 7448 if (exit_addr) { 7449 | jne &exit_addr 7450 | jp &exit_addr 7451 } else { 7452 | jne => target_label 7453 | jp => target_label 7454 } 7455 break; 7456 case ZEND_IS_NOT_IDENTICAL: 7457 if (exit_addr) { 7458 | jp >1 7459 | je &exit_addr 7460 |1: 7461 } else { 7462 | jne => target_label 7463 | jp => target_label 7464 } 7465 break; 7466 case ZEND_IS_SMALLER: 7467 if (swap) { 7468 if (exit_addr) { 7469 | ja &exit_addr 7470 } else { 7471 | ja => target_label 7472 } 7473 } else { 7474 | jp >1 7475 if (exit_addr) { 7476 | jb &exit_addr 7477 } else { 7478 | jb => target_label 7479 } 7480 |1: 7481 } 7482 break; 7483 case ZEND_IS_SMALLER_OR_EQUAL: 7484 if (swap) { 7485 if (exit_addr) { 7486 | jae &exit_addr 7487 } else { 7488 | jae => target_label 7489 } 7490 } else { 7491 | jp >1 7492 if (exit_addr) { 7493 | jbe &exit_addr 7494 } else { 7495 | jbe => target_label 7496 } 7497 |1: 7498 } 7499 break; 7500 default: 7501 ZEND_UNREACHABLE(); 7502 } 7503 } else if (smart_branch_opcode == ZEND_JMPZ_EX) { 7504 switch (opline->opcode) { 7505 case ZEND_IS_EQUAL: 7506 case ZEND_IS_IDENTICAL: 7507 case ZEND_CASE: 7508 case ZEND_CASE_STRICT: 7509 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7510 | jne => target_label 7511 | jp => target_label 7512 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7513 break; 7514 case ZEND_IS_NOT_EQUAL: 7515 case ZEND_IS_NOT_IDENTICAL: 7516 | jp >1 7517 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7518 | je => target_label 7519 |1: 7520 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7521 break; 7522 case ZEND_IS_SMALLER: 7523 if (swap) { 7524 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7525 | jbe => target_label 7526 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7527 } else { 7528 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7529 | jae => target_label 7530 | jp => target_label 7531 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7532 } 7533 break; 7534 case ZEND_IS_SMALLER_OR_EQUAL: 7535 if (swap) { 7536 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7537 | jb => target_label 7538 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7539 } else { 7540 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7541 | ja => target_label 7542 | jp => target_label 7543 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7544 } 7545 break; 7546 default: 7547 ZEND_UNREACHABLE(); 7548 } 7549 } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { 7550 switch (opline->opcode) { 7551 case ZEND_IS_EQUAL: 7552 case ZEND_IS_IDENTICAL: 7553 case ZEND_CASE: 7554 case ZEND_CASE_STRICT: 7555 | jp >1 7556 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7557 | je => target_label 7558 |1: 7559 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7560 break; 7561 case ZEND_IS_NOT_EQUAL: 7562 case ZEND_IS_NOT_IDENTICAL: 7563 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7564 | jne => target_label 7565 | jp => target_label 7566 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7567 break; 7568 case ZEND_IS_SMALLER: 7569 if (swap) { 7570 | seta al 7571 | movzx eax, al 7572 | lea eax, [eax + 2] 7573 | SET_ZVAL_TYPE_INFO res_addr, eax 7574 | ja => target_label 7575 } else { 7576 | jp >1 7577 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7578 | jb => target_label 7579 |1: 7580 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7581 } 7582 break; 7583 case ZEND_IS_SMALLER_OR_EQUAL: 7584 if (swap) { 7585 | setae al 7586 | movzx eax, al 7587 | lea eax, [eax + 2] 7588 | SET_ZVAL_TYPE_INFO res_addr, eax 7589 | jae => target_label 7590 } else { 7591 | jp >1 7592 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7593 | jbe => target_label 7594 |1: 7595 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7596 } 7597 break; 7598 default: 7599 ZEND_UNREACHABLE(); 7600 } 7601 } else { 7602 ZEND_UNREACHABLE(); 7603 } 7604 } else { 7605 switch (opline->opcode) { 7606 case ZEND_IS_EQUAL: 7607 case ZEND_IS_IDENTICAL: 7608 case ZEND_CASE: 7609 case ZEND_CASE_STRICT: 7610 | jp >1 7611 | mov eax, IS_TRUE 7612 | je >2 7613 |1: 7614 | mov eax, IS_FALSE 7615 |2: 7616 break; 7617 case ZEND_IS_NOT_EQUAL: 7618 case ZEND_IS_NOT_IDENTICAL: 7619 | jp >1 7620 | mov eax, IS_FALSE 7621 | je >2 7622 |1: 7623 | mov eax, IS_TRUE 7624 |2: 7625 break; 7626 case ZEND_IS_SMALLER: 7627 if (swap) { 7628 | seta al 7629 | movzx eax, al 7630 | add eax, 2 7631 } else { 7632 | jp >1 7633 | mov eax, IS_TRUE 7634 | jb >2 7635 |1: 7636 | mov eax, IS_FALSE 7637 |2: 7638 } 7639 break; 7640 case ZEND_IS_SMALLER_OR_EQUAL: 7641 if (swap) { 7642 | setae al 7643 | movzx eax, al 7644 | add eax, 2 7645 } else { 7646 | jp >1 7647 | mov eax, IS_TRUE 7648 | jbe >2 7649 |1: 7650 | mov eax, IS_FALSE 7651 |2: 7652 } 7653 break; 7654 default: 7655 ZEND_UNREACHABLE(); 7656 } 7657 | SET_ZVAL_TYPE_INFO res_addr, eax 7658 } 7659 7660 return 1; 7661} 7662 7663static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7664{ 7665 zend_reg tmp_reg = ZREG_XMM0; 7666 7667 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_R0 7668 | DOUBLE_CMP tmp_reg, op2_addr 7669 7670 return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); 7671} 7672 7673static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7674{ 7675 zend_reg tmp_reg = ZREG_XMM0; 7676 7677 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_R0 7678 | DOUBLE_CMP tmp_reg, op1_addr 7679 7680 return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); 7681} 7682 7683static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7684{ 7685 bool swap = 0; 7686 7687 if (Z_MODE(op1_addr) == IS_REG) { 7688 | DOUBLE_CMP Z_REG(op1_addr), op2_addr 7689 } else if (Z_MODE(op2_addr) == IS_REG) { 7690 | DOUBLE_CMP Z_REG(op2_addr), op1_addr 7691 swap = 1; 7692 } else { 7693 zend_reg tmp_reg = ZREG_XMM0; 7694 7695 | DOUBLE_GET_ZVAL_DVAL tmp_reg, op1_addr 7696 | DOUBLE_CMP tmp_reg, op2_addr 7697 } 7698 7699 return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); 7700} 7701 7702static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7703{ 7704 | test, eax, eax 7705 if (smart_branch_opcode) { 7706 if (smart_branch_opcode == ZEND_JMPZ_EX || 7707 smart_branch_opcode == ZEND_JMPNZ_EX) { 7708 switch (opline->opcode) { 7709 case ZEND_IS_EQUAL: 7710 case ZEND_CASE: 7711 | sete al 7712 break; 7713 case ZEND_IS_NOT_EQUAL: 7714 | setne al 7715 break; 7716 case ZEND_IS_SMALLER: 7717 | setl al 7718 break; 7719 case ZEND_IS_SMALLER_OR_EQUAL: 7720 | setle al 7721 break; 7722 default: 7723 ZEND_UNREACHABLE(); 7724 } 7725 | movzx eax, al 7726 | lea eax, [eax + 2] 7727 | SET_ZVAL_TYPE_INFO res_addr, eax 7728 } 7729 if (smart_branch_opcode == ZEND_JMPZ || 7730 smart_branch_opcode == ZEND_JMPZ_EX) { 7731 switch (opline->opcode) { 7732 case ZEND_IS_EQUAL: 7733 case ZEND_CASE: 7734 if (exit_addr) { 7735 | jne &exit_addr 7736 } else { 7737 | jne => target_label 7738 } 7739 break; 7740 case ZEND_IS_NOT_EQUAL: 7741 if (exit_addr) { 7742 | je &exit_addr 7743 } else { 7744 | je => target_label 7745 } 7746 break; 7747 case ZEND_IS_SMALLER: 7748 if (exit_addr) { 7749 | jge &exit_addr 7750 } else { 7751 | jge => target_label 7752 } 7753 break; 7754 case ZEND_IS_SMALLER_OR_EQUAL: 7755 if (exit_addr) { 7756 | jg &exit_addr 7757 } else { 7758 | jg => target_label 7759 } 7760 break; 7761 default: 7762 ZEND_UNREACHABLE(); 7763 } 7764 } else if (smart_branch_opcode == ZEND_JMPNZ || 7765 smart_branch_opcode == ZEND_JMPNZ_EX) { 7766 switch (opline->opcode) { 7767 case ZEND_IS_EQUAL: 7768 case ZEND_CASE: 7769 if (exit_addr) { 7770 | je &exit_addr 7771 } else { 7772 | je => target_label 7773 } 7774 break; 7775 case ZEND_IS_NOT_EQUAL: 7776 if (exit_addr) { 7777 | jne &exit_addr 7778 } else { 7779 | jne => target_label 7780 } 7781 break; 7782 case ZEND_IS_SMALLER: 7783 if (exit_addr) { 7784 | jl &exit_addr 7785 } else { 7786 | jl => target_label 7787 } 7788 break; 7789 case ZEND_IS_SMALLER_OR_EQUAL: 7790 if (exit_addr) { 7791 | jle &exit_addr 7792 } else { 7793 | jle => target_label 7794 } 7795 break; 7796 default: 7797 ZEND_UNREACHABLE(); 7798 } 7799 } else { 7800 ZEND_UNREACHABLE(); 7801 } 7802 } else { 7803 switch (opline->opcode) { 7804 case ZEND_IS_EQUAL: 7805 case ZEND_CASE: 7806 | sete al 7807 break; 7808 case ZEND_IS_NOT_EQUAL: 7809 | setne al 7810 break; 7811 case ZEND_IS_SMALLER: 7812 | setl al 7813 break; 7814 case ZEND_IS_SMALLER_OR_EQUAL: 7815 | setle al 7816 break; 7817 default: 7818 ZEND_UNREACHABLE(); 7819 } 7820 | movzx eax, al 7821 | add eax, 2 7822 | SET_ZVAL_TYPE_INFO res_addr, eax 7823 } 7824 7825 return 1; 7826} 7827 7828static int zend_jit_cmp(dasm_State **Dst, 7829 const zend_op *opline, 7830 uint32_t op1_info, 7831 zend_ssa_range *op1_range, 7832 zend_jit_addr op1_addr, 7833 uint32_t op2_info, 7834 zend_ssa_range *op2_range, 7835 zend_jit_addr op2_addr, 7836 zend_jit_addr res_addr, 7837 int may_throw, 7838 zend_uchar smart_branch_opcode, 7839 uint32_t target_label, 7840 uint32_t target_label2, 7841 const void *exit_addr, 7842 bool skip_comparison) 7843{ 7844 bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); 7845 bool has_slow; 7846 7847 has_slow = 7848 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7849 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7850 ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7851 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); 7852 7853 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 7854 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 7855 if (op1_info & MAY_BE_DOUBLE) { 7856 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4 7857 } else { 7858 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 7859 } 7860 } 7861 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 7862 if (op2_info & MAY_BE_DOUBLE) { 7863 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 7864 |.cold_code 7865 |3: 7866 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7867 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7868 } 7869 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7870 return 0; 7871 } 7872 | jmp >6 7873 |.code 7874 } else { 7875 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7876 } 7877 } 7878 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 7879 return 0; 7880 } 7881 if (op1_info & MAY_BE_DOUBLE) { 7882 |.cold_code 7883 |4: 7884 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7885 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7886 } 7887 if (op2_info & MAY_BE_DOUBLE) { 7888 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7889 if (!same_ops) { 7890 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5 7891 } else { 7892 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7893 } 7894 } 7895 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7896 return 0; 7897 } 7898 | jmp >6 7899 } 7900 if (!same_ops) { 7901 |5: 7902 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7903 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7904 } 7905 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7906 return 0; 7907 } 7908 | jmp >6 7909 } 7910 |.code 7911 } 7912 } else if ((op1_info & MAY_BE_DOUBLE) && 7913 !(op1_info & MAY_BE_LONG) && 7914 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7915 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7916 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7917 } 7918 if (op2_info & MAY_BE_DOUBLE) { 7919 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7920 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7921 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3 7922 } else { 7923 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7924 } 7925 } 7926 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7927 return 0; 7928 } 7929 } 7930 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7931 if (op2_info & MAY_BE_DOUBLE) { 7932 |.cold_code 7933 } 7934 |3: 7935 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7936 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7937 } 7938 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7939 return 0; 7940 } 7941 if (op2_info & MAY_BE_DOUBLE) { 7942 | jmp >6 7943 |.code 7944 } 7945 } 7946 } else if ((op2_info & MAY_BE_DOUBLE) && 7947 !(op2_info & MAY_BE_LONG) && 7948 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7949 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7950 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7951 } 7952 if (op1_info & MAY_BE_DOUBLE) { 7953 if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7954 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7955 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3 7956 } else { 7957 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7958 } 7959 } 7960 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7961 return 0; 7962 } 7963 } 7964 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7965 if (op1_info & MAY_BE_DOUBLE) { 7966 |.cold_code 7967 } 7968 |3: 7969 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7970 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 7971 } 7972 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7973 return 0; 7974 } 7975 if (op1_info & MAY_BE_DOUBLE) { 7976 | jmp >6 7977 |.code 7978 } 7979 } 7980 } 7981 7982 if (has_slow || 7983 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7984 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 7985 if (has_slow) { 7986 |.cold_code 7987 |9: 7988 } 7989 | SET_EX_OPLINE opline, r0 7990 if (Z_MODE(op1_addr) == IS_REG) { 7991 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 7992 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 7993 return 0; 7994 } 7995 op1_addr = real_addr; 7996 } 7997 if (Z_MODE(op2_addr) == IS_REG) { 7998 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 7999 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 8000 return 0; 8001 } 8002 op2_addr = real_addr; 8003 } 8004 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8005 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { 8006 | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 8007 | mov FCARG1a, opline->op1.var 8008 | EXT_CALL zend_jit_undefined_op_helper, r0 8009 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8010 |1: 8011 } 8012 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { 8013 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 8014 | mov T1, FCARG1a // save 8015 | mov FCARG1a, opline->op2.var 8016 | EXT_CALL zend_jit_undefined_op_helper, r0 8017 | mov FCARG1a, T1 // restore 8018 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8019 | jmp >2 8020 |1: 8021 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8022 |2: 8023 } else { 8024 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8025 } 8026 | EXT_CALL zend_compare, r0 8027 if ((opline->opcode != ZEND_CASE && 8028 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8029 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8030 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8031 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8032 | mov dword T1, eax // save 8033 if (opline->opcode != ZEND_CASE) { 8034 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, NULL 8035 } 8036 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL 8037 if (may_throw) { 8038 zend_jit_check_exception_undef_result(Dst, opline); 8039 } 8040 | mov eax, dword T1 // restore 8041 } else if (may_throw) { 8042#if ZTS 8043 | mov dword T1, eax // save 8044#else 8045 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(exception)))) { 8046 | mov dword T1, eax // save 8047 } 8048#endif 8049 zend_jit_check_exception_undef_result(Dst, opline); 8050#if ZTS 8051 | mov eax, dword T1 // restore 8052#else 8053 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(exception)))) { 8054 | mov eax, dword T1 // restore 8055 } 8056#endif 8057 } 8058 if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8059 return 0; 8060 } 8061 if (has_slow) { 8062 | jmp >6 8063 |.code 8064 } 8065 } 8066 8067 |6: 8068 8069 return 1; 8070} 8071 8072static int zend_jit_identical(dasm_State **Dst, 8073 const zend_op *opline, 8074 uint32_t op1_info, 8075 zend_ssa_range *op1_range, 8076 zend_jit_addr op1_addr, 8077 uint32_t op2_info, 8078 zend_ssa_range *op2_range, 8079 zend_jit_addr op2_addr, 8080 zend_jit_addr res_addr, 8081 int may_throw, 8082 zend_uchar smart_branch_opcode, 8083 uint32_t target_label, 8084 uint32_t target_label2, 8085 const void *exit_addr, 8086 bool skip_comparison) 8087{ 8088 uint32_t identical_label = (uint32_t)-1; 8089 uint32_t not_identical_label = (uint32_t)-1; 8090 8091 if (smart_branch_opcode && !exit_addr) { 8092 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8093 if (smart_branch_opcode == ZEND_JMPZ) { 8094 not_identical_label = target_label; 8095 } else if (smart_branch_opcode == ZEND_JMPNZ) { 8096 identical_label = target_label; 8097 } else { 8098 ZEND_UNREACHABLE(); 8099 } 8100 } else { 8101 if (smart_branch_opcode == ZEND_JMPZ) { 8102 identical_label = target_label; 8103 } else if (smart_branch_opcode == ZEND_JMPNZ) { 8104 not_identical_label = target_label; 8105 } else { 8106 ZEND_UNREACHABLE(); 8107 } 8108 } 8109 } 8110 8111 if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && 8112 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { 8113 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 8114 return 0; 8115 } 8116 return 1; 8117 } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && 8118 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { 8119 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8120 return 0; 8121 } 8122 return 1; 8123 } 8124 8125 if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { 8126 op1_info |= MAY_BE_NULL; 8127 op2_info |= MAY_BE_NULL; 8128 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8129 | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 8130 |.cold_code 8131 |1: 8132 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8133 | SET_EX_OPLINE opline, r0 8134 | mov FCARG1d, opline->op1.var 8135 | EXT_CALL zend_jit_undefined_op_helper, r0 8136 if (may_throw) { 8137 zend_jit_check_exception_undef_result(Dst, opline); 8138 } 8139 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8140 | jmp >1 8141 |.code 8142 |1: 8143 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8144 | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 8145 |.cold_code 8146 |1: 8147 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8148 | SET_EX_OPLINE opline, r0 8149 | mov aword T1, FCARG1a // save 8150 | mov FCARG1d, opline->op2.var 8151 | EXT_CALL zend_jit_undefined_op_helper, r0 8152 if (may_throw) { 8153 zend_jit_check_exception_undef_result(Dst, opline); 8154 } 8155 | mov FCARG1a, aword T1 // restore 8156 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8157 | jmp >1 8158 |.code 8159 |1: 8160 } else if (op1_info & MAY_BE_UNDEF) { 8161 op1_info |= MAY_BE_NULL; 8162 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8163 | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 8164 |.cold_code 8165 |1: 8166 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8167 | SET_EX_OPLINE opline, r0 8168 | mov FCARG1d, opline->op1.var 8169 | EXT_CALL zend_jit_undefined_op_helper, r0 8170 if (may_throw) { 8171 zend_jit_check_exception_undef_result(Dst, opline); 8172 } 8173 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8174 | jmp >1 8175 |.code 8176 |1: 8177 if (opline->op2_type != IS_CONST) { 8178 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8179 } 8180 } else if (op2_info & MAY_BE_UNDEF) { 8181 op2_info |= MAY_BE_NULL; 8182 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8183 | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 8184 |.cold_code 8185 |1: 8186 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8187 | SET_EX_OPLINE opline, r0 8188 | mov FCARG1d, opline->op2.var 8189 | EXT_CALL zend_jit_undefined_op_helper, r0 8190 if (may_throw) { 8191 zend_jit_check_exception_undef_result(Dst, opline); 8192 } 8193 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8194 | jmp >1 8195 |.code 8196 |1: 8197 if (opline->op1_type != IS_CONST) { 8198 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8199 } 8200 } else if ((op1_info & op2_info & MAY_BE_ANY) != 0) { 8201 if (opline->op1_type != IS_CONST) { 8202 if (Z_MODE(op1_addr) == IS_REG) { 8203 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8204 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 8205 return 0; 8206 } 8207 op1_addr = real_addr; 8208 } 8209 } 8210 if (opline->op2_type != IS_CONST) { 8211 if (Z_MODE(op2_addr) == IS_REG) { 8212 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 8213 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 8214 return 0; 8215 } 8216 op2_addr = real_addr; 8217 } 8218 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8219 } 8220 if (opline->op1_type != IS_CONST) { 8221 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8222 } 8223 } 8224 8225 if ((op1_info & op2_info & MAY_BE_ANY) == 0) { 8226 if ((opline->opcode != ZEND_CASE_STRICT && 8227 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8228 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8229 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8230 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8231 if (opline->opcode != ZEND_CASE_STRICT) { 8232 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8233 } 8234 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8235 } 8236 if (smart_branch_opcode) { 8237 if (may_throw) { 8238 zend_jit_check_exception_undef_result(Dst, opline); 8239 } 8240 if (exit_addr) { 8241 if (smart_branch_opcode == ZEND_JMPZ) { 8242 | jmp &exit_addr 8243 } 8244 } else if (not_identical_label != (uint32_t)-1) { 8245 | jmp =>not_identical_label 8246 } 8247 } else { 8248 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) 8249 if (may_throw) { 8250 zend_jit_check_exception(Dst); 8251 } 8252 } 8253 return 1; 8254 } 8255 8256 if (opline->op1_type & (IS_CV|IS_VAR)) { 8257 | ZVAL_DEREF FCARG1a, op1_info 8258 } 8259 if (opline->op2_type & (IS_CV|IS_VAR)) { 8260 | ZVAL_DEREF FCARG2a, op2_info 8261 } 8262 8263 if (has_concrete_type(op1_info) 8264 && has_concrete_type(op2_info) 8265 && concrete_type(op1_info) == concrete_type(op2_info) 8266 && concrete_type(op1_info) <= IS_TRUE) { 8267 if (smart_branch_opcode) { 8268 if (exit_addr) { 8269 if (smart_branch_opcode == ZEND_JMPNZ) { 8270 | jmp &exit_addr 8271 } 8272 } else if (identical_label != (uint32_t)-1) { 8273 | jmp =>identical_label 8274 } 8275 } else { 8276 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) 8277 } 8278 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { 8279 if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { 8280 if (smart_branch_opcode) { 8281 if (exit_addr) { 8282 if (smart_branch_opcode == ZEND_JMPNZ) { 8283 | jmp &exit_addr 8284 } 8285 } else if (identical_label != (uint32_t)-1) { 8286 | jmp =>identical_label 8287 } 8288 } else { 8289 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) 8290 } 8291 } else { 8292 if (smart_branch_opcode) { 8293 if (exit_addr) { 8294 if (smart_branch_opcode == ZEND_JMPZ) { 8295 | jmp &exit_addr 8296 } 8297 } else if (not_identical_label != (uint32_t)-1) { 8298 | jmp =>not_identical_label 8299 } 8300 } else { 8301 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) 8302 } 8303 } 8304 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { 8305 zval *val = Z_ZV(op1_addr); 8306 8307 | cmp byte [FCARG2a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) 8308 if (smart_branch_opcode) { 8309 if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { 8310 | jne >8 8311 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8312 if (may_throw) { 8313 zend_jit_check_exception_undef_result(Dst, opline); 8314 } 8315 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8316 | jmp &exit_addr 8317 } else if (identical_label != (uint32_t)-1) { 8318 | jmp =>identical_label 8319 } else { 8320 | jmp >9 8321 } 8322 |8: 8323 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8324 | je &exit_addr 8325 } else if (identical_label != (uint32_t)-1) { 8326 | je =>identical_label 8327 } else { 8328 | je >9 8329 } 8330 } else { 8331 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8332 | sete al 8333 } else { 8334 | setne al 8335 } 8336 | movzx eax, al 8337 | lea eax, [eax + 2] 8338 | SET_ZVAL_TYPE_INFO res_addr, eax 8339 } 8340 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8341 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 8342 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8343 if (may_throw) { 8344 zend_jit_check_exception_undef_result(Dst, opline); 8345 } 8346 } 8347 if (exit_addr) { 8348 if (smart_branch_opcode == ZEND_JMPZ) { 8349 | jmp &exit_addr 8350 } 8351 } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { 8352 | jmp =>not_identical_label 8353 } 8354 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { 8355 zval *val = Z_ZV(op2_addr); 8356 8357 | cmp byte [FCARG1a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) 8358 if (smart_branch_opcode) { 8359 if (opline->opcode != ZEND_CASE_STRICT 8360 && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { 8361 | jne >8 8362 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8363 if (may_throw) { 8364 zend_jit_check_exception_undef_result(Dst, opline); 8365 } 8366 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8367 | jmp &exit_addr 8368 } else if (identical_label != (uint32_t)-1) { 8369 | jmp =>identical_label 8370 } else { 8371 | jmp >9 8372 } 8373 |8: 8374 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8375 | je &exit_addr 8376 } else if (identical_label != (uint32_t)-1) { 8377 | je =>identical_label 8378 } else { 8379 | je >9 8380 } 8381 } else { 8382 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8383 | sete al 8384 } else { 8385 | setne al 8386 } 8387 | movzx eax, al 8388 | lea eax, [eax + 2] 8389 | SET_ZVAL_TYPE_INFO res_addr, eax 8390 } 8391 if (opline->opcode != ZEND_CASE_STRICT 8392 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8393 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 8394 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8395 if (may_throw) { 8396 zend_jit_check_exception_undef_result(Dst, opline); 8397 } 8398 } 8399 if (smart_branch_opcode) { 8400 if (exit_addr) { 8401 if (smart_branch_opcode == ZEND_JMPZ) { 8402 | jmp &exit_addr 8403 } 8404 } else if (not_identical_label != (uint32_t)-1) { 8405 | jmp =>not_identical_label 8406 } 8407 } 8408 } else { 8409 if (opline->op1_type == IS_CONST) { 8410 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8411 } 8412 if (opline->op2_type == IS_CONST) { 8413 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8414 } 8415 | EXT_CALL zend_is_identical, r0 8416 if ((opline->opcode != ZEND_CASE_STRICT && 8417 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8418 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8419 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8420 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8421 | mov aword T1, r0 // save 8422 if (opline->opcode != ZEND_CASE_STRICT) { 8423 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8424 } 8425 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8426 if (may_throw) { 8427 zend_jit_check_exception_undef_result(Dst, opline); 8428 } 8429 | mov r0, aword T1 // restore 8430 } 8431 if (smart_branch_opcode) { 8432 | test al, al 8433 if (exit_addr) { 8434 if (smart_branch_opcode == ZEND_JMPNZ) { 8435 | jnz &exit_addr 8436 } else { 8437 | jz &exit_addr 8438 } 8439 } else if (not_identical_label != (uint32_t)-1) { 8440 | jz =>not_identical_label 8441 if (identical_label != (uint32_t)-1) { 8442 | jmp =>identical_label 8443 } 8444 } else if (identical_label != (uint32_t)-1) { 8445 | jnz =>identical_label 8446 } 8447 } else { 8448 | movzx eax, al 8449 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8450 | lea eax, [eax + 2] 8451 } else { 8452 | neg eax 8453 | lea eax, [eax + 3] 8454 } 8455 | SET_ZVAL_TYPE_INFO res_addr, eax 8456 } 8457 } 8458 8459 |9: 8460 if (may_throw) { 8461 zend_jit_check_exception(Dst); 8462 } 8463 return 1; 8464} 8465 8466static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr) 8467{ 8468 uint32_t true_label = -1; 8469 uint32_t false_label = -1; 8470 bool set_bool = 0; 8471 bool set_bool_not = 0; 8472 bool set_delayed = 0; 8473 bool jmp_done = 0; 8474 8475 if (branch_opcode == ZEND_BOOL) { 8476 set_bool = 1; 8477 } else if (branch_opcode == ZEND_BOOL_NOT) { 8478 set_bool = 1; 8479 set_bool_not = 1; 8480 } else if (branch_opcode == ZEND_JMPZ) { 8481 false_label = target_label; 8482 } else if (branch_opcode == ZEND_JMPNZ) { 8483 true_label = target_label; 8484 } else if (branch_opcode == ZEND_JMPZ_EX) { 8485 set_bool = 1; 8486 false_label = target_label; 8487 } else if (branch_opcode == ZEND_JMPNZ_EX) { 8488 set_bool = 1; 8489 true_label = target_label; 8490 } else { 8491 ZEND_UNREACHABLE(); 8492 } 8493 8494 if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { 8495 if (zend_is_true(Z_ZV(op1_addr))) { 8496 /* Always TRUE */ 8497 if (set_bool) { 8498 if (set_bool_not) { 8499 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8500 } else { 8501 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8502 } 8503 } 8504 if (true_label != (uint32_t)-1) { 8505 | jmp =>true_label; 8506 } 8507 } else { 8508 /* Always FALSE */ 8509 if (set_bool) { 8510 if (set_bool_not) { 8511 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8512 } else { 8513 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8514 } 8515 } 8516 if (false_label != (uint32_t)-1) { 8517 | jmp =>false_label; 8518 } 8519 } 8520 return 1; 8521 } 8522 8523 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) { 8524 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8525 | ZVAL_DEREF FCARG1a, op1_info 8526 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 8527 } 8528 8529 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { 8530 if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { 8531 /* Always TRUE */ 8532 if (set_bool) { 8533 if (set_bool_not) { 8534 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8535 } else { 8536 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8537 } 8538 } 8539 if (true_label != (uint32_t)-1) { 8540 | jmp =>true_label; 8541 } 8542 } else { 8543 if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { 8544 /* Always FALSE */ 8545 if (set_bool) { 8546 if (set_bool_not) { 8547 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8548 } else { 8549 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8550 } 8551 } 8552 } else { 8553 | CMP_ZVAL_TYPE op1_addr, IS_TRUE 8554 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 8555 if ((op1_info & MAY_BE_LONG) && 8556 !(op1_info & MAY_BE_UNDEF) && 8557 !set_bool) { 8558 if (exit_addr) { 8559 if (branch_opcode == ZEND_JMPNZ) { 8560 | jl >9 8561 } else { 8562 | jl &exit_addr 8563 } 8564 } else if (false_label != (uint32_t)-1) { 8565 | jl =>false_label 8566 } else { 8567 | jl >9 8568 } 8569 jmp_done = 1; 8570 } else { 8571 | jg >2 8572 } 8573 } 8574 if (!(op1_info & MAY_BE_TRUE)) { 8575 /* It's FALSE */ 8576 if (set_bool) { 8577 if (set_bool_not) { 8578 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8579 } else { 8580 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8581 } 8582 } 8583 } else { 8584 if (exit_addr) { 8585 if (set_bool) { 8586 | jne >1 8587 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8588 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8589 | jmp &exit_addr 8590 } else { 8591 | jmp >9 8592 } 8593 |1: 8594 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8595 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8596 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8597 | jne &exit_addr 8598 } 8599 } 8600 } else { 8601 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8602 | je &exit_addr 8603 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8604 | jne &exit_addr 8605 } else { 8606 | je >9 8607 } 8608 } 8609 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8610 if (set_bool) { 8611 | jne >1 8612 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8613 if (true_label != (uint32_t)-1) { 8614 | jmp =>true_label 8615 } else { 8616 | jmp >9 8617 } 8618 |1: 8619 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8620 } else { 8621 if (true_label != (uint32_t)-1) { 8622 | je =>true_label 8623 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8624 | jne =>false_label 8625 jmp_done = 1; 8626 } else { 8627 | je >9 8628 } 8629 } 8630 } else if (set_bool) { 8631 | sete al 8632 | movzx eax, al 8633 if (set_bool_not) { 8634 | neg eax 8635 | add eax, 3 8636 } else { 8637 | add eax, 2 8638 } 8639 if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { 8640 set_delayed = 1; 8641 } else { 8642 | SET_ZVAL_TYPE_INFO res_addr, eax 8643 } 8644 } 8645 } 8646 } 8647 8648 /* It's FALSE, but may be UNDEF */ 8649 if (op1_info & MAY_BE_UNDEF) { 8650 if (op1_info & MAY_BE_ANY) { 8651 if (set_delayed) { 8652 | CMP_ZVAL_TYPE op1_addr, IS_UNDEF 8653 | SET_ZVAL_TYPE_INFO res_addr, eax 8654 | jz >1 8655 } else { 8656 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 8657 } 8658 |.cold_code 8659 |1: 8660 } 8661 | mov FCARG1d, opline->op1.var 8662 | SET_EX_OPLINE opline, r0 8663 | EXT_CALL zend_jit_undefined_op_helper, r0 8664 8665 if (may_throw) { 8666 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 8667 return 0; 8668 } 8669 } 8670 8671 if (exit_addr) { 8672 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8673 | jmp &exit_addr 8674 } 8675 } else if (false_label != (uint32_t)-1) { 8676 | jmp =>false_label 8677 } 8678 if (op1_info & MAY_BE_ANY) { 8679 if (exit_addr) { 8680 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8681 | jmp >9 8682 } 8683 } else if (false_label == (uint32_t)-1) { 8684 | jmp >9 8685 } 8686 |.code 8687 } 8688 } 8689 8690 if (!jmp_done) { 8691 if (exit_addr) { 8692 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8693 if (op1_info & MAY_BE_LONG) { 8694 | jmp >9 8695 } 8696 } else if (op1_info & MAY_BE_LONG) { 8697 | jmp &exit_addr 8698 } 8699 } else if (false_label != (uint32_t)-1) { 8700 | jmp =>false_label 8701 } else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 8702 | jmp >9 8703 } 8704 } 8705 } 8706 } 8707 8708 if (op1_info & MAY_BE_LONG) { 8709 |2: 8710 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8711 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 8712 } 8713 if (Z_MODE(op1_addr) == IS_REG) { 8714 | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) 8715 } else { 8716 | LONG_OP_WITH_CONST, cmp, op1_addr, Z_L(0) 8717 } 8718 if (set_bool) { 8719 | setne al 8720 | movzx eax, al 8721 if (set_bool_not) { 8722 | neg eax 8723 | add eax, 3 8724 } else { 8725 | lea eax, [eax + 2] 8726 } 8727 | SET_ZVAL_TYPE_INFO res_addr, eax 8728 } 8729 if (exit_addr) { 8730 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8731 | jne &exit_addr 8732 } else { 8733 | je &exit_addr 8734 } 8735 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8736 if (true_label != (uint32_t)-1) { 8737 | jne =>true_label 8738 if (false_label != (uint32_t)-1) { 8739 | jmp =>false_label 8740 } 8741 } else { 8742 | je =>false_label 8743 } 8744 } 8745 } 8746 8747 if ((op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) == MAY_BE_DOUBLE) { 8748 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8749 |.cold_code 8750 } 8751 |2: 8752 if (CAN_USE_AVX()) { 8753 | vxorps xmm0, xmm0, xmm0 8754 } else { 8755 | xorps xmm0, xmm0 8756 } 8757 | DOUBLE_CMP ZREG_XMM0, op1_addr 8758 8759 if (set_bool) { 8760 if (exit_addr) { 8761 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8762 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8763 | jp &exit_addr 8764 | jne &exit_addr 8765 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8766 } else { 8767 | jp >1 8768 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8769 | je &exit_addr 8770 |1: 8771 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8772 } 8773 } else if (false_label != (uint32_t)-1) { // JMPZ_EX (p=>true, z=>false, false=>jmp) 8774 | jp >1 8775 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8776 | je => false_label 8777 |1: 8778 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8779 } else if (true_label != (uint32_t)-1) { // JMPNZ_EX (p=>true, z=>false, true=>jmp) 8780 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8781 | jp => true_label 8782 | jne => true_label 8783 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8784 } else if (set_bool_not) { // BOOL_NOT (p=>false, z=>true) 8785 | mov eax, IS_FALSE 8786 | jp >1 8787 | jne >1 8788 | mov eax, IS_TRUE 8789 |1: 8790 | SET_ZVAL_TYPE_INFO res_addr, eax 8791 } else { // BOOL (p=>true, z=>false) 8792 | mov eax, IS_TRUE 8793 | jp >1 8794 | jne >1 8795 | mov eax, IS_FALSE 8796 |1: 8797 | SET_ZVAL_TYPE_INFO res_addr, eax 8798 } 8799 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8800 | jmp >9 8801 |.code 8802 } 8803 } else { 8804 if (exit_addr) { 8805 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8806 | jp &exit_addr 8807 | jne &exit_addr 8808 |1: 8809 } else { 8810 | jp >1 8811 | je &exit_addr 8812 |1: 8813 } 8814 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8815 | jmp >9 8816 } 8817 } else { 8818 ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); 8819 if (false_label != (uint32_t)-1 ) { 8820 | jp >1 8821 | je => false_label 8822 |1: 8823 if (true_label != (uint32_t)-1) { 8824 | jmp =>true_label 8825 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8826 | jmp >9 8827 } 8828 } else { 8829 | jp => true_label 8830 | jne => true_label 8831 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8832 | jmp >9 8833 } 8834 } 8835 } 8836 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8837 |.code 8838 } 8839 } 8840 } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8841 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8842 |.cold_code 8843 |2: 8844 } 8845 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8846 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8847 } 8848 | SET_EX_OPLINE opline, r0 8849 | EXT_CALL zend_is_true, r0 8850 8851 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8852 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8853 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8854 8855 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8856 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3 8857 } 8858 | GET_ZVAL_PTR FCARG1a, op1_addr 8859 | GC_DELREF FCARG1a 8860 | jnz >3 8861 | mov aword T1, r0 // save 8862 | ZVAL_DTOR_FUNC op1_info, opline 8863 | mov r0, aword T1 // restore 8864 |3: 8865 } 8866 if (may_throw) { 8867 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r1 8868 | jne ->exception_handler_undef 8869 } 8870 8871 if (set_bool) { 8872 if (set_bool_not) { 8873 | neg eax 8874 | add eax, 3 8875 } else { 8876 | add eax, 2 8877 } 8878 | SET_ZVAL_TYPE_INFO res_addr, eax 8879 if (exit_addr) { 8880 | CMP_ZVAL_TYPE res_addr, IS_FALSE 8881 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8882 | jne &exit_addr 8883 } else { 8884 | je &exit_addr 8885 } 8886 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8887 | CMP_ZVAL_TYPE res_addr, IS_FALSE 8888 if (true_label != (uint32_t)-1) { 8889 | jne =>true_label 8890 if (false_label != (uint32_t)-1) { 8891 | jmp =>false_label 8892 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8893 | jmp >9 8894 } 8895 } else { 8896 | je =>false_label 8897 } 8898 } 8899 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8900 | jmp >9 8901 |.code 8902 } 8903 } else { 8904 | test r0, r0 8905 if (exit_addr) { 8906 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8907 | jne &exit_addr 8908 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8909 | jmp >9 8910 } 8911 } else { 8912 | je &exit_addr 8913 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8914 | jmp >9 8915 } 8916 } 8917 } else if (true_label != (uint32_t)-1) { 8918 | jne =>true_label 8919 if (false_label != (uint32_t)-1) { 8920 | jmp =>false_label 8921 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8922 | jmp >9 8923 } 8924 } else { 8925 | je =>false_label 8926 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8927 | jmp >9 8928 } 8929 } 8930 8931 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8932 |.code 8933 } 8934 } 8935 } 8936 8937 |9: 8938 8939 return 1; 8940} 8941 8942static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) 8943{ 8944 if (op1_addr != op1_def_addr) { 8945 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 8946 return 0; 8947 } 8948 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 8949 op1_addr = op1_def_addr; 8950 } 8951 } 8952 8953 if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0, 1)) { 8954 return 0; 8955 } 8956 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8957 return 0; 8958 } 8959 if (op1_info & MAY_BE_UNDEF) { 8960 zend_jit_check_exception(Dst); 8961 } 8962 return 1; 8963} 8964 8965static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 8966{ 8967 ZEND_ASSERT(opline->op1_type == IS_CV); 8968 8969 if (op2_addr != op2_def_addr) { 8970 if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { 8971 return 0; 8972 } 8973 if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { 8974 op2_addr = op2_def_addr; 8975 } 8976 } 8977 8978 if (Z_MODE(op1_addr) != IS_REG 8979 && Z_MODE(op1_use_addr) == IS_REG 8980 && !Z_LOAD(op1_use_addr) 8981 && !Z_STORE(op1_use_addr)) { 8982 /* Force type update */ 8983 op1_info |= MAY_BE_UNDEF; 8984 } 8985 if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, 8986 may_throw)) { 8987 return 0; 8988 } 8989 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { 8990 return 0; 8991 } 8992 if (opline->result_type != IS_UNUSED) { 8993 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8994 return 0; 8995 } 8996 } 8997 8998 return 1; 8999} 9000 9001/* copy of hidden zend_closure */ 9002typedef struct _zend_closure { 9003 zend_object std; 9004 zend_function func; 9005 zval this_ptr; 9006 zend_class_entry *called_scope; 9007 zif_handler orig_internal_handler; 9008} zend_closure; 9009 9010static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) 9011{ 9012 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9013 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9014 9015 if (!exit_addr) { 9016 return 0; 9017 } 9018 9019 | // Check Stack Overflow 9020 | MEM_LOAD_ZTS r1, aword, executor_globals, vm_stack_end, r0 9021 | MEM_LOAD_OP_ZTS sub, r1, aword, executor_globals, vm_stack_top, r0 9022 | cmp r1, used_stack 9023 | jb &exit_addr 9024 9025 return 1; 9026} 9027 9028static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack) 9029{ 9030 uint32_t used_stack; 9031 bool stack_check = 1; 9032 9033 if (func) { 9034 used_stack = zend_vm_calc_used_stack(opline->extended_value, func); 9035 if ((int)used_stack <= checked_stack) { 9036 stack_check = 0; 9037 } 9038 } else { 9039 used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value + ZEND_OBSERVER_ENABLED) * sizeof(zval); 9040 9041 | // if (EXPECTED(ZEND_USER_CODE(func->type))) { 9042 if (!is_closure) { 9043 | test byte [r0 + offsetof(zend_function, type)], 1 9044 | mov FCARG1a, used_stack 9045 | jnz >1 9046 } else { 9047 | mov FCARG1a, used_stack 9048 } 9049 | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); 9050 | mov edx, opline->extended_value 9051 if (!is_closure) { 9052 | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)] 9053 | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)] 9054 | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)] 9055 | sub edx, dword [r0 + offsetof(zend_function, op_array.T)] 9056 } else { 9057 | cmp edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] 9058 | cmova edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] 9059 | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.last_var)] 9060 | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.T)] 9061 } 9062 | shl edx, 4 9063 |.if X64 9064 | movsxd r2, edx 9065 |.endif 9066 | sub FCARG1a, r2 9067 |1: 9068 } 9069 9070 zend_jit_start_reuse_ip(); 9071 9072 | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { 9073 | MEM_LOAD_ZTS RX, aword, executor_globals, vm_stack_top, RX 9074 9075 if (stack_check) { 9076 | // Check Stack Overflow 9077 | MEM_LOAD_ZTS r2, aword, executor_globals, vm_stack_end, r2 9078 | sub r2, RX 9079 if (func) { 9080 | cmp r2, used_stack 9081 } else { 9082 | cmp r2, FCARG1a 9083 } 9084 9085 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9086 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9087 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9088 9089 if (!exit_addr) { 9090 return 0; 9091 } 9092 9093 | jb &exit_addr 9094 } else { 9095 | jb >1 9096 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); 9097 |.cold_code 9098 |1: 9099 if (func) { 9100 | mov FCARG1d, used_stack 9101 } 9102#ifdef _WIN32 9103 if (0) { 9104#else 9105 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 9106#endif 9107 | SET_EX_OPLINE opline, r0 9108 | EXT_CALL zend_jit_int_extend_stack_helper, r0 9109 } else { 9110 if (!is_closure) { 9111 if (func 9112 && op_array == &func->op_array 9113 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9114 && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { 9115 | LOAD_ADDR FCARG2a, func 9116 } else { 9117 | mov FCARG2a, r0 9118 } 9119 } else { 9120 | lea FCARG2a, aword [r0 + offsetof(zend_closure, func)] 9121 } 9122 | SET_EX_OPLINE opline, r0 9123 | EXT_CALL zend_jit_extend_stack_helper, r0 9124 } 9125 | mov RX, r0 9126 | jmp >1 9127 |.code 9128 } 9129 } 9130 9131 if (func) { 9132 | MEM_UPDATE_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2 9133 } else { 9134 | MEM_UPDATE_ZTS add, aword, executor_globals, vm_stack_top, FCARG1a, r2 9135 } 9136 | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); 9137 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { 9138 | // ZEND_SET_CALL_INFO(call, 0, call_info); 9139 | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) 9140 } 9141#ifdef _WIN32 9142 if (0) { 9143#else 9144 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 9145#endif 9146 | // call->func = func; 9147 |1: 9148 | ADDR_STORE aword EX:RX->func, func, r1 9149 } else { 9150 if (!is_closure) { 9151 | // call->func = func; 9152 if (func 9153 && op_array == &func->op_array 9154 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9155 && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { 9156 | ADDR_STORE aword EX:RX->func, func, r1 9157 } else { 9158 | mov aword EX:RX->func, r0 9159 } 9160 } else { 9161 | // call->func = &closure->func; 9162 | lea r1, aword [r0 + offsetof(zend_closure, func)] 9163 | mov aword EX:RX->func, r1 9164 } 9165 |1: 9166 } 9167 if (opline->opcode == ZEND_INIT_METHOD_CALL) { 9168 | // Z_PTR(call->This) = obj; 9169 | mov r1, aword T1 9170 | mov aword EX:RX->This.value.ptr, r1 9171 if (opline->op1_type == IS_UNUSED || delayed_fetch_this) { 9172 | // call->call_info |= ZEND_CALL_HAS_THIS; 9173 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9174 | mov dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS 9175 } else { 9176 | or dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS 9177 } 9178 } else { 9179 if (opline->op1_type == IS_CV) { 9180 | // GC_ADDREF(obj); 9181 | add dword [r1], 1 9182 } 9183 | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; 9184 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9185 | mov dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 9186 } else { 9187 | or dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 9188 } 9189 } 9190 } else if (!is_closure) { 9191 | // Z_CE(call->This) = called_scope; 9192 | mov aword EX:RX->This.value.ptr, 0 9193 } else { 9194 if (opline->op2_type == IS_CV) { 9195 | // GC_ADDREF(closure); 9196 | add dword [r0], 1 9197 } 9198 | // object_or_called_scope = closure->called_scope; 9199 | mov r1, aword [r0 + offsetof(zend_closure, called_scope)] 9200 | mov aword EX:RX->This.value.ptr, r1 9201 | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | 9202 | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); 9203 | mov edx, dword [r0 + offsetof(zend_closure, func.common.fn_flags)] 9204 | and edx, ZEND_ACC_FAKE_CLOSURE 9205 | or edx, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE) 9206 | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { 9207 | cmp byte [r0 + offsetof(zend_closure, this_ptr.u1.v.type)], IS_UNDEF 9208 | jz >1 9209 | // call_info |= ZEND_CALL_HAS_THIS; 9210 | or edx, ZEND_CALL_HAS_THIS 9211 | // object_or_called_scope = Z_OBJ(closure->this_ptr); 9212 | mov r1, aword [r0 + offsetof(zend_closure, this_ptr.value.ptr)] 9213 |1: 9214 | // ZEND_SET_CALL_INFO(call, 0, call_info); 9215 | or dword EX:RX->This.u1.type_info, edx 9216 | // Z_PTR(call->This) = object_or_called_scope; 9217 | mov aword EX:RX->This.value.ptr, r1 9218 | cmp aword [r0 + offsetof(zend_closure, func.op_array.run_time_cache__ptr)], 0 9219 | jnz >1 9220 | lea FCARG1a, aword [r0 + offsetof(zend_closure, func)] 9221 | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 9222 |1: 9223 } 9224 | // ZEND_CALL_NUM_ARGS(call) = num_args; 9225 | mov dword EX:RX->This.u2.num_args, opline->extended_value 9226 return 1; 9227} 9228 9229static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) 9230{ 9231 int32_t exit_point; 9232 const void *exit_addr; 9233 9234 if (func->type == ZEND_INTERNAL_FUNCTION) { 9235#ifdef ZEND_WIN32 9236 // TODO: ASLR may cause different addresses in different workers ??? 9237 return 0; 9238#endif 9239 } else if (func->type == ZEND_USER_FUNCTION) { 9240 if (!zend_accel_in_shm(func->op_array.opcodes)) { 9241 /* op_array and op_array->opcodes are not persistent. We can't link. */ 9242 return 0; 9243 } 9244 } else { 9245 ZEND_UNREACHABLE(); 9246 return 0; 9247 } 9248 9249 exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); 9250 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9251 if (!exit_addr) { 9252 return 0; 9253 } 9254 9255 | // call = EX(call); 9256 | mov r1, EX->call 9257 while (level > 0) { 9258 | mov r1, EX:r1->prev_execute_data 9259 level--; 9260 } 9261 9262 if (func->type == ZEND_USER_FUNCTION && 9263 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 9264 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 9265 !func->common.function_name)) { 9266 const zend_op *opcodes = func->op_array.opcodes; 9267 9268 | mov r1, aword EX:r1->func 9269 | .if X64 9270 || if (!IS_SIGNED_32BIT(opcodes)) { 9271 | mov64 r2, ((ptrdiff_t)opcodes) 9272 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], r2 9273 || } else { 9274 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes 9275 || } 9276 | .else 9277 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes 9278 | .endif 9279 | jne &exit_addr 9280 } else { 9281 | .if X64 9282 || if (!IS_SIGNED_32BIT(func)) { 9283 | mov64 r2, ((ptrdiff_t)func) 9284 | cmp aword EX:r1->func, r2 9285 || } else { 9286 | cmp aword EX:r1->func, func 9287 || } 9288 | .else 9289 | cmp aword EX:r1->func, func 9290 | .endif 9291 | jne &exit_addr 9292 } 9293 9294 return 1; 9295} 9296 9297static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack) 9298{ 9299 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9300 zend_call_info *call_info = NULL; 9301 zend_function *func = NULL; 9302 9303 if (delayed_call_chain) { 9304 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9305 return 0; 9306 } 9307 } 9308 9309 if (info) { 9310 call_info = info->callee_info; 9311 while (call_info && call_info->caller_init_opline != opline) { 9312 call_info = call_info->next_callee; 9313 } 9314 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9315 func = call_info->callee_func; 9316 } 9317 } 9318 9319 if (!func 9320 && trace 9321 && trace->op == ZEND_JIT_TRACE_INIT_CALL) { 9322#ifdef _WIN32 9323 /* ASLR */ 9324 if (trace->func->type != ZEND_INTERNAL_FUNCTION) { 9325 func = (zend_function*)trace->func; 9326 } 9327#else 9328 func = (zend_function*)trace->func; 9329#endif 9330 } 9331 9332#ifdef _WIN32 9333 if (0) { 9334#else 9335 if (opline->opcode == ZEND_INIT_FCALL 9336 && func 9337 && func->type == ZEND_INTERNAL_FUNCTION) { 9338#endif 9339 /* load constant address later */ 9340 } else if (func && op_array == &func->op_array) { 9341 /* recursive call */ 9342 if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) || 9343 (sizeof(void*) == 8 && !IS_SIGNED_32BIT(func))) { 9344 | mov r0, EX->func 9345 } 9346 } else { 9347 | // if (CACHED_PTR(opline->result.num)) 9348 | mov r2, EX->run_time_cache 9349 | mov r0, aword [r2 + opline->result.num] 9350 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9351 && func 9352 && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) 9353 && opline->opcode != ZEND_INIT_FCALL) { 9354 /* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */ 9355 | .if X64 9356 || if (!IS_SIGNED_32BIT(func)) { 9357 | mov64 r1, ((ptrdiff_t)func) 9358 | cmp r0, r1 9359 || } else { 9360 | cmp r0, func 9361 || } 9362 | .else 9363 | cmp r0, func 9364 | .endif 9365 | jnz >1 9366 |.cold_code 9367 |1: 9368 } else { 9369 | test r0, r0 9370 | jz >1 9371 } 9372 |.cold_code 9373 |1: 9374 if (opline->opcode == ZEND_INIT_FCALL 9375 && func 9376 && func->type == ZEND_USER_FUNCTION 9377 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { 9378 | LOAD_ADDR FCARG1a, func 9379 | mov aword [r2 + opline->result.num], FCARG1a 9380 | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 9381 | jmp >3 9382 } else { 9383 zval *zv = RT_CONSTANT(opline, opline->op2); 9384 9385 if (opline->opcode == ZEND_INIT_FCALL) { 9386 | LOAD_ADDR FCARG1a, Z_STR_P(zv); 9387 | lea FCARG2a, aword [r2 + opline->result.num] 9388 | EXT_CALL zend_jit_find_func_helper, r0 9389 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { 9390 | LOAD_ADDR FCARG1a, Z_STR_P(zv + 1); 9391 | lea FCARG2a, aword [r2 + opline->result.num] 9392 | EXT_CALL zend_jit_find_func_helper, r0 9393 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { 9394 | LOAD_ADDR FCARG1a, zv; 9395 | lea FCARG2a, aword [r2 + opline->result.num] 9396 | EXT_CALL zend_jit_find_ns_func_helper, r0 9397 } else { 9398 ZEND_UNREACHABLE(); 9399 } 9400 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9401 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 9402 func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0); 9403 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9404 9405 if (!exit_addr) { 9406 return 0; 9407 } 9408 if (!func || opline->opcode == ZEND_INIT_FCALL) { 9409 | test r0, r0 9410 | jnz >3 9411 } else if (func->type == ZEND_USER_FUNCTION 9412 && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { 9413 const zend_op *opcodes = func->op_array.opcodes; 9414 9415 | .if X64 9416 || if (!IS_SIGNED_32BIT(opcodes)) { 9417 | mov64 r1, ((ptrdiff_t)opcodes) 9418 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 9419 || } else { 9420 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9421 || } 9422 | .else 9423 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9424 | .endif 9425 | jz >3 9426 } else { 9427 | .if X64 9428 || if (!IS_SIGNED_32BIT(func)) { 9429 | mov64 r1, ((ptrdiff_t)func) 9430 | cmp r0, r1 9431 || } else { 9432 | cmp r0, func 9433 || } 9434 | .else 9435 | cmp r0, func 9436 | .endif 9437 | jz >3 9438 } 9439 | jmp &exit_addr 9440 } else { 9441 | test r0, r0 9442 | jnz >3 9443 | // SAVE_OPLINE(); 9444 | SET_EX_OPLINE opline, r0 9445 | jmp ->undefined_function 9446 } 9447 } 9448 |.code 9449 |3: 9450 } 9451 9452 if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, checked_stack)) { 9453 return 0; 9454 } 9455 9456 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9457 if (!zend_jit_save_call_chain(Dst, call_level)) { 9458 return 0; 9459 } 9460 } else { 9461 delayed_call_chain = 1; 9462 delayed_call_level = call_level; 9463 } 9464 9465 return 1; 9466} 9467 9468static int zend_jit_init_method_call(dasm_State **Dst, 9469 const zend_op *opline, 9470 uint32_t b, 9471 const zend_op_array *op_array, 9472 zend_ssa *ssa, 9473 const zend_ssa_op *ssa_op, 9474 int call_level, 9475 uint32_t op1_info, 9476 zend_jit_addr op1_addr, 9477 zend_class_entry *ce, 9478 bool ce_is_instanceof, 9479 bool on_this, 9480 bool delayed_fetch_this, 9481 zend_class_entry *trace_ce, 9482 zend_jit_trace_rec *trace, 9483 int checked_stack, 9484 bool polymorphic_side_trace) 9485{ 9486 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9487 zend_call_info *call_info = NULL; 9488 zend_function *func = NULL; 9489 zval *function_name; 9490 9491 ZEND_ASSERT(opline->op2_type == IS_CONST); 9492 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 9493 9494 function_name = RT_CONSTANT(opline, opline->op2); 9495 9496 if (info) { 9497 call_info = info->callee_info; 9498 while (call_info && call_info->caller_init_opline != opline) { 9499 call_info = call_info->next_callee; 9500 } 9501 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9502 func = call_info->callee_func; 9503 } 9504 } 9505 9506 if (polymorphic_side_trace) { 9507 /* function is passed in r0 from parent_trace */ 9508 } else { 9509 if (on_this) { 9510 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 9511 9512 | GET_ZVAL_PTR FCARG1a, this_addr 9513 } else { 9514 if (op1_info & MAY_BE_REF) { 9515 if (opline->op1_type == IS_CV) { 9516 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 9517 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9518 } 9519 | ZVAL_DEREF FCARG1a, op1_info 9520 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 9521 } else { 9522 /* Hack: Convert reference to regular value to simplify JIT code */ 9523 ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); 9524 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 9525 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9526 | EXT_CALL zend_jit_unref_helper, r0 9527 |1: 9528 } 9529 } 9530 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 9531 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9532 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9533 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9534 9535 if (!exit_addr) { 9536 return 0; 9537 } 9538 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 9539 } else { 9540 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 9541 |.cold_code 9542 |1: 9543 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 9544 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9545 } 9546 | SET_EX_OPLINE opline, r0 9547 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9548 | EXT_CALL zend_jit_invalid_method_call_tmp, r0 9549 } else { 9550 | EXT_CALL zend_jit_invalid_method_call, r0 9551 } 9552 | jmp ->exception_handler 9553 |.code 9554 } 9555 } 9556 | GET_ZVAL_PTR FCARG1a, op1_addr 9557 } 9558 9559 if (delayed_call_chain) { 9560 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9561 return 0; 9562 } 9563 } 9564 9565 | mov aword T1, FCARG1a // save 9566 9567 if (func) { 9568 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9569 | mov r0, EX->run_time_cache 9570 | mov r0, aword [r0 + opline->result.num + sizeof(void*)] 9571 | test r0, r0 9572 | jz >1 9573 } else { 9574 | // if (CACHED_PTR(opline->result.num) == obj->ce)) { 9575 | mov r0, EX->run_time_cache 9576 | mov r2, aword [r0 + opline->result.num] 9577 | cmp r2, [FCARG1a + offsetof(zend_object, ce)] 9578 | jnz >1 9579 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9580 | mov r0, aword [r0 + opline->result.num + sizeof(void*)] 9581 } 9582 9583 |.cold_code 9584 |1: 9585 | LOAD_ADDR FCARG2a, function_name 9586 |.if X64 9587 | lea CARG3, aword T1 9588 |.else 9589 | lea r0, aword T1 9590 | sub r4, 12 9591 | push r0 9592 |.endif 9593 | SET_EX_OPLINE opline, r0 9594 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9595 | EXT_CALL zend_jit_find_method_tmp_helper, r0 9596 } else { 9597 | EXT_CALL zend_jit_find_method_helper, r0 9598 } 9599 |.if not(X64) 9600 | add r4, 12 9601 |.endif 9602 | test r0, r0 9603 | jnz >2 9604 | jmp ->exception_handler 9605 |.code 9606 |2: 9607 } 9608 9609 if ((!func || zend_jit_may_be_modified(func, op_array)) 9610 && trace 9611 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9612 && trace->func 9613#ifdef _WIN32 9614 && trace->func->type != ZEND_INTERNAL_FUNCTION 9615#endif 9616 ) { 9617 int32_t exit_point; 9618 const void *exit_addr; 9619 9620 exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); 9621 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9622 if (!exit_addr) { 9623 return 0; 9624 } 9625 9626 func = (zend_function*)trace->func; 9627 9628 if (func->type == ZEND_USER_FUNCTION && 9629 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 9630 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 9631 !func->common.function_name)) { 9632 const zend_op *opcodes = func->op_array.opcodes; 9633 9634 | .if X64 9635 || if (!IS_SIGNED_32BIT(opcodes)) { 9636 | mov64 r1, ((ptrdiff_t)opcodes) 9637 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 9638 || } else { 9639 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9640 || } 9641 | .else 9642 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9643 | .endif 9644 | jne &exit_addr 9645 } else { 9646 | .if X64 9647 || if (!IS_SIGNED_32BIT(func)) { 9648 | mov64 r1, ((ptrdiff_t)func) 9649 | cmp r0, r1 9650 || } else { 9651 | cmp r0, func 9652 || } 9653 | .else 9654 | cmp r0, func 9655 | .endif 9656 | jne &exit_addr 9657 } 9658 } 9659 9660 if (!func) { 9661 | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { 9662 | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_STATIC 9663 | jnz >1 9664 |.cold_code 9665 |1: 9666 } 9667 9668 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { 9669 | mov FCARG1a, aword T1 // restore 9670 | mov FCARG2a, r0 9671 |.if X64 9672 | mov CARG3d, opline->extended_value 9673 |.else 9674 | sub r4, 12 9675 | push opline->extended_value 9676 |.endif 9677 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9678 | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, r0 9679 } else { 9680 | EXT_CALL zend_jit_push_static_metod_call_frame, r0 9681 } 9682 |.if not(X64) 9683 | add r4, 12 9684 |.endif 9685 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) { 9686 | test r0, r0 9687 | jz ->exception_handler 9688 } 9689 | mov RX, r0 9690 } 9691 9692 if (!func) { 9693 | jmp >9 9694 |.code 9695 } 9696 9697 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { 9698 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, delayed_fetch_this, checked_stack)) { 9699 return 0; 9700 } 9701 } 9702 9703 if (!func) { 9704 |9: 9705 } 9706 zend_jit_start_reuse_ip(); 9707 9708 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9709 if (!zend_jit_save_call_chain(Dst, call_level)) { 9710 return 0; 9711 } 9712 } else { 9713 delayed_call_chain = 1; 9714 delayed_call_level = call_level; 9715 } 9716 9717 return 1; 9718} 9719 9720static int zend_jit_init_closure_call(dasm_State **Dst, 9721 const zend_op *opline, 9722 uint32_t b, 9723 const zend_op_array *op_array, 9724 zend_ssa *ssa, 9725 const zend_ssa_op *ssa_op, 9726 int call_level, 9727 zend_jit_trace_rec *trace, 9728 int checked_stack) 9729{ 9730 zend_function *func = NULL; 9731 zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 9732 9733 | GET_ZVAL_PTR r0, op2_addr 9734 9735 if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure 9736 && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { 9737 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9738 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9739 9740 if (!exit_addr) { 9741 return 0; 9742 } 9743 9744 |.if X64 9745 || if (!IS_SIGNED_32BIT(zend_ce_closure)) { 9746 | mov64 FCARG1a, ((ptrdiff_t)zend_ce_closure) 9747 | cmp aword [r0 + offsetof(zend_object, ce)], FCARG1a 9748 || } else { 9749 | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure 9750 || } 9751 |.else 9752 | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure 9753 |.endif 9754 | jne &exit_addr 9755 if (ssa->var_info && ssa_op->op2_use >= 0) { 9756 ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; 9757 ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; 9758 ssa->var_info[ssa_op->op2_use].is_instanceof = 0; 9759 } 9760 } 9761 9762 if (trace 9763 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9764 && trace->func 9765 && trace->func->type == ZEND_USER_FUNCTION) { 9766 const zend_op *opcodes; 9767 int32_t exit_point; 9768 const void *exit_addr; 9769 9770 func = (zend_function*)trace->func; 9771 opcodes = func->op_array.opcodes; 9772 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); 9773 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9774 if (!exit_addr) { 9775 return 0; 9776 } 9777 9778 | .if X64 9779 || if (!IS_SIGNED_32BIT(opcodes)) { 9780 | mov64 FCARG1a, ((ptrdiff_t)opcodes) 9781 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], FCARG1a 9782 || } else { 9783 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes 9784 || } 9785 | .else 9786 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes 9787 | .endif 9788 | jne &exit_addr 9789 } 9790 9791 if (delayed_call_chain) { 9792 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9793 return 0; 9794 } 9795 } 9796 9797 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, checked_stack)) { 9798 return 0; 9799 } 9800 9801 if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9802 if (!zend_jit_save_call_chain(Dst, call_level)) { 9803 return 0; 9804 } 9805 } else { 9806 delayed_call_chain = 1; 9807 delayed_call_level = call_level; 9808 } 9809 9810 if (trace 9811 && trace->op == ZEND_JIT_TRACE_END 9812 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 9813 if (!zend_jit_set_valid_ip(Dst, opline + 1)) { 9814 return 0; 9815 } 9816 } 9817 9818 return 1; 9819} 9820 9821static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) 9822{ 9823 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9824 zend_call_info *call_info = NULL; 9825 const zend_function *func = NULL; 9826 uint32_t i; 9827 zend_jit_addr res_addr; 9828 uint32_t call_num_args = 0; 9829 bool unknown_num_args = 0; 9830 const void *exit_addr = NULL; 9831 const zend_op *prev_opline; 9832 9833 if (RETURN_VALUE_USED(opline)) { 9834 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 9835 } else { 9836 /* CPU stack allocated temporary zval */ 9837 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, TMP_ZVAL_OFFSET); 9838 } 9839 9840 prev_opline = opline - 1; 9841 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { 9842 prev_opline--; 9843 } 9844 if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || 9845 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 9846 unknown_num_args = 1; 9847 } 9848 9849 if (info) { 9850 call_info = info->callee_info; 9851 while (call_info && call_info->caller_call_opline != opline) { 9852 call_info = call_info->next_callee; 9853 } 9854 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9855 func = call_info->callee_func; 9856 } 9857 if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) 9858 && JIT_G(current_frame) 9859 && JIT_G(current_frame)->call 9860 && !JIT_G(current_frame)->call->func) { 9861 call_info = NULL; func = NULL; /* megamorphic call from trait */ 9862 } 9863 } 9864 if (!func) { 9865 /* resolve function at run time */ 9866 } else if (func->type == ZEND_USER_FUNCTION) { 9867 ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); 9868 call_num_args = call_info->num_args; 9869 } else if (func->type == ZEND_INTERNAL_FUNCTION) { 9870 ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); 9871 call_num_args = call_info->num_args; 9872 } else { 9873 ZEND_UNREACHABLE(); 9874 } 9875 9876 if (trace && !func) { 9877 if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { 9878 ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); 9879#ifndef ZEND_WIN32 9880 // TODO: ASLR may cause different addresses in different workers ??? 9881 func = trace->func; 9882 if (JIT_G(current_frame) && 9883 JIT_G(current_frame)->call && 9884 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9885 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9886 } else { 9887 unknown_num_args = 1; 9888 } 9889#endif 9890 } else if (trace->op == ZEND_JIT_TRACE_ENTER) { 9891 ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); 9892 if (zend_accel_in_shm(trace->func->op_array.opcodes)) { 9893 func = trace->func; 9894 if (JIT_G(current_frame) && 9895 JIT_G(current_frame)->call && 9896 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9897 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9898 } else { 9899 unknown_num_args = 1; 9900 } 9901 } 9902 } 9903 } 9904 9905 bool may_have_extra_named_params = 9906 opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && 9907 (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); 9908 9909 if (!reuse_ip) { 9910 zend_jit_start_reuse_ip(); 9911 | // call = EX(call); 9912 | mov RX, EX->call 9913 } 9914 zend_jit_stop_reuse_ip(); 9915 9916 | // fbc = call->func; 9917 | // mov r2, EX:RX->func ??? 9918 | // SAVE_OPLINE(); 9919 | SET_EX_OPLINE opline, r0 9920 9921 if (opline->opcode == ZEND_DO_FCALL) { 9922 if (!func) { 9923 if (trace) { 9924 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9925 9926 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9927 if (!exit_addr) { 9928 return 0; 9929 } 9930 | mov r0, EX:RX->func 9931 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 9932 | jnz &exit_addr 9933 } 9934 } 9935 } 9936 9937 if (!delayed_call_chain) { 9938 if (call_level == 1) { 9939 | mov aword EX->call, 0 9940 } else { 9941 | //EX(call) = call->prev_execute_data; 9942 | mov r0, EX:RX->prev_execute_data 9943 | mov EX->call, r0 9944 } 9945 } 9946 delayed_call_chain = 0; 9947 9948 | //call->prev_execute_data = execute_data; 9949 | mov EX:RX->prev_execute_data, EX 9950 9951 if (!func) { 9952 | mov r0, EX:RX->func 9953 } 9954 9955 if (opline->opcode == ZEND_DO_FCALL) { 9956 if (!func) { 9957 if (!trace) { 9958 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 9959 | jnz >1 9960 |.cold_code 9961 |1: 9962 if (!GCC_GLOBAL_REGS) { 9963 | mov FCARG1a, RX 9964 } 9965 | EXT_CALL zend_jit_deprecated_helper, r0 9966 | test al, al 9967 | mov r0, EX:RX->func // reload 9968 | jne >1 9969 | jmp ->exception_handler 9970 |.code 9971 |1: 9972 } 9973 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 9974 if (!GCC_GLOBAL_REGS) { 9975 | mov FCARG1a, RX 9976 } 9977 | EXT_CALL zend_jit_deprecated_helper, r0 9978 | test al, al 9979 | je ->exception_handler 9980 } 9981 } 9982 9983 if (!func 9984 && opline->opcode != ZEND_DO_UCALL 9985 && opline->opcode != ZEND_DO_ICALL) { 9986 | cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION 9987 | jne >8 9988 } 9989 9990 if ((!func || func->type == ZEND_USER_FUNCTION) 9991 && opline->opcode != ZEND_DO_ICALL) { 9992 | // EX(call) = NULL; 9993 | mov aword EX:RX->call, 0 9994 9995 if (RETURN_VALUE_USED(opline)) { 9996 | // EX(return_value) = EX_VAR(opline->result.var); 9997 | LOAD_ZVAL_ADDR r2, res_addr 9998 | mov aword EX:RX->return_value, r2 9999 } else { 10000 | // EX(return_value) = 0; 10001 | mov aword EX:RX->return_value, 0 10002 } 10003 10004 //EX_LOAD_RUN_TIME_CACHE(op_array); 10005 if (!func || func->op_array.cache_size) { 10006 if (func && op_array == &func->op_array) { 10007 /* recursive call */ 10008 if (trace || func->op_array.cache_size > sizeof(void*)) { 10009 | mov r2, EX->run_time_cache 10010 | mov EX:RX->run_time_cache, r2 10011 } 10012 } else { 10013 if (func 10014 && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE) 10015 && ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { 10016 | MEM_LOAD_ZTS r2, aword, compiler_globals, map_ptr_base, r1 10017 | mov r2, aword [r2 + (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache)] 10018 } else if ((func && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) || 10019 (JIT_G(current_frame) && 10020 JIT_G(current_frame)->call && 10021 TRACE_FRAME_IS_CLOSURE_CALL(JIT_G(current_frame)->call))) { 10022 /* Closures always use direct pointers */ 10023 | mov r0, EX:RX->func 10024 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 10025 } else { 10026 if (func) { 10027 | mov r0, EX:RX->func 10028 } 10029 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 10030 | test r2, 1 10031 | jz >1 10032 | MEM_LOAD_OP_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 10033 | mov r2, aword [r2] 10034 |1: 10035 } 10036 | mov EX:RX->run_time_cache, r2 10037 } 10038 } 10039 10040 | // EG(current_execute_data) = execute_data; 10041 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, RX, r1 10042 | mov FP, RX 10043 10044 | // opline = op_array->opcodes; 10045 if (func && !unknown_num_args) { 10046 10047 for (i = call_num_args; i < func->op_array.last_var; i++) { 10048 uint32_t n = EX_NUM_TO_VAR(i); 10049 | SET_Z_TYPE_INFO RX + n, IS_UNDEF 10050 } 10051 10052 if (call_num_args <= func->op_array.num_args) { 10053 if (!trace || (trace->op == ZEND_JIT_TRACE_END 10054 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10055 uint32_t num_args; 10056 10057 if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { 10058 if (trace) { 10059 num_args = 0; 10060 } else if (call_info) { 10061 num_args = skip_valid_arguments(op_array, ssa, call_info); 10062 } else { 10063 num_args = call_num_args; 10064 } 10065 } else { 10066 num_args = call_num_args; 10067 } 10068 if (zend_accel_in_shm(func->op_array.opcodes)) { 10069 | LOAD_IP_ADDR (func->op_array.opcodes + num_args) 10070 } else { 10071 | mov r0, EX->func 10072 if (GCC_GLOBAL_REGS) { 10073 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10074 if (num_args) { 10075 | add IP, (num_args * sizeof(zend_op)) 10076 } 10077 } else { 10078 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10079 if (num_args) { 10080 | add FCARG1a, (num_args * sizeof(zend_op)) 10081 } 10082 | mov aword EX->opline, FCARG1a 10083 } 10084 } 10085 10086 if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array 10087 && num_args >= op_array->required_num_args) { 10088 /* recursive call */ 10089 if (ZEND_OBSERVER_ENABLED) { 10090 | SAVE_IP 10091 | mov FCARG1a, FP 10092 | EXT_CALL zend_observer_fcall_begin, r0 10093 } 10094#ifdef CONTEXT_THREADED_JIT 10095 | call >1 10096 |.cold_code 10097 |1: 10098 | pop r0 10099 | jmp =>num_args 10100 |.code 10101#else 10102 | jmp =>num_args 10103#endif 10104 return 1; 10105 } 10106 } 10107 } else { 10108 if (!trace || (trace->op == ZEND_JIT_TRACE_END 10109 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10110 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 10111 | LOAD_IP_ADDR (func->op_array.opcodes) 10112 } else if (GCC_GLOBAL_REGS) { 10113 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10114 } else { 10115 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10116 | mov aword EX->opline, FCARG1a 10117 } 10118 } 10119 if (!GCC_GLOBAL_REGS) { 10120 | mov FCARG1a, FP 10121 } 10122 | EXT_CALL zend_jit_copy_extra_args_helper, r0 10123 } 10124 } else { 10125 | // opline = op_array->opcodes 10126 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 10127 | LOAD_IP_ADDR (func->op_array.opcodes) 10128 } else if (GCC_GLOBAL_REGS) { 10129 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10130 } else { 10131 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10132 | mov aword EX->opline, FCARG1a 10133 } 10134 if (func) { 10135 | // num_args = EX_NUM_ARGS(); 10136 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] 10137 | // if (UNEXPECTED(num_args > first_extra_arg)) 10138 | cmp ecx, (func->op_array.num_args) 10139 } else { 10140 | // first_extra_arg = op_array->num_args; 10141 | mov edx, dword [r0 + offsetof(zend_op_array, num_args)] 10142 | // num_args = EX_NUM_ARGS(); 10143 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] 10144 | // if (UNEXPECTED(num_args > first_extra_arg)) 10145 | cmp ecx, edx 10146 } 10147 | jg >1 10148 |.cold_code 10149 |1: 10150 if (!GCC_GLOBAL_REGS) { 10151 | mov FCARG1a, FP 10152 } 10153 | EXT_CALL zend_jit_copy_extra_args_helper, r0 10154 if (!func) { 10155 | mov r0, EX->func // reload 10156 } 10157 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload 10158 | jmp >1 10159 |.code 10160 if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { 10161 if (!func) { 10162 | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) 10163 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS 10164 | jnz >1 10165 } 10166 | // opline += num_args; 10167 |.if X64 10168 || ZEND_ASSERT(sizeof(zend_op) == 32); 10169 | mov edx, ecx 10170 | shl r2, 5 10171 |.else 10172 | imul r2, ecx, sizeof(zend_op) 10173 |.endif 10174 | ADD_IP r2 10175 } 10176 |1: 10177 | // if (EXPECTED((int)num_args < op_array->last_var)) { 10178 if (func) { 10179 | mov edx, (func->op_array.last_var) 10180 } else { 10181 | mov edx, dword [r0 + offsetof(zend_op_array, last_var)] 10182 } 10183 | sub edx, ecx 10184 | jle >3 //??? 10185 | // zval *var = EX_VAR_NUM(num_args); 10186// |.if X64 10187// | movsxd r1, ecx 10188// |.endif 10189 | shl r1, 4 10190 | lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))] 10191 |2: 10192 | SET_Z_TYPE_INFO r1, IS_UNDEF 10193 | sub edx, 1 10194 | lea r1, [r1 + 16] 10195 | jne <2 10196 |3: 10197 } 10198 10199 if (ZEND_OBSERVER_ENABLED) { 10200 if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10201 ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END); 10202 | SET_EX_OPLINE trace[1].opline, r0 10203 } else { 10204 | SAVE_IP 10205 } 10206 | mov FCARG1a, FP 10207 | EXT_CALL zend_observer_fcall_begin, r0 10208 } 10209 10210 if (trace) { 10211 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 10212 | jmp >9 10213 } 10214 } else { 10215#ifdef CONTEXT_THREADED_JIT 10216 | call ->context_threaded_call 10217 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 10218 | jmp >9 10219 } 10220 | call ->context_threaded_call 10221 if (!func) { 10222 | jmp >9 10223 } 10224#else 10225 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10226 | ADD_HYBRID_SPAD 10227 | JMP_IP 10228 } else if (GCC_GLOBAL_REGS) { 10229 | add r4, SPAD // stack alignment 10230 | JMP_IP 10231 } else { 10232 | mov FP, aword T2 // restore FP 10233 | mov RX, aword T3 // restore IP 10234 | add r4, NR_SPAD // stack alignment 10235 | mov r0, 1 // ZEND_VM_ENTER 10236 | ret 10237 } 10238 } 10239#endif 10240 } 10241 10242 if ((!func || func->type == ZEND_INTERNAL_FUNCTION) 10243 && (opline->opcode != ZEND_DO_UCALL)) { 10244 if (!func && (opline->opcode != ZEND_DO_ICALL)) { 10245 |8: 10246 } 10247 if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { 10248 if (!func) { 10249 if (trace) { 10250 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10251 10252 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10253 if (!exit_addr) { 10254 return 0; 10255 } 10256 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10257 | jnz &exit_addr 10258 } else { 10259 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10260 | jnz >1 10261 |.cold_code 10262 |1: 10263 if (!GCC_GLOBAL_REGS) { 10264 | mov FCARG1a, RX 10265 } 10266 | EXT_CALL zend_jit_deprecated_helper, r0 10267 | test al, al 10268 | mov r0, EX:RX->func // reload 10269 | jne >1 10270 | jmp ->exception_handler 10271 |.code 10272 |1: 10273 } 10274 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 10275 if (!GCC_GLOBAL_REGS) { 10276 | mov FCARG1a, RX 10277 } 10278 | EXT_CALL zend_jit_deprecated_helper, r0 10279 | test al, al 10280 | je ->exception_handler 10281 | mov r0, EX:RX->func // reload 10282 } 10283 } 10284 10285 | // EG(current_execute_data) = execute_data; 10286 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, RX, r1 10287 10288 if (ZEND_OBSERVER_ENABLED) { 10289 | mov FCARG1a, RX 10290 | EXT_CALL zend_observer_fcall_begin, r0 10291 | mov r0, EX:RX->func // reload 10292 } 10293 10294 | // ZVAL_NULL(EX_VAR(opline->result.var)); 10295 | LOAD_ZVAL_ADDR FCARG2a, res_addr 10296 | SET_Z_TYPE_INFO FCARG2a, IS_NULL 10297 10298 zend_jit_reset_last_valid_opline(); 10299 10300 | // (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret); 10301 if (zend_execute_internal) { 10302 |.if X64 10303 | // CARG2 and FCARG2a are identical 10304 | mov CARG1, RX 10305 |.else 10306 | mov aword A2, FCARG2a 10307 | mov aword A1, RX 10308 |.endif 10309 | EXT_CALL zend_execute_internal, r0 10310 } else { 10311 | mov FCARG1a, RX 10312 if (func) { 10313 | EXT_CALL func->internal_function.handler, r0 10314 } else { 10315 | call aword [r0 + offsetof(zend_internal_function, handler)] 10316 } 10317 } 10318 10319 if (ZEND_OBSERVER_ENABLED) { 10320 | LOAD_ZVAL_ADDR FCARG2a, res_addr 10321 | mov FCARG1a, RX 10322 | EXT_CALL zend_observer_fcall_end, r0 10323 } 10324 10325 | // EG(current_execute_data) = execute_data; 10326 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, FP, r0 10327 10328 | // zend_vm_stack_free_args(call); 10329 if (func && !unknown_num_args) { 10330 for (i = 0; i < call_num_args; i++ ) { 10331 if (zend_jit_needs_arg_dtor(func, i, call_info)) { 10332 uint32_t offset = EX_NUM_TO_VAR(i); 10333 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, opline 10334 } 10335 } 10336 } else { 10337 | mov FCARG1a, RX 10338 | EXT_CALL zend_jit_vm_stack_free_args_helper, r0 10339 } 10340 if (may_have_extra_named_params) { 10341 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24) 10342 | jnz >1 10343 |.cold_code 10344 |1: 10345 | mov FCARG1a, aword [RX + offsetof(zend_execute_data, extra_named_params)] 10346 | EXT_CALL zend_free_extra_named_params, r0 10347 | jmp >2 10348 |.code 10349 |2: 10350 } 10351 10352 |8: 10353 if (opline->opcode == ZEND_DO_FCALL) { 10354 // TODO: optimize ??? 10355 | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) 10356 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_RELEASE_THIS >> 16) 10357 | jnz >1 10358 |.cold_code 10359 |1: 10360 | GET_Z_PTR FCARG1a, RX + offsetof(zend_execute_data, This) 10361 | // OBJ_RELEASE(object); 10362 | OBJ_RELEASE ZREG_FCARG1, >2 10363 | jmp >2 10364 |.code 10365 |2: 10366 } 10367 10368 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10369 !JIT_G(current_frame) || 10370 !JIT_G(current_frame)->call || 10371 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || 10372 prev_opline->opcode == ZEND_SEND_UNPACK || 10373 prev_opline->opcode == ZEND_SEND_ARRAY || 10374 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 10375 10376 | // zend_vm_stack_free_call_frame(call); 10377 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16) 10378 | jnz >1 10379 |.cold_code 10380 |1: 10381 | mov FCARG1a, RX 10382 | EXT_CALL zend_jit_free_call_frame, r0 10383 | jmp >1 10384 |.code 10385 } 10386 | MEM_STORE_ZTS aword, executor_globals, vm_stack_top, RX, r0 10387 |1: 10388 10389 if (!RETURN_VALUE_USED(opline)) { 10390 zend_class_entry *ce; 10391 bool ce_is_instanceof; 10392 uint32_t func_info = call_info ? 10393 zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : 10394 (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); 10395 10396 /* If an exception is thrown, the return_value may stay at the 10397 * original value of null. */ 10398 func_info |= MAY_BE_NULL; 10399 10400 if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10401 | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline 10402 } 10403 } 10404 10405 | // if (UNEXPECTED(EG(exception) != NULL)) { 10406 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 10407 | jne ->icall_throw_handler 10408 10409 // TODO: Can we avoid checking for interrupts after each call ??? 10410 if (trace && last_valid_opline != opline) { 10411 int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); 10412 10413 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10414 if (!exit_addr) { 10415 return 0; 10416 } 10417 } else { 10418 exit_addr = NULL; 10419 } 10420 if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { 10421 return 0; 10422 } 10423 10424 if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { 10425 | LOAD_IP_ADDR (opline + 1) 10426 } else if (trace 10427 && trace->op == ZEND_JIT_TRACE_END 10428 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 10429 | LOAD_IP_ADDR (opline + 1) 10430 } 10431 } 10432 10433 if (!func) { 10434 |9: 10435 } 10436 10437 return 1; 10438} 10439 10440static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) 10441{ 10442 uint32_t arg_num = opline->op2.num; 10443 zend_jit_addr arg_addr; 10444 10445 ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); 10446 10447 if (!zend_jit_reuse_ip(Dst)) { 10448 return 0; 10449 } 10450 10451 if (opline->opcode == ZEND_SEND_VAL_EX) { 10452 uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); 10453 10454 ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); 10455 10456 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10457 && JIT_G(current_frame) 10458 && JIT_G(current_frame)->call 10459 && JIT_G(current_frame)->call->func) { 10460 if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10461 /* Don't generate code that always throws exception */ 10462 return 0; 10463 } 10464 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10465 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10466 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10467 if (!exit_addr) { 10468 return 0; 10469 } 10470 | mov r0, EX:RX->func 10471 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10472 | jnz &exit_addr 10473 } else { 10474 | mov r0, EX:RX->func 10475 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10476 | jnz >1 10477 |.cold_code 10478 |1: 10479 if (Z_MODE(op1_addr) == IS_REG) { 10480 /* set type to avoid zval_ptr_dtor() on uninitialized value */ 10481 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 10482 | SET_ZVAL_TYPE_INFO addr, IS_UNDEF 10483 } 10484 | SET_EX_OPLINE opline, r0 10485 | jmp ->throw_cannot_pass_by_ref 10486 |.code 10487 10488 } 10489 } 10490 10491 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10492 10493 if (opline->op1_type == IS_CONST) { 10494 zval *zv = RT_CONSTANT(opline, opline->op1); 10495 10496 | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 10497 if (Z_REFCOUNTED_P(zv)) { 10498 | ADDREF_CONST zv, r0 10499 } 10500 } else { 10501 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10502 } 10503 10504 return 1; 10505} 10506 10507static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) 10508{ 10509 | mov FCARG1a, EX->call 10510 | test byte [FCARG1a + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_MAY_HAVE_UNDEF >> 24) 10511 | jnz >1 10512 |.cold_code 10513 |1: 10514 | SET_EX_OPLINE opline, r0 10515 | EXT_CALL zend_handle_undef_args, r0 10516 | test r0, r0 10517 | jnz ->exception_handler 10518 | jmp >2 10519 |.code 10520 |2: 10521 10522 return 1; 10523} 10524 10525static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) 10526{ 10527 zend_jit_addr op1_addr, arg_addr, ref_addr; 10528 10529 op1_addr = OP1_ADDR(); 10530 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10531 10532 if (!zend_jit_reuse_ip(Dst)) { 10533 return 0; 10534 } 10535 10536 if (opline->op1_type == IS_VAR) { 10537 if (op1_info & MAY_BE_INDIRECT) { 10538 | LOAD_ZVAL_ADDR r0, op1_addr 10539 | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { 10540 | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 10541 | // ret = Z_INDIRECT_P(ret); 10542 | GET_Z_PTR r0, r0 10543 |1: 10544 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 10545 } 10546 } else if (opline->op1_type == IS_CV) { 10547 if (op1_info & MAY_BE_UNDEF) { 10548 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10549 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 10550 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 10551 | jmp >2 10552 |1: 10553 } 10554 op1_info &= ~MAY_BE_UNDEF; 10555 op1_info |= MAY_BE_NULL; 10556 } 10557 } else { 10558 ZEND_UNREACHABLE(); 10559 } 10560 10561 if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { 10562 if (op1_info & MAY_BE_REF) { 10563 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2 10564 | GET_ZVAL_PTR r1, op1_addr 10565 | GC_ADDREF r1 10566 | SET_ZVAL_PTR arg_addr, r1 10567 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX 10568 | jmp >6 10569 } 10570 |2: 10571 | // ZVAL_NEW_REF(arg, varptr); 10572 if (opline->op1_type == IS_VAR) { 10573 if (Z_REG(op1_addr) != ZREG_R0 || Z_OFFSET(op1_addr) != 0) { 10574 | LOAD_ZVAL_ADDR r0, op1_addr 10575 } 10576 | mov aword T1, r0 // save 10577 } 10578 | EMALLOC sizeof(zend_reference), op_array, opline 10579 | mov dword [r0], 2 10580 | mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], GC_REFERENCE 10581 | mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0 10582 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); 10583 if (opline->op1_type == IS_VAR) { 10584 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); 10585 10586 | mov r1, aword T1 // restore 10587 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R2, ZREG_R2 10588 | SET_ZVAL_PTR val_addr, r0 10589 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX 10590 } else { 10591 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10592 | SET_ZVAL_PTR op1_addr, r0 10593 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 10594 } 10595 | SET_ZVAL_PTR arg_addr, r0 10596 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX 10597 } 10598 10599 |6: 10600 | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline 10601 |7: 10602 10603 return 1; 10604} 10605 10606static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) 10607{ 10608 uint32_t arg_num = opline->op2.num; 10609 zend_jit_addr arg_addr; 10610 10611 ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && 10612 opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || 10613 arg_num <= MAX_ARG_FLAG_NUM); 10614 10615 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10616 10617 if (!zend_jit_reuse_ip(Dst)) { 10618 return 0; 10619 } 10620 10621 if (opline->opcode == ZEND_SEND_VAR_EX) { 10622 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10623 && JIT_G(current_frame) 10624 && JIT_G(current_frame)->call 10625 && JIT_G(current_frame)->call->func) { 10626 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10627 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10628 return 0; 10629 } 10630 return 1; 10631 } 10632 } else { 10633 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10634 10635 | mov r0, EX:RX->func 10636 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10637 | jnz >1 10638 |.cold_code 10639 |1: 10640 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10641 return 0; 10642 } 10643 | jmp >7 10644 |.code 10645 } 10646 } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { 10647 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10648 && JIT_G(current_frame) 10649 && JIT_G(current_frame)->call 10650 && JIT_G(current_frame)->call->func) { 10651 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10652 10653 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10654 10655 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10656 if (!(op1_info & MAY_BE_REF)) { 10657 /* Don't generate code that always throws exception */ 10658 return 0; 10659 } else { 10660 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10661 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10662 if (!exit_addr) { 10663 return 0; 10664 } 10665 | cmp cl, IS_REFERENCE 10666 | jne &exit_addr 10667 } 10668 } 10669 return 1; 10670 } 10671 } else { 10672 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10673 10674 | mov r0, EX:RX->func 10675 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10676 | jnz >1 10677 |.cold_code 10678 |1: 10679 10680 mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); 10681 10682 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10683 if (op1_info & MAY_BE_REF) { 10684 | cmp cl, IS_REFERENCE 10685 | je >7 10686 } 10687 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10688 | jnz >7 10689 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10690 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10691 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10692 if (!exit_addr) { 10693 return 0; 10694 } 10695 | jmp &exit_addr 10696 } else { 10697 | SET_EX_OPLINE opline, r0 10698 | LOAD_ZVAL_ADDR FCARG1a, arg_addr 10699 | EXT_CALL zend_jit_only_vars_by_reference, r0 10700 if (!zend_jit_check_exception(Dst)) { 10701 return 0; 10702 } 10703 | jmp >7 10704 } 10705 10706 |.code 10707 } 10708 } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { 10709 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10710 && JIT_G(current_frame) 10711 && JIT_G(current_frame)->call 10712 && JIT_G(current_frame)->call->func) { 10713 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10714 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10715 return 0; 10716 } 10717 return 1; 10718 } 10719 } else { 10720 | test dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10721 | jnz >1 10722 |.cold_code 10723 |1: 10724 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10725 return 0; 10726 } 10727 | jmp >7 10728 |.code 10729 } 10730 } 10731 10732 if (op1_info & MAY_BE_UNDEF) { 10733 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10734 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 10735 |.cold_code 10736 |1: 10737 } 10738 10739 | SET_EX_OPLINE opline, r0 10740 | mov FCARG1d, opline->op1.var 10741 | EXT_CALL zend_jit_undefined_op_helper, r0 10742 | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL 10743 | test r0, r0 10744 | jz ->exception_handler 10745 10746 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10747 | jmp >7 10748 |.code 10749 } else { 10750 |7: 10751 return 1; 10752 } 10753 } 10754 10755 if (opline->opcode == ZEND_SEND_VAR_NO_REF) { 10756 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10757 if (op1_info & MAY_BE_REF) { 10758 | cmp cl, IS_REFERENCE 10759 | je >7 10760 } 10761 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10762 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10763 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10764 if (!exit_addr) { 10765 return 0; 10766 } 10767 | jmp &exit_addr 10768 } else { 10769 | SET_EX_OPLINE opline, r0 10770 | LOAD_ZVAL_ADDR FCARG1a, arg_addr 10771 | EXT_CALL zend_jit_only_vars_by_reference, r0 10772 if (!zend_jit_check_exception(Dst)) { 10773 return 0; 10774 } 10775 } 10776 } else { 10777 if (op1_info & MAY_BE_REF) { 10778 if (opline->op1_type == IS_CV) { 10779 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 10780 10781 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 10782 | ZVAL_DEREF FCARG1a, op1_info 10783 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R0, ZREG_R2 10784 | TRY_ADDREF op1_info, ah, r2 10785 } else { 10786 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 8); 10787 10788 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 10789 |.cold_code 10790 |1: 10791 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 10792 | GET_ZVAL_PTR FCARG1a, op1_addr 10793 | // ZVAL_COPY_VALUE(return_value, &ref->value); 10794 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R0, ZREG_R2 10795 | GC_DELREF FCARG1a 10796 | je >1 10797 | IF_NOT_REFCOUNTED ah, >2 10798 | GC_ADDREF r2 10799 | jmp >2 10800 |1: 10801 | EFREE_REG_REFERENCE 10802 | jmp >2 10803 |.code 10804 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10805 |2: 10806 } 10807 } else { 10808 if (op1_addr != op1_def_addr) { 10809 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 10810 return 0; 10811 } 10812 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 10813 op1_addr= op1_def_addr; 10814 } 10815 } 10816 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10817 if (opline->op1_type == IS_CV) { 10818 | TRY_ADDREF op1_info, ah, r2 10819 } 10820 } 10821 } 10822 |7: 10823 10824 return 1; 10825} 10826 10827static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) 10828{ 10829 uint32_t arg_num = opline->op2.num; 10830 10831 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10832 && JIT_G(current_frame) 10833 && JIT_G(current_frame)->call 10834 && JIT_G(current_frame)->call->func) { 10835 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10836 if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { 10837 TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); 10838 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10839 || if (reuse_ip) { 10840 | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10841 || } else { 10842 | mov r0, EX->call 10843 | or dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10844 || } 10845 } 10846 } else { 10847 if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 10848 TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); 10849 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10850 || if (reuse_ip) { 10851 | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10852 || } else { 10853 | mov r0, EX->call 10854 | and dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10855 || } 10856 } 10857 } 10858 } else { 10859 // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { 10860 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10861 10862 if (!zend_jit_reuse_ip(Dst)) { 10863 return 0; 10864 } 10865 10866 | mov r0, EX:RX->func 10867 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10868 | jnz >1 10869 |.cold_code 10870 |1: 10871 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10872 | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10873 | jmp >1 10874 |.code 10875 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10876 | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10877 |1: 10878 } 10879 10880 return 1; 10881} 10882 10883static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) 10884{ 10885 if (smart_branch_opcode) { 10886 if (smart_branch_opcode == ZEND_JMPZ) { 10887 if (jmp) { 10888 | jmp >7 10889 } 10890 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10891 | jmp =>target_label 10892 } else { 10893 ZEND_UNREACHABLE(); 10894 } 10895 } else { 10896 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10897 10898 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 10899 if (jmp) { 10900 | jmp >7 10901 } 10902 } 10903 10904 return 1; 10905} 10906 10907static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) 10908{ 10909 if (smart_branch_opcode) { 10910 if (smart_branch_opcode == ZEND_JMPZ) { 10911 | jmp =>target_label 10912 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10913 if (jmp) { 10914 | jmp >7 10915 } 10916 } else { 10917 ZEND_UNREACHABLE(); 10918 } 10919 } else { 10920 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10921 10922 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 10923 if (jmp) { 10924 | jmp >7 10925 } 10926 } 10927 10928 return 1; 10929} 10930 10931static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 10932{ 10933 uint32_t defined_label = (uint32_t)-1; 10934 uint32_t undefined_label = (uint32_t)-1; 10935 zval *zv = RT_CONSTANT(opline, opline->op1); 10936 zend_jit_addr res_addr = 0; 10937 10938 if (smart_branch_opcode && !exit_addr) { 10939 if (smart_branch_opcode == ZEND_JMPZ) { 10940 undefined_label = target_label; 10941 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10942 defined_label = target_label; 10943 } else { 10944 ZEND_UNREACHABLE(); 10945 } 10946 } 10947 10948 | // if (CACHED_PTR(opline->extended_value)) { 10949 | mov r0, EX->run_time_cache 10950 | mov r0, aword [r0 + opline->extended_value] 10951 | test r0, r0 10952 | jz >1 10953 | test r0, 0x1 10954 | jnz >4 10955 |.cold_code 10956 |4: 10957 | MEM_LOAD_ZTS FCARG1a, aword, executor_globals, zend_constants, FCARG1a 10958 | shr r0, 1 10959 | cmp dword [FCARG1a + offsetof(HashTable, nNumOfElements)], eax 10960 10961 if (smart_branch_opcode) { 10962 if (exit_addr) { 10963 if (smart_branch_opcode == ZEND_JMPZ) { 10964 | jz &exit_addr 10965 } else { 10966 | jz >3 10967 } 10968 } else if (undefined_label != (uint32_t)-1) { 10969 | jz =>undefined_label 10970 } else { 10971 | jz >3 10972 } 10973 } else { 10974 | jz >2 10975 } 10976 |1: 10977 | SET_EX_OPLINE opline, r0 10978 | LOAD_ADDR FCARG1a, zv 10979 | EXT_CALL zend_jit_check_constant, r0 10980 | test r0, r0 10981 if (exit_addr) { 10982 if (smart_branch_opcode == ZEND_JMPNZ) { 10983 | jz >3 10984 } else { 10985 | jnz >3 10986 } 10987 | jmp &exit_addr 10988 } else if (smart_branch_opcode) { 10989 if (undefined_label != (uint32_t)-1) { 10990 | jz =>undefined_label 10991 } else { 10992 | jz >3 10993 } 10994 if (defined_label != (uint32_t)-1) { 10995 | jmp =>defined_label 10996 } else { 10997 | jmp >3 10998 } 10999 } else { 11000 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11001 | jnz >1 11002 |2: 11003 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 11004 | jmp >3 11005 } 11006 |.code 11007 if (smart_branch_opcode) { 11008 if (exit_addr) { 11009 if (smart_branch_opcode == ZEND_JMPNZ) { 11010 | jmp &exit_addr 11011 } 11012 } else if (defined_label != (uint32_t)-1) { 11013 | jmp =>defined_label 11014 } 11015 } else { 11016 |1: 11017 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 11018 } 11019 |3: 11020 11021 return 1; 11022} 11023 11024static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 11025{ 11026 uint32_t mask; 11027 zend_jit_addr op1_addr = OP1_ADDR(); 11028 11029 // TODO: support for is_resource() ??? 11030 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); 11031 11032 if (op1_info & MAY_BE_UNDEF) { 11033 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11034 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 11035 |.cold_code 11036 |1: 11037 } 11038 | SET_EX_OPLINE opline, r0 11039 | mov FCARG1d, opline->op1.var 11040 | EXT_CALL zend_jit_undefined_op_helper, r0 11041 zend_jit_check_exception_undef_result(Dst, opline); 11042 if (opline->extended_value & MAY_BE_NULL) { 11043 if (exit_addr) { 11044 if (smart_branch_opcode == ZEND_JMPNZ) { 11045 | jmp &exit_addr 11046 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 11047 | jmp >7 11048 } 11049 } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { 11050 return 0; 11051 } 11052 } else { 11053 if (exit_addr) { 11054 if (smart_branch_opcode == ZEND_JMPZ) { 11055 | jmp &exit_addr 11056 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 11057 | jmp >7 11058 } 11059 } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { 11060 return 0; 11061 } 11062 } 11063 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11064 |.code 11065 } 11066 } 11067 11068 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11069 mask = opline->extended_value; 11070 if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { 11071 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11072 if (exit_addr) { 11073 if (smart_branch_opcode == ZEND_JMPNZ) { 11074 | jmp &exit_addr 11075 } 11076 } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { 11077 return 0; 11078 } 11079 } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { 11080 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11081 if (exit_addr) { 11082 if (smart_branch_opcode == ZEND_JMPZ) { 11083 | jmp &exit_addr 11084 } 11085 } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { 11086 return 0; 11087 } 11088 } else { 11089 bool invert = 0; 11090 zend_uchar type; 11091 11092 switch (mask) { 11093 case MAY_BE_NULL: type = IS_NULL; break; 11094 case MAY_BE_FALSE: type = IS_FALSE; break; 11095 case MAY_BE_TRUE: type = IS_TRUE; break; 11096 case MAY_BE_LONG: type = IS_LONG; break; 11097 case MAY_BE_DOUBLE: type = IS_DOUBLE; break; 11098 case MAY_BE_STRING: type = IS_STRING; break; 11099 case MAY_BE_ARRAY: type = IS_ARRAY; break; 11100 case MAY_BE_OBJECT: type = IS_OBJECT; break; 11101 case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break; 11102 case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break; 11103 case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break; 11104 case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break; 11105 case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break; 11106 case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break; 11107 case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break; 11108 case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break; 11109 case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break; 11110 default: 11111 type = 0; 11112 } 11113 11114 if (op1_info & MAY_BE_REF) { 11115 | LOAD_ZVAL_ADDR r0, op1_addr 11116 | ZVAL_DEREF r0, op1_info 11117 } 11118 if (type == 0) { 11119 if (smart_branch_opcode && 11120 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11121 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11122 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11123 | // if (Z_REFCOUNTED_P(cv)) { 11124 | IF_ZVAL_REFCOUNTED op1_addr, >1 11125 |.cold_code 11126 |1: 11127 } 11128 | // if (!Z_DELREF_P(cv)) { 11129 | GET_ZVAL_PTR FCARG1a, op1_addr 11130 | GC_DELREF FCARG1a 11131 if (RC_MAY_BE_1(op1_info)) { 11132 if (RC_MAY_BE_N(op1_info)) { 11133 | jnz >3 11134 } 11135 if (op1_info & MAY_BE_REF) { 11136 | mov al, byte [r0 + 8] 11137 } else { 11138 | mov al, byte [FP + opline->op1.var + 8] 11139 } 11140 | mov byte T1, al // save 11141 | // zval_dtor_func(r); 11142 | ZVAL_DTOR_FUNC op1_info, opline 11143 | mov cl, byte T1 // restore 11144 |jmp >2 11145 } 11146 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11147 if (!RC_MAY_BE_1(op1_info)) { 11148 | jmp >3 11149 } 11150 |.code 11151 } 11152 |3: 11153 if (op1_info & MAY_BE_REF) { 11154 | mov cl, byte [r0 + 8] 11155 } else { 11156 | mov cl, byte [FP + opline->op1.var + 8] 11157 } 11158 |2: 11159 } else { 11160 if (op1_info & MAY_BE_REF) { 11161 | mov cl, byte [r0 + 8] 11162 } else { 11163 | mov cl, byte [FP + opline->op1.var + 8] 11164 } 11165 } 11166 | mov eax, 1 11167 | shl eax, cl 11168 | test eax, mask 11169 if (exit_addr) { 11170 if (smart_branch_opcode == ZEND_JMPNZ) { 11171 | jne &exit_addr 11172 } else { 11173 | je &exit_addr 11174 } 11175 } else if (smart_branch_opcode) { 11176 if (smart_branch_opcode == ZEND_JMPZ) { 11177 | je =>target_label 11178 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11179 | jne =>target_label 11180 } else { 11181 ZEND_UNREACHABLE(); 11182 } 11183 } else { 11184 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11185 11186 | setne al 11187 | movzx eax, al 11188 | add eax, 2 11189 | SET_ZVAL_TYPE_INFO res_addr, eax 11190 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11191 } 11192 } else { 11193 if (smart_branch_opcode && 11194 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11195 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11196 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11197 | // if (Z_REFCOUNTED_P(cv)) { 11198 | IF_ZVAL_REFCOUNTED op1_addr, >1 11199 |.cold_code 11200 |1: 11201 } 11202 | // if (!Z_DELREF_P(cv)) { 11203 | GET_ZVAL_PTR FCARG1a, op1_addr 11204 | GC_DELREF FCARG1a 11205 if (RC_MAY_BE_1(op1_info)) { 11206 if (RC_MAY_BE_N(op1_info)) { 11207 | jnz >3 11208 } 11209 if (op1_info & MAY_BE_REF) { 11210 | mov al, byte [r0 + 8] 11211 } else { 11212 | mov al, byte [FP + opline->op1.var + 8] 11213 } 11214 | mov byte T1, al // save 11215 | // zval_dtor_func(r); 11216 | ZVAL_DTOR_FUNC op1_info, opline 11217 | mov cl, byte T1 // restore 11218 |jmp >2 11219 } 11220 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11221 if (!RC_MAY_BE_1(op1_info)) { 11222 | jmp >3 11223 } 11224 |.code 11225 } 11226 |3: 11227 if (op1_info & MAY_BE_REF) { 11228 | mov cl, byte [r0 + 8] 11229 } else { 11230 | mov cl, byte [FP + opline->op1.var + 8] 11231 } 11232 |2: 11233 | cmp cl, type 11234 } else { 11235 if (op1_info & MAY_BE_REF) { 11236 | cmp byte [r0 + 8], type 11237 } else { 11238 | cmp byte [FP + opline->op1.var + 8], type 11239 } 11240 } 11241 if (exit_addr) { 11242 if (invert) { 11243 if (smart_branch_opcode == ZEND_JMPNZ) { 11244 | jne &exit_addr 11245 } else { 11246 | je &exit_addr 11247 } 11248 } else { 11249 if (smart_branch_opcode == ZEND_JMPNZ) { 11250 | je &exit_addr 11251 } else { 11252 | jne &exit_addr 11253 } 11254 } 11255 } else if (smart_branch_opcode) { 11256 if (invert) { 11257 if (smart_branch_opcode == ZEND_JMPZ) { 11258 | je =>target_label 11259 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11260 | jne =>target_label 11261 } else { 11262 ZEND_UNREACHABLE(); 11263 } 11264 } else { 11265 if (smart_branch_opcode == ZEND_JMPZ) { 11266 | jne =>target_label 11267 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11268 | je =>target_label 11269 } else { 11270 ZEND_UNREACHABLE(); 11271 } 11272 } 11273 } else { 11274 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11275 11276 if (invert) { 11277 | setne al 11278 } else { 11279 | sete al 11280 } 11281 | movzx eax, al 11282 | add eax, 2 11283 | SET_ZVAL_TYPE_INFO res_addr, eax 11284 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11285 } 11286 } 11287 } 11288 } 11289 11290 |7: 11291 11292 return 1; 11293} 11294 11295static int zend_jit_leave_frame(dasm_State **Dst) 11296{ 11297 | // EG(current_execute_data) = EX(prev_execute_data); 11298 | mov r0, EX->prev_execute_data 11299 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, r0, r2 11300 return 1; 11301} 11302 11303static int zend_jit_free_cvs(dasm_State **Dst) 11304{ 11305 | // EG(current_execute_data) = EX(prev_execute_data); 11306 | mov FCARG1a, EX->prev_execute_data 11307 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, FCARG1a, r0 11308 | // zend_free_compiled_variables(execute_data); 11309 | mov FCARG1a, FP 11310 | EXT_CALL zend_free_compiled_variables, r0 11311 return 1; 11312} 11313 11314static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) 11315{ 11316 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11317 uint32_t offset = EX_NUM_TO_VAR(var); 11318 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, NULL 11319 } 11320 return 1; 11321} 11322 11323static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) 11324{ 11325 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11326 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, 1, opline 11327 } 11328 return 1; 11329} 11330 11331static int zend_jit_leave_func(dasm_State **Dst, 11332 const zend_op_array *op_array, 11333 const zend_op *opline, 11334 uint32_t op1_info, 11335 bool left_frame, 11336 zend_jit_trace_rec *trace, 11337 zend_jit_trace_info *trace_info, 11338 int indirect_var_access, 11339 int may_throw) 11340{ 11341 bool may_be_top_frame = 11342 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11343 !JIT_G(current_frame) || 11344 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); 11345 bool may_need_call_helper = 11346 indirect_var_access || /* may have symbol table */ 11347 !op_array->function_name || /* may have symbol table */ 11348 may_be_top_frame || 11349 (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ 11350 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11351 !JIT_G(current_frame) || 11352 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ 11353 (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ 11354 bool may_need_release_this = 11355 !(op_array->fn_flags & ZEND_ACC_CLOSURE) && 11356 op_array->scope && 11357 !(op_array->fn_flags & ZEND_ACC_STATIC) && 11358 (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11359 !JIT_G(current_frame) || 11360 !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame))); 11361 11362 if (may_need_release_this) { 11363 | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)] 11364 } 11365 if (may_need_call_helper) { 11366 if (!left_frame) { 11367 left_frame = 1; 11368 if (!zend_jit_leave_frame(Dst)) { 11369 return 0; 11370 } 11371 } 11372 /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ 11373 if (may_need_release_this) { 11374 | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) 11375 } else { 11376 | test dword [FP + offsetof(zend_execute_data, This.u1.type_info)], (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) 11377 } 11378 if (trace && trace->op != ZEND_JIT_TRACE_END) { 11379 | jnz >1 11380 |.cold_code 11381 |1: 11382 if (!GCC_GLOBAL_REGS) { 11383 | mov FCARG1a, FP 11384 } 11385 | EXT_CALL zend_jit_leave_func_helper, r0 11386 11387 if (may_be_top_frame) { 11388 // TODO: try to avoid this check ??? 11389 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 11390#if 0 11391 /* this check should be handled by the following OPLINE guard */ 11392 | cmp IP, zend_jit_halt_op 11393 | je ->trace_halt 11394#endif 11395 } else if (GCC_GLOBAL_REGS) { 11396 | test IP, IP 11397 | je ->trace_halt 11398 } else { 11399 | test eax, eax 11400 | jl ->trace_halt 11401 } 11402 } 11403 11404 if (!GCC_GLOBAL_REGS) { 11405 | // execute_data = EG(current_execute_data) 11406 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 11407 } 11408 | jmp >8 11409 |.code 11410 } else { 11411 | jnz ->leave_function_handler 11412 } 11413 } 11414 11415 if (op_array->fn_flags & ZEND_ACC_CLOSURE) { 11416 if (!left_frame) { 11417 left_frame = 1; 11418 if (!zend_jit_leave_frame(Dst)) { 11419 return 0; 11420 } 11421 } 11422 | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); 11423 | mov FCARG1a, EX->func 11424 | sub FCARG1a, sizeof(zend_object) 11425 | OBJ_RELEASE ZREG_FCARG1, >4 11426 |4: 11427 } else if (may_need_release_this) { 11428 if (!left_frame) { 11429 left_frame = 1; 11430 if (!zend_jit_leave_frame(Dst)) { 11431 return 0; 11432 } 11433 } 11434 if (!JIT_G(current_frame) || !TRACE_FRAME_ALWAYS_RELEASE_THIS(JIT_G(current_frame))) { 11435 | // if (call_info & ZEND_CALL_RELEASE_THIS) 11436 | test FCARG1d, ZEND_CALL_RELEASE_THIS 11437 | je >4 11438 } 11439 | // zend_object *object = Z_OBJ(execute_data->This); 11440 | mov FCARG1a, EX->This.value.obj 11441 | // OBJ_RELEASE(object); 11442 | OBJ_RELEASE ZREG_FCARG1, >4 11443 |4: 11444 // TODO: avoid EG(excption) check for $this->foo() calls 11445 may_throw = 1; 11446 } 11447 11448 | // EG(vm_stack_top) = (zval*)execute_data; 11449 | MEM_STORE_ZTS aword, executor_globals, vm_stack_top, FP, r0 11450 | // execute_data = EX(prev_execute_data); 11451 | mov FP, EX->prev_execute_data 11452 11453 if (!left_frame) { 11454 | // EG(current_execute_data) = execute_data; 11455 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, FP, r0 11456 } 11457 11458 |9: 11459 if (trace) { 11460 if (trace->op != ZEND_JIT_TRACE_END 11461 && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 11462 zend_jit_reset_last_valid_opline(); 11463 } else { 11464 | LOAD_IP 11465 | ADD_IP sizeof(zend_op) 11466 } 11467 11468 |8: 11469 11470 if (trace->op == ZEND_JIT_TRACE_BACK 11471 && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 11472 const zend_op *next_opline = trace->opline; 11473 11474 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11475 && (op1_info & MAY_BE_RC1) 11476 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { 11477 /* exception might be thrown during destruction of unused return value */ 11478 | // if (EG(exception)) 11479 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 11480 | jne ->leave_throw_handler 11481 } 11482 do { 11483 trace++; 11484 } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); 11485 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 11486 next_opline = trace->opline; 11487 ZEND_ASSERT(next_opline != NULL); 11488 11489 if (trace->op == ZEND_JIT_TRACE_END 11490 && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { 11491 trace_info->flags |= ZEND_JIT_TRACE_LOOP; 11492 | CMP_IP next_opline 11493 | je =>0 // LOOP 11494#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 11495 | JMP_IP 11496#else 11497 | jmp ->trace_escape 11498#endif 11499 } else { 11500 | CMP_IP next_opline 11501 | jne ->trace_escape 11502 } 11503 11504 zend_jit_set_last_valid_opline(trace->opline); 11505 11506 return 1; 11507 } else if (may_throw || 11508 (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11509 && (op1_info & MAY_BE_RC1) 11510 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) 11511 && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { 11512 | // if (EG(exception)) 11513 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 11514 | jne ->leave_throw_handler 11515 } 11516 11517 return 1; 11518 } else { 11519 | // if (EG(exception)) 11520 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 11521 | LOAD_IP 11522 | jne ->leave_throw_handler 11523 | // opline = EX(opline) + 1 11524 | ADD_IP sizeof(zend_op) 11525 } 11526 11527 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 11528 | ADD_HYBRID_SPAD 11529#ifdef CONTEXT_THREADED_JIT 11530 | push aword [IP] 11531 | ret 11532#else 11533 | JMP_IP 11534#endif 11535 } else if (GCC_GLOBAL_REGS) { 11536 | add r4, SPAD // stack alignment 11537#ifdef CONTEXT_THREADED_JIT 11538 | push aword [IP] 11539 | ret 11540#else 11541 | JMP_IP 11542#endif 11543 } else { 11544#ifdef CONTEXT_THREADED_JIT 11545 ZEND_UNREACHABLE(); 11546 // TODO: context threading can't work without GLOBAL REGS because we have to change 11547 // the value of execute_data in execute_ex() 11548 | mov FCARG1a, FP 11549 | mov r0, aword [FP] 11550 | mov FP, aword T2 // restore FP 11551 | mov RX, aword T3 // restore IP 11552 | add r4, NR_SPAD // stack alignment 11553 | push aword [r0] 11554 | ret 11555#else 11556 | mov FP, aword T2 // restore FP 11557 | mov RX, aword T3 // restore IP 11558 | add r4, NR_SPAD // stack alignment 11559 | mov r0, 2 // ZEND_VM_LEAVE 11560 | ret 11561#endif 11562 } 11563 11564 return 1; 11565} 11566 11567static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) 11568{ 11569 zend_jit_addr ret_addr; 11570 int8_t return_value_used; 11571 11572 ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); 11573 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); 11574 11575 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { 11576 if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { 11577 return_value_used = 1; 11578 } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { 11579 return_value_used = 0; 11580 } else { 11581 return_value_used = -1; 11582 } 11583 } else { 11584 return_value_used = -1; 11585 } 11586 11587 if (ZEND_OBSERVER_ENABLED) { 11588 if (Z_MODE(op1_addr) == IS_REG) { 11589 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 11590 11591 if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { 11592 return 0; 11593 } 11594 op1_addr = dst; 11595 } 11596 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 11597 | mov FCARG1a, FP 11598 | SET_EX_OPLINE opline, r0 11599 | EXT_CALL zend_observer_fcall_end, r0 11600 } 11601 11602 // if (!EX(return_value)) 11603 if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) { 11604 if (return_value_used != 0) { 11605 | mov r2, EX->return_value 11606 } 11607 if (return_value_used == -1) { 11608 | test r2, r2 11609 } 11610 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); 11611 } else { 11612 if (return_value_used != 0) { 11613 | mov r1, EX->return_value 11614 } 11615 if (return_value_used == -1) { 11616 | test r1, r1 11617 } 11618 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); 11619 } 11620 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11621 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11622 if (return_value_used == -1) { 11623 | jz >1 11624 |.cold_code 11625 |1: 11626 } 11627 if (return_value_used != 1) { 11628 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11629 if (jit_return_label >= 0) { 11630 | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label 11631 } else { 11632 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9 11633 } 11634 } 11635 | GET_ZVAL_PTR FCARG1a, op1_addr 11636 | GC_DELREF FCARG1a 11637 if (RC_MAY_BE_1(op1_info)) { 11638 if (RC_MAY_BE_N(op1_info)) { 11639 if (jit_return_label >= 0) { 11640 | jnz =>jit_return_label 11641 } else { 11642 | jnz >9 11643 } 11644 } 11645 | //SAVE_OPLINE() 11646 | ZVAL_DTOR_FUNC op1_info, opline 11647 | //????mov r1, EX->return_value // reload ??? 11648 } 11649 if (return_value_used == -1) { 11650 if (jit_return_label >= 0) { 11651 | jmp =>jit_return_label 11652 } else { 11653 | jmp >9 11654 } 11655 |.code 11656 } 11657 } 11658 } else if (return_value_used == -1) { 11659 if (jit_return_label >= 0) { 11660 | jz =>jit_return_label 11661 } else { 11662 | jz >9 11663 } 11664 } 11665 11666 if (return_value_used == 0) { 11667 |9: 11668 return 1; 11669 } 11670 11671 if (opline->op1_type == IS_CONST) { 11672 zval *zv = RT_CONSTANT(opline, opline->op1); 11673 | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 11674 if (Z_REFCOUNTED_P(zv)) { 11675 | ADDREF_CONST zv, r0 11676 } 11677 } else if (opline->op1_type == IS_TMP_VAR) { 11678 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11679 } else if (opline->op1_type == IS_CV) { 11680 if (op1_info & MAY_BE_REF) { 11681 | LOAD_ZVAL_ADDR r0, op1_addr 11682 | ZVAL_DEREF r0, op1_info 11683 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 11684 } 11685 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11686 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 11687 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11688 (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || 11689 !op_array->function_name) { 11690 | TRY_ADDREF op1_info, ah, r2 11691 } else if (return_value_used != 1) { 11692 | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); 11693 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 11694 } 11695 } 11696 } else { 11697 if (op1_info & MAY_BE_REF) { 11698 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); 11699 11700 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 11701 |.cold_code 11702 |1: 11703 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 11704 | GET_ZVAL_PTR r0, op1_addr 11705 | // ZVAL_COPY_VALUE(return_value, &ref->value); 11706 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R2, ZREG_R2 11707 | GC_DELREF r0 11708 | je >2 11709 | // if (IS_REFCOUNTED()) 11710 if (jit_return_label >= 0) { 11711 | IF_NOT_REFCOUNTED dh, =>jit_return_label 11712 } else { 11713 | IF_NOT_REFCOUNTED dh, >9 11714 } 11715 | // ADDREF 11716 | GET_ZVAL_PTR r2, ret_addr // reload 11717 | GC_ADDREF r2 11718 if (jit_return_label >= 0) { 11719 | jmp =>jit_return_label 11720 } else { 11721 | jmp >9 11722 } 11723 |2: 11724 | EFREE_REFERENCE r0 11725 if (jit_return_label >= 0) { 11726 | jmp =>jit_return_label 11727 } else { 11728 | jmp >9 11729 } 11730 |.code 11731 } 11732 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11733 } 11734 11735 |9: 11736 return 1; 11737} 11738 11739static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) 11740{ 11741 ZEND_ASSERT(type_reg == ZREG_R2); 11742 11743 |.if not(X64) 11744 || if (Z_REG(val_addr) == ZREG_R1) { 11745 | GET_ZVAL_W2 r0, val_addr 11746 || } 11747 |.endif 11748 | GET_ZVAL_PTR r1, val_addr 11749 |.if not(X64) 11750 || if (Z_REG(val_addr) != ZREG_R1) { 11751 | GET_ZVAL_W2 r0, val_addr 11752 || } 11753 |.endif 11754 | IF_NOT_REFCOUNTED dh, >2 11755 | IF_NOT_TYPE dl, IS_REFERENCE, >1 11756 | GET_Z_TYPE_INFO edx, r1+offsetof(zend_reference, val) 11757 |.if not(X64) 11758 | GET_Z_W2 r0, r1+offsetof(zend_reference, val) 11759 |.endif 11760 | GET_Z_PTR r1, r1+offsetof(zend_reference, val) 11761 | IF_NOT_REFCOUNTED dh, >2 11762 |1: 11763 | GC_ADDREF r1 11764 |2: 11765 | SET_ZVAL_PTR res_addr, r1 11766 |.if not(X64) 11767 | SET_ZVAL_W2 res_addr, r0 11768 |.endif 11769 | SET_ZVAL_TYPE_INFO res_addr, edx 11770 11771 return 1; 11772} 11773 11774static int zend_jit_fetch_dim_read(dasm_State **Dst, 11775 const zend_op *opline, 11776 zend_ssa *ssa, 11777 const zend_ssa_op *ssa_op, 11778 uint32_t op1_info, 11779 zend_jit_addr op1_addr, 11780 bool op1_avoid_refcounting, 11781 uint32_t op2_info, 11782 uint32_t res_info, 11783 zend_jit_addr res_addr, 11784 uint8_t dim_type) 11785{ 11786 zend_jit_addr orig_op1_addr, op2_addr; 11787 const void *exit_addr = NULL; 11788 const void *not_found_exit_addr = NULL; 11789 const void *res_exit_addr = NULL; 11790 bool result_avoid_refcounting = 0; 11791 uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; 11792 int may_throw = 0; 11793 11794 orig_op1_addr = OP1_ADDR(); 11795 op2_addr = OP2_ADDR(); 11796 11797 if (opline->opcode != ZEND_FETCH_DIM_IS 11798 && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11799 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 11800 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11801 if (!exit_addr) { 11802 return 0; 11803 } 11804 } 11805 11806 if ((res_info & MAY_BE_GUARD) 11807 && JIT_G(current_frame) 11808 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { 11809 uint32_t flags = 0; 11810 uint32_t old_op1_info = 0; 11811 uint32_t old_info; 11812 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 11813 int32_t exit_point; 11814 11815 if (opline->opcode != ZEND_FETCH_LIST_R 11816 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11817 && !op1_avoid_refcounting) { 11818 flags |= ZEND_JIT_EXIT_FREE_OP1; 11819 } 11820 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) 11821 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11822 flags |= ZEND_JIT_EXIT_FREE_OP2; 11823 } 11824 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 11825 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 11826 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 11827 && (ssa_op+1)->op1_use == ssa_op->result_def 11828 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) 11829 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 11830 result_avoid_refcounting = 1; 11831 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 11832 } 11833 11834 if (op1_avoid_refcounting) { 11835 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 11836 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 11837 } 11838 11839 if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { 11840 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11841 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 11842 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 11843 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11844 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11845 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11846 if (!res_exit_addr) { 11847 return 0; 11848 } 11849 res_info &= ~MAY_BE_GUARD; 11850 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 11851 } 11852 11853 if (opline->opcode == ZEND_FETCH_DIM_IS 11854 && !(res_info & MAY_BE_NULL)) { 11855 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11856 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); 11857 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); 11858 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11859 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11860 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11861 if (!not_found_exit_addr) { 11862 return 0; 11863 } 11864 } 11865 11866 if (op1_avoid_refcounting) { 11867 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 11868 } 11869 } 11870 11871 if (op1_info & MAY_BE_REF) { 11872 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 11873 | ZVAL_DEREF FCARG1a, op1_info 11874 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11875 } 11876 11877 if (op1_info & MAY_BE_ARRAY) { 11878 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11879 if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { 11880 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr 11881 } else { 11882 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 11883 } 11884 } 11885 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 11886 if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) || 11887 (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) { 11888 may_throw = 1; 11889 } 11890 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, dim_type, res_exit_addr, not_found_exit_addr, exit_addr)) { 11891 return 0; 11892 } 11893 } 11894 11895 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 11896 if (op1_info & MAY_BE_ARRAY) { 11897 |.cold_code 11898 |7: 11899 } 11900 11901 if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { 11902 may_throw = 1; 11903 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { 11904 if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { 11905 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr 11906 } else { 11907 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 11908 } 11909 } 11910 | SET_EX_OPLINE opline, r0 11911 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 11912 if (opline->opcode != ZEND_FETCH_DIM_IS) { 11913 if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { 11914 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 11915 | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, r0 11916 } else { 11917 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 11918 | EXT_CALL zend_jit_fetch_dim_str_r_helper, r0 11919 } 11920 | SET_ZVAL_PTR res_addr, r0 11921 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING 11922 } else { 11923 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 11924 |.if X64 11925 | LOAD_ZVAL_ADDR CARG3, res_addr 11926 |.else 11927 | sub r4, 12 11928 | PUSH_ZVAL_ADDR res_addr, r0 11929 |.endif 11930 | EXT_CALL zend_jit_fetch_dim_str_is_helper, r0 11931 |.if not(X64) 11932 | add r4, 12 11933 |.endif 11934 } 11935 if ((op1_info & MAY_BE_ARRAY) || 11936 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { 11937 | jmp >9 // END 11938 } 11939 |6: 11940 } 11941 11942 if (op1_info & MAY_BE_OBJECT) { 11943 may_throw = 1; 11944 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { 11945 if (exit_addr) { 11946 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 11947 } else { 11948 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6 11949 } 11950 } 11951 | SET_EX_OPLINE opline, r0 11952 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11953 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 11954 } 11955 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11956 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11957 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 11958 } else { 11959 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 11960 } 11961 |.if X64 11962 | LOAD_ZVAL_ADDR CARG3, res_addr 11963 |.else 11964 | sub r4, 12 11965 | PUSH_ZVAL_ADDR res_addr, r0 11966 |.endif 11967 if (opline->opcode != ZEND_FETCH_DIM_IS) { 11968 | EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0 11969 } else { 11970 | EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0 11971 } 11972 |.if not(X64) 11973 | add r4, 12 11974 |.endif 11975 if ((op1_info & MAY_BE_ARRAY) || 11976 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 11977 | jmp >9 // END 11978 } 11979 |6: 11980 } 11981 11982 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) 11983 && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 11984 if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { 11985 | SET_EX_OPLINE opline, r0 11986 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { 11987 may_throw = 1; 11988 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 11989 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 11990 | mov FCARG1d, opline->op1.var 11991 | EXT_CALL zend_jit_undefined_op_helper, r0 11992 |1: 11993 } 11994 11995 if (op2_info & MAY_BE_UNDEF) { 11996 may_throw = 1; 11997 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 11998 | mov FCARG1d, opline->op2.var 11999 | EXT_CALL zend_jit_undefined_op_helper, r0 12000 |1: 12001 } 12002 } 12003 12004 if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { 12005 may_throw = 1; 12006 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { 12007 | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr 12008 } else { 12009 | SET_EX_OPLINE opline, r0 12010 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || 12011 Z_REG(op1_addr) != ZREG_FCARG1 || 12012 Z_OFFSET(op1_addr) != 0) { 12013 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12014 } 12015 } 12016 | EXT_CALL zend_jit_invalid_array_access, r0 12017 } 12018 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 12019 if (op1_info & MAY_BE_ARRAY) { 12020 | jmp >9 // END 12021 } 12022 } 12023 12024 if (op1_info & MAY_BE_ARRAY) { 12025 |.code 12026 } 12027 } 12028 12029 if (op1_info & MAY_BE_ARRAY) { 12030 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 12031 12032 |8: 12033 if (res_exit_addr) { 12034 zend_uchar type = concrete_type(res_info); 12035 12036 if ((op1_info & MAY_BE_ARRAY_OF_REF) 12037 && dim_type != IS_UNKNOWN 12038 && dim_type != IS_REFERENCE) { 12039 if (type < IS_STRING) { 12040 | IF_NOT_ZVAL_TYPE val_addr, type, >1 12041 |.cold_code 12042 |1: 12043 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, &res_exit_addr 12044 | GET_Z_PTR r0, r0 12045 | add r0, offsetof(zend_reference, val) 12046 | IF_ZVAL_TYPE val_addr, type, >1 12047 | jmp &res_exit_addr 12048 |.code 12049 |1: 12050 } else { 12051 | GET_ZVAL_TYPE_INFO edx, val_addr 12052 | IF_NOT_TYPE dl, type, >1 12053 |.cold_code 12054 |1: 12055 | IF_NOT_TYPE dl, IS_REFERENCE, &res_exit_addr 12056 | GET_Z_PTR r0, r0 12057 | add r0, offsetof(zend_reference, val) 12058 | GET_ZVAL_TYPE_INFO edx, val_addr 12059 | IF_TYPE dl, type, >1 12060 | jmp &res_exit_addr 12061 |.code 12062 |1: 12063 } 12064 } else { 12065 if (op1_info & MAY_BE_ARRAY_OF_REF) { 12066 | ZVAL_DEREF r0, MAY_BE_REF 12067 } 12068 if (type < IS_STRING) { 12069 | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr 12070 } else { 12071 | GET_ZVAL_TYPE_INFO edx, val_addr 12072 | IF_NOT_TYPE dl, type, &res_exit_addr 12073 } 12074 } 12075 12076 | // ZVAL_COPY 12077 |7: 12078 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 12079 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 12080 if (type < IS_STRING) { 12081 if (Z_REG(res_addr) != ZREG_FP || 12082 JIT_G(current_frame) == NULL || 12083 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 12084 | SET_ZVAL_TYPE_INFO res_addr, type 12085 } 12086 } else { 12087 | SET_ZVAL_TYPE_INFO res_addr, edx 12088 if (!result_avoid_refcounting) { 12089 | TRY_ADDREF res_info, dh, r1 12090 } 12091 } 12092 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 12093 return 0; 12094 } 12095 } else if (op1_info & MAY_BE_ARRAY_OF_REF) { 12096 | // ZVAL_COPY_DEREF 12097 | GET_ZVAL_TYPE_INFO Rd(ZREG_R2), val_addr 12098 if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_R2)) { 12099 return 0; 12100 } 12101 } else { 12102 | // ZVAL_COPY 12103 | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_R1, ZREG_R2 12104 | TRY_ADDREF res_info, ch, r2 12105 } 12106 } 12107 |9: // END 12108 12109#ifdef ZEND_JIT_USE_RC_INFERENCE 12110 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 12111 /* Magic offsetGet() may increase refcount of the key */ 12112 op2_info |= MAY_BE_RCN; 12113 } 12114#endif 12115 12116 if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) { 12117 if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) { 12118 may_throw = 1; 12119 } 12120 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12121 } 12122 if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { 12123 if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) { 12124 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 12125 may_throw = 1; 12126 } 12127 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12128 } 12129 } 12130 12131 if (may_throw) { 12132 if (!zend_jit_check_exception(Dst)) { 12133 return 0; 12134 } 12135 } 12136 12137 return 1; 12138} 12139 12140static int zend_jit_fetch_dim(dasm_State **Dst, 12141 const zend_op *opline, 12142 uint32_t op1_info, 12143 zend_jit_addr op1_addr, 12144 uint32_t op2_info, 12145 zend_jit_addr res_addr, 12146 uint8_t dim_type) 12147{ 12148 zend_jit_addr op2_addr; 12149 int may_throw = 0; 12150 12151 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 12152 12153 if (opline->opcode == ZEND_FETCH_DIM_RW) { 12154 | SET_EX_OPLINE opline, r0 12155 } 12156 if (op1_info & MAY_BE_REF) { 12157 may_throw = 1; 12158 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12159 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 12160 | GET_Z_PTR FCARG2a, FCARG1a 12161 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 12162 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 12163 | jmp >3 12164 |.cold_code 12165 |2: 12166 if (opline->opcode != ZEND_FETCH_DIM_RW) { 12167 | SET_EX_OPLINE opline, r0 12168 } 12169 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 12170 | test r0, r0 12171 | mov FCARG1a, r0 12172 | jne >1 12173 | jmp ->exception_handler_undef 12174 |.code 12175 |1: 12176 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12177 } 12178 12179 if (op1_info & MAY_BE_ARRAY) { 12180 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12181 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12182 } 12183 |3: 12184 | SEPARATE_ARRAY op1_addr, op1_info, 1 12185 } 12186 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 12187 if (op1_info & MAY_BE_ARRAY) { 12188 |.cold_code 12189 |7: 12190 } 12191 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 12192 | CMP_ZVAL_TYPE op1_addr, IS_NULL 12193 | jg >7 12194 } 12195 if (Z_REG(op1_addr) != ZREG_FP) { 12196 | mov T1, Ra(Z_REG(op1_addr)) // save 12197 } 12198 if ((op1_info & MAY_BE_UNDEF) 12199 && opline->opcode == ZEND_FETCH_DIM_RW) { 12200 may_throw = 1; 12201 if (op1_info & MAY_BE_NULL) { 12202 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 12203 } 12204 | mov FCARG1a, opline->op1.var 12205 | EXT_CALL zend_jit_undefined_op_helper, r0 12206 |1: 12207 } 12208 | // ZVAL_ARR(container, zend_new_array(8)); 12209 | EXT_CALL _zend_new_array_0, r0 12210 if (Z_REG(op1_addr) != ZREG_FP) { 12211 | mov Ra(Z_REG(op1_addr)), T1 // restore 12212 } 12213 | SET_ZVAL_LVAL op1_addr, r0 12214 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 12215 | mov FCARG1a, r0 12216 if (op1_info & MAY_BE_ARRAY) { 12217 | jmp >1 12218 |.code 12219 |1: 12220 } 12221 } 12222 12223 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 12224 |6: 12225 if (opline->op2_type == IS_UNUSED) { 12226 may_throw = 1; 12227 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 12228 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 12229 | EXT_CALL zend_hash_next_index_insert, r0 12230 | // if (UNEXPECTED(!var_ptr)) { 12231 | test r0, r0 12232 | jz >1 12233 |.cold_code 12234 |1: 12235 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 12236 | CANNOT_ADD_ELEMENT opline 12237 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF 12238 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 12239 | jmp >8 12240 |.code 12241 | SET_ZVAL_PTR res_addr, r0 12242 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 12243 } else { 12244 uint32_t type; 12245 12246 switch (opline->opcode) { 12247 case ZEND_FETCH_DIM_W: 12248 case ZEND_FETCH_LIST_W: 12249 type = BP_VAR_W; 12250 break; 12251 case ZEND_FETCH_DIM_RW: 12252 may_throw = 1; 12253 type = BP_VAR_RW; 12254 break; 12255 case ZEND_FETCH_DIM_UNSET: 12256 type = BP_VAR_UNSET; 12257 break; 12258 default: 12259 ZEND_UNREACHABLE(); 12260 } 12261 12262 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 12263 may_throw = 1; 12264 } 12265 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 12266 return 0; 12267 } 12268 12269 |8: 12270 | SET_ZVAL_PTR res_addr, r0 12271 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 12272 12273 if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 12274 |.cold_code 12275 |9: 12276 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 12277 | jmp >8 12278 |.code 12279 } 12280 } 12281 } 12282 12283 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 12284 may_throw = 1; 12285 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 12286 |.cold_code 12287 |7: 12288 } 12289 12290 if (opline->opcode != ZEND_FETCH_DIM_RW) { 12291 | SET_EX_OPLINE opline, r0 12292 } 12293 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12294 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12295 } 12296 if (opline->op2_type == IS_UNUSED) { 12297 | xor FCARG2a, FCARG2a 12298 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12299 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12300 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12301 } else { 12302 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12303 } 12304 |.if X64 12305 | LOAD_ZVAL_ADDR CARG3, res_addr 12306 |.else 12307 | sub r4, 12 12308 | PUSH_ZVAL_ADDR res_addr, r0 12309 |.endif 12310 switch (opline->opcode) { 12311 case ZEND_FETCH_DIM_W: 12312 case ZEND_FETCH_LIST_W: 12313 | EXT_CALL zend_jit_fetch_dim_obj_w_helper, r0 12314 break; 12315 case ZEND_FETCH_DIM_RW: 12316 | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, r0 12317 break; 12318// case ZEND_FETCH_DIM_UNSET: 12319// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0 12320// break; 12321 default: 12322 ZEND_UNREACHABLE(); 12323 } 12324 |.if not(X64) 12325 | add r4, 12 12326 |.endif 12327 12328 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 12329 | jmp >8 // END 12330 |.code 12331 } 12332 } 12333 12334#ifdef ZEND_JIT_USE_RC_INFERENCE 12335 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 12336 /* ASSIGN_DIM may increase refcount of the key */ 12337 op2_info |= MAY_BE_RCN; 12338 } 12339#endif 12340 12341 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 12342 && (op2_info & MAY_HAVE_DTOR) 12343 && (op2_info & MAY_BE_RC1)) { 12344 may_throw = 1; 12345 } 12346 |8: 12347 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12348 12349 if (may_throw) { 12350 if (!zend_jit_check_exception(Dst)) { 12351 return 0; 12352 } 12353 } 12354 12355 return 1; 12356} 12357 12358static int zend_jit_isset_isempty_dim(dasm_State **Dst, 12359 const zend_op *opline, 12360 uint32_t op1_info, 12361 zend_jit_addr op1_addr, 12362 bool op1_avoid_refcounting, 12363 uint32_t op2_info, 12364 uint8_t dim_type, 12365 int may_throw, 12366 zend_uchar smart_branch_opcode, 12367 uint32_t target_label, 12368 uint32_t target_label2, 12369 const void *exit_addr) 12370{ 12371 zend_jit_addr op2_addr, res_addr; 12372 12373 // TODO: support for empty() ??? 12374 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 12375 12376 op2_addr = OP2_ADDR(); 12377 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12378 12379 if (op1_info & MAY_BE_REF) { 12380 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12381 | ZVAL_DEREF FCARG1a, op1_info 12382 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12383 } 12384 12385 if (op1_info & MAY_BE_ARRAY) { 12386 const void *found_exit_addr = NULL; 12387 const void *not_found_exit_addr = NULL; 12388 12389 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12390 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12391 } 12392 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 12393 if (exit_addr 12394 && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) 12395 && !may_throw 12396 && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) 12397 && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { 12398 if (smart_branch_opcode == ZEND_JMPNZ) { 12399 found_exit_addr = exit_addr; 12400 } else { 12401 not_found_exit_addr = exit_addr; 12402 } 12403 } 12404 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, dim_type, found_exit_addr, not_found_exit_addr, NULL)) { 12405 return 0; 12406 } 12407 12408 if (found_exit_addr) { 12409 |9: 12410 return 1; 12411 } else if (not_found_exit_addr) { 12412 |8: 12413 return 1; 12414 } 12415 } 12416 12417 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 12418 if (op1_info & MAY_BE_ARRAY) { 12419 |.cold_code 12420 |7: 12421 } 12422 12423 if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { 12424 | SET_EX_OPLINE opline, r0 12425 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12426 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12427 } 12428 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12429 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12430 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12431 } else { 12432 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12433 } 12434 | EXT_CALL zend_jit_isset_dim_helper, r0 12435 | test r0, r0 12436 | jz >9 12437 if (op1_info & MAY_BE_ARRAY) { 12438 | jmp >8 12439 |.code 12440 } 12441 } else { 12442 if (op2_info & MAY_BE_UNDEF) { 12443 if (op2_info & MAY_BE_ANY) { 12444 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 12445 } 12446 | SET_EX_OPLINE opline, r0 12447 | mov FCARG1d, opline->op2.var 12448 | EXT_CALL zend_jit_undefined_op_helper, r0 12449 |1: 12450 } 12451 if (op1_info & MAY_BE_ARRAY) { 12452 | jmp >9 12453 |.code 12454 } 12455 } 12456 } 12457 12458#ifdef ZEND_JIT_USE_RC_INFERENCE 12459 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 12460 /* Magic offsetExists() may increase refcount of the key */ 12461 op2_info |= MAY_BE_RCN; 12462 } 12463#endif 12464 12465 if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { 12466 |8: 12467 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12468 if (!op1_avoid_refcounting) { 12469 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12470 } 12471 if (may_throw) { 12472 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 12473 return 0; 12474 } 12475 } 12476 if (!(opline->extended_value & ZEND_ISEMPTY)) { 12477 if (exit_addr) { 12478 if (smart_branch_opcode == ZEND_JMPNZ) { 12479 | jmp &exit_addr 12480 } else { 12481 | jmp >8 12482 } 12483 } else if (smart_branch_opcode) { 12484 if (smart_branch_opcode == ZEND_JMPZ) { 12485 | jmp =>target_label2 12486 } else if (smart_branch_opcode == ZEND_JMPNZ) { 12487 | jmp =>target_label 12488 } else { 12489 ZEND_UNREACHABLE(); 12490 } 12491 } else { 12492 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 12493 | jmp >8 12494 } 12495 } else { 12496 | NIY // TODO: support for empty() 12497 } 12498 } 12499 12500 |9: // not found 12501 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12502 if (!op1_avoid_refcounting) { 12503 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12504 } 12505 if (may_throw) { 12506 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 12507 return 0; 12508 } 12509 } 12510 if (!(opline->extended_value & ZEND_ISEMPTY)) { 12511 if (exit_addr) { 12512 if (smart_branch_opcode == ZEND_JMPZ) { 12513 | jmp &exit_addr 12514 } 12515 } else if (smart_branch_opcode) { 12516 if (smart_branch_opcode == ZEND_JMPZ) { 12517 | jmp =>target_label 12518 } else if (smart_branch_opcode == ZEND_JMPNZ) { 12519 } else { 12520 ZEND_UNREACHABLE(); 12521 } 12522 } else { 12523 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 12524 } 12525 } else { 12526 | NIY // TODO: support for empty() 12527 } 12528 12529 |8: 12530 12531 return 1; 12532} 12533 12534static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 12535{ 12536 zend_jit_addr op1_addr = OP1_ADDR(); 12537 zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); 12538 12539 | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; 12540 | mov FCARG2a, EX->run_time_cache 12541 | mov r0, aword [FCARG2a + opline->extended_value] 12542 | sub r0, 1 12543 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) 12544 | MEM_LOAD_ZTS ecx, dword, executor_globals, symbol_table.nNumUsed, r1 12545 |.if X64 12546 | shl r1, 5 12547 |.else 12548 | imul r1, sizeof(Bucket) 12549 |.endif 12550 | cmp r0, r1 12551 | jae >9 12552 | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); 12553 | MEM_LOAD_OP_ZTS add, r0, aword, executor_globals, symbol_table.arData, r1 12554 | IF_NOT_Z_TYPE r0, IS_REFERENCE, >9 12555 | // (EXPECTED(p->key == varname)) 12556 | ADDR_CMP aword [r0 + offsetof(Bucket, key)], varname, r1 12557 | jne >9 12558 | GET_Z_PTR r0, r0 12559 | GC_ADDREF r0 12560 |1: 12561 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 12562 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12563 | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) 12564 | IF_ZVAL_REFCOUNTED op1_addr, >2 12565 |.cold_code 12566 |2: 12567 } 12568 | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); 12569 | GET_ZVAL_PTR FCARG1a, op1_addr 12570 | // ZVAL_REF(variable_ptr, ref) 12571 | SET_ZVAL_PTR op1_addr, r0 12572 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 12573 | // if (GC_DELREF(garbage) == 0) 12574 | GC_DELREF FCARG1a 12575 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 12576 | jnz >3 12577 } else { 12578 | jnz >5 12579 } 12580 | ZVAL_DTOR_FUNC op1_info, opline 12581 | jmp >5 12582 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 12583 |3: 12584 | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) 12585 | IF_GC_MAY_NOT_LEAK FCARG1a, >5 12586 | EXT_CALL gc_possible_root, r1 12587 | jmp >5 12588 } 12589 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12590 |.code 12591 } 12592 } 12593 12594 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12595 | // ZVAL_REF(variable_ptr, ref) 12596 | SET_ZVAL_PTR op1_addr, r0 12597 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 12598 } 12599 |5: 12600 //END of handler 12601 12602 |.cold_code 12603 |9: 12604 | LOAD_ADDR FCARG1a, (ptrdiff_t)varname 12605 if (opline->extended_value) { 12606 | add FCARG2a, opline->extended_value 12607 } 12608 | EXT_CALL zend_jit_fetch_global_helper, r0 12609 | jmp <1 12610 |.code 12611 12612 return 1; 12613} 12614 12615static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception) 12616{ 12617 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12618 bool in_cold = 0; 12619 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 12620 zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1 : ZREG_R0; 12621 12622 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12623 && JIT_G(current_frame) 12624 && JIT_G(current_frame)->prev) { 12625 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 12626 uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)); 12627 12628 if (type != IS_UNKNOWN && (type_mask & (1u << type))) { 12629 return 1; 12630 } 12631 } 12632 12633 if (ZEND_ARG_SEND_MODE(arg_info)) { 12634 if (opline->opcode == ZEND_RECV_INIT) { 12635 | LOAD_ZVAL_ADDR Ra(tmp_reg), res_addr 12636 | ZVAL_DEREF Ra(tmp_reg), MAY_BE_REF 12637 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); 12638 } else { 12639 | GET_ZVAL_PTR Ra(tmp_reg), res_addr 12640 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); 12641 } 12642 } 12643 12644 if (type_mask != 0) { 12645 if (is_power_of_two(type_mask)) { 12646 uint32_t type_code = concrete_type(type_mask); 12647 | IF_NOT_ZVAL_TYPE res_addr, type_code, >1 12648 } else { 12649 | mov edx, 1 12650 | mov cl, byte [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)+offsetof(zval, u1.v.type)] 12651 | shl edx, cl 12652 | test edx, type_mask 12653 | je >1 12654 } 12655 12656 |.cold_code 12657 |1: 12658 12659 in_cold = 1; 12660 } 12661 12662 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 12663 | LOAD_ZVAL_ADDR FCARG1a, res_addr 12664 } 12665 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12666 | SET_EX_OPLINE opline, r0 12667 } else { 12668 | ADDR_STORE aword EX->opline, opline, r0 12669 } 12670 | LOAD_ADDR FCARG2a, (ptrdiff_t)arg_info 12671 | EXT_CALL zend_jit_verify_arg_slow, r0 12672 12673 if (check_exception) { 12674 | test al, al 12675 if (in_cold) { 12676 | jnz >1 12677 | jmp ->exception_handler 12678 |.code 12679 |1: 12680 } else { 12681 | jz ->exception_handler 12682 } 12683 } else if (in_cold) { 12684 | jmp >1 12685 |.code 12686 |1: 12687 } 12688 12689 return 1; 12690} 12691 12692static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) 12693{ 12694 uint32_t arg_num = opline->op1.num; 12695 zend_arg_info *arg_info = NULL; 12696 12697 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12698 if (EXPECTED(arg_num <= op_array->num_args)) { 12699 arg_info = &op_array->arg_info[arg_num-1]; 12700 } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { 12701 arg_info = &op_array->arg_info[op_array->num_args]; 12702 } 12703 if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { 12704 arg_info = NULL; 12705 } 12706 } 12707 12708 if (arg_info || (opline+1)->opcode != ZEND_RECV) { 12709 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12710 if (!JIT_G(current_frame) || 12711 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 || 12712 arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 12713 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12714 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12715 12716 if (!exit_addr) { 12717 return 0; 12718 } 12719 | cmp dword EX->This.u2.num_args, arg_num 12720 | jb &exit_addr 12721 } 12722 } else { 12723 | cmp dword EX->This.u2.num_args, arg_num 12724 | jb >1 12725 |.cold_code 12726 |1: 12727 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12728 | SET_EX_OPLINE opline, r0 12729 } else { 12730 | ADDR_STORE aword EX->opline, opline, r0 12731 } 12732 | mov FCARG1a, FP 12733 | EXT_CALL zend_missing_arg_error, r0 12734 | jmp ->exception_handler 12735 |.code 12736 } 12737 } 12738 12739 if (arg_info) { 12740 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { 12741 return 0; 12742 } 12743 } 12744 12745 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12746 if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { 12747 | LOAD_IP_ADDR (opline + 1) 12748 zend_jit_set_last_valid_opline(opline + 1); 12749 } 12750 } 12751 12752 return 1; 12753} 12754 12755static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw) 12756{ 12757 uint32_t arg_num = opline->op1.num; 12758 zval *zv = RT_CONSTANT(opline, opline->op2); 12759 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12760 12761 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12762 && JIT_G(current_frame) 12763 && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) { 12764 if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 12765 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0 12766 if (Z_REFCOUNTED_P(zv)) { 12767 | ADDREF_CONST zv, r0 12768 } 12769 } 12770 } else { 12771 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 12772 (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 12773 | cmp dword EX->This.u2.num_args, arg_num 12774 | jae >5 12775 } 12776 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0 12777 if (Z_REFCOUNTED_P(zv)) { 12778 | ADDREF_CONST zv, r0 12779 } 12780 } 12781 12782 if (Z_CONSTANT_P(zv)) { 12783 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12784 | SET_EX_OPLINE opline, r0 12785 } else { 12786 | ADDR_STORE aword EX->opline, opline, r0 12787 } 12788 | LOAD_ZVAL_ADDR FCARG1a, res_addr 12789 | mov r0, EX->func 12790 | mov FCARG2a, [r0 + offsetof(zend_op_array, scope)] 12791 | .if X64 12792 | EXT_CALL zval_update_constant_ex, r0 12793 | .else 12794 ||#if (PHP_VERSION_ID < 80100) && (SIZEOF_SIZE_T == 4) 12795 | EXT_CALL zval_jit_update_constant_ex, r0 12796 ||#else 12797 | EXT_CALL zval_update_constant_ex, r0 12798 ||#endif 12799 | .endif 12800 | test al, al 12801 | jnz >1 12802 |.cold_code 12803 |1: 12804 | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline 12805 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF 12806 | jmp ->exception_handler 12807 |.code 12808 } 12809 12810 |5: 12811 12812 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12813 do { 12814 zend_arg_info *arg_info; 12815 12816 if (arg_num <= op_array->num_args) { 12817 arg_info = &op_array->arg_info[arg_num-1]; 12818 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { 12819 arg_info = &op_array->arg_info[op_array->num_args]; 12820 } else { 12821 break; 12822 } 12823 if (!ZEND_TYPE_IS_SET(arg_info->type)) { 12824 break; 12825 } 12826 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { 12827 return 0; 12828 } 12829 } while (0); 12830 } 12831 12832 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12833 if (is_last) { 12834 | LOAD_IP_ADDR (opline + 1) 12835 zend_jit_set_last_valid_opline(opline + 1); 12836 } 12837 } 12838 12839 return 1; 12840} 12841 12842static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) 12843{ 12844 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 12845 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12846 12847 if (!exit_addr) { 12848 return 0; 12849 } 12850 12851 |.if X64 12852 || if (!IS_SIGNED_32BIT(ce)) { 12853 | mov64 r0, ((ptrdiff_t)ce) 12854 | cmp aword [FCARG1a + offsetof(zend_object, ce)], r0 12855 || } else { 12856 | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce 12857 || } 12858 |.else 12859 | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce 12860 |.endif 12861 | jne &exit_addr 12862 12863 return 1; 12864} 12865 12866static int zend_jit_fetch_obj(dasm_State **Dst, 12867 const zend_op *opline, 12868 const zend_op_array *op_array, 12869 zend_ssa *ssa, 12870 const zend_ssa_op *ssa_op, 12871 uint32_t op1_info, 12872 zend_jit_addr op1_addr, 12873 bool op1_indirect, 12874 zend_class_entry *ce, 12875 bool ce_is_instanceof, 12876 bool on_this, 12877 bool delayed_fetch_this, 12878 bool op1_avoid_refcounting, 12879 zend_class_entry *trace_ce, 12880 uint8_t prop_type, 12881 int may_throw) 12882{ 12883 zval *member; 12884 zend_property_info *prop_info; 12885 bool may_be_dynamic = 1; 12886 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12887 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 12888 zend_jit_addr prop_addr; 12889 uint32_t res_info = RES_INFO(); 12890 bool type_loaded = 0; 12891 12892 ZEND_ASSERT(opline->op2_type == IS_CONST); 12893 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 12894 12895 member = RT_CONSTANT(opline, opline->op2); 12896 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 12897 prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); 12898 12899 if (on_this) { 12900 | GET_ZVAL_PTR FCARG1a, this_addr 12901 } else { 12902 if (opline->op1_type == IS_VAR 12903 && opline->opcode == ZEND_FETCH_OBJ_W 12904 && (op1_info & MAY_BE_INDIRECT) 12905 && Z_REG(op1_addr) == ZREG_FP) { 12906 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12907 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 12908 | GET_Z_PTR FCARG1a, FCARG1a 12909 |1: 12910 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12911 } 12912 if (op1_info & MAY_BE_REF) { 12913 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12914 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12915 } 12916 | ZVAL_DEREF FCARG1a, op1_info 12917 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12918 } 12919 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 12920 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12921 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12922 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12923 12924 if (!exit_addr) { 12925 return 0; 12926 } 12927 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 12928 } else { 12929 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7 12930 } 12931 } 12932 | GET_ZVAL_PTR FCARG1a, op1_addr 12933 } 12934 12935 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12936 prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename); 12937 if (prop_info) { 12938 ce = trace_ce; 12939 ce_is_instanceof = 0; 12940 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 12941 if (on_this && JIT_G(current_frame) 12942 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 12943 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 12944 } else if (zend_jit_class_guard(Dst, opline, ce)) { 12945 if (on_this && JIT_G(current_frame)) { 12946 JIT_G(current_frame)->ce = ce; 12947 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 12948 } 12949 } else { 12950 return 0; 12951 } 12952 if (ssa->var_info && ssa_op->op1_use >= 0) { 12953 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 12954 ssa->var_info[ssa_op->op1_use].ce = ce; 12955 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 12956 } 12957 } 12958 } 12959 } 12960 12961 if (!prop_info) { 12962 | mov r0, EX->run_time_cache 12963 | mov r2, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)] 12964 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 12965 | jne >5 12966 | mov r0, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)] 12967 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); 12968 if (may_be_dynamic) { 12969 | test r0, r0 12970 if (opline->opcode == ZEND_FETCH_OBJ_W) { 12971 | jl >5 12972 } else { 12973 | jl >8 // dynamic property 12974 } 12975 } 12976 | mov edx, dword [FCARG1a + r0 + 8] 12977 | IF_UNDEF dl, >5 12978 | add FCARG1a, r0 12979 type_loaded = 1; 12980 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12981 if (opline->opcode == ZEND_FETCH_OBJ_W 12982 && (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) { 12983 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 12984 12985 | mov r0, EX->run_time_cache 12986 | mov FCARG2a, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2] 12987 | test FCARG2a, FCARG2a 12988 | jnz >1 12989 |.cold_code 12990 |1: 12991 | test dword [FCARG2a + offsetof(zend_property_info, flags)], ZEND_ACC_READONLY 12992 if (flags) { 12993 | jz >3 12994 } else { 12995 | jz >4 12996 } 12997 | IF_NOT_Z_TYPE FCARG1a, IS_OBJECT, >2 12998 | GET_Z_PTR r0, FCARG1a 12999 | GC_ADDREF r0 13000 | SET_ZVAL_PTR res_addr, r0 13001 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX 13002 | jmp >9 13003 |2: 13004 | mov FCARG1a, FCARG2a 13005 | SET_EX_OPLINE opline, r0 13006 | EXT_CALL zend_readonly_property_modification_error, r0 13007 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 13008 | jmp >9 13009 |3: 13010 if (flags == ZEND_FETCH_DIM_WRITE) { 13011 | SET_EX_OPLINE opline, r0 13012 | EXT_CALL zend_jit_check_array_promotion, r0 13013 | jmp >9 13014 } else if (flags == ZEND_FETCH_REF) { 13015 |.if X64 13016 | LOAD_ZVAL_ADDR CARG3, res_addr 13017 |.else 13018 | sub r4, 12 13019 | PUSH_ZVAL_ADDR res_addr, r0 13020 |.endif 13021 | EXT_CALL zend_jit_create_typed_ref, r0 13022 |.if not(X64) 13023 | add r4, 12 13024 |.endif 13025 | jmp >9 13026 } else { 13027 ZEND_ASSERT(flags == 0); 13028 } 13029 |.code 13030 |4: 13031 } 13032 } else { 13033 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13034 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13035 if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { 13036 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 13037 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13038 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13039 13040 if (!exit_addr) { 13041 return 0; 13042 } 13043 type_loaded = 1; 13044 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13045 | IF_UNDEF dl, &exit_addr 13046 } 13047 } else { 13048 type_loaded = 1; 13049 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13050 | IF_UNDEF dl, >5 13051 } 13052 if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) { 13053 if (!type_loaded) { 13054 type_loaded = 1; 13055 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13056 } 13057 | IF_NOT_TYPE dl, IS_OBJECT, >4 13058 | GET_ZVAL_PTR r0, prop_addr 13059 | GC_ADDREF r0 13060 | SET_ZVAL_PTR res_addr, r0 13061 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX 13062 | jmp >9 13063 |.cold_code 13064 |4: 13065 | LOAD_ADDR FCARG1a, prop_info 13066 | SET_EX_OPLINE opline, r0 13067 | EXT_CALL zend_readonly_property_modification_error, r0 13068 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 13069 | jmp >9 13070 |.code 13071 } 13072 if (opline->opcode == ZEND_FETCH_OBJ_W 13073 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) 13074 && ZEND_TYPE_IS_SET(prop_info->type)) { 13075 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 13076 13077 if (flags == ZEND_FETCH_DIM_WRITE) { 13078 if ((ZEND_TYPE_FULL_MASK(prop_info->type) & MAY_BE_ARRAY) == 0) { 13079 if (!type_loaded) { 13080 type_loaded = 1; 13081 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13082 } 13083 | cmp dl, IS_FALSE 13084 | jle >1 13085 |.cold_code 13086 |1: 13087 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13088 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13089 } 13090 | LOAD_ADDR FCARG2a, prop_info 13091 | SET_EX_OPLINE opline, r0 13092 | EXT_CALL zend_jit_check_array_promotion, r0 13093 | jmp >9 13094 |.code 13095 } 13096 } else if (flags == ZEND_FETCH_REF) { 13097 if (!type_loaded) { 13098 type_loaded = 1; 13099 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13100 } 13101 | IF_TYPE dl, IS_REFERENCE, >1 13102 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13103 | LOAD_ADDR FCARG2a, prop_info 13104 } else { 13105 int prop_info_offset = 13106 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13107 13108 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13109 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13110 | mov FCARG2a, aword[r0 + prop_info_offset] 13111 } 13112 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13113 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13114 } 13115 |.if X64 13116 | LOAD_ZVAL_ADDR CARG3, res_addr 13117 |.else 13118 | sub r4, 12 13119 | PUSH_ZVAL_ADDR res_addr, r0 13120 |.endif 13121 | EXT_CALL zend_jit_create_typed_ref, r0 13122 |.if not(X64) 13123 | add r4, 12 13124 |.endif 13125 | jmp >9 13126 |1: 13127 } else { 13128 ZEND_UNREACHABLE(); 13129 } 13130 } 13131 } 13132 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13133 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13134 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13135 } 13136 | SET_ZVAL_PTR res_addr, FCARG1a 13137 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 13138 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { 13139 ssa->var_info[ssa_op->result_def].indirect_reference = 1; 13140 } 13141 } else { 13142 bool result_avoid_refcounting = 0; 13143 13144 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { 13145 uint32_t flags = 0; 13146 uint32_t old_info; 13147 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 13148 int32_t exit_point; 13149 const void *exit_addr; 13150 zend_uchar type; 13151 zend_jit_addr val_addr = prop_addr; 13152 13153 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 13154 && !delayed_fetch_this 13155 && !op1_avoid_refcounting) { 13156 flags = ZEND_JIT_EXIT_FREE_OP1; 13157 } 13158 13159 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 13160 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 13161 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 13162 && (ssa_op+1)->op1_use == ssa_op->result_def 13163 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 13164 result_avoid_refcounting = 1; 13165 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 13166 } 13167 13168 type = concrete_type(res_info); 13169 13170 if (prop_type != IS_UNKNOWN 13171 && prop_type != IS_UNDEF 13172 && prop_type != IS_REFERENCE 13173 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) { 13174 exit_point = zend_jit_trace_get_exit_point(opline, 0); 13175 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13176 if (!exit_addr) { 13177 return 0; 13178 } 13179 } else { 13180 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 13181 | LOAD_ZVAL_ADDR r0, prop_addr 13182 if (op1_avoid_refcounting) { 13183 SET_STACK_REG(JIT_G(current_frame)->stack, 13184 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 13185 } 13186 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 13187 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 13188 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 13189 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 13190 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 13191 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13192 if (!exit_addr) { 13193 return 0; 13194 } 13195 13196 if (!type_loaded) { 13197 type_loaded = 1; 13198 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13199 } 13200 | // ZVAL_DEREF() 13201 | IF_NOT_TYPE dl, IS_REFERENCE, >1 13202 | GET_Z_PTR r0, r0 13203 | add r0, offsetof(zend_reference, val) 13204 | GET_ZVAL_TYPE_INFO edx, val_addr 13205 } 13206 res_info &= ~MAY_BE_GUARD; 13207 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 13208 if (type < IS_STRING) { 13209 |1: 13210 if (type_loaded) { 13211 | IF_NOT_TYPE dl, type, &exit_addr 13212 } else { 13213 | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr 13214 } 13215 } else { 13216 if (!type_loaded) { 13217 type_loaded = 1; 13218 | GET_ZVAL_TYPE_INFO edx, val_addr 13219 } 13220 |1: 13221 | IF_NOT_TYPE dl, type, &exit_addr 13222 } 13223 | // ZVAL_COPY 13224 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 13225 if (type < IS_STRING) { 13226 if (Z_REG(res_addr) != ZREG_FP || 13227 JIT_G(current_frame) == NULL || 13228 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 13229 | SET_ZVAL_TYPE_INFO res_addr, type 13230 } 13231 } else { 13232 | SET_ZVAL_TYPE_INFO res_addr, edx 13233 if (!result_avoid_refcounting) { 13234 | TRY_ADDREF res_info, dh, r1 13235 } 13236 } 13237 } else { 13238 if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_R2)) { 13239 return 0; 13240 } 13241 } 13242 } 13243 13244 if (op1_avoid_refcounting) { 13245 SET_STACK_REG(JIT_G(current_frame)->stack, 13246 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 13247 } 13248 13249 |.cold_code 13250 13251 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { 13252 |5: 13253 | SET_EX_OPLINE opline, r0 13254 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13255 | EXT_CALL zend_jit_fetch_obj_w_slow, r0 13256 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13257 | EXT_CALL zend_jit_fetch_obj_r_slow, r0 13258 } else { 13259 | EXT_CALL zend_jit_fetch_obj_is_slow, r0 13260 } 13261 | jmp >9 13262 } 13263 13264 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 13265 |7: 13266 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13267 | SET_EX_OPLINE opline, r0 13268 if (opline->opcode != ZEND_FETCH_OBJ_W 13269 && (op1_info & MAY_BE_UNDEF)) { 13270 zend_jit_addr orig_op1_addr = OP1_ADDR(); 13271 13272 if (op1_info & MAY_BE_ANY) { 13273 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 13274 } 13275 | mov FCARG1d, opline->op1.var 13276 | EXT_CALL zend_jit_undefined_op_helper, r0 13277 |1: 13278 | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr 13279 } else if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13280 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13281 } 13282 | LOAD_ADDR FCARG2a, Z_STRVAL_P(member) 13283 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13284 | EXT_CALL zend_jit_invalid_property_write, r0 13285 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 13286 } else { 13287 | EXT_CALL zend_jit_invalid_property_read, r0 13288 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 13289 } 13290 | jmp >9 13291 } else { 13292 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 13293 | jmp >9 13294 } 13295 } 13296 13297 if (!prop_info 13298 && may_be_dynamic 13299 && opline->opcode != ZEND_FETCH_OBJ_W) { 13300 |8: 13301 | mov FCARG2a, r0 13302 | SET_EX_OPLINE opline, r0 13303 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13304 | EXT_CALL zend_jit_fetch_obj_r_dynamic, r0 13305 } else { 13306 | EXT_CALL zend_jit_fetch_obj_is_dynamic, r0 13307 } 13308 | jmp >9 13309 } 13310 13311 |.code; 13312 |9: // END 13313 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13314 if (opline->op1_type == IS_VAR 13315 && opline->opcode == ZEND_FETCH_OBJ_W 13316 && (op1_info & MAY_BE_RC1)) { 13317 zend_jit_addr orig_op1_addr = OP1_ADDR(); 13318 13319 | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1 13320 | GET_ZVAL_PTR FCARG1a, orig_op1_addr 13321 | GC_DELREF FCARG1a 13322 | jnz >1 13323 | SET_EX_OPLINE opline, r0 13324 | EXT_CALL zend_jit_extract_helper, r0 13325 |1: 13326 } else if (!op1_avoid_refcounting) { 13327 if (on_this) { 13328 op1_info &= ~MAY_BE_RC1; 13329 } 13330 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 13331 } 13332 } 13333 13334 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 13335 && prop_info 13336 && (opline->opcode != ZEND_FETCH_OBJ_W || 13337 !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) || 13338 !ZEND_TYPE_IS_SET(prop_info->type)) 13339 && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) { 13340 may_throw = 0; 13341 } 13342 13343 if (may_throw) { 13344 if (!zend_jit_check_exception(Dst)) { 13345 return 0; 13346 } 13347 } 13348 13349 return 1; 13350} 13351 13352static int zend_jit_incdec_obj(dasm_State **Dst, 13353 const zend_op *opline, 13354 const zend_op_array *op_array, 13355 zend_ssa *ssa, 13356 const zend_ssa_op *ssa_op, 13357 uint32_t op1_info, 13358 zend_jit_addr op1_addr, 13359 bool op1_indirect, 13360 zend_class_entry *ce, 13361 bool ce_is_instanceof, 13362 bool on_this, 13363 bool delayed_fetch_this, 13364 zend_class_entry *trace_ce, 13365 uint8_t prop_type) 13366{ 13367 zval *member; 13368 zend_string *name; 13369 zend_property_info *prop_info; 13370 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13371 zend_jit_addr res_addr = 0; 13372 zend_jit_addr prop_addr; 13373 bool needs_slow_path = 0; 13374 bool use_prop_guard = 0; 13375 bool may_throw = 0; 13376 uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0; 13377 13378 ZEND_ASSERT(opline->op2_type == IS_CONST); 13379 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13380 13381 if (opline->result_type != IS_UNUSED) { 13382 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 13383 } 13384 13385 member = RT_CONSTANT(opline, opline->op2); 13386 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13387 name = Z_STR_P(member); 13388 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13389 13390 if (on_this) { 13391 | GET_ZVAL_PTR FCARG1a, this_addr 13392 } else { 13393 if (opline->op1_type == IS_VAR 13394 && (op1_info & MAY_BE_INDIRECT) 13395 && Z_REG(op1_addr) == ZREG_FP) { 13396 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13397 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13398 | GET_Z_PTR FCARG1a, FCARG1a 13399 |1: 13400 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13401 } 13402 if (op1_info & MAY_BE_REF) { 13403 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13404 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13405 } 13406 | ZVAL_DEREF FCARG1a, op1_info 13407 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13408 } 13409 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13410 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13411 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13412 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13413 13414 if (!exit_addr) { 13415 return 0; 13416 } 13417 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13418 } else { 13419 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 13420 |.cold_code 13421 |1: 13422 | SET_EX_OPLINE opline, r0 13423 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13424 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13425 } 13426 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 13427 | EXT_CALL zend_jit_invalid_property_incdec, r0 13428 | jmp ->exception_handler 13429 |.code 13430 } 13431 } 13432 | GET_ZVAL_PTR FCARG1a, op1_addr 13433 } 13434 13435 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13436 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13437 if (prop_info) { 13438 ce = trace_ce; 13439 ce_is_instanceof = 0; 13440 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13441 if (on_this && JIT_G(current_frame) 13442 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13443 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13444 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13445 if (on_this && JIT_G(current_frame)) { 13446 JIT_G(current_frame)->ce = ce; 13447 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13448 } 13449 } else { 13450 return 0; 13451 } 13452 if (ssa->var_info && ssa_op->op1_use >= 0) { 13453 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13454 ssa->var_info[ssa_op->op1_use].ce = ce; 13455 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13456 } 13457 if (ssa->var_info && ssa_op->op1_def >= 0) { 13458 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13459 ssa->var_info[ssa_op->op1_def].ce = ce; 13460 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13461 } 13462 } 13463 } 13464 } 13465 13466 use_prop_guard = (prop_type != IS_UNKNOWN 13467 && prop_type != IS_UNDEF 13468 && prop_type != IS_REFERENCE 13469 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 13470 13471 if (!prop_info) { 13472 needs_slow_path = 1; 13473 13474 | mov r0, EX->run_time_cache 13475 | mov r2, aword [r0 + opline->extended_value] 13476 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13477 | jne >7 13478 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 13479 | cmp aword [r0 + opline->extended_value + sizeof(void*) * 2], 0 13480 | jnz >7 13481 } 13482 | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] 13483 | test r0, r0 13484 | jl >7 13485 if (!use_prop_guard) { 13486 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 13487 } 13488 | add FCARG1a, r0 13489 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13490 } else { 13491 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13492 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 13493 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13494 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13495 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13496 13497 if (!exit_addr) { 13498 return 0; 13499 } 13500 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 13501 } else { 13502 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 13503 needs_slow_path = 1; 13504 } 13505 } 13506 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13507 may_throw = 1; 13508 | SET_EX_OPLINE opline, r0 13509 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13510 | LOAD_ADDR FCARG2a, prop_info 13511 } else { 13512 int prop_info_offset = 13513 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13514 13515 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13516 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13517 | mov FCARG2a, aword[r0 + prop_info_offset] 13518 } 13519 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13520 if (opline->result_type == IS_UNUSED) { 13521 switch (opline->opcode) { 13522 case ZEND_PRE_INC_OBJ: 13523 case ZEND_POST_INC_OBJ: 13524 | EXT_CALL zend_jit_inc_typed_prop, r0 13525 break; 13526 case ZEND_PRE_DEC_OBJ: 13527 case ZEND_POST_DEC_OBJ: 13528 | EXT_CALL zend_jit_dec_typed_prop, r0 13529 break; 13530 default: 13531 ZEND_UNREACHABLE(); 13532 } 13533 } else { 13534 |.if X64 13535 | LOAD_ZVAL_ADDR CARG3, res_addr 13536 |.else 13537 | sub r4, 12 13538 | PUSH_ZVAL_ADDR res_addr, r0 13539 |.endif 13540 switch (opline->opcode) { 13541 case ZEND_PRE_INC_OBJ: 13542 | EXT_CALL zend_jit_pre_inc_typed_prop, r0 13543 break; 13544 case ZEND_PRE_DEC_OBJ: 13545 | EXT_CALL zend_jit_pre_dec_typed_prop, r0 13546 break; 13547 case ZEND_POST_INC_OBJ: 13548 | EXT_CALL zend_jit_post_inc_typed_prop, r0 13549 break; 13550 case ZEND_POST_DEC_OBJ: 13551 | EXT_CALL zend_jit_post_dec_typed_prop, r0 13552 break; 13553 default: 13554 ZEND_UNREACHABLE(); 13555 } 13556 |.if not(X64) 13557 | add r4, 12 13558 |.endif 13559 } 13560 } 13561 } 13562 13563 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13564 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13565 zend_jit_addr var_addr = prop_addr; 13566 13567 if (use_prop_guard) { 13568 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 13569 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13570 if (!exit_addr) { 13571 return 0; 13572 } 13573 13574 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr 13575 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 13576 } 13577 13578 if (var_info & MAY_BE_REF) { 13579 may_throw = 1; 13580 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13581 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13582 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13583 } 13584 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 13585 | GET_ZVAL_PTR FCARG1a, var_addr 13586 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 13587 | jnz >1 13588 | lea FCARG1a, aword [FCARG1a + offsetof(zend_reference, val)] 13589 |.cold_code 13590 |1: 13591 if (opline) { 13592 | SET_EX_OPLINE opline, r0 13593 } 13594 if (opline->result_type == IS_UNUSED) { 13595 | xor FCARG2a, FCARG2a 13596 } else { 13597 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13598 } 13599 switch (opline->opcode) { 13600 case ZEND_PRE_INC_OBJ: 13601 | EXT_CALL zend_jit_pre_inc_typed_ref, r0 13602 break; 13603 case ZEND_PRE_DEC_OBJ: 13604 | EXT_CALL zend_jit_pre_dec_typed_ref, r0 13605 break; 13606 case ZEND_POST_INC_OBJ: 13607 | EXT_CALL zend_jit_post_inc_typed_ref, r0 13608 break; 13609 case ZEND_POST_DEC_OBJ: 13610 | EXT_CALL zend_jit_post_dec_typed_ref, r0 13611 break; 13612 default: 13613 ZEND_UNREACHABLE(); 13614 } 13615 | jmp >9 13616 |.code 13617 |2: 13618 } 13619 13620 if (var_info & MAY_BE_LONG) { 13621 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 13622 | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2 13623 } 13624 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 13625 if (opline->result_type != IS_UNUSED) { 13626 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R1, ZREG_R2 13627 } 13628 } 13629 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13630 | LONG_OP_WITH_32BIT_CONST add, var_addr, Z_L(1) 13631 } else { 13632 | LONG_OP_WITH_32BIT_CONST sub, var_addr, Z_L(1) 13633 } 13634 | jo >3 13635 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { 13636 if (opline->result_type != IS_UNUSED) { 13637 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R0, ZREG_R2 13638 } 13639 } 13640 |.cold_code 13641 } 13642 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 13643 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13644 may_throw = 1; 13645 } 13646 if (var_info & MAY_BE_LONG) { 13647 |2: 13648 } 13649 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 13650 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13651 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13652 } 13653 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 13654 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_R0, ZREG_R2 13655 | TRY_ADDREF MAY_BE_ANY, ah, r2 13656 } 13657 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13658 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13659 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13660 | EXT_CALL zend_jit_pre_inc, r0 13661 } else { 13662 | EXT_CALL increment_function, r0 13663 } 13664 } else { 13665 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13666 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13667 | EXT_CALL zend_jit_pre_dec, r0 13668 } else { 13669 | EXT_CALL decrement_function, r0 13670 } 13671 } 13672 if (var_info & MAY_BE_LONG) { 13673 | jmp >4 13674 } 13675 } 13676 if (var_info & MAY_BE_LONG) { 13677 |3: 13678 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13679 |.if X64 13680 | mov64 rax, 0x43e0000000000000 13681 | SET_ZVAL_LVAL var_addr, rax 13682 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13683 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13684 | SET_ZVAL_LVAL res_addr, rax 13685 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13686 } 13687 |.else 13688 | SET_ZVAL_LVAL var_addr, 0 13689 | SET_ZVAL_W2 var_addr, 0x41e00000 13690 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13691 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13692 | SET_ZVAL_LVAL res_addr, 0 13693 | SET_ZVAL_W2 res_addr, 0x41e00000 13694 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13695 } 13696 |.endif 13697 } else { 13698 |.if X64 13699 | mov64 rax, 0xc3e0000000000000 13700 | SET_ZVAL_LVAL var_addr, rax 13701 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13702 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13703 | SET_ZVAL_LVAL res_addr, rax 13704 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13705 } 13706 |.else 13707 | SET_ZVAL_LVAL var_addr, 0x00200000 13708 | SET_ZVAL_W2 var_addr, 0xc1e00000 13709 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13710 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13711 | SET_ZVAL_LVAL res_addr, 0x00200000 13712 | SET_ZVAL_W2 res_addr, 0xc1e00000 13713 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13714 } 13715 |.endif 13716 } 13717 if (opline->result_type != IS_UNUSED 13718 && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) 13719 && prop_info 13720 && !ZEND_TYPE_IS_SET(prop_info->type) 13721 && (res_info & MAY_BE_GUARD) 13722 && (res_info & MAY_BE_LONG)) { 13723 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 13724 uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 13725 int32_t exit_point; 13726 const void *exit_addr; 13727 13728 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 13729 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 13730 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13731 if (!exit_addr) { 13732 return 0; 13733 } 13734 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 13735 ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD; 13736 | jmp &exit_addr 13737 |.code 13738 } else { 13739 | jmp >4 13740 |.code 13741 |4: 13742 } 13743 } 13744 } 13745 13746 if (needs_slow_path) { 13747 may_throw = 1; 13748 |.cold_code 13749 |7: 13750 | SET_EX_OPLINE opline, r0 13751 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13752 | LOAD_ADDR FCARG2a, name 13753 |.if X64 13754 | mov CARG3, EX->run_time_cache 13755 | add CARG3, opline->extended_value 13756 if (opline->result_type == IS_UNUSED) { 13757 | xor CARG4, CARG4 13758 } else { 13759 | LOAD_ZVAL_ADDR CARG4, res_addr 13760 } 13761 |.else 13762 | sub r4, 8 13763 if (opline->result_type == IS_UNUSED) { 13764 | push 0 13765 } else { 13766 | PUSH_ZVAL_ADDR res_addr, r0 13767 } 13768 | mov r0, EX->run_time_cache 13769 | add r0, opline->extended_value 13770 | push r0 13771 |.endif 13772 13773 switch (opline->opcode) { 13774 case ZEND_PRE_INC_OBJ: 13775 | EXT_CALL zend_jit_pre_inc_obj_helper, r0 13776 break; 13777 case ZEND_PRE_DEC_OBJ: 13778 | EXT_CALL zend_jit_pre_dec_obj_helper, r0 13779 break; 13780 case ZEND_POST_INC_OBJ: 13781 | EXT_CALL zend_jit_post_inc_obj_helper, r0 13782 break; 13783 case ZEND_POST_DEC_OBJ: 13784 | EXT_CALL zend_jit_post_dec_obj_helper, r0 13785 break; 13786 default: 13787 ZEND_UNREACHABLE(); 13788 } 13789 13790 |.if not(X64) 13791 | add r4, 8 13792 |.endif 13793 13794 | jmp >9 13795 |.code 13796 } 13797 13798 |9: 13799 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13800 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 13801 may_throw = 1; 13802 } 13803 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 13804 } 13805 13806 if (may_throw) { 13807 if (!zend_jit_check_exception(Dst)) { 13808 return 0; 13809 } 13810 } 13811 13812 return 1; 13813} 13814 13815static int zend_jit_assign_obj_op(dasm_State **Dst, 13816 const zend_op *opline, 13817 const zend_op_array *op_array, 13818 zend_ssa *ssa, 13819 const zend_ssa_op *ssa_op, 13820 uint32_t op1_info, 13821 zend_jit_addr op1_addr, 13822 uint32_t val_info, 13823 zend_ssa_range *val_range, 13824 bool op1_indirect, 13825 zend_class_entry *ce, 13826 bool ce_is_instanceof, 13827 bool on_this, 13828 bool delayed_fetch_this, 13829 zend_class_entry *trace_ce, 13830 uint8_t prop_type) 13831{ 13832 zval *member; 13833 zend_string *name; 13834 zend_property_info *prop_info; 13835 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13836 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13837 zend_jit_addr prop_addr; 13838 bool needs_slow_path = 0; 13839 bool use_prop_guard = 0; 13840 bool may_throw = 0; 13841 binary_op_type binary_op = get_binary_op(opline->extended_value); 13842 13843 ZEND_ASSERT(opline->op2_type == IS_CONST); 13844 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13845 ZEND_ASSERT(opline->result_type == IS_UNUSED); 13846 13847 member = RT_CONSTANT(opline, opline->op2); 13848 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13849 name = Z_STR_P(member); 13850 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13851 13852 if (on_this) { 13853 | GET_ZVAL_PTR FCARG1a, this_addr 13854 } else { 13855 if (opline->op1_type == IS_VAR 13856 && (op1_info & MAY_BE_INDIRECT) 13857 && Z_REG(op1_addr) == ZREG_FP) { 13858 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13859 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13860 | GET_Z_PTR FCARG1a, FCARG1a 13861 |1: 13862 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13863 } 13864 if (op1_info & MAY_BE_REF) { 13865 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13866 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13867 } 13868 | ZVAL_DEREF FCARG1a, op1_info 13869 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13870 } 13871 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13872 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13873 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13874 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13875 13876 if (!exit_addr) { 13877 return 0; 13878 } 13879 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13880 } else { 13881 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 13882 |.cold_code 13883 |1: 13884 | SET_EX_OPLINE opline, r0 13885 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13886 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13887 } 13888 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 13889 if (op1_info & MAY_BE_UNDEF) { 13890 | EXT_CALL zend_jit_invalid_property_assign_op, r0 13891 } else { 13892 | EXT_CALL zend_jit_invalid_property_assign, r0 13893 } 13894 may_throw = 1; 13895 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13896 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13897 may_throw = 1; 13898 | jmp >8 13899 } else { 13900 | jmp >9 13901 } 13902 |.code 13903 } 13904 } 13905 | GET_ZVAL_PTR FCARG1a, op1_addr 13906 } 13907 13908 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13909 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13910 if (prop_info) { 13911 ce = trace_ce; 13912 ce_is_instanceof = 0; 13913 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13914 if (on_this && JIT_G(current_frame) 13915 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13916 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13917 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13918 if (on_this && JIT_G(current_frame)) { 13919 JIT_G(current_frame)->ce = ce; 13920 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13921 } 13922 } else { 13923 return 0; 13924 } 13925 if (ssa->var_info && ssa_op->op1_use >= 0) { 13926 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13927 ssa->var_info[ssa_op->op1_use].ce = ce; 13928 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13929 } 13930 if (ssa->var_info && ssa_op->op1_def >= 0) { 13931 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13932 ssa->var_info[ssa_op->op1_def].ce = ce; 13933 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13934 } 13935 } 13936 } 13937 } 13938 13939 use_prop_guard = (prop_type != IS_UNKNOWN 13940 && prop_type != IS_UNDEF 13941 && prop_type != IS_REFERENCE 13942 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 13943 13944 if (!prop_info) { 13945 needs_slow_path = 1; 13946 13947 | mov r0, EX->run_time_cache 13948 | mov r2, aword [r0 + (opline+1)->extended_value] 13949 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13950 | jne >7 13951 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 13952 | cmp aword [r0 + (opline+1)->extended_value + sizeof(void*) * 2], 0 13953 | jnz >7 13954 } 13955 | mov r0, aword [r0 + (opline+1)->extended_value + sizeof(void*)] 13956 | test r0, r0 13957 | jl >7 13958 if (!use_prop_guard) { 13959 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 13960 } 13961 | add FCARG1a, r0 13962 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13963 } else { 13964 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13965 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 13966 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13967 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13968 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13969 13970 if (!exit_addr) { 13971 return 0; 13972 } 13973 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 13974 } else { 13975 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 13976 needs_slow_path = 1; 13977 } 13978 } 13979 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13980 uint32_t info = val_info; 13981 13982 may_throw = 1; 13983 13984 if (opline) { 13985 | SET_EX_OPLINE opline, r0 13986 } 13987 13988 | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1 13989 |.cold_code 13990 |1: 13991 | GET_ZVAL_PTR FCARG1a, prop_addr 13992 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 13993 | LOAD_ZVAL_ADDR FCARG2a, val_addr 13994 } 13995 |.if X64 13996 | LOAD_ADDR CARG3, binary_op 13997 |.else 13998 | sub r4, 12 13999 | PUSH_ADDR binary_op, r0 14000 |.endif 14001 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 14002 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14003 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 14004 } else { 14005 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 14006 } 14007 |.if not(X64) 14008 | add r4, 12 14009 |.endif 14010 | jmp >9 14011 |.code 14012 14013 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14014 14015 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 14016 | LOAD_ADDR FCARG2a, prop_info 14017 } else { 14018 int prop_info_offset = 14019 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 14020 14021 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 14022 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 14023 | mov FCARG2a, aword[r0 + prop_info_offset] 14024 } 14025 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 14026 |.if X64 14027 | LOAD_ZVAL_ADDR CARG3, val_addr 14028 | LOAD_ADDR CARG4, binary_op 14029 |.else 14030 | sub r4, 8 14031 | PUSH_ADDR binary_op, r0 14032 | PUSH_ZVAL_ADDR val_addr, r0 14033 |.endif 14034 14035 | EXT_CALL zend_jit_assign_op_to_typed_prop, r0 14036 14037 |.if not(X64) 14038 | add r4, 8 14039 |.endif 14040 14041 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14042 info |= MAY_BE_RC1|MAY_BE_RCN; 14043 } 14044 14045 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL 14046 } 14047 } 14048 14049 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 14050 zend_jit_addr var_addr = prop_addr; 14051 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 14052 uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 14053 14054 if (use_prop_guard) { 14055 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14056 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14057 if (!exit_addr) { 14058 return 0; 14059 } 14060 14061 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr 14062 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 14063 } 14064 14065 if (var_info & MAY_BE_REF) { 14066 may_throw = 1; 14067 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 14068 | LOAD_ZVAL_ADDR r0, prop_addr 14069 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 14070 | GET_ZVAL_PTR FCARG1a, var_addr 14071 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 14072 | jnz >1 14073 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] 14074 |.cold_code 14075 |1: 14076 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 14077 | LOAD_ZVAL_ADDR FCARG2a, val_addr 14078 } 14079 if (opline) { 14080 | SET_EX_OPLINE opline, r0 14081 } 14082 |.if X64 14083 | LOAD_ADDR CARG3, binary_op 14084 |.else 14085 | sub r4, 12 14086 | PUSH_ADDR binary_op, r0 14087 |.endif 14088 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 14089 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14090 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 14091 } else { 14092 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 14093 } 14094 |.if not(X64) 14095 | add r4, 12 14096 |.endif 14097 | jmp >9 14098 |.code 14099 |2: 14100 var_info &= ~MAY_BE_REF; 14101 } 14102 14103 switch (opline->extended_value) { 14104 case ZEND_ADD: 14105 case ZEND_SUB: 14106 case ZEND_MUL: 14107 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14108 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14109 if (opline->extended_value != ZEND_ADD || 14110 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 14111 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 14112 may_throw = 1; 14113 } 14114 } 14115 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, 14116 1 /* may overflow */, 0)) { 14117 return 0; 14118 } 14119 break; 14120 case ZEND_BW_OR: 14121 case ZEND_BW_AND: 14122 case ZEND_BW_XOR: 14123 may_throw = 1; 14124 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14125 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14126 if ((var_info & MAY_BE_ANY) != MAY_BE_STRING || 14127 (val_info & MAY_BE_ANY) != MAY_BE_STRING) { 14128 may_throw = 1; 14129 } 14130 } 14131 goto long_math; 14132 case ZEND_SL: 14133 case ZEND_SR: 14134 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14135 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14136 may_throw = 1; 14137 } 14138 if ((opline+1)->op1_type != IS_CONST || 14139 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 14140 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) { 14141 may_throw = 1; 14142 } 14143 goto long_math; 14144 case ZEND_MOD: 14145 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14146 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14147 if (opline->extended_value != ZEND_ADD || 14148 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 14149 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 14150 may_throw = 1; 14151 } 14152 } 14153 if ((opline+1)->op1_type != IS_CONST || 14154 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 14155 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) { 14156 may_throw = 1; 14157 } 14158long_math: 14159 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 14160 IS_CV, opline->op1, var_addr, var_info, NULL, 14161 (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 14162 val_range, 14163 0, var_addr, var_def_info, var_info, /* may throw */ 1)) { 14164 return 0; 14165 } 14166 break; 14167 case ZEND_CONCAT: 14168 may_throw = 1; 14169 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, 14170 0)) { 14171 return 0; 14172 } 14173 break; 14174 default: 14175 ZEND_UNREACHABLE(); 14176 } 14177 } 14178 14179 if (needs_slow_path) { 14180 may_throw = 1; 14181 |.cold_code 14182 |7: 14183 | SET_EX_OPLINE opline, r0 14184 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 14185 | LOAD_ADDR FCARG2a, name 14186 |.if X64 14187 | LOAD_ZVAL_ADDR CARG3, val_addr 14188 | mov CARG4, EX->run_time_cache 14189 | add CARG4, (opline+1)->extended_value 14190 |.if X64WIN 14191 | LOAD_ADDR r0, binary_op 14192 | mov aword A5, r0 14193 |.else 14194 | LOAD_ADDR CARG5, binary_op 14195 |.endif 14196 |.else 14197 | sub r4, 4 14198 | PUSH_ADDR binary_op, r0 14199 | mov r0, EX->run_time_cache 14200 | add r0, (opline+1)->extended_value 14201 | push r0 14202 | PUSH_ZVAL_ADDR val_addr, r0 14203 |.endif 14204 14205 | EXT_CALL zend_jit_assign_obj_op_helper, r0 14206 14207 |.if not(X64) 14208 | add r4, 4 14209 |.endif 14210 14211 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14212 val_info |= MAY_BE_RC1|MAY_BE_RCN; 14213 } 14214 14215 |8: 14216 | // FREE_OP_DATA(); 14217 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14218 | jmp >9 14219 |.code 14220 } 14221 14222 |9: 14223 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 14224 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 14225 may_throw = 1; 14226 } 14227 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 14228 } 14229 14230 if (may_throw) { 14231 if (!zend_jit_check_exception(Dst)) { 14232 return 0; 14233 } 14234 } 14235 14236 return 1; 14237} 14238 14239static int zend_jit_assign_obj(dasm_State **Dst, 14240 const zend_op *opline, 14241 const zend_op_array *op_array, 14242 zend_ssa *ssa, 14243 const zend_ssa_op *ssa_op, 14244 uint32_t op1_info, 14245 zend_jit_addr op1_addr, 14246 uint32_t val_info, 14247 bool op1_indirect, 14248 zend_class_entry *ce, 14249 bool ce_is_instanceof, 14250 bool on_this, 14251 bool delayed_fetch_this, 14252 zend_class_entry *trace_ce, 14253 uint8_t prop_type, 14254 int may_throw) 14255{ 14256 zval *member; 14257 zend_string *name; 14258 zend_property_info *prop_info; 14259 zend_jit_addr val_addr = OP1_DATA_ADDR(); 14260 zend_jit_addr res_addr = 0; 14261 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 14262 zend_jit_addr prop_addr; 14263 bool needs_slow_path = 0; 14264 bool needs_val_dtor = 0; 14265 14266 if (RETURN_VALUE_USED(opline)) { 14267 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14268 } 14269 14270 ZEND_ASSERT(opline->op2_type == IS_CONST); 14271 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 14272 14273 member = RT_CONSTANT(opline, opline->op2); 14274 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 14275 name = Z_STR_P(member); 14276 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 14277 14278 if (on_this) { 14279 | GET_ZVAL_PTR FCARG1a, this_addr 14280 } else { 14281 if (opline->op1_type == IS_VAR 14282 && (op1_info & MAY_BE_INDIRECT) 14283 && Z_REG(op1_addr) == ZREG_FP) { 14284 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14285 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 14286 | GET_Z_PTR FCARG1a, FCARG1a 14287 |1: 14288 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14289 } 14290 if (op1_info & MAY_BE_REF) { 14291 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14292 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14293 } 14294 | ZVAL_DEREF FCARG1a, op1_info 14295 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14296 } 14297 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 14298 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14299 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14300 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14301 14302 if (!exit_addr) { 14303 return 0; 14304 } 14305 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 14306 } else { 14307 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 14308 |.cold_code 14309 |1: 14310 | SET_EX_OPLINE opline, r0 14311 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14312 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14313 } 14314 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 14315 | EXT_CALL zend_jit_invalid_property_assign, r0 14316 if (RETURN_VALUE_USED(opline)) { 14317 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 14318 } 14319 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 14320 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14321 needs_val_dtor = 1; 14322 | jmp >7 14323 } else { 14324 | jmp >9 14325 } 14326 |.code 14327 } 14328 } 14329 | GET_ZVAL_PTR FCARG1a, op1_addr 14330 } 14331 14332 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 14333 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 14334 if (prop_info) { 14335 ce = trace_ce; 14336 ce_is_instanceof = 0; 14337 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 14338 if (on_this && JIT_G(current_frame) 14339 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 14340 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 14341 } else if (zend_jit_class_guard(Dst, opline, ce)) { 14342 if (on_this && JIT_G(current_frame)) { 14343 JIT_G(current_frame)->ce = ce; 14344 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 14345 } 14346 } else { 14347 return 0; 14348 } 14349 if (ssa->var_info && ssa_op->op1_use >= 0) { 14350 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 14351 ssa->var_info[ssa_op->op1_use].ce = ce; 14352 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 14353 } 14354 if (ssa->var_info && ssa_op->op1_def >= 0) { 14355 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 14356 ssa->var_info[ssa_op->op1_def].ce = ce; 14357 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 14358 } 14359 } 14360 } 14361 } 14362 14363 if (!prop_info) { 14364 needs_slow_path = 1; 14365 14366 | mov r0, EX->run_time_cache 14367 | mov r2, aword [r0 + opline->extended_value] 14368 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 14369 | jne >5 14370 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 14371 | mov FCARG2a, aword [r0 + opline->extended_value + sizeof(void*) * 2] 14372 } 14373 | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] 14374 | test r0, r0 14375 | jl >5 14376 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >5 14377 | add FCARG1a, r0 14378 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14379 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 14380 | test FCARG2a, FCARG2a 14381 | jnz >1 14382 |.cold_code 14383 |1: 14384 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14385 | SET_EX_OPLINE opline, r0 14386 |.if X64 14387 | LOAD_ZVAL_ADDR CARG3, val_addr 14388 if (RETURN_VALUE_USED(opline)) { 14389 | LOAD_ZVAL_ADDR CARG4, res_addr 14390 } else { 14391 | xor CARG4, CARG4 14392 } 14393 |.else 14394 | sub r4, 8 14395 if (RETURN_VALUE_USED(opline)) { 14396 | PUSH_ZVAL_ADDR res_addr, r0 14397 } else { 14398 | push 0 14399 } 14400 | PUSH_ZVAL_ADDR val_addr, r0 14401 |.endif 14402 14403 | EXT_CALL zend_jit_assign_to_typed_prop, r0 14404 14405 |.if not(X64) 14406 | add r4, 8 14407 |.endif 14408 14409 if ((opline+1)->op1_type == IS_CONST) { 14410 | // TODO: ??? 14411 | // if (Z_TYPE_P(value) == orig_type) { 14412 | // CACHE_PTR_EX(cache_slot + 2, NULL); 14413 } 14414 14415 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 14416 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14417 | jmp >7 14418 } else { 14419 | jmp >9 14420 } 14421 |.code 14422 } 14423 } else { 14424 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 14425 if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) { 14426 // Undefined property with magic __get()/__set() 14427 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14428 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14429 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14430 14431 if (!exit_addr) { 14432 return 0; 14433 } 14434 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 14435 } else { 14436 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >5 14437 needs_slow_path = 1; 14438 } 14439 } 14440 if (ZEND_TYPE_IS_SET(prop_info->type)) { 14441 uint32_t info = val_info; 14442 14443 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14444 | SET_EX_OPLINE opline, r0 14445 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 14446 | LOAD_ADDR FCARG2a, prop_info 14447 } else { 14448 int prop_info_offset = 14449 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 14450 14451 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 14452 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 14453 | mov FCARG2a, aword[r0 + prop_info_offset] 14454 } 14455 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 14456 |.if X64 14457 | LOAD_ZVAL_ADDR CARG3, val_addr 14458 if (RETURN_VALUE_USED(opline)) { 14459 | LOAD_ZVAL_ADDR CARG4, res_addr 14460 } else { 14461 | xor CARG4, CARG4 14462 } 14463 |.else 14464 | sub r4, 8 14465 if (RETURN_VALUE_USED(opline)) { 14466 | PUSH_ZVAL_ADDR res_addr, r0 14467 } else { 14468 | push 0 14469 } 14470 | PUSH_ZVAL_ADDR val_addr, r0 14471 |.endif 14472 14473 | EXT_CALL zend_jit_assign_to_typed_prop, r0 14474 14475 |.if not(X64) 14476 | add r4, 8 14477 |.endif 14478 14479 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14480 info |= MAY_BE_RC1|MAY_BE_RCN; 14481 } 14482 14483 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL 14484 } 14485 } 14486 14487 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 14488 // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); 14489 if (opline->result_type == IS_UNUSED) { 14490 if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 14491 return 0; 14492 } 14493 } else { 14494 if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 14495 return 0; 14496 } 14497 } 14498 } 14499 14500 if (needs_slow_path) { 14501 |.cold_code 14502 |5: 14503 | SET_EX_OPLINE opline, r0 14504 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 14505 | LOAD_ADDR FCARG2a, name 14506 |.if X64 14507 | LOAD_ZVAL_ADDR CARG3, val_addr 14508 | mov CARG4, EX->run_time_cache 14509 | add CARG4, opline->extended_value 14510 if (RETURN_VALUE_USED(opline)) { 14511 |.if X64WIN 14512 | LOAD_ZVAL_ADDR r0, res_addr 14513 | mov aword A5, r0 14514 |.else 14515 | LOAD_ZVAL_ADDR CARG5, res_addr 14516 |.endif 14517 } else { 14518 |.if X64WIN 14519 | mov aword A5, 0 14520 |.else 14521 | xor CARG5, CARG5 14522 |.endif 14523 } 14524 |.else 14525 | sub r4, 4 14526 if (RETURN_VALUE_USED(opline)) { 14527 | PUSH_ZVAL_ADDR res_addr, r0 14528 } else { 14529 | push 0 14530 } 14531 | mov r0, EX->run_time_cache 14532 | add r0, opline->extended_value 14533 | push r0 14534 | PUSH_ZVAL_ADDR val_addr, r0 14535 |.endif 14536 14537 | EXT_CALL zend_jit_assign_obj_helper, r0 14538 14539 |.if not(X64) 14540 | add r4, 4 14541 |.endif 14542 14543 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14544 val_info |= MAY_BE_RC1|MAY_BE_RCN; 14545 } 14546 14547 |7: 14548 | // FREE_OP_DATA(); 14549 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14550 | jmp >9 14551 |.code 14552 } else if (needs_val_dtor) { 14553 |.cold_code 14554 |7: 14555 | // FREE_OP_DATA(); 14556 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14557 | jmp >9 14558 |.code 14559 } 14560 14561 |9: 14562 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 14563 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 14564 } 14565 14566 if (may_throw) { 14567 if (!zend_jit_check_exception(Dst)) { 14568 return 0; 14569 } 14570 } 14571 14572 return 1; 14573} 14574 14575static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) 14576{ 14577 zend_jit_addr op1_addr = OP1_ADDR(); 14578 14579 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 14580 if (may_throw) { 14581 | SET_EX_OPLINE opline, r0 14582 } 14583 if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { 14584 if (op1_info & MAY_BE_ARRAY) { 14585 | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7 14586 } 14587 | mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_iter_idx)] 14588 | cmp FCARG1d, -1 14589 | je >7 14590 | EXT_CALL zend_hash_iterator_del, r0 14591 |7: 14592 } 14593 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline 14594 if (may_throw) { 14595 if (!zend_jit_check_exception(Dst)) { 14596 return 0; 14597 } 14598 } 14599 } 14600 14601 return 1; 14602} 14603 14604static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 14605{ 14606 if (opline->op1_type == IS_CONST) { 14607 zval *zv; 14608 size_t len; 14609 14610 zv = RT_CONSTANT(opline, opline->op1); 14611 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14612 len = Z_STRLEN_P(zv); 14613 14614 if (len > 0) { 14615 const char *str = Z_STRVAL_P(zv); 14616 14617 | SET_EX_OPLINE opline, r0 14618 |.if X64 14619 | LOAD_ADDR CARG1, str 14620 | LOAD_ADDR CARG2, len 14621 | EXT_CALL zend_write, r0 14622 |.else 14623 | mov aword A2, len 14624 | mov aword A1, str 14625 | EXT_CALL zend_write, r0 14626 |.endif 14627 if (!zend_jit_check_exception(Dst)) { 14628 return 0; 14629 } 14630 } 14631 } else { 14632 zend_jit_addr op1_addr = OP1_ADDR(); 14633 14634 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14635 14636 | SET_EX_OPLINE opline, r0 14637 | GET_ZVAL_PTR r0, op1_addr 14638 |.if X64 14639 | lea CARG1, aword [r0 + offsetof(zend_string, val)] 14640 | mov CARG2, aword [r0 + offsetof(zend_string, len)] 14641 | EXT_CALL zend_write, r0 14642 |.else 14643 | add r0, offsetof(zend_string, val) 14644 | mov aword A1, r0 14645 | mov r0, aword [r0 + (offsetof(zend_string, len)-offsetof(zend_string, val))] 14646 | mov aword A2, r0 14647 | EXT_CALL zend_write, r0 14648 |.endif 14649 if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { 14650 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline 14651 } 14652 if (!zend_jit_check_exception(Dst)) { 14653 return 0; 14654 } 14655 } 14656 return 1; 14657} 14658 14659static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) 14660{ 14661 if (opline->op1_type == IS_CONST) { 14662 zval *zv; 14663 size_t len; 14664 14665 zv = RT_CONSTANT(opline, opline->op1); 14666 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14667 len = Z_STRLEN_P(zv); 14668 14669 | SET_ZVAL_LVAL res_addr, len 14670 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 14671 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14672 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14673 return 0; 14674 } 14675 } else { 14676 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14677 14678 if (Z_MODE(res_addr) == IS_REG) { 14679 | GET_ZVAL_PTR Ra(Z_REG(res_addr)), op1_addr 14680 | mov Ra(Z_REG(res_addr)), aword [Ra(Z_REG(res_addr))+offsetof(zend_string, len)] 14681 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14682 return 0; 14683 } 14684 } else { 14685 | GET_ZVAL_PTR r0, op1_addr 14686 | mov r0, aword [r0 + offsetof(zend_string, len)] 14687 | SET_ZVAL_LVAL res_addr, r0 14688 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14689 } 14690 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 14691 } 14692 return 1; 14693} 14694 14695static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) 14696{ 14697 if (opline->op1_type == IS_CONST) { 14698 zval *zv; 14699 zend_long count; 14700 14701 zv = RT_CONSTANT(opline, opline->op1); 14702 ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); 14703 count = zend_hash_num_elements(Z_ARRVAL_P(zv)); 14704 14705 | SET_ZVAL_LVAL res_addr, count 14706 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 14707 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14708 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14709 return 0; 14710 } 14711 } else { 14712 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY); 14713 // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+. 14714 14715 if (Z_MODE(res_addr) == IS_REG) { 14716 | GET_ZVAL_PTR Ra(Z_REG(res_addr)), op1_addr 14717 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 14718 | mov Rd(Z_REG(res_addr)), dword [Ra(Z_REG(res_addr))+offsetof(HashTable, nNumOfElements)] 14719 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14720 return 0; 14721 } 14722 } else { 14723 | GET_ZVAL_PTR r0, op1_addr 14724 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 14725 | mov eax, dword [r0 + offsetof(HashTable, nNumOfElements)] 14726 | SET_ZVAL_LVAL res_addr, r0 14727 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14728 } 14729 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 14730 } 14731 14732 if (may_throw) { 14733 return zend_jit_check_exception(Dst); 14734 } 14735 return 1; 14736} 14737 14738static int zend_jit_load_this(dasm_State **Dst, uint32_t var) 14739{ 14740 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 14741 14742 | mov FCARG1a, aword EX->This.value.ptr 14743 | SET_ZVAL_PTR var_addr, FCARG1a 14744 | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX 14745 | GC_ADDREF FCARG1a 14746 14747 return 1; 14748} 14749 14750static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only) 14751{ 14752 if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) { 14753 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14754 if (!JIT_G(current_frame) || 14755 !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { 14756 14757 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14758 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14759 14760 if (!exit_addr) { 14761 return 0; 14762 } 14763 14764 | cmp byte EX->This.u1.v.type, IS_OBJECT 14765 | jne &exit_addr 14766 14767 if (JIT_G(current_frame)) { 14768 TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); 14769 } 14770 } 14771 } else { 14772 14773 | cmp byte EX->This.u1.v.type, IS_OBJECT 14774 | jne >1 14775 |.cold_code 14776 |1: 14777 | SET_EX_OPLINE opline, r0 14778 | jmp ->invalid_this 14779 |.code 14780 } 14781 } 14782 14783 if (!check_only) { 14784 if (!zend_jit_load_this(Dst, opline->result.var)) { 14785 return 0; 14786 } 14787 } 14788 14789 return 1; 14790} 14791 14792static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) 14793{ 14794 uint32_t count; 14795 Bucket *p; 14796 const zend_op *target; 14797 int b; 14798 int32_t exit_point; 14799 const void *exit_addr; 14800 14801 | test r0, r0 14802 if (default_label) { 14803 | jz &default_label 14804 } else if (next_opline) { 14805 | jz >3 14806 } else { 14807 | jz =>default_b 14808 } 14809 | LOAD_ADDR FCARG1a, jumptable 14810 | sub r0, aword [FCARG1a + offsetof(HashTable, arData)] 14811 if (HT_IS_PACKED(jumptable)) { 14812 | mov FCARG1a, (sizeof(zval) / sizeof(void*)) 14813 } else { 14814 | mov FCARG1a, (sizeof(Bucket) / sizeof(void*)) 14815 } 14816 |.if X64 14817 | cqo 14818 |.else 14819 | cdq 14820 |.endif 14821 | idiv FCARG1a 14822 |.if X64 14823 if (!IS_32BIT(dasm_end)) { 14824 | lea FCARG1a, aword [>4] 14825 | jmp aword [FCARG1a + r0] 14826 } else { 14827 | jmp aword [r0 + >4] 14828 } 14829 |.else 14830 | jmp aword [r0 + >4] 14831 |.endif 14832 |.jmp_table 14833 |.align aword 14834 |4: 14835 if (trace_info) { 14836 trace_info->jmp_table_size += zend_hash_num_elements(jumptable); 14837 } 14838 14839 count = jumptable->nNumUsed; 14840 p = jumptable->arData; 14841 do { 14842 if (Z_TYPE(p->val) == IS_UNDEF) { 14843 if (default_label) { 14844 | .aword &default_label 14845 } else if (next_opline) { 14846 | .aword >3 14847 } else { 14848 | .aword =>default_b 14849 } 14850 } else { 14851 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 14852 if (!next_opline) { 14853 b = ssa->cfg.map[target - op_array->opcodes]; 14854 | .aword =>b 14855 } else if (next_opline == target) { 14856 | .aword >3 14857 } else { 14858 exit_point = zend_jit_trace_get_exit_point(target, 0); 14859 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14860 if (!exit_addr) { 14861 return 0; 14862 } 14863 | .aword &exit_addr 14864 } 14865 } 14866 if (HT_IS_PACKED(jumptable)) { 14867 p = (Bucket*)(((zval*)p)+1); 14868 } else { 14869 p++; 14870 } 14871 count--; 14872 } while (count); 14873 |.code 14874 14875 return 1; 14876} 14877 14878static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) 14879{ 14880 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 14881 const zend_op *next_opline = NULL; 14882 14883 if (trace) { 14884 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 14885 ZEND_ASSERT(trace->opline != NULL); 14886 next_opline = trace->opline; 14887 } 14888 14889 if (opline->op1_type == IS_CONST) { 14890 zval *zv = RT_CONSTANT(opline, opline->op1); 14891 zval *jump_zv = NULL; 14892 int b; 14893 14894 if (opline->opcode == ZEND_SWITCH_LONG) { 14895 if (Z_TYPE_P(zv) == IS_LONG) { 14896 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14897 } 14898 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14899 if (Z_TYPE_P(zv) == IS_STRING) { 14900 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 14901 } 14902 } else if (opline->opcode == ZEND_MATCH) { 14903 if (Z_TYPE_P(zv) == IS_LONG) { 14904 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14905 } else if (Z_TYPE_P(zv) == IS_STRING) { 14906 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 14907 } 14908 } else { 14909 ZEND_UNREACHABLE(); 14910 } 14911 if (next_opline) { 14912 const zend_op *target; 14913 14914 if (jump_zv != NULL) { 14915 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); 14916 } else { 14917 target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14918 } 14919 ZEND_ASSERT(target == next_opline); 14920 } else { 14921 if (jump_zv != NULL) { 14922 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; 14923 } else { 14924 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; 14925 } 14926 | jmp =>b 14927 } 14928 } else { 14929 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; 14930 uint32_t op1_info = OP1_INFO(); 14931 zend_jit_addr op1_addr = OP1_ADDR(); 14932 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14933 const zend_op *target; 14934 int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; 14935 int b; 14936 int32_t exit_point; 14937 const void *fallback_label = NULL; 14938 const void *default_label = NULL; 14939 const void *exit_addr; 14940 14941 if (next_opline) { 14942 if (next_opline != opline + 1) { 14943 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 14944 fallback_label = zend_jit_trace_get_exit_addr(exit_point); 14945 if (!fallback_label) { 14946 return 0; 14947 } 14948 } 14949 if (next_opline != default_opline) { 14950 exit_point = zend_jit_trace_get_exit_point(default_opline, 0); 14951 default_label = zend_jit_trace_get_exit_addr(exit_point); 14952 if (!default_label) { 14953 return 0; 14954 } 14955 } 14956 } 14957 14958 if (opline->opcode == ZEND_SWITCH_LONG) { 14959 if (op1_info & MAY_BE_LONG) { 14960 if (op1_info & MAY_BE_REF) { 14961 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1 14962 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr 14963 |.cold_code 14964 |1: 14965 | // ZVAL_DEREF(op) 14966 if (fallback_label) { 14967 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label 14968 } else { 14969 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 14970 } 14971 | GET_ZVAL_PTR FCARG2a, op1_addr 14972 if (fallback_label) { 14973 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, &fallback_label 14974 } else { 14975 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3 14976 } 14977 | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.lval)] 14978 | jmp >2 14979 |.code 14980 |2: 14981 } else { 14982 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14983 if (fallback_label) { 14984 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label 14985 } else { 14986 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 14987 } 14988 } 14989 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr 14990 } 14991 if (HT_IS_PACKED(jumptable)) { 14992 uint32_t count = jumptable->nNumUsed; 14993 zval *zv = jumptable->arPacked; 14994 14995 | cmp FCARG2a, jumptable->nNumUsed 14996 if (default_label) { 14997 | jae &default_label 14998 } else if (next_opline) { 14999 | jae >3 15000 } else { 15001 | jae =>default_b 15002 } 15003 |.if X64 15004 if (!IS_32BIT(dasm_end)) { 15005 | lea r0, aword [>4] 15006 | jmp aword [r0 + FCARG2a * 8] 15007 } else { 15008 | jmp aword [FCARG2a * 8 + >4] 15009 } 15010 |.else 15011 | jmp aword [FCARG2a * 4 + >4] 15012 |.endif 15013 |.jmp_table 15014 |.align aword 15015 |4: 15016 if (trace_info) { 15017 trace_info->jmp_table_size += count; 15018 } 15019 do { 15020 if (Z_TYPE_P(zv) == IS_UNDEF) { 15021 if (default_label) { 15022 | .aword &default_label 15023 } else if (next_opline) { 15024 | .aword >3 15025 } else { 15026 | .aword =>default_b 15027 } 15028 } else { 15029 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); 15030 if (!next_opline) { 15031 b = ssa->cfg.map[target - op_array->opcodes]; 15032 | .aword =>b 15033 } else if (next_opline == target) { 15034 | .aword >3 15035 } else { 15036 exit_point = zend_jit_trace_get_exit_point(target, 0); 15037 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15038 if (!exit_addr) { 15039 return 0; 15040 } 15041 | .aword &exit_addr 15042 } 15043 } 15044 zv++; 15045 count--; 15046 } while (count); 15047 |.code 15048 |3: 15049 } else { 15050 | LOAD_ADDR FCARG1a, jumptable 15051 | EXT_CALL zend_hash_index_find, r0 15052 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 15053 return 0; 15054 } 15055 |3: 15056 } 15057 } 15058 } else if (opline->opcode == ZEND_SWITCH_STRING) { 15059 if (op1_info & MAY_BE_STRING) { 15060 if (op1_info & MAY_BE_REF) { 15061 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1 15062 | GET_ZVAL_PTR FCARG2a, op1_addr 15063 |.cold_code 15064 |1: 15065 | // ZVAL_DEREF(op) 15066 if (fallback_label) { 15067 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label 15068 } else { 15069 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 15070 } 15071 | GET_ZVAL_PTR FCARG2a, op1_addr 15072 if (fallback_label) { 15073 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, &fallback_label 15074 } else { 15075 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3 15076 } 15077 | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.ptr)] 15078 | jmp >2 15079 |.code 15080 |2: 15081 } else { 15082 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { 15083 if (fallback_label) { 15084 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label 15085 } else { 15086 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 15087 } 15088 } 15089 | GET_ZVAL_PTR FCARG2a, op1_addr 15090 } 15091 | LOAD_ADDR FCARG1a, jumptable 15092 | EXT_CALL zend_hash_find, r0 15093 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 15094 return 0; 15095 } 15096 |3: 15097 } 15098 } else if (opline->opcode == ZEND_MATCH) { 15099 if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { 15100 if (op1_info & MAY_BE_REF) { 15101 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 15102 | ZVAL_DEREF FCARG2a, op1_info 15103 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 15104 } 15105 | LOAD_ADDR FCARG1a, jumptable 15106 if (op1_info & MAY_BE_LONG) { 15107 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 15108 if (op1_info & MAY_BE_STRING) { 15109 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5 15110 } else if (op1_info & MAY_BE_UNDEF) { 15111 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 15112 } else if (default_label) { 15113 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label 15114 } else if (next_opline) { 15115 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 15116 } else { 15117 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b 15118 } 15119 } 15120 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr 15121 | EXT_CALL zend_hash_index_find, r0 15122 if (op1_info & MAY_BE_STRING) { 15123 | jmp >2 15124 } 15125 } 15126 if (op1_info & MAY_BE_STRING) { 15127 |5: 15128 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { 15129 if (op1_info & MAY_BE_UNDEF) { 15130 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 15131 } else if (default_label) { 15132 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label 15133 } else if (next_opline) { 15134 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 15135 } else { 15136 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b 15137 } 15138 } 15139 | GET_ZVAL_PTR FCARG2a, op1_addr 15140 | EXT_CALL zend_hash_find, r0 15141 } 15142 |2: 15143 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 15144 return 0; 15145 } 15146 } 15147 if (op1_info & MAY_BE_UNDEF) { 15148 |6: 15149 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { 15150 if (default_label) { 15151 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label 15152 } else if (next_opline) { 15153 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3 15154 } else { 15155 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b 15156 } 15157 } 15158 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 15159 | SET_EX_OPLINE opline, r0 15160 | mov FCARG1d, opline->op1.var 15161 | EXT_CALL zend_jit_undefined_op_helper, r0 15162 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 15163 return 0; 15164 } 15165 } 15166 if (default_label) { 15167 | jmp &default_label 15168 } else if (next_opline) { 15169 | jmp >3 15170 } else { 15171 | jmp =>default_b 15172 } 15173 |3: 15174 } else { 15175 ZEND_UNREACHABLE(); 15176 } 15177 } 15178 return 1; 15179} 15180 15181static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) 15182{ 15183 zend_arg_info *arg_info = &op_array->arg_info[-1]; 15184 ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); 15185 zend_jit_addr op1_addr = OP1_ADDR(); 15186 bool needs_slow_check = 1; 15187 bool slow_check_in_cold = 1; 15188 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 15189 15190 if (type_mask == 0) { 15191 slow_check_in_cold = 0; 15192 } else { 15193 if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { 15194 slow_check_in_cold = 0; 15195 } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { 15196 needs_slow_check = 0; 15197 } else if (is_power_of_two(type_mask)) { 15198 uint32_t type_code = concrete_type(type_mask); 15199 | IF_NOT_ZVAL_TYPE op1_addr, type_code, >6 15200 } else { 15201 | mov edx, 1 15202 | GET_ZVAL_TYPE cl, op1_addr 15203 | shl edx, cl 15204 | test edx, type_mask 15205 | je >6 15206 } 15207 } 15208 if (needs_slow_check) { 15209 if (slow_check_in_cold) { 15210 |.cold_code 15211 |6: 15212 } 15213 | SET_EX_OPLINE opline, r1 15214 if (op1_info & MAY_BE_UNDEF) { 15215 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7 15216 | mov FCARG1a, opline->op1.var 15217 | EXT_CALL zend_jit_undefined_op_helper, FCARG2a 15218 | test r0, r0 15219 | jz ->exception_handler 15220 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 15221 | jmp >8 15222 } 15223 |7: 15224 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 15225 |8: 15226 | mov FCARG2a, EX->func 15227 |.if X64 15228 | LOAD_ADDR CARG3, (ptrdiff_t)arg_info 15229 | mov r0, EX->run_time_cache 15230 | lea CARG4, aword [r0+opline->op2.num] 15231 | EXT_CALL zend_jit_verify_return_slow, r0 15232 |.else 15233 | sub r4, 8 15234 | mov r0, EX->run_time_cache 15235 | add r0, opline->op2.num 15236 | push r0 15237 | push (ptrdiff_t)arg_info 15238 | EXT_CALL zend_jit_verify_return_slow, r0 15239 | add r4, 8 15240 |.endif 15241 if (!zend_jit_check_exception(Dst)) { 15242 return 0; 15243 } 15244 if (slow_check_in_cold) { 15245 | jmp >9 15246 |.code 15247 } 15248 } 15249 |9: 15250 return 1; 15251} 15252 15253static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 15254{ 15255 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15256 15257 // TODO: support for empty() ??? 15258 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 15259 15260 if (op1_info & MAY_BE_REF) { 15261 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 15262 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 15263 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 15264 } 15265 | ZVAL_DEREF FCARG1a, op1_info 15266 |1: 15267 } 15268 15269 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { 15270 if (exit_addr) { 15271 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); 15272 } else if (smart_branch_opcode) { 15273 if (smart_branch_opcode == ZEND_JMPNZ) { 15274 | jmp =>target_label 15275 } 15276 } else { 15277 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 15278 } 15279 } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { 15280 if (exit_addr) { 15281 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); 15282 } else if (smart_branch_opcode) { 15283 if (smart_branch_opcode != ZEND_JMPNZ) { 15284 | jmp =>target_label 15285 } 15286 } else { 15287 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 15288 } 15289 } else { 15290 ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); 15291 | cmp byte [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)], IS_NULL 15292 if (exit_addr) { 15293 if (smart_branch_opcode == ZEND_JMPNZ) { 15294 | jg &exit_addr 15295 } else { 15296 | jle &exit_addr 15297 } 15298 } else if (smart_branch_opcode) { 15299 if (smart_branch_opcode == ZEND_JMPZ) { 15300 | jle =>target_label 15301 } else if (smart_branch_opcode == ZEND_JMPNZ) { 15302 | jg =>target_label 15303 } else { 15304 ZEND_UNREACHABLE(); 15305 } 15306 } else { 15307 | setg al 15308 | movzx eax, al 15309 | lea eax, [eax + IS_FALSE] 15310 | SET_ZVAL_TYPE_INFO res_addr, eax 15311 } 15312 } 15313 15314 return 1; 15315} 15316 15317static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 15318{ 15319 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15320 15321 if (opline->op1_type == IS_CONST) { 15322 zval *zv = RT_CONSTANT(opline, opline->op1); 15323 15324 | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 15325 if (Z_REFCOUNTED_P(zv)) { 15326 | ADDREF_CONST zv, r0 15327 } 15328 } else { 15329 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 15330 15331 | // ZVAL_COPY(res, value); 15332 | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_FCARG1 15333 if (opline->op1_type == IS_CV) { 15334 | TRY_ADDREF op1_info, ah, FCARG1a 15335 } 15336 } 15337 | // Z_FE_POS_P(res) = 0; 15338 | mov dword [FP + opline->result.var + offsetof(zval, u2.fe_pos)], 0 15339 15340 return 1; 15341} 15342 15343static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr) 15344{ 15345 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 15346 15347 if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) { 15348 /* empty array */ 15349 if (exit_addr) { 15350 if (exit_opcode == ZEND_JMP) { 15351 | jmp &exit_addr 15352 } 15353 } else { 15354 | jmp =>target_label 15355 } 15356 return 1; 15357 } 15358 15359 | // array = EX_VAR(opline->op1.var); 15360 | // fe_ht = Z_ARRVAL_P(array); 15361 | GET_ZVAL_PTR FCARG1a, op1_addr 15362 15363 if (op1_info & MAY_BE_PACKED_GUARD) { 15364 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 15365 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15366 15367 if (!exit_addr) { 15368 return 0; 15369 } 15370 if (op1_info & MAY_BE_ARRAY_PACKED) { 15371 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 15372 | jz &exit_addr 15373 } else { 15374 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 15375 | jnz &exit_addr 15376 } 15377 } 15378 15379 | // pos = Z_FE_POS_P(array); 15380 | mov eax, dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)] 15381 15382 if (MAY_BE_HASH(op1_info)) { 15383 if (MAY_BE_PACKED(op1_info)) { 15384 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 15385 | jnz >2 15386 } 15387 15388 | // p = fe_ht->arData + pos; 15389 |.if X64 15390 || ZEND_ASSERT(sizeof(Bucket) == 32); 15391 | mov FCARG2d, eax 15392 | shl FCARG2a, 5 15393 |.else 15394 | imul FCARG2a, r0, sizeof(Bucket) 15395 |.endif 15396 | add FCARG2a, aword [FCARG1a + offsetof(zend_array, arData)] 15397 |1: 15398 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 15399 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], eax 15400 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 15401 | // ZEND_VM_CONTINUE(); 15402 if (exit_addr) { 15403 if (exit_opcode == ZEND_JMP) { 15404 | jbe &exit_addr 15405 } else { 15406 | jbe >3 15407 } 15408 } else { 15409 | jbe =>target_label 15410 } 15411 | // pos++; 15412 | add eax, 1 15413 | // value_type = Z_TYPE_INFO_P(value); 15414 | // if (EXPECTED(value_type != IS_UNDEF)) { 15415 if (!exit_addr || exit_opcode == ZEND_JMP) { 15416 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >3 15417 } else { 15418 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, &exit_addr 15419 } 15420 | // p++; 15421 | add FCARG2a, sizeof(Bucket) 15422 | jmp <1 15423 if (MAY_BE_PACKED(op1_info)) { 15424 |2: 15425 } 15426 } 15427 if (MAY_BE_PACKED(op1_info)) { 15428 | // p = fe_ht->arPacked + pos; 15429 || ZEND_ASSERT(sizeof(zval) == 16); 15430 | mov FCARG2d, eax 15431 | shl FCARG2a, 4 15432 | add FCARG2a, aword [FCARG1a + offsetof(zend_array, arPacked)] 15433 |1: 15434 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 15435 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], eax 15436 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 15437 | // ZEND_VM_CONTINUE(); 15438 if (exit_addr) { 15439 if (exit_opcode == ZEND_JMP) { 15440 | jbe &exit_addr 15441 } else { 15442 | jbe >4 15443 } 15444 } else { 15445 | jbe =>target_label 15446 } 15447 | // pos++; 15448 | add eax, 1 15449 | // value_type = Z_TYPE_INFO_P(value); 15450 | // if (EXPECTED(value_type != IS_UNDEF)) { 15451 if (!exit_addr || exit_opcode == ZEND_JMP) { 15452 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >4 15453 } else { 15454 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, &exit_addr 15455 } 15456 | // p++; 15457 | add FCARG2a, sizeof(zval) 15458 | jmp <1 15459 } 15460 15461 15462 if (!exit_addr || exit_opcode == ZEND_JMP) { 15463 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 15464 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 15465 uint32_t val_info; 15466 15467 if (RETURN_VALUE_USED(opline)) { 15468 zend_jit_addr res_addr = RES_ADDR(); 15469 15470 if (MAY_BE_HASH(op1_info)) { 15471 |3: 15472 | // Z_FE_POS_P(array) = pos + 1; 15473 | mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], eax 15474 15475 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) 15476 && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { 15477 | // if (!p->key) { 15478 | cmp aword [FCARG2a + offsetof(Bucket, key)], 0 15479 | jz >2 15480 } 15481 if (op1_info & MAY_BE_ARRAY_KEY_STRING) { 15482 | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); 15483 | mov r0, aword [FCARG2a + offsetof(Bucket, key)] 15484 | SET_ZVAL_PTR res_addr, r0 15485 | test dword [r0 + offsetof(zend_refcounted, gc.u.type_info)], IS_STR_INTERNED 15486 | jz >1 15487 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING 15488 | jmp >3 15489 |1: 15490 | GC_ADDREF r0 15491 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX 15492 15493 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) || MAY_BE_PACKED(op1_info)) { 15494 | jmp >3 15495 |2: 15496 } 15497 } 15498 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 15499 | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); 15500 | mov r0, aword [FCARG2a + offsetof(Bucket, h)] 15501 | SET_ZVAL_LVAL res_addr, r0 15502 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 15503 if (MAY_BE_PACKED(op1_info)) { 15504 | jmp >3 15505 } 15506 } 15507 } 15508 if (MAY_BE_PACKED(op1_info)) { 15509 |4: 15510 | // Z_FE_POS_P(array) = pos + 1; 15511 | mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], eax 15512 | sub r0, 1 15513 | SET_ZVAL_LVAL res_addr, r0 15514 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 15515 } 15516 |3: 15517 } else { 15518 |3: 15519 |4: 15520 | // Z_FE_POS_P(array) = pos + 1; 15521 | mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], eax 15522 } 15523 15524 val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); 15525 if (val_info & MAY_BE_ARRAY) { 15526 val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 15527 } 15528 if (op1_info & MAY_BE_ARRAY_OF_REF) { 15529 val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | 15530 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 15531 } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 15532 val_info |= MAY_BE_RC1 | MAY_BE_RCN; 15533 } 15534 15535 if (opline->op2_type == IS_CV) { 15536 | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); 15537 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { 15538 return 0; 15539 } 15540 } else { 15541 | // ZVAL_COPY(res, value); 15542 | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_R0, ZREG_FCARG1 15543 | TRY_ADDREF val_info, ah, FCARG1a 15544 } 15545 } else { 15546 |3: 15547 |4: 15548 } 15549 15550 return 1; 15551} 15552 15553static int zend_jit_fetch_constant(dasm_State **Dst, 15554 const zend_op *opline, 15555 const zend_op_array *op_array, 15556 zend_ssa *ssa, 15557 const zend_ssa_op *ssa_op, 15558 zend_jit_addr res_addr) 15559{ 15560 zval *zv = RT_CONSTANT(opline, opline->op2) + 1; 15561 zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 15562 uint32_t res_info = RES_INFO(); 15563 15564 | // c = CACHED_PTR(opline->extended_value); 15565 | mov FCARG1a, EX->run_time_cache 15566 | mov r0, aword [FCARG1a + opline->extended_value] 15567 | // if (c != NULL) 15568 | test r0, r0 15569 | jz >9 15570 if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) { 15571 | // if (!IS_SPECIAL_CACHE_VAL(c)) 15572 | test r0, CACHE_SPECIAL 15573 | jnz >9 15574 } 15575 |8: 15576 15577 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { 15578 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 15579 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 15580 int32_t exit_point; 15581 const void *exit_addr = NULL; 15582 15583 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 15584 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 15585 exit_point = zend_jit_trace_get_exit_point(opline+1, 0); 15586 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 15587 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15588 if (!exit_addr) { 15589 return 0; 15590 } 15591 res_info &= ~MAY_BE_GUARD; 15592 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 15593 15594 zend_uchar type = concrete_type(res_info); 15595 15596 if (type < IS_STRING) { 15597 | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr 15598 } else { 15599 | GET_ZVAL_TYPE_INFO edx, const_addr 15600 | IF_NOT_TYPE dl, type, &exit_addr 15601 } 15602 | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_R0, ZREG_R1 15603 if (type < IS_STRING) { 15604 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 15605 | SET_ZVAL_TYPE_INFO res_addr, type 15606 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 15607 return 0; 15608 } 15609 } else { 15610 | SET_ZVAL_TYPE_INFO res_addr, edx 15611 | TRY_ADDREF res_info, dh, r1 15612 } 15613 } else { 15614 | // ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup) 15615 | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_R0, ZREG_R1 15616 | TRY_ADDREF MAY_BE_ANY, ah, r1 15617 } 15618 15619 |.cold_code 15620 |9: 15621 | // SAVE_OPLINE(); 15622 | SET_EX_OPLINE opline, r0 15623 | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); 15624 | LOAD_ADDR FCARG1a, zv 15625 | mov FCARG2a, opline->op1.num 15626 | EXT_CALL zend_jit_get_constant, r0 15627 | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); 15628 | test r0, r0 15629 | jnz <8 15630 | jmp ->exception_handler 15631 |.code 15632 15633 return 1; 15634} 15635 15636static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 15637{ 15638 HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 15639 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15640 15641 ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); 15642 ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); 15643 15644 | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); 15645 | LOAD_ADDR FCARG1a, ht 15646 if (opline->op1_type != IS_CONST) { 15647 | GET_ZVAL_PTR FCARG2a, op1_addr 15648 | EXT_CALL zend_hash_find, r0 15649 } else { 15650 zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); 15651 | LOAD_ADDR FCARG2a, str 15652 | EXT_CALL zend_hash_find_known_hash, r0 15653 } 15654 | test r0, r0 15655 if (exit_addr) { 15656 if (smart_branch_opcode == ZEND_JMPZ) { 15657 | jz &exit_addr 15658 } else { 15659 | jnz &exit_addr 15660 } 15661 } else if (smart_branch_opcode) { 15662 if (smart_branch_opcode == ZEND_JMPZ) { 15663 | jz =>target_label 15664 } else if (smart_branch_opcode == ZEND_JMPNZ) { 15665 | jnz =>target_label 15666 } else { 15667 ZEND_UNREACHABLE(); 15668 } 15669 } else { 15670 | setnz al 15671 | movzx eax, al 15672 | lea eax, [eax + IS_FALSE] 15673 | SET_ZVAL_TYPE_INFO res_addr, eax 15674 } 15675 15676 return 1; 15677} 15678 15679static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info) 15680{ 15681 uint32_t offset; 15682 15683 offset = (opline->opcode == ZEND_ROPE_INIT) ? 15684 opline->result.var : 15685 opline->op1.var + opline->extended_value * sizeof(zend_string*); 15686 15687 if (opline->op2_type == IS_CONST) { 15688 zval *zv = RT_CONSTANT(opline, opline->op2); 15689 zend_string *str; 15690 15691 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 15692 str = Z_STR_P(zv); 15693 | ADDR_STORE aword [FP + offset], str, r0 15694 } else { 15695 zend_jit_addr op2_addr = OP2_ADDR(); 15696 15697 ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 15698 15699 | GET_ZVAL_PTR r1, op2_addr 15700 | mov aword [FP + offset], r1 15701 if (opline->op2_type == IS_CV) { 15702 | GET_ZVAL_TYPE_INFO eax, op2_addr 15703 | TRY_ADDREF op2_info, ah, r1 15704 } 15705 } 15706 15707 if (opline->opcode == ZEND_ROPE_END) { 15708 zend_jit_addr res_addr = RES_ADDR(); 15709 15710 | lea FCARG1a, [FP + opline->op1.var] 15711 | mov FCARG2d, opline->extended_value 15712 | EXT_CALL zend_jit_rope_end, r0 15713 | SET_ZVAL_PTR res_addr, r0 15714 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX 15715 } 15716 15717 return 1; 15718} 15719 15720static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) 15721{ 15722 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15723 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15724 15725 if (!exit_addr) { 15726 return 0; 15727 } 15728 | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr 15729 15730 return 1; 15731} 15732 15733static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard) 15734{ 15735 zend_jit_addr var_addr = *var_addr_ptr; 15736 uint32_t var_info = *var_info_ptr; 15737 const void *exit_addr = NULL; 15738 15739 if (add_ref_guard || add_type_guard) { 15740 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15741 15742 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15743 if (!exit_addr) { 15744 return 0; 15745 } 15746 } 15747 15748 if (add_ref_guard) { 15749 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr 15750 } 15751 if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { 15752 /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ 15753 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 15754 | LOAD_ZVAL_ADDR FCARG1a, var_addr 15755 } 15756 | EXT_CALL zend_jit_unref_helper, r0 15757 } else { 15758 | GET_ZVAL_PTR FCARG1a, var_addr 15759 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 15760 *var_addr_ptr = var_addr; 15761 } 15762 15763 if (var_type != IS_UNKNOWN) { 15764 var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); 15765 } 15766 if (add_type_guard 15767 && var_type != IS_UNKNOWN 15768 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 15769 | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr 15770 15771 ZEND_ASSERT(var_info & (1 << var_type)); 15772 if (var_type < IS_STRING) { 15773 var_info = (1 << var_type); 15774 } else if (var_type != IS_ARRAY) { 15775 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 15776 } else { 15777 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 15778 } 15779 15780 *var_info_ptr = var_info; 15781 } else { 15782 var_info &= ~MAY_BE_REF; 15783 *var_info_ptr = var_info; 15784 } 15785 *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */ 15786 15787 return 1; 15788} 15789 15790static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard) 15791{ 15792 zend_jit_addr var_addr = *var_addr_ptr; 15793 uint32_t var_info = *var_info_ptr; 15794 int32_t exit_point; 15795 const void *exit_addr; 15796 15797 if (add_indirect_guard) { 15798 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15799 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15800 15801 if (!exit_addr) { 15802 return 0; 15803 } 15804 | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr 15805 | GET_ZVAL_PTR FCARG1a, var_addr 15806 } else { 15807 /* May be already loaded into FCARG1a or RAX by previous FETCH_OBJ_W/DIM_W */ 15808 if (opline->op1_type != IS_VAR || 15809 (opline-1)->result_type != IS_VAR || 15810 (opline-1)->result.var != opline->op1.var || 15811 (opline-1)->op1_type == IS_VAR || 15812 (opline-1)->op2_type == IS_VAR || 15813 (opline-1)->op2_type == IS_TMP_VAR) { 15814 | GET_ZVAL_PTR FCARG1a, var_addr 15815 } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { 15816 | mov FCARG1a, r0 15817 } 15818 } 15819 *var_info_ptr &= ~MAY_BE_INDIRECT; 15820 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 15821 *var_addr_ptr = var_addr; 15822 15823 if (var_type != IS_UNKNOWN) { 15824 var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); 15825 } 15826 if (!(var_type & IS_TRACE_REFERENCE) 15827 && var_type != IS_UNKNOWN 15828 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 15829 exit_point = zend_jit_trace_get_exit_point(opline, 0); 15830 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15831 15832 if (!exit_addr) { 15833 return 0; 15834 } 15835 15836 | IF_NOT_Z_TYPE FCARG1a, var_type, &exit_addr 15837 15838 //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); 15839 ZEND_ASSERT(var_info & (1 << var_type)); 15840 if (var_type < IS_STRING) { 15841 var_info = (1 << var_type); 15842 } else if (var_type != IS_ARRAY) { 15843 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 15844 } else { 15845 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 15846 } 15847 15848 *var_info_ptr = var_info; 15849 } 15850 15851 return 1; 15852} 15853 15854static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) 15855{ 15856 if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { 15857 return 0; 15858 } 15859 15860 switch (opline->opcode) { 15861 case ZEND_QM_ASSIGN: 15862 case ZEND_SEND_VAR: 15863 case ZEND_ASSIGN: 15864 case ZEND_PRE_INC: 15865 case ZEND_PRE_DEC: 15866 case ZEND_POST_INC: 15867 case ZEND_POST_DEC: 15868 return 1; 15869 case ZEND_ADD: 15870 case ZEND_SUB: 15871 case ZEND_MUL: 15872 case ZEND_BW_OR: 15873 case ZEND_BW_AND: 15874 case ZEND_BW_XOR: 15875 if (def_var == ssa_op->result_def && 15876 use_var == ssa_op->op1_use) { 15877 return 1; 15878 } 15879 break; 15880 default: 15881 break; 15882 } 15883 return 0; 15884} 15885 15886static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) 15887{ 15888 uint32_t op1_info, op2_info; 15889 15890 switch (opline->opcode) { 15891 case ZEND_SEND_VAR: 15892 case ZEND_SEND_VAL: 15893 case ZEND_SEND_VAL_EX: 15894 return (opline->op2_type != IS_CONST); 15895 case ZEND_QM_ASSIGN: 15896 case ZEND_IS_SMALLER: 15897 case ZEND_IS_SMALLER_OR_EQUAL: 15898 case ZEND_IS_EQUAL: 15899 case ZEND_IS_NOT_EQUAL: 15900 case ZEND_IS_IDENTICAL: 15901 case ZEND_IS_NOT_IDENTICAL: 15902 case ZEND_CASE: 15903 return 1; 15904 case ZEND_RETURN: 15905 return (op_array->type != ZEND_EVAL_CODE && op_array->function_name); 15906 case ZEND_ASSIGN: 15907 op1_info = OP1_INFO(); 15908 op2_info = OP2_INFO(); 15909 return 15910 opline->op1_type == IS_CV && 15911 !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) && 15912 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); 15913 case ZEND_ADD: 15914 case ZEND_SUB: 15915 case ZEND_MUL: 15916 op1_info = OP1_INFO(); 15917 op2_info = OP2_INFO(); 15918 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); 15919 case ZEND_BW_OR: 15920 case ZEND_BW_AND: 15921 case ZEND_BW_XOR: 15922 case ZEND_SL: 15923 case ZEND_SR: 15924 case ZEND_MOD: 15925 op1_info = OP1_INFO(); 15926 op2_info = OP2_INFO(); 15927 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); 15928 case ZEND_PRE_INC: 15929 case ZEND_PRE_DEC: 15930 case ZEND_POST_INC: 15931 case ZEND_POST_DEC: 15932 op1_info = OP1_INFO(); 15933 op2_info = OP1_DEF_INFO(); 15934 return opline->op1_type == IS_CV 15935 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)) 15936 && (op2_info & MAY_BE_LONG); 15937 case ZEND_STRLEN: 15938 op1_info = OP1_INFO(); 15939 return (opline->op1_type & (IS_CV|IS_CONST)) 15940 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING; 15941 case ZEND_COUNT: 15942 op1_info = OP1_INFO(); 15943 return (opline->op1_type & (IS_CV|IS_CONST)) 15944 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY; 15945 case ZEND_JMPZ: 15946 case ZEND_JMPNZ: 15947 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 15948 if (!ssa->cfg.map) { 15949 return 0; 15950 } 15951 if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start && 15952 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { 15953 return 0; 15954 } 15955 } 15956 ZEND_FALLTHROUGH; 15957 case ZEND_BOOL: 15958 case ZEND_BOOL_NOT: 15959 case ZEND_JMPZ_EX: 15960 case ZEND_JMPNZ_EX: 15961 return 1; 15962 case ZEND_FETCH_CONSTANT: 15963 return 1; 15964 case ZEND_FETCH_DIM_R: 15965 op1_info = OP1_INFO(); 15966 op2_info = OP2_INFO(); 15967 if (trace 15968 && trace->op1_type != IS_UNKNOWN 15969 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { 15970 op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); 15971 } 15972 return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && 15973 (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && 15974 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || 15975 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && 15976 (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); 15977 } 15978 return 0; 15979} 15980 15981static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) 15982{ 15983 if (ssa->vars[var].no_val) { 15984 /* we don't need the value */ 15985 return 0; 15986 } 15987 15988 if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { 15989 /* Disable global register allocation, 15990 * register allocation for SSA variables connected through Phi functions 15991 */ 15992 if (ssa->vars[var].definition_phi) { 15993 return 0; 15994 } 15995 if (ssa->vars[var].phi_use_chain) { 15996 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; 15997 do { 15998 if (!ssa->vars[phi->ssa_var].no_val) { 15999 return 0; 16000 } 16001 phi = zend_ssa_next_use_phi(ssa, var, phi); 16002 } while (phi); 16003 } 16004 } 16005 16006 if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && 16007 ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { 16008 /* bad type */ 16009 return 0; 16010 } 16011 16012 return 1; 16013} 16014 16015static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) 16016{ 16017 if (!zend_jit_var_supports_reg(ssa, var)) { 16018 return 0; 16019 } 16020 16021 if (ssa->vars[var].definition >= 0) { 16022 uint32_t def = ssa->vars[var].definition; 16023 if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { 16024 return 0; 16025 } 16026 } 16027 16028 if (ssa->vars[var].use_chain >= 0) { 16029 int use = ssa->vars[var].use_chain; 16030 16031 do { 16032 if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && 16033 !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { 16034 return 0; 16035 } 16036 use = zend_ssa_next_use(ssa->ops, var, use); 16037 } while (use >= 0); 16038 } 16039 16040 return 1; 16041} 16042 16043static bool zend_needs_extra_reg_for_const(const zend_op *opline, zend_uchar op_type, znode_op op) 16044{ 16045|.if X64 16046|| if (op_type == IS_CONST) { 16047|| zval *zv = RT_CONSTANT(opline, op); 16048|| if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0 && !IS_SIGNED_32BIT(zv)) { 16049|| return 1; 16050|| } else if (Z_TYPE_P(zv) == IS_LONG && !IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 16051|| return 1; 16052|| } 16053|| } 16054|.endif 16055 return 0; 16056} 16057 16058static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 16059{ 16060 uint32_t op1_info, op2_info; 16061 16062 switch (opline->opcode) { 16063 case ZEND_FETCH_DIM_R: 16064 op1_info = OP1_INFO(); 16065 op2_info = OP2_INFO(); 16066 if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && 16067 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || 16068 ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && 16069 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { 16070 return ZEND_REGSET(ZREG_FCARG1); 16071 } 16072 break; 16073 default: 16074 break; 16075 } 16076 16077 return ZEND_REGSET_EMPTY; 16078} 16079 16080static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 16081{ 16082 uint32_t op1_info, op2_info, res_info; 16083 zend_regset regset = ZEND_REGSET_SCRATCH; 16084 16085 switch (opline->opcode) { 16086 case ZEND_NOP: 16087 case ZEND_OP_DATA: 16088 case ZEND_JMP: 16089 case ZEND_RETURN: 16090 regset = ZEND_REGSET_EMPTY; 16091 break; 16092 case ZEND_QM_ASSIGN: 16093 if (ssa_op->op1_def == current_var || 16094 ssa_op->result_def == current_var) { 16095 regset = ZEND_REGSET_EMPTY; 16096 break; 16097 } 16098 /* break missing intentionally */ 16099 case ZEND_SEND_VAL: 16100 case ZEND_SEND_VAL_EX: 16101 if (opline->op2_type == IS_CONST) { 16102 break; 16103 } 16104 if (ssa_op->op1_use == current_var) { 16105 regset = ZEND_REGSET(ZREG_R0); 16106 break; 16107 } 16108 op1_info = OP1_INFO(); 16109 if (!(op1_info & MAY_BE_UNDEF)) { 16110 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 16111 regset = ZEND_REGSET(ZREG_XMM0); 16112 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 16113 regset = ZEND_REGSET(ZREG_R0); 16114 } else { 16115 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 16116 } 16117 } 16118 break; 16119 case ZEND_SEND_VAR: 16120 if (opline->op2_type == IS_CONST) { 16121 break; 16122 } 16123 if (ssa_op->op1_use == current_var || 16124 ssa_op->op1_def == current_var) { 16125 regset = ZEND_REGSET_EMPTY; 16126 break; 16127 } 16128 op1_info = OP1_INFO(); 16129 if (!(op1_info & MAY_BE_UNDEF)) { 16130 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 16131 regset = ZEND_REGSET(ZREG_XMM0); 16132 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 16133 } else { 16134 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 16135 if (op1_info & MAY_BE_REF) { 16136 ZEND_REGSET_INCL(regset, ZREG_R1); 16137 } 16138 } 16139 } 16140 break; 16141 case ZEND_ASSIGN: 16142 if (ssa_op->op2_use == current_var || 16143 ssa_op->op2_def == current_var || 16144 ssa_op->op1_def == current_var || 16145 ssa_op->result_def == current_var) { 16146 regset = ZEND_REGSET_EMPTY; 16147 break; 16148 } 16149 op1_info = OP1_INFO(); 16150 op2_info = OP2_INFO(); 16151 if (opline->op1_type == IS_CV 16152 && !(op2_info & MAY_BE_UNDEF) 16153 && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 16154 if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 16155 regset = ZEND_REGSET(ZREG_XMM0); 16156 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 16157 regset = ZEND_REGSET(ZREG_R0); 16158 } else { 16159 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 16160 } 16161 } 16162 break; 16163 case ZEND_PRE_INC: 16164 case ZEND_PRE_DEC: 16165 case ZEND_POST_INC: 16166 case ZEND_POST_DEC: 16167 if (ssa_op->op1_use == current_var || 16168 ssa_op->op1_def == current_var || 16169 ssa_op->result_def == current_var) { 16170 regset = ZEND_REGSET_EMPTY; 16171 break; 16172 } 16173 op1_info = OP1_INFO(); 16174 if (opline->op1_type == IS_CV 16175 && (op1_info & MAY_BE_LONG) 16176 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16177 regset = ZEND_REGSET_EMPTY; 16178 if (op1_info & MAY_BE_DOUBLE) { 16179 regset = ZEND_REGSET(ZREG_XMM0); 16180 } 16181 if (opline->result_type != IS_UNUSED && (op1_info & MAY_BE_LONG)) { 16182 ZEND_REGSET_INCL(regset, ZREG_R1); 16183 } 16184 } 16185 break; 16186 case ZEND_ADD: 16187 case ZEND_SUB: 16188 case ZEND_MUL: 16189 op1_info = OP1_INFO(); 16190 op2_info = OP2_INFO(); 16191 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 16192 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16193 16194 regset = ZEND_REGSET_EMPTY; 16195 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 16196 if (ssa_op->result_def != current_var && 16197 (ssa_op->op1_use != current_var || !last_use)) { 16198 ZEND_REGSET_INCL(regset, ZREG_R0); 16199 } 16200 res_info = RES_INFO(); 16201 if (res_info & MAY_BE_DOUBLE) { 16202 ZEND_REGSET_INCL(regset, ZREG_R0); 16203 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16204 ZEND_REGSET_INCL(regset, ZREG_XMM1); 16205 } else if (res_info & MAY_BE_GUARD) { 16206 ZEND_REGSET_INCL(regset, ZREG_R0); 16207 } 16208 } 16209 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 16210 if (opline->op1_type == IS_CONST) { 16211 ZEND_REGSET_INCL(regset, ZREG_R0); 16212 } 16213 if (ssa_op->result_def != current_var) { 16214 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16215 } 16216 } 16217 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 16218 if (opline->op2_type == IS_CONST) { 16219 ZEND_REGSET_INCL(regset, ZREG_R0); 16220 } 16221 if (zend_is_commutative(opline->opcode)) { 16222 if (ssa_op->result_def != current_var) { 16223 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16224 } 16225 } else { 16226 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16227 if (ssa_op->result_def != current_var && 16228 (ssa_op->op1_use != current_var || !last_use)) { 16229 ZEND_REGSET_INCL(regset, ZREG_XMM1); 16230 } 16231 } 16232 } 16233 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 16234 if (ssa_op->result_def != current_var && 16235 (ssa_op->op1_use != current_var || !last_use) && 16236 (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { 16237 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16238 } 16239 } 16240 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16241 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16242 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 16243 ZEND_REGSET_INCL(regset, ZREG_R0); 16244 } else { 16245 ZEND_REGSET_INCL(regset, ZREG_R1); 16246 } 16247 } 16248 } 16249 break; 16250 case ZEND_BW_OR: 16251 case ZEND_BW_AND: 16252 case ZEND_BW_XOR: 16253 op1_info = OP1_INFO(); 16254 op2_info = OP2_INFO(); 16255 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16256 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16257 regset = ZEND_REGSET_EMPTY; 16258 if (ssa_op->result_def != current_var && 16259 (ssa_op->op1_use != current_var || !last_use)) { 16260 ZEND_REGSET_INCL(regset, ZREG_R0); 16261 } 16262 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16263 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16264 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 16265 ZEND_REGSET_INCL(regset, ZREG_R0); 16266 } else { 16267 ZEND_REGSET_INCL(regset, ZREG_R1); 16268 } 16269 } 16270 } 16271 break; 16272 case ZEND_SL: 16273 case ZEND_SR: 16274 op1_info = OP1_INFO(); 16275 op2_info = OP2_INFO(); 16276 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16277 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16278bw_op: 16279 regset = ZEND_REGSET_EMPTY; 16280 if (ssa_op->result_def != current_var && 16281 (ssa_op->op1_use != current_var || !last_use)) { 16282 ZEND_REGSET_INCL(regset, ZREG_R0); 16283 } 16284 if (opline->op2_type != IS_CONST && ssa_op->op2_use != current_var) { 16285 ZEND_REGSET_INCL(regset, ZREG_R1); 16286 } 16287 } 16288 break; 16289 case ZEND_MOD: 16290 op1_info = OP1_INFO(); 16291 op2_info = OP2_INFO(); 16292 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16293 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16294 if (opline->op2_type == IS_CONST && 16295 opline->op1_type != IS_CONST && 16296 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG && 16297 zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2))) && 16298 OP1_HAS_RANGE() && 16299 OP1_MIN_RANGE() >= 0) { 16300 /* MOD is going to be optimized into AND */ 16301 goto bw_op; 16302 } else { 16303 regset = ZEND_REGSET_EMPTY; 16304 ZEND_REGSET_INCL(regset, ZREG_R0); 16305 ZEND_REGSET_INCL(regset, ZREG_R2); 16306 if (opline->op2_type == IS_CONST) { 16307 ZEND_REGSET_INCL(regset, ZREG_R1); 16308 } 16309 } 16310 } 16311 break; 16312 case ZEND_IS_SMALLER: 16313 case ZEND_IS_SMALLER_OR_EQUAL: 16314 case ZEND_IS_EQUAL: 16315 case ZEND_IS_NOT_EQUAL: 16316 case ZEND_IS_IDENTICAL: 16317 case ZEND_IS_NOT_IDENTICAL: 16318 case ZEND_CASE: 16319 op1_info = OP1_INFO(); 16320 op2_info = OP2_INFO(); 16321 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 16322 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16323 regset = ZEND_REGSET_EMPTY; 16324 if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { 16325 ZEND_REGSET_INCL(regset, ZREG_R0); 16326 } 16327 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && 16328 opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { 16329 if (ssa_op->op1_use != current_var && 16330 ssa_op->op2_use != current_var) { 16331 ZEND_REGSET_INCL(regset, ZREG_R0); 16332 } 16333 } 16334 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 16335 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16336 } 16337 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 16338 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16339 } 16340 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 16341 if (ssa_op->op1_use != current_var && 16342 ssa_op->op2_use != current_var) { 16343 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16344 } 16345 } 16346 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16347 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16348 ZEND_REGSET_INCL(regset, ZREG_R0); 16349 } 16350 } 16351 break; 16352 case ZEND_BOOL: 16353 case ZEND_BOOL_NOT: 16354 case ZEND_JMPZ: 16355 case ZEND_JMPNZ: 16356 case ZEND_JMPZ_EX: 16357 case ZEND_JMPNZ_EX: 16358 op1_info = OP1_INFO(); 16359 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16360 regset = ZEND_REGSET_EMPTY; 16361 if (op1_info & MAY_BE_DOUBLE) { 16362 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16363 } 16364 if (opline->opcode == ZEND_BOOL || 16365 opline->opcode == ZEND_BOOL_NOT || 16366 opline->opcode == ZEND_JMPZ_EX || 16367 opline->opcode == ZEND_JMPNZ_EX) { 16368 ZEND_REGSET_INCL(regset, ZREG_R0); 16369 } 16370 } 16371 break; 16372 case ZEND_DO_UCALL: 16373 case ZEND_DO_FCALL: 16374 case ZEND_DO_FCALL_BY_NAME: 16375 case ZEND_INCLUDE_OR_EVAL: 16376 case ZEND_GENERATOR_CREATE: 16377 case ZEND_YIELD: 16378 case ZEND_YIELD_FROM: 16379 regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); 16380 break; 16381 default: 16382 break; 16383 } 16384 16385 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 16386 if (ssa_op == ssa->ops 16387 && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL 16388 && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { 16389 ZEND_REGSET_INCL(regset, ZREG_R0); 16390 ZEND_REGSET_INCL(regset, ZREG_R1); 16391 } 16392 } 16393 16394 /* %r0 is used to check EG(vm_interrupt) */ 16395 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 16396 if (ssa_op == ssa->ops 16397 && (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP || 16398 JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) { 16399#if ZTS 16400 ZEND_REGSET_INCL(regset, ZREG_R0); 16401#else 16402 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { 16403 ZEND_REGSET_INCL(regset, ZREG_R0); 16404 } 16405#endif 16406 } 16407 } else { 16408 uint32_t b = ssa->cfg.map[ssa_op - ssa->ops]; 16409 16410 if ((ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) != 0 16411 && ssa->cfg.blocks[b].start == ssa_op - ssa->ops) { 16412#if ZTS 16413 ZEND_REGSET_INCL(regset, ZREG_R0); 16414#else 16415 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { 16416 ZEND_REGSET_INCL(regset, ZREG_R0); 16417 } 16418#endif 16419 } 16420 } 16421 16422 return regset; 16423} 16424 16425/* 16426 * Local variables: 16427 * tab-width: 4 16428 * c-basic-offset: 4 16429 * indent-tabs-mode: t 16430 * End: 16431 */ 16432
