1/* 2 * +----------------------------------------------------------------------+ 3 * | Zend JIT | 4 * +----------------------------------------------------------------------+ 5 * | Copyright (c) The PHP Group | 6 * +----------------------------------------------------------------------+ 7 * | This source file is subject to version 3.01 of the PHP license, | 8 * | that is bundled with this package in the file LICENSE, and is | 9 * | available through the world-wide-web at the following url: | 10 * | https://www.php.net/license/3_01.txt | 11 * | If you did not receive a copy of the PHP license and are unable to | 12 * | obtain it through the world-wide-web, please send a note to | 13 * | license@php.net so we can mail you a copy immediately. | 14 * +----------------------------------------------------------------------+ 15 * | Authors: Dmitry Stogov <dmitry@php.net> | 16 * | Xinchen Hui <laruence@php.net> | 17 * +----------------------------------------------------------------------+ 18 */ 19 20|.if X64 21 |.arch x64 22|.else 23 |.arch x86 24|.endif 25 26|.if X64WIN 27 |.define FP, r14 28 |.define IP, r15 29 |.define IPl, r15d 30 |.define RX, r15 // the same as VM IP reused as a general purpose reg 31 |.define CARG1, rcx // x64/POSIX C call arguments. 32 |.define CARG2, rdx 33 |.define CARG3, r8 34 |.define CARG4, r9 35 |.define CARG1d, ecx 36 |.define CARG2d, edx 37 |.define CARG3d, r8d 38 |.define CARG4d, r9d 39 |.define FCARG1a, CARG1 // Simulate x86 fastcall. 40 |.define FCARG2a, CARG2 41 |.define FCARG1d, CARG1d 42 |.define FCARG2d, CARG2d 43 |.define SPAD, 0x58 // padding for CPU stack alignment 44 |.define NR_SPAD, 0x58 // padding for CPU stack alignment 45 |.define T3, [r4+0x50] // Used to store old value of IP 46 |.define T2, [r4+0x48] // Used to store old value of FP 47 |.define T1, [r4+0x40] 48 |.define A6, [r4+0x28] // preallocated slot for 6-th argument 49 |.define A5, [r4+0x20] // preallocated slot for 5-th argument 50|.elif X64 51 |.define FP, r14 52 |.define IP, r15 53 |.define IPl, r15d 54 |.define RX, r15 // the same as VM IP reused as a general purpose reg 55 |.define CARG1, rdi // x64/POSIX C call arguments. 56 |.define CARG2, rsi 57 |.define CARG3, rdx 58 |.define CARG4, rcx 59 |.define CARG5, r8 60 |.define CARG6, r9 61 |.define CARG1d, edi 62 |.define CARG2d, esi 63 |.define CARG3d, edx 64 |.define CARG4d, ecx 65 |.define CARG5d, r8d 66 |.define CARG6d, r9d 67 |.define FCARG1a, CARG1 // Simulate x86 fastcall. 68 |.define FCARG2a, CARG2 69 |.define FCARG1d, CARG1d 70 |.define FCARG2d, CARG2d 71 |.define SPAD, 0x18 // padding for CPU stack alignment 72 |.define NR_SPAD, 0x28 // padding for CPU stack alignment 73 |.define T3, [r4+0x20] // Used to store old value of IP (CALL VM only) 74 |.define T2, [r4+0x18] // Used to store old value of FP (CALL VM only) 75 |.define T1, [r4] 76|.else 77 |.define FP, esi 78 |.define IP, edi 79 |.define IPl, edi 80 |.define RX, edi // the same as VM IP reused as a general purpose reg 81 |.define FCARG1a, ecx // x86 fastcall arguments. 82 |.define FCARG2a, edx 83 |.define FCARG1d, ecx 84 |.define FCARG2d, edx 85 |.define SPAD, 0x1c // padding for CPU stack alignment 86 |.define NR_SPAD, 0x1c // padding for CPU stack alignment 87 |.define T3, [r4+0x18] // Used to store old value of IP (CALL VM only) 88 |.define T2, [r4+0x14] // Used to store old value of FP (CALL VM only) 89 |.define T1, [r4] 90 |.define A4, [r4+0xC] // preallocated slots for arguments of "cdecl" functions (intersect with T1) 91 |.define A3, [r4+0x8] 92 |.define A2, [r4+0x4] 93 |.define A1, [r4] 94|.endif 95 96|.define HYBRID_SPAD, 16 // padding for stack alignment 97 98#ifdef _WIN64 99# define TMP_ZVAL_OFFSET 0x20 100#else 101# define TMP_ZVAL_OFFSET 0 102#endif 103 104#define DASM_ALIGNMENT 16 105 106/* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to 107 * guarantee proper alignment of 128-bit SSE data allocated on stack. 108 * With broken alignment any execution of SSE code, including calls to 109 * memcpy() and others, may lead to crash. 110 */ 111 112const char* zend_reg_name[] = { 113#if defined(__x86_64__) || defined(_M_X64) 114 "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", 115 "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", 116 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", 117 "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15" 118#else 119 "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", 120 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" 121#endif 122}; 123 124/* Simulate x86 fastcall */ 125#ifdef _WIN64 126# define ZREG_FCARG1 ZREG_RCX 127# define ZREG_FCARG2 ZREG_RDX 128#elif defined(__x86_64__) 129# define ZREG_FCARG1 ZREG_RDI 130# define ZREG_FCARG2 ZREG_RSI 131#else 132# define ZREG_FCARG1 ZREG_RCX 133# define ZREG_FCARG2 ZREG_RDX 134#endif 135 136|.type EX, zend_execute_data, FP 137|.type OP, zend_op 138|.type ZVAL, zval 139|.actionlist dasm_actions 140|.globals zend_lb 141|.section code, cold_code, jmp_table 142 143static void* dasm_labels[zend_lb_MAX]; 144 145#if ZTS 146static size_t tsrm_ls_cache_tcb_offset = 0; 147static size_t tsrm_tls_index; 148static size_t tsrm_tls_offset; 149#endif 150 151#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) 152 153#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) 154 155/* Call range is before or after 2GB */ 156#define MAY_USE_32BIT_ADDR(addr) \ 157 (IS_SIGNED_32BIT((char*)(addr) - (char*)dasm_buf) && \ 158 IS_SIGNED_32BIT((char*)(addr) - (char*)dasm_end)) 159 160#define CAN_USE_AVX() (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) 161 162/* Not Implemented Yet */ 163|.macro NIY 164|| //ZEND_ASSERT(0); 165| int3 166|.endmacro 167 168|.macro NIY_STUB 169|| //ZEND_ASSERT(0); 170| int3 171|.endmacro 172 173|.macro ADD_HYBRID_SPAD 174||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 175| add r4, HYBRID_SPAD 176||#endif 177|.endmacro 178 179|.macro SUB_HYBRID_SPAD 180||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 181| sub r4, HYBRID_SPAD 182||#endif 183|.endmacro 184 185|.macro LOAD_ADDR, reg, addr 186| .if X64 187|| if (IS_SIGNED_32BIT(addr)) { 188| mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 <imm-32-bit> 189|| } else { 190| mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 <imm-64-bit> 191|| } 192| .else 193| mov reg, ((ptrdiff_t)addr) 194| .endif 195|.endmacro 196 197|.macro LOAD_TSRM_CACHE, reg 198| .if X64WIN 199| gs 200| mov reg, aword [0x58] 201| mov reg, aword [reg+tsrm_tls_index] 202| mov reg, aword [reg+tsrm_tls_offset] 203| .elif WIN 204| fs 205| mov reg, aword [0x2c] 206| mov reg, aword [reg+tsrm_tls_index] 207| mov reg, aword [reg+tsrm_tls_offset] 208| .elif X64APPLE 209| gs 210|| if (tsrm_ls_cache_tcb_offset) { 211| mov reg, aword [tsrm_ls_cache_tcb_offset] 212|| } else { 213| mov reg, aword [tsrm_tls_index] 214| mov reg, aword [reg+tsrm_tls_offset] 215|| } 216| .elif X64 217| fs 218|| if (tsrm_ls_cache_tcb_offset) { 219| mov reg, aword [tsrm_ls_cache_tcb_offset] 220|| } else { 221| mov reg, [0x8] 222| mov reg, aword [reg+tsrm_tls_index] 223| mov reg, aword [reg+tsrm_tls_offset] 224|| } 225| .else 226| gs 227|| if (tsrm_ls_cache_tcb_offset) { 228| mov reg, aword [tsrm_ls_cache_tcb_offset] 229|| } else { 230| mov reg, [0x4] 231| mov reg, aword [reg+tsrm_tls_index] 232| mov reg, aword [reg+tsrm_tls_offset] 233|| } 234| .endif 235|.endmacro 236 237|.macro LOAD_ADDR_ZTS, reg, struct, field 238| .if ZTS 239| LOAD_TSRM_CACHE reg 240| lea reg, aword [reg + (struct.._offset + offsetof(zend_..struct, field))] 241| .else 242| LOAD_ADDR reg, &struct.field 243| .endif 244|.endmacro 245 246|.macro PUSH_ADDR, addr, tmp_reg 247| .if X64 248|| if (IS_SIGNED_32BIT(addr)) { 249| push ((ptrdiff_t)addr) 250|| } else { 251| mov64 tmp_reg, ((ptrdiff_t)addr) 252| push tmp_reg 253|| } 254| .else 255| push ((ptrdiff_t)addr) 256| .endif 257|.endmacro 258 259|.macro ADDR_STORE, mem, addr, tmp_reg 260| .if X64 261|| if (IS_SIGNED_32BIT(addr)) { 262| mov mem, ((ptrdiff_t)addr) 263|| } else { 264| mov64 tmp_reg, ((ptrdiff_t)addr) 265| mov mem, tmp_reg 266|| } 267| .else 268| mov mem, ((ptrdiff_t)addr) 269| .endif 270|.endmacro 271 272|.macro ADDR_CMP, mem, addr, tmp_reg 273| .if X64 274|| if (IS_SIGNED_32BIT(addr)) { 275| cmp mem, ((ptrdiff_t)addr) 276|| } else { 277| mov64 tmp_reg, ((ptrdiff_t)addr) 278| cmp mem, tmp_reg 279|| } 280| .else 281| cmp mem, ((ptrdiff_t)addr) 282| .endif 283|.endmacro 284 285|.macro PUSH_ADDR_ZTS, struct, field, tmp_reg 286| .if ZTS 287| LOAD_TSRM_CACHE tmp_reg 288| lea tmp_reg, aword [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))] 289| push tmp_reg 290| .else 291| PUSH_ADDR &struct.field, tmp_reg 292| .endif 293|.endmacro 294 295|.macro _MEM_OP, mem_ins, prefix, addr, op2, tmp_reg 296| .if X64 297|| if (IS_SIGNED_32BIT(addr)) { 298| mem_ins prefix [addr], op2 299|| } else { 300| mov64 tmp_reg, ((ptrdiff_t)addr) 301| mem_ins prefix [tmp_reg], op2 302|| } 303| .else 304| mem_ins prefix [addr], op2 305| .endif 306|.endmacro 307 308|.macro MEM_LOAD_OP, mem_ins, reg, prefix, addr, tmp_reg 309| .if X64 310|| if (IS_SIGNED_32BIT(addr)) { 311| mem_ins reg, prefix [addr] 312|| } else { 313| mov64 tmp_reg, ((ptrdiff_t)addr) 314| mem_ins reg, prefix [tmp_reg] 315|| } 316| .else 317| mem_ins reg, prefix [addr] 318| .endif 319|.endmacro 320 321|.macro MEM_LOAD, op1, prefix, addr, tmp_reg 322| MEM_LOAD_OP mov, op1, prefix, addr, tmp_reg 323|.endmacro 324 325|.macro _MEM_OP_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg 326| .if ZTS 327| LOAD_TSRM_CACHE tmp_reg 328| mem_ins prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))], op2 329| .else 330| _MEM_OP mem_ins, prefix, &struct.field, op2, tmp_reg 331| .endif 332|.endmacro 333 334|.macro MEM_STORE_ZTS, prefix, struct, field, op2, tmp_reg 335| _MEM_OP_ZTS mov, prefix, struct, field, op2, tmp_reg 336|.endmacro 337 338|.macro MEM_CMP_ZTS, prefix, struct, field, op2, tmp_reg 339| _MEM_OP_ZTS cmp, prefix, struct, field, op2, tmp_reg 340|.endmacro 341 342|.macro MEM_UPDATE_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg 343| _MEM_OP_ZTS mem_ins, prefix, struct, field, op2, tmp_reg 344|.endmacro 345 346|.macro MEM_LOAD_OP_ZTS, mem_ins, reg, prefix, struct, field, tmp_reg 347| .if ZTS 348| LOAD_TSRM_CACHE tmp_reg 349| mem_ins reg, prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))] 350| .else 351| MEM_LOAD_OP mem_ins, reg, prefix, &struct.field, tmp_reg 352| .endif 353|.endmacro 354 355|.macro MEM_LOAD_ZTS, reg, prefix, struct, field, tmp_reg 356| MEM_LOAD_OP_ZTS mov, reg, prefix, struct, field, tmp_reg 357|.endmacro 358 359|.macro EXT_CALL, func, tmp_reg 360| .if X64 361|| if (MAY_USE_32BIT_ADDR(func)) { 362| call qword &func 363|| } else { 364| LOAD_ADDR tmp_reg, func 365| call tmp_reg 366|| } 367| .else 368| call dword &func 369| .endif 370|.endmacro 371 372|.macro EXT_JMP, func, tmp_reg 373| .if X64 374|| if (MAY_USE_32BIT_ADDR(func)) { 375| jmp qword &func 376|| } else { 377| LOAD_ADDR tmp_reg, func 378| jmp tmp_reg 379|| } 380| .else 381| jmp dword &func 382| .endif 383|.endmacro 384 385|.macro SAVE_IP 386|| if (GCC_GLOBAL_REGS) { 387| mov aword EX->opline, IP 388|| } 389|.endmacro 390 391|.macro LOAD_IP 392|| if (GCC_GLOBAL_REGS) { 393| mov IP, aword EX->opline 394|| } 395|.endmacro 396 397|.macro LOAD_IP_ADDR, addr 398|| if (GCC_GLOBAL_REGS) { 399| LOAD_ADDR IP, addr 400|| } else { 401| ADDR_STORE aword EX->opline, addr, RX 402|| } 403|.endmacro 404 405|.macro LOAD_IP_ADDR_ZTS, struct, field 406| .if ZTS 407|| if (GCC_GLOBAL_REGS) { 408| LOAD_TSRM_CACHE IP 409| lea IP, aword [IP + (struct.._offset + offsetof(zend_..struct, field))] 410|| } else { 411| LOAD_TSRM_CACHE RX 412| lea RX, aword [RX + (struct.._offset + offsetof(zend_..struct, field))] 413| mov aword EX->opline, RX 414|| } 415| .else 416| LOAD_IP_ADDR &struct.field 417| .endif 418|.endmacro 419 420|.macro GET_IP, reg 421|| if (GCC_GLOBAL_REGS) { 422| mov reg, IP 423|| } else { 424| mov reg, aword EX->opline 425|| } 426|.endmacro 427 428|.macro ADD_IP, val 429|| if (GCC_GLOBAL_REGS) { 430| add IP, val 431|| } else { 432| add aword EX->opline, val 433|| } 434|.endmacro 435 436|.macro JMP_IP 437|| if (GCC_GLOBAL_REGS) { 438| jmp aword [IP] 439|| } else { 440| mov r0, aword EX:FCARG1a->opline 441| jmp aword [r0] 442|| } 443|.endmacro 444 445/* In 64-bit build we compare only low 32-bits. 446 * x86_64 cmp instruction doesn't support immediate 64-bit operand, and full 447 * comparison would require an additional load of 64-bit address into register. 448 * This is not a problem at all, while JIT buffer size is less than 4GB. 449 */ 450|.macro CMP_IP, addr 451|| if (GCC_GLOBAL_REGS) { 452| cmp IPl, addr 453|| } else { 454| cmp dword EX->opline, addr 455|| } 456|.endmacro 457 458|.macro LOAD_ZVAL_ADDR, reg, addr 459|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 460| LOAD_ADDR reg, Z_ZV(addr) 461|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 462|| if (Z_OFFSET(addr)) { 463| lea reg, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 464|| } else { 465| mov reg, Ra(Z_REG(addr)) 466|| } 467|| } else { 468|| ZEND_UNREACHABLE(); 469|| } 470|.endmacro 471 472|.macro PUSH_ZVAL_ADDR, addr, tmp_reg 473|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 474| PUSH_ADDR Z_ZV(addr), tmp_reg 475|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 476|| if (Z_OFFSET(addr)) { 477| lea tmp_reg, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 478| push tmp_reg 479|| } else { 480| push Ra(Z_REG(addr)) 481|| } 482|| } else { 483|| ZEND_UNREACHABLE(); 484|| } 485|.endmacro 486 487|.macro GET_Z_TYPE_INFO, reg, zv 488| mov reg, dword [zv+offsetof(zval,u1.type_info)] 489|.endmacro 490 491|.macro SET_Z_TYPE_INFO, zv, type 492| mov dword [zv+offsetof(zval,u1.type_info)], type 493|.endmacro 494 495|.macro GET_ZVAL_TYPE, reg, addr 496|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 497| mov reg, byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.v.type)] 498|.endmacro 499 500|.macro GET_ZVAL_TYPE_INFO, reg, addr 501|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 502| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)] 503|.endmacro 504 505|.macro SET_ZVAL_TYPE_INFO, addr, type 506|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 507| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)], type 508|.endmacro 509 510|.macro GET_Z_PTR, reg, zv 511| mov reg, aword [zv] 512|.endmacro 513 514|.macro GET_Z_W2, reg, zv 515| mov reg, dword [zv+4] 516|.endmacro 517 518|.macro SET_Z_W2, zv, reg 519| mov dword [zv+4], reg 520|.endmacro 521 522|.macro GET_ZVAL_PTR, reg, addr 523|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 524| mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 525|.endmacro 526 527|.macro SET_ZVAL_PTR, addr, val 528|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 529| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], val 530|.endmacro 531 532|.macro GET_ZVAL_W2, reg, addr 533|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 534| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4] 535|.endmacro 536 537|.macro SET_ZVAL_W2, addr, val 538|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 539| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4], val 540|.endmacro 541 542|.macro UNDEF_OPLINE_RESULT 543| mov r0, EX->opline 544| mov eax, dword OP:r0->result.var 545| SET_Z_TYPE_INFO FP + r0, IS_UNDEF 546|.endmacro 547 548|.macro UNDEF_OPLINE_RESULT_IF_USED 549| test byte OP:RX->result_type, (IS_TMP_VAR|IS_VAR) 550| jz >1 551| mov eax, dword OP:RX->result.var 552| SET_Z_TYPE_INFO FP + r0, IS_UNDEF 553|1: 554|.endmacro 555 556|.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2 557|| if (CAN_USE_AVX()) { 558| avx_ins op1, op2 559|| } else { 560| sse_ins op1, op2 561|| } 562|.endmacro 563 564|.macro SSE_OP, sse_ins, reg, addr, tmp_reg 565|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 566| MEM_LOAD_OP sse_ins, xmm(reg-ZREG_XMM0), qword, Z_ZV(addr), tmp_reg 567|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 568| sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 569|| } else if (Z_MODE(addr) == IS_REG) { 570| sse_ins xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 571|| } else { 572|| ZEND_UNREACHABLE(); 573|| } 574|.endmacro 575 576|.macro DOUBLE_CMP, reg, addr 577|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 578| .if X64 579|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 580| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 581|| } else { 582| LOAD_ADDR r0, Z_ZV(addr) 583| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [r0] 584|| } 585| .else 586| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 587| .endif 588|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 589| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 590|| } else if (Z_MODE(addr) == IS_REG) { 591| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 592|| } else { 593|| ZEND_UNREACHABLE(); 594|| } 595|.endmacro 596 597|.macro DOUBLE_GET_LONG, reg, lval, tmp_reg 598|| if (lval == 0) { 599|| if (CAN_USE_AVX()) { 600| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 601|| } else { 602| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 603|| } 604|| } else { 605|.if X64 606|| if (!IS_SIGNED_32BIT(lval)) { 607| mov64 Ra(tmp_reg), lval 608|| } else { 609| mov Ra(tmp_reg), lval 610|| } 611|.else 612| mov Ra(tmp_reg), lval 613|.endif 614|| if (CAN_USE_AVX()) { 615| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 616| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(tmp_reg) 617|| } else { 618| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 619| cvtsi2sd, xmm(reg-ZREG_XMM0), Ra(tmp_reg) 620|| } 621|| } 622|.endmacro 623 624|.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg 625|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 626| DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg 627|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 628|| if (CAN_USE_AVX()) { 629| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 630| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 631|| } else { 632| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 633| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 634|| } 635|| } else if (Z_MODE(addr) == IS_REG) { 636|| if (CAN_USE_AVX()) { 637| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 638| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) 639|| } else { 640| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 641| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) 642|| } 643|| } else { 644|| ZEND_UNREACHABLE(); 645|| } 646|.endmacro 647 648|.macro DOUBLE_GET_ZVAL_DVAL, reg, addr 649|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { 650|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 651| .if X64 652|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 653| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 654|| } else { 655| LOAD_ADDR r0, Z_ZV(addr) 656| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [r0] 657|| } 658| .else 659| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 660| .endif 661|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 662| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 663|| } else if (Z_MODE(addr) == IS_REG) { 664| SSE_AVX_INS movaps, vmovaps, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 665|| } else { 666|| ZEND_UNREACHABLE(); 667|| } 668|| } 669|.endmacro 670 671|.macro SSE_MATH, opcode, reg, addr, tmp_reg 672|| switch (opcode) { 673|| case ZEND_ADD: 674| SSE_OP addsd, reg, addr, tmp_reg 675|| break; 676|| case ZEND_SUB: 677| SSE_OP subsd, reg, addr, tmp_reg 678|| break; 679|| case ZEND_MUL: 680| SSE_OP mulsd, reg, addr, tmp_reg 681|| break; 682|| case ZEND_DIV: 683| SSE_OP divsd, reg, addr, tmp_reg 684|| break; 685|| } 686|.endmacro 687 688|.macro SSE_MATH_REG, opcode, dst_reg, src_reg 689|| switch (opcode) { 690|| case ZEND_ADD: 691| addsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 692|| break; 693|| case ZEND_SUB: 694| subsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 695|| break; 696|| case ZEND_MUL: 697| mulsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 698|| break; 699|| case ZEND_DIV: 700| divsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 701|| break; 702|| } 703|.endmacro 704 705|.macro DOUBLE_SET_ZVAL_DVAL, addr, reg 706|| if (Z_MODE(addr) == IS_REG) { 707|| if (reg != Z_REG(addr)) { 708| SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0) 709|| } 710|| } else { 711|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 712| SSE_AVX_INS movsd, vmovsd, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0) 713|| } 714|.endmacro 715 716|.macro AVX_OP, avx_ins, reg, op1_reg, addr, tmp_reg 717|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 718| .if X64 719|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 720| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Z_ZV(addr)] 721|| } else { 722| mov64 tmp_reg, ((ptrdiff_t)Z_ZV(addr)) 723| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [tmp_reg] 724|| } 725| .else 726| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [addr] 727| .endif 728|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 729| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 730|| } else if (Z_MODE(addr) == IS_REG) { 731| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 732|| } else { 733|| ZEND_UNREACHABLE(); 734|| } 735|.endmacro 736 737|.macro AVX_MATH, opcode, reg, op1_reg, addr, tmp_reg 738|| switch (opcode) { 739|| case ZEND_ADD: 740| AVX_OP vaddsd, reg, op1_reg, addr, tmp_reg 741|| break; 742|| case ZEND_SUB: 743| AVX_OP vsubsd, reg, op1_reg, addr, tmp_reg 744|| break; 745|| case ZEND_MUL: 746| AVX_OP vmulsd, reg, op1_reg, addr, tmp_reg 747|| break; 748|| case ZEND_DIV: 749| AVX_OP vdivsd, reg, op1_reg, addr, tmp_reg 750|| break; 751|| } 752|.endmacro 753 754|.macro AVX_MATH_REG, opcode, dst_reg, op1_reg, src_reg 755|| switch (opcode) { 756|| case ZEND_ADD: 757| vaddsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 758|| break; 759|| case ZEND_SUB: 760| vsubsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 761|| break; 762|| case ZEND_MUL: 763| vmulsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 764|| break; 765|| case ZEND_DIV: 766| vdivsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 767|| break; 768|| } 769|.endmacro 770 771|.macro LONG_OP, long_ins, reg, addr, tmp_reg 772|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 773| .if X64 774|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { 775| mov64 tmp_reg, Z_LVAL_P(Z_ZV(addr)) 776| long_ins Ra(reg), tmp_reg 777|| } else { 778| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) 779|| } 780| .else 781| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) 782| .endif 783|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 784| long_ins Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 785|| } else if (Z_MODE(addr) == IS_REG) { 786| long_ins Ra(reg), Ra(Z_REG(addr)) 787|| } else { 788|| ZEND_UNREACHABLE(); 789|| } 790|.endmacro 791 792|.macro LONG_OP_WITH_32BIT_CONST, long_ins, op1_addr, lval 793|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { 794| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 795|| } else if (Z_MODE(op1_addr) == IS_REG) { 796| long_ins Ra(Z_REG(op1_addr)), lval 797|| } else { 798|| ZEND_UNREACHABLE(); 799|| } 800|.endmacro 801 802|.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval 803|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { 804| .if X64 805|| if (!IS_SIGNED_32BIT(lval)) { 806| mov64 r0, lval 807| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0 808|| } else { 809| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 810|| } 811| .else 812| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 813| .endif 814|| } else if (Z_MODE(op1_addr) == IS_REG) { 815| .if X64 816|| if (!IS_SIGNED_32BIT(lval)) { 817| mov64 r0, lval 818| long_ins Ra(Z_REG(op1_addr)), r0 819|| } else { 820| long_ins Ra(Z_REG(op1_addr)), lval 821|| } 822| .else 823| long_ins Ra(Z_REG(op1_addr)), lval 824| .endif 825|| } else { 826|| ZEND_UNREACHABLE(); 827|| } 828|.endmacro 829 830|.macro GET_ZVAL_LVAL, reg, addr 831|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 832|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { 833| xor Ra(reg), Ra(reg) 834|| } else { 835| .if X64 836|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { 837| mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr)) 838|| } else { 839| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) 840|| } 841| .else 842| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) 843| .endif 844|| } 845|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 846| mov Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 847|| } else if (Z_MODE(addr) == IS_REG) { 848|| if (reg != Z_REG(addr)) { 849| mov Ra(reg), Ra(Z_REG(addr)) 850|| } 851|| } else { 852|| ZEND_UNREACHABLE(); 853|| } 854|.endmacro 855 856|.macro LONG_MATH, opcode, reg, addr, tmp_reg 857|| switch (opcode) { 858|| case ZEND_ADD: 859| LONG_OP add, reg, addr, Ra(tmp_reg) 860|| break; 861|| case ZEND_SUB: 862| LONG_OP sub, reg, addr, Ra(tmp_reg) 863|| break; 864|| case ZEND_MUL: 865| LONG_OP imul, reg, addr, Ra(tmp_reg) 866|| break; 867|| case ZEND_BW_OR: 868| LONG_OP or, reg, addr, Ra(tmp_reg) 869|| break; 870|| case ZEND_BW_AND: 871| LONG_OP and, reg, addr, Ra(tmp_reg) 872|| break; 873|| case ZEND_BW_XOR: 874| LONG_OP xor, reg, addr, Ra(tmp_reg) 875|| break; 876|| default: 877|| ZEND_UNREACHABLE(); 878|| } 879|.endmacro 880 881|.macro LONG_MATH_REG, opcode, dst_reg, src_reg 882|| switch (opcode) { 883|| case ZEND_ADD: 884| add dst_reg, src_reg 885|| break; 886|| case ZEND_SUB: 887| sub dst_reg, src_reg 888|| break; 889|| case ZEND_MUL: 890| imul dst_reg, src_reg 891|| break; 892|| case ZEND_BW_OR: 893| or dst_reg, src_reg 894|| break; 895|| case ZEND_BW_AND: 896| and dst_reg, src_reg 897|| break; 898|| case ZEND_BW_XOR: 899| xor dst_reg, src_reg 900|| break; 901|| default: 902|| ZEND_UNREACHABLE(); 903|| } 904|.endmacro 905 906|.macro SET_ZVAL_LVAL, addr, lval 907|| if (Z_MODE(addr) == IS_REG) { 908| mov Ra(Z_REG(addr)), lval 909|| } else { 910|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 911| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval 912|| } 913|.endmacro 914 915|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg 916|| if (Z_TYPE_P(zv) > IS_TRUE) { 917|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 918|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; 919|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { 920|| if (CAN_USE_AVX()) { 921| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 922|| } else { 923| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 924|| } 925| .if X64 926|| } else if (!IS_SIGNED_32BIT(zv)) { 927| mov64 Ra(tmp_reg), ((uintptr_t)zv) 928| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] 929| .endif 930|| } else { 931| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] 932|| } 933| DOUBLE_SET_ZVAL_DVAL dst_addr, dst_reg 934|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 935|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; 936| DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), ZREG_R0 937| DOUBLE_SET_ZVAL_DVAL dst_addr, dst_reg 938|| } else if (Z_LVAL_P(zv) == 0 && Z_MODE(dst_addr) == IS_REG) { 939| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) 940|| } else { 941| .if X64 942|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 943|| if (Z_MODE(dst_addr) == IS_REG) { 944| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) 945|| } else { 946| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) 947| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) 948|| } 949|| } else { 950| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 951|| } 952| .else 953| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 954| .endif 955|| } 956|| } 957|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 958|| if (dst_def_info == MAY_BE_DOUBLE) { 959|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 960| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE 961|| } 962|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 963| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) 964|| } 965|| } 966|.endmacro 967 968|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg 969|| if (Z_TYPE_P(zv) > IS_TRUE) { 970|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 971|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? 972|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0); 973|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { 974|| if (CAN_USE_AVX()) { 975| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 976|| } else { 977| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 978|| } 979| .if X64 980|| } else if (!IS_SIGNED_32BIT(zv)) { 981| mov64 Ra(tmp_reg), ((uintptr_t)zv) 982| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] 983| .endif 984|| } else { 985| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] 986|| } 987| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 988| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 989|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 990|| if (Z_MODE(dst_addr) == IS_REG) { 991| DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), ZREG_R0 992| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) 993|| } else if (Z_MODE(res_addr) == IS_REG) { 994| DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), ZREG_R0 995| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) 996|| } else { 997| DOUBLE_GET_LONG ZREG_XMM0, Z_LVAL_P(zv), ZREG_R0 998| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 999| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 1000|| } 1001|| } else if (Z_LVAL_P(zv) == 0 && (Z_MODE(dst_addr) == IS_REG || Z_MODE(res_addr) == IS_REG)) { 1002|| if (Z_MODE(dst_addr) == IS_REG) { 1003| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) 1004| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1005|| } else { 1006| xor Ra(Z_REG(res_addr)), Ra(Z_REG(res_addr)) 1007| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1008|| } 1009|| } else { 1010| .if X64 1011|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1012|| if (Z_MODE(dst_addr) == IS_REG) { 1013| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) 1014| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1015|| } else if (Z_MODE(res_addr) == IS_REG) { 1016| mov64 Ra(Z_REG(res_addr)), ((uintptr_t)Z_LVAL_P(zv)) 1017| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1018|| } else { 1019| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) 1020| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) 1021| SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 1022|| } 1023|| } else if (Z_MODE(dst_addr) == IS_REG) { 1024| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1025| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1026|| } else if (Z_MODE(res_addr) == IS_REG) { 1027| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1028| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1029|| } else { 1030| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1031| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1032|| } 1033| .else 1034|| if (Z_MODE(dst_addr) == IS_REG) { 1035| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1036| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1037|| } else if (Z_MODE(res_addr) == IS_REG) { 1038| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1039| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1040|| } else { 1041| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1042| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1043|| } 1044| .endif 1045|| } 1046|| } 1047|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1048|| if (dst_def_info == MAY_BE_DOUBLE) { 1049|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1050| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE 1051|| } 1052|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1053| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) 1054|| } 1055|| } 1056|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1057|| if (dst_def_info == MAY_BE_DOUBLE) { 1058| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 1059|| } else { 1060| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv) 1061|| } 1062|| } 1063|.endmacro 1064 1065/* the same as above, but "src" may overlap with "tmp_reg1" */ 1066|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1067| ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1068|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1069|| !(src_info & MAY_BE_GUARD) && 1070|| has_concrete_type(src_info & MAY_BE_ANY)) { 1071|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1072|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) { 1073|| uint8_t type = concrete_type(src_info); 1074| SET_ZVAL_TYPE_INFO dst_addr, type 1075|| } 1076|| } 1077|| } else { 1078| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr 1079| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) 1080|| } 1081|.endmacro 1082 1083|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1084|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1085|| if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) { 1086|| if (Z_MODE(src_addr) == IS_REG) { 1087|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1088| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) 1089|| } 1090|| } else if (Z_MODE(dst_addr) == IS_REG) { 1091| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr 1092|| } else { 1093| GET_ZVAL_LVAL tmp_reg2, src_addr 1094| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) 1095|| } 1096|| } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 1097|| if (Z_MODE(src_addr) == IS_REG) { 1098| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) 1099|| } else if (Z_MODE(dst_addr) == IS_REG) { 1100| DOUBLE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr 1101|| } else { 1102| DOUBLE_GET_ZVAL_DVAL ZREG_XMM0, src_addr 1103| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1104|| } 1105|| } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) { 1106| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1107| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1108|| } else { 1109| .if X64 1110| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1111| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1112| .else 1113|| if ((tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr))) { 1114| GET_ZVAL_W2 Ra(tmp_reg2), src_addr 1115| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) 1116| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1117| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1118|| } else { 1119| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1120| GET_ZVAL_W2 Ra(tmp_reg1), src_addr 1121| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1122| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) 1123|| } 1124| .endif 1125|| } 1126|| } 1127|.endmacro 1128 1129|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, tmp_reg1, tmp_reg2 1130|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1131|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) { 1132|| if (Z_MODE(src_addr) == IS_REG) { 1133|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1134| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) 1135|| } 1136|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) { 1137| SET_ZVAL_LVAL res_addr, Ra(Z_REG(src_addr)) 1138|| } 1139|| } else if (Z_MODE(dst_addr) == IS_REG) { 1140| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr 1141|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) { 1142| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1143|| } 1144|| } else if (Z_MODE(res_addr) == IS_REG) { 1145| GET_ZVAL_LVAL Z_REG(res_addr), src_addr 1146| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1147|| } else { 1148| GET_ZVAL_LVAL tmp_reg2, src_addr 1149| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) 1150| SET_ZVAL_LVAL res_addr, Ra(tmp_reg2) 1151|| } 1152|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 1153|| if (Z_MODE(src_addr) == IS_REG) { 1154| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) 1155| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(src_addr) 1156|| } else if (Z_MODE(dst_addr) == IS_REG) { 1157| DOUBLE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr 1158| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) 1159|| } else if (Z_MODE(res_addr) == IS_REG) { 1160| DOUBLE_GET_ZVAL_DVAL Z_REG(res_addr), src_addr 1161| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) 1162|| } else { 1163| DOUBLE_GET_ZVAL_DVAL ZREG_XMM0, src_addr 1164| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1165| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 1166|| } 1167|| } else if (!(src_info & MAY_BE_DOUBLE)) { 1168| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1169| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1170| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1171|| } else { 1172| .if X64 1173| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1174| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1175| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1176| .else 1177|| if (tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) { 1178| GET_ZVAL_W2 Ra(tmp_reg2), src_addr 1179| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) 1180| SET_ZVAL_W2 res_addr, Ra(tmp_reg2) 1181| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1182| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1183| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1184|| } else { 1185| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1186| GET_ZVAL_W2 Ra(tmp_reg1), src_addr 1187| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1188| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1189| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) 1190| SET_ZVAL_W2 res_addr, Ra(tmp_reg1) 1191|| } 1192| .endif 1193|| } 1194|| } 1195|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1196|| has_concrete_type(src_info & MAY_BE_ANY)) { 1197|| uint8_t type = concrete_type(src_info); 1198|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1199|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { 1200| SET_ZVAL_TYPE_INFO dst_addr, type 1201|| } 1202|| } 1203|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1204| SET_ZVAL_TYPE_INFO res_addr, type 1205|| } 1206|| } else { 1207| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr 1208| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) 1209| SET_ZVAL_TYPE_INFO res_addr, Rd(tmp_reg1) 1210|| } 1211|.endmacro 1212 1213|.macro IF_UNDEF, type_reg, label 1214| test type_reg, type_reg 1215| je label 1216|.endmacro 1217 1218|.macro IF_TYPE, type, val, label 1219| cmp type, val 1220| je label 1221|.endmacro 1222 1223|.macro IF_NOT_TYPE, type, val, label 1224| cmp type, val 1225| jne label 1226|.endmacro 1227 1228|.macro IF_Z_TYPE, zv, val, label 1229| IF_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label 1230|.endmacro 1231 1232|.macro IF_NOT_Z_TYPE, zv, val, label 1233| IF_NOT_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label 1234|.endmacro 1235 1236|.macro CMP_ZVAL_TYPE, addr, val 1237|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1238| cmp byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val 1239|.endmacro 1240 1241|.macro IF_ZVAL_TYPE, addr, val, label 1242|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1243| IF_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label 1244|.endmacro 1245 1246|.macro IF_NOT_ZVAL_TYPE, addr, val, label 1247|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1248| IF_NOT_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label 1249|.endmacro 1250 1251|.macro IF_FLAGS, type_flags, mask, label 1252| test type_flags, mask 1253| jnz label 1254|.endmacro 1255 1256|.macro IF_NOT_FLAGS, type_flags, mask, label 1257| test type_flags, mask 1258| jz label 1259|.endmacro 1260 1261|.macro IF_REFCOUNTED, type_flags, label 1262| IF_FLAGS type_flags, IS_TYPE_REFCOUNTED, label 1263|.endmacro 1264 1265|.macro IF_NOT_REFCOUNTED, type_flags, label 1266| //IF_NOT_FLAGS type_flags, IS_TYPE_REFCOUNTED, label 1267| test type_flags, type_flags 1268| jz label 1269|.endmacro 1270 1271|.macro IF_ZVAL_FLAGS, addr, mask, label 1272|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1273| IF_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label 1274|.endmacro 1275 1276|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label 1277|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1278| IF_NOT_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label 1279|.endmacro 1280 1281|.macro IF_ZVAL_REFCOUNTED, addr, label 1282| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label 1283|.endmacro 1284 1285|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label 1286| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label 1287|.endmacro 1288 1289|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label 1290| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label 1291|.endmacro 1292 1293|.macro GC_ADDREF, zv 1294| add dword [zv], 1 1295|.endmacro 1296 1297|.macro GC_DELREF, zv 1298| sub dword [zv], 1 1299|.endmacro 1300 1301|.macro IF_GC_MAY_NOT_LEAK, ptr, label 1302| test dword [ptr+4],(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)) 1303| jne label 1304|.endmacro 1305 1306|.macro ADDREF_CONST, zv, tmp_reg 1307| .if X64 1308|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1309| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) 1310| add dword [tmp_reg], 1 1311|| } else { 1312| add dword [Z_LVAL_P(zv)], 1 1313|| } 1314| .else 1315| add dword [Z_LVAL_P(zv)], 1 1316| .endif 1317|.endmacro 1318 1319|.macro ADDREF_CONST_2, zv, tmp_reg 1320| .if X64 1321|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1322| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) 1323| add dword [tmp_reg], 2 1324|| } else { 1325| add dword [Z_LVAL_P(zv)], 2 1326|| } 1327| .else 1328| add dword [Z_LVAL_P(zv)], 2 1329| .endif 1330|.endmacro 1331 1332|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg 1333|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1334|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1335| IF_NOT_REFCOUNTED type_flags_reg, >1 1336|| } 1337| GC_ADDREF value_ptr_reg 1338|1: 1339|| } 1340|.endmacro 1341 1342|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg 1343|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1344|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1345| IF_NOT_REFCOUNTED type_flags_reg, >1 1346|| } 1347| add dword [value_ptr_reg], 2 1348|1: 1349|| } 1350|.endmacro 1351 1352|.macro ZVAL_DEREF, reg, info 1353|| if (info & MAY_BE_REF) { 1354| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1 1355| GET_Z_PTR reg, reg 1356| add reg, offsetof(zend_reference, val) 1357|1: 1358|| } 1359|.endmacro 1360 1361|.macro SET_EX_OPLINE, op, tmp_reg 1362|| if (op == last_valid_opline) { 1363|| zend_jit_use_last_valid_opline(); 1364| SAVE_IP 1365|| } else { 1366| ADDR_STORE aword EX->opline, op, tmp_reg 1367|| if (!GCC_GLOBAL_REGS) { 1368|| zend_jit_reset_last_valid_opline(); 1369|| } 1370|| } 1371|.endmacro 1372 1373// zval should be in FCARG1a 1374|.macro ZVAL_DTOR_FUNC, var_info, opline // arg1 must be in FCARG1a 1375|| do { 1376|| if (!((var_info) & MAY_BE_GUARD) 1377|| && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1378|| uint8_t type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); 1379|| if (type == IS_STRING && !ZEND_DEBUG) { 1380| EXT_CALL _efree, r0 1381|| break; 1382|| } else if (type == IS_ARRAY) { 1383|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { 1384|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { 1385| SET_EX_OPLINE opline, r0 1386|| } 1387| EXT_CALL zend_array_destroy, r0 1388|| } else { 1389| EXT_CALL zend_jit_array_free, r0 1390|| } 1391|| break; 1392|| } else if (type == IS_OBJECT) { 1393|| if (opline) { 1394| SET_EX_OPLINE opline, r0 1395|| } 1396| EXT_CALL zend_objects_store_del, r0 1397|| break; 1398|| } 1399|| } 1400|| if (opline) { 1401| SET_EX_OPLINE opline, r0 1402|| } 1403| EXT_CALL rc_dtor_func, r0 1404|| } while(0); 1405|.endmacro 1406 1407|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline 1408|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) { 1409|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1410| // if (Z_REFCOUNTED_P(cv)) { 1411|| if (cold) { 1412| IF_ZVAL_REFCOUNTED addr, >1 1413|.cold_code 1414|1: 1415|| } else { 1416| IF_NOT_ZVAL_REFCOUNTED addr, >4 1417|| } 1418|| } 1419| // if (!Z_DELREF_P(cv)) { 1420| GET_ZVAL_PTR FCARG1a, addr 1421| GC_DELREF FCARG1a 1422|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) { 1423|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) { 1424|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1425| jnz >3 1426|| } else { 1427| jnz >4 1428|| } 1429|| } 1430| // zval_dtor_func(r); 1431| ZVAL_DTOR_FUNC op_info, opline 1432|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1433| jmp >4 1434|| } 1435|3: 1436|| } 1437|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1438|| if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) { 1439|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 1440| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1 1441| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4 1442| GET_ZVAL_PTR FCARG1a, ref_addr 1443|1: 1444|| } 1445| IF_GC_MAY_NOT_LEAK FCARG1a, >4 1446| // gc_possible_root(Z_COUNTED_P(z)) 1447| EXT_CALL gc_possible_root, r0 1448|| } 1449|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { 1450| jmp >4 1451|.code 1452|| } 1453|4: 1454|| } 1455|.endmacro 1456 1457|.macro FREE_OP, op_type, op, op_info, cold, opline 1458|| if (op_type & (IS_VAR|IS_TMP_VAR)) { 1459| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, opline 1460|| } 1461|.endmacro 1462 1463|.macro SEPARATE_ARRAY, addr, op_info, cold 1464|| if (RC_MAY_BE_N(op_info)) { 1465|| if (Z_REG(addr) != ZREG_FP) { 1466| GET_ZVAL_LVAL ZREG_R0, addr 1467|| if (RC_MAY_BE_1(op_info)) { 1468| cmp dword [r0], 1 // if (GC_REFCOUNT() > 1) 1469| jbe >2 1470|| } 1471|| if (Z_REG(addr) != ZREG_FCARG1 || Z_OFFSET(addr) != 0) { 1472| LOAD_ZVAL_ADDR FCARG1a, addr 1473|| } 1474| EXT_CALL zend_jit_zval_array_dup, r0 1475|2: 1476| mov FCARG1a, r0 1477|| } else { 1478| GET_ZVAL_LVAL ZREG_FCARG1, addr 1479|| if (RC_MAY_BE_1(op_info)) { 1480| cmp dword [FCARG1a], 1 // if (GC_REFCOUNT() > 1) 1481|| if (cold) { 1482| ja >1 1483|.cold_code 1484|1: 1485|| } else { 1486| jbe >2 1487|| } 1488|| } 1489| IF_NOT_ZVAL_REFCOUNTED addr, >1 1490| GC_DELREF FCARG1a 1491|1: 1492| EXT_CALL zend_array_dup, r0 1493| SET_ZVAL_PTR addr, r0 1494| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX 1495| mov FCARG1a, r0 1496|| if (RC_MAY_BE_1(op_info)) { 1497|| if (cold) { 1498| jmp >2 1499|.code 1500|| } 1501|| } 1502|2: 1503|| } 1504|| } else { 1505| GET_ZVAL_LVAL ZREG_FCARG1, addr 1506|| } 1507|.endmacro 1508 1509|.macro EFREE_REG_REFERENCE 1510||#if ZEND_DEBUG 1511| xor FCARG2a, FCARG2a // filename 1512| .if X64WIN 1513| xor CARG3d, CARG3d // lineno 1514| xor CARG4, CARG4 1515| mov aword A5, 0 1516| EXT_CALL _efree, r0 1517| .elif X64 1518| xor CARG3d, CARG3d // lineno 1519| xor CARG4, CARG4 1520| xor CARG5, CARG5 1521| EXT_CALL _efree, r0 1522| .else 1523| sub r4, 4 1524| push 0 1525| push 0 1526| push 0 // lineno 1527| EXT_CALL _efree, r0 1528| add r4, 4 1529| .endif 1530||#else 1531||#ifdef HAVE_BUILTIN_CONSTANT_P 1532| EXT_CALL _efree_32, r0 1533||#else 1534| EXT_CALL _efree, r0 1535||#endif 1536||#endif 1537|.endmacro 1538 1539|.macro EFREE_REFERENCE, ptr 1540| mov FCARG1a, ptr 1541| EFREE_REG_REFERENCE 1542|.endmacro 1543 1544|.macro EMALLOC, size, op_array, opline 1545||#if ZEND_DEBUG 1546|| const char *filename = op_array->filename ? op_array->filename->val : NULL; 1547| mov FCARG1a, size 1548| LOAD_ADDR FCARG2a, filename 1549| .if X64WIN 1550| mov CARG3d, opline->lineno 1551| xor CARG4, CARG4 1552| mov aword A5, 0 1553| EXT_CALL _emalloc, r0 1554| .elif X64 1555| mov CARG3d, opline->lineno 1556| xor CARG4, CARG4 1557| xor CARG5, CARG5 1558| EXT_CALL _emalloc, r0 1559| .else 1560| sub r4, 4 1561| push 0 1562| push 0 1563| push opline->lineno 1564| EXT_CALL _emalloc, r0 1565| add r4, 4 1566| .endif 1567||#else 1568||#ifdef HAVE_BUILTIN_CONSTANT_P 1569|| if (size > 24 && size <= 32) { 1570| EXT_CALL _emalloc_32, r0 1571|| } else { 1572| mov FCARG1a, size 1573| EXT_CALL _emalloc, r0 1574|| } 1575||#else 1576| mov FCARG1a, size 1577| EXT_CALL _emalloc, r0 1578||#endif 1579||#endif 1580|.endmacro 1581 1582|.macro OBJ_RELEASE, reg, exit_label 1583| GC_DELREF Ra(reg) 1584| jne >1 1585| // zend_objects_store_del(obj); 1586|| if (reg != ZREG_FCARG1) { 1587| mov FCARG1a, Ra(reg) 1588|| } 1589| EXT_CALL zend_objects_store_del, r0 1590| jmp exit_label 1591|1: 1592| IF_GC_MAY_NOT_LEAK Ra(reg), >1 1593| // gc_possible_root(obj) 1594|| if (reg != ZREG_FCARG1) { 1595| mov FCARG1a, Ra(reg) 1596|| } 1597| EXT_CALL gc_possible_root, r0 1598|1: 1599|.endmacro 1600 1601|.macro UNDEFINED_OFFSET, opline 1602|| if (opline == last_valid_opline) { 1603|| zend_jit_use_last_valid_opline(); 1604| call ->undefined_offset_ex 1605|| } else { 1606| SET_EX_OPLINE opline, r0 1607| call ->undefined_offset 1608|| } 1609|.endmacro 1610 1611|.macro UNDEFINED_INDEX, opline 1612|| if (opline == last_valid_opline) { 1613|| zend_jit_use_last_valid_opline(); 1614| call ->undefined_index_ex 1615|| } else { 1616| SET_EX_OPLINE opline, r0 1617| call ->undefined_index 1618|| } 1619|.endmacro 1620 1621|.macro CANNOT_ADD_ELEMENT, opline 1622|| if (opline == last_valid_opline) { 1623|| zend_jit_use_last_valid_opline(); 1624| call ->cannot_add_element_ex 1625|| } else { 1626| SET_EX_OPLINE opline, r0 1627| call ->cannot_add_element 1628|| } 1629|.endmacro 1630 1631|.macro ENDBR 1632||#if defined (__CET__) && (__CET__ & 1) != 0 1633| .if X64 1634| endbr64 1635| .else 1636| endbr32 1637| .endif 1638||#endif 1639|.endmacro 1640 1641#if defined (__CET__) && (__CET__ & 1) != 0 1642# define ENDBR_PADDING 4 1643#else 1644# define ENDBR_PADDING 0 1645#endif 1646 1647static bool reuse_ip = 0; 1648static bool delayed_call_chain = 0; 1649static uint32_t delayed_call_level = 0; 1650static const zend_op *last_valid_opline = NULL; 1651static bool use_last_vald_opline = 0; 1652static bool track_last_valid_opline = 0; 1653static int jit_return_label = -1; 1654static uint32_t current_trace_num = 0; 1655static uint32_t allowed_opt_flags = 0; 1656 1657static void zend_jit_track_last_valid_opline(void) 1658{ 1659 use_last_vald_opline = 0; 1660 track_last_valid_opline = 1; 1661} 1662 1663static void zend_jit_use_last_valid_opline(void) 1664{ 1665 if (track_last_valid_opline) { 1666 use_last_vald_opline = 1; 1667 track_last_valid_opline = 0; 1668 } 1669} 1670 1671static bool zend_jit_trace_uses_initial_ip(void) 1672{ 1673 return use_last_vald_opline; 1674} 1675 1676static void zend_jit_set_last_valid_opline(const zend_op *target_opline) 1677{ 1678 if (!reuse_ip) { 1679 track_last_valid_opline = 0; 1680 last_valid_opline = target_opline; 1681 } 1682} 1683 1684static void zend_jit_reset_last_valid_opline(void) 1685{ 1686 track_last_valid_opline = 0; 1687 last_valid_opline = NULL; 1688} 1689 1690static void zend_jit_start_reuse_ip(void) 1691{ 1692 zend_jit_reset_last_valid_opline(); 1693 reuse_ip = 1; 1694} 1695 1696static int zend_jit_reuse_ip(dasm_State **Dst) 1697{ 1698 if (!reuse_ip) { 1699 zend_jit_start_reuse_ip(); 1700 | // call = EX(call); 1701 | mov RX, EX->call 1702 } 1703 return 1; 1704} 1705 1706static void zend_jit_stop_reuse_ip(void) 1707{ 1708 reuse_ip = 0; 1709} 1710 1711static int zend_jit_interrupt_handler_stub(dasm_State **Dst) 1712{ 1713 |->interrupt_handler: 1714 | SAVE_IP 1715 | //EG(vm_interrupt) = 0; 1716 | MEM_STORE_ZTS byte, executor_globals, vm_interrupt, 0, r0 1717 | //if (EG(timed_out)) { 1718 | MEM_CMP_ZTS byte, executor_globals, timed_out, 0, r0 1719 | je >1 1720 | //zend_timeout(); 1721 | EXT_CALL zend_timeout, r0 1722 |1: 1723 | //} else if (zend_interrupt_function) { 1724 if (zend_interrupt_function) { 1725 | //zend_interrupt_function(execute_data); 1726 |.if X64 1727 | mov CARG1, FP 1728 | EXT_CALL zend_interrupt_function, r0 1729 |.else 1730 | mov aword A1, FP 1731 | EXT_CALL zend_interrupt_function, r0 1732 |.endif 1733 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 1734 | je >1 1735 | EXT_CALL zend_jit_exception_in_interrupt_handler_helper, r0 1736 |1: 1737 | //ZEND_VM_ENTER(); 1738 | //execute_data = EG(current_execute_data); 1739 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 1740 | LOAD_IP 1741 } 1742 | //ZEND_VM_CONTINUE() 1743 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1744 | ADD_HYBRID_SPAD 1745 | JMP_IP 1746 } else if (GCC_GLOBAL_REGS) { 1747 | add r4, SPAD // stack alignment 1748 | JMP_IP 1749 } else { 1750 | mov FP, aword T2 // restore FP 1751 | mov RX, aword T3 // restore IP 1752 | add r4, NR_SPAD // stack alignment 1753 | mov r0, 1 // ZEND_VM_ENTER 1754 | ret 1755 } 1756 1757 return 1; 1758} 1759 1760static int zend_jit_exception_handler_stub(dasm_State **Dst) 1761{ 1762 |->exception_handler: 1763 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1764 const void *handler = zend_get_opcode_handler_func(EG(exception_op)); 1765 1766 | ADD_HYBRID_SPAD 1767 | EXT_CALL handler, r0 1768 | JMP_IP 1769 } else { 1770 const void *handler = EG(exception_op)->handler; 1771 1772 if (GCC_GLOBAL_REGS) { 1773 | add r4, SPAD // stack alignment 1774 | EXT_JMP handler, r0 1775 } else { 1776 | mov FCARG1a, FP 1777 | EXT_CALL handler, r0 1778 | mov FP, aword T2 // restore FP 1779 | mov RX, aword T3 // restore IP 1780 | add r4, NR_SPAD // stack alignment 1781 | test eax, eax 1782 | jl >1 1783 | mov r0, 1 // ZEND_VM_ENTER 1784 |1: 1785 | ret 1786 } 1787 } 1788 1789 return 1; 1790} 1791 1792static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) 1793{ 1794 |->exception_handler_undef: 1795 | MEM_LOAD_ZTS r0, aword, executor_globals, opline_before_exception, r0 1796 | test byte OP:r0->result_type, (IS_TMP_VAR|IS_VAR) 1797 | jz >1 1798 | mov eax, dword OP:r0->result.var 1799 | SET_Z_TYPE_INFO FP + r0, IS_UNDEF 1800 |1: 1801 | jmp ->exception_handler 1802 1803 return 1; 1804} 1805 1806 1807static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst) 1808{ 1809 |->exception_handler_free_op1_op2: 1810 | UNDEF_OPLINE_RESULT_IF_USED 1811 | test byte OP:RX->op1_type, (IS_TMP_VAR|IS_VAR) 1812 | je >9 1813 | mov eax, dword OP:RX->op1.var 1814 | add r0, FP 1815 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1816 |9: 1817 | test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR) 1818 | je >9 1819 | mov eax, dword OP:RX->op2.var 1820 | add r0, FP 1821 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1822 |9: 1823 | jmp ->exception_handler 1824 return 1; 1825} 1826 1827static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst) 1828{ 1829 |->exception_handler_free_op2: 1830 | MEM_LOAD_ZTS RX, aword, executor_globals, opline_before_exception, r0 1831 | UNDEF_OPLINE_RESULT_IF_USED 1832 | test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR) 1833 | je >9 1834 | mov eax, dword OP:RX->op2.var 1835 | add r0, FP 1836 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1837 |9: 1838 | jmp ->exception_handler 1839 return 1; 1840} 1841 1842static int zend_jit_leave_function_stub(dasm_State **Dst) 1843{ 1844 |->leave_function_handler: 1845 | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)] 1846 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1847 | test FCARG1d, ZEND_CALL_TOP 1848 | jnz >1 1849 | EXT_CALL zend_jit_leave_nested_func_helper, r0 1850 | ADD_HYBRID_SPAD 1851 | JMP_IP 1852 |1: 1853 | EXT_CALL zend_jit_leave_top_func_helper, r0 1854 | ADD_HYBRID_SPAD 1855 | JMP_IP 1856 } else { 1857 if (GCC_GLOBAL_REGS) { 1858 | add r4, SPAD 1859 } else { 1860 | mov FCARG2a, FP 1861 | mov FP, aword T2 // restore FP 1862 | mov RX, aword T3 // restore IP 1863 | add r4, NR_SPAD 1864 } 1865 | test FCARG1d, ZEND_CALL_TOP 1866 | jnz >1 1867 | EXT_JMP zend_jit_leave_nested_func_helper, r0 1868 |1: 1869 | EXT_JMP zend_jit_leave_top_func_helper, r0 1870 } 1871 1872 return 1; 1873} 1874 1875static int zend_jit_leave_throw_stub(dasm_State **Dst) 1876{ 1877 |->leave_throw_handler: 1878 | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { 1879 if (GCC_GLOBAL_REGS) { 1880 | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION 1881 | je >5 1882 | // EG(opline_before_exception) = opline; 1883 | MEM_STORE_ZTS aword, executor_globals, opline_before_exception, IP, r0 1884 |5: 1885 | // opline = EG(exception_op); 1886 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1887 | mov aword EX->opline, IP 1888 | // HANDLE_EXCEPTION() 1889 | jmp ->exception_handler 1890 } else { 1891 | GET_IP FCARG1a 1892 | cmp byte OP:FCARG1a->opcode, ZEND_HANDLE_EXCEPTION 1893 | je >5 1894 | // EG(opline_before_exception) = opline; 1895 | MEM_STORE_ZTS aword, executor_globals, opline_before_exception, FCARG1a, r0 1896 |5: 1897 | // opline = EG(exception_op); 1898 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1899 | mov FP, aword T2 // restore FP 1900 | mov RX, aword T3 // restore IP 1901 | add r4, NR_SPAD // stack alignment 1902 | mov r0, 2 // ZEND_VM_LEAVE 1903 | ret 1904 } 1905 1906 return 1; 1907} 1908 1909static int zend_jit_icall_throw_stub(dasm_State **Dst) 1910{ 1911 |->icall_throw_handler: 1912 | // zend_rethrow_exception(zend_execute_data *execute_data) 1913 | mov IP, aword EX->opline 1914 | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { 1915 | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION 1916 | je >1 1917 | // EG(opline_before_exception) = opline; 1918 | MEM_STORE_ZTS aword, executor_globals, opline_before_exception, IP, r0 1919 |1: 1920 | // opline = EG(exception_op); 1921 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1922 || if (GCC_GLOBAL_REGS) { 1923 | mov aword EX->opline, IP 1924 || } 1925 | // HANDLE_EXCEPTION() 1926 | jmp ->exception_handler 1927 1928 return 1; 1929} 1930 1931static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) 1932{ 1933 |->throw_cannot_pass_by_ref: 1934 | mov r0, EX->opline 1935 | mov ecx, dword OP:r0->result.var 1936 | SET_Z_TYPE_INFO RX+r1, IS_UNDEF 1937 | // last EX(call) frame may be delayed 1938 | cmp RX, EX->call 1939 | je >1 1940 | mov r1, EX->call 1941 | mov EX:RX->prev_execute_data, r1 1942 | mov EX->call, RX 1943 |1: 1944 | mov RX, r0 1945 | mov FCARG1d, dword OP:r0->op2.num 1946 | EXT_CALL zend_cannot_pass_by_reference, r0 1947 | cmp byte OP:RX->op1_type, IS_TMP_VAR 1948 | jne >9 1949 | mov eax, dword OP:RX->op1.var 1950 | add r0, FP 1951 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1952 |9: 1953 | jmp ->exception_handler 1954 1955 return 1; 1956} 1957 1958static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) 1959{ 1960 |->undefined_offset_ex: 1961 | SAVE_IP 1962 | jmp ->undefined_offset 1963 1964 return 1; 1965} 1966 1967static int zend_jit_undefined_offset_stub(dasm_State **Dst) 1968{ 1969 |->undefined_offset: 1970 || if (!GCC_GLOBAL_REGS) { 1971 | mov FCARG1a, FP 1972 || } 1973 | EXT_JMP zend_jit_undefined_long_key, r0 1974 1975 return 1; 1976} 1977 1978static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) 1979{ 1980 |->undefined_index_ex: 1981 | SAVE_IP 1982 | jmp ->undefined_index 1983 1984 return 1; 1985} 1986 1987static int zend_jit_undefined_index_stub(dasm_State **Dst) 1988{ 1989 |->undefined_index: 1990 || if (!GCC_GLOBAL_REGS) { 1991 | mov FCARG1a, FP 1992 || } 1993 | EXT_JMP zend_jit_undefined_string_key, r0 1994 1995 return 1; 1996} 1997 1998static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) 1999{ 2000 |->cannot_add_element_ex: 2001 | SAVE_IP 2002 | jmp ->cannot_add_element 2003 2004 return 1; 2005} 2006 2007static int zend_jit_cannot_add_element_stub(dasm_State **Dst) 2008{ 2009 |->cannot_add_element: 2010 |.if X64WIN 2011 | sub r4, 0x28 2012 |.elif X64 2013 | sub r4, 8 2014 |.else 2015 | sub r4, 12 2016 |.endif 2017 | mov r0, EX->opline 2018 | cmp byte OP:r0->result_type, IS_UNUSED 2019 | jz >1 2020 | mov eax, dword OP:r0->result.var 2021 | SET_Z_TYPE_INFO FP + r0, IS_NULL 2022 |1: 2023 |.if X64WIN 2024 | xor CARG1, CARG1 2025 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2026 | EXT_CALL zend_throw_error, r0 2027 | add r4, 0x28 2028 |.elif X64 2029 | xor CARG1, CARG1 2030 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2031 | EXT_CALL zend_throw_error, r0 2032 | add r4, 8 2033 |.else 2034 | sub r4, 8 2035 | push "Cannot add element to the array as the next element is already occupied" 2036 | push 0 2037 | EXT_CALL zend_throw_error, r0 2038 | add r4, 28 2039 |.endif 2040 | ret 2041 2042 return 1; 2043} 2044 2045static int zend_jit_undefined_function_stub(dasm_State **Dst) 2046{ 2047 |->undefined_function: 2048 | mov r0, aword EX->opline 2049 |.if X64 2050 | xor CARG1, CARG1 2051 | LOAD_ADDR CARG2, "Call to undefined function %s()" 2052 | movsxd CARG3, dword [r0 + offsetof(zend_op, op2.constant)] 2053 | mov CARG3, aword [r0 + CARG3] 2054 | add CARG3, offsetof(zend_string, val) 2055 | EXT_CALL zend_throw_error, r0 2056 |.else 2057 | mov r0, aword [r0 + offsetof(zend_op, op2.zv)] 2058 | mov r0, aword [r0] 2059 | add r0, offsetof(zend_string, val) 2060 | mov aword A3, r0 2061 | mov aword A2, "Call to undefined function %s()" 2062 | mov aword A1, 0 2063 | EXT_CALL zend_throw_error, r0 2064 |.endif 2065 | jmp ->exception_handler 2066 return 1; 2067} 2068 2069static int zend_jit_negative_shift_stub(dasm_State **Dst) 2070{ 2071 |->negative_shift: 2072 | mov RX, EX->opline 2073 |.if X64 2074 |.if WIN 2075 | LOAD_ADDR CARG1, &zend_ce_arithmetic_error 2076 | mov CARG1, aword [CARG1] 2077 |.else 2078 | LOAD_ADDR CARG1, zend_ce_arithmetic_error 2079 |.endif 2080 | LOAD_ADDR CARG2, "Bit shift by negative number" 2081 | EXT_CALL zend_throw_error, r0 2082 |.else 2083 | sub r4, 8 2084 | push "Bit shift by negative number" 2085 |.if WIN 2086 | LOAD_ADDR r0, &zend_ce_arithmetic_error 2087 | push aword [r0] 2088 |.else 2089 | PUSH_ADDR zend_ce_arithmetic_error, r0 2090 |.endif 2091 | EXT_CALL zend_throw_error, r0 2092 | add r4, 16 2093 |.endif 2094 | jmp ->exception_handler_free_op1_op2 2095 return 1; 2096} 2097 2098static int zend_jit_mod_by_zero_stub(dasm_State **Dst) 2099{ 2100 |->mod_by_zero: 2101 | mov RX, EX->opline 2102 |.if X64 2103 |.if WIN 2104 | LOAD_ADDR CARG1, &zend_ce_division_by_zero_error 2105 | mov CARG1, aword [CARG1] 2106 |.else 2107 | LOAD_ADDR CARG1, zend_ce_division_by_zero_error 2108 |.endif 2109 | LOAD_ADDR CARG2, "Modulo by zero" 2110 | EXT_CALL zend_throw_error, r0 2111 |.else 2112 | sub r4, 8 2113 | push "Modulo by zero" 2114 |.if WIN 2115 | LOAD_ADDR r0, &zend_ce_division_by_zero_error 2116 | push aword [r0] 2117 |.else 2118 | PUSH_ADDR zend_ce_division_by_zero_error, r0 2119 |.endif 2120 | EXT_CALL zend_throw_error, r0 2121 | add r4, 16 2122 |.endif 2123 | jmp ->exception_handler_free_op1_op2 2124 return 1; 2125} 2126 2127static int zend_jit_invalid_this_stub(dasm_State **Dst) 2128{ 2129 |->invalid_this: 2130 | UNDEF_OPLINE_RESULT 2131 |.if X64 2132 | xor CARG1, CARG1 2133 | LOAD_ADDR CARG2, "Using $this when not in object context" 2134 | EXT_CALL zend_throw_error, r0 2135 |.else 2136 | sub r4, 8 2137 | push "Using $this when not in object context" 2138 | push 0 2139 | EXT_CALL zend_throw_error, r0 2140 | add r4, 16 2141 |.endif 2142 | jmp ->exception_handler 2143 return 1; 2144} 2145 2146static int zend_jit_double_one_stub(dasm_State **Dst) 2147{ 2148 |->one: 2149 |.dword 0, 0x3ff00000 2150 return 1; 2151} 2152 2153static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) 2154{ 2155 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2156 return 1; 2157 } 2158 2159 |->hybrid_runtime_jit: 2160 | EXT_CALL zend_runtime_jit, r0 2161 | JMP_IP 2162 return 1; 2163} 2164 2165static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) 2166{ 2167 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2168 return 1; 2169 } 2170 2171 |->hybrid_profile_jit: 2172 | // ++zend_jit_profile_counter; 2173 | .if X64 2174 | LOAD_ADDR r0, &zend_jit_profile_counter 2175 | inc aword [r0] 2176 | .else 2177 | inc aword [&zend_jit_profile_counter] 2178 | .endif 2179 | // op_array = (zend_op_array*)EX(func); 2180 | mov r0, EX->func 2181 | // run_time_cache = EX(run_time_cache); 2182 | mov r2, EX->run_time_cache 2183 | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); 2184 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2185 | // ++ZEND_COUNTER_INFO(op_array) 2186 | inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void*)] 2187 | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() 2188 | jmp aword [r0 + offsetof(zend_jit_op_array_extension, orig_handler)] 2189 return 1; 2190} 2191 2192static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) 2193{ 2194 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2195 return 1; 2196 } 2197 2198 |->hybrid_hot_code: 2199 | mov word [r2], ZEND_JIT_COUNTER_INIT 2200 | mov FCARG1a, FP 2201 | GET_IP FCARG2a 2202 | EXT_CALL zend_jit_hot_func, r0 2203 | JMP_IP 2204 return 1; 2205} 2206 2207/* 2208 * This code is based Mike Pall's "Hashed profile counters" idea, implemented 2209 * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual 2210 * property disclosure and research opportunities" email 2211 * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html 2212 * 2213 * In addition we use a variation of Knuth's multiplicative hash function 2214 * described at https://code.i-harness.com/en/q/a21ce 2215 * 2216 * uint64_t hash(uint64_t x) { 2217 * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; 2218 * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; 2219 * x = x ^ (x >> 31); 2220 * return x; 2221 * } 2222 * 2223 * uint_32_t hash(uint32_t x) { 2224 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2225 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2226 * x = (x >> 16) ^ x; 2227 * return x; 2228 * } 2229 * 2230 */ 2231static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) 2232{ 2233 | ENDBR 2234 | mov r0, EX->func 2235 | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2236 | mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)] 2237 | sub word [r2], cost 2238 | jle ->hybrid_hot_code 2239 | GET_IP r2 2240 | sub r2, aword [r0 + offsetof(zend_op_array, opcodes)] 2241 | // divide by sizeof(zend_op) 2242 | .if X64 2243 || ZEND_ASSERT(sizeof(zend_op) == 32); 2244 | sar r2, 2 2245 | .else 2246 || ZEND_ASSERT(sizeof(zend_op) == 28); 2247 | imul r2, 0xb6db6db7 2248 | .endif 2249 | .if X64 2250 | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2251 | .else 2252 | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2253 | .endif 2254 return 1; 2255} 2256 2257static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) 2258{ 2259 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2260 return 1; 2261 } 2262 2263 |->hybrid_func_hot_counter: 2264 2265 return zend_jit_hybrid_hot_counter_stub(Dst, 2266 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2267} 2268 2269static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) 2270{ 2271 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2272 return 1; 2273 } 2274 2275 |->hybrid_loop_hot_counter: 2276 2277 return zend_jit_hybrid_hot_counter_stub(Dst, 2278 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2279} 2280 2281static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) 2282{ 2283 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2284 return 1; 2285 } 2286 2287 |->hybrid_hot_trace: 2288 | mov word [r2], ZEND_JIT_COUNTER_INIT 2289 | mov FCARG1a, FP 2290 | GET_IP FCARG2a 2291 | EXT_CALL zend_jit_trace_hot_root, r0 2292 | test eax, eax // TODO : remove this check at least for HYBRID VM ??? 2293 | jl >1 2294 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 2295 | LOAD_IP 2296 | JMP_IP 2297 |1: 2298 | EXT_JMP zend_jit_halt_op->handler, r0 2299 return 1; 2300} 2301 2302static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) 2303{ 2304 | ENDBR 2305 | mov r0, EX->func 2306 | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2307 | mov r1, aword [r1 + offsetof(zend_jit_op_array_trace_extension, offset)] 2308 | mov r2, aword [IP + r1 + offsetof(zend_op_trace_info, counter)] 2309 | sub word [r2], cost 2310 | jle ->hybrid_hot_trace 2311 | jmp aword [IP + r1] 2312 return 1; 2313} 2314 2315static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) 2316{ 2317 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2318 return 1; 2319 } 2320 2321 |->hybrid_func_trace_counter: 2322 2323 return zend_jit_hybrid_trace_counter_stub(Dst, 2324 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2325} 2326 2327static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) 2328{ 2329 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { 2330 return 1; 2331 } 2332 2333 |->hybrid_ret_trace_counter: 2334 2335 return zend_jit_hybrid_trace_counter_stub(Dst, 2336 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); 2337} 2338 2339static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) 2340{ 2341 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2342 return 1; 2343 } 2344 2345 |->hybrid_loop_trace_counter: 2346 2347 return zend_jit_hybrid_trace_counter_stub(Dst, 2348 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2349} 2350 2351static int zend_jit_trace_halt_stub(dasm_State **Dst) 2352{ 2353 |->trace_halt: 2354 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2355 | ADD_HYBRID_SPAD 2356 | EXT_JMP zend_jit_halt_op->handler, r0 2357 } else if (GCC_GLOBAL_REGS) { 2358 | add r4, SPAD // stack alignment 2359 | xor IP, IP // PC must be zero 2360 | ret 2361 } else { 2362 | mov FP, aword T2 // restore FP 2363 | mov RX, aword T3 // restore IP 2364 | add r4, NR_SPAD // stack alignment 2365 | mov r0, -1 // ZEND_VM_RETURN 2366 | ret 2367 } 2368 return 1; 2369} 2370 2371static int zend_jit_trace_exit_stub(dasm_State **Dst) 2372{ 2373 |->trace_exit: 2374 | 2375 | // Save CPU registers 2376 |.if X64 2377 | sub r4, 16*8+16*8-8 /* CPU regs + SSE regs */ 2378 | mov aword [r4+15*8], r15 2379 | mov aword [r4+11*8], r11 2380 | mov aword [r4+10*8], r10 2381 | mov aword [r4+9*8], r9 2382 | mov aword [r4+8*8], r8 2383 | mov aword [r4+7*8], rdi 2384 | mov aword [r4+6*8], rsi 2385 | mov aword [r4+2*8], rdx 2386 | mov aword [r4+1*8], rcx 2387 | mov aword [r4+0*8], rax 2388 | mov FCARG1a, aword [r4+16*8+16*8-8] // exit_num = POP 2389 | mov FCARG2a, r4 2390 | movsd qword [r4+16*8+15*8], xmm15 2391 | movsd qword [r4+16*8+14*8], xmm14 2392 | movsd qword [r4+16*8+13*8], xmm13 2393 | movsd qword [r4+16*8+12*8], xmm12 2394 | movsd qword [r4+16*8+11*8], xmm11 2395 | movsd qword [r4+16*8+10*8], xmm10 2396 | movsd qword [r4+16*8+9*8], xmm9 2397 | movsd qword [r4+16*8+8*8], xmm8 2398 | movsd qword [r4+16*8+7*8], xmm7 2399 | movsd qword [r4+16*8+6*8], xmm6 2400 | movsd qword [r4+16*8+5*8], xmm5 2401 | movsd qword [r4+16*8+4*8], xmm4 2402 | movsd qword [r4+16*8+3*8], xmm3 2403 | movsd qword [r4+16*8+2*8], xmm2 2404 | movsd qword [r4+16*8+1*8], xmm1 2405 | movsd qword [r4+16*8+0*8], xmm0 2406 |.if X64WIN 2407 | sub r4, 32 /* shadow space */ 2408 |.endif 2409 |.else 2410 | sub r4, 8*4+8*8-4 /* CPU regs + SSE regs */ 2411 | mov aword [r4+7*4], edi 2412 | mov aword [r4+2*4], edx 2413 | mov aword [r4+1*4], ecx 2414 | mov aword [r4+0*4], eax 2415 | mov FCARG1a, aword [r4+8*4+8*8-4] // exit_num = POP 2416 | mov FCARG2a, r4 2417 | movsd qword [r4+8*4+7*8], xmm7 2418 | movsd qword [r4+8*4+6*8], xmm6 2419 | movsd qword [r4+8*4+5*8], xmm5 2420 | movsd qword [r4+8*4+4*8], xmm4 2421 | movsd qword [r4+8*4+3*8], xmm3 2422 | movsd qword [r4+8*4+2*8], xmm2 2423 | movsd qword [r4+8*4+1*8], xmm1 2424 | movsd qword [r4+8*4+0*8], xmm0 2425 |.endif 2426 | 2427 | // EX(opline) = opline 2428 | SAVE_IP 2429 | // zend_jit_trace_exit(trace_num, exit_num) 2430 | EXT_CALL zend_jit_trace_exit, r0 2431 |.if X64WIN 2432 | add r4, 16*8+16*8+32 /* CPU regs + SSE regs + shadow space */ 2433 |.elif X64 2434 | add r4, 16*8+16*8 /* CPU regs + SSE regs */ 2435 |.else 2436 | add r4, 8*4+8*8 /* CPU regs + SSE regs */ 2437 |.endif 2438 2439 | test eax, eax 2440 | jne >1 2441 2442 | // execute_data = EG(current_execute_data) 2443 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 2444 | // opline = EX(opline) 2445 | LOAD_IP 2446 2447 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2448 | ADD_HYBRID_SPAD 2449 | JMP_IP 2450 } else if (GCC_GLOBAL_REGS) { 2451 | add r4, SPAD // stack alignment 2452 | JMP_IP 2453 } else { 2454 | mov FP, aword T2 // restore FP 2455 | mov RX, aword T3 // restore IP 2456 | add r4, NR_SPAD // stack alignment 2457 | mov r0, 1 // ZEND_VM_ENTER 2458 | ret 2459 } 2460 2461 |1: 2462 | jl ->trace_halt 2463 2464 | // execute_data = EG(current_execute_data) 2465 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 2466 | // opline = EX(opline) 2467 | LOAD_IP 2468 2469 | // check for interrupt (try to avoid this ???) 2470 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 2471 | jne ->interrupt_handler 2472 2473 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2474 | ADD_HYBRID_SPAD 2475 | mov r0, EX->func 2476 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2477 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2478 | jmp aword [IP + r0] 2479 } else if (GCC_GLOBAL_REGS) { 2480 | add r4, SPAD // stack alignment 2481 | mov r0, EX->func 2482 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2483 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2484 | jmp aword [IP + r0] 2485 } else { 2486 | mov IP, aword EX->opline 2487 | mov FCARG1a, FP 2488 | mov r0, EX->func 2489 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2490 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2491 | call aword [IP + r0] 2492 | test eax, eax 2493 | jl ->trace_halt 2494 | mov FP, aword T2 // restore FP 2495 | mov RX, aword T3 // restore IP 2496 | add r4, NR_SPAD // stack alignment 2497 | mov r0, 1 // ZEND_VM_ENTER 2498 | ret 2499 } 2500 2501 return 1; 2502} 2503 2504static int zend_jit_trace_escape_stub(dasm_State **Dst) 2505{ 2506 |->trace_escape: 2507 | 2508 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2509 | ADD_HYBRID_SPAD 2510 | JMP_IP 2511 } else if (GCC_GLOBAL_REGS) { 2512 | add r4, SPAD // stack alignment 2513 | JMP_IP 2514 } else { 2515 | mov FP, aword T2 // restore FP 2516 | mov RX, aword T3 // restore IP 2517 | add r4, NR_SPAD // stack alignment 2518 | mov r0, 1 // ZEND_VM_ENTER 2519 | ret 2520 } 2521 2522 return 1; 2523} 2524 2525/* Keep 32 exit points in a single code block */ 2526#define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes 2527#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points 2528 2529static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) 2530{ 2531 uint32_t i; 2532 2533 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP - 1; i++) { 2534 | push byte i 2535 | .byte 0xeb, (4*(ZEND_JIT_EXIT_POINTS_PER_GROUP-i)-6) // jmp >1 2536 } 2537 | push byte i 2538 |// 1: 2539 | add aword [r4], n 2540 | jmp ->trace_exit 2541 2542 return 1; 2543} 2544 2545#ifdef CONTEXT_THREADED_JIT 2546static int zend_jit_context_threaded_call_stub(dasm_State **Dst) 2547{ 2548 |->context_threaded_call: 2549 | pop r0 2550 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2551 | ADD_HYBRID_SPAD 2552 | jmp aword [IP] 2553 } else if (GCC_GLOBAL_REGS) { 2554 | add r4, SPAD // stack alignment 2555 | jmp aword [IP] 2556 } else { 2557 ZEND_UNREACHABLE(); 2558 // TODO: context threading can't work without GLOBAL REGS because we have to change 2559 // the value of execute_data in execute_ex() 2560 | mov FCARG1a, FP 2561 | mov r0, aword [FP] 2562 | mov FP, aword T2 // restore FP 2563 | mov RX, aword T3 // restore IP 2564 | add r4, NR_SPAD // stack alignment 2565 | jmp aword [r0] 2566 } 2567 return 1; 2568} 2569#endif 2570 2571static int zend_jit_assign_const_stub(dasm_State **Dst) 2572{ 2573 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2574 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2575 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2576 2577 |->assign_const: 2578 |.if X64WIN 2579 | sub r4, 0x28 2580 |.elif X64 2581 | sub r4, 8 2582 |.else 2583 | sub r4, 12 2584 |.endif 2585 if (!zend_jit_assign_to_variable( 2586 Dst, NULL, 2587 var_addr, var_addr, -1, -1, 2588 IS_CONST, val_addr, val_info, 2589 0, 0)) { 2590 return 0; 2591 } 2592 |.if X64WIN 2593 | add r4, 0x28 2594 |.elif X64 2595 | add r4, 8 2596 |.else 2597 | add r4, 12 2598 |.endif 2599 | ret 2600 return 1; 2601} 2602 2603static int zend_jit_assign_tmp_stub(dasm_State **Dst) 2604{ 2605 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2606 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2607 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2608 2609 |->assign_tmp: 2610 |.if X64WIN 2611 | sub r4, 0x28 2612 |.elif X64 2613 | sub r4, 8 2614 |.else 2615 | sub r4, 12 2616 |.endif 2617 if (!zend_jit_assign_to_variable( 2618 Dst, NULL, 2619 var_addr, var_addr, -1, -1, 2620 IS_TMP_VAR, val_addr, val_info, 2621 0, 0)) { 2622 return 0; 2623 } 2624 |.if X64WIN 2625 | add r4, 0x28 2626 |.elif X64 2627 | add r4, 8 2628 |.else 2629 | add r4, 12 2630 |.endif 2631 | ret 2632 return 1; 2633} 2634 2635static int zend_jit_assign_var_stub(dasm_State **Dst) 2636{ 2637 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2638 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2639 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; 2640 2641 |->assign_var: 2642 |.if X64WIN 2643 | sub r4, 0x28 2644 |.elif X64 2645 | sub r4, 8 2646 |.else 2647 | sub r4, 12 2648 |.endif 2649 if (!zend_jit_assign_to_variable( 2650 Dst, NULL, 2651 var_addr, var_addr, -1, -1, 2652 IS_VAR, val_addr, val_info, 2653 0, 0)) { 2654 return 0; 2655 } 2656 |.if X64WIN 2657 | add r4, 0x28 2658 |.elif X64 2659 | add r4, 8 2660 |.else 2661 | add r4, 12 2662 |.endif 2663 | ret 2664 return 1; 2665} 2666 2667static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) 2668{ 2669 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2670 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2671 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; 2672 2673 |->assign_cv_noref: 2674 |.if X64WIN 2675 | sub r4, 0x28 2676 |.elif X64 2677 | sub r4, 8 2678 |.else 2679 | sub r4, 12 2680 |.endif 2681 if (!zend_jit_assign_to_variable( 2682 Dst, NULL, 2683 var_addr, var_addr, -1, -1, 2684 IS_CV, val_addr, val_info, 2685 0, 0)) { 2686 return 0; 2687 } 2688 |.if X64WIN 2689 | add r4, 0x28 2690 |.elif X64 2691 | add r4, 8 2692 |.else 2693 | add r4, 12 2694 |.endif 2695 | ret 2696 return 1; 2697} 2698 2699static int zend_jit_assign_cv_stub(dasm_State **Dst) 2700{ 2701 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2702 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2703 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; 2704 2705 |->assign_cv: 2706 |.if X64WIN 2707 | sub r4, 0x28 2708 |.elif X64 2709 | sub r4, 8 2710 |.else 2711 | sub r4, 12 2712 |.endif 2713 if (!zend_jit_assign_to_variable( 2714 Dst, NULL, 2715 var_addr, var_addr, -1, -1, 2716 IS_CV, val_addr, val_info, 2717 0, 0)) { 2718 return 0; 2719 } 2720 |.if X64WIN 2721 | add r4, 0x28 2722 |.elif X64 2723 | add r4, 8 2724 |.else 2725 | add r4, 12 2726 |.endif 2727 | ret 2728 return 1; 2729} 2730 2731static const zend_jit_stub zend_jit_stubs[] = { 2732 JIT_STUB(interrupt_handler, SP_ADJ_JIT, SP_ADJ_VM), 2733 JIT_STUB(exception_handler, SP_ADJ_JIT, SP_ADJ_VM), 2734 JIT_STUB(exception_handler_undef, SP_ADJ_JIT, SP_ADJ_VM), 2735 JIT_STUB(exception_handler_free_op1_op2, SP_ADJ_JIT, SP_ADJ_VM), 2736 JIT_STUB(exception_handler_free_op2, SP_ADJ_JIT, SP_ADJ_VM), 2737 JIT_STUB(leave_function, SP_ADJ_JIT, SP_ADJ_VM), 2738 JIT_STUB(leave_throw, SP_ADJ_JIT, SP_ADJ_VM), 2739 JIT_STUB(icall_throw, SP_ADJ_JIT, SP_ADJ_VM), 2740 JIT_STUB(throw_cannot_pass_by_ref, SP_ADJ_JIT, SP_ADJ_VM), 2741 JIT_STUB(undefined_offset, SP_ADJ_JIT, SP_ADJ_VM), 2742 JIT_STUB(undefined_index, SP_ADJ_JIT, SP_ADJ_VM), 2743 JIT_STUB(cannot_add_element, SP_ADJ_JIT, SP_ADJ_VM), 2744 JIT_STUB(undefined_offset_ex, SP_ADJ_JIT, SP_ADJ_VM), 2745 JIT_STUB(undefined_index_ex, SP_ADJ_JIT, SP_ADJ_VM), 2746 JIT_STUB(cannot_add_element_ex, SP_ADJ_JIT, SP_ADJ_VM), 2747 JIT_STUB(undefined_function, SP_ADJ_JIT, SP_ADJ_VM), 2748 JIT_STUB(negative_shift, SP_ADJ_JIT, SP_ADJ_VM), 2749 JIT_STUB(mod_by_zero, SP_ADJ_JIT, SP_ADJ_VM), 2750 JIT_STUB(invalid_this, SP_ADJ_JIT, SP_ADJ_VM), 2751 JIT_STUB(trace_halt, SP_ADJ_JIT, SP_ADJ_VM), 2752 JIT_STUB(trace_exit, SP_ADJ_JIT, SP_ADJ_VM), 2753 JIT_STUB(trace_escape, SP_ADJ_JIT, SP_ADJ_VM), 2754 JIT_STUB(hybrid_runtime_jit, SP_ADJ_VM, SP_ADJ_NONE), 2755 JIT_STUB(hybrid_profile_jit, SP_ADJ_VM, SP_ADJ_NONE), 2756 JIT_STUB(hybrid_hot_code, SP_ADJ_VM, SP_ADJ_NONE), 2757 JIT_STUB(hybrid_func_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2758 JIT_STUB(hybrid_loop_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2759 JIT_STUB(hybrid_hot_trace, SP_ADJ_VM, SP_ADJ_NONE), 2760 JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2761 JIT_STUB(hybrid_ret_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2762 JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2763 JIT_STUB(assign_const, SP_ADJ_RET, SP_ADJ_ASSIGN), 2764 JIT_STUB(assign_tmp, SP_ADJ_RET, SP_ADJ_ASSIGN), 2765 JIT_STUB(assign_var, SP_ADJ_RET, SP_ADJ_ASSIGN), 2766 JIT_STUB(assign_cv_noref, SP_ADJ_RET, SP_ADJ_ASSIGN), 2767 JIT_STUB(assign_cv, SP_ADJ_RET, SP_ADJ_ASSIGN), 2768 JIT_STUB(double_one, SP_ADJ_NONE, SP_ADJ_NONE), 2769#ifdef CONTEXT_THREADED_JIT 2770 JIT_STUB(context_threaded_call, SP_ADJ_RET, SP_ADJ_NONE), 2771#endif 2772}; 2773 2774#if ZTS && defined(ZEND_WIN32) 2775extern uint32_t _tls_index; 2776extern char *_tls_start; 2777extern char *_tls_end; 2778#endif 2779 2780#ifdef HAVE_GDB 2781typedef struct _Unwind_Context _Unwind_Context; 2782typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *); 2783extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *); 2784extern uintptr_t _Unwind_GetCFA(_Unwind_Context *); 2785 2786typedef struct _zend_jit_unwind_arg { 2787 int cnt; 2788 uintptr_t cfa[3]; 2789} zend_jit_unwind_arg; 2790 2791static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a) 2792{ 2793 zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a; 2794 arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx); 2795 arg->cnt++; 2796 if (arg->cnt == 3) { 2797 return 5; // _URC_END_OF_STACK 2798 } 2799 return 0; // _URC_NO_REASON; 2800} 2801 2802static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) 2803{ 2804 zend_jit_unwind_arg arg; 2805 2806 memset(&arg, 0, sizeof(arg)); 2807 _Unwind_Backtrace(zend_jit_unwind_cb, &arg); 2808 if (arg.cnt == 3) { 2809 sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1]; 2810 } 2811} 2812 2813extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data); 2814 2815static zend_never_inline void zend_jit_set_sp_adj_vm(void) 2816{ 2817 void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *); 2818 2819 orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data; 2820 zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data; 2821 execute_ex(NULL); // set sp_adj[SP_ADJ_VM] 2822 zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data; 2823} 2824#endif 2825 2826static int zend_jit_setup(void) 2827{ 2828 if (!zend_cpu_supports_sse2()) { 2829 zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); 2830 return FAILURE; 2831 } 2832 allowed_opt_flags = 0; 2833 if (zend_cpu_supports_avx()) { 2834 allowed_opt_flags |= ZEND_JIT_CPU_AVX; 2835 } 2836 2837#if ZTS 2838# ifdef _WIN64 2839 tsrm_tls_index = _tls_index * sizeof(void*); 2840 2841 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ 2842 /* Probably, it might be better solution */ 2843 do { 2844 void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index]; 2845 void *val = _tsrm_ls_cache; 2846 size_t offset = 0; 2847 size_t size = (char*)&_tls_end - (char*)&_tls_start; 2848 2849 while (offset < size) { 2850 if (*tls_mem == val) { 2851 tsrm_tls_offset = offset; 2852 break; 2853 } 2854 tls_mem++; 2855 offset += sizeof(void*); 2856 } 2857 if (offset >= size) { 2858 // TODO: error message ??? 2859 return FAILURE; 2860 } 2861 } while(0); 2862# elif ZEND_WIN32 2863 tsrm_tls_index = _tls_index * sizeof(void*); 2864 2865 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ 2866 /* Probably, it might be better solution */ 2867 do { 2868 void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index]; 2869 void *val = _tsrm_ls_cache; 2870 size_t offset = 0; 2871 size_t size = (char*)&_tls_end - (char*)&_tls_start; 2872 2873 while (offset < size) { 2874 if (*tls_mem == val) { 2875 tsrm_tls_offset = offset; 2876 break; 2877 } 2878 tls_mem++; 2879 offset += sizeof(void*); 2880 } 2881 if (offset >= size) { 2882 // TODO: error message ??? 2883 return FAILURE; 2884 } 2885 } while(0); 2886# elif defined(__APPLE__) && defined(__x86_64__) 2887 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2888 if (tsrm_ls_cache_tcb_offset == 0) { 2889 size_t *ti; 2890 __asm__( 2891 "leaq __tsrm_ls_cache(%%rip),%0" 2892 : "=r" (ti)); 2893 tsrm_tls_offset = ti[2]; 2894 tsrm_tls_index = ti[1] * 8; 2895 } 2896# elif defined(__GNUC__) && defined(__x86_64__) 2897 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2898 if (tsrm_ls_cache_tcb_offset == 0) { 2899#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__) 2900 size_t ret; 2901 2902 asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0" 2903 : "=r" (ret)); 2904 tsrm_ls_cache_tcb_offset = ret; 2905#elif defined(__MUSL__) 2906 size_t *ti; 2907 2908 __asm__( 2909 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" 2910 : "=a" (ti)); 2911 tsrm_tls_offset = ti[1]; 2912 tsrm_tls_index = ti[0] * 8; 2913#elif defined(__FreeBSD__) 2914 size_t *ti; 2915 2916 __asm__( 2917 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" 2918 : "=a" (ti)); 2919 tsrm_tls_offset = ti[1]; 2920 /* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/bf56e8b9c8639ac4447d223b83cdc128107cc3cd/libexec/rtld-elf/rtld.c#L5260) */ 2921 tsrm_tls_index = (ti[0] + 1) * 8; 2922#else 2923 size_t *ti; 2924 2925 __asm__( 2926 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" 2927 : "=a" (ti)); 2928 tsrm_tls_offset = ti[1]; 2929 tsrm_tls_index = ti[0] * 16; 2930#endif 2931 } 2932# elif defined(__GNUC__) && defined(__i386__) 2933 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2934 if (tsrm_ls_cache_tcb_offset == 0) { 2935#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__) 2936 size_t ret; 2937 2938 asm ("leal _tsrm_ls_cache@ntpoff,%0\n" 2939 : "=a" (ret)); 2940 tsrm_ls_cache_tcb_offset = ret; 2941#else 2942 size_t *ti, _ebx, _ecx, _edx; 2943 2944 __asm__( 2945 "call 1f\n" 2946 ".subsection 1\n" 2947 "1:\tmovl (%%esp), %%ebx\n\t" 2948 "ret\n" 2949 ".previous\n\t" 2950 "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" 2951 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t" 2952 "call ___tls_get_addr@plt\n\t" 2953 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n" 2954 : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx)); 2955 tsrm_tls_offset = ti[1]; 2956 tsrm_tls_index = ti[0] * 8; 2957#endif 2958 } 2959# endif 2960#endif 2961 2962 memset(sp_adj, 0, sizeof(sp_adj)); 2963#ifdef HAVE_GDB 2964 sp_adj[SP_ADJ_RET] = sizeof(void*); 2965 |.if X64WIN 2966 || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 0x28; // sub r4, 0x28 2967 |.elif X64 2968 || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 8; // sub r4, 8 2969 |.else 2970 || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 12; // sub r4, 12 2971 |.endif 2972 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2973 zend_jit_set_sp_adj_vm(); // set sp_adj[SP_ADJ_VM] 2974#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 2975 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD 2976#else 2977 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM]; 2978#endif 2979 } else if (GCC_GLOBAL_REGS) { 2980 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD; // sub r4, SPAD 2981 } else { 2982 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD; // sub r4, NR_SPAD 2983 } 2984#endif 2985 2986 return SUCCESS; 2987} 2988 2989static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) 2990{ 2991 | int3 2992 return 1; 2993} 2994 2995static int zend_jit_align_func(dasm_State **Dst) 2996{ 2997 reuse_ip = 0; 2998 delayed_call_chain = 0; 2999 last_valid_opline = NULL; 3000 use_last_vald_opline = 0; 3001 track_last_valid_opline = 0; 3002 jit_return_label = -1; 3003 |.align 16 3004 return 1; 3005} 3006 3007static int zend_jit_align_stub(dasm_State **Dst) 3008{ 3009 |.align 16 3010 return 1; 3011} 3012 3013static int zend_jit_prologue(dasm_State **Dst) 3014{ 3015 | ENDBR 3016 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3017 | SUB_HYBRID_SPAD 3018 } else if (GCC_GLOBAL_REGS) { 3019 | sub r4, SPAD // stack alignment 3020 } else { 3021 | sub r4, NR_SPAD // stack alignment 3022 | mov aword T2, FP // save FP 3023 | mov aword T3, RX // save IP 3024 | mov FP, FCARG1a 3025 } 3026 return 1; 3027} 3028 3029static int zend_jit_label(dasm_State **Dst, unsigned int label) 3030{ 3031 |=>label: 3032 return 1; 3033} 3034 3035static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) 3036{ 3037 | // call->prev_execute_data = EX(call); 3038 if (call_level == 1) { 3039 | mov aword EX:RX->prev_execute_data, 0 3040 } else { 3041 | mov r0, EX->call 3042 | mov EX:RX->prev_execute_data, r0 3043 } 3044 | // EX(call) = call; 3045 | mov EX->call, RX 3046 3047 delayed_call_chain = 0; 3048 3049 return 1; 3050} 3051 3052static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) 3053{ 3054 if (last_valid_opline == opline) { 3055 zend_jit_use_last_valid_opline(); 3056 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 3057 zend_jit_use_last_valid_opline(); 3058 | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); 3059 } else { 3060 | LOAD_IP_ADDR opline 3061 } 3062 zend_jit_set_last_valid_opline(opline); 3063 3064 return 1; 3065} 3066 3067static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg) 3068{ 3069 if (last_valid_opline == opline) { 3070 zend_jit_use_last_valid_opline(); 3071 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 3072 zend_jit_use_last_valid_opline(); 3073 | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); 3074 } else if (!GCC_GLOBAL_REGS && set_ip_reg) { 3075 | LOAD_ADDR RX, opline 3076 | mov aword EX->opline, RX 3077 } else { 3078 | LOAD_IP_ADDR opline 3079 } 3080 zend_jit_set_last_valid_opline(opline); 3081 3082 return 1; 3083} 3084 3085static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) 3086{ 3087 if (delayed_call_chain) { 3088 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 3089 return 0; 3090 } 3091 } 3092 if (!zend_jit_set_ip(Dst, opline)) { 3093 return 0; 3094 } 3095 reuse_ip = 0; 3096 return 1; 3097} 3098 3099static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) 3100{ 3101#if 0 3102 if (!zend_jit_set_valid_ip(Dst, opline)) { 3103 return 0; 3104 } 3105 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3106 | jne ->interrupt_handler 3107#else 3108 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3109 if (exit_addr) { 3110 | jne &exit_addr 3111 } else if (last_valid_opline == opline) { 3112 || zend_jit_use_last_valid_opline(); 3113 | jne ->interrupt_handler 3114 } else { 3115 | jne >1 3116 |.cold_code 3117 |1: 3118 | LOAD_IP_ADDR opline 3119 | jmp ->interrupt_handler 3120 |.code 3121 } 3122#endif 3123 return 1; 3124} 3125 3126static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) 3127{ 3128 if (timeout_exit_addr) { 3129 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3130 | je =>loop_label 3131 | jmp &timeout_exit_addr 3132 } else { 3133 | jmp =>loop_label 3134 } 3135 return 1; 3136} 3137 3138static int zend_jit_check_exception(dasm_State **Dst) 3139{ 3140 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 3141 | jne ->exception_handler 3142 return 1; 3143} 3144 3145static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) 3146{ 3147 if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { 3148 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 3149 | jne ->exception_handler_undef 3150 return 1; 3151 } 3152 return zend_jit_check_exception(Dst); 3153} 3154 3155static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) 3156{ 3157 zend_regset regset = ZEND_REGSET_SCRATCH; 3158 3159#if ZTS 3160 if (1) { 3161#else 3162 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(jit_trace_num)))) { 3163#endif 3164 /* assignment to EG(jit_trace_num) shouldn't clober CPU register used by deoptimizer */ 3165 if (parent) { 3166 int i; 3167 int parent_vars_count = parent->exit_info[exit_num].stack_size; 3168 zend_jit_trace_stack *parent_stack = 3169 parent->stack_map + 3170 parent->exit_info[exit_num].stack_offset; 3171 3172 for (i = 0; i < parent_vars_count; i++) { 3173 if (STACK_REG(parent_stack, i) != ZREG_NONE) { 3174 if (STACK_REG(parent_stack, i) < ZREG_NUM) { 3175 ZEND_REGSET_EXCL(regset, STACK_REG(parent_stack, i)); 3176 } else if (STACK_REG(parent_stack, i) == ZREG_ZVAL_COPY_GPR0) { 3177 ZEND_REGSET_EXCL(regset, ZREG_R0); 3178 } 3179 } 3180 } 3181 } 3182 } 3183 3184 if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { 3185 ZEND_REGSET_EXCL(regset, ZREG_R0); 3186 } 3187 3188 current_trace_num = trace_num; 3189 3190 | // EG(jit_trace_num) = trace_num; 3191 if (regset == ZEND_REGSET_EMPTY) { 3192 | push r0 3193 | MEM_STORE_ZTS dword, executor_globals, jit_trace_num, trace_num, r0 3194 | pop r0 3195 } else { 3196 zend_reg tmp = ZEND_REGSET_FIRST(regset); 3197 3198 | MEM_STORE_ZTS dword, executor_globals, jit_trace_num, trace_num, Ra(tmp) 3199 (void)tmp; 3200 } 3201 3202 return 1; 3203} 3204 3205static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t) 3206{ 3207 |.cold_code 3208 |=>1: // end of the code 3209 |.code 3210 return 1; 3211} 3212 3213/* This taken from LuaJIT. Thanks to Mike Pall. */ 3214static uint32_t _asm_x86_inslen(const uint8_t* p) 3215{ 3216 static const uint8_t map_op1[256] = { 3217 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x20, 3218 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51, 3219 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, 3220 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, 3221#if defined(__x86_64__) || defined(_M_X64) 3222 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x14,0x14,0x14,0x14,0x14,0x14,0x14,0x14, 3223#else 3224 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, 3225#endif 3226 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, 3227 0x51,0x51,0x92,0x92,0x10,0x10,0x12,0x11,0x45,0x86,0x52,0x93,0x51,0x51,0x51,0x51, 3228 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 3229 0x93,0x86,0x93,0x93,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 3230 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x47,0x51,0x51,0x51,0x51,0x51, 3231#if defined(__x86_64__) || defined(_M_X64) 3232 0x59,0x59,0x59,0x59,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, 3233#else 3234 0x55,0x55,0x55,0x55,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, 3235#endif 3236 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x05,0x05,0x05,0x05,0x05,0x05,0x05,0x05, 3237 0x93,0x93,0x53,0x51,0x70,0x71,0x93,0x86,0x54,0x51,0x53,0x51,0x51,0x52,0x51,0x51, 3238 0x92,0x92,0x92,0x92,0x52,0x52,0x51,0x51,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 3239 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x45,0x45,0x47,0x52,0x51,0x51,0x51,0x51, 3240 0x10,0x51,0x10,0x10,0x51,0x51,0x63,0x66,0x51,0x51,0x51,0x51,0x51,0x51,0x92,0x92 3241 }; 3242 static const uint8_t map_op2[256] = { 3243 0x93,0x93,0x93,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x51,0x52,0x51,0x93,0x52,0x94, 3244 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3245 0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3246 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x34,0x51,0x35,0x51,0x51,0x51,0x51,0x51, 3247 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3248 0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3249 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3250 0x94,0x54,0x54,0x54,0x93,0x93,0x93,0x52,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3251 0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46, 3252 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3253 0x52,0x52,0x52,0x93,0x94,0x93,0x51,0x51,0x52,0x52,0x52,0x93,0x94,0x93,0x93,0x93, 3254 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x94,0x93,0x93,0x93,0x93,0x93, 3255 0x93,0x93,0x94,0x93,0x94,0x94,0x94,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 3256 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3257 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3258 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x52 3259 }; 3260 uint32_t result = 0; 3261 uint32_t prefixes = 0; 3262 uint32_t x = map_op1[*p]; 3263 3264 for (;;) { 3265 switch (x >> 4) { 3266 case 0: 3267 return result + x + (prefixes & 4); 3268 case 1: 3269 prefixes |= x; 3270 x = map_op1[*++p]; 3271 result++; 3272 break; 3273 case 2: 3274 x = map_op2[*++p]; 3275 break; 3276 case 3: 3277 p++; 3278 goto mrm; 3279 case 4: 3280 result -= (prefixes & 2); 3281 /* fallthrough */ 3282 case 5: 3283 return result + (x & 15); 3284 case 6: /* Group 3. */ 3285 if (p[1] & 0x38) { 3286 x = 2; 3287 } else if ((prefixes & 2) && (x == 0x66)) { 3288 x = 4; 3289 } 3290 goto mrm; 3291 case 7: /* VEX c4/c5. */ 3292#if !defined(__x86_64__) && !defined(_M_X64) 3293 if (p[1] < 0xc0) { 3294 x = 2; 3295 goto mrm; 3296 } 3297#endif 3298 if (x == 0x70) { 3299 x = *++p & 0x1f; 3300 result++; 3301 if (x >= 2) { 3302 p += 2; 3303 result += 2; 3304 goto mrm; 3305 } 3306 } 3307 p++; 3308 result++; 3309 x = map_op2[*++p]; 3310 break; 3311 case 8: 3312 result -= (prefixes & 2); 3313 /* fallthrough */ 3314 case 9: 3315mrm: 3316 /* ModR/M and possibly SIB. */ 3317 result += (x & 15); 3318 x = *++p; 3319 switch (x >> 6) { 3320 case 0: 3321 if ((x & 7) == 5) { 3322 return result + 4; 3323 } 3324 break; 3325 case 1: 3326 result++; 3327 break; 3328 case 2: 3329 result += 4; 3330 break; 3331 case 3: 3332 return result; 3333 } 3334 if ((x & 7) == 4) { 3335 result++; 3336 if (x < 0x40 && (p[1] & 7) == 5) { 3337 result += 4; 3338 } 3339 } 3340 return result; 3341 } 3342 } 3343} 3344 3345typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t); 3346typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t); 3347 3348static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) 3349{ 3350 int ret = 0; 3351 uint8_t *p, *end; 3352 3353 if (jmp_table_size) { 3354 const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*))); 3355 3356 do { 3357 if (*jmp_slot == from_addr) { 3358 *jmp_slot = to_addr; 3359 ret++; 3360 } 3361 jmp_slot++; 3362 } while (--jmp_table_size); 3363 } 3364 3365 p = (uint8_t*)code; 3366 end = p + size - 5; 3367 while (p < end) { 3368 if ((*(unaligned_uint16_t*)p & 0xf0ff) == 0x800f && p + *(unaligned_int32_t*)(p+2) == (uint8_t*)from_addr - 6) { 3369 *(unaligned_int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6)); 3370 ret++; 3371 } else if (*p == 0xe9 && p + *(unaligned_int32_t*)(p+1) == (uint8_t*)from_addr - 5) { 3372 *(unaligned_int32_t*)(p+1) = ((uint8_t*)to_addr - (p + 5)); 3373 ret++; 3374 } 3375 p += _asm_x86_inslen(p); 3376 } 3377#ifdef HAVE_VALGRIND 3378 VALGRIND_DISCARD_TRANSLATIONS(code, size); 3379#endif 3380 return ret; 3381} 3382 3383static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) 3384{ 3385 return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); 3386} 3387 3388static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) 3389{ 3390 const void *link_addr; 3391 size_t prologue_size; 3392 3393 /* Skip prologue. */ 3394 // TODO: don't hardcode this ??? 3395 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3396#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 3397 prologue_size = 0; 3398#elif defined(__x86_64__) || defined(_M_X64) 3399 // sub r4, HYBRID_SPAD 3400 prologue_size = 4; 3401#else 3402 // sub r4, HYBRID_SPAD 3403 prologue_size = 3; 3404#endif 3405 } else if (GCC_GLOBAL_REGS) { 3406 // sub r4, SPAD // stack alignment 3407#if defined(__x86_64__) || defined(_M_X64) 3408 prologue_size = 4; 3409#else 3410 prologue_size = 3; 3411#endif 3412 } else { 3413 // sub r4, NR_SPAD // stack alignment 3414 // mov aword T2, FP // save FP 3415 // mov aword T3, RX // save IP 3416 // mov FP, FCARG1a 3417#if defined(__x86_64__) || defined(_M_X64) 3418 prologue_size = 17; 3419#else 3420 prologue_size = 13; 3421#endif 3422 } 3423 link_addr = (const void*)((const char*)t->code_start + prologue_size + ENDBR_PADDING); 3424 3425 if (timeout_exit_addr) { 3426 /* Check timeout for links to LOOP */ 3427 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3428 | je &link_addr 3429 | jmp &timeout_exit_addr 3430 } else { 3431 | jmp &link_addr 3432 } 3433 return 1; 3434} 3435 3436static int zend_jit_trace_return(dasm_State **Dst, bool original_handler, const zend_op *opline) 3437{ 3438#if 0 3439 | jmp ->trace_escape 3440#else 3441 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3442 | ADD_HYBRID_SPAD 3443 if (!original_handler) { 3444 | JMP_IP 3445 } else { 3446 | mov r0, EX->func 3447 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3448 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3449 | jmp aword [IP + r0] 3450 } 3451 } else if (GCC_GLOBAL_REGS) { 3452 | add r4, SPAD // stack alignment 3453 if (!original_handler) { 3454 | JMP_IP 3455 } else { 3456 | mov r0, EX->func 3457 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3458 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3459 | jmp aword [IP + r0] 3460 } 3461 } else { 3462 if (original_handler) { 3463 | mov FCARG1a, FP 3464 | mov r0, EX->func 3465 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3466 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3467 | call aword [IP + r0] 3468 } 3469 | mov FP, aword T2 // restore FP 3470 | mov RX, aword T3 // restore IP 3471 | add r4, NR_SPAD // stack alignment 3472 if (!original_handler || !opline || 3473 (opline->opcode != ZEND_RETURN 3474 && opline->opcode != ZEND_RETURN_BY_REF 3475 && opline->opcode != ZEND_GENERATOR_RETURN 3476 && opline->opcode != ZEND_GENERATOR_CREATE 3477 && opline->opcode != ZEND_YIELD 3478 && opline->opcode != ZEND_YIELD_FROM)) { 3479 | mov r0, 2 // ZEND_VM_LEAVE 3480 } 3481 | ret 3482 } 3483#endif 3484 return 1; 3485} 3486 3487static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) 3488{ 3489 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3490 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3491 3492 if (!exit_addr) { 3493 return 0; 3494 } 3495 | IF_NOT_Z_TYPE FP + var, type, &exit_addr 3496 3497 return 1; 3498} 3499 3500static int zend_jit_scalar_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var) 3501{ 3502 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3503 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3504 3505 if (!exit_addr) { 3506 return 0; 3507 } 3508 | cmp byte [FP+var+offsetof(zval, u1.v.type)], IS_STRING 3509 | jae &exit_addr 3510 3511 return 1; 3512} 3513 3514static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) 3515{ 3516 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 3517 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3518 3519 if (!exit_addr) { 3520 return 0; 3521 } 3522 3523 | GET_ZVAL_LVAL ZREG_FCARG1, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var) 3524 if (op_info & MAY_BE_ARRAY_PACKED) { 3525 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 3526 | jz &exit_addr 3527 } else { 3528 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 3529 | jnz &exit_addr 3530 } 3531 3532 return 1; 3533} 3534 3535static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) 3536{ 3537 zend_jit_op_array_trace_extension *jit_extension = 3538 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); 3539 size_t offset = jit_extension->offset; 3540 const void *handler = 3541 (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; 3542 3543 if (!zend_jit_set_valid_ip(Dst, opline)) { 3544 return 0; 3545 } 3546 if (!GCC_GLOBAL_REGS) { 3547 | mov FCARG1a, FP 3548 } 3549 | EXT_CALL handler, r0 3550 if (may_throw 3551 && opline->opcode != ZEND_RETURN 3552 && opline->opcode != ZEND_RETURN_BY_REF) { 3553 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r1 3554 | jne ->exception_handler 3555 } 3556 3557 while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { 3558 trace++; 3559 } 3560 3561 if (!GCC_GLOBAL_REGS 3562 && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { 3563 if (opline->opcode == ZEND_RETURN || 3564 opline->opcode == ZEND_RETURN_BY_REF || 3565 opline->opcode == ZEND_DO_UCALL || 3566 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3567 opline->opcode == ZEND_DO_FCALL || 3568 opline->opcode == ZEND_GENERATOR_CREATE) { 3569 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r1 3570 } 3571 } 3572 3573 if (zend_jit_trace_may_exit(op_array, opline)) { 3574 if (opline->opcode == ZEND_RETURN || 3575 opline->opcode == ZEND_RETURN_BY_REF || 3576 opline->opcode == ZEND_GENERATOR_CREATE) { 3577 3578 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3579 if (trace->op != ZEND_JIT_TRACE_END || 3580 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && 3581 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 3582 /* this check may be handled by the following OPLINE guard or jmp [IP] */ 3583 | cmp IP, zend_jit_halt_op 3584 | je ->trace_halt 3585 } 3586 } else if (GCC_GLOBAL_REGS) { 3587 | test IP, IP 3588 | je ->trace_halt 3589 } else { 3590 | test eax, eax 3591 | jl ->trace_halt 3592 } 3593 } else if (opline->opcode == ZEND_EXIT || 3594 opline->opcode == ZEND_GENERATOR_RETURN || 3595 opline->opcode == ZEND_YIELD || 3596 opline->opcode == ZEND_YIELD_FROM) { 3597 | jmp ->trace_halt 3598 } 3599 if (trace->op != ZEND_JIT_TRACE_END || 3600 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && 3601 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 3602 3603 const zend_op *next_opline = trace->opline; 3604 const zend_op *exit_opline = NULL; 3605 uint32_t exit_point; 3606 const void *exit_addr; 3607 uint32_t old_info = 0; 3608 uint32_t old_res_info = 0; 3609 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 3610 3611 if (zend_is_smart_branch(opline)) { 3612 bool exit_if_true = 0; 3613 exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); 3614 } else { 3615 switch (opline->opcode) { 3616 case ZEND_JMPZ: 3617 case ZEND_JMPNZ: 3618 case ZEND_JMPZ_EX: 3619 case ZEND_JMPNZ_EX: 3620 case ZEND_JMP_SET: 3621 case ZEND_COALESCE: 3622 case ZEND_JMP_NULL: 3623 case ZEND_FE_RESET_R: 3624 case ZEND_FE_RESET_RW: 3625 exit_opline = (trace->opline == opline + 1) ? 3626 OP_JMP_ADDR(opline, opline->op2) : 3627 opline + 1; 3628 break; 3629 case ZEND_FE_FETCH_R: 3630 case ZEND_FE_FETCH_RW: 3631 exit_opline = (trace->opline == opline + 1) ? 3632 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3633 opline + 1; 3634 break; 3635 3636 } 3637 } 3638 3639 switch (opline->opcode) { 3640 case ZEND_FE_FETCH_R: 3641 case ZEND_FE_FETCH_RW: 3642 if (opline->op2_type != IS_UNUSED) { 3643 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); 3644 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); 3645 } 3646 break; 3647 case ZEND_BIND_INIT_STATIC_OR_JMP: 3648 if (opline->op1_type == IS_CV) { 3649 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 3650 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1); 3651 } 3652 break; 3653 } 3654 3655 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3656 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3657 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 3658 } 3659 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); 3660 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3661 3662 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3663 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3664 } 3665 switch (opline->opcode) { 3666 case ZEND_FE_FETCH_R: 3667 case ZEND_FE_FETCH_RW: 3668 if (opline->op2_type != IS_UNUSED) { 3669 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); 3670 } 3671 break; 3672 case ZEND_BIND_INIT_STATIC_OR_JMP: 3673 if (opline->op1_type == IS_CV) { 3674 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info); 3675 } 3676 break; 3677 } 3678 3679 if (!exit_addr) { 3680 return 0; 3681 } 3682 | CMP_IP next_opline 3683 | jne &exit_addr 3684 } 3685 } 3686 3687 zend_jit_set_last_valid_opline(trace->opline); 3688 3689 return 1; 3690} 3691 3692static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) 3693{ 3694 const void *handler; 3695 3696 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3697 handler = zend_get_opcode_handler_func(opline); 3698 } else { 3699 handler = opline->handler; 3700 } 3701 3702 if (!zend_jit_set_valid_ip(Dst, opline)) { 3703 return 0; 3704 } 3705 if (!GCC_GLOBAL_REGS) { 3706 | mov FCARG1a, FP 3707 } 3708 | EXT_CALL handler, r0 3709 if (may_throw) { 3710 zend_jit_check_exception(Dst); 3711 } 3712 3713 /* Skip the following OP_DATA */ 3714 switch (opline->opcode) { 3715 case ZEND_ASSIGN_DIM: 3716 case ZEND_ASSIGN_OBJ: 3717 case ZEND_ASSIGN_STATIC_PROP: 3718 case ZEND_ASSIGN_DIM_OP: 3719 case ZEND_ASSIGN_OBJ_OP: 3720 case ZEND_ASSIGN_STATIC_PROP_OP: 3721 case ZEND_ASSIGN_STATIC_PROP_REF: 3722 case ZEND_ASSIGN_OBJ_REF: 3723 zend_jit_set_last_valid_opline(opline + 2); 3724 break; 3725 default: 3726 zend_jit_set_last_valid_opline(opline + 1); 3727 break; 3728 } 3729 3730 return 1; 3731} 3732 3733static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) 3734{ 3735 if (!zend_jit_set_valid_ip(Dst, opline)) { 3736 return 0; 3737 } 3738 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3739 if (opline->opcode == ZEND_DO_UCALL || 3740 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3741 opline->opcode == ZEND_DO_FCALL || 3742 opline->opcode == ZEND_RETURN) { 3743 3744 /* Use inlined HYBRID VM handler */ 3745 const void *handler = opline->handler; 3746 3747 | ADD_HYBRID_SPAD 3748 | EXT_JMP handler, r0 3749 } else { 3750 const void *handler = zend_get_opcode_handler_func(opline); 3751 3752 | EXT_CALL handler, r0 3753 | ADD_HYBRID_SPAD 3754 | JMP_IP 3755 } 3756 } else { 3757 const void *handler = opline->handler; 3758 3759 if (GCC_GLOBAL_REGS) { 3760 | add r4, SPAD // stack alignment 3761 } else { 3762 | mov FCARG1a, FP 3763 | mov FP, aword T2 // restore FP 3764 | mov RX, aword T3 // restore IP 3765 | add r4, NR_SPAD // stack alignment 3766 } 3767 | EXT_JMP handler, r0 3768 } 3769 zend_jit_reset_last_valid_opline(); 3770 return 1; 3771} 3772 3773static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) 3774{ 3775 uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); 3776 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3777 3778 if (!exit_addr) { 3779 return 0; 3780 } 3781 | CMP_IP opline 3782 | jne &exit_addr 3783 3784 zend_jit_set_last_valid_opline(opline); 3785 3786 return 1; 3787} 3788 3789static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) 3790{ 3791 | jmp =>target_label 3792 return 1; 3793} 3794 3795static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) 3796{ 3797 | CMP_IP next_opline 3798 | jne =>target_label 3799 3800 zend_jit_set_last_valid_opline(next_opline); 3801 3802 return 1; 3803} 3804 3805#ifdef CONTEXT_THREADED_JIT 3806static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3807{ 3808 if (!zend_jit_handler(Dst, opline, 1)) return 0; 3809 if (opline->opcode == ZEND_DO_UCALL) { 3810 | call ->context_threaded_call 3811 } else { 3812 const zend_op *next_opline = opline + 1; 3813 3814 | CMP_IP next_opline 3815 | je =>next_block 3816 | call ->context_threaded_call 3817 } 3818 return 1; 3819} 3820#endif 3821 3822static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3823{ 3824#ifdef CONTEXT_THREADED_JIT 3825 return zend_jit_context_threaded_call(Dst, opline, next_block); 3826#else 3827 return zend_jit_tail_handler(Dst, opline); 3828#endif 3829} 3830 3831static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type) 3832{ 3833 ZEND_ASSERT(Z_MODE(src) == IS_REG); 3834 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); 3835 3836 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3837 | SET_ZVAL_LVAL dst, Ra(Z_REG(src)) 3838 if (set_type && 3839 (Z_REG(dst) != ZREG_FP || 3840 !JIT_G(current_frame) || 3841 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) { 3842 | SET_ZVAL_TYPE_INFO dst, IS_LONG 3843 } 3844 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3845 | DOUBLE_SET_ZVAL_DVAL dst, Z_REG(src) 3846 if (set_type && 3847 (Z_REG(dst) != ZREG_FP || 3848 !JIT_G(current_frame) || 3849 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) { 3850 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 3851 } 3852 } else { 3853 ZEND_UNREACHABLE(); 3854 } 3855 return 1; 3856} 3857 3858static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3859{ 3860 ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); 3861 ZEND_ASSERT(Z_MODE(dst) == IS_REG); 3862 3863 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3864 | GET_ZVAL_LVAL Z_REG(dst), src 3865 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3866 | DOUBLE_GET_ZVAL_DVAL Z_REG(dst), src 3867 } else { 3868 ZEND_UNREACHABLE(); 3869 } 3870 return 1; 3871} 3872 3873static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type) 3874{ 3875 zend_jit_addr src = ZEND_ADDR_REG(reg); 3876 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3877 3878 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3879} 3880 3881static int zend_jit_store_var_type(dasm_State **Dst, int var, uint32_t type) 3882{ 3883 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3884 3885 | SET_ZVAL_TYPE_INFO dst, type 3886 return 1; 3887} 3888 3889static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) 3890{ 3891 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3892 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3893 return zend_jit_spill_store(Dst, src, dst, info, 1); 3894 } 3895 return 1; 3896} 3897 3898static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) 3899{ 3900 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3901 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3902 bool set_type = 1; 3903 3904 if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == 3905 (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { 3906 if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { 3907 set_type = 0; 3908 } 3909 } 3910 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3911 } 3912 return 1; 3913} 3914 3915static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) 3916{ 3917 zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3918 zend_jit_addr dst = ZEND_ADDR_REG(reg); 3919 3920 return zend_jit_load_reg(Dst, src, dst, info); 3921} 3922 3923static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, uint8_t op_type, zend_jit_addr addr, znode_op op) 3924{ 3925 if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) { 3926 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 3927 | SET_ZVAL_TYPE_INFO dst, IS_UNDEF 3928 } 3929 return 1; 3930} 3931 3932static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3933{ 3934 if (!zend_jit_same_addr(src, dst)) { 3935 if (Z_MODE(src) == IS_REG) { 3936 if (Z_MODE(dst) == IS_REG) { 3937 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3938 | mov Ra(Z_REG(dst)), Ra(Z_REG(src)) 3939 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3940 | SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0) 3941 } else { 3942 ZEND_UNREACHABLE(); 3943 } 3944 if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) { 3945 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3946 3947 if (!zend_jit_spill_store(Dst, dst, var_addr, info, 3948 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3949 JIT_G(current_frame) == NULL || 3950 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3951 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3952 )) { 3953 return 0; 3954 } 3955 } 3956 } else if (Z_MODE(dst) == IS_MEM_ZVAL) { 3957 if (!Z_LOAD(src) && !Z_STORE(src)) { 3958 if (!zend_jit_spill_store(Dst, src, dst, info, 3959 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3960 JIT_G(current_frame) == NULL || 3961 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3962 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3963 )) { 3964 return 0; 3965 } 3966 } 3967 } else { 3968 ZEND_UNREACHABLE(); 3969 } 3970 } else if (Z_MODE(src) == IS_MEM_ZVAL) { 3971 if (Z_MODE(dst) == IS_REG) { 3972 if (!zend_jit_load_reg(Dst, src, dst, info)) { 3973 return 0; 3974 } 3975 } else { 3976 ZEND_UNREACHABLE(); 3977 } 3978 } else { 3979 ZEND_UNREACHABLE(); 3980 } 3981 } else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) { 3982 dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3983 if (!zend_jit_spill_store(Dst, src, dst, info, 3984 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3985 JIT_G(current_frame) == NULL || 3986 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3987 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3988 )) { 3989 return 0; 3990 } 3991 } 3992 return 1; 3993} 3994 3995static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) 3996{ 3997 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 3998 3999 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1 4000 4001 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { 4002 if (!zend_jit_save_call_chain(Dst, -1)) { 4003 return 0; 4004 } 4005 } 4006 4007 ZEND_ASSERT(opline); 4008 4009 if ((opline-1)->opcode != ZEND_FETCH_CONSTANT 4010 && (opline-1)->opcode != ZEND_FETCH_LIST_R 4011 && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR)) 4012 && !(flags & ZEND_JIT_EXIT_FREE_OP1)) { 4013 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var); 4014 4015 | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 4016 | GET_ZVAL_PTR r0, val_addr 4017 | GC_ADDREF r0 4018 |2: 4019 } 4020 4021 | LOAD_IP_ADDR (opline - 1) 4022 | jmp ->trace_escape 4023 |1: 4024 4025 return 1; 4026} 4027 4028static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) 4029{ 4030 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 4031 4032 if (reg == ZREG_LONG_MIN_MINUS_1) { 4033 |.if X64 4034 | SET_ZVAL_LVAL dst, 0x00000000 4035 | SET_ZVAL_W2 dst, 0xc3e00000 4036 |.else 4037 | SET_ZVAL_LVAL dst, 0x00200000 4038 | SET_ZVAL_W2 dst, 0xc1e00000 4039 |.endif 4040 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 4041 } else if (reg == ZREG_LONG_MIN) { 4042 |.if X64 4043 | SET_ZVAL_LVAL dst, 0x00000000 4044 | SET_ZVAL_W2 dst, 0x80000000 4045 |.else 4046 | SET_ZVAL_LVAL dst, ZEND_LONG_MIN 4047 |.endif 4048 | SET_ZVAL_TYPE_INFO dst, IS_LONG 4049 } else if (reg == ZREG_LONG_MAX) { 4050 |.if X64 4051 | SET_ZVAL_LVAL dst, 0xffffffff 4052 | SET_ZVAL_W2 dst, 0x7fffffff 4053 |.else 4054 | SET_ZVAL_LVAL dst, ZEND_LONG_MAX 4055 |.endif 4056 | SET_ZVAL_TYPE_INFO dst, IS_LONG 4057 } else if (reg == ZREG_LONG_MAX_PLUS_1) { 4058 |.if X64 4059 | SET_ZVAL_LVAL dst, 0 4060 | SET_ZVAL_W2 dst, 0x43e00000 4061 |.else 4062 | SET_ZVAL_LVAL dst, 0 4063 | SET_ZVAL_W2 dst, 0x41e00000 4064 |.endif 4065 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 4066 } else if (reg == ZREG_NULL) { 4067 | SET_ZVAL_TYPE_INFO dst, IS_NULL 4068 } else if (reg == ZREG_ZVAL_TRY_ADDREF) { 4069 | IF_NOT_ZVAL_REFCOUNTED dst, >1 4070 | GET_ZVAL_PTR r1, dst 4071 | GC_ADDREF r1 4072 |1: 4073 } else if (reg == ZREG_ZVAL_COPY_GPR0) { 4074 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 4075 4076 | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_R1, ZREG_R2 4077 | TRY_ADDREF -1, ch, r2 4078 } else { 4079 ZEND_UNREACHABLE(); 4080 } 4081 return 1; 4082} 4083 4084static int zend_jit_free_trampoline(dasm_State **Dst) 4085{ 4086 | /// if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) 4087 | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_CALL_VIA_TRAMPOLINE 4088 | jz >1 4089 | mov FCARG1a, r0 4090 | EXT_CALL zend_jit_free_trampoline_helper, r0 4091 |1: 4092 return 1; 4093} 4094 4095static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 4096{ 4097 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { 4098 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 4099 } 4100 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4101 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4102 } 4103 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { 4104 return 0; 4105 } 4106 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4107 | LONG_OP_WITH_32BIT_CONST add, op1_def_addr, Z_L(1) 4108 } else { 4109 | LONG_OP_WITH_32BIT_CONST sub, op1_def_addr, Z_L(1) 4110 } 4111 4112 if (may_overflow && 4113 (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || 4114 ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { 4115 int32_t exit_point; 4116 const void *exit_addr; 4117 zend_jit_trace_stack *stack; 4118 uint32_t old_op1_info, old_res_info = 0; 4119 4120 stack = JIT_G(current_frame)->stack; 4121 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 4122 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); 4123 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4124 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); 4125 } else { 4126 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); 4127 } 4128 if (opline->result_type != IS_UNUSED) { 4129 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 4130 if (opline->opcode == ZEND_PRE_INC) { 4131 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 4132 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); 4133 } else if (opline->opcode == ZEND_PRE_DEC) { 4134 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 4135 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); 4136 } else if (opline->opcode == ZEND_POST_INC) { 4137 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 4138 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); 4139 } else if (opline->opcode == ZEND_POST_DEC) { 4140 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 4141 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); 4142 } 4143 } 4144 4145 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 4146 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4147 if (!exit_addr) { 4148 return 0; 4149 } 4150 | jo &exit_addr 4151 4152 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4153 opline->result_type != IS_UNUSED) { 4154 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4155 } 4156 4157 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 4158 if (opline->result_type != IS_UNUSED) { 4159 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 4160 } 4161 } else if (may_overflow) { 4162 | jo >1 4163 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4164 opline->result_type != IS_UNUSED) { 4165 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4166 } 4167 |.cold_code 4168 |1: 4169 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4170 |.if X64 4171 | mov64 rax, 0x43e0000000000000 4172 | SET_ZVAL_LVAL op1_def_addr, rax 4173 |.else 4174 | SET_ZVAL_LVAL op1_def_addr, 0 4175 | SET_ZVAL_W2 op1_def_addr, 0x41e00000 4176 |.endif 4177 } else { 4178 |.if X64 4179 | mov64 rax, 0xc3e0000000000000 4180 | SET_ZVAL_LVAL op1_def_addr, rax 4181 |.else 4182 | SET_ZVAL_LVAL op1_def_addr, 0x00200000 4183 | SET_ZVAL_W2 op1_def_addr, 0xc1e00000 4184 |.endif 4185 } 4186 if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { 4187 | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE 4188 } 4189 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4190 opline->result_type != IS_UNUSED) { 4191 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1 4192 } 4193 | jmp >3 4194 |.code 4195 } else { 4196 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4197 opline->result_type != IS_UNUSED) { 4198 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4199 } 4200 } 4201 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 4202 |.cold_code 4203 |2: 4204 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4205 | SET_EX_OPLINE opline, r0 4206 if (op1_info & MAY_BE_UNDEF) { 4207 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2 4208 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 4209 | mov FCARG1d, opline->op1.var 4210 | EXT_CALL zend_jit_undefined_op_helper, r0 4211 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 4212 op1_info |= MAY_BE_NULL; 4213 } 4214 |2: 4215 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 4216 4217 | // ZVAL_DEREF(var_ptr); 4218 if (op1_info & MAY_BE_REF) { 4219 | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >2 4220 | GET_Z_PTR FCARG1a, FCARG1a 4221 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 4222 | jz >1 4223 if (RETURN_VALUE_USED(opline)) { 4224 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4225 } else { 4226 | xor FCARG2a, FCARG2a 4227 } 4228 if (opline->opcode == ZEND_PRE_INC) { 4229 | EXT_CALL zend_jit_pre_inc_typed_ref, r0 4230 } else if (opline->opcode == ZEND_PRE_DEC) { 4231 | EXT_CALL zend_jit_pre_dec_typed_ref, r0 4232 } else if (opline->opcode == ZEND_POST_INC) { 4233 | EXT_CALL zend_jit_post_inc_typed_ref, r0 4234 } else if (opline->opcode == ZEND_POST_DEC) { 4235 | EXT_CALL zend_jit_post_dec_typed_ref, r0 4236 } else { 4237 ZEND_UNREACHABLE(); 4238 } 4239 zend_jit_check_exception(Dst); 4240 | jmp >3 4241 |1: 4242 | lea FCARG1a, [FCARG1a + offsetof(zend_reference, val)] 4243 |2: 4244 } 4245 4246 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4247 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 4248 4249 | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2 4250 | TRY_ADDREF op1_info, ah, r2 4251 } 4252 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4253 if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { 4254 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4255 | EXT_CALL zend_jit_pre_inc, r0 4256 } else { 4257 | EXT_CALL increment_function, r0 4258 } 4259 } else { 4260 if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { 4261 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4262 | EXT_CALL zend_jit_pre_dec, r0 4263 } else { 4264 | EXT_CALL decrement_function, r0 4265 } 4266 } 4267 if (may_throw) { 4268 zend_jit_check_exception(Dst); 4269 } 4270 } else { 4271 zend_reg tmp_reg; 4272 4273 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4274 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2 4275 } 4276 if (Z_MODE(op1_def_addr) == IS_REG) { 4277 tmp_reg = Z_REG(op1_def_addr); 4278 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4279 tmp_reg = Z_REG(op1_addr); 4280 } else { 4281 tmp_reg = ZREG_XMM0; 4282 } 4283 | DOUBLE_GET_ZVAL_DVAL tmp_reg, op1_addr 4284 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4285 if (CAN_USE_AVX()) { 4286 | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] 4287 } else { 4288 | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] 4289 } 4290 } else { 4291 if (CAN_USE_AVX()) { 4292 | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] 4293 } else { 4294 | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] 4295 } 4296 } 4297 | DOUBLE_SET_ZVAL_DVAL op1_def_addr, tmp_reg 4298 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4299 opline->result_type != IS_UNUSED) { 4300 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1 4301 | TRY_ADDREF op1_def_info, ah, r1 4302 } 4303 } 4304 | jmp >3 4305 |.code 4306 } 4307 |3: 4308 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { 4309 return 0; 4310 } 4311 if (opline->result_type != IS_UNUSED) { 4312 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4313 return 0; 4314 } 4315 } 4316 return 1; 4317} 4318 4319static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) 4320{ 4321 if ((opline+1)->opcode == ZEND_OP_DATA 4322 && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) 4323 && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { 4324 return 1; 4325 } 4326 return 4327 ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4328 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || 4329 ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4330 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || 4331 ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4332 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); 4333} 4334 4335static int zend_jit_math_long_long(dasm_State **Dst, 4336 const zend_op *opline, 4337 uint8_t opcode, 4338 zend_jit_addr op1_addr, 4339 zend_jit_addr op2_addr, 4340 zend_jit_addr res_addr, 4341 uint32_t res_info, 4342 uint32_t res_use_info, 4343 int may_overflow) 4344{ 4345 bool must_set_cflags = 0; 4346 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4347 zend_reg result_reg; 4348 zend_reg tmp_reg = ZREG_R0; 4349 4350 if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { 4351 if (may_overflow && (res_info & MAY_BE_GUARD) 4352 && JIT_G(current_frame) 4353 && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { 4354 result_reg = ZREG_R0; 4355 } else { 4356 result_reg = Z_REG(res_addr); 4357 } 4358 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { 4359 result_reg = Z_REG(op1_addr); 4360 } else if (Z_REG(res_addr) != ZREG_R0) { 4361 result_reg = ZREG_R0; 4362 } else { 4363 /* ASSIGN_DIM_OP */ 4364 result_reg = ZREG_FCARG1; 4365 tmp_reg = ZREG_FCARG1; 4366 } 4367 4368 if (may_overflow) { 4369 must_set_cflags = 1; 4370 } else { 4371 const zend_op *next_opline = opline + 1; 4372 4373 if (next_opline->opcode == ZEND_IS_EQUAL || 4374 next_opline->opcode == ZEND_IS_NOT_EQUAL || 4375 next_opline->opcode == ZEND_IS_SMALLER || 4376 next_opline->opcode == ZEND_IS_SMALLER_OR_EQUAL || 4377 next_opline->opcode == ZEND_CASE || 4378 next_opline->opcode == ZEND_IS_IDENTICAL || 4379 next_opline->opcode == ZEND_IS_NOT_IDENTICAL || 4380 next_opline->opcode == ZEND_CASE_STRICT) { 4381 if (next_opline->op1_type == IS_CONST 4382 && Z_TYPE_P(RT_CONSTANT(next_opline, next_opline->op1)) == IS_LONG 4383 && Z_LVAL_P(RT_CONSTANT(next_opline, next_opline->op1)) == 0 4384 && next_opline->op2_type == opline->result_type 4385 && next_opline->op2.var == opline->result.var) { 4386 must_set_cflags = 1; 4387 } else if (next_opline->op2_type == IS_CONST 4388 && Z_TYPE_P(RT_CONSTANT(next_opline, next_opline->op2)) == IS_LONG 4389 && Z_LVAL_P(RT_CONSTANT(next_opline, next_opline->op2)) == 0 4390 && next_opline->op2_type == opline->result_type 4391 && next_opline->op2.var == opline->result.var) { 4392 must_set_cflags = 1; 4393 } 4394 } 4395 } 4396 4397 if (opcode == ZEND_MUL && 4398 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4399 Z_LVAL_P(Z_ZV(op2_addr)) == 2) { 4400 if (Z_MODE(op1_addr) == IS_REG && !must_set_cflags) { 4401 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] 4402 } else { 4403 | GET_ZVAL_LVAL result_reg, op1_addr 4404 | add Ra(result_reg), Ra(result_reg) 4405 } 4406 } else if (opcode == ZEND_MUL && 4407 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4408 !must_set_cflags && 4409 Z_LVAL_P(Z_ZV(op2_addr)) > 0 && 4410 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { 4411 | GET_ZVAL_LVAL result_reg, op1_addr 4412 | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4413 } else if (opcode == ZEND_MUL && 4414 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4415 Z_LVAL_P(Z_ZV(op1_addr)) == 2) { 4416 if (Z_MODE(op2_addr) == IS_REG && !must_set_cflags) { 4417 | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Ra(Z_REG(op2_addr))] 4418 } else { 4419 | GET_ZVAL_LVAL result_reg, op2_addr 4420 | add Ra(result_reg), Ra(result_reg) 4421 } 4422 } else if (opcode == ZEND_MUL && 4423 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4424 !must_set_cflags && 4425 Z_LVAL_P(Z_ZV(op1_addr)) > 0 && 4426 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { 4427 | GET_ZVAL_LVAL result_reg, op2_addr 4428 | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) 4429 } else if (opcode == ZEND_DIV && 4430 (Z_MODE(op2_addr) == IS_CONST_ZVAL && 4431 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { 4432 | GET_ZVAL_LVAL result_reg, op1_addr 4433 | shr Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4434 } else if (opcode == ZEND_ADD && 4435 !must_set_cflags && 4436 Z_MODE(op1_addr) == IS_REG && 4437 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4438 IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr)))) { 4439 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Z_LVAL_P(Z_ZV(op2_addr))] 4440 } else if (opcode == ZEND_ADD && 4441 !must_set_cflags && 4442 Z_MODE(op2_addr) == IS_REG && 4443 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4444 IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr)))) { 4445 | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Z_LVAL_P(Z_ZV(op1_addr))] 4446 } else if (opcode == ZEND_SUB && 4447 !must_set_cflags && 4448 Z_MODE(op1_addr) == IS_REG && 4449 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4450 IS_SIGNED_32BIT(-Z_LVAL_P(Z_ZV(op2_addr)))) { 4451 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))-Z_LVAL_P(Z_ZV(op2_addr))] 4452 } else { 4453 | GET_ZVAL_LVAL result_reg, op1_addr 4454 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4455 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4456 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4457 /* +/- 0 */ 4458 may_overflow = 0; 4459 } else if (same_ops && opcode != ZEND_DIV) { 4460 | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) 4461 } else { 4462 zend_reg tmp_reg; 4463 4464 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4465 if (Z_REG(res_addr) != ZREG_R0 && result_reg != ZREG_R0) { 4466 tmp_reg = ZREG_R0; 4467 } else if (Z_REG(res_addr) != ZREG_R1 && result_reg != ZREG_R1) { 4468 tmp_reg = ZREG_R1; 4469 } else { 4470 tmp_reg = ZREG_R2; 4471 } 4472 } else if (result_reg != ZREG_R0) { 4473 tmp_reg = ZREG_R0; 4474 } else { 4475 tmp_reg = ZREG_R1; 4476 } 4477 | LONG_MATH opcode, result_reg, op2_addr, tmp_reg 4478 (void)tmp_reg; 4479 } 4480 } 4481 if (may_overflow) { 4482 if (res_info & MAY_BE_GUARD) { 4483 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 4484 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4485 if (!exit_addr) { 4486 return 0; 4487 } 4488 if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { 4489 | jo &exit_addr 4490 if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { 4491 | mov Ra(Z_REG(res_addr)), Ra(result_reg) 4492 } 4493 } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 4494 | jno &exit_addr 4495 } else { 4496 ZEND_UNREACHABLE(); 4497 } 4498 } else { 4499 if (res_info & MAY_BE_LONG) { 4500 | jo >1 4501 } else { 4502 | jno >1 4503 } 4504 } 4505 } 4506 4507 if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { 4508 | SET_ZVAL_LVAL res_addr, Ra(result_reg) 4509 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4510 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4511 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 4512 } 4513 } 4514 } 4515 4516 if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { 4517 zend_reg tmp_reg1 = ZREG_XMM0; 4518 zend_reg tmp_reg2 = ZREG_XMM1; 4519 4520 if (res_info & MAY_BE_LONG) { 4521 |.cold_code 4522 |1: 4523 } 4524 4525 do { 4526 if ((sizeof(void*) == 8 || Z_MODE(res_addr) != IS_REG) && 4527 ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || 4528 (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1))) { 4529 if (opcode == ZEND_ADD) { 4530 |.if X64 4531 | mov64 Ra(tmp_reg), 0x43e0000000000000 4532 if (Z_MODE(res_addr) == IS_REG) { 4533 | movd xmm(Z_REG(res_addr)-ZREG_XMM0), Ra(tmp_reg) 4534 } else { 4535 | SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 4536 } 4537 |.else 4538 | SET_ZVAL_LVAL res_addr, 0 4539 | SET_ZVAL_W2 res_addr, 0x41e00000 4540 |.endif 4541 break; 4542 } else if (opcode == ZEND_SUB) { 4543 |.if X64 4544 | mov64 Ra(tmp_reg), 0xc3e0000000000000 4545 if (Z_MODE(res_addr) == IS_REG) { 4546 | movd xmm(Z_REG(res_addr)-ZREG_XMM0), Ra(tmp_reg) 4547 } else { 4548 | SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 4549 } 4550 |.else 4551 | SET_ZVAL_LVAL res_addr, 0x00200000 4552 | SET_ZVAL_W2 res_addr, 0xc1e00000 4553 |.endif 4554 break; 4555 } 4556 } 4557 4558 | DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg 4559 | DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg 4560 if (CAN_USE_AVX()) { 4561 | AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 4562 } else { 4563 | SSE_MATH_REG opcode, tmp_reg1, tmp_reg2 4564 } 4565 | DOUBLE_SET_ZVAL_DVAL res_addr, tmp_reg1 4566 } while (0); 4567 4568 if (Z_MODE(res_addr) == IS_MEM_ZVAL 4569 && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4570 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4571 } 4572 if (res_info & MAY_BE_LONG) { 4573 | jmp >2 4574 |.code 4575 } 4576 |2: 4577 } 4578 4579 return 1; 4580} 4581 4582static int zend_jit_math_long_double(dasm_State **Dst, 4583 uint8_t opcode, 4584 zend_jit_addr op1_addr, 4585 zend_jit_addr op2_addr, 4586 zend_jit_addr res_addr, 4587 uint32_t res_use_info) 4588{ 4589 zend_reg result_reg = 4590 (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; 4591 zend_reg tmp_reg; 4592 4593 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4594 /* ASSIGN_DIM_OP */ 4595 tmp_reg = ZREG_R1; 4596 } else { 4597 tmp_reg = ZREG_R0; 4598 } 4599 4600 | DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, tmp_reg 4601 4602 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4603 /* ASSIGN_DIM_OP */ 4604 if (CAN_USE_AVX()) { 4605 | AVX_MATH opcode, result_reg, result_reg, op2_addr, r1 4606 } else { 4607 | SSE_MATH opcode, result_reg, op2_addr, r1 4608 } 4609 } else { 4610 if (CAN_USE_AVX()) { 4611 | AVX_MATH opcode, result_reg, result_reg, op2_addr, r0 4612 } else { 4613 | SSE_MATH opcode, result_reg, op2_addr, r0 4614 } 4615 } 4616 | DOUBLE_SET_ZVAL_DVAL res_addr, result_reg 4617 4618 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4619 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4620 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4621 } 4622 } 4623 4624 return 1; 4625} 4626 4627static int zend_jit_math_double_long(dasm_State **Dst, 4628 uint8_t opcode, 4629 zend_jit_addr op1_addr, 4630 zend_jit_addr op2_addr, 4631 zend_jit_addr res_addr, 4632 uint32_t res_use_info) 4633{ 4634 zend_reg result_reg, tmp_reg_gp; 4635 4636 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4637 /* ASSIGN_DIM_OP */ 4638 tmp_reg_gp = ZREG_R1; 4639 } else { 4640 tmp_reg_gp = ZREG_R0; 4641 } 4642 4643 if (zend_is_commutative(opcode) 4644 && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { 4645 if (Z_MODE(res_addr) == IS_REG) { 4646 result_reg = Z_REG(res_addr); 4647 } else { 4648 result_reg = ZREG_XMM0; 4649 } 4650 | DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, tmp_reg_gp 4651 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4652 /* ASSIGN_DIM_OP */ 4653 if (CAN_USE_AVX()) { 4654 | AVX_MATH opcode, result_reg, result_reg, op1_addr, r1 4655 } else { 4656 | SSE_MATH opcode, result_reg, op1_addr, r1 4657 } 4658 } else { 4659 if (CAN_USE_AVX()) { 4660 | AVX_MATH opcode, result_reg, result_reg, op1_addr, r0 4661 } else { 4662 | SSE_MATH opcode, result_reg, op1_addr, r0 4663 } 4664 } 4665 } else { 4666 zend_reg tmp_reg; 4667 4668 if (Z_MODE(res_addr) == IS_REG) { 4669 result_reg = Z_REG(res_addr); 4670 tmp_reg = (result_reg == ZREG_XMM0) ? ZREG_XMM1 : ZREG_XMM0; 4671 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4672 result_reg = Z_REG(op1_addr); 4673 tmp_reg = ZREG_XMM0; 4674 } else { 4675 result_reg = ZREG_XMM0; 4676 tmp_reg = ZREG_XMM1; 4677 } 4678 if (CAN_USE_AVX()) { 4679 zend_reg op1_reg; 4680 4681 if (Z_MODE(op1_addr) == IS_REG) { 4682 op1_reg = Z_REG(op1_addr); 4683 } else { 4684 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4685 op1_reg = result_reg; 4686 } 4687 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4688 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4689 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4690 /* +/- 0 */ 4691 } else { 4692 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp 4693 | AVX_MATH_REG opcode, result_reg, op1_reg, tmp_reg 4694 } 4695 } else { 4696 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4697 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4698 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4699 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4700 /* +/- 0 */ 4701 } else { 4702 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp 4703 | SSE_MATH_REG opcode, result_reg, tmp_reg 4704 } 4705 } 4706 } 4707 | DOUBLE_SET_ZVAL_DVAL res_addr, result_reg 4708 4709 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4710 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4711 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4712 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4713 } 4714 } 4715 } 4716 4717 return 1; 4718} 4719 4720static int zend_jit_math_double_double(dasm_State **Dst, 4721 uint8_t opcode, 4722 zend_jit_addr op1_addr, 4723 zend_jit_addr op2_addr, 4724 zend_jit_addr res_addr, 4725 uint32_t res_use_info) 4726{ 4727 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4728 zend_reg result_reg; 4729 4730 if (Z_MODE(res_addr) == IS_REG) { 4731 result_reg = Z_REG(res_addr); 4732 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4733 result_reg = Z_REG(op1_addr); 4734 } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { 4735 result_reg = Z_REG(op2_addr); 4736 } else { 4737 result_reg = ZREG_XMM0; 4738 } 4739 4740 if (CAN_USE_AVX()) { 4741 zend_reg op1_reg; 4742 zend_jit_addr val_addr; 4743 4744 if (Z_MODE(op1_addr) == IS_REG) { 4745 op1_reg = Z_REG(op1_addr); 4746 val_addr = op2_addr; 4747 } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4748 op1_reg = Z_REG(op2_addr); 4749 val_addr = op1_addr; 4750 } else { 4751 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4752 op1_reg = result_reg; 4753 val_addr = op2_addr; 4754 } 4755 if ((opcode == ZEND_MUL) && 4756 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4757 | AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg 4758 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4759 /* ASSIGN_DIM_OP */ 4760 | AVX_MATH opcode, result_reg, op1_reg, val_addr, r1 4761 } else { 4762 | AVX_MATH opcode, result_reg, op1_reg, val_addr, r0 4763 } 4764 } else { 4765 zend_jit_addr val_addr; 4766 4767 if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4768 | DOUBLE_GET_ZVAL_DVAL result_reg, op2_addr 4769 val_addr = op1_addr; 4770 } else { 4771 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4772 val_addr = op2_addr; 4773 } 4774 if (same_ops) { 4775 | SSE_MATH_REG opcode, result_reg, result_reg 4776 } else if ((opcode == ZEND_MUL) && 4777 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4778 | SSE_MATH_REG ZEND_ADD, result_reg, result_reg 4779 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4780 /* ASSIGN_DIM_OP */ 4781 | SSE_MATH opcode, result_reg, val_addr, r1 4782 } else { 4783 | SSE_MATH opcode, result_reg, val_addr, r0 4784 } 4785 } 4786 | DOUBLE_SET_ZVAL_DVAL res_addr, result_reg 4787 4788 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4789 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4790 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4791 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4792 } 4793 } 4794 } 4795 4796 return 1; 4797} 4798 4799static int zend_jit_math_helper(dasm_State **Dst, 4800 const zend_op *opline, 4801 uint8_t opcode, 4802 uint8_t op1_type, 4803 znode_op op1, 4804 zend_jit_addr op1_addr, 4805 uint32_t op1_info, 4806 uint8_t op2_type, 4807 znode_op op2, 4808 zend_jit_addr op2_addr, 4809 uint32_t op2_info, 4810 uint32_t res_var, 4811 zend_jit_addr res_addr, 4812 uint32_t res_info, 4813 uint32_t res_use_info, 4814 int may_overflow, 4815 int may_throw) 4816/* Labels: 1,2,3,4,5,6 */ 4817{ 4818 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4819 4820 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4821 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { 4822 if (op1_info & MAY_BE_DOUBLE) { 4823 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 4824 } else { 4825 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 4826 } 4827 } 4828 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { 4829 if (op2_info & MAY_BE_DOUBLE) { 4830 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1 4831 |.cold_code 4832 |1: 4833 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4834 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4835 } 4836 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4837 return 0; 4838 } 4839 | jmp >5 4840 |.code 4841 } else { 4842 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4843 } 4844 } 4845 if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { 4846 return 0; 4847 } 4848 if (op1_info & MAY_BE_DOUBLE) { 4849 |.cold_code 4850 |3: 4851 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4852 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4853 } 4854 if (op2_info & MAY_BE_DOUBLE) { 4855 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4856 if (!same_ops) { 4857 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1 4858 } else { 4859 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6 4860 } 4861 } 4862 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4863 return 0; 4864 } 4865 | jmp >5 4866 } 4867 if (!same_ops) { 4868 |1: 4869 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4870 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4871 } 4872 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4873 return 0; 4874 } 4875 | jmp >5 4876 } 4877 |.code 4878 } 4879 } else if ((op1_info & MAY_BE_DOUBLE) && 4880 !(op1_info & MAY_BE_LONG) && 4881 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4882 (res_info & MAY_BE_DOUBLE)) { 4883 if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4884 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4885 } 4886 if (op2_info & MAY_BE_DOUBLE) { 4887 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4888 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4889 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1 4890 } else { 4891 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4892 } 4893 } 4894 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4895 return 0; 4896 } 4897 } 4898 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4899 if (op2_info & MAY_BE_DOUBLE) { 4900 |.cold_code 4901 } 4902 |1: 4903 if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4904 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4905 } 4906 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4907 return 0; 4908 } 4909 if (op2_info & MAY_BE_DOUBLE) { 4910 | jmp >5 4911 |.code 4912 } 4913 } 4914 } else if ((op2_info & MAY_BE_DOUBLE) && 4915 !(op2_info & MAY_BE_LONG) && 4916 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4917 (res_info & MAY_BE_DOUBLE)) { 4918 if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4919 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4920 } 4921 if (op1_info & MAY_BE_DOUBLE) { 4922 if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4923 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4924 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1 4925 } else { 4926 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4927 } 4928 } 4929 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4930 return 0; 4931 } 4932 } 4933 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4934 if (op1_info & MAY_BE_DOUBLE) { 4935 |.cold_code 4936 } 4937 |1: 4938 if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4939 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 4940 } 4941 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4942 return 0; 4943 } 4944 if (op1_info & MAY_BE_DOUBLE) { 4945 | jmp >5 4946 |.code 4947 } 4948 } 4949 } 4950 4951 |5: 4952 4953 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 4954 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 4955 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4956 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4957 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4958 |.cold_code 4959 } 4960 |6: 4961 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4962 if (Z_MODE(res_addr) == IS_REG) { 4963 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4964 | LOAD_ZVAL_ADDR FCARG1a, real_addr 4965 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4966 | LOAD_ZVAL_ADDR FCARG1a, res_addr 4967 } 4968 if (Z_MODE(op1_addr) == IS_REG) { 4969 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4970 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4971 return 0; 4972 } 4973 op1_addr = real_addr; 4974 } 4975 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 4976 } else { 4977 if (Z_MODE(op1_addr) == IS_REG) { 4978 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4979 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4980 return 0; 4981 } 4982 op1_addr = real_addr; 4983 } 4984 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 4985 if (Z_MODE(res_addr) == IS_REG) { 4986 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4987 | LOAD_ZVAL_ADDR FCARG1a, real_addr 4988 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4989 | LOAD_ZVAL_ADDR FCARG1a, res_addr 4990 } 4991 } 4992 4993 if (Z_MODE(op2_addr) == IS_REG) { 4994 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 4995 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 4996 return 0; 4997 } 4998 op2_addr = real_addr; 4999 } 5000 |.if X64 5001 | LOAD_ZVAL_ADDR CARG3, op2_addr 5002 |.else 5003 | sub r4, 12 5004 | PUSH_ZVAL_ADDR op2_addr, r0 5005 |.endif 5006 | SET_EX_OPLINE opline, r0 5007 if (opcode == ZEND_ADD) { 5008 | EXT_CALL add_function, r0 5009 } else if (opcode == ZEND_SUB) { 5010 | EXT_CALL sub_function, r0 5011 } else if (opcode == ZEND_MUL) { 5012 | EXT_CALL mul_function, r0 5013 } else if (opcode == ZEND_DIV) { 5014 | EXT_CALL div_function, r0 5015 } else { 5016 ZEND_UNREACHABLE(); 5017 } 5018 |.if not(X64) 5019 | add r4, 12 5020 |.endif 5021 | FREE_OP op1_type, op1, op1_info, 0, NULL 5022 | FREE_OP op2_type, op2, op2_info, 0, NULL 5023 if (may_throw) { 5024 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 5025 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 5026 | jne ->exception_handler_free_op2 5027 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5028 zend_jit_check_exception_undef_result(Dst, opline); 5029 } else { 5030 zend_jit_check_exception(Dst); 5031 } 5032 } 5033 if (Z_MODE(res_addr) == IS_REG) { 5034 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5035 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 5036 return 0; 5037 } 5038 } 5039 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 5040 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 5041 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 5042 | jmp <5 5043 |.code 5044 } 5045 } 5046 5047 return 1; 5048} 5049 5050static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 5051{ 5052 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5053 ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 5054 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); 5055 5056 if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { 5057 return 0; 5058 } 5059 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 5060 return 0; 5061 } 5062 return 1; 5063} 5064 5065static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr) 5066{ 5067 if (Z_MODE(op2_addr) != IS_MEM_ZVAL || Z_REG(op2_addr) != ZREG_FCARG1) { 5068 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 5069 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5070 } else if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG2) { 5071 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5072 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 5073 } else { 5074 | GET_ZVAL_LVAL ZREG_R0, op2_addr 5075 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 5076 | mov FCARG2a, r0 5077 } 5078 | EXT_CALL zend_jit_add_arrays_helper, r0 5079 | SET_ZVAL_PTR res_addr, r0 5080 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX 5081 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 5082 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 5083 return 1; 5084} 5085 5086static int zend_jit_long_math_helper(dasm_State **Dst, 5087 const zend_op *opline, 5088 uint8_t opcode, 5089 uint8_t op1_type, 5090 znode_op op1, 5091 zend_jit_addr op1_addr, 5092 uint32_t op1_info, 5093 zend_ssa_range *op1_range, 5094 uint8_t op2_type, 5095 znode_op op2, 5096 zend_jit_addr op2_addr, 5097 uint32_t op2_info, 5098 zend_ssa_range *op2_range, 5099 uint32_t res_var, 5100 zend_jit_addr res_addr, 5101 uint32_t res_info, 5102 uint32_t res_use_info, 5103 int may_throw) 5104/* Labels: 6 */ 5105{ 5106 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 5107 zend_reg result_reg; 5108 5109 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 5110 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 5111 } 5112 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 5113 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 5114 } 5115 5116 if (opcode == ZEND_MOD) { 5117 result_reg = ZREG_RAX; 5118 } else if (Z_MODE(res_addr) == IS_REG) { 5119 if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) 5120 && opline->op2_type != IS_CONST) { 5121 result_reg = ZREG_R0; 5122 } else { 5123 result_reg = Z_REG(res_addr); 5124 } 5125 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 5126 result_reg = Z_REG(op1_addr); 5127 } else if (Z_REG(res_addr) != ZREG_R0) { 5128 result_reg = ZREG_R0; 5129 } else { 5130 /* ASSIGN_DIM_OP */ 5131 if (ZREG_FCARG1 == ZREG_RCX 5132 && (opcode == ZEND_SL || opcode == ZEND_SR) 5133 && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 5134 result_reg = ZREG_R2; 5135 } else { 5136 result_reg = ZREG_FCARG1; 5137 } 5138 } 5139 5140 if (opcode == ZEND_SL) { 5141 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5142 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5143 5144 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 5145 if (EXPECTED(op2_lval > 0)) { 5146 | xor Ra(result_reg), Ra(result_reg) 5147 } else { 5148 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5149 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5150 | SET_EX_OPLINE opline, r0 5151 | jmp ->negative_shift 5152 } 5153 } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { 5154 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] 5155 } else { 5156 | GET_ZVAL_LVAL result_reg, op1_addr 5157 | shl Ra(result_reg), op2_lval 5158 } 5159 } else { 5160 if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { 5161 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5162 } 5163 if (!op2_range || 5164 op2_range->min < 0 || 5165 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 5166 | cmp r1, (SIZEOF_ZEND_LONG*8) 5167 | jae >1 5168 |.cold_code 5169 |1: 5170 | cmp r1, 0 5171 | mov Ra(result_reg), 0 5172 | jg >1 5173 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5174 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5175 | SET_EX_OPLINE opline, r0 5176 | jmp ->negative_shift 5177 |.code 5178 } 5179 | GET_ZVAL_LVAL result_reg, op1_addr 5180 | shl Ra(result_reg), cl 5181 |1: 5182 } 5183 } else if (opcode == ZEND_SR) { 5184 | GET_ZVAL_LVAL result_reg, op1_addr 5185 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5186 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5187 5188 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 5189 if (EXPECTED(op2_lval > 0)) { 5190 | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1 5191 } else { 5192 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5193 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5194 | SET_EX_OPLINE opline, r0 5195 | jmp ->negative_shift 5196 } 5197 } else { 5198 | sar Ra(result_reg), op2_lval 5199 } 5200 } else { 5201 if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { 5202 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5203 } 5204 if (!op2_range || 5205 op2_range->min < 0 || 5206 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 5207 | cmp r1, (SIZEOF_ZEND_LONG*8) 5208 | jae >1 5209 |.cold_code 5210 |1: 5211 | cmp r1, 0 5212 | mov r1, (SIZEOF_ZEND_LONG * 8) - 1 5213 | jg >1 5214 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5215 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5216 | SET_EX_OPLINE opline, r0 5217 | jmp ->negative_shift 5218 |.code 5219 } 5220 |1: 5221 | sar Ra(result_reg), cl 5222 } 5223 } else if (opcode == ZEND_MOD) { 5224 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5225 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5226 5227 if (op2_lval == 0) { 5228 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5229 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5230 | SET_EX_OPLINE opline, r0 5231 | jmp ->mod_by_zero 5232 } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) { 5233 zval tmp; 5234 zend_jit_addr tmp_addr; 5235 zend_reg tmp_reg; 5236 5237 /* Optimisation for mod of power of 2 */ 5238 ZVAL_LONG(&tmp, op2_lval - 1); 5239 tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp); 5240 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 5241 tmp_reg = ZREG_R1; 5242 } else if (result_reg != ZREG_R0) { 5243 tmp_reg = ZREG_R0; 5244 } else { 5245 tmp_reg = ZREG_R1; 5246 } 5247 | GET_ZVAL_LVAL result_reg, op1_addr 5248 | LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, tmp_reg 5249 (void)tmp_reg; 5250 } else { 5251 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5252 | mov aword T1, r0 // save 5253 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) { 5254 | mov aword T1, Ra(ZREG_RCX) // save 5255 } 5256 result_reg = ZREG_RDX; 5257 if (op2_lval == -1) { 5258 | xor Ra(result_reg), Ra(result_reg) 5259 } else { 5260 | GET_ZVAL_LVAL ZREG_RAX, op1_addr 5261 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5262 |.if X64 5263 | cqo 5264 |.else 5265 | cdq 5266 |.endif 5267 | idiv Ra(ZREG_RCX) 5268 } 5269 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5270 | mov r0, aword T1 // restore 5271 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) { 5272 | mov Ra(ZREG_RCX), aword T1 // restore 5273 } 5274 } 5275 } else { 5276 if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { 5277 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5278 | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], 0 5279 } else if (Z_MODE(op2_addr) == IS_REG) { 5280 | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) 5281 } 5282 | jz >1 5283 |.cold_code 5284 |1: 5285 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5286 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5287 | SET_EX_OPLINE opline, r0 5288 | jmp ->mod_by_zero 5289 |.code 5290 } 5291 5292 /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ 5293 if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { 5294 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5295 | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], -1 5296 } else if (Z_MODE(op2_addr) == IS_REG) { 5297 | cmp Ra(Z_REG(op2_addr)), -1 5298 } 5299 | jz >1 5300 |.cold_code 5301 |1: 5302 | SET_ZVAL_LVAL res_addr, 0 5303 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5304 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 5305 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 5306 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 5307 } 5308 } 5309 } 5310 | jmp >5 5311 |.code 5312 } 5313 5314 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5315 | mov aword T1, r0 // save 5316 } 5317 result_reg = ZREG_RDX; 5318 | GET_ZVAL_LVAL ZREG_RAX, op1_addr 5319 |.if X64 5320 | cqo 5321 |.else 5322 | cdq 5323 |.endif 5324 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5325 | idiv aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)] 5326 } else if (Z_MODE(op2_addr) == IS_REG) { 5327 | idiv Ra(Z_REG(op2_addr)) 5328 } 5329 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5330 | mov r0, aword T1 // restore 5331 } 5332 } 5333 } else if (same_ops) { 5334 | GET_ZVAL_LVAL result_reg, op1_addr 5335 | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) 5336 } else { 5337 zend_reg tmp_reg; 5338 5339 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5340 if (Z_REG(res_addr) != ZREG_R0 && result_reg != ZREG_R0) { 5341 tmp_reg = ZREG_R0; 5342 } else if (Z_REG(res_addr) != ZREG_R1 && result_reg != ZREG_R1) { 5343 tmp_reg = ZREG_R1; 5344 } else { 5345 tmp_reg = ZREG_R2; 5346 } 5347 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R1) { 5348 tmp_reg = ZREG_R0; 5349 } else if (result_reg != ZREG_R0) { 5350 tmp_reg = ZREG_R0; 5351 } else { 5352 tmp_reg = ZREG_R1; 5353 } 5354 | GET_ZVAL_LVAL result_reg, op1_addr 5355 | LONG_MATH opcode, result_reg, op2_addr, tmp_reg 5356 (void)tmp_reg; 5357 } 5358 5359 if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { 5360 | SET_ZVAL_LVAL res_addr, Ra(result_reg) 5361 } 5362 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5363 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 5364 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 5365 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 5366 } 5367 } 5368 } 5369 5370 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || 5371 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 5372 if ((op1_info & MAY_BE_LONG) && 5373 (op2_info & MAY_BE_LONG)) { 5374 |.cold_code 5375 } 5376 |6: 5377 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 5378 if (Z_MODE(res_addr) == IS_REG) { 5379 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5380 | LOAD_ZVAL_ADDR FCARG1a, real_addr 5381 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5382 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5383 } 5384 if (Z_MODE(op1_addr) == IS_REG) { 5385 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 5386 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 5387 return 0; 5388 } 5389 op1_addr = real_addr; 5390 } 5391 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5392 } else { 5393 if (Z_MODE(op1_addr) == IS_REG) { 5394 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 5395 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 5396 return 0; 5397 } 5398 op1_addr = real_addr; 5399 } 5400 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5401 if (Z_MODE(res_addr) == IS_REG) { 5402 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5403 | LOAD_ZVAL_ADDR FCARG1a, real_addr 5404 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5405 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5406 } 5407 } 5408 if (Z_MODE(op2_addr) == IS_REG) { 5409 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 5410 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 5411 return 0; 5412 } 5413 op2_addr = real_addr; 5414 } 5415 |.if X64 5416 | LOAD_ZVAL_ADDR CARG3, op2_addr 5417 |.else 5418 | sub r4, 12 5419 | PUSH_ZVAL_ADDR op2_addr, r0 5420 |.endif 5421 | SET_EX_OPLINE opline, r0 5422 if (opcode == ZEND_BW_OR) { 5423 | EXT_CALL bitwise_or_function, r0 5424 } else if (opcode == ZEND_BW_AND) { 5425 | EXT_CALL bitwise_and_function, r0 5426 } else if (opcode == ZEND_BW_XOR) { 5427 | EXT_CALL bitwise_xor_function, r0 5428 } else if (opcode == ZEND_SL) { 5429 | EXT_CALL shift_left_function, r0 5430 } else if (opcode == ZEND_SR) { 5431 | EXT_CALL shift_right_function, r0 5432 } else if (opcode == ZEND_MOD) { 5433 | EXT_CALL mod_function, r0 5434 } else { 5435 ZEND_UNREACHABLE(); 5436 } 5437 |.if not(X64) 5438 | add r4, 12 5439 |.endif 5440 if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) { 5441 /* compound assignment may decrement "op2" refcount */ 5442 op2_info |= MAY_BE_RC1; 5443 } 5444 | FREE_OP op1_type, op1, op1_info, 0, NULL 5445 | FREE_OP op2_type, op2, op2_info, 0, NULL 5446 if (may_throw) { 5447 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 5448 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 5449 | jne ->exception_handler_free_op2 5450 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5451 zend_jit_check_exception_undef_result(Dst, opline); 5452 } else { 5453 zend_jit_check_exception(Dst); 5454 } 5455 } 5456 if (Z_MODE(res_addr) == IS_REG) { 5457 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5458 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 5459 return 0; 5460 } 5461 } 5462 if ((op1_info & MAY_BE_LONG) && 5463 (op2_info & MAY_BE_LONG)) { 5464 | jmp >5 5465 |.code 5466 } 5467 } 5468 |5: 5469 5470 return 1; 5471} 5472 5473static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 5474{ 5475 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5476 ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); 5477 5478 if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, 5479 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 5480 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 5481 opline->result.var, res_addr, res_info, res_use_info, may_throw)) { 5482 return 0; 5483 } 5484 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 5485 return 0; 5486 } 5487 return 1; 5488} 5489 5490static int zend_jit_concat_helper(dasm_State **Dst, 5491 const zend_op *opline, 5492 uint8_t op1_type, 5493 znode_op op1, 5494 zend_jit_addr op1_addr, 5495 uint32_t op1_info, 5496 uint8_t op2_type, 5497 znode_op op2, 5498 zend_jit_addr op2_addr, 5499 uint32_t op2_info, 5500 zend_jit_addr res_addr, 5501 int may_throw) 5502{ 5503#if 1 5504 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5505 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5506 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 5507 } 5508 if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5509 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6 5510 } 5511 if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { 5512 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5513 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5514 } 5515 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 5516 | EXT_CALL zend_jit_fast_assign_concat_helper, r0 5517 /* concatenation with itself may reduce refcount */ 5518 op2_info |= MAY_BE_RC1; 5519 } else { 5520 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5521 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5522 } 5523 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5524 |.if X64 5525 | LOAD_ZVAL_ADDR CARG3, op2_addr 5526 |.else 5527 | sub r4, 12 5528 | PUSH_ZVAL_ADDR op2_addr, r0 5529 |.endif 5530 if (op1_type == IS_CV || op1_type == IS_CONST) { 5531 | EXT_CALL zend_jit_fast_concat_helper, r0 5532 } else { 5533 | EXT_CALL zend_jit_fast_concat_tmp_helper, r0 5534 } 5535 |.if not(X64) 5536 | add r4, 12 5537 |.endif 5538 } 5539 /* concatenation with empty string may increase refcount */ 5540 op2_info |= MAY_BE_RCN; 5541 | FREE_OP op2_type, op2, op2_info, 0, opline 5542 |5: 5543 } 5544 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || 5545 (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { 5546 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5547 |.cold_code 5548 |6: 5549 } 5550#endif 5551 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 5552 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5553 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5554 } 5555 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5556 } else { 5557 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5558 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5559 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5560 } 5561 } 5562 |.if X64 5563 | LOAD_ZVAL_ADDR CARG3, op2_addr 5564 |.else 5565 | sub r4, 12 5566 | PUSH_ZVAL_ADDR op2_addr, r0 5567 |.endif 5568 | SET_EX_OPLINE opline, r0 5569 | EXT_CALL concat_function, r0 5570 |.if not(X64) 5571 | add r4, 12 5572 |.endif 5573 /* concatenation with empty string may increase refcount */ 5574 op1_info |= MAY_BE_RCN; 5575 op2_info |= MAY_BE_RCN; 5576 | FREE_OP op1_type, op1, op1_info, 0, NULL 5577 | FREE_OP op2_type, op2, op2_info, 0, NULL 5578 if (may_throw) { 5579 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5580 zend_jit_check_exception_undef_result(Dst, opline); 5581 } else { 5582 zend_jit_check_exception(Dst); 5583 } 5584 } 5585#if 1 5586 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5587 | jmp <5 5588 |.code 5589 } 5590 } 5591#endif 5592 5593 return 1; 5594} 5595 5596static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) 5597{ 5598 zend_jit_addr op1_addr, op2_addr; 5599 5600 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5601 ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); 5602 5603 op1_addr = OP1_ADDR(); 5604 op2_addr = OP2_ADDR(); 5605 5606 return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); 5607} 5608 5609static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) 5610/* Labels: 1,2,3,4,5 */ 5611{ 5612 zend_jit_addr op2_addr = OP2_ADDR(); 5613 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 5614 5615 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 5616 && type == BP_VAR_R 5617 && !exit_addr) { 5618 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5619 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5620 if (!exit_addr) { 5621 return 0; 5622 } 5623 } 5624 5625 if (op2_info & MAY_BE_LONG) { 5626 bool op2_loaded = 0; 5627 bool packed_loaded = 0; 5628 bool bad_packed_key = 0; 5629 5630 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { 5631 | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) 5632 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 5633 } 5634 if (op1_info & MAY_BE_PACKED_GUARD) { 5635 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 5636 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5637 5638 if (!exit_addr) { 5639 return 0; 5640 } 5641 if (op1_info & MAY_BE_ARRAY_PACKED) { 5642 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5643 | jz &exit_addr 5644 } else { 5645 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5646 | jnz &exit_addr 5647 } 5648 } 5649 if (type == BP_VAR_W) { 5650 | // hval = Z_LVAL_P(dim); 5651 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5652 op2_loaded = 1; 5653 } 5654 if (op1_info & MAY_BE_ARRAY_PACKED) { 5655 zend_long val = -1; 5656 5657 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5658 val = Z_LVAL_P(Z_ZV(op2_addr)); 5659 if (val >= 0 && val < HT_MAX_SIZE) { 5660 packed_loaded = 1; 5661 } else { 5662 bad_packed_key = 1; 5663 } 5664 } else { 5665 if (!op2_loaded) { 5666 | // hval = Z_LVAL_P(dim); 5667 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5668 op2_loaded = 1; 5669 } 5670 packed_loaded = 1; 5671 } 5672 5673 if (dim_type == IS_UNDEF && type == BP_VAR_W && packed_loaded) { 5674 /* don't generate "fast" code for packed array */ 5675 packed_loaded = 0; 5676 } 5677 5678 if (packed_loaded) { 5679 | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); 5680 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5681 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5682 | jz >4 // HASH_FIND 5683 } 5684 | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) 5685 |.if X64 5686 | mov eax, dword [FCARG1a + offsetof(zend_array, nNumUsed)] 5687 if (val == 0) { 5688 | test r0, r0 5689 } else if (val > 0 && !op2_loaded) { 5690 | cmp r0, val 5691 } else { 5692 | cmp r0, FCARG2a 5693 } 5694 |.else 5695 if (val >= 0 && !op2_loaded) { 5696 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val 5697 } else { 5698 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a 5699 } 5700 |.endif 5701 if (type == BP_JIT_IS) { 5702 if (not_found_exit_addr) { 5703 | jbe ¬_found_exit_addr 5704 } else { 5705 | jbe >9 // NOT_FOUND 5706 } 5707 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5708 | jbe &exit_addr 5709 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5710 | jbe ¬_found_exit_addr 5711 } else if (type == BP_VAR_RW && not_found_exit_addr) { 5712 | jbe ¬_found_exit_addr 5713 } else if (type == BP_VAR_IS && found_exit_addr) { 5714 | jbe >7 // NOT_FOUND 5715 } else { 5716 | jbe >2 // NOT_FOUND 5717 } 5718 | // _ret = &_ht->arPacked[h]; 5719 if (val >= 0) { 5720 | mov r0, aword [FCARG1a + offsetof(zend_array, arPacked)] 5721 if (val != 0) { 5722 | add r0, val * sizeof(zval) 5723 } 5724 } else { 5725 |.if X64 5726 | mov r0, FCARG2a 5727 | shl r0, 4 5728 |.else 5729 | imul r0, FCARG2a, sizeof(zval) 5730 |.endif 5731 | add r0, aword [FCARG1a + offsetof(zend_array, arPacked)] 5732 } 5733 } 5734 } 5735 switch (type) { 5736 case BP_JIT_IS: 5737 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5738 if (packed_loaded) { 5739 | jmp >5 5740 } 5741 |4: 5742 if (!op2_loaded) { 5743 | // hval = Z_LVAL_P(dim); 5744 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5745 } 5746 if (packed_loaded) { 5747 | EXT_CALL _zend_hash_index_find, r0 5748 } else { 5749 | EXT_CALL zend_hash_index_find, r0 5750 } 5751 | test r0, r0 5752 if (not_found_exit_addr) { 5753 | jz ¬_found_exit_addr 5754 } else { 5755 | jz >9 // NOT_FOUND 5756 } 5757 if (op2_info & MAY_BE_STRING) { 5758 | jmp >5 5759 } 5760 } else if (packed_loaded) { 5761 if (op2_info & MAY_BE_STRING) { 5762 | jmp >5 5763 } 5764 } else if (not_found_exit_addr) { 5765 | jmp ¬_found_exit_addr 5766 } else { 5767 | jmp >9 // NOT_FOUND 5768 } 5769 break; 5770 case BP_VAR_R: 5771 case BP_VAR_IS: 5772 case BP_VAR_UNSET: 5773 if (packed_loaded) { 5774 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5775 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5776 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5777 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 5778 if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5779 | IF_Z_TYPE r0, IS_UNDEF, &exit_addr 5780 } 5781 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5782 | IF_Z_TYPE r0, IS_UNDEF, ¬_found_exit_addr 5783 } else if (type == BP_VAR_IS && found_exit_addr) { 5784 | IF_Z_TYPE r0, IS_UNDEF, >7 // NOT_FOUND 5785 } else { 5786 | IF_Z_TYPE r0, IS_UNDEF, >2 // NOT_FOUND 5787 } 5788 } 5789 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_NUMERIC_HASH))) { 5790 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5791 | jmp &exit_addr 5792 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5793 | jmp ¬_found_exit_addr 5794 } else if (type == BP_VAR_IS && found_exit_addr) { 5795 | jmp >7 // NOT_FOUND 5796 } else { 5797 | jmp >2 // NOT_FOUND 5798 } 5799 } 5800 if (!packed_loaded || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5801 |4: 5802 if (!op2_loaded) { 5803 | // hval = Z_LVAL_P(dim); 5804 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5805 } 5806 if (packed_loaded) { 5807 | EXT_CALL _zend_hash_index_find, r0 5808 } else { 5809 | EXT_CALL zend_hash_index_find, r0 5810 } 5811 | test r0, r0 5812 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5813 | jz &exit_addr 5814 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5815 | jz ¬_found_exit_addr 5816 } else if (type == BP_VAR_IS && found_exit_addr) { 5817 | jz >7 // NOT_FOUND 5818 } else { 5819 | jz >2 // NOT_FOUND 5820 } 5821 } 5822 |.cold_code 5823 |2: 5824 switch (type) { 5825 case BP_VAR_R: 5826 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5827 | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); 5828 | // retval = &EG(uninitialized_zval); 5829 | UNDEFINED_OFFSET opline 5830 | jmp >9 5831 } 5832 break; 5833 case BP_VAR_IS: 5834 case BP_VAR_UNSET: 5835 if (!not_found_exit_addr && !found_exit_addr) { 5836 | // retval = &EG(uninitialized_zval); 5837 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 5838 | jmp >9 5839 } 5840 break; 5841 default: 5842 ZEND_UNREACHABLE(); 5843 } 5844 |.code 5845 break; 5846 case BP_VAR_RW: 5847 if (packed_loaded && !not_found_exit_addr) { 5848 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5849 } 5850 if (!packed_loaded || 5851 !not_found_exit_addr || 5852 (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5853 if (packed_loaded && not_found_exit_addr) { 5854 |.cold_code 5855 } 5856 |2: 5857 |4: 5858 if (!op2_loaded) { 5859 | // hval = Z_LVAL_P(dim); 5860 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5861 } 5862 if (packed_loaded) { 5863 | EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, r0 5864 } else { 5865 | EXT_CALL zend_jit_hash_index_lookup_rw, r0 5866 } 5867 | test r0, r0 5868 if (not_found_exit_addr) { 5869 if (packed_loaded) { 5870 | jnz >8 5871 | jmp ¬_found_exit_addr 5872 |.code 5873 } else { 5874 | jz ¬_found_exit_addr 5875 } 5876 } else { 5877 | jz >9 5878 } 5879 } 5880 break; 5881 case BP_VAR_W: 5882 if (packed_loaded) { 5883 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5884 } 5885 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) { 5886 |2: 5887 |4: 5888 if (!op2_loaded) { 5889 | // hval = Z_LVAL_P(dim); 5890 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5891 } 5892 | EXT_CALL zend_hash_index_lookup, r0 5893 } 5894 break; 5895 default: 5896 ZEND_UNREACHABLE(); 5897 } 5898 5899 if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { 5900 | jmp >8 5901 } 5902 } 5903 5904 if (op2_info & MAY_BE_STRING) { 5905 |3: 5906 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5907 | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) 5908 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3 5909 } 5910 | // offset_key = Z_STR_P(dim); 5911 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5912 | // retval = zend_hash_find(ht, offset_key); 5913 switch (type) { 5914 case BP_JIT_IS: 5915 if (opline->op2_type != IS_CONST) { 5916 | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' 5917 | jle >1 5918 |.cold_code 5919 |1: 5920 | EXT_CALL zend_jit_symtable_find, r0 5921 | jmp >1 5922 |.code 5923 | EXT_CALL zend_hash_find, r0 5924 |1: 5925 } else { 5926 | EXT_CALL zend_hash_find_known_hash, r0 5927 } 5928 | test r0, r0 5929 if (not_found_exit_addr) { 5930 | jz ¬_found_exit_addr 5931 } else { 5932 | jz >9 // NOT_FOUND 5933 } 5934 break; 5935 case BP_VAR_R: 5936 case BP_VAR_IS: 5937 case BP_VAR_UNSET: 5938 if (opline->op2_type != IS_CONST) { 5939 | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' 5940 | jle >1 5941 |.cold_code 5942 |1: 5943 | EXT_CALL zend_jit_symtable_find, r0 5944 | jmp >1 5945 |.code 5946 | EXT_CALL zend_hash_find, r0 5947 |1: 5948 } else { 5949 | EXT_CALL zend_hash_find_known_hash, r0 5950 } 5951 | test r0, r0 5952 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5953 | jz &exit_addr 5954 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5955 | jz ¬_found_exit_addr 5956 } else if (type == BP_VAR_IS && found_exit_addr) { 5957 | jz >7 // NOT_FOUND 5958 } else { 5959 | jz >2 // NOT_FOUND 5960 |.cold_code 5961 |2: 5962 switch (type) { 5963 case BP_VAR_R: 5964 // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); 5965 | UNDEFINED_INDEX opline 5966 | jmp >9 5967 break; 5968 case BP_VAR_IS: 5969 case BP_VAR_UNSET: 5970 | // retval = &EG(uninitialized_zval); 5971 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 5972 | jmp >9 5973 break; 5974 default: 5975 ZEND_UNREACHABLE(); 5976 } 5977 |.code 5978 } 5979 break; 5980 case BP_VAR_RW: 5981 if (opline->op2_type != IS_CONST) { 5982 | EXT_CALL zend_jit_symtable_lookup_rw, r0 5983 } else { 5984 | EXT_CALL zend_jit_hash_lookup_rw, r0 5985 } 5986 | test r0, r0 5987 if (not_found_exit_addr) { 5988 | jz ¬_found_exit_addr 5989 } else { 5990 | jz >9 5991 } 5992 break; 5993 case BP_VAR_W: 5994 if (opline->op2_type != IS_CONST) { 5995 | EXT_CALL zend_jit_symtable_lookup_w, r0 5996 } else { 5997 | EXT_CALL zend_hash_lookup, r0 5998 } 5999 break; 6000 default: 6001 ZEND_UNREACHABLE(); 6002 } 6003 } 6004 6005 if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { 6006 |5: 6007 if (op1_info & MAY_BE_ARRAY_OF_REF) { 6008 | ZVAL_DEREF r0, MAY_BE_REF 6009 } 6010 | cmp byte [r0 + 8], IS_NULL 6011 if (not_found_exit_addr) { 6012 | jle ¬_found_exit_addr 6013 } else if (found_exit_addr) { 6014 | jg &found_exit_addr 6015 } else { 6016 | jle >9 // NOT FOUND 6017 } 6018 } 6019 6020 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 6021 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 6022 |.cold_code 6023 |3: 6024 } 6025 if (type != BP_VAR_RW) { 6026 | SET_EX_OPLINE opline, r0 6027 } 6028 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6029 switch (type) { 6030 case BP_VAR_R: 6031 |.if X64 6032 | LOAD_ZVAL_ADDR CARG3, res_addr 6033 |.else 6034 | sub r4, 12 6035 | PUSH_ZVAL_ADDR res_addr, r0 6036 |.endif 6037 | EXT_CALL zend_jit_fetch_dim_r_helper, r0 6038 |.if not(X64) 6039 | add r4, 12 6040 |.endif 6041 | jmp >9 6042 break; 6043 case BP_JIT_IS: 6044 | EXT_CALL zend_jit_fetch_dim_isset_helper, r0 6045 | test r0, r0 6046 if (not_found_exit_addr) { 6047 | je ¬_found_exit_addr 6048 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 6049 | jmp >8 6050 } 6051 } else if (found_exit_addr) { 6052 | jne &found_exit_addr 6053 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 6054 | jmp >9 6055 } 6056 } else { 6057 | jne >8 6058 | jmp >9 6059 } 6060 break; 6061 case BP_VAR_IS: 6062 case BP_VAR_UNSET: 6063 |.if X64 6064 | LOAD_ZVAL_ADDR CARG3, res_addr 6065 |.else 6066 | sub r4, 12 6067 | PUSH_ZVAL_ADDR res_addr, r0 6068 |.endif 6069 | EXT_CALL zend_jit_fetch_dim_is_helper, r0 6070 |.if not(X64) 6071 | add r4, 12 6072 |.endif 6073 | jmp >9 6074 break; 6075 case BP_VAR_RW: 6076 | EXT_CALL zend_jit_fetch_dim_rw_helper, r0 6077 | test r0, r0 6078 | jne >8 6079 | jmp >9 6080 break; 6081 case BP_VAR_W: 6082 | EXT_CALL zend_jit_fetch_dim_w_helper, r0 6083 | test r0, r0 6084 | jne >8 6085 | jmp >9 6086 break; 6087 default: 6088 ZEND_UNREACHABLE(); 6089 } 6090 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 6091 |.code 6092 } 6093 } 6094 6095 return 1; 6096} 6097 6098static int zend_jit_simple_assign(dasm_State **Dst, 6099 const zend_op *opline, 6100 zend_jit_addr var_addr, 6101 uint32_t var_info, 6102 uint32_t var_def_info, 6103 uint8_t val_type, 6104 zend_jit_addr val_addr, 6105 uint32_t val_info, 6106 zend_jit_addr res_addr, 6107 int in_cold, 6108 int save_r1, 6109 bool check_exception) 6110/* Labels: 1,2,3 */ 6111{ 6112 zend_reg tmp_reg; 6113 6114 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_R0) { 6115 tmp_reg = ZREG_R0; 6116 } else { 6117 /* ASSIGN_DIM */ 6118 tmp_reg = ZREG_FCARG1; 6119 } 6120 6121 if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 6122 zval *zv = Z_ZV(val_addr); 6123 6124 if (!res_addr) { 6125 | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg 6126 } else { 6127 | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg 6128 } 6129 if (Z_REFCOUNTED_P(zv)) { 6130 if (!res_addr) { 6131 | ADDREF_CONST zv, Ra(tmp_reg) 6132 } else { 6133 | ADDREF_CONST_2 zv, Ra(tmp_reg) 6134 } 6135 } 6136 } else { 6137 if (val_info & MAY_BE_UNDEF) { 6138 if (in_cold) { 6139 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2 6140 } else { 6141 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 6142 |.cold_code 6143 |1: 6144 } 6145 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 6146 if (save_r1) { 6147 | mov aword T1, FCARG1a // save 6148 } 6149 | SET_ZVAL_TYPE_INFO var_addr, IS_NULL 6150 if (res_addr) { 6151 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 6152 } 6153 if (opline) { 6154 | SET_EX_OPLINE opline, Ra(tmp_reg) 6155 } 6156 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); 6157 | mov FCARG1d, Z_OFFSET(val_addr) 6158 | EXT_CALL zend_jit_undefined_op_helper, r0 6159 if (check_exception) { 6160 | test r0, r0 6161 | jz ->exception_handler_undef 6162 } 6163 if (save_r1) { 6164 | mov FCARG1a, aword T1 // restore 6165 } 6166 | jmp >3 6167 if (in_cold) { 6168 |2: 6169 } else { 6170 |.code 6171 } 6172 } 6173 if (val_info & MAY_BE_REF) { 6174 if (val_type == IS_CV) { 6175 ZEND_ASSERT(Z_REG(var_addr) != ZREG_R2); 6176 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_R2 || Z_OFFSET(val_addr) != 0) { 6177 | LOAD_ZVAL_ADDR r2, val_addr 6178 } 6179 | ZVAL_DEREF r2, val_info 6180 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); 6181 } else { 6182 zend_jit_addr ref_addr; 6183 zend_reg type_reg = tmp_reg; 6184 6185 if (in_cold) { 6186 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 6187 } else { 6188 | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1 6189 |.cold_code 6190 |1: 6191 } 6192 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 6193 | GET_ZVAL_PTR r2, val_addr 6194 | GC_DELREF r2 6195 | // ZVAL_COPY_VALUE(return_value, &ref->value); 6196 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8); 6197 if (!res_addr) { 6198 | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, type_reg, tmp_reg 6199 } else { 6200 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, type_reg, tmp_reg 6201 } 6202 | je >2 6203 if (tmp_reg == ZREG_R0) { 6204 | IF_NOT_REFCOUNTED ah, >3 6205 } else { 6206 | IF_NOT_FLAGS Rd(tmp_reg), (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), >3 6207 } 6208 | GET_ZVAL_PTR Ra(tmp_reg), var_addr 6209 6210 if (!res_addr) { 6211 | GC_ADDREF Ra(tmp_reg) 6212 } else { 6213 | add dword [Ra(tmp_reg)], 2 6214 } 6215 | jmp >3 6216 |2: 6217 if (res_addr) { 6218 if (tmp_reg == ZREG_R0) { 6219 | IF_NOT_REFCOUNTED ah, >2 6220 } else { 6221 | IF_NOT_FLAGS Rd(tmp_reg), (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), >2 6222 } 6223 | GET_ZVAL_PTR Ra(tmp_reg), var_addr 6224 | GC_ADDREF Ra(tmp_reg) 6225 |2: 6226 } 6227 if (save_r1) { 6228 | mov aword T1, FCARG1a // save 6229 } 6230 | EFREE_REFERENCE r2 6231 if (save_r1) { 6232 | mov FCARG1a, aword T1 // restore 6233 } 6234 | jmp >3 6235 if (in_cold) { 6236 |1: 6237 } else { 6238 |.code 6239 } 6240 } 6241 } 6242 6243 if (!res_addr) { 6244 | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, tmp_reg 6245 } else { 6246 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_R2, tmp_reg 6247 } 6248 6249 if (val_type == IS_CV) { 6250 if (!res_addr) { 6251 | TRY_ADDREF val_info, dh, Ra(tmp_reg) 6252 } else { 6253 | TRY_ADDREF_2 val_info, dh, Ra(tmp_reg) 6254 } 6255 } else { 6256 if (res_addr) { 6257 | TRY_ADDREF val_info, dh, Ra(tmp_reg) 6258 } 6259 } 6260 |3: 6261 } 6262 return 1; 6263} 6264 6265static int zend_jit_assign_to_typed_ref(dasm_State **Dst, 6266 const zend_op *opline, 6267 uint8_t val_type, 6268 zend_jit_addr val_addr, 6269 zend_jit_addr res_addr, 6270 bool check_exception) 6271{ 6272 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 6273 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6274 | jnz >2 6275 |.cold_code 6276 |2: 6277 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 6278 | LOAD_ZVAL_ADDR FCARG2a, val_addr 6279 } 6280 if (opline) { 6281 | SET_EX_OPLINE opline, r0 6282 } 6283 if (!res_addr) { 6284 if (val_type == IS_CONST) { 6285 | EXT_CALL zend_jit_assign_const_to_typed_ref, r0 6286 } else if (val_type == IS_TMP_VAR) { 6287 | EXT_CALL zend_jit_assign_tmp_to_typed_ref, r0 6288 } else if (val_type == IS_VAR) { 6289 | EXT_CALL zend_jit_assign_var_to_typed_ref, r0 6290 } else if (val_type == IS_CV) { 6291 | EXT_CALL zend_jit_assign_cv_to_typed_ref, r0 6292 } else { 6293 ZEND_UNREACHABLE(); 6294 } 6295 } else { 6296 |.if X64 6297 | LOAD_ZVAL_ADDR CARG3, res_addr 6298 |.else 6299 | sub r4, 12 6300 | PUSH_ZVAL_ADDR res_addr, r0 6301 |.endif 6302 if (val_type == IS_CONST) { 6303 | EXT_CALL zend_jit_assign_const_to_typed_ref2, r0 6304 } else if (val_type == IS_TMP_VAR) { 6305 | EXT_CALL zend_jit_assign_tmp_to_typed_ref2, r0 6306 } else if (val_type == IS_VAR) { 6307 | EXT_CALL zend_jit_assign_var_to_typed_ref2, r0 6308 } else if (val_type == IS_CV) { 6309 | EXT_CALL zend_jit_assign_cv_to_typed_ref2, r0 6310 } else { 6311 ZEND_UNREACHABLE(); 6312 } 6313 |.if not(X64) 6314 | add r4, 12 6315 |.endif 6316 } 6317 if (check_exception) { 6318 | // if (UNEXPECTED(EG(exception) != NULL)) { 6319 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 6320 | je >8 // END OF zend_jit_assign_to_variable() 6321 | jmp ->exception_handler 6322 } else { 6323 | jmp >8 6324 } 6325 |.code 6326 6327 return 1; 6328} 6329 6330static int zend_jit_assign_to_variable_call(dasm_State **Dst, 6331 const zend_op *opline, 6332 zend_jit_addr __var_use_addr, 6333 zend_jit_addr var_addr, 6334 uint32_t __var_info, 6335 uint32_t __var_def_info, 6336 uint8_t val_type, 6337 zend_jit_addr val_addr, 6338 uint32_t val_info, 6339 zend_jit_addr __res_addr, 6340 bool __check_exception) 6341{ 6342 if (val_info & MAY_BE_UNDEF) { 6343 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 6344 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6345 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6346 6347 if (!exit_addr) { 6348 return 0; 6349 } 6350 6351 | IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr 6352 } else { 6353 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 6354 |.cold_code 6355 |1: 6356 ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP); 6357 if (Z_REG(var_addr) != ZREG_FP) { 6358 | mov aword T1, Ra(Z_REG(var_addr)) // save 6359 } 6360 | SET_EX_OPLINE opline, r0 6361 | mov FCARG1d, Z_OFFSET(val_addr) 6362 | EXT_CALL zend_jit_undefined_op_helper, r0 6363 if (Z_REG(var_addr) != ZREG_FP) { 6364 | mov Ra(Z_REG(var_addr)), aword T1 // restore 6365 } 6366 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 6367 | LOAD_ZVAL_ADDR FCARG1a, var_addr 6368 } 6369 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6370 | call ->assign_const 6371 | jmp >9 6372 |.code 6373 } 6374 } 6375 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 6376 | LOAD_ZVAL_ADDR FCARG1a, var_addr 6377 } 6378 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 6379 | LOAD_ZVAL_ADDR FCARG2a, val_addr 6380 } 6381 if (opline) { 6382 | SET_EX_OPLINE opline, r0 6383 } 6384 if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 6385 | call ->assign_tmp 6386 } else if (val_type == IS_CONST) { 6387 | call ->assign_const 6388 } else if (val_type == IS_TMP_VAR) { 6389 | call ->assign_tmp 6390 } else if (val_type == IS_VAR) { 6391 if (!(val_info & MAY_BE_REF)) { 6392 | call ->assign_tmp 6393 } else { 6394 | call ->assign_var 6395 } 6396 } else if (val_type == IS_CV) { 6397 if (!(val_info & MAY_BE_REF)) { 6398 | call ->assign_cv_noref 6399 } else { 6400 | call ->assign_cv 6401 } 6402 if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 6403 |9: 6404 } 6405 } else { 6406 ZEND_UNREACHABLE(); 6407 } 6408 6409 return 1; 6410} 6411 6412static int zend_jit_assign_to_variable(dasm_State **Dst, 6413 const zend_op *opline, 6414 zend_jit_addr var_use_addr, 6415 zend_jit_addr var_addr, 6416 uint32_t var_info, 6417 uint32_t var_def_info, 6418 uint8_t val_type, 6419 zend_jit_addr val_addr, 6420 uint32_t val_info, 6421 zend_jit_addr res_addr, 6422 bool check_exception) 6423/* Labels: 1,2,3,4,5,8 */ 6424{ 6425 int done = 0; 6426 zend_reg ref_reg, tmp_reg; 6427 6428 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_R0) { 6429 ref_reg = ZREG_FCARG1; 6430 tmp_reg = ZREG_R0; 6431 } else { 6432 /* ASSIGN_DIM */ 6433 ref_reg = ZREG_R0; 6434 tmp_reg = ZREG_FCARG1; 6435 } 6436 6437 if (var_info & MAY_BE_REF) { 6438 if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { 6439 | LOAD_ZVAL_ADDR Ra(ref_reg), var_use_addr 6440 var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); 6441 } 6442 | // if (Z_ISREF_P(variable_ptr)) { 6443 | IF_NOT_Z_TYPE, Ra(ref_reg), IS_REFERENCE, >3 6444 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 6445 | GET_Z_PTR FCARG1a, Ra(ref_reg) 6446 if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) { 6447 return 0; 6448 } 6449 | lea Ra(ref_reg), [FCARG1a + offsetof(zend_reference, val)] 6450 |3: 6451 } 6452 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6453 if (RC_MAY_BE_1(var_info)) { 6454 int in_cold = 0; 6455 6456 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6457 | IF_ZVAL_REFCOUNTED var_use_addr, >1 6458 |.cold_code 6459 |1: 6460 in_cold = 1; 6461 } 6462 if (Z_REG(var_use_addr) == ZREG_FCARG1 || Z_REG(var_use_addr) == ZREG_R0) { 6463 bool keep_gc = 0; 6464 6465 | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr 6466 if (tmp_reg == ZREG_FCARG1) { 6467 if (Z_MODE(val_addr) == IS_REG) { 6468 keep_gc = 1; 6469 } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { 6470 keep_gc = 1; 6471 } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 6472 if (sizeof(void*) == 4) { 6473 keep_gc = 1; 6474 } else { 6475 zval *zv = Z_ZV(val_addr); 6476 6477 if (Z_TYPE_P(zv) == IS_DOUBLE) { 6478 if (Z_DVAL_P(zv) == 0 || IS_SIGNED_32BIT(zv)) { 6479 keep_gc = 1; 6480 } 6481 } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 6482 keep_gc = 1; 6483 } 6484 } 6485 } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { 6486 if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 6487 keep_gc = 1; 6488 } 6489 } 6490 } 6491 if (!keep_gc) { 6492 | mov aword T1, Ra(tmp_reg) // save 6493 } 6494 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0, 0)) { 6495 return 0; 6496 } 6497 if (!keep_gc) { 6498 | mov FCARG1a, aword T1 // restore 6499 } 6500 } else { 6501 | GET_ZVAL_PTR FCARG1a, var_use_addr 6502 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1, 0)) { 6503 return 0; 6504 } 6505 } 6506 | GC_DELREF FCARG1a 6507 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 6508 | jnz >4 6509 } else { 6510 | jnz >8 6511 } 6512 | ZVAL_DTOR_FUNC var_info, opline 6513 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { 6514 if (check_exception && !(val_info & MAY_BE_UNDEF)) { 6515 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 6516 | je >8 6517 | jmp ->exception_handler 6518 } else { 6519 | jmp >8 6520 } 6521 } 6522 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 6523 |4: 6524 | IF_GC_MAY_NOT_LEAK FCARG1a, >8 6525 | EXT_CALL gc_possible_root, r0 6526 if (in_cold) { 6527 | jmp >8 6528 } 6529 } 6530 if (check_exception && (val_info & MAY_BE_UNDEF)) { 6531 |8: 6532 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 6533 | je >8 6534 | jmp ->exception_handler 6535 } 6536 if (in_cold) { 6537 |.code 6538 } else { 6539 done = 1; 6540 } 6541 } else /* if (RC_MAY_BE_N(var_info)) */ { 6542 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6543 | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5 6544 } 6545 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { 6546 if (Z_REG(var_use_addr) != ZREG_FP) { 6547 | mov T1, Ra(Z_REG(var_use_addr)) // save 6548 } 6549 | GET_ZVAL_PTR FCARG1a, var_use_addr 6550 | GC_DELREF FCARG1a 6551 | IF_GC_MAY_NOT_LEAK FCARG1a, >5 6552 | EXT_CALL gc_possible_root, r0 6553 if (Z_REG(var_use_addr) != ZREG_FP) { 6554 | mov Ra(Z_REG(var_use_addr)), T1 // restore 6555 } 6556 } else { 6557 | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr 6558 | GC_DELREF Ra(tmp_reg) 6559 } 6560 |5: 6561 } 6562 } 6563 6564 if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0, check_exception)) { 6565 return 0; 6566 } 6567 6568 |8: 6569 6570 return 1; 6571} 6572 6573static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw) 6574{ 6575 zend_jit_addr op2_addr, op3_addr, res_addr; 6576 6577 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6578 op3_addr = OP1_DATA_ADDR(); 6579 if (opline->result_type == IS_UNUSED) { 6580 res_addr = 0; 6581 } else { 6582 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 6583 } 6584 6585 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { 6586 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6587 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6588 6589 if (!exit_addr) { 6590 return 0; 6591 } 6592 6593 | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr 6594 6595 val_info &= ~MAY_BE_UNDEF; 6596 } 6597 6598 if (op1_info & MAY_BE_REF) { 6599 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6600 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 6601 | GET_Z_PTR FCARG2a, FCARG1a 6602 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 6603 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 6604 | jmp >3 6605 |.cold_code 6606 |2: 6607 | SET_EX_OPLINE opline, r0 6608 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 6609 | test r0, r0 6610 | mov FCARG1a, r0 6611 | jne >1 6612 | jmp ->exception_handler_undef 6613 |.code 6614 |1: 6615 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6616 } 6617 6618 if (op1_info & MAY_BE_ARRAY) { 6619 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6620 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 6621 } 6622 |3: 6623 | SEPARATE_ARRAY op1_addr, op1_info, 1 6624 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6625 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6626 | CMP_ZVAL_TYPE op1_addr, IS_NULL 6627 | jg >7 6628 } 6629 | // ZVAL_ARR(container, zend_new_array(8)); 6630 if (Z_REG(op1_addr) != ZREG_FP) { 6631 | mov T1, Ra(Z_REG(op1_addr)) // save 6632 } 6633 | EXT_CALL _zend_new_array_0, r0 6634 if (Z_REG(op1_addr) != ZREG_FP) { 6635 | mov Ra(Z_REG(op1_addr)), T1 // restore 6636 } 6637 | SET_ZVAL_LVAL op1_addr, r0 6638 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6639 | mov FCARG1a, r0 6640 } 6641 6642 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6643 |6: 6644 if (opline->op2_type == IS_UNUSED) { 6645 uint32_t var_info = MAY_BE_NULL; 6646 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6647 6648 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6649 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6650 | EXT_CALL zend_hash_next_index_insert, r0 6651 | // if (UNEXPECTED(!var_ptr)) { 6652 | test r0, r0 6653 | jz >1 6654 |.cold_code 6655 |1: 6656 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6657 | CANNOT_ADD_ELEMENT opline 6658 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6659 | jmp >9 6660 |.code 6661 6662 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0, 0)) { 6663 return 0; 6664 } 6665 } else { 6666 uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6667 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6668 6669 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 6670 return 0; 6671 } 6672 6673 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6674 var_info |= MAY_BE_REF; 6675 } 6676 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6677 var_info |= MAY_BE_RC1; 6678 } 6679 6680 |8: 6681 | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); 6682 if (opline->op1_type == IS_VAR) { 6683 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6684 if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6685 return 0; 6686 } 6687 } else { 6688 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6689 return 0; 6690 } 6691 } 6692 } 6693 } 6694 6695 if (((op1_info & MAY_BE_ARRAY) && 6696 (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) || 6697 (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY)))) { 6698 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6699 |.cold_code 6700 |7: 6701 } 6702 6703 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) && 6704 (op1_info & MAY_BE_ARRAY)) { 6705 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6706 | CMP_ZVAL_TYPE op1_addr, IS_NULL 6707 | jg >2 6708 } 6709 | // ZVAL_ARR(container, zend_new_array(8)); 6710 if (Z_REG(op1_addr) != ZREG_FP) { 6711 | mov T1, Ra(Z_REG(op1_addr)) // save 6712 } 6713 | EXT_CALL _zend_new_array_0, r0 6714 if (Z_REG(op1_addr) != ZREG_FP) { 6715 | mov Ra(Z_REG(op1_addr)), T1 // restore 6716 } 6717 | SET_ZVAL_LVAL op1_addr, r0 6718 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6719 | mov FCARG1a, r0 6720 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); 6721 | jmp <6 6722 |2: 6723 } 6724 6725 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6726 | SET_EX_OPLINE opline, r0 6727 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6728 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6729 } 6730 if (opline->op2_type == IS_UNUSED) { 6731 | xor FCARG2a, FCARG2a 6732 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6733 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6734 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 6735 } else { 6736 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6737 } 6738 |.if not(X64) 6739 | sub r4, 8 6740 |.endif 6741 if (opline->result_type == IS_UNUSED) { 6742 |.if X64 6743 | xor CARG4, CARG4 6744 |.else 6745 | push 0 6746 |.endif 6747 } else { 6748 |.if X64 6749 | LOAD_ZVAL_ADDR CARG4, res_addr 6750 |.else 6751 | PUSH_ZVAL_ADDR res_addr, r0 6752 |.endif 6753 } 6754 |.if X64 6755 | LOAD_ZVAL_ADDR CARG3, op3_addr 6756 |.else 6757 | PUSH_ZVAL_ADDR op3_addr, r0 6758 |.endif 6759 | EXT_CALL zend_jit_assign_dim_helper, r0 6760 |.if not(X64) 6761 | add r4, 8 6762 |.endif 6763 6764#ifdef ZEND_JIT_USE_RC_INFERENCE 6765 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { 6766 /* ASSIGN_DIM may increase refcount of the value */ 6767 val_info |= MAY_BE_RCN; 6768 } 6769#endif 6770 6771 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, NULL 6772 } 6773 6774 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6775 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6776 | jmp >9 // END 6777 } 6778 |.code 6779 } 6780 } 6781 6782#ifdef ZEND_JIT_USE_RC_INFERENCE 6783 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 6784 /* ASSIGN_DIM may increase refcount of the key */ 6785 op2_info |= MAY_BE_RCN; 6786 } 6787#endif 6788 6789 |9: 6790 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 6791 6792 if (may_throw) { 6793 zend_jit_check_exception(Dst); 6794 } 6795 6796 return 1; 6797} 6798 6799static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) 6800{ 6801 zend_jit_addr op2_addr, op3_addr, var_addr; 6802 const void *not_found_exit_addr = NULL; 6803 uint32_t var_info = MAY_BE_NULL; 6804 6805 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6806 6807 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6808 op3_addr = OP1_DATA_ADDR(); 6809 6810 | SET_EX_OPLINE opline, r0 6811 if (op1_info & MAY_BE_REF) { 6812 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6813 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 6814 | GET_Z_PTR FCARG2a, FCARG1a 6815 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 6816 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 6817 | jmp >3 6818 |.cold_code 6819 |2: 6820 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 6821 | test r0, r0 6822 | mov FCARG1a, r0 6823 | jne >1 6824 | jmp ->exception_handler_undef 6825 |.code 6826 |1: 6827 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6828 } 6829 6830 if (op1_info & MAY_BE_ARRAY) { 6831 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6832 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 6833 } 6834 |3: 6835 | SEPARATE_ARRAY op1_addr, op1_info, 1 6836 } 6837 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6838 if (op1_info & MAY_BE_ARRAY) { 6839 |.cold_code 6840 |7: 6841 } 6842 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6843 | CMP_ZVAL_TYPE op1_addr, IS_NULL 6844 | jg >7 6845 } 6846 if (Z_REG(op1_addr) != ZREG_FP) { 6847 | mov T1, Ra(Z_REG(op1_addr)) // save 6848 } 6849 if (op1_info & MAY_BE_UNDEF) { 6850 if (op1_info & MAY_BE_NULL) { 6851 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 6852 } 6853 | mov FCARG1a, opline->op1.var 6854 | EXT_CALL zend_jit_undefined_op_helper, r0 6855 |1: 6856 } 6857 | // ZVAL_ARR(container, zend_new_array(8)); 6858 | EXT_CALL _zend_new_array_0, r0 6859 if (Z_REG(op1_addr) != ZREG_FP) { 6860 | mov Ra(Z_REG(op1_addr)), T1 // restore 6861 } 6862 | SET_ZVAL_LVAL op1_addr, r0 6863 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6864 | mov FCARG1a, r0 6865 if (op1_info & MAY_BE_ARRAY) { 6866 | jmp >1 6867 |.code 6868 |1: 6869 } 6870 } 6871 6872 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6873 uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); 6874 6875 |6: 6876 if (opline->op2_type == IS_UNUSED) { 6877 var_info = MAY_BE_NULL; 6878 6879 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6880 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6881 | EXT_CALL zend_hash_next_index_insert, r0 6882 | // if (UNEXPECTED(!var_ptr)) { 6883 | test r0, r0 6884 | jz >1 6885 |.cold_code 6886 |1: 6887 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6888 | CANNOT_ADD_ELEMENT opline 6889 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6890 | jmp >9 6891 |.code 6892 } else { 6893 var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6894 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6895 var_info |= MAY_BE_REF; 6896 } 6897 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6898 var_info |= MAY_BE_RC1; 6899 } 6900 6901 if (dim_type != IS_UNKNOWN 6902 && dim_type != IS_UNDEF 6903 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY 6904 && (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) 6905 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 6906 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 6907 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6908 if (!not_found_exit_addr) { 6909 return 0; 6910 } 6911 } 6912 6913 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, not_found_exit_addr, NULL)) { 6914 return 0; 6915 } 6916 6917 |8: 6918 if (not_found_exit_addr && dim_type != IS_REFERENCE) { 6919 | IF_NOT_Z_TYPE, r0, dim_type, ¬_found_exit_addr 6920 var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 6921 } 6922 if (var_info & MAY_BE_REF) { 6923 binary_op_type binary_op = get_binary_op(opline->extended_value); 6924 | IF_NOT_Z_TYPE, r0, IS_REFERENCE, >1 6925 | GET_Z_PTR FCARG1a, r0 6926 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6927 | jnz >2 6928 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] 6929 |.cold_code 6930 |2: 6931 | LOAD_ZVAL_ADDR FCARG2a, op3_addr 6932 |.if X64 6933 | LOAD_ADDR CARG3, binary_op 6934 |.else 6935 | sub r4, 12 6936 | PUSH_ADDR binary_op, r0 6937 |.endif 6938 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 6939 && (op1_data_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6940 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 6941 } else { 6942 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 6943 } 6944 |.if not(X64) 6945 | add r4, 12 6946 |.endif 6947 | jmp >9 6948 |.code 6949 |1: 6950 } 6951 } 6952 6953 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6954 switch (opline->extended_value) { 6955 case ZEND_ADD: 6956 case ZEND_SUB: 6957 case ZEND_MUL: 6958 case ZEND_DIV: 6959 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, 6960 1 /* may overflow */, may_throw)) { 6961 return 0; 6962 } 6963 break; 6964 case ZEND_BW_OR: 6965 case ZEND_BW_AND: 6966 case ZEND_BW_XOR: 6967 case ZEND_SL: 6968 case ZEND_SR: 6969 case ZEND_MOD: 6970 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6971 IS_CV, opline->op1, var_addr, var_info, NULL, 6972 (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 6973 op1_data_range, 6974 0, var_addr, var_def_info, var_info, may_throw)) { 6975 return 0; 6976 } 6977 break; 6978 case ZEND_CONCAT: 6979 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, 6980 may_throw)) { 6981 return 0; 6982 } 6983 break; 6984 default: 6985 ZEND_UNREACHABLE(); 6986 } 6987 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 6988 } 6989 6990 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6991 binary_op_type binary_op; 6992 6993 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6994 |.cold_code 6995 |7: 6996 } 6997 6998 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6999 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 7000 } 7001 if (opline->op2_type == IS_UNUSED) { 7002 | xor FCARG2a, FCARG2a 7003 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 7004 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 7005 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 7006 } else { 7007 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 7008 } 7009 binary_op = get_binary_op(opline->extended_value); 7010 |.if X64 7011 | LOAD_ZVAL_ADDR CARG3, op3_addr 7012 | LOAD_ADDR CARG4, binary_op 7013 |.else 7014 | sub r4, 8 7015 | PUSH_ADDR binary_op, r0 7016 | PUSH_ZVAL_ADDR op3_addr, r0 7017 |.endif 7018 | EXT_CALL zend_jit_assign_dim_op_helper, r0 7019 |.if not(X64) 7020 | add r4, 8 7021 |.endif 7022 7023 |9: 7024 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, NULL 7025 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL 7026 if (may_throw) { 7027 zend_jit_check_exception(Dst); 7028 } 7029 7030 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 7031 | jmp >9 // END 7032 |.code 7033 |9: 7034 } 7035 } else if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) 7036 && (!not_found_exit_addr || (var_info & MAY_BE_REF))) { 7037 |.cold_code 7038 |9: 7039 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, opline 7040 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 7041 if (may_throw) { 7042 zend_jit_check_exception(Dst); 7043 } 7044 | jmp >9 7045 |.code 7046 |9: 7047 } 7048 7049 return 1; 7050} 7051 7052static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) 7053{ 7054 zend_jit_addr op1_addr, op2_addr; 7055 7056 ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); 7057 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 7058 7059 op1_addr = OP1_ADDR(); 7060 op2_addr = OP2_ADDR(); 7061 7062 if (op1_info & MAY_BE_REF) { 7063 binary_op_type binary_op = get_binary_op(opline->extended_value); 7064 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 7065 | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >1 7066 | GET_Z_PTR FCARG1a, FCARG1a 7067 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 7068 | jnz >2 7069 | add FCARG1a, offsetof(zend_reference, val) 7070 |.cold_code 7071 |2: 7072 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 7073 |.if X64 7074 | LOAD_ADDR CARG3, binary_op 7075 |.else 7076 | sub r4, 12 7077 | PUSH_ADDR binary_op, r0 7078 |.endif 7079 | SET_EX_OPLINE opline, r0 7080 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 7081 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 7082 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 7083 } else { 7084 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 7085 } 7086 |.if not(X64) 7087 | add r4, 12 7088 |.endif 7089 zend_jit_check_exception(Dst); 7090 | jmp >9 7091 |.code 7092 |1: 7093 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 7094 } 7095 7096 int result; 7097 switch (opline->extended_value) { 7098 case ZEND_ADD: 7099 case ZEND_SUB: 7100 case ZEND_MUL: 7101 case ZEND_DIV: 7102 result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); 7103 break; 7104 case ZEND_BW_OR: 7105 case ZEND_BW_AND: 7106 case ZEND_BW_XOR: 7107 case ZEND_SL: 7108 case ZEND_SR: 7109 case ZEND_MOD: 7110 result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, 7111 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 7112 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 7113 opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); 7114 break; 7115 case ZEND_CONCAT: 7116 result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); 7117 break; 7118 default: 7119 ZEND_UNREACHABLE(); 7120 } 7121 |9: 7122 return result; 7123} 7124 7125static int zend_jit_cmp_long_long(dasm_State **Dst, 7126 const zend_op *opline, 7127 zend_ssa_range *op1_range, 7128 zend_jit_addr op1_addr, 7129 zend_ssa_range *op2_range, 7130 zend_jit_addr op2_addr, 7131 zend_jit_addr res_addr, 7132 uint8_t smart_branch_opcode, 7133 uint32_t target_label, 7134 uint32_t target_label2, 7135 const void *exit_addr, 7136 bool skip_comparison) 7137{ 7138 bool swap = 0; 7139 bool result; 7140 7141 if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { 7142 if (!smart_branch_opcode || 7143 smart_branch_opcode == ZEND_JMPZ_EX || 7144 smart_branch_opcode == ZEND_JMPNZ_EX) { 7145 | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE) 7146 } 7147 if (smart_branch_opcode && !exit_addr) { 7148 if (smart_branch_opcode == ZEND_JMPZ || 7149 smart_branch_opcode == ZEND_JMPZ_EX) { 7150 if (!result) { 7151 | jmp => target_label 7152 } 7153 } else if (smart_branch_opcode == ZEND_JMPNZ || 7154 smart_branch_opcode == ZEND_JMPNZ_EX) { 7155 if (result) { 7156 | jmp => target_label 7157 } 7158 } else { 7159 ZEND_UNREACHABLE(); 7160 } 7161 } 7162 return 1; 7163 } 7164 7165 if (skip_comparison) { 7166 if (Z_MODE(op1_addr) != IS_REG && 7167 (Z_MODE(op2_addr) == IS_REG || 7168 (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { 7169 swap = 1; 7170 } 7171 } else if (Z_MODE(op1_addr) == IS_REG) { 7172 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 7173 | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) 7174 } else { 7175 | LONG_OP cmp, Z_REG(op1_addr), op2_addr, r0 7176 } 7177 } else if (Z_MODE(op2_addr) == IS_REG) { 7178 if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { 7179 | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) 7180 } else { 7181 | LONG_OP cmp, Z_REG(op2_addr), op1_addr, r0 7182 } 7183 swap = 1; 7184 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 7185 | LONG_OP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) 7186 swap = 1; 7187 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { 7188 | LONG_OP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)) 7189 } else { 7190 | GET_ZVAL_LVAL ZREG_R0, op1_addr 7191 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 7192 | test r0, r0 7193 } else { 7194 | LONG_OP cmp, ZREG_R0, op2_addr, r0 7195 } 7196 } 7197 7198 if (smart_branch_opcode) { 7199 if (smart_branch_opcode == ZEND_JMPZ_EX || 7200 smart_branch_opcode == ZEND_JMPNZ_EX) { 7201 7202 switch (opline->opcode) { 7203 case ZEND_IS_EQUAL: 7204 case ZEND_IS_IDENTICAL: 7205 case ZEND_CASE: 7206 case ZEND_CASE_STRICT: 7207 | sete al 7208 break; 7209 case ZEND_IS_NOT_EQUAL: 7210 case ZEND_IS_NOT_IDENTICAL: 7211 | setne al 7212 break; 7213 case ZEND_IS_SMALLER: 7214 if (swap) { 7215 | setg al 7216 } else { 7217 | setl al 7218 } 7219 break; 7220 case ZEND_IS_SMALLER_OR_EQUAL: 7221 if (swap) { 7222 | setge al 7223 } else { 7224 | setle al 7225 } 7226 break; 7227 default: 7228 ZEND_UNREACHABLE(); 7229 } 7230 | movzx eax, al 7231 | lea eax, [eax + 2] 7232 | SET_ZVAL_TYPE_INFO res_addr, eax 7233 } 7234 if (smart_branch_opcode == ZEND_JMPZ || 7235 smart_branch_opcode == ZEND_JMPZ_EX) { 7236 switch (opline->opcode) { 7237 case ZEND_IS_EQUAL: 7238 case ZEND_IS_IDENTICAL: 7239 case ZEND_CASE: 7240 case ZEND_CASE_STRICT: 7241 if (exit_addr) { 7242 | jne &exit_addr 7243 } else { 7244 | jne => target_label 7245 } 7246 break; 7247 case ZEND_IS_NOT_EQUAL: 7248 if (exit_addr) { 7249 | je &exit_addr 7250 } else { 7251 | je => target_label 7252 } 7253 break; 7254 case ZEND_IS_NOT_IDENTICAL: 7255 if (exit_addr) { 7256 | jne &exit_addr 7257 } else { 7258 | je => target_label 7259 } 7260 break; 7261 case ZEND_IS_SMALLER: 7262 if (swap) { 7263 if (exit_addr) { 7264 | jle &exit_addr 7265 } else { 7266 | jle => target_label 7267 } 7268 } else { 7269 if (exit_addr) { 7270 | jge &exit_addr 7271 } else { 7272 | jge => target_label 7273 } 7274 } 7275 break; 7276 case ZEND_IS_SMALLER_OR_EQUAL: 7277 if (swap) { 7278 if (exit_addr) { 7279 | jl &exit_addr 7280 } else { 7281 | jl => target_label 7282 } 7283 } else { 7284 if (exit_addr) { 7285 | jg &exit_addr 7286 } else { 7287 | jg => target_label 7288 } 7289 } 7290 break; 7291 default: 7292 ZEND_UNREACHABLE(); 7293 } 7294 } else if (smart_branch_opcode == ZEND_JMPNZ || 7295 smart_branch_opcode == ZEND_JMPNZ_EX) { 7296 switch (opline->opcode) { 7297 case ZEND_IS_EQUAL: 7298 case ZEND_IS_IDENTICAL: 7299 case ZEND_CASE: 7300 case ZEND_CASE_STRICT: 7301 if (exit_addr) { 7302 | je &exit_addr 7303 } else { 7304 | je => target_label 7305 } 7306 break; 7307 case ZEND_IS_NOT_EQUAL: 7308 if (exit_addr) { 7309 | jne &exit_addr 7310 } else { 7311 | jne => target_label 7312 } 7313 break; 7314 case ZEND_IS_NOT_IDENTICAL: 7315 if (exit_addr) { 7316 | je &exit_addr 7317 } else { 7318 | jne => target_label 7319 } 7320 break; 7321 case ZEND_IS_SMALLER: 7322 if (swap) { 7323 if (exit_addr) { 7324 | jg &exit_addr 7325 } else { 7326 | jg => target_label 7327 } 7328 } else { 7329 if (exit_addr) { 7330 | jl &exit_addr 7331 } else { 7332 | jl => target_label 7333 } 7334 } 7335 break; 7336 case ZEND_IS_SMALLER_OR_EQUAL: 7337 if (swap) { 7338 if (exit_addr) { 7339 | jge &exit_addr 7340 } else { 7341 | jge => target_label 7342 } 7343 } else { 7344 if (exit_addr) { 7345 | jle &exit_addr 7346 } else { 7347 | jle => target_label 7348 } 7349 } 7350 break; 7351 default: 7352 ZEND_UNREACHABLE(); 7353 } 7354 } else { 7355 ZEND_UNREACHABLE(); 7356 } 7357 } else { 7358 switch (opline->opcode) { 7359 case ZEND_IS_EQUAL: 7360 case ZEND_IS_IDENTICAL: 7361 case ZEND_CASE: 7362 case ZEND_CASE_STRICT: 7363 | sete al 7364 break; 7365 case ZEND_IS_NOT_EQUAL: 7366 case ZEND_IS_NOT_IDENTICAL: 7367 | setne al 7368 break; 7369 case ZEND_IS_SMALLER: 7370 if (swap) { 7371 | setg al 7372 } else { 7373 | setl al 7374 } 7375 break; 7376 case ZEND_IS_SMALLER_OR_EQUAL: 7377 if (swap) { 7378 | setge al 7379 } else { 7380 | setle al 7381 } 7382 break; 7383 default: 7384 ZEND_UNREACHABLE(); 7385 } 7386 | movzx eax, al 7387 | add eax, 2 7388 | SET_ZVAL_TYPE_INFO res_addr, eax 7389 } 7390 7391 return 1; 7392} 7393 7394static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7395{ 7396 if (smart_branch_opcode) { 7397 if (smart_branch_opcode == ZEND_JMPZ) { 7398 switch (opline->opcode) { 7399 case ZEND_IS_EQUAL: 7400 case ZEND_IS_IDENTICAL: 7401 case ZEND_CASE: 7402 case ZEND_CASE_STRICT: 7403 if (exit_addr) { 7404 | jne &exit_addr 7405 | jp &exit_addr 7406 } else { 7407 | jne => target_label 7408 | jp => target_label 7409 } 7410 break; 7411 case ZEND_IS_NOT_EQUAL: 7412 | jp >1 7413 if (exit_addr) { 7414 | je &exit_addr 7415 } else { 7416 | je => target_label 7417 } 7418 |1: 7419 break; 7420 case ZEND_IS_NOT_IDENTICAL: 7421 if (exit_addr) { 7422 | jne &exit_addr 7423 | jp &exit_addr 7424 } else { 7425 | jp >1 7426 | je => target_label 7427 |1: 7428 } 7429 break; 7430 case ZEND_IS_SMALLER: 7431 if (swap) { 7432 if (exit_addr) { 7433 | jbe &exit_addr 7434 } else { 7435 | jbe => target_label 7436 } 7437 } else { 7438 if (exit_addr) { 7439 | jae &exit_addr 7440 | jp &exit_addr 7441 } else { 7442 | jae => target_label 7443 | jp => target_label 7444 } 7445 } 7446 break; 7447 case ZEND_IS_SMALLER_OR_EQUAL: 7448 if (swap) { 7449 if (exit_addr) { 7450 | jb &exit_addr 7451 } else { 7452 | jb => target_label 7453 } 7454 } else { 7455 if (exit_addr) { 7456 | ja &exit_addr 7457 | jp &exit_addr 7458 } else { 7459 | ja => target_label 7460 | jp => target_label 7461 } 7462 } 7463 break; 7464 default: 7465 ZEND_UNREACHABLE(); 7466 } 7467 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7468 switch (opline->opcode) { 7469 case ZEND_IS_EQUAL: 7470 case ZEND_IS_IDENTICAL: 7471 case ZEND_CASE: 7472 case ZEND_CASE_STRICT: 7473 | jp >1 7474 if (exit_addr) { 7475 | je &exit_addr 7476 } else { 7477 | je => target_label 7478 } 7479 |1: 7480 break; 7481 case ZEND_IS_NOT_EQUAL: 7482 if (exit_addr) { 7483 | jne &exit_addr 7484 | jp &exit_addr 7485 } else { 7486 | jne => target_label 7487 | jp => target_label 7488 } 7489 break; 7490 case ZEND_IS_NOT_IDENTICAL: 7491 if (exit_addr) { 7492 | jp >1 7493 | je &exit_addr 7494 |1: 7495 } else { 7496 | jne => target_label 7497 | jp => target_label 7498 } 7499 break; 7500 case ZEND_IS_SMALLER: 7501 if (swap) { 7502 if (exit_addr) { 7503 | ja &exit_addr 7504 } else { 7505 | ja => target_label 7506 } 7507 } else { 7508 | jp >1 7509 if (exit_addr) { 7510 | jb &exit_addr 7511 } else { 7512 | jb => target_label 7513 } 7514 |1: 7515 } 7516 break; 7517 case ZEND_IS_SMALLER_OR_EQUAL: 7518 if (swap) { 7519 if (exit_addr) { 7520 | jae &exit_addr 7521 } else { 7522 | jae => target_label 7523 } 7524 } else { 7525 | jp >1 7526 if (exit_addr) { 7527 | jbe &exit_addr 7528 } else { 7529 | jbe => target_label 7530 } 7531 |1: 7532 } 7533 break; 7534 default: 7535 ZEND_UNREACHABLE(); 7536 } 7537 } else if (smart_branch_opcode == ZEND_JMPZ_EX) { 7538 switch (opline->opcode) { 7539 case ZEND_IS_EQUAL: 7540 case ZEND_IS_IDENTICAL: 7541 case ZEND_CASE: 7542 case ZEND_CASE_STRICT: 7543 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7544 | jne => target_label 7545 | jp => target_label 7546 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7547 break; 7548 case ZEND_IS_NOT_EQUAL: 7549 case ZEND_IS_NOT_IDENTICAL: 7550 | jp >1 7551 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7552 | je => target_label 7553 |1: 7554 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7555 break; 7556 case ZEND_IS_SMALLER: 7557 if (swap) { 7558 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7559 | jbe => target_label 7560 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7561 } else { 7562 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7563 | jae => target_label 7564 | jp => target_label 7565 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7566 } 7567 break; 7568 case ZEND_IS_SMALLER_OR_EQUAL: 7569 if (swap) { 7570 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7571 | jb => target_label 7572 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7573 } else { 7574 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7575 | ja => target_label 7576 | jp => target_label 7577 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7578 } 7579 break; 7580 default: 7581 ZEND_UNREACHABLE(); 7582 } 7583 } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { 7584 switch (opline->opcode) { 7585 case ZEND_IS_EQUAL: 7586 case ZEND_IS_IDENTICAL: 7587 case ZEND_CASE: 7588 case ZEND_CASE_STRICT: 7589 | jp >1 7590 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7591 | je => target_label 7592 |1: 7593 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7594 break; 7595 case ZEND_IS_NOT_EQUAL: 7596 case ZEND_IS_NOT_IDENTICAL: 7597 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7598 | jne => target_label 7599 | jp => target_label 7600 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7601 break; 7602 case ZEND_IS_SMALLER: 7603 if (swap) { 7604 | seta al 7605 | movzx eax, al 7606 | lea eax, [eax + 2] 7607 | SET_ZVAL_TYPE_INFO res_addr, eax 7608 | ja => target_label 7609 } else { 7610 | jp >1 7611 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7612 | jb => target_label 7613 |1: 7614 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7615 } 7616 break; 7617 case ZEND_IS_SMALLER_OR_EQUAL: 7618 if (swap) { 7619 | setae al 7620 | movzx eax, al 7621 | lea eax, [eax + 2] 7622 | SET_ZVAL_TYPE_INFO res_addr, eax 7623 | jae => target_label 7624 } else { 7625 | jp >1 7626 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7627 | jbe => target_label 7628 |1: 7629 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7630 } 7631 break; 7632 default: 7633 ZEND_UNREACHABLE(); 7634 } 7635 } else { 7636 ZEND_UNREACHABLE(); 7637 } 7638 } else { 7639 switch (opline->opcode) { 7640 case ZEND_IS_EQUAL: 7641 case ZEND_IS_IDENTICAL: 7642 case ZEND_CASE: 7643 case ZEND_CASE_STRICT: 7644 | jp >1 7645 | mov eax, IS_TRUE 7646 | je >2 7647 |1: 7648 | mov eax, IS_FALSE 7649 |2: 7650 break; 7651 case ZEND_IS_NOT_EQUAL: 7652 case ZEND_IS_NOT_IDENTICAL: 7653 | jp >1 7654 | mov eax, IS_FALSE 7655 | je >2 7656 |1: 7657 | mov eax, IS_TRUE 7658 |2: 7659 break; 7660 case ZEND_IS_SMALLER: 7661 if (swap) { 7662 | seta al 7663 | movzx eax, al 7664 | add eax, 2 7665 } else { 7666 | jp >1 7667 | mov eax, IS_TRUE 7668 | jb >2 7669 |1: 7670 | mov eax, IS_FALSE 7671 |2: 7672 } 7673 break; 7674 case ZEND_IS_SMALLER_OR_EQUAL: 7675 if (swap) { 7676 | setae al 7677 | movzx eax, al 7678 | add eax, 2 7679 } else { 7680 | jp >1 7681 | mov eax, IS_TRUE 7682 | jbe >2 7683 |1: 7684 | mov eax, IS_FALSE 7685 |2: 7686 } 7687 break; 7688 default: 7689 ZEND_UNREACHABLE(); 7690 } 7691 | SET_ZVAL_TYPE_INFO res_addr, eax 7692 } 7693 7694 return 1; 7695} 7696 7697static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7698{ 7699 zend_reg tmp_reg = ZREG_XMM0; 7700 7701 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_R0 7702 | DOUBLE_CMP tmp_reg, op2_addr 7703 7704 return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); 7705} 7706 7707static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7708{ 7709 zend_reg tmp_reg = ZREG_XMM0; 7710 7711 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_R0 7712 | DOUBLE_CMP tmp_reg, op1_addr 7713 7714 return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); 7715} 7716 7717static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7718{ 7719 bool swap = 0; 7720 7721 if (Z_MODE(op1_addr) == IS_REG) { 7722 | DOUBLE_CMP Z_REG(op1_addr), op2_addr 7723 } else if (Z_MODE(op2_addr) == IS_REG) { 7724 | DOUBLE_CMP Z_REG(op2_addr), op1_addr 7725 swap = 1; 7726 } else { 7727 zend_reg tmp_reg = ZREG_XMM0; 7728 7729 | DOUBLE_GET_ZVAL_DVAL tmp_reg, op1_addr 7730 | DOUBLE_CMP tmp_reg, op2_addr 7731 } 7732 7733 return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); 7734} 7735 7736static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7737{ 7738 | test, eax, eax 7739 if (smart_branch_opcode) { 7740 if (smart_branch_opcode == ZEND_JMPZ_EX || 7741 smart_branch_opcode == ZEND_JMPNZ_EX) { 7742 switch (opline->opcode) { 7743 case ZEND_IS_EQUAL: 7744 case ZEND_CASE: 7745 | sete al 7746 break; 7747 case ZEND_IS_NOT_EQUAL: 7748 | setne al 7749 break; 7750 case ZEND_IS_SMALLER: 7751 | setl al 7752 break; 7753 case ZEND_IS_SMALLER_OR_EQUAL: 7754 | setle al 7755 break; 7756 default: 7757 ZEND_UNREACHABLE(); 7758 } 7759 | movzx eax, al 7760 | lea eax, [eax + 2] 7761 | SET_ZVAL_TYPE_INFO res_addr, eax 7762 } 7763 if (smart_branch_opcode == ZEND_JMPZ || 7764 smart_branch_opcode == ZEND_JMPZ_EX) { 7765 switch (opline->opcode) { 7766 case ZEND_IS_EQUAL: 7767 case ZEND_CASE: 7768 if (exit_addr) { 7769 | jne &exit_addr 7770 } else { 7771 | jne => target_label 7772 } 7773 break; 7774 case ZEND_IS_NOT_EQUAL: 7775 if (exit_addr) { 7776 | je &exit_addr 7777 } else { 7778 | je => target_label 7779 } 7780 break; 7781 case ZEND_IS_SMALLER: 7782 if (exit_addr) { 7783 | jge &exit_addr 7784 } else { 7785 | jge => target_label 7786 } 7787 break; 7788 case ZEND_IS_SMALLER_OR_EQUAL: 7789 if (exit_addr) { 7790 | jg &exit_addr 7791 } else { 7792 | jg => target_label 7793 } 7794 break; 7795 default: 7796 ZEND_UNREACHABLE(); 7797 } 7798 } else if (smart_branch_opcode == ZEND_JMPNZ || 7799 smart_branch_opcode == ZEND_JMPNZ_EX) { 7800 switch (opline->opcode) { 7801 case ZEND_IS_EQUAL: 7802 case ZEND_CASE: 7803 if (exit_addr) { 7804 | je &exit_addr 7805 } else { 7806 | je => target_label 7807 } 7808 break; 7809 case ZEND_IS_NOT_EQUAL: 7810 if (exit_addr) { 7811 | jne &exit_addr 7812 } else { 7813 | jne => target_label 7814 } 7815 break; 7816 case ZEND_IS_SMALLER: 7817 if (exit_addr) { 7818 | jl &exit_addr 7819 } else { 7820 | jl => target_label 7821 } 7822 break; 7823 case ZEND_IS_SMALLER_OR_EQUAL: 7824 if (exit_addr) { 7825 | jle &exit_addr 7826 } else { 7827 | jle => target_label 7828 } 7829 break; 7830 default: 7831 ZEND_UNREACHABLE(); 7832 } 7833 } else { 7834 ZEND_UNREACHABLE(); 7835 } 7836 } else { 7837 switch (opline->opcode) { 7838 case ZEND_IS_EQUAL: 7839 case ZEND_CASE: 7840 | sete al 7841 break; 7842 case ZEND_IS_NOT_EQUAL: 7843 | setne al 7844 break; 7845 case ZEND_IS_SMALLER: 7846 | setl al 7847 break; 7848 case ZEND_IS_SMALLER_OR_EQUAL: 7849 | setle al 7850 break; 7851 default: 7852 ZEND_UNREACHABLE(); 7853 } 7854 | movzx eax, al 7855 | add eax, 2 7856 | SET_ZVAL_TYPE_INFO res_addr, eax 7857 } 7858 7859 return 1; 7860} 7861 7862static int zend_jit_cmp(dasm_State **Dst, 7863 const zend_op *opline, 7864 uint32_t op1_info, 7865 zend_ssa_range *op1_range, 7866 zend_jit_addr op1_addr, 7867 uint32_t op2_info, 7868 zend_ssa_range *op2_range, 7869 zend_jit_addr op2_addr, 7870 zend_jit_addr res_addr, 7871 int may_throw, 7872 uint8_t smart_branch_opcode, 7873 uint32_t target_label, 7874 uint32_t target_label2, 7875 const void *exit_addr, 7876 bool skip_comparison) 7877{ 7878 bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); 7879 bool has_slow; 7880 7881 has_slow = 7882 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7883 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7884 ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7885 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); 7886 7887 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 7888 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 7889 if (op1_info & MAY_BE_DOUBLE) { 7890 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4 7891 } else { 7892 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 7893 } 7894 } 7895 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 7896 if (op2_info & MAY_BE_DOUBLE) { 7897 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 7898 |.cold_code 7899 |3: 7900 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7901 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7902 } 7903 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7904 return 0; 7905 } 7906 | jmp >6 7907 |.code 7908 } else { 7909 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7910 } 7911 } 7912 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 7913 return 0; 7914 } 7915 if (op1_info & MAY_BE_DOUBLE) { 7916 |.cold_code 7917 |4: 7918 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7919 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7920 } 7921 if (op2_info & MAY_BE_DOUBLE) { 7922 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7923 if (!same_ops) { 7924 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5 7925 } else { 7926 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7927 } 7928 } 7929 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7930 return 0; 7931 } 7932 | jmp >6 7933 } 7934 if (!same_ops) { 7935 |5: 7936 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7937 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7938 } 7939 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7940 return 0; 7941 } 7942 | jmp >6 7943 } 7944 |.code 7945 } 7946 } else if ((op1_info & MAY_BE_DOUBLE) && 7947 !(op1_info & MAY_BE_LONG) && 7948 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7949 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7950 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7951 } 7952 if (op2_info & MAY_BE_DOUBLE) { 7953 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7954 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7955 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3 7956 } else { 7957 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7958 } 7959 } 7960 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7961 return 0; 7962 } 7963 } 7964 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7965 if (op2_info & MAY_BE_DOUBLE) { 7966 |.cold_code 7967 } 7968 |3: 7969 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7970 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7971 } 7972 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7973 return 0; 7974 } 7975 if (op2_info & MAY_BE_DOUBLE) { 7976 | jmp >6 7977 |.code 7978 } 7979 } 7980 } else if ((op2_info & MAY_BE_DOUBLE) && 7981 !(op2_info & MAY_BE_LONG) && 7982 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7983 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7984 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7985 } 7986 if (op1_info & MAY_BE_DOUBLE) { 7987 if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7988 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7989 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3 7990 } else { 7991 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7992 } 7993 } 7994 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7995 return 0; 7996 } 7997 } 7998 if (!same_ops && (op1_info & MAY_BE_LONG)) { 7999 if (op1_info & MAY_BE_DOUBLE) { 8000 |.cold_code 8001 } 8002 |3: 8003 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 8004 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 8005 } 8006 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8007 return 0; 8008 } 8009 if (op1_info & MAY_BE_DOUBLE) { 8010 | jmp >6 8011 |.code 8012 } 8013 } 8014 } 8015 8016 if (has_slow || 8017 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 8018 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 8019 if (has_slow) { 8020 |.cold_code 8021 |9: 8022 } 8023 | SET_EX_OPLINE opline, r0 8024 if (Z_MODE(op1_addr) == IS_REG) { 8025 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8026 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 8027 return 0; 8028 } 8029 op1_addr = real_addr; 8030 } 8031 if (Z_MODE(op2_addr) == IS_REG) { 8032 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 8033 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 8034 return 0; 8035 } 8036 op2_addr = real_addr; 8037 } 8038 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8039 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { 8040 | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 8041 | mov FCARG1a, opline->op1.var 8042 | EXT_CALL zend_jit_undefined_op_helper, r0 8043 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8044 |1: 8045 } 8046 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { 8047 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 8048 | mov T1, FCARG1a // save 8049 | mov FCARG1a, opline->op2.var 8050 | EXT_CALL zend_jit_undefined_op_helper, r0 8051 | mov FCARG1a, T1 // restore 8052 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8053 | jmp >2 8054 |1: 8055 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8056 |2: 8057 } else { 8058 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8059 } 8060 | EXT_CALL zend_compare, r0 8061 if ((opline->opcode != ZEND_CASE && 8062 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8063 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8064 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8065 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8066 | mov dword T1, eax // save 8067 if (opline->opcode != ZEND_CASE) { 8068 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, NULL 8069 } 8070 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL 8071 if (may_throw) { 8072 zend_jit_check_exception_undef_result(Dst, opline); 8073 } 8074 | mov eax, dword T1 // restore 8075 } else if (may_throw) { 8076#if ZTS 8077 | mov dword T1, eax // save 8078#else 8079 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(exception)))) { 8080 | mov dword T1, eax // save 8081 } 8082#endif 8083 zend_jit_check_exception_undef_result(Dst, opline); 8084#if ZTS 8085 | mov eax, dword T1 // restore 8086#else 8087 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(exception)))) { 8088 | mov eax, dword T1 // restore 8089 } 8090#endif 8091 } 8092 if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8093 return 0; 8094 } 8095 if (has_slow) { 8096 | jmp >6 8097 |.code 8098 } 8099 } 8100 8101 |6: 8102 8103 return 1; 8104} 8105 8106static int zend_jit_identical(dasm_State **Dst, 8107 const zend_op *opline, 8108 uint32_t op1_info, 8109 zend_ssa_range *op1_range, 8110 zend_jit_addr op1_addr, 8111 uint32_t op2_info, 8112 zend_ssa_range *op2_range, 8113 zend_jit_addr op2_addr, 8114 zend_jit_addr res_addr, 8115 int may_throw, 8116 uint8_t smart_branch_opcode, 8117 uint32_t target_label, 8118 uint32_t target_label2, 8119 const void *exit_addr, 8120 bool skip_comparison) 8121{ 8122 uint32_t identical_label = (uint32_t)-1; 8123 uint32_t not_identical_label = (uint32_t)-1; 8124 8125 if (smart_branch_opcode && !exit_addr) { 8126 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8127 if (smart_branch_opcode == ZEND_JMPZ) { 8128 not_identical_label = target_label; 8129 } else if (smart_branch_opcode == ZEND_JMPNZ) { 8130 identical_label = target_label; 8131 } else { 8132 ZEND_UNREACHABLE(); 8133 } 8134 } else { 8135 if (smart_branch_opcode == ZEND_JMPZ) { 8136 identical_label = target_label; 8137 } else if (smart_branch_opcode == ZEND_JMPNZ) { 8138 not_identical_label = target_label; 8139 } else { 8140 ZEND_UNREACHABLE(); 8141 } 8142 } 8143 } 8144 8145 if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && 8146 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { 8147 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 8148 return 0; 8149 } 8150 return 1; 8151 } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && 8152 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { 8153 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8154 return 0; 8155 } 8156 return 1; 8157 } 8158 8159 if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { 8160 op1_info |= MAY_BE_NULL; 8161 op2_info |= MAY_BE_NULL; 8162 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8163 | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 8164 |.cold_code 8165 |1: 8166 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8167 | SET_EX_OPLINE opline, r0 8168 | mov FCARG1d, opline->op1.var 8169 | EXT_CALL zend_jit_undefined_op_helper, r0 8170 if (may_throw) { 8171 zend_jit_check_exception_undef_result(Dst, opline); 8172 } 8173 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8174 | jmp >1 8175 |.code 8176 |1: 8177 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8178 | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 8179 |.cold_code 8180 |1: 8181 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8182 | SET_EX_OPLINE opline, r0 8183 | mov aword T1, FCARG1a // save 8184 | mov FCARG1d, opline->op2.var 8185 | EXT_CALL zend_jit_undefined_op_helper, r0 8186 if (may_throw) { 8187 zend_jit_check_exception_undef_result(Dst, opline); 8188 } 8189 | mov FCARG1a, aword T1 // restore 8190 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8191 | jmp >1 8192 |.code 8193 |1: 8194 } else if (op1_info & MAY_BE_UNDEF) { 8195 op1_info |= MAY_BE_NULL; 8196 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8197 | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 8198 |.cold_code 8199 |1: 8200 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8201 | SET_EX_OPLINE opline, r0 8202 | mov FCARG1d, opline->op1.var 8203 | EXT_CALL zend_jit_undefined_op_helper, r0 8204 if (may_throw) { 8205 zend_jit_check_exception_undef_result(Dst, opline); 8206 } 8207 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8208 | jmp >1 8209 |.code 8210 |1: 8211 if (opline->op2_type != IS_CONST) { 8212 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8213 } 8214 } else if (op2_info & MAY_BE_UNDEF) { 8215 op2_info |= MAY_BE_NULL; 8216 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8217 | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 8218 |.cold_code 8219 |1: 8220 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8221 | SET_EX_OPLINE opline, r0 8222 | mov FCARG1d, opline->op2.var 8223 | EXT_CALL zend_jit_undefined_op_helper, r0 8224 if (may_throw) { 8225 zend_jit_check_exception_undef_result(Dst, opline); 8226 } 8227 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8228 | jmp >1 8229 |.code 8230 |1: 8231 if (opline->op1_type != IS_CONST) { 8232 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8233 } 8234 } else if ((op1_info & op2_info & MAY_BE_ANY) != 0) { 8235 if (opline->op1_type != IS_CONST) { 8236 if (Z_MODE(op1_addr) == IS_REG) { 8237 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8238 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 8239 return 0; 8240 } 8241 op1_addr = real_addr; 8242 } 8243 } 8244 if (opline->op2_type != IS_CONST) { 8245 if (Z_MODE(op2_addr) == IS_REG) { 8246 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 8247 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 8248 return 0; 8249 } 8250 op2_addr = real_addr; 8251 } 8252 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8253 } 8254 if (opline->op1_type != IS_CONST) { 8255 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8256 } 8257 } 8258 8259 if ((op1_info & op2_info & MAY_BE_ANY) == 0) { 8260 if ((opline->opcode != ZEND_CASE_STRICT && 8261 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8262 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8263 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8264 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8265 if (opline->opcode != ZEND_CASE_STRICT) { 8266 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8267 } 8268 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8269 } 8270 if (smart_branch_opcode) { 8271 if (may_throw) { 8272 zend_jit_check_exception_undef_result(Dst, opline); 8273 } 8274 if (exit_addr) { 8275 if (smart_branch_opcode == ZEND_JMPZ) { 8276 | jmp &exit_addr 8277 } 8278 } else if (not_identical_label != (uint32_t)-1) { 8279 | jmp =>not_identical_label 8280 } 8281 } else { 8282 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) 8283 if (may_throw) { 8284 zend_jit_check_exception(Dst); 8285 } 8286 } 8287 return 1; 8288 } 8289 8290 if (opline->op1_type & (IS_CV|IS_VAR)) { 8291 | ZVAL_DEREF FCARG1a, op1_info 8292 } 8293 if (opline->op2_type & (IS_CV|IS_VAR)) { 8294 | ZVAL_DEREF FCARG2a, op2_info 8295 } 8296 8297 if (has_concrete_type(op1_info) 8298 && has_concrete_type(op2_info) 8299 && concrete_type(op1_info) == concrete_type(op2_info) 8300 && concrete_type(op1_info) <= IS_TRUE) { 8301 if (smart_branch_opcode) { 8302 if (exit_addr) { 8303 if (smart_branch_opcode == ZEND_JMPNZ) { 8304 | jmp &exit_addr 8305 } 8306 } else if (identical_label != (uint32_t)-1) { 8307 | jmp =>identical_label 8308 } 8309 } else { 8310 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) 8311 } 8312 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { 8313 if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { 8314 if (smart_branch_opcode) { 8315 if (exit_addr) { 8316 if (smart_branch_opcode == ZEND_JMPNZ) { 8317 | jmp &exit_addr 8318 } 8319 } else if (identical_label != (uint32_t)-1) { 8320 | jmp =>identical_label 8321 } 8322 } else { 8323 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) 8324 } 8325 } else { 8326 if (smart_branch_opcode) { 8327 if (exit_addr) { 8328 if (smart_branch_opcode == ZEND_JMPZ) { 8329 | jmp &exit_addr 8330 } 8331 } else if (not_identical_label != (uint32_t)-1) { 8332 | jmp =>not_identical_label 8333 } 8334 } else { 8335 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) 8336 } 8337 } 8338 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { 8339 zval *val = Z_ZV(op1_addr); 8340 8341 | cmp byte [FCARG2a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) 8342 if (smart_branch_opcode) { 8343 if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { 8344 | jne >8 8345 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8346 if (may_throw) { 8347 zend_jit_check_exception_undef_result(Dst, opline); 8348 } 8349 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8350 | jmp &exit_addr 8351 } else if (identical_label != (uint32_t)-1) { 8352 | jmp =>identical_label 8353 } else { 8354 | jmp >9 8355 } 8356 |8: 8357 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8358 | je &exit_addr 8359 } else if (identical_label != (uint32_t)-1) { 8360 | je =>identical_label 8361 } else { 8362 | je >9 8363 } 8364 } else { 8365 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8366 | sete al 8367 } else { 8368 | setne al 8369 } 8370 | movzx eax, al 8371 | lea eax, [eax + 2] 8372 | SET_ZVAL_TYPE_INFO res_addr, eax 8373 } 8374 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8375 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 8376 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8377 if (may_throw) { 8378 zend_jit_check_exception_undef_result(Dst, opline); 8379 } 8380 } 8381 if (exit_addr) { 8382 if (smart_branch_opcode == ZEND_JMPZ) { 8383 | jmp &exit_addr 8384 } 8385 } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { 8386 | jmp =>not_identical_label 8387 } 8388 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { 8389 zval *val = Z_ZV(op2_addr); 8390 8391 | cmp byte [FCARG1a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) 8392 if (smart_branch_opcode) { 8393 if (opline->opcode != ZEND_CASE_STRICT 8394 && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { 8395 | jne >8 8396 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8397 if (may_throw) { 8398 zend_jit_check_exception_undef_result(Dst, opline); 8399 } 8400 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8401 | jmp &exit_addr 8402 } else if (identical_label != (uint32_t)-1) { 8403 | jmp =>identical_label 8404 } else { 8405 | jmp >9 8406 } 8407 |8: 8408 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8409 | je &exit_addr 8410 } else if (identical_label != (uint32_t)-1) { 8411 | je =>identical_label 8412 } else { 8413 | je >9 8414 } 8415 } else { 8416 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8417 | sete al 8418 } else { 8419 | setne al 8420 } 8421 | movzx eax, al 8422 | lea eax, [eax + 2] 8423 | SET_ZVAL_TYPE_INFO res_addr, eax 8424 } 8425 if (opline->opcode != ZEND_CASE_STRICT 8426 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8427 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 8428 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8429 if (may_throw) { 8430 zend_jit_check_exception_undef_result(Dst, opline); 8431 } 8432 } 8433 if (smart_branch_opcode) { 8434 if (exit_addr) { 8435 if (smart_branch_opcode == ZEND_JMPZ) { 8436 | jmp &exit_addr 8437 } 8438 } else if (not_identical_label != (uint32_t)-1) { 8439 | jmp =>not_identical_label 8440 } 8441 } 8442 } else { 8443 if (opline->op1_type == IS_CONST) { 8444 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8445 } 8446 if (opline->op2_type == IS_CONST) { 8447 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8448 } 8449 | EXT_CALL zend_is_identical, r0 8450 if ((opline->opcode != ZEND_CASE_STRICT && 8451 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8452 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8453 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8454 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8455 | mov aword T1, r0 // save 8456 if (opline->opcode != ZEND_CASE_STRICT) { 8457 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8458 } 8459 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8460 if (may_throw) { 8461 zend_jit_check_exception_undef_result(Dst, opline); 8462 } 8463 | mov r0, aword T1 // restore 8464 } 8465 if (smart_branch_opcode) { 8466 | test al, al 8467 if (exit_addr) { 8468 if (smart_branch_opcode == ZEND_JMPNZ) { 8469 | jnz &exit_addr 8470 } else { 8471 | jz &exit_addr 8472 } 8473 } else if (not_identical_label != (uint32_t)-1) { 8474 | jz =>not_identical_label 8475 if (identical_label != (uint32_t)-1) { 8476 | jmp =>identical_label 8477 } 8478 } else if (identical_label != (uint32_t)-1) { 8479 | jnz =>identical_label 8480 } 8481 } else { 8482 | movzx eax, al 8483 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8484 | lea eax, [eax + 2] 8485 } else { 8486 | neg eax 8487 | lea eax, [eax + 3] 8488 } 8489 | SET_ZVAL_TYPE_INFO res_addr, eax 8490 } 8491 } 8492 8493 |9: 8494 if (may_throw) { 8495 zend_jit_check_exception(Dst); 8496 } 8497 return 1; 8498} 8499 8500static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, uint8_t branch_opcode, const void *exit_addr) 8501{ 8502 uint32_t true_label = -1; 8503 uint32_t false_label = -1; 8504 bool set_bool = 0; 8505 bool set_bool_not = 0; 8506 bool set_delayed = 0; 8507 bool jmp_done = 0; 8508 8509 if (branch_opcode == ZEND_BOOL) { 8510 set_bool = 1; 8511 } else if (branch_opcode == ZEND_BOOL_NOT) { 8512 set_bool = 1; 8513 set_bool_not = 1; 8514 } else if (branch_opcode == ZEND_JMPZ) { 8515 false_label = target_label; 8516 } else if (branch_opcode == ZEND_JMPNZ) { 8517 true_label = target_label; 8518 } else if (branch_opcode == ZEND_JMPZ_EX) { 8519 set_bool = 1; 8520 false_label = target_label; 8521 } else if (branch_opcode == ZEND_JMPNZ_EX) { 8522 set_bool = 1; 8523 true_label = target_label; 8524 } else { 8525 ZEND_UNREACHABLE(); 8526 } 8527 8528 if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { 8529 if (zend_is_true(Z_ZV(op1_addr))) { 8530 /* Always TRUE */ 8531 if (set_bool) { 8532 if (set_bool_not) { 8533 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8534 } else { 8535 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8536 } 8537 } 8538 if (true_label != (uint32_t)-1) { 8539 | jmp =>true_label; 8540 } 8541 } else { 8542 /* Always FALSE */ 8543 if (set_bool) { 8544 if (set_bool_not) { 8545 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8546 } else { 8547 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8548 } 8549 } 8550 if (false_label != (uint32_t)-1) { 8551 | jmp =>false_label; 8552 } 8553 } 8554 return 1; 8555 } 8556 8557 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) { 8558 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8559 | ZVAL_DEREF FCARG1a, op1_info 8560 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 8561 } 8562 8563 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { 8564 if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { 8565 /* Always TRUE */ 8566 if (set_bool) { 8567 if (set_bool_not) { 8568 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8569 } else { 8570 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8571 } 8572 } 8573 if (true_label != (uint32_t)-1) { 8574 | jmp =>true_label; 8575 } 8576 } else { 8577 if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { 8578 /* Always FALSE */ 8579 if (set_bool) { 8580 if (set_bool_not) { 8581 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8582 } else { 8583 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8584 } 8585 } 8586 } else { 8587 | CMP_ZVAL_TYPE op1_addr, IS_TRUE 8588 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 8589 if ((op1_info & MAY_BE_LONG) && 8590 !(op1_info & MAY_BE_UNDEF) && 8591 !set_bool) { 8592 if (exit_addr) { 8593 if (branch_opcode == ZEND_JMPNZ) { 8594 | jl >9 8595 } else { 8596 | jl &exit_addr 8597 } 8598 } else if (false_label != (uint32_t)-1) { 8599 | jl =>false_label 8600 } else { 8601 | jl >9 8602 } 8603 jmp_done = 1; 8604 } else { 8605 | jg >2 8606 } 8607 } 8608 if (!(op1_info & MAY_BE_TRUE)) { 8609 /* It's FALSE */ 8610 if (set_bool) { 8611 if (set_bool_not) { 8612 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8613 } else { 8614 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8615 } 8616 } 8617 } else { 8618 if (exit_addr) { 8619 if (set_bool) { 8620 | jne >1 8621 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8622 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8623 | jmp &exit_addr 8624 } else { 8625 | jmp >9 8626 } 8627 |1: 8628 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8629 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8630 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8631 | jne &exit_addr 8632 } 8633 } 8634 } else { 8635 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8636 | je &exit_addr 8637 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8638 | jne &exit_addr 8639 } else { 8640 | je >9 8641 } 8642 } 8643 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8644 if (set_bool) { 8645 | jne >1 8646 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8647 if (true_label != (uint32_t)-1) { 8648 | jmp =>true_label 8649 } else { 8650 | jmp >9 8651 } 8652 |1: 8653 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8654 } else { 8655 if (true_label != (uint32_t)-1) { 8656 | je =>true_label 8657 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8658 | jne =>false_label 8659 jmp_done = 1; 8660 } else { 8661 | je >9 8662 } 8663 } 8664 } else if (set_bool) { 8665 | sete al 8666 | movzx eax, al 8667 if (set_bool_not) { 8668 | neg eax 8669 | add eax, 3 8670 } else { 8671 | add eax, 2 8672 } 8673 if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { 8674 set_delayed = 1; 8675 } else { 8676 | SET_ZVAL_TYPE_INFO res_addr, eax 8677 } 8678 } 8679 } 8680 } 8681 8682 /* It's FALSE, but may be UNDEF */ 8683 if (op1_info & MAY_BE_UNDEF) { 8684 if (op1_info & MAY_BE_ANY) { 8685 if (set_delayed) { 8686 | CMP_ZVAL_TYPE op1_addr, IS_UNDEF 8687 | SET_ZVAL_TYPE_INFO res_addr, eax 8688 | jz >1 8689 } else { 8690 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 8691 } 8692 |.cold_code 8693 |1: 8694 } 8695 | mov FCARG1d, opline->op1.var 8696 | SET_EX_OPLINE opline, r0 8697 | EXT_CALL zend_jit_undefined_op_helper, r0 8698 8699 if (may_throw) { 8700 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 8701 return 0; 8702 } 8703 } 8704 8705 if (exit_addr) { 8706 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8707 | jmp &exit_addr 8708 } 8709 } else if (false_label != (uint32_t)-1) { 8710 | jmp =>false_label 8711 } 8712 if (op1_info & MAY_BE_ANY) { 8713 if (exit_addr) { 8714 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8715 | jmp >9 8716 } 8717 } else if (false_label == (uint32_t)-1) { 8718 | jmp >9 8719 } 8720 |.code 8721 } 8722 } 8723 8724 if (!jmp_done) { 8725 if (exit_addr) { 8726 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8727 if (op1_info & MAY_BE_LONG) { 8728 | jmp >9 8729 } 8730 } else if (op1_info & MAY_BE_LONG) { 8731 | jmp &exit_addr 8732 } 8733 } else if (false_label != (uint32_t)-1) { 8734 | jmp =>false_label 8735 } else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 8736 | jmp >9 8737 } 8738 } 8739 } 8740 } 8741 8742 if (op1_info & MAY_BE_LONG) { 8743 |2: 8744 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8745 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 8746 } 8747 if (Z_MODE(op1_addr) == IS_REG) { 8748 | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) 8749 } else { 8750 | LONG_OP_WITH_CONST, cmp, op1_addr, Z_L(0) 8751 } 8752 if (set_bool) { 8753 | setne al 8754 | movzx eax, al 8755 if (set_bool_not) { 8756 | neg eax 8757 | add eax, 3 8758 } else { 8759 | lea eax, [eax + 2] 8760 } 8761 | SET_ZVAL_TYPE_INFO res_addr, eax 8762 } 8763 if (exit_addr) { 8764 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8765 | jne &exit_addr 8766 } else { 8767 | je &exit_addr 8768 } 8769 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8770 if (true_label != (uint32_t)-1) { 8771 | jne =>true_label 8772 if (false_label != (uint32_t)-1) { 8773 | jmp =>false_label 8774 } 8775 } else { 8776 | je =>false_label 8777 } 8778 } 8779 } 8780 8781 if ((op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) == MAY_BE_DOUBLE) { 8782 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8783 |.cold_code 8784 } 8785 |2: 8786 if (CAN_USE_AVX()) { 8787 | vxorps xmm0, xmm0, xmm0 8788 } else { 8789 | xorps xmm0, xmm0 8790 } 8791 | DOUBLE_CMP ZREG_XMM0, op1_addr 8792 8793 if (set_bool) { 8794 if (exit_addr) { 8795 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8796 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8797 | jp &exit_addr 8798 | jne &exit_addr 8799 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8800 } else { 8801 | jp >1 8802 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8803 | je &exit_addr 8804 |1: 8805 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8806 } 8807 } else if (false_label != (uint32_t)-1) { // JMPZ_EX (p=>true, z=>false, false=>jmp) 8808 | jp >1 8809 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8810 | je => false_label 8811 |1: 8812 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8813 } else if (true_label != (uint32_t)-1) { // JMPNZ_EX (p=>true, z=>false, true=>jmp) 8814 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8815 | jp => true_label 8816 | jne => true_label 8817 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8818 } else if (set_bool_not) { // BOOL_NOT (p=>false, z=>true) 8819 | mov eax, IS_FALSE 8820 | jp >1 8821 | jne >1 8822 | mov eax, IS_TRUE 8823 |1: 8824 | SET_ZVAL_TYPE_INFO res_addr, eax 8825 } else { // BOOL (p=>true, z=>false) 8826 | mov eax, IS_TRUE 8827 | jp >1 8828 | jne >1 8829 | mov eax, IS_FALSE 8830 |1: 8831 | SET_ZVAL_TYPE_INFO res_addr, eax 8832 } 8833 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8834 | jmp >9 8835 |.code 8836 } 8837 } else { 8838 if (exit_addr) { 8839 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8840 | jp &exit_addr 8841 | jne &exit_addr 8842 |1: 8843 } else { 8844 | jp >1 8845 | je &exit_addr 8846 |1: 8847 } 8848 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8849 | jmp >9 8850 } 8851 } else { 8852 ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); 8853 if (false_label != (uint32_t)-1 ) { 8854 | jp >1 8855 | je => false_label 8856 |1: 8857 if (true_label != (uint32_t)-1) { 8858 | jmp =>true_label 8859 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8860 | jmp >9 8861 } 8862 } else { 8863 | jp => true_label 8864 | jne => true_label 8865 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8866 | jmp >9 8867 } 8868 } 8869 } 8870 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8871 |.code 8872 } 8873 } 8874 } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8875 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8876 |.cold_code 8877 |2: 8878 } 8879 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8880 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8881 } 8882 | SET_EX_OPLINE opline, r0 8883 | EXT_CALL zend_is_true, r0 8884 8885 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8886 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8887 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8888 8889 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8890 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3 8891 } 8892 | GET_ZVAL_PTR FCARG1a, op1_addr 8893 | GC_DELREF FCARG1a 8894 | jnz >3 8895 | mov aword T1, r0 // save 8896 | ZVAL_DTOR_FUNC op1_info, opline 8897 | mov r0, aword T1 // restore 8898 |3: 8899 } 8900 if (may_throw) { 8901 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r1 8902 | jne ->exception_handler_undef 8903 } 8904 8905 if (set_bool) { 8906 if (set_bool_not) { 8907 | neg eax 8908 | add eax, 3 8909 } else { 8910 | add eax, 2 8911 } 8912 | SET_ZVAL_TYPE_INFO res_addr, eax 8913 if (exit_addr) { 8914 | CMP_ZVAL_TYPE res_addr, IS_FALSE 8915 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8916 | jne &exit_addr 8917 } else { 8918 | je &exit_addr 8919 } 8920 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8921 | CMP_ZVAL_TYPE res_addr, IS_FALSE 8922 if (true_label != (uint32_t)-1) { 8923 | jne =>true_label 8924 if (false_label != (uint32_t)-1) { 8925 | jmp =>false_label 8926 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8927 | jmp >9 8928 } 8929 } else { 8930 | je =>false_label 8931 } 8932 } 8933 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8934 | jmp >9 8935 |.code 8936 } 8937 } else { 8938 | test r0, r0 8939 if (exit_addr) { 8940 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8941 | jne &exit_addr 8942 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8943 | jmp >9 8944 } 8945 } else { 8946 | je &exit_addr 8947 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8948 | jmp >9 8949 } 8950 } 8951 } else if (true_label != (uint32_t)-1) { 8952 | jne =>true_label 8953 if (false_label != (uint32_t)-1) { 8954 | jmp =>false_label 8955 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8956 | jmp >9 8957 } 8958 } else { 8959 | je =>false_label 8960 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8961 | jmp >9 8962 } 8963 } 8964 8965 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8966 |.code 8967 } 8968 } 8969 } 8970 8971 |9: 8972 8973 return 1; 8974} 8975 8976static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) 8977{ 8978 if (op1_addr != op1_def_addr) { 8979 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 8980 return 0; 8981 } 8982 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 8983 op1_addr = op1_def_addr; 8984 } 8985 } 8986 8987 if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0, 1)) { 8988 return 0; 8989 } 8990 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 8991 return 0; 8992 } 8993 if (op1_info & MAY_BE_UNDEF) { 8994 zend_jit_check_exception(Dst); 8995 } 8996 return 1; 8997} 8998 8999static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 9000{ 9001 ZEND_ASSERT(opline->op1_type == IS_CV); 9002 9003 if (op2_addr != op2_def_addr) { 9004 if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { 9005 return 0; 9006 } 9007 if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { 9008 op2_addr = op2_def_addr; 9009 } 9010 } 9011 9012 if (Z_MODE(op1_addr) != IS_REG 9013 && Z_MODE(op1_use_addr) == IS_REG 9014 && !Z_LOAD(op1_use_addr) 9015 && !Z_STORE(op1_use_addr)) { 9016 /* Force type update */ 9017 op1_info |= MAY_BE_UNDEF; 9018 } 9019 if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, 9020 may_throw)) { 9021 return 0; 9022 } 9023 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { 9024 return 0; 9025 } 9026 if (opline->result_type != IS_UNUSED) { 9027 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 9028 return 0; 9029 } 9030 } 9031 9032 return 1; 9033} 9034 9035/* copy of hidden zend_closure */ 9036typedef struct _zend_closure { 9037 zend_object std; 9038 zend_function func; 9039 zval this_ptr; 9040 zend_class_entry *called_scope; 9041 zif_handler orig_internal_handler; 9042} zend_closure; 9043 9044static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) 9045{ 9046 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9047 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9048 9049 if (!exit_addr) { 9050 return 0; 9051 } 9052 9053 | // Check Stack Overflow 9054 | MEM_LOAD_ZTS r1, aword, executor_globals, vm_stack_end, r0 9055 | MEM_LOAD_OP_ZTS sub, r1, aword, executor_globals, vm_stack_top, r0 9056 | cmp r1, used_stack 9057 | jb &exit_addr 9058 9059 return 1; 9060} 9061 9062static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack) 9063{ 9064 uint32_t used_stack; 9065 bool stack_check = 1; 9066 9067 if (func) { 9068 used_stack = zend_vm_calc_used_stack(opline->extended_value, func); 9069 if ((int)used_stack <= checked_stack) { 9070 stack_check = 0; 9071 } 9072 } else { 9073 used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value + ZEND_OBSERVER_ENABLED) * sizeof(zval); 9074 9075 | // if (EXPECTED(ZEND_USER_CODE(func->type))) { 9076 if (!is_closure) { 9077 | test byte [r0 + offsetof(zend_function, type)], 1 9078 | mov FCARG1a, used_stack 9079 | jnz >1 9080 } else { 9081 | mov FCARG1a, used_stack 9082 } 9083 | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); 9084 | mov edx, opline->extended_value 9085 if (!is_closure) { 9086 | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)] 9087 | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)] 9088 | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)] 9089 | sub edx, dword [r0 + offsetof(zend_function, op_array.T)] 9090 } else { 9091 | cmp edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] 9092 | cmova edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] 9093 | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.last_var)] 9094 | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.T)] 9095 } 9096 | shl edx, 4 9097 |.if X64 9098 | movsxd r2, edx 9099 |.endif 9100 | sub FCARG1a, r2 9101 |1: 9102 } 9103 9104 zend_jit_start_reuse_ip(); 9105 9106 | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { 9107 | MEM_LOAD_ZTS RX, aword, executor_globals, vm_stack_top, RX 9108 9109 if (stack_check) { 9110 | // Check Stack Overflow 9111 | MEM_LOAD_ZTS r2, aword, executor_globals, vm_stack_end, r2 9112 | sub r2, RX 9113 if (func) { 9114 | cmp r2, used_stack 9115 } else { 9116 | cmp r2, FCARG1a 9117 } 9118 9119 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9120 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9121 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9122 9123 if (!exit_addr) { 9124 return 0; 9125 } 9126 9127 | jb &exit_addr 9128 } else { 9129 | jb >1 9130 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); 9131 |.cold_code 9132 |1: 9133 if (func) { 9134 | mov FCARG1d, used_stack 9135 } 9136#ifdef _WIN32 9137 if (0) { 9138#else 9139 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 9140#endif 9141 | SET_EX_OPLINE opline, r0 9142 | EXT_CALL zend_jit_int_extend_stack_helper, r0 9143 } else { 9144 if (!is_closure) { 9145 if (func 9146 && op_array == &func->op_array 9147 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9148 && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { 9149 | LOAD_ADDR FCARG2a, func 9150 } else { 9151 | mov FCARG2a, r0 9152 } 9153 } else { 9154 | lea FCARG2a, aword [r0 + offsetof(zend_closure, func)] 9155 } 9156 | SET_EX_OPLINE opline, r0 9157 | EXT_CALL zend_jit_extend_stack_helper, r0 9158 } 9159 | mov RX, r0 9160 | jmp >1 9161 |.code 9162 } 9163 } 9164 9165 if (func) { 9166 | MEM_UPDATE_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2 9167 } else { 9168 | MEM_UPDATE_ZTS add, aword, executor_globals, vm_stack_top, FCARG1a, r2 9169 } 9170 | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); 9171 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { 9172 | // ZEND_SET_CALL_INFO(call, 0, call_info); 9173 | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) 9174 } 9175#ifdef _WIN32 9176 if (0) { 9177#else 9178 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 9179#endif 9180 | // call->func = func; 9181 |1: 9182 | ADDR_STORE aword EX:RX->func, func, r1 9183 } else { 9184 if (!is_closure) { 9185 | // call->func = func; 9186 if (func 9187 && op_array == &func->op_array 9188 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9189 && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { 9190 | ADDR_STORE aword EX:RX->func, func, r1 9191 } else { 9192 | mov aword EX:RX->func, r0 9193 } 9194 } else { 9195 | // call->func = &closure->func; 9196 | lea r1, aword [r0 + offsetof(zend_closure, func)] 9197 | mov aword EX:RX->func, r1 9198 } 9199 |1: 9200 } 9201 if (opline->opcode == ZEND_INIT_METHOD_CALL) { 9202 | // Z_PTR(call->This) = obj; 9203 | mov r1, aword T1 9204 | mov aword EX:RX->This.value.ptr, r1 9205 if (opline->op1_type == IS_UNUSED || delayed_fetch_this) { 9206 | // call->call_info |= ZEND_CALL_HAS_THIS; 9207 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9208 | mov dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS 9209 } else { 9210 | or dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS 9211 } 9212 } else { 9213 if (opline->op1_type == IS_CV) { 9214 | // GC_ADDREF(obj); 9215 | add dword [r1], 1 9216 } 9217 | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; 9218 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9219 | mov dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 9220 } else { 9221 | or dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 9222 } 9223 } 9224 } else if (!is_closure) { 9225 | // Z_CE(call->This) = called_scope; 9226 | mov aword EX:RX->This.value.ptr, 0 9227 } else { 9228 if (opline->op2_type == IS_CV) { 9229 | // GC_ADDREF(closure); 9230 | add dword [r0], 1 9231 } 9232 | // object_or_called_scope = closure->called_scope; 9233 | mov r1, aword [r0 + offsetof(zend_closure, called_scope)] 9234 | mov aword EX:RX->This.value.ptr, r1 9235 | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | 9236 | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); 9237 | mov edx, dword [r0 + offsetof(zend_closure, func.common.fn_flags)] 9238 | and edx, ZEND_ACC_FAKE_CLOSURE 9239 | or edx, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE) 9240 | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { 9241 | cmp byte [r0 + offsetof(zend_closure, this_ptr.u1.v.type)], IS_UNDEF 9242 | jz >1 9243 | // call_info |= ZEND_CALL_HAS_THIS; 9244 | or edx, ZEND_CALL_HAS_THIS 9245 | // object_or_called_scope = Z_OBJ(closure->this_ptr); 9246 | mov r1, aword [r0 + offsetof(zend_closure, this_ptr.value.ptr)] 9247 |1: 9248 | // ZEND_SET_CALL_INFO(call, 0, call_info); 9249 | or dword EX:RX->This.u1.type_info, edx 9250 | // Z_PTR(call->This) = object_or_called_scope; 9251 | mov aword EX:RX->This.value.ptr, r1 9252 | cmp aword [r0 + offsetof(zend_closure, func.op_array.run_time_cache__ptr)], 0 9253 | jnz >1 9254 | lea FCARG1a, aword [r0 + offsetof(zend_closure, func)] 9255 | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 9256 |1: 9257 } 9258 | // ZEND_CALL_NUM_ARGS(call) = num_args; 9259 | mov dword EX:RX->This.u2.num_args, opline->extended_value 9260 return 1; 9261} 9262 9263static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) 9264{ 9265 int32_t exit_point; 9266 const void *exit_addr; 9267 9268 if (func->type == ZEND_INTERNAL_FUNCTION) { 9269#ifdef ZEND_WIN32 9270 // TODO: ASLR may cause different addresses in different workers ??? 9271 return 0; 9272#endif 9273 } else if (func->type == ZEND_USER_FUNCTION) { 9274 if (!zend_accel_in_shm(func->op_array.opcodes)) { 9275 /* op_array and op_array->opcodes are not persistent. We can't link. */ 9276 return 0; 9277 } 9278 } else { 9279 ZEND_UNREACHABLE(); 9280 return 0; 9281 } 9282 9283 exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); 9284 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9285 if (!exit_addr) { 9286 return 0; 9287 } 9288 9289 | // call = EX(call); 9290 | mov r1, EX->call 9291 while (level > 0) { 9292 | mov r1, EX:r1->prev_execute_data 9293 level--; 9294 } 9295 9296 if (func->type == ZEND_USER_FUNCTION && 9297 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 9298 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 9299 !func->common.function_name)) { 9300 const zend_op *opcodes = func->op_array.opcodes; 9301 9302 | mov r1, aword EX:r1->func 9303 | .if X64 9304 || if (!IS_SIGNED_32BIT(opcodes)) { 9305 | mov64 r2, ((ptrdiff_t)opcodes) 9306 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], r2 9307 || } else { 9308 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes 9309 || } 9310 | .else 9311 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes 9312 | .endif 9313 | jne &exit_addr 9314 } else { 9315 | .if X64 9316 || if (!IS_SIGNED_32BIT(func)) { 9317 | mov64 r2, ((ptrdiff_t)func) 9318 | cmp aword EX:r1->func, r2 9319 || } else { 9320 | cmp aword EX:r1->func, func 9321 || } 9322 | .else 9323 | cmp aword EX:r1->func, func 9324 | .endif 9325 | jne &exit_addr 9326 } 9327 9328 return 1; 9329} 9330 9331static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack) 9332{ 9333 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9334 zend_call_info *call_info = NULL; 9335 zend_function *func = NULL; 9336 9337 if (delayed_call_chain) { 9338 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9339 return 0; 9340 } 9341 } 9342 9343 if (info) { 9344 call_info = info->callee_info; 9345 while (call_info && call_info->caller_init_opline != opline) { 9346 call_info = call_info->next_callee; 9347 } 9348 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9349 func = call_info->callee_func; 9350 } 9351 } 9352 9353 if (!func 9354 && trace 9355 && trace->op == ZEND_JIT_TRACE_INIT_CALL) { 9356#ifdef _WIN32 9357 /* ASLR */ 9358 if (trace->func->type != ZEND_INTERNAL_FUNCTION) { 9359 func = (zend_function*)trace->func; 9360 } 9361#else 9362 func = (zend_function*)trace->func; 9363#endif 9364 } 9365 9366#ifdef _WIN32 9367 if (0) { 9368#else 9369 if (opline->opcode == ZEND_INIT_FCALL 9370 && func 9371 && func->type == ZEND_INTERNAL_FUNCTION) { 9372#endif 9373 /* load constant address later */ 9374 } else if (func && op_array == &func->op_array) { 9375 /* recursive call */ 9376 if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) || 9377 (sizeof(void*) == 8 && !IS_SIGNED_32BIT(func))) { 9378 | mov r0, EX->func 9379 } 9380 } else { 9381 | // if (CACHED_PTR(opline->result.num)) 9382 | mov r2, EX->run_time_cache 9383 | mov r0, aword [r2 + opline->result.num] 9384 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9385 && func 9386 && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) 9387 && opline->opcode != ZEND_INIT_FCALL) { 9388 /* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */ 9389 | .if X64 9390 || if (!IS_SIGNED_32BIT(func)) { 9391 | mov64 r1, ((ptrdiff_t)func) 9392 | cmp r0, r1 9393 || } else { 9394 | cmp r0, func 9395 || } 9396 | .else 9397 | cmp r0, func 9398 | .endif 9399 | jnz >1 9400 |.cold_code 9401 |1: 9402 } else { 9403 | test r0, r0 9404 | jz >1 9405 } 9406 |.cold_code 9407 |1: 9408 if (opline->opcode == ZEND_INIT_FCALL 9409 && func 9410 && func->type == ZEND_USER_FUNCTION 9411 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { 9412 | LOAD_ADDR FCARG1a, func 9413 | mov aword [r2 + opline->result.num], FCARG1a 9414 | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 9415 | jmp >3 9416 } else { 9417 zval *zv = RT_CONSTANT(opline, opline->op2); 9418 9419 if (opline->opcode == ZEND_INIT_FCALL) { 9420 | LOAD_ADDR FCARG1a, Z_STR_P(zv); 9421 | lea FCARG2a, aword [r2 + opline->result.num] 9422 | EXT_CALL zend_jit_find_func_helper, r0 9423 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { 9424 | LOAD_ADDR FCARG1a, Z_STR_P(zv + 1); 9425 | lea FCARG2a, aword [r2 + opline->result.num] 9426 | EXT_CALL zend_jit_find_func_helper, r0 9427 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { 9428 | LOAD_ADDR FCARG1a, zv; 9429 | lea FCARG2a, aword [r2 + opline->result.num] 9430 | EXT_CALL zend_jit_find_ns_func_helper, r0 9431 } else { 9432 ZEND_UNREACHABLE(); 9433 } 9434 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9435 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 9436 func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0); 9437 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9438 9439 if (!exit_addr) { 9440 return 0; 9441 } 9442 if (!func || opline->opcode == ZEND_INIT_FCALL) { 9443 | test r0, r0 9444 | jnz >3 9445 } else if (func->type == ZEND_USER_FUNCTION 9446 && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { 9447 const zend_op *opcodes = func->op_array.opcodes; 9448 9449 | .if X64 9450 || if (!IS_SIGNED_32BIT(opcodes)) { 9451 | mov64 r1, ((ptrdiff_t)opcodes) 9452 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 9453 || } else { 9454 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9455 || } 9456 | .else 9457 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9458 | .endif 9459 | jz >3 9460 } else { 9461 | .if X64 9462 || if (!IS_SIGNED_32BIT(func)) { 9463 | mov64 r1, ((ptrdiff_t)func) 9464 | cmp r0, r1 9465 || } else { 9466 | cmp r0, func 9467 || } 9468 | .else 9469 | cmp r0, func 9470 | .endif 9471 | jz >3 9472 } 9473 | jmp &exit_addr 9474 } else { 9475 | test r0, r0 9476 | jnz >3 9477 | // SAVE_OPLINE(); 9478 | SET_EX_OPLINE opline, r0 9479 | jmp ->undefined_function 9480 } 9481 } 9482 |.code 9483 |3: 9484 } 9485 9486 if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, checked_stack)) { 9487 return 0; 9488 } 9489 9490 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9491 if (!zend_jit_save_call_chain(Dst, call_level)) { 9492 return 0; 9493 } 9494 } else { 9495 delayed_call_chain = 1; 9496 delayed_call_level = call_level; 9497 } 9498 9499 return 1; 9500} 9501 9502static int zend_jit_init_method_call(dasm_State **Dst, 9503 const zend_op *opline, 9504 uint32_t b, 9505 const zend_op_array *op_array, 9506 zend_ssa *ssa, 9507 const zend_ssa_op *ssa_op, 9508 int call_level, 9509 uint32_t op1_info, 9510 zend_jit_addr op1_addr, 9511 zend_class_entry *ce, 9512 bool ce_is_instanceof, 9513 bool on_this, 9514 bool delayed_fetch_this, 9515 zend_class_entry *trace_ce, 9516 zend_jit_trace_rec *trace, 9517 int checked_stack, 9518 bool polymorphic_side_trace) 9519{ 9520 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9521 zend_call_info *call_info = NULL; 9522 zend_function *func = NULL; 9523 zval *function_name; 9524 9525 ZEND_ASSERT(opline->op2_type == IS_CONST); 9526 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 9527 9528 function_name = RT_CONSTANT(opline, opline->op2); 9529 9530 if (info) { 9531 call_info = info->callee_info; 9532 while (call_info && call_info->caller_init_opline != opline) { 9533 call_info = call_info->next_callee; 9534 } 9535 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9536 func = call_info->callee_func; 9537 } 9538 } 9539 9540 if (polymorphic_side_trace) { 9541 /* function is passed in r0 from parent_trace */ 9542 } else { 9543 if (on_this) { 9544 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 9545 9546 | GET_ZVAL_PTR FCARG1a, this_addr 9547 } else { 9548 if (op1_info & MAY_BE_REF) { 9549 if (opline->op1_type == IS_CV) { 9550 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 9551 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9552 } 9553 | ZVAL_DEREF FCARG1a, op1_info 9554 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 9555 } else { 9556 /* Hack: Convert reference to regular value to simplify JIT code */ 9557 ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); 9558 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 9559 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9560 | EXT_CALL zend_jit_unref_helper, r0 9561 |1: 9562 } 9563 } 9564 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 9565 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9566 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9567 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9568 9569 if (!exit_addr) { 9570 return 0; 9571 } 9572 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 9573 } else { 9574 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 9575 |.cold_code 9576 |1: 9577 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 9578 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9579 } 9580 | SET_EX_OPLINE opline, r0 9581 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9582 | EXT_CALL zend_jit_invalid_method_call_tmp, r0 9583 } else { 9584 | EXT_CALL zend_jit_invalid_method_call, r0 9585 } 9586 | jmp ->exception_handler 9587 |.code 9588 } 9589 } 9590 | GET_ZVAL_PTR FCARG1a, op1_addr 9591 } 9592 9593 if (delayed_call_chain) { 9594 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9595 return 0; 9596 } 9597 } 9598 9599 | mov aword T1, FCARG1a // save 9600 9601 if (func) { 9602 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9603 | mov r0, EX->run_time_cache 9604 | mov r0, aword [r0 + opline->result.num + sizeof(void*)] 9605 | test r0, r0 9606 | jz >1 9607 } else { 9608 | // if (CACHED_PTR(opline->result.num) == obj->ce)) { 9609 | mov r0, EX->run_time_cache 9610 | mov r2, aword [r0 + opline->result.num] 9611 | cmp r2, [FCARG1a + offsetof(zend_object, ce)] 9612 | jnz >1 9613 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9614 | mov r0, aword [r0 + opline->result.num + sizeof(void*)] 9615 } 9616 9617 |.cold_code 9618 |1: 9619 | LOAD_ADDR FCARG2a, function_name 9620 |.if X64 9621 | lea CARG3, aword T1 9622 |.else 9623 | lea r0, aword T1 9624 | sub r4, 12 9625 | push r0 9626 |.endif 9627 | SET_EX_OPLINE opline, r0 9628 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9629 | EXT_CALL zend_jit_find_method_tmp_helper, r0 9630 } else { 9631 | EXT_CALL zend_jit_find_method_helper, r0 9632 } 9633 |.if not(X64) 9634 | add r4, 12 9635 |.endif 9636 | test r0, r0 9637 | jnz >2 9638 | jmp ->exception_handler 9639 |.code 9640 |2: 9641 } 9642 9643 if ((!func || zend_jit_may_be_modified(func, op_array)) 9644 && trace 9645 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9646 && trace->func 9647#ifdef _WIN32 9648 && trace->func->type != ZEND_INTERNAL_FUNCTION 9649#endif 9650 ) { 9651 int32_t exit_point; 9652 const void *exit_addr; 9653 9654 exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); 9655 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9656 if (!exit_addr) { 9657 return 0; 9658 } 9659 9660 func = (zend_function*)trace->func; 9661 9662 if (func->type == ZEND_USER_FUNCTION && 9663 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 9664 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 9665 !func->common.function_name)) { 9666 const zend_op *opcodes = func->op_array.opcodes; 9667 9668 | .if X64 9669 || if (!IS_SIGNED_32BIT(opcodes)) { 9670 | mov64 r1, ((ptrdiff_t)opcodes) 9671 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 9672 || } else { 9673 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9674 || } 9675 | .else 9676 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9677 | .endif 9678 | jne &exit_addr 9679 } else { 9680 | .if X64 9681 || if (!IS_SIGNED_32BIT(func)) { 9682 | mov64 r1, ((ptrdiff_t)func) 9683 | cmp r0, r1 9684 || } else { 9685 | cmp r0, func 9686 || } 9687 | .else 9688 | cmp r0, func 9689 | .endif 9690 | jne &exit_addr 9691 } 9692 } 9693 9694 if (!func) { 9695 | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { 9696 | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_STATIC 9697 | jnz >1 9698 |.cold_code 9699 |1: 9700 } 9701 9702 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { 9703 | mov FCARG1a, aword T1 // restore 9704 | mov FCARG2a, r0 9705 |.if X64 9706 | mov CARG3d, opline->extended_value 9707 |.else 9708 | sub r4, 12 9709 | push opline->extended_value 9710 |.endif 9711 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9712 | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, r0 9713 } else { 9714 | EXT_CALL zend_jit_push_static_metod_call_frame, r0 9715 } 9716 |.if not(X64) 9717 | add r4, 12 9718 |.endif 9719 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) { 9720 | test r0, r0 9721 | jz ->exception_handler 9722 } 9723 | mov RX, r0 9724 } 9725 9726 if (!func) { 9727 | jmp >9 9728 |.code 9729 } 9730 9731 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { 9732 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, delayed_fetch_this, checked_stack)) { 9733 return 0; 9734 } 9735 } 9736 9737 if (!func) { 9738 |9: 9739 } 9740 zend_jit_start_reuse_ip(); 9741 9742 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9743 if (!zend_jit_save_call_chain(Dst, call_level)) { 9744 return 0; 9745 } 9746 } else { 9747 delayed_call_chain = 1; 9748 delayed_call_level = call_level; 9749 } 9750 9751 return 1; 9752} 9753 9754static int zend_jit_init_closure_call(dasm_State **Dst, 9755 const zend_op *opline, 9756 uint32_t b, 9757 const zend_op_array *op_array, 9758 zend_ssa *ssa, 9759 const zend_ssa_op *ssa_op, 9760 int call_level, 9761 zend_jit_trace_rec *trace, 9762 int checked_stack) 9763{ 9764 zend_function *func = NULL; 9765 zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 9766 9767 | GET_ZVAL_PTR r0, op2_addr 9768 9769 if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure 9770 && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { 9771 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9772 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9773 9774 if (!exit_addr) { 9775 return 0; 9776 } 9777 9778 |.if X64 9779 || if (!IS_SIGNED_32BIT(zend_ce_closure)) { 9780 | mov64 FCARG1a, ((ptrdiff_t)zend_ce_closure) 9781 | cmp aword [r0 + offsetof(zend_object, ce)], FCARG1a 9782 || } else { 9783 | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure 9784 || } 9785 |.else 9786 | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure 9787 |.endif 9788 | jne &exit_addr 9789 if (ssa->var_info && ssa_op->op2_use >= 0) { 9790 ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; 9791 ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; 9792 ssa->var_info[ssa_op->op2_use].is_instanceof = 0; 9793 } 9794 } 9795 9796 if (trace 9797 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9798 && trace->func 9799 && trace->func->type == ZEND_USER_FUNCTION) { 9800 const zend_op *opcodes; 9801 int32_t exit_point; 9802 const void *exit_addr; 9803 9804 func = (zend_function*)trace->func; 9805 opcodes = func->op_array.opcodes; 9806 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); 9807 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9808 if (!exit_addr) { 9809 return 0; 9810 } 9811 9812 | .if X64 9813 || if (!IS_SIGNED_32BIT(opcodes)) { 9814 | mov64 FCARG1a, ((ptrdiff_t)opcodes) 9815 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], FCARG1a 9816 || } else { 9817 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes 9818 || } 9819 | .else 9820 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes 9821 | .endif 9822 | jne &exit_addr 9823 } 9824 9825 if (delayed_call_chain) { 9826 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9827 return 0; 9828 } 9829 } 9830 9831 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, checked_stack)) { 9832 return 0; 9833 } 9834 9835 if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9836 if (!zend_jit_save_call_chain(Dst, call_level)) { 9837 return 0; 9838 } 9839 } else { 9840 delayed_call_chain = 1; 9841 delayed_call_level = call_level; 9842 } 9843 9844 if (trace 9845 && trace->op == ZEND_JIT_TRACE_END 9846 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 9847 if (!zend_jit_set_valid_ip(Dst, opline + 1)) { 9848 return 0; 9849 } 9850 } 9851 9852 return 1; 9853} 9854 9855static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) 9856{ 9857 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9858 zend_call_info *call_info = NULL; 9859 const zend_function *func = NULL; 9860 uint32_t i; 9861 zend_jit_addr res_addr; 9862 uint32_t call_num_args = 0; 9863 bool unknown_num_args = 0; 9864 const void *exit_addr = NULL; 9865 const zend_op *prev_opline; 9866 9867 if (RETURN_VALUE_USED(opline)) { 9868 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 9869 } else { 9870 /* CPU stack allocated temporary zval */ 9871 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, TMP_ZVAL_OFFSET); 9872 } 9873 9874 prev_opline = opline - 1; 9875 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { 9876 prev_opline--; 9877 } 9878 if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || 9879 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 9880 unknown_num_args = 1; 9881 } 9882 9883 if (info) { 9884 call_info = info->callee_info; 9885 while (call_info && call_info->caller_call_opline != opline) { 9886 call_info = call_info->next_callee; 9887 } 9888 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9889 func = call_info->callee_func; 9890 } 9891 if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) 9892 && JIT_G(current_frame) 9893 && JIT_G(current_frame)->call 9894 && !JIT_G(current_frame)->call->func) { 9895 call_info = NULL; func = NULL; /* megamorphic call from trait */ 9896 } 9897 } 9898 if (!func) { 9899 /* resolve function at run time */ 9900 } else if (func->type == ZEND_USER_FUNCTION) { 9901 ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); 9902 call_num_args = call_info->num_args; 9903 } else if (func->type == ZEND_INTERNAL_FUNCTION) { 9904 ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); 9905 call_num_args = call_info->num_args; 9906 } else { 9907 ZEND_UNREACHABLE(); 9908 } 9909 9910 if (trace && !func) { 9911 if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { 9912 ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); 9913#ifndef ZEND_WIN32 9914 // TODO: ASLR may cause different addresses in different workers ??? 9915 func = trace->func; 9916 if (JIT_G(current_frame) && 9917 JIT_G(current_frame)->call && 9918 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9919 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9920 } else { 9921 unknown_num_args = 1; 9922 } 9923#endif 9924 } else if (trace->op == ZEND_JIT_TRACE_ENTER) { 9925 ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); 9926 if (zend_accel_in_shm(trace->func->op_array.opcodes)) { 9927 func = trace->func; 9928 if (JIT_G(current_frame) && 9929 JIT_G(current_frame)->call && 9930 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 9931 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 9932 } else { 9933 unknown_num_args = 1; 9934 } 9935 } 9936 } 9937 } 9938 9939 bool may_have_extra_named_params = 9940 opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && 9941 (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); 9942 9943 if (!reuse_ip) { 9944 zend_jit_start_reuse_ip(); 9945 | // call = EX(call); 9946 | mov RX, EX->call 9947 } 9948 zend_jit_stop_reuse_ip(); 9949 9950 | // fbc = call->func; 9951 | // mov r2, EX:RX->func ??? 9952 | // SAVE_OPLINE(); 9953 | SET_EX_OPLINE opline, r0 9954 9955 if (opline->opcode == ZEND_DO_FCALL) { 9956 if (!func) { 9957 if (trace) { 9958 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9959 9960 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9961 if (!exit_addr) { 9962 return 0; 9963 } 9964 | mov r0, EX:RX->func 9965 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 9966 | jnz &exit_addr 9967 } 9968 } 9969 } 9970 9971 if (!delayed_call_chain) { 9972 if (call_level == 1) { 9973 | mov aword EX->call, 0 9974 } else { 9975 | //EX(call) = call->prev_execute_data; 9976 | mov r0, EX:RX->prev_execute_data 9977 | mov EX->call, r0 9978 } 9979 } 9980 delayed_call_chain = 0; 9981 9982 | //call->prev_execute_data = execute_data; 9983 | mov EX:RX->prev_execute_data, EX 9984 9985 if (!func) { 9986 | mov r0, EX:RX->func 9987 } 9988 9989 if (opline->opcode == ZEND_DO_FCALL) { 9990 if (!func) { 9991 if (!trace) { 9992 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 9993 | jnz >1 9994 |.cold_code 9995 |1: 9996 if (!GCC_GLOBAL_REGS) { 9997 | mov FCARG1a, RX 9998 } 9999 | EXT_CALL zend_jit_deprecated_helper, r0 10000 | test al, al 10001 | mov r0, EX:RX->func // reload 10002 | jne >1 10003 | jmp ->exception_handler 10004 |.code 10005 |1: 10006 } 10007 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 10008 if (!GCC_GLOBAL_REGS) { 10009 | mov FCARG1a, RX 10010 } 10011 | EXT_CALL zend_jit_deprecated_helper, r0 10012 | test al, al 10013 | je ->exception_handler 10014 } 10015 } 10016 10017 if (!func 10018 && opline->opcode != ZEND_DO_UCALL 10019 && opline->opcode != ZEND_DO_ICALL) { 10020 | cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION 10021 | jne >8 10022 } 10023 10024 if ((!func || func->type == ZEND_USER_FUNCTION) 10025 && opline->opcode != ZEND_DO_ICALL) { 10026 | // EX(call) = NULL; 10027 | mov aword EX:RX->call, 0 10028 10029 if (RETURN_VALUE_USED(opline)) { 10030 | // EX(return_value) = EX_VAR(opline->result.var); 10031 | LOAD_ZVAL_ADDR r2, res_addr 10032 | mov aword EX:RX->return_value, r2 10033 } else { 10034 | // EX(return_value) = 0; 10035 | mov aword EX:RX->return_value, 0 10036 } 10037 10038 //EX_LOAD_RUN_TIME_CACHE(op_array); 10039 if (!func || func->op_array.cache_size) { 10040 if (func && op_array == &func->op_array) { 10041 /* recursive call */ 10042 if (trace || func->op_array.cache_size > sizeof(void*)) { 10043 | mov r2, EX->run_time_cache 10044 | mov EX:RX->run_time_cache, r2 10045 } 10046 } else { 10047 if (func 10048 && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE) 10049 && ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { 10050 | MEM_LOAD_ZTS r2, aword, compiler_globals, map_ptr_base, r1 10051 | mov r2, aword [r2 + (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache)] 10052 } else if ((func && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) || 10053 (JIT_G(current_frame) && 10054 JIT_G(current_frame)->call && 10055 TRACE_FRAME_IS_CLOSURE_CALL(JIT_G(current_frame)->call))) { 10056 /* Closures always use direct pointers */ 10057 | mov r0, EX:RX->func 10058 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 10059 } else { 10060 if (func) { 10061 | mov r0, EX:RX->func 10062 } 10063 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 10064 | test r2, 1 10065 | jz >1 10066 | MEM_LOAD_OP_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 10067 | mov r2, aword [r2] 10068 |1: 10069 } 10070 | mov EX:RX->run_time_cache, r2 10071 } 10072 } 10073 10074 | // EG(current_execute_data) = execute_data; 10075 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, RX, r1 10076 | mov FP, RX 10077 10078 | // opline = op_array->opcodes; 10079 if (func && !unknown_num_args) { 10080 10081 for (i = call_num_args; i < func->op_array.last_var; i++) { 10082 uint32_t n = EX_NUM_TO_VAR(i); 10083 | SET_Z_TYPE_INFO RX + n, IS_UNDEF 10084 } 10085 10086 if (call_num_args <= func->op_array.num_args) { 10087 if (!trace || (trace->op == ZEND_JIT_TRACE_END 10088 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10089 uint32_t num_args; 10090 10091 if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { 10092 if (trace) { 10093 num_args = 0; 10094 } else if (call_info) { 10095 num_args = skip_valid_arguments(op_array, ssa, call_info); 10096 } else { 10097 num_args = call_num_args; 10098 } 10099 } else { 10100 num_args = call_num_args; 10101 } 10102 if (zend_accel_in_shm(func->op_array.opcodes)) { 10103 | LOAD_IP_ADDR (func->op_array.opcodes + num_args) 10104 } else { 10105 | mov r0, EX->func 10106 if (GCC_GLOBAL_REGS) { 10107 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10108 if (num_args) { 10109 | add IP, (num_args * sizeof(zend_op)) 10110 } 10111 } else { 10112 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10113 if (num_args) { 10114 | add FCARG1a, (num_args * sizeof(zend_op)) 10115 } 10116 | mov aword EX->opline, FCARG1a 10117 } 10118 } 10119 10120 if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array 10121 && num_args >= op_array->required_num_args) { 10122 /* recursive call */ 10123 if (ZEND_OBSERVER_ENABLED) { 10124 | SAVE_IP 10125 | mov FCARG1a, FP 10126 | EXT_CALL zend_observer_fcall_begin, r0 10127 } 10128#ifdef CONTEXT_THREADED_JIT 10129 | call >1 10130 |.cold_code 10131 |1: 10132 | pop r0 10133 | jmp =>num_args 10134 |.code 10135#else 10136 | jmp =>num_args 10137#endif 10138 return 1; 10139 } 10140 } 10141 } else { 10142 if (!trace || (trace->op == ZEND_JIT_TRACE_END 10143 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10144 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 10145 | LOAD_IP_ADDR (func->op_array.opcodes) 10146 } else if (GCC_GLOBAL_REGS) { 10147 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10148 } else { 10149 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10150 | mov aword EX->opline, FCARG1a 10151 } 10152 } 10153 if (!GCC_GLOBAL_REGS) { 10154 | mov FCARG1a, FP 10155 } 10156 | EXT_CALL zend_jit_copy_extra_args_helper, r0 10157 } 10158 } else { 10159 | // opline = op_array->opcodes 10160 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 10161 | LOAD_IP_ADDR (func->op_array.opcodes) 10162 } else if (GCC_GLOBAL_REGS) { 10163 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10164 } else { 10165 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10166 | mov aword EX->opline, FCARG1a 10167 } 10168 if (func) { 10169 | // num_args = EX_NUM_ARGS(); 10170 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] 10171 | // if (UNEXPECTED(num_args > first_extra_arg)) 10172 | cmp ecx, (func->op_array.num_args) 10173 } else { 10174 | // first_extra_arg = op_array->num_args; 10175 | mov edx, dword [r0 + offsetof(zend_op_array, num_args)] 10176 | // num_args = EX_NUM_ARGS(); 10177 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] 10178 | // if (UNEXPECTED(num_args > first_extra_arg)) 10179 | cmp ecx, edx 10180 } 10181 | jg >1 10182 |.cold_code 10183 |1: 10184 if (!GCC_GLOBAL_REGS) { 10185 | mov FCARG1a, FP 10186 } 10187 | EXT_CALL zend_jit_copy_extra_args_helper, r0 10188 if (!func) { 10189 | mov r0, EX->func // reload 10190 } 10191 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload 10192 | jmp >1 10193 |.code 10194 if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { 10195 if (!func) { 10196 | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) 10197 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS 10198 | jnz >1 10199 } 10200 | // opline += num_args; 10201 |.if X64 10202 || ZEND_ASSERT(sizeof(zend_op) == 32); 10203 | mov edx, ecx 10204 | shl r2, 5 10205 |.else 10206 | imul r2, ecx, sizeof(zend_op) 10207 |.endif 10208 | ADD_IP r2 10209 } 10210 |1: 10211 | // if (EXPECTED((int)num_args < op_array->last_var)) { 10212 if (func) { 10213 | mov edx, (func->op_array.last_var) 10214 } else { 10215 | mov edx, dword [r0 + offsetof(zend_op_array, last_var)] 10216 } 10217 | sub edx, ecx 10218 | jle >3 //??? 10219 | // zval *var = EX_VAR_NUM(num_args); 10220// |.if X64 10221// | movsxd r1, ecx 10222// |.endif 10223 | shl r1, 4 10224 | lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))] 10225 |2: 10226 | SET_Z_TYPE_INFO r1, IS_UNDEF 10227 | sub edx, 1 10228 | lea r1, [r1 + 16] 10229 | jne <2 10230 |3: 10231 } 10232 10233 if (ZEND_OBSERVER_ENABLED) { 10234 if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10235 ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END); 10236 | SET_EX_OPLINE trace[1].opline, r0 10237 } else { 10238 | SAVE_IP 10239 } 10240 | mov FCARG1a, FP 10241 | EXT_CALL zend_observer_fcall_begin, r0 10242 } 10243 10244 if (trace) { 10245 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 10246 | jmp >9 10247 } 10248 } else { 10249#ifdef CONTEXT_THREADED_JIT 10250 | call ->context_threaded_call 10251 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 10252 | jmp >9 10253 } 10254 | call ->context_threaded_call 10255 if (!func) { 10256 | jmp >9 10257 } 10258#else 10259 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10260 | ADD_HYBRID_SPAD 10261 | JMP_IP 10262 } else if (GCC_GLOBAL_REGS) { 10263 | add r4, SPAD // stack alignment 10264 | JMP_IP 10265 } else { 10266 | mov FP, aword T2 // restore FP 10267 | mov RX, aword T3 // restore IP 10268 | add r4, NR_SPAD // stack alignment 10269 | mov r0, 1 // ZEND_VM_ENTER 10270 | ret 10271 } 10272 } 10273#endif 10274 } 10275 10276 if ((!func || func->type == ZEND_INTERNAL_FUNCTION) 10277 && (opline->opcode != ZEND_DO_UCALL)) { 10278 if (!func && (opline->opcode != ZEND_DO_ICALL)) { 10279 |8: 10280 } 10281 if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { 10282 if (!func) { 10283 if (trace) { 10284 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10285 10286 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10287 if (!exit_addr) { 10288 return 0; 10289 } 10290 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10291 | jnz &exit_addr 10292 } else { 10293 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10294 | jnz >1 10295 |.cold_code 10296 |1: 10297 if (!GCC_GLOBAL_REGS) { 10298 | mov FCARG1a, RX 10299 } 10300 | EXT_CALL zend_jit_deprecated_helper, r0 10301 | test al, al 10302 | mov r0, EX:RX->func // reload 10303 | jne >1 10304 | jmp ->exception_handler 10305 |.code 10306 |1: 10307 } 10308 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 10309 if (!GCC_GLOBAL_REGS) { 10310 | mov FCARG1a, RX 10311 } 10312 | EXT_CALL zend_jit_deprecated_helper, r0 10313 | test al, al 10314 | je ->exception_handler 10315 | mov r0, EX:RX->func // reload 10316 } 10317 } 10318 10319 | // EG(current_execute_data) = execute_data; 10320 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, RX, r1 10321 10322 if (ZEND_OBSERVER_ENABLED) { 10323 | mov FCARG1a, RX 10324 | EXT_CALL zend_observer_fcall_begin, r0 10325 | mov r0, EX:RX->func // reload 10326 } 10327 10328 | // ZVAL_NULL(EX_VAR(opline->result.var)); 10329 | LOAD_ZVAL_ADDR FCARG2a, res_addr 10330 | SET_Z_TYPE_INFO FCARG2a, IS_NULL 10331 10332 zend_jit_reset_last_valid_opline(); 10333 10334 | // (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret); 10335 if (zend_execute_internal) { 10336 |.if X64 10337 | // CARG2 and FCARG2a are identical 10338 | mov CARG1, RX 10339 |.else 10340 | mov aword A2, FCARG2a 10341 | mov aword A1, RX 10342 |.endif 10343 | EXT_CALL zend_execute_internal, r0 10344 } else { 10345 | mov FCARG1a, RX 10346 if (func) { 10347 | EXT_CALL func->internal_function.handler, r0 10348 } else { 10349 | call aword [r0 + offsetof(zend_internal_function, handler)] 10350 } 10351 } 10352 10353 if (ZEND_OBSERVER_ENABLED) { 10354 | LOAD_ZVAL_ADDR FCARG2a, res_addr 10355 | mov FCARG1a, RX 10356 | EXT_CALL zend_observer_fcall_end, r0 10357 } 10358 10359 | // EG(current_execute_data) = execute_data; 10360 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, FP, r0 10361 10362 | // zend_vm_stack_free_args(call); 10363 if (func && !unknown_num_args) { 10364 for (i = 0; i < call_num_args; i++ ) { 10365 if (zend_jit_needs_arg_dtor(func, i, call_info)) { 10366 uint32_t offset = EX_NUM_TO_VAR(i); 10367 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, opline 10368 } 10369 } 10370 } else { 10371 | mov FCARG1a, RX 10372 | EXT_CALL zend_jit_vm_stack_free_args_helper, r0 10373 } 10374 if (may_have_extra_named_params) { 10375 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24) 10376 | jnz >1 10377 |.cold_code 10378 |1: 10379 | mov FCARG1a, aword [RX + offsetof(zend_execute_data, extra_named_params)] 10380 | EXT_CALL zend_free_extra_named_params, r0 10381 | jmp >2 10382 |.code 10383 |2: 10384 } 10385 10386 |8: 10387 if (opline->opcode == ZEND_DO_FCALL) { 10388 // TODO: optimize ??? 10389 | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) 10390 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_RELEASE_THIS >> 16) 10391 | jnz >1 10392 |.cold_code 10393 |1: 10394 | GET_Z_PTR FCARG1a, RX + offsetof(zend_execute_data, This) 10395 | // OBJ_RELEASE(object); 10396 | OBJ_RELEASE ZREG_FCARG1, >2 10397 | jmp >2 10398 |.code 10399 |2: 10400 } 10401 10402 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10403 !JIT_G(current_frame) || 10404 !JIT_G(current_frame)->call || 10405 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || 10406 prev_opline->opcode == ZEND_SEND_UNPACK || 10407 prev_opline->opcode == ZEND_SEND_ARRAY || 10408 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 10409 10410 | // zend_vm_stack_free_call_frame(call); 10411 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16) 10412 | jnz >1 10413 |.cold_code 10414 |1: 10415 | mov FCARG1a, RX 10416 | EXT_CALL zend_jit_free_call_frame, r0 10417 | jmp >1 10418 |.code 10419 } 10420 | MEM_STORE_ZTS aword, executor_globals, vm_stack_top, RX, r0 10421 |1: 10422 10423 if (!RETURN_VALUE_USED(opline)) { 10424 zend_class_entry *ce; 10425 bool ce_is_instanceof; 10426 uint32_t func_info = call_info ? 10427 zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : 10428 (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); 10429 10430 /* If an exception is thrown, the return_value may stay at the 10431 * original value of null. */ 10432 func_info |= MAY_BE_NULL; 10433 10434 if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10435 | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline 10436 } 10437 } 10438 10439 | // if (UNEXPECTED(EG(exception) != NULL)) { 10440 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 10441 | jne ->icall_throw_handler 10442 10443 // TODO: Can we avoid checking for interrupts after each call ??? 10444 if (trace && last_valid_opline != opline) { 10445 int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); 10446 10447 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10448 if (!exit_addr) { 10449 return 0; 10450 } 10451 } else { 10452 exit_addr = NULL; 10453 } 10454 if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { 10455 return 0; 10456 } 10457 10458 if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { 10459 | LOAD_IP_ADDR (opline + 1) 10460 } else if (trace 10461 && trace->op == ZEND_JIT_TRACE_END 10462 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 10463 | LOAD_IP_ADDR (opline + 1) 10464 } 10465 } 10466 10467 if (!func) { 10468 |9: 10469 } 10470 10471 return 1; 10472} 10473 10474static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) 10475{ 10476 uint32_t arg_num = opline->op2.num; 10477 zend_jit_addr arg_addr; 10478 10479 ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); 10480 10481 if (!zend_jit_reuse_ip(Dst)) { 10482 return 0; 10483 } 10484 10485 if (opline->opcode == ZEND_SEND_VAL_EX) { 10486 uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); 10487 10488 ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); 10489 10490 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10491 && JIT_G(current_frame) 10492 && JIT_G(current_frame)->call 10493 && JIT_G(current_frame)->call->func) { 10494 if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10495 /* Don't generate code that always throws exception */ 10496 return 0; 10497 } 10498 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10499 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10500 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10501 if (!exit_addr) { 10502 return 0; 10503 } 10504 | mov r0, EX:RX->func 10505 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10506 | jnz &exit_addr 10507 } else { 10508 | mov r0, EX:RX->func 10509 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10510 | jnz >1 10511 |.cold_code 10512 |1: 10513 if (Z_MODE(op1_addr) == IS_REG) { 10514 /* set type to avoid zval_ptr_dtor() on uninitialized value */ 10515 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 10516 | SET_ZVAL_TYPE_INFO addr, IS_UNDEF 10517 } 10518 | SET_EX_OPLINE opline, r0 10519 | jmp ->throw_cannot_pass_by_ref 10520 |.code 10521 10522 } 10523 } 10524 10525 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10526 10527 if (opline->op1_type == IS_CONST) { 10528 zval *zv = RT_CONSTANT(opline, opline->op1); 10529 10530 | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 10531 if (Z_REFCOUNTED_P(zv)) { 10532 | ADDREF_CONST zv, r0 10533 } 10534 } else { 10535 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10536 } 10537 10538 return 1; 10539} 10540 10541static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) 10542{ 10543 | mov FCARG1a, EX->call 10544 | test byte [FCARG1a + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_MAY_HAVE_UNDEF >> 24) 10545 | jnz >1 10546 |.cold_code 10547 |1: 10548 | SET_EX_OPLINE opline, r0 10549 | EXT_CALL zend_handle_undef_args, r0 10550 | test r0, r0 10551 | jnz ->exception_handler 10552 | jmp >2 10553 |.code 10554 |2: 10555 10556 return 1; 10557} 10558 10559static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) 10560{ 10561 zend_jit_addr op1_addr, arg_addr, ref_addr; 10562 10563 op1_addr = OP1_ADDR(); 10564 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10565 10566 if (!zend_jit_reuse_ip(Dst)) { 10567 return 0; 10568 } 10569 10570 if (opline->op1_type == IS_VAR) { 10571 if (op1_info & MAY_BE_INDIRECT) { 10572 | LOAD_ZVAL_ADDR r0, op1_addr 10573 | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { 10574 | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 10575 | // ret = Z_INDIRECT_P(ret); 10576 | GET_Z_PTR r0, r0 10577 |1: 10578 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 10579 } 10580 } else if (opline->op1_type == IS_CV) { 10581 if (op1_info & MAY_BE_UNDEF) { 10582 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10583 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 10584 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 10585 | jmp >2 10586 |1: 10587 } 10588 op1_info &= ~MAY_BE_UNDEF; 10589 op1_info |= MAY_BE_NULL; 10590 } 10591 } else { 10592 ZEND_UNREACHABLE(); 10593 } 10594 10595 if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { 10596 if (op1_info & MAY_BE_REF) { 10597 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2 10598 | GET_ZVAL_PTR r1, op1_addr 10599 | GC_ADDREF r1 10600 | SET_ZVAL_PTR arg_addr, r1 10601 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX 10602 | jmp >6 10603 } 10604 |2: 10605 | // ZVAL_NEW_REF(arg, varptr); 10606 if (opline->op1_type == IS_VAR) { 10607 if (Z_REG(op1_addr) != ZREG_R0 || Z_OFFSET(op1_addr) != 0) { 10608 | LOAD_ZVAL_ADDR r0, op1_addr 10609 } 10610 | mov aword T1, r0 // save 10611 } 10612 | EMALLOC sizeof(zend_reference), op_array, opline 10613 | mov dword [r0], 2 10614 | mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], GC_REFERENCE 10615 | mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0 10616 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); 10617 if (opline->op1_type == IS_VAR) { 10618 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); 10619 10620 | mov r1, aword T1 // restore 10621 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R2, ZREG_R2 10622 | SET_ZVAL_PTR val_addr, r0 10623 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX 10624 } else { 10625 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10626 | SET_ZVAL_PTR op1_addr, r0 10627 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 10628 } 10629 | SET_ZVAL_PTR arg_addr, r0 10630 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX 10631 } 10632 10633 |6: 10634 | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline 10635 |7: 10636 10637 return 1; 10638} 10639 10640static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) 10641{ 10642 uint32_t arg_num = opline->op2.num; 10643 zend_jit_addr arg_addr; 10644 10645 ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && 10646 opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || 10647 arg_num <= MAX_ARG_FLAG_NUM); 10648 10649 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10650 10651 if (!zend_jit_reuse_ip(Dst)) { 10652 return 0; 10653 } 10654 10655 if (opline->opcode == ZEND_SEND_VAR_EX) { 10656 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10657 && JIT_G(current_frame) 10658 && JIT_G(current_frame)->call 10659 && JIT_G(current_frame)->call->func) { 10660 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10661 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10662 return 0; 10663 } 10664 return 1; 10665 } 10666 } else { 10667 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10668 10669 | mov r0, EX:RX->func 10670 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10671 | jnz >1 10672 |.cold_code 10673 |1: 10674 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10675 return 0; 10676 } 10677 | jmp >7 10678 |.code 10679 } 10680 } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { 10681 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10682 && JIT_G(current_frame) 10683 && JIT_G(current_frame)->call 10684 && JIT_G(current_frame)->call->func) { 10685 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10686 10687 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10688 10689 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10690 if (!(op1_info & MAY_BE_REF)) { 10691 /* Don't generate code that always throws exception */ 10692 return 0; 10693 } else { 10694 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10695 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10696 if (!exit_addr) { 10697 return 0; 10698 } 10699 | cmp cl, IS_REFERENCE 10700 | jne &exit_addr 10701 } 10702 } 10703 return 1; 10704 } 10705 } else { 10706 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10707 10708 | mov r0, EX:RX->func 10709 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10710 | jnz >1 10711 |.cold_code 10712 |1: 10713 10714 mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); 10715 10716 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10717 if (op1_info & MAY_BE_REF) { 10718 | cmp cl, IS_REFERENCE 10719 | je >7 10720 } 10721 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10722 | jnz >7 10723 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10724 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10725 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10726 if (!exit_addr) { 10727 return 0; 10728 } 10729 | jmp &exit_addr 10730 } else { 10731 | SET_EX_OPLINE opline, r0 10732 | LOAD_ZVAL_ADDR FCARG1a, arg_addr 10733 | EXT_CALL zend_jit_only_vars_by_reference, r0 10734 if (!zend_jit_check_exception(Dst)) { 10735 return 0; 10736 } 10737 | jmp >7 10738 } 10739 10740 |.code 10741 } 10742 } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { 10743 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10744 && JIT_G(current_frame) 10745 && JIT_G(current_frame)->call 10746 && JIT_G(current_frame)->call->func) { 10747 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10748 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10749 return 0; 10750 } 10751 return 1; 10752 } 10753 } else { 10754 | test dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10755 | jnz >1 10756 |.cold_code 10757 |1: 10758 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10759 return 0; 10760 } 10761 | jmp >7 10762 |.code 10763 } 10764 } 10765 10766 if (op1_info & MAY_BE_UNDEF) { 10767 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10768 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 10769 |.cold_code 10770 |1: 10771 } 10772 10773 | SET_EX_OPLINE opline, r0 10774 | mov FCARG1d, opline->op1.var 10775 | EXT_CALL zend_jit_undefined_op_helper, r0 10776 | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL 10777 | test r0, r0 10778 | jz ->exception_handler 10779 10780 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10781 | jmp >7 10782 |.code 10783 } else { 10784 |7: 10785 return 1; 10786 } 10787 } 10788 10789 if (opline->opcode == ZEND_SEND_VAR_NO_REF) { 10790 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10791 if (op1_info & MAY_BE_REF) { 10792 | cmp cl, IS_REFERENCE 10793 | je >7 10794 } 10795 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10796 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10797 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10798 if (!exit_addr) { 10799 return 0; 10800 } 10801 | jmp &exit_addr 10802 } else { 10803 | SET_EX_OPLINE opline, r0 10804 | LOAD_ZVAL_ADDR FCARG1a, arg_addr 10805 | EXT_CALL zend_jit_only_vars_by_reference, r0 10806 if (!zend_jit_check_exception(Dst)) { 10807 return 0; 10808 } 10809 } 10810 } else { 10811 if (op1_info & MAY_BE_REF) { 10812 if (opline->op1_type == IS_CV) { 10813 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 10814 10815 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 10816 | ZVAL_DEREF FCARG1a, op1_info 10817 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R0, ZREG_R2 10818 | TRY_ADDREF op1_info, ah, r2 10819 } else { 10820 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 8); 10821 10822 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 10823 |.cold_code 10824 |1: 10825 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 10826 | GET_ZVAL_PTR FCARG1a, op1_addr 10827 | // ZVAL_COPY_VALUE(return_value, &ref->value); 10828 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R0, ZREG_R2 10829 | GC_DELREF FCARG1a 10830 | je >1 10831 | IF_NOT_REFCOUNTED ah, >2 10832 | GC_ADDREF r2 10833 | jmp >2 10834 |1: 10835 | EFREE_REG_REFERENCE 10836 | jmp >2 10837 |.code 10838 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10839 |2: 10840 } 10841 } else { 10842 if (op1_addr != op1_def_addr) { 10843 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 10844 return 0; 10845 } 10846 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 10847 op1_addr= op1_def_addr; 10848 } 10849 } 10850 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10851 if (opline->op1_type == IS_CV) { 10852 | TRY_ADDREF op1_info, ah, r2 10853 } 10854 } 10855 } 10856 |7: 10857 10858 return 1; 10859} 10860 10861static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) 10862{ 10863 uint32_t arg_num = opline->op2.num; 10864 10865 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10866 && JIT_G(current_frame) 10867 && JIT_G(current_frame)->call 10868 && JIT_G(current_frame)->call->func) { 10869 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10870 if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { 10871 TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); 10872 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10873 || if (reuse_ip) { 10874 | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10875 || } else { 10876 | mov r0, EX->call 10877 | or dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10878 || } 10879 } 10880 } else { 10881 if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 10882 TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); 10883 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10884 || if (reuse_ip) { 10885 | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10886 || } else { 10887 | mov r0, EX->call 10888 | and dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10889 || } 10890 } 10891 } 10892 } else { 10893 // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { 10894 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10895 10896 if (!zend_jit_reuse_ip(Dst)) { 10897 return 0; 10898 } 10899 10900 | mov r0, EX:RX->func 10901 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10902 | jnz >1 10903 |.cold_code 10904 |1: 10905 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10906 | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10907 | jmp >1 10908 |.code 10909 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10910 | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10911 |1: 10912 } 10913 10914 return 1; 10915} 10916 10917static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2) 10918{ 10919 if (smart_branch_opcode) { 10920 if (smart_branch_opcode == ZEND_JMPZ) { 10921 if (jmp) { 10922 | jmp >7 10923 } 10924 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10925 | jmp =>target_label 10926 } else { 10927 ZEND_UNREACHABLE(); 10928 } 10929 } else { 10930 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10931 10932 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 10933 if (jmp) { 10934 | jmp >7 10935 } 10936 } 10937 10938 return 1; 10939} 10940 10941static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, uint8_t smart_branch_opcode, uint32_t target_label) 10942{ 10943 if (smart_branch_opcode) { 10944 if (smart_branch_opcode == ZEND_JMPZ) { 10945 | jmp =>target_label 10946 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10947 if (jmp) { 10948 | jmp >7 10949 } 10950 } else { 10951 ZEND_UNREACHABLE(); 10952 } 10953 } else { 10954 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10955 10956 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 10957 if (jmp) { 10958 | jmp >7 10959 } 10960 } 10961 10962 return 1; 10963} 10964 10965static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 10966{ 10967 uint32_t defined_label = (uint32_t)-1; 10968 uint32_t undefined_label = (uint32_t)-1; 10969 zval *zv = RT_CONSTANT(opline, opline->op1); 10970 zend_jit_addr res_addr = 0; 10971 10972 if (smart_branch_opcode && !exit_addr) { 10973 if (smart_branch_opcode == ZEND_JMPZ) { 10974 undefined_label = target_label; 10975 } else if (smart_branch_opcode == ZEND_JMPNZ) { 10976 defined_label = target_label; 10977 } else { 10978 ZEND_UNREACHABLE(); 10979 } 10980 } 10981 10982 | // if (CACHED_PTR(opline->extended_value)) { 10983 | mov r0, EX->run_time_cache 10984 | mov r0, aword [r0 + opline->extended_value] 10985 | test r0, r0 10986 | jz >1 10987 | test r0, 0x1 10988 | jnz >4 10989 |.cold_code 10990 |4: 10991 | MEM_LOAD_ZTS FCARG1a, aword, executor_globals, zend_constants, FCARG1a 10992 | shr r0, 1 10993 | cmp dword [FCARG1a + offsetof(HashTable, nNumOfElements)], eax 10994 10995 if (smart_branch_opcode) { 10996 if (exit_addr) { 10997 if (smart_branch_opcode == ZEND_JMPZ) { 10998 | jz &exit_addr 10999 } else { 11000 | jz >3 11001 } 11002 } else if (undefined_label != (uint32_t)-1) { 11003 | jz =>undefined_label 11004 } else { 11005 | jz >3 11006 } 11007 } else { 11008 | jz >2 11009 } 11010 |1: 11011 | SET_EX_OPLINE opline, r0 11012 | LOAD_ADDR FCARG1a, zv 11013 | EXT_CALL zend_jit_check_constant, r0 11014 | test r0, r0 11015 if (exit_addr) { 11016 if (smart_branch_opcode == ZEND_JMPNZ) { 11017 | jz >3 11018 } else { 11019 | jnz >3 11020 } 11021 | jmp &exit_addr 11022 } else if (smart_branch_opcode) { 11023 if (undefined_label != (uint32_t)-1) { 11024 | jz =>undefined_label 11025 } else { 11026 | jz >3 11027 } 11028 if (defined_label != (uint32_t)-1) { 11029 | jmp =>defined_label 11030 } else { 11031 | jmp >3 11032 } 11033 } else { 11034 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11035 | jnz >1 11036 |2: 11037 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 11038 | jmp >3 11039 } 11040 |.code 11041 if (smart_branch_opcode) { 11042 if (exit_addr) { 11043 if (smart_branch_opcode == ZEND_JMPNZ) { 11044 | jmp &exit_addr 11045 } 11046 } else if (defined_label != (uint32_t)-1) { 11047 | jmp =>defined_label 11048 } 11049 } else { 11050 |1: 11051 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 11052 } 11053 |3: 11054 11055 return 1; 11056} 11057 11058static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 11059{ 11060 uint32_t mask; 11061 zend_jit_addr op1_addr = OP1_ADDR(); 11062 11063 // TODO: support for is_resource() ??? 11064 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); 11065 11066 if (op1_info & MAY_BE_UNDEF) { 11067 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11068 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 11069 |.cold_code 11070 |1: 11071 } 11072 | SET_EX_OPLINE opline, r0 11073 | mov FCARG1d, opline->op1.var 11074 | EXT_CALL zend_jit_undefined_op_helper, r0 11075 zend_jit_check_exception_undef_result(Dst, opline); 11076 if (opline->extended_value & MAY_BE_NULL) { 11077 if (exit_addr) { 11078 if (smart_branch_opcode == ZEND_JMPNZ) { 11079 | jmp &exit_addr 11080 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 11081 | jmp >7 11082 } 11083 } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { 11084 return 0; 11085 } 11086 } else { 11087 if (exit_addr) { 11088 if (smart_branch_opcode == ZEND_JMPZ) { 11089 | jmp &exit_addr 11090 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 11091 | jmp >7 11092 } 11093 } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { 11094 return 0; 11095 } 11096 } 11097 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11098 |.code 11099 } 11100 } 11101 11102 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11103 mask = opline->extended_value; 11104 if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { 11105 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11106 if (exit_addr) { 11107 if (smart_branch_opcode == ZEND_JMPNZ) { 11108 | jmp &exit_addr 11109 } 11110 } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { 11111 return 0; 11112 } 11113 } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { 11114 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11115 if (exit_addr) { 11116 if (smart_branch_opcode == ZEND_JMPZ) { 11117 | jmp &exit_addr 11118 } 11119 } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { 11120 return 0; 11121 } 11122 } else { 11123 bool invert = 0; 11124 uint8_t type; 11125 11126 switch (mask) { 11127 case MAY_BE_NULL: type = IS_NULL; break; 11128 case MAY_BE_FALSE: type = IS_FALSE; break; 11129 case MAY_BE_TRUE: type = IS_TRUE; break; 11130 case MAY_BE_LONG: type = IS_LONG; break; 11131 case MAY_BE_DOUBLE: type = IS_DOUBLE; break; 11132 case MAY_BE_STRING: type = IS_STRING; break; 11133 case MAY_BE_ARRAY: type = IS_ARRAY; break; 11134 case MAY_BE_OBJECT: type = IS_OBJECT; break; 11135 case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break; 11136 case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break; 11137 case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break; 11138 case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break; 11139 case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break; 11140 case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break; 11141 case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break; 11142 case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break; 11143 case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break; 11144 default: 11145 type = 0; 11146 } 11147 11148 if (op1_info & MAY_BE_REF) { 11149 | LOAD_ZVAL_ADDR r0, op1_addr 11150 | ZVAL_DEREF r0, op1_info 11151 } 11152 if (type == 0) { 11153 if (smart_branch_opcode && 11154 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11155 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11156 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11157 | // if (Z_REFCOUNTED_P(cv)) { 11158 | IF_ZVAL_REFCOUNTED op1_addr, >1 11159 |.cold_code 11160 |1: 11161 } 11162 | // if (!Z_DELREF_P(cv)) { 11163 | GET_ZVAL_PTR FCARG1a, op1_addr 11164 | GC_DELREF FCARG1a 11165 if (RC_MAY_BE_1(op1_info)) { 11166 if (RC_MAY_BE_N(op1_info)) { 11167 | jnz >3 11168 } 11169 if (op1_info & MAY_BE_REF) { 11170 | mov al, byte [r0 + 8] 11171 } else { 11172 | mov al, byte [FP + opline->op1.var + 8] 11173 } 11174 | mov byte T1, al // save 11175 | // zval_dtor_func(r); 11176 | ZVAL_DTOR_FUNC op1_info, opline 11177 | mov cl, byte T1 // restore 11178 |jmp >2 11179 } 11180 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11181 if (!RC_MAY_BE_1(op1_info)) { 11182 | jmp >3 11183 } 11184 |.code 11185 } 11186 |3: 11187 if (op1_info & MAY_BE_REF) { 11188 | mov cl, byte [r0 + 8] 11189 } else { 11190 | mov cl, byte [FP + opline->op1.var + 8] 11191 } 11192 |2: 11193 } else { 11194 if (op1_info & MAY_BE_REF) { 11195 | mov cl, byte [r0 + 8] 11196 } else { 11197 | mov cl, byte [FP + opline->op1.var + 8] 11198 } 11199 } 11200 | mov eax, 1 11201 | shl eax, cl 11202 | test eax, mask 11203 if (exit_addr) { 11204 if (smart_branch_opcode == ZEND_JMPNZ) { 11205 | jne &exit_addr 11206 } else { 11207 | je &exit_addr 11208 } 11209 } else if (smart_branch_opcode) { 11210 if (smart_branch_opcode == ZEND_JMPZ) { 11211 | je =>target_label 11212 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11213 | jne =>target_label 11214 } else { 11215 ZEND_UNREACHABLE(); 11216 } 11217 } else { 11218 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11219 11220 | setne al 11221 | movzx eax, al 11222 | add eax, 2 11223 | SET_ZVAL_TYPE_INFO res_addr, eax 11224 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11225 } 11226 } else { 11227 if (smart_branch_opcode && 11228 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11229 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11230 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11231 | // if (Z_REFCOUNTED_P(cv)) { 11232 | IF_ZVAL_REFCOUNTED op1_addr, >1 11233 |.cold_code 11234 |1: 11235 } 11236 | // if (!Z_DELREF_P(cv)) { 11237 | GET_ZVAL_PTR FCARG1a, op1_addr 11238 | GC_DELREF FCARG1a 11239 if (RC_MAY_BE_1(op1_info)) { 11240 if (RC_MAY_BE_N(op1_info)) { 11241 | jnz >3 11242 } 11243 if (op1_info & MAY_BE_REF) { 11244 | mov al, byte [r0 + 8] 11245 } else { 11246 | mov al, byte [FP + opline->op1.var + 8] 11247 } 11248 | mov byte T1, al // save 11249 | // zval_dtor_func(r); 11250 | ZVAL_DTOR_FUNC op1_info, opline 11251 | mov cl, byte T1 // restore 11252 |jmp >2 11253 } 11254 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11255 if (!RC_MAY_BE_1(op1_info)) { 11256 | jmp >3 11257 } 11258 |.code 11259 } 11260 |3: 11261 if (op1_info & MAY_BE_REF) { 11262 | mov cl, byte [r0 + 8] 11263 } else { 11264 | mov cl, byte [FP + opline->op1.var + 8] 11265 } 11266 |2: 11267 | cmp cl, type 11268 } else { 11269 if (op1_info & MAY_BE_REF) { 11270 | cmp byte [r0 + 8], type 11271 } else { 11272 | cmp byte [FP + opline->op1.var + 8], type 11273 } 11274 } 11275 if (exit_addr) { 11276 if (invert) { 11277 if (smart_branch_opcode == ZEND_JMPNZ) { 11278 | jne &exit_addr 11279 } else { 11280 | je &exit_addr 11281 } 11282 } else { 11283 if (smart_branch_opcode == ZEND_JMPNZ) { 11284 | je &exit_addr 11285 } else { 11286 | jne &exit_addr 11287 } 11288 } 11289 } else if (smart_branch_opcode) { 11290 if (invert) { 11291 if (smart_branch_opcode == ZEND_JMPZ) { 11292 | je =>target_label 11293 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11294 | jne =>target_label 11295 } else { 11296 ZEND_UNREACHABLE(); 11297 } 11298 } else { 11299 if (smart_branch_opcode == ZEND_JMPZ) { 11300 | jne =>target_label 11301 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11302 | je =>target_label 11303 } else { 11304 ZEND_UNREACHABLE(); 11305 } 11306 } 11307 } else { 11308 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11309 11310 if (invert) { 11311 | setne al 11312 } else { 11313 | sete al 11314 } 11315 | movzx eax, al 11316 | add eax, 2 11317 | SET_ZVAL_TYPE_INFO res_addr, eax 11318 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11319 } 11320 } 11321 } 11322 } 11323 11324 |7: 11325 11326 return 1; 11327} 11328 11329static int zend_jit_leave_frame(dasm_State **Dst) 11330{ 11331 | // EG(current_execute_data) = EX(prev_execute_data); 11332 | mov r0, EX->prev_execute_data 11333 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, r0, r2 11334 return 1; 11335} 11336 11337static int zend_jit_free_cvs(dasm_State **Dst) 11338{ 11339 | // EG(current_execute_data) = EX(prev_execute_data); 11340 | mov FCARG1a, EX->prev_execute_data 11341 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, FCARG1a, r0 11342 | // zend_free_compiled_variables(execute_data); 11343 | mov FCARG1a, FP 11344 | EXT_CALL zend_free_compiled_variables, r0 11345 return 1; 11346} 11347 11348static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) 11349{ 11350 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11351 uint32_t offset = EX_NUM_TO_VAR(var); 11352 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, NULL 11353 } 11354 return 1; 11355} 11356 11357static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) 11358{ 11359 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11360 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, 1, opline 11361 } 11362 return 1; 11363} 11364 11365static int zend_jit_leave_func(dasm_State **Dst, 11366 const zend_op_array *op_array, 11367 const zend_op *opline, 11368 uint32_t op1_info, 11369 bool left_frame, 11370 zend_jit_trace_rec *trace, 11371 zend_jit_trace_info *trace_info, 11372 int indirect_var_access, 11373 int may_throw) 11374{ 11375 bool may_be_top_frame = 11376 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11377 !JIT_G(current_frame) || 11378 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); 11379 bool may_need_call_helper = 11380 indirect_var_access || /* may have symbol table */ 11381 !op_array->function_name || /* may have symbol table */ 11382 may_be_top_frame || 11383 (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ 11384 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11385 !JIT_G(current_frame) || 11386 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ 11387 (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ 11388 bool may_need_release_this = 11389 !(op_array->fn_flags & ZEND_ACC_CLOSURE) && 11390 op_array->scope && 11391 !(op_array->fn_flags & ZEND_ACC_STATIC) && 11392 (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11393 !JIT_G(current_frame) || 11394 !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame))); 11395 11396 if (may_need_release_this) { 11397 | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)] 11398 } 11399 if (may_need_call_helper) { 11400 if (!left_frame) { 11401 left_frame = 1; 11402 if (!zend_jit_leave_frame(Dst)) { 11403 return 0; 11404 } 11405 } 11406 /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ 11407 if (may_need_release_this) { 11408 | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) 11409 } else { 11410 | test dword [FP + offsetof(zend_execute_data, This.u1.type_info)], (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) 11411 } 11412 if (trace && trace->op != ZEND_JIT_TRACE_END) { 11413 | jnz >1 11414 |.cold_code 11415 |1: 11416 if (!GCC_GLOBAL_REGS) { 11417 | mov FCARG1a, FP 11418 } 11419 | EXT_CALL zend_jit_leave_func_helper, r0 11420 11421 if (may_be_top_frame) { 11422 // TODO: try to avoid this check ??? 11423 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 11424#if 0 11425 /* this check should be handled by the following OPLINE guard */ 11426 | cmp IP, zend_jit_halt_op 11427 | je ->trace_halt 11428#endif 11429 } else if (GCC_GLOBAL_REGS) { 11430 | test IP, IP 11431 | je ->trace_halt 11432 } else { 11433 | test eax, eax 11434 | jl ->trace_halt 11435 } 11436 } 11437 11438 if (!GCC_GLOBAL_REGS) { 11439 | // execute_data = EG(current_execute_data) 11440 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 11441 } 11442 | jmp >8 11443 |.code 11444 } else { 11445 | jnz ->leave_function_handler 11446 } 11447 } 11448 11449 if (op_array->fn_flags & ZEND_ACC_CLOSURE) { 11450 if (!left_frame) { 11451 left_frame = 1; 11452 if (!zend_jit_leave_frame(Dst)) { 11453 return 0; 11454 } 11455 } 11456 | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); 11457 | mov FCARG1a, EX->func 11458 | sub FCARG1a, sizeof(zend_object) 11459 | OBJ_RELEASE ZREG_FCARG1, >4 11460 |4: 11461 } else if (may_need_release_this) { 11462 if (!left_frame) { 11463 left_frame = 1; 11464 if (!zend_jit_leave_frame(Dst)) { 11465 return 0; 11466 } 11467 } 11468 if (!JIT_G(current_frame) || !TRACE_FRAME_ALWAYS_RELEASE_THIS(JIT_G(current_frame))) { 11469 | // if (call_info & ZEND_CALL_RELEASE_THIS) 11470 | test FCARG1d, ZEND_CALL_RELEASE_THIS 11471 | je >4 11472 } 11473 | // zend_object *object = Z_OBJ(execute_data->This); 11474 | mov FCARG1a, EX->This.value.obj 11475 | // OBJ_RELEASE(object); 11476 | OBJ_RELEASE ZREG_FCARG1, >4 11477 |4: 11478 // TODO: avoid EG(excption) check for $this->foo() calls 11479 may_throw = 1; 11480 } 11481 11482 | // EG(vm_stack_top) = (zval*)execute_data; 11483 | MEM_STORE_ZTS aword, executor_globals, vm_stack_top, FP, r0 11484 | // execute_data = EX(prev_execute_data); 11485 | mov FP, EX->prev_execute_data 11486 11487 if (!left_frame) { 11488 | // EG(current_execute_data) = execute_data; 11489 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, FP, r0 11490 } 11491 11492 |9: 11493 if (trace) { 11494 if (trace->op != ZEND_JIT_TRACE_END 11495 && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 11496 zend_jit_reset_last_valid_opline(); 11497 } else { 11498 | LOAD_IP 11499 | ADD_IP sizeof(zend_op) 11500 } 11501 11502 |8: 11503 11504 if (trace->op == ZEND_JIT_TRACE_BACK 11505 && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 11506 const zend_op *next_opline = trace->opline; 11507 11508 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11509 && (op1_info & MAY_BE_RC1) 11510 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { 11511 /* exception might be thrown during destruction of unused return value */ 11512 | // if (EG(exception)) 11513 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 11514 | jne ->leave_throw_handler 11515 } 11516 do { 11517 trace++; 11518 } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); 11519 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 11520 next_opline = trace->opline; 11521 ZEND_ASSERT(next_opline != NULL); 11522 11523 if (trace->op == ZEND_JIT_TRACE_END 11524 && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { 11525 trace_info->flags |= ZEND_JIT_TRACE_LOOP; 11526 | CMP_IP next_opline 11527 | je =>0 // LOOP 11528#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 11529 | JMP_IP 11530#else 11531 | jmp ->trace_escape 11532#endif 11533 } else { 11534 | CMP_IP next_opline 11535 | jne ->trace_escape 11536 } 11537 11538 zend_jit_set_last_valid_opline(trace->opline); 11539 11540 return 1; 11541 } else if (may_throw || 11542 (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11543 && (op1_info & MAY_BE_RC1) 11544 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) 11545 && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { 11546 | // if (EG(exception)) 11547 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 11548 | jne ->leave_throw_handler 11549 } 11550 11551 return 1; 11552 } else { 11553 | // if (EG(exception)) 11554 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 11555 | LOAD_IP 11556 | jne ->leave_throw_handler 11557 | // opline = EX(opline) + 1 11558 | ADD_IP sizeof(zend_op) 11559 } 11560 11561 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 11562 | ADD_HYBRID_SPAD 11563#ifdef CONTEXT_THREADED_JIT 11564 | push aword [IP] 11565 | ret 11566#else 11567 | JMP_IP 11568#endif 11569 } else if (GCC_GLOBAL_REGS) { 11570 | add r4, SPAD // stack alignment 11571#ifdef CONTEXT_THREADED_JIT 11572 | push aword [IP] 11573 | ret 11574#else 11575 | JMP_IP 11576#endif 11577 } else { 11578#ifdef CONTEXT_THREADED_JIT 11579 ZEND_UNREACHABLE(); 11580 // TODO: context threading can't work without GLOBAL REGS because we have to change 11581 // the value of execute_data in execute_ex() 11582 | mov FCARG1a, FP 11583 | mov r0, aword [FP] 11584 | mov FP, aword T2 // restore FP 11585 | mov RX, aword T3 // restore IP 11586 | add r4, NR_SPAD // stack alignment 11587 | push aword [r0] 11588 | ret 11589#else 11590 | mov FP, aword T2 // restore FP 11591 | mov RX, aword T3 // restore IP 11592 | add r4, NR_SPAD // stack alignment 11593 | mov r0, 2 // ZEND_VM_LEAVE 11594 | ret 11595#endif 11596 } 11597 11598 return 1; 11599} 11600 11601static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) 11602{ 11603 zend_jit_addr ret_addr; 11604 int8_t return_value_used; 11605 11606 ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); 11607 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); 11608 11609 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { 11610 if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { 11611 return_value_used = 1; 11612 } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { 11613 return_value_used = 0; 11614 } else { 11615 return_value_used = -1; 11616 } 11617 } else { 11618 return_value_used = -1; 11619 } 11620 11621 if (ZEND_OBSERVER_ENABLED) { 11622 if (Z_MODE(op1_addr) == IS_REG) { 11623 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 11624 11625 if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { 11626 return 0; 11627 } 11628 op1_addr = dst; 11629 } 11630 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 11631 | mov FCARG1a, FP 11632 | SET_EX_OPLINE opline, r0 11633 | EXT_CALL zend_observer_fcall_end, r0 11634 } 11635 11636 // if (!EX(return_value)) 11637 if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) { 11638 if (return_value_used != 0) { 11639 | mov r2, EX->return_value 11640 } 11641 if (return_value_used == -1) { 11642 | test r2, r2 11643 } 11644 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); 11645 } else { 11646 if (return_value_used != 0) { 11647 | mov r1, EX->return_value 11648 } 11649 if (return_value_used == -1) { 11650 | test r1, r1 11651 } 11652 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); 11653 } 11654 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11655 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11656 if (return_value_used == -1) { 11657 | jz >1 11658 |.cold_code 11659 |1: 11660 } 11661 if (return_value_used != 1) { 11662 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11663 if (jit_return_label >= 0) { 11664 | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label 11665 } else { 11666 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9 11667 } 11668 } 11669 | GET_ZVAL_PTR FCARG1a, op1_addr 11670 | GC_DELREF FCARG1a 11671 if (RC_MAY_BE_1(op1_info)) { 11672 if (RC_MAY_BE_N(op1_info)) { 11673 if (jit_return_label >= 0) { 11674 | jnz =>jit_return_label 11675 } else { 11676 | jnz >9 11677 } 11678 } 11679 | //SAVE_OPLINE() 11680 | ZVAL_DTOR_FUNC op1_info, opline 11681 | //????mov r1, EX->return_value // reload ??? 11682 } 11683 if (return_value_used == -1) { 11684 if (jit_return_label >= 0) { 11685 | jmp =>jit_return_label 11686 } else { 11687 | jmp >9 11688 } 11689 |.code 11690 } 11691 } 11692 } else if (return_value_used == -1) { 11693 if (jit_return_label >= 0) { 11694 | jz =>jit_return_label 11695 } else { 11696 | jz >9 11697 } 11698 } 11699 11700 if (return_value_used == 0) { 11701 |9: 11702 return 1; 11703 } 11704 11705 if (opline->op1_type == IS_CONST) { 11706 zval *zv = RT_CONSTANT(opline, opline->op1); 11707 | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 11708 if (Z_REFCOUNTED_P(zv)) { 11709 | ADDREF_CONST zv, r0 11710 } 11711 } else if (opline->op1_type == IS_TMP_VAR) { 11712 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11713 } else if (opline->op1_type == IS_CV) { 11714 if (op1_info & MAY_BE_REF) { 11715 | LOAD_ZVAL_ADDR r0, op1_addr 11716 | ZVAL_DEREF r0, op1_info 11717 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 11718 } 11719 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11720 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 11721 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11722 (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || 11723 !op_array->function_name) { 11724 | TRY_ADDREF op1_info, ah, r2 11725 } else if (return_value_used != 1) { 11726 | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); 11727 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 11728 } 11729 } 11730 } else { 11731 if (op1_info & MAY_BE_REF) { 11732 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); 11733 11734 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 11735 |.cold_code 11736 |1: 11737 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 11738 | GET_ZVAL_PTR r0, op1_addr 11739 | // ZVAL_COPY_VALUE(return_value, &ref->value); 11740 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R2, ZREG_R2 11741 | GC_DELREF r0 11742 | je >2 11743 | // if (IS_REFCOUNTED()) 11744 if (jit_return_label >= 0) { 11745 | IF_NOT_REFCOUNTED dh, =>jit_return_label 11746 } else { 11747 | IF_NOT_REFCOUNTED dh, >9 11748 } 11749 | // ADDREF 11750 | GET_ZVAL_PTR r2, ret_addr // reload 11751 | GC_ADDREF r2 11752 if (jit_return_label >= 0) { 11753 | jmp =>jit_return_label 11754 } else { 11755 | jmp >9 11756 } 11757 |2: 11758 | EFREE_REFERENCE r0 11759 if (jit_return_label >= 0) { 11760 | jmp =>jit_return_label 11761 } else { 11762 | jmp >9 11763 } 11764 |.code 11765 } 11766 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11767 } 11768 11769 |9: 11770 return 1; 11771} 11772 11773static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) 11774{ 11775 ZEND_ASSERT(type_reg == ZREG_R2); 11776 11777 |.if not(X64) 11778 || if (Z_REG(val_addr) == ZREG_R1) { 11779 | GET_ZVAL_W2 r0, val_addr 11780 || } 11781 |.endif 11782 | GET_ZVAL_PTR r1, val_addr 11783 |.if not(X64) 11784 || if (Z_REG(val_addr) != ZREG_R1) { 11785 | GET_ZVAL_W2 r0, val_addr 11786 || } 11787 |.endif 11788 | IF_NOT_REFCOUNTED dh, >2 11789 | IF_NOT_TYPE dl, IS_REFERENCE, >1 11790 | GET_Z_TYPE_INFO edx, r1+offsetof(zend_reference, val) 11791 |.if not(X64) 11792 | GET_Z_W2 r0, r1+offsetof(zend_reference, val) 11793 |.endif 11794 | GET_Z_PTR r1, r1+offsetof(zend_reference, val) 11795 | IF_NOT_REFCOUNTED dh, >2 11796 |1: 11797 | GC_ADDREF r1 11798 |2: 11799 | SET_ZVAL_PTR res_addr, r1 11800 |.if not(X64) 11801 | SET_ZVAL_W2 res_addr, r0 11802 |.endif 11803 | SET_ZVAL_TYPE_INFO res_addr, edx 11804 11805 return 1; 11806} 11807 11808static int zend_jit_fetch_dim_read(dasm_State **Dst, 11809 const zend_op *opline, 11810 zend_ssa *ssa, 11811 const zend_ssa_op *ssa_op, 11812 uint32_t op1_info, 11813 zend_jit_addr op1_addr, 11814 bool op1_avoid_refcounting, 11815 uint32_t op2_info, 11816 uint32_t res_info, 11817 zend_jit_addr res_addr, 11818 uint8_t dim_type) 11819{ 11820 zend_jit_addr orig_op1_addr, op2_addr; 11821 const void *exit_addr = NULL; 11822 const void *not_found_exit_addr = NULL; 11823 const void *res_exit_addr = NULL; 11824 bool result_avoid_refcounting = 0; 11825 uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; 11826 int may_throw = 0; 11827 11828 orig_op1_addr = OP1_ADDR(); 11829 op2_addr = OP2_ADDR(); 11830 11831 if (opline->opcode != ZEND_FETCH_DIM_IS 11832 && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11833 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 11834 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11835 if (!exit_addr) { 11836 return 0; 11837 } 11838 } 11839 11840 if ((res_info & MAY_BE_GUARD) 11841 && JIT_G(current_frame) 11842 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { 11843 uint32_t flags = 0; 11844 uint32_t old_op1_info = 0; 11845 uint32_t old_info; 11846 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 11847 int32_t exit_point; 11848 11849 if (opline->opcode != ZEND_FETCH_LIST_R 11850 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11851 && !op1_avoid_refcounting) { 11852 flags |= ZEND_JIT_EXIT_FREE_OP1; 11853 } 11854 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) 11855 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11856 flags |= ZEND_JIT_EXIT_FREE_OP2; 11857 } 11858 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 11859 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 11860 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 11861 && (ssa_op+1)->op1_use == ssa_op->result_def 11862 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) 11863 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 11864 result_avoid_refcounting = 1; 11865 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 11866 } 11867 11868 if (op1_avoid_refcounting) { 11869 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 11870 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 11871 } 11872 11873 if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { 11874 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11875 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 11876 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 11877 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11878 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11879 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11880 if (!res_exit_addr) { 11881 return 0; 11882 } 11883 res_info &= ~MAY_BE_GUARD; 11884 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 11885 } 11886 11887 if (opline->opcode == ZEND_FETCH_DIM_IS 11888 && !(res_info & MAY_BE_NULL)) { 11889 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11890 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); 11891 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); 11892 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11893 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 11894 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11895 if (!not_found_exit_addr) { 11896 return 0; 11897 } 11898 } 11899 11900 if (op1_avoid_refcounting) { 11901 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 11902 } 11903 } 11904 11905 if (op1_info & MAY_BE_REF) { 11906 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 11907 | ZVAL_DEREF FCARG1a, op1_info 11908 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 11909 } 11910 11911 if (op1_info & MAY_BE_ARRAY) { 11912 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 11913 if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { 11914 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr 11915 } else { 11916 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 11917 } 11918 } 11919 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 11920 if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) || 11921 (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) { 11922 may_throw = 1; 11923 } 11924 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, dim_type, res_exit_addr, not_found_exit_addr, exit_addr)) { 11925 return 0; 11926 } 11927 } 11928 11929 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 11930 if (op1_info & MAY_BE_ARRAY) { 11931 |.cold_code 11932 |7: 11933 } 11934 11935 if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { 11936 may_throw = 1; 11937 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { 11938 if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { 11939 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr 11940 } else { 11941 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 11942 } 11943 } 11944 | SET_EX_OPLINE opline, r0 11945 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 11946 if (opline->opcode != ZEND_FETCH_DIM_IS) { 11947 if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { 11948 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 11949 | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, r0 11950 } else { 11951 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 11952 | EXT_CALL zend_jit_fetch_dim_str_r_helper, r0 11953 } 11954 | SET_ZVAL_PTR res_addr, r0 11955 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING 11956 } else { 11957 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 11958 |.if X64 11959 | LOAD_ZVAL_ADDR CARG3, res_addr 11960 |.else 11961 | sub r4, 12 11962 | PUSH_ZVAL_ADDR res_addr, r0 11963 |.endif 11964 | EXT_CALL zend_jit_fetch_dim_str_is_helper, r0 11965 |.if not(X64) 11966 | add r4, 12 11967 |.endif 11968 } 11969 if ((op1_info & MAY_BE_ARRAY) || 11970 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { 11971 | jmp >9 // END 11972 } 11973 |6: 11974 } 11975 11976 if (op1_info & MAY_BE_OBJECT) { 11977 may_throw = 1; 11978 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { 11979 if (exit_addr) { 11980 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 11981 } else { 11982 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6 11983 } 11984 } 11985 | SET_EX_OPLINE opline, r0 11986 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 11987 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 11988 } 11989 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 11990 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 11991 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 11992 } else { 11993 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 11994 } 11995 |.if X64 11996 | LOAD_ZVAL_ADDR CARG3, res_addr 11997 |.else 11998 | sub r4, 12 11999 | PUSH_ZVAL_ADDR res_addr, r0 12000 |.endif 12001 if (opline->opcode != ZEND_FETCH_DIM_IS) { 12002 | EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0 12003 } else { 12004 | EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0 12005 } 12006 |.if not(X64) 12007 | add r4, 12 12008 |.endif 12009 if ((op1_info & MAY_BE_ARRAY) || 12010 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 12011 | jmp >9 // END 12012 } 12013 |6: 12014 } 12015 12016 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) 12017 && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 12018 if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { 12019 | SET_EX_OPLINE opline, r0 12020 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { 12021 may_throw = 1; 12022 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 12023 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 12024 | mov FCARG1d, opline->op1.var 12025 | EXT_CALL zend_jit_undefined_op_helper, r0 12026 |1: 12027 } 12028 12029 if (op2_info & MAY_BE_UNDEF) { 12030 may_throw = 1; 12031 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 12032 | mov FCARG1d, opline->op2.var 12033 | EXT_CALL zend_jit_undefined_op_helper, r0 12034 |1: 12035 } 12036 } 12037 12038 if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { 12039 may_throw = 1; 12040 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { 12041 | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr 12042 } else { 12043 | SET_EX_OPLINE opline, r0 12044 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || 12045 Z_REG(op1_addr) != ZREG_FCARG1 || 12046 Z_OFFSET(op1_addr) != 0) { 12047 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12048 } 12049 } 12050 | EXT_CALL zend_jit_invalid_array_access, r0 12051 } 12052 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 12053 if (op1_info & MAY_BE_ARRAY) { 12054 | jmp >9 // END 12055 } 12056 } 12057 12058 if (op1_info & MAY_BE_ARRAY) { 12059 |.code 12060 } 12061 } 12062 12063 if (op1_info & MAY_BE_ARRAY) { 12064 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 12065 12066 |8: 12067 if (res_exit_addr) { 12068 uint8_t type = concrete_type(res_info); 12069 12070 if ((op1_info & MAY_BE_ARRAY_OF_REF) 12071 && dim_type != IS_UNKNOWN 12072 && dim_type != IS_REFERENCE) { 12073 if (type < IS_STRING) { 12074 | IF_NOT_ZVAL_TYPE val_addr, type, >1 12075 |.cold_code 12076 |1: 12077 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, &res_exit_addr 12078 | GET_Z_PTR r0, r0 12079 | add r0, offsetof(zend_reference, val) 12080 | IF_ZVAL_TYPE val_addr, type, >1 12081 | jmp &res_exit_addr 12082 |.code 12083 |1: 12084 } else { 12085 | GET_ZVAL_TYPE_INFO edx, val_addr 12086 | IF_NOT_TYPE dl, type, >1 12087 |.cold_code 12088 |1: 12089 | IF_NOT_TYPE dl, IS_REFERENCE, &res_exit_addr 12090 | GET_Z_PTR r0, r0 12091 | add r0, offsetof(zend_reference, val) 12092 | GET_ZVAL_TYPE_INFO edx, val_addr 12093 | IF_TYPE dl, type, >1 12094 | jmp &res_exit_addr 12095 |.code 12096 |1: 12097 } 12098 } else { 12099 if (op1_info & MAY_BE_ARRAY_OF_REF) { 12100 | ZVAL_DEREF r0, MAY_BE_REF 12101 } 12102 if (type < IS_STRING) { 12103 | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr 12104 } else { 12105 | GET_ZVAL_TYPE_INFO edx, val_addr 12106 | IF_NOT_TYPE dl, type, &res_exit_addr 12107 } 12108 } 12109 12110 | // ZVAL_COPY 12111 |7: 12112 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 12113 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 12114 if (type < IS_STRING) { 12115 if (Z_REG(res_addr) != ZREG_FP || 12116 JIT_G(current_frame) == NULL || 12117 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 12118 | SET_ZVAL_TYPE_INFO res_addr, type 12119 } 12120 } else { 12121 | SET_ZVAL_TYPE_INFO res_addr, edx 12122 if (!result_avoid_refcounting) { 12123 | TRY_ADDREF res_info, dh, r1 12124 } 12125 } 12126 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 12127 return 0; 12128 } 12129 } else if (op1_info & MAY_BE_ARRAY_OF_REF) { 12130 | // ZVAL_COPY_DEREF 12131 | GET_ZVAL_TYPE_INFO Rd(ZREG_R2), val_addr 12132 if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_R2)) { 12133 return 0; 12134 } 12135 } else { 12136 | // ZVAL_COPY 12137 | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_R1, ZREG_R2 12138 | TRY_ADDREF res_info, ch, r2 12139 } 12140 } 12141 |9: // END 12142 12143#ifdef ZEND_JIT_USE_RC_INFERENCE 12144 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 12145 /* Magic offsetGet() may increase refcount of the key */ 12146 op2_info |= MAY_BE_RCN; 12147 } 12148#endif 12149 12150 if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) { 12151 if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) { 12152 may_throw = 1; 12153 } 12154 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12155 } 12156 if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { 12157 if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) { 12158 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 12159 may_throw = 1; 12160 } 12161 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12162 } 12163 } 12164 12165 if (may_throw) { 12166 if (!zend_jit_check_exception(Dst)) { 12167 return 0; 12168 } 12169 } 12170 12171 return 1; 12172} 12173 12174static int zend_jit_fetch_dim(dasm_State **Dst, 12175 const zend_op *opline, 12176 uint32_t op1_info, 12177 zend_jit_addr op1_addr, 12178 uint32_t op2_info, 12179 zend_jit_addr res_addr, 12180 uint8_t dim_type) 12181{ 12182 zend_jit_addr op2_addr; 12183 int may_throw = 0; 12184 12185 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 12186 12187 if (opline->opcode == ZEND_FETCH_DIM_RW) { 12188 | SET_EX_OPLINE opline, r0 12189 } 12190 if (op1_info & MAY_BE_REF) { 12191 may_throw = 1; 12192 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12193 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 12194 | GET_Z_PTR FCARG2a, FCARG1a 12195 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 12196 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 12197 | jmp >3 12198 |.cold_code 12199 |2: 12200 if (opline->opcode != ZEND_FETCH_DIM_RW) { 12201 | SET_EX_OPLINE opline, r0 12202 } 12203 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 12204 | test r0, r0 12205 | mov FCARG1a, r0 12206 | jne >1 12207 | jmp ->exception_handler_undef 12208 |.code 12209 |1: 12210 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12211 } 12212 12213 if (op1_info & MAY_BE_ARRAY) { 12214 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12215 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12216 } 12217 |3: 12218 | SEPARATE_ARRAY op1_addr, op1_info, 1 12219 } 12220 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 12221 if (op1_info & MAY_BE_ARRAY) { 12222 |.cold_code 12223 |7: 12224 } 12225 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 12226 | CMP_ZVAL_TYPE op1_addr, IS_NULL 12227 | jg >7 12228 } 12229 if (Z_REG(op1_addr) != ZREG_FP) { 12230 | mov T1, Ra(Z_REG(op1_addr)) // save 12231 } 12232 if ((op1_info & MAY_BE_UNDEF) 12233 && opline->opcode == ZEND_FETCH_DIM_RW) { 12234 may_throw = 1; 12235 if (op1_info & MAY_BE_NULL) { 12236 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 12237 } 12238 | mov FCARG1a, opline->op1.var 12239 | EXT_CALL zend_jit_undefined_op_helper, r0 12240 |1: 12241 } 12242 | // ZVAL_ARR(container, zend_new_array(8)); 12243 | EXT_CALL _zend_new_array_0, r0 12244 if (Z_REG(op1_addr) != ZREG_FP) { 12245 | mov Ra(Z_REG(op1_addr)), T1 // restore 12246 } 12247 | SET_ZVAL_LVAL op1_addr, r0 12248 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 12249 | mov FCARG1a, r0 12250 if (op1_info & MAY_BE_ARRAY) { 12251 | jmp >1 12252 |.code 12253 |1: 12254 } 12255 } 12256 12257 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 12258 |6: 12259 if (opline->op2_type == IS_UNUSED) { 12260 may_throw = 1; 12261 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 12262 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 12263 | EXT_CALL zend_hash_next_index_insert, r0 12264 | // if (UNEXPECTED(!var_ptr)) { 12265 | test r0, r0 12266 | jz >1 12267 |.cold_code 12268 |1: 12269 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 12270 | CANNOT_ADD_ELEMENT opline 12271 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF 12272 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 12273 | jmp >8 12274 |.code 12275 | SET_ZVAL_PTR res_addr, r0 12276 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 12277 } else { 12278 uint32_t type; 12279 12280 switch (opline->opcode) { 12281 case ZEND_FETCH_DIM_W: 12282 case ZEND_FETCH_LIST_W: 12283 type = BP_VAR_W; 12284 break; 12285 case ZEND_FETCH_DIM_RW: 12286 may_throw = 1; 12287 type = BP_VAR_RW; 12288 break; 12289 case ZEND_FETCH_DIM_UNSET: 12290 type = BP_VAR_UNSET; 12291 break; 12292 default: 12293 ZEND_UNREACHABLE(); 12294 } 12295 12296 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 12297 may_throw = 1; 12298 } 12299 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 12300 return 0; 12301 } 12302 12303 |8: 12304 | SET_ZVAL_PTR res_addr, r0 12305 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 12306 12307 if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 12308 |.cold_code 12309 |9: 12310 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 12311 | jmp >8 12312 |.code 12313 } 12314 } 12315 } 12316 12317 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 12318 may_throw = 1; 12319 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 12320 |.cold_code 12321 |7: 12322 } 12323 12324 if (opline->opcode != ZEND_FETCH_DIM_RW) { 12325 | SET_EX_OPLINE opline, r0 12326 } 12327 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12328 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12329 } 12330 if (opline->op2_type == IS_UNUSED) { 12331 | xor FCARG2a, FCARG2a 12332 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12333 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12334 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12335 } else { 12336 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12337 } 12338 |.if X64 12339 | LOAD_ZVAL_ADDR CARG3, res_addr 12340 |.else 12341 | sub r4, 12 12342 | PUSH_ZVAL_ADDR res_addr, r0 12343 |.endif 12344 switch (opline->opcode) { 12345 case ZEND_FETCH_DIM_W: 12346 case ZEND_FETCH_LIST_W: 12347 | EXT_CALL zend_jit_fetch_dim_obj_w_helper, r0 12348 break; 12349 case ZEND_FETCH_DIM_RW: 12350 | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, r0 12351 break; 12352// case ZEND_FETCH_DIM_UNSET: 12353// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0 12354// break; 12355 default: 12356 ZEND_UNREACHABLE(); 12357 } 12358 |.if not(X64) 12359 | add r4, 12 12360 |.endif 12361 12362 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 12363 | jmp >8 // END 12364 |.code 12365 } 12366 } 12367 12368#ifdef ZEND_JIT_USE_RC_INFERENCE 12369 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 12370 /* ASSIGN_DIM may increase refcount of the key */ 12371 op2_info |= MAY_BE_RCN; 12372 } 12373#endif 12374 12375 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 12376 && (op2_info & MAY_HAVE_DTOR) 12377 && (op2_info & MAY_BE_RC1)) { 12378 may_throw = 1; 12379 } 12380 |8: 12381 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12382 12383 if (may_throw) { 12384 if (!zend_jit_check_exception(Dst)) { 12385 return 0; 12386 } 12387 } 12388 12389 return 1; 12390} 12391 12392static int zend_jit_isset_isempty_dim(dasm_State **Dst, 12393 const zend_op *opline, 12394 uint32_t op1_info, 12395 zend_jit_addr op1_addr, 12396 bool op1_avoid_refcounting, 12397 uint32_t op2_info, 12398 uint8_t dim_type, 12399 int may_throw, 12400 uint8_t smart_branch_opcode, 12401 uint32_t target_label, 12402 uint32_t target_label2, 12403 const void *exit_addr) 12404{ 12405 zend_jit_addr op2_addr, res_addr; 12406 12407 // TODO: support for empty() ??? 12408 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 12409 12410 op2_addr = OP2_ADDR(); 12411 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12412 12413 if (op1_info & MAY_BE_REF) { 12414 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12415 | ZVAL_DEREF FCARG1a, op1_info 12416 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12417 } 12418 12419 if (op1_info & MAY_BE_ARRAY) { 12420 const void *found_exit_addr = NULL; 12421 const void *not_found_exit_addr = NULL; 12422 12423 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12424 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12425 } 12426 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 12427 if (exit_addr 12428 && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) 12429 && !may_throw 12430 && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) 12431 && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { 12432 if (smart_branch_opcode == ZEND_JMPNZ) { 12433 found_exit_addr = exit_addr; 12434 } else { 12435 not_found_exit_addr = exit_addr; 12436 } 12437 } 12438 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, dim_type, found_exit_addr, not_found_exit_addr, NULL)) { 12439 return 0; 12440 } 12441 12442 if (found_exit_addr) { 12443 |9: 12444 return 1; 12445 } else if (not_found_exit_addr) { 12446 |8: 12447 return 1; 12448 } 12449 } 12450 12451 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 12452 if (op1_info & MAY_BE_ARRAY) { 12453 |.cold_code 12454 |7: 12455 } 12456 12457 if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { 12458 | SET_EX_OPLINE opline, r0 12459 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12460 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12461 } 12462 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12463 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12464 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12465 } else { 12466 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12467 } 12468 | EXT_CALL zend_jit_isset_dim_helper, r0 12469 | test r0, r0 12470 | jz >9 12471 if (op1_info & MAY_BE_ARRAY) { 12472 | jmp >8 12473 |.code 12474 } 12475 } else { 12476 if (op2_info & MAY_BE_UNDEF) { 12477 if (op2_info & MAY_BE_ANY) { 12478 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 12479 } 12480 | SET_EX_OPLINE opline, r0 12481 | mov FCARG1d, opline->op2.var 12482 | EXT_CALL zend_jit_undefined_op_helper, r0 12483 |1: 12484 } 12485 if (op1_info & MAY_BE_ARRAY) { 12486 | jmp >9 12487 |.code 12488 } 12489 } 12490 } 12491 12492#ifdef ZEND_JIT_USE_RC_INFERENCE 12493 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 12494 /* Magic offsetExists() may increase refcount of the key */ 12495 op2_info |= MAY_BE_RCN; 12496 } 12497#endif 12498 12499 if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { 12500 |8: 12501 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12502 if (!op1_avoid_refcounting) { 12503 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12504 } 12505 if (may_throw) { 12506 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 12507 return 0; 12508 } 12509 } 12510 if (!(opline->extended_value & ZEND_ISEMPTY)) { 12511 if (exit_addr) { 12512 if (smart_branch_opcode == ZEND_JMPNZ) { 12513 | jmp &exit_addr 12514 } else { 12515 | jmp >8 12516 } 12517 } else if (smart_branch_opcode) { 12518 if (smart_branch_opcode == ZEND_JMPZ) { 12519 | jmp =>target_label2 12520 } else if (smart_branch_opcode == ZEND_JMPNZ) { 12521 | jmp =>target_label 12522 } else { 12523 ZEND_UNREACHABLE(); 12524 } 12525 } else { 12526 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 12527 | jmp >8 12528 } 12529 } else { 12530 | NIY // TODO: support for empty() 12531 } 12532 } 12533 12534 |9: // not found 12535 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12536 if (!op1_avoid_refcounting) { 12537 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12538 } 12539 if (may_throw) { 12540 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 12541 return 0; 12542 } 12543 } 12544 if (!(opline->extended_value & ZEND_ISEMPTY)) { 12545 if (exit_addr) { 12546 if (smart_branch_opcode == ZEND_JMPZ) { 12547 | jmp &exit_addr 12548 } 12549 } else if (smart_branch_opcode) { 12550 if (smart_branch_opcode == ZEND_JMPZ) { 12551 | jmp =>target_label 12552 } else if (smart_branch_opcode == ZEND_JMPNZ) { 12553 } else { 12554 ZEND_UNREACHABLE(); 12555 } 12556 } else { 12557 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 12558 } 12559 } else { 12560 | NIY // TODO: support for empty() 12561 } 12562 12563 |8: 12564 12565 return 1; 12566} 12567 12568static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 12569{ 12570 zend_jit_addr op1_addr = OP1_ADDR(); 12571 zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); 12572 12573 | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; 12574 | mov FCARG2a, EX->run_time_cache 12575 | mov r0, aword [FCARG2a + opline->extended_value] 12576 | sub r0, 1 12577 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) 12578 | MEM_LOAD_ZTS ecx, dword, executor_globals, symbol_table.nNumUsed, r1 12579 |.if X64 12580 | shl r1, 5 12581 |.else 12582 | imul r1, sizeof(Bucket) 12583 |.endif 12584 | cmp r0, r1 12585 | jae >9 12586 | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); 12587 | MEM_LOAD_OP_ZTS add, r0, aword, executor_globals, symbol_table.arData, r1 12588 | IF_NOT_Z_TYPE r0, IS_REFERENCE, >9 12589 | // (EXPECTED(p->key == varname)) 12590 | ADDR_CMP aword [r0 + offsetof(Bucket, key)], varname, r1 12591 | jne >9 12592 | GET_Z_PTR r0, r0 12593 | GC_ADDREF r0 12594 |1: 12595 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 12596 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12597 | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) 12598 | IF_ZVAL_REFCOUNTED op1_addr, >2 12599 |.cold_code 12600 |2: 12601 } 12602 | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); 12603 | GET_ZVAL_PTR FCARG1a, op1_addr 12604 | // ZVAL_REF(variable_ptr, ref) 12605 | SET_ZVAL_PTR op1_addr, r0 12606 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 12607 | // if (GC_DELREF(garbage) == 0) 12608 | GC_DELREF FCARG1a 12609 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 12610 | jnz >3 12611 } else { 12612 | jnz >5 12613 } 12614 | ZVAL_DTOR_FUNC op1_info, opline 12615 | jmp >5 12616 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 12617 |3: 12618 | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) 12619 | IF_GC_MAY_NOT_LEAK FCARG1a, >5 12620 | EXT_CALL gc_possible_root, r1 12621 | jmp >5 12622 } 12623 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12624 |.code 12625 } 12626 } 12627 12628 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12629 | // ZVAL_REF(variable_ptr, ref) 12630 | SET_ZVAL_PTR op1_addr, r0 12631 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 12632 } 12633 |5: 12634 //END of handler 12635 12636 |.cold_code 12637 |9: 12638 | LOAD_ADDR FCARG1a, (ptrdiff_t)varname 12639 if (opline->extended_value) { 12640 | add FCARG2a, opline->extended_value 12641 } 12642 | EXT_CALL zend_jit_fetch_global_helper, r0 12643 | jmp <1 12644 |.code 12645 12646 return 1; 12647} 12648 12649static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception) 12650{ 12651 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12652 bool in_cold = 0; 12653 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 12654 zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1 : ZREG_R0; 12655 12656 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12657 && JIT_G(current_frame) 12658 && JIT_G(current_frame)->prev) { 12659 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 12660 uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)); 12661 12662 if (type != IS_UNKNOWN && (type_mask & (1u << type))) { 12663 return 1; 12664 } 12665 } 12666 12667 if (ZEND_ARG_SEND_MODE(arg_info)) { 12668 if (opline->opcode == ZEND_RECV_INIT) { 12669 | LOAD_ZVAL_ADDR Ra(tmp_reg), res_addr 12670 | ZVAL_DEREF Ra(tmp_reg), MAY_BE_REF 12671 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); 12672 } else { 12673 | GET_ZVAL_PTR Ra(tmp_reg), res_addr 12674 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); 12675 } 12676 } 12677 12678 if (type_mask != 0) { 12679 if (is_power_of_two(type_mask)) { 12680 uint32_t type_code = concrete_type(type_mask); 12681 | IF_NOT_ZVAL_TYPE res_addr, type_code, >1 12682 } else { 12683 | mov edx, 1 12684 | mov cl, byte [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)+offsetof(zval, u1.v.type)] 12685 | shl edx, cl 12686 | test edx, type_mask 12687 | je >1 12688 } 12689 12690 |.cold_code 12691 |1: 12692 12693 in_cold = 1; 12694 } 12695 12696 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 12697 | LOAD_ZVAL_ADDR FCARG1a, res_addr 12698 } 12699 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12700 | SET_EX_OPLINE opline, r0 12701 } else { 12702 | ADDR_STORE aword EX->opline, opline, r0 12703 } 12704 | LOAD_ADDR FCARG2a, (ptrdiff_t)arg_info 12705 | EXT_CALL zend_jit_verify_arg_slow, r0 12706 12707 if (check_exception) { 12708 | test al, al 12709 if (in_cold) { 12710 | jnz >1 12711 | jmp ->exception_handler 12712 |.code 12713 |1: 12714 } else { 12715 | jz ->exception_handler 12716 } 12717 } else if (in_cold) { 12718 | jmp >1 12719 |.code 12720 |1: 12721 } 12722 12723 return 1; 12724} 12725 12726static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) 12727{ 12728 uint32_t arg_num = opline->op1.num; 12729 zend_arg_info *arg_info = NULL; 12730 12731 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12732 if (EXPECTED(arg_num <= op_array->num_args)) { 12733 arg_info = &op_array->arg_info[arg_num-1]; 12734 } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { 12735 arg_info = &op_array->arg_info[op_array->num_args]; 12736 } 12737 if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { 12738 arg_info = NULL; 12739 } 12740 } 12741 12742 if (arg_info || (opline+1)->opcode != ZEND_RECV) { 12743 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12744 if (!JIT_G(current_frame) || 12745 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 || 12746 arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 12747 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12748 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12749 12750 if (!exit_addr) { 12751 return 0; 12752 } 12753 | cmp dword EX->This.u2.num_args, arg_num 12754 | jb &exit_addr 12755 } 12756 } else { 12757 | cmp dword EX->This.u2.num_args, arg_num 12758 | jb >1 12759 |.cold_code 12760 |1: 12761 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12762 | SET_EX_OPLINE opline, r0 12763 } else { 12764 | ADDR_STORE aword EX->opline, opline, r0 12765 } 12766 | mov FCARG1a, FP 12767 | EXT_CALL zend_missing_arg_error, r0 12768 | jmp ->exception_handler 12769 |.code 12770 } 12771 } 12772 12773 if (arg_info) { 12774 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { 12775 return 0; 12776 } 12777 } 12778 12779 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12780 if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { 12781 | LOAD_IP_ADDR (opline + 1) 12782 zend_jit_set_last_valid_opline(opline + 1); 12783 } 12784 } 12785 12786 return 1; 12787} 12788 12789static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw) 12790{ 12791 uint32_t arg_num = opline->op1.num; 12792 zval *zv = RT_CONSTANT(opline, opline->op2); 12793 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12794 12795 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12796 && JIT_G(current_frame) 12797 && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) { 12798 if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 12799 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0 12800 if (Z_REFCOUNTED_P(zv)) { 12801 | ADDREF_CONST zv, r0 12802 } 12803 } 12804 } else { 12805 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 12806 (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 12807 | cmp dword EX->This.u2.num_args, arg_num 12808 | jae >5 12809 } 12810 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0 12811 if (Z_REFCOUNTED_P(zv)) { 12812 | ADDREF_CONST zv, r0 12813 } 12814 } 12815 12816 if (Z_CONSTANT_P(zv)) { 12817 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12818 | SET_EX_OPLINE opline, r0 12819 } else { 12820 | ADDR_STORE aword EX->opline, opline, r0 12821 } 12822 | LOAD_ZVAL_ADDR FCARG1a, res_addr 12823 | mov r0, EX->func 12824 | mov FCARG2a, [r0 + offsetof(zend_op_array, scope)] 12825 | .if X64 12826 | EXT_CALL zval_update_constant_ex, r0 12827 | .else 12828 | EXT_CALL zval_update_constant_ex, r0 12829 | .endif 12830 | test al, al 12831 | jnz >1 12832 |.cold_code 12833 |1: 12834 | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline 12835 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF 12836 | jmp ->exception_handler 12837 |.code 12838 } 12839 12840 |5: 12841 12842 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12843 do { 12844 zend_arg_info *arg_info; 12845 12846 if (arg_num <= op_array->num_args) { 12847 arg_info = &op_array->arg_info[arg_num-1]; 12848 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { 12849 arg_info = &op_array->arg_info[op_array->num_args]; 12850 } else { 12851 break; 12852 } 12853 if (!ZEND_TYPE_IS_SET(arg_info->type)) { 12854 break; 12855 } 12856 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { 12857 return 0; 12858 } 12859 } while (0); 12860 } 12861 12862 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12863 if (is_last) { 12864 | LOAD_IP_ADDR (opline + 1) 12865 zend_jit_set_last_valid_opline(opline + 1); 12866 } 12867 } 12868 12869 return 1; 12870} 12871 12872static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) 12873{ 12874 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 12875 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12876 12877 if (!exit_addr) { 12878 return 0; 12879 } 12880 12881 |.if X64 12882 || if (!IS_SIGNED_32BIT(ce)) { 12883 | mov64 r0, ((ptrdiff_t)ce) 12884 | cmp aword [FCARG1a + offsetof(zend_object, ce)], r0 12885 || } else { 12886 | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce 12887 || } 12888 |.else 12889 | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce 12890 |.endif 12891 | jne &exit_addr 12892 12893 return 1; 12894} 12895 12896static int zend_jit_fetch_obj(dasm_State **Dst, 12897 const zend_op *opline, 12898 const zend_op_array *op_array, 12899 zend_ssa *ssa, 12900 const zend_ssa_op *ssa_op, 12901 uint32_t op1_info, 12902 zend_jit_addr op1_addr, 12903 bool op1_indirect, 12904 zend_class_entry *ce, 12905 bool ce_is_instanceof, 12906 bool on_this, 12907 bool delayed_fetch_this, 12908 bool op1_avoid_refcounting, 12909 zend_class_entry *trace_ce, 12910 uint8_t prop_type, 12911 int may_throw) 12912{ 12913 zval *member; 12914 zend_property_info *prop_info; 12915 bool may_be_dynamic = 1; 12916 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12917 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 12918 zend_jit_addr prop_addr; 12919 uint32_t res_info = RES_INFO(); 12920 bool type_loaded = 0; 12921 12922 ZEND_ASSERT(opline->op2_type == IS_CONST); 12923 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 12924 12925 member = RT_CONSTANT(opline, opline->op2); 12926 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 12927 prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); 12928 12929 if (on_this) { 12930 | GET_ZVAL_PTR FCARG1a, this_addr 12931 } else { 12932 if (opline->op1_type == IS_VAR 12933 && opline->opcode == ZEND_FETCH_OBJ_W 12934 && (op1_info & MAY_BE_INDIRECT) 12935 && Z_REG(op1_addr) == ZREG_FP) { 12936 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12937 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 12938 | GET_Z_PTR FCARG1a, FCARG1a 12939 |1: 12940 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12941 } 12942 if (op1_info & MAY_BE_REF) { 12943 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12944 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12945 } 12946 | ZVAL_DEREF FCARG1a, op1_info 12947 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12948 } 12949 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 12950 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12951 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12952 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12953 12954 if (!exit_addr) { 12955 return 0; 12956 } 12957 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 12958 } else { 12959 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7 12960 } 12961 } 12962 | GET_ZVAL_PTR FCARG1a, op1_addr 12963 } 12964 12965 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12966 prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename); 12967 if (prop_info) { 12968 ce = trace_ce; 12969 ce_is_instanceof = 0; 12970 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 12971 if (on_this && JIT_G(current_frame) 12972 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 12973 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 12974 } else if (zend_jit_class_guard(Dst, opline, ce)) { 12975 if (on_this && JIT_G(current_frame)) { 12976 JIT_G(current_frame)->ce = ce; 12977 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 12978 } 12979 } else { 12980 return 0; 12981 } 12982 if (ssa->var_info && ssa_op->op1_use >= 0) { 12983 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 12984 ssa->var_info[ssa_op->op1_use].ce = ce; 12985 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 12986 } 12987 } 12988 } 12989 } 12990 12991 if (!prop_info) { 12992 | mov r0, EX->run_time_cache 12993 | mov r2, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)] 12994 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 12995 | jne >5 12996 | mov r0, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)] 12997 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array); 12998 if (may_be_dynamic) { 12999 | test r0, r0 13000 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13001 | jl >5 13002 } else { 13003 | jl >8 // dynamic property 13004 } 13005 } 13006 | mov edx, dword [FCARG1a + r0 + 8] 13007 | IF_UNDEF dl, >5 13008 | add FCARG1a, r0 13009 type_loaded = 1; 13010 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13011 if (opline->opcode == ZEND_FETCH_OBJ_W 13012 && (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) { 13013 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 13014 13015 | mov r0, EX->run_time_cache 13016 | mov FCARG2a, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2] 13017 | test FCARG2a, FCARG2a 13018 | jnz >1 13019 |.cold_code 13020 |1: 13021 | test dword [FCARG2a + offsetof(zend_property_info, flags)], ZEND_ACC_READONLY 13022 if (flags) { 13023 | jz >3 13024 } else { 13025 | jz >4 13026 } 13027 | IF_NOT_Z_TYPE FCARG1a, IS_OBJECT, >2 13028 | GET_Z_PTR r0, FCARG1a 13029 | GC_ADDREF r0 13030 | SET_ZVAL_PTR res_addr, r0 13031 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX 13032 | jmp >9 13033 |2: 13034 | mov eax, dword [FCARG1a + offsetof(zval, u2.extra)] 13035 | test eax, IS_PROP_REINITABLE 13036 | jz >6 13037 | and eax, ~IS_PROP_REINITABLE 13038 | mov dword [FCARG1a + offsetof(zval, u2.extra)], eax 13039 if (flags) { 13040 | jmp >3 13041 } else { 13042 | jmp >4 13043 } 13044 |6: 13045 | mov FCARG1a, FCARG2a 13046 | SET_EX_OPLINE opline, r0 13047 | EXT_CALL zend_readonly_property_modification_error, r0 13048 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 13049 | jmp >9 13050 |3: 13051 if (flags == ZEND_FETCH_DIM_WRITE) { 13052 | SET_EX_OPLINE opline, r0 13053 | EXT_CALL zend_jit_check_array_promotion, r0 13054 | jmp >9 13055 } else if (flags == ZEND_FETCH_REF) { 13056 |.if X64 13057 | LOAD_ZVAL_ADDR CARG3, res_addr 13058 |.else 13059 | sub r4, 12 13060 | PUSH_ZVAL_ADDR res_addr, r0 13061 |.endif 13062 | EXT_CALL zend_jit_create_typed_ref, r0 13063 |.if not(X64) 13064 | add r4, 12 13065 |.endif 13066 | jmp >9 13067 } else { 13068 ZEND_ASSERT(flags == 0); 13069 } 13070 |.code 13071 |4: 13072 } 13073 } else { 13074 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13075 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13076 if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { 13077 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 13078 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13079 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13080 13081 if (!exit_addr) { 13082 return 0; 13083 } 13084 type_loaded = 1; 13085 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13086 | IF_UNDEF dl, &exit_addr 13087 } 13088 } else { 13089 type_loaded = 1; 13090 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13091 | IF_UNDEF dl, >5 13092 } 13093 if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) { 13094 if (!type_loaded) { 13095 type_loaded = 1; 13096 | mov edx, dword [FCARG1a + prop_info->offset + offsetof(zval, u1.type_info)] 13097 } 13098 | IF_NOT_TYPE dl, IS_OBJECT, >4 13099 | GET_ZVAL_PTR r0, prop_addr 13100 | GC_ADDREF r0 13101 | SET_ZVAL_PTR res_addr, r0 13102 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX 13103 | jmp >9 13104 |.cold_code 13105 |4: 13106 | mov eax, dword [FCARG1a + prop_info->offset + offsetof(zval, u2.extra)] 13107 | test eax, IS_PROP_REINITABLE 13108 | jz >6 13109 | and eax, ~IS_PROP_REINITABLE 13110 | mov dword [FCARG1a + prop_info->offset + offsetof(zval, u2.extra)], eax 13111 | jmp >4 13112 |6: 13113 | LOAD_ADDR FCARG1a, prop_info 13114 | SET_EX_OPLINE opline, r0 13115 | EXT_CALL zend_readonly_property_modification_error, r0 13116 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 13117 | jmp >9 13118 |.code 13119 |4: 13120 } 13121 if (opline->opcode == ZEND_FETCH_OBJ_W 13122 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) 13123 && ZEND_TYPE_IS_SET(prop_info->type)) { 13124 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 13125 13126 if (flags == ZEND_FETCH_DIM_WRITE) { 13127 if ((ZEND_TYPE_FULL_MASK(prop_info->type) & MAY_BE_ARRAY) == 0) { 13128 if (!type_loaded) { 13129 type_loaded = 1; 13130 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13131 } 13132 | cmp dl, IS_FALSE 13133 | jle >1 13134 |.cold_code 13135 |1: 13136 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13137 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13138 } 13139 | LOAD_ADDR FCARG2a, prop_info 13140 | SET_EX_OPLINE opline, r0 13141 | EXT_CALL zend_jit_check_array_promotion, r0 13142 | jmp >9 13143 |.code 13144 } 13145 } else if (flags == ZEND_FETCH_REF) { 13146 if (!type_loaded) { 13147 type_loaded = 1; 13148 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13149 } 13150 | IF_TYPE dl, IS_REFERENCE, >1 13151 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13152 | LOAD_ADDR FCARG2a, prop_info 13153 } else { 13154 int prop_info_offset = 13155 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13156 13157 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13158 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13159 | mov FCARG2a, aword[r0 + prop_info_offset] 13160 } 13161 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13162 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13163 } 13164 |.if X64 13165 | LOAD_ZVAL_ADDR CARG3, res_addr 13166 |.else 13167 | sub r4, 12 13168 | PUSH_ZVAL_ADDR res_addr, r0 13169 |.endif 13170 | EXT_CALL zend_jit_create_typed_ref, r0 13171 |.if not(X64) 13172 | add r4, 12 13173 |.endif 13174 | jmp >9 13175 |1: 13176 } else { 13177 ZEND_UNREACHABLE(); 13178 } 13179 } 13180 } 13181 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13182 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13183 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13184 } 13185 | SET_ZVAL_PTR res_addr, FCARG1a 13186 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 13187 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { 13188 ssa->var_info[ssa_op->result_def].indirect_reference = 1; 13189 } 13190 } else { 13191 bool result_avoid_refcounting = 0; 13192 13193 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { 13194 uint32_t flags = 0; 13195 uint32_t old_info; 13196 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 13197 int32_t exit_point; 13198 const void *exit_addr; 13199 uint8_t type; 13200 zend_jit_addr val_addr = prop_addr; 13201 13202 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 13203 && !delayed_fetch_this 13204 && !op1_avoid_refcounting) { 13205 flags = ZEND_JIT_EXIT_FREE_OP1; 13206 } 13207 13208 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 13209 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 13210 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 13211 && (ssa_op+1)->op1_use == ssa_op->result_def 13212 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 13213 result_avoid_refcounting = 1; 13214 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 13215 } 13216 13217 type = concrete_type(res_info); 13218 13219 if (prop_type != IS_UNKNOWN 13220 && prop_type != IS_UNDEF 13221 && prop_type != IS_REFERENCE 13222 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) { 13223 exit_point = zend_jit_trace_get_exit_point(opline, 0); 13224 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13225 if (!exit_addr) { 13226 return 0; 13227 } 13228 } else { 13229 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 13230 | LOAD_ZVAL_ADDR r0, prop_addr 13231 if (op1_avoid_refcounting) { 13232 SET_STACK_REG(JIT_G(current_frame)->stack, 13233 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 13234 } 13235 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 13236 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 13237 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 13238 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 13239 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 13240 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13241 if (!exit_addr) { 13242 return 0; 13243 } 13244 13245 if (!type_loaded) { 13246 type_loaded = 1; 13247 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13248 } 13249 | // ZVAL_DEREF() 13250 | IF_NOT_TYPE dl, IS_REFERENCE, >1 13251 | GET_Z_PTR r0, r0 13252 | add r0, offsetof(zend_reference, val) 13253 | GET_ZVAL_TYPE_INFO edx, val_addr 13254 } 13255 res_info &= ~MAY_BE_GUARD; 13256 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 13257 if (type < IS_STRING) { 13258 |1: 13259 if (type_loaded) { 13260 | IF_NOT_TYPE dl, type, &exit_addr 13261 } else { 13262 | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr 13263 } 13264 } else { 13265 if (!type_loaded) { 13266 type_loaded = 1; 13267 | GET_ZVAL_TYPE_INFO edx, val_addr 13268 } 13269 |1: 13270 | IF_NOT_TYPE dl, type, &exit_addr 13271 } 13272 | // ZVAL_COPY 13273 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 13274 if (type < IS_STRING) { 13275 if (Z_REG(res_addr) != ZREG_FP || 13276 JIT_G(current_frame) == NULL || 13277 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 13278 | SET_ZVAL_TYPE_INFO res_addr, type 13279 } 13280 } else { 13281 | SET_ZVAL_TYPE_INFO res_addr, edx 13282 if (!result_avoid_refcounting) { 13283 | TRY_ADDREF res_info, dh, r1 13284 } 13285 } 13286 } else { 13287 if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_R2)) { 13288 return 0; 13289 } 13290 } 13291 } 13292 13293 if (op1_avoid_refcounting) { 13294 SET_STACK_REG(JIT_G(current_frame)->stack, 13295 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 13296 } 13297 13298 |.cold_code 13299 13300 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { 13301 |5: 13302 | SET_EX_OPLINE opline, r0 13303 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13304 | EXT_CALL zend_jit_fetch_obj_w_slow, r0 13305 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13306 | EXT_CALL zend_jit_fetch_obj_r_slow, r0 13307 } else { 13308 | EXT_CALL zend_jit_fetch_obj_is_slow, r0 13309 } 13310 | jmp >9 13311 } 13312 13313 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 13314 |7: 13315 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13316 | SET_EX_OPLINE opline, r0 13317 if (opline->opcode != ZEND_FETCH_OBJ_W 13318 && (op1_info & MAY_BE_UNDEF)) { 13319 zend_jit_addr orig_op1_addr = OP1_ADDR(); 13320 13321 if (op1_info & MAY_BE_ANY) { 13322 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 13323 } 13324 | mov FCARG1d, opline->op1.var 13325 | EXT_CALL zend_jit_undefined_op_helper, r0 13326 |1: 13327 | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr 13328 } else if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13329 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13330 } 13331 | LOAD_ADDR FCARG2a, Z_STRVAL_P(member) 13332 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13333 | EXT_CALL zend_jit_invalid_property_write, r0 13334 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 13335 } else { 13336 | EXT_CALL zend_jit_invalid_property_read, r0 13337 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 13338 } 13339 | jmp >9 13340 } else { 13341 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 13342 | jmp >9 13343 } 13344 } 13345 13346 if (!prop_info 13347 && may_be_dynamic 13348 && opline->opcode != ZEND_FETCH_OBJ_W) { 13349 |8: 13350 | mov FCARG2a, r0 13351 | SET_EX_OPLINE opline, r0 13352 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13353 | EXT_CALL zend_jit_fetch_obj_r_dynamic, r0 13354 } else { 13355 | EXT_CALL zend_jit_fetch_obj_is_dynamic, r0 13356 } 13357 | jmp >9 13358 } 13359 13360 |.code; 13361 |9: // END 13362 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13363 if (opline->op1_type == IS_VAR 13364 && opline->opcode == ZEND_FETCH_OBJ_W 13365 && (op1_info & MAY_BE_RC1)) { 13366 zend_jit_addr orig_op1_addr = OP1_ADDR(); 13367 13368 | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1 13369 | GET_ZVAL_PTR FCARG1a, orig_op1_addr 13370 | GC_DELREF FCARG1a 13371 | jnz >1 13372 | SET_EX_OPLINE opline, r0 13373 | EXT_CALL zend_jit_extract_helper, r0 13374 |1: 13375 } else if (!op1_avoid_refcounting) { 13376 if (on_this) { 13377 op1_info &= ~MAY_BE_RC1; 13378 } 13379 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 13380 } 13381 } 13382 13383 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 13384 && prop_info 13385 && (opline->opcode != ZEND_FETCH_OBJ_W || 13386 !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) || 13387 !ZEND_TYPE_IS_SET(prop_info->type)) 13388 && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) { 13389 may_throw = 0; 13390 } 13391 13392 if (may_throw) { 13393 if (!zend_jit_check_exception(Dst)) { 13394 return 0; 13395 } 13396 } 13397 13398 return 1; 13399} 13400 13401static int zend_jit_incdec_obj(dasm_State **Dst, 13402 const zend_op *opline, 13403 const zend_op_array *op_array, 13404 zend_ssa *ssa, 13405 const zend_ssa_op *ssa_op, 13406 uint32_t op1_info, 13407 zend_jit_addr op1_addr, 13408 bool op1_indirect, 13409 zend_class_entry *ce, 13410 bool ce_is_instanceof, 13411 bool on_this, 13412 bool delayed_fetch_this, 13413 zend_class_entry *trace_ce, 13414 uint8_t prop_type) 13415{ 13416 zval *member; 13417 zend_string *name; 13418 zend_property_info *prop_info; 13419 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13420 zend_jit_addr res_addr = 0; 13421 zend_jit_addr prop_addr; 13422 bool needs_slow_path = 0; 13423 bool use_prop_guard = 0; 13424 bool may_throw = 0; 13425 uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0; 13426 13427 ZEND_ASSERT(opline->op2_type == IS_CONST); 13428 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13429 13430 if (opline->result_type != IS_UNUSED) { 13431 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 13432 } 13433 13434 member = RT_CONSTANT(opline, opline->op2); 13435 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13436 name = Z_STR_P(member); 13437 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13438 13439 if (on_this) { 13440 | GET_ZVAL_PTR FCARG1a, this_addr 13441 } else { 13442 if (opline->op1_type == IS_VAR 13443 && (op1_info & MAY_BE_INDIRECT) 13444 && Z_REG(op1_addr) == ZREG_FP) { 13445 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13446 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13447 | GET_Z_PTR FCARG1a, FCARG1a 13448 |1: 13449 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13450 } 13451 if (op1_info & MAY_BE_REF) { 13452 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13453 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13454 } 13455 | ZVAL_DEREF FCARG1a, op1_info 13456 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13457 } 13458 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13459 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13460 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13461 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13462 13463 if (!exit_addr) { 13464 return 0; 13465 } 13466 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13467 } else { 13468 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 13469 |.cold_code 13470 |1: 13471 | SET_EX_OPLINE opline, r0 13472 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13473 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13474 } 13475 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 13476 | EXT_CALL zend_jit_invalid_property_incdec, r0 13477 | jmp ->exception_handler 13478 |.code 13479 } 13480 } 13481 | GET_ZVAL_PTR FCARG1a, op1_addr 13482 } 13483 13484 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13485 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13486 if (prop_info) { 13487 ce = trace_ce; 13488 ce_is_instanceof = 0; 13489 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13490 if (on_this && JIT_G(current_frame) 13491 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13492 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13493 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13494 if (on_this && JIT_G(current_frame)) { 13495 JIT_G(current_frame)->ce = ce; 13496 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13497 } 13498 } else { 13499 return 0; 13500 } 13501 if (ssa->var_info && ssa_op->op1_use >= 0) { 13502 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13503 ssa->var_info[ssa_op->op1_use].ce = ce; 13504 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13505 } 13506 if (ssa->var_info && ssa_op->op1_def >= 0) { 13507 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13508 ssa->var_info[ssa_op->op1_def].ce = ce; 13509 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13510 } 13511 } 13512 } 13513 } 13514 13515 use_prop_guard = (prop_type != IS_UNKNOWN 13516 && prop_type != IS_UNDEF 13517 && prop_type != IS_REFERENCE 13518 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 13519 13520 if (!prop_info) { 13521 needs_slow_path = 1; 13522 13523 | mov r0, EX->run_time_cache 13524 | mov r2, aword [r0 + opline->extended_value] 13525 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13526 | jne >7 13527 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 13528 | cmp aword [r0 + opline->extended_value + sizeof(void*) * 2], 0 13529 | jnz >7 13530 } 13531 | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] 13532 | test r0, r0 13533 | jl >7 13534 if (!use_prop_guard) { 13535 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 13536 } 13537 | add FCARG1a, r0 13538 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13539 } else { 13540 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13541 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 13542 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13543 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13544 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13545 13546 if (!exit_addr) { 13547 return 0; 13548 } 13549 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 13550 } else { 13551 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 13552 needs_slow_path = 1; 13553 } 13554 } 13555 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13556 may_throw = 1; 13557 | SET_EX_OPLINE opline, r0 13558 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13559 | LOAD_ADDR FCARG2a, prop_info 13560 } else { 13561 int prop_info_offset = 13562 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13563 13564 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13565 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13566 | mov FCARG2a, aword[r0 + prop_info_offset] 13567 } 13568 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13569 if (opline->result_type == IS_UNUSED) { 13570 switch (opline->opcode) { 13571 case ZEND_PRE_INC_OBJ: 13572 case ZEND_POST_INC_OBJ: 13573 | EXT_CALL zend_jit_inc_typed_prop, r0 13574 break; 13575 case ZEND_PRE_DEC_OBJ: 13576 case ZEND_POST_DEC_OBJ: 13577 | EXT_CALL zend_jit_dec_typed_prop, r0 13578 break; 13579 default: 13580 ZEND_UNREACHABLE(); 13581 } 13582 } else { 13583 |.if X64 13584 | LOAD_ZVAL_ADDR CARG3, res_addr 13585 |.else 13586 | sub r4, 12 13587 | PUSH_ZVAL_ADDR res_addr, r0 13588 |.endif 13589 switch (opline->opcode) { 13590 case ZEND_PRE_INC_OBJ: 13591 | EXT_CALL zend_jit_pre_inc_typed_prop, r0 13592 break; 13593 case ZEND_PRE_DEC_OBJ: 13594 | EXT_CALL zend_jit_pre_dec_typed_prop, r0 13595 break; 13596 case ZEND_POST_INC_OBJ: 13597 | EXT_CALL zend_jit_post_inc_typed_prop, r0 13598 break; 13599 case ZEND_POST_DEC_OBJ: 13600 | EXT_CALL zend_jit_post_dec_typed_prop, r0 13601 break; 13602 default: 13603 ZEND_UNREACHABLE(); 13604 } 13605 |.if not(X64) 13606 | add r4, 12 13607 |.endif 13608 } 13609 } 13610 } 13611 13612 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13613 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13614 zend_jit_addr var_addr = prop_addr; 13615 13616 if (use_prop_guard) { 13617 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 13618 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13619 if (!exit_addr) { 13620 return 0; 13621 } 13622 13623 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr 13624 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 13625 } 13626 13627 if (var_info & MAY_BE_REF) { 13628 may_throw = 1; 13629 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13630 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13631 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13632 } 13633 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 13634 | GET_ZVAL_PTR FCARG1a, var_addr 13635 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 13636 | jnz >1 13637 | lea FCARG1a, aword [FCARG1a + offsetof(zend_reference, val)] 13638 |.cold_code 13639 |1: 13640 if (opline) { 13641 | SET_EX_OPLINE opline, r0 13642 } 13643 if (opline->result_type == IS_UNUSED) { 13644 | xor FCARG2a, FCARG2a 13645 } else { 13646 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13647 } 13648 switch (opline->opcode) { 13649 case ZEND_PRE_INC_OBJ: 13650 | EXT_CALL zend_jit_pre_inc_typed_ref, r0 13651 break; 13652 case ZEND_PRE_DEC_OBJ: 13653 | EXT_CALL zend_jit_pre_dec_typed_ref, r0 13654 break; 13655 case ZEND_POST_INC_OBJ: 13656 | EXT_CALL zend_jit_post_inc_typed_ref, r0 13657 break; 13658 case ZEND_POST_DEC_OBJ: 13659 | EXT_CALL zend_jit_post_dec_typed_ref, r0 13660 break; 13661 default: 13662 ZEND_UNREACHABLE(); 13663 } 13664 | jmp >9 13665 |.code 13666 |2: 13667 } 13668 13669 if (var_info & MAY_BE_LONG) { 13670 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 13671 | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2 13672 } 13673 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 13674 if (opline->result_type != IS_UNUSED) { 13675 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R1, ZREG_R2 13676 } 13677 } 13678 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13679 | LONG_OP_WITH_32BIT_CONST add, var_addr, Z_L(1) 13680 } else { 13681 | LONG_OP_WITH_32BIT_CONST sub, var_addr, Z_L(1) 13682 } 13683 | jo >3 13684 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { 13685 if (opline->result_type != IS_UNUSED) { 13686 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R0, ZREG_R2 13687 } 13688 } 13689 |.cold_code 13690 } 13691 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 13692 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13693 may_throw = 1; 13694 } 13695 if (var_info & MAY_BE_LONG) { 13696 |2: 13697 } 13698 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 13699 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13700 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13701 } 13702 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 13703 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_R0, ZREG_R2 13704 | TRY_ADDREF MAY_BE_ANY, ah, r2 13705 } 13706 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13707 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13708 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13709 | EXT_CALL zend_jit_pre_inc, r0 13710 } else { 13711 | EXT_CALL increment_function, r0 13712 } 13713 } else { 13714 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13715 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13716 | EXT_CALL zend_jit_pre_dec, r0 13717 } else { 13718 | EXT_CALL decrement_function, r0 13719 } 13720 } 13721 if (var_info & MAY_BE_LONG) { 13722 | jmp >4 13723 } 13724 } 13725 if (var_info & MAY_BE_LONG) { 13726 |3: 13727 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13728 |.if X64 13729 | mov64 rax, 0x43e0000000000000 13730 | SET_ZVAL_LVAL var_addr, rax 13731 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13732 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13733 | SET_ZVAL_LVAL res_addr, rax 13734 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13735 } 13736 |.else 13737 | SET_ZVAL_LVAL var_addr, 0 13738 | SET_ZVAL_W2 var_addr, 0x41e00000 13739 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13740 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13741 | SET_ZVAL_LVAL res_addr, 0 13742 | SET_ZVAL_W2 res_addr, 0x41e00000 13743 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13744 } 13745 |.endif 13746 } else { 13747 |.if X64 13748 | mov64 rax, 0xc3e0000000000000 13749 | SET_ZVAL_LVAL var_addr, rax 13750 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13751 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13752 | SET_ZVAL_LVAL res_addr, rax 13753 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13754 } 13755 |.else 13756 | SET_ZVAL_LVAL var_addr, 0x00200000 13757 | SET_ZVAL_W2 var_addr, 0xc1e00000 13758 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13759 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13760 | SET_ZVAL_LVAL res_addr, 0x00200000 13761 | SET_ZVAL_W2 res_addr, 0xc1e00000 13762 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13763 } 13764 |.endif 13765 } 13766 if (opline->result_type != IS_UNUSED 13767 && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) 13768 && prop_info 13769 && !ZEND_TYPE_IS_SET(prop_info->type) 13770 && (res_info & MAY_BE_GUARD) 13771 && (res_info & MAY_BE_LONG)) { 13772 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 13773 uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 13774 int32_t exit_point; 13775 const void *exit_addr; 13776 13777 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 13778 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 13779 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13780 if (!exit_addr) { 13781 return 0; 13782 } 13783 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 13784 ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD; 13785 | jmp &exit_addr 13786 |.code 13787 } else { 13788 | jmp >4 13789 |.code 13790 |4: 13791 } 13792 } 13793 } 13794 13795 if (needs_slow_path) { 13796 may_throw = 1; 13797 |.cold_code 13798 |7: 13799 | SET_EX_OPLINE opline, r0 13800 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13801 | LOAD_ADDR FCARG2a, name 13802 |.if X64 13803 | mov CARG3, EX->run_time_cache 13804 | add CARG3, opline->extended_value 13805 if (opline->result_type == IS_UNUSED) { 13806 | xor CARG4, CARG4 13807 } else { 13808 | LOAD_ZVAL_ADDR CARG4, res_addr 13809 } 13810 |.else 13811 | sub r4, 8 13812 if (opline->result_type == IS_UNUSED) { 13813 | push 0 13814 } else { 13815 | PUSH_ZVAL_ADDR res_addr, r0 13816 } 13817 | mov r0, EX->run_time_cache 13818 | add r0, opline->extended_value 13819 | push r0 13820 |.endif 13821 13822 switch (opline->opcode) { 13823 case ZEND_PRE_INC_OBJ: 13824 | EXT_CALL zend_jit_pre_inc_obj_helper, r0 13825 break; 13826 case ZEND_PRE_DEC_OBJ: 13827 | EXT_CALL zend_jit_pre_dec_obj_helper, r0 13828 break; 13829 case ZEND_POST_INC_OBJ: 13830 | EXT_CALL zend_jit_post_inc_obj_helper, r0 13831 break; 13832 case ZEND_POST_DEC_OBJ: 13833 | EXT_CALL zend_jit_post_dec_obj_helper, r0 13834 break; 13835 default: 13836 ZEND_UNREACHABLE(); 13837 } 13838 13839 |.if not(X64) 13840 | add r4, 8 13841 |.endif 13842 13843 | jmp >9 13844 |.code 13845 } 13846 13847 |9: 13848 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13849 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 13850 may_throw = 1; 13851 } 13852 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 13853 } 13854 13855 if (may_throw) { 13856 if (!zend_jit_check_exception(Dst)) { 13857 return 0; 13858 } 13859 } 13860 13861 return 1; 13862} 13863 13864static int zend_jit_assign_obj_op(dasm_State **Dst, 13865 const zend_op *opline, 13866 const zend_op_array *op_array, 13867 zend_ssa *ssa, 13868 const zend_ssa_op *ssa_op, 13869 uint32_t op1_info, 13870 zend_jit_addr op1_addr, 13871 uint32_t val_info, 13872 zend_ssa_range *val_range, 13873 bool op1_indirect, 13874 zend_class_entry *ce, 13875 bool ce_is_instanceof, 13876 bool on_this, 13877 bool delayed_fetch_this, 13878 zend_class_entry *trace_ce, 13879 uint8_t prop_type) 13880{ 13881 zval *member; 13882 zend_string *name; 13883 zend_property_info *prop_info; 13884 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13885 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13886 zend_jit_addr prop_addr; 13887 bool needs_slow_path = 0; 13888 bool use_prop_guard = 0; 13889 bool may_throw = 0; 13890 binary_op_type binary_op = get_binary_op(opline->extended_value); 13891 13892 ZEND_ASSERT(opline->op2_type == IS_CONST); 13893 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13894 ZEND_ASSERT(opline->result_type == IS_UNUSED); 13895 13896 member = RT_CONSTANT(opline, opline->op2); 13897 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13898 name = Z_STR_P(member); 13899 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13900 13901 if (on_this) { 13902 | GET_ZVAL_PTR FCARG1a, this_addr 13903 } else { 13904 if (opline->op1_type == IS_VAR 13905 && (op1_info & MAY_BE_INDIRECT) 13906 && Z_REG(op1_addr) == ZREG_FP) { 13907 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13908 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13909 | GET_Z_PTR FCARG1a, FCARG1a 13910 |1: 13911 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13912 } 13913 if (op1_info & MAY_BE_REF) { 13914 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13915 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13916 } 13917 | ZVAL_DEREF FCARG1a, op1_info 13918 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13919 } 13920 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13921 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13922 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13923 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13924 13925 if (!exit_addr) { 13926 return 0; 13927 } 13928 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13929 } else { 13930 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 13931 |.cold_code 13932 |1: 13933 | SET_EX_OPLINE opline, r0 13934 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13935 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13936 } 13937 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 13938 if (op1_info & MAY_BE_UNDEF) { 13939 | EXT_CALL zend_jit_invalid_property_assign_op, r0 13940 } else { 13941 | EXT_CALL zend_jit_invalid_property_assign, r0 13942 } 13943 may_throw = 1; 13944 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13945 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13946 may_throw = 1; 13947 | jmp >8 13948 } else { 13949 | jmp >9 13950 } 13951 |.code 13952 } 13953 } 13954 | GET_ZVAL_PTR FCARG1a, op1_addr 13955 } 13956 13957 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13958 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13959 if (prop_info) { 13960 ce = trace_ce; 13961 ce_is_instanceof = 0; 13962 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13963 if (on_this && JIT_G(current_frame) 13964 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13965 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13966 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13967 if (on_this && JIT_G(current_frame)) { 13968 JIT_G(current_frame)->ce = ce; 13969 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13970 } 13971 } else { 13972 return 0; 13973 } 13974 if (ssa->var_info && ssa_op->op1_use >= 0) { 13975 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13976 ssa->var_info[ssa_op->op1_use].ce = ce; 13977 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13978 } 13979 if (ssa->var_info && ssa_op->op1_def >= 0) { 13980 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13981 ssa->var_info[ssa_op->op1_def].ce = ce; 13982 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13983 } 13984 } 13985 } 13986 } 13987 13988 use_prop_guard = (prop_type != IS_UNKNOWN 13989 && prop_type != IS_UNDEF 13990 && prop_type != IS_REFERENCE 13991 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 13992 13993 if (!prop_info) { 13994 needs_slow_path = 1; 13995 13996 | mov r0, EX->run_time_cache 13997 | mov r2, aword [r0 + (opline+1)->extended_value] 13998 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13999 | jne >7 14000 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 14001 | cmp aword [r0 + (opline+1)->extended_value + sizeof(void*) * 2], 0 14002 | jnz >7 14003 } 14004 | mov r0, aword [r0 + (opline+1)->extended_value + sizeof(void*)] 14005 | test r0, r0 14006 | jl >7 14007 if (!use_prop_guard) { 14008 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 14009 } 14010 | add FCARG1a, r0 14011 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14012 } else { 14013 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 14014 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 14015 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14016 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14017 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14018 14019 if (!exit_addr) { 14020 return 0; 14021 } 14022 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 14023 } else { 14024 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 14025 needs_slow_path = 1; 14026 } 14027 } 14028 if (ZEND_TYPE_IS_SET(prop_info->type)) { 14029 uint32_t info = val_info; 14030 14031 may_throw = 1; 14032 14033 if (opline) { 14034 | SET_EX_OPLINE opline, r0 14035 } 14036 14037 | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1 14038 |.cold_code 14039 |1: 14040 | GET_ZVAL_PTR FCARG1a, prop_addr 14041 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 14042 | LOAD_ZVAL_ADDR FCARG2a, val_addr 14043 } 14044 |.if X64 14045 | LOAD_ADDR CARG3, binary_op 14046 |.else 14047 | sub r4, 12 14048 | PUSH_ADDR binary_op, r0 14049 |.endif 14050 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 14051 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14052 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 14053 } else { 14054 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 14055 } 14056 |.if not(X64) 14057 | add r4, 12 14058 |.endif 14059 | jmp >9 14060 |.code 14061 14062 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14063 14064 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 14065 | LOAD_ADDR FCARG2a, prop_info 14066 } else { 14067 int prop_info_offset = 14068 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 14069 14070 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 14071 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 14072 | mov FCARG2a, aword[r0 + prop_info_offset] 14073 } 14074 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 14075 |.if X64 14076 | LOAD_ZVAL_ADDR CARG3, val_addr 14077 | LOAD_ADDR CARG4, binary_op 14078 |.else 14079 | sub r4, 8 14080 | PUSH_ADDR binary_op, r0 14081 | PUSH_ZVAL_ADDR val_addr, r0 14082 |.endif 14083 14084 | EXT_CALL zend_jit_assign_op_to_typed_prop, r0 14085 14086 |.if not(X64) 14087 | add r4, 8 14088 |.endif 14089 14090 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14091 info |= MAY_BE_RC1|MAY_BE_RCN; 14092 } 14093 14094 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL 14095 } 14096 } 14097 14098 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 14099 zend_jit_addr var_addr = prop_addr; 14100 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 14101 uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 14102 14103 if (use_prop_guard) { 14104 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14105 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14106 if (!exit_addr) { 14107 return 0; 14108 } 14109 14110 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr 14111 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 14112 } 14113 14114 if (var_info & MAY_BE_REF) { 14115 may_throw = 1; 14116 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 14117 | LOAD_ZVAL_ADDR r0, prop_addr 14118 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 14119 | GET_ZVAL_PTR FCARG1a, var_addr 14120 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 14121 | jnz >1 14122 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] 14123 |.cold_code 14124 |1: 14125 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 14126 | LOAD_ZVAL_ADDR FCARG2a, val_addr 14127 } 14128 if (opline) { 14129 | SET_EX_OPLINE opline, r0 14130 } 14131 |.if X64 14132 | LOAD_ADDR CARG3, binary_op 14133 |.else 14134 | sub r4, 12 14135 | PUSH_ADDR binary_op, r0 14136 |.endif 14137 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 14138 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14139 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 14140 } else { 14141 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 14142 } 14143 |.if not(X64) 14144 | add r4, 12 14145 |.endif 14146 | jmp >9 14147 |.code 14148 |2: 14149 var_info &= ~MAY_BE_REF; 14150 } 14151 14152 switch (opline->extended_value) { 14153 case ZEND_ADD: 14154 case ZEND_SUB: 14155 case ZEND_MUL: 14156 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14157 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14158 if (opline->extended_value != ZEND_ADD || 14159 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 14160 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 14161 may_throw = 1; 14162 } 14163 } 14164 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, 14165 1 /* may overflow */, 0)) { 14166 return 0; 14167 } 14168 break; 14169 case ZEND_BW_OR: 14170 case ZEND_BW_AND: 14171 case ZEND_BW_XOR: 14172 may_throw = 1; 14173 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14174 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14175 if ((var_info & MAY_BE_ANY) != MAY_BE_STRING || 14176 (val_info & MAY_BE_ANY) != MAY_BE_STRING) { 14177 may_throw = 1; 14178 } 14179 } 14180 goto long_math; 14181 case ZEND_SL: 14182 case ZEND_SR: 14183 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14184 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14185 may_throw = 1; 14186 } 14187 if ((opline+1)->op1_type != IS_CONST || 14188 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 14189 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) { 14190 may_throw = 1; 14191 } 14192 goto long_math; 14193 case ZEND_MOD: 14194 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14195 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14196 if (opline->extended_value != ZEND_ADD || 14197 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 14198 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 14199 may_throw = 1; 14200 } 14201 } 14202 if ((opline+1)->op1_type != IS_CONST || 14203 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 14204 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) { 14205 may_throw = 1; 14206 } 14207long_math: 14208 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 14209 IS_CV, opline->op1, var_addr, var_info, NULL, 14210 (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 14211 val_range, 14212 0, var_addr, var_def_info, var_info, /* may throw */ 1)) { 14213 return 0; 14214 } 14215 break; 14216 case ZEND_CONCAT: 14217 may_throw = 1; 14218 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, 14219 0)) { 14220 return 0; 14221 } 14222 break; 14223 default: 14224 ZEND_UNREACHABLE(); 14225 } 14226 } 14227 14228 if (needs_slow_path) { 14229 may_throw = 1; 14230 |.cold_code 14231 |7: 14232 | SET_EX_OPLINE opline, r0 14233 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 14234 | LOAD_ADDR FCARG2a, name 14235 |.if X64 14236 | LOAD_ZVAL_ADDR CARG3, val_addr 14237 | mov CARG4, EX->run_time_cache 14238 | add CARG4, (opline+1)->extended_value 14239 |.if X64WIN 14240 | LOAD_ADDR r0, binary_op 14241 | mov aword A5, r0 14242 |.else 14243 | LOAD_ADDR CARG5, binary_op 14244 |.endif 14245 |.else 14246 | sub r4, 4 14247 | PUSH_ADDR binary_op, r0 14248 | mov r0, EX->run_time_cache 14249 | add r0, (opline+1)->extended_value 14250 | push r0 14251 | PUSH_ZVAL_ADDR val_addr, r0 14252 |.endif 14253 14254 | EXT_CALL zend_jit_assign_obj_op_helper, r0 14255 14256 |.if not(X64) 14257 | add r4, 4 14258 |.endif 14259 14260 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14261 val_info |= MAY_BE_RC1|MAY_BE_RCN; 14262 } 14263 14264 |8: 14265 | // FREE_OP_DATA(); 14266 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14267 | jmp >9 14268 |.code 14269 } 14270 14271 |9: 14272 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 14273 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 14274 may_throw = 1; 14275 } 14276 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 14277 } 14278 14279 if (may_throw) { 14280 if (!zend_jit_check_exception(Dst)) { 14281 return 0; 14282 } 14283 } 14284 14285 return 1; 14286} 14287 14288static int zend_jit_assign_obj(dasm_State **Dst, 14289 const zend_op *opline, 14290 const zend_op_array *op_array, 14291 zend_ssa *ssa, 14292 const zend_ssa_op *ssa_op, 14293 uint32_t op1_info, 14294 zend_jit_addr op1_addr, 14295 uint32_t val_info, 14296 bool op1_indirect, 14297 zend_class_entry *ce, 14298 bool ce_is_instanceof, 14299 bool on_this, 14300 bool delayed_fetch_this, 14301 zend_class_entry *trace_ce, 14302 uint8_t prop_type, 14303 int may_throw) 14304{ 14305 zval *member; 14306 zend_string *name; 14307 zend_property_info *prop_info; 14308 zend_jit_addr val_addr = OP1_DATA_ADDR(); 14309 zend_jit_addr res_addr = 0; 14310 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 14311 zend_jit_addr prop_addr; 14312 bool needs_slow_path = 0; 14313 bool needs_val_dtor = 0; 14314 14315 if (RETURN_VALUE_USED(opline)) { 14316 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14317 } 14318 14319 ZEND_ASSERT(opline->op2_type == IS_CONST); 14320 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 14321 14322 member = RT_CONSTANT(opline, opline->op2); 14323 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 14324 name = Z_STR_P(member); 14325 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 14326 14327 if (on_this) { 14328 | GET_ZVAL_PTR FCARG1a, this_addr 14329 } else { 14330 if (opline->op1_type == IS_VAR 14331 && (op1_info & MAY_BE_INDIRECT) 14332 && Z_REG(op1_addr) == ZREG_FP) { 14333 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14334 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 14335 | GET_Z_PTR FCARG1a, FCARG1a 14336 |1: 14337 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14338 } 14339 if (op1_info & MAY_BE_REF) { 14340 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14341 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14342 } 14343 | ZVAL_DEREF FCARG1a, op1_info 14344 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14345 } 14346 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 14347 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14348 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14349 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14350 14351 if (!exit_addr) { 14352 return 0; 14353 } 14354 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 14355 } else { 14356 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 14357 |.cold_code 14358 |1: 14359 | SET_EX_OPLINE opline, r0 14360 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14361 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14362 } 14363 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 14364 | EXT_CALL zend_jit_invalid_property_assign, r0 14365 if (RETURN_VALUE_USED(opline)) { 14366 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 14367 } 14368 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 14369 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14370 needs_val_dtor = 1; 14371 | jmp >7 14372 } else { 14373 | jmp >9 14374 } 14375 |.code 14376 } 14377 } 14378 | GET_ZVAL_PTR FCARG1a, op1_addr 14379 } 14380 14381 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 14382 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 14383 if (prop_info) { 14384 ce = trace_ce; 14385 ce_is_instanceof = 0; 14386 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 14387 if (on_this && JIT_G(current_frame) 14388 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 14389 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 14390 } else if (zend_jit_class_guard(Dst, opline, ce)) { 14391 if (on_this && JIT_G(current_frame)) { 14392 JIT_G(current_frame)->ce = ce; 14393 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 14394 } 14395 } else { 14396 return 0; 14397 } 14398 if (ssa->var_info && ssa_op->op1_use >= 0) { 14399 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 14400 ssa->var_info[ssa_op->op1_use].ce = ce; 14401 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 14402 } 14403 if (ssa->var_info && ssa_op->op1_def >= 0) { 14404 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 14405 ssa->var_info[ssa_op->op1_def].ce = ce; 14406 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 14407 } 14408 } 14409 } 14410 } 14411 14412 if (!prop_info) { 14413 needs_slow_path = 1; 14414 14415 | mov r0, EX->run_time_cache 14416 | mov r2, aword [r0 + opline->extended_value] 14417 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 14418 | jne >5 14419 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 14420 | mov FCARG2a, aword [r0 + opline->extended_value + sizeof(void*) * 2] 14421 } 14422 | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] 14423 | test r0, r0 14424 | jl >5 14425 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >5 14426 | add FCARG1a, r0 14427 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14428 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 14429 | test FCARG2a, FCARG2a 14430 | jnz >1 14431 |.cold_code 14432 |1: 14433 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14434 | SET_EX_OPLINE opline, r0 14435 |.if X64 14436 | LOAD_ZVAL_ADDR CARG3, val_addr 14437 if (RETURN_VALUE_USED(opline)) { 14438 | LOAD_ZVAL_ADDR CARG4, res_addr 14439 } else { 14440 | xor CARG4, CARG4 14441 } 14442 |.else 14443 | sub r4, 8 14444 if (RETURN_VALUE_USED(opline)) { 14445 | PUSH_ZVAL_ADDR res_addr, r0 14446 } else { 14447 | push 0 14448 } 14449 | PUSH_ZVAL_ADDR val_addr, r0 14450 |.endif 14451 14452 | EXT_CALL zend_jit_assign_to_typed_prop, r0 14453 14454 |.if not(X64) 14455 | add r4, 8 14456 |.endif 14457 14458 if ((opline+1)->op1_type == IS_CONST) { 14459 | // TODO: ??? 14460 | // if (Z_TYPE_P(value) == orig_type) { 14461 | // CACHE_PTR_EX(cache_slot + 2, NULL); 14462 } 14463 14464 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 14465 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14466 | jmp >7 14467 } else { 14468 | jmp >9 14469 } 14470 |.code 14471 } 14472 } else { 14473 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 14474 if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) { 14475 // Undefined property with magic __get()/__set() 14476 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14477 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14478 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14479 14480 if (!exit_addr) { 14481 return 0; 14482 } 14483 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 14484 } else { 14485 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >5 14486 needs_slow_path = 1; 14487 } 14488 } 14489 if (ZEND_TYPE_IS_SET(prop_info->type)) { 14490 uint32_t info = val_info; 14491 14492 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14493 | SET_EX_OPLINE opline, r0 14494 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 14495 | LOAD_ADDR FCARG2a, prop_info 14496 } else { 14497 int prop_info_offset = 14498 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 14499 14500 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 14501 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 14502 | mov FCARG2a, aword[r0 + prop_info_offset] 14503 } 14504 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 14505 |.if X64 14506 | LOAD_ZVAL_ADDR CARG3, val_addr 14507 if (RETURN_VALUE_USED(opline)) { 14508 | LOAD_ZVAL_ADDR CARG4, res_addr 14509 } else { 14510 | xor CARG4, CARG4 14511 } 14512 |.else 14513 | sub r4, 8 14514 if (RETURN_VALUE_USED(opline)) { 14515 | PUSH_ZVAL_ADDR res_addr, r0 14516 } else { 14517 | push 0 14518 } 14519 | PUSH_ZVAL_ADDR val_addr, r0 14520 |.endif 14521 14522 | EXT_CALL zend_jit_assign_to_typed_prop, r0 14523 14524 |.if not(X64) 14525 | add r4, 8 14526 |.endif 14527 14528 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14529 info |= MAY_BE_RC1|MAY_BE_RCN; 14530 } 14531 14532 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL 14533 } 14534 } 14535 14536 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 14537 // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); 14538 if (opline->result_type == IS_UNUSED) { 14539 if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 14540 return 0; 14541 } 14542 } else { 14543 if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 14544 return 0; 14545 } 14546 } 14547 } 14548 14549 if (needs_slow_path) { 14550 |.cold_code 14551 |5: 14552 | SET_EX_OPLINE opline, r0 14553 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 14554 | LOAD_ADDR FCARG2a, name 14555 |.if X64 14556 | LOAD_ZVAL_ADDR CARG3, val_addr 14557 | mov CARG4, EX->run_time_cache 14558 | add CARG4, opline->extended_value 14559 if (RETURN_VALUE_USED(opline)) { 14560 |.if X64WIN 14561 | LOAD_ZVAL_ADDR r0, res_addr 14562 | mov aword A5, r0 14563 |.else 14564 | LOAD_ZVAL_ADDR CARG5, res_addr 14565 |.endif 14566 } else { 14567 |.if X64WIN 14568 | mov aword A5, 0 14569 |.else 14570 | xor CARG5, CARG5 14571 |.endif 14572 } 14573 |.else 14574 | sub r4, 4 14575 if (RETURN_VALUE_USED(opline)) { 14576 | PUSH_ZVAL_ADDR res_addr, r0 14577 } else { 14578 | push 0 14579 } 14580 | mov r0, EX->run_time_cache 14581 | add r0, opline->extended_value 14582 | push r0 14583 | PUSH_ZVAL_ADDR val_addr, r0 14584 |.endif 14585 14586 | EXT_CALL zend_jit_assign_obj_helper, r0 14587 14588 |.if not(X64) 14589 | add r4, 4 14590 |.endif 14591 14592 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14593 val_info |= MAY_BE_RC1|MAY_BE_RCN; 14594 } 14595 14596 |7: 14597 | // FREE_OP_DATA(); 14598 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14599 | jmp >9 14600 |.code 14601 } else if (needs_val_dtor) { 14602 |.cold_code 14603 |7: 14604 | // FREE_OP_DATA(); 14605 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14606 | jmp >9 14607 |.code 14608 } 14609 14610 |9: 14611 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 14612 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 14613 } 14614 14615 if (may_throw) { 14616 if (!zend_jit_check_exception(Dst)) { 14617 return 0; 14618 } 14619 } 14620 14621 return 1; 14622} 14623 14624static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) 14625{ 14626 zend_jit_addr op1_addr = OP1_ADDR(); 14627 14628 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 14629 if (may_throw) { 14630 | SET_EX_OPLINE opline, r0 14631 } 14632 if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { 14633 if (op1_info & MAY_BE_ARRAY) { 14634 | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7 14635 } 14636 | mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_iter_idx)] 14637 | cmp FCARG1d, -1 14638 | je >7 14639 | EXT_CALL zend_hash_iterator_del, r0 14640 |7: 14641 } 14642 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline 14643 if (may_throw) { 14644 if (!zend_jit_check_exception(Dst)) { 14645 return 0; 14646 } 14647 } 14648 } 14649 14650 return 1; 14651} 14652 14653static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 14654{ 14655 if (opline->op1_type == IS_CONST) { 14656 zval *zv; 14657 size_t len; 14658 14659 zv = RT_CONSTANT(opline, opline->op1); 14660 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14661 len = Z_STRLEN_P(zv); 14662 14663 if (len > 0) { 14664 const char *str = Z_STRVAL_P(zv); 14665 14666 | SET_EX_OPLINE opline, r0 14667 |.if X64 14668 | LOAD_ADDR CARG1, str 14669 | LOAD_ADDR CARG2, len 14670 | EXT_CALL zend_write, r0 14671 |.else 14672 | mov aword A2, len 14673 | mov aword A1, str 14674 | EXT_CALL zend_write, r0 14675 |.endif 14676 if (!zend_jit_check_exception(Dst)) { 14677 return 0; 14678 } 14679 } 14680 } else { 14681 zend_jit_addr op1_addr = OP1_ADDR(); 14682 14683 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14684 14685 | SET_EX_OPLINE opline, r0 14686 | GET_ZVAL_PTR r0, op1_addr 14687 |.if X64 14688 | lea CARG1, aword [r0 + offsetof(zend_string, val)] 14689 | mov CARG2, aword [r0 + offsetof(zend_string, len)] 14690 | EXT_CALL zend_write, r0 14691 |.else 14692 | add r0, offsetof(zend_string, val) 14693 | mov aword A1, r0 14694 | mov r0, aword [r0 + (offsetof(zend_string, len)-offsetof(zend_string, val))] 14695 | mov aword A2, r0 14696 | EXT_CALL zend_write, r0 14697 |.endif 14698 if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { 14699 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline 14700 } 14701 if (!zend_jit_check_exception(Dst)) { 14702 return 0; 14703 } 14704 } 14705 return 1; 14706} 14707 14708static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) 14709{ 14710 if (opline->op1_type == IS_CONST) { 14711 zval *zv; 14712 size_t len; 14713 14714 zv = RT_CONSTANT(opline, opline->op1); 14715 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14716 len = Z_STRLEN_P(zv); 14717 14718 | SET_ZVAL_LVAL res_addr, len 14719 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 14720 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14721 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14722 return 0; 14723 } 14724 } else { 14725 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14726 14727 if (Z_MODE(res_addr) == IS_REG) { 14728 | GET_ZVAL_PTR Ra(Z_REG(res_addr)), op1_addr 14729 | mov Ra(Z_REG(res_addr)), aword [Ra(Z_REG(res_addr))+offsetof(zend_string, len)] 14730 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14731 return 0; 14732 } 14733 } else { 14734 | GET_ZVAL_PTR r0, op1_addr 14735 | mov r0, aword [r0 + offsetof(zend_string, len)] 14736 | SET_ZVAL_LVAL res_addr, r0 14737 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14738 } 14739 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 14740 } 14741 return 1; 14742} 14743 14744static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) 14745{ 14746 if (opline->op1_type == IS_CONST) { 14747 zval *zv; 14748 zend_long count; 14749 14750 zv = RT_CONSTANT(opline, opline->op1); 14751 ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); 14752 count = zend_hash_num_elements(Z_ARRVAL_P(zv)); 14753 14754 | SET_ZVAL_LVAL res_addr, count 14755 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 14756 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14757 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14758 return 0; 14759 } 14760 } else { 14761 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY); 14762 // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+. 14763 14764 if (Z_MODE(res_addr) == IS_REG) { 14765 | GET_ZVAL_PTR Ra(Z_REG(res_addr)), op1_addr 14766 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 14767 | mov Rd(Z_REG(res_addr)), dword [Ra(Z_REG(res_addr))+offsetof(HashTable, nNumOfElements)] 14768 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14769 return 0; 14770 } 14771 } else { 14772 | GET_ZVAL_PTR r0, op1_addr 14773 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 14774 | mov eax, dword [r0 + offsetof(HashTable, nNumOfElements)] 14775 | SET_ZVAL_LVAL res_addr, r0 14776 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14777 } 14778 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 14779 } 14780 14781 if (may_throw) { 14782 return zend_jit_check_exception(Dst); 14783 } 14784 return 1; 14785} 14786 14787static int zend_jit_load_this(dasm_State **Dst, uint32_t var) 14788{ 14789 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 14790 14791 | mov FCARG1a, aword EX->This.value.ptr 14792 | SET_ZVAL_PTR var_addr, FCARG1a 14793 | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX 14794 | GC_ADDREF FCARG1a 14795 14796 return 1; 14797} 14798 14799static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only) 14800{ 14801 if (!op_array->scope || 14802 (op_array->fn_flags & ZEND_ACC_STATIC) || 14803 ((op_array->fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_IMMUTABLE)) == ZEND_ACC_CLOSURE)) { 14804 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14805 if (!JIT_G(current_frame) || 14806 !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { 14807 14808 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14809 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14810 14811 if (!exit_addr) { 14812 return 0; 14813 } 14814 14815 | cmp byte EX->This.u1.v.type, IS_OBJECT 14816 | jne &exit_addr 14817 14818 if (JIT_G(current_frame)) { 14819 TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); 14820 } 14821 } 14822 } else { 14823 14824 | cmp byte EX->This.u1.v.type, IS_OBJECT 14825 | jne >1 14826 |.cold_code 14827 |1: 14828 | SET_EX_OPLINE opline, r0 14829 | jmp ->invalid_this 14830 |.code 14831 } 14832 } 14833 14834 if (!check_only) { 14835 if (!zend_jit_load_this(Dst, opline->result.var)) { 14836 return 0; 14837 } 14838 } 14839 14840 return 1; 14841} 14842 14843static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) 14844{ 14845 uint32_t count; 14846 Bucket *p; 14847 const zend_op *target; 14848 int b; 14849 int32_t exit_point; 14850 const void *exit_addr; 14851 14852 | test r0, r0 14853 if (default_label) { 14854 | jz &default_label 14855 } else if (next_opline) { 14856 | jz >3 14857 } else { 14858 | jz =>default_b 14859 } 14860 | LOAD_ADDR FCARG1a, jumptable 14861 | sub r0, aword [FCARG1a + offsetof(HashTable, arData)] 14862 if (HT_IS_PACKED(jumptable)) { 14863 | mov FCARG1a, (sizeof(zval) / sizeof(void*)) 14864 } else { 14865 | mov FCARG1a, (sizeof(Bucket) / sizeof(void*)) 14866 } 14867 |.if X64 14868 | cqo 14869 |.else 14870 | cdq 14871 |.endif 14872 | idiv FCARG1a 14873 |.if X64 14874 if (!IS_32BIT(dasm_end)) { 14875 | lea FCARG1a, aword [>4] 14876 | jmp aword [FCARG1a + r0] 14877 } else { 14878 | jmp aword [r0 + >4] 14879 } 14880 |.else 14881 | jmp aword [r0 + >4] 14882 |.endif 14883 |.jmp_table 14884 |.align aword 14885 |4: 14886 if (trace_info) { 14887 trace_info->jmp_table_size += zend_hash_num_elements(jumptable); 14888 } 14889 14890 count = jumptable->nNumUsed; 14891 p = jumptable->arData; 14892 do { 14893 if (Z_TYPE(p->val) == IS_UNDEF) { 14894 if (default_label) { 14895 | .aword &default_label 14896 } else if (next_opline) { 14897 | .aword >3 14898 } else { 14899 | .aword =>default_b 14900 } 14901 } else { 14902 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 14903 if (!next_opline) { 14904 b = ssa->cfg.map[target - op_array->opcodes]; 14905 | .aword =>b 14906 } else if (next_opline == target) { 14907 | .aword >3 14908 } else { 14909 exit_point = zend_jit_trace_get_exit_point(target, 0); 14910 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14911 if (!exit_addr) { 14912 return 0; 14913 } 14914 | .aword &exit_addr 14915 } 14916 } 14917 if (HT_IS_PACKED(jumptable)) { 14918 p = (Bucket*)(((zval*)p)+1); 14919 } else { 14920 p++; 14921 } 14922 count--; 14923 } while (count); 14924 |.code 14925 14926 return 1; 14927} 14928 14929static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) 14930{ 14931 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 14932 const zend_op *next_opline = NULL; 14933 14934 if (trace) { 14935 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 14936 ZEND_ASSERT(trace->opline != NULL); 14937 next_opline = trace->opline; 14938 } 14939 14940 if (opline->op1_type == IS_CONST) { 14941 zval *zv = RT_CONSTANT(opline, opline->op1); 14942 zval *jump_zv = NULL; 14943 int b; 14944 14945 if (opline->opcode == ZEND_SWITCH_LONG) { 14946 if (Z_TYPE_P(zv) == IS_LONG) { 14947 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14948 } 14949 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14950 if (Z_TYPE_P(zv) == IS_STRING) { 14951 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 14952 } 14953 } else if (opline->opcode == ZEND_MATCH) { 14954 if (Z_TYPE_P(zv) == IS_LONG) { 14955 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14956 } else if (Z_TYPE_P(zv) == IS_STRING) { 14957 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 14958 } 14959 } else { 14960 ZEND_UNREACHABLE(); 14961 } 14962 if (next_opline) { 14963 const zend_op *target; 14964 14965 if (jump_zv != NULL) { 14966 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); 14967 } else { 14968 target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14969 } 14970 ZEND_ASSERT(target == next_opline); 14971 } else { 14972 if (jump_zv != NULL) { 14973 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; 14974 } else { 14975 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; 14976 } 14977 | jmp =>b 14978 } 14979 } else { 14980 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; 14981 uint32_t op1_info = OP1_INFO(); 14982 zend_jit_addr op1_addr = OP1_ADDR(); 14983 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14984 const zend_op *target; 14985 int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; 14986 int b; 14987 int32_t exit_point; 14988 const void *fallback_label = NULL; 14989 const void *default_label = NULL; 14990 const void *exit_addr; 14991 14992 if (next_opline) { 14993 if (next_opline != opline + 1) { 14994 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 14995 fallback_label = zend_jit_trace_get_exit_addr(exit_point); 14996 if (!fallback_label) { 14997 return 0; 14998 } 14999 } 15000 if (next_opline != default_opline) { 15001 exit_point = zend_jit_trace_get_exit_point(default_opline, 0); 15002 default_label = zend_jit_trace_get_exit_addr(exit_point); 15003 if (!default_label) { 15004 return 0; 15005 } 15006 } 15007 } 15008 15009 if (opline->opcode == ZEND_SWITCH_LONG) { 15010 if (op1_info & MAY_BE_LONG) { 15011 if (op1_info & MAY_BE_REF) { 15012 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1 15013 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr 15014 |.cold_code 15015 |1: 15016 | // ZVAL_DEREF(op) 15017 if (fallback_label) { 15018 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label 15019 } else { 15020 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 15021 } 15022 | GET_ZVAL_PTR FCARG2a, op1_addr 15023 if (fallback_label) { 15024 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, &fallback_label 15025 } else { 15026 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3 15027 } 15028 | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.lval)] 15029 | jmp >2 15030 |.code 15031 |2: 15032 } else { 15033 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 15034 if (fallback_label) { 15035 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label 15036 } else { 15037 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 15038 } 15039 } 15040 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr 15041 } 15042 if (HT_IS_PACKED(jumptable)) { 15043 uint32_t count = jumptable->nNumUsed; 15044 zval *zv = jumptable->arPacked; 15045 15046 | cmp FCARG2a, jumptable->nNumUsed 15047 if (default_label) { 15048 | jae &default_label 15049 } else if (next_opline) { 15050 | jae >3 15051 } else { 15052 | jae =>default_b 15053 } 15054 |.if X64 15055 if (!IS_32BIT(dasm_end)) { 15056 | lea r0, aword [>4] 15057 | jmp aword [r0 + FCARG2a * 8] 15058 } else { 15059 | jmp aword [FCARG2a * 8 + >4] 15060 } 15061 |.else 15062 | jmp aword [FCARG2a * 4 + >4] 15063 |.endif 15064 |.jmp_table 15065 |.align aword 15066 |4: 15067 if (trace_info) { 15068 trace_info->jmp_table_size += count; 15069 } 15070 do { 15071 if (Z_TYPE_P(zv) == IS_UNDEF) { 15072 if (default_label) { 15073 | .aword &default_label 15074 } else if (next_opline) { 15075 | .aword >3 15076 } else { 15077 | .aword =>default_b 15078 } 15079 } else { 15080 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); 15081 if (!next_opline) { 15082 b = ssa->cfg.map[target - op_array->opcodes]; 15083 | .aword =>b 15084 } else if (next_opline == target) { 15085 | .aword >3 15086 } else { 15087 exit_point = zend_jit_trace_get_exit_point(target, 0); 15088 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15089 if (!exit_addr) { 15090 return 0; 15091 } 15092 | .aword &exit_addr 15093 } 15094 } 15095 zv++; 15096 count--; 15097 } while (count); 15098 |.code 15099 |3: 15100 } else { 15101 | LOAD_ADDR FCARG1a, jumptable 15102 | EXT_CALL zend_hash_index_find, r0 15103 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 15104 return 0; 15105 } 15106 |3: 15107 } 15108 } 15109 } else if (opline->opcode == ZEND_SWITCH_STRING) { 15110 if (op1_info & MAY_BE_STRING) { 15111 if (op1_info & MAY_BE_REF) { 15112 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1 15113 | GET_ZVAL_PTR FCARG2a, op1_addr 15114 |.cold_code 15115 |1: 15116 | // ZVAL_DEREF(op) 15117 if (fallback_label) { 15118 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label 15119 } else { 15120 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 15121 } 15122 | GET_ZVAL_PTR FCARG2a, op1_addr 15123 if (fallback_label) { 15124 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, &fallback_label 15125 } else { 15126 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3 15127 } 15128 | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.ptr)] 15129 | jmp >2 15130 |.code 15131 |2: 15132 } else { 15133 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { 15134 if (fallback_label) { 15135 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label 15136 } else { 15137 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 15138 } 15139 } 15140 | GET_ZVAL_PTR FCARG2a, op1_addr 15141 } 15142 | LOAD_ADDR FCARG1a, jumptable 15143 | EXT_CALL zend_hash_find, r0 15144 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 15145 return 0; 15146 } 15147 |3: 15148 } 15149 } else if (opline->opcode == ZEND_MATCH) { 15150 if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { 15151 if (op1_info & MAY_BE_REF) { 15152 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 15153 | ZVAL_DEREF FCARG2a, op1_info 15154 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 15155 } 15156 | LOAD_ADDR FCARG1a, jumptable 15157 if (op1_info & MAY_BE_LONG) { 15158 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 15159 if (op1_info & MAY_BE_STRING) { 15160 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5 15161 } else if (op1_info & MAY_BE_UNDEF) { 15162 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 15163 } else if (default_label) { 15164 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label 15165 } else if (next_opline) { 15166 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 15167 } else { 15168 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b 15169 } 15170 } 15171 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr 15172 | EXT_CALL zend_hash_index_find, r0 15173 if (op1_info & MAY_BE_STRING) { 15174 | jmp >2 15175 } 15176 } 15177 if (op1_info & MAY_BE_STRING) { 15178 |5: 15179 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { 15180 if (op1_info & MAY_BE_UNDEF) { 15181 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 15182 } else if (default_label) { 15183 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label 15184 } else if (next_opline) { 15185 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 15186 } else { 15187 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b 15188 } 15189 } 15190 | GET_ZVAL_PTR FCARG2a, op1_addr 15191 | EXT_CALL zend_hash_find, r0 15192 } 15193 |2: 15194 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 15195 return 0; 15196 } 15197 } 15198 if (op1_info & MAY_BE_UNDEF) { 15199 |6: 15200 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { 15201 if (default_label) { 15202 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label 15203 } else if (next_opline) { 15204 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3 15205 } else { 15206 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b 15207 } 15208 } 15209 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 15210 | SET_EX_OPLINE opline, r0 15211 | mov FCARG1d, opline->op1.var 15212 | EXT_CALL zend_jit_undefined_op_helper, r0 15213 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 15214 return 0; 15215 } 15216 } 15217 if (default_label) { 15218 | jmp &default_label 15219 } else if (next_opline) { 15220 | jmp >3 15221 } else { 15222 | jmp =>default_b 15223 } 15224 |3: 15225 } else { 15226 ZEND_UNREACHABLE(); 15227 } 15228 } 15229 return 1; 15230} 15231 15232static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) 15233{ 15234 zend_arg_info *arg_info = &op_array->arg_info[-1]; 15235 ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); 15236 zend_jit_addr op1_addr = OP1_ADDR(); 15237 bool needs_slow_check = 1; 15238 bool slow_check_in_cold = 1; 15239 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 15240 15241 if (type_mask == 0) { 15242 slow_check_in_cold = 0; 15243 } else { 15244 if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { 15245 slow_check_in_cold = 0; 15246 } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { 15247 needs_slow_check = 0; 15248 } else if (is_power_of_two(type_mask)) { 15249 uint32_t type_code = concrete_type(type_mask); 15250 | IF_NOT_ZVAL_TYPE op1_addr, type_code, >6 15251 } else { 15252 | mov edx, 1 15253 | GET_ZVAL_TYPE cl, op1_addr 15254 | shl edx, cl 15255 | test edx, type_mask 15256 | je >6 15257 } 15258 } 15259 if (needs_slow_check) { 15260 if (slow_check_in_cold) { 15261 |.cold_code 15262 |6: 15263 } 15264 | SET_EX_OPLINE opline, r1 15265 if (op1_info & MAY_BE_UNDEF) { 15266 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7 15267 | mov FCARG1a, opline->op1.var 15268 | EXT_CALL zend_jit_undefined_op_helper, FCARG2a 15269 | test r0, r0 15270 | jz ->exception_handler 15271 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 15272 | jmp >8 15273 } 15274 |7: 15275 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 15276 |8: 15277 | mov FCARG2a, EX->func 15278 |.if X64 15279 | LOAD_ADDR CARG3, (ptrdiff_t)arg_info 15280 | mov r0, EX->run_time_cache 15281 | lea CARG4, aword [r0+opline->op2.num] 15282 | EXT_CALL zend_jit_verify_return_slow, r0 15283 |.else 15284 | sub r4, 8 15285 | mov r0, EX->run_time_cache 15286 | add r0, opline->op2.num 15287 | push r0 15288 | push (ptrdiff_t)arg_info 15289 | EXT_CALL zend_jit_verify_return_slow, r0 15290 | add r4, 8 15291 |.endif 15292 if (!zend_jit_check_exception(Dst)) { 15293 return 0; 15294 } 15295 if (slow_check_in_cold) { 15296 | jmp >9 15297 |.code 15298 } 15299 } 15300 |9: 15301 return 1; 15302} 15303 15304static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 15305{ 15306 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15307 15308 // TODO: support for empty() ??? 15309 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 15310 15311 if (op1_info & MAY_BE_REF) { 15312 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 15313 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 15314 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 15315 } 15316 | ZVAL_DEREF FCARG1a, op1_info 15317 |1: 15318 } 15319 15320 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { 15321 if (exit_addr) { 15322 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); 15323 } else if (smart_branch_opcode) { 15324 if (smart_branch_opcode == ZEND_JMPNZ) { 15325 | jmp =>target_label 15326 } 15327 } else { 15328 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 15329 } 15330 } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { 15331 if (exit_addr) { 15332 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); 15333 } else if (smart_branch_opcode) { 15334 if (smart_branch_opcode != ZEND_JMPNZ) { 15335 | jmp =>target_label 15336 } 15337 } else { 15338 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 15339 } 15340 } else { 15341 ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); 15342 | cmp byte [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)], IS_NULL 15343 if (exit_addr) { 15344 if (smart_branch_opcode == ZEND_JMPNZ) { 15345 | jg &exit_addr 15346 } else { 15347 | jle &exit_addr 15348 } 15349 } else if (smart_branch_opcode) { 15350 if (smart_branch_opcode == ZEND_JMPZ) { 15351 | jle =>target_label 15352 } else if (smart_branch_opcode == ZEND_JMPNZ) { 15353 | jg =>target_label 15354 } else { 15355 ZEND_UNREACHABLE(); 15356 } 15357 } else { 15358 | setg al 15359 | movzx eax, al 15360 | lea eax, [eax + IS_FALSE] 15361 | SET_ZVAL_TYPE_INFO res_addr, eax 15362 } 15363 } 15364 15365 return 1; 15366} 15367 15368static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 15369{ 15370 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15371 15372 if (opline->op1_type == IS_CONST) { 15373 zval *zv = RT_CONSTANT(opline, opline->op1); 15374 15375 | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 15376 if (Z_REFCOUNTED_P(zv)) { 15377 | ADDREF_CONST zv, r0 15378 } 15379 } else { 15380 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 15381 15382 | // ZVAL_COPY(res, value); 15383 | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_FCARG1 15384 if (opline->op1_type == IS_CV) { 15385 | TRY_ADDREF op1_info, ah, FCARG1a 15386 } 15387 } 15388 | // Z_FE_POS_P(res) = 0; 15389 | mov dword [FP + opline->result.var + offsetof(zval, u2.fe_pos)], 0 15390 15391 return 1; 15392} 15393 15394static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, uint8_t exit_opcode, const void *exit_addr) 15395{ 15396 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 15397 15398 if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) { 15399 /* empty array */ 15400 if (exit_addr) { 15401 if (exit_opcode == ZEND_JMP) { 15402 | jmp &exit_addr 15403 } 15404 } else { 15405 | jmp =>target_label 15406 } 15407 return 1; 15408 } 15409 15410 | // array = EX_VAR(opline->op1.var); 15411 | // fe_ht = Z_ARRVAL_P(array); 15412 | GET_ZVAL_PTR FCARG1a, op1_addr 15413 15414 if (op1_info & MAY_BE_PACKED_GUARD) { 15415 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 15416 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15417 15418 if (!exit_addr) { 15419 return 0; 15420 } 15421 if (op1_info & MAY_BE_ARRAY_PACKED) { 15422 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 15423 | jz &exit_addr 15424 } else { 15425 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 15426 | jnz &exit_addr 15427 } 15428 } 15429 15430 | // pos = Z_FE_POS_P(array); 15431 | mov eax, dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)] 15432 15433 if (MAY_BE_HASH(op1_info)) { 15434 if (MAY_BE_PACKED(op1_info)) { 15435 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 15436 | jnz >2 15437 } 15438 15439 | // p = fe_ht->arData + pos; 15440 |.if X64 15441 || ZEND_ASSERT(sizeof(Bucket) == 32); 15442 | mov FCARG2d, eax 15443 | shl FCARG2a, 5 15444 |.else 15445 | imul FCARG2a, r0, sizeof(Bucket) 15446 |.endif 15447 | add FCARG2a, aword [FCARG1a + offsetof(zend_array, arData)] 15448 |1: 15449 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 15450 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], eax 15451 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 15452 | // ZEND_VM_CONTINUE(); 15453 if (exit_addr) { 15454 if (exit_opcode == ZEND_JMP) { 15455 | jbe &exit_addr 15456 } else { 15457 | jbe >3 15458 } 15459 } else { 15460 | jbe =>target_label 15461 } 15462 | // pos++; 15463 | add eax, 1 15464 | // value_type = Z_TYPE_INFO_P(value); 15465 | // if (EXPECTED(value_type != IS_UNDEF)) { 15466 if (!exit_addr || exit_opcode == ZEND_JMP) { 15467 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >3 15468 } else { 15469 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, &exit_addr 15470 } 15471 | // p++; 15472 | add FCARG2a, sizeof(Bucket) 15473 | jmp <1 15474 if (MAY_BE_PACKED(op1_info)) { 15475 |2: 15476 } 15477 } 15478 if (MAY_BE_PACKED(op1_info)) { 15479 | // p = fe_ht->arPacked + pos; 15480 || ZEND_ASSERT(sizeof(zval) == 16); 15481 | mov FCARG2d, eax 15482 | shl FCARG2a, 4 15483 | add FCARG2a, aword [FCARG1a + offsetof(zend_array, arPacked)] 15484 |1: 15485 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 15486 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], eax 15487 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 15488 | // ZEND_VM_CONTINUE(); 15489 if (exit_addr) { 15490 if (exit_opcode == ZEND_JMP) { 15491 | jbe &exit_addr 15492 } else { 15493 | jbe >4 15494 } 15495 } else { 15496 | jbe =>target_label 15497 } 15498 | // pos++; 15499 | add eax, 1 15500 | // value_type = Z_TYPE_INFO_P(value); 15501 | // if (EXPECTED(value_type != IS_UNDEF)) { 15502 if (!exit_addr || exit_opcode == ZEND_JMP) { 15503 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >4 15504 } else { 15505 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, &exit_addr 15506 } 15507 | // p++; 15508 | add FCARG2a, sizeof(zval) 15509 | jmp <1 15510 } 15511 15512 15513 if (!exit_addr || exit_opcode == ZEND_JMP) { 15514 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 15515 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 15516 uint32_t val_info; 15517 15518 if (RETURN_VALUE_USED(opline)) { 15519 zend_jit_addr res_addr = RES_ADDR(); 15520 15521 if (MAY_BE_HASH(op1_info)) { 15522 |3: 15523 | // Z_FE_POS_P(array) = pos + 1; 15524 | mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], eax 15525 15526 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) 15527 && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { 15528 | // if (!p->key) { 15529 | cmp aword [FCARG2a + offsetof(Bucket, key)], 0 15530 | jz >2 15531 } 15532 if (op1_info & MAY_BE_ARRAY_KEY_STRING) { 15533 | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); 15534 | mov r0, aword [FCARG2a + offsetof(Bucket, key)] 15535 | SET_ZVAL_PTR res_addr, r0 15536 | test dword [r0 + offsetof(zend_refcounted, gc.u.type_info)], IS_STR_INTERNED 15537 | jz >1 15538 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING 15539 | jmp >3 15540 |1: 15541 | GC_ADDREF r0 15542 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX 15543 15544 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) || MAY_BE_PACKED(op1_info)) { 15545 | jmp >3 15546 |2: 15547 } 15548 } 15549 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 15550 | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); 15551 | mov r0, aword [FCARG2a + offsetof(Bucket, h)] 15552 | SET_ZVAL_LVAL res_addr, r0 15553 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 15554 if (MAY_BE_PACKED(op1_info)) { 15555 | jmp >3 15556 } 15557 } 15558 } 15559 if (MAY_BE_PACKED(op1_info)) { 15560 |4: 15561 | // Z_FE_POS_P(array) = pos + 1; 15562 | mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], eax 15563 | sub r0, 1 15564 | SET_ZVAL_LVAL res_addr, r0 15565 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 15566 } 15567 |3: 15568 } else { 15569 |3: 15570 |4: 15571 | // Z_FE_POS_P(array) = pos + 1; 15572 | mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], eax 15573 } 15574 15575 val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); 15576 if (val_info & MAY_BE_ARRAY) { 15577 val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 15578 } 15579 if (op1_info & MAY_BE_ARRAY_OF_REF) { 15580 val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | 15581 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 15582 } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 15583 val_info |= MAY_BE_RC1 | MAY_BE_RCN; 15584 } 15585 15586 if (opline->op2_type == IS_CV) { 15587 | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); 15588 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { 15589 return 0; 15590 } 15591 } else { 15592 | // ZVAL_COPY(res, value); 15593 | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_R0, ZREG_FCARG1 15594 | TRY_ADDREF val_info, ah, FCARG1a 15595 } 15596 } else { 15597 |3: 15598 |4: 15599 } 15600 15601 return 1; 15602} 15603 15604static int zend_jit_fetch_constant(dasm_State **Dst, 15605 const zend_op *opline, 15606 const zend_op_array *op_array, 15607 zend_ssa *ssa, 15608 const zend_ssa_op *ssa_op, 15609 zend_jit_addr res_addr) 15610{ 15611 zval *zv = RT_CONSTANT(opline, opline->op2) + 1; 15612 zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 15613 uint32_t res_info = RES_INFO(); 15614 15615 | // c = CACHED_PTR(opline->extended_value); 15616 | mov FCARG1a, EX->run_time_cache 15617 | mov r0, aword [FCARG1a + opline->extended_value] 15618 | // if (c != NULL) 15619 | test r0, r0 15620 | jz >9 15621 if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) { 15622 | // if (!IS_SPECIAL_CACHE_VAL(c)) 15623 | test r0, CACHE_SPECIAL 15624 | jnz >9 15625 } 15626 |8: 15627 15628 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { 15629 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 15630 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 15631 int32_t exit_point; 15632 const void *exit_addr = NULL; 15633 15634 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 15635 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 15636 exit_point = zend_jit_trace_get_exit_point(opline+1, 0); 15637 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 15638 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15639 if (!exit_addr) { 15640 return 0; 15641 } 15642 res_info &= ~MAY_BE_GUARD; 15643 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 15644 15645 uint8_t type = concrete_type(res_info); 15646 15647 if (type < IS_STRING) { 15648 | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr 15649 } else { 15650 | GET_ZVAL_TYPE_INFO edx, const_addr 15651 | IF_NOT_TYPE dl, type, &exit_addr 15652 } 15653 | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_R0, ZREG_R1 15654 if (type < IS_STRING) { 15655 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 15656 | SET_ZVAL_TYPE_INFO res_addr, type 15657 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 15658 return 0; 15659 } 15660 } else { 15661 | SET_ZVAL_TYPE_INFO res_addr, edx 15662 | TRY_ADDREF res_info, dh, r1 15663 } 15664 } else { 15665 | // ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup) 15666 | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_R0, ZREG_R1 15667 | TRY_ADDREF MAY_BE_ANY, ah, r1 15668 } 15669 15670 |.cold_code 15671 |9: 15672 | // SAVE_OPLINE(); 15673 | SET_EX_OPLINE opline, r0 15674 | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); 15675 | LOAD_ADDR FCARG1a, zv 15676 | mov FCARG2a, opline->op1.num 15677 | EXT_CALL zend_jit_get_constant, r0 15678 | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); 15679 | test r0, r0 15680 | jnz <8 15681 | jmp ->exception_handler 15682 |.code 15683 15684 return 1; 15685} 15686 15687static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 15688{ 15689 HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 15690 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15691 15692 ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); 15693 ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); 15694 15695 | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); 15696 | LOAD_ADDR FCARG1a, ht 15697 if (opline->op1_type != IS_CONST) { 15698 | GET_ZVAL_PTR FCARG2a, op1_addr 15699 | EXT_CALL zend_hash_find, r0 15700 } else { 15701 zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); 15702 | LOAD_ADDR FCARG2a, str 15703 | EXT_CALL zend_hash_find_known_hash, r0 15704 } 15705 | test r0, r0 15706 if (exit_addr) { 15707 if (smart_branch_opcode == ZEND_JMPZ) { 15708 | jz &exit_addr 15709 } else { 15710 | jnz &exit_addr 15711 } 15712 } else if (smart_branch_opcode) { 15713 if (smart_branch_opcode == ZEND_JMPZ) { 15714 | jz =>target_label 15715 } else if (smart_branch_opcode == ZEND_JMPNZ) { 15716 | jnz =>target_label 15717 } else { 15718 ZEND_UNREACHABLE(); 15719 } 15720 } else { 15721 | setnz al 15722 | movzx eax, al 15723 | lea eax, [eax + IS_FALSE] 15724 | SET_ZVAL_TYPE_INFO res_addr, eax 15725 } 15726 15727 return 1; 15728} 15729 15730static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info) 15731{ 15732 uint32_t offset; 15733 15734 offset = (opline->opcode == ZEND_ROPE_INIT) ? 15735 opline->result.var : 15736 opline->op1.var + opline->extended_value * sizeof(zend_string*); 15737 15738 if (opline->op2_type == IS_CONST) { 15739 zval *zv = RT_CONSTANT(opline, opline->op2); 15740 zend_string *str; 15741 15742 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 15743 str = Z_STR_P(zv); 15744 | ADDR_STORE aword [FP + offset], str, r0 15745 } else { 15746 zend_jit_addr op2_addr = OP2_ADDR(); 15747 15748 ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 15749 15750 | GET_ZVAL_PTR r1, op2_addr 15751 | mov aword [FP + offset], r1 15752 if (opline->op2_type == IS_CV) { 15753 | GET_ZVAL_TYPE_INFO eax, op2_addr 15754 | TRY_ADDREF op2_info, ah, r1 15755 } 15756 } 15757 15758 if (opline->opcode == ZEND_ROPE_END) { 15759 zend_jit_addr res_addr = RES_ADDR(); 15760 15761 | lea FCARG1a, [FP + opline->op1.var] 15762 | mov FCARG2d, opline->extended_value 15763 | EXT_CALL zend_jit_rope_end, r0 15764 | SET_ZVAL_PTR res_addr, r0 15765 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX 15766 } 15767 15768 return 1; 15769} 15770 15771static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) 15772{ 15773 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15774 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15775 15776 if (!exit_addr) { 15777 return 0; 15778 } 15779 | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr 15780 15781 return 1; 15782} 15783 15784static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard) 15785{ 15786 zend_jit_addr var_addr = *var_addr_ptr; 15787 uint32_t var_info = *var_info_ptr; 15788 const void *exit_addr = NULL; 15789 15790 if (add_ref_guard || add_type_guard) { 15791 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15792 15793 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15794 if (!exit_addr) { 15795 return 0; 15796 } 15797 } 15798 15799 if (add_ref_guard) { 15800 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr 15801 } 15802 if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { 15803 /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ 15804 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 15805 | LOAD_ZVAL_ADDR FCARG1a, var_addr 15806 } 15807 | EXT_CALL zend_jit_unref_helper, r0 15808 } else { 15809 | GET_ZVAL_PTR FCARG1a, var_addr 15810 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 15811 *var_addr_ptr = var_addr; 15812 } 15813 15814 if (var_type != IS_UNKNOWN) { 15815 var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); 15816 } 15817 if (add_type_guard 15818 && var_type != IS_UNKNOWN 15819 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 15820 | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr 15821 15822 ZEND_ASSERT(var_info & (1 << var_type)); 15823 if (var_type < IS_STRING) { 15824 var_info = (1 << var_type); 15825 } else if (var_type != IS_ARRAY) { 15826 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 15827 } else { 15828 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 15829 } 15830 15831 *var_info_ptr = var_info; 15832 } else { 15833 var_info &= ~MAY_BE_REF; 15834 *var_info_ptr = var_info; 15835 } 15836 *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */ 15837 15838 return 1; 15839} 15840 15841static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard) 15842{ 15843 zend_jit_addr var_addr = *var_addr_ptr; 15844 uint32_t var_info = *var_info_ptr; 15845 int32_t exit_point; 15846 const void *exit_addr; 15847 15848 if (add_indirect_guard) { 15849 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15850 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15851 15852 if (!exit_addr) { 15853 return 0; 15854 } 15855 | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr 15856 | GET_ZVAL_PTR FCARG1a, var_addr 15857 } else { 15858 /* May be already loaded into FCARG1a or RAX by previous FETCH_OBJ_W/DIM_W */ 15859 if (opline->op1_type != IS_VAR || 15860 (opline-1)->result_type != IS_VAR || 15861 (opline-1)->result.var != opline->op1.var || 15862 (opline-1)->op1_type == IS_VAR || 15863 (opline-1)->op2_type == IS_VAR || 15864 (opline-1)->op2_type == IS_TMP_VAR) { 15865 | GET_ZVAL_PTR FCARG1a, var_addr 15866 } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { 15867 | mov FCARG1a, r0 15868 } 15869 } 15870 *var_info_ptr &= ~MAY_BE_INDIRECT; 15871 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 15872 *var_addr_ptr = var_addr; 15873 15874 if (var_type != IS_UNKNOWN) { 15875 var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); 15876 } 15877 if (!(var_type & IS_TRACE_REFERENCE) 15878 && var_type != IS_UNKNOWN 15879 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 15880 exit_point = zend_jit_trace_get_exit_point(opline, 0); 15881 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15882 15883 if (!exit_addr) { 15884 return 0; 15885 } 15886 15887 | IF_NOT_Z_TYPE FCARG1a, var_type, &exit_addr 15888 15889 //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); 15890 ZEND_ASSERT(var_info & (1 << var_type)); 15891 if (var_type < IS_STRING) { 15892 var_info = (1 << var_type); 15893 } else if (var_type != IS_ARRAY) { 15894 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 15895 } else { 15896 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 15897 } 15898 15899 *var_info_ptr = var_info; 15900 } 15901 15902 return 1; 15903} 15904 15905static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) 15906{ 15907 if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { 15908 return 0; 15909 } 15910 15911 switch (opline->opcode) { 15912 case ZEND_QM_ASSIGN: 15913 case ZEND_SEND_VAR: 15914 case ZEND_ASSIGN: 15915 case ZEND_PRE_INC: 15916 case ZEND_PRE_DEC: 15917 case ZEND_POST_INC: 15918 case ZEND_POST_DEC: 15919 return 1; 15920 case ZEND_ADD: 15921 case ZEND_SUB: 15922 case ZEND_MUL: 15923 case ZEND_BW_OR: 15924 case ZEND_BW_AND: 15925 case ZEND_BW_XOR: 15926 if (def_var == ssa_op->result_def && 15927 use_var == ssa_op->op1_use) { 15928 return 1; 15929 } 15930 break; 15931 default: 15932 break; 15933 } 15934 return 0; 15935} 15936 15937static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) 15938{ 15939 uint32_t op1_info, op2_info; 15940 15941 switch (opline->opcode) { 15942 case ZEND_SEND_VAR: 15943 case ZEND_SEND_VAL: 15944 case ZEND_SEND_VAL_EX: 15945 return (opline->op2_type != IS_CONST); 15946 case ZEND_QM_ASSIGN: 15947 case ZEND_IS_SMALLER: 15948 case ZEND_IS_SMALLER_OR_EQUAL: 15949 case ZEND_IS_EQUAL: 15950 case ZEND_IS_NOT_EQUAL: 15951 case ZEND_IS_IDENTICAL: 15952 case ZEND_IS_NOT_IDENTICAL: 15953 case ZEND_CASE: 15954 return 1; 15955 case ZEND_RETURN: 15956 return (op_array->type != ZEND_EVAL_CODE && op_array->function_name); 15957 case ZEND_ASSIGN: 15958 op1_info = OP1_INFO(); 15959 op2_info = OP2_INFO(); 15960 return 15961 opline->op1_type == IS_CV && 15962 !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) && 15963 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); 15964 case ZEND_ADD: 15965 case ZEND_SUB: 15966 case ZEND_MUL: 15967 op1_info = OP1_INFO(); 15968 op2_info = OP2_INFO(); 15969 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); 15970 case ZEND_BW_OR: 15971 case ZEND_BW_AND: 15972 case ZEND_BW_XOR: 15973 case ZEND_SL: 15974 case ZEND_SR: 15975 case ZEND_MOD: 15976 op1_info = OP1_INFO(); 15977 op2_info = OP2_INFO(); 15978 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); 15979 case ZEND_PRE_INC: 15980 case ZEND_PRE_DEC: 15981 case ZEND_POST_INC: 15982 case ZEND_POST_DEC: 15983 op1_info = OP1_INFO(); 15984 op2_info = OP1_DEF_INFO(); 15985 return opline->op1_type == IS_CV 15986 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)) 15987 && (op2_info & MAY_BE_LONG); 15988 case ZEND_STRLEN: 15989 op1_info = OP1_INFO(); 15990 return (opline->op1_type & (IS_CV|IS_CONST)) 15991 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING; 15992 case ZEND_COUNT: 15993 op1_info = OP1_INFO(); 15994 return (opline->op1_type & (IS_CV|IS_CONST)) 15995 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY; 15996 case ZEND_JMPZ: 15997 case ZEND_JMPNZ: 15998 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 15999 if (!ssa->cfg.map) { 16000 return 0; 16001 } 16002 if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start && 16003 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { 16004 return 0; 16005 } 16006 } 16007 ZEND_FALLTHROUGH; 16008 case ZEND_BOOL: 16009 case ZEND_BOOL_NOT: 16010 case ZEND_JMPZ_EX: 16011 case ZEND_JMPNZ_EX: 16012 return 1; 16013 case ZEND_FETCH_CONSTANT: 16014 return 1; 16015 case ZEND_FETCH_DIM_R: 16016 op1_info = OP1_INFO(); 16017 op2_info = OP2_INFO(); 16018 if (trace 16019 && trace->op1_type != IS_UNKNOWN 16020 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { 16021 op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); 16022 } 16023 return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && 16024 (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && 16025 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || 16026 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && 16027 (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); 16028 } 16029 return 0; 16030} 16031 16032static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) 16033{ 16034 if (ssa->vars[var].no_val) { 16035 /* we don't need the value */ 16036 return 0; 16037 } 16038 16039 if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { 16040 /* Disable global register allocation, 16041 * register allocation for SSA variables connected through Phi functions 16042 */ 16043 if (ssa->vars[var].definition_phi) { 16044 return 0; 16045 } 16046 if (ssa->vars[var].phi_use_chain) { 16047 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; 16048 do { 16049 if (!ssa->vars[phi->ssa_var].no_val) { 16050 return 0; 16051 } 16052 phi = zend_ssa_next_use_phi(ssa, var, phi); 16053 } while (phi); 16054 } 16055 } 16056 16057 if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && 16058 ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { 16059 /* bad type */ 16060 return 0; 16061 } 16062 16063 return 1; 16064} 16065 16066static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) 16067{ 16068 if (!zend_jit_var_supports_reg(ssa, var)) { 16069 return 0; 16070 } 16071 16072 if (ssa->vars[var].definition >= 0) { 16073 uint32_t def = ssa->vars[var].definition; 16074 if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { 16075 return 0; 16076 } 16077 } 16078 16079 if (ssa->vars[var].use_chain >= 0) { 16080 int use = ssa->vars[var].use_chain; 16081 16082 do { 16083 if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && 16084 !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { 16085 return 0; 16086 } 16087 use = zend_ssa_next_use(ssa->ops, var, use); 16088 } while (use >= 0); 16089 } 16090 16091 return 1; 16092} 16093 16094static bool zend_needs_extra_reg_for_const(const zend_op *opline, uint8_t op_type, znode_op op) 16095{ 16096|.if X64 16097|| if (op_type == IS_CONST) { 16098|| zval *zv = RT_CONSTANT(opline, op); 16099|| if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0 && !IS_SIGNED_32BIT(zv)) { 16100|| return 1; 16101|| } else if (Z_TYPE_P(zv) == IS_LONG && !IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 16102|| return 1; 16103|| } 16104|| } 16105|.endif 16106 return 0; 16107} 16108 16109static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 16110{ 16111 uint32_t op1_info, op2_info; 16112 16113 switch (opline->opcode) { 16114 case ZEND_FETCH_DIM_R: 16115 op1_info = OP1_INFO(); 16116 op2_info = OP2_INFO(); 16117 if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && 16118 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || 16119 ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && 16120 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { 16121 return ZEND_REGSET(ZREG_FCARG1); 16122 } 16123 break; 16124 default: 16125 break; 16126 } 16127 16128 return ZEND_REGSET_EMPTY; 16129} 16130 16131static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 16132{ 16133 uint32_t op1_info, op2_info, res_info; 16134 zend_regset regset = ZEND_REGSET_SCRATCH; 16135 16136 switch (opline->opcode) { 16137 case ZEND_NOP: 16138 case ZEND_OP_DATA: 16139 case ZEND_JMP: 16140 case ZEND_RETURN: 16141 regset = ZEND_REGSET_EMPTY; 16142 break; 16143 case ZEND_QM_ASSIGN: 16144 if (ssa_op->op1_def == current_var || 16145 ssa_op->result_def == current_var) { 16146 regset = ZEND_REGSET_EMPTY; 16147 break; 16148 } 16149 /* break missing intentionally */ 16150 case ZEND_SEND_VAL: 16151 case ZEND_SEND_VAL_EX: 16152 if (opline->op2_type == IS_CONST) { 16153 break; 16154 } 16155 if (ssa_op->op1_use == current_var) { 16156 regset = ZEND_REGSET(ZREG_R0); 16157 break; 16158 } 16159 op1_info = OP1_INFO(); 16160 if (!(op1_info & MAY_BE_UNDEF)) { 16161 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 16162 regset = ZEND_REGSET(ZREG_XMM0); 16163 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 16164 regset = ZEND_REGSET(ZREG_R0); 16165 } else { 16166 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 16167 } 16168 } 16169 break; 16170 case ZEND_SEND_VAR: 16171 if (opline->op2_type == IS_CONST) { 16172 break; 16173 } 16174 if (ssa_op->op1_use == current_var || 16175 ssa_op->op1_def == current_var) { 16176 regset = ZEND_REGSET_EMPTY; 16177 break; 16178 } 16179 op1_info = OP1_INFO(); 16180 if (!(op1_info & MAY_BE_UNDEF)) { 16181 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 16182 regset = ZEND_REGSET(ZREG_XMM0); 16183 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 16184 } else { 16185 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 16186 if (op1_info & MAY_BE_REF) { 16187 ZEND_REGSET_INCL(regset, ZREG_R1); 16188 } 16189 } 16190 } 16191 break; 16192 case ZEND_ASSIGN: 16193 if (ssa_op->op2_use == current_var || 16194 ssa_op->op2_def == current_var || 16195 ssa_op->op1_def == current_var || 16196 ssa_op->result_def == current_var) { 16197 regset = ZEND_REGSET_EMPTY; 16198 break; 16199 } 16200 op1_info = OP1_INFO(); 16201 op2_info = OP2_INFO(); 16202 if (opline->op1_type == IS_CV 16203 && !(op2_info & MAY_BE_UNDEF) 16204 && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 16205 if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 16206 regset = ZEND_REGSET(ZREG_XMM0); 16207 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 16208 regset = ZEND_REGSET(ZREG_R0); 16209 } else { 16210 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 16211 } 16212 } 16213 break; 16214 case ZEND_PRE_INC: 16215 case ZEND_PRE_DEC: 16216 case ZEND_POST_INC: 16217 case ZEND_POST_DEC: 16218 if (ssa_op->op1_use == current_var || 16219 ssa_op->op1_def == current_var || 16220 ssa_op->result_def == current_var) { 16221 regset = ZEND_REGSET_EMPTY; 16222 break; 16223 } 16224 op1_info = OP1_INFO(); 16225 if (opline->op1_type == IS_CV 16226 && (op1_info & MAY_BE_LONG) 16227 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16228 regset = ZEND_REGSET_EMPTY; 16229 if (op1_info & MAY_BE_DOUBLE) { 16230 regset = ZEND_REGSET(ZREG_XMM0); 16231 } 16232 if (opline->result_type != IS_UNUSED && (op1_info & MAY_BE_LONG)) { 16233 ZEND_REGSET_INCL(regset, ZREG_R1); 16234 } 16235 } 16236 break; 16237 case ZEND_ADD: 16238 case ZEND_SUB: 16239 case ZEND_MUL: 16240 op1_info = OP1_INFO(); 16241 op2_info = OP2_INFO(); 16242 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 16243 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16244 16245 regset = ZEND_REGSET_EMPTY; 16246 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 16247 if (ssa_op->result_def != current_var && 16248 (ssa_op->op1_use != current_var || !last_use)) { 16249 ZEND_REGSET_INCL(regset, ZREG_R0); 16250 } 16251 res_info = RES_INFO(); 16252 if (res_info & MAY_BE_DOUBLE) { 16253 ZEND_REGSET_INCL(regset, ZREG_R0); 16254 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16255 ZEND_REGSET_INCL(regset, ZREG_XMM1); 16256 } else if (res_info & MAY_BE_GUARD) { 16257 ZEND_REGSET_INCL(regset, ZREG_R0); 16258 } 16259 } 16260 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 16261 if (opline->op1_type == IS_CONST) { 16262 ZEND_REGSET_INCL(regset, ZREG_R0); 16263 } 16264 if (ssa_op->result_def != current_var) { 16265 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16266 } 16267 } 16268 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 16269 if (opline->op2_type == IS_CONST) { 16270 ZEND_REGSET_INCL(regset, ZREG_R0); 16271 } 16272 if (zend_is_commutative(opline->opcode)) { 16273 if (ssa_op->result_def != current_var) { 16274 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16275 } 16276 } else { 16277 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16278 if (ssa_op->result_def != current_var && 16279 (ssa_op->op1_use != current_var || !last_use)) { 16280 ZEND_REGSET_INCL(regset, ZREG_XMM1); 16281 } 16282 } 16283 } 16284 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 16285 if (ssa_op->result_def != current_var && 16286 (ssa_op->op1_use != current_var || !last_use) && 16287 (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { 16288 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16289 } 16290 } 16291 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16292 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16293 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 16294 ZEND_REGSET_INCL(regset, ZREG_R0); 16295 } else { 16296 ZEND_REGSET_INCL(regset, ZREG_R1); 16297 } 16298 } 16299 } 16300 break; 16301 case ZEND_BW_OR: 16302 case ZEND_BW_AND: 16303 case ZEND_BW_XOR: 16304 op1_info = OP1_INFO(); 16305 op2_info = OP2_INFO(); 16306 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16307 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16308 regset = ZEND_REGSET_EMPTY; 16309 if (ssa_op->result_def != current_var && 16310 (ssa_op->op1_use != current_var || !last_use)) { 16311 ZEND_REGSET_INCL(regset, ZREG_R0); 16312 } 16313 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16314 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16315 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 16316 ZEND_REGSET_INCL(regset, ZREG_R0); 16317 } else { 16318 ZEND_REGSET_INCL(regset, ZREG_R1); 16319 } 16320 } 16321 } 16322 break; 16323 case ZEND_SL: 16324 case ZEND_SR: 16325 op1_info = OP1_INFO(); 16326 op2_info = OP2_INFO(); 16327 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16328 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16329bw_op: 16330 regset = ZEND_REGSET_EMPTY; 16331 if (ssa_op->result_def != current_var && 16332 (ssa_op->op1_use != current_var || !last_use)) { 16333 ZEND_REGSET_INCL(regset, ZREG_R0); 16334 } 16335 if (opline->op2_type != IS_CONST && ssa_op->op2_use != current_var) { 16336 ZEND_REGSET_INCL(regset, ZREG_R1); 16337 } 16338 } 16339 break; 16340 case ZEND_MOD: 16341 op1_info = OP1_INFO(); 16342 op2_info = OP2_INFO(); 16343 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16344 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16345 if (opline->op2_type == IS_CONST && 16346 opline->op1_type != IS_CONST && 16347 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG && 16348 zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2))) && 16349 OP1_HAS_RANGE() && 16350 OP1_MIN_RANGE() >= 0) { 16351 /* MOD is going to be optimized into AND */ 16352 goto bw_op; 16353 } else { 16354 regset = ZEND_REGSET_EMPTY; 16355 ZEND_REGSET_INCL(regset, ZREG_R0); 16356 ZEND_REGSET_INCL(regset, ZREG_R2); 16357 if (opline->op2_type == IS_CONST) { 16358 ZEND_REGSET_INCL(regset, ZREG_R1); 16359 } 16360 } 16361 } 16362 break; 16363 case ZEND_IS_SMALLER: 16364 case ZEND_IS_SMALLER_OR_EQUAL: 16365 case ZEND_IS_EQUAL: 16366 case ZEND_IS_NOT_EQUAL: 16367 case ZEND_IS_IDENTICAL: 16368 case ZEND_IS_NOT_IDENTICAL: 16369 case ZEND_CASE: 16370 op1_info = OP1_INFO(); 16371 op2_info = OP2_INFO(); 16372 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 16373 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16374 regset = ZEND_REGSET_EMPTY; 16375 if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { 16376 ZEND_REGSET_INCL(regset, ZREG_R0); 16377 } 16378 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && 16379 opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { 16380 if (ssa_op->op1_use != current_var && 16381 ssa_op->op2_use != current_var) { 16382 ZEND_REGSET_INCL(regset, ZREG_R0); 16383 } 16384 } 16385 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 16386 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16387 } 16388 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 16389 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16390 } 16391 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 16392 if (ssa_op->op1_use != current_var && 16393 ssa_op->op2_use != current_var) { 16394 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16395 } 16396 } 16397 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16398 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16399 ZEND_REGSET_INCL(regset, ZREG_R0); 16400 } 16401 } 16402 break; 16403 case ZEND_BOOL: 16404 case ZEND_BOOL_NOT: 16405 case ZEND_JMPZ: 16406 case ZEND_JMPNZ: 16407 case ZEND_JMPZ_EX: 16408 case ZEND_JMPNZ_EX: 16409 op1_info = OP1_INFO(); 16410 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16411 regset = ZEND_REGSET_EMPTY; 16412 if (op1_info & MAY_BE_DOUBLE) { 16413 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16414 } 16415 if (opline->opcode == ZEND_BOOL || 16416 opline->opcode == ZEND_BOOL_NOT || 16417 opline->opcode == ZEND_JMPZ_EX || 16418 opline->opcode == ZEND_JMPNZ_EX) { 16419 ZEND_REGSET_INCL(regset, ZREG_R0); 16420 } 16421 } 16422 break; 16423 case ZEND_DO_UCALL: 16424 case ZEND_DO_FCALL: 16425 case ZEND_DO_FCALL_BY_NAME: 16426 case ZEND_INCLUDE_OR_EVAL: 16427 case ZEND_GENERATOR_CREATE: 16428 case ZEND_YIELD: 16429 case ZEND_YIELD_FROM: 16430 regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); 16431 break; 16432 default: 16433 break; 16434 } 16435 16436 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 16437 if (ssa_op == ssa->ops 16438 && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL 16439 && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { 16440 ZEND_REGSET_INCL(regset, ZREG_R0); 16441 ZEND_REGSET_INCL(regset, ZREG_R1); 16442 } 16443 } 16444 16445 /* %r0 is used to check EG(vm_interrupt) */ 16446 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 16447 if (ssa_op == ssa->ops 16448 && (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP || 16449 JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) { 16450#if ZTS 16451 ZEND_REGSET_INCL(regset, ZREG_R0); 16452#else 16453 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { 16454 ZEND_REGSET_INCL(regset, ZREG_R0); 16455 } 16456#endif 16457 } 16458 } else { 16459 uint32_t b = ssa->cfg.map[ssa_op - ssa->ops]; 16460 16461 if ((ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) != 0 16462 && ssa->cfg.blocks[b].start == ssa_op - ssa->ops) { 16463#if ZTS 16464 ZEND_REGSET_INCL(regset, ZREG_R0); 16465#else 16466 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { 16467 ZEND_REGSET_INCL(regset, ZREG_R0); 16468 } 16469#endif 16470 } 16471 } 16472 16473 return regset; 16474} 16475 16476/* 16477 * Local variables: 16478 * tab-width: 4 16479 * c-basic-offset: 4 16480 * indent-tabs-mode: t 16481 * End: 16482 */ 16483
