1/* 2 * +----------------------------------------------------------------------+ 3 * | Zend JIT | 4 * +----------------------------------------------------------------------+ 5 * | Copyright (c) The PHP Group | 6 * +----------------------------------------------------------------------+ 7 * | This source file is subject to version 3.01 of the PHP license, | 8 * | that is bundled with this package in the file LICENSE, and is | 9 * | available through the world-wide-web at the following url: | 10 * | https://www.php.net/license/3_01.txt | 11 * | If you did not receive a copy of the PHP license and are unable to | 12 * | obtain it through the world-wide-web, please send a note to | 13 * | license@php.net so we can mail you a copy immediately. | 14 * +----------------------------------------------------------------------+ 15 * | Authors: Dmitry Stogov <dmitry@php.net> | 16 * | Xinchen Hui <laruence@php.net> | 17 * +----------------------------------------------------------------------+ 18 */ 19 20|.if X64 21 |.arch x64 22|.else 23 |.arch x86 24|.endif 25 26|.if X64WIN 27 |.define FP, r14 28 |.define IP, r15 29 |.define IPl, r15d 30 |.define RX, r15 // the same as VM IP reused as a general purpose reg 31 |.define CARG1, rcx // x64/POSIX C call arguments. 32 |.define CARG2, rdx 33 |.define CARG3, r8 34 |.define CARG4, r9 35 |.define CARG1d, ecx 36 |.define CARG2d, edx 37 |.define CARG3d, r8d 38 |.define CARG4d, r9d 39 |.define FCARG1a, CARG1 // Simulate x86 fastcall. 40 |.define FCARG2a, CARG2 41 |.define FCARG1d, CARG1d 42 |.define FCARG2d, CARG2d 43 |.define SPAD, 0x58 // padding for CPU stack alignment 44 |.define NR_SPAD, 0x58 // padding for CPU stack alignment 45 |.define T3, [r4+0x50] // Used to store old value of IP 46 |.define T2, [r4+0x48] // Used to store old value of FP 47 |.define T1, [r4+0x40] 48 |.define A6, [r4+0x28] // preallocated slot for 6-th argument 49 |.define A5, [r4+0x20] // preallocated slot for 5-th argument 50|.elif X64 51 |.define FP, r14 52 |.define IP, r15 53 |.define IPl, r15d 54 |.define RX, r15 // the same as VM IP reused as a general purpose reg 55 |.define CARG1, rdi // x64/POSIX C call arguments. 56 |.define CARG2, rsi 57 |.define CARG3, rdx 58 |.define CARG4, rcx 59 |.define CARG5, r8 60 |.define CARG6, r9 61 |.define CARG1d, edi 62 |.define CARG2d, esi 63 |.define CARG3d, edx 64 |.define CARG4d, ecx 65 |.define CARG5d, r8d 66 |.define CARG6d, r9d 67 |.define FCARG1a, CARG1 // Simulate x86 fastcall. 68 |.define FCARG2a, CARG2 69 |.define FCARG1d, CARG1d 70 |.define FCARG2d, CARG2d 71 |.define SPAD, 0x18 // padding for CPU stack alignment 72 |.define NR_SPAD, 0x28 // padding for CPU stack alignment 73 |.define T3, [r4+0x20] // Used to store old value of IP (CALL VM only) 74 |.define T2, [r4+0x18] // Used to store old value of FP (CALL VM only) 75 |.define T1, [r4] 76|.else 77 |.define FP, esi 78 |.define IP, edi 79 |.define IPl, edi 80 |.define RX, edi // the same as VM IP reused as a general purpose reg 81 |.define FCARG1a, ecx // x86 fastcall arguments. 82 |.define FCARG2a, edx 83 |.define FCARG1d, ecx 84 |.define FCARG2d, edx 85 |.define SPAD, 0x1c // padding for CPU stack alignment 86 |.define NR_SPAD, 0x1c // padding for CPU stack alignment 87 |.define T3, [r4+0x18] // Used to store old value of IP (CALL VM only) 88 |.define T2, [r4+0x14] // Used to store old value of FP (CALL VM only) 89 |.define T1, [r4] 90 |.define A4, [r4+0xC] // preallocated slots for arguments of "cdecl" functions (intersect with T1) 91 |.define A3, [r4+0x8] 92 |.define A2, [r4+0x4] 93 |.define A1, [r4] 94|.endif 95 96|.define HYBRID_SPAD, 16 // padding for stack alignment 97 98#ifdef _WIN64 99# define TMP_ZVAL_OFFSET 0x20 100#else 101# define TMP_ZVAL_OFFSET 0 102#endif 103 104#define DASM_ALIGNMENT 16 105 106/* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to 107 * guarantee proper alignment of 128-bit SSE data allocated on stack. 108 * With broken alignment any execution of SSE code, including calls to 109 * memcpy() and others, may lead to crash. 110 */ 111 112const char* zend_reg_name[] = { 113#if defined(__x86_64__) || defined(_M_X64) 114 "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", 115 "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", 116 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", 117 "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15" 118#else 119 "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", 120 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" 121#endif 122}; 123 124/* Simulate x86 fastcall */ 125#ifdef _WIN64 126# define ZREG_FCARG1 ZREG_RCX 127# define ZREG_FCARG2 ZREG_RDX 128#elif defined(__x86_64__) 129# define ZREG_FCARG1 ZREG_RDI 130# define ZREG_FCARG2 ZREG_RSI 131#else 132# define ZREG_FCARG1 ZREG_RCX 133# define ZREG_FCARG2 ZREG_RDX 134#endif 135 136|.type EX, zend_execute_data, FP 137|.type OP, zend_op 138|.type ZVAL, zval 139|.actionlist dasm_actions 140|.globals zend_lb 141|.section code, cold_code, jmp_table 142 143static void* dasm_labels[zend_lb_MAX]; 144 145#if ZTS 146static size_t tsrm_ls_cache_tcb_offset = 0; 147static size_t tsrm_tls_index; 148static size_t tsrm_tls_offset; 149#endif 150 151#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) 152 153#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) 154 155/* Call range is before or after 2GB */ 156#define MAY_USE_32BIT_ADDR(addr) \ 157 (IS_SIGNED_32BIT((char*)(addr) - (char*)dasm_buf) && \ 158 IS_SIGNED_32BIT((char*)(addr) - (char*)dasm_end)) 159 160#define CAN_USE_AVX() (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) 161 162/* Not Implemented Yet */ 163|.macro NIY 164|| //ZEND_ASSERT(0); 165| int3 166|.endmacro 167 168|.macro NIY_STUB 169|| //ZEND_ASSERT(0); 170| int3 171|.endmacro 172 173|.macro ADD_HYBRID_SPAD 174||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 175| add r4, HYBRID_SPAD 176||#endif 177|.endmacro 178 179|.macro SUB_HYBRID_SPAD 180||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 181| sub r4, HYBRID_SPAD 182||#endif 183|.endmacro 184 185|.macro LOAD_ADDR, reg, addr 186| .if X64 187|| if (IS_SIGNED_32BIT(addr)) { 188| mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 <imm-32-bit> 189|| } else { 190| mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 <imm-64-bit> 191|| } 192| .else 193| mov reg, ((ptrdiff_t)addr) 194| .endif 195|.endmacro 196 197|.macro LOAD_TSRM_CACHE, reg 198| .if X64WIN 199| gs 200| mov reg, aword [0x58] 201| mov reg, aword [reg+tsrm_tls_index] 202| mov reg, aword [reg+tsrm_tls_offset] 203| .elif WIN 204| fs 205| mov reg, aword [0x2c] 206| mov reg, aword [reg+tsrm_tls_index] 207| mov reg, aword [reg+tsrm_tls_offset] 208| .elif X64APPLE 209| gs 210|| if (tsrm_ls_cache_tcb_offset) { 211| mov reg, aword [tsrm_ls_cache_tcb_offset] 212|| } else { 213| mov reg, aword [tsrm_tls_index] 214| mov reg, aword [reg+tsrm_tls_offset] 215|| } 216| .elif X64 217| fs 218|| if (tsrm_ls_cache_tcb_offset) { 219| mov reg, aword [tsrm_ls_cache_tcb_offset] 220|| } else { 221| mov reg, [0x8] 222| mov reg, aword [reg+tsrm_tls_index] 223| mov reg, aword [reg+tsrm_tls_offset] 224|| } 225| .else 226| gs 227|| if (tsrm_ls_cache_tcb_offset) { 228| mov reg, aword [tsrm_ls_cache_tcb_offset] 229|| } else { 230| mov reg, [0x4] 231| mov reg, aword [reg+tsrm_tls_index] 232| mov reg, aword [reg+tsrm_tls_offset] 233|| } 234| .endif 235|.endmacro 236 237|.macro LOAD_ADDR_ZTS, reg, struct, field 238| .if ZTS 239| LOAD_TSRM_CACHE reg 240| lea reg, aword [reg + (struct.._offset + offsetof(zend_..struct, field))] 241| .else 242| LOAD_ADDR reg, &struct.field 243| .endif 244|.endmacro 245 246|.macro PUSH_ADDR, addr, tmp_reg 247| .if X64 248|| if (IS_SIGNED_32BIT(addr)) { 249| push ((ptrdiff_t)addr) 250|| } else { 251| mov64 tmp_reg, ((ptrdiff_t)addr) 252| push tmp_reg 253|| } 254| .else 255| push ((ptrdiff_t)addr) 256| .endif 257|.endmacro 258 259|.macro ADDR_STORE, mem, addr, tmp_reg 260| .if X64 261|| if (IS_SIGNED_32BIT(addr)) { 262| mov mem, ((ptrdiff_t)addr) 263|| } else { 264| mov64 tmp_reg, ((ptrdiff_t)addr) 265| mov mem, tmp_reg 266|| } 267| .else 268| mov mem, ((ptrdiff_t)addr) 269| .endif 270|.endmacro 271 272|.macro ADDR_CMP, mem, addr, tmp_reg 273| .if X64 274|| if (IS_SIGNED_32BIT(addr)) { 275| cmp mem, ((ptrdiff_t)addr) 276|| } else { 277| mov64 tmp_reg, ((ptrdiff_t)addr) 278| cmp mem, tmp_reg 279|| } 280| .else 281| cmp mem, ((ptrdiff_t)addr) 282| .endif 283|.endmacro 284 285|.macro PUSH_ADDR_ZTS, struct, field, tmp_reg 286| .if ZTS 287| LOAD_TSRM_CACHE tmp_reg 288| lea tmp_reg, aword [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))] 289| push tmp_reg 290| .else 291| PUSH_ADDR &struct.field, tmp_reg 292| .endif 293|.endmacro 294 295|.macro _MEM_OP, mem_ins, prefix, addr, op2, tmp_reg 296| .if X64 297|| if (IS_SIGNED_32BIT(addr)) { 298| mem_ins prefix [addr], op2 299|| } else { 300| mov64 tmp_reg, ((ptrdiff_t)addr) 301| mem_ins prefix [tmp_reg], op2 302|| } 303| .else 304| mem_ins prefix [addr], op2 305| .endif 306|.endmacro 307 308|.macro MEM_LOAD_OP, mem_ins, reg, prefix, addr, tmp_reg 309| .if X64 310|| if (IS_SIGNED_32BIT(addr)) { 311| mem_ins reg, prefix [addr] 312|| } else { 313| mov64 tmp_reg, ((ptrdiff_t)addr) 314| mem_ins reg, prefix [tmp_reg] 315|| } 316| .else 317| mem_ins reg, prefix [addr] 318| .endif 319|.endmacro 320 321|.macro MEM_LOAD, op1, prefix, addr, tmp_reg 322| MEM_LOAD_OP mov, op1, prefix, addr, tmp_reg 323|.endmacro 324 325|.macro _MEM_OP_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg 326| .if ZTS 327| LOAD_TSRM_CACHE tmp_reg 328| mem_ins prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))], op2 329| .else 330| _MEM_OP mem_ins, prefix, &struct.field, op2, tmp_reg 331| .endif 332|.endmacro 333 334|.macro MEM_STORE_ZTS, prefix, struct, field, op2, tmp_reg 335| _MEM_OP_ZTS mov, prefix, struct, field, op2, tmp_reg 336|.endmacro 337 338|.macro MEM_CMP_ZTS, prefix, struct, field, op2, tmp_reg 339| _MEM_OP_ZTS cmp, prefix, struct, field, op2, tmp_reg 340|.endmacro 341 342|.macro MEM_UPDATE_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg 343| _MEM_OP_ZTS mem_ins, prefix, struct, field, op2, tmp_reg 344|.endmacro 345 346|.macro MEM_LOAD_OP_ZTS, mem_ins, reg, prefix, struct, field, tmp_reg 347| .if ZTS 348| LOAD_TSRM_CACHE tmp_reg 349| mem_ins reg, prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))] 350| .else 351| MEM_LOAD_OP mem_ins, reg, prefix, &struct.field, tmp_reg 352| .endif 353|.endmacro 354 355|.macro MEM_LOAD_ZTS, reg, prefix, struct, field, tmp_reg 356| MEM_LOAD_OP_ZTS mov, reg, prefix, struct, field, tmp_reg 357|.endmacro 358 359|.macro EXT_CALL, func, tmp_reg 360| .if X64 361|| if (MAY_USE_32BIT_ADDR(func)) { 362| call qword &func 363|| } else { 364| LOAD_ADDR tmp_reg, func 365| call tmp_reg 366|| } 367| .else 368| call dword &func 369| .endif 370|.endmacro 371 372|.macro EXT_JMP, func, tmp_reg 373| .if X64 374|| if (MAY_USE_32BIT_ADDR(func)) { 375| jmp qword &func 376|| } else { 377| LOAD_ADDR tmp_reg, func 378| jmp tmp_reg 379|| } 380| .else 381| jmp dword &func 382| .endif 383|.endmacro 384 385|.macro SAVE_IP 386|| if (GCC_GLOBAL_REGS) { 387| mov aword EX->opline, IP 388|| } 389|.endmacro 390 391|.macro LOAD_IP 392|| if (GCC_GLOBAL_REGS) { 393| mov IP, aword EX->opline 394|| } 395|.endmacro 396 397|.macro LOAD_IP_ADDR, addr 398|| if (GCC_GLOBAL_REGS) { 399| LOAD_ADDR IP, addr 400|| } else { 401| ADDR_STORE aword EX->opline, addr, RX 402|| } 403|.endmacro 404 405|.macro LOAD_IP_ADDR_ZTS, struct, field 406| .if ZTS 407|| if (GCC_GLOBAL_REGS) { 408| LOAD_TSRM_CACHE IP 409| mov IP, aword [IP + (struct.._offset + offsetof(zend_..struct, field))] 410|| } else { 411| LOAD_TSRM_CACHE RX 412| lea RX, aword [RX + (struct.._offset + offsetof(zend_..struct, field))] 413| mov aword EX->opline, RX 414|| } 415| .else 416| LOAD_IP_ADDR &struct.field 417| .endif 418|.endmacro 419 420|.macro GET_IP, reg 421|| if (GCC_GLOBAL_REGS) { 422| mov reg, IP 423|| } else { 424| mov reg, aword EX->opline 425|| } 426|.endmacro 427 428|.macro ADD_IP, val 429|| if (GCC_GLOBAL_REGS) { 430| add IP, val 431|| } else { 432| add aword EX->opline, val 433|| } 434|.endmacro 435 436|.macro JMP_IP 437|| if (GCC_GLOBAL_REGS) { 438| jmp aword [IP] 439|| } else { 440| mov r0, aword EX:FCARG1a->opline 441| jmp aword [r0] 442|| } 443|.endmacro 444 445/* In 64-bit build we compare only low 32-bits. 446 * x86_64 cmp instruction doesn't support immediate 64-bit operand, and full 447 * comparison would require an additional load of 64-bit address into register. 448 * This is not a problem at all, while JIT buffer size is less than 4GB. 449 */ 450|.macro CMP_IP, addr 451|| if (GCC_GLOBAL_REGS) { 452| cmp IPl, addr 453|| } else { 454| cmp dword EX->opline, addr 455|| } 456|.endmacro 457 458|.macro LOAD_ZVAL_ADDR, reg, addr 459|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 460| LOAD_ADDR reg, Z_ZV(addr) 461|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 462|| if (Z_OFFSET(addr)) { 463| lea reg, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 464|| } else { 465| mov reg, Ra(Z_REG(addr)) 466|| } 467|| } else { 468|| ZEND_UNREACHABLE(); 469|| } 470|.endmacro 471 472|.macro PUSH_ZVAL_ADDR, addr, tmp_reg 473|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 474| PUSH_ADDR Z_ZV(addr), tmp_reg 475|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 476|| if (Z_OFFSET(addr)) { 477| lea tmp_reg, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 478| push tmp_reg 479|| } else { 480| push Ra(Z_REG(addr)) 481|| } 482|| } else { 483|| ZEND_UNREACHABLE(); 484|| } 485|.endmacro 486 487|.macro GET_Z_TYPE_INFO, reg, zv 488| mov reg, dword [zv+offsetof(zval,u1.type_info)] 489|.endmacro 490 491|.macro SET_Z_TYPE_INFO, zv, type 492| mov dword [zv+offsetof(zval,u1.type_info)], type 493|.endmacro 494 495|.macro GET_ZVAL_TYPE, reg, addr 496|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 497| mov reg, byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.v.type)] 498|.endmacro 499 500|.macro GET_ZVAL_TYPE_INFO, reg, addr 501|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 502| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)] 503|.endmacro 504 505|.macro SET_ZVAL_TYPE_INFO, addr, type 506|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 507| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)], type 508|.endmacro 509 510|.macro GET_Z_PTR, reg, zv 511| mov reg, aword [zv] 512|.endmacro 513 514|.macro GET_Z_W2, reg, zv 515| mov reg, dword [zv+4] 516|.endmacro 517 518|.macro SET_Z_W2, zv, reg 519| mov dword [zv+4], reg 520|.endmacro 521 522|.macro GET_ZVAL_PTR, reg, addr 523|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 524| mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 525|.endmacro 526 527|.macro SET_ZVAL_PTR, addr, val 528|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 529| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], val 530|.endmacro 531 532|.macro GET_ZVAL_W2, reg, addr 533|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 534| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4] 535|.endmacro 536 537|.macro SET_ZVAL_W2, addr, val 538|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 539| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4], val 540|.endmacro 541 542|.macro UNDEF_OPLINE_RESULT 543| mov r0, EX->opline 544| mov eax, dword OP:r0->result.var 545| SET_Z_TYPE_INFO FP + r0, IS_UNDEF 546|.endmacro 547 548|.macro UNDEF_OPLINE_RESULT_IF_USED 549| test byte OP:RX->result_type, (IS_TMP_VAR|IS_VAR) 550| jz >1 551| mov eax, dword OP:RX->result.var 552| SET_Z_TYPE_INFO FP + r0, IS_UNDEF 553|1: 554|.endmacro 555 556|.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2 557|| if (CAN_USE_AVX()) { 558| avx_ins op1, op2 559|| } else { 560| sse_ins op1, op2 561|| } 562|.endmacro 563 564|.macro SSE_OP, sse_ins, reg, addr, tmp_reg 565|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 566| MEM_LOAD_OP sse_ins, xmm(reg-ZREG_XMM0), qword, Z_ZV(addr), tmp_reg 567|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 568| sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 569|| } else if (Z_MODE(addr) == IS_REG) { 570| sse_ins xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 571|| } else { 572|| ZEND_UNREACHABLE(); 573|| } 574|.endmacro 575 576|.macro DOUBLE_CMP, reg, addr 577|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 578| .if X64 579|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 580| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 581|| } else { 582| LOAD_ADDR r0, Z_ZV(addr) 583| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [r0] 584|| } 585| .else 586| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 587| .endif 588|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 589| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 590|| } else if (Z_MODE(addr) == IS_REG) { 591| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 592|| } else { 593|| ZEND_UNREACHABLE(); 594|| } 595|.endmacro 596 597|.macro DOUBLE_GET_LONG, reg, lval, tmp_reg 598|| if (lval == 0) { 599|| if (CAN_USE_AVX()) { 600| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 601|| } else { 602| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 603|| } 604|| } else { 605|.if X64 606|| if (!IS_SIGNED_32BIT(lval)) { 607| mov64 Ra(tmp_reg), lval 608|| } else { 609| mov Ra(tmp_reg), lval 610|| } 611|.else 612| mov Ra(tmp_reg), lval 613|.endif 614|| if (CAN_USE_AVX()) { 615| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 616| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(tmp_reg) 617|| } else { 618| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 619| cvtsi2sd, xmm(reg-ZREG_XMM0), Ra(tmp_reg) 620|| } 621|| } 622|.endmacro 623 624|.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg 625|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 626| DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg 627|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 628|| if (CAN_USE_AVX()) { 629| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 630| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 631|| } else { 632| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 633| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 634|| } 635|| } else if (Z_MODE(addr) == IS_REG) { 636|| if (CAN_USE_AVX()) { 637| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 638| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) 639|| } else { 640| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 641| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) 642|| } 643|| } else { 644|| ZEND_UNREACHABLE(); 645|| } 646|.endmacro 647 648|.macro DOUBLE_GET_ZVAL_DVAL, reg, addr 649|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { 650|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 651| .if X64 652|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 653| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 654|| } else { 655| LOAD_ADDR r0, Z_ZV(addr) 656| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [r0] 657|| } 658| .else 659| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 660| .endif 661|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 662| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 663|| } else if (Z_MODE(addr) == IS_REG) { 664| SSE_AVX_INS movaps, vmovaps, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 665|| } else { 666|| ZEND_UNREACHABLE(); 667|| } 668|| } 669|.endmacro 670 671|.macro SSE_MATH, opcode, reg, addr, tmp_reg 672|| switch (opcode) { 673|| case ZEND_ADD: 674| SSE_OP addsd, reg, addr, tmp_reg 675|| break; 676|| case ZEND_SUB: 677| SSE_OP subsd, reg, addr, tmp_reg 678|| break; 679|| case ZEND_MUL: 680| SSE_OP mulsd, reg, addr, tmp_reg 681|| break; 682|| case ZEND_DIV: 683| SSE_OP divsd, reg, addr, tmp_reg 684|| break; 685|| } 686|.endmacro 687 688|.macro SSE_MATH_REG, opcode, dst_reg, src_reg 689|| switch (opcode) { 690|| case ZEND_ADD: 691| addsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 692|| break; 693|| case ZEND_SUB: 694| subsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 695|| break; 696|| case ZEND_MUL: 697| mulsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 698|| break; 699|| case ZEND_DIV: 700| divsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 701|| break; 702|| } 703|.endmacro 704 705|.macro DOUBLE_SET_ZVAL_DVAL, addr, reg 706|| if (Z_MODE(addr) == IS_REG) { 707|| if (reg != Z_REG(addr)) { 708| SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0) 709|| } 710|| } else { 711|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 712| SSE_AVX_INS movsd, vmovsd, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0) 713|| } 714|.endmacro 715 716|.macro AVX_OP, avx_ins, reg, op1_reg, addr, tmp_reg 717|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 718| .if X64 719|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 720| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Z_ZV(addr)] 721|| } else { 722| mov64 tmp_reg, ((ptrdiff_t)Z_ZV(addr)) 723| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [tmp_reg] 724|| } 725| .else 726| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [addr] 727| .endif 728|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 729| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 730|| } else if (Z_MODE(addr) == IS_REG) { 731| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 732|| } else { 733|| ZEND_UNREACHABLE(); 734|| } 735|.endmacro 736 737|.macro AVX_MATH, opcode, reg, op1_reg, addr, tmp_reg 738|| switch (opcode) { 739|| case ZEND_ADD: 740| AVX_OP vaddsd, reg, op1_reg, addr, tmp_reg 741|| break; 742|| case ZEND_SUB: 743| AVX_OP vsubsd, reg, op1_reg, addr, tmp_reg 744|| break; 745|| case ZEND_MUL: 746| AVX_OP vmulsd, reg, op1_reg, addr, tmp_reg 747|| break; 748|| case ZEND_DIV: 749| AVX_OP vdivsd, reg, op1_reg, addr, tmp_reg 750|| break; 751|| } 752|.endmacro 753 754|.macro AVX_MATH_REG, opcode, dst_reg, op1_reg, src_reg 755|| switch (opcode) { 756|| case ZEND_ADD: 757| vaddsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 758|| break; 759|| case ZEND_SUB: 760| vsubsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 761|| break; 762|| case ZEND_MUL: 763| vmulsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 764|| break; 765|| case ZEND_DIV: 766| vdivsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 767|| break; 768|| } 769|.endmacro 770 771|.macro LONG_OP, long_ins, reg, addr, tmp_reg 772|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 773| .if X64 774|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { 775| mov64 tmp_reg, Z_LVAL_P(Z_ZV(addr)) 776| long_ins Ra(reg), tmp_reg 777|| } else { 778| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) 779|| } 780| .else 781| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) 782| .endif 783|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 784| long_ins Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 785|| } else if (Z_MODE(addr) == IS_REG) { 786| long_ins Ra(reg), Ra(Z_REG(addr)) 787|| } else { 788|| ZEND_UNREACHABLE(); 789|| } 790|.endmacro 791 792|.macro LONG_OP_WITH_32BIT_CONST, long_ins, op1_addr, lval 793|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { 794| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 795|| } else if (Z_MODE(op1_addr) == IS_REG) { 796| long_ins Ra(Z_REG(op1_addr)), lval 797|| } else { 798|| ZEND_UNREACHABLE(); 799|| } 800|.endmacro 801 802|.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval 803|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { 804| .if X64 805|| if (!IS_SIGNED_32BIT(lval)) { 806| mov64 r0, lval 807| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0 808|| } else { 809| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 810|| } 811| .else 812| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 813| .endif 814|| } else if (Z_MODE(op1_addr) == IS_REG) { 815| .if X64 816|| if (!IS_SIGNED_32BIT(lval)) { 817| mov64 r0, lval 818| long_ins Ra(Z_REG(op1_addr)), r0 819|| } else { 820| long_ins Ra(Z_REG(op1_addr)), lval 821|| } 822| .else 823| long_ins Ra(Z_REG(op1_addr)), lval 824| .endif 825|| } else { 826|| ZEND_UNREACHABLE(); 827|| } 828|.endmacro 829 830|.macro GET_ZVAL_LVAL, reg, addr 831|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 832|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { 833| xor Ra(reg), Ra(reg) 834|| } else { 835| .if X64 836|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { 837| mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr)) 838|| } else { 839| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) 840|| } 841| .else 842| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) 843| .endif 844|| } 845|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 846| mov Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 847|| } else if (Z_MODE(addr) == IS_REG) { 848|| if (reg != Z_REG(addr)) { 849| mov Ra(reg), Ra(Z_REG(addr)) 850|| } 851|| } else { 852|| ZEND_UNREACHABLE(); 853|| } 854|.endmacro 855 856|.macro LONG_MATH, opcode, reg, addr, tmp_reg 857|| switch (opcode) { 858|| case ZEND_ADD: 859| LONG_OP add, reg, addr, Ra(tmp_reg) 860|| break; 861|| case ZEND_SUB: 862| LONG_OP sub, reg, addr, Ra(tmp_reg) 863|| break; 864|| case ZEND_MUL: 865| LONG_OP imul, reg, addr, Ra(tmp_reg) 866|| break; 867|| case ZEND_BW_OR: 868| LONG_OP or, reg, addr, Ra(tmp_reg) 869|| break; 870|| case ZEND_BW_AND: 871| LONG_OP and, reg, addr, Ra(tmp_reg) 872|| break; 873|| case ZEND_BW_XOR: 874| LONG_OP xor, reg, addr, Ra(tmp_reg) 875|| break; 876|| default: 877|| ZEND_UNREACHABLE(); 878|| } 879|.endmacro 880 881|.macro LONG_MATH_REG, opcode, dst_reg, src_reg 882|| switch (opcode) { 883|| case ZEND_ADD: 884| add dst_reg, src_reg 885|| break; 886|| case ZEND_SUB: 887| sub dst_reg, src_reg 888|| break; 889|| case ZEND_MUL: 890| imul dst_reg, src_reg 891|| break; 892|| case ZEND_BW_OR: 893| or dst_reg, src_reg 894|| break; 895|| case ZEND_BW_AND: 896| and dst_reg, src_reg 897|| break; 898|| case ZEND_BW_XOR: 899| xor dst_reg, src_reg 900|| break; 901|| default: 902|| ZEND_UNREACHABLE(); 903|| } 904|.endmacro 905 906|.macro SET_ZVAL_LVAL, addr, lval 907|| if (Z_MODE(addr) == IS_REG) { 908| mov Ra(Z_REG(addr)), lval 909|| } else { 910|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 911| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval 912|| } 913|.endmacro 914 915|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg 916|| if (Z_TYPE_P(zv) > IS_TRUE) { 917|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 918|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; 919|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { 920|| if (CAN_USE_AVX()) { 921| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 922|| } else { 923| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 924|| } 925| .if X64 926|| } else if (!IS_SIGNED_32BIT(zv)) { 927| mov64 Ra(tmp_reg), ((uintptr_t)zv) 928| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] 929| .endif 930|| } else { 931| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] 932|| } 933| DOUBLE_SET_ZVAL_DVAL dst_addr, dst_reg 934|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 935|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; 936| DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), ZREG_R0 937| DOUBLE_SET_ZVAL_DVAL dst_addr, dst_reg 938|| } else if (Z_LVAL_P(zv) == 0 && Z_MODE(dst_addr) == IS_REG) { 939| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) 940|| } else { 941| .if X64 942|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 943|| if (Z_MODE(dst_addr) == IS_REG) { 944| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) 945|| } else { 946| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) 947| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) 948|| } 949|| } else { 950| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 951|| } 952| .else 953| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 954| .endif 955|| } 956|| } 957|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 958|| if (dst_def_info == MAY_BE_DOUBLE) { 959|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 960| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE 961|| } 962|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 963| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) 964|| } 965|| } 966|.endmacro 967 968|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg 969|| if (Z_TYPE_P(zv) > IS_TRUE) { 970|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 971|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? 972|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0); 973|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { 974|| if (CAN_USE_AVX()) { 975| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 976|| } else { 977| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 978|| } 979| .if X64 980|| } else if (!IS_SIGNED_32BIT(zv)) { 981| mov64 Ra(tmp_reg), ((uintptr_t)zv) 982| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] 983| .endif 984|| } else { 985| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] 986|| } 987| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 988| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 989|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 990|| if (Z_MODE(dst_addr) == IS_REG) { 991| DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), ZREG_R0 992| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) 993|| } else if (Z_MODE(res_addr) == IS_REG) { 994| DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), ZREG_R0 995| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) 996|| } else { 997| DOUBLE_GET_LONG ZREG_XMM0, Z_LVAL_P(zv), ZREG_R0 998| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 999| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 1000|| } 1001|| } else if (Z_LVAL_P(zv) == 0 && (Z_MODE(dst_addr) == IS_REG || Z_MODE(res_addr) == IS_REG)) { 1002|| if (Z_MODE(dst_addr) == IS_REG) { 1003| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) 1004| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1005|| } else { 1006| xor Ra(Z_REG(res_addr)), Ra(Z_REG(res_addr)) 1007| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1008|| } 1009|| } else { 1010| .if X64 1011|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1012|| if (Z_MODE(dst_addr) == IS_REG) { 1013| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) 1014| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1015|| } else if (Z_MODE(res_addr) == IS_REG) { 1016| mov64 Ra(Z_REG(res_addr)), ((uintptr_t)Z_LVAL_P(zv)) 1017| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1018|| } else { 1019| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) 1020| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) 1021| SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 1022|| } 1023|| } else if (Z_MODE(dst_addr) == IS_REG) { 1024| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1025| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1026|| } else if (Z_MODE(res_addr) == IS_REG) { 1027| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1028| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1029|| } else { 1030| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1031| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1032|| } 1033| .else 1034|| if (Z_MODE(dst_addr) == IS_REG) { 1035| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1036| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1037|| } else if (Z_MODE(res_addr) == IS_REG) { 1038| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1039| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1040|| } else { 1041| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1042| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1043|| } 1044| .endif 1045|| } 1046|| } 1047|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1048|| if (dst_def_info == MAY_BE_DOUBLE) { 1049|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1050| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE 1051|| } 1052|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1053| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) 1054|| } 1055|| } 1056|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1057|| if (dst_def_info == MAY_BE_DOUBLE) { 1058| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 1059|| } else { 1060| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv) 1061|| } 1062|| } 1063|.endmacro 1064 1065/* the same as above, but "src" may overlap with "tmp_reg1" */ 1066|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1067| ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1068|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1069|| !(src_info & MAY_BE_GUARD) && 1070|| has_concrete_type(src_info & MAY_BE_ANY)) { 1071|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1072|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) { 1073|| zend_uchar type = concrete_type(src_info); 1074| SET_ZVAL_TYPE_INFO dst_addr, type 1075|| } 1076|| } 1077|| } else { 1078| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr 1079| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) 1080|| } 1081|.endmacro 1082 1083|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1084|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1085|| if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) { 1086|| if (Z_MODE(src_addr) == IS_REG) { 1087|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1088| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) 1089|| } 1090|| } else if (Z_MODE(dst_addr) == IS_REG) { 1091| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr 1092|| } else { 1093| GET_ZVAL_LVAL tmp_reg2, src_addr 1094| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) 1095|| } 1096|| } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 1097|| if (Z_MODE(src_addr) == IS_REG) { 1098| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) 1099|| } else if (Z_MODE(dst_addr) == IS_REG) { 1100| DOUBLE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr 1101|| } else { 1102| DOUBLE_GET_ZVAL_DVAL ZREG_XMM0, src_addr 1103| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1104|| } 1105|| } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) { 1106| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1107| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1108|| } else { 1109| .if X64 1110| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1111| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1112| .else 1113|| if ((tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr))) { 1114| GET_ZVAL_W2 Ra(tmp_reg2), src_addr 1115| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) 1116| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1117| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1118|| } else { 1119| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1120| GET_ZVAL_W2 Ra(tmp_reg1), src_addr 1121| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1122| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) 1123|| } 1124| .endif 1125|| } 1126|| } 1127|.endmacro 1128 1129|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, tmp_reg1, tmp_reg2 1130|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1131|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) { 1132|| if (Z_MODE(src_addr) == IS_REG) { 1133|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1134| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) 1135|| } 1136|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) { 1137| SET_ZVAL_LVAL res_addr, Ra(Z_REG(src_addr)) 1138|| } 1139|| } else if (Z_MODE(dst_addr) == IS_REG) { 1140| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr 1141|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) { 1142| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1143|| } 1144|| } else if (Z_MODE(res_addr) == IS_REG) { 1145| GET_ZVAL_LVAL Z_REG(res_addr), src_addr 1146| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1147|| } else { 1148| GET_ZVAL_LVAL tmp_reg2, src_addr 1149| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) 1150| SET_ZVAL_LVAL res_addr, Ra(tmp_reg2) 1151|| } 1152|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 1153|| if (Z_MODE(src_addr) == IS_REG) { 1154| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) 1155| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(src_addr) 1156|| } else if (Z_MODE(dst_addr) == IS_REG) { 1157| DOUBLE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr 1158| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) 1159|| } else if (Z_MODE(res_addr) == IS_REG) { 1160| DOUBLE_GET_ZVAL_DVAL Z_REG(res_addr), src_addr 1161| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) 1162|| } else { 1163| DOUBLE_GET_ZVAL_DVAL ZREG_XMM0, src_addr 1164| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1165| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 1166|| } 1167|| } else if (!(src_info & MAY_BE_DOUBLE)) { 1168| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1169| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1170| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1171|| } else { 1172| .if X64 1173| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1174| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1175| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1176| .else 1177|| if (tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) { 1178| GET_ZVAL_W2 Ra(tmp_reg2), src_addr 1179| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) 1180| SET_ZVAL_W2 res_addr, Ra(tmp_reg2) 1181| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1182| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1183| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1184|| } else { 1185| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1186| GET_ZVAL_W2 Ra(tmp_reg1), src_addr 1187| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1188| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1189| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) 1190| SET_ZVAL_W2 res_addr, Ra(tmp_reg1) 1191|| } 1192| .endif 1193|| } 1194|| } 1195|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1196|| has_concrete_type(src_info & MAY_BE_ANY)) { 1197|| zend_uchar type = concrete_type(src_info); 1198|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1199|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { 1200| SET_ZVAL_TYPE_INFO dst_addr, type 1201|| } 1202|| } 1203|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1204| SET_ZVAL_TYPE_INFO res_addr, type 1205|| } 1206|| } else { 1207| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr 1208| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) 1209| SET_ZVAL_TYPE_INFO res_addr, Rd(tmp_reg1) 1210|| } 1211|.endmacro 1212 1213|.macro IF_UNDEF, type_reg, label 1214| test type_reg, type_reg 1215| je label 1216|.endmacro 1217 1218|.macro IF_TYPE, type, val, label 1219| cmp type, val 1220| je label 1221|.endmacro 1222 1223|.macro IF_NOT_TYPE, type, val, label 1224| cmp type, val 1225| jne label 1226|.endmacro 1227 1228|.macro IF_Z_TYPE, zv, val, label 1229| IF_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label 1230|.endmacro 1231 1232|.macro IF_NOT_Z_TYPE, zv, val, label 1233| IF_NOT_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label 1234|.endmacro 1235 1236|.macro CMP_ZVAL_TYPE, addr, val 1237|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1238| cmp byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val 1239|.endmacro 1240 1241|.macro IF_ZVAL_TYPE, addr, val, label 1242|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1243| IF_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label 1244|.endmacro 1245 1246|.macro IF_NOT_ZVAL_TYPE, addr, val, label 1247|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1248| IF_NOT_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label 1249|.endmacro 1250 1251|.macro IF_FLAGS, type_flags, mask, label 1252| test type_flags, mask 1253| jnz label 1254|.endmacro 1255 1256|.macro IF_NOT_FLAGS, type_flags, mask, label 1257| test type_flags, mask 1258| jz label 1259|.endmacro 1260 1261|.macro IF_REFCOUNTED, type_flags, label 1262| IF_FLAGS type_flags, IS_TYPE_REFCOUNTED, label 1263|.endmacro 1264 1265|.macro IF_NOT_REFCOUNTED, type_flags, label 1266| //IF_NOT_FLAGS type_flags, IS_TYPE_REFCOUNTED, label 1267| test type_flags, type_flags 1268| jz label 1269|.endmacro 1270 1271|.macro IF_ZVAL_FLAGS, addr, mask, label 1272|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1273| IF_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label 1274|.endmacro 1275 1276|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label 1277|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1278| IF_NOT_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label 1279|.endmacro 1280 1281|.macro IF_ZVAL_REFCOUNTED, addr, label 1282| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label 1283|.endmacro 1284 1285|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label 1286| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label 1287|.endmacro 1288 1289|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label 1290| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label 1291|.endmacro 1292 1293|.macro GC_ADDREF, zv 1294| add dword [zv], 1 1295|.endmacro 1296 1297|.macro GC_DELREF, zv 1298| sub dword [zv], 1 1299|.endmacro 1300 1301|.macro IF_GC_MAY_NOT_LEAK, ptr, label 1302| test dword [ptr+4],(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)) 1303| jne label 1304|.endmacro 1305 1306|.macro ADDREF_CONST, zv, tmp_reg 1307| .if X64 1308|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1309| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) 1310| add dword [tmp_reg], 1 1311|| } else { 1312| add dword [Z_LVAL_P(zv)], 1 1313|| } 1314| .else 1315| add dword [Z_LVAL_P(zv)], 1 1316| .endif 1317|.endmacro 1318 1319|.macro ADDREF_CONST_2, zv, tmp_reg 1320| .if X64 1321|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1322| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) 1323| add dword [tmp_reg], 2 1324|| } else { 1325| add dword [Z_LVAL_P(zv)], 2 1326|| } 1327| .else 1328| add dword [Z_LVAL_P(zv)], 2 1329| .endif 1330|.endmacro 1331 1332|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg 1333|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1334|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1335| IF_NOT_REFCOUNTED type_flags_reg, >1 1336|| } 1337| GC_ADDREF value_ptr_reg 1338|1: 1339|| } 1340|.endmacro 1341 1342|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg 1343|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1344|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1345| IF_NOT_REFCOUNTED type_flags_reg, >1 1346|| } 1347| add dword [value_ptr_reg], 2 1348|1: 1349|| } 1350|.endmacro 1351 1352|.macro ZVAL_DEREF, reg, info 1353|| if (info & MAY_BE_REF) { 1354| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1 1355| GET_Z_PTR reg, reg 1356| add reg, offsetof(zend_reference, val) 1357|1: 1358|| } 1359|.endmacro 1360 1361|.macro SET_EX_OPLINE, op, tmp_reg 1362|| if (op == last_valid_opline) { 1363|| zend_jit_use_last_valid_opline(); 1364| SAVE_IP 1365|| } else { 1366| ADDR_STORE aword EX->opline, op, tmp_reg 1367|| if (!GCC_GLOBAL_REGS) { 1368|| zend_jit_reset_last_valid_opline(); 1369|| } 1370|| } 1371|.endmacro 1372 1373// zval should be in FCARG1a 1374|.macro ZVAL_DTOR_FUNC, var_info, opline // arg1 must be in FCARG1a 1375|| do { 1376|| if (!((var_info) & MAY_BE_GUARD) 1377|| && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1378|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); 1379|| if (type == IS_STRING && !ZEND_DEBUG) { 1380| EXT_CALL _efree, r0 1381|| break; 1382|| } else if (type == IS_ARRAY) { 1383|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { 1384|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { 1385| SET_EX_OPLINE opline, r0 1386|| } 1387| EXT_CALL zend_array_destroy, r0 1388|| } else { 1389| EXT_CALL zend_jit_array_free, r0 1390|| } 1391|| break; 1392|| } else if (type == IS_OBJECT) { 1393|| if (opline) { 1394| SET_EX_OPLINE opline, r0 1395|| } 1396| EXT_CALL zend_objects_store_del, r0 1397|| break; 1398|| } 1399|| } 1400|| if (opline) { 1401| SET_EX_OPLINE opline, r0 1402|| } 1403| EXT_CALL rc_dtor_func, r0 1404|| } while(0); 1405|.endmacro 1406 1407|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline 1408|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) { 1409|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1410| // if (Z_REFCOUNTED_P(cv)) { 1411|| if (cold) { 1412| IF_ZVAL_REFCOUNTED addr, >1 1413|.cold_code 1414|1: 1415|| } else { 1416| IF_NOT_ZVAL_REFCOUNTED addr, >4 1417|| } 1418|| } 1419| // if (!Z_DELREF_P(cv)) { 1420| GET_ZVAL_PTR FCARG1a, addr 1421| GC_DELREF FCARG1a 1422|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) { 1423|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) { 1424|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1425| jnz >3 1426|| } else { 1427| jnz >4 1428|| } 1429|| } 1430| // zval_dtor_func(r); 1431| ZVAL_DTOR_FUNC op_info, opline 1432|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1433| jmp >4 1434|| } 1435|3: 1436|| } 1437|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1438|| if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) { 1439|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 1440| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1 1441| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4 1442| GET_ZVAL_PTR FCARG1a, ref_addr 1443|1: 1444|| } 1445| IF_GC_MAY_NOT_LEAK FCARG1a, >4 1446| // gc_possible_root(Z_COUNTED_P(z)) 1447| EXT_CALL gc_possible_root, r0 1448|| } 1449|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { 1450| jmp >4 1451|.code 1452|| } 1453|4: 1454|| } 1455|.endmacro 1456 1457|.macro FREE_OP, op_type, op, op_info, cold, opline 1458|| if (op_type & (IS_VAR|IS_TMP_VAR)) { 1459| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, opline 1460|| } 1461|.endmacro 1462 1463|.macro SEPARATE_ARRAY, addr, op_info, cold 1464|| if (RC_MAY_BE_N(op_info)) { 1465|| if (Z_REG(addr) != ZREG_FP) { 1466| GET_ZVAL_LVAL ZREG_R0, addr 1467|| if (RC_MAY_BE_1(op_info)) { 1468| cmp dword [r0], 1 // if (GC_REFCOUNT() > 1) 1469| jbe >2 1470|| } 1471|| if (Z_REG(addr) != ZREG_FCARG1 || Z_OFFSET(addr) != 0) { 1472| LOAD_ZVAL_ADDR FCARG1a, addr 1473|| } 1474| EXT_CALL zend_jit_zval_array_dup, r0 1475|2: 1476| mov FCARG1a, r0 1477|| } else { 1478| GET_ZVAL_LVAL ZREG_FCARG1, addr 1479|| if (RC_MAY_BE_1(op_info)) { 1480| cmp dword [FCARG1a], 1 // if (GC_REFCOUNT() > 1) 1481|| if (cold) { 1482| ja >1 1483|.cold_code 1484|1: 1485|| } else { 1486| jbe >2 1487|| } 1488|| } 1489| IF_NOT_ZVAL_REFCOUNTED addr, >1 1490| GC_DELREF FCARG1a 1491|1: 1492| EXT_CALL zend_array_dup, r0 1493| SET_ZVAL_PTR addr, r0 1494| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX 1495| mov FCARG1a, r0 1496|| if (RC_MAY_BE_1(op_info)) { 1497|| if (cold) { 1498| jmp >2 1499|.code 1500|| } 1501|| } 1502|2: 1503|| } 1504|| } else { 1505| GET_ZVAL_LVAL ZREG_FCARG1, addr 1506|| } 1507|.endmacro 1508 1509|.macro EFREE_REG_REFERENCE 1510||#if ZEND_DEBUG 1511| xor FCARG2a, FCARG2a // filename 1512| .if X64WIN 1513| xor CARG3d, CARG3d // lineno 1514| xor CARG4, CARG4 1515| mov aword A5, 0 1516| EXT_CALL _efree, r0 1517| .elif X64 1518| xor CARG3d, CARG3d // lineno 1519| xor CARG4, CARG4 1520| xor CARG5, CARG5 1521| EXT_CALL _efree, r0 1522| .else 1523| sub r4, 4 1524| push 0 1525| push 0 1526| push 0 // lineno 1527| EXT_CALL _efree, r0 1528| add r4, 4 1529| .endif 1530||#else 1531||#ifdef HAVE_BUILTIN_CONSTANT_P 1532| EXT_CALL _efree_32, r0 1533||#else 1534| EXT_CALL _efree, r0 1535||#endif 1536||#endif 1537|.endmacro 1538 1539|.macro EFREE_REFERENCE, ptr 1540| mov FCARG1a, ptr 1541| EFREE_REG_REFERENCE 1542|.endmacro 1543 1544|.macro EMALLOC, size, op_array, opline 1545||#if ZEND_DEBUG 1546|| const char *filename = op_array->filename ? op_array->filename->val : NULL; 1547| mov FCARG1a, size 1548| LOAD_ADDR FCARG2a, filename 1549| .if X64WIN 1550| mov CARG3d, opline->lineno 1551| xor CARG4, CARG4 1552| mov aword A5, 0 1553| EXT_CALL _emalloc, r0 1554| .elif X64 1555| mov CARG3d, opline->lineno 1556| xor CARG4, CARG4 1557| xor CARG5, CARG5 1558| EXT_CALL _emalloc, r0 1559| .else 1560| sub r4, 4 1561| push 0 1562| push 0 1563| push opline->lineno 1564| EXT_CALL _emalloc, r0 1565| add r4, 4 1566| .endif 1567||#else 1568||#ifdef HAVE_BUILTIN_CONSTANT_P 1569|| if (size > 24 && size <= 32) { 1570| EXT_CALL _emalloc_32, r0 1571|| } else { 1572| mov FCARG1a, size 1573| EXT_CALL _emalloc, r0 1574|| } 1575||#else 1576| mov FCARG1a, size 1577| EXT_CALL _emalloc, r0 1578||#endif 1579||#endif 1580|.endmacro 1581 1582|.macro OBJ_RELEASE, reg, exit_label 1583| GC_DELREF Ra(reg) 1584| jne >1 1585| // zend_objects_store_del(obj); 1586|| if (reg != ZREG_FCARG1) { 1587| mov FCARG1a, Ra(reg) 1588|| } 1589| EXT_CALL zend_objects_store_del, r0 1590| jmp exit_label 1591|1: 1592| IF_GC_MAY_NOT_LEAK Ra(reg), >1 1593| // gc_possible_root(obj) 1594|| if (reg != ZREG_FCARG1) { 1595| mov FCARG1a, Ra(reg) 1596|| } 1597| EXT_CALL gc_possible_root, r0 1598|1: 1599|.endmacro 1600 1601|.macro UNDEFINED_OFFSET, opline 1602|| if (opline == last_valid_opline) { 1603|| zend_jit_use_last_valid_opline(); 1604| call ->undefined_offset_ex 1605|| } else { 1606| SET_EX_OPLINE opline, r0 1607| call ->undefined_offset 1608|| } 1609|.endmacro 1610 1611|.macro UNDEFINED_INDEX, opline 1612|| if (opline == last_valid_opline) { 1613|| zend_jit_use_last_valid_opline(); 1614| call ->undefined_index_ex 1615|| } else { 1616| SET_EX_OPLINE opline, r0 1617| call ->undefined_index 1618|| } 1619|.endmacro 1620 1621|.macro CANNOT_ADD_ELEMENT, opline 1622|| if (opline == last_valid_opline) { 1623|| zend_jit_use_last_valid_opline(); 1624| call ->cannot_add_element_ex 1625|| } else { 1626| SET_EX_OPLINE opline, r0 1627| call ->cannot_add_element 1628|| } 1629|.endmacro 1630 1631static bool reuse_ip = 0; 1632static bool delayed_call_chain = 0; 1633static uint32_t delayed_call_level = 0; 1634static const zend_op *last_valid_opline = NULL; 1635static bool use_last_vald_opline = 0; 1636static bool track_last_valid_opline = 0; 1637static int jit_return_label = -1; 1638static uint32_t current_trace_num = 0; 1639static uint32_t allowed_opt_flags = 0; 1640 1641static void zend_jit_track_last_valid_opline(void) 1642{ 1643 use_last_vald_opline = 0; 1644 track_last_valid_opline = 1; 1645} 1646 1647static void zend_jit_use_last_valid_opline(void) 1648{ 1649 if (track_last_valid_opline) { 1650 use_last_vald_opline = 1; 1651 track_last_valid_opline = 0; 1652 } 1653} 1654 1655static bool zend_jit_trace_uses_initial_ip(void) 1656{ 1657 return use_last_vald_opline; 1658} 1659 1660static void zend_jit_set_last_valid_opline(const zend_op *target_opline) 1661{ 1662 if (!reuse_ip) { 1663 track_last_valid_opline = 0; 1664 last_valid_opline = target_opline; 1665 } 1666} 1667 1668static void zend_jit_reset_last_valid_opline(void) 1669{ 1670 track_last_valid_opline = 0; 1671 last_valid_opline = NULL; 1672} 1673 1674static void zend_jit_start_reuse_ip(void) 1675{ 1676 zend_jit_reset_last_valid_opline(); 1677 reuse_ip = 1; 1678} 1679 1680static int zend_jit_reuse_ip(dasm_State **Dst) 1681{ 1682 if (!reuse_ip) { 1683 zend_jit_start_reuse_ip(); 1684 | // call = EX(call); 1685 | mov RX, EX->call 1686 } 1687 return 1; 1688} 1689 1690static void zend_jit_stop_reuse_ip(void) 1691{ 1692 reuse_ip = 0; 1693} 1694 1695static int zend_jit_interrupt_handler_stub(dasm_State **Dst) 1696{ 1697 |->interrupt_handler: 1698 | SAVE_IP 1699 | //EG(vm_interrupt) = 0; 1700 | MEM_STORE_ZTS byte, executor_globals, vm_interrupt, 0, r0 1701 | //if (EG(timed_out)) { 1702 | MEM_CMP_ZTS byte, executor_globals, timed_out, 0, r0 1703 | je >1 1704 | //zend_timeout(); 1705 | EXT_CALL zend_timeout, r0 1706 |1: 1707 | //} else if (zend_interrupt_function) { 1708 if (zend_interrupt_function) { 1709 | //zend_interrupt_function(execute_data); 1710 |.if X64 1711 | mov CARG1, FP 1712 | EXT_CALL zend_interrupt_function, r0 1713 |.else 1714 | mov aword A1, FP 1715 | EXT_CALL zend_interrupt_function, r0 1716 |.endif 1717 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 1718 | je >1 1719 | EXT_CALL zend_jit_exception_in_interrupt_handler_helper, r0 1720 |1: 1721 | //ZEND_VM_ENTER(); 1722 | //execute_data = EG(current_execute_data); 1723 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 1724 | LOAD_IP 1725 } 1726 | //ZEND_VM_CONTINUE() 1727 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1728 | ADD_HYBRID_SPAD 1729 | JMP_IP 1730 } else if (GCC_GLOBAL_REGS) { 1731 | add r4, SPAD // stack alignment 1732 | JMP_IP 1733 } else { 1734 | mov FP, aword T2 // restore FP 1735 | mov RX, aword T3 // restore IP 1736 | add r4, NR_SPAD // stack alignment 1737 | mov r0, 1 // ZEND_VM_ENTER 1738 | ret 1739 } 1740 1741 return 1; 1742} 1743 1744static int zend_jit_exception_handler_stub(dasm_State **Dst) 1745{ 1746 |->exception_handler: 1747 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1748 const void *handler = zend_get_opcode_handler_func(EG(exception_op)); 1749 1750 | ADD_HYBRID_SPAD 1751 | EXT_CALL handler, r0 1752 | JMP_IP 1753 } else { 1754 const void *handler = EG(exception_op)->handler; 1755 1756 if (GCC_GLOBAL_REGS) { 1757 | add r4, SPAD // stack alignment 1758 | EXT_JMP handler, r0 1759 } else { 1760 | mov FCARG1a, FP 1761 | EXT_CALL handler, r0 1762 | mov FP, aword T2 // restore FP 1763 | mov RX, aword T3 // restore IP 1764 | add r4, NR_SPAD // stack alignment 1765 | test eax, eax 1766 | jl >1 1767 | mov r0, 1 // ZEND_VM_ENTER 1768 |1: 1769 | ret 1770 } 1771 } 1772 1773 return 1; 1774} 1775 1776static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) 1777{ 1778 |->exception_handler_undef: 1779 | MEM_LOAD_ZTS r0, aword, executor_globals, opline_before_exception, r0 1780 | test byte OP:r0->result_type, (IS_TMP_VAR|IS_VAR) 1781 | jz >1 1782 | mov eax, dword OP:r0->result.var 1783 | SET_Z_TYPE_INFO FP + r0, IS_UNDEF 1784 |1: 1785 | jmp ->exception_handler 1786 1787 return 1; 1788} 1789 1790 1791static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst) 1792{ 1793 |->exception_handler_free_op1_op2: 1794 | UNDEF_OPLINE_RESULT_IF_USED 1795 | test byte OP:RX->op1_type, (IS_TMP_VAR|IS_VAR) 1796 | je >9 1797 | mov eax, dword OP:RX->op1.var 1798 | add r0, FP 1799 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1800 |9: 1801 | test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR) 1802 | je >9 1803 | mov eax, dword OP:RX->op2.var 1804 | add r0, FP 1805 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1806 |9: 1807 | jmp ->exception_handler 1808 return 1; 1809} 1810 1811static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst) 1812{ 1813 |->exception_handler_free_op2: 1814 | MEM_LOAD_ZTS RX, aword, executor_globals, opline_before_exception, r0 1815 | UNDEF_OPLINE_RESULT_IF_USED 1816 | test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR) 1817 | je >9 1818 | mov eax, dword OP:RX->op2.var 1819 | add r0, FP 1820 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1821 |9: 1822 | jmp ->exception_handler 1823 return 1; 1824} 1825 1826static int zend_jit_leave_function_stub(dasm_State **Dst) 1827{ 1828 |->leave_function_handler: 1829 | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)] 1830 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1831 | test FCARG1d, ZEND_CALL_TOP 1832 | jnz >1 1833 | EXT_CALL zend_jit_leave_nested_func_helper, r0 1834 | ADD_HYBRID_SPAD 1835 | JMP_IP 1836 |1: 1837 | EXT_CALL zend_jit_leave_top_func_helper, r0 1838 | ADD_HYBRID_SPAD 1839 | JMP_IP 1840 } else { 1841 if (GCC_GLOBAL_REGS) { 1842 | add r4, SPAD 1843 } else { 1844 | mov FCARG2a, FP 1845 | mov FP, aword T2 // restore FP 1846 | mov RX, aword T3 // restore IP 1847 | add r4, NR_SPAD 1848 } 1849 | test FCARG1d, ZEND_CALL_TOP 1850 | jnz >1 1851 | EXT_JMP zend_jit_leave_nested_func_helper, r0 1852 |1: 1853 | EXT_JMP zend_jit_leave_top_func_helper, r0 1854 } 1855 1856 return 1; 1857} 1858 1859static int zend_jit_leave_throw_stub(dasm_State **Dst) 1860{ 1861 |->leave_throw_handler: 1862 | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { 1863 if (GCC_GLOBAL_REGS) { 1864 | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION 1865 | je >5 1866 | // EG(opline_before_exception) = opline; 1867 | MEM_STORE_ZTS aword, executor_globals, opline_before_exception, IP, r0 1868 |5: 1869 | // opline = EG(exception_op); 1870 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1871 | // HANDLE_EXCEPTION() 1872 | jmp ->exception_handler 1873 } else { 1874 | GET_IP FCARG1a 1875 | cmp byte OP:FCARG1a->opcode, ZEND_HANDLE_EXCEPTION 1876 | je >5 1877 | // EG(opline_before_exception) = opline; 1878 | MEM_STORE_ZTS aword, executor_globals, opline_before_exception, FCARG1a, r0 1879 |5: 1880 | // opline = EG(exception_op); 1881 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1882 | mov FP, aword T2 // restore FP 1883 | mov RX, aword T3 // restore IP 1884 | add r4, NR_SPAD // stack alignment 1885 | mov r0, 2 // ZEND_VM_LEAVE 1886 | ret 1887 } 1888 1889 return 1; 1890} 1891 1892static int zend_jit_icall_throw_stub(dasm_State **Dst) 1893{ 1894 |->icall_throw_handler: 1895 | // zend_rethrow_exception(zend_execute_data *execute_data) 1896 | mov IP, aword EX->opline 1897 | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { 1898 | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION 1899 | je >1 1900 | // EG(opline_before_exception) = opline; 1901 | MEM_STORE_ZTS aword, executor_globals, opline_before_exception, IP, r0 1902 |1: 1903 | // opline = EG(exception_op); 1904 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1905 || if (GCC_GLOBAL_REGS) { 1906 | mov aword EX->opline, IP 1907 || } 1908 | // HANDLE_EXCEPTION() 1909 | jmp ->exception_handler 1910 1911 return 1; 1912} 1913 1914static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) 1915{ 1916 |->throw_cannot_pass_by_ref: 1917 | mov r0, EX->opline 1918 | mov ecx, dword OP:r0->result.var 1919 | SET_Z_TYPE_INFO RX+r1, IS_UNDEF 1920 | // last EX(call) frame may be delayed 1921 | cmp RX, EX->call 1922 | je >1 1923 | mov r1, EX->call 1924 | mov EX:RX->prev_execute_data, r1 1925 | mov EX->call, RX 1926 |1: 1927 | mov RX, r0 1928 | mov FCARG1d, dword OP:r0->op2.num 1929 | EXT_CALL zend_cannot_pass_by_reference, r0 1930 | cmp byte OP:RX->op1_type, IS_TMP_VAR 1931 | jne >9 1932 | mov eax, dword OP:RX->op1.var 1933 | add r0, FP 1934 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1935 |9: 1936 | jmp ->exception_handler 1937 1938 return 1; 1939} 1940 1941static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) 1942{ 1943 |->undefined_offset_ex: 1944 | SAVE_IP 1945 | jmp ->undefined_offset 1946 1947 return 1; 1948} 1949 1950static int zend_jit_undefined_offset_stub(dasm_State **Dst) 1951{ 1952 |->undefined_offset: 1953 |.if X64WIN 1954 | sub r4, 0x28 1955 |.elif X64 1956 | sub r4, 8 1957 |.else 1958 | sub r4, 12 1959 |.endif 1960 | mov r0, EX->opline 1961 | mov ecx, dword OP:r0->result.var 1962 | cmp byte OP:r0->op2_type, IS_CONST 1963 | SET_Z_TYPE_INFO FP + r1, IS_NULL 1964 | jne >2 1965 |.if X64 1966 | movsxd r1, dword OP:r0->op2.constant 1967 | add r0, r1 1968 |.else 1969 | mov r0, aword OP:r0->op2.zv 1970 |.endif 1971 | jmp >3 1972 |2: 1973 | mov eax, dword OP:r0->op2.var 1974 | add r0, FP 1975 |3: 1976 |.if X64WIN 1977 | mov CARG1, E_WARNING 1978 | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT 1979 | mov CARG3, aword [r0] 1980 | EXT_CALL zend_error, r0 1981 | add r4, 0x28 // stack alignment 1982 |.elif X64 1983 | mov CARG1, E_WARNING 1984 | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT 1985 | mov CARG3, aword [r0] 1986 | EXT_CALL zend_error, r0 1987 | add r4, 8 // stack alignment 1988 |.else 1989 | sub r4, 4 1990 | push aword [r0] 1991 | push "Undefined array key " ZEND_LONG_FMT 1992 | push E_WARNING 1993 | EXT_CALL zend_error, r0 1994 | add r4, 28 1995 |.endif 1996 | ret 1997 1998 return 1; 1999} 2000 2001static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) 2002{ 2003 |->undefined_index_ex: 2004 | SAVE_IP 2005 | jmp ->undefined_index 2006 2007 return 1; 2008} 2009 2010static int zend_jit_undefined_index_stub(dasm_State **Dst) 2011{ 2012 |->undefined_index: 2013 |.if X64WIN 2014 | sub r4, 0x28 2015 |.elif X64 2016 | sub r4, 8 2017 |.else 2018 | sub r4, 12 2019 |.endif 2020 | mov r0, EX->opline 2021 | mov ecx, dword OP:r0->result.var 2022 | cmp byte OP:r0->op2_type, IS_CONST 2023 | SET_Z_TYPE_INFO FP + r1, IS_NULL 2024 | jne >2 2025 |.if X64 2026 | movsxd r1, dword OP:r0->op2.constant 2027 | add r0, r1 2028 |.else 2029 | mov r0, aword OP:r0->op2.zv 2030 |.endif 2031 | jmp >3 2032 |2: 2033 | mov eax, dword OP:r0->op2.var 2034 | add r0, FP 2035 |3: 2036 |.if X64WIN 2037 | mov CARG1, E_WARNING 2038 | LOAD_ADDR CARG2, "Undefined array key \"%s\"" 2039 | mov CARG3, aword [r0] 2040 | add CARG3, offsetof(zend_string, val) 2041 | EXT_CALL zend_error, r0 2042 | add r4, 0x28 2043 |.elif X64 2044 | mov CARG1, E_WARNING 2045 | LOAD_ADDR CARG2, "Undefined array key \"%s\"" 2046 | mov CARG3, aword [r0] 2047 | add CARG3, offsetof(zend_string, val) 2048 | EXT_CALL zend_error, r0 2049 | add r4, 8 2050 |.else 2051 | sub r4, 4 2052 | mov r0, aword [r0] 2053 | add r0, offsetof(zend_string, val) 2054 | push r0 2055 | push "Undefined array key \"%s\"" 2056 | push E_WARNING 2057 | EXT_CALL zend_error, r0 2058 | add r4, 28 2059 |.endif 2060 | ret 2061 2062 return 1; 2063} 2064 2065static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) 2066{ 2067 |->cannot_add_element_ex: 2068 | SAVE_IP 2069 | jmp ->cannot_add_element 2070 2071 return 1; 2072} 2073 2074static int zend_jit_cannot_add_element_stub(dasm_State **Dst) 2075{ 2076 |->cannot_add_element: 2077 |.if X64WIN 2078 | sub r4, 0x28 2079 |.elif X64 2080 | sub r4, 8 2081 |.else 2082 | sub r4, 12 2083 |.endif 2084 | mov r0, EX->opline 2085 | cmp byte OP:r0->result_type, IS_UNUSED 2086 | jz >1 2087 | mov eax, dword OP:r0->result.var 2088 | SET_Z_TYPE_INFO FP + r0, IS_NULL 2089 |1: 2090 |.if X64WIN 2091 | xor CARG1, CARG1 2092 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2093 | EXT_CALL zend_throw_error, r0 2094 | add r4, 0x28 2095 |.elif X64 2096 | xor CARG1, CARG1 2097 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2098 | EXT_CALL zend_throw_error, r0 2099 | add r4, 8 2100 |.else 2101 | sub r4, 8 2102 | push "Cannot add element to the array as the next element is already occupied" 2103 | push 0 2104 | EXT_CALL zend_throw_error, r0 2105 | add r4, 28 2106 |.endif 2107 | ret 2108 2109 return 1; 2110} 2111 2112static int zend_jit_undefined_function_stub(dasm_State **Dst) 2113{ 2114 |->undefined_function: 2115 | mov r0, aword EX->opline 2116 |.if X64 2117 | xor CARG1, CARG1 2118 | LOAD_ADDR CARG2, "Call to undefined function %s()" 2119 | movsxd CARG3, dword [r0 + offsetof(zend_op, op2.constant)] 2120 | mov CARG3, aword [r0 + CARG3] 2121 | add CARG3, offsetof(zend_string, val) 2122 | EXT_CALL zend_throw_error, r0 2123 |.else 2124 | mov r0, aword [r0 + offsetof(zend_op, op2.zv)] 2125 | mov r0, aword [r0] 2126 | add r0, offsetof(zend_string, val) 2127 | mov aword A3, r0 2128 | mov aword A2, "Call to undefined function %s()" 2129 | mov aword A1, 0 2130 | EXT_CALL zend_throw_error, r0 2131 |.endif 2132 | jmp ->exception_handler 2133 return 1; 2134} 2135 2136static int zend_jit_negative_shift_stub(dasm_State **Dst) 2137{ 2138 |->negative_shift: 2139 | mov RX, EX->opline 2140 |.if X64 2141 |.if WIN 2142 | LOAD_ADDR CARG1, &zend_ce_arithmetic_error 2143 | mov CARG1, aword [CARG1] 2144 |.else 2145 | LOAD_ADDR CARG1, zend_ce_arithmetic_error 2146 |.endif 2147 | LOAD_ADDR CARG2, "Bit shift by negative number" 2148 | EXT_CALL zend_throw_error, r0 2149 |.else 2150 | sub r4, 8 2151 | push "Bit shift by negative number" 2152 |.if WIN 2153 | LOAD_ADDR r0, &zend_ce_arithmetic_error 2154 | push aword [r0] 2155 |.else 2156 | PUSH_ADDR zend_ce_arithmetic_error, r0 2157 |.endif 2158 | EXT_CALL zend_throw_error, r0 2159 | add r4, 16 2160 |.endif 2161 | jmp ->exception_handler_free_op1_op2 2162 return 1; 2163} 2164 2165static int zend_jit_mod_by_zero_stub(dasm_State **Dst) 2166{ 2167 |->mod_by_zero: 2168 | mov RX, EX->opline 2169 |.if X64 2170 |.if WIN 2171 | LOAD_ADDR CARG1, &zend_ce_division_by_zero_error 2172 | mov CARG1, aword [CARG1] 2173 |.else 2174 | LOAD_ADDR CARG1, zend_ce_division_by_zero_error 2175 |.endif 2176 | LOAD_ADDR CARG2, "Modulo by zero" 2177 | EXT_CALL zend_throw_error, r0 2178 |.else 2179 | sub r4, 8 2180 | push "Modulo by zero" 2181 |.if WIN 2182 | LOAD_ADDR r0, &zend_ce_division_by_zero_error 2183 | push aword [r0] 2184 |.else 2185 | PUSH_ADDR zend_ce_division_by_zero_error, r0 2186 |.endif 2187 | EXT_CALL zend_throw_error, r0 2188 | add r4, 16 2189 |.endif 2190 | jmp ->exception_handler_free_op1_op2 2191 return 1; 2192} 2193 2194static int zend_jit_invalid_this_stub(dasm_State **Dst) 2195{ 2196 |->invalid_this: 2197 | UNDEF_OPLINE_RESULT 2198 |.if X64 2199 | xor CARG1, CARG1 2200 | LOAD_ADDR CARG2, "Using $this when not in object context" 2201 | EXT_CALL zend_throw_error, r0 2202 |.else 2203 | sub r4, 8 2204 | push "Using $this when not in object context" 2205 | push 0 2206 | EXT_CALL zend_throw_error, r0 2207 | add r4, 16 2208 |.endif 2209 | jmp ->exception_handler 2210 return 1; 2211} 2212 2213static int zend_jit_double_one_stub(dasm_State **Dst) 2214{ 2215 |->one: 2216 |.dword 0, 0x3ff00000 2217 return 1; 2218} 2219 2220static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) 2221{ 2222 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2223 return 1; 2224 } 2225 2226 |->hybrid_runtime_jit: 2227 | EXT_CALL zend_runtime_jit, r0 2228 | JMP_IP 2229 return 1; 2230} 2231 2232static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) 2233{ 2234 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2235 return 1; 2236 } 2237 2238 |->hybrid_profile_jit: 2239 | // ++zend_jit_profile_counter; 2240 | .if X64 2241 | LOAD_ADDR r0, &zend_jit_profile_counter 2242 | inc aword [r0] 2243 | .else 2244 | inc aword [&zend_jit_profile_counter] 2245 | .endif 2246 | // op_array = (zend_op_array*)EX(func); 2247 | mov r0, EX->func 2248 | // run_time_cache = EX(run_time_cache); 2249 | mov r2, EX->run_time_cache 2250 | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); 2251 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2252 | // ++ZEND_COUNTER_INFO(op_array) 2253 | inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void*)] 2254 | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() 2255 | jmp aword [r0 + offsetof(zend_jit_op_array_extension, orig_handler)] 2256 return 1; 2257} 2258 2259static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) 2260{ 2261 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2262 return 1; 2263 } 2264 2265 |->hybrid_hot_code: 2266 | mov word [r2], ZEND_JIT_COUNTER_INIT 2267 | mov FCARG1a, FP 2268 | GET_IP FCARG2a 2269 | EXT_CALL zend_jit_hot_func, r0 2270 | JMP_IP 2271 return 1; 2272} 2273 2274/* 2275 * This code is based Mike Pall's "Hashed profile counters" idea, implemented 2276 * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual 2277 * property disclosure and research opportunities" email 2278 * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html 2279 * 2280 * In addition we use a variation of Knuth's multiplicative hash function 2281 * described at https://code.i-harness.com/en/q/a21ce 2282 * 2283 * uint64_t hash(uint64_t x) { 2284 * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; 2285 * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; 2286 * x = x ^ (x >> 31); 2287 * return x; 2288 * } 2289 * 2290 * uint_32_t hash(uint32_t x) { 2291 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2292 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2293 * x = (x >> 16) ^ x; 2294 * return x; 2295 * } 2296 * 2297 */ 2298static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) 2299{ 2300 | mov r0, EX->func 2301 | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2302 | mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)] 2303 | sub word [r2], cost 2304 | jle ->hybrid_hot_code 2305 | GET_IP r2 2306 | sub r2, aword [r0 + offsetof(zend_op_array, opcodes)] 2307 | // divide by sizeof(zend_op) 2308 | .if X64 2309 || ZEND_ASSERT(sizeof(zend_op) == 32); 2310 | sar r2, 2 2311 | .else 2312 || ZEND_ASSERT(sizeof(zend_op) == 28); 2313 | imul r2, 0xb6db6db7 2314 | .endif 2315 | .if X64 2316 | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2317 | .else 2318 | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2319 | .endif 2320 return 1; 2321} 2322 2323static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) 2324{ 2325 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2326 return 1; 2327 } 2328 2329 |->hybrid_func_hot_counter: 2330 2331 return zend_jit_hybrid_hot_counter_stub(Dst, 2332 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2333} 2334 2335static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) 2336{ 2337 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2338 return 1; 2339 } 2340 2341 |->hybrid_loop_hot_counter: 2342 2343 return zend_jit_hybrid_hot_counter_stub(Dst, 2344 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2345} 2346 2347static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) 2348{ 2349 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2350 return 1; 2351 } 2352 2353 |->hybrid_hot_trace: 2354 | mov word [r2], ZEND_JIT_COUNTER_INIT 2355 | mov FCARG1a, FP 2356 | GET_IP FCARG2a 2357 | EXT_CALL zend_jit_trace_hot_root, r0 2358 | test eax, eax // TODO : remove this check at least for HYBRID VM ??? 2359 | jl >1 2360 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 2361 | LOAD_IP 2362 | JMP_IP 2363 |1: 2364 | EXT_JMP zend_jit_halt_op->handler, r0 2365 return 1; 2366} 2367 2368static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) 2369{ 2370 | mov r0, EX->func 2371 | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2372 | mov r1, aword [r1 + offsetof(zend_jit_op_array_trace_extension, offset)] 2373 | mov r2, aword [IP + r1 + offsetof(zend_op_trace_info, counter)] 2374 | sub word [r2], cost 2375 | jle ->hybrid_hot_trace 2376 | jmp aword [IP + r1] 2377 return 1; 2378} 2379 2380static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) 2381{ 2382 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2383 return 1; 2384 } 2385 2386 |->hybrid_func_trace_counter: 2387 2388 return zend_jit_hybrid_trace_counter_stub(Dst, 2389 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2390} 2391 2392static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) 2393{ 2394 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { 2395 return 1; 2396 } 2397 2398 |->hybrid_ret_trace_counter: 2399 2400 return zend_jit_hybrid_trace_counter_stub(Dst, 2401 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); 2402} 2403 2404static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) 2405{ 2406 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2407 return 1; 2408 } 2409 2410 |->hybrid_loop_trace_counter: 2411 2412 return zend_jit_hybrid_trace_counter_stub(Dst, 2413 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2414} 2415 2416static int zend_jit_trace_halt_stub(dasm_State **Dst) 2417{ 2418 |->trace_halt: 2419 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2420 | ADD_HYBRID_SPAD 2421 | EXT_JMP zend_jit_halt_op->handler, r0 2422 } else if (GCC_GLOBAL_REGS) { 2423 | add r4, SPAD // stack alignment 2424 | xor IP, IP // PC must be zero 2425 | ret 2426 } else { 2427 | mov FP, aword T2 // restore FP 2428 | mov RX, aword T3 // restore IP 2429 | add r4, NR_SPAD // stack alignment 2430 | mov r0, -1 // ZEND_VM_RETURN 2431 | ret 2432 } 2433 return 1; 2434} 2435 2436static int zend_jit_trace_exit_stub(dasm_State **Dst) 2437{ 2438 |->trace_exit: 2439 | 2440 | // Save CPU registers 2441 |.if X64 2442 | sub r4, 16*8+16*8-8 /* CPU regs + SSE regs */ 2443 | mov aword [r4+15*8], r15 2444 | mov aword [r4+11*8], r11 2445 | mov aword [r4+10*8], r10 2446 | mov aword [r4+9*8], r9 2447 | mov aword [r4+8*8], r8 2448 | mov aword [r4+7*8], rdi 2449 | mov aword [r4+6*8], rsi 2450 | mov aword [r4+2*8], rdx 2451 | mov aword [r4+1*8], rcx 2452 | mov aword [r4+0*8], rax 2453 | mov FCARG1a, aword [r4+16*8+16*8-8] // exit_num = POP 2454 | mov FCARG2a, r4 2455 | movsd qword [r4+16*8+15*8], xmm15 2456 | movsd qword [r4+16*8+14*8], xmm14 2457 | movsd qword [r4+16*8+13*8], xmm13 2458 | movsd qword [r4+16*8+12*8], xmm12 2459 | movsd qword [r4+16*8+11*8], xmm11 2460 | movsd qword [r4+16*8+10*8], xmm10 2461 | movsd qword [r4+16*8+9*8], xmm9 2462 | movsd qword [r4+16*8+8*8], xmm8 2463 | movsd qword [r4+16*8+7*8], xmm7 2464 | movsd qword [r4+16*8+6*8], xmm6 2465 | movsd qword [r4+16*8+5*8], xmm5 2466 | movsd qword [r4+16*8+4*8], xmm4 2467 | movsd qword [r4+16*8+3*8], xmm3 2468 | movsd qword [r4+16*8+2*8], xmm2 2469 | movsd qword [r4+16*8+1*8], xmm1 2470 | movsd qword [r4+16*8+0*8], xmm0 2471 |.if X64WIN 2472 | sub r4, 32 /* shadow space */ 2473 |.endif 2474 |.else 2475 | sub r4, 8*4+8*8-4 /* CPU regs + SSE regs */ 2476 | mov aword [r4+7*4], edi 2477 | mov aword [r4+2*4], edx 2478 | mov aword [r4+1*4], ecx 2479 | mov aword [r4+0*4], eax 2480 | mov FCARG1a, aword [r4+8*4+8*8-4] // exit_num = POP 2481 | mov FCARG2a, r4 2482 | movsd qword [r4+8*4+7*8], xmm7 2483 | movsd qword [r4+8*4+6*8], xmm6 2484 | movsd qword [r4+8*4+5*8], xmm5 2485 | movsd qword [r4+8*4+4*8], xmm4 2486 | movsd qword [r4+8*4+3*8], xmm3 2487 | movsd qword [r4+8*4+2*8], xmm2 2488 | movsd qword [r4+8*4+1*8], xmm1 2489 | movsd qword [r4+8*4+0*8], xmm0 2490 |.endif 2491 | 2492 | // EX(opline) = opline 2493 | SAVE_IP 2494 | // zend_jit_trace_exit(trace_num, exit_num) 2495 | EXT_CALL zend_jit_trace_exit, r0 2496 |.if X64WIN 2497 | add r4, 16*8+16*8+32 /* CPU regs + SSE regs + shadow space */ 2498 |.elif X64 2499 | add r4, 16*8+16*8 /* CPU regs + SSE regs */ 2500 |.else 2501 | add r4, 8*4+8*8 /* CPU regs + SSE regs */ 2502 |.endif 2503 2504 | test eax, eax 2505 | jne >1 2506 2507 | // execute_data = EG(current_execute_data) 2508 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 2509 | // opline = EX(opline) 2510 | LOAD_IP 2511 2512 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2513 | ADD_HYBRID_SPAD 2514 | JMP_IP 2515 } else if (GCC_GLOBAL_REGS) { 2516 | add r4, SPAD // stack alignment 2517 | JMP_IP 2518 } else { 2519 | mov FP, aword T2 // restore FP 2520 | mov RX, aword T3 // restore IP 2521 | add r4, NR_SPAD // stack alignment 2522 | mov r0, 1 // ZEND_VM_ENTER 2523 | ret 2524 } 2525 2526 |1: 2527 | jl ->trace_halt 2528 2529 | // execute_data = EG(current_execute_data) 2530 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 2531 | // opline = EX(opline) 2532 | LOAD_IP 2533 2534 | // check for interrupt (try to avoid this ???) 2535 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 2536 | jne ->interrupt_handler 2537 2538 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2539 | ADD_HYBRID_SPAD 2540 | mov r0, EX->func 2541 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2542 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2543 | jmp aword [IP + r0] 2544 } else if (GCC_GLOBAL_REGS) { 2545 | add r4, SPAD // stack alignment 2546 | mov r0, EX->func 2547 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2548 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2549 | jmp aword [IP + r0] 2550 } else { 2551 | mov IP, aword EX->opline 2552 | mov FCARG1a, FP 2553 | mov r0, EX->func 2554 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2555 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2556 | call aword [IP + r0] 2557 | test eax, eax 2558 | jl ->trace_halt 2559 | mov FP, aword T2 // restore FP 2560 | mov RX, aword T3 // restore IP 2561 | add r4, NR_SPAD // stack alignment 2562 | mov r0, 1 // ZEND_VM_ENTER 2563 | ret 2564 } 2565 2566 return 1; 2567} 2568 2569static int zend_jit_trace_escape_stub(dasm_State **Dst) 2570{ 2571 |->trace_escape: 2572 | 2573 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2574 | ADD_HYBRID_SPAD 2575 | JMP_IP 2576 } else if (GCC_GLOBAL_REGS) { 2577 | add r4, SPAD // stack alignment 2578 | JMP_IP 2579 } else { 2580 | mov FP, aword T2 // restore FP 2581 | mov RX, aword T3 // restore IP 2582 | add r4, NR_SPAD // stack alignment 2583 | mov r0, 1 // ZEND_VM_ENTER 2584 | ret 2585 } 2586 2587 return 1; 2588} 2589 2590/* Keep 32 exit points in a single code block */ 2591#define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes 2592#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points 2593 2594static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) 2595{ 2596 uint32_t i; 2597 2598 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP - 1; i++) { 2599 | push byte i 2600 | .byte 0xeb, (4*(ZEND_JIT_EXIT_POINTS_PER_GROUP-i)-6) // jmp >1 2601 } 2602 | push byte i 2603 |// 1: 2604 | add aword [r4], n 2605 | jmp ->trace_exit 2606 2607 return 1; 2608} 2609 2610#ifdef CONTEXT_THREADED_JIT 2611static int zend_jit_context_threaded_call_stub(dasm_State **Dst) 2612{ 2613 |->context_threaded_call: 2614 | pop r0 2615 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2616 | ADD_HYBRID_SPAD 2617 | jmp aword [IP] 2618 } else if (GCC_GLOBAL_REGS) { 2619 | add r4, SPAD // stack alignment 2620 | jmp aword [IP] 2621 } else { 2622 ZEND_UNREACHABLE(); 2623 // TODO: context threading can't work without GLOBAL REGS because we have to change 2624 // the value of execute_data in execute_ex() 2625 | mov FCARG1a, FP 2626 | mov r0, aword [FP] 2627 | mov FP, aword T2 // restore FP 2628 | mov RX, aword T3 // restore IP 2629 | add r4, NR_SPAD // stack alignment 2630 | jmp aword [r0] 2631 } 2632 return 1; 2633} 2634#endif 2635 2636static int zend_jit_assign_const_stub(dasm_State **Dst) 2637{ 2638 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2639 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2640 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2641 2642 |->assign_const: 2643 |.if X64WIN 2644 | sub r4, 0x28 2645 |.elif X64 2646 | sub r4, 8 2647 |.else 2648 | sub r4, 12 2649 |.endif 2650 if (!zend_jit_assign_to_variable( 2651 Dst, NULL, 2652 var_addr, var_addr, -1, -1, 2653 IS_CONST, val_addr, val_info, 2654 0, 0)) { 2655 return 0; 2656 } 2657 |.if X64WIN 2658 | add r4, 0x28 2659 |.elif X64 2660 | add r4, 8 2661 |.else 2662 | add r4, 12 2663 |.endif 2664 | ret 2665 return 1; 2666} 2667 2668static int zend_jit_assign_tmp_stub(dasm_State **Dst) 2669{ 2670 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2671 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2672 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2673 2674 |->assign_tmp: 2675 |.if X64WIN 2676 | sub r4, 0x28 2677 |.elif X64 2678 | sub r4, 8 2679 |.else 2680 | sub r4, 12 2681 |.endif 2682 if (!zend_jit_assign_to_variable( 2683 Dst, NULL, 2684 var_addr, var_addr, -1, -1, 2685 IS_TMP_VAR, val_addr, val_info, 2686 0, 0)) { 2687 return 0; 2688 } 2689 |.if X64WIN 2690 | add r4, 0x28 2691 |.elif X64 2692 | add r4, 8 2693 |.else 2694 | add r4, 12 2695 |.endif 2696 | ret 2697 return 1; 2698} 2699 2700static int zend_jit_assign_var_stub(dasm_State **Dst) 2701{ 2702 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2703 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2704 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; 2705 2706 |->assign_var: 2707 |.if X64WIN 2708 | sub r4, 0x28 2709 |.elif X64 2710 | sub r4, 8 2711 |.else 2712 | sub r4, 12 2713 |.endif 2714 if (!zend_jit_assign_to_variable( 2715 Dst, NULL, 2716 var_addr, var_addr, -1, -1, 2717 IS_VAR, val_addr, val_info, 2718 0, 0)) { 2719 return 0; 2720 } 2721 |.if X64WIN 2722 | add r4, 0x28 2723 |.elif X64 2724 | add r4, 8 2725 |.else 2726 | add r4, 12 2727 |.endif 2728 | ret 2729 return 1; 2730} 2731 2732static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) 2733{ 2734 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2735 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2736 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; 2737 2738 |->assign_cv_noref: 2739 |.if X64WIN 2740 | sub r4, 0x28 2741 |.elif X64 2742 | sub r4, 8 2743 |.else 2744 | sub r4, 12 2745 |.endif 2746 if (!zend_jit_assign_to_variable( 2747 Dst, NULL, 2748 var_addr, var_addr, -1, -1, 2749 IS_CV, val_addr, val_info, 2750 0, 0)) { 2751 return 0; 2752 } 2753 |.if X64WIN 2754 | add r4, 0x28 2755 |.elif X64 2756 | add r4, 8 2757 |.else 2758 | add r4, 12 2759 |.endif 2760 | ret 2761 return 1; 2762} 2763 2764static int zend_jit_assign_cv_stub(dasm_State **Dst) 2765{ 2766 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 2767 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 2768 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; 2769 2770 |->assign_cv: 2771 |.if X64WIN 2772 | sub r4, 0x28 2773 |.elif X64 2774 | sub r4, 8 2775 |.else 2776 | sub r4, 12 2777 |.endif 2778 if (!zend_jit_assign_to_variable( 2779 Dst, NULL, 2780 var_addr, var_addr, -1, -1, 2781 IS_CV, val_addr, val_info, 2782 0, 0)) { 2783 return 0; 2784 } 2785 |.if X64WIN 2786 | add r4, 0x28 2787 |.elif X64 2788 | add r4, 8 2789 |.else 2790 | add r4, 12 2791 |.endif 2792 | ret 2793 return 1; 2794} 2795 2796static const zend_jit_stub zend_jit_stubs[] = { 2797 JIT_STUB(interrupt_handler, SP_ADJ_JIT, SP_ADJ_VM), 2798 JIT_STUB(exception_handler, SP_ADJ_JIT, SP_ADJ_VM), 2799 JIT_STUB(exception_handler_undef, SP_ADJ_JIT, SP_ADJ_VM), 2800 JIT_STUB(exception_handler_free_op1_op2, SP_ADJ_JIT, SP_ADJ_VM), 2801 JIT_STUB(exception_handler_free_op2, SP_ADJ_JIT, SP_ADJ_VM), 2802 JIT_STUB(leave_function, SP_ADJ_JIT, SP_ADJ_VM), 2803 JIT_STUB(leave_throw, SP_ADJ_JIT, SP_ADJ_VM), 2804 JIT_STUB(icall_throw, SP_ADJ_JIT, SP_ADJ_VM), 2805 JIT_STUB(throw_cannot_pass_by_ref, SP_ADJ_JIT, SP_ADJ_VM), 2806 JIT_STUB(undefined_offset, SP_ADJ_JIT, SP_ADJ_VM), 2807 JIT_STUB(undefined_index, SP_ADJ_JIT, SP_ADJ_VM), 2808 JIT_STUB(cannot_add_element, SP_ADJ_JIT, SP_ADJ_VM), 2809 JIT_STUB(undefined_offset_ex, SP_ADJ_JIT, SP_ADJ_VM), 2810 JIT_STUB(undefined_index_ex, SP_ADJ_JIT, SP_ADJ_VM), 2811 JIT_STUB(cannot_add_element_ex, SP_ADJ_JIT, SP_ADJ_VM), 2812 JIT_STUB(undefined_function, SP_ADJ_JIT, SP_ADJ_VM), 2813 JIT_STUB(negative_shift, SP_ADJ_JIT, SP_ADJ_VM), 2814 JIT_STUB(mod_by_zero, SP_ADJ_JIT, SP_ADJ_VM), 2815 JIT_STUB(invalid_this, SP_ADJ_JIT, SP_ADJ_VM), 2816 JIT_STUB(trace_halt, SP_ADJ_JIT, SP_ADJ_VM), 2817 JIT_STUB(trace_exit, SP_ADJ_JIT, SP_ADJ_VM), 2818 JIT_STUB(trace_escape, SP_ADJ_JIT, SP_ADJ_VM), 2819 JIT_STUB(hybrid_runtime_jit, SP_ADJ_VM, SP_ADJ_NONE), 2820 JIT_STUB(hybrid_profile_jit, SP_ADJ_VM, SP_ADJ_NONE), 2821 JIT_STUB(hybrid_hot_code, SP_ADJ_VM, SP_ADJ_NONE), 2822 JIT_STUB(hybrid_func_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2823 JIT_STUB(hybrid_loop_hot_counter, SP_ADJ_VM, SP_ADJ_NONE), 2824 JIT_STUB(hybrid_hot_trace, SP_ADJ_VM, SP_ADJ_NONE), 2825 JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2826 JIT_STUB(hybrid_ret_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2827 JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM, SP_ADJ_NONE), 2828 JIT_STUB(assign_const, SP_ADJ_RET, SP_ADJ_ASSIGN), 2829 JIT_STUB(assign_tmp, SP_ADJ_RET, SP_ADJ_ASSIGN), 2830 JIT_STUB(assign_var, SP_ADJ_RET, SP_ADJ_ASSIGN), 2831 JIT_STUB(assign_cv_noref, SP_ADJ_RET, SP_ADJ_ASSIGN), 2832 JIT_STUB(assign_cv, SP_ADJ_RET, SP_ADJ_ASSIGN), 2833 JIT_STUB(double_one, SP_ADJ_NONE, SP_ADJ_NONE), 2834#ifdef CONTEXT_THREADED_JIT 2835 JIT_STUB(context_threaded_call, SP_ADJ_RET, SP_ADJ_NONE), 2836#endif 2837}; 2838 2839#if ZTS && defined(ZEND_WIN32) 2840extern uint32_t _tls_index; 2841extern char *_tls_start; 2842extern char *_tls_end; 2843#endif 2844 2845#ifdef HAVE_GDB 2846typedef struct _Unwind_Context _Unwind_Context; 2847typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *); 2848extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *); 2849extern uintptr_t _Unwind_GetCFA(_Unwind_Context *); 2850 2851typedef struct _zend_jit_unwind_arg { 2852 int cnt; 2853 uintptr_t cfa[3]; 2854} zend_jit_unwind_arg; 2855 2856static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a) 2857{ 2858 zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a; 2859 arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx); 2860 arg->cnt++; 2861 if (arg->cnt == 3) { 2862 return 5; // _URC_END_OF_STACK 2863 } 2864 return 0; // _URC_NO_REASON; 2865} 2866 2867static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) 2868{ 2869 zend_jit_unwind_arg arg; 2870 2871 memset(&arg, 0, sizeof(arg)); 2872 _Unwind_Backtrace(zend_jit_unwind_cb, &arg); 2873 if (arg.cnt == 3) { 2874 sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1]; 2875 } 2876} 2877 2878extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data); 2879 2880static zend_never_inline void zend_jit_set_sp_adj_vm(void) 2881{ 2882 void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *); 2883 2884 orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data; 2885 zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data; 2886 execute_ex(NULL); // set sp_adj[SP_ADJ_VM] 2887 zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data; 2888} 2889#endif 2890 2891static int zend_jit_setup(void) 2892{ 2893 if (!zend_cpu_supports_sse2()) { 2894 zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); 2895 return FAILURE; 2896 } 2897 allowed_opt_flags = 0; 2898 if (zend_cpu_supports_avx()) { 2899 allowed_opt_flags |= ZEND_JIT_CPU_AVX; 2900 } 2901 2902#if ZTS 2903# ifdef _WIN64 2904 tsrm_tls_index = _tls_index * sizeof(void*); 2905 2906 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ 2907 /* Probably, it might be better solution */ 2908 do { 2909 void ***tls_mem = ((void**)__readgsqword(0x58))[_tls_index]; 2910 void *val = _tsrm_ls_cache; 2911 size_t offset = 0; 2912 size_t size = (char*)&_tls_end - (char*)&_tls_start; 2913 2914 while (offset < size) { 2915 if (*tls_mem == val) { 2916 tsrm_tls_offset = offset; 2917 break; 2918 } 2919 tls_mem++; 2920 offset += sizeof(void*); 2921 } 2922 if (offset >= size) { 2923 // TODO: error message ??? 2924 return FAILURE; 2925 } 2926 } while(0); 2927# elif ZEND_WIN32 2928 tsrm_tls_index = _tls_index * sizeof(void*); 2929 2930 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ 2931 /* Probably, it might be better solution */ 2932 do { 2933 void ***tls_mem = ((void***)__readfsdword(0x2c))[_tls_index]; 2934 void *val = _tsrm_ls_cache; 2935 size_t offset = 0; 2936 size_t size = (char*)&_tls_end - (char*)&_tls_start; 2937 2938 while (offset < size) { 2939 if (*tls_mem == val) { 2940 tsrm_tls_offset = offset; 2941 break; 2942 } 2943 tls_mem++; 2944 offset += sizeof(void*); 2945 } 2946 if (offset >= size) { 2947 // TODO: error message ??? 2948 return FAILURE; 2949 } 2950 } while(0); 2951# elif defined(__APPLE__) && defined(__x86_64__) 2952 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2953 if (tsrm_ls_cache_tcb_offset == 0) { 2954 size_t *ti; 2955 __asm__( 2956 "leaq __tsrm_ls_cache(%%rip),%0" 2957 : "=r" (ti)); 2958 tsrm_tls_offset = ti[2]; 2959 tsrm_tls_index = ti[1] * 8; 2960 } 2961# elif defined(__GNUC__) && defined(__x86_64__) 2962 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2963 if (tsrm_ls_cache_tcb_offset == 0) { 2964#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__) 2965 size_t ret; 2966 2967 asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0" 2968 : "=r" (ret)); 2969 tsrm_ls_cache_tcb_offset = ret; 2970#else 2971 size_t *ti; 2972 2973 __asm__( 2974 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" 2975 : "=a" (ti)); 2976 tsrm_tls_offset = ti[1]; 2977 tsrm_tls_index = ti[0] * 16; 2978#endif 2979 } 2980# elif defined(__GNUC__) && defined(__i386__) 2981 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2982 if (tsrm_ls_cache_tcb_offset == 0) { 2983#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__) 2984 size_t ret; 2985 2986 asm ("leal _tsrm_ls_cache@ntpoff,%0\n" 2987 : "=a" (ret)); 2988 tsrm_ls_cache_tcb_offset = ret; 2989#else 2990 size_t *ti, _ebx, _ecx, _edx; 2991 2992 __asm__( 2993 "call 1f\n" 2994 ".subsection 1\n" 2995 "1:\tmovl (%%esp), %%ebx\n\t" 2996 "ret\n" 2997 ".previous\n\t" 2998 "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" 2999 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t" 3000 "call ___tls_get_addr@plt\n\t" 3001 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n" 3002 : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx)); 3003 tsrm_tls_offset = ti[1]; 3004 tsrm_tls_index = ti[0] * 8; 3005#endif 3006 } 3007# endif 3008#endif 3009 3010 memset(sp_adj, 0, sizeof(sp_adj)); 3011#ifdef HAVE_GDB 3012 sp_adj[SP_ADJ_RET] = sizeof(void*); 3013 |.if X64WIN 3014 || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 0x28; // sub r4, 0x28 3015 |.elif X64 3016 || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 8; // sub r4, 8 3017 |.else 3018 || sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 12; // sub r4, 12 3019 |.endif 3020 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3021 zend_jit_set_sp_adj_vm(); // set sp_adj[SP_ADJ_VM] 3022#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 3023 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD 3024#else 3025 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM]; 3026#endif 3027 } else if (GCC_GLOBAL_REGS) { 3028 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD; // sub r4, SPAD 3029 } else { 3030 || sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD; // sub r4, NR_SPAD 3031 } 3032#endif 3033 3034 return SUCCESS; 3035} 3036 3037static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) 3038{ 3039 | int3 3040 return 1; 3041} 3042 3043static int zend_jit_align_func(dasm_State **Dst) 3044{ 3045 reuse_ip = 0; 3046 delayed_call_chain = 0; 3047 last_valid_opline = NULL; 3048 use_last_vald_opline = 0; 3049 track_last_valid_opline = 0; 3050 jit_return_label = -1; 3051 |.align 16 3052 return 1; 3053} 3054 3055static int zend_jit_prologue(dasm_State **Dst) 3056{ 3057 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3058 | SUB_HYBRID_SPAD 3059 } else if (GCC_GLOBAL_REGS) { 3060 | sub r4, SPAD // stack alignment 3061 } else { 3062 | sub r4, NR_SPAD // stack alignment 3063 | mov aword T2, FP // save FP 3064 | mov aword T3, RX // save IP 3065 | mov FP, FCARG1a 3066 } 3067 return 1; 3068} 3069 3070static int zend_jit_label(dasm_State **Dst, unsigned int label) 3071{ 3072 |=>label: 3073 return 1; 3074} 3075 3076static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) 3077{ 3078 | // call->prev_execute_data = EX(call); 3079 if (call_level == 1) { 3080 | mov aword EX:RX->prev_execute_data, 0 3081 } else { 3082 | mov r0, EX->call 3083 | mov EX:RX->prev_execute_data, r0 3084 } 3085 | // EX(call) = call; 3086 | mov EX->call, RX 3087 3088 delayed_call_chain = 0; 3089 3090 return 1; 3091} 3092 3093static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) 3094{ 3095 if (last_valid_opline == opline) { 3096 zend_jit_use_last_valid_opline(); 3097 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 3098 zend_jit_use_last_valid_opline(); 3099 | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); 3100 } else { 3101 | LOAD_IP_ADDR opline 3102 } 3103 zend_jit_set_last_valid_opline(opline); 3104 3105 return 1; 3106} 3107 3108static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg) 3109{ 3110 if (last_valid_opline == opline) { 3111 zend_jit_use_last_valid_opline(); 3112 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 3113 zend_jit_use_last_valid_opline(); 3114 | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); 3115 } else if (!GCC_GLOBAL_REGS && set_ip_reg) { 3116 | LOAD_ADDR RX, opline 3117 | mov aword EX->opline, RX 3118 } else { 3119 | LOAD_IP_ADDR opline 3120 } 3121 zend_jit_set_last_valid_opline(opline); 3122 3123 return 1; 3124} 3125 3126static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) 3127{ 3128 if (delayed_call_chain) { 3129 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 3130 return 0; 3131 } 3132 } 3133 if (!zend_jit_set_ip(Dst, opline)) { 3134 return 0; 3135 } 3136 reuse_ip = 0; 3137 return 1; 3138} 3139 3140static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) 3141{ 3142#if 0 3143 if (!zend_jit_set_valid_ip(Dst, opline)) { 3144 return 0; 3145 } 3146 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3147 | jne ->interrupt_handler 3148#else 3149 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3150 if (exit_addr) { 3151 | jne &exit_addr 3152 } else if (last_valid_opline == opline) { 3153 || zend_jit_use_last_valid_opline(); 3154 | jne ->interrupt_handler 3155 } else { 3156 | jne >1 3157 |.cold_code 3158 |1: 3159 | LOAD_IP_ADDR opline 3160 | jmp ->interrupt_handler 3161 |.code 3162 } 3163#endif 3164 return 1; 3165} 3166 3167static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) 3168{ 3169 if (timeout_exit_addr) { 3170 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3171 | je =>loop_label 3172 | jmp &timeout_exit_addr 3173 } else { 3174 | jmp =>loop_label 3175 } 3176 return 1; 3177} 3178 3179static int zend_jit_check_exception(dasm_State **Dst) 3180{ 3181 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 3182 | jne ->exception_handler 3183 return 1; 3184} 3185 3186static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) 3187{ 3188 if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { 3189 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 3190 | jne ->exception_handler_undef 3191 return 1; 3192 } 3193 return zend_jit_check_exception(Dst); 3194} 3195 3196static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) 3197{ 3198 zend_regset regset = ZEND_REGSET_SCRATCH; 3199 3200#if ZTS 3201 if (1) { 3202#else 3203 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(jit_trace_num)))) { 3204#endif 3205 /* assignment to EG(jit_trace_num) shouldn't clober CPU register used by deoptimizer */ 3206 if (parent) { 3207 int i; 3208 int parent_vars_count = parent->exit_info[exit_num].stack_size; 3209 zend_jit_trace_stack *parent_stack = 3210 parent->stack_map + 3211 parent->exit_info[exit_num].stack_offset; 3212 3213 for (i = 0; i < parent_vars_count; i++) { 3214 if (STACK_REG(parent_stack, i) != ZREG_NONE) { 3215 if (STACK_REG(parent_stack, i) < ZREG_NUM) { 3216 ZEND_REGSET_EXCL(regset, STACK_REG(parent_stack, i)); 3217 } else if (STACK_REG(parent_stack, i) == ZREG_ZVAL_COPY_GPR0) { 3218 ZEND_REGSET_EXCL(regset, ZREG_R0); 3219 } 3220 } 3221 } 3222 } 3223 } 3224 3225 if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { 3226 ZEND_REGSET_EXCL(regset, ZREG_R0); 3227 } 3228 3229 current_trace_num = trace_num; 3230 3231 | // EG(jit_trace_num) = trace_num; 3232 if (regset == ZEND_REGSET_EMPTY) { 3233 | push r0 3234 | MEM_STORE_ZTS dword, executor_globals, jit_trace_num, trace_num, r0 3235 | pop r0 3236 } else { 3237 zend_reg tmp = ZEND_REGSET_FIRST(regset); 3238 3239 | MEM_STORE_ZTS dword, executor_globals, jit_trace_num, trace_num, Ra(tmp) 3240 (void)tmp; 3241 } 3242 3243 return 1; 3244} 3245 3246static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t) 3247{ 3248 |.cold_code 3249 |=>1: // end of the code 3250 |.code 3251 return 1; 3252} 3253 3254/* This taken from LuaJIT. Thanks to Mike Pall. */ 3255static uint32_t _asm_x86_inslen(const uint8_t* p) 3256{ 3257 static const uint8_t map_op1[256] = { 3258 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x20, 3259 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51, 3260 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, 3261 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, 3262#if defined(__x86_64__) || defined(_M_X64) 3263 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x14,0x14,0x14,0x14,0x14,0x14,0x14,0x14, 3264#else 3265 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, 3266#endif 3267 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, 3268 0x51,0x51,0x92,0x92,0x10,0x10,0x12,0x11,0x45,0x86,0x52,0x93,0x51,0x51,0x51,0x51, 3269 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 3270 0x93,0x86,0x93,0x93,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 3271 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x47,0x51,0x51,0x51,0x51,0x51, 3272#if defined(__x86_64__) || defined(_M_X64) 3273 0x59,0x59,0x59,0x59,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, 3274#else 3275 0x55,0x55,0x55,0x55,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, 3276#endif 3277 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x05,0x05,0x05,0x05,0x05,0x05,0x05,0x05, 3278 0x93,0x93,0x53,0x51,0x70,0x71,0x93,0x86,0x54,0x51,0x53,0x51,0x51,0x52,0x51,0x51, 3279 0x92,0x92,0x92,0x92,0x52,0x52,0x51,0x51,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 3280 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x45,0x45,0x47,0x52,0x51,0x51,0x51,0x51, 3281 0x10,0x51,0x10,0x10,0x51,0x51,0x63,0x66,0x51,0x51,0x51,0x51,0x51,0x51,0x92,0x92 3282 }; 3283 static const uint8_t map_op2[256] = { 3284 0x93,0x93,0x93,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x51,0x52,0x51,0x93,0x52,0x94, 3285 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3286 0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3287 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x34,0x51,0x35,0x51,0x51,0x51,0x51,0x51, 3288 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3289 0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3290 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3291 0x94,0x54,0x54,0x54,0x93,0x93,0x93,0x52,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3292 0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46, 3293 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3294 0x52,0x52,0x52,0x93,0x94,0x93,0x51,0x51,0x52,0x52,0x52,0x93,0x94,0x93,0x93,0x93, 3295 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x94,0x93,0x93,0x93,0x93,0x93, 3296 0x93,0x93,0x94,0x93,0x94,0x94,0x94,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 3297 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3298 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3299 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x52 3300 }; 3301 uint32_t result = 0; 3302 uint32_t prefixes = 0; 3303 uint32_t x = map_op1[*p]; 3304 3305 for (;;) { 3306 switch (x >> 4) { 3307 case 0: 3308 return result + x + (prefixes & 4); 3309 case 1: 3310 prefixes |= x; 3311 x = map_op1[*++p]; 3312 result++; 3313 break; 3314 case 2: 3315 x = map_op2[*++p]; 3316 break; 3317 case 3: 3318 p++; 3319 goto mrm; 3320 case 4: 3321 result -= (prefixes & 2); 3322 /* fallthrough */ 3323 case 5: 3324 return result + (x & 15); 3325 case 6: /* Group 3. */ 3326 if (p[1] & 0x38) { 3327 x = 2; 3328 } else if ((prefixes & 2) && (x == 0x66)) { 3329 x = 4; 3330 } 3331 goto mrm; 3332 case 7: /* VEX c4/c5. */ 3333#if !defined(__x86_64__) && !defined(_M_X64) 3334 if (p[1] < 0xc0) { 3335 x = 2; 3336 goto mrm; 3337 } 3338#endif 3339 if (x == 0x70) { 3340 x = *++p & 0x1f; 3341 result++; 3342 if (x >= 2) { 3343 p += 2; 3344 result += 2; 3345 goto mrm; 3346 } 3347 } 3348 p++; 3349 result++; 3350 x = map_op2[*++p]; 3351 break; 3352 case 8: 3353 result -= (prefixes & 2); 3354 /* fallthrough */ 3355 case 9: 3356mrm: 3357 /* ModR/M and possibly SIB. */ 3358 result += (x & 15); 3359 x = *++p; 3360 switch (x >> 6) { 3361 case 0: 3362 if ((x & 7) == 5) { 3363 return result + 4; 3364 } 3365 break; 3366 case 1: 3367 result++; 3368 break; 3369 case 2: 3370 result += 4; 3371 break; 3372 case 3: 3373 return result; 3374 } 3375 if ((x & 7) == 4) { 3376 result++; 3377 if (x < 0x40 && (p[1] & 7) == 5) { 3378 result += 4; 3379 } 3380 } 3381 return result; 3382 } 3383 } 3384} 3385 3386typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t); 3387typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t); 3388 3389static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) 3390{ 3391 int ret = 0; 3392 uint8_t *p, *end; 3393 3394 if (jmp_table_size) { 3395 const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*))); 3396 3397 do { 3398 if (*jmp_slot == from_addr) { 3399 *jmp_slot = to_addr; 3400 ret++; 3401 } 3402 jmp_slot++; 3403 } while (--jmp_table_size); 3404 } 3405 3406 p = (uint8_t*)code; 3407 end = p + size - 5; 3408 while (p < end) { 3409 if ((*(unaligned_uint16_t*)p & 0xf0ff) == 0x800f && p + *(unaligned_int32_t*)(p+2) == (uint8_t*)from_addr - 6) { 3410 *(unaligned_int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6)); 3411 ret++; 3412 } else if (*p == 0xe9 && p + *(unaligned_int32_t*)(p+1) == (uint8_t*)from_addr - 5) { 3413 *(unaligned_int32_t*)(p+1) = ((uint8_t*)to_addr - (p + 5)); 3414 ret++; 3415 } 3416 p += _asm_x86_inslen(p); 3417 } 3418#ifdef HAVE_VALGRIND 3419 VALGRIND_DISCARD_TRANSLATIONS(code, size); 3420#endif 3421 return ret; 3422} 3423 3424static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) 3425{ 3426 return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); 3427} 3428 3429static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) 3430{ 3431 const void *link_addr; 3432 size_t prologue_size; 3433 3434 /* Skip prologue. */ 3435 // TODO: don't hardcode this ??? 3436 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3437#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 3438 prologue_size = 0; 3439#elif defined(__x86_64__) || defined(_M_X64) 3440 // sub r4, HYBRID_SPAD 3441 prologue_size = 4; 3442#else 3443 // sub r4, HYBRID_SPAD 3444 prologue_size = 3; 3445#endif 3446 } else if (GCC_GLOBAL_REGS) { 3447 // sub r4, SPAD // stack alignment 3448#if defined(__x86_64__) || defined(_M_X64) 3449 prologue_size = 4; 3450#else 3451 prologue_size = 3; 3452#endif 3453 } else { 3454 // sub r4, NR_SPAD // stack alignment 3455 // mov aword T2, FP // save FP 3456 // mov aword T3, RX // save IP 3457 // mov FP, FCARG1a 3458#if defined(__x86_64__) || defined(_M_X64) 3459 prologue_size = 17; 3460#else 3461 prologue_size = 13; 3462#endif 3463 } 3464 link_addr = (const void*)((const char*)t->code_start + prologue_size); 3465 3466 if (timeout_exit_addr) { 3467 /* Check timeout for links to LOOP */ 3468 | MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0 3469 | je &link_addr 3470 | jmp &timeout_exit_addr 3471 } else { 3472 | jmp &link_addr 3473 } 3474 return 1; 3475} 3476 3477static int zend_jit_trace_return(dasm_State **Dst, bool original_handler, const zend_op *opline) 3478{ 3479#if 0 3480 | jmp ->trace_escape 3481#else 3482 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3483 | ADD_HYBRID_SPAD 3484 if (!original_handler) { 3485 | JMP_IP 3486 } else { 3487 | mov r0, EX->func 3488 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3489 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3490 | jmp aword [IP + r0] 3491 } 3492 } else if (GCC_GLOBAL_REGS) { 3493 | add r4, SPAD // stack alignment 3494 if (!original_handler) { 3495 | JMP_IP 3496 } else { 3497 | mov r0, EX->func 3498 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3499 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3500 | jmp aword [IP + r0] 3501 } 3502 } else { 3503 if (original_handler) { 3504 | mov FCARG1a, FP 3505 | mov r0, EX->func 3506 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3507 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3508 | call aword [IP + r0] 3509 } 3510 | mov FP, aword T2 // restore FP 3511 | mov RX, aword T3 // restore IP 3512 | add r4, NR_SPAD // stack alignment 3513 if (!original_handler || !opline || 3514 (opline->opcode != ZEND_RETURN 3515 && opline->opcode != ZEND_RETURN_BY_REF 3516 && opline->opcode != ZEND_GENERATOR_RETURN 3517 && opline->opcode != ZEND_GENERATOR_CREATE 3518 && opline->opcode != ZEND_YIELD 3519 && opline->opcode != ZEND_YIELD_FROM)) { 3520 | mov r0, 2 // ZEND_VM_LEAVE 3521 } 3522 | ret 3523 } 3524#endif 3525 return 1; 3526} 3527 3528static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) 3529{ 3530 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3531 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3532 3533 if (!exit_addr) { 3534 return 0; 3535 } 3536 | IF_NOT_Z_TYPE FP + var, type, &exit_addr 3537 3538 return 1; 3539} 3540 3541static int zend_jit_scalar_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var) 3542{ 3543 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3544 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3545 3546 if (!exit_addr) { 3547 return 0; 3548 } 3549 | cmp byte [FP+var+offsetof(zval, u1.v.type)], IS_STRING 3550 | jae &exit_addr 3551 3552 return 1; 3553} 3554 3555static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) 3556{ 3557 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 3558 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3559 3560 if (!exit_addr) { 3561 return 0; 3562 } 3563 3564 | GET_ZVAL_LVAL ZREG_FCARG1, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var) 3565 if (op_info & MAY_BE_ARRAY_PACKED) { 3566 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 3567 | jz &exit_addr 3568 } else { 3569 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 3570 | jnz &exit_addr 3571 } 3572 3573 return 1; 3574} 3575 3576static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) 3577{ 3578 zend_jit_op_array_trace_extension *jit_extension = 3579 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); 3580 size_t offset = jit_extension->offset; 3581 const void *handler = 3582 (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; 3583 3584 if (!zend_jit_set_valid_ip(Dst, opline)) { 3585 return 0; 3586 } 3587 if (!GCC_GLOBAL_REGS) { 3588 | mov FCARG1a, FP 3589 } 3590 | EXT_CALL handler, r0 3591 if (may_throw 3592 && opline->opcode != ZEND_RETURN 3593 && opline->opcode != ZEND_RETURN_BY_REF) { 3594 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r1 3595 | jne ->exception_handler 3596 } 3597 3598 while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { 3599 trace++; 3600 } 3601 3602 if (!GCC_GLOBAL_REGS 3603 && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { 3604 if (opline->opcode == ZEND_RETURN || 3605 opline->opcode == ZEND_RETURN_BY_REF || 3606 opline->opcode == ZEND_DO_UCALL || 3607 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3608 opline->opcode == ZEND_DO_FCALL || 3609 opline->opcode == ZEND_GENERATOR_CREATE) { 3610 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r1 3611 } 3612 } 3613 3614 if (zend_jit_trace_may_exit(op_array, opline)) { 3615 if (opline->opcode == ZEND_RETURN || 3616 opline->opcode == ZEND_RETURN_BY_REF || 3617 opline->opcode == ZEND_GENERATOR_CREATE) { 3618 3619 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3620 if (trace->op != ZEND_JIT_TRACE_END || 3621 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && 3622 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 3623 /* this check may be handled by the following OPLINE guard or jmp [IP] */ 3624 | cmp IP, zend_jit_halt_op 3625 | je ->trace_halt 3626 } 3627 } else if (GCC_GLOBAL_REGS) { 3628 | test IP, IP 3629 | je ->trace_halt 3630 } else { 3631 | test eax, eax 3632 | jl ->trace_halt 3633 } 3634 } else if (opline->opcode == ZEND_EXIT || 3635 opline->opcode == ZEND_GENERATOR_RETURN || 3636 opline->opcode == ZEND_YIELD || 3637 opline->opcode == ZEND_YIELD_FROM) { 3638 | jmp ->trace_halt 3639 } 3640 if (trace->op != ZEND_JIT_TRACE_END || 3641 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && 3642 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 3643 3644 const zend_op *next_opline = trace->opline; 3645 const zend_op *exit_opline = NULL; 3646 uint32_t exit_point; 3647 const void *exit_addr; 3648 uint32_t old_info = 0; 3649 uint32_t old_res_info = 0; 3650 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 3651 3652 if (zend_is_smart_branch(opline)) { 3653 bool exit_if_true = 0; 3654 exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); 3655 } else { 3656 switch (opline->opcode) { 3657 case ZEND_JMPZ: 3658 case ZEND_JMPNZ: 3659 case ZEND_JMPZ_EX: 3660 case ZEND_JMPNZ_EX: 3661 case ZEND_JMP_SET: 3662 case ZEND_COALESCE: 3663 case ZEND_JMP_NULL: 3664 case ZEND_FE_RESET_R: 3665 case ZEND_FE_RESET_RW: 3666 exit_opline = (trace->opline == opline + 1) ? 3667 OP_JMP_ADDR(opline, opline->op2) : 3668 opline + 1; 3669 break; 3670 case ZEND_JMPZNZ: 3671 exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ? 3672 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3673 OP_JMP_ADDR(opline, opline->op2); 3674 break; 3675 case ZEND_FE_FETCH_R: 3676 case ZEND_FE_FETCH_RW: 3677 exit_opline = (trace->opline == opline + 1) ? 3678 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3679 opline + 1; 3680 break; 3681 3682 } 3683 } 3684 3685 switch (opline->opcode) { 3686 case ZEND_FE_FETCH_R: 3687 case ZEND_FE_FETCH_RW: 3688 if (opline->op2_type != IS_UNUSED) { 3689 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); 3690 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); 3691 } 3692 break; 3693 } 3694 3695 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3696 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3697 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 3698 } 3699 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); 3700 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3701 3702 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3703 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3704 } 3705 switch (opline->opcode) { 3706 case ZEND_FE_FETCH_R: 3707 case ZEND_FE_FETCH_RW: 3708 if (opline->op2_type != IS_UNUSED) { 3709 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); 3710 } 3711 break; 3712 } 3713 3714 if (!exit_addr) { 3715 return 0; 3716 } 3717 | CMP_IP next_opline 3718 | jne &exit_addr 3719 } 3720 } 3721 3722 zend_jit_set_last_valid_opline(trace->opline); 3723 3724 return 1; 3725} 3726 3727static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) 3728{ 3729 const void *handler; 3730 3731 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3732 handler = zend_get_opcode_handler_func(opline); 3733 } else { 3734 handler = opline->handler; 3735 } 3736 3737 if (!zend_jit_set_valid_ip(Dst, opline)) { 3738 return 0; 3739 } 3740 if (!GCC_GLOBAL_REGS) { 3741 | mov FCARG1a, FP 3742 } 3743 | EXT_CALL handler, r0 3744 if (may_throw) { 3745 zend_jit_check_exception(Dst); 3746 } 3747 3748 /* Skip the following OP_DATA */ 3749 switch (opline->opcode) { 3750 case ZEND_ASSIGN_DIM: 3751 case ZEND_ASSIGN_OBJ: 3752 case ZEND_ASSIGN_STATIC_PROP: 3753 case ZEND_ASSIGN_DIM_OP: 3754 case ZEND_ASSIGN_OBJ_OP: 3755 case ZEND_ASSIGN_STATIC_PROP_OP: 3756 case ZEND_ASSIGN_STATIC_PROP_REF: 3757 case ZEND_ASSIGN_OBJ_REF: 3758 zend_jit_set_last_valid_opline(opline + 2); 3759 break; 3760 default: 3761 zend_jit_set_last_valid_opline(opline + 1); 3762 break; 3763 } 3764 3765 return 1; 3766} 3767 3768static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) 3769{ 3770 if (!zend_jit_set_valid_ip(Dst, opline)) { 3771 return 0; 3772 } 3773 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3774 if (opline->opcode == ZEND_DO_UCALL || 3775 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3776 opline->opcode == ZEND_DO_FCALL || 3777 opline->opcode == ZEND_RETURN) { 3778 3779 /* Use inlined HYBRID VM handler */ 3780 const void *handler = opline->handler; 3781 3782 | ADD_HYBRID_SPAD 3783 | EXT_JMP handler, r0 3784 } else { 3785 const void *handler = zend_get_opcode_handler_func(opline); 3786 3787 | EXT_CALL handler, r0 3788 | ADD_HYBRID_SPAD 3789 | JMP_IP 3790 } 3791 } else { 3792 const void *handler = opline->handler; 3793 3794 if (GCC_GLOBAL_REGS) { 3795 | add r4, SPAD // stack alignment 3796 } else { 3797 | mov FCARG1a, FP 3798 | mov FP, aword T2 // restore FP 3799 | mov RX, aword T3 // restore IP 3800 | add r4, NR_SPAD // stack alignment 3801 } 3802 | EXT_JMP handler, r0 3803 } 3804 zend_jit_reset_last_valid_opline(); 3805 return 1; 3806} 3807 3808static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) 3809{ 3810 uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); 3811 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3812 3813 if (!exit_addr) { 3814 return 0; 3815 } 3816 | CMP_IP opline 3817 | jne &exit_addr 3818 3819 zend_jit_set_last_valid_opline(opline); 3820 3821 return 1; 3822} 3823 3824static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) 3825{ 3826 | jmp =>target_label 3827 return 1; 3828} 3829 3830static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) 3831{ 3832 | CMP_IP next_opline 3833 | jne =>target_label 3834 3835 zend_jit_set_last_valid_opline(next_opline); 3836 3837 return 1; 3838} 3839 3840#ifdef CONTEXT_THREADED_JIT 3841static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3842{ 3843 if (!zend_jit_handler(Dst, opline, 1)) return 0; 3844 if (opline->opcode == ZEND_DO_UCALL) { 3845 | call ->context_threaded_call 3846 } else { 3847 const zend_op *next_opline = opline + 1; 3848 3849 | CMP_IP next_opline 3850 | je =>next_block 3851 | call ->context_threaded_call 3852 } 3853 return 1; 3854} 3855#endif 3856 3857static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3858{ 3859#ifdef CONTEXT_THREADED_JIT 3860 return zend_jit_context_threaded_call(Dst, opline, next_block); 3861#else 3862 return zend_jit_tail_handler(Dst, opline); 3863#endif 3864} 3865 3866static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type) 3867{ 3868 ZEND_ASSERT(Z_MODE(src) == IS_REG); 3869 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); 3870 3871 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3872 | SET_ZVAL_LVAL dst, Ra(Z_REG(src)) 3873 if (set_type && 3874 (Z_REG(dst) != ZREG_FP || 3875 !JIT_G(current_frame) || 3876 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) { 3877 | SET_ZVAL_TYPE_INFO dst, IS_LONG 3878 } 3879 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3880 | DOUBLE_SET_ZVAL_DVAL dst, Z_REG(src) 3881 if (set_type && 3882 (Z_REG(dst) != ZREG_FP || 3883 !JIT_G(current_frame) || 3884 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) { 3885 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 3886 } 3887 } else { 3888 ZEND_UNREACHABLE(); 3889 } 3890 return 1; 3891} 3892 3893static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3894{ 3895 ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); 3896 ZEND_ASSERT(Z_MODE(dst) == IS_REG); 3897 3898 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3899 | GET_ZVAL_LVAL Z_REG(dst), src 3900 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3901 | DOUBLE_GET_ZVAL_DVAL Z_REG(dst), src 3902 } else { 3903 ZEND_UNREACHABLE(); 3904 } 3905 return 1; 3906} 3907 3908static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type) 3909{ 3910 zend_jit_addr src = ZEND_ADDR_REG(reg); 3911 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3912 3913 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3914} 3915 3916static int zend_jit_store_var_type(dasm_State **Dst, int var, uint32_t type) 3917{ 3918 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3919 3920 | SET_ZVAL_TYPE_INFO dst, type 3921 return 1; 3922} 3923 3924static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) 3925{ 3926 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3927 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3928 return zend_jit_spill_store(Dst, src, dst, info, 1); 3929 } 3930 return 1; 3931} 3932 3933static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) 3934{ 3935 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3936 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3937 bool set_type = 1; 3938 3939 if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == 3940 (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { 3941 if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { 3942 set_type = 0; 3943 } 3944 } 3945 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3946 } 3947 return 1; 3948} 3949 3950static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) 3951{ 3952 zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3953 zend_jit_addr dst = ZEND_ADDR_REG(reg); 3954 3955 return zend_jit_load_reg(Dst, src, dst, info); 3956} 3957 3958static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, zend_uchar op_type, zend_jit_addr addr, znode_op op) 3959{ 3960 if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) { 3961 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 3962 | SET_ZVAL_TYPE_INFO dst, IS_UNDEF 3963 } 3964 return 1; 3965} 3966 3967static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3968{ 3969 if (!zend_jit_same_addr(src, dst)) { 3970 if (Z_MODE(src) == IS_REG) { 3971 if (Z_MODE(dst) == IS_REG) { 3972 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3973 | mov Ra(Z_REG(dst)), Ra(Z_REG(src)) 3974 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3975 | SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0) 3976 } else { 3977 ZEND_UNREACHABLE(); 3978 } 3979 if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) { 3980 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3981 3982 if (!zend_jit_spill_store(Dst, dst, var_addr, info, 3983 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3984 JIT_G(current_frame) == NULL || 3985 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3986 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3987 )) { 3988 return 0; 3989 } 3990 } 3991 } else if (Z_MODE(dst) == IS_MEM_ZVAL) { 3992 if (!Z_LOAD(src) && !Z_STORE(src)) { 3993 if (!zend_jit_spill_store(Dst, src, dst, info, 3994 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3995 JIT_G(current_frame) == NULL || 3996 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3997 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3998 )) { 3999 return 0; 4000 } 4001 } 4002 } else { 4003 ZEND_UNREACHABLE(); 4004 } 4005 } else if (Z_MODE(src) == IS_MEM_ZVAL) { 4006 if (Z_MODE(dst) == IS_REG) { 4007 if (!zend_jit_load_reg(Dst, src, dst, info)) { 4008 return 0; 4009 } 4010 } else { 4011 ZEND_UNREACHABLE(); 4012 } 4013 } else { 4014 ZEND_UNREACHABLE(); 4015 } 4016 } else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) { 4017 dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 4018 if (!zend_jit_spill_store(Dst, src, dst, info, 4019 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 4020 JIT_G(current_frame) == NULL || 4021 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 4022 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 4023 )) { 4024 return 0; 4025 } 4026 } 4027 return 1; 4028} 4029 4030static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) 4031{ 4032 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 4033 4034 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1 4035 4036 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { 4037 if (!zend_jit_save_call_chain(Dst, -1)) { 4038 return 0; 4039 } 4040 } 4041 4042 ZEND_ASSERT(opline); 4043 4044 if ((opline-1)->opcode != ZEND_FETCH_CONSTANT 4045 && (opline-1)->opcode != ZEND_FETCH_LIST_R 4046 && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR)) 4047 && !(flags & ZEND_JIT_EXIT_FREE_OP1)) { 4048 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var); 4049 4050 | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 4051 | GET_ZVAL_PTR r0, val_addr 4052 | GC_ADDREF r0 4053 |2: 4054 } 4055 4056 | LOAD_IP_ADDR (opline - 1) 4057 | jmp ->trace_escape 4058 |1: 4059 4060 return 1; 4061} 4062 4063static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) 4064{ 4065 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 4066 4067 if (reg == ZREG_LONG_MIN_MINUS_1) { 4068 |.if X64 4069 | SET_ZVAL_LVAL dst, 0x00000000 4070 | SET_ZVAL_W2 dst, 0xc3e00000 4071 |.else 4072 | SET_ZVAL_LVAL dst, 0x00200000 4073 | SET_ZVAL_W2 dst, 0xc1e00000 4074 |.endif 4075 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 4076 } else if (reg == ZREG_LONG_MIN) { 4077 |.if X64 4078 | SET_ZVAL_LVAL dst, 0x00000000 4079 | SET_ZVAL_W2 dst, 0x80000000 4080 |.else 4081 | SET_ZVAL_LVAL dst, ZEND_LONG_MIN 4082 |.endif 4083 | SET_ZVAL_TYPE_INFO dst, IS_LONG 4084 } else if (reg == ZREG_LONG_MAX) { 4085 |.if X64 4086 | SET_ZVAL_LVAL dst, 0xffffffff 4087 | SET_ZVAL_W2 dst, 0x7fffffff 4088 |.else 4089 | SET_ZVAL_LVAL dst, ZEND_LONG_MAX 4090 |.endif 4091 | SET_ZVAL_TYPE_INFO dst, IS_LONG 4092 } else if (reg == ZREG_LONG_MAX_PLUS_1) { 4093 |.if X64 4094 | SET_ZVAL_LVAL dst, 0 4095 | SET_ZVAL_W2 dst, 0x43e00000 4096 |.else 4097 | SET_ZVAL_LVAL dst, 0 4098 | SET_ZVAL_W2 dst, 0x41e00000 4099 |.endif 4100 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 4101 } else if (reg == ZREG_NULL) { 4102 | SET_ZVAL_TYPE_INFO dst, IS_NULL 4103 } else if (reg == ZREG_ZVAL_TRY_ADDREF) { 4104 | IF_NOT_ZVAL_REFCOUNTED dst, >1 4105 | GET_ZVAL_PTR r1, dst 4106 | GC_ADDREF r1 4107 |1: 4108 } else if (reg == ZREG_ZVAL_COPY_GPR0) { 4109 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 4110 4111 | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_R1, ZREG_R2 4112 | TRY_ADDREF -1, ch, r2 4113 } else { 4114 ZEND_UNREACHABLE(); 4115 } 4116 return 1; 4117} 4118 4119static int zend_jit_free_trampoline(dasm_State **Dst) 4120{ 4121 | /// if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) 4122 | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_CALL_VIA_TRAMPOLINE 4123 | jz >1 4124 | mov FCARG1a, r0 4125 | EXT_CALL zend_jit_free_trampoline_helper, r0 4126 |1: 4127 return 1; 4128} 4129 4130static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 4131{ 4132 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { 4133 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 4134 } 4135 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4136 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4137 } 4138 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { 4139 return 0; 4140 } 4141 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4142 | LONG_OP_WITH_32BIT_CONST add, op1_def_addr, Z_L(1) 4143 } else { 4144 | LONG_OP_WITH_32BIT_CONST sub, op1_def_addr, Z_L(1) 4145 } 4146 4147 if (may_overflow && 4148 (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || 4149 ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { 4150 int32_t exit_point; 4151 const void *exit_addr; 4152 zend_jit_trace_stack *stack; 4153 uint32_t old_op1_info, old_res_info = 0; 4154 4155 stack = JIT_G(current_frame)->stack; 4156 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 4157 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); 4158 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4159 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); 4160 } else { 4161 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); 4162 } 4163 if (opline->result_type != IS_UNUSED) { 4164 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 4165 if (opline->opcode == ZEND_PRE_INC) { 4166 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 4167 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); 4168 } else if (opline->opcode == ZEND_PRE_DEC) { 4169 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 4170 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); 4171 } else if (opline->opcode == ZEND_POST_INC) { 4172 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 4173 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); 4174 } else if (opline->opcode == ZEND_POST_DEC) { 4175 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 4176 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); 4177 } 4178 } 4179 4180 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 4181 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4182 if (!exit_addr) { 4183 return 0; 4184 } 4185 | jo &exit_addr 4186 4187 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4188 opline->result_type != IS_UNUSED) { 4189 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4190 } 4191 4192 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 4193 if (opline->result_type != IS_UNUSED) { 4194 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 4195 } 4196 } else if (may_overflow) { 4197 | jo >1 4198 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4199 opline->result_type != IS_UNUSED) { 4200 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4201 } 4202 |.cold_code 4203 |1: 4204 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4205 |.if X64 4206 | mov64 rax, 0x43e0000000000000 4207 | SET_ZVAL_LVAL op1_def_addr, rax 4208 |.else 4209 | SET_ZVAL_LVAL op1_def_addr, 0 4210 | SET_ZVAL_W2 op1_def_addr, 0x41e00000 4211 |.endif 4212 } else { 4213 |.if X64 4214 | mov64 rax, 0xc3e0000000000000 4215 | SET_ZVAL_LVAL op1_def_addr, rax 4216 |.else 4217 | SET_ZVAL_LVAL op1_def_addr, 0x00200000 4218 | SET_ZVAL_W2 op1_def_addr, 0xc1e00000 4219 |.endif 4220 } 4221 if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { 4222 | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE 4223 } 4224 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4225 opline->result_type != IS_UNUSED) { 4226 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1 4227 } 4228 | jmp >3 4229 |.code 4230 } else { 4231 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4232 opline->result_type != IS_UNUSED) { 4233 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4234 } 4235 } 4236 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 4237 |.cold_code 4238 |2: 4239 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4240 | SET_EX_OPLINE opline, r0 4241 if (op1_info & MAY_BE_UNDEF) { 4242 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2 4243 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 4244 | mov FCARG1d, opline->op1.var 4245 | EXT_CALL zend_jit_undefined_op_helper, r0 4246 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 4247 op1_info |= MAY_BE_NULL; 4248 } 4249 |2: 4250 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 4251 4252 | // ZVAL_DEREF(var_ptr); 4253 if (op1_info & MAY_BE_REF) { 4254 | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >2 4255 | GET_Z_PTR FCARG1a, FCARG1a 4256 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 4257 | jz >1 4258 if (RETURN_VALUE_USED(opline)) { 4259 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4260 } else { 4261 | xor FCARG2a, FCARG2a 4262 } 4263 if (opline->opcode == ZEND_PRE_INC) { 4264 | EXT_CALL zend_jit_pre_inc_typed_ref, r0 4265 } else if (opline->opcode == ZEND_PRE_DEC) { 4266 | EXT_CALL zend_jit_pre_dec_typed_ref, r0 4267 } else if (opline->opcode == ZEND_POST_INC) { 4268 | EXT_CALL zend_jit_post_inc_typed_ref, r0 4269 } else if (opline->opcode == ZEND_POST_DEC) { 4270 | EXT_CALL zend_jit_post_dec_typed_ref, r0 4271 } else { 4272 ZEND_UNREACHABLE(); 4273 } 4274 zend_jit_check_exception(Dst); 4275 | jmp >3 4276 |1: 4277 | lea FCARG1a, [FCARG1a + offsetof(zend_reference, val)] 4278 |2: 4279 } 4280 4281 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4282 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 4283 4284 | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2 4285 | TRY_ADDREF op1_info, ah, r2 4286 } 4287 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4288 if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { 4289 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4290 | EXT_CALL zend_jit_pre_inc, r0 4291 } else { 4292 | EXT_CALL increment_function, r0 4293 } 4294 } else { 4295 if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { 4296 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4297 | EXT_CALL zend_jit_pre_dec, r0 4298 } else { 4299 | EXT_CALL decrement_function, r0 4300 } 4301 } 4302 if (may_throw) { 4303 zend_jit_check_exception(Dst); 4304 } 4305 } else { 4306 zend_reg tmp_reg; 4307 4308 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4309 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2 4310 } 4311 if (Z_MODE(op1_def_addr) == IS_REG) { 4312 tmp_reg = Z_REG(op1_def_addr); 4313 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4314 tmp_reg = Z_REG(op1_addr); 4315 } else { 4316 tmp_reg = ZREG_XMM0; 4317 } 4318 | DOUBLE_GET_ZVAL_DVAL tmp_reg, op1_addr 4319 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4320 if (CAN_USE_AVX()) { 4321 | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] 4322 } else { 4323 | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] 4324 } 4325 } else { 4326 if (CAN_USE_AVX()) { 4327 | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] 4328 } else { 4329 | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] 4330 } 4331 } 4332 | DOUBLE_SET_ZVAL_DVAL op1_def_addr, tmp_reg 4333 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4334 opline->result_type != IS_UNUSED) { 4335 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1 4336 | TRY_ADDREF op1_def_info, ah, r1 4337 } 4338 } 4339 | jmp >3 4340 |.code 4341 } 4342 |3: 4343 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { 4344 return 0; 4345 } 4346 if (opline->result_type != IS_UNUSED) { 4347 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4348 return 0; 4349 } 4350 } 4351 return 1; 4352} 4353 4354static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) 4355{ 4356 if ((opline+1)->opcode == ZEND_OP_DATA 4357 && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) 4358 && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { 4359 return 1; 4360 } 4361 return 4362 ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4363 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || 4364 ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4365 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || 4366 ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4367 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); 4368} 4369 4370static int zend_jit_math_long_long(dasm_State **Dst, 4371 const zend_op *opline, 4372 zend_uchar opcode, 4373 zend_jit_addr op1_addr, 4374 zend_jit_addr op2_addr, 4375 zend_jit_addr res_addr, 4376 uint32_t res_info, 4377 uint32_t res_use_info, 4378 int may_overflow) 4379{ 4380 bool must_set_cflags = 0; 4381 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4382 zend_reg result_reg; 4383 zend_reg tmp_reg = ZREG_R0; 4384 4385 if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { 4386 if (may_overflow && (res_info & MAY_BE_GUARD) 4387 && JIT_G(current_frame) 4388 && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { 4389 result_reg = ZREG_R0; 4390 } else { 4391 result_reg = Z_REG(res_addr); 4392 } 4393 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { 4394 result_reg = Z_REG(op1_addr); 4395 } else if (Z_REG(res_addr) != ZREG_R0) { 4396 result_reg = ZREG_R0; 4397 } else { 4398 /* ASSIGN_DIM_OP */ 4399 result_reg = ZREG_FCARG1; 4400 tmp_reg = ZREG_FCARG1; 4401 } 4402 4403 if (may_overflow) { 4404 must_set_cflags = 1; 4405 } else { 4406 const zend_op *next_opline = opline + 1; 4407 4408 if (next_opline->opcode == ZEND_IS_EQUAL || 4409 next_opline->opcode == ZEND_IS_NOT_EQUAL || 4410 next_opline->opcode == ZEND_IS_SMALLER || 4411 next_opline->opcode == ZEND_IS_SMALLER_OR_EQUAL || 4412 next_opline->opcode == ZEND_CASE || 4413 next_opline->opcode == ZEND_IS_IDENTICAL || 4414 next_opline->opcode == ZEND_IS_NOT_IDENTICAL || 4415 next_opline->opcode == ZEND_CASE_STRICT) { 4416 if (next_opline->op1_type == IS_CONST 4417 && Z_TYPE_P(RT_CONSTANT(next_opline, next_opline->op1)) == IS_LONG 4418 && Z_LVAL_P(RT_CONSTANT(next_opline, next_opline->op1)) == 0 4419 && next_opline->op2_type == opline->result_type 4420 && next_opline->op2.var == opline->result.var) { 4421 must_set_cflags = 1; 4422 } else if (next_opline->op2_type == IS_CONST 4423 && Z_TYPE_P(RT_CONSTANT(next_opline, next_opline->op2)) == IS_LONG 4424 && Z_LVAL_P(RT_CONSTANT(next_opline, next_opline->op2)) == 0 4425 && next_opline->op2_type == opline->result_type 4426 && next_opline->op2.var == opline->result.var) { 4427 must_set_cflags = 1; 4428 } 4429 } 4430 } 4431 4432 if (opcode == ZEND_MUL && 4433 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4434 Z_LVAL_P(Z_ZV(op2_addr)) == 2) { 4435 if (Z_MODE(op1_addr) == IS_REG && !must_set_cflags) { 4436 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] 4437 } else { 4438 | GET_ZVAL_LVAL result_reg, op1_addr 4439 | add Ra(result_reg), Ra(result_reg) 4440 } 4441 } else if (opcode == ZEND_MUL && 4442 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4443 !must_set_cflags && 4444 Z_LVAL_P(Z_ZV(op2_addr)) > 0 && 4445 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { 4446 | GET_ZVAL_LVAL result_reg, op1_addr 4447 | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4448 } else if (opcode == ZEND_MUL && 4449 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4450 Z_LVAL_P(Z_ZV(op1_addr)) == 2) { 4451 if (Z_MODE(op2_addr) == IS_REG && !must_set_cflags) { 4452 | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Ra(Z_REG(op2_addr))] 4453 } else { 4454 | GET_ZVAL_LVAL result_reg, op2_addr 4455 | add Ra(result_reg), Ra(result_reg) 4456 } 4457 } else if (opcode == ZEND_MUL && 4458 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4459 !must_set_cflags && 4460 Z_LVAL_P(Z_ZV(op1_addr)) > 0 && 4461 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { 4462 | GET_ZVAL_LVAL result_reg, op2_addr 4463 | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) 4464 } else if (opcode == ZEND_DIV && 4465 (Z_MODE(op2_addr) == IS_CONST_ZVAL && 4466 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { 4467 | GET_ZVAL_LVAL result_reg, op1_addr 4468 | shr Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4469 } else if (opcode == ZEND_ADD && 4470 !must_set_cflags && 4471 Z_MODE(op1_addr) == IS_REG && 4472 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4473 IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr)))) { 4474 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Z_LVAL_P(Z_ZV(op2_addr))] 4475 } else if (opcode == ZEND_ADD && 4476 !must_set_cflags && 4477 Z_MODE(op2_addr) == IS_REG && 4478 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4479 IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr)))) { 4480 | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Z_LVAL_P(Z_ZV(op1_addr))] 4481 } else if (opcode == ZEND_SUB && 4482 !must_set_cflags && 4483 Z_MODE(op1_addr) == IS_REG && 4484 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4485 IS_SIGNED_32BIT(-Z_LVAL_P(Z_ZV(op2_addr)))) { 4486 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))-Z_LVAL_P(Z_ZV(op2_addr))] 4487 } else { 4488 | GET_ZVAL_LVAL result_reg, op1_addr 4489 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4490 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4491 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4492 /* +/- 0 */ 4493 may_overflow = 0; 4494 } else if (same_ops && opcode != ZEND_DIV) { 4495 | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) 4496 } else { 4497 zend_reg tmp_reg; 4498 4499 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4500 tmp_reg = ZREG_R1; 4501 } else if (result_reg != ZREG_R0) { 4502 tmp_reg = ZREG_R0; 4503 } else { 4504 tmp_reg = ZREG_R1; 4505 } 4506 | LONG_MATH opcode, result_reg, op2_addr, tmp_reg 4507 (void)tmp_reg; 4508 } 4509 } 4510 if (may_overflow) { 4511 if (res_info & MAY_BE_GUARD) { 4512 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 4513 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4514 if (!exit_addr) { 4515 return 0; 4516 } 4517 if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { 4518 | jo &exit_addr 4519 if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { 4520 | mov Ra(Z_REG(res_addr)), Ra(result_reg) 4521 } 4522 } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 4523 | jno &exit_addr 4524 } else { 4525 ZEND_UNREACHABLE(); 4526 } 4527 } else { 4528 if (res_info & MAY_BE_LONG) { 4529 | jo >1 4530 } else { 4531 | jno >1 4532 } 4533 } 4534 } 4535 4536 if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { 4537 | SET_ZVAL_LVAL res_addr, Ra(result_reg) 4538 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4539 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4540 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 4541 } 4542 } 4543 } 4544 4545 if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { 4546 zend_reg tmp_reg1 = ZREG_XMM0; 4547 zend_reg tmp_reg2 = ZREG_XMM1; 4548 4549 if (res_info & MAY_BE_LONG) { 4550 |.cold_code 4551 |1: 4552 } 4553 4554 do { 4555 if ((sizeof(void*) == 8 || Z_MODE(res_addr) != IS_REG) && 4556 ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || 4557 (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1))) { 4558 if (opcode == ZEND_ADD) { 4559 |.if X64 4560 | mov64 Ra(tmp_reg), 0x43e0000000000000 4561 if (Z_MODE(res_addr) == IS_REG) { 4562 | movd xmm(Z_REG(res_addr)-ZREG_XMM0), Ra(tmp_reg) 4563 } else { 4564 | SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 4565 } 4566 |.else 4567 | SET_ZVAL_LVAL res_addr, 0 4568 | SET_ZVAL_W2 res_addr, 0x41e00000 4569 |.endif 4570 break; 4571 } else if (opcode == ZEND_SUB) { 4572 |.if X64 4573 | mov64 Ra(tmp_reg), 0xc3e0000000000000 4574 if (Z_MODE(res_addr) == IS_REG) { 4575 | movd xmm(Z_REG(res_addr)-ZREG_XMM0), Ra(tmp_reg) 4576 } else { 4577 | SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 4578 } 4579 |.else 4580 | SET_ZVAL_LVAL res_addr, 0x00200000 4581 | SET_ZVAL_W2 res_addr, 0xc1e00000 4582 |.endif 4583 break; 4584 } 4585 } 4586 4587 | DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg 4588 | DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg 4589 if (CAN_USE_AVX()) { 4590 | AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 4591 } else { 4592 | SSE_MATH_REG opcode, tmp_reg1, tmp_reg2 4593 } 4594 | DOUBLE_SET_ZVAL_DVAL res_addr, tmp_reg1 4595 } while (0); 4596 4597 if (Z_MODE(res_addr) == IS_MEM_ZVAL 4598 && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4599 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4600 } 4601 if (res_info & MAY_BE_LONG) { 4602 | jmp >2 4603 |.code 4604 } 4605 |2: 4606 } 4607 4608 return 1; 4609} 4610 4611static int zend_jit_math_long_double(dasm_State **Dst, 4612 zend_uchar opcode, 4613 zend_jit_addr op1_addr, 4614 zend_jit_addr op2_addr, 4615 zend_jit_addr res_addr, 4616 uint32_t res_use_info) 4617{ 4618 zend_reg result_reg = 4619 (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; 4620 zend_reg tmp_reg; 4621 4622 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4623 /* ASSIGN_DIM_OP */ 4624 tmp_reg = ZREG_R1; 4625 } else { 4626 tmp_reg = ZREG_R0; 4627 } 4628 4629 | DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, tmp_reg 4630 4631 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4632 /* ASSIGN_DIM_OP */ 4633 if (CAN_USE_AVX()) { 4634 | AVX_MATH opcode, result_reg, result_reg, op2_addr, r1 4635 } else { 4636 | SSE_MATH opcode, result_reg, op2_addr, r1 4637 } 4638 } else { 4639 if (CAN_USE_AVX()) { 4640 | AVX_MATH opcode, result_reg, result_reg, op2_addr, r0 4641 } else { 4642 | SSE_MATH opcode, result_reg, op2_addr, r0 4643 } 4644 } 4645 | DOUBLE_SET_ZVAL_DVAL res_addr, result_reg 4646 4647 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4648 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4649 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4650 } 4651 } 4652 4653 return 1; 4654} 4655 4656static int zend_jit_math_double_long(dasm_State **Dst, 4657 zend_uchar opcode, 4658 zend_jit_addr op1_addr, 4659 zend_jit_addr op2_addr, 4660 zend_jit_addr res_addr, 4661 uint32_t res_use_info) 4662{ 4663 zend_reg result_reg, tmp_reg_gp; 4664 4665 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4666 /* ASSIGN_DIM_OP */ 4667 tmp_reg_gp = ZREG_R1; 4668 } else { 4669 tmp_reg_gp = ZREG_R0; 4670 } 4671 4672 if (zend_is_commutative(opcode) 4673 && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { 4674 if (Z_MODE(res_addr) == IS_REG) { 4675 result_reg = Z_REG(res_addr); 4676 } else { 4677 result_reg = ZREG_XMM0; 4678 } 4679 | DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, tmp_reg_gp 4680 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4681 /* ASSIGN_DIM_OP */ 4682 if (CAN_USE_AVX()) { 4683 | AVX_MATH opcode, result_reg, result_reg, op1_addr, r1 4684 } else { 4685 | SSE_MATH opcode, result_reg, op1_addr, r1 4686 } 4687 } else { 4688 if (CAN_USE_AVX()) { 4689 | AVX_MATH opcode, result_reg, result_reg, op1_addr, r0 4690 } else { 4691 | SSE_MATH opcode, result_reg, op1_addr, r0 4692 } 4693 } 4694 } else { 4695 zend_reg tmp_reg; 4696 4697 if (Z_MODE(res_addr) == IS_REG) { 4698 result_reg = Z_REG(res_addr); 4699 tmp_reg = (result_reg == ZREG_XMM0) ? ZREG_XMM1 : ZREG_XMM0; 4700 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4701 result_reg = Z_REG(op1_addr); 4702 tmp_reg = ZREG_XMM0; 4703 } else { 4704 result_reg = ZREG_XMM0; 4705 tmp_reg = ZREG_XMM1; 4706 } 4707 if (CAN_USE_AVX()) { 4708 zend_reg op1_reg; 4709 4710 if (Z_MODE(op1_addr) == IS_REG) { 4711 op1_reg = Z_REG(op1_addr); 4712 } else { 4713 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4714 op1_reg = result_reg; 4715 } 4716 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4717 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4718 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4719 /* +/- 0 */ 4720 } else { 4721 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp 4722 | AVX_MATH_REG opcode, result_reg, op1_reg, tmp_reg 4723 } 4724 } else { 4725 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4726 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4727 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4728 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4729 /* +/- 0 */ 4730 } else { 4731 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp 4732 | SSE_MATH_REG opcode, result_reg, tmp_reg 4733 } 4734 } 4735 } 4736 | DOUBLE_SET_ZVAL_DVAL res_addr, result_reg 4737 4738 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4739 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4740 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4741 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4742 } 4743 } 4744 } 4745 4746 return 1; 4747} 4748 4749static int zend_jit_math_double_double(dasm_State **Dst, 4750 zend_uchar opcode, 4751 zend_jit_addr op1_addr, 4752 zend_jit_addr op2_addr, 4753 zend_jit_addr res_addr, 4754 uint32_t res_use_info) 4755{ 4756 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4757 zend_reg result_reg; 4758 4759 if (Z_MODE(res_addr) == IS_REG) { 4760 result_reg = Z_REG(res_addr); 4761 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4762 result_reg = Z_REG(op1_addr); 4763 } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { 4764 result_reg = Z_REG(op2_addr); 4765 } else { 4766 result_reg = ZREG_XMM0; 4767 } 4768 4769 if (CAN_USE_AVX()) { 4770 zend_reg op1_reg; 4771 zend_jit_addr val_addr; 4772 4773 if (Z_MODE(op1_addr) == IS_REG) { 4774 op1_reg = Z_REG(op1_addr); 4775 val_addr = op2_addr; 4776 } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4777 op1_reg = Z_REG(op2_addr); 4778 val_addr = op1_addr; 4779 } else { 4780 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4781 op1_reg = result_reg; 4782 val_addr = op2_addr; 4783 } 4784 if ((opcode == ZEND_MUL) && 4785 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4786 | AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg 4787 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4788 /* ASSIGN_DIM_OP */ 4789 | AVX_MATH opcode, result_reg, op1_reg, val_addr, r1 4790 } else { 4791 | AVX_MATH opcode, result_reg, op1_reg, val_addr, r0 4792 } 4793 } else { 4794 zend_jit_addr val_addr; 4795 4796 if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4797 | DOUBLE_GET_ZVAL_DVAL result_reg, op2_addr 4798 val_addr = op1_addr; 4799 } else { 4800 | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr 4801 val_addr = op2_addr; 4802 } 4803 if (same_ops) { 4804 | SSE_MATH_REG opcode, result_reg, result_reg 4805 } else if ((opcode == ZEND_MUL) && 4806 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4807 | SSE_MATH_REG ZEND_ADD, result_reg, result_reg 4808 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4809 /* ASSIGN_DIM_OP */ 4810 | SSE_MATH opcode, result_reg, val_addr, r1 4811 } else { 4812 | SSE_MATH opcode, result_reg, val_addr, r0 4813 } 4814 } 4815 | DOUBLE_SET_ZVAL_DVAL res_addr, result_reg 4816 4817 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4818 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4819 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4820 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4821 } 4822 } 4823 } 4824 4825 return 1; 4826} 4827 4828static int zend_jit_math_helper(dasm_State **Dst, 4829 const zend_op *opline, 4830 zend_uchar opcode, 4831 zend_uchar op1_type, 4832 znode_op op1, 4833 zend_jit_addr op1_addr, 4834 uint32_t op1_info, 4835 zend_uchar op2_type, 4836 znode_op op2, 4837 zend_jit_addr op2_addr, 4838 uint32_t op2_info, 4839 uint32_t res_var, 4840 zend_jit_addr res_addr, 4841 uint32_t res_info, 4842 uint32_t res_use_info, 4843 int may_overflow, 4844 int may_throw) 4845/* Labels: 1,2,3,4,5,6 */ 4846{ 4847 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4848 4849 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4850 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { 4851 if (op1_info & MAY_BE_DOUBLE) { 4852 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 4853 } else { 4854 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 4855 } 4856 } 4857 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { 4858 if (op2_info & MAY_BE_DOUBLE) { 4859 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1 4860 |.cold_code 4861 |1: 4862 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4863 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4864 } 4865 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4866 return 0; 4867 } 4868 | jmp >5 4869 |.code 4870 } else { 4871 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4872 } 4873 } 4874 if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { 4875 return 0; 4876 } 4877 if (op1_info & MAY_BE_DOUBLE) { 4878 |.cold_code 4879 |3: 4880 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4881 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4882 } 4883 if (op2_info & MAY_BE_DOUBLE) { 4884 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4885 if (!same_ops) { 4886 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1 4887 } else { 4888 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6 4889 } 4890 } 4891 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4892 return 0; 4893 } 4894 | jmp >5 4895 } 4896 if (!same_ops) { 4897 |1: 4898 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4899 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4900 } 4901 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4902 return 0; 4903 } 4904 | jmp >5 4905 } 4906 |.code 4907 } 4908 } else if ((op1_info & MAY_BE_DOUBLE) && 4909 !(op1_info & MAY_BE_LONG) && 4910 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4911 (res_info & MAY_BE_DOUBLE)) { 4912 if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4913 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4914 } 4915 if (op2_info & MAY_BE_DOUBLE) { 4916 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4917 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4918 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1 4919 } else { 4920 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4921 } 4922 } 4923 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4924 return 0; 4925 } 4926 } 4927 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4928 if (op2_info & MAY_BE_DOUBLE) { 4929 |.cold_code 4930 } 4931 |1: 4932 if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4933 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4934 } 4935 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4936 return 0; 4937 } 4938 if (op2_info & MAY_BE_DOUBLE) { 4939 | jmp >5 4940 |.code 4941 } 4942 } 4943 } else if ((op2_info & MAY_BE_DOUBLE) && 4944 !(op2_info & MAY_BE_LONG) && 4945 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4946 (res_info & MAY_BE_DOUBLE)) { 4947 if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4948 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4949 } 4950 if (op1_info & MAY_BE_DOUBLE) { 4951 if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4952 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4953 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1 4954 } else { 4955 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4956 } 4957 } 4958 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4959 return 0; 4960 } 4961 } 4962 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4963 if (op1_info & MAY_BE_DOUBLE) { 4964 |.cold_code 4965 } 4966 |1: 4967 if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4968 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 4969 } 4970 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4971 return 0; 4972 } 4973 if (op1_info & MAY_BE_DOUBLE) { 4974 | jmp >5 4975 |.code 4976 } 4977 } 4978 } 4979 4980 |5: 4981 4982 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 4983 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 4984 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4985 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4986 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4987 |.cold_code 4988 } 4989 |6: 4990 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 4991 if (Z_MODE(res_addr) == IS_REG) { 4992 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4993 | LOAD_ZVAL_ADDR FCARG1a, real_addr 4994 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 4995 | LOAD_ZVAL_ADDR FCARG1a, res_addr 4996 } 4997 if (Z_MODE(op1_addr) == IS_REG) { 4998 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4999 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 5000 return 0; 5001 } 5002 op1_addr = real_addr; 5003 } 5004 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5005 } else { 5006 if (Z_MODE(op1_addr) == IS_REG) { 5007 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 5008 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 5009 return 0; 5010 } 5011 op1_addr = real_addr; 5012 } 5013 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5014 if (Z_MODE(res_addr) == IS_REG) { 5015 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5016 | LOAD_ZVAL_ADDR FCARG1a, real_addr 5017 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5018 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5019 } 5020 } 5021 5022 if (Z_MODE(op2_addr) == IS_REG) { 5023 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 5024 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 5025 return 0; 5026 } 5027 op2_addr = real_addr; 5028 } 5029 |.if X64 5030 | LOAD_ZVAL_ADDR CARG3, op2_addr 5031 |.else 5032 | sub r4, 12 5033 | PUSH_ZVAL_ADDR op2_addr, r0 5034 |.endif 5035 | SET_EX_OPLINE opline, r0 5036 if (opcode == ZEND_ADD) { 5037 | EXT_CALL add_function, r0 5038 } else if (opcode == ZEND_SUB) { 5039 | EXT_CALL sub_function, r0 5040 } else if (opcode == ZEND_MUL) { 5041 | EXT_CALL mul_function, r0 5042 } else if (opcode == ZEND_DIV) { 5043 | EXT_CALL div_function, r0 5044 } else { 5045 ZEND_UNREACHABLE(); 5046 } 5047 |.if not(X64) 5048 | add r4, 12 5049 |.endif 5050 | FREE_OP op1_type, op1, op1_info, 0, NULL 5051 | FREE_OP op2_type, op2, op2_info, 0, NULL 5052 if (may_throw) { 5053 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 5054 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 5055 | jne ->exception_handler_free_op2 5056 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5057 zend_jit_check_exception_undef_result(Dst, opline); 5058 } else { 5059 zend_jit_check_exception(Dst); 5060 } 5061 } 5062 if (Z_MODE(res_addr) == IS_REG) { 5063 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5064 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 5065 return 0; 5066 } 5067 } 5068 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 5069 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 5070 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 5071 | jmp <5 5072 |.code 5073 } 5074 } 5075 5076 return 1; 5077} 5078 5079static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 5080{ 5081 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5082 ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 5083 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); 5084 5085 if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { 5086 return 0; 5087 } 5088 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 5089 return 0; 5090 } 5091 return 1; 5092} 5093 5094static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr) 5095{ 5096 if (Z_MODE(op2_addr) != IS_MEM_ZVAL || Z_REG(op2_addr) != ZREG_FCARG1) { 5097 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 5098 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5099 } else if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG2) { 5100 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5101 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 5102 } else { 5103 | GET_ZVAL_LVAL ZREG_R0, op2_addr 5104 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 5105 | mov FCARG2a, r0 5106 } 5107 | EXT_CALL zend_jit_add_arrays_helper, r0 5108 | SET_ZVAL_PTR res_addr, r0 5109 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX 5110 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 5111 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 5112 return 1; 5113} 5114 5115static int zend_jit_long_math_helper(dasm_State **Dst, 5116 const zend_op *opline, 5117 zend_uchar opcode, 5118 zend_uchar op1_type, 5119 znode_op op1, 5120 zend_jit_addr op1_addr, 5121 uint32_t op1_info, 5122 zend_ssa_range *op1_range, 5123 zend_uchar op2_type, 5124 znode_op op2, 5125 zend_jit_addr op2_addr, 5126 uint32_t op2_info, 5127 zend_ssa_range *op2_range, 5128 uint32_t res_var, 5129 zend_jit_addr res_addr, 5130 uint32_t res_info, 5131 uint32_t res_use_info, 5132 int may_throw) 5133/* Labels: 6 */ 5134{ 5135 bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 5136 zend_reg result_reg; 5137 5138 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 5139 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 5140 } 5141 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 5142 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 5143 } 5144 5145 if (opcode == ZEND_MOD) { 5146 result_reg = ZREG_RAX; 5147 } else if (Z_MODE(res_addr) == IS_REG) { 5148 if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) 5149 && opline->op2_type != IS_CONST) { 5150 result_reg = ZREG_R0; 5151 } else { 5152 result_reg = Z_REG(res_addr); 5153 } 5154 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 5155 result_reg = Z_REG(op1_addr); 5156 } else if (Z_REG(res_addr) != ZREG_R0) { 5157 result_reg = ZREG_R0; 5158 } else { 5159 /* ASSIGN_DIM_OP */ 5160 if (ZREG_FCARG1 == ZREG_RCX 5161 && (opcode == ZEND_SL || opcode == ZEND_SR) 5162 && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 5163 result_reg = ZREG_R2; 5164 } else { 5165 result_reg = ZREG_FCARG1; 5166 } 5167 } 5168 5169 if (opcode == ZEND_SL) { 5170 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5171 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5172 5173 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 5174 if (EXPECTED(op2_lval > 0)) { 5175 | xor Ra(result_reg), Ra(result_reg) 5176 } else { 5177 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5178 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5179 | SET_EX_OPLINE opline, r0 5180 | jmp ->negative_shift 5181 } 5182 } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { 5183 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] 5184 } else { 5185 | GET_ZVAL_LVAL result_reg, op1_addr 5186 | shl Ra(result_reg), op2_lval 5187 } 5188 } else { 5189 if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { 5190 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5191 } 5192 if (!op2_range || 5193 op2_range->min < 0 || 5194 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 5195 | cmp r1, (SIZEOF_ZEND_LONG*8) 5196 | jae >1 5197 |.cold_code 5198 |1: 5199 | cmp r1, 0 5200 | mov Ra(result_reg), 0 5201 | jg >1 5202 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5203 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5204 | SET_EX_OPLINE opline, r0 5205 | jmp ->negative_shift 5206 |.code 5207 } 5208 | GET_ZVAL_LVAL result_reg, op1_addr 5209 | shl Ra(result_reg), cl 5210 |1: 5211 } 5212 } else if (opcode == ZEND_SR) { 5213 | GET_ZVAL_LVAL result_reg, op1_addr 5214 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5215 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5216 5217 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 5218 if (EXPECTED(op2_lval > 0)) { 5219 | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1 5220 } else { 5221 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5222 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5223 | SET_EX_OPLINE opline, r0 5224 | jmp ->negative_shift 5225 } 5226 } else { 5227 | sar Ra(result_reg), op2_lval 5228 } 5229 } else { 5230 if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { 5231 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5232 } 5233 if (!op2_range || 5234 op2_range->min < 0 || 5235 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 5236 | cmp r1, (SIZEOF_ZEND_LONG*8) 5237 | jae >1 5238 |.cold_code 5239 |1: 5240 | cmp r1, 0 5241 | mov r1, (SIZEOF_ZEND_LONG * 8) - 1 5242 | jg >1 5243 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5244 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5245 | SET_EX_OPLINE opline, r0 5246 | jmp ->negative_shift 5247 |.code 5248 } 5249 |1: 5250 | sar Ra(result_reg), cl 5251 } 5252 } else if (opcode == ZEND_MOD) { 5253 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5254 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5255 5256 if (op2_lval == 0) { 5257 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5258 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5259 | SET_EX_OPLINE opline, r0 5260 | jmp ->mod_by_zero 5261 } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) { 5262 zval tmp; 5263 zend_jit_addr tmp_addr; 5264 zend_reg tmp_reg; 5265 5266 /* Optimisation for mod of power of 2 */ 5267 ZVAL_LONG(&tmp, op2_lval - 1); 5268 tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp); 5269 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 5270 tmp_reg = ZREG_R1; 5271 } else if (result_reg != ZREG_R0) { 5272 tmp_reg = ZREG_R0; 5273 } else { 5274 tmp_reg = ZREG_R1; 5275 } 5276 | GET_ZVAL_LVAL result_reg, op1_addr 5277 | LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, tmp_reg 5278 (void)tmp_reg; 5279 } else { 5280 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5281 | mov aword T1, r0 // save 5282 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) { 5283 | mov aword T1, Ra(ZREG_RCX) // save 5284 } 5285 result_reg = ZREG_RDX; 5286 if (op2_lval == -1) { 5287 | xor Ra(result_reg), Ra(result_reg) 5288 } else { 5289 | GET_ZVAL_LVAL ZREG_RAX, op1_addr 5290 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5291 |.if X64 5292 | cqo 5293 |.else 5294 | cdq 5295 |.endif 5296 | idiv Ra(ZREG_RCX) 5297 } 5298 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5299 | mov r0, aword T1 // restore 5300 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) { 5301 | mov Ra(ZREG_RCX), aword T1 // restore 5302 } 5303 } 5304 } else { 5305 if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { 5306 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5307 | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], 0 5308 } else if (Z_MODE(op2_addr) == IS_REG) { 5309 | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) 5310 } 5311 | jz >1 5312 |.cold_code 5313 |1: 5314 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5315 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5316 | SET_EX_OPLINE opline, r0 5317 | jmp ->mod_by_zero 5318 |.code 5319 } 5320 5321 /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ 5322 if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { 5323 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5324 | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], -1 5325 } else if (Z_MODE(op2_addr) == IS_REG) { 5326 | cmp Ra(Z_REG(op2_addr)), -1 5327 } 5328 | jz >1 5329 |.cold_code 5330 |1: 5331 | SET_ZVAL_LVAL res_addr, 0 5332 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5333 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 5334 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 5335 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 5336 } 5337 } 5338 } 5339 | jmp >5 5340 |.code 5341 } 5342 5343 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5344 | mov aword T1, r0 // save 5345 } 5346 result_reg = ZREG_RDX; 5347 | GET_ZVAL_LVAL ZREG_RAX, op1_addr 5348 |.if X64 5349 | cqo 5350 |.else 5351 | cdq 5352 |.endif 5353 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5354 | idiv aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)] 5355 } else if (Z_MODE(op2_addr) == IS_REG) { 5356 | idiv Ra(Z_REG(op2_addr)) 5357 } 5358 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5359 | mov r0, aword T1 // restore 5360 } 5361 } 5362 } else if (same_ops) { 5363 | GET_ZVAL_LVAL result_reg, op1_addr 5364 | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) 5365 } else { 5366 zend_reg tmp_reg; 5367 5368 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 5369 tmp_reg = ZREG_R1; 5370 } else if (result_reg != ZREG_R0) { 5371 tmp_reg = ZREG_R0; 5372 } else { 5373 tmp_reg = ZREG_R1; 5374 } 5375 | GET_ZVAL_LVAL result_reg, op1_addr 5376 | LONG_MATH opcode, result_reg, op2_addr, tmp_reg 5377 (void)tmp_reg; 5378 } 5379 5380 if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { 5381 | SET_ZVAL_LVAL res_addr, Ra(result_reg) 5382 } 5383 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5384 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 5385 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 5386 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 5387 } 5388 } 5389 } 5390 5391 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || 5392 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 5393 if ((op1_info & MAY_BE_LONG) && 5394 (op2_info & MAY_BE_LONG)) { 5395 |.cold_code 5396 } 5397 |6: 5398 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 5399 if (Z_MODE(res_addr) == IS_REG) { 5400 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5401 | LOAD_ZVAL_ADDR FCARG1a, real_addr 5402 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5403 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5404 } 5405 if (Z_MODE(op1_addr) == IS_REG) { 5406 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 5407 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 5408 return 0; 5409 } 5410 op1_addr = real_addr; 5411 } 5412 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5413 } else { 5414 if (Z_MODE(op1_addr) == IS_REG) { 5415 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 5416 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 5417 return 0; 5418 } 5419 op1_addr = real_addr; 5420 } 5421 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5422 if (Z_MODE(res_addr) == IS_REG) { 5423 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5424 | LOAD_ZVAL_ADDR FCARG1a, real_addr 5425 } else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5426 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5427 } 5428 } 5429 if (Z_MODE(op2_addr) == IS_REG) { 5430 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 5431 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 5432 return 0; 5433 } 5434 op2_addr = real_addr; 5435 } 5436 |.if X64 5437 | LOAD_ZVAL_ADDR CARG3, op2_addr 5438 |.else 5439 | sub r4, 12 5440 | PUSH_ZVAL_ADDR op2_addr, r0 5441 |.endif 5442 | SET_EX_OPLINE opline, r0 5443 if (opcode == ZEND_BW_OR) { 5444 | EXT_CALL bitwise_or_function, r0 5445 } else if (opcode == ZEND_BW_AND) { 5446 | EXT_CALL bitwise_and_function, r0 5447 } else if (opcode == ZEND_BW_XOR) { 5448 | EXT_CALL bitwise_xor_function, r0 5449 } else if (opcode == ZEND_SL) { 5450 | EXT_CALL shift_left_function, r0 5451 } else if (opcode == ZEND_SR) { 5452 | EXT_CALL shift_right_function, r0 5453 } else if (opcode == ZEND_MOD) { 5454 | EXT_CALL mod_function, r0 5455 } else { 5456 ZEND_UNREACHABLE(); 5457 } 5458 |.if not(X64) 5459 | add r4, 12 5460 |.endif 5461 if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) { 5462 /* compound assignment may decrement "op2" refcount */ 5463 op2_info |= MAY_BE_RC1; 5464 } 5465 | FREE_OP op1_type, op1, op1_info, 0, NULL 5466 | FREE_OP op2_type, op2, op2_info, 0, NULL 5467 if (may_throw) { 5468 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 5469 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 5470 | jne ->exception_handler_free_op2 5471 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5472 zend_jit_check_exception_undef_result(Dst, opline); 5473 } else { 5474 zend_jit_check_exception(Dst); 5475 } 5476 } 5477 if (Z_MODE(res_addr) == IS_REG) { 5478 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5479 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 5480 return 0; 5481 } 5482 } 5483 if ((op1_info & MAY_BE_LONG) && 5484 (op2_info & MAY_BE_LONG)) { 5485 | jmp >5 5486 |.code 5487 } 5488 } 5489 |5: 5490 5491 return 1; 5492} 5493 5494static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 5495{ 5496 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5497 ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); 5498 5499 if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, 5500 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 5501 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 5502 opline->result.var, res_addr, res_info, res_use_info, may_throw)) { 5503 return 0; 5504 } 5505 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 5506 return 0; 5507 } 5508 return 1; 5509} 5510 5511static int zend_jit_concat_helper(dasm_State **Dst, 5512 const zend_op *opline, 5513 zend_uchar op1_type, 5514 znode_op op1, 5515 zend_jit_addr op1_addr, 5516 uint32_t op1_info, 5517 zend_uchar op2_type, 5518 znode_op op2, 5519 zend_jit_addr op2_addr, 5520 uint32_t op2_info, 5521 zend_jit_addr res_addr, 5522 int may_throw) 5523{ 5524#if 1 5525 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5526 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5527 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 5528 } 5529 if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5530 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6 5531 } 5532 if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { 5533 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5534 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5535 } 5536 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 5537 | EXT_CALL zend_jit_fast_assign_concat_helper, r0 5538 /* concatination with itself may reduce refcount */ 5539 op2_info |= MAY_BE_RC1; 5540 } else { 5541 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5542 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5543 } 5544 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5545 |.if X64 5546 | LOAD_ZVAL_ADDR CARG3, op2_addr 5547 |.else 5548 | sub r4, 12 5549 | PUSH_ZVAL_ADDR op2_addr, r0 5550 |.endif 5551 if (op1_type == IS_CV || op1_type == IS_CONST) { 5552 | EXT_CALL zend_jit_fast_concat_helper, r0 5553 } else { 5554 | EXT_CALL zend_jit_fast_concat_tmp_helper, r0 5555 } 5556 |.if not(X64) 5557 | add r4, 12 5558 |.endif 5559 } 5560 /* concatination with empty string may increase refcount */ 5561 op2_info |= MAY_BE_RCN; 5562 | FREE_OP op2_type, op2, op2_info, 0, opline 5563 |5: 5564 } 5565 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || 5566 (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { 5567 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5568 |.cold_code 5569 |6: 5570 } 5571#endif 5572 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) { 5573 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5574 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5575 } 5576 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5577 } else { 5578 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5579 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 5580 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5581 } 5582 } 5583 |.if X64 5584 | LOAD_ZVAL_ADDR CARG3, op2_addr 5585 |.else 5586 | sub r4, 12 5587 | PUSH_ZVAL_ADDR op2_addr, r0 5588 |.endif 5589 | SET_EX_OPLINE opline, r0 5590 | EXT_CALL concat_function, r0 5591 |.if not(X64) 5592 | add r4, 12 5593 |.endif 5594 /* concatination with empty string may increase refcount */ 5595 op1_info |= MAY_BE_RCN; 5596 op2_info |= MAY_BE_RCN; 5597 | FREE_OP op1_type, op1, op1_info, 0, NULL 5598 | FREE_OP op2_type, op2, op2_info, 0, NULL 5599 if (may_throw) { 5600 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5601 zend_jit_check_exception_undef_result(Dst, opline); 5602 } else { 5603 zend_jit_check_exception(Dst); 5604 } 5605 } 5606#if 1 5607 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5608 | jmp <5 5609 |.code 5610 } 5611 } 5612#endif 5613 5614 return 1; 5615} 5616 5617static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) 5618{ 5619 zend_jit_addr op1_addr, op2_addr; 5620 5621 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5622 ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); 5623 5624 op1_addr = OP1_ADDR(); 5625 op2_addr = OP2_ADDR(); 5626 5627 return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); 5628} 5629 5630static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) 5631/* Labels: 1,2,3,4,5 */ 5632{ 5633 zend_jit_addr op2_addr = OP2_ADDR(); 5634 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 5635 5636 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 5637 && type == BP_VAR_R 5638 && !exit_addr) { 5639 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5640 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5641 if (!exit_addr) { 5642 return 0; 5643 } 5644 } 5645 5646 if (op2_info & MAY_BE_LONG) { 5647 bool op2_loaded = 0; 5648 bool packed_loaded = 0; 5649 bool bad_packed_key = 0; 5650 5651 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { 5652 | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) 5653 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 5654 } 5655 if (op1_info & MAY_BE_PACKED_GUARD) { 5656 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 5657 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5658 5659 if (!exit_addr) { 5660 return 0; 5661 } 5662 if (op1_info & MAY_BE_ARRAY_PACKED) { 5663 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5664 | jz &exit_addr 5665 } else { 5666 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5667 | jnz &exit_addr 5668 } 5669 } 5670 if (type == BP_VAR_W) { 5671 | // hval = Z_LVAL_P(dim); 5672 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5673 op2_loaded = 1; 5674 } 5675 if (op1_info & MAY_BE_ARRAY_PACKED) { 5676 zend_long val = -1; 5677 5678 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5679 val = Z_LVAL_P(Z_ZV(op2_addr)); 5680 if (val >= 0 && val < HT_MAX_SIZE) { 5681 packed_loaded = 1; 5682 } else { 5683 bad_packed_key = 1; 5684 } 5685 } else { 5686 if (!op2_loaded) { 5687 | // hval = Z_LVAL_P(dim); 5688 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5689 op2_loaded = 1; 5690 } 5691 packed_loaded = 1; 5692 } 5693 5694 if (dim_type == IS_UNDEF && type == BP_VAR_W && packed_loaded) { 5695 /* don't generate "fast" code for packed array */ 5696 packed_loaded = 0; 5697 } 5698 5699 if (packed_loaded) { 5700 | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); 5701 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5702 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5703 | jz >4 // HASH_FIND 5704 } 5705 | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) 5706 |.if X64 5707 | mov eax, dword [FCARG1a + offsetof(zend_array, nNumUsed)] 5708 if (val == 0) { 5709 | test r0, r0 5710 } else if (val > 0 && !op2_loaded) { 5711 | cmp r0, val 5712 } else { 5713 | cmp r0, FCARG2a 5714 } 5715 |.else 5716 if (val >= 0 && !op2_loaded) { 5717 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val 5718 } else { 5719 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a 5720 } 5721 |.endif 5722 if (type == BP_JIT_IS) { 5723 if (not_found_exit_addr) { 5724 | jbe ¬_found_exit_addr 5725 } else { 5726 | jbe >9 // NOT_FOUND 5727 } 5728 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5729 | jbe &exit_addr 5730 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5731 | jbe ¬_found_exit_addr 5732 } else if (type == BP_VAR_RW && not_found_exit_addr) { 5733 | jbe ¬_found_exit_addr 5734 } else if (type == BP_VAR_IS && found_exit_addr) { 5735 | jbe >7 // NOT_FOUND 5736 } else { 5737 | jbe >2 // NOT_FOUND 5738 } 5739 | // _ret = &_ht->arData[_h].val; 5740 if (val >= 0) { 5741 | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] 5742 if (val != 0) { 5743 | add r0, val * sizeof(Bucket) 5744 } 5745 } else { 5746 |.if X64 5747 | mov r0, FCARG2a 5748 | shl r0, 5 5749 |.else 5750 | imul r0, FCARG2a, sizeof(Bucket) 5751 |.endif 5752 | add r0, aword [FCARG1a + offsetof(zend_array, arData)] 5753 } 5754 } 5755 } 5756 switch (type) { 5757 case BP_JIT_IS: 5758 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5759 if (packed_loaded) { 5760 | jmp >5 5761 } 5762 |4: 5763 if (!op2_loaded) { 5764 | // hval = Z_LVAL_P(dim); 5765 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5766 } 5767 if (packed_loaded) { 5768 | EXT_CALL _zend_hash_index_find, r0 5769 } else { 5770 | EXT_CALL zend_hash_index_find, r0 5771 } 5772 | test r0, r0 5773 if (not_found_exit_addr) { 5774 | jz ¬_found_exit_addr 5775 } else { 5776 | jz >9 // NOT_FOUND 5777 } 5778 if (op2_info & MAY_BE_STRING) { 5779 | jmp >5 5780 } 5781 } else if (packed_loaded) { 5782 if (op2_info & MAY_BE_STRING) { 5783 | jmp >5 5784 } 5785 } else if (not_found_exit_addr) { 5786 | jmp ¬_found_exit_addr 5787 } else { 5788 | jmp >9 // NOT_FOUND 5789 } 5790 break; 5791 case BP_VAR_R: 5792 case BP_VAR_IS: 5793 case BP_VAR_UNSET: 5794 if (packed_loaded) { 5795 if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { 5796 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5797 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5798 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 5799 if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5800 | IF_Z_TYPE r0, IS_UNDEF, &exit_addr 5801 } 5802 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5803 | IF_Z_TYPE r0, IS_UNDEF, ¬_found_exit_addr 5804 } else if (type == BP_VAR_IS && found_exit_addr) { 5805 | IF_Z_TYPE r0, IS_UNDEF, >7 // NOT_FOUND 5806 } else { 5807 | IF_Z_TYPE r0, IS_UNDEF, >2 // NOT_FOUND 5808 } 5809 } 5810 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_NUMERIC_HASH))) { 5811 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5812 | jmp &exit_addr 5813 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5814 | jmp ¬_found_exit_addr 5815 } else if (type == BP_VAR_IS && found_exit_addr) { 5816 | jmp >7 // NOT_FOUND 5817 } else { 5818 | jmp >2 // NOT_FOUND 5819 } 5820 } 5821 if (!packed_loaded || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5822 |4: 5823 if (!op2_loaded) { 5824 | // hval = Z_LVAL_P(dim); 5825 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5826 } 5827 if (packed_loaded) { 5828 | EXT_CALL _zend_hash_index_find, r0 5829 } else { 5830 | EXT_CALL zend_hash_index_find, r0 5831 } 5832 | test r0, r0 5833 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5834 | jz &exit_addr 5835 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5836 | jz ¬_found_exit_addr 5837 } else if (type == BP_VAR_IS && found_exit_addr) { 5838 | jz >7 // NOT_FOUND 5839 } else { 5840 | jz >2 // NOT_FOUND 5841 } 5842 } 5843 |.cold_code 5844 |2: 5845 switch (type) { 5846 case BP_VAR_R: 5847 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5848 | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); 5849 | // retval = &EG(uninitialized_zval); 5850 | UNDEFINED_OFFSET opline 5851 | jmp >9 5852 } 5853 break; 5854 case BP_VAR_IS: 5855 case BP_VAR_UNSET: 5856 if (!not_found_exit_addr && !found_exit_addr) { 5857 | // retval = &EG(uninitialized_zval); 5858 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 5859 | jmp >9 5860 } 5861 break; 5862 default: 5863 ZEND_UNREACHABLE(); 5864 } 5865 |.code 5866 break; 5867 case BP_VAR_RW: 5868 if (packed_loaded && !not_found_exit_addr) { 5869 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5870 } 5871 if (!packed_loaded || 5872 !not_found_exit_addr || 5873 (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { 5874 if (packed_loaded && not_found_exit_addr) { 5875 |.cold_code 5876 } 5877 |2: 5878 |4: 5879 if (!op2_loaded) { 5880 | // hval = Z_LVAL_P(dim); 5881 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5882 } 5883 if (packed_loaded) { 5884 | EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, r0 5885 } else { 5886 | EXT_CALL zend_jit_hash_index_lookup_rw, r0 5887 } 5888 | test r0, r0 5889 if (not_found_exit_addr) { 5890 if (packed_loaded) { 5891 | jnz >8 5892 | jmp ¬_found_exit_addr 5893 |.code 5894 } else { 5895 | jz ¬_found_exit_addr 5896 } 5897 } else { 5898 | jz >9 5899 } 5900 } 5901 break; 5902 case BP_VAR_W: 5903 if (packed_loaded) { 5904 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5905 } 5906 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) { 5907 |2: 5908 |4: 5909 if (!op2_loaded) { 5910 | // hval = Z_LVAL_P(dim); 5911 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5912 } 5913 | EXT_CALL zend_hash_index_lookup, r0 5914 } 5915 break; 5916 default: 5917 ZEND_UNREACHABLE(); 5918 } 5919 5920 if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { 5921 | jmp >8 5922 } 5923 } 5924 5925 if (op2_info & MAY_BE_STRING) { 5926 |3: 5927 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5928 | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) 5929 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3 5930 } 5931 | // offset_key = Z_STR_P(dim); 5932 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 5933 | // retval = zend_hash_find(ht, offset_key); 5934 switch (type) { 5935 case BP_JIT_IS: 5936 if (opline->op2_type != IS_CONST) { 5937 | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' 5938 | jle >1 5939 |.cold_code 5940 |1: 5941 | EXT_CALL zend_jit_symtable_find, r0 5942 | jmp >1 5943 |.code 5944 | EXT_CALL zend_hash_find, r0 5945 |1: 5946 } else { 5947 | EXT_CALL zend_hash_find_known_hash, r0 5948 } 5949 | test r0, r0 5950 if (not_found_exit_addr) { 5951 | jz ¬_found_exit_addr 5952 } else { 5953 | jz >9 // NOT_FOUND 5954 } 5955 break; 5956 case BP_VAR_R: 5957 case BP_VAR_IS: 5958 case BP_VAR_UNSET: 5959 if (opline->op2_type != IS_CONST) { 5960 | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' 5961 | jle >1 5962 |.cold_code 5963 |1: 5964 | EXT_CALL zend_jit_symtable_find, r0 5965 | jmp >1 5966 |.code 5967 | EXT_CALL zend_hash_find, r0 5968 |1: 5969 } else { 5970 | EXT_CALL zend_hash_find_known_hash, r0 5971 } 5972 | test r0, r0 5973 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5974 | jz &exit_addr 5975 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5976 | jz ¬_found_exit_addr 5977 } else if (type == BP_VAR_IS && found_exit_addr) { 5978 | jz >7 // NOT_FOUND 5979 } else { 5980 | jz >2 // NOT_FOUND 5981 |.cold_code 5982 |2: 5983 switch (type) { 5984 case BP_VAR_R: 5985 // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); 5986 | UNDEFINED_INDEX opline 5987 | jmp >9 5988 break; 5989 case BP_VAR_IS: 5990 case BP_VAR_UNSET: 5991 | // retval = &EG(uninitialized_zval); 5992 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 5993 | jmp >9 5994 break; 5995 default: 5996 ZEND_UNREACHABLE(); 5997 } 5998 |.code 5999 } 6000 break; 6001 case BP_VAR_RW: 6002 if (opline->op2_type != IS_CONST) { 6003 | EXT_CALL zend_jit_symtable_lookup_rw, r0 6004 } else { 6005 | EXT_CALL zend_jit_hash_lookup_rw, r0 6006 } 6007 | test r0, r0 6008 if (not_found_exit_addr) { 6009 | jz ¬_found_exit_addr 6010 } else { 6011 | jz >9 6012 } 6013 break; 6014 case BP_VAR_W: 6015 if (opline->op2_type != IS_CONST) { 6016 | EXT_CALL zend_jit_symtable_lookup_w, r0 6017 } else { 6018 | EXT_CALL zend_hash_lookup, r0 6019 } 6020 break; 6021 default: 6022 ZEND_UNREACHABLE(); 6023 } 6024 } 6025 6026 if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { 6027 |5: 6028 if (op1_info & MAY_BE_ARRAY_OF_REF) { 6029 | ZVAL_DEREF r0, MAY_BE_REF 6030 } 6031 | cmp byte [r0 + 8], IS_NULL 6032 if (not_found_exit_addr) { 6033 | jle ¬_found_exit_addr 6034 } else if (found_exit_addr) { 6035 | jg &found_exit_addr 6036 } else { 6037 | jle >9 // NOT FOUND 6038 } 6039 } 6040 6041 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 6042 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 6043 |.cold_code 6044 |3: 6045 } 6046 if (type != BP_VAR_RW) { 6047 | SET_EX_OPLINE opline, r0 6048 } 6049 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6050 switch (type) { 6051 case BP_VAR_R: 6052 |.if X64 6053 | LOAD_ZVAL_ADDR CARG3, res_addr 6054 |.else 6055 | sub r4, 12 6056 | PUSH_ZVAL_ADDR res_addr, r0 6057 |.endif 6058 | EXT_CALL zend_jit_fetch_dim_r_helper, r0 6059 |.if not(X64) 6060 | add r4, 12 6061 |.endif 6062 | jmp >9 6063 break; 6064 case BP_JIT_IS: 6065 | EXT_CALL zend_jit_fetch_dim_isset_helper, r0 6066 | test r0, r0 6067 if (not_found_exit_addr) { 6068 | je ¬_found_exit_addr 6069 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 6070 | jmp >8 6071 } 6072 } else if (found_exit_addr) { 6073 | jne &found_exit_addr 6074 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 6075 | jmp >9 6076 } 6077 } else { 6078 | jne >8 6079 | jmp >9 6080 } 6081 break; 6082 case BP_VAR_IS: 6083 case BP_VAR_UNSET: 6084 |.if X64 6085 | LOAD_ZVAL_ADDR CARG3, res_addr 6086 |.else 6087 | sub r4, 12 6088 | PUSH_ZVAL_ADDR res_addr, r0 6089 |.endif 6090 | EXT_CALL zend_jit_fetch_dim_is_helper, r0 6091 |.if not(X64) 6092 | add r4, 12 6093 |.endif 6094 | jmp >9 6095 break; 6096 case BP_VAR_RW: 6097 | EXT_CALL zend_jit_fetch_dim_rw_helper, r0 6098 | test r0, r0 6099 | jne >8 6100 | jmp >9 6101 break; 6102 case BP_VAR_W: 6103 | EXT_CALL zend_jit_fetch_dim_w_helper, r0 6104 | test r0, r0 6105 | jne >8 6106 | jmp >9 6107 break; 6108 default: 6109 ZEND_UNREACHABLE(); 6110 } 6111 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 6112 |.code 6113 } 6114 } 6115 6116 return 1; 6117} 6118 6119static int zend_jit_simple_assign(dasm_State **Dst, 6120 const zend_op *opline, 6121 zend_jit_addr var_addr, 6122 uint32_t var_info, 6123 uint32_t var_def_info, 6124 zend_uchar val_type, 6125 zend_jit_addr val_addr, 6126 uint32_t val_info, 6127 zend_jit_addr res_addr, 6128 int in_cold, 6129 int save_r1, 6130 bool check_exception) 6131/* Labels: 1,2,3 */ 6132{ 6133 zend_reg tmp_reg; 6134 6135 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_R0) { 6136 tmp_reg = ZREG_R0; 6137 } else { 6138 /* ASSIGN_DIM */ 6139 tmp_reg = ZREG_FCARG1; 6140 } 6141 6142 if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 6143 zval *zv = Z_ZV(val_addr); 6144 6145 if (!res_addr) { 6146 | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg 6147 } else { 6148 | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg 6149 } 6150 if (Z_REFCOUNTED_P(zv)) { 6151 if (!res_addr) { 6152 | ADDREF_CONST zv, Ra(tmp_reg) 6153 } else { 6154 | ADDREF_CONST_2 zv, Ra(tmp_reg) 6155 } 6156 } 6157 } else { 6158 if (val_info & MAY_BE_UNDEF) { 6159 if (in_cold) { 6160 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2 6161 } else { 6162 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 6163 |.cold_code 6164 |1: 6165 } 6166 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 6167 if (save_r1) { 6168 | mov aword T1, FCARG1a // save 6169 } 6170 | SET_ZVAL_TYPE_INFO var_addr, IS_NULL 6171 if (res_addr) { 6172 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 6173 } 6174 if (opline) { 6175 | SET_EX_OPLINE opline, Ra(tmp_reg) 6176 } 6177 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); 6178 | mov FCARG1d, Z_OFFSET(val_addr) 6179 | EXT_CALL zend_jit_undefined_op_helper, r0 6180 if (check_exception) { 6181 | test r0, r0 6182 | jz ->exception_handler_undef 6183 } 6184 if (save_r1) { 6185 | mov FCARG1a, aword T1 // restore 6186 } 6187 | jmp >3 6188 if (in_cold) { 6189 |2: 6190 } else { 6191 |.code 6192 } 6193 } 6194 if (val_info & MAY_BE_REF) { 6195 if (val_type == IS_CV) { 6196 ZEND_ASSERT(Z_REG(var_addr) != ZREG_R2); 6197 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_R2 || Z_OFFSET(val_addr) != 0) { 6198 | LOAD_ZVAL_ADDR r2, val_addr 6199 } 6200 | ZVAL_DEREF r2, val_info 6201 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); 6202 } else { 6203 zend_jit_addr ref_addr; 6204 zend_reg type_reg = tmp_reg; 6205 6206 if (in_cold) { 6207 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 6208 } else { 6209 | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1 6210 |.cold_code 6211 |1: 6212 } 6213 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 6214 | GET_ZVAL_PTR r2, val_addr 6215 | GC_DELREF r2 6216 | // ZVAL_COPY_VALUE(return_value, &ref->value); 6217 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8); 6218 if (!res_addr) { 6219 | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, type_reg, tmp_reg 6220 } else { 6221 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, type_reg, tmp_reg 6222 } 6223 | je >2 6224 if (tmp_reg == ZREG_R0) { 6225 | IF_NOT_REFCOUNTED ah, >3 6226 } else { 6227 | IF_NOT_FLAGS Rd(tmp_reg), (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), >3 6228 } 6229 | GET_ZVAL_PTR Ra(tmp_reg), var_addr 6230 6231 if (!res_addr) { 6232 | GC_ADDREF Ra(tmp_reg) 6233 } else { 6234 | add dword [Ra(tmp_reg)], 2 6235 } 6236 | jmp >3 6237 |2: 6238 if (res_addr) { 6239 if (tmp_reg == ZREG_R0) { 6240 | IF_NOT_REFCOUNTED ah, >2 6241 } else { 6242 | IF_NOT_FLAGS Rd(tmp_reg), (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), >2 6243 } 6244 | GET_ZVAL_PTR Ra(tmp_reg), var_addr 6245 | GC_ADDREF Ra(tmp_reg) 6246 |2: 6247 } 6248 if (save_r1) { 6249 | mov aword T1, FCARG1a // save 6250 } 6251 | EFREE_REFERENCE r2 6252 if (save_r1) { 6253 | mov FCARG1a, aword T1 // restore 6254 } 6255 | jmp >3 6256 if (in_cold) { 6257 |1: 6258 } else { 6259 |.code 6260 } 6261 } 6262 } 6263 6264 if (!res_addr) { 6265 | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, tmp_reg 6266 } else { 6267 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_R2, tmp_reg 6268 } 6269 6270 if (val_type == IS_CV) { 6271 if (!res_addr) { 6272 | TRY_ADDREF val_info, dh, Ra(tmp_reg) 6273 } else { 6274 | TRY_ADDREF_2 val_info, dh, Ra(tmp_reg) 6275 } 6276 } else { 6277 if (res_addr) { 6278 | TRY_ADDREF val_info, dh, Ra(tmp_reg) 6279 } 6280 } 6281 |3: 6282 } 6283 return 1; 6284} 6285 6286static int zend_jit_assign_to_typed_ref(dasm_State **Dst, 6287 const zend_op *opline, 6288 zend_uchar val_type, 6289 zend_jit_addr val_addr, 6290 zend_jit_addr res_addr, 6291 bool check_exception) 6292{ 6293 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 6294 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6295 | jnz >2 6296 |.cold_code 6297 |2: 6298 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 6299 | LOAD_ZVAL_ADDR FCARG2a, val_addr 6300 } 6301 if (opline) { 6302 | SET_EX_OPLINE opline, r0 6303 } 6304 if (val_type == IS_CONST) { 6305 | EXT_CALL zend_jit_assign_const_to_typed_ref, r0 6306 } else if (val_type == IS_TMP_VAR) { 6307 | EXT_CALL zend_jit_assign_tmp_to_typed_ref, r0 6308 } else if (val_type == IS_VAR) { 6309 | EXT_CALL zend_jit_assign_var_to_typed_ref, r0 6310 } else if (val_type == IS_CV) { 6311 | EXT_CALL zend_jit_assign_cv_to_typed_ref, r0 6312 } else { 6313 ZEND_UNREACHABLE(); 6314 } 6315 if (res_addr) { 6316 zend_jit_addr ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6317 6318 | ZVAL_COPY_VALUE res_addr, -1, ret_addr, -1, ZREG_R1, ZREG_R2 6319 | TRY_ADDREF -1, ch, r2 6320 } 6321 if (check_exception) { 6322 | // if (UNEXPECTED(EG(exception) != NULL)) { 6323 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 6324 | je >8 // END OF zend_jit_assign_to_variable() 6325 | jmp ->exception_handler 6326 } else { 6327 | jmp >8 6328 } 6329 |.code 6330 6331 return 1; 6332} 6333 6334static int zend_jit_assign_to_variable_call(dasm_State **Dst, 6335 const zend_op *opline, 6336 zend_jit_addr __var_use_addr, 6337 zend_jit_addr var_addr, 6338 uint32_t __var_info, 6339 uint32_t __var_def_info, 6340 zend_uchar val_type, 6341 zend_jit_addr val_addr, 6342 uint32_t val_info, 6343 zend_jit_addr __res_addr, 6344 bool __check_exception) 6345{ 6346 if (val_info & MAY_BE_UNDEF) { 6347 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 6348 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6349 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6350 6351 if (!exit_addr) { 6352 return 0; 6353 } 6354 6355 | IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr 6356 } else { 6357 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 6358 |.cold_code 6359 |1: 6360 ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP); 6361 if (Z_REG(var_addr) != ZREG_FP) { 6362 | mov aword T1, Ra(Z_REG(var_addr)) // save 6363 } 6364 | SET_EX_OPLINE opline, r0 6365 | mov FCARG1d, Z_OFFSET(val_addr) 6366 | EXT_CALL zend_jit_undefined_op_helper, r0 6367 if (Z_REG(var_addr) != ZREG_FP) { 6368 | mov Ra(Z_REG(var_addr)), aword T1 // restore 6369 } 6370 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 6371 | LOAD_ZVAL_ADDR FCARG1a, var_addr 6372 } 6373 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6374 | call ->assign_const 6375 | jmp >9 6376 |.code 6377 } 6378 } 6379 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 6380 | LOAD_ZVAL_ADDR FCARG1a, var_addr 6381 } 6382 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 6383 | LOAD_ZVAL_ADDR FCARG2a, val_addr 6384 } 6385 if (opline) { 6386 | SET_EX_OPLINE opline, r0 6387 } 6388 if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 6389 | call ->assign_tmp 6390 } else if (val_type == IS_CONST) { 6391 | call ->assign_const 6392 } else if (val_type == IS_TMP_VAR) { 6393 | call ->assign_tmp 6394 } else if (val_type == IS_VAR) { 6395 if (!(val_info & MAY_BE_REF)) { 6396 | call ->assign_tmp 6397 } else { 6398 | call ->assign_var 6399 } 6400 } else if (val_type == IS_CV) { 6401 if (!(val_info & MAY_BE_REF)) { 6402 | call ->assign_cv_noref 6403 } else { 6404 | call ->assign_cv 6405 } 6406 if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 6407 |9: 6408 } 6409 } else { 6410 ZEND_UNREACHABLE(); 6411 } 6412 6413 return 1; 6414} 6415 6416static int zend_jit_assign_to_variable(dasm_State **Dst, 6417 const zend_op *opline, 6418 zend_jit_addr var_use_addr, 6419 zend_jit_addr var_addr, 6420 uint32_t var_info, 6421 uint32_t var_def_info, 6422 zend_uchar val_type, 6423 zend_jit_addr val_addr, 6424 uint32_t val_info, 6425 zend_jit_addr res_addr, 6426 bool check_exception) 6427/* Labels: 1,2,3,4,5,8 */ 6428{ 6429 int done = 0; 6430 zend_reg ref_reg, tmp_reg; 6431 6432 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_R0) { 6433 ref_reg = ZREG_FCARG1; 6434 tmp_reg = ZREG_R0; 6435 } else { 6436 /* ASSIGN_DIM */ 6437 ref_reg = ZREG_R0; 6438 tmp_reg = ZREG_FCARG1; 6439 } 6440 6441 if (var_info & MAY_BE_REF) { 6442 if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { 6443 | LOAD_ZVAL_ADDR Ra(ref_reg), var_use_addr 6444 var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); 6445 } 6446 | // if (Z_ISREF_P(variable_ptr)) { 6447 | IF_NOT_Z_TYPE, Ra(ref_reg), IS_REFERENCE, >3 6448 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 6449 | GET_Z_PTR FCARG1a, Ra(ref_reg) 6450 if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) { 6451 return 0; 6452 } 6453 | lea Ra(ref_reg), [FCARG1a + offsetof(zend_reference, val)] 6454 |3: 6455 } 6456 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6457 if (RC_MAY_BE_1(var_info)) { 6458 int in_cold = 0; 6459 6460 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6461 | IF_ZVAL_REFCOUNTED var_use_addr, >1 6462 |.cold_code 6463 |1: 6464 in_cold = 1; 6465 } 6466 if (Z_REG(var_use_addr) == ZREG_FCARG1 || Z_REG(var_use_addr) == ZREG_R0) { 6467 bool keep_gc = 0; 6468 6469 | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr 6470 if (tmp_reg == ZREG_FCARG1) { 6471 if (Z_MODE(val_addr) == IS_REG) { 6472 keep_gc = 1; 6473 } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { 6474 keep_gc = 1; 6475 } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 6476 if (sizeof(void*) == 4) { 6477 keep_gc = 1; 6478 } else { 6479 zval *zv = Z_ZV(val_addr); 6480 6481 if (Z_TYPE_P(zv) == IS_DOUBLE) { 6482 if (Z_DVAL_P(zv) == 0 || IS_SIGNED_32BIT(zv)) { 6483 keep_gc = 1; 6484 } 6485 } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 6486 keep_gc = 1; 6487 } 6488 } 6489 } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { 6490 if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 6491 keep_gc = 1; 6492 } 6493 } 6494 } 6495 if (!keep_gc) { 6496 | mov aword T1, Ra(tmp_reg) // save 6497 } 6498 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0, 0)) { 6499 return 0; 6500 } 6501 if (!keep_gc) { 6502 | mov FCARG1a, aword T1 // restore 6503 } 6504 } else { 6505 | GET_ZVAL_PTR FCARG1a, var_use_addr 6506 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1, 0)) { 6507 return 0; 6508 } 6509 } 6510 | GC_DELREF FCARG1a 6511 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 6512 | jnz >4 6513 } else { 6514 | jnz >8 6515 } 6516 | ZVAL_DTOR_FUNC var_info, opline 6517 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { 6518 if (check_exception && !(val_info & MAY_BE_UNDEF)) { 6519 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 6520 | je >8 6521 | jmp ->exception_handler 6522 } else { 6523 | jmp >8 6524 } 6525 } 6526 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 6527 |4: 6528 | IF_GC_MAY_NOT_LEAK FCARG1a, >8 6529 | EXT_CALL gc_possible_root, r0 6530 if (in_cold) { 6531 | jmp >8 6532 } 6533 } 6534 if (check_exception && (val_info & MAY_BE_UNDEF)) { 6535 |8: 6536 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 6537 | je >8 6538 | jmp ->exception_handler 6539 } 6540 if (in_cold) { 6541 |.code 6542 } else { 6543 done = 1; 6544 } 6545 } else /* if (RC_MAY_BE_N(var_info)) */ { 6546 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6547 | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5 6548 } 6549 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { 6550 if (Z_REG(var_use_addr) != ZREG_FP) { 6551 | mov T1, Ra(Z_REG(var_use_addr)) // save 6552 } 6553 | GET_ZVAL_PTR FCARG1a, var_use_addr 6554 | GC_DELREF FCARG1a 6555 | IF_GC_MAY_NOT_LEAK FCARG1a, >5 6556 | EXT_CALL gc_possible_root, r0 6557 if (Z_REG(var_use_addr) != ZREG_FP) { 6558 | mov Ra(Z_REG(var_use_addr)), T1 // restore 6559 } 6560 } else { 6561 | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr 6562 | GC_DELREF Ra(tmp_reg) 6563 } 6564 |5: 6565 } 6566 } 6567 6568 if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0, check_exception)) { 6569 return 0; 6570 } 6571 6572 |8: 6573 6574 return 1; 6575} 6576 6577static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw) 6578{ 6579 zend_jit_addr op2_addr, op3_addr, res_addr; 6580 6581 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6582 op3_addr = OP1_DATA_ADDR(); 6583 if (opline->result_type == IS_UNUSED) { 6584 res_addr = 0; 6585 } else { 6586 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 6587 } 6588 6589 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { 6590 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6591 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6592 6593 if (!exit_addr) { 6594 return 0; 6595 } 6596 6597 | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr 6598 6599 val_info &= ~MAY_BE_UNDEF; 6600 } 6601 6602 if (op1_info & MAY_BE_REF) { 6603 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6604 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 6605 | GET_Z_PTR FCARG2a, FCARG1a 6606 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 6607 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 6608 | jmp >3 6609 |.cold_code 6610 |2: 6611 | SET_EX_OPLINE opline, r0 6612 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 6613 | test r0, r0 6614 | mov FCARG1a, r0 6615 | jne >1 6616 | jmp ->exception_handler_undef 6617 |.code 6618 |1: 6619 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6620 } 6621 6622 if (op1_info & MAY_BE_ARRAY) { 6623 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6624 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 6625 } 6626 |3: 6627 | SEPARATE_ARRAY op1_addr, op1_info, 1 6628 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6629 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6630 | CMP_ZVAL_TYPE op1_addr, IS_NULL 6631 | jg >7 6632 } 6633 | // ZVAL_ARR(container, zend_new_array(8)); 6634 if (Z_REG(op1_addr) != ZREG_FP) { 6635 | mov T1, Ra(Z_REG(op1_addr)) // save 6636 } 6637 | EXT_CALL _zend_new_array_0, r0 6638 if (Z_REG(op1_addr) != ZREG_FP) { 6639 | mov Ra(Z_REG(op1_addr)), T1 // restore 6640 } 6641 | SET_ZVAL_LVAL op1_addr, r0 6642 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6643 | mov FCARG1a, r0 6644 } 6645 6646 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6647 |6: 6648 if (opline->op2_type == IS_UNUSED) { 6649 uint32_t var_info = MAY_BE_NULL; 6650 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6651 6652 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6653 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6654 | EXT_CALL zend_hash_next_index_insert, r0 6655 | // if (UNEXPECTED(!var_ptr)) { 6656 | test r0, r0 6657 | jz >1 6658 |.cold_code 6659 |1: 6660 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6661 | CANNOT_ADD_ELEMENT opline 6662 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6663 | jmp >9 6664 |.code 6665 6666 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0, 0)) { 6667 return 0; 6668 } 6669 } else { 6670 uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6671 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6672 6673 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 6674 return 0; 6675 } 6676 6677 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6678 var_info |= MAY_BE_REF; 6679 } 6680 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6681 var_info |= MAY_BE_RC1; 6682 } 6683 6684 |8: 6685 | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); 6686 if (opline->op1_type == IS_VAR) { 6687 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6688 if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6689 return 0; 6690 } 6691 } else { 6692 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6693 return 0; 6694 } 6695 } 6696 } 6697 } 6698 6699 if (((op1_info & MAY_BE_ARRAY) && 6700 (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) || 6701 (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY)))) { 6702 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6703 |.cold_code 6704 |7: 6705 } 6706 6707 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) && 6708 (op1_info & MAY_BE_ARRAY)) { 6709 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6710 | CMP_ZVAL_TYPE op1_addr, IS_NULL 6711 | jg >2 6712 } 6713 | // ZVAL_ARR(container, zend_new_array(8)); 6714 if (Z_REG(op1_addr) != ZREG_FP) { 6715 | mov T1, Ra(Z_REG(op1_addr)) // save 6716 } 6717 | EXT_CALL _zend_new_array_0, r0 6718 if (Z_REG(op1_addr) != ZREG_FP) { 6719 | mov Ra(Z_REG(op1_addr)), T1 // restore 6720 } 6721 | SET_ZVAL_LVAL op1_addr, r0 6722 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6723 | mov FCARG1a, r0 6724 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); 6725 | jmp <6 6726 |2: 6727 } 6728 6729 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6730 | SET_EX_OPLINE opline, r0 6731 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 6732 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6733 } 6734 if (opline->op2_type == IS_UNUSED) { 6735 | xor FCARG2a, FCARG2a 6736 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6737 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6738 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 6739 } else { 6740 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6741 } 6742 |.if not(X64) 6743 | sub r4, 8 6744 |.endif 6745 if (opline->result_type == IS_UNUSED) { 6746 |.if X64 6747 | xor CARG4, CARG4 6748 |.else 6749 | push 0 6750 |.endif 6751 } else { 6752 |.if X64 6753 | LOAD_ZVAL_ADDR CARG4, res_addr 6754 |.else 6755 | PUSH_ZVAL_ADDR res_addr, r0 6756 |.endif 6757 } 6758 |.if X64 6759 | LOAD_ZVAL_ADDR CARG3, op3_addr 6760 |.else 6761 | PUSH_ZVAL_ADDR op3_addr, r0 6762 |.endif 6763 | EXT_CALL zend_jit_assign_dim_helper, r0 6764 |.if not(X64) 6765 | add r4, 8 6766 |.endif 6767 6768#ifdef ZEND_JIT_USE_RC_INFERENCE 6769 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { 6770 /* ASSIGN_DIM may increase refcount of the value */ 6771 val_info |= MAY_BE_RCN; 6772 } 6773#endif 6774 6775 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, NULL 6776 } 6777 6778 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6779 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6780 | jmp >9 // END 6781 } 6782 |.code 6783 } 6784 } 6785 6786#ifdef ZEND_JIT_USE_RC_INFERENCE 6787 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 6788 /* ASSIGN_DIM may increase refcount of the key */ 6789 op2_info |= MAY_BE_RCN; 6790 } 6791#endif 6792 6793 |9: 6794 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 6795 6796 if (may_throw) { 6797 zend_jit_check_exception(Dst); 6798 } 6799 6800 return 1; 6801} 6802 6803static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) 6804{ 6805 zend_jit_addr op2_addr, op3_addr, var_addr; 6806 const void *not_found_exit_addr = NULL; 6807 uint32_t var_info = MAY_BE_NULL; 6808 6809 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6810 6811 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6812 op3_addr = OP1_DATA_ADDR(); 6813 6814 | SET_EX_OPLINE opline, r0 6815 if (op1_info & MAY_BE_REF) { 6816 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6817 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 6818 | GET_Z_PTR FCARG2a, FCARG1a 6819 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 6820 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 6821 | jmp >3 6822 |.cold_code 6823 |2: 6824 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 6825 | test r0, r0 6826 | mov FCARG1a, r0 6827 | jne >1 6828 | jmp ->exception_handler_undef 6829 |.code 6830 |1: 6831 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 6832 } 6833 6834 if (op1_info & MAY_BE_ARRAY) { 6835 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6836 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 6837 } 6838 |3: 6839 | SEPARATE_ARRAY op1_addr, op1_info, 1 6840 } 6841 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 6842 if (op1_info & MAY_BE_ARRAY) { 6843 |.cold_code 6844 |7: 6845 } 6846 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6847 | CMP_ZVAL_TYPE op1_addr, IS_NULL 6848 | jg >7 6849 } 6850 if (Z_REG(op1_addr) != ZREG_FP) { 6851 | mov T1, Ra(Z_REG(op1_addr)) // save 6852 } 6853 if (op1_info & MAY_BE_UNDEF) { 6854 if (op1_info & MAY_BE_NULL) { 6855 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 6856 } 6857 | mov FCARG1a, opline->op1.var 6858 | EXT_CALL zend_jit_undefined_op_helper, r0 6859 |1: 6860 } 6861 | // ZVAL_ARR(container, zend_new_array(8)); 6862 | EXT_CALL _zend_new_array_0, r0 6863 if (Z_REG(op1_addr) != ZREG_FP) { 6864 | mov Ra(Z_REG(op1_addr)), T1 // restore 6865 } 6866 | SET_ZVAL_LVAL op1_addr, r0 6867 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6868 | mov FCARG1a, r0 6869 if (op1_info & MAY_BE_ARRAY) { 6870 | jmp >1 6871 |.code 6872 |1: 6873 } 6874 } 6875 6876 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6877 uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); 6878 6879 |6: 6880 if (opline->op2_type == IS_UNUSED) { 6881 var_info = MAY_BE_NULL; 6882 6883 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6884 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6885 | EXT_CALL zend_hash_next_index_insert, r0 6886 | // if (UNEXPECTED(!var_ptr)) { 6887 | test r0, r0 6888 | jz >1 6889 |.cold_code 6890 |1: 6891 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6892 | CANNOT_ADD_ELEMENT opline 6893 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6894 | jmp >9 6895 |.code 6896 } else { 6897 var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6898 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6899 var_info |= MAY_BE_REF; 6900 } 6901 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6902 var_info |= MAY_BE_RC1; 6903 } 6904 6905 if (dim_type != IS_UNKNOWN 6906 && dim_type != IS_UNDEF 6907 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY 6908 && (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) 6909 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 6910 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 6911 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6912 if (!not_found_exit_addr) { 6913 return 0; 6914 } 6915 } 6916 6917 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, not_found_exit_addr, NULL)) { 6918 return 0; 6919 } 6920 6921 |8: 6922 if (not_found_exit_addr && dim_type != IS_REFERENCE) { 6923 | IF_NOT_Z_TYPE, r0, dim_type, ¬_found_exit_addr 6924 var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 6925 } 6926 if (var_info & MAY_BE_REF) { 6927 binary_op_type binary_op = get_binary_op(opline->extended_value); 6928 | IF_NOT_Z_TYPE, r0, IS_REFERENCE, >1 6929 | GET_Z_PTR FCARG1a, r0 6930 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6931 | jnz >2 6932 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] 6933 |.cold_code 6934 |2: 6935 | LOAD_ZVAL_ADDR FCARG2a, op3_addr 6936 |.if X64 6937 | LOAD_ADDR CARG3, binary_op 6938 |.else 6939 | sub r4, 12 6940 | PUSH_ADDR binary_op, r0 6941 |.endif 6942 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 6943 && (op1_data_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6944 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 6945 } else { 6946 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 6947 } 6948 |.if not(X64) 6949 | add r4, 12 6950 |.endif 6951 | jmp >9 6952 |.code 6953 |1: 6954 } 6955 } 6956 6957 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6958 switch (opline->extended_value) { 6959 case ZEND_ADD: 6960 case ZEND_SUB: 6961 case ZEND_MUL: 6962 case ZEND_DIV: 6963 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, 6964 1 /* may overflow */, may_throw)) { 6965 return 0; 6966 } 6967 break; 6968 case ZEND_BW_OR: 6969 case ZEND_BW_AND: 6970 case ZEND_BW_XOR: 6971 case ZEND_SL: 6972 case ZEND_SR: 6973 case ZEND_MOD: 6974 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6975 IS_CV, opline->op1, var_addr, var_info, NULL, 6976 (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 6977 op1_data_range, 6978 0, var_addr, var_def_info, var_info, may_throw)) { 6979 return 0; 6980 } 6981 break; 6982 case ZEND_CONCAT: 6983 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, 6984 may_throw)) { 6985 return 0; 6986 } 6987 break; 6988 default: 6989 ZEND_UNREACHABLE(); 6990 } 6991 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 6992 } 6993 6994 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 6995 binary_op_type binary_op; 6996 6997 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 6998 |.cold_code 6999 |7: 7000 } 7001 7002 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 7003 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 7004 } 7005 if (opline->op2_type == IS_UNUSED) { 7006 | xor FCARG2a, FCARG2a 7007 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 7008 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 7009 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 7010 } else { 7011 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 7012 } 7013 binary_op = get_binary_op(opline->extended_value); 7014 |.if X64 7015 | LOAD_ZVAL_ADDR CARG3, op3_addr 7016 | LOAD_ADDR CARG4, binary_op 7017 |.else 7018 | sub r4, 8 7019 | PUSH_ADDR binary_op, r0 7020 | PUSH_ZVAL_ADDR op3_addr, r0 7021 |.endif 7022 | EXT_CALL zend_jit_assign_dim_op_helper, r0 7023 |.if not(X64) 7024 | add r4, 8 7025 |.endif 7026 7027 |9: 7028 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, NULL 7029 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL 7030 if (may_throw) { 7031 zend_jit_check_exception(Dst); 7032 } 7033 7034 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 7035 | jmp >9 // END 7036 |.code 7037 |9: 7038 } 7039 } else if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) 7040 && (!not_found_exit_addr || (var_info & MAY_BE_REF))) { 7041 |.cold_code 7042 |9: 7043 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, opline 7044 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 7045 if (may_throw) { 7046 zend_jit_check_exception(Dst); 7047 } 7048 | jmp >9 7049 |.code 7050 |9: 7051 } 7052 7053 return 1; 7054} 7055 7056static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) 7057{ 7058 zend_jit_addr op1_addr, op2_addr; 7059 7060 ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); 7061 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 7062 7063 op1_addr = OP1_ADDR(); 7064 op2_addr = OP2_ADDR(); 7065 7066 if (op1_info & MAY_BE_REF) { 7067 binary_op_type binary_op = get_binary_op(opline->extended_value); 7068 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 7069 | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >1 7070 | GET_Z_PTR FCARG1a, FCARG1a 7071 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 7072 | jnz >2 7073 | add FCARG1a, offsetof(zend_reference, val) 7074 |.cold_code 7075 |2: 7076 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 7077 |.if X64 7078 | LOAD_ADDR CARG3, binary_op 7079 |.else 7080 | sub r4, 12 7081 | PUSH_ADDR binary_op, r0 7082 |.endif 7083 | SET_EX_OPLINE opline, r0 7084 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 7085 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 7086 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 7087 } else { 7088 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 7089 } 7090 |.if not(X64) 7091 | add r4, 12 7092 |.endif 7093 zend_jit_check_exception(Dst); 7094 | jmp >9 7095 |.code 7096 |1: 7097 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 7098 } 7099 7100 int result; 7101 switch (opline->extended_value) { 7102 case ZEND_ADD: 7103 case ZEND_SUB: 7104 case ZEND_MUL: 7105 case ZEND_DIV: 7106 result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); 7107 break; 7108 case ZEND_BW_OR: 7109 case ZEND_BW_AND: 7110 case ZEND_BW_XOR: 7111 case ZEND_SL: 7112 case ZEND_SR: 7113 case ZEND_MOD: 7114 result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, 7115 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 7116 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 7117 opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); 7118 break; 7119 case ZEND_CONCAT: 7120 result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); 7121 break; 7122 default: 7123 ZEND_UNREACHABLE(); 7124 } 7125 |9: 7126 return result; 7127} 7128 7129static int zend_jit_cmp_long_long(dasm_State **Dst, 7130 const zend_op *opline, 7131 zend_ssa_range *op1_range, 7132 zend_jit_addr op1_addr, 7133 zend_ssa_range *op2_range, 7134 zend_jit_addr op2_addr, 7135 zend_jit_addr res_addr, 7136 zend_uchar smart_branch_opcode, 7137 uint32_t target_label, 7138 uint32_t target_label2, 7139 const void *exit_addr, 7140 bool skip_comparison) 7141{ 7142 bool swap = 0; 7143 bool result; 7144 7145 if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { 7146 if (!smart_branch_opcode || 7147 smart_branch_opcode == ZEND_JMPZ_EX || 7148 smart_branch_opcode == ZEND_JMPNZ_EX) { 7149 | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE) 7150 } 7151 if (smart_branch_opcode && !exit_addr) { 7152 if (smart_branch_opcode == ZEND_JMPZ || 7153 smart_branch_opcode == ZEND_JMPZ_EX) { 7154 if (!result) { 7155 | jmp => target_label 7156 } 7157 } else if (smart_branch_opcode == ZEND_JMPNZ || 7158 smart_branch_opcode == ZEND_JMPNZ_EX) { 7159 if (result) { 7160 | jmp => target_label 7161 } 7162 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7163 if (!result) { 7164 | jmp => target_label 7165 } else { 7166 | jmp => target_label2 7167 } 7168 } else { 7169 ZEND_UNREACHABLE(); 7170 } 7171 } 7172 return 1; 7173 } 7174 7175 if (skip_comparison) { 7176 if (Z_MODE(op1_addr) != IS_REG && 7177 (Z_MODE(op2_addr) == IS_REG || 7178 (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { 7179 swap = 1; 7180 } 7181 } else if (Z_MODE(op1_addr) == IS_REG) { 7182 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 7183 | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) 7184 } else { 7185 | LONG_OP cmp, Z_REG(op1_addr), op2_addr, r0 7186 } 7187 } else if (Z_MODE(op2_addr) == IS_REG) { 7188 if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { 7189 | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) 7190 } else { 7191 | LONG_OP cmp, Z_REG(op2_addr), op1_addr, r0 7192 } 7193 swap = 1; 7194 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 7195 | LONG_OP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) 7196 swap = 1; 7197 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { 7198 | LONG_OP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)) 7199 } else { 7200 | GET_ZVAL_LVAL ZREG_R0, op1_addr 7201 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 7202 | test r0, r0 7203 } else { 7204 | LONG_OP cmp, ZREG_R0, op2_addr, r0 7205 } 7206 } 7207 7208 if (smart_branch_opcode) { 7209 if (smart_branch_opcode == ZEND_JMPZ_EX || 7210 smart_branch_opcode == ZEND_JMPNZ_EX) { 7211 7212 switch (opline->opcode) { 7213 case ZEND_IS_EQUAL: 7214 case ZEND_IS_IDENTICAL: 7215 case ZEND_CASE: 7216 case ZEND_CASE_STRICT: 7217 | sete al 7218 break; 7219 case ZEND_IS_NOT_EQUAL: 7220 case ZEND_IS_NOT_IDENTICAL: 7221 | setne al 7222 break; 7223 case ZEND_IS_SMALLER: 7224 if (swap) { 7225 | setg al 7226 } else { 7227 | setl al 7228 } 7229 break; 7230 case ZEND_IS_SMALLER_OR_EQUAL: 7231 if (swap) { 7232 | setge al 7233 } else { 7234 | setle al 7235 } 7236 break; 7237 default: 7238 ZEND_UNREACHABLE(); 7239 } 7240 | movzx eax, al 7241 | lea eax, [eax + 2] 7242 | SET_ZVAL_TYPE_INFO res_addr, eax 7243 } 7244 if (smart_branch_opcode == ZEND_JMPZ || 7245 smart_branch_opcode == ZEND_JMPZ_EX) { 7246 switch (opline->opcode) { 7247 case ZEND_IS_EQUAL: 7248 case ZEND_IS_IDENTICAL: 7249 case ZEND_CASE: 7250 case ZEND_CASE_STRICT: 7251 if (exit_addr) { 7252 | jne &exit_addr 7253 } else { 7254 | jne => target_label 7255 } 7256 break; 7257 case ZEND_IS_NOT_EQUAL: 7258 if (exit_addr) { 7259 | je &exit_addr 7260 } else { 7261 | je => target_label 7262 } 7263 break; 7264 case ZEND_IS_NOT_IDENTICAL: 7265 if (exit_addr) { 7266 | jne &exit_addr 7267 } else { 7268 | je => target_label 7269 } 7270 break; 7271 case ZEND_IS_SMALLER: 7272 if (swap) { 7273 if (exit_addr) { 7274 | jle &exit_addr 7275 } else { 7276 | jle => target_label 7277 } 7278 } else { 7279 if (exit_addr) { 7280 | jge &exit_addr 7281 } else { 7282 | jge => target_label 7283 } 7284 } 7285 break; 7286 case ZEND_IS_SMALLER_OR_EQUAL: 7287 if (swap) { 7288 if (exit_addr) { 7289 | jl &exit_addr 7290 } else { 7291 | jl => target_label 7292 } 7293 } else { 7294 if (exit_addr) { 7295 | jg &exit_addr 7296 } else { 7297 | jg => target_label 7298 } 7299 } 7300 break; 7301 default: 7302 ZEND_UNREACHABLE(); 7303 } 7304 } else if (smart_branch_opcode == ZEND_JMPNZ || 7305 smart_branch_opcode == ZEND_JMPNZ_EX) { 7306 switch (opline->opcode) { 7307 case ZEND_IS_EQUAL: 7308 case ZEND_IS_IDENTICAL: 7309 case ZEND_CASE: 7310 case ZEND_CASE_STRICT: 7311 if (exit_addr) { 7312 | je &exit_addr 7313 } else { 7314 | je => target_label 7315 } 7316 break; 7317 case ZEND_IS_NOT_EQUAL: 7318 if (exit_addr) { 7319 | jne &exit_addr 7320 } else { 7321 | jne => target_label 7322 } 7323 break; 7324 case ZEND_IS_NOT_IDENTICAL: 7325 if (exit_addr) { 7326 | je &exit_addr 7327 } else { 7328 | jne => target_label 7329 } 7330 break; 7331 case ZEND_IS_SMALLER: 7332 if (swap) { 7333 if (exit_addr) { 7334 | jg &exit_addr 7335 } else { 7336 | jg => target_label 7337 } 7338 } else { 7339 if (exit_addr) { 7340 | jl &exit_addr 7341 } else { 7342 | jl => target_label 7343 } 7344 } 7345 break; 7346 case ZEND_IS_SMALLER_OR_EQUAL: 7347 if (swap) { 7348 if (exit_addr) { 7349 | jge &exit_addr 7350 } else { 7351 | jge => target_label 7352 } 7353 } else { 7354 if (exit_addr) { 7355 | jle &exit_addr 7356 } else { 7357 | jle => target_label 7358 } 7359 } 7360 break; 7361 default: 7362 ZEND_UNREACHABLE(); 7363 } 7364 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7365 switch (opline->opcode) { 7366 case ZEND_IS_EQUAL: 7367 case ZEND_IS_IDENTICAL: 7368 case ZEND_CASE: 7369 case ZEND_CASE_STRICT: 7370 | jne => target_label 7371 break; 7372 case ZEND_IS_NOT_EQUAL: 7373 case ZEND_IS_NOT_IDENTICAL: 7374 | je => target_label 7375 break; 7376 case ZEND_IS_SMALLER: 7377 if (swap) { 7378 | jle => target_label 7379 } else { 7380 | jge => target_label 7381 } 7382 break; 7383 case ZEND_IS_SMALLER_OR_EQUAL: 7384 if (swap) { 7385 | jl => target_label 7386 } else { 7387 | jg => target_label 7388 } 7389 break; 7390 default: 7391 ZEND_UNREACHABLE(); 7392 } 7393 | jmp => target_label2 7394 } else { 7395 ZEND_UNREACHABLE(); 7396 } 7397 } else { 7398 switch (opline->opcode) { 7399 case ZEND_IS_EQUAL: 7400 case ZEND_IS_IDENTICAL: 7401 case ZEND_CASE: 7402 case ZEND_CASE_STRICT: 7403 | sete al 7404 break; 7405 case ZEND_IS_NOT_EQUAL: 7406 case ZEND_IS_NOT_IDENTICAL: 7407 | setne al 7408 break; 7409 case ZEND_IS_SMALLER: 7410 if (swap) { 7411 | setg al 7412 } else { 7413 | setl al 7414 } 7415 break; 7416 case ZEND_IS_SMALLER_OR_EQUAL: 7417 if (swap) { 7418 | setge al 7419 } else { 7420 | setle al 7421 } 7422 break; 7423 default: 7424 ZEND_UNREACHABLE(); 7425 } 7426 | movzx eax, al 7427 | add eax, 2 7428 | SET_ZVAL_TYPE_INFO res_addr, eax 7429 } 7430 7431 return 1; 7432} 7433 7434static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7435{ 7436 if (smart_branch_opcode) { 7437 if (smart_branch_opcode == ZEND_JMPZ) { 7438 switch (opline->opcode) { 7439 case ZEND_IS_EQUAL: 7440 case ZEND_IS_IDENTICAL: 7441 case ZEND_CASE: 7442 case ZEND_CASE_STRICT: 7443 if (exit_addr) { 7444 | jne &exit_addr 7445 | jp &exit_addr 7446 } else { 7447 | jne => target_label 7448 | jp => target_label 7449 } 7450 break; 7451 case ZEND_IS_NOT_EQUAL: 7452 | jp >1 7453 if (exit_addr) { 7454 | je &exit_addr 7455 } else { 7456 | je => target_label 7457 } 7458 |1: 7459 break; 7460 case ZEND_IS_NOT_IDENTICAL: 7461 if (exit_addr) { 7462 | jne &exit_addr 7463 | jp &exit_addr 7464 } else { 7465 | jp >1 7466 | je => target_label 7467 |1: 7468 } 7469 break; 7470 case ZEND_IS_SMALLER: 7471 if (swap) { 7472 if (exit_addr) { 7473 | jbe &exit_addr 7474 } else { 7475 | jbe => target_label 7476 } 7477 } else { 7478 if (exit_addr) { 7479 | jae &exit_addr 7480 | jp &exit_addr 7481 } else { 7482 | jae => target_label 7483 | jp => target_label 7484 } 7485 } 7486 break; 7487 case ZEND_IS_SMALLER_OR_EQUAL: 7488 if (swap) { 7489 if (exit_addr) { 7490 | jb &exit_addr 7491 } else { 7492 | jb => target_label 7493 } 7494 } else { 7495 if (exit_addr) { 7496 | ja &exit_addr 7497 | jp &exit_addr 7498 } else { 7499 | ja => target_label 7500 | jp => target_label 7501 } 7502 } 7503 break; 7504 default: 7505 ZEND_UNREACHABLE(); 7506 } 7507 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7508 switch (opline->opcode) { 7509 case ZEND_IS_EQUAL: 7510 case ZEND_IS_IDENTICAL: 7511 case ZEND_CASE: 7512 case ZEND_CASE_STRICT: 7513 | jp >1 7514 if (exit_addr) { 7515 | je &exit_addr 7516 } else { 7517 | je => target_label 7518 } 7519 |1: 7520 break; 7521 case ZEND_IS_NOT_EQUAL: 7522 if (exit_addr) { 7523 | jne &exit_addr 7524 | jp &exit_addr 7525 } else { 7526 | jne => target_label 7527 | jp => target_label 7528 } 7529 break; 7530 case ZEND_IS_NOT_IDENTICAL: 7531 if (exit_addr) { 7532 | jp >1 7533 | je &exit_addr 7534 |1: 7535 } else { 7536 | jne => target_label 7537 | jp => target_label 7538 } 7539 break; 7540 case ZEND_IS_SMALLER: 7541 if (swap) { 7542 if (exit_addr) { 7543 | ja &exit_addr 7544 } else { 7545 | ja => target_label 7546 } 7547 } else { 7548 | jp >1 7549 if (exit_addr) { 7550 | jb &exit_addr 7551 } else { 7552 | jb => target_label 7553 } 7554 |1: 7555 } 7556 break; 7557 case ZEND_IS_SMALLER_OR_EQUAL: 7558 if (swap) { 7559 if (exit_addr) { 7560 | jae &exit_addr 7561 } else { 7562 | jae => target_label 7563 } 7564 } else { 7565 | jp >1 7566 if (exit_addr) { 7567 | jbe &exit_addr 7568 } else { 7569 | jbe => target_label 7570 } 7571 |1: 7572 } 7573 break; 7574 default: 7575 ZEND_UNREACHABLE(); 7576 } 7577 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7578 switch (opline->opcode) { 7579 case ZEND_IS_EQUAL: 7580 case ZEND_IS_IDENTICAL: 7581 case ZEND_CASE: 7582 case ZEND_CASE_STRICT: 7583 | jne => target_label 7584 | jp => target_label 7585 break; 7586 case ZEND_IS_NOT_EQUAL: 7587 case ZEND_IS_NOT_IDENTICAL: 7588 | jp => target_label2 7589 | je => target_label 7590 break; 7591 case ZEND_IS_SMALLER: 7592 if (swap) { 7593 | jbe => target_label 7594 } else { 7595 | jae => target_label 7596 | jp => target_label 7597 } 7598 break; 7599 case ZEND_IS_SMALLER_OR_EQUAL: 7600 if (swap) { 7601 | jb => target_label 7602 } else { 7603 | ja => target_label 7604 | jp => target_label 7605 } 7606 break; 7607 default: 7608 ZEND_UNREACHABLE(); 7609 } 7610 | jmp => target_label2 7611 } else if (smart_branch_opcode == ZEND_JMPZ_EX) { 7612 switch (opline->opcode) { 7613 case ZEND_IS_EQUAL: 7614 case ZEND_IS_IDENTICAL: 7615 case ZEND_CASE: 7616 case ZEND_CASE_STRICT: 7617 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7618 | jne => target_label 7619 | jp => target_label 7620 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7621 break; 7622 case ZEND_IS_NOT_EQUAL: 7623 case ZEND_IS_NOT_IDENTICAL: 7624 | jp >1 7625 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7626 | je => target_label 7627 |1: 7628 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7629 break; 7630 case ZEND_IS_SMALLER: 7631 if (swap) { 7632 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7633 | jbe => target_label 7634 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7635 } else { 7636 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7637 | jae => target_label 7638 | jp => target_label 7639 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7640 } 7641 break; 7642 case ZEND_IS_SMALLER_OR_EQUAL: 7643 if (swap) { 7644 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7645 | jb => target_label 7646 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7647 } else { 7648 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7649 | ja => target_label 7650 | jp => target_label 7651 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7652 } 7653 break; 7654 default: 7655 ZEND_UNREACHABLE(); 7656 } 7657 } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { 7658 switch (opline->opcode) { 7659 case ZEND_IS_EQUAL: 7660 case ZEND_IS_IDENTICAL: 7661 case ZEND_CASE: 7662 case ZEND_CASE_STRICT: 7663 | jp >1 7664 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7665 | je => target_label 7666 |1: 7667 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7668 break; 7669 case ZEND_IS_NOT_EQUAL: 7670 case ZEND_IS_NOT_IDENTICAL: 7671 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7672 | jne => target_label 7673 | jp => target_label 7674 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7675 break; 7676 case ZEND_IS_SMALLER: 7677 if (swap) { 7678 | seta al 7679 | movzx eax, al 7680 | lea eax, [eax + 2] 7681 | SET_ZVAL_TYPE_INFO res_addr, eax 7682 | ja => target_label 7683 } else { 7684 | jp >1 7685 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7686 | jb => target_label 7687 |1: 7688 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7689 } 7690 break; 7691 case ZEND_IS_SMALLER_OR_EQUAL: 7692 if (swap) { 7693 | setae al 7694 | movzx eax, al 7695 | lea eax, [eax + 2] 7696 | SET_ZVAL_TYPE_INFO res_addr, eax 7697 | jae => target_label 7698 } else { 7699 | jp >1 7700 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7701 | jbe => target_label 7702 |1: 7703 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7704 } 7705 break; 7706 default: 7707 ZEND_UNREACHABLE(); 7708 } 7709 } else { 7710 ZEND_UNREACHABLE(); 7711 } 7712 } else { 7713 switch (opline->opcode) { 7714 case ZEND_IS_EQUAL: 7715 case ZEND_IS_IDENTICAL: 7716 case ZEND_CASE: 7717 case ZEND_CASE_STRICT: 7718 | jp >1 7719 | mov eax, IS_TRUE 7720 | je >2 7721 |1: 7722 | mov eax, IS_FALSE 7723 |2: 7724 break; 7725 case ZEND_IS_NOT_EQUAL: 7726 case ZEND_IS_NOT_IDENTICAL: 7727 | jp >1 7728 | mov eax, IS_FALSE 7729 | je >2 7730 |1: 7731 | mov eax, IS_TRUE 7732 |2: 7733 break; 7734 case ZEND_IS_SMALLER: 7735 if (swap) { 7736 | seta al 7737 | movzx eax, al 7738 | add eax, 2 7739 } else { 7740 | jp >1 7741 | mov eax, IS_TRUE 7742 | jb >2 7743 |1: 7744 | mov eax, IS_FALSE 7745 |2: 7746 } 7747 break; 7748 case ZEND_IS_SMALLER_OR_EQUAL: 7749 if (swap) { 7750 | setae al 7751 | movzx eax, al 7752 | add eax, 2 7753 } else { 7754 | jp >1 7755 | mov eax, IS_TRUE 7756 | jbe >2 7757 |1: 7758 | mov eax, IS_FALSE 7759 |2: 7760 } 7761 break; 7762 default: 7763 ZEND_UNREACHABLE(); 7764 } 7765 | SET_ZVAL_TYPE_INFO res_addr, eax 7766 } 7767 7768 return 1; 7769} 7770 7771static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7772{ 7773 zend_reg tmp_reg = ZREG_XMM0; 7774 7775 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_R0 7776 | DOUBLE_CMP tmp_reg, op2_addr 7777 7778 return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); 7779} 7780 7781static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7782{ 7783 zend_reg tmp_reg = ZREG_XMM0; 7784 7785 | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_R0 7786 | DOUBLE_CMP tmp_reg, op1_addr 7787 7788 return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); 7789} 7790 7791static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7792{ 7793 bool swap = 0; 7794 7795 if (Z_MODE(op1_addr) == IS_REG) { 7796 | DOUBLE_CMP Z_REG(op1_addr), op2_addr 7797 } else if (Z_MODE(op2_addr) == IS_REG) { 7798 | DOUBLE_CMP Z_REG(op2_addr), op1_addr 7799 swap = 1; 7800 } else { 7801 zend_reg tmp_reg = ZREG_XMM0; 7802 7803 | DOUBLE_GET_ZVAL_DVAL tmp_reg, op1_addr 7804 | DOUBLE_CMP tmp_reg, op2_addr 7805 } 7806 7807 return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); 7808} 7809 7810static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7811{ 7812 | test, eax, eax 7813 if (smart_branch_opcode) { 7814 if (smart_branch_opcode == ZEND_JMPZ_EX || 7815 smart_branch_opcode == ZEND_JMPNZ_EX) { 7816 switch (opline->opcode) { 7817 case ZEND_IS_EQUAL: 7818 case ZEND_CASE: 7819 | sete al 7820 break; 7821 case ZEND_IS_NOT_EQUAL: 7822 | setne al 7823 break; 7824 case ZEND_IS_SMALLER: 7825 | setl al 7826 break; 7827 case ZEND_IS_SMALLER_OR_EQUAL: 7828 | setle al 7829 break; 7830 default: 7831 ZEND_UNREACHABLE(); 7832 } 7833 | movzx eax, al 7834 | lea eax, [eax + 2] 7835 | SET_ZVAL_TYPE_INFO res_addr, eax 7836 } 7837 if (smart_branch_opcode == ZEND_JMPZ || 7838 smart_branch_opcode == ZEND_JMPZ_EX) { 7839 switch (opline->opcode) { 7840 case ZEND_IS_EQUAL: 7841 case ZEND_CASE: 7842 if (exit_addr) { 7843 | jne &exit_addr 7844 } else { 7845 | jne => target_label 7846 } 7847 break; 7848 case ZEND_IS_NOT_EQUAL: 7849 if (exit_addr) { 7850 | je &exit_addr 7851 } else { 7852 | je => target_label 7853 } 7854 break; 7855 case ZEND_IS_SMALLER: 7856 if (exit_addr) { 7857 | jge &exit_addr 7858 } else { 7859 | jge => target_label 7860 } 7861 break; 7862 case ZEND_IS_SMALLER_OR_EQUAL: 7863 if (exit_addr) { 7864 | jg &exit_addr 7865 } else { 7866 | jg => target_label 7867 } 7868 break; 7869 default: 7870 ZEND_UNREACHABLE(); 7871 } 7872 } else if (smart_branch_opcode == ZEND_JMPNZ || 7873 smart_branch_opcode == ZEND_JMPNZ_EX) { 7874 switch (opline->opcode) { 7875 case ZEND_IS_EQUAL: 7876 case ZEND_CASE: 7877 if (exit_addr) { 7878 | je &exit_addr 7879 } else { 7880 | je => target_label 7881 } 7882 break; 7883 case ZEND_IS_NOT_EQUAL: 7884 if (exit_addr) { 7885 | jne &exit_addr 7886 } else { 7887 | jne => target_label 7888 } 7889 break; 7890 case ZEND_IS_SMALLER: 7891 if (exit_addr) { 7892 | jl &exit_addr 7893 } else { 7894 | jl => target_label 7895 } 7896 break; 7897 case ZEND_IS_SMALLER_OR_EQUAL: 7898 if (exit_addr) { 7899 | jle &exit_addr 7900 } else { 7901 | jle => target_label 7902 } 7903 break; 7904 default: 7905 ZEND_UNREACHABLE(); 7906 } 7907 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7908 switch (opline->opcode) { 7909 case ZEND_IS_EQUAL: 7910 case ZEND_CASE: 7911 | jne => target_label 7912 break; 7913 case ZEND_IS_NOT_EQUAL: 7914 | je => target_label 7915 break; 7916 case ZEND_IS_SMALLER: 7917 | jge => target_label 7918 break; 7919 case ZEND_IS_SMALLER_OR_EQUAL: 7920 | jg => target_label 7921 break; 7922 default: 7923 ZEND_UNREACHABLE(); 7924 } 7925 | jmp => target_label2 7926 } else { 7927 ZEND_UNREACHABLE(); 7928 } 7929 } else { 7930 switch (opline->opcode) { 7931 case ZEND_IS_EQUAL: 7932 case ZEND_CASE: 7933 | sete al 7934 break; 7935 case ZEND_IS_NOT_EQUAL: 7936 | setne al 7937 break; 7938 case ZEND_IS_SMALLER: 7939 | setl al 7940 break; 7941 case ZEND_IS_SMALLER_OR_EQUAL: 7942 | setle al 7943 break; 7944 default: 7945 ZEND_UNREACHABLE(); 7946 } 7947 | movzx eax, al 7948 | add eax, 2 7949 | SET_ZVAL_TYPE_INFO res_addr, eax 7950 } 7951 7952 return 1; 7953} 7954 7955static int zend_jit_cmp(dasm_State **Dst, 7956 const zend_op *opline, 7957 uint32_t op1_info, 7958 zend_ssa_range *op1_range, 7959 zend_jit_addr op1_addr, 7960 uint32_t op2_info, 7961 zend_ssa_range *op2_range, 7962 zend_jit_addr op2_addr, 7963 zend_jit_addr res_addr, 7964 int may_throw, 7965 zend_uchar smart_branch_opcode, 7966 uint32_t target_label, 7967 uint32_t target_label2, 7968 const void *exit_addr, 7969 bool skip_comparison) 7970{ 7971 bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); 7972 bool has_slow; 7973 7974 has_slow = 7975 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7976 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7977 ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7978 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); 7979 7980 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 7981 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 7982 if (op1_info & MAY_BE_DOUBLE) { 7983 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4 7984 } else { 7985 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 7986 } 7987 } 7988 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 7989 if (op2_info & MAY_BE_DOUBLE) { 7990 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 7991 |.cold_code 7992 |3: 7993 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7994 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7995 } 7996 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7997 return 0; 7998 } 7999 | jmp >6 8000 |.code 8001 } else { 8002 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 8003 } 8004 } 8005 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 8006 return 0; 8007 } 8008 if (op1_info & MAY_BE_DOUBLE) { 8009 |.cold_code 8010 |4: 8011 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 8012 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 8013 } 8014 if (op2_info & MAY_BE_DOUBLE) { 8015 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 8016 if (!same_ops) { 8017 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5 8018 } else { 8019 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 8020 } 8021 } 8022 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8023 return 0; 8024 } 8025 | jmp >6 8026 } 8027 if (!same_ops) { 8028 |5: 8029 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 8030 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 8031 } 8032 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8033 return 0; 8034 } 8035 | jmp >6 8036 } 8037 |.code 8038 } 8039 } else if ((op1_info & MAY_BE_DOUBLE) && 8040 !(op1_info & MAY_BE_LONG) && 8041 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 8042 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 8043 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 8044 } 8045 if (op2_info & MAY_BE_DOUBLE) { 8046 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 8047 if (!same_ops && (op2_info & MAY_BE_LONG)) { 8048 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3 8049 } else { 8050 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 8051 } 8052 } 8053 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8054 return 0; 8055 } 8056 } 8057 if (!same_ops && (op2_info & MAY_BE_LONG)) { 8058 if (op2_info & MAY_BE_DOUBLE) { 8059 |.cold_code 8060 } 8061 |3: 8062 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 8063 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 8064 } 8065 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8066 return 0; 8067 } 8068 if (op2_info & MAY_BE_DOUBLE) { 8069 | jmp >6 8070 |.code 8071 } 8072 } 8073 } else if ((op2_info & MAY_BE_DOUBLE) && 8074 !(op2_info & MAY_BE_LONG) && 8075 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 8076 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 8077 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 8078 } 8079 if (op1_info & MAY_BE_DOUBLE) { 8080 if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 8081 if (!same_ops && (op1_info & MAY_BE_LONG)) { 8082 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3 8083 } else { 8084 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 8085 } 8086 } 8087 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8088 return 0; 8089 } 8090 } 8091 if (!same_ops && (op1_info & MAY_BE_LONG)) { 8092 if (op1_info & MAY_BE_DOUBLE) { 8093 |.cold_code 8094 } 8095 |3: 8096 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 8097 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 8098 } 8099 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8100 return 0; 8101 } 8102 if (op1_info & MAY_BE_DOUBLE) { 8103 | jmp >6 8104 |.code 8105 } 8106 } 8107 } 8108 8109 if (has_slow || 8110 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 8111 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 8112 if (has_slow) { 8113 |.cold_code 8114 |9: 8115 } 8116 | SET_EX_OPLINE opline, r0 8117 if (Z_MODE(op1_addr) == IS_REG) { 8118 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8119 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 8120 return 0; 8121 } 8122 op1_addr = real_addr; 8123 } 8124 if (Z_MODE(op2_addr) == IS_REG) { 8125 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 8126 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 8127 return 0; 8128 } 8129 op2_addr = real_addr; 8130 } 8131 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8132 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { 8133 | IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1 8134 | mov FCARG1a, opline->op1.var 8135 | EXT_CALL zend_jit_undefined_op_helper, r0 8136 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8137 |1: 8138 } 8139 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { 8140 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 8141 | mov T1, FCARG1a // save 8142 | mov FCARG1a, opline->op2.var 8143 | EXT_CALL zend_jit_undefined_op_helper, r0 8144 | mov FCARG1a, T1 // restore 8145 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8146 | jmp >2 8147 |1: 8148 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8149 |2: 8150 } else { 8151 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8152 } 8153 | EXT_CALL zend_compare, r0 8154 if ((opline->opcode != ZEND_CASE && 8155 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8156 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8157 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8158 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8159 | mov dword T1, eax // save 8160 if (opline->opcode != ZEND_CASE) { 8161 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, NULL 8162 } 8163 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL 8164 if (may_throw) { 8165 zend_jit_check_exception_undef_result(Dst, opline); 8166 } 8167 | mov eax, dword T1 // restore 8168 } else if (may_throw) { 8169#if ZTS 8170 | mov dword T1, eax // save 8171#else 8172 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(exception)))) { 8173 | mov dword T1, eax // save 8174 } 8175#endif 8176 zend_jit_check_exception_undef_result(Dst, opline); 8177#if ZTS 8178 | mov eax, dword T1 // restore 8179#else 8180 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(exception)))) { 8181 | mov eax, dword T1 // restore 8182 } 8183#endif 8184 } 8185 if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8186 return 0; 8187 } 8188 if (has_slow) { 8189 | jmp >6 8190 |.code 8191 } 8192 } 8193 8194 |6: 8195 8196 return 1; 8197} 8198 8199static int zend_jit_identical(dasm_State **Dst, 8200 const zend_op *opline, 8201 uint32_t op1_info, 8202 zend_ssa_range *op1_range, 8203 zend_jit_addr op1_addr, 8204 uint32_t op2_info, 8205 zend_ssa_range *op2_range, 8206 zend_jit_addr op2_addr, 8207 zend_jit_addr res_addr, 8208 int may_throw, 8209 zend_uchar smart_branch_opcode, 8210 uint32_t target_label, 8211 uint32_t target_label2, 8212 const void *exit_addr, 8213 bool skip_comparison) 8214{ 8215 uint32_t identical_label = (uint32_t)-1; 8216 uint32_t not_identical_label = (uint32_t)-1; 8217 8218 if (smart_branch_opcode && !exit_addr) { 8219 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8220 if (smart_branch_opcode == ZEND_JMPZ) { 8221 not_identical_label = target_label; 8222 } else if (smart_branch_opcode == ZEND_JMPNZ) { 8223 identical_label = target_label; 8224 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 8225 not_identical_label = target_label; 8226 identical_label = target_label2; 8227 } else { 8228 ZEND_UNREACHABLE(); 8229 } 8230 } else { 8231 if (smart_branch_opcode == ZEND_JMPZ) { 8232 identical_label = target_label; 8233 } else if (smart_branch_opcode == ZEND_JMPNZ) { 8234 not_identical_label = target_label; 8235 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 8236 identical_label = target_label; 8237 not_identical_label = target_label2; 8238 } else { 8239 ZEND_UNREACHABLE(); 8240 } 8241 } 8242 } 8243 8244 if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && 8245 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { 8246 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 8247 return 0; 8248 } 8249 return 1; 8250 } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && 8251 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { 8252 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8253 return 0; 8254 } 8255 return 1; 8256 } 8257 8258 if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { 8259 op1_info |= MAY_BE_NULL; 8260 op2_info |= MAY_BE_NULL; 8261 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8262 | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 8263 |.cold_code 8264 |1: 8265 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8266 | SET_EX_OPLINE opline, r0 8267 | mov FCARG1d, opline->op1.var 8268 | EXT_CALL zend_jit_undefined_op_helper, r0 8269 if (may_throw) { 8270 zend_jit_check_exception_undef_result(Dst, opline); 8271 } 8272 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8273 | jmp >1 8274 |.code 8275 |1: 8276 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8277 | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 8278 |.cold_code 8279 |1: 8280 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8281 | SET_EX_OPLINE opline, r0 8282 | mov aword T1, FCARG1a // save 8283 | mov FCARG1d, opline->op2.var 8284 | EXT_CALL zend_jit_undefined_op_helper, r0 8285 if (may_throw) { 8286 zend_jit_check_exception_undef_result(Dst, opline); 8287 } 8288 | mov FCARG1a, aword T1 // restore 8289 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8290 | jmp >1 8291 |.code 8292 |1: 8293 } else if (op1_info & MAY_BE_UNDEF) { 8294 op1_info |= MAY_BE_NULL; 8295 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8296 | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 8297 |.cold_code 8298 |1: 8299 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8300 | SET_EX_OPLINE opline, r0 8301 | mov FCARG1d, opline->op1.var 8302 | EXT_CALL zend_jit_undefined_op_helper, r0 8303 if (may_throw) { 8304 zend_jit_check_exception_undef_result(Dst, opline); 8305 } 8306 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8307 | jmp >1 8308 |.code 8309 |1: 8310 if (opline->op2_type != IS_CONST) { 8311 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8312 } 8313 } else if (op2_info & MAY_BE_UNDEF) { 8314 op2_info |= MAY_BE_NULL; 8315 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8316 | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 8317 |.cold_code 8318 |1: 8319 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8320 | SET_EX_OPLINE opline, r0 8321 | mov FCARG1d, opline->op2.var 8322 | EXT_CALL zend_jit_undefined_op_helper, r0 8323 if (may_throw) { 8324 zend_jit_check_exception_undef_result(Dst, opline); 8325 } 8326 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8327 | jmp >1 8328 |.code 8329 |1: 8330 if (opline->op1_type != IS_CONST) { 8331 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8332 } 8333 } else if ((op1_info & op2_info & MAY_BE_ANY) != 0) { 8334 if (opline->op1_type != IS_CONST) { 8335 if (Z_MODE(op1_addr) == IS_REG) { 8336 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8337 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 8338 return 0; 8339 } 8340 op1_addr = real_addr; 8341 } 8342 } 8343 if (opline->op2_type != IS_CONST) { 8344 if (Z_MODE(op2_addr) == IS_REG) { 8345 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 8346 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 8347 return 0; 8348 } 8349 op2_addr = real_addr; 8350 } 8351 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8352 } 8353 if (opline->op1_type != IS_CONST) { 8354 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8355 } 8356 } 8357 8358 if ((op1_info & op2_info & MAY_BE_ANY) == 0) { 8359 if ((opline->opcode != ZEND_CASE_STRICT && 8360 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8361 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8362 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8363 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8364 if (opline->opcode != ZEND_CASE_STRICT) { 8365 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8366 } 8367 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8368 } 8369 if (smart_branch_opcode) { 8370 if (may_throw) { 8371 zend_jit_check_exception_undef_result(Dst, opline); 8372 } 8373 if (exit_addr) { 8374 if (smart_branch_opcode == ZEND_JMPZ) { 8375 | jmp &exit_addr 8376 } 8377 } else if (not_identical_label != (uint32_t)-1) { 8378 | jmp =>not_identical_label 8379 } 8380 } else { 8381 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) 8382 if (may_throw) { 8383 zend_jit_check_exception(Dst); 8384 } 8385 } 8386 return 1; 8387 } 8388 8389 if (opline->op1_type & (IS_CV|IS_VAR)) { 8390 | ZVAL_DEREF FCARG1a, op1_info 8391 } 8392 if (opline->op2_type & (IS_CV|IS_VAR)) { 8393 | ZVAL_DEREF FCARG2a, op2_info 8394 } 8395 8396 if (has_concrete_type(op1_info) 8397 && has_concrete_type(op2_info) 8398 && concrete_type(op1_info) == concrete_type(op2_info) 8399 && concrete_type(op1_info) <= IS_TRUE) { 8400 if (smart_branch_opcode) { 8401 if (exit_addr) { 8402 if (smart_branch_opcode == ZEND_JMPNZ) { 8403 | jmp &exit_addr 8404 } 8405 } else if (identical_label != (uint32_t)-1) { 8406 | jmp =>identical_label 8407 } 8408 } else { 8409 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) 8410 } 8411 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { 8412 if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { 8413 if (smart_branch_opcode) { 8414 if (exit_addr) { 8415 if (smart_branch_opcode == ZEND_JMPNZ) { 8416 | jmp &exit_addr 8417 } 8418 } else if (identical_label != (uint32_t)-1) { 8419 | jmp =>identical_label 8420 } 8421 } else { 8422 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) 8423 } 8424 } else { 8425 if (smart_branch_opcode) { 8426 if (exit_addr) { 8427 if (smart_branch_opcode == ZEND_JMPZ) { 8428 | jmp &exit_addr 8429 } 8430 } else if (not_identical_label != (uint32_t)-1) { 8431 | jmp =>not_identical_label 8432 } 8433 } else { 8434 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) 8435 } 8436 } 8437 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { 8438 zval *val = Z_ZV(op1_addr); 8439 8440 | cmp byte [FCARG2a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) 8441 if (smart_branch_opcode) { 8442 if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { 8443 | jne >8 8444 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8445 if (may_throw) { 8446 zend_jit_check_exception_undef_result(Dst, opline); 8447 } 8448 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8449 | jmp &exit_addr 8450 } else if (identical_label != (uint32_t)-1) { 8451 | jmp =>identical_label 8452 } else { 8453 | jmp >9 8454 } 8455 |8: 8456 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8457 | je &exit_addr 8458 } else if (identical_label != (uint32_t)-1) { 8459 | je =>identical_label 8460 } else { 8461 | je >9 8462 } 8463 } else { 8464 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8465 | sete al 8466 } else { 8467 | setne al 8468 } 8469 | movzx eax, al 8470 | lea eax, [eax + 2] 8471 | SET_ZVAL_TYPE_INFO res_addr, eax 8472 } 8473 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8474 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 8475 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8476 if (may_throw) { 8477 zend_jit_check_exception_undef_result(Dst, opline); 8478 } 8479 } 8480 if (exit_addr) { 8481 if (smart_branch_opcode == ZEND_JMPZ) { 8482 | jmp &exit_addr 8483 } 8484 } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { 8485 | jmp =>not_identical_label 8486 } 8487 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { 8488 zval *val = Z_ZV(op2_addr); 8489 8490 | cmp byte [FCARG1a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) 8491 if (smart_branch_opcode) { 8492 if (opline->opcode != ZEND_CASE_STRICT 8493 && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { 8494 | jne >8 8495 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8496 if (may_throw) { 8497 zend_jit_check_exception_undef_result(Dst, opline); 8498 } 8499 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8500 | jmp &exit_addr 8501 } else if (identical_label != (uint32_t)-1) { 8502 | jmp =>identical_label 8503 } else { 8504 | jmp >9 8505 } 8506 |8: 8507 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8508 | je &exit_addr 8509 } else if (identical_label != (uint32_t)-1) { 8510 | je =>identical_label 8511 } else { 8512 | je >9 8513 } 8514 } else { 8515 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8516 | sete al 8517 } else { 8518 | setne al 8519 } 8520 | movzx eax, al 8521 | lea eax, [eax + 2] 8522 | SET_ZVAL_TYPE_INFO res_addr, eax 8523 } 8524 if (opline->opcode != ZEND_CASE_STRICT 8525 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8526 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 8527 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8528 if (may_throw) { 8529 zend_jit_check_exception_undef_result(Dst, opline); 8530 } 8531 } 8532 if (smart_branch_opcode) { 8533 if (exit_addr) { 8534 if (smart_branch_opcode == ZEND_JMPZ) { 8535 | jmp &exit_addr 8536 } 8537 } else if (not_identical_label != (uint32_t)-1) { 8538 | jmp =>not_identical_label 8539 } 8540 } 8541 } else { 8542 if (opline->op1_type == IS_CONST) { 8543 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8544 } 8545 if (opline->op2_type == IS_CONST) { 8546 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8547 } 8548 | EXT_CALL zend_is_identical, r0 8549 if ((opline->opcode != ZEND_CASE_STRICT && 8550 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8551 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8552 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8553 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8554 | mov aword T1, r0 // save 8555 if (opline->opcode != ZEND_CASE_STRICT) { 8556 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8557 } 8558 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8559 if (may_throw) { 8560 zend_jit_check_exception_undef_result(Dst, opline); 8561 } 8562 | mov r0, aword T1 // restore 8563 } 8564 if (smart_branch_opcode) { 8565 | test al, al 8566 if (exit_addr) { 8567 if (smart_branch_opcode == ZEND_JMPNZ) { 8568 | jnz &exit_addr 8569 } else { 8570 | jz &exit_addr 8571 } 8572 } else if (not_identical_label != (uint32_t)-1) { 8573 | jz =>not_identical_label 8574 if (identical_label != (uint32_t)-1) { 8575 | jmp =>identical_label 8576 } 8577 } else if (identical_label != (uint32_t)-1) { 8578 | jnz =>identical_label 8579 } 8580 } else { 8581 | movzx eax, al 8582 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8583 | lea eax, [eax + 2] 8584 } else { 8585 | neg eax 8586 | lea eax, [eax + 3] 8587 } 8588 | SET_ZVAL_TYPE_INFO res_addr, eax 8589 } 8590 } 8591 8592 |9: 8593 if (may_throw) { 8594 zend_jit_check_exception(Dst); 8595 } 8596 return 1; 8597} 8598 8599static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr) 8600{ 8601 uint32_t true_label = -1; 8602 uint32_t false_label = -1; 8603 bool set_bool = 0; 8604 bool set_bool_not = 0; 8605 bool set_delayed = 0; 8606 bool jmp_done = 0; 8607 8608 if (branch_opcode == ZEND_BOOL) { 8609 set_bool = 1; 8610 } else if (branch_opcode == ZEND_BOOL_NOT) { 8611 set_bool = 1; 8612 set_bool_not = 1; 8613 } else if (branch_opcode == ZEND_JMPZ) { 8614 false_label = target_label; 8615 } else if (branch_opcode == ZEND_JMPNZ) { 8616 true_label = target_label; 8617 } else if (branch_opcode == ZEND_JMPZNZ) { 8618 true_label = target_label2; 8619 false_label = target_label; 8620 } else if (branch_opcode == ZEND_JMPZ_EX) { 8621 set_bool = 1; 8622 false_label = target_label; 8623 } else if (branch_opcode == ZEND_JMPNZ_EX) { 8624 set_bool = 1; 8625 true_label = target_label; 8626 } else { 8627 ZEND_UNREACHABLE(); 8628 } 8629 8630 if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { 8631 if (zend_is_true(Z_ZV(op1_addr))) { 8632 /* Always TRUE */ 8633 if (set_bool) { 8634 if (set_bool_not) { 8635 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8636 } else { 8637 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8638 } 8639 } 8640 if (true_label != (uint32_t)-1) { 8641 | jmp =>true_label; 8642 } 8643 } else { 8644 /* Always FALSE */ 8645 if (set_bool) { 8646 if (set_bool_not) { 8647 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8648 } else { 8649 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8650 } 8651 } 8652 if (false_label != (uint32_t)-1) { 8653 | jmp =>false_label; 8654 } 8655 } 8656 return 1; 8657 } 8658 8659 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) { 8660 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8661 | ZVAL_DEREF FCARG1a, op1_info 8662 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 8663 } 8664 8665 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { 8666 if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { 8667 /* Always TRUE */ 8668 if (set_bool) { 8669 if (set_bool_not) { 8670 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8671 } else { 8672 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8673 } 8674 } 8675 if (true_label != (uint32_t)-1) { 8676 | jmp =>true_label; 8677 } 8678 } else { 8679 if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { 8680 /* Always FALSE */ 8681 if (set_bool) { 8682 if (set_bool_not) { 8683 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8684 } else { 8685 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8686 } 8687 } 8688 } else { 8689 | CMP_ZVAL_TYPE op1_addr, IS_TRUE 8690 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 8691 if ((op1_info & MAY_BE_LONG) && 8692 !(op1_info & MAY_BE_UNDEF) && 8693 !set_bool) { 8694 if (exit_addr) { 8695 if (branch_opcode == ZEND_JMPNZ) { 8696 | jl >9 8697 } else { 8698 | jl &exit_addr 8699 } 8700 } else if (false_label != (uint32_t)-1) { 8701 | jl =>false_label 8702 } else { 8703 | jl >9 8704 } 8705 jmp_done = 1; 8706 } else { 8707 | jg >2 8708 } 8709 } 8710 if (!(op1_info & MAY_BE_TRUE)) { 8711 /* It's FALSE */ 8712 if (set_bool) { 8713 if (set_bool_not) { 8714 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8715 } else { 8716 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8717 } 8718 } 8719 } else { 8720 if (exit_addr) { 8721 if (set_bool) { 8722 | jne >1 8723 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8724 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8725 | jmp &exit_addr 8726 } else { 8727 | jmp >9 8728 } 8729 |1: 8730 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8731 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8732 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8733 | jne &exit_addr 8734 } 8735 } 8736 } else { 8737 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8738 | je &exit_addr 8739 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8740 | jne &exit_addr 8741 } else { 8742 | je >9 8743 } 8744 } 8745 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8746 if (set_bool) { 8747 | jne >1 8748 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8749 if (true_label != (uint32_t)-1) { 8750 | jmp =>true_label 8751 } else { 8752 | jmp >9 8753 } 8754 |1: 8755 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8756 } else { 8757 if (true_label != (uint32_t)-1) { 8758 | je =>true_label 8759 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8760 | jne =>false_label 8761 jmp_done = 1; 8762 } else { 8763 | je >9 8764 } 8765 } 8766 } else if (set_bool) { 8767 | sete al 8768 | movzx eax, al 8769 if (set_bool_not) { 8770 | neg eax 8771 | add eax, 3 8772 } else { 8773 | add eax, 2 8774 } 8775 if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { 8776 set_delayed = 1; 8777 } else { 8778 | SET_ZVAL_TYPE_INFO res_addr, eax 8779 } 8780 } 8781 } 8782 } 8783 8784 /* It's FALSE, but may be UNDEF */ 8785 if (op1_info & MAY_BE_UNDEF) { 8786 if (op1_info & MAY_BE_ANY) { 8787 if (set_delayed) { 8788 | CMP_ZVAL_TYPE op1_addr, IS_UNDEF 8789 | SET_ZVAL_TYPE_INFO res_addr, eax 8790 | jz >1 8791 } else { 8792 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 8793 } 8794 |.cold_code 8795 |1: 8796 } 8797 | mov FCARG1d, opline->op1.var 8798 | SET_EX_OPLINE opline, r0 8799 | EXT_CALL zend_jit_undefined_op_helper, r0 8800 8801 if (may_throw) { 8802 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 8803 return 0; 8804 } 8805 } 8806 8807 if (exit_addr) { 8808 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8809 | jmp &exit_addr 8810 } 8811 } else if (false_label != (uint32_t)-1) { 8812 | jmp =>false_label 8813 } 8814 if (op1_info & MAY_BE_ANY) { 8815 if (exit_addr) { 8816 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8817 | jmp >9 8818 } 8819 } else if (false_label == (uint32_t)-1) { 8820 | jmp >9 8821 } 8822 |.code 8823 } 8824 } 8825 8826 if (!jmp_done) { 8827 if (exit_addr) { 8828 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8829 if (op1_info & MAY_BE_LONG) { 8830 | jmp >9 8831 } 8832 } else if (op1_info & MAY_BE_LONG) { 8833 | jmp &exit_addr 8834 } 8835 } else if (false_label != (uint32_t)-1) { 8836 | jmp =>false_label 8837 } else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 8838 | jmp >9 8839 } 8840 } 8841 } 8842 } 8843 8844 if (op1_info & MAY_BE_LONG) { 8845 |2: 8846 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8847 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 8848 } 8849 if (Z_MODE(op1_addr) == IS_REG) { 8850 | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) 8851 } else { 8852 | LONG_OP_WITH_CONST, cmp, op1_addr, Z_L(0) 8853 } 8854 if (set_bool) { 8855 | setne al 8856 | movzx eax, al 8857 if (set_bool_not) { 8858 | neg eax 8859 | add eax, 3 8860 } else { 8861 | lea eax, [eax + 2] 8862 } 8863 | SET_ZVAL_TYPE_INFO res_addr, eax 8864 } 8865 if (exit_addr) { 8866 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8867 | jne &exit_addr 8868 } else { 8869 | je &exit_addr 8870 } 8871 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8872 if (true_label != (uint32_t)-1) { 8873 | jne =>true_label 8874 if (false_label != (uint32_t)-1) { 8875 | jmp =>false_label 8876 } 8877 } else { 8878 | je =>false_label 8879 } 8880 } 8881 } 8882 8883 if ((op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) == MAY_BE_DOUBLE) { 8884 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8885 |.cold_code 8886 } 8887 |2: 8888 if (CAN_USE_AVX()) { 8889 | vxorps xmm0, xmm0, xmm0 8890 } else { 8891 | xorps xmm0, xmm0 8892 } 8893 | DOUBLE_CMP ZREG_XMM0, op1_addr 8894 8895 if (set_bool) { 8896 if (exit_addr) { 8897 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8898 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8899 | jp &exit_addr 8900 | jne &exit_addr 8901 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8902 } else { 8903 | jp >1 8904 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8905 | je &exit_addr 8906 |1: 8907 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8908 } 8909 } else if (false_label != (uint32_t)-1) { // JMPZ_EX (p=>true, z=>false, false=>jmp) 8910 | jp >1 8911 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8912 | je => false_label 8913 |1: 8914 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8915 } else if (true_label != (uint32_t)-1) { // JMPNZ_EX (p=>true, z=>false, true=>jmp) 8916 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8917 | jp => true_label 8918 | jne => true_label 8919 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8920 } else if (set_bool_not) { // BOOL_NOT (p=>false, z=>true) 8921 | mov eax, IS_FALSE 8922 | jp >1 8923 | jne >1 8924 | mov eax, IS_TRUE 8925 |1: 8926 | SET_ZVAL_TYPE_INFO res_addr, eax 8927 } else { // BOOL (p=>true, z=>false) 8928 | mov eax, IS_TRUE 8929 | jp >1 8930 | jne >1 8931 | mov eax, IS_FALSE 8932 |1: 8933 | SET_ZVAL_TYPE_INFO res_addr, eax 8934 } 8935 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8936 | jmp >9 8937 |.code 8938 } 8939 } else { 8940 if (exit_addr) { 8941 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8942 | jp &exit_addr 8943 | jne &exit_addr 8944 |1: 8945 } else { 8946 | jp >1 8947 | je &exit_addr 8948 |1: 8949 } 8950 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8951 | jmp >9 8952 } 8953 } else { 8954 ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); 8955 if (false_label != (uint32_t)-1 ) { 8956 | jp >1 8957 | je => false_label 8958 |1: 8959 if (true_label != (uint32_t)-1) { 8960 | jmp =>true_label 8961 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8962 | jmp >9 8963 } 8964 } else { 8965 | jp => true_label 8966 | jne => true_label 8967 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8968 | jmp >9 8969 } 8970 } 8971 } 8972 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8973 |.code 8974 } 8975 } 8976 } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8977 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8978 |.cold_code 8979 |2: 8980 } 8981 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 8982 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8983 } 8984 | SET_EX_OPLINE opline, r0 8985 | EXT_CALL zend_is_true, r0 8986 8987 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8988 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8989 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8990 8991 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8992 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3 8993 } 8994 | GET_ZVAL_PTR FCARG1a, op1_addr 8995 | GC_DELREF FCARG1a 8996 | jnz >3 8997 | mov aword T1, r0 // save 8998 | ZVAL_DTOR_FUNC op1_info, opline 8999 | mov r0, aword T1 // restore 9000 |3: 9001 } 9002 if (may_throw) { 9003 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r1 9004 | jne ->exception_handler_undef 9005 } 9006 9007 if (set_bool) { 9008 if (set_bool_not) { 9009 | neg eax 9010 | add eax, 3 9011 } else { 9012 | add eax, 2 9013 } 9014 | SET_ZVAL_TYPE_INFO res_addr, eax 9015 if (exit_addr) { 9016 | CMP_ZVAL_TYPE res_addr, IS_FALSE 9017 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 9018 | jne &exit_addr 9019 } else { 9020 | je &exit_addr 9021 } 9022 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 9023 | CMP_ZVAL_TYPE res_addr, IS_FALSE 9024 if (true_label != (uint32_t)-1) { 9025 | jne =>true_label 9026 if (false_label != (uint32_t)-1) { 9027 | jmp =>false_label 9028 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 9029 | jmp >9 9030 } 9031 } else { 9032 | je =>false_label 9033 } 9034 } 9035 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 9036 | jmp >9 9037 |.code 9038 } 9039 } else { 9040 | test r0, r0 9041 if (exit_addr) { 9042 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 9043 | jne &exit_addr 9044 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 9045 | jmp >9 9046 } 9047 } else { 9048 | je &exit_addr 9049 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 9050 | jmp >9 9051 } 9052 } 9053 } else if (true_label != (uint32_t)-1) { 9054 | jne =>true_label 9055 if (false_label != (uint32_t)-1) { 9056 | jmp =>false_label 9057 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 9058 | jmp >9 9059 } 9060 } else { 9061 | je =>false_label 9062 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 9063 | jmp >9 9064 } 9065 } 9066 9067 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 9068 |.code 9069 } 9070 } 9071 } 9072 9073 |9: 9074 9075 return 1; 9076} 9077 9078static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) 9079{ 9080 if (op1_addr != op1_def_addr) { 9081 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 9082 return 0; 9083 } 9084 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 9085 op1_addr = op1_def_addr; 9086 } 9087 } 9088 9089 if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0, 1)) { 9090 return 0; 9091 } 9092 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 9093 return 0; 9094 } 9095 if (op1_info & MAY_BE_UNDEF) { 9096 zend_jit_check_exception(Dst); 9097 } 9098 return 1; 9099} 9100 9101static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 9102{ 9103 ZEND_ASSERT(opline->op1_type == IS_CV); 9104 9105 if (op2_addr != op2_def_addr) { 9106 if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { 9107 return 0; 9108 } 9109 if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { 9110 op2_addr = op2_def_addr; 9111 } 9112 } 9113 9114 if (Z_MODE(op1_addr) != IS_REG 9115 && Z_MODE(op1_use_addr) == IS_REG 9116 && !Z_LOAD(op1_use_addr) 9117 && !Z_STORE(op1_use_addr)) { 9118 /* Force type update */ 9119 op1_info |= MAY_BE_UNDEF; 9120 } 9121 if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, 9122 may_throw)) { 9123 return 0; 9124 } 9125 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { 9126 return 0; 9127 } 9128 if (opline->result_type != IS_UNUSED) { 9129 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 9130 return 0; 9131 } 9132 } 9133 9134 return 1; 9135} 9136 9137/* copy of hidden zend_closure */ 9138typedef struct _zend_closure { 9139 zend_object std; 9140 zend_function func; 9141 zval this_ptr; 9142 zend_class_entry *called_scope; 9143 zif_handler orig_internal_handler; 9144} zend_closure; 9145 9146static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) 9147{ 9148 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9149 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9150 9151 if (!exit_addr) { 9152 return 0; 9153 } 9154 9155 | // Check Stack Overflow 9156 | MEM_LOAD_ZTS r1, aword, executor_globals, vm_stack_end, r0 9157 | MEM_LOAD_OP_ZTS sub, r1, aword, executor_globals, vm_stack_top, r0 9158 | cmp r1, used_stack 9159 | jb &exit_addr 9160 9161 return 1; 9162} 9163 9164static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack) 9165{ 9166 uint32_t used_stack; 9167 bool stack_check = 1; 9168 9169 if (func) { 9170 used_stack = zend_vm_calc_used_stack(opline->extended_value, func); 9171 if ((int)used_stack <= checked_stack) { 9172 stack_check = 0; 9173 } 9174 } else { 9175 used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval); 9176 9177 | // if (EXPECTED(ZEND_USER_CODE(func->type))) { 9178 if (!is_closure) { 9179 | test byte [r0 + offsetof(zend_function, type)], 1 9180 | mov FCARG1a, used_stack 9181 | jnz >1 9182 } else { 9183 | mov FCARG1a, used_stack 9184 } 9185 | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); 9186 | mov edx, opline->extended_value 9187 if (!is_closure) { 9188 | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)] 9189 | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)] 9190 | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)] 9191 | sub edx, dword [r0 + offsetof(zend_function, op_array.T)] 9192 } else { 9193 | cmp edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] 9194 | cmova edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] 9195 | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.last_var)] 9196 | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.T)] 9197 } 9198 | shl edx, 4 9199 |.if X64 9200 | movsxd r2, edx 9201 |.endif 9202 | sub FCARG1a, r2 9203 |1: 9204 } 9205 9206 zend_jit_start_reuse_ip(); 9207 9208 | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { 9209 | MEM_LOAD_ZTS RX, aword, executor_globals, vm_stack_top, RX 9210 9211 if (stack_check) { 9212 | // Check Stack Overflow 9213 | MEM_LOAD_ZTS r2, aword, executor_globals, vm_stack_end, r2 9214 | sub r2, RX 9215 if (func) { 9216 | cmp r2, used_stack 9217 } else { 9218 | cmp r2, FCARG1a 9219 } 9220 9221 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9222 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9223 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9224 9225 if (!exit_addr) { 9226 return 0; 9227 } 9228 9229 | jb &exit_addr 9230 } else { 9231 | jb >1 9232 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); 9233 |.cold_code 9234 |1: 9235 if (func) { 9236 | mov FCARG1d, used_stack 9237 } 9238#ifdef _WIN32 9239 if (0) { 9240#else 9241 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 9242#endif 9243 | SET_EX_OPLINE opline, r0 9244 | EXT_CALL zend_jit_int_extend_stack_helper, r0 9245 } else { 9246 if (!is_closure) { 9247 if (func 9248 && op_array == &func->op_array 9249 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9250 && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { 9251 | LOAD_ADDR FCARG2a, func 9252 } else { 9253 | mov FCARG2a, r0 9254 } 9255 } else { 9256 | lea FCARG2a, aword [r0 + offsetof(zend_closure, func)] 9257 } 9258 | SET_EX_OPLINE opline, r0 9259 | EXT_CALL zend_jit_extend_stack_helper, r0 9260 } 9261 | mov RX, r0 9262 | jmp >1 9263 |.code 9264 } 9265 } 9266 9267 if (func) { 9268 | MEM_UPDATE_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2 9269 } else { 9270 | MEM_UPDATE_ZTS add, aword, executor_globals, vm_stack_top, FCARG1a, r2 9271 } 9272 | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); 9273 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { 9274 | // ZEND_SET_CALL_INFO(call, 0, call_info); 9275 | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) 9276 } 9277#ifdef _WIN32 9278 if (0) { 9279#else 9280 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 9281#endif 9282 | // call->func = func; 9283 |1: 9284 | ADDR_STORE aword EX:RX->func, func, r1 9285 } else { 9286 if (!is_closure) { 9287 | // call->func = func; 9288 if (func 9289 && op_array == &func->op_array 9290 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9291 && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { 9292 | ADDR_STORE aword EX:RX->func, func, r1 9293 } else { 9294 | mov aword EX:RX->func, r0 9295 } 9296 } else { 9297 | // call->func = &closure->func; 9298 | lea r1, aword [r0 + offsetof(zend_closure, func)] 9299 | mov aword EX:RX->func, r1 9300 } 9301 |1: 9302 } 9303 if (opline->opcode == ZEND_INIT_METHOD_CALL) { 9304 | // Z_PTR(call->This) = obj; 9305 | mov r1, aword T1 9306 | mov aword EX:RX->This.value.ptr, r1 9307 if (opline->op1_type == IS_UNUSED || delayed_fetch_this) { 9308 | // call->call_info |= ZEND_CALL_HAS_THIS; 9309 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9310 | mov dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS 9311 } else { 9312 | or dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS 9313 } 9314 } else { 9315 if (opline->op1_type == IS_CV) { 9316 | // GC_ADDREF(obj); 9317 | add dword [r1], 1 9318 } 9319 | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; 9320 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9321 | mov dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 9322 } else { 9323 | or dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 9324 } 9325 } 9326 } else if (!is_closure) { 9327 | // Z_CE(call->This) = called_scope; 9328 | mov aword EX:RX->This.value.ptr, 0 9329 } else { 9330 if (opline->op2_type == IS_CV) { 9331 | // GC_ADDREF(closure); 9332 | add dword [r0], 1 9333 } 9334 | // object_or_called_scope = closure->called_scope; 9335 | mov r1, aword [r0 + offsetof(zend_closure, called_scope)] 9336 | mov aword EX:RX->This.value.ptr, r1 9337 | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | 9338 | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); 9339 | mov edx, dword [r0 + offsetof(zend_closure, func.common.fn_flags)] 9340 | and edx, ZEND_ACC_FAKE_CLOSURE 9341 | or edx, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE) 9342 | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { 9343 | cmp byte [r0 + offsetof(zend_closure, this_ptr.u1.v.type)], IS_UNDEF 9344 | jz >1 9345 | // call_info |= ZEND_CALL_HAS_THIS; 9346 | or edx, ZEND_CALL_HAS_THIS 9347 | // object_or_called_scope = Z_OBJ(closure->this_ptr); 9348 | mov r1, aword [r0 + offsetof(zend_closure, this_ptr.value.ptr)] 9349 |1: 9350 | // ZEND_SET_CALL_INFO(call, 0, call_info); 9351 | or dword EX:RX->This.u1.type_info, edx 9352 | // Z_PTR(call->This) = object_or_called_scope; 9353 | mov aword EX:RX->This.value.ptr, r1 9354 | cmp aword [r0 + offsetof(zend_closure, func.op_array.run_time_cache__ptr)], 0 9355 | jnz >1 9356 | lea FCARG1a, aword [r0 + offsetof(zend_closure, func)] 9357 | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 9358 |1: 9359 } 9360 | // ZEND_CALL_NUM_ARGS(call) = num_args; 9361 | mov dword EX:RX->This.u2.num_args, opline->extended_value 9362 return 1; 9363} 9364 9365static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) 9366{ 9367 int32_t exit_point; 9368 const void *exit_addr; 9369 9370 if (func->type == ZEND_INTERNAL_FUNCTION) { 9371#ifdef ZEND_WIN32 9372 // TODO: ASLR may cause different addresses in different workers ??? 9373 return 0; 9374#endif 9375 } else if (func->type == ZEND_USER_FUNCTION) { 9376 if (!zend_accel_in_shm(func->op_array.opcodes)) { 9377 /* op_array and op_array->opcodes are not persistent. We can't link. */ 9378 return 0; 9379 } 9380 } else { 9381 ZEND_UNREACHABLE(); 9382 return 0; 9383 } 9384 9385 exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); 9386 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9387 if (!exit_addr) { 9388 return 0; 9389 } 9390 9391 | // call = EX(call); 9392 | mov r1, EX->call 9393 while (level > 0) { 9394 | mov r1, EX:r1->prev_execute_data 9395 level--; 9396 } 9397 9398 if (func->type == ZEND_USER_FUNCTION && 9399 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 9400 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 9401 !func->common.function_name)) { 9402 const zend_op *opcodes = func->op_array.opcodes; 9403 9404 | mov r1, aword EX:r1->func 9405 | .if X64 9406 || if (!IS_SIGNED_32BIT(opcodes)) { 9407 | mov64 r2, ((ptrdiff_t)opcodes) 9408 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], r2 9409 || } else { 9410 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes 9411 || } 9412 | .else 9413 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes 9414 | .endif 9415 | jne &exit_addr 9416 } else { 9417 | .if X64 9418 || if (!IS_SIGNED_32BIT(func)) { 9419 | mov64 r2, ((ptrdiff_t)func) 9420 | cmp aword EX:r1->func, r2 9421 || } else { 9422 | cmp aword EX:r1->func, func 9423 || } 9424 | .else 9425 | cmp aword EX:r1->func, func 9426 | .endif 9427 | jne &exit_addr 9428 } 9429 9430 return 1; 9431} 9432 9433static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack) 9434{ 9435 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9436 zend_call_info *call_info = NULL; 9437 zend_function *func = NULL; 9438 9439 if (delayed_call_chain) { 9440 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9441 return 0; 9442 } 9443 } 9444 9445 if (info) { 9446 call_info = info->callee_info; 9447 while (call_info && call_info->caller_init_opline != opline) { 9448 call_info = call_info->next_callee; 9449 } 9450 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9451 func = call_info->callee_func; 9452 } 9453 } 9454 9455 if (!func 9456 && trace 9457 && trace->op == ZEND_JIT_TRACE_INIT_CALL) { 9458#ifdef _WIN32 9459 /* ASLR */ 9460 if (trace->func->type != ZEND_INTERNAL_FUNCTION) { 9461 func = (zend_function*)trace->func; 9462 } 9463#else 9464 func = (zend_function*)trace->func; 9465#endif 9466 } 9467 9468#ifdef _WIN32 9469 if (0) { 9470#else 9471 if (opline->opcode == ZEND_INIT_FCALL 9472 && func 9473 && func->type == ZEND_INTERNAL_FUNCTION) { 9474#endif 9475 /* load constant address later */ 9476 } else if (func && op_array == &func->op_array) { 9477 /* recursive call */ 9478 if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) || 9479 (sizeof(void*) == 8 && !IS_SIGNED_32BIT(func))) { 9480 | mov r0, EX->func 9481 } 9482 } else { 9483 | // if (CACHED_PTR(opline->result.num)) 9484 | mov r2, EX->run_time_cache 9485 | mov r0, aword [r2 + opline->result.num] 9486 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 9487 && func 9488 && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) 9489 && opline->opcode != ZEND_INIT_FCALL) { 9490 /* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */ 9491 | .if X64 9492 || if (!IS_SIGNED_32BIT(func)) { 9493 | mov64 r1, ((ptrdiff_t)func) 9494 | cmp r0, r1 9495 || } else { 9496 | cmp r0, func 9497 || } 9498 | .else 9499 | cmp r0, func 9500 | .endif 9501 | jnz >1 9502 |.cold_code 9503 |1: 9504 } else { 9505 | test r0, r0 9506 | jz >1 9507 } 9508 |.cold_code 9509 |1: 9510 if (opline->opcode == ZEND_INIT_FCALL 9511 && func 9512 && func->type == ZEND_USER_FUNCTION 9513 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { 9514 | LOAD_ADDR FCARG1a, func 9515 | mov aword [r2 + opline->result.num], FCARG1a 9516 | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 9517 | jmp >3 9518 } else { 9519 zval *zv = RT_CONSTANT(opline, opline->op2); 9520 9521 if (opline->opcode == ZEND_INIT_FCALL) { 9522 | LOAD_ADDR FCARG1a, Z_STR_P(zv); 9523 | lea FCARG2a, aword [r2 + opline->result.num] 9524 | EXT_CALL zend_jit_find_func_helper, r0 9525 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { 9526 | LOAD_ADDR FCARG1a, Z_STR_P(zv + 1); 9527 | lea FCARG2a, aword [r2 + opline->result.num] 9528 | EXT_CALL zend_jit_find_func_helper, r0 9529 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { 9530 | LOAD_ADDR FCARG1a, zv; 9531 | lea FCARG2a, aword [r2 + opline->result.num] 9532 | EXT_CALL zend_jit_find_ns_func_helper, r0 9533 } else { 9534 ZEND_UNREACHABLE(); 9535 } 9536 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9537 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 9538 func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0); 9539 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9540 9541 if (!exit_addr) { 9542 return 0; 9543 } 9544 if (!func || opline->opcode == ZEND_INIT_FCALL) { 9545 | test r0, r0 9546 | jnz >3 9547 } else if (func->type == ZEND_USER_FUNCTION 9548 && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { 9549 const zend_op *opcodes = func->op_array.opcodes; 9550 9551 | .if X64 9552 || if (!IS_SIGNED_32BIT(opcodes)) { 9553 | mov64 r1, ((ptrdiff_t)opcodes) 9554 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 9555 || } else { 9556 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9557 || } 9558 | .else 9559 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9560 | .endif 9561 | jz >3 9562 } else { 9563 | .if X64 9564 || if (!IS_SIGNED_32BIT(func)) { 9565 | mov64 r1, ((ptrdiff_t)func) 9566 | cmp r0, r1 9567 || } else { 9568 | cmp r0, func 9569 || } 9570 | .else 9571 | cmp r0, func 9572 | .endif 9573 | jz >3 9574 } 9575 | jmp &exit_addr 9576 } else { 9577 | test r0, r0 9578 | jnz >3 9579 | // SAVE_OPLINE(); 9580 | SET_EX_OPLINE opline, r0 9581 | jmp ->undefined_function 9582 } 9583 } 9584 |.code 9585 |3: 9586 } 9587 9588 if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, checked_stack)) { 9589 return 0; 9590 } 9591 9592 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9593 if (!zend_jit_save_call_chain(Dst, call_level)) { 9594 return 0; 9595 } 9596 } else { 9597 delayed_call_chain = 1; 9598 delayed_call_level = call_level; 9599 } 9600 9601 return 1; 9602} 9603 9604static int zend_jit_init_method_call(dasm_State **Dst, 9605 const zend_op *opline, 9606 uint32_t b, 9607 const zend_op_array *op_array, 9608 zend_ssa *ssa, 9609 const zend_ssa_op *ssa_op, 9610 int call_level, 9611 uint32_t op1_info, 9612 zend_jit_addr op1_addr, 9613 zend_class_entry *ce, 9614 bool ce_is_instanceof, 9615 bool on_this, 9616 bool delayed_fetch_this, 9617 zend_class_entry *trace_ce, 9618 zend_jit_trace_rec *trace, 9619 int checked_stack, 9620 bool polymorphic_side_trace) 9621{ 9622 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9623 zend_call_info *call_info = NULL; 9624 zend_function *func = NULL; 9625 zval *function_name; 9626 9627 ZEND_ASSERT(opline->op2_type == IS_CONST); 9628 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 9629 9630 function_name = RT_CONSTANT(opline, opline->op2); 9631 9632 if (info) { 9633 call_info = info->callee_info; 9634 while (call_info && call_info->caller_init_opline != opline) { 9635 call_info = call_info->next_callee; 9636 } 9637 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9638 func = call_info->callee_func; 9639 } 9640 } 9641 9642 if (polymorphic_side_trace) { 9643 /* function is passed in r0 from parent_trace */ 9644 } else { 9645 if (on_this) { 9646 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 9647 9648 | GET_ZVAL_PTR FCARG1a, this_addr 9649 } else { 9650 if (op1_info & MAY_BE_REF) { 9651 if (opline->op1_type == IS_CV) { 9652 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 9653 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9654 } 9655 | ZVAL_DEREF FCARG1a, op1_info 9656 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 9657 } else { 9658 /* Hack: Convert reference to regular value to simplify JIT code */ 9659 ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); 9660 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 9661 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9662 | EXT_CALL zend_jit_unref_helper, r0 9663 |1: 9664 } 9665 } 9666 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 9667 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9668 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9669 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9670 9671 if (!exit_addr) { 9672 return 0; 9673 } 9674 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 9675 } else { 9676 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 9677 |.cold_code 9678 |1: 9679 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 9680 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9681 } 9682 | SET_EX_OPLINE opline, r0 9683 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9684 | EXT_CALL zend_jit_invalid_method_call_tmp, r0 9685 } else { 9686 | EXT_CALL zend_jit_invalid_method_call, r0 9687 } 9688 | jmp ->exception_handler 9689 |.code 9690 } 9691 } 9692 | GET_ZVAL_PTR FCARG1a, op1_addr 9693 } 9694 9695 if (delayed_call_chain) { 9696 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9697 return 0; 9698 } 9699 } 9700 9701 | mov aword T1, FCARG1a // save 9702 9703 if (func) { 9704 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9705 | mov r0, EX->run_time_cache 9706 | mov r0, aword [r0 + opline->result.num + sizeof(void*)] 9707 | test r0, r0 9708 | jz >1 9709 } else { 9710 | // if (CACHED_PTR(opline->result.num) == obj->ce)) { 9711 | mov r0, EX->run_time_cache 9712 | mov r2, aword [r0 + opline->result.num] 9713 | cmp r2, [FCARG1a + offsetof(zend_object, ce)] 9714 | jnz >1 9715 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9716 | mov r0, aword [r0 + opline->result.num + sizeof(void*)] 9717 } 9718 9719 |.cold_code 9720 |1: 9721 | LOAD_ADDR FCARG2a, function_name 9722 |.if X64 9723 | lea CARG3, aword T1 9724 |.else 9725 | lea r0, aword T1 9726 | sub r4, 12 9727 | push r0 9728 |.endif 9729 | SET_EX_OPLINE opline, r0 9730 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9731 | EXT_CALL zend_jit_find_method_tmp_helper, r0 9732 } else { 9733 | EXT_CALL zend_jit_find_method_helper, r0 9734 } 9735 |.if not(X64) 9736 | add r4, 12 9737 |.endif 9738 | test r0, r0 9739 | jnz >2 9740 | jmp ->exception_handler 9741 |.code 9742 |2: 9743 } 9744 9745 if ((!func || zend_jit_may_be_modified(func, op_array)) 9746 && trace 9747 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9748 && trace->func 9749#ifdef _WIN32 9750 && trace->func->type != ZEND_INTERNAL_FUNCTION 9751#endif 9752 ) { 9753 int32_t exit_point; 9754 const void *exit_addr; 9755 9756 exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); 9757 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9758 if (!exit_addr) { 9759 return 0; 9760 } 9761 9762 func = (zend_function*)trace->func; 9763 9764 if (func->type == ZEND_USER_FUNCTION && 9765 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 9766 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 9767 !func->common.function_name)) { 9768 const zend_op *opcodes = func->op_array.opcodes; 9769 9770 | .if X64 9771 || if (!IS_SIGNED_32BIT(opcodes)) { 9772 | mov64 r1, ((ptrdiff_t)opcodes) 9773 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 9774 || } else { 9775 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9776 || } 9777 | .else 9778 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9779 | .endif 9780 | jne &exit_addr 9781 } else { 9782 | .if X64 9783 || if (!IS_SIGNED_32BIT(func)) { 9784 | mov64 r1, ((ptrdiff_t)func) 9785 | cmp r0, r1 9786 || } else { 9787 | cmp r0, func 9788 || } 9789 | .else 9790 | cmp r0, func 9791 | .endif 9792 | jne &exit_addr 9793 } 9794 } 9795 9796 if (!func) { 9797 | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { 9798 | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_STATIC 9799 | jnz >1 9800 |.cold_code 9801 |1: 9802 } 9803 9804 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { 9805 | mov FCARG1a, aword T1 // restore 9806 | mov FCARG2a, r0 9807 |.if X64 9808 | mov CARG3d, opline->extended_value 9809 |.else 9810 | sub r4, 12 9811 | push opline->extended_value 9812 |.endif 9813 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { 9814 | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, r0 9815 } else { 9816 | EXT_CALL zend_jit_push_static_metod_call_frame, r0 9817 } 9818 |.if not(X64) 9819 | add r4, 12 9820 |.endif 9821 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) { 9822 | test r0, r0 9823 | jz ->exception_handler 9824 } 9825 | mov RX, r0 9826 } 9827 9828 if (!func) { 9829 | jmp >9 9830 |.code 9831 } 9832 9833 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { 9834 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, delayed_fetch_this, checked_stack)) { 9835 return 0; 9836 } 9837 } 9838 9839 if (!func) { 9840 |9: 9841 } 9842 zend_jit_start_reuse_ip(); 9843 9844 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9845 if (!zend_jit_save_call_chain(Dst, call_level)) { 9846 return 0; 9847 } 9848 } else { 9849 delayed_call_chain = 1; 9850 delayed_call_level = call_level; 9851 } 9852 9853 return 1; 9854} 9855 9856static int zend_jit_init_closure_call(dasm_State **Dst, 9857 const zend_op *opline, 9858 uint32_t b, 9859 const zend_op_array *op_array, 9860 zend_ssa *ssa, 9861 const zend_ssa_op *ssa_op, 9862 int call_level, 9863 zend_jit_trace_rec *trace, 9864 int checked_stack) 9865{ 9866 zend_function *func = NULL; 9867 zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 9868 9869 | GET_ZVAL_PTR r0, op2_addr 9870 9871 if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure 9872 && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { 9873 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9874 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9875 9876 if (!exit_addr) { 9877 return 0; 9878 } 9879 9880 |.if X64 9881 || if (!IS_SIGNED_32BIT(zend_ce_closure)) { 9882 | mov64 FCARG1a, ((ptrdiff_t)zend_ce_closure) 9883 | cmp aword [r0 + offsetof(zend_object, ce)], FCARG1a 9884 || } else { 9885 | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure 9886 || } 9887 |.else 9888 | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure 9889 |.endif 9890 | jne &exit_addr 9891 if (ssa->var_info && ssa_op->op2_use >= 0) { 9892 ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; 9893 ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; 9894 ssa->var_info[ssa_op->op2_use].is_instanceof = 0; 9895 } 9896 } 9897 9898 if (trace 9899 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9900 && trace->func 9901 && trace->func->type == ZEND_USER_FUNCTION) { 9902 const zend_op *opcodes; 9903 int32_t exit_point; 9904 const void *exit_addr; 9905 9906 func = (zend_function*)trace->func; 9907 opcodes = func->op_array.opcodes; 9908 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); 9909 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9910 if (!exit_addr) { 9911 return 0; 9912 } 9913 9914 | .if X64 9915 || if (!IS_SIGNED_32BIT(opcodes)) { 9916 | mov64 FCARG1a, ((ptrdiff_t)opcodes) 9917 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], FCARG1a 9918 || } else { 9919 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes 9920 || } 9921 | .else 9922 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes 9923 | .endif 9924 | jne &exit_addr 9925 } 9926 9927 if (delayed_call_chain) { 9928 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9929 return 0; 9930 } 9931 } 9932 9933 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, checked_stack)) { 9934 return 0; 9935 } 9936 9937 if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9938 if (!zend_jit_save_call_chain(Dst, call_level)) { 9939 return 0; 9940 } 9941 } else { 9942 delayed_call_chain = 1; 9943 delayed_call_level = call_level; 9944 } 9945 9946 if (trace 9947 && trace->op == ZEND_JIT_TRACE_END 9948 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 9949 if (!zend_jit_set_valid_ip(Dst, opline + 1)) { 9950 return 0; 9951 } 9952 } 9953 9954 return 1; 9955} 9956 9957static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) 9958{ 9959 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9960 zend_call_info *call_info = NULL; 9961 const zend_function *func = NULL; 9962 uint32_t i; 9963 zend_jit_addr res_addr; 9964 uint32_t call_num_args = 0; 9965 bool unknown_num_args = 0; 9966 const void *exit_addr = NULL; 9967 const zend_op *prev_opline; 9968 9969 if (RETURN_VALUE_USED(opline)) { 9970 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 9971 } else { 9972 /* CPU stack allocated temporary zval */ 9973 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, TMP_ZVAL_OFFSET); 9974 } 9975 9976 prev_opline = opline - 1; 9977 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { 9978 prev_opline--; 9979 } 9980 if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || 9981 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 9982 unknown_num_args = 1; 9983 } 9984 9985 if (info) { 9986 call_info = info->callee_info; 9987 while (call_info && call_info->caller_call_opline != opline) { 9988 call_info = call_info->next_callee; 9989 } 9990 if (call_info && call_info->callee_func && !call_info->is_prototype) { 9991 func = call_info->callee_func; 9992 } 9993 if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) 9994 && JIT_G(current_frame) 9995 && JIT_G(current_frame)->call 9996 && !JIT_G(current_frame)->call->func) { 9997 call_info = NULL; func = NULL; /* megamorphic call from trait */ 9998 } 9999 } 10000 if (!func) { 10001 /* resolve function at run time */ 10002 } else if (func->type == ZEND_USER_FUNCTION) { 10003 ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); 10004 call_num_args = call_info->num_args; 10005 } else if (func->type == ZEND_INTERNAL_FUNCTION) { 10006 ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); 10007 call_num_args = call_info->num_args; 10008 } else { 10009 ZEND_UNREACHABLE(); 10010 } 10011 10012 if (trace && !func) { 10013 if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { 10014 ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); 10015#ifndef ZEND_WIN32 10016 // TODO: ASLR may cause different addresses in different workers ??? 10017 func = trace->func; 10018 if (JIT_G(current_frame) && 10019 JIT_G(current_frame)->call && 10020 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 10021 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 10022 } else { 10023 unknown_num_args = 1; 10024 } 10025#endif 10026 } else if (trace->op == ZEND_JIT_TRACE_ENTER) { 10027 ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); 10028 if (zend_accel_in_shm(trace->func->op_array.opcodes)) { 10029 func = trace->func; 10030 if (JIT_G(current_frame) && 10031 JIT_G(current_frame)->call && 10032 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 10033 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 10034 } else { 10035 unknown_num_args = 1; 10036 } 10037 } 10038 } 10039 } 10040 10041 bool may_have_extra_named_params = 10042 opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && 10043 (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); 10044 10045 if (!reuse_ip) { 10046 zend_jit_start_reuse_ip(); 10047 | // call = EX(call); 10048 | mov RX, EX->call 10049 } 10050 zend_jit_stop_reuse_ip(); 10051 10052 | // fbc = call->func; 10053 | // mov r2, EX:RX->func ??? 10054 | // SAVE_OPLINE(); 10055 | SET_EX_OPLINE opline, r0 10056 10057 if (opline->opcode == ZEND_DO_FCALL) { 10058 if (!func) { 10059 if (trace) { 10060 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10061 10062 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10063 if (!exit_addr) { 10064 return 0; 10065 } 10066 | mov r0, EX:RX->func 10067 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10068 | jnz &exit_addr 10069 } 10070 } 10071 } 10072 10073 if (!delayed_call_chain) { 10074 if (call_level == 1) { 10075 | mov aword EX->call, 0 10076 } else { 10077 | //EX(call) = call->prev_execute_data; 10078 | mov r0, EX:RX->prev_execute_data 10079 | mov EX->call, r0 10080 } 10081 } 10082 delayed_call_chain = 0; 10083 10084 | //call->prev_execute_data = execute_data; 10085 | mov EX:RX->prev_execute_data, EX 10086 10087 if (!func) { 10088 | mov r0, EX:RX->func 10089 } 10090 10091 if (opline->opcode == ZEND_DO_FCALL) { 10092 if (!func) { 10093 if (!trace) { 10094 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10095 | jnz >1 10096 |.cold_code 10097 |1: 10098 if (!GCC_GLOBAL_REGS) { 10099 | mov FCARG1a, RX 10100 } 10101 | EXT_CALL zend_jit_deprecated_helper, r0 10102 | test al, al 10103 | mov r0, EX:RX->func // reload 10104 | jne >1 10105 | jmp ->exception_handler 10106 |.code 10107 |1: 10108 } 10109 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 10110 if (!GCC_GLOBAL_REGS) { 10111 | mov FCARG1a, RX 10112 } 10113 | EXT_CALL zend_jit_deprecated_helper, r0 10114 | test al, al 10115 | je ->exception_handler 10116 } 10117 } 10118 10119 if (!func 10120 && opline->opcode != ZEND_DO_UCALL 10121 && opline->opcode != ZEND_DO_ICALL) { 10122 | cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION 10123 | jne >8 10124 } 10125 10126 if ((!func || func->type == ZEND_USER_FUNCTION) 10127 && opline->opcode != ZEND_DO_ICALL) { 10128 | // EX(call) = NULL; 10129 | mov aword EX:RX->call, 0 10130 10131 if (RETURN_VALUE_USED(opline)) { 10132 | // EX(return_value) = EX_VAR(opline->result.var); 10133 | LOAD_ZVAL_ADDR r2, res_addr 10134 | mov aword EX:RX->return_value, r2 10135 } else { 10136 | // EX(return_value) = 0; 10137 | mov aword EX:RX->return_value, 0 10138 } 10139 10140 //EX_LOAD_RUN_TIME_CACHE(op_array); 10141 if (!func || func->op_array.cache_size) { 10142 if (func && op_array == &func->op_array) { 10143 /* recursive call */ 10144 if (trace || func->op_array.cache_size > sizeof(void*)) { 10145 | mov r2, EX->run_time_cache 10146 | mov EX:RX->run_time_cache, r2 10147 } 10148 } else { 10149#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR 10150 if (func) { 10151 | mov r0, EX:RX->func 10152 } 10153 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 10154 | mov r2, aword [r2] 10155#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET 10156 if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { 10157 if (ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { 10158 | MEM_LOAD_ZTS r2, aword, compiler_globals, map_ptr_base, r1 10159 | mov r2, aword [r2 + (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache)] 10160 } else if ((func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 10161 && (!func->op_array.scope || (func->op_array.scope->ce_flags & ZEND_ACC_LINKED))) { 10162 if (func) { 10163 | mov r0, EX:RX->func 10164 } 10165 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 10166 | MEM_LOAD_OP_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 10167 | mov r2, aword [r2] 10168 } else { 10169 /* the called op_array may be not persisted yet */ 10170 if (func) { 10171 | mov r0, EX:RX->func 10172 } 10173 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 10174 | test r2, 1 10175 | jz >1 10176 | MEM_LOAD_OP_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 10177 |1: 10178 | mov r2, aword [r2] 10179 } 10180 } else { 10181 if (func) { 10182 | mov r0, EX:RX->func 10183 } 10184 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 10185 | test r2, 1 10186 | jz >1 10187 | MEM_LOAD_OP_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 10188 |1: 10189 | mov r2, aword [r2] 10190 } 10191#else 10192# error "Unknown ZEND_MAP_PTR_KIND" 10193#endif 10194 | mov EX:RX->run_time_cache, r2 10195 } 10196 } 10197 10198 | // EG(current_execute_data) = execute_data; 10199 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, RX, r1 10200 | mov FP, RX 10201 10202 | // opline = op_array->opcodes; 10203 if (func && !unknown_num_args) { 10204 10205 for (i = call_num_args; i < func->op_array.last_var; i++) { 10206 uint32_t n = EX_NUM_TO_VAR(i); 10207 | SET_Z_TYPE_INFO RX + n, IS_UNDEF 10208 } 10209 10210 if (call_num_args <= func->op_array.num_args) { 10211 if (!trace || (trace->op == ZEND_JIT_TRACE_END 10212 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10213 uint32_t num_args; 10214 10215 if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { 10216 if (trace) { 10217 num_args = 0; 10218 } else if (call_info) { 10219 num_args = skip_valid_arguments(op_array, ssa, call_info); 10220 } else { 10221 num_args = call_num_args; 10222 } 10223 } else { 10224 num_args = call_num_args; 10225 } 10226 if (zend_accel_in_shm(func->op_array.opcodes)) { 10227 | LOAD_IP_ADDR (func->op_array.opcodes + num_args) 10228 } else { 10229 | mov r0, EX->func 10230 if (GCC_GLOBAL_REGS) { 10231 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10232 if (num_args) { 10233 | add IP, (num_args * sizeof(zend_op)) 10234 } 10235 } else { 10236 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10237 if (num_args) { 10238 | add FCARG1a, (num_args * sizeof(zend_op)) 10239 } 10240 | mov aword EX->opline, FCARG1a 10241 } 10242 } 10243 10244 if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array 10245 && num_args >= op_array->required_num_args) { 10246 /* recursive call */ 10247 if (ZEND_OBSERVER_ENABLED) { 10248 | SAVE_IP 10249 | mov FCARG1a, FP 10250 | EXT_CALL zend_observer_fcall_begin, r0 10251 } 10252#ifdef CONTEXT_THREADED_JIT 10253 | call >1 10254 |.cold_code 10255 |1: 10256 | pop r0 10257 | jmp =>num_args 10258 |.code 10259#else 10260 | jmp =>num_args 10261#endif 10262 return 1; 10263 } 10264 } 10265 } else { 10266 if (!trace || (trace->op == ZEND_JIT_TRACE_END 10267 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10268 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 10269 | LOAD_IP_ADDR (func->op_array.opcodes) 10270 } else if (GCC_GLOBAL_REGS) { 10271 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10272 } else { 10273 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10274 | mov aword EX->opline, FCARG1a 10275 } 10276 } 10277 if (!GCC_GLOBAL_REGS) { 10278 | mov FCARG1a, FP 10279 } 10280 | EXT_CALL zend_jit_copy_extra_args_helper, r0 10281 } 10282 } else { 10283 | // opline = op_array->opcodes 10284 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 10285 | LOAD_IP_ADDR (func->op_array.opcodes) 10286 } else if (GCC_GLOBAL_REGS) { 10287 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10288 } else { 10289 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10290 | mov aword EX->opline, FCARG1a 10291 } 10292 if (func) { 10293 | // num_args = EX_NUM_ARGS(); 10294 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] 10295 | // if (UNEXPECTED(num_args > first_extra_arg)) 10296 | cmp ecx, (func->op_array.num_args) 10297 } else { 10298 | // first_extra_arg = op_array->num_args; 10299 | mov edx, dword [r0 + offsetof(zend_op_array, num_args)] 10300 | // num_args = EX_NUM_ARGS(); 10301 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] 10302 | // if (UNEXPECTED(num_args > first_extra_arg)) 10303 | cmp ecx, edx 10304 } 10305 | jg >1 10306 |.cold_code 10307 |1: 10308 if (!GCC_GLOBAL_REGS) { 10309 | mov FCARG1a, FP 10310 } 10311 | EXT_CALL zend_jit_copy_extra_args_helper, r0 10312 if (!func) { 10313 | mov r0, EX->func // reload 10314 } 10315 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload 10316 | jmp >1 10317 |.code 10318 if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { 10319 if (!func) { 10320 | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) 10321 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS 10322 | jnz >1 10323 } 10324 | // opline += num_args; 10325 |.if X64 10326 || ZEND_ASSERT(sizeof(zend_op) == 32); 10327 | mov edx, ecx 10328 | shl r2, 5 10329 |.else 10330 | imul r2, ecx, sizeof(zend_op) 10331 |.endif 10332 | ADD_IP r2 10333 } 10334 |1: 10335 | // if (EXPECTED((int)num_args < op_array->last_var)) { 10336 if (func) { 10337 | mov edx, (func->op_array.last_var) 10338 } else { 10339 | mov edx, dword [r0 + offsetof(zend_op_array, last_var)] 10340 } 10341 | sub edx, ecx 10342 | jle >3 //??? 10343 | // zval *var = EX_VAR_NUM(num_args); 10344// |.if X64 10345// | movsxd r1, ecx 10346// |.endif 10347 | shl r1, 4 10348 | lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))] 10349 |2: 10350 | SET_Z_TYPE_INFO r1, IS_UNDEF 10351 | sub edx, 1 10352 | lea r1, [r1 + 16] 10353 | jne <2 10354 |3: 10355 } 10356 10357 if (ZEND_OBSERVER_ENABLED) { 10358 | SAVE_IP 10359 | mov FCARG1a, FP 10360 | EXT_CALL zend_observer_fcall_begin, r0 10361 } 10362 10363 if (trace) { 10364 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 10365 | jmp >9 10366 } 10367 } else { 10368#ifdef CONTEXT_THREADED_JIT 10369 | call ->context_threaded_call 10370 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 10371 | jmp >9 10372 } 10373 | call ->context_threaded_call 10374 if (!func) { 10375 | jmp >9 10376 } 10377#else 10378 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10379 | ADD_HYBRID_SPAD 10380 | JMP_IP 10381 } else if (GCC_GLOBAL_REGS) { 10382 | add r4, SPAD // stack alignment 10383 | JMP_IP 10384 } else { 10385 | mov FP, aword T2 // restore FP 10386 | mov RX, aword T3 // restore IP 10387 | add r4, NR_SPAD // stack alignment 10388 | mov r0, 1 // ZEND_VM_ENTER 10389 | ret 10390 } 10391 } 10392#endif 10393 } 10394 10395 if ((!func || func->type == ZEND_INTERNAL_FUNCTION) 10396 && (opline->opcode != ZEND_DO_UCALL)) { 10397 if (!func && (opline->opcode != ZEND_DO_ICALL)) { 10398 |8: 10399 } 10400 if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { 10401 if (!func) { 10402 if (trace) { 10403 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10404 10405 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10406 if (!exit_addr) { 10407 return 0; 10408 } 10409 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10410 | jnz &exit_addr 10411 } else { 10412 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10413 | jnz >1 10414 |.cold_code 10415 |1: 10416 if (!GCC_GLOBAL_REGS) { 10417 | mov FCARG1a, RX 10418 } 10419 | EXT_CALL zend_jit_deprecated_helper, r0 10420 | test al, al 10421 | mov r0, EX:RX->func // reload 10422 | jne >1 10423 | jmp ->exception_handler 10424 |.code 10425 |1: 10426 } 10427 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 10428 if (!GCC_GLOBAL_REGS) { 10429 | mov FCARG1a, RX 10430 } 10431 | EXT_CALL zend_jit_deprecated_helper, r0 10432 | test al, al 10433 | je ->exception_handler 10434 | mov r0, EX:RX->func // reload 10435 } 10436 } 10437 10438 | // ZVAL_NULL(EX_VAR(opline->result.var)); 10439 | LOAD_ZVAL_ADDR FCARG2a, res_addr 10440 | SET_Z_TYPE_INFO FCARG2a, IS_NULL 10441 10442 | // EG(current_execute_data) = execute_data; 10443 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, RX, r1 10444 10445 zend_jit_reset_last_valid_opline(); 10446 10447 | // (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret); 10448 if (zend_execute_internal) { 10449 |.if X64 10450 | // CARG2 and FCARG2a are identical 10451 | mov CARG1, RX 10452 |.else 10453 | mov aword A2, FCARG2a 10454 | mov aword A1, RX 10455 |.endif 10456 | EXT_CALL zend_execute_internal, r0 10457 } else { 10458 | mov FCARG1a, RX 10459 if (func) { 10460 | EXT_CALL func->internal_function.handler, r0 10461 } else { 10462 | call aword [r0 + offsetof(zend_internal_function, handler)] 10463 } 10464 } 10465 10466 | // EG(current_execute_data) = execute_data; 10467 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, FP, r0 10468 10469 | // zend_vm_stack_free_args(call); 10470 if (func && !unknown_num_args) { 10471 for (i = 0; i < call_num_args; i++ ) { 10472 if (zend_jit_needs_arg_dtor(func, i, call_info)) { 10473 uint32_t offset = EX_NUM_TO_VAR(i); 10474 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, opline 10475 } 10476 } 10477 } else { 10478 | mov FCARG1a, RX 10479 | EXT_CALL zend_jit_vm_stack_free_args_helper, r0 10480 } 10481 if (may_have_extra_named_params) { 10482 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24) 10483 | jnz >1 10484 |.cold_code 10485 |1: 10486 | mov FCARG1a, aword [RX + offsetof(zend_execute_data, extra_named_params)] 10487 | EXT_CALL zend_free_extra_named_params, r0 10488 | jmp >2 10489 |.code 10490 |2: 10491 } 10492 10493 |8: 10494 if (opline->opcode == ZEND_DO_FCALL) { 10495 // TODO: optimize ??? 10496 | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) 10497 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_RELEASE_THIS >> 16) 10498 | jnz >1 10499 |.cold_code 10500 |1: 10501 | GET_Z_PTR FCARG1a, RX + offsetof(zend_execute_data, This) 10502 | // OBJ_RELEASE(object); 10503 | OBJ_RELEASE ZREG_FCARG1, >2 10504 | jmp >2 10505 |.code 10506 |2: 10507 } 10508 10509 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10510 !JIT_G(current_frame) || 10511 !JIT_G(current_frame)->call || 10512 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || 10513 prev_opline->opcode == ZEND_SEND_UNPACK || 10514 prev_opline->opcode == ZEND_SEND_ARRAY || 10515 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 10516 10517 | // zend_vm_stack_free_call_frame(call); 10518 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16) 10519 | jnz >1 10520 |.cold_code 10521 |1: 10522 | mov FCARG1a, RX 10523 | EXT_CALL zend_jit_free_call_frame, r0 10524 | jmp >1 10525 |.code 10526 } 10527 | MEM_STORE_ZTS aword, executor_globals, vm_stack_top, RX, r0 10528 |1: 10529 10530 if (!RETURN_VALUE_USED(opline)) { 10531 zend_class_entry *ce; 10532 bool ce_is_instanceof; 10533 uint32_t func_info = call_info ? 10534 zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : 10535 (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); 10536 10537 /* If an exception is thrown, the return_value may stay at the 10538 * original value of null. */ 10539 func_info |= MAY_BE_NULL; 10540 10541 if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10542 | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline 10543 } 10544 } 10545 10546 | // if (UNEXPECTED(EG(exception) != NULL)) { 10547 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 10548 | jne ->icall_throw_handler 10549 10550 // TODO: Can we avoid checking for interrupts after each call ??? 10551 if (trace && last_valid_opline != opline) { 10552 int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); 10553 10554 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10555 if (!exit_addr) { 10556 return 0; 10557 } 10558 } else { 10559 exit_addr = NULL; 10560 } 10561 if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { 10562 return 0; 10563 } 10564 10565 if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { 10566 | LOAD_IP_ADDR (opline + 1) 10567 } else if (trace 10568 && trace->op == ZEND_JIT_TRACE_END 10569 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 10570 | LOAD_IP_ADDR (opline + 1) 10571 } 10572 } 10573 10574 if (!func) { 10575 |9: 10576 } 10577 10578 return 1; 10579} 10580 10581static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) 10582{ 10583 uint32_t arg_num = opline->op2.num; 10584 zend_jit_addr arg_addr; 10585 10586 ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); 10587 10588 if (!zend_jit_reuse_ip(Dst)) { 10589 return 0; 10590 } 10591 10592 if (opline->opcode == ZEND_SEND_VAL_EX) { 10593 uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); 10594 10595 ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); 10596 10597 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10598 && JIT_G(current_frame) 10599 && JIT_G(current_frame)->call 10600 && JIT_G(current_frame)->call->func) { 10601 if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10602 /* Don't generate code that always throws exception */ 10603 return 0; 10604 } 10605 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10606 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10607 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10608 if (!exit_addr) { 10609 return 0; 10610 } 10611 | mov r0, EX:RX->func 10612 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10613 | jnz &exit_addr 10614 } else { 10615 | mov r0, EX:RX->func 10616 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10617 | jnz >1 10618 |.cold_code 10619 |1: 10620 if (Z_MODE(op1_addr) == IS_REG) { 10621 /* set type to avoid zval_ptr_dtor() on uninitialized value */ 10622 zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 10623 | SET_ZVAL_TYPE_INFO addr, IS_UNDEF 10624 } 10625 | SET_EX_OPLINE opline, r0 10626 | jmp ->throw_cannot_pass_by_ref 10627 |.code 10628 10629 } 10630 } 10631 10632 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10633 10634 if (opline->op1_type == IS_CONST) { 10635 zval *zv = RT_CONSTANT(opline, opline->op1); 10636 10637 | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 10638 if (Z_REFCOUNTED_P(zv)) { 10639 | ADDREF_CONST zv, r0 10640 } 10641 } else { 10642 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10643 } 10644 10645 return 1; 10646} 10647 10648static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) 10649{ 10650 | mov FCARG1a, EX->call 10651 | test byte [FCARG1a + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_MAY_HAVE_UNDEF >> 24) 10652 | jnz >1 10653 |.cold_code 10654 |1: 10655 | SET_EX_OPLINE opline, r0 10656 | EXT_CALL zend_handle_undef_args, r0 10657 | test r0, r0 10658 | jnz ->exception_handler 10659 | jmp >2 10660 |.code 10661 |2: 10662 10663 return 1; 10664} 10665 10666static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) 10667{ 10668 zend_jit_addr op1_addr, arg_addr, ref_addr; 10669 10670 op1_addr = OP1_ADDR(); 10671 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10672 10673 if (!zend_jit_reuse_ip(Dst)) { 10674 return 0; 10675 } 10676 10677 if (opline->op1_type == IS_VAR) { 10678 if (op1_info & MAY_BE_INDIRECT) { 10679 | LOAD_ZVAL_ADDR r0, op1_addr 10680 | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { 10681 | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 10682 | // ret = Z_INDIRECT_P(ret); 10683 | GET_Z_PTR r0, r0 10684 |1: 10685 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 10686 } 10687 } else if (opline->op1_type == IS_CV) { 10688 if (op1_info & MAY_BE_UNDEF) { 10689 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10690 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 10691 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 10692 | jmp >2 10693 |1: 10694 } 10695 op1_info &= ~MAY_BE_UNDEF; 10696 op1_info |= MAY_BE_NULL; 10697 } 10698 } else { 10699 ZEND_UNREACHABLE(); 10700 } 10701 10702 if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { 10703 if (op1_info & MAY_BE_REF) { 10704 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2 10705 | GET_ZVAL_PTR r1, op1_addr 10706 | GC_ADDREF r1 10707 | SET_ZVAL_PTR arg_addr, r1 10708 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX 10709 | jmp >6 10710 } 10711 |2: 10712 | // ZVAL_NEW_REF(arg, varptr); 10713 if (opline->op1_type == IS_VAR) { 10714 if (Z_REG(op1_addr) != ZREG_R0 || Z_OFFSET(op1_addr) != 0) { 10715 | LOAD_ZVAL_ADDR r0, op1_addr 10716 } 10717 | mov aword T1, r0 // save 10718 } 10719 | EMALLOC sizeof(zend_reference), op_array, opline 10720 | mov dword [r0], 2 10721 | mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], GC_REFERENCE 10722 | mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0 10723 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); 10724 if (opline->op1_type == IS_VAR) { 10725 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); 10726 10727 | mov r1, aword T1 // restore 10728 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R2, ZREG_R2 10729 | SET_ZVAL_PTR val_addr, r0 10730 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX 10731 } else { 10732 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10733 | SET_ZVAL_PTR op1_addr, r0 10734 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 10735 } 10736 | SET_ZVAL_PTR arg_addr, r0 10737 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX 10738 } 10739 10740 |6: 10741 | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline 10742 |7: 10743 10744 return 1; 10745} 10746 10747static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) 10748{ 10749 uint32_t arg_num = opline->op2.num; 10750 zend_jit_addr arg_addr; 10751 10752 ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && 10753 opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || 10754 arg_num <= MAX_ARG_FLAG_NUM); 10755 10756 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10757 10758 if (!zend_jit_reuse_ip(Dst)) { 10759 return 0; 10760 } 10761 10762 if (opline->opcode == ZEND_SEND_VAR_EX) { 10763 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10764 && JIT_G(current_frame) 10765 && JIT_G(current_frame)->call 10766 && JIT_G(current_frame)->call->func) { 10767 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10768 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10769 return 0; 10770 } 10771 return 1; 10772 } 10773 } else { 10774 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10775 10776 | mov r0, EX:RX->func 10777 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10778 | jnz >1 10779 |.cold_code 10780 |1: 10781 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10782 return 0; 10783 } 10784 | jmp >7 10785 |.code 10786 } 10787 } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { 10788 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10789 && JIT_G(current_frame) 10790 && JIT_G(current_frame)->call 10791 && JIT_G(current_frame)->call->func) { 10792 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10793 10794 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10795 10796 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10797 if (!(op1_info & MAY_BE_REF)) { 10798 /* Don't generate code that always throws exception */ 10799 return 0; 10800 } else { 10801 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10802 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10803 if (!exit_addr) { 10804 return 0; 10805 } 10806 | cmp cl, IS_REFERENCE 10807 | jne &exit_addr 10808 } 10809 } 10810 return 1; 10811 } 10812 } else { 10813 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10814 10815 | mov r0, EX:RX->func 10816 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10817 | jnz >1 10818 |.cold_code 10819 |1: 10820 10821 mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); 10822 10823 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10824 if (op1_info & MAY_BE_REF) { 10825 | cmp cl, IS_REFERENCE 10826 | je >7 10827 } 10828 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10829 | jnz >7 10830 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10831 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10832 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10833 if (!exit_addr) { 10834 return 0; 10835 } 10836 | jmp &exit_addr 10837 } else { 10838 | SET_EX_OPLINE opline, r0 10839 | LOAD_ZVAL_ADDR FCARG1a, arg_addr 10840 | EXT_CALL zend_jit_only_vars_by_reference, r0 10841 if (!zend_jit_check_exception(Dst)) { 10842 return 0; 10843 } 10844 | jmp >7 10845 } 10846 10847 |.code 10848 } 10849 } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { 10850 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10851 && JIT_G(current_frame) 10852 && JIT_G(current_frame)->call 10853 && JIT_G(current_frame)->call->func) { 10854 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10855 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10856 return 0; 10857 } 10858 return 1; 10859 } 10860 } else { 10861 | test dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10862 | jnz >1 10863 |.cold_code 10864 |1: 10865 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10866 return 0; 10867 } 10868 | jmp >7 10869 |.code 10870 } 10871 } 10872 10873 if (op1_info & MAY_BE_UNDEF) { 10874 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10875 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 10876 |.cold_code 10877 |1: 10878 } 10879 10880 | SET_EX_OPLINE opline, r0 10881 | mov FCARG1d, opline->op1.var 10882 | EXT_CALL zend_jit_undefined_op_helper, r0 10883 | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL 10884 | test r0, r0 10885 | jz ->exception_handler 10886 10887 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10888 | jmp >7 10889 |.code 10890 } else { 10891 |7: 10892 return 1; 10893 } 10894 } 10895 10896 if (opline->opcode == ZEND_SEND_VAR_NO_REF) { 10897 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10898 if (op1_info & MAY_BE_REF) { 10899 | cmp cl, IS_REFERENCE 10900 | je >7 10901 } 10902 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10903 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10904 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10905 if (!exit_addr) { 10906 return 0; 10907 } 10908 | jmp &exit_addr 10909 } else { 10910 | SET_EX_OPLINE opline, r0 10911 | LOAD_ZVAL_ADDR FCARG1a, arg_addr 10912 | EXT_CALL zend_jit_only_vars_by_reference, r0 10913 if (!zend_jit_check_exception(Dst)) { 10914 return 0; 10915 } 10916 } 10917 } else { 10918 if (op1_info & MAY_BE_REF) { 10919 if (opline->op1_type == IS_CV) { 10920 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 10921 10922 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 10923 | ZVAL_DEREF FCARG1a, op1_info 10924 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R0, ZREG_R2 10925 | TRY_ADDREF op1_info, ah, r2 10926 } else { 10927 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 8); 10928 10929 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 10930 |.cold_code 10931 |1: 10932 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 10933 | GET_ZVAL_PTR FCARG1a, op1_addr 10934 | // ZVAL_COPY_VALUE(return_value, &ref->value); 10935 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R0, ZREG_R2 10936 | GC_DELREF FCARG1a 10937 | je >1 10938 | IF_NOT_REFCOUNTED ah, >2 10939 | GC_ADDREF r2 10940 | jmp >2 10941 |1: 10942 | EFREE_REG_REFERENCE 10943 | jmp >2 10944 |.code 10945 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10946 |2: 10947 } 10948 } else { 10949 if (op1_addr != op1_def_addr) { 10950 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 10951 return 0; 10952 } 10953 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 10954 op1_addr= op1_def_addr; 10955 } 10956 } 10957 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10958 if (opline->op1_type == IS_CV) { 10959 | TRY_ADDREF op1_info, ah, r2 10960 } 10961 } 10962 } 10963 |7: 10964 10965 return 1; 10966} 10967 10968static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) 10969{ 10970 uint32_t arg_num = opline->op2.num; 10971 10972 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10973 && JIT_G(current_frame) 10974 && JIT_G(current_frame)->call 10975 && JIT_G(current_frame)->call->func) { 10976 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10977 if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { 10978 TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); 10979 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10980 || if (reuse_ip) { 10981 | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10982 || } else { 10983 | mov r0, EX->call 10984 | or dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10985 || } 10986 } 10987 } else { 10988 if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 10989 TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); 10990 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 10991 || if (reuse_ip) { 10992 | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10993 || } else { 10994 | mov r0, EX->call 10995 | and dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 10996 || } 10997 } 10998 } 10999 } else { 11000 // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { 11001 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 11002 11003 if (!zend_jit_reuse_ip(Dst)) { 11004 return 0; 11005 } 11006 11007 | mov r0, EX:RX->func 11008 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 11009 | jnz >1 11010 |.cold_code 11011 |1: 11012 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 11013 | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 11014 | jmp >1 11015 |.code 11016 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 11017 | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 11018 |1: 11019 } 11020 11021 return 1; 11022} 11023 11024static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) 11025{ 11026 if (smart_branch_opcode) { 11027 if (smart_branch_opcode == ZEND_JMPZ) { 11028 if (jmp) { 11029 | jmp >7 11030 } 11031 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11032 | jmp =>target_label 11033 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11034 | jmp =>target_label2 11035 } else { 11036 ZEND_UNREACHABLE(); 11037 } 11038 } else { 11039 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11040 11041 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 11042 if (jmp) { 11043 | jmp >7 11044 } 11045 } 11046 11047 return 1; 11048} 11049 11050static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) 11051{ 11052 if (smart_branch_opcode) { 11053 if (smart_branch_opcode == ZEND_JMPZ) { 11054 | jmp =>target_label 11055 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11056 if (jmp) { 11057 | jmp >7 11058 } 11059 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11060 | jmp =>target_label 11061 } else { 11062 ZEND_UNREACHABLE(); 11063 } 11064 } else { 11065 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11066 11067 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 11068 if (jmp) { 11069 | jmp >7 11070 } 11071 } 11072 11073 return 1; 11074} 11075 11076static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 11077{ 11078 uint32_t defined_label = (uint32_t)-1; 11079 uint32_t undefined_label = (uint32_t)-1; 11080 zval *zv = RT_CONSTANT(opline, opline->op1); 11081 zend_jit_addr res_addr = 0; 11082 11083 if (smart_branch_opcode && !exit_addr) { 11084 if (smart_branch_opcode == ZEND_JMPZ) { 11085 undefined_label = target_label; 11086 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11087 defined_label = target_label; 11088 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11089 undefined_label = target_label; 11090 defined_label = target_label2; 11091 } else { 11092 ZEND_UNREACHABLE(); 11093 } 11094 } 11095 11096 | // if (CACHED_PTR(opline->extended_value)) { 11097 | mov r0, EX->run_time_cache 11098 | mov r0, aword [r0 + opline->extended_value] 11099 | test r0, r0 11100 | jz >1 11101 | test r0, 0x1 11102 | jnz >4 11103 |.cold_code 11104 |4: 11105 | MEM_LOAD_ZTS FCARG1a, aword, executor_globals, zend_constants, FCARG1a 11106 | shr r0, 1 11107 | cmp dword [FCARG1a + offsetof(HashTable, nNumOfElements)], eax 11108 11109 if (smart_branch_opcode) { 11110 if (exit_addr) { 11111 if (smart_branch_opcode == ZEND_JMPZ) { 11112 | jz &exit_addr 11113 } else { 11114 | jz >3 11115 } 11116 } else if (undefined_label != (uint32_t)-1) { 11117 | jz =>undefined_label 11118 } else { 11119 | jz >3 11120 } 11121 } else { 11122 | jz >2 11123 } 11124 |1: 11125 | SET_EX_OPLINE opline, r0 11126 | LOAD_ADDR FCARG1a, zv 11127 | EXT_CALL zend_jit_check_constant, r0 11128 | test r0, r0 11129 if (exit_addr) { 11130 if (smart_branch_opcode == ZEND_JMPNZ) { 11131 | jz >3 11132 } else { 11133 | jnz >3 11134 } 11135 | jmp &exit_addr 11136 } else if (smart_branch_opcode) { 11137 if (undefined_label != (uint32_t)-1) { 11138 | jz =>undefined_label 11139 } else { 11140 | jz >3 11141 } 11142 if (defined_label != (uint32_t)-1) { 11143 | jmp =>defined_label 11144 } else { 11145 | jmp >3 11146 } 11147 } else { 11148 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11149 | jnz >1 11150 |2: 11151 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 11152 | jmp >3 11153 } 11154 |.code 11155 if (smart_branch_opcode) { 11156 if (exit_addr) { 11157 if (smart_branch_opcode == ZEND_JMPNZ) { 11158 | jmp &exit_addr 11159 } 11160 } else if (defined_label != (uint32_t)-1) { 11161 | jmp =>defined_label 11162 } 11163 } else { 11164 |1: 11165 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 11166 } 11167 |3: 11168 11169 return 1; 11170} 11171 11172static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 11173{ 11174 uint32_t mask; 11175 zend_jit_addr op1_addr = OP1_ADDR(); 11176 11177 // TODO: support for is_resource() ??? 11178 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); 11179 11180 if (op1_info & MAY_BE_UNDEF) { 11181 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11182 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 11183 |.cold_code 11184 |1: 11185 } 11186 | SET_EX_OPLINE opline, r0 11187 | mov FCARG1d, opline->op1.var 11188 | EXT_CALL zend_jit_undefined_op_helper, r0 11189 zend_jit_check_exception_undef_result(Dst, opline); 11190 if (opline->extended_value & MAY_BE_NULL) { 11191 if (exit_addr) { 11192 if (smart_branch_opcode == ZEND_JMPNZ) { 11193 | jmp &exit_addr 11194 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 11195 | jmp >7 11196 } 11197 } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { 11198 return 0; 11199 } 11200 } else { 11201 if (exit_addr) { 11202 if (smart_branch_opcode == ZEND_JMPZ) { 11203 | jmp &exit_addr 11204 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 11205 | jmp >7 11206 } 11207 } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { 11208 return 0; 11209 } 11210 } 11211 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11212 |.code 11213 } 11214 } 11215 11216 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11217 mask = opline->extended_value; 11218 if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { 11219 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11220 if (exit_addr) { 11221 if (smart_branch_opcode == ZEND_JMPNZ) { 11222 | jmp &exit_addr 11223 } 11224 } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { 11225 return 0; 11226 } 11227 } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { 11228 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11229 if (exit_addr) { 11230 if (smart_branch_opcode == ZEND_JMPZ) { 11231 | jmp &exit_addr 11232 } 11233 } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { 11234 return 0; 11235 } 11236 } else { 11237 bool invert = 0; 11238 zend_uchar type; 11239 11240 switch (mask) { 11241 case MAY_BE_NULL: type = IS_NULL; break; 11242 case MAY_BE_FALSE: type = IS_FALSE; break; 11243 case MAY_BE_TRUE: type = IS_TRUE; break; 11244 case MAY_BE_LONG: type = IS_LONG; break; 11245 case MAY_BE_DOUBLE: type = IS_DOUBLE; break; 11246 case MAY_BE_STRING: type = IS_STRING; break; 11247 case MAY_BE_ARRAY: type = IS_ARRAY; break; 11248 case MAY_BE_OBJECT: type = IS_OBJECT; break; 11249 case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break; 11250 case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break; 11251 case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break; 11252 case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break; 11253 case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break; 11254 case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break; 11255 case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break; 11256 case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break; 11257 case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break; 11258 default: 11259 type = 0; 11260 } 11261 11262 if (op1_info & MAY_BE_REF) { 11263 | LOAD_ZVAL_ADDR r0, op1_addr 11264 | ZVAL_DEREF r0, op1_info 11265 } 11266 if (type == 0) { 11267 if (smart_branch_opcode && 11268 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11269 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11270 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11271 | // if (Z_REFCOUNTED_P(cv)) { 11272 | IF_ZVAL_REFCOUNTED op1_addr, >1 11273 |.cold_code 11274 |1: 11275 } 11276 | // if (!Z_DELREF_P(cv)) { 11277 | GET_ZVAL_PTR FCARG1a, op1_addr 11278 | GC_DELREF FCARG1a 11279 if (RC_MAY_BE_1(op1_info)) { 11280 if (RC_MAY_BE_N(op1_info)) { 11281 | jnz >3 11282 } 11283 if (op1_info & MAY_BE_REF) { 11284 | mov al, byte [r0 + 8] 11285 } else { 11286 | mov al, byte [FP + opline->op1.var + 8] 11287 } 11288 | mov byte T1, al // save 11289 | // zval_dtor_func(r); 11290 | ZVAL_DTOR_FUNC op1_info, opline 11291 | mov cl, byte T1 // restore 11292 |jmp >2 11293 } 11294 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11295 if (!RC_MAY_BE_1(op1_info)) { 11296 | jmp >3 11297 } 11298 |.code 11299 } 11300 |3: 11301 if (op1_info & MAY_BE_REF) { 11302 | mov cl, byte [r0 + 8] 11303 } else { 11304 | mov cl, byte [FP + opline->op1.var + 8] 11305 } 11306 |2: 11307 } else { 11308 if (op1_info & MAY_BE_REF) { 11309 | mov cl, byte [r0 + 8] 11310 } else { 11311 | mov cl, byte [FP + opline->op1.var + 8] 11312 } 11313 } 11314 | mov eax, 1 11315 | shl eax, cl 11316 | test eax, mask 11317 if (exit_addr) { 11318 if (smart_branch_opcode == ZEND_JMPNZ) { 11319 | jne &exit_addr 11320 } else { 11321 | je &exit_addr 11322 } 11323 } else if (smart_branch_opcode) { 11324 if (smart_branch_opcode == ZEND_JMPZ) { 11325 | je =>target_label 11326 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11327 | jne =>target_label 11328 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11329 | je =>target_label 11330 | jmp =>target_label2 11331 } else { 11332 ZEND_UNREACHABLE(); 11333 } 11334 } else { 11335 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11336 11337 | setne al 11338 | movzx eax, al 11339 | add eax, 2 11340 | SET_ZVAL_TYPE_INFO res_addr, eax 11341 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11342 } 11343 } else { 11344 if (smart_branch_opcode && 11345 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11346 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11347 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11348 | // if (Z_REFCOUNTED_P(cv)) { 11349 | IF_ZVAL_REFCOUNTED op1_addr, >1 11350 |.cold_code 11351 |1: 11352 } 11353 | // if (!Z_DELREF_P(cv)) { 11354 | GET_ZVAL_PTR FCARG1a, op1_addr 11355 | GC_DELREF FCARG1a 11356 if (RC_MAY_BE_1(op1_info)) { 11357 if (RC_MAY_BE_N(op1_info)) { 11358 | jnz >3 11359 } 11360 if (op1_info & MAY_BE_REF) { 11361 | mov al, byte [r0 + 8] 11362 } else { 11363 | mov al, byte [FP + opline->op1.var + 8] 11364 } 11365 | mov byte T1, al // save 11366 | // zval_dtor_func(r); 11367 | ZVAL_DTOR_FUNC op1_info, opline 11368 | mov cl, byte T1 // restore 11369 |jmp >2 11370 } 11371 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11372 if (!RC_MAY_BE_1(op1_info)) { 11373 | jmp >3 11374 } 11375 |.code 11376 } 11377 |3: 11378 if (op1_info & MAY_BE_REF) { 11379 | mov cl, byte [r0 + 8] 11380 } else { 11381 | mov cl, byte [FP + opline->op1.var + 8] 11382 } 11383 |2: 11384 | cmp cl, type 11385 } else { 11386 if (op1_info & MAY_BE_REF) { 11387 | cmp byte [r0 + 8], type 11388 } else { 11389 | cmp byte [FP + opline->op1.var + 8], type 11390 } 11391 } 11392 if (exit_addr) { 11393 if (invert) { 11394 if (smart_branch_opcode == ZEND_JMPNZ) { 11395 | jne &exit_addr 11396 } else { 11397 | je &exit_addr 11398 } 11399 } else { 11400 if (smart_branch_opcode == ZEND_JMPNZ) { 11401 | je &exit_addr 11402 } else { 11403 | jne &exit_addr 11404 } 11405 } 11406 } else if (smart_branch_opcode) { 11407 if (invert) { 11408 if (smart_branch_opcode == ZEND_JMPZ) { 11409 | je =>target_label 11410 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11411 | jne =>target_label 11412 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11413 | je =>target_label 11414 | jmp =>target_label2 11415 } else { 11416 ZEND_UNREACHABLE(); 11417 } 11418 } else { 11419 if (smart_branch_opcode == ZEND_JMPZ) { 11420 | jne =>target_label 11421 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11422 | je =>target_label 11423 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11424 | jne =>target_label 11425 | jmp =>target_label2 11426 } else { 11427 ZEND_UNREACHABLE(); 11428 } 11429 } 11430 } else { 11431 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11432 11433 if (invert) { 11434 | setne al 11435 } else { 11436 | sete al 11437 } 11438 | movzx eax, al 11439 | add eax, 2 11440 | SET_ZVAL_TYPE_INFO res_addr, eax 11441 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11442 } 11443 } 11444 } 11445 } 11446 11447 |7: 11448 11449 return 1; 11450} 11451 11452static int zend_jit_leave_frame(dasm_State **Dst) 11453{ 11454 | // EG(current_execute_data) = EX(prev_execute_data); 11455 | mov r0, EX->prev_execute_data 11456 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, r0, r2 11457 return 1; 11458} 11459 11460static int zend_jit_free_cvs(dasm_State **Dst) 11461{ 11462 | // EG(current_execute_data) = EX(prev_execute_data); 11463 | mov FCARG1a, EX->prev_execute_data 11464 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, FCARG1a, r0 11465 | // zend_free_compiled_variables(execute_data); 11466 | mov FCARG1a, FP 11467 | EXT_CALL zend_free_compiled_variables, r0 11468 return 1; 11469} 11470 11471static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) 11472{ 11473 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11474 uint32_t offset = EX_NUM_TO_VAR(var); 11475 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, NULL 11476 } 11477 return 1; 11478} 11479 11480static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) 11481{ 11482 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11483 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, 1, opline 11484 } 11485 return 1; 11486} 11487 11488static int zend_jit_leave_func(dasm_State **Dst, 11489 const zend_op_array *op_array, 11490 const zend_op *opline, 11491 uint32_t op1_info, 11492 bool left_frame, 11493 zend_jit_trace_rec *trace, 11494 zend_jit_trace_info *trace_info, 11495 int indirect_var_access, 11496 int may_throw) 11497{ 11498 bool may_be_top_frame = 11499 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11500 !JIT_G(current_frame) || 11501 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); 11502 bool may_need_call_helper = 11503 indirect_var_access || /* may have symbol table */ 11504 !op_array->function_name || /* may have symbol table */ 11505 may_be_top_frame || 11506 (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ 11507 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11508 !JIT_G(current_frame) || 11509 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ 11510 (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ 11511 bool may_need_release_this = 11512 !(op_array->fn_flags & ZEND_ACC_CLOSURE) && 11513 op_array->scope && 11514 !(op_array->fn_flags & ZEND_ACC_STATIC) && 11515 (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11516 !JIT_G(current_frame) || 11517 !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame))); 11518 11519 if (may_need_release_this) { 11520 | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)] 11521 } 11522 if (may_need_call_helper) { 11523 if (!left_frame) { 11524 left_frame = 1; 11525 if (!zend_jit_leave_frame(Dst)) { 11526 return 0; 11527 } 11528 } 11529 /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ 11530 if (may_need_release_this) { 11531 | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) 11532 } else { 11533 | test dword [FP + offsetof(zend_execute_data, This.u1.type_info)], (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) 11534 } 11535 if (trace && trace->op != ZEND_JIT_TRACE_END) { 11536 | jnz >1 11537 |.cold_code 11538 |1: 11539 if (!GCC_GLOBAL_REGS) { 11540 | mov FCARG1a, FP 11541 } 11542 | EXT_CALL zend_jit_leave_func_helper, r0 11543 11544 if (may_be_top_frame) { 11545 // TODO: try to avoid this check ??? 11546 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 11547#if 0 11548 /* this check should be handled by the following OPLINE guard */ 11549 | cmp IP, zend_jit_halt_op 11550 | je ->trace_halt 11551#endif 11552 } else if (GCC_GLOBAL_REGS) { 11553 | test IP, IP 11554 | je ->trace_halt 11555 } else { 11556 | test eax, eax 11557 | jl ->trace_halt 11558 } 11559 } 11560 11561 if (!GCC_GLOBAL_REGS) { 11562 | // execute_data = EG(current_execute_data) 11563 | MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0 11564 } 11565 | jmp >8 11566 |.code 11567 } else { 11568 | jnz ->leave_function_handler 11569 } 11570 } 11571 11572 if (op_array->fn_flags & ZEND_ACC_CLOSURE) { 11573 if (!left_frame) { 11574 left_frame = 1; 11575 if (!zend_jit_leave_frame(Dst)) { 11576 return 0; 11577 } 11578 } 11579 | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); 11580 | mov FCARG1a, EX->func 11581 | sub FCARG1a, sizeof(zend_object) 11582 | OBJ_RELEASE ZREG_FCARG1, >4 11583 |4: 11584 } else if (may_need_release_this) { 11585 if (!left_frame) { 11586 left_frame = 1; 11587 if (!zend_jit_leave_frame(Dst)) { 11588 return 0; 11589 } 11590 } 11591 | // if (call_info & ZEND_CALL_RELEASE_THIS) 11592 | test FCARG1d, ZEND_CALL_RELEASE_THIS 11593 | je >4 11594 | // zend_object *object = Z_OBJ(execute_data->This); 11595 | mov FCARG1a, EX->This.value.obj 11596 | // OBJ_RELEASE(object); 11597 | OBJ_RELEASE ZREG_FCARG1, >4 11598 |4: 11599 // TODO: avoid EG(excption) check for $this->foo() calls 11600 may_throw = 1; 11601 } 11602 11603 | // EG(vm_stack_top) = (zval*)execute_data; 11604 | MEM_STORE_ZTS aword, executor_globals, vm_stack_top, FP, r0 11605 | // execute_data = EX(prev_execute_data); 11606 | mov FP, EX->prev_execute_data 11607 11608 if (!left_frame) { 11609 | // EG(current_execute_data) = execute_data; 11610 | MEM_STORE_ZTS aword, executor_globals, current_execute_data, FP, r0 11611 } 11612 11613 |9: 11614 if (trace) { 11615 if (trace->op != ZEND_JIT_TRACE_END 11616 && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 11617 zend_jit_reset_last_valid_opline(); 11618 } else { 11619 | LOAD_IP 11620 | ADD_IP sizeof(zend_op) 11621 } 11622 11623 |8: 11624 11625 if (trace->op == ZEND_JIT_TRACE_BACK 11626 && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 11627 const zend_op *next_opline = trace->opline; 11628 11629 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11630 && (op1_info & MAY_BE_RC1) 11631 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { 11632 /* exception might be thrown during destruction of unused return value */ 11633 | // if (EG(exception)) 11634 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 11635 | jne ->leave_throw_handler 11636 } 11637 do { 11638 trace++; 11639 } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); 11640 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 11641 next_opline = trace->opline; 11642 ZEND_ASSERT(next_opline != NULL); 11643 11644 if (trace->op == ZEND_JIT_TRACE_END 11645 && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { 11646 trace_info->flags |= ZEND_JIT_TRACE_LOOP; 11647 | CMP_IP next_opline 11648 | je =>0 // LOOP 11649#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 11650 | JMP_IP 11651#else 11652 | jmp ->trace_escape 11653#endif 11654 } else { 11655 | CMP_IP next_opline 11656 | jne ->trace_escape 11657 } 11658 11659 zend_jit_set_last_valid_opline(trace->opline); 11660 11661 return 1; 11662 } else if (may_throw || 11663 (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11664 && (op1_info & MAY_BE_RC1) 11665 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) 11666 && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { 11667 | // if (EG(exception)) 11668 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 11669 | jne ->leave_throw_handler 11670 } 11671 11672 return 1; 11673 } else { 11674 | // if (EG(exception)) 11675 | MEM_CMP_ZTS aword, executor_globals, exception, 0, r0 11676 | LOAD_IP 11677 | jne ->leave_throw_handler 11678 | // opline = EX(opline) + 1 11679 | ADD_IP sizeof(zend_op) 11680 } 11681 11682 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 11683 | ADD_HYBRID_SPAD 11684#ifdef CONTEXT_THREADED_JIT 11685 | push aword [IP] 11686 | ret 11687#else 11688 | JMP_IP 11689#endif 11690 } else if (GCC_GLOBAL_REGS) { 11691 | add r4, SPAD // stack alignment 11692#ifdef CONTEXT_THREADED_JIT 11693 | push aword [IP] 11694 | ret 11695#else 11696 | JMP_IP 11697#endif 11698 } else { 11699#ifdef CONTEXT_THREADED_JIT 11700 ZEND_UNREACHABLE(); 11701 // TODO: context threading can't work without GLOBAL REGS because we have to change 11702 // the value of execute_data in execute_ex() 11703 | mov FCARG1a, FP 11704 | mov r0, aword [FP] 11705 | mov FP, aword T2 // restore FP 11706 | mov RX, aword T3 // restore IP 11707 | add r4, NR_SPAD // stack alignment 11708 | push aword [r0] 11709 | ret 11710#else 11711 | mov FP, aword T2 // restore FP 11712 | mov RX, aword T3 // restore IP 11713 | add r4, NR_SPAD // stack alignment 11714 | mov r0, 2 // ZEND_VM_LEAVE 11715 | ret 11716#endif 11717 } 11718 11719 return 1; 11720} 11721 11722static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) 11723{ 11724 zend_jit_addr ret_addr; 11725 int8_t return_value_used; 11726 11727 ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); 11728 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); 11729 11730 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { 11731 if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { 11732 return_value_used = 1; 11733 } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { 11734 return_value_used = 0; 11735 } else { 11736 return_value_used = -1; 11737 } 11738 } else { 11739 return_value_used = -1; 11740 } 11741 11742 if (ZEND_OBSERVER_ENABLED) { 11743 if (Z_MODE(op1_addr) == IS_REG) { 11744 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 11745 11746 if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { 11747 return 0; 11748 } 11749 op1_addr = dst; 11750 } 11751 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 11752 | mov FCARG1a, FP 11753 | SET_EX_OPLINE opline, r0 11754 | EXT_CALL zend_observer_fcall_end, r0 11755 } 11756 11757 // if (!EX(return_value)) 11758 if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) { 11759 if (return_value_used != 0) { 11760 | mov r2, EX->return_value 11761 } 11762 if (return_value_used == -1) { 11763 | test r2, r2 11764 } 11765 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); 11766 } else { 11767 if (return_value_used != 0) { 11768 | mov r1, EX->return_value 11769 } 11770 if (return_value_used == -1) { 11771 | test r1, r1 11772 } 11773 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); 11774 } 11775 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11776 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11777 if (return_value_used == -1) { 11778 | jz >1 11779 |.cold_code 11780 |1: 11781 } 11782 if (return_value_used != 1) { 11783 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11784 if (jit_return_label >= 0) { 11785 | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label 11786 } else { 11787 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9 11788 } 11789 } 11790 | GET_ZVAL_PTR FCARG1a, op1_addr 11791 | GC_DELREF FCARG1a 11792 if (RC_MAY_BE_1(op1_info)) { 11793 if (RC_MAY_BE_N(op1_info)) { 11794 if (jit_return_label >= 0) { 11795 | jnz =>jit_return_label 11796 } else { 11797 | jnz >9 11798 } 11799 } 11800 | //SAVE_OPLINE() 11801 | ZVAL_DTOR_FUNC op1_info, opline 11802 | //????mov r1, EX->return_value // reload ??? 11803 } 11804 if (return_value_used == -1) { 11805 if (jit_return_label >= 0) { 11806 | jmp =>jit_return_label 11807 } else { 11808 | jmp >9 11809 } 11810 |.code 11811 } 11812 } 11813 } else if (return_value_used == -1) { 11814 if (jit_return_label >= 0) { 11815 | jz =>jit_return_label 11816 } else { 11817 | jz >9 11818 } 11819 } 11820 11821 if (return_value_used == 0) { 11822 |9: 11823 return 1; 11824 } 11825 11826 if (opline->op1_type == IS_CONST) { 11827 zval *zv = RT_CONSTANT(opline, opline->op1); 11828 | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 11829 if (Z_REFCOUNTED_P(zv)) { 11830 | ADDREF_CONST zv, r0 11831 } 11832 } else if (opline->op1_type == IS_TMP_VAR) { 11833 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11834 } else if (opline->op1_type == IS_CV) { 11835 if (op1_info & MAY_BE_REF) { 11836 | LOAD_ZVAL_ADDR r0, op1_addr 11837 | ZVAL_DEREF r0, op1_info 11838 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 11839 } 11840 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11841 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 11842 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11843 (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || 11844 !op_array->function_name) { 11845 | TRY_ADDREF op1_info, ah, r2 11846 } else if (return_value_used != 1) { 11847 | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); 11848 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 11849 } 11850 } 11851 } else { 11852 if (op1_info & MAY_BE_REF) { 11853 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); 11854 11855 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 11856 |.cold_code 11857 |1: 11858 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 11859 | GET_ZVAL_PTR r0, op1_addr 11860 | // ZVAL_COPY_VALUE(return_value, &ref->value); 11861 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R2, ZREG_R2 11862 | GC_DELREF r0 11863 | je >2 11864 | // if (IS_REFCOUNTED()) 11865 if (jit_return_label >= 0) { 11866 | IF_NOT_REFCOUNTED dh, =>jit_return_label 11867 } else { 11868 | IF_NOT_REFCOUNTED dh, >9 11869 } 11870 | // ADDREF 11871 | GET_ZVAL_PTR r2, ret_addr // reload 11872 | GC_ADDREF r2 11873 if (jit_return_label >= 0) { 11874 | jmp =>jit_return_label 11875 } else { 11876 | jmp >9 11877 } 11878 |2: 11879 | EFREE_REFERENCE r0 11880 if (jit_return_label >= 0) { 11881 | jmp =>jit_return_label 11882 } else { 11883 | jmp >9 11884 } 11885 |.code 11886 } 11887 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11888 } 11889 11890 |9: 11891 return 1; 11892} 11893 11894static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) 11895{ 11896 ZEND_ASSERT(type_reg == ZREG_R2); 11897 11898 |.if not(X64) 11899 || if (Z_REG(val_addr) == ZREG_R1) { 11900 | GET_ZVAL_W2 r0, val_addr 11901 || } 11902 |.endif 11903 | GET_ZVAL_PTR r1, val_addr 11904 |.if not(X64) 11905 || if (Z_REG(val_addr) != ZREG_R1) { 11906 | GET_ZVAL_W2 r0, val_addr 11907 || } 11908 |.endif 11909 | IF_NOT_REFCOUNTED dh, >2 11910 | IF_NOT_TYPE dl, IS_REFERENCE, >1 11911 | GET_Z_TYPE_INFO edx, r1+offsetof(zend_reference, val) 11912 |.if not(X64) 11913 | GET_Z_W2 r0, r1+offsetof(zend_reference, val) 11914 |.endif 11915 | GET_Z_PTR r1, r1+offsetof(zend_reference, val) 11916 | IF_NOT_REFCOUNTED dh, >2 11917 |1: 11918 | GC_ADDREF r1 11919 |2: 11920 | SET_ZVAL_PTR res_addr, r1 11921 |.if not(X64) 11922 | SET_ZVAL_W2 res_addr, r0 11923 |.endif 11924 | SET_ZVAL_TYPE_INFO res_addr, edx 11925 11926 return 1; 11927} 11928 11929static int zend_jit_fetch_dim_read(dasm_State **Dst, 11930 const zend_op *opline, 11931 zend_ssa *ssa, 11932 const zend_ssa_op *ssa_op, 11933 uint32_t op1_info, 11934 zend_jit_addr op1_addr, 11935 bool op1_avoid_refcounting, 11936 uint32_t op2_info, 11937 uint32_t res_info, 11938 zend_jit_addr res_addr, 11939 uint8_t dim_type) 11940{ 11941 zend_jit_addr orig_op1_addr, op2_addr; 11942 const void *exit_addr = NULL; 11943 const void *not_found_exit_addr = NULL; 11944 const void *res_exit_addr = NULL; 11945 bool result_avoid_refcounting = 0; 11946 uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; 11947 int may_throw = 0; 11948 11949 orig_op1_addr = OP1_ADDR(); 11950 op2_addr = OP2_ADDR(); 11951 11952 if (opline->opcode != ZEND_FETCH_DIM_IS 11953 && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 11954 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 11955 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 11956 if (!exit_addr) { 11957 return 0; 11958 } 11959 } 11960 11961 if ((res_info & MAY_BE_GUARD) 11962 && JIT_G(current_frame) 11963 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { 11964 uint32_t flags = 0; 11965 uint32_t old_op1_info = 0; 11966 uint32_t old_info; 11967 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 11968 int32_t exit_point; 11969 11970 if (opline->opcode != ZEND_FETCH_LIST_R 11971 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11972 && !op1_avoid_refcounting) { 11973 flags |= ZEND_JIT_EXIT_FREE_OP1; 11974 } 11975 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) 11976 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11977 flags |= ZEND_JIT_EXIT_FREE_OP2; 11978 } 11979 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 11980 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 11981 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 11982 && (ssa_op+1)->op1_use == ssa_op->result_def 11983 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) 11984 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 11985 result_avoid_refcounting = 1; 11986 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 11987 } 11988 11989 if (op1_avoid_refcounting) { 11990 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 11991 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 11992 } 11993 11994 if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { 11995 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 11996 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 11997 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 11998 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 11999 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 12000 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12001 if (!res_exit_addr) { 12002 return 0; 12003 } 12004 res_info &= ~MAY_BE_GUARD; 12005 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 12006 } 12007 12008 if (opline->opcode == ZEND_FETCH_DIM_IS 12009 && !(res_info & MAY_BE_NULL)) { 12010 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 12011 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); 12012 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); 12013 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 12014 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 12015 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12016 if (!not_found_exit_addr) { 12017 return 0; 12018 } 12019 } 12020 12021 if (op1_avoid_refcounting) { 12022 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 12023 } 12024 } 12025 12026 if (op1_info & MAY_BE_REF) { 12027 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12028 | ZVAL_DEREF FCARG1a, op1_info 12029 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12030 } 12031 12032 if (op1_info & MAY_BE_ARRAY) { 12033 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12034 if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { 12035 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr 12036 } else { 12037 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12038 } 12039 } 12040 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 12041 if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) || 12042 (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) { 12043 may_throw = 1; 12044 } 12045 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, dim_type, res_exit_addr, not_found_exit_addr, exit_addr)) { 12046 return 0; 12047 } 12048 } 12049 12050 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 12051 if (op1_info & MAY_BE_ARRAY) { 12052 |.cold_code 12053 |7: 12054 } 12055 12056 if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { 12057 may_throw = 1; 12058 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { 12059 if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { 12060 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr 12061 } else { 12062 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 12063 } 12064 } 12065 | SET_EX_OPLINE opline, r0 12066 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 12067 if (opline->opcode != ZEND_FETCH_DIM_IS) { 12068 if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { 12069 | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr 12070 | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, r0 12071 } else { 12072 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12073 | EXT_CALL zend_jit_fetch_dim_str_r_helper, r0 12074 } 12075 | SET_ZVAL_PTR res_addr, r0 12076 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING 12077 } else { 12078 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12079 |.if X64 12080 | LOAD_ZVAL_ADDR CARG3, res_addr 12081 |.else 12082 | sub r4, 12 12083 | PUSH_ZVAL_ADDR res_addr, r0 12084 |.endif 12085 | EXT_CALL zend_jit_fetch_dim_str_is_helper, r0 12086 |.if not(X64) 12087 | add r4, 12 12088 |.endif 12089 } 12090 if ((op1_info & MAY_BE_ARRAY) || 12091 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { 12092 | jmp >9 // END 12093 } 12094 |6: 12095 } 12096 12097 if (op1_info & MAY_BE_OBJECT) { 12098 may_throw = 1; 12099 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { 12100 if (exit_addr) { 12101 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 12102 } else { 12103 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6 12104 } 12105 } 12106 | SET_EX_OPLINE opline, r0 12107 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12108 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12109 } 12110 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12111 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12112 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12113 } else { 12114 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12115 } 12116 |.if X64 12117 | LOAD_ZVAL_ADDR CARG3, res_addr 12118 |.else 12119 | sub r4, 12 12120 | PUSH_ZVAL_ADDR res_addr, r0 12121 |.endif 12122 if (opline->opcode != ZEND_FETCH_DIM_IS) { 12123 | EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0 12124 } else { 12125 | EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0 12126 } 12127 |.if not(X64) 12128 | add r4, 12 12129 |.endif 12130 if ((op1_info & MAY_BE_ARRAY) || 12131 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 12132 | jmp >9 // END 12133 } 12134 |6: 12135 } 12136 12137 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) 12138 && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 12139 if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { 12140 | SET_EX_OPLINE opline, r0 12141 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { 12142 may_throw = 1; 12143 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 12144 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 12145 | mov FCARG1d, opline->op1.var 12146 | EXT_CALL zend_jit_undefined_op_helper, r0 12147 |1: 12148 } 12149 12150 if (op2_info & MAY_BE_UNDEF) { 12151 may_throw = 1; 12152 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 12153 | mov FCARG1d, opline->op2.var 12154 | EXT_CALL zend_jit_undefined_op_helper, r0 12155 |1: 12156 } 12157 } 12158 12159 if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { 12160 may_throw = 1; 12161 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { 12162 | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr 12163 } else { 12164 | SET_EX_OPLINE opline, r0 12165 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || 12166 Z_REG(op1_addr) != ZREG_FCARG1 || 12167 Z_OFFSET(op1_addr) != 0) { 12168 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12169 } 12170 } 12171 | EXT_CALL zend_jit_invalid_array_access, r0 12172 } 12173 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 12174 if (op1_info & MAY_BE_ARRAY) { 12175 | jmp >9 // END 12176 } 12177 } 12178 12179 if (op1_info & MAY_BE_ARRAY) { 12180 |.code 12181 } 12182 } 12183 12184 if (op1_info & MAY_BE_ARRAY) { 12185 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 12186 12187 |8: 12188 if (res_exit_addr) { 12189 zend_uchar type = concrete_type(res_info); 12190 12191 if ((op1_info & MAY_BE_ARRAY_OF_REF) 12192 && dim_type != IS_UNKNOWN 12193 && dim_type != IS_REFERENCE) { 12194 if (type < IS_STRING) { 12195 | IF_NOT_ZVAL_TYPE val_addr, type, >1 12196 |.cold_code 12197 |1: 12198 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, &res_exit_addr 12199 | GET_Z_PTR r0, r0 12200 | add r0, offsetof(zend_reference, val) 12201 | IF_ZVAL_TYPE val_addr, type, >1 12202 | jmp &res_exit_addr 12203 |.code 12204 |1: 12205 } else { 12206 | GET_ZVAL_TYPE_INFO edx, val_addr 12207 | IF_NOT_TYPE dl, type, >1 12208 |.cold_code 12209 |1: 12210 | IF_NOT_TYPE dl, IS_REFERENCE, &res_exit_addr 12211 | GET_Z_PTR r0, r0 12212 | add r0, offsetof(zend_reference, val) 12213 | GET_ZVAL_TYPE_INFO edx, val_addr 12214 | IF_TYPE dl, type, >1 12215 | jmp &res_exit_addr 12216 |.code 12217 |1: 12218 } 12219 } else { 12220 if (op1_info & MAY_BE_ARRAY_OF_REF) { 12221 | ZVAL_DEREF r0, MAY_BE_REF 12222 } 12223 if (type < IS_STRING) { 12224 | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr 12225 } else { 12226 | GET_ZVAL_TYPE_INFO edx, val_addr 12227 | IF_NOT_TYPE dl, type, &res_exit_addr 12228 } 12229 } 12230 12231 | // ZVAL_COPY 12232 |7: 12233 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 12234 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 12235 if (type < IS_STRING) { 12236 if (Z_REG(res_addr) != ZREG_FP || 12237 JIT_G(current_frame) == NULL || 12238 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 12239 | SET_ZVAL_TYPE_INFO res_addr, type 12240 } 12241 } else { 12242 | SET_ZVAL_TYPE_INFO res_addr, edx 12243 if (!result_avoid_refcounting) { 12244 | TRY_ADDREF res_info, dh, r1 12245 } 12246 } 12247 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 12248 return 0; 12249 } 12250 } else if (op1_info & MAY_BE_ARRAY_OF_REF) { 12251 | // ZVAL_COPY_DEREF 12252 | GET_ZVAL_TYPE_INFO Rd(ZREG_R2), val_addr 12253 if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_R2)) { 12254 return 0; 12255 } 12256 } else { 12257 | // ZVAL_COPY 12258 | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_R1, ZREG_R2 12259 | TRY_ADDREF res_info, ch, r2 12260 } 12261 } 12262 |9: // END 12263 12264#ifdef ZEND_JIT_USE_RC_INFERENCE 12265 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 12266 /* Magic offsetGet() may increase refcount of the key */ 12267 op2_info |= MAY_BE_RCN; 12268 } 12269#endif 12270 12271 if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) { 12272 if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) { 12273 may_throw = 1; 12274 } 12275 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12276 } 12277 if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { 12278 if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) { 12279 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 12280 may_throw = 1; 12281 } 12282 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12283 } 12284 } 12285 12286 if (may_throw) { 12287 if (!zend_jit_check_exception(Dst)) { 12288 return 0; 12289 } 12290 } 12291 12292 return 1; 12293} 12294 12295static int zend_jit_fetch_dim(dasm_State **Dst, 12296 const zend_op *opline, 12297 uint32_t op1_info, 12298 zend_jit_addr op1_addr, 12299 uint32_t op2_info, 12300 zend_jit_addr res_addr, 12301 uint8_t dim_type) 12302{ 12303 zend_jit_addr op2_addr; 12304 int may_throw = 0; 12305 12306 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 12307 12308 if (opline->opcode == ZEND_FETCH_DIM_RW) { 12309 | SET_EX_OPLINE opline, r0 12310 } 12311 if (op1_info & MAY_BE_REF) { 12312 may_throw = 1; 12313 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12314 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 12315 | GET_Z_PTR FCARG2a, FCARG1a 12316 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 12317 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 12318 | jmp >3 12319 |.cold_code 12320 |2: 12321 if (opline->opcode != ZEND_FETCH_DIM_RW) { 12322 | SET_EX_OPLINE opline, r0 12323 } 12324 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 12325 | test r0, r0 12326 | mov FCARG1a, r0 12327 | jne >1 12328 | jmp ->exception_handler_undef 12329 |.code 12330 |1: 12331 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12332 } 12333 12334 if (op1_info & MAY_BE_ARRAY) { 12335 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12336 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12337 } 12338 |3: 12339 | SEPARATE_ARRAY op1_addr, op1_info, 1 12340 } 12341 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { 12342 if (op1_info & MAY_BE_ARRAY) { 12343 |.cold_code 12344 |7: 12345 } 12346 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 12347 | CMP_ZVAL_TYPE op1_addr, IS_NULL 12348 | jg >7 12349 } 12350 if (Z_REG(op1_addr) != ZREG_FP) { 12351 | mov T1, Ra(Z_REG(op1_addr)) // save 12352 } 12353 if ((op1_info & MAY_BE_UNDEF) 12354 && opline->opcode == ZEND_FETCH_DIM_RW) { 12355 may_throw = 1; 12356 if (op1_info & MAY_BE_NULL) { 12357 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 12358 } 12359 | mov FCARG1a, opline->op1.var 12360 | EXT_CALL zend_jit_undefined_op_helper, r0 12361 |1: 12362 } 12363 | // ZVAL_ARR(container, zend_new_array(8)); 12364 | EXT_CALL _zend_new_array_0, r0 12365 if (Z_REG(op1_addr) != ZREG_FP) { 12366 | mov Ra(Z_REG(op1_addr)), T1 // restore 12367 } 12368 | SET_ZVAL_LVAL op1_addr, r0 12369 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 12370 | mov FCARG1a, r0 12371 if (op1_info & MAY_BE_ARRAY) { 12372 | jmp >1 12373 |.code 12374 |1: 12375 } 12376 } 12377 12378 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 12379 |6: 12380 if (opline->op2_type == IS_UNUSED) { 12381 may_throw = 1; 12382 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 12383 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 12384 | EXT_CALL zend_hash_next_index_insert, r0 12385 | // if (UNEXPECTED(!var_ptr)) { 12386 | test r0, r0 12387 | jz >1 12388 |.cold_code 12389 |1: 12390 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 12391 | CANNOT_ADD_ELEMENT opline 12392 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF 12393 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 12394 | jmp >8 12395 |.code 12396 | SET_ZVAL_PTR res_addr, r0 12397 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 12398 } else { 12399 uint32_t type; 12400 12401 switch (opline->opcode) { 12402 case ZEND_FETCH_DIM_W: 12403 case ZEND_FETCH_LIST_W: 12404 type = BP_VAR_W; 12405 break; 12406 case ZEND_FETCH_DIM_RW: 12407 may_throw = 1; 12408 type = BP_VAR_RW; 12409 break; 12410 case ZEND_FETCH_DIM_UNSET: 12411 type = BP_VAR_UNSET; 12412 break; 12413 default: 12414 ZEND_UNREACHABLE(); 12415 } 12416 12417 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 12418 may_throw = 1; 12419 } 12420 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { 12421 return 0; 12422 } 12423 12424 |8: 12425 | SET_ZVAL_PTR res_addr, r0 12426 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 12427 12428 if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 12429 |.cold_code 12430 |9: 12431 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 12432 | jmp >8 12433 |.code 12434 } 12435 } 12436 } 12437 12438 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { 12439 may_throw = 1; 12440 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 12441 |.cold_code 12442 |7: 12443 } 12444 12445 if (opline->opcode != ZEND_FETCH_DIM_RW) { 12446 | SET_EX_OPLINE opline, r0 12447 } 12448 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12449 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12450 } 12451 if (opline->op2_type == IS_UNUSED) { 12452 | xor FCARG2a, FCARG2a 12453 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12454 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12455 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12456 } else { 12457 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12458 } 12459 |.if X64 12460 | LOAD_ZVAL_ADDR CARG3, res_addr 12461 |.else 12462 | sub r4, 12 12463 | PUSH_ZVAL_ADDR res_addr, r0 12464 |.endif 12465 switch (opline->opcode) { 12466 case ZEND_FETCH_DIM_W: 12467 case ZEND_FETCH_LIST_W: 12468 | EXT_CALL zend_jit_fetch_dim_obj_w_helper, r0 12469 break; 12470 case ZEND_FETCH_DIM_RW: 12471 | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, r0 12472 break; 12473// case ZEND_FETCH_DIM_UNSET: 12474// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0 12475// break; 12476 default: 12477 ZEND_UNREACHABLE(); 12478 } 12479 |.if not(X64) 12480 | add r4, 12 12481 |.endif 12482 12483 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { 12484 | jmp >8 // END 12485 |.code 12486 } 12487 } 12488 12489#ifdef ZEND_JIT_USE_RC_INFERENCE 12490 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 12491 /* ASSIGN_DIM may increase refcount of the key */ 12492 op2_info |= MAY_BE_RCN; 12493 } 12494#endif 12495 12496 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 12497 && (op2_info & MAY_HAVE_DTOR) 12498 && (op2_info & MAY_BE_RC1)) { 12499 may_throw = 1; 12500 } 12501 |8: 12502 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12503 12504 if (may_throw) { 12505 if (!zend_jit_check_exception(Dst)) { 12506 return 0; 12507 } 12508 } 12509 12510 return 1; 12511} 12512 12513static int zend_jit_isset_isempty_dim(dasm_State **Dst, 12514 const zend_op *opline, 12515 uint32_t op1_info, 12516 zend_jit_addr op1_addr, 12517 bool op1_avoid_refcounting, 12518 uint32_t op2_info, 12519 uint8_t dim_type, 12520 int may_throw, 12521 zend_uchar smart_branch_opcode, 12522 uint32_t target_label, 12523 uint32_t target_label2, 12524 const void *exit_addr) 12525{ 12526 zend_jit_addr op2_addr, res_addr; 12527 12528 // TODO: support for empty() ??? 12529 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 12530 12531 op2_addr = OP2_ADDR(); 12532 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12533 12534 if (op1_info & MAY_BE_REF) { 12535 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12536 | ZVAL_DEREF FCARG1a, op1_info 12537 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 12538 } 12539 12540 if (op1_info & MAY_BE_ARRAY) { 12541 const void *found_exit_addr = NULL; 12542 const void *not_found_exit_addr = NULL; 12543 12544 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12545 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12546 } 12547 | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr 12548 if (exit_addr 12549 && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) 12550 && !may_throw 12551 && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) 12552 && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { 12553 if (smart_branch_opcode == ZEND_JMPNZ) { 12554 found_exit_addr = exit_addr; 12555 } else { 12556 not_found_exit_addr = exit_addr; 12557 } 12558 } 12559 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, dim_type, found_exit_addr, not_found_exit_addr, NULL)) { 12560 return 0; 12561 } 12562 12563 if (found_exit_addr) { 12564 |9: 12565 return 1; 12566 } else if (not_found_exit_addr) { 12567 |8: 12568 return 1; 12569 } 12570 } 12571 12572 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 12573 if (op1_info & MAY_BE_ARRAY) { 12574 |.cold_code 12575 |7: 12576 } 12577 12578 if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { 12579 | SET_EX_OPLINE opline, r0 12580 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 12581 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12582 } 12583 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12584 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12585 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12586 } else { 12587 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12588 } 12589 | EXT_CALL zend_jit_isset_dim_helper, r0 12590 | test r0, r0 12591 | jz >9 12592 if (op1_info & MAY_BE_ARRAY) { 12593 | jmp >8 12594 |.code 12595 } 12596 } else { 12597 if (op2_info & MAY_BE_UNDEF) { 12598 if (op2_info & MAY_BE_ANY) { 12599 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 12600 } 12601 | SET_EX_OPLINE opline, r0 12602 | mov FCARG1d, opline->op2.var 12603 | EXT_CALL zend_jit_undefined_op_helper, r0 12604 |1: 12605 } 12606 if (op1_info & MAY_BE_ARRAY) { 12607 | jmp >9 12608 |.code 12609 } 12610 } 12611 } 12612 12613#ifdef ZEND_JIT_USE_RC_INFERENCE 12614 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 12615 /* Magic offsetExists() may increase refcount of the key */ 12616 op2_info |= MAY_BE_RCN; 12617 } 12618#endif 12619 12620 if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { 12621 |8: 12622 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12623 if (!op1_avoid_refcounting) { 12624 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12625 } 12626 if (may_throw) { 12627 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 12628 return 0; 12629 } 12630 } 12631 if (!(opline->extended_value & ZEND_ISEMPTY)) { 12632 if (exit_addr) { 12633 if (smart_branch_opcode == ZEND_JMPNZ) { 12634 | jmp &exit_addr 12635 } else { 12636 | jmp >8 12637 } 12638 } else if (smart_branch_opcode) { 12639 if (smart_branch_opcode == ZEND_JMPZ) { 12640 | jmp =>target_label2 12641 } else if (smart_branch_opcode == ZEND_JMPNZ) { 12642 | jmp =>target_label 12643 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 12644 | jmp =>target_label2 12645 } else { 12646 ZEND_UNREACHABLE(); 12647 } 12648 } else { 12649 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 12650 | jmp >8 12651 } 12652 } else { 12653 | NIY // TODO: support for empty() 12654 } 12655 } 12656 12657 |9: // not found 12658 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12659 if (!op1_avoid_refcounting) { 12660 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12661 } 12662 if (may_throw) { 12663 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 12664 return 0; 12665 } 12666 } 12667 if (!(opline->extended_value & ZEND_ISEMPTY)) { 12668 if (exit_addr) { 12669 if (smart_branch_opcode == ZEND_JMPZ) { 12670 | jmp &exit_addr 12671 } 12672 } else if (smart_branch_opcode) { 12673 if (smart_branch_opcode == ZEND_JMPZ) { 12674 | jmp =>target_label 12675 } else if (smart_branch_opcode == ZEND_JMPNZ) { 12676 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 12677 | jmp =>target_label 12678 } else { 12679 ZEND_UNREACHABLE(); 12680 } 12681 } else { 12682 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 12683 } 12684 } else { 12685 | NIY // TODO: support for empty() 12686 } 12687 12688 |8: 12689 12690 return 1; 12691} 12692 12693static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 12694{ 12695 zend_jit_addr op1_addr = OP1_ADDR(); 12696 zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); 12697 12698 | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; 12699 | mov FCARG2a, EX->run_time_cache 12700 | mov r0, aword [FCARG2a + opline->extended_value] 12701 | sub r0, 1 12702 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) 12703 | MEM_LOAD_ZTS ecx, dword, executor_globals, symbol_table.nNumUsed, r1 12704 |.if X64 12705 | shl r1, 5 12706 |.else 12707 | imul r1, sizeof(Bucket) 12708 |.endif 12709 | cmp r0, r1 12710 | jae >9 12711 | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); 12712 | MEM_LOAD_OP_ZTS add, r0, aword, executor_globals, symbol_table.arData, r1 12713 | IF_NOT_Z_TYPE r0, IS_REFERENCE, >9 12714 | // (EXPECTED(p->key == varname)) 12715 | ADDR_CMP aword [r0 + offsetof(Bucket, key)], varname, r1 12716 | jne >9 12717 | GET_Z_PTR r0, r0 12718 | GC_ADDREF r0 12719 |1: 12720 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 12721 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12722 | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) 12723 | IF_ZVAL_REFCOUNTED op1_addr, >2 12724 |.cold_code 12725 |2: 12726 } 12727 | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); 12728 | GET_ZVAL_PTR FCARG1a, op1_addr 12729 | // ZVAL_REF(variable_ptr, ref) 12730 | SET_ZVAL_PTR op1_addr, r0 12731 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 12732 | // if (GC_DELREF(garbage) == 0) 12733 | GC_DELREF FCARG1a 12734 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 12735 | jnz >3 12736 } else { 12737 | jnz >5 12738 } 12739 | ZVAL_DTOR_FUNC op1_info, opline 12740 | jmp >5 12741 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 12742 |3: 12743 | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) 12744 | IF_GC_MAY_NOT_LEAK FCARG1a, >5 12745 | EXT_CALL gc_possible_root, r1 12746 | jmp >5 12747 } 12748 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12749 |.code 12750 } 12751 } 12752 12753 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12754 | // ZVAL_REF(variable_ptr, ref) 12755 | SET_ZVAL_PTR op1_addr, r0 12756 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 12757 } 12758 |5: 12759 //END of handler 12760 12761 |.cold_code 12762 |9: 12763 | LOAD_ADDR FCARG1a, (ptrdiff_t)varname 12764 if (opline->extended_value) { 12765 | add FCARG2a, opline->extended_value 12766 } 12767 | EXT_CALL zend_jit_fetch_global_helper, r0 12768 | jmp <1 12769 |.code 12770 12771 return 1; 12772} 12773 12774static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception) 12775{ 12776 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12777 bool in_cold = 0; 12778 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 12779 zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1 : ZREG_R0; 12780 12781 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12782 && JIT_G(current_frame) 12783 && JIT_G(current_frame)->prev) { 12784 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 12785 uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)); 12786 12787 if (type != IS_UNKNOWN && (type_mask & (1u << type))) { 12788 return 1; 12789 } 12790 } 12791 12792 if (ZEND_ARG_SEND_MODE(arg_info)) { 12793 if (opline->opcode == ZEND_RECV_INIT) { 12794 | LOAD_ZVAL_ADDR Ra(tmp_reg), res_addr 12795 | ZVAL_DEREF Ra(tmp_reg), MAY_BE_REF 12796 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); 12797 } else { 12798 | GET_ZVAL_PTR Ra(tmp_reg), res_addr 12799 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); 12800 } 12801 } 12802 12803 if (type_mask != 0) { 12804 if (is_power_of_two(type_mask)) { 12805 uint32_t type_code = concrete_type(type_mask); 12806 | IF_NOT_ZVAL_TYPE res_addr, type_code, >1 12807 } else { 12808 | mov edx, 1 12809 | mov cl, byte [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)+offsetof(zval, u1.v.type)] 12810 | shl edx, cl 12811 | test edx, type_mask 12812 | je >1 12813 } 12814 12815 |.cold_code 12816 |1: 12817 12818 in_cold = 1; 12819 } 12820 12821 if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) { 12822 | LOAD_ZVAL_ADDR FCARG1a, res_addr 12823 } 12824 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12825 | SET_EX_OPLINE opline, r0 12826 } else { 12827 | ADDR_STORE aword EX->opline, opline, r0 12828 } 12829 | LOAD_ADDR FCARG2a, (ptrdiff_t)arg_info 12830 | EXT_CALL zend_jit_verify_arg_slow, r0 12831 12832 if (check_exception) { 12833 | test al, al 12834 if (in_cold) { 12835 | jnz >1 12836 | jmp ->exception_handler 12837 |.code 12838 |1: 12839 } else { 12840 | jz ->exception_handler 12841 } 12842 } else if (in_cold) { 12843 | jmp >1 12844 |.code 12845 |1: 12846 } 12847 12848 return 1; 12849} 12850 12851static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) 12852{ 12853 uint32_t arg_num = opline->op1.num; 12854 zend_arg_info *arg_info = NULL; 12855 12856 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12857 if (EXPECTED(arg_num <= op_array->num_args)) { 12858 arg_info = &op_array->arg_info[arg_num-1]; 12859 } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { 12860 arg_info = &op_array->arg_info[op_array->num_args]; 12861 } 12862 if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { 12863 arg_info = NULL; 12864 } 12865 } 12866 12867 if (arg_info || (opline+1)->opcode != ZEND_RECV) { 12868 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12869 if (!JIT_G(current_frame) || 12870 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 || 12871 arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 12872 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12873 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12874 12875 if (!exit_addr) { 12876 return 0; 12877 } 12878 | cmp dword EX->This.u2.num_args, arg_num 12879 | jb &exit_addr 12880 } 12881 } else { 12882 | cmp dword EX->This.u2.num_args, arg_num 12883 | jb >1 12884 |.cold_code 12885 |1: 12886 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12887 | SET_EX_OPLINE opline, r0 12888 } else { 12889 | ADDR_STORE aword EX->opline, opline, r0 12890 } 12891 | mov FCARG1a, FP 12892 | EXT_CALL zend_missing_arg_error, r0 12893 | jmp ->exception_handler 12894 |.code 12895 } 12896 } 12897 12898 if (arg_info) { 12899 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { 12900 return 0; 12901 } 12902 } 12903 12904 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12905 if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { 12906 | LOAD_IP_ADDR (opline + 1) 12907 zend_jit_set_last_valid_opline(opline + 1); 12908 } 12909 } 12910 12911 return 1; 12912} 12913 12914static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw) 12915{ 12916 uint32_t arg_num = opline->op1.num; 12917 zval *zv = RT_CONSTANT(opline, opline->op2); 12918 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12919 12920 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 12921 && JIT_G(current_frame) 12922 && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) { 12923 if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { 12924 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0 12925 if (Z_REFCOUNTED_P(zv)) { 12926 | ADDREF_CONST zv, r0 12927 } 12928 } 12929 } else { 12930 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 12931 (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 12932 | cmp dword EX->This.u2.num_args, arg_num 12933 | jae >5 12934 } 12935 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0 12936 if (Z_REFCOUNTED_P(zv)) { 12937 | ADDREF_CONST zv, r0 12938 } 12939 } 12940 12941 if (Z_CONSTANT_P(zv)) { 12942 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12943 | SET_EX_OPLINE opline, r0 12944 } else { 12945 | ADDR_STORE aword EX->opline, opline, r0 12946 } 12947 | LOAD_ZVAL_ADDR FCARG1a, res_addr 12948 | mov r0, EX->func 12949 | mov FCARG2a, [r0 + offsetof(zend_op_array, scope)] 12950 | .if X64 12951 | EXT_CALL zval_update_constant_ex, r0 12952 | .else 12953 ||#if (PHP_VERSION_ID < 80100) && (SIZEOF_SIZE_T == 4) 12954 | EXT_CALL zval_jit_update_constant_ex, r0 12955 ||#else 12956 | EXT_CALL zval_update_constant_ex, r0 12957 ||#endif 12958 | .endif 12959 | test al, al 12960 | jnz >1 12961 |.cold_code 12962 |1: 12963 | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline 12964 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF 12965 | jmp ->exception_handler 12966 |.code 12967 } 12968 12969 |5: 12970 12971 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12972 do { 12973 zend_arg_info *arg_info; 12974 12975 if (arg_num <= op_array->num_args) { 12976 arg_info = &op_array->arg_info[arg_num-1]; 12977 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { 12978 arg_info = &op_array->arg_info[op_array->num_args]; 12979 } else { 12980 break; 12981 } 12982 if (!ZEND_TYPE_IS_SET(arg_info->type)) { 12983 break; 12984 } 12985 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { 12986 return 0; 12987 } 12988 } while (0); 12989 } 12990 12991 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12992 if (is_last) { 12993 | LOAD_IP_ADDR (opline + 1) 12994 zend_jit_set_last_valid_opline(opline + 1); 12995 } 12996 } 12997 12998 return 1; 12999} 13000 13001static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) 13002{ 13003 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 13004 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13005 13006 if (!exit_addr) { 13007 return 0; 13008 } 13009 13010 |.if X64 13011 || if (!IS_SIGNED_32BIT(ce)) { 13012 | mov64 r0, ((ptrdiff_t)ce) 13013 | cmp aword [FCARG1a + offsetof(zend_object, ce)], r0 13014 || } else { 13015 | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce 13016 || } 13017 |.else 13018 | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce 13019 |.endif 13020 | jne &exit_addr 13021 13022 return 1; 13023} 13024 13025static int zend_jit_fetch_obj(dasm_State **Dst, 13026 const zend_op *opline, 13027 const zend_op_array *op_array, 13028 zend_ssa *ssa, 13029 const zend_ssa_op *ssa_op, 13030 uint32_t op1_info, 13031 zend_jit_addr op1_addr, 13032 bool op1_indirect, 13033 zend_class_entry *ce, 13034 bool ce_is_instanceof, 13035 bool on_this, 13036 bool delayed_fetch_this, 13037 bool op1_avoid_refcounting, 13038 zend_class_entry *trace_ce, 13039 uint8_t prop_type, 13040 int may_throw) 13041{ 13042 zval *member; 13043 zend_property_info *prop_info; 13044 bool may_be_dynamic = 1; 13045 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 13046 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13047 zend_jit_addr prop_addr; 13048 uint32_t res_info = RES_INFO(); 13049 bool type_loaded = 0; 13050 13051 ZEND_ASSERT(opline->op2_type == IS_CONST); 13052 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13053 13054 member = RT_CONSTANT(opline, opline->op2); 13055 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13056 prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); 13057 13058 if (on_this) { 13059 | GET_ZVAL_PTR FCARG1a, this_addr 13060 } else { 13061 if (opline->op1_type == IS_VAR 13062 && opline->opcode == ZEND_FETCH_OBJ_W 13063 && (op1_info & MAY_BE_INDIRECT) 13064 && Z_REG(op1_addr) == ZREG_FP) { 13065 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13066 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13067 | GET_Z_PTR FCARG1a, FCARG1a 13068 |1: 13069 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13070 } 13071 if (op1_info & MAY_BE_REF) { 13072 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13073 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13074 } 13075 | ZVAL_DEREF FCARG1a, op1_info 13076 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13077 } 13078 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13079 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13080 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13081 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13082 13083 if (!exit_addr) { 13084 return 0; 13085 } 13086 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13087 } else { 13088 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7 13089 } 13090 } 13091 | GET_ZVAL_PTR FCARG1a, op1_addr 13092 } 13093 13094 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13095 prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename); 13096 if (prop_info) { 13097 ce = trace_ce; 13098 ce_is_instanceof = 0; 13099 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13100 if (on_this && JIT_G(current_frame) 13101 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13102 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13103 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13104 if (on_this && JIT_G(current_frame)) { 13105 JIT_G(current_frame)->ce = ce; 13106 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13107 } 13108 } else { 13109 return 0; 13110 } 13111 if (ssa->var_info && ssa_op->op1_use >= 0) { 13112 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13113 ssa->var_info[ssa_op->op1_use].ce = ce; 13114 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13115 } 13116 } 13117 } 13118 } 13119 13120 if (!prop_info) { 13121 | mov r0, EX->run_time_cache 13122 | mov r2, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)] 13123 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13124 | jne >5 13125 | mov r0, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)] 13126 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); 13127 if (may_be_dynamic) { 13128 | test r0, r0 13129 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13130 | jl >5 13131 } else { 13132 | jl >8 // dynamic property 13133 } 13134 } 13135 | mov edx, dword [FCARG1a + r0 + 8] 13136 | IF_UNDEF dl, >5 13137 | add FCARG1a, r0 13138 type_loaded = 1; 13139 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13140 if (opline->opcode == ZEND_FETCH_OBJ_W 13141 && (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) { 13142 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 13143 13144 | mov r0, EX->run_time_cache 13145 | mov FCARG2a, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2] 13146 | test FCARG2a, FCARG2a 13147 | jnz >1 13148 |.cold_code 13149 |1: 13150 | test dword [FCARG2a + offsetof(zend_property_info, flags)], ZEND_ACC_READONLY 13151 if (flags) { 13152 | jz >3 13153 } else { 13154 | jz >4 13155 } 13156 | IF_NOT_Z_TYPE FCARG1a, IS_OBJECT, >2 13157 | GET_Z_PTR r0, FCARG1a 13158 | GC_ADDREF r0 13159 | SET_ZVAL_PTR res_addr, r0 13160 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX 13161 | jmp >9 13162 |2: 13163 | mov FCARG1a, FCARG2a 13164 | SET_EX_OPLINE opline, r0 13165 | EXT_CALL zend_readonly_property_modification_error, r0 13166 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 13167 | jmp >9 13168 |3: 13169 if (flags == ZEND_FETCH_DIM_WRITE) { 13170 | SET_EX_OPLINE opline, r0 13171 | EXT_CALL zend_jit_check_array_promotion, r0 13172 | jmp >9 13173 } else if (flags == ZEND_FETCH_REF) { 13174 |.if X64 13175 | LOAD_ZVAL_ADDR CARG3, res_addr 13176 |.else 13177 | sub r4, 12 13178 | PUSH_ZVAL_ADDR res_addr, r0 13179 |.endif 13180 | EXT_CALL zend_jit_create_typed_ref, r0 13181 |.if not(X64) 13182 | add r4, 12 13183 |.endif 13184 | jmp >9 13185 } else { 13186 ZEND_ASSERT(flags == 0); 13187 } 13188 |.code 13189 |4: 13190 } 13191 } else { 13192 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13193 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13194 if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { 13195 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 13196 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13197 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13198 13199 if (!exit_addr) { 13200 return 0; 13201 } 13202 type_loaded = 1; 13203 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13204 | IF_UNDEF dl, &exit_addr 13205 } 13206 } else { 13207 type_loaded = 1; 13208 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13209 | IF_UNDEF dl, >5 13210 } 13211 if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) { 13212 if (!type_loaded) { 13213 type_loaded = 1; 13214 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13215 } 13216 | IF_NOT_TYPE dl, IS_OBJECT, >4 13217 | GET_ZVAL_PTR r0, prop_addr 13218 | GC_ADDREF r0 13219 | SET_ZVAL_PTR res_addr, r0 13220 | SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX 13221 | jmp >9 13222 |.cold_code 13223 |4: 13224 | LOAD_ADDR FCARG1a, prop_info 13225 | SET_EX_OPLINE opline, r0 13226 | EXT_CALL zend_readonly_property_modification_error, r0 13227 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 13228 | jmp >9 13229 |.code 13230 } 13231 if (opline->opcode == ZEND_FETCH_OBJ_W 13232 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) 13233 && ZEND_TYPE_IS_SET(prop_info->type)) { 13234 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 13235 13236 if (flags == ZEND_FETCH_DIM_WRITE) { 13237 if ((ZEND_TYPE_FULL_MASK(prop_info->type) & (MAY_BE_ITERABLE|MAY_BE_ARRAY)) == 0) { 13238 if (!type_loaded) { 13239 type_loaded = 1; 13240 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13241 } 13242 | cmp dl, IS_FALSE 13243 | jle >1 13244 |.cold_code 13245 |1: 13246 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13247 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13248 } 13249 | LOAD_ADDR FCARG2a, prop_info 13250 | SET_EX_OPLINE opline, r0 13251 | EXT_CALL zend_jit_check_array_promotion, r0 13252 | jmp >9 13253 |.code 13254 } 13255 } else if (flags == ZEND_FETCH_REF) { 13256 if (!type_loaded) { 13257 type_loaded = 1; 13258 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13259 } 13260 | IF_TYPE dl, IS_REFERENCE, >1 13261 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13262 | LOAD_ADDR FCARG2a, prop_info 13263 } else { 13264 int prop_info_offset = 13265 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13266 13267 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13268 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13269 | mov FCARG2a, aword[r0 + prop_info_offset] 13270 } 13271 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13272 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13273 } 13274 |.if X64 13275 | LOAD_ZVAL_ADDR CARG3, res_addr 13276 |.else 13277 | sub r4, 12 13278 | PUSH_ZVAL_ADDR res_addr, r0 13279 |.endif 13280 | EXT_CALL zend_jit_create_typed_ref, r0 13281 |.if not(X64) 13282 | add r4, 12 13283 |.endif 13284 | jmp >9 13285 |1: 13286 } else { 13287 ZEND_UNREACHABLE(); 13288 } 13289 } 13290 } 13291 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13292 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13293 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13294 } 13295 | SET_ZVAL_PTR res_addr, FCARG1a 13296 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 13297 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { 13298 ssa->var_info[ssa_op->result_def].indirect_reference = 1; 13299 } 13300 } else { 13301 bool result_avoid_refcounting = 0; 13302 13303 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { 13304 uint32_t flags = 0; 13305 uint32_t old_info; 13306 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 13307 int32_t exit_point; 13308 const void *exit_addr; 13309 zend_uchar type; 13310 zend_jit_addr val_addr = prop_addr; 13311 13312 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 13313 && !delayed_fetch_this 13314 && !op1_avoid_refcounting) { 13315 flags = ZEND_JIT_EXIT_FREE_OP1; 13316 } 13317 13318 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 13319 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 13320 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 13321 && (ssa_op+1)->op1_use == ssa_op->result_def 13322 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 13323 result_avoid_refcounting = 1; 13324 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 13325 } 13326 13327 type = concrete_type(res_info); 13328 13329 if (prop_type != IS_UNKNOWN 13330 && prop_type != IS_UNDEF 13331 && prop_type != IS_REFERENCE 13332 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) { 13333 exit_point = zend_jit_trace_get_exit_point(opline, 0); 13334 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13335 if (!exit_addr) { 13336 return 0; 13337 } 13338 } else { 13339 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 13340 | LOAD_ZVAL_ADDR r0, prop_addr 13341 if (op1_avoid_refcounting) { 13342 SET_STACK_REG(JIT_G(current_frame)->stack, 13343 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 13344 } 13345 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 13346 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 13347 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 13348 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 13349 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 13350 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13351 if (!exit_addr) { 13352 return 0; 13353 } 13354 13355 if (!type_loaded) { 13356 type_loaded = 1; 13357 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13358 } 13359 | // ZVAL_DEREF() 13360 | IF_NOT_TYPE dl, IS_REFERENCE, >1 13361 | GET_Z_PTR r0, r0 13362 | add r0, offsetof(zend_reference, val) 13363 | GET_ZVAL_TYPE_INFO edx, val_addr 13364 } 13365 res_info &= ~MAY_BE_GUARD; 13366 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 13367 if (type < IS_STRING) { 13368 |1: 13369 if (type_loaded) { 13370 | IF_NOT_TYPE dl, type, &exit_addr 13371 } else { 13372 | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr 13373 } 13374 } else { 13375 if (!type_loaded) { 13376 type_loaded = 1; 13377 | GET_ZVAL_TYPE_INFO edx, val_addr 13378 } 13379 |1: 13380 | IF_NOT_TYPE dl, type, &exit_addr 13381 } 13382 | // ZVAL_COPY 13383 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 13384 if (type < IS_STRING) { 13385 if (Z_REG(res_addr) != ZREG_FP || 13386 JIT_G(current_frame) == NULL || 13387 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 13388 | SET_ZVAL_TYPE_INFO res_addr, type 13389 } 13390 } else { 13391 | SET_ZVAL_TYPE_INFO res_addr, edx 13392 if (!result_avoid_refcounting) { 13393 | TRY_ADDREF res_info, dh, r1 13394 } 13395 } 13396 } else { 13397 if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_R2)) { 13398 return 0; 13399 } 13400 } 13401 } 13402 13403 if (op1_avoid_refcounting) { 13404 SET_STACK_REG(JIT_G(current_frame)->stack, 13405 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 13406 } 13407 13408 |.cold_code 13409 13410 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { 13411 |5: 13412 | SET_EX_OPLINE opline, r0 13413 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13414 | EXT_CALL zend_jit_fetch_obj_w_slow, r0 13415 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13416 | EXT_CALL zend_jit_fetch_obj_r_slow, r0 13417 } else { 13418 | EXT_CALL zend_jit_fetch_obj_is_slow, r0 13419 } 13420 | jmp >9 13421 } 13422 13423 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 13424 |7: 13425 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13426 | SET_EX_OPLINE opline, r0 13427 if (opline->opcode != ZEND_FETCH_OBJ_W 13428 && (op1_info & MAY_BE_UNDEF)) { 13429 zend_jit_addr orig_op1_addr = OP1_ADDR(); 13430 13431 if (op1_info & MAY_BE_ANY) { 13432 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 13433 } 13434 | mov FCARG1d, opline->op1.var 13435 | EXT_CALL zend_jit_undefined_op_helper, r0 13436 |1: 13437 | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr 13438 } else if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13439 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13440 } 13441 | LOAD_ADDR FCARG2a, Z_STRVAL_P(member) 13442 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13443 | EXT_CALL zend_jit_invalid_property_write, r0 13444 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 13445 } else { 13446 | EXT_CALL zend_jit_invalid_property_read, r0 13447 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 13448 } 13449 | jmp >9 13450 } else { 13451 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 13452 | jmp >9 13453 } 13454 } 13455 13456 if (!prop_info 13457 && may_be_dynamic 13458 && opline->opcode != ZEND_FETCH_OBJ_W) { 13459 |8: 13460 | mov FCARG2a, r0 13461 | SET_EX_OPLINE opline, r0 13462 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13463 | EXT_CALL zend_jit_fetch_obj_r_dynamic, r0 13464 } else { 13465 | EXT_CALL zend_jit_fetch_obj_is_dynamic, r0 13466 } 13467 | jmp >9 13468 } 13469 13470 |.code; 13471 |9: // END 13472 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13473 if (opline->op1_type == IS_VAR 13474 && opline->opcode == ZEND_FETCH_OBJ_W 13475 && (op1_info & MAY_BE_RC1)) { 13476 zend_jit_addr orig_op1_addr = OP1_ADDR(); 13477 13478 | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1 13479 | GET_ZVAL_PTR FCARG1a, orig_op1_addr 13480 | GC_DELREF FCARG1a 13481 | jnz >1 13482 | SET_EX_OPLINE opline, r0 13483 | EXT_CALL zend_jit_extract_helper, r0 13484 |1: 13485 } else if (!op1_avoid_refcounting) { 13486 if (on_this) { 13487 op1_info &= ~MAY_BE_RC1; 13488 } 13489 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 13490 } 13491 } 13492 13493 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 13494 && prop_info 13495 && (opline->opcode != ZEND_FETCH_OBJ_W || 13496 !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) || 13497 !ZEND_TYPE_IS_SET(prop_info->type)) 13498 && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) { 13499 may_throw = 0; 13500 } 13501 13502 if (may_throw) { 13503 if (!zend_jit_check_exception(Dst)) { 13504 return 0; 13505 } 13506 } 13507 13508 return 1; 13509} 13510 13511static int zend_jit_incdec_obj(dasm_State **Dst, 13512 const zend_op *opline, 13513 const zend_op_array *op_array, 13514 zend_ssa *ssa, 13515 const zend_ssa_op *ssa_op, 13516 uint32_t op1_info, 13517 zend_jit_addr op1_addr, 13518 bool op1_indirect, 13519 zend_class_entry *ce, 13520 bool ce_is_instanceof, 13521 bool on_this, 13522 bool delayed_fetch_this, 13523 zend_class_entry *trace_ce, 13524 uint8_t prop_type) 13525{ 13526 zval *member; 13527 zend_string *name; 13528 zend_property_info *prop_info; 13529 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13530 zend_jit_addr res_addr = 0; 13531 zend_jit_addr prop_addr; 13532 bool needs_slow_path = 0; 13533 bool use_prop_guard = 0; 13534 bool may_throw = 0; 13535 uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0; 13536 13537 ZEND_ASSERT(opline->op2_type == IS_CONST); 13538 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13539 13540 if (opline->result_type != IS_UNUSED) { 13541 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 13542 } 13543 13544 member = RT_CONSTANT(opline, opline->op2); 13545 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13546 name = Z_STR_P(member); 13547 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 13548 13549 if (on_this) { 13550 | GET_ZVAL_PTR FCARG1a, this_addr 13551 } else { 13552 if (opline->op1_type == IS_VAR 13553 && (op1_info & MAY_BE_INDIRECT) 13554 && Z_REG(op1_addr) == ZREG_FP) { 13555 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13556 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13557 | GET_Z_PTR FCARG1a, FCARG1a 13558 |1: 13559 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13560 } 13561 if (op1_info & MAY_BE_REF) { 13562 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13563 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13564 } 13565 | ZVAL_DEREF FCARG1a, op1_info 13566 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13567 } 13568 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13569 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13570 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13571 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13572 13573 if (!exit_addr) { 13574 return 0; 13575 } 13576 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13577 } else { 13578 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 13579 |.cold_code 13580 |1: 13581 | SET_EX_OPLINE opline, r0 13582 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 13583 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13584 } 13585 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 13586 | EXT_CALL zend_jit_invalid_property_incdec, r0 13587 | jmp ->exception_handler 13588 |.code 13589 } 13590 } 13591 | GET_ZVAL_PTR FCARG1a, op1_addr 13592 } 13593 13594 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13595 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 13596 if (prop_info) { 13597 ce = trace_ce; 13598 ce_is_instanceof = 0; 13599 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13600 if (on_this && JIT_G(current_frame) 13601 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 13602 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 13603 } else if (zend_jit_class_guard(Dst, opline, ce)) { 13604 if (on_this && JIT_G(current_frame)) { 13605 JIT_G(current_frame)->ce = ce; 13606 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 13607 } 13608 } else { 13609 return 0; 13610 } 13611 if (ssa->var_info && ssa_op->op1_use >= 0) { 13612 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13613 ssa->var_info[ssa_op->op1_use].ce = ce; 13614 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13615 } 13616 if (ssa->var_info && ssa_op->op1_def >= 0) { 13617 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13618 ssa->var_info[ssa_op->op1_def].ce = ce; 13619 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13620 } 13621 } 13622 } 13623 } 13624 13625 use_prop_guard = (prop_type != IS_UNKNOWN 13626 && prop_type != IS_UNDEF 13627 && prop_type != IS_REFERENCE 13628 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 13629 13630 if (!prop_info) { 13631 needs_slow_path = 1; 13632 13633 | mov r0, EX->run_time_cache 13634 | mov r2, aword [r0 + opline->extended_value] 13635 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13636 | jne >7 13637 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 13638 | cmp aword [r0 + opline->extended_value + sizeof(void*) * 2], 0 13639 | jnz >7 13640 } 13641 | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] 13642 | test r0, r0 13643 | jl >7 13644 if (!use_prop_guard) { 13645 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 13646 } 13647 | add FCARG1a, r0 13648 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13649 } else { 13650 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 13651 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 13652 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13653 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13654 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13655 13656 if (!exit_addr) { 13657 return 0; 13658 } 13659 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 13660 } else { 13661 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 13662 needs_slow_path = 1; 13663 } 13664 } 13665 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13666 may_throw = 1; 13667 | SET_EX_OPLINE opline, r0 13668 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13669 | LOAD_ADDR FCARG2a, prop_info 13670 } else { 13671 int prop_info_offset = 13672 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13673 13674 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13675 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13676 | mov FCARG2a, aword[r0 + prop_info_offset] 13677 } 13678 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13679 if (opline->result_type == IS_UNUSED) { 13680 switch (opline->opcode) { 13681 case ZEND_PRE_INC_OBJ: 13682 case ZEND_POST_INC_OBJ: 13683 | EXT_CALL zend_jit_inc_typed_prop, r0 13684 break; 13685 case ZEND_PRE_DEC_OBJ: 13686 case ZEND_POST_DEC_OBJ: 13687 | EXT_CALL zend_jit_dec_typed_prop, r0 13688 break; 13689 default: 13690 ZEND_UNREACHABLE(); 13691 } 13692 } else { 13693 |.if X64 13694 | LOAD_ZVAL_ADDR CARG3, res_addr 13695 |.else 13696 | sub r4, 12 13697 | PUSH_ZVAL_ADDR res_addr, r0 13698 |.endif 13699 switch (opline->opcode) { 13700 case ZEND_PRE_INC_OBJ: 13701 | EXT_CALL zend_jit_pre_inc_typed_prop, r0 13702 break; 13703 case ZEND_PRE_DEC_OBJ: 13704 | EXT_CALL zend_jit_pre_dec_typed_prop, r0 13705 break; 13706 case ZEND_POST_INC_OBJ: 13707 | EXT_CALL zend_jit_post_inc_typed_prop, r0 13708 break; 13709 case ZEND_POST_DEC_OBJ: 13710 | EXT_CALL zend_jit_post_dec_typed_prop, r0 13711 break; 13712 default: 13713 ZEND_UNREACHABLE(); 13714 } 13715 |.if not(X64) 13716 | add r4, 12 13717 |.endif 13718 } 13719 } 13720 } 13721 13722 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13723 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 13724 zend_jit_addr var_addr = prop_addr; 13725 13726 if (use_prop_guard) { 13727 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 13728 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13729 if (!exit_addr) { 13730 return 0; 13731 } 13732 13733 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr 13734 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 13735 } 13736 13737 if (var_info & MAY_BE_REF) { 13738 may_throw = 1; 13739 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13740 if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) { 13741 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13742 } 13743 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 13744 | GET_ZVAL_PTR FCARG1a, var_addr 13745 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 13746 | jnz >1 13747 | lea FCARG1a, aword [FCARG1a + offsetof(zend_reference, val)] 13748 |.cold_code 13749 |1: 13750 if (opline) { 13751 | SET_EX_OPLINE opline, r0 13752 } 13753 if (opline->result_type == IS_UNUSED) { 13754 | xor FCARG2a, FCARG2a 13755 } else { 13756 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13757 } 13758 switch (opline->opcode) { 13759 case ZEND_PRE_INC_OBJ: 13760 | EXT_CALL zend_jit_pre_inc_typed_ref, r0 13761 break; 13762 case ZEND_PRE_DEC_OBJ: 13763 | EXT_CALL zend_jit_pre_dec_typed_ref, r0 13764 break; 13765 case ZEND_POST_INC_OBJ: 13766 | EXT_CALL zend_jit_post_inc_typed_ref, r0 13767 break; 13768 case ZEND_POST_DEC_OBJ: 13769 | EXT_CALL zend_jit_post_dec_typed_ref, r0 13770 break; 13771 default: 13772 ZEND_UNREACHABLE(); 13773 } 13774 | jmp >9 13775 |.code 13776 |2: 13777 } 13778 13779 if (var_info & MAY_BE_LONG) { 13780 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 13781 | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2 13782 } 13783 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 13784 if (opline->result_type != IS_UNUSED) { 13785 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R1, ZREG_R2 13786 } 13787 } 13788 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13789 | LONG_OP_WITH_32BIT_CONST add, var_addr, Z_L(1) 13790 } else { 13791 | LONG_OP_WITH_32BIT_CONST sub, var_addr, Z_L(1) 13792 } 13793 | jo >3 13794 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { 13795 if (opline->result_type != IS_UNUSED) { 13796 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R0, ZREG_R2 13797 } 13798 } 13799 |.cold_code 13800 } 13801 if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { 13802 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 13803 may_throw = 1; 13804 } 13805 if (var_info & MAY_BE_LONG) { 13806 |2: 13807 } 13808 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 13809 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 13810 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13811 } 13812 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 13813 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_R0, ZREG_R2 13814 | TRY_ADDREF MAY_BE_ANY, ah, r2 13815 } 13816 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13817 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13818 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13819 | EXT_CALL zend_jit_pre_inc, r0 13820 } else { 13821 | EXT_CALL increment_function, r0 13822 } 13823 } else { 13824 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13825 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13826 | EXT_CALL zend_jit_pre_dec, r0 13827 } else { 13828 | EXT_CALL decrement_function, r0 13829 } 13830 } 13831 if (var_info & MAY_BE_LONG) { 13832 | jmp >4 13833 } 13834 } 13835 if (var_info & MAY_BE_LONG) { 13836 |3: 13837 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13838 |.if X64 13839 | mov64 rax, 0x43e0000000000000 13840 | SET_ZVAL_LVAL var_addr, rax 13841 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13842 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13843 | SET_ZVAL_LVAL res_addr, rax 13844 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13845 } 13846 |.else 13847 | SET_ZVAL_LVAL var_addr, 0 13848 | SET_ZVAL_W2 var_addr, 0x41e00000 13849 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13850 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13851 | SET_ZVAL_LVAL res_addr, 0 13852 | SET_ZVAL_W2 res_addr, 0x41e00000 13853 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13854 } 13855 |.endif 13856 } else { 13857 |.if X64 13858 | mov64 rax, 0xc3e0000000000000 13859 | SET_ZVAL_LVAL var_addr, rax 13860 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13861 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13862 | SET_ZVAL_LVAL res_addr, rax 13863 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13864 } 13865 |.else 13866 | SET_ZVAL_LVAL var_addr, 0x00200000 13867 | SET_ZVAL_W2 var_addr, 0xc1e00000 13868 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13869 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13870 | SET_ZVAL_LVAL res_addr, 0x00200000 13871 | SET_ZVAL_W2 res_addr, 0xc1e00000 13872 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13873 } 13874 |.endif 13875 } 13876 if (opline->result_type != IS_UNUSED 13877 && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) 13878 && prop_info 13879 && !ZEND_TYPE_IS_SET(prop_info->type) 13880 && (res_info & MAY_BE_GUARD) 13881 && (res_info & MAY_BE_LONG)) { 13882 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 13883 uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 13884 int32_t exit_point; 13885 const void *exit_addr; 13886 13887 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 13888 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 13889 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13890 if (!exit_addr) { 13891 return 0; 13892 } 13893 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 13894 ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD; 13895 | jmp &exit_addr 13896 |.code 13897 } else { 13898 | jmp >4 13899 |.code 13900 |4: 13901 } 13902 } 13903 } 13904 13905 if (needs_slow_path) { 13906 may_throw = 1; 13907 |.cold_code 13908 |7: 13909 | SET_EX_OPLINE opline, r0 13910 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13911 | LOAD_ADDR FCARG2a, name 13912 |.if X64 13913 | mov CARG3, EX->run_time_cache 13914 | add CARG3, opline->extended_value 13915 if (opline->result_type == IS_UNUSED) { 13916 | xor CARG4, CARG4 13917 } else { 13918 | LOAD_ZVAL_ADDR CARG4, res_addr 13919 } 13920 |.else 13921 | sub r4, 8 13922 if (opline->result_type == IS_UNUSED) { 13923 | push 0 13924 } else { 13925 | PUSH_ZVAL_ADDR res_addr, r0 13926 } 13927 | mov r0, EX->run_time_cache 13928 | add r0, opline->extended_value 13929 | push r0 13930 |.endif 13931 13932 switch (opline->opcode) { 13933 case ZEND_PRE_INC_OBJ: 13934 | EXT_CALL zend_jit_pre_inc_obj_helper, r0 13935 break; 13936 case ZEND_PRE_DEC_OBJ: 13937 | EXT_CALL zend_jit_pre_dec_obj_helper, r0 13938 break; 13939 case ZEND_POST_INC_OBJ: 13940 | EXT_CALL zend_jit_post_inc_obj_helper, r0 13941 break; 13942 case ZEND_POST_DEC_OBJ: 13943 | EXT_CALL zend_jit_post_dec_obj_helper, r0 13944 break; 13945 default: 13946 ZEND_UNREACHABLE(); 13947 } 13948 13949 |.if not(X64) 13950 | add r4, 8 13951 |.endif 13952 13953 | jmp >9 13954 |.code 13955 } 13956 13957 |9: 13958 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 13959 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 13960 may_throw = 1; 13961 } 13962 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 13963 } 13964 13965 if (may_throw) { 13966 if (!zend_jit_check_exception(Dst)) { 13967 return 0; 13968 } 13969 } 13970 13971 return 1; 13972} 13973 13974static int zend_jit_assign_obj_op(dasm_State **Dst, 13975 const zend_op *opline, 13976 const zend_op_array *op_array, 13977 zend_ssa *ssa, 13978 const zend_ssa_op *ssa_op, 13979 uint32_t op1_info, 13980 zend_jit_addr op1_addr, 13981 uint32_t val_info, 13982 zend_ssa_range *val_range, 13983 bool op1_indirect, 13984 zend_class_entry *ce, 13985 bool ce_is_instanceof, 13986 bool on_this, 13987 bool delayed_fetch_this, 13988 zend_class_entry *trace_ce, 13989 uint8_t prop_type) 13990{ 13991 zval *member; 13992 zend_string *name; 13993 zend_property_info *prop_info; 13994 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13995 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13996 zend_jit_addr prop_addr; 13997 bool needs_slow_path = 0; 13998 bool use_prop_guard = 0; 13999 bool may_throw = 0; 14000 binary_op_type binary_op = get_binary_op(opline->extended_value); 14001 14002 ZEND_ASSERT(opline->op2_type == IS_CONST); 14003 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 14004 ZEND_ASSERT(opline->result_type == IS_UNUSED); 14005 14006 member = RT_CONSTANT(opline, opline->op2); 14007 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 14008 name = Z_STR_P(member); 14009 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 14010 14011 if (on_this) { 14012 | GET_ZVAL_PTR FCARG1a, this_addr 14013 } else { 14014 if (opline->op1_type == IS_VAR 14015 && (op1_info & MAY_BE_INDIRECT) 14016 && Z_REG(op1_addr) == ZREG_FP) { 14017 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14018 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 14019 | GET_Z_PTR FCARG1a, FCARG1a 14020 |1: 14021 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14022 } 14023 if (op1_info & MAY_BE_REF) { 14024 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14025 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14026 } 14027 | ZVAL_DEREF FCARG1a, op1_info 14028 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14029 } 14030 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 14031 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14032 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14033 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14034 14035 if (!exit_addr) { 14036 return 0; 14037 } 14038 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 14039 } else { 14040 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 14041 |.cold_code 14042 |1: 14043 | SET_EX_OPLINE opline, r0 14044 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14045 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14046 } 14047 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 14048 if (op1_info & MAY_BE_UNDEF) { 14049 | EXT_CALL zend_jit_invalid_property_assign_op, r0 14050 } else { 14051 | EXT_CALL zend_jit_invalid_property_assign, r0 14052 } 14053 may_throw = 1; 14054 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 14055 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14056 may_throw = 1; 14057 | jmp >8 14058 } else { 14059 | jmp >9 14060 } 14061 |.code 14062 } 14063 } 14064 | GET_ZVAL_PTR FCARG1a, op1_addr 14065 } 14066 14067 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 14068 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 14069 if (prop_info) { 14070 ce = trace_ce; 14071 ce_is_instanceof = 0; 14072 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 14073 if (on_this && JIT_G(current_frame) 14074 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 14075 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 14076 } else if (zend_jit_class_guard(Dst, opline, ce)) { 14077 if (on_this && JIT_G(current_frame)) { 14078 JIT_G(current_frame)->ce = ce; 14079 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 14080 } 14081 } else { 14082 return 0; 14083 } 14084 if (ssa->var_info && ssa_op->op1_use >= 0) { 14085 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 14086 ssa->var_info[ssa_op->op1_use].ce = ce; 14087 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 14088 } 14089 if (ssa->var_info && ssa_op->op1_def >= 0) { 14090 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 14091 ssa->var_info[ssa_op->op1_def].ce = ce; 14092 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 14093 } 14094 } 14095 } 14096 } 14097 14098 use_prop_guard = (prop_type != IS_UNKNOWN 14099 && prop_type != IS_UNDEF 14100 && prop_type != IS_REFERENCE 14101 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); 14102 14103 if (!prop_info) { 14104 needs_slow_path = 1; 14105 14106 | mov r0, EX->run_time_cache 14107 | mov r2, aword [r0 + (opline+1)->extended_value] 14108 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 14109 | jne >7 14110 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 14111 | cmp aword [r0 + (opline+1)->extended_value + sizeof(void*) * 2], 0 14112 | jnz >7 14113 } 14114 | mov r0, aword [r0 + (opline+1)->extended_value + sizeof(void*)] 14115 | test r0, r0 14116 | jl >7 14117 if (!use_prop_guard) { 14118 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 14119 } 14120 | add FCARG1a, r0 14121 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14122 } else { 14123 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 14124 if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { 14125 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14126 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14127 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14128 14129 if (!exit_addr) { 14130 return 0; 14131 } 14132 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 14133 } else { 14134 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 14135 needs_slow_path = 1; 14136 } 14137 } 14138 if (ZEND_TYPE_IS_SET(prop_info->type)) { 14139 uint32_t info = val_info; 14140 14141 may_throw = 1; 14142 14143 if (opline) { 14144 | SET_EX_OPLINE opline, r0 14145 } 14146 14147 | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1 14148 |.cold_code 14149 |1: 14150 | GET_ZVAL_PTR FCARG1a, prop_addr 14151 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 14152 | LOAD_ZVAL_ADDR FCARG2a, val_addr 14153 } 14154 |.if X64 14155 | LOAD_ADDR CARG3, binary_op 14156 |.else 14157 | sub r4, 12 14158 | PUSH_ADDR binary_op, r0 14159 |.endif 14160 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 14161 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14162 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 14163 } else { 14164 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 14165 } 14166 |.if not(X64) 14167 | add r4, 12 14168 |.endif 14169 | jmp >9 14170 |.code 14171 14172 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14173 14174 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 14175 | LOAD_ADDR FCARG2a, prop_info 14176 } else { 14177 int prop_info_offset = 14178 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 14179 14180 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 14181 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 14182 | mov FCARG2a, aword[r0 + prop_info_offset] 14183 } 14184 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 14185 |.if X64 14186 | LOAD_ZVAL_ADDR CARG3, val_addr 14187 | LOAD_ADDR CARG4, binary_op 14188 |.else 14189 | sub r4, 8 14190 | PUSH_ADDR binary_op, r0 14191 | PUSH_ZVAL_ADDR val_addr, r0 14192 |.endif 14193 14194 | EXT_CALL zend_jit_assign_op_to_typed_prop, r0 14195 14196 |.if not(X64) 14197 | add r4, 8 14198 |.endif 14199 14200 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14201 info |= MAY_BE_RC1|MAY_BE_RCN; 14202 } 14203 14204 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL 14205 } 14206 } 14207 14208 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 14209 zend_jit_addr var_addr = prop_addr; 14210 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 14211 uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 14212 14213 if (use_prop_guard) { 14214 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 14215 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14216 if (!exit_addr) { 14217 return 0; 14218 } 14219 14220 | IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr 14221 var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); 14222 } 14223 14224 if (var_info & MAY_BE_REF) { 14225 may_throw = 1; 14226 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 14227 | LOAD_ZVAL_ADDR r0, prop_addr 14228 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 14229 | GET_ZVAL_PTR FCARG1a, var_addr 14230 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 14231 | jnz >1 14232 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] 14233 |.cold_code 14234 |1: 14235 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) { 14236 | LOAD_ZVAL_ADDR FCARG2a, val_addr 14237 } 14238 if (opline) { 14239 | SET_EX_OPLINE opline, r0 14240 } 14241 |.if X64 14242 | LOAD_ADDR CARG3, binary_op 14243 |.else 14244 | sub r4, 12 14245 | PUSH_ADDR binary_op, r0 14246 |.endif 14247 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 14248 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14249 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 14250 } else { 14251 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 14252 } 14253 |.if not(X64) 14254 | add r4, 12 14255 |.endif 14256 | jmp >9 14257 |.code 14258 |2: 14259 var_info &= ~MAY_BE_REF; 14260 } 14261 14262 switch (opline->extended_value) { 14263 case ZEND_ADD: 14264 case ZEND_SUB: 14265 case ZEND_MUL: 14266 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14267 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14268 if (opline->extended_value != ZEND_ADD || 14269 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 14270 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 14271 may_throw = 1; 14272 } 14273 } 14274 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, 14275 1 /* may overflow */, 0)) { 14276 return 0; 14277 } 14278 break; 14279 case ZEND_BW_OR: 14280 case ZEND_BW_AND: 14281 case ZEND_BW_XOR: 14282 may_throw = 1; 14283 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14284 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14285 if ((var_info & MAY_BE_ANY) != MAY_BE_STRING || 14286 (val_info & MAY_BE_ANY) != MAY_BE_STRING) { 14287 may_throw = 1; 14288 } 14289 } 14290 goto long_math; 14291 case ZEND_SL: 14292 case ZEND_SR: 14293 if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14294 (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14295 may_throw = 1; 14296 } 14297 if ((opline+1)->op1_type != IS_CONST || 14298 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 14299 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) { 14300 may_throw = 1; 14301 } 14302 goto long_math; 14303 case ZEND_MOD: 14304 if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || 14305 (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14306 if (opline->extended_value != ZEND_ADD || 14307 (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || 14308 (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { 14309 may_throw = 1; 14310 } 14311 } 14312 if ((opline+1)->op1_type != IS_CONST || 14313 Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || 14314 Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) { 14315 may_throw = 1; 14316 } 14317long_math: 14318 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 14319 IS_CV, opline->op1, var_addr, var_info, NULL, 14320 (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 14321 val_range, 14322 0, var_addr, var_def_info, var_info, /* may throw */ 1)) { 14323 return 0; 14324 } 14325 break; 14326 case ZEND_CONCAT: 14327 may_throw = 1; 14328 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, 14329 0)) { 14330 return 0; 14331 } 14332 break; 14333 default: 14334 ZEND_UNREACHABLE(); 14335 } 14336 } 14337 14338 if (needs_slow_path) { 14339 may_throw = 1; 14340 |.cold_code 14341 |7: 14342 | SET_EX_OPLINE opline, r0 14343 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 14344 | LOAD_ADDR FCARG2a, name 14345 |.if X64 14346 | LOAD_ZVAL_ADDR CARG3, val_addr 14347 | mov CARG4, EX->run_time_cache 14348 | add CARG4, (opline+1)->extended_value 14349 |.if X64WIN 14350 | LOAD_ADDR r0, binary_op 14351 | mov aword A5, r0 14352 |.else 14353 | LOAD_ADDR CARG5, binary_op 14354 |.endif 14355 |.else 14356 | sub r4, 4 14357 | PUSH_ADDR binary_op, r0 14358 | mov r0, EX->run_time_cache 14359 | add r0, (opline+1)->extended_value 14360 | push r0 14361 | PUSH_ZVAL_ADDR val_addr, r0 14362 |.endif 14363 14364 | EXT_CALL zend_jit_assign_obj_op_helper, r0 14365 14366 |.if not(X64) 14367 | add r4, 4 14368 |.endif 14369 14370 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14371 val_info |= MAY_BE_RC1|MAY_BE_RCN; 14372 } 14373 14374 |8: 14375 | // FREE_OP_DATA(); 14376 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14377 | jmp >9 14378 |.code 14379 } 14380 14381 |9: 14382 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 14383 if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { 14384 may_throw = 1; 14385 } 14386 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 14387 } 14388 14389 if (may_throw) { 14390 if (!zend_jit_check_exception(Dst)) { 14391 return 0; 14392 } 14393 } 14394 14395 return 1; 14396} 14397 14398static int zend_jit_assign_obj(dasm_State **Dst, 14399 const zend_op *opline, 14400 const zend_op_array *op_array, 14401 zend_ssa *ssa, 14402 const zend_ssa_op *ssa_op, 14403 uint32_t op1_info, 14404 zend_jit_addr op1_addr, 14405 uint32_t val_info, 14406 bool op1_indirect, 14407 zend_class_entry *ce, 14408 bool ce_is_instanceof, 14409 bool on_this, 14410 bool delayed_fetch_this, 14411 zend_class_entry *trace_ce, 14412 uint8_t prop_type, 14413 int may_throw) 14414{ 14415 zval *member; 14416 zend_string *name; 14417 zend_property_info *prop_info; 14418 zend_jit_addr val_addr = OP1_DATA_ADDR(); 14419 zend_jit_addr res_addr = 0; 14420 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 14421 zend_jit_addr prop_addr; 14422 bool needs_slow_path = 0; 14423 bool needs_val_dtor = 0; 14424 14425 if (RETURN_VALUE_USED(opline)) { 14426 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14427 } 14428 14429 ZEND_ASSERT(opline->op2_type == IS_CONST); 14430 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 14431 14432 member = RT_CONSTANT(opline, opline->op2); 14433 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 14434 name = Z_STR_P(member); 14435 prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); 14436 14437 if (on_this) { 14438 | GET_ZVAL_PTR FCARG1a, this_addr 14439 } else { 14440 if (opline->op1_type == IS_VAR 14441 && (op1_info & MAY_BE_INDIRECT) 14442 && Z_REG(op1_addr) == ZREG_FP) { 14443 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14444 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 14445 | GET_Z_PTR FCARG1a, FCARG1a 14446 |1: 14447 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14448 } 14449 if (op1_info & MAY_BE_REF) { 14450 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14451 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14452 } 14453 | ZVAL_DEREF FCARG1a, op1_info 14454 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14455 } 14456 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 14457 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14458 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14459 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14460 14461 if (!exit_addr) { 14462 return 0; 14463 } 14464 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 14465 } else { 14466 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 14467 |.cold_code 14468 |1: 14469 | SET_EX_OPLINE opline, r0 14470 if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 14471 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14472 } 14473 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 14474 | EXT_CALL zend_jit_invalid_property_assign, r0 14475 if (RETURN_VALUE_USED(opline)) { 14476 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 14477 } 14478 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 14479 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14480 needs_val_dtor = 1; 14481 | jmp >7 14482 } else { 14483 | jmp >9 14484 } 14485 |.code 14486 } 14487 } 14488 | GET_ZVAL_PTR FCARG1a, op1_addr 14489 } 14490 14491 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 14492 prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); 14493 if (prop_info) { 14494 ce = trace_ce; 14495 ce_is_instanceof = 0; 14496 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 14497 if (on_this && JIT_G(current_frame) 14498 && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { 14499 ZEND_ASSERT(JIT_G(current_frame)->ce == ce); 14500 } else if (zend_jit_class_guard(Dst, opline, ce)) { 14501 if (on_this && JIT_G(current_frame)) { 14502 JIT_G(current_frame)->ce = ce; 14503 TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); 14504 } 14505 } else { 14506 return 0; 14507 } 14508 if (ssa->var_info && ssa_op->op1_use >= 0) { 14509 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 14510 ssa->var_info[ssa_op->op1_use].ce = ce; 14511 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 14512 } 14513 if (ssa->var_info && ssa_op->op1_def >= 0) { 14514 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 14515 ssa->var_info[ssa_op->op1_def].ce = ce; 14516 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 14517 } 14518 } 14519 } 14520 } 14521 14522 if (!prop_info) { 14523 needs_slow_path = 1; 14524 14525 | mov r0, EX->run_time_cache 14526 | mov r2, aword [r0 + opline->extended_value] 14527 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 14528 | jne >5 14529 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 14530 | mov FCARG2a, aword [r0 + opline->extended_value + sizeof(void*) * 2] 14531 } 14532 | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] 14533 | test r0, r0 14534 | jl >5 14535 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >5 14536 | add FCARG1a, r0 14537 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 14538 if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { 14539 | test FCARG2a, FCARG2a 14540 | jnz >1 14541 |.cold_code 14542 |1: 14543 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14544 | SET_EX_OPLINE opline, r0 14545 |.if X64 14546 | LOAD_ZVAL_ADDR CARG3, val_addr 14547 if (RETURN_VALUE_USED(opline)) { 14548 | LOAD_ZVAL_ADDR CARG4, res_addr 14549 } else { 14550 | xor CARG4, CARG4 14551 } 14552 |.else 14553 | sub r4, 8 14554 if (RETURN_VALUE_USED(opline)) { 14555 | PUSH_ZVAL_ADDR res_addr, r0 14556 } else { 14557 | push 0 14558 } 14559 | PUSH_ZVAL_ADDR val_addr, r0 14560 |.endif 14561 14562 | EXT_CALL zend_jit_assign_to_typed_prop, r0 14563 14564 |.if not(X64) 14565 | add r4, 8 14566 |.endif 14567 14568 if ((opline+1)->op1_type == IS_CONST) { 14569 | // TODO: ??? 14570 | // if (Z_TYPE_P(value) == orig_type) { 14571 | // CACHE_PTR_EX(cache_slot + 2, NULL); 14572 } 14573 14574 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 14575 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14576 | jmp >7 14577 } else { 14578 | jmp >9 14579 } 14580 |.code 14581 } 14582 } else { 14583 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset); 14584 if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) { 14585 // Undefined property with magic __get()/__set() 14586 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14587 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14588 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14589 14590 if (!exit_addr) { 14591 return 0; 14592 } 14593 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 14594 } else { 14595 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >5 14596 needs_slow_path = 1; 14597 } 14598 } 14599 if (ZEND_TYPE_IS_SET(prop_info->type)) { 14600 uint32_t info = val_info; 14601 14602 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14603 | SET_EX_OPLINE opline, r0 14604 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 14605 | LOAD_ADDR FCARG2a, prop_info 14606 } else { 14607 int prop_info_offset = 14608 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 14609 14610 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 14611 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 14612 | mov FCARG2a, aword[r0 + prop_info_offset] 14613 } 14614 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 14615 |.if X64 14616 | LOAD_ZVAL_ADDR CARG3, val_addr 14617 if (RETURN_VALUE_USED(opline)) { 14618 | LOAD_ZVAL_ADDR CARG4, res_addr 14619 } else { 14620 | xor CARG4, CARG4 14621 } 14622 |.else 14623 | sub r4, 8 14624 if (RETURN_VALUE_USED(opline)) { 14625 | PUSH_ZVAL_ADDR res_addr, r0 14626 } else { 14627 | push 0 14628 } 14629 | PUSH_ZVAL_ADDR val_addr, r0 14630 |.endif 14631 14632 | EXT_CALL zend_jit_assign_to_typed_prop, r0 14633 14634 |.if not(X64) 14635 | add r4, 8 14636 |.endif 14637 14638 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14639 info |= MAY_BE_RC1|MAY_BE_RCN; 14640 } 14641 14642 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL 14643 } 14644 } 14645 14646 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 14647 // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); 14648 if (opline->result_type == IS_UNUSED) { 14649 if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 14650 return 0; 14651 } 14652 } else { 14653 if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 14654 return 0; 14655 } 14656 } 14657 } 14658 14659 if (needs_slow_path) { 14660 |.cold_code 14661 |5: 14662 | SET_EX_OPLINE opline, r0 14663 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 14664 | LOAD_ADDR FCARG2a, name 14665 |.if X64 14666 | LOAD_ZVAL_ADDR CARG3, val_addr 14667 | mov CARG4, EX->run_time_cache 14668 | add CARG4, opline->extended_value 14669 if (RETURN_VALUE_USED(opline)) { 14670 |.if X64WIN 14671 | LOAD_ZVAL_ADDR r0, res_addr 14672 | mov aword A5, r0 14673 |.else 14674 | LOAD_ZVAL_ADDR CARG5, res_addr 14675 |.endif 14676 } else { 14677 |.if X64WIN 14678 | mov aword A5, 0 14679 |.else 14680 | xor CARG5, CARG5 14681 |.endif 14682 } 14683 |.else 14684 | sub r4, 4 14685 if (RETURN_VALUE_USED(opline)) { 14686 | PUSH_ZVAL_ADDR res_addr, r0 14687 } else { 14688 | push 0 14689 } 14690 | mov r0, EX->run_time_cache 14691 | add r0, opline->extended_value 14692 | push r0 14693 | PUSH_ZVAL_ADDR val_addr, r0 14694 |.endif 14695 14696 | EXT_CALL zend_jit_assign_obj_helper, r0 14697 14698 |.if not(X64) 14699 | add r4, 4 14700 |.endif 14701 14702 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14703 val_info |= MAY_BE_RC1|MAY_BE_RCN; 14704 } 14705 14706 |7: 14707 | // FREE_OP_DATA(); 14708 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14709 | jmp >9 14710 |.code 14711 } else if (needs_val_dtor) { 14712 |.cold_code 14713 |7: 14714 | // FREE_OP_DATA(); 14715 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14716 | jmp >9 14717 |.code 14718 } 14719 14720 |9: 14721 if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { 14722 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 14723 } 14724 14725 if (may_throw) { 14726 if (!zend_jit_check_exception(Dst)) { 14727 return 0; 14728 } 14729 } 14730 14731 return 1; 14732} 14733 14734static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) 14735{ 14736 zend_jit_addr op1_addr = OP1_ADDR(); 14737 14738 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 14739 if (may_throw) { 14740 | SET_EX_OPLINE opline, r0 14741 } 14742 if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { 14743 if (op1_info & MAY_BE_ARRAY) { 14744 | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7 14745 } 14746 | mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_iter_idx)] 14747 | cmp FCARG1d, -1 14748 | je >7 14749 | EXT_CALL zend_hash_iterator_del, r0 14750 |7: 14751 } 14752 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline 14753 if (may_throw) { 14754 if (!zend_jit_check_exception(Dst)) { 14755 return 0; 14756 } 14757 } 14758 } 14759 14760 return 1; 14761} 14762 14763static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 14764{ 14765 if (opline->op1_type == IS_CONST) { 14766 zval *zv; 14767 size_t len; 14768 14769 zv = RT_CONSTANT(opline, opline->op1); 14770 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14771 len = Z_STRLEN_P(zv); 14772 14773 if (len > 0) { 14774 const char *str = Z_STRVAL_P(zv); 14775 14776 | SET_EX_OPLINE opline, r0 14777 |.if X64 14778 | LOAD_ADDR CARG1, str 14779 | LOAD_ADDR CARG2, len 14780 | EXT_CALL zend_write, r0 14781 |.else 14782 | mov aword A2, len 14783 | mov aword A1, str 14784 | EXT_CALL zend_write, r0 14785 |.endif 14786 if (!zend_jit_check_exception(Dst)) { 14787 return 0; 14788 } 14789 } 14790 } else { 14791 zend_jit_addr op1_addr = OP1_ADDR(); 14792 14793 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14794 14795 | SET_EX_OPLINE opline, r0 14796 | GET_ZVAL_PTR r0, op1_addr 14797 |.if X64 14798 | lea CARG1, aword [r0 + offsetof(zend_string, val)] 14799 | mov CARG2, aword [r0 + offsetof(zend_string, len)] 14800 | EXT_CALL zend_write, r0 14801 |.else 14802 | add r0, offsetof(zend_string, val) 14803 | mov aword A1, r0 14804 | mov r0, aword [r0 + (offsetof(zend_string, len)-offsetof(zend_string, val))] 14805 | mov aword A2, r0 14806 | EXT_CALL zend_write, r0 14807 |.endif 14808 if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { 14809 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline 14810 } 14811 if (!zend_jit_check_exception(Dst)) { 14812 return 0; 14813 } 14814 } 14815 return 1; 14816} 14817 14818static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) 14819{ 14820 if (opline->op1_type == IS_CONST) { 14821 zval *zv; 14822 size_t len; 14823 14824 zv = RT_CONSTANT(opline, opline->op1); 14825 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14826 len = Z_STRLEN_P(zv); 14827 14828 | SET_ZVAL_LVAL res_addr, len 14829 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 14830 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14831 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14832 return 0; 14833 } 14834 } else { 14835 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14836 14837 if (Z_MODE(res_addr) == IS_REG) { 14838 | GET_ZVAL_PTR Ra(Z_REG(res_addr)), op1_addr 14839 | mov Ra(Z_REG(res_addr)), aword [Ra(Z_REG(res_addr))+offsetof(zend_string, len)] 14840 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14841 return 0; 14842 } 14843 } else { 14844 | GET_ZVAL_PTR r0, op1_addr 14845 | mov r0, aword [r0 + offsetof(zend_string, len)] 14846 | SET_ZVAL_LVAL res_addr, r0 14847 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14848 } 14849 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 14850 } 14851 return 1; 14852} 14853 14854static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) 14855{ 14856 if (opline->op1_type == IS_CONST) { 14857 zval *zv; 14858 zend_long count; 14859 14860 zv = RT_CONSTANT(opline, opline->op1); 14861 ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); 14862 count = zend_hash_num_elements(Z_ARRVAL_P(zv)); 14863 14864 | SET_ZVAL_LVAL res_addr, count 14865 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 14866 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14867 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14868 return 0; 14869 } 14870 } else { 14871 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY); 14872 // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+. 14873 14874 if (Z_MODE(res_addr) == IS_REG) { 14875 | GET_ZVAL_PTR Ra(Z_REG(res_addr)), op1_addr 14876 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 14877 | mov Rd(Z_REG(res_addr)), dword [Ra(Z_REG(res_addr))+offsetof(HashTable, nNumOfElements)] 14878 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) { 14879 return 0; 14880 } 14881 } else { 14882 | GET_ZVAL_PTR r0, op1_addr 14883 // Sign-extend the 32-bit value to a potentially 64-bit zend_long 14884 | mov eax, dword [r0 + offsetof(HashTable, nNumOfElements)] 14885 | SET_ZVAL_LVAL res_addr, r0 14886 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14887 } 14888 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 14889 } 14890 14891 if (may_throw) { 14892 return zend_jit_check_exception(Dst); 14893 } 14894 return 1; 14895} 14896 14897static int zend_jit_load_this(dasm_State **Dst, uint32_t var) 14898{ 14899 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 14900 14901 | mov FCARG1a, aword EX->This.value.ptr 14902 | SET_ZVAL_PTR var_addr, FCARG1a 14903 | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX 14904 | GC_ADDREF FCARG1a 14905 14906 return 1; 14907} 14908 14909static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only) 14910{ 14911 if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) { 14912 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14913 if (!JIT_G(current_frame) || 14914 !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { 14915 14916 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14917 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14918 14919 if (!exit_addr) { 14920 return 0; 14921 } 14922 14923 | cmp byte EX->This.u1.v.type, IS_OBJECT 14924 | jne &exit_addr 14925 14926 if (JIT_G(current_frame)) { 14927 TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); 14928 } 14929 } 14930 } else { 14931 14932 | cmp byte EX->This.u1.v.type, IS_OBJECT 14933 | jne >1 14934 |.cold_code 14935 |1: 14936 | SET_EX_OPLINE opline, r0 14937 | jmp ->invalid_this 14938 |.code 14939 } 14940 } 14941 14942 if (!check_only) { 14943 if (!zend_jit_load_this(Dst, opline->result.var)) { 14944 return 0; 14945 } 14946 } 14947 14948 return 1; 14949} 14950 14951static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) 14952{ 14953 uint32_t count; 14954 Bucket *p; 14955 const zend_op *target; 14956 int b; 14957 int32_t exit_point; 14958 const void *exit_addr; 14959 14960 | test r0, r0 14961 if (default_label) { 14962 | jz &default_label 14963 } else if (next_opline) { 14964 | jz >3 14965 } else { 14966 | jz =>default_b 14967 } 14968 | LOAD_ADDR FCARG1a, jumptable 14969 | sub r0, aword [FCARG1a + offsetof(HashTable, arData)] 14970 | mov FCARG1a, (sizeof(Bucket) / sizeof(void*)) 14971 |.if X64 14972 | cqo 14973 |.else 14974 | cdq 14975 |.endif 14976 | idiv FCARG1a 14977 |.if X64 14978 if (!IS_32BIT(dasm_end)) { 14979 | lea FCARG1a, aword [>4] 14980 | jmp aword [FCARG1a + r0] 14981 } else { 14982 | jmp aword [r0 + >4] 14983 } 14984 |.else 14985 | jmp aword [r0 + >4] 14986 |.endif 14987 |.jmp_table 14988 |.align aword 14989 |4: 14990 if (trace_info) { 14991 trace_info->jmp_table_size += zend_hash_num_elements(jumptable); 14992 } 14993 14994 count = jumptable->nNumUsed; 14995 p = jumptable->arData; 14996 do { 14997 if (Z_TYPE(p->val) == IS_UNDEF) { 14998 if (default_label) { 14999 | .aword &default_label 15000 } else if (next_opline) { 15001 | .aword >3 15002 } else { 15003 | .aword =>default_b 15004 } 15005 } else { 15006 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 15007 if (!next_opline) { 15008 b = ssa->cfg.map[target - op_array->opcodes]; 15009 | .aword =>b 15010 } else if (next_opline == target) { 15011 | .aword >3 15012 } else { 15013 exit_point = zend_jit_trace_get_exit_point(target, 0); 15014 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15015 if (!exit_addr) { 15016 return 0; 15017 } 15018 | .aword &exit_addr 15019 } 15020 } 15021 p++; 15022 count--; 15023 } while (count); 15024 |.code 15025 15026 return 1; 15027} 15028 15029static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) 15030{ 15031 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 15032 const zend_op *next_opline = NULL; 15033 15034 if (trace) { 15035 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 15036 ZEND_ASSERT(trace->opline != NULL); 15037 next_opline = trace->opline; 15038 } 15039 15040 if (opline->op1_type == IS_CONST) { 15041 zval *zv = RT_CONSTANT(opline, opline->op1); 15042 zval *jump_zv = NULL; 15043 int b; 15044 15045 if (opline->opcode == ZEND_SWITCH_LONG) { 15046 if (Z_TYPE_P(zv) == IS_LONG) { 15047 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 15048 } 15049 } else if (opline->opcode == ZEND_SWITCH_STRING) { 15050 if (Z_TYPE_P(zv) == IS_STRING) { 15051 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 15052 } 15053 } else if (opline->opcode == ZEND_MATCH) { 15054 if (Z_TYPE_P(zv) == IS_LONG) { 15055 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 15056 } else if (Z_TYPE_P(zv) == IS_STRING) { 15057 jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); 15058 } 15059 } else { 15060 ZEND_UNREACHABLE(); 15061 } 15062 if (next_opline) { 15063 const zend_op *target; 15064 15065 if (jump_zv != NULL) { 15066 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); 15067 } else { 15068 target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 15069 } 15070 ZEND_ASSERT(target == next_opline); 15071 } else { 15072 if (jump_zv != NULL) { 15073 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; 15074 } else { 15075 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; 15076 } 15077 | jmp =>b 15078 } 15079 } else { 15080 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; 15081 uint32_t op1_info = OP1_INFO(); 15082 zend_jit_addr op1_addr = OP1_ADDR(); 15083 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 15084 const zend_op *target; 15085 int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; 15086 int b; 15087 int32_t exit_point; 15088 const void *fallback_label = NULL; 15089 const void *default_label = NULL; 15090 const void *exit_addr; 15091 15092 if (next_opline) { 15093 if (next_opline != opline + 1) { 15094 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 15095 fallback_label = zend_jit_trace_get_exit_addr(exit_point); 15096 if (!fallback_label) { 15097 return 0; 15098 } 15099 } 15100 if (next_opline != default_opline) { 15101 exit_point = zend_jit_trace_get_exit_point(default_opline, 0); 15102 default_label = zend_jit_trace_get_exit_addr(exit_point); 15103 if (!default_label) { 15104 return 0; 15105 } 15106 } 15107 } 15108 15109 if (opline->opcode == ZEND_SWITCH_LONG) { 15110 if (op1_info & MAY_BE_LONG) { 15111 if (op1_info & MAY_BE_REF) { 15112 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1 15113 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr 15114 |.cold_code 15115 |1: 15116 | // ZVAL_DEREF(op) 15117 if (fallback_label) { 15118 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label 15119 } else { 15120 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 15121 } 15122 | GET_ZVAL_PTR FCARG2a, op1_addr 15123 if (fallback_label) { 15124 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, &fallback_label 15125 } else { 15126 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3 15127 } 15128 | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.lval)] 15129 | jmp >2 15130 |.code 15131 |2: 15132 } else { 15133 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 15134 if (fallback_label) { 15135 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label 15136 } else { 15137 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 15138 } 15139 } 15140 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr 15141 } 15142 if (HT_IS_PACKED(jumptable)) { 15143 uint32_t count = jumptable->nNumUsed; 15144 Bucket *p = jumptable->arData; 15145 15146 | cmp FCARG2a, jumptable->nNumUsed 15147 if (default_label) { 15148 | jae &default_label 15149 } else if (next_opline) { 15150 | jae >3 15151 } else { 15152 | jae =>default_b 15153 } 15154 |.if X64 15155 if (!IS_32BIT(dasm_end)) { 15156 | lea r0, aword [>4] 15157 | jmp aword [r0 + FCARG2a * 8] 15158 } else { 15159 | jmp aword [FCARG2a * 8 + >4] 15160 } 15161 |.else 15162 | jmp aword [FCARG2a * 4 + >4] 15163 |.endif 15164 |.jmp_table 15165 |.align aword 15166 |4: 15167 if (trace_info) { 15168 trace_info->jmp_table_size += count; 15169 } 15170 p = jumptable->arData; 15171 do { 15172 if (Z_TYPE(p->val) == IS_UNDEF) { 15173 if (default_label) { 15174 | .aword &default_label 15175 } else if (next_opline) { 15176 | .aword >3 15177 } else { 15178 | .aword =>default_b 15179 } 15180 } else { 15181 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 15182 if (!next_opline) { 15183 b = ssa->cfg.map[target - op_array->opcodes]; 15184 | .aword =>b 15185 } else if (next_opline == target) { 15186 | .aword >3 15187 } else { 15188 exit_point = zend_jit_trace_get_exit_point(target, 0); 15189 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15190 if (!exit_addr) { 15191 return 0; 15192 } 15193 | .aword &exit_addr 15194 } 15195 } 15196 p++; 15197 count--; 15198 } while (count); 15199 |.code 15200 |3: 15201 } else { 15202 | LOAD_ADDR FCARG1a, jumptable 15203 | EXT_CALL zend_hash_index_find, r0 15204 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 15205 return 0; 15206 } 15207 |3: 15208 } 15209 } 15210 } else if (opline->opcode == ZEND_SWITCH_STRING) { 15211 if (op1_info & MAY_BE_STRING) { 15212 if (op1_info & MAY_BE_REF) { 15213 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1 15214 | GET_ZVAL_PTR FCARG2a, op1_addr 15215 |.cold_code 15216 |1: 15217 | // ZVAL_DEREF(op) 15218 if (fallback_label) { 15219 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label 15220 } else { 15221 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 15222 } 15223 | GET_ZVAL_PTR FCARG2a, op1_addr 15224 if (fallback_label) { 15225 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, &fallback_label 15226 } else { 15227 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3 15228 } 15229 | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.ptr)] 15230 | jmp >2 15231 |.code 15232 |2: 15233 } else { 15234 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { 15235 if (fallback_label) { 15236 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label 15237 } else { 15238 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 15239 } 15240 } 15241 | GET_ZVAL_PTR FCARG2a, op1_addr 15242 } 15243 | LOAD_ADDR FCARG1a, jumptable 15244 | EXT_CALL zend_hash_find, r0 15245 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 15246 return 0; 15247 } 15248 |3: 15249 } 15250 } else if (opline->opcode == ZEND_MATCH) { 15251 if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { 15252 if (op1_info & MAY_BE_REF) { 15253 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 15254 | ZVAL_DEREF FCARG2a, op1_info 15255 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 15256 } 15257 | LOAD_ADDR FCARG1a, jumptable 15258 if (op1_info & MAY_BE_LONG) { 15259 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 15260 if (op1_info & MAY_BE_STRING) { 15261 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5 15262 } else if (op1_info & MAY_BE_UNDEF) { 15263 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 15264 } else if (default_label) { 15265 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label 15266 } else if (next_opline) { 15267 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 15268 } else { 15269 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b 15270 } 15271 } 15272 | GET_ZVAL_LVAL ZREG_FCARG2, op1_addr 15273 | EXT_CALL zend_hash_index_find, r0 15274 if (op1_info & MAY_BE_STRING) { 15275 | jmp >2 15276 } 15277 } 15278 if (op1_info & MAY_BE_STRING) { 15279 |5: 15280 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { 15281 if (op1_info & MAY_BE_UNDEF) { 15282 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 15283 } else if (default_label) { 15284 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label 15285 } else if (next_opline) { 15286 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 15287 } else { 15288 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b 15289 } 15290 } 15291 | GET_ZVAL_PTR FCARG2a, op1_addr 15292 | EXT_CALL zend_hash_find, r0 15293 } 15294 |2: 15295 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 15296 return 0; 15297 } 15298 } 15299 if (op1_info & MAY_BE_UNDEF) { 15300 |6: 15301 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { 15302 if (default_label) { 15303 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label 15304 } else if (next_opline) { 15305 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3 15306 } else { 15307 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b 15308 } 15309 } 15310 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 15311 | SET_EX_OPLINE opline, r0 15312 | mov FCARG1d, opline->op1.var 15313 | EXT_CALL zend_jit_undefined_op_helper, r0 15314 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 15315 return 0; 15316 } 15317 } 15318 if (default_label) { 15319 | jmp &default_label 15320 } else if (next_opline) { 15321 | jmp >3 15322 } else { 15323 | jmp =>default_b 15324 } 15325 |3: 15326 } else { 15327 ZEND_UNREACHABLE(); 15328 } 15329 } 15330 return 1; 15331} 15332 15333static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) 15334{ 15335 zend_arg_info *arg_info = &op_array->arg_info[-1]; 15336 ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); 15337 zend_jit_addr op1_addr = OP1_ADDR(); 15338 bool needs_slow_check = 1; 15339 bool slow_check_in_cold = 1; 15340 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 15341 15342 if (type_mask == 0) { 15343 slow_check_in_cold = 0; 15344 } else { 15345 if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { 15346 slow_check_in_cold = 0; 15347 } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { 15348 needs_slow_check = 0; 15349 } else if (is_power_of_two(type_mask)) { 15350 uint32_t type_code = concrete_type(type_mask); 15351 | IF_NOT_ZVAL_TYPE op1_addr, type_code, >6 15352 } else { 15353 | mov edx, 1 15354 | GET_ZVAL_TYPE cl, op1_addr 15355 | shl edx, cl 15356 | test edx, type_mask 15357 | je >6 15358 } 15359 } 15360 if (needs_slow_check) { 15361 if (slow_check_in_cold) { 15362 |.cold_code 15363 |6: 15364 } 15365 | SET_EX_OPLINE opline, r1 15366 if (op1_info & MAY_BE_UNDEF) { 15367 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7 15368 | mov FCARG1a, opline->op1.var 15369 | EXT_CALL zend_jit_undefined_op_helper, FCARG2a 15370 | test r0, r0 15371 | jz ->exception_handler 15372 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 15373 | jmp >8 15374 } 15375 |7: 15376 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 15377 |8: 15378 | mov FCARG2a, EX->func 15379 |.if X64 15380 | LOAD_ADDR CARG3, (ptrdiff_t)arg_info 15381 | mov r0, EX->run_time_cache 15382 | lea CARG4, aword [r0+opline->op2.num] 15383 | EXT_CALL zend_jit_verify_return_slow, r0 15384 |.else 15385 | sub r4, 8 15386 | mov r0, EX->run_time_cache 15387 | add r0, opline->op2.num 15388 | push r0 15389 | push (ptrdiff_t)arg_info 15390 | EXT_CALL zend_jit_verify_return_slow, r0 15391 | add r4, 8 15392 |.endif 15393 if (!zend_jit_check_exception(Dst)) { 15394 return 0; 15395 } 15396 if (slow_check_in_cold) { 15397 | jmp >9 15398 |.code 15399 } 15400 } 15401 |9: 15402 return 1; 15403} 15404 15405static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 15406{ 15407 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15408 15409 // TODO: support for empty() ??? 15410 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 15411 15412 if (op1_info & MAY_BE_REF) { 15413 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) { 15414 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 15415 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 15416 } 15417 | ZVAL_DEREF FCARG1a, op1_info 15418 |1: 15419 } 15420 15421 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { 15422 if (exit_addr) { 15423 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); 15424 } else if (smart_branch_opcode) { 15425 if (smart_branch_opcode == ZEND_JMPNZ) { 15426 | jmp =>target_label 15427 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 15428 | jmp =>target_label2 15429 } 15430 } else { 15431 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 15432 } 15433 } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { 15434 if (exit_addr) { 15435 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); 15436 } else if (smart_branch_opcode) { 15437 if (smart_branch_opcode != ZEND_JMPNZ) { 15438 | jmp =>target_label 15439 } 15440 } else { 15441 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 15442 } 15443 } else { 15444 ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); 15445 | cmp byte [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)], IS_NULL 15446 if (exit_addr) { 15447 if (smart_branch_opcode == ZEND_JMPNZ) { 15448 | jg &exit_addr 15449 } else { 15450 | jle &exit_addr 15451 } 15452 } else if (smart_branch_opcode) { 15453 if (smart_branch_opcode == ZEND_JMPZ) { 15454 | jle =>target_label 15455 } else if (smart_branch_opcode == ZEND_JMPNZ) { 15456 | jg =>target_label 15457 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 15458 | jle =>target_label 15459 | jmp =>target_label2 15460 } else { 15461 ZEND_UNREACHABLE(); 15462 } 15463 } else { 15464 | setg al 15465 | movzx eax, al 15466 | lea eax, [eax + IS_FALSE] 15467 | SET_ZVAL_TYPE_INFO res_addr, eax 15468 } 15469 } 15470 15471 return 1; 15472} 15473 15474static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 15475{ 15476 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15477 15478 if (opline->op1_type == IS_CONST) { 15479 zval *zv = RT_CONSTANT(opline, opline->op1); 15480 15481 | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 15482 if (Z_REFCOUNTED_P(zv)) { 15483 | ADDREF_CONST zv, r0 15484 } 15485 } else { 15486 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 15487 15488 | // ZVAL_COPY(res, value); 15489 | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_FCARG1 15490 if (opline->op1_type == IS_CV) { 15491 | TRY_ADDREF op1_info, ah, FCARG1a 15492 } 15493 } 15494 | // Z_FE_POS_P(res) = 0; 15495 | mov dword [FP + opline->result.var + offsetof(zval, u2.fe_pos)], 0 15496 15497 return 1; 15498} 15499 15500static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr) 15501{ 15502 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 15503 15504 if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) { 15505 /* empty array */ 15506 if (exit_addr) { 15507 if (exit_opcode == ZEND_JMP) { 15508 | jmp &exit_addr 15509 } 15510 } else { 15511 | jmp =>target_label 15512 } 15513 return 1; 15514 } 15515 15516 | // array = EX_VAR(opline->op1.var); 15517 | // fe_ht = Z_ARRVAL_P(array); 15518 | GET_ZVAL_PTR FCARG1a, op1_addr 15519 | // pos = Z_FE_POS_P(array); 15520 | mov eax, dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)] 15521 | // p = fe_ht->arData + pos; 15522 |.if X64 15523 || ZEND_ASSERT(sizeof(Bucket) == 32); 15524 | mov FCARG2d, eax 15525 | shl FCARG2a, 5 15526 |.else 15527 | imul FCARG2a, r0, sizeof(Bucket) 15528 |.endif 15529 | add FCARG2a, aword [FCARG1a + offsetof(zend_array, arData)] 15530 |1: 15531 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 15532 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], eax 15533 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 15534 | // ZEND_VM_CONTINUE(); 15535 if (exit_addr) { 15536 if (exit_opcode == ZEND_JMP) { 15537 | jbe &exit_addr 15538 } else { 15539 | jbe >3 15540 } 15541 } else { 15542 | jbe =>target_label 15543 } 15544 | // pos++; 15545 | add eax, 1 15546 | // value_type = Z_TYPE_INFO_P(value); 15547 | // if (EXPECTED(value_type != IS_UNDEF)) { 15548 if (!exit_addr || exit_opcode == ZEND_JMP) { 15549 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >3 15550 } else { 15551 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, &exit_addr 15552 } 15553 | // p++; 15554 | add FCARG2a, sizeof(Bucket) 15555 | jmp <1 15556 |3: 15557 15558 if (!exit_addr || exit_opcode == ZEND_JMP) { 15559 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0); 15560 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 15561 uint32_t val_info; 15562 15563 | // Z_FE_POS_P(array) = pos + 1; 15564 | mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], eax 15565 15566 if (RETURN_VALUE_USED(opline)) { 15567 zend_jit_addr res_addr = RES_ADDR(); 15568 15569 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) 15570 && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { 15571 | // if (!p->key) { 15572 | cmp aword [FCARG2a + offsetof(Bucket, key)], 0 15573 | jz >2 15574 } 15575 if (op1_info & MAY_BE_ARRAY_KEY_STRING) { 15576 | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); 15577 | mov r0, aword [FCARG2a + offsetof(Bucket, key)] 15578 | SET_ZVAL_PTR res_addr, r0 15579 | test dword [r0 + offsetof(zend_refcounted, gc.u.type_info)], IS_STR_INTERNED 15580 | jz >1 15581 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING 15582 | jmp >3 15583 |1: 15584 | GC_ADDREF r0 15585 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX 15586 15587 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 15588 | jmp >3 15589 |2: 15590 } 15591 } 15592 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 15593 | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); 15594 | mov r0, aword [FCARG2a + offsetof(Bucket, h)] 15595 | SET_ZVAL_LVAL res_addr, r0 15596 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 15597 } 15598 |3: 15599 } 15600 15601 val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); 15602 if (val_info & MAY_BE_ARRAY) { 15603 val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 15604 } 15605 if (op1_info & MAY_BE_ARRAY_OF_REF) { 15606 val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | 15607 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 15608 } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 15609 val_info |= MAY_BE_RC1 | MAY_BE_RCN; 15610 } 15611 15612 if (opline->op2_type == IS_CV) { 15613 | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); 15614 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { 15615 return 0; 15616 } 15617 } else { 15618 | // ZVAL_COPY(res, value); 15619 | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_R0, ZREG_FCARG1 15620 | TRY_ADDREF val_info, ah, FCARG1a 15621 } 15622 } 15623 15624 return 1; 15625} 15626 15627static int zend_jit_fetch_constant(dasm_State **Dst, 15628 const zend_op *opline, 15629 const zend_op_array *op_array, 15630 zend_ssa *ssa, 15631 const zend_ssa_op *ssa_op, 15632 zend_jit_addr res_addr) 15633{ 15634 zval *zv = RT_CONSTANT(opline, opline->op2) + 1; 15635 zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 15636 uint32_t res_info = RES_INFO(); 15637 15638 | // c = CACHED_PTR(opline->extended_value); 15639 | mov FCARG1a, EX->run_time_cache 15640 | mov r0, aword [FCARG1a + opline->extended_value] 15641 | // if (c != NULL) 15642 | test r0, r0 15643 | jz >9 15644 if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) { 15645 | // if (!IS_SPECIAL_CACHE_VAL(c)) 15646 | test r0, CACHE_SPECIAL 15647 | jnz >9 15648 } 15649 |8: 15650 15651 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { 15652 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 15653 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 15654 int32_t exit_point; 15655 const void *exit_addr = NULL; 15656 15657 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 15658 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); 15659 exit_point = zend_jit_trace_get_exit_point(opline+1, 0); 15660 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 15661 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15662 if (!exit_addr) { 15663 return 0; 15664 } 15665 res_info &= ~MAY_BE_GUARD; 15666 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 15667 15668 zend_uchar type = concrete_type(res_info); 15669 15670 if (type < IS_STRING) { 15671 | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr 15672 } else { 15673 | GET_ZVAL_TYPE_INFO edx, const_addr 15674 | IF_NOT_TYPE dl, type, &exit_addr 15675 } 15676 | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_R0, ZREG_R1 15677 if (type < IS_STRING) { 15678 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 15679 | SET_ZVAL_TYPE_INFO res_addr, type 15680 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 15681 return 0; 15682 } 15683 } else { 15684 | SET_ZVAL_TYPE_INFO res_addr, edx 15685 | TRY_ADDREF res_info, dh, r1 15686 } 15687 } else { 15688 | // ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup) 15689 | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_R0, ZREG_R1 15690 | TRY_ADDREF MAY_BE_ANY, ah, r1 15691 } 15692 15693 |.cold_code 15694 |9: 15695 | // SAVE_OPLINE(); 15696 | SET_EX_OPLINE opline, r0 15697 | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); 15698 | LOAD_ADDR FCARG1a, zv 15699 | mov FCARG2a, opline->op1.num 15700 | EXT_CALL zend_jit_get_constant, r0 15701 | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); 15702 | test r0, r0 15703 | jnz <8 15704 | jmp ->exception_handler 15705 |.code 15706 15707 return 1; 15708} 15709 15710static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 15711{ 15712 HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 15713 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15714 15715 ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); 15716 ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); 15717 15718 | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); 15719 | LOAD_ADDR FCARG1a, ht 15720 if (opline->op1_type != IS_CONST) { 15721 | GET_ZVAL_PTR FCARG2a, op1_addr 15722 | EXT_CALL zend_hash_find, r0 15723 } else { 15724 zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); 15725 | LOAD_ADDR FCARG2a, str 15726 | EXT_CALL zend_hash_find_known_hash, r0 15727 } 15728 | test r0, r0 15729 if (exit_addr) { 15730 if (smart_branch_opcode == ZEND_JMPZ) { 15731 | jz &exit_addr 15732 } else { 15733 | jnz &exit_addr 15734 } 15735 } else if (smart_branch_opcode) { 15736 if (smart_branch_opcode == ZEND_JMPZ) { 15737 | jz =>target_label 15738 } else if (smart_branch_opcode == ZEND_JMPNZ) { 15739 | jnz =>target_label 15740 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 15741 | jz =>target_label 15742 | jmp =>target_label2 15743 } else { 15744 ZEND_UNREACHABLE(); 15745 } 15746 } else { 15747 | setnz al 15748 | movzx eax, al 15749 | lea eax, [eax + IS_FALSE] 15750 | SET_ZVAL_TYPE_INFO res_addr, eax 15751 } 15752 15753 return 1; 15754} 15755 15756static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info) 15757{ 15758 uint32_t offset; 15759 15760 offset = (opline->opcode == ZEND_ROPE_INIT) ? 15761 opline->result.var : 15762 opline->op1.var + opline->extended_value * sizeof(zend_string*); 15763 15764 if (opline->op2_type == IS_CONST) { 15765 zval *zv = RT_CONSTANT(opline, opline->op2); 15766 zend_string *str; 15767 15768 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 15769 str = Z_STR_P(zv); 15770 | ADDR_STORE aword [FP + offset], str, r0 15771 } else { 15772 zend_jit_addr op2_addr = OP2_ADDR(); 15773 15774 ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 15775 15776 | GET_ZVAL_PTR r1, op2_addr 15777 | mov aword [FP + offset], r1 15778 if (opline->op2_type == IS_CV) { 15779 | GET_ZVAL_TYPE_INFO eax, op2_addr 15780 | TRY_ADDREF op2_info, ah, r1 15781 } 15782 } 15783 15784 if (opline->opcode == ZEND_ROPE_END) { 15785 zend_jit_addr res_addr = RES_ADDR(); 15786 15787 | lea FCARG1a, [FP + opline->op1.var] 15788 | mov FCARG2d, opline->extended_value 15789 | EXT_CALL zend_jit_rope_end, r0 15790 | SET_ZVAL_PTR res_addr, r0 15791 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX 15792 } 15793 15794 return 1; 15795} 15796 15797static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) 15798{ 15799 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15800 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15801 15802 if (!exit_addr) { 15803 return 0; 15804 } 15805 | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr 15806 15807 return 1; 15808} 15809 15810static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard) 15811{ 15812 zend_jit_addr var_addr = *var_addr_ptr; 15813 uint32_t var_info = *var_info_ptr; 15814 const void *exit_addr = NULL; 15815 15816 if (add_ref_guard || add_type_guard) { 15817 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15818 15819 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15820 if (!exit_addr) { 15821 return 0; 15822 } 15823 } 15824 15825 if (add_ref_guard) { 15826 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr 15827 } 15828 if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { 15829 /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ 15830 if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) { 15831 | LOAD_ZVAL_ADDR FCARG1a, var_addr 15832 } 15833 | EXT_CALL zend_jit_unref_helper, r0 15834 } else { 15835 | GET_ZVAL_PTR FCARG1a, var_addr 15836 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val)); 15837 *var_addr_ptr = var_addr; 15838 } 15839 15840 if (var_type != IS_UNKNOWN) { 15841 var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); 15842 } 15843 if (add_type_guard 15844 && var_type != IS_UNKNOWN 15845 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 15846 | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr 15847 15848 ZEND_ASSERT(var_info & (1 << var_type)); 15849 if (var_type < IS_STRING) { 15850 var_info = (1 << var_type); 15851 } else if (var_type != IS_ARRAY) { 15852 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 15853 } else { 15854 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 15855 } 15856 15857 *var_info_ptr = var_info; 15858 } else { 15859 var_info &= ~MAY_BE_REF; 15860 *var_info_ptr = var_info; 15861 } 15862 *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */ 15863 15864 return 1; 15865} 15866 15867static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard) 15868{ 15869 zend_jit_addr var_addr = *var_addr_ptr; 15870 uint32_t var_info = *var_info_ptr; 15871 int32_t exit_point; 15872 const void *exit_addr; 15873 15874 if (add_indirect_guard) { 15875 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15876 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15877 15878 if (!exit_addr) { 15879 return 0; 15880 } 15881 | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr 15882 | GET_ZVAL_PTR FCARG1a, var_addr 15883 } else { 15884 /* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */ 15885 if (opline->op1_type != IS_VAR || 15886 (opline-1)->result_type != IS_VAR || 15887 (opline-1)->result.var != opline->op1.var || 15888 (opline-1)->op1_type == IS_VAR || 15889 (opline-1)->op2_type == IS_VAR || 15890 (opline-1)->op2_type == IS_TMP_VAR) { 15891 | GET_ZVAL_PTR FCARG1a, var_addr 15892 } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { 15893 | mov FCARG1a, r0 15894 } 15895 } 15896 *var_info_ptr &= ~MAY_BE_INDIRECT; 15897 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0); 15898 *var_addr_ptr = var_addr; 15899 15900 if (var_type != IS_UNKNOWN) { 15901 var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); 15902 } 15903 if (!(var_type & IS_TRACE_REFERENCE) 15904 && var_type != IS_UNKNOWN 15905 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 15906 exit_point = zend_jit_trace_get_exit_point(opline, 0); 15907 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15908 15909 if (!exit_addr) { 15910 return 0; 15911 } 15912 15913 | IF_NOT_Z_TYPE FCARG1a, var_type, &exit_addr 15914 15915 //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); 15916 ZEND_ASSERT(var_info & (1 << var_type)); 15917 if (var_type < IS_STRING) { 15918 var_info = (1 << var_type); 15919 } else if (var_type != IS_ARRAY) { 15920 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 15921 } else { 15922 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 15923 } 15924 15925 *var_info_ptr = var_info; 15926 } 15927 15928 return 1; 15929} 15930 15931static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) 15932{ 15933 if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { 15934 return 0; 15935 } 15936 15937 switch (opline->opcode) { 15938 case ZEND_QM_ASSIGN: 15939 case ZEND_SEND_VAR: 15940 case ZEND_ASSIGN: 15941 case ZEND_PRE_INC: 15942 case ZEND_PRE_DEC: 15943 case ZEND_POST_INC: 15944 case ZEND_POST_DEC: 15945 return 1; 15946 case ZEND_ADD: 15947 case ZEND_SUB: 15948 case ZEND_MUL: 15949 case ZEND_BW_OR: 15950 case ZEND_BW_AND: 15951 case ZEND_BW_XOR: 15952 if (def_var == ssa_op->result_def && 15953 use_var == ssa_op->op1_use) { 15954 return 1; 15955 } 15956 break; 15957 default: 15958 break; 15959 } 15960 return 0; 15961} 15962 15963static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) 15964{ 15965 uint32_t op1_info, op2_info; 15966 15967 switch (opline->opcode) { 15968 case ZEND_SEND_VAR: 15969 case ZEND_SEND_VAL: 15970 case ZEND_SEND_VAL_EX: 15971 return (opline->op2_type != IS_CONST); 15972 case ZEND_QM_ASSIGN: 15973 case ZEND_IS_SMALLER: 15974 case ZEND_IS_SMALLER_OR_EQUAL: 15975 case ZEND_IS_EQUAL: 15976 case ZEND_IS_NOT_EQUAL: 15977 case ZEND_IS_IDENTICAL: 15978 case ZEND_IS_NOT_IDENTICAL: 15979 case ZEND_CASE: 15980 return 1; 15981 case ZEND_RETURN: 15982 return (op_array->type != ZEND_EVAL_CODE && op_array->function_name); 15983 case ZEND_ASSIGN: 15984 op1_info = OP1_INFO(); 15985 op2_info = OP2_INFO(); 15986 return 15987 opline->op1_type == IS_CV && 15988 !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) && 15989 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); 15990 case ZEND_ADD: 15991 case ZEND_SUB: 15992 case ZEND_MUL: 15993 op1_info = OP1_INFO(); 15994 op2_info = OP2_INFO(); 15995 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); 15996 case ZEND_BW_OR: 15997 case ZEND_BW_AND: 15998 case ZEND_BW_XOR: 15999 case ZEND_SL: 16000 case ZEND_SR: 16001 case ZEND_MOD: 16002 op1_info = OP1_INFO(); 16003 op2_info = OP2_INFO(); 16004 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); 16005 case ZEND_PRE_INC: 16006 case ZEND_PRE_DEC: 16007 case ZEND_POST_INC: 16008 case ZEND_POST_DEC: 16009 op1_info = OP1_INFO(); 16010 op2_info = OP1_DEF_INFO(); 16011 return opline->op1_type == IS_CV 16012 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)) 16013 && (op2_info & MAY_BE_LONG); 16014 case ZEND_STRLEN: 16015 op1_info = OP1_INFO(); 16016 return (opline->op1_type & (IS_CV|IS_CONST)) 16017 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING; 16018 case ZEND_COUNT: 16019 op1_info = OP1_INFO(); 16020 return (opline->op1_type & (IS_CV|IS_CONST)) 16021 && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY; 16022 case ZEND_JMPZ: 16023 case ZEND_JMPNZ: 16024 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 16025 if (!ssa->cfg.map) { 16026 return 0; 16027 } 16028 if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start && 16029 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { 16030 return 0; 16031 } 16032 } 16033 ZEND_FALLTHROUGH; 16034 case ZEND_BOOL: 16035 case ZEND_BOOL_NOT: 16036 case ZEND_JMPZNZ: 16037 case ZEND_JMPZ_EX: 16038 case ZEND_JMPNZ_EX: 16039 return 1; 16040 case ZEND_FETCH_CONSTANT: 16041 return 1; 16042 case ZEND_FETCH_DIM_R: 16043 op1_info = OP1_INFO(); 16044 op2_info = OP2_INFO(); 16045 if (trace 16046 && trace->op1_type != IS_UNKNOWN 16047 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { 16048 op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); 16049 } 16050 return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && 16051 (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && 16052 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || 16053 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && 16054 (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); 16055 } 16056 return 0; 16057} 16058 16059static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) 16060{ 16061 if (ssa->vars[var].no_val) { 16062 /* we don't need the value */ 16063 return 0; 16064 } 16065 16066 if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { 16067 /* Disable global register allocation, 16068 * register allocation for SSA variables connected through Phi functions 16069 */ 16070 if (ssa->vars[var].definition_phi) { 16071 return 0; 16072 } 16073 if (ssa->vars[var].phi_use_chain) { 16074 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; 16075 do { 16076 if (!ssa->vars[phi->ssa_var].no_val) { 16077 return 0; 16078 } 16079 phi = zend_ssa_next_use_phi(ssa, var, phi); 16080 } while (phi); 16081 } 16082 } 16083 16084 if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && 16085 ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { 16086 /* bad type */ 16087 return 0; 16088 } 16089 16090 return 1; 16091} 16092 16093static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) 16094{ 16095 if (!zend_jit_var_supports_reg(ssa, var)) { 16096 return 0; 16097 } 16098 16099 if (ssa->vars[var].definition >= 0) { 16100 uint32_t def = ssa->vars[var].definition; 16101 if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { 16102 return 0; 16103 } 16104 } 16105 16106 if (ssa->vars[var].use_chain >= 0) { 16107 int use = ssa->vars[var].use_chain; 16108 16109 do { 16110 if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && 16111 !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { 16112 return 0; 16113 } 16114 use = zend_ssa_next_use(ssa->ops, var, use); 16115 } while (use >= 0); 16116 } 16117 16118 return 1; 16119} 16120 16121static bool zend_needs_extra_reg_for_const(const zend_op *opline, zend_uchar op_type, znode_op op) 16122{ 16123|.if X64 16124|| if (op_type == IS_CONST) { 16125|| zval *zv = RT_CONSTANT(opline, op); 16126|| if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0 && !IS_SIGNED_32BIT(zv)) { 16127|| return 1; 16128|| } else if (Z_TYPE_P(zv) == IS_LONG && !IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 16129|| return 1; 16130|| } 16131|| } 16132|.endif 16133 return 0; 16134} 16135 16136static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 16137{ 16138 uint32_t op1_info, op2_info; 16139 16140 switch (opline->opcode) { 16141 case ZEND_FETCH_DIM_R: 16142 op1_info = OP1_INFO(); 16143 op2_info = OP2_INFO(); 16144 if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && 16145 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || 16146 ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && 16147 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { 16148 return ZEND_REGSET(ZREG_FCARG1); 16149 } 16150 break; 16151 default: 16152 break; 16153 } 16154 16155 return ZEND_REGSET_EMPTY; 16156} 16157 16158static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) 16159{ 16160 uint32_t op1_info, op2_info, res_info; 16161 zend_regset regset = ZEND_REGSET_SCRATCH; 16162 16163 switch (opline->opcode) { 16164 case ZEND_NOP: 16165 case ZEND_OP_DATA: 16166 case ZEND_JMP: 16167 case ZEND_RETURN: 16168 regset = ZEND_REGSET_EMPTY; 16169 break; 16170 case ZEND_QM_ASSIGN: 16171 if (ssa_op->op1_def == current_var || 16172 ssa_op->result_def == current_var) { 16173 regset = ZEND_REGSET_EMPTY; 16174 break; 16175 } 16176 /* break missing intentionally */ 16177 case ZEND_SEND_VAL: 16178 case ZEND_SEND_VAL_EX: 16179 if (opline->op2_type == IS_CONST) { 16180 break; 16181 } 16182 if (ssa_op->op1_use == current_var) { 16183 regset = ZEND_REGSET(ZREG_R0); 16184 break; 16185 } 16186 op1_info = OP1_INFO(); 16187 if (!(op1_info & MAY_BE_UNDEF)) { 16188 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 16189 regset = ZEND_REGSET(ZREG_XMM0); 16190 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 16191 regset = ZEND_REGSET(ZREG_R0); 16192 } else { 16193 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 16194 } 16195 } 16196 break; 16197 case ZEND_SEND_VAR: 16198 if (opline->op2_type == IS_CONST) { 16199 break; 16200 } 16201 if (ssa_op->op1_use == current_var || 16202 ssa_op->op1_def == current_var) { 16203 regset = ZEND_REGSET_EMPTY; 16204 break; 16205 } 16206 op1_info = OP1_INFO(); 16207 if (!(op1_info & MAY_BE_UNDEF)) { 16208 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 16209 regset = ZEND_REGSET(ZREG_XMM0); 16210 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 16211 } else { 16212 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 16213 if (op1_info & MAY_BE_REF) { 16214 ZEND_REGSET_INCL(regset, ZREG_R1); 16215 } 16216 } 16217 } 16218 break; 16219 case ZEND_ASSIGN: 16220 if (ssa_op->op2_use == current_var || 16221 ssa_op->op2_def == current_var || 16222 ssa_op->op1_def == current_var || 16223 ssa_op->result_def == current_var) { 16224 regset = ZEND_REGSET_EMPTY; 16225 break; 16226 } 16227 op1_info = OP1_INFO(); 16228 op2_info = OP2_INFO(); 16229 if (opline->op1_type == IS_CV 16230 && !(op2_info & MAY_BE_UNDEF) 16231 && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 16232 if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 16233 regset = ZEND_REGSET(ZREG_XMM0); 16234 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 16235 regset = ZEND_REGSET(ZREG_R0); 16236 } else { 16237 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 16238 } 16239 } 16240 break; 16241 case ZEND_PRE_INC: 16242 case ZEND_PRE_DEC: 16243 case ZEND_POST_INC: 16244 case ZEND_POST_DEC: 16245 if (ssa_op->op1_use == current_var || 16246 ssa_op->op1_def == current_var || 16247 ssa_op->result_def == current_var) { 16248 regset = ZEND_REGSET_EMPTY; 16249 break; 16250 } 16251 op1_info = OP1_INFO(); 16252 if (opline->op1_type == IS_CV 16253 && (op1_info & MAY_BE_LONG) 16254 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16255 regset = ZEND_REGSET_EMPTY; 16256 if (op1_info & MAY_BE_DOUBLE) { 16257 regset = ZEND_REGSET(ZREG_XMM0); 16258 } 16259 if (opline->result_type != IS_UNUSED && (op1_info & MAY_BE_LONG)) { 16260 ZEND_REGSET_INCL(regset, ZREG_R1); 16261 } 16262 } 16263 break; 16264 case ZEND_ADD: 16265 case ZEND_SUB: 16266 case ZEND_MUL: 16267 op1_info = OP1_INFO(); 16268 op2_info = OP2_INFO(); 16269 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 16270 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16271 16272 regset = ZEND_REGSET_EMPTY; 16273 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 16274 if (ssa_op->result_def != current_var && 16275 (ssa_op->op1_use != current_var || !last_use)) { 16276 ZEND_REGSET_INCL(regset, ZREG_R0); 16277 } 16278 res_info = RES_INFO(); 16279 if (res_info & MAY_BE_DOUBLE) { 16280 ZEND_REGSET_INCL(regset, ZREG_R0); 16281 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16282 ZEND_REGSET_INCL(regset, ZREG_XMM1); 16283 } else if (res_info & MAY_BE_GUARD) { 16284 ZEND_REGSET_INCL(regset, ZREG_R0); 16285 } 16286 } 16287 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 16288 if (opline->op1_type == IS_CONST) { 16289 ZEND_REGSET_INCL(regset, ZREG_R0); 16290 } 16291 if (ssa_op->result_def != current_var) { 16292 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16293 } 16294 } 16295 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 16296 if (opline->op2_type == IS_CONST) { 16297 ZEND_REGSET_INCL(regset, ZREG_R0); 16298 } 16299 if (zend_is_commutative(opline->opcode)) { 16300 if (ssa_op->result_def != current_var) { 16301 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16302 } 16303 } else { 16304 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16305 if (ssa_op->result_def != current_var && 16306 (ssa_op->op1_use != current_var || !last_use)) { 16307 ZEND_REGSET_INCL(regset, ZREG_XMM1); 16308 } 16309 } 16310 } 16311 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 16312 if (ssa_op->result_def != current_var && 16313 (ssa_op->op1_use != current_var || !last_use) && 16314 (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { 16315 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16316 } 16317 } 16318 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16319 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16320 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 16321 ZEND_REGSET_INCL(regset, ZREG_R0); 16322 } else { 16323 ZEND_REGSET_INCL(regset, ZREG_R1); 16324 } 16325 } 16326 } 16327 break; 16328 case ZEND_BW_OR: 16329 case ZEND_BW_AND: 16330 case ZEND_BW_XOR: 16331 op1_info = OP1_INFO(); 16332 op2_info = OP2_INFO(); 16333 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16334 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16335 regset = ZEND_REGSET_EMPTY; 16336 if (ssa_op->result_def != current_var && 16337 (ssa_op->op1_use != current_var || !last_use)) { 16338 ZEND_REGSET_INCL(regset, ZREG_R0); 16339 } 16340 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16341 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16342 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 16343 ZEND_REGSET_INCL(regset, ZREG_R0); 16344 } else { 16345 ZEND_REGSET_INCL(regset, ZREG_R1); 16346 } 16347 } 16348 } 16349 break; 16350 case ZEND_SL: 16351 case ZEND_SR: 16352 op1_info = OP1_INFO(); 16353 op2_info = OP2_INFO(); 16354 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16355 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16356bw_op: 16357 regset = ZEND_REGSET_EMPTY; 16358 if (ssa_op->result_def != current_var && 16359 (ssa_op->op1_use != current_var || !last_use)) { 16360 ZEND_REGSET_INCL(regset, ZREG_R0); 16361 } 16362 if (opline->op2_type != IS_CONST && ssa_op->op2_use != current_var) { 16363 ZEND_REGSET_INCL(regset, ZREG_R1); 16364 } 16365 } 16366 break; 16367 case ZEND_MOD: 16368 op1_info = OP1_INFO(); 16369 op2_info = OP2_INFO(); 16370 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16371 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16372 if (opline->op2_type == IS_CONST && 16373 opline->op1_type != IS_CONST && 16374 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG && 16375 zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2))) && 16376 OP1_HAS_RANGE() && 16377 OP1_MIN_RANGE() >= 0) { 16378 /* MOD is going to be optimized into AND */ 16379 goto bw_op; 16380 } else { 16381 regset = ZEND_REGSET_EMPTY; 16382 ZEND_REGSET_INCL(regset, ZREG_R0); 16383 ZEND_REGSET_INCL(regset, ZREG_R2); 16384 if (opline->op2_type == IS_CONST) { 16385 ZEND_REGSET_INCL(regset, ZREG_R1); 16386 } 16387 } 16388 } 16389 break; 16390 case ZEND_IS_SMALLER: 16391 case ZEND_IS_SMALLER_OR_EQUAL: 16392 case ZEND_IS_EQUAL: 16393 case ZEND_IS_NOT_EQUAL: 16394 case ZEND_IS_IDENTICAL: 16395 case ZEND_IS_NOT_IDENTICAL: 16396 case ZEND_CASE: 16397 op1_info = OP1_INFO(); 16398 op2_info = OP2_INFO(); 16399 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 16400 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16401 regset = ZEND_REGSET_EMPTY; 16402 if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { 16403 ZEND_REGSET_INCL(regset, ZREG_R0); 16404 } 16405 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && 16406 opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { 16407 if (ssa_op->op1_use != current_var && 16408 ssa_op->op2_use != current_var) { 16409 ZEND_REGSET_INCL(regset, ZREG_R0); 16410 } 16411 } 16412 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 16413 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16414 } 16415 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 16416 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16417 } 16418 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 16419 if (ssa_op->op1_use != current_var && 16420 ssa_op->op2_use != current_var) { 16421 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16422 } 16423 } 16424 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16425 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16426 ZEND_REGSET_INCL(regset, ZREG_R0); 16427 } 16428 } 16429 break; 16430 case ZEND_BOOL: 16431 case ZEND_BOOL_NOT: 16432 case ZEND_JMPZ: 16433 case ZEND_JMPNZ: 16434 case ZEND_JMPZNZ: 16435 case ZEND_JMPZ_EX: 16436 case ZEND_JMPNZ_EX: 16437 op1_info = OP1_INFO(); 16438 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16439 regset = ZEND_REGSET_EMPTY; 16440 if (op1_info & MAY_BE_DOUBLE) { 16441 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16442 } 16443 if (opline->opcode == ZEND_BOOL || 16444 opline->opcode == ZEND_BOOL_NOT || 16445 opline->opcode == ZEND_JMPZ_EX || 16446 opline->opcode == ZEND_JMPNZ_EX) { 16447 ZEND_REGSET_INCL(regset, ZREG_R0); 16448 } 16449 } 16450 break; 16451 case ZEND_DO_UCALL: 16452 case ZEND_DO_FCALL: 16453 case ZEND_DO_FCALL_BY_NAME: 16454 case ZEND_INCLUDE_OR_EVAL: 16455 case ZEND_GENERATOR_CREATE: 16456 case ZEND_YIELD: 16457 case ZEND_YIELD_FROM: 16458 regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); 16459 break; 16460 default: 16461 break; 16462 } 16463 16464 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 16465 if (ssa_op == ssa->ops 16466 && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL 16467 && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { 16468 ZEND_REGSET_INCL(regset, ZREG_R0); 16469 ZEND_REGSET_INCL(regset, ZREG_R1); 16470 } 16471 } 16472 16473 /* %r0 is used to check EG(vm_interrupt) */ 16474 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 16475 if (ssa_op == ssa->ops 16476 && (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP || 16477 JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) { 16478#if ZTS 16479 ZEND_REGSET_INCL(regset, ZREG_R0); 16480#else 16481 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { 16482 ZEND_REGSET_INCL(regset, ZREG_R0); 16483 } 16484#endif 16485 } 16486 } else { 16487 uint32_t b = ssa->cfg.map[ssa_op - ssa->ops]; 16488 16489 if ((ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) != 0 16490 && ssa->cfg.blocks[b].start == ssa_op - ssa->ops) { 16491#if ZTS 16492 ZEND_REGSET_INCL(regset, ZREG_R0); 16493#else 16494 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { 16495 ZEND_REGSET_INCL(regset, ZREG_R0); 16496 } 16497#endif 16498 } 16499 } 16500 16501 return regset; 16502} 16503 16504/* 16505 * Local variables: 16506 * tab-width: 4 16507 * c-basic-offset: 4 16508 * indent-tabs-mode: t 16509 * End: 16510 */ 16511
