1/* 2 * +----------------------------------------------------------------------+ 3 * | Zend JIT | 4 * +----------------------------------------------------------------------+ 5 * | Copyright (c) The PHP Group | 6 * +----------------------------------------------------------------------+ 7 * | This source file is subject to version 3.01 of the PHP license, | 8 * | that is bundled with this package in the file LICENSE, and is | 9 * | available through the world-wide-web at the following url: | 10 * | http://www.php.net/license/3_01.txt | 11 * | If you did not receive a copy of the PHP license and are unable to | 12 * | obtain it through the world-wide-web, please send a note to | 13 * | license@php.net so we can mail you a copy immediately. | 14 * +----------------------------------------------------------------------+ 15 * | Authors: Dmitry Stogov <dmitry@php.net> | 16 * | Xinchen Hui <laruence@php.net> | 17 * +----------------------------------------------------------------------+ 18 */ 19 20|.if X64 21 |.arch x64 22|.else 23 |.arch x86 24|.endif 25 26|.if X64WIN 27 |.define FP, r14 28 |.define IP, r15 29 |.define IPl, r15d 30 |.define RX, r15 // the same as VM IP reused as a general purpose reg 31 |.define CARG1, rcx // x64/POSIX C call arguments. 32 |.define CARG2, rdx 33 |.define CARG3, r8 34 |.define CARG4, r9 35 |.define CARG1d, ecx 36 |.define CARG2d, edx 37 |.define CARG3d, r8d 38 |.define CARG4d, r9d 39 |.define FCARG1a, CARG1 // Simulate x86 fastcall. 40 |.define FCARG2a, CARG2 41 |.define FCARG1d, CARG1d 42 |.define FCARG2d, CARG2d 43 |.define SPAD, 0x58 // padding for CPU stack alignment 44 |.define NR_SPAD, 0x58 // padding for CPU stack alignment 45 |.define T3, [r4+0x50] // Used to store old value of IP 46 |.define T2, [r4+0x48] // Used to store old value of FP 47 |.define T1, [r4+0x40] 48 |.define A6, [r4+0x28] // preallocated slot for 6-th argument 49 |.define A5, [r4+0x20] // preallocated slot for 5-th argument 50|.elif X64 51 |.define FP, r14 52 |.define IP, r15 53 |.define IPl, r15d 54 |.define RX, r15 // the same as VM IP reused as a general purpose reg 55 |.define CARG1, rdi // x64/POSIX C call arguments. 56 |.define CARG2, rsi 57 |.define CARG3, rdx 58 |.define CARG4, rcx 59 |.define CARG5, r8 60 |.define CARG6, r9 61 |.define CARG1d, edi 62 |.define CARG2d, esi 63 |.define CARG3d, edx 64 |.define CARG4d, ecx 65 |.define CARG5d, r8d 66 |.define CARG6d, r9d 67 |.define FCARG1a, CARG1 // Simulate x86 fastcall. 68 |.define FCARG2a, CARG2 69 |.define FCARG1d, CARG1d 70 |.define FCARG2d, CARG2d 71 |.define SPAD, 0x18 // padding for CPU stack alignment 72 |.define NR_SPAD, 0x28 // padding for CPU stack alignment 73 |.define T3, [r4+0x20] // Used to store old value of IP (CALL VM only) 74 |.define T2, [r4+0x18] // Used to store old value of FP (CALL VM only) 75 |.define T1, [r4] 76|.else 77 |.define FP, esi 78 |.define IP, edi 79 |.define IPl, edi 80 |.define RX, edi // the same as VM IP reused as a general purpose reg 81 |.define FCARG1a, ecx // x86 fastcall arguments. 82 |.define FCARG2a, edx 83 |.define FCARG1d, ecx 84 |.define FCARG2d, edx 85 |.define SPAD, 0x1c // padding for CPU stack alignment 86 |.define NR_SPAD, 0x1c // padding for CPU stack alignment 87 |.define T3, [r4+0x18] // Used to store old value of IP (CALL VM only) 88 |.define T2, [r4+0x14] // Used to store old value of FP (CALL VM only) 89 |.define T1, [r4] 90 |.define A4, [r4+0xC] // preallocated slots for arguments of "cdecl" functions (intersect with T1) 91 |.define A3, [r4+0x8] 92 |.define A2, [r4+0x4] 93 |.define A1, [r4] 94|.endif 95 96|.define HYBRID_SPAD, 16 // padding for stack alignment 97 98#ifdef _WIN64 99# define TMP_ZVAL_OFFSET 0x20 100#else 101# define TMP_ZVAL_OFFSET 0 102#endif 103 104#define DASM_ALIGNMENT 16 105 106/* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to 107 * guarantee proper alignment of 128-bit SSE data allocated on stack. 108 * With broken alignment any execution of SSE code, including calls to 109 * memcpy() and others, may lead to crash. 110 */ 111 112#include "Zend/zend_cpuinfo.h" 113#include "jit/zend_jit_x86.h" 114 115#ifdef HAVE_VALGRIND 116# include <valgrind/valgrind.h> 117#endif 118 119/* The generated code may contain tautological comparisons, ignore them. */ 120#if defined(__clang__) 121# pragma clang diagnostic push 122# pragma clang diagnostic ignored "-Wtautological-compare" 123# pragma clang diagnostic ignored "-Wstring-compare" 124#endif 125 126const char* zend_reg_name[] = { 127#if defined(__x86_64__) || defined(_M_X64) 128 "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", 129 "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", 130 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", 131 "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15" 132#else 133 "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", 134 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" 135#endif 136}; 137 138#ifdef HAVE_GCC_GLOBAL_REGS 139# define GCC_GLOBAL_REGS 1 140#else 141# define GCC_GLOBAL_REGS 0 142#endif 143 144#if ZTS 145static size_t tsrm_ls_cache_tcb_offset = 0; 146static size_t tsrm_tls_index; 147static size_t tsrm_tls_offset; 148#endif 149 150/* By default avoid JITing inline handlers if it does not seem profitable due to lack of 151 * type information. Disabling this option allows testing some JIT handlers in the 152 * presence of try/catch blocks, which prevent SSA construction. */ 153#ifndef PROFITABILITY_CHECKS 154# define PROFITABILITY_CHECKS 1 155#endif 156 157|.type EX, zend_execute_data, FP 158|.type OP, zend_op 159|.type ZVAL, zval 160 161|.actionlist dasm_actions 162 163|.globals zend_lb 164static void* dasm_labels[zend_lb_MAX]; 165 166|.section code, cold_code, jmp_table 167 168#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) 169 170#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) 171 172#define BP_JIT_IS 6 173 174 175#define CAN_USE_AVX() (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) 176 177|.macro ADD_HYBRID_SPAD 178||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 179| add r4, HYBRID_SPAD 180||#endif 181|.endmacro 182 183|.macro SUB_HYBRID_SPAD 184||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 185| sub r4, HYBRID_SPAD 186||#endif 187|.endmacro 188 189|.macro LOAD_ADDR, reg, addr 190| .if X64 191|| if (IS_SIGNED_32BIT(addr)) { 192| mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 <imm-32-bit> 193|| } else { 194| mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 <imm-64-bit> 195|| } 196| .else 197| mov reg, ((ptrdiff_t)addr) 198| .endif 199|.endmacro 200 201|.macro LOAD_TSRM_CACHE, reg 202| .if X64WIN 203| gs 204| mov reg, aword [0x58] 205| mov reg, aword [reg+tsrm_tls_index] 206| mov reg, aword [reg+tsrm_tls_offset] 207| .elif WIN 208| fs 209| mov reg, aword [0x2c] 210| mov reg, aword [reg+tsrm_tls_index] 211| mov reg, aword [reg+tsrm_tls_offset] 212| .elif X64APPLE 213| gs 214|| if (tsrm_ls_cache_tcb_offset) { 215| mov reg, aword [tsrm_ls_cache_tcb_offset] 216|| } else { 217| mov reg, aword [tsrm_tls_index] 218| mov reg, aword [reg+tsrm_tls_offset] 219|| } 220| .elif X64 221| fs 222|| if (tsrm_ls_cache_tcb_offset) { 223| mov reg, aword [tsrm_ls_cache_tcb_offset] 224|| } else { 225| mov reg, [0x8] 226| mov reg, aword [reg+tsrm_tls_index] 227| mov reg, aword [reg+tsrm_tls_offset] 228|| } 229| .else 230| gs 231|| if (tsrm_ls_cache_tcb_offset) { 232| mov reg, aword [tsrm_ls_cache_tcb_offset] 233|| } else { 234| mov reg, [0x4] 235| mov reg, aword [reg+tsrm_tls_index] 236| mov reg, aword [reg+tsrm_tls_offset] 237|| } 238| .endif 239|.endmacro 240 241|.macro LOAD_ADDR_ZTS, reg, struct, field 242| .if ZTS 243| LOAD_TSRM_CACHE reg 244| lea reg, aword [reg + (struct.._offset + offsetof(zend_..struct, field))] 245| .else 246| LOAD_ADDR reg, &struct.field 247| .endif 248|.endmacro 249 250|.macro ADDR_OP1, addr_ins, addr, tmp_reg 251| .if X64 252|| if (IS_SIGNED_32BIT(addr)) { 253| addr_ins ((ptrdiff_t)addr) 254|| } else { 255| mov64 tmp_reg, ((ptrdiff_t)addr) 256| addr_ins tmp_reg 257|| } 258| .else 259| addr_ins ((ptrdiff_t)addr) 260| .endif 261|.endmacro 262 263|.macro ADDR_OP2_2, addr_ins, op1, addr, tmp_reg 264| .if X64 265|| if (IS_SIGNED_32BIT(addr)) { 266| addr_ins op1, ((ptrdiff_t)addr) 267|| } else { 268| mov64 tmp_reg, ((ptrdiff_t)addr) 269| addr_ins op1, tmp_reg 270|| } 271| .else 272| addr_ins op1, ((ptrdiff_t)addr) 273| .endif 274|.endmacro 275 276|.macro PUSH_ADDR, addr, tmp_reg 277| ADDR_OP1 push, addr, tmp_reg 278|.endmacro 279 280|.macro PUSH_ADDR_ZTS, struct, field, tmp_reg 281| .if ZTS 282| LOAD_TSRM_CACHE tmp_reg 283| lea tmp_reg, aword [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))] 284| push tmp_reg 285| .else 286| ADDR_OP1 push, &struct.field, tmp_reg 287| .endif 288|.endmacro 289 290|.macro MEM_OP1, mem_ins, prefix, addr, tmp_reg 291| .if X64 292|| if (IS_SIGNED_32BIT(addr)) { 293| mem_ins prefix [addr] 294|| } else { 295| mov64 tmp_reg, ((ptrdiff_t)addr) 296| mem_ins prefix [tmp_reg] 297|| } 298| .else 299| mem_ins prefix [addr] 300| .endif 301|.endmacro 302 303|.macro MEM_OP2_1, mem_ins, prefix, addr, op2, tmp_reg 304| .if X64 305|| if (IS_SIGNED_32BIT(addr)) { 306| mem_ins prefix [addr], op2 307|| } else { 308| mov64 tmp_reg, ((ptrdiff_t)addr) 309| mem_ins prefix [tmp_reg], op2 310|| } 311| .else 312| mem_ins prefix [addr], op2 313| .endif 314|.endmacro 315 316|.macro MEM_OP2_2, mem_ins, op1, prefix, addr, tmp_reg 317| .if X64 318|| if (IS_SIGNED_32BIT(addr)) { 319| mem_ins op1, prefix [addr] 320|| } else { 321| mov64 tmp_reg, ((ptrdiff_t)addr) 322| mem_ins op1, prefix [tmp_reg] 323|| } 324| .else 325| mem_ins op1, prefix [addr] 326| .endif 327|.endmacro 328 329|.macro MEM_OP2_1_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg 330| .if ZTS 331| LOAD_TSRM_CACHE tmp_reg 332| mem_ins prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))], op2 333| .else 334| MEM_OP2_1 mem_ins, prefix, &struct.field, op2, tmp_reg 335| .endif 336|.endmacro 337 338|.macro MEM_OP2_2_ZTS, mem_ins, op1, prefix, struct, field, tmp_reg 339| .if ZTS 340| LOAD_TSRM_CACHE tmp_reg 341| mem_ins op1, prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))] 342| .else 343| MEM_OP2_2 mem_ins, op1, prefix, &struct.field, tmp_reg 344| .endif 345|.endmacro 346 347|.macro MEM_OP3_3, mem_ins, op1, op2, prefix, addr, tmp_reg 348| .if X64 349|| if (IS_SIGNED_32BIT(addr)) { 350| mem_ins op1, op2, prefix [addr] 351|| } else { 352| mov64 tmp_reg, ((ptrdiff_t)addr) 353| mem_ins op1, op2, prefix [tmp_reg] 354|| } 355| .else 356| mem_ins op1, op2, prefix [addr] 357| .endif 358|.endmacro 359 360|.macro LOAD_BASE_ADDR, reg, base, offset 361|| if (offset) { 362| lea reg, qword [Ra(base)+offset] 363|| } else { 364| mov reg, Ra(base) 365|| } 366|.endmacro 367 368|.macro PUSH_BASE_ADDR, base, offset, tmp_reg 369|| if (offset) { 370| lea tmp_reg, qword [Ra(base)+offset] 371| push tmp_reg 372|| } else { 373| push Ra(base) 374|| } 375|.endmacro 376 377|.macro EXT_CALL, func, tmp_reg 378| .if X64 379|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) { 380| call qword &func 381|| } else { 382| LOAD_ADDR tmp_reg, func 383| call tmp_reg 384|| } 385| .else 386| call dword &func 387| .endif 388|.endmacro 389 390|.macro EXT_JMP, func, tmp_reg 391| .if X64 392|| if (IS_32BIT(dasm_end) && IS_32BIT(func)) { 393| jmp qword &func 394|| } else { 395| LOAD_ADDR tmp_reg, func 396| jmp tmp_reg 397|| } 398| .else 399| jmp dword &func 400| .endif 401|.endmacro 402 403|.macro SAVE_IP 404|| if (GCC_GLOBAL_REGS) { 405| mov aword EX->opline, IP 406|| } 407|.endmacro 408 409|.macro LOAD_IP 410|| if (GCC_GLOBAL_REGS) { 411| mov IP, aword EX->opline 412|| } 413|.endmacro 414 415|.macro LOAD_IP_ADDR, addr 416|| if (GCC_GLOBAL_REGS) { 417| LOAD_ADDR IP, addr 418|| } else { 419| ADDR_OP2_2 mov, aword EX->opline, addr, RX 420|| } 421|.endmacro 422 423|.macro LOAD_IP_ADDR_ZTS, struct, field 424| .if ZTS 425|| if (GCC_GLOBAL_REGS) { 426| LOAD_TSRM_CACHE IP 427| mov IP, aword [IP + (struct.._offset + offsetof(zend_..struct, field))] 428|| } else { 429| LOAD_TSRM_CACHE RX 430| lea RX, aword [RX + (struct.._offset + offsetof(zend_..struct, field))] 431| mov aword EX->opline, RX 432|| } 433| .else 434| LOAD_IP_ADDR &struct.field 435| .endif 436|.endmacro 437 438|.macro GET_IP, reg 439|| if (GCC_GLOBAL_REGS) { 440| mov reg, IP 441|| } else { 442| mov reg, aword EX->opline 443|| } 444|.endmacro 445 446|.macro ADD_IP, val 447|| if (GCC_GLOBAL_REGS) { 448| add IP, val 449|| } else { 450| add aword EX->opline, val 451|| } 452|.endmacro 453 454|.macro JMP_IP 455|| if (GCC_GLOBAL_REGS) { 456| jmp aword [IP] 457|| } else { 458| mov r0, aword EX:FCARG1a->opline 459| jmp aword [r0] 460|| } 461|.endmacro 462 463/* In 64-bit build we compare only low 32-bits. 464 * x86_64 cmp instruction doesn't support immediate 64-bit operand, and full 465 * comparison would require an additional load of 64-bit address into register. 466 * This is not a problem at all, while JIT buffer size is less than 4GB. 467 */ 468|.macro CMP_IP, addr 469|| if (GCC_GLOBAL_REGS) { 470| cmp IPl, addr 471|| } else { 472| cmp dword EX->opline, addr 473|| } 474|.endmacro 475 476|.macro LOAD_ZVAL_ADDR, reg, addr 477|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 478| LOAD_ADDR reg, Z_ZV(addr) 479|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 480| LOAD_BASE_ADDR reg, Z_REG(addr), Z_OFFSET(addr) 481|| } else { 482|| ZEND_UNREACHABLE(); 483|| } 484|.endmacro 485 486|.macro PUSH_ZVAL_ADDR, addr, tmp_reg 487|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 488| PUSH_ADDR Z_ZV(addr), tmp_reg 489|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 490| PUSH_BASE_ADDR Z_REG(addr), Z_OFFSET(addr), tmp_reg 491|| } else { 492|| ZEND_UNREACHABLE(); 493|| } 494|.endmacro 495 496|.macro GET_Z_TYPE_INFO, reg, zv 497| mov reg, dword [zv+offsetof(zval,u1.type_info)] 498|.endmacro 499 500|.macro SET_Z_TYPE_INFO, zv, type 501| mov dword [zv+offsetof(zval,u1.type_info)], type 502|.endmacro 503 504|.macro GET_ZVAL_TYPE, reg, addr 505|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 506| mov reg, byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.v.type)] 507|.endmacro 508 509|.macro GET_ZVAL_TYPE_INFO, reg, addr 510|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 511| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)] 512|.endmacro 513 514|.macro SET_ZVAL_TYPE_INFO, addr, type 515|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 516| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)], type 517|.endmacro 518 519|.macro GET_Z_PTR, reg, zv 520| mov reg, aword [zv] 521|.endmacro 522 523|.macro SET_Z_PTR, zv, val 524| mov aword [zv], val 525|.endmacro 526 527|.macro GET_Z_W2, reg, zv 528| mov reg, dword [zv+4] 529|.endmacro 530 531|.macro SET_Z_W2, zv, reg 532| mov dword [zv+4], reg 533|.endmacro 534 535|.macro GET_ZVAL_PTR, reg, addr 536|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 537| mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 538|.endmacro 539 540|.macro SET_ZVAL_PTR, addr, val 541|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 542| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], val 543|.endmacro 544 545|.macro GET_ZVAL_W2, reg, addr 546|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 547| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4] 548|.endmacro 549 550|.macro SET_ZVAL_W2, addr, val 551|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 552| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4], val 553|.endmacro 554 555|.macro UNDEF_OPLINE_RESULT 556| mov r0, EX->opline 557| mov eax, dword OP:r0->result.var 558| SET_Z_TYPE_INFO FP + r0, IS_UNDEF 559|.endmacro 560 561|.macro UNDEF_OPLINE_RESULT_IF_USED 562| test byte OP:RX->result_type, (IS_TMP_VAR|IS_VAR) 563| jz >1 564| mov eax, dword OP:RX->result.var 565| SET_Z_TYPE_INFO FP + r0, IS_UNDEF 566|1: 567|.endmacro 568 569|.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2 570|| if (CAN_USE_AVX()) { 571| avx_ins op1, op2 572|| } else { 573| sse_ins op1, op2 574|| } 575|.endmacro 576 577|.macro SSE_OP, sse_ins, reg, addr, tmp_reg 578|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 579| MEM_OP2_2 sse_ins, xmm(reg-ZREG_XMM0), qword, Z_ZV(addr), tmp_reg 580|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 581| sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 582|| } else if (Z_MODE(addr) == IS_REG) { 583| sse_ins xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 584|| } else { 585|| ZEND_UNREACHABLE(); 586|| } 587|.endmacro 588 589|.macro SSE_AVX_OP, sse_ins, avx_ins, reg, addr 590|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 591| .if X64 592|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 593| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 594|| } else { 595| LOAD_ADDR r0, Z_ZV(addr) 596| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [r0] 597|| } 598| .else 599| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 600| .endif 601|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 602| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 603|| } else if (Z_MODE(addr) == IS_REG) { 604| SSE_AVX_INS sse_ins, avx_ins, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 605|| } else { 606|| ZEND_UNREACHABLE(); 607|| } 608|.endmacro 609 610|.macro SSE_GET_LONG, reg, lval, tmp_reg 611|| if (lval == 0) { 612|| if (CAN_USE_AVX()) { 613| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 614|| } else { 615| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 616|| } 617|| } else { 618|.if X64 619|| if (!IS_SIGNED_32BIT(lval)) { 620| mov64 Ra(tmp_reg), lval 621|| } else { 622| mov Ra(tmp_reg), lval 623|| } 624|.else 625| mov Ra(tmp_reg), lval 626|.endif 627|| if (CAN_USE_AVX()) { 628| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 629| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(tmp_reg) 630|| } else { 631| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 632| cvtsi2sd, xmm(reg-ZREG_XMM0), Ra(tmp_reg) 633|| } 634|| } 635|.endmacro 636 637|.macro SSE_GET_ZVAL_LVAL, reg, addr, tmp_reg 638|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 639| SSE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg 640|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 641|| if (CAN_USE_AVX()) { 642| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 643| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 644|| } else { 645| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 646| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 647|| } 648|| } else if (Z_MODE(addr) == IS_REG) { 649|| if (CAN_USE_AVX()) { 650| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 651| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) 652|| } else { 653| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0) 654| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr)) 655|| } 656|| } else { 657|| ZEND_UNREACHABLE(); 658|| } 659|.endmacro 660 661|.macro SSE_GET_ZVAL_DVAL, reg, addr 662|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { 663|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 664| .if X64 665|| if (IS_SIGNED_32BIT(Z_ZV(addr))) { 666| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 667|| } else { 668| LOAD_ADDR r0, Z_ZV(addr) 669| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [r0] 670|| } 671| .else 672| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)] 673| .endif 674|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 675| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 676|| } else if (Z_MODE(addr) == IS_REG) { 677| SSE_AVX_INS movaps, vmovaps, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 678|| } else { 679|| ZEND_UNREACHABLE(); 680|| } 681|| } 682|.endmacro 683 684|.macro SSE_MATH, opcode, reg, addr, tmp_reg 685|| switch (opcode) { 686|| case ZEND_ADD: 687| SSE_OP addsd, reg, addr, tmp_reg 688|| break; 689|| case ZEND_SUB: 690| SSE_OP subsd, reg, addr, tmp_reg 691|| break; 692|| case ZEND_MUL: 693| SSE_OP mulsd, reg, addr, tmp_reg 694|| break; 695|| case ZEND_DIV: 696| SSE_OP divsd, reg, addr, tmp_reg 697|| break; 698|| } 699|.endmacro 700 701|.macro SSE_MATH_REG, opcode, dst_reg, src_reg 702|| switch (opcode) { 703|| case ZEND_ADD: 704| addsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 705|| break; 706|| case ZEND_SUB: 707| subsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 708|| break; 709|| case ZEND_MUL: 710| mulsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 711|| break; 712|| case ZEND_DIV: 713| divsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 714|| break; 715|| } 716|.endmacro 717 718|.macro SSE_SET_ZVAL_DVAL, addr, reg 719|| if (Z_MODE(addr) == IS_REG) { 720|| if (reg != Z_REG(addr)) { 721| SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0) 722|| } 723|| } else { 724|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 725| SSE_AVX_INS movsd, vmovsd, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0) 726|| } 727|.endmacro 728 729|.macro AVX_OP, avx_ins, reg, op1_reg, addr, tmp_reg 730|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 731| MEM_OP3_3 avx_ins, xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword, Z_ZV(addr), tmp_reg 732|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 733| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 734|| } else if (Z_MODE(addr) == IS_REG) { 735| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0) 736|| } else { 737|| ZEND_UNREACHABLE(); 738|| } 739|.endmacro 740 741|.macro AVX_MATH, opcode, reg, op1_reg, addr, tmp_reg 742|| switch (opcode) { 743|| case ZEND_ADD: 744| AVX_OP vaddsd, reg, op1_reg, addr, tmp_reg 745|| break; 746|| case ZEND_SUB: 747| AVX_OP vsubsd, reg, op1_reg, addr, tmp_reg 748|| break; 749|| case ZEND_MUL: 750| AVX_OP vmulsd, reg, op1_reg, addr, tmp_reg 751|| break; 752|| case ZEND_DIV: 753| AVX_OP vdivsd, reg, op1_reg, addr, tmp_reg 754|| break; 755|| } 756|.endmacro 757 758|.macro AVX_MATH_REG, opcode, dst_reg, op1_reg, src_reg 759|| switch (opcode) { 760|| case ZEND_ADD: 761| vaddsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 762|| break; 763|| case ZEND_SUB: 764| vsubsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 765|| break; 766|| case ZEND_MUL: 767| vmulsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 768|| break; 769|| case ZEND_DIV: 770| vdivsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0) 771|| break; 772|| } 773|.endmacro 774 775|.macro LONG_OP, long_ins, reg, addr, tmp_reg 776|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 777| .if X64 778|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { 779| mov64 tmp_reg, Z_LVAL_P(Z_ZV(addr)) 780| long_ins Ra(reg), tmp_reg 781|| } else { 782| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) 783|| } 784| .else 785| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr)) 786| .endif 787|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 788| long_ins Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 789|| } else if (Z_MODE(addr) == IS_REG) { 790| long_ins Ra(reg), Ra(Z_REG(addr)) 791|| } else { 792|| ZEND_UNREACHABLE(); 793|| } 794|.endmacro 795 796|.macro LONG_OP_WITH_32BIT_CONST, long_ins, op1_addr, lval 797|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { 798| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 799|| } else if (Z_MODE(op1_addr) == IS_REG) { 800| long_ins Ra(Z_REG(op1_addr)), lval 801|| } else { 802|| ZEND_UNREACHABLE(); 803|| } 804|.endmacro 805 806|.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval 807|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { 808| .if X64 809|| if (!IS_SIGNED_32BIT(lval)) { 810| mov64 r0, lval 811| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0 812|| } else { 813| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 814|| } 815| .else 816| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval 817| .endif 818|| } else if (Z_MODE(op1_addr) == IS_REG) { 819| .if X64 820|| if (!IS_SIGNED_32BIT(lval)) { 821| mov64 r0, lval 822| long_ins Ra(Z_REG(op1_addr)), r0 823|| } else { 824| long_ins Ra(Z_REG(op1_addr)), lval 825|| } 826| .else 827| long_ins Ra(Z_REG(op1_addr)), lval 828| .endif 829|| } else { 830|| ZEND_UNREACHABLE(); 831|| } 832|.endmacro 833 834|.macro GET_ZVAL_LVAL, reg, addr 835|| if (Z_MODE(addr) == IS_CONST_ZVAL) { 836|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { 837| xor Ra(reg), Ra(reg) 838|| } else { 839| .if X64 840|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) { 841| mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr)) 842|| } else { 843| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) 844|| } 845| .else 846| mov Ra(reg), Z_LVAL_P(Z_ZV(addr)) 847| .endif 848|| } 849|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { 850| mov Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)] 851|| } else if (Z_MODE(addr) == IS_REG) { 852|| if (reg != Z_REG(addr)) { 853| mov Ra(reg), Ra(Z_REG(addr)) 854|| } 855|| } else { 856|| ZEND_UNREACHABLE(); 857|| } 858|.endmacro 859 860|.macro LONG_MATH, opcode, reg, addr, tmp_reg 861|| switch (opcode) { 862|| case ZEND_ADD: 863| LONG_OP add, reg, addr, Ra(tmp_reg) 864|| break; 865|| case ZEND_SUB: 866| LONG_OP sub, reg, addr, Ra(tmp_reg) 867|| break; 868|| case ZEND_MUL: 869| LONG_OP imul, reg, addr, Ra(tmp_reg) 870|| break; 871|| case ZEND_BW_OR: 872| LONG_OP or, reg, addr, Ra(tmp_reg) 873|| break; 874|| case ZEND_BW_AND: 875| LONG_OP and, reg, addr, Ra(tmp_reg) 876|| break; 877|| case ZEND_BW_XOR: 878| LONG_OP xor, reg, addr, Ra(tmp_reg) 879|| break; 880|| default: 881|| ZEND_UNREACHABLE(); 882|| } 883|.endmacro 884 885|.macro LONG_MATH_REG, opcode, dst_reg, src_reg 886|| switch (opcode) { 887|| case ZEND_ADD: 888| add dst_reg, src_reg 889|| break; 890|| case ZEND_SUB: 891| sub dst_reg, src_reg 892|| break; 893|| case ZEND_MUL: 894| imul dst_reg, src_reg 895|| break; 896|| case ZEND_BW_OR: 897| or dst_reg, src_reg 898|| break; 899|| case ZEND_BW_AND: 900| and dst_reg, src_reg 901|| break; 902|| case ZEND_BW_XOR: 903| xor dst_reg, src_reg 904|| break; 905|| default: 906|| ZEND_UNREACHABLE(); 907|| } 908|.endmacro 909 910|.macro SET_ZVAL_LVAL, addr, lval 911|| if (Z_MODE(addr) == IS_REG) { 912| mov Ra(Z_REG(addr)), lval 913|| } else { 914|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 915| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval 916|| } 917|.endmacro 918 919|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg 920|| if (Z_TYPE_P(zv) > IS_TRUE) { 921|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 922|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; 923|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { 924|| if (CAN_USE_AVX()) { 925| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 926|| } else { 927| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 928|| } 929| .if X64 930|| } else if (!IS_SIGNED_32BIT(zv)) { 931| mov64 Ra(tmp_reg), ((uintptr_t)zv) 932| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] 933| .endif 934|| } else { 935| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] 936|| } 937| SSE_SET_ZVAL_DVAL dst_addr, dst_reg 938|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 939|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0; 940| SSE_GET_LONG dst_reg, Z_LVAL_P(zv), ZREG_R0 941| SSE_SET_ZVAL_DVAL dst_addr, dst_reg 942|| } else if (Z_LVAL_P(zv) == 0 && Z_MODE(dst_addr) == IS_REG) { 943| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) 944|| } else { 945| .if X64 946|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 947|| if (Z_MODE(dst_addr) == IS_REG) { 948| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) 949|| } else { 950| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) 951| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) 952|| } 953|| } else { 954| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 955|| } 956| .else 957| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 958| .endif 959|| } 960|| } 961|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 962|| if (dst_def_info == MAY_BE_DOUBLE) { 963|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 964| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE 965|| } 966|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 967| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) 968|| } 969|| } 970|.endmacro 971 972|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg 973|| if (Z_TYPE_P(zv) > IS_TRUE) { 974|| if (Z_TYPE_P(zv) == IS_DOUBLE) { 975|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? 976|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0); 977|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) { 978|| if (CAN_USE_AVX()) { 979| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 980|| } else { 981| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0) 982|| } 983| .if X64 984|| } else if (!IS_SIGNED_32BIT(zv)) { 985| mov64 Ra(tmp_reg), ((uintptr_t)zv) 986| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)] 987| .endif 988|| } else { 989| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)] 990|| } 991| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 992| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 993|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { 994|| if (Z_MODE(dst_addr) == IS_REG) { 995| SSE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), ZREG_R0 996| SSE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) 997|| } else if (Z_MODE(res_addr) == IS_REG) { 998| SSE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), ZREG_R0 999| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) 1000|| } else { 1001| SSE_GET_LONG ZREG_XMM0, Z_LVAL_P(zv), ZREG_R0 1002| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1003| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 1004|| } 1005|| } else if (Z_LVAL_P(zv) == 0 && (Z_MODE(dst_addr) == IS_REG || Z_MODE(res_addr) == IS_REG)) { 1006|| if (Z_MODE(dst_addr) == IS_REG) { 1007| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr)) 1008| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1009|| } else { 1010| xor Ra(Z_REG(res_addr)), Ra(Z_REG(res_addr)) 1011| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1012|| } 1013|| } else { 1014| .if X64 1015|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1016|| if (Z_MODE(dst_addr) == IS_REG) { 1017| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv)) 1018| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1019|| } else if (Z_MODE(res_addr) == IS_REG) { 1020| mov64 Ra(Z_REG(res_addr)), ((uintptr_t)Z_LVAL_P(zv)) 1021| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1022|| } else { 1023| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv)) 1024| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg) 1025| SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 1026|| } 1027|| } else if (Z_MODE(dst_addr) == IS_REG) { 1028| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1029| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1030|| } else if (Z_MODE(res_addr) == IS_REG) { 1031| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1032| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1033|| } else { 1034| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1035| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1036|| } 1037| .else 1038|| if (Z_MODE(dst_addr) == IS_REG) { 1039| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1040| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1041|| } else if (Z_MODE(res_addr) == IS_REG) { 1042| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1043| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1044|| } else { 1045| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv) 1046| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv) 1047|| } 1048| .endif 1049|| } 1050|| } 1051|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1052|| if (dst_def_info == MAY_BE_DOUBLE) { 1053|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 1054| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE 1055|| } 1056|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) { 1057| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv) 1058|| } 1059|| } 1060|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1061|| if (dst_def_info == MAY_BE_DOUBLE) { 1062| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 1063|| } else { 1064| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv) 1065|| } 1066|| } 1067|.endmacro 1068 1069/* the same as above, but "src" may overlap with "tmp_reg1" */ 1070|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1071| ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1072|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1073|| !(src_info & MAY_BE_GUARD) && 1074|| has_concrete_type(src_info & MAY_BE_ANY)) { 1075|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1076|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) { 1077|| zend_uchar type = concrete_type(src_info); 1078| SET_ZVAL_TYPE_INFO dst_addr, type 1079|| } 1080|| } 1081|| } else { 1082| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr 1083| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) 1084|| } 1085|.endmacro 1086 1087|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2 1088|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1089|| if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) { 1090|| if (Z_MODE(src_addr) == IS_REG) { 1091|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1092| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) 1093|| } 1094|| } else if (Z_MODE(dst_addr) == IS_REG) { 1095| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr 1096|| } else { 1097| GET_ZVAL_LVAL tmp_reg2, src_addr 1098| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) 1099|| } 1100|| } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 1101|| if (Z_MODE(src_addr) == IS_REG) { 1102| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) 1103|| } else if (Z_MODE(dst_addr) == IS_REG) { 1104| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr 1105|| } else { 1106| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr 1107| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1108|| } 1109|| } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) { 1110| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1111| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1112|| } else { 1113| .if X64 1114| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1115| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1116| .else 1117|| if ((tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr))) { 1118| GET_ZVAL_W2 Ra(tmp_reg2), src_addr 1119| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) 1120| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1121| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1122|| } else { 1123| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1124| GET_ZVAL_W2 Ra(tmp_reg1), src_addr 1125| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1126| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) 1127|| } 1128| .endif 1129|| } 1130|| } 1131|.endmacro 1132 1133|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, tmp_reg1, tmp_reg2 1134|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 1135|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) { 1136|| if (Z_MODE(src_addr) == IS_REG) { 1137|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { 1138| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr)) 1139|| } 1140|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) { 1141| SET_ZVAL_LVAL res_addr, Ra(Z_REG(src_addr)) 1142|| } 1143|| } else if (Z_MODE(dst_addr) == IS_REG) { 1144| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr 1145|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) { 1146| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr)) 1147|| } 1148|| } else if (Z_MODE(res_addr) == IS_REG) { 1149| GET_ZVAL_LVAL Z_REG(res_addr), src_addr 1150| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr)) 1151|| } else { 1152| GET_ZVAL_LVAL tmp_reg2, src_addr 1153| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2) 1154| SET_ZVAL_LVAL res_addr, Ra(tmp_reg2) 1155|| } 1156|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 1157|| if (Z_MODE(src_addr) == IS_REG) { 1158| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr) 1159| SSE_SET_ZVAL_DVAL res_addr, Z_REG(src_addr) 1160|| } else if (Z_MODE(dst_addr) == IS_REG) { 1161| SSE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr 1162| SSE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr) 1163|| } else if (Z_MODE(res_addr) == IS_REG) { 1164| SSE_GET_ZVAL_DVAL Z_REG(res_addr), src_addr 1165| SSE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr) 1166|| } else { 1167| SSE_GET_ZVAL_DVAL ZREG_XMM0, src_addr 1168| SSE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0 1169| SSE_SET_ZVAL_DVAL res_addr, ZREG_XMM0 1170|| } 1171|| } else if (!(src_info & MAY_BE_DOUBLE)) { 1172| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1173| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1174| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1175|| } else { 1176| .if X64 1177| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1178| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1179| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1180| .else 1181|| if (tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) { 1182| GET_ZVAL_W2 Ra(tmp_reg2), src_addr 1183| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2) 1184| SET_ZVAL_W2 res_addr, Ra(tmp_reg2) 1185| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1186| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1187| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1188|| } else { 1189| GET_ZVAL_PTR Ra(tmp_reg2), src_addr 1190| GET_ZVAL_W2 Ra(tmp_reg1), src_addr 1191| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2) 1192| SET_ZVAL_PTR res_addr, Ra(tmp_reg2) 1193| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1) 1194| SET_ZVAL_W2 res_addr, Ra(tmp_reg1) 1195|| } 1196| .endif 1197|| } 1198|| } 1199|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && 1200|| has_concrete_type(src_info & MAY_BE_ANY)) { 1201|| zend_uchar type = concrete_type(src_info); 1202|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { 1203|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { 1204| SET_ZVAL_TYPE_INFO dst_addr, type 1205|| } 1206|| } 1207|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 1208| SET_ZVAL_TYPE_INFO res_addr, type 1209|| } 1210|| } else { 1211| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr 1212| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1) 1213| SET_ZVAL_TYPE_INFO res_addr, Rd(tmp_reg1) 1214|| } 1215|.endmacro 1216 1217|.macro IF_UNDEF, type_reg, label 1218| test type_reg, type_reg 1219| je label 1220|.endmacro 1221 1222|.macro IF_TYPE, type, val, label 1223| cmp type, val 1224| je label 1225|.endmacro 1226 1227|.macro IF_NOT_TYPE, type, val, label 1228| cmp type, val 1229| jne label 1230|.endmacro 1231 1232|.macro IF_Z_TYPE, zv, val, label 1233| IF_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label 1234|.endmacro 1235 1236|.macro IF_NOT_Z_TYPE, zv, val, label 1237| IF_NOT_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label 1238|.endmacro 1239 1240|.macro CMP_ZVAL_TYPE, addr, val 1241|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1242| cmp byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val 1243|.endmacro 1244 1245|.macro IF_ZVAL_TYPE, addr, val, label 1246|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1247| IF_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label 1248|.endmacro 1249 1250|.macro IF_NOT_ZVAL_TYPE, addr, val, label 1251|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1252| IF_NOT_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label 1253|.endmacro 1254 1255|.macro IF_FLAGS, type_flags, mask, label 1256| test type_flags, mask 1257| jnz label 1258|.endmacro 1259 1260|.macro IF_NOT_FLAGS, type_flags, mask, label 1261| test type_flags, mask 1262| jz label 1263|.endmacro 1264 1265|.macro IF_REFCOUNTED, type_flags, label 1266| IF_FLAGS type_flags, IS_TYPE_REFCOUNTED, label 1267|.endmacro 1268 1269|.macro IF_NOT_REFCOUNTED, type_flags, label 1270| //IF_NOT_FLAGS type_flags, IS_TYPE_REFCOUNTED, label 1271| test type_flags, type_flags 1272| jz label 1273|.endmacro 1274 1275|.macro IF_ZVAL_FLAGS, addr, mask, label 1276|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1277| IF_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label 1278|.endmacro 1279 1280|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label 1281|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); 1282| IF_NOT_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label 1283|.endmacro 1284 1285|.macro IF_ZVAL_REFCOUNTED, addr, label 1286| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label 1287|.endmacro 1288 1289|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label 1290| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label 1291|.endmacro 1292 1293|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label 1294| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label 1295|.endmacro 1296 1297|.macro GC_ADDREF, zv 1298| add dword [zv], 1 1299|.endmacro 1300 1301|.macro GC_DELREF, zv 1302| sub dword [zv], 1 1303|.endmacro 1304 1305|.macro IF_GC_MAY_NOT_LEAK, ptr, label 1306| test dword [ptr+4],(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)) 1307| jne label 1308|.endmacro 1309 1310|.macro ADDREF_CONST, zv, tmp_reg 1311| .if X64 1312|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1313| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) 1314| add dword [tmp_reg], 1 1315|| } else { 1316| add dword [Z_LVAL_P(zv)], 1 1317|| } 1318| .else 1319| add dword [Z_LVAL_P(zv)], 1 1320| .endif 1321|.endmacro 1322 1323|.macro ADDREF_CONST_2, zv, tmp_reg 1324| .if X64 1325|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 1326| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) 1327| add dword [tmp_reg], 2 1328|| } else { 1329| add dword [Z_LVAL_P(zv)], 2 1330|| } 1331| .else 1332| add dword [Z_LVAL_P(zv)], 2 1333| .endif 1334|.endmacro 1335 1336|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg 1337|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1338|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1339| IF_NOT_REFCOUNTED type_flags_reg, >1 1340|| } 1341| GC_ADDREF value_ptr_reg 1342|1: 1343|| } 1344|.endmacro 1345 1346|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg 1347|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 1348|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1349| IF_NOT_REFCOUNTED type_flags_reg, >1 1350|| } 1351| add dword [value_ptr_reg], 2 1352|1: 1353|| } 1354|.endmacro 1355 1356|.macro ZVAL_DEREF, reg, info 1357|| if (info & MAY_BE_REF) { 1358| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1 1359| GET_Z_PTR reg, reg 1360| add reg, offsetof(zend_reference, val) 1361|1: 1362|| } 1363|.endmacro 1364 1365|.macro SET_EX_OPLINE, op, tmp_reg 1366|| if (op == last_valid_opline) { 1367|| zend_jit_use_last_valid_opline(); 1368| SAVE_IP 1369|| } else { 1370| ADDR_OP2_2 mov, aword EX->opline, op, tmp_reg 1371|| if (!GCC_GLOBAL_REGS) { 1372|| zend_jit_reset_last_valid_opline(); 1373|| } 1374|| } 1375|.endmacro 1376 1377// zval should be in FCARG1a 1378|.macro ZVAL_DTOR_FUNC, var_info, opline // arg1 must be in FCARG1a 1379|| do { 1380|| if (!((var_info) & MAY_BE_GUARD) 1381|| && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1382|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); 1383|| if (type == IS_STRING && !ZEND_DEBUG) { 1384| EXT_CALL _efree, r0 1385|| break; 1386|| } else if (type == IS_ARRAY) { 1387|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { 1388|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { 1389| SET_EX_OPLINE opline, r0 1390|| } 1391| EXT_CALL zend_array_destroy, r0 1392|| } else { 1393| EXT_CALL zend_jit_array_free, r0 1394|| } 1395|| break; 1396|| } else if (type == IS_OBJECT) { 1397|| if (opline) { 1398| SET_EX_OPLINE opline, r0 1399|| } 1400| EXT_CALL zend_objects_store_del, r0 1401|| break; 1402|| } 1403|| } 1404|| if (opline) { 1405| SET_EX_OPLINE opline, r0 1406|| } 1407| EXT_CALL rc_dtor_func, r0 1408|| } while(0); 1409|.endmacro 1410 1411|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline 1412|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) { 1413|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 1414| // if (Z_REFCOUNTED_P(cv)) { 1415|| if (cold) { 1416| IF_ZVAL_REFCOUNTED addr, >1 1417|.cold_code 1418|1: 1419|| } else { 1420| IF_NOT_ZVAL_REFCOUNTED addr, >4 1421|| } 1422|| } 1423| // if (!Z_DELREF_P(cv)) { 1424| GET_ZVAL_PTR FCARG1a, addr 1425| GC_DELREF FCARG1a 1426|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) { 1427|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) { 1428|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1429| jnz >3 1430|| } else { 1431| jnz >4 1432|| } 1433|| } 1434| // zval_dtor_func(r); 1435| ZVAL_DTOR_FUNC op_info, opline 1436|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1437| jmp >4 1438|| } 1439|3: 1440|| } 1441|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) { 1442|| if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) { 1443|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offsetof(zend_reference, val)); 1444| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1 1445| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4 1446| GET_ZVAL_PTR FCARG1a, ref_addr 1447|1: 1448|| } 1449| IF_GC_MAY_NOT_LEAK FCARG1a, >4 1450| // gc_possible_root(Z_COUNTED_P(z)) 1451| EXT_CALL gc_possible_root, r0 1452|| } 1453|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { 1454| jmp >4 1455|.code 1456|| } 1457|4: 1458|| } 1459|.endmacro 1460 1461|.macro FREE_OP, op_type, op, op_info, cold, opline 1462|| if (op_type & (IS_VAR|IS_TMP_VAR)) { 1463| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, opline 1464|| } 1465|.endmacro 1466 1467|.macro SEPARATE_ARRAY, addr, op_info, cold 1468|| if (RC_MAY_BE_N(op_info)) { 1469|| if (Z_REG(addr) != ZREG_FP) { 1470| GET_ZVAL_LVAL ZREG_R0, addr 1471|| if (RC_MAY_BE_1(op_info)) { 1472| cmp dword [r0], 1 // if (GC_REFCOUNT() > 1) 1473| jbe >2 1474|| } 1475|| if (Z_REG(addr) != ZREG_FCARG1a || Z_OFFSET(addr) != 0) { 1476| LOAD_ZVAL_ADDR FCARG1a, addr 1477|| } 1478| EXT_CALL zend_jit_zval_array_dup, r0 1479|2: 1480| mov FCARG1a, r0 1481|| } else { 1482| GET_ZVAL_LVAL ZREG_FCARG1a, addr 1483|| if (RC_MAY_BE_1(op_info)) { 1484| cmp dword [FCARG1a], 1 // if (GC_REFCOUNT() > 1) 1485|| if (cold) { 1486| ja >1 1487|.cold_code 1488|1: 1489|| } else { 1490| jbe >2 1491|| } 1492|| } 1493| IF_NOT_ZVAL_REFCOUNTED addr, >1 1494| GC_DELREF FCARG1a 1495|1: 1496| EXT_CALL zend_array_dup, r0 1497| SET_ZVAL_PTR addr, r0 1498| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX 1499| mov FCARG1a, r0 1500|| if (RC_MAY_BE_1(op_info)) { 1501|| if (cold) { 1502| jmp >2 1503|.code 1504|| } 1505|| } 1506|2: 1507|| } 1508|| } else { 1509| GET_ZVAL_LVAL ZREG_FCARG1a, addr 1510|| } 1511|.endmacro 1512 1513|.macro EFREE_REG_REFERENCE 1514||#if ZEND_DEBUG 1515| xor FCARG2a, FCARG2a // filename 1516| .if X64WIN 1517| xor CARG3d, CARG3d // lineno 1518| xor CARG4, CARG4 1519| mov aword A5, 0 1520| EXT_CALL _efree, r0 1521| .elif X64 1522| xor CARG3d, CARG3d // lineno 1523| xor CARG4, CARG4 1524| xor CARG5, CARG5 1525| EXT_CALL _efree, r0 1526| .else 1527| sub r4, 4 1528| push 0 1529| push 0 1530| push 0 // lineno 1531| EXT_CALL _efree, r0 1532| add r4, 4 1533| .endif 1534||#else 1535||#ifdef HAVE_BUILTIN_CONSTANT_P 1536| EXT_CALL _efree_32, r0 1537||#else 1538| EXT_CALL _efree, r0 1539||#endif 1540||#endif 1541|.endmacro 1542 1543|.macro EFREE_REFERENCE, ptr 1544| mov FCARG1a, ptr 1545| EFREE_REG_REFERENCE 1546|.endmacro 1547 1548|.macro EMALLOC, size, op_array, opline 1549||#if ZEND_DEBUG 1550|| const char *filename = op_array->filename ? op_array->filename->val : NULL; 1551| mov FCARG1a, size 1552| LOAD_ADDR FCARG2a, filename 1553| .if X64WIN 1554| mov CARG3d, opline->lineno 1555| xor CARG4, CARG4 1556| mov aword A5, 0 1557| EXT_CALL _emalloc, r0 1558| .elif X64 1559| mov CARG3d, opline->lineno 1560| xor CARG4, CARG4 1561| xor CARG5, CARG5 1562| EXT_CALL _emalloc, r0 1563| .else 1564| sub r4, 4 1565| push 0 1566| push 0 1567| push opline->lineno 1568| EXT_CALL _emalloc, r0 1569| add r4, 4 1570| .endif 1571||#else 1572||#ifdef HAVE_BUILTIN_CONSTANT_P 1573|| if (size > 24 && size <= 32) { 1574| EXT_CALL _emalloc_32, r0 1575|| } else { 1576| mov FCARG1a, size 1577| EXT_CALL _emalloc, r0 1578|| } 1579||#else 1580| mov FCARG1a, size 1581| EXT_CALL _emalloc, r0 1582||#endif 1583||#endif 1584|.endmacro 1585 1586|.macro OBJ_RELEASE, reg, exit_label 1587| GC_DELREF Ra(reg) 1588| jne >1 1589| // zend_objects_store_del(obj); 1590|| if (reg != ZREG_FCARG1a) { 1591| mov FCARG1a, Ra(reg) 1592|| } 1593| EXT_CALL zend_objects_store_del, r0 1594| jmp exit_label 1595|1: 1596| IF_GC_MAY_NOT_LEAK Ra(reg), >1 1597| // gc_possible_root(obj) 1598|| if (reg != ZREG_FCARG1a) { 1599| mov FCARG1a, Ra(reg) 1600|| } 1601| EXT_CALL gc_possible_root, r0 1602|1: 1603|.endmacro 1604 1605|.macro UNDEFINED_OFFSET, opline 1606|| if (opline == last_valid_opline) { 1607|| zend_jit_use_last_valid_opline(); 1608| call ->undefined_offset_ex 1609|| } else { 1610| SET_EX_OPLINE opline, r0 1611| call ->undefined_offset 1612|| } 1613|.endmacro 1614 1615|.macro UNDEFINED_INDEX, opline 1616|| if (opline == last_valid_opline) { 1617|| zend_jit_use_last_valid_opline(); 1618| call ->undefined_index_ex 1619|| } else { 1620| SET_EX_OPLINE opline, r0 1621| call ->undefined_index 1622|| } 1623|.endmacro 1624 1625|.macro CANNOT_ADD_ELEMENT, opline 1626|| if (opline == last_valid_opline) { 1627|| zend_jit_use_last_valid_opline(); 1628| call ->cannot_add_element_ex 1629|| } else { 1630| SET_EX_OPLINE opline, r0 1631| call ->cannot_add_element 1632|| } 1633|.endmacro 1634 1635static zend_bool reuse_ip = 0; 1636static zend_bool delayed_call_chain = 0; 1637static uint32_t delayed_call_level = 0; 1638static const zend_op *last_valid_opline = NULL; 1639static zend_bool use_last_vald_opline = 0; 1640static zend_bool track_last_valid_opline = 0; 1641static int jit_return_label = -1; 1642static uint32_t current_trace_num = 0; 1643static uint32_t allowed_opt_flags = 0; 1644 1645static void zend_jit_track_last_valid_opline(void) 1646{ 1647 use_last_vald_opline = 0; 1648 track_last_valid_opline = 1; 1649} 1650 1651static void zend_jit_use_last_valid_opline(void) 1652{ 1653 if (track_last_valid_opline) { 1654 use_last_vald_opline = 1; 1655 track_last_valid_opline = 0; 1656 } 1657} 1658 1659static zend_bool zend_jit_trace_uses_initial_ip(void) 1660{ 1661 return use_last_vald_opline; 1662} 1663 1664static void zend_jit_set_last_valid_opline(const zend_op *target_opline) 1665{ 1666 if (!reuse_ip) { 1667 track_last_valid_opline = 0; 1668 last_valid_opline = target_opline; 1669 } 1670} 1671 1672static void zend_jit_reset_last_valid_opline(void) 1673{ 1674 track_last_valid_opline = 0; 1675 last_valid_opline = NULL; 1676} 1677 1678static void zend_jit_start_reuse_ip(void) 1679{ 1680 zend_jit_reset_last_valid_opline(); 1681 reuse_ip = 1; 1682} 1683 1684static int zend_jit_reuse_ip(dasm_State **Dst) 1685{ 1686 if (!reuse_ip) { 1687 zend_jit_start_reuse_ip(); 1688 | // call = EX(call); 1689 | mov RX, EX->call 1690 } 1691 return 1; 1692} 1693 1694static void zend_jit_stop_reuse_ip(void) 1695{ 1696 reuse_ip = 0; 1697} 1698 1699/* bit helpers */ 1700 1701/* from http://aggregate.org/MAGIC/ */ 1702static uint32_t ones32(uint32_t x) 1703{ 1704 x -= ((x >> 1) & 0x55555555); 1705 x = (((x >> 2) & 0x33333333) + (x & 0x33333333)); 1706 x = (((x >> 4) + x) & 0x0f0f0f0f); 1707 x += (x >> 8); 1708 x += (x >> 16); 1709 return x & 0x0000003f; 1710} 1711 1712static uint32_t floor_log2(uint32_t x) 1713{ 1714 ZEND_ASSERT(x != 0); 1715 x |= (x >> 1); 1716 x |= (x >> 2); 1717 x |= (x >> 4); 1718 x |= (x >> 8); 1719 x |= (x >> 16); 1720 return ones32(x) - 1; 1721} 1722 1723static zend_bool is_power_of_two(uint32_t x) 1724{ 1725 return !(x & (x - 1)) && x != 0; 1726} 1727 1728static zend_bool has_concrete_type(uint32_t value_type) 1729{ 1730 return is_power_of_two (value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); 1731} 1732 1733static uint32_t concrete_type(uint32_t value_type) 1734{ 1735 return floor_log2(value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); 1736} 1737 1738static inline zend_bool is_signed(double d) 1739{ 1740 return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0; 1741} 1742 1743static int zend_jit_interrupt_handler_stub(dasm_State **Dst) 1744{ 1745 |->interrupt_handler: 1746 | SAVE_IP 1747 | //EG(vm_interrupt) = 0; 1748 | MEM_OP2_1_ZTS mov, byte, executor_globals, vm_interrupt, 0, r0 1749 | //if (EG(timed_out)) { 1750 | MEM_OP2_1_ZTS cmp, byte, executor_globals, timed_out, 0, r0 1751 | je >1 1752 | //zend_timeout(); 1753 | EXT_CALL zend_timeout, r0 1754 |1: 1755 | //} else if (zend_interrupt_function) { 1756 if (zend_interrupt_function) { 1757 | //zend_interrupt_function(execute_data); 1758 |.if X64 1759 | mov CARG1, FP 1760 | EXT_CALL zend_interrupt_function, r0 1761 |.else 1762 | mov aword A1, FP 1763 | EXT_CALL zend_interrupt_function, r0 1764 |.endif 1765 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 1766 | je >1 1767 | EXT_CALL zend_jit_exception_in_interrupt_handler_helper, r0 1768 |1: 1769 | //ZEND_VM_ENTER(); 1770 | //execute_data = EG(current_execute_data); 1771 | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 1772 | LOAD_IP 1773 } 1774 | //ZEND_VM_CONTINUE() 1775 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1776 | ADD_HYBRID_SPAD 1777 | JMP_IP 1778 } else if (GCC_GLOBAL_REGS) { 1779 | add r4, SPAD // stack alignment 1780 | JMP_IP 1781 } else { 1782 | mov FP, aword T2 // restore FP 1783 | mov RX, aword T3 // restore IP 1784 | add r4, NR_SPAD // stack alignment 1785 | mov r0, 1 // ZEND_VM_ENTER 1786 | ret 1787 } 1788 1789 return 1; 1790} 1791 1792static int zend_jit_exception_handler_stub(dasm_State **Dst) 1793{ 1794 |->exception_handler: 1795 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1796 const void *handler = zend_get_opcode_handler_func(EG(exception_op)); 1797 1798 | ADD_HYBRID_SPAD 1799 | EXT_CALL handler, r0 1800 | JMP_IP 1801 } else { 1802 const void *handler = EG(exception_op)->handler; 1803 1804 if (GCC_GLOBAL_REGS) { 1805 | add r4, SPAD // stack alignment 1806 | EXT_JMP handler, r0 1807 } else { 1808 | mov FCARG1a, FP 1809 | EXT_CALL handler, r0 1810 | mov FP, aword T2 // restore FP 1811 | mov RX, aword T3 // restore IP 1812 | add r4, NR_SPAD // stack alignment 1813 | test eax, eax 1814 | jl >1 1815 | mov r0, 1 // ZEND_VM_ENTER 1816 |1: 1817 | ret 1818 } 1819 } 1820 1821 return 1; 1822} 1823 1824static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) 1825{ 1826 |->exception_handler_undef: 1827 | MEM_OP2_2_ZTS mov, r0, aword, executor_globals, opline_before_exception, r0 1828 | test byte OP:r0->result_type, (IS_TMP_VAR|IS_VAR) 1829 | jz >1 1830 | mov eax, dword OP:r0->result.var 1831 | SET_Z_TYPE_INFO FP + r0, IS_UNDEF 1832 |1: 1833 | jmp ->exception_handler 1834 1835 return 1; 1836} 1837 1838 1839static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst) 1840{ 1841 |->exception_handler_free_op1_op2: 1842 | UNDEF_OPLINE_RESULT_IF_USED 1843 | test byte OP:RX->op1_type, (IS_TMP_VAR|IS_VAR) 1844 | je >9 1845 | mov eax, dword OP:RX->op1.var 1846 | add r0, FP 1847 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1848 |9: 1849 | test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR) 1850 | je >9 1851 | mov eax, dword OP:RX->op2.var 1852 | add r0, FP 1853 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1854 |9: 1855 | jmp ->exception_handler 1856 return 1; 1857} 1858 1859static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst) 1860{ 1861 |->exception_handler_free_op2: 1862 | MEM_OP2_2_ZTS mov, RX, aword, executor_globals, opline_before_exception, r0 1863 | UNDEF_OPLINE_RESULT_IF_USED 1864 | test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR) 1865 | je >9 1866 | mov eax, dword OP:RX->op2.var 1867 | add r0, FP 1868 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1869 |9: 1870 | jmp ->exception_handler 1871 return 1; 1872} 1873 1874static int zend_jit_leave_function_stub(dasm_State **Dst) 1875{ 1876 |->leave_function_handler: 1877 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 1878 | test FCARG1d, ZEND_CALL_TOP 1879 | jnz >1 1880 | EXT_CALL zend_jit_leave_nested_func_helper, r0 1881 | ADD_HYBRID_SPAD 1882 | JMP_IP 1883 |1: 1884 | EXT_CALL zend_jit_leave_top_func_helper, r0 1885 | ADD_HYBRID_SPAD 1886 | JMP_IP 1887 } else { 1888 if (GCC_GLOBAL_REGS) { 1889 | add r4, SPAD 1890 } else { 1891 | mov FCARG2a, FP 1892 | mov FP, aword T2 // restore FP 1893 | mov RX, aword T3 // restore IP 1894 | add r4, NR_SPAD 1895 } 1896 | test FCARG1d, ZEND_CALL_TOP 1897 | jnz >1 1898 | EXT_JMP zend_jit_leave_nested_func_helper, r0 1899 |1: 1900 | EXT_JMP zend_jit_leave_top_func_helper, r0 1901 } 1902 1903 return 1; 1904} 1905 1906static int zend_jit_leave_throw_stub(dasm_State **Dst) 1907{ 1908 |->leave_throw_handler: 1909 | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { 1910 if (GCC_GLOBAL_REGS) { 1911 | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION 1912 | je >5 1913 | // EG(opline_before_exception) = opline; 1914 | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, IP, r0 1915 |5: 1916 | // opline = EG(exception_op); 1917 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1918 | // HANDLE_EXCEPTION() 1919 | jmp ->exception_handler 1920 } else { 1921 | GET_IP FCARG1a 1922 | cmp byte OP:FCARG1a->opcode, ZEND_HANDLE_EXCEPTION 1923 | je >5 1924 | // EG(opline_before_exception) = opline; 1925 | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, FCARG1a, r0 1926 |5: 1927 | // opline = EG(exception_op); 1928 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1929 | mov FP, aword T2 // restore FP 1930 | mov RX, aword T3 // restore IP 1931 | add r4, NR_SPAD // stack alignment 1932 | mov r0, 2 // ZEND_VM_LEAVE 1933 | ret 1934 } 1935 1936 return 1; 1937} 1938 1939static int zend_jit_icall_throw_stub(dasm_State **Dst) 1940{ 1941 |->icall_throw_handler: 1942 | // zend_rethrow_exception(zend_execute_data *execute_data) 1943 | mov IP, aword EX->opline 1944 | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { 1945 | cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION 1946 | je >1 1947 | // EG(opline_before_exception) = opline; 1948 | MEM_OP2_1_ZTS mov, aword, executor_globals, opline_before_exception, IP, r0 1949 |1: 1950 | // opline = EG(exception_op); 1951 | LOAD_IP_ADDR_ZTS executor_globals, exception_op 1952 || if (GCC_GLOBAL_REGS) { 1953 | mov aword EX->opline, IP 1954 || } 1955 | // HANDLE_EXCEPTION() 1956 | jmp ->exception_handler 1957 1958 return 1; 1959} 1960 1961static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) 1962{ 1963 |->throw_cannot_pass_by_ref: 1964 | mov r0, EX->opline 1965 | mov ecx, dword OP:r0->result.var 1966 | SET_Z_TYPE_INFO RX+r1, IS_UNDEF 1967 | // last EX(call) frame may be delayed 1968 | cmp RX, EX->call 1969 | je >1 1970 | mov r1, EX->call 1971 | mov EX:RX->prev_execute_data, r1 1972 | mov EX->call, RX 1973 |1: 1974 | mov RX, r0 1975 | mov FCARG1d, dword OP:r0->op2.num 1976 | EXT_CALL zend_cannot_pass_by_reference, r0 1977 | cmp byte OP:RX->op1_type, IS_TMP_VAR 1978 | jne >9 1979 | mov eax, dword OP:RX->op1.var 1980 | add r0, FP 1981 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL 1982 |9: 1983 | jmp ->exception_handler 1984 1985 return 1; 1986} 1987 1988static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) 1989{ 1990 |->undefined_offset_ex: 1991 | SAVE_IP 1992 | jmp ->undefined_offset 1993 1994 return 1; 1995} 1996 1997static int zend_jit_undefined_offset_stub(dasm_State **Dst) 1998{ 1999 |->undefined_offset: 2000 |.if X64WIN 2001 | sub r4, 0x28 2002 |.elif X64 2003 | sub r4, 8 2004 |.else 2005 | sub r4, 12 2006 |.endif 2007 | mov r0, EX->opline 2008 | mov ecx, dword OP:r0->result.var 2009 | cmp byte OP:r0->op2_type, IS_CONST 2010 | SET_Z_TYPE_INFO FP + r1, IS_NULL 2011 | jne >2 2012 |.if X64 2013 | movsxd r1, dword OP:r0->op2.constant 2014 | add r0, r1 2015 |.else 2016 | mov r0, aword OP:r0->op2.zv 2017 |.endif 2018 | jmp >3 2019 |2: 2020 | mov eax, dword OP:r0->op2.var 2021 | add r0, FP 2022 |3: 2023 |.if X64WIN 2024 | mov CARG1, E_WARNING 2025 | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT 2026 | mov CARG3, aword [r0] 2027 | EXT_CALL zend_error, r0 2028 | add r4, 0x28 // stack alignment 2029 |.elif X64 2030 | mov CARG1, E_WARNING 2031 | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT 2032 | mov CARG3, aword [r0] 2033 | EXT_CALL zend_error, r0 2034 | add r4, 8 // stack alignment 2035 |.else 2036 | sub r4, 4 2037 | push aword [r0] 2038 | push "Undefined array key " ZEND_LONG_FMT 2039 | push E_WARNING 2040 | EXT_CALL zend_error, r0 2041 | add r4, 28 2042 |.endif 2043 | ret 2044 2045 return 1; 2046} 2047 2048static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) 2049{ 2050 |->undefined_index_ex: 2051 | SAVE_IP 2052 | jmp ->undefined_index 2053 2054 return 1; 2055} 2056 2057static int zend_jit_undefined_index_stub(dasm_State **Dst) 2058{ 2059 |->undefined_index: 2060 |.if X64WIN 2061 | sub r4, 0x28 2062 |.elif X64 2063 | sub r4, 8 2064 |.else 2065 | sub r4, 12 2066 |.endif 2067 | mov r0, EX->opline 2068 | mov ecx, dword OP:r0->result.var 2069 | cmp byte OP:r0->op2_type, IS_CONST 2070 | SET_Z_TYPE_INFO FP + r1, IS_NULL 2071 | jne >2 2072 |.if X64 2073 | movsxd r1, dword OP:r0->op2.constant 2074 | add r0, r1 2075 |.else 2076 | mov r0, aword OP:r0->op2.zv 2077 |.endif 2078 | jmp >3 2079 |2: 2080 | mov eax, dword OP:r0->op2.var 2081 | add r0, FP 2082 |3: 2083 |.if X64WIN 2084 | mov CARG1, E_WARNING 2085 | LOAD_ADDR CARG2, "Undefined array key \"%s\"" 2086 | mov CARG3, aword [r0] 2087 | add CARG3, offsetof(zend_string, val) 2088 | EXT_CALL zend_error, r0 2089 | add r4, 0x28 2090 |.elif X64 2091 | mov CARG1, E_WARNING 2092 | LOAD_ADDR CARG2, "Undefined array key \"%s\"" 2093 | mov CARG3, aword [r0] 2094 | add CARG3, offsetof(zend_string, val) 2095 | EXT_CALL zend_error, r0 2096 | add r4, 8 2097 |.else 2098 | sub r4, 4 2099 | mov r0, aword [r0] 2100 | add r0, offsetof(zend_string, val) 2101 | push r0 2102 | push "Undefined array key \"%s\"" 2103 | push E_WARNING 2104 | EXT_CALL zend_error, r0 2105 | add r4, 28 2106 |.endif 2107 | ret 2108 2109 return 1; 2110} 2111 2112static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) 2113{ 2114 |->cannot_add_element_ex: 2115 | SAVE_IP 2116 | jmp ->cannot_add_element 2117 2118 return 1; 2119} 2120 2121static int zend_jit_cannot_add_element_stub(dasm_State **Dst) 2122{ 2123 |->cannot_add_element: 2124 |.if X64WIN 2125 | sub r4, 0x28 2126 |.elif X64 2127 | sub r4, 8 2128 |.else 2129 | sub r4, 12 2130 |.endif 2131 | mov r0, EX->opline 2132 | cmp byte OP:r0->result_type, IS_UNUSED 2133 | jz >1 2134 | mov eax, dword OP:r0->result.var 2135 | SET_Z_TYPE_INFO FP + r0, IS_NULL 2136 |1: 2137 |.if X64WIN 2138 | xor CARG1, CARG1 2139 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2140 | EXT_CALL zend_throw_error, r0 2141 | add r4, 0x28 2142 |.elif X64 2143 | xor CARG1, CARG1 2144 | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" 2145 | EXT_CALL zend_throw_error, r0 2146 | add r4, 8 2147 |.else 2148 | sub r4, 8 2149 | push "Cannot add element to the array as the next element is already occupied" 2150 | push 0 2151 | EXT_CALL zend_throw_error, r0 2152 | add r4, 28 2153 |.endif 2154 | ret 2155 2156 return 1; 2157} 2158 2159static int zend_jit_undefined_function_stub(dasm_State **Dst) 2160{ 2161 |->undefined_function: 2162 | mov r0, aword EX->opline 2163 |.if X64 2164 | xor CARG1, CARG1 2165 | LOAD_ADDR CARG2, "Call to undefined function %s()" 2166 | movsxd CARG3, dword [r0 + offsetof(zend_op, op2.constant)] 2167 | mov CARG3, aword [r0 + CARG3] 2168 | add CARG3, offsetof(zend_string, val) 2169 | EXT_CALL zend_throw_error, r0 2170 |.else 2171 | mov r0, aword [r0 + offsetof(zend_op, op2.zv)] 2172 | mov r0, aword [r0] 2173 | add r0, offsetof(zend_string, val) 2174 | mov aword A3, r0 2175 | mov aword A2, "Call to undefined function %s()" 2176 | mov aword A1, 0 2177 | EXT_CALL zend_throw_error, r0 2178 |.endif 2179 | jmp ->exception_handler 2180 return 1; 2181} 2182 2183static int zend_jit_negative_shift_stub(dasm_State **Dst) 2184{ 2185 |->negative_shift: 2186 | mov RX, EX->opline 2187 |.if X64 2188 |.if WIN 2189 | LOAD_ADDR CARG1, &zend_ce_arithmetic_error 2190 | mov CARG1, aword [CARG1] 2191 |.else 2192 | LOAD_ADDR CARG1, zend_ce_arithmetic_error 2193 |.endif 2194 | LOAD_ADDR CARG2, "Bit shift by negative number" 2195 | EXT_CALL zend_throw_error, r0 2196 |.else 2197 | sub r4, 8 2198 | push "Bit shift by negative number" 2199 |.if WIN 2200 | LOAD_ADDR r0, &zend_ce_arithmetic_error 2201 | push aword [r0] 2202 |.else 2203 | PUSH_ADDR zend_ce_arithmetic_error, r0 2204 |.endif 2205 | EXT_CALL zend_throw_error, r0 2206 | add r4, 16 2207 |.endif 2208 | jmp ->exception_handler_free_op1_op2 2209 return 1; 2210} 2211 2212static int zend_jit_mod_by_zero_stub(dasm_State **Dst) 2213{ 2214 |->mod_by_zero: 2215 | mov RX, EX->opline 2216 |.if X64 2217 |.if WIN 2218 | LOAD_ADDR CARG1, &zend_ce_division_by_zero_error 2219 | mov CARG1, aword [CARG1] 2220 |.else 2221 | LOAD_ADDR CARG1, zend_ce_division_by_zero_error 2222 |.endif 2223 | LOAD_ADDR CARG2, "Modulo by zero" 2224 | EXT_CALL zend_throw_error, r0 2225 |.else 2226 | sub r4, 8 2227 | push "Modulo by zero" 2228 |.if WIN 2229 | LOAD_ADDR r0, &zend_ce_division_by_zero_error 2230 | push aword [r0] 2231 |.else 2232 | PUSH_ADDR zend_ce_division_by_zero_error, r0 2233 |.endif 2234 | EXT_CALL zend_throw_error, r0 2235 | add r4, 16 2236 |.endif 2237 | jmp ->exception_handler_free_op1_op2 2238 return 1; 2239} 2240 2241static int zend_jit_invalid_this_stub(dasm_State **Dst) 2242{ 2243 |->invalid_this: 2244 | UNDEF_OPLINE_RESULT 2245 |.if X64 2246 | xor CARG1, CARG1 2247 | LOAD_ADDR CARG2, "Using $this when not in object context" 2248 | EXT_CALL zend_throw_error, r0 2249 |.else 2250 | sub r4, 8 2251 | push "Using $this when not in object context" 2252 | push 0 2253 | EXT_CALL zend_throw_error, r0 2254 | add r4, 16 2255 |.endif 2256 | jmp ->exception_handler 2257 return 1; 2258} 2259 2260static int zend_jit_double_one_stub(dasm_State **Dst) 2261{ 2262 |->one: 2263 |.dword 0, 0x3ff00000 2264 return 1; 2265} 2266 2267static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) 2268{ 2269 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2270 return 1; 2271 } 2272 2273 |->hybrid_runtime_jit: 2274 | EXT_CALL zend_runtime_jit, r0 2275 | JMP_IP 2276 return 1; 2277} 2278 2279static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) 2280{ 2281 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2282 return 1; 2283 } 2284 2285 |->hybrid_profile_jit: 2286 | // ++zend_jit_profile_counter; 2287 | .if X64 2288 | LOAD_ADDR r0, &zend_jit_profile_counter 2289 | inc aword [r0] 2290 | .else 2291 | inc aword [&zend_jit_profile_counter] 2292 | .endif 2293 | // op_array = (zend_op_array*)EX(func); 2294 | mov r0, EX->func 2295 | // run_time_cache = EX(run_time_cache); 2296 | mov r2, EX->run_time_cache 2297 | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); 2298 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2299 | // ++ZEND_COUNTER_INFO(op_array) 2300 | inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void*)] 2301 | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() 2302 | jmp aword [r0 + offsetof(zend_jit_op_array_extension, orig_handler)] 2303 return 1; 2304} 2305 2306static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) 2307{ 2308 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2309 return 1; 2310 } 2311 2312 |->hybrid_hot_code: 2313 | mov word [r2], ZEND_JIT_COUNTER_INIT 2314 | mov FCARG1a, FP 2315 | GET_IP FCARG2a 2316 | EXT_CALL zend_jit_hot_func, r0 2317 | JMP_IP 2318 return 1; 2319} 2320 2321/* 2322 * This code is based Mike Pall's "Hashed profile counters" idea, implemented 2323 * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual 2324 * property disclosure and research opportunities" email 2325 * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html 2326 * 2327 * In addition we use a variation of Knuth's multiplicative hash function 2328 * described at https://code.i-harness.com/en/q/a21ce 2329 * 2330 * uint64_t hash(uint64_t x) { 2331 * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; 2332 * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; 2333 * x = x ^ (x >> 31); 2334 * return x; 2335 * } 2336 * 2337 * uint_32_t hash(uint32_t x) { 2338 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2339 * x = ((x >> 16) ^ x) * 0x45d9f3b; 2340 * x = (x >> 16) ^ x; 2341 * return x; 2342 * } 2343 * 2344 */ 2345static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) 2346{ 2347 | mov r0, EX->func 2348 | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2349 | mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)] 2350 | sub word [r2], cost 2351 | jle ->hybrid_hot_code 2352 | GET_IP r2 2353 | sub r2, aword [r0 + offsetof(zend_op_array, opcodes)] 2354 | // divide by sizeof(zend_op) 2355 | .if X64 2356 || ZEND_ASSERT(sizeof(zend_op) == 32); 2357 | sar r2, 2 2358 | .else 2359 || ZEND_ASSERT(sizeof(zend_op) == 28); 2360 | imul r2, 0xb6db6db7 2361 | .endif 2362 | .if X64 2363 | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2364 | .else 2365 | jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)] 2366 | .endif 2367 return 1; 2368} 2369 2370static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) 2371{ 2372 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2373 return 1; 2374 } 2375 2376 |->hybrid_func_hot_counter: 2377 2378 return zend_jit_hybrid_hot_counter_stub(Dst, 2379 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2380} 2381 2382static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) 2383{ 2384 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2385 return 1; 2386 } 2387 2388 |->hybrid_loop_hot_counter: 2389 2390 return zend_jit_hybrid_hot_counter_stub(Dst, 2391 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2392} 2393 2394static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) 2395{ 2396 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { 2397 return 1; 2398 } 2399 2400 |->hybrid_hot_trace: 2401 | mov word [r2], ZEND_JIT_COUNTER_INIT 2402 | mov FCARG1a, FP 2403 | GET_IP FCARG2a 2404 | EXT_CALL zend_jit_trace_hot_root, r0 2405 | test eax, eax // TODO : remove this check at least for HYBRID VM ??? 2406 | jl >1 2407 | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 2408 | LOAD_IP 2409 | JMP_IP 2410 |1: 2411 | EXT_JMP zend_jit_halt_op->handler, r0 2412 return 1; 2413} 2414 2415static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) 2416{ 2417 | mov r0, EX->func 2418 | mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2419 | mov r1, aword [r1 + offsetof(zend_jit_op_array_trace_extension, offset)] 2420 | mov r2, aword [IP + r1 + offsetof(zend_op_trace_info, counter)] 2421 | sub word [r2], cost 2422 | jle ->hybrid_hot_trace 2423 | jmp aword [IP + r1] 2424 return 1; 2425} 2426 2427static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) 2428{ 2429 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { 2430 return 1; 2431 } 2432 2433 |->hybrid_func_trace_counter: 2434 2435 return zend_jit_hybrid_trace_counter_stub(Dst, 2436 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); 2437} 2438 2439static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) 2440{ 2441 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { 2442 return 1; 2443 } 2444 2445 |->hybrid_ret_trace_counter: 2446 2447 return zend_jit_hybrid_trace_counter_stub(Dst, 2448 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); 2449} 2450 2451static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) 2452{ 2453 if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { 2454 return 1; 2455 } 2456 2457 |->hybrid_loop_trace_counter: 2458 2459 return zend_jit_hybrid_trace_counter_stub(Dst, 2460 ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); 2461} 2462 2463static int zend_jit_trace_halt_stub(dasm_State **Dst) 2464{ 2465 |->trace_halt: 2466 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2467 | ADD_HYBRID_SPAD 2468 | EXT_JMP zend_jit_halt_op->handler, r0 2469 } else if (GCC_GLOBAL_REGS) { 2470 | add r4, SPAD // stack alignment 2471 | xor IP, IP // PC must be zero 2472 | ret 2473 } else { 2474 | mov FP, aword T2 // restore FP 2475 | mov RX, aword T3 // restore IP 2476 | add r4, NR_SPAD // stack alignment 2477 | mov r0, -1 // ZEND_VM_RETURN 2478 | ret 2479 } 2480 return 1; 2481} 2482 2483static int zend_jit_trace_exit_stub(dasm_State **Dst) 2484{ 2485 |->trace_exit: 2486 | 2487 | // Save CPU registers 2488 |.if X64 2489 | sub r4, 16*8+16*8-8 /* CPU regs + SSE regs */ 2490 | mov aword [r4+15*8], r15 2491 | mov aword [r4+11*8], r11 2492 | mov aword [r4+10*8], r10 2493 | mov aword [r4+9*8], r9 2494 | mov aword [r4+8*8], r8 2495 | mov aword [r4+7*8], rdi 2496 | mov aword [r4+6*8], rsi 2497 | mov aword [r4+2*8], rdx 2498 | mov aword [r4+1*8], rcx 2499 | mov aword [r4+0*8], rax 2500 | mov FCARG1a, aword [r4+16*8+16*8-8] // exit_num = POP 2501 | mov FCARG2a, r4 2502 | movsd qword [r4+16*8+15*8], xmm15 2503 | movsd qword [r4+16*8+14*8], xmm14 2504 | movsd qword [r4+16*8+13*8], xmm13 2505 | movsd qword [r4+16*8+12*8], xmm12 2506 | movsd qword [r4+16*8+11*8], xmm11 2507 | movsd qword [r4+16*8+10*8], xmm10 2508 | movsd qword [r4+16*8+9*8], xmm9 2509 | movsd qword [r4+16*8+8*8], xmm8 2510 | movsd qword [r4+16*8+7*8], xmm7 2511 | movsd qword [r4+16*8+6*8], xmm6 2512 | movsd qword [r4+16*8+5*8], xmm5 2513 | movsd qword [r4+16*8+4*8], xmm4 2514 | movsd qword [r4+16*8+3*8], xmm3 2515 | movsd qword [r4+16*8+2*8], xmm2 2516 | movsd qword [r4+16*8+1*8], xmm1 2517 | movsd qword [r4+16*8+0*8], xmm0 2518 |.if X64WIN 2519 | sub r4, 32 /* shadow space */ 2520 |.endif 2521 |.else 2522 | sub r4, 8*4+8*8-4 /* CPU regs + SSE regs */ 2523 | mov aword [r4+7*4], edi 2524 | mov aword [r4+2*4], edx 2525 | mov aword [r4+1*4], ecx 2526 | mov aword [r4+0*4], eax 2527 | mov FCARG1a, aword [r4+8*4+8*8-4] // exit_num = POP 2528 | mov FCARG2a, r4 2529 | movsd qword [r4+8*4+7*8], xmm7 2530 | movsd qword [r4+8*4+6*8], xmm6 2531 | movsd qword [r4+8*4+5*8], xmm5 2532 | movsd qword [r4+8*4+4*8], xmm4 2533 | movsd qword [r4+8*4+3*8], xmm3 2534 | movsd qword [r4+8*4+2*8], xmm2 2535 | movsd qword [r4+8*4+1*8], xmm1 2536 | movsd qword [r4+8*4+0*8], xmm0 2537 |.endif 2538 | 2539 | // EX(opline) = opline 2540 | SAVE_IP 2541 | // zend_jit_trace_exit(trace_num, exit_num) 2542 | EXT_CALL zend_jit_trace_exit, r0 2543 |.if X64WIN 2544 | add r4, 16*8+16*8+32 /* CPU regs + SSE regs + shadow space */ 2545 |.elif X64 2546 | add r4, 16*8+16*8 /* CPU regs + SSE regs */ 2547 |.else 2548 | add r4, 8*4+8*8 /* CPU regs + SSE regs */ 2549 |.endif 2550 2551 | test eax, eax 2552 | jne >1 2553 2554 | // execute_data = EG(current_execute_data) 2555 | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 2556 | // opline = EX(opline) 2557 | LOAD_IP 2558 2559 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2560 | ADD_HYBRID_SPAD 2561 | JMP_IP 2562 } else if (GCC_GLOBAL_REGS) { 2563 | add r4, SPAD // stack alignment 2564 | JMP_IP 2565 } else { 2566 | mov FP, aword T2 // restore FP 2567 | mov RX, aword T3 // restore IP 2568 | add r4, NR_SPAD // stack alignment 2569 | mov r0, 1 // ZEND_VM_ENTER 2570 | ret 2571 } 2572 2573 |1: 2574 | jl ->trace_halt 2575 2576 | // execute_data = EG(current_execute_data) 2577 | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 2578 | // opline = EX(opline) 2579 | LOAD_IP 2580 2581 | // check for interrupt (try to avoid this ???) 2582 | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 2583 | jne ->interrupt_handler 2584 2585 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2586 | ADD_HYBRID_SPAD 2587 | mov r0, EX->func 2588 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2589 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2590 | jmp aword [IP + r0] 2591 } else if (GCC_GLOBAL_REGS) { 2592 | add r4, SPAD // stack alignment 2593 | mov r0, EX->func 2594 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2595 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2596 | jmp aword [IP + r0] 2597 } else { 2598 | mov IP, aword EX->opline 2599 | mov FCARG1a, FP 2600 | mov r0, EX->func 2601 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 2602 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 2603 | call aword [IP + r0] 2604 | test eax, eax 2605 | jl ->trace_halt 2606 | mov FP, aword T2 // restore FP 2607 | mov RX, aword T3 // restore IP 2608 | add r4, NR_SPAD // stack alignment 2609 | mov r0, 1 // ZEND_VM_ENTER 2610 | ret 2611 } 2612 2613 return 1; 2614} 2615 2616static int zend_jit_trace_escape_stub(dasm_State **Dst) 2617{ 2618 |->trace_escape: 2619 | 2620 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2621 | ADD_HYBRID_SPAD 2622 | JMP_IP 2623 } else if (GCC_GLOBAL_REGS) { 2624 | add r4, SPAD // stack alignment 2625 | JMP_IP 2626 } else { 2627 | mov FP, aword T2 // restore FP 2628 | mov RX, aword T3 // restore IP 2629 | add r4, NR_SPAD // stack alignment 2630 | mov r0, 1 // ZEND_VM_ENTER 2631 | ret 2632 } 2633 2634 return 1; 2635} 2636 2637/* Keep 32 exit points in a single code block */ 2638#define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes 2639#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points 2640 2641static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) 2642{ 2643 uint32_t i; 2644 2645 for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP - 1; i++) { 2646 | push byte i 2647 | .byte 0xeb, (4*(ZEND_JIT_EXIT_POINTS_PER_GROUP-i)-6) // jmp >1 2648 } 2649 | push byte i 2650 |// 1: 2651 | add aword [r4], n 2652 | jmp ->trace_exit 2653 2654 return 1; 2655} 2656 2657#ifdef CONTEXT_THREADED_JIT 2658static int zend_jit_context_threaded_call_stub(dasm_State **Dst) 2659{ 2660 |->context_threaded_call: 2661 | pop r0 2662 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 2663 | ADD_HYBRID_SPAD 2664 | jmp aword [IP] 2665 } else if (GCC_GLOBAL_REGS) { 2666 | add r4, SPAD // stack alignment 2667 | jmp aword [IP] 2668 } else { 2669 ZEND_UNREACHABLE(); 2670 // TODO: context threading can't work without GLOBAL REGS because we have to change 2671 // the value of execute_data in execute_ex() 2672 | mov FCARG1a, FP 2673 | mov r0, aword [FP] 2674 | mov FP, aword T2 // restore FP 2675 | mov RX, aword T3 // restore IP 2676 | add r4, NR_SPAD // stack alignment 2677 | jmp aword [r0] 2678 } 2679 return 1; 2680} 2681#endif 2682 2683static int zend_jit_assign_to_variable(dasm_State **Dst, 2684 const zend_op *opline, 2685 zend_jit_addr var_use_addr, 2686 zend_jit_addr var_addr, 2687 uint32_t var_info, 2688 uint32_t var_def_info, 2689 zend_uchar val_type, 2690 zend_jit_addr val_addr, 2691 uint32_t val_info, 2692 zend_jit_addr res_addr, 2693 zend_bool check_exception); 2694 2695static int zend_jit_assign_const_stub(dasm_State **Dst) 2696{ 2697 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 2698 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 2699 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2700 2701 |->assign_const: 2702 |.if X64WIN 2703 | sub r4, 0x28 2704 |.elif X64 2705 | sub r4, 8 2706 |.else 2707 | sub r4, 12 2708 |.endif 2709 if (!zend_jit_assign_to_variable( 2710 Dst, NULL, 2711 var_addr, var_addr, -1, -1, 2712 IS_CONST, val_addr, val_info, 2713 0, 0)) { 2714 return 0; 2715 } 2716 |.if X64WIN 2717 | add r4, 0x28 2718 |.elif X64 2719 | add r4, 8 2720 |.else 2721 | add r4, 12 2722 |.endif 2723 | ret 2724 return 1; 2725} 2726 2727static int zend_jit_assign_tmp_stub(dasm_State **Dst) 2728{ 2729 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 2730 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 2731 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; 2732 2733 |->assign_tmp: 2734 |.if X64WIN 2735 | sub r4, 0x28 2736 |.elif X64 2737 | sub r4, 8 2738 |.else 2739 | sub r4, 12 2740 |.endif 2741 if (!zend_jit_assign_to_variable( 2742 Dst, NULL, 2743 var_addr, var_addr, -1, -1, 2744 IS_TMP_VAR, val_addr, val_info, 2745 0, 0)) { 2746 return 0; 2747 } 2748 |.if X64WIN 2749 | add r4, 0x28 2750 |.elif X64 2751 | add r4, 8 2752 |.else 2753 | add r4, 12 2754 |.endif 2755 | ret 2756 return 1; 2757} 2758 2759static int zend_jit_assign_var_stub(dasm_State **Dst) 2760{ 2761 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 2762 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 2763 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; 2764 2765 |->assign_var: 2766 |.if X64WIN 2767 | sub r4, 0x28 2768 |.elif X64 2769 | sub r4, 8 2770 |.else 2771 | sub r4, 12 2772 |.endif 2773 if (!zend_jit_assign_to_variable( 2774 Dst, NULL, 2775 var_addr, var_addr, -1, -1, 2776 IS_VAR, val_addr, val_info, 2777 0, 0)) { 2778 return 0; 2779 } 2780 |.if X64WIN 2781 | add r4, 0x28 2782 |.elif X64 2783 | add r4, 8 2784 |.else 2785 | add r4, 12 2786 |.endif 2787 | ret 2788 return 1; 2789} 2790 2791static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) 2792{ 2793 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 2794 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 2795 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; 2796 2797 |->assign_cv_noref: 2798 |.if X64WIN 2799 | sub r4, 0x28 2800 |.elif X64 2801 | sub r4, 8 2802 |.else 2803 | sub r4, 12 2804 |.endif 2805 if (!zend_jit_assign_to_variable( 2806 Dst, NULL, 2807 var_addr, var_addr, -1, -1, 2808 IS_CV, val_addr, val_info, 2809 0, 0)) { 2810 return 0; 2811 } 2812 |.if X64WIN 2813 | add r4, 0x28 2814 |.elif X64 2815 | add r4, 8 2816 |.else 2817 | add r4, 12 2818 |.endif 2819 | ret 2820 return 1; 2821} 2822 2823static int zend_jit_assign_cv_stub(dasm_State **Dst) 2824{ 2825 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 2826 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 2827 uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; 2828 2829 |->assign_cv: 2830 |.if X64WIN 2831 | sub r4, 0x28 2832 |.elif X64 2833 | sub r4, 8 2834 |.else 2835 | sub r4, 12 2836 |.endif 2837 if (!zend_jit_assign_to_variable( 2838 Dst, NULL, 2839 var_addr, var_addr, -1, -1, 2840 IS_CV, val_addr, val_info, 2841 0, 0)) { 2842 return 0; 2843 } 2844 |.if X64WIN 2845 | add r4, 0x28 2846 |.elif X64 2847 | add r4, 8 2848 |.else 2849 | add r4, 12 2850 |.endif 2851 | ret 2852 return 1; 2853} 2854 2855static const zend_jit_stub zend_jit_stubs[] = { 2856 JIT_STUB(interrupt_handler), 2857 JIT_STUB(exception_handler), 2858 JIT_STUB(exception_handler_undef), 2859 JIT_STUB(exception_handler_free_op1_op2), 2860 JIT_STUB(exception_handler_free_op2), 2861 JIT_STUB(leave_function), 2862 JIT_STUB(leave_throw), 2863 JIT_STUB(icall_throw), 2864 JIT_STUB(throw_cannot_pass_by_ref), 2865 JIT_STUB(undefined_offset), 2866 JIT_STUB(undefined_index), 2867 JIT_STUB(cannot_add_element), 2868 JIT_STUB(undefined_offset_ex), 2869 JIT_STUB(undefined_index_ex), 2870 JIT_STUB(cannot_add_element_ex), 2871 JIT_STUB(undefined_function), 2872 JIT_STUB(negative_shift), 2873 JIT_STUB(mod_by_zero), 2874 JIT_STUB(invalid_this), 2875 JIT_STUB(trace_halt), 2876 JIT_STUB(trace_exit), 2877 JIT_STUB(trace_escape), 2878 JIT_STUB(hybrid_runtime_jit), 2879 JIT_STUB(hybrid_profile_jit), 2880 JIT_STUB(hybrid_hot_code), 2881 JIT_STUB(hybrid_func_hot_counter), 2882 JIT_STUB(hybrid_loop_hot_counter), 2883 JIT_STUB(hybrid_hot_trace), 2884 JIT_STUB(hybrid_func_trace_counter), 2885 JIT_STUB(hybrid_ret_trace_counter), 2886 JIT_STUB(hybrid_loop_trace_counter), 2887 JIT_STUB(assign_const), 2888 JIT_STUB(assign_tmp), 2889 JIT_STUB(assign_var), 2890 JIT_STUB(assign_cv_noref), 2891 JIT_STUB(assign_cv), 2892 JIT_STUB(double_one), 2893#ifdef CONTEXT_THREADED_JIT 2894 JIT_STUB(context_threaded_call), 2895#endif 2896}; 2897 2898#if ZTS && defined(ZEND_WIN32) 2899extern uint32_t _tls_index; 2900extern char *_tls_start; 2901extern char *_tls_end; 2902#endif 2903 2904static int zend_jit_setup(void) 2905{ 2906 if (!zend_cpu_supports_sse2()) { 2907 zend_error(E_CORE_ERROR, "CPU doesn't support SSE2"); 2908 return FAILURE; 2909 } 2910 allowed_opt_flags = 0; 2911 if (zend_cpu_supports_avx()) { 2912 allowed_opt_flags |= ZEND_JIT_CPU_AVX; 2913 } 2914 2915#if ZTS 2916# ifdef _WIN64 2917 tsrm_tls_index = _tls_index * sizeof(void*); 2918 2919 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ 2920 /* Probably, it might be better solution */ 2921 do { 2922 void ***tls_mem = ((void**)__readgsqword(0x58))[_tls_index]; 2923 void *val = _tsrm_ls_cache; 2924 size_t offset = 0; 2925 size_t size = (char*)&_tls_end - (char*)&_tls_start; 2926 2927 while (offset < size) { 2928 if (*tls_mem == val) { 2929 tsrm_tls_offset = offset; 2930 break; 2931 } 2932 tls_mem++; 2933 offset += sizeof(void*); 2934 } 2935 if (offset >= size) { 2936 // TODO: error message ??? 2937 return FAILURE; 2938 } 2939 } while(0); 2940# elif ZEND_WIN32 2941 tsrm_tls_index = _tls_index * sizeof(void*); 2942 2943 /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ 2944 /* Probably, it might be better solution */ 2945 do { 2946 void ***tls_mem = ((void***)__readfsdword(0x2c))[_tls_index]; 2947 void *val = _tsrm_ls_cache; 2948 size_t offset = 0; 2949 size_t size = (char*)&_tls_end - (char*)&_tls_start; 2950 2951 while (offset < size) { 2952 if (*tls_mem == val) { 2953 tsrm_tls_offset = offset; 2954 break; 2955 } 2956 tls_mem++; 2957 offset += sizeof(void*); 2958 } 2959 if (offset >= size) { 2960 // TODO: error message ??? 2961 return FAILURE; 2962 } 2963 } while(0); 2964# elif defined(__APPLE__) && defined(__x86_64__) 2965 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2966 if (tsrm_ls_cache_tcb_offset == 0) { 2967 size_t *ti; 2968 __asm__( 2969 "leaq __tsrm_ls_cache(%%rip),%0" 2970 : "=r" (ti)); 2971 tsrm_tls_offset = ti[2]; 2972 tsrm_tls_index = ti[1] * 8; 2973 } 2974# elif defined(__GNUC__) && defined(__x86_64__) 2975 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2976 if (tsrm_ls_cache_tcb_offset == 0) { 2977#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__) 2978 size_t ret; 2979 2980 asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0" 2981 : "=r" (ret)); 2982 tsrm_ls_cache_tcb_offset = ret; 2983#else 2984 size_t *ti; 2985 2986 __asm__( 2987 "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" 2988 : "=a" (ti)); 2989 tsrm_tls_offset = ti[1]; 2990 tsrm_tls_index = ti[0] * 16; 2991#endif 2992 } 2993# elif defined(__GNUC__) && defined(__i386__) 2994 tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); 2995 if (tsrm_ls_cache_tcb_offset == 0) { 2996#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__) 2997 size_t ret; 2998 2999 asm ("leal _tsrm_ls_cache@ntpoff,%0\n" 3000 : "=a" (ret)); 3001 tsrm_ls_cache_tcb_offset = ret; 3002#else 3003 size_t *ti, _ebx, _ecx, _edx; 3004 3005 __asm__( 3006 "call 1f\n" 3007 ".subsection 1\n" 3008 "1:\tmovl (%%esp), %%ebx\n\t" 3009 "ret\n" 3010 ".previous\n\t" 3011 "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" 3012 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t" 3013 "call ___tls_get_addr@plt\n\t" 3014 "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n" 3015 : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx)); 3016 tsrm_tls_offset = ti[1]; 3017 tsrm_tls_index = ti[0] * 8; 3018#endif 3019 } 3020# endif 3021#endif 3022 3023 return SUCCESS; 3024} 3025 3026static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) 3027{ 3028 | int3 3029 return 1; 3030} 3031 3032static int zend_jit_align_func(dasm_State **Dst) 3033{ 3034 reuse_ip = 0; 3035 delayed_call_chain = 0; 3036 last_valid_opline = NULL; 3037 use_last_vald_opline = 0; 3038 track_last_valid_opline = 0; 3039 jit_return_label = -1; 3040 |.align 16 3041 return 1; 3042} 3043 3044static int zend_jit_prologue(dasm_State **Dst) 3045{ 3046 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3047 | SUB_HYBRID_SPAD 3048 } else if (GCC_GLOBAL_REGS) { 3049 | sub r4, SPAD // stack alignment 3050 } else { 3051 | sub r4, NR_SPAD // stack alignment 3052 | mov aword T2, FP // save FP 3053 | mov aword T3, RX // save IP 3054 | mov FP, FCARG1a 3055 } 3056 return 1; 3057} 3058 3059static int zend_jit_label(dasm_State **Dst, unsigned int label) 3060{ 3061 |=>label: 3062 return 1; 3063} 3064 3065static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) 3066{ 3067 | // call->prev_execute_data = EX(call); 3068 if (call_level == 1) { 3069 | mov aword EX:RX->prev_execute_data, 0 3070 } else { 3071 | mov r0, EX->call 3072 | mov EX:RX->prev_execute_data, r0 3073 } 3074 | // EX(call) = call; 3075 | mov EX->call, RX 3076 3077 delayed_call_chain = 0; 3078 3079 return 1; 3080} 3081 3082static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) 3083{ 3084 if (last_valid_opline == opline) { 3085 zend_jit_use_last_valid_opline(); 3086 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 3087 zend_jit_use_last_valid_opline(); 3088 | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); 3089 } else { 3090 | LOAD_IP_ADDR opline 3091 } 3092 zend_jit_set_last_valid_opline(opline); 3093 3094 return 1; 3095} 3096 3097static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg) 3098{ 3099 if (last_valid_opline == opline) { 3100 zend_jit_use_last_valid_opline(); 3101 } else if (GCC_GLOBAL_REGS && last_valid_opline) { 3102 zend_jit_use_last_valid_opline(); 3103 | ADD_IP (opline - last_valid_opline) * sizeof(zend_op); 3104 } else if (!GCC_GLOBAL_REGS && set_ip_reg) { 3105 | LOAD_ADDR RX, opline 3106 | mov aword EX->opline, RX 3107 } else { 3108 | LOAD_IP_ADDR opline 3109 } 3110 zend_jit_set_last_valid_opline(opline); 3111 3112 return 1; 3113} 3114 3115static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) 3116{ 3117 if (delayed_call_chain) { 3118 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 3119 return 0; 3120 } 3121 } 3122 if (!zend_jit_set_ip(Dst, opline)) { 3123 return 0; 3124 } 3125 reuse_ip = 0; 3126 return 1; 3127} 3128 3129static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) 3130{ 3131#if 0 3132 if (!zend_jit_set_valid_ip(Dst, opline)) { 3133 return 0; 3134 } 3135 | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 3136 | jne ->interrupt_handler 3137#else 3138 | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 3139 if (exit_addr) { 3140 | jne &exit_addr 3141 } else if (last_valid_opline == opline) { 3142 || zend_jit_use_last_valid_opline(); 3143 | jne ->interrupt_handler 3144 } else { 3145 | jne >1 3146 |.cold_code 3147 |1: 3148 | LOAD_IP_ADDR opline 3149 | jmp ->interrupt_handler 3150 |.code 3151 } 3152#endif 3153 return 1; 3154} 3155 3156static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) 3157{ 3158 if (timeout_exit_addr) { 3159 | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 3160 | je =>loop_label 3161 | jmp &timeout_exit_addr 3162 } else { 3163 | jmp =>loop_label 3164 } 3165 return 1; 3166} 3167 3168static int zend_jit_check_exception(dasm_State **Dst) 3169{ 3170 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 3171 | jne ->exception_handler 3172 return 1; 3173} 3174 3175static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) 3176{ 3177 if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { 3178 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 3179 | jne ->exception_handler_undef 3180 return 1; 3181 } 3182 return zend_jit_check_exception(Dst); 3183} 3184 3185static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) 3186{ 3187 zend_regset regset = ZEND_REGSET_SCRATCH; 3188 3189#if ZTS 3190 if (1) { 3191#else 3192 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(jit_trace_num)))) { 3193#endif 3194 /* assignment to EG(jit_trace_num) shouldn't clober CPU register used by deoptimizer */ 3195 if (parent) { 3196 int i; 3197 int parent_vars_count = parent->exit_info[exit_num].stack_size; 3198 zend_jit_trace_stack *parent_stack = 3199 parent->stack_map + 3200 parent->exit_info[exit_num].stack_offset; 3201 3202 for (i = 0; i < parent_vars_count; i++) { 3203 if (STACK_REG(parent_stack, i) != ZREG_NONE) { 3204 if (STACK_REG(parent_stack, i) < ZREG_NUM) { 3205 ZEND_REGSET_EXCL(regset, STACK_REG(parent_stack, i)); 3206 } else if (STACK_REG(parent_stack, i) == ZREG_ZVAL_COPY_R0) { 3207 ZEND_REGSET_EXCL(regset, ZREG_R0); 3208 } 3209 } 3210 } 3211 } 3212 } 3213 3214 if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { 3215 ZEND_REGSET_EXCL(regset, ZREG_R0); 3216 } 3217 3218 current_trace_num = trace_num; 3219 3220 | // EG(jit_trace_num) = trace_num; 3221 if (regset == ZEND_REGSET_EMPTY) { 3222 | push r0 3223 | MEM_OP2_1_ZTS mov, dword, executor_globals, jit_trace_num, trace_num, r0 3224 | pop r0 3225 } else { 3226 zend_reg tmp = ZEND_REGSET_FIRST(regset); 3227 3228 | MEM_OP2_1_ZTS mov, dword, executor_globals, jit_trace_num, trace_num, Ra(tmp) 3229 (void)tmp; 3230 } 3231 3232 return 1; 3233} 3234 3235/* This taken from LuaJIT. Thanks to Mike Pall. */ 3236static uint32_t _asm_x86_inslen(const uint8_t* p) 3237{ 3238 static const uint8_t map_op1[256] = { 3239 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x20, 3240 0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51, 3241 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, 3242 0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51, 3243#if defined(__x86_64__) || defined(_M_X64) 3244 0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x14,0x14,0x14,0x14,0x14,0x14,0x14,0x14, 3245#else 3246 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, 3247#endif 3248 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51, 3249 0x51,0x51,0x92,0x92,0x10,0x10,0x12,0x11,0x45,0x86,0x52,0x93,0x51,0x51,0x51,0x51, 3250 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 3251 0x93,0x86,0x93,0x93,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 3252 0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x47,0x51,0x51,0x51,0x51,0x51, 3253#if defined(__x86_64__) || defined(_M_X64) 3254 0x59,0x59,0x59,0x59,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, 3255#else 3256 0x55,0x55,0x55,0x55,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51, 3257#endif 3258 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x05,0x05,0x05,0x05,0x05,0x05,0x05,0x05, 3259 0x93,0x93,0x53,0x51,0x70,0x71,0x93,0x86,0x54,0x51,0x53,0x51,0x51,0x52,0x51,0x51, 3260 0x92,0x92,0x92,0x92,0x52,0x52,0x51,0x51,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92, 3261 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x45,0x45,0x47,0x52,0x51,0x51,0x51,0x51, 3262 0x10,0x51,0x10,0x10,0x51,0x51,0x63,0x66,0x51,0x51,0x51,0x51,0x51,0x51,0x92,0x92 3263 }; 3264 static const uint8_t map_op2[256] = { 3265 0x93,0x93,0x93,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x51,0x52,0x51,0x93,0x52,0x94, 3266 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3267 0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3268 0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x34,0x51,0x35,0x51,0x51,0x51,0x51,0x51, 3269 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3270 0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3271 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3272 0x94,0x54,0x54,0x54,0x93,0x93,0x93,0x52,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3273 0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46, 3274 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3275 0x52,0x52,0x52,0x93,0x94,0x93,0x51,0x51,0x52,0x52,0x52,0x93,0x94,0x93,0x93,0x93, 3276 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x94,0x93,0x93,0x93,0x93,0x93, 3277 0x93,0x93,0x94,0x93,0x94,0x94,0x94,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52, 3278 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3279 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93, 3280 0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x52 3281 }; 3282 uint32_t result = 0; 3283 uint32_t prefixes = 0; 3284 uint32_t x = map_op1[*p]; 3285 3286 for (;;) { 3287 switch (x >> 4) { 3288 case 0: 3289 return result + x + (prefixes & 4); 3290 case 1: 3291 prefixes |= x; 3292 x = map_op1[*++p]; 3293 result++; 3294 break; 3295 case 2: 3296 x = map_op2[*++p]; 3297 break; 3298 case 3: 3299 p++; 3300 goto mrm; 3301 case 4: 3302 result -= (prefixes & 2); 3303 /* fallthrough */ 3304 case 5: 3305 return result + (x & 15); 3306 case 6: /* Group 3. */ 3307 if (p[1] & 0x38) { 3308 x = 2; 3309 } else if ((prefixes & 2) && (x == 0x66)) { 3310 x = 4; 3311 } 3312 goto mrm; 3313 case 7: /* VEX c4/c5. */ 3314#if !defined(__x86_64__) && !defined(_M_X64) 3315 if (p[1] < 0xc0) { 3316 x = 2; 3317 goto mrm; 3318 } 3319#endif 3320 if (x == 0x70) { 3321 x = *++p & 0x1f; 3322 result++; 3323 if (x >= 2) { 3324 p += 2; 3325 result += 2; 3326 goto mrm; 3327 } 3328 } 3329 p++; 3330 result++; 3331 x = map_op2[*++p]; 3332 break; 3333 case 8: 3334 result -= (prefixes & 2); 3335 /* fallthrough */ 3336 case 9: 3337mrm: 3338 /* ModR/M and possibly SIB. */ 3339 result += (x & 15); 3340 x = *++p; 3341 switch (x >> 6) { 3342 case 0: 3343 if ((x & 7) == 5) { 3344 return result + 4; 3345 } 3346 break; 3347 case 1: 3348 result++; 3349 break; 3350 case 2: 3351 result += 4; 3352 break; 3353 case 3: 3354 return result; 3355 } 3356 if ((x & 7) == 4) { 3357 result++; 3358 if (x < 0x40 && (p[1] & 7) == 5) { 3359 result += 4; 3360 } 3361 } 3362 return result; 3363 } 3364 } 3365} 3366 3367typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t); 3368typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t); 3369 3370static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) 3371{ 3372 int ret = 0; 3373 uint8_t *p, *end; 3374 3375 if (jmp_table_size) { 3376 const void **jmp_slot = (const void **)((char*)code + size); 3377 3378 size -= jmp_table_size * sizeof(void*); 3379 do { 3380 jmp_slot--; 3381 if (*jmp_slot == from_addr) { 3382 *jmp_slot = to_addr; 3383 ret++; 3384 } 3385 } while (--jmp_table_size); 3386 } 3387 3388 p = (uint8_t*)code; 3389 end = p + size - 5; 3390 while (p < end) { 3391 if ((*(unaligned_uint16_t*)p & 0xf0ff) == 0x800f && p + *(unaligned_int32_t*)(p+2) == (uint8_t*)from_addr - 6) { 3392 *(unaligned_int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6)); 3393 ret++; 3394 } else if (*p == 0xe9 && p + *(unaligned_int32_t*)(p+1) == (uint8_t*)from_addr - 5) { 3395 *(unaligned_int32_t*)(p+1) = ((uint8_t*)to_addr - (p + 5)); 3396 ret++; 3397 } 3398 p += _asm_x86_inslen(p); 3399 } 3400#ifdef HAVE_VALGRIND 3401 VALGRIND_DISCARD_TRANSLATIONS(code, size); 3402#endif 3403 return ret; 3404} 3405 3406static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) 3407{ 3408 return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); 3409} 3410 3411static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) 3412{ 3413 const void *link_addr; 3414 size_t prologue_size; 3415 3416 /* Skip prologue. */ 3417 // TODO: don't hardcode this ??? 3418 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3419#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 3420 prologue_size = 0; 3421#elif defined(__x86_64__) || defined(_M_X64) 3422 // sub r4, HYBRID_SPAD 3423 prologue_size = 4; 3424#else 3425 // sub r4, HYBRID_SPAD 3426 prologue_size = 3; 3427#endif 3428 } else if (GCC_GLOBAL_REGS) { 3429 // sub r4, SPAD // stack alignment 3430#if defined(__x86_64__) || defined(_M_X64) 3431 prologue_size = 4; 3432#else 3433 prologue_size = 3; 3434#endif 3435 } else { 3436 // sub r4, NR_SPAD // stack alignment 3437 // mov aword T2, FP // save FP 3438 // mov aword T3, RX // save IP 3439 // mov FP, FCARG1a 3440#if defined(__x86_64__) || defined(_M_X64) 3441 prologue_size = 17; 3442#else 3443 prologue_size = 13; 3444#endif 3445 } 3446 link_addr = (const void*)((const char*)t->code_start + prologue_size); 3447 3448 if (timeout_exit_addr) { 3449 /* Check timeout for links to LOOP */ 3450 | MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0 3451 | je &link_addr 3452 | jmp &timeout_exit_addr 3453 } else { 3454 | jmp &link_addr 3455 } 3456 return 1; 3457} 3458 3459static int zend_jit_trace_return(dasm_State **Dst, zend_bool original_handler, const zend_op *opline) 3460{ 3461#if 0 3462 | jmp ->trace_escape 3463#else 3464 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3465 | ADD_HYBRID_SPAD 3466 if (!original_handler) { 3467 | JMP_IP 3468 } else { 3469 | mov r0, EX->func 3470 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3471 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3472 | jmp aword [IP + r0] 3473 } 3474 } else if (GCC_GLOBAL_REGS) { 3475 | add r4, SPAD // stack alignment 3476 if (!original_handler) { 3477 | JMP_IP 3478 } else { 3479 | mov r0, EX->func 3480 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3481 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3482 | jmp aword [IP + r0] 3483 } 3484 } else { 3485 if (original_handler) { 3486 | mov FCARG1a, FP 3487 | mov r0, EX->func 3488 | mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])] 3489 | mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)] 3490 | call aword [IP + r0] 3491 } 3492 | mov FP, aword T2 // restore FP 3493 | mov RX, aword T3 // restore IP 3494 | add r4, NR_SPAD // stack alignment 3495 if (!original_handler || !opline || 3496 (opline->opcode != ZEND_RETURN 3497 && opline->opcode != ZEND_RETURN_BY_REF 3498 && opline->opcode != ZEND_GENERATOR_RETURN 3499 && opline->opcode != ZEND_GENERATOR_CREATE 3500 && opline->opcode != ZEND_YIELD 3501 && opline->opcode != ZEND_YIELD_FROM)) { 3502 | mov r0, 2 // ZEND_VM_LEAVE 3503 } 3504 | ret 3505 } 3506#endif 3507 return 1; 3508} 3509 3510static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) 3511{ 3512 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 3513 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3514 3515 if (!exit_addr) { 3516 return 0; 3517 } 3518 | IF_NOT_Z_TYPE FP + var, type, &exit_addr 3519 3520 return 1; 3521} 3522 3523static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) 3524{ 3525 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 3526 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3527 3528 if (!exit_addr) { 3529 return 0; 3530 } 3531 3532 | GET_ZVAL_LVAL ZREG_FCARG1a, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var) 3533 if (op_info & MAY_BE_ARRAY_PACKED) { 3534 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 3535 | jz &exit_addr 3536 } else { 3537 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 3538 | jnz &exit_addr 3539 } 3540 3541 return 1; 3542} 3543 3544static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) 3545{ 3546 zend_jit_op_array_trace_extension *jit_extension = 3547 (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); 3548 size_t offset = jit_extension->offset; 3549 const void *handler = 3550 (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; 3551 3552 if (!zend_jit_set_valid_ip(Dst, opline)) { 3553 return 0; 3554 } 3555 if (!GCC_GLOBAL_REGS) { 3556 | mov FCARG1a, FP 3557 } 3558 | EXT_CALL handler, r0 3559 if (may_throw 3560 && opline->opcode != ZEND_RETURN 3561 && opline->opcode != ZEND_RETURN_BY_REF) { 3562 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r1 3563 | jne ->exception_handler 3564 } 3565 3566 while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { 3567 trace++; 3568 } 3569 3570 if (!GCC_GLOBAL_REGS 3571 && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { 3572 if (opline->opcode == ZEND_RETURN || 3573 opline->opcode == ZEND_RETURN_BY_REF || 3574 opline->opcode == ZEND_DO_UCALL || 3575 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3576 opline->opcode == ZEND_DO_FCALL || 3577 opline->opcode == ZEND_GENERATOR_CREATE) { 3578 | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r1 3579 } 3580 } 3581 3582 if (zend_jit_trace_may_exit(op_array, opline)) { 3583 if (opline->opcode == ZEND_RETURN || 3584 opline->opcode == ZEND_RETURN_BY_REF || 3585 opline->opcode == ZEND_GENERATOR_CREATE) { 3586 3587 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3588#if 0 3589 /* this check should be handled by the following OPLINE guard or jmp [IP] */ 3590 | cmp IP, zend_jit_halt_op 3591 | je ->trace_halt 3592#endif 3593 } else if (GCC_GLOBAL_REGS) { 3594 | test IP, IP 3595 | je ->trace_halt 3596 } else { 3597 | test eax, eax 3598 | jl ->trace_halt 3599 } 3600 } else if (opline->opcode == ZEND_EXIT || 3601 opline->opcode == ZEND_GENERATOR_RETURN || 3602 opline->opcode == ZEND_YIELD || 3603 opline->opcode == ZEND_YIELD_FROM) { 3604 | jmp ->trace_halt 3605 } 3606 if (trace->op != ZEND_JIT_TRACE_END || 3607 (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && 3608 trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { 3609 3610 const zend_op *next_opline = trace->opline; 3611 const zend_op *exit_opline = NULL; 3612 uint32_t exit_point; 3613 const void *exit_addr; 3614 uint32_t old_info = 0; 3615 uint32_t old_res_info = 0; 3616 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 3617 3618 if (zend_is_smart_branch(opline)) { 3619 zend_bool exit_if_true = 0; 3620 exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); 3621 } else { 3622 switch (opline->opcode) { 3623 case ZEND_JMPZ: 3624 case ZEND_JMPNZ: 3625 case ZEND_JMPZ_EX: 3626 case ZEND_JMPNZ_EX: 3627 case ZEND_JMP_SET: 3628 case ZEND_COALESCE: 3629 case ZEND_JMP_NULL: 3630 case ZEND_FE_RESET_R: 3631 case ZEND_FE_RESET_RW: 3632 exit_opline = (trace->opline == opline + 1) ? 3633 OP_JMP_ADDR(opline, opline->op2) : 3634 opline + 1; 3635 break; 3636 case ZEND_JMPZNZ: 3637 exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ? 3638 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3639 OP_JMP_ADDR(opline, opline->op2); 3640 break; 3641 case ZEND_FE_FETCH_R: 3642 case ZEND_FE_FETCH_RW: 3643 exit_opline = (trace->opline == opline + 1) ? 3644 ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : 3645 opline + 1; 3646 break; 3647 3648 } 3649 } 3650 3651 switch (opline->opcode) { 3652 case ZEND_FE_FETCH_R: 3653 case ZEND_FE_FETCH_RW: 3654 if (opline->op2_type != IS_UNUSED) { 3655 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); 3656 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); 3657 } 3658 break; 3659 } 3660 3661 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3662 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 3663 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 3664 } 3665 exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); 3666 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3667 3668 if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { 3669 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 3670 } 3671 switch (opline->opcode) { 3672 case ZEND_FE_FETCH_R: 3673 case ZEND_FE_FETCH_RW: 3674 if (opline->op2_type != IS_UNUSED) { 3675 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); 3676 } 3677 break; 3678 } 3679 3680 if (!exit_addr) { 3681 return 0; 3682 } 3683 | CMP_IP next_opline 3684 | jne &exit_addr 3685 } 3686 } 3687 3688 zend_jit_set_last_valid_opline(trace->opline); 3689 3690 return 1; 3691} 3692 3693static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) 3694{ 3695 const void *handler; 3696 3697 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3698 handler = zend_get_opcode_handler_func(opline); 3699 } else { 3700 handler = opline->handler; 3701 } 3702 3703 if (!zend_jit_set_valid_ip(Dst, opline)) { 3704 return 0; 3705 } 3706 if (!GCC_GLOBAL_REGS) { 3707 | mov FCARG1a, FP 3708 } 3709 | EXT_CALL handler, r0 3710 if (may_throw) { 3711 zend_jit_check_exception(Dst); 3712 } 3713 3714 /* Skip the following OP_DATA */ 3715 switch (opline->opcode) { 3716 case ZEND_ASSIGN_DIM: 3717 case ZEND_ASSIGN_OBJ: 3718 case ZEND_ASSIGN_STATIC_PROP: 3719 case ZEND_ASSIGN_DIM_OP: 3720 case ZEND_ASSIGN_OBJ_OP: 3721 case ZEND_ASSIGN_STATIC_PROP_OP: 3722 case ZEND_ASSIGN_STATIC_PROP_REF: 3723 case ZEND_ASSIGN_OBJ_REF: 3724 zend_jit_set_last_valid_opline(opline + 2); 3725 break; 3726 default: 3727 zend_jit_set_last_valid_opline(opline + 1); 3728 break; 3729 } 3730 3731 return 1; 3732} 3733 3734static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) 3735{ 3736 if (!zend_jit_set_valid_ip(Dst, opline)) { 3737 return 0; 3738 } 3739 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 3740 if (opline->opcode == ZEND_DO_UCALL || 3741 opline->opcode == ZEND_DO_FCALL_BY_NAME || 3742 opline->opcode == ZEND_DO_FCALL || 3743 opline->opcode == ZEND_RETURN) { 3744 3745 /* Use inlined HYBRID VM handler */ 3746 const void *handler = opline->handler; 3747 3748 | ADD_HYBRID_SPAD 3749 | EXT_JMP handler, r0 3750 } else { 3751 const void *handler = zend_get_opcode_handler_func(opline); 3752 3753 | EXT_CALL handler, r0 3754 | ADD_HYBRID_SPAD 3755 | JMP_IP 3756 } 3757 } else { 3758 const void *handler = opline->handler; 3759 3760 if (GCC_GLOBAL_REGS) { 3761 | add r4, SPAD // stack alignment 3762 } else { 3763 | mov FCARG1a, FP 3764 | mov FP, aword T2 // restore FP 3765 | mov RX, aword T3 // restore IP 3766 | add r4, NR_SPAD // stack alignment 3767 } 3768 | EXT_JMP handler, r0 3769 } 3770 zend_jit_reset_last_valid_opline(); 3771 return 1; 3772} 3773 3774static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) 3775{ 3776 uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); 3777 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 3778 3779 if (!exit_addr) { 3780 return 0; 3781 } 3782 | CMP_IP opline 3783 | jne &exit_addr 3784 3785 zend_jit_set_last_valid_opline(opline); 3786 3787 return 1; 3788} 3789 3790static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) 3791{ 3792 | jmp =>target_label 3793 return 1; 3794} 3795 3796static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) 3797{ 3798 | CMP_IP next_opline 3799 | jne =>target_label 3800 3801 zend_jit_set_last_valid_opline(next_opline); 3802 3803 return 1; 3804} 3805 3806#ifdef CONTEXT_THREADED_JIT 3807static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3808{ 3809 if (!zend_jit_handler(Dst, opline, 1)) return 0; 3810 if (opline->opcode == ZEND_DO_UCALL) { 3811 | call ->context_threaded_call 3812 } else { 3813 const zend_op *next_opline = opline + 1; 3814 3815 | CMP_IP next_opline 3816 | je =>next_block 3817 | call ->context_threaded_call 3818 } 3819 return 1; 3820} 3821#endif 3822 3823static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) 3824{ 3825#ifdef CONTEXT_THREADED_JIT 3826 return zend_jit_context_threaded_call(Dst, opline, next_block); 3827#else 3828 return zend_jit_tail_handler(Dst, opline); 3829#endif 3830} 3831 3832static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, zend_bool set_type) 3833{ 3834 ZEND_ASSERT(Z_MODE(src) == IS_REG); 3835 ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); 3836 3837 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3838 | SET_ZVAL_LVAL dst, Ra(Z_REG(src)) 3839 if (set_type) { 3840 | SET_ZVAL_TYPE_INFO dst, IS_LONG 3841 } 3842 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3843 | SSE_SET_ZVAL_DVAL dst, Z_REG(src) 3844 if (set_type) { 3845 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 3846 } 3847 } else { 3848 ZEND_UNREACHABLE(); 3849 } 3850 return 1; 3851} 3852 3853static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3854{ 3855 ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); 3856 ZEND_ASSERT(Z_MODE(dst) == IS_REG); 3857 3858 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3859 | GET_ZVAL_LVAL Z_REG(dst), src 3860 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3861 | SSE_GET_ZVAL_DVAL Z_REG(dst), src 3862 } else { 3863 ZEND_UNREACHABLE(); 3864 } 3865 return 1; 3866} 3867 3868static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, zend_bool set_type) 3869{ 3870 zend_jit_addr src = ZEND_ADDR_REG(reg); 3871 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3872 3873 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3874} 3875 3876static int zend_jit_store_var_type(dasm_State **Dst, int var, uint8_t type) 3877{ 3878 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3879 3880 | SET_ZVAL_TYPE_INFO dst, type 3881 return 1; 3882} 3883 3884static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) 3885{ 3886 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3887 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3888 return zend_jit_spill_store(Dst, src, dst, info, 1); 3889 } 3890 return 1; 3891} 3892 3893static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) 3894{ 3895 if (Z_MODE(src) == IS_REG && Z_STORE(src)) { 3896 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3897 zend_bool set_type = 1; 3898 3899 if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == 3900 (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { 3901 if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { 3902 set_type = 0; 3903 } 3904 } 3905 return zend_jit_spill_store(Dst, src, dst, info, set_type); 3906 } 3907 return 1; 3908} 3909 3910static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) 3911{ 3912 zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 3913 zend_jit_addr dst = ZEND_ADDR_REG(reg); 3914 3915 return zend_jit_load_reg(Dst, src, dst, info); 3916} 3917 3918static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, zend_uchar op_type, zend_jit_addr addr, znode_op op) 3919{ 3920 if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) { 3921 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); 3922 | SET_ZVAL_TYPE_INFO dst, IS_UNDEF 3923 } 3924 return 1; 3925} 3926 3927static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) 3928{ 3929 if (!zend_jit_same_addr(src, dst)) { 3930 if (Z_MODE(src) == IS_REG) { 3931 if (Z_MODE(dst) == IS_REG) { 3932 if ((info & MAY_BE_ANY) == MAY_BE_LONG) { 3933 | mov Ra(Z_REG(dst)), Ra(Z_REG(src)) 3934 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 3935 | SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0) 3936 } else { 3937 ZEND_UNREACHABLE(); 3938 } 3939 if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) { 3940 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3941 3942 if (!zend_jit_spill_store(Dst, dst, var_addr, info, 3943 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3944 JIT_G(current_frame) == NULL || 3945 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3946 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3947 )) { 3948 return 0; 3949 } 3950 } 3951 } else if (Z_MODE(dst) == IS_MEM_ZVAL) { 3952 if (!Z_LOAD(src) && !Z_STORE(src)) { 3953 if (!zend_jit_spill_store(Dst, src, dst, info, 3954 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3955 JIT_G(current_frame) == NULL || 3956 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3957 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3958 )) { 3959 return 0; 3960 } 3961 } 3962 } else { 3963 ZEND_UNREACHABLE(); 3964 } 3965 } else if (Z_MODE(src) == IS_MEM_ZVAL) { 3966 if (Z_MODE(dst) == IS_REG) { 3967 if (!zend_jit_load_reg(Dst, src, dst, info)) { 3968 return 0; 3969 } 3970 } else { 3971 ZEND_UNREACHABLE(); 3972 } 3973 } else { 3974 ZEND_UNREACHABLE(); 3975 } 3976 } else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) { 3977 dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 3978 if (!zend_jit_spill_store(Dst, src, dst, info, 3979 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 3980 JIT_G(current_frame) == NULL || 3981 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || 3982 (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) 3983 )) { 3984 return 0; 3985 } 3986 } 3987 return 1; 3988} 3989 3990static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) 3991{ 3992 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 3993 3994 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1 3995 3996 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { 3997 if (!zend_jit_save_call_chain(Dst, -1)) { 3998 return 0; 3999 } 4000 } 4001 4002 ZEND_ASSERT(opline); 4003 4004 if ((opline-1)->opcode != ZEND_FETCH_CONSTANT 4005 && (opline-1)->opcode != ZEND_FETCH_LIST_R 4006 && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR)) 4007 && !(flags & ZEND_JIT_EXIT_FREE_OP1)) { 4008 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var); 4009 4010 | IF_NOT_ZVAL_REFCOUNTED val_addr, >2 4011 | GET_ZVAL_PTR r0, val_addr 4012 | GC_ADDREF r0 4013 |2: 4014 } 4015 4016 | LOAD_IP_ADDR (opline - 1) 4017 | jmp ->trace_escape 4018 |1: 4019 4020 return 1; 4021} 4022 4023static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) 4024{ 4025 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); 4026 4027 if (reg == ZREG_LONG_MIN_MINUS_1) { 4028 |.if X64 4029 | SET_ZVAL_LVAL dst, 0x00000000 4030 | SET_ZVAL_W2 dst, 0xc3e00000 4031 |.else 4032 | SET_ZVAL_LVAL dst, 0x00200000 4033 | SET_ZVAL_W2 dst, 0xc1e00000 4034 |.endif 4035 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 4036 } else if (reg == ZREG_LONG_MIN) { 4037 |.if X64 4038 | SET_ZVAL_LVAL dst, 0x00000000 4039 | SET_ZVAL_W2 dst, 0x80000000 4040 |.else 4041 | SET_ZVAL_LVAL dst, ZEND_LONG_MIN 4042 |.endif 4043 | SET_ZVAL_TYPE_INFO dst, IS_LONG 4044 } else if (reg == ZREG_LONG_MAX) { 4045 |.if X64 4046 | SET_ZVAL_LVAL dst, 0xffffffff 4047 | SET_ZVAL_W2 dst, 0x7fffffff 4048 |.else 4049 | SET_ZVAL_LVAL dst, ZEND_LONG_MAX 4050 |.endif 4051 | SET_ZVAL_TYPE_INFO dst, IS_LONG 4052 } else if (reg == ZREG_LONG_MAX_PLUS_1) { 4053 |.if X64 4054 | SET_ZVAL_LVAL dst, 0 4055 | SET_ZVAL_W2 dst, 0x43e00000 4056 |.else 4057 | SET_ZVAL_LVAL dst, 0 4058 | SET_ZVAL_W2 dst, 0x41e00000 4059 |.endif 4060 | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE 4061 } else if (reg == ZREG_NULL) { 4062 | SET_ZVAL_TYPE_INFO dst, IS_NULL 4063 } else if (reg == ZREG_ZVAL_TRY_ADDREF) { 4064 | IF_NOT_ZVAL_REFCOUNTED dst, >1 4065 | GET_ZVAL_PTR r1, dst 4066 | GC_ADDREF r1 4067 |1: 4068 } else if (reg == ZREG_ZVAL_COPY_R0) { 4069 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 4070 4071 | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_R1, ZREG_R2 4072 | TRY_ADDREF -1, ch, r2 4073 } else { 4074 ZEND_UNREACHABLE(); 4075 } 4076 return 1; 4077} 4078 4079static int zend_jit_free_trampoline(dasm_State **Dst) 4080{ 4081 | /// if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) 4082 | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_CALL_VIA_TRAMPOLINE 4083 | jz >1 4084 | mov FCARG1a, r0 4085 | EXT_CALL zend_jit_free_trampoline_helper, r0 4086 |1: 4087 return 1; 4088} 4089 4090static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 4091{ 4092 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { 4093 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 4094 } 4095 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4096 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4097 } 4098 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { 4099 return 0; 4100 } 4101 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4102 | LONG_OP_WITH_32BIT_CONST add, op1_def_addr, Z_L(1) 4103 } else { 4104 | LONG_OP_WITH_32BIT_CONST sub, op1_def_addr, Z_L(1) 4105 } 4106 4107 if (may_overflow && 4108 (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || 4109 ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { 4110 int32_t exit_point; 4111 const void *exit_addr; 4112 zend_jit_trace_stack *stack; 4113 uint32_t old_op1_info, old_res_info = 0; 4114 4115 stack = JIT_G(current_frame)->stack; 4116 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 4117 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); 4118 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4119 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); 4120 } else { 4121 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); 4122 } 4123 if (opline->result_type != IS_UNUSED) { 4124 old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 4125 if (opline->opcode == ZEND_PRE_INC) { 4126 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 4127 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); 4128 } else if (opline->opcode == ZEND_PRE_DEC) { 4129 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); 4130 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); 4131 } else if (opline->opcode == ZEND_POST_INC) { 4132 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 4133 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); 4134 } else if (opline->opcode == ZEND_POST_DEC) { 4135 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); 4136 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); 4137 } 4138 } 4139 4140 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 4141 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4142 | jo &exit_addr 4143 4144 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4145 opline->result_type != IS_UNUSED) { 4146 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4147 } 4148 4149 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 4150 if (opline->result_type != IS_UNUSED) { 4151 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); 4152 } 4153 } else if (may_overflow) { 4154 | jo >1 4155 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4156 opline->result_type != IS_UNUSED) { 4157 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4158 } 4159 |.cold_code 4160 |1: 4161 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4162 |.if X64 4163 | mov64 rax, 0x43e0000000000000 4164 | SET_ZVAL_LVAL op1_def_addr, rax 4165 |.else 4166 | SET_ZVAL_LVAL op1_def_addr, 0 4167 | SET_ZVAL_W2 op1_def_addr, 0x41e00000 4168 |.endif 4169 } else { 4170 |.if X64 4171 | mov64 rax, 0xc3e0000000000000 4172 | SET_ZVAL_LVAL op1_def_addr, rax 4173 |.else 4174 | SET_ZVAL_LVAL op1_def_addr, 0x00200000 4175 | SET_ZVAL_W2 op1_def_addr, 0xc1e00000 4176 |.endif 4177 } 4178 if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { 4179 | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE 4180 } 4181 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4182 opline->result_type != IS_UNUSED) { 4183 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1 4184 } 4185 | jmp >3 4186 |.code 4187 } else { 4188 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4189 opline->result_type != IS_UNUSED) { 4190 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1 4191 } 4192 } 4193 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 4194 |.cold_code 4195 |2: 4196 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4197 | SET_EX_OPLINE opline, r0 4198 if (op1_info & MAY_BE_UNDEF) { 4199 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2 4200 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 4201 | mov FCARG1d, opline->op1.var 4202 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 4203 | EXT_CALL zend_jit_undefined_op_helper, r0 4204 op1_info |= MAY_BE_NULL; 4205 } 4206 |2: 4207 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 4208 4209 | // ZVAL_DEREF(var_ptr); 4210 if (op1_info & MAY_BE_REF) { 4211 | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >2 4212 | GET_Z_PTR FCARG1a, FCARG1a 4213 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 4214 | jz >1 4215 if (RETURN_VALUE_USED(opline)) { 4216 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4217 } else { 4218 | xor FCARG2a, FCARG2a 4219 } 4220 if (opline->opcode == ZEND_PRE_INC) { 4221 | EXT_CALL zend_jit_pre_inc_typed_ref, r0 4222 } else if (opline->opcode == ZEND_PRE_DEC) { 4223 | EXT_CALL zend_jit_pre_dec_typed_ref, r0 4224 } else if (opline->opcode == ZEND_POST_INC) { 4225 | EXT_CALL zend_jit_post_inc_typed_ref, r0 4226 } else if (opline->opcode == ZEND_POST_DEC) { 4227 | EXT_CALL zend_jit_post_dec_typed_ref, r0 4228 } else { 4229 ZEND_UNREACHABLE(); 4230 } 4231 zend_jit_check_exception(Dst); 4232 | jmp >3 4233 |1: 4234 | lea FCARG1a, [FCARG1a + offsetof(zend_reference, val)] 4235 |2: 4236 } 4237 4238 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4239 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 4240 4241 | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2 4242 | TRY_ADDREF op1_info, ah, r2 4243 } 4244 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4245 if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { 4246 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4247 | EXT_CALL zend_jit_pre_inc, r0 4248 } else { 4249 | EXT_CALL increment_function, r0 4250 } 4251 } else { 4252 if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { 4253 | LOAD_ZVAL_ADDR FCARG2a, res_addr 4254 | EXT_CALL zend_jit_pre_dec, r0 4255 } else { 4256 | EXT_CALL decrement_function, r0 4257 } 4258 } 4259 if (may_throw) { 4260 zend_jit_check_exception(Dst); 4261 } 4262 } else { 4263 zend_reg tmp_reg; 4264 4265 if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { 4266 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2 4267 } 4268 if (Z_MODE(op1_def_addr) == IS_REG) { 4269 tmp_reg = Z_REG(op1_def_addr); 4270 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4271 tmp_reg = Z_REG(op1_addr); 4272 } else { 4273 tmp_reg = ZREG_XMM0; 4274 } 4275 | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr 4276 if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { 4277 if (CAN_USE_AVX()) { 4278 | vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] 4279 } else { 4280 | addsd xmm(tmp_reg-ZREG_XMM0), qword [->one] 4281 } 4282 } else { 4283 if (CAN_USE_AVX()) { 4284 | vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one] 4285 } else { 4286 | subsd xmm(tmp_reg-ZREG_XMM0), qword [->one] 4287 } 4288 } 4289 | SSE_SET_ZVAL_DVAL op1_def_addr, tmp_reg 4290 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && 4291 opline->result_type != IS_UNUSED) { 4292 | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1 4293 | TRY_ADDREF op1_def_info, ah, r1 4294 } 4295 } 4296 | jmp >3 4297 |.code 4298 } 4299 |3: 4300 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { 4301 return 0; 4302 } 4303 if (opline->result_type != IS_UNUSED) { 4304 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4305 return 0; 4306 } 4307 } 4308 return 1; 4309} 4310 4311static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) 4312{ 4313 if ((opline+1)->opcode == ZEND_OP_DATA 4314 && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) 4315 && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { 4316 return 1; 4317 } 4318 return 4319 ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4320 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || 4321 ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4322 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || 4323 ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && 4324 JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); 4325} 4326 4327static int zend_jit_math_long_long(dasm_State **Dst, 4328 const zend_op *opline, 4329 zend_uchar opcode, 4330 zend_jit_addr op1_addr, 4331 zend_jit_addr op2_addr, 4332 zend_jit_addr res_addr, 4333 uint32_t res_info, 4334 uint32_t res_use_info, 4335 int may_overflow) 4336{ 4337 zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4338 zend_reg result_reg; 4339 zend_reg tmp_reg = ZREG_R0; 4340 4341 if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { 4342 if (may_overflow && (res_info & MAY_BE_GUARD) 4343 && JIT_G(current_frame) 4344 && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { 4345 result_reg = ZREG_R0; 4346 } else { 4347 result_reg = Z_REG(res_addr); 4348 } 4349 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { 4350 result_reg = Z_REG(op1_addr); 4351 } else if (Z_REG(res_addr) != ZREG_R0) { 4352 result_reg = ZREG_R0; 4353 } else { 4354 /* ASSIGN_DIM_OP */ 4355 result_reg = ZREG_FCARG1a; 4356 tmp_reg = ZREG_FCARG1a; 4357 } 4358 4359 if (opcode == ZEND_MUL && 4360 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4361 Z_LVAL_P(Z_ZV(op2_addr)) == 2) { 4362 if (Z_MODE(op1_addr) == IS_REG && !may_overflow) { 4363 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] 4364 } else { 4365 | GET_ZVAL_LVAL result_reg, op1_addr 4366 | add Ra(result_reg), Ra(result_reg) 4367 } 4368 } else if (opcode == ZEND_MUL && 4369 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4370 !may_overflow && 4371 Z_LVAL_P(Z_ZV(op2_addr)) > 0 && 4372 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { 4373 | GET_ZVAL_LVAL result_reg, op1_addr 4374 | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4375 } else if (opcode == ZEND_MUL && 4376 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4377 Z_LVAL_P(Z_ZV(op1_addr)) == 2) { 4378 if (Z_MODE(op2_addr) == IS_REG && !may_overflow) { 4379 | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Ra(Z_REG(op2_addr))] 4380 } else { 4381 | GET_ZVAL_LVAL result_reg, op2_addr 4382 | add Ra(result_reg), Ra(result_reg) 4383 } 4384 } else if (opcode == ZEND_MUL && 4385 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4386 !may_overflow && 4387 Z_LVAL_P(Z_ZV(op1_addr)) > 0 && 4388 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { 4389 | GET_ZVAL_LVAL result_reg, op2_addr 4390 | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) 4391 } else if (opcode == ZEND_DIV && 4392 (Z_MODE(op2_addr) == IS_CONST_ZVAL && 4393 zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { 4394 | GET_ZVAL_LVAL result_reg, op1_addr 4395 | shr Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) 4396 } else if (opcode == ZEND_ADD && 4397 !may_overflow && 4398 Z_MODE(op1_addr) == IS_REG && 4399 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4400 IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr)))) { 4401 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Z_LVAL_P(Z_ZV(op2_addr))] 4402 } else if (opcode == ZEND_ADD && 4403 !may_overflow && 4404 Z_MODE(op2_addr) == IS_REG && 4405 Z_MODE(op1_addr) == IS_CONST_ZVAL && 4406 IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr)))) { 4407 | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Z_LVAL_P(Z_ZV(op1_addr))] 4408 } else if (opcode == ZEND_SUB && 4409 !may_overflow && 4410 Z_MODE(op1_addr) == IS_REG && 4411 Z_MODE(op2_addr) == IS_CONST_ZVAL && 4412 IS_SIGNED_32BIT(-Z_LVAL_P(Z_ZV(op2_addr)))) { 4413 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))-Z_LVAL_P(Z_ZV(op2_addr))] 4414 } else { 4415 | GET_ZVAL_LVAL result_reg, op1_addr 4416 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4417 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4418 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4419 /* +/- 0 */ 4420 may_overflow = 0; 4421 } else if (same_ops && opcode != ZEND_DIV) { 4422 | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) 4423 } else { 4424 zend_reg tmp_reg; 4425 4426 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4427 tmp_reg = ZREG_R1; 4428 } else if (result_reg != ZREG_R0) { 4429 tmp_reg = ZREG_R0; 4430 } else { 4431 tmp_reg = ZREG_R1; 4432 } 4433 | LONG_MATH opcode, result_reg, op2_addr, tmp_reg 4434 (void)tmp_reg; 4435 } 4436 } 4437 if (may_overflow) { 4438 if (res_info & MAY_BE_GUARD) { 4439 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 4440 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 4441 if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { 4442 | jo &exit_addr 4443 if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { 4444 | mov Ra(Z_REG(res_addr)), Ra(result_reg) 4445 } 4446 } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 4447 | jno &exit_addr 4448 } else { 4449 ZEND_UNREACHABLE(); 4450 } 4451 } else { 4452 if (res_info & MAY_BE_LONG) { 4453 | jo >1 4454 } else { 4455 | jno >1 4456 } 4457 } 4458 } 4459 4460 if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { 4461 | SET_ZVAL_LVAL res_addr, Ra(result_reg) 4462 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4463 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 4464 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 4465 } 4466 } 4467 } 4468 4469 if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { 4470 zend_reg tmp_reg1 = ZREG_XMM0; 4471 zend_reg tmp_reg2 = ZREG_XMM1; 4472 4473 if (res_info & MAY_BE_LONG) { 4474 |.cold_code 4475 |1: 4476 } 4477 4478 do { 4479 if ((sizeof(void*) == 8 || Z_MODE(res_addr) != IS_REG) && 4480 ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || 4481 (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1))) { 4482 if (opcode == ZEND_ADD) { 4483 |.if X64 4484 | mov64 Ra(tmp_reg), 0x43e0000000000000 4485 if (Z_MODE(res_addr) == IS_REG) { 4486 | movd xmm(Z_REG(res_addr)-ZREG_XMM0), Ra(tmp_reg) 4487 } else { 4488 | SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 4489 } 4490 |.else 4491 | SET_ZVAL_LVAL res_addr, 0 4492 | SET_ZVAL_W2 res_addr, 0x41e00000 4493 |.endif 4494 break; 4495 } else if (opcode == ZEND_SUB) { 4496 |.if X64 4497 | mov64 Ra(tmp_reg), 0xc3e0000000000000 4498 if (Z_MODE(res_addr) == IS_REG) { 4499 | movd xmm(Z_REG(res_addr)-ZREG_XMM0), Ra(tmp_reg) 4500 } else { 4501 | SET_ZVAL_LVAL res_addr, Ra(tmp_reg) 4502 } 4503 |.else 4504 | SET_ZVAL_LVAL res_addr, 0x00200000 4505 | SET_ZVAL_W2 res_addr, 0xc1e00000 4506 |.endif 4507 break; 4508 } 4509 } 4510 4511 | SSE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg 4512 | SSE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg 4513 if (CAN_USE_AVX()) { 4514 | AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 4515 } else { 4516 | SSE_MATH_REG opcode, tmp_reg1, tmp_reg2 4517 } 4518 | SSE_SET_ZVAL_DVAL res_addr, tmp_reg1 4519 } while (0); 4520 4521 if (Z_MODE(res_addr) == IS_MEM_ZVAL 4522 && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4523 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4524 } 4525 if (res_info & MAY_BE_LONG) { 4526 | jmp >2 4527 |.code 4528 } 4529 |2: 4530 } 4531 4532 return 1; 4533} 4534 4535static int zend_jit_math_long_double(dasm_State **Dst, 4536 zend_uchar opcode, 4537 zend_jit_addr op1_addr, 4538 zend_jit_addr op2_addr, 4539 zend_jit_addr res_addr, 4540 uint32_t res_use_info) 4541{ 4542 zend_reg result_reg = 4543 (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0; 4544 zend_reg tmp_reg; 4545 4546 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4547 /* ASSIGN_DIM_OP */ 4548 tmp_reg = ZREG_R1; 4549 } else { 4550 tmp_reg = ZREG_R0; 4551 } 4552 4553 | SSE_GET_ZVAL_LVAL result_reg, op1_addr, tmp_reg 4554 4555 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4556 /* ASSIGN_DIM_OP */ 4557 if (CAN_USE_AVX()) { 4558 | AVX_MATH opcode, result_reg, result_reg, op2_addr, r1 4559 } else { 4560 | SSE_MATH opcode, result_reg, op2_addr, r1 4561 } 4562 } else { 4563 if (CAN_USE_AVX()) { 4564 | AVX_MATH opcode, result_reg, result_reg, op2_addr, r0 4565 } else { 4566 | SSE_MATH opcode, result_reg, op2_addr, r0 4567 } 4568 } 4569 | SSE_SET_ZVAL_DVAL res_addr, result_reg 4570 4571 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4572 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4573 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4574 } 4575 } 4576 4577 return 1; 4578} 4579 4580static int zend_jit_math_double_long(dasm_State **Dst, 4581 zend_uchar opcode, 4582 zend_jit_addr op1_addr, 4583 zend_jit_addr op2_addr, 4584 zend_jit_addr res_addr, 4585 uint32_t res_use_info) 4586{ 4587 zend_reg result_reg, tmp_reg_gp; 4588 4589 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4590 /* ASSIGN_DIM_OP */ 4591 tmp_reg_gp = ZREG_R1; 4592 } else { 4593 tmp_reg_gp = ZREG_R0; 4594 } 4595 4596 if (zend_is_commutative(opcode) 4597 && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { 4598 if (Z_MODE(res_addr) == IS_REG) { 4599 result_reg = Z_REG(res_addr); 4600 } else { 4601 result_reg = ZREG_XMM0; 4602 } 4603 | SSE_GET_ZVAL_LVAL result_reg, op2_addr, tmp_reg_gp 4604 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4605 /* ASSIGN_DIM_OP */ 4606 if (CAN_USE_AVX()) { 4607 | AVX_MATH opcode, result_reg, result_reg, op1_addr, r1 4608 } else { 4609 | SSE_MATH opcode, result_reg, op1_addr, r1 4610 } 4611 } else { 4612 if (CAN_USE_AVX()) { 4613 | AVX_MATH opcode, result_reg, result_reg, op1_addr, r0 4614 } else { 4615 | SSE_MATH opcode, result_reg, op1_addr, r0 4616 } 4617 } 4618 } else { 4619 zend_reg tmp_reg; 4620 4621 if (Z_MODE(res_addr) == IS_REG) { 4622 result_reg = Z_REG(res_addr); 4623 tmp_reg = (result_reg == ZREG_XMM0) ? ZREG_XMM1 : ZREG_XMM0; 4624 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4625 result_reg = Z_REG(op1_addr); 4626 tmp_reg = ZREG_XMM0; 4627 } else { 4628 result_reg = ZREG_XMM0; 4629 tmp_reg = ZREG_XMM1; 4630 } 4631 if (CAN_USE_AVX()) { 4632 zend_reg op1_reg; 4633 4634 if (Z_MODE(op1_addr) == IS_REG) { 4635 op1_reg = Z_REG(op1_addr); 4636 } else { 4637 | SSE_GET_ZVAL_DVAL result_reg, op1_addr 4638 op1_reg = result_reg; 4639 } 4640 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4641 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4642 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4643 /* +/- 0 */ 4644 } else { 4645 | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp 4646 | AVX_MATH_REG opcode, result_reg, op1_reg, tmp_reg 4647 } 4648 } else { 4649 | SSE_GET_ZVAL_DVAL result_reg, op1_addr 4650 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) 4651 && Z_MODE(op2_addr) == IS_CONST_ZVAL 4652 && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 4653 /* +/- 0 */ 4654 } else { 4655 | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp 4656 | SSE_MATH_REG opcode, result_reg, tmp_reg 4657 } 4658 } 4659 } 4660 | SSE_SET_ZVAL_DVAL res_addr, result_reg 4661 4662 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4663 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4664 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4665 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4666 } 4667 } 4668 } 4669 4670 return 1; 4671} 4672 4673static int zend_jit_math_double_double(dasm_State **Dst, 4674 zend_uchar opcode, 4675 zend_jit_addr op1_addr, 4676 zend_jit_addr op2_addr, 4677 zend_jit_addr res_addr, 4678 uint32_t res_use_info) 4679{ 4680 zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4681 zend_reg result_reg; 4682 4683 if (Z_MODE(res_addr) == IS_REG) { 4684 result_reg = Z_REG(res_addr); 4685 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 4686 result_reg = Z_REG(op1_addr); 4687 } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { 4688 result_reg = Z_REG(op2_addr); 4689 } else { 4690 result_reg = ZREG_XMM0; 4691 } 4692 4693 if (CAN_USE_AVX()) { 4694 zend_reg op1_reg; 4695 zend_jit_addr val_addr; 4696 4697 if (Z_MODE(op1_addr) == IS_REG) { 4698 op1_reg = Z_REG(op1_addr); 4699 val_addr = op2_addr; 4700 } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4701 op1_reg = Z_REG(op2_addr); 4702 val_addr = op1_addr; 4703 } else { 4704 | SSE_GET_ZVAL_DVAL result_reg, op1_addr 4705 op1_reg = result_reg; 4706 val_addr = op2_addr; 4707 } 4708 if ((opcode == ZEND_MUL) && 4709 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4710 | AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg 4711 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4712 /* ASSIGN_DIM_OP */ 4713 | AVX_MATH opcode, result_reg, op1_reg, val_addr, r1 4714 } else { 4715 | AVX_MATH opcode, result_reg, op1_reg, val_addr, r0 4716 } 4717 } else { 4718 zend_jit_addr val_addr; 4719 4720 if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { 4721 | SSE_GET_ZVAL_DVAL result_reg, op2_addr 4722 val_addr = op1_addr; 4723 } else { 4724 | SSE_GET_ZVAL_DVAL result_reg, op1_addr 4725 val_addr = op2_addr; 4726 } 4727 if (same_ops) { 4728 | SSE_MATH_REG opcode, result_reg, result_reg 4729 } else if ((opcode == ZEND_MUL) && 4730 Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { 4731 | SSE_MATH_REG ZEND_ADD, result_reg, result_reg 4732 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 4733 /* ASSIGN_DIM_OP */ 4734 | SSE_MATH opcode, result_reg, val_addr, r1 4735 } else { 4736 | SSE_MATH opcode, result_reg, val_addr, r0 4737 } 4738 } 4739 | SSE_SET_ZVAL_DVAL res_addr, result_reg 4740 4741 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 4742 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 4743 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { 4744 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 4745 } 4746 } 4747 } 4748 4749 return 1; 4750} 4751 4752static int zend_jit_math_helper(dasm_State **Dst, 4753 const zend_op *opline, 4754 zend_uchar opcode, 4755 zend_uchar op1_type, 4756 znode_op op1, 4757 zend_jit_addr op1_addr, 4758 uint32_t op1_info, 4759 zend_uchar op2_type, 4760 znode_op op2, 4761 zend_jit_addr op2_addr, 4762 uint32_t op2_info, 4763 uint32_t res_var, 4764 zend_jit_addr res_addr, 4765 uint32_t res_info, 4766 uint32_t res_use_info, 4767 int may_overflow, 4768 int may_throw) 4769/* Labels: 1,2,3,4,5,6 */ 4770{ 4771 zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 4772 4773 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4774 if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { 4775 if (op1_info & MAY_BE_DOUBLE) { 4776 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 4777 } else { 4778 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 4779 } 4780 } 4781 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { 4782 if (op2_info & MAY_BE_DOUBLE) { 4783 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1 4784 |.cold_code 4785 |1: 4786 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4787 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4788 } 4789 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4790 return 0; 4791 } 4792 | jmp >5 4793 |.code 4794 } else { 4795 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4796 } 4797 } 4798 if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { 4799 return 0; 4800 } 4801 if (op1_info & MAY_BE_DOUBLE) { 4802 |.cold_code 4803 |3: 4804 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4805 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4806 } 4807 if (op2_info & MAY_BE_DOUBLE) { 4808 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4809 if (!same_ops) { 4810 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1 4811 } else { 4812 | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6 4813 } 4814 } 4815 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4816 return 0; 4817 } 4818 | jmp >5 4819 } 4820 if (!same_ops) { 4821 |1: 4822 if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 4823 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4824 } 4825 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4826 return 0; 4827 } 4828 | jmp >5 4829 } 4830 |.code 4831 } 4832 } else if ((op1_info & MAY_BE_DOUBLE) && 4833 !(op1_info & MAY_BE_LONG) && 4834 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4835 (res_info & MAY_BE_DOUBLE)) { 4836 if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4837 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4838 } 4839 if (op2_info & MAY_BE_DOUBLE) { 4840 if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4841 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4842 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1 4843 } else { 4844 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4845 } 4846 } 4847 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4848 return 0; 4849 } 4850 } 4851 if (!same_ops && (op2_info & MAY_BE_LONG)) { 4852 if (op2_info & MAY_BE_DOUBLE) { 4853 |.cold_code 4854 } 4855 |1: 4856 if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4857 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 4858 } 4859 if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4860 return 0; 4861 } 4862 if (op2_info & MAY_BE_DOUBLE) { 4863 | jmp >5 4864 |.code 4865 } 4866 } 4867 } else if ((op2_info & MAY_BE_DOUBLE) && 4868 !(op2_info & MAY_BE_LONG) && 4869 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4870 (res_info & MAY_BE_DOUBLE)) { 4871 if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { 4872 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6 4873 } 4874 if (op1_info & MAY_BE_DOUBLE) { 4875 if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { 4876 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4877 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1 4878 } else { 4879 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6 4880 } 4881 } 4882 if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4883 return 0; 4884 } 4885 } 4886 if (!same_ops && (op1_info & MAY_BE_LONG)) { 4887 if (op1_info & MAY_BE_DOUBLE) { 4888 |.cold_code 4889 } 4890 |1: 4891 if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 4892 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 4893 } 4894 if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { 4895 return 0; 4896 } 4897 if (op1_info & MAY_BE_DOUBLE) { 4898 | jmp >5 4899 |.code 4900 } 4901 } 4902 } 4903 4904 |5: 4905 4906 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 4907 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 4908 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4909 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4910 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4911 |.cold_code 4912 } 4913 |6: 4914 if (Z_MODE(res_addr) == IS_REG) { 4915 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4916 | LOAD_ZVAL_ADDR FCARG1a, real_addr 4917 } else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { 4918 | LOAD_ZVAL_ADDR FCARG1a, res_addr 4919 } 4920 if (Z_MODE(op1_addr) == IS_REG) { 4921 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 4922 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 4923 return 0; 4924 } 4925 op1_addr = real_addr; 4926 } 4927 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 4928 if (Z_MODE(op2_addr) == IS_REG) { 4929 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 4930 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 4931 return 0; 4932 } 4933 op2_addr = real_addr; 4934 } 4935 |.if X64 4936 | LOAD_ZVAL_ADDR CARG3, op2_addr 4937 |.else 4938 | sub r4, 12 4939 | PUSH_ZVAL_ADDR op2_addr, r0 4940 |.endif 4941 | SET_EX_OPLINE opline, r0 4942 if (opcode == ZEND_ADD) { 4943 | EXT_CALL add_function, r0 4944 } else if (opcode == ZEND_SUB) { 4945 | EXT_CALL sub_function, r0 4946 } else if (opcode == ZEND_MUL) { 4947 | EXT_CALL mul_function, r0 4948 } else if (opcode == ZEND_DIV) { 4949 | EXT_CALL div_function, r0 4950 } else { 4951 ZEND_UNREACHABLE(); 4952 } 4953 |.if not(X64) 4954 | add r4, 12 4955 |.endif 4956 | FREE_OP op1_type, op1, op1_info, 0, opline 4957 | FREE_OP op2_type, op2, op2_info, 0, opline 4958 if (may_throw) { 4959 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 4960 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 4961 | jne ->exception_handler_free_op2 4962 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 4963 zend_jit_check_exception_undef_result(Dst, opline); 4964 } else { 4965 zend_jit_check_exception(Dst); 4966 } 4967 } 4968 if (Z_MODE(res_addr) == IS_REG) { 4969 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 4970 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 4971 return 0; 4972 } 4973 } 4974 if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4975 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4976 (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 4977 | jmp <5 4978 |.code 4979 } 4980 } 4981 4982 return 1; 4983} 4984 4985static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) 4986{ 4987 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 4988 ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 4989 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); 4990 4991 if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { 4992 return 0; 4993 } 4994 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 4995 return 0; 4996 } 4997 return 1; 4998} 4999 5000static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr) 5001{ 5002 zend_jit_addr op1_addr = OP1_ADDR(); 5003 zend_jit_addr op2_addr = OP2_ADDR(); 5004 5005 | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr 5006 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5007 | EXT_CALL zend_jit_add_arrays_helper, r0 5008 | SET_ZVAL_PTR res_addr, r0 5009 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX 5010 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 5011 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 5012 return 1; 5013} 5014 5015static int zend_jit_long_math_helper(dasm_State **Dst, 5016 const zend_op *opline, 5017 zend_uchar opcode, 5018 zend_uchar op1_type, 5019 znode_op op1, 5020 zend_jit_addr op1_addr, 5021 uint32_t op1_info, 5022 zend_ssa_range *op1_range, 5023 zend_uchar op2_type, 5024 znode_op op2, 5025 zend_jit_addr op2_addr, 5026 uint32_t op2_info, 5027 zend_ssa_range *op2_range, 5028 uint32_t res_var, 5029 zend_jit_addr res_addr, 5030 uint32_t res_info, 5031 uint32_t res_use_info, 5032 int may_throw) 5033/* Labels: 6 */ 5034{ 5035 zend_bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); 5036 zend_reg result_reg; 5037 5038 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 5039 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 5040 } 5041 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 5042 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6 5043 } 5044 5045 if (opcode == ZEND_MOD) { 5046 result_reg = ZREG_RAX; 5047 } else if (Z_MODE(res_addr) == IS_REG) { 5048 if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) 5049 && opline->op2_type != IS_CONST) { 5050 result_reg = ZREG_R0; 5051 } else { 5052 result_reg = Z_REG(res_addr); 5053 } 5054 } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { 5055 result_reg = Z_REG(op1_addr); 5056 } else if (Z_REG(res_addr) != ZREG_R0) { 5057 result_reg = ZREG_R0; 5058 } else { 5059 /* ASSIGN_DIM_OP */ 5060 if (sizeof(void*) == 4 5061 && (opcode == ZEND_SL || opcode == ZEND_SR) 5062 && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 5063 result_reg = ZREG_R2; 5064 } else { 5065 result_reg = ZREG_FCARG1a; 5066 } 5067 } 5068 5069 if (opcode == ZEND_SL) { 5070 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5071 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5072 5073 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 5074 if (EXPECTED(op2_lval > 0)) { 5075 | xor Ra(result_reg), Ra(result_reg) 5076 } else { 5077 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5078 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5079 | SET_EX_OPLINE opline, r0 5080 | jmp ->negative_shift 5081 } 5082 } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { 5083 | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] 5084 } else { 5085 | GET_ZVAL_LVAL result_reg, op1_addr 5086 | shl Ra(result_reg), op2_lval 5087 } 5088 } else { 5089 if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { 5090 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5091 } 5092 if (!op2_range || 5093 op2_range->min < 0 || 5094 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 5095 | cmp r1, (SIZEOF_ZEND_LONG*8) 5096 | jae >1 5097 |.cold_code 5098 |1: 5099 | cmp r1, 0 5100 | mov Ra(result_reg), 0 5101 | jg >1 5102 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5103 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5104 | SET_EX_OPLINE opline, r0 5105 | jmp ->negative_shift 5106 |.code 5107 } 5108 | GET_ZVAL_LVAL result_reg, op1_addr 5109 | shl Ra(result_reg), cl 5110 |1: 5111 } 5112 } else if (opcode == ZEND_SR) { 5113 | GET_ZVAL_LVAL result_reg, op1_addr 5114 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5115 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5116 5117 if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { 5118 if (EXPECTED(op2_lval > 0)) { 5119 | sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1 5120 } else { 5121 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5122 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5123 | SET_EX_OPLINE opline, r0 5124 | jmp ->negative_shift 5125 } 5126 } else { 5127 | sar Ra(result_reg), op2_lval 5128 } 5129 } else { 5130 if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) { 5131 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5132 } 5133 if (!op2_range || 5134 op2_range->min < 0 || 5135 op2_range->max >= SIZEOF_ZEND_LONG * 8) { 5136 | cmp r1, (SIZEOF_ZEND_LONG*8) 5137 | jae >1 5138 |.cold_code 5139 |1: 5140 | cmp r1, 0 5141 | mov r1, (SIZEOF_ZEND_LONG * 8) - 1 5142 | jg >1 5143 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5144 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5145 | SET_EX_OPLINE opline, r0 5146 | jmp ->negative_shift 5147 |.code 5148 } 5149 |1: 5150 | sar Ra(result_reg), cl 5151 } 5152 } else if (opcode == ZEND_MOD) { 5153 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5154 zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); 5155 5156 if (op2_lval == 0) { 5157 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5158 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5159 | SET_EX_OPLINE opline, r0 5160 | jmp ->mod_by_zero 5161 } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) { 5162 zval tmp; 5163 zend_jit_addr tmp_addr; 5164 zend_reg tmp_reg; 5165 5166 /* Optimisation for mod of power of 2 */ 5167 ZVAL_LONG(&tmp, op2_lval - 1); 5168 tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp); 5169 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 5170 tmp_reg = ZREG_R1; 5171 } else if (result_reg != ZREG_R0) { 5172 tmp_reg = ZREG_R0; 5173 } else { 5174 tmp_reg = ZREG_R1; 5175 } 5176 | GET_ZVAL_LVAL result_reg, op1_addr 5177 | LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, tmp_reg 5178 (void)tmp_reg; 5179 } else { 5180 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5181 | mov aword T1, r0 // save 5182 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) { 5183 | mov aword T1, Ra(ZREG_RCX) // save 5184 } 5185 result_reg = ZREG_RDX; 5186 if (op2_lval == -1) { 5187 | xor Ra(result_reg), Ra(result_reg) 5188 } else { 5189 | GET_ZVAL_LVAL ZREG_RAX, op1_addr 5190 | GET_ZVAL_LVAL ZREG_RCX, op2_addr 5191 |.if X64 5192 | cqo 5193 |.else 5194 | cdq 5195 |.endif 5196 | idiv Ra(ZREG_RCX) 5197 } 5198 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5199 | mov r0, aword T1 // restore 5200 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) { 5201 | mov Ra(ZREG_RCX), aword T1 // restore 5202 } 5203 } 5204 } else { 5205 if (!op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { 5206 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5207 | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], 0 5208 } else if (Z_MODE(op2_addr) == IS_REG) { 5209 | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) 5210 } 5211 | jz >1 5212 |.cold_code 5213 |1: 5214 zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1); 5215 zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2); 5216 | SET_EX_OPLINE opline, r0 5217 | jmp ->mod_by_zero 5218 |.code 5219 } 5220 5221 /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ 5222 if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { 5223 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5224 | cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], -1 5225 } else if (Z_MODE(op2_addr) == IS_REG) { 5226 | cmp Ra(Z_REG(op2_addr)), -1 5227 } 5228 | jz >1 5229 |.cold_code 5230 |1: 5231 | SET_ZVAL_LVAL res_addr, 0 5232 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5233 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 5234 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 5235 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 5236 } 5237 } 5238 } 5239 | jmp >5 5240 |.code 5241 } 5242 5243 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5244 | mov aword T1, r0 // save 5245 } 5246 result_reg = ZREG_RDX; 5247 | GET_ZVAL_LVAL ZREG_RAX, op1_addr 5248 |.if X64 5249 | cqo 5250 |.else 5251 | cdq 5252 |.endif 5253 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { 5254 | idiv aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)] 5255 } else if (Z_MODE(op2_addr) == IS_REG) { 5256 | idiv Ra(Z_REG(op2_addr)) 5257 } 5258 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) { 5259 | mov r0, aword T1 // restore 5260 } 5261 } 5262 } else if (same_ops) { 5263 | GET_ZVAL_LVAL result_reg, op1_addr 5264 | LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg) 5265 } else { 5266 zend_reg tmp_reg; 5267 5268 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) { 5269 tmp_reg = ZREG_R1; 5270 } else if (result_reg != ZREG_R0) { 5271 tmp_reg = ZREG_R0; 5272 } else { 5273 tmp_reg = ZREG_R1; 5274 } 5275 | GET_ZVAL_LVAL result_reg, op1_addr 5276 | LONG_MATH opcode, result_reg, op2_addr, tmp_reg 5277 (void)tmp_reg; 5278 } 5279 5280 if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { 5281 | SET_ZVAL_LVAL res_addr, Ra(result_reg) 5282 } 5283 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 5284 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { 5285 if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { 5286 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 5287 } 5288 } 5289 } 5290 5291 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || 5292 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 5293 if ((op1_info & MAY_BE_LONG) && 5294 (op2_info & MAY_BE_LONG)) { 5295 |.cold_code 5296 } 5297 |6: 5298 if (Z_MODE(res_addr) == IS_REG) { 5299 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5300 | LOAD_ZVAL_ADDR FCARG1a, real_addr 5301 } else if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { 5302 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5303 } 5304 if (Z_MODE(op1_addr) == IS_REG) { 5305 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); 5306 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 5307 return 0; 5308 } 5309 op1_addr = real_addr; 5310 } 5311 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5312 if (Z_MODE(op2_addr) == IS_REG) { 5313 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); 5314 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 5315 return 0; 5316 } 5317 op2_addr = real_addr; 5318 } 5319 |.if X64 5320 | LOAD_ZVAL_ADDR CARG3, op2_addr 5321 |.else 5322 | sub r4, 12 5323 | PUSH_ZVAL_ADDR op2_addr, r0 5324 |.endif 5325 | SET_EX_OPLINE opline, r0 5326 if (opcode == ZEND_BW_OR) { 5327 | EXT_CALL bitwise_or_function, r0 5328 } else if (opcode == ZEND_BW_AND) { 5329 | EXT_CALL bitwise_and_function, r0 5330 } else if (opcode == ZEND_BW_XOR) { 5331 | EXT_CALL bitwise_xor_function, r0 5332 } else if (opcode == ZEND_SL) { 5333 | EXT_CALL shift_left_function, r0 5334 } else if (opcode == ZEND_SR) { 5335 | EXT_CALL shift_right_function, r0 5336 } else if (opcode == ZEND_MOD) { 5337 | EXT_CALL mod_function, r0 5338 } else { 5339 ZEND_UNREACHABLE(); 5340 } 5341 |.if not(X64) 5342 | add r4, 12 5343 |.endif 5344 if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) { 5345 /* compound assignment may decrement "op2" refcount */ 5346 op2_info |= MAY_BE_RC1; 5347 } 5348 | FREE_OP op1_type, op1, op1_info, 0, opline 5349 | FREE_OP op2_type, op2, op2_info, 0, opline 5350 if (may_throw) { 5351 if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { 5352 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 5353 | jne ->exception_handler_free_op2 5354 } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5355 zend_jit_check_exception_undef_result(Dst, opline); 5356 } else { 5357 zend_jit_check_exception(Dst); 5358 } 5359 } 5360 if (Z_MODE(res_addr) == IS_REG) { 5361 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); 5362 if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { 5363 return 0; 5364 } 5365 } 5366 if ((op1_info & MAY_BE_LONG) && 5367 (op2_info & MAY_BE_LONG)) { 5368 | jmp >5 5369 |.code 5370 } 5371 } 5372 |5: 5373 5374 return 1; 5375} 5376 5377static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 5378{ 5379 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5380 ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); 5381 5382 if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, 5383 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 5384 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 5385 opline->result.var, res_addr, res_info, res_use_info, may_throw)) { 5386 return 0; 5387 } 5388 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 5389 return 0; 5390 } 5391 return 1; 5392} 5393 5394static int zend_jit_concat_helper(dasm_State **Dst, 5395 const zend_op *opline, 5396 zend_uchar op1_type, 5397 znode_op op1, 5398 zend_jit_addr op1_addr, 5399 uint32_t op1_info, 5400 zend_uchar op2_type, 5401 znode_op op2, 5402 zend_jit_addr op2_addr, 5403 uint32_t op2_info, 5404 zend_jit_addr res_addr, 5405 int may_throw) 5406{ 5407#if 1 5408 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5409 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5410 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 5411 } 5412 if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { 5413 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6 5414 } 5415 if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { 5416 if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { 5417 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5418 } 5419 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 5420 | EXT_CALL zend_jit_fast_assign_concat_helper, r0 5421 /* concatination with itself may reduce refcount */ 5422 op2_info |= MAY_BE_RC1; 5423 } else { 5424 if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { 5425 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5426 } 5427 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5428 |.if X64 5429 | LOAD_ZVAL_ADDR CARG3, op2_addr 5430 |.else 5431 | sub r4, 12 5432 | PUSH_ZVAL_ADDR op2_addr, r0 5433 |.endif 5434 | EXT_CALL zend_jit_fast_concat_helper, r0 5435 |.if not(X64) 5436 | add r4, 12 5437 |.endif 5438 } 5439 /* concatination with empty string may increase refcount */ 5440 op1_info |= MAY_BE_RCN; 5441 op2_info |= MAY_BE_RCN; 5442 | FREE_OP op1_type, op1, op1_info, 0, opline 5443 | FREE_OP op2_type, op2, op2_info, 0, opline 5444 |5: 5445 } 5446 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || 5447 (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { 5448 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5449 |.cold_code 5450 |6: 5451 } 5452#endif 5453 if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { 5454 | LOAD_ZVAL_ADDR FCARG1a, res_addr 5455 } 5456 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 5457 |.if X64 5458 | LOAD_ZVAL_ADDR CARG3, op2_addr 5459 |.else 5460 | sub r4, 12 5461 | PUSH_ZVAL_ADDR op2_addr, r0 5462 |.endif 5463 | SET_EX_OPLINE opline, r0 5464 | EXT_CALL concat_function, r0 5465 |.if not(X64) 5466 | add r4, 12 5467 |.endif 5468 /* concatination with empty string may increase refcount */ 5469 op1_info |= MAY_BE_RCN; 5470 op2_info |= MAY_BE_RCN; 5471 | FREE_OP op1_type, op1, op1_info, 0, opline 5472 | FREE_OP op2_type, op2, op2_info, 0, opline 5473 if (may_throw) { 5474 if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { 5475 zend_jit_check_exception_undef_result(Dst, opline); 5476 } else { 5477 zend_jit_check_exception(Dst); 5478 } 5479 } 5480#if 1 5481 if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { 5482 | jmp <5 5483 |.code 5484 } 5485 } 5486#endif 5487 5488 return 1; 5489} 5490 5491static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) 5492{ 5493 zend_jit_addr op1_addr, op2_addr; 5494 5495 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 5496 ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); 5497 5498 op1_addr = OP1_ADDR(); 5499 op2_addr = OP2_ADDR(); 5500 5501 return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); 5502} 5503 5504static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) 5505/* Labels: 1,2,3,4,5 */ 5506{ 5507 zend_jit_addr op2_addr = OP2_ADDR(); 5508 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 5509 5510 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 5511 && type == BP_VAR_R 5512 && !exit_addr) { 5513 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 5514 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5515 if (!exit_addr) { 5516 return 0; 5517 } 5518 } 5519 5520 if (op2_info & MAY_BE_LONG) { 5521 zend_bool op2_loaded = 0; 5522 zend_bool packed_loaded = 0; 5523 zend_bool bad_packed_key = 0; 5524 5525 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { 5526 | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) 5527 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 5528 } 5529 if (op1_info & MAY_BE_PACKED_GUARD) { 5530 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); 5531 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 5532 5533 if (!exit_addr) { 5534 return 0; 5535 } 5536 if (op1_info & MAY_BE_ARRAY_PACKED) { 5537 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5538 | jz &exit_addr 5539 } else { 5540 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5541 | jnz &exit_addr 5542 } 5543 } 5544 if (type == BP_VAR_W) { 5545 | // hval = Z_LVAL_P(dim); 5546 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5547 op2_loaded = 1; 5548 } 5549 if (op1_info & MAY_BE_ARRAY_PACKED) { 5550 zend_long val = -1; 5551 5552 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 5553 val = Z_LVAL_P(Z_ZV(op2_addr)); 5554 if (val >= 0 && val < HT_MAX_SIZE) { 5555 packed_loaded = 1; 5556 } else { 5557 bad_packed_key = 1; 5558 } 5559 } else { 5560 if (!op2_loaded) { 5561 | // hval = Z_LVAL_P(dim); 5562 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5563 op2_loaded = 1; 5564 } 5565 packed_loaded = 1; 5566 } 5567 if (packed_loaded) { 5568 | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); 5569 if (op1_info & MAY_BE_ARRAY_HASH) { 5570 | test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED 5571 | jz >4 // HASH_FIND 5572 } 5573 | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) 5574 |.if X64 5575 | mov eax, dword [FCARG1a + offsetof(zend_array, nNumUsed)] 5576 if (val == 0) { 5577 | test r0, r0 5578 } else if (val > 0 && !op2_loaded) { 5579 | cmp r0, val 5580 } else { 5581 | cmp r0, FCARG2a 5582 } 5583 |.else 5584 if (val >= 0 && !op2_loaded) { 5585 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val 5586 } else { 5587 | cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a 5588 } 5589 |.endif 5590 if (type == BP_JIT_IS) { 5591 if (not_found_exit_addr) { 5592 | jbe ¬_found_exit_addr 5593 } else { 5594 | jbe >9 // NOT_FOUND 5595 } 5596 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5597 | jbe &exit_addr 5598 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5599 | jbe ¬_found_exit_addr 5600 } else if (type == BP_VAR_IS && found_exit_addr) { 5601 | jbe >7 // NOT_FOUND 5602 } else { 5603 | jbe >2 // NOT_FOUND 5604 } 5605 | // _ret = &_ht->arData[_h].val; 5606 if (val >= 0) { 5607 | mov r0, aword [FCARG1a + offsetof(zend_array, arData)] 5608 if (val != 0) { 5609 | add r0, val * sizeof(Bucket) 5610 } 5611 } else { 5612 |.if X64 5613 | mov r0, FCARG2a 5614 | shl r0, 5 5615 |.else 5616 | imul r0, FCARG2a, sizeof(Bucket) 5617 |.endif 5618 | add r0, aword [FCARG1a + offsetof(zend_array, arData)] 5619 } 5620 } 5621 } 5622 switch (type) { 5623 case BP_JIT_IS: 5624 if (op1_info & MAY_BE_ARRAY_HASH) { 5625 if (packed_loaded) { 5626 | jmp >5 5627 } 5628 |4: 5629 if (!op2_loaded) { 5630 | // hval = Z_LVAL_P(dim); 5631 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5632 } 5633 if (packed_loaded) { 5634 | EXT_CALL _zend_hash_index_find, r0 5635 } else { 5636 | EXT_CALL zend_hash_index_find, r0 5637 } 5638 | test r0, r0 5639 if (not_found_exit_addr) { 5640 | jz ¬_found_exit_addr 5641 } else { 5642 | jz >9 // NOT_FOUND 5643 } 5644 if (op2_info & MAY_BE_STRING) { 5645 | jmp >5 5646 } 5647 } else if (packed_loaded) { 5648 if (op2_info & MAY_BE_STRING) { 5649 | jmp >5 5650 } 5651 } else if (not_found_exit_addr) { 5652 | jmp ¬_found_exit_addr 5653 } else { 5654 | jmp >9 // NOT_FOUND 5655 } 5656 break; 5657 case BP_VAR_R: 5658 case BP_VAR_IS: 5659 case BP_VAR_UNSET: 5660 if (packed_loaded) { 5661 if (op1_info & MAY_BE_ARRAY_HASH) { 5662 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5663 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5664 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 5665 if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_HASH)) { 5666 | IF_Z_TYPE r0, IS_UNDEF, &exit_addr 5667 } 5668 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5669 | IF_Z_TYPE r0, IS_UNDEF, ¬_found_exit_addr 5670 } else if (type == BP_VAR_IS && found_exit_addr) { 5671 | IF_Z_TYPE r0, IS_UNDEF, >7 // NOT_FOUND 5672 } else { 5673 | IF_Z_TYPE r0, IS_UNDEF, >2 // NOT_FOUND 5674 } 5675 } 5676 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_HASH))) { 5677 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5678 | jmp &exit_addr 5679 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5680 | jmp ¬_found_exit_addr 5681 } else if (type == BP_VAR_IS && found_exit_addr) { 5682 | jmp >7 // NOT_FOUND 5683 } else { 5684 | jmp >2 // NOT_FOUND 5685 } 5686 } 5687 if (op1_info & MAY_BE_ARRAY_HASH) { 5688 |4: 5689 if (!op2_loaded) { 5690 | // hval = Z_LVAL_P(dim); 5691 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5692 } 5693 if (packed_loaded) { 5694 | EXT_CALL _zend_hash_index_find, r0 5695 } else { 5696 | EXT_CALL zend_hash_index_find, r0 5697 } 5698 | test r0, r0 5699 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5700 | jz &exit_addr 5701 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5702 | jz ¬_found_exit_addr 5703 } else if (type == BP_VAR_IS && found_exit_addr) { 5704 | jz >7 // NOT_FOUND 5705 } else { 5706 | jz >2 // NOT_FOUND 5707 } 5708 } 5709 |.cold_code 5710 |2: 5711 switch (type) { 5712 case BP_VAR_R: 5713 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5714 | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); 5715 | // retval = &EG(uninitialized_zval); 5716 | UNDEFINED_OFFSET opline 5717 | jmp >9 5718 } 5719 break; 5720 case BP_VAR_IS: 5721 case BP_VAR_UNSET: 5722 if (!not_found_exit_addr && !found_exit_addr) { 5723 | // retval = &EG(uninitialized_zval); 5724 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 5725 | jmp >9 5726 } 5727 break; 5728 default: 5729 ZEND_UNREACHABLE(); 5730 } 5731 |.code 5732 break; 5733 case BP_VAR_RW: 5734 if (packed_loaded) { 5735 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5736 } 5737 |2: 5738 |4: 5739 if (!op2_loaded) { 5740 | // hval = Z_LVAL_P(dim); 5741 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5742 } 5743 | SET_EX_OPLINE opline, r0 5744 if (packed_loaded) { 5745 | EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, r0 5746 } else { 5747 | EXT_CALL zend_jit_hash_index_lookup_rw, r0 5748 } 5749 | test r0, r0 5750 | jz >9 5751 break; 5752 case BP_VAR_W: 5753 if (packed_loaded) { 5754 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5755 } 5756 if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || packed_loaded || bad_packed_key) { 5757 |2: 5758 | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); 5759 if (!op2_loaded) { 5760 | // hval = Z_LVAL_P(dim); 5761 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5762 } 5763 |.if X64 5764 | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval 5765 |.else 5766 | sub r4, 12 5767 | PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0 5768 |.endif 5769 | EXT_CALL zend_hash_index_add_new, r0 5770 |.if not(X64) 5771 | add r4, 12 5772 |.endif 5773 if (op1_info & MAY_BE_ARRAY_HASH) { 5774 | jmp >8 5775 } 5776 } 5777 if (op1_info & MAY_BE_ARRAY_HASH) { 5778 |4: 5779 if (!op2_loaded) { 5780 | // hval = Z_LVAL_P(dim); 5781 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5782 } 5783 | EXT_CALL zend_jit_hash_index_lookup_w, r0 5784 } 5785 break; 5786 default: 5787 ZEND_UNREACHABLE(); 5788 } 5789 5790 if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { 5791 | jmp >8 5792 } 5793 } 5794 5795 if (op2_info & MAY_BE_STRING) { 5796 |3: 5797 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5798 | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) 5799 | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3 5800 } 5801 | // offset_key = Z_STR_P(dim); 5802 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 5803 | // retval = zend_hash_find(ht, offset_key); 5804 switch (type) { 5805 case BP_JIT_IS: 5806 if (opline->op2_type != IS_CONST) { 5807 | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' 5808 | jle >1 5809 |.cold_code 5810 |1: 5811 | EXT_CALL zend_jit_symtable_find, r0 5812 | jmp >1 5813 |.code 5814 | EXT_CALL zend_hash_find, r0 5815 |1: 5816 } else { 5817 | EXT_CALL _zend_hash_find_known_hash, r0 5818 } 5819 | test r0, r0 5820 if (not_found_exit_addr) { 5821 | jz ¬_found_exit_addr 5822 } else { 5823 | jz >9 // NOT_FOUND 5824 } 5825 | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) 5826 | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 5827 | GET_Z_PTR r0, r0 5828 |1: 5829 break; 5830 case BP_VAR_R: 5831 case BP_VAR_IS: 5832 case BP_VAR_UNSET: 5833 if (opline->op2_type != IS_CONST) { 5834 | cmp byte [FCARG2a + offsetof(zend_string, val)], '9' 5835 | jle >1 5836 |.cold_code 5837 |1: 5838 | EXT_CALL zend_jit_symtable_find, r0 5839 | jmp >1 5840 |.code 5841 | EXT_CALL zend_hash_find, r0 5842 |1: 5843 } else { 5844 | EXT_CALL _zend_hash_find_known_hash, r0 5845 } 5846 | test r0, r0 5847 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5848 | jz &exit_addr 5849 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5850 | jz ¬_found_exit_addr 5851 } else if (type == BP_VAR_IS && found_exit_addr) { 5852 | jz >7 // NOT_FOUND 5853 } else { 5854 | jz >2 // NOT_FOUND 5855 } 5856 | // if (UNEXPECTED(Z_TYPE_P(retval) == IS_INDIRECT)) 5857 | IF_Z_TYPE r0, IS_INDIRECT, >1 // SLOW 5858 |.cold_code 5859 |1: 5860 | // retval = Z_INDIRECT_P(retval); 5861 | GET_Z_PTR r0, r0 5862 | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 5863 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { 5864 | jmp &exit_addr 5865 } else if (type == BP_VAR_IS && not_found_exit_addr) { 5866 | jmp ¬_found_exit_addr 5867 } 5868 |2: 5869 switch (type) { 5870 case BP_VAR_R: 5871 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 5872 // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); 5873 | UNDEFINED_INDEX opline 5874 | jmp >9 5875 } 5876 break; 5877 case BP_VAR_IS: 5878 case BP_VAR_UNSET: 5879 if (!not_found_exit_addr && !found_exit_addr) { 5880 | // retval = &EG(uninitialized_zval); 5881 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 5882 | jmp >9 5883 } 5884 break; 5885 default: 5886 ZEND_UNREACHABLE(); 5887 } 5888 |.code 5889 break; 5890 case BP_VAR_RW: 5891 | SET_EX_OPLINE opline, r0 5892 if (opline->op2_type != IS_CONST) { 5893 | EXT_CALL zend_jit_symtable_lookup_rw, r0 5894 } else { 5895 | EXT_CALL zend_jit_hash_lookup_rw, r0 5896 } 5897 | test r0, r0 5898 | jz >9 5899 break; 5900 case BP_VAR_W: 5901 if (opline->op2_type != IS_CONST) { 5902 | EXT_CALL zend_jit_symtable_lookup_w, r0 5903 } else { 5904 | EXT_CALL zend_jit_hash_lookup_w, r0 5905 } 5906 break; 5907 default: 5908 ZEND_UNREACHABLE(); 5909 } 5910 } 5911 5912 if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { 5913 |5: 5914 if (op1_info & MAY_BE_ARRAY_OF_REF) { 5915 | ZVAL_DEREF r0, MAY_BE_REF 5916 } 5917 | cmp byte [r0 + 8], IS_NULL 5918 if (not_found_exit_addr) { 5919 | jle ¬_found_exit_addr 5920 } else if (found_exit_addr) { 5921 | jg &found_exit_addr 5922 } else { 5923 | jle >9 // NOT FOUND 5924 } 5925 } 5926 5927 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { 5928 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5929 |.cold_code 5930 |3: 5931 } 5932 | SET_EX_OPLINE opline, r0 5933 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 5934 switch (type) { 5935 case BP_VAR_R: 5936 |.if X64 5937 | LOAD_ZVAL_ADDR CARG3, res_addr 5938 |.else 5939 | sub r4, 12 5940 | PUSH_ZVAL_ADDR res_addr, r0 5941 |.endif 5942 | EXT_CALL zend_jit_fetch_dim_r_helper, r0 5943 |.if not(X64) 5944 | add r4, 12 5945 |.endif 5946 | jmp >9 5947 break; 5948 case BP_JIT_IS: 5949 | EXT_CALL zend_jit_fetch_dim_isset_helper, r0 5950 | test r0, r0 5951 if (not_found_exit_addr) { 5952 | je ¬_found_exit_addr 5953 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5954 | jmp >8 5955 } 5956 } else if (found_exit_addr) { 5957 | jne &found_exit_addr 5958 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5959 | jmp >9 5960 } 5961 } else { 5962 | jne >8 5963 | jmp >9 5964 } 5965 break; 5966 case BP_VAR_IS: 5967 case BP_VAR_UNSET: 5968 |.if X64 5969 | LOAD_ZVAL_ADDR CARG3, res_addr 5970 |.else 5971 | sub r4, 12 5972 | PUSH_ZVAL_ADDR res_addr, r0 5973 |.endif 5974 | EXT_CALL zend_jit_fetch_dim_is_helper, r0 5975 |.if not(X64) 5976 | add r4, 12 5977 |.endif 5978 | jmp >9 5979 break; 5980 case BP_VAR_RW: 5981 | EXT_CALL zend_jit_fetch_dim_rw_helper, r0 5982 | test r0, r0 5983 | jne >8 5984 | jmp >9 5985 break; 5986 case BP_VAR_W: 5987 | EXT_CALL zend_jit_fetch_dim_w_helper, r0 5988 | test r0, r0 5989 | jne >8 5990 | jmp >9 5991 break; 5992 default: 5993 ZEND_UNREACHABLE(); 5994 } 5995 if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { 5996 |.code 5997 } 5998 } 5999 6000 return 1; 6001} 6002 6003static int zend_jit_simple_assign(dasm_State **Dst, 6004 const zend_op *opline, 6005 zend_jit_addr var_addr, 6006 uint32_t var_info, 6007 uint32_t var_def_info, 6008 zend_uchar val_type, 6009 zend_jit_addr val_addr, 6010 uint32_t val_info, 6011 zend_jit_addr res_addr, 6012 int in_cold, 6013 int save_r1) 6014/* Labels: 1,2,3 */ 6015{ 6016 zend_reg tmp_reg; 6017 6018 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_R0) { 6019 tmp_reg = ZREG_R0; 6020 } else { 6021 /* ASSIGN_DIM */ 6022 tmp_reg = ZREG_FCARG1a; 6023 } 6024 6025 if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 6026 zval *zv = Z_ZV(val_addr); 6027 6028 if (!res_addr) { 6029 | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg 6030 } else { 6031 | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg 6032 } 6033 if (Z_REFCOUNTED_P(zv)) { 6034 if (!res_addr) { 6035 | ADDREF_CONST zv, Ra(tmp_reg) 6036 } else { 6037 | ADDREF_CONST_2 zv, Ra(tmp_reg) 6038 } 6039 } 6040 } else { 6041 if (val_info & MAY_BE_UNDEF) { 6042 if (in_cold) { 6043 | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2 6044 } else { 6045 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 6046 |.cold_code 6047 |1: 6048 } 6049 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 6050 if (save_r1) { 6051 | mov aword T1, FCARG1a // save 6052 } 6053 | SET_ZVAL_TYPE_INFO var_addr, IS_NULL 6054 if (res_addr) { 6055 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 6056 } 6057 if (opline) { 6058 | SET_EX_OPLINE opline, Ra(tmp_reg) 6059 } 6060 ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); 6061 | mov FCARG1d, Z_OFFSET(val_addr) 6062 | EXT_CALL zend_jit_undefined_op_helper, r0 6063 | test r0, r0 6064 | jz ->exception_handler_undef 6065 if (save_r1) { 6066 | mov FCARG1a, aword T1 // restore 6067 } 6068 | jmp >3 6069 if (in_cold) { 6070 |2: 6071 } else { 6072 |.code 6073 } 6074 } 6075 if (val_info & MAY_BE_REF) { 6076 if (val_type == IS_CV) { 6077 ZEND_ASSERT(Z_REG(var_addr) != ZREG_R2); 6078 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_R2 || Z_OFFSET(val_addr) != 0) { 6079 | LOAD_ZVAL_ADDR r2, val_addr 6080 } 6081 | ZVAL_DEREF r2, val_info 6082 val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); 6083 } else { 6084 zend_jit_addr ref_addr; 6085 zend_reg type_reg = tmp_reg; 6086 6087 if (in_cold) { 6088 | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1 6089 } else { 6090 | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1 6091 |.cold_code 6092 |1: 6093 } 6094 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 6095 | GET_ZVAL_PTR r2, val_addr 6096 | GC_DELREF r2 6097 | // ZVAL_COPY_VALUE(return_value, &ref->value); 6098 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8); 6099 if (!res_addr) { 6100 | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, type_reg, tmp_reg 6101 } else { 6102 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, type_reg, tmp_reg 6103 } 6104 | je >2 6105 if (tmp_reg == ZREG_R0) { 6106 | IF_NOT_REFCOUNTED ah, >3 6107 } else { 6108 | IF_NOT_FLAGS Rd(tmp_reg), (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), >3 6109 } 6110 | GET_ZVAL_PTR Ra(tmp_reg), var_addr 6111 6112 if (!res_addr) { 6113 | GC_ADDREF Ra(tmp_reg) 6114 } else { 6115 | add dword [Ra(tmp_reg)], 2 6116 } 6117 | jmp >3 6118 |2: 6119 if (res_addr) { 6120 if (tmp_reg == ZREG_R0) { 6121 | IF_NOT_REFCOUNTED ah, >2 6122 } else { 6123 | IF_NOT_FLAGS Rd(tmp_reg), (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), >2 6124 } 6125 | GET_ZVAL_PTR Ra(tmp_reg), var_addr 6126 | GC_ADDREF Ra(tmp_reg) 6127 |2: 6128 } 6129 if (save_r1) { 6130 | mov aword T1, FCARG1a // save 6131 } 6132 | EFREE_REFERENCE r2 6133 if (save_r1) { 6134 | mov FCARG1a, aword T1 // restore 6135 } 6136 | jmp >3 6137 if (in_cold) { 6138 |1: 6139 } else { 6140 |.code 6141 } 6142 } 6143 } 6144 6145 if (!res_addr) { 6146 | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, tmp_reg 6147 } else { 6148 | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_R2, tmp_reg 6149 } 6150 6151 if (val_type == IS_CV) { 6152 if (!res_addr) { 6153 | TRY_ADDREF val_info, dh, Ra(tmp_reg) 6154 } else { 6155 | TRY_ADDREF_2 val_info, dh, Ra(tmp_reg) 6156 } 6157 } else { 6158 if (res_addr) { 6159 | TRY_ADDREF val_info, dh, Ra(tmp_reg) 6160 } 6161 } 6162 |3: 6163 } 6164 return 1; 6165} 6166 6167static int zend_jit_assign_to_typed_ref(dasm_State **Dst, 6168 const zend_op *opline, 6169 zend_uchar val_type, 6170 zend_jit_addr val_addr, 6171 zend_jit_addr res_addr, 6172 zend_bool check_exception) 6173{ 6174 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 6175 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6176 | jnz >2 6177 |.cold_code 6178 |2: 6179 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) { 6180 | LOAD_ZVAL_ADDR FCARG2a, val_addr 6181 } 6182 if (opline) { 6183 | SET_EX_OPLINE opline, r0 6184 } 6185 if (val_type == IS_CONST) { 6186 | EXT_CALL zend_jit_assign_const_to_typed_ref, r0 6187 } else if (val_type == IS_TMP_VAR) { 6188 | EXT_CALL zend_jit_assign_tmp_to_typed_ref, r0 6189 } else if (val_type == IS_VAR) { 6190 | EXT_CALL zend_jit_assign_var_to_typed_ref, r0 6191 } else if (val_type == IS_CV) { 6192 | EXT_CALL zend_jit_assign_cv_to_typed_ref, r0 6193 } else { 6194 ZEND_UNREACHABLE(); 6195 } 6196 if (res_addr) { 6197 zend_jit_addr ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6198 6199 | ZVAL_COPY_VALUE res_addr, -1, ret_addr, -1, ZREG_R1, ZREG_R2 6200 | TRY_ADDREF -1, ch, r2 6201 } 6202 if (check_exception) { 6203 | // if (UNEXPECTED(EG(exception) != NULL)) { 6204 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 6205 | je >8 // END OF zend_jit_assign_to_variable() 6206 | jmp ->exception_handler 6207 } else { 6208 | jmp >8 6209 } 6210 |.code 6211 6212 return 1; 6213} 6214 6215static int zend_jit_assign_to_variable_call(dasm_State **Dst, 6216 const zend_op *opline, 6217 zend_jit_addr __var_use_addr, 6218 zend_jit_addr var_addr, 6219 uint32_t __var_info, 6220 uint32_t __var_def_info, 6221 zend_uchar val_type, 6222 zend_jit_addr val_addr, 6223 uint32_t val_info, 6224 zend_jit_addr __res_addr, 6225 zend_bool __check_exception) 6226{ 6227 if (val_info & MAY_BE_UNDEF) { 6228 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 6229 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6230 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6231 6232 if (!exit_addr) { 6233 return 0; 6234 } 6235 6236 | IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr 6237 } else { 6238 | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1 6239 |.cold_code 6240 |1: 6241 ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP); 6242 if (Z_REG(var_addr) != ZREG_FP) { 6243 | mov aword T1, Ra(Z_REG(var_addr)) // save 6244 } 6245 | SET_EX_OPLINE opline, r0 6246 | mov FCARG1d, Z_OFFSET(val_addr) 6247 | EXT_CALL zend_jit_undefined_op_helper, r0 6248 if (Z_REG(var_addr) != ZREG_FP) { 6249 | mov Ra(Z_REG(var_addr)), aword T1 // restore 6250 } 6251 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1a || Z_OFFSET(var_addr) != 0) { 6252 | LOAD_ZVAL_ADDR FCARG1a, var_addr 6253 } 6254 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6255 | call ->assign_const 6256 | jmp >9 6257 |.code 6258 } 6259 } 6260 if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1a || Z_OFFSET(var_addr) != 0) { 6261 | LOAD_ZVAL_ADDR FCARG1a, var_addr 6262 } 6263 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) { 6264 | LOAD_ZVAL_ADDR FCARG2a, val_addr 6265 } 6266 if (opline) { 6267 | SET_EX_OPLINE opline, r0 6268 } 6269 if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 6270 | call ->assign_tmp 6271 } else if (val_type == IS_CONST) { 6272 | call ->assign_const 6273 } else if (val_type == IS_TMP_VAR) { 6274 | call ->assign_tmp 6275 } else if (val_type == IS_VAR) { 6276 if (!(val_info & MAY_BE_REF)) { 6277 | call ->assign_tmp 6278 } else { 6279 | call ->assign_var 6280 } 6281 } else if (val_type == IS_CV) { 6282 if (!(val_info & MAY_BE_REF)) { 6283 | call ->assign_cv_noref 6284 } else { 6285 | call ->assign_cv 6286 } 6287 if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 6288 |9: 6289 } 6290 } else { 6291 ZEND_UNREACHABLE(); 6292 } 6293 6294 return 1; 6295} 6296 6297static int zend_jit_assign_to_variable(dasm_State **Dst, 6298 const zend_op *opline, 6299 zend_jit_addr var_use_addr, 6300 zend_jit_addr var_addr, 6301 uint32_t var_info, 6302 uint32_t var_def_info, 6303 zend_uchar val_type, 6304 zend_jit_addr val_addr, 6305 uint32_t val_info, 6306 zend_jit_addr res_addr, 6307 zend_bool check_exception) 6308/* Labels: 1,2,3,4,5,8 */ 6309{ 6310 int done = 0; 6311 zend_reg ref_reg, tmp_reg; 6312 6313 if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_R0) { 6314 ref_reg = ZREG_FCARG1a; 6315 tmp_reg = ZREG_R0; 6316 } else { 6317 /* ASSIGN_DIM */ 6318 ref_reg = ZREG_R0; 6319 tmp_reg = ZREG_FCARG1a; 6320 } 6321 6322 if (var_info & MAY_BE_REF) { 6323 if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { 6324 | LOAD_ZVAL_ADDR Ra(ref_reg), var_use_addr 6325 var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); 6326 } 6327 | // if (Z_ISREF_P(variable_ptr)) { 6328 | IF_NOT_Z_TYPE, Ra(ref_reg), IS_REFERENCE, >3 6329 | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { 6330 | GET_Z_PTR FCARG1a, Ra(ref_reg) 6331 if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) { 6332 return 0; 6333 } 6334 | lea Ra(ref_reg), [FCARG1a + offsetof(zend_reference, val)] 6335 |3: 6336 } 6337 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6338 if (RC_MAY_BE_1(var_info)) { 6339 int in_cold = 0; 6340 6341 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6342 | IF_ZVAL_REFCOUNTED var_use_addr, >1 6343 |.cold_code 6344 |1: 6345 in_cold = 1; 6346 } 6347 if (Z_REG(var_use_addr) == ZREG_FCARG1a || Z_REG(var_use_addr) == ZREG_R0) { 6348 zend_bool keep_gc = 0; 6349 6350 | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr 6351 if (tmp_reg == ZREG_FCARG1a) { 6352 if (Z_MODE(val_addr) == IS_REG) { 6353 keep_gc = 1; 6354 } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { 6355 keep_gc = 1; 6356 } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { 6357 if (sizeof(void*) == 4) { 6358 keep_gc = 1; 6359 } else { 6360 zval *zv = Z_ZV(val_addr); 6361 6362 if (Z_TYPE_P(zv) == IS_DOUBLE) { 6363 if (Z_DVAL_P(zv) == 0 || IS_SIGNED_32BIT(zv)) { 6364 keep_gc = 1; 6365 } 6366 } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 6367 keep_gc = 1; 6368 } 6369 } 6370 } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { 6371 if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { 6372 keep_gc = 1; 6373 } 6374 } 6375 } 6376 if (!keep_gc) { 6377 | mov aword T1, Ra(tmp_reg) // save 6378 } 6379 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0)) { 6380 return 0; 6381 } 6382 if (!keep_gc) { 6383 | mov FCARG1a, aword T1 // restore 6384 } 6385 } else { 6386 | GET_ZVAL_PTR FCARG1a, var_use_addr 6387 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1)) { 6388 return 0; 6389 } 6390 } 6391 | GC_DELREF FCARG1a 6392 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 6393 | jnz >4 6394 } else { 6395 | jnz >8 6396 } 6397 | ZVAL_DTOR_FUNC var_info, opline 6398 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { 6399 if (check_exception) { 6400 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 6401 | je >8 6402 | jmp ->exception_handler 6403 } else { 6404 | jmp >8 6405 } 6406 } 6407 if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { 6408 |4: 6409 | IF_GC_MAY_NOT_LEAK FCARG1a, >8 6410 | EXT_CALL gc_possible_root, r0 6411 if (in_cold) { 6412 | jmp >8 6413 } 6414 } 6415 if (in_cold) { 6416 |.code 6417 } else { 6418 done = 1; 6419 } 6420 } else /* if (RC_MAY_BE_N(var_info)) */ { 6421 if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6422 | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5 6423 } 6424 if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { 6425 if (Z_REG(var_use_addr) == ZREG_FP) { 6426 | mov T1, Ra(Z_REG(var_use_addr)) // save 6427 } 6428 | GET_ZVAL_PTR FCARG1a, var_use_addr 6429 | GC_DELREF FCARG1a 6430 | IF_GC_MAY_NOT_LEAK FCARG1a, >5 6431 | EXT_CALL gc_possible_root, r0 6432 if (Z_REG(var_use_addr) != ZREG_FP) { 6433 | mov Ra(Z_REG(var_use_addr)), T1 // restore 6434 } 6435 } else { 6436 | GET_ZVAL_PTR Ra(tmp_reg), var_use_addr 6437 | GC_DELREF Ra(tmp_reg) 6438 } 6439 |5: 6440 } 6441 } 6442 6443 if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0)) { 6444 return 0; 6445 } 6446 6447 |8: 6448 6449 return 1; 6450} 6451 6452static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, int may_throw) 6453{ 6454 zend_jit_addr op2_addr, op3_addr, res_addr; 6455 6456 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6457 op3_addr = OP1_DATA_ADDR(); 6458 if (opline->result_type == IS_UNUSED) { 6459 res_addr = 0; 6460 } else { 6461 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 6462 } 6463 6464 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { 6465 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 6466 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 6467 6468 if (!exit_addr) { 6469 return 0; 6470 } 6471 6472 | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr 6473 6474 val_info &= ~MAY_BE_UNDEF; 6475 } 6476 6477 if (op1_info & MAY_BE_REF) { 6478 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6479 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 6480 | GET_Z_PTR FCARG2a, FCARG1a 6481 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 6482 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 6483 | jmp >3 6484 |.cold_code 6485 |2: 6486 | SET_EX_OPLINE opline, r0 6487 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 6488 | test r0, r0 6489 | mov FCARG1a, r0 6490 | jne >1 6491 | jmp ->exception_handler_undef 6492 |.code 6493 |1: 6494 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 6495 } 6496 6497 if (op1_info & MAY_BE_ARRAY) { 6498 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6499 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 6500 } 6501 |3: 6502 | SEPARATE_ARRAY op1_addr, op1_info, 1 6503 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { 6504 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 6505 | CMP_ZVAL_TYPE op1_addr, IS_FALSE 6506 | jg >7 6507 } 6508 | // ZVAL_ARR(container, zend_new_array(8)); 6509 if (Z_REG(op1_addr) != ZREG_FP) { 6510 | mov T1, Ra(Z_REG(op1_addr)) // save 6511 } 6512 | EXT_CALL _zend_new_array_0, r0 6513 if (Z_REG(op1_addr) != ZREG_FP) { 6514 | mov Ra(Z_REG(op1_addr)), T1 // restore 6515 } 6516 | SET_ZVAL_LVAL op1_addr, r0 6517 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6518 | mov FCARG1a, r0 6519 } 6520 6521 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 6522 |6: 6523 if (opline->op2_type == IS_UNUSED) { 6524 uint32_t var_info = MAY_BE_NULL; 6525 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6526 6527 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6528 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6529 | EXT_CALL zend_hash_next_index_insert, r0 6530 | // if (UNEXPECTED(!var_ptr)) { 6531 | test r0, r0 6532 | jz >1 6533 |.cold_code 6534 |1: 6535 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6536 | CANNOT_ADD_ELEMENT opline 6537 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6538 | jmp >9 6539 |.code 6540 6541 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) { 6542 return 0; 6543 } 6544 } else { 6545 uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6546 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6547 6548 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, NULL, NULL, NULL)) { 6549 return 0; 6550 } 6551 6552 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6553 var_info |= MAY_BE_REF; 6554 } 6555 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6556 var_info |= MAY_BE_RC1; 6557 } 6558 6559 |8: 6560 | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); 6561 if (opline->op1_type == IS_VAR) { 6562 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6563 if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6564 return 0; 6565 } 6566 } else { 6567 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { 6568 return 0; 6569 } 6570 } 6571 } 6572 } 6573 6574 if (((op1_info & MAY_BE_ARRAY) && 6575 (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE))) || 6576 (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) { 6577 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 6578 |.cold_code 6579 |7: 6580 } 6581 6582 if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) && 6583 (op1_info & MAY_BE_ARRAY)) { 6584 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 6585 | CMP_ZVAL_TYPE op1_addr, IS_FALSE 6586 | jg >2 6587 } 6588 | // ZVAL_ARR(container, zend_new_array(8)); 6589 if (Z_REG(op1_addr) != ZREG_FP) { 6590 | mov T1, Ra(Z_REG(op1_addr)) // save 6591 } 6592 | EXT_CALL _zend_new_array_0, r0 6593 if (Z_REG(op1_addr) != ZREG_FP) { 6594 | mov Ra(Z_REG(op1_addr)), T1 // restore 6595 } 6596 | SET_ZVAL_LVAL op1_addr, r0 6597 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6598 | mov FCARG1a, r0 6599 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); 6600 | jmp <6 6601 |2: 6602 } 6603 6604 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 6605 | SET_EX_OPLINE opline, r0 6606 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 6607 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6608 } 6609 if (opline->op2_type == IS_UNUSED) { 6610 | xor FCARG2a, FCARG2a 6611 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6612 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6613 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 6614 } else { 6615 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6616 } 6617 |.if not(X64) 6618 | sub r4, 8 6619 |.endif 6620 if (opline->result_type == IS_UNUSED) { 6621 |.if X64 6622 | xor CARG4, CARG4 6623 |.else 6624 | push 0 6625 |.endif 6626 } else { 6627 |.if X64 6628 | LOAD_ZVAL_ADDR CARG4, res_addr 6629 |.else 6630 | PUSH_ZVAL_ADDR res_addr, r0 6631 |.endif 6632 } 6633 |.if X64 6634 | LOAD_ZVAL_ADDR CARG3, op3_addr 6635 |.else 6636 | PUSH_ZVAL_ADDR op3_addr, r0 6637 |.endif 6638 | EXT_CALL zend_jit_assign_dim_helper, r0 6639 |.if not(X64) 6640 | add r4, 8 6641 |.endif 6642 6643#ifdef ZEND_JIT_USE_RC_INFERENCE 6644 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { 6645 /* ASSIGN_DIM may increase refcount of the value */ 6646 val_info |= MAY_BE_RCN; 6647 } 6648#endif 6649 6650 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 6651 } 6652 6653 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 6654 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 6655 | jmp >9 // END 6656 } 6657 |.code 6658 } 6659 } 6660 6661#ifdef ZEND_JIT_USE_RC_INFERENCE 6662 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 6663 /* ASSIGN_DIM may increase refcount of the key */ 6664 op2_info |= MAY_BE_RCN; 6665 } 6666#endif 6667 6668 |9: 6669 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 6670 6671 if (may_throw) { 6672 zend_jit_check_exception(Dst); 6673 } 6674 6675 return 1; 6676} 6677 6678static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, int may_throw) 6679{ 6680 zend_jit_addr op2_addr, op3_addr, var_addr; 6681 6682 ZEND_ASSERT(opline->result_type == IS_UNUSED); 6683 6684 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 6685 op3_addr = OP1_DATA_ADDR(); 6686 6687 if (op1_info & MAY_BE_REF) { 6688 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6689 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 6690 | GET_Z_PTR FCARG2a, FCARG1a 6691 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 6692 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 6693 | jmp >3 6694 |.cold_code 6695 |2: 6696 | SET_EX_OPLINE opline, r0 6697 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 6698 | test r0, r0 6699 | mov FCARG1a, r0 6700 | jne >1 6701 | jmp ->exception_handler_undef 6702 |.code 6703 |1: 6704 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 6705 } 6706 6707 if (op1_info & MAY_BE_ARRAY) { 6708 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 6709 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 6710 } 6711 |3: 6712 | SEPARATE_ARRAY op1_addr, op1_info, 1 6713 } 6714 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { 6715 if (op1_info & MAY_BE_ARRAY) { 6716 |.cold_code 6717 |7: 6718 } 6719 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 6720 | CMP_ZVAL_TYPE op1_addr, IS_FALSE 6721 | jg >7 6722 } 6723 if (Z_REG(op1_addr) != ZREG_FP) { 6724 | mov T1, Ra(Z_REG(op1_addr)) // save 6725 } 6726 if (op1_info & MAY_BE_UNDEF) { 6727 if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { 6728 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 6729 } 6730 | SET_EX_OPLINE opline, r0 6731 | mov FCARG1a, opline->op1.var 6732 | EXT_CALL zend_jit_undefined_op_helper, r0 6733 |1: 6734 } 6735 | // ZVAL_ARR(container, zend_new_array(8)); 6736 | EXT_CALL _zend_new_array_0, r0 6737 if (Z_REG(op1_addr) != ZREG_FP) { 6738 | mov Ra(Z_REG(op1_addr)), T1 // restore 6739 } 6740 | SET_ZVAL_LVAL op1_addr, r0 6741 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 6742 | mov FCARG1a, r0 6743 if (op1_info & MAY_BE_ARRAY) { 6744 | jmp >1 6745 |.code 6746 |1: 6747 } 6748 } 6749 6750 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 6751 uint32_t var_info; 6752 uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); 6753 6754 |6: 6755 if (opline->op2_type == IS_UNUSED) { 6756 var_info = MAY_BE_NULL; 6757 6758 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 6759 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 6760 | EXT_CALL zend_hash_next_index_insert, r0 6761 | // if (UNEXPECTED(!var_ptr)) { 6762 | test r0, r0 6763 | jz >1 6764 |.cold_code 6765 |1: 6766 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 6767 | CANNOT_ADD_ELEMENT opline 6768 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 6769 | jmp >9 6770 |.code 6771 } else { 6772 var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); 6773 if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { 6774 var_info |= MAY_BE_REF; 6775 } 6776 if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 6777 var_info |= MAY_BE_RC1; 6778 } 6779 6780 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, NULL, NULL, NULL)) { 6781 return 0; 6782 } 6783 6784 |8: 6785 if (op1_info & (MAY_BE_ARRAY_OF_REF)) { 6786 binary_op_type binary_op = get_binary_op(opline->extended_value); 6787 | IF_NOT_Z_TYPE, r0, IS_REFERENCE, >1 6788 | GET_Z_PTR FCARG1a, r0 6789 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6790 | jnz >2 6791 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] 6792 |.cold_code 6793 |2: 6794 | LOAD_ZVAL_ADDR FCARG2a, op3_addr 6795 |.if X64 6796 | LOAD_ADDR CARG3, binary_op 6797 |.else 6798 | sub r4, 12 6799 | PUSH_ADDR binary_op, r0 6800 |.endif 6801 | SET_EX_OPLINE opline, r0 6802 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 6803 && (op1_data_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6804 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 6805 } else { 6806 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 6807 } 6808 |.if not(X64) 6809 | add r4, 12 6810 |.endif 6811 zend_jit_check_exception(Dst); 6812 | jmp >9 6813 |.code 6814 |1: 6815 } 6816 } 6817 6818 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 6819 switch (opline->extended_value) { 6820 case ZEND_ADD: 6821 case ZEND_SUB: 6822 case ZEND_MUL: 6823 case ZEND_DIV: 6824 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, 6825 1 /* may overflow */, may_throw)) { 6826 return 0; 6827 } 6828 break; 6829 case ZEND_BW_OR: 6830 case ZEND_BW_AND: 6831 case ZEND_BW_XOR: 6832 case ZEND_SL: 6833 case ZEND_SR: 6834 case ZEND_MOD: 6835 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6836 IS_CV, opline->op1, var_addr, var_info, NULL, 6837 (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 6838 op1_data_range, 6839 0, var_addr, var_def_info, var_info, may_throw)) { 6840 return 0; 6841 } 6842 break; 6843 case ZEND_CONCAT: 6844 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, 6845 may_throw)) { 6846 return 0; 6847 } 6848 break; 6849 default: 6850 ZEND_UNREACHABLE(); 6851 } 6852 } 6853 6854 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 6855 binary_op_type binary_op; 6856 6857 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 6858 |.cold_code 6859 |7: 6860 } 6861 6862 | SET_EX_OPLINE opline, r0 6863 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 6864 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6865 } 6866 if (opline->op2_type == IS_UNUSED) { 6867 | xor FCARG2a, FCARG2a 6868 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 6869 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 6870 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 6871 } else { 6872 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6873 } 6874 binary_op = get_binary_op(opline->extended_value); 6875 |.if X64 6876 | LOAD_ZVAL_ADDR CARG3, op3_addr 6877 | LOAD_ADDR CARG4, binary_op 6878 |.else 6879 | sub r4, 8 6880 | PUSH_ADDR binary_op, r0 6881 | PUSH_ZVAL_ADDR op3_addr, r0 6882 |.endif 6883 | EXT_CALL zend_jit_assign_dim_op_helper, r0 6884 |.if not(X64) 6885 | add r4, 8 6886 |.endif 6887 if (!zend_jit_check_exception(Dst)) { 6888 return 0; 6889 } 6890 6891 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 6892 | jmp >9 // END 6893 |.code 6894 } 6895 } 6896 6897 |9: 6898 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 6899 6900 return 1; 6901} 6902 6903static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) 6904{ 6905 zend_jit_addr op1_addr, op2_addr; 6906 6907 ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); 6908 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); 6909 6910 op1_addr = OP1_ADDR(); 6911 op2_addr = OP2_ADDR(); 6912 6913 if (op1_info & MAY_BE_REF) { 6914 binary_op_type binary_op = get_binary_op(opline->extended_value); 6915 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 6916 | IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >1 6917 | GET_Z_PTR FCARG1a, FCARG1a 6918 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 6919 | jnz >2 6920 | add FCARG1a, offsetof(zend_reference, val) 6921 |.cold_code 6922 |2: 6923 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 6924 |.if X64 6925 | LOAD_ADDR CARG3, binary_op 6926 |.else 6927 | sub r4, 12 6928 | PUSH_ADDR binary_op, r0 6929 |.endif 6930 | SET_EX_OPLINE opline, r0 6931 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) 6932 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 6933 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 6934 } else { 6935 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 6936 } 6937 |.if not(X64) 6938 | add r4, 12 6939 |.endif 6940 zend_jit_check_exception(Dst); 6941 | jmp >9 6942 |.code 6943 |1: 6944 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 6945 } 6946 6947 int result; 6948 switch (opline->extended_value) { 6949 case ZEND_ADD: 6950 case ZEND_SUB: 6951 case ZEND_MUL: 6952 case ZEND_DIV: 6953 result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); 6954 break; 6955 case ZEND_BW_OR: 6956 case ZEND_BW_AND: 6957 case ZEND_BW_XOR: 6958 case ZEND_SL: 6959 case ZEND_SR: 6960 case ZEND_MOD: 6961 result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, 6962 opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, 6963 opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, 6964 opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); 6965 break; 6966 case ZEND_CONCAT: 6967 result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); 6968 break; 6969 default: 6970 ZEND_UNREACHABLE(); 6971 } 6972 |9: 6973 return result; 6974} 6975 6976static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, 6977 zend_ssa_range *op1_range, 6978 zend_jit_addr op1_addr, 6979 zend_ssa_range *op2_range, 6980 zend_jit_addr op2_addr, 6981 zend_bool *result) 6982{ 6983 zend_long op1_min; 6984 zend_long op1_max; 6985 zend_long op2_min; 6986 zend_long op2_max; 6987 6988 if (op1_range) { 6989 op1_min = op1_range->min; 6990 op1_max = op1_range->max; 6991 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { 6992 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG); 6993 op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr)); 6994 } else { 6995 return 0; 6996 } 6997 6998 if (op2_range) { 6999 op2_min = op2_range->min; 7000 op2_max = op2_range->max; 7001 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { 7002 ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG); 7003 op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr)); 7004 } else { 7005 return 0; 7006 } 7007 7008 switch (opline->opcode) { 7009 case ZEND_IS_EQUAL: 7010 case ZEND_IS_IDENTICAL: 7011 case ZEND_CASE: 7012 case ZEND_CASE_STRICT: 7013 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { 7014 *result = 1; 7015 return 1; 7016 } else if (op1_max < op2_min || op1_min > op2_max) { 7017 *result = 0; 7018 return 1; 7019 } 7020 return 0; 7021 case ZEND_IS_NOT_EQUAL: 7022 case ZEND_IS_NOT_IDENTICAL: 7023 if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { 7024 *result = 0; 7025 return 1; 7026 } else if (op1_max < op2_min || op1_min > op2_max) { 7027 *result = 1; 7028 return 1; 7029 } 7030 return 0; 7031 case ZEND_IS_SMALLER: 7032 if (op1_max < op2_min) { 7033 *result = 1; 7034 return 1; 7035 } else if (op1_min >= op2_max) { 7036 *result = 0; 7037 return 1; 7038 } 7039 return 0; 7040 case ZEND_IS_SMALLER_OR_EQUAL: 7041 if (op1_max <= op2_min) { 7042 *result = 1; 7043 return 1; 7044 } else if (op1_min > op2_max) { 7045 *result = 0; 7046 return 1; 7047 } 7048 return 0; 7049 default: 7050 ZEND_UNREACHABLE(); 7051 } 7052 return 0; 7053} 7054 7055static int zend_jit_cmp_long_long(dasm_State **Dst, 7056 const zend_op *opline, 7057 zend_ssa_range *op1_range, 7058 zend_jit_addr op1_addr, 7059 zend_ssa_range *op2_range, 7060 zend_jit_addr op2_addr, 7061 zend_jit_addr res_addr, 7062 zend_uchar smart_branch_opcode, 7063 uint32_t target_label, 7064 uint32_t target_label2, 7065 const void *exit_addr, 7066 zend_bool skip_comparison) 7067{ 7068 zend_bool swap = 0; 7069 zend_bool result; 7070 7071 if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { 7072 if (!smart_branch_opcode || 7073 smart_branch_opcode == ZEND_JMPZ_EX || 7074 smart_branch_opcode == ZEND_JMPNZ_EX) { 7075 | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE) 7076 } 7077 if (smart_branch_opcode && !exit_addr) { 7078 if (smart_branch_opcode == ZEND_JMPZ || 7079 smart_branch_opcode == ZEND_JMPZ_EX) { 7080 if (!result) { 7081 | jmp => target_label 7082 } 7083 } else if (smart_branch_opcode == ZEND_JMPNZ || 7084 smart_branch_opcode == ZEND_JMPNZ_EX) { 7085 if (result) { 7086 | jmp => target_label 7087 } 7088 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7089 if (!result) { 7090 | jmp => target_label 7091 } else { 7092 | jmp => target_label2 7093 } 7094 } else { 7095 ZEND_UNREACHABLE(); 7096 } 7097 } 7098 return 1; 7099 } 7100 7101 if (skip_comparison) { 7102 if (Z_MODE(op1_addr) != IS_REG && 7103 (Z_MODE(op2_addr) == IS_REG || 7104 (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { 7105 swap = 1; 7106 } 7107 } else if (Z_MODE(op1_addr) == IS_REG) { 7108 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 7109 | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) 7110 } else { 7111 | LONG_OP cmp, Z_REG(op1_addr), op2_addr, r0 7112 } 7113 } else if (Z_MODE(op2_addr) == IS_REG) { 7114 if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { 7115 | test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr)) 7116 } else { 7117 | LONG_OP cmp, Z_REG(op2_addr), op1_addr, r0 7118 } 7119 swap = 1; 7120 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { 7121 | LONG_OP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)) 7122 swap = 1; 7123 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { 7124 | LONG_OP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)) 7125 } else { 7126 | GET_ZVAL_LVAL ZREG_R0, op1_addr 7127 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { 7128 | test r0, r0 7129 } else { 7130 | LONG_OP cmp, ZREG_R0, op2_addr, r0 7131 } 7132 } 7133 7134 if (smart_branch_opcode) { 7135 if (smart_branch_opcode == ZEND_JMPZ_EX || 7136 smart_branch_opcode == ZEND_JMPNZ_EX) { 7137 7138 switch (opline->opcode) { 7139 case ZEND_IS_EQUAL: 7140 case ZEND_IS_IDENTICAL: 7141 case ZEND_CASE: 7142 case ZEND_CASE_STRICT: 7143 | sete al 7144 break; 7145 case ZEND_IS_NOT_EQUAL: 7146 case ZEND_IS_NOT_IDENTICAL: 7147 | setne al 7148 break; 7149 case ZEND_IS_SMALLER: 7150 if (swap) { 7151 | setg al 7152 } else { 7153 | setl al 7154 } 7155 break; 7156 case ZEND_IS_SMALLER_OR_EQUAL: 7157 if (swap) { 7158 | setge al 7159 } else { 7160 | setle al 7161 } 7162 break; 7163 default: 7164 ZEND_UNREACHABLE(); 7165 } 7166 | movzx eax, al 7167 | lea eax, [eax + 2] 7168 | SET_ZVAL_TYPE_INFO res_addr, eax 7169 } 7170 if (smart_branch_opcode == ZEND_JMPZ || 7171 smart_branch_opcode == ZEND_JMPZ_EX) { 7172 switch (opline->opcode) { 7173 case ZEND_IS_EQUAL: 7174 case ZEND_IS_IDENTICAL: 7175 case ZEND_CASE: 7176 case ZEND_CASE_STRICT: 7177 if (exit_addr) { 7178 | jne &exit_addr 7179 } else { 7180 | jne => target_label 7181 } 7182 break; 7183 case ZEND_IS_NOT_EQUAL: 7184 if (exit_addr) { 7185 | je &exit_addr 7186 } else { 7187 | je => target_label 7188 } 7189 break; 7190 case ZEND_IS_NOT_IDENTICAL: 7191 if (exit_addr) { 7192 | jne &exit_addr 7193 } else { 7194 | je => target_label 7195 } 7196 break; 7197 case ZEND_IS_SMALLER: 7198 if (swap) { 7199 if (exit_addr) { 7200 | jle &exit_addr 7201 } else { 7202 | jle => target_label 7203 } 7204 } else { 7205 if (exit_addr) { 7206 | jge &exit_addr 7207 } else { 7208 | jge => target_label 7209 } 7210 } 7211 break; 7212 case ZEND_IS_SMALLER_OR_EQUAL: 7213 if (swap) { 7214 if (exit_addr) { 7215 | jl &exit_addr 7216 } else { 7217 | jl => target_label 7218 } 7219 } else { 7220 if (exit_addr) { 7221 | jg &exit_addr 7222 } else { 7223 | jg => target_label 7224 } 7225 } 7226 break; 7227 default: 7228 ZEND_UNREACHABLE(); 7229 } 7230 } else if (smart_branch_opcode == ZEND_JMPNZ || 7231 smart_branch_opcode == ZEND_JMPNZ_EX) { 7232 switch (opline->opcode) { 7233 case ZEND_IS_EQUAL: 7234 case ZEND_IS_IDENTICAL: 7235 case ZEND_CASE: 7236 case ZEND_CASE_STRICT: 7237 if (exit_addr) { 7238 | je &exit_addr 7239 } else { 7240 | je => target_label 7241 } 7242 break; 7243 case ZEND_IS_NOT_EQUAL: 7244 if (exit_addr) { 7245 | jne &exit_addr 7246 } else { 7247 | jne => target_label 7248 } 7249 break; 7250 case ZEND_IS_NOT_IDENTICAL: 7251 if (exit_addr) { 7252 | je &exit_addr 7253 } else { 7254 | jne => target_label 7255 } 7256 break; 7257 case ZEND_IS_SMALLER: 7258 if (swap) { 7259 if (exit_addr) { 7260 | jg &exit_addr 7261 } else { 7262 | jg => target_label 7263 } 7264 } else { 7265 if (exit_addr) { 7266 | jl &exit_addr 7267 } else { 7268 | jl => target_label 7269 } 7270 } 7271 break; 7272 case ZEND_IS_SMALLER_OR_EQUAL: 7273 if (swap) { 7274 if (exit_addr) { 7275 | jge &exit_addr 7276 } else { 7277 | jge => target_label 7278 } 7279 } else { 7280 if (exit_addr) { 7281 | jle &exit_addr 7282 } else { 7283 | jle => target_label 7284 } 7285 } 7286 break; 7287 default: 7288 ZEND_UNREACHABLE(); 7289 } 7290 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7291 switch (opline->opcode) { 7292 case ZEND_IS_EQUAL: 7293 case ZEND_IS_IDENTICAL: 7294 case ZEND_CASE: 7295 case ZEND_CASE_STRICT: 7296 | jne => target_label 7297 break; 7298 case ZEND_IS_NOT_EQUAL: 7299 case ZEND_IS_NOT_IDENTICAL: 7300 | je => target_label 7301 break; 7302 case ZEND_IS_SMALLER: 7303 if (swap) { 7304 | jle => target_label 7305 } else { 7306 | jge => target_label 7307 } 7308 break; 7309 case ZEND_IS_SMALLER_OR_EQUAL: 7310 if (swap) { 7311 | jl => target_label 7312 } else { 7313 | jg => target_label 7314 } 7315 break; 7316 default: 7317 ZEND_UNREACHABLE(); 7318 } 7319 | jmp => target_label2 7320 } else { 7321 ZEND_UNREACHABLE(); 7322 } 7323 } else { 7324 switch (opline->opcode) { 7325 case ZEND_IS_EQUAL: 7326 case ZEND_IS_IDENTICAL: 7327 case ZEND_CASE: 7328 case ZEND_CASE_STRICT: 7329 | sete al 7330 break; 7331 case ZEND_IS_NOT_EQUAL: 7332 case ZEND_IS_NOT_IDENTICAL: 7333 | setne al 7334 break; 7335 case ZEND_IS_SMALLER: 7336 if (swap) { 7337 | setg al 7338 } else { 7339 | setl al 7340 } 7341 break; 7342 case ZEND_IS_SMALLER_OR_EQUAL: 7343 if (swap) { 7344 | setge al 7345 } else { 7346 | setle al 7347 } 7348 break; 7349 default: 7350 ZEND_UNREACHABLE(); 7351 } 7352 | movzx eax, al 7353 | add eax, 2 7354 | SET_ZVAL_TYPE_INFO res_addr, eax 7355 } 7356 7357 return 1; 7358} 7359 7360static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7361{ 7362 if (smart_branch_opcode) { 7363 if (smart_branch_opcode == ZEND_JMPZ) { 7364 switch (opline->opcode) { 7365 case ZEND_IS_EQUAL: 7366 case ZEND_IS_IDENTICAL: 7367 case ZEND_CASE: 7368 case ZEND_CASE_STRICT: 7369 if (exit_addr) { 7370 | jne &exit_addr 7371 | jp &exit_addr 7372 } else { 7373 | jne => target_label 7374 | jp => target_label 7375 } 7376 break; 7377 case ZEND_IS_NOT_EQUAL: 7378 | jp >1 7379 if (exit_addr) { 7380 | je &exit_addr 7381 } else { 7382 | je => target_label 7383 } 7384 |1: 7385 break; 7386 case ZEND_IS_NOT_IDENTICAL: 7387 if (exit_addr) { 7388 | jne &exit_addr 7389 | jp &exit_addr 7390 } else { 7391 | jp >1 7392 | je => target_label 7393 |1: 7394 } 7395 break; 7396 case ZEND_IS_SMALLER: 7397 if (swap) { 7398 if (exit_addr) { 7399 | jbe &exit_addr 7400 } else { 7401 | jbe => target_label 7402 } 7403 } else { 7404 if (exit_addr) { 7405 | jae &exit_addr 7406 | jp &exit_addr 7407 } else { 7408 | jae => target_label 7409 | jp => target_label 7410 } 7411 } 7412 break; 7413 case ZEND_IS_SMALLER_OR_EQUAL: 7414 if (swap) { 7415 if (exit_addr) { 7416 | jb &exit_addr 7417 } else { 7418 | jb => target_label 7419 } 7420 } else { 7421 if (exit_addr) { 7422 | ja &exit_addr 7423 | jp &exit_addr 7424 } else { 7425 | ja => target_label 7426 | jp => target_label 7427 } 7428 } 7429 break; 7430 default: 7431 ZEND_UNREACHABLE(); 7432 } 7433 } else if (smart_branch_opcode == ZEND_JMPNZ) { 7434 switch (opline->opcode) { 7435 case ZEND_IS_EQUAL: 7436 case ZEND_IS_IDENTICAL: 7437 case ZEND_CASE: 7438 case ZEND_CASE_STRICT: 7439 | jp >1 7440 if (exit_addr) { 7441 | je &exit_addr 7442 } else { 7443 | je => target_label 7444 } 7445 |1: 7446 break; 7447 case ZEND_IS_NOT_EQUAL: 7448 if (exit_addr) { 7449 | jne &exit_addr 7450 | jp &exit_addr 7451 } else { 7452 | jne => target_label 7453 | jp => target_label 7454 } 7455 break; 7456 case ZEND_IS_NOT_IDENTICAL: 7457 if (exit_addr) { 7458 | jp >1 7459 | je &exit_addr 7460 |1: 7461 } else { 7462 | jne => target_label 7463 | jp => target_label 7464 } 7465 break; 7466 case ZEND_IS_SMALLER: 7467 if (swap) { 7468 if (exit_addr) { 7469 | ja &exit_addr 7470 } else { 7471 | ja => target_label 7472 } 7473 } else { 7474 | jp >1 7475 if (exit_addr) { 7476 | jb &exit_addr 7477 } else { 7478 | jb => target_label 7479 } 7480 |1: 7481 } 7482 break; 7483 case ZEND_IS_SMALLER_OR_EQUAL: 7484 if (swap) { 7485 if (exit_addr) { 7486 | jae &exit_addr 7487 } else { 7488 | jae => target_label 7489 } 7490 } else { 7491 | jp >1 7492 if (exit_addr) { 7493 | jbe &exit_addr 7494 } else { 7495 | jbe => target_label 7496 } 7497 |1: 7498 } 7499 break; 7500 default: 7501 ZEND_UNREACHABLE(); 7502 } 7503 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7504 switch (opline->opcode) { 7505 case ZEND_IS_EQUAL: 7506 case ZEND_IS_IDENTICAL: 7507 case ZEND_CASE: 7508 case ZEND_CASE_STRICT: 7509 | jne => target_label 7510 | jp => target_label 7511 break; 7512 case ZEND_IS_NOT_EQUAL: 7513 case ZEND_IS_NOT_IDENTICAL: 7514 | jp => target_label2 7515 | je => target_label 7516 break; 7517 case ZEND_IS_SMALLER: 7518 if (swap) { 7519 | jbe => target_label 7520 } else { 7521 | jae => target_label 7522 | jp => target_label 7523 } 7524 break; 7525 case ZEND_IS_SMALLER_OR_EQUAL: 7526 if (swap) { 7527 | jb => target_label 7528 } else { 7529 | ja => target_label 7530 | jp => target_label 7531 } 7532 break; 7533 default: 7534 ZEND_UNREACHABLE(); 7535 } 7536 | jmp => target_label2 7537 } else if (smart_branch_opcode == ZEND_JMPZ_EX) { 7538 switch (opline->opcode) { 7539 case ZEND_IS_EQUAL: 7540 case ZEND_IS_IDENTICAL: 7541 case ZEND_CASE: 7542 case ZEND_CASE_STRICT: 7543 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7544 | jne => target_label 7545 | jp => target_label 7546 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7547 break; 7548 case ZEND_IS_NOT_EQUAL: 7549 case ZEND_IS_NOT_IDENTICAL: 7550 | jp >1 7551 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7552 | je => target_label 7553 |1: 7554 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7555 break; 7556 case ZEND_IS_SMALLER: 7557 if (swap) { 7558 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7559 | jbe => target_label 7560 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7561 } else { 7562 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7563 | jae => target_label 7564 | jp => target_label 7565 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7566 } 7567 break; 7568 case ZEND_IS_SMALLER_OR_EQUAL: 7569 if (swap) { 7570 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7571 | jb => target_label 7572 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7573 } else { 7574 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7575 | ja => target_label 7576 | jp => target_label 7577 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7578 } 7579 break; 7580 default: 7581 ZEND_UNREACHABLE(); 7582 } 7583 } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { 7584 switch (opline->opcode) { 7585 case ZEND_IS_EQUAL: 7586 case ZEND_IS_IDENTICAL: 7587 case ZEND_CASE: 7588 case ZEND_CASE_STRICT: 7589 | jp >1 7590 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7591 | je => target_label 7592 |1: 7593 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7594 break; 7595 case ZEND_IS_NOT_EQUAL: 7596 case ZEND_IS_NOT_IDENTICAL: 7597 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7598 | jne => target_label 7599 | jp => target_label 7600 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7601 break; 7602 case ZEND_IS_SMALLER: 7603 if (swap) { 7604 | seta al 7605 | movzx eax, al 7606 | lea eax, [eax + 2] 7607 | SET_ZVAL_TYPE_INFO res_addr, eax 7608 | ja => target_label 7609 } else { 7610 | jp >1 7611 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7612 | jb => target_label 7613 |1: 7614 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7615 } 7616 break; 7617 case ZEND_IS_SMALLER_OR_EQUAL: 7618 if (swap) { 7619 | setae al 7620 | movzx eax, al 7621 | lea eax, [eax + 2] 7622 | SET_ZVAL_TYPE_INFO res_addr, eax 7623 | jae => target_label 7624 } else { 7625 | jp >1 7626 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 7627 | jbe => target_label 7628 |1: 7629 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 7630 } 7631 break; 7632 default: 7633 ZEND_UNREACHABLE(); 7634 } 7635 } else { 7636 ZEND_UNREACHABLE(); 7637 } 7638 } else { 7639 switch (opline->opcode) { 7640 case ZEND_IS_EQUAL: 7641 case ZEND_IS_IDENTICAL: 7642 case ZEND_CASE: 7643 case ZEND_CASE_STRICT: 7644 | jp >1 7645 | mov eax, IS_TRUE 7646 | je >2 7647 |1: 7648 | mov eax, IS_FALSE 7649 |2: 7650 break; 7651 case ZEND_IS_NOT_EQUAL: 7652 case ZEND_IS_NOT_IDENTICAL: 7653 | jp >1 7654 | mov eax, IS_FALSE 7655 | je >2 7656 |1: 7657 | mov eax, IS_TRUE 7658 |2: 7659 break; 7660 case ZEND_IS_SMALLER: 7661 if (swap) { 7662 | seta al 7663 | movzx eax, al 7664 | add eax, 2 7665 } else { 7666 | jp >1 7667 | mov eax, IS_TRUE 7668 | jb >2 7669 |1: 7670 | mov eax, IS_FALSE 7671 |2: 7672 } 7673 break; 7674 case ZEND_IS_SMALLER_OR_EQUAL: 7675 if (swap) { 7676 | setae al 7677 | movzx eax, al 7678 | add eax, 2 7679 } else { 7680 | jp >1 7681 | mov eax, IS_TRUE 7682 | jbe >2 7683 |1: 7684 | mov eax, IS_FALSE 7685 |2: 7686 } 7687 break; 7688 default: 7689 ZEND_UNREACHABLE(); 7690 } 7691 | SET_ZVAL_TYPE_INFO res_addr, eax 7692 } 7693 7694 return 1; 7695} 7696 7697static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7698{ 7699 zend_reg tmp_reg = ZREG_XMM0; 7700 7701 | SSE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_R0 7702 | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr 7703 7704 return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); 7705} 7706 7707static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7708{ 7709 zend_reg tmp_reg = ZREG_XMM0; 7710 7711 | SSE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_R0 7712 | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op1_addr 7713 7714 return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); 7715} 7716 7717static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7718{ 7719 zend_bool swap = 0; 7720 7721 if (Z_MODE(op1_addr) == IS_REG) { 7722 | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op1_addr), op2_addr 7723 } else if (Z_MODE(op2_addr) == IS_REG) { 7724 | SSE_AVX_OP ucomisd, vucomisd, Z_REG(op2_addr), op1_addr 7725 swap = 1; 7726 } else { 7727 zend_reg tmp_reg = ZREG_XMM0; 7728 7729 | SSE_GET_ZVAL_DVAL tmp_reg, op1_addr 7730 | SSE_AVX_OP ucomisd, vucomisd, tmp_reg, op2_addr 7731 } 7732 7733 return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); 7734} 7735 7736static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 7737{ 7738 | LONG_OP_WITH_CONST cmp, res_addr, Z_L(0) 7739 if (smart_branch_opcode) { 7740 if (smart_branch_opcode == ZEND_JMPZ_EX || 7741 smart_branch_opcode == ZEND_JMPNZ_EX) { 7742 switch (opline->opcode) { 7743 case ZEND_IS_EQUAL: 7744 case ZEND_CASE: 7745 | sete al 7746 break; 7747 case ZEND_IS_NOT_EQUAL: 7748 | setne al 7749 break; 7750 case ZEND_IS_SMALLER: 7751 | setl al 7752 break; 7753 case ZEND_IS_SMALLER_OR_EQUAL: 7754 | setle al 7755 break; 7756 default: 7757 ZEND_UNREACHABLE(); 7758 } 7759 | movzx eax, al 7760 | lea eax, [eax + 2] 7761 | SET_ZVAL_TYPE_INFO res_addr, eax 7762 } 7763 if (smart_branch_opcode == ZEND_JMPZ || 7764 smart_branch_opcode == ZEND_JMPZ_EX) { 7765 switch (opline->opcode) { 7766 case ZEND_IS_EQUAL: 7767 case ZEND_CASE: 7768 if (exit_addr) { 7769 | jne &exit_addr 7770 } else { 7771 | jne => target_label 7772 } 7773 break; 7774 case ZEND_IS_NOT_EQUAL: 7775 if (exit_addr) { 7776 | je &exit_addr 7777 } else { 7778 | je => target_label 7779 } 7780 break; 7781 case ZEND_IS_SMALLER: 7782 if (exit_addr) { 7783 | jge &exit_addr 7784 } else { 7785 | jge => target_label 7786 } 7787 break; 7788 case ZEND_IS_SMALLER_OR_EQUAL: 7789 if (exit_addr) { 7790 | jg &exit_addr 7791 } else { 7792 | jg => target_label 7793 } 7794 break; 7795 default: 7796 ZEND_UNREACHABLE(); 7797 } 7798 } else if (smart_branch_opcode == ZEND_JMPNZ || 7799 smart_branch_opcode == ZEND_JMPNZ_EX) { 7800 switch (opline->opcode) { 7801 case ZEND_IS_EQUAL: 7802 case ZEND_CASE: 7803 if (exit_addr) { 7804 | je &exit_addr 7805 } else { 7806 | je => target_label 7807 } 7808 break; 7809 case ZEND_IS_NOT_EQUAL: 7810 if (exit_addr) { 7811 | jne &exit_addr 7812 } else { 7813 | jne => target_label 7814 } 7815 break; 7816 case ZEND_IS_SMALLER: 7817 if (exit_addr) { 7818 | jl &exit_addr 7819 } else { 7820 | jl => target_label 7821 } 7822 break; 7823 case ZEND_IS_SMALLER_OR_EQUAL: 7824 if (exit_addr) { 7825 | jle &exit_addr 7826 } else { 7827 | jle => target_label 7828 } 7829 break; 7830 default: 7831 ZEND_UNREACHABLE(); 7832 } 7833 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 7834 switch (opline->opcode) { 7835 case ZEND_IS_EQUAL: 7836 case ZEND_CASE: 7837 | jne => target_label 7838 break; 7839 case ZEND_IS_NOT_EQUAL: 7840 | je => target_label 7841 break; 7842 case ZEND_IS_SMALLER: 7843 | jge => target_label 7844 break; 7845 case ZEND_IS_SMALLER_OR_EQUAL: 7846 | jg => target_label 7847 break; 7848 default: 7849 ZEND_UNREACHABLE(); 7850 } 7851 | jmp => target_label2 7852 } else { 7853 ZEND_UNREACHABLE(); 7854 } 7855 } else { 7856 switch (opline->opcode) { 7857 case ZEND_IS_EQUAL: 7858 case ZEND_CASE: 7859 | sete al 7860 break; 7861 case ZEND_IS_NOT_EQUAL: 7862 | setne al 7863 break; 7864 case ZEND_IS_SMALLER: 7865 | setl al 7866 break; 7867 case ZEND_IS_SMALLER_OR_EQUAL: 7868 | setle al 7869 break; 7870 default: 7871 ZEND_UNREACHABLE(); 7872 } 7873 | movzx eax, al 7874 | add eax, 2 7875 | SET_ZVAL_TYPE_INFO res_addr, eax 7876 } 7877 7878 return 1; 7879} 7880 7881static int zend_jit_cmp(dasm_State **Dst, 7882 const zend_op *opline, 7883 uint32_t op1_info, 7884 zend_ssa_range *op1_range, 7885 zend_jit_addr op1_addr, 7886 uint32_t op2_info, 7887 zend_ssa_range *op2_range, 7888 zend_jit_addr op2_addr, 7889 zend_jit_addr res_addr, 7890 int may_throw, 7891 zend_uchar smart_branch_opcode, 7892 uint32_t target_label, 7893 uint32_t target_label2, 7894 const void *exit_addr, 7895 zend_bool skip_comparison) 7896{ 7897 zend_bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); 7898 zend_bool has_slow; 7899 7900 has_slow = 7901 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7902 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && 7903 ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 7904 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); 7905 7906 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 7907 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 7908 if (op1_info & MAY_BE_DOUBLE) { 7909 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4 7910 } else { 7911 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 7912 } 7913 } 7914 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { 7915 if (op2_info & MAY_BE_DOUBLE) { 7916 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3 7917 |.cold_code 7918 |3: 7919 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7920 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7921 } 7922 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7923 return 0; 7924 } 7925 | jmp >6 7926 |.code 7927 } else { 7928 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7929 } 7930 } 7931 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 7932 return 0; 7933 } 7934 if (op1_info & MAY_BE_DOUBLE) { 7935 |.cold_code 7936 |4: 7937 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7938 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7939 } 7940 if (op2_info & MAY_BE_DOUBLE) { 7941 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7942 if (!same_ops) { 7943 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5 7944 } else { 7945 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7946 } 7947 } 7948 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7949 return 0; 7950 } 7951 | jmp >6 7952 } 7953 if (!same_ops) { 7954 |5: 7955 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { 7956 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7957 } 7958 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7959 return 0; 7960 } 7961 | jmp >6 7962 } 7963 |.code 7964 } 7965 } else if ((op1_info & MAY_BE_DOUBLE) && 7966 !(op1_info & MAY_BE_LONG) && 7967 (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 7968 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 7969 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 7970 } 7971 if (op2_info & MAY_BE_DOUBLE) { 7972 if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 7973 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7974 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3 7975 } else { 7976 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 7977 } 7978 } 7979 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7980 return 0; 7981 } 7982 } 7983 if (!same_ops && (op2_info & MAY_BE_LONG)) { 7984 if (op2_info & MAY_BE_DOUBLE) { 7985 |.cold_code 7986 } 7987 |3: 7988 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 7989 | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9 7990 } 7991 if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 7992 return 0; 7993 } 7994 if (op2_info & MAY_BE_DOUBLE) { 7995 | jmp >6 7996 |.code 7997 } 7998 } 7999 } else if ((op2_info & MAY_BE_DOUBLE) && 8000 !(op2_info & MAY_BE_LONG) && 8001 (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { 8002 if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { 8003 | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9 8004 } 8005 if (op1_info & MAY_BE_DOUBLE) { 8006 if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { 8007 if (!same_ops && (op1_info & MAY_BE_LONG)) { 8008 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3 8009 } else { 8010 | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9 8011 } 8012 } 8013 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8014 return 0; 8015 } 8016 } 8017 if (!same_ops && (op1_info & MAY_BE_LONG)) { 8018 if (op1_info & MAY_BE_DOUBLE) { 8019 |.cold_code 8020 } 8021 |3: 8022 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { 8023 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9 8024 } 8025 if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8026 return 0; 8027 } 8028 if (op1_info & MAY_BE_DOUBLE) { 8029 | jmp >6 8030 |.code 8031 } 8032 } 8033 } 8034 8035 if (has_slow || 8036 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || 8037 (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 8038 if (has_slow) { 8039 |.cold_code 8040 |9: 8041 } 8042 | SET_EX_OPLINE opline, r0 8043 if (Z_MODE(op1_addr) == IS_REG) { 8044 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8045 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 8046 return 0; 8047 } 8048 op1_addr = real_addr; 8049 } 8050 if (Z_MODE(op2_addr) == IS_REG) { 8051 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 8052 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 8053 return 0; 8054 } 8055 op2_addr = real_addr; 8056 } 8057 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 8058 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { 8059 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >1 8060 | mov FCARG1a, opline->op1.var 8061 | EXT_CALL zend_jit_undefined_op_helper, r0 8062 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8063 |1: 8064 } 8065 if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { 8066 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 8067 | mov T1, FCARG2a // save 8068 | mov FCARG1a, opline->op2.var 8069 | EXT_CALL zend_jit_undefined_op_helper, r0 8070 | mov FCARG2a, T1 // restore 8071 |.if X64 8072 | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval 8073 |.else 8074 | sub r4, 12 8075 | PUSH_ADDR_ZTS executor_globals, uninitialized_zval, r0 8076 |.endif 8077 | jmp >2 8078 |1: 8079 |.if X64 8080 | LOAD_ZVAL_ADDR CARG3, op2_addr 8081 |.else 8082 | sub r4, 12 8083 | PUSH_ZVAL_ADDR op2_addr, r0 8084 |.endif 8085 |2: 8086 } else { 8087 |.if X64 8088 | LOAD_ZVAL_ADDR CARG3, op2_addr 8089 |.else 8090 | sub r4, 12 8091 | PUSH_ZVAL_ADDR op2_addr, r0 8092 |.endif 8093 } 8094 | LOAD_ZVAL_ADDR FCARG1a, res_addr 8095 | EXT_CALL compare_function, r0 8096 |.if not(X64) 8097 | add r4, 12 8098 |.endif 8099 if (opline->opcode != ZEND_CASE) { 8100 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 8101 } 8102 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 8103 if (may_throw) { 8104 zend_jit_check_exception_undef_result(Dst, opline); 8105 } 8106 if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8107 return 0; 8108 } 8109 if (has_slow) { 8110 | jmp >6 8111 |.code 8112 } 8113 } 8114 8115 |6: 8116 8117 return 1; 8118} 8119 8120static int zend_jit_identical(dasm_State **Dst, 8121 const zend_op *opline, 8122 uint32_t op1_info, 8123 zend_ssa_range *op1_range, 8124 zend_jit_addr op1_addr, 8125 uint32_t op2_info, 8126 zend_ssa_range *op2_range, 8127 zend_jit_addr op2_addr, 8128 zend_jit_addr res_addr, 8129 int may_throw, 8130 zend_uchar smart_branch_opcode, 8131 uint32_t target_label, 8132 uint32_t target_label2, 8133 const void *exit_addr, 8134 zend_bool skip_comparison) 8135{ 8136 uint32_t identical_label = (uint32_t)-1; 8137 uint32_t not_identical_label = (uint32_t)-1; 8138 8139 if (smart_branch_opcode && !exit_addr) { 8140 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8141 if (smart_branch_opcode == ZEND_JMPZ) { 8142 not_identical_label = target_label; 8143 } else if (smart_branch_opcode == ZEND_JMPNZ) { 8144 identical_label = target_label; 8145 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 8146 not_identical_label = target_label; 8147 identical_label = target_label2; 8148 } else { 8149 ZEND_UNREACHABLE(); 8150 } 8151 } else { 8152 if (smart_branch_opcode == ZEND_JMPZ) { 8153 identical_label = target_label; 8154 } else if (smart_branch_opcode == ZEND_JMPNZ) { 8155 not_identical_label = target_label; 8156 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 8157 identical_label = target_label; 8158 not_identical_label = target_label2; 8159 } else { 8160 ZEND_UNREACHABLE(); 8161 } 8162 } 8163 } 8164 8165 if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && 8166 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { 8167 if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { 8168 return 0; 8169 } 8170 return 1; 8171 } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && 8172 (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { 8173 if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { 8174 return 0; 8175 } 8176 return 1; 8177 } 8178 8179 if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { 8180 op1_info |= MAY_BE_NULL; 8181 op2_info |= MAY_BE_NULL; 8182 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8183 | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 8184 |.cold_code 8185 |1: 8186 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8187 | SET_EX_OPLINE opline, r0 8188 | mov FCARG1d, opline->op1.var 8189 | EXT_CALL zend_jit_undefined_op_helper, r0 8190 if (may_throw) { 8191 zend_jit_check_exception_undef_result(Dst, opline); 8192 } 8193 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8194 | jmp >1 8195 |.code 8196 |1: 8197 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8198 | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 8199 |.cold_code 8200 |1: 8201 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8202 | SET_EX_OPLINE opline, r0 8203 | mov aword T1, FCARG1a // save 8204 | mov FCARG1d, opline->op2.var 8205 | EXT_CALL zend_jit_undefined_op_helper, r0 8206 if (may_throw) { 8207 zend_jit_check_exception_undef_result(Dst, opline); 8208 } 8209 | mov FCARG1a, aword T1 // restore 8210 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8211 | jmp >1 8212 |.code 8213 |1: 8214 } else if (op1_info & MAY_BE_UNDEF) { 8215 op1_info |= MAY_BE_NULL; 8216 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8217 | IF_Z_TYPE FCARG1a, IS_UNDEF, >1 8218 |.cold_code 8219 |1: 8220 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8221 | SET_EX_OPLINE opline, r0 8222 | mov FCARG1d, opline->op1.var 8223 | EXT_CALL zend_jit_undefined_op_helper, r0 8224 if (may_throw) { 8225 zend_jit_check_exception_undef_result(Dst, opline); 8226 } 8227 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 8228 | jmp >1 8229 |.code 8230 |1: 8231 if (opline->op2_type != IS_CONST) { 8232 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8233 } 8234 } else if (op2_info & MAY_BE_UNDEF) { 8235 op2_info |= MAY_BE_NULL; 8236 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8237 | IF_Z_TYPE FCARG2a, IS_UNDEF, >1 8238 |.cold_code 8239 |1: 8240 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 8241 | SET_EX_OPLINE opline, r0 8242 | mov FCARG1d, opline->op2.var 8243 | EXT_CALL zend_jit_undefined_op_helper, r0 8244 if (may_throw) { 8245 zend_jit_check_exception_undef_result(Dst, opline); 8246 } 8247 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 8248 | jmp >1 8249 |.code 8250 |1: 8251 if (opline->op1_type != IS_CONST) { 8252 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8253 } 8254 } else { 8255 if (opline->op1_type != IS_CONST) { 8256 if (Z_MODE(op1_addr) == IS_REG) { 8257 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8258 if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { 8259 return 0; 8260 } 8261 op1_addr = real_addr; 8262 } 8263 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8264 } 8265 if (opline->op2_type != IS_CONST) { 8266 if (Z_MODE(op2_addr) == IS_REG) { 8267 zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 8268 if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { 8269 return 0; 8270 } 8271 op2_addr = real_addr; 8272 } 8273 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8274 } 8275 } 8276 if (opline->op1_type & (IS_CV|IS_VAR)) { 8277 | ZVAL_DEREF FCARG1a, op1_info 8278 } 8279 if (opline->op2_type & (IS_CV|IS_VAR)) { 8280 | ZVAL_DEREF FCARG2a, op2_info 8281 } 8282 8283 if ((op1_info & op2_info & MAY_BE_ANY) == 0) { 8284 if ((opline->opcode != ZEND_CASE_STRICT && 8285 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8286 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8287 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8288 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8289 | SET_EX_OPLINE opline, r0 8290 if (opline->opcode != ZEND_CASE_STRICT) { 8291 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8292 } 8293 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8294 } 8295 if (smart_branch_opcode) { 8296 zend_jit_check_exception_undef_result(Dst, opline); 8297 if (exit_addr) { 8298 if (smart_branch_opcode == ZEND_JMPZ) { 8299 | jmp &exit_addr 8300 } 8301 } else if (not_identical_label != (uint32_t)-1) { 8302 | jmp =>not_identical_label 8303 } 8304 } else { 8305 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) 8306 zend_jit_check_exception(Dst); 8307 } 8308 } else if (has_concrete_type(op1_info) && 8309 has_concrete_type(op2_info) && 8310 concrete_type(op1_info) == concrete_type(op2_info) && 8311 concrete_type(op1_info) <= IS_TRUE) { 8312 if (smart_branch_opcode) { 8313 if (exit_addr) { 8314 if (smart_branch_opcode == ZEND_JMPNZ) { 8315 | jmp &exit_addr 8316 } 8317 } else if (identical_label != (uint32_t)-1) { 8318 | jmp =>identical_label 8319 } 8320 } else { 8321 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) 8322 } 8323 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { 8324 if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { 8325 if (smart_branch_opcode) { 8326 if (exit_addr) { 8327 if (smart_branch_opcode == ZEND_JMPNZ) { 8328 | jmp &exit_addr 8329 } 8330 } else if (identical_label != (uint32_t)-1) { 8331 | jmp =>identical_label 8332 } 8333 } else { 8334 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE) 8335 } 8336 } else { 8337 if (smart_branch_opcode) { 8338 if (exit_addr) { 8339 if (smart_branch_opcode == ZEND_JMPZ) { 8340 | jmp &exit_addr 8341 } 8342 } else if (not_identical_label != (uint32_t)-1) { 8343 | jmp =>not_identical_label 8344 } 8345 } else { 8346 | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE) 8347 } 8348 } 8349 } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { 8350 zval *val = Z_ZV(op1_addr); 8351 8352 | cmp byte [FCARG2a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) 8353 if (smart_branch_opcode) { 8354 if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { 8355 | jne >8 8356 | SET_EX_OPLINE opline, r0 8357 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8358 zend_jit_check_exception_undef_result(Dst, opline); 8359 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8360 | jmp &exit_addr 8361 } else if (identical_label != (uint32_t)-1) { 8362 | jmp =>identical_label 8363 } else { 8364 | jmp >9 8365 } 8366 |8: 8367 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8368 | je &exit_addr 8369 } else if (identical_label != (uint32_t)-1) { 8370 | je =>identical_label 8371 } else { 8372 | je >9 8373 } 8374 } else { 8375 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8376 | sete al 8377 } else { 8378 | setne al 8379 } 8380 | movzx eax, al 8381 | lea eax, [eax + 2] 8382 | SET_ZVAL_TYPE_INFO res_addr, eax 8383 } 8384 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8385 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 8386 | SET_EX_OPLINE opline, r0 8387 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8388 zend_jit_check_exception_undef_result(Dst, opline); 8389 } 8390 if (exit_addr) { 8391 if (smart_branch_opcode == ZEND_JMPZ) { 8392 | jmp &exit_addr 8393 } 8394 } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { 8395 | jmp =>not_identical_label 8396 } 8397 } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { 8398 zval *val = Z_ZV(op2_addr); 8399 8400 | cmp byte [FCARG1a + offsetof(zval, u1.v.type)], Z_TYPE_P(val) 8401 if (smart_branch_opcode) { 8402 if (opline->opcode != ZEND_CASE_STRICT 8403 && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { 8404 | jne >8 8405 | SET_EX_OPLINE opline, r0 8406 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8407 zend_jit_check_exception_undef_result(Dst, opline); 8408 if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8409 | jmp &exit_addr 8410 } else if (identical_label != (uint32_t)-1) { 8411 | jmp =>identical_label 8412 } else { 8413 | jmp >9 8414 } 8415 |8: 8416 } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { 8417 | je &exit_addr 8418 } else if (identical_label != (uint32_t)-1) { 8419 | je =>identical_label 8420 } else { 8421 | je >9 8422 } 8423 } else { 8424 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8425 | sete al 8426 } else { 8427 | setne al 8428 } 8429 | movzx eax, al 8430 | lea eax, [eax + 2] 8431 | SET_ZVAL_TYPE_INFO res_addr, eax 8432 } 8433 if (opline->opcode != ZEND_CASE_STRICT 8434 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8435 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 8436 | SET_EX_OPLINE opline, r0 8437 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8438 zend_jit_check_exception_undef_result(Dst, opline); 8439 } 8440 if (smart_branch_opcode) { 8441 if (exit_addr) { 8442 if (smart_branch_opcode == ZEND_JMPZ) { 8443 | jmp &exit_addr 8444 } 8445 } else if (not_identical_label != (uint32_t)-1) { 8446 | jmp =>not_identical_label 8447 } 8448 } 8449 } else { 8450 if (opline->op1_type == IS_CONST) { 8451 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8452 } 8453 if (opline->op2_type == IS_CONST) { 8454 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 8455 } 8456 | EXT_CALL zend_is_identical, r0 8457 if ((opline->opcode != ZEND_CASE_STRICT && 8458 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8459 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || 8460 ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && 8461 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { 8462 | mov aword T1, r0 // save 8463 | SET_EX_OPLINE opline, r0 8464 if (opline->opcode != ZEND_CASE_STRICT) { 8465 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 8466 } 8467 | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline 8468 zend_jit_check_exception_undef_result(Dst, opline); 8469 | mov r0, aword T1 // restore 8470 } 8471 if (smart_branch_opcode) { 8472 | test al, al 8473 if (exit_addr) { 8474 if (smart_branch_opcode == ZEND_JMPNZ) { 8475 | jnz &exit_addr 8476 } else { 8477 | jz &exit_addr 8478 } 8479 } else if (not_identical_label != (uint32_t)-1) { 8480 | jz =>not_identical_label 8481 if (identical_label != (uint32_t)-1) { 8482 | jmp =>identical_label 8483 } 8484 } else if (identical_label != (uint32_t)-1) { 8485 | jnz =>identical_label 8486 } 8487 } else { 8488 | movzx eax, al 8489 if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { 8490 | lea eax, [eax + 2] 8491 } else { 8492 | neg eax 8493 | lea eax, [eax + 3] 8494 } 8495 | SET_ZVAL_TYPE_INFO res_addr, eax 8496 } 8497 } 8498 8499 |9: 8500 if (may_throw) { 8501 zend_jit_check_exception(Dst); 8502 } 8503 return 1; 8504} 8505 8506static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr) 8507{ 8508 uint32_t true_label = -1; 8509 uint32_t false_label = -1; 8510 zend_bool set_bool = 0; 8511 zend_bool set_bool_not = 0; 8512 zend_bool set_delayed = 0; 8513 zend_bool jmp_done = 0; 8514 8515 if (branch_opcode == ZEND_BOOL) { 8516 set_bool = 1; 8517 } else if (branch_opcode == ZEND_BOOL_NOT) { 8518 set_bool = 1; 8519 set_bool_not = 1; 8520 } else if (branch_opcode == ZEND_JMPZ) { 8521 false_label = target_label; 8522 } else if (branch_opcode == ZEND_JMPNZ) { 8523 true_label = target_label; 8524 } else if (branch_opcode == ZEND_JMPZNZ) { 8525 true_label = target_label2; 8526 false_label = target_label; 8527 } else if (branch_opcode == ZEND_JMPZ_EX) { 8528 set_bool = 1; 8529 false_label = target_label; 8530 } else if (branch_opcode == ZEND_JMPNZ_EX) { 8531 set_bool = 1; 8532 true_label = target_label; 8533 } else { 8534 ZEND_UNREACHABLE(); 8535 } 8536 8537 if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { 8538 if (zend_is_true(Z_ZV(op1_addr))) { 8539 /* Always TRUE */ 8540 if (set_bool) { 8541 if (set_bool_not) { 8542 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8543 } else { 8544 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8545 } 8546 } 8547 if (true_label != (uint32_t)-1) { 8548 | jmp =>true_label; 8549 } 8550 } else { 8551 /* Always FALSE */ 8552 if (set_bool) { 8553 if (set_bool_not) { 8554 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8555 } else { 8556 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8557 } 8558 } 8559 if (false_label != (uint32_t)-1) { 8560 | jmp =>false_label; 8561 } 8562 } 8563 return 1; 8564 } 8565 8566 if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) { 8567 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8568 | ZVAL_DEREF FCARG1a, op1_info 8569 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 8570 } 8571 8572 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { 8573 if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { 8574 /* Always TRUE */ 8575 if (set_bool) { 8576 if (set_bool_not) { 8577 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8578 } else { 8579 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8580 } 8581 } 8582 if (true_label != (uint32_t)-1) { 8583 | jmp =>true_label; 8584 } 8585 } else { 8586 if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { 8587 /* Always FALSE */ 8588 if (set_bool) { 8589 if (set_bool_not) { 8590 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8591 } else { 8592 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8593 } 8594 } 8595 } else { 8596 | CMP_ZVAL_TYPE op1_addr, IS_TRUE 8597 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { 8598 if ((op1_info & MAY_BE_LONG) && 8599 !(op1_info & MAY_BE_UNDEF) && 8600 !set_bool) { 8601 if (exit_addr) { 8602 if (branch_opcode == ZEND_JMPNZ) { 8603 | jl >9 8604 } else { 8605 | jl &exit_addr 8606 } 8607 } else if (false_label != (uint32_t)-1) { 8608 | jl =>false_label 8609 } else { 8610 | jl >9 8611 } 8612 jmp_done = 1; 8613 } else { 8614 | jg >2 8615 } 8616 } 8617 if (!(op1_info & MAY_BE_TRUE)) { 8618 /* It's FALSE */ 8619 if (set_bool) { 8620 if (set_bool_not) { 8621 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8622 } else { 8623 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8624 } 8625 } 8626 } else { 8627 if (exit_addr) { 8628 if (set_bool) { 8629 | jne >1 8630 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8631 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8632 | jmp &exit_addr 8633 } else { 8634 | jmp >9 8635 } 8636 |1: 8637 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8638 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8639 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8640 | jne &exit_addr 8641 } 8642 } 8643 } else { 8644 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8645 | je &exit_addr 8646 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8647 | jne &exit_addr 8648 } else { 8649 | je >9 8650 } 8651 } 8652 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8653 if (set_bool) { 8654 | jne >1 8655 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8656 if (true_label != (uint32_t)-1) { 8657 | jmp =>true_label 8658 } else { 8659 | jmp >9 8660 } 8661 |1: 8662 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8663 } else { 8664 if (true_label != (uint32_t)-1) { 8665 | je =>true_label 8666 } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { 8667 | jne =>false_label 8668 jmp_done = 1; 8669 } else { 8670 | je >9 8671 } 8672 } 8673 } else if (set_bool) { 8674 | sete al 8675 | movzx eax, al 8676 if (set_bool_not) { 8677 | neg eax 8678 | add eax, 3 8679 } else { 8680 | add eax, 2 8681 } 8682 if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { 8683 set_delayed = 1; 8684 } else { 8685 | SET_ZVAL_TYPE_INFO res_addr, eax 8686 } 8687 } 8688 } 8689 } 8690 8691 /* It's FALSE, but may be UNDEF */ 8692 if (op1_info & MAY_BE_UNDEF) { 8693 if (op1_info & MAY_BE_ANY) { 8694 if (set_delayed) { 8695 | CMP_ZVAL_TYPE op1_addr, IS_UNDEF 8696 | SET_ZVAL_TYPE_INFO res_addr, eax 8697 | jz >1 8698 } else { 8699 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 8700 } 8701 |.cold_code 8702 |1: 8703 } 8704 | mov FCARG1d, opline->op1.var 8705 | SET_EX_OPLINE opline, r0 8706 | EXT_CALL zend_jit_undefined_op_helper, r0 8707 8708 if (may_throw) { 8709 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 8710 return 0; 8711 } 8712 } 8713 8714 if (exit_addr) { 8715 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { 8716 | jmp &exit_addr 8717 } 8718 } else if (false_label != (uint32_t)-1) { 8719 | jmp =>false_label 8720 } 8721 if (op1_info & MAY_BE_ANY) { 8722 if (exit_addr) { 8723 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8724 | jmp >9 8725 } 8726 } else if (false_label == (uint32_t)-1) { 8727 | jmp >9 8728 } 8729 |.code 8730 } 8731 } 8732 8733 if (!jmp_done) { 8734 if (exit_addr) { 8735 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8736 if (op1_info & MAY_BE_LONG) { 8737 | jmp >9 8738 } 8739 } else if (op1_info & MAY_BE_LONG) { 8740 | jmp &exit_addr 8741 } 8742 } else if (false_label != (uint32_t)-1) { 8743 | jmp =>false_label 8744 } else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { 8745 | jmp >9 8746 } 8747 } 8748 } 8749 } 8750 8751 if (op1_info & MAY_BE_LONG) { 8752 |2: 8753 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8754 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2 8755 } 8756 if (Z_MODE(op1_addr) == IS_REG) { 8757 | test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr)) 8758 } else { 8759 | LONG_OP_WITH_CONST, cmp, op1_addr, Z_L(0) 8760 } 8761 if (set_bool) { 8762 | setne al 8763 | movzx eax, al 8764 if (set_bool_not) { 8765 | neg eax 8766 | add eax, 3 8767 } else { 8768 | lea eax, [eax + 2] 8769 } 8770 | SET_ZVAL_TYPE_INFO res_addr, eax 8771 } 8772 if (exit_addr) { 8773 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8774 | jne &exit_addr 8775 } else { 8776 | je &exit_addr 8777 } 8778 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8779 if (true_label != (uint32_t)-1) { 8780 | jne =>true_label 8781 if (false_label != (uint32_t)-1) { 8782 | jmp =>false_label 8783 } 8784 } else { 8785 | je =>false_label 8786 } 8787 } 8788 } 8789 8790 if ((op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) == MAY_BE_DOUBLE) { 8791 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8792 |.cold_code 8793 } 8794 |2: 8795 if (CAN_USE_AVX()) { 8796 | vxorps xmm0, xmm0, xmm0 8797 } else { 8798 | xorps xmm0, xmm0 8799 } 8800 | SSE_AVX_OP ucomisd, vucomisd, ZREG_XMM0, op1_addr 8801 8802 if (set_bool) { 8803 if (exit_addr) { 8804 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8805 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8806 | jp &exit_addr 8807 | jne &exit_addr 8808 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8809 } else { 8810 | jp >1 8811 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8812 | je &exit_addr 8813 |1: 8814 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8815 } 8816 } else if (false_label != (uint32_t)-1) { // JMPZ_EX (p=>true, z=>false, false=>jmp) 8817 | jp >1 8818 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8819 | je => false_label 8820 |1: 8821 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8822 } else if (true_label != (uint32_t)-1) { // JMPNZ_EX (p=>true, z=>false, true=>jmp) 8823 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 8824 | jp => true_label 8825 | jne => true_label 8826 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 8827 } else if (set_bool_not) { // BOOL_NOT (p=>false, z=>true) 8828 | mov eax, IS_FALSE 8829 | jp >1 8830 | jne >1 8831 | mov eax, IS_TRUE 8832 |1: 8833 | SET_ZVAL_TYPE_INFO res_addr, eax 8834 } else { // BOOL (p=>true, z=>false) 8835 | mov eax, IS_TRUE 8836 | jp >1 8837 | jne >1 8838 | mov eax, IS_FALSE 8839 |1: 8840 | SET_ZVAL_TYPE_INFO res_addr, eax 8841 } 8842 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8843 | jmp >9 8844 |.code 8845 } 8846 } else { 8847 if (exit_addr) { 8848 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8849 | jp &exit_addr 8850 | jne &exit_addr 8851 |1: 8852 } else { 8853 | jp >1 8854 | je &exit_addr 8855 |1: 8856 } 8857 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8858 | jmp >9 8859 } 8860 } else { 8861 ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); 8862 if (false_label != (uint32_t)-1 ) { 8863 | jp >1 8864 | je => false_label 8865 |1: 8866 if (true_label != (uint32_t)-1) { 8867 | jmp =>true_label 8868 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8869 | jmp >9 8870 } 8871 } else { 8872 | jp => true_label 8873 | jne => true_label 8874 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8875 | jmp >9 8876 } 8877 } 8878 } 8879 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8880 |.code 8881 } 8882 } 8883 } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { 8884 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8885 |.cold_code 8886 |2: 8887 } 8888 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 8889 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 8890 } 8891 | SET_EX_OPLINE opline, r0 8892 | EXT_CALL zend_is_true, r0 8893 8894 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 8895 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8896 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 8897 8898 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 8899 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3 8900 } 8901 | GET_ZVAL_PTR FCARG1a, op1_addr 8902 | GC_DELREF FCARG1a 8903 | jnz >3 8904 | mov aword T1, r0 // save 8905 | ZVAL_DTOR_FUNC op1_info, opline 8906 | mov r0, aword T1 // restore 8907 |3: 8908 } 8909 if (may_throw) { 8910 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r1 8911 | jne ->exception_handler_undef 8912 } 8913 8914 if (set_bool) { 8915 if (set_bool_not) { 8916 | neg eax 8917 | add eax, 3 8918 } else { 8919 | add eax, 2 8920 } 8921 | SET_ZVAL_TYPE_INFO res_addr, eax 8922 if (exit_addr) { 8923 | CMP_ZVAL_TYPE res_addr, IS_FALSE 8924 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8925 | jne &exit_addr 8926 } else { 8927 | je &exit_addr 8928 } 8929 } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { 8930 | CMP_ZVAL_TYPE res_addr, IS_FALSE 8931 if (true_label != (uint32_t)-1) { 8932 | jne =>true_label 8933 if (false_label != (uint32_t)-1) { 8934 | jmp =>false_label 8935 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8936 | jmp >9 8937 } 8938 } else { 8939 | je =>false_label 8940 } 8941 } 8942 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8943 | jmp >9 8944 |.code 8945 } 8946 } else { 8947 | test r0, r0 8948 if (exit_addr) { 8949 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { 8950 | jne &exit_addr 8951 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8952 | jmp >9 8953 } 8954 } else { 8955 | je &exit_addr 8956 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8957 | jmp >9 8958 } 8959 } 8960 } else if (true_label != (uint32_t)-1) { 8961 | jne =>true_label 8962 if (false_label != (uint32_t)-1) { 8963 | jmp =>false_label 8964 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8965 | jmp >9 8966 } 8967 } else { 8968 | je =>false_label 8969 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8970 | jmp >9 8971 } 8972 } 8973 8974 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { 8975 |.code 8976 } 8977 } 8978 } 8979 8980 |9: 8981 8982 return 1; 8983} 8984 8985static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) 8986{ 8987 if (op1_addr != op1_def_addr) { 8988 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 8989 return 0; 8990 } 8991 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 8992 op1_addr = op1_def_addr; 8993 } 8994 } 8995 8996 if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0)) { 8997 return 0; 8998 } 8999 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 9000 return 0; 9001 } 9002 if (op1_info & MAY_BE_UNDEF) { 9003 zend_jit_check_exception(Dst); 9004 } 9005 return 1; 9006} 9007 9008static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) 9009{ 9010 ZEND_ASSERT(opline->op1_type == IS_CV); 9011 9012 if (op2_addr != op2_def_addr) { 9013 if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { 9014 return 0; 9015 } 9016 if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { 9017 op2_addr = op2_def_addr; 9018 } 9019 } 9020 9021 if (Z_MODE(op1_addr) != IS_REG 9022 && Z_MODE(op1_use_addr) == IS_REG 9023 && !Z_LOAD(op1_use_addr) 9024 && !Z_STORE(op1_use_addr)) { 9025 /* Force type update */ 9026 op1_info |= MAY_BE_UNDEF; 9027 } 9028 if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, 9029 may_throw)) { 9030 return 0; 9031 } 9032 if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { 9033 return 0; 9034 } 9035 if (opline->result_type != IS_UNUSED) { 9036 if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 9037 return 0; 9038 } 9039 } 9040 9041 return 1; 9042} 9043 9044/* copy of hidden zend_closure */ 9045typedef struct _zend_closure { 9046 zend_object std; 9047 zend_function func; 9048 zval this_ptr; 9049 zend_class_entry *called_scope; 9050 zif_handler orig_internal_handler; 9051} zend_closure; 9052 9053static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) 9054{ 9055 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9056 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9057 9058 if (!exit_addr) { 9059 return 0; 9060 } 9061 9062 | // Check Stack Overflow 9063 | MEM_OP2_2_ZTS mov, r1, aword, executor_globals, vm_stack_end, r0 9064 | MEM_OP2_2_ZTS sub, r1, aword, executor_globals, vm_stack_top, r0 9065 | cmp r1, used_stack 9066 | jb &exit_addr 9067 9068 return 1; 9069} 9070 9071static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, zend_bool is_closure, zend_bool use_this, zend_bool stack_check) 9072{ 9073 uint32_t used_stack; 9074 9075 if (func) { 9076 used_stack = zend_vm_calc_used_stack(opline->extended_value, func); 9077 } else { 9078 used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval); 9079 9080 | // if (EXPECTED(ZEND_USER_CODE(func->type))) { 9081 if (!is_closure) { 9082 | test byte [r0 + offsetof(zend_function, type)], 1 9083 | mov FCARG1a, used_stack 9084 | jnz >1 9085 } else { 9086 | mov FCARG1a, used_stack 9087 } 9088 | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); 9089 | mov edx, opline->extended_value 9090 if (!is_closure) { 9091 | cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)] 9092 | cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)] 9093 | sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)] 9094 | sub edx, dword [r0 + offsetof(zend_function, op_array.T)] 9095 } else { 9096 | cmp edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] 9097 | cmova edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)] 9098 | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.last_var)] 9099 | sub edx, dword [r0 + offsetof(zend_closure, func.op_array.T)] 9100 } 9101 | shl edx, 4 9102 |.if X64 9103 | movsxd r2, edx 9104 |.endif 9105 | sub FCARG1a, r2 9106 |1: 9107 } 9108 9109 zend_jit_start_reuse_ip(); 9110 9111 | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { 9112 | MEM_OP2_2_ZTS mov, RX, aword, executor_globals, vm_stack_top, RX 9113 9114 if (stack_check) { 9115 | // Check Stack Overflow 9116 | MEM_OP2_2_ZTS mov, r2, aword, executor_globals, vm_stack_end, r2 9117 | sub r2, RX 9118 if (func) { 9119 | cmp r2, used_stack 9120 } else { 9121 | cmp r2, FCARG1a 9122 } 9123 9124 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9125 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9126 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9127 9128 if (!exit_addr) { 9129 return 0; 9130 } 9131 9132 | jb &exit_addr 9133 } else { 9134 | jb >1 9135 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); 9136 |.cold_code 9137 |1: 9138 if (func) { 9139 | mov FCARG1d, used_stack 9140 } 9141#ifdef _WIN32 9142 if (0) { 9143#else 9144 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 9145#endif 9146 | SET_EX_OPLINE opline, r0 9147 | EXT_CALL zend_jit_int_extend_stack_helper, r0 9148 } else { 9149 if (!is_closure) { 9150 if (func 9151 && op_array == &func->op_array 9152 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9153 && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { 9154 | LOAD_ADDR FCARG2a, func 9155 } else { 9156 | mov FCARG2a, r0 9157 } 9158 } else { 9159 | lea FCARG2a, aword [r0 + offsetof(zend_closure, func)] 9160 } 9161 | SET_EX_OPLINE opline, r0 9162 | EXT_CALL zend_jit_extend_stack_helper, r0 9163 } 9164 | mov RX, r0 9165 | jmp >1 9166 |.code 9167 } 9168 } 9169 9170 if (func) { 9171 | MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2 9172 } else { 9173 | MEM_OP2_1_ZTS add, aword, executor_globals, vm_stack_top, FCARG1a, r2 9174 } 9175 | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); 9176 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { 9177 | // ZEND_SET_CALL_INFO(call, 0, call_info); 9178 | mov dword EX:RX->This.u1.type_info, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) 9179 } 9180#ifdef _WIN32 9181 if (0) { 9182#else 9183 if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { 9184#endif 9185 | // call->func = func; 9186 |1: 9187 | ADDR_OP2_2 mov, aword EX:RX->func, func, r1 9188 } else { 9189 if (!is_closure) { 9190 | // call->func = func; 9191 if (func 9192 && op_array == &func->op_array 9193 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) 9194 && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { 9195 | ADDR_OP2_2 mov, aword EX:RX->func, func, r1 9196 } else { 9197 | mov aword EX:RX->func, r0 9198 } 9199 } else { 9200 | // call->func = &closure->func; 9201 | lea r1, aword [r0 + offsetof(zend_closure, func)] 9202 | mov aword EX:RX->func, r1 9203 } 9204 |1: 9205 } 9206 if (opline->opcode == ZEND_INIT_METHOD_CALL) { 9207 | // Z_PTR(call->This) = obj; 9208 | mov r1, aword T1 9209 | mov aword EX:RX->This.value.ptr, r1 9210 if (opline->op1_type == IS_UNUSED || use_this) { 9211 | // call->call_info |= ZEND_CALL_HAS_THIS; 9212 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9213 | mov dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS 9214 } else { 9215 | or dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS 9216 } 9217 } else { 9218 if (opline->op1_type == IS_CV) { 9219 | // GC_ADDREF(obj); 9220 | add dword [r1], 1 9221 } 9222 | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; 9223 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9224 | mov dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 9225 } else { 9226 | or dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) 9227 } 9228 } 9229 } else if (!is_closure) { 9230 | // Z_CE(call->This) = called_scope; 9231 | mov aword EX:RX->This.value.ptr, 0 9232 } else { 9233 if (opline->op2_type == IS_CV) { 9234 | // GC_ADDREF(closure); 9235 | add dword [r0], 1 9236 } 9237 | // object_or_called_scope = closure->called_scope; 9238 | mov r1, aword [r0 + offsetof(zend_closure, called_scope)] 9239 | mov aword EX:RX->This.value.ptr, r1 9240 | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | 9241 | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); 9242 | mov edx, dword [r0 + offsetof(zend_closure, func.common.fn_flags)] 9243 | and edx, ZEND_ACC_FAKE_CLOSURE 9244 | or edx, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE) 9245 | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { 9246 | cmp byte [r0 + offsetof(zend_closure, this_ptr.u1.v.type)], IS_UNDEF 9247 | jz >1 9248 | // call_info |= ZEND_CALL_HAS_THIS; 9249 | or edx, ZEND_CALL_HAS_THIS 9250 | // object_or_called_scope = Z_OBJ(closure->this_ptr); 9251 | mov r1, aword [r0 + offsetof(zend_closure, this_ptr.value.ptr)] 9252 |1: 9253 | // ZEND_SET_CALL_INFO(call, 0, call_info); 9254 | or dword EX:RX->This.u1.type_info, edx 9255 | // Z_PTR(call->This) = object_or_called_scope; 9256 | mov aword EX:RX->This.value.ptr, r1 9257 | cmp aword [r0 + offsetof(zend_closure, func.op_array.run_time_cache__ptr)], 0 9258 | jnz >1 9259 | lea FCARG1a, aword [r0 + offsetof(zend_closure, func)] 9260 | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 9261 |1: 9262 } 9263 | // ZEND_CALL_NUM_ARGS(call) = num_args; 9264 | mov dword EX:RX->This.u2.num_args, opline->extended_value 9265 return 1; 9266} 9267 9268static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace) 9269{ 9270 int skip; 9271 9272 if (trace) { 9273 zend_jit_trace_rec *p = trace; 9274 9275 ssa_op++; 9276 while (1) { 9277 if (p->op == ZEND_JIT_TRACE_VM) { 9278 switch (p->opline->opcode) { 9279 case ZEND_SEND_ARRAY: 9280 case ZEND_SEND_USER: 9281 case ZEND_SEND_UNPACK: 9282 case ZEND_INIT_FCALL: 9283 case ZEND_INIT_METHOD_CALL: 9284 case ZEND_INIT_STATIC_METHOD_CALL: 9285 case ZEND_INIT_FCALL_BY_NAME: 9286 case ZEND_INIT_NS_FCALL_BY_NAME: 9287 case ZEND_INIT_DYNAMIC_CALL: 9288 case ZEND_NEW: 9289 case ZEND_INIT_USER_CALL: 9290 case ZEND_FAST_CALL: 9291 case ZEND_JMP: 9292 case ZEND_JMPZNZ: 9293 case ZEND_JMPZ: 9294 case ZEND_JMPNZ: 9295 case ZEND_JMPZ_EX: 9296 case ZEND_JMPNZ_EX: 9297 case ZEND_FE_RESET_R: 9298 case ZEND_FE_RESET_RW: 9299 case ZEND_JMP_SET: 9300 case ZEND_COALESCE: 9301 case ZEND_JMP_NULL: 9302 case ZEND_ASSERT_CHECK: 9303 case ZEND_CATCH: 9304 case ZEND_DECLARE_ANON_CLASS: 9305 case ZEND_FE_FETCH_R: 9306 case ZEND_FE_FETCH_RW: 9307 return 1; 9308 case ZEND_DO_ICALL: 9309 case ZEND_DO_UCALL: 9310 case ZEND_DO_FCALL_BY_NAME: 9311 case ZEND_DO_FCALL: 9312 return 0; 9313 case ZEND_SEND_VAL: 9314 case ZEND_SEND_VAR: 9315 case ZEND_SEND_VAL_EX: 9316 case ZEND_SEND_VAR_EX: 9317 case ZEND_SEND_FUNC_ARG: 9318 case ZEND_SEND_REF: 9319 case ZEND_SEND_VAR_NO_REF: 9320 case ZEND_SEND_VAR_NO_REF_EX: 9321 /* skip */ 9322 break; 9323 default: 9324 if (zend_may_throw(opline, ssa_op, op_array, ssa)) { 9325 return 1; 9326 } 9327 } 9328 ssa_op += zend_jit_trace_op_len(opline); 9329 } else if (p->op == ZEND_JIT_TRACE_ENTER || 9330 p->op == ZEND_JIT_TRACE_BACK || 9331 p->op == ZEND_JIT_TRACE_END) { 9332 return 1; 9333 } 9334 p++; 9335 } 9336 } 9337 9338 if (!call_info) { 9339 const zend_op *end = op_array->opcodes + op_array->last; 9340 9341 opline++; 9342 ssa_op++; 9343 skip = (call_level == 1); 9344 while (opline != end) { 9345 if (!skip) { 9346 if (zend_may_throw(opline, ssa_op, op_array, ssa)) { 9347 return 1; 9348 } 9349 } 9350 switch (opline->opcode) { 9351 case ZEND_SEND_VAL: 9352 case ZEND_SEND_VAR: 9353 case ZEND_SEND_VAL_EX: 9354 case ZEND_SEND_VAR_EX: 9355 case ZEND_SEND_FUNC_ARG: 9356 case ZEND_SEND_REF: 9357 case ZEND_SEND_VAR_NO_REF: 9358 case ZEND_SEND_VAR_NO_REF_EX: 9359 skip = 0; 9360 break; 9361 case ZEND_SEND_ARRAY: 9362 case ZEND_SEND_USER: 9363 case ZEND_SEND_UNPACK: 9364 case ZEND_INIT_FCALL: 9365 case ZEND_INIT_METHOD_CALL: 9366 case ZEND_INIT_STATIC_METHOD_CALL: 9367 case ZEND_INIT_FCALL_BY_NAME: 9368 case ZEND_INIT_NS_FCALL_BY_NAME: 9369 case ZEND_INIT_DYNAMIC_CALL: 9370 case ZEND_NEW: 9371 case ZEND_INIT_USER_CALL: 9372 case ZEND_FAST_CALL: 9373 case ZEND_JMP: 9374 case ZEND_JMPZNZ: 9375 case ZEND_JMPZ: 9376 case ZEND_JMPNZ: 9377 case ZEND_JMPZ_EX: 9378 case ZEND_JMPNZ_EX: 9379 case ZEND_FE_RESET_R: 9380 case ZEND_FE_RESET_RW: 9381 case ZEND_JMP_SET: 9382 case ZEND_COALESCE: 9383 case ZEND_JMP_NULL: 9384 case ZEND_ASSERT_CHECK: 9385 case ZEND_CATCH: 9386 case ZEND_DECLARE_ANON_CLASS: 9387 case ZEND_FE_FETCH_R: 9388 case ZEND_FE_FETCH_RW: 9389 return 1; 9390 case ZEND_DO_ICALL: 9391 case ZEND_DO_UCALL: 9392 case ZEND_DO_FCALL_BY_NAME: 9393 case ZEND_DO_FCALL: 9394 end = opline; 9395 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { 9396 /* INIT_FCALL and DO_FCALL in different BasicBlocks */ 9397 return 1; 9398 } 9399 return 0; 9400 } 9401 opline++; 9402 ssa_op++; 9403 } 9404 9405 return 1; 9406 } else { 9407 const zend_op *end = call_info->caller_call_opline; 9408 9409 if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { 9410 /* INIT_FCALL and DO_FCALL in different BasicBlocks */ 9411 return 1; 9412 } 9413 9414 opline++; 9415 ssa_op++; 9416 skip = (call_level == 1); 9417 while (opline != end) { 9418 if (skip) { 9419 switch (opline->opcode) { 9420 case ZEND_SEND_VAL: 9421 case ZEND_SEND_VAR: 9422 case ZEND_SEND_VAL_EX: 9423 case ZEND_SEND_VAR_EX: 9424 case ZEND_SEND_FUNC_ARG: 9425 case ZEND_SEND_REF: 9426 case ZEND_SEND_VAR_NO_REF: 9427 case ZEND_SEND_VAR_NO_REF_EX: 9428 skip = 0; 9429 break; 9430 case ZEND_SEND_ARRAY: 9431 case ZEND_SEND_USER: 9432 case ZEND_SEND_UNPACK: 9433 return 1; 9434 } 9435 } else { 9436 if (zend_may_throw(opline, ssa_op, op_array, ssa)) { 9437 return 1; 9438 } 9439 } 9440 opline++; 9441 ssa_op++; 9442 } 9443 9444 return 0; 9445 } 9446} 9447 9448static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) 9449{ 9450 int32_t exit_point; 9451 const void *exit_addr; 9452 9453 if (func->type == ZEND_INTERNAL_FUNCTION) { 9454#ifdef ZEND_WIN32 9455 // TODO: ASLR may cause different addresses in different workers ??? 9456 return 0; 9457#endif 9458 } else if (func->type == ZEND_USER_FUNCTION) { 9459 if (!zend_accel_in_shm(func->op_array.opcodes)) { 9460 /* op_array and op_array->opcodes are not persistent. We can't link. */ 9461 return 0; 9462 } 9463 } else { 9464 ZEND_UNREACHABLE(); 9465 return 0; 9466 } 9467 9468 exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); 9469 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9470 if (!exit_addr) { 9471 return 0; 9472 } 9473 9474 | // call = EX(call); 9475 | mov r1, EX->call 9476 while (level > 0) { 9477 | mov r1, EX:r1->prev_execute_data 9478 level--; 9479 } 9480 9481 if (func->type == ZEND_USER_FUNCTION && 9482 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 9483 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 9484 !func->common.function_name)) { 9485 const zend_op *opcodes = func->op_array.opcodes; 9486 9487 | mov r1, aword EX:r1->func 9488 | .if X64 9489 || if (!IS_SIGNED_32BIT(opcodes)) { 9490 | mov64 r2, ((ptrdiff_t)opcodes) 9491 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], r2 9492 || } else { 9493 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes 9494 || } 9495 | .else 9496 | cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes 9497 | .endif 9498 | jne &exit_addr 9499 } else { 9500 | .if X64 9501 || if (!IS_SIGNED_32BIT(func)) { 9502 | mov64 r2, ((ptrdiff_t)func) 9503 | cmp aword EX:r1->func, r2 9504 || } else { 9505 | cmp aword EX:r1->func, func 9506 || } 9507 | .else 9508 | cmp aword EX:r1->func, func 9509 | .endif 9510 | jne &exit_addr 9511 } 9512 9513 return 1; 9514} 9515 9516static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, zend_bool stack_check) 9517{ 9518 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9519 zend_call_info *call_info = NULL; 9520 zend_function *func = NULL; 9521 9522 if (delayed_call_chain) { 9523 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9524 return 0; 9525 } 9526 } 9527 9528 if (info) { 9529 call_info = info->callee_info; 9530 while (call_info && call_info->caller_init_opline != opline) { 9531 call_info = call_info->next_callee; 9532 } 9533 if (call_info && call_info->callee_func) { 9534 func = call_info->callee_func; 9535 } 9536 } 9537 9538 if (!func 9539 && trace 9540 && trace->op == ZEND_JIT_TRACE_INIT_CALL) { 9541#ifdef _WIN32 9542 /* ASLR */ 9543 if (trace->func->type != ZEND_INTERNAL_FUNCTION) { 9544 func = (zend_function*)trace->func; 9545 } 9546#else 9547 func = (zend_function*)trace->func; 9548#endif 9549 } 9550 9551#ifdef _WIN32 9552 if (0) { 9553#else 9554 if (opline->opcode == ZEND_INIT_FCALL 9555 && func 9556 && func->type == ZEND_INTERNAL_FUNCTION) { 9557#endif 9558 /* load constant address later */ 9559 } else if (func && op_array == &func->op_array) { 9560 /* recursive call */ 9561 if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) || 9562 (sizeof(void*) == 8 && !IS_SIGNED_32BIT(func))) { 9563 | mov r0, EX->func 9564 } 9565 } else { 9566 | // if (CACHED_PTR(opline->result.num)) 9567 | mov r0, EX->run_time_cache 9568 | mov r0, aword [r0 + opline->result.num] 9569 | test r0, r0 9570 | jz >1 9571 |.cold_code 9572 |1: 9573 if (opline->opcode == ZEND_INIT_FCALL 9574 && func 9575 && func->type == ZEND_USER_FUNCTION 9576 && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { 9577 | LOAD_ADDR FCARG1a, func 9578 | EXT_CALL zend_jit_init_func_run_time_cache_helper, r0 9579 | mov r1, EX->run_time_cache 9580 | mov aword [r1 + opline->result.num], r0 9581 | jmp >3 9582 } else { 9583 zval *zv = RT_CONSTANT(opline, opline->op2); 9584 9585 if (opline->opcode == ZEND_INIT_FCALL) { 9586 | LOAD_ADDR FCARG1a, Z_STR_P(zv); 9587 | EXT_CALL zend_jit_find_func_helper, r0 9588 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { 9589 | LOAD_ADDR FCARG1a, Z_STR_P(zv + 1); 9590 | EXT_CALL zend_jit_find_func_helper, r0 9591 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { 9592 | LOAD_ADDR FCARG1a, zv; 9593 | EXT_CALL zend_jit_find_ns_func_helper, r0 9594 } else { 9595 ZEND_UNREACHABLE(); 9596 } 9597 | // CACHE_PTR(opline->result.num, fbc); 9598 | mov r1, EX->run_time_cache 9599 | mov aword [r1 + opline->result.num], r0 9600 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9601 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 9602 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9603 9604 if (!exit_addr) { 9605 return 0; 9606 } 9607 9608 if (!func || opline->opcode == ZEND_INIT_FCALL) { 9609 | test r0, r0 9610 | jnz >3 9611 } else if (func->type == ZEND_USER_FUNCTION 9612 && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { 9613 const zend_op *opcodes = func->op_array.opcodes; 9614 9615 | .if X64 9616 || if (!IS_SIGNED_32BIT(opcodes)) { 9617 | mov64 r1, ((ptrdiff_t)opcodes) 9618 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 9619 || } else { 9620 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9621 || } 9622 | .else 9623 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9624 | .endif 9625 | jz >3 9626 } else { 9627 | .if X64 9628 || if (!IS_SIGNED_32BIT(func)) { 9629 | mov64 r1, ((ptrdiff_t)func) 9630 | cmp r0, r1 9631 || } else { 9632 | cmp r0, func 9633 || } 9634 | .else 9635 | cmp r0, func 9636 | .endif 9637 | jz >3 9638 } 9639 | jmp &exit_addr 9640 } else { 9641 | test r0, r0 9642 | jnz >3 9643 | // SAVE_OPLINE(); 9644 | SET_EX_OPLINE opline, r0 9645 | jmp ->undefined_function 9646 } 9647 } 9648 |.code 9649 |3: 9650 } 9651 9652 if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, stack_check)) { 9653 return 0; 9654 } 9655 9656 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9657 if (!zend_jit_save_call_chain(Dst, call_level)) { 9658 return 0; 9659 } 9660 } else { 9661 delayed_call_chain = 1; 9662 delayed_call_level = call_level; 9663 } 9664 9665 return 1; 9666} 9667 9668static int zend_jit_init_method_call(dasm_State **Dst, 9669 const zend_op *opline, 9670 uint32_t b, 9671 const zend_op_array *op_array, 9672 zend_ssa *ssa, 9673 const zend_ssa_op *ssa_op, 9674 int call_level, 9675 uint32_t op1_info, 9676 zend_jit_addr op1_addr, 9677 zend_class_entry *ce, 9678 zend_bool ce_is_instanceof, 9679 zend_bool use_this, 9680 zend_class_entry *trace_ce, 9681 zend_jit_trace_rec *trace, 9682 zend_bool stack_check, 9683 zend_bool polymorphic_side_trace) 9684{ 9685 zend_func_info *info = ZEND_FUNC_INFO(op_array); 9686 zend_call_info *call_info = NULL; 9687 zend_function *func = NULL; 9688 zval *function_name; 9689 9690 ZEND_ASSERT(opline->op2_type == IS_CONST); 9691 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 9692 9693 function_name = RT_CONSTANT(opline, opline->op2); 9694 9695 if (info) { 9696 call_info = info->callee_info; 9697 while (call_info && call_info->caller_init_opline != opline) { 9698 call_info = call_info->next_callee; 9699 } 9700 if (call_info && call_info->callee_func) { 9701 func = call_info->callee_func; 9702 } 9703 } 9704 9705 if (polymorphic_side_trace) { 9706 /* function is passed in r0 from parent_trace */ 9707 } else { 9708 if (opline->op1_type == IS_UNUSED || use_this) { 9709 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 9710 9711 | GET_ZVAL_PTR FCARG1a, this_addr 9712 } else { 9713 if (op1_info & MAY_BE_REF) { 9714 if (opline->op1_type == IS_CV) { 9715 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 9716 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9717 } 9718 | ZVAL_DEREF FCARG1a, op1_info 9719 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 9720 } else { 9721 /* Hack: Convert reference to regular value to simplify JIT code */ 9722 ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); 9723 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 9724 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9725 | EXT_CALL zend_jit_unref_helper, r0 9726 |1: 9727 } 9728 } 9729 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 9730 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 9731 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9732 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9733 9734 if (!exit_addr) { 9735 return 0; 9736 } 9737 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 9738 } else { 9739 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 9740 |.cold_code 9741 |1: 9742 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 9743 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 9744 } 9745 | SET_EX_OPLINE opline, r0 9746 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { 9747 | EXT_CALL zend_jit_invalid_method_call_tmp, r0 9748 } else { 9749 | EXT_CALL zend_jit_invalid_method_call, r0 9750 } 9751 | jmp ->exception_handler 9752 |.code 9753 } 9754 } 9755 | GET_ZVAL_PTR FCARG1a, op1_addr 9756 } 9757 9758 if (delayed_call_chain) { 9759 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9760 return 0; 9761 } 9762 } 9763 9764 | mov aword T1, FCARG1a // save 9765 9766 if (func) { 9767 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9768 | mov r0, EX->run_time_cache 9769 | mov r0, aword [r0 + opline->result.num + sizeof(void*)] 9770 | test r0, r0 9771 | jz >1 9772 } else { 9773 | // if (CACHED_PTR(opline->result.num) == obj->ce)) { 9774 | mov r0, EX->run_time_cache 9775 | mov r2, aword [r0 + opline->result.num] 9776 | cmp r2, [FCARG1a + offsetof(zend_object, ce)] 9777 | jnz >1 9778 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); 9779 | mov r0, aword [r0 + opline->result.num + sizeof(void*)] 9780 } 9781 9782 |.cold_code 9783 |1: 9784 | LOAD_ADDR FCARG2a, function_name 9785 |.if X64 9786 | lea CARG3, aword T1 9787 |.else 9788 | lea r0, aword T1 9789 | sub r4, 12 9790 | push r0 9791 |.endif 9792 | SET_EX_OPLINE opline, r0 9793 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { 9794 | EXT_CALL zend_jit_find_method_tmp_helper, r0 9795 } else { 9796 | EXT_CALL zend_jit_find_method_helper, r0 9797 } 9798 |.if not(X64) 9799 | add r4, 12 9800 |.endif 9801 | test r0, r0 9802 | jnz >2 9803 | jmp ->exception_handler 9804 |.code 9805 |2: 9806 } 9807 9808 if (!func 9809 && trace 9810 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9811 && trace->func 9812#ifdef _WIN32 9813 && trace->func->type != ZEND_INTERNAL_FUNCTION 9814#endif 9815 ) { 9816 int32_t exit_point; 9817 const void *exit_addr; 9818 9819 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_METHOD_CALL); 9820 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9821 if (!exit_addr) { 9822 return 0; 9823 } 9824 9825 func = (zend_function*)trace->func; 9826 9827 if (func->type == ZEND_USER_FUNCTION && 9828 (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || 9829 (func->common.fn_flags & ZEND_ACC_CLOSURE) || 9830 !func->common.function_name)) { 9831 const zend_op *opcodes = func->op_array.opcodes; 9832 9833 | .if X64 9834 || if (!IS_SIGNED_32BIT(opcodes)) { 9835 | mov64 r1, ((ptrdiff_t)opcodes) 9836 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1 9837 || } else { 9838 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9839 || } 9840 | .else 9841 | cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes 9842 | .endif 9843 | jne &exit_addr 9844 } else { 9845 | .if X64 9846 || if (!IS_SIGNED_32BIT(func)) { 9847 | mov64 r1, ((ptrdiff_t)func) 9848 | cmp r0, r1 9849 || } else { 9850 | cmp r0, func 9851 || } 9852 | .else 9853 | cmp r0, func 9854 | .endif 9855 | jne &exit_addr 9856 } 9857 } 9858 9859 if (!func) { 9860 | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { 9861 | test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_STATIC 9862 | jnz >1 9863 |.cold_code 9864 |1: 9865 } 9866 9867 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { 9868 | mov FCARG1a, aword T1 // restore 9869 | mov FCARG2a, r0 9870 |.if X64 9871 | mov CARG3d, opline->extended_value 9872 |.else 9873 | sub r4, 12 9874 | push opline->extended_value 9875 |.endif 9876 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { 9877 | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, r0 9878 } else { 9879 | EXT_CALL zend_jit_push_static_metod_call_frame, r0 9880 } 9881 |.if not(X64) 9882 | add r4, 12 9883 |.endif 9884 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !use_this)) { 9885 | test r0, r0 9886 | jz ->exception_handler 9887 } 9888 | mov RX, r0 9889 } 9890 9891 if (!func) { 9892 | jmp >9 9893 |.code 9894 } 9895 9896 if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { 9897 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, use_this, stack_check)) { 9898 return 0; 9899 } 9900 } 9901 9902 if (!func) { 9903 |9: 9904 } 9905 zend_jit_start_reuse_ip(); 9906 9907 if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 9908 if (!zend_jit_save_call_chain(Dst, call_level)) { 9909 return 0; 9910 } 9911 } else { 9912 delayed_call_chain = 1; 9913 delayed_call_level = call_level; 9914 } 9915 9916 return 1; 9917} 9918 9919static int zend_jit_init_closure_call(dasm_State **Dst, 9920 const zend_op *opline, 9921 uint32_t b, 9922 const zend_op_array *op_array, 9923 zend_ssa *ssa, 9924 const zend_ssa_op *ssa_op, 9925 int call_level, 9926 zend_jit_trace_rec *trace, 9927 zend_bool stack_check) 9928{ 9929 zend_function *func = NULL; 9930 zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 9931 9932 | GET_ZVAL_PTR r0, op2_addr 9933 9934 if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure 9935 && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { 9936 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 9937 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9938 9939 if (!exit_addr) { 9940 return 0; 9941 } 9942 9943 |.if X64 9944 || if (!IS_SIGNED_32BIT(zend_ce_closure)) { 9945 | mov64 FCARG1a, ((ptrdiff_t)zend_ce_closure) 9946 | cmp aword [r0 + offsetof(zend_object, ce)], FCARG1a 9947 || } else { 9948 | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure 9949 || } 9950 |.else 9951 | cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure 9952 |.endif 9953 | jne &exit_addr 9954 if (ssa->var_info && ssa_op->op2_use >= 0) { 9955 ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; 9956 ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; 9957 ssa->var_info[ssa_op->op2_use].is_instanceof = 0; 9958 } 9959 } 9960 9961 if (trace 9962 && trace->op == ZEND_JIT_TRACE_INIT_CALL 9963 && trace->func 9964 && trace->func->type == ZEND_USER_FUNCTION) { 9965 const zend_op *opcodes; 9966 int32_t exit_point; 9967 const void *exit_addr; 9968 9969 func = (zend_function*)trace->func; 9970 opcodes = func->op_array.opcodes; 9971 exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); 9972 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 9973 if (!exit_addr) { 9974 return 0; 9975 } 9976 9977 | .if X64 9978 || if (!IS_SIGNED_32BIT(opcodes)) { 9979 | mov64 FCARG1a, ((ptrdiff_t)opcodes) 9980 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], FCARG1a 9981 || } else { 9982 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes 9983 || } 9984 | .else 9985 | cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes 9986 | .endif 9987 | jne &exit_addr 9988 } 9989 9990 if (delayed_call_chain) { 9991 if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { 9992 return 0; 9993 } 9994 } 9995 9996 if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, stack_check)) { 9997 return 0; 9998 } 9999 10000 if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { 10001 if (!zend_jit_save_call_chain(Dst, call_level)) { 10002 return 0; 10003 } 10004 } else { 10005 delayed_call_chain = 1; 10006 delayed_call_level = call_level; 10007 } 10008 10009 if (trace 10010 && trace->op == ZEND_JIT_TRACE_END 10011 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 10012 if (!zend_jit_set_valid_ip(Dst, opline + 1)) { 10013 return 0; 10014 } 10015 } 10016 10017 return 1; 10018} 10019 10020static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info) 10021{ 10022 uint32_t num_args = 0; 10023 zend_function *func = call_info->callee_func; 10024 10025 while (num_args < call_info->num_args) { 10026 zend_arg_info *arg_info = func->op_array.arg_info + num_args; 10027 10028 if (ZEND_TYPE_IS_SET(arg_info->type)) { 10029 if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) { 10030 zend_op *opline = call_info->arg_info[num_args].opline; 10031 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; 10032 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type); 10033 if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) { 10034 break; 10035 } 10036 } else { 10037 break; 10038 } 10039 } 10040 num_args++; 10041 } 10042 return num_args; 10043} 10044 10045static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) 10046{ 10047 zend_func_info *info = ZEND_FUNC_INFO(op_array); 10048 zend_call_info *call_info = NULL; 10049 const zend_function *func = NULL; 10050 uint32_t i; 10051 zend_jit_addr res_addr; 10052 uint32_t call_num_args = 0; 10053 zend_bool unknown_num_args = 0; 10054 const void *exit_addr = NULL; 10055 const zend_op *prev_opline; 10056 10057 if (RETURN_VALUE_USED(opline)) { 10058 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 10059 } else { 10060 /* CPU stack allocated temporary zval */ 10061 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, TMP_ZVAL_OFFSET); 10062 } 10063 10064 prev_opline = opline - 1; 10065 while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { 10066 prev_opline--; 10067 } 10068 if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || 10069 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 10070 unknown_num_args = 1; 10071 } 10072 10073 if (info) { 10074 call_info = info->callee_info; 10075 while (call_info && call_info->caller_call_opline != opline) { 10076 call_info = call_info->next_callee; 10077 } 10078 if (call_info && call_info->callee_func) { 10079 func = call_info->callee_func; 10080 } 10081 if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) 10082 && JIT_G(current_frame) 10083 && JIT_G(current_frame)->call 10084 && !JIT_G(current_frame)->call->func) { 10085 call_info = NULL; func = NULL; /* megamorphic call from trait */ 10086 } 10087 } 10088 if (!func) { 10089 /* resolve function at run time */ 10090 } else if (func->type == ZEND_USER_FUNCTION) { 10091 ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); 10092 call_num_args = call_info->num_args; 10093 } else if (func->type == ZEND_INTERNAL_FUNCTION) { 10094 ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); 10095 call_num_args = call_info->num_args; 10096 } else { 10097 ZEND_UNREACHABLE(); 10098 } 10099 10100 if (trace && !func) { 10101 if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { 10102 ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); 10103#ifndef ZEND_WIN32 10104 // TODO: ASLR may cause different addresses in different workers ??? 10105 func = trace->func; 10106 if (JIT_G(current_frame) && 10107 JIT_G(current_frame)->call && 10108 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 10109 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 10110 } else { 10111 unknown_num_args = 1; 10112 } 10113#endif 10114 } else if (trace->op == ZEND_JIT_TRACE_ENTER) { 10115 ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); 10116 if (zend_accel_in_shm(trace->func->op_array.opcodes)) { 10117 func = trace->func; 10118 if (JIT_G(current_frame) && 10119 JIT_G(current_frame)->call && 10120 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { 10121 call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); 10122 } else { 10123 unknown_num_args = 1; 10124 } 10125 } 10126 } 10127 } 10128 10129 bool may_have_extra_named_params = 10130 opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && 10131 (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); 10132 10133 if (!reuse_ip) { 10134 zend_jit_start_reuse_ip(); 10135 | // call = EX(call); 10136 | mov RX, EX->call 10137 } 10138 zend_jit_stop_reuse_ip(); 10139 10140 | // fbc = call->func; 10141 | // mov r2, EX:RX->func ??? 10142 | // SAVE_OPLINE(); 10143 | SET_EX_OPLINE opline, r0 10144 10145 if (opline->opcode == ZEND_DO_FCALL) { 10146 if (!func) { 10147 if (trace) { 10148 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10149 10150 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10151 if (!exit_addr) { 10152 return 0; 10153 } 10154 | mov r0, EX:RX->func 10155 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10156 | jnz &exit_addr 10157 } 10158 } 10159 } 10160 10161 if (!delayed_call_chain) { 10162 if (call_level == 1) { 10163 | mov aword EX->call, 0 10164 } else { 10165 | //EX(call) = call->prev_execute_data; 10166 | mov r0, EX:RX->prev_execute_data 10167 | mov EX->call, r0 10168 } 10169 } 10170 delayed_call_chain = 0; 10171 10172 | //call->prev_execute_data = execute_data; 10173 | mov EX:RX->prev_execute_data, EX 10174 10175 if (!func) { 10176 | mov r0, EX:RX->func 10177 } 10178 10179 if (opline->opcode == ZEND_DO_FCALL) { 10180 if (!func) { 10181 if (!trace) { 10182 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10183 | jnz >1 10184 |.cold_code 10185 |1: 10186 if (!GCC_GLOBAL_REGS) { 10187 | mov FCARG1a, RX 10188 } 10189 | EXT_CALL zend_jit_deprecated_helper, r0 10190 | test al, al 10191 | mov r0, EX:RX->func // reload 10192 | jne >1 10193 | jmp ->exception_handler 10194 |.code 10195 |1: 10196 } 10197 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 10198 if (!GCC_GLOBAL_REGS) { 10199 | mov FCARG1a, RX 10200 } 10201 | EXT_CALL zend_jit_deprecated_helper, r0 10202 | test al, al 10203 | je ->exception_handler 10204 } 10205 } 10206 10207 if (!func 10208 && opline->opcode != ZEND_DO_UCALL 10209 && opline->opcode != ZEND_DO_ICALL) { 10210 | cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION 10211 | jne >8 10212 } 10213 10214 if ((!func || func->type == ZEND_USER_FUNCTION) 10215 && opline->opcode != ZEND_DO_ICALL) { 10216 | // EX(call) = NULL; 10217 | mov aword EX:RX->call, 0 10218 10219 if (RETURN_VALUE_USED(opline)) { 10220 | // EX(return_value) = EX_VAR(opline->result.var); 10221 | LOAD_ZVAL_ADDR r2, res_addr 10222 | mov aword EX:RX->return_value, r2 10223 } else { 10224 | // EX(return_value) = 0; 10225 | mov aword EX:RX->return_value, 0 10226 } 10227 10228 //EX_LOAD_RUN_TIME_CACHE(op_array); 10229 if (!func || func->op_array.cache_size) { 10230 if (func && op_array == &func->op_array) { 10231 /* recursive call */ 10232 if (trace || func->op_array.cache_size > sizeof(void*)) { 10233 | mov r2, EX->run_time_cache 10234 | mov EX:RX->run_time_cache, r2 10235 } 10236 } else { 10237 if (func) { 10238 | mov r0, EX:RX->func 10239 } 10240 | mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)] 10241#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR 10242 | mov r2, aword [r2] 10243#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET 10244 if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { 10245 if (ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { 10246 | MEM_OP2_2_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 10247 } else if (!zend_accel_in_shm(func->op_array.opcodes)) { 10248 /* the called op_array may be not persisted yet */ 10249 | test r2, 1 10250 | jz >1 10251 | MEM_OP2_2_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 10252 |1: 10253 } 10254 | mov r2, aword [r2] 10255 } else { 10256 | test r2, 1 10257 | jz >1 10258 | MEM_OP2_2_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1 10259 |1: 10260 | mov r2, aword [r2] 10261 } 10262#else 10263# error "Unknown ZEND_MAP_PTR_KIND" 10264#endif 10265 | mov EX:RX->run_time_cache, r2 10266 } 10267 } 10268 10269 | // EG(current_execute_data) = execute_data; 10270 | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1 10271 | mov FP, RX 10272 10273 | // opline = op_array->opcodes; 10274 if (func && !unknown_num_args) { 10275 10276 for (i = call_num_args; i < func->op_array.last_var; i++) { 10277 uint32_t n = EX_NUM_TO_VAR(i); 10278 | SET_Z_TYPE_INFO RX + n, IS_UNDEF 10279 } 10280 10281 if (call_num_args <= func->op_array.num_args) { 10282 if (!trace || (trace->op == ZEND_JIT_TRACE_END 10283 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10284 uint32_t num_args; 10285 10286 if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { 10287 if (trace) { 10288 num_args = 0; 10289 } else if (call_info) { 10290 num_args = skip_valid_arguments(op_array, ssa, call_info); 10291 } else { 10292 num_args = call_num_args; 10293 } 10294 } else { 10295 num_args = call_num_args; 10296 } 10297 if (zend_accel_in_shm(func->op_array.opcodes)) { 10298 | LOAD_IP_ADDR (func->op_array.opcodes + num_args) 10299 } else { 10300 | mov r0, EX->func 10301 if (GCC_GLOBAL_REGS) { 10302 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10303 if (num_args) { 10304 | add IP, (num_args * sizeof(zend_op)) 10305 } 10306 } else { 10307 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10308 if (num_args) { 10309 | add FCARG1a, (num_args * sizeof(zend_op)) 10310 } 10311 | mov aword EX->opline, FCARG1a 10312 } 10313 } 10314 10315 if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array 10316 && num_args >= op_array->required_num_args) { 10317 /* recursive call */ 10318 if (ZEND_OBSERVER_ENABLED) { 10319 | SAVE_IP 10320 | mov FCARG1a, FP 10321 | EXT_CALL zend_observer_fcall_begin, r0 10322 } 10323#ifdef CONTEXT_THREADED_JIT 10324 | call >1 10325 |.cold_code 10326 |1: 10327 | pop r0 10328 | jmp =>num_args 10329 |.code 10330#else 10331 | jmp =>num_args 10332#endif 10333 return 1; 10334 } 10335 } 10336 } else { 10337 if (!trace || (trace->op == ZEND_JIT_TRACE_END 10338 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { 10339 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 10340 | LOAD_IP_ADDR (func->op_array.opcodes) 10341 } else if (GCC_GLOBAL_REGS) { 10342 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10343 } else { 10344 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10345 | mov aword EX->opline, FCARG1a 10346 } 10347 } 10348 if (!GCC_GLOBAL_REGS) { 10349 | mov FCARG1a, FP 10350 } 10351 | EXT_CALL zend_jit_copy_extra_args_helper, r0 10352 } 10353 } else { 10354 | // opline = op_array->opcodes 10355 if (func && zend_accel_in_shm(func->op_array.opcodes)) { 10356 | LOAD_IP_ADDR (func->op_array.opcodes) 10357 } else if (GCC_GLOBAL_REGS) { 10358 | mov IP, aword [r0 + offsetof(zend_op_array, opcodes)] 10359 } else { 10360 | mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)] 10361 | mov aword EX->opline, FCARG1a 10362 } 10363 if (func) { 10364 | // num_args = EX_NUM_ARGS(); 10365 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] 10366 | // if (UNEXPECTED(num_args > first_extra_arg)) 10367 | cmp ecx, (func->op_array.num_args) 10368 } else { 10369 | // first_extra_arg = op_array->num_args; 10370 | mov edx, dword [r0 + offsetof(zend_op_array, num_args)] 10371 | // num_args = EX_NUM_ARGS(); 10372 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] 10373 | // if (UNEXPECTED(num_args > first_extra_arg)) 10374 | cmp ecx, edx 10375 } 10376 | jg >1 10377 |.cold_code 10378 |1: 10379 if (!GCC_GLOBAL_REGS) { 10380 | mov FCARG1a, FP 10381 } 10382 | EXT_CALL zend_jit_copy_extra_args_helper, r0 10383 if (!func) { 10384 | mov r0, EX->func // reload 10385 } 10386 | mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload 10387 | jmp >1 10388 |.code 10389 if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { 10390 if (!func) { 10391 | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) 10392 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS 10393 | jnz >1 10394 } 10395 | // opline += num_args; 10396 |.if X64 10397 || ZEND_ASSERT(sizeof(zend_op) == 32); 10398 | mov edx, ecx 10399 | shl r2, 5 10400 |.else 10401 | imul r2, ecx, sizeof(zend_op) 10402 |.endif 10403 | ADD_IP r2 10404 } 10405 |1: 10406 | // if (EXPECTED((int)num_args < op_array->last_var)) { 10407 if (func) { 10408 | mov edx, (func->op_array.last_var) 10409 } else { 10410 | mov edx, dword [r0 + offsetof(zend_op_array, last_var)] 10411 } 10412 | sub edx, ecx 10413 | jle >3 //??? 10414 | // zval *var = EX_VAR_NUM(num_args); 10415// |.if X64 10416// | movsxd r1, ecx 10417// |.endif 10418 | shl r1, 4 10419 | lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))] 10420 |2: 10421 | SET_Z_TYPE_INFO r1, IS_UNDEF 10422 | sub edx, 1 10423 | lea r1, [r1 + 16] 10424 | jne <2 10425 |3: 10426 } 10427 10428 if (ZEND_OBSERVER_ENABLED) { 10429 | SAVE_IP 10430 | mov FCARG1a, FP 10431 | EXT_CALL zend_observer_fcall_begin, r0 10432 } 10433 10434 if (trace) { 10435 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 10436 | jmp >9 10437 } 10438 } else { 10439#ifdef CONTEXT_THREADED_JIT 10440 | call ->context_threaded_call 10441 if (!func && (opline->opcode != ZEND_DO_UCALL)) { 10442 | jmp >9 10443 } 10444 | call ->context_threaded_call 10445 if (!func) { 10446 | jmp >9 10447 } 10448#else 10449 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 10450 | ADD_HYBRID_SPAD 10451 | JMP_IP 10452 } else if (GCC_GLOBAL_REGS) { 10453 | add r4, SPAD // stack alignment 10454 | JMP_IP 10455 } else { 10456 | mov FP, aword T2 // restore FP 10457 | mov RX, aword T3 // restore IP 10458 | add r4, NR_SPAD // stack alignment 10459 | mov r0, 1 // ZEND_VM_ENTER 10460 | ret 10461 } 10462 } 10463#endif 10464 } 10465 10466 if ((!func || func->type == ZEND_INTERNAL_FUNCTION) 10467 && (opline->opcode != ZEND_DO_UCALL)) { 10468 if (!func && (opline->opcode != ZEND_DO_ICALL)) { 10469 |8: 10470 } 10471 if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { 10472 if (!func) { 10473 if (trace) { 10474 uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10475 10476 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10477 if (!exit_addr) { 10478 return 0; 10479 } 10480 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10481 | jnz &exit_addr 10482 } else { 10483 | test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED 10484 | jnz >1 10485 |.cold_code 10486 |1: 10487 if (!GCC_GLOBAL_REGS) { 10488 | mov FCARG1a, RX 10489 } 10490 | EXT_CALL zend_jit_deprecated_helper, r0 10491 | test al, al 10492 | mov r0, EX:RX->func // reload 10493 | jne >1 10494 | jmp ->exception_handler 10495 |.code 10496 |1: 10497 } 10498 } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { 10499 if (!GCC_GLOBAL_REGS) { 10500 | mov FCARG1a, RX 10501 } 10502 | EXT_CALL zend_jit_deprecated_helper, r0 10503 | test al, al 10504 | je ->exception_handler 10505 | mov r0, EX:RX->func // reload 10506 } 10507 } 10508 10509 | // ZVAL_NULL(EX_VAR(opline->result.var)); 10510 | LOAD_ZVAL_ADDR FCARG2a, res_addr 10511 | SET_Z_TYPE_INFO FCARG2a, IS_NULL 10512 10513 | // EG(current_execute_data) = execute_data; 10514 | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, RX, r1 10515 10516 zend_jit_reset_last_valid_opline(); 10517 10518 | // fbc->internal_function.handler(call, ret); 10519 | mov FCARG1a, RX 10520 if (func) { 10521 | EXT_CALL func->internal_function.handler, r0 10522 } else { 10523 | call aword [r0 + offsetof(zend_internal_function, handler)] 10524 } 10525 10526 | // EG(current_execute_data) = execute_data; 10527 | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0 10528 10529 | // zend_vm_stack_free_args(call); 10530 if (func && !unknown_num_args) { 10531 for (i = 0; i < call_num_args; i++ ) { 10532 uint32_t offset = EX_NUM_TO_VAR(i); 10533 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, opline 10534 } 10535 } else { 10536 | mov FCARG1a, RX 10537 | EXT_CALL zend_jit_vm_stack_free_args_helper, r0 10538 } 10539 if (may_have_extra_named_params) { 10540 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24) 10541 | jnz >1 10542 |.cold_code 10543 |1: 10544 | mov FCARG1a, aword [RX + offsetof(zend_execute_data, extra_named_params)] 10545 | EXT_CALL zend_free_extra_named_params, r0 10546 | jmp >2 10547 |.code 10548 |2: 10549 } 10550 10551 |8: 10552 if (opline->opcode == ZEND_DO_FCALL) { 10553 // TODO: optimize ??? 10554 | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) 10555 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_RELEASE_THIS >> 16) 10556 | jnz >1 10557 |.cold_code 10558 |1: 10559 | GET_Z_PTR FCARG1a, RX + offsetof(zend_execute_data, This) 10560 | // OBJ_RELEASE(object); 10561 | OBJ_RELEASE ZREG_FCARG1a, >2 10562 | jmp >2 10563 |.code 10564 |2: 10565 } 10566 10567 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 10568 !JIT_G(current_frame) || 10569 !JIT_G(current_frame)->call || 10570 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || 10571 prev_opline->opcode == ZEND_SEND_UNPACK || 10572 prev_opline->opcode == ZEND_SEND_ARRAY || 10573 prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { 10574 10575 | // zend_vm_stack_free_call_frame(call); 10576 | test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16) 10577 | jnz >1 10578 |.cold_code 10579 |1: 10580 | mov FCARG1a, RX 10581 | EXT_CALL zend_jit_free_call_frame, r0 10582 | jmp >1 10583 |.code 10584 } 10585 | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, RX, r0 10586 |1: 10587 10588 if (!RETURN_VALUE_USED(opline)) { 10589 zend_class_entry *ce; 10590 zend_bool ce_is_instanceof; 10591 uint32_t func_info = call_info ? 10592 zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : 10593 (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); 10594 10595 /* If an exception is thrown, the return_value may stay at the 10596 * original value of null. */ 10597 func_info |= MAY_BE_NULL; 10598 10599 if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 10600 | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline 10601 } 10602 } 10603 10604 | // if (UNEXPECTED(EG(exception) != NULL)) { 10605 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 10606 | jne ->icall_throw_handler 10607 10608 // TODO: Can we avoid checking for interrupts after each call ??? 10609 if (trace && last_valid_opline != opline) { 10610 int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); 10611 10612 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10613 if (!exit_addr) { 10614 return 0; 10615 } 10616 } else { 10617 exit_addr = NULL; 10618 } 10619 if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { 10620 return 0; 10621 } 10622 10623 if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { 10624 | LOAD_IP_ADDR (opline + 1) 10625 } else if (trace 10626 && trace->op == ZEND_JIT_TRACE_END 10627 && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { 10628 | LOAD_IP_ADDR (opline + 1) 10629 } 10630 } 10631 10632 if (!func) { 10633 |9: 10634 } 10635 10636 return 1; 10637} 10638 10639static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) 10640{ 10641 uint32_t arg_num = opline->op2.num; 10642 zend_jit_addr arg_addr; 10643 10644 ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); 10645 10646 if (!zend_jit_reuse_ip(Dst)) { 10647 return 0; 10648 } 10649 10650 if (opline->opcode == ZEND_SEND_VAL_EX) { 10651 uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); 10652 10653 ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); 10654 10655 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10656 && JIT_G(current_frame) 10657 && JIT_G(current_frame)->call 10658 && JIT_G(current_frame)->call->func) { 10659 if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10660 /* Don't generate code that always throws exception */ 10661 return 0; 10662 } 10663 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10664 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10665 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10666 if (!exit_addr) { 10667 return 0; 10668 } 10669 | mov r0, EX:RX->func 10670 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10671 | jnz &exit_addr 10672 } else { 10673 | mov r0, EX:RX->func 10674 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10675 | jnz >1 10676 |.cold_code 10677 |1: 10678 | SET_EX_OPLINE opline, r0 10679 | jmp ->throw_cannot_pass_by_ref 10680 |.code 10681 10682 } 10683 } 10684 10685 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10686 10687 if (opline->op1_type == IS_CONST) { 10688 zval *zv = RT_CONSTANT(opline, opline->op1); 10689 10690 | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 10691 if (Z_REFCOUNTED_P(zv)) { 10692 | ADDREF_CONST zv, r0 10693 } 10694 } else { 10695 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10696 } 10697 10698 return 1; 10699} 10700 10701static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) 10702{ 10703 | mov FCARG1a, EX->call 10704 | test byte [FCARG1a + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_MAY_HAVE_UNDEF >> 24) 10705 | jnz >1 10706 |.cold_code 10707 |1: 10708 | SET_EX_OPLINE opline, r0 10709 | EXT_CALL zend_handle_undef_args, r0 10710 | test r0, r0 10711 | jnz ->exception_handler 10712 | jmp >2 10713 |.code 10714 |2: 10715 10716 return 1; 10717} 10718 10719static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) 10720{ 10721 zend_jit_addr op1_addr, arg_addr, ref_addr; 10722 10723 op1_addr = OP1_ADDR(); 10724 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10725 10726 if (!zend_jit_reuse_ip(Dst)) { 10727 return 0; 10728 } 10729 10730 if (opline->op1_type == IS_VAR) { 10731 if (op1_info & MAY_BE_INDIRECT) { 10732 | LOAD_ZVAL_ADDR r0, op1_addr 10733 | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { 10734 | IF_NOT_Z_TYPE r0, IS_INDIRECT, >1 10735 | // ret = Z_INDIRECT_P(ret); 10736 | GET_Z_PTR r0, r0 10737 |1: 10738 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 10739 } 10740 } else if (opline->op1_type == IS_CV) { 10741 if (op1_info & MAY_BE_UNDEF) { 10742 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10743 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 10744 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 10745 | jmp >2 10746 |1: 10747 } 10748 op1_info &= ~MAY_BE_UNDEF; 10749 op1_info |= MAY_BE_NULL; 10750 } 10751 } else { 10752 ZEND_UNREACHABLE(); 10753 } 10754 10755 if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { 10756 if (op1_info & MAY_BE_REF) { 10757 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2 10758 | GET_ZVAL_PTR r1, op1_addr 10759 | GC_ADDREF r1 10760 | SET_ZVAL_PTR arg_addr, r1 10761 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX 10762 | jmp >6 10763 } 10764 |2: 10765 | // ZVAL_NEW_REF(arg, varptr); 10766 if (opline->op1_type == IS_VAR) { 10767 if (Z_REG(op1_addr) != ZREG_R0 || Z_OFFSET(op1_addr) != 0) { 10768 | LOAD_ZVAL_ADDR r0, op1_addr 10769 } 10770 | mov aword T1, r0 // save 10771 } 10772 | EMALLOC sizeof(zend_reference), op_array, opline 10773 | mov dword [r0], 2 10774 | mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], GC_REFERENCE 10775 | mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0 10776 ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); 10777 if (opline->op1_type == IS_VAR) { 10778 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); 10779 10780 | mov r1, aword T1 // restore 10781 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R2, ZREG_R2 10782 | SET_ZVAL_PTR val_addr, r0 10783 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX 10784 } else { 10785 | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10786 | SET_ZVAL_PTR op1_addr, r0 10787 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 10788 } 10789 | SET_ZVAL_PTR arg_addr, r0 10790 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX 10791 } 10792 10793 |6: 10794 | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline 10795 |7: 10796 10797 return 1; 10798} 10799 10800static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) 10801{ 10802 uint32_t arg_num = opline->op2.num; 10803 zend_jit_addr arg_addr; 10804 10805 ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && 10806 opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || 10807 arg_num <= MAX_ARG_FLAG_NUM); 10808 10809 arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); 10810 10811 if (!zend_jit_reuse_ip(Dst)) { 10812 return 0; 10813 } 10814 10815 if (opline->opcode == ZEND_SEND_VAR_EX) { 10816 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10817 && JIT_G(current_frame) 10818 && JIT_G(current_frame)->call 10819 && JIT_G(current_frame)->call->func) { 10820 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10821 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10822 return 0; 10823 } 10824 return 1; 10825 } 10826 } else { 10827 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10828 10829 | mov r0, EX:RX->func 10830 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10831 | jnz >1 10832 |.cold_code 10833 |1: 10834 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10835 return 0; 10836 } 10837 | jmp >7 10838 |.code 10839 } 10840 } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { 10841 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10842 && JIT_G(current_frame) 10843 && JIT_G(current_frame)->call 10844 && JIT_G(current_frame)->call->func) { 10845 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10846 10847 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10848 10849 if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10850 if (!(op1_info & MAY_BE_REF)) { 10851 /* Don't generate code that always throws exception */ 10852 return 0; 10853 } else { 10854 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10855 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10856 if (!exit_addr) { 10857 return 0; 10858 } 10859 | cmp cl, IS_REFERENCE 10860 | jne &exit_addr 10861 } 10862 } 10863 return 1; 10864 } 10865 } else { 10866 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 10867 10868 | mov r0, EX:RX->func 10869 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10870 | jnz >1 10871 |.cold_code 10872 |1: 10873 10874 mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); 10875 10876 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10877 if (op1_info & MAY_BE_REF) { 10878 | cmp cl, IS_REFERENCE 10879 | je >7 10880 } 10881 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 10882 | jnz >7 10883 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10884 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10885 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10886 if (!exit_addr) { 10887 return 0; 10888 } 10889 | jmp &exit_addr 10890 } else { 10891 | SET_EX_OPLINE opline, r0 10892 | LOAD_ZVAL_ADDR FCARG1a, arg_addr 10893 | EXT_CALL zend_jit_only_vars_by_reference, r0 10894 if (!zend_jit_check_exception(Dst)) { 10895 return 0; 10896 } 10897 | jmp >7 10898 } 10899 10900 |.code 10901 } 10902 } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { 10903 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 10904 && JIT_G(current_frame) 10905 && JIT_G(current_frame)->call 10906 && JIT_G(current_frame)->call->func) { 10907 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 10908 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { 10909 return 0; 10910 } 10911 return 1; 10912 } 10913 } else { 10914 | test dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 10915 | jnz >1 10916 |.cold_code 10917 |1: 10918 if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { 10919 return 0; 10920 } 10921 | jmp >7 10922 |.code 10923 } 10924 } 10925 10926 if (op1_info & MAY_BE_UNDEF) { 10927 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10928 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 10929 |.cold_code 10930 |1: 10931 } 10932 10933 | SET_EX_OPLINE opline, r0 10934 | mov FCARG1d, opline->op1.var 10935 | EXT_CALL zend_jit_undefined_op_helper, r0 10936 | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL 10937 | test r0, r0 10938 | jz ->exception_handler 10939 10940 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 10941 | jmp >7 10942 |.code 10943 } else { 10944 |7: 10945 return 1; 10946 } 10947 } 10948 10949 if (opline->opcode == ZEND_SEND_VAR_NO_REF) { 10950 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2 10951 if (op1_info & MAY_BE_REF) { 10952 | cmp cl, IS_REFERENCE 10953 | je >7 10954 } 10955 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 10956 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 10957 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 10958 if (!exit_addr) { 10959 return 0; 10960 } 10961 | jmp &exit_addr 10962 } else { 10963 | SET_EX_OPLINE opline, r0 10964 | LOAD_ZVAL_ADDR FCARG1a, arg_addr 10965 | EXT_CALL zend_jit_only_vars_by_reference, r0 10966 if (!zend_jit_check_exception(Dst)) { 10967 return 0; 10968 } 10969 } 10970 } else { 10971 if (op1_info & MAY_BE_REF) { 10972 if (opline->op1_type == IS_CV) { 10973 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 10974 10975 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 10976 | ZVAL_DEREF FCARG1a, op1_info 10977 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R0, ZREG_R2 10978 | TRY_ADDREF op1_info, ah, r2 10979 } else { 10980 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 8); 10981 10982 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 10983 |.cold_code 10984 |1: 10985 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 10986 | GET_ZVAL_PTR FCARG1a, op1_addr 10987 | // ZVAL_COPY_VALUE(return_value, &ref->value); 10988 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R0, ZREG_R2 10989 | GC_DELREF FCARG1a 10990 | je >1 10991 | IF_NOT_REFCOUNTED ah, >2 10992 | GC_ADDREF r2 10993 | jmp >2 10994 |1: 10995 | EFREE_REG_REFERENCE 10996 | jmp >2 10997 |.code 10998 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 10999 |2: 11000 } 11001 } else { 11002 if (op1_addr != op1_def_addr) { 11003 if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { 11004 return 0; 11005 } 11006 if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { 11007 op1_addr= op1_def_addr; 11008 } 11009 } 11010 | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11011 if (opline->op1_type == IS_CV) { 11012 | TRY_ADDREF op1_info, ah, r2 11013 } 11014 } 11015 } 11016 |7: 11017 11018 return 1; 11019} 11020 11021static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) 11022{ 11023 uint32_t arg_num = opline->op2.num; 11024 11025 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 11026 && JIT_G(current_frame) 11027 && JIT_G(current_frame)->call 11028 && JIT_G(current_frame)->call->func) { 11029 if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { 11030 if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { 11031 TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); 11032 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 11033 || if (reuse_ip) { 11034 | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 11035 || } else { 11036 | mov r0, EX->call 11037 | or dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 11038 || } 11039 } 11040 } else { 11041 if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 11042 TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); 11043 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 11044 || if (reuse_ip) { 11045 | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 11046 || } else { 11047 | mov r0, EX->call 11048 | and dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 11049 || } 11050 } 11051 } 11052 } else { 11053 // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { 11054 uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); 11055 11056 if (!zend_jit_reuse_ip(Dst)) { 11057 return 0; 11058 } 11059 11060 | mov r0, EX:RX->func 11061 | test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask 11062 | jnz >1 11063 |.cold_code 11064 |1: 11065 | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 11066 | or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF 11067 | jmp >1 11068 |.code 11069 | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); 11070 | and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF 11071 |1: 11072 } 11073 11074 return 1; 11075} 11076 11077static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) 11078{ 11079 if (smart_branch_opcode) { 11080 if (smart_branch_opcode == ZEND_JMPZ) { 11081 if (jmp) { 11082 | jmp >7 11083 } 11084 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11085 | jmp =>target_label 11086 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11087 | jmp =>target_label2 11088 } else { 11089 ZEND_UNREACHABLE(); 11090 } 11091 } else { 11092 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11093 11094 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 11095 if (jmp) { 11096 | jmp >7 11097 } 11098 } 11099 11100 return 1; 11101} 11102 11103static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) 11104{ 11105 if (smart_branch_opcode) { 11106 if (smart_branch_opcode == ZEND_JMPZ) { 11107 | jmp =>target_label 11108 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11109 if (jmp) { 11110 | jmp >7 11111 } 11112 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11113 | jmp =>target_label 11114 } else { 11115 ZEND_UNREACHABLE(); 11116 } 11117 } else { 11118 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11119 11120 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 11121 if (jmp) { 11122 | jmp >7 11123 } 11124 } 11125 11126 return 1; 11127} 11128 11129static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 11130{ 11131 uint32_t defined_label = (uint32_t)-1; 11132 uint32_t undefined_label = (uint32_t)-1; 11133 zval *zv = RT_CONSTANT(opline, opline->op1); 11134 zend_jit_addr res_addr = 0; 11135 11136 if (smart_branch_opcode && !exit_addr) { 11137 if (smart_branch_opcode == ZEND_JMPZ) { 11138 undefined_label = target_label; 11139 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11140 defined_label = target_label; 11141 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11142 undefined_label = target_label; 11143 defined_label = target_label2; 11144 } else { 11145 ZEND_UNREACHABLE(); 11146 } 11147 } 11148 11149 | // if (CACHED_PTR(opline->extended_value)) { 11150 | mov r0, EX->run_time_cache 11151 | mov r0, aword [r0 + opline->extended_value] 11152 | test r0, r0 11153 | jz >1 11154 | test r0, 0x1 11155 | jnz >4 11156 |.cold_code 11157 |4: 11158 | MEM_OP2_2_ZTS mov, FCARG1a, aword, executor_globals, zend_constants, FCARG1a 11159 | shr r0, 1 11160 | cmp dword [FCARG1a + offsetof(HashTable, nNumOfElements)], eax 11161 11162 if (smart_branch_opcode) { 11163 if (exit_addr) { 11164 if (smart_branch_opcode == ZEND_JMPZ) { 11165 | jz &exit_addr 11166 } else { 11167 | jz >3 11168 } 11169 } else if (undefined_label != (uint32_t)-1) { 11170 | jz =>undefined_label 11171 } else { 11172 | jz >3 11173 } 11174 } else { 11175 | jz >2 11176 } 11177 |1: 11178 | SET_EX_OPLINE opline, r0 11179 | LOAD_ADDR FCARG1a, zv 11180 | EXT_CALL zend_jit_check_constant, r0 11181 | test r0, r0 11182 if (exit_addr) { 11183 if (smart_branch_opcode == ZEND_JMPNZ) { 11184 | jz >3 11185 } else { 11186 | jnz >3 11187 } 11188 | jmp &exit_addr 11189 } else if (smart_branch_opcode) { 11190 if (undefined_label != (uint32_t)-1) { 11191 | jz =>undefined_label 11192 } else { 11193 | jz >3 11194 } 11195 if (defined_label != (uint32_t)-1) { 11196 | jmp =>defined_label 11197 } else { 11198 | jmp >3 11199 } 11200 } else { 11201 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11202 | jnz >1 11203 |2: 11204 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 11205 | jmp >3 11206 } 11207 |.code 11208 if (smart_branch_opcode) { 11209 if (exit_addr) { 11210 if (smart_branch_opcode == ZEND_JMPNZ) { 11211 | jmp &exit_addr 11212 } 11213 } else if (defined_label != (uint32_t)-1) { 11214 | jmp =>defined_label 11215 } 11216 } else { 11217 |1: 11218 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 11219 } 11220 |3: 11221 11222 return 1; 11223} 11224 11225static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 11226{ 11227 uint32_t mask; 11228 zend_uchar type; 11229 zend_jit_addr op1_addr = OP1_ADDR(); 11230 11231 // TODO: support for is_resource() ??? 11232 ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); 11233 11234 if (op1_info & MAY_BE_UNDEF) { 11235 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11236 | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1 11237 |.cold_code 11238 |1: 11239 } 11240 | SET_EX_OPLINE opline, r0 11241 | mov FCARG1d, opline->op1.var 11242 | EXT_CALL zend_jit_undefined_op_helper, r0 11243 zend_jit_check_exception_undef_result(Dst, opline); 11244 if (opline->extended_value & MAY_BE_NULL) { 11245 if (exit_addr) { 11246 if (smart_branch_opcode == ZEND_JMPNZ) { 11247 | jmp &exit_addr 11248 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 11249 | jmp >7 11250 } 11251 } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { 11252 return 0; 11253 } 11254 } else { 11255 if (exit_addr) { 11256 if (smart_branch_opcode == ZEND_JMPZ) { 11257 | jmp &exit_addr 11258 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { 11259 | jmp >7 11260 } 11261 } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { 11262 return 0; 11263 } 11264 } 11265 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11266 |.code 11267 } 11268 } 11269 11270 if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { 11271 mask = opline->extended_value; 11272 switch (mask) { 11273 case MAY_BE_NULL: type = IS_NULL; break; 11274 case MAY_BE_FALSE: type = IS_FALSE; break; 11275 case MAY_BE_TRUE: type = IS_TRUE; break; 11276 case MAY_BE_LONG: type = IS_LONG; break; 11277 case MAY_BE_DOUBLE: type = IS_DOUBLE; break; 11278 case MAY_BE_STRING: type = IS_STRING; break; 11279 case MAY_BE_ARRAY: type = IS_ARRAY; break; 11280 case MAY_BE_OBJECT: type = IS_OBJECT; break; 11281 default: 11282 type = 0; 11283 } 11284 11285 if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { 11286 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11287 if (exit_addr) { 11288 if (smart_branch_opcode == ZEND_JMPNZ) { 11289 | jmp &exit_addr 11290 } 11291 } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { 11292 return 0; 11293 } 11294 } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { 11295 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11296 if (exit_addr) { 11297 if (smart_branch_opcode == ZEND_JMPZ) { 11298 | jmp &exit_addr 11299 } 11300 } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { 11301 return 0; 11302 } 11303 } else { 11304 if (op1_info & MAY_BE_REF) { 11305 | LOAD_ZVAL_ADDR r0, op1_addr 11306 | ZVAL_DEREF r0, op1_info 11307 } 11308 if (type == 0) { 11309 if (smart_branch_opcode && 11310 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11311 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11312 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11313 | // if (Z_REFCOUNTED_P(cv)) { 11314 | IF_ZVAL_REFCOUNTED op1_addr, >1 11315 |.cold_code 11316 |1: 11317 } 11318 | // if (!Z_DELREF_P(cv)) { 11319 | GET_ZVAL_PTR FCARG1a, op1_addr 11320 | GC_DELREF FCARG1a 11321 if (RC_MAY_BE_1(op1_info)) { 11322 if (RC_MAY_BE_N(op1_info)) { 11323 | jnz >3 11324 } 11325 if (op1_info & MAY_BE_REF) { 11326 | mov al, byte [r0 + 8] 11327 } else { 11328 | mov al, byte [FP + opline->op1.var + 8] 11329 } 11330 | mov byte T1, al // save 11331 | // zval_dtor_func(r); 11332 | ZVAL_DTOR_FUNC op1_info, opline 11333 | mov cl, byte T1 // restore 11334 |jmp >2 11335 } 11336 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11337 if (!RC_MAY_BE_1(op1_info)) { 11338 | jmp >3 11339 } 11340 |.code 11341 } 11342 |3: 11343 if (op1_info & MAY_BE_REF) { 11344 | mov cl, byte [r0 + 8] 11345 } else { 11346 | mov cl, byte [FP + opline->op1.var + 8] 11347 } 11348 |2: 11349 } else { 11350 if (op1_info & MAY_BE_REF) { 11351 | mov cl, byte [r0 + 8] 11352 } else { 11353 | mov cl, byte [FP + opline->op1.var + 8] 11354 } 11355 } 11356 | mov eax, 1 11357 | shl eax, cl 11358 | test eax, mask 11359 if (exit_addr) { 11360 if (smart_branch_opcode == ZEND_JMPNZ) { 11361 | jne &exit_addr 11362 } else { 11363 | je &exit_addr 11364 } 11365 } else if (smart_branch_opcode) { 11366 if (smart_branch_opcode == ZEND_JMPZ) { 11367 | je =>target_label 11368 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11369 | jne =>target_label 11370 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11371 | je =>target_label 11372 | jmp =>target_label2 11373 } else { 11374 ZEND_UNREACHABLE(); 11375 } 11376 } else { 11377 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11378 11379 | setne al 11380 | movzx eax, al 11381 | add eax, 2 11382 | SET_ZVAL_TYPE_INFO res_addr, eax 11383 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11384 } 11385 } else { 11386 if (smart_branch_opcode && 11387 (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11388 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11389 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11390 | // if (Z_REFCOUNTED_P(cv)) { 11391 | IF_ZVAL_REFCOUNTED op1_addr, >1 11392 |.cold_code 11393 |1: 11394 } 11395 | // if (!Z_DELREF_P(cv)) { 11396 | GET_ZVAL_PTR FCARG1a, op1_addr 11397 | GC_DELREF FCARG1a 11398 if (RC_MAY_BE_1(op1_info)) { 11399 if (RC_MAY_BE_N(op1_info)) { 11400 | jnz >3 11401 } 11402 if (op1_info & MAY_BE_REF) { 11403 | mov al, byte [r0 + 8] 11404 } else { 11405 | mov al, byte [FP + opline->op1.var + 8] 11406 } 11407 | mov byte T1, al // save 11408 | // zval_dtor_func(r); 11409 | ZVAL_DTOR_FUNC op1_info, opline 11410 | mov cl, byte T1 // restore 11411 |jmp >2 11412 } 11413 if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11414 if (!RC_MAY_BE_1(op1_info)) { 11415 | jmp >3 11416 } 11417 |.code 11418 } 11419 |3: 11420 if (op1_info & MAY_BE_REF) { 11421 | mov cl, byte [r0 + 8] 11422 } else { 11423 | mov cl, byte [FP + opline->op1.var + 8] 11424 } 11425 |2: 11426 | cmp cl, type 11427 } else { 11428 if (op1_info & MAY_BE_REF) { 11429 | cmp byte [r0 + 8], type 11430 } else { 11431 | cmp byte [FP + opline->op1.var + 8], type 11432 } 11433 } 11434 if (exit_addr) { 11435 if (smart_branch_opcode == ZEND_JMPNZ) { 11436 | je &exit_addr 11437 } else { 11438 | jne &exit_addr 11439 } 11440 } else if (smart_branch_opcode) { 11441 if (smart_branch_opcode == ZEND_JMPZ) { 11442 | jne =>target_label 11443 } else if (smart_branch_opcode == ZEND_JMPNZ) { 11444 | je =>target_label 11445 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 11446 | jne =>target_label 11447 | jmp =>target_label2 11448 } else { 11449 ZEND_UNREACHABLE(); 11450 } 11451 } else { 11452 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 11453 11454 | sete al 11455 | movzx eax, al 11456 | add eax, 2 11457 | SET_ZVAL_TYPE_INFO res_addr, eax 11458 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 11459 } 11460 } 11461 } 11462 } 11463 11464 |7: 11465 11466 return 1; 11467} 11468 11469static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var) 11470{ 11471 uint32_t j, info; 11472 11473 if (ssa->vars && ssa->var_info) { 11474 info = ssa->var_info[var].type; 11475 for (j = op_array->last_var; j < ssa->vars_count; j++) { 11476 if (ssa->vars[j].var == var) { 11477 info |= ssa->var_info[j].type; 11478 } 11479 } 11480 } else { 11481 info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF | 11482 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 11483 } 11484 11485#ifdef ZEND_JIT_USE_RC_INFERENCE 11486 /* Refcount may be increased by RETURN opcode */ 11487 if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) { 11488 for (j = 0; j < ssa->cfg.blocks_count; j++) { 11489 if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) && 11490 ssa->cfg.blocks[j].len > 0) { 11491 const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1; 11492 11493 if (opline->opcode == ZEND_RETURN) { 11494 if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) { 11495 info |= MAY_BE_RCN; 11496 break; 11497 } 11498 } 11499 } 11500 } 11501 } 11502#endif 11503 11504 return info; 11505} 11506 11507static int zend_jit_leave_frame(dasm_State **Dst) 11508{ 11509 | // EG(current_execute_data) = EX(prev_execute_data); 11510 | mov r0, EX->prev_execute_data 11511 | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, r0, r2 11512 return 1; 11513} 11514 11515static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) 11516{ 11517 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11518 uint32_t offset = EX_NUM_TO_VAR(var); 11519 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, NULL 11520 } 11521 return 1; 11522} 11523 11524static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) 11525{ 11526 if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 11527 | ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, 1, opline 11528 } 11529 return 1; 11530} 11531 11532static int zend_jit_leave_func(dasm_State **Dst, 11533 const zend_op_array *op_array, 11534 const zend_op *opline, 11535 uint32_t op1_info, 11536 zend_bool left_frame, 11537 zend_jit_trace_rec *trace, 11538 zend_jit_trace_info *trace_info, 11539 int indirect_var_access, 11540 int may_throw) 11541{ 11542 zend_bool may_be_top_frame = 11543 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11544 !JIT_G(current_frame) || 11545 !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); 11546 zend_bool may_need_call_helper = 11547 indirect_var_access || /* may have symbol table */ 11548 !op_array->function_name || /* may have symbol table */ 11549 may_be_top_frame || 11550 (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ 11551 JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11552 !JIT_G(current_frame) || 11553 TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ 11554 (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ 11555 zend_bool may_need_release_this = 11556 !(op_array->fn_flags & ZEND_ACC_CLOSURE) && 11557 op_array->scope && 11558 !(op_array->fn_flags & ZEND_ACC_STATIC) && 11559 (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11560 !JIT_G(current_frame) || 11561 !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame))); 11562 11563 if (may_need_call_helper || may_need_release_this) { 11564 | mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)] 11565 } 11566 if (may_need_call_helper) { 11567 if (!left_frame) { 11568 left_frame = 1; 11569 if (!zend_jit_leave_frame(Dst)) { 11570 return 0; 11571 } 11572 } 11573 /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ 11574 | test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) 11575 if (trace && trace->op != ZEND_JIT_TRACE_END) { 11576 | jnz >1 11577 |.cold_code 11578 |1: 11579 if (!GCC_GLOBAL_REGS) { 11580 | mov FCARG2a, FP 11581 } 11582 | EXT_CALL zend_jit_leave_func_helper, r0 11583 11584 if (may_be_top_frame) { 11585 // TODO: try to avoid this check ??? 11586 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 11587#if 0 11588 /* this check should be handled by the following OPLINE guard */ 11589 | cmp IP, zend_jit_halt_op 11590 | je ->trace_halt 11591#endif 11592 } else if (GCC_GLOBAL_REGS) { 11593 | test IP, IP 11594 | je ->trace_halt 11595 } else { 11596 | test eax, eax 11597 | jl ->trace_halt 11598 } 11599 } 11600 11601 if (!GCC_GLOBAL_REGS) { 11602 | // execute_data = EG(current_execute_data) 11603 | MEM_OP2_2_ZTS mov, FP, aword, executor_globals, current_execute_data, r0 11604 } 11605 | jmp >8 11606 |.code 11607 } else { 11608 | jnz ->leave_function_handler 11609 } 11610 } 11611 11612 if (op_array->fn_flags & ZEND_ACC_CLOSURE) { 11613 if (!left_frame) { 11614 left_frame = 1; 11615 if (!zend_jit_leave_frame(Dst)) { 11616 return 0; 11617 } 11618 } 11619 | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); 11620 | mov FCARG1a, EX->func 11621 | sub FCARG1a, sizeof(zend_object) 11622 | OBJ_RELEASE ZREG_FCARG1a, >4 11623 |4: 11624 } else if (may_need_release_this) { 11625 if (!left_frame) { 11626 left_frame = 1; 11627 if (!zend_jit_leave_frame(Dst)) { 11628 return 0; 11629 } 11630 } 11631 | // if (call_info & ZEND_CALL_RELEASE_THIS) 11632 | test FCARG1d, ZEND_CALL_RELEASE_THIS 11633 | je >4 11634 | // zend_object *object = Z_OBJ(execute_data->This); 11635 | mov FCARG1a, EX->This.value.obj 11636 | // OBJ_RELEASE(object); 11637 | OBJ_RELEASE ZREG_FCARG1a, >4 11638 |4: 11639 // TODO: avoid EG(excption) check for $this->foo() calls 11640 may_throw = 1; 11641 } 11642 11643 | // EG(vm_stack_top) = (zval*)execute_data; 11644 | MEM_OP2_1_ZTS mov, aword, executor_globals, vm_stack_top, FP, r0 11645 | // execute_data = EX(prev_execute_data); 11646 | mov FP, EX->prev_execute_data 11647 11648 if (!left_frame) { 11649 | // EG(current_execute_data) = execute_data; 11650 | MEM_OP2_1_ZTS mov, aword, executor_globals, current_execute_data, FP, r0 11651 } 11652 11653 |9: 11654 if (trace) { 11655 if (trace->op != ZEND_JIT_TRACE_END 11656 && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 11657 zend_jit_reset_last_valid_opline(); 11658 } else { 11659 | LOAD_IP 11660 | ADD_IP sizeof(zend_op) 11661 } 11662 11663 |8: 11664 11665 if (trace->op == ZEND_JIT_TRACE_BACK 11666 && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { 11667 const zend_op *next_opline = trace->opline; 11668 11669 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11670 && (op1_info & MAY_BE_RC1) 11671 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { 11672 /* exception might be thrown during destruction of unused return value */ 11673 | // if (EG(exception)) 11674 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 11675 | jne ->leave_throw_handler 11676 } 11677 do { 11678 trace++; 11679 } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); 11680 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 11681 next_opline = trace->opline; 11682 ZEND_ASSERT(next_opline != NULL); 11683 11684 if (trace->op == ZEND_JIT_TRACE_END 11685 && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { 11686 trace_info->flags |= ZEND_JIT_TRACE_LOOP; 11687 | CMP_IP next_opline 11688 | je =>0 // LOOP 11689#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 11690 | JMP_IP 11691#else 11692 | jmp ->trace_escape 11693#endif 11694 } else { 11695 | CMP_IP next_opline 11696 | jne ->trace_escape 11697 } 11698 11699 zend_jit_set_last_valid_opline(trace->opline); 11700 11701 return 1; 11702 } else if (may_throw || 11703 (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 11704 && (op1_info & MAY_BE_RC1) 11705 && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) 11706 && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { 11707 | // if (EG(exception)) 11708 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 11709 | jne ->leave_throw_handler 11710 } 11711 11712 return 1; 11713 } else { 11714 | // if (EG(exception)) 11715 | MEM_OP2_1_ZTS cmp, aword, executor_globals, exception, 0, r0 11716 | LOAD_IP 11717 | jne ->leave_throw_handler 11718 | // opline = EX(opline) + 1 11719 | ADD_IP sizeof(zend_op) 11720 } 11721 11722 if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { 11723 | ADD_HYBRID_SPAD 11724#ifdef CONTEXT_THREADED_JIT 11725 | push aword [IP] 11726 | ret 11727#else 11728 | JMP_IP 11729#endif 11730 } else if (GCC_GLOBAL_REGS) { 11731 | add r4, SPAD // stack alignment 11732#ifdef CONTEXT_THREADED_JIT 11733 | push aword [IP] 11734 | ret 11735#else 11736 | JMP_IP 11737#endif 11738 } else { 11739#ifdef CONTEXT_THREADED_JIT 11740 ZEND_UNREACHABLE(); 11741 // TODO: context threading can't work without GLOBAL REGS because we have to change 11742 // the value of execute_data in execute_ex() 11743 | mov FCARG1a, FP 11744 | mov r0, aword [FP] 11745 | mov FP, aword T2 // restore FP 11746 | mov RX, aword T3 // restore IP 11747 | add r4, NR_SPAD // stack alignment 11748 | push aword [r0] 11749 | ret 11750#else 11751 | mov FP, aword T2 // restore FP 11752 | mov RX, aword T3 // restore IP 11753 | add r4, NR_SPAD // stack alignment 11754 | mov r0, 2 // ZEND_VM_LEAVE 11755 | ret 11756#endif 11757 } 11758 11759 return 1; 11760} 11761 11762static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) 11763{ 11764 zend_jit_addr ret_addr; 11765 int8_t return_value_used; 11766 11767 ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); 11768 ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); 11769 11770 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { 11771 if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { 11772 return_value_used = 1; 11773 } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { 11774 return_value_used = 0; 11775 } else { 11776 return_value_used = -1; 11777 } 11778 } else { 11779 return_value_used = -1; 11780 } 11781 11782 if (ZEND_OBSERVER_ENABLED) { 11783 if (Z_MODE(op1_addr) == IS_REG) { 11784 zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 11785 11786 if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { 11787 return 0; 11788 } 11789 op1_addr = dst; 11790 } 11791 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 11792 | mov FCARG1a, FP 11793 | SET_EX_OPLINE opline, r0 11794 | EXT_CALL zend_observer_fcall_end, r0 11795 } 11796 11797 // if (!EX(return_value)) 11798 if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) { 11799 if (return_value_used != 0) { 11800 | mov r2, EX->return_value 11801 } 11802 if (return_value_used == -1) { 11803 | test r2, r2 11804 } 11805 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0); 11806 } else { 11807 if (return_value_used != 0) { 11808 | mov r1, EX->return_value 11809 } 11810 if (return_value_used == -1) { 11811 | test r1, r1 11812 } 11813 ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0); 11814 } 11815 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && 11816 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11817 if (return_value_used == -1) { 11818 | jz >1 11819 |.cold_code 11820 |1: 11821 } 11822 if (return_value_used != 1) { 11823 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 11824 if (jit_return_label >= 0) { 11825 | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label 11826 } else { 11827 | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9 11828 } 11829 } 11830 | GET_ZVAL_PTR FCARG1a, op1_addr 11831 | GC_DELREF FCARG1a 11832 if (RC_MAY_BE_1(op1_info)) { 11833 if (RC_MAY_BE_N(op1_info)) { 11834 if (jit_return_label >= 0) { 11835 | jnz =>jit_return_label 11836 } else { 11837 | jnz >9 11838 } 11839 } 11840 | //SAVE_OPLINE() 11841 | ZVAL_DTOR_FUNC op1_info, opline 11842 | //????mov r1, EX->return_value // reload ??? 11843 } 11844 if (return_value_used == -1) { 11845 if (jit_return_label >= 0) { 11846 | jmp =>jit_return_label 11847 } else { 11848 | jmp >9 11849 } 11850 |.code 11851 } 11852 } 11853 } else if (return_value_used == -1) { 11854 if (jit_return_label >= 0) { 11855 | jz =>jit_return_label 11856 } else { 11857 | jz >9 11858 } 11859 } 11860 11861 if (return_value_used == 0) { 11862 |9: 11863 return 1; 11864 } 11865 11866 if (opline->op1_type == IS_CONST) { 11867 zval *zv = RT_CONSTANT(opline, opline->op1); 11868 | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 11869 if (Z_REFCOUNTED_P(zv)) { 11870 | ADDREF_CONST zv, r0 11871 } 11872 } else if (opline->op1_type == IS_TMP_VAR) { 11873 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11874 } else if (opline->op1_type == IS_CV) { 11875 if (op1_info & MAY_BE_REF) { 11876 | LOAD_ZVAL_ADDR r0, op1_addr 11877 | ZVAL_DEREF r0, op1_info 11878 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 11879 } 11880 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11881 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 11882 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 11883 (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || 11884 !op_array->function_name) { 11885 | TRY_ADDREF op1_info, ah, r2 11886 } else if (return_value_used != 1) { 11887 | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); 11888 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL 11889 } 11890 } 11891 } else { 11892 if (op1_info & MAY_BE_REF) { 11893 zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val)); 11894 11895 | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1 11896 |.cold_code 11897 |1: 11898 | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); 11899 | GET_ZVAL_PTR r0, op1_addr 11900 | // ZVAL_COPY_VALUE(return_value, &ref->value); 11901 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R2, ZREG_R2 11902 | GC_DELREF r0 11903 | je >2 11904 | // if (IS_REFCOUNTED()) 11905 if (jit_return_label >= 0) { 11906 | IF_NOT_REFCOUNTED dh, =>jit_return_label 11907 } else { 11908 | IF_NOT_REFCOUNTED dh, >9 11909 } 11910 | // ADDREF 11911 | GET_ZVAL_PTR r2, ret_addr // reload 11912 | GC_ADDREF r2 11913 if (jit_return_label >= 0) { 11914 | jmp =>jit_return_label 11915 } else { 11916 | jmp >9 11917 } 11918 |2: 11919 | EFREE_REFERENCE r0 11920 if (jit_return_label >= 0) { 11921 | jmp =>jit_return_label 11922 } else { 11923 | jmp >9 11924 } 11925 |.code 11926 } 11927 | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2 11928 } 11929 11930 |9: 11931 return 1; 11932} 11933 11934static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) 11935{ 11936 ZEND_ASSERT(type_reg == ZREG_R2); 11937 11938 |.if not(X64) 11939 || if (Z_REG(val_addr) == ZREG_R1) { 11940 | GET_ZVAL_W2 r0, val_addr 11941 || } 11942 |.endif 11943 | GET_ZVAL_PTR r1, val_addr 11944 |.if not(X64) 11945 || if (Z_REG(val_addr) != ZREG_R1) { 11946 | GET_ZVAL_W2 r0, val_addr 11947 || } 11948 |.endif 11949 | IF_NOT_REFCOUNTED dh, >2 11950 | IF_NOT_TYPE dl, IS_REFERENCE, >1 11951 | GET_Z_TYPE_INFO edx, r1+offsetof(zend_reference, val) 11952 |.if not(X64) 11953 | GET_Z_W2 r0, r1+offsetof(zend_reference, val) 11954 |.endif 11955 | GET_Z_PTR r1, r1+offsetof(zend_reference, val) 11956 | IF_NOT_REFCOUNTED dh, >2 11957 |1: 11958 | GC_ADDREF r1 11959 |2: 11960 | SET_ZVAL_PTR res_addr, r1 11961 |.if not(X64) 11962 | SET_ZVAL_W2 res_addr, r0 11963 |.endif 11964 | SET_ZVAL_TYPE_INFO res_addr, edx 11965 11966 return 1; 11967} 11968 11969static zend_bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_info) 11970{ 11971 switch (opline->opcode) { 11972 case ZEND_FETCH_OBJ_FUNC_ARG: 11973 if (!JIT_G(current_frame) || 11974 !JIT_G(current_frame)->call->func || 11975 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 11976 return 0; 11977 } 11978 /* break missing intentionally */ 11979 case ZEND_FETCH_OBJ_R: 11980 case ZEND_FETCH_OBJ_IS: 11981 if ((op1_info & MAY_BE_OBJECT) 11982 && opline->op2_type == IS_CONST 11983 && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING 11984 && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') { 11985 return 1; 11986 } 11987 break; 11988 case ZEND_FETCH_DIM_FUNC_ARG: 11989 if (!JIT_G(current_frame) || 11990 !JIT_G(current_frame)->call->func || 11991 !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { 11992 return 0; 11993 } 11994 /* break missing intentionally */ 11995 case ZEND_FETCH_DIM_R: 11996 case ZEND_FETCH_DIM_IS: 11997 return 1; 11998 case ZEND_ISSET_ISEMPTY_DIM_OBJ: 11999 if (!(opline->extended_value & ZEND_ISEMPTY)) { 12000 return 1; 12001 } 12002 break; 12003 } 12004 return 0; 12005} 12006 12007static int zend_jit_fetch_dim_read(dasm_State **Dst, 12008 const zend_op *opline, 12009 zend_ssa *ssa, 12010 const zend_ssa_op *ssa_op, 12011 uint32_t op1_info, 12012 zend_jit_addr op1_addr, 12013 zend_bool op1_avoid_refcounting, 12014 uint32_t op2_info, 12015 uint32_t res_info, 12016 zend_jit_addr res_addr, 12017 int may_throw) 12018{ 12019 zend_jit_addr orig_op1_addr, op2_addr; 12020 const void *exit_addr = NULL; 12021 const void *not_found_exit_addr = NULL; 12022 const void *res_exit_addr = NULL; 12023 zend_bool result_avoid_refcounting = 0; 12024 uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; 12025 12026 orig_op1_addr = OP1_ADDR(); 12027 op2_addr = OP2_ADDR(); 12028 12029 if (opline->opcode != ZEND_FETCH_DIM_IS 12030 && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12031 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12032 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12033 if (!exit_addr) { 12034 return 0; 12035 } 12036 } 12037 12038 if ((res_info & MAY_BE_GUARD) 12039 && JIT_G(current_frame) 12040 && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { 12041 uint32_t flags = 0; 12042 uint32_t old_op1_info = 0; 12043 uint32_t old_info; 12044 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 12045 int32_t exit_point; 12046 12047 if (opline->opcode != ZEND_FETCH_LIST_R 12048 && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) 12049 && !op1_avoid_refcounting) { 12050 flags |= ZEND_JIT_EXIT_FREE_OP1; 12051 } 12052 if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) 12053 && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12054 flags |= ZEND_JIT_EXIT_FREE_OP2; 12055 } 12056 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 12057 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 12058 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 12059 && (ssa_op+1)->op1_use == ssa_op->result_def 12060 && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) 12061 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 12062 result_avoid_refcounting = 1; 12063 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 12064 } 12065 12066 if (op1_avoid_refcounting) { 12067 old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); 12068 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 12069 } 12070 12071 if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { 12072 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 12073 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 12074 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_R0); 12075 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 12076 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 12077 res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12078 if (!res_exit_addr) { 12079 return 0; 12080 } 12081 res_info &= ~MAY_BE_GUARD; 12082 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 12083 } 12084 12085 if (opline->opcode == ZEND_FETCH_DIM_IS 12086 && !(res_info & MAY_BE_NULL)) { 12087 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 12088 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); 12089 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); 12090 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 12091 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 12092 not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12093 if (!not_found_exit_addr) { 12094 return 0; 12095 } 12096 } 12097 12098 if (op1_avoid_refcounting) { 12099 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); 12100 } 12101 } 12102 12103 if (op1_info & MAY_BE_REF) { 12104 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12105 | ZVAL_DEREF FCARG1a, op1_info 12106 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 12107 } 12108 12109 if (op1_info & MAY_BE_ARRAY) { 12110 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12111 if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { 12112 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr 12113 } else { 12114 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12115 } 12116 } 12117 | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr 12118 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, res_exit_addr, not_found_exit_addr, exit_addr)) { 12119 return 0; 12120 } 12121 } 12122 12123 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 12124 if (op1_info & MAY_BE_ARRAY) { 12125 |.cold_code 12126 |7: 12127 } 12128 12129 if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { 12130 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { 12131 if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { 12132 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr 12133 } else { 12134 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 12135 } 12136 } 12137 | SET_EX_OPLINE opline, r0 12138 | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr 12139 if (opline->opcode != ZEND_FETCH_DIM_IS) { 12140 if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { 12141 | GET_ZVAL_LVAL ZREG_FCARG2a, op2_addr 12142 | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, r0 12143 } else { 12144 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12145 | EXT_CALL zend_jit_fetch_dim_str_r_helper, r0 12146 } 12147 | SET_ZVAL_PTR res_addr, r0 12148 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING 12149 } else { 12150 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12151 |.if X64 12152 | LOAD_ZVAL_ADDR CARG3, res_addr 12153 |.else 12154 | sub r4, 12 12155 | PUSH_ZVAL_ADDR res_addr, r0 12156 |.endif 12157 | EXT_CALL zend_jit_fetch_dim_str_is_helper, r0 12158 |.if not(X64) 12159 | add r4, 12 12160 |.endif 12161 } 12162 if ((op1_info & MAY_BE_ARRAY) || 12163 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { 12164 | jmp >9 // END 12165 } 12166 |6: 12167 } 12168 12169 if (op1_info & MAY_BE_OBJECT) { 12170 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { 12171 if (exit_addr) { 12172 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 12173 } else { 12174 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6 12175 } 12176 } 12177 | SET_EX_OPLINE opline, r0 12178 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 12179 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12180 } 12181 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12182 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12183 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12184 } else { 12185 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12186 } 12187 |.if X64 12188 | LOAD_ZVAL_ADDR CARG3, res_addr 12189 |.else 12190 | sub r4, 12 12191 | PUSH_ZVAL_ADDR res_addr, r0 12192 |.endif 12193 if (opline->opcode != ZEND_FETCH_DIM_IS) { 12194 | EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0 12195 } else { 12196 | EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0 12197 } 12198 |.if not(X64) 12199 | add r4, 12 12200 |.endif 12201 if ((op1_info & MAY_BE_ARRAY) || 12202 (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 12203 | jmp >9 // END 12204 } 12205 |6: 12206 } 12207 12208 if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) 12209 && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { 12210 if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { 12211 | SET_EX_OPLINE opline, r0 12212 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { 12213 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 12214 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 12215 | mov FCARG1d, opline->op1.var 12216 | EXT_CALL zend_jit_undefined_op_helper, r0 12217 |1: 12218 } 12219 12220 if (op2_info & MAY_BE_UNDEF) { 12221 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 12222 | mov FCARG1d, opline->op2.var 12223 | EXT_CALL zend_jit_undefined_op_helper, r0 12224 |1: 12225 } 12226 } 12227 12228 if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { 12229 if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { 12230 | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr 12231 } else { 12232 | SET_EX_OPLINE opline, r0 12233 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || 12234 Z_REG(op1_addr) != ZREG_FCARG1a || 12235 Z_OFFSET(op1_addr) != 0) { 12236 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12237 } 12238 } 12239 | EXT_CALL zend_jit_invalid_array_access, r0 12240 } 12241 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 12242 if (op1_info & MAY_BE_ARRAY) { 12243 | jmp >9 // END 12244 } 12245 } 12246 12247 if (op1_info & MAY_BE_ARRAY) { 12248 |.code 12249 } 12250 } 12251 12252 if (op1_info & MAY_BE_ARRAY) { 12253 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 12254 12255 |8: 12256 if (res_exit_addr) { 12257 zend_uchar type = concrete_type(res_info); 12258 12259 if (op1_info & MAY_BE_ARRAY_OF_REF) { 12260 | ZVAL_DEREF r0, MAY_BE_REF 12261 } 12262 if (type < IS_STRING) { 12263 | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr 12264 } else { 12265 | GET_ZVAL_TYPE_INFO edx, val_addr 12266 | IF_NOT_TYPE dl, type, &res_exit_addr 12267 } 12268 | // ZVAL_COPY 12269 |7: 12270 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 12271 if (Z_MODE(res_addr) == IS_MEM_ZVAL) { 12272 if (type < IS_STRING) { 12273 if (Z_REG(res_addr) != ZREG_FP || 12274 JIT_G(current_frame) == NULL || 12275 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 12276 | SET_ZVAL_TYPE_INFO res_addr, type 12277 } 12278 } else { 12279 | SET_ZVAL_TYPE_INFO res_addr, edx 12280 if (!result_avoid_refcounting) { 12281 | TRY_ADDREF res_info, dh, r1 12282 } 12283 } 12284 } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { 12285 return 0; 12286 } 12287 } else if (op1_info & MAY_BE_ARRAY_OF_REF) { 12288 | // ZVAL_COPY_DEREF 12289 | GET_ZVAL_TYPE_INFO Rd(ZREG_R2), val_addr 12290 if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_R2)) { 12291 return 0; 12292 } 12293 } else { 12294 | // ZVAL_COPY 12295 | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_R1, ZREG_R2 12296 | TRY_ADDREF res_info, ch, r2 12297 } 12298 } 12299 |9: // END 12300 12301#ifdef ZEND_JIT_USE_RC_INFERENCE 12302 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 12303 /* Magic offsetGet() may increase refcount of the key */ 12304 op2_info |= MAY_BE_RCN; 12305 } 12306#endif 12307 12308 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12309 if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { 12310 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12311 } 12312 12313 if (may_throw) { 12314 if (!zend_jit_check_exception(Dst)) { 12315 return 0; 12316 } 12317 } 12318 12319 return 1; 12320} 12321 12322static int zend_jit_fetch_dim(dasm_State **Dst, 12323 const zend_op *opline, 12324 uint32_t op1_info, 12325 zend_jit_addr op1_addr, 12326 uint32_t op2_info, 12327 zend_jit_addr res_addr, 12328 int may_throw) 12329{ 12330 zend_jit_addr op2_addr; 12331 12332 op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; 12333 12334 if (op1_info & MAY_BE_REF) { 12335 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12336 | IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1 12337 | GET_Z_PTR FCARG2a, FCARG1a 12338 | IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2 12339 | lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)] 12340 | jmp >3 12341 |.cold_code 12342 |2: 12343 | SET_EX_OPLINE opline, r0 12344 | EXT_CALL zend_jit_prepare_assign_dim_ref, r0 12345 | test r0, r0 12346 | mov FCARG1a, r0 12347 | jne >1 12348 | jmp ->exception_handler_undef 12349 |.code 12350 |1: 12351 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 12352 } 12353 12354 if (op1_info & MAY_BE_ARRAY) { 12355 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12356 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12357 } 12358 |3: 12359 | SEPARATE_ARRAY op1_addr, op1_info, 1 12360 } 12361 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { 12362 if (op1_info & MAY_BE_ARRAY) { 12363 |.cold_code 12364 |7: 12365 } 12366 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 12367 | CMP_ZVAL_TYPE op1_addr, IS_FALSE 12368 | jg >7 12369 } 12370 if (Z_REG(op1_addr) != ZREG_FP) { 12371 | mov T1, Ra(Z_REG(op1_addr)) // save 12372 } 12373 if ((op1_info & MAY_BE_UNDEF) 12374 && opline->opcode == ZEND_FETCH_DIM_RW) { 12375 if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { 12376 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 12377 } 12378 | SET_EX_OPLINE opline, r0 12379 | mov FCARG1a, opline->op1.var 12380 | EXT_CALL zend_jit_undefined_op_helper, r0 12381 |1: 12382 } 12383 | // ZVAL_ARR(container, zend_new_array(8)); 12384 | EXT_CALL _zend_new_array_0, r0 12385 if (Z_REG(op1_addr) != ZREG_FP) { 12386 | mov Ra(Z_REG(op1_addr)), T1 // restore 12387 } 12388 | SET_ZVAL_LVAL op1_addr, r0 12389 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX 12390 | mov FCARG1a, r0 12391 if (op1_info & MAY_BE_ARRAY) { 12392 | jmp >1 12393 |.code 12394 |1: 12395 } 12396 } 12397 12398 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 12399 |6: 12400 if (opline->op2_type == IS_UNUSED) { 12401 | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); 12402 | LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval 12403 | EXT_CALL zend_hash_next_index_insert, r0 12404 | // if (UNEXPECTED(!var_ptr)) { 12405 | test r0, r0 12406 | jz >1 12407 |.cold_code 12408 |1: 12409 | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); 12410 | CANNOT_ADD_ELEMENT opline 12411 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF 12412 | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); 12413 | jmp >8 12414 |.code 12415 | SET_ZVAL_PTR res_addr, r0 12416 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 12417 } else { 12418 uint32_t type; 12419 12420 switch (opline->opcode) { 12421 case ZEND_FETCH_DIM_W: 12422 case ZEND_FETCH_LIST_W: 12423 type = BP_VAR_W; 12424 break; 12425 case ZEND_FETCH_DIM_RW: 12426 type = BP_VAR_RW; 12427 break; 12428 case ZEND_FETCH_DIM_UNSET: 12429 type = BP_VAR_UNSET; 12430 break; 12431 default: 12432 ZEND_UNREACHABLE(); 12433 } 12434 12435 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, NULL, NULL, NULL)) { 12436 return 0; 12437 } 12438 12439 |8: 12440 | SET_ZVAL_PTR res_addr, r0 12441 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 12442 12443 if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { 12444 |.cold_code 12445 |9: 12446 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 12447 | jmp >8 12448 |.code 12449 } 12450 } 12451 } 12452 12453 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { 12454 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 12455 |.cold_code 12456 |7: 12457 } 12458 12459 | SET_EX_OPLINE opline, r0 12460 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 12461 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12462 } 12463 if (opline->op2_type == IS_UNUSED) { 12464 | xor FCARG2a, FCARG2a 12465 } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12466 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12467 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12468 } else { 12469 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12470 } 12471 |.if X64 12472 | LOAD_ZVAL_ADDR CARG3, res_addr 12473 |.else 12474 | sub r4, 12 12475 | PUSH_ZVAL_ADDR res_addr, r0 12476 |.endif 12477 switch (opline->opcode) { 12478 case ZEND_FETCH_DIM_W: 12479 case ZEND_FETCH_LIST_W: 12480 | EXT_CALL zend_jit_fetch_dim_obj_w_helper, r0 12481 break; 12482 case ZEND_FETCH_DIM_RW: 12483 | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, r0 12484 break; 12485// case ZEND_FETCH_DIM_UNSET: 12486// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0 12487// break; 12488 default: 12489 ZEND_UNREACHABLE(); 12490 } 12491 |.if not(X64) 12492 | add r4, 12 12493 |.endif 12494 12495 if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { 12496 | jmp >8 // END 12497 |.code 12498 } 12499 } 12500 12501#ifdef ZEND_JIT_USE_RC_INFERENCE 12502 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { 12503 /* ASSIGN_DIM may increase refcount of the key */ 12504 op2_info |= MAY_BE_RCN; 12505 } 12506#endif 12507 12508 |8: 12509 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12510 12511 if (may_throw) { 12512 if (!zend_jit_check_exception(Dst)) { 12513 return 0; 12514 } 12515 } 12516 12517 return 1; 12518} 12519 12520static int zend_jit_isset_isempty_dim(dasm_State **Dst, 12521 const zend_op *opline, 12522 uint32_t op1_info, 12523 zend_jit_addr op1_addr, 12524 zend_bool op1_avoid_refcounting, 12525 uint32_t op2_info, 12526 int may_throw, 12527 zend_uchar smart_branch_opcode, 12528 uint32_t target_label, 12529 uint32_t target_label2, 12530 const void *exit_addr) 12531{ 12532 zend_jit_addr op2_addr, res_addr; 12533 12534 // TODO: support for empty() ??? 12535 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 12536 12537 op2_addr = OP2_ADDR(); 12538 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12539 12540 if (op1_info & MAY_BE_REF) { 12541 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12542 | ZVAL_DEREF FCARG1a, op1_info 12543 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 12544 } 12545 12546 if (op1_info & MAY_BE_ARRAY) { 12547 const void *found_exit_addr = NULL; 12548 const void *not_found_exit_addr = NULL; 12549 12550 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { 12551 | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7 12552 } 12553 | GET_ZVAL_LVAL ZREG_FCARG1a, op1_addr 12554 if (exit_addr 12555 && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) 12556 && !may_throw 12557 && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) 12558 && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { 12559 if (smart_branch_opcode == ZEND_JMPNZ) { 12560 found_exit_addr = exit_addr; 12561 } else { 12562 not_found_exit_addr = exit_addr; 12563 } 12564 } 12565 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, found_exit_addr, not_found_exit_addr, NULL)) { 12566 return 0; 12567 } 12568 12569 if (found_exit_addr) { 12570 |9: 12571 return 1; 12572 } else if (not_found_exit_addr) { 12573 |8: 12574 return 1; 12575 } 12576 } 12577 12578 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { 12579 if (op1_info & MAY_BE_ARRAY) { 12580 |.cold_code 12581 |7: 12582 } 12583 12584 if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { 12585 | SET_EX_OPLINE opline, r0 12586 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 12587 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 12588 } 12589 if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { 12590 ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); 12591 | LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1) 12592 } else { 12593 | LOAD_ZVAL_ADDR FCARG2a, op2_addr 12594 } 12595 | EXT_CALL zend_jit_isset_dim_helper, r0 12596 | test r0, r0 12597 | jz >9 12598 if (op1_info & MAY_BE_ARRAY) { 12599 | jmp >8 12600 |.code 12601 } 12602 } else { 12603 if (op2_info & MAY_BE_UNDEF) { 12604 if (op2_info & MAY_BE_ANY) { 12605 | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1 12606 } 12607 | SET_EX_OPLINE opline, r0 12608 | mov FCARG1d, opline->op2.var 12609 | EXT_CALL zend_jit_undefined_op_helper, r0 12610 |1: 12611 } 12612 if (op1_info & MAY_BE_ARRAY) { 12613 | jmp >9 12614 |.code 12615 } 12616 } 12617 } 12618 12619#ifdef ZEND_JIT_USE_RC_INFERENCE 12620 if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { 12621 /* Magic offsetExists() may increase refcount of the key */ 12622 op2_info |= MAY_BE_RCN; 12623 } 12624#endif 12625 12626 if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { 12627 |8: 12628 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12629 if (!op1_avoid_refcounting) { 12630 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12631 } 12632 if (may_throw) { 12633 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 12634 return 0; 12635 } 12636 } 12637 if (!(opline->extended_value & ZEND_ISEMPTY)) { 12638 if (exit_addr) { 12639 if (smart_branch_opcode == ZEND_JMPNZ) { 12640 | jmp &exit_addr 12641 } else { 12642 | jmp >8 12643 } 12644 } else if (smart_branch_opcode) { 12645 if (smart_branch_opcode == ZEND_JMPZ) { 12646 | jmp =>target_label2 12647 } else if (smart_branch_opcode == ZEND_JMPNZ) { 12648 | jmp =>target_label 12649 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 12650 | jmp =>target_label2 12651 } else { 12652 ZEND_UNREACHABLE(); 12653 } 12654 } else { 12655 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 12656 | jmp >8 12657 } 12658 } else { 12659 | //???? 12660 | int3 12661 } 12662 } 12663 12664 |9: // not found 12665 | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline 12666 if (!op1_avoid_refcounting) { 12667 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 12668 } 12669 if (may_throw) { 12670 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 12671 return 0; 12672 } 12673 } 12674 if (!(opline->extended_value & ZEND_ISEMPTY)) { 12675 if (exit_addr) { 12676 if (smart_branch_opcode == ZEND_JMPZ) { 12677 | jmp &exit_addr 12678 } 12679 } else if (smart_branch_opcode) { 12680 if (smart_branch_opcode == ZEND_JMPZ) { 12681 | jmp =>target_label 12682 } else if (smart_branch_opcode == ZEND_JMPNZ) { 12683 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 12684 | jmp =>target_label 12685 } else { 12686 ZEND_UNREACHABLE(); 12687 } 12688 } else { 12689 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 12690 } 12691 } else { 12692 | //???? 12693 | int3 12694 } 12695 12696 |8: 12697 12698 return 1; 12699} 12700 12701static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 12702{ 12703 zend_jit_addr op1_addr = OP1_ADDR(); 12704 zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); 12705 12706 | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; 12707 | mov r0, EX->run_time_cache 12708 | mov r0, aword [r0 + opline->extended_value] 12709 | sub r0, 1 12710 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) 12711 | MEM_OP2_2_ZTS mov, ecx, dword, executor_globals, symbol_table.nNumUsed, r1 12712 |.if X64 12713 | shl r1, 5 12714 |.else 12715 | imul r1, sizeof(Bucket) 12716 |.endif 12717 | cmp r0, r1 12718 | jae >9 12719 | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); 12720 | MEM_OP2_2_ZTS add, r0, aword, executor_globals, symbol_table.arData, r1 12721 | IF_NOT_Z_TYPE r0, IS_REFERENCE, >9 12722 | // (EXPECTED(p->key == varname)) 12723 | ADDR_OP2_2 cmp, aword [r0 + offsetof(Bucket, key)], varname, r1 12724 | jne >9 12725 | GET_Z_PTR r0, r0 12726 | GC_ADDREF r0 12727 |1: 12728 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 12729 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12730 | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) 12731 | IF_ZVAL_REFCOUNTED op1_addr, >2 12732 |.cold_code 12733 |2: 12734 } 12735 | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); 12736 | GET_ZVAL_PTR FCARG1a, op1_addr 12737 | // ZVAL_REF(variable_ptr, ref) 12738 | SET_ZVAL_PTR op1_addr, r0 12739 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 12740 | // if (GC_DELREF(garbage) == 0) 12741 | GC_DELREF FCARG1a 12742 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 12743 | jnz >3 12744 } else { 12745 | jnz >5 12746 } 12747 | ZVAL_DTOR_FUNC op1_info, opline 12748 | jmp >5 12749 if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { 12750 |3: 12751 | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) 12752 | IF_GC_MAY_NOT_LEAK FCARG1a, >5 12753 | EXT_CALL gc_possible_root, r1 12754 | jmp >5 12755 } 12756 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12757 |.code 12758 } 12759 } 12760 12761 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 12762 | // ZVAL_REF(variable_ptr, ref) 12763 | SET_ZVAL_PTR op1_addr, r0 12764 | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX 12765 } 12766 |5: 12767 //END of handler 12768 12769 |.cold_code 12770 |9: 12771 | LOAD_ADDR FCARG1a, (ptrdiff_t)varname 12772 | mov FCARG2a, EX->run_time_cache 12773 if (opline->extended_value) { 12774 | add FCARG2a, opline->extended_value 12775 } 12776 | EXT_CALL zend_jit_fetch_global_helper, r0 12777 | jmp <1 12778 |.code 12779 12780 return 1; 12781} 12782 12783static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, zend_bool check_exception) 12784{ 12785 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12786 zend_bool in_cold = 0; 12787 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 12788 zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1a : ZREG_R0; 12789 12790 if (ZEND_ARG_SEND_MODE(arg_info)) { 12791 if (opline->opcode == ZEND_RECV_INIT) { 12792 | LOAD_ZVAL_ADDR Ra(tmp_reg), res_addr 12793 | ZVAL_DEREF Ra(tmp_reg), MAY_BE_REF 12794 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); 12795 } else { 12796 | GET_ZVAL_PTR Ra(tmp_reg), res_addr 12797 res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); 12798 } 12799 } 12800 12801 if (type_mask != 0) { 12802 if (is_power_of_two(type_mask)) { 12803 uint32_t type_code = concrete_type(type_mask); 12804 | IF_NOT_ZVAL_TYPE res_addr, type_code, >1 12805 } else { 12806 | mov edx, 1 12807 | mov cl, byte [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)+offsetof(zval, u1.v.type)] 12808 | shl edx, cl 12809 | test edx, type_mask 12810 | je >1 12811 } 12812 12813 |.cold_code 12814 |1: 12815 12816 in_cold = 1; 12817 } 12818 12819 if (Z_REG(res_addr) != ZREG_FCARG1a || Z_OFFSET(res_addr) != 0) { 12820 | LOAD_ZVAL_ADDR FCARG1a, res_addr 12821 } 12822 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12823 | SET_EX_OPLINE opline, r0 12824 } else { 12825 | ADDR_OP2_2 mov, aword EX->opline, opline, r0 12826 } 12827 | LOAD_ADDR FCARG2a, (ptrdiff_t)arg_info 12828 | EXT_CALL zend_jit_verify_arg_slow, r0 12829 12830 if (check_exception) { 12831 | test al, al 12832 if (in_cold) { 12833 | jnz >1 12834 | jmp ->exception_handler 12835 |.code 12836 |1: 12837 } else { 12838 | jz ->exception_handler 12839 } 12840 } else if (in_cold) { 12841 | jmp >1 12842 |.code 12843 |1: 12844 } 12845 12846 return 1; 12847} 12848 12849static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) 12850{ 12851 uint32_t arg_num = opline->op1.num; 12852 zend_arg_info *arg_info = NULL; 12853 12854 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12855 if (EXPECTED(arg_num <= op_array->num_args)) { 12856 arg_info = &op_array->arg_info[arg_num-1]; 12857 } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { 12858 arg_info = &op_array->arg_info[op_array->num_args]; 12859 } 12860 if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { 12861 arg_info = NULL; 12862 } 12863 } 12864 12865 if (arg_info || (opline+1)->opcode != ZEND_RECV) { 12866 | cmp dword EX->This.u2.num_args, arg_num 12867 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12868 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 12869 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 12870 12871 if (!exit_addr) { 12872 return 0; 12873 } 12874 | jb &exit_addr 12875 } else { 12876 | jb >1 12877 |.cold_code 12878 |1: 12879 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12880 | SET_EX_OPLINE opline, r0 12881 } else { 12882 | ADDR_OP2_2 mov, aword EX->opline, opline, r0 12883 } 12884 | mov FCARG1a, FP 12885 | EXT_CALL zend_missing_arg_error, r0 12886 | jmp ->exception_handler 12887 |.code 12888 } 12889 } 12890 12891 if (arg_info) { 12892 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { 12893 return 0; 12894 } 12895 } 12896 12897 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12898 if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { 12899 | LOAD_IP_ADDR (opline + 1) 12900 zend_jit_set_last_valid_opline(opline + 1); 12901 } 12902 } 12903 12904 return 1; 12905} 12906 12907static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_bool is_last, int may_throw) 12908{ 12909 uint32_t arg_num = opline->op1.num; 12910 zval *zv = RT_CONSTANT(opline, opline->op2); 12911 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 12912 12913 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || 12914 (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 12915 | cmp dword EX->This.u2.num_args, arg_num 12916 | jae >5 12917 } 12918 | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0 12919 if (Z_REFCOUNTED_P(zv)) { 12920 | ADDREF_CONST zv, r0 12921 } 12922 12923 if (Z_CONSTANT_P(zv)) { 12924 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 12925 | SET_EX_OPLINE opline, r0 12926 } else { 12927 | ADDR_OP2_2 mov, aword EX->opline, opline, r0 12928 } 12929 | LOAD_ZVAL_ADDR FCARG1a, res_addr 12930 | mov r0, EX->func 12931 | mov FCARG2a, [r0 + offsetof(zend_op_array, scope)] 12932 | .if X64 12933 | EXT_CALL zval_update_constant_ex, r0 12934 | .else 12935 ||#if (PHP_VERSION_ID < 80100) && (SIZEOF_SIZE_T == 4) 12936 | EXT_CALL zval_jit_update_constant_ex, r0 12937 ||#else 12938 | EXT_CALL zval_update_constant_ex, r0 12939 ||#endif 12940 | .endif 12941 | test al, al 12942 | jnz >1 12943 |.cold_code 12944 |1: 12945 | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline 12946 | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF 12947 | jmp ->exception_handler 12948 |.code 12949 } 12950 12951 |5: 12952 12953 if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { 12954 do { 12955 zend_arg_info *arg_info; 12956 12957 if (arg_num <= op_array->num_args) { 12958 arg_info = &op_array->arg_info[arg_num-1]; 12959 } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { 12960 arg_info = &op_array->arg_info[op_array->num_args]; 12961 } else { 12962 break; 12963 } 12964 if (!ZEND_TYPE_IS_SET(arg_info->type)) { 12965 break; 12966 } 12967 if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { 12968 return 0; 12969 } 12970 } while (0); 12971 } 12972 12973 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 12974 if (is_last) { 12975 | LOAD_IP_ADDR (opline + 1) 12976 zend_jit_set_last_valid_opline(opline + 1); 12977 } 12978 } 12979 12980 return 1; 12981} 12982 12983static zend_property_info* zend_get_known_property_info(zend_class_entry *ce, zend_string *member, zend_bool on_this, zend_string *filename) 12984{ 12985 zend_property_info *info = NULL; 12986 12987 if (!ce || 12988 !(ce->ce_flags & ZEND_ACC_LINKED) || 12989 (ce->ce_flags & ZEND_ACC_TRAIT) || 12990 ce->create_object) { 12991 return NULL; 12992 } 12993 12994 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 12995 if (ce->info.user.filename != filename) { 12996 /* class declaration might be changed independently */ 12997 return NULL; 12998 } 12999 13000 if (ce->parent) { 13001 zend_class_entry *parent = ce->parent; 13002 13003 do { 13004 if (parent->type == ZEND_INTERNAL_CLASS) { 13005 break; 13006 } else if (parent->info.user.filename != filename) { 13007 /* some of parents class declarations might be changed independently */ 13008 /* TODO: this check may be not enough, because even 13009 * in the same it's possible to conditionally define 13010 * few classes with the same name, and "parent" may 13011 * change from request to request. 13012 */ 13013 return NULL; 13014 } 13015 parent = parent->parent; 13016 } while (parent); 13017 } 13018 } 13019 13020 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); 13021 if (info == NULL || 13022 !IS_VALID_PROPERTY_OFFSET(info->offset) || 13023 (info->flags & ZEND_ACC_STATIC)) { 13024 return NULL; 13025 } 13026 13027 if (!(info->flags & ZEND_ACC_PUBLIC) && 13028 (!on_this || info->ce != ce)) { 13029 return NULL; 13030 } 13031 13032 return info; 13033} 13034 13035static zend_bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, zend_bool on_this, zend_string *filename) 13036{ 13037 zend_property_info *info; 13038 13039 if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) { 13040 return 1; 13041 } 13042 13043 if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13044 if (ce->info.user.filename != filename) { 13045 /* class declaration might be changed independently */ 13046 return 1; 13047 } 13048 } 13049 13050 info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); 13051 if (info == NULL || 13052 !IS_VALID_PROPERTY_OFFSET(info->offset) || 13053 (info->flags & ZEND_ACC_STATIC)) { 13054 return 1; 13055 } 13056 13057 if (!(info->flags & ZEND_ACC_PUBLIC) && 13058 (!on_this || info->ce != ce)) { 13059 return 1; 13060 } 13061 13062 return 0; 13063} 13064 13065static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) 13066{ 13067 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 13068 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13069 13070 if (!exit_addr) { 13071 return 0; 13072 } 13073 13074 |.if X64 13075 || if (!IS_SIGNED_32BIT(ce)) { 13076 | mov64 r0, ((ptrdiff_t)ce) 13077 | cmp aword [FCARG1a + offsetof(zend_object, ce)], r0 13078 || } else { 13079 | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce 13080 || } 13081 |.else 13082 | cmp aword [FCARG1a + offsetof(zend_object, ce)], ce 13083 |.endif 13084 | jne &exit_addr 13085 13086 return 1; 13087} 13088 13089static int zend_jit_fetch_obj(dasm_State **Dst, 13090 const zend_op *opline, 13091 const zend_op_array *op_array, 13092 zend_ssa *ssa, 13093 const zend_ssa_op *ssa_op, 13094 uint32_t op1_info, 13095 zend_jit_addr op1_addr, 13096 zend_bool op1_indirect, 13097 zend_class_entry *ce, 13098 zend_bool ce_is_instanceof, 13099 zend_bool use_this, 13100 zend_bool op1_avoid_refcounting, 13101 zend_class_entry *trace_ce, 13102 int may_throw) 13103{ 13104 zval *member; 13105 zend_property_info *prop_info; 13106 zend_bool may_be_dynamic = 1; 13107 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 13108 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13109 zend_jit_addr prop_addr; 13110 uint32_t res_info = RES_INFO(); 13111 13112 ZEND_ASSERT(opline->op2_type == IS_CONST); 13113 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13114 13115 member = RT_CONSTANT(opline, opline->op2); 13116 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13117 prop_info = zend_get_known_property_info(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); 13118 13119 if (opline->op1_type == IS_UNUSED || use_this) { 13120 | GET_ZVAL_PTR FCARG1a, this_addr 13121 } else { 13122 if (opline->op1_type == IS_VAR 13123 && opline->opcode == ZEND_FETCH_OBJ_W 13124 && (op1_info & MAY_BE_INDIRECT) 13125 && Z_REG(op1_addr) == ZREG_FP) { 13126 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13127 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13128 | GET_Z_PTR FCARG1a, FCARG1a 13129 |1: 13130 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13131 } 13132 if (op1_info & MAY_BE_REF) { 13133 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 13134 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13135 } 13136 | ZVAL_DEREF FCARG1a, op1_info 13137 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13138 } 13139 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13140 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13141 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13142 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13143 13144 if (!exit_addr) { 13145 return 0; 13146 } 13147 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13148 } else { 13149 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7 13150 } 13151 } 13152 | GET_ZVAL_PTR FCARG1a, op1_addr 13153 } 13154 13155 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13156 prop_info = zend_get_known_property_info(trace_ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); 13157 if (prop_info) { 13158 ce = trace_ce; 13159 ce_is_instanceof = 0; 13160 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13161 if (!zend_jit_class_guard(Dst, opline, trace_ce)) { 13162 return 0; 13163 } 13164 if (ssa->var_info && ssa_op->op1_use >= 0) { 13165 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13166 ssa->var_info[ssa_op->op1_use].ce = ce; 13167 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13168 } 13169 } 13170 } 13171 } 13172 13173 if (!prop_info) { 13174 | mov r0, EX->run_time_cache 13175 | mov r2, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)] 13176 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13177 | jne >5 13178 | mov r0, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)] 13179 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); 13180 if (may_be_dynamic) { 13181 | test r0, r0 13182 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13183 | jl >5 13184 } else { 13185 | jl >8 // dynamic property 13186 } 13187 } 13188 | mov edx, dword [FCARG1a + r0 + 8] 13189 | IF_UNDEF dl, >5 13190 | add FCARG1a, r0 13191 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13192 if (opline->opcode == ZEND_FETCH_OBJ_W 13193 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) 13194 && (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS))) { 13195 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 13196 13197 | mov r0, EX->run_time_cache 13198 | mov FCARG2a, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2] 13199 | test FCARG2a, FCARG2a 13200 | jnz >1 13201 |.cold_code 13202 |1: 13203 if (flags == ZEND_FETCH_DIM_WRITE) { 13204 | SET_EX_OPLINE opline, r0 13205 | EXT_CALL zend_jit_check_array_promotion, r0 13206 | jmp >9 13207 } else if (flags == ZEND_FETCH_REF) { 13208 |.if X64 13209 | LOAD_ZVAL_ADDR CARG3, res_addr 13210 |.else 13211 | sub r4, 12 13212 | PUSH_ZVAL_ADDR res_addr, r0 13213 |.endif 13214 | EXT_CALL zend_jit_create_typed_ref, r0 13215 |.if not(X64) 13216 | add r4, 12 13217 |.endif 13218 | jmp >9 13219 } else { 13220 ZEND_UNREACHABLE(); 13221 } 13222 |.code 13223 } 13224 } else { 13225 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset); 13226 | mov edx, dword [FCARG1a + prop_info->offset + 8] 13227 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13228 if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { 13229 /* perform IS_UNDEF check only after result type guard (during deoptimization) */ 13230 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13231 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13232 13233 if (!exit_addr) { 13234 return 0; 13235 } 13236 | IF_UNDEF dl, &exit_addr 13237 } 13238 } else { 13239 | IF_UNDEF dl, >5 13240 } 13241 if (opline->opcode == ZEND_FETCH_OBJ_W 13242 && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) 13243 && ZEND_TYPE_IS_SET(prop_info->type)) { 13244 uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; 13245 13246 if (flags == ZEND_FETCH_DIM_WRITE) { 13247 if ((ZEND_TYPE_FULL_MASK(prop_info->type) & (MAY_BE_ITERABLE|MAY_BE_ARRAY)) == 0) { 13248 | cmp dl, IS_FALSE 13249 | jle >1 13250 |.cold_code 13251 |1: 13252 if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) { 13253 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13254 } 13255 | LOAD_ADDR FCARG2a, prop_info 13256 | SET_EX_OPLINE opline, r0 13257 | EXT_CALL zend_jit_check_array_promotion, r0 13258 | jmp >9 13259 |.code 13260 } 13261 } else if (flags == ZEND_FETCH_REF) { 13262 | IF_TYPE dl, IS_REFERENCE, >1 13263 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13264 | LOAD_ADDR FCARG2a, prop_info 13265 } else { 13266 int prop_info_offset = 13267 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13268 13269 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13270 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13271 | mov FCARG2a, aword[r0 + prop_info_offset] 13272 } 13273 if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) { 13274 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13275 } 13276 |.if X64 13277 | LOAD_ZVAL_ADDR CARG3, res_addr 13278 |.else 13279 | sub r4, 12 13280 | PUSH_ZVAL_ADDR res_addr, r0 13281 |.endif 13282 | EXT_CALL zend_jit_create_typed_ref, r0 13283 |.if not(X64) 13284 | add r4, 12 13285 |.endif 13286 | jmp >9 13287 |1: 13288 } else { 13289 ZEND_UNREACHABLE(); 13290 } 13291 } 13292 } 13293 if (op1_avoid_refcounting) { 13294 SET_STACK_REG(JIT_G(current_frame)->stack, 13295 EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); 13296 } 13297 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13298 if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) { 13299 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13300 } 13301 | SET_ZVAL_PTR res_addr, FCARG1a 13302 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT 13303 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { 13304 ssa->var_info[ssa_op->result_def].indirect_reference = 1; 13305 } 13306 } else { 13307 zend_bool result_avoid_refcounting = 0; 13308 13309 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { 13310 uint32_t flags = 0; 13311 uint32_t old_info; 13312 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 13313 int32_t exit_point; 13314 const void *exit_addr; 13315 zend_uchar type; 13316 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 13317 13318 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) 13319 && !use_this 13320 && !op1_avoid_refcounting) { 13321 flags = ZEND_JIT_EXIT_FREE_OP1; 13322 } 13323 13324 | LOAD_ZVAL_ADDR r0, prop_addr 13325 13326 if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) 13327 && !(flags & ZEND_JIT_EXIT_FREE_OP1) 13328 && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) 13329 && (ssa_op+1)->op1_use == ssa_op->result_def 13330 && zend_jit_may_avoid_refcounting(opline+1, res_info)) { 13331 result_avoid_refcounting = 1; 13332 ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; 13333 } 13334 13335 old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 13336 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 13337 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_R0); 13338 exit_point = zend_jit_trace_get_exit_point(opline+1, flags); 13339 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 13340 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13341 if (!exit_addr) { 13342 return 0; 13343 } 13344 13345 res_info &= ~MAY_BE_GUARD; 13346 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 13347 type = concrete_type(res_info); 13348 13349 | // ZVAL_DEREF() 13350 | IF_NOT_TYPE dl, IS_REFERENCE, >1 13351 | GET_Z_PTR r0, r0 13352 | add r0, offsetof(zend_reference, val) 13353 if (type < IS_STRING) { 13354 |1: 13355 | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr 13356 } else { 13357 | GET_ZVAL_TYPE_INFO edx, val_addr 13358 |1: 13359 | IF_NOT_TYPE dl, type, &exit_addr 13360 } 13361 | // ZVAL_COPY 13362 | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1 13363 if (type < IS_STRING) { 13364 if (Z_REG(res_addr) != ZREG_FP || 13365 JIT_G(current_frame) == NULL || 13366 STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { 13367 | SET_ZVAL_TYPE_INFO res_addr, type 13368 } 13369 } else { 13370 | SET_ZVAL_TYPE_INFO res_addr, edx 13371 if (!result_avoid_refcounting) { 13372 | TRY_ADDREF res_info, dh, r1 13373 } 13374 } 13375 } else { 13376 if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_R2)) { 13377 return 0; 13378 } 13379 } 13380 } 13381 13382 |.cold_code 13383 13384 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { 13385 |5: 13386 | SET_EX_OPLINE opline, r0 13387 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13388 | EXT_CALL zend_jit_fetch_obj_w_slow, r0 13389 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13390 | EXT_CALL zend_jit_fetch_obj_r_slow, r0 13391 } else { 13392 | EXT_CALL zend_jit_fetch_obj_is_slow, r0 13393 } 13394 | jmp >9 13395 } 13396 13397 if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 13398 |7: 13399 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13400 | SET_EX_OPLINE opline, r0 13401 if (opline->opcode != ZEND_FETCH_OBJ_W 13402 && (op1_info & MAY_BE_UNDEF)) { 13403 zend_jit_addr orig_op1_addr = OP1_ADDR(); 13404 13405 if (op1_info & MAY_BE_ANY) { 13406 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1 13407 } 13408 | mov FCARG1d, opline->op1.var 13409 | EXT_CALL zend_jit_undefined_op_helper, r0 13410 |1: 13411 | LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr 13412 } else if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 13413 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13414 } 13415 | LOAD_ADDR FCARG2a, Z_STRVAL_P(member) 13416 if (opline->opcode == ZEND_FETCH_OBJ_W) { 13417 | EXT_CALL zend_jit_invalid_property_write, r0 13418 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR 13419 } else { 13420 | EXT_CALL zend_jit_invalid_property_read, r0 13421 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 13422 } 13423 | jmp >9 13424 } else { 13425 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 13426 | jmp >9 13427 } 13428 } 13429 13430 if (!prop_info 13431 && may_be_dynamic 13432 && opline->opcode != ZEND_FETCH_OBJ_W) { 13433 |8: 13434 | mov FCARG2a, r0 13435 | SET_EX_OPLINE opline, r0 13436 if (opline->opcode != ZEND_FETCH_OBJ_IS) { 13437 | EXT_CALL zend_jit_fetch_obj_r_dynamic, r0 13438 } else { 13439 | EXT_CALL zend_jit_fetch_obj_is_dynamic, r0 13440 } 13441 | jmp >9 13442 } 13443 13444 |.code; 13445 |9: // END 13446 if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { 13447 if (opline->op1_type == IS_VAR 13448 && opline->opcode == ZEND_FETCH_OBJ_W 13449 && (op1_info & MAY_BE_RC1)) { 13450 zend_jit_addr orig_op1_addr = OP1_ADDR(); 13451 13452 | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1 13453 | GET_ZVAL_PTR FCARG1a, orig_op1_addr 13454 | GC_DELREF FCARG1a 13455 | jnz >1 13456 | SET_EX_OPLINE opline, r0 13457 | EXT_CALL zend_jit_extract_helper, r0 13458 |1: 13459 } else if (!op1_avoid_refcounting) { 13460 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 13461 } 13462 } 13463 13464 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE 13465 && prop_info 13466 && (opline->opcode != ZEND_FETCH_OBJ_W || 13467 !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) || 13468 !ZEND_TYPE_IS_SET(prop_info->type)) 13469 && opline->op1_type != IS_VAR 13470 && opline->op1_type != IS_TMP_VAR) { 13471 may_throw = 0; 13472 } 13473 13474 if (may_throw) { 13475 if (!zend_jit_check_exception(Dst)) { 13476 return 0; 13477 } 13478 } 13479 13480 return 1; 13481} 13482 13483static int zend_jit_incdec_obj(dasm_State **Dst, 13484 const zend_op *opline, 13485 const zend_op_array *op_array, 13486 zend_ssa *ssa, 13487 const zend_ssa_op *ssa_op, 13488 uint32_t op1_info, 13489 zend_jit_addr op1_addr, 13490 zend_bool op1_indirect, 13491 zend_class_entry *ce, 13492 zend_bool ce_is_instanceof, 13493 zend_bool use_this, 13494 zend_class_entry *trace_ce, 13495 int may_throw) 13496{ 13497 zval *member; 13498 zend_string *name; 13499 zend_property_info *prop_info; 13500 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13501 zend_jit_addr res_addr = 0; 13502 zend_jit_addr prop_addr; 13503 zend_bool needs_slow_path = 0; 13504 13505 ZEND_ASSERT(opline->op2_type == IS_CONST); 13506 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13507 13508 if (opline->result_type != IS_UNUSED) { 13509 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 13510 } 13511 13512 member = RT_CONSTANT(opline, opline->op2); 13513 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13514 name = Z_STR_P(member); 13515 prop_info = zend_get_known_property_info(ce, name, opline->op1_type == IS_UNUSED, op_array->filename); 13516 13517 if (opline->op1_type == IS_UNUSED || use_this) { 13518 | GET_ZVAL_PTR FCARG1a, this_addr 13519 } else { 13520 if (opline->op1_type == IS_VAR 13521 && (op1_info & MAY_BE_INDIRECT) 13522 && Z_REG(op1_addr) == ZREG_FP) { 13523 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13524 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13525 | GET_Z_PTR FCARG1a, FCARG1a 13526 |1: 13527 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13528 } 13529 if (op1_info & MAY_BE_REF) { 13530 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 13531 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13532 } 13533 | ZVAL_DEREF FCARG1a, op1_info 13534 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13535 } 13536 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13537 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13538 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13539 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13540 13541 if (!exit_addr) { 13542 return 0; 13543 } 13544 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13545 } else { 13546 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 13547 |.cold_code 13548 |1: 13549 | SET_EX_OPLINE opline, r0 13550 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 13551 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13552 } 13553 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 13554 | EXT_CALL zend_jit_invalid_property_incdec, r0 13555 | jmp ->exception_handler 13556 |.code 13557 } 13558 } 13559 | GET_ZVAL_PTR FCARG1a, op1_addr 13560 } 13561 13562 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13563 prop_info = zend_get_known_property_info(trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); 13564 if (prop_info) { 13565 ce = trace_ce; 13566 ce_is_instanceof = 0; 13567 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13568 if (!zend_jit_class_guard(Dst, opline, trace_ce)) { 13569 return 0; 13570 } 13571 if (ssa->var_info && ssa_op->op1_use >= 0) { 13572 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13573 ssa->var_info[ssa_op->op1_use].ce = ce; 13574 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13575 } 13576 if (ssa->var_info && ssa_op->op1_def >= 0) { 13577 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13578 ssa->var_info[ssa_op->op1_def].ce = ce; 13579 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13580 } 13581 } 13582 } 13583 } 13584 13585 if (!prop_info) { 13586 needs_slow_path = 1; 13587 13588 | mov r0, EX->run_time_cache 13589 | mov r2, aword [r0 + opline->extended_value] 13590 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13591 | jne >7 13592 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 13593 | cmp aword [r0 + opline->extended_value + sizeof(void*) * 2], 0 13594 | jnz >7 13595 } 13596 | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] 13597 | test r0, r0 13598 | jl >7 13599 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 13600 | add FCARG1a, r0 13601 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13602 } else { 13603 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset); 13604 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13605 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13606 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13607 13608 if (!exit_addr) { 13609 return 0; 13610 } 13611 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 13612 } else { 13613 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 13614 needs_slow_path = 1; 13615 } 13616 if (ZEND_TYPE_IS_SET(prop_info->type)) { 13617 | SET_EX_OPLINE opline, r0 13618 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 13619 | LOAD_ADDR FCARG2a, prop_info 13620 } else { 13621 int prop_info_offset = 13622 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 13623 13624 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 13625 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 13626 | mov FCARG2a, aword[r0 + prop_info_offset] 13627 } 13628 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13629 if (opline->result_type == IS_UNUSED) { 13630 switch (opline->opcode) { 13631 case ZEND_PRE_INC_OBJ: 13632 case ZEND_POST_INC_OBJ: 13633 | EXT_CALL zend_jit_inc_typed_prop, r0 13634 break; 13635 case ZEND_PRE_DEC_OBJ: 13636 case ZEND_POST_DEC_OBJ: 13637 | EXT_CALL zend_jit_dec_typed_prop, r0 13638 break; 13639 default: 13640 ZEND_UNREACHABLE(); 13641 } 13642 } else { 13643 |.if X64 13644 | LOAD_ZVAL_ADDR CARG3, res_addr 13645 |.else 13646 | sub r4, 12 13647 | PUSH_ZVAL_ADDR res_addr, r0 13648 |.endif 13649 switch (opline->opcode) { 13650 case ZEND_PRE_INC_OBJ: 13651 | EXT_CALL zend_jit_pre_inc_typed_prop, r0 13652 break; 13653 case ZEND_PRE_DEC_OBJ: 13654 | EXT_CALL zend_jit_pre_dec_typed_prop, r0 13655 break; 13656 case ZEND_POST_INC_OBJ: 13657 | EXT_CALL zend_jit_post_inc_typed_prop, r0 13658 break; 13659 case ZEND_POST_DEC_OBJ: 13660 | EXT_CALL zend_jit_post_dec_typed_prop, r0 13661 break; 13662 default: 13663 ZEND_UNREACHABLE(); 13664 } 13665 |.if not(X64) 13666 | add r4, 12 13667 |.endif 13668 } 13669 } 13670 } 13671 13672 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 13673 zend_jit_addr var_addr = prop_addr; 13674 13675 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13676 if (Z_REG(prop_addr) != ZREG_FCARG1a || Z_OFFSET(prop_addr) != 0) { 13677 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 13678 } 13679 13680 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 13681 | GET_ZVAL_PTR FCARG1a, var_addr 13682 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 13683 | jnz >1 13684 | lea FCARG1a, aword [FCARG1a + offsetof(zend_reference, val)] 13685 |.cold_code 13686 |1: 13687 if (opline) { 13688 | SET_EX_OPLINE opline, r0 13689 } 13690 if (opline->result_type == IS_UNUSED) { 13691 | xor FCARG2a, FCARG2a 13692 } else { 13693 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13694 } 13695 switch (opline->opcode) { 13696 case ZEND_PRE_INC_OBJ: 13697 | EXT_CALL zend_jit_pre_inc_typed_ref, r0 13698 break; 13699 case ZEND_PRE_DEC_OBJ: 13700 | EXT_CALL zend_jit_pre_dec_typed_ref, r0 13701 break; 13702 case ZEND_POST_INC_OBJ: 13703 | EXT_CALL zend_jit_post_inc_typed_ref, r0 13704 break; 13705 case ZEND_POST_DEC_OBJ: 13706 | EXT_CALL zend_jit_post_dec_typed_ref, r0 13707 break; 13708 default: 13709 ZEND_UNREACHABLE(); 13710 } 13711 | jmp >9 13712 |.code 13713 13714 |2: 13715 | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2 13716 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 13717 if (opline->result_type != IS_UNUSED) { 13718 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R1, ZREG_R2 13719 } 13720 } 13721 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13722 | LONG_OP_WITH_32BIT_CONST add, var_addr, Z_L(1) 13723 } else { 13724 | LONG_OP_WITH_32BIT_CONST sub, var_addr, Z_L(1) 13725 } 13726 | jo >3 13727 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { 13728 if (opline->result_type != IS_UNUSED) { 13729 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R0, ZREG_R2 13730 } 13731 } 13732 |.cold_code 13733 |2: 13734 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { 13735 | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_R0, ZREG_R2 13736 | TRY_ADDREF MAY_BE_ANY, ah, r2 13737 } 13738 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13739 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13740 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13741 | EXT_CALL zend_jit_pre_inc, r0 13742 } else { 13743 | EXT_CALL increment_function, r0 13744 } 13745 } else { 13746 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13747 | LOAD_ZVAL_ADDR FCARG2a, res_addr 13748 | EXT_CALL zend_jit_pre_dec, r0 13749 } else { 13750 | EXT_CALL decrement_function, r0 13751 } 13752 } 13753 | jmp >4 13754 13755 |3: 13756 if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { 13757 |.if X64 13758 | mov64 rax, 0x43e0000000000000 13759 | SET_ZVAL_LVAL var_addr, rax 13760 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13761 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13762 | SET_ZVAL_LVAL res_addr, rax 13763 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13764 } 13765 |.else 13766 | SET_ZVAL_LVAL var_addr, 0 13767 | SET_ZVAL_W2 var_addr, 0x41e00000 13768 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13769 if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { 13770 | SET_ZVAL_LVAL res_addr, 0 13771 | SET_ZVAL_W2 res_addr, 0x41e00000 13772 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13773 } 13774 |.endif 13775 } else { 13776 |.if X64 13777 | mov64 rax, 0xc3e0000000000000 13778 | SET_ZVAL_LVAL var_addr, rax 13779 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13780 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13781 | SET_ZVAL_LVAL res_addr, rax 13782 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13783 } 13784 |.else 13785 | SET_ZVAL_LVAL var_addr, 0x00200000 13786 | SET_ZVAL_W2 var_addr, 0xc1e00000 13787 | SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE 13788 if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { 13789 | SET_ZVAL_LVAL res_addr, 0x00200000 13790 | SET_ZVAL_W2 res_addr, 0xc1e00000 13791 | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE 13792 } 13793 |.endif 13794 } 13795 | jmp >4 13796 |.code 13797 |4: 13798 } 13799 13800 if (needs_slow_path) { 13801 |.cold_code 13802 |7: 13803 | SET_EX_OPLINE opline, r0 13804 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 13805 | LOAD_ADDR FCARG2a, name 13806 |.if X64 13807 | mov CARG3, EX->run_time_cache 13808 | add CARG3, opline->extended_value 13809 if (opline->result_type == IS_UNUSED) { 13810 | xor CARG4, CARG4 13811 } else { 13812 | LOAD_ZVAL_ADDR CARG4, res_addr 13813 } 13814 |.else 13815 | sub r4, 8 13816 if (opline->result_type == IS_UNUSED) { 13817 | push 0 13818 } else { 13819 | PUSH_ZVAL_ADDR res_addr, r0 13820 } 13821 | mov r0, EX->run_time_cache 13822 | add r0, opline->extended_value 13823 | push r0 13824 |.endif 13825 13826 switch (opline->opcode) { 13827 case ZEND_PRE_INC_OBJ: 13828 | EXT_CALL zend_jit_pre_inc_obj_helper, r0 13829 break; 13830 case ZEND_PRE_DEC_OBJ: 13831 | EXT_CALL zend_jit_pre_dec_obj_helper, r0 13832 break; 13833 case ZEND_POST_INC_OBJ: 13834 | EXT_CALL zend_jit_post_inc_obj_helper, r0 13835 break; 13836 case ZEND_POST_DEC_OBJ: 13837 | EXT_CALL zend_jit_post_dec_obj_helper, r0 13838 break; 13839 default: 13840 ZEND_UNREACHABLE(); 13841 } 13842 13843 |.if not(X64) 13844 | add r4, 8 13845 |.endif 13846 13847 | jmp >9 13848 |.code 13849 } 13850 13851 |9: 13852 if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { 13853 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 13854 } 13855 13856 if (may_throw) { 13857 if (!zend_jit_check_exception(Dst)) { 13858 return 0; 13859 } 13860 } 13861 13862 return 1; 13863} 13864 13865static int zend_jit_assign_obj_op(dasm_State **Dst, 13866 const zend_op *opline, 13867 const zend_op_array *op_array, 13868 zend_ssa *ssa, 13869 const zend_ssa_op *ssa_op, 13870 uint32_t op1_info, 13871 zend_jit_addr op1_addr, 13872 uint32_t val_info, 13873 zend_ssa_range *val_range, 13874 zend_bool op1_indirect, 13875 zend_class_entry *ce, 13876 zend_bool ce_is_instanceof, 13877 zend_bool use_this, 13878 zend_class_entry *trace_ce, 13879 int may_throw) 13880{ 13881 zval *member; 13882 zend_string *name; 13883 zend_property_info *prop_info; 13884 zend_jit_addr val_addr = OP1_DATA_ADDR(); 13885 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 13886 zend_jit_addr prop_addr; 13887 zend_bool needs_slow_path = 0; 13888 binary_op_type binary_op = get_binary_op(opline->extended_value); 13889 13890 ZEND_ASSERT(opline->op2_type == IS_CONST); 13891 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 13892 ZEND_ASSERT(opline->result_type == IS_UNUSED); 13893 13894 member = RT_CONSTANT(opline, opline->op2); 13895 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 13896 name = Z_STR_P(member); 13897 prop_info = zend_get_known_property_info(ce, name, opline->op1_type == IS_UNUSED, op_array->filename); 13898 13899 if (opline->op1_type == IS_UNUSED || use_this) { 13900 | GET_ZVAL_PTR FCARG1a, this_addr 13901 } else { 13902 if (opline->op1_type == IS_VAR 13903 && (op1_info & MAY_BE_INDIRECT) 13904 && Z_REG(op1_addr) == ZREG_FP) { 13905 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13906 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 13907 | GET_Z_PTR FCARG1a, FCARG1a 13908 |1: 13909 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13910 } 13911 if (op1_info & MAY_BE_REF) { 13912 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 13913 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13914 } 13915 | ZVAL_DEREF FCARG1a, op1_info 13916 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13917 } 13918 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 13919 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13920 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13921 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13922 13923 if (!exit_addr) { 13924 return 0; 13925 } 13926 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 13927 } else { 13928 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 13929 |.cold_code 13930 |1: 13931 | SET_EX_OPLINE opline, r0 13932 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 13933 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 13934 } 13935 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 13936 if (op1_info & MAY_BE_UNDEF) { 13937 | EXT_CALL zend_jit_invalid_property_assign_op, r0 13938 } else { 13939 | EXT_CALL zend_jit_invalid_property_assign, r0 13940 } 13941 may_throw = 1; 13942 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 13943 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 13944 | jmp >8 13945 } else { 13946 | jmp >9 13947 } 13948 |.code 13949 } 13950 } 13951 | GET_ZVAL_PTR FCARG1a, op1_addr 13952 } 13953 13954 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 13955 prop_info = zend_get_known_property_info(trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); 13956 if (prop_info) { 13957 ce = trace_ce; 13958 ce_is_instanceof = 0; 13959 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 13960 if (!zend_jit_class_guard(Dst, opline, trace_ce)) { 13961 return 0; 13962 } 13963 if (ssa->var_info && ssa_op->op1_use >= 0) { 13964 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 13965 ssa->var_info[ssa_op->op1_use].ce = ce; 13966 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 13967 } 13968 if (ssa->var_info && ssa_op->op1_def >= 0) { 13969 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 13970 ssa->var_info[ssa_op->op1_def].ce = ce; 13971 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 13972 } 13973 } 13974 } 13975 } 13976 13977 if (!prop_info) { 13978 needs_slow_path = 1; 13979 13980 | mov r0, EX->run_time_cache 13981 | mov r2, aword [r0 + (opline+1)->extended_value] 13982 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 13983 | jne >7 13984 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 13985 | cmp aword [r0 + ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2], 0 13986 | jnz >7 13987 } 13988 | mov r0, aword [r0 + (opline+1)->extended_value + sizeof(void*)] 13989 | test r0, r0 13990 | jl >7 13991 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7 13992 | add FCARG1a, r0 13993 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 13994 } else { 13995 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset); 13996 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 13997 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 13998 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 13999 14000 if (!exit_addr) { 14001 return 0; 14002 } 14003 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 14004 } else { 14005 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7 14006 needs_slow_path = 1; 14007 } 14008 if (ZEND_TYPE_IS_SET(prop_info->type)) { 14009 uint32_t info = val_info; 14010 14011 if (opline) { 14012 | SET_EX_OPLINE opline, r0 14013 } 14014 14015 | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1 14016 |.cold_code 14017 |1: 14018 | GET_ZVAL_PTR FCARG1a, prop_addr 14019 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) { 14020 | LOAD_ZVAL_ADDR FCARG2a, val_addr 14021 } 14022 |.if X64 14023 | LOAD_ADDR CARG3, binary_op 14024 |.else 14025 | sub r4, 12 14026 | PUSH_ADDR binary_op, r0 14027 |.endif 14028 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 14029 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14030 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 14031 } else { 14032 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 14033 } 14034 |.if not(X64) 14035 | add r4, 12 14036 |.endif 14037 | jmp >9 14038 |.code 14039 14040 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14041 14042 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 14043 | LOAD_ADDR FCARG2a, prop_info 14044 } else { 14045 int prop_info_offset = 14046 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 14047 14048 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 14049 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 14050 | mov FCARG2a, aword[r0 + prop_info_offset] 14051 } 14052 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 14053 |.if X64 14054 | LOAD_ZVAL_ADDR CARG3, val_addr 14055 | LOAD_ADDR CARG4, binary_op 14056 |.else 14057 | sub r4, 8 14058 | PUSH_ADDR binary_op, r0 14059 | PUSH_ZVAL_ADDR val_addr, r0 14060 |.endif 14061 14062 | EXT_CALL zend_jit_assign_op_to_typed_prop, r0 14063 14064 |.if not(X64) 14065 | add r4, 8 14066 |.endif 14067 14068 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14069 info |= MAY_BE_RC1|MAY_BE_RCN; 14070 } 14071 14072 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, opline 14073 } 14074 } 14075 14076 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 14077 zend_jit_addr var_addr = prop_addr; 14078 uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 14079 uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; 14080 14081 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 14082 | LOAD_ZVAL_ADDR r0, prop_addr 14083 14084 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2 14085 | GET_ZVAL_PTR FCARG1a, var_addr 14086 | cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0 14087 | jnz >1 14088 | lea r0, aword [FCARG1a + offsetof(zend_reference, val)] 14089 |.cold_code 14090 |1: 14091 if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2a || Z_OFFSET(val_addr) != 0) { 14092 | LOAD_ZVAL_ADDR FCARG2a, val_addr 14093 } 14094 if (opline) { 14095 | SET_EX_OPLINE opline, r0 14096 } 14097 |.if X64 14098 | LOAD_ADDR CARG3, binary_op 14099 |.else 14100 | sub r4, 12 14101 | PUSH_ADDR binary_op, r0 14102 |.endif 14103 if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) 14104 && (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14105 | EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0 14106 } else { 14107 | EXT_CALL zend_jit_assign_op_to_typed_ref, r0 14108 } 14109 |.if not(X64) 14110 | add r4, 12 14111 |.endif 14112 | jmp >9 14113 |.code 14114 |2: 14115 14116 switch (opline->extended_value) { 14117 case ZEND_ADD: 14118 case ZEND_SUB: 14119 case ZEND_MUL: 14120 case ZEND_DIV: 14121 if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, 14122 1 /* may overflow */, 0)) { 14123 return 0; 14124 } 14125 break; 14126 case ZEND_BW_OR: 14127 case ZEND_BW_AND: 14128 case ZEND_BW_XOR: 14129 case ZEND_SL: 14130 case ZEND_SR: 14131 case ZEND_MOD: 14132 if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, 14133 IS_CV, opline->op1, var_addr, var_info, NULL, 14134 (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 14135 val_range, 14136 0, var_addr, var_def_info, var_info, 0)) { 14137 return 0; 14138 } 14139 break; 14140 case ZEND_CONCAT: 14141 if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, 14142 0)) { 14143 return 0; 14144 } 14145 break; 14146 default: 14147 ZEND_UNREACHABLE(); 14148 } 14149 } 14150 14151 if (needs_slow_path) { 14152 |.cold_code 14153 |7: 14154 | SET_EX_OPLINE opline, r0 14155 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 14156 | LOAD_ADDR FCARG2a, name 14157 |.if X64 14158 | LOAD_ZVAL_ADDR CARG3, val_addr 14159 | mov CARG4, EX->run_time_cache 14160 | add CARG4, (opline+1)->extended_value 14161 |.if X64WIN 14162 | LOAD_ADDR r0, binary_op 14163 | mov aword A5, r0 14164 |.else 14165 | LOAD_ADDR CARG5, binary_op 14166 |.endif 14167 |.else 14168 | sub r4, 4 14169 | PUSH_ADDR binary_op, r0 14170 | mov r0, EX->run_time_cache 14171 | add r0, (opline+1)->extended_value 14172 | push r0 14173 | PUSH_ZVAL_ADDR val_addr, r0 14174 |.endif 14175 14176 | EXT_CALL zend_jit_assign_obj_op_helper, r0 14177 14178 |.if not(X64) 14179 | add r4, 4 14180 |.endif 14181 14182 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14183 val_info |= MAY_BE_RC1|MAY_BE_RCN; 14184 } 14185 14186 |8: 14187 | // FREE_OP_DATA(); 14188 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14189 | jmp >9 14190 |.code 14191 } 14192 14193 |9: 14194 if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { 14195 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 14196 } 14197 14198 if (may_throw) { 14199 if (!zend_jit_check_exception(Dst)) { 14200 return 0; 14201 } 14202 } 14203 14204 return 1; 14205} 14206 14207static int zend_jit_assign_obj(dasm_State **Dst, 14208 const zend_op *opline, 14209 const zend_op_array *op_array, 14210 zend_ssa *ssa, 14211 const zend_ssa_op *ssa_op, 14212 uint32_t op1_info, 14213 zend_jit_addr op1_addr, 14214 uint32_t val_info, 14215 zend_bool op1_indirect, 14216 zend_class_entry *ce, 14217 zend_bool ce_is_instanceof, 14218 zend_bool use_this, 14219 zend_class_entry *trace_ce, 14220 int may_throw) 14221{ 14222 zval *member; 14223 zend_string *name; 14224 zend_property_info *prop_info; 14225 zend_jit_addr val_addr = OP1_DATA_ADDR(); 14226 zend_jit_addr res_addr = 0; 14227 zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); 14228 zend_jit_addr prop_addr; 14229 zend_bool needs_slow_path = 0; 14230 zend_bool needs_val_dtor = 0; 14231 14232 if (RETURN_VALUE_USED(opline)) { 14233 res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 14234 } 14235 14236 ZEND_ASSERT(opline->op2_type == IS_CONST); 14237 ZEND_ASSERT(op1_info & MAY_BE_OBJECT); 14238 14239 member = RT_CONSTANT(opline, opline->op2); 14240 ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); 14241 name = Z_STR_P(member); 14242 prop_info = zend_get_known_property_info(ce, name, opline->op1_type == IS_UNUSED, op_array->filename); 14243 14244 if (opline->op1_type == IS_UNUSED || use_this) { 14245 | GET_ZVAL_PTR FCARG1a, this_addr 14246 } else { 14247 if (opline->op1_type == IS_VAR 14248 && (op1_info & MAY_BE_INDIRECT) 14249 && Z_REG(op1_addr) == ZREG_FP) { 14250 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14251 | IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1 14252 | GET_Z_PTR FCARG1a, FCARG1a 14253 |1: 14254 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 14255 } 14256 if (op1_info & MAY_BE_REF) { 14257 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 14258 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14259 } 14260 | ZVAL_DEREF FCARG1a, op1_info 14261 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 14262 } 14263 if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { 14264 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14265 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14266 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14267 14268 if (!exit_addr) { 14269 return 0; 14270 } 14271 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr 14272 } else { 14273 | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1 14274 |.cold_code 14275 |1: 14276 | SET_EX_OPLINE opline, r0 14277 if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 14278 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 14279 } 14280 | LOAD_ADDR FCARG2a, ZSTR_VAL(name) 14281 | EXT_CALL zend_jit_invalid_property_assign, r0 14282 if (RETURN_VALUE_USED(opline)) { 14283 | SET_ZVAL_TYPE_INFO res_addr, IS_NULL 14284 } 14285 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 14286 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14287 needs_val_dtor = 1; 14288 | jmp >7 14289 } else { 14290 | jmp >9 14291 } 14292 |.code 14293 } 14294 } 14295 | GET_ZVAL_PTR FCARG1a, op1_addr 14296 } 14297 14298 if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { 14299 prop_info = zend_get_known_property_info(trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); 14300 if (prop_info) { 14301 ce = trace_ce; 14302 ce_is_instanceof = 0; 14303 if (!(op1_info & MAY_BE_CLASS_GUARD)) { 14304 if (!zend_jit_class_guard(Dst, opline, trace_ce)) { 14305 return 0; 14306 } 14307 if (ssa->var_info && ssa_op->op1_use >= 0) { 14308 ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; 14309 ssa->var_info[ssa_op->op1_use].ce = ce; 14310 ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; 14311 } 14312 if (ssa->var_info && ssa_op->op1_def >= 0) { 14313 ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; 14314 ssa->var_info[ssa_op->op1_def].ce = ce; 14315 ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; 14316 } 14317 } 14318 } 14319 } 14320 14321 if (!prop_info) { 14322 needs_slow_path = 1; 14323 14324 | mov r0, EX->run_time_cache 14325 | mov r2, aword [r0 + opline->extended_value] 14326 | cmp r2, aword [FCARG1a + offsetof(zend_object, ce)] 14327 | jne >5 14328 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 14329 | mov FCARG2a, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2] 14330 } 14331 | mov r0, aword [r0 + opline->extended_value + sizeof(void*)] 14332 | test r0, r0 14333 | jl >5 14334 | IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >5 14335 | add FCARG1a, r0 14336 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 14337 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { 14338 | test FCARG2a, FCARG2a 14339 | jnz >1 14340 |.cold_code 14341 |1: 14342 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14343 | SET_EX_OPLINE opline, r0 14344 |.if X64 14345 | LOAD_ZVAL_ADDR CARG3, val_addr 14346 if (RETURN_VALUE_USED(opline)) { 14347 | LOAD_ZVAL_ADDR CARG4, res_addr 14348 } else { 14349 | xor CARG4, CARG4 14350 } 14351 |.else 14352 | sub r4, 8 14353 if (RETURN_VALUE_USED(opline)) { 14354 | PUSH_ZVAL_ADDR res_addr, r0 14355 } else { 14356 | push 0 14357 } 14358 | PUSH_ZVAL_ADDR val_addr, r0 14359 |.endif 14360 14361 | EXT_CALL zend_jit_assign_to_typed_prop, r0 14362 14363 |.if not(X64) 14364 | add r4, 8 14365 |.endif 14366 14367 if ((opline+1)->op1_type == IS_CONST) { 14368 | // TODO: ??? 14369 | // if (Z_TYPE_P(value) == orig_type) { 14370 | // CACHE_PTR_EX(cache_slot + 2, NULL); 14371 } 14372 14373 if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) 14374 && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { 14375 | jmp >7 14376 } else { 14377 | jmp >9 14378 } 14379 |.code 14380 } 14381 } else { 14382 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, prop_info->offset); 14383 if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set) { 14384 // Undefined property with magic __get()/__set() 14385 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14386 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14387 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14388 14389 if (!exit_addr) { 14390 return 0; 14391 } 14392 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr 14393 } else { 14394 | IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >5 14395 needs_slow_path = 1; 14396 } 14397 } 14398 if (ZEND_TYPE_IS_SET(prop_info->type)) { 14399 uint32_t info = val_info; 14400 14401 | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); 14402 | SET_EX_OPLINE opline, r0 14403 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { 14404 | LOAD_ADDR FCARG2a, prop_info 14405 } else { 14406 int prop_info_offset = 14407 (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); 14408 14409 | mov r0, aword [FCARG1a + offsetof(zend_object, ce)] 14410 | mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)] 14411 | mov FCARG2a, aword[r0 + prop_info_offset] 14412 } 14413 | LOAD_ZVAL_ADDR FCARG1a, prop_addr 14414 |.if X64 14415 | LOAD_ZVAL_ADDR CARG3, val_addr 14416 if (RETURN_VALUE_USED(opline)) { 14417 | LOAD_ZVAL_ADDR CARG4, res_addr 14418 } else { 14419 | xor CARG4, CARG4 14420 } 14421 |.else 14422 | sub r4, 8 14423 if (RETURN_VALUE_USED(opline)) { 14424 | PUSH_ZVAL_ADDR res_addr, r0 14425 } else { 14426 | push 0 14427 } 14428 | PUSH_ZVAL_ADDR val_addr, r0 14429 |.endif 14430 14431 | EXT_CALL zend_jit_assign_to_typed_prop, r0 14432 14433 |.if not(X64) 14434 | add r4, 8 14435 |.endif 14436 14437 if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14438 info |= MAY_BE_RC1|MAY_BE_RCN; 14439 } 14440 14441 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, opline 14442 } 14443 } 14444 14445 if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { 14446 // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); 14447 if (opline->result_type == IS_UNUSED) { 14448 if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 14449 return 0; 14450 } 14451 } else { 14452 if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { 14453 return 0; 14454 } 14455 } 14456 } 14457 14458 if (needs_slow_path) { 14459 |.cold_code 14460 |5: 14461 | SET_EX_OPLINE opline, r0 14462 | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); 14463 | LOAD_ADDR FCARG2a, name 14464 |.if X64 14465 | LOAD_ZVAL_ADDR CARG3, val_addr 14466 | mov CARG4, EX->run_time_cache 14467 | add CARG4, opline->extended_value 14468 if (RETURN_VALUE_USED(opline)) { 14469 |.if X64WIN 14470 | LOAD_ZVAL_ADDR r0, res_addr 14471 | mov aword A5, r0 14472 |.else 14473 | LOAD_ZVAL_ADDR CARG5, res_addr 14474 |.endif 14475 } else { 14476 |.if X64WIN 14477 | mov aword A5, 0 14478 |.else 14479 | xor CARG5, CARG5 14480 |.endif 14481 } 14482 |.else 14483 | sub r4, 4 14484 if (RETURN_VALUE_USED(opline)) { 14485 | PUSH_ZVAL_ADDR res_addr, r0 14486 } else { 14487 | push 0 14488 } 14489 | mov r0, EX->run_time_cache 14490 | add r0, opline->extended_value 14491 | push r0 14492 | PUSH_ZVAL_ADDR val_addr, r0 14493 |.endif 14494 14495 | EXT_CALL zend_jit_assign_obj_helper, r0 14496 14497 |.if not(X64) 14498 | add r4, 4 14499 |.endif 14500 14501 if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 14502 val_info |= MAY_BE_RC1|MAY_BE_RCN; 14503 } 14504 14505 |7: 14506 | // FREE_OP_DATA(); 14507 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14508 | jmp >9 14509 |.code 14510 } else if (needs_val_dtor) { 14511 |.cold_code 14512 |7: 14513 | // FREE_OP_DATA(); 14514 | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline 14515 | jmp >9 14516 |.code 14517 } 14518 14519 |9: 14520 if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { 14521 | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline 14522 } 14523 14524 if (may_throw) { 14525 if (!zend_jit_check_exception(Dst)) { 14526 return 0; 14527 } 14528 } 14529 14530 return 1; 14531} 14532 14533static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) 14534{ 14535 zend_jit_addr op1_addr = OP1_ADDR(); 14536 14537 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { 14538 if (may_throw) { 14539 | SET_EX_OPLINE opline, r0 14540 } 14541 if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { 14542 if (op1_info & MAY_BE_ARRAY) { 14543 | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7 14544 } 14545 | mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_iter_idx)] 14546 | cmp FCARG1d, -1 14547 | je >7 14548 | EXT_CALL zend_hash_iterator_del, r0 14549 |7: 14550 } 14551 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline 14552 if (may_throw) { 14553 if (!zend_jit_check_exception(Dst)) { 14554 return 0; 14555 } 14556 } 14557 } 14558 14559 return 1; 14560} 14561 14562static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 14563{ 14564 if (opline->op1_type == IS_CONST) { 14565 zval *zv; 14566 size_t len; 14567 14568 zv = RT_CONSTANT(opline, opline->op1); 14569 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14570 len = Z_STRLEN_P(zv); 14571 14572 if (len > 0) { 14573 const char *str = Z_STRVAL_P(zv); 14574 14575 | SET_EX_OPLINE opline, r0 14576 |.if X64 14577 | LOAD_ADDR CARG1, str 14578 | LOAD_ADDR CARG2, len 14579 | EXT_CALL zend_write, r0 14580 |.else 14581 | mov aword A2, len 14582 | mov aword A1, str 14583 | EXT_CALL zend_write, r0 14584 |.endif 14585 if (!zend_jit_check_exception(Dst)) { 14586 return 0; 14587 } 14588 } 14589 } else { 14590 zend_jit_addr op1_addr = OP1_ADDR(); 14591 14592 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14593 14594 | SET_EX_OPLINE opline, r0 14595 | GET_ZVAL_PTR r0, op1_addr 14596 |.if X64 14597 | lea CARG1, aword [r0 + offsetof(zend_string, val)] 14598 | mov CARG2, aword [r0 + offsetof(zend_string, len)] 14599 | EXT_CALL zend_write, r0 14600 |.else 14601 | add r0, offsetof(zend_string, val) 14602 | mov aword A1, r0 14603 | mov r0, aword [r0 + (offsetof(zend_string, len)-offsetof(zend_string, val))] 14604 | mov aword A2, r0 14605 | EXT_CALL zend_write, r0 14606 |.endif 14607 if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { 14608 | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline 14609 } 14610 if (!zend_jit_check_exception(Dst)) { 14611 return 0; 14612 } 14613 } 14614 return 1; 14615} 14616 14617static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) 14618{ 14619 zend_jit_addr res_addr = RES_ADDR(); 14620 14621 if (opline->op1_type == IS_CONST) { 14622 zval *zv; 14623 size_t len; 14624 14625 zv = RT_CONSTANT(opline, opline->op1); 14626 ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); 14627 len = Z_STRLEN_P(zv); 14628 14629 | SET_ZVAL_LVAL res_addr, len 14630 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14631 } else { 14632 ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); 14633 14634 | GET_ZVAL_PTR r0, op1_addr 14635 | mov r0, aword [r0 + offsetof(zend_string, len)] 14636 | SET_ZVAL_LVAL res_addr, r0 14637 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 14638 | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline 14639 } 14640 return 1; 14641} 14642 14643static int zend_jit_load_this(dasm_State **Dst, uint32_t var) 14644{ 14645 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); 14646 14647 | mov FCARG1a, aword EX->This.value.ptr 14648 | SET_ZVAL_PTR var_addr, FCARG1a 14649 | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX 14650 | GC_ADDREF FCARG1a 14651 14652 return 1; 14653} 14654 14655static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_bool check_only) 14656{ 14657 if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) { 14658 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 14659 if (!JIT_G(current_frame) || 14660 !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { 14661 14662 int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); 14663 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14664 14665 | cmp byte EX->This.u1.v.type, IS_OBJECT 14666 | jne &exit_addr 14667 14668 if (JIT_G(current_frame)) { 14669 TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); 14670 } 14671 } 14672 } else { 14673 14674 | cmp byte EX->This.u1.v.type, IS_OBJECT 14675 | jne >1 14676 |.cold_code 14677 |1: 14678 | SET_EX_OPLINE opline, r0 14679 | jmp ->invalid_this 14680 |.code 14681 } 14682 } 14683 14684 if (!check_only) { 14685 if (!zend_jit_load_this(Dst, opline->result.var)) { 14686 return 0; 14687 } 14688 } 14689 14690 return 1; 14691} 14692 14693static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) 14694{ 14695 uint32_t count; 14696 Bucket *p; 14697 const zend_op *target; 14698 int b; 14699 int32_t exit_point; 14700 const void *exit_addr; 14701 14702 | test r0, r0 14703 if (default_label) { 14704 | jz &default_label 14705 } else if (next_opline) { 14706 | jz >3 14707 } else { 14708 | jz =>default_b 14709 } 14710 | LOAD_ADDR FCARG1a, jumptable 14711 | sub r0, aword [FCARG1a + offsetof(HashTable, arData)] 14712 | mov FCARG1a, (sizeof(Bucket) / sizeof(void*)) 14713 |.if X64 14714 | cqo 14715 |.else 14716 | cdq 14717 |.endif 14718 | idiv FCARG1a 14719 |.if X64 14720 if (!IS_32BIT(dasm_end)) { 14721 | lea FCARG1a, aword [>4] 14722 | jmp aword [FCARG1a + r0] 14723 } else { 14724 | jmp aword [r0 + >4] 14725 } 14726 |.else 14727 | jmp aword [r0 + >4] 14728 |.endif 14729 |.jmp_table 14730 |.align aword 14731 |4: 14732 if (trace_info) { 14733 trace_info->jmp_table_size += zend_hash_num_elements(jumptable); 14734 } 14735 14736 count = jumptable->nNumUsed; 14737 p = jumptable->arData; 14738 do { 14739 if (Z_TYPE(p->val) == IS_UNDEF) { 14740 if (default_label) { 14741 | .aword &default_label 14742 } else if (next_opline) { 14743 | .aword >3 14744 } else { 14745 | .aword =>default_b 14746 } 14747 } else { 14748 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 14749 if (!next_opline) { 14750 b = ssa->cfg.map[target - op_array->opcodes]; 14751 | .aword =>b 14752 } else if (next_opline == target) { 14753 | .aword >3 14754 } else { 14755 exit_point = zend_jit_trace_get_exit_point(target, 0); 14756 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14757 | .aword &exit_addr 14758 } 14759 } 14760 p++; 14761 count--; 14762 } while (count); 14763 |.code 14764 14765 return 1; 14766} 14767 14768static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) 14769{ 14770 HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 14771 const zend_op *next_opline = NULL; 14772 14773 if (trace) { 14774 ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); 14775 ZEND_ASSERT(trace->opline != NULL); 14776 next_opline = trace->opline; 14777 } 14778 14779 if (opline->op1_type == IS_CONST) { 14780 zval *zv = RT_CONSTANT(opline, opline->op1); 14781 zval *jump_zv = NULL; 14782 int b; 14783 14784 if (opline->opcode == ZEND_SWITCH_LONG) { 14785 if (Z_TYPE_P(zv) == IS_LONG) { 14786 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14787 } 14788 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14789 if (Z_TYPE_P(zv) == IS_STRING) { 14790 jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1); 14791 } 14792 } else if (opline->opcode == ZEND_MATCH) { 14793 if (Z_TYPE_P(zv) == IS_LONG) { 14794 jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); 14795 } else if (Z_TYPE_P(zv) == IS_STRING) { 14796 jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1); 14797 } 14798 } else { 14799 ZEND_UNREACHABLE(); 14800 } 14801 if (next_opline) { 14802 const zend_op *target; 14803 14804 if (jump_zv != NULL) { 14805 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); 14806 } else { 14807 target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14808 } 14809 ZEND_ASSERT(target == next_opline); 14810 } else { 14811 if (jump_zv != NULL) { 14812 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; 14813 } else { 14814 b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; 14815 } 14816 | jmp =>b 14817 } 14818 } else { 14819 zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; 14820 uint32_t op1_info = OP1_INFO(); 14821 zend_jit_addr op1_addr = OP1_ADDR(); 14822 const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); 14823 const zend_op *target; 14824 int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; 14825 int b; 14826 int32_t exit_point; 14827 const void *fallback_label = NULL; 14828 const void *default_label = NULL; 14829 const void *exit_addr; 14830 14831 if (next_opline) { 14832 if (next_opline != opline + 1) { 14833 exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); 14834 fallback_label = zend_jit_trace_get_exit_addr(exit_point); 14835 } 14836 if (next_opline != default_opline) { 14837 exit_point = zend_jit_trace_get_exit_point(default_opline, 0); 14838 default_label = zend_jit_trace_get_exit_addr(exit_point); 14839 } 14840 } 14841 14842 if (opline->opcode == ZEND_SWITCH_LONG) { 14843 if (op1_info & MAY_BE_LONG) { 14844 if (op1_info & MAY_BE_REF) { 14845 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1 14846 | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr 14847 |.cold_code 14848 |1: 14849 | // ZVAL_DEREF(op) 14850 if (fallback_label) { 14851 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label 14852 } else { 14853 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 14854 } 14855 | GET_ZVAL_PTR FCARG2a, op1_addr 14856 if (fallback_label) { 14857 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, &fallback_label 14858 } else { 14859 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3 14860 } 14861 | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.lval)] 14862 | jmp >2 14863 |.code 14864 |2: 14865 } else { 14866 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14867 if (fallback_label) { 14868 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label 14869 } else { 14870 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 14871 } 14872 } 14873 | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr 14874 } 14875 if (HT_IS_PACKED(jumptable)) { 14876 uint32_t count = jumptable->nNumUsed; 14877 Bucket *p = jumptable->arData; 14878 14879 | cmp FCARG2a, jumptable->nNumUsed 14880 if (default_label) { 14881 | jae &default_label 14882 } else if (next_opline) { 14883 | jae >3 14884 } else { 14885 | jae =>default_b 14886 } 14887 |.if X64 14888 if (!IS_32BIT(dasm_end)) { 14889 | lea r0, aword [>4] 14890 | jmp aword [r0 + FCARG2a * 8] 14891 } else { 14892 | jmp aword [FCARG2a * 8 + >4] 14893 } 14894 |.else 14895 | jmp aword [FCARG2a * 4 + >4] 14896 |.endif 14897 |.jmp_table 14898 |.align aword 14899 |4: 14900 if (trace_info) { 14901 trace_info->jmp_table_size += count; 14902 } 14903 p = jumptable->arData; 14904 do { 14905 if (Z_TYPE(p->val) == IS_UNDEF) { 14906 if (default_label) { 14907 | .aword &default_label 14908 } else if (next_opline) { 14909 | .aword >3 14910 } else { 14911 | .aword =>default_b 14912 } 14913 } else { 14914 target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); 14915 if (!next_opline) { 14916 b = ssa->cfg.map[target - op_array->opcodes]; 14917 | .aword =>b 14918 } else if (next_opline == target) { 14919 | .aword >3 14920 } else { 14921 exit_point = zend_jit_trace_get_exit_point(target, 0); 14922 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 14923 | .aword &exit_addr 14924 } 14925 } 14926 p++; 14927 count--; 14928 } while (count); 14929 |.code 14930 |3: 14931 } else { 14932 | LOAD_ADDR FCARG1a, jumptable 14933 | EXT_CALL zend_hash_index_find, r0 14934 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14935 return 0; 14936 } 14937 |3: 14938 } 14939 } 14940 } else if (opline->opcode == ZEND_SWITCH_STRING) { 14941 if (op1_info & MAY_BE_STRING) { 14942 if (op1_info & MAY_BE_REF) { 14943 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1 14944 | GET_ZVAL_PTR FCARG2a, op1_addr 14945 |.cold_code 14946 |1: 14947 | // ZVAL_DEREF(op) 14948 if (fallback_label) { 14949 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label 14950 } else { 14951 | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3 14952 } 14953 | GET_ZVAL_PTR FCARG2a, op1_addr 14954 if (fallback_label) { 14955 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, &fallback_label 14956 } else { 14957 | IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3 14958 } 14959 | mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.ptr)] 14960 | jmp >2 14961 |.code 14962 |2: 14963 } else { 14964 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { 14965 if (fallback_label) { 14966 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label 14967 } else { 14968 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 14969 } 14970 } 14971 | GET_ZVAL_PTR FCARG2a, op1_addr 14972 } 14973 | LOAD_ADDR FCARG1a, jumptable 14974 | EXT_CALL zend_hash_find, r0 14975 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 14976 return 0; 14977 } 14978 |3: 14979 } 14980 } else if (opline->opcode == ZEND_MATCH) { 14981 if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { 14982 if (op1_info & MAY_BE_REF) { 14983 | LOAD_ZVAL_ADDR FCARG2a, op1_addr 14984 | ZVAL_DEREF FCARG2a, op1_info 14985 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 14986 } 14987 | LOAD_ADDR FCARG1a, jumptable 14988 if (op1_info & MAY_BE_LONG) { 14989 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { 14990 if (op1_info & MAY_BE_STRING) { 14991 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5 14992 } else if (op1_info & MAY_BE_UNDEF) { 14993 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6 14994 } else if (default_label) { 14995 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label 14996 } else if (next_opline) { 14997 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3 14998 } else { 14999 | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b 15000 } 15001 } 15002 | GET_ZVAL_LVAL ZREG_FCARG2a, op1_addr 15003 | EXT_CALL zend_hash_index_find, r0 15004 if (op1_info & MAY_BE_STRING) { 15005 | jmp >2 15006 } 15007 } 15008 if (op1_info & MAY_BE_STRING) { 15009 |5: 15010 if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { 15011 if (op1_info & MAY_BE_UNDEF) { 15012 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6 15013 } else if (default_label) { 15014 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label 15015 } else if (next_opline) { 15016 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3 15017 } else { 15018 | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b 15019 } 15020 } 15021 | GET_ZVAL_PTR FCARG2a, op1_addr 15022 | EXT_CALL zend_hash_find, r0 15023 } 15024 |2: 15025 if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { 15026 return 0; 15027 } 15028 } 15029 if (op1_info & MAY_BE_UNDEF) { 15030 |6: 15031 if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { 15032 if (default_label) { 15033 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label 15034 } else if (next_opline) { 15035 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3 15036 } else { 15037 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b 15038 } 15039 } 15040 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); 15041 | SET_EX_OPLINE opline, r0 15042 | mov FCARG1d, opline->op1.var 15043 | EXT_CALL zend_jit_undefined_op_helper, r0 15044 if (!zend_jit_check_exception_undef_result(Dst, opline)) { 15045 return 0; 15046 } 15047 } 15048 if (default_label) { 15049 | jmp &default_label 15050 } else if (next_opline) { 15051 | jmp >3 15052 } else { 15053 | jmp =>default_b 15054 } 15055 |3: 15056 } else { 15057 ZEND_UNREACHABLE(); 15058 } 15059 } 15060 return 1; 15061} 15062 15063static zend_bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) 15064{ 15065 zend_arg_info *arg_info = &op_array->arg_info[-1]; 15066 ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); 15067 zend_jit_addr op1_addr = OP1_ADDR(); 15068 zend_bool needs_slow_check = 1; 15069 zend_bool slow_check_in_cold = 1; 15070 uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; 15071 15072 if (type_mask == 0) { 15073 slow_check_in_cold = 0; 15074 } else { 15075 if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { 15076 slow_check_in_cold = 0; 15077 } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { 15078 needs_slow_check = 0; 15079 } else if (is_power_of_two(type_mask)) { 15080 uint32_t type_code = concrete_type(type_mask); 15081 | IF_NOT_ZVAL_TYPE op1_addr, type_code, >6 15082 } else { 15083 | mov edx, 1 15084 | GET_ZVAL_TYPE cl, op1_addr 15085 | shl edx, cl 15086 | test edx, type_mask 15087 | je >6 15088 } 15089 } 15090 if (needs_slow_check) { 15091 if (slow_check_in_cold) { 15092 |.cold_code 15093 |6: 15094 } 15095 | SET_EX_OPLINE opline, r1 15096 if (op1_info & MAY_BE_UNDEF) { 15097 | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7 15098 | mov FCARG1a, opline->op1.var 15099 | EXT_CALL zend_jit_undefined_op_helper, FCARG2a 15100 | test r0, r0 15101 | jz ->exception_handler 15102 | LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval 15103 | jmp >8 15104 } 15105 |7: 15106 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 15107 |8: 15108 | mov FCARG2a, EX->func 15109 |.if X64 15110 | LOAD_ADDR CARG3, (ptrdiff_t)arg_info 15111 | mov r0, EX->run_time_cache 15112 | lea CARG4, aword [r0+opline->op2.num] 15113 | EXT_CALL zend_jit_verify_return_slow, r0 15114 |.else 15115 | sub r4, 8 15116 | mov r0, EX->run_time_cache 15117 | add r0, opline->op2.num 15118 | push r0 15119 | push (ptrdiff_t)arg_info 15120 | EXT_CALL zend_jit_verify_return_slow, r0 15121 | add r4, 8 15122 |.endif 15123 if (!zend_jit_check_exception(Dst)) { 15124 return 0; 15125 } 15126 if (slow_check_in_cold) { 15127 | jmp >9 15128 |.code 15129 } 15130 } 15131 |9: 15132 return 1; 15133} 15134 15135static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 15136{ 15137 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15138 15139 // TODO: support for empty() ??? 15140 ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); 15141 15142 if (op1_info & MAY_BE_REF) { 15143 if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) { 15144 | LOAD_ZVAL_ADDR FCARG1a, op1_addr 15145 op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 15146 } 15147 | ZVAL_DEREF FCARG1a, op1_info 15148 |1: 15149 } 15150 15151 if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { 15152 if (exit_addr) { 15153 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); 15154 } else if (smart_branch_opcode) { 15155 if (smart_branch_opcode == ZEND_JMPNZ) { 15156 | jmp =>target_label 15157 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 15158 | jmp =>target_label2 15159 } 15160 } else { 15161 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE 15162 } 15163 } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { 15164 if (exit_addr) { 15165 ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); 15166 } else if (smart_branch_opcode) { 15167 if (smart_branch_opcode != ZEND_JMPNZ) { 15168 | jmp =>target_label 15169 } 15170 } else { 15171 | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE 15172 } 15173 } else { 15174 ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); 15175 | cmp byte [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)], IS_NULL 15176 if (exit_addr) { 15177 if (smart_branch_opcode == ZEND_JMPNZ) { 15178 | jg &exit_addr 15179 } else { 15180 | jle &exit_addr 15181 } 15182 } else if (smart_branch_opcode) { 15183 if (smart_branch_opcode == ZEND_JMPZ) { 15184 | jle =>target_label 15185 } else if (smart_branch_opcode == ZEND_JMPNZ) { 15186 | jg =>target_label 15187 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 15188 | jle =>target_label 15189 | jmp =>target_label2 15190 } else { 15191 ZEND_UNREACHABLE(); 15192 } 15193 } else { 15194 | setg al 15195 | movzx eax, al 15196 | lea eax, [eax + IS_FALSE] 15197 | SET_ZVAL_TYPE_INFO res_addr, eax 15198 } 15199 } 15200 15201 return 1; 15202} 15203 15204static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) 15205{ 15206 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15207 15208 if (opline->op1_type == IS_CONST) { 15209 zval *zv = RT_CONSTANT(opline, opline->op1); 15210 15211 | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0 15212 if (Z_REFCOUNTED_P(zv)) { 15213 | ADDREF_CONST zv, r0 15214 } 15215 } else { 15216 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 15217 15218 | // ZVAL_COPY(res, value); 15219 | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_FCARG1a 15220 if (opline->op1_type == IS_CV) { 15221 | TRY_ADDREF op1_info, ah, FCARG1a 15222 } 15223 } 15224 | // Z_FE_POS_P(res) = 0; 15225 | mov dword [FP + opline->result.var + offsetof(zval, u2.fe_pos)], 0 15226 15227 return 1; 15228} 15229 15230static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr) 15231{ 15232 zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); 15233 15234 | // array = EX_VAR(opline->op1.var); 15235 | // fe_ht = Z_ARRVAL_P(array); 15236 | GET_ZVAL_PTR FCARG2a, op1_addr 15237 | // pos = Z_FE_POS_P(array); 15238 | mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)] 15239 | // p = fe_ht->arData + pos; 15240 |.if X64 15241 || ZEND_ASSERT(sizeof(Bucket) == 32); 15242 | mov eax, FCARG1d 15243 | shl r0, 5 15244 |.else 15245 | imul r0, FCARG1a, sizeof(Bucket) 15246 |.endif 15247 | add r0, aword [FCARG2a + offsetof(zend_array, arData)] 15248 |1: 15249 | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { 15250 | cmp dword [FCARG2a + offsetof(zend_array, nNumUsed)], FCARG1d 15251 | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); 15252 | // ZEND_VM_CONTINUE(); 15253 if (exit_addr) { 15254 if (exit_opcode == ZEND_JMP) { 15255 | jbe &exit_addr 15256 } else { 15257 | jbe >3 15258 } 15259 } else { 15260 | jbe =>target_label 15261 } 15262 | // pos++; 15263 | add FCARG1d, 1 15264 | // value_type = Z_TYPE_INFO_P(value); 15265 | // if (EXPECTED(value_type != IS_UNDEF)) { 15266 | IF_Z_TYPE r0, IS_UNDEF, >2 15267 if (!exit_addr || exit_opcode == ZEND_JMP) { 15268 | IF_NOT_Z_TYPE r0, IS_INDIRECT, >3 15269 } else { 15270 | IF_NOT_Z_TYPE r0, IS_INDIRECT, &exit_addr 15271 } 15272 | // value = Z_INDIRECT_P(value); 15273 | GET_Z_PTR FCARG2a, r0 15274 | // value_type = Z_TYPE_INFO_P(value); 15275 | // if (EXPECTED(value_type != IS_UNDEF)) { 15276 if (!exit_addr || exit_opcode == ZEND_JMP) { 15277 | IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >4 15278 } else { 15279 | IF_NOT_Z_TYPE r0, IS_UNDEF, &exit_addr 15280 } 15281 | GET_ZVAL_PTR FCARG2a, op1_addr // reload 15282 |2: 15283 | // p++; 15284 | add r0, sizeof(Bucket) 15285 | jmp <1 15286 |3: 15287 15288 if (!exit_addr || exit_opcode == ZEND_JMP) { 15289 zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2a, 0); 15290 zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); 15291 uint32_t val_info; 15292 15293 | mov FCARG2a, r0 15294 |4: 15295 | // Z_FE_POS_P(array) = pos + 1; 15296 | mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], FCARG1d 15297 15298 if (RETURN_VALUE_USED(opline)) { 15299 zend_jit_addr res_addr = RES_ADDR(); 15300 15301 if ((op1_info & MAY_BE_ARRAY_KEY_LONG) 15302 && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { 15303 | // if (!p->key) { 15304 | cmp aword [r0 + offsetof(Bucket, key)], 0 15305 | jz >2 15306 } 15307 if (op1_info & MAY_BE_ARRAY_KEY_STRING) { 15308 | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); 15309 | mov FCARG1a, aword [r0 + offsetof(Bucket, key)] 15310 | SET_ZVAL_PTR res_addr, FCARG1a 15311 | test dword [FCARG1a + offsetof(zend_refcounted, gc.u.type_info)], IS_STR_INTERNED 15312 | jz >1 15313 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING 15314 | jmp >3 15315 |1: 15316 | GC_ADDREF FCARG1a 15317 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX 15318 15319 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 15320 | jmp >3 15321 |2: 15322 } 15323 } 15324 if (op1_info & MAY_BE_ARRAY_KEY_LONG) { 15325 | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); 15326 | mov FCARG1a, aword [r0 + offsetof(Bucket, h)] 15327 | SET_ZVAL_LVAL res_addr, FCARG1a 15328 | SET_ZVAL_TYPE_INFO res_addr, IS_LONG 15329 } 15330 |3: 15331 } 15332 15333 val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); 15334 if (val_info & MAY_BE_ARRAY) { 15335 val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 15336 } 15337 if (op1_info & MAY_BE_ARRAY_OF_REF) { 15338 val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | 15339 MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; 15340 } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { 15341 val_info |= MAY_BE_RC1 | MAY_BE_RCN; 15342 } 15343 15344 if (opline->op2_type == IS_CV) { 15345 | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); 15346 if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { 15347 return 0; 15348 } 15349 } else { 15350 | // ZVAL_COPY(res, value); 15351 | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_R0, ZREG_FCARG1a 15352 | TRY_ADDREF val_info, ah, FCARG1a 15353 } 15354 } 15355 15356 return 1; 15357} 15358 15359static int zend_jit_fetch_constant(dasm_State **Dst, 15360 const zend_op *opline, 15361 const zend_op_array *op_array, 15362 zend_ssa *ssa, 15363 const zend_ssa_op *ssa_op) 15364{ 15365 zval *zv = RT_CONSTANT(opline, opline->op2) + 1; 15366 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15367 zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); 15368 uint32_t res_info = RES_INFO(); 15369 15370 | // c = CACHED_PTR(opline->extended_value); 15371 | mov FCARG1a, EX->run_time_cache 15372 | mov r0, aword [FCARG1a + opline->extended_value] 15373 | // if (c != NULL) 15374 | test r0, r0 15375 | jz >9 15376 | // if (!IS_SPECIAL_CACHE_VAL(c)) 15377 | test r0, CACHE_SPECIAL 15378 | jnz >9 15379 |8: 15380 15381 if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { 15382 zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; 15383 uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); 15384 int32_t exit_point; 15385 const void *exit_addr = NULL; 15386 15387 SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); 15388 SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_R0); 15389 exit_point = zend_jit_trace_get_exit_point(opline+1, 0); 15390 SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); 15391 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15392 if (!exit_addr) { 15393 return 0; 15394 } 15395 res_info &= ~MAY_BE_GUARD; 15396 ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; 15397 15398 zend_uchar type = concrete_type(res_info); 15399 15400 if (type < IS_STRING) { 15401 | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr 15402 } else { 15403 | GET_ZVAL_TYPE_INFO edx, const_addr 15404 | IF_NOT_TYPE dl, type, &exit_addr 15405 } 15406 | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_R0, ZREG_R1 15407 if (type < IS_STRING) { 15408 | SET_ZVAL_TYPE_INFO res_addr, type 15409 } else { 15410 | SET_ZVAL_TYPE_INFO res_addr, edx 15411 | TRY_ADDREF res_info, dh, r1 15412 } 15413 } else { 15414 | // ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup) 15415 | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_R0, ZREG_R1 15416 | TRY_ADDREF MAY_BE_ANY, ah, r1 15417 } 15418 15419 |.cold_code 15420 |9: 15421 | // SAVE_OPLINE(); 15422 | SET_EX_OPLINE opline, r0 15423 | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); 15424 | LOAD_ADDR FCARG1a, zv 15425 | mov FCARG2a, opline->op1.num 15426 | EXT_CALL zend_jit_get_constant, r0 15427 | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); 15428 | test r0, r0 15429 | jnz <8 15430 | jmp ->exception_handler 15431 |.code 15432 15433 return 1; 15434} 15435 15436static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) 15437{ 15438 HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); 15439 zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); 15440 15441 ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); 15442 ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); 15443 15444 | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); 15445 | LOAD_ADDR FCARG1a, ht 15446 if (opline->op1_type != IS_CONST) { 15447 | GET_ZVAL_PTR FCARG2a, op1_addr 15448 | EXT_CALL zend_hash_find, r0 15449 } else { 15450 zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); 15451 | LOAD_ADDR FCARG2a, str 15452 | EXT_CALL _zend_hash_find_known_hash, r0 15453 } 15454 | test r0, r0 15455 if (exit_addr) { 15456 if (smart_branch_opcode == ZEND_JMPZ) { 15457 | jz &exit_addr 15458 } else { 15459 | jnz &exit_addr 15460 } 15461 } else if (smart_branch_opcode) { 15462 if (smart_branch_opcode == ZEND_JMPZ) { 15463 | jz =>target_label 15464 } else if (smart_branch_opcode == ZEND_JMPNZ) { 15465 | jnz =>target_label 15466 } else if (smart_branch_opcode == ZEND_JMPZNZ) { 15467 | jz =>target_label 15468 | jmp =>target_label2 15469 } else { 15470 ZEND_UNREACHABLE(); 15471 } 15472 } else { 15473 | setnz al 15474 | movzx eax, al 15475 | lea eax, [eax + IS_FALSE] 15476 | SET_ZVAL_TYPE_INFO res_addr, eax 15477 } 15478 15479 return 1; 15480} 15481 15482static zend_bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) 15483{ 15484 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15485 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15486 15487 if (!exit_addr) { 15488 return 0; 15489 } 15490 | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr 15491 15492 return 1; 15493} 15494 15495static zend_bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, zend_bool add_ref_guard, zend_bool add_type_guard) 15496{ 15497 zend_jit_addr var_addr = *var_addr_ptr; 15498 uint32_t var_info = *var_info_ptr; 15499 const void *exit_addr = NULL; 15500 15501 if (add_ref_guard || add_type_guard) { 15502 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15503 15504 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15505 if (!exit_addr) { 15506 return 0; 15507 } 15508 } 15509 15510 if (add_ref_guard) { 15511 | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr 15512 } 15513 if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { 15514 /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ 15515 if (Z_REG(var_addr) != ZREG_FCARG1a || Z_OFFSET(var_addr) != 0) { 15516 | LOAD_ZVAL_ADDR FCARG1a, var_addr 15517 } 15518 | EXT_CALL zend_jit_unref_helper, r0 15519 } else { 15520 | GET_ZVAL_PTR FCARG1a, var_addr 15521 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, offsetof(zend_reference, val)); 15522 *var_addr_ptr = var_addr; 15523 } 15524 15525 if (var_type != IS_UNKNOWN) { 15526 var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); 15527 } 15528 if (add_type_guard 15529 && var_type != IS_UNKNOWN 15530 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 15531 | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr 15532 15533 ZEND_ASSERT(var_info & (1 << var_type)); 15534 if (var_type < IS_STRING) { 15535 var_info = (1 << var_type); 15536 } else if (var_type != IS_ARRAY) { 15537 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 15538 } else { 15539 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 15540 } 15541 15542 *var_info_ptr = var_info; 15543 } else { 15544 var_info &= ~MAY_BE_REF; 15545 *var_info_ptr = var_info; 15546 } 15547 *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */ 15548 15549 return 1; 15550} 15551 15552static zend_bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, zend_bool add_indirect_guard) 15553{ 15554 zend_jit_addr var_addr = *var_addr_ptr; 15555 uint32_t var_info = *var_info_ptr; 15556 int32_t exit_point; 15557 const void *exit_addr; 15558 15559 if (add_indirect_guard) { 15560 int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); 15561 const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15562 15563 if (!exit_addr) { 15564 return 0; 15565 } 15566 | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr 15567 | GET_ZVAL_PTR FCARG1a, var_addr 15568 } else { 15569 /* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */ 15570 if (opline->op1_type != IS_VAR || 15571 (opline-1)->result_type != IS_VAR || 15572 (opline-1)->result.var != opline->op1.var || 15573 (opline-1)->op2_type == IS_VAR || 15574 (opline-1)->op2_type == IS_TMP_VAR) { 15575 | GET_ZVAL_PTR FCARG1a, var_addr 15576 } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { 15577 | mov FCARG1a, r0 15578 } 15579 } 15580 *var_info_ptr &= ~MAY_BE_INDIRECT; 15581 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0); 15582 *var_addr_ptr = var_addr; 15583 15584 if (var_type != IS_UNKNOWN) { 15585 var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); 15586 } 15587 if (!(var_type & IS_TRACE_REFERENCE) 15588 && var_type != IS_UNKNOWN 15589 && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { 15590 exit_point = zend_jit_trace_get_exit_point(opline, 0); 15591 exit_addr = zend_jit_trace_get_exit_addr(exit_point); 15592 15593 if (!exit_addr) { 15594 return 0; 15595 } 15596 15597 | IF_NOT_Z_TYPE FCARG1a, var_type, &exit_addr 15598 15599 //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); 15600 ZEND_ASSERT(var_info & (1 << var_type)); 15601 if (var_type < IS_STRING) { 15602 var_info = (1 << var_type); 15603 } else if (var_type != IS_ARRAY) { 15604 var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); 15605 } else { 15606 var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); 15607 } 15608 15609 *var_info_ptr = var_info; 15610 } 15611 15612 return 1; 15613} 15614 15615static zend_bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) 15616{ 15617 if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { 15618 return 0; 15619 } 15620 15621 switch (opline->opcode) { 15622 case ZEND_QM_ASSIGN: 15623 case ZEND_SEND_VAR: 15624 case ZEND_ASSIGN: 15625 case ZEND_PRE_INC: 15626 case ZEND_PRE_DEC: 15627 case ZEND_POST_INC: 15628 case ZEND_POST_DEC: 15629 return 1; 15630 case ZEND_ADD: 15631 case ZEND_SUB: 15632 case ZEND_MUL: 15633 case ZEND_BW_OR: 15634 case ZEND_BW_AND: 15635 case ZEND_BW_XOR: 15636 if (def_var == ssa_op->result_def && 15637 use_var == ssa_op->op1_use) { 15638 return 1; 15639 } 15640 break; 15641 default: 15642 break; 15643 } 15644 return 0; 15645} 15646 15647static zend_bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) 15648{ 15649 uint32_t op1_info, op2_info; 15650 15651 switch (opline->opcode) { 15652 case ZEND_SEND_VAR: 15653 case ZEND_SEND_VAL: 15654 case ZEND_SEND_VAL_EX: 15655 return (opline->op2_type != IS_CONST); 15656 case ZEND_QM_ASSIGN: 15657 case ZEND_IS_SMALLER: 15658 case ZEND_IS_SMALLER_OR_EQUAL: 15659 case ZEND_IS_EQUAL: 15660 case ZEND_IS_NOT_EQUAL: 15661 case ZEND_IS_IDENTICAL: 15662 case ZEND_IS_NOT_IDENTICAL: 15663 case ZEND_CASE: 15664 return 1; 15665 case ZEND_RETURN: 15666 return (op_array->type != ZEND_EVAL_CODE && op_array->function_name); 15667 case ZEND_ASSIGN: 15668 op1_info = OP1_INFO(); 15669 op2_info = OP2_INFO(); 15670 return 15671 opline->op1_type == IS_CV && 15672 !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) && 15673 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); 15674 case ZEND_ADD: 15675 case ZEND_SUB: 15676 case ZEND_MUL: 15677 op1_info = OP1_INFO(); 15678 op2_info = OP2_INFO(); 15679 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); 15680 case ZEND_BW_OR: 15681 case ZEND_BW_AND: 15682 case ZEND_BW_XOR: 15683 case ZEND_SL: 15684 case ZEND_SR: 15685 case ZEND_MOD: 15686 op1_info = OP1_INFO(); 15687 op2_info = OP2_INFO(); 15688 return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); 15689 case ZEND_PRE_INC: 15690 case ZEND_PRE_DEC: 15691 case ZEND_POST_INC: 15692 case ZEND_POST_DEC: 15693 op1_info = OP1_INFO(); 15694 return opline->op1_type == IS_CV && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); 15695 case ZEND_JMPZ: 15696 case ZEND_JMPNZ: 15697 if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { 15698 if (!ssa->cfg.map) { 15699 return 0; 15700 } 15701 if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start && 15702 ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { 15703 return 0; 15704 } 15705 } 15706 /* break missing intentionally */ 15707 case ZEND_BOOL: 15708 case ZEND_BOOL_NOT: 15709 case ZEND_JMPZNZ: 15710 case ZEND_JMPZ_EX: 15711 case ZEND_JMPNZ_EX: 15712 return 1; 15713 case ZEND_FETCH_DIM_R: 15714 op1_info = OP1_INFO(); 15715 op2_info = OP2_INFO(); 15716 if (trace 15717 && trace->op1_type != IS_UNKNOWN 15718 && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { 15719 op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); 15720 } 15721 return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && 15722 (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && 15723 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || 15724 (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && 15725 (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); 15726 } 15727 return 0; 15728} 15729 15730static zend_bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) 15731{ 15732 if (ssa->vars[var].no_val) { 15733 /* we don't need the value */ 15734 return 0; 15735 } 15736 15737 if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { 15738 /* Disable global register allocation, 15739 * register allocation for SSA variables connected through Phi functions 15740 */ 15741 if (ssa->vars[var].definition_phi) { 15742 return 0; 15743 } 15744 if (ssa->vars[var].phi_use_chain) { 15745 zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; 15746 do { 15747 if (!ssa->vars[phi->ssa_var].no_val) { 15748 return 0; 15749 } 15750 phi = zend_ssa_next_use_phi(ssa, var, phi); 15751 } while (phi); 15752 } 15753 } 15754 15755 if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && 15756 ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { 15757 /* bad type */ 15758 return 0; 15759 } 15760 15761 return 1; 15762} 15763 15764static zend_bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) 15765{ 15766 if (!zend_jit_var_supports_reg(ssa, var)) { 15767 return 0; 15768 } 15769 15770 if (ssa->vars[var].definition >= 0) { 15771 uint32_t def = ssa->vars[var].definition; 15772 if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { 15773 return 0; 15774 } 15775 } 15776 15777 if (ssa->vars[var].use_chain >= 0) { 15778 int use = ssa->vars[var].use_chain; 15779 15780 do { 15781 if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && 15782 !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { 15783 return 0; 15784 } 15785 use = zend_ssa_next_use(ssa->ops, var, use); 15786 } while (use >= 0); 15787 } 15788 15789 return 1; 15790} 15791 15792static zend_bool zend_needs_extra_reg_for_const(const zend_op *opline, zend_uchar op_type, znode_op op) 15793{ 15794|.if X64 15795|| if (op_type == IS_CONST) { 15796|| zval *zv = RT_CONSTANT(opline, op); 15797|| if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0 && !IS_SIGNED_32BIT(zv)) { 15798|| return 1; 15799|| } else if (Z_TYPE_P(zv) == IS_LONG && !IS_SIGNED_32BIT(Z_LVAL_P(zv))) { 15800|| return 1; 15801|| } 15802|| } 15803|.endif 15804 return 0; 15805} 15806 15807static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, zend_bool last_use) 15808{ 15809 uint32_t op1_info, op2_info; 15810 15811 switch (opline->opcode) { 15812 case ZEND_FETCH_DIM_R: 15813 op1_info = OP1_INFO(); 15814 op2_info = OP2_INFO(); 15815 if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && 15816 (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || 15817 ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && 15818 (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { 15819 return ZEND_REGSET(ZREG_FCARG1a); 15820 } 15821 break; 15822 default: 15823 break; 15824 } 15825 15826 return ZEND_REGSET_EMPTY; 15827} 15828 15829static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, zend_bool last_use) 15830{ 15831 uint32_t op1_info, op2_info, res_info; 15832 zend_regset regset = ZEND_REGSET_SCRATCH; 15833 15834 switch (opline->opcode) { 15835 case ZEND_NOP: 15836 case ZEND_OP_DATA: 15837 case ZEND_JMP: 15838 case ZEND_RETURN: 15839 regset = ZEND_REGSET_EMPTY; 15840 break; 15841 case ZEND_QM_ASSIGN: 15842 if (ssa_op->op1_def == current_var || 15843 ssa_op->result_def == current_var) { 15844 regset = ZEND_REGSET_EMPTY; 15845 break; 15846 } 15847 /* break missing intentionally */ 15848 case ZEND_SEND_VAL: 15849 case ZEND_SEND_VAL_EX: 15850 if (opline->op2_type == IS_CONST) { 15851 break; 15852 } 15853 if (ssa_op->op1_use == current_var) { 15854 regset = ZEND_REGSET(ZREG_R0); 15855 break; 15856 } 15857 op1_info = OP1_INFO(); 15858 if (!(op1_info & MAY_BE_UNDEF)) { 15859 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15860 regset = ZEND_REGSET(ZREG_XMM0); 15861 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15862 regset = ZEND_REGSET(ZREG_R0); 15863 } else { 15864 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 15865 } 15866 } 15867 break; 15868 case ZEND_SEND_VAR: 15869 if (opline->op2_type == IS_CONST) { 15870 break; 15871 } 15872 if (ssa_op->op1_use == current_var || 15873 ssa_op->op1_def == current_var) { 15874 regset = ZEND_REGSET_EMPTY; 15875 break; 15876 } 15877 op1_info = OP1_INFO(); 15878 if (!(op1_info & MAY_BE_UNDEF)) { 15879 if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15880 regset = ZEND_REGSET(ZREG_XMM0); 15881 } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15882 } else { 15883 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 15884 if (op1_info & MAY_BE_REF) { 15885 ZEND_REGSET_INCL(regset, ZREG_R1); 15886 } 15887 } 15888 } 15889 break; 15890 case ZEND_ASSIGN: 15891 if (ssa_op->op2_use == current_var || 15892 ssa_op->op2_def == current_var || 15893 ssa_op->op1_def == current_var || 15894 ssa_op->result_def == current_var) { 15895 regset = ZEND_REGSET_EMPTY; 15896 break; 15897 } 15898 op1_info = OP1_INFO(); 15899 op2_info = OP2_INFO(); 15900 if (opline->op1_type == IS_CV 15901 && !(op2_info & MAY_BE_UNDEF) 15902 && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { 15903 if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { 15904 regset = ZEND_REGSET(ZREG_XMM0); 15905 } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { 15906 regset = ZEND_REGSET(ZREG_R0); 15907 } else { 15908 regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2)); 15909 } 15910 } 15911 break; 15912 case ZEND_PRE_INC: 15913 case ZEND_PRE_DEC: 15914 case ZEND_POST_INC: 15915 case ZEND_POST_DEC: 15916 if (ssa_op->op1_use == current_var || 15917 ssa_op->op1_def == current_var || 15918 ssa_op->result_def == current_var) { 15919 regset = ZEND_REGSET_EMPTY; 15920 break; 15921 } 15922 op1_info = OP1_INFO(); 15923 if (opline->op1_type == IS_CV 15924 && (op1_info & MAY_BE_LONG) 15925 && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15926 regset = ZEND_REGSET_EMPTY; 15927 if (op1_info & MAY_BE_DOUBLE) { 15928 regset = ZEND_REGSET(ZREG_XMM0); 15929 } 15930 if (opline->result_type != IS_UNUSED && (op1_info & MAY_BE_LONG)) { 15931 ZEND_REGSET_INCL(regset, ZREG_R1); 15932 } 15933 } 15934 break; 15935 case ZEND_ADD: 15936 case ZEND_SUB: 15937 case ZEND_MUL: 15938 op1_info = OP1_INFO(); 15939 op2_info = OP2_INFO(); 15940 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 15941 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 15942 15943 regset = ZEND_REGSET_EMPTY; 15944 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { 15945 if (ssa_op->result_def != current_var && 15946 (ssa_op->op1_use != current_var || !last_use)) { 15947 ZEND_REGSET_INCL(regset, ZREG_R0); 15948 } 15949 res_info = RES_INFO(); 15950 if (res_info & MAY_BE_DOUBLE) { 15951 ZEND_REGSET_INCL(regset, ZREG_R0); 15952 ZEND_REGSET_INCL(regset, ZREG_XMM0); 15953 ZEND_REGSET_INCL(regset, ZREG_XMM1); 15954 } else if (res_info & MAY_BE_GUARD) { 15955 ZEND_REGSET_INCL(regset, ZREG_R0); 15956 } 15957 } 15958 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 15959 if (ssa_op->result_def != current_var) { 15960 ZEND_REGSET_INCL(regset, ZREG_XMM0); 15961 } 15962 } 15963 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 15964 if (zend_is_commutative(opline->opcode)) { 15965 if (ssa_op->result_def != current_var) { 15966 ZEND_REGSET_INCL(regset, ZREG_XMM0); 15967 } 15968 } else { 15969 ZEND_REGSET_INCL(regset, ZREG_XMM0); 15970 if (ssa_op->result_def != current_var && 15971 (ssa_op->op1_use != current_var || !last_use)) { 15972 ZEND_REGSET_INCL(regset, ZREG_XMM1); 15973 } 15974 } 15975 } 15976 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 15977 if (ssa_op->result_def != current_var && 15978 (ssa_op->op1_use != current_var || !last_use) && 15979 (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { 15980 ZEND_REGSET_INCL(regset, ZREG_XMM0); 15981 } 15982 } 15983 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 15984 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 15985 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 15986 ZEND_REGSET_INCL(regset, ZREG_R0); 15987 } else { 15988 ZEND_REGSET_INCL(regset, ZREG_R1); 15989 } 15990 } 15991 } 15992 break; 15993 case ZEND_BW_OR: 15994 case ZEND_BW_AND: 15995 case ZEND_BW_XOR: 15996 op1_info = OP1_INFO(); 15997 op2_info = OP2_INFO(); 15998 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 15999 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16000 regset = ZEND_REGSET_EMPTY; 16001 if (ssa_op->result_def != current_var && 16002 (ssa_op->op1_use != current_var || !last_use)) { 16003 ZEND_REGSET_INCL(regset, ZREG_R0); 16004 } 16005 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16006 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16007 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 16008 ZEND_REGSET_INCL(regset, ZREG_R0); 16009 } else { 16010 ZEND_REGSET_INCL(regset, ZREG_R1); 16011 } 16012 } 16013 } 16014 break; 16015 case ZEND_SL: 16016 case ZEND_SR: 16017 op1_info = OP1_INFO(); 16018 op2_info = OP2_INFO(); 16019 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16020 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16021 regset = ZEND_REGSET_EMPTY; 16022 if (ssa_op->result_def != current_var && 16023 (ssa_op->op1_use != current_var || !last_use)) { 16024 ZEND_REGSET_INCL(regset, ZREG_R0); 16025 } 16026 if (opline->op2_type != IS_CONST && ssa_op->op2_use != current_var) { 16027 ZEND_REGSET_INCL(regset, ZREG_R1); 16028 } 16029 } 16030 break; 16031 case ZEND_MOD: 16032 op1_info = OP1_INFO(); 16033 op2_info = OP2_INFO(); 16034 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && 16035 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { 16036 regset = ZEND_REGSET_EMPTY; 16037 if (opline->op2_type == IS_CONST && 16038 opline->op1_type != IS_CONST && 16039 Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG && 16040 zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2))) && 16041 OP1_HAS_RANGE() && 16042 OP1_MIN_RANGE() >= 0) { 16043 if (ssa_op->result_def != current_var && 16044 (ssa_op->op1_use != current_var || !last_use)) { 16045 ZEND_REGSET_INCL(regset, ZREG_R0); 16046 } 16047 if (sizeof(void*) == 8 16048 && !IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) - 1)) { 16049 if (!ZEND_REGSET_IN(regset, ZREG_R0)) { 16050 ZEND_REGSET_INCL(regset, ZREG_R0); 16051 } else { 16052 ZEND_REGSET_INCL(regset, ZREG_R1); 16053 } 16054 } 16055 } else { 16056 ZEND_REGSET_INCL(regset, ZREG_R0); 16057 ZEND_REGSET_INCL(regset, ZREG_R2); 16058 if (opline->op2_type == IS_CONST) { 16059 ZEND_REGSET_INCL(regset, ZREG_R1); 16060 } 16061 } 16062 } 16063 break; 16064 case ZEND_IS_SMALLER: 16065 case ZEND_IS_SMALLER_OR_EQUAL: 16066 case ZEND_IS_EQUAL: 16067 case ZEND_IS_NOT_EQUAL: 16068 case ZEND_IS_IDENTICAL: 16069 case ZEND_IS_NOT_IDENTICAL: 16070 case ZEND_CASE: 16071 op1_info = OP1_INFO(); 16072 op2_info = OP2_INFO(); 16073 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && 16074 !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16075 regset = ZEND_REGSET_EMPTY; 16076 if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { 16077 ZEND_REGSET_INCL(regset, ZREG_R0); 16078 } 16079 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && 16080 opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { 16081 if (ssa_op->op1_use != current_var && 16082 ssa_op->op2_use != current_var) { 16083 ZEND_REGSET_INCL(regset, ZREG_R0); 16084 } 16085 } 16086 if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { 16087 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16088 } 16089 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { 16090 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16091 } 16092 if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { 16093 if (ssa_op->op1_use != current_var && 16094 ssa_op->op2_use != current_var) { 16095 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16096 } 16097 } 16098 if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || 16099 zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { 16100 ZEND_REGSET_INCL(regset, ZREG_R0); 16101 } 16102 } 16103 break; 16104 case ZEND_BOOL: 16105 case ZEND_BOOL_NOT: 16106 case ZEND_JMPZ: 16107 case ZEND_JMPNZ: 16108 case ZEND_JMPZNZ: 16109 case ZEND_JMPZ_EX: 16110 case ZEND_JMPNZ_EX: 16111 op1_info = OP1_INFO(); 16112 if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { 16113 regset = ZEND_REGSET_EMPTY; 16114 if (op1_info & MAY_BE_DOUBLE) { 16115 ZEND_REGSET_INCL(regset, ZREG_XMM0); 16116 } 16117 if (opline->opcode == ZEND_BOOL || 16118 opline->opcode == ZEND_BOOL_NOT || 16119 opline->opcode == ZEND_JMPZ_EX || 16120 opline->opcode == ZEND_JMPNZ_EX) { 16121 ZEND_REGSET_INCL(regset, ZREG_R0); 16122 } 16123 } 16124 break; 16125 case ZEND_DO_UCALL: 16126 case ZEND_DO_FCALL: 16127 case ZEND_DO_FCALL_BY_NAME: 16128 case ZEND_INCLUDE_OR_EVAL: 16129 case ZEND_GENERATOR_CREATE: 16130 case ZEND_YIELD: 16131 case ZEND_YIELD_FROM: 16132 regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); 16133 break; 16134 default: 16135 break; 16136 } 16137 16138 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 16139 if (ssa_op == ssa->ops 16140 && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL 16141 && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { 16142 ZEND_REGSET_INCL(regset, ZREG_R0); 16143 ZEND_REGSET_INCL(regset, ZREG_R1); 16144 } 16145 } 16146 16147 /* %r0 is used to check EG(vm_interrupt) */ 16148 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { 16149 if (ssa_op == ssa->ops 16150 && (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP || 16151 JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) { 16152#if ZTS 16153 ZEND_REGSET_INCL(regset, ZREG_R0); 16154#else 16155 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { 16156 ZEND_REGSET_INCL(regset, ZREG_R0); 16157 } 16158#endif 16159 } 16160 } else { 16161 uint32_t b = ssa->cfg.map[ssa_op - ssa->ops]; 16162 16163 if ((ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) != 0 16164 && ssa->cfg.blocks[b].start == ssa_op - ssa->ops) { 16165#if ZTS 16166 ZEND_REGSET_INCL(regset, ZREG_R0); 16167#else 16168 if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { 16169 ZEND_REGSET_INCL(regset, ZREG_R0); 16170 } 16171#endif 16172 } 16173 } 16174 16175 return regset; 16176} 16177 16178#if defined(__clang__) 16179# pragma clang diagnostic pop 16180#endif 16181 16182/* 16183 * Local variables: 16184 * tab-width: 4 16185 * c-basic-offset: 4 16186 * indent-tabs-mode: t 16187 * End: 16188 */ 16189
