1 /*
2 +----------------------------------------------------------------------+
3 | Zend Engine |
4 +----------------------------------------------------------------------+
5 | Copyright (c) Zend Technologies Ltd. (http://www.zend.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 2.00 of the Zend license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.zend.com/license/2_00.txt. |
11 | If you did not receive a copy of the Zend license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@zend.com so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Aaron Piotrowski <aaron@trowski.com> |
16 | Martin Schröder <m.schroeder2007@gmail.com> |
17 +----------------------------------------------------------------------+
18 */
19
20 #include "zend.h"
21 #include "zend_API.h"
22 #include "zend_ini.h"
23 #include "zend_vm.h"
24 #include "zend_exceptions.h"
25 #include "zend_builtin_functions.h"
26 #include "zend_observer.h"
27 #include "zend_mmap.h"
28 #include "zend_compile.h"
29 #include "zend_closures.h"
30
31 #include "zend_fibers.h"
32 #include "zend_fibers_arginfo.h"
33
34 #ifdef HAVE_VALGRIND
35 # include <valgrind/valgrind.h>
36 #endif
37
38 #ifdef ZEND_FIBER_UCONTEXT
39 # include <ucontext.h>
40 #endif
41
42 #ifndef ZEND_WIN32
43 # include <unistd.h>
44 # include <sys/mman.h>
45 # include <limits.h>
46
47 # if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
48 # define MAP_ANONYMOUS MAP_ANON
49 # endif
50
51 /* FreeBSD require a first (i.e. addr) argument of mmap(2) is not NULL
52 * if MAP_STACK is passed.
53 * http://www.FreeBSD.org/cgi/query-pr.cgi?pr=158755 */
54 # if !defined(MAP_STACK) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
55 # undef MAP_STACK
56 # define MAP_STACK 0
57 # endif
58
59 # ifndef MAP_FAILED
60 # define MAP_FAILED ((void * ) -1)
61 # endif
62 #endif
63
64 #ifdef __SANITIZE_ADDRESS__
65 # include <sanitizer/asan_interface.h>
66 # include <sanitizer/common_interface_defs.h>
67 #endif
68
69 # if defined __CET__
70 # include <cet.h>
71 # define SHSTK_ENABLED (__CET__ & 0x2)
72 # define BOOST_CONTEXT_SHADOW_STACK (SHSTK_ENABLED && SHADOW_STACK_SYSCALL)
73 # define __NR_map_shadow_stack 451
74 # ifndef SHADOW_STACK_SET_TOKEN
75 # define SHADOW_STACK_SET_TOKEN 0x1
76 #endif
77 #endif
78
79 /* Encapsulates the fiber C stack with extension for debugging tools. */
80 struct _zend_fiber_stack {
81 void *pointer;
82 size_t size;
83
84 #ifdef HAVE_VALGRIND
85 unsigned int valgrind_stack_id;
86 #endif
87
88 #ifdef __SANITIZE_ADDRESS__
89 const void *asan_pointer;
90 size_t asan_size;
91 #endif
92
93 #ifdef ZEND_FIBER_UCONTEXT
94 /* Embedded ucontext to avoid unnecessary memory allocations. */
95 ucontext_t ucontext;
96 #elif BOOST_CONTEXT_SHADOW_STACK
97 /* Shadow stack: base, size */
98 void *ss_base;
99 size_t ss_size;
100 #endif
101 };
102
103 /* Zend VM state that needs to be captured / restored during fiber context switch. */
104 typedef struct _zend_fiber_vm_state {
105 zend_vm_stack vm_stack;
106 zval *vm_stack_top;
107 zval *vm_stack_end;
108 size_t vm_stack_page_size;
109 zend_execute_data *current_execute_data;
110 int error_reporting;
111 uint32_t jit_trace_num;
112 JMP_BUF *bailout;
113 zend_fiber *active_fiber;
114 #ifdef ZEND_CHECK_STACK_LIMIT
115 void *stack_base;
116 void *stack_limit;
117 #endif
118 } zend_fiber_vm_state;
119
zend_fiber_capture_vm_state(zend_fiber_vm_state * state)120 static zend_always_inline void zend_fiber_capture_vm_state(zend_fiber_vm_state *state)
121 {
122 state->vm_stack = EG(vm_stack);
123 state->vm_stack_top = EG(vm_stack_top);
124 state->vm_stack_end = EG(vm_stack_end);
125 state->vm_stack_page_size = EG(vm_stack_page_size);
126 state->current_execute_data = EG(current_execute_data);
127 state->error_reporting = EG(error_reporting);
128 state->jit_trace_num = EG(jit_trace_num);
129 state->bailout = EG(bailout);
130 state->active_fiber = EG(active_fiber);
131 #ifdef ZEND_CHECK_STACK_LIMIT
132 state->stack_base = EG(stack_base);
133 state->stack_limit = EG(stack_limit);
134 #endif
135 }
136
zend_fiber_restore_vm_state(zend_fiber_vm_state * state)137 static zend_always_inline void zend_fiber_restore_vm_state(zend_fiber_vm_state *state)
138 {
139 EG(vm_stack) = state->vm_stack;
140 EG(vm_stack_top) = state->vm_stack_top;
141 EG(vm_stack_end) = state->vm_stack_end;
142 EG(vm_stack_page_size) = state->vm_stack_page_size;
143 EG(current_execute_data) = state->current_execute_data;
144 EG(error_reporting) = state->error_reporting;
145 EG(jit_trace_num) = state->jit_trace_num;
146 EG(bailout) = state->bailout;
147 EG(active_fiber) = state->active_fiber;
148 #ifdef ZEND_CHECK_STACK_LIMIT
149 EG(stack_base) = state->stack_base;
150 EG(stack_limit) = state->stack_limit;
151 #endif
152 }
153
154 #ifdef ZEND_FIBER_UCONTEXT
155 ZEND_TLS zend_fiber_transfer *transfer_data;
156 #else
157 /* boost_context_data is our customized definition of struct transfer_t as
158 * provided by boost.context in fcontext.hpp:
159 *
160 * typedef void* fcontext_t;
161 *
162 * struct transfer_t {
163 * fcontext_t fctx;
164 * void *data;
165 * }; */
166
167 typedef struct {
168 void *handle;
169 zend_fiber_transfer *transfer;
170 } boost_context_data;
171
172 /* These functions are defined in assembler files provided by boost.context (located in "Zend/asm"). */
173 extern void *make_fcontext(void *sp, size_t size, void (*fn)(boost_context_data));
174 extern ZEND_INDIRECT_RETURN boost_context_data jump_fcontext(void *to, zend_fiber_transfer *transfer);
175 #endif
176
177 ZEND_API zend_class_entry *zend_ce_fiber;
178 static zend_class_entry *zend_ce_fiber_error;
179
180 static zend_object_handlers zend_fiber_handlers;
181
182 static zend_function zend_fiber_function = { ZEND_INTERNAL_FUNCTION };
183
184 ZEND_TLS uint32_t zend_fiber_switch_blocking = 0;
185
186 #define ZEND_FIBER_DEFAULT_PAGE_SIZE 4096
187
zend_fiber_get_page_size(void)188 static size_t zend_fiber_get_page_size(void)
189 {
190 static size_t page_size = 0;
191
192 if (!page_size) {
193 page_size = zend_get_page_size();
194 if (!page_size || (page_size & (page_size - 1))) {
195 /* anyway, we have to return a valid result */
196 page_size = ZEND_FIBER_DEFAULT_PAGE_SIZE;
197 }
198 }
199
200 return page_size;
201 }
202
zend_fiber_stack_allocate(size_t size)203 static zend_fiber_stack *zend_fiber_stack_allocate(size_t size)
204 {
205 void *pointer;
206 const size_t page_size = zend_fiber_get_page_size();
207 const size_t minimum_stack_size = page_size + ZEND_FIBER_GUARD_PAGES * page_size;
208
209 if (size < minimum_stack_size) {
210 zend_throw_exception_ex(NULL, 0, "Fiber stack size is too small, it needs to be at least %zu bytes", minimum_stack_size);
211 return NULL;
212 }
213
214 const size_t stack_size = (size + page_size - 1) / page_size * page_size;
215 const size_t alloc_size = stack_size + ZEND_FIBER_GUARD_PAGES * page_size;
216
217 #ifdef ZEND_WIN32
218 pointer = VirtualAlloc(0, alloc_size, MEM_COMMIT, PAGE_READWRITE);
219
220 if (!pointer) {
221 DWORD err = GetLastError();
222 char *errmsg = php_win32_error_to_msg(err);
223 zend_throw_exception_ex(NULL, 0, "Fiber stack allocate failed: VirtualAlloc failed: [0x%08lx] %s", err, errmsg[0] ? errmsg : "Unknown");
224 php_win32_error_msg_free(errmsg);
225 return NULL;
226 }
227
228 # if ZEND_FIBER_GUARD_PAGES
229 DWORD protect;
230
231 if (!VirtualProtect(pointer, ZEND_FIBER_GUARD_PAGES * page_size, PAGE_READWRITE | PAGE_GUARD, &protect)) {
232 DWORD err = GetLastError();
233 char *errmsg = php_win32_error_to_msg(err);
234 zend_throw_exception_ex(NULL, 0, "Fiber stack protect failed: VirtualProtect failed: [0x%08lx] %s", err, errmsg[0] ? errmsg : "Unknown");
235 php_win32_error_msg_free(errmsg);
236 VirtualFree(pointer, 0, MEM_RELEASE);
237 return NULL;
238 }
239 # endif
240 #else
241 pointer = mmap(NULL, alloc_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
242
243 if (pointer == MAP_FAILED) {
244 zend_throw_exception_ex(NULL, 0, "Fiber stack allocate failed: mmap failed: %s (%d)", strerror(errno), errno);
245 return NULL;
246 }
247
248 #if defined(MADV_NOHUGEPAGE)
249 /* Multiple reasons to fail, ignore all errors only needed
250 * for linux < 6.8 */
251 (void) madvise(pointer, alloc_size, MADV_NOHUGEPAGE);
252 #endif
253
254 zend_mmap_set_name(pointer, alloc_size, "zend_fiber_stack");
255
256 # if ZEND_FIBER_GUARD_PAGES
257 if (mprotect(pointer, ZEND_FIBER_GUARD_PAGES * page_size, PROT_NONE) < 0) {
258 zend_throw_exception_ex(NULL, 0, "Fiber stack protect failed: mprotect failed: %s (%d)", strerror(errno), errno);
259 munmap(pointer, alloc_size);
260 return NULL;
261 }
262 # endif
263 #endif
264
265 zend_fiber_stack *stack = emalloc(sizeof(zend_fiber_stack));
266
267 stack->pointer = (void *) ((uintptr_t) pointer + ZEND_FIBER_GUARD_PAGES * page_size);
268 stack->size = stack_size;
269
270 #if !defined(ZEND_FIBER_UCONTEXT) && BOOST_CONTEXT_SHADOW_STACK
271 /* shadow stack saves ret address only, need less space */
272 stack->ss_size= stack_size >> 5;
273
274 /* align shadow stack to 8 bytes. */
275 stack->ss_size = (stack->ss_size + 7) & ~7;
276
277 /* issue syscall to create shadow stack for the new fcontext */
278 /* SHADOW_STACK_SET_TOKEN option will put "restore token" on the new shadow stack */
279 stack->ss_base = (void *)syscall(__NR_map_shadow_stack, 0, stack->ss_size, SHADOW_STACK_SET_TOKEN);
280
281 if (stack->ss_base == MAP_FAILED) {
282 zend_throw_exception_ex(NULL, 0, "Fiber shadow stack allocate failed: mmap failed: %s (%d)", strerror(errno), errno);
283 return NULL;
284 }
285 #endif
286
287 #ifdef VALGRIND_STACK_REGISTER
288 uintptr_t base = (uintptr_t) stack->pointer;
289 stack->valgrind_stack_id = VALGRIND_STACK_REGISTER(base, base + stack->size);
290 #endif
291
292 #ifdef __SANITIZE_ADDRESS__
293 stack->asan_pointer = stack->pointer;
294 stack->asan_size = stack->size;
295 #endif
296
297 return stack;
298 }
299
zend_fiber_stack_free(zend_fiber_stack * stack)300 static void zend_fiber_stack_free(zend_fiber_stack *stack)
301 {
302 #ifdef VALGRIND_STACK_DEREGISTER
303 VALGRIND_STACK_DEREGISTER(stack->valgrind_stack_id);
304 #endif
305
306 const size_t page_size = zend_fiber_get_page_size();
307
308 void *pointer = (void *) ((uintptr_t) stack->pointer - ZEND_FIBER_GUARD_PAGES * page_size);
309
310 #ifdef __SANITIZE_ADDRESS__
311 /* If another mmap happens after unmapping, it may trigger the stale stack red zones
312 * so we have to unpoison it before unmapping. */
313 ASAN_UNPOISON_MEMORY_REGION(pointer, stack->size + ZEND_FIBER_GUARD_PAGES * page_size);
314 #endif
315
316 #ifdef ZEND_WIN32
317 VirtualFree(pointer, 0, MEM_RELEASE);
318 #else
319 munmap(pointer, stack->size + ZEND_FIBER_GUARD_PAGES * page_size);
320 #endif
321
322 #if !defined(ZEND_FIBER_UCONTEXT) && BOOST_CONTEXT_SHADOW_STACK
323 munmap(stack->ss_base, stack->ss_size);
324 #endif
325
326 efree(stack);
327 }
328
329 #ifdef ZEND_CHECK_STACK_LIMIT
zend_fiber_stack_limit(zend_fiber_stack * stack)330 ZEND_API void* zend_fiber_stack_limit(zend_fiber_stack *stack)
331 {
332 zend_ulong reserve = EG(reserved_stack_size);
333
334 #ifdef __APPLE__
335 /* On Apple Clang, the stack probing function ___chkstk_darwin incorrectly
336 * probes a location that is twice the entered function's stack usage away
337 * from the stack pointer, when using an alternative stack.
338 * https://openradar.appspot.com/radar?id=5497722702397440
339 */
340 reserve = reserve * 2;
341 #endif
342
343 /* stack->pointer is the end of the stack */
344 return (int8_t*)stack->pointer + reserve;
345 }
346
zend_fiber_stack_base(zend_fiber_stack * stack)347 ZEND_API void* zend_fiber_stack_base(zend_fiber_stack *stack)
348 {
349 return (void*)((uintptr_t)stack->pointer + stack->size);
350 }
351 #endif
352
353 #ifdef ZEND_FIBER_UCONTEXT
zend_fiber_trampoline(void)354 static ZEND_NORETURN void zend_fiber_trampoline(void)
355 #else
356 static ZEND_NORETURN void zend_fiber_trampoline(boost_context_data data)
357 #endif
358 {
359 /* Initialize transfer struct with a copy of passed data. */
360 #ifdef ZEND_FIBER_UCONTEXT
361 zend_fiber_transfer transfer = *transfer_data;
362 #else
363 zend_fiber_transfer transfer = *data.transfer;
364 #endif
365
366 zend_fiber_context *from = transfer.context;
367
368 #ifdef __SANITIZE_ADDRESS__
369 __sanitizer_finish_switch_fiber(NULL, &from->stack->asan_pointer, &from->stack->asan_size);
370 #endif
371
372 #ifndef ZEND_FIBER_UCONTEXT
373 /* Get the context that resumed us and update its handle to allow for symmetric coroutines. */
374 from->handle = data.handle;
375 #endif
376
377 /* Ensure that previous fiber will be cleaned up (needed by symmetric coroutines). */
378 if (from->status == ZEND_FIBER_STATUS_DEAD) {
379 zend_fiber_destroy_context(from);
380 }
381
382 zend_fiber_context *context = EG(current_fiber_context);
383
384 context->function(&transfer);
385 context->status = ZEND_FIBER_STATUS_DEAD;
386
387 /* Final context switch, the fiber must not be resumed afterwards! */
388 zend_fiber_switch_context(&transfer);
389
390 /* Abort here because we are in an inconsistent program state. */
391 abort();
392 }
393
zend_fiber_switch_block(void)394 ZEND_API void zend_fiber_switch_block(void)
395 {
396 ++zend_fiber_switch_blocking;
397 }
398
zend_fiber_switch_unblock(void)399 ZEND_API void zend_fiber_switch_unblock(void)
400 {
401 ZEND_ASSERT(zend_fiber_switch_blocking && "Fiber switching was not blocked");
402 --zend_fiber_switch_blocking;
403 }
404
zend_fiber_switch_blocked(void)405 ZEND_API bool zend_fiber_switch_blocked(void)
406 {
407 return zend_fiber_switch_blocking;
408 }
409
zend_fiber_init_context(zend_fiber_context * context,void * kind,zend_fiber_coroutine coroutine,size_t stack_size)410 ZEND_API zend_result zend_fiber_init_context(zend_fiber_context *context, void *kind, zend_fiber_coroutine coroutine, size_t stack_size)
411 {
412 context->stack = zend_fiber_stack_allocate(stack_size);
413
414 if (UNEXPECTED(!context->stack)) {
415 return FAILURE;
416 }
417
418 #ifdef ZEND_FIBER_UCONTEXT
419 ucontext_t *handle = &context->stack->ucontext;
420
421 getcontext(handle);
422
423 handle->uc_stack.ss_size = context->stack->size;
424 handle->uc_stack.ss_sp = context->stack->pointer;
425 handle->uc_stack.ss_flags = 0;
426 handle->uc_link = NULL;
427
428 makecontext(handle, (void (*)(void)) zend_fiber_trampoline, 0);
429
430 context->handle = handle;
431 #else
432 // Stack grows down, calculate the top of the stack. make_fcontext then shifts pointer to lower 16-byte boundary.
433 void *stack = (void *) ((uintptr_t) context->stack->pointer + context->stack->size);
434
435 #if BOOST_CONTEXT_SHADOW_STACK
436 // pass the shadow stack pointer to make_fcontext
437 // i.e., link the new shadow stack with the new fcontext
438 // TODO should be a better way?
439 *((unsigned long*) (stack - 8)) = (unsigned long)context->stack->ss_base + context->stack->ss_size;
440 #endif
441
442 context->handle = make_fcontext(stack, context->stack->size, zend_fiber_trampoline);
443 ZEND_ASSERT(context->handle != NULL && "make_fcontext() never returns NULL");
444 #endif
445
446 context->kind = kind;
447 context->function = coroutine;
448
449 // Set status in case memory has not been zeroed.
450 context->status = ZEND_FIBER_STATUS_INIT;
451
452 zend_observer_fiber_init_notify(context);
453
454 return SUCCESS;
455 }
456
zend_fiber_destroy_context(zend_fiber_context * context)457 ZEND_API void zend_fiber_destroy_context(zend_fiber_context *context)
458 {
459 zend_observer_fiber_destroy_notify(context);
460
461 if (context->cleanup) {
462 context->cleanup(context);
463 }
464
465 zend_fiber_stack_free(context->stack);
466 }
467
zend_fiber_switch_context(zend_fiber_transfer * transfer)468 ZEND_API void zend_fiber_switch_context(zend_fiber_transfer *transfer)
469 {
470 zend_fiber_context *from = EG(current_fiber_context);
471 zend_fiber_context *to = transfer->context;
472 zend_fiber_vm_state state;
473
474 ZEND_ASSERT(to && to->handle && to->status != ZEND_FIBER_STATUS_DEAD && "Invalid fiber context");
475 ZEND_ASSERT(from && "From fiber context must be present");
476 ZEND_ASSERT(to != from && "Cannot switch into the running fiber context");
477
478 /* Assert that all error transfers hold a Throwable value. */
479 ZEND_ASSERT((
480 !(transfer->flags & ZEND_FIBER_TRANSFER_FLAG_ERROR) ||
481 (Z_TYPE(transfer->value) == IS_OBJECT && (
482 zend_is_unwind_exit(Z_OBJ(transfer->value)) ||
483 zend_is_graceful_exit(Z_OBJ(transfer->value)) ||
484 instanceof_function(Z_OBJCE(transfer->value), zend_ce_throwable)
485 ))
486 ) && "Error transfer requires a throwable value");
487
488 zend_observer_fiber_switch_notify(from, to);
489
490 zend_fiber_capture_vm_state(&state);
491
492 to->status = ZEND_FIBER_STATUS_RUNNING;
493
494 if (EXPECTED(from->status == ZEND_FIBER_STATUS_RUNNING)) {
495 from->status = ZEND_FIBER_STATUS_SUSPENDED;
496 }
497
498 /* Update transfer context with the current fiber before switching. */
499 transfer->context = from;
500
501 EG(current_fiber_context) = to;
502
503 #ifdef __SANITIZE_ADDRESS__
504 void *fake_stack = NULL;
505 __sanitizer_start_switch_fiber(
506 from->status != ZEND_FIBER_STATUS_DEAD ? &fake_stack : NULL,
507 to->stack->asan_pointer,
508 to->stack->asan_size);
509 #endif
510
511 #ifdef ZEND_FIBER_UCONTEXT
512 transfer_data = transfer;
513
514 swapcontext(from->handle, to->handle);
515
516 /* Copy transfer struct because it might live on the other fiber's stack that will eventually be destroyed. */
517 *transfer = *transfer_data;
518 #else
519 boost_context_data data = jump_fcontext(to->handle, transfer);
520
521 /* Copy transfer struct because it might live on the other fiber's stack that will eventually be destroyed. */
522 *transfer = *data.transfer;
523 #endif
524
525 to = transfer->context;
526
527 #ifndef ZEND_FIBER_UCONTEXT
528 /* Get the context that resumed us and update its handle to allow for symmetric coroutines. */
529 to->handle = data.handle;
530 #endif
531
532 #ifdef __SANITIZE_ADDRESS__
533 __sanitizer_finish_switch_fiber(fake_stack, &to->stack->asan_pointer, &to->stack->asan_size);
534 #endif
535
536 EG(current_fiber_context) = from;
537
538 zend_fiber_restore_vm_state(&state);
539
540 /* Destroy prior context if it has been marked as dead. */
541 if (to->status == ZEND_FIBER_STATUS_DEAD) {
542 zend_fiber_destroy_context(to);
543 }
544 }
545
zend_fiber_cleanup(zend_fiber_context * context)546 static void zend_fiber_cleanup(zend_fiber_context *context)
547 {
548 zend_fiber *fiber = zend_fiber_from_context(context);
549
550 zend_vm_stack current_stack = EG(vm_stack);
551 EG(vm_stack) = fiber->vm_stack;
552 zend_vm_stack_destroy();
553 EG(vm_stack) = current_stack;
554 fiber->execute_data = NULL;
555 fiber->stack_bottom = NULL;
556 fiber->caller = NULL;
557 }
558
zend_fiber_execute(zend_fiber_transfer * transfer)559 static ZEND_STACK_ALIGNED void zend_fiber_execute(zend_fiber_transfer *transfer)
560 {
561 ZEND_ASSERT(Z_TYPE(transfer->value) == IS_NULL && "Initial transfer value to fiber context must be NULL");
562 ZEND_ASSERT(!transfer->flags && "No flags should be set on initial transfer");
563
564 zend_fiber *fiber = EG(active_fiber);
565
566 /* Determine the current error_reporting ini setting. */
567 zend_long error_reporting = INI_INT("error_reporting");
568 /* If error_reporting is 0 and not explicitly set to 0, INI_STR returns a null pointer. */
569 if (!error_reporting && !INI_STR("error_reporting")) {
570 error_reporting = E_ALL;
571 }
572
573 EG(vm_stack) = NULL;
574
575 zend_first_try {
576 zend_vm_stack stack = zend_vm_stack_new_page(ZEND_FIBER_VM_STACK_SIZE, NULL);
577 EG(vm_stack) = stack;
578 EG(vm_stack_top) = stack->top + ZEND_CALL_FRAME_SLOT;
579 EG(vm_stack_end) = stack->end;
580 EG(vm_stack_page_size) = ZEND_FIBER_VM_STACK_SIZE;
581
582 fiber->execute_data = (zend_execute_data *) stack->top;
583 fiber->stack_bottom = fiber->execute_data;
584
585 memset(fiber->execute_data, 0, sizeof(zend_execute_data));
586
587 fiber->execute_data->func = &zend_fiber_function;
588 fiber->stack_bottom->prev_execute_data = EG(current_execute_data);
589
590 EG(current_execute_data) = fiber->execute_data;
591 EG(jit_trace_num) = 0;
592 EG(error_reporting) = error_reporting;
593
594 #ifdef ZEND_CHECK_STACK_LIMIT
595 EG(stack_base) = zend_fiber_stack_base(fiber->context.stack);
596 EG(stack_limit) = zend_fiber_stack_limit(fiber->context.stack);
597 #endif
598
599 fiber->fci.retval = &fiber->result;
600
601 zend_call_function(&fiber->fci, &fiber->fci_cache);
602
603 /* Cleanup callback and unset field to prevent GC / duplicate dtor issues. */
604 zval_ptr_dtor(&fiber->fci.function_name);
605 ZVAL_UNDEF(&fiber->fci.function_name);
606
607 if (EG(exception)) {
608 if (!(fiber->flags & ZEND_FIBER_FLAG_DESTROYED)
609 || !(zend_is_graceful_exit(EG(exception)) || zend_is_unwind_exit(EG(exception)))
610 ) {
611 fiber->flags |= ZEND_FIBER_FLAG_THREW;
612 transfer->flags = ZEND_FIBER_TRANSFER_FLAG_ERROR;
613
614 ZVAL_OBJ_COPY(&transfer->value, EG(exception));
615 }
616
617 zend_clear_exception();
618 }
619 } zend_catch {
620 fiber->flags |= ZEND_FIBER_FLAG_BAILOUT;
621 transfer->flags = ZEND_FIBER_TRANSFER_FLAG_BAILOUT;
622 } zend_end_try();
623
624 fiber->context.cleanup = &zend_fiber_cleanup;
625 fiber->vm_stack = EG(vm_stack);
626
627 transfer->context = fiber->caller;
628 }
629
630 /* Handles forwarding of result / error from a transfer into the running fiber. */
zend_fiber_delegate_transfer_result(zend_fiber_transfer * transfer,INTERNAL_FUNCTION_PARAMETERS)631 static zend_always_inline void zend_fiber_delegate_transfer_result(
632 zend_fiber_transfer *transfer, INTERNAL_FUNCTION_PARAMETERS
633 ) {
634 if (transfer->flags & ZEND_FIBER_TRANSFER_FLAG_ERROR) {
635 /* Use internal throw to skip the Throwable-check that would fail for (graceful) exit. */
636 zend_throw_exception_internal(Z_OBJ(transfer->value));
637 RETURN_THROWS();
638 }
639
640 RETURN_COPY_VALUE(&transfer->value);
641 }
642
zend_fiber_switch_to(zend_fiber_context * context,zval * value,bool exception)643 static zend_always_inline zend_fiber_transfer zend_fiber_switch_to(
644 zend_fiber_context *context, zval *value, bool exception
645 ) {
646 zend_fiber_transfer transfer = {
647 .context = context,
648 .flags = exception ? ZEND_FIBER_TRANSFER_FLAG_ERROR : 0,
649 };
650
651 if (value) {
652 ZVAL_COPY(&transfer.value, value);
653 } else {
654 ZVAL_NULL(&transfer.value);
655 }
656
657 zend_fiber_switch_context(&transfer);
658
659 /* Forward bailout into current fiber. */
660 if (UNEXPECTED(transfer.flags & ZEND_FIBER_TRANSFER_FLAG_BAILOUT)) {
661 EG(active_fiber) = NULL;
662 zend_bailout();
663 }
664
665 return transfer;
666 }
667
zend_fiber_resume(zend_fiber * fiber,zval * value,bool exception)668 static zend_always_inline zend_fiber_transfer zend_fiber_resume(zend_fiber *fiber, zval *value, bool exception)
669 {
670 zend_fiber *previous = EG(active_fiber);
671
672 if (previous) {
673 previous->execute_data = EG(current_execute_data);
674 }
675
676 fiber->caller = EG(current_fiber_context);
677 EG(active_fiber) = fiber;
678
679 zend_fiber_transfer transfer = zend_fiber_switch_to(fiber->previous, value, exception);
680
681 EG(active_fiber) = previous;
682
683 return transfer;
684 }
685
zend_fiber_suspend(zend_fiber * fiber,zval * value)686 static zend_always_inline zend_fiber_transfer zend_fiber_suspend(zend_fiber *fiber, zval *value)
687 {
688 ZEND_ASSERT(fiber->caller != NULL);
689
690 zend_fiber_context *caller = fiber->caller;
691 fiber->previous = EG(current_fiber_context);
692 fiber->caller = NULL;
693 fiber->execute_data = EG(current_execute_data);
694
695 return zend_fiber_switch_to(caller, value, false);
696 }
697
zend_fiber_object_create(zend_class_entry * ce)698 static zend_object *zend_fiber_object_create(zend_class_entry *ce)
699 {
700 zend_fiber *fiber = emalloc(sizeof(zend_fiber));
701 memset(fiber, 0, sizeof(zend_fiber));
702
703 zend_object_std_init(&fiber->std, ce);
704 return &fiber->std;
705 }
706
zend_fiber_object_destroy(zend_object * object)707 static void zend_fiber_object_destroy(zend_object *object)
708 {
709 zend_fiber *fiber = (zend_fiber *) object;
710
711 if (fiber->context.status != ZEND_FIBER_STATUS_SUSPENDED) {
712 return;
713 }
714
715 zend_object *exception = EG(exception);
716 EG(exception) = NULL;
717
718 zval graceful_exit;
719 ZVAL_OBJ(&graceful_exit, zend_create_graceful_exit());
720
721 fiber->flags |= ZEND_FIBER_FLAG_DESTROYED;
722
723 zend_fiber_transfer transfer = zend_fiber_resume(fiber, &graceful_exit, true);
724
725 zval_ptr_dtor(&graceful_exit);
726
727 if (transfer.flags & ZEND_FIBER_TRANSFER_FLAG_ERROR) {
728 EG(exception) = Z_OBJ(transfer.value);
729
730 if (!exception && EG(current_execute_data) && EG(current_execute_data)->func
731 && ZEND_USER_CODE(EG(current_execute_data)->func->common.type)) {
732 zend_rethrow_exception(EG(current_execute_data));
733 }
734
735 zend_exception_set_previous(EG(exception), exception);
736
737 if (!EG(current_execute_data)) {
738 zend_exception_error(EG(exception), E_ERROR);
739 }
740 } else {
741 zval_ptr_dtor(&transfer.value);
742 EG(exception) = exception;
743 }
744 }
745
zend_fiber_object_free(zend_object * object)746 static void zend_fiber_object_free(zend_object *object)
747 {
748 zend_fiber *fiber = (zend_fiber *) object;
749
750 zval_ptr_dtor(&fiber->fci.function_name);
751 zval_ptr_dtor(&fiber->result);
752
753 zend_object_std_dtor(&fiber->std);
754 }
755
zend_fiber_object_gc(zend_object * object,zval ** table,int * num)756 static HashTable *zend_fiber_object_gc(zend_object *object, zval **table, int *num)
757 {
758 zend_fiber *fiber = (zend_fiber *) object;
759 zend_get_gc_buffer *buf = zend_get_gc_buffer_create();
760
761 zend_get_gc_buffer_add_zval(buf, &fiber->fci.function_name);
762 zend_get_gc_buffer_add_zval(buf, &fiber->result);
763
764 if (fiber->context.status != ZEND_FIBER_STATUS_SUSPENDED || fiber->caller != NULL) {
765 zend_get_gc_buffer_use(buf, table, num);
766 return NULL;
767 }
768
769 HashTable *lastSymTable = NULL;
770 zend_execute_data *ex = fiber->execute_data;
771 for (; ex; ex = ex->prev_execute_data) {
772 HashTable *symTable = zend_unfinished_execution_gc_ex(ex, ex->func && ZEND_USER_CODE(ex->func->type) ? ex->call : NULL, buf, false);
773 if (symTable) {
774 if (lastSymTable) {
775 zval *val;
776 ZEND_HASH_FOREACH_VAL(lastSymTable, val) {
777 if (EXPECTED(Z_TYPE_P(val) == IS_INDIRECT)) {
778 val = Z_INDIRECT_P(val);
779 }
780 zend_get_gc_buffer_add_zval(buf, val);
781 } ZEND_HASH_FOREACH_END();
782 }
783 lastSymTable = symTable;
784 }
785 }
786
787 zend_get_gc_buffer_use(buf, table, num);
788
789 return lastSymTable;
790 }
791
ZEND_METHOD(Fiber,__construct)792 ZEND_METHOD(Fiber, __construct)
793 {
794 zend_fcall_info fci;
795 zend_fcall_info_cache fcc;
796
797 ZEND_PARSE_PARAMETERS_START(1, 1)
798 Z_PARAM_FUNC(fci, fcc)
799 ZEND_PARSE_PARAMETERS_END();
800
801 zend_fiber *fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
802
803 if (UNEXPECTED(fiber->context.status != ZEND_FIBER_STATUS_INIT || Z_TYPE(fiber->fci.function_name) != IS_UNDEF)) {
804 zend_throw_error(zend_ce_fiber_error, "Cannot call constructor twice");
805 RETURN_THROWS();
806 }
807
808 fiber->fci = fci;
809 fiber->fci_cache = fcc;
810
811 // Keep a reference to closures or callable objects while the fiber is running.
812 Z_TRY_ADDREF(fiber->fci.function_name);
813 }
814
ZEND_METHOD(Fiber,start)815 ZEND_METHOD(Fiber, start)
816 {
817 zend_fiber *fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
818
819 ZEND_PARSE_PARAMETERS_START(0, -1)
820 Z_PARAM_VARIADIC_WITH_NAMED(fiber->fci.params, fiber->fci.param_count, fiber->fci.named_params);
821 ZEND_PARSE_PARAMETERS_END();
822
823 if (UNEXPECTED(zend_fiber_switch_blocked())) {
824 zend_throw_error(zend_ce_fiber_error, "Cannot switch fibers in current execution context");
825 RETURN_THROWS();
826 }
827
828 if (fiber->context.status != ZEND_FIBER_STATUS_INIT) {
829 zend_throw_error(zend_ce_fiber_error, "Cannot start a fiber that has already been started");
830 RETURN_THROWS();
831 }
832
833 if (zend_fiber_init_context(&fiber->context, zend_ce_fiber, zend_fiber_execute, EG(fiber_stack_size)) == FAILURE) {
834 RETURN_THROWS();
835 }
836
837 fiber->previous = &fiber->context;
838
839 zend_fiber_transfer transfer = zend_fiber_resume(fiber, NULL, false);
840
841 zend_fiber_delegate_transfer_result(&transfer, INTERNAL_FUNCTION_PARAM_PASSTHRU);
842 }
843
ZEND_METHOD(Fiber,suspend)844 ZEND_METHOD(Fiber, suspend)
845 {
846 zval *value = NULL;
847
848 ZEND_PARSE_PARAMETERS_START(0, 1)
849 Z_PARAM_OPTIONAL
850 Z_PARAM_ZVAL(value);
851 ZEND_PARSE_PARAMETERS_END();
852
853 zend_fiber *fiber = EG(active_fiber);
854
855 if (UNEXPECTED(!fiber)) {
856 zend_throw_error(zend_ce_fiber_error, "Cannot suspend outside of a fiber");
857 RETURN_THROWS();
858 }
859
860 if (UNEXPECTED(fiber->flags & ZEND_FIBER_FLAG_DESTROYED)) {
861 zend_throw_error(zend_ce_fiber_error, "Cannot suspend in a force-closed fiber");
862 RETURN_THROWS();
863 }
864
865 if (UNEXPECTED(zend_fiber_switch_blocked())) {
866 zend_throw_error(zend_ce_fiber_error, "Cannot switch fibers in current execution context");
867 RETURN_THROWS();
868 }
869
870 ZEND_ASSERT(fiber->context.status == ZEND_FIBER_STATUS_RUNNING || fiber->context.status == ZEND_FIBER_STATUS_SUSPENDED);
871
872 fiber->stack_bottom->prev_execute_data = NULL;
873
874 zend_fiber_transfer transfer = zend_fiber_suspend(fiber, value);
875
876 zend_fiber_delegate_transfer_result(&transfer, INTERNAL_FUNCTION_PARAM_PASSTHRU);
877 }
878
ZEND_METHOD(Fiber,resume)879 ZEND_METHOD(Fiber, resume)
880 {
881 zend_fiber *fiber;
882 zval *value = NULL;
883
884 ZEND_PARSE_PARAMETERS_START(0, 1)
885 Z_PARAM_OPTIONAL
886 Z_PARAM_ZVAL(value);
887 ZEND_PARSE_PARAMETERS_END();
888
889 if (UNEXPECTED(zend_fiber_switch_blocked())) {
890 zend_throw_error(zend_ce_fiber_error, "Cannot switch fibers in current execution context");
891 RETURN_THROWS();
892 }
893
894 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
895
896 if (UNEXPECTED(fiber->context.status != ZEND_FIBER_STATUS_SUSPENDED || fiber->caller != NULL)) {
897 zend_throw_error(zend_ce_fiber_error, "Cannot resume a fiber that is not suspended");
898 RETURN_THROWS();
899 }
900
901 fiber->stack_bottom->prev_execute_data = EG(current_execute_data);
902
903 zend_fiber_transfer transfer = zend_fiber_resume(fiber, value, false);
904
905 zend_fiber_delegate_transfer_result(&transfer, INTERNAL_FUNCTION_PARAM_PASSTHRU);
906 }
907
ZEND_METHOD(Fiber,throw)908 ZEND_METHOD(Fiber, throw)
909 {
910 zend_fiber *fiber;
911 zval *exception;
912
913 ZEND_PARSE_PARAMETERS_START(1, 1)
914 Z_PARAM_OBJECT_OF_CLASS(exception, zend_ce_throwable)
915 ZEND_PARSE_PARAMETERS_END();
916
917 if (UNEXPECTED(zend_fiber_switch_blocked())) {
918 zend_throw_error(zend_ce_fiber_error, "Cannot switch fibers in current execution context");
919 RETURN_THROWS();
920 }
921
922 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
923
924 if (UNEXPECTED(fiber->context.status != ZEND_FIBER_STATUS_SUSPENDED || fiber->caller != NULL)) {
925 zend_throw_error(zend_ce_fiber_error, "Cannot resume a fiber that is not suspended");
926 RETURN_THROWS();
927 }
928
929 fiber->stack_bottom->prev_execute_data = EG(current_execute_data);
930
931 zend_fiber_transfer transfer = zend_fiber_resume(fiber, exception, true);
932
933 zend_fiber_delegate_transfer_result(&transfer, INTERNAL_FUNCTION_PARAM_PASSTHRU);
934 }
935
ZEND_METHOD(Fiber,isStarted)936 ZEND_METHOD(Fiber, isStarted)
937 {
938 zend_fiber *fiber;
939
940 ZEND_PARSE_PARAMETERS_NONE();
941
942 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
943
944 RETURN_BOOL(fiber->context.status != ZEND_FIBER_STATUS_INIT);
945 }
946
ZEND_METHOD(Fiber,isSuspended)947 ZEND_METHOD(Fiber, isSuspended)
948 {
949 zend_fiber *fiber;
950
951 ZEND_PARSE_PARAMETERS_NONE();
952
953 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
954
955 RETURN_BOOL(fiber->context.status == ZEND_FIBER_STATUS_SUSPENDED && fiber->caller == NULL);
956 }
957
ZEND_METHOD(Fiber,isRunning)958 ZEND_METHOD(Fiber, isRunning)
959 {
960 zend_fiber *fiber;
961
962 ZEND_PARSE_PARAMETERS_NONE();
963
964 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
965
966 RETURN_BOOL(fiber->context.status == ZEND_FIBER_STATUS_RUNNING || fiber->caller != NULL);
967 }
968
ZEND_METHOD(Fiber,isTerminated)969 ZEND_METHOD(Fiber, isTerminated)
970 {
971 zend_fiber *fiber;
972
973 ZEND_PARSE_PARAMETERS_NONE();
974
975 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
976
977 RETURN_BOOL(fiber->context.status == ZEND_FIBER_STATUS_DEAD);
978 }
979
ZEND_METHOD(Fiber,getReturn)980 ZEND_METHOD(Fiber, getReturn)
981 {
982 zend_fiber *fiber;
983 const char *message;
984
985 ZEND_PARSE_PARAMETERS_NONE();
986
987 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
988
989 if (fiber->context.status == ZEND_FIBER_STATUS_DEAD) {
990 if (fiber->flags & ZEND_FIBER_FLAG_THREW) {
991 message = "The fiber threw an exception";
992 } else if (fiber->flags & ZEND_FIBER_FLAG_BAILOUT) {
993 message = "The fiber exited with a fatal error";
994 } else {
995 RETURN_COPY_DEREF(&fiber->result);
996 }
997 } else if (fiber->context.status == ZEND_FIBER_STATUS_INIT) {
998 message = "The fiber has not been started";
999 } else {
1000 message = "The fiber has not returned";
1001 }
1002
1003 zend_throw_error(zend_ce_fiber_error, "Cannot get fiber return value: %s", message);
1004 RETURN_THROWS();
1005 }
1006
ZEND_METHOD(Fiber,getCurrent)1007 ZEND_METHOD(Fiber, getCurrent)
1008 {
1009 ZEND_PARSE_PARAMETERS_NONE();
1010
1011 zend_fiber *fiber = EG(active_fiber);
1012
1013 if (!fiber) {
1014 RETURN_NULL();
1015 }
1016
1017 RETURN_OBJ_COPY(&fiber->std);
1018 }
1019
ZEND_METHOD(FiberError,__construct)1020 ZEND_METHOD(FiberError, __construct)
1021 {
1022 zend_throw_error(
1023 NULL,
1024 "The \"%s\" class is reserved for internal use and cannot be manually instantiated",
1025 ZSTR_VAL(Z_OBJCE_P(ZEND_THIS)->name)
1026 );
1027 }
1028
1029
zend_register_fiber_ce(void)1030 void zend_register_fiber_ce(void)
1031 {
1032 zend_ce_fiber = register_class_Fiber();
1033 zend_ce_fiber->create_object = zend_fiber_object_create;
1034 zend_ce_fiber->default_object_handlers = &zend_fiber_handlers;
1035
1036 zend_fiber_handlers = std_object_handlers;
1037 zend_fiber_handlers.dtor_obj = zend_fiber_object_destroy;
1038 zend_fiber_handlers.free_obj = zend_fiber_object_free;
1039 zend_fiber_handlers.get_gc = zend_fiber_object_gc;
1040 zend_fiber_handlers.clone_obj = NULL;
1041
1042 zend_ce_fiber_error = register_class_FiberError(zend_ce_error);
1043 zend_ce_fiber_error->create_object = zend_ce_error->create_object;
1044 }
1045
zend_fiber_init(void)1046 void zend_fiber_init(void)
1047 {
1048 zend_fiber_context *context = ecalloc(1, sizeof(zend_fiber_context));
1049
1050 #if defined(__SANITIZE_ADDRESS__) || defined(ZEND_FIBER_UCONTEXT)
1051 // Main fiber stack is only needed if ASan or ucontext is enabled.
1052 context->stack = emalloc(sizeof(zend_fiber_stack));
1053
1054 #ifdef ZEND_FIBER_UCONTEXT
1055 context->handle = &context->stack->ucontext;
1056 #endif
1057 #endif
1058
1059 context->status = ZEND_FIBER_STATUS_RUNNING;
1060
1061 EG(main_fiber_context) = context;
1062 EG(current_fiber_context) = context;
1063 EG(active_fiber) = NULL;
1064
1065 zend_fiber_switch_blocking = 0;
1066 }
1067
zend_fiber_shutdown(void)1068 void zend_fiber_shutdown(void)
1069 {
1070 #if defined(__SANITIZE_ADDRESS__) || defined(ZEND_FIBER_UCONTEXT)
1071 efree(EG(main_fiber_context)->stack);
1072 #endif
1073
1074 efree(EG(main_fiber_context));
1075
1076 zend_fiber_switch_block();
1077 }
1078
