1 /*
2 +----------------------------------------------------------------------+
3 | Zend Engine |
4 +----------------------------------------------------------------------+
5 | Copyright (c) Zend Technologies Ltd. (http://www.zend.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 2.00 of the Zend license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.zend.com/license/2_00.txt. |
11 | If you did not receive a copy of the Zend license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@zend.com so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Aaron Piotrowski <aaron@trowski.com> |
16 | Martin Schröder <m.schroeder2007@gmail.com> |
17 +----------------------------------------------------------------------+
18 */
19
20 #include "zend.h"
21 #include "zend_API.h"
22 #include "zend_ini.h"
23 #include "zend_vm.h"
24 #include "zend_exceptions.h"
25 #include "zend_builtin_functions.h"
26 #include "zend_observer.h"
27 #include "zend_mmap.h"
28 #include "zend_compile.h"
29 #include "zend_closures.h"
30
31 #include "zend_fibers.h"
32 #include "zend_fibers_arginfo.h"
33
34 #ifdef HAVE_VALGRIND
35 # include <valgrind/valgrind.h>
36 #endif
37
38 #ifdef ZEND_FIBER_UCONTEXT
39 # include <ucontext.h>
40 #endif
41
42 #ifndef ZEND_WIN32
43 # include <unistd.h>
44 # include <sys/mman.h>
45 # include <limits.h>
46
47 # if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
48 # define MAP_ANONYMOUS MAP_ANON
49 # endif
50
51 /* FreeBSD require a first (i.e. addr) argument of mmap(2) is not NULL
52 * if MAP_STACK is passed.
53 * http://www.FreeBSD.org/cgi/query-pr.cgi?pr=158755 */
54 # if !defined(MAP_STACK) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
55 # undef MAP_STACK
56 # define MAP_STACK 0
57 # endif
58
59 # ifndef MAP_FAILED
60 # define MAP_FAILED ((void * ) -1)
61 # endif
62 #endif
63
64 #ifdef __SANITIZE_ADDRESS__
65 # include <sanitizer/asan_interface.h>
66 # include <sanitizer/common_interface_defs.h>
67 #endif
68
69 # if defined __CET__
70 # include <cet.h>
71 # define SHSTK_ENABLED (__CET__ & 0x2)
72 # define BOOST_CONTEXT_SHADOW_STACK (SHSTK_ENABLED && SHADOW_STACK_SYSCALL)
73 # define __NR_map_shadow_stack 451
74 # ifndef SHADOW_STACK_SET_TOKEN
75 # define SHADOW_STACK_SET_TOKEN 0x1
76 #endif
77 #endif
78
79 /* Encapsulates the fiber C stack with extension for debugging tools. */
80 struct _zend_fiber_stack {
81 void *pointer;
82 size_t size;
83
84 #ifdef HAVE_VALGRIND
85 unsigned int valgrind_stack_id;
86 #endif
87
88 #ifdef __SANITIZE_ADDRESS__
89 const void *asan_pointer;
90 size_t asan_size;
91 #endif
92
93 #ifdef ZEND_FIBER_UCONTEXT
94 /* Embedded ucontext to avoid unnecessary memory allocations. */
95 ucontext_t ucontext;
96 #elif BOOST_CONTEXT_SHADOW_STACK
97 /* Shadow stack: base, size */
98 void *ss_base;
99 size_t ss_size;
100 #endif
101 };
102
103 /* Zend VM state that needs to be captured / restored during fiber context switch. */
104 typedef struct _zend_fiber_vm_state {
105 zend_vm_stack vm_stack;
106 zval *vm_stack_top;
107 zval *vm_stack_end;
108 size_t vm_stack_page_size;
109 zend_execute_data *current_execute_data;
110 int error_reporting;
111 uint32_t jit_trace_num;
112 JMP_BUF *bailout;
113 zend_fiber *active_fiber;
114 #ifdef ZEND_CHECK_STACK_LIMIT
115 void *stack_base;
116 void *stack_limit;
117 #endif
118 } zend_fiber_vm_state;
119
zend_fiber_capture_vm_state(zend_fiber_vm_state * state)120 static zend_always_inline void zend_fiber_capture_vm_state(zend_fiber_vm_state *state)
121 {
122 state->vm_stack = EG(vm_stack);
123 state->vm_stack_top = EG(vm_stack_top);
124 state->vm_stack_end = EG(vm_stack_end);
125 state->vm_stack_page_size = EG(vm_stack_page_size);
126 state->current_execute_data = EG(current_execute_data);
127 state->error_reporting = EG(error_reporting);
128 state->jit_trace_num = EG(jit_trace_num);
129 state->bailout = EG(bailout);
130 state->active_fiber = EG(active_fiber);
131 #ifdef ZEND_CHECK_STACK_LIMIT
132 state->stack_base = EG(stack_base);
133 state->stack_limit = EG(stack_limit);
134 #endif
135 }
136
zend_fiber_restore_vm_state(zend_fiber_vm_state * state)137 static zend_always_inline void zend_fiber_restore_vm_state(zend_fiber_vm_state *state)
138 {
139 EG(vm_stack) = state->vm_stack;
140 EG(vm_stack_top) = state->vm_stack_top;
141 EG(vm_stack_end) = state->vm_stack_end;
142 EG(vm_stack_page_size) = state->vm_stack_page_size;
143 EG(current_execute_data) = state->current_execute_data;
144 EG(error_reporting) = state->error_reporting;
145 EG(jit_trace_num) = state->jit_trace_num;
146 EG(bailout) = state->bailout;
147 EG(active_fiber) = state->active_fiber;
148 #ifdef ZEND_CHECK_STACK_LIMIT
149 EG(stack_base) = state->stack_base;
150 EG(stack_limit) = state->stack_limit;
151 #endif
152 }
153
154 #ifdef ZEND_FIBER_UCONTEXT
155 ZEND_TLS zend_fiber_transfer *transfer_data;
156 #else
157 /* boost_context_data is our customized definition of struct transfer_t as
158 * provided by boost.context in fcontext.hpp:
159 *
160 * typedef void* fcontext_t;
161 *
162 * struct transfer_t {
163 * fcontext_t fctx;
164 * void *data;
165 * }; */
166
167 typedef struct {
168 void *handle;
169 zend_fiber_transfer *transfer;
170 } boost_context_data;
171
172 /* These functions are defined in assembler files provided by boost.context (located in "Zend/asm"). */
173 extern void *make_fcontext(void *sp, size_t size, void (*fn)(boost_context_data));
174 extern ZEND_INDIRECT_RETURN boost_context_data jump_fcontext(void *to, zend_fiber_transfer *transfer);
175 #endif
176
177 ZEND_API zend_class_entry *zend_ce_fiber;
178 static zend_class_entry *zend_ce_fiber_error;
179
180 static zend_object_handlers zend_fiber_handlers;
181
182 static zend_function zend_fiber_function = { ZEND_INTERNAL_FUNCTION };
183
184 ZEND_TLS uint32_t zend_fiber_switch_blocking = 0;
185
186 #define ZEND_FIBER_DEFAULT_PAGE_SIZE 4096
187
zend_fiber_get_page_size(void)188 static size_t zend_fiber_get_page_size(void)
189 {
190 static size_t page_size = 0;
191
192 if (!page_size) {
193 page_size = zend_get_page_size();
194 if (!page_size || (page_size & (page_size - 1))) {
195 /* anyway, we have to return a valid result */
196 page_size = ZEND_FIBER_DEFAULT_PAGE_SIZE;
197 }
198 }
199
200 return page_size;
201 }
202
zend_fiber_stack_allocate(size_t size)203 static zend_fiber_stack *zend_fiber_stack_allocate(size_t size)
204 {
205 void *pointer;
206 const size_t page_size = zend_fiber_get_page_size();
207 const size_t minimum_stack_size = page_size + ZEND_FIBER_GUARD_PAGES * page_size;
208
209 if (size < minimum_stack_size) {
210 zend_throw_exception_ex(NULL, 0, "Fiber stack size is too small, it needs to be at least %zu bytes", minimum_stack_size);
211 return NULL;
212 }
213
214 const size_t stack_size = (size + page_size - 1) / page_size * page_size;
215 const size_t alloc_size = stack_size + ZEND_FIBER_GUARD_PAGES * page_size;
216
217 #ifdef ZEND_WIN32
218 pointer = VirtualAlloc(0, alloc_size, MEM_COMMIT, PAGE_READWRITE);
219
220 if (!pointer) {
221 DWORD err = GetLastError();
222 char *errmsg = php_win32_error_to_msg(err);
223 zend_throw_exception_ex(NULL, 0, "Fiber stack allocate failed: VirtualAlloc failed: [0x%08lx] %s", err, errmsg[0] ? errmsg : "Unknown");
224 php_win32_error_msg_free(errmsg);
225 return NULL;
226 }
227
228 # if ZEND_FIBER_GUARD_PAGES
229 DWORD protect;
230
231 if (!VirtualProtect(pointer, ZEND_FIBER_GUARD_PAGES * page_size, PAGE_READWRITE | PAGE_GUARD, &protect)) {
232 DWORD err = GetLastError();
233 char *errmsg = php_win32_error_to_msg(err);
234 zend_throw_exception_ex(NULL, 0, "Fiber stack protect failed: VirtualProtect failed: [0x%08lx] %s", err, errmsg[0] ? errmsg : "Unknown");
235 php_win32_error_msg_free(errmsg);
236 VirtualFree(pointer, 0, MEM_RELEASE);
237 return NULL;
238 }
239 # endif
240 #else
241 pointer = mmap(NULL, alloc_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
242
243 if (pointer == MAP_FAILED) {
244 zend_throw_exception_ex(NULL, 0, "Fiber stack allocate failed: mmap failed: %s (%d)", strerror(errno), errno);
245 return NULL;
246 }
247
248 zend_mmap_set_name(pointer, alloc_size, "zend_fiber_stack");
249
250 # if ZEND_FIBER_GUARD_PAGES
251 if (mprotect(pointer, ZEND_FIBER_GUARD_PAGES * page_size, PROT_NONE) < 0) {
252 zend_throw_exception_ex(NULL, 0, "Fiber stack protect failed: mprotect failed: %s (%d)", strerror(errno), errno);
253 munmap(pointer, alloc_size);
254 return NULL;
255 }
256 # endif
257 #endif
258
259 zend_fiber_stack *stack = emalloc(sizeof(zend_fiber_stack));
260
261 stack->pointer = (void *) ((uintptr_t) pointer + ZEND_FIBER_GUARD_PAGES * page_size);
262 stack->size = stack_size;
263
264 #if !defined(ZEND_FIBER_UCONTEXT) && BOOST_CONTEXT_SHADOW_STACK
265 /* shadow stack saves ret address only, need less space */
266 stack->ss_size= stack_size >> 5;
267
268 /* align shadow stack to 8 bytes. */
269 stack->ss_size = (stack->ss_size + 7) & ~7;
270
271 /* issue syscall to create shadow stack for the new fcontext */
272 /* SHADOW_STACK_SET_TOKEN option will put "restore token" on the new shadow stack */
273 stack->ss_base = (void *)syscall(__NR_map_shadow_stack, 0, stack->ss_size, SHADOW_STACK_SET_TOKEN);
274
275 if (stack->ss_base == MAP_FAILED) {
276 zend_throw_exception_ex(NULL, 0, "Fiber shadow stack allocate failed: mmap failed: %s (%d)", strerror(errno), errno);
277 return NULL;
278 }
279 #endif
280
281 #ifdef VALGRIND_STACK_REGISTER
282 uintptr_t base = (uintptr_t) stack->pointer;
283 stack->valgrind_stack_id = VALGRIND_STACK_REGISTER(base, base + stack->size);
284 #endif
285
286 #ifdef __SANITIZE_ADDRESS__
287 stack->asan_pointer = stack->pointer;
288 stack->asan_size = stack->size;
289 #endif
290
291 return stack;
292 }
293
zend_fiber_stack_free(zend_fiber_stack * stack)294 static void zend_fiber_stack_free(zend_fiber_stack *stack)
295 {
296 #ifdef VALGRIND_STACK_DEREGISTER
297 VALGRIND_STACK_DEREGISTER(stack->valgrind_stack_id);
298 #endif
299
300 const size_t page_size = zend_fiber_get_page_size();
301
302 void *pointer = (void *) ((uintptr_t) stack->pointer - ZEND_FIBER_GUARD_PAGES * page_size);
303
304 #ifdef __SANITIZE_ADDRESS__
305 /* If another mmap happens after unmapping, it may trigger the stale stack red zones
306 * so we have to unpoison it before unmapping. */
307 ASAN_UNPOISON_MEMORY_REGION(pointer, stack->size + ZEND_FIBER_GUARD_PAGES * page_size);
308 #endif
309
310 #ifdef ZEND_WIN32
311 VirtualFree(pointer, 0, MEM_RELEASE);
312 #else
313 munmap(pointer, stack->size + ZEND_FIBER_GUARD_PAGES * page_size);
314 #endif
315
316 #if !defined(ZEND_FIBER_UCONTEXT) && BOOST_CONTEXT_SHADOW_STACK
317 munmap(stack->ss_base, stack->ss_size);
318 #endif
319
320 efree(stack);
321 }
322
323 #ifdef ZEND_CHECK_STACK_LIMIT
zend_fiber_stack_limit(zend_fiber_stack * stack)324 ZEND_API void* zend_fiber_stack_limit(zend_fiber_stack *stack)
325 {
326 zend_ulong reserve = EG(reserved_stack_size);
327
328 #ifdef __APPLE__
329 /* On Apple Clang, the stack probing function ___chkstk_darwin incorrectly
330 * probes a location that is twice the entered function's stack usage away
331 * from the stack pointer, when using an alternative stack.
332 * https://openradar.appspot.com/radar?id=5497722702397440
333 */
334 reserve = reserve * 2;
335 #endif
336
337 /* stack->pointer is the end of the stack */
338 return (int8_t*)stack->pointer + reserve;
339 }
340
zend_fiber_stack_base(zend_fiber_stack * stack)341 ZEND_API void* zend_fiber_stack_base(zend_fiber_stack *stack)
342 {
343 return (void*)((uintptr_t)stack->pointer + stack->size);
344 }
345 #endif
346
347 #ifdef ZEND_FIBER_UCONTEXT
zend_fiber_trampoline(void)348 static ZEND_NORETURN void zend_fiber_trampoline(void)
349 #else
350 static ZEND_NORETURN void zend_fiber_trampoline(boost_context_data data)
351 #endif
352 {
353 /* Initialize transfer struct with a copy of passed data. */
354 #ifdef ZEND_FIBER_UCONTEXT
355 zend_fiber_transfer transfer = *transfer_data;
356 #else
357 zend_fiber_transfer transfer = *data.transfer;
358 #endif
359
360 zend_fiber_context *from = transfer.context;
361
362 #ifdef __SANITIZE_ADDRESS__
363 __sanitizer_finish_switch_fiber(NULL, &from->stack->asan_pointer, &from->stack->asan_size);
364 #endif
365
366 #ifndef ZEND_FIBER_UCONTEXT
367 /* Get the context that resumed us and update its handle to allow for symmetric coroutines. */
368 from->handle = data.handle;
369 #endif
370
371 /* Ensure that previous fiber will be cleaned up (needed by symmetric coroutines). */
372 if (from->status == ZEND_FIBER_STATUS_DEAD) {
373 zend_fiber_destroy_context(from);
374 }
375
376 zend_fiber_context *context = EG(current_fiber_context);
377
378 context->function(&transfer);
379 context->status = ZEND_FIBER_STATUS_DEAD;
380
381 /* Final context switch, the fiber must not be resumed afterwards! */
382 zend_fiber_switch_context(&transfer);
383
384 /* Abort here because we are in an inconsistent program state. */
385 abort();
386 }
387
zend_fiber_switch_block(void)388 ZEND_API void zend_fiber_switch_block(void)
389 {
390 ++zend_fiber_switch_blocking;
391 }
392
zend_fiber_switch_unblock(void)393 ZEND_API void zend_fiber_switch_unblock(void)
394 {
395 ZEND_ASSERT(zend_fiber_switch_blocking && "Fiber switching was not blocked");
396 --zend_fiber_switch_blocking;
397 }
398
zend_fiber_switch_blocked(void)399 ZEND_API bool zend_fiber_switch_blocked(void)
400 {
401 return zend_fiber_switch_blocking;
402 }
403
zend_fiber_init_context(zend_fiber_context * context,void * kind,zend_fiber_coroutine coroutine,size_t stack_size)404 ZEND_API zend_result zend_fiber_init_context(zend_fiber_context *context, void *kind, zend_fiber_coroutine coroutine, size_t stack_size)
405 {
406 context->stack = zend_fiber_stack_allocate(stack_size);
407
408 if (UNEXPECTED(!context->stack)) {
409 return FAILURE;
410 }
411
412 #ifdef ZEND_FIBER_UCONTEXT
413 ucontext_t *handle = &context->stack->ucontext;
414
415 getcontext(handle);
416
417 handle->uc_stack.ss_size = context->stack->size;
418 handle->uc_stack.ss_sp = context->stack->pointer;
419 handle->uc_stack.ss_flags = 0;
420 handle->uc_link = NULL;
421
422 makecontext(handle, (void (*)(void)) zend_fiber_trampoline, 0);
423
424 context->handle = handle;
425 #else
426 // Stack grows down, calculate the top of the stack. make_fcontext then shifts pointer to lower 16-byte boundary.
427 void *stack = (void *) ((uintptr_t) context->stack->pointer + context->stack->size);
428
429 #if BOOST_CONTEXT_SHADOW_STACK
430 // pass the shadow stack pointer to make_fcontext
431 // i.e., link the new shadow stack with the new fcontext
432 // TODO should be a better way?
433 *((unsigned long*) (stack - 8)) = (unsigned long)context->stack->ss_base + context->stack->ss_size;
434 #endif
435
436 context->handle = make_fcontext(stack, context->stack->size, zend_fiber_trampoline);
437 ZEND_ASSERT(context->handle != NULL && "make_fcontext() never returns NULL");
438 #endif
439
440 context->kind = kind;
441 context->function = coroutine;
442
443 // Set status in case memory has not been zeroed.
444 context->status = ZEND_FIBER_STATUS_INIT;
445
446 zend_observer_fiber_init_notify(context);
447
448 return SUCCESS;
449 }
450
zend_fiber_destroy_context(zend_fiber_context * context)451 ZEND_API void zend_fiber_destroy_context(zend_fiber_context *context)
452 {
453 zend_observer_fiber_destroy_notify(context);
454
455 if (context->cleanup) {
456 context->cleanup(context);
457 }
458
459 zend_fiber_stack_free(context->stack);
460 }
461
zend_fiber_switch_context(zend_fiber_transfer * transfer)462 ZEND_API void zend_fiber_switch_context(zend_fiber_transfer *transfer)
463 {
464 zend_fiber_context *from = EG(current_fiber_context);
465 zend_fiber_context *to = transfer->context;
466 zend_fiber_vm_state state;
467
468 ZEND_ASSERT(to && to->handle && to->status != ZEND_FIBER_STATUS_DEAD && "Invalid fiber context");
469 ZEND_ASSERT(from && "From fiber context must be present");
470 ZEND_ASSERT(to != from && "Cannot switch into the running fiber context");
471
472 /* Assert that all error transfers hold a Throwable value. */
473 ZEND_ASSERT((
474 !(transfer->flags & ZEND_FIBER_TRANSFER_FLAG_ERROR) ||
475 (Z_TYPE(transfer->value) == IS_OBJECT && (
476 zend_is_unwind_exit(Z_OBJ(transfer->value)) ||
477 zend_is_graceful_exit(Z_OBJ(transfer->value)) ||
478 instanceof_function(Z_OBJCE(transfer->value), zend_ce_throwable)
479 ))
480 ) && "Error transfer requires a throwable value");
481
482 zend_observer_fiber_switch_notify(from, to);
483
484 zend_fiber_capture_vm_state(&state);
485
486 to->status = ZEND_FIBER_STATUS_RUNNING;
487
488 if (EXPECTED(from->status == ZEND_FIBER_STATUS_RUNNING)) {
489 from->status = ZEND_FIBER_STATUS_SUSPENDED;
490 }
491
492 /* Update transfer context with the current fiber before switching. */
493 transfer->context = from;
494
495 EG(current_fiber_context) = to;
496
497 #ifdef __SANITIZE_ADDRESS__
498 void *fake_stack = NULL;
499 __sanitizer_start_switch_fiber(
500 from->status != ZEND_FIBER_STATUS_DEAD ? &fake_stack : NULL,
501 to->stack->asan_pointer,
502 to->stack->asan_size);
503 #endif
504
505 #ifdef ZEND_FIBER_UCONTEXT
506 transfer_data = transfer;
507
508 swapcontext(from->handle, to->handle);
509
510 /* Copy transfer struct because it might live on the other fiber's stack that will eventually be destroyed. */
511 *transfer = *transfer_data;
512 #else
513 boost_context_data data = jump_fcontext(to->handle, transfer);
514
515 /* Copy transfer struct because it might live on the other fiber's stack that will eventually be destroyed. */
516 *transfer = *data.transfer;
517 #endif
518
519 to = transfer->context;
520
521 #ifndef ZEND_FIBER_UCONTEXT
522 /* Get the context that resumed us and update its handle to allow for symmetric coroutines. */
523 to->handle = data.handle;
524 #endif
525
526 #ifdef __SANITIZE_ADDRESS__
527 __sanitizer_finish_switch_fiber(fake_stack, &to->stack->asan_pointer, &to->stack->asan_size);
528 #endif
529
530 EG(current_fiber_context) = from;
531
532 zend_fiber_restore_vm_state(&state);
533
534 /* Destroy prior context if it has been marked as dead. */
535 if (to->status == ZEND_FIBER_STATUS_DEAD) {
536 zend_fiber_destroy_context(to);
537 }
538 }
539
zend_fiber_cleanup(zend_fiber_context * context)540 static void zend_fiber_cleanup(zend_fiber_context *context)
541 {
542 zend_fiber *fiber = zend_fiber_from_context(context);
543
544 zend_vm_stack current_stack = EG(vm_stack);
545 EG(vm_stack) = fiber->vm_stack;
546 zend_vm_stack_destroy();
547 EG(vm_stack) = current_stack;
548 fiber->execute_data = NULL;
549 fiber->stack_bottom = NULL;
550 fiber->caller = NULL;
551 }
552
zend_fiber_execute(zend_fiber_transfer * transfer)553 static ZEND_STACK_ALIGNED void zend_fiber_execute(zend_fiber_transfer *transfer)
554 {
555 ZEND_ASSERT(Z_TYPE(transfer->value) == IS_NULL && "Initial transfer value to fiber context must be NULL");
556 ZEND_ASSERT(!transfer->flags && "No flags should be set on initial transfer");
557
558 zend_fiber *fiber = EG(active_fiber);
559
560 /* Determine the current error_reporting ini setting. */
561 zend_long error_reporting = INI_INT("error_reporting");
562 /* If error_reporting is 0 and not explicitly set to 0, INI_STR returns a null pointer. */
563 if (!error_reporting && !INI_STR("error_reporting")) {
564 error_reporting = E_ALL;
565 }
566
567 EG(vm_stack) = NULL;
568
569 zend_first_try {
570 zend_vm_stack stack = zend_vm_stack_new_page(ZEND_FIBER_VM_STACK_SIZE, NULL);
571 EG(vm_stack) = stack;
572 EG(vm_stack_top) = stack->top + ZEND_CALL_FRAME_SLOT;
573 EG(vm_stack_end) = stack->end;
574 EG(vm_stack_page_size) = ZEND_FIBER_VM_STACK_SIZE;
575
576 fiber->execute_data = (zend_execute_data *) stack->top;
577 fiber->stack_bottom = fiber->execute_data;
578
579 memset(fiber->execute_data, 0, sizeof(zend_execute_data));
580
581 fiber->execute_data->func = &zend_fiber_function;
582 fiber->stack_bottom->prev_execute_data = EG(current_execute_data);
583
584 EG(current_execute_data) = fiber->execute_data;
585 EG(jit_trace_num) = 0;
586 EG(error_reporting) = error_reporting;
587
588 #ifdef ZEND_CHECK_STACK_LIMIT
589 EG(stack_base) = zend_fiber_stack_base(fiber->context.stack);
590 EG(stack_limit) = zend_fiber_stack_limit(fiber->context.stack);
591 #endif
592
593 fiber->fci.retval = &fiber->result;
594
595 zend_call_function(&fiber->fci, &fiber->fci_cache);
596
597 /* Cleanup callback and unset field to prevent GC / duplicate dtor issues. */
598 zval_ptr_dtor(&fiber->fci.function_name);
599 ZVAL_UNDEF(&fiber->fci.function_name);
600
601 if (EG(exception)) {
602 if (!(fiber->flags & ZEND_FIBER_FLAG_DESTROYED)
603 || !(zend_is_graceful_exit(EG(exception)) || zend_is_unwind_exit(EG(exception)))
604 ) {
605 fiber->flags |= ZEND_FIBER_FLAG_THREW;
606 transfer->flags = ZEND_FIBER_TRANSFER_FLAG_ERROR;
607
608 ZVAL_OBJ_COPY(&transfer->value, EG(exception));
609 }
610
611 zend_clear_exception();
612 }
613 } zend_catch {
614 fiber->flags |= ZEND_FIBER_FLAG_BAILOUT;
615 transfer->flags = ZEND_FIBER_TRANSFER_FLAG_BAILOUT;
616 } zend_end_try();
617
618 fiber->context.cleanup = &zend_fiber_cleanup;
619 fiber->vm_stack = EG(vm_stack);
620
621 transfer->context = fiber->caller;
622 }
623
624 /* Handles forwarding of result / error from a transfer into the running fiber. */
zend_fiber_delegate_transfer_result(zend_fiber_transfer * transfer,INTERNAL_FUNCTION_PARAMETERS)625 static zend_always_inline void zend_fiber_delegate_transfer_result(
626 zend_fiber_transfer *transfer, INTERNAL_FUNCTION_PARAMETERS
627 ) {
628 if (transfer->flags & ZEND_FIBER_TRANSFER_FLAG_ERROR) {
629 /* Use internal throw to skip the Throwable-check that would fail for (graceful) exit. */
630 zend_throw_exception_internal(Z_OBJ(transfer->value));
631 RETURN_THROWS();
632 }
633
634 RETURN_COPY_VALUE(&transfer->value);
635 }
636
zend_fiber_switch_to(zend_fiber_context * context,zval * value,bool exception)637 static zend_always_inline zend_fiber_transfer zend_fiber_switch_to(
638 zend_fiber_context *context, zval *value, bool exception
639 ) {
640 zend_fiber_transfer transfer = {
641 .context = context,
642 .flags = exception ? ZEND_FIBER_TRANSFER_FLAG_ERROR : 0,
643 };
644
645 if (value) {
646 ZVAL_COPY(&transfer.value, value);
647 } else {
648 ZVAL_NULL(&transfer.value);
649 }
650
651 zend_fiber_switch_context(&transfer);
652
653 /* Forward bailout into current fiber. */
654 if (UNEXPECTED(transfer.flags & ZEND_FIBER_TRANSFER_FLAG_BAILOUT)) {
655 EG(active_fiber) = NULL;
656 zend_bailout();
657 }
658
659 return transfer;
660 }
661
zend_fiber_resume(zend_fiber * fiber,zval * value,bool exception)662 static zend_always_inline zend_fiber_transfer zend_fiber_resume(zend_fiber *fiber, zval *value, bool exception)
663 {
664 zend_fiber *previous = EG(active_fiber);
665
666 if (previous) {
667 previous->execute_data = EG(current_execute_data);
668 }
669
670 fiber->caller = EG(current_fiber_context);
671 EG(active_fiber) = fiber;
672
673 zend_fiber_transfer transfer = zend_fiber_switch_to(fiber->previous, value, exception);
674
675 EG(active_fiber) = previous;
676
677 return transfer;
678 }
679
zend_fiber_suspend(zend_fiber * fiber,zval * value)680 static zend_always_inline zend_fiber_transfer zend_fiber_suspend(zend_fiber *fiber, zval *value)
681 {
682 ZEND_ASSERT(fiber->caller != NULL);
683
684 zend_fiber_context *caller = fiber->caller;
685 fiber->previous = EG(current_fiber_context);
686 fiber->caller = NULL;
687 fiber->execute_data = EG(current_execute_data);
688
689 return zend_fiber_switch_to(caller, value, false);
690 }
691
zend_fiber_object_create(zend_class_entry * ce)692 static zend_object *zend_fiber_object_create(zend_class_entry *ce)
693 {
694 zend_fiber *fiber = emalloc(sizeof(zend_fiber));
695 memset(fiber, 0, sizeof(zend_fiber));
696
697 zend_object_std_init(&fiber->std, ce);
698 return &fiber->std;
699 }
700
zend_fiber_object_destroy(zend_object * object)701 static void zend_fiber_object_destroy(zend_object *object)
702 {
703 zend_fiber *fiber = (zend_fiber *) object;
704
705 if (fiber->context.status != ZEND_FIBER_STATUS_SUSPENDED) {
706 return;
707 }
708
709 zend_object *exception = EG(exception);
710 EG(exception) = NULL;
711
712 zval graceful_exit;
713 ZVAL_OBJ(&graceful_exit, zend_create_graceful_exit());
714
715 fiber->flags |= ZEND_FIBER_FLAG_DESTROYED;
716
717 zend_fiber_transfer transfer = zend_fiber_resume(fiber, &graceful_exit, true);
718
719 zval_ptr_dtor(&graceful_exit);
720
721 if (transfer.flags & ZEND_FIBER_TRANSFER_FLAG_ERROR) {
722 EG(exception) = Z_OBJ(transfer.value);
723
724 if (!exception && EG(current_execute_data) && EG(current_execute_data)->func
725 && ZEND_USER_CODE(EG(current_execute_data)->func->common.type)) {
726 zend_rethrow_exception(EG(current_execute_data));
727 }
728
729 zend_exception_set_previous(EG(exception), exception);
730
731 if (!EG(current_execute_data)) {
732 zend_exception_error(EG(exception), E_ERROR);
733 }
734 } else {
735 zval_ptr_dtor(&transfer.value);
736 EG(exception) = exception;
737 }
738 }
739
zend_fiber_object_free(zend_object * object)740 static void zend_fiber_object_free(zend_object *object)
741 {
742 zend_fiber *fiber = (zend_fiber *) object;
743
744 zval_ptr_dtor(&fiber->fci.function_name);
745 zval_ptr_dtor(&fiber->result);
746
747 zend_object_std_dtor(&fiber->std);
748 }
749
zend_fiber_object_gc(zend_object * object,zval ** table,int * num)750 static HashTable *zend_fiber_object_gc(zend_object *object, zval **table, int *num)
751 {
752 zend_fiber *fiber = (zend_fiber *) object;
753 zend_get_gc_buffer *buf = zend_get_gc_buffer_create();
754
755 zend_get_gc_buffer_add_zval(buf, &fiber->fci.function_name);
756 zend_get_gc_buffer_add_zval(buf, &fiber->result);
757
758 if (fiber->context.status != ZEND_FIBER_STATUS_SUSPENDED || fiber->caller != NULL) {
759 zend_get_gc_buffer_use(buf, table, num);
760 return NULL;
761 }
762
763 HashTable *lastSymTable = NULL;
764 zend_execute_data *ex = fiber->execute_data;
765 for (; ex; ex = ex->prev_execute_data) {
766 HashTable *symTable = zend_unfinished_execution_gc_ex(ex, ex->func && ZEND_USER_CODE(ex->func->type) ? ex->call : NULL, buf, false);
767 if (symTable) {
768 if (lastSymTable) {
769 zval *val;
770 ZEND_HASH_FOREACH_VAL(lastSymTable, val) {
771 if (EXPECTED(Z_TYPE_P(val) == IS_INDIRECT)) {
772 val = Z_INDIRECT_P(val);
773 }
774 zend_get_gc_buffer_add_zval(buf, val);
775 } ZEND_HASH_FOREACH_END();
776 }
777 lastSymTable = symTable;
778 }
779 }
780
781 zend_get_gc_buffer_use(buf, table, num);
782
783 return lastSymTable;
784 }
785
ZEND_METHOD(Fiber,__construct)786 ZEND_METHOD(Fiber, __construct)
787 {
788 zend_fcall_info fci;
789 zend_fcall_info_cache fcc;
790
791 ZEND_PARSE_PARAMETERS_START(1, 1)
792 Z_PARAM_FUNC(fci, fcc)
793 ZEND_PARSE_PARAMETERS_END();
794
795 zend_fiber *fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
796
797 if (UNEXPECTED(fiber->context.status != ZEND_FIBER_STATUS_INIT || Z_TYPE(fiber->fci.function_name) != IS_UNDEF)) {
798 zend_throw_error(zend_ce_fiber_error, "Cannot call constructor twice");
799 RETURN_THROWS();
800 }
801
802 fiber->fci = fci;
803 fiber->fci_cache = fcc;
804
805 // Keep a reference to closures or callable objects while the fiber is running.
806 Z_TRY_ADDREF(fiber->fci.function_name);
807 }
808
ZEND_METHOD(Fiber,start)809 ZEND_METHOD(Fiber, start)
810 {
811 zend_fiber *fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
812
813 ZEND_PARSE_PARAMETERS_START(0, -1)
814 Z_PARAM_VARIADIC_WITH_NAMED(fiber->fci.params, fiber->fci.param_count, fiber->fci.named_params);
815 ZEND_PARSE_PARAMETERS_END();
816
817 if (UNEXPECTED(zend_fiber_switch_blocked())) {
818 zend_throw_error(zend_ce_fiber_error, "Cannot switch fibers in current execution context");
819 RETURN_THROWS();
820 }
821
822 if (fiber->context.status != ZEND_FIBER_STATUS_INIT) {
823 zend_throw_error(zend_ce_fiber_error, "Cannot start a fiber that has already been started");
824 RETURN_THROWS();
825 }
826
827 if (zend_fiber_init_context(&fiber->context, zend_ce_fiber, zend_fiber_execute, EG(fiber_stack_size)) == FAILURE) {
828 RETURN_THROWS();
829 }
830
831 fiber->previous = &fiber->context;
832
833 zend_fiber_transfer transfer = zend_fiber_resume(fiber, NULL, false);
834
835 zend_fiber_delegate_transfer_result(&transfer, INTERNAL_FUNCTION_PARAM_PASSTHRU);
836 }
837
ZEND_METHOD(Fiber,suspend)838 ZEND_METHOD(Fiber, suspend)
839 {
840 zval *value = NULL;
841
842 ZEND_PARSE_PARAMETERS_START(0, 1)
843 Z_PARAM_OPTIONAL
844 Z_PARAM_ZVAL(value);
845 ZEND_PARSE_PARAMETERS_END();
846
847 zend_fiber *fiber = EG(active_fiber);
848
849 if (UNEXPECTED(!fiber)) {
850 zend_throw_error(zend_ce_fiber_error, "Cannot suspend outside of a fiber");
851 RETURN_THROWS();
852 }
853
854 if (UNEXPECTED(fiber->flags & ZEND_FIBER_FLAG_DESTROYED)) {
855 zend_throw_error(zend_ce_fiber_error, "Cannot suspend in a force-closed fiber");
856 RETURN_THROWS();
857 }
858
859 if (UNEXPECTED(zend_fiber_switch_blocked())) {
860 zend_throw_error(zend_ce_fiber_error, "Cannot switch fibers in current execution context");
861 RETURN_THROWS();
862 }
863
864 ZEND_ASSERT(fiber->context.status == ZEND_FIBER_STATUS_RUNNING || fiber->context.status == ZEND_FIBER_STATUS_SUSPENDED);
865
866 fiber->stack_bottom->prev_execute_data = NULL;
867
868 zend_fiber_transfer transfer = zend_fiber_suspend(fiber, value);
869
870 zend_fiber_delegate_transfer_result(&transfer, INTERNAL_FUNCTION_PARAM_PASSTHRU);
871 }
872
ZEND_METHOD(Fiber,resume)873 ZEND_METHOD(Fiber, resume)
874 {
875 zend_fiber *fiber;
876 zval *value = NULL;
877
878 ZEND_PARSE_PARAMETERS_START(0, 1)
879 Z_PARAM_OPTIONAL
880 Z_PARAM_ZVAL(value);
881 ZEND_PARSE_PARAMETERS_END();
882
883 if (UNEXPECTED(zend_fiber_switch_blocked())) {
884 zend_throw_error(zend_ce_fiber_error, "Cannot switch fibers in current execution context");
885 RETURN_THROWS();
886 }
887
888 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
889
890 if (UNEXPECTED(fiber->context.status != ZEND_FIBER_STATUS_SUSPENDED || fiber->caller != NULL)) {
891 zend_throw_error(zend_ce_fiber_error, "Cannot resume a fiber that is not suspended");
892 RETURN_THROWS();
893 }
894
895 fiber->stack_bottom->prev_execute_data = EG(current_execute_data);
896
897 zend_fiber_transfer transfer = zend_fiber_resume(fiber, value, false);
898
899 zend_fiber_delegate_transfer_result(&transfer, INTERNAL_FUNCTION_PARAM_PASSTHRU);
900 }
901
ZEND_METHOD(Fiber,throw)902 ZEND_METHOD(Fiber, throw)
903 {
904 zend_fiber *fiber;
905 zval *exception;
906
907 ZEND_PARSE_PARAMETERS_START(1, 1)
908 Z_PARAM_OBJECT_OF_CLASS(exception, zend_ce_throwable)
909 ZEND_PARSE_PARAMETERS_END();
910
911 if (UNEXPECTED(zend_fiber_switch_blocked())) {
912 zend_throw_error(zend_ce_fiber_error, "Cannot switch fibers in current execution context");
913 RETURN_THROWS();
914 }
915
916 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
917
918 if (UNEXPECTED(fiber->context.status != ZEND_FIBER_STATUS_SUSPENDED || fiber->caller != NULL)) {
919 zend_throw_error(zend_ce_fiber_error, "Cannot resume a fiber that is not suspended");
920 RETURN_THROWS();
921 }
922
923 fiber->stack_bottom->prev_execute_data = EG(current_execute_data);
924
925 zend_fiber_transfer transfer = zend_fiber_resume(fiber, exception, true);
926
927 zend_fiber_delegate_transfer_result(&transfer, INTERNAL_FUNCTION_PARAM_PASSTHRU);
928 }
929
ZEND_METHOD(Fiber,isStarted)930 ZEND_METHOD(Fiber, isStarted)
931 {
932 zend_fiber *fiber;
933
934 ZEND_PARSE_PARAMETERS_NONE();
935
936 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
937
938 RETURN_BOOL(fiber->context.status != ZEND_FIBER_STATUS_INIT);
939 }
940
ZEND_METHOD(Fiber,isSuspended)941 ZEND_METHOD(Fiber, isSuspended)
942 {
943 zend_fiber *fiber;
944
945 ZEND_PARSE_PARAMETERS_NONE();
946
947 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
948
949 RETURN_BOOL(fiber->context.status == ZEND_FIBER_STATUS_SUSPENDED && fiber->caller == NULL);
950 }
951
ZEND_METHOD(Fiber,isRunning)952 ZEND_METHOD(Fiber, isRunning)
953 {
954 zend_fiber *fiber;
955
956 ZEND_PARSE_PARAMETERS_NONE();
957
958 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
959
960 RETURN_BOOL(fiber->context.status == ZEND_FIBER_STATUS_RUNNING || fiber->caller != NULL);
961 }
962
ZEND_METHOD(Fiber,isTerminated)963 ZEND_METHOD(Fiber, isTerminated)
964 {
965 zend_fiber *fiber;
966
967 ZEND_PARSE_PARAMETERS_NONE();
968
969 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
970
971 RETURN_BOOL(fiber->context.status == ZEND_FIBER_STATUS_DEAD);
972 }
973
ZEND_METHOD(Fiber,getReturn)974 ZEND_METHOD(Fiber, getReturn)
975 {
976 zend_fiber *fiber;
977 const char *message;
978
979 ZEND_PARSE_PARAMETERS_NONE();
980
981 fiber = (zend_fiber *) Z_OBJ_P(ZEND_THIS);
982
983 if (fiber->context.status == ZEND_FIBER_STATUS_DEAD) {
984 if (fiber->flags & ZEND_FIBER_FLAG_THREW) {
985 message = "The fiber threw an exception";
986 } else if (fiber->flags & ZEND_FIBER_FLAG_BAILOUT) {
987 message = "The fiber exited with a fatal error";
988 } else {
989 RETURN_COPY_DEREF(&fiber->result);
990 }
991 } else if (fiber->context.status == ZEND_FIBER_STATUS_INIT) {
992 message = "The fiber has not been started";
993 } else {
994 message = "The fiber has not returned";
995 }
996
997 zend_throw_error(zend_ce_fiber_error, "Cannot get fiber return value: %s", message);
998 RETURN_THROWS();
999 }
1000
ZEND_METHOD(Fiber,getCurrent)1001 ZEND_METHOD(Fiber, getCurrent)
1002 {
1003 ZEND_PARSE_PARAMETERS_NONE();
1004
1005 zend_fiber *fiber = EG(active_fiber);
1006
1007 if (!fiber) {
1008 RETURN_NULL();
1009 }
1010
1011 RETURN_OBJ_COPY(&fiber->std);
1012 }
1013
ZEND_METHOD(FiberError,__construct)1014 ZEND_METHOD(FiberError, __construct)
1015 {
1016 zend_throw_error(
1017 NULL,
1018 "The \"%s\" class is reserved for internal use and cannot be manually instantiated",
1019 ZSTR_VAL(Z_OBJCE_P(ZEND_THIS)->name)
1020 );
1021 }
1022
1023
zend_register_fiber_ce(void)1024 void zend_register_fiber_ce(void)
1025 {
1026 zend_ce_fiber = register_class_Fiber();
1027 zend_ce_fiber->create_object = zend_fiber_object_create;
1028 zend_ce_fiber->default_object_handlers = &zend_fiber_handlers;
1029
1030 zend_fiber_handlers = std_object_handlers;
1031 zend_fiber_handlers.dtor_obj = zend_fiber_object_destroy;
1032 zend_fiber_handlers.free_obj = zend_fiber_object_free;
1033 zend_fiber_handlers.get_gc = zend_fiber_object_gc;
1034 zend_fiber_handlers.clone_obj = NULL;
1035
1036 zend_ce_fiber_error = register_class_FiberError(zend_ce_error);
1037 zend_ce_fiber_error->create_object = zend_ce_error->create_object;
1038 }
1039
zend_fiber_init(void)1040 void zend_fiber_init(void)
1041 {
1042 zend_fiber_context *context = ecalloc(1, sizeof(zend_fiber_context));
1043
1044 #if defined(__SANITIZE_ADDRESS__) || defined(ZEND_FIBER_UCONTEXT)
1045 // Main fiber stack is only needed if ASan or ucontext is enabled.
1046 context->stack = emalloc(sizeof(zend_fiber_stack));
1047
1048 #ifdef ZEND_FIBER_UCONTEXT
1049 context->handle = &context->stack->ucontext;
1050 #endif
1051 #endif
1052
1053 context->status = ZEND_FIBER_STATUS_RUNNING;
1054
1055 EG(main_fiber_context) = context;
1056 EG(current_fiber_context) = context;
1057 EG(active_fiber) = NULL;
1058
1059 zend_fiber_switch_blocking = 0;
1060 }
1061
zend_fiber_shutdown(void)1062 void zend_fiber_shutdown(void)
1063 {
1064 #if defined(__SANITIZE_ADDRESS__) || defined(ZEND_FIBER_UCONTEXT)
1065 efree(EG(main_fiber_context)->stack);
1066 #endif
1067
1068 efree(EG(main_fiber_context));
1069
1070 zend_fiber_switch_block();
1071 }
1072
