Ruby: cont.c Source File

Ruby 4.0.5p0 (2026-05-20 revision 64336ffd0ee9e1f4c05891695a3d7b49cb709721)

cont.c
1/**********************************************************************
2
3 cont.c -
4
5 $Author$
6 created at: Thu May 23 09:03:43 2007
7
8 Copyright (C) 2007 Koichi Sasada
9
10**********************************************************************/
11
12#include "ruby/internal/config.h"
13
14#ifndef _WIN32
15#include <unistd.h>
16#include <sys/mman.h>
17#endif
18
19// On Solaris, madvise() is NOT declared for SUS (XPG4v2) or later,
20// but MADV_* macros are defined when __EXTENSIONS__ is defined.
21#ifdef NEED_MADVICE_PROTOTYPE_USING_CADDR_T
22#include <sys/types.h>
23extern int madvise(caddr_t, size_t, int);
24#endif
25
26#include COROUTINE_H
27
28#include "eval_intern.h"
29#include "internal.h"
30#include "internal/cont.h"
31#include "internal/thread.h"
32#include "internal/error.h"
33#include "internal/eval.h"
34#include "internal/gc.h"
35#include "internal/proc.h"
36#include "internal/sanitizers.h"
37#include "internal/warnings.h"
38#include "ruby/fiber/scheduler.h"
39#include "yjit.h"
40#include "vm_core.h"
41#include "vm_sync.h"
42#include "id_table.h"
43#include "ractor_core.h"
44
45static const int DEBUG = 0;
46
47#define RB_PAGE_SIZE (pagesize)
48#define RB_PAGE_MASK (~(RB_PAGE_SIZE - 1))
49static long pagesize;
50
51static const rb_data_type_t cont_data_type, fiber_data_type;
52static VALUE rb_cContinuation;
53static VALUE rb_cFiber;
54static VALUE rb_eFiberError;
55#ifdef RB_EXPERIMENTAL_FIBER_POOL
56static VALUE rb_cFiberPool;
57#endif
58
59#define CAPTURE_JUST_VALID_VM_STACK 1
60
61// Defined in `coroutine/$arch/Context.h`:
62#ifdef COROUTINE_LIMITED_ADDRESS_SPACE
63#define FIBER_POOL_ALLOCATION_FREE
64#define FIBER_POOL_INITIAL_SIZE 8
65#define FIBER_POOL_ALLOCATION_MAXIMUM_SIZE 32
66#else
67#define FIBER_POOL_INITIAL_SIZE 32
68#define FIBER_POOL_ALLOCATION_MAXIMUM_SIZE 1024
69#endif
70#ifdef RB_EXPERIMENTAL_FIBER_POOL
71#define FIBER_POOL_ALLOCATION_FREE
72#endif
73
74enum context_type {
75 CONTINUATION_CONTEXT = 0,
76 FIBER_CONTEXT = 1
77};
78
79 struct cont_saved_vm_stack {
80 VALUE *ptr;
81#ifdef CAPTURE_JUST_VALID_VM_STACK
82 size_t slen; /* length of stack (head of ec->vm_stack) */
83 size_t clen; /* length of control frames (tail of ec->vm_stack) */
84#endif
85};
86
87struct fiber_pool;
88
89// Represents a single stack.
90 struct fiber_pool_stack {
91 // A pointer to the memory allocation (lowest address) for the stack.
92 void * base;
93
94 // The current stack pointer, taking into account the direction of the stack.
95 void * current;
96
97 // The size of the stack excluding any guard pages.
98 size_t size;
99
100 // The available stack capacity w.r.t. the current stack offset.
101 size_t available;
102
103 // The pool this stack should be allocated from.
104 struct fiber_pool * pool;
105
106 // If the stack is allocated, the allocation it came from.
107 struct fiber_pool_allocation * allocation;
108};
109
110// A linked list of vacant (unused) stacks.
111// This structure is stored in the first page of a stack if it is not in use.
112// @sa fiber_pool_vacancy_pointer
113 struct fiber_pool_vacancy {
114 // Details about the vacant stack:
115 struct fiber_pool_stack stack;
116
117 // The vacancy linked list.
118#ifdef FIBER_POOL_ALLOCATION_FREE
119 struct fiber_pool_vacancy * previous;
120#endif
121 struct fiber_pool_vacancy * next;
122};
123
124// Manages singly linked list of mapped regions of memory which contains 1 more more stack:
125//
126// base = +-------------------------------+-----------------------+ +
127// |VM Stack |VM Stack | | |
128// | | | | |
129// | | | | |
130// +-------------------------------+ | |
131// |Machine Stack |Machine Stack | | |
132// | | | | |
133// | | | | |
134// | | | . . . . | | size
135// | | | | |
136// | | | | |
137// | | | | |
138// | | | | |
139// | | | | |
140// +-------------------------------+ | |
141// |Guard Page |Guard Page | | |
142// +-------------------------------+-----------------------+ v
143//
144// +------------------------------------------------------->
145//
146// count
147//
148 struct fiber_pool_allocation {
149 // A pointer to the memory mapped region.
150 void * base;
151
152 // The size of the individual stacks.
153 size_t size;
154
155 // The stride of individual stacks (including any guard pages or other accounting details).
156 size_t stride;
157
158 // The number of stacks that were allocated.
159 size_t count;
160
161#ifdef FIBER_POOL_ALLOCATION_FREE
162 // The number of stacks used in this allocation.
163 size_t used;
164#endif
165
166 struct fiber_pool * pool;
167
168 // The allocation linked list.
169#ifdef FIBER_POOL_ALLOCATION_FREE
170 struct fiber_pool_allocation * previous;
171#endif
172 struct fiber_pool_allocation * next;
173};
174
175// A fiber pool manages vacant stacks to reduce the overhead of creating fibers.
176 struct fiber_pool {
177 // A singly-linked list of allocations which contain 1 or more stacks each.
178 struct fiber_pool_allocation * allocations;
179
180 // Free list that provides O(1) stack "allocation".
181 struct fiber_pool_vacancy * vacancies;
182
183 // The size of the stack allocations (excluding any guard page).
184 size_t size;
185
186 // The total number of stacks that have been allocated in this pool.
187 size_t count;
188
189 // The initial number of stacks to allocate.
190 size_t initial_count;
191
192 // Whether to madvise(free) the stack or not.
193 // If this value is set to 1, the stack will be madvise(free)ed
194 // (or equivalent), where possible, when it is returned to the pool.
195 int free_stacks;
196
197 // The number of stacks that have been used in this pool.
198 size_t used;
199
200 // The amount to allocate for the vm_stack.
201 size_t vm_stack_size;
202};
203
204// Continuation contexts used by JITs
205 struct rb_jit_cont {
206 rb_execution_context_t *ec; // continuation ec
207 struct rb_jit_cont *prev, *next; // used to form lists
208};
209
210// Doubly linked list for enumerating all on-stack ISEQs.
211static struct rb_jit_cont *first_jit_cont;
212
213 typedef struct rb_context_struct {
214 enum context_type type;
215 int argc;
216 int kw_splat;
217 VALUE self;
218 VALUE value;
219
220 struct cont_saved_vm_stack saved_vm_stack;
221
222 struct {
223 VALUE *stack;
224 VALUE *stack_src;
225 size_t stack_size;
226 } machine;
227 rb_execution_context_t saved_ec;
228 rb_jmpbuf_t jmpbuf;
229 struct rb_jit_cont *jit_cont; // Continuation contexts for JITs
230} rb_context_t;
231
232/*
233 * Fiber status:
234 * [Fiber.new] ------> FIBER_CREATED ----> [Fiber#kill] --> |
235 * | [Fiber#resume] |
236 * v |
237 * +--> FIBER_RESUMED ----> [return] ------> |
238 * [Fiber#resume] | | [Fiber.yield/transfer] |
239 * [Fiber#transfer] | v |
240 * +--- FIBER_SUSPENDED --> [Fiber#kill] --> |
241 * |
242 * |
243 * FIBER_TERMINATED <-------------------+
244 */
245enum fiber_status {
246 FIBER_CREATED,
247 FIBER_RESUMED,
248 FIBER_SUSPENDED,
249 FIBER_TERMINATED
250};
251
252#define FIBER_CREATED_P(fiber) ((fiber)->status == FIBER_CREATED)
253#define FIBER_RESUMED_P(fiber) ((fiber)->status == FIBER_RESUMED)
254#define FIBER_SUSPENDED_P(fiber) ((fiber)->status == FIBER_SUSPENDED)
255#define FIBER_TERMINATED_P(fiber) ((fiber)->status == FIBER_TERMINATED)
256#define FIBER_RUNNABLE_P(fiber) (FIBER_CREATED_P(fiber) || FIBER_SUSPENDED_P(fiber))
257
258 struct rb_fiber_struct {
259 rb_context_t cont;
260 VALUE first_proc;
261 struct rb_fiber_struct *prev;
262 struct rb_fiber_struct *resuming_fiber;
263
264 BITFIELD(enum fiber_status, status, 2);
265 /* Whether the fiber is allowed to implicitly yield. */
266 unsigned int yielding : 1;
267 unsigned int blocking : 1;
268
269 unsigned int killed : 1;
270
271 struct coroutine_context context;
272 struct fiber_pool_stack stack;
273};
274
275static struct fiber_pool shared_fiber_pool = {NULL, NULL, 0, 0, 0, 0};
276
277void
278rb_free_shared_fiber_pool(void)
279{
280 struct fiber_pool_allocation *allocations = shared_fiber_pool.allocations;
281 while (allocations) {
282 struct fiber_pool_allocation *next = allocations->next;
283 xfree(allocations);
284 allocations = next;
285 }
286}
287
288static ID fiber_initialize_keywords[3] = {0};
289
290/*
291 * FreeBSD require a first (i.e. addr) argument of mmap(2) is not NULL
292 * if MAP_STACK is passed.
293 * https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=158755
294 */
295#if defined(MAP_STACK) && !defined(__FreeBSD__) && !defined(__FreeBSD_kernel__)
296#define FIBER_STACK_FLAGS (MAP_PRIVATE | MAP_ANON | MAP_STACK)
297#else
298#define FIBER_STACK_FLAGS (MAP_PRIVATE | MAP_ANON)
299#endif
300
301#define ERRNOMSG strerror(errno)
302
303// Locates the stack vacancy details for the given stack.
304inline static struct fiber_pool_vacancy *
305fiber_pool_vacancy_pointer(void * base, size_t size)
306{
307 STACK_GROW_DIR_DETECTION;
308
309 return (struct fiber_pool_vacancy *)(
310 (char*)base + STACK_DIR_UPPER(0, size - RB_PAGE_SIZE)
311 );
312}
313
314#if defined(COROUTINE_SANITIZE_ADDRESS)
315// Compute the base pointer for a vacant stack, for the area which can be poisoned.
316inline static void *
317fiber_pool_stack_poison_base(struct fiber_pool_stack * stack)
318{
319 STACK_GROW_DIR_DETECTION;
320
321 return (char*)stack->base + STACK_DIR_UPPER(RB_PAGE_SIZE, 0);
322}
323
324// Compute the size of the vacant stack, for the area that can be poisoned.
325inline static size_t
326fiber_pool_stack_poison_size(struct fiber_pool_stack * stack)
327{
328 return stack->size - RB_PAGE_SIZE;
329}
330#endif
331
332// Reset the current stack pointer and available size of the given stack.
333inline static void
334fiber_pool_stack_reset(struct fiber_pool_stack * stack)
335{
336 STACK_GROW_DIR_DETECTION;
337
338 stack->current = (char*)stack->base + STACK_DIR_UPPER(0, stack->size);
339 stack->available = stack->size;
340}
341
342// A pointer to the base of the current unused portion of the stack.
343inline static void *
344fiber_pool_stack_base(struct fiber_pool_stack * stack)
345{
346 STACK_GROW_DIR_DETECTION;
347
348 VM_ASSERT(stack->current);
349
350 return STACK_DIR_UPPER(stack->current, (char*)stack->current - stack->available);
351}
352
353// Allocate some memory from the stack. Used to allocate vm_stack inline with machine stack.
354// @sa fiber_initialize_coroutine
355inline static void *
356fiber_pool_stack_alloca(struct fiber_pool_stack * stack, size_t offset)
357{
358 STACK_GROW_DIR_DETECTION;
359
360 if (DEBUG) fprintf(stderr, "fiber_pool_stack_alloca(%p): %"PRIuSIZE"/%"PRIuSIZE"\n", (void*)stack, offset, stack->available);
361 VM_ASSERT(stack->available >= offset);
362
363 // The pointer to the memory being allocated:
364 void * pointer = STACK_DIR_UPPER(stack->current, (char*)stack->current - offset);
365
366 // Move the stack pointer:
367 stack->current = STACK_DIR_UPPER((char*)stack->current + offset, (char*)stack->current - offset);
368 stack->available -= offset;
369
370 return pointer;
371}
372
373// Reset the current stack pointer and available size of the given stack.
374inline static void
375fiber_pool_vacancy_reset(struct fiber_pool_vacancy * vacancy)
376{
377 fiber_pool_stack_reset(&vacancy->stack);
378
379 // Consume one page of the stack because it's used for the vacancy list:
380 fiber_pool_stack_alloca(&vacancy->stack, RB_PAGE_SIZE);
381}
382
383inline static struct fiber_pool_vacancy *
384fiber_pool_vacancy_push(struct fiber_pool_vacancy * vacancy, struct fiber_pool_vacancy * head)
385{
386 vacancy->next = head;
387
388#ifdef FIBER_POOL_ALLOCATION_FREE
389 if (head) {
390 head->previous = vacancy;
391 vacancy->previous = NULL;
392 }
393#endif
394
395 return vacancy;
396}
397
398#ifdef FIBER_POOL_ALLOCATION_FREE
399static void
400fiber_pool_vacancy_remove(struct fiber_pool_vacancy * vacancy)
401{
402 if (vacancy->next) {
403 vacancy->next->previous = vacancy->previous;
404 }
405
406 if (vacancy->previous) {
407 vacancy->previous->next = vacancy->next;
408 }
409 else {
410 // It's the head of the list:
411 vacancy->stack.pool->vacancies = vacancy->next;
412 }
413}
414
415inline static struct fiber_pool_vacancy *
416fiber_pool_vacancy_pop(struct fiber_pool * pool)
417{
418 struct fiber_pool_vacancy * vacancy = pool->vacancies;
419
420 if (vacancy) {
421 fiber_pool_vacancy_remove(vacancy);
422 }
423
424 return vacancy;
425}
426#else
427inline static struct fiber_pool_vacancy *
428fiber_pool_vacancy_pop(struct fiber_pool * pool)
429{
430 struct fiber_pool_vacancy * vacancy = pool->vacancies;
431
432 if (vacancy) {
433 pool->vacancies = vacancy->next;
434 }
435
436 return vacancy;
437}
438#endif
439
440// Initialize the vacant stack. The [base, size] allocation should not include the guard page.
441// @param base The pointer to the lowest address of the allocated memory.
442// @param size The size of the allocated memory.
443inline static struct fiber_pool_vacancy *
444fiber_pool_vacancy_initialize(struct fiber_pool * fiber_pool, struct fiber_pool_vacancy * vacancies, void * base, size_t size)
445{
446 struct fiber_pool_vacancy * vacancy = fiber_pool_vacancy_pointer(base, size);
447
448 vacancy->stack.base = base;
449 vacancy->stack.size = size;
450
451 fiber_pool_vacancy_reset(vacancy);
452
453 vacancy->stack.pool = fiber_pool;
454
455 return fiber_pool_vacancy_push(vacancy, vacancies);
456}
457
458// Allocate a maximum of count stacks, size given by stride.
459// @param count the number of stacks to allocate / were allocated.
460// @param stride the size of the individual stacks.
461// @return [void *] the allocated memory or NULL if allocation failed.
462inline static void *
463fiber_pool_allocate_memory(size_t * count, size_t stride)
464{
465 // We use a divide-by-2 strategy to try and allocate memory. We are trying
466 // to allocate `count` stacks. In normal situation, this won't fail. But
467 // if we ran out of address space, or we are allocating more memory than
468 // the system would allow (e.g. overcommit * physical memory + swap), we
469 // divide count by two and try again. This condition should only be
470 // encountered in edge cases, but we handle it here gracefully.
471 while (*count > 1) {
472#if defined(_WIN32)
473 void * base = VirtualAlloc(0, (*count)*stride, MEM_COMMIT, PAGE_READWRITE);
474
475 if (!base) {
476 errno = rb_w32_map_errno(GetLastError());
477 *count = (*count) >> 1;
478 }
479 else {
480 return base;
481 }
482#else
483 errno = 0;
484 size_t mmap_size = (*count)*stride;
485 void * base = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, FIBER_STACK_FLAGS, -1, 0);
486
487 if (base == MAP_FAILED) {
488 // If the allocation fails, count = count / 2, and try again.
489 *count = (*count) >> 1;
490 }
491 else {
492 ruby_annotate_mmap(base, mmap_size, "Ruby:fiber_pool_allocate_memory");
493#if defined(MADV_FREE_REUSE)
494 // On Mac MADV_FREE_REUSE is necessary for the task_info api
495 // to keep the accounting accurate as possible when a page is marked as reusable
496 // it can possibly not occurring at first call thus re-iterating if necessary.
497 while (madvise(base, mmap_size, MADV_FREE_REUSE) == -1 && errno == EAGAIN);
498#endif
499 return base;
500 }
501#endif
502 }
503
504 return NULL;
505}
506
507// Given an existing fiber pool, expand it by the specified number of stacks.
508//
509// @param count the maximum number of stacks to allocate.
510// @return the new allocation on success, or NULL on failure with errno set.
511// @raise NoMemoryError if the struct or memory allocation fails.
512//
513// Call from fiber_pool_stack_acquire_expand with VM lock held, or from
514// fiber_pool_initialize before the pool is shared across threads.
515// @sa fiber_pool_allocation_free
516static struct fiber_pool_allocation *
517fiber_pool_expand(struct fiber_pool * fiber_pool, size_t count)
518{
519 STACK_GROW_DIR_DETECTION;
520
521 size_t size = fiber_pool->size;
522 size_t stride = size + RB_PAGE_SIZE;
523
524 // Allocate metadata before mmap: ruby_xmalloc (RB_ALLOC) raises on failure and
525 // must not run after base is mapped, or the region would leak.
526 struct fiber_pool_allocation * allocation = RB_ALLOC(struct fiber_pool_allocation);
527
528 // Allocate the memory required for the stacks:
529 void * base = fiber_pool_allocate_memory(&count, stride);
530
531 if (base == NULL) {
532 if (!errno) errno = ENOMEM;
533 ruby_xfree(allocation);
534 return NULL;
535 }
536
537 struct fiber_pool_vacancy * vacancies = fiber_pool->vacancies;
538
539 // Initialize fiber pool allocation:
540 allocation->base = base;
541 allocation->size = size;
542 allocation->stride = stride;
543 allocation->count = count;
544#ifdef FIBER_POOL_ALLOCATION_FREE
545 allocation->used = 0;
546#endif
547 allocation->pool = fiber_pool;
548
549 if (DEBUG) {
550 fprintf(stderr, "fiber_pool_expand(%"PRIuSIZE"): %p, %"PRIuSIZE"/%"PRIuSIZE" x [%"PRIuSIZE":%"PRIuSIZE"]\n",
551 count, (void*)fiber_pool, fiber_pool->used, fiber_pool->count, size, fiber_pool->vm_stack_size);
552 }
553
554 // Iterate over all stacks, initializing the vacancy list:
555 for (size_t i = 0; i < count; i += 1) {
556 void * base = (char*)allocation->base + (stride * i);
557 void * page = (char*)base + STACK_DIR_UPPER(size, 0);
558#if defined(_WIN32)
559 DWORD old_protect;
560
561 if (!VirtualProtect(page, RB_PAGE_SIZE, PAGE_READWRITE | PAGE_GUARD, &old_protect)) {
562 int error = rb_w32_map_errno(GetLastError());
563 VirtualFree(allocation->base, 0, MEM_RELEASE);
564 ruby_xfree(allocation);
565 errno = error;
566 return NULL;
567 }
568#elif defined(__wasi__)
569 // wasi-libc's mprotect emulation doesn't support PROT_NONE.
570 (void)page;
571#else
572 if (mprotect(page, RB_PAGE_SIZE, PROT_NONE) < 0) {
573 int error = errno;
574 if (!error) error = ENOMEM;
575 munmap(allocation->base, count*stride);
576 ruby_xfree(allocation);
577 errno = error;
578 return NULL;
579 }
580#endif
581
582 vacancies = fiber_pool_vacancy_initialize(
583 fiber_pool, vacancies,
584 (char*)base + STACK_DIR_UPPER(0, RB_PAGE_SIZE),
585 size
586 );
587
588#ifdef FIBER_POOL_ALLOCATION_FREE
589 vacancies->stack.allocation = allocation;
590#endif
591 }
592
593 // Insert the allocation into the head of the pool:
594 allocation->next = fiber_pool->allocations;
595
596#ifdef FIBER_POOL_ALLOCATION_FREE
597 if (allocation->next) {
598 allocation->next->previous = allocation;
599 }
600
601 allocation->previous = NULL;
602#endif
603
604 fiber_pool->allocations = allocation;
605 fiber_pool->vacancies = vacancies;
606 fiber_pool->count += count;
607
608 return allocation;
609}
610
611// Initialize the specified fiber pool with the given number of stacks.
612// @param vm_stack_size The size of the vm stack to allocate.
613static void
614fiber_pool_initialize(struct fiber_pool * fiber_pool, size_t size, size_t count, size_t vm_stack_size)
615{
616 VM_ASSERT(vm_stack_size < size);
617
618 fiber_pool->allocations = NULL;
619 fiber_pool->vacancies = NULL;
620 fiber_pool->size = ((size / RB_PAGE_SIZE) + 1) * RB_PAGE_SIZE;
621 fiber_pool->count = 0;
622 fiber_pool->initial_count = count;
623 fiber_pool->free_stacks = 1;
624 fiber_pool->used = 0;
625
626 fiber_pool->vm_stack_size = vm_stack_size;
627
628 if (RB_UNLIKELY(!fiber_pool_expand(fiber_pool, count))) {
629 rb_raise(rb_eFiberError, "can't allocate initial fiber stacks (%"PRIuSIZE" x %"PRIuSIZE" bytes): %s", count, fiber_pool->size, strerror(errno));
630 }
631}
632
633#ifdef FIBER_POOL_ALLOCATION_FREE
634// Free the list of fiber pool allocations.
635static void
636fiber_pool_allocation_free(struct fiber_pool_allocation * allocation)
637{
638 STACK_GROW_DIR_DETECTION;
639
640 VM_ASSERT(allocation->used == 0);
641
642 if (DEBUG) fprintf(stderr, "fiber_pool_allocation_free: %p base=%p count=%"PRIuSIZE"\n", (void*)allocation, allocation->base, allocation->count);
643
644 size_t i;
645 for (i = 0; i < allocation->count; i += 1) {
646 void * base = (char*)allocation->base + (allocation->stride * i) + STACK_DIR_UPPER(0, RB_PAGE_SIZE);
647
648 struct fiber_pool_vacancy * vacancy = fiber_pool_vacancy_pointer(base, allocation->size);
649
650 // Pop the vacant stack off the free list:
651 fiber_pool_vacancy_remove(vacancy);
652 }
653
654#ifdef _WIN32
655 VirtualFree(allocation->base, 0, MEM_RELEASE);
656#else
657 munmap(allocation->base, allocation->stride * allocation->count);
658#endif
659
660 if (allocation->previous) {
661 allocation->previous->next = allocation->next;
662 }
663 else {
664 // We are the head of the list, so update the pool:
665 allocation->pool->allocations = allocation->next;
666 }
667
668 if (allocation->next) {
669 allocation->next->previous = allocation->previous;
670 }
671
672 allocation->pool->count -= allocation->count;
673
674 ruby_xfree(allocation);
675}
676#endif
677
678// Number of stacks to request when expanding the pool (clamped to min/max).
679static inline size_t
680fiber_pool_stack_expand_count(const struct fiber_pool *pool)
681{
682 const size_t maximum = FIBER_POOL_ALLOCATION_MAXIMUM_SIZE;
683 const size_t minimum = pool->initial_count;
684
685 size_t count = pool->count;
686 if (count > maximum) count = maximum;
687 if (count < minimum) count = minimum;
688
689 return count;
690}
691
692// When the vacancy list is empty, grow the pool (and run GC only if mmap fails). Caller holds the VM lock.
693// Returns NULL if expansion failed after GC + retry; errno is set. Otherwise returns a vacancy.
694static struct fiber_pool_vacancy *
695fiber_pool_stack_acquire_expand(struct fiber_pool *fiber_pool)
696{
697 size_t count = fiber_pool_stack_expand_count(fiber_pool);
698
699 if (DEBUG) fprintf(stderr, "fiber_pool_stack_acquire: expanding fiber pool by %"PRIuSIZE" stacks\n", count);
700
701 struct fiber_pool_vacancy *vacancy = NULL;
702
703 if (RB_LIKELY(fiber_pool_expand(fiber_pool, count))) {
704 return fiber_pool_vacancy_pop(fiber_pool);
705 }
706 else {
707 if (DEBUG) fprintf(stderr, "fiber_pool_stack_acquire: expand failed (%s), collecting garbage\n", strerror(errno));
708
709 rb_gc();
710
711 // After running GC, the vacancy list may have some stacks:
712 vacancy = fiber_pool_vacancy_pop(fiber_pool);
713 if (RB_LIKELY(vacancy)) {
714 return vacancy;
715 }
716
717 // Try to expand the fiber pool again:
718 if (RB_LIKELY(fiber_pool_expand(fiber_pool, count))) {
719 return fiber_pool_vacancy_pop(fiber_pool);
720 }
721 else {
722 // Okay, we really failed to acquire a stack. Give up and return NULL with errno set:
723 return NULL;
724 }
725 }
726}
727
728// Acquire a stack from the given fiber pool. If none are available, allocate more.
729static struct fiber_pool_stack
730fiber_pool_stack_acquire(struct fiber_pool * fiber_pool)
731{
732 struct fiber_pool_vacancy * vacancy;
733
734 unsigned int lev;
735 RB_VM_LOCK_ENTER_LEV(&lev);
736 {
737 // Fast path: try to acquire a stack from the vacancy list:
738 vacancy = fiber_pool_vacancy_pop(fiber_pool);
739
740 if (DEBUG) fprintf(stderr, "fiber_pool_stack_acquire: %p used=%"PRIuSIZE"\n", (void*)fiber_pool->vacancies, fiber_pool->used);
741
742 // Slow path: If the pool has no vacancies, expand first. Only run GC when expansion fails (e.g. mmap), so we can reclaim stacks from dead fibers before retrying:
743 if (RB_UNLIKELY(!vacancy)) {
744 vacancy = fiber_pool_stack_acquire_expand(fiber_pool);
745
746 // If expansion failed, raise an error:
747 if (RB_UNLIKELY(!vacancy)) {
748 RB_VM_LOCK_LEAVE_LEV(&lev);
749 rb_raise(rb_eFiberError, "can't allocate fiber stack: %s", strerror(errno));
750 }
751 }
752
753 VM_ASSERT(vacancy);
754 VM_ASSERT(vacancy->stack.base);
755
756#if defined(COROUTINE_SANITIZE_ADDRESS)
757 __asan_unpoison_memory_region(fiber_pool_stack_poison_base(&vacancy->stack), fiber_pool_stack_poison_size(&vacancy->stack));
758#endif
759
760 // Take the top item from the free list:
761 fiber_pool->used += 1;
762
763#ifdef FIBER_POOL_ALLOCATION_FREE
764 vacancy->stack.allocation->used += 1;
765#endif
766
767 fiber_pool_stack_reset(&vacancy->stack);
768 }
769 RB_VM_LOCK_LEAVE_LEV(&lev);
770
771 return vacancy->stack;
772}
773
774// We advise the operating system that the stack memory pages are no longer being used.
775// This introduce some performance overhead but allows system to relaim memory when there is pressure.
776static inline void
777fiber_pool_stack_free(struct fiber_pool_stack * stack)
778{
779 void * base = fiber_pool_stack_base(stack);
780 size_t size = stack->available;
781
782 // If this is not true, the vacancy information will almost certainly be destroyed:
783 VM_ASSERT(size <= (stack->size - RB_PAGE_SIZE));
784
785 int advice = stack->pool->free_stacks >> 1;
786
787 if (DEBUG) fprintf(stderr, "fiber_pool_stack_free: %p+%"PRIuSIZE" [base=%p, size=%"PRIuSIZE"] advice=%d\n", base, size, stack->base, stack->size, advice);
788
789 // The pages being used by the stack can be returned back to the system.
790 // That doesn't change the page mapping, but it does allow the system to
791 // reclaim the physical memory.
792 // Since we no longer care about the data itself, we don't need to page
793 // out to disk, since that is costly. Not all systems support that, so
794 // we try our best to select the most efficient implementation.
795 // In addition, it's actually slightly desirable to not do anything here,
796 // but that results in higher memory usage.
797
798#ifdef __wasi__
799 // WebAssembly doesn't support madvise, so we just don't do anything.
800#elif VM_CHECK_MODE > 0 && defined(MADV_DONTNEED)
801 if (!advice) advice = MADV_DONTNEED;
802 // This immediately discards the pages and the memory is reset to zero.
803 madvise(base, size, advice);
804#elif defined(MADV_FREE_REUSABLE)
805 if (!advice) advice = MADV_FREE_REUSABLE;
806 // Darwin / macOS / iOS.
807 // Acknowledge the kernel down to the task info api we make this
808 // page reusable for future use.
809 // As for MADV_FREE_REUSABLE below we ensure in the rare occasions the task was not
810 // completed at the time of the call to re-iterate.
811 while (madvise(base, size, advice) == -1 && errno == EAGAIN);
812#elif defined(MADV_FREE)
813 if (!advice) advice = MADV_FREE;
814 // Recent Linux.
815 madvise(base, size, advice);
816#elif defined(MADV_DONTNEED)
817 if (!advice) advice = MADV_DONTNEED;
818 // Old Linux.
819 madvise(base, size, advice);
820#elif defined(POSIX_MADV_DONTNEED)
821 if (!advice) advice = POSIX_MADV_DONTNEED;
822 // Solaris?
823 posix_madvise(base, size, advice);
824#elif defined(_WIN32)
825 VirtualAlloc(base, size, MEM_RESET, PAGE_READWRITE);
826 // Not available in all versions of Windows.
827 //DiscardVirtualMemory(base, size);
828#endif
829
830#if defined(COROUTINE_SANITIZE_ADDRESS)
831 __asan_poison_memory_region(fiber_pool_stack_poison_base(stack), fiber_pool_stack_poison_size(stack));
832#endif
833}
834
835// Release and return a stack to the vacancy list.
836static void
837fiber_pool_stack_release(struct fiber_pool_stack * stack)
838{
839 struct fiber_pool * pool = stack->pool;
840 struct fiber_pool_vacancy * vacancy = fiber_pool_vacancy_pointer(stack->base, stack->size);
841
842 if (DEBUG) fprintf(stderr, "fiber_pool_stack_release: %p used=%"PRIuSIZE"\n", stack->base, stack->pool->used);
843
844 // Copy the stack details into the vacancy area:
845 vacancy->stack = *stack;
846 // After this point, be careful about updating/using state in stack, since it's copied to the vacancy area.
847
848 // Reset the stack pointers and reserve space for the vacancy data:
849 fiber_pool_vacancy_reset(vacancy);
850
851 // Push the vacancy into the vancancies list:
852 pool->vacancies = fiber_pool_vacancy_push(vacancy, pool->vacancies);
853 pool->used -= 1;
854
855#ifdef FIBER_POOL_ALLOCATION_FREE
856 struct fiber_pool_allocation * allocation = stack->allocation;
857
858 allocation->used -= 1;
859
860 // Release address space and/or dirty memory:
861 if (allocation->used == 0) {
862 fiber_pool_allocation_free(allocation);
863 }
864 else if (stack->pool->free_stacks) {
865 fiber_pool_stack_free(&vacancy->stack);
866 }
867#else
868 // This is entirely optional, but clears the dirty flag from the stack
869 // memory, so it won't get swapped to disk when there is memory pressure:
870 if (stack->pool->free_stacks) {
871 fiber_pool_stack_free(&vacancy->stack);
872 }
873#endif
874}
875
876static inline void
877ec_switch(rb_thread_t *th, rb_fiber_t *fiber)
878{
879 rb_execution_context_t *ec = &fiber->cont.saved_ec;
880#ifdef RUBY_ASAN_ENABLED
881 ec->machine.asan_fake_stack_handle = asan_get_thread_fake_stack_handle();
882#endif
883 rb_ractor_set_current_ec(th->ractor, th->ec = ec);
884 // ruby_current_execution_context_ptr = th->ec = ec;
885
886 /*
887 * timer-thread may set trap interrupt on previous th->ec at any time;
888 * ensure we do not delay (or lose) the trap interrupt handling.
889 */
890 if (th->vm->ractor.main_thread == th &&
891 rb_signal_buff_size() > 0) {
892 RUBY_VM_SET_TRAP_INTERRUPT(ec);
893 }
894
895 VM_ASSERT(ec->fiber_ptr->cont.self == 0 || ec->vm_stack != NULL);
896}
897
898static inline void
899fiber_restore_thread(rb_thread_t *th, rb_fiber_t *fiber)
900{
901 ec_switch(th, fiber);
902 VM_ASSERT(th->ec->fiber_ptr == fiber);
903}
904
905#ifndef COROUTINE_DECL
906# define COROUTINE_DECL COROUTINE
907#endif
908NORETURN(static COROUTINE_DECL fiber_entry(struct coroutine_context * from, struct coroutine_context * to));
909static COROUTINE
910fiber_entry(struct coroutine_context * from, struct coroutine_context * to)
911{
912 rb_fiber_t *fiber = to->argument;
913
914#if defined(COROUTINE_SANITIZE_ADDRESS)
915 // Address sanitizer will copy the previous stack base and stack size into
916 // the "from" fiber. `coroutine_initialize_main` doesn't generally know the
917 // stack bounds (base + size). Therefore, the main fiber `stack_base` and
918 // `stack_size` will be NULL/0. It's specifically important in that case to
919 // get the (base+size) of the previous fiber and save it, so that later when
920 // we return to the main coroutine, we don't supply (NULL, 0) to
921 // __sanitizer_start_switch_fiber which royally messes up the internal state
922 // of ASAN and causes (sometimes) the following message:
923 // "WARNING: ASan is ignoring requested __asan_handle_no_return"
924 __sanitizer_finish_switch_fiber(to->fake_stack, (const void**)&from->stack_base, &from->stack_size);
925#endif
926
927 rb_thread_t *thread = fiber->cont.saved_ec.thread_ptr;
928
929#ifdef COROUTINE_PTHREAD_CONTEXT
930 ruby_thread_set_native(thread);
931#endif
932
933 fiber_restore_thread(thread, fiber);
934
935 rb_fiber_start(fiber);
936
937#ifndef COROUTINE_PTHREAD_CONTEXT
938 VM_UNREACHABLE(fiber_entry);
939#endif
940}
941
942// Initialize a fiber's coroutine's machine stack and vm stack.
943static VALUE *
944fiber_initialize_coroutine(rb_fiber_t *fiber, size_t * vm_stack_size)
945{
946 struct fiber_pool * fiber_pool = fiber->stack.pool;
947 rb_execution_context_t *sec = &fiber->cont.saved_ec;
948 void * vm_stack = NULL;
949
950 VM_ASSERT(fiber_pool != NULL);
951
952 fiber->stack = fiber_pool_stack_acquire(fiber_pool);
953 vm_stack = fiber_pool_stack_alloca(&fiber->stack, fiber_pool->vm_stack_size);
954 *vm_stack_size = fiber_pool->vm_stack_size;
955
956 coroutine_initialize(&fiber->context, fiber_entry, fiber_pool_stack_base(&fiber->stack), fiber->stack.available);
957
958 // The stack for this execution context is the one we allocated:
959 sec->machine.stack_start = fiber->stack.current;
960 sec->machine.stack_maxsize = fiber->stack.available;
961
962 fiber->context.argument = (void*)fiber;
963
964 return vm_stack;
965}
966
967// Release the stack from the fiber, it's execution context, and return it to
968// the fiber pool.
969static void
970fiber_stack_release(rb_fiber_t * fiber)
971{
972 rb_execution_context_t *ec = &fiber->cont.saved_ec;
973
974 if (DEBUG) fprintf(stderr, "fiber_stack_release: %p, stack.base=%p\n", (void*)fiber, fiber->stack.base);
975
976 // Return the stack back to the fiber pool if it wasn't already:
977 if (fiber->stack.base) {
978 fiber_pool_stack_release(&fiber->stack);
979 fiber->stack.base = NULL;
980 }
981
982 // The stack is no longer associated with this execution context:
983 rb_ec_clear_vm_stack(ec);
984}
985
986static void
987fiber_stack_release_locked(rb_fiber_t *fiber)
988{
989 if (!ruby_vm_during_cleanup) {
990 // We can't try to acquire the VM lock here because MMTK calls free in its own native thread which has no ec.
991 // This assertion will fail on MMTK but we currently don't have CI for debug releases of MMTK, so we can assert for now.
992 ASSERT_vm_locking_with_barrier();
993 }
994 fiber_stack_release(fiber);
995}
996
997static const char *
998fiber_status_name(enum fiber_status s)
999{
1000 switch (s) {
1001 case FIBER_CREATED: return "created";
1002 case FIBER_RESUMED: return "resumed";
1003 case FIBER_SUSPENDED: return "suspended";
1004 case FIBER_TERMINATED: return "terminated";
1005 }
1006 VM_UNREACHABLE(fiber_status_name);
1007 return NULL;
1008}
1009
1010static void
1011fiber_verify(const rb_fiber_t *fiber)
1012{
1013#if VM_CHECK_MODE > 0
1014 VM_ASSERT(fiber->cont.saved_ec.fiber_ptr == fiber);
1015
1016 switch (fiber->status) {
1017 case FIBER_RESUMED:
1018 VM_ASSERT(fiber->cont.saved_ec.vm_stack != NULL);
1019 break;
1020 case FIBER_SUSPENDED:
1021 VM_ASSERT(fiber->cont.saved_ec.vm_stack != NULL);
1022 break;
1023 case FIBER_CREATED:
1024 case FIBER_TERMINATED:
1025 /* TODO */
1026 break;
1027 default:
1028 VM_UNREACHABLE(fiber_verify);
1029 }
1030#endif
1031}
1032
1033inline static void
1034fiber_status_set(rb_fiber_t *fiber, enum fiber_status s)
1035{
1036 // if (DEBUG) fprintf(stderr, "fiber: %p, status: %s -> %s\n", (void *)fiber, fiber_status_name(fiber->status), fiber_status_name(s));
1037 VM_ASSERT(!FIBER_TERMINATED_P(fiber));
1038 VM_ASSERT(fiber->status != s);
1039 fiber_verify(fiber);
1040 fiber->status = s;
1041}
1042
1043static rb_context_t *
1044cont_ptr(VALUE obj)
1045{
1046 rb_context_t *cont;
1047
1048 TypedData_Get_Struct(obj, rb_context_t, &cont_data_type, cont);
1049
1050 return cont;
1051}
1052
1053static rb_fiber_t *
1054fiber_ptr(VALUE obj)
1055{
1056 rb_fiber_t *fiber;
1057
1058 TypedData_Get_Struct(obj, rb_fiber_t, &fiber_data_type, fiber);
1059 if (!fiber) rb_raise(rb_eFiberError, "uninitialized fiber");
1060
1061 return fiber;
1062}
1063
1064NOINLINE(static VALUE cont_capture(volatile int *volatile stat));
1065
1066#define THREAD_MUST_BE_RUNNING(th) do { \
1067 if (!(th)->ec->tag) rb_raise(rb_eThreadError, "not running thread"); \
1068 } while (0)
1069
1070rb_thread_t*
1071rb_fiber_threadptr(const rb_fiber_t *fiber)
1072{
1073 return fiber->cont.saved_ec.thread_ptr;
1074}
1075
1076static VALUE
1077cont_thread_value(const rb_context_t *cont)
1078{
1079 return cont->saved_ec.thread_ptr->self;
1080}
1081
1082static void
1083cont_compact(void *ptr)
1084{
1085 rb_context_t *cont = ptr;
1086
1087 if (cont->self) {
1088 cont->self = rb_gc_location(cont->self);
1089 }
1090 cont->value = rb_gc_location(cont->value);
1091 rb_execution_context_update(&cont->saved_ec);
1092}
1093
1094static void
1095cont_mark(void *ptr)
1096{
1097 rb_context_t *cont = ptr;
1098
1099 RUBY_MARK_ENTER("cont");
1100 if (cont->self) {
1101 rb_gc_mark_movable(cont->self);
1102 }
1103 rb_gc_mark_movable(cont->value);
1104
1105 rb_execution_context_mark(&cont->saved_ec);
1106 rb_gc_mark(cont_thread_value(cont));
1107
1108 if (cont->saved_vm_stack.ptr) {
1109#ifdef CAPTURE_JUST_VALID_VM_STACK
1110 rb_gc_mark_locations(cont->saved_vm_stack.ptr,
1111 cont->saved_vm_stack.ptr + cont->saved_vm_stack.slen + cont->saved_vm_stack.clen);
1112#else
1113 rb_gc_mark_locations(cont->saved_vm_stack.ptr,
1114 cont->saved_vm_stack.ptr, cont->saved_ec.stack_size);
1115#endif
1116 }
1117
1118 if (cont->machine.stack) {
1119 if (cont->type == CONTINUATION_CONTEXT) {
1120 /* cont */
1121 rb_gc_mark_locations(cont->machine.stack,
1122 cont->machine.stack + cont->machine.stack_size);
1123 }
1124 else {
1125 /* fiber machine context is marked as part of rb_execution_context_mark, no need to
1126 * do anything here. */
1127 }
1128 }
1129
1130 RUBY_MARK_LEAVE("cont");
1131}
1132
1133#if 0
1134static int
1135fiber_is_root_p(const rb_fiber_t *fiber)
1136{
1137 return fiber == fiber->cont.saved_ec.thread_ptr->root_fiber;
1138}
1139#endif
1140
1141static void jit_cont_free(struct rb_jit_cont *cont);
1142
1143static void
1144cont_free(void *ptr)
1145{
1146 rb_context_t *cont = ptr;
1147
1148 RUBY_FREE_ENTER("cont");
1149
1150 if (cont->type == CONTINUATION_CONTEXT) {
1151 ruby_xfree(cont->saved_ec.vm_stack);
1152 RUBY_FREE_UNLESS_NULL(cont->machine.stack);
1153 }
1154 else {
1155 rb_fiber_t *fiber = (rb_fiber_t*)cont;
1156 coroutine_destroy(&fiber->context);
1157 fiber_stack_release_locked(fiber);
1158 }
1159
1160 RUBY_FREE_UNLESS_NULL(cont->saved_vm_stack.ptr);
1161
1162 VM_ASSERT(cont->jit_cont != NULL);
1163 jit_cont_free(cont->jit_cont);
1164 /* free rb_cont_t or rb_fiber_t */
1165 ruby_xfree(ptr);
1166 RUBY_FREE_LEAVE("cont");
1167}
1168
1169static size_t
1170cont_memsize(const void *ptr)
1171{
1172 const rb_context_t *cont = ptr;
1173 size_t size = 0;
1174
1175 size = sizeof(*cont);
1176 if (cont->saved_vm_stack.ptr) {
1177#ifdef CAPTURE_JUST_VALID_VM_STACK
1178 size_t n = (cont->saved_vm_stack.slen + cont->saved_vm_stack.clen);
1179#else
1180 size_t n = cont->saved_ec.vm_stack_size;
1181#endif
1182 size += n * sizeof(*cont->saved_vm_stack.ptr);
1183 }
1184
1185 if (cont->machine.stack) {
1186 size += cont->machine.stack_size * sizeof(*cont->machine.stack);
1187 }
1188
1189 return size;
1190}
1191
1192void
1193rb_fiber_update_self(rb_fiber_t *fiber)
1194{
1195 if (fiber->cont.self) {
1196 fiber->cont.self = rb_gc_location(fiber->cont.self);
1197 }
1198 else {
1199 rb_execution_context_update(&fiber->cont.saved_ec);
1200 }
1201}
1202
1203void
1204rb_fiber_mark_self(const rb_fiber_t *fiber)
1205{
1206 if (fiber->cont.self) {
1207 rb_gc_mark_movable(fiber->cont.self);
1208 }
1209 else {
1210 rb_execution_context_mark(&fiber->cont.saved_ec);
1211 }
1212}
1213
1214static void
1215fiber_compact(void *ptr)
1216{
1217 rb_fiber_t *fiber = ptr;
1218 fiber->first_proc = rb_gc_location(fiber->first_proc);
1219
1220 if (fiber->prev) rb_fiber_update_self(fiber->prev);
1221
1222 cont_compact(&fiber->cont);
1223 fiber_verify(fiber);
1224}
1225
1226static void
1227fiber_mark(void *ptr)
1228{
1229 rb_fiber_t *fiber = ptr;
1230 RUBY_MARK_ENTER("cont");
1231 fiber_verify(fiber);
1232 rb_gc_mark_movable(fiber->first_proc);
1233 if (fiber->prev) rb_fiber_mark_self(fiber->prev);
1234 cont_mark(&fiber->cont);
1235 RUBY_MARK_LEAVE("cont");
1236}
1237
1238static void
1239fiber_free(void *ptr)
1240{
1241 rb_fiber_t *fiber = ptr;
1242 RUBY_FREE_ENTER("fiber");
1243
1244 if (DEBUG) fprintf(stderr, "fiber_free: %p[%p]\n", (void *)fiber, fiber->stack.base);
1245
1246 if (fiber->cont.saved_ec.local_storage) {
1247 rb_id_table_free(fiber->cont.saved_ec.local_storage);
1248 }
1249
1250 cont_free(&fiber->cont);
1251 RUBY_FREE_LEAVE("fiber");
1252}
1253
1254static size_t
1255fiber_memsize(const void *ptr)
1256{
1257 const rb_fiber_t *fiber = ptr;
1258 size_t size = sizeof(*fiber);
1259 const rb_execution_context_t *saved_ec = &fiber->cont.saved_ec;
1260 const rb_thread_t *th = rb_ec_thread_ptr(saved_ec);
1261
1262 /*
1263 * vm.c::thread_memsize already counts th->ec->local_storage
1264 */
1265 if (saved_ec->local_storage && fiber != th->root_fiber) {
1266 size += rb_id_table_memsize(saved_ec->local_storage);
1267 size += rb_obj_memsize_of(saved_ec->storage);
1268 }
1269
1270 size += cont_memsize(&fiber->cont);
1271 return size;
1272}
1273
1274 VALUE
1275rb_obj_is_fiber(VALUE obj)
1276{
1277 return RBOOL(rb_typeddata_is_kind_of(obj, &fiber_data_type));
1278}
1279
1280static void
1281cont_save_machine_stack(rb_thread_t *th, rb_context_t *cont)
1282{
1283 size_t size;
1284
1285 SET_MACHINE_STACK_END(&th->ec->machine.stack_end);
1286
1287 if (th->ec->machine.stack_start > th->ec->machine.stack_end) {
1288 size = cont->machine.stack_size = th->ec->machine.stack_start - th->ec->machine.stack_end;
1289 cont->machine.stack_src = th->ec->machine.stack_end;
1290 }
1291 else {
1292 size = cont->machine.stack_size = th->ec->machine.stack_end - th->ec->machine.stack_start;
1293 cont->machine.stack_src = th->ec->machine.stack_start;
1294 }
1295
1296 if (cont->machine.stack) {
1297 REALLOC_N(cont->machine.stack, VALUE, size);
1298 }
1299 else {
1300 cont->machine.stack = ALLOC_N(VALUE, size);
1301 }
1302
1303 FLUSH_REGISTER_WINDOWS;
1304 asan_unpoison_memory_region(cont->machine.stack_src, size, false);
1305 MEMCPY(cont->machine.stack, cont->machine.stack_src, VALUE, size);
1306}
1307
1308static const rb_data_type_t cont_data_type = {
1309 "continuation",
1310 {cont_mark, cont_free, cont_memsize, cont_compact},
1311 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
1312};
1313
1314static inline void
1315cont_save_thread(rb_context_t *cont, rb_thread_t *th)
1316{
1317 rb_execution_context_t *sec = &cont->saved_ec;
1318
1319 VM_ASSERT(th->status == THREAD_RUNNABLE);
1320
1321 /* save thread context */
1322 *sec = *th->ec;
1323
1324 /* saved_ec->machine.stack_end should be NULL */
1325 /* because it may happen GC afterward */
1326 sec->machine.stack_end = NULL;
1327}
1328
1329static rb_nativethread_lock_t jit_cont_lock;
1330
1331// Register a new continuation with execution context `ec`. Return JIT info about
1332// the continuation.
1333static struct rb_jit_cont *
1334jit_cont_new(rb_execution_context_t *ec)
1335{
1336 struct rb_jit_cont *cont;
1337
1338 // We need to use calloc instead of something like ZALLOC to avoid triggering GC here.
1339 // When this function is called from rb_thread_alloc through rb_threadptr_root_fiber_setup,
1340 // the thread is still being prepared and marking it causes SEGV.
1341 cont = calloc(1, sizeof(struct rb_jit_cont));
1342 if (cont == NULL)
1343 rb_memerror();
1344 cont->ec = ec;
1345
1346 rb_native_mutex_lock(&jit_cont_lock);
1347 if (first_jit_cont == NULL) {
1348 cont->next = cont->prev = NULL;
1349 }
1350 else {
1351 cont->prev = NULL;
1352 cont->next = first_jit_cont;
1353 first_jit_cont->prev = cont;
1354 }
1355 first_jit_cont = cont;
1356 rb_native_mutex_unlock(&jit_cont_lock);
1357
1358 return cont;
1359}
1360
1361// Unregister continuation `cont`.
1362static void
1363jit_cont_free(struct rb_jit_cont *cont)
1364{
1365 if (!cont) return;
1366
1367 rb_native_mutex_lock(&jit_cont_lock);
1368 if (cont == first_jit_cont) {
1369 first_jit_cont = cont->next;
1370 if (first_jit_cont != NULL)
1371 first_jit_cont->prev = NULL;
1372 }
1373 else {
1374 cont->prev->next = cont->next;
1375 if (cont->next != NULL)
1376 cont->next->prev = cont->prev;
1377 }
1378 rb_native_mutex_unlock(&jit_cont_lock);
1379
1380 free(cont);
1381}
1382
1383// Call a given callback against all on-stack ISEQs.
1384void
1385rb_jit_cont_each_iseq(rb_iseq_callback callback, void *data)
1386{
1387 struct rb_jit_cont *cont;
1388 for (cont = first_jit_cont; cont != NULL; cont = cont->next) {
1389 if (cont->ec->vm_stack == NULL)
1390 continue;
1391
1392 const rb_control_frame_t *cfp = cont->ec->cfp;
1393 while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) {
1394 if (cfp->pc && cfp->iseq && imemo_type((VALUE)cfp->iseq) == imemo_iseq) {
1395 callback(cfp->iseq, data);
1396 }
1397 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1398 }
1399 }
1400}
1401
1402#if USE_YJIT
1403// Update the jit_return of all CFPs to leave_exit unless it's leave_exception or not set.
1404// This prevents jit_exec_exception from jumping to the caller after invalidation.
1405void
1406rb_yjit_cancel_jit_return(void *leave_exit, void *leave_exception)
1407{
1408 struct rb_jit_cont *cont;
1409 for (cont = first_jit_cont; cont != NULL; cont = cont->next) {
1410 if (cont->ec->vm_stack == NULL)
1411 continue;
1412
1413 const rb_control_frame_t *cfp = cont->ec->cfp;
1414 while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) {
1415 if (cfp->jit_return && cfp->jit_return != leave_exception) {
1416 ((rb_control_frame_t *)cfp)->jit_return = leave_exit;
1417 }
1418 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1419 }
1420 }
1421}
1422#endif
1423
1424// Finish working with jit_cont.
1425void
1426rb_jit_cont_finish(void)
1427{
1428 struct rb_jit_cont *cont, *next;
1429 for (cont = first_jit_cont; cont != NULL; cont = next) {
1430 next = cont->next;
1431 free(cont); // Don't use xfree because it's allocated by calloc.
1432 }
1433 rb_native_mutex_destroy(&jit_cont_lock);
1434}
1435
1436static void
1437cont_init_jit_cont(rb_context_t *cont)
1438{
1439 VM_ASSERT(cont->jit_cont == NULL);
1440 // We always allocate this since YJIT may be enabled later
1441 cont->jit_cont = jit_cont_new(&(cont->saved_ec));
1442}
1443
1444struct rb_execution_context_struct *
1445rb_fiberptr_get_ec(struct rb_fiber_struct *fiber)
1446{
1447 return &fiber->cont.saved_ec;
1448}
1449
1450static void
1451cont_init(rb_context_t *cont, rb_thread_t *th)
1452{
1453 /* save thread context */
1454 cont_save_thread(cont, th);
1455 cont->saved_ec.thread_ptr = th;
1456 cont->saved_ec.local_storage = NULL;
1457 cont->saved_ec.local_storage_recursive_hash = Qnil;
1458 cont->saved_ec.local_storage_recursive_hash_for_trace = Qnil;
1459 cont_init_jit_cont(cont);
1460}
1461
1462static rb_context_t *
1463cont_new(VALUE klass)
1464{
1465 rb_context_t *cont;
1466 volatile VALUE contval;
1467 rb_thread_t *th = GET_THREAD();
1468
1469 THREAD_MUST_BE_RUNNING(th);
1470 contval = TypedData_Make_Struct(klass, rb_context_t, &cont_data_type, cont);
1471 cont->self = contval;
1472 cont_init(cont, th);
1473 return cont;
1474}
1475
1476 VALUE
1477rb_fiberptr_self(struct rb_fiber_struct *fiber)
1478{
1479 return fiber->cont.self;
1480}
1481
1482unsigned int
1483rb_fiberptr_blocking(struct rb_fiber_struct *fiber)
1484{
1485 return fiber->blocking;
1486}
1487
1488// Initialize the jit_cont_lock
1489void
1490rb_jit_cont_init(void)
1491{
1492 rb_native_mutex_initialize(&jit_cont_lock);
1493}
1494
1495#if 0
1496void
1497show_vm_stack(const rb_execution_context_t *ec)
1498{
1499 VALUE *p = ec->vm_stack;
1500 while (p < ec->cfp->sp) {
1501 fprintf(stderr, "%3d ", (int)(p - ec->vm_stack));
1502 rb_obj_info_dump(*p);
1503 p++;
1504 }
1505}
1506
1507void
1508show_vm_pcs(const rb_control_frame_t *cfp,
1509 const rb_control_frame_t *end_of_cfp)
1510{
1511 int i=0;
1512 while (cfp != end_of_cfp) {
1513 int pc = 0;
1514 if (cfp->iseq) {
1515 pc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded;
1516 }
1517 fprintf(stderr, "%2d pc: %d\n", i++, pc);
1518 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
1519 }
1520}
1521#endif
1522
1523static VALUE
1524cont_capture(volatile int *volatile stat)
1525{
1526 rb_context_t *volatile cont;
1527 rb_thread_t *th = GET_THREAD();
1528 volatile VALUE contval;
1529 const rb_execution_context_t *ec = th->ec;
1530
1531 THREAD_MUST_BE_RUNNING(th);
1532 rb_vm_stack_to_heap(th->ec);
1533 cont = cont_new(rb_cContinuation);
1534 contval = cont->self;
1535
1536#ifdef CAPTURE_JUST_VALID_VM_STACK
1537 cont->saved_vm_stack.slen = ec->cfp->sp - ec->vm_stack;
1538 cont->saved_vm_stack.clen = ec->vm_stack + ec->vm_stack_size - (VALUE*)ec->cfp;
1539 cont->saved_vm_stack.ptr = ALLOC_N(VALUE, cont->saved_vm_stack.slen + cont->saved_vm_stack.clen);
1540 MEMCPY(cont->saved_vm_stack.ptr,
1541 ec->vm_stack,
1542 VALUE, cont->saved_vm_stack.slen);
1543 MEMCPY(cont->saved_vm_stack.ptr + cont->saved_vm_stack.slen,
1544 (VALUE*)ec->cfp,
1545 VALUE,
1546 cont->saved_vm_stack.clen);
1547#else
1548 cont->saved_vm_stack.ptr = ALLOC_N(VALUE, ec->vm_stack_size);
1549 MEMCPY(cont->saved_vm_stack.ptr, ec->vm_stack, VALUE, ec->vm_stack_size);
1550#endif
1551 // At this point, `cfp` is valid but `vm_stack` should be cleared:
1552 rb_ec_set_vm_stack(&cont->saved_ec, NULL, 0);
1553 VM_ASSERT(cont->saved_ec.cfp != NULL);
1554 cont_save_machine_stack(th, cont);
1555
1556 if (ruby_setjmp(cont->jmpbuf)) {
1557 VALUE value;
1558
1559 VAR_INITIALIZED(cont);
1560 value = cont->value;
1561 if (cont->argc == -1) rb_exc_raise(value);
1562 cont->value = Qnil;
1563 *stat = 1;
1564 return value;
1565 }
1566 else {
1567 *stat = 0;
1568 return contval;
1569 }
1570}
1571
1572static inline void
1573cont_restore_thread(rb_context_t *cont)
1574{
1575 rb_thread_t *th = GET_THREAD();
1576
1577 /* restore thread context */
1578 if (cont->type == CONTINUATION_CONTEXT) {
1579 /* continuation */
1580 rb_execution_context_t *sec = &cont->saved_ec;
1581 rb_fiber_t *fiber = NULL;
1582
1583 if (sec->fiber_ptr != NULL) {
1584 fiber = sec->fiber_ptr;
1585 }
1586 else if (th->root_fiber) {
1587 fiber = th->root_fiber;
1588 }
1589
1590 if (fiber && th->ec != &fiber->cont.saved_ec) {
1591 ec_switch(th, fiber);
1592 }
1593
1594 if (th->ec->trace_arg != sec->trace_arg) {
1595 rb_raise(rb_eRuntimeError, "can't call across trace_func");
1596 }
1597
1598#if defined(__wasm__) && !defined(__EMSCRIPTEN__)
1599 if (th->ec->tag != sec->tag) {
1600 /* find the lowest common ancestor tag of the current EC and the saved EC */
1601
1602 struct rb_vm_tag *lowest_common_ancestor = NULL;
1603 size_t num_tags = 0;
1604 size_t num_saved_tags = 0;
1605 for (struct rb_vm_tag *tag = th->ec->tag; tag != NULL; tag = tag->prev) {
1606 ++num_tags;
1607 }
1608 for (struct rb_vm_tag *tag = sec->tag; tag != NULL; tag = tag->prev) {
1609 ++num_saved_tags;
1610 }
1611
1612 size_t min_tags = num_tags <= num_saved_tags ? num_tags : num_saved_tags;
1613
1614 struct rb_vm_tag *tag = th->ec->tag;
1615 while (num_tags > min_tags) {
1616 tag = tag->prev;
1617 --num_tags;
1618 }
1619
1620 struct rb_vm_tag *saved_tag = sec->tag;
1621 while (num_saved_tags > min_tags) {
1622 saved_tag = saved_tag->prev;
1623 --num_saved_tags;
1624 }
1625
1626 while (min_tags > 0) {
1627 if (tag == saved_tag) {
1628 lowest_common_ancestor = tag;
1629 break;
1630 }
1631 tag = tag->prev;
1632 saved_tag = saved_tag->prev;
1633 --min_tags;
1634 }
1635
1636 /* free all the jump buffers between the current EC's tag and the lowest common ancestor tag */
1637 for (struct rb_vm_tag *tag = th->ec->tag; tag != lowest_common_ancestor; tag = tag->prev) {
1638 rb_vm_tag_jmpbuf_deinit(&tag->buf);
1639 }
1640 }
1641#endif
1642
1643 /* copy vm stack */
1644#ifdef CAPTURE_JUST_VALID_VM_STACK
1645 MEMCPY(th->ec->vm_stack,
1646 cont->saved_vm_stack.ptr,
1647 VALUE, cont->saved_vm_stack.slen);
1648 MEMCPY(th->ec->vm_stack + th->ec->vm_stack_size - cont->saved_vm_stack.clen,
1649 cont->saved_vm_stack.ptr + cont->saved_vm_stack.slen,
1650 VALUE, cont->saved_vm_stack.clen);
1651#else
1652 MEMCPY(th->ec->vm_stack, cont->saved_vm_stack.ptr, VALUE, sec->vm_stack_size);
1653#endif
1654 /* other members of ec */
1655
1656 th->ec->cfp = sec->cfp;
1657 th->ec->raised_flag = sec->raised_flag;
1658 th->ec->tag = sec->tag;
1659 th->ec->root_lep = sec->root_lep;
1660 th->ec->root_svar = sec->root_svar;
1661 th->ec->errinfo = sec->errinfo;
1662
1663 VM_ASSERT(th->ec->vm_stack != NULL);
1664 }
1665 else {
1666 /* fiber */
1667 fiber_restore_thread(th, (rb_fiber_t*)cont);
1668 }
1669}
1670
1671NOINLINE(static void fiber_setcontext(rb_fiber_t *new_fiber, rb_fiber_t *old_fiber));
1672
1673static void
1674fiber_setcontext(rb_fiber_t *new_fiber, rb_fiber_t *old_fiber)
1675{
1676 rb_thread_t *th = GET_THREAD();
1677
1678 /* save old_fiber's machine stack - to ensure efficient garbage collection */
1679 if (!FIBER_TERMINATED_P(old_fiber)) {
1680 STACK_GROW_DIR_DETECTION;
1681 SET_MACHINE_STACK_END(&th->ec->machine.stack_end);
1682 if (STACK_DIR_UPPER(0, 1)) {
1683 old_fiber->cont.machine.stack_size = th->ec->machine.stack_start - th->ec->machine.stack_end;
1684 old_fiber->cont.machine.stack = th->ec->machine.stack_end;
1685 }
1686 else {
1687 old_fiber->cont.machine.stack_size = th->ec->machine.stack_end - th->ec->machine.stack_start;
1688 old_fiber->cont.machine.stack = th->ec->machine.stack_start;
1689 }
1690 }
1691
1692 /* these values are used in rb_gc_mark_machine_context to mark the fiber's stack. */
1693 old_fiber->cont.saved_ec.machine.stack_start = th->ec->machine.stack_start;
1694 old_fiber->cont.saved_ec.machine.stack_end = FIBER_TERMINATED_P(old_fiber) ? NULL : th->ec->machine.stack_end;
1695
1696
1697 // if (DEBUG) fprintf(stderr, "fiber_setcontext: %p[%p] -> %p[%p]\n", (void*)old_fiber, old_fiber->stack.base, (void*)new_fiber, new_fiber->stack.base);
1698
1699#if defined(COROUTINE_SANITIZE_ADDRESS)
1700 __sanitizer_start_switch_fiber(FIBER_TERMINATED_P(old_fiber) ? NULL : &old_fiber->context.fake_stack, new_fiber->context.stack_base, new_fiber->context.stack_size);
1701#endif
1702
1703 /* swap machine context */
1704 struct coroutine_context * from = coroutine_transfer(&old_fiber->context, &new_fiber->context);
1705
1706#if defined(COROUTINE_SANITIZE_ADDRESS)
1707 __sanitizer_finish_switch_fiber(old_fiber->context.fake_stack, NULL, NULL);
1708#endif
1709
1710 if (from == NULL) {
1711 rb_syserr_fail(errno, "coroutine_transfer");
1712 }
1713
1714 /* restore thread context */
1715 fiber_restore_thread(th, old_fiber);
1716
1717 // It's possible to get here, and new_fiber is already freed.
1718 // if (DEBUG) fprintf(stderr, "fiber_setcontext: %p[%p] <- %p[%p]\n", (void*)old_fiber, old_fiber->stack.base, (void*)new_fiber, new_fiber->stack.base);
1719}
1720
1721NOINLINE(NORETURN(static void cont_restore_1(rb_context_t *)));
1722
1723static void
1724cont_restore_1(rb_context_t *cont)
1725{
1726 cont_restore_thread(cont);
1727
1728 /* restore machine stack */
1729#if (defined(_M_AMD64) && !defined(__MINGW64__)) || defined(_M_ARM64)
1730 {
1731 /* workaround for x64 and arm64 SEH on Windows */
1732 jmp_buf buf;
1733 setjmp(buf);
1734 _JUMP_BUFFER *bp = (void*)&cont->jmpbuf;
1735 bp->Frame = ((_JUMP_BUFFER*)((void*)&buf))->Frame;
1736 }
1737#endif
1738 if (cont->machine.stack_src) {
1739 FLUSH_REGISTER_WINDOWS;
1740 MEMCPY(cont->machine.stack_src, cont->machine.stack,
1741 VALUE, cont->machine.stack_size);
1742 }
1743
1744 ruby_longjmp(cont->jmpbuf, 1);
1745}
1746
1747NORETURN(NOINLINE(static void cont_restore_0(rb_context_t *, VALUE *)));
1748
1749static void
1750cont_restore_0(rb_context_t *cont, VALUE *addr_in_prev_frame)
1751{
1752 if (cont->machine.stack_src) {
1753#ifdef HAVE_ALLOCA
1754#define STACK_PAD_SIZE 1
1755#else
1756#define STACK_PAD_SIZE 1024
1757#endif
1758 VALUE space[STACK_PAD_SIZE];
1759
1760#if !STACK_GROW_DIRECTION
1761 if (addr_in_prev_frame > &space[0]) {
1762 /* Stack grows downward */
1763#endif
1764#if STACK_GROW_DIRECTION <= 0
1765 volatile VALUE *const end = cont->machine.stack_src;
1766 if (&space[0] > end) {
1767# ifdef HAVE_ALLOCA
1768 volatile VALUE *sp = ALLOCA_N(VALUE, &space[0] - end);
1769 // We need to make sure that the stack pointer is moved,
1770 // but some compilers may remove the allocation by optimization.
1771 // We hope that the following read/write will prevent such an optimization.
1772 *sp = Qfalse;
1773 space[0] = *sp;
1774# else
1775 cont_restore_0(cont, &space[0]);
1776# endif
1777 }
1778#endif
1779#if !STACK_GROW_DIRECTION
1780 }
1781 else {
1782 /* Stack grows upward */
1783#endif
1784#if STACK_GROW_DIRECTION >= 0
1785 volatile VALUE *const end = cont->machine.stack_src + cont->machine.stack_size;
1786 if (&space[STACK_PAD_SIZE] < end) {
1787# ifdef HAVE_ALLOCA
1788 volatile VALUE *sp = ALLOCA_N(VALUE, end - &space[STACK_PAD_SIZE]);
1789 space[0] = *sp;
1790# else
1791 cont_restore_0(cont, &space[STACK_PAD_SIZE-1]);
1792# endif
1793 }
1794#endif
1795#if !STACK_GROW_DIRECTION
1796 }
1797#endif
1798 }
1799 cont_restore_1(cont);
1800}
1801
1802/*
1803 * Document-class: Continuation
1804 *
1805 * Continuation objects are generated by Kernel#callcc,
1806 * after having +require+d <i>continuation</i>. They hold
1807 * a return address and execution context, allowing a nonlocal return
1808 * to the end of the #callcc block from anywhere within a
1809 * program. Continuations are somewhat analogous to a structured
1810 * version of C's <code>setjmp/longjmp</code> (although they contain
1811 * more state, so you might consider them closer to threads).
1812 *
1813 * For instance:
1814 *
1815 * require "continuation"
1816 * arr = [ "Freddie", "Herbie", "Ron", "Max", "Ringo" ]
1817 * callcc{|cc| $cc = cc}
1818 * puts(message = arr.shift)
1819 * $cc.call unless message =~ /Max/
1820 *
1821 * <em>produces:</em>
1822 *
1823 * Freddie
1824 * Herbie
1825 * Ron
1826 * Max
1827 *
1828 * Also you can call callcc in other methods:
1829 *
1830 * require "continuation"
1831 *
1832 * def g
1833 * arr = [ "Freddie", "Herbie", "Ron", "Max", "Ringo" ]
1834 * cc = callcc { |cc| cc }
1835 * puts arr.shift
1836 * return cc, arr.size
1837 * end
1838 *
1839 * def f
1840 * c, size = g
1841 * c.call(c) if size > 1
1842 * end
1843 *
1844 * f
1845 *
1846 * This (somewhat contrived) example allows the inner loop to abandon
1847 * processing early:
1848 *
1849 * require "continuation"
1850 * callcc {|cont|
1851 * for i in 0..4
1852 * print "#{i}: "
1853 * for j in i*5...(i+1)*5
1854 * cont.call() if j == 17
1855 * printf "%3d", j
1856 * end
1857 * end
1858 * }
1859 * puts
1860 *
1861 * <em>produces:</em>
1862 *
1863 * 0: 0 1 2 3 4
1864 * 1: 5 6 7 8 9
1865 * 2: 10 11 12 13 14
1866 * 3: 15 16
1867 */
1868
1869/*
1870 * call-seq:
1871 * callcc {|cont| block } -> obj
1872 *
1873 * Generates a Continuation object, which it passes to
1874 * the associated block. You need to <code>require
1875 * 'continuation'</code> before using this method. Performing a
1876 * <em>cont</em><code>.call</code> will cause the #callcc
1877 * to return (as will falling through the end of the block). The
1878 * value returned by the #callcc is the value of the
1879 * block, or the value passed to <em>cont</em><code>.call</code>. See
1880 * class Continuation for more details. Also see
1881 * Kernel#throw for an alternative mechanism for
1882 * unwinding a call stack.
1883 */
1884
1885static VALUE
1886rb_callcc(VALUE self)
1887{
1888 volatile int called;
1889 volatile VALUE val = cont_capture(&called);
1890
1891 if (called) {
1892 return val;
1893 }
1894 else {
1895 return rb_yield(val);
1896 }
1897}
1898#ifdef RUBY_ASAN_ENABLED
1899/* callcc can't possibly work with ASAN; see bug #20273. Also this function
1900 * definition below avoids a "defined and not used" warning. */
1901MAYBE_UNUSED(static void notusing_callcc(void)) { rb_callcc(Qnil); }
1902# define rb_callcc rb_f_notimplement
1903#endif
1904
1905
1906static VALUE
1907make_passing_arg(int argc, const VALUE *argv)
1908{
1909 switch (argc) {
1910 case -1:
1911 return argv[0];
1912 case 0:
1913 return Qnil;
1914 case 1:
1915 return argv[0];
1916 default:
1917 return rb_ary_new4(argc, argv);
1918 }
1919}
1920
1921typedef VALUE e_proc(VALUE);
1922
1923NORETURN(static VALUE rb_cont_call(int argc, VALUE *argv, VALUE contval));
1924
1925/*
1926 * call-seq:
1927 * cont.call(args, ...)
1928 * cont[args, ...]
1929 *
1930 * Invokes the continuation. The program continues from the end of
1931 * the #callcc block. If no arguments are given, the original #callcc
1932 * returns +nil+. If one argument is given, #callcc returns
1933 * it. Otherwise, an array containing <i>args</i> is returned.
1934 *
1935 * callcc {|cont| cont.call } #=> nil
1936 * callcc {|cont| cont.call 1 } #=> 1
1937 * callcc {|cont| cont.call 1, 2, 3 } #=> [1, 2, 3]
1938 */
1939
1940static VALUE
1941rb_cont_call(int argc, VALUE *argv, VALUE contval)
1942{
1943 rb_context_t *cont = cont_ptr(contval);
1944 rb_thread_t *th = GET_THREAD();
1945
1946 if (cont_thread_value(cont) != th->self) {
1947 rb_raise(rb_eRuntimeError, "continuation called across threads");
1948 }
1949 if (cont->saved_ec.fiber_ptr) {
1950 if (th->ec->fiber_ptr != cont->saved_ec.fiber_ptr) {
1951 rb_raise(rb_eRuntimeError, "continuation called across fiber");
1952 }
1953 }
1954
1955 cont->argc = argc;
1956 cont->value = make_passing_arg(argc, argv);
1957
1958 cont_restore_0(cont, &contval);
1959 UNREACHABLE_RETURN(Qnil);
1960}
1961
1962/*********/
1963/* fiber */
1964/*********/
1965
1966/*
1967 * Document-class: Fiber
1968 *
1969 * Fibers are primitives for implementing light weight cooperative
1970 * concurrency in Ruby. Basically they are a means of creating code blocks
1971 * that can be paused and resumed, much like threads. The main difference
1972 * is that they are never preempted and that the scheduling must be done by
1973 * the programmer and not the VM.
1974 *
1975 * As opposed to other stackless light weight concurrency models, each fiber
1976 * comes with a stack. This enables the fiber to be paused from deeply
1977 * nested function calls within the fiber block. See the ruby(1)
1978 * manpage to configure the size of the fiber stack(s).
1979 *
1980 * When a fiber is created it will not run automatically. Rather it must
1981 * be explicitly asked to run using the Fiber#resume method.
1982 * The code running inside the fiber can give up control by calling
1983 * Fiber.yield in which case it yields control back to caller (the
1984 * caller of the Fiber#resume).
1985 *
1986 * Upon yielding or termination the Fiber returns the value of the last
1987 * executed expression
1988 *
1989 * For instance:
1990 *
1991 * fiber = Fiber.new do
1992 * Fiber.yield 1
1993 * 2
1994 * end
1995 *
1996 * puts fiber.resume
1997 * puts fiber.resume
1998 * puts fiber.resume
1999 *
2000 * <em>produces</em>
2001 *
2002 * 1
2003 * 2
2004 * FiberError: dead fiber called
2005 *
2006 * The Fiber#resume method accepts an arbitrary number of parameters,
2007 * if it is the first call to #resume then they will be passed as
2008 * block arguments. Otherwise they will be the return value of the
2009 * call to Fiber.yield
2010 *
2011 * Example:
2012 *
2013 * fiber = Fiber.new do |first|
2014 * second = Fiber.yield first + 2
2015 * end
2016 *
2017 * puts fiber.resume 10
2018 * puts fiber.resume 1_000_000
2019 * puts fiber.resume "The fiber will be dead before I can cause trouble"
2020 *
2021 * <em>produces</em>
2022 *
2023 * 12
2024 * 1000000
2025 * FiberError: dead fiber called
2026 *
2027 * == Non-blocking Fibers
2028 *
2029 * The concept of <em>non-blocking fiber</em> was introduced in Ruby 3.0.
2030 * A non-blocking fiber, when reaching an operation that would normally block
2031 * the fiber (like <code>sleep</code>, or wait for another process or I/O)
2032 * will yield control to other fibers and allow the <em>scheduler</em> to
2033 * handle blocking and waking up (resuming) this fiber when it can proceed.
2034 *
2035 * For a Fiber to behave as non-blocking, it need to be created in Fiber.new with
2036 * <tt>blocking: false</tt> (which is the default), and Fiber.scheduler
2037 * should be set with Fiber.set_scheduler. If Fiber.scheduler is not set in
2038 * the current thread, blocking and non-blocking fibers' behavior is identical.
2039 *
2040 * Ruby doesn't provide a scheduler class: it is expected to be implemented by
2041 * the user and correspond to Fiber::Scheduler.
2042 *
2043 * There is also Fiber.schedule method, which is expected to immediately perform
2044 * the given block in a non-blocking manner. Its actual implementation is up to
2045 * the scheduler.
2046 *
2047 */
2048
2049static const rb_data_type_t fiber_data_type = {
2050 "fiber",
2051 {fiber_mark, fiber_free, fiber_memsize, fiber_compact,},
2052 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
2053};
2054
2055static VALUE
2056fiber_alloc(VALUE klass)
2057{
2058 return TypedData_Wrap_Struct(klass, &fiber_data_type, 0);
2059}
2060
2061static rb_serial_t
2062next_ec_serial(rb_ractor_t *cr)
2063{
2064 return cr->next_ec_serial++;
2065}
2066
2067static rb_fiber_t*
2068fiber_t_alloc(VALUE fiber_value, unsigned int blocking)
2069{
2070 rb_fiber_t *fiber;
2071 rb_thread_t *th = GET_THREAD();
2072
2073 if (DATA_PTR(fiber_value) != 0) {
2074 rb_raise(rb_eRuntimeError, "cannot initialize twice");
2075 }
2076
2077 THREAD_MUST_BE_RUNNING(th);
2078 fiber = ZALLOC(rb_fiber_t);
2079 fiber->cont.self = fiber_value;
2080 fiber->cont.type = FIBER_CONTEXT;
2081 fiber->blocking = blocking;
2082 fiber->killed = 0;
2083 cont_init(&fiber->cont, th);
2084
2085 fiber->cont.saved_ec.fiber_ptr = fiber;
2086 fiber->cont.saved_ec.serial = next_ec_serial(th->ractor);
2087 rb_ec_clear_vm_stack(&fiber->cont.saved_ec);
2088
2089 fiber->prev = NULL;
2090
2091 /* fiber->status == 0 == CREATED
2092 * So that we don't need to set status: fiber_status_set(fiber, FIBER_CREATED); */
2093 VM_ASSERT(FIBER_CREATED_P(fiber));
2094
2095 DATA_PTR(fiber_value) = fiber;
2096
2097 return fiber;
2098}
2099
2100static rb_fiber_t *
2101root_fiber_alloc(rb_thread_t *th)
2102{
2103 VALUE fiber_value = fiber_alloc(rb_cFiber);
2104 rb_fiber_t *fiber = th->ec->fiber_ptr;
2105
2106 VM_ASSERT(DATA_PTR(fiber_value) == NULL);
2107 VM_ASSERT(fiber->cont.type == FIBER_CONTEXT);
2108 VM_ASSERT(FIBER_RESUMED_P(fiber));
2109
2110 th->root_fiber = fiber;
2111 DATA_PTR(fiber_value) = fiber;
2112 fiber->cont.self = fiber_value;
2113
2114 coroutine_initialize_main(&fiber->context);
2115
2116 return fiber;
2117}
2118
2119static inline rb_fiber_t*
2120fiber_current(void)
2121{
2122 rb_execution_context_t *ec = GET_EC();
2123 if (ec->fiber_ptr->cont.self == 0) {
2124 root_fiber_alloc(rb_ec_thread_ptr(ec));
2125 }
2126 return ec->fiber_ptr;
2127}
2128
2129static inline VALUE
2130current_fiber_storage(void)
2131{
2132 rb_execution_context_t *ec = GET_EC();
2133 return ec->storage;
2134}
2135
2136static inline VALUE
2137inherit_fiber_storage(void)
2138{
2139 return rb_obj_dup(current_fiber_storage());
2140}
2141
2142static inline void
2143fiber_storage_set(struct rb_fiber_struct *fiber, VALUE storage)
2144{
2145 fiber->cont.saved_ec.storage = storage;
2146}
2147
2148static inline VALUE
2149fiber_storage_get(rb_fiber_t *fiber, int allocate)
2150{
2151 VALUE storage = fiber->cont.saved_ec.storage;
2152 if (storage == Qnil && allocate) {
2153 storage = rb_hash_new();
2154 fiber_storage_set(fiber, storage);
2155 }
2156 return storage;
2157}
2158
2159static void
2160storage_access_must_be_from_same_fiber(VALUE self)
2161{
2162 rb_fiber_t *fiber = fiber_ptr(self);
2163 rb_fiber_t *current = fiber_current();
2164 if (fiber != current) {
2165 rb_raise(rb_eArgError, "Fiber storage can only be accessed from the Fiber it belongs to");
2166 }
2167}
2168
2175static VALUE
2176rb_fiber_storage_get(VALUE self)
2177{
2178 storage_access_must_be_from_same_fiber(self);
2179
2180 VALUE storage = fiber_storage_get(fiber_ptr(self), FALSE);
2181
2182 if (storage == Qnil) {
2183 return Qnil;
2184 }
2185 else {
2186 return rb_obj_dup(storage);
2187 }
2188}
2189
2190static int
2191fiber_storage_validate_each(VALUE key, VALUE value, VALUE _argument)
2192{
2193 Check_Type(key, T_SYMBOL);
2194
2195 return ST_CONTINUE;
2196}
2197
2198static void
2199fiber_storage_validate(VALUE value)
2200{
2201 // nil is an allowed value and will be lazily initialized.
2202 if (value == Qnil) return;
2203
2204 if (!RB_TYPE_P(value, T_HASH)) {
2205 rb_raise(rb_eTypeError, "storage must be a hash");
2206 }
2207
2208 if (RB_OBJ_FROZEN(value)) {
2209 rb_raise(rb_eFrozenError, "storage must not be frozen");
2210 }
2211
2212 rb_hash_foreach(value, fiber_storage_validate_each, Qundef);
2213}
2214
2237static VALUE
2238rb_fiber_storage_set(VALUE self, VALUE value)
2239{
2240 if (rb_warning_category_enabled_p(RB_WARN_CATEGORY_EXPERIMENTAL)) {
2241 rb_category_warn(RB_WARN_CATEGORY_EXPERIMENTAL,
2242 "Fiber#storage= is experimental and may be removed in the future!");
2243 }
2244
2245 storage_access_must_be_from_same_fiber(self);
2246 fiber_storage_validate(value);
2247
2248 fiber_ptr(self)->cont.saved_ec.storage = rb_obj_dup(value);
2249 return value;
2250}
2251
2262static VALUE
2263rb_fiber_storage_aref(VALUE class, VALUE key)
2264{
2265 key = rb_to_symbol(key);
2266
2267 VALUE storage = fiber_storage_get(fiber_current(), FALSE);
2268 if (storage == Qnil) return Qnil;
2269
2270 return rb_hash_aref(storage, key);
2271}
2272
2283static VALUE
2284rb_fiber_storage_aset(VALUE class, VALUE key, VALUE value)
2285{
2286 key = rb_to_symbol(key);
2287
2288 VALUE storage = fiber_storage_get(fiber_current(), value != Qnil);
2289 if (storage == Qnil) return Qnil;
2290
2291 if (value == Qnil) {
2292 return rb_hash_delete(storage, key);
2293 }
2294 else {
2295 return rb_hash_aset(storage, key, value);
2296 }
2297}
2298
2299static VALUE
2300fiber_initialize(VALUE self, VALUE proc, struct fiber_pool * fiber_pool, unsigned int blocking, VALUE storage)
2301{
2302 if (storage == Qundef || storage == Qtrue) {
2303 // The default, inherit storage (dup) from the current fiber:
2304 storage = inherit_fiber_storage();
2305 }
2306 else /* nil, hash, etc. */ {
2307 fiber_storage_validate(storage);
2308 storage = rb_obj_dup(storage);
2309 }
2310
2311 rb_fiber_t *fiber = fiber_t_alloc(self, blocking);
2312
2313 fiber->cont.saved_ec.storage = storage;
2314 fiber->first_proc = proc;
2315 fiber->stack.base = NULL;
2316 fiber->stack.pool = fiber_pool;
2317
2318 return self;
2319}
2320
2321static void
2322fiber_prepare_stack(rb_fiber_t *fiber)
2323{
2324 rb_context_t *cont = &fiber->cont;
2325 rb_execution_context_t *sec = &cont->saved_ec;
2326
2327 size_t vm_stack_size = 0;
2328 VALUE *vm_stack = fiber_initialize_coroutine(fiber, &vm_stack_size);
2329
2330 /* initialize cont */
2331 cont->saved_vm_stack.ptr = NULL;
2332 rb_ec_initialize_vm_stack(sec, vm_stack, vm_stack_size / sizeof(VALUE));
2333
2334 sec->tag = NULL;
2335 sec->local_storage = NULL;
2336 sec->local_storage_recursive_hash = Qnil;
2337 sec->local_storage_recursive_hash_for_trace = Qnil;
2338}
2339
2340static struct fiber_pool *
2341rb_fiber_pool_default(VALUE pool)
2342{
2343 return &shared_fiber_pool;
2344}
2345
2346 VALUE rb_fiber_inherit_storage(struct rb_execution_context_struct *ec, struct rb_fiber_struct *fiber)
2347{
2348 VALUE storage = rb_obj_dup(ec->storage);
2349 fiber->cont.saved_ec.storage = storage;
2350 return storage;
2351}
2352
2353/* :nodoc: */
2354static VALUE
2355rb_fiber_initialize_kw(int argc, VALUE* argv, VALUE self, int kw_splat)
2356{
2357 VALUE pool = Qnil;
2358 VALUE blocking = Qfalse;
2359 VALUE storage = Qundef;
2360
2361 if (kw_splat != RB_NO_KEYWORDS) {
2362 VALUE options = Qnil;
2363 VALUE arguments[3] = {Qundef};
2364
2365 argc = rb_scan_args_kw(kw_splat, argc, argv, ":", &options);
2366 rb_get_kwargs(options, fiber_initialize_keywords, 0, 3, arguments);
2367
2368 if (!UNDEF_P(arguments[0])) {
2369 blocking = arguments[0];
2370 }
2371
2372 if (!UNDEF_P(arguments[1])) {
2373 pool = arguments[1];
2374 }
2375
2376 storage = arguments[2];
2377 }
2378
2379 return fiber_initialize(self, rb_block_proc(), rb_fiber_pool_default(pool), RTEST(blocking), storage);
2380}
2381
2382/*
2383 * call-seq:
2384 * Fiber.new(blocking: false, storage: true) { |*args| ... } -> fiber
2385 *
2386 * Creates new Fiber. Initially, the fiber is not running and can be resumed
2387 * with #resume. Arguments to the first #resume call will be passed to the
2388 * block:
2389 *
2390 * f = Fiber.new do |initial|
2391 * current = initial
2392 * loop do
2393 * puts "current: #{current.inspect}"
2394 * current = Fiber.yield
2395 * end
2396 * end
2397 * f.resume(100) # prints: current: 100
2398 * f.resume(1, 2, 3) # prints: current: [1, 2, 3]
2399 * f.resume # prints: current: nil
2400 * # ... and so on ...
2401 *
2402 * If <tt>blocking: false</tt> is passed to <tt>Fiber.new</tt>, _and_ current
2403 * thread has a Fiber.scheduler defined, the Fiber becomes non-blocking (see
2404 * "Non-blocking Fibers" section in class docs).
2405 *
2406 * If the <tt>storage</tt> is unspecified, the default is to inherit a copy of
2407 * the storage from the current fiber. This is the same as specifying
2408 * <tt>storage: true</tt>.
2409 *
2410 * Fiber[:x] = 1
2411 * Fiber.new do
2412 * Fiber[:x] # => 1
2413 * Fiber[:x] = 2
2414 * end.resume
2415 * Fiber[:x] # => 1
2416 *
2417 * If the given <tt>storage</tt> is <tt>nil</tt>, this function will lazy
2418 * initialize the internal storage, which starts as an empty hash.
2419 *
2420 * Fiber[:x] = "Hello World"
2421 * Fiber.new(storage: nil) do
2422 * Fiber[:x] # nil
2423 * end
2424 *
2425 * Otherwise, the given <tt>storage</tt> is used as the new fiber's storage,
2426 * and it must be an instance of Hash.
2427 *
2428 * Explicitly using <tt>storage: true</tt> is currently experimental and may
2429 * change in the future.
2430 */
2431static VALUE
2432rb_fiber_initialize(int argc, VALUE* argv, VALUE self)
2433{
2434 return rb_fiber_initialize_kw(argc, argv, self, rb_keyword_given_p());
2435}
2436
2437 VALUE
2438rb_fiber_new_storage(rb_block_call_func_t func, VALUE obj, VALUE storage)
2439{
2440 return fiber_initialize(fiber_alloc(rb_cFiber), rb_proc_new(func, obj), rb_fiber_pool_default(Qnil), 0, storage);
2441}
2442
2443 VALUE
2444rb_fiber_new(rb_block_call_func_t func, VALUE obj)
2445{
2446 return rb_fiber_new_storage(func, obj, Qtrue);
2447}
2448
2449static VALUE
2450rb_fiber_s_schedule_kw(int argc, VALUE* argv, int kw_splat)
2451{
2452 rb_thread_t * th = GET_THREAD();
2453 VALUE scheduler = th->scheduler;
2454 VALUE fiber = Qnil;
2455
2456 if (scheduler != Qnil) {
2457 fiber = rb_fiber_scheduler_fiber(scheduler, argc, argv, kw_splat);
2458 }
2459 else {
2460 rb_raise(rb_eRuntimeError, "No scheduler is available!");
2461 }
2462
2463 return fiber;
2464}
2465
2466/*
2467 * call-seq:
2468 * Fiber.schedule { |*args| ... } -> fiber
2469 *
2470 * The method is <em>expected</em> to immediately run the provided block of code in a
2471 * separate non-blocking fiber.
2472 *
2473 * puts "Go to sleep!"
2474 *
2475 * Fiber.set_scheduler(MyScheduler.new)
2476 *
2477 * Fiber.schedule do
2478 * puts "Going to sleep"
2479 * sleep(1)
2480 * puts "I slept well"
2481 * end
2482 *
2483 * puts "Wakey-wakey, sleepyhead"
2484 *
2485 * Assuming MyScheduler is properly implemented, this program will produce:
2486 *
2487 * Go to sleep!
2488 * Going to sleep
2489 * Wakey-wakey, sleepyhead
2490 * ...1 sec pause here...
2491 * I slept well
2492 *
2493 * ...e.g. on the first blocking operation inside the Fiber (<tt>sleep(1)</tt>),
2494 * the control is yielded to the outside code (main fiber), and <em>at the end
2495 * of that execution</em>, the scheduler takes care of properly resuming all the
2496 * blocked fibers.
2497 *
2498 * Note that the behavior described above is how the method is <em>expected</em>
2499 * to behave, actual behavior is up to the current scheduler's implementation of
2500 * Fiber::Scheduler#fiber method. Ruby doesn't enforce this method to
2501 * behave in any particular way.
2502 *
2503 * If the scheduler is not set, the method raises
2504 * <tt>RuntimeError (No scheduler is available!)</tt>.
2505 *
2506 */
2507static VALUE
2508rb_fiber_s_schedule(int argc, VALUE *argv, VALUE obj)
2509{
2510 return rb_fiber_s_schedule_kw(argc, argv, rb_keyword_given_p());
2511}
2512
2513/*
2514 * call-seq:
2515 * Fiber.scheduler -> obj or nil
2516 *
2517 * Returns the Fiber scheduler, that was last set for the current thread with Fiber.set_scheduler.
2518 * Returns +nil+ if no scheduler is set (which is the default), and non-blocking fibers'
2519 * behavior is the same as blocking.
2520 * (see "Non-blocking fibers" section in class docs for details about the scheduler concept).
2521 *
2522 */
2523static VALUE
2524rb_fiber_s_scheduler(VALUE klass)
2525{
2526 return rb_fiber_scheduler_get();
2527}
2528
2529/*
2530 * call-seq:
2531 * Fiber.current_scheduler -> obj or nil
2532 *
2533 * Returns the Fiber scheduler, that was last set for the current thread with Fiber.set_scheduler
2534 * if and only if the current fiber is non-blocking.
2535 *
2536 */
2537static VALUE
2538rb_fiber_current_scheduler(VALUE klass)
2539{
2540 return rb_fiber_scheduler_current();
2541}
2542
2543/*
2544 * call-seq:
2545 * Fiber.set_scheduler(scheduler) -> scheduler
2546 *
2547 * Sets the Fiber scheduler for the current thread. If the scheduler is set, non-blocking
2548 * fibers (created by Fiber.new with <tt>blocking: false</tt>, or by Fiber.schedule)
2549 * call that scheduler's hook methods on potentially blocking operations, and the current
2550 * thread will call scheduler's +close+ method on finalization (allowing the scheduler to
2551 * properly manage all non-finished fibers).
2552 *
2553 * +scheduler+ can be an object of any class corresponding to Fiber::Scheduler. Its
2554 * implementation is up to the user.
2555 *
2556 * See also the "Non-blocking fibers" section in class docs.
2557 *
2558 */
2559static VALUE
2560rb_fiber_set_scheduler(VALUE klass, VALUE scheduler)
2561{
2562 return rb_fiber_scheduler_set(scheduler);
2563}
2564
2565NORETURN(static void rb_fiber_terminate(rb_fiber_t *fiber, int need_interrupt, VALUE err));
2566
2567void
2568rb_fiber_start(rb_fiber_t *fiber)
2569{
2570 rb_thread_t * volatile th = fiber->cont.saved_ec.thread_ptr;
2571
2572 rb_proc_t *proc;
2573 enum ruby_tag_type state;
2574
2575 VM_ASSERT(th->ec == GET_EC());
2576 VM_ASSERT(FIBER_RESUMED_P(fiber));
2577
2578 if (fiber->blocking) {
2579 th->blocking += 1;
2580 }
2581
2582 EC_PUSH_TAG(th->ec);
2583 if ((state = EC_EXEC_TAG()) == TAG_NONE) {
2584 rb_context_t *cont = &VAR_FROM_MEMORY(fiber)->cont;
2585 int argc;
2586 const VALUE *argv, args = cont->value;
2587 GetProcPtr(fiber->first_proc, proc);
2588 argv = (argc = cont->argc) > 1 ? RARRAY_CONST_PTR(args) : &args;
2589 cont->value = Qnil;
2590 th->ec->errinfo = Qnil;
2591 th->ec->root_lep = rb_vm_proc_local_ep(fiber->first_proc);
2592 th->ec->root_svar = Qfalse;
2593
2594 EXEC_EVENT_HOOK(th->ec, RUBY_EVENT_FIBER_SWITCH, th->self, 0, 0, 0, Qnil);
2595 cont->value = rb_vm_invoke_proc(th->ec, proc, argc, argv, cont->kw_splat, VM_BLOCK_HANDLER_NONE);
2596 }
2597 EC_POP_TAG();
2598
2599 int need_interrupt = TRUE;
2600 VALUE err = Qfalse;
2601 if (state) {
2602 err = th->ec->errinfo;
2603 VM_ASSERT(FIBER_RESUMED_P(fiber));
2604
2605 if (state == TAG_RAISE) {
2606 // noop...
2607 }
2608 else if (state == TAG_FATAL && err == RUBY_FATAL_FIBER_KILLED) {
2609 need_interrupt = FALSE;
2610 err = Qfalse;
2611 }
2612 else if (state == TAG_FATAL) {
2613 rb_threadptr_pending_interrupt_enque(th, err);
2614 }
2615 else {
2616 err = rb_vm_make_jump_tag_but_local_jump(state, err);
2617 }
2618 }
2619
2620 rb_fiber_terminate(fiber, need_interrupt, err);
2621}
2622
2623// Set up a "root fiber", which is the fiber that every Ractor has.
2624void
2625rb_threadptr_root_fiber_setup(rb_thread_t *th)
2626{
2627 rb_fiber_t *fiber = ruby_mimcalloc(1, sizeof(rb_fiber_t));
2628 if (!fiber) {
2629 rb_bug("%s", strerror(errno)); /* ... is it possible to call rb_bug here? */
2630 }
2631 fiber->cont.type = FIBER_CONTEXT;
2632 fiber->cont.saved_ec.fiber_ptr = fiber;
2633 fiber->cont.saved_ec.serial = next_ec_serial(th->ractor);
2634 fiber->cont.saved_ec.thread_ptr = th;
2635 fiber->blocking = 1;
2636 fiber->killed = 0;
2637 fiber_status_set(fiber, FIBER_RESUMED); /* skip CREATED */
2638 th->ec = &fiber->cont.saved_ec;
2639 cont_init_jit_cont(&fiber->cont);
2640}
2641
2642void
2643rb_threadptr_root_fiber_release(rb_thread_t *th)
2644{
2645 if (th->root_fiber) {
2646 /* ignore. A root fiber object will free th->ec */
2647 }
2648 else {
2649 rb_execution_context_t *ec = rb_current_execution_context(false);
2650
2651 VM_ASSERT(th->ec->fiber_ptr->cont.type == FIBER_CONTEXT);
2652 VM_ASSERT(th->ec->fiber_ptr->cont.self == 0);
2653
2654 if (ec && th->ec == ec) {
2655 rb_ractor_set_current_ec(th->ractor, NULL);
2656 }
2657 fiber_free(th->ec->fiber_ptr);
2658 th->ec = NULL;
2659 }
2660}
2661
2662void
2663rb_threadptr_root_fiber_terminate(rb_thread_t *th)
2664{
2665 rb_fiber_t *fiber = th->ec->fiber_ptr;
2666
2667 fiber->status = FIBER_TERMINATED;
2668
2669 // The vm_stack is `alloca`ed on the thread stack, so it's gone too:
2670 rb_ec_clear_vm_stack(th->ec);
2671}
2672
2673static inline rb_fiber_t*
2674return_fiber(bool terminate)
2675{
2676 rb_fiber_t *fiber = fiber_current();
2677 rb_fiber_t *prev = fiber->prev;
2678
2679 if (prev) {
2680 fiber->prev = NULL;
2681 prev->resuming_fiber = NULL;
2682 return prev;
2683 }
2684 else {
2685 if (!terminate) {
2686 rb_raise(rb_eFiberError, "attempt to yield on a not resumed fiber");
2687 }
2688
2689 rb_thread_t *th = GET_THREAD();
2690 rb_fiber_t *root_fiber = th->root_fiber;
2691
2692 VM_ASSERT(root_fiber != NULL);
2693
2694 // search resuming fiber
2695 for (fiber = root_fiber; fiber->resuming_fiber; fiber = fiber->resuming_fiber) {
2696 }
2697
2698 return fiber;
2699 }
2700}
2701
2702 VALUE
2703rb_fiber_current(void)
2704{
2705 return fiber_current()->cont.self;
2706}
2707
2708// Prepare to execute next_fiber on the given thread.
2709static inline void
2710fiber_store(rb_fiber_t *next_fiber, rb_thread_t *th)
2711{
2712 rb_fiber_t *fiber;
2713
2714 if (th->ec->fiber_ptr != NULL) {
2715 fiber = th->ec->fiber_ptr;
2716 }
2717 else {
2718 /* create root fiber */
2719 fiber = root_fiber_alloc(th);
2720 }
2721
2722 if (FIBER_CREATED_P(next_fiber)) {
2723 fiber_prepare_stack(next_fiber);
2724 }
2725
2726 VM_ASSERT(FIBER_RESUMED_P(fiber) || FIBER_TERMINATED_P(fiber));
2727 VM_ASSERT(FIBER_RUNNABLE_P(next_fiber));
2728
2729 if (FIBER_RESUMED_P(fiber)) fiber_status_set(fiber, FIBER_SUSPENDED);
2730
2731 fiber_status_set(next_fiber, FIBER_RESUMED);
2732 fiber_setcontext(next_fiber, fiber);
2733}
2734
2735static void
2736fiber_check_killed(rb_fiber_t *fiber)
2737{
2738 VM_ASSERT(fiber == fiber_current());
2739
2740 if (fiber->killed) {
2741 rb_thread_t *thread = fiber->cont.saved_ec.thread_ptr;
2742
2743 thread->ec->errinfo = RUBY_FATAL_FIBER_KILLED;
2744 EC_JUMP_TAG(thread->ec, RUBY_TAG_FATAL);
2745 }
2746}
2747
2748static inline VALUE
2749fiber_switch(rb_fiber_t *fiber, int argc, const VALUE *argv, int kw_splat, rb_fiber_t *resuming_fiber, bool yielding)
2750{
2751 VALUE value;
2752 rb_context_t *cont = &fiber->cont;
2753 rb_thread_t *th = GET_THREAD();
2754
2755 /* make sure the root_fiber object is available */
2756 if (th->root_fiber == NULL) root_fiber_alloc(th);
2757
2758 if (th->ec->fiber_ptr == fiber) {
2759 /* ignore fiber context switch
2760 * because destination fiber is the same as current fiber
2761 */
2762 return make_passing_arg(argc, argv);
2763 }
2764
2765 if (cont_thread_value(cont) != th->self) {
2766 rb_raise(rb_eFiberError, "fiber called across threads");
2767 }
2768
2769 if (FIBER_TERMINATED_P(fiber)) {
2770 value = rb_exc_new2(rb_eFiberError, "dead fiber called");
2771
2772 if (!FIBER_TERMINATED_P(th->ec->fiber_ptr)) {
2773 rb_exc_raise(value);
2774 VM_UNREACHABLE(fiber_switch);
2775 }
2776 else {
2777 /* th->ec->fiber_ptr is also dead => switch to root fiber */
2778 /* (this means we're being called from rb_fiber_terminate, */
2779 /* and the terminated fiber's return_fiber() is already dead) */
2780 VM_ASSERT(FIBER_SUSPENDED_P(th->root_fiber));
2781
2782 cont = &th->root_fiber->cont;
2783 cont->argc = -1;
2784 cont->value = value;
2785
2786 fiber_setcontext(th->root_fiber, th->ec->fiber_ptr);
2787
2788 VM_UNREACHABLE(fiber_switch);
2789 }
2790 }
2791
2792 VM_ASSERT(FIBER_RUNNABLE_P(fiber));
2793
2794 rb_fiber_t *current_fiber = fiber_current();
2795
2796 VM_ASSERT(!current_fiber->resuming_fiber);
2797
2798 if (resuming_fiber) {
2799 current_fiber->resuming_fiber = resuming_fiber;
2800 fiber->prev = fiber_current();
2801 fiber->yielding = 0;
2802 }
2803
2804 VM_ASSERT(!current_fiber->yielding);
2805 if (yielding) {
2806 current_fiber->yielding = 1;
2807 }
2808
2809 if (current_fiber->blocking) {
2810 th->blocking -= 1;
2811 }
2812
2813 cont->argc = argc;
2814 cont->kw_splat = kw_splat;
2815 cont->value = make_passing_arg(argc, argv);
2816
2817 fiber_store(fiber, th);
2818
2819 // We cannot free the stack until the pthread is joined:
2820#ifndef COROUTINE_PTHREAD_CONTEXT
2821 if (FIBER_TERMINATED_P(fiber)) {
2822 RB_VM_LOCKING() {
2823 fiber_stack_release(fiber);
2824 }
2825 }
2826#endif
2827
2828 if (fiber_current()->blocking) {
2829 th->blocking += 1;
2830 }
2831
2832 RUBY_VM_CHECK_INTS(th->ec);
2833
2834 EXEC_EVENT_HOOK(th->ec, RUBY_EVENT_FIBER_SWITCH, th->self, 0, 0, 0, Qnil);
2835
2836 current_fiber = th->ec->fiber_ptr;
2837 value = current_fiber->cont.value;
2838
2839 fiber_check_killed(current_fiber);
2840
2841 if (current_fiber->cont.argc == -1) {
2842 // Fiber#raise will trigger this path.
2843 rb_exc_raise(value);
2844 }
2845
2846 return value;
2847}
2848
2849 VALUE
2850rb_fiber_transfer(VALUE fiber_value, int argc, const VALUE *argv)
2851{
2852 return fiber_switch(fiber_ptr(fiber_value), argc, argv, RB_NO_KEYWORDS, NULL, false);
2853}
2854
2855/*
2856 * call-seq:
2857 * fiber.blocking? -> true or false
2858 *
2859 * Returns +true+ if +fiber+ is blocking and +false+ otherwise.
2860 * Fiber is non-blocking if it was created via passing <tt>blocking: false</tt>
2861 * to Fiber.new, or via Fiber.schedule.
2862 *
2863 * Note that, even if the method returns +false+, the fiber behaves differently
2864 * only if Fiber.scheduler is set in the current thread.
2865 *
2866 * See the "Non-blocking fibers" section in class docs for details.
2867 *
2868 */
2869 VALUE
2870rb_fiber_blocking_p(VALUE fiber)
2871{
2872 return RBOOL(fiber_ptr(fiber)->blocking);
2873}
2874
2875static VALUE
2876fiber_blocking_yield(VALUE fiber_value)
2877{
2878 rb_fiber_t *fiber = fiber_ptr(fiber_value);
2879 rb_thread_t * volatile th = fiber->cont.saved_ec.thread_ptr;
2880
2881 VM_ASSERT(fiber->blocking == 0);
2882
2883 // fiber->blocking is `unsigned int : 1`, so we use it as a boolean:
2884 fiber->blocking = 1;
2885
2886 // Once the fiber is blocking, and current, we increment the thread blocking state:
2887 th->blocking += 1;
2888
2889 return rb_yield(fiber_value);
2890}
2891
2892static VALUE
2893fiber_blocking_ensure(VALUE fiber_value)
2894{
2895 rb_fiber_t *fiber = fiber_ptr(fiber_value);
2896 rb_thread_t * volatile th = fiber->cont.saved_ec.thread_ptr;
2897
2898 // We are no longer blocking:
2899 fiber->blocking = 0;
2900 th->blocking -= 1;
2901
2902 return Qnil;
2903}
2904
2905/*
2906 * call-seq:
2907 * Fiber.blocking{|fiber| ...} -> result
2908 *
2909 * Forces the fiber to be blocking for the duration of the block. Returns the
2910 * result of the block.
2911 *
2912 * See the "Non-blocking fibers" section in class docs for details.
2913 *
2914 */
2915 VALUE
2916rb_fiber_blocking(VALUE class)
2917{
2918 VALUE fiber_value = rb_fiber_current();
2919 rb_fiber_t *fiber = fiber_ptr(fiber_value);
2920
2921 // If we are already blocking, this is essentially a no-op:
2922 if (fiber->blocking) {
2923 return rb_yield(fiber_value);
2924 }
2925 else {
2926 return rb_ensure(fiber_blocking_yield, fiber_value, fiber_blocking_ensure, fiber_value);
2927 }
2928}
2929
2930/*
2931 * call-seq:
2932 * Fiber.blocking? -> false or 1
2933 *
2934 * Returns +false+ if the current fiber is non-blocking.
2935 * Fiber is non-blocking if it was created via passing <tt>blocking: false</tt>
2936 * to Fiber.new, or via Fiber.schedule.
2937 *
2938 * If the current Fiber is blocking, the method returns 1.
2939 * Future developments may allow for situations where larger integers
2940 * could be returned.
2941 *
2942 * Note that, even if the method returns +false+, Fiber behaves differently
2943 * only if Fiber.scheduler is set in the current thread.
2944 *
2945 * See the "Non-blocking fibers" section in class docs for details.
2946 *
2947 */
2948static VALUE
2949rb_fiber_s_blocking_p(VALUE klass)
2950{
2951 rb_thread_t *thread = GET_THREAD();
2952 unsigned blocking = thread->blocking;
2953
2954 if (blocking == 0)
2955 return Qfalse;
2956
2957 return INT2NUM(blocking);
2958}
2959
2960void
2961rb_fiber_close(rb_fiber_t *fiber)
2962{
2963 fiber_status_set(fiber, FIBER_TERMINATED);
2964 rb_ec_close(&fiber->cont.saved_ec);
2965}
2966
2967static void
2968rb_fiber_terminate(rb_fiber_t *fiber, int need_interrupt, VALUE error)
2969{
2970 VALUE value = fiber->cont.value;
2971
2972 VM_ASSERT(FIBER_RESUMED_P(fiber));
2973 rb_fiber_close(fiber);
2974
2975 fiber->cont.machine.stack = NULL;
2976 fiber->cont.machine.stack_size = 0;
2977
2978 rb_fiber_t *next_fiber = return_fiber(true);
2979
2980 if (need_interrupt) RUBY_VM_SET_INTERRUPT(&next_fiber->cont.saved_ec);
2981
2982 if (RTEST(error))
2983 fiber_switch(next_fiber, -1, &error, RB_NO_KEYWORDS, NULL, false);
2984 else
2985 fiber_switch(next_fiber, 1, &value, RB_NO_KEYWORDS, NULL, false);
2986 ruby_stop(0);
2987}
2988
2989static VALUE
2990fiber_resume_kw(rb_fiber_t *fiber, int argc, const VALUE *argv, int kw_splat)
2991{
2992 rb_fiber_t *current_fiber = fiber_current();
2993
2994 if (argc == -1 && FIBER_CREATED_P(fiber)) {
2995 rb_raise(rb_eFiberError, "cannot raise exception on unborn fiber");
2996 }
2997 else if (FIBER_TERMINATED_P(fiber)) {
2998 rb_raise(rb_eFiberError, "attempt to resume a terminated fiber");
2999 }
3000 else if (fiber == current_fiber) {
3001 rb_raise(rb_eFiberError, "attempt to resume the current fiber");
3002 }
3003 else if (fiber->prev != NULL) {
3004 rb_raise(rb_eFiberError, "attempt to resume a resumed fiber (double resume)");
3005 }
3006 else if (fiber->resuming_fiber) {
3007 rb_raise(rb_eFiberError, "attempt to resume a resuming fiber");
3008 }
3009 else if (fiber->prev == NULL &&
3010 (!fiber->yielding && fiber->status != FIBER_CREATED)) {
3011 rb_raise(rb_eFiberError, "attempt to resume a transferring fiber");
3012 }
3013
3014 return fiber_switch(fiber, argc, argv, kw_splat, fiber, false);
3015}
3016
3017 VALUE
3018rb_fiber_resume_kw(VALUE self, int argc, const VALUE *argv, int kw_splat)
3019{
3020 return fiber_resume_kw(fiber_ptr(self), argc, argv, kw_splat);
3021}
3022
3023 VALUE
3024rb_fiber_resume(VALUE self, int argc, const VALUE *argv)
3025{
3026 return fiber_resume_kw(fiber_ptr(self), argc, argv, RB_NO_KEYWORDS);
3027}
3028
3029 VALUE
3030rb_fiber_yield_kw(int argc, const VALUE *argv, int kw_splat)
3031{
3032 return fiber_switch(return_fiber(false), argc, argv, kw_splat, NULL, true);
3033}
3034
3035 VALUE
3036rb_fiber_yield(int argc, const VALUE *argv)
3037{
3038 return fiber_switch(return_fiber(false), argc, argv, RB_NO_KEYWORDS, NULL, true);
3039}
3040
3041void
3042rb_fiber_reset_root_local_storage(rb_thread_t *th)
3043{
3044 if (th->root_fiber && th->root_fiber != th->ec->fiber_ptr) {
3045 th->ec->local_storage = th->root_fiber->cont.saved_ec.local_storage;
3046 }
3047}
3048
3049/*
3050 * call-seq:
3051 * fiber.alive? -> true or false
3052 *
3053 * Returns true if the fiber can still be resumed (or transferred
3054 * to). After finishing execution of the fiber block this method will
3055 * always return +false+.
3056 */
3057 VALUE
3058rb_fiber_alive_p(VALUE fiber_value)
3059{
3060 return RBOOL(!FIBER_TERMINATED_P(fiber_ptr(fiber_value)));
3061}
3062
3063/*
3064 * call-seq:
3065 * fiber.resume(args, ...) -> obj
3066 *
3067 * Resumes the fiber from the point at which the last Fiber.yield was
3068 * called, or starts running it if it is the first call to
3069 * #resume. Arguments passed to resume will be the value of the
3070 * Fiber.yield expression or will be passed as block parameters to
3071 * the fiber's block if this is the first #resume.
3072 *
3073 * Alternatively, when resume is called it evaluates to the arguments passed
3074 * to the next Fiber.yield statement inside the fiber's block
3075 * or to the block value if it runs to completion without any
3076 * Fiber.yield
3077 */
3078static VALUE
3079rb_fiber_m_resume(int argc, VALUE *argv, VALUE fiber)
3080{
3081 return rb_fiber_resume_kw(fiber, argc, argv, rb_keyword_given_p());
3082}
3083
3084/*
3085 * call-seq:
3086 * fiber.backtrace -> array
3087 * fiber.backtrace(start) -> array
3088 * fiber.backtrace(start, count) -> array
3089 * fiber.backtrace(start..end) -> array
3090 *
3091 * Returns the current execution stack of the fiber. +start+, +count+ and +end+ allow
3092 * to select only parts of the backtrace.
3093 *
3094 * def level3
3095 * Fiber.yield
3096 * end
3097 *
3098 * def level2
3099 * level3
3100 * end
3101 *
3102 * def level1
3103 * level2
3104 * end
3105 *
3106 * f = Fiber.new { level1 }
3107 *
3108 * # It is empty before the fiber started
3109 * f.backtrace
3110 * #=> []
3111 *
3112 * f.resume
3113 *
3114 * f.backtrace
3115 * #=> ["test.rb:2:in `yield'", "test.rb:2:in `level3'", "test.rb:6:in `level2'", "test.rb:10:in `level1'", "test.rb:13:in `block in <main>'"]
3116 * p f.backtrace(1) # start from the item 1
3117 * #=> ["test.rb:2:in `level3'", "test.rb:6:in `level2'", "test.rb:10:in `level1'", "test.rb:13:in `block in <main>'"]
3118 * p f.backtrace(2, 2) # start from item 2, take 2
3119 * #=> ["test.rb:6:in `level2'", "test.rb:10:in `level1'"]
3120 * p f.backtrace(1..3) # take items from 1 to 3
3121 * #=> ["test.rb:2:in `level3'", "test.rb:6:in `level2'", "test.rb:10:in `level1'"]
3122 *
3123 * f.resume
3124 *
3125 * # It is nil after the fiber is finished
3126 * f.backtrace
3127 * #=> nil
3128 *
3129 */
3130static VALUE
3131rb_fiber_backtrace(int argc, VALUE *argv, VALUE fiber)
3132{
3133 return rb_vm_backtrace(argc, argv, &fiber_ptr(fiber)->cont.saved_ec);
3134}
3135
3136/*
3137 * call-seq:
3138 * fiber.backtrace_locations -> array
3139 * fiber.backtrace_locations(start) -> array
3140 * fiber.backtrace_locations(start, count) -> array
3141 * fiber.backtrace_locations(start..end) -> array
3142 *
3143 * Like #backtrace, but returns each line of the execution stack as a
3144 * Thread::Backtrace::Location. Accepts the same arguments as #backtrace.
3145 *
3146 * f = Fiber.new { Fiber.yield }
3147 * f.resume
3148 * loc = f.backtrace_locations.first
3149 * loc.label #=> "yield"
3150 * loc.path #=> "test.rb"
3151 * loc.lineno #=> 1
3152 *
3153 *
3154 */
3155static VALUE
3156rb_fiber_backtrace_locations(int argc, VALUE *argv, VALUE fiber)
3157{
3158 return rb_vm_backtrace_locations(argc, argv, &fiber_ptr(fiber)->cont.saved_ec);
3159}
3160
3161/*
3162 * call-seq:
3163 * fiber.transfer(args, ...) -> obj
3164 *
3165 * Transfer control to another fiber, resuming it from where it last
3166 * stopped or starting it if it was not resumed before. The calling
3167 * fiber will be suspended much like in a call to
3168 * Fiber.yield.
3169 *
3170 * The fiber which receives the transfer call treats it much like
3171 * a resume call. Arguments passed to transfer are treated like those
3172 * passed to resume.
3173 *
3174 * The two style of control passing to and from fiber (one is #resume and
3175 * Fiber::yield, another is #transfer to and from fiber) can't be freely
3176 * mixed.
3177 *
3178 * * If the Fiber's lifecycle had started with transfer, it will never
3179 * be able to yield or be resumed control passing, only
3180 * finish or transfer back. (It still can resume other fibers that
3181 * are allowed to be resumed.)
3182 * * If the Fiber's lifecycle had started with resume, it can yield
3183 * or transfer to another Fiber, but can receive control back only
3184 * the way compatible with the way it was given away: if it had
3185 * transferred, it only can be transferred back, and if it had
3186 * yielded, it only can be resumed back. After that, it again can
3187 * transfer or yield.
3188 *
3189 * If those rules are broken FiberError is raised.
3190 *
3191 * For an individual Fiber design, yield/resume is easier to use
3192 * (the Fiber just gives away control, it doesn't need to think
3193 * about who the control is given to), while transfer is more flexible
3194 * for complex cases, allowing to build arbitrary graphs of Fibers
3195 * dependent on each other.
3196 *
3197 *
3198 * Example:
3199 *
3200 * manager = nil # For local var to be visible inside worker block
3201 *
3202 * # This fiber would be started with transfer
3203 * # It can't yield, and can't be resumed
3204 * worker = Fiber.new { |work|
3205 * puts "Worker: starts"
3206 * puts "Worker: Performed #{work.inspect}, transferring back"
3207 * # Fiber.yield # this would raise FiberError: attempt to yield on a not resumed fiber
3208 * # manager.resume # this would raise FiberError: attempt to resume a resumed fiber (double resume)
3209 * manager.transfer(work.capitalize)
3210 * }
3211 *
3212 * # This fiber would be started with resume
3213 * # It can yield or transfer, and can be transferred
3214 * # back or resumed
3215 * manager = Fiber.new {
3216 * puts "Manager: starts"
3217 * puts "Manager: transferring 'something' to worker"
3218 * result = worker.transfer('something')
3219 * puts "Manager: worker returned #{result.inspect}"
3220 * # worker.resume # this would raise FiberError: attempt to resume a transferring fiber
3221 * Fiber.yield # this is OK, the fiber transferred from and to, now it can yield
3222 * puts "Manager: finished"
3223 * }
3224 *
3225 * puts "Starting the manager"
3226 * manager.resume
3227 * puts "Resuming the manager"
3228 * # manager.transfer # this would raise FiberError: attempt to transfer to a yielding fiber
3229 * manager.resume
3230 *
3231 * <em>produces</em>
3232 *
3233 * Starting the manager
3234 * Manager: starts
3235 * Manager: transferring 'something' to worker
3236 * Worker: starts
3237 * Worker: Performed "something", transferring back
3238 * Manager: worker returned "Something"
3239 * Resuming the manager
3240 * Manager: finished
3241 *
3242 */
3243static VALUE
3244rb_fiber_m_transfer(int argc, VALUE *argv, VALUE self)
3245{
3246 return rb_fiber_transfer_kw(self, argc, argv, rb_keyword_given_p());
3247}
3248
3249static VALUE
3250fiber_transfer_kw(rb_fiber_t *fiber, int argc, const VALUE *argv, int kw_splat)
3251{
3252 if (fiber->resuming_fiber) {
3253 rb_raise(rb_eFiberError, "attempt to transfer to a resuming fiber");
3254 }
3255
3256 if (fiber->yielding) {
3257 rb_raise(rb_eFiberError, "attempt to transfer to a yielding fiber");
3258 }
3259
3260 return fiber_switch(fiber, argc, argv, kw_splat, NULL, false);
3261}
3262
3263 VALUE
3264rb_fiber_transfer_kw(VALUE self, int argc, const VALUE *argv, int kw_splat)
3265{
3266 return fiber_transfer_kw(fiber_ptr(self), argc, argv, kw_splat);
3267}
3268
3269/*
3270 * call-seq:
3271 * Fiber.yield(args, ...) -> obj
3272 *
3273 * Yields control back to the context that resumed the fiber, passing
3274 * along any arguments that were passed to it. The fiber will resume
3275 * processing at this point when #resume is called next.
3276 * Any arguments passed to the next #resume will be the value that
3277 * this Fiber.yield expression evaluates to.
3278 */
3279static VALUE
3280rb_fiber_s_yield(int argc, VALUE *argv, VALUE klass)
3281{
3282 return rb_fiber_yield_kw(argc, argv, rb_keyword_given_p());
3283}
3284
3285static VALUE
3286fiber_raise(rb_fiber_t *fiber, VALUE exception)
3287{
3288 if (fiber == fiber_current()) {
3289 rb_exc_raise(exception);
3290 }
3291 else if (fiber->resuming_fiber) {
3292 return fiber_raise(fiber->resuming_fiber, exception);
3293 }
3294 else if (FIBER_SUSPENDED_P(fiber) && !fiber->yielding) {
3295 return fiber_transfer_kw(fiber, -1, &exception, RB_NO_KEYWORDS);
3296 }
3297 else {
3298 return fiber_resume_kw(fiber, -1, &exception, RB_NO_KEYWORDS);
3299 }
3300}
3301
3302 VALUE
3303rb_fiber_raise(VALUE fiber, int argc, VALUE *argv)
3304{
3305 VALUE exception = rb_exception_setup(argc, argv);
3306
3307 return fiber_raise(fiber_ptr(fiber), exception);
3308}
3309
3310/*
3311 * call-seq:
3312 * raise(exception, message = exception.to_s, backtrace = nil, cause: $!)
3313 * raise(message = nil, cause: $!)
3314 *
3315 * Raises an exception in the fiber at the point at which the last
3316 * +Fiber.yield+ was called.
3317 *
3318 * f = Fiber.new {
3319 * puts "Before the yield"
3320 * Fiber.yield 1 # -- exception will be raised here
3321 * puts "After the yield"
3322 * }
3323 *
3324 * p f.resume
3325 * f.raise "Gotcha"
3326 *
3327 * Output
3328 *
3329 * Before the first yield
3330 * 1
3331 * t.rb:8:in 'Fiber.yield': Gotcha (RuntimeError)
3332 * from t.rb:8:in 'block in <main>'
3333 *
3334 * If the fiber has not been started or has
3335 * already run to completion, raises +FiberError+. If the fiber is
3336 * yielding, it is resumed. If it is transferring, it is transferred into.
3337 * But if it is resuming, raises +FiberError+.
3338 *
3339 * Raises +FiberError+ if called on a Fiber belonging to another +Thread+.
3340 *
3341 * See Kernel#raise for more information on arguments.
3342 *
3343 */
3344static VALUE
3345rb_fiber_m_raise(int argc, VALUE *argv, VALUE self)
3346{
3347 return rb_fiber_raise(self, argc, argv);
3348}
3349
3350/*
3351 * call-seq:
3352 * fiber.kill -> nil
3353 *
3354 * Terminates the fiber by raising an uncatchable exception.
3355 * It only terminates the given fiber and no other fiber, returning +nil+ to
3356 * another fiber if that fiber was calling #resume or #transfer.
3357 *
3358 * <tt>Fiber#kill</tt> only interrupts another fiber when it is in Fiber.yield.
3359 * If called on the current fiber then it raises that exception at the <tt>Fiber#kill</tt> call site.
3360 *
3361 * If the fiber has not been started, transition directly to the terminated state.
3362 *
3363 * If the fiber is already terminated, does nothing.
3364 *
3365 * Raises FiberError if called on a fiber belonging to another thread.
3366 */
3367static VALUE
3368rb_fiber_m_kill(VALUE self)
3369{
3370 rb_fiber_t *fiber = fiber_ptr(self);
3371
3372 if (fiber->killed) return Qfalse;
3373 fiber->killed = 1;
3374
3375 if (fiber->status == FIBER_CREATED) {
3376 fiber->status = FIBER_TERMINATED;
3377 }
3378 else if (fiber->status != FIBER_TERMINATED) {
3379 if (fiber_current() == fiber) {
3380 fiber_check_killed(fiber);
3381 }
3382 else {
3383 fiber_raise(fiber_ptr(self), Qnil);
3384 }
3385 }
3386
3387 return self;
3388}
3389
3390/*
3391 * call-seq:
3392 * Fiber.current -> fiber
3393 *
3394 * Returns the current fiber. If you are not running in the context of
3395 * a fiber this method will return the root fiber.
3396 */
3397static VALUE
3398rb_fiber_s_current(VALUE klass)
3399{
3400 return rb_fiber_current();
3401}
3402
3403static VALUE
3404fiber_to_s(VALUE fiber_value)
3405{
3406 const rb_fiber_t *fiber = fiber_ptr(fiber_value);
3407 const rb_proc_t *proc;
3408 char status_info[0x20];
3409
3410 if (fiber->resuming_fiber) {
3411 snprintf(status_info, 0x20, " (%s by resuming)", fiber_status_name(fiber->status));
3412 }
3413 else {
3414 snprintf(status_info, 0x20, " (%s)", fiber_status_name(fiber->status));
3415 }
3416
3417 if (!rb_obj_is_proc(fiber->first_proc)) {
3418 VALUE str = rb_any_to_s(fiber_value);
3419 strlcat(status_info, ">", sizeof(status_info));
3420 rb_str_set_len(str, RSTRING_LEN(str)-1);
3421 rb_str_cat_cstr(str, status_info);
3422 return str;
3423 }
3424 GetProcPtr(fiber->first_proc, proc);
3425 return rb_block_to_s(fiber_value, &proc->block, status_info);
3426}
3427
3428#ifdef HAVE_WORKING_FORK
3429void
3430rb_fiber_atfork(rb_thread_t *th)
3431{
3432 if (th->root_fiber) {
3433 if (&th->root_fiber->cont.saved_ec != th->ec) {
3434 th->root_fiber = th->ec->fiber_ptr;
3435 }
3436 th->root_fiber->prev = 0;
3437 th->root_fiber->blocking = 1;
3438 th->blocking = 1;
3439 }
3440}
3441#endif
3442
3443#ifdef RB_EXPERIMENTAL_FIBER_POOL
3444static void
3445fiber_pool_free(void *ptr)
3446{
3447 struct fiber_pool * fiber_pool = ptr;
3448 RUBY_FREE_ENTER("fiber_pool");
3449
3450 fiber_pool_allocation_free(fiber_pool->allocations);
3451 ruby_xfree(fiber_pool);
3452
3453 RUBY_FREE_LEAVE("fiber_pool");
3454}
3455
3456static size_t
3457fiber_pool_memsize(const void *ptr)
3458{
3459 const struct fiber_pool * fiber_pool = ptr;
3460 size_t size = sizeof(*fiber_pool);
3461
3462 size += fiber_pool->count * fiber_pool->size;
3463
3464 return size;
3465}
3466
3467static const rb_data_type_t FiberPoolDataType = {
3468 "fiber_pool",
3469 {NULL, fiber_pool_free, fiber_pool_memsize,},
3470 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
3471};
3472
3473static VALUE
3474fiber_pool_alloc(VALUE klass)
3475{
3476 struct fiber_pool *fiber_pool;
3477
3478 return TypedData_Make_Struct(klass, struct fiber_pool, &FiberPoolDataType, fiber_pool);
3479}
3480
3481static VALUE
3482rb_fiber_pool_initialize(int argc, VALUE* argv, VALUE self)
3483{
3484 rb_thread_t *th = GET_THREAD();
3485 VALUE size = Qnil, count = Qnil, vm_stack_size = Qnil;
3486 struct fiber_pool * fiber_pool = NULL;
3487
3488 // Maybe these should be keyword arguments.
3489 rb_scan_args(argc, argv, "03", &size, &count, &vm_stack_size);
3490
3491 if (NIL_P(size)) {
3492 size = SIZET2NUM(th->vm->default_params.fiber_machine_stack_size);
3493 }
3494
3495 if (NIL_P(count)) {
3496 count = INT2NUM(128);
3497 }
3498
3499 if (NIL_P(vm_stack_size)) {
3500 vm_stack_size = SIZET2NUM(th->vm->default_params.fiber_vm_stack_size);
3501 }
3502
3503 TypedData_Get_Struct(self, struct fiber_pool, &FiberPoolDataType, fiber_pool);
3504
3505 fiber_pool_initialize(fiber_pool, NUM2SIZET(size), NUM2SIZET(count), NUM2SIZET(vm_stack_size));
3506
3507 return self;
3508}
3509#endif
3510
3511/*
3512 * Document-class: FiberError
3513 *
3514 * Raised when an invalid operation is attempted on a Fiber, in
3515 * particular when attempting to call/resume a dead fiber,
3516 * attempting to yield from the root fiber, or calling a fiber across
3517 * threads.
3518 *
3519 * fiber = Fiber.new{}
3520 * fiber.resume #=> nil
3521 * fiber.resume #=> FiberError: dead fiber called
3522 */
3523
3524void
3525Init_Cont(void)
3526{
3527 rb_thread_t *th = GET_THREAD();
3528 size_t vm_stack_size = th->vm->default_params.fiber_vm_stack_size;
3529 size_t machine_stack_size = th->vm->default_params.fiber_machine_stack_size;
3530 size_t stack_size = machine_stack_size + vm_stack_size;
3531
3532#ifdef _WIN32
3533 SYSTEM_INFO info;
3534 GetSystemInfo(&info);
3535 pagesize = info.dwPageSize;
3536#else /* not WIN32 */
3537 pagesize = sysconf(_SC_PAGESIZE);
3538#endif
3539 SET_MACHINE_STACK_END(&th->ec->machine.stack_end);
3540
3541 fiber_pool_initialize(&shared_fiber_pool, stack_size, FIBER_POOL_INITIAL_SIZE, vm_stack_size);
3542
3543 fiber_initialize_keywords[0] = rb_intern_const("blocking");
3544 fiber_initialize_keywords[1] = rb_intern_const("pool");
3545 fiber_initialize_keywords[2] = rb_intern_const("storage");
3546
3547 const char *fiber_shared_fiber_pool_free_stacks = getenv("RUBY_SHARED_FIBER_POOL_FREE_STACKS");
3548 if (fiber_shared_fiber_pool_free_stacks) {
3549 shared_fiber_pool.free_stacks = atoi(fiber_shared_fiber_pool_free_stacks);
3550
3551 if (shared_fiber_pool.free_stacks < 0) {
3552 rb_warn("Setting RUBY_SHARED_FIBER_POOL_FREE_STACKS to a negative value is not allowed.");
3553 shared_fiber_pool.free_stacks = 0;
3554 }
3555
3556 if (shared_fiber_pool.free_stacks > 1) {
3557 rb_warn("Setting RUBY_SHARED_FIBER_POOL_FREE_STACKS to a value greater than 1 is operating system specific, and may cause crashes.");
3558 }
3559 }
3560
3561 rb_cFiber = rb_define_class("Fiber", rb_cObject);
3562 rb_define_alloc_func(rb_cFiber, fiber_alloc);
3563 rb_eFiberError = rb_define_class("FiberError", rb_eStandardError);
3564 rb_define_singleton_method(rb_cFiber, "yield", rb_fiber_s_yield, -1);
3565 rb_define_singleton_method(rb_cFiber, "current", rb_fiber_s_current, 0);
3566 rb_define_singleton_method(rb_cFiber, "blocking", rb_fiber_blocking, 0);
3567 rb_define_singleton_method(rb_cFiber, "[]", rb_fiber_storage_aref, 1);
3568 rb_define_singleton_method(rb_cFiber, "[]=", rb_fiber_storage_aset, 2);
3569
3570 rb_define_method(rb_cFiber, "initialize", rb_fiber_initialize, -1);
3571 rb_define_method(rb_cFiber, "blocking?", rb_fiber_blocking_p, 0);
3572 rb_define_method(rb_cFiber, "storage", rb_fiber_storage_get, 0);
3573 rb_define_method(rb_cFiber, "storage=", rb_fiber_storage_set, 1);
3574 rb_define_method(rb_cFiber, "resume", rb_fiber_m_resume, -1);
3575 rb_define_method(rb_cFiber, "raise", rb_fiber_m_raise, -1);
3576 rb_define_method(rb_cFiber, "kill", rb_fiber_m_kill, 0);
3577 rb_define_method(rb_cFiber, "backtrace", rb_fiber_backtrace, -1);
3578 rb_define_method(rb_cFiber, "backtrace_locations", rb_fiber_backtrace_locations, -1);
3579 rb_define_method(rb_cFiber, "to_s", fiber_to_s, 0);
3580 rb_define_alias(rb_cFiber, "inspect", "to_s");
3581 rb_define_method(rb_cFiber, "transfer", rb_fiber_m_transfer, -1);
3582 rb_define_method(rb_cFiber, "alive?", rb_fiber_alive_p, 0);
3583
3584 rb_define_singleton_method(rb_cFiber, "blocking?", rb_fiber_s_blocking_p, 0);
3585 rb_define_singleton_method(rb_cFiber, "scheduler", rb_fiber_s_scheduler, 0);
3586 rb_define_singleton_method(rb_cFiber, "set_scheduler", rb_fiber_set_scheduler, 1);
3587 rb_define_singleton_method(rb_cFiber, "current_scheduler", rb_fiber_current_scheduler, 0);
3588
3589 rb_define_singleton_method(rb_cFiber, "schedule", rb_fiber_s_schedule, -1);
3590
3591#ifdef RB_EXPERIMENTAL_FIBER_POOL
3592 /*
3593 * Document-class: Fiber::Pool
3594 * :nodoc: experimental
3595 */
3596 rb_cFiberPool = rb_define_class_under(rb_cFiber, "Pool", rb_cObject);
3597 rb_define_alloc_func(rb_cFiberPool, fiber_pool_alloc);
3598 rb_define_method(rb_cFiberPool, "initialize", rb_fiber_pool_initialize, -1);
3599#endif
3600
3601 rb_provide("fiber.so");
3602}
3603
3604RUBY_SYMBOL_EXPORT_BEGIN
3605
3606void
3607ruby_Init_Continuation_body(void)
3608{
3609 rb_cContinuation = rb_define_class("Continuation", rb_cObject);
3610 rb_undef_alloc_func(rb_cContinuation);
3611 rb_undef_method(CLASS_OF(rb_cContinuation), "new");
3612 rb_define_method(rb_cContinuation, "call", rb_cont_call, -1);
3613 rb_define_method(rb_cContinuation, "[]", rb_cont_call, -1);
3614 rb_define_global_function("callcc", rb_callcc, 0);
3615}
3616
3617RUBY_SYMBOL_EXPORT_END
rb_define_method
#define rb_define_method(klass, mid, func, arity)
Defines klass#mid.
Definition cxxanyargs.hpp:670
rb_define_singleton_method
#define rb_define_singleton_method(klass, mid, func, arity)
Defines klass.mid.
Definition cxxanyargs.hpp:685
rb_define_global_function
#define rb_define_global_function(mid, func, arity)
Defines rb_mKernel #mid.
Definition cxxanyargs.hpp:695
RUBY_EVENT_FIBER_SWITCH
#define RUBY_EVENT_FIBER_SWITCH
Encountered a Fiber#yield.
Definition event.h:59
RB_OBJ_FROZEN
static bool RB_OBJ_FROZEN(VALUE obj)
Checks if an object is frozen.
Definition fl_type.h:892
rb_define_class
VALUE rb_define_class(const char *name, VALUE super)
Defines a top-level class.
Definition class.c:1478
rb_define_class_under
VALUE rb_define_class_under(VALUE outer, const char *name, VALUE super)
Defines a class under the namespace of outer.
Definition class.c:1509
rb_define_alias
void rb_define_alias(VALUE klass, const char *name1, const char *name2)
Defines an alias of a method.
Definition class.c:2843
rb_undef_method
void rb_undef_method(VALUE klass, const char *name)
Defines an undef of a method.
Definition class.c:2655
rb_scan_args_kw
int rb_scan_args_kw(int kw_flag, int argc, const VALUE *argv, const char *fmt,...)
Identical to rb_scan_args(), except it also accepts kw_splat.
Definition class.c:3146
rb_scan_args
int rb_scan_args(int argc, const VALUE *argv, const char *fmt,...)
Retrieves argument from argc and argv to given VALUE references according to the format string.
Definition class.c:3133
rb_keyword_given_p
int rb_keyword_given_p(void)
Determines if the current method is given a keyword argument.
Definition eval.c:1023
rb_get_kwargs
int rb_get_kwargs(VALUE keyword_hash, const ID *table, int required, int optional, VALUE *values)
Keyword argument deconstructor.
Definition class.c:2922
REALLOC_N
#define REALLOC_N
Old name of RB_REALLOC_N.
Definition memory.h:403
xfree
#define xfree
Old name of ruby_xfree.
Definition xmalloc.h:58
Qundef
#define Qundef
Old name of RUBY_Qundef.
Definition special_consts.h:62
UNREACHABLE_RETURN
#define UNREACHABLE_RETURN
Old name of RBIMPL_UNREACHABLE_RETURN.
Definition assume.h:29
ZALLOC
#define ZALLOC
Old name of RB_ZALLOC.
Definition memory.h:402
CLASS_OF
#define CLASS_OF
Old name of rb_class_of.
Definition globals.h:205
rb_ary_new4
#define rb_ary_new4
Old name of rb_ary_new_from_values.
Definition array.h:659
SIZET2NUM
#define SIZET2NUM
Old name of RB_SIZE2NUM.
Definition size_t.h:62
rb_exc_new2
#define rb_exc_new2
Old name of rb_exc_new_cstr.
Definition error.h:37
T_HASH
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
ALLOC_N
#define ALLOC_N
Old name of RB_ALLOC_N.
Definition memory.h:399
Qtrue
#define Qtrue
Old name of RUBY_Qtrue.
Definition special_consts.h:61
INT2NUM
#define INT2NUM
Old name of RB_INT2NUM.
Definition int.h:43
Qnil
#define Qnil
Old name of RUBY_Qnil.
Definition special_consts.h:60
Qfalse
#define Qfalse
Old name of RUBY_Qfalse.
Definition special_consts.h:59
NIL_P
#define NIL_P
Old name of RB_NIL_P.
Definition special_consts.h:55
T_SYMBOL
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition value_type.h:80
NUM2SIZET
#define NUM2SIZET
Old name of RB_NUM2SIZE.
Definition size_t.h:61
ruby_stop
void ruby_stop(int ex)
Calls ruby_cleanup() and exits the process.
Definition eval.c:290
rb_category_warn
void rb_category_warn(rb_warning_category_t category, const char *fmt,...)
Identical to rb_category_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:476
rb_exc_raise
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
Definition eval.c:653
rb_typeddata_is_kind_of
int rb_typeddata_is_kind_of(VALUE obj, const rb_data_type_t *data_type)
Checks if the given object is of given kind.
Definition error.c:1381
rb_syserr_fail
void rb_syserr_fail(int e, const char *mesg)
Raises appropriate exception that represents a C errno.
Definition error.c:3909
rb_eStandardError
VALUE rb_eStandardError
StandardError exception.
Definition error.c:1428
rb_eFrozenError
VALUE rb_eFrozenError
FrozenError exception.
Definition error.c:1430
rb_eTypeError
VALUE rb_eTypeError
TypeError exception.
Definition error.c:1431
rb_eRuntimeError
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1429
rb_warn
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:466
RB_WARN_CATEGORY_EXPERIMENTAL
@ RB_WARN_CATEGORY_EXPERIMENTAL
Warning is for experimental features.
Definition error.h:51
rb_any_to_s
VALUE rb_any_to_s(VALUE obj)
Generates a textual representation of the given object.
Definition object.c:675
rb_obj_dup
VALUE rb_obj_dup(VALUE obj)
Duplicates the given object.
Definition object.c:582
rb_memerror
void rb_memerror(void)
Triggers out-of-memory error.
Definition gc.c:5075
rb_gc
void rb_gc(void)
Triggers a GC process.
Definition gc.c:4260
rb_fiber_current
VALUE rb_fiber_current(void)
Queries the fiber which is calling this function.
Definition cont.c:2703
rb_hash_new
VALUE rb_hash_new(void)
Creates a new, empty hash object.
Definition hash.c:1464
rb_provide
void rb_provide(const char *feature)
Declares that the given feature is already provided by someone else.
Definition load.c:695
rb_block_proc
VALUE rb_block_proc(void)
Constructs a Proc object from implicitly passed components.
Definition proc.c:983
rb_obj_is_proc
VALUE rb_obj_is_proc(VALUE recv)
Queries if the given object is a proc.
Definition proc.c:120
rb_str_set_len
void rb_str_set_len(VALUE str, long len)
Overwrites the length of the string.
Definition string.c:3389
rb_str_cat_cstr
#define rb_str_cat_cstr(buf, str)
Identical to rb_str_cat(), except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1657
rb_undef_alloc_func
void rb_undef_alloc_func(VALUE klass)
Deletes the allocator function of a class.
Definition vm_method.c:1664
rb_define_alloc_func
void rb_define_alloc_func(VALUE klass, rb_alloc_func_t func)
Sets the allocator function of a class.
rb_intern_const
static ID rb_intern_const(const char *str)
This is a "tiny optimisation" over rb_intern().
Definition symbol.h:285
rb_to_symbol
VALUE rb_to_symbol(VALUE name)
Identical to rb_intern_str(), except it generates a dynamic symbol if necessary.
Definition string.c:12674
rb_yield
VALUE rb_yield(VALUE val)
Yields the block.
Definition vm_eval.c:1372
rb_block_call_func_t
rb_block_call_func * rb_block_call_func_t
Shorthand type that represents an iterator-written-in-C function pointer.
Definition iterator.h:88
MEMCPY
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:372
ALLOCA_N
#define ALLOCA_N(type, n)
Definition memory.h:292
RB_ALLOC
#define RB_ALLOC(type)
Shorthand of RB_ALLOC_N with n=1.
Definition memory.h:213
ruby::backward::cxxanyargs::rb_proc_new
VALUE rb_proc_new(type *q, VALUE w)
Creates a rb_cProc instance.
Definition cxxanyargs.hpp:394
ruby::backward::cxxanyargs::rb_hash_foreach
void rb_hash_foreach(VALUE q, int_type *w, VALUE e)
Iteration over the given hash.
Definition cxxanyargs.hpp:482
ruby::backward::cxxanyargs::rb_ensure
VALUE rb_ensure(type *q, VALUE w, type *e, VALUE r)
An equivalent of ensure clause.
Definition cxxanyargs.hpp:310
RARRAY_CONST_PTR
#define RARRAY_CONST_PTR
Just another name of rb_array_const_ptr.
Definition rarray.h:52
DATA_PTR
#define DATA_PTR(obj)
Convenient getter macro.
Definition rdata.h:67
TypedData_Get_Struct
#define TypedData_Get_Struct(obj, type, data_type, sval)
Obtains a C struct from inside of a wrapper Ruby object.
Definition rtypeddata.h:649
TypedData_Wrap_Struct
#define TypedData_Wrap_Struct(klass, data_type, sval)
Converts sval, a pointer to your struct, into a Ruby object.
Definition rtypeddata.h:461
rb_data_type_t
struct rb_data_type_struct rb_data_type_t
This is the struct that holds necessary info for a struct.
Definition rtypeddata.h:205
TypedData_Make_Struct
#define TypedData_Make_Struct(klass, type, data_type, sval)
Identical to TypedData_Wrap_Struct, except it allocates a new data region internally instead of takin...
Definition rtypeddata.h:508
errno
#define errno
Ractor-aware version of errno.
Definition ruby.h:388
RB_NO_KEYWORDS
#define RB_NO_KEYWORDS
Do not pass keywords.
Definition scan_args.h:69
scheduler.h
Scheduler APIs.
rb_fiber_scheduler_current
VALUE rb_fiber_scheduler_current(void)
Identical to rb_fiber_scheduler_get(), except it also returns RUBY_Qnil in case of a blocking fiber.
Definition scheduler.c:471
rb_fiber_scheduler_set
VALUE rb_fiber_scheduler_set(VALUE scheduler)
Destructively assigns the passed scheduler to that of the current thread that is calling this functio...
Definition scheduler.c:433
rb_fiber_scheduler_get
VALUE rb_fiber_scheduler_get(void)
Queries the current scheduler of the current thread that is calling this function.
Definition scheduler.c:383
rb_fiber_scheduler_fiber
VALUE rb_fiber_scheduler_fiber(VALUE scheduler, int argc, VALUE *argv, int kw_splat)
Create and schedule a non-blocking fiber.
Definition scheduler.c:1191
RTEST
#define RTEST
This is an old name of RB_TEST.
Definition special_consts.h:51
cont_saved_vm_stack
Definition cont.c:79
coroutine_context
Definition Context.h:36
fiber_pool_allocation
Definition cont.c:148
fiber_pool_stack
Definition cont.c:90
fiber_pool_vacancy
Definition cont.c:113
fiber_pool
Definition cont.c:176
rb_context_struct
Definition cont.c:213
rb_execution_context_struct
Definition vm_core.h:1033
rb_fiber_struct
Definition cont.c:258
rb_jit_cont
Definition cont.c:205
rb_proc_t
Definition vm_core.h:1287
rb_vm_tag
Definition vm_core.h:1004
rb_native_mutex_lock
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
rb_native_mutex_initialize
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
rb_native_mutex_unlock
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
rb_native_mutex_destroy
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
ID
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
Check_Type
static void Check_Type(VALUE v, enum ruby_value_type t)
Identical to RB_TYPE_P(), except it raises exceptions on predication failure.
Definition value_type.h:433
RB_TYPE_P
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:376