Ruby: thread_pthread.c Source File

Ruby 4.0.5p0 (2026-05-20 revision 64336ffd0ee9e1f4c05891695a3d7b49cb709721)

thread_pthread.c
1/* -*-c-*- */
2/**********************************************************************
3
4 thread_pthread.c -
5
6 $Author$
7
8 Copyright (C) 2004-2007 Koichi Sasada
9
10**********************************************************************/
11
12#ifdef THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION
13
14#include "internal/gc.h"
15#include "internal/sanitizers.h"
16
17#ifdef HAVE_SYS_RESOURCE_H
18#include <sys/resource.h>
19#endif
20#ifdef HAVE_THR_STKSEGMENT
21#include <thread.h>
22#endif
23#if defined(HAVE_FCNTL_H)
24#include <fcntl.h>
25#elif defined(HAVE_SYS_FCNTL_H)
26#include <sys/fcntl.h>
27#endif
28#ifdef HAVE_SYS_PRCTL_H
29#include <sys/prctl.h>
30#endif
31#if defined(HAVE_SYS_TIME_H)
32#include <sys/time.h>
33#endif
34#if defined(__HAIKU__)
35#include <kernel/OS.h>
36#endif
37#ifdef __linux__
38#include <sys/syscall.h> /* for SYS_gettid */
39#endif
40#include <time.h>
41#include <signal.h>
42
43#if defined __APPLE__
44# include <AvailabilityMacros.h>
45#endif
46
47#if defined(HAVE_SYS_EVENTFD_H) && defined(HAVE_EVENTFD)
48# define USE_EVENTFD (1)
49# include <sys/eventfd.h>
50#else
51# define USE_EVENTFD (0)
52#endif
53
54#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \
55 defined(CLOCK_REALTIME) && defined(CLOCK_MONOTONIC) && \
56 defined(HAVE_CLOCK_GETTIME)
57static pthread_condattr_t condattr_mono;
58static pthread_condattr_t *condattr_monotonic = &condattr_mono;
59#else
60static const void *const condattr_monotonic = NULL;
61#endif
62
63#include COROUTINE_H
64
65#ifndef HAVE_SYS_EVENT_H
66#define HAVE_SYS_EVENT_H 0
67#endif
68
69#ifndef HAVE_SYS_EPOLL_H
70#define HAVE_SYS_EPOLL_H 0
71#else
72// force setting for debug
73// #undef HAVE_SYS_EPOLL_H
74// #define HAVE_SYS_EPOLL_H 0
75#endif
76
77#ifndef USE_MN_THREADS
78 #if defined(__EMSCRIPTEN__) || defined(COROUTINE_PTHREAD_CONTEXT)
79 // on __EMSCRIPTEN__ provides epoll* declarations, but no implementations.
80 // on COROUTINE_PTHREAD_CONTEXT, it doesn't worth to use it.
81 #define USE_MN_THREADS 0
82 #elif HAVE_SYS_EPOLL_H
83 #include <sys/epoll.h>
84 #define USE_MN_THREADS 1
85 #elif HAVE_SYS_EVENT_H
86 #include <sys/event.h>
87 #define USE_MN_THREADS 1
88 #else
89 #define USE_MN_THREADS 0
90 #endif
91#endif
92
93#ifdef HAVE_SCHED_YIELD
94#define native_thread_yield() (void)sched_yield()
95#else
96#define native_thread_yield() ((void)0)
97#endif
98
99// native thread wrappers
100
101#define NATIVE_MUTEX_LOCK_DEBUG 0
102#define NATIVE_MUTEX_LOCK_DEBUG_YIELD 0
103
104static void
105mutex_debug(const char *msg, void *lock)
106{
107 if (NATIVE_MUTEX_LOCK_DEBUG) {
108 int r;
109 static pthread_mutex_t dbglock = PTHREAD_MUTEX_INITIALIZER;
110
111 if ((r = pthread_mutex_lock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
112 fprintf(stdout, "%s: %p\n", msg, lock);
113 if ((r = pthread_mutex_unlock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
114 }
115}
116
117void
118 rb_native_mutex_lock(pthread_mutex_t *lock)
119{
120 int r;
121#if NATIVE_MUTEX_LOCK_DEBUG_YIELD
122 native_thread_yield();
123#endif
124 mutex_debug("lock", lock);
125 if ((r = pthread_mutex_lock(lock)) != 0) {
126 rb_bug_errno("pthread_mutex_lock", r);
127 }
128}
129
130void
131 rb_native_mutex_unlock(pthread_mutex_t *lock)
132{
133 int r;
134 mutex_debug("unlock", lock);
135 if ((r = pthread_mutex_unlock(lock)) != 0) {
136 rb_bug_errno("pthread_mutex_unlock", r);
137 }
138}
139
140int
141 rb_native_mutex_trylock(pthread_mutex_t *lock)
142{
143 int r;
144 mutex_debug("trylock", lock);
145 if ((r = pthread_mutex_trylock(lock)) != 0) {
146 if (r == EBUSY) {
147 return EBUSY;
148 }
149 else {
150 rb_bug_errno("pthread_mutex_trylock", r);
151 }
152 }
153 return 0;
154}
155
156void
157 rb_native_mutex_initialize(pthread_mutex_t *lock)
158{
159 int r = pthread_mutex_init(lock, 0);
160 mutex_debug("init", lock);
161 if (r != 0) {
162 rb_bug_errno("pthread_mutex_init", r);
163 }
164}
165
166void
167 rb_native_mutex_destroy(pthread_mutex_t *lock)
168{
169 int r = pthread_mutex_destroy(lock);
170 mutex_debug("destroy", lock);
171 if (r != 0) {
172 rb_bug_errno("pthread_mutex_destroy", r);
173 }
174}
175
176void
177 rb_native_cond_initialize(rb_nativethread_cond_t *cond)
178{
179 int r = pthread_cond_init(cond, condattr_monotonic);
180 if (r != 0) {
181 rb_bug_errno("pthread_cond_init", r);
182 }
183}
184
185void
186 rb_native_cond_destroy(rb_nativethread_cond_t *cond)
187{
188 int r = pthread_cond_destroy(cond);
189 if (r != 0) {
190 rb_bug_errno("pthread_cond_destroy", r);
191 }
192}
193
194/*
195 * In OS X 10.7 (Lion), pthread_cond_signal and pthread_cond_broadcast return
196 * EAGAIN after retrying 8192 times. You can see them in the following page:
197 *
198 * http://www.opensource.apple.com/source/Libc/Libc-763.11/pthreads/pthread_cond.c
199 *
200 * The following rb_native_cond_signal and rb_native_cond_broadcast functions
201 * need to retrying until pthread functions don't return EAGAIN.
202 */
203
204void
205 rb_native_cond_signal(rb_nativethread_cond_t *cond)
206{
207 int r;
208 do {
209 r = pthread_cond_signal(cond);
210 } while (r == EAGAIN);
211 if (r != 0) {
212 rb_bug_errno("pthread_cond_signal", r);
213 }
214}
215
216void
217 rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
218{
219 int r;
220 do {
221 r = pthread_cond_broadcast(cond);
222 } while (r == EAGAIN);
223 if (r != 0) {
224 rb_bug_errno("rb_native_cond_broadcast", r);
225 }
226}
227
228void
229 rb_native_cond_wait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex)
230{
231 int r = pthread_cond_wait(cond, mutex);
232 if (r != 0) {
233 rb_bug_errno("pthread_cond_wait", r);
234 }
235}
236
237static int
238native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex, const rb_hrtime_t *abs)
239{
240 int r;
241 struct timespec ts;
242
243 /*
244 * An old Linux may return EINTR. Even though POSIX says
245 * "These functions shall not return an error code of [EINTR]".
246 * http://pubs.opengroup.org/onlinepubs/009695399/functions/pthread_cond_timedwait.html
247 * Let's hide it from arch generic code.
248 */
249 do {
250 rb_hrtime2timespec(&ts, abs);
251 r = pthread_cond_timedwait(cond, mutex, &ts);
252 } while (r == EINTR);
253
254 if (r != 0 && r != ETIMEDOUT) {
255 rb_bug_errno("pthread_cond_timedwait", r);
256 }
257
258 return r;
259}
260
261static rb_hrtime_t
262native_cond_timeout(rb_nativethread_cond_t *cond, const rb_hrtime_t rel)
263{
264 if (condattr_monotonic) {
265 return rb_hrtime_add(rb_hrtime_now(), rel);
266 }
267 else {
268 struct timespec ts;
269
270 rb_timespec_now(&ts);
271 return rb_hrtime_add(rb_timespec2hrtime(&ts), rel);
272 }
273}
274
275void
276 rb_native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex, unsigned long msec)
277{
278 rb_hrtime_t hrmsec = native_cond_timeout(cond, RB_HRTIME_PER_MSEC * msec);
279 native_cond_timedwait(cond, mutex, &hrmsec);
280}
281
282// thread scheduling
283
284static rb_internal_thread_event_hook_t *rb_internal_thread_event_hooks = NULL;
285static void rb_thread_execute_hooks(rb_event_flag_t event, rb_thread_t *th);
286
287#if 0
288static const char *
289event_name(rb_event_flag_t event)
290{
291 switch (event) {
292 case RUBY_INTERNAL_THREAD_EVENT_STARTED:
293 return "STARTED";
294 case RUBY_INTERNAL_THREAD_EVENT_READY:
295 return "READY";
296 case RUBY_INTERNAL_THREAD_EVENT_RESUMED:
297 return "RESUMED";
298 case RUBY_INTERNAL_THREAD_EVENT_SUSPENDED:
299 return "SUSPENDED";
300 case RUBY_INTERNAL_THREAD_EVENT_EXITED:
301 return "EXITED";
302 }
303 return "no-event";
304}
305
306#define RB_INTERNAL_THREAD_HOOK(event, th) \
307 if (UNLIKELY(rb_internal_thread_event_hooks)) { \
308 fprintf(stderr, "[thread=%"PRIxVALUE"] %s in %s (%s:%d)\n", th->self, event_name(event), __func__, __FILE__, __LINE__); \
309 rb_thread_execute_hooks(event, th); \
310 }
311#else
312#define RB_INTERNAL_THREAD_HOOK(event, th) if (UNLIKELY(rb_internal_thread_event_hooks)) { rb_thread_execute_hooks(event, th); }
313#endif
314
315static rb_serial_t current_fork_gen = 1; /* We can't use GET_VM()->fork_gen */
316
317#if defined(SIGVTALRM) && !defined(__EMSCRIPTEN__)
318# define USE_UBF_LIST 1
319#endif
320
321static void threadptr_trap_interrupt(rb_thread_t *);
322
323static void native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt);
324static void native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt);
325static void native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th);
326
327static void ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r);
328static void timer_thread_wakeup(void);
329static void timer_thread_wakeup_locked(rb_vm_t *vm);
330static void timer_thread_wakeup_force(void);
331static void thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th);
332static void coroutine_transfer0(struct coroutine_context *transfer_from,
333 struct coroutine_context *transfer_to, bool to_dead);
334
335#define thread_sched_dump(s) thread_sched_dump_(__FILE__, __LINE__, s)
336
337static bool
338th_has_dedicated_nt(const rb_thread_t *th)
339{
340 // TODO: th->has_dedicated_nt
341 return th->nt->dedicated > 0;
342}
343
344 RBIMPL_ATTR_MAYBE_UNUSED()
345static void
346thread_sched_dump_(const char *file, int line, struct rb_thread_sched *sched)
347{
348 fprintf(stderr, "@%s:%d running:%d\n", file, line, sched->running ? (int)sched->running->serial : -1);
349 rb_thread_t *th;
350 int i = 0;
351 ccan_list_for_each(&sched->readyq, th, sched.node.readyq) {
352 i++; if (i>10) rb_bug("too many");
353 fprintf(stderr, " ready:%d (%sNT:%d)\n", th->serial,
354 th->nt ? (th->nt->dedicated ? "D" : "S") : "x",
355 th->nt ? (int)th->nt->serial : -1);
356 }
357}
358
359#define ractor_sched_dump(s) ractor_sched_dump_(__FILE__, __LINE__, s)
360
361 RBIMPL_ATTR_MAYBE_UNUSED()
362static void
363ractor_sched_dump_(const char *file, int line, rb_vm_t *vm)
364{
365 rb_ractor_t *r;
366
367 fprintf(stderr, "ractor_sched_dump %s:%d\n", file, line);
368
369 int i = 0;
370 ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) {
371 i++;
372 if (i>10) rb_bug("!!");
373 fprintf(stderr, " %d ready:%d\n", i, rb_ractor_id(r));
374 }
375}
376
377#define thread_sched_lock(a, b) thread_sched_lock_(a, b, __FILE__, __LINE__)
378#define thread_sched_unlock(a, b) thread_sched_unlock_(a, b, __FILE__, __LINE__)
379
380static void
381thread_sched_set_locked(struct rb_thread_sched *sched, rb_thread_t *th)
382{
383#if VM_CHECK_MODE > 0
384 VM_ASSERT(sched->lock_owner == NULL);
385
386 sched->lock_owner = th;
387#endif
388}
389
390static void
391thread_sched_set_unlocked(struct rb_thread_sched *sched, rb_thread_t *th)
392{
393#if VM_CHECK_MODE > 0
394 VM_ASSERT(sched->lock_owner == th);
395
396 sched->lock_owner = NULL;
397#endif
398}
399
400static void
401thread_sched_lock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line)
402{
403 rb_native_mutex_lock(&sched->lock_);
404
405#if VM_CHECK_MODE
406 RUBY_DEBUG_LOG2(file, line, "r:%d th:%u", th ? (int)rb_ractor_id(th->ractor) : -1, rb_th_serial(th));
407#else
408 RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th));
409#endif
410
411 thread_sched_set_locked(sched, th);
412}
413
414static void
415thread_sched_unlock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line)
416{
417 RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th));
418
419 thread_sched_set_unlocked(sched, th);
420
421 rb_native_mutex_unlock(&sched->lock_);
422}
423
424static void
425ASSERT_thread_sched_locked(struct rb_thread_sched *sched, rb_thread_t *th)
426{
427 VM_ASSERT(rb_native_mutex_trylock(&sched->lock_) == EBUSY);
428
429#if VM_CHECK_MODE
430 if (th) {
431 VM_ASSERT(sched->lock_owner == th);
432 }
433 else {
434 VM_ASSERT(sched->lock_owner != NULL);
435 }
436#endif
437}
438
439#define ractor_sched_lock(a, b) ractor_sched_lock_(a, b, __FILE__, __LINE__)
440#define ractor_sched_unlock(a, b) ractor_sched_unlock_(a, b, __FILE__, __LINE__)
441
442 RBIMPL_ATTR_MAYBE_UNUSED()
443static unsigned int
444rb_ractor_serial(const rb_ractor_t *r)
445{
446 if (r) {
447 return rb_ractor_id(r);
448 }
449 else {
450 return 0;
451 }
452}
453
454static void
455ractor_sched_set_locked(rb_vm_t *vm, rb_ractor_t *cr)
456{
457#if VM_CHECK_MODE > 0
458 VM_ASSERT(vm->ractor.sched.lock_owner == NULL);
459 VM_ASSERT(vm->ractor.sched.locked == false);
460
461 vm->ractor.sched.lock_owner = cr;
462 vm->ractor.sched.locked = true;
463#endif
464}
465
466static void
467ractor_sched_set_unlocked(rb_vm_t *vm, rb_ractor_t *cr)
468{
469#if VM_CHECK_MODE > 0
470 VM_ASSERT(vm->ractor.sched.locked);
471 VM_ASSERT(vm->ractor.sched.lock_owner == cr);
472
473 vm->ractor.sched.locked = false;
474 vm->ractor.sched.lock_owner = NULL;
475#endif
476}
477
478static void
479ractor_sched_lock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line)
480{
481 rb_native_mutex_lock(&vm->ractor.sched.lock);
482
483#if VM_CHECK_MODE
484 RUBY_DEBUG_LOG2(file, line, "cr:%u prev_owner:%u", rb_ractor_serial(cr), rb_ractor_serial(vm->ractor.sched.lock_owner));
485#else
486 RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr));
487#endif
488
489 ractor_sched_set_locked(vm, cr);
490}
491
492static void
493ractor_sched_unlock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line)
494{
495 RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr));
496
497 ractor_sched_set_unlocked(vm, cr);
498 rb_native_mutex_unlock(&vm->ractor.sched.lock);
499}
500
501static void
502ASSERT_ractor_sched_locked(rb_vm_t *vm, rb_ractor_t *cr)
503{
504 VM_ASSERT(rb_native_mutex_trylock(&vm->ractor.sched.lock) == EBUSY);
505 VM_ASSERT(vm->ractor.sched.locked);
506 VM_ASSERT(cr == NULL || vm->ractor.sched.lock_owner == cr);
507}
508
509 RBIMPL_ATTR_MAYBE_UNUSED()
510static bool
511ractor_sched_running_threads_contain_p(rb_vm_t *vm, rb_thread_t *th)
512{
513 rb_thread_t *rth;
514 ccan_list_for_each(&vm->ractor.sched.running_threads, rth, sched.node.running_threads) {
515 if (rth == th) return true;
516 }
517 return false;
518}
519
520 RBIMPL_ATTR_MAYBE_UNUSED()
521static unsigned int
522ractor_sched_running_threads_size(rb_vm_t *vm)
523{
524 rb_thread_t *th;
525 unsigned int i = 0;
526 ccan_list_for_each(&vm->ractor.sched.running_threads, th, sched.node.running_threads) {
527 i++;
528 }
529 return i;
530}
531
532 RBIMPL_ATTR_MAYBE_UNUSED()
533static unsigned int
534ractor_sched_timeslice_threads_size(rb_vm_t *vm)
535{
536 rb_thread_t *th;
537 unsigned int i = 0;
538 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, th, sched.node.timeslice_threads) {
539 i++;
540 }
541 return i;
542}
543
544 RBIMPL_ATTR_MAYBE_UNUSED()
545static bool
546ractor_sched_timeslice_threads_contain_p(rb_vm_t *vm, rb_thread_t *th)
547{
548 rb_thread_t *rth;
549 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, rth, sched.node.timeslice_threads) {
550 if (rth == th) return true;
551 }
552 return false;
553}
554
555static void ractor_sched_barrier_join_signal_locked(rb_vm_t *vm);
556
557// setup timeslice signals by the timer thread.
558static void
559thread_sched_setup_running_threads(struct rb_thread_sched *sched, rb_ractor_t *cr, rb_vm_t *vm,
560 rb_thread_t *add_th, rb_thread_t *del_th, rb_thread_t *add_timeslice_th)
561{
562#if USE_RUBY_DEBUG_LOG
563 unsigned int prev_running_cnt = vm->ractor.sched.running_cnt;
564#endif
565
566 rb_thread_t *del_timeslice_th;
567
568 if (del_th && sched->is_running_timeslice) {
569 del_timeslice_th = del_th;
570 sched->is_running_timeslice = false;
571 }
572 else {
573 del_timeslice_th = NULL;
574 }
575
576 RUBY_DEBUG_LOG("+:%u -:%u +ts:%u -ts:%u",
577 rb_th_serial(add_th), rb_th_serial(del_th),
578 rb_th_serial(add_timeslice_th), rb_th_serial(del_timeslice_th));
579
580 ractor_sched_lock(vm, cr);
581 {
582 // update running_threads
583 if (del_th) {
584 VM_ASSERT(ractor_sched_running_threads_contain_p(vm, del_th));
585 VM_ASSERT(del_timeslice_th != NULL ||
586 !ractor_sched_timeslice_threads_contain_p(vm, del_th));
587
588 ccan_list_del_init(&del_th->sched.node.running_threads);
589 vm->ractor.sched.running_cnt--;
590
591 if (UNLIKELY(vm->ractor.sched.barrier_waiting)) {
592 ractor_sched_barrier_join_signal_locked(vm);
593 }
594 sched->is_running = false;
595 }
596
597 if (add_th) {
598 if (vm->ractor.sched.barrier_waiting) {
599 // TODO: GC barrier check?
600 RUBY_DEBUG_LOG("barrier_waiting");
601 RUBY_VM_SET_VM_BARRIER_INTERRUPT(add_th->ec);
602 }
603
604 VM_ASSERT(!ractor_sched_running_threads_contain_p(vm, add_th));
605 VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(vm, add_th));
606
607 ccan_list_add(&vm->ractor.sched.running_threads, &add_th->sched.node.running_threads);
608 vm->ractor.sched.running_cnt++;
609 sched->is_running = true;
610 }
611
612 if (add_timeslice_th) {
613 // update timeslice threads
614 int was_empty = ccan_list_empty(&vm->ractor.sched.timeslice_threads);
615 VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(vm, add_timeslice_th));
616 ccan_list_add(&vm->ractor.sched.timeslice_threads, &add_timeslice_th->sched.node.timeslice_threads);
617 sched->is_running_timeslice = true;
618 if (was_empty) {
619 timer_thread_wakeup_locked(vm);
620 }
621 }
622
623 if (del_timeslice_th) {
624 VM_ASSERT(ractor_sched_timeslice_threads_contain_p(vm, del_timeslice_th));
625 ccan_list_del_init(&del_timeslice_th->sched.node.timeslice_threads);
626 }
627
628 VM_ASSERT(ractor_sched_running_threads_size(vm) == vm->ractor.sched.running_cnt);
629 VM_ASSERT(ractor_sched_timeslice_threads_size(vm) <= vm->ractor.sched.running_cnt);
630 }
631 ractor_sched_unlock(vm, cr);
632
633 //RUBY_DEBUG_LOG("+:%u -:%u +ts:%u -ts:%u run:%u->%u",
634 // rb_th_serial(add_th), rb_th_serial(del_th),
635 // rb_th_serial(add_timeslice_th), rb_th_serial(del_timeslice_th),
636 RUBY_DEBUG_LOG("run:%u->%u", prev_running_cnt, vm->ractor.sched.running_cnt);
637}
638
639static void
640thread_sched_add_running_thread(struct rb_thread_sched *sched, rb_thread_t *th)
641{
642 ASSERT_thread_sched_locked(sched, th);
643 VM_ASSERT(sched->running == th);
644
645 rb_vm_t *vm = th->vm;
646 thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL, ccan_list_empty(&sched->readyq) ? NULL : th);
647}
648
649static void
650thread_sched_del_running_thread(struct rb_thread_sched *sched, rb_thread_t *th)
651{
652 ASSERT_thread_sched_locked(sched, th);
653
654 rb_vm_t *vm = th->vm;
655 thread_sched_setup_running_threads(sched, th->ractor, vm, NULL, th, NULL);
656}
657
658void
659rb_add_running_thread(rb_thread_t *th)
660{
661 struct rb_thread_sched *sched = TH_SCHED(th);
662
663 thread_sched_lock(sched, th);
664 {
665 thread_sched_add_running_thread(sched, th);
666 }
667 thread_sched_unlock(sched, th);
668}
669
670void
671rb_del_running_thread(rb_thread_t *th)
672{
673 struct rb_thread_sched *sched = TH_SCHED(th);
674
675 thread_sched_lock(sched, th);
676 {
677 thread_sched_del_running_thread(sched, th);
678 }
679 thread_sched_unlock(sched, th);
680}
681
682// setup current or next running thread
683// sched->running should be set only on this function.
684//
685// if th is NULL, there is no running threads.
686static void
687thread_sched_set_running(struct rb_thread_sched *sched, rb_thread_t *th)
688{
689 RUBY_DEBUG_LOG("th:%u->th:%u", rb_th_serial(sched->running), rb_th_serial(th));
690 VM_ASSERT(sched->running != th);
691
692 sched->running = th;
693}
694
695 RBIMPL_ATTR_MAYBE_UNUSED()
696static bool
697thread_sched_readyq_contain_p(struct rb_thread_sched *sched, rb_thread_t *th)
698{
699 rb_thread_t *rth;
700 ccan_list_for_each(&sched->readyq, rth, sched.node.readyq) {
701 if (rth == th) return true;
702 }
703 return false;
704}
705
706// deque thread from the ready queue.
707// if the ready queue is empty, return NULL.
708//
709// return deque'ed running thread (or NULL).
710static rb_thread_t *
711thread_sched_deq(struct rb_thread_sched *sched)
712{
713 ASSERT_thread_sched_locked(sched, NULL);
714 rb_thread_t *next_th;
715
716 VM_ASSERT(sched->running != NULL);
717
718 if (ccan_list_empty(&sched->readyq)) {
719 next_th = NULL;
720 }
721 else {
722 next_th = ccan_list_pop(&sched->readyq, rb_thread_t, sched.node.readyq);
723
724 VM_ASSERT(sched->readyq_cnt > 0);
725 sched->readyq_cnt--;
726 ccan_list_node_init(&next_th->sched.node.readyq);
727 }
728
729 RUBY_DEBUG_LOG("next_th:%u readyq_cnt:%d", rb_th_serial(next_th), sched->readyq_cnt);
730
731 return next_th;
732}
733
734// enqueue ready thread to the ready queue.
735static void
736thread_sched_enq(struct rb_thread_sched *sched, rb_thread_t *ready_th)
737{
738 ASSERT_thread_sched_locked(sched, NULL);
739 RUBY_DEBUG_LOG("ready_th:%u readyq_cnt:%d", rb_th_serial(ready_th), sched->readyq_cnt);
740
741 VM_ASSERT(sched->running != NULL);
742 VM_ASSERT(!thread_sched_readyq_contain_p(sched, ready_th));
743
744 if (sched->is_running) {
745 if (ccan_list_empty(&sched->readyq)) {
746 // add sched->running to timeslice
747 thread_sched_setup_running_threads(sched, ready_th->ractor, ready_th->vm, NULL, NULL, sched->running);
748 }
749 }
750 else {
751 // ractor_sched lock is needed
752 // VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(ready_th->vm, sched->running));
753 }
754
755 ccan_list_add_tail(&sched->readyq, &ready_th->sched.node.readyq);
756 sched->readyq_cnt++;
757}
758
759// DNT: kick condvar
760// SNT: TODO
761static void
762thread_sched_wakeup_running_thread(struct rb_thread_sched *sched, rb_thread_t *next_th, bool will_switch)
763{
764 ASSERT_thread_sched_locked(sched, NULL);
765 VM_ASSERT(sched->running == next_th);
766
767 if (next_th) {
768 if (next_th->nt) {
769 if (th_has_dedicated_nt(next_th)) {
770 RUBY_DEBUG_LOG("pinning th:%u", next_th->serial);
771 rb_native_cond_signal(&next_th->nt->cond.readyq);
772 }
773 else {
774 // TODO
775 RUBY_DEBUG_LOG("th:%u is already running.", next_th->serial);
776 }
777 }
778 else {
779 if (will_switch) {
780 RUBY_DEBUG_LOG("th:%u (do nothing)", rb_th_serial(next_th));
781 }
782 else {
783 RUBY_DEBUG_LOG("th:%u (enq)", rb_th_serial(next_th));
784 ractor_sched_enq(next_th->vm, next_th->ractor);
785 }
786 }
787 }
788 else {
789 RUBY_DEBUG_LOG("no waiting threads%s", "");
790 }
791}
792
793// waiting -> ready (locked)
794static void
795thread_sched_to_ready_common(struct rb_thread_sched *sched, rb_thread_t *th, bool wakeup, bool will_switch)
796{
797 RUBY_DEBUG_LOG("th:%u running:%u redyq_cnt:%d", rb_th_serial(th), rb_th_serial(sched->running), sched->readyq_cnt);
798
799 VM_ASSERT(sched->running != th);
800 VM_ASSERT(!thread_sched_readyq_contain_p(sched, th));
801 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_READY, th);
802
803 if (sched->running == NULL) {
804 thread_sched_set_running(sched, th);
805 if (wakeup) thread_sched_wakeup_running_thread(sched, th, will_switch);
806 }
807 else {
808 thread_sched_enq(sched, th);
809 }
810}
811
812// waiting -> ready
813//
814// `th` had became "waiting" state by `thread_sched_to_waiting`
815// and `thread_sched_to_ready` enqueue `th` to the thread ready queue.
816 RBIMPL_ATTR_MAYBE_UNUSED()
817static void
818thread_sched_to_ready(struct rb_thread_sched *sched, rb_thread_t *th)
819{
820 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
821
822 thread_sched_lock(sched, th);
823 {
824 thread_sched_to_ready_common(sched, th, true, false);
825 }
826 thread_sched_unlock(sched, th);
827}
828
829// wait until sched->running is `th`.
830static void
831thread_sched_wait_running_turn(struct rb_thread_sched *sched, rb_thread_t *th, bool can_direct_transfer)
832{
833 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
834
835 ASSERT_thread_sched_locked(sched, th);
836 VM_ASSERT(th == rb_ec_thread_ptr(rb_current_ec_noinline()));
837
838 if (th != sched->running) {
839 // already deleted from running threads
840 // VM_ASSERT(!ractor_sched_running_threads_contain_p(th->vm, th)); // need locking
841
842 // wait for execution right
843 rb_thread_t *next_th;
844 while((next_th = sched->running) != th) {
845 if (th_has_dedicated_nt(th)) {
846 RUBY_DEBUG_LOG("(nt) sleep th:%u running:%u", rb_th_serial(th), rb_th_serial(sched->running));
847
848 thread_sched_set_unlocked(sched, th);
849 {
850 RUBY_DEBUG_LOG("nt:%d cond:%p", th->nt->serial, &th->nt->cond.readyq);
851 rb_native_cond_wait(&th->nt->cond.readyq, &sched->lock_);
852 }
853 thread_sched_set_locked(sched, th);
854
855 RUBY_DEBUG_LOG("(nt) wakeup %s", sched->running == th ? "success" : "failed");
856 if (th == sched->running) {
857 rb_ractor_thread_switch(th->ractor, th, false);
858 }
859 }
860 else {
861 // search another ready thread
862 if (can_direct_transfer &&
863 (next_th = sched->running) != NULL &&
864 !next_th->nt // next_th is running or has dedicated nt
865 ) {
866
867 RUBY_DEBUG_LOG("th:%u->%u (direct)", rb_th_serial(th), rb_th_serial(next_th));
868
869 thread_sched_set_unlocked(sched, th);
870 {
871 rb_ractor_set_current_ec(th->ractor, NULL);
872 thread_sched_switch(th, next_th);
873 }
874 thread_sched_set_locked(sched, th);
875 }
876 else {
877 // search another ready ractor
878 struct rb_native_thread *nt = th->nt;
879 native_thread_assign(NULL, th);
880
881 RUBY_DEBUG_LOG("th:%u->%u (ractor scheduling)", rb_th_serial(th), rb_th_serial(next_th));
882
883 thread_sched_set_unlocked(sched, th);
884 {
885 rb_ractor_set_current_ec(th->ractor, NULL);
886 coroutine_transfer0(th->sched.context, nt->nt_context, false);
887 }
888 thread_sched_set_locked(sched, th);
889 }
890
891 VM_ASSERT(rb_current_ec_noinline() == th->ec);
892 }
893 }
894
895 VM_ASSERT(th->nt != NULL);
896 VM_ASSERT(rb_current_ec_noinline() == th->ec);
897 VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none);
898
899 // add th to running threads
900 thread_sched_add_running_thread(sched, th);
901 }
902
903 // VM_ASSERT(ractor_sched_running_threads_contain_p(th->vm, th)); need locking
904 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_RESUMED, th);
905}
906
907// waiting -> ready -> running (locked)
908static void
909thread_sched_to_running_common(struct rb_thread_sched *sched, rb_thread_t *th)
910{
911 RUBY_DEBUG_LOG("th:%u dedicated:%d", rb_th_serial(th), th_has_dedicated_nt(th));
912
913 VM_ASSERT(sched->running != th);
914 VM_ASSERT(th_has_dedicated_nt(th));
915 VM_ASSERT(GET_THREAD() == th);
916
917 native_thread_dedicated_dec(th->vm, th->ractor, th->nt);
918
919 // waiting -> ready
920 thread_sched_to_ready_common(sched, th, false, false);
921
922 if (sched->running == th) {
923 thread_sched_add_running_thread(sched, th);
924 }
925
926 // TODO: check SNT number
927 thread_sched_wait_running_turn(sched, th, false);
928}
929
930// waiting -> ready -> running
931//
932// `th` had been waiting by `thread_sched_to_waiting()`
933// and run a dedicated task (like waitpid and so on).
934// After the dedicated task, this function is called
935// to join a normal thread-scheduling.
936static void
937thread_sched_to_running(struct rb_thread_sched *sched, rb_thread_t *th)
938{
939 thread_sched_lock(sched, th);
940 {
941 thread_sched_to_running_common(sched, th);
942 }
943 thread_sched_unlock(sched, th);
944}
945
946// resume a next thread in the thread ready queue.
947//
948// deque next running thread from the ready thread queue and
949// resume this thread if available.
950//
951// If the next therad has a dedicated native thraed, simply signal to resume.
952// Otherwise, make the ractor ready and other nt will run the ractor and the thread.
953static void
954thread_sched_wakeup_next_thread(struct rb_thread_sched *sched, rb_thread_t *th, bool will_switch)
955{
956 ASSERT_thread_sched_locked(sched, th);
957
958 VM_ASSERT(sched->running == th);
959 VM_ASSERT(sched->running->nt != NULL);
960
961 rb_thread_t *next_th = thread_sched_deq(sched);
962
963 RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th));
964 VM_ASSERT(th != next_th);
965
966 thread_sched_set_running(sched, next_th);
967 VM_ASSERT(next_th == sched->running);
968 thread_sched_wakeup_running_thread(sched, next_th, will_switch);
969
970 if (th != next_th) {
971 thread_sched_del_running_thread(sched, th);
972 }
973}
974
975// running -> dead (locked)
976static void
977thread_sched_to_dead_common(struct rb_thread_sched *sched, rb_thread_t *th)
978{
979 RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated);
980
981 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
982
983 thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th));
984
985 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_EXITED, th);
986}
987
988// running -> dead
989static void
990thread_sched_to_dead(struct rb_thread_sched *sched, rb_thread_t *th)
991{
992 thread_sched_lock(sched, th);
993 {
994 thread_sched_to_dead_common(sched, th);
995 }
996 thread_sched_unlock(sched, th);
997}
998
999// running -> waiting (locked)
1000//
1001// This thread will run dedicated task (th->nt->dedicated++).
1002static void
1003thread_sched_to_waiting_common(struct rb_thread_sched *sched, rb_thread_t *th)
1004{
1005 RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated);
1006
1007 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1008
1009 native_thread_dedicated_inc(th->vm, th->ractor, th->nt);
1010 thread_sched_wakeup_next_thread(sched, th, false);
1011}
1012
1013// running -> waiting
1014//
1015// This thread will run a dedicated task.
1016static void
1017thread_sched_to_waiting(struct rb_thread_sched *sched, rb_thread_t *th)
1018{
1019 thread_sched_lock(sched, th);
1020 {
1021 thread_sched_to_waiting_common(sched, th);
1022 }
1023 thread_sched_unlock(sched, th);
1024}
1025
1026// mini utility func
1027// return true if any there are any interrupts
1028static bool
1029ubf_set(rb_thread_t *th, rb_unblock_function_t *func, void *arg)
1030{
1031 VM_ASSERT(func != NULL);
1032
1033 retry:
1034 if (RUBY_VM_INTERRUPTED(th->ec)) {
1035 RUBY_DEBUG_LOG("interrupted:0x%x", th->ec->interrupt_flag);
1036 return true;
1037 }
1038
1039 rb_native_mutex_lock(&th->interrupt_lock);
1040 {
1041 if (!th->ec->raised_flag && RUBY_VM_INTERRUPTED(th->ec)) {
1042 rb_native_mutex_unlock(&th->interrupt_lock);
1043 goto retry;
1044 }
1045
1046 VM_ASSERT(th->unblock.func == NULL);
1047 th->unblock.func = func;
1048 th->unblock.arg = arg;
1049 }
1050 rb_native_mutex_unlock(&th->interrupt_lock);
1051
1052 return false;
1053}
1054
1055static void
1056ubf_clear(rb_thread_t *th)
1057{
1058 if (th->unblock.func) {
1059 rb_native_mutex_lock(&th->interrupt_lock);
1060 {
1061 th->unblock.func = NULL;
1062 th->unblock.arg = NULL;
1063 }
1064 rb_native_mutex_unlock(&th->interrupt_lock);
1065 }
1066}
1067
1068static void
1069ubf_waiting(void *ptr)
1070{
1071 rb_thread_t *th = (rb_thread_t *)ptr;
1072 struct rb_thread_sched *sched = TH_SCHED(th);
1073
1074 // only once. it is safe because th->interrupt_lock is already acquired.
1075 th->unblock.func = NULL;
1076 th->unblock.arg = NULL;
1077
1078 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
1079
1080 thread_sched_lock(sched, th);
1081 {
1082 if (sched->running == th) {
1083 // not sleeping yet.
1084 }
1085 else {
1086 thread_sched_to_ready_common(sched, th, true, false);
1087 }
1088 }
1089 thread_sched_unlock(sched, th);
1090}
1091
1092// running -> waiting
1093//
1094// This thread will sleep until other thread wakeup the thread.
1095static void
1096thread_sched_to_waiting_until_wakeup(struct rb_thread_sched *sched, rb_thread_t *th)
1097{
1098 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
1099
1100 RB_VM_SAVE_MACHINE_CONTEXT(th);
1101
1102
1103 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1104
1105 thread_sched_lock(sched, th);
1106 {
1107 // NOTE: there's a lock ordering inversion here with the ubf call, but it's benign.
1108 if (ubf_set(th, ubf_waiting, (void *)th)) {
1109 RUBY_DEBUG_LOG("th:%u interrupted", rb_th_serial(th));
1110 }
1111 else {
1112 bool can_direct_transfer = !th_has_dedicated_nt(th);
1113 // NOTE: th->status is set before and after this sleep outside of this function in `sleep_forever`
1114 thread_sched_wakeup_next_thread(sched, th, can_direct_transfer);
1115 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1116 }
1117 }
1118 thread_sched_unlock(sched, th);
1119
1120 ubf_clear(th);
1121}
1122
1123// run another thread in the ready queue.
1124// continue to run if there are no ready threads.
1125static void
1126thread_sched_yield(struct rb_thread_sched *sched, rb_thread_t *th)
1127{
1128 RUBY_DEBUG_LOG("th:%d sched->readyq_cnt:%d", (int)th->serial, sched->readyq_cnt);
1129
1130 thread_sched_lock(sched, th);
1131 {
1132 if (!ccan_list_empty(&sched->readyq)) {
1133 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1134 thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th));
1135 bool can_direct_transfer = !th_has_dedicated_nt(th);
1136 thread_sched_to_ready_common(sched, th, false, can_direct_transfer);
1137 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1138 th->status = THREAD_RUNNABLE;
1139 }
1140 else {
1141 VM_ASSERT(sched->readyq_cnt == 0);
1142 }
1143 }
1144 thread_sched_unlock(sched, th);
1145}
1146
1147void
1148rb_thread_sched_init(struct rb_thread_sched *sched, bool atfork)
1149{
1150 rb_native_mutex_initialize(&sched->lock_);
1151
1152#if VM_CHECK_MODE
1153 sched->lock_owner = NULL;
1154#endif
1155
1156 ccan_list_head_init(&sched->readyq);
1157 sched->readyq_cnt = 0;
1158
1159#if USE_MN_THREADS
1160 if (!atfork) sched->enable_mn_threads = true; // MN is enabled on Ractors
1161#endif
1162}
1163
1164static void
1165coroutine_transfer0(struct coroutine_context *transfer_from, struct coroutine_context *transfer_to, bool to_dead)
1166{
1167#ifdef RUBY_ASAN_ENABLED
1168 void **fake_stack = to_dead ? NULL : &transfer_from->fake_stack;
1169 __sanitizer_start_switch_fiber(fake_stack, transfer_to->stack_base, transfer_to->stack_size);
1170#endif
1171
1172 RBIMPL_ATTR_MAYBE_UNUSED()
1173 struct coroutine_context *returning_from = coroutine_transfer(transfer_from, transfer_to);
1174
1175 /* if to_dead was passed, the caller is promising that this coroutine is finished and it should
1176 * never be resumed! */
1177 VM_ASSERT(!to_dead);
1178#ifdef RUBY_ASAN_ENABLED
1179 __sanitizer_finish_switch_fiber(transfer_from->fake_stack,
1180 (const void**)&returning_from->stack_base, &returning_from->stack_size);
1181#endif
1182
1183}
1184
1185static void
1186thread_sched_switch0(struct coroutine_context *current_cont, rb_thread_t *next_th, struct rb_native_thread *nt, bool to_dead)
1187{
1188 VM_ASSERT(!nt->dedicated);
1189 VM_ASSERT(next_th->nt == NULL);
1190
1191 RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th));
1192
1193 ruby_thread_set_native(next_th);
1194 native_thread_assign(nt, next_th);
1195
1196 coroutine_transfer0(current_cont, next_th->sched.context, to_dead);
1197}
1198
1199static void
1200thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th)
1201{
1202 struct rb_native_thread *nt = cth->nt;
1203 native_thread_assign(NULL, cth);
1204 RUBY_DEBUG_LOG("th:%u->%u on nt:%d", rb_th_serial(cth), rb_th_serial(next_th), nt->serial);
1205 thread_sched_switch0(cth->sched.context, next_th, nt, cth->status == THREAD_KILLED);
1206}
1207
1208#if VM_CHECK_MODE > 0
1209 RBIMPL_ATTR_MAYBE_UNUSED()
1210static unsigned int
1211grq_size(rb_vm_t *vm, rb_ractor_t *cr)
1212{
1213 ASSERT_ractor_sched_locked(vm, cr);
1214
1215 rb_ractor_t *r, *prev_r = NULL;
1216 unsigned int i = 0;
1217
1218 ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) {
1219 i++;
1220
1221 VM_ASSERT(r != prev_r);
1222 prev_r = r;
1223 }
1224 return i;
1225}
1226#endif
1227
1228static void
1229ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r)
1230{
1231 struct rb_thread_sched *sched = &r->threads.sched;
1232 rb_ractor_t *cr = NULL; // timer thread can call this function
1233
1234 VM_ASSERT(sched->running != NULL);
1235 VM_ASSERT(sched->running->nt == NULL);
1236
1237 ractor_sched_lock(vm, cr);
1238 {
1239#if VM_CHECK_MODE > 0
1240 // check if grq contains r
1241 rb_ractor_t *tr;
1242 ccan_list_for_each(&vm->ractor.sched.grq, tr, threads.sched.grq_node) {
1243 VM_ASSERT(r != tr);
1244 }
1245#endif
1246
1247 ccan_list_add_tail(&vm->ractor.sched.grq, &sched->grq_node);
1248 vm->ractor.sched.grq_cnt++;
1249 VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt);
1250
1251 RUBY_DEBUG_LOG("r:%u th:%u grq_cnt:%u", rb_ractor_id(r), rb_th_serial(sched->running), vm->ractor.sched.grq_cnt);
1252
1253 rb_native_cond_signal(&vm->ractor.sched.cond);
1254
1255 // ractor_sched_dump(vm);
1256 }
1257 ractor_sched_unlock(vm, cr);
1258}
1259
1260
1261#ifndef SNT_KEEP_SECONDS
1262#define SNT_KEEP_SECONDS 0
1263#endif
1264
1265#ifndef MINIMUM_SNT
1266// make at least MINIMUM_SNT snts for debug.
1267#define MINIMUM_SNT 0
1268#endif
1269
1270static rb_ractor_t *
1271ractor_sched_deq(rb_vm_t *vm, rb_ractor_t *cr)
1272{
1273 rb_ractor_t *r;
1274
1275 ractor_sched_lock(vm, cr);
1276 {
1277 RUBY_DEBUG_LOG("empty? %d", ccan_list_empty(&vm->ractor.sched.grq));
1278 // ractor_sched_dump(vm);
1279
1280 VM_ASSERT(rb_current_execution_context(false) == NULL);
1281 VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt);
1282
1283 while ((r = ccan_list_pop(&vm->ractor.sched.grq, rb_ractor_t, threads.sched.grq_node)) == NULL) {
1284 RUBY_DEBUG_LOG("wait grq_cnt:%d", (int)vm->ractor.sched.grq_cnt);
1285
1286#if SNT_KEEP_SECONDS > 0
1287 rb_hrtime_t abs = rb_hrtime_add(rb_hrtime_now(), RB_HRTIME_PER_SEC * SNT_KEEP_SECONDS);
1288 if (native_cond_timedwait(&vm->ractor.sched.cond, &vm->ractor.sched.lock, &abs) == ETIMEDOUT) {
1289 RUBY_DEBUG_LOG("timeout, grq_cnt:%d", (int)vm->ractor.sched.grq_cnt);
1290 VM_ASSERT(r == NULL);
1291 vm->ractor.sched.snt_cnt--;
1292 vm->ractor.sched.running_cnt--;
1293 break;
1294 }
1295 else {
1296 RUBY_DEBUG_LOG("wakeup grq_cnt:%d", (int)vm->ractor.sched.grq_cnt);
1297 }
1298#else
1299 ractor_sched_set_unlocked(vm, cr);
1300 rb_native_cond_wait(&vm->ractor.sched.cond, &vm->ractor.sched.lock);
1301 ractor_sched_set_locked(vm, cr);
1302
1303 RUBY_DEBUG_LOG("wakeup grq_cnt:%d", (int)vm->ractor.sched.grq_cnt);
1304#endif
1305 }
1306
1307 VM_ASSERT(rb_current_execution_context(false) == NULL);
1308
1309 if (r) {
1310 VM_ASSERT(vm->ractor.sched.grq_cnt > 0);
1311 vm->ractor.sched.grq_cnt--;
1312 RUBY_DEBUG_LOG("r:%d grq_cnt:%u", (int)rb_ractor_id(r), vm->ractor.sched.grq_cnt);
1313 }
1314 else {
1315 VM_ASSERT(SNT_KEEP_SECONDS > 0);
1316 // timeout
1317 }
1318 }
1319 ractor_sched_unlock(vm, cr);
1320
1321 return r;
1322}
1323
1324void rb_ractor_lock_self(rb_ractor_t *r);
1325void rb_ractor_unlock_self(rb_ractor_t *r);
1326
1327// The current thread for a ractor is put to "sleep" (descheduled in the STOPPED_FOREVER state) waiting for
1328// a ractor action to wake it up.
1329void
1330rb_ractor_sched_wait(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf, void *ubf_arg)
1331{
1332 // ractor lock of cr is acquired
1333
1334 RUBY_DEBUG_LOG("start%s", "");
1335
1336 rb_thread_t * volatile th = rb_ec_thread_ptr(ec);
1337 struct rb_thread_sched *sched = TH_SCHED(th);
1338
1339 if (ubf_set(th, ubf, ubf_arg)) {
1340 // interrupted
1341 return;
1342 }
1343
1344 thread_sched_lock(sched, th);
1345 rb_ractor_unlock_self(cr);
1346 {
1347 // setup sleep
1348 bool can_direct_transfer = !th_has_dedicated_nt(th);
1349 RB_VM_SAVE_MACHINE_CONTEXT(th);
1350 th->status = THREAD_STOPPED_FOREVER;
1351 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1352 thread_sched_wakeup_next_thread(sched, th, can_direct_transfer);
1353 // sleep
1354 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1355 th->status = THREAD_RUNNABLE;
1356 }
1357 thread_sched_unlock(sched, th);
1358 rb_ractor_lock_self(cr);
1359
1360 ubf_clear(th);
1361
1362 RUBY_DEBUG_LOG("end%s", "");
1363}
1364
1365void
1366rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *r_th)
1367{
1368 // ractor lock of r is NOT acquired
1369 struct rb_thread_sched *sched = TH_SCHED(r_th);
1370
1371 RUBY_DEBUG_LOG("r:%u th:%d", (unsigned int)rb_ractor_id(r), r_th->serial);
1372
1373 thread_sched_lock(sched, r_th);
1374 {
1375 if (r_th->status == THREAD_STOPPED_FOREVER) {
1376 thread_sched_to_ready_common(sched, r_th, true, false);
1377 }
1378 }
1379 thread_sched_unlock(sched, r_th);
1380}
1381
1382static bool
1383ractor_sched_barrier_completed_p(rb_vm_t *vm)
1384{
1385 RUBY_DEBUG_LOG("run:%u wait:%u", vm->ractor.sched.running_cnt, vm->ractor.sched.barrier_waiting_cnt);
1386 VM_ASSERT(vm->ractor.sched.running_cnt - 1 >= vm->ractor.sched.barrier_waiting_cnt);
1387
1388 return (vm->ractor.sched.running_cnt - vm->ractor.sched.barrier_waiting_cnt) == 1;
1389}
1390
1391void
1392rb_ractor_sched_barrier_start(rb_vm_t *vm, rb_ractor_t *cr)
1393{
1394 VM_ASSERT(cr == GET_RACTOR());
1395 VM_ASSERT(vm->ractor.sync.lock_owner == cr); // VM is locked
1396 VM_ASSERT(!vm->ractor.sched.barrier_waiting);
1397 VM_ASSERT(vm->ractor.sched.barrier_waiting_cnt == 0);
1398 VM_ASSERT(vm->ractor.sched.barrier_ractor == NULL);
1399 VM_ASSERT(vm->ractor.sched.barrier_lock_rec == 0);
1400
1401 RUBY_DEBUG_LOG("start serial:%u", vm->ractor.sched.barrier_serial);
1402
1403 unsigned int lock_rec;
1404
1405 ractor_sched_lock(vm, cr);
1406 {
1407 vm->ractor.sched.barrier_waiting = true;
1408 vm->ractor.sched.barrier_ractor = cr;
1409 vm->ractor.sched.barrier_lock_rec = vm->ractor.sync.lock_rec;
1410
1411 // release VM lock
1412 lock_rec = vm->ractor.sync.lock_rec;
1413 vm->ractor.sync.lock_rec = 0;
1414 vm->ractor.sync.lock_owner = NULL;
1415 rb_native_mutex_unlock(&vm->ractor.sync.lock);
1416
1417 // interrupts all running threads
1418 rb_thread_t *ith;
1419 ccan_list_for_each(&vm->ractor.sched.running_threads, ith, sched.node.running_threads) {
1420 if (ith->ractor != cr) {
1421 RUBY_DEBUG_LOG("barrier request to th:%u", rb_th_serial(ith));
1422 RUBY_VM_SET_VM_BARRIER_INTERRUPT(ith->ec);
1423 }
1424 }
1425
1426 // wait for other ractors
1427 while (!ractor_sched_barrier_completed_p(vm)) {
1428 ractor_sched_set_unlocked(vm, cr);
1429 rb_native_cond_wait(&vm->ractor.sched.barrier_complete_cond, &vm->ractor.sched.lock);
1430 ractor_sched_set_locked(vm, cr);
1431 }
1432
1433 RUBY_DEBUG_LOG("completed seirial:%u", vm->ractor.sched.barrier_serial);
1434
1435 // no other ractors are there
1436 vm->ractor.sched.barrier_serial++;
1437 vm->ractor.sched.barrier_waiting_cnt = 0;
1438 rb_native_cond_broadcast(&vm->ractor.sched.barrier_release_cond);
1439
1440 // acquire VM lock
1441 rb_native_mutex_lock(&vm->ractor.sync.lock);
1442 vm->ractor.sync.lock_rec = lock_rec;
1443 vm->ractor.sync.lock_owner = cr;
1444 }
1445
1446 // do not release ractor_sched_lock and there is no newly added (resumed) thread
1447 // thread_sched_setup_running_threads
1448}
1449
1450// called from vm_lock_leave if the vm_lock used for barrierred
1451void
1452rb_ractor_sched_barrier_end(rb_vm_t *vm, rb_ractor_t *cr)
1453{
1454 RUBY_DEBUG_LOG("serial:%u", (unsigned int)vm->ractor.sched.barrier_serial - 1);
1455 VM_ASSERT(vm->ractor.sched.barrier_waiting);
1456 VM_ASSERT(vm->ractor.sched.barrier_ractor);
1457 VM_ASSERT(vm->ractor.sched.barrier_lock_rec > 0);
1458
1459 vm->ractor.sched.barrier_waiting = false;
1460 vm->ractor.sched.barrier_ractor = NULL;
1461 vm->ractor.sched.barrier_lock_rec = 0;
1462 ractor_sched_unlock(vm, cr);
1463}
1464
1465static void
1466ractor_sched_barrier_join_signal_locked(rb_vm_t *vm)
1467{
1468 if (ractor_sched_barrier_completed_p(vm)) {
1469 rb_native_cond_signal(&vm->ractor.sched.barrier_complete_cond);
1470 }
1471}
1472
1473static void
1474ractor_sched_barrier_join_wait_locked(rb_vm_t *vm, rb_thread_t *th)
1475{
1476 VM_ASSERT(vm->ractor.sched.barrier_waiting);
1477
1478 unsigned int barrier_serial = vm->ractor.sched.barrier_serial;
1479
1480 while (vm->ractor.sched.barrier_serial == barrier_serial) {
1481 RUBY_DEBUG_LOG("sleep serial:%u", barrier_serial);
1482 RB_VM_SAVE_MACHINE_CONTEXT(th);
1483
1484 rb_ractor_t *cr = th->ractor;
1485 ractor_sched_set_unlocked(vm, cr);
1486 rb_native_cond_wait(&vm->ractor.sched.barrier_release_cond, &vm->ractor.sched.lock);
1487 ractor_sched_set_locked(vm, cr);
1488
1489 RUBY_DEBUG_LOG("wakeup serial:%u", barrier_serial);
1490 }
1491}
1492
1493void
1494rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr)
1495{
1496 VM_ASSERT(cr->threads.sched.running != NULL); // running ractor
1497 VM_ASSERT(cr == GET_RACTOR());
1498 VM_ASSERT(vm->ractor.sync.lock_owner == NULL); // VM is locked, but owner == NULL
1499 VM_ASSERT(vm->ractor.sched.barrier_waiting); // VM needs barrier sync
1500
1501#if USE_RUBY_DEBUG_LOG || VM_CHECK_MODE > 0
1502 unsigned int barrier_serial = vm->ractor.sched.barrier_serial;
1503#endif
1504
1505 RUBY_DEBUG_LOG("join");
1506
1507 rb_native_mutex_unlock(&vm->ractor.sync.lock);
1508 {
1509 VM_ASSERT(vm->ractor.sched.barrier_waiting); // VM needs barrier sync
1510 VM_ASSERT(vm->ractor.sched.barrier_serial == barrier_serial);
1511
1512 ractor_sched_lock(vm, cr);
1513 {
1514 // running_cnt
1515 vm->ractor.sched.barrier_waiting_cnt++;
1516 RUBY_DEBUG_LOG("waiting_cnt:%u serial:%u", vm->ractor.sched.barrier_waiting_cnt, barrier_serial);
1517
1518 ractor_sched_barrier_join_signal_locked(vm);
1519 ractor_sched_barrier_join_wait_locked(vm, cr->threads.sched.running);
1520 }
1521 ractor_sched_unlock(vm, cr);
1522 }
1523
1524 rb_native_mutex_lock(&vm->ractor.sync.lock);
1525 // VM locked here
1526}
1527
1528#if 0
1529// TODO
1530
1531static void clear_thread_cache_altstack(void);
1532
1533static void
1534rb_thread_sched_destroy(struct rb_thread_sched *sched)
1535{
1536 /*
1537 * only called once at VM shutdown (not atfork), another thread
1538 * may still grab vm->gvl.lock when calling gvl_release at
1539 * the end of thread_start_func_2
1540 */
1541 if (0) {
1542 rb_native_mutex_destroy(&sched->lock);
1543 }
1544 clear_thread_cache_altstack();
1545}
1546#endif
1547
1548#ifdef RB_THREAD_T_HAS_NATIVE_ID
1549static int
1550get_native_thread_id(void)
1551{
1552#ifdef __linux__
1553 return (int)syscall(SYS_gettid);
1554#elif defined(__FreeBSD__)
1555 return pthread_getthreadid_np();
1556#endif
1557}
1558#endif
1559
1560#if defined(HAVE_WORKING_FORK)
1561void rb_internal_thread_event_hooks_rw_lock_atfork(void);
1562
1563static void
1564thread_sched_atfork(struct rb_thread_sched *sched)
1565{
1566 current_fork_gen++;
1567 rb_thread_sched_init(sched, true);
1568 rb_thread_t *th = GET_THREAD();
1569 rb_vm_t *vm = GET_VM();
1570
1571 if (th_has_dedicated_nt(th)) {
1572 vm->ractor.sched.snt_cnt = 0;
1573 }
1574 else {
1575 vm->ractor.sched.snt_cnt = 1;
1576 }
1577 vm->ractor.sched.running_cnt = 0;
1578
1579 rb_native_mutex_initialize(&vm->ractor.sched.lock);
1580#if VM_CHECK_MODE > 0
1581 vm->ractor.sched.lock_owner = NULL;
1582 vm->ractor.sched.locked = false;
1583#endif
1584
1585 // rb_native_cond_destroy(&vm->ractor.sched.cond);
1586 rb_native_cond_initialize(&vm->ractor.sched.cond);
1587 rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond);
1588 rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond);
1589
1590 ccan_list_head_init(&vm->ractor.sched.grq);
1591 ccan_list_head_init(&vm->ractor.sched.timeslice_threads);
1592 ccan_list_head_init(&vm->ractor.sched.running_threads);
1593
1594 rb_internal_thread_event_hooks_rw_lock_atfork();
1595
1596 VM_ASSERT(sched->is_running);
1597 sched->is_running_timeslice = false;
1598
1599 if (sched->running != th) {
1600 thread_sched_to_running(sched, th);
1601 }
1602 else {
1603 thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL, NULL);
1604 }
1605
1606#ifdef RB_THREAD_T_HAS_NATIVE_ID
1607 if (th->nt) {
1608 th->nt->tid = get_native_thread_id();
1609 }
1610#endif
1611}
1612
1613#endif
1614
1615#ifdef RB_THREAD_LOCAL_SPECIFIER
1616static RB_THREAD_LOCAL_SPECIFIER rb_thread_t *ruby_native_thread;
1617#else
1618static pthread_key_t ruby_native_thread_key;
1619#endif
1620
1621static void
1622null_func(int i)
1623{
1624 /* null */
1625 // This function can be called from signal handler
1626 // RUBY_DEBUG_LOG("i:%d", i);
1627}
1628
1629rb_thread_t *
1630ruby_thread_from_native(void)
1631{
1632#ifdef RB_THREAD_LOCAL_SPECIFIER
1633 return ruby_native_thread;
1634#else
1635 return pthread_getspecific(ruby_native_thread_key);
1636#endif
1637}
1638
1639int
1640ruby_thread_set_native(rb_thread_t *th)
1641{
1642 if (th) {
1643#ifdef USE_UBF_LIST
1644 ccan_list_node_init(&th->sched.node.ubf);
1645#endif
1646 }
1647
1648 // setup TLS
1649
1650 if (th && th->ec) {
1651 rb_ractor_set_current_ec(th->ractor, th->ec);
1652 }
1653#ifdef RB_THREAD_LOCAL_SPECIFIER
1654 ruby_native_thread = th;
1655 return 1;
1656#else
1657 return pthread_setspecific(ruby_native_thread_key, th) == 0;
1658#endif
1659}
1660
1661static void native_thread_setup(struct rb_native_thread *nt);
1662static void native_thread_setup_on_thread(struct rb_native_thread *nt);
1663
1664void
1665Init_native_thread(rb_thread_t *main_th)
1666{
1667#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK)
1668 if (condattr_monotonic) {
1669 int r = pthread_condattr_init(condattr_monotonic);
1670 if (r == 0) {
1671 r = pthread_condattr_setclock(condattr_monotonic, CLOCK_MONOTONIC);
1672 }
1673 if (r) condattr_monotonic = NULL;
1674 }
1675#endif
1676
1677#ifndef RB_THREAD_LOCAL_SPECIFIER
1678 if (pthread_key_create(&ruby_native_thread_key, 0) == EAGAIN) {
1679 rb_bug("pthread_key_create failed (ruby_native_thread_key)");
1680 }
1681 if (pthread_key_create(&ruby_current_ec_key, 0) == EAGAIN) {
1682 rb_bug("pthread_key_create failed (ruby_current_ec_key)");
1683 }
1684#endif
1685 ruby_posix_signal(SIGVTALRM, null_func);
1686
1687 // setup vm
1688 rb_vm_t *vm = main_th->vm;
1689 rb_native_mutex_initialize(&vm->ractor.sched.lock);
1690 rb_native_cond_initialize(&vm->ractor.sched.cond);
1691 rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond);
1692 rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond);
1693
1694 ccan_list_head_init(&vm->ractor.sched.grq);
1695 ccan_list_head_init(&vm->ractor.sched.timeslice_threads);
1696 ccan_list_head_init(&vm->ractor.sched.running_threads);
1697
1698 // setup main thread
1699 main_th->nt->thread_id = pthread_self();
1700 main_th->nt->serial = 1;
1701#ifdef RUBY_NT_SERIAL
1702 ruby_nt_serial = 1;
1703#endif
1704 ruby_thread_set_native(main_th);
1705 native_thread_setup(main_th->nt);
1706 native_thread_setup_on_thread(main_th->nt);
1707
1708 TH_SCHED(main_th)->running = main_th;
1709 main_th->has_dedicated_nt = 1;
1710
1711 thread_sched_setup_running_threads(TH_SCHED(main_th), main_th->ractor, vm, main_th, NULL, NULL);
1712
1713 // setup main NT
1714 main_th->nt->dedicated = 1;
1715 main_th->nt->vm = vm;
1716
1717 // setup mn
1718 vm->ractor.sched.dnt_cnt = 1;
1719}
1720
1721extern int ruby_mn_threads_enabled;
1722
1723void
1724ruby_mn_threads_params(void)
1725{
1726 rb_vm_t *vm = GET_VM();
1727 rb_ractor_t *main_ractor = GET_RACTOR();
1728
1729 const char *mn_threads_cstr = getenv("RUBY_MN_THREADS");
1730 bool enable_mn_threads = false;
1731
1732 if (USE_MN_THREADS && mn_threads_cstr && (enable_mn_threads = atoi(mn_threads_cstr) > 0)) {
1733 // enabled
1734 ruby_mn_threads_enabled = 1;
1735 }
1736 main_ractor->threads.sched.enable_mn_threads = enable_mn_threads;
1737
1738 const char *max_cpu_cstr = getenv("RUBY_MAX_CPU");
1739 const int default_max_cpu = 8; // TODO: CPU num?
1740 int max_cpu = default_max_cpu;
1741
1742 if (USE_MN_THREADS && max_cpu_cstr) {
1743 int given_max_cpu = atoi(max_cpu_cstr);
1744 if (given_max_cpu > 0) {
1745 max_cpu = given_max_cpu;
1746 }
1747 }
1748
1749 vm->ractor.sched.max_cpu = max_cpu;
1750}
1751
1752static void
1753native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt)
1754{
1755 RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated + 1);
1756
1757 if (nt->dedicated == 0) {
1758 ractor_sched_lock(vm, cr);
1759 {
1760 vm->ractor.sched.snt_cnt--;
1761 vm->ractor.sched.dnt_cnt++;
1762 }
1763 ractor_sched_unlock(vm, cr);
1764 }
1765
1766 nt->dedicated++;
1767}
1768
1769static void
1770native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt)
1771{
1772 RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated - 1);
1773 VM_ASSERT(nt->dedicated > 0);
1774 nt->dedicated--;
1775
1776 if (nt->dedicated == 0) {
1777 ractor_sched_lock(vm, cr);
1778 {
1779 nt->vm->ractor.sched.snt_cnt++;
1780 nt->vm->ractor.sched.dnt_cnt--;
1781 }
1782 ractor_sched_unlock(vm, cr);
1783 }
1784}
1785
1786static void
1787native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th)
1788{
1789#if USE_RUBY_DEBUG_LOG
1790 if (nt) {
1791 if (th->nt) {
1792 RUBY_DEBUG_LOG("th:%d nt:%d->%d", (int)th->serial, (int)th->nt->serial, (int)nt->serial);
1793 }
1794 else {
1795 RUBY_DEBUG_LOG("th:%d nt:NULL->%d", (int)th->serial, (int)nt->serial);
1796 }
1797 }
1798 else {
1799 if (th->nt) {
1800 RUBY_DEBUG_LOG("th:%d nt:%d->NULL", (int)th->serial, (int)th->nt->serial);
1801 }
1802 else {
1803 RUBY_DEBUG_LOG("th:%d nt:NULL->NULL", (int)th->serial);
1804 }
1805 }
1806#endif
1807
1808 th->nt = nt;
1809}
1810
1811static void
1812native_thread_destroy_atfork(struct rb_native_thread *nt)
1813{
1814 if (nt) {
1815 /* We can't call rb_native_cond_destroy here because according to the
1816 * specs of pthread_cond_destroy:
1817 *
1818 * Attempting to destroy a condition variable upon which other threads
1819 * are currently blocked results in undefined behavior.
1820 *
1821 * Specifically, glibc's pthread_cond_destroy waits on all the other
1822 * listeners. Since after forking all the threads are dead, the condition
1823 * variable's listeners will never wake up, so it will hang forever.
1824 */
1825
1826 RB_ALTSTACK_FREE(nt->altstack);
1827 ruby_xfree(nt->nt_context);
1828 ruby_xfree(nt);
1829 }
1830}
1831
1832static void
1833native_thread_destroy(struct rb_native_thread *nt)
1834{
1835 if (nt) {
1836 rb_native_cond_destroy(&nt->cond.readyq);
1837
1838 if (&nt->cond.readyq != &nt->cond.intr) {
1839 rb_native_cond_destroy(&nt->cond.intr);
1840 }
1841
1842 native_thread_destroy_atfork(nt);
1843 }
1844}
1845
1846#if defined HAVE_PTHREAD_GETATTR_NP || defined HAVE_PTHREAD_ATTR_GET_NP
1847#define STACKADDR_AVAILABLE 1
1848#elif defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP
1849#define STACKADDR_AVAILABLE 1
1850#undef MAINSTACKADDR_AVAILABLE
1851#define MAINSTACKADDR_AVAILABLE 1
1852void *pthread_get_stackaddr_np(pthread_t);
1853size_t pthread_get_stacksize_np(pthread_t);
1854#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
1855#define STACKADDR_AVAILABLE 1
1856#elif defined HAVE_PTHREAD_GETTHRDS_NP
1857#define STACKADDR_AVAILABLE 1
1858#elif defined __HAIKU__
1859#define STACKADDR_AVAILABLE 1
1860#endif
1861
1862#ifndef MAINSTACKADDR_AVAILABLE
1863# ifdef STACKADDR_AVAILABLE
1864# define MAINSTACKADDR_AVAILABLE 1
1865# else
1866# define MAINSTACKADDR_AVAILABLE 0
1867# endif
1868#endif
1869#if MAINSTACKADDR_AVAILABLE && !defined(get_main_stack)
1870# define get_main_stack(addr, size) get_stack(addr, size)
1871#endif
1872
1873#ifdef STACKADDR_AVAILABLE
1874/*
1875 * Get the initial address and size of current thread's stack
1876 */
1877static int
1878get_stack(void **addr, size_t *size)
1879{
1880#define CHECK_ERR(expr) \
1881 {int err = (expr); if (err) return err;}
1882#ifdef HAVE_PTHREAD_GETATTR_NP /* Linux */
1883 pthread_attr_t attr;
1884 size_t guard = 0;
1885 STACK_GROW_DIR_DETECTION;
1886 CHECK_ERR(pthread_getattr_np(pthread_self(), &attr));
1887# ifdef HAVE_PTHREAD_ATTR_GETSTACK
1888 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
1889 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
1890# else
1891 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
1892 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
1893# endif
1894# ifdef HAVE_PTHREAD_ATTR_GETGUARDSIZE
1895 CHECK_ERR(pthread_attr_getguardsize(&attr, &guard));
1896# else
1897 guard = getpagesize();
1898# endif
1899 *size -= guard;
1900 pthread_attr_destroy(&attr);
1901#elif defined HAVE_PTHREAD_ATTR_GET_NP /* FreeBSD, DragonFly BSD, NetBSD */
1902 pthread_attr_t attr;
1903 CHECK_ERR(pthread_attr_init(&attr));
1904 CHECK_ERR(pthread_attr_get_np(pthread_self(), &attr));
1905# ifdef HAVE_PTHREAD_ATTR_GETSTACK
1906 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
1907# else
1908 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
1909 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
1910# endif
1911 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
1912 pthread_attr_destroy(&attr);
1913#elif (defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP) /* MacOS X */
1914 pthread_t th = pthread_self();
1915 *addr = pthread_get_stackaddr_np(th);
1916 *size = pthread_get_stacksize_np(th);
1917#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
1918 stack_t stk;
1919# if defined HAVE_THR_STKSEGMENT /* Solaris */
1920 CHECK_ERR(thr_stksegment(&stk));
1921# else /* OpenBSD */
1922 CHECK_ERR(pthread_stackseg_np(pthread_self(), &stk));
1923# endif
1924 *addr = stk.ss_sp;
1925 *size = stk.ss_size;
1926#elif defined HAVE_PTHREAD_GETTHRDS_NP /* AIX */
1927 pthread_t th = pthread_self();
1928 struct __pthrdsinfo thinfo;
1929 char reg[256];
1930 int regsiz=sizeof(reg);
1931 CHECK_ERR(pthread_getthrds_np(&th, PTHRDSINFO_QUERY_ALL,
1932 &thinfo, sizeof(thinfo),
1933 &reg, &regsiz));
1934 *addr = thinfo.__pi_stackaddr;
1935 /* Must not use thinfo.__pi_stacksize for size.
1936 It is around 3KB smaller than the correct size
1937 calculated by thinfo.__pi_stackend - thinfo.__pi_stackaddr. */
1938 *size = thinfo.__pi_stackend - thinfo.__pi_stackaddr;
1939 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
1940#elif defined __HAIKU__
1941 thread_info info;
1942 STACK_GROW_DIR_DETECTION;
1943 CHECK_ERR(get_thread_info(find_thread(NULL), &info));
1944 *addr = info.stack_base;
1945 *size = (uintptr_t)info.stack_end - (uintptr_t)info.stack_base;
1946 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
1947#else
1948#error STACKADDR_AVAILABLE is defined but not implemented.
1949#endif
1950 return 0;
1951#undef CHECK_ERR
1952}
1953#endif
1954
1955static struct {
1956 rb_nativethread_id_t id;
1957 size_t stack_maxsize;
1958 VALUE *stack_start;
1959} native_main_thread;
1960
1961#ifdef STACK_END_ADDRESS
1962extern void *STACK_END_ADDRESS;
1963#endif
1964
1965enum {
1966 RUBY_STACK_SPACE_LIMIT = 1024 * 1024, /* 1024KB */
1967 RUBY_STACK_SPACE_RATIO = 5
1968};
1969
1970static size_t
1971space_size(size_t stack_size)
1972{
1973 size_t space_size = stack_size / RUBY_STACK_SPACE_RATIO;
1974 if (space_size > RUBY_STACK_SPACE_LIMIT) {
1975 return RUBY_STACK_SPACE_LIMIT;
1976 }
1977 else {
1978 return space_size;
1979 }
1980}
1981
1982static void
1983native_thread_init_main_thread_stack(void *addr)
1984{
1985 native_main_thread.id = pthread_self();
1986#ifdef RUBY_ASAN_ENABLED
1987 addr = asan_get_real_stack_addr((void *)addr);
1988#endif
1989
1990#if MAINSTACKADDR_AVAILABLE
1991 if (native_main_thread.stack_maxsize) return;
1992 {
1993 void* stackaddr;
1994 size_t size;
1995 if (get_main_stack(&stackaddr, &size) == 0) {
1996 native_main_thread.stack_maxsize = size;
1997 native_main_thread.stack_start = stackaddr;
1998 goto bound_check;
1999 }
2000 }
2001#endif
2002#ifdef STACK_END_ADDRESS
2003 native_main_thread.stack_start = STACK_END_ADDRESS;
2004#else
2005 if (!native_main_thread.stack_start ||
2006 STACK_UPPER((VALUE *)(void *)&addr,
2007 native_main_thread.stack_start > (VALUE *)addr,
2008 native_main_thread.stack_start < (VALUE *)addr)) {
2009 native_main_thread.stack_start = (VALUE *)addr;
2010 }
2011#endif
2012 {
2013#if defined(HAVE_GETRLIMIT)
2014#if defined(PTHREAD_STACK_DEFAULT)
2015# if PTHREAD_STACK_DEFAULT < RUBY_STACK_SPACE*5
2016# error "PTHREAD_STACK_DEFAULT is too small"
2017# endif
2018 size_t size = PTHREAD_STACK_DEFAULT;
2019#else
2020 size_t size = RUBY_VM_THREAD_VM_STACK_SIZE;
2021#endif
2022 size_t space;
2023 int pagesize = getpagesize();
2024 struct rlimit rlim;
2025 STACK_GROW_DIR_DETECTION;
2026 if (getrlimit(RLIMIT_STACK, &rlim) == 0) {
2027 size = (size_t)rlim.rlim_cur;
2028 }
2029 addr = native_main_thread.stack_start;
2030 if (IS_STACK_DIR_UPPER()) {
2031 space = ((size_t)((char *)addr + size) / pagesize) * pagesize - (size_t)addr;
2032 }
2033 else {
2034 space = (size_t)addr - ((size_t)((char *)addr - size) / pagesize + 1) * pagesize;
2035 }
2036 native_main_thread.stack_maxsize = space;
2037#endif
2038 }
2039
2040#if MAINSTACKADDR_AVAILABLE
2041 bound_check:
2042#endif
2043 /* If addr is out of range of main-thread stack range estimation, */
2044 /* it should be on co-routine (alternative stack). [Feature #2294] */
2045 {
2046 void *start, *end;
2047 STACK_GROW_DIR_DETECTION;
2048
2049 if (IS_STACK_DIR_UPPER()) {
2050 start = native_main_thread.stack_start;
2051 end = (char *)native_main_thread.stack_start + native_main_thread.stack_maxsize;
2052 }
2053 else {
2054 start = (char *)native_main_thread.stack_start - native_main_thread.stack_maxsize;
2055 end = native_main_thread.stack_start;
2056 }
2057
2058 if ((void *)addr < start || (void *)addr > end) {
2059 /* out of range */
2060 native_main_thread.stack_start = (VALUE *)addr;
2061 native_main_thread.stack_maxsize = 0; /* unknown */
2062 }
2063 }
2064}
2065
2066#define CHECK_ERR(expr) \
2067 {int err = (expr); if (err) {rb_bug_errno(#expr, err);}}
2068
2069static int
2070native_thread_init_stack(rb_thread_t *th, void *local_in_parent_frame)
2071{
2072 rb_nativethread_id_t curr = pthread_self();
2073#ifdef RUBY_ASAN_ENABLED
2074 local_in_parent_frame = asan_get_real_stack_addr(local_in_parent_frame);
2075 th->ec->machine.asan_fake_stack_handle = asan_get_thread_fake_stack_handle();
2076#endif
2077
2078 if (!native_main_thread.id) {
2079 /* This thread is the first thread, must be the main thread -
2080 * configure the native_main_thread object */
2081 native_thread_init_main_thread_stack(local_in_parent_frame);
2082 }
2083
2084 if (pthread_equal(curr, native_main_thread.id)) {
2085 th->ec->machine.stack_start = native_main_thread.stack_start;
2086 th->ec->machine.stack_maxsize = native_main_thread.stack_maxsize;
2087 }
2088 else {
2089#ifdef STACKADDR_AVAILABLE
2090 if (th_has_dedicated_nt(th)) {
2091 void *start;
2092 size_t size;
2093
2094 if (get_stack(&start, &size) == 0) {
2095 uintptr_t diff = (uintptr_t)start - (uintptr_t)local_in_parent_frame;
2096 th->ec->machine.stack_start = local_in_parent_frame;
2097 th->ec->machine.stack_maxsize = size - diff;
2098 }
2099 }
2100#else
2101 rb_raise(rb_eNotImpError, "ruby engine can initialize only in the main thread");
2102#endif
2103 }
2104
2105 return 0;
2106}
2107
2108struct nt_param {
2109 rb_vm_t *vm;
2110 struct rb_native_thread *nt;
2111};
2112
2113static void *
2114nt_start(void *ptr);
2115
2116static int
2117native_thread_create0(struct rb_native_thread *nt)
2118{
2119 int err = 0;
2120 pthread_attr_t attr;
2121
2122 const size_t stack_size = nt->vm->default_params.thread_machine_stack_size;
2123 const size_t space = space_size(stack_size);
2124
2125 nt->machine_stack_maxsize = stack_size - space;
2126
2127#ifdef USE_SIGALTSTACK
2128 nt->altstack = rb_allocate_sigaltstack();
2129#endif
2130
2131 CHECK_ERR(pthread_attr_init(&attr));
2132
2133# ifdef PTHREAD_STACK_MIN
2134 RUBY_DEBUG_LOG("stack size: %lu", (unsigned long)stack_size);
2135 CHECK_ERR(pthread_attr_setstacksize(&attr, stack_size));
2136# endif
2137
2138# ifdef HAVE_PTHREAD_ATTR_SETINHERITSCHED
2139 CHECK_ERR(pthread_attr_setinheritsched(&attr, PTHREAD_INHERIT_SCHED));
2140# endif
2141 CHECK_ERR(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED));
2142
2143 err = pthread_create(&nt->thread_id, &attr, nt_start, nt);
2144
2145 RUBY_DEBUG_LOG("nt:%d err:%d", (int)nt->serial, err);
2146
2147 CHECK_ERR(pthread_attr_destroy(&attr));
2148
2149 return err;
2150}
2151
2152static void
2153native_thread_setup(struct rb_native_thread *nt)
2154{
2155 // init cond
2156 rb_native_cond_initialize(&nt->cond.readyq);
2157
2158 if (&nt->cond.readyq != &nt->cond.intr) {
2159 rb_native_cond_initialize(&nt->cond.intr);
2160 }
2161}
2162
2163static void
2164native_thread_setup_on_thread(struct rb_native_thread *nt)
2165{
2166 // init tid
2167#ifdef RB_THREAD_T_HAS_NATIVE_ID
2168 nt->tid = get_native_thread_id();
2169#endif
2170
2171 // init signal handler
2172 RB_ALTSTACK_INIT(nt->altstack, nt->altstack);
2173}
2174
2175static struct rb_native_thread *
2176native_thread_alloc(void)
2177{
2178 struct rb_native_thread *nt = ZALLOC(struct rb_native_thread);
2179 native_thread_setup(nt);
2180
2181#if USE_MN_THREADS
2182 nt->nt_context = ruby_xmalloc(sizeof(struct coroutine_context));
2183#endif
2184
2185#if USE_RUBY_DEBUG_LOG
2186 static rb_atomic_t nt_serial = 2;
2187 nt->serial = RUBY_ATOMIC_FETCH_ADD(nt_serial, 1);
2188#endif
2189 return nt;
2190}
2191
2192static int
2193native_thread_create_dedicated(rb_thread_t *th)
2194{
2195 th->nt = native_thread_alloc();
2196 th->nt->vm = th->vm;
2197 th->nt->running_thread = th;
2198 th->nt->dedicated = 1;
2199
2200 // vm stack
2201 size_t vm_stack_word_size = th->vm->default_params.thread_vm_stack_size / sizeof(VALUE);
2202 void *vm_stack = ruby_xmalloc(vm_stack_word_size * sizeof(VALUE));
2203 th->sched.malloc_stack = true;
2204 rb_ec_initialize_vm_stack(th->ec, vm_stack, vm_stack_word_size);
2205 th->sched.context_stack = vm_stack;
2206
2207
2208 int err = native_thread_create0(th->nt);
2209 if (!err) {
2210 // setup
2211 thread_sched_to_ready(TH_SCHED(th), th);
2212 }
2213 return err;
2214}
2215
2216static void
2217call_thread_start_func_2(rb_thread_t *th)
2218{
2219 /* Capture the address of a local in this stack frame to mark the beginning of the
2220 machine stack for this thread. This is required even if we can tell the real
2221 stack beginning from the pthread API in native_thread_init_stack, because
2222 glibc stores some of its own data on the stack before calling into user code
2223 on a new thread, and replacing that data on fiber-switch would break it (see
2224 bug #13887) */
2225 VALUE stack_start = 0;
2226 VALUE *stack_start_addr = asan_get_real_stack_addr(&stack_start);
2227
2228 native_thread_init_stack(th, stack_start_addr);
2229 thread_start_func_2(th, th->ec->machine.stack_start);
2230}
2231
2232static void *
2233nt_start(void *ptr)
2234{
2235 struct rb_native_thread *nt = (struct rb_native_thread *)ptr;
2236 rb_vm_t *vm = nt->vm;
2237
2238 native_thread_setup_on_thread(nt);
2239
2240 // init tid
2241#ifdef RB_THREAD_T_HAS_NATIVE_ID
2242 nt->tid = get_native_thread_id();
2243#endif
2244
2245#if USE_RUBY_DEBUG_LOG && defined(RUBY_NT_SERIAL)
2246 ruby_nt_serial = nt->serial;
2247#endif
2248
2249 RUBY_DEBUG_LOG("nt:%u", nt->serial);
2250
2251 if (!nt->dedicated) {
2252 coroutine_initialize_main(nt->nt_context);
2253 }
2254
2255 while (1) {
2256 if (nt->dedicated) {
2257 // wait running turn
2258 rb_thread_t *th = nt->running_thread;
2259 struct rb_thread_sched *sched = TH_SCHED(th);
2260
2261 RUBY_DEBUG_LOG("on dedicated th:%u", rb_th_serial(th));
2262 ruby_thread_set_native(th);
2263
2264 thread_sched_lock(sched, th);
2265 {
2266 if (sched->running == th) {
2267 thread_sched_add_running_thread(sched, th);
2268 }
2269 thread_sched_wait_running_turn(sched, th, false);
2270 }
2271 thread_sched_unlock(sched, th);
2272
2273 // start threads
2274 call_thread_start_func_2(th);
2275 break; // TODO: allow to change to the SNT
2276 }
2277 else {
2278 RUBY_DEBUG_LOG("check next");
2279 rb_ractor_t *r = ractor_sched_deq(vm, NULL);
2280
2281 if (r) {
2282 struct rb_thread_sched *sched = &r->threads.sched;
2283
2284 thread_sched_lock(sched, NULL);
2285 {
2286 rb_thread_t *next_th = sched->running;
2287
2288 if (next_th && next_th->nt == NULL) {
2289 RUBY_DEBUG_LOG("nt:%d next_th:%d", (int)nt->serial, (int)next_th->serial);
2290 thread_sched_switch0(nt->nt_context, next_th, nt, false);
2291 }
2292 else {
2293 RUBY_DEBUG_LOG("no schedulable threads -- next_th:%p", next_th);
2294 }
2295 }
2296 thread_sched_unlock(sched, NULL);
2297 }
2298 else {
2299 // timeout -> deleted.
2300 break;
2301 }
2302
2303 if (nt->dedicated) {
2304 // SNT becomes DNT while running
2305 break;
2306 }
2307 }
2308 }
2309
2310 return NULL;
2311}
2312
2313static int native_thread_create_shared(rb_thread_t *th);
2314
2315#if USE_MN_THREADS
2316static void nt_free_stack(void *mstack);
2317#endif
2318
2319void
2320rb_threadptr_remove(rb_thread_t *th)
2321{
2322#if USE_MN_THREADS
2323 if (th->sched.malloc_stack) {
2324 // dedicated
2325 return;
2326 }
2327 else {
2328 rb_vm_t *vm = th->vm;
2329 th->sched.finished = false;
2330
2331 RB_VM_LOCKING() {
2332 ccan_list_add(&vm->ractor.sched.zombie_threads, &th->sched.node.zombie_threads);
2333 }
2334 }
2335#endif
2336}
2337
2338void
2339rb_threadptr_sched_free(rb_thread_t *th)
2340{
2341#if USE_MN_THREADS
2342 if (th->sched.malloc_stack) {
2343 // has dedicated
2344 ruby_xfree(th->sched.context_stack);
2345 native_thread_destroy(th->nt);
2346 }
2347 else {
2348 nt_free_stack(th->sched.context_stack);
2349 // TODO: how to free nt and nt->altstack?
2350 }
2351
2352 ruby_xfree(th->sched.context);
2353 th->sched.context = NULL;
2354 // VM_ASSERT(th->sched.context == NULL);
2355#else
2356 ruby_xfree(th->sched.context_stack);
2357 native_thread_destroy(th->nt);
2358#endif
2359
2360 th->nt = NULL;
2361}
2362
2363void
2364rb_thread_sched_mark_zombies(rb_vm_t *vm)
2365{
2366 if (!ccan_list_empty(&vm->ractor.sched.zombie_threads)) {
2367 rb_thread_t *zombie_th, *next_zombie_th;
2368 ccan_list_for_each_safe(&vm->ractor.sched.zombie_threads, zombie_th, next_zombie_th, sched.node.zombie_threads) {
2369 if (zombie_th->sched.finished) {
2370 ccan_list_del_init(&zombie_th->sched.node.zombie_threads);
2371 }
2372 else {
2373 rb_gc_mark(zombie_th->self);
2374 }
2375 }
2376 }
2377}
2378
2379static int
2380native_thread_create(rb_thread_t *th)
2381{
2382 VM_ASSERT(th->nt == 0);
2383 RUBY_DEBUG_LOG("th:%d has_dnt:%d", th->serial, th->has_dedicated_nt);
2384 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_STARTED, th);
2385
2386 if (!th->ractor->threads.sched.enable_mn_threads) {
2387 th->has_dedicated_nt = 1;
2388 }
2389
2390 if (th->has_dedicated_nt) {
2391 return native_thread_create_dedicated(th);
2392 }
2393 else {
2394 return native_thread_create_shared(th);
2395 }
2396}
2397
2398#if USE_NATIVE_THREAD_PRIORITY
2399
2400static void
2401native_thread_apply_priority(rb_thread_t *th)
2402{
2403#if defined(_POSIX_PRIORITY_SCHEDULING) && (_POSIX_PRIORITY_SCHEDULING > 0)
2404 struct sched_param sp;
2405 int policy;
2406 int priority = 0 - th->priority;
2407 int max, min;
2408 pthread_getschedparam(th->nt->thread_id, &policy, &sp);
2409 max = sched_get_priority_max(policy);
2410 min = sched_get_priority_min(policy);
2411
2412 if (min > priority) {
2413 priority = min;
2414 }
2415 else if (max < priority) {
2416 priority = max;
2417 }
2418
2419 sp.sched_priority = priority;
2420 pthread_setschedparam(th->nt->thread_id, policy, &sp);
2421#else
2422 /* not touched */
2423#endif
2424}
2425
2426#endif /* USE_NATIVE_THREAD_PRIORITY */
2427
2428static int
2429native_fd_select(int n, rb_fdset_t *readfds, rb_fdset_t *writefds, rb_fdset_t *exceptfds, struct timeval *timeout, rb_thread_t *th)
2430{
2431 return rb_fd_select(n, readfds, writefds, exceptfds, timeout);
2432}
2433
2434static void
2435ubf_pthread_cond_signal(void *ptr)
2436{
2437 rb_thread_t *th = (rb_thread_t *)ptr;
2438 RUBY_DEBUG_LOG("th:%u on nt:%d", rb_th_serial(th), (int)th->nt->serial);
2439 rb_native_cond_signal(&th->nt->cond.intr);
2440}
2441
2442static void
2443native_cond_sleep(rb_thread_t *th, rb_hrtime_t *rel)
2444{
2445 rb_nativethread_lock_t *lock = &th->interrupt_lock;
2446 rb_nativethread_cond_t *cond = &th->nt->cond.intr;
2447
2448 /* Solaris cond_timedwait() return EINVAL if an argument is greater than
2449 * current_time + 100,000,000. So cut up to 100,000,000. This is
2450 * considered as a kind of spurious wakeup. The caller to native_sleep
2451 * should care about spurious wakeup.
2452 *
2453 * See also [Bug #1341] [ruby-core:29702]
2454 * http://download.oracle.com/docs/cd/E19683-01/816-0216/6m6ngupgv/index.html
2455 */
2456 const rb_hrtime_t max = (rb_hrtime_t)100000000 * RB_HRTIME_PER_SEC;
2457
2458 THREAD_BLOCKING_BEGIN(th);
2459 {
2460 rb_native_mutex_lock(lock);
2461 th->unblock.func = ubf_pthread_cond_signal;
2462 th->unblock.arg = th;
2463
2464 if (RUBY_VM_INTERRUPTED(th->ec)) {
2465 /* interrupted. return immediate */
2466 RUBY_DEBUG_LOG("interrupted before sleep th:%u", rb_th_serial(th));
2467 }
2468 else {
2469 if (!rel) {
2470 rb_native_cond_wait(cond, lock);
2471 }
2472 else {
2473 rb_hrtime_t end;
2474
2475 if (*rel > max) {
2476 *rel = max;
2477 }
2478
2479 end = native_cond_timeout(cond, *rel);
2480 native_cond_timedwait(cond, lock, &end);
2481 }
2482 }
2483 th->unblock.func = 0;
2484
2485 rb_native_mutex_unlock(lock);
2486 }
2487 THREAD_BLOCKING_END(th);
2488
2489 RUBY_DEBUG_LOG("done th:%u", rb_th_serial(th));
2490}
2491
2492#ifdef USE_UBF_LIST
2493static CCAN_LIST_HEAD(ubf_list_head);
2494static rb_nativethread_lock_t ubf_list_lock = RB_NATIVETHREAD_LOCK_INIT;
2495
2496static void
2497ubf_list_atfork(void)
2498{
2499 ccan_list_head_init(&ubf_list_head);
2500 rb_native_mutex_initialize(&ubf_list_lock);
2501}
2502
2503 RBIMPL_ATTR_MAYBE_UNUSED()
2504static bool
2505ubf_list_contain_p(rb_thread_t *th)
2506{
2507 rb_thread_t *list_th;
2508 ccan_list_for_each(&ubf_list_head, list_th, sched.node.ubf) {
2509 if (list_th == th) return true;
2510 }
2511 return false;
2512}
2513
2514/* The thread 'th' is registered to be trying unblock. */
2515static void
2516register_ubf_list(rb_thread_t *th)
2517{
2518 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
2519 struct ccan_list_node *node = &th->sched.node.ubf;
2520
2521 VM_ASSERT(th->unblock.func != NULL);
2522
2523 rb_native_mutex_lock(&ubf_list_lock);
2524 {
2525 // check not connected yet
2526 if (ccan_list_empty((struct ccan_list_head*)node)) {
2527 VM_ASSERT(!ubf_list_contain_p(th));
2528 ccan_list_add(&ubf_list_head, node);
2529 }
2530 }
2531 rb_native_mutex_unlock(&ubf_list_lock);
2532
2533 timer_thread_wakeup();
2534}
2535
2536/* The thread 'th' is unblocked. It no longer need to be registered. */
2537static void
2538unregister_ubf_list(rb_thread_t *th)
2539{
2540 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
2541 struct ccan_list_node *node = &th->sched.node.ubf;
2542
2543 /* we can't allow re-entry into ubf_list_head */
2544 VM_ASSERT(th->unblock.func == NULL);
2545
2546 if (!ccan_list_empty((struct ccan_list_head*)node)) {
2547 rb_native_mutex_lock(&ubf_list_lock);
2548 {
2549 VM_ASSERT(ubf_list_contain_p(th));
2550 ccan_list_del_init(node);
2551 }
2552 rb_native_mutex_unlock(&ubf_list_lock);
2553 }
2554}
2555
2556/*
2557 * send a signal to intent that a target thread return from blocking syscall.
2558 * Maybe any signal is ok, but we chose SIGVTALRM.
2559 */
2560static void
2561ubf_wakeup_thread(rb_thread_t *th)
2562{
2563 RUBY_DEBUG_LOG("th:%u thread_id:%p", rb_th_serial(th), (void *)th->nt->thread_id);
2564
2565 pthread_kill(th->nt->thread_id, SIGVTALRM);
2566}
2567
2568static void
2569ubf_select(void *ptr)
2570{
2571 rb_thread_t *th = (rb_thread_t *)ptr;
2572 RUBY_DEBUG_LOG("wakeup th:%u", rb_th_serial(th));
2573 ubf_wakeup_thread(th);
2574 register_ubf_list(th);
2575}
2576
2577static bool
2578ubf_threads_empty(void)
2579{
2580 return ccan_list_empty(&ubf_list_head) != 0;
2581}
2582
2583static void
2584ubf_wakeup_all_threads(void)
2585{
2586 rb_thread_t *th;
2587 rb_native_mutex_lock(&ubf_list_lock);
2588 {
2589 ccan_list_for_each(&ubf_list_head, th, sched.node.ubf) {
2590 ubf_wakeup_thread(th);
2591 }
2592 }
2593 rb_native_mutex_unlock(&ubf_list_lock);
2594}
2595
2596#else /* USE_UBF_LIST */
2597#define register_ubf_list(th) (void)(th)
2598#define unregister_ubf_list(th) (void)(th)
2599#define ubf_select 0
2600static void ubf_wakeup_all_threads(void) { return; }
2601static bool ubf_threads_empty(void) { return true; }
2602#define ubf_list_atfork() do {} while (0)
2603#endif /* USE_UBF_LIST */
2604
2605#define TT_DEBUG 0
2606#define WRITE_CONST(fd, str) (void)(write((fd),(str),sizeof(str)-1)<0)
2607
2608void
2609rb_thread_wakeup_timer_thread(int sig)
2610{
2611 // This function can be called from signal handlers so that
2612 // pthread_mutex_lock() should not be used.
2613
2614 // wakeup timer thread
2615 timer_thread_wakeup_force();
2616
2617 // interrupt main thread if main thread is available
2618 if (RUBY_ATOMIC_LOAD(system_working)) {
2619 rb_vm_t *vm = GET_VM();
2620 rb_thread_t *main_th = vm->ractor.main_thread;
2621
2622 if (main_th) {
2623 volatile rb_execution_context_t *main_th_ec = ACCESS_ONCE(rb_execution_context_t *, main_th->ec);
2624
2625 if (main_th_ec) {
2626 RUBY_VM_SET_TRAP_INTERRUPT(main_th_ec);
2627
2628 if (vm->ubf_async_safe && main_th->unblock.func) {
2629 (main_th->unblock.func)(main_th->unblock.arg);
2630 }
2631 }
2632 }
2633 }
2634}
2635
2636#define CLOSE_INVALIDATE_PAIR(expr) \
2637 close_invalidate_pair(expr,"close_invalidate: "#expr)
2638static void
2639close_invalidate(int *fdp, const char *msg)
2640{
2641 int fd = *fdp;
2642
2643 *fdp = -1;
2644 if (close(fd) < 0) {
2645 async_bug_fd(msg, errno, fd);
2646 }
2647}
2648
2649static void
2650close_invalidate_pair(int fds[2], const char *msg)
2651{
2652 if (USE_EVENTFD && fds[0] == fds[1]) {
2653 fds[1] = -1; // disable write port first
2654 close_invalidate(&fds[0], msg);
2655 }
2656 else {
2657 close_invalidate(&fds[1], msg);
2658 close_invalidate(&fds[0], msg);
2659 }
2660}
2661
2662static void
2663set_nonblock(int fd)
2664{
2665 int oflags;
2666 int err;
2667
2668 oflags = fcntl(fd, F_GETFL);
2669 if (oflags == -1)
2670 rb_sys_fail(0);
2671 oflags |= O_NONBLOCK;
2672 err = fcntl(fd, F_SETFL, oflags);
2673 if (err == -1)
2674 rb_sys_fail(0);
2675}
2676
2677/* communication pipe with timer thread and signal handler */
2678static void
2679setup_communication_pipe_internal(int pipes[2])
2680{
2681 int err;
2682
2683 if (pipes[0] > 0 || pipes[1] > 0) {
2684 VM_ASSERT(pipes[0] > 0);
2685 VM_ASSERT(pipes[1] > 0);
2686 return;
2687 }
2688
2689 /*
2690 * Don't bother with eventfd on ancient Linux 2.6.22..2.6.26 which were
2691 * missing EFD_* flags, they can fall back to pipe
2692 */
2693#if USE_EVENTFD && defined(EFD_NONBLOCK) && defined(EFD_CLOEXEC)
2694 pipes[0] = pipes[1] = eventfd(0, EFD_NONBLOCK|EFD_CLOEXEC);
2695
2696 if (pipes[0] >= 0) {
2697 rb_update_max_fd(pipes[0]);
2698 return;
2699 }
2700#endif
2701
2702 err = rb_cloexec_pipe(pipes);
2703 if (err != 0) {
2704 rb_bug("can not create communication pipe");
2705 }
2706 rb_update_max_fd(pipes[0]);
2707 rb_update_max_fd(pipes[1]);
2708 set_nonblock(pipes[0]);
2709 set_nonblock(pipes[1]);
2710}
2711
2712#if !defined(SET_CURRENT_THREAD_NAME) && defined(__linux__) && defined(PR_SET_NAME)
2713# define SET_CURRENT_THREAD_NAME(name) prctl(PR_SET_NAME, name)
2714#endif
2715
2716enum {
2717 THREAD_NAME_MAX =
2718#if defined(__linux__)
2719 16
2720#elif defined(__APPLE__)
2721/* Undocumented, and main thread seems unlimited */
2722 64
2723#else
2724 16
2725#endif
2726};
2727
2728static VALUE threadptr_invoke_proc_location(rb_thread_t *th);
2729
2730static void
2731native_set_thread_name(rb_thread_t *th)
2732{
2733#ifdef SET_CURRENT_THREAD_NAME
2734 VALUE loc;
2735 if (!NIL_P(loc = th->name)) {
2736 SET_CURRENT_THREAD_NAME(RSTRING_PTR(loc));
2737 }
2738 else if ((loc = threadptr_invoke_proc_location(th)) != Qnil) {
2739 char *name, *p;
2740 char buf[THREAD_NAME_MAX];
2741 size_t len;
2742 int n;
2743
2744 name = RSTRING_PTR(RARRAY_AREF(loc, 0));
2745 p = strrchr(name, '/'); /* show only the basename of the path. */
2746 if (p && p[1])
2747 name = p + 1;
2748
2749 n = snprintf(buf, sizeof(buf), "%s:%d", name, NUM2INT(RARRAY_AREF(loc, 1)));
2750 RB_GC_GUARD(loc);
2751
2752 len = (size_t)n;
2753 if (len >= sizeof(buf)) {
2754 buf[sizeof(buf)-2] = '*';
2755 buf[sizeof(buf)-1] = '\0';
2756 }
2757 SET_CURRENT_THREAD_NAME(buf);
2758 }
2759#endif
2760}
2761
2762static void
2763native_set_another_thread_name(rb_nativethread_id_t thread_id, VALUE name)
2764{
2765#if defined SET_ANOTHER_THREAD_NAME || defined SET_CURRENT_THREAD_NAME
2766 char buf[THREAD_NAME_MAX];
2767 const char *s = "";
2768# if !defined SET_ANOTHER_THREAD_NAME
2769 if (!pthread_equal(pthread_self(), thread_id)) return;
2770# endif
2771 if (!NIL_P(name)) {
2772 long n;
2773 RSTRING_GETMEM(name, s, n);
2774 if (n >= (int)sizeof(buf)) {
2775 memcpy(buf, s, sizeof(buf)-1);
2776 buf[sizeof(buf)-1] = '\0';
2777 s = buf;
2778 }
2779 }
2780# if defined SET_ANOTHER_THREAD_NAME
2781 SET_ANOTHER_THREAD_NAME(thread_id, s);
2782# elif defined SET_CURRENT_THREAD_NAME
2783 SET_CURRENT_THREAD_NAME(s);
2784# endif
2785#endif
2786}
2787
2788#if defined(RB_THREAD_T_HAS_NATIVE_ID) || defined(__APPLE__)
2789static VALUE
2790native_thread_native_thread_id(rb_thread_t *target_th)
2791{
2792 if (!target_th->nt) return Qnil;
2793
2794#ifdef RB_THREAD_T_HAS_NATIVE_ID
2795 int tid = target_th->nt->tid;
2796 if (tid == 0) return Qnil;
2797 return INT2FIX(tid);
2798#elif defined(__APPLE__)
2799 uint64_t tid;
2800/* The first condition is needed because MAC_OS_X_VERSION_10_6
2801 is not defined on 10.5, and while __POWERPC__ takes care of ppc/ppc64,
2802 i386 will be broken without this. Note, 10.5 is supported with GCC upstream,
2803 so it has C++17 and everything needed to build modern Ruby. */
2804# if (!defined(MAC_OS_X_VERSION_10_6) || \
2805 (MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_X_VERSION_10_6) || \
2806 defined(__POWERPC__) /* never defined for PowerPC platforms */)
2807 const bool no_pthread_threadid_np = true;
2808# define NO_PTHREAD_MACH_THREAD_NP 1
2809# elif MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_6
2810 const bool no_pthread_threadid_np = false;
2811# else
2812# if !(defined(__has_attribute) && __has_attribute(availability))
2813 /* __API_AVAILABLE macro does nothing on gcc */
2814 __attribute__((weak)) int pthread_threadid_np(pthread_t, uint64_t*);
2815# endif
2816 /* Check weakly linked symbol */
2817 const bool no_pthread_threadid_np = !&pthread_threadid_np;
2818# endif
2819 if (no_pthread_threadid_np) {
2820 return ULL2NUM(pthread_mach_thread_np(pthread_self()));
2821 }
2822# ifndef NO_PTHREAD_MACH_THREAD_NP
2823 int e = pthread_threadid_np(target_th->nt->thread_id, &tid);
2824 if (e != 0) rb_syserr_fail(e, "pthread_threadid_np");
2825 return ULL2NUM((unsigned long long)tid);
2826# endif
2827#endif
2828}
2829# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 1
2830#else
2831# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 0
2832#endif
2833
2834static struct {
2835 rb_serial_t created_fork_gen;
2836 pthread_t pthread_id;
2837
2838 int comm_fds[2]; // r, w
2839
2840#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
2841 int event_fd; // kernel event queue fd (epoll/kqueue)
2842#endif
2843#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
2844#define EPOLL_EVENTS_MAX 0x10
2845 struct epoll_event finished_events[EPOLL_EVENTS_MAX];
2846#elif HAVE_SYS_EVENT_H && USE_MN_THREADS
2847#define KQUEUE_EVENTS_MAX 0x10
2848 struct kevent finished_events[KQUEUE_EVENTS_MAX];
2849#endif
2850
2851 // waiting threads list
2852 struct ccan_list_head waiting; // waiting threads in ractors
2853 pthread_mutex_t waiting_lock;
2854} timer_th = {
2855 .created_fork_gen = 0,
2856};
2857
2858#define TIMER_THREAD_CREATED_P() (timer_th.created_fork_gen == current_fork_gen)
2859
2860static void timer_thread_check_timeslice(rb_vm_t *vm);
2861static int timer_thread_set_timeout(rb_vm_t *vm);
2862static void timer_thread_wakeup_thread(rb_thread_t *th, uint32_t event_serial);
2863
2864#include "thread_pthread_mn.c"
2865
2866static rb_thread_t *
2867thread_sched_waiting_thread(struct rb_thread_sched_waiting *w)
2868{
2869 if (w) {
2870 return (rb_thread_t *)((size_t)w - offsetof(rb_thread_t, sched.waiting_reason));
2871 }
2872 else {
2873 return NULL;
2874 }
2875}
2876
2877static int
2878timer_thread_set_timeout(rb_vm_t *vm)
2879{
2880#if 0
2881 return 10; // ms
2882#else
2883 int timeout = -1;
2884
2885 ractor_sched_lock(vm, NULL);
2886 {
2887 if ( !ccan_list_empty(&vm->ractor.sched.timeslice_threads) // (1-1) Provide time slice for active NTs
2888 || !ubf_threads_empty() // (1-3) Periodic UBF
2889 || vm->ractor.sched.grq_cnt > 0 // (1-4) Lazy GRQ deq start
2890 ) {
2891
2892 RUBY_DEBUG_LOG("timeslice:%d ubf:%d grq:%d",
2893 !ccan_list_empty(&vm->ractor.sched.timeslice_threads),
2894 !ubf_threads_empty(),
2895 (vm->ractor.sched.grq_cnt > 0));
2896
2897 timeout = 10; // ms
2898 vm->ractor.sched.timeslice_wait_inf = false;
2899 }
2900 else {
2901 vm->ractor.sched.timeslice_wait_inf = true;
2902 }
2903 }
2904 ractor_sched_unlock(vm, NULL);
2905
2906 // Always check waiting threads to find minimum timeout
2907 // even when scheduler has work (grq_cnt > 0)
2908 rb_native_mutex_lock(&timer_th.waiting_lock);
2909 {
2910 struct rb_thread_sched_waiting *w = ccan_list_top(&timer_th.waiting, struct rb_thread_sched_waiting, node);
2911 rb_thread_t *th = thread_sched_waiting_thread(w);
2912
2913 if (th && (th->sched.waiting_reason.flags & thread_sched_waiting_timeout)) {
2914 rb_hrtime_t now = rb_hrtime_now();
2915 rb_hrtime_t hrrel = rb_hrtime_sub(th->sched.waiting_reason.data.timeout, now);
2916
2917 RUBY_DEBUG_LOG("th:%u now:%lu rel:%lu", rb_th_serial(th), (unsigned long)now, (unsigned long)hrrel);
2918
2919 // TODO: overflow?
2920 int thread_timeout = (int)((hrrel + RB_HRTIME_PER_MSEC - 1) / RB_HRTIME_PER_MSEC); // ms
2921
2922 // Use minimum of scheduler timeout and thread sleep timeout
2923 if (timeout < 0 || thread_timeout < timeout) {
2924 timeout = thread_timeout;
2925 }
2926 }
2927 }
2928 rb_native_mutex_unlock(&timer_th.waiting_lock);
2929
2930 RUBY_DEBUG_LOG("timeout:%d inf:%d", timeout, (int)vm->ractor.sched.timeslice_wait_inf);
2931
2932 // fprintf(stderr, "timeout:%d\n", timeout);
2933 return timeout;
2934#endif
2935}
2936
2937static void
2938timer_thread_check_signal(rb_vm_t *vm)
2939{
2940 // ruby_sigchld_handler(vm); TODO
2941
2942 int signum = rb_signal_buff_size();
2943 if (UNLIKELY(signum > 0) && vm->ractor.main_thread) {
2944 RUBY_DEBUG_LOG("signum:%d", signum);
2945 threadptr_trap_interrupt(vm->ractor.main_thread);
2946 }
2947}
2948
2949static bool
2950timer_thread_check_exceed(rb_hrtime_t abs, rb_hrtime_t now)
2951{
2952 if (abs < now) {
2953 return true;
2954 }
2955 else if (abs - now < RB_HRTIME_PER_MSEC) {
2956 return true; // too short time
2957 }
2958 else {
2959 return false;
2960 }
2961}
2962
2963static rb_thread_t *
2964timer_thread_deq_wakeup(rb_vm_t *vm, rb_hrtime_t now, uint32_t *event_serial)
2965{
2966 struct rb_thread_sched_waiting *w = ccan_list_top(&timer_th.waiting, struct rb_thread_sched_waiting, node);
2967
2968 if (w != NULL &&
2969 (w->flags & thread_sched_waiting_timeout) &&
2970 timer_thread_check_exceed(w->data.timeout, now)) {
2971
2972 RUBY_DEBUG_LOG("wakeup th:%u", rb_th_serial(thread_sched_waiting_thread(w)));
2973
2974 // delete from waiting list
2975 ccan_list_del_init(&w->node);
2976
2977 // setup result
2978 w->flags = thread_sched_waiting_none;
2979 w->data.result = 0;
2980
2981 rb_thread_t *th = thread_sched_waiting_thread(w);
2982 *event_serial = w->data.event_serial;
2983 return th;
2984 }
2985
2986 return NULL;
2987}
2988
2989static void
2990timer_thread_wakeup_thread_locked(struct rb_thread_sched *sched, rb_thread_t *th, uint32_t event_serial)
2991{
2992 if (sched->running != th && th->sched.event_serial == event_serial) {
2993 thread_sched_to_ready_common(sched, th, true, false);
2994 }
2995}
2996
2997static void
2998timer_thread_wakeup_thread(rb_thread_t *th, uint32_t event_serial)
2999{
3000 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
3001 struct rb_thread_sched *sched = TH_SCHED(th);
3002
3003 thread_sched_lock(sched, th);
3004 {
3005 timer_thread_wakeup_thread_locked(sched, th, event_serial);
3006 }
3007 thread_sched_unlock(sched, th);
3008}
3009
3010static void
3011timer_thread_check_timeout(rb_vm_t *vm)
3012{
3013 rb_hrtime_t now = rb_hrtime_now();
3014 rb_thread_t *th;
3015 uint32_t event_serial;
3016
3017 rb_native_mutex_lock(&timer_th.waiting_lock);
3018 {
3019 while ((th = timer_thread_deq_wakeup(vm, now, &event_serial)) != NULL) {
3020 rb_native_mutex_unlock(&timer_th.waiting_lock);
3021 timer_thread_wakeup_thread(th, event_serial);
3022 rb_native_mutex_lock(&timer_th.waiting_lock);
3023 }
3024 }
3025 rb_native_mutex_unlock(&timer_th.waiting_lock);
3026}
3027
3028static void
3029timer_thread_check_timeslice(rb_vm_t *vm)
3030{
3031 // TODO: check time
3032 rb_thread_t *th;
3033 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, th, sched.node.timeslice_threads) {
3034 RUBY_DEBUG_LOG("timeslice th:%u", rb_th_serial(th));
3035 RUBY_VM_SET_TIMER_INTERRUPT(th->ec);
3036 }
3037}
3038
3039void
3040rb_assert_sig(void)
3041{
3042 sigset_t oldmask;
3043 pthread_sigmask(0, NULL, &oldmask);
3044 if (sigismember(&oldmask, SIGVTALRM)) {
3045 rb_bug("!!!");
3046 }
3047 else {
3048 RUBY_DEBUG_LOG("ok");
3049 }
3050}
3051
3052static void *
3053timer_thread_func(void *ptr)
3054{
3055 rb_vm_t *vm = (rb_vm_t *)ptr;
3056#if defined(RUBY_NT_SERIAL)
3057 ruby_nt_serial = (rb_atomic_t)-1;
3058#endif
3059
3060 RUBY_DEBUG_LOG("started%s", "");
3061
3062 while (RUBY_ATOMIC_LOAD(system_working)) {
3063 timer_thread_check_signal(vm);
3064 timer_thread_check_timeout(vm);
3065 ubf_wakeup_all_threads();
3066
3067 RUBY_DEBUG_LOG("system_working:%d", RUBY_ATOMIC_LOAD(system_working));
3068 timer_thread_polling(vm);
3069 }
3070
3071 RUBY_DEBUG_LOG("terminated");
3072 return NULL;
3073}
3074
3075/* only use signal-safe system calls here */
3076static void
3077signal_communication_pipe(int fd)
3078{
3079#if USE_EVENTFD
3080 const uint64_t buff = 1;
3081#else
3082 const char buff = '!';
3083#endif
3084 ssize_t result;
3085
3086 /* already opened */
3087 if (fd >= 0) {
3088 retry:
3089 if ((result = write(fd, &buff, sizeof(buff))) <= 0) {
3090 int e = errno;
3091 switch (e) {
3092 case EINTR: goto retry;
3093 case EAGAIN:
3094#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN
3095 case EWOULDBLOCK:
3096#endif
3097 break;
3098 default:
3099 async_bug_fd("rb_thread_wakeup_timer_thread: write", e, fd);
3100 }
3101 }
3102 if (TT_DEBUG) WRITE_CONST(2, "rb_thread_wakeup_timer_thread: write\n");
3103 }
3104 else {
3105 // ignore wakeup
3106 }
3107}
3108
3109static void
3110timer_thread_wakeup_force(void)
3111{
3112 // should not use RUBY_DEBUG_LOG() because it can be called within signal handlers.
3113 signal_communication_pipe(timer_th.comm_fds[1]);
3114}
3115
3116static void
3117timer_thread_wakeup_locked(rb_vm_t *vm)
3118{
3119 // should be locked before.
3120 ASSERT_ractor_sched_locked(vm, NULL);
3121
3122 if (timer_th.created_fork_gen == current_fork_gen) {
3123 if (vm->ractor.sched.timeslice_wait_inf) {
3124 RUBY_DEBUG_LOG("wakeup with fd:%d", timer_th.comm_fds[1]);
3125 timer_thread_wakeup_force();
3126 }
3127 else {
3128 RUBY_DEBUG_LOG("will be wakeup...");
3129 }
3130 }
3131}
3132
3133static void
3134timer_thread_wakeup(void)
3135{
3136 rb_vm_t *vm = GET_VM();
3137
3138 ractor_sched_lock(vm, NULL);
3139 {
3140 timer_thread_wakeup_locked(vm);
3141 }
3142 ractor_sched_unlock(vm, NULL);
3143}
3144
3145static void
3146rb_thread_create_timer_thread(void)
3147{
3148 rb_serial_t created_fork_gen = timer_th.created_fork_gen;
3149
3150 RUBY_DEBUG_LOG("fork_gen create:%d current:%d", (int)created_fork_gen, (int)current_fork_gen);
3151
3152 timer_th.created_fork_gen = current_fork_gen;
3153
3154 if (created_fork_gen != current_fork_gen) {
3155 if (created_fork_gen != 0) {
3156 RUBY_DEBUG_LOG("forked child process");
3157
3158 CLOSE_INVALIDATE_PAIR(timer_th.comm_fds);
3159#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
3160 close_invalidate(&timer_th.event_fd, "close event_fd");
3161#endif
3162 rb_native_mutex_destroy(&timer_th.waiting_lock);
3163 }
3164
3165 ccan_list_head_init(&timer_th.waiting);
3166 rb_native_mutex_initialize(&timer_th.waiting_lock);
3167
3168 // open communication channel
3169 setup_communication_pipe_internal(timer_th.comm_fds);
3170
3171 // open event fd
3172 timer_thread_setup_mn();
3173 }
3174
3175 pthread_create(&timer_th.pthread_id, NULL, timer_thread_func, GET_VM());
3176}
3177
3178static int
3179native_stop_timer_thread(void)
3180{
3181 RUBY_ATOMIC_SET(system_working, 0);
3182
3183 RUBY_DEBUG_LOG("wakeup send %d", timer_th.comm_fds[1]);
3184 timer_thread_wakeup_force();
3185 RUBY_DEBUG_LOG("wakeup sent");
3186 pthread_join(timer_th.pthread_id, NULL);
3187
3188 if (TT_DEBUG) fprintf(stderr, "stop timer thread\n");
3189
3190 return 1;
3191}
3192
3193static void
3194native_reset_timer_thread(void)
3195{
3196 //
3197}
3198
3199#ifdef HAVE_SIGALTSTACK
3200int
3201ruby_stack_overflowed_p(const rb_thread_t *th, const void *addr)
3202{
3203 void *base;
3204 size_t size;
3205 const size_t water_mark = 1024 * 1024;
3206 STACK_GROW_DIR_DETECTION;
3207
3208 if (th) {
3209 size = th->ec->machine.stack_maxsize;
3210 base = (char *)th->ec->machine.stack_start - STACK_DIR_UPPER(0, size);
3211 }
3212#ifdef STACKADDR_AVAILABLE
3213 else if (get_stack(&base, &size) == 0) {
3214# ifdef __APPLE__
3215 if (pthread_equal(th->nt->thread_id, native_main_thread.id)) {
3216 struct rlimit rlim;
3217 if (getrlimit(RLIMIT_STACK, &rlim) == 0 && rlim.rlim_cur > size) {
3218 size = (size_t)rlim.rlim_cur;
3219 }
3220 }
3221# endif
3222 base = (char *)base + STACK_DIR_UPPER(+size, -size);
3223 }
3224#endif
3225 else {
3226 return 0;
3227 }
3228
3229 size /= RUBY_STACK_SPACE_RATIO;
3230 if (size > water_mark) size = water_mark;
3231 if (IS_STACK_DIR_UPPER()) {
3232 if (size > ~(size_t)base+1) size = ~(size_t)base+1;
3233 if (addr > base && addr <= (void *)((char *)base + size)) return 1;
3234 }
3235 else {
3236 if (size > (size_t)base) size = (size_t)base;
3237 if (addr > (void *)((char *)base - size) && addr <= base) return 1;
3238 }
3239 return 0;
3240}
3241#endif
3242
3243int
3244 rb_reserved_fd_p(int fd)
3245{
3246 /* no false-positive if out-of-FD at startup */
3247 if (fd < 0) return 0;
3248
3249 if (fd == timer_th.comm_fds[0] ||
3250 fd == timer_th.comm_fds[1]
3251#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
3252 || fd == timer_th.event_fd
3253#endif
3254 ) {
3255 goto check_fork_gen;
3256 }
3257 return 0;
3258
3259 check_fork_gen:
3260 if (timer_th.created_fork_gen == current_fork_gen) {
3261 /* async-signal-safe */
3262 return 1;
3263 }
3264 else {
3265 return 0;
3266 }
3267}
3268
3269rb_nativethread_id_t
3270 rb_nativethread_self(void)
3271{
3272 return pthread_self();
3273}
3274
3275#if defined(USE_POLL) && !defined(HAVE_PPOLL)
3276/* TODO: don't ignore sigmask */
3277static int
3278ruby_ppoll(struct pollfd *fds, nfds_t nfds,
3279 const struct timespec *ts, const sigset_t *sigmask)
3280{
3281 int timeout_ms;
3282
3283 if (ts) {
3284 int tmp, tmp2;
3285
3286 if (ts->tv_sec > INT_MAX/1000)
3287 timeout_ms = INT_MAX;
3288 else {
3289 tmp = (int)(ts->tv_sec * 1000);
3290 /* round up 1ns to 1ms to avoid excessive wakeups for <1ms sleep */
3291 tmp2 = (int)((ts->tv_nsec + 999999L) / (1000L * 1000L));
3292 if (INT_MAX - tmp < tmp2)
3293 timeout_ms = INT_MAX;
3294 else
3295 timeout_ms = (int)(tmp + tmp2);
3296 }
3297 }
3298 else
3299 timeout_ms = -1;
3300
3301 return poll(fds, nfds, timeout_ms);
3302}
3303# define ppoll(fds,nfds,ts,sigmask) ruby_ppoll((fds),(nfds),(ts),(sigmask))
3304#endif
3305
3306/*
3307 * Single CPU setups benefit from explicit sched_yield() before ppoll(),
3308 * since threads may be too starved to enter the GVL waitqueue for
3309 * us to detect contention. Instead, we want to kick other threads
3310 * so they can run and possibly prevent us from entering slow paths
3311 * in ppoll() or similar syscalls.
3312 *
3313 * Confirmed on FreeBSD 11.2 and Linux 4.19.
3314 * [ruby-core:90417] [Bug #15398]
3315 */
3316#define THREAD_BLOCKING_YIELD(th) do { \
3317 const rb_thread_t *next_th; \
3318 struct rb_thread_sched *sched = TH_SCHED(th); \
3319 RB_VM_SAVE_MACHINE_CONTEXT(th); \
3320 thread_sched_to_waiting(sched, (th)); \
3321 next_th = sched->running; \
3322 rb_native_mutex_unlock(&sched->lock_); \
3323 native_thread_yield(); /* TODO: needed? */ \
3324 if (!next_th && rb_ractor_living_thread_num(th->ractor) > 1) { \
3325 native_thread_yield(); \
3326 }
3327
3328static void
3329native_sleep(rb_thread_t *th, rb_hrtime_t *rel)
3330{
3331 struct rb_thread_sched *sched = TH_SCHED(th);
3332
3333 RUBY_DEBUG_LOG("rel:%d", rel ? (int)*rel : 0);
3334 if (rel) {
3335 if (th_has_dedicated_nt(th)) {
3336 native_cond_sleep(th, rel);
3337 }
3338 else {
3339 thread_sched_wait_events(sched, th, -1, thread_sched_waiting_timeout, rel);
3340 }
3341 }
3342 else {
3343 thread_sched_to_waiting_until_wakeup(sched, th);
3344 }
3345
3346 RUBY_DEBUG_LOG("wakeup");
3347}
3348
3349// fork read-write lock (only for pthread)
3350static pthread_rwlock_t rb_thread_fork_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
3351
3352void
3353rb_thread_release_fork_lock(void)
3354{
3355 int r;
3356 if ((r = pthread_rwlock_unlock(&rb_thread_fork_rw_lock))) {
3357 rb_bug_errno("pthread_rwlock_unlock", r);
3358 }
3359}
3360
3361void
3362rb_thread_reset_fork_lock(void)
3363{
3364 int r;
3365 if ((r = pthread_rwlock_destroy(&rb_thread_fork_rw_lock))) {
3366 rb_bug_errno("pthread_rwlock_destroy", r);
3367 }
3368
3369 if ((r = pthread_rwlock_init(&rb_thread_fork_rw_lock, NULL))) {
3370 rb_bug_errno("pthread_rwlock_init", r);
3371 }
3372}
3373
3374void *
3375rb_thread_prevent_fork(void *(*func)(void *), void *data)
3376{
3377 int r;
3378 if ((r = pthread_rwlock_rdlock(&rb_thread_fork_rw_lock))) {
3379 rb_bug_errno("pthread_rwlock_rdlock", r);
3380 }
3381 void *result = func(data);
3382 rb_thread_release_fork_lock();
3383 return result;
3384}
3385
3386void
3387rb_thread_acquire_fork_lock(void)
3388{
3389 int r;
3390 if ((r = pthread_rwlock_wrlock(&rb_thread_fork_rw_lock))) {
3391 rb_bug_errno("pthread_rwlock_wrlock", r);
3392 }
3393}
3394
3395// thread internal event hooks (only for pthread)
3396
3397struct rb_internal_thread_event_hook {
3398 rb_internal_thread_event_callback callback;
3399 rb_event_flag_t event;
3400 void *user_data;
3401
3402 struct rb_internal_thread_event_hook *next;
3403};
3404
3405static pthread_rwlock_t rb_internal_thread_event_hooks_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
3406
3407#if defined(HAVE_WORKING_FORK)
3408void
3409rb_internal_thread_event_hooks_rw_lock_atfork(void)
3410{
3411 // After fork(), this rwlock may have been held by a now-dead thread.
3412 //
3413 // pthread_rwlock_destroy() on a held lock is undefined behavior, and
3414 // pthread_rwlock_init() on an already-initialized lock is also undefined
3415 // behavior
3416 //
3417 // Direct assignment of PTHREAD_RWLOCK_INITIALIZER is safe and portable.
3418 rb_internal_thread_event_hooks_rw_lock =
3419 (pthread_rwlock_t)PTHREAD_RWLOCK_INITIALIZER;
3420}
3421#endif
3422
3423rb_internal_thread_event_hook_t *
3424 rb_internal_thread_add_event_hook(rb_internal_thread_event_callback callback, rb_event_flag_t internal_event, void *user_data)
3425{
3426 rb_internal_thread_event_hook_t *hook = ALLOC_N(rb_internal_thread_event_hook_t, 1);
3427 hook->callback = callback;
3428 hook->user_data = user_data;
3429 hook->event = internal_event;
3430
3431 int r;
3432 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
3433 rb_bug_errno("pthread_rwlock_wrlock", r);
3434 }
3435
3436 hook->next = rb_internal_thread_event_hooks;
3437 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook);
3438
3439 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3440 rb_bug_errno("pthread_rwlock_unlock", r);
3441 }
3442 return hook;
3443}
3444
3445bool
3446 rb_internal_thread_remove_event_hook(rb_internal_thread_event_hook_t * hook)
3447{
3448 int r;
3449 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
3450 rb_bug_errno("pthread_rwlock_wrlock", r);
3451 }
3452
3453 bool success = FALSE;
3454
3455 if (rb_internal_thread_event_hooks == hook) {
3456 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook->next);
3457 success = TRUE;
3458 }
3459 else {
3460 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
3461
3462 do {
3463 if (h->next == hook) {
3464 h->next = hook->next;
3465 success = TRUE;
3466 break;
3467 }
3468 } while ((h = h->next));
3469 }
3470
3471 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3472 rb_bug_errno("pthread_rwlock_unlock", r);
3473 }
3474
3475 if (success) {
3476 ruby_xfree(hook);
3477 }
3478 return success;
3479}
3480
3481static void
3482rb_thread_execute_hooks(rb_event_flag_t event, rb_thread_t *th)
3483{
3484 int r;
3485 if ((r = pthread_rwlock_rdlock(&rb_internal_thread_event_hooks_rw_lock))) {
3486 rb_bug_errno("pthread_rwlock_rdlock", r);
3487 }
3488
3489 if (rb_internal_thread_event_hooks) {
3490 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
3491 do {
3492 if (h->event & event) {
3493 rb_internal_thread_event_data_t event_data = {
3494 .thread = th->self,
3495 };
3496 (*h->callback)(event, &event_data, h->user_data);
3497 }
3498 } while((h = h->next));
3499 }
3500 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3501 rb_bug_errno("pthread_rwlock_unlock", r);
3502 }
3503}
3504
3505// return true if the current thread acquires DNT.
3506// return false if the current thread already acquires DNT.
3507bool
3508 rb_thread_lock_native_thread(void)
3509{
3510 rb_thread_t *th = GET_THREAD();
3511 bool is_snt = th->nt->dedicated == 0;
3512 native_thread_dedicated_inc(th->vm, th->ractor, th->nt);
3513
3514 return is_snt;
3515}
3516
3517void
3518rb_thread_malloc_stack_set(rb_thread_t *th, void *stack)
3519{
3520 th->sched.malloc_stack = true;
3521 th->sched.context_stack = stack;
3522}
3523
3524#endif /* THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION */
rb_atomic_t
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
Definition atomic.h:69
RUBY_ATOMIC_FETCH_ADD
#define RUBY_ATOMIC_FETCH_ADD(var, val)
Atomically replaces the value pointed by var with the result of addition of val to the old value of v...
Definition atomic.h:118
RUBY_ATOMIC_LOAD
#define RUBY_ATOMIC_LOAD(var)
Atomic load.
Definition atomic.h:175
RUBY_ATOMIC_SET
#define RUBY_ATOMIC_SET(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except for the return type.
Definition atomic.h:185
rb_event_flag_t
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
INT2FIX
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
ZALLOC
#define ZALLOC
Old name of RB_ZALLOC.
Definition memory.h:402
ALLOC_N
#define ALLOC_N
Old name of RB_ALLOC_N.
Definition memory.h:399
ULL2NUM
#define ULL2NUM
Old name of RB_ULL2NUM.
Definition long_long.h:31
NUM2INT
#define NUM2INT
Old name of RB_NUM2INT.
Definition int.h:44
Qnil
#define Qnil
Old name of RUBY_Qnil.
Definition special_consts.h:60
NIL_P
#define NIL_P
Old name of RB_NIL_P.
Definition special_consts.h:55
rb_eNotImpError
VALUE rb_eNotImpError
NotImplementedError exception.
Definition error.c:1441
rb_syserr_fail
void rb_syserr_fail(int e, const char *mesg)
Raises appropriate exception that represents a C errno.
Definition error.c:3909
rb_bug_errno
void rb_bug_errno(const char *mesg, int errno_arg)
This is a wrapper of rb_bug() which automatically constructs appropriate message from the passed errn...
Definition error.c:1141
rb_cloexec_pipe
int rb_cloexec_pipe(int fildes[2])
Opens a pipe with closing on exec.
Definition io.c:427
rb_update_max_fd
void rb_update_max_fd(int fd)
Informs the interpreter that the passed fd can be the max.
Definition io.c:248
rb_reserved_fd_p
int rb_reserved_fd_p(int fd)
Queries if the given FD is reserved or not.
rb_unblock_function_t
void rb_unblock_function_t(void *)
This is the type of UBFs.
Definition thread.h:336
rb_timespec_now
void rb_timespec_now(struct timespec *ts)
Fills the current time into the given struct.
Definition time.c:2003
len
int len
Length of the buffer.
Definition io.h:8
RUBY_INTERNAL_THREAD_EVENT_RESUMED
#define RUBY_INTERNAL_THREAD_EVENT_RESUMED
Triggered when a thread successfully acquired the GVL.
Definition thread.h:238
rb_internal_thread_add_event_hook
rb_internal_thread_event_hook_t * rb_internal_thread_add_event_hook(rb_internal_thread_event_callback func, rb_event_flag_t events, void *data)
Registers a thread event hook function.
RUBY_INTERNAL_THREAD_EVENT_EXITED
#define RUBY_INTERNAL_THREAD_EVENT_EXITED
Triggered when a thread exits.
Definition thread.h:252
RUBY_INTERNAL_THREAD_EVENT_SUSPENDED
#define RUBY_INTERNAL_THREAD_EVENT_SUSPENDED
Triggered when a thread released the GVL.
Definition thread.h:245
rb_thread_lock_native_thread
bool rb_thread_lock_native_thread(void)
Declare the current Ruby thread should acquire a dedicated native thread on M:N thread scheduler.
RUBY_INTERNAL_THREAD_EVENT_STARTED
#define RUBY_INTERNAL_THREAD_EVENT_STARTED
Triggered when a new thread is started.
Definition thread.h:224
rb_internal_thread_remove_event_hook
bool rb_internal_thread_remove_event_hook(rb_internal_thread_event_hook_t *hook)
Unregister the passed hook.
RUBY_INTERNAL_THREAD_EVENT_READY
#define RUBY_INTERNAL_THREAD_EVENT_READY
Triggered when a thread attempt to acquire the GVL.
Definition thread.h:231
RBIMPL_ATTR_MAYBE_UNUSED
#define RBIMPL_ATTR_MAYBE_UNUSED()
Wraps (or simulates) [[maybe_unused]].
Definition maybe_unused.h:35
RB_GC_GUARD
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:167
rb_fd_select
#define rb_fd_select
Waits for multiple file descriptors at once.
Definition posix.h:66
RARRAY_AREF
#define RARRAY_AREF(a, i)
Definition rarray.h:403
RSTRING_GETMEM
#define RSTRING_GETMEM(str, ptrvar, lenvar)
Convenient macro to obtain the contents and length at once.
Definition rstring.h:450
errno
#define errno
Ractor-aware version of errno.
Definition ruby.h:388
coroutine_context
Definition Context.h:36
rb_fdset_t
The data structure which wraps the fd_set bitmap used by select(2).
Definition largesize.h:71
rb_native_thread
Definition thread_none.h:11
rb_thread_sched_waiting
Definition thread_pthread.h:26
rb_thread_sched
Definition thread_none.h:16
timespec
Definition missing.h:74
timeval
Definition missing.h:63
tr
Definition string.c:8221
rb_nativethread_self
rb_nativethread_id_t rb_nativethread_self(void)
Queries the ID of the native thread that is calling this function.
rb_native_mutex_lock
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
rb_native_cond_initialize
void rb_native_cond_initialize(rb_nativethread_cond_t *cond)
Fills the passed condition variable with an initial value.
rb_native_mutex_trylock
int rb_native_mutex_trylock(rb_nativethread_lock_t *lock)
Identical to rb_native_mutex_lock(), except it doesn't block in case rb_native_mutex_lock() would.
rb_native_cond_broadcast
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
rb_native_mutex_initialize
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
rb_native_mutex_unlock
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
rb_native_mutex_destroy
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
rb_native_cond_destroy
void rb_native_cond_destroy(rb_nativethread_cond_t *cond)
Destroys the passed condition variable.
rb_native_cond_signal
void rb_native_cond_signal(rb_nativethread_cond_t *cond)
Signals a condition variable.
rb_native_cond_wait
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.
rb_native_cond_timedwait
void rb_native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, unsigned long msec)
Identical to rb_native_cond_wait(), except it additionally takes timeout in msec resolution.
__attribute__
Definition vm_insnhelper.c:2399
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
ruby_xfree
void ruby_xfree(void *ptr)
Deallocates a storage instance.
Definition gc.c:5262
ruby_xmalloc
void * ruby_xmalloc(size_t size)
Allocates a storage instance.
Definition gc.c:5124