X-Git-Url: https://git.saurik.com/apple/libpthread.git/blobdiff_plain/76b7b9a2a65d05f65ded82a6675bf63a7f569766..4d9153646b79616988baf143c3597b37a778e989:/src/pthread.c?ds=inline diff --git a/src/pthread.c b/src/pthread.c index 8e63bd3..57ff8e9 100644 --- a/src/pthread.c +++ b/src/pthread.c @@ -49,13 +49,7 @@ * POSIX Pthread Library */ -#include "resolver.h" #include "internal.h" -#include "private.h" -#include "workqueue_private.h" -#include "introspection_private.h" -#include "qos_private.h" -#include "tsd_private.h" #include #include @@ -63,376 +57,180 @@ #include #include #include +#include #include #include #include #include +#include #include #include #define __APPLE_API_PRIVATE #include +#if __has_include() +#include +#endif // __has_include() +#include +#include -#include <_simple.h> -#include -#include - -extern int __sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen); -extern void __exit(int) __attribute__((noreturn)); -extern int __pthread_kill(mach_port_t, int); - -extern struct _pthread _thread; -extern int default_priority; - +// Default stack size is 512KB; independent of the main thread's stack size. +#define DEFAULT_STACK_SIZE (size_t)(512 * 1024) // -// Global variables +// Global constants // -static void (*exitf)(int) = __exit; -PTHREAD_NOEXPORT void* (*_pthread_malloc)(size_t) = NULL; -PTHREAD_NOEXPORT void (*_pthread_free)(void *) = NULL; - -#if PTHREAD_DEBUG_LOG -#include -int _pthread_debuglog; -uint64_t _pthread_debugstart; -#endif - -// This global should be used (carefully) by anyone needing to know if a -// pthread (other than the main thread) has been created. -int __is_threaded = 0; - -int __unix_conforming = 0; +/* + * The pthread may be offset into a page. In that event, by contract + * with the kernel, the allocation will extend PTHREAD_SIZE from the + * start of the next page. There's also one page worth of allocation + * below stacksize for the guard page. + */ +#define PTHREAD_SIZE ((size_t)mach_vm_round_page(sizeof(struct pthread_s))) +#define PTHREAD_ALLOCADDR(stackaddr, stacksize) ((stackaddr - stacksize) - vm_page_size) +#define PTHREAD_ALLOCSIZE(stackaddr, stacksize) ((round_page((uintptr_t)stackaddr) + PTHREAD_SIZE) - (uintptr_t)PTHREAD_ALLOCADDR(stackaddr, stacksize)) -// _pthread_list_lock protects _pthread_count, access to the __pthread_head -// list, and the parentcheck, childrun and childexit flags of the pthread -// structure. Externally imported by pthread_cancelable.c. -PTHREAD_NOEXPORT _pthread_lock _pthread_list_lock = _PTHREAD_LOCK_INITIALIZER; -PTHREAD_NOEXPORT struct __pthread_list __pthread_head = TAILQ_HEAD_INITIALIZER(__pthread_head); -static int _pthread_count = 1; +static const pthread_attr_t _pthread_attr_default = { + .sig = _PTHREAD_ATTR_SIG, + .stacksize = 0, + .detached = PTHREAD_CREATE_JOINABLE, + .inherit = _PTHREAD_DEFAULT_INHERITSCHED, + .policy = _PTHREAD_DEFAULT_POLICY, + .defaultguardpage = true, + // compile time constant for _pthread_default_priority(0) + .qosclass = (1U << (THREAD_QOS_LEGACY - 1 + _PTHREAD_PRIORITY_QOS_CLASS_SHIFT)) | + ((uint8_t)-1 & _PTHREAD_PRIORITY_PRIORITY_MASK), +}; #if PTHREAD_LAYOUT_SPI const struct pthread_layout_offsets_s pthread_layout_offsets = { .plo_version = 1, - .plo_pthread_tsd_base_offset = offsetof(struct _pthread, tsd), + .plo_pthread_tsd_base_offset = offsetof(struct pthread_s, tsd), .plo_pthread_tsd_base_address_offset = 0, - .plo_pthread_tsd_entry_size = sizeof(((struct _pthread *)NULL)->tsd[0]), + .plo_pthread_tsd_entry_size = sizeof(((struct pthread_s *)NULL)->tsd[0]), }; #endif // PTHREAD_LAYOUT_SPI // -// Static variables +// Global exported variables // -// Mach message notification that a thread needs to be recycled. -typedef struct _pthread_reap_msg_t { - mach_msg_header_t header; - pthread_t thread; - mach_msg_trailer_t trailer; -} pthread_reap_msg_t; +// This global should be used (carefully) by anyone needing to know if a +// pthread (other than the main thread) has been created. +int __is_threaded = 0; +const int __unix_conforming = 1; // we're always conformant, but it's exported -/* - * The pthread may be offset into a page. In that event, by contract - * with the kernel, the allocation will extend PTHREAD_SIZE from the - * start of the next page. There's also one page worth of allocation - * below stacksize for the guard page. - */ -#define PTHREAD_SIZE ((size_t)mach_vm_round_page(sizeof(struct _pthread))) -#define PTHREAD_ALLOCADDR(stackaddr, stacksize) ((stackaddr - stacksize) - vm_page_size) -#define PTHREAD_ALLOCSIZE(stackaddr, stacksize) ((round_page((uintptr_t)stackaddr) + PTHREAD_SIZE) - (uintptr_t)PTHREAD_ALLOCADDR(stackaddr, stacksize)) +// +// Global internal variables +// + +// _pthread_list_lock protects _pthread_count, access to the __pthread_head +// list. Externally imported by pthread_cancelable.c. +struct __pthread_list __pthread_head = TAILQ_HEAD_INITIALIZER(__pthread_head); +_pthread_lock _pthread_list_lock = _PTHREAD_LOCK_INITIALIZER; -static pthread_attr_t _pthread_attr_default = { }; +uint32_t _main_qos; +#if VARIANT_DYLD // The main thread's pthread_t -PTHREAD_NOEXPORT struct _pthread _thread __attribute__((aligned(64))) = { }; +struct pthread_s _main_thread OS_ALIGNED(64); +#else // VARIANT_DYLD +pthread_t _main_thread_ptr; +void *(*_pthread_malloc)(size_t); +void (*_pthread_free)(void *); +#endif // VARIANT_DYLD + +#if PTHREAD_DEBUG_LOG +#include +int _pthread_debuglog; +uint64_t _pthread_debugstart; +#endif -PTHREAD_NOEXPORT int default_priority; -static int max_priority; -static int min_priority; +// +// Global static variables +// +static bool __workq_newapi; +static uint8_t default_priority; +#if !VARIANT_DYLD +static uint8_t max_priority; +static uint8_t min_priority; +#endif // !VARIANT_DYLD +static int _pthread_count = 1; static int pthread_concurrency; +uintptr_t _pthread_ptr_munge_token; + +static void (*exitf)(int) = __exit; // work queue support data -static void (*__libdispatch_workerfunction)(pthread_priority_t) = NULL; -static void (*__libdispatch_keventfunction)(void **events, int *nevents) = NULL; -static void (*__libdispatch_workloopfunction)(uint64_t *workloop_id, void **events, int *nevents) = NULL; -static int __libdispatch_offset; +OS_NORETURN OS_COLD +static void +__pthread_invalid_keventfunction(void **events, int *nevents) +{ + PTHREAD_CLIENT_CRASH(0, "Invalid kqworkq setup"); +} -// supported feature set -int __pthread_supported_features; -static bool __workq_newapi; +OS_NORETURN OS_COLD +static void +__pthread_invalid_workloopfunction(uint64_t *workloop_id, void **events, int *nevents) +{ + PTHREAD_CLIENT_CRASH(0, "Invalid kqwl setup"); +} +static pthread_workqueue_function2_t __libdispatch_workerfunction; +static pthread_workqueue_function_kevent_t __libdispatch_keventfunction = &__pthread_invalid_keventfunction; +static pthread_workqueue_function_workloop_t __libdispatch_workloopfunction = &__pthread_invalid_workloopfunction; +static int __pthread_supported_features; // supported feature set + +#if defined(__i386__) || defined(__x86_64__) +static mach_vm_address_t __pthread_stack_hint = 0xB0000000; +#elif defined(__arm__) || defined(__arm64__) +static mach_vm_address_t __pthread_stack_hint = 0x30000000; +#else +#error no __pthread_stack_hint for this architecture +#endif // // Function prototypes // // pthread primitives -static int _pthread_allocate(pthread_t *thread, const pthread_attr_t *attrs, void **stack); -static int _pthread_deallocate(pthread_t t); - -static void _pthread_terminate_invoke(pthread_t t); - -static inline void _pthread_struct_init(pthread_t t, - const pthread_attr_t *attrs, - void *stack, - size_t stacksize, - void *freeaddr, - size_t freesize); +static inline void _pthread_struct_init(pthread_t t, const pthread_attr_t *attrs, + void *stack, size_t stacksize, void *freeaddr, size_t freesize); -static inline void _pthread_set_self_internal(pthread_t, bool needs_tsd_base_set); +#if VARIANT_DYLD +static void _pthread_set_self_dyld(void); +#endif // VARIANT_DYLD +static inline void _pthread_set_self_internal(pthread_t); -static void _pthread_dealloc_reply_port(pthread_t t); -static void _pthread_dealloc_special_reply_port(pthread_t t); - -static inline void __pthread_add_thread(pthread_t t, const pthread_attr_t *attr, bool parent, bool from_mach_thread); -static inline int __pthread_remove_thread(pthread_t t, bool child, bool *should_exit); +static inline void __pthread_started_thread(pthread_t t); static void _pthread_exit(pthread_t self, void *value_ptr) __dead2; -static inline void _pthread_introspection_thread_create(pthread_t t, bool destroy); +static inline void _pthread_introspection_thread_create(pthread_t t); static inline void _pthread_introspection_thread_start(pthread_t t); -static inline void _pthread_introspection_thread_terminate(pthread_t t, void *freeaddr, size_t freesize, bool destroy); +static inline void _pthread_introspection_thread_terminate(pthread_t t); static inline void _pthread_introspection_thread_destroy(pthread_t t); -extern void _pthread_set_self(pthread_t); -extern void start_wqthread(pthread_t self, mach_port_t kport, void *stackaddr, void *unused, int reuse); // trampoline into _pthread_wqthread -extern void thread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags); // trampoline into _pthread_start - -/* Compatibility: previous pthread API used WORKQUEUE_OVERCOMMIT to request overcommit threads from - * the kernel. This definition is kept here, in userspace only, to perform the compatibility shimm - * from old API requests to the new kext conventions. - */ -#define WORKQUEUE_OVERCOMMIT 0x10000 - /* * Flags filed passed to bsdthread_create and back in pthread_start -31 <---------------------------------> 0 -_________________________________________ -| flags(8) | policy(8) | importance(16) | ------------------------------------------ -*/ - -#define PTHREAD_START_CUSTOM 0x01000000 + * 31 <---------------------------------> 0 + * _________________________________________ + * | flags(8) | policy(8) | importance(16) | + * ----------------------------------------- + */ +#define PTHREAD_START_CUSTOM 0x01000000 // #define PTHREAD_START_SETSCHED 0x02000000 -#define PTHREAD_START_DETACHED 0x04000000 +// was PTHREAD_START_DETACHED 0x04000000 #define PTHREAD_START_QOSCLASS 0x08000000 #define PTHREAD_START_TSD_BASE_SET 0x10000000 +#define PTHREAD_START_SUSPENDED 0x20000000 #define PTHREAD_START_QOSCLASS_MASK 0x00ffffff #define PTHREAD_START_POLICY_BITSHIFT 16 #define PTHREAD_START_POLICY_MASK 0xff #define PTHREAD_START_IMPORTANCE_MASK 0xffff -static int pthread_setschedparam_internal(pthread_t, mach_port_t, int, const struct sched_param *); -extern pthread_t __bsdthread_create(void *(*func)(void *), void * func_arg, void * stack, pthread_t thread, unsigned int flags); -extern int __bsdthread_register(void (*)(pthread_t, mach_port_t, void *(*)(void *), void *, size_t, unsigned int), void (*)(pthread_t, mach_port_t, void *, void *, int), int,void (*)(pthread_t, mach_port_t, void *(*)(void *), void *, size_t, unsigned int), int32_t *,__uint64_t); -extern int __bsdthread_terminate(void * freeaddr, size_t freesize, mach_port_t kport, mach_port_t joinsem); -extern __uint64_t __thread_selfid( void ); - -extern int __workq_open(void); -extern int __workq_kernreturn(int, void *, int, int); - -#if defined(__i386__) || defined(__x86_64__) -static const mach_vm_address_t PTHREAD_STACK_HINT = 0xB0000000; -#else -#error no PTHREAD_STACK_HINT for this architecture -#endif - -// Check that offsets of _PTHREAD_STRUCT_DIRECT_*_OFFSET values hasn't changed -_Static_assert(offsetof(struct _pthread, tsd) + _PTHREAD_STRUCT_DIRECT_THREADID_OFFSET - == offsetof(struct _pthread, thread_id), - "_PTHREAD_STRUCT_DIRECT_THREADID_OFFSET is correct"); - -// Allocate a thread structure, stack and guard page. -// -// The thread structure may optionally be placed in the same allocation as the -// stack, residing above the top of the stack. This cannot be done if a -// custom stack address is provided. -// -// Similarly the guard page cannot be allocated if a custom stack address is -// provided. -// -// The allocated thread structure is initialized with values that indicate how -// it should be freed. - -static int -_pthread_allocate(pthread_t *thread, const pthread_attr_t *attrs, void **stack) -{ - int res; - kern_return_t kr; - pthread_t t = NULL; - mach_vm_address_t allocaddr = PTHREAD_STACK_HINT; - size_t allocsize = 0; - size_t guardsize = 0; - size_t stacksize = 0; - - PTHREAD_ASSERT(attrs->stacksize >= PTHREAD_STACK_MIN); - - *thread = NULL; - *stack = NULL; - - // Allocate a pthread structure if necessary - - if (attrs->stackaddr != NULL) { - PTHREAD_ASSERT(((uintptr_t)attrs->stackaddr % vm_page_size) == 0); - *stack = attrs->stackaddr; - allocsize = PTHREAD_SIZE; - } else { - guardsize = attrs->guardsize; - stacksize = attrs->stacksize; - allocsize = stacksize + guardsize + PTHREAD_SIZE; - } - - kr = mach_vm_map(mach_task_self(), - &allocaddr, - allocsize, - vm_page_size - 1, - VM_MAKE_TAG(VM_MEMORY_STACK)| VM_FLAGS_ANYWHERE, - MEMORY_OBJECT_NULL, - 0, - FALSE, - VM_PROT_DEFAULT, - VM_PROT_ALL, - VM_INHERIT_DEFAULT); - - if (kr != KERN_SUCCESS) { - kr = mach_vm_allocate(mach_task_self(), - &allocaddr, - allocsize, - VM_MAKE_TAG(VM_MEMORY_STACK)| VM_FLAGS_ANYWHERE); - } - - if (kr == KERN_SUCCESS) { - // The stack grows down. - // Set the guard page at the lowest address of the - // newly allocated stack. Return the highest address - // of the stack. - if (guardsize) { - (void)mach_vm_protect(mach_task_self(), allocaddr, guardsize, FALSE, VM_PROT_NONE); - } - - // Thread structure resides at the top of the stack. - t = (void *)(allocaddr + stacksize + guardsize); - if (stacksize) { - // Returns the top of the stack. - *stack = t; - } - } - - if (t != NULL) { - _pthread_struct_init(t, attrs, - *stack, attrs->stacksize, - allocaddr, allocsize); - *thread = t; - res = 0; - } else { - res = EAGAIN; - } - return res; -} - -static int -_pthread_deallocate(pthread_t t) -{ - // Don't free the main thread. - if (t != &_thread) { - kern_return_t ret; - ret = mach_vm_deallocate(mach_task_self(), t->freeaddr, t->freesize); - PTHREAD_ASSERT(ret == KERN_SUCCESS); - } - return 0; -} - -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wreturn-stack-address" - -PTHREAD_NOINLINE -static void* -_pthread_current_stack_address(void) -{ - int a; - return &a; -} - -#pragma clang diagnostic pop - -// Terminates the thread if called from the currently running thread. -PTHREAD_NORETURN PTHREAD_NOINLINE PTHREAD_NOT_TAIL_CALLED -static void -_pthread_terminate(pthread_t t) -{ - PTHREAD_ASSERT(t == pthread_self()); - - uintptr_t freeaddr = (uintptr_t)t->freeaddr; - size_t freesize = t->freesize; - - // the size of just the stack - size_t freesize_stack = t->freesize; - - // We usually pass our structure+stack to bsdthread_terminate to free, but - // if we get told to keep the pthread_t structure around then we need to - // adjust the free size and addr in the pthread_t to just refer to the - // structure and not the stack. If we do end up deallocating the - // structure, this is useless work since no one can read the result, but we - // can't do it after the call to pthread_remove_thread because it isn't - // safe to dereference t after that. - if ((void*)t > t->freeaddr && (void*)t < t->freeaddr + t->freesize){ - // Check to ensure the pthread structure itself is part of the - // allocation described by freeaddr/freesize, in which case we split and - // only deallocate the area below the pthread structure. In the event of a - // custom stack, the freeaddr/size will be the pthread structure itself, in - // which case we shouldn't free anything (the final else case). - freesize_stack = trunc_page((uintptr_t)t - (uintptr_t)freeaddr); - - // describe just the remainder for deallocation when the pthread_t goes away - t->freeaddr += freesize_stack; - t->freesize -= freesize_stack; - } else if (t == &_thread){ - freeaddr = t->stackaddr - pthread_get_stacksize_np(t); - uintptr_t stackborder = trunc_page((uintptr_t)_pthread_current_stack_address()); - freesize_stack = stackborder - freeaddr; - } else { - freesize_stack = 0; - } - - mach_port_t kport = _pthread_kernel_thread(t); - semaphore_t joinsem = t->joiner_notify; - - _pthread_dealloc_special_reply_port(t); - _pthread_dealloc_reply_port(t); - - // After the call to __pthread_remove_thread, it is not safe to - // dereference the pthread_t structure. - - bool destroy, should_exit; - destroy = (__pthread_remove_thread(t, true, &should_exit) != EBUSY); - - if (!destroy || t == &_thread) { - // Use the adjusted freesize of just the stack that we computed above. - freesize = freesize_stack; - } - - // Check if there is nothing to free because the thread has a custom - // stack allocation and is joinable. - if (freesize == 0) { - freeaddr = 0; - } - _pthread_introspection_thread_terminate(t, freeaddr, freesize, destroy); - if (should_exit) { - exitf(0); - } - - __bsdthread_terminate((void *)freeaddr, freesize, kport, joinsem); - PTHREAD_ABORT("thread %p didn't terminate", t); -} - -PTHREAD_NORETURN -static void -_pthread_terminate_invoke(pthread_t t) -{ - _pthread_terminate(t); -} +#pragma mark pthread attrs int pthread_attr_destroy(pthread_attr_t *attr) @@ -467,12 +265,24 @@ pthread_attr_getinheritsched(const pthread_attr_t *attr, int *inheritsched) return ret; } +static OS_ALWAYS_INLINE void +_pthread_attr_get_schedparam(const pthread_attr_t *attr, + struct sched_param *param) +{ + if (attr->schedset) { + *param = attr->param; + } else { + param->sched_priority = default_priority; + param->quantum = 10; /* quantum isn't public yet */ + } +} + int pthread_attr_getschedparam(const pthread_attr_t *attr, struct sched_param *param) { int ret = EINVAL; if (attr->sig == _PTHREAD_ATTR_SIG) { - *param = attr->param; + _pthread_attr_get_schedparam(attr, param); ret = 0; } return ret; @@ -489,24 +299,10 @@ pthread_attr_getschedpolicy(const pthread_attr_t *attr, int *policy) return ret; } -// Default stack size is 512KB; independent of the main thread's stack size. -static const size_t DEFAULT_STACK_SIZE = 512 * 1024; - int pthread_attr_init(pthread_attr_t *attr) { - attr->stacksize = DEFAULT_STACK_SIZE; - attr->stackaddr = NULL; - attr->sig = _PTHREAD_ATTR_SIG; - attr->param.sched_priority = default_priority; - attr->param.quantum = 10; /* quantum isn't public yet */ - attr->detached = PTHREAD_CREATE_JOINABLE; - attr->inherit = _PTHREAD_DEFAULT_INHERITSCHED; - attr->policy = _PTHREAD_DEFAULT_POLICY; - attr->fastpath = 1; - attr->schedset = 0; - attr->guardsize = vm_page_size; - attr->qosclass = _pthread_priority_make_newest(QOS_CLASS_DEFAULT, 0, 0); + *attr = _pthread_attr_default; return 0; } @@ -515,8 +311,8 @@ pthread_attr_setdetachstate(pthread_attr_t *attr, int detachstate) { int ret = EINVAL; if (attr->sig == _PTHREAD_ATTR_SIG && - (detachstate == PTHREAD_CREATE_JOINABLE || - detachstate == PTHREAD_CREATE_DETACHED)) { + (detachstate == PTHREAD_CREATE_JOINABLE || + detachstate == PTHREAD_CREATE_DETACHED)) { attr->detached = detachstate; ret = 0; } @@ -528,8 +324,8 @@ pthread_attr_setinheritsched(pthread_attr_t *attr, int inheritsched) { int ret = EINVAL; if (attr->sig == _PTHREAD_ATTR_SIG && - (inheritsched == PTHREAD_INHERIT_SCHED || - inheritsched == PTHREAD_EXPLICIT_SCHED)) { + (inheritsched == PTHREAD_INHERIT_SCHED || + inheritsched == PTHREAD_EXPLICIT_SCHED)) { attr->inherit = inheritsched; ret = 0; } @@ -549,16 +345,20 @@ pthread_attr_setschedparam(pthread_attr_t *attr, const struct sched_param *param return ret; } +#define _PTHREAD_POLICY_IS_FIXEDPRI(x) ((x) == SCHED_RR || (x) == SCHED_FIFO) + int pthread_attr_setschedpolicy(pthread_attr_t *attr, int policy) { int ret = EINVAL; - if (attr->sig == _PTHREAD_ATTR_SIG && - (policy == SCHED_OTHER || - policy == SCHED_RR || - policy == SCHED_FIFO)) { + if (attr->sig == _PTHREAD_ATTR_SIG && (policy == SCHED_OTHER || + policy == SCHED_RR || policy == SCHED_FIFO)) { + if (!_PTHREAD_POLICY_IS_FIXEDPRI(policy)) { + /* non-fixedpri policy should remove cpupercent */ + attr->cpupercentset = 0; + } attr->policy = policy; - attr->schedset = 1; + attr->policyset = 1; ret = 0; } return ret; @@ -606,21 +406,27 @@ pthread_attr_setstackaddr(pthread_attr_t *attr, void *stackaddr) { int ret = EINVAL; if (attr->sig == _PTHREAD_ATTR_SIG && - ((uintptr_t)stackaddr % vm_page_size) == 0) { + ((mach_vm_address_t)stackaddr & vm_page_mask) == 0) { attr->stackaddr = stackaddr; - attr->fastpath = 0; + attr->defaultguardpage = false; attr->guardsize = 0; ret = 0; } return ret; } +static inline size_t +_pthread_attr_stacksize(const pthread_attr_t *attr) +{ + return attr->stacksize ? attr->stacksize : DEFAULT_STACK_SIZE; +} + int pthread_attr_getstacksize(const pthread_attr_t *attr, size_t *stacksize) { int ret = EINVAL; if (attr->sig == _PTHREAD_ATTR_SIG) { - *stacksize = attr->stacksize; + *stacksize = _pthread_attr_stacksize(attr); ret = 0; } return ret; @@ -629,10 +435,17 @@ pthread_attr_getstacksize(const pthread_attr_t *attr, size_t *stacksize) int pthread_attr_setstacksize(pthread_attr_t *attr, size_t stacksize) { +#if TARGET_OS_OSX + // If the caller is doing something reasonable, help them out. + if (stacksize % 0x1000 == 0) { + stacksize = round_page(stacksize); + } +#endif // TARGET_OS_OSX + int ret = EINVAL; if (attr->sig == _PTHREAD_ATTR_SIG && - (stacksize % vm_page_size) == 0 && - stacksize >= PTHREAD_STACK_MIN) { + ((stacksize & vm_page_mask) == 0) && + stacksize >= PTHREAD_STACK_MIN) { attr->stacksize = stacksize; ret = 0; } @@ -645,7 +458,7 @@ pthread_attr_getstack(const pthread_attr_t *attr, void **stackaddr, size_t * sta int ret = EINVAL; if (attr->sig == _PTHREAD_ATTR_SIG) { *stackaddr = (void *)((uintptr_t)attr->stackaddr - attr->stacksize); - *stacksize = attr->stacksize; + *stacksize = _pthread_attr_stacksize(attr); ret = 0; } return ret; @@ -658,12 +471,11 @@ pthread_attr_setstack(pthread_attr_t *attr, void *stackaddr, size_t stacksize) { int ret = EINVAL; if (attr->sig == _PTHREAD_ATTR_SIG && - ((uintptr_t)stackaddr % vm_page_size) == 0 && - (stacksize % vm_page_size) == 0 && - stacksize >= PTHREAD_STACK_MIN) { + (((mach_vm_address_t)stackaddr & vm_page_mask) == 0) && + ((stacksize & vm_page_mask) == 0) && + stacksize >= PTHREAD_STACK_MIN) { attr->stackaddr = (void *)((uintptr_t)stackaddr + stacksize); - attr->stacksize = stacksize; - attr->fastpath = 0; + attr->stacksize = stacksize; ret = 0; } return ret; @@ -672,134 +484,432 @@ pthread_attr_setstack(pthread_attr_t *attr, void *stackaddr, size_t stacksize) int pthread_attr_setguardsize(pthread_attr_t *attr, size_t guardsize) { +#if TARGET_OS_OSX + // If the caller is doing something reasonable, help them out. + if (guardsize % 0x1000 == 0) { + guardsize = round_page(guardsize); + } +#endif // TARGET_OS_OSX + int ret = EINVAL; - if (attr->sig == _PTHREAD_ATTR_SIG) { - /* Guardsize of 0 is valid, ot means no guard */ - if ((guardsize % vm_page_size) == 0) { - attr->guardsize = guardsize; - attr->fastpath = 0; - ret = 0; - } + if (attr->sig == _PTHREAD_ATTR_SIG && + (guardsize & vm_page_mask) == 0) { + /* Guardsize of 0 is valid, means no guard */ + attr->defaultguardpage = false; + attr->guardsize = guardsize; + ret = 0; } return ret; } +static inline size_t +_pthread_attr_guardsize(const pthread_attr_t *attr) +{ + return attr->defaultguardpage ? vm_page_size : attr->guardsize; +} + int pthread_attr_getguardsize(const pthread_attr_t *attr, size_t *guardsize) { int ret = EINVAL; if (attr->sig == _PTHREAD_ATTR_SIG) { - *guardsize = attr->guardsize; + *guardsize = _pthread_attr_guardsize(attr); ret = 0; } return ret; } +int +pthread_attr_setcpupercent_np(pthread_attr_t *attr, int percent, + unsigned long refillms) +{ + int ret = EINVAL; + if (attr->sig == _PTHREAD_ATTR_SIG && percent < UINT8_MAX && + refillms < _PTHREAD_ATTR_REFILLMS_MAX && attr->policyset && + _PTHREAD_POLICY_IS_FIXEDPRI(attr->policy)) { + attr->cpupercent = percent; + attr->refillms = (uint32_t)(refillms & 0x00ffffff); + attr->cpupercentset = 1; + ret = 0; + } + return ret; +} + +#pragma mark pthread lifetime + +// Allocate a thread structure, stack and guard page. +// +// The thread structure may optionally be placed in the same allocation as the +// stack, residing above the top of the stack. This cannot be done if a +// custom stack address is provided. +// +// Similarly the guard page cannot be allocated if a custom stack address is +// provided. +// +// The allocated thread structure is initialized with values that indicate how +// it should be freed. + +static pthread_t +_pthread_allocate(const pthread_attr_t *attrs, void **stack, + bool from_mach_thread) +{ + mach_vm_address_t allocaddr = __pthread_stack_hint; + size_t allocsize, guardsize, stacksize, pthreadoff; + kern_return_t kr; + pthread_t t; + + if (os_unlikely(attrs->stacksize != 0 && + attrs->stacksize < PTHREAD_STACK_MIN)) { + PTHREAD_CLIENT_CRASH(attrs->stacksize, "Stack size in attrs is too small"); + } + + if (os_unlikely((mach_vm_address_t)attrs->stackaddr & vm_page_mask)) { + PTHREAD_CLIENT_CRASH(attrs->stackaddr, "Unaligned stack addr in attrs"); + } + + // Allocate a pthread structure if necessary + + if (attrs->stackaddr != NULL) { + allocsize = PTHREAD_SIZE; + guardsize = 0; + pthreadoff = 0; + // if the attrs struct specifies a custom + // stack address but not a custom size, using ->stacksize here instead + // of _pthread_attr_stacksize stores stacksize as zero, indicating + // that the stack size is unknown. + stacksize = attrs->stacksize; + } else { + guardsize = _pthread_attr_guardsize(attrs); + stacksize = _pthread_attr_stacksize(attrs) + PTHREAD_T_OFFSET; + pthreadoff = stacksize + guardsize; + allocsize = pthreadoff + PTHREAD_SIZE; + allocsize = mach_vm_round_page(allocsize); + } + + kr = mach_vm_map(mach_task_self(), &allocaddr, allocsize, vm_page_size - 1, + VM_MAKE_TAG(VM_MEMORY_STACK)| VM_FLAGS_ANYWHERE, MEMORY_OBJECT_NULL, + 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); + + if (kr != KERN_SUCCESS) { + kr = mach_vm_allocate(mach_task_self(), &allocaddr, allocsize, + VM_MAKE_TAG(VM_MEMORY_STACK)| VM_FLAGS_ANYWHERE); + } else if (__syscall_logger && !from_mach_thread) { + // libsyscall will not output malloc stack logging events when + // VM_MEMORY_STACK is passed in to facilitate mach thread promotion. + // To avoid losing the stack traces for normal p-thread create + // operations, libpthread must pretend to be the vm syscall and log + // the allocations. + int eventTypeFlags = stack_logging_type_vm_allocate | + stack_logging_type_mapped_file_or_shared_mem; + __syscall_logger(eventTypeFlags | VM_MAKE_TAG(VM_MEMORY_STACK), + (uintptr_t)mach_task_self(), (uintptr_t)allocsize, 0, + (uintptr_t)allocaddr, 0); + } + + if (kr != KERN_SUCCESS) { + *stack = NULL; + return NULL; + } else if (__syscall_logger && !from_mach_thread) { + // libsyscall will not output malloc stack logging events when + // VM_MEMORY_STACK is passed in to facilitate mach thread promotion. + // To avoid losing the stack traces for normal p-thread create + // operations, libpthread must pretend to be the vm syscall and log + // the allocations. + int eventTypeFlags = stack_logging_type_vm_allocate; + __syscall_logger(eventTypeFlags | VM_MAKE_TAG(VM_MEMORY_STACK), + (uintptr_t)mach_task_self(), (uintptr_t)allocsize, 0, + (uintptr_t)allocaddr, 0); + } + + // The stack grows down. + // Set the guard page at the lowest address of the + // newly allocated stack. Return the highest address + // of the stack. + if (guardsize) { + (void)mach_vm_protect(mach_task_self(), allocaddr, guardsize, + FALSE, VM_PROT_NONE); + } + + // Thread structure resides at the top of the stack (when using a + // custom stack, allocsize == PTHREAD_SIZE, so places the pthread_t + // at allocaddr). + t = (pthread_t)(allocaddr + pthreadoff); + if (attrs->stackaddr) { + *stack = attrs->stackaddr; + } else { + *stack = t; + } + + _pthread_struct_init(t, attrs, *stack, stacksize, allocaddr, allocsize); + return t; +} + +OS_NOINLINE +void +_pthread_deallocate(pthread_t t, bool from_mach_thread) +{ + kern_return_t ret; + + // Don't free the main thread. + if (t != main_thread()) { + if (!from_mach_thread) { // see __pthread_add_thread + _pthread_introspection_thread_destroy(t); + } + ret = mach_vm_deallocate(mach_task_self(), t->freeaddr, t->freesize); + if (ret != KERN_SUCCESS) { + PTHREAD_INTERNAL_CRASH(ret, "Unable to deallocate stack"); + } + } +} + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-stack-address" + +OS_NOINLINE +static void* +_pthread_current_stack_address(void) +{ + int a; + return &a; +} + +#pragma clang diagnostic pop -/* - * Create and start execution of a new thread. - */ -PTHREAD_NOINLINE PTHREAD_NORETURN static void -_pthread_body(pthread_t self, bool needs_tsd_base_set) +_pthread_joiner_wake(pthread_t thread) { - _pthread_set_self_internal(self, needs_tsd_base_set); - __pthread_add_thread(self, NULL, false, false); - void *result = (self->fun)(self->arg); + uint32_t *exit_gate = &thread->tl_exit_gate; + + for (;;) { + int ret = __ulock_wake(UL_UNFAIR_LOCK | ULF_NO_ERRNO, exit_gate, 0); + if (ret == 0 || ret == -ENOENT) { + return; + } + if (ret != -EINTR) { + PTHREAD_INTERNAL_CRASH(-ret, "pthread_join() wake failure"); + } + } +} + +static void +_pthread_dealloc_reply_port(pthread_t self) +{ + mach_port_t port = _pthread_tsd_slot(self, MIG_REPLY); + if (port != MACH_PORT_NULL) { + // this will also set the TSD to MACH_PORT_NULL + mig_dealloc_reply_port(port); + } +} + +static void +_pthread_dealloc_special_reply_port(pthread_t self) +{ + mach_port_t port = _pthread_tsd_slot(self, MACH_SPECIAL_REPLY); + if (port != MACH_PORT_NULL) { + _pthread_tsd_slot(self, MACH_SPECIAL_REPLY) = MACH_PORT_NULL; + thread_destruct_special_reply_port(port, THREAD_SPECIAL_REPLY_PORT_ALL); + } +} + +// Terminates the thread if called from the currently running thread. +OS_NORETURN OS_NOINLINE OS_NOT_TAIL_CALLED +static void +_pthread_terminate(pthread_t t, void *exit_value) +{ + _pthread_introspection_thread_terminate(t); + + uintptr_t freeaddr = (uintptr_t)t->freeaddr; + size_t freesize = t->freesize; + bool should_exit; + + // the size of just the stack + size_t freesize_stack = t->freesize; + + // We usually pass our structure+stack to bsdthread_terminate to free, but + // if we get told to keep the pthread_t structure around then we need to + // adjust the free size and addr in the pthread_t to just refer to the + // structure and not the stack. If we do end up deallocating the + // structure, this is useless work since no one can read the result, but we + // can't do it after the call to pthread_remove_thread because it isn't + // safe to dereference t after that. + if ((void*)t > t->freeaddr && (void*)t < t->freeaddr + t->freesize){ + // Check to ensure the pthread structure itself is part of the + // allocation described by freeaddr/freesize, in which case we split and + // only deallocate the area below the pthread structure. In the event of a + // custom stack, the freeaddr/size will be the pthread structure itself, in + // which case we shouldn't free anything (the final else case). + freesize_stack = trunc_page((uintptr_t)t - (uintptr_t)freeaddr); + + // describe just the remainder for deallocation when the pthread_t goes away + t->freeaddr += freesize_stack; + t->freesize -= freesize_stack; + } else if (t == main_thread()) { + freeaddr = t->stackaddr - pthread_get_stacksize_np(t); + uintptr_t stackborder = trunc_page((uintptr_t)_pthread_current_stack_address()); + freesize_stack = stackborder - freeaddr; + } else { + freesize_stack = 0; + } - _pthread_exit(self, result); -} + mach_port_t kport = _pthread_tsd_slot(t, MACH_THREAD_SELF); + bool keep_thread_struct = false, needs_wake = false; + semaphore_t custom_stack_sema = MACH_PORT_NULL; -PTHREAD_NORETURN -void -_pthread_start(pthread_t self, - mach_port_t kport, - void *(*fun)(void *), - void *arg, - size_t stacksize, - unsigned int pflags) -{ - if ((pflags & PTHREAD_START_CUSTOM) == 0) { - void *stackaddr = self; - _pthread_struct_init(self, &_pthread_attr_default, - stackaddr, stacksize, - PTHREAD_ALLOCADDR(stackaddr, stacksize), PTHREAD_ALLOCSIZE(stackaddr, stacksize)); - - if (pflags & PTHREAD_START_SETSCHED) { - self->policy = ((pflags >> PTHREAD_START_POLICY_BITSHIFT) & PTHREAD_START_POLICY_MASK); - self->param.sched_priority = (pflags & PTHREAD_START_IMPORTANCE_MASK); - } + _pthread_dealloc_special_reply_port(t); + _pthread_dealloc_reply_port(t); - if ((pflags & PTHREAD_START_DETACHED) == PTHREAD_START_DETACHED) { - self->detached &= ~PTHREAD_CREATE_JOINABLE; - self->detached |= PTHREAD_CREATE_DETACHED; + _pthread_lock_lock(&_pthread_list_lock); + + // This piece of code interacts with pthread_join. It will always: + // - set tl_exit_gate to MACH_PORT_DEAD (thread exited) + // - set tl_exit_value to the value passed to pthread_exit() + // - decrement _pthread_count, so that we can exit the process when all + // threads exited even if not all of them were joined. + t->tl_exit_gate = MACH_PORT_DEAD; + t->tl_exit_value = exit_value; + should_exit = (--_pthread_count <= 0); + + // If we see a joiner, we prepost that the join has to succeed, + // and the joiner is committed to finish (even if it was canceled) + if (t->tl_join_ctx) { + custom_stack_sema = _pthread_joiner_prepost_wake(t); // unsets tl_joinable + needs_wake = true; + } + + // Joinable threads that have no joiner yet are kept on the thread list + // so that pthread_join() can later discover the thread when it is joined, + // and will have to do the pthread_t cleanup. + if (t->tl_joinable) { + t->tl_joiner_cleans_up = keep_thread_struct = true; + } else { + TAILQ_REMOVE(&__pthread_head, t, tl_plist); + } + + _pthread_lock_unlock(&_pthread_list_lock); + + if (needs_wake) { + // When we found a waiter, we want to drop the very contended list lock + // before we do the syscall in _pthread_joiner_wake(). Then, we decide + // who gets to cleanup the pthread_t between the joiner and the exiting + // thread: + // - the joiner tries to set tl_join_ctx to NULL + // - the exiting thread tries to set tl_joiner_cleans_up to true + // Whoever does it first commits the other guy to cleanup the pthread_t + _pthread_joiner_wake(t); + _pthread_lock_lock(&_pthread_list_lock); + if (t->tl_join_ctx) { + t->tl_joiner_cleans_up = true; + keep_thread_struct = true; } + _pthread_lock_unlock(&_pthread_list_lock); } - if ((pflags & PTHREAD_START_QOSCLASS) != 0) { - /* The QoS class is cached in the TSD of the pthread, so to reflect the - * class that the kernel brought us up at, the TSD must be primed from the - * flags parameter. - */ - self->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = (pflags & PTHREAD_START_QOSCLASS_MASK); + // + // /!\ dereferencing `t` past this point is not safe /!\ + // + + if (keep_thread_struct || t == main_thread()) { + // Use the adjusted freesize of just the stack that we computed above. + freesize = freesize_stack; } else { - /* Give the thread a default QoS tier, of zero. */ - self->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = _pthread_priority_make_newest(QOS_CLASS_UNSPECIFIED, 0, 0); + _pthread_introspection_thread_destroy(t); } - bool thread_tsd_bsd_set = (bool)(pflags & PTHREAD_START_TSD_BASE_SET); + // Check if there is nothing to free because the thread has a custom + // stack allocation and is joinable. + if (freesize == 0) { + freeaddr = 0; + } + if (should_exit) { + exitf(0); + } + __bsdthread_terminate((void *)freeaddr, freesize, kport, custom_stack_sema); + PTHREAD_INTERNAL_CRASH(t, "thread didn't terminate"); +} -#if DEBUG - PTHREAD_ASSERT(MACH_PORT_VALID(kport)); - PTHREAD_ASSERT(_pthread_kernel_thread(self) == kport); +OS_NORETURN +static void +_pthread_terminate_invoke(pthread_t t, void *exit_value) +{ +#if PTHREAD_T_OFFSET + void *p = NULL; + // During pthread termination there is a race + // between pthread_join and pthread_terminate; if the joiner is responsible + // for cleaning up the pthread_t struct, then it may destroy some part of the + // stack with it on 16k OSes. So that this doesn't cause _pthread_terminate() + // to crash because its stack has been removed from under its feet, just make + // sure termination happens in a part of the stack that is not on the same + // page as the pthread_t. + if (trunc_page((uintptr_t)__builtin_frame_address(0)) == + trunc_page((uintptr_t)t)) { + p = alloca(PTHREAD_T_OFFSET); + } + // And this __asm__ volatile is needed to stop the compiler from optimising + // away the alloca() completely. + __asm__ volatile ("" : : "r"(p) ); #endif - // will mark the thread initialized - _pthread_markcancel_if_canceled(self, kport); + _pthread_terminate(t, exit_value); +} - self->fun = fun; - self->arg = arg; +#pragma mark pthread start / body - _pthread_body(self, !thread_tsd_bsd_set); +void +_pthread_start(pthread_t self, mach_port_t kport, + __unused void *(*fun)(void *), __unused void *arg, + __unused size_t stacksize, unsigned int pflags) +{ + if (os_unlikely(pflags & PTHREAD_START_SUSPENDED)) { + PTHREAD_INTERNAL_CRASH(pflags, + "kernel without PTHREAD_START_SUSPENDED support"); + } + if (os_unlikely((pflags & PTHREAD_START_TSD_BASE_SET) == 0)) { + PTHREAD_INTERNAL_CRASH(pflags, + "thread_set_tsd_base() wasn't called by the kernel"); + } + PTHREAD_DEBUG_ASSERT(MACH_PORT_VALID(kport)); + PTHREAD_DEBUG_ASSERT(_pthread_tsd_slot(self, MACH_THREAD_SELF) == kport); + _pthread_validate_signature(self); + _pthread_markcancel_if_canceled(self, kport); + + _pthread_set_self_internal(self); + __pthread_started_thread(self); + _pthread_exit(self, (self->fun)(self->arg)); } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void -_pthread_struct_init(pthread_t t, - const pthread_attr_t *attrs, - void *stackaddr, - size_t stacksize, - void *freeaddr, - size_t freesize) -{ -#if DEBUG - PTHREAD_ASSERT(t->sig != _PTHREAD_SIG); -#endif +_pthread_struct_init(pthread_t t, const pthread_attr_t *attrs, + void *stackaddr, size_t stacksize, void *freeaddr, size_t freesize) +{ + _pthread_init_signature(t); + _pthread_tsd_slot(t, PTHREAD_SELF) = t; + _pthread_tsd_slot(t, ERRNO) = &t->err_no; + if (attrs->schedset == 0) { + _pthread_tsd_slot(t, PTHREAD_QOS_CLASS) = attrs->qosclass; + } else { + _pthread_tsd_slot(t, PTHREAD_QOS_CLASS) = + _pthread_unspecified_priority(); + } + _pthread_tsd_slot(t, PTR_MUNGE) = _pthread_ptr_munge_token; + t->tl_has_custom_stack = (attrs->stackaddr != NULL); - t->sig = _PTHREAD_SIG; - t->tsd[_PTHREAD_TSD_SLOT_PTHREAD_SELF] = t; - t->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = _pthread_priority_make_newest(QOS_CLASS_UNSPECIFIED, 0, 0); - _PTHREAD_LOCK_INIT(t->lock); + _pthread_lock_init(&t->lock); t->stackaddr = stackaddr; - t->stacksize = stacksize; + t->stackbottom = stackaddr - stacksize; t->freeaddr = freeaddr; t->freesize = freesize; - t->guardsize = attrs->guardsize; - t->detached = attrs->detached; + t->guardsize = _pthread_attr_guardsize(attrs); + t->tl_joinable = (attrs->detached == PTHREAD_CREATE_JOINABLE); t->inherit = attrs->inherit; - t->policy = attrs->policy; + t->tl_policy = attrs->policy; t->schedset = attrs->schedset; - t->param = attrs->param; + _pthread_attr_get_schedparam(attrs, &t->tl_param); t->cancel_state = PTHREAD_CANCEL_ENABLE | PTHREAD_CANCEL_DEFERRED; } -/* Need to deprecate this in future */ -int -_pthread_is_threaded(void) -{ - return __is_threaded; -} +#pragma mark pthread public interface /* Non portable public api to know whether this process has(had) atleast one thread * apart from main thread. There could be race if there is a thread in the process of @@ -812,37 +922,33 @@ pthread_is_threaded_np(void) return __is_threaded; } - -PTHREAD_NOEXPORT_VARIANT mach_port_t pthread_mach_thread_np(pthread_t t) { mach_port_t kport = MACH_PORT_NULL; - (void)_pthread_is_valid(t, 0, &kport); + (void)_pthread_is_valid(t, &kport); return kport; } -PTHREAD_NOEXPORT_VARIANT pthread_t pthread_from_mach_thread_np(mach_port_t kernel_thread) { - struct _pthread *p = NULL; + pthread_t p = NULL; /* No need to wait as mach port is already known */ - _PTHREAD_LOCK(_pthread_list_lock); + _pthread_lock_lock(&_pthread_list_lock); - TAILQ_FOREACH(p, &__pthread_head, plist) { - if (_pthread_kernel_thread(p) == kernel_thread) { + TAILQ_FOREACH(p, &__pthread_head, tl_plist) { + if (_pthread_tsd_slot(p, MACH_THREAD_SELF) == kernel_thread) { break; } } - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); return p; } -PTHREAD_NOEXPORT_VARIANT size_t pthread_get_stacksize_np(pthread_t t) { @@ -852,7 +958,7 @@ pthread_get_stacksize_np(pthread_t t) return ESRCH; // XXX bug? } -#if !defined(__arm__) && !defined(__arm64__) +#if TARGET_OS_OSX // The default rlimit based allocations will be provided with a stacksize // of the current limit and a freesize of the max. However, custom // allocations will just have the guard page to free. If we aren't in the @@ -863,149 +969,105 @@ pthread_get_stacksize_np(pthread_t t) // // Of course, on arm rlim_cur == rlim_max and there's only the one guard // page. So, we can skip all this there. - if (t == &_thread && t->stacksize + vm_page_size != t->freesize) { - // We want to call getrlimit() just once, as it's relatively expensive - static size_t rlimit_stack; + if (t == main_thread()) { + size_t stacksize = t->stackaddr - t->stackbottom; + + if (stacksize + vm_page_size != t->freesize) { + // We want to call getrlimit() just once, as it's relatively + // expensive + static size_t rlimit_stack; - if (rlimit_stack == 0) { - struct rlimit limit; - int ret = getrlimit(RLIMIT_STACK, &limit); + if (rlimit_stack == 0) { + struct rlimit limit; + int ret = getrlimit(RLIMIT_STACK, &limit); - if (ret == 0) { - rlimit_stack = (size_t) limit.rlim_cur; + if (ret == 0) { + rlimit_stack = (size_t) limit.rlim_cur; + } } - } - if (rlimit_stack == 0 || rlimit_stack > t->freesize) { - return t->stacksize; - } else { - return rlimit_stack; + if (rlimit_stack == 0 || rlimit_stack > t->freesize) { + return stacksize; + } else { + return round_page(rlimit_stack); + } } } -#endif /* !defined(__arm__) && !defined(__arm64__) */ +#endif /* TARGET_OS_OSX */ - if (t == pthread_self() || t == &_thread) { - return t->stacksize; + if (t == pthread_self() || t == main_thread()) { + size = t->stackaddr - t->stackbottom;; + goto out; } - _PTHREAD_LOCK(_pthread_list_lock); - - if (_pthread_is_valid_locked(t)) { - size = t->stacksize; - } else { - size = ESRCH; // XXX bug? + if (_pthread_validate_thread_and_list_lock(t)) { + size = t->stackaddr - t->stackbottom;; + _pthread_lock_unlock(&_pthread_list_lock); } - _PTHREAD_UNLOCK(_pthread_list_lock); - - return size; +out: + // binary compatibility issues force us to return + // DEFAULT_STACK_SIZE here when we do not know the size of the stack + return size ? size : DEFAULT_STACK_SIZE; } -PTHREAD_NOEXPORT_VARIANT void * pthread_get_stackaddr_np(pthread_t t) { - void *addr = NULL; - - if (t == NULL) { - return (void *)(uintptr_t)ESRCH; // XXX bug? - } - // since the main thread will not get de-allocated from underneath us - if (t == pthread_self() || t == &_thread) { + if (t == pthread_self() || t == main_thread()) { return t->stackaddr; } - _PTHREAD_LOCK(_pthread_list_lock); - - if (_pthread_is_valid_locked(t)) { - addr = t->stackaddr; - } else { - addr = (void *)(uintptr_t)ESRCH; // XXX bug? + if (!_pthread_validate_thread_and_list_lock(t)) { + return (void *)(uintptr_t)ESRCH; // XXX bug? } - _PTHREAD_UNLOCK(_pthread_list_lock); - + void *addr = t->stackaddr; + _pthread_lock_unlock(&_pthread_list_lock); return addr; } - -static mach_port_t -_pthread_reply_port(pthread_t t) -{ - void *p; - if (t == NULL) { - p = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_MIG_REPLY); - } else { - p = t->tsd[_PTHREAD_TSD_SLOT_MIG_REPLY]; - } - return (mach_port_t)(uintptr_t)p; -} - -static void -_pthread_set_reply_port(pthread_t t, mach_port_t reply_port) -{ - void *p = (void *)(uintptr_t)reply_port; - if (t == NULL) { - _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_MIG_REPLY, p); - } else { - t->tsd[_PTHREAD_TSD_SLOT_MIG_REPLY] = p; - } -} - -static void -_pthread_dealloc_reply_port(pthread_t t) -{ - mach_port_t reply_port = _pthread_reply_port(t); - if (reply_port != MACH_PORT_NULL) { - mig_dealloc_reply_port(reply_port); - } -} - -static mach_port_t -_pthread_special_reply_port(pthread_t t) -{ - void *p; - if (t == NULL) { - p = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_MACH_SPECIAL_REPLY); - } else { - p = t->tsd[_PTHREAD_TSD_SLOT_MACH_SPECIAL_REPLY]; - } - return (mach_port_t)(uintptr_t)p; -} - -static void -_pthread_dealloc_special_reply_port(pthread_t t) -{ - mach_port_t special_reply_port = _pthread_special_reply_port(t); - if (special_reply_port != MACH_PORT_NULL) { - mach_port_mod_refs(mach_task_self(), special_reply_port, - MACH_PORT_RIGHT_RECEIVE, -1); - } -} - pthread_t pthread_main_thread_np(void) { - return &_thread; + return main_thread(); } /* returns non-zero if the current thread is the main thread */ int pthread_main_np(void) { - pthread_t self = pthread_self(); - - return ((self->detached & _PTHREAD_CREATE_PARENT) == _PTHREAD_CREATE_PARENT); + return pthread_self() == main_thread(); } +static int +_pthread_threadid_slow(pthread_t thread, uint64_t *thread_id) +{ + unsigned int info_count = THREAD_IDENTIFIER_INFO_COUNT; + mach_port_t thport = _pthread_tsd_slot(thread, MACH_THREAD_SELF); + struct thread_identifier_info info; + kern_return_t kr; + + kr = thread_info(thport, THREAD_IDENTIFIER_INFO, + (thread_info_t)&info, &info_count); + if (kr == KERN_SUCCESS && info.thread_id) { + *thread_id = info.thread_id; +#if __LP64__ + os_atomic_store(&thread->thread_id, info.thread_id, relaxed); +#else + os_atomic_store_wide(&thread->thread_id, info.thread_id, relaxed); +#endif + return 0; + } + return EINVAL; +} -/* if we are passed in a pthread_t that is NULL, then we return - the current thread's thread_id. So folks don't have to call - pthread_self, in addition to us doing it, if they just want - their thread_id. -*/ -PTHREAD_NOEXPORT_VARIANT +/* + * if we are passed in a pthread_t that is NULL, then we return the current + * thread's thread_id. So folks don't have to call pthread_self, in addition to + * us doing it, if they just want their thread_id. + */ int pthread_threadid_np(pthread_t thread, uint64_t *thread_id) { @@ -1018,41 +1080,53 @@ pthread_threadid_np(pthread_t thread, uint64_t *thread_id) if (thread == NULL || thread == self) { *thread_id = self->thread_id; + } else if (!_pthread_validate_thread_and_list_lock(thread)) { + res = ESRCH; } else { - _PTHREAD_LOCK(_pthread_list_lock); - if (!_pthread_is_valid_locked(thread)) { - res = ESRCH; - } else if (thread->thread_id == 0) { - res = EINVAL; - } else { - *thread_id = thread->thread_id; +#if __LP64__ + *thread_id = os_atomic_load(&thread->thread_id, relaxed); +#else + *thread_id = os_atomic_load_wide(&thread->thread_id, relaxed); +#endif + if (os_unlikely(*thread_id == 0)) { + // there is a race at init because the thread sets its own TID. + // correct this by asking mach + res = _pthread_threadid_slow(thread, thread_id); } - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); } return res; } -PTHREAD_NOEXPORT_VARIANT +int +pthread_cpu_number_np(size_t *cpu_id) +{ + if (cpu_id == NULL) { + errno = EINVAL; + return errno; + } + + *cpu_id = _os_cpu_number(); + return 0; +} + int pthread_getname_np(pthread_t thread, char *threadname, size_t len) { - int res = 0; + if (thread == pthread_self()) { + strlcpy(threadname, thread->pthread_name, len); + return 0; + } - if (thread == NULL) { + if (!_pthread_validate_thread_and_list_lock(thread)) { return ESRCH; } - _PTHREAD_LOCK(_pthread_list_lock); - if (_pthread_is_valid_locked(thread)) { - strlcpy(threadname, thread->pthread_name, len); - } else { - res = ESRCH; - } - _PTHREAD_UNLOCK(_pthread_list_lock); - return res; + strlcpy(threadname, thread->pthread_name, len); + _pthread_lock_unlock(&_pthread_list_lock); + return 0; } - int pthread_setname_np(const char *name) { @@ -1064,7 +1138,8 @@ pthread_setname_np(const char *name) len = strlen(name); } - /* protytype is in pthread_internals.h */ + _pthread_validate_signature(self); + res = __proc_info(5, getpid(), 2, (uint64_t)0, (void*)name, (int)len); if (res == 0) { if (len > 0) { @@ -1077,339 +1152,193 @@ pthread_setname_np(const char *name) } -PTHREAD_ALWAYS_INLINE -static inline void -__pthread_add_thread(pthread_t t, const pthread_attr_t *attrs, - bool parent, bool from_mach_thread) +void +pthread_jit_write_protect_np(int enable) { - bool should_deallocate = false; - bool should_add = true; - - mach_port_t kport = _pthread_kernel_thread(t); - if (os_slowpath(!MACH_PORT_VALID(kport))) { - PTHREAD_CLIENT_CRASH(kport, - "Unable to allocate thread port, possible port leak"); - } + if (!os_thread_self_restrict_rwx_is_supported()) { + return; + } - if (from_mach_thread) { - _PTHREAD_LOCK_FROM_MACH_THREAD(_pthread_list_lock); - } else { - _PTHREAD_LOCK(_pthread_list_lock); - } - - // The parent and child threads race to add the thread to the list. - // When called by the parent: - // - set parentcheck to true - // - back off if childrun is true - // When called by the child: - // - set childrun to true - // - back off if parentcheck is true - if (parent) { - t->parentcheck = 1; - if (t->childrun) { - // child got here first, don't add. - should_add = false; - } - - // If the child exits before we check in then it has to keep - // the thread structure memory alive so our dereferences above - // are valid. If it's a detached thread, then no joiner will - // deallocate the thread structure itself. So we do it here. - if (t->childexit) { - should_add = false; - should_deallocate = ((t->detached & PTHREAD_CREATE_DETACHED) == PTHREAD_CREATE_DETACHED); - } - } else { - t->childrun = 1; - if (t->parentcheck) { - // Parent got here first, don't add. - should_add = false; - } - if (t->wqthread) { - // Work queue threads have no parent. Simulate. - t->parentcheck = 1; - } - } - - if (should_add) { - TAILQ_INSERT_TAIL(&__pthread_head, t, plist); - _pthread_count++; + if (enable) { + os_thread_self_restrict_rwx_to_rx(); + } else { + os_thread_self_restrict_rwx_to_rw(); + } +} - /* - * Set some initial values which we know in the pthread structure in - * case folks try to get the values before the thread can set them. - */ - if (parent && attrs && attrs->schedset == 0) { - t->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = attrs->qosclass; - } - } +int pthread_jit_write_protect_supported_np() +{ + return os_thread_self_restrict_rwx_is_supported(); +} - if (from_mach_thread){ - _PTHREAD_UNLOCK_FROM_MACH_THREAD(_pthread_list_lock); - } else { - _PTHREAD_UNLOCK(_pthread_list_lock); - } +OS_ALWAYS_INLINE +static inline void +__pthread_add_thread(pthread_t t, mach_port_t self, bool from_mach_thread) +{ + _pthread_lock_lock(&_pthread_list_lock, self); + TAILQ_INSERT_TAIL(&__pthread_head, t, tl_plist); + _pthread_count++; + _pthread_lock_unlock(&_pthread_list_lock, self); - if (parent) { - if (!from_mach_thread) { - // PR-26275485: Mach threads will likely crash trying to run - // introspection code. Since the fall out from the introspection - // code not seeing the injected thread is likely less than crashing - // in the introspection code, just don't make the call. - _pthread_introspection_thread_create(t, should_deallocate); - } - if (should_deallocate) { - _pthread_deallocate(t); - } - } else { - _pthread_introspection_thread_start(t); + if (!from_mach_thread) { + // PR-26275485: Mach threads will likely crash trying to run + // introspection code. Since the fall out from the introspection + // code not seeing the injected thread is likely less than crashing + // in the introspection code, just don't make the call. + _pthread_introspection_thread_create(t); } } -// must always inline this function to avoid epilogues -// Returns EBUSY if the thread structure should be kept alive (is joinable). -// Returns ESRCH if the thread structure is no longer valid (was detached). -PTHREAD_ALWAYS_INLINE -static inline int -__pthread_remove_thread(pthread_t t, bool child, bool *should_exit) +OS_ALWAYS_INLINE +static inline void +__pthread_undo_add_thread(pthread_t t, mach_port_t self) { - int ret = 0; - - bool should_remove = true; - - _PTHREAD_LOCK(_pthread_list_lock); - - // When a thread removes itself: - // - Set the childexit flag indicating that the thread has exited. - // - Return false if parentcheck is zero (must keep structure) - // - If the thread is joinable, keep it on the list so that - // the join operation succeeds. Still decrement the running - // thread count so that we exit if no threads are running. - // - Update the running thread count. - // When another thread removes a joinable thread: - // - CAREFUL not to dereference the thread before verifying that the - // reference is still valid using _pthread_is_valid_locked(). - // - Remove the thread from the list. - - if (child) { - t->childexit = 1; - if (t->parentcheck == 0) { - ret = EBUSY; - } - if ((t->detached & PTHREAD_CREATE_JOINABLE) != 0) { - ret = EBUSY; - should_remove = false; - } - *should_exit = (--_pthread_count <= 0); - } else if (!_pthread_is_valid_locked(t)) { - ret = ESRCH; - should_remove = false; - } else if ((t->detached & PTHREAD_CREATE_JOINABLE) == 0) { - // If we found a thread but it's not joinable, bail. - ret = ESRCH; - should_remove = false; - } else if (t->parentcheck == 0) { - // If we're not the child thread *and* the parent has not finished - // creating the thread yet, then we are another thread that's joining - // and we cannot deallocate the pthread. - ret = EBUSY; - } - if (should_remove) { - TAILQ_REMOVE(&__pthread_head, t, plist); - } - - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_lock(&_pthread_list_lock, self); + TAILQ_REMOVE(&__pthread_head, t, tl_plist); + _pthread_count--; + _pthread_lock_unlock(&_pthread_list_lock, self); +} - return ret; +OS_ALWAYS_INLINE +static inline void +__pthread_started_thread(pthread_t t) +{ + mach_port_t kport = _pthread_tsd_slot(t, MACH_THREAD_SELF); + if (os_unlikely(!MACH_PORT_VALID(kport))) { + PTHREAD_CLIENT_CRASH(kport, + "Unable to allocate thread port, possible port leak"); + } + _pthread_introspection_thread_start(t); } +#define _PTHREAD_CREATE_NONE 0x0 +#define _PTHREAD_CREATE_FROM_MACH_THREAD 0x1 +#define _PTHREAD_CREATE_SUSPENDED 0x2 + static int -_pthread_create(pthread_t *thread, - const pthread_attr_t *attr, - void *(*start_routine)(void *), - void *arg, - bool from_mach_thread) +_pthread_create(pthread_t *thread, const pthread_attr_t *attrs, + void *(*start_routine)(void *), void *arg, unsigned int create_flags) { pthread_t t = NULL; - unsigned int flags = 0; + void *stack = NULL; + bool from_mach_thread = (create_flags & _PTHREAD_CREATE_FROM_MACH_THREAD); + mach_port_t self_kport; + int rc = 0; - pthread_attr_t *attrs = (pthread_attr_t *)attr; if (attrs == NULL) { attrs = &_pthread_attr_default; } else if (attrs->sig != _PTHREAD_ATTR_SIG) { return EINVAL; } - if (attrs->detached == PTHREAD_CREATE_DETACHED) { - flags |= PTHREAD_START_DETACHED; - } - + unsigned int flags = PTHREAD_START_CUSTOM; if (attrs->schedset != 0) { + struct sched_param p; + _pthread_attr_get_schedparam(attrs, &p); flags |= PTHREAD_START_SETSCHED; flags |= ((attrs->policy & PTHREAD_START_POLICY_MASK) << PTHREAD_START_POLICY_BITSHIFT); - flags |= (attrs->param.sched_priority & PTHREAD_START_IMPORTANCE_MASK); + flags |= (p.sched_priority & PTHREAD_START_IMPORTANCE_MASK); } else if (attrs->qosclass != 0) { flags |= PTHREAD_START_QOSCLASS; flags |= (attrs->qosclass & PTHREAD_START_QOSCLASS_MASK); } + if (create_flags & _PTHREAD_CREATE_SUSPENDED) { + flags |= PTHREAD_START_SUSPENDED; + } __is_threaded = 1; - void *stack; + t = _pthread_allocate(attrs, &stack, from_mach_thread); + if (t == NULL) { + return EAGAIN; + } - if (attrs->fastpath) { - // kernel will allocate thread and stack, pass stacksize. - stack = (void *)attrs->stacksize; + if (os_unlikely(from_mach_thread)) { + self_kport = mach_thread_self(); } else { - // allocate the thread and its stack - flags |= PTHREAD_START_CUSTOM; - - int res; - res = _pthread_allocate(&t, attrs, &stack); - if (res) { - return res; - } - - t->arg = arg; - t->fun = start_routine; + self_kport = _pthread_mach_thread_self_direct(); } - pthread_t t2; - t2 = __bsdthread_create(start_routine, arg, stack, t, flags); - if (t2 == (pthread_t)-1) { + t->arg = arg; + t->fun = start_routine; + __pthread_add_thread(t, self_kport, from_mach_thread); + + if (__bsdthread_create(start_routine, arg, stack, t, flags) == + (pthread_t)-1) { if (errno == EMFILE) { PTHREAD_CLIENT_CRASH(0, "Unable to allocate thread port, possible port leak"); } - if (flags & PTHREAD_START_CUSTOM) { - // free the thread and stack if we allocated it - _pthread_deallocate(t); - } - return EAGAIN; + __pthread_undo_add_thread(t, self_kport); + _pthread_deallocate(t, from_mach_thread); + t = NULL; + rc = EAGAIN; } - if (t == NULL) { - t = t2; + if (from_mach_thread) { + mach_port_deallocate(mach_task_self(), self_kport); } - __pthread_add_thread(t, attrs, true, from_mach_thread); - // n.b. if a thread is created detached and exits, t will be invalid *thread = t; - return 0; + return rc; } int -pthread_create(pthread_t *thread, - const pthread_attr_t *attr, - void *(*start_routine)(void *), - void *arg) +pthread_create(pthread_t *thread, const pthread_attr_t *attr, + void *(*start_routine)(void *), void *arg) { - return _pthread_create(thread, attr, start_routine, arg, false); + unsigned int flags = _PTHREAD_CREATE_NONE; + return _pthread_create(thread, attr, start_routine, arg, flags); } int -pthread_create_from_mach_thread(pthread_t *thread, - const pthread_attr_t *attr, - void *(*start_routine)(void *), - void *arg) -{ - return _pthread_create(thread, attr, start_routine, arg, true); -} - -PTHREAD_NORETURN -static void -_pthread_suspended_body(pthread_t self) +pthread_create_from_mach_thread(pthread_t *thread, const pthread_attr_t *attr, + void *(*start_routine)(void *), void *arg) { - _pthread_set_self(self); - __pthread_add_thread(self, NULL, false, false); - _pthread_exit(self, (self->fun)(self->arg)); + unsigned int flags = _PTHREAD_CREATE_FROM_MACH_THREAD; + return _pthread_create(thread, attr, start_routine, arg, flags); } int -pthread_create_suspended_np(pthread_t *thread, - const pthread_attr_t *attr, - void *(*start_routine)(void *), - void *arg) +pthread_create_suspended_np(pthread_t *thread, const pthread_attr_t *attr, + void *(*start_routine)(void *), void *arg) { - int res; - void *stack; - mach_port_t kernel_thread = MACH_PORT_NULL; - - const pthread_attr_t *attrs = attr; - if (attrs == NULL) { - attrs = &_pthread_attr_default; - } else if (attrs->sig != _PTHREAD_ATTR_SIG) { - return EINVAL; - } - - pthread_t t; - res = _pthread_allocate(&t, attrs, &stack); - if (res) { - return res; - } - - *thread = t; - - kern_return_t kr; - kr = thread_create(mach_task_self(), &kernel_thread); - if (kr != KERN_SUCCESS) { - //PTHREAD_ABORT("thread_create() failed: %d", kern_res); - return EINVAL; /* Need better error here? */ - } - - _pthread_set_kernel_thread(t, kernel_thread); - (void)pthread_setschedparam_internal(t, kernel_thread, t->policy, &t->param); - - __is_threaded = 1; - - t->arg = arg; - t->fun = start_routine; - - t->cancel_state |= _PTHREAD_CANCEL_INITIALIZED; - __pthread_add_thread(t, NULL, true, false); - - // Set up a suspended thread. - _pthread_setup(t, _pthread_suspended_body, stack, 1, 0); - return res; + unsigned int flags = _PTHREAD_CREATE_SUSPENDED; + return _pthread_create(thread, attr, start_routine, arg, flags); } - -PTHREAD_NOEXPORT_VARIANT int pthread_detach(pthread_t thread) { int res = 0; - bool join = false; - semaphore_t sema = SEMAPHORE_NULL; + bool join = false, wake = false; - if (!_pthread_is_valid(thread, PTHREAD_IS_VALID_LOCK_THREAD, NULL)) { - return ESRCH; // Not a valid thread to detach. + if (!_pthread_validate_thread_and_list_lock(thread)) { + return ESRCH; } - if ((thread->detached & PTHREAD_CREATE_DETACHED) || - !(thread->detached & PTHREAD_CREATE_JOINABLE)) { + if (!thread->tl_joinable) { res = EINVAL; - } else if (thread->detached & _PTHREAD_EXITED) { + } else if (thread->tl_exit_gate == MACH_PORT_DEAD) { // Join the thread if it's already exited. join = true; } else { - thread->detached &= ~PTHREAD_CREATE_JOINABLE; - thread->detached |= PTHREAD_CREATE_DETACHED; - sema = thread->joiner_notify; + thread->tl_joinable = false; // _pthread_joiner_prepost_wake uses this + if (thread->tl_join_ctx) { + (void)_pthread_joiner_prepost_wake(thread); + wake = true; + } } - - _PTHREAD_UNLOCK(thread->lock); + _pthread_lock_unlock(&_pthread_list_lock); if (join) { pthread_join(thread, NULL); - } else if (sema) { - semaphore_signal(sema); + } else if (wake) { + _pthread_joiner_wake(thread); } - return res; } -PTHREAD_NOEXPORT_VARIANT int pthread_kill(pthread_t th, int sig) { @@ -1418,13 +1347,10 @@ pthread_kill(pthread_t th, int sig) } mach_port_t kport = MACH_PORT_NULL; - if (!_pthread_is_valid(th, 0, &kport)) { - return ESRCH; // Not a valid thread. - } - - // Don't signal workqueue threads. - if (th->wqthread != 0 && th->wqkillset == 0) { - return ENOTSUP; + { + if (!_pthread_is_valid(th, &kport)) { + return ESRCH; + } } int ret = __pthread_kill(kport, sig); @@ -1435,35 +1361,21 @@ pthread_kill(pthread_t th, int sig) return ret; } -PTHREAD_NOEXPORT_VARIANT int __pthread_workqueue_setkill(int enable) { - pthread_t self = pthread_self(); - - _PTHREAD_LOCK(self->lock); - self->wqkillset = enable ? 1 : 0; - _PTHREAD_UNLOCK(self->lock); - - return 0; -} - - -/* For compatibility... */ - -pthread_t -_pthread_self(void) { - return pthread_self(); + { + return __bsdthread_ctl(BSDTHREAD_CTL_WORKQ_ALLOW_KILL, enable, 0, 0); + } } /* * Terminate a thread. */ -int __disable_threadsignal(int); -PTHREAD_NORETURN +OS_NORETURN static void -_pthread_exit(pthread_t self, void *value_ptr) +_pthread_exit(pthread_t self, void *exit_value) { struct __darwin_pthread_handler_rec *handler; @@ -1471,7 +1383,7 @@ _pthread_exit(pthread_t self, void *value_ptr) __disable_threadsignal(1); // Set cancel state to disable and type to deferred - _pthread_setcancelstate_exit(self, value_ptr, __unix_conforming); + _pthread_setcancelstate_exit(self, exit_value); while ((handler = self->__cleanup_stack) != 0) { (handler->__routine)(handler->__arg); @@ -1479,139 +1391,115 @@ _pthread_exit(pthread_t self, void *value_ptr) } _pthread_tsd_cleanup(self); - _PTHREAD_LOCK(self->lock); - self->detached |= _PTHREAD_EXITED; - self->exit_value = value_ptr; - - if ((self->detached & PTHREAD_CREATE_JOINABLE) && - self->joiner_notify == SEMAPHORE_NULL) { - self->joiner_notify = (semaphore_t)os_get_cached_semaphore(); - } - _PTHREAD_UNLOCK(self->lock); - // Clear per-thread semaphore cache os_put_cached_semaphore(SEMAPHORE_NULL); - _pthread_terminate_invoke(self); + _pthread_terminate_invoke(self, exit_value); } void -pthread_exit(void *value_ptr) +pthread_exit(void *exit_value) { pthread_t self = pthread_self(); - if (self->wqthread == 0) { - _pthread_exit(self, value_ptr); - } else { - PTHREAD_ABORT("pthread_exit() may only be called against threads created via pthread_create()"); + if (os_unlikely(self->wqthread)) { + PTHREAD_CLIENT_CRASH(0, "pthread_exit() called from a thread " + "not created by pthread_create()"); } + _pthread_validate_signature(self); + _pthread_exit(self, exit_value); } - -PTHREAD_NOEXPORT_VARIANT int -pthread_getschedparam(pthread_t thread, - int *policy, - struct sched_param *param) +pthread_self_is_exiting_np(void) { - int ret = 0; + return (os_atomic_load(&pthread_self()->cancel_state, relaxed) & + _PTHREAD_CANCEL_EXITING) != 0; +} - if (thread == NULL) { +int +pthread_getschedparam(pthread_t thread, int *policy, struct sched_param *param) +{ + if (!_pthread_validate_thread_and_list_lock(thread)) { return ESRCH; } - _PTHREAD_LOCK(_pthread_list_lock); - - if (_pthread_is_valid_locked(thread)) { - if (policy) { - *policy = thread->policy; - } - if (param) { - *param = thread->param; - } - } else { - ret = ESRCH; - } - - _PTHREAD_UNLOCK(_pthread_list_lock); - - return ret; + if (policy) *policy = thread->tl_policy; + if (param) *param = thread->tl_param; + _pthread_lock_unlock(&_pthread_list_lock); + return 0; } - -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int -pthread_setschedparam_internal(pthread_t thread, - mach_port_t kport, - int policy, - const struct sched_param *param) +pthread_setschedparam_internal(pthread_t thread, mach_port_t kport, int policy, + const struct sched_param *param) { policy_base_data_t bases; policy_base_t base; mach_msg_type_number_t count; kern_return_t ret; + if (os_unlikely(thread->wqthread)) { + return ENOTSUP; + } + switch (policy) { - case SCHED_OTHER: - bases.ts.base_priority = param->sched_priority; - base = (policy_base_t)&bases.ts; - count = POLICY_TIMESHARE_BASE_COUNT; - break; - case SCHED_FIFO: - bases.fifo.base_priority = param->sched_priority; - base = (policy_base_t)&bases.fifo; - count = POLICY_FIFO_BASE_COUNT; - break; - case SCHED_RR: - bases.rr.base_priority = param->sched_priority; - /* quantum isn't public yet */ - bases.rr.quantum = param->quantum; - base = (policy_base_t)&bases.rr; - count = POLICY_RR_BASE_COUNT; - break; - default: - return EINVAL; + case SCHED_OTHER: + bases.ts.base_priority = param->sched_priority; + base = (policy_base_t)&bases.ts; + count = POLICY_TIMESHARE_BASE_COUNT; + break; + case SCHED_FIFO: + bases.fifo.base_priority = param->sched_priority; + base = (policy_base_t)&bases.fifo; + count = POLICY_FIFO_BASE_COUNT; + break; + case SCHED_RR: + bases.rr.base_priority = param->sched_priority; + /* quantum isn't public yet */ + bases.rr.quantum = param->quantum; + base = (policy_base_t)&bases.rr; + count = POLICY_RR_BASE_COUNT; + break; + default: + return EINVAL; } ret = thread_policy(kport, policy, base, count, TRUE); return (ret != KERN_SUCCESS) ? EINVAL : 0; } - -PTHREAD_NOEXPORT_VARIANT int pthread_setschedparam(pthread_t t, int policy, const struct sched_param *param) { mach_port_t kport = MACH_PORT_NULL; - int res; int bypass = 1; // since the main thread will not get de-allocated from underneath us - if (t == pthread_self() || t == &_thread) { - kport = _pthread_kernel_thread(t); + if (t == pthread_self() || t == main_thread()) { + _pthread_validate_signature(t); + kport = _pthread_tsd_slot(t, MACH_THREAD_SELF); } else { bypass = 0; - (void)_pthread_is_valid(t, 0, &kport); + if (!_pthread_is_valid(t, &kport)) { + return ESRCH; + } } - res = pthread_setschedparam_internal(t, kport, policy, param); - if (res == 0) { - if (bypass == 0) { - // Ensure the thread is still valid. - _PTHREAD_LOCK(_pthread_list_lock); - if (_pthread_is_valid_locked(t)) { - t->policy = policy; - t->param = *param; - } else { - res = ESRCH; - } - _PTHREAD_UNLOCK(_pthread_list_lock); - } else { - t->policy = policy; - t->param = *param; - } + int res = pthread_setschedparam_internal(t, kport, policy, param); + if (res) return res; + + if (bypass) { + _pthread_lock_lock(&_pthread_list_lock); + } else if (!_pthread_validate_thread_and_list_lock(t)) { + // Ensure the thread is still valid. + return ESRCH; } - return res; -} + t->tl_policy = policy; + t->tl_param = *param; + _pthread_lock_unlock(&_pthread_list_lock); + return 0; +} int sched_get_priority_min(int policy) @@ -1631,42 +1519,62 @@ pthread_equal(pthread_t t1, pthread_t t2) return (t1 == t2); } -/* - * Force LLVM not to optimise this to a call to __pthread_set_self, if it does - * then _pthread_set_self won't be bound when secondary threads try and start up. - */ -PTHREAD_NOINLINE +OS_NOINLINE void _pthread_set_self(pthread_t p) { - return _pthread_set_self_internal(p, true); +#if VARIANT_DYLD + if (os_likely(!p)) { + return _pthread_set_self_dyld(); + } +#endif // VARIANT_DYLD + _pthread_set_self_internal(p); + _thread_set_tsd_base(&p->tsd[0]); } -PTHREAD_ALWAYS_INLINE -static inline void -_pthread_set_self_internal(pthread_t p, bool needs_tsd_base_set) +#if VARIANT_DYLD +// _pthread_set_self_dyld is noinline+noexport to allow the option for +// static libsyscall to adopt this as the entry point from mach_init if +// desired +OS_NOINLINE +static void +_pthread_set_self_dyld(void) { - if (p == NULL) { - p = &_thread; - } + pthread_t p = main_thread(); + p->thread_id = __thread_selfid(); - uint64_t tid = __thread_selfid(); - if (tid == -1ull) { - PTHREAD_ABORT("failed to set thread_id"); + if (os_unlikely(p->thread_id == -1ull)) { + PTHREAD_INTERNAL_CRASH(0, "failed to set thread_id"); } - p->tsd[_PTHREAD_TSD_SLOT_PTHREAD_SELF] = p; - p->tsd[_PTHREAD_TSD_SLOT_ERRNO] = &p->err_no; - p->thread_id = tid; + // pthread self and the errno address are the + // bare minimium TSD setup that dyld needs to actually function. Without + // this, TSD access will fail and crash if it uses bits of Libc prior to + // library initialization. __pthread_init will finish the initialization + // during library init. + _pthread_tsd_slot(p, PTHREAD_SELF) = p; + _pthread_tsd_slot(p, ERRNO) = &p->err_no; + _thread_set_tsd_base(&p->tsd[0]); +} +#endif // VARIANT_DYLD + +OS_ALWAYS_INLINE +static inline void +_pthread_set_self_internal(pthread_t p) +{ +#if __LP64__ + os_atomic_store(&p->thread_id, __thread_selfid(), relaxed); +#else + os_atomic_store_wide(&p->thread_id, __thread_selfid(), relaxed); +#endif - if (needs_tsd_base_set) { - _thread_set_tsd_base(&p->tsd[0]); + if (os_unlikely(p->thread_id == -1ull)) { + PTHREAD_INTERNAL_CRASH(0, "failed to set thread_id"); } } - // pthread_once should have an acquire barrier -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void _os_once_acquire(os_once_t *predicate, void *context, os_function_t function) { @@ -1691,7 +1599,6 @@ __pthread_once_handler(void *context) ctx->pthread_once->sig = _PTHREAD_ONCE_SIG; } -PTHREAD_NOEXPORT_VARIANT int pthread_once(pthread_once_t *once_control, void (*init_routine)(void)) { @@ -1702,7 +1609,6 @@ pthread_once(pthread_once_t *once_control, void (*init_routine)(void)) return 0; } - int pthread_getconcurrency(void) { @@ -1719,6 +1625,11 @@ pthread_setconcurrency(int new_level) return 0; } +/* + * Perform package initialization - called automatically when application starts + */ + +#if !VARIANT_DYLD static unsigned long _pthread_strtoul(const char *p, const char **endptr, int base) { @@ -1777,36 +1688,50 @@ out: return ret; } -#if !defined(VARIANT_STATIC) -void * -malloc(size_t sz) +static void +parse_ptr_munge_params(const char *envp[], const char *apple[]) { - if (_pthread_malloc) { - return _pthread_malloc(sz); - } else { - return NULL; + const char *p, *s; + uintptr_t token = 0; + p = _simple_getenv(apple, "ptr_munge"); + if (p) { + token = _pthread_strtoul(p, &s, 16); + bzero((char *)p, strlen(p)); + } + /* + * In DEBUG we allow the environment variable to override the kernel + * security setting, including setting it to 0 which is helpful during + * debugging sessions. + * + * For other cases, the token must be set by the kernel or the environment + * variable to a non 0 value. + */ +#if !DEBUG + if (!token) { +#endif + p = _simple_getenv(envp, "PTHREAD_PTR_MUNGE_TOKEN"); + if (p) { + uintptr_t t = _pthread_strtoul(p, &s, 16); + if (t) token = t; + } +#if !DEBUG } -} -void -free(void *p) -{ - if (_pthread_free) { - _pthread_free(p); + if (!token) { + PTHREAD_INTERNAL_CRASH(token, "Token from the kernel is 0"); } -} -#endif // VARIANT_STATIC +#endif // !DEBUG -/* - * Perform package initialization - called automatically when application starts - */ -struct ProgramVars; /* forward reference */ + _pthread_ptr_munge_token = token; + // we need to refresh the main thread signature now that we changed + // the munge token. We need to do it while TSAN will not look at it + _pthread_init_signature(_main_thread_ptr); +} int __pthread_init(const struct _libpthread_functions *pthread_funcs, - const char *envp[] __unused, - const char *apple[], - const struct ProgramVars *vars __unused) + const char *envp[], const char *apple[], + const struct ProgramVars *vars __unused) { // Save our provided pushed-down functions if (pthread_funcs) { @@ -1818,6 +1743,17 @@ __pthread_init(const struct _libpthread_functions *pthread_funcs, } } + // libpthread.a in dyld "owns" the main thread structure itself and sets + // up the tsd to point to it. So take the pthread_self() from there + // and make it our main thread point. + pthread_t thread = _pthread_self_direct(); + if (os_unlikely(thread == NULL)) { + PTHREAD_INTERNAL_CRASH(0, "PTHREAD_SELF TSD not initialized"); + } + _main_thread_ptr = thread; + // this needs to be done early so that pthread_self() works in TSAN + _pthread_init_signature(thread); + // // Get host information // @@ -1829,11 +1765,11 @@ __pthread_init(const struct _libpthread_functions *pthread_funcs, host_t host = mach_host_self(); kr = host_info(host, flavor, (host_info_t)&priority_info, &count); if (kr != KERN_SUCCESS) { - PTHREAD_ABORT("host_info(mach_host_self(), ...) failed: %s", mach_error_string(kr)); + PTHREAD_INTERNAL_CRASH(kr, "host_info() failed"); } else { - default_priority = priority_info.user_priority; - min_priority = priority_info.minimum_priority; - max_priority = priority_info.maximum_priority; + default_priority = (uint8_t)priority_info.user_priority; + min_priority = (uint8_t)priority_info.minimum_priority; + max_priority = (uint8_t)priority_info.maximum_priority; } mach_port_deallocate(mach_task_self(), host); @@ -1863,12 +1799,14 @@ __pthread_init(const struct _libpthread_functions *pthread_funcs, allocsize = 0; } - pthread_t thread = &_thread; - pthread_attr_init(&_pthread_attr_default); + // Initialize random ptr_munge token from the kernel. + parse_ptr_munge_params(envp, apple); + + PTHREAD_DEBUG_ASSERT(_pthread_attr_default.qosclass == + _pthread_default_priority(0)); _pthread_struct_init(thread, &_pthread_attr_default, - stackaddr, stacksize, - allocaddr, allocsize); - thread->detached = PTHREAD_CREATE_JOINABLE; + stackaddr, stacksize, allocaddr, allocsize); + thread->tl_joinable = true; // Finish initialization with common code that is reinvoked on the // child side of a fork. @@ -1897,133 +1835,156 @@ __pthread_init(const struct _libpthread_functions *pthread_funcs, return 0; } +#endif // !VARIANT_DYLD -PTHREAD_NOEXPORT void +void _pthread_main_thread_init(pthread_t p) { TAILQ_INIT(&__pthread_head); - _PTHREAD_LOCK_INIT(_pthread_list_lock); - - // Re-use the main thread's static storage if no thread was provided. - if (p == NULL) { - if (_thread.tsd[0] != 0) { - bzero(&_thread, sizeof(struct _pthread)); - } - p = &_thread; - } - - _PTHREAD_LOCK_INIT(p->lock); - _pthread_set_kernel_thread(p, mach_thread_self()); - _pthread_set_reply_port(p, mach_reply_port()); + _pthread_lock_init(&_pthread_list_lock); + _pthread_lock_init(&p->lock); p->__cleanup_stack = NULL; - p->joiner_notify = SEMAPHORE_NULL; - p->joiner = MACH_PORT_NULL; - p->detached |= _PTHREAD_CREATE_PARENT; - p->tsd[__TSD_SEMAPHORE_CACHE] = (void*)SEMAPHORE_NULL; - p->cancel_state |= _PTHREAD_CANCEL_INITIALIZED; + p->tl_join_ctx = NULL; + p->tl_exit_gate = MACH_PORT_NULL; + _pthread_tsd_slot(p, MACH_THREAD_SELF) = mach_thread_self(); + _pthread_tsd_slot(p, MIG_REPLY) = mach_reply_port(); + _pthread_tsd_slot(p, MACH_SPECIAL_REPLY) = MACH_PORT_NULL; + _pthread_tsd_slot(p, SEMAPHORE_CACHE) = SEMAPHORE_NULL; // Initialize the list of threads with the new main thread. - TAILQ_INSERT_HEAD(&__pthread_head, p, plist); + TAILQ_INSERT_HEAD(&__pthread_head, p, tl_plist); _pthread_count = 1; - _pthread_set_self(p); _pthread_introspection_thread_start(p); } -int -_pthread_join_cleanup(pthread_t thread, void ** value_ptr, int conforming) +void +_pthread_main_thread_postfork_init(pthread_t p) { - int ret = __pthread_remove_thread(thread, false, NULL); - if (ret != 0 && ret != EBUSY) { - // Returns ESRCH if the thread was not created joinable. - return ret; - } - - if (value_ptr) { - *value_ptr = _pthread_get_exit_value(thread, conforming); - } - _pthread_introspection_thread_destroy(thread); - if (ret != EBUSY) { - // __pthread_remove_thread returns EBUSY if the parent has not - // finished creating the thread (and is still expecting the pthread_t - // to be alive). - _pthread_deallocate(thread); - } - return 0; + _pthread_main_thread_init(p); + _pthread_set_self_internal(p); } int sched_yield(void) { - swtch_pri(0); - return 0; + swtch_pri(0); + return 0; } -// XXX remove -void -cthread_yield(void) +// Libsystem knows about this symbol and exports it to libsyscall +int +pthread_current_stack_contains_np(const void *addr, size_t length) { - sched_yield(); -} + uintptr_t begin = (uintptr_t) addr, end; + uintptr_t stack_base = (uintptr_t) _pthread_self_direct()->stackbottom; + uintptr_t stack_top = (uintptr_t) _pthread_self_direct()->stackaddr; -void -pthread_yield_np(void) -{ - sched_yield(); -} + if (stack_base == stack_top) { + return -ENOTSUP; + } + if (__builtin_add_overflow(begin, length, &end)) { + return -EINVAL; + } + + return stack_base <= begin && end <= stack_top; +} +// Libsystem knows about this symbol and exports it to libsyscall -PTHREAD_NOEXPORT_VARIANT void -_pthread_clear_qos_tsd(mach_port_t thread_port) +_pthread_clear_qos_tsd(mach_port_t port) { - if (thread_port == MACH_PORT_NULL || (uintptr_t)_pthread_getspecific_direct(_PTHREAD_TSD_SLOT_MACH_THREAD_SELF) == thread_port) { + pthread_priority_t pp = _pthread_unspecified_priority(); + + if (port == MACH_PORT_NULL || _pthread_mach_thread_self_direct() == port) { /* Clear the current thread's TSD, that can be done inline. */ - _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, _pthread_priority_make_newest(QOS_CLASS_UNSPECIFIED, 0, 0)); + _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, pp); } else { pthread_t p; - _PTHREAD_LOCK(_pthread_list_lock); + _pthread_lock_lock(&_pthread_list_lock); - TAILQ_FOREACH(p, &__pthread_head, plist) { - mach_port_t kp = _pthread_kernel_thread(p); - if (thread_port == kp) { - p->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = _pthread_priority_make_newest(QOS_CLASS_UNSPECIFIED, 0, 0); + TAILQ_FOREACH(p, &__pthread_head, tl_plist) { + mach_port_t kp = _pthread_tsd_slot(p, MACH_THREAD_SELF); + if (port == kp) { + _pthread_tsd_slot(p, PTHREAD_QOS_CLASS) = pp; break; } } - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); } } +#pragma mark pthread/stack_np.h public interface + +#if defined(__i386__) || defined(__x86_64__) || defined(__arm__) || defined(__arm64__) +#if __ARM64_ARCH_8_32__ +/* + * arm64_32 uses 64-bit sizes for the frame pointer and + * return address of a stack frame. + */ +typedef uint64_t frame_data_addr_t; +#else +typedef uintptr_t frame_data_addr_t; +#endif + +struct frame_data { + frame_data_addr_t frame_addr_next; + frame_data_addr_t ret_addr; +}; +#else +#error ********** Unimplemented architecture +#endif + +uintptr_t +pthread_stack_frame_decode_np(uintptr_t frame_addr, uintptr_t *return_addr) +{ + struct frame_data *frame = (struct frame_data *)frame_addr; + + if (return_addr) { +#if __has_feature(ptrauth_calls) + *return_addr = (uintptr_t)ptrauth_strip((void *)frame->ret_addr, + ptrauth_key_return_address); +#else + *return_addr = (uintptr_t)frame->ret_addr; +#endif /* __has_feature(ptrauth_calls) */ + } + +#if __has_feature(ptrauth_calls) + return (uintptr_t)ptrauth_strip((void *)frame->frame_addr_next, + ptrauth_key_frame_pointer); +#endif /* __has_feature(ptrauth_calls) */ + return (uintptr_t)frame->frame_addr_next; +} -/***** pthread workqueue support routines *****/ +#pragma mark pthread workqueue support routines -PTHREAD_NOEXPORT void +void _pthread_bsdthread_init(struct _pthread_registration_data *data) { bzero(data, sizeof(*data)); data->version = sizeof(struct _pthread_registration_data); data->dispatch_queue_offset = __PTK_LIBDISPATCH_KEY0 * sizeof(void *); data->return_to_kernel_offset = __TSD_RETURN_TO_KERNEL * sizeof(void *); - data->tsd_offset = offsetof(struct _pthread, tsd); + data->tsd_offset = offsetof(struct pthread_s, tsd); data->mach_thread_self_offset = __TSD_MACH_THREAD_SELF * sizeof(void *); + data->joinable_offset_bits = CHAR_BIT * (offsetof(struct pthread_s, tl_policy) + 1); - int rv = __bsdthread_register(thread_start, - start_wqthread, (int)PTHREAD_SIZE, - (void*)data, (uintptr_t)sizeof(*data), - data->dispatch_queue_offset); + int rv = __bsdthread_register(thread_start, start_wqthread, (int)PTHREAD_SIZE, + (void*)data, (uintptr_t)sizeof(*data), data->dispatch_queue_offset); if (rv > 0) { - if ((rv & PTHREAD_FEATURE_QOS_DEFAULT) == 0) { - PTHREAD_INTERNAL_CRASH(rv, - "Missing required support for QOS_CLASS_DEFAULT"); - } - if ((rv & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) { - PTHREAD_INTERNAL_CRASH(rv, - "Missing required support for QOS_CLASS_MAINTENANCE"); + int required_features = + PTHREAD_FEATURE_FINEPRIO | + PTHREAD_FEATURE_BSDTHREADCTL | + PTHREAD_FEATURE_SETSELF | + PTHREAD_FEATURE_QOS_MAINTENANCE | + PTHREAD_FEATURE_QOS_DEFAULT; + if ((rv & required_features) != required_features) { + PTHREAD_INTERNAL_CRASH(rv, "Missing required kernel support"); } __pthread_supported_features = rv; } @@ -2039,9 +2000,13 @@ _pthread_bsdthread_init(struct _pthread_registration_data *data) pthread_priority_t main_qos = (pthread_priority_t)data->main_qos; - if (_pthread_priority_get_qos_newest(main_qos) != QOS_CLASS_UNSPECIFIED) { + if (_pthread_priority_thread_qos(main_qos) != THREAD_QOS_UNSPECIFIED) { _pthread_set_main_qos(main_qos); - _thread.tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = main_qos; + _pthread_tsd_slot(main_thread(), PTHREAD_QOS_CLASS) = main_qos; + } + + if (data->stack_addr_hint) { + __pthread_stack_hint = data->stack_addr_hint; } if (__libdispatch_workerfunction != NULL) { @@ -2050,191 +2015,170 @@ _pthread_bsdthread_init(struct _pthread_registration_data *data) } } -// workqueue entry point from kernel -PTHREAD_NORETURN -void -_pthread_wqthread(pthread_t self, mach_port_t kport, void *stacklowaddr, void *keventlist, int flags, int nkevents) +OS_NOINLINE +static void +_pthread_wqthread_legacy_worker_wrap(pthread_priority_t pp) { - PTHREAD_ASSERT(flags & WQ_FLAG_THREAD_NEWSPI); - - bool thread_reuse = flags & WQ_FLAG_THREAD_REUSE; - bool overcommit = flags & WQ_FLAG_THREAD_OVERCOMMIT; - bool kevent = flags & WQ_FLAG_THREAD_KEVENT; - bool workloop = (flags & WQ_FLAG_THREAD_WORKLOOP) && - __libdispatch_workloopfunction != NULL; - PTHREAD_ASSERT((!kevent) || (__libdispatch_keventfunction != NULL)); - PTHREAD_ASSERT(!workloop || kevent); - - pthread_priority_t priority = 0; - unsigned long priority_flags = 0; - - if (overcommit) - priority_flags |= _PTHREAD_PRIORITY_OVERCOMMIT_FLAG; - if (flags & WQ_FLAG_THREAD_EVENT_MANAGER) - priority_flags |= _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG; - if (kevent) - priority_flags |= _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG; + /* Old thread priorities are inverted from where we have them in + * the new flexible priority scheme. The highest priority is zero, + * up to 2, with background at 3. + */ + pthread_workqueue_function_t func = (pthread_workqueue_function_t)__libdispatch_workerfunction; + bool overcommit = (pp & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG); + int opts = overcommit ? WORKQ_ADDTHREADS_OPTION_OVERCOMMIT : 0; + + switch (_pthread_priority_thread_qos(pp)) { + case THREAD_QOS_USER_INITIATED: + return (*func)(WORKQ_HIGH_PRIOQUEUE, opts, NULL); + case THREAD_QOS_LEGACY: + /* B&I builders can't pass a QOS_CLASS_DEFAULT thread to dispatch, for fear of the QoS being + * picked up by NSThread (et al) and transported around the system. So change the TSD to + * make this thread look like QOS_CLASS_USER_INITIATED even though it will still run as legacy. + */ + _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, + _pthread_priority_make_from_thread_qos(THREAD_QOS_USER_INITIATED, 0, 0)); + return (*func)(WORKQ_DEFAULT_PRIOQUEUE, opts, NULL); + case THREAD_QOS_UTILITY: + return (*func)(WORKQ_LOW_PRIOQUEUE, opts, NULL); + case THREAD_QOS_BACKGROUND: + return (*func)(WORKQ_BG_PRIOQUEUE, opts, NULL); + } + PTHREAD_INTERNAL_CRASH(pp, "Invalid pthread priority for the legacy interface"); +} - int thread_class = flags & WQ_FLAG_THREAD_PRIOMASK; - priority = _pthread_priority_make_newest(thread_class, 0, priority_flags); +OS_ALWAYS_INLINE +static inline pthread_priority_t +_pthread_wqthread_priority(int flags) +{ + pthread_priority_t pp = 0; + thread_qos_t qos; - if (!thread_reuse) { - // New thread created by kernel, needs initialization. - void *stackaddr = self; - size_t stacksize = (uintptr_t)self - (uintptr_t)stacklowaddr; + if (flags & WQ_FLAG_THREAD_KEVENT) { + pp |= _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG; + } + if (flags & WQ_FLAG_THREAD_EVENT_MANAGER) { + return pp | _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG; + } - _pthread_struct_init(self, &_pthread_attr_default, - stackaddr, stacksize, - PTHREAD_ALLOCADDR(stackaddr, stacksize), PTHREAD_ALLOCSIZE(stackaddr, stacksize)); + if (flags & WQ_FLAG_THREAD_OVERCOMMIT) { + pp |= _PTHREAD_PRIORITY_OVERCOMMIT_FLAG; + } + if (flags & WQ_FLAG_THREAD_PRIO_QOS) { + qos = (thread_qos_t)(flags & WQ_FLAG_THREAD_PRIO_MASK); + pp = _pthread_priority_make_from_thread_qos(qos, 0, pp); + } else if (flags & WQ_FLAG_THREAD_PRIO_SCHED) { + pp |= _PTHREAD_PRIORITY_SCHED_PRI_MASK; + pp |= (flags & WQ_FLAG_THREAD_PRIO_MASK); + } else { + PTHREAD_INTERNAL_CRASH(flags, "Missing priority"); + } + return pp; +} - _pthread_set_kernel_thread(self, kport); - self->wqthread = 1; - self->wqkillset = 0; - self->cancel_state |= _PTHREAD_CANCEL_INITIALIZED; +OS_NOINLINE +static void +_pthread_wqthread_setup(pthread_t self, mach_port_t kport, void *stacklowaddr, + int flags) +{ + void *stackaddr = self; + size_t stacksize = (uintptr_t)self - (uintptr_t)stacklowaddr; - // Not a joinable thread. - self->detached &= ~PTHREAD_CREATE_JOINABLE; - self->detached |= PTHREAD_CREATE_DETACHED; + _pthread_struct_init(self, &_pthread_attr_default, stackaddr, stacksize, + PTHREAD_ALLOCADDR(stackaddr, stacksize), + PTHREAD_ALLOCSIZE(stackaddr, stacksize)); - // Update the running thread count and set childrun bit. - bool thread_tsd_base_set = (bool)(flags & WQ_FLAG_THREAD_TSD_BASE_SET); - _pthread_set_self_internal(self, !thread_tsd_base_set); - _pthread_introspection_thread_create(self, false); - __pthread_add_thread(self, NULL, false, false); - } + _pthread_tsd_slot(self, MACH_THREAD_SELF) = kport; + self->wqthread = 1; + self->wqkillset = 0; + self->tl_joinable = false; - // If we're running with fine-grained priority, we also need to - // set this thread to have the QoS class provided to use by the kernel - if (__pthread_supported_features & PTHREAD_FEATURE_FINEPRIO) { - _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, _pthread_priority_make_newest(thread_class, 0, priority_flags)); + // Update the running thread count and set childrun bit. + if (os_unlikely((flags & WQ_FLAG_THREAD_TSD_BASE_SET) == 0)) { + PTHREAD_INTERNAL_CRASH(flags, + "thread_set_tsd_base() wasn't called by the kernel"); } + _pthread_set_self_internal(self); + __pthread_add_thread(self, kport, false); + __pthread_started_thread(self); +} -#if WQ_DEBUG - PTHREAD_ASSERT(self); - PTHREAD_ASSERT(self == pthread_self()); -#endif // WQ_DEBUG +OS_NORETURN OS_NOINLINE +static void +_pthread_wqthread_exit(pthread_t self) +{ + const thread_qos_t WORKQ_THREAD_QOS_CLEANUP = THREAD_QOS_LEGACY; + pthread_priority_t pp = _pthread_tsd_slot(self, PTHREAD_QOS_CLASS); + thread_qos_t qos; - if (workloop) { - self->fun = (void *(*)(void*))__libdispatch_workloopfunction; - } else if (kevent){ - self->fun = (void *(*)(void*))__libdispatch_keventfunction; - } else { - self->fun = (void *(*)(void*))__libdispatch_workerfunction; + qos = _pthread_priority_thread_qos(pp); + if (qos == THREAD_QOS_UNSPECIFIED || qos > WORKQ_THREAD_QOS_CLEANUP) { + // Reset QoS to something low for the cleanup process + pp = _pthread_priority_make_from_thread_qos(WORKQ_THREAD_QOS_CLEANUP, 0, 0); + _pthread_tsd_slot(self, PTHREAD_QOS_CLASS) = pp; } - self->arg = (void *)(uintptr_t)thread_class; - - if (kevent && keventlist && nkevents > 0){ - int errors_out; - kevent_errors_retry: - if (workloop) { - kqueue_id_t kevent_id = *(kqueue_id_t*)((char*)keventlist - sizeof(kqueue_id_t)); - kqueue_id_t kevent_id_in = kevent_id; - (__libdispatch_workloopfunction)(&kevent_id, &keventlist, &nkevents); - PTHREAD_ASSERT(kevent_id == kevent_id_in || nkevents == 0); - errors_out = __workq_kernreturn(WQOPS_THREAD_WORKLOOP_RETURN, keventlist, nkevents, 0); - } else { - (__libdispatch_keventfunction)(&keventlist, &nkevents); - errors_out = __workq_kernreturn(WQOPS_THREAD_KEVENT_RETURN, keventlist, nkevents, 0); - } - - if (errors_out > 0){ - nkevents = errors_out; - goto kevent_errors_retry; - } else if (errors_out < 0){ - PTHREAD_ABORT("kevent return produced an error: %d", errno); - } - goto thexit; - } else if (kevent){ - if (workloop) { - (__libdispatch_workloopfunction)(0, NULL, NULL); - __workq_kernreturn(WQOPS_THREAD_WORKLOOP_RETURN, NULL, 0, -1); - } else { - (__libdispatch_keventfunction)(NULL, NULL); - __workq_kernreturn(WQOPS_THREAD_KEVENT_RETURN, NULL, 0, 0); - } + _pthread_exit(self, NULL); +} - goto thexit; - } - - if (__pthread_supported_features & PTHREAD_FEATURE_FINEPRIO) { - if (!__workq_newapi) { - /* Old thread priorities are inverted from where we have them in - * the new flexible priority scheme. The highest priority is zero, - * up to 2, with background at 3. - */ - pthread_workqueue_function_t func = (pthread_workqueue_function_t)__libdispatch_workerfunction; - - int opts = overcommit ? WORKQ_ADDTHREADS_OPTION_OVERCOMMIT : 0; - - if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_DEFAULT) == 0) { - /* Dirty hack to support kernels that don't have QOS_CLASS_DEFAULT. */ - switch (thread_class) { - case QOS_CLASS_USER_INTERACTIVE: - thread_class = QOS_CLASS_USER_INITIATED; - break; - case QOS_CLASS_USER_INITIATED: - thread_class = QOS_CLASS_DEFAULT; - break; - default: - break; - } - } +// workqueue entry point from kernel +void +_pthread_wqthread(pthread_t self, mach_port_t kport, void *stacklowaddr, + void *keventlist, int flags, int nkevents) +{ + if ((flags & WQ_FLAG_THREAD_REUSE) == 0) { + _pthread_wqthread_setup(self, kport, stacklowaddr, flags); + } - switch (thread_class) { - /* QOS_CLASS_USER_INTERACTIVE is not currently requested by for old dispatch priority compatibility */ - case QOS_CLASS_USER_INITIATED: - (*func)(WORKQ_HIGH_PRIOQUEUE, opts, NULL); - break; - - case QOS_CLASS_DEFAULT: - /* B&I builders can't pass a QOS_CLASS_DEFAULT thread to dispatch, for fear of the QoS being - * picked up by NSThread (et al) and transported around the system. So change the TSD to - * make this thread look like QOS_CLASS_USER_INITIATED even though it will still run as legacy. - */ - _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, _pthread_priority_make_newest(QOS_CLASS_USER_INITIATED, 0, 0)); - (*func)(WORKQ_DEFAULT_PRIOQUEUE, opts, NULL); - break; - - case QOS_CLASS_UTILITY: - (*func)(WORKQ_LOW_PRIOQUEUE, opts, NULL); - break; - - case QOS_CLASS_BACKGROUND: - (*func)(WORKQ_BG_PRIOQUEUE, opts, NULL); - break; - - /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */ - } + pthread_priority_t pp; - } else { - /* "New" API, where dispatch is expecting to be given the thread priority */ - (*__libdispatch_workerfunction)(priority); - } + if (flags & WQ_FLAG_THREAD_OUTSIDEQOS) { + self->wq_outsideqos = 1; + pp = _pthread_priority_make_from_thread_qos(THREAD_QOS_LEGACY, 0, + _PTHREAD_PRIORITY_FALLBACK_FLAG); } else { - /* We're the new library running on an old kext, so thread_class is really the workq priority. */ - pthread_workqueue_function_t func = (pthread_workqueue_function_t)__libdispatch_workerfunction; - int options = overcommit ? WORKQ_ADDTHREADS_OPTION_OVERCOMMIT : 0; - (*func)(thread_class, options, NULL); + self->wq_outsideqos = 0; + pp = _pthread_wqthread_priority(flags); } - __workq_kernreturn(WQOPS_THREAD_RETURN, NULL, 0, 0); + self->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = (void *)pp; -thexit: - { - pthread_priority_t current_priority = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS); - if ((current_priority & _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG) || - (_pthread_priority_get_qos_newest(current_priority) > WQ_THREAD_CLEANUP_QOS)) { - // Reset QoS to something low for the cleanup process - priority = _pthread_priority_make_newest(WQ_THREAD_CLEANUP_QOS, 0, 0); - _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority); + // avoid spills on the stack hard to keep used stack space minimal + if (os_unlikely(nkevents == WORKQ_EXIT_THREAD_NKEVENT)) { + _pthread_wqthread_exit(self); + } else if (flags & WQ_FLAG_THREAD_WORKLOOP) { + kqueue_id_t *kqidptr = (kqueue_id_t *)keventlist - 1; + self->fun = (void *(*)(void*))__libdispatch_workloopfunction; + self->arg = keventlist; + self->wq_nevents = nkevents; + (*__libdispatch_workloopfunction)(kqidptr, &self->arg, &self->wq_nevents); + __workq_kernreturn(WQOPS_THREAD_WORKLOOP_RETURN, self->arg, self->wq_nevents, 0); + } else if (flags & WQ_FLAG_THREAD_KEVENT) { + self->fun = (void *(*)(void*))__libdispatch_keventfunction; + self->arg = keventlist; + self->wq_nevents = nkevents; + (*__libdispatch_keventfunction)(&self->arg, &self->wq_nevents); + __workq_kernreturn(WQOPS_THREAD_KEVENT_RETURN, self->arg, self->wq_nevents, 0); + } else { + self->fun = (void *(*)(void*))__libdispatch_workerfunction; + self->arg = (void *)(uintptr_t)pp; + self->wq_nevents = 0; + if (os_likely(__workq_newapi)) { + (*__libdispatch_workerfunction)(pp); + } else { + _pthread_wqthread_legacy_worker_wrap(pp); } + __workq_kernreturn(WQOPS_THREAD_RETURN, NULL, 0, 0); } - _pthread_exit(self, NULL); + _os_set_crash_log_cause_and_message(self->err_no, + "BUG IN LIBPTHREAD: __workq_kernreturn returned"); + /* + * 52858993: we should never return but the compiler insists on outlining, + * so the __builtin_trap() is in _start_wqthread in pthread_asm.s + */ } -/***** pthread workqueue API for libdispatch *****/ +#pragma mark pthread workqueue API for libdispatch _Static_assert(WORKQ_KEVENT_EVENT_BUFFER_LEN == WQ_KEVENT_LIST_LEN, "Kernel and userland should agree on the event list size"); @@ -2242,33 +2186,70 @@ _Static_assert(WORKQ_KEVENT_EVENT_BUFFER_LEN == WQ_KEVENT_LIST_LEN, void pthread_workqueue_setdispatchoffset_np(int offset) { - __libdispatch_offset = offset; + __workq_kernreturn(WQOPS_QUEUE_NEWSPISUPP, NULL, offset, 0x00); } -static int -pthread_workqueue_setdispatch_with_workloop_np(pthread_workqueue_function2_t queue_func, - pthread_workqueue_function_kevent_t kevent_func, - pthread_workqueue_function_workloop_t workloop_func) +int +pthread_workqueue_setup(struct pthread_workqueue_config *cfg, size_t cfg_size) { - int res = EBUSY; + int rv = EBUSY; + struct workq_dispatch_config wdc_cfg; + size_t min_size = 0; + + if (cfg_size < sizeof(uint32_t)) { + return EINVAL; + } + + switch (cfg->version) { + case 1: + min_size = offsetof(struct pthread_workqueue_config, queue_label_offs); + break; + case 2: + min_size = sizeof(struct pthread_workqueue_config); + break; + default: + return EINVAL; + } + + if (!cfg || cfg_size < min_size) { + return EINVAL; + } + + if (cfg->flags & ~PTHREAD_WORKQUEUE_CONFIG_SUPPORTED_FLAGS || + cfg->version < PTHREAD_WORKQUEUE_CONFIG_MIN_SUPPORTED_VERSION) { + return ENOTSUP; + } + if (__libdispatch_workerfunction == NULL) { - // Check whether the kernel supports new SPIs - res = __workq_kernreturn(WQOPS_QUEUE_NEWSPISUPP, NULL, __libdispatch_offset, kevent_func != NULL ? 0x01 : 0x00); - if (res == -1){ - res = ENOTSUP; + __workq_newapi = true; + + wdc_cfg.wdc_version = WORKQ_DISPATCH_CONFIG_VERSION; + wdc_cfg.wdc_flags = 0; + wdc_cfg.wdc_queue_serialno_offs = cfg->queue_serialno_offs; +#if WORKQ_DISPATCH_CONFIG_VERSION >= 2 + wdc_cfg.wdc_queue_label_offs = cfg->queue_label_offs; +#endif + + // Tell the kernel about dispatch internals + rv = (int) __workq_kernreturn(WQOPS_SETUP_DISPATCH, &wdc_cfg, sizeof(wdc_cfg), 0); + if (rv == -1) { + return errno; } else { - __libdispatch_workerfunction = queue_func; - __libdispatch_keventfunction = kevent_func; - __libdispatch_workloopfunction = workloop_func; + __libdispatch_keventfunction = cfg->kevent_cb; + __libdispatch_workloopfunction = cfg->workloop_cb; + __libdispatch_workerfunction = cfg->workq_cb; // Prepare the kernel for workq action (void)__workq_open(); if (__is_threaded == 0) { __is_threaded = 1; } + + return 0; } } - return res; + + return rv; } int @@ -2277,15 +2258,17 @@ _pthread_workqueue_init_with_workloop(pthread_workqueue_function2_t queue_func, pthread_workqueue_function_workloop_t workloop_func, int offset, int flags) { - if (flags != 0) { - return ENOTSUP; - } - - __workq_newapi = true; - __libdispatch_offset = offset; + struct pthread_workqueue_config cfg = { + .version = PTHREAD_WORKQUEUE_CONFIG_VERSION, + .flags = 0, + .workq_cb = queue_func, + .kevent_cb = kevent_func, + .workloop_cb = workloop_func, + .queue_serialno_offs = offset, + .queue_label_offs = 0, + }; - int rv = pthread_workqueue_setdispatch_with_workloop_np(queue_func, kevent_func, workloop_func); - return rv; + return pthread_workqueue_setup(&cfg, sizeof(cfg)); } int @@ -2305,7 +2288,17 @@ _pthread_workqueue_init(pthread_workqueue_function2_t func, int offset, int flag int pthread_workqueue_setdispatch_np(pthread_workqueue_function_t worker_func) { - return pthread_workqueue_setdispatch_with_workloop_np((pthread_workqueue_function2_t)worker_func, NULL, NULL); + struct pthread_workqueue_config cfg = { + .version = PTHREAD_WORKQUEUE_CONFIG_VERSION, + .flags = 0, + .workq_cb = (uint64_t)(pthread_workqueue_function2_t)worker_func, + .kevent_cb = 0, + .workloop_cb = 0, + .queue_serialno_offs = 0, + .queue_label_offs = 0, + }; + + return pthread_workqueue_setup(&cfg, sizeof(cfg)); } int @@ -2329,42 +2322,18 @@ pthread_workqueue_addthreads_np(int queue_priority, int options, int numthreads) } pthread_priority_t kp = 0; + int compat_priority = queue_priority & WQ_FLAG_THREAD_PRIO_MASK; + int flags = 0; - if (__pthread_supported_features & PTHREAD_FEATURE_FINEPRIO) { - /* The new kernel API takes the new QoS class + relative priority style of - * priority. This entry point is here for compatibility with old libdispatch - * versions (ie. the simulator). We request the corresponding new bracket - * from the kernel, then on the way out run all dispatch queues that were - * requested. - */ - - int compat_priority = queue_priority & WQ_FLAG_THREAD_PRIOMASK; - int flags = 0; - - /* To make sure the library does not issue more threads to dispatch than - * were requested, the total number of active requests is recorded in - * __workq_requests. - */ - if (options & WORKQ_ADDTHREADS_OPTION_OVERCOMMIT) { - flags = _PTHREAD_PRIORITY_OVERCOMMIT_FLAG; - } + if (options & WORKQ_ADDTHREADS_OPTION_OVERCOMMIT) { + flags = _PTHREAD_PRIORITY_OVERCOMMIT_FLAG; + } #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdeprecated-declarations" - kp = _pthread_qos_class_encode_workqueue(compat_priority, flags); + kp = _pthread_qos_class_encode_workqueue(compat_priority, flags); #pragma clang diagnostic pop - } else { - /* Running on the old kernel, queue_priority is what we pass directly to - * the syscall. - */ - kp = queue_priority & WQ_FLAG_THREAD_PRIOMASK; - - if (options & WORKQ_ADDTHREADS_OPTION_OVERCOMMIT) { - kp |= WORKQUEUE_OVERCOMMIT; - } - } - res = __workq_kernreturn(WQOPS_QUEUE_REQTHREADS, NULL, numthreads, (int)kp); if (res == -1) { res = errno; @@ -2391,9 +2360,17 @@ _pthread_workqueue_addthreads(int numthreads, pthread_priority_t priority) return EPERM; } - if ((__pthread_supported_features & PTHREAD_FEATURE_FINEPRIO) == 0) { - return ENOTSUP; - } +#if TARGET_OS_OSX + // Legacy simulators fail to boot + // + // Older sims set the deprecated _PTHREAD_PRIORITY_ROOTQUEUE_FLAG wrongly, + // which is aliased to _PTHREAD_PRIORITY_SCHED_PRI_FLAG and that XNU + // validates and rejects. + // + // As a workaround, forcefully unset this bit that cannot be set here + // anyway. + priority &= ~_PTHREAD_PRIORITY_SCHED_PRI_FLAG; +#endif res = __workq_kernreturn(WQOPS_QUEUE_REQTHREADS, NULL, numthreads, (int)priority); if (res == -1) { @@ -2412,9 +2389,60 @@ _pthread_workqueue_set_event_manager_priority(pthread_priority_t priority) return res; } -/* - * Introspection SPI for libpthread. - */ +int +_pthread_workloop_create(uint64_t workloop_id, uint64_t options, pthread_attr_t *attr) +{ + struct kqueue_workloop_params params = { + .kqwlp_version = sizeof(struct kqueue_workloop_params), + .kqwlp_id = workloop_id, + .kqwlp_flags = 0, + }; + + if (!attr) { + return EINVAL; + } + + if (attr->schedset) { + params.kqwlp_flags |= KQ_WORKLOOP_CREATE_SCHED_PRI; + params.kqwlp_sched_pri = attr->param.sched_priority; + } + + if (attr->policyset) { + params.kqwlp_flags |= KQ_WORKLOOP_CREATE_SCHED_POL; + params.kqwlp_sched_pol = attr->policy; + } + + if (attr->cpupercentset) { + params.kqwlp_flags |= KQ_WORKLOOP_CREATE_CPU_PERCENT; + params.kqwlp_cpu_percent = attr->cpupercent; + params.kqwlp_cpu_refillms = attr->refillms; + } + + int res = __kqueue_workloop_ctl(KQ_WORKLOOP_CREATE, 0, ¶ms, + sizeof(params)); + if (res == -1) { + res = errno; + } + return res; +} + +int +_pthread_workloop_destroy(uint64_t workloop_id) +{ + struct kqueue_workloop_params params = { + .kqwlp_version = sizeof(struct kqueue_workloop_params), + .kqwlp_id = workloop_id, + }; + + int res = __kqueue_workloop_ctl(KQ_WORKLOOP_DESTROY, 0, ¶ms, + sizeof(params)); + if (res == -1) { + res = errno; + } + return res; +} + +#pragma mark Introspection SPI for libpthread. static pthread_introspection_hook_t _pthread_introspection_hook; @@ -2426,37 +2454,47 @@ pthread_introspection_hook_install(pthread_introspection_hook_t hook) return prev; } -PTHREAD_NOINLINE +static inline void +_pthread_introspection_call_hook(unsigned int event, + pthread_t thread, void *addr, size_t size) +{ + pthread_t self = pthread_self(); + uint16_t old = self->introspection; + self->introspection = (uint16_t)event; + _pthread_introspection_hook(event, thread, addr, size); + self->introspection = old; +} + +OS_NOINLINE static void -_pthread_introspection_hook_callout_thread_create(pthread_t t, bool destroy) +_pthread_introspection_hook_callout_thread_create(pthread_t t) { - _pthread_introspection_hook(PTHREAD_INTROSPECTION_THREAD_CREATE, t, t, + _pthread_introspection_call_hook(PTHREAD_INTROSPECTION_THREAD_CREATE, t, t, PTHREAD_SIZE); - if (!destroy) return; - _pthread_introspection_thread_destroy(t); } static inline void -_pthread_introspection_thread_create(pthread_t t, bool destroy) +_pthread_introspection_thread_create(pthread_t t) { if (os_fastpath(!_pthread_introspection_hook)) return; - _pthread_introspection_hook_callout_thread_create(t, destroy); + _pthread_introspection_hook_callout_thread_create(t); } -PTHREAD_NOINLINE +OS_NOINLINE static void _pthread_introspection_hook_callout_thread_start(pthread_t t) { size_t freesize; void *freeaddr; - if (t == &_thread) { - freesize = t->stacksize + t->guardsize; + if (t == main_thread()) { + size_t stacksize = t->stackaddr - t->stackbottom; + freesize = stacksize + t->guardsize; freeaddr = t->stackaddr - freesize; } else { freesize = t->freesize - PTHREAD_SIZE; freeaddr = t->freeaddr; } - _pthread_introspection_hook(PTHREAD_INTROSPECTION_THREAD_START, t, + _pthread_introspection_call_hook(PTHREAD_INTROSPECTION_THREAD_START, t, freeaddr, freesize); } @@ -2467,35 +2505,36 @@ _pthread_introspection_thread_start(pthread_t t) _pthread_introspection_hook_callout_thread_start(t); } -PTHREAD_NOINLINE +OS_NOINLINE static void -_pthread_introspection_hook_callout_thread_terminate(pthread_t t, - void *freeaddr, size_t freesize, bool destroy) +_pthread_introspection_hook_callout_thread_terminate(pthread_t t) { - if (destroy && freesize) { - freesize -= PTHREAD_SIZE; + size_t freesize; + void *freeaddr; + if (t == main_thread()) { + size_t stacksize = t->stackaddr - t->stackbottom; + freesize = stacksize + t->guardsize; + freeaddr = t->stackaddr - freesize; + } else { + freesize = t->freesize - PTHREAD_SIZE; + freeaddr = t->freeaddr; } - _pthread_introspection_hook(PTHREAD_INTROSPECTION_THREAD_TERMINATE, t, + _pthread_introspection_call_hook(PTHREAD_INTROSPECTION_THREAD_TERMINATE, t, freeaddr, freesize); - if (!destroy) return; - _pthread_introspection_thread_destroy(t); } static inline void -_pthread_introspection_thread_terminate(pthread_t t, void *freeaddr, - size_t freesize, bool destroy) +_pthread_introspection_thread_terminate(pthread_t t) { if (os_fastpath(!_pthread_introspection_hook)) return; - _pthread_introspection_hook_callout_thread_terminate(t, freeaddr, freesize, - destroy); + _pthread_introspection_hook_callout_thread_terminate(t); } -PTHREAD_NOINLINE +OS_NOINLINE static void _pthread_introspection_hook_callout_thread_destroy(pthread_t t) { - if (t == &_thread) return; - _pthread_introspection_hook(PTHREAD_INTROSPECTION_THREAD_DESTROY, t, t, + _pthread_introspection_call_hook(PTHREAD_INTROSPECTION_THREAD_DESTROY, t, t, PTHREAD_SIZE); } @@ -2506,3 +2545,39 @@ _pthread_introspection_thread_destroy(pthread_t t) _pthread_introspection_hook_callout_thread_destroy(t); } +#pragma mark libplatform shims +#if !VARIANT_DYLD + +#include + +// pthread_setup initializes large structures to 0, +// which the compiler turns into a library call to memset. +// +// To avoid linking against Libc, provide a simple wrapper +// that calls through to the libplatform primitives + +#undef memset +PTHREAD_NOEXPORT +void * +memset(void *b, int c, size_t len) +{ + return _platform_memset(b, c, len); +} + +#undef bzero +PTHREAD_NOEXPORT +void +bzero(void *s, size_t n) +{ + _platform_bzero(s, n); +} + +#undef memcpy +PTHREAD_NOEXPORT +void * +memcpy(void* a, const void* b, unsigned long s) +{ + return _platform_memmove(a, b, s); +} + +#endif // !VARIANT_DYLD