+ return ret;
+}
+
+#pragma mark pthread lifetime
+
+// Allocate a thread structure, stack and guard page.
+//
+// The thread structure may optionally be placed in the same allocation as the
+// stack, residing above the top of the stack. This cannot be done if a
+// custom stack address is provided.
+//
+// Similarly the guard page cannot be allocated if a custom stack address is
+// provided.
+//
+// The allocated thread structure is initialized with values that indicate how
+// it should be freed.
+
+static pthread_t
+_pthread_allocate(const pthread_attr_t *attrs, void **stack,
+ bool from_mach_thread)
+{
+ mach_vm_address_t allocaddr = __pthread_stack_hint;
+ size_t allocsize, guardsize, stacksize, pthreadoff;
+ kern_return_t kr;
+ pthread_t t;
+
+ if (os_unlikely(attrs->stacksize != 0 &&
+ attrs->stacksize < PTHREAD_STACK_MIN)) {
+ PTHREAD_CLIENT_CRASH(attrs->stacksize, "Stack size in attrs is too small");
+ }
+
+ if (os_unlikely(((uintptr_t)attrs->stackaddr % vm_page_size) != 0)) {
+ PTHREAD_CLIENT_CRASH(attrs->stacksize, "Unaligned stack addr in attrs");
+ }
+
+ // Allocate a pthread structure if necessary
+
+ if (attrs->stackaddr != NULL) {
+ allocsize = PTHREAD_SIZE;
+ guardsize = 0;
+ pthreadoff = 0;
+ // <rdar://problem/42588315> if the attrs struct specifies a custom
+ // stack address but not a custom size, using ->stacksize here instead
+ // of _pthread_attr_stacksize stores stacksize as zero, indicating
+ // that the stack size is unknown.
+ stacksize = attrs->stacksize;
+ } else {
+ guardsize = _pthread_attr_guardsize(attrs);
+ stacksize = _pthread_attr_stacksize(attrs) + PTHREAD_T_OFFSET;
+ pthreadoff = stacksize + guardsize;
+ allocsize = pthreadoff + PTHREAD_SIZE;
+ allocsize = mach_vm_round_page(allocsize);
+ }
+
+ kr = mach_vm_map(mach_task_self(), &allocaddr, allocsize, vm_page_size - 1,
+ VM_MAKE_TAG(VM_MEMORY_STACK)| VM_FLAGS_ANYWHERE, MEMORY_OBJECT_NULL,
+ 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+
+ if (kr != KERN_SUCCESS) {
+ kr = mach_vm_allocate(mach_task_self(), &allocaddr, allocsize,
+ VM_MAKE_TAG(VM_MEMORY_STACK)| VM_FLAGS_ANYWHERE);
+ } else if (__syscall_logger && !from_mach_thread) {
+ // libsyscall will not output malloc stack logging events when
+ // VM_MEMORY_STACK is passed in to facilitate mach thread promotion.
+ // To avoid losing the stack traces for normal p-thread create
+ // operations, libpthread must pretend to be the vm syscall and log
+ // the allocations. <rdar://36418708>
+ int eventTypeFlags = stack_logging_type_vm_allocate |
+ stack_logging_type_mapped_file_or_shared_mem;
+ __syscall_logger(eventTypeFlags | VM_MAKE_TAG(VM_MEMORY_STACK),
+ (uintptr_t)mach_task_self(), (uintptr_t)allocsize, 0,
+ (uintptr_t)allocaddr, 0);
+ }
+
+ if (kr != KERN_SUCCESS) {
+ *stack = NULL;
+ return NULL;
+ } else if (__syscall_logger && !from_mach_thread) {
+ // libsyscall will not output malloc stack logging events when
+ // VM_MEMORY_STACK is passed in to facilitate mach thread promotion.
+ // To avoid losing the stack traces for normal p-thread create
+ // operations, libpthread must pretend to be the vm syscall and log
+ // the allocations. <rdar://36418708>
+ int eventTypeFlags = stack_logging_type_vm_allocate;
+ __syscall_logger(eventTypeFlags | VM_MAKE_TAG(VM_MEMORY_STACK),
+ (uintptr_t)mach_task_self(), (uintptr_t)allocsize, 0,
+ (uintptr_t)allocaddr, 0);
+ }
+
+ // The stack grows down.
+ // Set the guard page at the lowest address of the
+ // newly allocated stack. Return the highest address
+ // of the stack.
+ if (guardsize) {
+ (void)mach_vm_protect(mach_task_self(), allocaddr, guardsize,
+ FALSE, VM_PROT_NONE);
+ }
+
+ // Thread structure resides at the top of the stack (when using a
+ // custom stack, allocsize == PTHREAD_SIZE, so places the pthread_t
+ // at allocaddr).
+ t = (pthread_t)(allocaddr + pthreadoff);
+ if (attrs->stackaddr) {
+ *stack = attrs->stackaddr;
+ } else {
+ *stack = t;
+ }
+
+ _pthread_struct_init(t, attrs, *stack, stacksize, allocaddr, allocsize);
+ return t;
+}
+
+PTHREAD_NOINLINE
+void
+_pthread_deallocate(pthread_t t, bool from_mach_thread)
+{
+ kern_return_t ret;
+
+ // Don't free the main thread.
+ if (t != main_thread()) {
+ if (!from_mach_thread) { // see __pthread_add_thread
+ _pthread_introspection_thread_destroy(t);
+ }
+ ret = mach_vm_deallocate(mach_task_self(), t->freeaddr, t->freesize);
+ if (ret != KERN_SUCCESS) {
+ PTHREAD_INTERNAL_CRASH(ret, "Unable to deallocate stack");
+ }
+ }
+}
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wreturn-stack-address"
+
+PTHREAD_NOINLINE
+static void*
+_pthread_current_stack_address(void)
+{
+ int a;
+ return &a;
+}
+
+#pragma clang diagnostic pop
+
+void
+_pthread_joiner_wake(pthread_t thread)
+{
+ uint32_t *exit_gate = &thread->tl_exit_gate;
+
+ for (;;) {
+ int ret = __ulock_wake(UL_UNFAIR_LOCK | ULF_NO_ERRNO, exit_gate, 0);
+ if (ret == 0 || ret == -ENOENT) {
+ return;
+ }
+ if (ret != -EINTR) {
+ PTHREAD_INTERNAL_CRASH(-ret, "pthread_join() wake failure");
+ }
+ }
+}
+
+// Terminates the thread if called from the currently running thread.
+PTHREAD_NORETURN PTHREAD_NOINLINE PTHREAD_NOT_TAIL_CALLED
+static void
+_pthread_terminate(pthread_t t, void *exit_value)
+{
+ _pthread_introspection_thread_terminate(t);
+
+ uintptr_t freeaddr = (uintptr_t)t->freeaddr;
+ size_t freesize = t->freesize;
+ bool should_exit;
+
+ // the size of just the stack
+ size_t freesize_stack = t->freesize;
+
+ // We usually pass our structure+stack to bsdthread_terminate to free, but
+ // if we get told to keep the pthread_t structure around then we need to
+ // adjust the free size and addr in the pthread_t to just refer to the
+ // structure and not the stack. If we do end up deallocating the
+ // structure, this is useless work since no one can read the result, but we
+ // can't do it after the call to pthread_remove_thread because it isn't
+ // safe to dereference t after that.
+ if ((void*)t > t->freeaddr && (void*)t < t->freeaddr + t->freesize){
+ // Check to ensure the pthread structure itself is part of the
+ // allocation described by freeaddr/freesize, in which case we split and
+ // only deallocate the area below the pthread structure. In the event of a
+ // custom stack, the freeaddr/size will be the pthread structure itself, in
+ // which case we shouldn't free anything (the final else case).
+ freesize_stack = trunc_page((uintptr_t)t - (uintptr_t)freeaddr);
+
+ // describe just the remainder for deallocation when the pthread_t goes away
+ t->freeaddr += freesize_stack;
+ t->freesize -= freesize_stack;
+ } else if (t == main_thread()) {
+ freeaddr = t->stackaddr - pthread_get_stacksize_np(t);
+ uintptr_t stackborder = trunc_page((uintptr_t)_pthread_current_stack_address());
+ freesize_stack = stackborder - freeaddr;
+ } else {
+ freesize_stack = 0;
+ }
projects / apple / libpthread.git / blobdiff