From c1f56ec94c29e576da2d6d7164bee1bdb2f02471 Mon Sep 17 00:00:00 2001 From: Apple Date: Wed, 18 Nov 2020 23:17:44 +0000 Subject: [PATCH] libpthread-454.40.3.tar.gz --- {pthread => include/pthread}/introspection.h | 44 + {pthread => include/pthread}/pthread.h | 30 +- {pthread => include/pthread}/pthread_impl.h | 0 {pthread => include/pthread}/pthread_spis.h | 2 +- {pthread => include/pthread}/qos.h | 0 {pthread => include/pthread}/sched.h | 4 +- {pthread => include/pthread}/spawn.h | 0 {pthread => include/pthread}/stack_np.h | 0 .../sys}/_pthread/_pthread_attr_t.h | 12 +- .../sys}/_pthread/_pthread_cond_t.h | 8 +- .../sys}/_pthread/_pthread_condattr_t.h | 8 +- .../sys}/_pthread/_pthread_key_t.h | 0 .../sys}/_pthread/_pthread_mutex_t.h | 0 .../sys}/_pthread/_pthread_mutexattr_t.h | 0 .../sys}/_pthread/_pthread_once_t.h | 0 .../sys}/_pthread/_pthread_rwlock_t.h | 8 +- .../sys}/_pthread/_pthread_rwlockattr_t.h | 8 +- {sys => include/sys}/_pthread/_pthread_t.h | 8 +- .../sys}/_pthread/_pthread_types.h | 8 +- {sys => include/sys}/qos.h | 0 include/sys_pthread_types.modulemap | 8 + kern/kern_init.c | 2 +- kern/kern_internal.h | 16 +- kern/kern_support.c | 5 +- kern/kern_synch.c | 10 +- kern/kern_trace.h | 44 +- kern/workqueue_internal.h | 50 -- libpthread.xcodeproj/project.pbxproj | 198 +++-- man/pthread_jit_write_protect_np.3 | 71 ++ private/{ => pthread}/dependency_private.h | 0 private/{ => pthread}/introspection_private.h | 0 private/{ => pthread}/posix_sched.h | 0 private/{ => pthread}/private.h | 106 ++- private/{ => pthread}/qos.h | 0 private/{ => pthread}/qos_private.h | 0 private/{ => pthread}/spinlock_private.h | 0 private/{ => pthread}/tsd_private.h | 62 +- private/{ => pthread}/workqueue_private.h | 3 - {sys => private/sys}/qos_private.h | 0 src/exports_internal.h | 165 ++++ src/imports_internal.h | 87 ++ src/inline_internal.h | 249 ++++++ src/internal.h | 749 +----------------- src/mk_pthread_impl.c | 110 --- src/{offsets.h => offsets_internal.h} | 26 +- src/prototypes_internal.h | 128 +++ src/pthread.c | 637 +++++++-------- src/pthread_asm.s | 2 +- src/pthread_atfork.c | 31 +- src/pthread_cancelable.c | 333 +++----- src/pthread_cond.c | 660 ++++++++++----- src/pthread_cwd.c | 29 +- src/pthread_dependency.c | 31 +- src/pthread_mutex.c | 716 +++++++++++------ src/pthread_rwlock.c | 213 ++--- src/pthread_tsd.c | 83 +- src/qos.c | 70 +- src/types_internal.h | 507 ++++++++++++ src/variants/pthread_cancelable_legacy.c | 34 - src/variants/pthread_cond_legacy.c | 34 - src/variants/pthread_mutex_legacy.c | 34 - src/variants/pthread_rwlock_legacy.c | 37 - tests/Makefile | 6 +- tests/cond.c | 452 +++++++++++ tests/mutex.c | 47 +- tests/pthread_attr_setstacksize.c | 1 + tests/pthread_cwd.c | 2 - tests/pthread_exit.c | 17 + tests/pthread_introspection.c | 30 +- ...ad_jit_write_protection-entitlements.plist | 9 + tests/pthread_jit_write_protection.c | 353 +++++++++ tests/tsd.c | 26 + tests/wq_event_manager.c | 4 +- tests/wq_kevent.c | 4 +- tests/wq_kevent_stress.c | 4 +- xcodescripts/install-manpages.sh | 5 +- xcodescripts/install-sys-headers.sh | 45 +- xcodescripts/kext.xcconfig | 2 +- xcodescripts/pthread-i386.aliases | 21 + xcodescripts/pthread-tapi.xcconfig | 11 + xcodescripts/pthread.aliases | 4 + xcodescripts/pthread.dirty | 1 - xcodescripts/pthread.xcconfig | 41 +- xcodescripts/pthread_driverkit.xcconfig | 2 + xcodescripts/resolver.xcconfig | 1 + xcodescripts/static.xcconfig | 1 + 86 files changed, 4206 insertions(+), 2563 deletions(-) rename {pthread => include/pthread}/introspection.h (72%) rename {pthread => include/pthread}/pthread.h (94%) rename {pthread => include/pthread}/pthread_impl.h (100%) rename {pthread => include/pthread}/pthread_spis.h (97%) rename {pthread => include/pthread}/qos.h (100%) rename {pthread => include/pthread}/sched.h (95%) rename {pthread => include/pthread}/spawn.h (100%) rename {pthread => include/pthread}/stack_np.h (100%) rename {sys => include/sys}/_pthread/_pthread_attr_t.h (95%) rename {sys => include/sys}/_pthread/_pthread_cond_t.h (98%) rename {sys => include/sys}/_pthread/_pthread_condattr_t.h (98%) rename {sys => include/sys}/_pthread/_pthread_key_t.h (100%) rename {sys => include/sys}/_pthread/_pthread_mutex_t.h (100%) rename {sys => include/sys}/_pthread/_pthread_mutexattr_t.h (100%) rename {sys => include/sys}/_pthread/_pthread_once_t.h (100%) rename {sys => include/sys}/_pthread/_pthread_rwlock_t.h (98%) rename {sys => include/sys}/_pthread/_pthread_rwlockattr_t.h (98%) rename {sys => include/sys}/_pthread/_pthread_t.h (98%) rename {sys => include/sys}/_pthread/_pthread_types.h (99%) rename {sys => include/sys}/qos.h (100%) create mode 100644 include/sys_pthread_types.modulemap delete mode 100644 kern/workqueue_internal.h create mode 100644 man/pthread_jit_write_protect_np.3 rename private/{ => pthread}/dependency_private.h (100%) rename private/{ => pthread}/introspection_private.h (100%) rename private/{ => pthread}/posix_sched.h (100%) rename private/{ => pthread}/private.h (61%) rename private/{ => pthread}/qos.h (100%) rename private/{ => pthread}/qos_private.h (100%) rename private/{ => pthread}/spinlock_private.h (100%) rename private/{ => pthread}/tsd_private.h (85%) rename private/{ => pthread}/workqueue_private.h (99%) rename {sys => private/sys}/qos_private.h (100%) create mode 100644 src/exports_internal.h create mode 100644 src/imports_internal.h create mode 100644 src/inline_internal.h delete mode 100644 src/mk_pthread_impl.c rename src/{offsets.h => offsets_internal.h} (83%) create mode 100644 src/prototypes_internal.h create mode 100644 src/types_internal.h delete mode 100644 src/variants/pthread_cancelable_legacy.c delete mode 100644 src/variants/pthread_cond_legacy.c delete mode 100644 src/variants/pthread_mutex_legacy.c delete mode 100644 src/variants/pthread_rwlock_legacy.c create mode 100644 tests/pthread_jit_write_protection-entitlements.plist create mode 100644 tests/pthread_jit_write_protection.c create mode 100644 xcodescripts/pthread-i386.aliases create mode 100644 xcodescripts/pthread-tapi.xcconfig create mode 100644 xcodescripts/pthread_driverkit.xcconfig diff --git a/pthread/introspection.h b/include/pthread/introspection.h similarity index 72% rename from pthread/introspection.h rename to include/pthread/introspection.h index 10b719a..7aa9f41 100644 --- a/pthread/introspection.h +++ b/include/pthread/introspection.h @@ -128,6 +128,50 @@ __attribute__((__nonnull__, __warn_unused_result__)) extern pthread_introspection_hook_t pthread_introspection_hook_install(pthread_introspection_hook_t hook); +/*! + * @function pthread_introspection_setspecific_np + * + * @abstract + * Performs the moral equivalent of pthread_setspecific() on a target thread + * during the @c PTHREAD_INTROSPECTION_START callback. + * + * @description + * This function is only valid to call during the delivery + * of an @c PTHREAD_INTROSPECTION_THREAD_CREATE introspection hook. + * + * Using this function outside of this context is undefined. + * + * If the created thread is started, then the destructor for this key + * will be called when the thread is terminated. However if the thread + * is not started, it will not be called, and + * pthread_introspection_getspecific_np() must be called manually during + * the @c PTHREAD_INTROSPECTION_THREAD_DESTROY callback to perform manual + * cleanup. + */ +__API_AVAILABLE(macos(10.16), ios(14.0), tvos(14.0), watchos(7.0)) +int +pthread_introspection_setspecific_np(pthread_t thread, + pthread_key_t key, const void * _Nullable value); + +/*! + * @function pthread_introspection_getspecific_np + * + * @abstract + * Performs the moral equivalent of pthread_getspecific() on a target thread + * during the @c PTHREAD_INTROSPECTION_THREAD_DESTROY callback. + * + * @description + * This function is only valid to call during the delivery + * of an @c PTHREAD_INTROSPECTION_THREAD_DESTROY introspection hook. + * + * If the thread was started then this will always return NULL even + * when pthread_introspection_setspecific_np() was used. + */ +__API_AVAILABLE(macos(10.16), ios(14.0), tvos(14.0), watchos(7.0)) +void * _Nullable +pthread_introspection_getspecific_np(pthread_t _Nonnull thread, + pthread_key_t key); + __END_DECLS #endif diff --git a/pthread/pthread.h b/include/pthread/pthread.h similarity index 94% rename from pthread/pthread.h rename to include/pthread/pthread.h index a042c82..5d52d71 100644 --- a/pthread/pthread.h +++ b/include/pthread/pthread.h @@ -53,9 +53,6 @@ #define _PTHREAD_H #include <_types.h> -#ifndef __POSIX_LIB__ -#include -#endif #include #include #include @@ -559,6 +556,33 @@ int pthread_sigmask(int, const sigset_t * _Nullable, sigset_t * _Nullable) __API_AVAILABLE(macos(10.4), ios(2.0)) void pthread_yield_np(void); +__API_AVAILABLE(macos(10.16)) +__SPI_AVAILABLE(ios(14.0), tvos(14.0), watchos(7.0), bridgeos(5.0), driverkit(20.0)) +void pthread_jit_write_protect_np(int enabled); + +__API_AVAILABLE(macos(10.16)) +__SPI_AVAILABLE(ios(14.0), tvos(14.0), watchos(7.0), bridgeos(5.0), driverkit(20.0)) +int pthread_jit_write_protect_supported_np(void); + +/*! + * @function pthread_cpu_number_np + * + * @param cpu_number_out + * The CPU number that the thread was running on at the time of query. + * This cpu number is in the interval [0, ncpus) (from sysctlbyname("hw.ncpu")) + * + * @result + * This function returns 0 or the value of errno if an error occurred. + * + * @note + * Optimizations of per-CPU datastructures based on the result of this function + * still require synchronization since it is not guaranteed that the thread will + * still be on the same CPU by the time the function returns. + */ +__API_AVAILABLE(macos(11.0), ios(14.2), tvos(14.2), watchos(7.1)) +int +pthread_cpu_number_np(size_t *cpu_number_out); + #endif /* (!_POSIX_C_SOURCE && !_XOPEN_SOURCE) || _DARWIN_C_SOURCE || __cplusplus */ __END_DECLS #if __has_feature(assume_nonnull) diff --git a/pthread/pthread_impl.h b/include/pthread/pthread_impl.h similarity index 100% rename from pthread/pthread_impl.h rename to include/pthread/pthread_impl.h diff --git a/pthread/pthread_spis.h b/include/pthread/pthread_spis.h similarity index 97% rename from pthread/pthread_spis.h rename to include/pthread/pthread_spis.h index 91fb641..e1e7930 100644 --- a/pthread/pthread_spis.h +++ b/include/pthread/pthread_spis.h @@ -73,7 +73,7 @@ __BEGIN_DECLS #endif /* (!_POSIX_C_SOURCE && !_XOPEN_SOURCE) || _DARWIN_C_SOURCE */ -__API_AVAILABLE(macos(10.11)) +__API_AVAILABLE(macos(10.11)) __API_UNAVAILABLE(ios, tvos, watchos, bridgeos) void _pthread_mutex_enable_legacy_mode(void); /* diff --git a/pthread/qos.h b/include/pthread/qos.h similarity index 100% rename from pthread/qos.h rename to include/pthread/qos.h diff --git a/pthread/sched.h b/include/pthread/sched.h similarity index 95% rename from pthread/sched.h rename to include/pthread/sched.h index 91efb4e..2ef6e2b 100644 --- a/pthread/sched.h +++ b/include/pthread/sched.h @@ -25,7 +25,7 @@ #define _SCHED_H_ #include -#include +#include __BEGIN_DECLS /* @@ -33,6 +33,8 @@ __BEGIN_DECLS */ #ifndef __POSIX_LIB__ struct sched_param { int sched_priority; char __opaque[__SCHED_PARAM_SIZE__]; }; +#else +struct sched_param; #endif extern int sched_yield(void); diff --git a/pthread/spawn.h b/include/pthread/spawn.h similarity index 100% rename from pthread/spawn.h rename to include/pthread/spawn.h diff --git a/pthread/stack_np.h b/include/pthread/stack_np.h similarity index 100% rename from pthread/stack_np.h rename to include/pthread/stack_np.h diff --git a/sys/_pthread/_pthread_attr_t.h b/include/sys/_pthread/_pthread_attr_t.h similarity index 95% rename from sys/_pthread/_pthread_attr_t.h rename to include/sys/_pthread/_pthread_attr_t.h index cba5882..94db170 100644 --- a/sys/_pthread/_pthread_attr_t.h +++ b/include/sys/_pthread/_pthread_attr_t.h @@ -2,7 +2,7 @@ * Copyright (c) 2003-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,11 +22,11 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ -#ifndef _PTHREAD_ATTR_T -#define _PTHREAD_ATTR_T +#ifndef _PTHREAD_ATTR_T +#define _PTHREAD_ATTR_T #include /* __darwin_pthread_attr_t */ typedef __darwin_pthread_attr_t pthread_attr_t; #endif /* _PTHREAD_ATTR_T */ diff --git a/sys/_pthread/_pthread_cond_t.h b/include/sys/_pthread/_pthread_cond_t.h similarity index 98% rename from sys/_pthread/_pthread_cond_t.h rename to include/sys/_pthread/_pthread_cond_t.h index b6a7b42..4f9a05f 100644 --- a/sys/_pthread/_pthread_cond_t.h +++ b/include/sys/_pthread/_pthread_cond_t.h @@ -2,7 +2,7 @@ * Copyright (c) 2003-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _PTHREAD_COND_T diff --git a/sys/_pthread/_pthread_condattr_t.h b/include/sys/_pthread/_pthread_condattr_t.h similarity index 98% rename from sys/_pthread/_pthread_condattr_t.h rename to include/sys/_pthread/_pthread_condattr_t.h index 6e9227a..a18e5a8 100644 --- a/sys/_pthread/_pthread_condattr_t.h +++ b/include/sys/_pthread/_pthread_condattr_t.h @@ -2,7 +2,7 @@ * Copyright (c) 2003-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _PTHREAD_CONDATTR_T diff --git a/sys/_pthread/_pthread_key_t.h b/include/sys/_pthread/_pthread_key_t.h similarity index 100% rename from sys/_pthread/_pthread_key_t.h rename to include/sys/_pthread/_pthread_key_t.h diff --git a/sys/_pthread/_pthread_mutex_t.h b/include/sys/_pthread/_pthread_mutex_t.h similarity index 100% rename from sys/_pthread/_pthread_mutex_t.h rename to include/sys/_pthread/_pthread_mutex_t.h diff --git a/sys/_pthread/_pthread_mutexattr_t.h b/include/sys/_pthread/_pthread_mutexattr_t.h similarity index 100% rename from sys/_pthread/_pthread_mutexattr_t.h rename to include/sys/_pthread/_pthread_mutexattr_t.h diff --git a/sys/_pthread/_pthread_once_t.h b/include/sys/_pthread/_pthread_once_t.h similarity index 100% rename from sys/_pthread/_pthread_once_t.h rename to include/sys/_pthread/_pthread_once_t.h diff --git a/sys/_pthread/_pthread_rwlock_t.h b/include/sys/_pthread/_pthread_rwlock_t.h similarity index 98% rename from sys/_pthread/_pthread_rwlock_t.h rename to include/sys/_pthread/_pthread_rwlock_t.h index 75c4e35..0c61c89 100644 --- a/sys/_pthread/_pthread_rwlock_t.h +++ b/include/sys/_pthread/_pthread_rwlock_t.h @@ -2,7 +2,7 @@ * Copyright (c) 2003-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _PTHREAD_RWLOCK_T diff --git a/sys/_pthread/_pthread_rwlockattr_t.h b/include/sys/_pthread/_pthread_rwlockattr_t.h similarity index 98% rename from sys/_pthread/_pthread_rwlockattr_t.h rename to include/sys/_pthread/_pthread_rwlockattr_t.h index 6ccd234..936a565 100644 --- a/sys/_pthread/_pthread_rwlockattr_t.h +++ b/include/sys/_pthread/_pthread_rwlockattr_t.h @@ -2,7 +2,7 @@ * Copyright (c) 2003-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _PTHREAD_RWLOCKATTR_T diff --git a/sys/_pthread/_pthread_t.h b/include/sys/_pthread/_pthread_t.h similarity index 98% rename from sys/_pthread/_pthread_t.h rename to include/sys/_pthread/_pthread_t.h index 4d9e3da..519f6e0 100644 --- a/sys/_pthread/_pthread_t.h +++ b/include/sys/_pthread/_pthread_t.h @@ -2,7 +2,7 @@ * Copyright (c) 2003-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _PTHREAD_T diff --git a/sys/_pthread/_pthread_types.h b/include/sys/_pthread/_pthread_types.h similarity index 99% rename from sys/_pthread/_pthread_types.h rename to include/sys/_pthread/_pthread_types.h index d9d51b8..123c31a 100644 --- a/sys/_pthread/_pthread_types.h +++ b/include/sys/_pthread/_pthread_types.h @@ -2,7 +2,7 @@ * Copyright (c) 2003-2013 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ diff --git a/sys/qos.h b/include/sys/qos.h similarity index 100% rename from sys/qos.h rename to include/sys/qos.h diff --git a/include/sys_pthread_types.modulemap b/include/sys_pthread_types.modulemap new file mode 100644 index 0000000..36590aa --- /dev/null +++ b/include/sys_pthread_types.modulemap @@ -0,0 +1,8 @@ +// Module map for sys/_pthread/_pthread_types.h. +// +// Expected to be included as a submodule of the top-level module for stdint.h. + +module Darwin_C_stdint._pthread_types { + export * + header "sys/_pthread/_pthread_types.h" +} diff --git a/kern/kern_init.c b/kern/kern_init.c index 3321483..871b0d8 100644 --- a/kern/kern_init.c +++ b/kern/kern_init.c @@ -8,7 +8,7 @@ #include #include -#include "kern_internal.h" +#include "kern/kern_internal.h" kern_return_t pthread_start(kmod_info_t * ki, void *d); kern_return_t pthread_stop(kmod_info_t *ki, void *d); diff --git a/kern/kern_internal.h b/kern/kern_internal.h index bb29cdc..fb84250 100644 --- a/kern/kern_internal.h +++ b/kern/kern_internal.h @@ -41,19 +41,12 @@ struct ksyn_waitq_element; #include #include #include - -#ifdef __arm64__ -#define PTHREAD_INLINE_RMW_ATOMICS 0 -#else -#define PTHREAD_INLINE_RMW_ATOMICS 1 -#endif #endif // KERNEL #include "kern/synch_internal.h" -#include "kern/workqueue_internal.h" #include "kern/kern_trace.h" #include "pthread/qos.h" -#include "private/qos_private.h" +#include "pthread/qos_private.h" /* pthread userspace SPI feature checking, these constants are returned from bsdthread_register, * as a bitmask, to inform userspace of the supported feature set. Old releases of OS X return @@ -88,7 +81,11 @@ struct _pthread_registration_data { uint32_t return_to_kernel_offset; /* copy-in */ uint32_t mach_thread_self_offset; /* copy-in */ mach_vm_address_t stack_addr_hint; /* copy-out */ +#define _PTHREAD_REG_DEFAULT_POLICY_MASK 0xff +#define _PTHREAD_REG_DEFAULT_USE_ULOCK 0x100 +#define _PTHREAD_REG_DEFAULT_USE_ADAPTIVE_SPIN 0x200 uint32_t mutex_default_policy; /* copy-out */ + uint32_t joinable_offset_bits; /* copy-in */ } __attribute__ ((packed)); /* @@ -207,5 +204,8 @@ workq_markfree_threadstack(proc_t p, thread_t th, vm_map_t vmap, #endif // KERNEL +// magical `nkevents` values for _pthread_wqthread +#define WORKQ_EXIT_THREAD_NKEVENT (-1) + #endif /* _SYS_PTHREAD_INTERNAL_H_ */ diff --git a/kern/kern_support.c b/kern/kern_support.c index 0e576c2..4c7848f 100644 --- a/kern/kern_support.c +++ b/kern/kern_support.c @@ -102,10 +102,7 @@ extern void panic(const char *string, ...) __printflike(1,2) __dead2; #include #include -#include -#include - -#include "kern_internal.h" +#include "kern/kern_internal.h" #ifndef WQ_SETUP_EXIT_THREAD #define WQ_SETUP_EXIT_THREAD 8 diff --git a/kern/kern_synch.c b/kern/kern_synch.c index 930abc0..eed69a7 100644 --- a/kern/kern_synch.c +++ b/kern/kern_synch.c @@ -80,13 +80,9 @@ #include #include -#include - -#include - -#include "kern_internal.h" -#include "synch_internal.h" -#include "kern_trace.h" +#include "kern/kern_internal.h" +#include "kern/synch_internal.h" +#include "kern/kern_trace.h" typedef struct uthread *uthread_t; diff --git a/kern/kern_trace.h b/kern/kern_trace.h index 2e59edc..ce87532 100644 --- a/kern/kern_trace.h +++ b/kern/kern_trace.h @@ -42,6 +42,8 @@ // WQ_TRACE_REQUESTS_SUBCLASS is 2, in xnu # define _TRACE_SUB_MUTEX 3 # define _TRACE_SUB_CONDVAR 4 +# define _TRACE_SUB_ULMUTEX 5 +# define _TRACE_SUB_ULCOND 6 #ifndef _PTHREAD_BUILDING_CODES_ @@ -67,12 +69,6 @@ VM_UNSLIDE(void* ptr) # define PTHREAD_TRACE(x,a,b,c,d) \ { if (pthread_debug_tracing) { KERNEL_DEBUG_CONSTANT(TRACE_##x, a, b, c, d, 0); } } -# define PTHREAD_TRACE_WQ(x,a,b,c,d) \ - { if (pthread_debug_tracing) { KERNEL_DEBUG_CONSTANT(TRACE_##x, VM_UNSLIDE(a), b, c, d, 0); } } - -# define PTHREAD_TRACE_WQ_REQ(x,a,b,c,d,e) \ - { if (pthread_debug_tracing) { KERNEL_DEBUG_CONSTANT(TRACE_##x, VM_UNSLIDE(a), VM_UNSLIDE(b), c, d, e); } } - #else // KERNEL #if ENABLE_USERSPACE_TRACE @@ -107,32 +103,6 @@ TRACE_CODE(pthread_thread_create, _TRACE_SUB_DEFAULT, 0x10); TRACE_CODE(pthread_thread_terminate, _TRACE_SUB_DEFAULT, 0x20); TRACE_CODE(pthread_set_qos_self, _TRACE_SUB_DEFAULT, 0x30); -// workqueue trace points -TRACE_CODE(wq_pthread_exit, _TRACE_SUB_WORKQUEUE, 0x01); -TRACE_CODE(wq_workqueue_exit, _TRACE_SUB_WORKQUEUE, 0x02); -TRACE_CODE(wq_runthread, _TRACE_SUB_WORKQUEUE, 0x03); -TRACE_CODE(wq_runitem, _TRACE_SUB_WORKQUEUE, 0x04); -TRACE_CODE(wq_thread_block, _TRACE_SUB_WORKQUEUE, 0x9); -TRACE_CODE(wq_thactive_update, _TRACE_SUB_WORKQUEUE, 0xa); -TRACE_CODE(wq_add_timer, _TRACE_SUB_WORKQUEUE, 0xb); -TRACE_CODE(wq_start_add_timer, _TRACE_SUB_WORKQUEUE, 0x0c); -TRACE_CODE(wq_override_start, _TRACE_SUB_WORKQUEUE, 0x12); -TRACE_CODE(wq_override_end, _TRACE_SUB_WORKQUEUE, 0x13); -TRACE_CODE(wq_override_dispatch, _TRACE_SUB_WORKQUEUE, 0x14); -TRACE_CODE(wq_override_reset, _TRACE_SUB_WORKQUEUE, 0x15); -TRACE_CODE(wq_thread_create_failed, _TRACE_SUB_WORKQUEUE, 0x1d); -TRACE_CODE(wq_thread_create, _TRACE_SUB_WORKQUEUE, 0x1f); -TRACE_CODE(wq_run_threadreq, _TRACE_SUB_WORKQUEUE, 0x20); -TRACE_CODE(wq_run_threadreq_mgr_merge, _TRACE_SUB_WORKQUEUE, 0x21); -TRACE_CODE(wq_run_threadreq_req_select, _TRACE_SUB_WORKQUEUE, 0x22); -TRACE_CODE(wq_run_threadreq_thread_select, _TRACE_SUB_WORKQUEUE, 0x23); -TRACE_CODE(wq_thread_reset_priority, _TRACE_SUB_WORKQUEUE, 0x24); -TRACE_CODE(wq_constrained_admission, _TRACE_SUB_WORKQUEUE, 0x25); -TRACE_CODE(wq_wqops_reqthreads, _TRACE_SUB_WORKQUEUE, 0x26); -TRACE_CODE(wq_kevent_reqthreads, _TRACE_SUB_WORKQUEUE, 0x27); -TRACE_CODE(wq_thread_park, _TRACE_SUB_WORKQUEUE, 0x28); -TRACE_CODE(wq_thread_squash, _TRACE_SUB_WORKQUEUE, 0x29); - // synch trace points TRACE_CODE(psynch_mutex_ulock, _TRACE_SUB_MUTEX, 0x0); TRACE_CODE(psynch_mutex_utrylock_failed, _TRACE_SUB_MUTEX, 0x1); @@ -161,4 +131,14 @@ TRACE_CODE(psynch_cvar_broadcast, _TRACE_SUB_CONDVAR, 0x5); TRACE_CODE(psynch_cvar_zeroed, _TRACE_SUB_CONDVAR, 0x6); TRACE_CODE(psynch_cvar_updateval, _TRACE_SUB_CONDVAR, 0x7); +TRACE_CODE(ulmutex_lock, _TRACE_SUB_ULMUTEX, 0x0); +TRACE_CODE(ulmutex_trylock, _TRACE_SUB_ULMUTEX, 0x1); +TRACE_CODE(ulmutex_lock_wait, _TRACE_SUB_ULMUTEX, 0x2); +TRACE_CODE(ulmutex_unlock, _TRACE_SUB_ULMUTEX, 0x3); +TRACE_CODE(ulmutex_unlock_wake, _TRACE_SUB_ULMUTEX, 0x4); +TRACE_CODE(ulmutex_unlock_steal, _TRACE_SUB_ULMUTEX, 0x5); + +TRACE_CODE(ulcond_wait, _TRACE_SUB_ULCOND, 0x0); +TRACE_CODE(ulcond_signal, _TRACE_SUB_ULCOND, 0x1); + #endif // _KERN_TRACE_H_ diff --git a/kern/workqueue_internal.h b/kern/workqueue_internal.h deleted file mode 100644 index c044fe7..0000000 --- a/kern/workqueue_internal.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2014 Apple Computer, Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -#ifndef _WORKQUEUE_INTERNAL_H_ -#define _WORKQUEUE_INTERNAL_H_ - -/* These definitions are shared between the kext and userspace inside the pthread project. Consolidating - * duplicate definitions that used to exist in both projects, when separate. - */ - -// Sometimes something gets passed a bucket number and we need a way to express -// that it's actually the event manager. Use the (0)th bucket for that. -#define WORKQ_THREAD_QOS_MIN (THREAD_QOS_MAINTENANCE) -#define WORKQ_THREAD_QOS_MAX (THREAD_QOS_LAST - 1) -#define WORKQ_THREAD_QOS_CLEANUP (THREAD_QOS_LEGACY) -#define WORKQ_THREAD_QOS_MANAGER (THREAD_QOS_LAST) // outside of MIN/MAX - -#define WORKQ_NUM_QOS_BUCKETS (WORKQ_THREAD_QOS_MAX) -#define WORKQ_NUM_BUCKETS (WORKQ_THREAD_QOS_MAX + 1) -#define WORKQ_IDX(qos) ((qos) - 1) // 0 based index - -// magical `nkevents` values for _pthread_wqthread -#define WORKQ_EXIT_THREAD_NKEVENT (-1) - -#endif // _WORKQUEUE_INTERNAL_H_ diff --git a/libpthread.xcodeproj/project.pbxproj b/libpthread.xcodeproj/project.pbxproj index 5c193fd..0c86611 100644 --- a/libpthread.xcodeproj/project.pbxproj +++ b/libpthread.xcodeproj/project.pbxproj @@ -83,10 +83,10 @@ /* Begin PBXBuildFile section */ 6E2A3BBE2101222F0003B53B /* stack_np.h in Headers */ = {isa = PBXBuildFile; fileRef = 6E2A3BBD210122230003B53B /* stack_np.h */; settings = {ATTRIBUTES = (Public, ); }; }; - 6E2A3BBF210122300003B53B /* stack_np.h in Headers */ = {isa = PBXBuildFile; fileRef = 6E2A3BBD210122230003B53B /* stack_np.h */; settings = {ATTRIBUTES = (Public, ); }; }; + 6E2A3BBF210122300003B53B /* stack_np.h in Headers */ = {isa = PBXBuildFile; fileRef = 6E2A3BBD210122230003B53B /* stack_np.h */; }; 6E2A3BC0210122340003B53B /* stack_np.h in Headers */ = {isa = PBXBuildFile; fileRef = 6E2A3BBD210122230003B53B /* stack_np.h */; }; 6E5869C720C9040A00F1CB75 /* dependency_private.h in Headers */ = {isa = PBXBuildFile; fileRef = 6E5869C620C8FE8300F1CB75 /* dependency_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; - 6E5869C820C9040B00F1CB75 /* dependency_private.h in Headers */ = {isa = PBXBuildFile; fileRef = 6E5869C620C8FE8300F1CB75 /* dependency_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; + 6E5869C820C9040B00F1CB75 /* dependency_private.h in Headers */ = {isa = PBXBuildFile; fileRef = 6E5869C620C8FE8300F1CB75 /* dependency_private.h */; }; 6E5869C920C9040C00F1CB75 /* dependency_private.h in Headers */ = {isa = PBXBuildFile; fileRef = 6E5869C620C8FE8300F1CB75 /* dependency_private.h */; }; 6E5869CB20C9043200F1CB75 /* pthread_dependency.c in Sources */ = {isa = PBXBuildFile; fileRef = 6E5869CA20C9043200F1CB75 /* pthread_dependency.c */; }; 6E5869CC20C9043B00F1CB75 /* pthread_dependency.c in Sources */ = {isa = PBXBuildFile; fileRef = 6E5869CA20C9043200F1CB75 /* pthread_dependency.c */; }; @@ -106,10 +106,6 @@ 6E8C165C1B14F08A00C8987C /* pthread_rwlock.c in Sources */ = {isa = PBXBuildFile; fileRef = C9A325F615B7513200270056 /* pthread_rwlock.c */; }; 6E8C165D1B14F08A00C8987C /* pthread_tsd.c in Sources */ = {isa = PBXBuildFile; fileRef = C9A325F815B7513200270056 /* pthread_tsd.c */; }; 6E8C165E1B14F08A00C8987C /* pthread_cancelable_cancel.c in Sources */ = {isa = PBXBuildFile; fileRef = C9A1BF5215C9A9F5006BB313 /* pthread_cancelable_cancel.c */; }; - 6E8C165F1B14F08A00C8987C /* pthread_cancelable_legacy.c in Sources */ = {isa = PBXBuildFile; fileRef = C9A1BF5415C9CB9D006BB313 /* pthread_cancelable_legacy.c */; }; - 6E8C16601B14F08A00C8987C /* pthread_cond_legacy.c in Sources */ = {isa = PBXBuildFile; fileRef = C975D5D615C9CECA0098ECD8 /* pthread_cond_legacy.c */; }; - 6E8C16611B14F08A00C8987C /* pthread_mutex_legacy.c in Sources */ = {isa = PBXBuildFile; fileRef = C975D5D815C9CEEA0098ECD8 /* pthread_mutex_legacy.c */; }; - 6E8C16621B14F08A00C8987C /* pthread_rwlock_legacy.c in Sources */ = {isa = PBXBuildFile; fileRef = C975D5DA15C9CEFA0098ECD8 /* pthread_rwlock_legacy.c */; }; 6E8C16651B14F08A00C8987C /* pthread_atfork.c in Sources */ = {isa = PBXBuildFile; fileRef = C90E7AB415DC40D900A06D48 /* pthread_atfork.c */; }; 6E8C16661B14F08A00C8987C /* pthread_asm.s in Sources */ = {isa = PBXBuildFile; fileRef = C99AD87D15DF04D10009A6F8 /* pthread_asm.s */; }; 6E8C16691B14F08A00C8987C /* qos.h in Headers */ = {isa = PBXBuildFile; fileRef = C9244C1A185FCFED00075748 /* qos.h */; }; @@ -129,6 +125,27 @@ 6EB232CC1B0EB2F0005915CE /* resolver.c in Sources */ = {isa = PBXBuildFile; fileRef = 6EB232C91B0EB29D005915CE /* resolver.c */; }; 6EB232CE1B0EB31B005915CE /* resolver.c in Sources */ = {isa = PBXBuildFile; fileRef = 6EB232C91B0EB29D005915CE /* resolver.c */; }; 6EB232D01B0EB325005915CE /* resolver.c in Sources */ = {isa = PBXBuildFile; fileRef = 6EB232C91B0EB29D005915CE /* resolver.c */; }; + 6ED6E65522BFCD1200CE82C2 /* internal.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F315B7513200270056 /* internal.h */; }; + 6ED6E65622BFCD1200CE82C2 /* types_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E63F22BFC64700CE82C2 /* types_internal.h */; }; + 6ED6E65722BFCD1200CE82C2 /* offsets_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6E514A0220B67C0900844EE1 /* offsets_internal.h */; }; + 6ED6E65822BFCD1300CE82C2 /* internal.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F315B7513200270056 /* internal.h */; }; + 6ED6E65922BFCD1300CE82C2 /* types_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E63F22BFC64700CE82C2 /* types_internal.h */; }; + 6ED6E65A22BFCD1300CE82C2 /* offsets_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6E514A0220B67C0900844EE1 /* offsets_internal.h */; }; + 6ED6E65B22BFCD1400CE82C2 /* internal.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F315B7513200270056 /* internal.h */; }; + 6ED6E65C22BFCD1400CE82C2 /* types_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E63F22BFC64700CE82C2 /* types_internal.h */; }; + 6ED6E65D22BFCD1400CE82C2 /* offsets_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6E514A0220B67C0900844EE1 /* offsets_internal.h */; }; + 6ED6E66022BFF81D00CE82C2 /* exports_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E65F22BFF5A500CE82C2 /* exports_internal.h */; }; + 6ED6E66122BFF81D00CE82C2 /* inline_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E65E22BFF03C00CE82C2 /* inline_internal.h */; }; + 6ED6E66222BFF81E00CE82C2 /* exports_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E65F22BFF5A500CE82C2 /* exports_internal.h */; }; + 6ED6E66322BFF81E00CE82C2 /* inline_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E65E22BFF03C00CE82C2 /* inline_internal.h */; }; + 6ED6E66422BFF81F00CE82C2 /* exports_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E65F22BFF5A500CE82C2 /* exports_internal.h */; }; + 6ED6E66522BFF81F00CE82C2 /* inline_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E65E22BFF03C00CE82C2 /* inline_internal.h */; }; + 6ED6E66D22BFFC3700CE82C2 /* imports_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E66C22BFFC3200CE82C2 /* imports_internal.h */; }; + 6ED6E66E22BFFC3700CE82C2 /* imports_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E66C22BFFC3200CE82C2 /* imports_internal.h */; }; + 6ED6E66F22BFFC3800CE82C2 /* imports_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E66C22BFFC3200CE82C2 /* imports_internal.h */; }; + 6ED6E67422C0034300CE82C2 /* prototypes_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E67322C0033B00CE82C2 /* prototypes_internal.h */; }; + 6ED6E67522C0034300CE82C2 /* prototypes_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E67322C0033B00CE82C2 /* prototypes_internal.h */; }; + 6ED6E67622C0034400CE82C2 /* prototypes_internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 6ED6E67322C0033B00CE82C2 /* prototypes_internal.h */; }; 74E594931613AAF4006C417B /* pthread.c in Sources */ = {isa = PBXBuildFile; fileRef = C9A325FA15B7513200270056 /* pthread.c */; }; 74E594941613AAF4006C417B /* pthread_cancelable.c in Sources */ = {isa = PBXBuildFile; fileRef = C9A325F115B7513200270056 /* pthread_cancelable.c */; }; 74E594951613AAF4006C417B /* pthread_cond.c in Sources */ = {isa = PBXBuildFile; fileRef = C9A325F215B7513200270056 /* pthread_cond.c */; }; @@ -170,9 +187,6 @@ C9244C1D1860D8EF00075748 /* qos.c in Sources */ = {isa = PBXBuildFile; fileRef = C9244C1C1860D8EF00075748 /* qos.c */; }; C9244C1E1860D96D00075748 /* qos.c in Sources */ = {isa = PBXBuildFile; fileRef = C9244C1C1860D8EF00075748 /* qos.c */; }; C9244C1F1860D96E00075748 /* qos.c in Sources */ = {isa = PBXBuildFile; fileRef = C9244C1C1860D8EF00075748 /* qos.c */; }; - C975D5D715C9CECA0098ECD8 /* pthread_cond_legacy.c in Sources */ = {isa = PBXBuildFile; fileRef = C975D5D615C9CECA0098ECD8 /* pthread_cond_legacy.c */; }; - C975D5D915C9CEEA0098ECD8 /* pthread_mutex_legacy.c in Sources */ = {isa = PBXBuildFile; fileRef = C975D5D815C9CEEA0098ECD8 /* pthread_mutex_legacy.c */; }; - C975D5DB15C9CEFA0098ECD8 /* pthread_rwlock_legacy.c in Sources */ = {isa = PBXBuildFile; fileRef = C975D5DA15C9CEFA0098ECD8 /* pthread_rwlock_legacy.c */; }; C98C95D918FF1F4E005654FB /* spawn.h in Headers */ = {isa = PBXBuildFile; fileRef = C98C95D818FF1F4E005654FB /* spawn.h */; settings = {ATTRIBUTES = (Public, ); }; }; C99AD87B15DEC4BC0009A6F8 /* posix_sched.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F015B7513200270056 /* posix_sched.h */; settings = {ATTRIBUTES = (Private, ); }; }; C99AD87C15DEC5290009A6F8 /* spinlock_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F715B7513200270056 /* spinlock_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; @@ -188,7 +202,6 @@ C9A1BF4F15C9A598006BB313 /* pthread_spis.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A3260015B7513700270056 /* pthread_spis.h */; settings = {ATTRIBUTES = (Public, ); }; }; C9A1BF5015C9A59B006BB313 /* sched.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A3260115B7513700270056 /* sched.h */; settings = {ATTRIBUTES = (Public, ); }; }; C9A1BF5315C9A9F5006BB313 /* pthread_cancelable_cancel.c in Sources */ = {isa = PBXBuildFile; fileRef = C9A1BF5215C9A9F5006BB313 /* pthread_cancelable_cancel.c */; }; - C9A1BF5515C9CB9D006BB313 /* pthread_cancelable_legacy.c in Sources */ = {isa = PBXBuildFile; fileRef = C9A1BF5415C9CB9D006BB313 /* pthread_cancelable_legacy.c */; }; C9BB478B15E6ABD900F135B7 /* workqueue_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F915B7513200270056 /* workqueue_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; C9BB478D15E6ADF700F135B7 /* tsd_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F415B7513200270056 /* tsd_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; C9CCFB9D18B6D0910060CAAE /* qos_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C99B17DA189C2E1B00991D38 /* qos_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; @@ -230,27 +243,23 @@ E4F449B11E82D03500A7FB9A /* pthread_cwd.c in Sources */ = {isa = PBXBuildFile; fileRef = 924D8EDE1C11832A002AC2BC /* pthread_cwd.c */; }; E4F449B21E82D03500A7FB9A /* pthread_tsd.c in Sources */ = {isa = PBXBuildFile; fileRef = C9A325F815B7513200270056 /* pthread_tsd.c */; }; E4F449B31E82D03500A7FB9A /* pthread_cancelable_cancel.c in Sources */ = {isa = PBXBuildFile; fileRef = C9A1BF5215C9A9F5006BB313 /* pthread_cancelable_cancel.c */; }; - E4F449B41E82D03500A7FB9A /* pthread_cancelable_legacy.c in Sources */ = {isa = PBXBuildFile; fileRef = C9A1BF5415C9CB9D006BB313 /* pthread_cancelable_legacy.c */; }; - E4F449B51E82D03500A7FB9A /* pthread_cond_legacy.c in Sources */ = {isa = PBXBuildFile; fileRef = C975D5D615C9CECA0098ECD8 /* pthread_cond_legacy.c */; }; - E4F449B61E82D03500A7FB9A /* pthread_mutex_legacy.c in Sources */ = {isa = PBXBuildFile; fileRef = C975D5D815C9CEEA0098ECD8 /* pthread_mutex_legacy.c */; }; - E4F449B71E82D03500A7FB9A /* pthread_rwlock_legacy.c in Sources */ = {isa = PBXBuildFile; fileRef = C975D5DA15C9CEFA0098ECD8 /* pthread_rwlock_legacy.c */; }; E4F449BA1E82D03500A7FB9A /* pthread_atfork.c in Sources */ = {isa = PBXBuildFile; fileRef = C90E7AB415DC40D900A06D48 /* pthread_atfork.c */; }; E4F449BB1E82D03500A7FB9A /* pthread_asm.s in Sources */ = {isa = PBXBuildFile; fileRef = C99AD87D15DF04D10009A6F8 /* pthread_asm.s */; }; - E4F449BE1E82D03500A7FB9A /* qos.h in Headers */ = {isa = PBXBuildFile; fileRef = C9244C1A185FCFED00075748 /* qos.h */; settings = {ATTRIBUTES = (Public, ); }; }; - E4F449BF1E82D03500A7FB9A /* pthread.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325FE15B7513700270056 /* pthread.h */; settings = {ATTRIBUTES = (Public, ); }; }; - E4F449C01E82D03500A7FB9A /* pthread_impl.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325FF15B7513700270056 /* pthread_impl.h */; settings = {ATTRIBUTES = (Public, ); }; }; - E4F449C11E82D03500A7FB9A /* qos.h in Headers */ = {isa = PBXBuildFile; fileRef = E4063CF21906B4FB000202F9 /* qos.h */; settings = {ATTRIBUTES = (Private, ); }; }; - E4F449C21E82D03500A7FB9A /* pthread_spis.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A3260015B7513700270056 /* pthread_spis.h */; settings = {ATTRIBUTES = (Public, ); }; }; - E4F449C31E82D03500A7FB9A /* introspection.h in Headers */ = {isa = PBXBuildFile; fileRef = 9202B2301D1A5B3F00945880 /* introspection.h */; settings = {ATTRIBUTES = (Public, ); }; }; - E4F449C41E82D03500A7FB9A /* sched.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A3260115B7513700270056 /* sched.h */; settings = {ATTRIBUTES = (Public, ); }; }; - E4F449C51E82D03500A7FB9A /* introspection_private.h in Headers */ = {isa = PBXBuildFile; fileRef = E4657D4017284F7B007D1847 /* introspection_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; - E4F449C61E82D03500A7FB9A /* tsd_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F415B7513200270056 /* tsd_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; - E4F449C71E82D03500A7FB9A /* posix_sched.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F015B7513200270056 /* posix_sched.h */; settings = {ATTRIBUTES = (Private, ); }; }; - E4F449C81E82D03500A7FB9A /* qos_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C99B17DA189C2E1B00991D38 /* qos_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; - E4F449C91E82D03500A7FB9A /* spawn.h in Headers */ = {isa = PBXBuildFile; fileRef = C98C95D818FF1F4E005654FB /* spawn.h */; settings = {ATTRIBUTES = (Public, ); }; }; - E4F449CA1E82D03500A7FB9A /* spinlock_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F715B7513200270056 /* spinlock_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; - E4F449CB1E82D03500A7FB9A /* workqueue_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F915B7513200270056 /* workqueue_private.h */; settings = {ATTRIBUTES = (Private, ); }; }; - E4F449CC1E82D03500A7FB9A /* private.h in Headers */ = {isa = PBXBuildFile; fileRef = C9153095167ACC22006BB094 /* private.h */; settings = {ATTRIBUTES = (Private, ); }; }; + E4F449BE1E82D03500A7FB9A /* qos.h in Headers */ = {isa = PBXBuildFile; fileRef = C9244C1A185FCFED00075748 /* qos.h */; }; + E4F449BF1E82D03500A7FB9A /* pthread.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325FE15B7513700270056 /* pthread.h */; }; + E4F449C01E82D03500A7FB9A /* pthread_impl.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325FF15B7513700270056 /* pthread_impl.h */; }; + E4F449C11E82D03500A7FB9A /* qos.h in Headers */ = {isa = PBXBuildFile; fileRef = E4063CF21906B4FB000202F9 /* qos.h */; }; + E4F449C21E82D03500A7FB9A /* pthread_spis.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A3260015B7513700270056 /* pthread_spis.h */; }; + E4F449C31E82D03500A7FB9A /* introspection.h in Headers */ = {isa = PBXBuildFile; fileRef = 9202B2301D1A5B3F00945880 /* introspection.h */; }; + E4F449C41E82D03500A7FB9A /* sched.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A3260115B7513700270056 /* sched.h */; }; + E4F449C51E82D03500A7FB9A /* introspection_private.h in Headers */ = {isa = PBXBuildFile; fileRef = E4657D4017284F7B007D1847 /* introspection_private.h */; }; + E4F449C61E82D03500A7FB9A /* tsd_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F415B7513200270056 /* tsd_private.h */; }; + E4F449C71E82D03500A7FB9A /* posix_sched.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F015B7513200270056 /* posix_sched.h */; }; + E4F449C81E82D03500A7FB9A /* qos_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C99B17DA189C2E1B00991D38 /* qos_private.h */; }; + E4F449C91E82D03500A7FB9A /* spawn.h in Headers */ = {isa = PBXBuildFile; fileRef = C98C95D818FF1F4E005654FB /* spawn.h */; }; + E4F449CA1E82D03500A7FB9A /* spinlock_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F715B7513200270056 /* spinlock_private.h */; }; + E4F449CB1E82D03500A7FB9A /* workqueue_private.h in Headers */ = {isa = PBXBuildFile; fileRef = C9A325F915B7513200270056 /* workqueue_private.h */; }; + E4F449CC1E82D03500A7FB9A /* private.h in Headers */ = {isa = PBXBuildFile; fileRef = C9153095167ACC22006BB094 /* private.h */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -356,13 +365,19 @@ /* Begin PBXFileReference section */ 6E2A3BBD210122230003B53B /* stack_np.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = stack_np.h; sourceTree = ""; }; - 6E514A0220B67C0900844EE1 /* offsets.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = offsets.h; sourceTree = ""; }; + 6E39974822D3C82C0057625B /* pthread_driverkit.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = pthread_driverkit.xcconfig; sourceTree = ""; }; + 6E514A0220B67C0900844EE1 /* offsets_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = offsets_internal.h; sourceTree = ""; }; 6E5869C620C8FE8300F1CB75 /* dependency_private.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = dependency_private.h; sourceTree = ""; }; 6E5869CA20C9043200F1CB75 /* pthread_dependency.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = pthread_dependency.c; sourceTree = ""; }; 6E8C16801B14F08A00C8987C /* libsystem_pthread.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; includeInIndex = 0; path = libsystem_pthread.dylib; sourceTree = BUILT_PRODUCTS_DIR; }; 6E8C16851B14F14000C8987C /* pthread_introspection.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = pthread_introspection.xcconfig; sourceTree = ""; }; 6EB232C91B0EB29D005915CE /* resolver.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = resolver.c; sourceTree = ""; }; 6EB232CA1B0EB29D005915CE /* resolver.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = resolver.h; sourceTree = ""; }; + 6ED6E63F22BFC64700CE82C2 /* types_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = types_internal.h; sourceTree = ""; }; + 6ED6E65E22BFF03C00CE82C2 /* inline_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = inline_internal.h; sourceTree = ""; }; + 6ED6E65F22BFF5A500CE82C2 /* exports_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = exports_internal.h; sourceTree = ""; }; + 6ED6E66C22BFFC3200CE82C2 /* imports_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = imports_internal.h; sourceTree = ""; }; + 6ED6E67322C0033B00CE82C2 /* prototypes_internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = prototypes_internal.h; sourceTree = ""; }; 74E594A41613AAF4006C417B /* libpthread_eOS.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libpthread_eOS.a; sourceTree = BUILT_PRODUCTS_DIR; }; 9202B2301D1A5B3F00945880 /* introspection.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = introspection.h; sourceTree = ""; }; 9235CA551CA48D010015C92B /* kext_development.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = kext_development.xcconfig; sourceTree = ""; }; @@ -431,19 +446,14 @@ C948FCEF15D187FA00180BF5 /* pthread_setcancelstate.3 */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = pthread_setcancelstate.3; sourceTree = ""; }; C948FCF015D187FA00180BF5 /* pthread_setspecific.3 */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = pthread_setspecific.3; sourceTree = ""; }; C948FCF115D187FA00180BF5 /* pthread.3 */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = pthread.3; sourceTree = ""; }; - C975D5D615C9CECA0098ECD8 /* pthread_cond_legacy.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pthread_cond_legacy.c; sourceTree = ""; }; - C975D5D815C9CEEA0098ECD8 /* pthread_mutex_legacy.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pthread_mutex_legacy.c; sourceTree = ""; }; - C975D5DA15C9CEFA0098ECD8 /* pthread_rwlock_legacy.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pthread_rwlock_legacy.c; sourceTree = ""; }; C979E9FB18A1BC2A000951E5 /* kern_trace.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = kern_trace.h; sourceTree = ""; }; C979E9FC18A2BF2C000951E5 /* install-codes.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = "install-codes.sh"; sourceTree = ""; }; - C98005141899BD2000368E4D /* workqueue_internal.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = workqueue_internal.h; sourceTree = ""; }; C98C95D818FF1F4E005654FB /* spawn.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = spawn.h; sourceTree = ""; }; C99AD87D15DF04D10009A6F8 /* pthread_asm.s */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.asm; path = pthread_asm.s; sourceTree = ""; }; C99B17DA189C2E1B00991D38 /* qos_private.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = qos_private.h; sourceTree = ""; }; C99EA612161F8288003EBC56 /* eos.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = eos.xcconfig; sourceTree = ""; }; C9A1998A1C8E271F00CE102A /* tests */ = {isa = PBXFileReference; lastKnownFileType = folder; path = tests; sourceTree = ""; }; C9A1BF5215C9A9F5006BB313 /* pthread_cancelable_cancel.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pthread_cancelable_cancel.c; sourceTree = ""; }; - C9A1BF5415C9CB9D006BB313 /* pthread_cancelable_legacy.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = pthread_cancelable_legacy.c; sourceTree = ""; }; C9A325E215B7347000270056 /* libsystem_pthread.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; includeInIndex = 0; path = libsystem_pthread.dylib; sourceTree = BUILT_PRODUCTS_DIR; }; C9A325EF15B7513200270056 /* plockstat.d */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.dtrace; path = plockstat.d; sourceTree = ""; }; C9A325F015B7513200270056 /* posix_sched.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = posix_sched.h; sourceTree = ""; }; @@ -549,6 +559,39 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ + 6E8C8C2E22BDC9DE00136359 /* pthread */ = { + isa = PBXGroup; + children = ( + 6E5869C620C8FE8300F1CB75 /* dependency_private.h */, + E4657D4017284F7B007D1847 /* introspection_private.h */, + C9A325F015B7513200270056 /* posix_sched.h */, + C9153095167ACC22006BB094 /* private.h */, + C99B17DA189C2E1B00991D38 /* qos_private.h */, + E4063CF21906B4FB000202F9 /* qos.h */, + C9A325F715B7513200270056 /* spinlock_private.h */, + C9A325F415B7513200270056 /* tsd_private.h */, + C9A325F915B7513200270056 /* workqueue_private.h */, + ); + path = pthread; + sourceTree = ""; + }; + 6E8C8C2F22BDCD4600136359 /* include */ = { + isa = PBXGroup; + children = ( + C9A325FD15B7513700270056 /* pthread */, + FC5A372217CEB3D6008C323E /* sys */, + ); + path = include; + sourceTree = ""; + }; + 6E8C8C3022BDD05700136359 /* sys */ = { + isa = PBXGroup; + children = ( + A98FE72D19479F7C007718DA /* qos_private.h */, + ); + path = sys; + sourceTree = ""; + }; 9240BF331AA669EB003C99B4 /* tools */ = { isa = PBXGroup; children = ( @@ -568,7 +611,6 @@ C9169DDC1603DE84005A2F8C /* kern_support.c */, C979E9FB18A1BC2A000951E5 /* kern_trace.h */, FC30E28D16A747AD00A25B5F /* synch_internal.h */, - C98005141899BD2000368E4D /* workqueue_internal.h */, ); path = kern; sourceTree = ""; @@ -635,10 +677,6 @@ isa = PBXGroup; children = ( C9A1BF5215C9A9F5006BB313 /* pthread_cancelable_cancel.c */, - C9A1BF5415C9CB9D006BB313 /* pthread_cancelable_legacy.c */, - C975D5D615C9CECA0098ECD8 /* pthread_cond_legacy.c */, - C975D5D815C9CEEA0098ECD8 /* pthread_mutex_legacy.c */, - C975D5DA15C9CEFA0098ECD8 /* pthread_rwlock_legacy.c */, ); path = variants; sourceTree = ""; @@ -646,14 +684,13 @@ C9A325D715B7347000270056 = { isa = PBXGroup; children = ( - C9169DD91603DE68005A2F8C /* kern */, - C9A960B218452B0700AE10C8 /* lldbmacros */, - C9A325FD15B7513700270056 /* pthread */, - FC5A372217CEB3D6008C323E /* sys */, + 6E8C8C2F22BDCD4600136359 /* include */, C9D70EBE167AC7D100D52713 /* private */, - C948FCC015D187AD00180BF5 /* man */, + C9169DD91603DE68005A2F8C /* kern */, C9A325ED15B74FB600270056 /* src */, C9A1998A1C8E271F00CE102A /* tests */, + C9A960B218452B0700AE10C8 /* lldbmacros */, + C948FCC015D187AD00180BF5 /* man */, 9240BF331AA669EB003C99B4 /* tools */, C9A3260B15B759A100270056 /* xcodescripts */, C9CA27DA1602813000259F78 /* Frameworks */, @@ -681,8 +718,13 @@ C9A325ED15B74FB600270056 /* src */ = { isa = PBXGroup; children = ( - 6E514A0220B67C0900844EE1 /* offsets.h */, C9A325F315B7513200270056 /* internal.h */, + 6ED6E63F22BFC64700CE82C2 /* types_internal.h */, + 6E514A0220B67C0900844EE1 /* offsets_internal.h */, + 6ED6E66C22BFFC3200CE82C2 /* imports_internal.h */, + 6ED6E67322C0033B00CE82C2 /* prototypes_internal.h */, + 6ED6E65F22BFF5A500CE82C2 /* exports_internal.h */, + 6ED6E65E22BFF03C00CE82C2 /* inline_internal.h */, C9A325EF15B7513200270056 /* plockstat.d */, C9A325FA15B7513200270056 /* pthread.c */, C99AD87D15DF04D10009A6F8 /* pthread_asm.s */, @@ -705,13 +747,13 @@ isa = PBXGroup; children = ( 9202B2301D1A5B3F00945880 /* introspection.h */, - C9A325FE15B7513700270056 /* pthread.h */, C9A325FF15B7513700270056 /* pthread_impl.h */, C9A3260015B7513700270056 /* pthread_spis.h */, + C9A325FE15B7513700270056 /* pthread.h */, + C9244C1A185FCFED00075748 /* qos.h */, C9A3260115B7513700270056 /* sched.h */, C98C95D818FF1F4E005654FB /* spawn.h */, 6E2A3BBD210122230003B53B /* stack_np.h */, - C9244C1A185FCFED00075748 /* qos.h */, ); path = pthread; sourceTree = ""; @@ -724,6 +766,7 @@ C9A3260C15B759B600270056 /* pthread.xcconfig */, E4F449A31E82CF0100A7FB9A /* resolver.xcconfig */, E41505E81E818D4D00F243FB /* resolved.xcconfig */, + 6E39974822D3C82C0057625B /* pthread_driverkit.xcconfig */, 6E8C16851B14F14000C8987C /* pthread_introspection.xcconfig */, C99EA612161F8288003EBC56 /* eos.xcconfig */, C04545B91C584F8B006A53B3 /* static.xcconfig */, @@ -765,15 +808,8 @@ C9D70EBE167AC7D100D52713 /* private */ = { isa = PBXGroup; children = ( - C9153095167ACC22006BB094 /* private.h */, - C9A325F015B7513200270056 /* posix_sched.h */, - C9A325F715B7513200270056 /* spinlock_private.h */, - C9A325F415B7513200270056 /* tsd_private.h */, - C9A325F915B7513200270056 /* workqueue_private.h */, - E4657D4017284F7B007D1847 /* introspection_private.h */, - C99B17DA189C2E1B00991D38 /* qos_private.h */, - E4063CF21906B4FB000202F9 /* qos.h */, - 6E5869C620C8FE8300F1CB75 /* dependency_private.h */, + 6E8C8C2E22BDC9DE00136359 /* pthread */, + 6E8C8C3022BDD05700136359 /* sys */, ); path = private; sourceTree = ""; @@ -791,9 +827,8 @@ FC5A372217CEB3D6008C323E /* sys */ = { isa = PBXGroup; children = ( - E4D962F919086AD600E8A9F2 /* qos.h */, FC5A372317CEB3D6008C323E /* _pthread */, - A98FE72D19479F7C007718DA /* qos_private.h */, + E4D962F919086AD600E8A9F2 /* qos.h */, ); path = sys; sourceTree = ""; @@ -823,20 +858,27 @@ isa = PBXHeadersBuildPhase; buildActionMask = 2147483647; files = ( + 6ED6E67622C0034400CE82C2 /* prototypes_internal.h in Headers */, + 6ED6E66422BFF81F00CE82C2 /* exports_internal.h in Headers */, 6E2A3BC0210122340003B53B /* stack_np.h in Headers */, 6E8C16711B14F08A00C8987C /* posix_sched.h in Headers */, 6E8C166F1B14F08A00C8987C /* introspection_private.h in Headers */, + 6ED6E65522BFCD1200CE82C2 /* internal.h in Headers */, E41A64AE1E83C470009479A9 /* introspection.h in Headers */, 6E8C166C1B14F08A00C8987C /* qos.h in Headers */, + 6ED6E65722BFCD1200CE82C2 /* offsets_internal.h in Headers */, + 6ED6E65622BFCD1200CE82C2 /* types_internal.h in Headers */, 6E8C16701B14F08A00C8987C /* tsd_private.h in Headers */, 6E8C16731B14F08A00C8987C /* qos_private.h in Headers */, 6E8C16761B14F08A00C8987C /* spinlock_private.h in Headers */, 6E8C16771B14F08A00C8987C /* workqueue_private.h in Headers */, 6E8C16781B14F08A00C8987C /* private.h in Headers */, + 6ED6E66522BFF81F00CE82C2 /* inline_internal.h in Headers */, 6E8C16691B14F08A00C8987C /* qos.h in Headers */, 6E8C166A1B14F08A00C8987C /* pthread.h in Headers */, 6E8C166B1B14F08A00C8987C /* pthread_impl.h in Headers */, 6E8C166D1B14F08A00C8987C /* pthread_spis.h in Headers */, + 6ED6E66F22BFFC3800CE82C2 /* imports_internal.h in Headers */, 6E8C166E1B14F08A00C8987C /* sched.h in Headers */, 6E5869C920C9040C00F1CB75 /* dependency_private.h in Headers */, 6E8C16751B14F08A00C8987C /* spawn.h in Headers */, @@ -868,20 +910,27 @@ isa = PBXHeadersBuildPhase; buildActionMask = 2147483647; files = ( + 6ED6E67422C0034300CE82C2 /* prototypes_internal.h in Headers */, + 6ED6E66022BFF81D00CE82C2 /* exports_internal.h in Headers */, 6E2A3BBE2101222F0003B53B /* stack_np.h in Headers */, C9244C1B185FD33000075748 /* qos.h in Headers */, C9A1BF4D15C9A58E006BB313 /* pthread.h in Headers */, + 6ED6E65B22BFCD1400CE82C2 /* internal.h in Headers */, C9A1BF4E15C9A594006BB313 /* pthread_impl.h in Headers */, E4063CF31906B75A000202F9 /* qos.h in Headers */, + 6ED6E65D22BFCD1400CE82C2 /* offsets_internal.h in Headers */, + 6ED6E65C22BFCD1400CE82C2 /* types_internal.h in Headers */, C9A1BF4F15C9A598006BB313 /* pthread_spis.h in Headers */, 9202B2321D1A5B6200945880 /* introspection.h in Headers */, C9A1BF5015C9A59B006BB313 /* sched.h in Headers */, E4657D4117284F7B007D1847 /* introspection_private.h in Headers */, C9BB478D15E6ADF700F135B7 /* tsd_private.h in Headers */, + 6ED6E66122BFF81D00CE82C2 /* inline_internal.h in Headers */, C99AD87B15DEC4BC0009A6F8 /* posix_sched.h in Headers */, C9CCFB9D18B6D0910060CAAE /* qos_private.h in Headers */, C98C95D918FF1F4E005654FB /* spawn.h in Headers */, C99AD87C15DEC5290009A6F8 /* spinlock_private.h in Headers */, + 6ED6E66D22BFFC3700CE82C2 /* imports_internal.h in Headers */, C9BB478B15E6ABD900F135B7 /* workqueue_private.h in Headers */, 6E5869C720C9040A00F1CB75 /* dependency_private.h in Headers */, C9153096167ACC2B006BB094 /* private.h in Headers */, @@ -899,20 +948,27 @@ isa = PBXHeadersBuildPhase; buildActionMask = 2147483647; files = ( + 6ED6E67522C0034300CE82C2 /* prototypes_internal.h in Headers */, + 6ED6E66222BFF81E00CE82C2 /* exports_internal.h in Headers */, 6E2A3BBF210122300003B53B /* stack_np.h in Headers */, E4F449BE1E82D03500A7FB9A /* qos.h in Headers */, E4F449BF1E82D03500A7FB9A /* pthread.h in Headers */, + 6ED6E65822BFCD1300CE82C2 /* internal.h in Headers */, E4F449C01E82D03500A7FB9A /* pthread_impl.h in Headers */, E4F449C11E82D03500A7FB9A /* qos.h in Headers */, + 6ED6E65A22BFCD1300CE82C2 /* offsets_internal.h in Headers */, + 6ED6E65922BFCD1300CE82C2 /* types_internal.h in Headers */, E4F449C21E82D03500A7FB9A /* pthread_spis.h in Headers */, E4F449C31E82D03500A7FB9A /* introspection.h in Headers */, E4F449C41E82D03500A7FB9A /* sched.h in Headers */, E4F449C51E82D03500A7FB9A /* introspection_private.h in Headers */, E4F449C61E82D03500A7FB9A /* tsd_private.h in Headers */, + 6ED6E66322BFF81E00CE82C2 /* inline_internal.h in Headers */, E4F449C71E82D03500A7FB9A /* posix_sched.h in Headers */, E4F449C81E82D03500A7FB9A /* qos_private.h in Headers */, E4F449C91E82D03500A7FB9A /* spawn.h in Headers */, E4F449CA1E82D03500A7FB9A /* spinlock_private.h in Headers */, + 6ED6E66E22BFFC3700CE82C2 /* imports_internal.h in Headers */, E4F449CB1E82D03500A7FB9A /* workqueue_private.h in Headers */, 6E5869C820C9040B00F1CB75 /* dependency_private.h in Headers */, E4F449CC1E82D03500A7FB9A /* private.h in Headers */, @@ -1110,6 +1166,7 @@ C9A325D915B7347000270056 /* Project object */ = { isa = PBXProject; attributes = { + DefaultBuildSystemTypeForWorkspace = Latest; LastUpgradeCheck = 1100; ORGANIZATIONNAME = ""; TargetAttributes = { @@ -1126,6 +1183,7 @@ developmentRegion = English; hasScannedForEncodings = 0; knownRegions = ( + English, en, ); mainGroup = C9A325D715B7347000270056; @@ -1195,7 +1253,7 @@ ); runOnlyForDeploymentPostprocessing = 1; shellPath = "/bin/bash -e -x"; - shellScript = ". \"${SCRIPT_INPUT_FILE_0}\" /bin/ln -sf \"../../../..${INSTALL_PATH}/libpthread.a\" \"${DSTROOT}/usr/local/lib/loaderd/libpthread.a\""; + shellScript = ". \"${SCRIPT_INPUT_FILE_0}\" /bin/ln -sf \"../../../..${INSTALL_PATH}/libpthread.a\" \"${DSTROOT}/usr/local/lib/loaderd/libpthread.a\"\n"; showEnvVarsInLog = 0; }; C979E9FD18A2BF3D000951E5 /* Install Codes file */ = { @@ -1391,10 +1449,6 @@ 6E8C165C1B14F08A00C8987C /* pthread_rwlock.c in Sources */, 6E8C165D1B14F08A00C8987C /* pthread_tsd.c in Sources */, 6E8C165E1B14F08A00C8987C /* pthread_cancelable_cancel.c in Sources */, - 6E8C165F1B14F08A00C8987C /* pthread_cancelable_legacy.c in Sources */, - 6E8C16601B14F08A00C8987C /* pthread_cond_legacy.c in Sources */, - 6E8C16611B14F08A00C8987C /* pthread_mutex_legacy.c in Sources */, - 6E8C16621B14F08A00C8987C /* pthread_rwlock_legacy.c in Sources */, 6E8C16651B14F08A00C8987C /* pthread_atfork.c in Sources */, 6E5869CD20C9043B00F1CB75 /* pthread_dependency.c in Sources */, 6E8C16661B14F08A00C8987C /* pthread_asm.s in Sources */, @@ -1473,10 +1527,6 @@ 924D8EE11C11833E002AC2BC /* pthread_cwd.c in Sources */, C9A1BF4C15C9A578006BB313 /* pthread_tsd.c in Sources */, C9A1BF5315C9A9F5006BB313 /* pthread_cancelable_cancel.c in Sources */, - C9A1BF5515C9CB9D006BB313 /* pthread_cancelable_legacy.c in Sources */, - C975D5D715C9CECA0098ECD8 /* pthread_cond_legacy.c in Sources */, - C975D5D915C9CEEA0098ECD8 /* pthread_mutex_legacy.c in Sources */, - C975D5DB15C9CEFA0098ECD8 /* pthread_rwlock_legacy.c in Sources */, C90E7AB815DC40D900A06D48 /* pthread_atfork.c in Sources */, 6E5869CB20C9043200F1CB75 /* pthread_dependency.c in Sources */, C99AD88015E2D8B50009A6F8 /* pthread_asm.s in Sources */, @@ -1547,10 +1597,6 @@ E4F449B11E82D03500A7FB9A /* pthread_cwd.c in Sources */, E4F449B21E82D03500A7FB9A /* pthread_tsd.c in Sources */, E4F449B31E82D03500A7FB9A /* pthread_cancelable_cancel.c in Sources */, - E4F449B41E82D03500A7FB9A /* pthread_cancelable_legacy.c in Sources */, - E4F449B51E82D03500A7FB9A /* pthread_cond_legacy.c in Sources */, - E4F449B61E82D03500A7FB9A /* pthread_mutex_legacy.c in Sources */, - E4F449B71E82D03500A7FB9A /* pthread_rwlock_legacy.c in Sources */, E4F449BA1E82D03500A7FB9A /* pthread_atfork.c in Sources */, 6E5869CC20C9043B00F1CB75 /* pthread_dependency.c in Sources */, E4F449BB1E82D03500A7FB9A /* pthread_asm.s in Sources */, @@ -1675,7 +1721,6 @@ 9235CA4D1CA48CEA0015C92B /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { - SUPPORTED_PLATFORMS = "macosx iphoneos appletvos watchos"; }; name = Debug; }; @@ -1731,23 +1776,18 @@ 9235CA541CA48CEA0015C92B /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { - SDKROOT = macosx.internal; - SUPPORTED_PLATFORMS = "iphoneos macosx watchos appletvos"; }; name = Debug; }; 92799B461B96A5FE00861404 /* Release */ = { isa = XCBuildConfiguration; buildSettings = { - SUPPORTED_PLATFORMS = "macosx iphoneos appletvos watchos"; }; name = Release; }; 92B275F21BCE4C5E007D06D7 /* Release */ = { isa = XCBuildConfiguration; buildSettings = { - SDKROOT = macosx.internal; - SUPPORTED_PLATFORMS = "iphoneos macosx watchos appletvos"; }; name = Release; }; @@ -1841,6 +1881,7 @@ }; E4B7FCB022000AF50010A840 /* Release */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 6E39974822D3C82C0057625B /* pthread_driverkit.xcconfig */; buildSettings = { PRODUCT_NAME = "$(TARGET_NAME)"; }; @@ -1848,6 +1889,7 @@ }; E4B7FCB122000AF50010A840 /* Debug */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 6E39974822D3C82C0057625B /* pthread_driverkit.xcconfig */; buildSettings = { PRODUCT_NAME = "$(TARGET_NAME)"; }; diff --git a/man/pthread_jit_write_protect_np.3 b/man/pthread_jit_write_protect_np.3 new file mode 100644 index 0000000..44ac5d3 --- /dev/null +++ b/man/pthread_jit_write_protect_np.3 @@ -0,0 +1,71 @@ +.\" Copyright (c) 2020 Apple Inc. All rights reserved. +.\" +.\" @APPLE_LICENSE_HEADER_START@ +.\" +.\" This file contains Original Code and/or Modifications of Original Code +.\" as defined in and that are subject to the Apple Public Source License +.\" Version 2.0 (the 'License'). You may not use this file except in +.\" compliance with the License. Please obtain a copy of the License at +.\" http://www.opensource.apple.com/apsl/ and read it before using this +.\" file. +.\" +.\" The Original Code and all software distributed under the License are +.\" distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER +.\" EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, +.\" INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, +.\" FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. +.\" Please see the License for the specific language governing rights and +.\" limitations under the License. +.\" +.\" @APPLE_LICENSE_HEADER_END@ +.\" +.Dd May 1, 2020 +.Dt PTHREAD_JIT_WRITE_PROTECT_NP 3 +.Os +.Sh NAME +.Nm pthread_jit_write_protect_supported_np , +.Nm pthread_jit_write_protect_np +.Nd thread JIT region write protection settings +.Sh SYNOPSIS +.In pthread.h +.Ft int +.Fn pthread_jit_write_protect_supported_np "void" +.Ft void +.Fn pthread_jit_write_protect_np "int enabled" +.Sh DESCRIPTION +The +.Fn pthread_jit_write_protect_supported_np +function returns whether the +.Fn pthread_jit_write_protect_np +API is supported on this platform. If +.Fn pthread_jit_write_protect_np +API is supported on this platform, +.Fn pthread_jit_write_protect_np +must be called to toggle per-thread write protection on the MAP_JIT region before the thread writes to or executes from the MAP_JIT region. +.Pp +The +.Fn pthread_jit_write_protect_np +function sets whether MAP_JIT region write protection is enabled for this thread. +.Pp +On platforms where +.Fn pthread_jit_write_protect_supported_np +is true, MAP_JIT regions are never writeable and executable simultaneously. +When write protection is enabled for the thread, writes by the thread to the MAP_JIT region are denied and the MAP_JIT region is executable. +When write protection is disabled for the thread, writes by the thread to the MAP_JIT region are allowed and the MAP_JIT region is not executable. +Pass a non-zero value for the +.Fa enabled +parameter to enable thread JIT region write protection and allow execution. Pass a zero value for the +.Fa enabled +parameter to disable thread JIT write protection and deny execution. +.Pp +On platforms where +.Fn pthread_jit_write_protect_supported_np +is not supported, MAP_JIT regions are both simultaenously writeable and executable. Calls to +.Fn pthread_jit_write_protect_np +are no-ops on unsupported platforms. +.Sh RETURN VALUES +If supported, the +.Fn pthread_jit_write_protect_supported_np +function will return one. Otherwise the function will return zero. +.Sh SEE ALSO +.Xr mmap 2 diff --git a/private/dependency_private.h b/private/pthread/dependency_private.h similarity index 100% rename from private/dependency_private.h rename to private/pthread/dependency_private.h diff --git a/private/introspection_private.h b/private/pthread/introspection_private.h similarity index 100% rename from private/introspection_private.h rename to private/pthread/introspection_private.h diff --git a/private/posix_sched.h b/private/pthread/posix_sched.h similarity index 100% rename from private/posix_sched.h rename to private/pthread/posix_sched.h diff --git a/private/private.h b/private/pthread/private.h similarity index 61% rename from private/private.h rename to private/pthread/private.h index 33d5e25..fdde859 100644 --- a/private/private.h +++ b/private/pthread/private.h @@ -28,20 +28,6 @@ #include #include -/* get the thread specific errno value */ -__header_always_inline int -_pthread_get_errno_direct(void) -{ - return *(int*)_pthread_getspecific_direct(_PTHREAD_TSD_SLOT_ERRNO); -} - -/* set the thread specific errno value */ -__header_always_inline void -_pthread_set_errno_direct(int value) -{ - *((int*)_pthread_getspecific_direct(_PTHREAD_TSD_SLOT_ERRNO)) = value; -} - __API_AVAILABLE(macos(10.9), ios(7.0)) pthread_t pthread_main_thread_np(void); @@ -105,32 +91,98 @@ int pthread_attr_setcpupercent_np(pthread_attr_t * __restrict, int, unsigned lon __API_AVAILABLE(macos(10.15), ios(13.0), tvos(13.0), watchos(6.0)) int pthread_current_stack_contains_np(const void *, size_t); -#ifdef _os_tsd_get_base +/*! + * @function pthread_self_is_exiting_np + * + * @abstract + * Returns whether the current thread is exiting. + * + * @discussion + * This can be useful for certain introspection tools to know that malloc/free + * is called from the TSD destruction codepath. + * + * @result + * 0 if the thread is not exiting + * 1 if the thread is exiting + */ +__API_AVAILABLE(macos(10.16), ios(14.0), tvos(14.0), watchos(7.0)) +int pthread_self_is_exiting_np(void); #ifdef __LP64__ -#define _PTHREAD_STRUCT_DIRECT_THREADID_OFFSET -8 +#define _PTHREAD_STRUCT_DIRECT_THREADID_OFFSET -8 +#define _PTHREAD_STRUCT_DIRECT_TSD_OFFSET 224 +#else +#define _PTHREAD_STRUCT_DIRECT_THREADID_OFFSET -16 +#define _PTHREAD_STRUCT_DIRECT_TSD_OFFSET 176 +#endif + +#if !TARGET_OS_SIMULATOR +#if defined(__i386__) +#define _pthread_direct_tsd_relative_access(type, offset) \ + (type *)((char *)_pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF) + \ + _PTHREAD_STRUCT_DIRECT_TSD_OFFSET + _PTHREAD_STRUCT_DIRECT_##offset##_OFFSET) +#elif defined(OS_GS_RELATIVE) +#define _pthread_direct_tsd_relative_access(type, offset) \ + (type OS_GS_RELATIVE *)(_PTHREAD_STRUCT_DIRECT_##offset##_OFFSET) +#elif defined(_os_tsd_get_base) +#define _pthread_direct_tsd_relative_access(type, offset) \ + (type *)((char *)_os_tsd_get_base() + _PTHREAD_STRUCT_DIRECT_##offset##_OFFSET) #else -#define _PTHREAD_STRUCT_DIRECT_THREADID_OFFSET -16 +#error "unknown configuration" #endif +#endif // !TARGET_OS_SIMULATOR /* N.B. DO NOT USE UNLESS YOU ARE REBUILT AS PART OF AN OS TRAIN WORLDBUILD */ -__header_always_inline uint64_t +__header_always_inline __pure2 uint64_t _pthread_threadid_self_np_direct(void) { -#ifndef __i386__ - if (_pthread_has_direct_tsd()) { -#ifdef OS_GS_RELATIVE - return *(uint64_t OS_GS_RELATIVE *)(_PTHREAD_STRUCT_DIRECT_THREADID_OFFSET); +#ifdef _pthread_direct_tsd_relative_access + return *_pthread_direct_tsd_relative_access(uint64_t, THREADID); #else - return *(uint64_t*)((char *)_os_tsd_get_base() + _PTHREAD_STRUCT_DIRECT_THREADID_OFFSET); -#endif - } -#endif uint64_t threadid = 0; pthread_threadid_np(NULL, &threadid); return threadid; +#endif +} + +__header_always_inline __pure2 pthread_t +_pthread_self_direct(void) +{ +#if TARGET_OS_SIMULATOR || defined(__i386__) || defined(__x86_64__) + return (pthread_t)_pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF); +#elif defined(__arm__) || defined(__arm64__) + uintptr_t tsd_base = (uintptr_t)_os_tsd_get_base(); + return (pthread_t)(tsd_base - _PTHREAD_STRUCT_DIRECT_TSD_OFFSET); +#else +#error unsupported architecture +#endif +} + +__header_always_inline __pure2 mach_port_t +_pthread_mach_thread_self_direct(void) +{ + return (mach_port_t)(uintptr_t) + _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_MACH_THREAD_SELF); +} + +__header_always_inline int * +_pthread_errno_address_direct(void) +{ + return (int *)_pthread_getspecific_direct(_PTHREAD_TSD_SLOT_ERRNO); +} + +/* get the thread specific errno value */ +__header_always_inline int +_pthread_get_errno_direct(void) +{ + return *_pthread_errno_address_direct(); } -#endif // _os_tsd_get_base +/* set the thread specific errno value */ +__header_always_inline void +_pthread_set_errno_direct(int value) +{ + *_pthread_errno_address_direct() = value; +} #endif // __PTHREAD_PRIVATE_H__ diff --git a/private/qos.h b/private/pthread/qos.h similarity index 100% rename from private/qos.h rename to private/pthread/qos.h diff --git a/private/qos_private.h b/private/pthread/qos_private.h similarity index 100% rename from private/qos_private.h rename to private/pthread/qos_private.h diff --git a/private/spinlock_private.h b/private/pthread/spinlock_private.h similarity index 100% rename from private/spinlock_private.h rename to private/pthread/spinlock_private.h diff --git a/private/tsd_private.h b/private/pthread/tsd_private.h similarity index 85% rename from private/tsd_private.h rename to private/pthread/tsd_private.h index e83ea66..11daba0 100644 --- a/private/tsd_private.h +++ b/private/pthread/tsd_private.h @@ -56,38 +56,28 @@ #include #include -#ifndef __TSD_MACH_THREAD_SELF -#define __TSD_MACH_THREAD_SELF 3 -#endif - -#ifndef __TSD_THREAD_QOS_CLASS -#define __TSD_THREAD_QOS_CLASS 4 -#endif - -#ifndef __TSD_RETURN_TO_KERNEL -#define __TSD_RETURN_TO_KERNEL 5 -#endif - -#ifndef __TSD_PTR_MUNGE -#define __TSD_PTR_MUNGE 7 -#endif - -#ifndef __TSD_MACH_SPECIAL_REPLY -#define __TSD_MACH_SPECIAL_REPLY 8 -#endif - /* Constant TSD slots for inline pthread_getspecific() usage. */ /* Keys 0 - 9 are for Libsyscall/libplatform usage */ -#define _PTHREAD_TSD_SLOT_PTHREAD_SELF __TSD_THREAD_SELF -#define _PTHREAD_TSD_SLOT_ERRNO __TSD_ERRNO -#define _PTHREAD_TSD_SLOT_MIG_REPLY __TSD_MIG_REPLY -#define _PTHREAD_TSD_SLOT_MACH_THREAD_SELF __TSD_MACH_THREAD_SELF -#define _PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS __TSD_THREAD_QOS_CLASS -#define _PTHREAD_TSD_SLOT_RETURN_TO_KERNEL __TSD_RETURN_TO_KERNEL -#define _PTHREAD_TSD_SLOT_PTR_MUNGE __TSD_PTR_MUNGE -#define _PTHREAD_TSD_SLOT_MACH_SPECIAL_REPLY __TSD_MACH_SPECIAL_REPLY -//#define _PTHREAD_TSD_SLOT_SEMAPHORE_CACHE __TSD_SEMAPHORE_CACHE +#define _PTHREAD_TSD_SLOT_PTHREAD_SELF __TSD_THREAD_SELF +#define _PTHREAD_TSD_SLOT_ERRNO __TSD_ERRNO +#define _PTHREAD_TSD_SLOT_MIG_REPLY __TSD_MIG_REPLY +#define _PTHREAD_TSD_SLOT_MACH_THREAD_SELF __TSD_MACH_THREAD_SELF +#define _PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS __TSD_THREAD_QOS_CLASS +#define _PTHREAD_TSD_SLOT_RETURN_TO_KERNEL __TSD_RETURN_TO_KERNEL +#define _PTHREAD_TSD_SLOT_PTR_MUNGE __TSD_PTR_MUNGE +#define _PTHREAD_TSD_SLOT_MACH_SPECIAL_REPLY __TSD_MACH_SPECIAL_REPLY +#define _PTHREAD_TSD_SLOT_SEMAPHORE_CACHE __TSD_SEMAPHORE_CACHE + +#define _PTHREAD_TSD_SLOT_PTHREAD_SELF_TYPE pthread_t +#define _PTHREAD_TSD_SLOT_ERRNO_TYPE int * +#define _PTHREAD_TSD_SLOT_MIG_REPLY_TYPE mach_port_t +#define _PTHREAD_TSD_SLOT_MACH_THREAD_SELF_TYPE mach_port_t +#define _PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS_TYPE pthread_priority_t +#define _PTHREAD_TSD_SLOT_RETURN_TO_KERNEL_TYPE uintptr_t +#define _PTHREAD_TSD_SLOT_PTR_MUNGE_TYPE uintptr_t +#define _PTHREAD_TSD_SLOT_MACH_SPECIAL_REPLY_TYPE mach_port_t +#define _PTHREAD_TSD_SLOT_SEMAPHORE_CACHE_TYPE semaphore_t /* * Windows 64-bit ABI bakes %gs relative accesses into its code in the same @@ -225,6 +215,20 @@ #define __PTK_FRAMEWORK_SWIFT_KEY8 108 #define __PTK_FRAMEWORK_SWIFT_KEY9 109 +/* Keys 110-115 for libmalloc */ +#define __PTK_LIBMALLOC_KEY0 110 +#define __PTK_LIBMALLOC_KEY1 111 +#define __PTK_LIBMALLOC_KEY2 112 +#define __PTK_LIBMALLOC_KEY3 113 +#define __PTK_LIBMALLOC_KEY4 114 + +/* Keys 115-120 for libdispatch workgroups */ +#define __PTK_LIBDISPATCH_WORKGROUP_KEY0 115 +#define __PTK_LIBDISPATCH_WORKGROUP_KEY1 116 +#define __PTK_LIBDISPATCH_WORKGROUP_KEY2 117 +#define __PTK_LIBDISPATCH_WORKGROUP_KEY3 118 +#define __PTK_LIBDISPATCH_WORKGROUP_KEY4 119 + /* Keys 190 - 194 are for the use of PerfUtils */ #define __PTK_PERF_UTILS_KEY0 190 #define __PTK_PERF_UTILS_KEY1 191 diff --git a/private/workqueue_private.h b/private/pthread/workqueue_private.h similarity index 99% rename from private/workqueue_private.h rename to private/pthread/workqueue_private.h index 11b0e5e..ccbc103 100644 --- a/private/workqueue_private.h +++ b/private/pthread/workqueue_private.h @@ -29,10 +29,7 @@ #include #include #include -#include -#ifndef _PTHREAD_BUILDING_PTHREAD_ #include -#endif #define PTHREAD_WORKQUEUE_SPI_VERSION 20170201 diff --git a/sys/qos_private.h b/private/sys/qos_private.h similarity index 100% rename from sys/qos_private.h rename to private/sys/qos_private.h diff --git a/src/exports_internal.h b/src/exports_internal.h new file mode 100644 index 0000000..bb79f62 --- /dev/null +++ b/src/exports_internal.h @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2019 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#ifndef __LIBPTHREAD_EXPORTS_INTERNAL_H__ +#define __LIBPTHREAD_EXPORTS_INTERNAL_H__ + +/*! + * @file exports_internal.h + * + * @brief + * This file has prototypes for symbols / functions that are exported + * without a public header. + * + * @discussion + * This header is also fed to TAPI and needs to work without internal.h + */ + +#include +#include +#include + +struct ProgramVars; + +OS_EXPORT int __is_threaded; + +OS_EXPORT const int __unix_conforming; + +OS_EXPORT +void +_pthread_set_self(pthread_t); + +OS_EXPORT +pthread_t +_pthread_self(void); + +OS_EXPORT +int +__pthread_init(const struct _libpthread_functions *pthread_funcs, + const char *envp[], const char *apple[], + const struct ProgramVars *vars); + +OS_EXPORT OS_NORETURN +void +thread_start(pthread_t self, mach_port_t kport, + void *(*fun)(void *), void *arg, + size_t stacksize, unsigned int flags); // trampoline into _pthread_start + +OS_EXPORT OS_NORETURN +void +_pthread_start(pthread_t thread, mach_port_t kport, + void *(*fun)(void *), void *arg, + size_t stacksize, unsigned int flags); + +OS_EXPORT OS_NORETURN +void +start_wqthread(pthread_t self, mach_port_t kport, + void *stackaddr, void *unused, int reuse); // trampoline into _start_wqthread + +OS_EXPORT +void +_pthread_wqthread(pthread_t self, mach_port_t kport, + void *stackaddr, void *keventlist, int flags, int nkevents); + +#if defined(__x86_64__) || defined(__i386__) || defined(__arm64__) +OS_EXPORT +void +___chkstk_darwin(void); + +OS_EXPORT +void +thread_chkstk_darwin(void); +#endif // defined(__x86_64__) || defined(__i386__) || defined(__arm64__) + +#pragma mark - exports with prototypes from not in libpthread + +OS_EXPORT +int +sigwait(const sigset_t *, int *) __DARWIN_ALIAS_C(sigwait); + +#pragma mark - shared with libsystem_kernel.dylib +/* Note: these can't use _pthread_malloc / _pthread_free unconditionally */ + +OS_EXPORT +void +_pthread_clear_qos_tsd(mach_port_t kport); + +OS_EXPORT +void +_pthread_exit_if_canceled(int error); + +#pragma mark - atfork libSystem integration + +OS_EXPORT void _pthread_atfork_prepare_handlers(void); +OS_EXPORT void _pthread_atfork_prepare(void); +OS_EXPORT void _pthread_atfork_parent(void); +OS_EXPORT void _pthread_atfork_parent_handlers(void); +OS_EXPORT void _pthread_atfork_child(void); +OS_EXPORT void _pthread_atfork_child_handlers(void); +OS_EXPORT void _pthread_fork_prepare(void); +OS_EXPORT void _pthread_fork_parent(void); +OS_EXPORT void _pthread_fork_child(void); +OS_EXPORT void _pthread_fork_child_postinit(void); + +#pragma mark - TAPI +#ifdef __clang_tapi__ + +#define declare_symbol(s) OS_EXPORT void __tapi_##s(void) asm("_" #s) + +#if TARGET_OS_OSX && defined(__i386__) +// TAPI will see the $UNIX2003 redirected symbols +declare_symbol(pthread_cancel); +declare_symbol(pthread_cond_init); +declare_symbol(pthread_cond_timedwait); +declare_symbol(pthread_cond_wait); +declare_symbol(pthread_join); +declare_symbol(pthread_mutexattr_destroy); +declare_symbol(pthread_rwlock_destroy); +declare_symbol(pthread_rwlock_init); +declare_symbol(pthread_rwlock_rdlock); +declare_symbol(pthread_rwlock_tryrdlock); +declare_symbol(pthread_rwlock_trywrlock); +declare_symbol(pthread_rwlock_unlock); +declare_symbol(pthread_rwlock_wrlock); +declare_symbol(pthread_setcancelstate); +declare_symbol(pthread_setcanceltype); +declare_symbol(pthread_sigmask); +declare_symbol(pthread_testcancel); +declare_symbol(sigwait); +// TAPI will see the $NOCANCEL$UNIX2003 redirected symbols +declare_symbol(pthread_cond_timedwait$UNIX2003); +declare_symbol(pthread_cond_wait$UNIX2003); +declare_symbol(pthread_join$UNIX2003); +declare_symbol(sigwait$UNIX2003); +#else +// TAPI will see the $NOCANCEL redirected symbols +declare_symbol(pthread_cond_timedwait); +declare_symbol(pthread_cond_wait); +declare_symbol(pthread_join); +declare_symbol(sigwait); +#endif + +#undef declare_symbol + +#endif // __clang_tapi__ +#endif // __LIBPTHREAD_EXPORTS_INTERNAL_H__ diff --git a/src/imports_internal.h b/src/imports_internal.h new file mode 100644 index 0000000..19262a8 --- /dev/null +++ b/src/imports_internal.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#ifndef __LIBPTHREAD_IMPORTS_INTERNAL_H__ +#define __LIBPTHREAD_IMPORTS_INTERNAL_H__ + +/*! + * @file imports_internal.h + * + * @brief + * This file lists prototypes that do not have a header on the system, + * like syscalls, that we need to import in pthread. + */ + +#include +#include +#include +#include + +extern boolean_t swtch_pri(int); + +// Defined in libsyscall; initialized in libmalloc +extern malloc_logger_t *__syscall_logger; + +// syscalls + +extern uint32_t __psynch_mutexwait(pthread_mutex_t *mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags); +extern uint32_t __psynch_mutexdrop(pthread_mutex_t *mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags); + +extern uint32_t __psynch_cvbroad(pthread_cond_t *cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, pthread_mutex_t *mutex, uint64_t mugen, uint64_t tid); +extern uint32_t __psynch_cvsignal(pthread_cond_t *cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, pthread_mutex_t *mutex, uint64_t mugen, uint64_t tid, uint32_t flags); +extern uint32_t __psynch_cvwait(pthread_cond_t *cv, uint64_t cvlsgen, uint32_t cvugen, pthread_mutex_t *mutex, uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec); +extern uint32_t __psynch_cvclrprepost(void *cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags); + +extern uint32_t __psynch_rw_longrdlock(pthread_rwlock_t *rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); +extern uint32_t __psynch_rw_yieldwrlock(pthread_rwlock_t *rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); +extern int __psynch_rw_downgrade(pthread_rwlock_t *rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); +extern uint32_t __psynch_rw_upgrade(pthread_rwlock_t *rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); +extern uint32_t __psynch_rw_rdlock(pthread_rwlock_t *rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); +extern uint32_t __psynch_rw_wrlock(pthread_rwlock_t *rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); +extern uint32_t __psynch_rw_unlock(pthread_rwlock_t *rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); +extern uint32_t __psynch_rw_unlock2(pthread_rwlock_t *rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); + +extern uint32_t __bsdthread_ctl(uintptr_t cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3); +extern pthread_t __bsdthread_create(void *(*func)(void *), void *func_arg, void *stack, pthread_t thread, unsigned int flags); +extern int __bsdthread_register(void (*)(pthread_t, mach_port_t, void *(*)(void *), void *, size_t, unsigned int), void (*)(pthread_t, mach_port_t, void *, void *, int), int,void (*)(pthread_t, mach_port_t, void *(*)(void *), void *, size_t, unsigned int), int32_t *,__uint64_t); +extern int __bsdthread_terminate(void *freeaddr, size_t freesize, mach_port_t kport, mach_port_t joinsem); + +extern uint64_t __thread_selfid(void); +extern int __disable_threadsignal(int); + +extern int __pthread_canceled(int); +extern int __pthread_chdir(const char *path); +extern int __pthread_fchdir(int fd); +extern int __pthread_kill(mach_port_t, int); +extern int __pthread_markcancel(mach_port_t); +extern int __pthread_sigmask(int, const sigset_t *, sigset_t *); + +extern int __gettimeofday(struct timeval *, struct timezone *); +extern void __exit(int) __attribute__((noreturn)); +extern int __proc_info(int callnum, int pid, int flavor, uint64_t arg, void *buffer, int buffersize); +extern int __semwait_signal_nocancel(int, int, int, int, __int64_t, __int32_t); +extern int __sigwait(const sigset_t *set, int *sig); +extern int __sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); + + +#endif // __LIBPTHREAD_IMPORTS_INTERNAL_H__ diff --git a/src/inline_internal.h b/src/inline_internal.h new file mode 100644 index 0000000..bc6d65a --- /dev/null +++ b/src/inline_internal.h @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2019 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#ifndef __LIBPTHREAD_INLINE_INTERNAL_H__ +#define __LIBPTHREAD_INLINE_INTERNAL_H__ + +/*! + * @file inline_internal.h + * + * @brief + * This file exposes inline helpers that are generally useful in libpthread. + */ + +#define PTHREAD_INTERNAL_CRASH(c, x) OS_BUG_INTERNAL(c, "LIBPTHREAD", x) +#define PTHREAD_CLIENT_CRASH(c, x) OS_BUG_CLIENT(c, "LIBPTHREAD", x) +#ifdef DEBUG +#define PTHREAD_DEBUG_ASSERT(b) \ + do { \ + if (os_unlikely(!(b))) { \ + PTHREAD_INTERNAL_CRASH(0, "Assertion failed: " #b); \ + } \ + } while (0) +#else +#define PTHREAD_DEBUG_ASSERT(b) ((void)0) +#endif + +#pragma mark _pthread_mutex_check_signature + +OS_ALWAYS_INLINE +static inline bool +_pthread_mutex_check_signature_fast(pthread_mutex_t *mutex) +{ + return (mutex->sig == _PTHREAD_MUTEX_SIG_fast); +} + +OS_ALWAYS_INLINE +static inline bool +_pthread_mutex_check_signature(const pthread_mutex_t *mutex) +{ + // TODO: PTHREAD_STRICT candidate + return ((mutex->sig & _PTHREAD_MUTEX_SIG_MASK) == _PTHREAD_MUTEX_SIG_CMP); +} + +OS_ALWAYS_INLINE +static inline bool +_pthread_mutex_check_signature_init(const pthread_mutex_t *mutex) +{ + return ((mutex->sig & _PTHREAD_MUTEX_SIG_init_MASK) == + _PTHREAD_MUTEX_SIG_init_CMP); +} + +#pragma mark pthread mutex accessors + +OS_ALWAYS_INLINE +static inline bool +_pthread_mutex_uses_ulock(pthread_mutex_t *mutex) +{ + return mutex->mtxopts.options.ulock; +} + +#pragma mark _pthread_rwlock_check_signature + +OS_ALWAYS_INLINE +static inline bool +_pthread_rwlock_check_signature(const pthread_rwlock_t *rwlock) +{ + return (rwlock->sig == _PTHREAD_RWLOCK_SIG); +} + +OS_ALWAYS_INLINE +static inline bool +_pthread_rwlock_check_signature_init(const pthread_rwlock_t *rwlock) +{ + return (rwlock->sig == _PTHREAD_RWLOCK_SIG_init); +} + +#pragma mark unfair lock wrappers + +#define _PTHREAD_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT +#define _PTHREAD_LOCK_OPTIONS \ + (OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION | OS_UNFAIR_LOCK_ADAPTIVE_SPIN) + +OS_ALWAYS_INLINE +static inline void +_pthread_lock_init(os_unfair_lock_t lock) +{ + *lock = _PTHREAD_LOCK_INITIALIZER; +} + +OS_OVERLOADABLE OS_ALWAYS_INLINE +static inline void +_pthread_lock_lock(os_unfair_lock_t lock) +{ +#if OS_UNFAIR_LOCK_INLINE + os_unfair_lock_lock_with_options_inline(lock, _PTHREAD_LOCK_OPTIONS); +#else + os_unfair_lock_lock_with_options(lock, _PTHREAD_LOCK_OPTIONS); +#endif +} + +OS_OVERLOADABLE OS_ALWAYS_INLINE +static inline void +_pthread_lock_lock(os_unfair_lock_t lock, mach_port_t mts) +{ +#if OS_UNFAIR_LOCK_INLINE + os_unfair_lock_lock_no_tsd_inline(lock, _PTHREAD_LOCK_OPTIONS, mts); +#else + os_unfair_lock_lock_no_tsd(lock, _PTHREAD_LOCK_OPTIONS, mts); +#endif +} + +OS_OVERLOADABLE OS_ALWAYS_INLINE +static inline void +_pthread_lock_unlock(os_unfair_lock_t lock) +{ +#if OS_UNFAIR_LOCK_INLINE + os_unfair_lock_unlock_inline(lock); +#else + os_unfair_lock_unlock(lock); +#endif +} + +OS_OVERLOADABLE OS_ALWAYS_INLINE +static inline void +_pthread_lock_unlock(os_unfair_lock_t lock, mach_port_t mts) +{ +#if OS_UNFAIR_LOCK_INLINE + os_unfair_lock_unlock_no_tsd_inline(lock, mts); +#else + os_unfair_lock_unlock_no_tsd(lock, mts); +#endif +} + +#pragma mark pthread accessors + +// Internal references to pthread_self() use TSD slot 0 directly. +#define pthread_self() _pthread_self_direct() + +// Internal references to errno use TSD slot 1 directly. +#undef errno +#define errno (*_pthread_errno_address_direct()) + +#define _pthread_tsd_slot(th, name) \ + (*(_PTHREAD_TSD_SLOT_##name##_TYPE *)(uintptr_t *)&(th)->tsd[_PTHREAD_TSD_SLOT_##name]) + +OS_ALWAYS_INLINE +static inline void +_pthread_validate_signature(pthread_t thread) +{ + pthread_t th = (pthread_t)(thread->sig ^ _pthread_ptr_munge_token); +#if __has_feature(ptrauth_calls) + th = ptrauth_auth_data(th, ptrauth_key_process_dependent_data, + ptrauth_string_discriminator("pthread.signature")); +#endif + if (os_unlikely(th != thread)) { + /* OS_REASON_LIBSYSTEM_CODE_PTHREAD_CORRUPTION == 4 */ + abort_with_reason(OS_REASON_LIBSYSTEM, 4, "pthread_t was corrupted", 0); + } +} + +OS_ALWAYS_INLINE +static inline void +_pthread_init_signature(pthread_t thread) +{ + pthread_t th = thread; +#if __has_feature(ptrauth_calls) + th = ptrauth_sign_unauthenticated(th, ptrauth_key_process_dependent_data, + ptrauth_string_discriminator("pthread.signature")); +#endif + thread->sig = (uintptr_t)th ^ _pthread_ptr_munge_token; +} + +/* + * ALWAYS called without list lock and return with list lock held on success + * + * This weird calling convention exists because this function will sometimes + * drop the lock, and it's best callers don't have to remember this. + */ +OS_ALWAYS_INLINE +static inline bool +_pthread_validate_thread_and_list_lock(pthread_t thread) +{ + pthread_t p; + if (thread == NULL) return false; + _pthread_lock_lock(&_pthread_list_lock); + TAILQ_FOREACH(p, &__pthread_head, tl_plist) { + if (p != thread) continue; + _pthread_validate_signature(p); + return true; + } + _pthread_lock_unlock(&_pthread_list_lock); + + return false; +} + +OS_ALWAYS_INLINE +static inline bool +_pthread_is_valid(pthread_t thread, mach_port_t *portp) +{ + mach_port_t kport = MACH_PORT_NULL; + bool valid; + + if (thread == pthread_self()) { + _pthread_validate_signature(thread); + valid = true; + kport = _pthread_tsd_slot(thread, MACH_THREAD_SELF); + } else if (!_pthread_validate_thread_and_list_lock(thread)) { + valid = false; + } else { + kport = _pthread_tsd_slot(thread, MACH_THREAD_SELF); + valid = true; + _pthread_lock_unlock(&_pthread_list_lock); + } + + if (portp != NULL) { + *portp = kport; + } + return valid; +} + +OS_ALWAYS_INLINE OS_CONST +static inline pthread_globals_t +_pthread_globals(void) +{ + return os_alloc_once(OS_ALLOC_ONCE_KEY_LIBSYSTEM_PTHREAD, + sizeof(struct pthread_globals_s), NULL); +} + +#endif // __LIBPTHREAD_INLINE_INTERNAL_H__ diff --git a/src/internal.h b/src/internal.h index a98d86f..766ffdb 100644 --- a/src/internal.h +++ b/src/internal.h @@ -52,738 +52,57 @@ #ifndef _POSIX_PTHREAD_INTERNALS_H #define _POSIX_PTHREAD_INTERNALS_H -#define _PTHREAD_BUILDING_PTHREAD_ - -// suppress pthread_attr_t typedef in sys/signal.h -#define _PTHREAD_ATTR_T -struct _pthread_attr_t; /* forward reference */ -typedef struct _pthread_attr_t pthread_attr_t; +#ifndef PTHREAD_LAYOUT_SPI +#define PTHREAD_LAYOUT_SPI 1 +#endif -#include <_simple.h> -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include - -#define __OS_EXPOSE_INTERNALS__ 1 -#include -#include -#include #if TARGET_IPHONE_SIMULATOR #error Unsupported target #endif +#include "types_internal.h" // has to come first as it hides the SDK types +#include "offsets_internal.h" // included to validate the offsets at build time -#define PTHREAD_INTERNAL_CRASH(c, x) do { \ - _os_set_crash_log_cause_and_message((c), \ - "BUG IN LIBPTHREAD: " x); \ - __builtin_trap(); \ - } while (0) - -#define PTHREAD_CLIENT_CRASH(c, x) do { \ - _os_set_crash_log_cause_and_message((c), \ - "BUG IN CLIENT OF LIBPTHREAD: " x); \ - __builtin_trap(); \ - } while (0) - -#ifndef __POSIX_LIB__ -#define __POSIX_LIB__ -#endif - -#ifndef PTHREAD_LAYOUT_SPI -#define PTHREAD_LAYOUT_SPI 1 -#endif - -#include "posix_sched.h" -#include "tsd_private.h" -#include "spinlock_private.h" - -#define PTHREAD_EXPORT extern __attribute__((visibility("default"))) -#define PTHREAD_EXTERN extern -#define PTHREAD_NOEXPORT __attribute__((visibility("hidden"))) -#define PTHREAD_NOEXPORT_VARIANT -#define PTHREAD_NORETURN __attribute__((__noreturn__)) -#define PTHREAD_ALWAYS_INLINE __attribute__((always_inline)) -#define PTHREAD_NOINLINE __attribute__((noinline)) -#define PTHREAD_WEAK __attribute__((weak)) -#define PTHREAD_USED __attribute__((used)) -#define PTHREAD_NOT_TAIL_CALLED __attribute__((__not_tail_called__)) +#include <_simple.h> +#include +#include +#include +#include -#define OS_UNFAIR_LOCK_INLINE 1 +#include +#include +#include #include -typedef os_unfair_lock _pthread_lock; -#define _PTHREAD_LOCK_INITIALIZER OS_UNFAIR_LOCK_INIT -#define _PTHREAD_LOCK_INIT(lock) ((lock) = (_pthread_lock)_PTHREAD_LOCK_INITIALIZER) -#define _PTHREAD_LOCK(lock) os_unfair_lock_lock_with_options_inline(&(lock), OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION) -#define _PTHREAD_LOCK_FROM_MACH_THREAD(lock) os_unfair_lock_lock_inline_no_tsd_4libpthread(&(lock)) -#define _PTHREAD_UNLOCK(lock) os_unfair_lock_unlock_inline(&(lock)) -#define _PTHREAD_UNLOCK_FROM_MACH_THREAD(lock) os_unfair_lock_unlock_inline_no_tsd_4libpthread(&(lock)) - -#define _PTHREAD_POLICY_IS_FIXEDPRI(x) ((x) == SCHED_RR || (x) == SCHED_FIFO) - -extern int __is_threaded; -extern int __unix_conforming; -PTHREAD_NOEXPORT -extern uintptr_t _pthread_ptr_munge_token; - -// List of all pthreads in the process. -TAILQ_HEAD(__pthread_list, _pthread); -PTHREAD_NOEXPORT extern struct __pthread_list __pthread_head; - -// Lock protects access to above list. -PTHREAD_NOEXPORT extern _pthread_lock _pthread_list_lock; - -PTHREAD_NOEXPORT extern uint32_t _main_qos; - -#if PTHREAD_DEBUG_LOG -#include -PTHREAD_NOEXPORT extern int _pthread_debuglog; -PTHREAD_NOEXPORT extern uint64_t _pthread_debugstart; -#endif - -/* - * Compiled-in limits - */ -#if TARGET_OS_EMBEDDED -#define _EXTERNAL_POSIX_THREAD_KEYS_MAX 256 -#define _INTERNAL_POSIX_THREAD_KEYS_MAX 256 -#define _INTERNAL_POSIX_THREAD_KEYS_END 512 -#else -#define _EXTERNAL_POSIX_THREAD_KEYS_MAX 512 -#define _INTERNAL_POSIX_THREAD_KEYS_MAX 256 -#define _INTERNAL_POSIX_THREAD_KEYS_END 768 -#endif - -#if defined(__arm64__) -/* Pull the pthread_t into the same page as the top of the stack so we dirty one less page. - * The _pthread struct at the top of the stack shouldn't be page-aligned - */ -#define PTHREAD_T_OFFSET (12*1024) -#else -#define PTHREAD_T_OFFSET 0 -#endif - -#define MAXTHREADNAMESIZE 64 -#define _PTHREAD_T -typedef struct _pthread { - // - // ABI - These fields are externally known as struct _opaque_pthread_t. - // - long sig; - struct __darwin_pthread_handler_rec *__cleanup_stack; - - // - // SPI - These fields are private. - // - - // - // Fields protected by _pthread_list_lock - // - - TAILQ_ENTRY(_pthread) tl_plist; // global thread list [aligned] - struct pthread_join_context_s *tl_join_ctx; - void *tl_exit_value; - uint32_t tl_policy:8, - tl_joinable:1, - tl_joiner_cleans_up:1, - tl_has_custom_stack:1, - __tl_pad:21; - // MACH_PORT_NULL if no joiner - // tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] when has a joiner - // MACH_PORT_DEAD if the thread exited - uint32_t tl_exit_gate; - struct sched_param tl_param; - void *__unused_padding; - - // - // Fields protected by pthread_t::lock - // - - _pthread_lock lock; - uint16_t max_tsd_key; - uint16_t inherit:8, - kernalloc:1, - schedset:1, - wqthread:1, - wqkillset:1, - __flags_pad:4; - - char pthread_name[MAXTHREADNAMESIZE]; // includes NUL [aligned] - - void *(*fun)(void *); // thread start routine - void *arg; // thread start routine argument - int wq_nevents; // wqthreads (workloop / kevent) - bool wq_outsideqos; - uint8_t canceled; // 4597450 set if conformant cancelation happened - uint16_t cancel_state; // whether the thread can be canceled [atomic] - errno_t cancel_error; - errno_t err_no; // thread-local errno - - void *stackaddr; // base of the stack (page aligned) - void *stackbottom; // stackaddr - stacksize - void *freeaddr; // stack/thread allocation base address - size_t freesize; // stack/thread allocation size - size_t guardsize; // guard page size in bytes - - // tsd-base relative accessed elements - __attribute__((aligned(8))) - uint64_t thread_id; // 64-bit unique thread id - - /* Thread Specific Data slots - * - * The offset of this field from the start of the structure is difficult to - * change on OS X because of a thorny bitcompat issue: mono has hard coded - * the value into their source. Newer versions of mono will fall back to - * scanning to determine it at runtime, but there's lots of software built - * with older mono that won't. We will have to break them someday... - */ - __attribute__ ((aligned (16))) - void *tsd[_EXTERNAL_POSIX_THREAD_KEYS_MAX + _INTERNAL_POSIX_THREAD_KEYS_MAX]; -} *pthread_t; - -#define _PTHREAD_ATTR_REFILLMS_MAX ((2<<24) - 1) -struct _pthread_attr_t { - long sig; - size_t guardsize; // size in bytes of stack overflow guard area - void *stackaddr; // stack base; vm_page_size aligned - size_t stacksize; // stack size; multiple of vm_page_size and >= PTHREAD_STACK_MIN - union { - struct sched_param param; // [aligned] - unsigned long qosclass; // pthread_priority_t - }; - uint32_t - detached:8, - inherit:8, - policy:8, - schedset:1, - qosset:1, - policyset:1, - cpupercentset:1, - defaultguardpage:1, - unused:3; - uint32_t - cpupercent:8, - refillms:24; -#if defined(__LP64__) - uint32_t _reserved[4]; -#else - uint32_t _reserved[2]; -#endif -}; - -/* - * Mutex attributes - */ - -#define _PTHREAD_MUTEXATTR_T -typedef struct { - long sig; - int prioceiling; - uint32_t protocol:2, - type:2, - pshared:2, - opt:3, - unused:23; -} pthread_mutexattr_t; - -struct _pthread_mutex_options { - uint32_t protocol:2, - type:2, - pshared:2, - policy:3, - hold:2, - misalign:1, - notify:1, - mutex:1, - unused:2, - lock_count:16; -}; -// -#define _PTHREAD_MUTEX_POLICY_LAST (PTHREAD_MUTEX_POLICY_FIRSTFIT_NP + 1) -#define _PTHREAD_MTX_OPT_POLICY_FAIRSHARE 1 -#define _PTHREAD_MTX_OPT_POLICY_FIRSTFIT 2 -#define _PTHREAD_MTX_OPT_POLICY_DEFAULT _PTHREAD_MTX_OPT_POLICY_FIRSTFIT -// The following _pthread_mutex_options defintions exist in synch_internal.h -// such that the kernel extension can test for flags. They must be kept in -// sync with the bit values in the struct above. -// _PTHREAD_MTX_OPT_PSHARED 0x010 -// _PTHREAD_MTX_OPT_NOTIFY 0x1000 -// _PTHREAD_MTX_OPT_MUTEX 0x2000 - -// The fixed mask is used to mask out portions of the mutex options that -// change on a regular basis (notify, lock_count). -#define _PTHREAD_MTX_OPT_FIXED_MASK 0x27ff - -typedef struct { - long sig; - _pthread_lock lock; - union { - uint32_t value; - struct _pthread_mutex_options options; - } mtxopts; - int16_t prioceiling; - int16_t priority; -#if defined(__LP64__) - uint32_t _pad; -#endif - uint32_t m_tid[2]; // thread id of thread that has mutex locked - uint32_t m_seq[2]; // mutex sequence id - uint32_t m_mis[2]; // for misaligned locks m_tid/m_seq will span into here -#if defined(__LP64__) - uint32_t _reserved[4]; -#else - uint32_t _reserved[1]; -#endif -} _pthread_mutex; - - -#define _PTHREAD_CONDATTR_T -typedef struct { - long sig; - uint32_t pshared:2, - unsupported:30; -} pthread_condattr_t; - - -typedef struct { - long sig; - _pthread_lock lock; - uint32_t unused:29, - misalign:1, - pshared:2; - _pthread_mutex *busy; - uint32_t c_seq[3]; -#if defined(__LP64__) - uint32_t _reserved[3]; -#endif -} _pthread_cond; - - -#define _PTHREAD_ONCE_T -typedef struct { - long sig; - os_once_t once; -} pthread_once_t; - - -#define _PTHREAD_RWLOCKATTR_T -typedef struct { - long sig; - int pshared; -#if defined(__LP64__) - uint32_t _reserved[3]; -#else - uint32_t _reserved[2]; -#endif -} pthread_rwlockattr_t; - +#include +#include +#include -typedef struct { - long sig; - _pthread_lock lock; - uint32_t unused:29, - misalign:1, - pshared:2; - uint32_t rw_flags; -#if defined(__LP64__) - uint32_t _pad; -#endif - uint32_t rw_tid[2]; // thread id of thread that has exclusive (write) lock - uint32_t rw_seq[4]; // rw sequence id (at 128-bit aligned boundary) - uint32_t rw_mis[4]; // for misaligned locks rw_seq will span into here -#if defined(__LP64__) - uint32_t _reserved[34]; -#else - uint32_t _reserved[18]; -#endif -} _pthread_rwlock; +#include +#include #include "pthread.h" #include "pthread_spis.h" +#include "pthread/private.h" +#include "pthread/dependency_private.h" +#include "pthread/spinlock_private.h" +#include "pthread/workqueue_private.h" +#include "pthread/introspection_private.h" +#include "pthread/qos_private.h" +#include "pthread/tsd_private.h" +#include "pthread/stack_np.h" + +#include "imports_internal.h" +#include "prototypes_internal.h" +#include "exports_internal.h" +#include "inline_internal.h" -_Static_assert(sizeof(_pthread_mutex) == sizeof(pthread_mutex_t), - "Incorrect _pthread_mutex structure size"); - -_Static_assert(sizeof(_pthread_rwlock) == sizeof(pthread_rwlock_t), - "Incorrect _pthread_rwlock structure size"); - -// Internal references to pthread_self() use TSD slot 0 directly. -__header_always_inline __pure2 pthread_t -_pthread_self_direct(void) -{ -#if TARGET_OS_SIMULATOR || defined(__i386__) || defined(__x86_64__) - return (pthread_t)_pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_SELF); -#elif defined(__arm__) || defined(__arm64__) - uintptr_t tsd_base = (uintptr_t)_os_tsd_get_base(); - return (pthread_t)(tsd_base - offsetof(struct _pthread, tsd)); -#else -#error unsupported architecture -#endif -} -#define pthread_self() _pthread_self_direct() - -PTHREAD_ALWAYS_INLINE -inline static uint64_t __attribute__((__pure__)) -_pthread_selfid_direct(void) -{ - return (_pthread_self_direct())->thread_id; -} - -#define _PTHREAD_DEFAULT_INHERITSCHED PTHREAD_INHERIT_SCHED -#define _PTHREAD_DEFAULT_PROTOCOL PTHREAD_PRIO_NONE -#define _PTHREAD_DEFAULT_PRIOCEILING 0 -#define _PTHREAD_DEFAULT_POLICY SCHED_OTHER -#define _PTHREAD_DEFAULT_STACKSIZE 0x80000 /* 512K */ -#define _PTHREAD_DEFAULT_PSHARED PTHREAD_PROCESS_PRIVATE - -#define _PTHREAD_NO_SIG 0x00000000 -#define _PTHREAD_MUTEX_ATTR_SIG 0x4D545841 /* 'MTXA' */ -#define _PTHREAD_MUTEX_SIG 0x4D555458 /* 'MUTX' */ -#define _PTHREAD_MUTEX_SIG_fast 0x4D55545A /* 'MUTZ' */ -#define _PTHREAD_MUTEX_SIG_MASK 0xfffffffd -#define _PTHREAD_MUTEX_SIG_CMP 0x4D555458 /* _PTHREAD_MUTEX_SIG & _PTHREAD_MUTEX_SIG_MASK */ -#define _PTHREAD_MUTEX_SIG_init 0x32AAABA7 /* [almost] ~'MUTX' */ -#define _PTHREAD_ERRORCHECK_MUTEX_SIG_init 0x32AAABA1 -#define _PTHREAD_RECURSIVE_MUTEX_SIG_init 0x32AAABA2 -#define _PTHREAD_FIRSTFIT_MUTEX_SIG_init 0x32AAABA3 -#define _PTHREAD_MUTEX_SIG_init_MASK 0xfffffff0 -#define _PTHREAD_MUTEX_SIG_init_CMP 0x32AAABA0 -#define _PTHREAD_COND_ATTR_SIG 0x434E4441 /* 'CNDA' */ -#define _PTHREAD_COND_SIG 0x434F4E44 /* 'COND' */ -#define _PTHREAD_COND_SIG_init 0x3CB0B1BB /* [almost] ~'COND' */ -#define _PTHREAD_ATTR_SIG 0x54484441 /* 'THDA' */ -#define _PTHREAD_ONCE_SIG 0x4F4E4345 /* 'ONCE' */ -#define _PTHREAD_ONCE_SIG_init 0x30B1BCBA /* [almost] ~'ONCE' */ -#define _PTHREAD_SIG 0x54485244 /* 'THRD' */ -#define _PTHREAD_RWLOCK_ATTR_SIG 0x52574C41 /* 'RWLA' */ -#define _PTHREAD_RWLOCK_SIG 0x52574C4B /* 'RWLK' */ -#define _PTHREAD_RWLOCK_SIG_init 0x2DA8B3B4 /* [almost] ~'RWLK' */ - - -#define _PTHREAD_KERN_COND_SIG 0x12345678 /* */ -#define _PTHREAD_KERN_MUTEX_SIG 0x34567812 /* */ -#define _PTHREAD_KERN_RWLOCK_SIG 0x56781234 /* */ - -#if defined(DEBUG) -#define _PTHREAD_MUTEX_OWNER_SELF pthread_self() -#else -#define _PTHREAD_MUTEX_OWNER_SELF (pthread_t)0x12141968 -#endif -#define _PTHREAD_MUTEX_OWNER_SWITCHING (pthread_t)(~0) - -#define _PTHREAD_CANCEL_STATE_MASK 0x01 -#define _PTHREAD_CANCEL_TYPE_MASK 0x02 -#define _PTHREAD_CANCEL_PENDING 0x10 /* pthread_cancel() has been called for this thread */ - -extern boolean_t swtch_pri(int); - +#include "pthread.h" +#include "pthread_spis.h" +#include "inline_internal.h" #include "kern/kern_internal.h" -/* Prototypes. */ - -/* Internal globals. */ -PTHREAD_NOEXPORT void _pthread_tsd_cleanup(pthread_t self); - -PTHREAD_NOEXPORT int _pthread_mutex_droplock(_pthread_mutex *mutex, uint32_t * flagp, uint32_t ** pmtxp, uint32_t * mgenp, uint32_t * ugenp); - -/* internally redirected upcalls. */ -PTHREAD_NOEXPORT void* malloc(size_t); -PTHREAD_NOEXPORT void free(void*); - -/* syscall interfaces */ -extern uint32_t __psynch_mutexwait(_pthread_mutex * mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags); -extern uint32_t __psynch_mutexdrop(_pthread_mutex * mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags); - -extern uint32_t __psynch_cvbroad(pthread_cond_t * cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, pthread_mutex_t * mutex, uint64_t mugen, uint64_t tid); -extern uint32_t __psynch_cvsignal(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, pthread_mutex_t * mutex, uint64_t mugen, uint64_t tid, uint32_t flags); -extern uint32_t __psynch_cvwait(pthread_cond_t * cv, uint64_t cvlsgen, uint32_t cvugen, pthread_mutex_t * mutex, uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec); -extern uint32_t __psynch_cvclrprepost(void * cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags); -extern uint32_t __psynch_rw_longrdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); -extern uint32_t __psynch_rw_yieldwrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); -extern int __psynch_rw_downgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); -extern uint32_t __psynch_rw_upgrade(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); -extern uint32_t __psynch_rw_rdlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); -extern uint32_t __psynch_rw_wrlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); -extern uint32_t __psynch_rw_unlock(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); -extern uint32_t __psynch_rw_unlock2(pthread_rwlock_t * rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags); -extern uint32_t __bsdthread_ctl(uintptr_t cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3); - -PTHREAD_EXTERN -int -__proc_info(int callnum, int pid, int flavor, uint64_t arg, void * buffer, int buffersize); - -PTHREAD_NOEXPORT -void -_pthread_deallocate(pthread_t t, bool from_mach_thread); - -PTHREAD_NOEXPORT -thread_qos_t -_pthread_qos_class_to_thread_qos(qos_class_t qos); - -PTHREAD_NOEXPORT -void -_pthread_set_main_qos(pthread_priority_t qos); - -PTHREAD_NOEXPORT -void -_pthread_key_global_init(const char *envp[]); - -PTHREAD_NOEXPORT -void -_pthread_mutex_global_init(const char *envp[], struct _pthread_registration_data *registration_data); - -PTHREAD_EXPORT -void -_pthread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags); - -PTHREAD_EXPORT -void -_pthread_wqthread(pthread_t self, mach_port_t kport, void *stackaddr, void *keventlist, int flags, int nkevents); - -PTHREAD_NOEXPORT -void -_pthread_main_thread_init(pthread_t p); - -PTHREAD_NOEXPORT -void -_pthread_main_thread_postfork_init(pthread_t p); - -PTHREAD_NOEXPORT -void -_pthread_bsdthread_init(struct _pthread_registration_data *data); - -PTHREAD_NOEXPORT_VARIANT -void -_pthread_clear_qos_tsd(mach_port_t thread_port); - -#define PTHREAD_CONFORM_DARWIN_LEGACY 0 -#define PTHREAD_CONFORM_UNIX03_NOCANCEL 1 -#define PTHREAD_CONFORM_UNIX03_CANCELABLE 2 - -PTHREAD_NOEXPORT_VARIANT -void -_pthread_testcancel(int conforming); - -PTHREAD_EXPORT -void -_pthread_exit_if_canceled(int error); - -PTHREAD_NOEXPORT -void -_pthread_markcancel_if_canceled(pthread_t thread, mach_port_t kport); - -PTHREAD_NOEXPORT -void -_pthread_setcancelstate_exit(pthread_t self, void *value_ptr); - -PTHREAD_NOEXPORT -semaphore_t -_pthread_joiner_prepost_wake(pthread_t thread); - -PTHREAD_ALWAYS_INLINE -static inline mach_port_t -_pthread_kernel_thread(pthread_t t) -{ - return t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF]; -} - -PTHREAD_ALWAYS_INLINE -static inline void -_pthread_set_kernel_thread(pthread_t t, mach_port_t p) -{ - t->tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] = p; -} - -#ifdef DEBUG -#define PTHREAD_DEBUG_ASSERT(b) \ - do { \ - if (os_unlikely(!(b))) { \ - PTHREAD_INTERNAL_CRASH(0, "Assertion failed: " #b); \ - } \ - } while (0) -#else -#define PTHREAD_DEBUG_ASSERT(b) ((void)0) -#endif - -#include -#include - -struct pthread_atfork_entry { - void (*prepare)(void); - void (*parent)(void); - void (*child)(void); -}; - -#define PTHREAD_ATFORK_INLINE_MAX 10 -#define PTHREAD_ATFORK_MAX (vm_page_size/sizeof(struct pthread_atfork_entry)) - -struct pthread_globals_s { - // atfork.c - pthread_t psaved_self; - _pthread_lock psaved_self_global_lock; - _pthread_lock pthread_atfork_lock; - - size_t atfork_count; - struct pthread_atfork_entry atfork_storage[PTHREAD_ATFORK_INLINE_MAX]; - struct pthread_atfork_entry *atfork; - uint16_t qmp_logical[THREAD_QOS_LAST]; - uint16_t qmp_physical[THREAD_QOS_LAST]; - -}; -typedef struct pthread_globals_s *pthread_globals_t; - -__attribute__((__pure__)) -static inline pthread_globals_t -_pthread_globals(void) -{ - return os_alloc_once(OS_ALLOC_ONCE_KEY_LIBSYSTEM_PTHREAD, - sizeof(struct pthread_globals_s), - NULL); -} - -#pragma mark _pthread_mutex_check_signature - -PTHREAD_ALWAYS_INLINE -static inline bool -_pthread_mutex_check_signature_fast(_pthread_mutex *mutex) -{ - return (mutex->sig == _PTHREAD_MUTEX_SIG_fast); -} - -PTHREAD_ALWAYS_INLINE -static inline bool -_pthread_mutex_check_signature(_pthread_mutex *mutex) -{ - return ((mutex->sig & _PTHREAD_MUTEX_SIG_MASK) == _PTHREAD_MUTEX_SIG_CMP); -} - -PTHREAD_ALWAYS_INLINE -static inline bool -_pthread_mutex_check_signature_init(_pthread_mutex *mutex) -{ - return ((mutex->sig & _PTHREAD_MUTEX_SIG_init_MASK) == - _PTHREAD_MUTEX_SIG_init_CMP); -} - -#pragma mark _pthread_rwlock_check_signature - -PTHREAD_ALWAYS_INLINE -static inline bool -_pthread_rwlock_check_signature(_pthread_rwlock *rwlock) -{ - return (rwlock->sig == _PTHREAD_RWLOCK_SIG); -} - -PTHREAD_ALWAYS_INLINE -static inline bool -_pthread_rwlock_check_signature_init(_pthread_rwlock *rwlock) -{ - return (rwlock->sig == _PTHREAD_RWLOCK_SIG_init); -} - -PTHREAD_ALWAYS_INLINE -static inline void -_pthread_validate_signature(pthread_t thread) -{ - pthread_t th = (pthread_t)(thread->sig ^ _pthread_ptr_munge_token); -#if __has_feature(ptrauth_calls) - th = ptrauth_auth_data(th, ptrauth_key_process_dependent_data, - ptrauth_string_discriminator("pthread.signature")); -#endif - if (os_unlikely(th != thread)) { - /* OS_REASON_LIBSYSTEM_CODE_PTHREAD_CORRUPTION == 4 */ - abort_with_reason(OS_REASON_LIBSYSTEM, 4, "pthread_t was corrupted", 0); - } -} - -PTHREAD_ALWAYS_INLINE -static inline void -_pthread_init_signature(pthread_t thread) -{ - pthread_t th = thread; -#if __has_feature(ptrauth_calls) - th = ptrauth_sign_unauthenticated(th, ptrauth_key_process_dependent_data, - ptrauth_string_discriminator("pthread.signature")); -#endif - thread->sig = (uintptr_t)th ^ _pthread_ptr_munge_token; -} - -/* - * ALWAYS called without list lock and return with list lock held on success - * - * This weird calling convention exists because this function will sometimes - * drop the lock, and it's best callers don't have to remember this. - */ -PTHREAD_ALWAYS_INLINE -static inline bool -_pthread_validate_thread_and_list_lock(pthread_t thread) -{ - pthread_t p; - if (thread == NULL) return false; - _PTHREAD_LOCK(_pthread_list_lock); - TAILQ_FOREACH(p, &__pthread_head, tl_plist) { - if (p != thread) continue; - _pthread_validate_signature(p); - return true; - } - _PTHREAD_UNLOCK(_pthread_list_lock); - - return false; -} - -PTHREAD_ALWAYS_INLINE -static inline bool -_pthread_is_valid(pthread_t thread, mach_port_t *portp) -{ - mach_port_t kport = MACH_PORT_NULL; - bool valid; - - if (thread == pthread_self()) { - _pthread_validate_signature(thread); - valid = true; - kport = _pthread_kernel_thread(thread); - } else if (!_pthread_validate_thread_and_list_lock(thread)) { - valid = false; - } else { - kport = _pthread_kernel_thread(thread); - valid = true; - _PTHREAD_UNLOCK(_pthread_list_lock); - } - - if (portp != NULL) { - *portp = kport; - } - return valid; -} - -PTHREAD_ALWAYS_INLINE -static inline void* -_pthread_atomic_xchg_ptr_inline(void **p, void *v) -{ - return os_atomic_xchg(p, v, seq_cst); -} - -PTHREAD_ALWAYS_INLINE -static inline uint32_t -_pthread_atomic_xchg_uint32_relaxed_inline(uint32_t *p,uint32_t v) -{ - return os_atomic_xchg(p, v, relaxed); -} - -#define _pthread_atomic_xchg_ptr(p, v) \ - _pthread_atomic_xchg_ptr_inline(p, v) -#define _pthread_atomic_xchg_uint32_relaxed(p, v) \ - _pthread_atomic_xchg_uint32_relaxed_inline(p, v) #endif /* _POSIX_PTHREAD_INTERNALS_H */ diff --git a/src/mk_pthread_impl.c b/src/mk_pthread_impl.c deleted file mode 100644 index 8654c45..0000000 --- a/src/mk_pthread_impl.c +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved. - * - * @APPLE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_LICENSE_HEADER_END@ - */ -/* - * Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991 - * All Rights Reserved - * - * Permission to use, copy, modify, and distribute this software and - * its documentation for any purpose and without fee is hereby granted, - * provided that the above copyright notice appears in all copies and - * that both the copyright notice and this permission notice appear in - * supporting documentation. - * - * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE - * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE. - * - * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR - * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM - * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT, - * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION - * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - * - */ -/* - * MkLinux - */ - -/* - * This program will generate the stuff necessary to "publish" the POSIX - * header in a machine dependent fashion. - */ - -#include -#include - -int -main(void) -{ - printf("#ifndef _PTHREAD_IMPL_H_\n"); - printf("#define _PTHREAD_IMPL_H_\n"); - printf("/*\n"); - printf(" * Internal implementation details\n"); - printf(" */\n"); - printf("\n"); - printf("#define __PTHREAD_SIZE__ %zd\n", sizeof(struct _pthread)-sizeof(long)); - printf("#define __PTHREAD_ATTR_SIZE__ %zd\n", sizeof(pthread_attr_t)-sizeof(long)); - printf("#define __PTHREAD_MUTEXATTR_SIZE__ %zd\n", sizeof(pthread_mutexattr_t)-sizeof(long)); - printf("#define __PTHREAD_MUTEX_SIZE__ %zd\n", sizeof(pthread_mutex_t)-sizeof(long)); - printf("#define __PTHREAD_CONDATTR_SIZE__ %zd\n", sizeof(pthread_condattr_t)-sizeof(long)); - printf("#define __PTHREAD_COND_SIZE__ %zd\n", sizeof(pthread_cond_t)-sizeof(long)); - printf("#define __PTHREAD_ONCE_SIZE__ %zd\n", sizeof(pthread_once_t)-sizeof(long)); - printf("#define __PTHREAD_sig_OFFSET__ %zd\n", offsetof(struct _pthread, sig)); - printf("#define __PTHREAD_cleanup_stack_OFFSET__ %zd\n", offsetof(struct _pthread, __cleanup_stack)); - printf("#define __PTHREAD_guardsize_OFFSET__ %zd\n", offsetof(struct _pthread, guardsize)); - printf("#define __PTHREAD_param_OFFSET__ %zd\n", offsetof(struct _pthread, param)); - printf("#define __PTHREAD_mutexes_OFFSET__ %zd\n", offsetof(struct _pthread, mutexes)); - printf("#define __PTHREAD_joiner_OFFSET__ %zd\n", offsetof(struct _pthread, joiner)); - printf("#define __PTHREAD_exit_value_OFFSET__ %zd\n", offsetof(struct _pthread, exit_value)); - printf("#define __PTHREAD_death_OFFSET__ %zd\n", offsetof(struct _pthread, death)); - printf("#define __PTHREAD_kernel_thread_OFFSET__ %zd\n", offsetof(struct _pthread, kernel_thread)); - printf("#define __PTHREAD_fun_OFFSET__ %zd\n", offsetof(struct _pthread, fun)); - printf("#define __PTHREAD_arg_OFFSET__ %zd\n", offsetof(struct _pthread, arg)); - printf("#define __PTHREAD_cancel_state_OFFSET__ %zd\n", offsetof(struct _pthread, cancel_state)); - printf("#define __PTHREAD_err_no_OFFSET__ %zd\n", offsetof(struct _pthread, err_no)); - printf("#define __PTHREAD_tsd_OFFSET__ %zd\n", offsetof(struct _pthread, tsd)); - printf("#define __PTHREAD_stackaddr_OFFSET__ %zd\n", offsetof(struct _pthread, stackaddr)); - printf("#define __PTHREAD_stacksize_OFFSET__ %zd\n", offsetof(struct _pthread, stacksize)); - printf("#define __PTHREAD_reply_port_OFFSET__ %zd\n", offsetof(struct _pthread, reply_port)); - printf("#define __PTHREAD_cthread_self_OFFSET__ %zd\n", offsetof(struct _pthread, cthread_self)); - printf("#define __PTHREAD_freeStackOnExit_OFFSET__ %zd\n", offsetof(struct _pthread, freeStackOnExit)); - printf("#define __PTHREAD_plist_OFFSET__ %zd\n", offsetof(struct _pthread, plist)); - printf("/*\n"); - printf(" * [Internal] data structure signatures\n"); - printf(" */\n"); - printf("#define _PTHREAD_MUTEX_SIG_init 0x%08X\n", _PTHREAD_MUTEX_SIG_init); - printf("#define _PTHREAD_COND_SIG_init 0x%08X\n", _PTHREAD_COND_SIG_init); - printf("#define _PTHREAD_ONCE_SIG_init 0x%08X\n", _PTHREAD_ONCE_SIG_init); - printf("/*\n"); - printf(" * POSIX scheduling policies \n"); - printf(" */\n"); - printf("#define SCHED_OTHER %d\n", SCHED_OTHER); - printf("#define SCHED_FIFO %d\n", SCHED_FIFO); - printf("#define SCHED_RR %d\n", SCHED_RR); - printf("\n"); - printf("#define __SCHED_PARAM_SIZE__ %ld\n", (long) sizeof(struct sched_param)-sizeof(int)); - printf("\n"); - printf("#endif _PTHREAD_IMPL_H_\n"); - - exit(0); -} diff --git a/src/offsets.h b/src/offsets_internal.h similarity index 83% rename from src/offsets.h rename to src/offsets_internal.h index 0e20385..1f26c00 100644 --- a/src/offsets.h +++ b/src/offsets_internal.h @@ -24,18 +24,6 @@ #ifndef _POSIX_PTHREAD_OFFSETS_H #define _POSIX_PTHREAD_OFFSETS_H -#ifndef __ASSEMBLER__ -#define check_backward_offset(field, value) \ - _Static_assert(offsetof(struct _pthread, tsd) + value == \ - offsetof(struct _pthread, field), #value " is correct") -#define check_forward_offset(field, value) \ - _Static_assert(offsetof(struct _pthread, field) == value, \ - #value " is correct") -#else -#define check_backward_offset(field, value) -#define check_forward_offset(field, value) -#endif // __ASSEMBLER__ - #if defined(__i386__) #define _PTHREAD_STRUCT_DIRECT_STACKADDR_OFFSET 140 #define _PTHREAD_STRUCT_DIRECT_STACKBOTTOM_OFFSET 144 @@ -47,6 +35,18 @@ #define _PTHREAD_STRUCT_DIRECT_STACKBOTTOM_OFFSET -32 #endif +#ifndef __ASSEMBLER__ +#include "pthread/private.h" // for other _PTHREAD_STRUCT_DIRECT_*_OFFSET + +#define check_backward_offset(field, value) \ + _Static_assert(offsetof(struct pthread_s, tsd) + value == \ + offsetof(struct pthread_s, field), #value " is correct") +#define check_forward_offset(field, value) \ + _Static_assert(offsetof(struct pthread_s, field) == value, \ + #value " is correct") + +check_forward_offset(tsd, _PTHREAD_STRUCT_DIRECT_TSD_OFFSET); +check_backward_offset(thread_id, _PTHREAD_STRUCT_DIRECT_THREADID_OFFSET); #if defined(__i386__) check_forward_offset(stackaddr, _PTHREAD_STRUCT_DIRECT_STACKADDR_OFFSET); check_forward_offset(stackbottom, _PTHREAD_STRUCT_DIRECT_STACKBOTTOM_OFFSET); @@ -55,4 +55,6 @@ check_backward_offset(stackaddr, _PTHREAD_STRUCT_DIRECT_STACKADDR_OFFSET); check_backward_offset(stackbottom, _PTHREAD_STRUCT_DIRECT_STACKBOTTOM_OFFSET); #endif +#endif // __ASSEMBLER__ + #endif /* _POSIX_PTHREAD_OFFSETS_H */ diff --git a/src/prototypes_internal.h b/src/prototypes_internal.h new file mode 100644 index 0000000..8fc4d4c --- /dev/null +++ b/src/prototypes_internal.h @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2019 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#ifndef __LIBPTHREAD_PROTOTYPES_INTERNAL_H__ +#define __LIBPTHREAD_PROTOTYPES_INTERNAL_H__ + +/*! + * @file prototypes_internal.h + * + * @brief + * This file has prototypes for symbols / functions private to libpthread. + */ + +#define PTHREAD_NOEXPORT __attribute__((visibility("hidden"))) +#define PTHREAD_NOEXPORT_VARIANT + + +#pragma GCC visibility push(hidden) + +/*! + * @macro main_thread() + * + * @brief + * Returns a pointer to the main thread. + * + * @discussion + * The main thread structure really lives in dyld, + * and when __pthread_init() is called, its pointer will be discovered + * and stashed in _main_thread_ptr which libpthread uses. + */ +#if VARIANT_DYLD +extern struct pthread_s _main_thread; +#define main_thread() (&_main_thread) +#define __pthread_mutex_default_opt_policy _PTHREAD_MTX_OPT_POLICY_DEFAULT +#define __pthread_mutex_use_ulock _PTHREAD_MTX_OPT_ULOCK_DEFAULT +#define __pthread_mutex_ulock_adaptive_spin _PTHREAD_MTX_OPT_ADAPTIVE_DEFAULT +#else // VARIANT_DYLD +extern pthread_t _main_thread_ptr; +#define main_thread() (_main_thread_ptr) +extern void *(*_pthread_malloc)(size_t); +extern void (*_pthread_free)(void *); +extern int __pthread_mutex_default_opt_policy; +extern bool __pthread_mutex_use_ulock; +extern bool __pthread_mutex_ulock_adaptive_spin; +#endif // VARIANT_DYLD + +extern struct __pthread_list __pthread_head; // List of all pthreads in the process. +extern _pthread_lock _pthread_list_lock; // Lock protects access to above list. +extern uint32_t _main_qos; +extern uintptr_t _pthread_ptr_munge_token; + +#if PTHREAD_DEBUG_LOG +#include +extern int _pthread_debuglog; +extern uint64_t _pthread_debugstart; +#endif + +/* pthread.c */ +void _pthread_deallocate(pthread_t t, bool from_mach_thread); +void _pthread_main_thread_init(pthread_t p); +void _pthread_main_thread_postfork_init(pthread_t p); +void _pthread_bsdthread_init(struct _pthread_registration_data *data); +void *_pthread_atomic_xchg_ptr(void **p, void *v); +uint32_t _pthread_atomic_xchg_uint32_relaxed(uint32_t *p, uint32_t v); + +/* pthread_cancelable.c */ +void _pthread_markcancel_if_canceled(pthread_t thread, mach_port_t kport); +void _pthread_setcancelstate_exit(pthread_t self, void *value_ptr); +semaphore_t _pthread_joiner_prepost_wake(pthread_t thread); +int _pthread_join(pthread_t thread, void **value_ptr, pthread_conformance_t); + +/* pthread_cond.c */ +int _pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex, + const struct timespec *abstime, int isRelative, pthread_conformance_t); +int _pthread_mutex_droplock(pthread_mutex_t *mutex, uint32_t *flagp, + uint32_t **pmtxp, uint32_t *mgenp, uint32_t *ugenp); + +/* pthread_dependency.c */ +void _pthread_dependency_fulfill_slow(pthread_dependency_t *pr, uint32_t old); + +/* pthread_mutex.c */ +OS_COLD OS_NORETURN +int _pthread_mutex_corruption_abort(pthread_mutex_t *mutex); +int _pthread_mutex_fairshare_lock_slow(pthread_mutex_t *mutex, bool trylock); +int _pthread_mutex_fairshare_unlock_slow(pthread_mutex_t *mutex); +int _pthread_mutex_ulock_lock(pthread_mutex_t *mutex, bool trylock); +int _pthread_mutex_ulock_unlock(pthread_mutex_t *mutex); +int _pthread_mutex_firstfit_lock_slow(pthread_mutex_t *mutex, bool trylock); +int _pthread_mutex_firstfit_unlock_slow(pthread_mutex_t *mutex); +int _pthread_mutex_lock_init_slow(pthread_mutex_t *mutex, bool trylock); +void _pthread_mutex_global_init(const char *envp[], struct _pthread_registration_data *registration_data); + +/* pthread_rwlock.c */ +enum rwlock_seqfields; +int _pthread_rwlock_lock_slow(pthread_rwlock_t *rwlock, bool readlock, bool trylock); +int _pthread_rwlock_unlock_slow(pthread_rwlock_t *rwlock, enum rwlock_seqfields updated_seqfields); + +/* pthread_tsd.c */ +void _pthread_tsd_cleanup(pthread_t self); +void _pthread_key_global_init(const char *envp[]); + +/* qos.c */ +thread_qos_t _pthread_qos_class_to_thread_qos(qos_class_t qos); +void _pthread_set_main_qos(pthread_priority_t qos); + +#pragma GCC visibility pop + +#endif // __LIBPTHREAD_PROTOTYPES_INTERNAL_H__ diff --git a/src/pthread.c b/src/pthread.c index a8729bb..57ff8e9 100644 --- a/src/pthread.c +++ b/src/pthread.c @@ -49,15 +49,7 @@ * POSIX Pthread Library */ -#include "resolver.h" #include "internal.h" -#include "private.h" -#include "workqueue_private.h" -#include "introspection_private.h" -#include "qos_private.h" -#include "tsd_private.h" -#include "pthread/stack_np.h" -#include "offsets.h" // included to validate the offsets at build time #include #include @@ -78,32 +70,12 @@ #if __has_include() #include #endif // __has_include() - -#include <_simple.h> -#include -#include - -#include - -// Defined in libsyscall; initialized in libmalloc -extern malloc_logger_t *__syscall_logger; - -extern int __sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen); -extern void __exit(int) __attribute__((noreturn)); -extern int __pthread_kill(mach_port_t, int); - -extern void _pthread_joiner_wake(pthread_t thread); - -#if !VARIANT_DYLD -PTHREAD_NOEXPORT extern struct _pthread *_main_thread_ptr; -#define main_thread() (_main_thread_ptr) -#endif // VARIANT_DYLD +#include +#include // Default stack size is 512KB; independent of the main thread's stack size. #define DEFAULT_STACK_SIZE (size_t)(512 * 1024) - // // Global constants // @@ -114,7 +86,7 @@ PTHREAD_NOEXPORT extern struct _pthread *_main_thread_ptr; * start of the next page. There's also one page worth of allocation * below stacksize for the guard page. */ -#define PTHREAD_SIZE ((size_t)mach_vm_round_page(sizeof(struct _pthread))) +#define PTHREAD_SIZE ((size_t)mach_vm_round_page(sizeof(struct pthread_s))) #define PTHREAD_ALLOCADDR(stackaddr, stacksize) ((stackaddr - stacksize) - vm_page_size) #define PTHREAD_ALLOCSIZE(stackaddr, stacksize) ((round_page((uintptr_t)stackaddr) + PTHREAD_SIZE) - (uintptr_t)PTHREAD_ALLOCADDR(stackaddr, stacksize)) @@ -134,9 +106,9 @@ static const pthread_attr_t _pthread_attr_default = { const struct pthread_layout_offsets_s pthread_layout_offsets = { .plo_version = 1, - .plo_pthread_tsd_base_offset = offsetof(struct _pthread, tsd), + .plo_pthread_tsd_base_offset = offsetof(struct pthread_s, tsd), .plo_pthread_tsd_base_address_offset = 0, - .plo_pthread_tsd_entry_size = sizeof(((struct _pthread *)NULL)->tsd[0]), + .plo_pthread_tsd_entry_size = sizeof(((struct pthread_s *)NULL)->tsd[0]), }; #endif // PTHREAD_LAYOUT_SPI @@ -148,7 +120,7 @@ const struct pthread_layout_offsets_s pthread_layout_offsets = { // This global should be used (carefully) by anyone needing to know if a // pthread (other than the main thread) has been created. int __is_threaded = 0; -int __unix_conforming = 0; +const int __unix_conforming = 1; // we're always conformant, but it's exported // // Global internal variables @@ -163,10 +135,11 @@ uint32_t _main_qos; #if VARIANT_DYLD // The main thread's pthread_t -struct _pthread _main_thread __attribute__((aligned(64))) = { }; -#define main_thread() (&_main_thread) +struct pthread_s _main_thread OS_ALIGNED(64); #else // VARIANT_DYLD -struct _pthread *_main_thread_ptr; +pthread_t _main_thread_ptr; +void *(*_pthread_malloc)(size_t); +void (*_pthread_free)(void *); #endif // VARIANT_DYLD #if PTHREAD_DEBUG_LOG @@ -189,20 +162,16 @@ static int pthread_concurrency; uintptr_t _pthread_ptr_munge_token; static void (*exitf)(int) = __exit; -#if !VARIANT_DYLD -static void *(*_pthread_malloc)(size_t) = NULL; -static void (*_pthread_free)(void *) = NULL; -#endif // !VARIANT_DYLD // work queue support data -PTHREAD_NORETURN +OS_NORETURN OS_COLD static void __pthread_invalid_keventfunction(void **events, int *nevents) { PTHREAD_CLIENT_CRASH(0, "Invalid kqworkq setup"); } -PTHREAD_NORETURN +OS_NORETURN OS_COLD static void __pthread_invalid_workloopfunction(uint64_t *workloop_id, void **events, int *nevents) { @@ -234,9 +203,6 @@ static void _pthread_set_self_dyld(void); #endif // VARIANT_DYLD static inline void _pthread_set_self_internal(pthread_t); -static void _pthread_dealloc_reply_port(pthread_t t); -static void _pthread_dealloc_special_reply_port(pthread_t t); - static inline void __pthread_started_thread(pthread_t t); static void _pthread_exit(pthread_t self, void *value_ptr) __dead2; @@ -246,10 +212,6 @@ static inline void _pthread_introspection_thread_start(pthread_t t); static inline void _pthread_introspection_thread_terminate(pthread_t t); static inline void _pthread_introspection_thread_destroy(pthread_t t); -extern void _pthread_set_self(pthread_t); -extern void start_wqthread(pthread_t self, mach_port_t kport, void *stackaddr, void *unused, int reuse); // trampoline into _pthread_wqthread -extern void thread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags); // trampoline into _pthread_start - /* * Flags filed passed to bsdthread_create and back in pthread_start * 31 <---------------------------------> 0 @@ -268,25 +230,8 @@ extern void thread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *) #define PTHREAD_START_POLICY_MASK 0xff #define PTHREAD_START_IMPORTANCE_MASK 0xffff -extern pthread_t __bsdthread_create(void *(*func)(void *), void * func_arg, void * stack, pthread_t thread, unsigned int flags); -extern int __bsdthread_register(void (*)(pthread_t, mach_port_t, void *(*)(void *), void *, size_t, unsigned int), void (*)(pthread_t, mach_port_t, void *, void *, int), int,void (*)(pthread_t, mach_port_t, void *(*)(void *), void *, size_t, unsigned int), int32_t *,__uint64_t); -extern int __bsdthread_terminate(void * freeaddr, size_t freesize, mach_port_t kport, mach_port_t joinsem); -extern __uint64_t __thread_selfid( void ); - -#if __LP64__ -_Static_assert(offsetof(struct _pthread, tsd) == 224, "TSD LP64 offset"); -#else -_Static_assert(offsetof(struct _pthread, tsd) == 176, "TSD ILP32 offset"); -#endif -_Static_assert(offsetof(struct _pthread, tsd) + _PTHREAD_STRUCT_DIRECT_THREADID_OFFSET - == offsetof(struct _pthread, thread_id), - "_PTHREAD_STRUCT_DIRECT_THREADID_OFFSET is correct"); - #pragma mark pthread attrs -_Static_assert(sizeof(struct _pthread_attr_t) == sizeof(__darwin_pthread_attr_t), - "internal pthread_attr_t == external pthread_attr_t"); - int pthread_attr_destroy(pthread_attr_t *attr) { @@ -320,7 +265,7 @@ pthread_attr_getinheritsched(const pthread_attr_t *attr, int *inheritsched) return ret; } -static PTHREAD_ALWAYS_INLINE void +static OS_ALWAYS_INLINE void _pthread_attr_get_schedparam(const pthread_attr_t *attr, struct sched_param *param) { @@ -400,6 +345,8 @@ pthread_attr_setschedparam(pthread_attr_t *attr, const struct sched_param *param return ret; } +#define _PTHREAD_POLICY_IS_FIXEDPRI(x) ((x) == SCHED_RR || (x) == SCHED_FIFO) + int pthread_attr_setschedpolicy(pthread_attr_t *attr, int policy) { @@ -459,7 +406,7 @@ pthread_attr_setstackaddr(pthread_attr_t *attr, void *stackaddr) { int ret = EINVAL; if (attr->sig == _PTHREAD_ATTR_SIG && - ((uintptr_t)stackaddr % vm_page_size) == 0) { + ((mach_vm_address_t)stackaddr & vm_page_mask) == 0) { attr->stackaddr = stackaddr; attr->defaultguardpage = false; attr->guardsize = 0; @@ -488,9 +435,16 @@ pthread_attr_getstacksize(const pthread_attr_t *attr, size_t *stacksize) int pthread_attr_setstacksize(pthread_attr_t *attr, size_t stacksize) { +#if TARGET_OS_OSX + // If the caller is doing something reasonable, help them out. + if (stacksize % 0x1000 == 0) { + stacksize = round_page(stacksize); + } +#endif // TARGET_OS_OSX + int ret = EINVAL; if (attr->sig == _PTHREAD_ATTR_SIG && - (stacksize % vm_page_size) == 0 && + ((stacksize & vm_page_mask) == 0) && stacksize >= PTHREAD_STACK_MIN) { attr->stacksize = stacksize; ret = 0; @@ -517,8 +471,8 @@ pthread_attr_setstack(pthread_attr_t *attr, void *stackaddr, size_t stacksize) { int ret = EINVAL; if (attr->sig == _PTHREAD_ATTR_SIG && - ((uintptr_t)stackaddr % vm_page_size) == 0 && - (stacksize % vm_page_size) == 0 && + (((mach_vm_address_t)stackaddr & vm_page_mask) == 0) && + ((stacksize & vm_page_mask) == 0) && stacksize >= PTHREAD_STACK_MIN) { attr->stackaddr = (void *)((uintptr_t)stackaddr + stacksize); attr->stacksize = stacksize; @@ -530,8 +484,16 @@ pthread_attr_setstack(pthread_attr_t *attr, void *stackaddr, size_t stacksize) int pthread_attr_setguardsize(pthread_attr_t *attr, size_t guardsize) { +#if TARGET_OS_OSX + // If the caller is doing something reasonable, help them out. + if (guardsize % 0x1000 == 0) { + guardsize = round_page(guardsize); + } +#endif // TARGET_OS_OSX + int ret = EINVAL; - if (attr->sig == _PTHREAD_ATTR_SIG && (guardsize % vm_page_size) == 0) { + if (attr->sig == _PTHREAD_ATTR_SIG && + (guardsize & vm_page_mask) == 0) { /* Guardsize of 0 is valid, means no guard */ attr->defaultguardpage = false; attr->guardsize = guardsize; @@ -601,8 +563,8 @@ _pthread_allocate(const pthread_attr_t *attrs, void **stack, PTHREAD_CLIENT_CRASH(attrs->stacksize, "Stack size in attrs is too small"); } - if (os_unlikely(((uintptr_t)attrs->stackaddr % vm_page_size) != 0)) { - PTHREAD_CLIENT_CRASH(attrs->stacksize, "Unaligned stack addr in attrs"); + if (os_unlikely((mach_vm_address_t)attrs->stackaddr & vm_page_mask)) { + PTHREAD_CLIENT_CRASH(attrs->stackaddr, "Unaligned stack addr in attrs"); } // Allocate a pthread structure if necessary @@ -655,8 +617,8 @@ _pthread_allocate(const pthread_attr_t *attrs, void **stack, // the allocations. int eventTypeFlags = stack_logging_type_vm_allocate; __syscall_logger(eventTypeFlags | VM_MAKE_TAG(VM_MEMORY_STACK), - (uintptr_t)mach_task_self(), (uintptr_t)allocsize, 0, - (uintptr_t)allocaddr, 0); + (uintptr_t)mach_task_self(), (uintptr_t)allocsize, 0, + (uintptr_t)allocaddr, 0); } // The stack grows down. @@ -682,7 +644,7 @@ _pthread_allocate(const pthread_attr_t *attrs, void **stack, return t; } -PTHREAD_NOINLINE +OS_NOINLINE void _pthread_deallocate(pthread_t t, bool from_mach_thread) { @@ -703,7 +665,7 @@ _pthread_deallocate(pthread_t t, bool from_mach_thread) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wreturn-stack-address" -PTHREAD_NOINLINE +OS_NOINLINE static void* _pthread_current_stack_address(void) { @@ -713,7 +675,7 @@ _pthread_current_stack_address(void) #pragma clang diagnostic pop -void +static void _pthread_joiner_wake(pthread_t thread) { uint32_t *exit_gate = &thread->tl_exit_gate; @@ -729,8 +691,28 @@ _pthread_joiner_wake(pthread_t thread) } } +static void +_pthread_dealloc_reply_port(pthread_t self) +{ + mach_port_t port = _pthread_tsd_slot(self, MIG_REPLY); + if (port != MACH_PORT_NULL) { + // this will also set the TSD to MACH_PORT_NULL + mig_dealloc_reply_port(port); + } +} + +static void +_pthread_dealloc_special_reply_port(pthread_t self) +{ + mach_port_t port = _pthread_tsd_slot(self, MACH_SPECIAL_REPLY); + if (port != MACH_PORT_NULL) { + _pthread_tsd_slot(self, MACH_SPECIAL_REPLY) = MACH_PORT_NULL; + thread_destruct_special_reply_port(port, THREAD_SPECIAL_REPLY_PORT_ALL); + } +} + // Terminates the thread if called from the currently running thread. -PTHREAD_NORETURN PTHREAD_NOINLINE PTHREAD_NOT_TAIL_CALLED +OS_NORETURN OS_NOINLINE OS_NOT_TAIL_CALLED static void _pthread_terminate(pthread_t t, void *exit_value) { @@ -769,14 +751,14 @@ _pthread_terminate(pthread_t t, void *exit_value) freesize_stack = 0; } - mach_port_t kport = _pthread_kernel_thread(t); + mach_port_t kport = _pthread_tsd_slot(t, MACH_THREAD_SELF); bool keep_thread_struct = false, needs_wake = false; semaphore_t custom_stack_sema = MACH_PORT_NULL; _pthread_dealloc_special_reply_port(t); _pthread_dealloc_reply_port(t); - _PTHREAD_LOCK(_pthread_list_lock); + _pthread_lock_lock(&_pthread_list_lock); // This piece of code interacts with pthread_join. It will always: // - set tl_exit_gate to MACH_PORT_DEAD (thread exited) @@ -803,7 +785,7 @@ _pthread_terminate(pthread_t t, void *exit_value) TAILQ_REMOVE(&__pthread_head, t, tl_plist); } - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); if (needs_wake) { // When we found a waiter, we want to drop the very contended list lock @@ -814,12 +796,12 @@ _pthread_terminate(pthread_t t, void *exit_value) // - the exiting thread tries to set tl_joiner_cleans_up to true // Whoever does it first commits the other guy to cleanup the pthread_t _pthread_joiner_wake(t); - _PTHREAD_LOCK(_pthread_list_lock); + _pthread_lock_lock(&_pthread_list_lock); if (t->tl_join_ctx) { t->tl_joiner_cleans_up = true; keep_thread_struct = true; } - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); } // @@ -845,7 +827,7 @@ _pthread_terminate(pthread_t t, void *exit_value) PTHREAD_INTERNAL_CRASH(t, "thread didn't terminate"); } -PTHREAD_NORETURN +OS_NORETURN static void _pthread_terminate_invoke(pthread_t t, void *exit_value) { @@ -871,7 +853,6 @@ _pthread_terminate_invoke(pthread_t t, void *exit_value) #pragma mark pthread start / body -PTHREAD_NORETURN void _pthread_start(pthread_t self, mach_port_t kport, __unused void *(*fun)(void *), __unused void *arg, @@ -886,7 +867,7 @@ _pthread_start(pthread_t self, mach_port_t kport, "thread_set_tsd_base() wasn't called by the kernel"); } PTHREAD_DEBUG_ASSERT(MACH_PORT_VALID(kport)); - PTHREAD_DEBUG_ASSERT(_pthread_kernel_thread(self) == kport); + PTHREAD_DEBUG_ASSERT(_pthread_tsd_slot(self, MACH_THREAD_SELF) == kport); _pthread_validate_signature(self); _pthread_markcancel_if_canceled(self, kport); @@ -895,24 +876,24 @@ _pthread_start(pthread_t self, mach_port_t kport, _pthread_exit(self, (self->fun)(self->arg)); } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void _pthread_struct_init(pthread_t t, const pthread_attr_t *attrs, void *stackaddr, size_t stacksize, void *freeaddr, size_t freesize) { _pthread_init_signature(t); - t->tsd[_PTHREAD_TSD_SLOT_PTHREAD_SELF] = t; - t->tsd[_PTHREAD_TSD_SLOT_ERRNO] = &t->err_no; + _pthread_tsd_slot(t, PTHREAD_SELF) = t; + _pthread_tsd_slot(t, ERRNO) = &t->err_no; if (attrs->schedset == 0) { - t->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = attrs->qosclass; + _pthread_tsd_slot(t, PTHREAD_QOS_CLASS) = attrs->qosclass; } else { - t->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = + _pthread_tsd_slot(t, PTHREAD_QOS_CLASS) = _pthread_unspecified_priority(); } - t->tsd[_PTHREAD_TSD_SLOT_PTR_MUNGE] = _pthread_ptr_munge_token; + _pthread_tsd_slot(t, PTR_MUNGE) = _pthread_ptr_munge_token; t->tl_has_custom_stack = (attrs->stackaddr != NULL); - _PTHREAD_LOCK_INIT(t->lock); + _pthread_lock_init(&t->lock); t->stackaddr = stackaddr; t->stackbottom = stackaddr - stacksize; @@ -930,13 +911,6 @@ _pthread_struct_init(pthread_t t, const pthread_attr_t *attrs, #pragma mark pthread public interface -/* Need to deprecate this in future */ -int -_pthread_is_threaded(void) -{ - return __is_threaded; -} - /* Non portable public api to know whether this process has(had) atleast one thread * apart from main thread. There could be race if there is a thread in the process of * creation at the time of call . It does not tell whether there are more than one thread @@ -948,8 +922,6 @@ pthread_is_threaded_np(void) return __is_threaded; } - -PTHREAD_NOEXPORT_VARIANT mach_port_t pthread_mach_thread_np(pthread_t t) { @@ -958,27 +930,25 @@ pthread_mach_thread_np(pthread_t t) return kport; } -PTHREAD_NOEXPORT_VARIANT pthread_t pthread_from_mach_thread_np(mach_port_t kernel_thread) { - struct _pthread *p = NULL; + pthread_t p = NULL; /* No need to wait as mach port is already known */ - _PTHREAD_LOCK(_pthread_list_lock); + _pthread_lock_lock(&_pthread_list_lock); TAILQ_FOREACH(p, &__pthread_head, tl_plist) { - if (_pthread_kernel_thread(p) == kernel_thread) { + if (_pthread_tsd_slot(p, MACH_THREAD_SELF) == kernel_thread) { break; } } - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); return p; } -PTHREAD_NOEXPORT_VARIANT size_t pthread_get_stacksize_np(pthread_t t) { @@ -1032,7 +1002,7 @@ pthread_get_stacksize_np(pthread_t t) if (_pthread_validate_thread_and_list_lock(t)) { size = t->stackaddr - t->stackbottom;; - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); } out: @@ -1041,7 +1011,6 @@ out: return size ? size : DEFAULT_STACK_SIZE; } -PTHREAD_NOEXPORT_VARIANT void * pthread_get_stackaddr_np(pthread_t t) { @@ -1055,65 +1024,10 @@ pthread_get_stackaddr_np(pthread_t t) } void *addr = t->stackaddr; - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); return addr; } - -static mach_port_t -_pthread_reply_port(pthread_t t) -{ - void *p; - if (t == NULL) { - p = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_MIG_REPLY); - } else { - p = t->tsd[_PTHREAD_TSD_SLOT_MIG_REPLY]; - } - return (mach_port_t)(uintptr_t)p; -} - -static void -_pthread_set_reply_port(pthread_t t, mach_port_t reply_port) -{ - void *p = (void *)(uintptr_t)reply_port; - if (t == NULL) { - _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_MIG_REPLY, p); - } else { - t->tsd[_PTHREAD_TSD_SLOT_MIG_REPLY] = p; - } -} - -static void -_pthread_dealloc_reply_port(pthread_t t) -{ - mach_port_t reply_port = _pthread_reply_port(t); - if (reply_port != MACH_PORT_NULL) { - mig_dealloc_reply_port(reply_port); - } -} - -static mach_port_t -_pthread_special_reply_port(pthread_t t) -{ - void *p; - if (t == NULL) { - p = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_MACH_SPECIAL_REPLY); - } else { - p = t->tsd[_PTHREAD_TSD_SLOT_MACH_SPECIAL_REPLY]; - } - return (mach_port_t)(uintptr_t)p; -} - -static void -_pthread_dealloc_special_reply_port(pthread_t t) -{ - mach_port_t special_reply_port = _pthread_special_reply_port(t); - if (special_reply_port != MACH_PORT_NULL) { - thread_destruct_special_reply_port(special_reply_port, - THREAD_SPECIAL_REPLY_PORT_ALL); - } -} - pthread_t pthread_main_thread_np(void) { @@ -1127,12 +1041,11 @@ pthread_main_np(void) return pthread_self() == main_thread(); } - static int _pthread_threadid_slow(pthread_t thread, uint64_t *thread_id) { unsigned int info_count = THREAD_IDENTIFIER_INFO_COUNT; - mach_port_t thport = _pthread_kernel_thread(thread); + mach_port_t thport = _pthread_tsd_slot(thread, MACH_THREAD_SELF); struct thread_identifier_info info; kern_return_t kr; @@ -1140,7 +1053,11 @@ _pthread_threadid_slow(pthread_t thread, uint64_t *thread_id) (thread_info_t)&info, &info_count); if (kr == KERN_SUCCESS && info.thread_id) { *thread_id = info.thread_id; +#if __LP64__ os_atomic_store(&thread->thread_id, info.thread_id, relaxed); +#else + os_atomic_store_wide(&thread->thread_id, info.thread_id, relaxed); +#endif return 0; } return EINVAL; @@ -1151,7 +1068,6 @@ _pthread_threadid_slow(pthread_t thread, uint64_t *thread_id) * thread's thread_id. So folks don't have to call pthread_self, in addition to * us doing it, if they just want their thread_id. */ -PTHREAD_NOEXPORT_VARIANT int pthread_threadid_np(pthread_t thread, uint64_t *thread_id) { @@ -1167,18 +1083,33 @@ pthread_threadid_np(pthread_t thread, uint64_t *thread_id) } else if (!_pthread_validate_thread_and_list_lock(thread)) { res = ESRCH; } else { +#if __LP64__ *thread_id = os_atomic_load(&thread->thread_id, relaxed); +#else + *thread_id = os_atomic_load_wide(&thread->thread_id, relaxed); +#endif if (os_unlikely(*thread_id == 0)) { // there is a race at init because the thread sets its own TID. // correct this by asking mach res = _pthread_threadid_slow(thread, thread_id); } - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); } return res; } -PTHREAD_NOEXPORT_VARIANT +int +pthread_cpu_number_np(size_t *cpu_id) +{ + if (cpu_id == NULL) { + errno = EINVAL; + return errno; + } + + *cpu_id = _os_cpu_number(); + return 0; +} + int pthread_getname_np(pthread_t thread, char *threadname, size_t len) { @@ -1192,11 +1123,10 @@ pthread_getname_np(pthread_t thread, char *threadname, size_t len) } strlcpy(threadname, thread->pthread_name, len); - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); return 0; } - int pthread_setname_np(const char *name) { @@ -1210,7 +1140,6 @@ pthread_setname_np(const char *name) _pthread_validate_signature(self); - /* protytype is in pthread_internals.h */ res = __proc_info(5, getpid(), 2, (uint64_t)0, (void*)name, (int)len); if (res == 0) { if (len > 0) { @@ -1223,24 +1152,33 @@ pthread_setname_np(const char *name) } -PTHREAD_ALWAYS_INLINE -static inline void -__pthread_add_thread(pthread_t t, bool from_mach_thread) +void +pthread_jit_write_protect_np(int enable) { - if (from_mach_thread) { - _PTHREAD_LOCK_FROM_MACH_THREAD(_pthread_list_lock); - } else { - _PTHREAD_LOCK(_pthread_list_lock); - } + if (!os_thread_self_restrict_rwx_is_supported()) { + return; + } + + if (enable) { + os_thread_self_restrict_rwx_to_rx(); + } else { + os_thread_self_restrict_rwx_to_rw(); + } +} + +int pthread_jit_write_protect_supported_np() +{ + return os_thread_self_restrict_rwx_is_supported(); +} +OS_ALWAYS_INLINE +static inline void +__pthread_add_thread(pthread_t t, mach_port_t self, bool from_mach_thread) +{ + _pthread_lock_lock(&_pthread_list_lock, self); TAILQ_INSERT_TAIL(&__pthread_head, t, tl_plist); _pthread_count++; - - if (from_mach_thread) { - _PTHREAD_UNLOCK_FROM_MACH_THREAD(_pthread_list_lock); - } else { - _PTHREAD_UNLOCK(_pthread_list_lock); - } + _pthread_lock_unlock(&_pthread_list_lock, self); if (!from_mach_thread) { // PR-26275485: Mach threads will likely crash trying to run @@ -1251,31 +1189,21 @@ __pthread_add_thread(pthread_t t, bool from_mach_thread) } } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void -__pthread_undo_add_thread(pthread_t t, bool from_mach_thread) +__pthread_undo_add_thread(pthread_t t, mach_port_t self) { - if (from_mach_thread) { - _PTHREAD_LOCK_FROM_MACH_THREAD(_pthread_list_lock); - } else { - _PTHREAD_LOCK(_pthread_list_lock); - } - + _pthread_lock_lock(&_pthread_list_lock, self); TAILQ_REMOVE(&__pthread_head, t, tl_plist); _pthread_count--; - - if (from_mach_thread) { - _PTHREAD_UNLOCK_FROM_MACH_THREAD(_pthread_list_lock); - } else { - _PTHREAD_UNLOCK(_pthread_list_lock); - } + _pthread_lock_unlock(&_pthread_list_lock, self); } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void __pthread_started_thread(pthread_t t) { - mach_port_t kport = _pthread_kernel_thread(t); + mach_port_t kport = _pthread_tsd_slot(t, MACH_THREAD_SELF); if (os_unlikely(!MACH_PORT_VALID(kport))) { PTHREAD_CLIENT_CRASH(kport, "Unable to allocate thread port, possible port leak"); @@ -1294,6 +1222,8 @@ _pthread_create(pthread_t *thread, const pthread_attr_t *attrs, pthread_t t = NULL; void *stack = NULL; bool from_mach_thread = (create_flags & _PTHREAD_CREATE_FROM_MACH_THREAD); + mach_port_t self_kport; + int rc = 0; if (attrs == NULL) { attrs = &_pthread_attr_default; @@ -1318,14 +1248,20 @@ _pthread_create(pthread_t *thread, const pthread_attr_t *attrs, __is_threaded = 1; - t =_pthread_allocate(attrs, &stack, from_mach_thread); + t = _pthread_allocate(attrs, &stack, from_mach_thread); if (t == NULL) { return EAGAIN; } + if (os_unlikely(from_mach_thread)) { + self_kport = mach_thread_self(); + } else { + self_kport = _pthread_mach_thread_self_direct(); + } + t->arg = arg; t->fun = start_routine; - __pthread_add_thread(t, from_mach_thread); + __pthread_add_thread(t, self_kport, from_mach_thread); if (__bsdthread_create(start_routine, arg, stack, t, flags) == (pthread_t)-1) { @@ -1333,14 +1269,18 @@ _pthread_create(pthread_t *thread, const pthread_attr_t *attrs, PTHREAD_CLIENT_CRASH(0, "Unable to allocate thread port, possible port leak"); } - __pthread_undo_add_thread(t, from_mach_thread); + __pthread_undo_add_thread(t, self_kport); _pthread_deallocate(t, from_mach_thread); - return EAGAIN; + t = NULL; + rc = EAGAIN; + } + if (from_mach_thread) { + mach_port_deallocate(mach_task_self(), self_kport); } // n.b. if a thread is created detached and exits, t will be invalid *thread = t; - return 0; + return rc; } int @@ -1367,8 +1307,6 @@ pthread_create_suspended_np(pthread_t *thread, const pthread_attr_t *attr, return _pthread_create(thread, attr, start_routine, arg, flags); } - -PTHREAD_NOEXPORT_VARIANT int pthread_detach(pthread_t thread) { @@ -1391,7 +1329,7 @@ pthread_detach(pthread_t thread) wake = true; } } - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); if (join) { pthread_join(thread, NULL); @@ -1401,7 +1339,6 @@ pthread_detach(pthread_t thread) return res; } -PTHREAD_NOEXPORT_VARIANT int pthread_kill(pthread_t th, int sig) { @@ -1424,7 +1361,6 @@ pthread_kill(pthread_t th, int sig) return ret; } -PTHREAD_NOEXPORT_VARIANT int __pthread_workqueue_setkill(int enable) { @@ -1433,21 +1369,11 @@ __pthread_workqueue_setkill(int enable) } } - -/* For compatibility... */ - -pthread_t -_pthread_self(void) -{ - return pthread_self(); -} - /* * Terminate a thread. */ -extern int __disable_threadsignal(int); -PTHREAD_NORETURN +OS_NORETURN static void _pthread_exit(pthread_t self, void *exit_value) { @@ -1483,8 +1409,13 @@ pthread_exit(void *exit_value) _pthread_exit(self, exit_value); } +int +pthread_self_is_exiting_np(void) +{ + return (os_atomic_load(&pthread_self()->cancel_state, relaxed) & + _PTHREAD_CANCEL_EXITING) != 0; +} -PTHREAD_NOEXPORT_VARIANT int pthread_getschedparam(pthread_t thread, int *policy, struct sched_param *param) { @@ -1494,13 +1425,11 @@ pthread_getschedparam(pthread_t thread, int *policy, struct sched_param *param) if (policy) *policy = thread->tl_policy; if (param) *param = thread->tl_param; - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); return 0; } - - -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int pthread_setschedparam_internal(pthread_t thread, mach_port_t kport, int policy, const struct sched_param *param) @@ -1510,32 +1439,35 @@ pthread_setschedparam_internal(pthread_t thread, mach_port_t kport, int policy, mach_msg_type_number_t count; kern_return_t ret; + if (os_unlikely(thread->wqthread)) { + return ENOTSUP; + } + switch (policy) { - case SCHED_OTHER: - bases.ts.base_priority = param->sched_priority; - base = (policy_base_t)&bases.ts; - count = POLICY_TIMESHARE_BASE_COUNT; - break; - case SCHED_FIFO: - bases.fifo.base_priority = param->sched_priority; - base = (policy_base_t)&bases.fifo; - count = POLICY_FIFO_BASE_COUNT; - break; - case SCHED_RR: - bases.rr.base_priority = param->sched_priority; - /* quantum isn't public yet */ - bases.rr.quantum = param->quantum; - base = (policy_base_t)&bases.rr; - count = POLICY_RR_BASE_COUNT; - break; - default: - return EINVAL; + case SCHED_OTHER: + bases.ts.base_priority = param->sched_priority; + base = (policy_base_t)&bases.ts; + count = POLICY_TIMESHARE_BASE_COUNT; + break; + case SCHED_FIFO: + bases.fifo.base_priority = param->sched_priority; + base = (policy_base_t)&bases.fifo; + count = POLICY_FIFO_BASE_COUNT; + break; + case SCHED_RR: + bases.rr.base_priority = param->sched_priority; + /* quantum isn't public yet */ + bases.rr.quantum = param->quantum; + base = (policy_base_t)&bases.rr; + count = POLICY_RR_BASE_COUNT; + break; + default: + return EINVAL; } ret = thread_policy(kport, policy, base, count, TRUE); return (ret != KERN_SUCCESS) ? EINVAL : 0; } -PTHREAD_NOEXPORT_VARIANT int pthread_setschedparam(pthread_t t, int policy, const struct sched_param *param) { @@ -1545,7 +1477,7 @@ pthread_setschedparam(pthread_t t, int policy, const struct sched_param *param) // since the main thread will not get de-allocated from underneath us if (t == pthread_self() || t == main_thread()) { _pthread_validate_signature(t); - kport = _pthread_kernel_thread(t); + kport = _pthread_tsd_slot(t, MACH_THREAD_SELF); } else { bypass = 0; if (!_pthread_is_valid(t, &kport)) { @@ -1557,7 +1489,7 @@ pthread_setschedparam(pthread_t t, int policy, const struct sched_param *param) if (res) return res; if (bypass) { - _PTHREAD_LOCK(_pthread_list_lock); + _pthread_lock_lock(&_pthread_list_lock); } else if (!_pthread_validate_thread_and_list_lock(t)) { // Ensure the thread is still valid. return ESRCH; @@ -1565,11 +1497,10 @@ pthread_setschedparam(pthread_t t, int policy, const struct sched_param *param) t->tl_policy = policy; t->tl_param = *param; - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); return 0; } - int sched_get_priority_min(int policy) { @@ -1588,11 +1519,7 @@ pthread_equal(pthread_t t1, pthread_t t2) return (t1 == t2); } -/* - * Force LLVM not to optimise this to a call to __pthread_set_self, if it does - * then _pthread_set_self won't be bound when secondary threads try and start up. - */ -PTHREAD_NOINLINE +OS_NOINLINE void _pthread_set_self(pthread_t p) { @@ -1609,8 +1536,8 @@ _pthread_set_self(pthread_t p) // _pthread_set_self_dyld is noinline+noexport to allow the option for // static libsyscall to adopt this as the entry point from mach_init if // desired -PTHREAD_NOINLINE PTHREAD_NOEXPORT -void +OS_NOINLINE +static void _pthread_set_self_dyld(void) { pthread_t p = main_thread(); @@ -1625,26 +1552,29 @@ _pthread_set_self_dyld(void) // this, TSD access will fail and crash if it uses bits of Libc prior to // library initialization. __pthread_init will finish the initialization // during library init. - p->tsd[_PTHREAD_TSD_SLOT_PTHREAD_SELF] = p; - p->tsd[_PTHREAD_TSD_SLOT_ERRNO] = &p->err_no; + _pthread_tsd_slot(p, PTHREAD_SELF) = p; + _pthread_tsd_slot(p, ERRNO) = &p->err_no; _thread_set_tsd_base(&p->tsd[0]); } #endif // VARIANT_DYLD -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void _pthread_set_self_internal(pthread_t p) { +#if __LP64__ os_atomic_store(&p->thread_id, __thread_selfid(), relaxed); +#else + os_atomic_store_wide(&p->thread_id, __thread_selfid(), relaxed); +#endif if (os_unlikely(p->thread_id == -1ull)) { PTHREAD_INTERNAL_CRASH(0, "failed to set thread_id"); } } - // pthread_once should have an acquire barrier -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void _os_once_acquire(os_once_t *predicate, void *context, os_function_t function) { @@ -1669,7 +1599,6 @@ __pthread_once_handler(void *context) ctx->pthread_once->sig = _PTHREAD_ONCE_SIG; } -PTHREAD_NOEXPORT_VARIANT int pthread_once(pthread_once_t *once_control, void (*init_routine)(void)) { @@ -1680,7 +1609,6 @@ pthread_once(pthread_once_t *once_control, void (*init_routine)(void)) return 0; } - int pthread_getconcurrency(void) { @@ -1697,30 +1625,9 @@ pthread_setconcurrency(int new_level) return 0; } -#if !defined(VARIANT_STATIC) -void * -malloc(size_t sz) -{ - if (_pthread_malloc) { - return _pthread_malloc(sz); - } else { - return NULL; - } -} - -void -free(void *p) -{ - if (_pthread_free) { - _pthread_free(p); - } -} -#endif // VARIANT_STATIC - /* * Perform package initialization - called automatically when application starts */ -struct ProgramVars; /* forward reference */ #if !VARIANT_DYLD static unsigned long @@ -1791,6 +1698,14 @@ parse_ptr_munge_params(const char *envp[], const char *apple[]) token = _pthread_strtoul(p, &s, 16); bzero((char *)p, strlen(p)); } + /* + * In DEBUG we allow the environment variable to override the kernel + * security setting, including setting it to 0 which is helpful during + * debugging sessions. + * + * For other cases, the token must be set by the kernel or the environment + * variable to a non 0 value. + */ #if !DEBUG if (!token) { #endif @@ -1805,7 +1720,7 @@ parse_ptr_munge_params(const char *envp[], const char *apple[]) if (!token) { PTHREAD_INTERNAL_CRASH(token, "Token from the kernel is 0"); } -#endif // DEBUG +#endif // !DEBUG _pthread_ptr_munge_token = token; // we need to refresh the main thread signature now that we changed @@ -1831,8 +1746,7 @@ __pthread_init(const struct _libpthread_functions *pthread_funcs, // libpthread.a in dyld "owns" the main thread structure itself and sets // up the tsd to point to it. So take the pthread_self() from there // and make it our main thread point. - pthread_t thread = (pthread_t)_pthread_getspecific_direct( - _PTHREAD_TSD_SLOT_PTHREAD_SELF); + pthread_t thread = _pthread_self_direct(); if (os_unlikely(thread == NULL)) { PTHREAD_INTERNAL_CRASH(0, "PTHREAD_SELF TSD not initialized"); } @@ -1923,19 +1837,19 @@ __pthread_init(const struct _libpthread_functions *pthread_funcs, } #endif // !VARIANT_DYLD -PTHREAD_NOEXPORT void +void _pthread_main_thread_init(pthread_t p) { TAILQ_INIT(&__pthread_head); - _PTHREAD_LOCK_INIT(_pthread_list_lock); - _PTHREAD_LOCK_INIT(p->lock); - _pthread_set_kernel_thread(p, mach_thread_self()); - _pthread_set_reply_port(p, mach_reply_port()); + _pthread_lock_init(&_pthread_list_lock); + _pthread_lock_init(&p->lock); p->__cleanup_stack = NULL; p->tl_join_ctx = NULL; p->tl_exit_gate = MACH_PORT_NULL; - p->tsd[__TSD_SEMAPHORE_CACHE] = (void*)(uintptr_t)SEMAPHORE_NULL; - p->tsd[__TSD_MACH_SPECIAL_REPLY] = 0; + _pthread_tsd_slot(p, MACH_THREAD_SELF) = mach_thread_self(); + _pthread_tsd_slot(p, MIG_REPLY) = mach_reply_port(); + _pthread_tsd_slot(p, MACH_SPECIAL_REPLY) = MACH_PORT_NULL; + _pthread_tsd_slot(p, SEMAPHORE_CACHE) = SEMAPHORE_NULL; // Initialize the list of threads with the new main thread. TAILQ_INSERT_HEAD(&__pthread_head, p, tl_plist); @@ -1944,7 +1858,6 @@ _pthread_main_thread_init(pthread_t p) _pthread_introspection_thread_start(p); } -PTHREAD_NOEXPORT void _pthread_main_thread_postfork_init(pthread_t p) { @@ -1959,19 +1872,6 @@ sched_yield(void) return 0; } -// XXX remove -void -cthread_yield(void) -{ - sched_yield(); -} - -void -pthread_yield_np(void) -{ - sched_yield(); -} - // Libsystem knows about this symbol and exports it to libsyscall int pthread_current_stack_contains_np(const void *addr, size_t length) @@ -1991,39 +1891,35 @@ pthread_current_stack_contains_np(const void *addr, size_t length) return stack_base <= begin && end <= stack_top; } - - // Libsystem knows about this symbol and exports it to libsyscall -PTHREAD_NOEXPORT_VARIANT + void -_pthread_clear_qos_tsd(mach_port_t thread_port) +_pthread_clear_qos_tsd(mach_port_t port) { - if (thread_port == MACH_PORT_NULL || (uintptr_t)_pthread_getspecific_direct(_PTHREAD_TSD_SLOT_MACH_THREAD_SELF) == thread_port) { + pthread_priority_t pp = _pthread_unspecified_priority(); + + if (port == MACH_PORT_NULL || _pthread_mach_thread_self_direct() == port) { /* Clear the current thread's TSD, that can be done inline. */ - _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, - _pthread_unspecified_priority()); + _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, pp); } else { pthread_t p; - _PTHREAD_LOCK(_pthread_list_lock); + _pthread_lock_lock(&_pthread_list_lock); TAILQ_FOREACH(p, &__pthread_head, tl_plist) { - mach_port_t kp = _pthread_kernel_thread(p); - if (thread_port == kp) { - p->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = - _pthread_unspecified_priority(); + mach_port_t kp = _pthread_tsd_slot(p, MACH_THREAD_SELF); + if (port == kp) { + _pthread_tsd_slot(p, PTHREAD_QOS_CLASS) = pp; break; } } - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); } } - #pragma mark pthread/stack_np.h public interface - #if defined(__i386__) || defined(__x86_64__) || defined(__arm__) || defined(__arm64__) #if __ARM64_ARCH_8_32__ /* @@ -2064,19 +1960,18 @@ pthread_stack_frame_decode_np(uintptr_t frame_addr, uintptr_t *return_addr) return (uintptr_t)frame->frame_addr_next; } - #pragma mark pthread workqueue support routines - -PTHREAD_NOEXPORT void +void _pthread_bsdthread_init(struct _pthread_registration_data *data) { bzero(data, sizeof(*data)); data->version = sizeof(struct _pthread_registration_data); data->dispatch_queue_offset = __PTK_LIBDISPATCH_KEY0 * sizeof(void *); data->return_to_kernel_offset = __TSD_RETURN_TO_KERNEL * sizeof(void *); - data->tsd_offset = offsetof(struct _pthread, tsd); + data->tsd_offset = offsetof(struct pthread_s, tsd); data->mach_thread_self_offset = __TSD_MACH_THREAD_SELF * sizeof(void *); + data->joinable_offset_bits = CHAR_BIT * (offsetof(struct pthread_s, tl_policy) + 1); int rv = __bsdthread_register(thread_start, start_wqthread, (int)PTHREAD_SIZE, (void*)data, (uintptr_t)sizeof(*data), data->dispatch_queue_offset); @@ -2107,7 +2002,7 @@ _pthread_bsdthread_init(struct _pthread_registration_data *data) if (_pthread_priority_thread_qos(main_qos) != THREAD_QOS_UNSPECIFIED) { _pthread_set_main_qos(main_qos); - main_thread()->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = main_qos; + _pthread_tsd_slot(main_thread(), PTHREAD_QOS_CLASS) = main_qos; } if (data->stack_addr_hint) { @@ -2120,7 +2015,7 @@ _pthread_bsdthread_init(struct _pthread_registration_data *data) } } -PTHREAD_NOINLINE +OS_NOINLINE static void _pthread_wqthread_legacy_worker_wrap(pthread_priority_t pp) { @@ -2151,7 +2046,7 @@ _pthread_wqthread_legacy_worker_wrap(pthread_priority_t pp) PTHREAD_INTERNAL_CRASH(pp, "Invalid pthread priority for the legacy interface"); } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline pthread_priority_t _pthread_wqthread_priority(int flags) { @@ -2180,7 +2075,7 @@ _pthread_wqthread_priority(int flags) return pp; } -PTHREAD_NOINLINE +OS_NOINLINE static void _pthread_wqthread_setup(pthread_t self, mach_port_t kport, void *stacklowaddr, int flags) @@ -2192,7 +2087,7 @@ _pthread_wqthread_setup(pthread_t self, mach_port_t kport, void *stacklowaddr, PTHREAD_ALLOCADDR(stackaddr, stacksize), PTHREAD_ALLOCSIZE(stackaddr, stacksize)); - _pthread_set_kernel_thread(self, kport); + _pthread_tsd_slot(self, MACH_THREAD_SELF) = kport; self->wqthread = 1; self->wqkillset = 0; self->tl_joinable = false; @@ -2203,23 +2098,23 @@ _pthread_wqthread_setup(pthread_t self, mach_port_t kport, void *stacklowaddr, "thread_set_tsd_base() wasn't called by the kernel"); } _pthread_set_self_internal(self); - __pthread_add_thread(self, false); + __pthread_add_thread(self, kport, false); __pthread_started_thread(self); } -PTHREAD_NORETURN PTHREAD_NOINLINE +OS_NORETURN OS_NOINLINE static void _pthread_wqthread_exit(pthread_t self) { - pthread_priority_t pp; + const thread_qos_t WORKQ_THREAD_QOS_CLEANUP = THREAD_QOS_LEGACY; + pthread_priority_t pp = _pthread_tsd_slot(self, PTHREAD_QOS_CLASS); thread_qos_t qos; - pp = (pthread_priority_t)self->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS]; qos = _pthread_priority_thread_qos(pp); if (qos == THREAD_QOS_UNSPECIFIED || qos > WORKQ_THREAD_QOS_CLEANUP) { // Reset QoS to something low for the cleanup process pp = _pthread_priority_make_from_thread_qos(WORKQ_THREAD_QOS_CLEANUP, 0, 0); - self->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = (void *)pp; + _pthread_tsd_slot(self, PTHREAD_QOS_CLASS) = pp; } _pthread_exit(self, NULL); @@ -2283,10 +2178,8 @@ _pthread_wqthread(pthread_t self, mach_port_t kport, void *stacklowaddr, */ } - #pragma mark pthread workqueue API for libdispatch - _Static_assert(WORKQ_KEVENT_EVENT_BUFFER_LEN == WQ_KEVENT_LIST_LEN, "Kernel and userland should agree on the event list size"); @@ -2308,15 +2201,15 @@ pthread_workqueue_setup(struct pthread_workqueue_config *cfg, size_t cfg_size) } switch (cfg->version) { - case 1: - min_size = offsetof(struct pthread_workqueue_config, queue_label_offs); - break; - case 2: - min_size = sizeof(struct pthread_workqueue_config); - break; - default: - return EINVAL; - } + case 1: + min_size = offsetof(struct pthread_workqueue_config, queue_label_offs); + break; + case 2: + min_size = sizeof(struct pthread_workqueue_config); + break; + default: + return EINVAL; + } if (!cfg || cfg_size < min_size) { return EINVAL; @@ -2549,10 +2442,8 @@ _pthread_workloop_destroy(uint64_t workloop_id) return res; } - #pragma mark Introspection SPI for libpthread. - static pthread_introspection_hook_t _pthread_introspection_hook; pthread_introspection_hook_t @@ -2563,11 +2454,22 @@ pthread_introspection_hook_install(pthread_introspection_hook_t hook) return prev; } -PTHREAD_NOINLINE +static inline void +_pthread_introspection_call_hook(unsigned int event, + pthread_t thread, void *addr, size_t size) +{ + pthread_t self = pthread_self(); + uint16_t old = self->introspection; + self->introspection = (uint16_t)event; + _pthread_introspection_hook(event, thread, addr, size); + self->introspection = old; +} + +OS_NOINLINE static void _pthread_introspection_hook_callout_thread_create(pthread_t t) { - _pthread_introspection_hook(PTHREAD_INTROSPECTION_THREAD_CREATE, t, t, + _pthread_introspection_call_hook(PTHREAD_INTROSPECTION_THREAD_CREATE, t, t, PTHREAD_SIZE); } @@ -2578,7 +2480,7 @@ _pthread_introspection_thread_create(pthread_t t) _pthread_introspection_hook_callout_thread_create(t); } -PTHREAD_NOINLINE +OS_NOINLINE static void _pthread_introspection_hook_callout_thread_start(pthread_t t) { @@ -2592,7 +2494,7 @@ _pthread_introspection_hook_callout_thread_start(pthread_t t) freesize = t->freesize - PTHREAD_SIZE; freeaddr = t->freeaddr; } - _pthread_introspection_hook(PTHREAD_INTROSPECTION_THREAD_START, t, + _pthread_introspection_call_hook(PTHREAD_INTROSPECTION_THREAD_START, t, freeaddr, freesize); } @@ -2603,7 +2505,7 @@ _pthread_introspection_thread_start(pthread_t t) _pthread_introspection_hook_callout_thread_start(t); } -PTHREAD_NOINLINE +OS_NOINLINE static void _pthread_introspection_hook_callout_thread_terminate(pthread_t t) { @@ -2617,7 +2519,7 @@ _pthread_introspection_hook_callout_thread_terminate(pthread_t t) freesize = t->freesize - PTHREAD_SIZE; freeaddr = t->freeaddr; } - _pthread_introspection_hook(PTHREAD_INTROSPECTION_THREAD_TERMINATE, t, + _pthread_introspection_call_hook(PTHREAD_INTROSPECTION_THREAD_TERMINATE, t, freeaddr, freesize); } @@ -2628,11 +2530,11 @@ _pthread_introspection_thread_terminate(pthread_t t) _pthread_introspection_hook_callout_thread_terminate(t); } -PTHREAD_NOINLINE +OS_NOINLINE static void _pthread_introspection_hook_callout_thread_destroy(pthread_t t) { - _pthread_introspection_hook(PTHREAD_INTROSPECTION_THREAD_DESTROY, t, t, + _pthread_introspection_call_hook(PTHREAD_INTROSPECTION_THREAD_DESTROY, t, t, PTHREAD_SIZE); } @@ -2643,9 +2545,8 @@ _pthread_introspection_thread_destroy(pthread_t t) _pthread_introspection_hook_callout_thread_destroy(t); } - -#if !VARIANT_DYLD #pragma mark libplatform shims +#if !VARIANT_DYLD #include diff --git a/src/pthread_asm.s b/src/pthread_asm.s index 2a02f09..efa5666 100644 --- a/src/pthread_asm.s +++ b/src/pthread_asm.s @@ -21,7 +21,7 @@ * @APPLE_LICENSE_HEADER_END@ */ -#include "offsets.h" +#include "offsets_internal.h" #if defined(__x86_64__) diff --git a/src/pthread_atfork.c b/src/pthread_atfork.c index edeec16..150c504 100644 --- a/src/pthread_atfork.c +++ b/src/pthread_atfork.c @@ -25,8 +25,8 @@ #include #include -#include +#if !VARIANT_DYLD int pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void)) { @@ -34,7 +34,7 @@ pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void)) size_t idx; pthread_globals_t globals = _pthread_globals(); - _PTHREAD_LOCK(globals->pthread_atfork_lock); + _pthread_lock_lock(&globals->pthread_atfork_lock); idx = globals->atfork_count++; if (idx == 0) { @@ -45,7 +45,7 @@ pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void)) kern_return_t kr; mach_vm_address_t storage = 0; mach_vm_size_t size = PTHREAD_ATFORK_MAX * sizeof(struct pthread_atfork_entry); - _PTHREAD_UNLOCK(globals->pthread_atfork_lock); + _pthread_lock_unlock(&globals->pthread_atfork_lock); kr = mach_vm_map(mach_task_self(), &storage, size, @@ -57,7 +57,7 @@ pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void)) VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT); - _PTHREAD_LOCK(globals->pthread_atfork_lock); + _pthread_lock_lock(&globals->pthread_atfork_lock); if (kr == KERN_SUCCESS) { if (globals->atfork == globals->atfork_storage) { globals->atfork = storage; @@ -65,9 +65,9 @@ pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void)) bzero(globals->atfork_storage, sizeof(globals->atfork_storage)); } else { // Another thread did vm_map first. - _PTHREAD_UNLOCK(globals->pthread_atfork_lock); + _pthread_lock_unlock(&globals->pthread_atfork_lock); mach_vm_deallocate(mach_task_self(), storage, size); - _PTHREAD_LOCK(globals->pthread_atfork_lock); + _pthread_lock_lock(&globals->pthread_atfork_lock); } } else { res = ENOMEM; @@ -82,7 +82,7 @@ pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void)) e->parent = parent; e->child = child; } - _PTHREAD_UNLOCK(globals->pthread_atfork_lock); + _pthread_lock_unlock(&globals->pthread_atfork_lock); return res; } @@ -95,7 +95,7 @@ _pthread_atfork_prepare_handlers(void) { pthread_globals_t globals = _pthread_globals(); - _PTHREAD_LOCK(globals->pthread_atfork_lock); + _pthread_lock_lock(&globals->pthread_atfork_lock); size_t idx; for (idx = globals->atfork_count; idx > 0; --idx) { struct pthread_atfork_entry *e = &globals->atfork[idx-1]; @@ -112,9 +112,9 @@ _pthread_atfork_prepare(void) { pthread_globals_t globals = _pthread_globals(); - _PTHREAD_LOCK(globals->psaved_self_global_lock); + _pthread_lock_lock(&globals->psaved_self_global_lock); globals->psaved_self = pthread_self(); - _PTHREAD_LOCK(globals->psaved_self->lock); + _pthread_lock_lock(&globals->psaved_self->lock); } // Called after the fork(2) system call returns to the parent process. @@ -125,8 +125,8 @@ _pthread_atfork_parent(void) { pthread_globals_t globals = _pthread_globals(); - _PTHREAD_UNLOCK(globals->psaved_self->lock); - _PTHREAD_UNLOCK(globals->psaved_self_global_lock); + _pthread_lock_unlock(&globals->psaved_self->lock); + _pthread_lock_unlock(&globals->psaved_self_global_lock); } // Iterate pthread_atfork parent handlers. @@ -143,7 +143,7 @@ _pthread_atfork_parent_handlers(void) e->parent(); } } - _PTHREAD_UNLOCK(globals->pthread_atfork_lock); + _pthread_lock_unlock(&globals->pthread_atfork_lock); } // Called after the fork(2) system call returns to the new child process. @@ -154,7 +154,7 @@ void _pthread_atfork_child(void) { pthread_globals_t globals = _pthread_globals(); - _PTHREAD_LOCK_INIT(globals->psaved_self_global_lock); + _pthread_lock_init(&globals->psaved_self_global_lock); __is_threaded = 0; _pthread_main_thread_postfork_init(globals->psaved_self); @@ -175,7 +175,7 @@ _pthread_atfork_child_handlers(void) e->child(); } } - _PTHREAD_LOCK_INIT(globals->pthread_atfork_lock); + _pthread_lock_init(&globals->pthread_atfork_lock); } // Preserve legacy symbols for older iOS simulators @@ -204,3 +204,4 @@ _pthread_fork_child_postinit(void) { _pthread_atfork_child_handlers(); } +#endif // !VARIANT_DYLD diff --git a/src/pthread_cancelable.c b/src/pthread_cancelable.c index f209672..171f86d 100644 --- a/src/pthread_cancelable.c +++ b/src/pthread_cancelable.c @@ -64,45 +64,14 @@ #include #include -extern int _pthread_cond_wait(pthread_cond_t *cond, - pthread_mutex_t *mutex, - const struct timespec *abstime, - int isRelative, - int isconforming); -extern int __sigwait(const sigset_t *set, int *sig); -extern int __pthread_sigmask(int, const sigset_t *, sigset_t *); -extern int __pthread_markcancel(mach_port_t); -extern int __pthread_canceled(int); -extern int __semwait_signal_nocancel(int, int, int, int, __int64_t, __int32_t); - - -PTHREAD_NOEXPORT -int _pthread_join(pthread_t thread, void **value_ptr, int conforming); - -static inline int -_pthread_conformance(void) -{ -#if __DARWIN_UNIX03 - if (__unix_conforming == 0) - __unix_conforming = 1; -#ifdef VARIANT_CANCELABLE - return PTHREAD_CONFORM_UNIX03_CANCELABLE; -#else /* !VARIANT_CANCELABLE */ - return PTHREAD_CONFORM_UNIX03_NOCANCEL; -#endif -#else /* __DARWIN_UNIX03 */ - return PTHREAD_CONFORM_DARWIN_LEGACY; -#endif /* __DARWIN_UNIX03 */ -} - -#ifndef VARIANT_CANCELABLE +#ifndef BUILDING_VARIANT /* [ */ -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int _pthread_update_cancel_state(pthread_t thread, int mask, int state) { - int oldstate, newstate; - os_atomic_rmw_loop2o(thread, cancel_state, oldstate, newstate, seq_cst, { + uint16_t oldstate, newstate; + os_atomic_rmw_loop(&thread->cancel_state, oldstate, newstate, relaxed, { newstate = oldstate; newstate &= ~mask; newstate |= state; @@ -110,6 +79,16 @@ _pthread_update_cancel_state(pthread_t thread, int mask, int state) return oldstate; } +/* When a thread exits set the cancellation state to DISABLE and DEFERRED */ +void +_pthread_setcancelstate_exit(pthread_t thread, void *value_ptr) +{ + _pthread_update_cancel_state(thread, + _PTHREAD_CANCEL_STATE_MASK | _PTHREAD_CANCEL_TYPE_MASK, + PTHREAD_CANCEL_DISABLE | PTHREAD_CANCEL_DEFERRED | + _PTHREAD_CANCEL_EXITING); +} + /* * Cancel a thread */ @@ -117,11 +96,6 @@ PTHREAD_NOEXPORT_VARIANT int pthread_cancel(pthread_t thread) { -#if __DARWIN_UNIX03 - if (__unix_conforming == 0) - __unix_conforming = 1; -#endif /* __DARWIN_UNIX03 */ - if (!_pthread_is_valid(thread, NULL)) { return(ESRCH); } @@ -130,93 +104,20 @@ pthread_cancel(pthread_t thread) if (thread->wqthread != 0) { return(ENOTSUP); } -#if __DARWIN_UNIX03 - int state = os_atomic_or2o(thread, cancel_state, _PTHREAD_CANCEL_PENDING, relaxed); + int state = os_atomic_or(&thread->cancel_state, _PTHREAD_CANCEL_PENDING, relaxed); if (state & PTHREAD_CANCEL_ENABLE) { - mach_port_t kport = _pthread_kernel_thread(thread); + mach_port_t kport = _pthread_tsd_slot(thread, MACH_THREAD_SELF); if (kport) __pthread_markcancel(kport); } -#else /* __DARWIN_UNIX03 */ - thread->cancel_state |= _PTHREAD_CANCEL_PENDING; -#endif /* __DARWIN_UNIX03 */ return (0); } - -void -pthread_testcancel(void) -{ - _pthread_testcancel(_pthread_conformance()); -} - -#ifndef BUILDING_VARIANT /* [ */ - -PTHREAD_NOEXPORT_VARIANT -void -_pthread_exit_if_canceled(int error) -{ - if (((error & 0xff) == EINTR) && __unix_conforming && (__pthread_canceled(0) == 0)) { - pthread_t self = pthread_self(); - - _pthread_validate_signature(self); - self->cancel_error = error; - self->canceled = true; - pthread_exit(PTHREAD_CANCELED); - } -} - - -static inline bool -_pthread_is_canceled(pthread_t thread) -{ - const int flags = (PTHREAD_CANCEL_ENABLE|_PTHREAD_CANCEL_PENDING); - int state = os_atomic_load2o(thread, cancel_state, seq_cst); - return (state & flags) == flags; -} - -PTHREAD_NOEXPORT_VARIANT -void -_pthread_testcancel(int isconforming) -{ - pthread_t self = pthread_self(); - if (_pthread_is_canceled(self)) { - _pthread_validate_signature(self); - // 4597450: begin - self->canceled = (isconforming != PTHREAD_CONFORM_DARWIN_LEGACY); - // 4597450: end - pthread_exit(isconforming ? PTHREAD_CANCELED : NULL); - } -} - -PTHREAD_NOEXPORT -void -_pthread_markcancel_if_canceled(pthread_t thread, mach_port_t kport) -{ - const int flags = (PTHREAD_CANCEL_ENABLE|_PTHREAD_CANCEL_PENDING); - int state = os_atomic_load2o(thread, cancel_state, relaxed); - if ((state & flags) == flags && __unix_conforming) { - __pthread_markcancel(kport); - } -} - -/* When a thread exits set the cancellation state to DISABLE and DEFERRED */ -PTHREAD_NOEXPORT -void -_pthread_setcancelstate_exit(pthread_t thread, void *value_ptr) -{ - _pthread_update_cancel_state(thread, - _PTHREAD_CANCEL_STATE_MASK | _PTHREAD_CANCEL_TYPE_MASK, - PTHREAD_CANCEL_DISABLE | PTHREAD_CANCEL_DEFERRED); -} - -#endif /* !BUILDING_VARIANT ] */ - /* * Query/update the cancelability 'state' of a thread */ -PTHREAD_ALWAYS_INLINE -static inline int -_pthread_setcancelstate_internal(int state, int *oldstateptr, int conforming) +PTHREAD_NOEXPORT_VARIANT +int +pthread_setcancelstate(int state, int *oldstateptr) { pthread_t self = pthread_self(); @@ -224,14 +125,10 @@ _pthread_setcancelstate_internal(int state, int *oldstateptr, int conforming) switch (state) { case PTHREAD_CANCEL_ENABLE: - if (conforming) { - __pthread_canceled(1); - } + __pthread_canceled(1); break; case PTHREAD_CANCEL_DISABLE: - if (conforming) { - __pthread_canceled(2); - } + __pthread_canceled(2); break; default: return EINVAL; @@ -241,27 +138,9 @@ _pthread_setcancelstate_internal(int state, int *oldstateptr, int conforming) if (oldstateptr) { *oldstateptr = oldstate & _PTHREAD_CANCEL_STATE_MASK; } - if (!conforming) { - /* See if we need to 'die' now... */ - _pthread_testcancel(PTHREAD_CONFORM_DARWIN_LEGACY); - } return 0; } -PTHREAD_NOEXPORT_VARIANT -int -pthread_setcancelstate(int state, int *oldstate) -{ -#if __DARWIN_UNIX03 - if (__unix_conforming == 0) { - __unix_conforming = 1; - } - return (_pthread_setcancelstate_internal(state, oldstate, 1)); -#else /* __DARWIN_UNIX03 */ - return (_pthread_setcancelstate_internal(state, oldstate, 0)); -#endif /* __DARWIN_UNIX03 */ -} - /* * Query/update the cancelability 'type' of a thread */ @@ -269,66 +148,86 @@ PTHREAD_NOEXPORT_VARIANT int pthread_setcanceltype(int type, int *oldtype) { - pthread_t self; + pthread_t self = pthread_self(); -#if __DARWIN_UNIX03 - if (__unix_conforming == 0) - __unix_conforming = 1; -#endif /* __DARWIN_UNIX03 */ + _pthread_validate_signature(self); if ((type != PTHREAD_CANCEL_DEFERRED) && (type != PTHREAD_CANCEL_ASYNCHRONOUS)) return EINVAL; - self = pthread_self(); - _pthread_validate_signature(self); int oldstate = _pthread_update_cancel_state(self, _PTHREAD_CANCEL_TYPE_MASK, type); if (oldtype) { *oldtype = oldstate & _PTHREAD_CANCEL_TYPE_MASK; } -#if !__DARWIN_UNIX03 - /* See if we need to 'die' now... */ - _pthread_testcancel(PTHREAD_CONFORM_DARWIN_LEGACY); -#endif /* __DARWIN_UNIX03 */ return (0); } -int -pthread_sigmask(int how, const sigset_t * set, sigset_t * oset) +OS_ALWAYS_INLINE +static inline bool +_pthread_is_canceled(pthread_t thread) { -#if __DARWIN_UNIX03 - int err = 0; - - if (__pthread_sigmask(how, set, oset) == -1) { - err = errno; - } - return(err); -#else /* __DARWIN_UNIX03 */ - return(__pthread_sigmask(how, set, oset)); -#endif /* __DARWIN_UNIX03 */ + const int flags = (PTHREAD_CANCEL_ENABLE|_PTHREAD_CANCEL_PENDING); + int state = os_atomic_load(&thread->cancel_state, seq_cst); + return (state & flags) == flags; } -#ifndef BUILDING_VARIANT /* [ */ - -typedef struct pthread_join_context_s { - pthread_t waiter; - void **value_ptr; - mach_port_t kport; - semaphore_t custom_stack_sema; - bool detached; -} pthread_join_context_s, *pthread_join_context_t; - +OS_ALWAYS_INLINE static inline void * _pthread_get_exit_value(pthread_t thread) { - if (__unix_conforming && _pthread_is_canceled(thread)) { + if (os_unlikely(_pthread_is_canceled(thread))) { return PTHREAD_CANCELED; } return thread->tl_exit_value; } +void +pthread_testcancel(void) +{ + pthread_t self = pthread_self(); + if (os_unlikely(_pthread_is_canceled(self))) { + _pthread_validate_signature(self); + // 4597450: begin + self->canceled = true; + // 4597450: end + pthread_exit(PTHREAD_CANCELED); + } +} + +void +_pthread_markcancel_if_canceled(pthread_t thread, mach_port_t kport) +{ + if (os_unlikely(_pthread_is_canceled(thread))) { + __pthread_markcancel(kport); + } +} + +void +_pthread_exit_if_canceled(int error) +{ + if ((error & 0xff) == EINTR && __pthread_canceled(0) == 0) { + pthread_t self = pthread_self(); + + _pthread_validate_signature(self); + self->cancel_error = error; + self->canceled = true; + pthread_exit(PTHREAD_CANCELED); + } +} + +int +pthread_sigmask(int how, const sigset_t * set, sigset_t * oset) +{ + int err = 0; + + if (__pthread_sigmask(how, set, oset) == -1) { + err = errno; + } + return(err); +} + // called with _pthread_list_lock held -PTHREAD_NOEXPORT semaphore_t _pthread_joiner_prepost_wake(pthread_t thread) { @@ -351,7 +250,7 @@ _pthread_joiner_abort_wait(pthread_t thread, pthread_join_context_t ctx) { bool aborted = false; - _PTHREAD_LOCK(_pthread_list_lock); + _pthread_lock_lock(&_pthread_list_lock); if (!ctx->detached && thread->tl_exit_gate != MACH_PORT_DEAD) { /* * _pthread_joiner_prepost_wake() didn't happen @@ -362,12 +261,13 @@ _pthread_joiner_abort_wait(pthread_t thread, pthread_join_context_t ctx) thread->tl_exit_gate = MACH_PORT_NULL; aborted = true; } - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); return aborted; } static int -_pthread_joiner_wait(pthread_t thread, pthread_join_context_t ctx, int conforming) +_pthread_joiner_wait(pthread_t thread, pthread_join_context_t ctx, + pthread_conformance_t conforming) { uint32_t *exit_gate = &thread->tl_exit_gate; int ulock_op = UL_UNFAIR_LOCK | ULF_NO_ERRNO; @@ -407,8 +307,8 @@ _pthread_joiner_wait(pthread_t thread, pthread_join_context_t ctx, int conformin * over the cancelation which will be acted upon at the next * cancelation point. */ - if (conforming == PTHREAD_CONFORM_UNIX03_CANCELABLE && - _pthread_is_canceled(ctx->waiter)) { + if (os_unlikely(conforming == PTHREAD_CONFORM_UNIX03_CANCELABLE && + _pthread_is_canceled(ctx->waiter))) { if (_pthread_joiner_abort_wait(thread, ctx)) { ctx->waiter->canceled = true; pthread_exit(PTHREAD_CANCELED); @@ -420,7 +320,7 @@ _pthread_joiner_wait(pthread_t thread, pthread_join_context_t ctx, int conformin bool cleanup = false; - _PTHREAD_LOCK(_pthread_list_lock); + _pthread_lock_lock(&_pthread_list_lock); // If pthread_detach() was called, we can't safely dereference the thread, // else, decide who gets to deallocate the thread (see _pthread_terminate). if (!ctx->detached) { @@ -428,7 +328,7 @@ _pthread_joiner_wait(pthread_t thread, pthread_join_context_t ctx, int conformin thread->tl_join_ctx = NULL; cleanup = thread->tl_joiner_cleans_up; } - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); if (cleanup) { _pthread_deallocate(thread, false); @@ -436,9 +336,9 @@ _pthread_joiner_wait(pthread_t thread, pthread_join_context_t ctx, int conformin return 0; } -PTHREAD_NOEXPORT PTHREAD_NOINLINE +OS_NOINLINE int -_pthread_join(pthread_t thread, void **value_ptr, int conforming) +_pthread_join(pthread_t thread, void **value_ptr, pthread_conformance_t conforming) { pthread_t self = pthread_self(); pthread_join_context_s ctx = { @@ -453,6 +353,7 @@ _pthread_join(pthread_t thread, void **value_ptr, int conforming) if (!_pthread_validate_thread_and_list_lock(thread)) { return ESRCH; } + _pthread_validate_signature(self); if (!thread->tl_joinable || (thread->tl_join_ctx != NULL)) { @@ -466,14 +367,14 @@ _pthread_join(pthread_t thread, void **value_ptr, int conforming) thread->tl_joinable = false; if (value_ptr) *value_ptr = _pthread_get_exit_value(thread); } else { - ctx.kport = _pthread_kernel_thread(thread); + ctx.kport = _pthread_tsd_slot(thread, MACH_THREAD_SELF); thread->tl_exit_gate = ctx.kport; thread->tl_join_ctx = &ctx; if (thread->tl_has_custom_stack) { ctx.custom_stack_sema = (semaphore_t)os_get_cached_semaphore(); } } - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); if (res == 0) { if (ctx.kport == MACH_PORT_NULL) { @@ -500,19 +401,30 @@ _pthread_join(pthread_t thread, void **value_ptr, int conforming) } #endif /* !BUILDING_VARIANT ] */ -#endif /* VARIANT_CANCELABLE */ -/* - * Wait for a thread to terminate and obtain its exit value. - */ +static inline pthread_conformance_t +_pthread_conformance(void) +{ +#ifdef VARIANT_CANCELABLE + return PTHREAD_CONFORM_UNIX03_CANCELABLE; +#else /* !VARIANT_CANCELABLE */ + return PTHREAD_CONFORM_UNIX03_NOCANCEL; +#endif +} + +static inline void +_pthread_testcancel_if_cancelable_variant(void) +{ +#ifdef VARIANT_CANCELABLE + pthread_testcancel(); +#endif +} + int pthread_join(pthread_t thread, void **value_ptr) { - int conforming = _pthread_conformance(); - if (conforming == PTHREAD_CONFORM_UNIX03_CANCELABLE) { - _pthread_testcancel(conforming); - } - return _pthread_join(thread, value_ptr, conforming); + _pthread_testcancel_if_cancelable_variant(); + return _pthread_join(thread, value_ptr, _pthread_conformance()); } int @@ -531,22 +443,14 @@ pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, int sigwait(const sigset_t * set, int * sig) { -#if __DARWIN_UNIX03 - int err = 0, conformance = _pthread_conformance(); + int err = 0; - if (__unix_conforming == 0) - __unix_conforming = 1; - - if (conformance == PTHREAD_CONFORM_UNIX03_CANCELABLE) { - _pthread_testcancel(conformance); - } + _pthread_testcancel_if_cancelable_variant(); if (__sigwait(set, sig) == -1) { err = errno; - if (conformance == PTHREAD_CONFORM_UNIX03_CANCELABLE) { - _pthread_testcancel(conformance); - } + _pthread_testcancel_if_cancelable_variant(); /* * EINTR that isn't a result of pthread_cancel() @@ -557,18 +461,5 @@ sigwait(const sigset_t * set, int * sig) } } return(err); -#else /* __DARWIN_UNIX03 */ - if (__sigwait(set, sig) == -1) { - /* - * EINTR that isn't a result of pthread_cancel() - * is translated to 0. - */ - if (errno != EINTR) { - return -1; - } - } - - return 0; -#endif /* __DARWIN_UNIX03 */ } diff --git a/src/pthread_cond.c b/src/pthread_cond.c index 725134f..f7e532c 100644 --- a/src/pthread_cond.c +++ b/src/pthread_cond.c @@ -58,15 +58,21 @@ #define PLOCKSTAT_MUTEX_RELEASE(x, y) #endif /* PLOCKSTAT */ -extern int __gettimeofday(struct timeval *, struct timezone *); - -PTHREAD_NOEXPORT -int _pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex, - const struct timespec *abstime, int isRelative, int isconforming); - -PTHREAD_ALWAYS_INLINE +typedef union { + uint64_t val; + struct { + uint32_t seq; + uint16_t waiters; + uint16_t signal; + }; +} pthread_ulock_cond_state_u; + +#define _PTHREAD_COND_WAITERS_INC \ + (1ull << (offsetof(pthread_ulock_cond_state_u, waiters) * CHAR_BIT)) + +OS_ALWAYS_INLINE static inline void -COND_GETSEQ_ADDR(_pthread_cond *cond, +COND_GETSEQ_ADDR(pthread_cond_t *cond, volatile uint64_t **c_lsseqaddr, volatile uint32_t **c_lseqcnt, volatile uint32_t **c_useqcnt, @@ -84,12 +90,25 @@ COND_GETSEQ_ADDR(_pthread_cond *cond, *c_lsseqaddr = (volatile uint64_t *)*c_lseqcnt; } +OS_ALWAYS_INLINE +static inline pthread_ulock_cond_state_u * +_pthread_ulock_cond_state(pthread_cond_t *cond) +{ + return (pthread_ulock_cond_state_u *)&cond->c_seq[cond->misalign]; +} + #ifndef BUILDING_VARIANT /* [ */ -static void _pthread_cond_cleanup(void *arg); -static void _pthread_cond_updateval(_pthread_cond *cond, _pthread_mutex *mutex, +static void _pthread_psynch_cond_cleanup(void *arg); +static void _pthread_cond_updateval(pthread_cond_t *cond, pthread_mutex_t *mutex, int error, uint32_t updateval); +static int +_pthread_ulock_cond_wait_complete(pthread_ulock_cond_state_u *state, + pthread_mutex_t *mutex, int rc); +static void +_pthread_ulock_cond_cleanup(void *arg); + int pthread_condattr_init(pthread_condattr_t *attr) @@ -122,12 +141,7 @@ pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared) { int res = EINVAL; if (attr->sig == _PTHREAD_COND_ATTR_SIG) { -#if __DARWIN_UNIX03 - if (pshared == PTHREAD_PROCESS_PRIVATE || pshared == PTHREAD_PROCESS_SHARED) -#else /* __DARWIN_UNIX03 */ - if (pshared == PTHREAD_PROCESS_PRIVATE) -#endif /* __DARWIN_UNIX03 */ - { + if (pshared == PTHREAD_PROCESS_PRIVATE || pshared == PTHREAD_PROCESS_SHARED) { attr->pshared = pshared; res = 0; } @@ -139,14 +153,16 @@ int pthread_cond_timedwait_relative_np(pthread_cond_t *cond, pthread_mutex_t *mutex, const struct timespec *abstime) { - return _pthread_cond_wait(cond, mutex, abstime, 1, 0); + return _pthread_cond_wait(cond, mutex, abstime, 1, + PTHREAD_CONFORM_UNIX03_NOCANCEL); } #endif /* !BUILDING_VARIANT ] */ -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int -_pthread_cond_init(_pthread_cond *cond, const pthread_condattr_t *attr, int conforming) +_pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr, + uint32_t sig) { volatile uint64_t *c_lsseqaddr; volatile uint32_t *c_lseqcnt, *c_useqcnt, *c_sseqcnt; @@ -157,80 +173,115 @@ _pthread_cond_init(_pthread_cond *cond, const pthread_condattr_t *attr, int conf cond->c_seq[2] = 0; cond->unused = 0; + // TODO: PTHREAD_STRICT candidate cond->misalign = (((uintptr_t)&cond->c_seq[0]) & 0x7) != 0; COND_GETSEQ_ADDR(cond, &c_lsseqaddr, &c_lseqcnt, &c_useqcnt, &c_sseqcnt); *c_sseqcnt = PTH_RWS_CV_CBIT; // set Sword to 0c - if (conforming) { - if (attr) { - cond->pshared = attr->pshared; - } else { - cond->pshared = _PTHREAD_DEFAULT_PSHARED; - } + if (attr) { + cond->pshared = attr->pshared; } else { cond->pshared = _PTHREAD_DEFAULT_PSHARED; } - long sig = _PTHREAD_COND_SIG; - // Ensure all contents are properly set before setting signature. #if defined(__LP64__) // For binary compatibility reasons we cannot require natural alignment of // the 64bit 'sig' long value in the struct. rdar://problem/21610439 - uint32_t *sig32_ptr = (uint32_t*)&cond->sig; - uint32_t *sig32_val = (uint32_t*)&sig; - *(sig32_ptr + 1) = *(sig32_val + 1); - os_atomic_store(sig32_ptr, *sig32_val, release); -#else - os_atomic_store2o(cond, sig, sig, release); + cond->sig._pad = 0; #endif + os_atomic_store(&cond->sig.val, sig, release); return 0; } #ifndef BUILDING_VARIANT /* [ */ -PTHREAD_NOINLINE +OS_ALWAYS_INLINE static int -_pthread_cond_check_init_slow(_pthread_cond *cond, bool *inited) +_pthread_cond_check_signature(pthread_cond_t *cond, uint32_t sig_current, + uint32_t *sig_inout) { - int res = EINVAL; - if (cond->sig == _PTHREAD_COND_SIG_init) { - _PTHREAD_LOCK(cond->lock); - if (cond->sig == _PTHREAD_COND_SIG_init) { - res = _pthread_cond_init(cond, NULL, 0); - if (inited) { - *inited = true; - } - } else if (cond->sig == _PTHREAD_COND_SIG) { - res = 0; + int res = 0; + switch (sig_current) { + case _PTHREAD_COND_SIG_init: + __builtin_unreachable(); + break; + case _PTHREAD_COND_SIG_pristine: + if (*sig_inout != _PTHREAD_COND_SIG_pristine) { + os_atomic_store(&cond->sig.val, *sig_inout, relaxed); } - _PTHREAD_UNLOCK(cond->lock); - } else if (cond->sig == _PTHREAD_COND_SIG) { - res = 0; + break; + case _PTHREAD_COND_SIG_psynch: + case _PTHREAD_COND_SIG_ulock: + if (*sig_inout == _PTHREAD_COND_SIG_pristine) { + *sig_inout = sig_current; + } else if (*sig_inout != sig_current) { + PTHREAD_INTERNAL_CRASH(0, "Mixed ulock and psych condvar use"); + } + break; + default: + // TODO: PTHREAD_STRICT candidate + res = EINVAL; + break; } + return res; } -PTHREAD_ALWAYS_INLINE -static inline int -_pthread_cond_check_init(_pthread_cond *cond, bool *inited) +OS_NOINLINE +static int +_pthread_cond_check_init_slow(pthread_cond_t *cond, uint32_t *sig_inout) { - int res = 0; - if (cond->sig != _PTHREAD_COND_SIG) { - return _pthread_cond_check_init_slow(cond, inited); + int res; + _pthread_lock_lock(&cond->lock); + + uint32_t sig_current = os_atomic_load(&cond->sig.val, relaxed); + if (sig_current == _PTHREAD_COND_SIG_init) { + res = _pthread_cond_init(cond, NULL, *sig_inout); + } else { + res = _pthread_cond_check_signature(cond, sig_current, sig_inout); } + + _pthread_lock_unlock(&cond->lock); return res; } +/* + * These routines maintain the signature of the condition variable, which + * encodes a small state machine: + * - a statically initialized condvar begins with SIG_init + * - explicit initialization via _cond_init() and implicit initialization + * transition to SIG_pristine, as there have been no waiters so we don't know + * what kind of mutex we'll be used with + * - the first _cond_wait() transitions to one of SIG_psynch or SIG_ulock + * according to the mutex being waited on + * + * On entry, *sig_inout is the furthest state we can transition to given the + * calling context. On exit, it is the actual state we observed, after any + * possible advancement. + */ +OS_ALWAYS_INLINE +static inline int +_pthread_cond_check_init(pthread_cond_t *cond, uint32_t *sig_inout) +{ + uint32_t sig_current = os_atomic_load(&cond->sig.val, relaxed); + if (sig_current == _PTHREAD_COND_SIG_init) { + return _pthread_cond_check_init_slow(cond, sig_inout); + } else { + return _pthread_cond_check_signature(cond, sig_current, sig_inout); + } +} + PTHREAD_NOEXPORT_VARIANT int -pthread_cond_destroy(pthread_cond_t *ocond) +pthread_cond_destroy(pthread_cond_t *cond) { - _pthread_cond *cond = (_pthread_cond *)ocond; int res = EINVAL; - if (cond->sig == _PTHREAD_COND_SIG) { - _PTHREAD_LOCK(cond->lock); + uint32_t sig = os_atomic_load(&cond->sig.val, relaxed); + switch (sig) { + case _PTHREAD_COND_SIG_psynch: + _pthread_lock_lock(&cond->lock); uint64_t oldval64, newval64; uint32_t lcntval, ucntval, scntval; @@ -261,30 +312,37 @@ pthread_cond_destroy(pthread_cond_t *ocond) flags |= _PTHREAD_MTX_OPT_PSHARED; } - cond->sig = _PTHREAD_NO_SIG; + os_atomic_store(&cond->sig.val, _PTHREAD_NO_SIG, relaxed); res = 0; - _PTHREAD_UNLOCK(cond->lock); + _pthread_lock_unlock(&cond->lock); if (needclearpre) { (void)__psynch_cvclrprepost(cond, lcntval, ucntval, scntval, 0, lcntval, flags); } - } else if (cond->sig == _PTHREAD_COND_SIG_init) { + break; + case _PTHREAD_COND_SIG_init: // Compatibility for misbehaving applications that attempt to // destroy a statically initialized condition variable. - cond->sig = _PTHREAD_NO_SIG; + // + // fall through + case _PTHREAD_COND_SIG_pristine: + case _PTHREAD_COND_SIG_ulock: + os_atomic_store(&cond->sig.val, _PTHREAD_NO_SIG, relaxed); res = 0; + break; + default: + // TODO: PTHREAD_STRICT candidate + break; } return res; } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int -_pthread_cond_signal(pthread_cond_t *ocond, bool broadcast, mach_port_t thread) +_pthread_psynch_cond_signal(pthread_cond_t *cond, bool broadcast, + mach_port_t thread) { - int res; - _pthread_cond *cond = (_pthread_cond *)ocond; - uint32_t updateval; uint32_t diffgen; uint32_t ulval; @@ -297,12 +355,6 @@ _pthread_cond_signal(pthread_cond_t *ocond, bool broadcast, mach_port_t thread) int retry_count = 0, uretry_count = 0; int ucountreset = 0; - bool inited = false; - res = _pthread_cond_check_init(cond, &inited); - if (res != 0 || inited == true) { - return res; - } - COND_GETSEQ_ADDR(cond, &c_lsseqaddr, &c_lseqcnt, &c_useqcnt, &c_sseqcnt); bool retry; @@ -394,9 +446,9 @@ _pthread_cond_signal(pthread_cond_t *ocond, bool broadcast, mach_port_t thread) if (broadcast) { // pass old U val so kernel will know the diffgen uint64_t cvudgen = ((uint64_t)ucntval << 32) | diffgen; - updateval = __psynch_cvbroad(ocond, cvlsgen, cvudgen, flags, NULL, 0, 0); + updateval = __psynch_cvbroad(cond, cvlsgen, cvudgen, flags, NULL, 0, 0); } else { - updateval = __psynch_cvsignal(ocond, cvlsgen, ucntval, thread, NULL, 0, 0, flags); + updateval = __psynch_cvsignal(cond, cvlsgen, ucntval, thread, NULL, 0, 0, flags); } if (updateval != (uint32_t)-1 && updateval != 0) { @@ -406,14 +458,95 @@ _pthread_cond_signal(pthread_cond_t *ocond, bool broadcast, mach_port_t thread) return 0; } +OS_ALWAYS_INLINE +static inline int +_pthread_ulock_cond_signal(pthread_cond_t *cond, bool broadcast, + mach_port_t thread) +{ + pthread_ulock_cond_state_u *state = _pthread_ulock_cond_state(cond); + + pthread_ulock_cond_state_u oldstate, newstate; + // release to pair with acquire after wait + os_atomic_rmw_loop(&state->val, oldstate.val, newstate.val, release, { + if (!oldstate.waiters || oldstate.waiters == oldstate.signal) { + os_atomic_rmw_loop_give_up(return 0); + } + + newstate = (pthread_ulock_cond_state_u){ + .seq = oldstate.seq + 1, + .waiters = oldstate.waiters, + .signal = broadcast ? oldstate.waiters : + MIN(oldstate.signal + 1, oldstate.waiters), + }; + }); + + PTHREAD_TRACE(ulcond_signal, cond, oldstate.val, newstate.val, broadcast); + + // Priority hole: if we're pre-empted here, nobody else can signal the + // waiter we took responsibility for signaling by incrementing the signal + // count. + + if (oldstate.signal < oldstate.waiters) { + uint32_t wake_op = UL_COMPARE_AND_WAIT | ULF_NO_ERRNO; + if (broadcast) { + wake_op |= ULF_WAKE_ALL; + } else if (thread) { + wake_op |= ULF_WAKE_THREAD; + } + + for (;;) { + int rc = __ulock_wake(wake_op, &state->seq, thread); + if (rc < 0) { + switch (-rc) { + case EINTR: + continue; + case ENOENT: + break; + case EALREADY: + if (!thread) { + PTHREAD_INTERNAL_CRASH(0, "EALREADY from ulock_wake"); + } + // Compatibility with psynch: promote to broadcast + return pthread_cond_broadcast(cond); + default: + PTHREAD_INTERNAL_CRASH(-rc, "ulock_wake failure"); + } + } + break; + } + } + + return 0; +} + +OS_ALWAYS_INLINE +static inline int +_pthread_cond_signal(pthread_cond_t *cond, bool broadcast, mach_port_t thread) +{ + uint32_t sig = _PTHREAD_COND_SIG_pristine; + int res = _pthread_cond_check_init(cond, &sig); + if (res != 0 || sig == _PTHREAD_COND_SIG_pristine) { + return res; + } + + switch (sig) { + case _PTHREAD_COND_SIG_psynch: + return _pthread_psynch_cond_signal(cond, broadcast, thread); + case _PTHREAD_COND_SIG_ulock: + return _pthread_ulock_cond_signal(cond, broadcast, thread); + default: + PTHREAD_INTERNAL_CRASH(sig, "impossible cond signature"); + } +} + /* * Signal a condition variable, waking up all threads waiting for it. */ PTHREAD_NOEXPORT_VARIANT int -pthread_cond_broadcast(pthread_cond_t *ocond) +pthread_cond_broadcast(pthread_cond_t *cond) { - return _pthread_cond_signal(ocond, true, MACH_PORT_NULL); + return _pthread_cond_signal(cond, true, MACH_PORT_NULL); } /* @@ -421,13 +554,13 @@ pthread_cond_broadcast(pthread_cond_t *ocond) */ PTHREAD_NOEXPORT_VARIANT int -pthread_cond_signal_thread_np(pthread_cond_t *ocond, pthread_t thread) +pthread_cond_signal_thread_np(pthread_cond_t *cond, pthread_t thread) { mach_port_t mp = MACH_PORT_NULL; if (thread) { mp = pthread_mach_thread_np((_Nonnull pthread_t)thread); } - return _pthread_cond_signal(ocond, false, mp); + return _pthread_cond_signal(cond, false, mp); } /* @@ -435,34 +568,17 @@ pthread_cond_signal_thread_np(pthread_cond_t *ocond, pthread_t thread) */ PTHREAD_NOEXPORT_VARIANT int -pthread_cond_signal(pthread_cond_t *ocond) +pthread_cond_signal(pthread_cond_t *cond) { - return _pthread_cond_signal(ocond, false, MACH_PORT_NULL); + return _pthread_cond_signal(cond, false, MACH_PORT_NULL); } -/* - * Manage a list of condition variables associated with a mutex - */ - -/* - * Suspend waiting for a condition variable. - * Note: we have to keep a list of condition variables which are using - * this same mutex variable so we can detect invalid 'destroy' sequences. - * If conformance is not cancelable, we skip the _pthread_testcancel(), - * but keep the remaining conforming behavior.. - */ -PTHREAD_NOEXPORT PTHREAD_NOINLINE -int -_pthread_cond_wait(pthread_cond_t *ocond, - pthread_mutex_t *omutex, - const struct timespec *abstime, - int isRelative, - int conforming) +static int +_pthread_psynch_cond_wait(pthread_cond_t *cond, + pthread_mutex_t *mutex, + const struct timespec *then, + pthread_conformance_t conforming) { - int res; - _pthread_cond *cond = (_pthread_cond *)ocond; - _pthread_mutex *mutex = (_pthread_mutex *)omutex; - struct timespec then = { 0, 0 }; uint32_t mtxgen, mtxugen, flags=0, updateval; uint32_t lcntval, ucntval, scntval; uint32_t nlval, ulval, savebits; @@ -470,26 +586,215 @@ _pthread_cond_wait(pthread_cond_t *ocond, volatile uint32_t *c_lseqcnt, *c_useqcnt, *c_sseqcnt; uint64_t oldval64, newval64, mugen, cvlsgen; uint32_t *npmtx = NULL; - int timeout_elapsed = 0; - res = _pthread_cond_check_init(cond, NULL); + COND_GETSEQ_ADDR(cond, &c_lsseqaddr, &c_lseqcnt, &c_useqcnt, &c_sseqcnt); + + do { + lcntval = *c_lseqcnt; + ucntval = *c_useqcnt; + scntval = *c_sseqcnt; + + oldval64 = (((uint64_t)scntval) << 32); + oldval64 |= lcntval; + + /* remove c and p bits on S word */ + savebits = scntval & PTH_RWS_CV_BITSALL; + ulval = (scntval & PTHRW_COUNT_MASK); + nlval = lcntval + PTHRW_INC; + newval64 = (((uint64_t)ulval) << 32); + newval64 |= nlval; + } while (!os_atomic_cmpxchg(c_lsseqaddr, oldval64, newval64, seq_cst)); + + cond->busy = mutex; + + int res = _pthread_mutex_droplock(mutex, &flags, &npmtx, &mtxgen, &mtxugen); + + /* TBD: cases are for normal (non owner for recursive mutex; error checking)*/ if (res != 0) { - return res; + return EINVAL; } + if ((flags & _PTHREAD_MTX_OPT_NOTIFY) == 0) { + npmtx = NULL; + mugen = 0; + } else { + mugen = ((uint64_t)mtxugen << 32) | mtxgen; + } + flags &= ~_PTHREAD_MTX_OPT_MUTEX; /* reset the mutex bit as this is cvar */ - if (conforming) { - if (!_pthread_mutex_check_signature(mutex) && - !_pthread_mutex_check_signature_init(mutex)) { - return EINVAL; + cvlsgen = ((uint64_t)(ulval | savebits)<< 32) | nlval; + + // SUSv3 requires pthread_cond_wait to be a cancellation point + if (conforming == PTHREAD_CONFORM_UNIX03_CANCELABLE) { + pthread_cleanup_push(_pthread_psynch_cond_cleanup, (void *)cond); + updateval = __psynch_cvwait(cond, cvlsgen, ucntval, (pthread_mutex_t *)npmtx, mugen, flags, (int64_t)(then->tv_sec), (int32_t)(then->tv_nsec)); + pthread_testcancel(); + pthread_cleanup_pop(0); + } else { + updateval = __psynch_cvwait(cond, cvlsgen, ucntval, (pthread_mutex_t *)npmtx, mugen, flags, (int64_t)(then->tv_sec), (int32_t)(then->tv_nsec)); + } + + if (updateval == (uint32_t)-1) { + int err = errno; + switch (err & 0xff) { + case ETIMEDOUT: + res = ETIMEDOUT; + break; + case EINTR: + // spurious wakeup (unless canceled) + res = 0; + break; + default: + res = EINVAL; + break; } - if (conforming == PTHREAD_CONFORM_UNIX03_CANCELABLE) { - _pthread_testcancel(conforming); + + // add unlock ref to show one less waiter + _pthread_cond_updateval(cond, mutex, err, 0); + } else if (updateval != 0) { + // Successful wait + // The return due to prepost and might have bit states + // update S and return for prepo if needed + _pthread_cond_updateval(cond, mutex, 0, updateval); + } + + pthread_mutex_lock(mutex); + + return res; +} + +struct pthread_ulock_cond_cancel_ctx_s { + pthread_cond_t *cond; + pthread_mutex_t *mutex; +}; + +static int +_pthread_ulock_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex, + const struct timespec *then, pthread_conformance_t conforming) +{ + bool cancelable = (conforming == PTHREAD_CONFORM_UNIX03_CANCELABLE); + + uint64_t timeout_ns = 0; + if (then->tv_sec || then->tv_nsec) { + // psynch compatibility: cast and bitwise-truncate tv_nsec + uint64_t fraction_ns = ((uint32_t)then->tv_nsec) & 0x3fffffff; + if (os_mul_and_add_overflow(then->tv_sec, NSEC_PER_SEC, fraction_ns, + &timeout_ns)) { + // saturate (can't wait longer than 584 years...) + timeout_ns = UINT64_MAX; + } + } + + pthread_ulock_cond_state_u *state = _pthread_ulock_cond_state(cond); + + pthread_ulock_cond_state_u origstate = { + .val = os_atomic_add(&state->val, _PTHREAD_COND_WAITERS_INC, relaxed) + }; + + int rc = _pthread_mutex_ulock_unlock(mutex); + if (rc) { + return _pthread_ulock_cond_wait_complete(state, NULL, rc); + } + + PTHREAD_TRACE(ulcond_wait, cond, origstate.val, timeout_ns, 0); + + do { + const uint32_t wait_op = UL_COMPARE_AND_WAIT | ULF_NO_ERRNO; + if (cancelable) { + struct pthread_ulock_cond_cancel_ctx_s ctx = { + .cond = cond, + .mutex = mutex, + }; + pthread_cleanup_push(_pthread_ulock_cond_cleanup, &ctx); + rc = __ulock_wait2(wait_op | ULF_WAIT_CANCEL_POINT, &state->seq, + origstate.seq, timeout_ns, 0); + pthread_testcancel(); + pthread_cleanup_pop(0); + } else { + rc = __ulock_wait2(wait_op, &state->seq, origstate.seq, timeout_ns, 0); + } + if (rc < 0) { + switch (-rc) { + case EFAULT: + continue; + case EINTR: + // "These functions shall not return an error code of [EINTR]." + // => promote to spurious wake-up + rc = 0; + goto out; + case ETIMEDOUT: + rc = ETIMEDOUT; + goto out; + default: + PTHREAD_INTERNAL_CRASH(-rc, "ulock_wait failure"); + } + } else { + // XXX for now don't care about other waiters + rc = 0; } + } while (os_atomic_load(&state->seq, relaxed) == origstate.seq); + +out: + return _pthread_ulock_cond_wait_complete(state, mutex, rc); +} + +static int +_pthread_ulock_cond_wait_complete(pthread_ulock_cond_state_u *state, + pthread_mutex_t *mutex, int rc) +{ + if (mutex) { + // XXX Check this return value? Historically we haven't, but if rc == 0 + // we could promote the return value to this one. + _pthread_mutex_ulock_lock(mutex, false); + } + + pthread_ulock_cond_state_u oldstate, newstate; + // acquire to pair with release upon signal + os_atomic_rmw_loop(&state->val, oldstate.val, newstate.val, acquire, { + newstate = (pthread_ulock_cond_state_u){ + .seq = oldstate.seq, + .waiters = oldstate.waiters - 1, + .signal = oldstate.signal ? oldstate.signal - 1 : 0, + }; + }); + + return rc; +} + +/* + * Suspend waiting for a condition variable. + * If conformance is not cancelable, we skip the pthread_testcancel(), + * but keep the remaining conforming behavior. + */ +PTHREAD_NOEXPORT OS_NOINLINE +int +_pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex, + const struct timespec *abstime, int isRelative, + pthread_conformance_t conforming) +{ + int res; + struct timespec then = { 0, 0 }; + bool timeout_elapsed = false; + + if (!_pthread_mutex_check_signature(mutex) && + !_pthread_mutex_check_signature_init(mutex)) { + return EINVAL; + } + + bool ulock = _pthread_mutex_uses_ulock(mutex); + uint32_t sig = ulock ? _PTHREAD_COND_SIG_ulock : _PTHREAD_COND_SIG_psynch; + res = _pthread_cond_check_init(cond, &sig); + if (res != 0) { + return res; + } + + if (conforming == PTHREAD_CONFORM_UNIX03_CANCELABLE) { + pthread_testcancel(); } /* send relative time to kernel */ if (abstime) { if (abstime->tv_nsec < 0 || abstime->tv_nsec >= NSEC_PER_SEC) { + // TODO: PTHREAD_STRICT candidate return EINVAL; } @@ -502,7 +807,7 @@ _pthread_cond_wait(pthread_cond_t *ocond, if ((abstime->tv_sec == now.tv_sec) ? (abstime->tv_nsec <= now.tv_nsec) : (abstime->tv_sec < now.tv_sec)) { - timeout_elapsed = 1; + timeout_elapsed = true; } else { /* Compute relative time to sleep */ then.tv_nsec = abstime->tv_nsec - now.tv_nsec; @@ -516,12 +821,13 @@ _pthread_cond_wait(pthread_cond_t *ocond, then.tv_sec = abstime->tv_sec; then.tv_nsec = abstime->tv_nsec; if ((then.tv_sec == 0) && (then.tv_nsec == 0)) { - timeout_elapsed = 1; + timeout_elapsed = true; } } } - if (cond->busy != NULL && cond->busy != mutex) { + if (!ulock && cond->busy != NULL && cond->busy != mutex) { + // TODO: PTHREAD_STRICT candidate return EINVAL; } @@ -530,97 +836,49 @@ _pthread_cond_wait(pthread_cond_t *ocond, * relock the mutex to allow other waiters to get in line and * modify the condition state. */ - if (timeout_elapsed) { - res = pthread_mutex_unlock(omutex); + if (timeout_elapsed) { + res = pthread_mutex_unlock(mutex); if (res != 0) { return res; } - res = pthread_mutex_lock(omutex); + res = pthread_mutex_lock(mutex); if (res != 0) { return res; } - return ETIMEDOUT; - } - - COND_GETSEQ_ADDR(cond, &c_lsseqaddr, &c_lseqcnt, &c_useqcnt, &c_sseqcnt); - do { - lcntval = *c_lseqcnt; - ucntval = *c_useqcnt; - scntval = *c_sseqcnt; - - oldval64 = (((uint64_t)scntval) << 32); - oldval64 |= lcntval; - - /* remove c and p bits on S word */ - savebits = scntval & PTH_RWS_CV_BITSALL; - ulval = (scntval & PTHRW_COUNT_MASK); - nlval = lcntval + PTHRW_INC; - newval64 = (((uint64_t)ulval) << 32); - newval64 |= nlval; - } while (!os_atomic_cmpxchg(c_lsseqaddr, oldval64, newval64, seq_cst)); - - cond->busy = mutex; - - res = _pthread_mutex_droplock(mutex, &flags, &npmtx, &mtxgen, &mtxugen); - - /* TBD: cases are for normal (non owner for recursive mutex; error checking)*/ - if (res != 0) { - return EINVAL; - } - if ((flags & _PTHREAD_MTX_OPT_NOTIFY) == 0) { - npmtx = NULL; - mugen = 0; - } else { - mugen = ((uint64_t)mtxugen << 32) | mtxgen; + return ETIMEDOUT; } - flags &= ~_PTHREAD_MTX_OPT_MUTEX; /* reset the mutex bit as this is cvar */ - cvlsgen = ((uint64_t)(ulval | savebits)<< 32) | nlval; - - // SUSv3 requires pthread_cond_wait to be a cancellation point - if (conforming) { - pthread_cleanup_push(_pthread_cond_cleanup, (void *)cond); - updateval = __psynch_cvwait(ocond, cvlsgen, ucntval, (pthread_mutex_t *)npmtx, mugen, flags, (int64_t)then.tv_sec, (int32_t)then.tv_nsec); - _pthread_testcancel(conforming); - pthread_cleanup_pop(0); + if (ulock) { + return _pthread_ulock_cond_wait(cond, mutex, &then, conforming); } else { - updateval = __psynch_cvwait(ocond, cvlsgen, ucntval, (pthread_mutex_t *)npmtx, mugen, flags, (int64_t)then.tv_sec, (int32_t)then.tv_nsec); - } - - if (updateval == (uint32_t)-1) { - int err = errno; - switch (err & 0xff) { - case ETIMEDOUT: - res = ETIMEDOUT; - break; - case EINTR: - // spurious wakeup (unless canceled) - res = 0; - break; - default: - res = EINVAL; - break; - } - - // add unlock ref to show one less waiter - _pthread_cond_updateval(cond, mutex, err, 0); - } else if (updateval != 0) { - // Successful wait - // The return due to prepost and might have bit states - // update S and return for prepo if needed - _pthread_cond_updateval(cond, mutex, 0, updateval); + return _pthread_psynch_cond_wait(cond, mutex, &then, conforming); } +} - pthread_mutex_lock(omutex); - - return res; +static void +_pthread_ulock_cond_cleanup(void *arg) +{ + struct pthread_ulock_cond_cancel_ctx_s *ctx = arg; + pthread_ulock_cond_state_u *state = _pthread_ulock_cond_state(ctx->cond); + + (void)_pthread_ulock_cond_wait_complete(state, ctx->mutex, 0); + + // "A thread that has been unblocked because it has been canceled while + // blocked in a call to pthread_cond_timedwait() or pthread_cond_wait() + // shall not consume any condition signal that may be directed concurrently + // at the condition variable if there are other threads blocked on the + // condition variable." + // + // Since we have no way to know if we've eaten somebody else's signal, just + // signal again pessimistically. + pthread_cond_signal(ctx->cond); } static void -_pthread_cond_cleanup(void *arg) +_pthread_psynch_cond_cleanup(void *arg) { - _pthread_cond *cond = (_pthread_cond *)arg; + pthread_cond_t *cond = (pthread_cond_t *)arg; pthread_t thread = pthread_self(); pthread_mutex_t *mutex; @@ -630,11 +888,10 @@ _pthread_cond_cleanup(void *arg) } // 4597450: end - mutex = (pthread_mutex_t *)cond->busy; + mutex = cond->busy; // add unlock ref to show one less waiter - _pthread_cond_updateval(cond, (_pthread_mutex *)mutex, - thread->cancel_error, 0); + _pthread_cond_updateval(cond, mutex, thread->cancel_error, 0); /* ** Can't do anything if this fails -- we're on the way out @@ -645,7 +902,7 @@ _pthread_cond_cleanup(void *arg) } static void -_pthread_cond_updateval(_pthread_cond *cond, _pthread_mutex *mutex, +_pthread_cond_updateval(pthread_cond_t *cond, pthread_mutex_t *mutex, int error, uint32_t updateval) { int needclearpre; @@ -731,18 +988,9 @@ _pthread_cond_updateval(_pthread_cond *cond, _pthread_mutex *mutex, PTHREAD_NOEXPORT_VARIANT int -pthread_cond_init(pthread_cond_t *ocond, const pthread_condattr_t *attr) +pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr) { - int conforming; - -#if __DARWIN_UNIX03 - conforming = 1; -#else /* __DARWIN_UNIX03 */ - conforming = 0; -#endif /* __DARWIN_UNIX03 */ - - _pthread_cond *cond = (_pthread_cond *)ocond; - _PTHREAD_LOCK_INIT(cond->lock); - return _pthread_cond_init(cond, attr, conforming); + _pthread_lock_init(&cond->lock); + return _pthread_cond_init(cond, attr, _PTHREAD_COND_SIG_pristine); } diff --git a/src/pthread_cwd.c b/src/pthread_cwd.c index 3126c3b..1a2af22 100644 --- a/src/pthread_cwd.c +++ b/src/pthread_cwd.c @@ -1,15 +1,34 @@ -#include -#include -#include +/* + * Copyright (c) 2019 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#include "internal.h" -extern int __pthread_chdir(const char *path); int pthread_chdir_np(const char *path) { return __pthread_chdir(path); } -extern int __pthread_fchdir(int fd); int pthread_fchdir_np(int fd) { diff --git a/src/pthread_dependency.c b/src/pthread_dependency.c index 3836f15..a907ac0 100644 --- a/src/pthread_dependency.c +++ b/src/pthread_dependency.c @@ -23,22 +23,10 @@ #include "resolver.h" #include "internal.h" -#include "dependency_private.h" -#include #define PREREQUISITE_FULFILLED (~0u) -PTHREAD_NOEXPORT -void _pthread_dependency_fulfill_slow(pthread_dependency_t *pr, uint32_t old); - -OS_ALWAYS_INLINE -static inline mach_port_t -_pthread_dependency_self(void) -{ - void *v = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_MACH_THREAD_SELF); - return (mach_port_t)(uintptr_t)v; -} - +#if !VARIANT_DYLD void pthread_dependency_init_np(pthread_dependency_t *pr, pthread_t pth, pthread_dependency_attr_t *attrs) @@ -54,7 +42,7 @@ _pthread_dependency_fulfill_slow(pthread_dependency_t *pr, uint32_t old) if (old == PREREQUISITE_FULFILLED) { PTHREAD_CLIENT_CRASH(0, "Fufilling pthread_dependency_t twice"); } - if (os_unlikely(old != _pthread_dependency_self())) { + if (os_unlikely(old != _pthread_mach_thread_self_direct())) { PTHREAD_CLIENT_CRASH(old, "Fulfilled a dependency " "not owned by current thread"); } @@ -70,6 +58,7 @@ _pthread_dependency_fulfill_slow(pthread_dependency_t *pr, uint32_t old) } +PTHREAD_NOEXPORT_VARIANT void pthread_dependency_fulfill_np(pthread_dependency_t *pr, void *value) { @@ -81,6 +70,7 @@ pthread_dependency_fulfill_np(pthread_dependency_t *pr, void *value) if (old != 0) _pthread_dependency_fulfill_slow(pr, old); } +PTHREAD_NOEXPORT_VARIANT void * pthread_dependency_wait_np(pthread_dependency_t *pr) { @@ -109,3 +99,16 @@ pthread_dependency_wait_np(pthread_dependency_t *pr) PTHREAD_CLIENT_CRASH(cur, "Corrupted pthread_dependency_t"); } +PTHREAD_NOEXPORT_VARIANT void* +_pthread_atomic_xchg_ptr(void **p, void *v) +{ + return os_atomic_xchg(p, v, seq_cst); +} + +PTHREAD_NOEXPORT_VARIANT uint32_t +_pthread_atomic_xchg_uint32_relaxed(uint32_t *p, uint32_t v) +{ + return os_atomic_xchg(p, v, relaxed); +} + +#endif // !VARIANT_DYLD diff --git a/src/pthread_mutex.c b/src/pthread_mutex.c index ad86eeb..a199031 100644 --- a/src/pthread_mutex.c +++ b/src/pthread_mutex.c @@ -61,8 +61,9 @@ /* This function is never called and exists to provide never-fired dtrace * probes so that user d scripts don't get errors. */ -PTHREAD_NOEXPORT PTHREAD_USED -void +OS_USED static void +_plockstat_never_fired(void); +static void _plockstat_never_fired(void) { PLOCKSTAT_MUTEX_SPIN(NULL); @@ -83,29 +84,11 @@ _plockstat_never_fired(void) #define PTHREAD_MUTEX_INIT_UNUSED 1 -PTHREAD_NOEXPORT PTHREAD_WEAK -int _pthread_mutex_lock_init_slow(_pthread_mutex *mutex, bool trylock); - -PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers -int _pthread_mutex_fairshare_lock_slow(_pthread_mutex *mutex, bool trylock); - -PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers -int _pthread_mutex_firstfit_lock_slow(_pthread_mutex *mutex, bool trylock); - -PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers -int _pthread_mutex_fairshare_unlock_slow(_pthread_mutex *mutex); - -PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers -int _pthread_mutex_firstfit_unlock_slow(_pthread_mutex *mutex); - -PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers -int _pthread_mutex_corruption_abort(_pthread_mutex *mutex); - -extern int __pthread_mutex_default_opt_policy PTHREAD_NOEXPORT; +#if !VARIANT_DYLD - -int __pthread_mutex_default_opt_policy PTHREAD_NOEXPORT = - _PTHREAD_MTX_OPT_POLICY_DEFAULT; +int __pthread_mutex_default_opt_policy = _PTHREAD_MTX_OPT_POLICY_DEFAULT; +bool __pthread_mutex_use_ulock = _PTHREAD_MTX_OPT_ULOCK_DEFAULT; +bool __pthread_mutex_ulock_adaptive_spin = _PTHREAD_MTX_OPT_ADAPTIVE_DEFAULT; static inline bool _pthread_mutex_policy_validate(int policy) @@ -126,14 +109,14 @@ _pthread_mutex_policy_to_opt(int policy) } } -PTHREAD_NOEXPORT void _pthread_mutex_global_init(const char *envp[], struct _pthread_registration_data *registration_data) { int opt = _PTHREAD_MTX_OPT_POLICY_DEFAULT; if (registration_data->mutex_default_policy) { - int policy = registration_data->mutex_default_policy; + int policy = registration_data->mutex_default_policy & + _PTHREAD_REG_DEFAULT_POLICY_MASK; if (_pthread_mutex_policy_validate(policy)) { opt = _pthread_mutex_policy_to_opt(policy); } @@ -150,12 +133,43 @@ _pthread_mutex_global_init(const char *envp[], if (opt != __pthread_mutex_default_opt_policy) { __pthread_mutex_default_opt_policy = opt; } + + bool use_ulock = _PTHREAD_MTX_OPT_ULOCK_DEFAULT; + if (_os_xbs_chrooted) { + use_ulock = false; + } else { + envvar = _simple_getenv(envp, "PTHREAD_MUTEX_USE_ULOCK"); + if (envvar) { + use_ulock = (envvar[0] == '1'); + } else if (registration_data->mutex_default_policy) { + use_ulock = registration_data->mutex_default_policy & + _PTHREAD_REG_DEFAULT_USE_ULOCK; + } + } + + if (use_ulock != __pthread_mutex_use_ulock) { + __pthread_mutex_use_ulock = use_ulock; + } + + bool adaptive_spin = _PTHREAD_MTX_OPT_ADAPTIVE_DEFAULT; + envvar = _simple_getenv(envp, "PTHREAD_MUTEX_ADAPTIVE_SPIN"); + if (envvar) { + adaptive_spin = (envvar[0] == '1'); + } else if (registration_data->mutex_default_policy) { + adaptive_spin = registration_data->mutex_default_policy & + _PTHREAD_REG_DEFAULT_USE_ADAPTIVE_SPIN; + } + + if (adaptive_spin != __pthread_mutex_ulock_adaptive_spin) { + __pthread_mutex_ulock_adaptive_spin = adaptive_spin; + } } +#endif // !VARIANT_DYLD -PTHREAD_ALWAYS_INLINE -static inline int _pthread_mutex_init(_pthread_mutex *mutex, +OS_ALWAYS_INLINE +static inline int _pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr, uint32_t static_type); typedef union mutex_seq { @@ -173,25 +187,25 @@ _Static_assert(sizeof(mutex_seq) == 2 * sizeof(uint32_t), #error MUTEX_GETSEQ_ADDR assumes little endian layout of 2 32-bit sequence words #endif -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void -MUTEX_GETSEQ_ADDR(_pthread_mutex *mutex, mutex_seq **seqaddr) +MUTEX_GETSEQ_ADDR(pthread_mutex_t *mutex, mutex_seq **seqaddr) { // 64-bit aligned address inside m_seq array (&m_seq[0] for aligned mutex) // We don't require more than byte alignment on OS X. rdar://22278325 - *seqaddr = (void *)(((uintptr_t)mutex->m_seq + 0x7ul) & ~0x7ul); + *seqaddr = (void *)(((uintptr_t)mutex->psynch.m_seq + 0x7ul) & ~0x7ul); } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void -MUTEX_GETTID_ADDR(_pthread_mutex *mutex, uint64_t **tidaddr) +MUTEX_GETTID_ADDR(pthread_mutex_t *mutex, uint64_t **tidaddr) { // 64-bit aligned address inside m_tid array (&m_tid[0] for aligned mutex) // We don't require more than byte alignment on OS X. rdar://22278325 - *tidaddr = (void*)(((uintptr_t)mutex->m_tid + 0x7ul) & ~0x7ul); + *tidaddr = (void*)(((uintptr_t)mutex->psynch.m_tid + 0x7ul) & ~0x7ul); } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void mutex_seq_load(mutex_seq *seqaddr, mutex_seq *oldseqval) { @@ -201,7 +215,7 @@ mutex_seq_load(mutex_seq *seqaddr, mutex_seq *oldseqval) #define mutex_seq_atomic_load(seqaddr, oldseqval, m) \ mutex_seq_atomic_load_##m(seqaddr, oldseqval) -PTHREAD_ALWAYS_INLINE PTHREAD_USED +OS_ALWAYS_INLINE OS_USED static inline bool mutex_seq_atomic_cmpxchgv_relaxed(mutex_seq *seqaddr, mutex_seq *oldseqval, mutex_seq *newseqval) @@ -210,7 +224,7 @@ mutex_seq_atomic_cmpxchgv_relaxed(mutex_seq *seqaddr, mutex_seq *oldseqval, newseqval->seq_LU, &oldseqval->seq_LU, relaxed); } -PTHREAD_ALWAYS_INLINE PTHREAD_USED +OS_ALWAYS_INLINE OS_USED static inline bool mutex_seq_atomic_cmpxchgv_acquire(mutex_seq *seqaddr, mutex_seq *oldseqval, mutex_seq *newseqval) @@ -219,7 +233,7 @@ mutex_seq_atomic_cmpxchgv_acquire(mutex_seq *seqaddr, mutex_seq *oldseqval, newseqval->seq_LU, &oldseqval->seq_LU, acquire); } -PTHREAD_ALWAYS_INLINE PTHREAD_USED +OS_ALWAYS_INLINE OS_USED static inline bool mutex_seq_atomic_cmpxchgv_release(mutex_seq *seqaddr, mutex_seq *oldseqval, mutex_seq *newseqval) @@ -237,7 +251,7 @@ mutex_seq_atomic_cmpxchgv_release(mutex_seq *seqaddr, mutex_seq *oldseqval, */ PTHREAD_NOEXPORT_VARIANT int -pthread_mutex_init(pthread_mutex_t *omutex, const pthread_mutexattr_t *attr) +pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr) { #if 0 /* conformance tests depend on not having this behavior */ @@ -245,46 +259,42 @@ pthread_mutex_init(pthread_mutex_t *omutex, const pthread_mutexattr_t *attr) if (_pthread_mutex_check_signature(mutex)) return EBUSY; #endif - _pthread_mutex *mutex = (_pthread_mutex *)omutex; - _PTHREAD_LOCK_INIT(mutex->lock); + _pthread_lock_init(&mutex->lock); return (_pthread_mutex_init(mutex, attr, 0x7)); } -PTHREAD_NOEXPORT_VARIANT + int pthread_mutex_getprioceiling(const pthread_mutex_t *omutex, int *prioceiling) { int res = EINVAL; - _pthread_mutex *mutex = (_pthread_mutex *)omutex; + pthread_mutex_t *mutex = (pthread_mutex_t *)omutex; if (_pthread_mutex_check_signature(mutex)) { - _PTHREAD_LOCK(mutex->lock); + _pthread_lock_lock(&mutex->lock); *prioceiling = mutex->prioceiling; res = 0; - _PTHREAD_UNLOCK(mutex->lock); + _pthread_lock_unlock(&mutex->lock); } return res; } -PTHREAD_NOEXPORT_VARIANT int -pthread_mutex_setprioceiling(pthread_mutex_t *omutex, int prioceiling, +pthread_mutex_setprioceiling(pthread_mutex_t *mutex, int prioceiling, int *old_prioceiling) { int res = EINVAL; - _pthread_mutex *mutex = (_pthread_mutex *)omutex; if (_pthread_mutex_check_signature(mutex)) { - _PTHREAD_LOCK(mutex->lock); + _pthread_lock_lock(&mutex->lock); if (prioceiling >= -999 && prioceiling <= 999) { *old_prioceiling = mutex->prioceiling; mutex->prioceiling = (int16_t)prioceiling; res = 0; } - _PTHREAD_UNLOCK(mutex->lock); + _pthread_lock_unlock(&mutex->lock); } return res; } - int pthread_mutexattr_getprioceiling(const pthread_mutexattr_t *attr, int *prioceiling) @@ -380,12 +390,12 @@ pthread_mutexattr_setprotocol(pthread_mutexattr_t *attr, int protocol) int res = EINVAL; if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) { switch (protocol) { - case PTHREAD_PRIO_NONE: - case PTHREAD_PRIO_INHERIT: - case PTHREAD_PRIO_PROTECT: - attr->protocol = protocol; - res = 0; - break; + case PTHREAD_PRIO_NONE: + case PTHREAD_PRIO_INHERIT: + case PTHREAD_PRIO_PROTECT: + attr->protocol = protocol; + res = 0; + break; } } return res; @@ -419,13 +429,13 @@ pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) int res = EINVAL; if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) { switch (type) { - case PTHREAD_MUTEX_NORMAL: - case PTHREAD_MUTEX_ERRORCHECK: - case PTHREAD_MUTEX_RECURSIVE: - //case PTHREAD_MUTEX_DEFAULT: - attr->type = type; - res = 0; - break; + case PTHREAD_MUTEX_NORMAL: + case PTHREAD_MUTEX_ERRORCHECK: + case PTHREAD_MUTEX_RECURSIVE: + //case PTHREAD_MUTEX_DEFAULT: + attr->type = type; + res = 0; + break; } } return res; @@ -435,19 +445,9 @@ int pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared) { int res = EINVAL; -#if __DARWIN_UNIX03 - if (__unix_conforming == 0) { - __unix_conforming = 1; - } -#endif /* __DARWIN_UNIX03 */ - if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) { -#if __DARWIN_UNIX03 if (( pshared == PTHREAD_PROCESS_PRIVATE) || (pshared == PTHREAD_PROCESS_SHARED)) -#else /* __DARWIN_UNIX03 */ - if ( pshared == PTHREAD_PROCESS_PRIVATE) -#endif /* __DARWIN_UNIX03 */ { attr->pshared = pshared; res = 0; @@ -456,23 +456,23 @@ pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared) return res; } -PTHREAD_NOEXPORT PTHREAD_NOINLINE PTHREAD_NORETURN +OS_NOINLINE int -_pthread_mutex_corruption_abort(_pthread_mutex *mutex) +_pthread_mutex_corruption_abort(pthread_mutex_t *mutex) { PTHREAD_CLIENT_CRASH(0, "pthread_mutex corruption: mutex owner changed " "in the middle of lock/unlock"); } -PTHREAD_NOINLINE +OS_NOINLINE static int -_pthread_mutex_check_init_slow(_pthread_mutex *mutex) +_pthread_mutex_check_init_slow(pthread_mutex_t *mutex) { int res = EINVAL; if (_pthread_mutex_check_signature_init(mutex)) { - _PTHREAD_LOCK(mutex->lock); + _pthread_lock_lock(&mutex->lock); if (_pthread_mutex_check_signature_init(mutex)) { // initialize a statically initialized mutex to provide // compatibility for misbehaving applications. @@ -481,7 +481,7 @@ _pthread_mutex_check_init_slow(_pthread_mutex *mutex) } else if (_pthread_mutex_check_signature(mutex)) { res = 0; } - _PTHREAD_UNLOCK(mutex->lock); + _pthread_lock_unlock(&mutex->lock); } else if (_pthread_mutex_check_signature(mutex)) { res = 0; } @@ -491,9 +491,9 @@ _pthread_mutex_check_init_slow(_pthread_mutex *mutex) return res; } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int -_pthread_mutex_check_init(_pthread_mutex *mutex) +_pthread_mutex_check_init(pthread_mutex_t *mutex) { int res = 0; if (!_pthread_mutex_check_signature(mutex)) { @@ -502,30 +502,30 @@ _pthread_mutex_check_init(_pthread_mutex *mutex) return res; } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline bool -_pthread_mutex_is_fairshare(_pthread_mutex *mutex) +_pthread_mutex_is_fairshare(pthread_mutex_t *mutex) { return (mutex->mtxopts.options.policy == _PTHREAD_MTX_OPT_POLICY_FAIRSHARE); } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline bool -_pthread_mutex_is_firstfit(_pthread_mutex *mutex) +_pthread_mutex_is_firstfit(pthread_mutex_t *mutex) { return (mutex->mtxopts.options.policy == _PTHREAD_MTX_OPT_POLICY_FIRSTFIT); } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline bool -_pthread_mutex_is_recursive(_pthread_mutex *mutex) +_pthread_mutex_is_recursive(pthread_mutex_t *mutex) { return (mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE); } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static int -_pthread_mutex_lock_handle_options(_pthread_mutex *mutex, bool trylock, +_pthread_mutex_lock_handle_options(pthread_mutex_t *mutex, bool trylock, uint64_t *tidaddr) { if (mutex->mtxopts.options.type == PTHREAD_MUTEX_NORMAL) { @@ -533,8 +533,8 @@ _pthread_mutex_lock_handle_options(_pthread_mutex *mutex, bool trylock, return 0; } - uint64_t selfid = _pthread_selfid_direct(); - if (os_atomic_load(tidaddr, relaxed) == selfid) { + uint64_t selfid = _pthread_threadid_self_np_direct(); + if (os_atomic_load_wide(tidaddr, relaxed) == selfid) { if (_pthread_mutex_is_recursive(mutex)) { if (mutex->mtxopts.options.lock_count < USHRT_MAX) { mutex->mtxopts.options.lock_count += 1; @@ -555,17 +555,17 @@ _pthread_mutex_lock_handle_options(_pthread_mutex *mutex, bool trylock, return 0; } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static int -_pthread_mutex_unlock_handle_options(_pthread_mutex *mutex, uint64_t *tidaddr) +_pthread_mutex_unlock_handle_options(pthread_mutex_t *mutex, uint64_t *tidaddr) { if (mutex->mtxopts.options.type == PTHREAD_MUTEX_NORMAL) { // NORMAL does not do EDEADLK checking return 0; } - uint64_t selfid = _pthread_selfid_direct(); - if (os_atomic_load(tidaddr, relaxed) != selfid) { + uint64_t selfid = _pthread_threadid_self_np_direct(); + if (os_atomic_load_wide(tidaddr, relaxed) != selfid) { return -EPERM; } else if (_pthread_mutex_is_recursive(mutex) && --mutex->mtxopts.options.lock_count) { @@ -604,9 +604,9 @@ _pthread_mutex_unlock_handle_options(_pthread_mutex *mutex, uint64_t *tidaddr) /* * Drop the mutex unlock references from cond_wait or mutex_unlock. */ -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int -_pthread_mutex_fairshare_unlock_updatebits(_pthread_mutex *mutex, +_pthread_mutex_fairshare_unlock_updatebits(pthread_mutex_t *mutex, uint32_t *flagsp, uint32_t **pmtxp, uint32_t *mgenp, uint32_t *ugenp) { uint32_t flags = mutex->mtxopts.value; @@ -638,7 +638,7 @@ _pthread_mutex_fairshare_unlock_updatebits(_pthread_mutex *mutex, bool clearnotify, spurious; do { newseq = oldseq; - oldtid = os_atomic_load(tidaddr, relaxed); + oldtid = os_atomic_load_wide(tidaddr, relaxed); clearnotify = false; spurious = false; @@ -703,9 +703,9 @@ _pthread_mutex_fairshare_unlock_updatebits(_pthread_mutex *mutex, return 0; } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int -_pthread_mutex_fairshare_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid) +_pthread_mutex_fairshare_lock_updatebits(pthread_mutex_t *mutex, uint64_t selfid) { bool firstfit = _pthread_mutex_is_firstfit(mutex); bool gotlock = true; @@ -739,7 +739,7 @@ _pthread_mutex_fairshare_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid) acquire)); if (gotlock) { - os_atomic_store(tidaddr, selfid, relaxed); + os_atomic_store_wide(tidaddr, selfid, relaxed); } PTHREAD_TRACE(psynch_mutex_lock_updatebits, mutex, oldseq.lgenval, @@ -750,14 +750,14 @@ _pthread_mutex_fairshare_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid) return gotlock ? 0 : 1; } -PTHREAD_NOINLINE +OS_NOINLINE static int -_pthread_mutex_fairshare_lock_wait(_pthread_mutex *mutex, mutex_seq newseq, +_pthread_mutex_fairshare_lock_wait(pthread_mutex_t *mutex, mutex_seq newseq, uint64_t oldtid) { uint64_t *tidaddr; MUTEX_GETTID_ADDR(mutex, &tidaddr); - uint64_t selfid = _pthread_selfid_direct(); + uint64_t selfid = _pthread_threadid_self_np_direct(); PLOCKSTAT_MUTEX_BLOCK((pthread_mutex_t *)mutex); do { @@ -765,7 +765,7 @@ _pthread_mutex_fairshare_lock_wait(_pthread_mutex *mutex, mutex_seq newseq, do { updateval = __psynch_mutexwait(mutex, newseq.lgenval, newseq.ugenval, oldtid, mutex->mtxopts.value); - oldtid = os_atomic_load(tidaddr, relaxed); + oldtid = os_atomic_load_wide(tidaddr, relaxed); } while (updateval == (uint32_t)-1); // returns 0 on succesful update; in firstfit it may fail with 1 @@ -775,12 +775,11 @@ _pthread_mutex_fairshare_lock_wait(_pthread_mutex *mutex, mutex_seq newseq, return 0; } -PTHREAD_NOEXPORT PTHREAD_NOINLINE +OS_NOINLINE int -_pthread_mutex_fairshare_lock_slow(_pthread_mutex *omutex, bool trylock) +_pthread_mutex_fairshare_lock_slow(pthread_mutex_t *mutex, bool trylock) { int res, recursive = 0; - _pthread_mutex *mutex = (_pthread_mutex *)omutex; mutex_seq *seqaddr; MUTEX_GETSEQ_ADDR(mutex, &seqaddr); @@ -790,7 +789,7 @@ _pthread_mutex_fairshare_lock_slow(_pthread_mutex *omutex, bool trylock) uint64_t *tidaddr; MUTEX_GETTID_ADDR(mutex, &tidaddr); - uint64_t oldtid, selfid = _pthread_selfid_direct(); + uint64_t oldtid, selfid = _pthread_threadid_self_np_direct(); res = _pthread_mutex_lock_handle_options(mutex, trylock, tidaddr); if (res > 0) { @@ -805,7 +804,7 @@ _pthread_mutex_fairshare_lock_slow(_pthread_mutex *omutex, bool trylock) bool gotlock; do { newseq = oldseq; - oldtid = os_atomic_load(tidaddr, relaxed); + oldtid = os_atomic_load_wide(tidaddr, relaxed); gotlock = ((oldseq.lgenval & PTH_RWL_EBIT) == 0); @@ -821,23 +820,23 @@ _pthread_mutex_fairshare_lock_slow(_pthread_mutex *omutex, bool trylock) } } while (!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, acquire)); - PTHREAD_TRACE(psynch_mutex_lock_updatebits, omutex, oldseq.lgenval, + PTHREAD_TRACE(psynch_mutex_lock_updatebits, mutex, oldseq.lgenval, newseq.lgenval, 0); if (gotlock) { - os_atomic_store(tidaddr, selfid, relaxed); + os_atomic_store_wide(tidaddr, selfid, relaxed); res = 0; - PTHREAD_TRACE(psynch_mutex_ulock, omutex, newseq.lgenval, + PTHREAD_TRACE(psynch_mutex_ulock, mutex, newseq.lgenval, newseq.ugenval, selfid); } else if (trylock) { res = EBUSY; - PTHREAD_TRACE(psynch_mutex_utrylock_failed, omutex, newseq.lgenval, + PTHREAD_TRACE(psynch_mutex_utrylock_failed, mutex, newseq.lgenval, newseq.ugenval, oldtid); } else { - PTHREAD_TRACE(psynch_mutex_ulock | DBG_FUNC_START, omutex, + PTHREAD_TRACE(psynch_mutex_ulock | DBG_FUNC_START, mutex, newseq.lgenval, newseq.ugenval, oldtid); res = _pthread_mutex_fairshare_lock_wait(mutex, newseq, oldtid); - PTHREAD_TRACE(psynch_mutex_ulock | DBG_FUNC_END, omutex, + PTHREAD_TRACE(psynch_mutex_ulock | DBG_FUNC_END, mutex, newseq.lgenval, newseq.ugenval, oldtid); } @@ -857,9 +856,9 @@ out: return res; } -PTHREAD_NOINLINE +OS_NOINLINE static inline int -_pthread_mutex_fairshare_lock(_pthread_mutex *mutex, bool trylock) +_pthread_mutex_fairshare_lock(pthread_mutex_t *mutex, bool trylock) { #if ENABLE_USERSPACE_TRACE return _pthread_mutex_fairshare_lock_slow(mutex, trylock); @@ -871,7 +870,7 @@ _pthread_mutex_fairshare_lock(_pthread_mutex *mutex, bool trylock) uint64_t *tidaddr; MUTEX_GETTID_ADDR(mutex, &tidaddr); - uint64_t selfid = _pthread_selfid_direct(); + uint64_t selfid = _pthread_threadid_self_np_direct(); mutex_seq *seqaddr; MUTEX_GETSEQ_ADDR(mutex, &seqaddr); @@ -905,7 +904,7 @@ _pthread_mutex_fairshare_lock(_pthread_mutex *mutex, bool trylock) acquire))); if (os_likely(gotlock)) { - os_atomic_store(tidaddr, selfid, relaxed); + os_atomic_store_wide(tidaddr, selfid, relaxed); return 0; } else if (trylock) { return EBUSY; @@ -914,9 +913,9 @@ _pthread_mutex_fairshare_lock(_pthread_mutex *mutex, bool trylock) } } -PTHREAD_NOINLINE +OS_NOINLINE static int -_pthread_mutex_fairshare_unlock_drop(_pthread_mutex *mutex, mutex_seq newseq, +_pthread_mutex_fairshare_unlock_drop(pthread_mutex_t *mutex, mutex_seq newseq, uint32_t flags) { int res; @@ -926,10 +925,10 @@ _pthread_mutex_fairshare_unlock_drop(_pthread_mutex *mutex, mutex_seq newseq, MUTEX_GETTID_ADDR(mutex, &tidaddr); PTHREAD_TRACE(psynch_mutex_uunlock | DBG_FUNC_START, mutex, newseq.lgenval, - newseq.ugenval, os_atomic_load(tidaddr, relaxed)); + newseq.ugenval, os_atomic_load_wide(tidaddr, relaxed)); updateval = __psynch_mutexdrop(mutex, newseq.lgenval, newseq.ugenval, - os_atomic_load(tidaddr, relaxed), flags); + os_atomic_load_wide(tidaddr, relaxed), flags); PTHREAD_TRACE(psynch_mutex_uunlock | DBG_FUNC_END, mutex, updateval, 0, 0); @@ -948,9 +947,9 @@ _pthread_mutex_fairshare_unlock_drop(_pthread_mutex *mutex, mutex_seq newseq, return 0; } -PTHREAD_NOEXPORT PTHREAD_NOINLINE +OS_NOINLINE int -_pthread_mutex_fairshare_unlock_slow(_pthread_mutex *mutex) +_pthread_mutex_fairshare_unlock_slow(pthread_mutex_t *mutex) { int res; mutex_seq newseq; @@ -966,15 +965,15 @@ _pthread_mutex_fairshare_unlock_slow(_pthread_mutex *mutex) uint64_t *tidaddr; MUTEX_GETTID_ADDR(mutex, &tidaddr); PTHREAD_TRACE(psynch_mutex_uunlock, mutex, newseq.lgenval, - newseq.ugenval, os_atomic_load(tidaddr, relaxed)); + newseq.ugenval, os_atomic_load_wide(tidaddr, relaxed)); } return 0; } -PTHREAD_NOINLINE +OS_NOINLINE static int -_pthread_mutex_fairshare_unlock(_pthread_mutex *mutex) +_pthread_mutex_fairshare_unlock(pthread_mutex_t *mutex) { #if ENABLE_USERSPACE_TRACE return _pthread_mutex_fairshare_unlock_slow(mutex); @@ -1004,7 +1003,7 @@ _pthread_mutex_fairshare_unlock(_pthread_mutex *mutex) // is no stale ownership information. If the CAS of the seqaddr // fails, we may loop, but it's still valid for the owner to be // SWITCHING/0 - os_atomic_store(tidaddr, 0, relaxed); + os_atomic_store_wide(tidaddr, 0, relaxed); do { newseq = oldseq; @@ -1028,11 +1027,205 @@ _pthread_mutex_fairshare_unlock(_pthread_mutex *mutex) return 0; } +#pragma mark ulock + +OS_ALWAYS_INLINE +static inline uint32_t +_pthread_mutex_ulock_self_owner_value(void) +{ + mach_port_t self_port = _pthread_mach_thread_self_direct(); + return self_port & _PTHREAD_MUTEX_ULOCK_OWNER_MASK; +} + +OS_NOINLINE +static int +_pthread_mutex_ulock_lock_slow(pthread_mutex_t *mutex, uint32_t self_ownerval, + uint32_t state) +{ + bool success = false, kernel_waiters = false; + + uint32_t wait_op = UL_UNFAIR_LOCK | ULF_NO_ERRNO; + if (__pthread_mutex_ulock_adaptive_spin) { + wait_op |= ULF_WAIT_ADAPTIVE_SPIN; + } + + PLOCKSTAT_MUTEX_BLOCK((pthread_mutex_t *)mutex); + do { + bool owner_dead = false; + + do { + uint32_t current_ownerval = state & _PTHREAD_MUTEX_ULOCK_OWNER_MASK; + if (os_unlikely(owner_dead)) { + // TODO: PTHREAD_STRICT candidate + // + // For a non-recursive mutex, this indicates that it's really + // being used as a semaphore: even though we're the current + // owner, in reality we're expecting another thread to 'unlock' + // this mutex on our behalf later. + // + // __ulock_wait(2) doesn't permit you to wait for yourself, so + // we need to first swap our ownership for the anonymous owner + current_ownerval = + MACH_PORT_DEAD & _PTHREAD_MUTEX_ULOCK_OWNER_MASK; + owner_dead = false; + } + uint32_t new_state = + current_ownerval | _PTHREAD_MUTEX_ULOCK_WAITERS_BIT; + success = os_atomic_cmpxchgv(&mutex->ulock.uval, state, new_state, + &state, relaxed); + if (!success) { + continue; + } + + int rc = __ulock_wait(wait_op, &mutex->ulock, new_state, 0); + + PTHREAD_TRACE(ulmutex_lock_wait, mutex, new_state, rc, 0); + + if (os_unlikely(rc < 0)) { + switch (-rc) { + case EINTR: + case EFAULT: + break; + case EOWNERDEAD: + owner_dead = true; + continue; + default: + PTHREAD_INTERNAL_CRASH(rc, "ulock_wait failure"); + } + } else if (rc > 0) { + kernel_waiters = true; + } + + state = os_atomic_load(&mutex->ulock.uval, relaxed); + } while (state != _PTHREAD_MUTEX_ULOCK_UNLOCKED_VALUE); + + uint32_t locked_state = self_ownerval; + if (kernel_waiters) { + locked_state |= _PTHREAD_MUTEX_ULOCK_WAITERS_BIT; + } + + success = os_atomic_cmpxchgv(&mutex->ulock.uval, state, locked_state, + &state, acquire); + } while (!success); + PLOCKSTAT_MUTEX_BLOCKED((pthread_mutex_t *)mutex, BLOCK_SUCCESS_PLOCKSTAT); + + return 0; +} + +PTHREAD_NOEXPORT_VARIANT +int +_pthread_mutex_ulock_lock(pthread_mutex_t *mutex, bool trylock) +{ + uint32_t unlocked = _PTHREAD_MUTEX_ULOCK_UNLOCKED_VALUE; + uint32_t locked = _pthread_mutex_ulock_self_owner_value(); + uint32_t state; + + bool success = os_atomic_cmpxchgv(&mutex->ulock.uval, unlocked, locked, + &state, acquire); + + if (trylock) { + PTHREAD_TRACE(ulmutex_trylock, mutex, locked, state, success); + } else { + PTHREAD_TRACE(ulmutex_lock, mutex, locked, state, success); + } + + int rc = 0; + if (!success) { + if (trylock) { + rc = EBUSY; + } else { + rc = _pthread_mutex_ulock_lock_slow(mutex, locked, state); + } + } + + if (rc) { + PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, rc); + } else { + PLOCKSTAT_MUTEX_ACQUIRE((pthread_mutex_t *)mutex, /* recursive */ 0, 0); + } + + return rc; +} + +OS_NOINLINE +static int +_pthread_mutex_ulock_unlock_slow(pthread_mutex_t *mutex, uint32_t self_ownerval, + uint32_t orig_state) +{ + if (os_unlikely(orig_state == _PTHREAD_MUTEX_ULOCK_UNLOCKED_VALUE)) { + // XXX This is illegal, but psynch permitted it... + // TODO: PTHREAD_STRICT candidate + return 0; + } + + uint32_t wake_flags = 0; + + uint32_t orig_ownerval = orig_state & _PTHREAD_MUTEX_ULOCK_OWNER_MASK; + bool orig_waiters = orig_state & _PTHREAD_MUTEX_ULOCK_WAITERS_BIT; + if (os_unlikely(orig_ownerval != self_ownerval)) { + // XXX This is illegal, but psynch permitted it... + // TODO: PTHREAD_STRICT candidate + if (!orig_waiters) { + return 0; + } + + wake_flags |= ULF_WAKE_ALLOW_NON_OWNER; + } else if (os_unlikely(!orig_waiters)) { + PTHREAD_INTERNAL_CRASH(0, "unlock_slow without orig_waiters"); + } + + for (;;) { + int rc = __ulock_wake(UL_UNFAIR_LOCK | ULF_NO_ERRNO | wake_flags, + &mutex->ulock, 0); + + PTHREAD_TRACE(ulmutex_unlock_wake, mutex, rc, 0, 0); + + if (os_unlikely(rc < 0)) { + switch (-rc) { + case EINTR: + continue; + case ENOENT: + break; + default: + PTHREAD_INTERNAL_CRASH(-rc, "ulock_wake failure"); + } + } + break; + } + + return 0; +} + +PTHREAD_NOEXPORT_VARIANT +int +_pthread_mutex_ulock_unlock(pthread_mutex_t *mutex) +{ + uint32_t locked_uncontended = _pthread_mutex_ulock_self_owner_value(); + uint32_t unlocked = _PTHREAD_MUTEX_ULOCK_UNLOCKED_VALUE; + uint32_t state = os_atomic_xchg(&mutex->ulock.uval, unlocked, release); + + PTHREAD_TRACE(ulmutex_unlock, mutex, locked_uncontended, state, 0); + + int rc = 0; + if (state != locked_uncontended) { + rc = _pthread_mutex_ulock_unlock_slow(mutex, locked_uncontended, + state); + } + + if (rc) { + PLOCKSTAT_MUTEX_ERROR((pthread_mutex_t *)mutex, rc); + } else { + PLOCKSTAT_MUTEX_RELEASE((pthread_mutex_t *)mutex, /* recursive */ 0); + } + + return rc; +} + #pragma mark firstfit -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int -_pthread_mutex_firstfit_unlock_updatebits(_pthread_mutex *mutex, +_pthread_mutex_firstfit_unlock_updatebits(pthread_mutex_t *mutex, uint32_t *flagsp, uint32_t **mutexp, uint32_t *lvalp, uint32_t *uvalp) { uint32_t flags = mutex->mtxopts.value & ~_PTHREAD_MTX_OPT_NOTIFY; @@ -1063,7 +1256,7 @@ _pthread_mutex_firstfit_unlock_updatebits(_pthread_mutex *mutex, do { newseq = oldseq; - oldtid = os_atomic_load(tidaddr, relaxed); + oldtid = os_atomic_load_wide(tidaddr, relaxed); // More than one kernel waiter means we need to do a wake. kernel_wake = diff_genseq(oldseq.lgenval, oldseq.ugenval) > 0; newseq.lgenval &= ~PTH_RWL_EBIT; @@ -1105,9 +1298,9 @@ _pthread_mutex_firstfit_unlock_updatebits(_pthread_mutex *mutex, return 0; } -PTHREAD_NOEXPORT PTHREAD_NOINLINE +OS_NOINLINE static int -_pthread_mutex_firstfit_wake(_pthread_mutex *mutex, mutex_seq newseq, +_pthread_mutex_firstfit_wake(pthread_mutex_t *mutex, mutex_seq newseq, uint32_t flags) { PTHREAD_TRACE(psynch_ffmutex_wake, mutex, newseq.lgenval, newseq.ugenval, @@ -1128,9 +1321,9 @@ _pthread_mutex_firstfit_wake(_pthread_mutex *mutex, mutex_seq newseq, return 0; } -PTHREAD_NOEXPORT PTHREAD_NOINLINE +OS_NOINLINE int -_pthread_mutex_firstfit_unlock_slow(_pthread_mutex *mutex) +_pthread_mutex_firstfit_unlock_slow(pthread_mutex_t *mutex) { mutex_seq newseq; uint32_t flags; @@ -1146,9 +1339,9 @@ _pthread_mutex_firstfit_unlock_slow(_pthread_mutex *mutex) return 0; } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static bool -_pthread_mutex_firstfit_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid, +_pthread_mutex_firstfit_lock_updatebits(pthread_mutex_t *mutex, uint64_t selfid, mutex_seq *newseqp) { bool gotlock; @@ -1182,7 +1375,7 @@ _pthread_mutex_firstfit_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid, } while (!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, acquire)); if (gotlock) { - os_atomic_store(tidaddr, selfid, relaxed); + os_atomic_store_wide(tidaddr, selfid, relaxed); } PTHREAD_TRACE(psynch_ffmutex_lock_updatebits | DBG_FUNC_END, mutex, @@ -1194,14 +1387,14 @@ _pthread_mutex_firstfit_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid, return gotlock; } -PTHREAD_NOINLINE +OS_NOINLINE static int -_pthread_mutex_firstfit_lock_wait(_pthread_mutex *mutex, mutex_seq newseq, +_pthread_mutex_firstfit_lock_wait(pthread_mutex_t *mutex, mutex_seq newseq, uint64_t oldtid) { uint64_t *tidaddr; MUTEX_GETTID_ADDR(mutex, &tidaddr); - uint64_t selfid = _pthread_selfid_direct(); + uint64_t selfid = _pthread_threadid_self_np_direct(); PLOCKSTAT_MUTEX_BLOCK((pthread_mutex_t *)mutex); do { @@ -1213,7 +1406,7 @@ _pthread_mutex_firstfit_lock_wait(_pthread_mutex *mutex, mutex_seq newseq, oldtid, mutex->mtxopts.value); PTHREAD_TRACE(psynch_ffmutex_wait | DBG_FUNC_END, mutex, uval, 0, 0); - oldtid = os_atomic_load(tidaddr, relaxed); + oldtid = os_atomic_load_wide(tidaddr, relaxed); } while (uval == (uint32_t)-1); } while (!_pthread_mutex_firstfit_lock_updatebits(mutex, selfid, &newseq)); PLOCKSTAT_MUTEX_BLOCKED((pthread_mutex_t *)mutex, BLOCK_SUCCESS_PLOCKSTAT); @@ -1221,9 +1414,9 @@ _pthread_mutex_firstfit_lock_wait(_pthread_mutex *mutex, mutex_seq newseq, return 0; } -PTHREAD_NOEXPORT PTHREAD_NOINLINE +OS_NOINLINE int -_pthread_mutex_firstfit_lock_slow(_pthread_mutex *mutex, bool trylock) +_pthread_mutex_firstfit_lock_slow(pthread_mutex_t *mutex, bool trylock) { int res, recursive = 0; @@ -1235,7 +1428,7 @@ _pthread_mutex_firstfit_lock_slow(_pthread_mutex *mutex, bool trylock) uint64_t *tidaddr; MUTEX_GETTID_ADDR(mutex, &tidaddr); - uint64_t oldtid, selfid = _pthread_selfid_direct(); + uint64_t oldtid, selfid = _pthread_threadid_self_np_direct(); res = _pthread_mutex_lock_handle_options(mutex, trylock, tidaddr); if (res > 0) { @@ -1253,7 +1446,7 @@ _pthread_mutex_firstfit_lock_slow(_pthread_mutex *mutex, bool trylock) bool gotlock; do { newseq = oldseq; - oldtid = os_atomic_load(tidaddr, relaxed); + oldtid = os_atomic_load_wide(tidaddr, relaxed); gotlock = is_rwl_ebit_clear(oldseq.lgenval); if (trylock && !gotlock) { @@ -1274,7 +1467,7 @@ _pthread_mutex_firstfit_lock_slow(_pthread_mutex *mutex, bool trylock) newseq.lgenval, newseq.ugenval, 0); if (gotlock) { - os_atomic_store(tidaddr, selfid, relaxed); + os_atomic_store_wide(tidaddr, selfid, relaxed); res = 0; PTHREAD_TRACE(psynch_mutex_ulock, mutex, newseq.lgenval, newseq.ugenval, selfid); @@ -1307,9 +1500,9 @@ out: #pragma mark fast path -PTHREAD_NOEXPORT PTHREAD_NOINLINE +OS_NOINLINE int -_pthread_mutex_droplock(_pthread_mutex *mutex, uint32_t *flagsp, +_pthread_mutex_droplock(pthread_mutex_t *mutex, uint32_t *flagsp, uint32_t **pmtxp, uint32_t *mgenp, uint32_t *ugenp) { if (_pthread_mutex_is_fairshare(mutex)) { @@ -1320,9 +1513,9 @@ _pthread_mutex_droplock(_pthread_mutex *mutex, uint32_t *flagsp, mgenp, ugenp); } -PTHREAD_NOEXPORT PTHREAD_NOINLINE +OS_NOINLINE int -_pthread_mutex_lock_init_slow(_pthread_mutex *mutex, bool trylock) +_pthread_mutex_lock_init_slow(pthread_mutex_t *mutex, bool trylock) { int res; @@ -1331,13 +1524,15 @@ _pthread_mutex_lock_init_slow(_pthread_mutex *mutex, bool trylock) if (os_unlikely(_pthread_mutex_is_fairshare(mutex))) { return _pthread_mutex_fairshare_lock_slow(mutex, trylock); + } else if (os_unlikely(_pthread_mutex_uses_ulock(mutex))) { + return _pthread_mutex_ulock_lock(mutex, trylock); } return _pthread_mutex_firstfit_lock_slow(mutex, trylock); } -PTHREAD_NOEXPORT PTHREAD_NOINLINE +OS_NOINLINE static int -_pthread_mutex_unlock_init_slow(_pthread_mutex *mutex) +_pthread_mutex_unlock_init_slow(pthread_mutex_t *mutex) { int res; @@ -1348,15 +1543,16 @@ _pthread_mutex_unlock_init_slow(_pthread_mutex *mutex) if (os_unlikely(_pthread_mutex_is_fairshare(mutex))) { return _pthread_mutex_fairshare_unlock_slow(mutex); + } else if (os_unlikely(_pthread_mutex_uses_ulock(mutex))) { + return _pthread_mutex_ulock_unlock(mutex); } return _pthread_mutex_firstfit_unlock_slow(mutex); } PTHREAD_NOEXPORT_VARIANT int -pthread_mutex_unlock(pthread_mutex_t *omutex) +pthread_mutex_unlock(pthread_mutex_t *mutex) { - _pthread_mutex *mutex = (_pthread_mutex *)omutex; if (os_unlikely(!_pthread_mutex_check_signature_fast(mutex))) { return _pthread_mutex_unlock_init_slow(mutex); } @@ -1365,6 +1561,10 @@ pthread_mutex_unlock(pthread_mutex_t *omutex) return _pthread_mutex_fairshare_unlock(mutex); } + if (os_unlikely(_pthread_mutex_uses_ulock(mutex))) { + return _pthread_mutex_ulock_unlock(mutex); + } + #if ENABLE_USERSPACE_TRACE return _pthread_mutex_firstfit_unlock_slow(mutex); #elif PLOCKSTAT @@ -1391,7 +1591,7 @@ pthread_mutex_unlock(pthread_mutex_t *omutex) // is no stale ownership information. If the CAS of the seqaddr // fails, we may loop, but it's still valid for the owner to be // SWITCHING/0 - os_atomic_store(tidaddr, 0, relaxed); + os_atomic_store_wide(tidaddr, 0, relaxed); do { newseq = oldseq; @@ -1409,34 +1609,17 @@ pthread_mutex_unlock(pthread_mutex_t *omutex) return 0; } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int -_pthread_mutex_firstfit_lock(pthread_mutex_t *omutex, bool trylock) +_pthread_mutex_firstfit_lock(pthread_mutex_t *mutex, bool trylock) { - _pthread_mutex *mutex = (_pthread_mutex *)omutex; - if (os_unlikely(!_pthread_mutex_check_signature_fast(mutex))) { - return _pthread_mutex_lock_init_slow(mutex, trylock); - } - - if (os_unlikely(_pthread_mutex_is_fairshare(mutex))) { - return _pthread_mutex_fairshare_lock(mutex, trylock); - } - -#if ENABLE_USERSPACE_TRACE - return _pthread_mutex_firstfit_lock_slow(mutex, trylock); -#elif PLOCKSTAT - if (PLOCKSTAT_MUTEX_ACQUIRE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED()) { - return _pthread_mutex_firstfit_lock_slow(mutex, trylock); - } -#endif - /* * This is the first-fit fast path. The fairshare fast-ish path is in - * _pthread_mutex_firstfit_lock() + * _pthread_mutex_fairshare_lock() */ uint64_t *tidaddr; MUTEX_GETTID_ADDR(mutex, &tidaddr); - uint64_t selfid = _pthread_selfid_direct(); + uint64_t selfid = _pthread_threadid_self_np_direct(); mutex_seq *seqaddr; MUTEX_GETSEQ_ADDR(mutex, &seqaddr); @@ -1474,7 +1657,7 @@ _pthread_mutex_firstfit_lock(pthread_mutex_t *omutex, bool trylock) acquire))); if (os_likely(gotlock)) { - os_atomic_store(tidaddr, selfid, relaxed); + os_atomic_store_wide(tidaddr, selfid, relaxed); return 0; } else if (trylock) { return EBUSY; @@ -1483,24 +1666,51 @@ _pthread_mutex_firstfit_lock(pthread_mutex_t *omutex, bool trylock) } } +OS_ALWAYS_INLINE +static inline int +_pthread_mutex_lock(pthread_mutex_t *mutex, bool trylock) +{ + if (os_unlikely(!_pthread_mutex_check_signature_fast(mutex))) { + return _pthread_mutex_lock_init_slow(mutex, trylock); + } + + if (os_unlikely(_pthread_mutex_is_fairshare(mutex))) { + return _pthread_mutex_fairshare_lock(mutex, trylock); + } + + if (os_unlikely(_pthread_mutex_uses_ulock(mutex))) { + return _pthread_mutex_ulock_lock(mutex, trylock); + } + +#if ENABLE_USERSPACE_TRACE + return _pthread_mutex_firstfit_lock_slow(mutex, trylock); +#elif PLOCKSTAT + if (PLOCKSTAT_MUTEX_ACQUIRE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED()) { + return _pthread_mutex_firstfit_lock_slow(mutex, trylock); + } +#endif + + return _pthread_mutex_firstfit_lock(mutex, trylock); +} + PTHREAD_NOEXPORT_VARIANT int pthread_mutex_lock(pthread_mutex_t *mutex) { - return _pthread_mutex_firstfit_lock(mutex, false); + return _pthread_mutex_lock(mutex, false); } PTHREAD_NOEXPORT_VARIANT int pthread_mutex_trylock(pthread_mutex_t *mutex) { - return _pthread_mutex_firstfit_lock(mutex, true); + return _pthread_mutex_lock(mutex, true); } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int -_pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr, +_pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr, uint32_t static_type) { mutex->mtxopts.value = 0; @@ -1516,19 +1726,19 @@ _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr, mutex->mtxopts.options.pshared = attr->pshared; } else { switch (static_type) { - case 1: - mutex->mtxopts.options.type = PTHREAD_MUTEX_ERRORCHECK; - break; - case 2: - mutex->mtxopts.options.type = PTHREAD_MUTEX_RECURSIVE; - break; - case 3: - /* firstfit fall thru */ - case 7: - mutex->mtxopts.options.type = PTHREAD_MUTEX_DEFAULT; - break; - default: - return EINVAL; + case 1: + mutex->mtxopts.options.type = PTHREAD_MUTEX_ERRORCHECK; + break; + case 2: + mutex->mtxopts.options.type = PTHREAD_MUTEX_RECURSIVE; + break; + case 3: + /* firstfit fall thru */ + case 7: + mutex->mtxopts.options.type = PTHREAD_MUTEX_DEFAULT; + break; + default: + return EINVAL; } mutex->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING; @@ -1540,23 +1750,9 @@ _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr, } mutex->mtxopts.options.pshared = _PTHREAD_DEFAULT_PSHARED; } - mutex->priority = 0; - - mutex_seq *seqaddr; - MUTEX_GETSEQ_ADDR(mutex, &seqaddr); - uint64_t *tidaddr; - MUTEX_GETTID_ADDR(mutex, &tidaddr); + mutex->priority = 0; -#if PTHREAD_MUTEX_INIT_UNUSED - if ((uint32_t*)tidaddr != mutex->m_tid) { - mutex->mtxopts.options.misalign = 1; - __builtin_memset(mutex->m_tid, 0xff, sizeof(mutex->m_tid)); - } - __builtin_memset(mutex->m_mis, 0xff, sizeof(mutex->m_mis)); -#endif // PTHREAD_MUTEX_INIT_UNUSED - *tidaddr = 0; - *seqaddr = (mutex_seq){ }; long sig = _PTHREAD_MUTEX_SIG; if (mutex->mtxopts.options.type == PTHREAD_MUTEX_NORMAL && @@ -1566,6 +1762,48 @@ _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr, sig = _PTHREAD_MUTEX_SIG_fast; } + // Criteria for ulock eligility: + // - not ERRORCHECK or RECURSIVE + // - not FAIRSHARE + // - not PROCESS_SHARED + // - checkfix for rdar://21813573 not active + // + // All of these should be addressed eventually. + if (mutex->mtxopts.options.type == PTHREAD_MUTEX_NORMAL && + mutex->mtxopts.options.policy == _PTHREAD_MTX_OPT_POLICY_FIRSTFIT && + mutex->mtxopts.options.pshared == PTHREAD_PROCESS_PRIVATE && + sig == _PTHREAD_MUTEX_SIG_fast) { + mutex->mtxopts.options.ulock = __pthread_mutex_use_ulock; + } else { + mutex->mtxopts.options.ulock = false; + } + + if (mutex->mtxopts.options.ulock) { +#if PTHREAD_MUTEX_INIT_UNUSED + __builtin_memset(&mutex->psynch, 0xff, sizeof(mutex->psynch)); +#endif // PTHREAD_MUTEX_INIT_UNUSED + + mutex->ulock = _PTHREAD_MUTEX_ULOCK_UNLOCKED; + } else { + mutex_seq *seqaddr; + MUTEX_GETSEQ_ADDR(mutex, &seqaddr); + + uint64_t *tidaddr; + MUTEX_GETTID_ADDR(mutex, &tidaddr); + +#if PTHREAD_MUTEX_INIT_UNUSED + if ((uint32_t*)tidaddr != mutex->psynch.m_tid) { + // TODO: PTHREAD_STRICT candidate + mutex->mtxopts.options.misalign = 1; + __builtin_memset(mutex->psynch.m_tid, 0xff, + sizeof(mutex->psynch.m_tid)); + } + __builtin_memset(mutex->psynch.m_mis, 0xff, sizeof(mutex->psynch.m_mis)); +#endif // PTHREAD_MUTEX_INIT_UNUSED + *tidaddr = 0; + *seqaddr = (mutex_seq){ }; + } + #if PTHREAD_MUTEX_INIT_UNUSED // For detecting copied mutexes and smashes during debugging uint32_t sig32 = (uint32_t)sig; @@ -1589,7 +1827,7 @@ _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr, *(sig32_ptr + 1) = *(sig32_val + 1); os_atomic_store(sig32_ptr, *sig32_val, release); #else - os_atomic_store2o(mutex, sig, sig, release); + os_atomic_store(&mutex->sig, sig, release); #endif return 0; @@ -1597,36 +1835,43 @@ _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr, PTHREAD_NOEXPORT_VARIANT int -pthread_mutex_destroy(pthread_mutex_t *omutex) +pthread_mutex_destroy(pthread_mutex_t *mutex) { - _pthread_mutex *mutex = (_pthread_mutex *)omutex; - int res = EINVAL; - _PTHREAD_LOCK(mutex->lock); + _pthread_lock_lock(&mutex->lock); if (_pthread_mutex_check_signature(mutex)) { - mutex_seq *seqaddr; - MUTEX_GETSEQ_ADDR(mutex, &seqaddr); - - mutex_seq seq; - mutex_seq_load(seqaddr, &seq); - - uint64_t *tidaddr; - MUTEX_GETTID_ADDR(mutex, &tidaddr); + // TODO: PTHREAD_STRICT candidate + res = EBUSY; - if ((os_atomic_load(tidaddr, relaxed) == 0) && - (seq.lgenval & PTHRW_COUNT_MASK) == - (seq.ugenval & PTHRW_COUNT_MASK)) { - mutex->sig = _PTHREAD_NO_SIG; + if (_pthread_mutex_uses_ulock(mutex) && + mutex->ulock.uval == _PTHREAD_MUTEX_ULOCK_UNLOCKED_VALUE) { res = 0; } else { - res = EBUSY; + mutex_seq *seqaddr; + MUTEX_GETSEQ_ADDR(mutex, &seqaddr); + + mutex_seq seq; + mutex_seq_load(seqaddr, &seq); + + uint64_t *tidaddr; + MUTEX_GETTID_ADDR(mutex, &tidaddr); + + if ((os_atomic_load_wide(tidaddr, relaxed) == 0) && + (seq.lgenval & PTHRW_COUNT_MASK) == + (seq.ugenval & PTHRW_COUNT_MASK)) { + res = 0; + } } } else if (_pthread_mutex_check_signature_init(mutex)) { - mutex->sig = _PTHREAD_NO_SIG; res = 0; } - _PTHREAD_UNLOCK(mutex->lock); + + if (res == 0) { + mutex->sig = _PTHREAD_NO_SIG; + } + + _pthread_lock_unlock(&mutex->lock); return res; } @@ -1639,14 +1884,9 @@ pthread_mutex_destroy(pthread_mutex_t *omutex) int pthread_mutexattr_destroy(pthread_mutexattr_t *attr) { -#if __DARWIN_UNIX03 - if (__unix_conforming == 0) { - __unix_conforming = 1; - } if (attr->sig != _PTHREAD_MUTEX_ATTR_SIG) { return EINVAL; } -#endif /* __DARWIN_UNIX03 */ attr->sig = _PTHREAD_NO_SIG; return 0; diff --git a/src/pthread_rwlock.c b/src/pthread_rwlock.c index dac212a..55834c9 100644 --- a/src/pthread_rwlock.c +++ b/src/pthread_rwlock.c @@ -57,9 +57,6 @@ #include "resolver.h" #include "internal.h" -#if DEBUG -#include // for bzero -#endif #ifdef PLOCKSTAT #include "plockstat.h" @@ -82,17 +79,6 @@ // maximum number of times a read lock may be obtained #define MAX_READ_LOCKS (INT_MAX - 1) -union rwlock_seq; // forward declaration -enum rwlock_seqfields; // forward declaration - -PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers -int _pthread_rwlock_lock_slow(pthread_rwlock_t *orwlock, bool readlock, - bool trylock); - -PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers -int _pthread_rwlock_unlock_slow(pthread_rwlock_t *orwlock, - enum rwlock_seqfields updated_seqfields); - #if defined(__LP64__) #define RWLOCK_USE_INT128 1 @@ -133,7 +119,7 @@ typedef enum rwlock_seqfields { if (_pthread_debuglog >= 0) { \ _simple_dprintf(_pthread_debuglog, "rw_" #op " %p tck %7llu thr %llx " \ "L %x -> %x S %x -> %x U %x -> %x updt %x\n", rwlock, \ - mach_absolute_time() - _pthread_debugstart, _pthread_selfid_direct(), \ + mach_absolute_time() - _pthread_debugstart, _pthread_threadid_self_np_direct(), \ (f) & RWLOCK_SEQ_LS ? (oldseq).lcntval : 0, \ (f) & RWLOCK_SEQ_LS ? (newseq).lcntval : 0, \ (f) & RWLOCK_SEQ_LS ? (oldseq).rw_seq : 0, \ @@ -148,23 +134,23 @@ typedef enum rwlock_seqfields { #error RWLOCK_GETSEQ_ADDR assumes little endian layout of sequence words #endif -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void -RWLOCK_GETSEQ_ADDR(_pthread_rwlock *rwlock, rwlock_seq **seqaddr) +RWLOCK_GETSEQ_ADDR(pthread_rwlock_t *rwlock, rwlock_seq **seqaddr) { // 128-bit aligned address inside rw_seq & rw_mis arrays *seqaddr = (void*)(((uintptr_t)rwlock->rw_seq + 0xful) & ~0xful); } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void -RWLOCK_GETTID_ADDR(_pthread_rwlock *rwlock, uint64_t **tidaddr) +RWLOCK_GETTID_ADDR(pthread_rwlock_t *rwlock, uint64_t **tidaddr) { // 64-bit aligned address inside rw_tid array (&rw_tid[0] for aligned lock) *tidaddr = (void*)(((uintptr_t)rwlock->rw_tid + 0x7ul) & ~0x7ul); } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void rwlock_seq_load(rwlock_seq *seqaddr, rwlock_seq *oldseqval, const rwlock_seqfields seqfields) @@ -191,7 +177,7 @@ rwlock_seq_load(rwlock_seq *seqaddr, rwlock_seq *oldseqval, } } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void rwlock_seq_atomic_load_relaxed(rwlock_seq *seqaddr, rwlock_seq *oldseqval, const rwlock_seqfields seqfields) @@ -203,19 +189,19 @@ rwlock_seq_atomic_load_relaxed(rwlock_seq *seqaddr, rwlock_seq *oldseqval, // Workaround clang armv81 codegen bug for 128bit os_atomic_load // rdar://problem/31213932 oldseqval->seq_LSU = seqaddr->seq_LSU; - while (!os_atomic_cmpxchgvw(&seqaddr->atomic_seq_LSU, + while (!os_atomic_cmpxchgv(&seqaddr->atomic_seq_LSU, oldseqval->seq_LSU, oldseqval->seq_LSU, &oldseqval->seq_LSU, relaxed)); #else - oldseqval->seq_LSU = os_atomic_load(&seqaddr->atomic_seq_LSU, relaxed); + oldseqval->seq_LSU = os_atomic_load_wide(&seqaddr->atomic_seq_LSU, relaxed); #endif #else - oldseqval->seq_LS = os_atomic_load(&seqaddr->atomic_seq_LS, relaxed); + oldseqval->seq_LS = os_atomic_load_wide(&seqaddr->atomic_seq_LS, relaxed); oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed); #endif break; case RWLOCK_SEQ_LS: - oldseqval->seq_LS = os_atomic_load(&seqaddr->atomic_seq_LS, relaxed); + oldseqval->seq_LS = os_atomic_load_wide(&seqaddr->atomic_seq_LS, relaxed); break; #if DEBUG // unused case RWLOCK_SEQ_U: @@ -230,7 +216,7 @@ rwlock_seq_atomic_load_relaxed(rwlock_seq *seqaddr, rwlock_seq *oldseqval, #define rwlock_seq_atomic_load(seqaddr, oldseqval, seqfields, m) \ rwlock_seq_atomic_load_##m(seqaddr, oldseqval, seqfields) -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline rwlock_seqfields rwlock_seq_atomic_cmpxchgv_relaxed(rwlock_seq *seqaddr, rwlock_seq *oldseqval, rwlock_seq *newseqval, const rwlock_seqfields seqfields) @@ -274,7 +260,7 @@ rwlock_seq_atomic_cmpxchgv_relaxed(rwlock_seq *seqaddr, rwlock_seq *oldseqval, return updated_seqfields; } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline rwlock_seqfields rwlock_seq_atomic_cmpxchgv_acquire(rwlock_seq *seqaddr, rwlock_seq *oldseqval, rwlock_seq *newseqval, const rwlock_seqfields seqfields) @@ -318,7 +304,7 @@ rwlock_seq_atomic_cmpxchgv_acquire(rwlock_seq *seqaddr, rwlock_seq *oldseqval, return updated_seqfields; } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline rwlock_seqfields rwlock_seq_atomic_cmpxchgv_release(rwlock_seq *seqaddr, rwlock_seq *oldseqval, rwlock_seq *newseqval, const rwlock_seqfields seqfields) @@ -340,7 +326,8 @@ rwlock_seq_atomic_cmpxchgv_release(rwlock_seq *seqaddr, rwlock_seq *oldseqval, if (!r) oldseqval->seq_U = newseqval->seq_U; updated_seqfields = r ? RWLOCK_SEQ_LSU : RWLOCK_SEQ_U; } else { - oldseqval->seq_LS = os_atomic_load(&seqaddr->atomic_seq_LS,relaxed); + oldseqval->seq_LS = os_atomic_load_wide(&seqaddr->atomic_seq_LS, + relaxed); } #endif break; @@ -399,12 +386,8 @@ pthread_rwlockattr_setpshared(pthread_rwlockattr_t * attr, int pshared) { int res = EINVAL; if (attr->sig == _PTHREAD_RWLOCK_ATTR_SIG) { -#if __DARWIN_UNIX03 if (( pshared == PTHREAD_PROCESS_PRIVATE) || (pshared == PTHREAD_PROCESS_SHARED)) -#else /* __DARWIN_UNIX03 */ - if ( pshared == PTHREAD_PROCESS_PRIVATE) -#endif /* __DARWIN_UNIX03 */ { attr->pshared = pshared ; res = 0; @@ -415,9 +398,9 @@ pthread_rwlockattr_setpshared(pthread_rwlockattr_t * attr, int pshared) #endif /* !BUILDING_VARIANT ] */ -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int -_pthread_rwlock_init(_pthread_rwlock *rwlock, const pthread_rwlockattr_t *attr) +_pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr) { uint64_t *tidaddr; RWLOCK_GETTID_ADDR(rwlock, &tidaddr); @@ -476,7 +459,7 @@ _pthread_rwlock_init(_pthread_rwlock *rwlock, const pthread_rwlockattr_t *attr) *(sig32_ptr + 1) = *(sig32_val + 1); os_atomic_store(sig32_ptr, *sig32_val, release); #else - os_atomic_store2o(rwlock, sig, sig, release); + os_atomic_store(&rwlock->sig, sig, release); #endif return 0; @@ -506,9 +489,9 @@ _pthread_rwlock_modbits(uint32_t lgenval, uint32_t updateval, uint32_t savebits) return(rval); } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline void -_pthread_rwlock_updateval(_pthread_rwlock *rwlock, uint32_t updateval) +_pthread_rwlock_updateval(pthread_rwlock_t *rwlock, uint32_t updateval) { bool isoverlap = (updateval & PTH_RWL_MBIT) != 0; @@ -536,10 +519,9 @@ _pthread_rwlock_updateval(_pthread_rwlock *rwlock, uint32_t updateval) RWLOCK_DEBUG_SEQ(update, rwlock, oldseq, newseq, updateval, RWLOCK_SEQ_LS); } -#if __DARWIN_UNIX03 -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int -_pthread_rwlock_check_busy(_pthread_rwlock *rwlock) +_pthread_rwlock_check_busy(pthread_rwlock_t *rwlock) { int res = 0; @@ -554,38 +536,32 @@ _pthread_rwlock_check_busy(_pthread_rwlock *rwlock) return res; } -#endif /* __DARWIN_UNIX03 */ PTHREAD_NOEXPORT_VARIANT int -pthread_rwlock_destroy(pthread_rwlock_t *orwlock) +pthread_rwlock_destroy(pthread_rwlock_t *rwlock) { int res = 0; - _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock; - _PTHREAD_LOCK(rwlock->lock); + _pthread_lock_lock(&rwlock->lock); if (_pthread_rwlock_check_signature(rwlock)) { -#if __DARWIN_UNIX03 res = _pthread_rwlock_check_busy(rwlock); -#endif /* __DARWIN_UNIX03 */ } else if (!_pthread_rwlock_check_signature_init(rwlock)) { res = EINVAL; } if (res == 0) { rwlock->sig = _PTHREAD_NO_SIG; } - _PTHREAD_UNLOCK(rwlock->lock); + _pthread_lock_unlock(&rwlock->lock); return res; } PTHREAD_NOEXPORT_VARIANT int -pthread_rwlock_init(pthread_rwlock_t *orwlock, const pthread_rwlockattr_t *attr) +pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr) { int res = 0; - _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock; -#if __DARWIN_UNIX03 if (attr && attr->sig != _PTHREAD_RWLOCK_ATTR_SIG) { res = EINVAL; } @@ -593,58 +569,54 @@ pthread_rwlock_init(pthread_rwlock_t *orwlock, const pthread_rwlockattr_t *attr) if (res == 0 && _pthread_rwlock_check_signature(rwlock)) { res = _pthread_rwlock_check_busy(rwlock); } -#endif if (res == 0) { - _PTHREAD_LOCK_INIT(rwlock->lock); + _pthread_lock_init(&rwlock->lock); res = _pthread_rwlock_init(rwlock, attr); } return res; } -PTHREAD_NOINLINE +OS_NOINLINE static int -_pthread_rwlock_check_init_slow(pthread_rwlock_t *orwlock) +_pthread_rwlock_check_init_slow(pthread_rwlock_t *rwlock) { int res = EINVAL; - _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock; if (_pthread_rwlock_check_signature_init(rwlock)) { - _PTHREAD_LOCK(rwlock->lock); + _pthread_lock_lock(&rwlock->lock); if (_pthread_rwlock_check_signature_init(rwlock)) { res = _pthread_rwlock_init(rwlock, NULL); } else if (_pthread_rwlock_check_signature(rwlock)){ res = 0; } - _PTHREAD_UNLOCK(rwlock->lock); + _pthread_lock_unlock(&rwlock->lock); } else if (_pthread_rwlock_check_signature(rwlock)){ res = 0; } if (res != 0) { - PLOCKSTAT_RW_ERROR(orwlock, READ_LOCK_PLOCKSTAT, res); + PLOCKSTAT_RW_ERROR(rwlock, READ_LOCK_PLOCKSTAT, res); } return res; } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int -_pthread_rwlock_check_init(pthread_rwlock_t *orwlock) +_pthread_rwlock_check_init(pthread_rwlock_t *rwlock) { int res = 0; - _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock; if (!_pthread_rwlock_check_signature(rwlock)) { - return _pthread_rwlock_check_init_slow(orwlock); + return _pthread_rwlock_check_init_slow(rwlock); } return res; } -PTHREAD_NOINLINE +OS_NOINLINE static int -_pthread_rwlock_lock_wait(pthread_rwlock_t *orwlock, bool readlock, +_pthread_rwlock_lock_wait(pthread_rwlock_t *rwlock, bool readlock, rwlock_seq newseq) { int res; - _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock; #ifdef PLOCKSTAT int plockstat = readlock ? READ_LOCK_PLOCKSTAT : WRITE_LOCK_PLOCKSTAT; @@ -660,14 +632,14 @@ _pthread_rwlock_lock_wait(pthread_rwlock_t *orwlock, bool readlock, uint32_t updateval; - PLOCKSTAT_RW_BLOCK(orwlock, plockstat); + PLOCKSTAT_RW_BLOCK(rwlock, plockstat); do { if (readlock) { - updateval = __psynch_rw_rdlock(orwlock, newseq.lcntval, + updateval = __psynch_rw_rdlock(rwlock, newseq.lcntval, newseq.ucntval, newseq.rw_seq, rwlock->rw_flags); } else { - updateval = __psynch_rw_wrlock(orwlock, newseq.lcntval, + updateval = __psynch_rw_wrlock(rwlock, newseq.lcntval, newseq.ucntval, newseq.rw_seq, rwlock->rw_flags); } if (updateval == (uint32_t)-1) { @@ -679,28 +651,27 @@ _pthread_rwlock_lock_wait(pthread_rwlock_t *orwlock, bool readlock, if (res == 0) { _pthread_rwlock_updateval(rwlock, updateval); - PLOCKSTAT_RW_BLOCKED(orwlock, plockstat, BLOCK_SUCCESS_PLOCKSTAT); + PLOCKSTAT_RW_BLOCKED(rwlock, plockstat, BLOCK_SUCCESS_PLOCKSTAT); } else { - PLOCKSTAT_RW_BLOCKED(orwlock, plockstat, BLOCK_FAIL_PLOCKSTAT); + PLOCKSTAT_RW_BLOCKED(rwlock, plockstat, BLOCK_FAIL_PLOCKSTAT); PTHREAD_INTERNAL_CRASH(res, "kernel rwlock returned unknown error"); } return res; } -PTHREAD_NOEXPORT PTHREAD_NOINLINE +OS_NOINLINE int -_pthread_rwlock_lock_slow(pthread_rwlock_t *orwlock, bool readlock, +_pthread_rwlock_lock_slow(pthread_rwlock_t *rwlock, bool readlock, bool trylock) { int res; - _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock; #ifdef PLOCKSTAT int plockstat = readlock ? READ_LOCK_PLOCKSTAT : WRITE_LOCK_PLOCKSTAT; #endif - res = _pthread_rwlock_check_init(orwlock); + res = _pthread_rwlock_check_init(rwlock); if (res != 0) return res; rwlock_seq *seqaddr; @@ -709,14 +680,12 @@ _pthread_rwlock_lock_slow(pthread_rwlock_t *orwlock, bool readlock, rwlock_seq oldseq, newseq; rwlock_seq_atomic_load(seqaddr, &oldseq, RWLOCK_SEQ_LSU, relaxed); -#if __DARWIN_UNIX03 uint64_t *tidaddr; RWLOCK_GETTID_ADDR(rwlock, &tidaddr); - uint64_t selfid = _pthread_selfid_direct(); + uint64_t selfid = _pthread_threadid_self_np_direct(); if (is_rwl_ebit_set(oldseq.lcntval)) { - if (os_atomic_load(tidaddr, relaxed) == selfid) return EDEADLK; + if (os_atomic_load_wide(tidaddr, relaxed) == selfid) return EDEADLK; } -#endif /* __DARWIN_UNIX03 */ int retry_count; bool gotlock; @@ -779,41 +748,38 @@ retry: RWLOCK_SEQ_LS, acquire)); if (gotlock) { -#if __DARWIN_UNIX03 - if (!readlock) os_atomic_store(tidaddr, selfid, relaxed); -#endif /* __DARWIN_UNIX03 */ + if (!readlock) os_atomic_store_wide(tidaddr, selfid, relaxed); res = 0; } else if (trylock) { res = EBUSY; } else { - res = _pthread_rwlock_lock_wait(orwlock, readlock, newseq); + res = _pthread_rwlock_lock_wait(rwlock, readlock, newseq); } out: #ifdef PLOCKSTAT if (res == 0) { - PLOCKSTAT_RW_ACQUIRE(orwlock, plockstat); + PLOCKSTAT_RW_ACQUIRE(rwlock, plockstat); } else { - PLOCKSTAT_RW_ERROR(orwlock, plockstat, res); + PLOCKSTAT_RW_ERROR(rwlock, plockstat, res); } #endif return res; } -PTHREAD_ALWAYS_INLINE +OS_ALWAYS_INLINE static inline int -_pthread_rwlock_lock(pthread_rwlock_t *orwlock, bool readlock, bool trylock) +_pthread_rwlock_lock(pthread_rwlock_t *rwlock, bool readlock, bool trylock) { - _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock; #if PLOCKSTAT if (PLOCKSTAT_RW_ACQUIRE_ENABLED() || PLOCKSTAT_RW_ERROR_ENABLED()) { - return _pthread_rwlock_lock_slow(orwlock, readlock, trylock); + return _pthread_rwlock_lock_slow(rwlock, readlock, trylock); } #endif if (os_unlikely(!_pthread_rwlock_check_signature(rwlock))) { - return _pthread_rwlock_lock_slow(orwlock, readlock, trylock); + return _pthread_rwlock_lock_slow(rwlock, readlock, trylock); } rwlock_seq *seqaddr; @@ -825,11 +791,9 @@ _pthread_rwlock_lock(pthread_rwlock_t *orwlock, bool readlock, bool trylock) // slowpath below (which has rwlock_seq_atomic_load) rwlock_seq_load(seqaddr, &oldseq, RWLOCK_SEQ_LSU); -#if __DARWIN_UNIX03 if (os_unlikely(is_rwl_ebit_set(oldseq.lcntval))) { - return _pthread_rwlock_lock_slow(orwlock, readlock, trylock); + return _pthread_rwlock_lock_slow(rwlock, readlock, trylock); } -#endif /* __DARWIN_UNIX03 */ bool gotlock; do { @@ -850,7 +814,7 @@ _pthread_rwlock_lock(pthread_rwlock_t *orwlock, bool readlock, bool trylock) if (readlock) { if (os_unlikely(diff_genseq(oldseq.lcntval, oldseq.ucntval) >= PTHRW_MAX_READERS)) { - return _pthread_rwlock_lock_slow(orwlock, readlock,trylock); + return _pthread_rwlock_lock_slow(rwlock, readlock, trylock); } // Need to update L (remove U bit) and S word newseq.lcntval &= ~PTH_RWL_UBIT; @@ -861,20 +825,18 @@ _pthread_rwlock_lock(pthread_rwlock_t *orwlock, bool readlock, bool trylock) newseq.lcntval += PTHRW_INC; newseq.rw_seq += PTHRW_INC; } else { - return _pthread_rwlock_lock_slow(orwlock, readlock, trylock); + return _pthread_rwlock_lock_slow(rwlock, readlock, trylock); } } while (os_unlikely(!rwlock_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, RWLOCK_SEQ_LS, acquire))); if (os_likely(gotlock)) { -#if __DARWIN_UNIX03 if (!readlock) { uint64_t *tidaddr; RWLOCK_GETTID_ADDR(rwlock, &tidaddr); - uint64_t selfid = _pthread_selfid_direct(); - os_atomic_store(tidaddr, selfid, relaxed); + uint64_t selfid = _pthread_threadid_self_np_direct(); + os_atomic_store_wide(tidaddr, selfid, relaxed); } -#endif /* __DARWIN_UNIX03 */ return 0; } else if (trylock) { return EBUSY; @@ -885,48 +847,47 @@ _pthread_rwlock_lock(pthread_rwlock_t *orwlock, bool readlock, bool trylock) PTHREAD_NOEXPORT_VARIANT int -pthread_rwlock_rdlock(pthread_rwlock_t *orwlock) +pthread_rwlock_rdlock(pthread_rwlock_t *rwlock) { // read lock, no try - return _pthread_rwlock_lock(orwlock, true, false); + return _pthread_rwlock_lock(rwlock, true, false); } PTHREAD_NOEXPORT_VARIANT int -pthread_rwlock_tryrdlock(pthread_rwlock_t *orwlock) +pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock) { // read lock, try lock - return _pthread_rwlock_lock(orwlock, true, true); + return _pthread_rwlock_lock(rwlock, true, true); } PTHREAD_NOEXPORT_VARIANT int -pthread_rwlock_wrlock(pthread_rwlock_t *orwlock) +pthread_rwlock_wrlock(pthread_rwlock_t *rwlock) { // write lock, no try - return _pthread_rwlock_lock(orwlock, false, false); + return _pthread_rwlock_lock(rwlock, false, false); } PTHREAD_NOEXPORT_VARIANT int -pthread_rwlock_trywrlock(pthread_rwlock_t *orwlock) +pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock) { // write lock, try lock - return _pthread_rwlock_lock(orwlock, false, true); + return _pthread_rwlock_lock(rwlock, false, true); } -PTHREAD_NOINLINE +OS_NOINLINE static int -_pthread_rwlock_unlock_drop(pthread_rwlock_t *orwlock, rwlock_seq oldseq, +_pthread_rwlock_unlock_drop(pthread_rwlock_t *rwlock, rwlock_seq oldseq, rwlock_seq newseq) { int res; - _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock; RWLOCK_DEBUG_SEQ(unlock, rwlock, oldseq, newseq, !droplock, RWLOCK_SEQ_LSU); uint32_t updateval; do { - updateval = __psynch_rw_unlock(orwlock, oldseq.lcntval, + updateval = __psynch_rw_unlock(rwlock, oldseq.lcntval, newseq.ucntval, newseq.rw_seq, rwlock->rw_flags); if (updateval == (uint32_t)-1) { res = errno; @@ -944,19 +905,18 @@ _pthread_rwlock_unlock_drop(pthread_rwlock_t *orwlock, rwlock_seq oldseq, return res; } -PTHREAD_NOEXPORT PTHREAD_NOINLINE +OS_NOINLINE int -_pthread_rwlock_unlock_slow(pthread_rwlock_t *orwlock, +_pthread_rwlock_unlock_slow(pthread_rwlock_t *rwlock, rwlock_seqfields updated_seqfields) { int res; - _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock; rwlock_seqfields seqfields = RWLOCK_SEQ_LSU; #ifdef PLOCKSTAT int wrlock = 0; #endif - res = _pthread_rwlock_check_init(orwlock); + res = _pthread_rwlock_check_init(rwlock); if (res != 0) return res; rwlock_seq *seqaddr; @@ -974,11 +934,9 @@ _pthread_rwlock_unlock_slow(pthread_rwlock_t *orwlock, #ifdef PLOCKSTAT wrlock = 1; #endif -#if __DARWIN_UNIX03 uint64_t *tidaddr; RWLOCK_GETTID_ADDR(rwlock, &tidaddr); - os_atomic_store(tidaddr, 0, relaxed); -#endif /* __DARWIN_UNIX03 */ + os_atomic_store_wide(tidaddr, 0, relaxed); } bool droplock; @@ -1022,30 +980,29 @@ _pthread_rwlock_unlock_slow(pthread_rwlock_t *orwlock, seqaddr, &oldseq, &newseq, seqfields, release))); if (droplock) { - res = _pthread_rwlock_unlock_drop(orwlock, oldseq, newseq); + res = _pthread_rwlock_unlock_drop(rwlock, oldseq, newseq); } - PLOCKSTAT_RW_RELEASE(orwlock, wrlock); + PLOCKSTAT_RW_RELEASE(rwlock, wrlock); return res; } PTHREAD_NOEXPORT_VARIANT int -pthread_rwlock_unlock(pthread_rwlock_t *orwlock) +pthread_rwlock_unlock(pthread_rwlock_t *rwlock) { - _pthread_rwlock *rwlock = (_pthread_rwlock *)orwlock; rwlock_seqfields seqfields = RWLOCK_SEQ_LSU; rwlock_seqfields updated_seqfields = RWLOCK_SEQ_NONE; #if PLOCKSTAT if (PLOCKSTAT_RW_RELEASE_ENABLED() || PLOCKSTAT_RW_ERROR_ENABLED()) { - return _pthread_rwlock_unlock_slow(orwlock, updated_seqfields); + return _pthread_rwlock_unlock_slow(rwlock, updated_seqfields); } #endif if (os_unlikely(!_pthread_rwlock_check_signature(rwlock))) { - return _pthread_rwlock_unlock_slow(orwlock, updated_seqfields); + return _pthread_rwlock_unlock_slow(rwlock, updated_seqfields); } rwlock_seq *seqaddr; @@ -1060,16 +1017,14 @@ pthread_rwlock_unlock(pthread_rwlock_t *orwlock) } if (is_rwl_ebit_set(oldseq.lcntval)) { -#if __DARWIN_UNIX03 uint64_t *tidaddr; RWLOCK_GETTID_ADDR(rwlock, &tidaddr); - os_atomic_store(tidaddr, 0, relaxed); -#endif /* __DARWIN_UNIX03 */ + os_atomic_store_wide(tidaddr, 0, relaxed); } do { if (updated_seqfields) { - return _pthread_rwlock_unlock_slow(orwlock, updated_seqfields); + return _pthread_rwlock_unlock_slow(rwlock, updated_seqfields); } newseq = oldseq; @@ -1089,7 +1044,7 @@ pthread_rwlock_unlock(pthread_rwlock_t *orwlock) // no L/S update if lock is not exclusive or no writer pending // kernel transition only needed if U == S } else { - return _pthread_rwlock_unlock_slow(orwlock, updated_seqfields); + return _pthread_rwlock_unlock_slow(rwlock, updated_seqfields); } } } while (os_unlikely(seqfields != (updated_seqfields = diff --git a/src/pthread_tsd.c b/src/pthread_tsd.c index c76bba6..daf7a05 100644 --- a/src/pthread_tsd.c +++ b/src/pthread_tsd.c @@ -52,7 +52,6 @@ */ #include "internal.h" -#include #ifndef PTHREAD_KEY_LEGACY_SUPPORT #if TARGET_OS_DRIVERKIT @@ -151,7 +150,7 @@ pthread_key_create(pthread_key_t *key, void (*destructor)(void *)) int res = EAGAIN; // Returns EAGAIN if key cannot be allocated. pthread_key_t k; - _PTHREAD_LOCK(__pthread_tsd_lock); + _pthread_lock_lock(&__pthread_tsd_lock); for (k = __pthread_tsd_start; k < __pthread_tsd_end; k++) { if (_pthread_key_set_destructor(k, destructor)) { *key = k; @@ -159,7 +158,7 @@ pthread_key_create(pthread_key_t *key, void (*destructor)(void *)) break; } } - _PTHREAD_UNLOCK(__pthread_tsd_lock); + _pthread_lock_unlock(&__pthread_tsd_lock); return res; } @@ -169,51 +168,58 @@ pthread_key_delete(pthread_key_t key) { int res = EINVAL; // Returns EINVAL if key is not allocated. - _PTHREAD_LOCK(__pthread_tsd_lock); + _pthread_lock_lock(&__pthread_tsd_lock); if (key >= __pthread_tsd_start && key < __pthread_tsd_end) { if (_pthread_key_unset_destructor(key)) { - struct _pthread *p; - _PTHREAD_LOCK(_pthread_list_lock); + pthread_t p; + _pthread_lock_lock(&_pthread_list_lock); TAILQ_FOREACH(p, &__pthread_head, tl_plist) { // No lock for word-sized write. p->tsd[key] = 0; } - _PTHREAD_UNLOCK(_pthread_list_lock); + _pthread_lock_unlock(&_pthread_list_lock); res = 0; } } - _PTHREAD_UNLOCK(__pthread_tsd_lock); + _pthread_lock_unlock(&__pthread_tsd_lock); return res; } -#endif // !VARIANT_DYLD -int -pthread_setspecific(pthread_key_t key, const void *value) +static inline int +_pthread_setspecific(pthread_t thread, pthread_key_t key, const void *value) { int res = EINVAL; -#if !VARIANT_DYLD if (key >= __pthread_tsd_first && key < __pthread_tsd_end) { bool created = _pthread_key_get_destructor(key, NULL); if (key < __pthread_tsd_start || created) { - struct _pthread *self = pthread_self(); - self->tsd[key] = (void *)value; + thread->tsd[key] = (void *)value; res = 0; if (key < __pthread_tsd_start) { // XXX: is this really necessary? _pthread_key_set_destructor(key, NULL); } - if (key > self->max_tsd_key) { - self->max_tsd_key = (uint16_t)key; + if (key > thread->max_tsd_key) { + thread->max_tsd_key = (uint16_t)key; } } } -#endif // !VARIANT_DYLD return res; } +#endif // !VARIANT_DYLD + +int +pthread_setspecific(pthread_key_t key, const void *value) +{ +#if VARIANT_DYLD + return ENOTSUP; +#else + return _pthread_setspecific(pthread_self(), key, value); +#endif // !VARIANT_DYLD +} int _pthread_setspecific_static(pthread_key_t key, void *value) @@ -237,6 +243,30 @@ pthread_getspecific(pthread_key_t key) } #if !VARIANT_DYLD +int +pthread_introspection_setspecific_np(pthread_t thread, + pthread_key_t key, const void *value) +{ + pthread_t self = _pthread_self(); + if (os_unlikely(self->introspection != PTHREAD_INTROSPECTION_THREAD_CREATE)) { + PTHREAD_CLIENT_CRASH(0, "Calling pthread_introspection_setspecific_np " + "outside of a CREATE introspection hook"); + } + return _pthread_setspecific(thread, key, value); + +} + +void * +pthread_introspection_getspecific_np(pthread_t thread, pthread_key_t key) +{ + pthread_t self = _pthread_self(); + if (os_unlikely(self->introspection != PTHREAD_INTROSPECTION_THREAD_DESTROY)) { + PTHREAD_CLIENT_CRASH(0, "Calling pthread_introspection_getspecific_np " + "outside of a DESTROY introspection hook"); + } + return thread->tsd[key]; +} + static void _pthread_tsd_cleanup_key(pthread_t self, pthread_key_t key) { @@ -252,9 +282,7 @@ _pthread_tsd_cleanup_key(pthread_t self, pthread_key_t key) } } } -#endif // !VARIANT_DYLD -#if !VARIANT_DYLD static void _pthread_tsd_cleanup_new(pthread_t self) { @@ -284,7 +312,6 @@ _pthread_tsd_behaviour_check(pthread_t self) // Iterate from dynamic-key start to dynamic-key end, if the key has both // a desctructor and a value then _pthread_tsd_cleanup_key would cause // us to re-trigger the destructor. - Dl_info i; pthread_key_t k; for (k = __pthread_tsd_start; k < __pthread_tsd_end; k++) { @@ -295,11 +322,14 @@ _pthread_tsd_behaviour_check(pthread_t self) if (value && destructor) { _simple_asl_log(ASL_LEVEL_ERR, "pthread_tsd", "warning: dynamic tsd keys dirty after static key cleanup loop."); - +#if 0 + // enable this for debugging + Dl_info i; if (dladdr(destructor, &i) == 0) { _simple_asl_log(ASL_LEVEL_ERR, "pthread_tsd", i.dli_fname); _simple_asl_log(ASL_LEVEL_ERR, "pthread_tsd", i.dli_saddr); } +#endif } } } @@ -361,12 +391,12 @@ pthread_key_init_np(int key, void (*destructor)(void *)) { int res = EINVAL; // Returns EINVAL if key is out of range. if (key >= __pthread_tsd_first && key < __pthread_tsd_start) { - _PTHREAD_LOCK(__pthread_tsd_lock); + _pthread_lock_lock(&__pthread_tsd_lock); _pthread_key_set_destructor(key, destructor); if (key > __pthread_tsd_max) { __pthread_tsd_max = key; } - _PTHREAD_UNLOCK(__pthread_tsd_lock); + _pthread_lock_unlock(&__pthread_tsd_lock); res = 0; } return res; @@ -381,3 +411,10 @@ pthread_self(void) _pthread_validate_signature(self); return self; } + +// rdar://57406917 +pthread_t +_pthread_self(void) +{ + return pthread_self(); +} diff --git a/src/qos.c b/src/qos.c index dea72f8..4985d9e 100644 --- a/src/qos.c +++ b/src/qos.c @@ -23,21 +23,39 @@ #include "internal.h" -#include <_simple.h> #include #include #include #include +#include #include #include -// TODO: remove me when internal.h can include *_private.h itself -#include "workqueue_private.h" -#include "qos_private.h" - #define PTHREAD_OVERRIDE_SIGNATURE (0x6f766572) #define PTHREAD_OVERRIDE_SIG_DEAD (0x7265766f) +#if !defined(VARIANT_STATIC) +// internally redirected upcalls in case qos overrides are used +// before __pthread_init has run +PTHREAD_NOEXPORT void * +malloc(size_t sz) +{ + if (os_likely(_pthread_malloc)) { + return _pthread_malloc(sz); + } else { + return NULL; + } +} + +PTHREAD_NOEXPORT void +free(void *p) +{ + if (os_likely(_pthread_free)) { + _pthread_free(p); + } +} +#endif // VARIANT_STATIC + struct pthread_override_s { uint32_t sig; @@ -153,7 +171,7 @@ pthread_set_qos_class_np(pthread_t thread, qos_class_t qc, int relpri) int pthread_get_qos_class_np(pthread_t thread, qos_class_t *qc, int *relpri) { - pthread_priority_t pp = thread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS]; + pthread_priority_t pp = _pthread_tsd_slot(thread, PTHREAD_QOS_CLASS); _pthread_priority_split(pp, qc, relpri); return 0; } @@ -553,14 +571,14 @@ int posix_spawnattr_set_qos_class_np(posix_spawnattr_t * __restrict __attr, qos_class_t __qos_class) { switch (__qos_class) { - case QOS_CLASS_UTILITY: - return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_UTILITY); - case QOS_CLASS_BACKGROUND: - return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_BACKGROUND); - case QOS_CLASS_MAINTENANCE: - return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE); - default: - return EINVAL; + case QOS_CLASS_UTILITY: + return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_UTILITY); + case QOS_CLASS_BACKGROUND: + return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_BACKGROUND); + case QOS_CLASS_MAINTENANCE: + return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE); + default: + return EINVAL; } } @@ -579,18 +597,18 @@ posix_spawnattr_get_qos_class_np(const posix_spawnattr_t *__restrict __attr, qos } switch (clamp) { - case POSIX_SPAWN_PROC_CLAMP_UTILITY: - *__qos_class = QOS_CLASS_UTILITY; - break; - case POSIX_SPAWN_PROC_CLAMP_BACKGROUND: - *__qos_class = QOS_CLASS_BACKGROUND; - break; - case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE: - *__qos_class = QOS_CLASS_MAINTENANCE; - break; - default: - *__qos_class = QOS_CLASS_UNSPECIFIED; - break; + case POSIX_SPAWN_PROC_CLAMP_UTILITY: + *__qos_class = QOS_CLASS_UTILITY; + break; + case POSIX_SPAWN_PROC_CLAMP_BACKGROUND: + *__qos_class = QOS_CLASS_BACKGROUND; + break; + case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE: + *__qos_class = QOS_CLASS_MAINTENANCE; + break; + default: + *__qos_class = QOS_CLASS_UNSPECIFIED; + break; } return 0; diff --git a/src/types_internal.h b/src/types_internal.h new file mode 100644 index 0000000..858d94f --- /dev/null +++ b/src/types_internal.h @@ -0,0 +1,507 @@ +/* + * Copyright (c) 2003-2019 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef __LIBPTHREAD_TYPES_INTERNAL_H__ +#define __LIBPTHREAD_TYPES_INTERNAL_H__ + +/*! + * @file types_internal.h + * + * @brief + * This file exposes all the internal pthread types used by the library. + * + * @discussion + * This header must be included first, as it masks the opaque definitions + * exposed to libpthread clients in the SDK. + */ + +#define _PTHREAD_ONCE_T +typedef struct pthread_once_s pthread_once_t; + +#define _PTHREAD_MUTEX_T +#define _PTHREAD_MUTEXATTR_T +typedef struct pthread_mutex_s pthread_mutex_t; +typedef struct pthread_mutexattr_s pthread_mutexattr_t; + +#define _PTHREAD_COND_T +#define _PTHREAD_CONDATTR_T +typedef struct pthread_cond_s pthread_cond_t; +typedef struct pthread_condattr_s pthread_condattr_t; + +#define _PTHREAD_RWLOCK_T +#define _PTHREAD_RWLOCKATTR_T +typedef struct pthread_rwlock_s pthread_rwlock_t; +typedef struct pthread_rwlockattr_s pthread_rwlockattr_t; + +#define _PTHREAD_T +#define _PTHREAD_ATTR_T +typedef struct pthread_s *pthread_t; +typedef struct pthread_attr_s pthread_attr_t; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "pthread/posix_sched.h" +#include "pthread/workqueue_private.h" +#include "sys/_pthread/_pthread_types.h" + +#pragma mark - constants + +#define _PTHREAD_NO_SIG 0x00000000 +#define _PTHREAD_MUTEX_ATTR_SIG 0x4D545841 /* 'MTXA' */ +#define _PTHREAD_MUTEX_SIG 0x4D555458 /* 'MUTX' */ +#define _PTHREAD_MUTEX_SIG_fast 0x4D55545A /* 'MUTZ' */ +#define _PTHREAD_MUTEX_SIG_MASK 0xfffffffd +#define _PTHREAD_MUTEX_SIG_CMP 0x4D555458 /* _PTHREAD_MUTEX_SIG & _PTHREAD_MUTEX_SIG_MASK */ +#define _PTHREAD_MUTEX_SIG_init 0x32AAABA7 /* [almost] ~'MUTX' */ +#define _PTHREAD_ERRORCHECK_MUTEX_SIG_init 0x32AAABA1 +#define _PTHREAD_RECURSIVE_MUTEX_SIG_init 0x32AAABA2 +#define _PTHREAD_FIRSTFIT_MUTEX_SIG_init 0x32AAABA3 +#define _PTHREAD_MUTEX_SIG_init_MASK 0xfffffff0 +#define _PTHREAD_MUTEX_SIG_init_CMP 0x32AAABA0 +#define _PTHREAD_COND_ATTR_SIG 0x434E4441 /* 'CNDA' */ +#define _PTHREAD_COND_SIG_init 0x3CB0B1BB /* [almost] ~'COND' */ +#define _PTHREAD_COND_SIG_pristine 0x434F4E44 /* 'COND' */ +#define _PTHREAD_COND_SIG_psynch 0x434F4E45 /* 'COND' + 0b01: 'CONE' */ +#define _PTHREAD_COND_SIG_ulock 0x434F4E46 /* 'COND' + 0b10: 'CONF' */ +#define _PTHREAD_ATTR_SIG 0x54484441 /* 'THDA' */ +#define _PTHREAD_ONCE_SIG 0x4F4E4345 /* 'ONCE' */ +#define _PTHREAD_ONCE_SIG_init 0x30B1BCBA /* [almost] ~'ONCE' */ +#define _PTHREAD_SIG 0x54485244 /* 'THRD' */ +#define _PTHREAD_RWLOCK_ATTR_SIG 0x52574C41 /* 'RWLA' */ +#define _PTHREAD_RWLOCK_SIG 0x52574C4B /* 'RWLK' */ +#define _PTHREAD_RWLOCK_SIG_init 0x2DA8B3B4 /* [almost] ~'RWLK' */ + +__enum_closed_decl(pthread_conformance_t, unsigned, { + PTHREAD_CONFORM_UNIX03_NOCANCEL = 1, + PTHREAD_CONFORM_UNIX03_CANCELABLE = 2, +}); + +/* Pull the pthread_t into the same page as the top of the stack so we dirty one less page. + * The pthread_s struct at the top of the stack shouldn't be page-aligned + */ +#if defined(__arm64__) +#define PTHREAD_T_OFFSET (12*1024) +#else +#define PTHREAD_T_OFFSET 0 +#endif + +#if TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR +#define _EXTERNAL_POSIX_THREAD_KEYS_MAX 256 +#define _INTERNAL_POSIX_THREAD_KEYS_MAX 256 +#define _INTERNAL_POSIX_THREAD_KEYS_END 512 +#else +#define _EXTERNAL_POSIX_THREAD_KEYS_MAX 512 +#define _INTERNAL_POSIX_THREAD_KEYS_MAX 256 +#define _INTERNAL_POSIX_THREAD_KEYS_END 768 +#endif + +#define PTHREAD_ATFORK_INLINE_MAX 10 + +#define MAXTHREADNAMESIZE 64 + +#define _PTHREAD_DEFAULT_INHERITSCHED PTHREAD_INHERIT_SCHED +#define _PTHREAD_DEFAULT_PROTOCOL PTHREAD_PRIO_NONE +#define _PTHREAD_DEFAULT_PRIOCEILING 0 +#define _PTHREAD_DEFAULT_POLICY SCHED_OTHER +#define _PTHREAD_DEFAULT_STACKSIZE 0x80000 /* 512K */ +#define _PTHREAD_DEFAULT_PSHARED PTHREAD_PROCESS_PRIVATE + +#define _PTHREAD_CANCEL_STATE_MASK 0x01 +#define _PTHREAD_CANCEL_TYPE_MASK 0x02 +#define _PTHREAD_CANCEL_PENDING 0x10 /* pthread_cancel() has been called for this thread */ +#define _PTHREAD_CANCEL_EXITING 0x20 + +#define pthread_assert_type_size(type) \ + static_assert(sizeof(struct type##_s) == sizeof(struct _opaque_##type##_t), "") +#define pthread_assert_type_alias(type, f1, f2) \ + static_assert(offsetof(struct type##_s, f1) == offsetof(struct _opaque_##type##_t, f2), "") + +typedef os_unfair_lock _pthread_lock; +struct _pthread_registration_data; + + +#pragma mark - pthread_once_t + +struct pthread_once_s { + long sig; + os_once_t once; +}; + +pthread_assert_type_size(pthread_once); +pthread_assert_type_alias(pthread_once, sig, __sig); + +#pragma mark - pthread_mutex_t, pthread_mutexattr_t + +#define _PTHREAD_MUTEX_POLICY_LAST (PTHREAD_MUTEX_POLICY_FIRSTFIT_NP + 1) +#define _PTHREAD_MTX_OPT_POLICY_FAIRSHARE 1 +#define _PTHREAD_MTX_OPT_POLICY_FIRSTFIT 2 +#define _PTHREAD_MTX_OPT_POLICY_DEFAULT _PTHREAD_MTX_OPT_POLICY_FIRSTFIT +// The following pthread_mutex_options_s defintions exist in synch_internal.h +// such that the kernel extension can test for flags. They must be kept in +// sync with the bit values in the struct above. +// _PTHREAD_MTX_OPT_PSHARED 0x010 +// _PTHREAD_MTX_OPT_NOTIFY 0x1000 +// _PTHREAD_MTX_OPT_MUTEX 0x2000 + +#define _PTHREAD_MTX_OPT_ULOCK_DEFAULT false +#define _PTHREAD_MTX_OPT_ADAPTIVE_DEFAULT false + +// The fixed mask is used to mask out portions of the mutex options that +// change on a regular basis (notify, lock_count). +#define _PTHREAD_MTX_OPT_FIXED_MASK 0x27ff + +struct pthread_mutex_options_s { + uint32_t + protocol:2, + type:2, + pshared:2, + policy:3, + hold:2, + misalign:1, + notify:1, + mutex:1, + ulock:1, + unused:1, + lock_count:16; +}; + +#define _PTHREAD_MUTEX_ULOCK_OWNER_MASK 0xfffffffcu +#define _PTHREAD_MUTEX_ULOCK_WAITERS_BIT 0x00000001u +#define _PTHREAD_MUTEX_ULOCK_UNLOCKED_VALUE 0x0u +#define _PTHREAD_MUTEX_ULOCK_UNLOCKED \ + ((struct _pthread_mutex_ulock_s){0}) + +typedef struct _pthread_mutex_ulock_s { + uint32_t uval; +} *_pthread_mutex_ulock_t; + +struct pthread_mutex_s { + long sig; + _pthread_lock lock; + union { + uint32_t value; + struct pthread_mutex_options_s options; + } mtxopts; + int16_t prioceiling; + int16_t priority; +#if defined(__LP64__) + uint32_t _pad; +#endif + union { + struct { + uint32_t m_tid[2]; // thread id of thread that has mutex locked + uint32_t m_seq[2]; // mutex sequence id + uint32_t m_mis[2]; // for misaligned locks m_tid/m_seq will span into here + } psynch; + struct _pthread_mutex_ulock_s ulock; + }; +#if defined(__LP64__) + uint32_t _reserved[4]; +#else + uint32_t _reserved[1]; +#endif +}; + +pthread_assert_type_size(pthread_mutex); +pthread_assert_type_alias(pthread_mutex, sig, __sig); + +struct pthread_mutexattr_s { + long sig; + int prioceiling; + uint32_t + protocol:2, + type:2, + pshared:2, + opt:3, + unused:23; +}; + +pthread_assert_type_size(pthread_mutexattr); +pthread_assert_type_alias(pthread_mutexattr, sig, __sig); + +#pragma mark - pthread_rwlock_t, pthread_rwlockattr_t + +struct pthread_rwlock_s { + long sig; + _pthread_lock lock; + uint32_t + unused:29, + misalign:1, + pshared:2; + uint32_t rw_flags; +#if defined(__LP64__) + uint32_t _pad; +#endif + uint32_t rw_tid[2]; // thread id of thread that has exclusive (write) lock + uint32_t rw_seq[4]; // rw sequence id (at 128-bit aligned boundary) + uint32_t rw_mis[4]; // for misaligned locks rw_seq will span into here +#if defined(__LP64__) + uint32_t _reserved[34]; +#else + uint32_t _reserved[18]; +#endif +}; + +pthread_assert_type_size(pthread_rwlock); +pthread_assert_type_alias(pthread_rwlock, sig, __sig); + +struct pthread_rwlockattr_s { + long sig; + int pshared; +#if defined(__LP64__) + uint32_t _reserved[3]; +#else + uint32_t _reserved[2]; +#endif +}; + +pthread_assert_type_size(pthread_rwlockattr); +pthread_assert_type_alias(pthread_rwlockattr, sig, __sig); + +#pragma mark - pthread_cond_t, pthread_condattr_t + +struct pthread_cond_s { + struct { + uint32_t val; +#if defined(__LP64__) + uint32_t _pad; +#endif + } sig; + _pthread_lock lock; + uint32_t + unused:29, + misalign:1, + pshared:2; + pthread_mutex_t *busy; + uint32_t c_seq[3]; +#if defined(__LP64__) + uint32_t _reserved[3]; +#endif +}; + +pthread_assert_type_size(pthread_cond); +pthread_assert_type_alias(pthread_cond, sig, __sig); + +struct pthread_condattr_s { + long sig; + uint32_t + pshared:2, + unsupported:30; +}; + +pthread_assert_type_size(pthread_condattr); +pthread_assert_type_alias(pthread_condattr, sig, __sig); + +#pragma mark - pthread_t, pthread_attr_t + +typedef struct pthread_join_context_s { + pthread_t waiter; + void **value_ptr; + mach_port_t kport; + semaphore_t custom_stack_sema; + bool detached; +} pthread_join_context_s, *pthread_join_context_t; + +#define MAXTHREADNAMESIZE 64 + +struct pthread_s { + long sig; + struct __darwin_pthread_handler_rec *__cleanup_stack; + + // + // Fields protected by _pthread_list_lock + // + + TAILQ_ENTRY(pthread_s) tl_plist; // global thread list [aligned] + struct pthread_join_context_s *tl_join_ctx; + void *tl_exit_value; + uint8_t tl_policy; + // pthread knows that tl_joinable bit comes immediately after tl_policy + uint8_t + tl_joinable:1, + tl_joiner_cleans_up:1, + tl_has_custom_stack:1, + __tl_pad:5; + uint16_t introspection; + // MACH_PORT_NULL if no joiner + // tsd[_PTHREAD_TSD_SLOT_MACH_THREAD_SELF] when has a joiner + // MACH_PORT_DEAD if the thread exited + uint32_t tl_exit_gate; + struct sched_param tl_param; + void *__unused_padding; + + // + // Fields protected by pthread_t::lock + // + + _pthread_lock lock; + uint16_t max_tsd_key; + uint16_t + inherit:8, + kernalloc:1, + schedset:1, + wqthread:1, + wqkillset:1, + __flags_pad:4; + + char pthread_name[MAXTHREADNAMESIZE]; // includes NUL [aligned] + + void *(*fun)(void *); // thread start routine + void *arg; // thread start routine argument + int wq_nevents; // wqthreads (workloop / kevent) + bool wq_outsideqos; + uint8_t canceled; // 4597450 set if conformant cancelation happened + uint16_t cancel_state; // whether the thread can be canceled [atomic] + errno_t cancel_error; + errno_t err_no; // thread-local errno + + void *stackaddr; // base of the stack (page aligned) + void *stackbottom; // stackaddr - stacksize + void *freeaddr; // stack/thread allocation base address + size_t freesize; // stack/thread allocation size + size_t guardsize; // guard page size in bytes + + // tsd-base relative accessed elements + __attribute__((aligned(8))) + uint64_t thread_id; // 64-bit unique thread id + + /* Thread Specific Data slots + * + * The offset of this field from the start of the structure is difficult to + * change on OS X because of a thorny bitcompat issue: mono has hard coded + * the value into their source. Newer versions of mono will fall back to + * scanning to determine it at runtime, but there's lots of software built + * with older mono that won't. We will have to break them someday... + */ + __attribute__ ((aligned (16))) + void *tsd[_EXTERNAL_POSIX_THREAD_KEYS_MAX + _INTERNAL_POSIX_THREAD_KEYS_MAX]; +}; + +TAILQ_HEAD(__pthread_list, pthread_s); + +#if 0 // pthread_t is never stack-allocated, so it doesn't matter +pthread_assert_type_size(pthread); +#endif +pthread_assert_type_alias(pthread, sig, __sig); +pthread_assert_type_alias(pthread, __cleanup_stack, __cleanup_stack); +#if __LP64__ +static_assert(offsetof(struct pthread_s, tsd) == 224, "TSD LP64 offset"); +#else +static_assert(offsetof(struct pthread_s, tsd) == 176, "TSD ILP32 offset"); +#endif + +#define _PTHREAD_ATTR_REFILLMS_MAX ((2<<24) - 1) + +struct pthread_attr_s { + long sig; + size_t guardsize; // size in bytes of stack overflow guard area + void *stackaddr; // stack base; vm_page_size aligned + size_t stacksize; // stack size; multiple of vm_page_size and >= PTHREAD_STACK_MIN + union { + struct sched_param param; // [aligned] + unsigned long qosclass; // pthread_priority_t + }; + uint32_t + detached:8, + inherit:8, + policy:8, + schedset:1, + qosset:1, + policyset:1, + cpupercentset:1, + defaultguardpage:1, + unused:3; + uint32_t + cpupercent:8, + refillms:24; +#if defined(__LP64__) + uint32_t _reserved[4]; +#else + uint32_t _reserved[2]; +#endif +}; + +pthread_assert_type_size(pthread_attr); +pthread_assert_type_alias(pthread_attr, sig, __sig); + +#pragma mark - atfork / qos + +struct pthread_atfork_entry { + void (*prepare)(void); + void (*parent)(void); + void (*child)(void); +}; + +#define PTHREAD_ATFORK_INLINE_MAX 10 +#if defined(__arm__) +// Hack. We don't want to depend on libcompiler_rt. armv7 implements integer +// division by calling into compiler_rt. vm_page_size isn't a constant and +// pthread_atfork_entry is 12 bytes so the compiler can't strength-reduce the +// division, so it generates a call into compiler_rt. +// So let's just use PAGE_MAX_SIZE on armv7, which is a constant. At worst +// this wastes a maybe dozen K if we are actaully running on a smaller page +// size than the max. +// At the time of this writing we don't have any supported iOS armv7 hardware +// that has different vm_page_size and PAGE_MAX_SIZE. +#define PTHREAD_ATFORK_MAX (PAGE_MAX_SIZE/sizeof(struct pthread_atfork_entry)) +#else // defined(__arm__) +#define PTHREAD_ATFORK_MAX (vm_page_size/sizeof(struct pthread_atfork_entry)) +#endif // defined(__arm__) + +struct pthread_globals_s { + // atfork.c + pthread_t psaved_self; + _pthread_lock psaved_self_global_lock; + _pthread_lock pthread_atfork_lock; + + size_t atfork_count; + struct pthread_atfork_entry atfork_storage[PTHREAD_ATFORK_INLINE_MAX]; + struct pthread_atfork_entry *atfork; + uint16_t qmp_logical[THREAD_QOS_LAST]; + uint16_t qmp_physical[THREAD_QOS_LAST]; + +}; +typedef struct pthread_globals_s *pthread_globals_t; + +#endif // __LIBPTHREAD_TYPES_INTERNAL_H__ diff --git a/src/variants/pthread_cancelable_legacy.c b/src/variants/pthread_cancelable_legacy.c deleted file mode 100644 index 31177a0..0000000 --- a/src/variants/pthread_cancelable_legacy.c +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2012 Apple Inc. All rights reserved. - * - * @APPLE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_LICENSE_HEADER_END@ - */ - -#include -#if defined(__i386__) - -#define BUILDING_VARIANT 1 - -#undef __DARWIN_UNIX03 -#define __DARWIN_UNIX03 0 - -#include "../pthread_cancelable.c" - -#endif diff --git a/src/variants/pthread_cond_legacy.c b/src/variants/pthread_cond_legacy.c deleted file mode 100644 index 75ee354..0000000 --- a/src/variants/pthread_cond_legacy.c +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2012 Apple Inc. All rights reserved. - * - * @APPLE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_LICENSE_HEADER_END@ - */ - -#include -#if defined(__i386__) - -#define BUILDING_VARIANT 1 - -#undef __DARWIN_UNIX03 -#define __DARWIN_UNIX03 0 - -#include "../pthread_cond.c" - -#endif diff --git a/src/variants/pthread_mutex_legacy.c b/src/variants/pthread_mutex_legacy.c deleted file mode 100644 index ce00957..0000000 --- a/src/variants/pthread_mutex_legacy.c +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2012 Apple Inc. All rights reserved. - * - * @APPLE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_LICENSE_HEADER_END@ - */ - -#include -#if defined(__i386__) - -#define BUILDING_VARIANT 1 - -#undef __DARWIN_UNIX03 -#define __DARWIN_UNIX03 0 - -#include "../pthread_mutex.c" - -#endif diff --git a/src/variants/pthread_rwlock_legacy.c b/src/variants/pthread_rwlock_legacy.c deleted file mode 100644 index de0e978..0000000 --- a/src/variants/pthread_rwlock_legacy.c +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2012 Apple Inc. All rights reserved. - * - * @APPLE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_LICENSE_HEADER_END@ - */ - -#include -#if defined(__i386__) - -#define BUILDING_VARIANT 1 - -#undef __DARWIN_UNIX03 -#define __DARWIN_UNIX03 0 - -#define _pthread_rwlock_lock_slow _pthread_rwlock_lock_legacy_slow -#define _pthread_rwlock_unlock_slow _pthread_rwlock_unlock_legacy_slow - -#include "../pthread_rwlock.c" - -#endif diff --git a/tests/Makefile b/tests/Makefile index f41b873..1988e63 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -55,6 +55,8 @@ TARGETS += setrlimit_sigsegv TARGETS += wq_limits TARGETS += add_timer_termination TARGETS += perf_contended_mutex_rwlock +TARGETS += pthread_jit_write_protection +pthread_jit_write_protection: CODE_SIGN_ENTITLEMENTS=pthread_jit_write_protection-entitlements.plist # this should be CUSTOM_TARGETS, see "Compatibility defines" in Makefile.targets OTHER_TARGETS := stackoverflow_crash @@ -65,7 +67,8 @@ OTHER_LTE_INCLUDE_FILES += \ OTHER_CFLAGS := -DDARWINTEST -Weverything \ -Wno-vla -Wno-bad-function-cast -Wno-missing-noreturn \ -Wno-missing-field-initializers -Wno-format-pedantic \ - -Wno-gnu-folding-constant -Wno-used-but-marked-unused + -Wno-gnu-folding-constant -Wno-used-but-marked-unused \ + -Wno-padded OTHER_LDFLAGS := -ldarwintest_utils #TARGETS += main_stack_legacy // Disabled by default due to linker warnings @@ -89,3 +92,4 @@ stackoverflow_crash: helpers/stackoverflow_crash.c install-stackoverflow_crash: stackoverflow_crash mkdir -p $(INSTALLDIR)/assets @cp $(SYMROOT)/assets/stackoverflow_crash $(INSTALLDIR)/assets + diff --git a/tests/cond.c b/tests/cond.c index 0fc91c2..8edf390 100644 --- a/tests/cond.c +++ b/tests/cond.c @@ -123,3 +123,455 @@ T_DECL(cond, "pthread_cond", T_ASSERT_POSIX_ZERO(pthread_join(p[i], NULL), NULL); } } + +#pragma mark invalid concurrent mutex use + +// XXX ulock-based condvars don't detect concurrent waiting for now +#if 0 + +static pthread_cond_t concurrent_cond = PTHREAD_COND_INITIALIZER; + +static void * +invalid_wait_thread(void *arg) +{ + pthread_mutex_t *mutex = arg; + + int rc = pthread_mutex_lock(mutex); + T_ASSERT_POSIX_ZERO(rc, "lock mutex"); + + while (true) { + rc = pthread_cond_wait(&concurrent_cond, mutex); + if (rc == EINVAL) { + T_PASS("Detected EINVAL"); + T_END; + } else { + T_ASSERT_POSIX_ZERO(rc, "cond_wait"); + } + } +} + +T_DECL(cond_invalid_concurrent_mutex, "Detect concurrent waits with different mutexes as invalid") +{ + int rc; + pthread_t threads[2]; + pthread_mutex_t mutexes[2] = { + PTHREAD_MUTEX_INITIALIZER, + PTHREAD_MUTEX_INITIALIZER, + }; + + for (int i = 0; i < 2; i++) { + rc = pthread_create(&threads[i], NULL, invalid_wait_thread, + &mutexes[i]); + T_ASSERT_POSIX_ZERO(rc, "pthread_create"); + } + + // will not return + pthread_join(threads[0], NULL); +} + +#endif + +#pragma mark mutex ping pong test + +struct cond_mutex_ping_pong_ctx_s { + pthread_mutex_t mutex; + int arrived; + int group; +}; + +static struct { + pthread_mutex_t sync_mutex; + pthread_cond_t sync_cond; + int group; + pthread_cond_t shared_cond; +} ping_pong = { + .sync_mutex = PTHREAD_MUTEX_INITIALIZER, + .sync_cond = PTHREAD_COND_INITIALIZER, + .group = 0, + .shared_cond = PTHREAD_COND_INITIALIZER, +}; + +#define PING_PONG_NGROUPS 2 +#define PING_PONG_GROUP_NTHREADS 3 +#define PING_PONG_ITERATIONS 5000 + +static void * +ping_pong_thread(void *arg) +{ + int rc; + struct cond_mutex_ping_pong_ctx_s *ctx = arg; + + for (int i = 1; i < PING_PONG_ITERATIONS; i++) { + if (i % 5000 == 0) { + T_LOG("Iteration %d", i); + } + + // wait for our turn to synchronize on the shared_cond barrier + rc = pthread_mutex_lock(&ping_pong.sync_mutex); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "lock sync_mutex"); + + while (ping_pong.group != ctx->group) { + rc = pthread_cond_wait(&ping_pong.sync_cond, &ping_pong.sync_mutex); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "sync cond_wait"); + } + + rc = pthread_mutex_unlock(&ping_pong.sync_mutex); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "unlock sync_mutex"); + + rc = pthread_mutex_lock(&ctx->mutex); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "lock mutex"); + + ctx->arrived++; + + if (ctx->arrived == i * PING_PONG_GROUP_NTHREADS) { + // end our turn with shared_cond + rc = pthread_cond_broadcast(&ping_pong.shared_cond); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "shared cond_broadcast"); + + // let the next group begin + rc = pthread_mutex_lock(&ping_pong.sync_mutex); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "lock sync_mutex"); + + ping_pong.group = (ping_pong.group + 1) % PING_PONG_NGROUPS; + + rc = pthread_mutex_unlock(&ping_pong.sync_mutex); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "unlock sync_mutex"); + + // for fun, do this broadcast outside the mutex + rc = pthread_cond_broadcast(&ping_pong.sync_cond); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "sync cond_broadcast"); + + } else { + while (ctx->arrived < i * PING_PONG_GROUP_NTHREADS) { + rc = pthread_cond_wait(&ping_pong.shared_cond, &ctx->mutex); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "shared cond_wait"); + + // TODO: assert that you now hold the correct group mutex + } + } + + rc = pthread_mutex_unlock(&ctx->mutex); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "unlock mutex"); + } + + return NULL; +} + +T_DECL(cond_mutex_ping_pong, "Wait on the same condition variable with different mutexes", + T_META_ENVVAR("PTHREAD_MUTEX_USE_ULOCK=1")) +{ + int rc; + pthread_t threads[PING_PONG_NGROUPS][PING_PONG_GROUP_NTHREADS]; + struct cond_mutex_ping_pong_ctx_s ctxs[PING_PONG_NGROUPS]; + + for (int i = 0; i < PING_PONG_NGROUPS; i++) { + struct cond_mutex_ping_pong_ctx_s *ctx = &ctxs[i]; + *ctx = (struct cond_mutex_ping_pong_ctx_s){ + .mutex = PTHREAD_MUTEX_INITIALIZER, + .group = i, + }; + + for (int j = 0; j < PING_PONG_GROUP_NTHREADS; j++) { + rc = pthread_create(&threads[i][j], NULL, ping_pong_thread, ctx); + T_ASSERT_POSIX_ZERO(rc, "pthread_create"); + } + } + + for (int i = 0; i < PING_PONG_NGROUPS; i++) { + for (int j = 0; j < PING_PONG_GROUP_NTHREADS; j++) { + rc = pthread_join(threads[i][j], NULL); + T_ASSERT_POSIX_ZERO(rc, "pthread_join"); + } + } +} + +#pragma mark signal_thread_np tests + +static struct signal_thread_ctx_s { + pthread_mutex_t mutex; + pthread_cond_t cond; + bool signaled; +} signal_thread_ctx = { + .mutex = PTHREAD_MUTEX_INITIALIZER, + .cond = PTHREAD_COND_INITIALIZER, +}; + +static void * +chosen_waiter(void *arg __unused) +{ + struct signal_thread_ctx_s *ctx = &signal_thread_ctx; + + int rc = pthread_mutex_lock(&ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "chosen waiter lock"); + + while (!ctx->signaled) { + rc = pthread_cond_wait(&ctx->cond, &ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "chosen waiter cond_wait"); + } + + T_PASS("chosen waiter woke"); + T_END; +} + +static void * +other_waiter_thread(void *arg __unused) +{ + struct signal_thread_ctx_s *ctx = &signal_thread_ctx; + + int rc = pthread_mutex_lock(&ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "other waiter lock"); + + while (true) { + rc = pthread_cond_wait(&ctx->cond, &ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "other waiter cond_wait"); + } + + T_ASSERT_FAIL("Not reached"); + return NULL; +} + +T_DECL(cond_signal_thread_np_waiting, "signal a specific thread that's waiting") +{ + int rc; + struct signal_thread_ctx_s *ctx = &signal_thread_ctx; + + pthread_attr_t other_attr; + rc = pthread_attr_init(&other_attr); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "pthread_attr_init"); + + rc = pthread_attr_set_qos_class_np(&other_attr, + QOS_CLASS_USER_INTERACTIVE, 0); + T_ASSERT_POSIX_ZERO(rc, "pthread_attr_set_qos_class_np"); + + pthread_t other; + rc = pthread_create(&other, &other_attr, other_waiter_thread, NULL); + T_ASSERT_POSIX_ZERO(rc, "create other thread"); + + pthread_t chosen; + rc = pthread_create(&chosen, NULL, chosen_waiter, NULL); + T_ASSERT_POSIX_ZERO(rc, "create chosen thread"); + + T_LOG("Waiting for threads to wait"); + sleep(5); + + rc = pthread_mutex_lock(&ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "lock mutex"); + + ctx->signaled = true; + + rc = pthread_mutex_unlock(&ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "unlock mutex"); + + rc = pthread_cond_signal_thread_np(&ctx->cond, chosen); + T_ASSERT_POSIX_ZERO(rc, "cond_signal_thread_np"); + + pthread_join(chosen, NULL); +} + +static void * +absent_chosen_waiter(void *arg __unused) +{ + T_LOG("chosen thread doing nothing forever"); + while (true) { + sleep(100); + } + + T_ASSERT_FAIL("Not reached"); + return NULL; +} + +static void * +not_absent_waiter(void *arg __unused) +{ + struct signal_thread_ctx_s *ctx = &signal_thread_ctx; + + int rc = pthread_mutex_lock(&ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "other waiter lock"); + + while (!ctx->signaled) { + rc = pthread_cond_wait(&ctx->cond, &ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "other waiter cond_wait"); + } + + T_PASS("other waiter woke"); + T_END; +} + +T_DECL(cond_signal_thread_np_not_waiting, "signal a specific thread that isn't waiting") +{ + int rc; + struct signal_thread_ctx_s *ctx = &signal_thread_ctx; + + pthread_t other; + rc = pthread_create(&other, NULL, not_absent_waiter, NULL); + T_ASSERT_POSIX_ZERO(rc, "create other thread"); + + pthread_t chosen; + rc = pthread_create(&chosen, NULL, absent_chosen_waiter, NULL); + T_ASSERT_POSIX_ZERO(rc, "create chosen thread"); + + T_LOG("Waiting for threads to wait"); + sleep(5); + + rc = pthread_mutex_lock(&ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "lock mutex"); + + ctx->signaled = true; + + rc = pthread_mutex_unlock(&ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "unlock mutex"); + + rc = pthread_cond_signal_thread_np(&ctx->cond, chosen); + T_ASSERT_POSIX_ZERO(rc, "cond_signal_thread_np"); + + pthread_join(other, NULL); +} + +#pragma mark cancel signal race test + +static struct cancel_signal_race_context_s { + pthread_mutex_t mutex; + pthread_cond_t cond; +} cancel_signal_race_context = { + .mutex = PTHREAD_MUTEX_INITIALIZER, + .cond = PTHREAD_COND_INITIALIZER, +}; + +static void +cancelee_cleanup_handler(void *arg __unused) +{ + T_LOG("cancelee cleanup handler"); + + struct cancel_signal_race_context_s *ctx = &cancel_signal_race_context; + int rc = pthread_mutex_unlock(&ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "cleanup mutex unlock"); +} + +static void * +cancelee_thread(void *arg __unused) +{ + struct cancel_signal_race_context_s *ctx = &cancel_signal_race_context; + + int rc = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "disabled cancelation of cancelee thread"); + + rc = pthread_mutex_lock(&ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "cancelee lock"); + + rc = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); + if (rc) { + // manual T_QUIET since we can't safely call into libdarwintest with + // cancelation enabled + T_ASSERT_POSIX_ZERO(rc, "cancelation re-enabled"); + } + + pthread_cleanup_push(cancelee_cleanup_handler, NULL); + + rc = pthread_cond_wait(&ctx->cond, &ctx->mutex); + + pthread_cleanup_pop(0); + + int rc2 = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL); + T_QUIET; T_ASSERT_POSIX_ZERO(rc2, "re-disabled cancelation of cancelee thread"); + + // If we make it here we didn't manage to exercise the race, but that's + // legal. + T_ASSERT_POSIX_ZERO(rc, "cancelee woke from cond_wait"); + + rc = pthread_mutex_unlock(&ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "cancelee unlocked"); + + return NULL; +} + +static struct { + int dummy; +} other_thread_timed_out; + +static void * +other_racing_thread(void *arg __unused) +{ + struct cancel_signal_race_context_s *ctx = &cancel_signal_race_context; + + int rc = pthread_mutex_lock(&ctx->mutex); + T_ASSERT_POSIX_ZERO(rc, "other lock"); + + struct timespec ts = { + .tv_sec = 10, + }; + + rc = pthread_cond_timedwait_relative_np(&ctx->cond, &ctx->mutex, &ts); + + int rc2 = pthread_mutex_unlock(&ctx->mutex); + T_ASSERT_POSIX_ZERO(rc2, "other thread unlocked"); + + if (rc == ETIMEDOUT) { + T_LOG("other thread timed out"); + return &other_thread_timed_out; + } else { + // XXX if we change the algorithm in a way that can lead to spurious + // wakeups then this logic might become invalid, but at this point it's + // not possible + T_ASSERT_POSIX_ZERO(rc, "other thread woke from wait"); + return NULL; + } +} + +T_DECL(cond_cancel_signal_race, "Validate waiter cancelation does not eat wakes", + T_META_ENVVAR("PTHREAD_MUTEX_USE_ULOCK=1")) +{ + int rc; + struct cancel_signal_race_context_s *ctx = &cancel_signal_race_context; + + pthread_attr_t cancelee_attr; + rc = pthread_attr_init(&cancelee_attr); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "pthread_attr_init"); + + rc = pthread_attr_set_qos_class_np(&cancelee_attr, + QOS_CLASS_USER_INTERACTIVE, 0); + T_ASSERT_POSIX_ZERO(rc, "pthread_attr_set_qos_class_np"); + + pthread_t cancelee; + rc = pthread_create(&cancelee, &cancelee_attr, cancelee_thread, NULL); + T_ASSERT_POSIX_SUCCESS(rc, "create cancelee"); + + pthread_attr_t other_attr; + rc = pthread_attr_init(&other_attr); + T_QUIET; T_ASSERT_POSIX_ZERO(rc, "pthread_attr_init"); + + rc = pthread_attr_set_qos_class_np(&other_attr, + QOS_CLASS_USER_INITIATED, 0); + T_ASSERT_POSIX_ZERO(rc, "pthread_attr_set_qos_class_np"); + + pthread_t other; + rc = pthread_create(&other, &other_attr, other_racing_thread, NULL); + T_ASSERT_POSIX_SUCCESS(rc, "create other thread"); + + // Give them time to wait + // TODO: find some reliable way of waiting until they're really blocked? + sleep(2); + + rc = pthread_cond_signal(&ctx->cond); + + // Now quickly cancel, hopefully before they make it to userspace + (void)pthread_cancel(cancelee); + + T_ASSERT_POSIX_ZERO(rc, "signal cancelee"); + + void *cancelee_retval, *other_retval; + + rc = pthread_join(cancelee, &cancelee_retval); + T_ASSERT_POSIX_ZERO(rc, "join cancelee"); + + rc = pthread_join(other, &other_retval); + T_ASSERT_POSIX_ZERO(rc, "join other"); + + if (cancelee_retval == PTHREAD_CANCELED) { + T_LOG("cancelee was canceled"); + T_ASSERT_EQ(other_retval, NULL, "other thread must have woken"); + } else { + T_LOG("cancelee was not canceled quickly enough"); + T_ASSERT_EQ(cancelee_retval, NULL, "cancelee returned success"); + T_ASSERT_EQ(other_retval, &other_thread_timed_out, "other thread timed out"); + } +} diff --git a/tests/mutex.c b/tests/mutex.c index 9fe0277..1eda7be 100644 --- a/tests/mutex.c +++ b/tests/mutex.c @@ -103,7 +103,7 @@ check_process_default_mutex_policy(int expected_policy) } T_DECL(mutex_default_policy, - "Tests that the default mutex policy is fairshare") + "Tests that the default mutex policy is firstfit") { check_process_default_mutex_policy(_PTHREAD_MUTEX_POLICY_FIRSTFIT); } @@ -111,10 +111,10 @@ T_DECL(mutex_default_policy, T_DECL(mutex_default_policy_sysctl, "Tests that setting the policy sysctl changes the default policy") { - int firstfit_default = _PTHREAD_MUTEX_POLICY_FIRSTFIT; + int fairshare_default = _PTHREAD_MUTEX_POLICY_FAIRSHARE; T_EXPECT_POSIX_ZERO( - sysctlbyname("kern.pthread_mutex_default_policy", NULL, NULL, &firstfit_default, sizeof(firstfit_default)), - "Changed the default policy sysctl to firstfit"); + sysctlbyname("kern.pthread_mutex_default_policy", NULL, NULL, &fairshare_default, sizeof(fairshare_default)), + "Changed the default policy sysctl to fairshare"); dt_helper_t helper = dt_child_helper("mutex_default_policy_sysctl_helper"); dt_run_helpers(&helper, 1, 5); @@ -122,19 +122,48 @@ T_DECL(mutex_default_policy_sysctl, T_HELPER_DECL(mutex_default_policy_sysctl_helper, "sysctl helper") { - check_process_default_mutex_policy(_PTHREAD_MUTEX_POLICY_FIRSTFIT); + check_process_default_mutex_policy(_PTHREAD_MUTEX_POLICY_FAIRSHARE); - int default_default = _PTHREAD_MUTEX_POLICY_FAIRSHARE; + int default_default = _PTHREAD_MUTEX_POLICY_FIRSTFIT; T_EXPECT_POSIX_ZERO( sysctlbyname("kern.pthread_mutex_default_policy", NULL, NULL, &default_default, sizeof(default_default)), - "Restored the default policy to fairshare"); + "Restored the default policy to firstfit"); T_END; } T_DECL(mutex_default_policy_envvar, "Tests that setting the policy environment variable changes the default policy", - T_META_ENVVAR("PTHREAD_MUTEX_DEFAULT_POLICY=3")) + T_META_ENVVAR("PTHREAD_MUTEX_DEFAULT_POLICY=1")) { - check_process_default_mutex_policy(_PTHREAD_MUTEX_POLICY_FIRSTFIT); + check_process_default_mutex_policy(_PTHREAD_MUTEX_POLICY_FAIRSHARE); +} + +static void * +mutex_as_semaphore_signaller(void *arg) +{ + pthread_mutex_t *mtx = arg; + int rc = pthread_mutex_unlock(mtx); + T_ASSERT_POSIX_ZERO(rc, "unlock"); + + return NULL; +} + +T_DECL(mutex_as_semaphore_lock_owned, "Recursively lock a normal mutex to use as a semaphore") +{ + pthread_t signaller; + pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; + int rc = pthread_mutex_lock(&mtx); + T_ASSERT_POSIX_ZERO(rc, "lock"); + + rc = pthread_create(&signaller, NULL, mutex_as_semaphore_signaller, &mtx); + T_ASSERT_POSIX_ZERO(rc, "create"); + + rc = pthread_mutex_lock(&mtx); + T_ASSERT_POSIX_ZERO(rc, "recursive lock"); + + rc = pthread_join(signaller, NULL); + T_ASSERT_POSIX_ZERO(rc, "join"); + + T_END; } diff --git a/tests/pthread_attr_setstacksize.c b/tests/pthread_attr_setstacksize.c index 845ff65..e12aec8 100644 --- a/tests/pthread_attr_setstacksize.c +++ b/tests/pthread_attr_setstacksize.c @@ -40,6 +40,7 @@ pthread_attr_setstacksize_func(void *arg) T_DECL(pthread_attr_setstacksize, "pthread_attr_setstacksize") { + T_LOG("vm_page_size: %lld vm_kernel_page_size: %lld round_page(MIN): %lld", vm_page_size, vm_kernel_page_size, round_page(PTHREAD_STACK_MIN)); size_t stacksizes[] = {PTHREAD_STACK_MIN, 1024ULL * 16, 1024ULL * 32, 1024ULL * 1024}; for (int i = 0; (size_t)i < sizeof(stacksizes)/sizeof(stacksizes[0]); i++){ pthread_t t = NULL; diff --git a/tests/pthread_cwd.c b/tests/pthread_cwd.c index 68490f8..45242cc 100644 --- a/tests/pthread_cwd.c +++ b/tests/pthread_cwd.c @@ -11,8 +11,6 @@ #include "darwintest_defaults.h" -#include "../src/pthread_cwd.c" - // /tmp is a symlink, so use full path for strict compare #define WORKDIR "/private/var/tmp/ptwork" #define WORKDIR1 WORKDIR "/one" diff --git a/tests/pthread_exit.c b/tests/pthread_exit.c index cd5b12a..d5d9427 100644 --- a/tests/pthread_exit.c +++ b/tests/pthread_exit.c @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -102,3 +103,19 @@ T_DECL(pthread_exit_detached, "pthread_exit with detached threads") } T_PASS("Success!"); } + +static void +key_dtor(void *value) +{ + T_ASSERT_EQ(1, pthread_self_is_exiting_np(), "exiting"); +} + +T_DECL(pthread_self_is_exiting_np, "pthread_self_is_exiting_np") +{ + pthread_key_t key; + + T_ASSERT_POSIX_ZERO(pthread_key_create(&key, key_dtor), NULL); + pthread_setspecific(key, (void *)-1); + T_ASSERT_EQ(0, pthread_self_is_exiting_np(), "not exiting"); + pthread_exit(NULL); +} diff --git a/tests/pthread_introspection.c b/tests/pthread_introspection.c index 5fc5459..5287dd3 100644 --- a/tests/pthread_introspection.c +++ b/tests/pthread_introspection.c @@ -6,6 +6,17 @@ #include "darwintest_defaults.h" static pthread_introspection_hook_t prev_pthread_introspection_hook; +static pthread_key_t key; +static atomic_int keys_set; +static atomic_int keys_cleared; +static const char value_for_key[] = "used as a TSD value"; + +static void +key_destructor(void *ctx) +{ + T_EXPECT_EQ(ctx, (void *)value_for_key, "check value"); + keys_cleared++; +} #define THREAD_COUNT 3 @@ -14,18 +25,30 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, { static atomic_int create_count; static atomic_int terminate_count; + static atomic_int destroy_count; uint64_t tid; pthread_threadid_np(NULL, &tid); - if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) { + switch (event) { + case PTHREAD_INTROSPECTION_THREAD_CREATE: + atomic_fetch_add(&keys_set, 1); + pthread_introspection_setspecific_np(thread, key, value_for_key); T_LOG("event = PTHREAD_INTROSPECTION_THREAD_CREATE, thread = %p:%lld, addr = %p, size = 0x%zx", thread, tid, addr, size); create_count++; - } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { + break; + case PTHREAD_INTROSPECTION_THREAD_TERMINATE: T_LOG("event = PTHREAD_INTROSPECTION_THREAD_TERMINATE, thread = %p:%lld, addr = %p, size = 0x%zx", thread, tid, addr, size); terminate_count++; T_ASSERT_GE(create_count, THREAD_COUNT, NULL); T_PASS("Got termination events"); + break; + case PTHREAD_INTROSPECTION_THREAD_DESTROY: + T_LOG("event = PTHREAD_INTROSPECTION_THREAD_DESTROY, thread = %p:%lld, addr = %p, size = 0x%zx", thread, tid, addr, size); + destroy_count++; + T_ASSERT_NULL(pthread_introspection_getspecific_np(thread, key), "should have cleared"); + T_ASSERT_NE(keys_cleared, 0, "should have cleared a key"); + T_PASS("Got destruction events"); T_END; } @@ -35,9 +58,10 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, } T_DECL(pthread_introspection, "PR-25679871", - T_META_TIMEOUT(30), T_META_ALL_VALID_ARCHS(YES)) + T_META_TIMEOUT(30), T_META_ALL_VALID_ARCHS(YES)) { prev_pthread_introspection_hook = pthread_introspection_hook_install(&my_pthread_introspection_hook); + pthread_key_create(&key, key_destructor); // minus one that comes after this block for (int i = 0; i < THREAD_COUNT - 1; i++) { diff --git a/tests/pthread_jit_write_protection-entitlements.plist b/tests/pthread_jit_write_protection-entitlements.plist new file mode 100644 index 0000000..2703dcd --- /dev/null +++ b/tests/pthread_jit_write_protection-entitlements.plist @@ -0,0 +1,9 @@ + + + + + dynamic-codesigning + + + + diff --git a/tests/pthread_jit_write_protection.c b/tests/pthread_jit_write_protection.c new file mode 100644 index 0000000..bb17876 --- /dev/null +++ b/tests/pthread_jit_write_protection.c @@ -0,0 +1,353 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#if __has_include() +#include +#endif + +T_GLOBAL_META(T_META_RUN_CONCURRENTLY(true)); + +/* Enumerations */ +typedef enum _access_type { + ACCESS_WRITE, + ACCESS_EXECUTE, +} access_type_t; + +typedef enum _fault_strategy { + FAULT_STRAT_NONE, + FAULT_STRAT_RX, + FAULT_STRAT_RW +} fault_strategy_t; + +/* Structures */ +typedef struct { + uint64_t fault_count; + fault_strategy_t fault_strategy; + bool fault_expected; +} fault_state_t; + +/* Globals */ +static void * rwx_addr = NULL; +static pthread_key_t jit_test_fault_state_key; + +/* + * Return instruction encodings; a default value is given so that this test can + * be built for an architecture that may not support the tested feature. + */ +#ifdef __arm__ +static uint32_t ret_encoding = 0xe12fff1e; +#elif defined(__arm64__) +static uint32_t ret_encoding = 0xd65f03c0; +#elif defined(__x86_64__) +static uint32_t ret_encoding = 0x909090c3;; +#else +#error "Unsupported architecture" +#endif + +/* Allocate a fault_state_t, and associate it with the current thread. */ +static fault_state_t * +fault_state_create(void) +{ + fault_state_t * fault_state = malloc(sizeof(fault_state_t)); + + if (fault_state) { + fault_state->fault_count = 0; + fault_state->fault_strategy = FAULT_STRAT_NONE; + fault_state->fault_expected = false; + + if (pthread_setspecific(jit_test_fault_state_key, fault_state)) { + free(fault_state); + fault_state = NULL; + } + } + + return fault_state; +} + +/* Disassociate the given fault state from the current thread, and destroy it. */ +static void +fault_state_destroy(void * fault_state) +{ + if (fault_state == NULL) { + T_ASSERT_FAIL("Attempted to fault_state_destroy NULL"); + } + + free(fault_state); +} + +/* + * A signal handler that attempts to resolve anticipated faults through use of + * the pthread_jit_write_protect functions. + */ +static void +access_failed_handler(int signum) +{ + fault_state_t * fault_state; + + /* This handler should ONLY handle SIGBUS. */ + if (signum != SIGBUS) { + T_ASSERT_FAIL("Unexpected signal sent to handler"); + } + + if (!(fault_state = pthread_getspecific(jit_test_fault_state_key))) { + T_ASSERT_FAIL("Failed to retrieve fault state"); + } + + if (!(fault_state->fault_expected)) { + T_ASSERT_FAIL("Unexpected fault taken"); + } + + /* We should not see a second fault. */ + fault_state->fault_expected = false; + + switch (fault_state->fault_strategy) { + case FAULT_STRAT_NONE: + T_ASSERT_FAIL("No fault strategy"); + + /* Just in case we try to do something different. */ + break; + case FAULT_STRAT_RX: + pthread_jit_write_protect_np(TRUE); + break; + case FAULT_STRAT_RW: + pthread_jit_write_protect_np(FALSE); + break; + } + + fault_state->fault_count++; +} + +/* + * Attempt the specified access; if the access faults, this will return true; + * otherwise, it will return false. + */ +static bool +does_access_fault(access_type_t access_type, void * addr) +{ + uint64_t old_fault_count; + uint64_t new_fault_count; + + fault_state_t * fault_state; + + struct sigaction old_action; /* Save area for any existing action. */ + struct sigaction new_action; /* The action we wish to install for SIGBUS. */ + + bool retval = false; + + void (*func)(void); + + new_action.sa_handler = access_failed_handler; /* A handler for write failures. */ + new_action.sa_mask = 0; /* Don't modify the mask. */ + new_action.sa_flags = 0; /* Flags? Who needs those? */ + + if (addr == NULL) { + T_ASSERT_FAIL("Access attempted against NULL"); + } + + if (!(fault_state = pthread_getspecific(jit_test_fault_state_key))) { + T_ASSERT_FAIL("Failed to retrieve fault state"); + } + + old_fault_count = fault_state->fault_count; + + /* Install a handler so that we can catch SIGBUS. */ + sigaction(SIGBUS, &new_action, &old_action); + + /* Perform the requested operation. */ + switch (access_type) { + case ACCESS_WRITE: + fault_state->fault_strategy = FAULT_STRAT_RW; + fault_state->fault_expected = true; + + __sync_synchronize(); + + /* Attempt to scrawl a return instruction to the given address. */ + *((volatile uint32_t *)addr) = ret_encoding; + + __sync_synchronize(); + + fault_state->fault_expected = false; + fault_state->fault_strategy = FAULT_STRAT_NONE; + + /* Invalidate the instruction cache line that we modified. */ + sys_cache_control(kCacheFunctionPrepareForExecution, addr, sizeof(ret_encoding)); + + break; + case ACCESS_EXECUTE: + /* This is a request to branch to the given address. */ +#if __has_feature(ptrauth_calls) + func = ptrauth_sign_unauthenticated((void *)addr, ptrauth_key_function_pointer, 0); +#else + func = (void (*)(void))addr; +#endif + + + fault_state->fault_strategy = FAULT_STRAT_RX; + fault_state->fault_expected = true; + + __sync_synchronize(); + + /* Branch. */ + func(); + + __sync_synchronize(); + + fault_state->fault_expected = false; + fault_state->fault_strategy = FAULT_STRAT_NONE; + + break; + } + + /* Restore the old SIGBUS handler. */ + sigaction(SIGBUS, &old_action, NULL); + + new_fault_count = fault_state->fault_count; + + if (new_fault_count > old_fault_count) { + /* Indicate that we took a fault. */ + retval = true; + } + + return retval; +} + +static void * +expect_write_fail_thread(__unused void * arg) +{ + fault_state_create(); + + if (does_access_fault(ACCESS_WRITE, rwx_addr)) { + pthread_exit((void *)0); + } else { + pthread_exit((void *)1); + } +} + +T_DECL(pthread_jit_write_protect, + "Verify that the pthread_jit_write_protect interfaces work correctly") +{ + void * addr = NULL; + size_t alloc_size = PAGE_SIZE; + fault_state_t * fault_state = NULL; + int err = 0; + bool key_created = false; + void * join_value = NULL; + pthread_t pthread; + bool expect_fault = pthread_jit_write_protect_supported_np(); + + T_SETUPBEGIN; + + /* Set up the necessary state for the test. */ + err = pthread_key_create(&jit_test_fault_state_key, fault_state_destroy); + + T_ASSERT_POSIX_ZERO(err, 0, "Create pthread key"); + + key_created = true; + + fault_state = fault_state_create(); + + T_ASSERT_NOTNULL(fault_state, "Create fault state"); + + /* + * Create a JIT enabled mapping that we can use to test restriction of + * RWX mappings. + */ + rwx_addr = mmap(addr, alloc_size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_ANON | MAP_PRIVATE | MAP_JIT, -1, 0); + + T_ASSERT_NE_PTR(rwx_addr, MAP_FAILED, "Map range as MAP_JIT"); + + T_SETUPEND; + + /* + * Validate that we fault when we should, and that we do not fault when + * we should not fault. + */ + pthread_jit_write_protect_np(FALSE); + + T_EXPECT_EQ(does_access_fault(ACCESS_WRITE, rwx_addr), 0, "Write with RWX->RW"); + + pthread_jit_write_protect_np(TRUE); + + T_EXPECT_EQ(does_access_fault(ACCESS_EXECUTE, rwx_addr), 0, "Execute with RWX->RX"); + + pthread_jit_write_protect_np(TRUE); + + T_EXPECT_EQ(does_access_fault(ACCESS_WRITE, rwx_addr), expect_fault, "Write with RWX->RX"); + + pthread_jit_write_protect_np(FALSE); + + T_EXPECT_EQ(does_access_fault(ACCESS_EXECUTE, rwx_addr), expect_fault, "Execute with RWX->RW"); + + pthread_jit_write_protect_np(FALSE); + + if (expect_fault) { + /* + * Create another thread for testing multithreading; mark this as setup + * as this test is not targeted towards the pthread create/join APIs. + */ + T_SETUPBEGIN; + + T_ASSERT_POSIX_ZERO(pthread_create(&pthread, NULL, expect_write_fail_thread, NULL), "pthread_create expect_write_fail_thread"); + + T_ASSERT_POSIX_ZERO(pthread_join(pthread, &join_value), "pthread_join expect_write_fail_thread"); + + T_SETUPEND; + + /* + * Validate that the other thread was unable to write to the JIT region + * without independently using the pthread_jit_write_protect code. + */ + T_ASSERT_NULL((join_value), "Write on other thread with RWX->RX, " + "RWX->RW on parent thread"); + } + + /* We're done with the test; tear down our extra state. */ + /* + * This would be better dealt with using T_ATEND, but this would require + * making many variables global. This can be changed in the future. + * For now, mark this as SETUP (even though this is really teardown). + */ + T_SETUPBEGIN; + + T_ASSERT_POSIX_SUCCESS(munmap(rwx_addr, alloc_size), "Unmap MAP_JIT mapping"); + + if (fault_state) { + T_ASSERT_POSIX_ZERO(pthread_setspecific(jit_test_fault_state_key, NULL), "Remove fault_state"); + + fault_state_destroy(fault_state); + } + + if (key_created) { + T_ASSERT_POSIX_ZERO(pthread_key_delete(jit_test_fault_state_key), "Delete fault state key"); + } + + T_SETUPEND; +} + +T_DECL(thread_self_restrict_rwx_perf, + "Test the performance of the thread_self_restrict_rwx interfaces", + T_META_TAG_PERF, T_META_CHECK_LEAKS(false)) +{ + dt_stat_time_t dt_stat_time; + + dt_stat_time = dt_stat_time_create("rx->rw->rx time"); + + T_STAT_MEASURE_LOOP(dt_stat_time) { + pthread_jit_write_protect_np(FALSE); + pthread_jit_write_protect_np(TRUE); + } + + dt_stat_finalize(dt_stat_time); +} diff --git a/tests/tsd.c b/tests/tsd.c index 259816d..d20dcf1 100644 --- a/tests/tsd.c +++ b/tests/tsd.c @@ -1,5 +1,6 @@ #include #include +#include #include "darwintest_defaults.h" @@ -41,3 +42,28 @@ T_DECL(tsd, "tsd", T_ASSERT_POSIX_ZERO(pthread_key_delete(key), NULL); } + +static uint32_t +get_ncpu(void) +{ + static uint32_t activecpu; + if (!activecpu) { + uint32_t n; + size_t s = sizeof(activecpu); + sysctlbyname("hw.activecpu", &n, &s, NULL, 0); + activecpu = n; + } + return activecpu; +} + +T_DECL(cpuid, "cpu id", T_META_ALL_VALID_ARCHS(YES)) +{ + pthread_t child; + + size_t cpu_id; + if (pthread_cpu_number_np(&cpu_id)) { + T_FAIL("Should not fail to get CPU id"); + } + + T_ASSERT_LE(cpu_id, get_ncpu(), "Got a valid CPU id"); +} diff --git a/tests/wq_event_manager.c b/tests/wq_event_manager.c index bed9faa..aeae982 100644 --- a/tests/wq_event_manager.c +++ b/tests/wq_event_manager.c @@ -10,8 +10,8 @@ #include -#include "../private/workqueue_private.h" -#include "../private/qos_private.h" +#include "../private/pthread/workqueue_private.h" +#include "../private/pthread/qos_private.h" #include "wq_kevent.h" diff --git a/tests/wq_kevent.c b/tests/wq_kevent.c index 2b02039..4e13a29 100644 --- a/tests/wq_kevent.c +++ b/tests/wq_kevent.c @@ -10,8 +10,8 @@ #include -#include "../private/workqueue_private.h" -#include "../private/qos_private.h" +#include "../private/pthread/workqueue_private.h" +#include "../private/pthread/qos_private.h" #include "wq_kevent.h" diff --git a/tests/wq_kevent_stress.c b/tests/wq_kevent_stress.c index 7d47def..f409405 100644 --- a/tests/wq_kevent_stress.c +++ b/tests/wq_kevent_stress.c @@ -11,8 +11,8 @@ #include -#include "../private/workqueue_private.h" -#include "../private/qos_private.h" +#include "../private/pthread/workqueue_private.h" +#include "../private/pthread/qos_private.h" #include "wq_kevent.h" diff --git a/xcodescripts/install-manpages.sh b/xcodescripts/install-manpages.sh index b8da6de..bf60694 100644 --- a/xcodescripts/install-manpages.sh +++ b/xcodescripts/install-manpages.sh @@ -86,7 +86,9 @@ BASE_PAGES="pthread.3 \ pthread_setname_np.3 \ pthread_setspecific.3 \ pthread_threadid_np.3 \ - pthread_yield_np.3" + pthread_yield_np.3 \ + pthread_jit_write_protect_np.3" + cp $BASE_PAGES "$DSTROOT"/usr/share/man/man3 @@ -113,6 +115,7 @@ chmod $INSTALL_MODE_FLAG $BASE_PAGES ln -fh pthread_getschedparam.3 pthread_setschedparam.3 ln -fh pthread_rwlock_rdlock.3 pthread_rwlock_tryrdlock.3 ln -fh pthread_rwlock_wrlock.3 pthread_rwlock_trywrlock.3 +ln -fh pthread_jit_write_protect_np.3 pthread_jit_write_protect_supported_np.3 for M in \ pthread_attr_destroy.3 \ diff --git a/xcodescripts/install-sys-headers.sh b/xcodescripts/install-sys-headers.sh index a5b4eba..7b391fc 100644 --- a/xcodescripts/install-sys-headers.sh +++ b/xcodescripts/install-sys-headers.sh @@ -25,39 +25,16 @@ set -e if [ "$ACTION" = build ]; then exit 0; fi -DSTROOT="${DSTROOT}/${SDK_INSTALL_HEADERS_ROOT}" +install_headers() +{ + mkdir -p "${DSTROOT}/${SDK_INSTALL_HEADERS_ROOT}$2" + cp -r "${SRCROOT}/$1" "${DSTROOT}/${SDK_INSTALL_HEADERS_ROOT}$2" -DESTDIR="$DSTROOT/usr/include/sys" -mkdir -p "$DESTDIR" -for X in \ - qos.h \ - ; do - cp "sys/$X" "$DESTDIR" -done - -DESTDIR="$DSTROOT/usr/local/include/sys" -mkdir -p "$DESTDIR" -for X in \ - qos_private.h \ - ; do - cp "sys/$X" "$DESTDIR" -done - -DESTDIR="$DSTROOT/usr/include/sys/_pthread" -mkdir -p "$DESTDIR" -for X in \ - _pthread_attr_t.h \ - _pthread_cond_t.h \ - _pthread_condattr_t.h \ - _pthread_key_t.h \ - _pthread_mutex_t.h \ - _pthread_mutexattr_t.h \ - _pthread_once_t.h \ - _pthread_rwlock_t.h \ - _pthread_rwlockattr_t.h \ - _pthread_t.h \ - _pthread_types.h \ - ; do - cp "sys/_pthread/$X" "$DESTDIR" -done + find "${DSTROOT}/${SDK_INSTALL_HEADERS_ROOT}$2" -type f -name *.h -print0 | \ + xargs -0I % unifdef -t ${COPY_HEADERS_UNIFDEF_FLAGS} -o "%" "%" + find "${DSTROOT}/${SDK_INSTALL_HEADERS_ROOT}$2" -type f -name *.modulemap -print0 | \ + xargs -0I % unifdef -t ${COPY_HEADERS_UNIFDEF_FLAGS} -o "%" "%" +} +install_headers "include/sys" "/usr/include" +install_headers "private/sys" "/usr/local/include" diff --git a/xcodescripts/kext.xcconfig b/xcodescripts/kext.xcconfig index c28a730..1138c46 100644 --- a/xcodescripts/kext.xcconfig +++ b/xcodescripts/kext.xcconfig @@ -22,7 +22,7 @@ PRODUCT_BUNDLE_IDENTIFIER = ${MODULE_NAME} INFOPLIST_FILE = kern/pthread-Info.plist ALWAYS_SEARCH_USER_PATHS = NO -SRCROOT_SEARCH_PATHS = $(SRCROOT) $(SRCROOT)/pthread $(SRCROOT)/private +SRCROOT_SEARCH_PATHS = $(SRCROOT)/private $(SRCROOT)/include $(SRCROOT) HEADER_SEARCH_PATHS = $(SDKROOT)/System/Library/Frameworks/Kernel.framework/PrivateHeaders $(SDKROOT)/System/Library/Frameworks/Kernel.framework/Headers $(SDKROOT)/System/Library/Frameworks/System.framework/PrivateHeaders $(SDKROOT)/System/Library/Frameworks/System.framework/Headers $(SRCROOT_SEARCH_PATHS) GCC_C_LANGUAGE_STANDARD = gnu11 CLANG_CXX_LANGUAGE_STANDARD = gnu++0x diff --git a/xcodescripts/pthread-i386.aliases b/xcodescripts/pthread-i386.aliases new file mode 100644 index 0000000..e4a1276 --- /dev/null +++ b/xcodescripts/pthread-i386.aliases @@ -0,0 +1,21 @@ +# aliases file for pthread old-symbol aliases +# + +_pthread_cancel$UNIX2003 _pthread_cancel +_pthread_cond_init$UNIX2003 _pthread_cond_init +_pthread_cond_timedwait$UNIX2003 _pthread_cond_timedwait +_pthread_cond_wait$UNIX2003 _pthread_cond_wait +_pthread_join$UNIX2003 _pthread_join +_pthread_mutexattr_destroy$UNIX2003 _pthread_mutexattr_destroy +_pthread_rwlock_destroy$UNIX2003 _pthread_rwlock_destroy +_pthread_rwlock_init$UNIX2003 _pthread_rwlock_init +_pthread_rwlock_rdlock$UNIX2003 _pthread_rwlock_rdlock +_pthread_rwlock_tryrdlock$UNIX2003 _pthread_rwlock_tryrdlock +_pthread_rwlock_trywrlock$UNIX2003 _pthread_rwlock_trywrlock +_pthread_rwlock_unlock$UNIX2003 _pthread_rwlock_unlock +_pthread_rwlock_wrlock$UNIX2003 _pthread_rwlock_wrlock +_pthread_setcancelstate$UNIX2003 _pthread_setcancelstate +_pthread_setcanceltype$UNIX2003 _pthread_setcanceltype +_pthread_sigmask$UNIX2003 _pthread_sigmask +_pthread_testcancel$UNIX2003 _pthread_testcancel +_sigwait$UNIX2003 _sigwait diff --git a/xcodescripts/pthread-tapi.xcconfig b/xcodescripts/pthread-tapi.xcconfig new file mode 100644 index 0000000..9bd92cd --- /dev/null +++ b/xcodescripts/pthread-tapi.xcconfig @@ -0,0 +1,11 @@ +// TAPI +SUPPORTS_TEXT_BASED_API = YES +SUPPORTS_TEXT_BASED_API[sdk=iphonesimulator*] = NO +TAPI_VERIFY_MODE = Pedantic + +TAPI_PREPROCESSOR = -DPTHREAD_LAYOUT_SPI=1 -Dposix_spawnattr_t="void *" +TAPI_ADD_PUBLIC_HEADERS = -extra-public-header $(DSTROOT)$(SDK_INSTALL_HEADERS_ROOT)/usr/include/pthread/sched.h -extra-public-header $(DSTROOT)$(SDK_INSTALL_HEADERS_ROOT)/usr/include/sys/qos.h +TAPI_ADD_PRIVATE_HEADERS = -extra-private-header $(PROJECT_DIR)/src/exports_internal.h +TAPI_EXCL_PRIVATE_HEADERS = -exclude-private-header $(DSTROOT)$(SDK_INSTALL_HEADERS_ROOT)/usr/local/include/pthread/qos.h -exclude-private-header $(DSTROOT)$(SDK_INSTALL_HEADERS_ROOT)/usr/local/include/pthread/spinlock_private.h +TAPI_ADD_ALIASES = -alias_list $(PROJECT_DIR)/xcodescripts/pthread.aliases +OTHER_TAPI_FLAGS = -umbrella System $(TAPI_ADD_PUBLIC_HEADERS) $(TAPI_ADD_PRIVATE_HEADERS) $(TAPI_EXCL_PRIVATE_HEADERS) $(TAPI_ADD_ALIASES) $(TAPI_PREPROCESSOR) diff --git a/xcodescripts/pthread.aliases b/xcodescripts/pthread.aliases index a71e089..1c40fe5 100644 --- a/xcodescripts/pthread.aliases +++ b/xcodescripts/pthread.aliases @@ -1,2 +1,6 @@ # aliases file for pthread old-symbol aliases # + +_pthread_is_threaded_np __pthread_is_threaded +_sched_yield _cthread_yield +_sched_yield _pthread_yield_np diff --git a/xcodescripts/pthread.dirty b/xcodescripts/pthread.dirty index 45990a8..99fbfd5 100644 --- a/xcodescripts/pthread.dirty +++ b/xcodescripts/pthread.dirty @@ -20,7 +20,6 @@ ___is_threaded ___pthread_supported_features ___pthread_tsd_lock ___pthread_tsd_max -___unix_conforming __main_qos __pthread_count __pthread_list_lock diff --git a/xcodescripts/pthread.xcconfig b/xcodescripts/pthread.xcconfig index 9094c6d..e5ca03d 100644 --- a/xcodescripts/pthread.xcconfig +++ b/xcodescripts/pthread.xcconfig @@ -19,7 +19,10 @@ PRODUCT_NAME = system_pthread PUBLIC_HEADERS_FOLDER_PATH = $(SDK_INSTALL_HEADERS_ROOT)/usr/include/pthread PRIVATE_HEADERS_FOLDER_PATH = $(SDK_INSTALL_HEADERS_ROOT)/usr/local/include/pthread -SRCROOT_SEARCH_PATHS = $(SRCROOT) $(SRCROOT)/private $(SRCROOT)/os $(SRCROOT)/src/resolver +COPY_HEADERS_RUN_UNIFDEF = YES +COPY_HEADERS_UNIFDEF_FLAGS = -U__PTHREAD_BUILDING_PTHREAD__ + +SRCROOT_SEARCH_PATHS = $(SRCROOT)/src/resolver $(SRCROOT)/private $(SRCROOT)/include $(SRCROOT) SYSTEM_FRAMEWORK_HEADERS = $(SDKROOT)/$(SDK_INSTALL_HEADERS_ROOT)/System/Library/Frameworks/System.framework/PrivateHeaders HEADER_SEARCH_PATHS = $($(PRODUCT_NAME)_SEARCH_PATHS) $(SRCROOT_SEARCH_PATHS) $(inherited) SYSTEM_HEADER_SEARCH_PATHS = $(SYSTEM_FRAMEWORK_HEADERS) $(SDKROOT)/$(SDK_INSTALL_HEADERS_ROOT)/usr/local/include $(SDKROOT)/$(SDK_INSTALL_HEADERS_ROOT)/usr/include @@ -38,7 +41,7 @@ CLANG_LINK_OBJC_RUNTIME = NO GCC_WARN_64_TO_32_BIT_CONVERSION = YES GCC_WARN_ABOUT_MISSING_FIELD_INITIALIZERS = YES GCC_WARN_ABOUT_MISSING_NEWLINE = YES -//GCC_WARN_ABOUT_MISSING_PROTOTYPES = YES +GCC_WARN_ABOUT_MISSING_PROTOTYPES = YES GCC_WARN_ABOUT_RETURN_TYPE = YES GCC_WARN_INITIALIZER_NOT_FULLY_BRACKETED = YES GCC_WARN_SIGN_COMPARE = YES @@ -64,11 +67,11 @@ CLANG_WARN_SUSPICIOUS_MOVE = YES CLANG_WARN_UNREACHABLE_CODE = YES CLANG_WARN__DUPLICATE_METHOD_MATCH = YES -DISABLED_WARNING_CFLAGS = -Wno-int-conversion -Wno-missing-prototypes -Wno-sign-compare -Wno-sign-conversion -Wno-unused-parameter -WARNING_CFLAGS = -Wall -Wextra -Warray-bounds-pointer-arithmetic -Wcomma -Wconditional-uninitialized -Wcovered-switch-default -Wdate-time -Wdeprecated -Wdouble-promotion -Wduplicate-enum -Wfloat-equal -Widiomatic-parentheses -Wignored-qualifiers -Wimplicit-fallthrough -Wmissing-noreturn -Wnullable-to-nonnull-conversion -Wover-aligned -Wpointer-arith -Wstatic-in-inline -Wtautological-compare -Wunguarded-availability -Wunused $(NO_WARNING_CFLAGS) $(DISABLED_WARNING_CFLAGS) -NO_WARNING_CFLAGS = -Wno-pedantic -Wno-bad-function-cast -Wno-c++98-compat-pedantic -Wno-cast-align -Wno-cast-qual -Wno-disabled-macro-expansion -Wno-documentation-unknown-command -Wno-format-nonliteral -Wno-missing-variable-declarations -Wno-packed -Wno-padded -Wno-reserved-id-macro -Wno-switch-enum -Wno-undef -Wno-unreachable-code-aggressive -Wno-unused-macros -Wno-used-but-marked-unused +DISABLED_WARNING_CFLAGS = -Wno-int-conversion -Wno-sign-compare -Wno-sign-conversion -Wno-unused-parameter +WARNING_CFLAGS = -Wall -Wextra -Wmost -Warray-bounds-pointer-arithmetic -Wcomma -Wconditional-uninitialized -Wcovered-switch-default -Wdate-time -Wdeprecated -Wdouble-promotion -Wduplicate-enum -Wfloat-equal -Widiomatic-parentheses -Wignored-qualifiers -Wimplicit-fallthrough -Wmissing-noreturn -Wnullable-to-nonnull-conversion -Wover-aligned -Wpointer-arith -Wstatic-in-inline -Wtautological-compare -Wunguarded-availability -Wunused -Watomic-implicit-seq-cst $(NO_WARNING_CFLAGS) $(DISABLED_WARNING_CFLAGS) +NO_WARNING_CFLAGS = -Wno-pedantic -Wno-bad-function-cast -Wno-c++98-compat-pedantic -Wno-cast-align -Wno-cast-qual -Wno-disabled-macro-expansion -Wno-documentation-unknown-command -Wno-format-nonliteral -Wno-packed -Wno-padded -Wno-reserved-id-macro -Wno-switch-enum -Wno-undef -Wno-unreachable-code-aggressive -Wno-unused-macros -Wno-used-but-marked-unused -BASE_PREPROCESSOR_MACROS = __LIBC__ __DARWIN_UNIX03=1 __DARWIN_64_BIT_INO_T=1 __DARWIN_NON_CANCELABLE=1 __DARWIN_VERS_1050=1 _FORTIFY_SOURCE=0 __PTHREAD_BUILDING_PTHREAD__=1 $(SIM_PREPROCESSOR_MACROS) __PTHREAD_EXPOSE_INTERNALS__ +BASE_PREPROCESSOR_MACROS = __LIBC__ __POSIX_LIB__ __DARWIN_UNIX03=1 __DARWIN_64_BIT_INO_T=1 __DARWIN_NON_CANCELABLE=1 __DARWIN_VERS_1050=1 _FORTIFY_SOURCE=0 __PTHREAD_BUILDING_PTHREAD__=1 $(SIM_PREPROCESSOR_MACROS) __PTHREAD_EXPOSE_INTERNALS__ OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY=1 GCC_PREPROCESSOR_DEFINITIONS = $(BASE_PREPROCESSOR_MACROS) $(PLATFORM_PREPROCESSOR_DEFINITIONS) // TODO: Remove -fstack-protector on _debug when it is moved to libplatform @@ -80,9 +83,29 @@ LINK_WITH_STANDARD_LIBRARIES = NO DYLIB_CURRENT_VERSION = $(RC_ProjectSourceVersion) DYLIB_COMPATIBILITY_VERSION = 1 DIRTY_LDFLAGS = -Wl,-dirty_data_list,$(SRCROOT)/xcodescripts/pthread.dirty -DIRTY_LDFLAGS[sdk=macos*] = -DYLIB_LDFLAGS = -Wl,-alias_list,$(SRCROOT)/xcodescripts/pthread.aliases -Wl,-umbrella,System -L$(SDK_INSTALL_ROOT)/usr/lib/system -lsystem_kernel -lsystem_platform -ldyld -lcompiler_rt -OTHER_LDFLAGS = $(DYLIB_LDFLAGS) $(DIRTY_LDFLAGS) $(CR_LDFLAGS) $(PLATFORM_LDFLAGS) $(SIMULATOR_LDFLAGS) + +LEGACY_ALIAS_LDFLAGS = +LEGACY_ALIAS_LDFLAGS[sdk=macos*][arch=i386*] = -Wl,-alias_list,$(SRCROOT)/xcodescripts/pthread-i386.aliases +ALIAS_LDFLAGS = -Wl,-alias_list,$(SRCROOT)/xcodescripts/pthread.aliases $(LEGACY_ALIAS_LDFLAGS) + +// rdar://problem/46882983&54282933 +// On macOS, to support the i386 watchOS Simulator, we will continue building +// libpthread with an i386 slice for the foreseeable future, even though the +// rest of the OS has dropped i386. (This also applies to libplatform and +// libsyscall). Normally, dylibs with any dependency on another dylib need +// to link libdyld for lazy stub binding. libdyld has many dependencies, so +// that would create a dependency cycle that leads to the whole libSystem +// umbrella keeping an i386 slice. Instead, ld64 has changed so that the +// i386 simulator_support slice of libpthread doesn't use lazy binding and so +// doesn't need -ldyld. +// So, to break the dependency cycle, macOS libpthread will not link libdyld. +// All other platforms (including DriverKit on macOS) will continue to link +// libdyld. +MACOS_NO_LIBDYLD_LDFLAGS = -ldyld +MACOS_NO_LIBDYLD_LDFLAGS[sdk=macos*] = + +DYLIB_LDFLAGS = -Wl,-umbrella,System -L$(SDK_INSTALL_ROOT)/usr/lib/system -lsystem_kernel -lsystem_platform $(MACOS_NO_LIBDYLD_LDFLAGS) // Don't add compiler_rt or libdyld +OTHER_LDFLAGS = $(ALIAS_LDFLAGS) $(DYLIB_LDFLAGS) $(DIRTY_LDFLAGS) $(CR_LDFLAGS) $(PLATFORM_LDFLAGS) $(SIMULATOR_LDFLAGS) SIMULATOR_LDFLAGS = SIMULATOR_LDFLAGS[sdk=macosx*] = -Wl,-simulator_support diff --git a/xcodescripts/pthread_driverkit.xcconfig b/xcodescripts/pthread_driverkit.xcconfig new file mode 100644 index 0000000..4e7bb7e --- /dev/null +++ b/xcodescripts/pthread_driverkit.xcconfig @@ -0,0 +1,2 @@ +#include "pthread.xcconfig" +#include "pthread-tapi.xcconfig" diff --git a/xcodescripts/resolver.xcconfig b/xcodescripts/resolver.xcconfig index 85729af..c3fcd04 100644 --- a/xcodescripts/resolver.xcconfig +++ b/xcodescripts/resolver.xcconfig @@ -1,2 +1,3 @@ #include "pthread.xcconfig" +#include "pthread-tapi.xcconfig" diff --git a/xcodescripts/static.xcconfig b/xcodescripts/static.xcconfig index 611c29c..44e1aa5 100644 --- a/xcodescripts/static.xcconfig +++ b/xcodescripts/static.xcconfig @@ -5,3 +5,4 @@ EXECUTABLE_PREFIX = lib PRODUCT_NAME = pthread GCC_PREPROCESSOR_DEFINITIONS = $(BASE_PREPROCESSOR_MACROS) VARIANT_STATIC=1 OTHER_LDFLAGS = +INSTALLHDRS_SCRIPT_PHASE = NO -- 2.45.2