From e45b469263efe5fed40a01d25b2ace7e42243579 Mon Sep 17 00:00:00 2001 From: Apple Date: Tue, 26 Sep 2017 16:51:55 +0000 Subject: [PATCH] libplatform-161.tar.gz --- .upstream_base_commits | 2 + include/libkern/OSAtomicQueue.h | 2 +- include/libkern/OSSpinLockDeprecated.h | 3 +- include/os/base.h | 6 + include/os/lock.h | 2 + internal/os/internal.h | 50 +-- man/atomic.3 | 1 + man/atomic_deprecated.3 | 6 +- man/manpages.lst | 2 + man/stdatomic.3 | 274 ++++++++++++++ {include => private}/_simple.h | 22 +- private/libkern/module.modulemap | 7 +- {include => private}/os/alloc_once_impl.h | 0 {include => private}/os/base_private.h | 0 {include => private}/os/internal/atomic.h | 25 -- {include => private}/os/internal/crashlog.h | 0 .../os/internal/internal_shared.h | 1 + src/os/lock_wfe.c => private/os/lock.h | 19 +- {include => private}/os/lock_private.h | 57 ++- {include => private}/os/once_private.h | 0 {include => private}/os/semaphore_private.h | 0 {include => private}/platform/compat.h | 0 .../platform/introspection_private.h | 0 {include => private}/platform/string.h | 0 src/cachecontrol/arm64/cache.s | 53 ++- src/init.c | 10 +- src/os/alloc_once.c | 8 +- src/os/atomic.c | 292 +++++++++++---- src/os/lock.c | 349 +++++------------- src/os/lock_internal.h | 2 - src/os/resolver.c | 106 ------ src/os/resolver.h | 126 ------- src/os/{lock_up.c => resolver/resolver.h} | 12 +- .../resolver_internal.h} | 14 +- src/os/semaphore.c | 8 +- src/simple/asl.c | 12 +- src/simple/string_io.c | 20 +- xcodeconfig/libplatform.aliases | 2 + xcodeconfig/libplatform.xcconfig | 7 +- xcodeconfig/os.xcconfig | 4 +- 40 files changed, 742 insertions(+), 762 deletions(-) create mode 100644 .upstream_base_commits create mode 100644 man/stdatomic.3 rename {include => private}/_simple.h (91%) rename {include => private}/os/alloc_once_impl.h (100%) rename {include => private}/os/base_private.h (100%) rename {include => private}/os/internal/atomic.h (91%) rename {include => private}/os/internal/crashlog.h (100%) rename {include => private}/os/internal/internal_shared.h (99%) rename src/os/lock_wfe.c => private/os/lock.h (65%) rename {include => private}/os/lock_private.h (96%) rename {include => private}/os/once_private.h (100%) rename {include => private}/os/semaphore_private.h (100%) rename {include => private}/platform/compat.h (100%) rename {include => private}/platform/introspection_private.h (100%) rename {include => private}/platform/string.h (100%) delete mode 100644 src/os/resolver.c delete mode 100644 src/os/resolver.h rename src/os/{lock_up.c => resolver/resolver.h} (80%) rename src/os/{atomic_up.c => resolver/resolver_internal.h} (67%) diff --git a/.upstream_base_commits b/.upstream_base_commits new file mode 100644 index 0000000..ebe5327 --- /dev/null +++ b/.upstream_base_commits @@ -0,0 +1,2 @@ +#freebsd = https://github.com/freebsd/freebsd.git +man/stdatomic.3 freebsd share/man/man3/ATOMIC_VAR_INIT.3 10701d40cc37d89e1a105f5fc5bf6c4a21fabf67 diff --git a/include/libkern/OSAtomicQueue.h b/include/libkern/OSAtomicQueue.h index 8ffa4be..103f1e8 100644 --- a/include/libkern/OSAtomicQueue.h +++ b/include/libkern/OSAtomicQueue.h @@ -42,7 +42,7 @@ __BEGIN_DECLS You should always initialize a queue head structure with the initialization vector {@link OS_ATOMIC_QUEUE_INIT} before use. */ -#if defined(__x86_64__) +#if defined(__LP64__) typedef volatile struct { void *opaque1; diff --git a/include/libkern/OSSpinLockDeprecated.h b/include/libkern/OSSpinLockDeprecated.h index 68b64cd..f22b141 100644 --- a/include/libkern/OSSpinLockDeprecated.h +++ b/include/libkern/OSSpinLockDeprecated.h @@ -182,7 +182,8 @@ __BEGIN_DECLS #define OS_SPINLOCK_INIT 0 typedef int32_t OSSpinLock OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock); -typedef volatile OSSpinLock *_os_nospin_lock_t; +typedef volatile OSSpinLock *_os_nospin_lock_t + OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_t); OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_lock) OS_NOSPIN_LOCK_AVAILABILITY diff --git a/include/os/base.h b/include/os/base.h index 77ab213..9fa30e4 100644 --- a/include/os/base.h +++ b/include/os/base.h @@ -122,6 +122,12 @@ #define OS_NOESCAPE #endif +#if defined(__cplusplus) && defined(__clang__) +#define OS_FALLTHROUGH [[clang::fallthrough]] +#else +#define OS_FALLTHROUGH +#endif + #if __has_feature(assume_nonnull) #define OS_ASSUME_NONNULL_BEGIN _Pragma("clang assume_nonnull begin") #define OS_ASSUME_NONNULL_END _Pragma("clang assume_nonnull end") diff --git a/include/os/lock.h b/include/os/lock.h index 4af8449..abc5109 100644 --- a/include/os/lock.h +++ b/include/os/lock.h @@ -79,6 +79,7 @@ typedef struct os_unfair_lock_s { uint32_t _os_unfair_lock_opaque; } os_unfair_lock, *os_unfair_lock_t; +#ifndef OS_UNFAIR_LOCK_INIT #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L #define OS_UNFAIR_LOCK_INIT ((os_unfair_lock){0}) #elif defined(__cplusplus) && __cplusplus >= 201103L @@ -88,6 +89,7 @@ typedef struct os_unfair_lock_s { #else #define OS_UNFAIR_LOCK_INIT {0} #endif +#endif // OS_UNFAIR_LOCK_INIT /*! * @function os_unfair_lock_lock diff --git a/internal/os/internal.h b/internal/os/internal.h index 4d7a083..1784216 100644 --- a/internal/os/internal.h +++ b/internal/os/internal.h @@ -52,47 +52,16 @@ #define __LIBPLATFORM_INTERNAL_CRASH__(rc, msg) \ __OS_CRASH__(rc, "BUG IN LIBPLATFORM: " msg) - #define __OS_EXPOSE_INTERNALS__ 1 #include "os/internal/internal_shared.h" #include "yield.h" -#if !VARIANT_NO_RESOLVERS -#if defined(_ARM_ARCH_7) && !defined(__ARM_ARCH_7S__) -#if OS_ATOMIC_UP -#define OS_VARIANT_SELECTOR up -#else -#define OS_VARIANT_SELECTOR mp -#endif -#endif -#if !defined(OS_VARIANT_SELECTOR) && defined(VARIANT_NO_RESOLVERS) -// forced up variant for no-barrier OSAtomics -#define OS_ATOMIC_NO_BARRIER_ONLY 1 -#define OS_VARIANT_SELECTOR up -#endif -#if (defined(_ARM_ARCH_7) || defined(__arm64__)) && \ - (!defined(OS_ATOMIC_WFE) && !OS_ATOMIC_UP) -#define OS_ATOMIC_WFE 0 -#endif -#ifdef OS_ATOMIC_WFE -#if OS_ATOMIC_WFE -#define OS_LOCK_VARIANT_SELECTOR wfe -#else -#define OS_LOCK_VARIANT_SELECTOR mp -#endif -#endif -#endif // !VARIANT_NO_RESOLVERS +#define OS_NOEXPORT extern __attribute__((__visibility__("hidden"))) #define OS_VARIANT(f, v) OS_CONCAT(f, OS_CONCAT($VARIANT$, v)) -#ifdef OS_VARIANT_SELECTOR -#define _OS_ATOMIC_ALIAS_PRIVATE_EXTERN(n) \ - ".private_extern _" OS_STRINGIFY(n) "\n\t" -#define OS_ATOMIC_EXPORT -#else #define _OS_ATOMIC_ALIAS_PRIVATE_EXTERN(n) #define OS_ATOMIC_EXPORT OS_EXPORT -#endif #define _OS_ATOMIC_ALIAS_GLOBL(n) \ ".globl _" OS_STRINGIFY(n) "\n\t" #ifdef __thumb__ @@ -122,21 +91,4 @@ __VA_ARGS__ \ } -#define _OS_VARIANT_UPMP_RESOLVER(s, v) \ - _OS_VARIANT_RESOLVER(s, v, \ - uint32_t *_c = (void*)(uintptr_t)_COMM_PAGE_CPU_CAPABILITIES; \ - if (*_c & kUP) { \ - extern void OS_VARIANT(s, up)(void); \ - return &OS_VARIANT(s, up); \ - } else { \ - extern void OS_VARIANT(s, mp)(void); \ - return &OS_VARIANT(s, mp); \ - }) - -#define OS_VARIANT_UPMP_RESOLVER(s) \ - _OS_VARIANT_UPMP_RESOLVER(s, default) - -#define OS_VARIANT_UPMP_RESOLVER_INTERNAL(s) \ - _OS_VARIANT_UPMP_RESOLVER(s, hidden) - #endif // __OS_INTERNAL_H__ diff --git a/man/atomic.3 b/man/atomic.3 index eee6e43..8a8d246 100644 --- a/man/atomic.3 +++ b/man/atomic.3 @@ -56,6 +56,7 @@ will return a ptr to mary. .Sh RETURN VALUES The dequeue operation returns the most recently enqueued element, or NULL if the list in empty. .Sh SEE ALSO +.Xr stdatomic 3 , .Xr atomic_deprecated 3 , .Xr spinlock_deprecated 3 .Sh HISTORY diff --git a/man/atomic_deprecated.3 b/man/atomic_deprecated.3 index 8db9557..138da5c 100644 --- a/man/atomic_deprecated.3 +++ b/man/atomic_deprecated.3 @@ -1,4 +1,4 @@ -.Dd May 26, 2004 +.Dd Mar 7, 2016 .Dt ATOMIC_DEPRECATED 3 .Os Darwin .Sh NAME @@ -132,7 +132,8 @@ These are deprecated interfaces for atomic and synchronization operations, provided for compatibility with legacy code. New code should use the C11 .In stdatomic.h -interfaces. +interfaces described in +.Xr stdatomic 3 . .Ef .Pp These functions are thread and multiprocessor safe. For each function, there @@ -224,6 +225,7 @@ The compare-and-swap operations return true if the comparison was equal, ie if the swap occured. The bit test and set/clear operations return the original value of the bit. .Sh SEE ALSO +.Xr stdatomic 3 , .Xr atomic 3 , .Xr spinlock_deprecated 3 .Sh HISTORY diff --git a/man/manpages.lst b/man/manpages.lst index 898750d..dd74ea3 100644 --- a/man/manpages.lst +++ b/man/manpages.lst @@ -11,3 +11,5 @@ makecontext.3 makecontext.3 swapcontext.3 setjmp.3 setjmp.3 _longjmp.3 _setjmp.3 longjmp.3 longjmperr.3 longjmperror.3 siglongjmp.3 sigsetjmp.3 spinlock_deprecated.3 spinlock_deprecated.3 OSSpinLockLock.3 OSSpinLockTry.3 OSSpinLockUnlock.3 ucontext.3 ucontext.3 +stdatomic.3 stdatomic.3 ATOMIC_VAR_INIT.3 atomic_compare_exchange_strong.3 atomic_compare_exchange_strong_explicit.3 atomic_compare_exchange_weak.3 atomic_compare_exchange_weak_explicit.3 atomic_exchange.3 atomic_exchange_explicit.3 atomic_fetch_add.3 atomic_fetch_add_explicit.3 atomic_fetch_and.3 atomic_fetch_and_explicit.3 atomic_fetch_or.3 atomic_fetch_or_explicit.3 atomic_fetch_sub.3 atomic_fetch_sub_explicit.3 atomic_fetch_xor.3 atomic_fetch_xor_explicit.3 atomic_init.3 atomic_is_lock_free.3 atomic_load.3 atomic_load_explicit.3 atomic_store.3 atomic_store_explicit.3 + diff --git a/man/stdatomic.3 b/man/stdatomic.3 new file mode 100644 index 0000000..b4ae319 --- /dev/null +++ b/man/stdatomic.3 @@ -0,0 +1,274 @@ +.\" Copyright (c) 2011 Ed Schouten +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd December 27, 2011 +.Dt stdatomic 3 +.Os +.Sh NAME +.Nm ATOMIC_VAR_INIT , +.Nm atomic_init , +.Nm atomic_load , +.Nm atomic_store , +.Nm atomic_exchange , +.Nm atomic_compare_exchange_strong , +.Nm atomic_compare_exchange_weak , +.Nm atomic_fetch_add , +.Nm atomic_fetch_and , +.Nm atomic_fetch_or , +.Nm atomic_fetch_sub , +.Nm atomic_fetch_xor , +.Nm atomic_is_lock_free +.Nd type-generic atomic operations +.Sh SYNOPSIS +.In stdatomic.h +.Pp +.Vt _Atomic(T) Va v No = \*[Fn-font]ATOMIC_VAR_INIT\*[No-font] Ns Pq Fa c ; +.Vt _Atomic T Va v No = \*[Fn-font]ATOMIC_VAR_INIT\*[No-font] Ns Pq Fa c ; +.Ft void +.Fn atomic_init "_Atomic(T) *object" "T value" +.Ft T +.Fn atomic_load "_Atomic(T) *object" +.Ft T +.Fn atomic_load_explicit "_Atomic(T) *object" "memory_order order" +.Ft void +.Fn atomic_store "_Atomic(T) *object" "T desired" +.Ft void +.Fn atomic_store_explicit "_Atomic(T) *object" "T desired" "memory_order order" +.Ft T +.Fn atomic_exchange "_Atomic(T) *object" "T desired" +.Ft T +.Fn atomic_exchange_explicit "_Atomic(T) *object" "T desired" "memory_order order" +.Ft _Bool +.Fn atomic_compare_exchange_strong "_Atomic(T) *object" "T *expected" "T desired" +.Ft _Bool +.Fn atomic_compare_exchange_strong_explicit "_Atomic(T) *object" "T *expected" "T desired" "memory_order success" "memory_order failure" +.Ft _Bool +.Fn atomic_compare_exchange_weak "_Atomic(T) *object" "T *expected" "T desired" +.Ft _Bool +.Fn atomic_compare_exchange_weak_explicit "_Atomic(T) *object" "T *expected" "T desired" "memory_order success" "memory_order failure" +.Ft T +.Fn atomic_fetch_add "_Atomic(T) *object" "T operand" +.Ft T +.Fn atomic_fetch_add_explicit "_Atomic(T) *object" "T operand" "memory_order order" +.Ft T +.Fn atomic_fetch_and "_Atomic(T) *object" "T operand" +.Ft T +.Fn atomic_fetch_and_explicit "_Atomic(T) *object" "T operand" "memory_order order" +.Ft T +.Fn atomic_fetch_or "_Atomic(T) *object" "T operand" +.Ft T +.Fn atomic_fetch_or_explicit "_Atomic(T) *object" "T operand" "memory_order order" +.Ft T +.Fn atomic_fetch_sub "_Atomic(T) *object" "T operand" +.Ft T +.Fn atomic_fetch_sub_explicit "_Atomic(T) *object" "T operand" "memory_order order" +.Ft T +.Fn atomic_fetch_xor "_Atomic(T) *object" "T operand" +.Ft T +.Fn atomic_fetch_xor_explicit "_Atomic(T) *object" "T operand" "memory_order order" +.Ft _Bool +.Fn atomic_is_lock_free "const _Atomic(T) *object" +.Sh DESCRIPTION +The header +.In stdatomic.h +provides type-generic operations on atomic operations. +.Pp +Atomic variables are declared using the +.Vt _Atomic() +type specifier or the +.Vt _Atomic +type qualifier. +Such variables are not type-compatible with their non-atomic +counterparts and may have different alignment. +.Pp +Operations on atomic variables that do not use the +.Fn atomic_ +interfaces, including compound assignment operations, will behave as if the +.Pf non- Fn _explicit +versions of those interfaces had been used. +.Pp +The +.Fn atomic_init +operation initializes the atomic variable +.Fa object +with +.Fa value . +Atomic variables can be initialized while being declared using +.Fn ATOMIC_VAR_INIT . +.Pp +The +.Fn atomic_load +operation returns the value of atomic variable +.Fa object . +The +.Fn atomic_store +operation sets the atomic variable +.Fa object +to the +.Fa desired +value. +.Pp +The +.Fn atomic_exchange +operation combines the behaviour of +.Fn atomic_load +and +.Fn atomic_store . +It sets the atomic variable +.Fa object +to the desired +.Fa value +and returns the original contents of the atomic variable. +.Pp +The +.Fn atomic_compare_exchange_strong +operation stores the +.Fa desired +value into atomic variable +.Fa object , +but only if the atomic variable is equal to the +.Fa expected +value. +Upon success, the operation returns +.Dv true . +Upon failure, the +.Fa expected +value is overwritten with the contents of the atomic variable and +.Dv false +is returned. +.Pp +The +.Fn atomic_compare_exchange_weak +operation is identical to +.Fn atomic_compare_exchange_strong , +but is allowed to fail even if atomic variable +.Fa object +is equal to the +.Fa expected +value. When an +.Fn atomic_compare_exchange +operation is in a loop, the weak version will yield better performance on +some platforms. When +.Fn atomic_compare_exchange_weak +would require a loop and +.Fn atomic_compare_exchange_strong +would not, the strong version is preferable. +.Pp +The +.Fn atomic_fetch_add +operation adds the value +.Fa operand +to atomic variable +.Fa object +and returns the original contents of the atomic variable. +.Pp +The +.Fn atomic_fetch_and +operation applies the +.Em and +operator to atomic variable +.Fa object +and value +.Fa operand +and stores the result into +.Fa object , +while returning the original contents of the atomic variable. +.Pp +The +.Fn atomic_fetch_or +operation applies the +.Em or +operator to atomic variable +.Fa object +and value +.Fa operand +and stores the result into +.Fa object , +while returning the original contents of the atomic variable. +.Pp +The +.Fn atomic_fetch_sub +operation subtracts the value +.Fa operand +from atomic variable +.Fa object +and returns the original contents of the atomic variable. +.Pp +The +.Fn atomic_fetch_xor +operation applies the +.Em xor +operator to atomic variable +.Fa object +and value +.Fa operand +and stores the result into +.Fa object , +while returning the original contents of the atomic variable. +.Pp +The +.Fn atomic_is_lock_free +operation returns whether atomic variable +.Fa object +uses locks to implement atomic operations. +.Sh MEMORY ORDER +C11 defines a memory model that may allow for the reordering of operations in the +absence of fences or explicit memory ordering operations. +The +.Pf non- Fn _explicit +interfaces use the strictest available memory order: sequential +consistency. The +.Fn _explicit +interfaces allow for configuration of the memory order operation which is +present. The types of available memory order operations are explained in +more detail in +.St -isoC-2011 . +.Pp +The +.Fa order +parameter of the +.Fn _explicit +interfaces can have one of the following values: +.Bl -tag -width memory_order_relaxed +.It Dv memory_order_relaxed +Operation does not order memory. +.It Dv memory_order_consume +Performs a consume operation. +.It Dv memory_order_acquire +Performs an acquire operation. +.It Dv memory_order_release +Performs a release operation. +.It Dv memory_order_acq_rel +Performs both an acquire and a release operation. +.It Dv memory_order_seq_cst +Provides sequential consistency. +.El +.Sh SEE ALSO +.Xr atomic 3 , +.Xr pthread 3 +.Sh STANDARDS +These interfaces conform to +.St -isoC-2011 . diff --git a/include/_simple.h b/private/_simple.h similarity index 91% rename from include/_simple.h rename to private/_simple.h index b3bd924..2ae8384 100644 --- a/include/_simple.h +++ b/private/_simple.h @@ -38,14 +38,14 @@ __BEGIN_DECLS * arguments from the va_list, and the results are written to the given * file descriptor. */ -void _simple_vdprintf(int __fd, const char *__fmt, va_list __ap); +void _simple_vdprintf(int __fd, const char *__fmt, va_list __ap) __printflike(2, 0); /* * A simplified fprintf variant. The format string is interpreted with * arguments from the variable argument list, and the results are written * to the given file descriptor. */ -void _simple_dprintf(int __fd, const char *__fmt, ...); +void _simple_dprintf(int __fd, const char *__fmt, ...) __printflike(2, 3); /* * A simplified string allocate routine. Pass the opaque pointer to structure @@ -59,32 +59,32 @@ _SIMPLE_STRING _simple_salloc(void); /* * The format string is interpreted with arguments from the va_list, and the * results are appended to the string maintained by the opaque structure, as - * returned by a previous call to _simple_salloc(). - * Always returns 0 on OS X >= 10.12 and iOS >= 10.0 + * returned by a previous call to _simple_salloc(). Non-zero is returned on + * out-of-memory error. */ -int _simple_vsprintf(_SIMPLE_STRING __b, const char *__fmt, va_list __ap); +int _simple_vsprintf(_SIMPLE_STRING __b, const char *__fmt, va_list __ap) __printflike(2, 0); /* * The format string is interpreted with arguments from the variable argument * list, and the results are appended to the string maintained by the opaque - * structure, as returned by a previous call to _simple_salloc(). - * Always returns 0 on OS X >= 10.12 and iOS >= 10.0 + * structure, as returned by a previous call to _simple_salloc(). Non-zero is + * returned on out-of-memory error. */ -int _simple_sprintf(_SIMPLE_STRING __b, const char *__fmt, ...); +int _simple_sprintf(_SIMPLE_STRING __b, const char *__fmt, ...) __printflike(2, 3); /* * Like _simple_vsprintf(), except __esc is a function to call on each * character; the function returns NULL if the character should be passed * as is, otherwise, the returned character string is used instead. */ -int _simple_vesprintf(_SIMPLE_STRING __b, _esc_func __esc, const char *__fmt, va_list __ap); +int _simple_vesprintf(_SIMPLE_STRING __b, _esc_func __esc, const char *__fmt, va_list __ap) __printflike(3, 0); /* * Like _simple_sprintf(), except __esc is a function to call on each * character; the function returns NULL if the character should be passed * as is, otherwise, the returned character string is used instead. */ -int _simple_esprintf(_SIMPLE_STRING __b, _esc_func __esc, const char *__fmt, ...); +int _simple_esprintf(_SIMPLE_STRING __b, _esc_func __esc, const char *__fmt, ...) __printflike(3, 4); /* * Return the null terminated string from the opaque structure, as returned @@ -100,7 +100,7 @@ void _simple_sresize(_SIMPLE_STRING __b); /* * Append the null-terminated string to the string associated with the opaque - * structure. Always returns 0 on OS X >= 10.12 and iOS >= 10.0 + * structure. Non-zero is returned on out-of-memory error. */ int _simple_sappend(_SIMPLE_STRING __b, const char *__str); diff --git a/private/libkern/module.modulemap b/private/libkern/module.modulemap index 72b650c..8b13789 100644 --- a/private/libkern/module.modulemap +++ b/private/libkern/module.modulemap @@ -1,6 +1 @@ -module libkernPrivate [system] [extern_c] { - module OSAtomic { - header "OSAtomic.h" - export Darwin.libkern.OSAtomic - } -} + diff --git a/include/os/alloc_once_impl.h b/private/os/alloc_once_impl.h similarity index 100% rename from include/os/alloc_once_impl.h rename to private/os/alloc_once_impl.h diff --git a/include/os/base_private.h b/private/os/base_private.h similarity index 100% rename from include/os/base_private.h rename to private/os/base_private.h diff --git a/include/os/internal/atomic.h b/private/os/internal/atomic.h similarity index 91% rename from include/os/internal/atomic.h rename to private/os/internal/atomic.h index 9b3294d..1d717a6 100644 --- a/include/os/internal/atomic.h +++ b/private/os/internal/atomic.h @@ -162,9 +162,6 @@ typedef enum _os_atomic_memory_order { #pragma mark generic #define os_atomic_thread_fence(m) _os_atomic_barrier(m) -// see comment in os_once.c -#define os_atomic_maximally_synchronizing_barrier() \ - _os_atomic_barrier(seq_cst) #define os_atomic_load2o(p, f, m) \ os_atomic_load(&(p)->f, m) @@ -246,28 +243,6 @@ typedef enum _os_atomic_memory_order { #pragma mark - #pragma mark x86 -#undef os_atomic_maximally_synchronizing_barrier -#ifdef __LP64__ -#define os_atomic_maximally_synchronizing_barrier() \ - ({ unsigned long _clbr; __asm__ __volatile__( \ - "cpuid" \ - : "=a" (_clbr) : "0" (0) : "rbx", "rcx", "rdx", "cc", "memory"); }) -#else -#ifdef __llvm__ -#define os_atomic_maximally_synchronizing_barrier() \ - ({ unsigned long _clbr; __asm__ __volatile__( \ - "cpuid" \ - : "=a" (_clbr) : "0" (0) : "ebx", "ecx", "edx", "cc", "memory"); }) -#else // gcc does not allow inline i386 asm to clobber ebx -#define os_atomic_maximally_synchronizing_barrier() \ - ({ unsigned long _clbr; __asm__ __volatile__( \ - "pushl %%ebx\n\t" \ - "cpuid\n\t" \ - "popl %%ebx" \ - : "=a" (_clbr) : "0" (0) : "ecx", "edx", "cc", "memory"); }) -#endif -#endif - #endif diff --git a/include/os/internal/crashlog.h b/private/os/internal/crashlog.h similarity index 100% rename from include/os/internal/crashlog.h rename to private/os/internal/crashlog.h diff --git a/include/os/internal/internal_shared.h b/private/os/internal/internal_shared.h similarity index 99% rename from include/os/internal/internal_shared.h rename to private/os/internal/internal_shared.h index 732c420..c0e4e81 100644 --- a/include/os/internal/internal_shared.h +++ b/private/os/internal/internal_shared.h @@ -53,6 +53,7 @@ #include #include + #endif // __OS_EXPOSE_INTERNALS__ #endif // __OS_INTERNAL_SHARED__ diff --git a/src/os/lock_wfe.c b/private/os/lock.h similarity index 65% rename from src/os/lock_wfe.c rename to private/os/lock.h index 36dd190..74bc470 100644 --- a/src/os/lock_wfe.c +++ b/private/os/lock.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Apple Inc. All rights reserved. + * Copyright (c) 2017 Apple Inc. All rights reserved. * * @APPLE_APACHE_LICENSE_HEADER_START@ * @@ -18,19 +18,10 @@ * @APPLE_APACHE_LICENSE_HEADER_END@ */ -#include +#ifndef __OS_LOCK_PRIVATE_UNFAIR_LOCK_INIT__ +#define __OS_LOCK_PRIVATE_UNFAIR_LOCK_INIT__ -#if TARGET_OS_EMBEDDED -#define OS_ATOMIC_WFE 1 -#include "lock_internal.h" +#include_next -#ifdef OS_LOCK_VARIANT_SELECTOR -#define OS_VARIANT_ONLY 1 -#define OS_LOCK_VARIANT_ONLY 1 -#include "lock.c" -#endif - -#endif // TARGET_OS_EMBEDDED - -struct _os_empty_files_are_not_c_files; +#endif // __OS_LOCK_PRIVATE_UNFAIR_LOCK_INIT__ diff --git a/include/os/lock_private.h b/private/os/lock_private.h similarity index 96% rename from include/os/lock_private.h rename to private/os/lock_private.h index 8c32ea6..00329e2 100644 --- a/include/os/lock_private.h +++ b/private/os/lock_private.h @@ -391,16 +391,24 @@ OS_UNFAIR_LOCK_AVAILABILITY OS_EXPORT OS_NOTHROW OS_NONNULL_ALL void os_unfair_lock_assert_not_owner(os_unfair_lock_t lock); -/*! @group os_unfair_lock variant for consumption by Libc +/*! @group os_unfair_lock no-TSD interfaces + * + * Like the above, but don't require being on a thread with valid TSD, so they + * can be called from injected mach-threads. The normal routines use the TSD + * value for mach_thread_self(), these routines use MACH_PORT_DEAD for the + * locked value instead. As a result, they will be unable to resolve priority + * inversions. + * + * This should only be used by libpthread. + * */ OS_UNFAIR_LOCK_AVAILABILITY OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -void os_unfair_lock_lock_with_options_4Libc(os_unfair_lock_t lock, - os_unfair_lock_options_t options); +void os_unfair_lock_lock_no_tsd_4libpthread(os_unfair_lock_t lock); OS_UNFAIR_LOCK_AVAILABILITY OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -void os_unfair_lock_unlock_4Libc(os_unfair_lock_t lock); +void os_unfair_lock_unlock_no_tsd_4libpthread(os_unfair_lock_t lock); __END_DECLS @@ -450,6 +458,16 @@ typedef _Atomic(os_unfair_lock) _os_atomic_unfair_lock; OS_ASSUME_NONNULL_BEGIN +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#define OS_UNFAIR_LOCK_UNLOCKED ((os_unfair_lock){0}) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#define OS_UNFAIR_LOCK_UNLOCKED (os_unfair_lock{}) +#elif defined(__cplusplus) +#define OS_UNFAIR_LOCK_UNLOCKED (os_unfair_lock()) +#else +#define OS_UNFAIR_LOCK_UNLOCKED {0} +#endif + /*! * @function os_unfair_lock_lock_inline * @@ -467,7 +485,7 @@ os_unfair_lock_lock_inline(os_unfair_lock_t lock) if (!_pthread_has_direct_tsd()) return os_unfair_lock_lock(lock); uintptr_t mts = (uintptr_t)_pthread_getspecific_direct( _PTHREAD_TSD_SLOT_MACH_THREAD_SELF); - os_unfair_lock unlocked = OS_UNFAIR_LOCK_INIT, locked = { mts }; + os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts }; if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( (_os_atomic_unfair_lock*)lock, &unlocked, locked, OSLOCK_STD(memory_order_acquire), @@ -499,7 +517,7 @@ os_unfair_lock_lock_with_options_inline(os_unfair_lock_t lock, } uintptr_t mts = (uintptr_t)_pthread_getspecific_direct( _PTHREAD_TSD_SLOT_MACH_THREAD_SELF); - os_unfair_lock unlocked = OS_UNFAIR_LOCK_INIT, locked = { mts }; + os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts }; if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( (_os_atomic_unfair_lock*)lock, &unlocked, locked, OSLOCK_STD(memory_order_acquire), @@ -534,7 +552,7 @@ os_unfair_lock_trylock_inline(os_unfair_lock_t lock) if (!_pthread_has_direct_tsd()) return os_unfair_lock_trylock(lock); uintptr_t mts = (uintptr_t)_pthread_getspecific_direct( _PTHREAD_TSD_SLOT_MACH_THREAD_SELF); - os_unfair_lock unlocked = OS_UNFAIR_LOCK_INIT, locked = { mts }; + os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts }; return OSLOCK_STD(atomic_compare_exchange_strong_explicit)( (_os_atomic_unfair_lock*)lock, &unlocked, locked, OSLOCK_STD(memory_order_acquire), OSLOCK_STD(memory_order_relaxed)); @@ -557,7 +575,7 @@ os_unfair_lock_unlock_inline(os_unfair_lock_t lock) if (!_pthread_has_direct_tsd()) return os_unfair_lock_unlock(lock); uintptr_t mts = (uintptr_t)_pthread_getspecific_direct( _PTHREAD_TSD_SLOT_MACH_THREAD_SELF); - os_unfair_lock unlocked = OS_UNFAIR_LOCK_INIT, locked = { mts }; + os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts }; if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( (_os_atomic_unfair_lock*)lock, &locked, unlocked, OSLOCK_STD(memory_order_release), @@ -566,25 +584,6 @@ os_unfair_lock_unlock_inline(os_unfair_lock_t lock) } } -/*! @group os_unfair_lock no-TSD interfaces - * - * Like the above, but don't require being on a thread with valid TSD, so they - * can be called from injected mach-threads. The normal routines use the TSD - * value for mach_thread_self(), these routines use MACH_PORT_DEAD for the - * locked value instead. As a result, they will be unable to resolve priority - * inversions. - * - * This should only be used by libpthread. - * - */ -OS_UNFAIR_LOCK_AVAILABILITY -OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -void os_unfair_lock_lock_no_tsd_4libpthread(os_unfair_lock_t lock); - -OS_UNFAIR_LOCK_AVAILABILITY -OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -void os_unfair_lock_unlock_no_tsd_4libpthread(os_unfair_lock_t lock); - /*! * @function os_unfair_lock_lock_inline_no_tsd_4libpthread * @@ -602,7 +601,7 @@ void os_unfair_lock_lock_inline_no_tsd_4libpthread(os_unfair_lock_t lock) { uintptr_t mts = (uintptr_t)MACH_PORT_DEAD; - os_unfair_lock unlocked = OS_UNFAIR_LOCK_INIT, locked = { mts }; + os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts }; if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( (_os_atomic_unfair_lock*)lock, &unlocked, locked, OSLOCK_STD(memory_order_acquire), @@ -628,7 +627,7 @@ void os_unfair_lock_unlock_inline_no_tsd_4libpthread(os_unfair_lock_t lock) { uintptr_t mts = (uintptr_t)MACH_PORT_DEAD; - os_unfair_lock unlocked = OS_UNFAIR_LOCK_INIT, locked = { mts }; + os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts }; if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( (_os_atomic_unfair_lock*)lock, &locked, unlocked, OSLOCK_STD(memory_order_release), diff --git a/include/os/once_private.h b/private/os/once_private.h similarity index 100% rename from include/os/once_private.h rename to private/os/once_private.h diff --git a/include/os/semaphore_private.h b/private/os/semaphore_private.h similarity index 100% rename from include/os/semaphore_private.h rename to private/os/semaphore_private.h diff --git a/include/platform/compat.h b/private/platform/compat.h similarity index 100% rename from include/platform/compat.h rename to private/platform/compat.h diff --git a/include/platform/introspection_private.h b/private/platform/introspection_private.h similarity index 100% rename from include/platform/introspection_private.h rename to private/platform/introspection_private.h diff --git a/include/platform/string.h b/private/platform/string.h similarity index 100% rename from include/platform/string.h rename to private/platform/string.h diff --git a/src/cachecontrol/arm64/cache.s b/src/cachecontrol/arm64/cache.s index d59afdd..000e581 100644 --- a/src/cachecontrol/arm64/cache.s +++ b/src/cachecontrol/arm64/cache.s @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2011-2017 Apple Computer, Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * @@ -23,21 +23,52 @@ #include -/* void sys_icache_invalidate(addr_t start, int length) */ +#define MMU_I_CLINE 6 // cache line size as 1< 0 ? + and x8, x0, #~((1<> MMU_I_CLINE) + 1; + while (count--) { + asm("ic ivau, %[addr]" :: [addr] "r" (addr) : "memory"); + addr += (1 << MMU_I_CLINE); + } + asm volatile("dsb ish\n\tisb" ::: "memory"); +} +#endif + diff --git a/src/init.c b/src/init.c index 7e8bc6a..7f43e47 100644 --- a/src/init.c +++ b/src/init.c @@ -21,21 +21,21 @@ * @APPLE_LICENSE_HEADER_END@ */ -#include #include +#include struct ProgramVars; /* forward reference */ extern void _simple_asl_init(const char *envp[], const struct ProgramVars *vars); extern void __pfz_setup(const char *apple[]); - -void __libplatform_init(void *future_use __unused, const char *envp[], const char *apple[], const struct ProgramVars *vars) { - +void +__libplatform_init(void *future_use __unused, const char *envp[], + const char *apple[], const struct ProgramVars *vars) +{ /* In the Simulator, we just provide _simple for dyld */ #if !TARGET_IPHONE_SIMULATOR __pfz_setup(apple); #endif _simple_asl_init(envp, vars); - } diff --git a/src/os/alloc_once.c b/src/os/alloc_once.c index a8ff358..e1ee7c5 100644 --- a/src/os/alloc_once.c +++ b/src/os/alloc_once.c @@ -27,6 +27,7 @@ */ #include "os/internal.h" +#include "resolver.h" #include "os/alloc_once_impl.h" #include #include @@ -42,7 +43,12 @@ typedef struct _os_alloc_heap_metadata_s { #define allocation_size (2 * vm_page_size) #define usable (allocation_size-sizeof(_os_alloc_heap_metadata_s)) -static void * volatile _os_alloc_heap; +OS_NOEXPORT void * volatile _os_alloc_heap; + +OS_ATOMIC_EXPORT void* _os_alloc_once(struct _os_alloc_once_s *slot, size_t sz, + os_function_t init); + +void * volatile _os_alloc_heap; /* * Simple allocator that doesn't have to worry about ever freeing allocations. diff --git a/src/os/atomic.c b/src/os/atomic.c index 6c846f6..180c090 100644 --- a/src/os/atomic.c +++ b/src/os/atomic.c @@ -19,42 +19,86 @@ */ #include "os/internal.h" -#include "libkern/OSAtomic.h" #include "resolver.h" +#include "libkern/OSAtomic.h" #if TARGET_OS_EMBEDDED +OS_ATOMIC_EXPORT +int32_t OSAtomicAdd32(int32_t v, volatile int32_t *p); OS_ATOMIC_EXPORT int32_t OSAtomicAdd32Barrier(int32_t v, volatile int32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicIncrement32(volatile int32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicIncrement32Barrier(volatile int32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicDecrement32(volatile int32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicDecrement32Barrier(volatile int32_t *p); OS_ATOMIC_EXPORT +int64_t OSAtomicAdd64(int64_t v, volatile int64_t *p); +OS_ATOMIC_EXPORT int64_t OSAtomicAdd64Barrier(int64_t v, volatile int64_t *p); OS_ATOMIC_EXPORT +int64_t OSAtomicIncrement64(volatile int64_t *p); +OS_ATOMIC_EXPORT int64_t OSAtomicIncrement64Barrier(volatile int64_t *p); OS_ATOMIC_EXPORT +int64_t OSAtomicDecrement64(volatile int64_t *p); +OS_ATOMIC_EXPORT int64_t OSAtomicDecrement64Barrier(volatile int64_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicAnd32(uint32_t v, volatile uint32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicAnd32Barrier(uint32_t v, volatile uint32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicAnd32Orig(uint32_t v, volatile uint32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicAnd32OrigBarrier(uint32_t v, volatile uint32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicOr32(uint32_t v, volatile uint32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicOr32Barrier(uint32_t v, volatile uint32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicOr32Orig(uint32_t v, volatile uint32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicOr32OrigBarrier(uint32_t v, volatile uint32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicXor32(uint32_t v, volatile uint32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicXor32Barrier(uint32_t v, volatile uint32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicXor32Orig(uint32_t v, volatile uint32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicXor32OrigBarrier(uint32_t v, volatile uint32_t *p); OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwap32(int32_t o, int32_t n, volatile int32_t *p); +OS_ATOMIC_EXPORT bool OSAtomicCompareAndSwap32Barrier(int32_t o, int32_t n, volatile int32_t *p); OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwapPtr(void *o, void *n, void * volatile *p); +OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwapPtrBarrier(void *o, void *n, void * volatile *p); +OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwapInt(int o, int n, volatile int *p); +OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwapIntBarrier(int o, int n, volatile int *p); +OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwapLong(long o, long n, volatile long *p); +OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwapLongBarrier(long o, long n, volatile long *p); +OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwap64(int64_t o, int64_t n, volatile int64_t *p); +OS_ATOMIC_EXPORT bool OSAtomicCompareAndSwap64Barrier(int64_t o, int64_t n, volatile int64_t *p); OS_ATOMIC_EXPORT +bool OSAtomicTestAndSet(uint32_t n, volatile void * p); +OS_ATOMIC_EXPORT bool OSAtomicTestAndSetBarrier(uint32_t n, volatile void * p); OS_ATOMIC_EXPORT +bool OSAtomicTestAndClear(uint32_t n, volatile void * p); +OS_ATOMIC_EXPORT bool OSAtomicTestAndClearBarrier(uint32_t n, volatile void * p); OS_ATOMIC_EXPORT void OSAtomicEnqueue(OSQueueHead *list, void *new, size_t offset); @@ -63,139 +107,232 @@ void* OSAtomicDequeue(OSQueueHead *list, size_t offset); OS_ATOMIC_EXPORT void OSMemoryBarrier(void); -#if OS_ATOMIC_UP -#define OS_ATOMIC_ALIAS_NO_BARRIER(n) OS_ATOMIC_EXPORT_ALIAS(n, n##Barrier) -#else -#define OS_ATOMIC_ALIAS_NO_BARRIER(n) -#endif +int32_t +OSAtomicAdd32(int32_t v, volatile int32_t *p) +{ + int32_t r = os_atomic_add(p, v, relaxed); + return r; +} int32_t OSAtomicAdd32Barrier(int32_t v, volatile int32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicAdd32); - int32_t r = os_atomic_add(p, v, acq_rel); + int32_t r = os_atomic_add(p, v, seq_cst); + return r; +} + +int32_t +OSAtomicIncrement32(volatile int32_t *p) +{ + int32_t r = os_atomic_add(p, 1, relaxed); return r; } int32_t OSAtomicIncrement32Barrier(volatile int32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicIncrement32); - int32_t r = os_atomic_add(p, 1, acq_rel); + int32_t r = os_atomic_add(p, 1, seq_cst); + return r; +} + +int32_t +OSAtomicDecrement32(volatile int32_t *p) +{ + int32_t r = os_atomic_add(p, -1, relaxed); return r; } int32_t OSAtomicDecrement32Barrier(volatile int32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicDecrement32); - int32_t r = os_atomic_add(p, -1, acq_rel); + int32_t r = os_atomic_add(p, -1, seq_cst); + return r; +} + +int64_t +OSAtomicAdd64(int64_t v, volatile int64_t *p) +{ + int64_t r = os_atomic_add(p, v, relaxed); return r; } int64_t OSAtomicAdd64Barrier(int64_t v, volatile int64_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicAdd64); - int64_t r = os_atomic_add(p, v, acq_rel); + int64_t r = os_atomic_add(p, v, seq_cst); + return r; +} + +int64_t +OSAtomicIncrement64(volatile int64_t *p) +{ + int64_t r = os_atomic_add(p, 1, relaxed); return r; } int64_t OSAtomicIncrement64Barrier(volatile int64_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicIncrement64); - int64_t r = os_atomic_add(p, 1, acq_rel); + int64_t r = os_atomic_add(p, 1, seq_cst); + return r; +} + +int64_t +OSAtomicDecrement64(volatile int64_t *p) +{ + int64_t r = os_atomic_add(p, -1, relaxed); return r; } int64_t OSAtomicDecrement64Barrier(volatile int64_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicDecrement64); - int64_t r = os_atomic_add(p, -1, acq_rel); + int64_t r = os_atomic_add(p, -1, seq_cst); return r; } +int32_t +OSAtomicAnd32(uint32_t v, volatile uint32_t *p) +{ + uint32_t r = os_atomic_and(p, v, relaxed); + return (int32_t)r; +} + int32_t OSAtomicAnd32Barrier(uint32_t v, volatile uint32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicAnd32); - uint32_t r = os_atomic_and(p, v, acq_rel); + uint32_t r = os_atomic_and(p, v, seq_cst); + return (int32_t)r; +} + +int32_t +OSAtomicAnd32Orig(uint32_t v, volatile uint32_t *p) +{ + uint32_t r = os_atomic_and_orig(p, v, relaxed); return (int32_t)r; } int32_t OSAtomicAnd32OrigBarrier(uint32_t v, volatile uint32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicAnd32Orig); - uint32_t r = os_atomic_and_orig(p, v, acq_rel); + uint32_t r = os_atomic_and_orig(p, v, seq_cst); + return (int32_t)r; +} + +int32_t +OSAtomicOr32(uint32_t v, volatile uint32_t *p) +{ + uint32_t r = os_atomic_or(p, v, relaxed); return (int32_t)r; } int32_t OSAtomicOr32Barrier(uint32_t v, volatile uint32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicOr32); - uint32_t r = os_atomic_or(p, v, acq_rel); + uint32_t r = os_atomic_or(p, v, seq_cst); + return (int32_t)r; +} + +int32_t +OSAtomicOr32Orig(uint32_t v, volatile uint32_t *p) +{ + uint32_t r = os_atomic_or_orig(p, v, relaxed); return (int32_t)r; } int32_t OSAtomicOr32OrigBarrier(uint32_t v, volatile uint32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicOr32Orig); - uint32_t r = os_atomic_or_orig(p, v, acq_rel); + uint32_t r = os_atomic_or_orig(p, v, seq_cst); + return (int32_t)r; +} + +int32_t +OSAtomicXor32(uint32_t v, volatile uint32_t *p) +{ + uint32_t r = os_atomic_xor(p, v, relaxed); return (int32_t)r; } int32_t OSAtomicXor32Barrier(uint32_t v, volatile uint32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicXor32); - uint32_t r = os_atomic_xor(p, v, acq_rel); + uint32_t r = os_atomic_xor(p, v, seq_cst); + return (int32_t)r; +} + +int32_t +OSAtomicXor32Orig(uint32_t v, volatile uint32_t *p) +{ + uint32_t r = os_atomic_xor_orig(p, v, relaxed); return (int32_t)r; } int32_t OSAtomicXor32OrigBarrier(uint32_t v, volatile uint32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicXor32Orig); - uint32_t r = os_atomic_xor_orig(p, v, acq_rel); + uint32_t r = os_atomic_xor_orig(p, v, seq_cst); return (int32_t)r; } +bool +OSAtomicCompareAndSwap32(int32_t o, int32_t n, volatile int32_t *p) +{ + return os_atomic_cmpxchg(p, o, n, relaxed); +} + bool OSAtomicCompareAndSwap32Barrier(int32_t o, int32_t n, volatile int32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwap32); - OS_ATOMIC_ALIAS(OSAtomicCompareAndSwapIntBarrier, - OSAtomicCompareAndSwap32Barrier); - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwapInt); -#ifndef __LP64__ - OS_ATOMIC_ALIAS(OSAtomicCompareAndSwapLongBarrier, - OSAtomicCompareAndSwap32Barrier); - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwapLong); - OS_ATOMIC_ALIAS(OSAtomicCompareAndSwapPtrBarrier, - OSAtomicCompareAndSwap32Barrier); - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwapPtr); -#endif - return os_atomic_cmpxchg(p, o, n, acq_rel); + return os_atomic_cmpxchg(p, o, n, seq_cst); +} + +bool +OSAtomicCompareAndSwapPtr(void *o, void *n, void * volatile *p) +{ + return os_atomic_cmpxchg(p, o, n, relaxed); +} + +bool +OSAtomicCompareAndSwapPtrBarrier(void *o, void *n, void * volatile *p) +{ + return os_atomic_cmpxchg(p, o, n, seq_cst); +} + +bool +OSAtomicCompareAndSwapInt(int o, int n, volatile int *p) +{ + return os_atomic_cmpxchg(p, o, n, relaxed); +} + +bool +OSAtomicCompareAndSwapIntBarrier(int o, int n, volatile int *p) +{ + return os_atomic_cmpxchg(p, o, n, seq_cst); +} + +bool +OSAtomicCompareAndSwapLong(long o, long n, volatile long *p) +{ + return os_atomic_cmpxchg(p, o, n, relaxed); +} + +bool +OSAtomicCompareAndSwapLongBarrier(long o, long n, volatile long *p) +{ + return os_atomic_cmpxchg(p, o, n, seq_cst); +} + +bool +OSAtomicCompareAndSwap64(int64_t o, int64_t n, volatile int64_t *p) +{ + return os_atomic_cmpxchg(p, o, n, relaxed); } bool OSAtomicCompareAndSwap64Barrier(int64_t o, int64_t n, volatile int64_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwap64); -#ifdef __LP64__ - OS_ATOMIC_ALIAS(OSAtomicCompareAndSwapLongBarrier, - OSAtomicCompareAndSwap64Barrier); - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwapLong); - OS_ATOMIC_ALIAS(OSAtomicCompareAndSwapPtrBarrier, - OSAtomicCompareAndSwap64Barrier); - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwapPtr); -#endif - return os_atomic_cmpxchg(p, o, n, acq_rel); + return os_atomic_cmpxchg(p, o, n, seq_cst); } static inline uint32_t* @@ -211,42 +348,60 @@ _OSAtomicTestPtrVal(uint32_t bit, volatile void *addr, uint32_t *vp) return (uint32_t*)((char*)a + 4 * (bit / 32)); } +bool +OSAtomicTestAndSet(uint32_t bit, volatile void *addr) +{ + uint32_t v; + volatile uint32_t *p = _OSAtomicTestPtrVal(bit, addr, &v); + uint32_t r = os_atomic_or_orig(p, v, relaxed); + return (r & v); +} + bool OSAtomicTestAndSetBarrier(uint32_t bit, volatile void *addr) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicTestAndSet); uint32_t v; volatile uint32_t *p = _OSAtomicTestPtrVal(bit, addr, &v); - uint32_t r = os_atomic_or_orig(p, v, acq_rel); + uint32_t r = os_atomic_or_orig(p, v, seq_cst); return (r & v); } bool -OSAtomicTestAndClearBarrier(uint32_t bit, volatile void *addr) +OSAtomicTestAndClear(uint32_t bit, volatile void *addr) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicTestAndClear); uint32_t v; volatile uint32_t *p = _OSAtomicTestPtrVal(bit, addr, &v); - uint32_t r = os_atomic_and_orig(p, ~v, acq_rel); + uint32_t r = os_atomic_and_orig(p, ~v, relaxed); return (r & v); } -#if !OS_ATOMIC_NO_BARRIER_ONLY +bool +OSAtomicTestAndClearBarrier(uint32_t bit, volatile void *addr) +{ + uint32_t v; + volatile uint32_t *p = _OSAtomicTestPtrVal(bit, addr, &v); + uint32_t r = os_atomic_and_orig(p, ~v, seq_cst); + return (r & v); +} -typedef volatile struct { - void * volatile item; - long unused; +typedef struct { + void *item; + long gencount; } _OSQueueHead; + void OSAtomicEnqueue(OSQueueHead *list, void *new, size_t offset) { void * volatile *headptr = &(((_OSQueueHead*)list)->item); void * volatile *nextptr = (void*)((char*)new + offset); - void *head = *headptr; + void *head, *next; + + head = os_atomic_load(headptr, relaxed); + next = new; do { *nextptr = head; - } while (!os_atomic_cmpxchgvw(headptr, head, new, &head, release)); + } while (!os_atomic_cmpxchgvw(headptr, head, next, &head, release)); } void* @@ -255,23 +410,22 @@ OSAtomicDequeue(OSQueueHead *list, size_t offset) void * volatile *headptr = &(((_OSQueueHead*)list)->item); void * volatile *nextptr; void *head, *next; - (void)os_atomic_rmw_loop(headptr, head, next, acquire, { - if (!head) { - os_atomic_rmw_loop_give_up(break); - } + + os_atomic_rmw_loop(headptr, head, next, acquire, { + if (!head) os_atomic_rmw_loop_give_up(break); nextptr = (void*)((char*)head + offset); next = *nextptr; }); return head; } + void OSMemoryBarrier(void) { os_atomic_thread_fence(seq_cst); } -#endif // !OS_ATOMIC_NO_BARRIER_ONLY #endif // TARGET_OS_EMBEDDED struct _os_empty_files_are_not_c_files; diff --git a/src/os/lock.c b/src/os/lock.c index 3d66adf..076c61b 100644 --- a/src/os/lock.c +++ b/src/os/lock.c @@ -19,11 +19,12 @@ */ #include "lock_internal.h" +#include "os/internal.h" +#include "resolver.h" #include "libkern/OSAtomic.h" #include "os/lock.h" #include "os/lock_private.h" #include "os/once_private.h" -#include "resolver.h" #include #include @@ -34,7 +35,9 @@ #pragma mark - #pragma mark _os_lock_base_t -#if !OS_VARIANT_ONLY +OS_NOINLINE OS_NORETURN OS_COLD +void _os_lock_corruption_abort(void *lock_ptr OS_UNUSED, uintptr_t lock_value); + OS_LOCK_STRUCT_DECL_INTERNAL(base); OS_USED static OS_LOCK_TYPE_STRUCT_DECL(base); @@ -57,41 +60,34 @@ os_lock_unlock(os_lock_t l) return l._osl_base->osl_type->osl_unlock(l); } -#endif //!OS_VARIANT_ONLY - OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_lock_corruption_abort(void *lock_ptr OS_UNUSED, uintptr_t lock_value) { __LIBPLATFORM_CLIENT_CRASH__(lock_value, "os_lock is corrupt"); } + #pragma mark - #pragma mark OSSpinLock -#ifdef OS_LOCK_VARIANT_SELECTOR -void _OSSpinLockLockSlow(volatile OSSpinLock *l); -#else -OS_NOINLINE OS_USED static void _OSSpinLockLockSlow(volatile OSSpinLock *l); -#endif // OS_LOCK_VARIANT_SELECTOR +OS_NOEXPORT OS_NOINLINE void _OSSpinLockLockSlow(volatile OSSpinLock *l); OS_ATOMIC_EXPORT void OSSpinLockLock(volatile OSSpinLock *l); OS_ATOMIC_EXPORT bool OSSpinLockTry(volatile OSSpinLock *l); OS_ATOMIC_EXPORT int spin_lock_try(volatile OSSpinLock *l); OS_ATOMIC_EXPORT void OSSpinLockUnlock(volatile OSSpinLock *l); +static const OSSpinLock _OSSpinLockLocked = TARGET_OS_EMBEDDED ? 1 : -1; + + #if OS_ATOMIC_UP // Don't spin on UP -#elif OS_ATOMIC_WFE -#define OS_LOCK_SPIN_SPIN_TRIES 100 -#define OS_LOCK_SPIN_PAUSE() os_hardware_wfe() #else #define OS_LOCK_SPIN_SPIN_TRIES 1000 #define OS_LOCK_SPIN_PAUSE() os_hardware_pause() #endif -static const OSSpinLock _OSSpinLockLocked = TARGET_OS_EMBEDDED ? 1 : -1; - OS_ALWAYS_INLINE static uint64_t _os_lock_yield_deadline(mach_msg_timeout_t timeout) @@ -145,7 +141,7 @@ _OSSpinLockLockSlow(volatile OSSpinLock *l) { return _OSSpinLockLockYield(l); // Don't spin on UP } -#else +#else // !OS_ATOMIC_UP void _OSSpinLockLockSlow(volatile OSSpinLock *l) { @@ -163,21 +159,17 @@ _spin: if (likely(r)) return; goto _spin; } -#endif +#endif // !OS_ATOMIC_UP -#ifdef OS_LOCK_VARIANT_SELECTOR -#undef _OSSpinLockLockSlow -extern void _OSSpinLockLockSlow(volatile OSSpinLock *l); -#endif -#if !OS_LOCK_VARIANT_ONLY #if OS_LOCK_OSSPINLOCK_IS_NOSPINLOCK && !TARGET_OS_SIMULATOR typedef struct _os_nospin_lock_s *_os_nospin_lock_t; -void _os_nospin_lock_lock(_os_nospin_lock_t lock); -bool _os_nospin_lock_trylock(_os_nospin_lock_t lock); -void _os_nospin_lock_unlock(_os_nospin_lock_t lock); + +OS_ATOMIC_EXPORT void _os_nospin_lock_lock(_os_nospin_lock_t lock); +OS_ATOMIC_EXPORT bool _os_nospin_lock_trylock(_os_nospin_lock_t lock); +OS_ATOMIC_EXPORT void _os_nospin_lock_unlock(_os_nospin_lock_t lock); void OSSpinLockLock(volatile OSSpinLock *l) @@ -257,26 +249,15 @@ OSSpinLockUnlock(volatile OSSpinLock *l) os_atomic_store(l, 0, release); } + #pragma mark - #pragma mark os_lock_spin_t OS_LOCK_STRUCT_DECL_INTERNAL(spin, OSSpinLock volatile osl_spinlock; ); -#if !OS_VARIANT_ONLY OS_LOCK_METHODS_DECL(spin); OS_LOCK_TYPE_INSTANCE(spin); -#endif // !OS_VARIANT_ONLY - -#ifdef OS_VARIANT_SELECTOR -#define _os_lock_spin_lock \ - OS_VARIANT(_os_lock_spin_lock, OS_VARIANT_SELECTOR) -#define _os_lock_spin_trylock \ - OS_VARIANT(_os_lock_spin_trylock, OS_VARIANT_SELECTOR) -#define _os_lock_spin_unlock \ - OS_VARIANT(_os_lock_spin_unlock, OS_VARIANT_SELECTOR) -OS_LOCK_METHODS_DECL(spin); -#endif // OS_VARIANT_SELECTOR void _os_lock_spin_lock(_os_lock_spin_t l) @@ -296,6 +277,7 @@ _os_lock_spin_unlock(_os_lock_spin_t l) return OSSpinLockUnlock(&l->osl_spinlock); } + #pragma mark - #pragma mark os_lock_owner_t @@ -304,6 +286,8 @@ _os_lock_spin_unlock(_os_lock_spin_t l) #endif typedef mach_port_name_t os_lock_owner_t; +#define OS_LOCK_NO_OWNER MACH_PORT_NULL + OS_ALWAYS_INLINE static inline os_lock_owner_t @@ -314,9 +298,6 @@ _os_lock_owner_get_self(void) return self; } -#define OS_LOCK_NO_OWNER MACH_PORT_NULL - -#if !OS_LOCK_VARIANT_ONLY OS_NOINLINE OS_NORETURN OS_COLD static void @@ -326,7 +307,6 @@ _os_lock_recursive_abort(os_lock_owner_t owner) "os_lock"); } -#endif //!OS_LOCK_VARIANT_ONLY #pragma mark - #pragma mark os_lock_handoff_t @@ -334,20 +314,8 @@ _os_lock_recursive_abort(os_lock_owner_t owner) OS_LOCK_STRUCT_DECL_INTERNAL(handoff, os_lock_owner_t volatile osl_owner; ); -#if !OS_VARIANT_ONLY OS_LOCK_METHODS_DECL(handoff); OS_LOCK_TYPE_INSTANCE(handoff); -#endif // !OS_VARIANT_ONLY - -#ifdef OS_VARIANT_SELECTOR -#define _os_lock_handoff_lock \ - OS_VARIANT(_os_lock_handoff_lock, OS_VARIANT_SELECTOR) -#define _os_lock_handoff_trylock \ - OS_VARIANT(_os_lock_handoff_trylock, OS_VARIANT_SELECTOR) -#define _os_lock_handoff_unlock \ - OS_VARIANT(_os_lock_handoff_unlock, OS_VARIANT_SELECTOR) -OS_LOCK_METHODS_DECL(handoff); -#endif // OS_VARIANT_SELECTOR #define OS_LOCK_HANDOFF_YIELD_TRIES 100 @@ -397,6 +365,7 @@ _os_lock_handoff_unlock(_os_lock_handoff_t l) os_atomic_store2o(l, osl_owner, MACH_PORT_NULL, release); } + #pragma mark - #pragma mark os_ulock_value_t @@ -411,13 +380,12 @@ typedef os_lock_owner_t os_ulock_value_t; #define OS_ULOCK_OWNER(value) ((value) | OS_ULOCK_NOWAITERS_BIT) #define OS_ULOCK_ANONYMOUS_OWNER MACH_PORT_DEAD -#define OS_ULOCK_IS_OWNER(value, self) ({ \ - os_lock_owner_t _owner = OS_ULOCK_OWNER(value); \ - (_owner == (self) && _owner != OS_ULOCK_ANONYMOUS_OWNER); }) -#define OS_ULOCK_IS_NOT_OWNER(value, self) ({ \ - os_lock_owner_t _owner = OS_ULOCK_OWNER(value); \ - (_owner != (self) && _owner != OS_ULOCK_ANONYMOUS_OWNER); }) - +#define OS_ULOCK_IS_OWNER(value, self, allow_anonymous_owner) ({ \ + os_lock_owner_t _owner = OS_ULOCK_OWNER(value); (_owner == (self)) && \ + (!(allow_anonymous_owner) || _owner != OS_ULOCK_ANONYMOUS_OWNER); }) +#define OS_ULOCK_IS_NOT_OWNER(value, self, allow_anonymous_owner) ({ \ + os_lock_owner_t _owner = OS_ULOCK_OWNER(value); (_owner != (self)) && \ + (!(allow_anonymous_owner) || _owner != OS_ULOCK_ANONYMOUS_OWNER); }) #pragma mark - #pragma mark os_unfair_lock @@ -439,15 +407,27 @@ OS_ATOMIC_EXPORT void os_unfair_lock_lock_no_tsd_4libpthread( os_unfair_lock_t lock); OS_ATOMIC_EXPORT void os_unfair_lock_unlock_no_tsd_4libpthread( os_unfair_lock_t lock); +OS_ATOMIC_EXPORT void os_unfair_lock_lock_with_options_4Libc( + os_unfair_lock_t lock, os_unfair_lock_options_t options); +OS_ATOMIC_EXPORT void os_unfair_lock_unlock_4Libc(os_unfair_lock_t lock); + +OS_NOINLINE OS_NORETURN OS_COLD +void _os_unfair_lock_recursive_abort(os_lock_owner_t owner); +OS_NOINLINE OS_NORETURN OS_COLD +void _os_unfair_lock_unowned_abort(os_lock_owner_t owner); +OS_NOINLINE OS_NORETURN OS_COLD +void _os_unfair_lock_corruption_abort(os_ulock_value_t current); _Static_assert(OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION == ULF_WAIT_WORKQ_DATA_CONTENTION, "check value for OS_UNFAIR_LOCK_OPTIONS_MASK"); #define OS_UNFAIR_LOCK_OPTIONS_MASK \ (os_unfair_lock_options_t)(OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION) +#define OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER 0x01000000u + OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_unfair_lock_recursive_abort(os_lock_owner_t owner) { __LIBPLATFORM_CLIENT_CRASH__(owner, "Trying to recursively lock an " @@ -455,7 +435,7 @@ _os_unfair_lock_recursive_abort(os_lock_owner_t owner) } OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_unfair_lock_unowned_abort(os_lock_owner_t owner) { __LIBPLATFORM_CLIENT_CRASH__(owner, "Unlock of an os_unfair_lock not " @@ -463,25 +443,29 @@ _os_unfair_lock_unowned_abort(os_lock_owner_t owner) } OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_unfair_lock_corruption_abort(os_ulock_value_t current) { __LIBPLATFORM_CLIENT_CRASH__(current, "os_unfair_lock is corrupt"); } + OS_NOINLINE static void _os_unfair_lock_lock_slow(_os_unfair_lock_t l, os_lock_owner_t self, os_unfair_lock_options_t options) { - os_ulock_value_t current, new, waiters_mask = 0; + os_unfair_lock_options_t allow_anonymous_owner = + options & OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER; + options &= ~OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER; if (unlikely(options & ~OS_UNFAIR_LOCK_OPTIONS_MASK)) { __LIBPLATFORM_CLIENT_CRASH__(options, "Invalid options"); } + os_ulock_value_t current, new, waiters_mask = 0; while (unlikely((current = os_atomic_load2o(l, oul_value, relaxed)) != OS_LOCK_NO_OWNER)) { _retry: - if (unlikely(OS_ULOCK_IS_OWNER(current, self))) { + if (unlikely(OS_ULOCK_IS_OWNER(current, self, allow_anonymous_owner))) { return _os_unfair_lock_recursive_abort(self); } new = current & ~OS_ULOCK_NOWAITERS_BIT; @@ -519,9 +503,12 @@ _retry: OS_NOINLINE static void _os_unfair_lock_unlock_slow(_os_unfair_lock_t l, os_ulock_value_t current, - os_lock_owner_t self) + os_lock_owner_t self, os_unfair_lock_options_t options) { - if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self))) { + os_unfair_lock_options_t allow_anonymous_owner = + options & OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER; + options &= ~OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER; + if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self, allow_anonymous_owner))) { return _os_unfair_lock_unowned_abort(OS_ULOCK_OWNER(current)); } if (current & OS_ULOCK_NOWAITERS_BIT) { @@ -581,7 +568,7 @@ os_unfair_lock_unlock(os_unfair_lock_t lock) os_ulock_value_t current; current = os_atomic_xchg2o(l, oul_value, OS_LOCK_NO_OWNER, release); if (likely(current == self)) return; - return _os_unfair_lock_unlock_slow(l, current, self); + return _os_unfair_lock_unlock_slow(l, current, self, 0); } void @@ -592,7 +579,8 @@ os_unfair_lock_lock_no_tsd_4libpthread(os_unfair_lock_t lock) bool r = os_atomic_cmpxchg2o(l, oul_value, OS_LOCK_NO_OWNER, self, acquire); if (likely(r)) return; return _os_unfair_lock_lock_slow(l, self, - OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION); + OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION| + OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER); } void @@ -603,120 +591,18 @@ os_unfair_lock_unlock_no_tsd_4libpthread(os_unfair_lock_t lock) os_ulock_value_t current; current = os_atomic_xchg2o(l, oul_value, OS_LOCK_NO_OWNER, release); if (likely(current == self)) return; - return _os_unfair_lock_unlock_slow(l, current, self); -} - -#pragma mark - -#pragma mark _os_lock_unfair_t 4Libc // - -OS_ATOMIC_EXPORT void os_unfair_lock_lock_with_options_4Libc( - os_unfair_lock_t lock, os_unfair_lock_options_t options); -OS_ATOMIC_EXPORT void os_unfair_lock_unlock_4Libc(os_unfair_lock_t lock); - -OS_NOINLINE -static void -_os_unfair_lock_lock_slow_4Libc(_os_unfair_lock_t l, os_lock_owner_t self, - os_unfair_lock_options_t options) -{ - os_ulock_value_t current, new, waiters_mask = 0; - if (unlikely(options & ~OS_UNFAIR_LOCK_OPTIONS_MASK)) { - __LIBPLATFORM_CLIENT_CRASH__(options, "Invalid options"); - } - while (unlikely((current = os_atomic_load2o(l, oul_value, relaxed)) != - OS_LOCK_NO_OWNER)) { -_retry: - if (unlikely(OS_ULOCK_IS_OWNER(current, self))) { - return _os_unfair_lock_recursive_abort(self); - } - new = current & ~OS_ULOCK_NOWAITERS_BIT; - if (current != new) { - // Clear nowaiters bit in lock value before waiting - if (!os_atomic_cmpxchgv2o(l, oul_value, current, new, ¤t, - relaxed)){ - continue; - } - current = new; - } - int ret = __ulock_wait(UL_UNFAIR_LOCK | ULF_NO_ERRNO | options, - l, current, 0); - if (unlikely(ret < 0)) { - switch (-ret) { - case EINTR: - case EFAULT: - continue; - case EOWNERDEAD: - // if we get an `EOWNERDEAD` it could be corruption of the lock - // so for the Libc locks, if we can steal the lock, assume - // it is corruption and pretend we got the lock with contention - new = self & ~OS_ULOCK_NOWAITERS_BIT; - if (os_atomic_cmpxchgv2o(l, oul_value, current, new, ¤t, - acquire)) { - return; - } - break; - default: - __LIBPLATFORM_INTERNAL_CRASH__(-ret, "ulock_wait failure"); - } - } - // If there are more waiters, unset nowaiters bit when acquiring lock - waiters_mask = (ret > 0) ? OS_ULOCK_NOWAITERS_BIT : 0; - } - new = self & ~waiters_mask; - bool r = os_atomic_cmpxchgv2o(l, oul_value, OS_LOCK_NO_OWNER, new, - ¤t, acquire); - if (unlikely(!r)) goto _retry; + return _os_unfair_lock_unlock_slow(l, current, self, + OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER); } -OS_NOINLINE -static void -_os_unfair_lock_unlock_slow_4Libc(_os_unfair_lock_t l) -{ - for (;;) { - int ret = __ulock_wake(UL_UNFAIR_LOCK | ULF_NO_ERRNO, l, 0); - if (unlikely(ret < 0)) { - switch (-ret) { - case EINTR: - continue; - case ENOENT: - break; - default: - __LIBPLATFORM_INTERNAL_CRASH__(-ret, "ulock_wake failure"); - } - } - break; - } -} -void -os_unfair_lock_lock_with_options_4Libc(os_unfair_lock_t lock, - os_unfair_lock_options_t options) -{ - _os_unfair_lock_t l = (_os_unfair_lock_t)lock; - os_lock_owner_t self = _os_lock_owner_get_self(); - bool r = os_atomic_cmpxchg2o(l, oul_value, OS_LOCK_NO_OWNER, self, acquire); - if (likely(r)) return; - return _os_unfair_lock_lock_slow_4Libc(l, self, options); -} - -void -os_unfair_lock_unlock_4Libc(os_unfair_lock_t lock) -{ - _os_unfair_lock_t l = (_os_unfair_lock_t)lock; - os_lock_owner_t self = _os_lock_owner_get_self(); - os_ulock_value_t current; - current = os_atomic_xchg2o(l, oul_value, OS_LOCK_NO_OWNER, release); - if (likely(current == self)) return; - return _os_unfair_lock_unlock_slow_4Libc(l); -} - -#if !OS_VARIANT_ONLY void os_unfair_lock_assert_owner(os_unfair_lock_t lock) { _os_unfair_lock_t l = (_os_unfair_lock_t)lock; os_lock_owner_t self = _os_lock_owner_get_self(); os_ulock_value_t current = os_atomic_load2o(l, oul_value, relaxed); - if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self))) { + if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self, 0))) { __LIBPLATFORM_CLIENT_CRASH__(current, "Assertion failed: " "Lock unexpectedly not owned by current thread"); } @@ -728,12 +614,12 @@ os_unfair_lock_assert_not_owner(os_unfair_lock_t lock) _os_unfair_lock_t l = (_os_unfair_lock_t)lock; os_lock_owner_t self = _os_lock_owner_get_self(); os_ulock_value_t current = os_atomic_load2o(l, oul_value, relaxed); - if (unlikely(OS_ULOCK_IS_OWNER(current, self))) { + if (unlikely(OS_ULOCK_IS_OWNER(current, self, 0))) { __LIBPLATFORM_CLIENT_CRASH__(current, "Assertion failed: " "Lock unexpectedly owned by current thread"); } } -#endif + #pragma mark - #pragma mark _os_lock_unfair_t @@ -741,20 +627,8 @@ os_unfair_lock_assert_not_owner(os_unfair_lock_t lock) OS_LOCK_STRUCT_DECL_INTERNAL(unfair, os_unfair_lock osl_unfair_lock; ); -#if !OS_VARIANT_ONLY OS_LOCK_METHODS_DECL(unfair); OS_LOCK_TYPE_INSTANCE(unfair); -#endif // !OS_VARIANT_ONLY - -#ifdef OS_VARIANT_SELECTOR -#define _os_lock_unfair_lock \ - OS_VARIANT(_os_lock_unfair_lock, OS_VARIANT_SELECTOR) -#define _os_lock_unfair_trylock \ - OS_VARIANT(_os_lock_unfair_trylock, OS_VARIANT_SELECTOR) -#define _os_lock_unfair_unlock \ - OS_VARIANT(_os_lock_unfair_unlock, OS_VARIANT_SELECTOR) -OS_LOCK_METHODS_DECL(unfair); -#endif // OS_VARIANT_SELECTOR void _os_lock_unfair_lock(_os_lock_unfair_t l) @@ -774,6 +648,7 @@ _os_lock_unfair_unlock(_os_lock_unfair_t l) return os_unfair_lock_unlock(&l->osl_unfair_lock); } + #pragma mark - #pragma mark _os_nospin_lock @@ -788,6 +663,7 @@ OS_ATOMIC_EXPORT void _os_nospin_lock_lock(_os_nospin_lock_t lock); OS_ATOMIC_EXPORT bool _os_nospin_lock_trylock(_os_nospin_lock_t lock); OS_ATOMIC_EXPORT void _os_nospin_lock_unlock(_os_nospin_lock_t lock); + OS_NOINLINE static void _os_nospin_lock_lock_slow(_os_nospin_lock_t l) @@ -886,26 +762,15 @@ _os_nospin_lock_unlock(_os_nospin_lock_t l) return _os_nospin_lock_unlock_slow(l, current); } + #pragma mark - #pragma mark _os_lock_nospin_t OS_LOCK_STRUCT_DECL_INTERNAL(nospin, _os_nospin_lock osl_nospin_lock; ); -#if !OS_VARIANT_ONLY OS_LOCK_METHODS_DECL(nospin); OS_LOCK_TYPE_INSTANCE(nospin); -#endif // !OS_VARIANT_ONLY - -#ifdef OS_VARIANT_SELECTOR -#define _os_lock_nospin_lock \ - OS_VARIANT(_os_lock_nospin_lock, OS_VARIANT_SELECTOR) -#define _os_lock_nospin_trylock \ - OS_VARIANT(_os_lock_nospin_trylock, OS_VARIANT_SELECTOR) -#define _os_lock_nospin_unlock \ - OS_VARIANT(_os_lock_nospin_unlock, OS_VARIANT_SELECTOR) -OS_LOCK_METHODS_DECL(nospin); -#endif // OS_VARIANT_SELECTOR void _os_lock_nospin_lock(_os_lock_nospin_t l) @@ -925,6 +790,7 @@ _os_lock_nospin_unlock(_os_lock_nospin_t l) return _os_nospin_lock_unlock(&l->osl_nospin_lock); } + #pragma mark - #pragma mark os_once_t @@ -942,7 +808,15 @@ OS_ATOMIC_EXPORT void _os_once(os_once_t *val, void *ctxt, os_function_t func); OS_ATOMIC_EXPORT void __os_once_reset(os_once_t *val); OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_once_gate_recursive_abort(os_lock_owner_t owner); +OS_NOINLINE OS_NORETURN OS_COLD +void _os_once_gate_unowned_abort(os_lock_owner_t owner); +OS_NOINLINE OS_NORETURN OS_COLD +void _os_once_gate_corruption_abort(os_ulock_value_t current); + + +OS_NOINLINE OS_NORETURN OS_COLD +void _os_once_gate_recursive_abort(os_lock_owner_t owner) { __LIBPLATFORM_CLIENT_CRASH__(owner, "Trying to recursively lock an " @@ -950,7 +824,7 @@ _os_once_gate_recursive_abort(os_lock_owner_t owner) } OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_once_gate_unowned_abort(os_lock_owner_t owner) { __LIBPLATFORM_CLIENT_CRASH__(owner, "Unlock of an os_once_t not " @@ -958,12 +832,13 @@ _os_once_gate_unowned_abort(os_lock_owner_t owner) } OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_once_gate_corruption_abort(os_ulock_value_t current) { __LIBPLATFORM_CLIENT_CRASH__(current, "os_once_t is corrupt"); } + OS_NOINLINE static void _os_once_gate_wait_slow(os_ulock_value_t *gate, os_lock_owner_t self) @@ -980,7 +855,7 @@ _os_once_gate_wait_slow(os_ulock_value_t *gate, os_lock_owner_t self) tid_new = tid_old & ~OS_ULOCK_NOWAITERS_BIT; if (tid_new == tid_old) os_atomic_rmw_loop_give_up(break); }); - if (unlikely(OS_ULOCK_IS_OWNER(tid_old, self))) { + if (unlikely(OS_ULOCK_IS_OWNER(tid_old, self, 0))) { return _os_once_gate_recursive_abort(self); } int ret = __ulock_wait(UL_UNFAIR_LOCK | ULF_NO_ERRNO, @@ -1005,7 +880,7 @@ static void _os_once_gate_broadcast_slow(os_ulock_value_t *gate, os_ulock_value_t current, os_lock_owner_t self) { - if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self))) { + if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self, 0))) { return _os_once_gate_unowned_abort(OS_ULOCK_OWNER(current)); } if (current & OS_ULOCK_NOWAITERS_BIT) { @@ -1033,60 +908,13 @@ static void _os_once_gate_set_value_and_broadcast(os_once_gate_t og, os_lock_owner_t self, os_once_t value) { - // The next barrier must be long and strong. - // - // The scenario: SMP systems with weakly ordered memory models - // and aggressive out-of-order instruction execution. - // - // The problem: - // - // The os_once*() wrapper macro causes the callee's - // instruction stream to look like this (pseudo-RISC): - // - // load r5, pred-addr - // cmpi r5, -1 - // beq 1f - // call os_once*() - // 1f: - // load r6, data-addr - // - // May be re-ordered like so: - // - // load r6, data-addr - // load r5, pred-addr - // cmpi r5, -1 - // beq 1f - // call os_once*() - // 1f: - // - // Normally, a barrier on the read side is used to workaround - // the weakly ordered memory model. But barriers are expensive - // and we only need to synchronize once! After func(ctxt) - // completes, the predicate will be marked as "done" and the - // branch predictor will correctly skip the call to - // os_once*(). - // - // A far faster alternative solution: Defeat the speculative - // read-ahead of peer CPUs. - // - // Modern architectures will throw away speculative results - // once a branch mis-prediction occurs. Therefore, if we can - // ensure that the predicate is not marked as being complete - // until long after the last store by func(ctxt), then we have - // defeated the read-ahead of peer CPUs. - // - // In other words, the last "store" by func(ctxt) must complete - // and then N cycles must elapse before ~0l is stored to *val. - // The value of N is whatever is sufficient to defeat the - // read-ahead mechanism of peer CPUs. - // - // On some CPUs, the most fully synchronizing instruction might - // need to be issued. - os_atomic_maximally_synchronizing_barrier(); - - // above assumed to contain release barrier - os_ulock_value_t current = - (os_ulock_value_t)os_atomic_xchg(&og->ogo_once, value, relaxed); + os_ulock_value_t current; +#if defined(__i386__) || defined(__x86_64__) + // On Intel, any load is a load-acquire, so we don't need to be fancy + current = (os_ulock_value_t)os_atomic_xchg(&og->ogo_once, value, release); +#else +# error os_once algorithm not available for this architecture +#endif if (likely(current == self)) return; _os_once_gate_broadcast_slow(&og->ogo_lock, current, self); } @@ -1116,7 +944,6 @@ _os_once(os_once_t *val, void *ctxt, os_function_t func) } } -#if !OS_VARIANT_ONLY #pragma mark - #pragma mark os_lock_eliding_t @@ -1145,5 +972,3 @@ OS_LOCK_METHODS_DECL(transactional); OS_LOCK_TYPE_INSTANCE(transactional); #endif // !TARGET_OS_IPHONE -#endif // !OS_VARIANT_ONLY -#endif // !OS_LOCK_VARIANT_ONLY diff --git a/src/os/lock_internal.h b/src/os/lock_internal.h index af6a862..01c2d80 100644 --- a/src/os/lock_internal.h +++ b/src/os/lock_internal.h @@ -56,6 +56,4 @@ .osl_unlock = _os_lock_##type##_unlock, \ } -#include "os/internal.h" - #endif // __OS_LOCK_INTERNAL__ diff --git a/src/os/resolver.c b/src/os/resolver.c deleted file mode 100644 index 2fa6b0a..0000000 --- a/src/os/resolver.c +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2013 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -#include "lock_internal.h" - -#ifdef OS_VARIANT_SELECTOR - -#if TARGET_OS_EMBEDDED -OS_VARIANT_UPMP_RESOLVER(OSAtomicAdd32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicIncrement32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicDecrement32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicAdd64Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicIncrement64Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicDecrement64Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicAnd32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicAnd32OrigBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicOr32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicOr32OrigBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicXor32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicXor32OrigBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicCompareAndSwap32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicCompareAndSwap64Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicCompareAndSwapIntBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicCompareAndSwapLongBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicCompareAndSwapPtrBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicTestAndSetBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicTestAndClearBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicEnqueue) -OS_VARIANT_UPMP_RESOLVER(OSAtomicDequeue) -OS_VARIANT_UPMP_RESOLVER(OSMemoryBarrier) - -OS_VARIANT_UPMP_RESOLVER(OSSpinLockLock) -OS_VARIANT_UPMP_RESOLVER(OSSpinLockTry) -OS_VARIANT_UPMP_RESOLVER(OSSpinLockUnlock) -OS_VARIANT_UPMP_RESOLVER(spin_lock) -OS_VARIANT_UPMP_RESOLVER(spin_lock_try) -OS_VARIANT_UPMP_RESOLVER(spin_unlock) -OS_VARIANT_UPMP_RESOLVER(_spin_lock) -OS_VARIANT_UPMP_RESOLVER(_spin_lock_try) -OS_VARIANT_UPMP_RESOLVER(_spin_unlock) - -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_lock) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_lock_with_options) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_trylock) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_unlock) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_lock_no_tsd_4libpthread) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_unlock_no_tsd_4libpthread) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_lock_with_options_4Libc) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_unlock_4Libc) -OS_VARIANT_UPMP_RESOLVER(_os_nospin_lock_lock) -OS_VARIANT_UPMP_RESOLVER(_os_nospin_lock_trylock) -OS_VARIANT_UPMP_RESOLVER(_os_nospin_lock_unlock) - -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_spin_lock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_spin_trylock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_spin_unlock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_handoff_lock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_handoff_trylock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_handoff_unlock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_unfair_lock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_unfair_trylock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_unfair_unlock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_nospin_lock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_nospin_trylock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_nospin_unlock) - -OS_VARIANT_UPMP_RESOLVER(_os_once) -OS_VARIANT_UPMP_RESOLVER(__os_once_reset) -#endif // TARGET_OS_EMBEDDED - -#endif // OS_VARIANT_SELECTOR - -#ifdef OS_LOCK_VARIANT_SELECTOR - -#define WFE_RESOLVER(s) \ - _OS_VARIANT_RESOLVER(s, hidden, \ - uint32_t *_c = (void*)(uintptr_t)_COMM_PAGE_CPU_CAPABILITIES; \ - if (*_c & kHasEvent) { \ - extern void OS_VARIANT(s, wfe)(void); \ - return &OS_VARIANT(s, wfe); \ - } else { \ - extern void OS_VARIANT(s, mp)(void); \ - return &OS_VARIANT(s, mp); \ - }) - -WFE_RESOLVER(_OSSpinLockLockSlow) - -#endif // OS_LOCK_VARIANT_SELECTOR - diff --git a/src/os/resolver.h b/src/os/resolver.h deleted file mode 100644 index 345a6b6..0000000 --- a/src/os/resolver.h +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2013 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -#ifndef __OS_RESOLVER_H__ -#define __OS_RESOLVER_H__ - -#include - -#ifdef OS_VARIANT_SELECTOR - -#if TARGET_OS_EMBEDDED -#define OSAtomicAdd32Barrier \ - OS_VARIANT(OSAtomicAdd32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicIncrement32Barrier \ - OS_VARIANT(OSAtomicIncrement32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicDecrement32Barrier \ - OS_VARIANT(OSAtomicDecrement32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicAdd64Barrier \ - OS_VARIANT(OSAtomicAdd64Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicIncrement64Barrier \ - OS_VARIANT(OSAtomicIncrement64Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicDecrement64Barrier \ - OS_VARIANT(OSAtomicDecrement64Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicAnd32Barrier \ - OS_VARIANT(OSAtomicAnd32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicAnd32OrigBarrier \ - OS_VARIANT(OSAtomicAnd32OrigBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicOr32Barrier \ - OS_VARIANT(OSAtomicOr32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicOr32OrigBarrier \ - OS_VARIANT(OSAtomicOr32OrigBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicXor32Barrier \ - OS_VARIANT(OSAtomicXor32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicXor32OrigBarrier \ - OS_VARIANT(OSAtomicXor32OrigBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicCompareAndSwap32Barrier \ - OS_VARIANT(OSAtomicCompareAndSwap32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicCompareAndSwap64Barrier \ - OS_VARIANT(OSAtomicCompareAndSwap64Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicCompareAndSwapIntBarrier \ - OS_VARIANT(OSAtomicCompareAndSwapIntBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicCompareAndSwapLongBarrier \ - OS_VARIANT(OSAtomicCompareAndSwapLongBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicCompareAndSwapPtrBarrier \ - OS_VARIANT(OSAtomicCompareAndSwapPtrBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicTestAndSetBarrier \ - OS_VARIANT(OSAtomicTestAndSetBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicTestAndClearBarrier \ - OS_VARIANT(OSAtomicTestAndClearBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicEnqueue \ - OS_VARIANT(OSAtomicEnqueue, OS_VARIANT_SELECTOR) -#define OSAtomicDequeue \ - OS_VARIANT(OSAtomicDequeue, OS_VARIANT_SELECTOR) -#define OSMemoryBarrier \ - OS_VARIANT(OSMemoryBarrier, OS_VARIANT_SELECTOR) - -#define OSSpinLockLock \ - OS_VARIANT(OSSpinLockLock, OS_VARIANT_SELECTOR) -#define OSSpinLockTry \ - OS_VARIANT(OSSpinLockTry, OS_VARIANT_SELECTOR) -#define OSSpinLockUnlock \ - OS_VARIANT(OSSpinLockUnlock, OS_VARIANT_SELECTOR) -#define spin_lock \ - OS_VARIANT(spin_lock, OS_VARIANT_SELECTOR) -#define spin_lock_try \ - OS_VARIANT(spin_lock_try, OS_VARIANT_SELECTOR) -#define spin_unlock \ - OS_VARIANT(spin_unlock, OS_VARIANT_SELECTOR) -#define _spin_lock \ - OS_VARIANT(_spin_lock, OS_VARIANT_SELECTOR) -#define _spin_lock_try \ - OS_VARIANT(_spin_lock_try, OS_VARIANT_SELECTOR) -#define _spin_unlock \ - OS_VARIANT(_spin_unlock, OS_VARIANT_SELECTOR) -#define os_unfair_lock_lock \ - OS_VARIANT(os_unfair_lock_lock, OS_VARIANT_SELECTOR) -#define os_unfair_lock_lock_with_options \ - OS_VARIANT(os_unfair_lock_lock_with_options, OS_VARIANT_SELECTOR) -#define os_unfair_lock_trylock \ - OS_VARIANT(os_unfair_lock_trylock, OS_VARIANT_SELECTOR) -#define os_unfair_lock_unlock \ - OS_VARIANT(os_unfair_lock_unlock, OS_VARIANT_SELECTOR) -#define os_unfair_lock_lock_no_tsd_4libpthread \ - OS_VARIANT(os_unfair_lock_lock_no_tsd_4libpthread, OS_VARIANT_SELECTOR) -#define os_unfair_lock_unlock_no_tsd_4libpthread \ - OS_VARIANT(os_unfair_lock_unlock_no_tsd_4libpthread, OS_VARIANT_SELECTOR) -#define os_unfair_lock_lock_with_options_4Libc \ - OS_VARIANT(os_unfair_lock_lock_with_options_4Libc, OS_VARIANT_SELECTOR) -#define os_unfair_lock_unlock_4Libc \ - OS_VARIANT(os_unfair_lock_unlock_4Libc, OS_VARIANT_SELECTOR) -#define _os_nospin_lock_lock \ - OS_VARIANT(_os_nospin_lock_lock, OS_VARIANT_SELECTOR) -#define _os_nospin_lock_trylock \ - OS_VARIANT(_os_nospin_lock_trylock, OS_VARIANT_SELECTOR) -#define _os_nospin_lock_unlock \ - OS_VARIANT(_os_nospin_lock_unlock, OS_VARIANT_SELECTOR) -#define _os_once \ - OS_VARIANT(_os_once, OS_VARIANT_SELECTOR) -#define __os_once_reset \ - OS_VARIANT(__os_once_reset, OS_VARIANT_SELECTOR) -#endif // TARGET_OS_EMBEDDED -#endif // OS_VARIANT_SELECTOR - -#ifdef OS_LOCK_VARIANT_SELECTOR -#define _OSSpinLockLockSlow \ - OS_VARIANT(_OSSpinLockLockSlow, OS_LOCK_VARIANT_SELECTOR) -#endif // OS_LOCK_VARIANT_SELECTOR - -#endif // __OS_RESOLVER_H__ diff --git a/src/os/lock_up.c b/src/os/resolver/resolver.h similarity index 80% rename from src/os/lock_up.c rename to src/os/resolver/resolver.h index e03a22d..284575b 100644 --- a/src/os/lock_up.c +++ b/src/os/resolver/resolver.h @@ -18,12 +18,10 @@ * @APPLE_APACHE_LICENSE_HEADER_END@ */ -#define OS_ATOMIC_UP 1 -#include "lock_internal.h" +#ifndef __OS_RESOLVER_H__ +#define __OS_RESOLVER_H__ -#ifdef OS_VARIANT_SELECTOR -#define OS_VARIANT_ONLY 1 -#include "lock.c" -#endif +#include "resolver_internal.h" -struct _os_empty_files_are_not_c_files; + +#endif // __OS_RESOLVER_H__ diff --git a/src/os/atomic_up.c b/src/os/resolver/resolver_internal.h similarity index 67% rename from src/os/atomic_up.c rename to src/os/resolver/resolver_internal.h index 9013643..e87c918 100644 --- a/src/os/atomic_up.c +++ b/src/os/resolver/resolver_internal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Apple Inc. All rights reserved. + * Copyright (c) 2017 Apple Inc. All rights reserved. * * @APPLE_APACHE_LICENSE_HEADER_START@ * @@ -18,16 +18,10 @@ * @APPLE_APACHE_LICENSE_HEADER_END@ */ -// Force up variant to be generated to get the no-barrier OSAtomics -#undef VARIANT_NO_RESOLVERS -#define VARIANT_NO_RESOLVERS 0 +#ifndef __OS_RESOLVER_INTERNAL_H__ +#define __OS_RESOLVER_INTERNAL_H__ -#define OS_ATOMIC_UP 1 #include "os/internal.h" -#ifdef OS_VARIANT_SELECTOR -#define OS_VARIANT_ONLY 1 -#include "atomic.c" -#endif -struct _os_empty_files_are_not_c_files; +#endif // __OS_RESOLVER_INTERNAL_H__ diff --git a/src/os/semaphore.c b/src/os/semaphore.c index d805afc..513de98 100644 --- a/src/os/semaphore.c +++ b/src/os/semaphore.c @@ -41,11 +41,9 @@ _os_semaphore_create(void) { semaphore_t s4; kern_return_t kr; - while (unlikely(kr = semaphore_create(mach_task_self(), &s4, - SYNC_POLICY_FIFO, 0))) { - OS_VERIFY_MIG(kr, "Allocating semaphore failed with MIG_REPLY_MISMATCH"); - thread_switch(MACH_PORT_NULL, SWITCH_OPTION_WAIT, 100); - } + kr = semaphore_create(mach_task_self(), &s4, SYNC_POLICY_FIFO, 0); + OS_VERIFY_MIG(kr, "Allocating semaphore failed with MIG_REPLY_MISMATCH"); + OS_SEMAPHORE_VERIFY_KR(kr, "Creating semaphore failed, possible port leak"); return (os_semaphore_t)s4; } diff --git a/src/simple/asl.c b/src/simple/asl.c index d970ea0..5511e0f 100644 --- a/src/simple/asl.c +++ b/src/simple/asl.c @@ -207,7 +207,7 @@ _simple_asl_msg_new(void) if (b == NULL) return NULL; - if (_simple_sprintf(b, " 0", 0)) + if (_simple_sprintf(b, " 0")) { _simple_sfree(b); return NULL; @@ -224,7 +224,7 @@ _simple_asl_msg_set(_SIMPLE_STRING __b, const char *__key, const char *__val) do { - if (_simple_sprintf(__b, " [", 0)) break; + if (_simple_sprintf(__b, " [")) break; if (_simple_esprintf(__b, _simple_asl_escape_key, "%s", __key)) break; if (__val != NULL) { @@ -265,13 +265,13 @@ _simple_asl_send(_SIMPLE_STRING __b) { char *cp; - if (_simple_sprintf(__b, " [PID ", 0)) break; + if (_simple_sprintf(__b, " [PID ")) break; if (_simple_esprintf(__b, _simple_asl_escape_val, "%u", getpid())) break; - if (_simple_sprintf(__b, "] [UID ", 0)) break; + if (_simple_sprintf(__b, "] [UID ")) break; if (_simple_esprintf(__b, _simple_asl_escape_val, "%u", getuid())) break; - if (_simple_sprintf(__b, "] [GID ", 0)) break; + if (_simple_sprintf(__b, "] [GID ")) break; if (_simple_esprintf(__b, _simple_asl_escape_val, "%u", getgid())) break; - if (_simple_sprintf(__b, "] [Time ", 0)) break; + if (_simple_sprintf(__b, "] [Time ")) break; if (_simple_esprintf(__b, _simple_asl_escape_val, "%lu", tv.tv_sec)) break; if (_simple_sappend(__b, "] [TimeNanoSec ")) break; if (_simple_esprintf(__b, _simple_asl_escape_val, "%d", tv.tv_usec * 1000)) break; diff --git a/src/simple/string_io.c b/src/simple/string_io.c index 2e581e8..4bffac6 100644 --- a/src/simple/string_io.c +++ b/src/simple/string_io.c @@ -37,8 +37,12 @@ #endif #define BUF_SIZE(s) (((BUF *)(s))->end - ((BUF *)(s))->buf + 1) +#if DEBUG +#define MYBUFSIZE 256 +#else /* we use a small buffer to minimize stack usage constraints */ #define MYBUFSIZE 32 +#endif typedef struct _BUF { char *buf; @@ -439,13 +443,10 @@ _simple_dprintf(int fd, const char *fmt, ...) _SIMPLE_STRING _simple_salloc(void) { - kern_return_t kr; BUF *b; - kr = vm_allocate(mach_task_self(), (vm_address_t *)&b, VM_PAGE_SIZE, 1); - if (kr) { - __LIBPLATFORM_CLIENT_CRASH__(kr, "Failed to allocate memory for string"); - } + if(vm_allocate(mach_task_self(), (vm_address_t *)&b, VM_PAGE_SIZE, 1)) + return NULL; b->ptr = b->buf = (char *)b + sizeof(BUF); b->end = (char *)b + VM_PAGE_SIZE - 1; b->full = _enlarge; @@ -455,7 +456,8 @@ _simple_salloc(void) /* * The format string is interpreted with arguments from the va_list, and the * results are appended to the string maintained by the opaque structure, as - * returned by a previous call to _simple_salloc(). Always returns 0. + * returned by a previous call to _simple_salloc(). Non-zero is returned on + * out-of-memory error. */ int _simple_vsprintf(_SIMPLE_STRING b, const char *fmt, va_list ap) @@ -466,8 +468,8 @@ _simple_vsprintf(_SIMPLE_STRING b, const char *fmt, va_list ap) /* * The format string is interpreted with arguments from the variable argument * list, and the results are appended to the string maintained by the opaque - * structure, as returned by a previous call to _simple_salloc(). - * Always returns 0. + * structure, as returned by a previous call to _simple_salloc(). Non-zero is + * returned on out-of-memory error. */ int _simple_sprintf(_SIMPLE_STRING b, const char *fmt, ...) @@ -532,7 +534,7 @@ _simple_sresize(_SIMPLE_STRING b) /* * Append the null-terminated string to the string associated with the opaque - * structure. Always returns 0. + * structure. Non-zero is returned on out-of-memory error. */ int _simple_sappend(_SIMPLE_STRING b, const char *str) diff --git a/xcodeconfig/libplatform.aliases b/xcodeconfig/libplatform.aliases index 842ba78..4e8ce39 100644 --- a/xcodeconfig/libplatform.aliases +++ b/xcodeconfig/libplatform.aliases @@ -1 +1,3 @@ __platform_bzero ___bzero +_os_unfair_lock_lock_with_options _os_unfair_lock_lock_with_options_4Libc +_os_unfair_lock_unlock _os_unfair_lock_unlock_4Libc diff --git a/xcodeconfig/libplatform.xcconfig b/xcodeconfig/libplatform.xcconfig index e607f4b..ce479ba 100644 --- a/xcodeconfig/libplatform.xcconfig +++ b/xcodeconfig/libplatform.xcconfig @@ -19,16 +19,16 @@ INSTALLHDRS_SCRIPT_PHASE = YES GCC_OPTIMIZATION_LEVEL = s // TODO: Remove -fno-stack-protector once it has been moved down (after libproc is moved down) -OTHER_CFLAGS = -fno-stack-protector -fdollars-in-identifiers -fno-common -fverbose-asm $(COMPILER_CFLAGS) -isystem $(SYSTEM_FRAMEWORK_HEADERS) +OTHER_CFLAGS = -fno-stack-protector -fdollars-in-identifiers -fno-common -fverbose-asm $(COMPILER_CFLAGS) $(PLATFORM_CFLAGS) -isystem $(SYSTEM_FRAMEWORK_HEADERS) OTHER_CFLAGS_normal = -momit-leaf-frame-pointer OTHER_CFLAGS_debug = -fno-inline -O0 -GCC_PREPROCESSOR_DEFINITIONS = $(GCC_PREPROCESSOR_DEFINITIONS_$(CURRENT_VARIANT)) $(OSATOMIC_PREPROCESSOR_DEFINITIONS) +GCC_PREPROCESSOR_DEFINITIONS = _FORTIFY_SOURCE=0 $(OSATOMIC_PREPROCESSOR_DEFINITIONS) $(PLATFORM_PREPROCESSOR_DEFINITIONS) $(GCC_PREPROCESSOR_DEFINITIONS_$(CURRENT_VARIANT)) GCC_PREPROCESSOR_DEFINITIONS_dyld = VARIANT_DYLD=1 VARIANT_NO_RESOLVERS=1 VARIANT_STATIC=1 GCC_PREPROCESSOR_DEFINITIONS_static = VARIANT_NO_RESOLVERS=1 VARIANT_STATIC=1 GCC_PREPROCESSOR_DEFINITIONS_debug = DEBUG=1 -OSATOMIC_PREPROCESSOR_DEFINITIONS = OSATOMIC_USE_INLINED=1 OSSPINLOCK_USE_INLINED=1 OS_UNFAIR_LOCK_INLINE=0 +OSATOMIC_PREPROCESSOR_DEFINITIONS = OSATOMIC_USE_INLINED=0 OSATOMIC_DEPRECATED=0 OSSPINLOCK_USE_INLINED=1 OS_UNFAIR_LOCK_INLINE=0 STRIP_INSTALLED_PRODUCT = $(STRIP_INSTALLED_PRODUCT_$(CURRENT_VARIANT)) STRIP_INSTALLED_PRODUCT_normal = YES @@ -63,6 +63,7 @@ OTHER_LIBTOOLFLAGS_libplatform_dyld = $(PLATFORM_LIBRARIES) OTHER_LIBTOOLFLAGS_libplatform_static = $(PLATFORM_LIBRARIES) OTHER_LIBTOOLFLAGS_libatomics = $(ATOMICS_LIBRARIES) OTHER_LIBTOOLFLAGS_libcachecontrol = $(CACHECONTROL_LIBRARIES) +OTHER_LIBTOOLFLAGS_libos = $(OS_LIBRARIES_$(CURRENT_VARIANT)) OTHER_LIBTOOLFLAGS_libsetjmp = $(SETJMP_LIBRARIES) OTHER_LIBTOOLFLAGS_libstring = $(STRING_LIBRARIES) OTHER_LIBTOOLFLAGS_libucontext = $(UCONTEXT_LIBRARIES) diff --git a/xcodeconfig/os.xcconfig b/xcodeconfig/os.xcconfig index b163c0f..3a94018 100644 --- a/xcodeconfig/os.xcconfig +++ b/xcodeconfig/os.xcconfig @@ -1,5 +1,3 @@ -#include "libplatform.xcconfig" - GCC_STRICT_ALIASING = YES GCC_SYMBOLS_PRIVATE_EXTERN = YES GCC_WARN_SHADOW = YES @@ -21,8 +19,10 @@ WARNING_CFLAGS = -Wall -Wextra -Waggregate-return -Wfloat-equal -Wpacked -Wmissi COMPILER_CFLAGS = -momit-leaf-frame-pointer OTHER_CFLAGS_debug = +SRCROOT_SEARCH_PATHS = $(inherited) $(SRCROOT)/src/os/resolver OSATOMIC_PREPROCESSOR_DEFINITIONS = OSATOMIC_USE_INLINED=0 OSATOMIC_DEPRECATED=0 OSSPINLOCK_USE_INLINED=0 OSSPINLOCK_DEPRECATED=0 PUBLIC_HEADERS_FOLDER_PATH = /usr/include/os PRIVATE_HEADERS_FOLDER_PATH = /usr/local/include/os OS_INTERNAL_HEADERS_FOLDER_PATH = /usr/local/include/os/internal + -- 2.47.2