From: Apple Date: Tue, 26 Sep 2017 16:51:55 +0000 (+0000) Subject: libplatform-161.tar.gz X-Git-Tag: macos-1013^0 X-Git-Url: https://git.saurik.com/apple/libplatform.git/commitdiff_plain/e45b469263efe5fed40a01d25b2ace7e42243579 libplatform-161.tar.gz --- diff --git a/.upstream_base_commits b/.upstream_base_commits new file mode 100644 index 0000000..ebe5327 --- /dev/null +++ b/.upstream_base_commits @@ -0,0 +1,2 @@ +#freebsd = https://github.com/freebsd/freebsd.git +man/stdatomic.3 freebsd share/man/man3/ATOMIC_VAR_INIT.3 10701d40cc37d89e1a105f5fc5bf6c4a21fabf67 diff --git a/include/_simple.h b/include/_simple.h deleted file mode 100644 index b3bd924..0000000 --- a/include/_simple.h +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (c) 2006, 2010, 2013 Apple Inc. All rights reserved. - * - * @APPLE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_LICENSE_HEADER_END@ - */ - -#ifndef _SYSTEM_SIMPLE_H_ -#define _SYSTEM_SIMPLE_H_ - -#include -#include - -#include - -typedef void *_SIMPLE_STRING; -typedef const char *_esc_func(unsigned char); - -__BEGIN_DECLS -/* - * A simplified vfprintf variant. The format string is interpreted with - * arguments from the va_list, and the results are written to the given - * file descriptor. - */ -void _simple_vdprintf(int __fd, const char *__fmt, va_list __ap); - -/* - * A simplified fprintf variant. The format string is interpreted with - * arguments from the variable argument list, and the results are written - * to the given file descriptor. - */ -void _simple_dprintf(int __fd, const char *__fmt, ...); - -/* - * A simplified string allocate routine. Pass the opaque pointer to structure - * to _simple_*sprintf() routines. Use _simple_string() to retrieve the - * current string (the string is guaranteed to be null terminated only on - * the call to _simple_string()). Use _simple_sfree() to free the structure - * and string memory. - */ -_SIMPLE_STRING _simple_salloc(void); - -/* - * The format string is interpreted with arguments from the va_list, and the - * results are appended to the string maintained by the opaque structure, as - * returned by a previous call to _simple_salloc(). - * Always returns 0 on OS X >= 10.12 and iOS >= 10.0 - */ -int _simple_vsprintf(_SIMPLE_STRING __b, const char *__fmt, va_list __ap); - -/* - * The format string is interpreted with arguments from the variable argument - * list, and the results are appended to the string maintained by the opaque - * structure, as returned by a previous call to _simple_salloc(). - * Always returns 0 on OS X >= 10.12 and iOS >= 10.0 - */ -int _simple_sprintf(_SIMPLE_STRING __b, const char *__fmt, ...); - -/* - * Like _simple_vsprintf(), except __esc is a function to call on each - * character; the function returns NULL if the character should be passed - * as is, otherwise, the returned character string is used instead. - */ -int _simple_vesprintf(_SIMPLE_STRING __b, _esc_func __esc, const char *__fmt, va_list __ap); - -/* - * Like _simple_sprintf(), except __esc is a function to call on each - * character; the function returns NULL if the character should be passed - * as is, otherwise, the returned character string is used instead. - */ -int _simple_esprintf(_SIMPLE_STRING __b, _esc_func __esc, const char *__fmt, ...); - -/* - * Return the null terminated string from the opaque structure, as returned - * by a previous call to _simple_salloc(). - */ -char *_simple_string(_SIMPLE_STRING __b); - -/* - * Reposition the pointer to the first null in the buffer. After a call to - * _simple_string, the buffer can be modified, and shrunk. - */ -void _simple_sresize(_SIMPLE_STRING __b); - -/* - * Append the null-terminated string to the string associated with the opaque - * structure. Always returns 0 on OS X >= 10.12 and iOS >= 10.0 - */ -int _simple_sappend(_SIMPLE_STRING __b, const char *__str); - -/* - * Like _simple_sappend(), except __esc is a function to call on each - * character; the function returns NULL if the character should be passed - * as is, otherwise, the returned character string is used instead. - */ -int _simple_esappend(_SIMPLE_STRING __b, _esc_func __esc, const char *__str); - -/* - * Write the string associated with the opaque structure to the file descriptor. - */ -void _simple_put(_SIMPLE_STRING __b, int __fd); - -/* - * Write the string associated with the opaque structure and a trailing newline, - * to the file descriptor. - */ -void _simple_putline(_SIMPLE_STRING __b, int __fd); - -/* - * Free the opaque structure, and the associated string. - */ -void _simple_sfree(_SIMPLE_STRING __b); - -/* - * Simplified ASL log interface; does not use malloc. Unfortunately, this - * requires knowledge of the format used by ASL. - */ -#ifndef ASL_LEVEL_DEBUG -#define ASL_LEVEL_EMERG 0 -#define ASL_LEVEL_ALERT 1 -#define ASL_LEVEL_CRIT 2 -#define ASL_LEVEL_ERR 3 -#define ASL_LEVEL_WARNING 4 -#define ASL_LEVEL_NOTICE 5 -#define ASL_LEVEL_INFO 6 -#define ASL_LEVEL_DEBUG 7 -#endif - -void _simple_asl_log(int __level, const char *__facility, const char *__message); -void _simple_asl_log_prog(int level, const char *facility, const char *message, const char *progname); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -_SIMPLE_STRING _simple_asl_msg_new(void); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -void _simple_asl_msg_set(_SIMPLE_STRING __b, const char *__key, const char *__val); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -void _simple_asl_send(_SIMPLE_STRING __b); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -const char *_simple_getenv(const char *envp[], const char *var); - -__END_DECLS - -#endif /* _SYSTEM_SIMPLE_H_ */ diff --git a/include/libkern/OSAtomicQueue.h b/include/libkern/OSAtomicQueue.h index 8ffa4be..103f1e8 100644 --- a/include/libkern/OSAtomicQueue.h +++ b/include/libkern/OSAtomicQueue.h @@ -42,7 +42,7 @@ __BEGIN_DECLS You should always initialize a queue head structure with the initialization vector {@link OS_ATOMIC_QUEUE_INIT} before use. */ -#if defined(__x86_64__) +#if defined(__LP64__) typedef volatile struct { void *opaque1; diff --git a/include/libkern/OSSpinLockDeprecated.h b/include/libkern/OSSpinLockDeprecated.h index 68b64cd..f22b141 100644 --- a/include/libkern/OSSpinLockDeprecated.h +++ b/include/libkern/OSSpinLockDeprecated.h @@ -182,7 +182,8 @@ __BEGIN_DECLS #define OS_SPINLOCK_INIT 0 typedef int32_t OSSpinLock OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock); -typedef volatile OSSpinLock *_os_nospin_lock_t; +typedef volatile OSSpinLock *_os_nospin_lock_t + OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_t); OSSPINLOCK_DEPRECATED_REPLACE_WITH(os_unfair_lock_lock) OS_NOSPIN_LOCK_AVAILABILITY diff --git a/include/os/alloc_once_impl.h b/include/os/alloc_once_impl.h deleted file mode 100644 index d821e18..0000000 --- a/include/os/alloc_once_impl.h +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2012-2013 Apple Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ - -#ifndef __OS_ALLOC_ONCE_IMPL__ -#define __OS_ALLOC_ONCE_IMPL__ - -#ifndef __OS_ALLOC_INDIRECT__ -#error "Please include instead of this file directly." -#endif - -#include -#include -#include -#include - -__BEGIN_DECLS - -#define OS_ALLOC_SPI_VERSION 20120430 - -#define OS_ALLOC_ONCE_KEY_MAX 100 - -typedef os_once_t os_alloc_token_t; -struct _os_alloc_once_s { - os_alloc_token_t once; - void *ptr; -}; - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_6_0) -extern struct _os_alloc_once_s _os_alloc_once_table[]; - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_6_0) -OS_EXPORT OS_NONNULL1 -void* -_os_alloc_once(struct _os_alloc_once_s *slot, size_t sz, os_function_t init); - -/* - * The region allocated by os_alloc_once is 0-filled when initially - * returned (or handed off to the initializer). - */ -OS_WARN_RESULT OS_NOTHROW OS_CONST -__header_always_inline void* -os_alloc_once(os_alloc_token_t token, size_t sz, os_function_t init) -{ - struct _os_alloc_once_s *slot = &_os_alloc_once_table[token]; - if (OS_EXPECT(slot->once, ~0l) != ~0l) { - void *ptr = _os_alloc_once(slot, sz, init); - OS_COMPILER_CAN_ASSUME(slot->once == ~0l); - return ptr; - } - return slot->ptr; -} - -__END_DECLS - -#endif // __OS_ALLOC_ONCE_IMPL__ diff --git a/include/os/base.h b/include/os/base.h index 77ab213..9fa30e4 100644 --- a/include/os/base.h +++ b/include/os/base.h @@ -122,6 +122,12 @@ #define OS_NOESCAPE #endif +#if defined(__cplusplus) && defined(__clang__) +#define OS_FALLTHROUGH [[clang::fallthrough]] +#else +#define OS_FALLTHROUGH +#endif + #if __has_feature(assume_nonnull) #define OS_ASSUME_NONNULL_BEGIN _Pragma("clang assume_nonnull begin") #define OS_ASSUME_NONNULL_END _Pragma("clang assume_nonnull end") diff --git a/include/os/base_private.h b/include/os/base_private.h deleted file mode 100644 index 2d38266..0000000 --- a/include/os/base_private.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2008-2013 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -#ifndef __OS_BASE_PRIVATE__ -#define __OS_BASE_PRIVATE__ - -#include - -#ifndef os_fastpath -#define os_fastpath(x) ((__typeof__(x))OS_EXPECT((long)(x), ~0l)) -#endif -#ifndef os_slowpath -#define os_slowpath(x) ((__typeof__(x))OS_EXPECT((long)(x), 0l)) -#endif -#ifndef os_likely -#define os_likely(x) OS_EXPECT(!!(x), 1) -#endif -#ifndef os_unlikely -#define os_unlikely(x) OS_EXPECT(!!(x), 0) -#endif - -#endif // __OS_BASE_PRIVATE__ diff --git a/include/os/internal/atomic.h b/include/os/internal/atomic.h deleted file mode 100644 index 9b3294d..0000000 --- a/include/os/internal/atomic.h +++ /dev/null @@ -1,277 +0,0 @@ -/* - * Copyright (c) 2008-2013 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -#ifndef __OS_INTERNAL_ATOMIC__ -#define __OS_INTERNAL_ATOMIC__ - -#ifndef __OS_EXPOSE_INTERNALS_INDIRECT__ -/* - * Use c11 or c++11 std::atomic from instead - * - * XXX /!\ WARNING /!\ XXX - * - * This header file describes INTERNAL interfaces to libplatform used by other - * libsystem targets, which are subject to change in future releases of OS X - * and iOS. Any applications relying on these interfaces WILL break. - * - * If you are not a libsystem target, you should NOT EVER use these headers. - * Not even a little. - * - * XXX /!\ WARNING /!\ XXX - */ -#error "Please #include instead of this file directly." -#else - -// generate error during codegen -#define _os_atomic_unimplemented() \ - ({ __asm__(".err unimplemented"); }) - -#pragma mark - -#pragma mark memory_order - -typedef enum _os_atomic_memory_order { - _os_atomic_memory_order_relaxed, - _os_atomic_memory_order_consume, - _os_atomic_memory_order_acquire, - _os_atomic_memory_order_release, - _os_atomic_memory_order_acq_rel, - _os_atomic_memory_order_seq_cst, - _os_atomic_memory_order_ordered, - _os_atomic_memory_order_dependency, -} _os_atomic_memory_order; - -#if !OS_ATOMIC_UP - -#define os_atomic_memory_order_relaxed _os_atomic_memory_order_relaxed -#define os_atomic_memory_order_acquire _os_atomic_memory_order_acquire -#define os_atomic_memory_order_release _os_atomic_memory_order_release -#define os_atomic_memory_order_acq_rel _os_atomic_memory_order_acq_rel -#define os_atomic_memory_order_seq_cst _os_atomic_memory_order_seq_cst -#define os_atomic_memory_order_ordered _os_atomic_memory_order_seq_cst -#define os_atomic_memory_order_dependency _os_atomic_memory_order_acquire - -#else // OS_ATOMIC_UP - -#define os_atomic_memory_order_relaxed _os_atomic_memory_order_relaxed -#define os_atomic_memory_order_acquire _os_atomic_memory_order_relaxed -#define os_atomic_memory_order_release _os_atomic_memory_order_relaxed -#define os_atomic_memory_order_acq_rel _os_atomic_memory_order_relaxed -#define os_atomic_memory_order_seq_cst _os_atomic_memory_order_relaxed -#define os_atomic_memory_order_ordered _os_atomic_memory_order_relaxed -#define os_atomic_memory_order_dependency _os_atomic_memory_order_relaxed - -#endif // OS_ATOMIC_UP - -#pragma mark - -#pragma mark c11 - -#if !__has_extension(c_atomic) -#error "Please use a C11 compiler" -#endif - -#define os_atomic(type) type _Atomic - -#define _os_atomic_c11_atomic(p) \ - ((typeof(*(p)) _Atomic *)(p)) - -// This removes the _Atomic and volatile qualifiers on the type of *p -#define _os_atomic_basetypeof(p) \ - typeof(__c11_atomic_load(_os_atomic_c11_atomic(p), \ - _os_atomic_memory_order_relaxed)) - -#define _os_atomic_baseptr(p) \ - ((_os_atomic_basetypeof(p) *)(p)) - -#define _os_atomic_barrier(m) \ - __c11_atomic_thread_fence(os_atomic_memory_order_##m) -#define os_atomic_load(p, m) \ - __c11_atomic_load(_os_atomic_c11_atomic(p), os_atomic_memory_order_##m) -#define os_atomic_store(p, v, m) \ - __c11_atomic_store(_os_atomic_c11_atomic(p), v, \ - os_atomic_memory_order_##m) -#define os_atomic_xchg(p, v, m) \ - __c11_atomic_exchange(_os_atomic_c11_atomic(p), v, \ - os_atomic_memory_order_##m) -#define os_atomic_cmpxchg(p, e, v, m) \ - ({ _os_atomic_basetypeof(p) _r = (e); \ - __c11_atomic_compare_exchange_strong(_os_atomic_c11_atomic(p), \ - &_r, v, os_atomic_memory_order_##m, os_atomic_memory_order_relaxed); }) -#define os_atomic_cmpxchgv(p, e, v, g, m) \ - ({ _os_atomic_basetypeof(p) _r = (e); _Bool _b = \ - __c11_atomic_compare_exchange_strong(_os_atomic_c11_atomic(p), \ - &_r, v, os_atomic_memory_order_##m, os_atomic_memory_order_relaxed); \ - *(g) = _r; _b; }) -#define os_atomic_cmpxchgvw(p, e, v, g, m) \ - ({ _os_atomic_basetypeof(p) _r = (e); _Bool _b = \ - __c11_atomic_compare_exchange_weak(_os_atomic_c11_atomic(p), \ - &_r, v, os_atomic_memory_order_##m, os_atomic_memory_order_relaxed); \ - *(g) = _r; _b; }) -#define _os_atomic_c11_op(p, v, m, o, op) \ - ({ _os_atomic_basetypeof(p) _v = (v), _r = \ - __c11_atomic_fetch_##o(_os_atomic_c11_atomic(p), _v, \ - os_atomic_memory_order_##m); (typeof(_r))(_r op _v); }) -#define _os_atomic_c11_op_orig(p, v, m, o, op) \ - __c11_atomic_fetch_##o(_os_atomic_c11_atomic(p), v, \ - os_atomic_memory_order_##m) - -#define os_atomic_add(p, v, m) \ - _os_atomic_c11_op((p), (v), m, add, +) -#define os_atomic_add_orig(p, v, m) \ - _os_atomic_c11_op_orig((p), (v), m, add, +) -#define os_atomic_sub(p, v, m) \ - _os_atomic_c11_op((p), (v), m, sub, -) -#define os_atomic_sub_orig(p, v, m) \ - _os_atomic_c11_op_orig((p), (v), m, sub, -) -#define os_atomic_and(p, v, m) \ - _os_atomic_c11_op((p), (v), m, and, &) -#define os_atomic_and_orig(p, v, m) \ - _os_atomic_c11_op_orig((p), (v), m, and, &) -#define os_atomic_or(p, v, m) \ - _os_atomic_c11_op((p), (v), m, or, |) -#define os_atomic_or_orig(p, v, m) \ - _os_atomic_c11_op_orig((p), (v), m, or, |) -#define os_atomic_xor(p, v, m) \ - _os_atomic_c11_op((p), (v), m, xor, ^) -#define os_atomic_xor_orig(p, v, m) \ - _os_atomic_c11_op_orig((p), (v), m, xor, ^) - -#define os_atomic_force_dependency_on(p, e) (p) -#define os_atomic_load_with_dependency_on(p, e) \ - os_atomic_load(os_atomic_force_dependency_on(p, e), relaxed) -#define os_atomic_load_with_dependency_on2o(p, f, e) \ - os_atomic_load_with_dependency_on(&(p)->f, e) - -#pragma mark - -#pragma mark generic - -#define os_atomic_thread_fence(m) _os_atomic_barrier(m) -// see comment in os_once.c -#define os_atomic_maximally_synchronizing_barrier() \ - _os_atomic_barrier(seq_cst) - -#define os_atomic_load2o(p, f, m) \ - os_atomic_load(&(p)->f, m) -#define os_atomic_store2o(p, f, v, m) \ - os_atomic_store(&(p)->f, (v), m) -#define os_atomic_xchg2o(p, f, v, m) \ - os_atomic_xchg(&(p)->f, (v), m) -#define os_atomic_cmpxchg2o(p, f, e, v, m) \ - os_atomic_cmpxchg(&(p)->f, (e), (v), m) -#define os_atomic_cmpxchgv2o(p, f, e, v, g, m) \ - os_atomic_cmpxchgv(&(p)->f, (e), (v), (g), m) -#define os_atomic_cmpxchgvw2o(p, f, e, v, g, m) \ - os_atomic_cmpxchgvw(&(p)->f, (e), (v), (g), m) -#define os_atomic_add2o(p, f, v, m) \ - os_atomic_add(&(p)->f, (v), m) -#define os_atomic_add_orig2o(p, f, v, m) \ - os_atomic_add_orig(&(p)->f, (v), m) -#define os_atomic_sub2o(p, f, v, m) \ - os_atomic_sub(&(p)->f, (v), m) -#define os_atomic_sub_orig2o(p, f, v, m) \ - os_atomic_sub_orig(&(p)->f, (v), m) -#define os_atomic_and2o(p, f, v, m) \ - os_atomic_and(&(p)->f, (v), m) -#define os_atomic_and_orig2o(p, f, v, m) \ - os_atomic_and_orig(&(p)->f, (v), m) -#define os_atomic_or2o(p, f, v, m) \ - os_atomic_or(&(p)->f, (v), m) -#define os_atomic_or_orig2o(p, f, v, m) \ - os_atomic_or_orig(&(p)->f, (v), m) -#define os_atomic_xor2o(p, f, v, m) \ - os_atomic_xor(&(p)->f, (v), m) -#define os_atomic_xor_orig2o(p, f, v, m) \ - os_atomic_xor_orig(&(p)->f, (v), m) - -#define os_atomic_inc(p, m) \ - os_atomic_add((p), 1, m) -#define os_atomic_inc_orig(p, m) \ - os_atomic_add_orig((p), 1, m) -#define os_atomic_inc2o(p, f, m) \ - os_atomic_add2o(p, f, 1, m) -#define os_atomic_inc_orig2o(p, f, m) \ - os_atomic_add_orig2o(p, f, 1, m) -#define os_atomic_dec(p, m) \ - os_atomic_sub((p), 1, m) -#define os_atomic_dec_orig(p, m) \ - os_atomic_sub_orig((p), 1, m) -#define os_atomic_dec2o(p, f, m) \ - os_atomic_sub2o(p, f, 1, m) -#define os_atomic_dec_orig2o(p, f, m) \ - os_atomic_sub_orig2o(p, f, 1, m) - -#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ - bool _result = false; \ - typeof(p) _p = (p); \ - ov = os_atomic_load(_p, relaxed); \ - do { \ - __VA_ARGS__; \ - _result = os_atomic_cmpxchgvw(_p, ov, nv, &ov, m); \ - } while (os_unlikely(!_result)); \ - _result; \ - }) -#define os_atomic_rmw_loop2o(p, f, ov, nv, m, ...) \ - os_atomic_rmw_loop(&(p)->f, ov, nv, m, __VA_ARGS__) -#define os_atomic_rmw_loop_give_up_with_fence(m, expr) \ - ({ os_atomic_thread_fence(m); expr; __builtin_unreachable(); }) -#define os_atomic_rmw_loop_give_up(expr) \ - os_atomic_rmw_loop_give_up_with_fence(relaxed, expr) - -#define os_atomic_tsx_xacq_cmpxchgv(p, e, v, g) \ - os_atomic_cmpxchgv((p), (e), (v), (g), acquire) -#define os_atomic_tsx_xrel_store(p, v) \ - os_atomic_store(p, v, release) -#define os_atomic_tsx_xacq_cmpxchgv2o(p, f, e, v, g) \ - os_atomic_tsx_xacq_cmpxchgv(&(p)->f, (e), (v), (g)) -#define os_atomic_tsx_xrel_store2o(p, f, v) \ - os_atomic_tsx_xrel_store(&(p)->f, (v)) - -#if defined(__x86_64__) || defined(__i386__) -#pragma mark - -#pragma mark x86 - -#undef os_atomic_maximally_synchronizing_barrier -#ifdef __LP64__ -#define os_atomic_maximally_synchronizing_barrier() \ - ({ unsigned long _clbr; __asm__ __volatile__( \ - "cpuid" \ - : "=a" (_clbr) : "0" (0) : "rbx", "rcx", "rdx", "cc", "memory"); }) -#else -#ifdef __llvm__ -#define os_atomic_maximally_synchronizing_barrier() \ - ({ unsigned long _clbr; __asm__ __volatile__( \ - "cpuid" \ - : "=a" (_clbr) : "0" (0) : "ebx", "ecx", "edx", "cc", "memory"); }) -#else // gcc does not allow inline i386 asm to clobber ebx -#define os_atomic_maximally_synchronizing_barrier() \ - ({ unsigned long _clbr; __asm__ __volatile__( \ - "pushl %%ebx\n\t" \ - "cpuid\n\t" \ - "popl %%ebx" \ - : "=a" (_clbr) : "0" (0) : "ecx", "edx", "cc", "memory"); }) -#endif -#endif - - -#endif - - -#endif // __OS_EXPOSE_INTERNALS_INDIRECT__ - -#endif // __OS_ATOMIC__ diff --git a/include/os/internal/crashlog.h b/include/os/internal/crashlog.h deleted file mode 100644 index 41417ab..0000000 --- a/include/os/internal/crashlog.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2015 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -#ifndef __OS_INTERNAL_CRASHLOG__ -#define __OS_INTERNAL_CRASHLOG__ - -#ifndef __OS_EXPOSE_INTERNALS_INDIRECT__ -/* - * XXX /!\ WARNING /!\ XXX - * - * This header file describes INTERNAL interfaces to libplatform used by other - * libsystem targets, which are subject to change in future releases of OS X - * and iOS. Any applications relying on these interfaces WILL break. - * - * If you are not a libsystem target, you should NOT EVER use these headers. - * Not even a little. - * - * XXX /!\ WARNING /!\ XXX - */ -#error "Please #include instead of this file directly." -#else - - -#define _os_set_crash_log_cause_and_message(ac, msg) ((void)(ac), (void)(msg)) -#define _os_set_crash_log_message(msg) ((void)(msg)) -#define _os_set_crash_log_message_dynamic(msg) ((void)(msg)) - - -#endif // __OS_EXPOSE_INTERNALS_INDIRECT__ - -#endif // __OS_INTERNAL_CRASHLOG__ diff --git a/include/os/internal/internal_shared.h b/include/os/internal/internal_shared.h deleted file mode 100644 index 732c420..0000000 --- a/include/os/internal/internal_shared.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2015 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -#ifndef __OS_INTERNAL_SHARED__ -#define __OS_INTERNAL_SHARED__ - -#ifndef __OS_EXPOSE_INTERNALS__ -/* - * XXX /!\ WARNING /!\ XXX - * - * This header file describes INTERNAL interfaces to libplatform used by other - * libsystem targets, which are subject to change in future releases of Mac - * OS X and iOS. Any applications relying on these interfaces WILL break. - * - * If you are not a libsystem target, you should NOT EVER use these headers. - * Not even a little. - * - * XXX /!\ WARNING /!\ XXX - */ -#error "these internals are not for general use outside of libsystem" -#else - -#ifndef __OS_EXPOSE_INTERNALS_INDIRECT__ -#define __OS_EXPOSE_INTERNALS_INDIRECT__ -#endif - -#include -#include -#include -#if defined(__arm__) || defined(__arm64__) -#include -#endif - -#include -#include -#include -#include - -#endif // __OS_EXPOSE_INTERNALS__ - -#endif // __OS_INTERNAL_SHARED__ diff --git a/include/os/lock.h b/include/os/lock.h index 4af8449..abc5109 100644 --- a/include/os/lock.h +++ b/include/os/lock.h @@ -79,6 +79,7 @@ typedef struct os_unfair_lock_s { uint32_t _os_unfair_lock_opaque; } os_unfair_lock, *os_unfair_lock_t; +#ifndef OS_UNFAIR_LOCK_INIT #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L #define OS_UNFAIR_LOCK_INIT ((os_unfair_lock){0}) #elif defined(__cplusplus) && __cplusplus >= 201103L @@ -88,6 +89,7 @@ typedef struct os_unfair_lock_s { #else #define OS_UNFAIR_LOCK_INIT {0} #endif +#endif // OS_UNFAIR_LOCK_INIT /*! * @function os_unfair_lock_lock diff --git a/include/os/lock_private.h b/include/os/lock_private.h deleted file mode 100644 index 8c32ea6..0000000 --- a/include/os/lock_private.h +++ /dev/null @@ -1,650 +0,0 @@ -/* - * Copyright (c) 2013-2016 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -#ifndef __OS_LOCK_PRIVATE__ -#define __OS_LOCK_PRIVATE__ - -#include -#include -#include -#include -#include -#include -#include -#include - -OS_ASSUME_NONNULL_BEGIN - -/*! @header - * Low-level lock SPI - */ - -#define OS_LOCK_SPI_VERSION 20160406 - -/*! - * @typedef os_lock_t - * - * @abstract - * Pointer to one of the os_lock variants. - */ - -#define OS_LOCK_TYPE_STRUCT(type) const struct _os_lock_type_##type##_s -#define OS_LOCK_TYPE_REF(type) _os_lock_type_##type -#define OS_LOCK_TYPE_DECL(type) OS_LOCK_TYPE_STRUCT(type) OS_LOCK_TYPE_REF(type) - -#define OS_LOCK(type) os_lock_##type##_s -#define OS_LOCK_STRUCT(type) struct OS_LOCK(type) - -#if defined(__cplusplus) && __cplusplus >= 201103L - -#define OS_LOCK_DECL(type, size) \ - typedef OS_LOCK_STRUCT(type) : public OS_LOCK(base) { \ - private: \ - OS_LOCK_TYPE_STRUCT(type) * const osl_type OS_UNUSED; \ - uintptr_t _osl_##type##_opaque[size-1] OS_UNUSED; \ - public: \ - constexpr OS_LOCK(type)() : \ - osl_type(&OS_LOCK_TYPE_REF(type)), _osl_##type##_opaque() {} \ - } OS_LOCK(type) -#define OS_LOCK_INIT(type) {} - -typedef OS_LOCK_STRUCT(base) { - protected: - constexpr OS_LOCK(base)() {} -} *os_lock_t; - -#else - -#define OS_LOCK_DECL(type, size) \ - typedef OS_LOCK_STRUCT(type) { \ - OS_LOCK_TYPE_STRUCT(type) * const osl_type; \ - uintptr_t _osl_##type##_opaque[size-1]; \ - } OS_LOCK(type) - -#define OS_LOCK_INIT(type) { .osl_type = &OS_LOCK_TYPE_REF(type), } - -#ifndef OS_LOCK_T_MEMBER -#define OS_LOCK_T_MEMBER(type) OS_LOCK_STRUCT(type) *_osl_##type -#endif - -typedef OS_TRANSPARENT_UNION union { - OS_LOCK_T_MEMBER(base); - OS_LOCK_T_MEMBER(unfair); - OS_LOCK_T_MEMBER(nospin); - OS_LOCK_T_MEMBER(spin); - OS_LOCK_T_MEMBER(handoff); - OS_LOCK_T_MEMBER(eliding); - OS_LOCK_T_MEMBER(transactional); -} os_lock_t; - -#endif - -/*! - * @typedef os_lock_unfair_s - * - * @abstract - * os_lock variant equivalent to os_unfair_lock. Does not spin on contention but - * waits in the kernel to be woken up by an unlock. The lock value contains - * ownership information that the system may use to attempt to resolve priority - * inversions. - * - * @discussion - * Intended as a replacement for os_lock_spin_s or OSSpinLock. Like with - * OSSpinLock there is no attempt at fairness or lock ordering, e.g. an unlocker - * can potentially immediately reacquire the lock before a woken up waiter gets - * an opportunity to attempt to acquire the lock, so starvation is possibile. - * - * Must be initialized with OS_LOCK_UNFAIR_INIT - */ -__OSX_AVAILABLE(10.12) __IOS_AVAILABLE(10.0) -__TVOS_AVAILABLE(10.0) __WATCHOS_AVAILABLE(3.0) -OS_EXPORT OS_LOCK_TYPE_DECL(unfair); -OS_LOCK_DECL(unfair, 2); -#define OS_LOCK_UNFAIR_INIT OS_LOCK_INIT(unfair) - -/*! - * @typedef os_lock_nospin_s - * - * @abstract - * os_lock variant that does not spin on contention but waits in the kernel to - * be woken up by an unlock. No attempt to resolve priority inversions is made - * so os_unfair_lock or os_lock_unfair_s should generally be preferred. - * - * @discussion - * Intended as a replacement for os_lock_spin_s or OSSpinLock. Like with - * OSSpinLock there is no attempt at fairness or lock ordering, e.g. an unlocker - * can potentially immediately reacquire the lock before a woken up waiter gets - * an opportunity to attempt to acquire the lock, so starvation is possibile. - * - * Must be initialized with OS_LOCK_NOSPIN_INIT - */ -__OSX_AVAILABLE(10.12) __IOS_AVAILABLE(10.0) -__TVOS_AVAILABLE(10.0) __WATCHOS_AVAILABLE(3.0) -OS_EXPORT OS_LOCK_TYPE_DECL(nospin); -OS_LOCK_DECL(nospin, 2); -#define OS_LOCK_NOSPIN_INIT OS_LOCK_INIT(nospin) - -/*! - * @typedef os_lock_spin_s - * - * @abstract - * Deprecated os_lock variant that on contention starts by spinning trying to - * acquire the lock, then depressing the priority of the current thread and - * finally blocking the thread waiting for the lock to become available. - * Equivalent to OSSpinLock and equally not recommended, see discussion in - * libkern/OSAtomic.h headerdoc. - * - * @discussion - * Spinlocks are intended to be held only for very brief periods of time. The - * critical section must not make syscalls and should avoid touching areas of - * memory that may trigger a page fault, in particular if the critical section - * may be executing on threads of widely differing priorities or on a mix of - * IO-throttled and unthrottled threads. - * - * Must be initialized with OS_LOCK_SPIN_INIT - */ -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -OS_EXPORT OS_LOCK_TYPE_DECL(spin); -OS_LOCK_DECL(spin, 2); -#define OS_LOCK_SPIN_INIT OS_LOCK_INIT(spin) - -/*! - * @typedef os_lock_handoff_s - * - * @abstract - * os_lock variant that on contention hands off the current kernel thread to the - * lock-owning userspace thread (if it is not running), temporarily overriding - * its priority and IO throttle if necessary. - * - * @discussion - * Intended for use in limited circumstances where the critical section might - * be executing on threads of widely differing priorities or on a mix of - * IO-throttled and unthrottled threads where the ordinary os_lock_spin_s would - * be likely to encounter a priority inversion. - * - * IMPORTANT: This lock variant is NOT intended as a general replacement for all - * uses of os_lock_spin_s or OSSpinLock. - * - * Must be initialized with OS_LOCK_HANDOFF_INIT - */ -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -OS_EXPORT OS_LOCK_TYPE_DECL(handoff); -OS_LOCK_DECL(handoff, 2); -#define OS_LOCK_HANDOFF_INIT OS_LOCK_INIT(handoff) - - -#if !TARGET_OS_IPHONE -/*! - * @typedef os_lock_eliding_s - * - * @abstract - * os_lock variant that uses hardware lock elision support if available to allow - * multiple processors to concurrently execute a critical section as long as - * they don't perform conflicting operations on each other's data. In case of - * conflict, the lock reverts to exclusive operation and os_lock_spin_s behavior - * on contention (at potential extra cost for the aborted attempt at lock-elided - * concurrent execution). If hardware HLE support is not present, this lock - * variant behaves like os_lock_spin_s. - * - * @discussion - * IMPORTANT: Use of this lock variant MUST be extensively tested on hardware - * with HLE support to ensure the data access pattern and length of the critical - * section allows lock-elided execution to succeed frequently enough to offset - * the cost of any aborted concurrent execution. - * - * Must be initialized with OS_LOCK_ELIDING_INIT - */ -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_NA) -OS_EXPORT OS_LOCK_TYPE_DECL(eliding); -OS_LOCK_DECL(eliding, 8) OS_ALIGNED(64); -#define OS_LOCK_ELIDING_INIT OS_LOCK_INIT(eliding) - -/*! - * @typedef os_lock_transactional_s - * - * @abstract - * os_lock variant that uses hardware restricted transactional memory support if - * available to allow multiple processors to concurrently execute the critical - * section as a transactional region. If transactional execution aborts, the - * lock reverts to exclusive operation and os_lock_spin_s behavior on contention - * (at potential extra cost for the aborted attempt at transactional concurrent - * execution). If hardware RTM support is not present, this lock variant behaves - * like os_lock_eliding_s. - * - * @discussion - * IMPORTANT: Use of this lock variant MUST be extensively tested on hardware - * with RTM support to ensure the data access pattern and length of the critical - * section allows transactional execution to succeed frequently enough to offset - * the cost of any aborted transactions. - * - * Must be initialized with OS_LOCK_TRANSACTIONAL_INIT - */ -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_NA) -OS_EXPORT OS_LOCK_TYPE_DECL(transactional); -OS_LOCK_DECL(transactional, 8) OS_ALIGNED(64); -#define OS_LOCK_TRANSACTIONAL_INIT OS_LOCK_INIT(transactional) -#endif - -__BEGIN_DECLS - -/*! - * @function os_lock_lock - * - * @abstract - * Locks an os_lock variant. - * - * @param lock - * Pointer to one of the os_lock variants. - */ -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -void os_lock_lock(os_lock_t lock); - -/*! - * @function os_lock_trylock - * - * @abstract - * Locks an os_lock variant if it is not already locked. - * - * @param lock - * Pointer to one of the os_lock variants. - * - * @result - * Returns true if the lock was succesfully locked and false if the lock was - * already locked. - */ -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -bool os_lock_trylock(os_lock_t lock); - -/*! - * @function os_lock_unlock - * - * @abstract - * Unlocks an os_lock variant. - * - * @param lock - * Pointer to one of the os_lock variants. - */ -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -void os_lock_unlock(os_lock_t lock); - -/*! @group os_unfair_lock SPI - * - * @abstract - * Replacement for the deprecated OSSpinLock. Does not spin on contention but - * waits in the kernel to be woken up by an unlock. The opaque lock value - * contains thread ownership information that the system may use to attempt to - * resolve priority inversions. - * - * This lock must be unlocked from the same thread that locked it, attemps to - * unlock from a different thread will cause an assertion aborting the process. - * - * This lock must not be accessed from multiple processes or threads via shared - * or multiply-mapped memory, the lock implementation relies on the address of - * the lock value and owning process. - * - * @discussion - * As with OSSpinLock there is no attempt at fairness or lock ordering, e.g. an - * unlocker can potentially immediately reacquire the lock before a woken up - * waiter gets an opportunity to attempt to acquire the lock. This may be - * advantageous for performance reasons, but also makes starvation of waiters a - * possibility. - * - * Must be initialized with OS_UNFAIR_LOCK_INIT - */ - -/*! - * @typedef os_unfair_lock_options_t - * - * @const OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION - * This flag informs the runtime that the specified lock is used for data - * synchronization and that the lock owner is always able to make progress - * toward releasing the lock without the help of another thread in the same - * process. This hint will cause the workqueue subsystem to not create new - * threads to offset for threads waiting for the lock. - * - * When this flag is used, the code running under the critical section should - * be well known and under your control (Generally it should not call into - * framework code). - */ -OS_ENUM(os_unfair_lock_options, uint32_t, - OS_UNFAIR_LOCK_NONE - OS_UNFAIR_LOCK_AVAILABILITY = 0x00000000, - OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION - OS_UNFAIR_LOCK_AVAILABILITY = 0x00010000, -); - -/*! - * @function os_unfair_lock_lock_with_options - * - * @abstract - * Locks an os_unfair_lock. - * - * @param lock - * Pointer to an os_unfair_lock. - * - * @param options - * Options to alter the behavior of the lock. See os_unfair_lock_options_t. - */ -OS_UNFAIR_LOCK_AVAILABILITY -OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -void os_unfair_lock_lock_with_options(os_unfair_lock_t lock, - os_unfair_lock_options_t options); - -/*! - * @function os_unfair_lock_assert_owner - * - * @abstract - * Asserts that the calling thread is the current owner of the specified - * unfair lock. - * - * @discussion - * If the lock is currently owned by the calling thread, this function returns. - * - * If the lock is unlocked or owned by a different thread, this function - * asserts and terminates the process. - * - * @param lock - * Pointer to an os_unfair_lock. - */ -OS_UNFAIR_LOCK_AVAILABILITY -OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -void os_unfair_lock_assert_owner(os_unfair_lock_t lock); - -/*! - * @function os_unfair_lock_assert_not_owner - * - * @abstract - * Asserts that the calling thread is not the current owner of the specified - * unfair lock. - * - * @discussion - * If the lock is unlocked or owned by a different thread, this function - * returns. - * - * If the lock is currently owned by the current thread, this function asserts - * and terminates the process. - * - * @param lock - * Pointer to an os_unfair_lock. - */ -OS_UNFAIR_LOCK_AVAILABILITY -OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -void os_unfair_lock_assert_not_owner(os_unfair_lock_t lock); - -/*! @group os_unfair_lock variant for consumption by Libc - */ -OS_UNFAIR_LOCK_AVAILABILITY -OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -void os_unfair_lock_lock_with_options_4Libc(os_unfair_lock_t lock, - os_unfair_lock_options_t options); - -OS_UNFAIR_LOCK_AVAILABILITY -OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -void os_unfair_lock_unlock_4Libc(os_unfair_lock_t lock); - -__END_DECLS - -OS_ASSUME_NONNULL_END - -/*! @group Inline os_unfair_lock interfaces - * - * Inline versions of the os_unfair_lock fastpath. - * - * Intended exclusively for special highly performance-sensitive cases where the - * function calls to the os_unfair_lock API entrypoints add measurable overhead. - * - * Do not use in frameworks to implement synchronization API primitives that are - * exposed to developers, that would lead to false positives for that API from - * tools such as ThreadSanitizer. - * - * !!!!!!!!!!!!!!!!!!!!! WARNING WARNING WARNING WARNING !!!!!!!!!!!!!!!!!!!!! - * DO NOT USE IN CODE THAT IS NOT PART OF THE OPERATING SYSTEM OR THAT IS NOT - * REBUILT AS PART OF AN OS WORLDBUILD. YOU HAVE BEEN WARNED! - * !!!!!!!!!!!!!!!!!!!!! WARNING WARNING WARNING WARNING !!!!!!!!!!!!!!!!!!!!! - * - * Define OS_UNFAIR_LOCK_INLINE=1 to indicate that you have read the warning - * above and still wish to use these interfaces. - */ - -#if defined(OS_UNFAIR_LOCK_INLINE) && OS_UNFAIR_LOCK_INLINE - -#include - -#ifdef __cplusplus -extern "C++" { -#if !(__has_include() && __has_feature(cxx_atomic)) -#error Cannot use inline os_unfair_lock without and C++11 atomics -#endif -#include -typedef std::atomic _os_atomic_unfair_lock; -#define OSLOCK_STD(_a) std::_a -__BEGIN_DECLS -#else -#if !(__has_include() && __has_extension(c_atomic)) -#error Cannot use inline os_unfair_lock without and C11 atomics -#endif -#include -typedef _Atomic(os_unfair_lock) _os_atomic_unfair_lock; -#define OSLOCK_STD(_a) _a -#endif - -OS_ASSUME_NONNULL_BEGIN - -/*! - * @function os_unfair_lock_lock_inline - * - * @abstract - * Locks an os_unfair_lock. - * - * @param lock - * Pointer to an os_unfair_lock. - */ -OS_UNFAIR_LOCK_AVAILABILITY -OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL -void -os_unfair_lock_lock_inline(os_unfair_lock_t lock) -{ - if (!_pthread_has_direct_tsd()) return os_unfair_lock_lock(lock); - uintptr_t mts = (uintptr_t)_pthread_getspecific_direct( - _PTHREAD_TSD_SLOT_MACH_THREAD_SELF); - os_unfair_lock unlocked = OS_UNFAIR_LOCK_INIT, locked = { mts }; - if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( - (_os_atomic_unfair_lock*)lock, &unlocked, locked, - OSLOCK_STD(memory_order_acquire), - OSLOCK_STD(memory_order_relaxed))) { - return os_unfair_lock_lock(lock); - } -} - -/*! - * @function os_unfair_lock_lock_with_options_inline - * - * @abstract - * Locks an os_unfair_lock. - * - * @param lock - * Pointer to an os_unfair_lock. - * - * @param options - * Options to alter the behavior of the lock. See os_unfair_lock_options_t. - */ -OS_UNFAIR_LOCK_AVAILABILITY -OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL -void -os_unfair_lock_lock_with_options_inline(os_unfair_lock_t lock, - os_unfair_lock_options_t options) -{ - if (!_pthread_has_direct_tsd()) { - return os_unfair_lock_lock_with_options(lock, options); - } - uintptr_t mts = (uintptr_t)_pthread_getspecific_direct( - _PTHREAD_TSD_SLOT_MACH_THREAD_SELF); - os_unfair_lock unlocked = OS_UNFAIR_LOCK_INIT, locked = { mts }; - if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( - (_os_atomic_unfair_lock*)lock, &unlocked, locked, - OSLOCK_STD(memory_order_acquire), - OSLOCK_STD(memory_order_relaxed))) { - return os_unfair_lock_lock_with_options(lock, options); - } -} - -/*! - * @function os_unfair_lock_trylock_inline - * - * @abstract - * Locks an os_unfair_lock if it is not already locked. - * - * @discussion - * It is invalid to surround this function with a retry loop, if this function - * returns false, the program must be able to proceed without having acquired - * the lock, or it must call os_unfair_lock_lock_inline() instead. - * - * @param lock - * Pointer to an os_unfair_lock. - * - * @result - * Returns true if the lock was succesfully locked and false if the lock was - * already locked. - */ -OS_UNFAIR_LOCK_AVAILABILITY -OS_INLINE OS_ALWAYS_INLINE OS_WARN_RESULT OS_NONNULL_ALL -bool -os_unfair_lock_trylock_inline(os_unfair_lock_t lock) -{ - if (!_pthread_has_direct_tsd()) return os_unfair_lock_trylock(lock); - uintptr_t mts = (uintptr_t)_pthread_getspecific_direct( - _PTHREAD_TSD_SLOT_MACH_THREAD_SELF); - os_unfair_lock unlocked = OS_UNFAIR_LOCK_INIT, locked = { mts }; - return OSLOCK_STD(atomic_compare_exchange_strong_explicit)( - (_os_atomic_unfair_lock*)lock, &unlocked, locked, - OSLOCK_STD(memory_order_acquire), OSLOCK_STD(memory_order_relaxed)); -} - -/*! - * @function os_unfair_lock_unlock_inline - * - * @abstract - * Unlocks an os_unfair_lock. - * - * @param lock - * Pointer to an os_unfair_lock. - */ -OS_UNFAIR_LOCK_AVAILABILITY -OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL -void -os_unfair_lock_unlock_inline(os_unfair_lock_t lock) -{ - if (!_pthread_has_direct_tsd()) return os_unfair_lock_unlock(lock); - uintptr_t mts = (uintptr_t)_pthread_getspecific_direct( - _PTHREAD_TSD_SLOT_MACH_THREAD_SELF); - os_unfair_lock unlocked = OS_UNFAIR_LOCK_INIT, locked = { mts }; - if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( - (_os_atomic_unfair_lock*)lock, &locked, unlocked, - OSLOCK_STD(memory_order_release), - OSLOCK_STD(memory_order_relaxed))) { - return os_unfair_lock_unlock(lock); - } -} - -/*! @group os_unfair_lock no-TSD interfaces - * - * Like the above, but don't require being on a thread with valid TSD, so they - * can be called from injected mach-threads. The normal routines use the TSD - * value for mach_thread_self(), these routines use MACH_PORT_DEAD for the - * locked value instead. As a result, they will be unable to resolve priority - * inversions. - * - * This should only be used by libpthread. - * - */ -OS_UNFAIR_LOCK_AVAILABILITY -OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -void os_unfair_lock_lock_no_tsd_4libpthread(os_unfair_lock_t lock); - -OS_UNFAIR_LOCK_AVAILABILITY -OS_EXPORT OS_NOTHROW OS_NONNULL_ALL -void os_unfair_lock_unlock_no_tsd_4libpthread(os_unfair_lock_t lock); - -/*! - * @function os_unfair_lock_lock_inline_no_tsd_4libpthread - * - * @abstract - * Locks an os_unfair_lock, without requiring valid TSD. - * - * This should only be used by libpthread. - * - * @param lock - * Pointer to an os_unfair_lock. - */ -OS_UNFAIR_LOCK_AVAILABILITY -OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL -void -os_unfair_lock_lock_inline_no_tsd_4libpthread(os_unfair_lock_t lock) -{ - uintptr_t mts = (uintptr_t)MACH_PORT_DEAD; - os_unfair_lock unlocked = OS_UNFAIR_LOCK_INIT, locked = { mts }; - if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( - (_os_atomic_unfair_lock*)lock, &unlocked, locked, - OSLOCK_STD(memory_order_acquire), - OSLOCK_STD(memory_order_relaxed))) { - return os_unfair_lock_lock_no_tsd_4libpthread(lock); - } -} - -/*! - * @function os_unfair_lock_unlock_inline_no_tsd_4libpthread - * - * @abstract - * Unlocks an os_unfair_lock, without requiring valid TSD. - * - * This should only be used by libpthread. - * - * @param lock - * Pointer to an os_unfair_lock. - */ -OS_UNFAIR_LOCK_AVAILABILITY -OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL -void -os_unfair_lock_unlock_inline_no_tsd_4libpthread(os_unfair_lock_t lock) -{ - uintptr_t mts = (uintptr_t)MACH_PORT_DEAD; - os_unfair_lock unlocked = OS_UNFAIR_LOCK_INIT, locked = { mts }; - if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( - (_os_atomic_unfair_lock*)lock, &locked, unlocked, - OSLOCK_STD(memory_order_release), - OSLOCK_STD(memory_order_relaxed))) { - return os_unfair_lock_unlock_no_tsd_4libpthread(lock); - } -} - -OS_ASSUME_NONNULL_END - -#undef OSLOCK_STD -#ifdef __cplusplus -__END_DECLS -} // extern "C++" -#endif - -#endif // OS_UNFAIR_LOCK_INLINE - -#endif // __OS_LOCK_PRIVATE__ diff --git a/include/os/once_private.h b/include/os/once_private.h deleted file mode 100644 index c93cc14..0000000 --- a/include/os/once_private.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2008-2013 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -#ifndef __OS_ONCE_PRIVATE__ -#define __OS_ONCE_PRIVATE__ - -#include -#include - -OS_ASSUME_NONNULL_BEGIN - -__BEGIN_DECLS - -#define OS_ONCE_SPI_VERSION 20130313 - -OS_SWIFT_UNAVAILABLE("Swift has lazy init") -typedef long os_once_t; - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -OS_EXPORT OS_NONNULL1 OS_NONNULL3 OS_NOTHROW -OS_SWIFT_UNAVAILABLE("Swift has lazy init") -void -_os_once(os_once_t *predicate, void *_Nullable context, os_function_t function); - -OS_NONNULL1 OS_NONNULL3 OS_NOTHROW -__header_always_inline void -os_once(os_once_t *predicate, void *_Nullable context, os_function_t function) -{ - if (OS_EXPECT(*predicate, ~0l) != ~0l) { - _os_once(predicate, context, function); - OS_COMPILER_CAN_ASSUME(*predicate == ~0l); - } else { - os_compiler_barrier(); - } -} - -/* This SPI is *strictly* for the use of pthread_once only. This is not - * safe in general use of os_once. - */ -__OSX_AVAILABLE_STARTING(__MAC_10_9, __IPHONE_7_0) -OS_EXPORT OS_NONNULL1 OS_NOTHROW -OS_SWIFT_UNAVAILABLE("Swift has lazy init") -void -__os_once_reset(os_once_t *val); - -__END_DECLS - -OS_ASSUME_NONNULL_END - -#endif // __OS_ONCE_PRIVATE__ diff --git a/include/os/semaphore_private.h b/include/os/semaphore_private.h deleted file mode 100644 index d6bd728..0000000 --- a/include/os/semaphore_private.h +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2008-2013 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -#ifndef __OS_SEMAPHORE_PRIVATE__ -#define __OS_SEMAPHORE_PRIVATE__ - -#include -#include -#include -#include - -OS_ASSUME_NONNULL_BEGIN - -__BEGIN_DECLS - -#define OS_SEMAPHORE_SPI_VERSION 20130313 - -typedef uintptr_t os_semaphore_t; - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -OS_EXPORT OS_WARN_RESULT OS_NOTHROW -os_semaphore_t _os_semaphore_create(void); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -OS_EXPORT OS_NOTHROW -void _os_semaphore_dispose(os_semaphore_t); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -OS_EXPORT OS_NOTHROW -void _os_semaphore_wait(os_semaphore_t); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -OS_EXPORT OS_NOTHROW -void _os_semaphore_signal(os_semaphore_t); - -OS_WARN_RESULT OS_NOTHROW -__header_always_inline os_semaphore_t -os_get_cached_semaphore(void) -{ - os_semaphore_t sema; - sema = (os_semaphore_t)_os_tsd_get_direct(__TSD_SEMAPHORE_CACHE); - if (os_unlikely(!sema)) { - return _os_semaphore_create(); - } - _os_tsd_set_direct(__TSD_SEMAPHORE_CACHE, 0); - return sema; -} - -OS_NOTHROW -__header_always_inline void -os_put_cached_semaphore(os_semaphore_t sema) -{ - os_semaphore_t old_sema; - old_sema = (os_semaphore_t)_os_tsd_get_direct(__TSD_SEMAPHORE_CACHE); - _os_tsd_set_direct(__TSD_SEMAPHORE_CACHE, (void*)sema); - if (os_unlikely(old_sema)) { - return _os_semaphore_dispose(old_sema); - } -} - -OS_NOTHROW -__header_always_inline void -os_semaphore_wait(os_semaphore_t sema) -{ - return _os_semaphore_wait(sema); -} - -OS_NOTHROW -__header_always_inline void -os_semaphore_signal(os_semaphore_t sema) -{ - return _os_semaphore_signal(sema); -} - -__END_DECLS - -OS_ASSUME_NONNULL_END - -#endif // __OS_SEMAPHORE_PRIVATE__ diff --git a/include/platform/compat.h b/include/platform/compat.h deleted file mode 100644 index a9f59fd..0000000 --- a/include/platform/compat.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2013 Apple Inc. All rights reserved. - * - * @APPLE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_LICENSE_HEADER_END@ - */ - -#ifndef _PLATFORM_COMPAT_H_ -#define _PLATFORM_COMPAT_H_ - -#include - -__BEGIN_DECLS - -/* Helpers for other common non-primitive routines */ - -__header_always_inline -size_t -_platform_strlen(const char *s) { - const char *t = _platform_memchr(s, '\0', SIZE_MAX); - return (uintptr_t)t - (uintptr_t)s; -} - -__header_always_inline -size_t -_platform_strlcpy(char * restrict dst, const char * restrict src, size_t maxlen) { - const size_t srclen = _platform_strlen(src); - if (srclen < maxlen) { - _platform_memmove(dst, src, srclen+1); - } else if (maxlen != 0) { - _platform_memmove(dst, src, maxlen-1); - dst[maxlen-1] = '\0'; - } - return srclen; -} - -__END_DECLS - -/* Compat macros for primitives */ -#define bzero _platform_bzero -#define memchr _platform_memchr -#define memcmp _platform_memcmp -#define memmove _platform_memmove -#define memccpy _platform_memccpy -#define memset _platform_memset -#define memset_pattern4 _platform_memset_pattern4 -#define memset_pattern8 _platform_memset_pattern8 -#define memset_pattern16 _platform_memset_pattern16 -#define strchr _platform_strchr -#define strcmp _platform_strcmp -#define strncmp _platform_strncmp - -/* Compat macros for non-primitive helpers */ -#define strlcpy _platform_strlcpy -#define strlen _platform_strlen - -#endif /* _PLATFORM_COMPAT_H_ */ diff --git a/include/platform/introspection_private.h b/include/platform/introspection_private.h deleted file mode 100644 index f09ee58..0000000 --- a/include/platform/introspection_private.h +++ /dev/null @@ -1,447 +0,0 @@ -#ifndef __PLATFORM_INTROSPECTION_H__ -#define __PLATFORM_INTROSPECTION_H__ - -#include -#include -#include - -#include - -#include - -typedef struct platform_task_s *platform_task_t; -typedef struct platform_thread_s *platform_thread_t; - -/*! - * @typedef platform_thread_id_t - * - * @discussion - * The type of the 64-bit system-wide unique thread ID. - */ -typedef uint64_t platform_thread_id_t; - -/*! @functiongroup Tasks */ - -/*! - * @function platform_task_attach - * - * @discussion - * Attach to a process (specified by its mach task port) for debugging. This - * function creates a new task handle which must be disposed by a call to - * platform_task_detach(). - * - * @param task - * On output, a newly created task handle. - * - * @param target - * The mach task port of the target process. - * - * @return - * KERN_SUCCESS if the process was successfully attached, otherwise a mach - * error code. - */ -kern_return_t -platform_task_attach(platform_task_t *task, task_t target); - -/*! - * @function platform_task_detach - * - * @discussion - * Detaches from the target task and deallocates all memory associated with - * the task handle. - * - * @param task - * The task handle to detach. - * - * @return - * KERN_SUCCESS if the process was successfully detached, otherwise a mach - * error code. - */ -kern_return_t -platform_task_detach(platform_task_t task); - -/*! - * @function platform_task_is_64_bit - * - * @discussion - * Returns true if the target task is LP64. - * - * @param task - * A handle to the target task. - * - * @return - * true if the target task is LP64, otherwise false. - */ -bool -platform_task_is_64_bit(platform_task_t task); - -/*! - * @function platform_task_suspend_threads - * - * @discussion - * Suspends all the threads in the target task. This differs from task_suspend - * in that the task itself is not suspended, only the individual threads. While - * this suspension is in effect, any newly created threads will be created in - * a suspended state. The debuger may resume an individual thread for execution - * using platform_thread_resume() or evaluate an expression in the context of - * the task or a specific thread using platform_task_perform() and - * platform_thread_perform(), respectively. All threads in the task may be - * resumed with platform_task_resume_threads(). - * - * @param task - * A handle to the target task. - * - * @return - * KERN_SUCCESS if the threads were successfully suspended, otherwise a mach - * error code. - */ -kern_return_t -platform_task_suspend_threads(platform_task_t task); - -/*! - * @function platform_task_resume_threads - * - * @discussion - * Resumes the threads in the target task. See platform_task_suspend_threads(). - * - * @param task - * A handle to the target task. - */ -kern_return_t -platform_task_resume_threads(platform_task_t task); - -/*! - * @function platform_task_perform - * - * @discussion - * Performs the specified function on a newly created thread in the target task. - * This newly created thread will execute even if the threads in the task are - * suspended as the result of a call to platform_task_suspend_threads(). - * - * The function and context addresses are in the virtual address space of the - * target task. It is the responsiblity of the debugger to have previously - * mapped executable text and data at these addresses in the target task. - * - * @param task - * A handle to the target task. - * - * @param func_addr - * The address (in the virtual address space of the target task) of the - * function to perform. The function should be of type (void (*)(void *)) - * and will be passed the value of the data_addr parameter. - * - * @param data_addr - * The address (in the virtual address space of the target task) of the - * data to pass as a parameter to the function to perform. - * - * @return - * KERN_SUCCESS if the function was successfully performed, otherwise a mach - * error code. - */ -kern_return_t -platform_task_perform(platform_task_t task, - mach_vm_address_t func_addr, - mach_vm_address_t data_addr); - -/*! - * @function platform_task_update_threads - * - * @discussion - * Updates an internal representation of all threads in the target task. The - * list of threads may then be iterated using platform_task_copy_next_thread(). - * - * Calling this function resets any iteration currently in progress and a - * subsequent call to platform_task_copy_next_thread() will return the first - * thread in the list. - * - * @param task - * A handle to the target task. - * - * @return - * KERN_SUCCESS if the threads were successfully updated, otherwise a mach - * error code. - */ -kern_return_t -platform_task_update_threads(platform_task_t task); - -/*! - * @function platform_task_copy_next_thread - * - * @discussion - * Iterates the list of threads in the task. Returns a copied thread handle - * which must subsequently be released using platform_thread_release(). - * - * The platform_task_update_threads() function must be called before this - * function will return any thread handles. A NULL pointer is returned to - * signify the end of the list - * - * @param task - * A handle to the target task. - * - * @return - * A thread handle which must be released using platform_thread_release(), - * or NULL, signifying the end of the list. - */ -platform_thread_t -platform_task_copy_next_thread(platform_task_t task); - -/*! @functiongroup Threads */ - -/*! - * @function platform_thread_get_unique_id - * - * @discussion - * Returns the 64-bit system-wide unique ID of the target thread. - * - * @param thread - * A handle to the target thread. - * - * @return - * The unique ID of the thread. - */ -platform_thread_id_t -platform_thread_get_unique_id(platform_thread_t thread); - -/*! - * @function platform_thread_release - * - * @discussion - * Releases a thread handle obtained by platform_task_copy_next_thread(). - * - * @param thread - * The thread handle to release. - */ -void -platform_thread_release(platform_thread_t thread); - -/*! - * @function platform_thread_abort_safely - * - * @discussion - * Similar to thread_abort_safely(). - * - * @param thread - * A handle to the thread to signal. - * - * @return - * KERN_SUCCESS if the thread was successfully signaled, otherwise a mach - * error code. - */ -kern_return_t -platform_thread_abort_safely(platform_thread_t thread); - -/*! - * @function platform_thread_suspend - * - * @discussion - * Suspends execution of a thread similar to thread_suspend(). See also - * platform_task_suspend_threads(). - * - * @param thread - * A handle to the thread to suspend. - * - * @return - * KERN_SUCCESS if the thread was successfully suspended, otherwise a mach - * error code. - */ -kern_return_t -platform_thread_suspend(platform_thread_t thread); - -/*! - * @function platform_thread_resume - * - * @discussion - * Suspends execution of a thread similar to thread_suspend(). See also - * platform_task_suspend_threads() and platform_task_resume_threads(). - * - * @param thread - * A handle to the thread to resume. - * - * @return - * KERN_SUCCESS if the thread was successfully resumed, otherwise a mach - * error code. -*/ -kern_return_t -platform_thread_resume(platform_thread_t thread); - -/*! - * @function platform_thread_info - * - * @discussion - * Similar to thread_info. Supported flavor structures: - * - THREAD_BASIC_INFO: struct thread_basic_info - * - THREAD_IDENTIFIER_INFO: struct thread_identifier_info - * - * @param thread - * A handle to the target thread. - * - * @param flavor - * The desired thread info structure. - * - * @param info - * A pointer to storage where the thread info structure should be written. - * - * @param size - * On input, the size in bytes of the storage where the thread info structure - * is to be written. On output, the size of the thread info structure in bytes. - * - * @return - * KERN_SUCCESS if the function was successfully performed, otherwise a mach - * error code. - */ -kern_return_t -platform_thread_info(platform_thread_t thread, - thread_flavor_t flavor, - void *info, - size_t *size); - -/*! - * @function platform_thread_get_state - * - * @discussion - * Similar to thread_get_state. Supported flavor structures: - * - x86_THREAD_STATE32: struct ... - * - x86_FLOAT_STATE32: struct ... - * - x86_EXCEPTION_STATE32: struct ... - * - x86_DEBUG_STATE32: struct ... - * - x86_AVX_STATE32: struct ... - * - x86_THREAD_STATE64: struct ... - * - x86_FLOAT_STATE64: struct ... - * - x86_EXCEPTION_STATE64: struct ... - * - x86_DEBUG_STATE64: struct ... - * - x86_AVX_STATE64: struct ... - * - ARM_THREAD_STATE32: struct ... - * - ARM_FLOAT_STATE32: struct ... - * - ARM_EXCEPTION_STATE32: struct ... - * - ARM_DEBUG_STATE32: struct ... - * - ARM_THREAD_STATE64: struct ... - * - ARM_FLOAT_STATE64: struct ... - * - ARM_EXCEPTION_STATE64: struct ... - * - ARM_DEBUG_STATE64: struct ... - * - ... - * - * @param thread - * A handle to the target thread. - * - * @param flavor - * The desired thread state structure. - * - * @param state - * A pointer to storage where the thread state structure should be written. - * - * @param size - * On input, the size in bytes of the storage where the thread state structure - * is to be written. On output, the size of the thread state structure in bytes. - * - * @return - * KERN_SUCCESS if the function was successfully performed, otherwise a mach - * error code. - */ -kern_return_t -platform_thread_get_state(platform_thread_t thread, - thread_state_flavor_t flavor, - void *state, - size_t *size); - -/*! - * @function platform_thread_set_state - * - * @discussion - * Similar to thread_set_state. Supported flavor structures: - * - x86_THREAD_STATE32: struct ... - * - x86_FLOAT_STATE32: struct ... - * - x86_EXCEPTION_STATE32: struct ... - * - x86_DEBUG_STATE32: struct ... - * - x86_AVX_STATE32: struct ... - * - x86_THREAD_STATE64: struct ... - * - x86_FLOAT_STATE64: struct ... - * - x86_EXCEPTION_STATE64: struct ... - * - x86_DEBUG_STATE64: struct ... - * - x86_AVX_STATE64: struct ... - * - ARM_THREAD_STATE32: struct ... - * - ARM_FLOAT_STATE32: struct ... - * - ARM_EXCEPTION_STATE32: struct ... - * - ARM_DEBUG_STATE32: struct ... - * - ARM_THREAD_STATE64: struct ... - * - ARM_FLOAT_STATE64: struct ... - * - ARM_EXCEPTION_STATE64: struct ... - * - ARM_DEBUG_STATE64: struct ... - * - ... - * - * @param thread - * A handle to the target thread. - * - * @param flavor - * The desired thread state structure. - * - * @param state - * A pointer to storage where the thread state structure should be written. - * - * @param size - * The size of the thread state structure in bytes. - * - * @return - * KERN_SUCCESS if the function was successfully performed, otherwise a mach - * error code. - */ -kern_return_t -platform_thread_set_state(platform_thread_t thread, - thread_state_flavor_t flavor, - const void *state, - size_t size); - -/*! - * @function platform_thread_perform - * - * @discussion - * Performs the specified function within the context of the specified thread - * in the target task. The function will execute in the style of an - * asynchronous signal handler even if the thread is suspended as the result - * of a call to platform_task_suspend_threads() or platform_thread_suspend(). - * The original state of the thread will be restored when the function returns. - * - * The function and context addresses are in the virtual address space of the - * target task. It is the responsiblity of the debugger to have previously - * mapped executable text and data at these addresses in the target task. - * - * See also platform_task_perform(). - * - * @param thread - * A handle to the target thread. - * - * @param func_addr - * The address (in the virtual address space of the target task) of the - * function to perform. The function should be of type (void (*)(void *)) - * and will be passed the value of the data_addr parameter. - * - * @param data_addr - * The address (in the virtual address space of the target task) of the - * data to pass as a parameter to the function to perform. - * - * @return - * KERN_SUCCESS if the function was successfully performed, otherwise a mach - * error code. - */ -kern_return_t -platform_thread_perform(platform_thread_t thread, - mach_vm_address_t func_addr, - mach_vm_address_t data_addr); - -/*! - * @function platform_thread_get_pthread - * - * @discussion - * Returns a pointer to mapped memory which represents the pthread_t of the - * target process. Any embedded pointers will need to be mapped into the current - * process space on a case-by-case basis. - * - * @param thread - * A handle to the target thread. - * - * @return - * A valid pointer. - */ -const void * -platform_thread_get_pthread(platform_thread_t thread); - -#endif // __PLATFORM_INTROSPECTION_H__ diff --git a/include/platform/string.h b/include/platform/string.h deleted file mode 100644 index c4d1c3f..0000000 --- a/include/platform/string.h +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2013 Apple Inc. All rights reserved. - * - * @APPLE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_LICENSE_HEADER_END@ - */ - -#ifndef _PLATFORM_STRING_H_ -#define _PLATFORM_STRING_H_ - -#include -#include -#include - -#include -#include - -#define _PLATFORM_OPTIMIZED_BZERO 0 -#define _PLATFORM_OPTIMIZED_MEMCCPY 0 -#define _PLATFORM_OPTIMIZED_MEMCHR 0 -#define _PLATFORM_OPTIMIZED_MEMCMP 0 -#define _PLATFORM_OPTIMIZED_MEMMOVE 0 -#define _PLATFORM_OPTIMIZED_MEMSET 0 -#define _PLATFORM_OPTIMIZED_MEMSET_PATTERN4 0 -#define _PLATFORM_OPTIMIZED_MEMSET_PATTERN8 0 -#define _PLATFORM_OPTIMIZED_MEMSET_PATTERN16 0 -#define _PLATFORM_OPTIMIZED_STRCHR 0 -#define _PLATFORM_OPTIMIZED_STRCMP 0 -#define _PLATFORM_OPTIMIZED_STRNCMP 0 - -/* Primitives used to implement C memory and string routines */ - -__BEGIN_DECLS - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -void -_platform_bzero(void *s, size_t n); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -void * -_platform_memchr(const void *s, int c, size_t n); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -int -_platform_memcmp(const void *s1, const void *s2, size_t n); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -void * -_platform_memmove(void *dst, const void *src, size_t n); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -void * -_platform_memccpy(void *restrict dst, const void *restrict src, int c, size_t n); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -void * -_platform_memset(void *b, int c, size_t len); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -void -_platform_memset_pattern4(void *b, const void *pattern4, size_t len); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -void -_platform_memset_pattern8(void *b, const void *pattern8, size_t len); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -void -_platform_memset_pattern16(void *b, const void *pattern16, size_t len); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -char * -_platform_strchr(const char *s, int c); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -int -_platform_strcmp(const char *s1, const char *s2); - -__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) -int -_platform_strncmp(const char *s1, const char *s2, size_t n); - -__END_DECLS - -#endif /* _PLATFORM_STRING_H_ */ diff --git a/internal/os/internal.h b/internal/os/internal.h index 4d7a083..1784216 100644 --- a/internal/os/internal.h +++ b/internal/os/internal.h @@ -52,47 +52,16 @@ #define __LIBPLATFORM_INTERNAL_CRASH__(rc, msg) \ __OS_CRASH__(rc, "BUG IN LIBPLATFORM: " msg) - #define __OS_EXPOSE_INTERNALS__ 1 #include "os/internal/internal_shared.h" #include "yield.h" -#if !VARIANT_NO_RESOLVERS -#if defined(_ARM_ARCH_7) && !defined(__ARM_ARCH_7S__) -#if OS_ATOMIC_UP -#define OS_VARIANT_SELECTOR up -#else -#define OS_VARIANT_SELECTOR mp -#endif -#endif -#if !defined(OS_VARIANT_SELECTOR) && defined(VARIANT_NO_RESOLVERS) -// forced up variant for no-barrier OSAtomics -#define OS_ATOMIC_NO_BARRIER_ONLY 1 -#define OS_VARIANT_SELECTOR up -#endif -#if (defined(_ARM_ARCH_7) || defined(__arm64__)) && \ - (!defined(OS_ATOMIC_WFE) && !OS_ATOMIC_UP) -#define OS_ATOMIC_WFE 0 -#endif -#ifdef OS_ATOMIC_WFE -#if OS_ATOMIC_WFE -#define OS_LOCK_VARIANT_SELECTOR wfe -#else -#define OS_LOCK_VARIANT_SELECTOR mp -#endif -#endif -#endif // !VARIANT_NO_RESOLVERS +#define OS_NOEXPORT extern __attribute__((__visibility__("hidden"))) #define OS_VARIANT(f, v) OS_CONCAT(f, OS_CONCAT($VARIANT$, v)) -#ifdef OS_VARIANT_SELECTOR -#define _OS_ATOMIC_ALIAS_PRIVATE_EXTERN(n) \ - ".private_extern _" OS_STRINGIFY(n) "\n\t" -#define OS_ATOMIC_EXPORT -#else #define _OS_ATOMIC_ALIAS_PRIVATE_EXTERN(n) #define OS_ATOMIC_EXPORT OS_EXPORT -#endif #define _OS_ATOMIC_ALIAS_GLOBL(n) \ ".globl _" OS_STRINGIFY(n) "\n\t" #ifdef __thumb__ @@ -122,21 +91,4 @@ __VA_ARGS__ \ } -#define _OS_VARIANT_UPMP_RESOLVER(s, v) \ - _OS_VARIANT_RESOLVER(s, v, \ - uint32_t *_c = (void*)(uintptr_t)_COMM_PAGE_CPU_CAPABILITIES; \ - if (*_c & kUP) { \ - extern void OS_VARIANT(s, up)(void); \ - return &OS_VARIANT(s, up); \ - } else { \ - extern void OS_VARIANT(s, mp)(void); \ - return &OS_VARIANT(s, mp); \ - }) - -#define OS_VARIANT_UPMP_RESOLVER(s) \ - _OS_VARIANT_UPMP_RESOLVER(s, default) - -#define OS_VARIANT_UPMP_RESOLVER_INTERNAL(s) \ - _OS_VARIANT_UPMP_RESOLVER(s, hidden) - #endif // __OS_INTERNAL_H__ diff --git a/man/atomic.3 b/man/atomic.3 index eee6e43..8a8d246 100644 --- a/man/atomic.3 +++ b/man/atomic.3 @@ -56,6 +56,7 @@ will return a ptr to mary. .Sh RETURN VALUES The dequeue operation returns the most recently enqueued element, or NULL if the list in empty. .Sh SEE ALSO +.Xr stdatomic 3 , .Xr atomic_deprecated 3 , .Xr spinlock_deprecated 3 .Sh HISTORY diff --git a/man/atomic_deprecated.3 b/man/atomic_deprecated.3 index 8db9557..138da5c 100644 --- a/man/atomic_deprecated.3 +++ b/man/atomic_deprecated.3 @@ -1,4 +1,4 @@ -.Dd May 26, 2004 +.Dd Mar 7, 2016 .Dt ATOMIC_DEPRECATED 3 .Os Darwin .Sh NAME @@ -132,7 +132,8 @@ These are deprecated interfaces for atomic and synchronization operations, provided for compatibility with legacy code. New code should use the C11 .In stdatomic.h -interfaces. +interfaces described in +.Xr stdatomic 3 . .Ef .Pp These functions are thread and multiprocessor safe. For each function, there @@ -224,6 +225,7 @@ The compare-and-swap operations return true if the comparison was equal, ie if the swap occured. The bit test and set/clear operations return the original value of the bit. .Sh SEE ALSO +.Xr stdatomic 3 , .Xr atomic 3 , .Xr spinlock_deprecated 3 .Sh HISTORY diff --git a/man/manpages.lst b/man/manpages.lst index 898750d..dd74ea3 100644 --- a/man/manpages.lst +++ b/man/manpages.lst @@ -11,3 +11,5 @@ makecontext.3 makecontext.3 swapcontext.3 setjmp.3 setjmp.3 _longjmp.3 _setjmp.3 longjmp.3 longjmperr.3 longjmperror.3 siglongjmp.3 sigsetjmp.3 spinlock_deprecated.3 spinlock_deprecated.3 OSSpinLockLock.3 OSSpinLockTry.3 OSSpinLockUnlock.3 ucontext.3 ucontext.3 +stdatomic.3 stdatomic.3 ATOMIC_VAR_INIT.3 atomic_compare_exchange_strong.3 atomic_compare_exchange_strong_explicit.3 atomic_compare_exchange_weak.3 atomic_compare_exchange_weak_explicit.3 atomic_exchange.3 atomic_exchange_explicit.3 atomic_fetch_add.3 atomic_fetch_add_explicit.3 atomic_fetch_and.3 atomic_fetch_and_explicit.3 atomic_fetch_or.3 atomic_fetch_or_explicit.3 atomic_fetch_sub.3 atomic_fetch_sub_explicit.3 atomic_fetch_xor.3 atomic_fetch_xor_explicit.3 atomic_init.3 atomic_is_lock_free.3 atomic_load.3 atomic_load_explicit.3 atomic_store.3 atomic_store_explicit.3 + diff --git a/man/stdatomic.3 b/man/stdatomic.3 new file mode 100644 index 0000000..b4ae319 --- /dev/null +++ b/man/stdatomic.3 @@ -0,0 +1,274 @@ +.\" Copyright (c) 2011 Ed Schouten +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd December 27, 2011 +.Dt stdatomic 3 +.Os +.Sh NAME +.Nm ATOMIC_VAR_INIT , +.Nm atomic_init , +.Nm atomic_load , +.Nm atomic_store , +.Nm atomic_exchange , +.Nm atomic_compare_exchange_strong , +.Nm atomic_compare_exchange_weak , +.Nm atomic_fetch_add , +.Nm atomic_fetch_and , +.Nm atomic_fetch_or , +.Nm atomic_fetch_sub , +.Nm atomic_fetch_xor , +.Nm atomic_is_lock_free +.Nd type-generic atomic operations +.Sh SYNOPSIS +.In stdatomic.h +.Pp +.Vt _Atomic(T) Va v No = \*[Fn-font]ATOMIC_VAR_INIT\*[No-font] Ns Pq Fa c ; +.Vt _Atomic T Va v No = \*[Fn-font]ATOMIC_VAR_INIT\*[No-font] Ns Pq Fa c ; +.Ft void +.Fn atomic_init "_Atomic(T) *object" "T value" +.Ft T +.Fn atomic_load "_Atomic(T) *object" +.Ft T +.Fn atomic_load_explicit "_Atomic(T) *object" "memory_order order" +.Ft void +.Fn atomic_store "_Atomic(T) *object" "T desired" +.Ft void +.Fn atomic_store_explicit "_Atomic(T) *object" "T desired" "memory_order order" +.Ft T +.Fn atomic_exchange "_Atomic(T) *object" "T desired" +.Ft T +.Fn atomic_exchange_explicit "_Atomic(T) *object" "T desired" "memory_order order" +.Ft _Bool +.Fn atomic_compare_exchange_strong "_Atomic(T) *object" "T *expected" "T desired" +.Ft _Bool +.Fn atomic_compare_exchange_strong_explicit "_Atomic(T) *object" "T *expected" "T desired" "memory_order success" "memory_order failure" +.Ft _Bool +.Fn atomic_compare_exchange_weak "_Atomic(T) *object" "T *expected" "T desired" +.Ft _Bool +.Fn atomic_compare_exchange_weak_explicit "_Atomic(T) *object" "T *expected" "T desired" "memory_order success" "memory_order failure" +.Ft T +.Fn atomic_fetch_add "_Atomic(T) *object" "T operand" +.Ft T +.Fn atomic_fetch_add_explicit "_Atomic(T) *object" "T operand" "memory_order order" +.Ft T +.Fn atomic_fetch_and "_Atomic(T) *object" "T operand" +.Ft T +.Fn atomic_fetch_and_explicit "_Atomic(T) *object" "T operand" "memory_order order" +.Ft T +.Fn atomic_fetch_or "_Atomic(T) *object" "T operand" +.Ft T +.Fn atomic_fetch_or_explicit "_Atomic(T) *object" "T operand" "memory_order order" +.Ft T +.Fn atomic_fetch_sub "_Atomic(T) *object" "T operand" +.Ft T +.Fn atomic_fetch_sub_explicit "_Atomic(T) *object" "T operand" "memory_order order" +.Ft T +.Fn atomic_fetch_xor "_Atomic(T) *object" "T operand" +.Ft T +.Fn atomic_fetch_xor_explicit "_Atomic(T) *object" "T operand" "memory_order order" +.Ft _Bool +.Fn atomic_is_lock_free "const _Atomic(T) *object" +.Sh DESCRIPTION +The header +.In stdatomic.h +provides type-generic operations on atomic operations. +.Pp +Atomic variables are declared using the +.Vt _Atomic() +type specifier or the +.Vt _Atomic +type qualifier. +Such variables are not type-compatible with their non-atomic +counterparts and may have different alignment. +.Pp +Operations on atomic variables that do not use the +.Fn atomic_ +interfaces, including compound assignment operations, will behave as if the +.Pf non- Fn _explicit +versions of those interfaces had been used. +.Pp +The +.Fn atomic_init +operation initializes the atomic variable +.Fa object +with +.Fa value . +Atomic variables can be initialized while being declared using +.Fn ATOMIC_VAR_INIT . +.Pp +The +.Fn atomic_load +operation returns the value of atomic variable +.Fa object . +The +.Fn atomic_store +operation sets the atomic variable +.Fa object +to the +.Fa desired +value. +.Pp +The +.Fn atomic_exchange +operation combines the behaviour of +.Fn atomic_load +and +.Fn atomic_store . +It sets the atomic variable +.Fa object +to the desired +.Fa value +and returns the original contents of the atomic variable. +.Pp +The +.Fn atomic_compare_exchange_strong +operation stores the +.Fa desired +value into atomic variable +.Fa object , +but only if the atomic variable is equal to the +.Fa expected +value. +Upon success, the operation returns +.Dv true . +Upon failure, the +.Fa expected +value is overwritten with the contents of the atomic variable and +.Dv false +is returned. +.Pp +The +.Fn atomic_compare_exchange_weak +operation is identical to +.Fn atomic_compare_exchange_strong , +but is allowed to fail even if atomic variable +.Fa object +is equal to the +.Fa expected +value. When an +.Fn atomic_compare_exchange +operation is in a loop, the weak version will yield better performance on +some platforms. When +.Fn atomic_compare_exchange_weak +would require a loop and +.Fn atomic_compare_exchange_strong +would not, the strong version is preferable. +.Pp +The +.Fn atomic_fetch_add +operation adds the value +.Fa operand +to atomic variable +.Fa object +and returns the original contents of the atomic variable. +.Pp +The +.Fn atomic_fetch_and +operation applies the +.Em and +operator to atomic variable +.Fa object +and value +.Fa operand +and stores the result into +.Fa object , +while returning the original contents of the atomic variable. +.Pp +The +.Fn atomic_fetch_or +operation applies the +.Em or +operator to atomic variable +.Fa object +and value +.Fa operand +and stores the result into +.Fa object , +while returning the original contents of the atomic variable. +.Pp +The +.Fn atomic_fetch_sub +operation subtracts the value +.Fa operand +from atomic variable +.Fa object +and returns the original contents of the atomic variable. +.Pp +The +.Fn atomic_fetch_xor +operation applies the +.Em xor +operator to atomic variable +.Fa object +and value +.Fa operand +and stores the result into +.Fa object , +while returning the original contents of the atomic variable. +.Pp +The +.Fn atomic_is_lock_free +operation returns whether atomic variable +.Fa object +uses locks to implement atomic operations. +.Sh MEMORY ORDER +C11 defines a memory model that may allow for the reordering of operations in the +absence of fences or explicit memory ordering operations. +The +.Pf non- Fn _explicit +interfaces use the strictest available memory order: sequential +consistency. The +.Fn _explicit +interfaces allow for configuration of the memory order operation which is +present. The types of available memory order operations are explained in +more detail in +.St -isoC-2011 . +.Pp +The +.Fa order +parameter of the +.Fn _explicit +interfaces can have one of the following values: +.Bl -tag -width memory_order_relaxed +.It Dv memory_order_relaxed +Operation does not order memory. +.It Dv memory_order_consume +Performs a consume operation. +.It Dv memory_order_acquire +Performs an acquire operation. +.It Dv memory_order_release +Performs a release operation. +.It Dv memory_order_acq_rel +Performs both an acquire and a release operation. +.It Dv memory_order_seq_cst +Provides sequential consistency. +.El +.Sh SEE ALSO +.Xr atomic 3 , +.Xr pthread 3 +.Sh STANDARDS +These interfaces conform to +.St -isoC-2011 . diff --git a/private/_simple.h b/private/_simple.h new file mode 100644 index 0000000..2ae8384 --- /dev/null +++ b/private/_simple.h @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2006, 2010, 2013 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#ifndef _SYSTEM_SIMPLE_H_ +#define _SYSTEM_SIMPLE_H_ + +#include +#include + +#include + +typedef void *_SIMPLE_STRING; +typedef const char *_esc_func(unsigned char); + +__BEGIN_DECLS +/* + * A simplified vfprintf variant. The format string is interpreted with + * arguments from the va_list, and the results are written to the given + * file descriptor. + */ +void _simple_vdprintf(int __fd, const char *__fmt, va_list __ap) __printflike(2, 0); + +/* + * A simplified fprintf variant. The format string is interpreted with + * arguments from the variable argument list, and the results are written + * to the given file descriptor. + */ +void _simple_dprintf(int __fd, const char *__fmt, ...) __printflike(2, 3); + +/* + * A simplified string allocate routine. Pass the opaque pointer to structure + * to _simple_*sprintf() routines. Use _simple_string() to retrieve the + * current string (the string is guaranteed to be null terminated only on + * the call to _simple_string()). Use _simple_sfree() to free the structure + * and string memory. + */ +_SIMPLE_STRING _simple_salloc(void); + +/* + * The format string is interpreted with arguments from the va_list, and the + * results are appended to the string maintained by the opaque structure, as + * returned by a previous call to _simple_salloc(). Non-zero is returned on + * out-of-memory error. + */ +int _simple_vsprintf(_SIMPLE_STRING __b, const char *__fmt, va_list __ap) __printflike(2, 0); + +/* + * The format string is interpreted with arguments from the variable argument + * list, and the results are appended to the string maintained by the opaque + * structure, as returned by a previous call to _simple_salloc(). Non-zero is + * returned on out-of-memory error. + */ +int _simple_sprintf(_SIMPLE_STRING __b, const char *__fmt, ...) __printflike(2, 3); + +/* + * Like _simple_vsprintf(), except __esc is a function to call on each + * character; the function returns NULL if the character should be passed + * as is, otherwise, the returned character string is used instead. + */ +int _simple_vesprintf(_SIMPLE_STRING __b, _esc_func __esc, const char *__fmt, va_list __ap) __printflike(3, 0); + +/* + * Like _simple_sprintf(), except __esc is a function to call on each + * character; the function returns NULL if the character should be passed + * as is, otherwise, the returned character string is used instead. + */ +int _simple_esprintf(_SIMPLE_STRING __b, _esc_func __esc, const char *__fmt, ...) __printflike(3, 4); + +/* + * Return the null terminated string from the opaque structure, as returned + * by a previous call to _simple_salloc(). + */ +char *_simple_string(_SIMPLE_STRING __b); + +/* + * Reposition the pointer to the first null in the buffer. After a call to + * _simple_string, the buffer can be modified, and shrunk. + */ +void _simple_sresize(_SIMPLE_STRING __b); + +/* + * Append the null-terminated string to the string associated with the opaque + * structure. Non-zero is returned on out-of-memory error. + */ +int _simple_sappend(_SIMPLE_STRING __b, const char *__str); + +/* + * Like _simple_sappend(), except __esc is a function to call on each + * character; the function returns NULL if the character should be passed + * as is, otherwise, the returned character string is used instead. + */ +int _simple_esappend(_SIMPLE_STRING __b, _esc_func __esc, const char *__str); + +/* + * Write the string associated with the opaque structure to the file descriptor. + */ +void _simple_put(_SIMPLE_STRING __b, int __fd); + +/* + * Write the string associated with the opaque structure and a trailing newline, + * to the file descriptor. + */ +void _simple_putline(_SIMPLE_STRING __b, int __fd); + +/* + * Free the opaque structure, and the associated string. + */ +void _simple_sfree(_SIMPLE_STRING __b); + +/* + * Simplified ASL log interface; does not use malloc. Unfortunately, this + * requires knowledge of the format used by ASL. + */ +#ifndef ASL_LEVEL_DEBUG +#define ASL_LEVEL_EMERG 0 +#define ASL_LEVEL_ALERT 1 +#define ASL_LEVEL_CRIT 2 +#define ASL_LEVEL_ERR 3 +#define ASL_LEVEL_WARNING 4 +#define ASL_LEVEL_NOTICE 5 +#define ASL_LEVEL_INFO 6 +#define ASL_LEVEL_DEBUG 7 +#endif + +void _simple_asl_log(int __level, const char *__facility, const char *__message); +void _simple_asl_log_prog(int level, const char *facility, const char *message, const char *progname); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +_SIMPLE_STRING _simple_asl_msg_new(void); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +void _simple_asl_msg_set(_SIMPLE_STRING __b, const char *__key, const char *__val); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +void _simple_asl_send(_SIMPLE_STRING __b); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +const char *_simple_getenv(const char *envp[], const char *var); + +__END_DECLS + +#endif /* _SYSTEM_SIMPLE_H_ */ diff --git a/private/libkern/module.modulemap b/private/libkern/module.modulemap index 72b650c..8b13789 100644 --- a/private/libkern/module.modulemap +++ b/private/libkern/module.modulemap @@ -1,6 +1 @@ -module libkernPrivate [system] [extern_c] { - module OSAtomic { - header "OSAtomic.h" - export Darwin.libkern.OSAtomic - } -} + diff --git a/private/os/alloc_once_impl.h b/private/os/alloc_once_impl.h new file mode 100644 index 0000000..d821e18 --- /dev/null +++ b/private/os/alloc_once_impl.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2012-2013 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef __OS_ALLOC_ONCE_IMPL__ +#define __OS_ALLOC_ONCE_IMPL__ + +#ifndef __OS_ALLOC_INDIRECT__ +#error "Please include instead of this file directly." +#endif + +#include +#include +#include +#include + +__BEGIN_DECLS + +#define OS_ALLOC_SPI_VERSION 20120430 + +#define OS_ALLOC_ONCE_KEY_MAX 100 + +typedef os_once_t os_alloc_token_t; +struct _os_alloc_once_s { + os_alloc_token_t once; + void *ptr; +}; + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_6_0) +extern struct _os_alloc_once_s _os_alloc_once_table[]; + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_6_0) +OS_EXPORT OS_NONNULL1 +void* +_os_alloc_once(struct _os_alloc_once_s *slot, size_t sz, os_function_t init); + +/* + * The region allocated by os_alloc_once is 0-filled when initially + * returned (or handed off to the initializer). + */ +OS_WARN_RESULT OS_NOTHROW OS_CONST +__header_always_inline void* +os_alloc_once(os_alloc_token_t token, size_t sz, os_function_t init) +{ + struct _os_alloc_once_s *slot = &_os_alloc_once_table[token]; + if (OS_EXPECT(slot->once, ~0l) != ~0l) { + void *ptr = _os_alloc_once(slot, sz, init); + OS_COMPILER_CAN_ASSUME(slot->once == ~0l); + return ptr; + } + return slot->ptr; +} + +__END_DECLS + +#endif // __OS_ALLOC_ONCE_IMPL__ diff --git a/private/os/base_private.h b/private/os/base_private.h new file mode 100644 index 0000000..2d38266 --- /dev/null +++ b/private/os/base_private.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2008-2013 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __OS_BASE_PRIVATE__ +#define __OS_BASE_PRIVATE__ + +#include + +#ifndef os_fastpath +#define os_fastpath(x) ((__typeof__(x))OS_EXPECT((long)(x), ~0l)) +#endif +#ifndef os_slowpath +#define os_slowpath(x) ((__typeof__(x))OS_EXPECT((long)(x), 0l)) +#endif +#ifndef os_likely +#define os_likely(x) OS_EXPECT(!!(x), 1) +#endif +#ifndef os_unlikely +#define os_unlikely(x) OS_EXPECT(!!(x), 0) +#endif + +#endif // __OS_BASE_PRIVATE__ diff --git a/private/os/internal/atomic.h b/private/os/internal/atomic.h new file mode 100644 index 0000000..1d717a6 --- /dev/null +++ b/private/os/internal/atomic.h @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2008-2013 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __OS_INTERNAL_ATOMIC__ +#define __OS_INTERNAL_ATOMIC__ + +#ifndef __OS_EXPOSE_INTERNALS_INDIRECT__ +/* + * Use c11 or c++11 std::atomic from instead + * + * XXX /!\ WARNING /!\ XXX + * + * This header file describes INTERNAL interfaces to libplatform used by other + * libsystem targets, which are subject to change in future releases of OS X + * and iOS. Any applications relying on these interfaces WILL break. + * + * If you are not a libsystem target, you should NOT EVER use these headers. + * Not even a little. + * + * XXX /!\ WARNING /!\ XXX + */ +#error "Please #include instead of this file directly." +#else + +// generate error during codegen +#define _os_atomic_unimplemented() \ + ({ __asm__(".err unimplemented"); }) + +#pragma mark - +#pragma mark memory_order + +typedef enum _os_atomic_memory_order { + _os_atomic_memory_order_relaxed, + _os_atomic_memory_order_consume, + _os_atomic_memory_order_acquire, + _os_atomic_memory_order_release, + _os_atomic_memory_order_acq_rel, + _os_atomic_memory_order_seq_cst, + _os_atomic_memory_order_ordered, + _os_atomic_memory_order_dependency, +} _os_atomic_memory_order; + +#if !OS_ATOMIC_UP + +#define os_atomic_memory_order_relaxed _os_atomic_memory_order_relaxed +#define os_atomic_memory_order_acquire _os_atomic_memory_order_acquire +#define os_atomic_memory_order_release _os_atomic_memory_order_release +#define os_atomic_memory_order_acq_rel _os_atomic_memory_order_acq_rel +#define os_atomic_memory_order_seq_cst _os_atomic_memory_order_seq_cst +#define os_atomic_memory_order_ordered _os_atomic_memory_order_seq_cst +#define os_atomic_memory_order_dependency _os_atomic_memory_order_acquire + +#else // OS_ATOMIC_UP + +#define os_atomic_memory_order_relaxed _os_atomic_memory_order_relaxed +#define os_atomic_memory_order_acquire _os_atomic_memory_order_relaxed +#define os_atomic_memory_order_release _os_atomic_memory_order_relaxed +#define os_atomic_memory_order_acq_rel _os_atomic_memory_order_relaxed +#define os_atomic_memory_order_seq_cst _os_atomic_memory_order_relaxed +#define os_atomic_memory_order_ordered _os_atomic_memory_order_relaxed +#define os_atomic_memory_order_dependency _os_atomic_memory_order_relaxed + +#endif // OS_ATOMIC_UP + +#pragma mark - +#pragma mark c11 + +#if !__has_extension(c_atomic) +#error "Please use a C11 compiler" +#endif + +#define os_atomic(type) type _Atomic + +#define _os_atomic_c11_atomic(p) \ + ((typeof(*(p)) _Atomic *)(p)) + +// This removes the _Atomic and volatile qualifiers on the type of *p +#define _os_atomic_basetypeof(p) \ + typeof(__c11_atomic_load(_os_atomic_c11_atomic(p), \ + _os_atomic_memory_order_relaxed)) + +#define _os_atomic_baseptr(p) \ + ((_os_atomic_basetypeof(p) *)(p)) + +#define _os_atomic_barrier(m) \ + __c11_atomic_thread_fence(os_atomic_memory_order_##m) +#define os_atomic_load(p, m) \ + __c11_atomic_load(_os_atomic_c11_atomic(p), os_atomic_memory_order_##m) +#define os_atomic_store(p, v, m) \ + __c11_atomic_store(_os_atomic_c11_atomic(p), v, \ + os_atomic_memory_order_##m) +#define os_atomic_xchg(p, v, m) \ + __c11_atomic_exchange(_os_atomic_c11_atomic(p), v, \ + os_atomic_memory_order_##m) +#define os_atomic_cmpxchg(p, e, v, m) \ + ({ _os_atomic_basetypeof(p) _r = (e); \ + __c11_atomic_compare_exchange_strong(_os_atomic_c11_atomic(p), \ + &_r, v, os_atomic_memory_order_##m, os_atomic_memory_order_relaxed); }) +#define os_atomic_cmpxchgv(p, e, v, g, m) \ + ({ _os_atomic_basetypeof(p) _r = (e); _Bool _b = \ + __c11_atomic_compare_exchange_strong(_os_atomic_c11_atomic(p), \ + &_r, v, os_atomic_memory_order_##m, os_atomic_memory_order_relaxed); \ + *(g) = _r; _b; }) +#define os_atomic_cmpxchgvw(p, e, v, g, m) \ + ({ _os_atomic_basetypeof(p) _r = (e); _Bool _b = \ + __c11_atomic_compare_exchange_weak(_os_atomic_c11_atomic(p), \ + &_r, v, os_atomic_memory_order_##m, os_atomic_memory_order_relaxed); \ + *(g) = _r; _b; }) +#define _os_atomic_c11_op(p, v, m, o, op) \ + ({ _os_atomic_basetypeof(p) _v = (v), _r = \ + __c11_atomic_fetch_##o(_os_atomic_c11_atomic(p), _v, \ + os_atomic_memory_order_##m); (typeof(_r))(_r op _v); }) +#define _os_atomic_c11_op_orig(p, v, m, o, op) \ + __c11_atomic_fetch_##o(_os_atomic_c11_atomic(p), v, \ + os_atomic_memory_order_##m) + +#define os_atomic_add(p, v, m) \ + _os_atomic_c11_op((p), (v), m, add, +) +#define os_atomic_add_orig(p, v, m) \ + _os_atomic_c11_op_orig((p), (v), m, add, +) +#define os_atomic_sub(p, v, m) \ + _os_atomic_c11_op((p), (v), m, sub, -) +#define os_atomic_sub_orig(p, v, m) \ + _os_atomic_c11_op_orig((p), (v), m, sub, -) +#define os_atomic_and(p, v, m) \ + _os_atomic_c11_op((p), (v), m, and, &) +#define os_atomic_and_orig(p, v, m) \ + _os_atomic_c11_op_orig((p), (v), m, and, &) +#define os_atomic_or(p, v, m) \ + _os_atomic_c11_op((p), (v), m, or, |) +#define os_atomic_or_orig(p, v, m) \ + _os_atomic_c11_op_orig((p), (v), m, or, |) +#define os_atomic_xor(p, v, m) \ + _os_atomic_c11_op((p), (v), m, xor, ^) +#define os_atomic_xor_orig(p, v, m) \ + _os_atomic_c11_op_orig((p), (v), m, xor, ^) + +#define os_atomic_force_dependency_on(p, e) (p) +#define os_atomic_load_with_dependency_on(p, e) \ + os_atomic_load(os_atomic_force_dependency_on(p, e), relaxed) +#define os_atomic_load_with_dependency_on2o(p, f, e) \ + os_atomic_load_with_dependency_on(&(p)->f, e) + +#pragma mark - +#pragma mark generic + +#define os_atomic_thread_fence(m) _os_atomic_barrier(m) + +#define os_atomic_load2o(p, f, m) \ + os_atomic_load(&(p)->f, m) +#define os_atomic_store2o(p, f, v, m) \ + os_atomic_store(&(p)->f, (v), m) +#define os_atomic_xchg2o(p, f, v, m) \ + os_atomic_xchg(&(p)->f, (v), m) +#define os_atomic_cmpxchg2o(p, f, e, v, m) \ + os_atomic_cmpxchg(&(p)->f, (e), (v), m) +#define os_atomic_cmpxchgv2o(p, f, e, v, g, m) \ + os_atomic_cmpxchgv(&(p)->f, (e), (v), (g), m) +#define os_atomic_cmpxchgvw2o(p, f, e, v, g, m) \ + os_atomic_cmpxchgvw(&(p)->f, (e), (v), (g), m) +#define os_atomic_add2o(p, f, v, m) \ + os_atomic_add(&(p)->f, (v), m) +#define os_atomic_add_orig2o(p, f, v, m) \ + os_atomic_add_orig(&(p)->f, (v), m) +#define os_atomic_sub2o(p, f, v, m) \ + os_atomic_sub(&(p)->f, (v), m) +#define os_atomic_sub_orig2o(p, f, v, m) \ + os_atomic_sub_orig(&(p)->f, (v), m) +#define os_atomic_and2o(p, f, v, m) \ + os_atomic_and(&(p)->f, (v), m) +#define os_atomic_and_orig2o(p, f, v, m) \ + os_atomic_and_orig(&(p)->f, (v), m) +#define os_atomic_or2o(p, f, v, m) \ + os_atomic_or(&(p)->f, (v), m) +#define os_atomic_or_orig2o(p, f, v, m) \ + os_atomic_or_orig(&(p)->f, (v), m) +#define os_atomic_xor2o(p, f, v, m) \ + os_atomic_xor(&(p)->f, (v), m) +#define os_atomic_xor_orig2o(p, f, v, m) \ + os_atomic_xor_orig(&(p)->f, (v), m) + +#define os_atomic_inc(p, m) \ + os_atomic_add((p), 1, m) +#define os_atomic_inc_orig(p, m) \ + os_atomic_add_orig((p), 1, m) +#define os_atomic_inc2o(p, f, m) \ + os_atomic_add2o(p, f, 1, m) +#define os_atomic_inc_orig2o(p, f, m) \ + os_atomic_add_orig2o(p, f, 1, m) +#define os_atomic_dec(p, m) \ + os_atomic_sub((p), 1, m) +#define os_atomic_dec_orig(p, m) \ + os_atomic_sub_orig((p), 1, m) +#define os_atomic_dec2o(p, f, m) \ + os_atomic_sub2o(p, f, 1, m) +#define os_atomic_dec_orig2o(p, f, m) \ + os_atomic_sub_orig2o(p, f, 1, m) + +#define os_atomic_rmw_loop(p, ov, nv, m, ...) ({ \ + bool _result = false; \ + typeof(p) _p = (p); \ + ov = os_atomic_load(_p, relaxed); \ + do { \ + __VA_ARGS__; \ + _result = os_atomic_cmpxchgvw(_p, ov, nv, &ov, m); \ + } while (os_unlikely(!_result)); \ + _result; \ + }) +#define os_atomic_rmw_loop2o(p, f, ov, nv, m, ...) \ + os_atomic_rmw_loop(&(p)->f, ov, nv, m, __VA_ARGS__) +#define os_atomic_rmw_loop_give_up_with_fence(m, expr) \ + ({ os_atomic_thread_fence(m); expr; __builtin_unreachable(); }) +#define os_atomic_rmw_loop_give_up(expr) \ + os_atomic_rmw_loop_give_up_with_fence(relaxed, expr) + +#define os_atomic_tsx_xacq_cmpxchgv(p, e, v, g) \ + os_atomic_cmpxchgv((p), (e), (v), (g), acquire) +#define os_atomic_tsx_xrel_store(p, v) \ + os_atomic_store(p, v, release) +#define os_atomic_tsx_xacq_cmpxchgv2o(p, f, e, v, g) \ + os_atomic_tsx_xacq_cmpxchgv(&(p)->f, (e), (v), (g)) +#define os_atomic_tsx_xrel_store2o(p, f, v) \ + os_atomic_tsx_xrel_store(&(p)->f, (v)) + +#if defined(__x86_64__) || defined(__i386__) +#pragma mark - +#pragma mark x86 + + +#endif + + +#endif // __OS_EXPOSE_INTERNALS_INDIRECT__ + +#endif // __OS_ATOMIC__ diff --git a/private/os/internal/crashlog.h b/private/os/internal/crashlog.h new file mode 100644 index 0000000..41417ab --- /dev/null +++ b/private/os/internal/crashlog.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2015 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __OS_INTERNAL_CRASHLOG__ +#define __OS_INTERNAL_CRASHLOG__ + +#ifndef __OS_EXPOSE_INTERNALS_INDIRECT__ +/* + * XXX /!\ WARNING /!\ XXX + * + * This header file describes INTERNAL interfaces to libplatform used by other + * libsystem targets, which are subject to change in future releases of OS X + * and iOS. Any applications relying on these interfaces WILL break. + * + * If you are not a libsystem target, you should NOT EVER use these headers. + * Not even a little. + * + * XXX /!\ WARNING /!\ XXX + */ +#error "Please #include instead of this file directly." +#else + + +#define _os_set_crash_log_cause_and_message(ac, msg) ((void)(ac), (void)(msg)) +#define _os_set_crash_log_message(msg) ((void)(msg)) +#define _os_set_crash_log_message_dynamic(msg) ((void)(msg)) + + +#endif // __OS_EXPOSE_INTERNALS_INDIRECT__ + +#endif // __OS_INTERNAL_CRASHLOG__ diff --git a/private/os/internal/internal_shared.h b/private/os/internal/internal_shared.h new file mode 100644 index 0000000..c0e4e81 --- /dev/null +++ b/private/os/internal/internal_shared.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2015 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __OS_INTERNAL_SHARED__ +#define __OS_INTERNAL_SHARED__ + +#ifndef __OS_EXPOSE_INTERNALS__ +/* + * XXX /!\ WARNING /!\ XXX + * + * This header file describes INTERNAL interfaces to libplatform used by other + * libsystem targets, which are subject to change in future releases of Mac + * OS X and iOS. Any applications relying on these interfaces WILL break. + * + * If you are not a libsystem target, you should NOT EVER use these headers. + * Not even a little. + * + * XXX /!\ WARNING /!\ XXX + */ +#error "these internals are not for general use outside of libsystem" +#else + +#ifndef __OS_EXPOSE_INTERNALS_INDIRECT__ +#define __OS_EXPOSE_INTERNALS_INDIRECT__ +#endif + +#include +#include +#include +#if defined(__arm__) || defined(__arm64__) +#include +#endif + +#include +#include +#include +#include + + +#endif // __OS_EXPOSE_INTERNALS__ + +#endif // __OS_INTERNAL_SHARED__ diff --git a/private/os/lock.h b/private/os/lock.h new file mode 100644 index 0000000..74bc470 --- /dev/null +++ b/private/os/lock.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __OS_LOCK_PRIVATE_UNFAIR_LOCK_INIT__ +#define __OS_LOCK_PRIVATE_UNFAIR_LOCK_INIT__ + + +#include_next + +#endif // __OS_LOCK_PRIVATE_UNFAIR_LOCK_INIT__ diff --git a/private/os/lock_private.h b/private/os/lock_private.h new file mode 100644 index 0000000..00329e2 --- /dev/null +++ b/private/os/lock_private.h @@ -0,0 +1,649 @@ +/* + * Copyright (c) 2013-2016 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __OS_LOCK_PRIVATE__ +#define __OS_LOCK_PRIVATE__ + +#include +#include +#include +#include +#include +#include +#include +#include + +OS_ASSUME_NONNULL_BEGIN + +/*! @header + * Low-level lock SPI + */ + +#define OS_LOCK_SPI_VERSION 20160406 + +/*! + * @typedef os_lock_t + * + * @abstract + * Pointer to one of the os_lock variants. + */ + +#define OS_LOCK_TYPE_STRUCT(type) const struct _os_lock_type_##type##_s +#define OS_LOCK_TYPE_REF(type) _os_lock_type_##type +#define OS_LOCK_TYPE_DECL(type) OS_LOCK_TYPE_STRUCT(type) OS_LOCK_TYPE_REF(type) + +#define OS_LOCK(type) os_lock_##type##_s +#define OS_LOCK_STRUCT(type) struct OS_LOCK(type) + +#if defined(__cplusplus) && __cplusplus >= 201103L + +#define OS_LOCK_DECL(type, size) \ + typedef OS_LOCK_STRUCT(type) : public OS_LOCK(base) { \ + private: \ + OS_LOCK_TYPE_STRUCT(type) * const osl_type OS_UNUSED; \ + uintptr_t _osl_##type##_opaque[size-1] OS_UNUSED; \ + public: \ + constexpr OS_LOCK(type)() : \ + osl_type(&OS_LOCK_TYPE_REF(type)), _osl_##type##_opaque() {} \ + } OS_LOCK(type) +#define OS_LOCK_INIT(type) {} + +typedef OS_LOCK_STRUCT(base) { + protected: + constexpr OS_LOCK(base)() {} +} *os_lock_t; + +#else + +#define OS_LOCK_DECL(type, size) \ + typedef OS_LOCK_STRUCT(type) { \ + OS_LOCK_TYPE_STRUCT(type) * const osl_type; \ + uintptr_t _osl_##type##_opaque[size-1]; \ + } OS_LOCK(type) + +#define OS_LOCK_INIT(type) { .osl_type = &OS_LOCK_TYPE_REF(type), } + +#ifndef OS_LOCK_T_MEMBER +#define OS_LOCK_T_MEMBER(type) OS_LOCK_STRUCT(type) *_osl_##type +#endif + +typedef OS_TRANSPARENT_UNION union { + OS_LOCK_T_MEMBER(base); + OS_LOCK_T_MEMBER(unfair); + OS_LOCK_T_MEMBER(nospin); + OS_LOCK_T_MEMBER(spin); + OS_LOCK_T_MEMBER(handoff); + OS_LOCK_T_MEMBER(eliding); + OS_LOCK_T_MEMBER(transactional); +} os_lock_t; + +#endif + +/*! + * @typedef os_lock_unfair_s + * + * @abstract + * os_lock variant equivalent to os_unfair_lock. Does not spin on contention but + * waits in the kernel to be woken up by an unlock. The lock value contains + * ownership information that the system may use to attempt to resolve priority + * inversions. + * + * @discussion + * Intended as a replacement for os_lock_spin_s or OSSpinLock. Like with + * OSSpinLock there is no attempt at fairness or lock ordering, e.g. an unlocker + * can potentially immediately reacquire the lock before a woken up waiter gets + * an opportunity to attempt to acquire the lock, so starvation is possibile. + * + * Must be initialized with OS_LOCK_UNFAIR_INIT + */ +__OSX_AVAILABLE(10.12) __IOS_AVAILABLE(10.0) +__TVOS_AVAILABLE(10.0) __WATCHOS_AVAILABLE(3.0) +OS_EXPORT OS_LOCK_TYPE_DECL(unfair); +OS_LOCK_DECL(unfair, 2); +#define OS_LOCK_UNFAIR_INIT OS_LOCK_INIT(unfair) + +/*! + * @typedef os_lock_nospin_s + * + * @abstract + * os_lock variant that does not spin on contention but waits in the kernel to + * be woken up by an unlock. No attempt to resolve priority inversions is made + * so os_unfair_lock or os_lock_unfair_s should generally be preferred. + * + * @discussion + * Intended as a replacement for os_lock_spin_s or OSSpinLock. Like with + * OSSpinLock there is no attempt at fairness or lock ordering, e.g. an unlocker + * can potentially immediately reacquire the lock before a woken up waiter gets + * an opportunity to attempt to acquire the lock, so starvation is possibile. + * + * Must be initialized with OS_LOCK_NOSPIN_INIT + */ +__OSX_AVAILABLE(10.12) __IOS_AVAILABLE(10.0) +__TVOS_AVAILABLE(10.0) __WATCHOS_AVAILABLE(3.0) +OS_EXPORT OS_LOCK_TYPE_DECL(nospin); +OS_LOCK_DECL(nospin, 2); +#define OS_LOCK_NOSPIN_INIT OS_LOCK_INIT(nospin) + +/*! + * @typedef os_lock_spin_s + * + * @abstract + * Deprecated os_lock variant that on contention starts by spinning trying to + * acquire the lock, then depressing the priority of the current thread and + * finally blocking the thread waiting for the lock to become available. + * Equivalent to OSSpinLock and equally not recommended, see discussion in + * libkern/OSAtomic.h headerdoc. + * + * @discussion + * Spinlocks are intended to be held only for very brief periods of time. The + * critical section must not make syscalls and should avoid touching areas of + * memory that may trigger a page fault, in particular if the critical section + * may be executing on threads of widely differing priorities or on a mix of + * IO-throttled and unthrottled threads. + * + * Must be initialized with OS_LOCK_SPIN_INIT + */ +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +OS_EXPORT OS_LOCK_TYPE_DECL(spin); +OS_LOCK_DECL(spin, 2); +#define OS_LOCK_SPIN_INIT OS_LOCK_INIT(spin) + +/*! + * @typedef os_lock_handoff_s + * + * @abstract + * os_lock variant that on contention hands off the current kernel thread to the + * lock-owning userspace thread (if it is not running), temporarily overriding + * its priority and IO throttle if necessary. + * + * @discussion + * Intended for use in limited circumstances where the critical section might + * be executing on threads of widely differing priorities or on a mix of + * IO-throttled and unthrottled threads where the ordinary os_lock_spin_s would + * be likely to encounter a priority inversion. + * + * IMPORTANT: This lock variant is NOT intended as a general replacement for all + * uses of os_lock_spin_s or OSSpinLock. + * + * Must be initialized with OS_LOCK_HANDOFF_INIT + */ +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +OS_EXPORT OS_LOCK_TYPE_DECL(handoff); +OS_LOCK_DECL(handoff, 2); +#define OS_LOCK_HANDOFF_INIT OS_LOCK_INIT(handoff) + + +#if !TARGET_OS_IPHONE +/*! + * @typedef os_lock_eliding_s + * + * @abstract + * os_lock variant that uses hardware lock elision support if available to allow + * multiple processors to concurrently execute a critical section as long as + * they don't perform conflicting operations on each other's data. In case of + * conflict, the lock reverts to exclusive operation and os_lock_spin_s behavior + * on contention (at potential extra cost for the aborted attempt at lock-elided + * concurrent execution). If hardware HLE support is not present, this lock + * variant behaves like os_lock_spin_s. + * + * @discussion + * IMPORTANT: Use of this lock variant MUST be extensively tested on hardware + * with HLE support to ensure the data access pattern and length of the critical + * section allows lock-elided execution to succeed frequently enough to offset + * the cost of any aborted concurrent execution. + * + * Must be initialized with OS_LOCK_ELIDING_INIT + */ +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_NA) +OS_EXPORT OS_LOCK_TYPE_DECL(eliding); +OS_LOCK_DECL(eliding, 8) OS_ALIGNED(64); +#define OS_LOCK_ELIDING_INIT OS_LOCK_INIT(eliding) + +/*! + * @typedef os_lock_transactional_s + * + * @abstract + * os_lock variant that uses hardware restricted transactional memory support if + * available to allow multiple processors to concurrently execute the critical + * section as a transactional region. If transactional execution aborts, the + * lock reverts to exclusive operation and os_lock_spin_s behavior on contention + * (at potential extra cost for the aborted attempt at transactional concurrent + * execution). If hardware RTM support is not present, this lock variant behaves + * like os_lock_eliding_s. + * + * @discussion + * IMPORTANT: Use of this lock variant MUST be extensively tested on hardware + * with RTM support to ensure the data access pattern and length of the critical + * section allows transactional execution to succeed frequently enough to offset + * the cost of any aborted transactions. + * + * Must be initialized with OS_LOCK_TRANSACTIONAL_INIT + */ +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_NA) +OS_EXPORT OS_LOCK_TYPE_DECL(transactional); +OS_LOCK_DECL(transactional, 8) OS_ALIGNED(64); +#define OS_LOCK_TRANSACTIONAL_INIT OS_LOCK_INIT(transactional) +#endif + +__BEGIN_DECLS + +/*! + * @function os_lock_lock + * + * @abstract + * Locks an os_lock variant. + * + * @param lock + * Pointer to one of the os_lock variants. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +OS_EXPORT OS_NOTHROW OS_NONNULL_ALL +void os_lock_lock(os_lock_t lock); + +/*! + * @function os_lock_trylock + * + * @abstract + * Locks an os_lock variant if it is not already locked. + * + * @param lock + * Pointer to one of the os_lock variants. + * + * @result + * Returns true if the lock was succesfully locked and false if the lock was + * already locked. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +OS_EXPORT OS_NOTHROW OS_NONNULL_ALL +bool os_lock_trylock(os_lock_t lock); + +/*! + * @function os_lock_unlock + * + * @abstract + * Unlocks an os_lock variant. + * + * @param lock + * Pointer to one of the os_lock variants. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +OS_EXPORT OS_NOTHROW OS_NONNULL_ALL +void os_lock_unlock(os_lock_t lock); + +/*! @group os_unfair_lock SPI + * + * @abstract + * Replacement for the deprecated OSSpinLock. Does not spin on contention but + * waits in the kernel to be woken up by an unlock. The opaque lock value + * contains thread ownership information that the system may use to attempt to + * resolve priority inversions. + * + * This lock must be unlocked from the same thread that locked it, attemps to + * unlock from a different thread will cause an assertion aborting the process. + * + * This lock must not be accessed from multiple processes or threads via shared + * or multiply-mapped memory, the lock implementation relies on the address of + * the lock value and owning process. + * + * @discussion + * As with OSSpinLock there is no attempt at fairness or lock ordering, e.g. an + * unlocker can potentially immediately reacquire the lock before a woken up + * waiter gets an opportunity to attempt to acquire the lock. This may be + * advantageous for performance reasons, but also makes starvation of waiters a + * possibility. + * + * Must be initialized with OS_UNFAIR_LOCK_INIT + */ + +/*! + * @typedef os_unfair_lock_options_t + * + * @const OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION + * This flag informs the runtime that the specified lock is used for data + * synchronization and that the lock owner is always able to make progress + * toward releasing the lock without the help of another thread in the same + * process. This hint will cause the workqueue subsystem to not create new + * threads to offset for threads waiting for the lock. + * + * When this flag is used, the code running under the critical section should + * be well known and under your control (Generally it should not call into + * framework code). + */ +OS_ENUM(os_unfair_lock_options, uint32_t, + OS_UNFAIR_LOCK_NONE + OS_UNFAIR_LOCK_AVAILABILITY = 0x00000000, + OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION + OS_UNFAIR_LOCK_AVAILABILITY = 0x00010000, +); + +/*! + * @function os_unfair_lock_lock_with_options + * + * @abstract + * Locks an os_unfair_lock. + * + * @param lock + * Pointer to an os_unfair_lock. + * + * @param options + * Options to alter the behavior of the lock. See os_unfair_lock_options_t. + */ +OS_UNFAIR_LOCK_AVAILABILITY +OS_EXPORT OS_NOTHROW OS_NONNULL_ALL +void os_unfair_lock_lock_with_options(os_unfair_lock_t lock, + os_unfair_lock_options_t options); + +/*! + * @function os_unfair_lock_assert_owner + * + * @abstract + * Asserts that the calling thread is the current owner of the specified + * unfair lock. + * + * @discussion + * If the lock is currently owned by the calling thread, this function returns. + * + * If the lock is unlocked or owned by a different thread, this function + * asserts and terminates the process. + * + * @param lock + * Pointer to an os_unfair_lock. + */ +OS_UNFAIR_LOCK_AVAILABILITY +OS_EXPORT OS_NOTHROW OS_NONNULL_ALL +void os_unfair_lock_assert_owner(os_unfair_lock_t lock); + +/*! + * @function os_unfair_lock_assert_not_owner + * + * @abstract + * Asserts that the calling thread is not the current owner of the specified + * unfair lock. + * + * @discussion + * If the lock is unlocked or owned by a different thread, this function + * returns. + * + * If the lock is currently owned by the current thread, this function asserts + * and terminates the process. + * + * @param lock + * Pointer to an os_unfair_lock. + */ +OS_UNFAIR_LOCK_AVAILABILITY +OS_EXPORT OS_NOTHROW OS_NONNULL_ALL +void os_unfair_lock_assert_not_owner(os_unfair_lock_t lock); + +/*! @group os_unfair_lock no-TSD interfaces + * + * Like the above, but don't require being on a thread with valid TSD, so they + * can be called from injected mach-threads. The normal routines use the TSD + * value for mach_thread_self(), these routines use MACH_PORT_DEAD for the + * locked value instead. As a result, they will be unable to resolve priority + * inversions. + * + * This should only be used by libpthread. + * + */ +OS_UNFAIR_LOCK_AVAILABILITY +OS_EXPORT OS_NOTHROW OS_NONNULL_ALL +void os_unfair_lock_lock_no_tsd_4libpthread(os_unfair_lock_t lock); + +OS_UNFAIR_LOCK_AVAILABILITY +OS_EXPORT OS_NOTHROW OS_NONNULL_ALL +void os_unfair_lock_unlock_no_tsd_4libpthread(os_unfair_lock_t lock); + +__END_DECLS + +OS_ASSUME_NONNULL_END + +/*! @group Inline os_unfair_lock interfaces + * + * Inline versions of the os_unfair_lock fastpath. + * + * Intended exclusively for special highly performance-sensitive cases where the + * function calls to the os_unfair_lock API entrypoints add measurable overhead. + * + * Do not use in frameworks to implement synchronization API primitives that are + * exposed to developers, that would lead to false positives for that API from + * tools such as ThreadSanitizer. + * + * !!!!!!!!!!!!!!!!!!!!! WARNING WARNING WARNING WARNING !!!!!!!!!!!!!!!!!!!!! + * DO NOT USE IN CODE THAT IS NOT PART OF THE OPERATING SYSTEM OR THAT IS NOT + * REBUILT AS PART OF AN OS WORLDBUILD. YOU HAVE BEEN WARNED! + * !!!!!!!!!!!!!!!!!!!!! WARNING WARNING WARNING WARNING !!!!!!!!!!!!!!!!!!!!! + * + * Define OS_UNFAIR_LOCK_INLINE=1 to indicate that you have read the warning + * above and still wish to use these interfaces. + */ + +#if defined(OS_UNFAIR_LOCK_INLINE) && OS_UNFAIR_LOCK_INLINE + +#include + +#ifdef __cplusplus +extern "C++" { +#if !(__has_include() && __has_feature(cxx_atomic)) +#error Cannot use inline os_unfair_lock without and C++11 atomics +#endif +#include +typedef std::atomic _os_atomic_unfair_lock; +#define OSLOCK_STD(_a) std::_a +__BEGIN_DECLS +#else +#if !(__has_include() && __has_extension(c_atomic)) +#error Cannot use inline os_unfair_lock without and C11 atomics +#endif +#include +typedef _Atomic(os_unfair_lock) _os_atomic_unfair_lock; +#define OSLOCK_STD(_a) _a +#endif + +OS_ASSUME_NONNULL_BEGIN + +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#define OS_UNFAIR_LOCK_UNLOCKED ((os_unfair_lock){0}) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#define OS_UNFAIR_LOCK_UNLOCKED (os_unfair_lock{}) +#elif defined(__cplusplus) +#define OS_UNFAIR_LOCK_UNLOCKED (os_unfair_lock()) +#else +#define OS_UNFAIR_LOCK_UNLOCKED {0} +#endif + +/*! + * @function os_unfair_lock_lock_inline + * + * @abstract + * Locks an os_unfair_lock. + * + * @param lock + * Pointer to an os_unfair_lock. + */ +OS_UNFAIR_LOCK_AVAILABILITY +OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL +void +os_unfair_lock_lock_inline(os_unfair_lock_t lock) +{ + if (!_pthread_has_direct_tsd()) return os_unfair_lock_lock(lock); + uintptr_t mts = (uintptr_t)_pthread_getspecific_direct( + _PTHREAD_TSD_SLOT_MACH_THREAD_SELF); + os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts }; + if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( + (_os_atomic_unfair_lock*)lock, &unlocked, locked, + OSLOCK_STD(memory_order_acquire), + OSLOCK_STD(memory_order_relaxed))) { + return os_unfair_lock_lock(lock); + } +} + +/*! + * @function os_unfair_lock_lock_with_options_inline + * + * @abstract + * Locks an os_unfair_lock. + * + * @param lock + * Pointer to an os_unfair_lock. + * + * @param options + * Options to alter the behavior of the lock. See os_unfair_lock_options_t. + */ +OS_UNFAIR_LOCK_AVAILABILITY +OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL +void +os_unfair_lock_lock_with_options_inline(os_unfair_lock_t lock, + os_unfair_lock_options_t options) +{ + if (!_pthread_has_direct_tsd()) { + return os_unfair_lock_lock_with_options(lock, options); + } + uintptr_t mts = (uintptr_t)_pthread_getspecific_direct( + _PTHREAD_TSD_SLOT_MACH_THREAD_SELF); + os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts }; + if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( + (_os_atomic_unfair_lock*)lock, &unlocked, locked, + OSLOCK_STD(memory_order_acquire), + OSLOCK_STD(memory_order_relaxed))) { + return os_unfair_lock_lock_with_options(lock, options); + } +} + +/*! + * @function os_unfair_lock_trylock_inline + * + * @abstract + * Locks an os_unfair_lock if it is not already locked. + * + * @discussion + * It is invalid to surround this function with a retry loop, if this function + * returns false, the program must be able to proceed without having acquired + * the lock, or it must call os_unfair_lock_lock_inline() instead. + * + * @param lock + * Pointer to an os_unfair_lock. + * + * @result + * Returns true if the lock was succesfully locked and false if the lock was + * already locked. + */ +OS_UNFAIR_LOCK_AVAILABILITY +OS_INLINE OS_ALWAYS_INLINE OS_WARN_RESULT OS_NONNULL_ALL +bool +os_unfair_lock_trylock_inline(os_unfair_lock_t lock) +{ + if (!_pthread_has_direct_tsd()) return os_unfair_lock_trylock(lock); + uintptr_t mts = (uintptr_t)_pthread_getspecific_direct( + _PTHREAD_TSD_SLOT_MACH_THREAD_SELF); + os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts }; + return OSLOCK_STD(atomic_compare_exchange_strong_explicit)( + (_os_atomic_unfair_lock*)lock, &unlocked, locked, + OSLOCK_STD(memory_order_acquire), OSLOCK_STD(memory_order_relaxed)); +} + +/*! + * @function os_unfair_lock_unlock_inline + * + * @abstract + * Unlocks an os_unfair_lock. + * + * @param lock + * Pointer to an os_unfair_lock. + */ +OS_UNFAIR_LOCK_AVAILABILITY +OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL +void +os_unfair_lock_unlock_inline(os_unfair_lock_t lock) +{ + if (!_pthread_has_direct_tsd()) return os_unfair_lock_unlock(lock); + uintptr_t mts = (uintptr_t)_pthread_getspecific_direct( + _PTHREAD_TSD_SLOT_MACH_THREAD_SELF); + os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts }; + if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( + (_os_atomic_unfair_lock*)lock, &locked, unlocked, + OSLOCK_STD(memory_order_release), + OSLOCK_STD(memory_order_relaxed))) { + return os_unfair_lock_unlock(lock); + } +} + +/*! + * @function os_unfair_lock_lock_inline_no_tsd_4libpthread + * + * @abstract + * Locks an os_unfair_lock, without requiring valid TSD. + * + * This should only be used by libpthread. + * + * @param lock + * Pointer to an os_unfair_lock. + */ +OS_UNFAIR_LOCK_AVAILABILITY +OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL +void +os_unfair_lock_lock_inline_no_tsd_4libpthread(os_unfair_lock_t lock) +{ + uintptr_t mts = (uintptr_t)MACH_PORT_DEAD; + os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts }; + if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( + (_os_atomic_unfair_lock*)lock, &unlocked, locked, + OSLOCK_STD(memory_order_acquire), + OSLOCK_STD(memory_order_relaxed))) { + return os_unfair_lock_lock_no_tsd_4libpthread(lock); + } +} + +/*! + * @function os_unfair_lock_unlock_inline_no_tsd_4libpthread + * + * @abstract + * Unlocks an os_unfair_lock, without requiring valid TSD. + * + * This should only be used by libpthread. + * + * @param lock + * Pointer to an os_unfair_lock. + */ +OS_UNFAIR_LOCK_AVAILABILITY +OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL +void +os_unfair_lock_unlock_inline_no_tsd_4libpthread(os_unfair_lock_t lock) +{ + uintptr_t mts = (uintptr_t)MACH_PORT_DEAD; + os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts }; + if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)( + (_os_atomic_unfair_lock*)lock, &locked, unlocked, + OSLOCK_STD(memory_order_release), + OSLOCK_STD(memory_order_relaxed))) { + return os_unfair_lock_unlock_no_tsd_4libpthread(lock); + } +} + +OS_ASSUME_NONNULL_END + +#undef OSLOCK_STD +#ifdef __cplusplus +__END_DECLS +} // extern "C++" +#endif + +#endif // OS_UNFAIR_LOCK_INLINE + +#endif // __OS_LOCK_PRIVATE__ diff --git a/private/os/once_private.h b/private/os/once_private.h new file mode 100644 index 0000000..c93cc14 --- /dev/null +++ b/private/os/once_private.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2008-2013 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __OS_ONCE_PRIVATE__ +#define __OS_ONCE_PRIVATE__ + +#include +#include + +OS_ASSUME_NONNULL_BEGIN + +__BEGIN_DECLS + +#define OS_ONCE_SPI_VERSION 20130313 + +OS_SWIFT_UNAVAILABLE("Swift has lazy init") +typedef long os_once_t; + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +OS_EXPORT OS_NONNULL1 OS_NONNULL3 OS_NOTHROW +OS_SWIFT_UNAVAILABLE("Swift has lazy init") +void +_os_once(os_once_t *predicate, void *_Nullable context, os_function_t function); + +OS_NONNULL1 OS_NONNULL3 OS_NOTHROW +__header_always_inline void +os_once(os_once_t *predicate, void *_Nullable context, os_function_t function) +{ + if (OS_EXPECT(*predicate, ~0l) != ~0l) { + _os_once(predicate, context, function); + OS_COMPILER_CAN_ASSUME(*predicate == ~0l); + } else { + os_compiler_barrier(); + } +} + +/* This SPI is *strictly* for the use of pthread_once only. This is not + * safe in general use of os_once. + */ +__OSX_AVAILABLE_STARTING(__MAC_10_9, __IPHONE_7_0) +OS_EXPORT OS_NONNULL1 OS_NOTHROW +OS_SWIFT_UNAVAILABLE("Swift has lazy init") +void +__os_once_reset(os_once_t *val); + +__END_DECLS + +OS_ASSUME_NONNULL_END + +#endif // __OS_ONCE_PRIVATE__ diff --git a/private/os/semaphore_private.h b/private/os/semaphore_private.h new file mode 100644 index 0000000..d6bd728 --- /dev/null +++ b/private/os/semaphore_private.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2008-2013 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __OS_SEMAPHORE_PRIVATE__ +#define __OS_SEMAPHORE_PRIVATE__ + +#include +#include +#include +#include + +OS_ASSUME_NONNULL_BEGIN + +__BEGIN_DECLS + +#define OS_SEMAPHORE_SPI_VERSION 20130313 + +typedef uintptr_t os_semaphore_t; + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +OS_EXPORT OS_WARN_RESULT OS_NOTHROW +os_semaphore_t _os_semaphore_create(void); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +OS_EXPORT OS_NOTHROW +void _os_semaphore_dispose(os_semaphore_t); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +OS_EXPORT OS_NOTHROW +void _os_semaphore_wait(os_semaphore_t); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +OS_EXPORT OS_NOTHROW +void _os_semaphore_signal(os_semaphore_t); + +OS_WARN_RESULT OS_NOTHROW +__header_always_inline os_semaphore_t +os_get_cached_semaphore(void) +{ + os_semaphore_t sema; + sema = (os_semaphore_t)_os_tsd_get_direct(__TSD_SEMAPHORE_CACHE); + if (os_unlikely(!sema)) { + return _os_semaphore_create(); + } + _os_tsd_set_direct(__TSD_SEMAPHORE_CACHE, 0); + return sema; +} + +OS_NOTHROW +__header_always_inline void +os_put_cached_semaphore(os_semaphore_t sema) +{ + os_semaphore_t old_sema; + old_sema = (os_semaphore_t)_os_tsd_get_direct(__TSD_SEMAPHORE_CACHE); + _os_tsd_set_direct(__TSD_SEMAPHORE_CACHE, (void*)sema); + if (os_unlikely(old_sema)) { + return _os_semaphore_dispose(old_sema); + } +} + +OS_NOTHROW +__header_always_inline void +os_semaphore_wait(os_semaphore_t sema) +{ + return _os_semaphore_wait(sema); +} + +OS_NOTHROW +__header_always_inline void +os_semaphore_signal(os_semaphore_t sema) +{ + return _os_semaphore_signal(sema); +} + +__END_DECLS + +OS_ASSUME_NONNULL_END + +#endif // __OS_SEMAPHORE_PRIVATE__ diff --git a/private/platform/compat.h b/private/platform/compat.h new file mode 100644 index 0000000..a9f59fd --- /dev/null +++ b/private/platform/compat.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2013 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#ifndef _PLATFORM_COMPAT_H_ +#define _PLATFORM_COMPAT_H_ + +#include + +__BEGIN_DECLS + +/* Helpers for other common non-primitive routines */ + +__header_always_inline +size_t +_platform_strlen(const char *s) { + const char *t = _platform_memchr(s, '\0', SIZE_MAX); + return (uintptr_t)t - (uintptr_t)s; +} + +__header_always_inline +size_t +_platform_strlcpy(char * restrict dst, const char * restrict src, size_t maxlen) { + const size_t srclen = _platform_strlen(src); + if (srclen < maxlen) { + _platform_memmove(dst, src, srclen+1); + } else if (maxlen != 0) { + _platform_memmove(dst, src, maxlen-1); + dst[maxlen-1] = '\0'; + } + return srclen; +} + +__END_DECLS + +/* Compat macros for primitives */ +#define bzero _platform_bzero +#define memchr _platform_memchr +#define memcmp _platform_memcmp +#define memmove _platform_memmove +#define memccpy _platform_memccpy +#define memset _platform_memset +#define memset_pattern4 _platform_memset_pattern4 +#define memset_pattern8 _platform_memset_pattern8 +#define memset_pattern16 _platform_memset_pattern16 +#define strchr _platform_strchr +#define strcmp _platform_strcmp +#define strncmp _platform_strncmp + +/* Compat macros for non-primitive helpers */ +#define strlcpy _platform_strlcpy +#define strlen _platform_strlen + +#endif /* _PLATFORM_COMPAT_H_ */ diff --git a/private/platform/introspection_private.h b/private/platform/introspection_private.h new file mode 100644 index 0000000..f09ee58 --- /dev/null +++ b/private/platform/introspection_private.h @@ -0,0 +1,447 @@ +#ifndef __PLATFORM_INTROSPECTION_H__ +#define __PLATFORM_INTROSPECTION_H__ + +#include +#include +#include + +#include + +#include + +typedef struct platform_task_s *platform_task_t; +typedef struct platform_thread_s *platform_thread_t; + +/*! + * @typedef platform_thread_id_t + * + * @discussion + * The type of the 64-bit system-wide unique thread ID. + */ +typedef uint64_t platform_thread_id_t; + +/*! @functiongroup Tasks */ + +/*! + * @function platform_task_attach + * + * @discussion + * Attach to a process (specified by its mach task port) for debugging. This + * function creates a new task handle which must be disposed by a call to + * platform_task_detach(). + * + * @param task + * On output, a newly created task handle. + * + * @param target + * The mach task port of the target process. + * + * @return + * KERN_SUCCESS if the process was successfully attached, otherwise a mach + * error code. + */ +kern_return_t +platform_task_attach(platform_task_t *task, task_t target); + +/*! + * @function platform_task_detach + * + * @discussion + * Detaches from the target task and deallocates all memory associated with + * the task handle. + * + * @param task + * The task handle to detach. + * + * @return + * KERN_SUCCESS if the process was successfully detached, otherwise a mach + * error code. + */ +kern_return_t +platform_task_detach(platform_task_t task); + +/*! + * @function platform_task_is_64_bit + * + * @discussion + * Returns true if the target task is LP64. + * + * @param task + * A handle to the target task. + * + * @return + * true if the target task is LP64, otherwise false. + */ +bool +platform_task_is_64_bit(platform_task_t task); + +/*! + * @function platform_task_suspend_threads + * + * @discussion + * Suspends all the threads in the target task. This differs from task_suspend + * in that the task itself is not suspended, only the individual threads. While + * this suspension is in effect, any newly created threads will be created in + * a suspended state. The debuger may resume an individual thread for execution + * using platform_thread_resume() or evaluate an expression in the context of + * the task or a specific thread using platform_task_perform() and + * platform_thread_perform(), respectively. All threads in the task may be + * resumed with platform_task_resume_threads(). + * + * @param task + * A handle to the target task. + * + * @return + * KERN_SUCCESS if the threads were successfully suspended, otherwise a mach + * error code. + */ +kern_return_t +platform_task_suspend_threads(platform_task_t task); + +/*! + * @function platform_task_resume_threads + * + * @discussion + * Resumes the threads in the target task. See platform_task_suspend_threads(). + * + * @param task + * A handle to the target task. + */ +kern_return_t +platform_task_resume_threads(platform_task_t task); + +/*! + * @function platform_task_perform + * + * @discussion + * Performs the specified function on a newly created thread in the target task. + * This newly created thread will execute even if the threads in the task are + * suspended as the result of a call to platform_task_suspend_threads(). + * + * The function and context addresses are in the virtual address space of the + * target task. It is the responsiblity of the debugger to have previously + * mapped executable text and data at these addresses in the target task. + * + * @param task + * A handle to the target task. + * + * @param func_addr + * The address (in the virtual address space of the target task) of the + * function to perform. The function should be of type (void (*)(void *)) + * and will be passed the value of the data_addr parameter. + * + * @param data_addr + * The address (in the virtual address space of the target task) of the + * data to pass as a parameter to the function to perform. + * + * @return + * KERN_SUCCESS if the function was successfully performed, otherwise a mach + * error code. + */ +kern_return_t +platform_task_perform(platform_task_t task, + mach_vm_address_t func_addr, + mach_vm_address_t data_addr); + +/*! + * @function platform_task_update_threads + * + * @discussion + * Updates an internal representation of all threads in the target task. The + * list of threads may then be iterated using platform_task_copy_next_thread(). + * + * Calling this function resets any iteration currently in progress and a + * subsequent call to platform_task_copy_next_thread() will return the first + * thread in the list. + * + * @param task + * A handle to the target task. + * + * @return + * KERN_SUCCESS if the threads were successfully updated, otherwise a mach + * error code. + */ +kern_return_t +platform_task_update_threads(platform_task_t task); + +/*! + * @function platform_task_copy_next_thread + * + * @discussion + * Iterates the list of threads in the task. Returns a copied thread handle + * which must subsequently be released using platform_thread_release(). + * + * The platform_task_update_threads() function must be called before this + * function will return any thread handles. A NULL pointer is returned to + * signify the end of the list + * + * @param task + * A handle to the target task. + * + * @return + * A thread handle which must be released using platform_thread_release(), + * or NULL, signifying the end of the list. + */ +platform_thread_t +platform_task_copy_next_thread(platform_task_t task); + +/*! @functiongroup Threads */ + +/*! + * @function platform_thread_get_unique_id + * + * @discussion + * Returns the 64-bit system-wide unique ID of the target thread. + * + * @param thread + * A handle to the target thread. + * + * @return + * The unique ID of the thread. + */ +platform_thread_id_t +platform_thread_get_unique_id(platform_thread_t thread); + +/*! + * @function platform_thread_release + * + * @discussion + * Releases a thread handle obtained by platform_task_copy_next_thread(). + * + * @param thread + * The thread handle to release. + */ +void +platform_thread_release(platform_thread_t thread); + +/*! + * @function platform_thread_abort_safely + * + * @discussion + * Similar to thread_abort_safely(). + * + * @param thread + * A handle to the thread to signal. + * + * @return + * KERN_SUCCESS if the thread was successfully signaled, otherwise a mach + * error code. + */ +kern_return_t +platform_thread_abort_safely(platform_thread_t thread); + +/*! + * @function platform_thread_suspend + * + * @discussion + * Suspends execution of a thread similar to thread_suspend(). See also + * platform_task_suspend_threads(). + * + * @param thread + * A handle to the thread to suspend. + * + * @return + * KERN_SUCCESS if the thread was successfully suspended, otherwise a mach + * error code. + */ +kern_return_t +platform_thread_suspend(platform_thread_t thread); + +/*! + * @function platform_thread_resume + * + * @discussion + * Suspends execution of a thread similar to thread_suspend(). See also + * platform_task_suspend_threads() and platform_task_resume_threads(). + * + * @param thread + * A handle to the thread to resume. + * + * @return + * KERN_SUCCESS if the thread was successfully resumed, otherwise a mach + * error code. +*/ +kern_return_t +platform_thread_resume(platform_thread_t thread); + +/*! + * @function platform_thread_info + * + * @discussion + * Similar to thread_info. Supported flavor structures: + * - THREAD_BASIC_INFO: struct thread_basic_info + * - THREAD_IDENTIFIER_INFO: struct thread_identifier_info + * + * @param thread + * A handle to the target thread. + * + * @param flavor + * The desired thread info structure. + * + * @param info + * A pointer to storage where the thread info structure should be written. + * + * @param size + * On input, the size in bytes of the storage where the thread info structure + * is to be written. On output, the size of the thread info structure in bytes. + * + * @return + * KERN_SUCCESS if the function was successfully performed, otherwise a mach + * error code. + */ +kern_return_t +platform_thread_info(platform_thread_t thread, + thread_flavor_t flavor, + void *info, + size_t *size); + +/*! + * @function platform_thread_get_state + * + * @discussion + * Similar to thread_get_state. Supported flavor structures: + * - x86_THREAD_STATE32: struct ... + * - x86_FLOAT_STATE32: struct ... + * - x86_EXCEPTION_STATE32: struct ... + * - x86_DEBUG_STATE32: struct ... + * - x86_AVX_STATE32: struct ... + * - x86_THREAD_STATE64: struct ... + * - x86_FLOAT_STATE64: struct ... + * - x86_EXCEPTION_STATE64: struct ... + * - x86_DEBUG_STATE64: struct ... + * - x86_AVX_STATE64: struct ... + * - ARM_THREAD_STATE32: struct ... + * - ARM_FLOAT_STATE32: struct ... + * - ARM_EXCEPTION_STATE32: struct ... + * - ARM_DEBUG_STATE32: struct ... + * - ARM_THREAD_STATE64: struct ... + * - ARM_FLOAT_STATE64: struct ... + * - ARM_EXCEPTION_STATE64: struct ... + * - ARM_DEBUG_STATE64: struct ... + * - ... + * + * @param thread + * A handle to the target thread. + * + * @param flavor + * The desired thread state structure. + * + * @param state + * A pointer to storage where the thread state structure should be written. + * + * @param size + * On input, the size in bytes of the storage where the thread state structure + * is to be written. On output, the size of the thread state structure in bytes. + * + * @return + * KERN_SUCCESS if the function was successfully performed, otherwise a mach + * error code. + */ +kern_return_t +platform_thread_get_state(platform_thread_t thread, + thread_state_flavor_t flavor, + void *state, + size_t *size); + +/*! + * @function platform_thread_set_state + * + * @discussion + * Similar to thread_set_state. Supported flavor structures: + * - x86_THREAD_STATE32: struct ... + * - x86_FLOAT_STATE32: struct ... + * - x86_EXCEPTION_STATE32: struct ... + * - x86_DEBUG_STATE32: struct ... + * - x86_AVX_STATE32: struct ... + * - x86_THREAD_STATE64: struct ... + * - x86_FLOAT_STATE64: struct ... + * - x86_EXCEPTION_STATE64: struct ... + * - x86_DEBUG_STATE64: struct ... + * - x86_AVX_STATE64: struct ... + * - ARM_THREAD_STATE32: struct ... + * - ARM_FLOAT_STATE32: struct ... + * - ARM_EXCEPTION_STATE32: struct ... + * - ARM_DEBUG_STATE32: struct ... + * - ARM_THREAD_STATE64: struct ... + * - ARM_FLOAT_STATE64: struct ... + * - ARM_EXCEPTION_STATE64: struct ... + * - ARM_DEBUG_STATE64: struct ... + * - ... + * + * @param thread + * A handle to the target thread. + * + * @param flavor + * The desired thread state structure. + * + * @param state + * A pointer to storage where the thread state structure should be written. + * + * @param size + * The size of the thread state structure in bytes. + * + * @return + * KERN_SUCCESS if the function was successfully performed, otherwise a mach + * error code. + */ +kern_return_t +platform_thread_set_state(platform_thread_t thread, + thread_state_flavor_t flavor, + const void *state, + size_t size); + +/*! + * @function platform_thread_perform + * + * @discussion + * Performs the specified function within the context of the specified thread + * in the target task. The function will execute in the style of an + * asynchronous signal handler even if the thread is suspended as the result + * of a call to platform_task_suspend_threads() or platform_thread_suspend(). + * The original state of the thread will be restored when the function returns. + * + * The function and context addresses are in the virtual address space of the + * target task. It is the responsiblity of the debugger to have previously + * mapped executable text and data at these addresses in the target task. + * + * See also platform_task_perform(). + * + * @param thread + * A handle to the target thread. + * + * @param func_addr + * The address (in the virtual address space of the target task) of the + * function to perform. The function should be of type (void (*)(void *)) + * and will be passed the value of the data_addr parameter. + * + * @param data_addr + * The address (in the virtual address space of the target task) of the + * data to pass as a parameter to the function to perform. + * + * @return + * KERN_SUCCESS if the function was successfully performed, otherwise a mach + * error code. + */ +kern_return_t +platform_thread_perform(platform_thread_t thread, + mach_vm_address_t func_addr, + mach_vm_address_t data_addr); + +/*! + * @function platform_thread_get_pthread + * + * @discussion + * Returns a pointer to mapped memory which represents the pthread_t of the + * target process. Any embedded pointers will need to be mapped into the current + * process space on a case-by-case basis. + * + * @param thread + * A handle to the target thread. + * + * @return + * A valid pointer. + */ +const void * +platform_thread_get_pthread(platform_thread_t thread); + +#endif // __PLATFORM_INTROSPECTION_H__ diff --git a/private/platform/string.h b/private/platform/string.h new file mode 100644 index 0000000..c4d1c3f --- /dev/null +++ b/private/platform/string.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2013 Apple Inc. All rights reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +#ifndef _PLATFORM_STRING_H_ +#define _PLATFORM_STRING_H_ + +#include +#include +#include + +#include +#include + +#define _PLATFORM_OPTIMIZED_BZERO 0 +#define _PLATFORM_OPTIMIZED_MEMCCPY 0 +#define _PLATFORM_OPTIMIZED_MEMCHR 0 +#define _PLATFORM_OPTIMIZED_MEMCMP 0 +#define _PLATFORM_OPTIMIZED_MEMMOVE 0 +#define _PLATFORM_OPTIMIZED_MEMSET 0 +#define _PLATFORM_OPTIMIZED_MEMSET_PATTERN4 0 +#define _PLATFORM_OPTIMIZED_MEMSET_PATTERN8 0 +#define _PLATFORM_OPTIMIZED_MEMSET_PATTERN16 0 +#define _PLATFORM_OPTIMIZED_STRCHR 0 +#define _PLATFORM_OPTIMIZED_STRCMP 0 +#define _PLATFORM_OPTIMIZED_STRNCMP 0 + +/* Primitives used to implement C memory and string routines */ + +__BEGIN_DECLS + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +void +_platform_bzero(void *s, size_t n); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +void * +_platform_memchr(const void *s, int c, size_t n); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +int +_platform_memcmp(const void *s1, const void *s2, size_t n); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +void * +_platform_memmove(void *dst, const void *src, size_t n); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +void * +_platform_memccpy(void *restrict dst, const void *restrict src, int c, size_t n); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +void * +_platform_memset(void *b, int c, size_t len); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +void +_platform_memset_pattern4(void *b, const void *pattern4, size_t len); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +void +_platform_memset_pattern8(void *b, const void *pattern8, size_t len); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +void +_platform_memset_pattern16(void *b, const void *pattern16, size_t len); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +char * +_platform_strchr(const char *s, int c); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +int +_platform_strcmp(const char *s1, const char *s2); + +__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0) +int +_platform_strncmp(const char *s1, const char *s2, size_t n); + +__END_DECLS + +#endif /* _PLATFORM_STRING_H_ */ diff --git a/src/cachecontrol/arm64/cache.s b/src/cachecontrol/arm64/cache.s index d59afdd..000e581 100644 --- a/src/cachecontrol/arm64/cache.s +++ b/src/cachecontrol/arm64/cache.s @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2011-2017 Apple Computer, Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * @@ -23,21 +23,52 @@ #include -/* void sys_icache_invalidate(addr_t start, int length) */ +#define MMU_I_CLINE 6 // cache line size as 1< 0 ? + and x8, x0, #~((1<> MMU_I_CLINE) + 1; + while (count--) { + asm("ic ivau, %[addr]" :: [addr] "r" (addr) : "memory"); + addr += (1 << MMU_I_CLINE); + } + asm volatile("dsb ish\n\tisb" ::: "memory"); +} +#endif + diff --git a/src/init.c b/src/init.c index 7e8bc6a..7f43e47 100644 --- a/src/init.c +++ b/src/init.c @@ -21,21 +21,21 @@ * @APPLE_LICENSE_HEADER_END@ */ -#include #include +#include struct ProgramVars; /* forward reference */ extern void _simple_asl_init(const char *envp[], const struct ProgramVars *vars); extern void __pfz_setup(const char *apple[]); - -void __libplatform_init(void *future_use __unused, const char *envp[], const char *apple[], const struct ProgramVars *vars) { - +void +__libplatform_init(void *future_use __unused, const char *envp[], + const char *apple[], const struct ProgramVars *vars) +{ /* In the Simulator, we just provide _simple for dyld */ #if !TARGET_IPHONE_SIMULATOR __pfz_setup(apple); #endif _simple_asl_init(envp, vars); - } diff --git a/src/os/alloc_once.c b/src/os/alloc_once.c index a8ff358..e1ee7c5 100644 --- a/src/os/alloc_once.c +++ b/src/os/alloc_once.c @@ -27,6 +27,7 @@ */ #include "os/internal.h" +#include "resolver.h" #include "os/alloc_once_impl.h" #include #include @@ -42,7 +43,12 @@ typedef struct _os_alloc_heap_metadata_s { #define allocation_size (2 * vm_page_size) #define usable (allocation_size-sizeof(_os_alloc_heap_metadata_s)) -static void * volatile _os_alloc_heap; +OS_NOEXPORT void * volatile _os_alloc_heap; + +OS_ATOMIC_EXPORT void* _os_alloc_once(struct _os_alloc_once_s *slot, size_t sz, + os_function_t init); + +void * volatile _os_alloc_heap; /* * Simple allocator that doesn't have to worry about ever freeing allocations. diff --git a/src/os/atomic.c b/src/os/atomic.c index 6c846f6..180c090 100644 --- a/src/os/atomic.c +++ b/src/os/atomic.c @@ -19,42 +19,86 @@ */ #include "os/internal.h" -#include "libkern/OSAtomic.h" #include "resolver.h" +#include "libkern/OSAtomic.h" #if TARGET_OS_EMBEDDED +OS_ATOMIC_EXPORT +int32_t OSAtomicAdd32(int32_t v, volatile int32_t *p); OS_ATOMIC_EXPORT int32_t OSAtomicAdd32Barrier(int32_t v, volatile int32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicIncrement32(volatile int32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicIncrement32Barrier(volatile int32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicDecrement32(volatile int32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicDecrement32Barrier(volatile int32_t *p); OS_ATOMIC_EXPORT +int64_t OSAtomicAdd64(int64_t v, volatile int64_t *p); +OS_ATOMIC_EXPORT int64_t OSAtomicAdd64Barrier(int64_t v, volatile int64_t *p); OS_ATOMIC_EXPORT +int64_t OSAtomicIncrement64(volatile int64_t *p); +OS_ATOMIC_EXPORT int64_t OSAtomicIncrement64Barrier(volatile int64_t *p); OS_ATOMIC_EXPORT +int64_t OSAtomicDecrement64(volatile int64_t *p); +OS_ATOMIC_EXPORT int64_t OSAtomicDecrement64Barrier(volatile int64_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicAnd32(uint32_t v, volatile uint32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicAnd32Barrier(uint32_t v, volatile uint32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicAnd32Orig(uint32_t v, volatile uint32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicAnd32OrigBarrier(uint32_t v, volatile uint32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicOr32(uint32_t v, volatile uint32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicOr32Barrier(uint32_t v, volatile uint32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicOr32Orig(uint32_t v, volatile uint32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicOr32OrigBarrier(uint32_t v, volatile uint32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicXor32(uint32_t v, volatile uint32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicXor32Barrier(uint32_t v, volatile uint32_t *p); OS_ATOMIC_EXPORT +int32_t OSAtomicXor32Orig(uint32_t v, volatile uint32_t *p); +OS_ATOMIC_EXPORT int32_t OSAtomicXor32OrigBarrier(uint32_t v, volatile uint32_t *p); OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwap32(int32_t o, int32_t n, volatile int32_t *p); +OS_ATOMIC_EXPORT bool OSAtomicCompareAndSwap32Barrier(int32_t o, int32_t n, volatile int32_t *p); OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwapPtr(void *o, void *n, void * volatile *p); +OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwapPtrBarrier(void *o, void *n, void * volatile *p); +OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwapInt(int o, int n, volatile int *p); +OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwapIntBarrier(int o, int n, volatile int *p); +OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwapLong(long o, long n, volatile long *p); +OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwapLongBarrier(long o, long n, volatile long *p); +OS_ATOMIC_EXPORT +bool OSAtomicCompareAndSwap64(int64_t o, int64_t n, volatile int64_t *p); +OS_ATOMIC_EXPORT bool OSAtomicCompareAndSwap64Barrier(int64_t o, int64_t n, volatile int64_t *p); OS_ATOMIC_EXPORT +bool OSAtomicTestAndSet(uint32_t n, volatile void * p); +OS_ATOMIC_EXPORT bool OSAtomicTestAndSetBarrier(uint32_t n, volatile void * p); OS_ATOMIC_EXPORT +bool OSAtomicTestAndClear(uint32_t n, volatile void * p); +OS_ATOMIC_EXPORT bool OSAtomicTestAndClearBarrier(uint32_t n, volatile void * p); OS_ATOMIC_EXPORT void OSAtomicEnqueue(OSQueueHead *list, void *new, size_t offset); @@ -63,139 +107,232 @@ void* OSAtomicDequeue(OSQueueHead *list, size_t offset); OS_ATOMIC_EXPORT void OSMemoryBarrier(void); -#if OS_ATOMIC_UP -#define OS_ATOMIC_ALIAS_NO_BARRIER(n) OS_ATOMIC_EXPORT_ALIAS(n, n##Barrier) -#else -#define OS_ATOMIC_ALIAS_NO_BARRIER(n) -#endif +int32_t +OSAtomicAdd32(int32_t v, volatile int32_t *p) +{ + int32_t r = os_atomic_add(p, v, relaxed); + return r; +} int32_t OSAtomicAdd32Barrier(int32_t v, volatile int32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicAdd32); - int32_t r = os_atomic_add(p, v, acq_rel); + int32_t r = os_atomic_add(p, v, seq_cst); + return r; +} + +int32_t +OSAtomicIncrement32(volatile int32_t *p) +{ + int32_t r = os_atomic_add(p, 1, relaxed); return r; } int32_t OSAtomicIncrement32Barrier(volatile int32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicIncrement32); - int32_t r = os_atomic_add(p, 1, acq_rel); + int32_t r = os_atomic_add(p, 1, seq_cst); + return r; +} + +int32_t +OSAtomicDecrement32(volatile int32_t *p) +{ + int32_t r = os_atomic_add(p, -1, relaxed); return r; } int32_t OSAtomicDecrement32Barrier(volatile int32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicDecrement32); - int32_t r = os_atomic_add(p, -1, acq_rel); + int32_t r = os_atomic_add(p, -1, seq_cst); + return r; +} + +int64_t +OSAtomicAdd64(int64_t v, volatile int64_t *p) +{ + int64_t r = os_atomic_add(p, v, relaxed); return r; } int64_t OSAtomicAdd64Barrier(int64_t v, volatile int64_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicAdd64); - int64_t r = os_atomic_add(p, v, acq_rel); + int64_t r = os_atomic_add(p, v, seq_cst); + return r; +} + +int64_t +OSAtomicIncrement64(volatile int64_t *p) +{ + int64_t r = os_atomic_add(p, 1, relaxed); return r; } int64_t OSAtomicIncrement64Barrier(volatile int64_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicIncrement64); - int64_t r = os_atomic_add(p, 1, acq_rel); + int64_t r = os_atomic_add(p, 1, seq_cst); + return r; +} + +int64_t +OSAtomicDecrement64(volatile int64_t *p) +{ + int64_t r = os_atomic_add(p, -1, relaxed); return r; } int64_t OSAtomicDecrement64Barrier(volatile int64_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicDecrement64); - int64_t r = os_atomic_add(p, -1, acq_rel); + int64_t r = os_atomic_add(p, -1, seq_cst); return r; } +int32_t +OSAtomicAnd32(uint32_t v, volatile uint32_t *p) +{ + uint32_t r = os_atomic_and(p, v, relaxed); + return (int32_t)r; +} + int32_t OSAtomicAnd32Barrier(uint32_t v, volatile uint32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicAnd32); - uint32_t r = os_atomic_and(p, v, acq_rel); + uint32_t r = os_atomic_and(p, v, seq_cst); + return (int32_t)r; +} + +int32_t +OSAtomicAnd32Orig(uint32_t v, volatile uint32_t *p) +{ + uint32_t r = os_atomic_and_orig(p, v, relaxed); return (int32_t)r; } int32_t OSAtomicAnd32OrigBarrier(uint32_t v, volatile uint32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicAnd32Orig); - uint32_t r = os_atomic_and_orig(p, v, acq_rel); + uint32_t r = os_atomic_and_orig(p, v, seq_cst); + return (int32_t)r; +} + +int32_t +OSAtomicOr32(uint32_t v, volatile uint32_t *p) +{ + uint32_t r = os_atomic_or(p, v, relaxed); return (int32_t)r; } int32_t OSAtomicOr32Barrier(uint32_t v, volatile uint32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicOr32); - uint32_t r = os_atomic_or(p, v, acq_rel); + uint32_t r = os_atomic_or(p, v, seq_cst); + return (int32_t)r; +} + +int32_t +OSAtomicOr32Orig(uint32_t v, volatile uint32_t *p) +{ + uint32_t r = os_atomic_or_orig(p, v, relaxed); return (int32_t)r; } int32_t OSAtomicOr32OrigBarrier(uint32_t v, volatile uint32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicOr32Orig); - uint32_t r = os_atomic_or_orig(p, v, acq_rel); + uint32_t r = os_atomic_or_orig(p, v, seq_cst); + return (int32_t)r; +} + +int32_t +OSAtomicXor32(uint32_t v, volatile uint32_t *p) +{ + uint32_t r = os_atomic_xor(p, v, relaxed); return (int32_t)r; } int32_t OSAtomicXor32Barrier(uint32_t v, volatile uint32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicXor32); - uint32_t r = os_atomic_xor(p, v, acq_rel); + uint32_t r = os_atomic_xor(p, v, seq_cst); + return (int32_t)r; +} + +int32_t +OSAtomicXor32Orig(uint32_t v, volatile uint32_t *p) +{ + uint32_t r = os_atomic_xor_orig(p, v, relaxed); return (int32_t)r; } int32_t OSAtomicXor32OrigBarrier(uint32_t v, volatile uint32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicXor32Orig); - uint32_t r = os_atomic_xor_orig(p, v, acq_rel); + uint32_t r = os_atomic_xor_orig(p, v, seq_cst); return (int32_t)r; } +bool +OSAtomicCompareAndSwap32(int32_t o, int32_t n, volatile int32_t *p) +{ + return os_atomic_cmpxchg(p, o, n, relaxed); +} + bool OSAtomicCompareAndSwap32Barrier(int32_t o, int32_t n, volatile int32_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwap32); - OS_ATOMIC_ALIAS(OSAtomicCompareAndSwapIntBarrier, - OSAtomicCompareAndSwap32Barrier); - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwapInt); -#ifndef __LP64__ - OS_ATOMIC_ALIAS(OSAtomicCompareAndSwapLongBarrier, - OSAtomicCompareAndSwap32Barrier); - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwapLong); - OS_ATOMIC_ALIAS(OSAtomicCompareAndSwapPtrBarrier, - OSAtomicCompareAndSwap32Barrier); - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwapPtr); -#endif - return os_atomic_cmpxchg(p, o, n, acq_rel); + return os_atomic_cmpxchg(p, o, n, seq_cst); +} + +bool +OSAtomicCompareAndSwapPtr(void *o, void *n, void * volatile *p) +{ + return os_atomic_cmpxchg(p, o, n, relaxed); +} + +bool +OSAtomicCompareAndSwapPtrBarrier(void *o, void *n, void * volatile *p) +{ + return os_atomic_cmpxchg(p, o, n, seq_cst); +} + +bool +OSAtomicCompareAndSwapInt(int o, int n, volatile int *p) +{ + return os_atomic_cmpxchg(p, o, n, relaxed); +} + +bool +OSAtomicCompareAndSwapIntBarrier(int o, int n, volatile int *p) +{ + return os_atomic_cmpxchg(p, o, n, seq_cst); +} + +bool +OSAtomicCompareAndSwapLong(long o, long n, volatile long *p) +{ + return os_atomic_cmpxchg(p, o, n, relaxed); +} + +bool +OSAtomicCompareAndSwapLongBarrier(long o, long n, volatile long *p) +{ + return os_atomic_cmpxchg(p, o, n, seq_cst); +} + +bool +OSAtomicCompareAndSwap64(int64_t o, int64_t n, volatile int64_t *p) +{ + return os_atomic_cmpxchg(p, o, n, relaxed); } bool OSAtomicCompareAndSwap64Barrier(int64_t o, int64_t n, volatile int64_t *p) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwap64); -#ifdef __LP64__ - OS_ATOMIC_ALIAS(OSAtomicCompareAndSwapLongBarrier, - OSAtomicCompareAndSwap64Barrier); - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwapLong); - OS_ATOMIC_ALIAS(OSAtomicCompareAndSwapPtrBarrier, - OSAtomicCompareAndSwap64Barrier); - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicCompareAndSwapPtr); -#endif - return os_atomic_cmpxchg(p, o, n, acq_rel); + return os_atomic_cmpxchg(p, o, n, seq_cst); } static inline uint32_t* @@ -211,42 +348,60 @@ _OSAtomicTestPtrVal(uint32_t bit, volatile void *addr, uint32_t *vp) return (uint32_t*)((char*)a + 4 * (bit / 32)); } +bool +OSAtomicTestAndSet(uint32_t bit, volatile void *addr) +{ + uint32_t v; + volatile uint32_t *p = _OSAtomicTestPtrVal(bit, addr, &v); + uint32_t r = os_atomic_or_orig(p, v, relaxed); + return (r & v); +} + bool OSAtomicTestAndSetBarrier(uint32_t bit, volatile void *addr) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicTestAndSet); uint32_t v; volatile uint32_t *p = _OSAtomicTestPtrVal(bit, addr, &v); - uint32_t r = os_atomic_or_orig(p, v, acq_rel); + uint32_t r = os_atomic_or_orig(p, v, seq_cst); return (r & v); } bool -OSAtomicTestAndClearBarrier(uint32_t bit, volatile void *addr) +OSAtomicTestAndClear(uint32_t bit, volatile void *addr) { - OS_ATOMIC_ALIAS_NO_BARRIER(OSAtomicTestAndClear); uint32_t v; volatile uint32_t *p = _OSAtomicTestPtrVal(bit, addr, &v); - uint32_t r = os_atomic_and_orig(p, ~v, acq_rel); + uint32_t r = os_atomic_and_orig(p, ~v, relaxed); return (r & v); } -#if !OS_ATOMIC_NO_BARRIER_ONLY +bool +OSAtomicTestAndClearBarrier(uint32_t bit, volatile void *addr) +{ + uint32_t v; + volatile uint32_t *p = _OSAtomicTestPtrVal(bit, addr, &v); + uint32_t r = os_atomic_and_orig(p, ~v, seq_cst); + return (r & v); +} -typedef volatile struct { - void * volatile item; - long unused; +typedef struct { + void *item; + long gencount; } _OSQueueHead; + void OSAtomicEnqueue(OSQueueHead *list, void *new, size_t offset) { void * volatile *headptr = &(((_OSQueueHead*)list)->item); void * volatile *nextptr = (void*)((char*)new + offset); - void *head = *headptr; + void *head, *next; + + head = os_atomic_load(headptr, relaxed); + next = new; do { *nextptr = head; - } while (!os_atomic_cmpxchgvw(headptr, head, new, &head, release)); + } while (!os_atomic_cmpxchgvw(headptr, head, next, &head, release)); } void* @@ -255,23 +410,22 @@ OSAtomicDequeue(OSQueueHead *list, size_t offset) void * volatile *headptr = &(((_OSQueueHead*)list)->item); void * volatile *nextptr; void *head, *next; - (void)os_atomic_rmw_loop(headptr, head, next, acquire, { - if (!head) { - os_atomic_rmw_loop_give_up(break); - } + + os_atomic_rmw_loop(headptr, head, next, acquire, { + if (!head) os_atomic_rmw_loop_give_up(break); nextptr = (void*)((char*)head + offset); next = *nextptr; }); return head; } + void OSMemoryBarrier(void) { os_atomic_thread_fence(seq_cst); } -#endif // !OS_ATOMIC_NO_BARRIER_ONLY #endif // TARGET_OS_EMBEDDED struct _os_empty_files_are_not_c_files; diff --git a/src/os/atomic_up.c b/src/os/atomic_up.c deleted file mode 100644 index 9013643..0000000 --- a/src/os/atomic_up.c +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2013 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -// Force up variant to be generated to get the no-barrier OSAtomics -#undef VARIANT_NO_RESOLVERS -#define VARIANT_NO_RESOLVERS 0 - -#define OS_ATOMIC_UP 1 -#include "os/internal.h" - -#ifdef OS_VARIANT_SELECTOR -#define OS_VARIANT_ONLY 1 -#include "atomic.c" -#endif - -struct _os_empty_files_are_not_c_files; diff --git a/src/os/lock.c b/src/os/lock.c index 3d66adf..076c61b 100644 --- a/src/os/lock.c +++ b/src/os/lock.c @@ -19,11 +19,12 @@ */ #include "lock_internal.h" +#include "os/internal.h" +#include "resolver.h" #include "libkern/OSAtomic.h" #include "os/lock.h" #include "os/lock_private.h" #include "os/once_private.h" -#include "resolver.h" #include #include @@ -34,7 +35,9 @@ #pragma mark - #pragma mark _os_lock_base_t -#if !OS_VARIANT_ONLY +OS_NOINLINE OS_NORETURN OS_COLD +void _os_lock_corruption_abort(void *lock_ptr OS_UNUSED, uintptr_t lock_value); + OS_LOCK_STRUCT_DECL_INTERNAL(base); OS_USED static OS_LOCK_TYPE_STRUCT_DECL(base); @@ -57,41 +60,34 @@ os_lock_unlock(os_lock_t l) return l._osl_base->osl_type->osl_unlock(l); } -#endif //!OS_VARIANT_ONLY - OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_lock_corruption_abort(void *lock_ptr OS_UNUSED, uintptr_t lock_value) { __LIBPLATFORM_CLIENT_CRASH__(lock_value, "os_lock is corrupt"); } + #pragma mark - #pragma mark OSSpinLock -#ifdef OS_LOCK_VARIANT_SELECTOR -void _OSSpinLockLockSlow(volatile OSSpinLock *l); -#else -OS_NOINLINE OS_USED static void _OSSpinLockLockSlow(volatile OSSpinLock *l); -#endif // OS_LOCK_VARIANT_SELECTOR +OS_NOEXPORT OS_NOINLINE void _OSSpinLockLockSlow(volatile OSSpinLock *l); OS_ATOMIC_EXPORT void OSSpinLockLock(volatile OSSpinLock *l); OS_ATOMIC_EXPORT bool OSSpinLockTry(volatile OSSpinLock *l); OS_ATOMIC_EXPORT int spin_lock_try(volatile OSSpinLock *l); OS_ATOMIC_EXPORT void OSSpinLockUnlock(volatile OSSpinLock *l); +static const OSSpinLock _OSSpinLockLocked = TARGET_OS_EMBEDDED ? 1 : -1; + + #if OS_ATOMIC_UP // Don't spin on UP -#elif OS_ATOMIC_WFE -#define OS_LOCK_SPIN_SPIN_TRIES 100 -#define OS_LOCK_SPIN_PAUSE() os_hardware_wfe() #else #define OS_LOCK_SPIN_SPIN_TRIES 1000 #define OS_LOCK_SPIN_PAUSE() os_hardware_pause() #endif -static const OSSpinLock _OSSpinLockLocked = TARGET_OS_EMBEDDED ? 1 : -1; - OS_ALWAYS_INLINE static uint64_t _os_lock_yield_deadline(mach_msg_timeout_t timeout) @@ -145,7 +141,7 @@ _OSSpinLockLockSlow(volatile OSSpinLock *l) { return _OSSpinLockLockYield(l); // Don't spin on UP } -#else +#else // !OS_ATOMIC_UP void _OSSpinLockLockSlow(volatile OSSpinLock *l) { @@ -163,21 +159,17 @@ _spin: if (likely(r)) return; goto _spin; } -#endif +#endif // !OS_ATOMIC_UP -#ifdef OS_LOCK_VARIANT_SELECTOR -#undef _OSSpinLockLockSlow -extern void _OSSpinLockLockSlow(volatile OSSpinLock *l); -#endif -#if !OS_LOCK_VARIANT_ONLY #if OS_LOCK_OSSPINLOCK_IS_NOSPINLOCK && !TARGET_OS_SIMULATOR typedef struct _os_nospin_lock_s *_os_nospin_lock_t; -void _os_nospin_lock_lock(_os_nospin_lock_t lock); -bool _os_nospin_lock_trylock(_os_nospin_lock_t lock); -void _os_nospin_lock_unlock(_os_nospin_lock_t lock); + +OS_ATOMIC_EXPORT void _os_nospin_lock_lock(_os_nospin_lock_t lock); +OS_ATOMIC_EXPORT bool _os_nospin_lock_trylock(_os_nospin_lock_t lock); +OS_ATOMIC_EXPORT void _os_nospin_lock_unlock(_os_nospin_lock_t lock); void OSSpinLockLock(volatile OSSpinLock *l) @@ -257,26 +249,15 @@ OSSpinLockUnlock(volatile OSSpinLock *l) os_atomic_store(l, 0, release); } + #pragma mark - #pragma mark os_lock_spin_t OS_LOCK_STRUCT_DECL_INTERNAL(spin, OSSpinLock volatile osl_spinlock; ); -#if !OS_VARIANT_ONLY OS_LOCK_METHODS_DECL(spin); OS_LOCK_TYPE_INSTANCE(spin); -#endif // !OS_VARIANT_ONLY - -#ifdef OS_VARIANT_SELECTOR -#define _os_lock_spin_lock \ - OS_VARIANT(_os_lock_spin_lock, OS_VARIANT_SELECTOR) -#define _os_lock_spin_trylock \ - OS_VARIANT(_os_lock_spin_trylock, OS_VARIANT_SELECTOR) -#define _os_lock_spin_unlock \ - OS_VARIANT(_os_lock_spin_unlock, OS_VARIANT_SELECTOR) -OS_LOCK_METHODS_DECL(spin); -#endif // OS_VARIANT_SELECTOR void _os_lock_spin_lock(_os_lock_spin_t l) @@ -296,6 +277,7 @@ _os_lock_spin_unlock(_os_lock_spin_t l) return OSSpinLockUnlock(&l->osl_spinlock); } + #pragma mark - #pragma mark os_lock_owner_t @@ -304,6 +286,8 @@ _os_lock_spin_unlock(_os_lock_spin_t l) #endif typedef mach_port_name_t os_lock_owner_t; +#define OS_LOCK_NO_OWNER MACH_PORT_NULL + OS_ALWAYS_INLINE static inline os_lock_owner_t @@ -314,9 +298,6 @@ _os_lock_owner_get_self(void) return self; } -#define OS_LOCK_NO_OWNER MACH_PORT_NULL - -#if !OS_LOCK_VARIANT_ONLY OS_NOINLINE OS_NORETURN OS_COLD static void @@ -326,7 +307,6 @@ _os_lock_recursive_abort(os_lock_owner_t owner) "os_lock"); } -#endif //!OS_LOCK_VARIANT_ONLY #pragma mark - #pragma mark os_lock_handoff_t @@ -334,20 +314,8 @@ _os_lock_recursive_abort(os_lock_owner_t owner) OS_LOCK_STRUCT_DECL_INTERNAL(handoff, os_lock_owner_t volatile osl_owner; ); -#if !OS_VARIANT_ONLY OS_LOCK_METHODS_DECL(handoff); OS_LOCK_TYPE_INSTANCE(handoff); -#endif // !OS_VARIANT_ONLY - -#ifdef OS_VARIANT_SELECTOR -#define _os_lock_handoff_lock \ - OS_VARIANT(_os_lock_handoff_lock, OS_VARIANT_SELECTOR) -#define _os_lock_handoff_trylock \ - OS_VARIANT(_os_lock_handoff_trylock, OS_VARIANT_SELECTOR) -#define _os_lock_handoff_unlock \ - OS_VARIANT(_os_lock_handoff_unlock, OS_VARIANT_SELECTOR) -OS_LOCK_METHODS_DECL(handoff); -#endif // OS_VARIANT_SELECTOR #define OS_LOCK_HANDOFF_YIELD_TRIES 100 @@ -397,6 +365,7 @@ _os_lock_handoff_unlock(_os_lock_handoff_t l) os_atomic_store2o(l, osl_owner, MACH_PORT_NULL, release); } + #pragma mark - #pragma mark os_ulock_value_t @@ -411,13 +380,12 @@ typedef os_lock_owner_t os_ulock_value_t; #define OS_ULOCK_OWNER(value) ((value) | OS_ULOCK_NOWAITERS_BIT) #define OS_ULOCK_ANONYMOUS_OWNER MACH_PORT_DEAD -#define OS_ULOCK_IS_OWNER(value, self) ({ \ - os_lock_owner_t _owner = OS_ULOCK_OWNER(value); \ - (_owner == (self) && _owner != OS_ULOCK_ANONYMOUS_OWNER); }) -#define OS_ULOCK_IS_NOT_OWNER(value, self) ({ \ - os_lock_owner_t _owner = OS_ULOCK_OWNER(value); \ - (_owner != (self) && _owner != OS_ULOCK_ANONYMOUS_OWNER); }) - +#define OS_ULOCK_IS_OWNER(value, self, allow_anonymous_owner) ({ \ + os_lock_owner_t _owner = OS_ULOCK_OWNER(value); (_owner == (self)) && \ + (!(allow_anonymous_owner) || _owner != OS_ULOCK_ANONYMOUS_OWNER); }) +#define OS_ULOCK_IS_NOT_OWNER(value, self, allow_anonymous_owner) ({ \ + os_lock_owner_t _owner = OS_ULOCK_OWNER(value); (_owner != (self)) && \ + (!(allow_anonymous_owner) || _owner != OS_ULOCK_ANONYMOUS_OWNER); }) #pragma mark - #pragma mark os_unfair_lock @@ -439,15 +407,27 @@ OS_ATOMIC_EXPORT void os_unfair_lock_lock_no_tsd_4libpthread( os_unfair_lock_t lock); OS_ATOMIC_EXPORT void os_unfair_lock_unlock_no_tsd_4libpthread( os_unfair_lock_t lock); +OS_ATOMIC_EXPORT void os_unfair_lock_lock_with_options_4Libc( + os_unfair_lock_t lock, os_unfair_lock_options_t options); +OS_ATOMIC_EXPORT void os_unfair_lock_unlock_4Libc(os_unfair_lock_t lock); + +OS_NOINLINE OS_NORETURN OS_COLD +void _os_unfair_lock_recursive_abort(os_lock_owner_t owner); +OS_NOINLINE OS_NORETURN OS_COLD +void _os_unfair_lock_unowned_abort(os_lock_owner_t owner); +OS_NOINLINE OS_NORETURN OS_COLD +void _os_unfair_lock_corruption_abort(os_ulock_value_t current); _Static_assert(OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION == ULF_WAIT_WORKQ_DATA_CONTENTION, "check value for OS_UNFAIR_LOCK_OPTIONS_MASK"); #define OS_UNFAIR_LOCK_OPTIONS_MASK \ (os_unfair_lock_options_t)(OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION) +#define OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER 0x01000000u + OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_unfair_lock_recursive_abort(os_lock_owner_t owner) { __LIBPLATFORM_CLIENT_CRASH__(owner, "Trying to recursively lock an " @@ -455,7 +435,7 @@ _os_unfair_lock_recursive_abort(os_lock_owner_t owner) } OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_unfair_lock_unowned_abort(os_lock_owner_t owner) { __LIBPLATFORM_CLIENT_CRASH__(owner, "Unlock of an os_unfair_lock not " @@ -463,25 +443,29 @@ _os_unfair_lock_unowned_abort(os_lock_owner_t owner) } OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_unfair_lock_corruption_abort(os_ulock_value_t current) { __LIBPLATFORM_CLIENT_CRASH__(current, "os_unfair_lock is corrupt"); } + OS_NOINLINE static void _os_unfair_lock_lock_slow(_os_unfair_lock_t l, os_lock_owner_t self, os_unfair_lock_options_t options) { - os_ulock_value_t current, new, waiters_mask = 0; + os_unfair_lock_options_t allow_anonymous_owner = + options & OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER; + options &= ~OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER; if (unlikely(options & ~OS_UNFAIR_LOCK_OPTIONS_MASK)) { __LIBPLATFORM_CLIENT_CRASH__(options, "Invalid options"); } + os_ulock_value_t current, new, waiters_mask = 0; while (unlikely((current = os_atomic_load2o(l, oul_value, relaxed)) != OS_LOCK_NO_OWNER)) { _retry: - if (unlikely(OS_ULOCK_IS_OWNER(current, self))) { + if (unlikely(OS_ULOCK_IS_OWNER(current, self, allow_anonymous_owner))) { return _os_unfair_lock_recursive_abort(self); } new = current & ~OS_ULOCK_NOWAITERS_BIT; @@ -519,9 +503,12 @@ _retry: OS_NOINLINE static void _os_unfair_lock_unlock_slow(_os_unfair_lock_t l, os_ulock_value_t current, - os_lock_owner_t self) + os_lock_owner_t self, os_unfair_lock_options_t options) { - if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self))) { + os_unfair_lock_options_t allow_anonymous_owner = + options & OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER; + options &= ~OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER; + if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self, allow_anonymous_owner))) { return _os_unfair_lock_unowned_abort(OS_ULOCK_OWNER(current)); } if (current & OS_ULOCK_NOWAITERS_BIT) { @@ -581,7 +568,7 @@ os_unfair_lock_unlock(os_unfair_lock_t lock) os_ulock_value_t current; current = os_atomic_xchg2o(l, oul_value, OS_LOCK_NO_OWNER, release); if (likely(current == self)) return; - return _os_unfair_lock_unlock_slow(l, current, self); + return _os_unfair_lock_unlock_slow(l, current, self, 0); } void @@ -592,7 +579,8 @@ os_unfair_lock_lock_no_tsd_4libpthread(os_unfair_lock_t lock) bool r = os_atomic_cmpxchg2o(l, oul_value, OS_LOCK_NO_OWNER, self, acquire); if (likely(r)) return; return _os_unfair_lock_lock_slow(l, self, - OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION); + OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION| + OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER); } void @@ -603,120 +591,18 @@ os_unfair_lock_unlock_no_tsd_4libpthread(os_unfair_lock_t lock) os_ulock_value_t current; current = os_atomic_xchg2o(l, oul_value, OS_LOCK_NO_OWNER, release); if (likely(current == self)) return; - return _os_unfair_lock_unlock_slow(l, current, self); -} - -#pragma mark - -#pragma mark _os_lock_unfair_t 4Libc // - -OS_ATOMIC_EXPORT void os_unfair_lock_lock_with_options_4Libc( - os_unfair_lock_t lock, os_unfair_lock_options_t options); -OS_ATOMIC_EXPORT void os_unfair_lock_unlock_4Libc(os_unfair_lock_t lock); - -OS_NOINLINE -static void -_os_unfair_lock_lock_slow_4Libc(_os_unfair_lock_t l, os_lock_owner_t self, - os_unfair_lock_options_t options) -{ - os_ulock_value_t current, new, waiters_mask = 0; - if (unlikely(options & ~OS_UNFAIR_LOCK_OPTIONS_MASK)) { - __LIBPLATFORM_CLIENT_CRASH__(options, "Invalid options"); - } - while (unlikely((current = os_atomic_load2o(l, oul_value, relaxed)) != - OS_LOCK_NO_OWNER)) { -_retry: - if (unlikely(OS_ULOCK_IS_OWNER(current, self))) { - return _os_unfair_lock_recursive_abort(self); - } - new = current & ~OS_ULOCK_NOWAITERS_BIT; - if (current != new) { - // Clear nowaiters bit in lock value before waiting - if (!os_atomic_cmpxchgv2o(l, oul_value, current, new, ¤t, - relaxed)){ - continue; - } - current = new; - } - int ret = __ulock_wait(UL_UNFAIR_LOCK | ULF_NO_ERRNO | options, - l, current, 0); - if (unlikely(ret < 0)) { - switch (-ret) { - case EINTR: - case EFAULT: - continue; - case EOWNERDEAD: - // if we get an `EOWNERDEAD` it could be corruption of the lock - // so for the Libc locks, if we can steal the lock, assume - // it is corruption and pretend we got the lock with contention - new = self & ~OS_ULOCK_NOWAITERS_BIT; - if (os_atomic_cmpxchgv2o(l, oul_value, current, new, ¤t, - acquire)) { - return; - } - break; - default: - __LIBPLATFORM_INTERNAL_CRASH__(-ret, "ulock_wait failure"); - } - } - // If there are more waiters, unset nowaiters bit when acquiring lock - waiters_mask = (ret > 0) ? OS_ULOCK_NOWAITERS_BIT : 0; - } - new = self & ~waiters_mask; - bool r = os_atomic_cmpxchgv2o(l, oul_value, OS_LOCK_NO_OWNER, new, - ¤t, acquire); - if (unlikely(!r)) goto _retry; + return _os_unfair_lock_unlock_slow(l, current, self, + OS_UNFAIR_LOCK_ALLOW_ANONYMOUS_OWNER); } -OS_NOINLINE -static void -_os_unfair_lock_unlock_slow_4Libc(_os_unfair_lock_t l) -{ - for (;;) { - int ret = __ulock_wake(UL_UNFAIR_LOCK | ULF_NO_ERRNO, l, 0); - if (unlikely(ret < 0)) { - switch (-ret) { - case EINTR: - continue; - case ENOENT: - break; - default: - __LIBPLATFORM_INTERNAL_CRASH__(-ret, "ulock_wake failure"); - } - } - break; - } -} -void -os_unfair_lock_lock_with_options_4Libc(os_unfair_lock_t lock, - os_unfair_lock_options_t options) -{ - _os_unfair_lock_t l = (_os_unfair_lock_t)lock; - os_lock_owner_t self = _os_lock_owner_get_self(); - bool r = os_atomic_cmpxchg2o(l, oul_value, OS_LOCK_NO_OWNER, self, acquire); - if (likely(r)) return; - return _os_unfair_lock_lock_slow_4Libc(l, self, options); -} - -void -os_unfair_lock_unlock_4Libc(os_unfair_lock_t lock) -{ - _os_unfair_lock_t l = (_os_unfair_lock_t)lock; - os_lock_owner_t self = _os_lock_owner_get_self(); - os_ulock_value_t current; - current = os_atomic_xchg2o(l, oul_value, OS_LOCK_NO_OWNER, release); - if (likely(current == self)) return; - return _os_unfair_lock_unlock_slow_4Libc(l); -} - -#if !OS_VARIANT_ONLY void os_unfair_lock_assert_owner(os_unfair_lock_t lock) { _os_unfair_lock_t l = (_os_unfair_lock_t)lock; os_lock_owner_t self = _os_lock_owner_get_self(); os_ulock_value_t current = os_atomic_load2o(l, oul_value, relaxed); - if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self))) { + if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self, 0))) { __LIBPLATFORM_CLIENT_CRASH__(current, "Assertion failed: " "Lock unexpectedly not owned by current thread"); } @@ -728,12 +614,12 @@ os_unfair_lock_assert_not_owner(os_unfair_lock_t lock) _os_unfair_lock_t l = (_os_unfair_lock_t)lock; os_lock_owner_t self = _os_lock_owner_get_self(); os_ulock_value_t current = os_atomic_load2o(l, oul_value, relaxed); - if (unlikely(OS_ULOCK_IS_OWNER(current, self))) { + if (unlikely(OS_ULOCK_IS_OWNER(current, self, 0))) { __LIBPLATFORM_CLIENT_CRASH__(current, "Assertion failed: " "Lock unexpectedly owned by current thread"); } } -#endif + #pragma mark - #pragma mark _os_lock_unfair_t @@ -741,20 +627,8 @@ os_unfair_lock_assert_not_owner(os_unfair_lock_t lock) OS_LOCK_STRUCT_DECL_INTERNAL(unfair, os_unfair_lock osl_unfair_lock; ); -#if !OS_VARIANT_ONLY OS_LOCK_METHODS_DECL(unfair); OS_LOCK_TYPE_INSTANCE(unfair); -#endif // !OS_VARIANT_ONLY - -#ifdef OS_VARIANT_SELECTOR -#define _os_lock_unfair_lock \ - OS_VARIANT(_os_lock_unfair_lock, OS_VARIANT_SELECTOR) -#define _os_lock_unfair_trylock \ - OS_VARIANT(_os_lock_unfair_trylock, OS_VARIANT_SELECTOR) -#define _os_lock_unfair_unlock \ - OS_VARIANT(_os_lock_unfair_unlock, OS_VARIANT_SELECTOR) -OS_LOCK_METHODS_DECL(unfair); -#endif // OS_VARIANT_SELECTOR void _os_lock_unfair_lock(_os_lock_unfair_t l) @@ -774,6 +648,7 @@ _os_lock_unfair_unlock(_os_lock_unfair_t l) return os_unfair_lock_unlock(&l->osl_unfair_lock); } + #pragma mark - #pragma mark _os_nospin_lock @@ -788,6 +663,7 @@ OS_ATOMIC_EXPORT void _os_nospin_lock_lock(_os_nospin_lock_t lock); OS_ATOMIC_EXPORT bool _os_nospin_lock_trylock(_os_nospin_lock_t lock); OS_ATOMIC_EXPORT void _os_nospin_lock_unlock(_os_nospin_lock_t lock); + OS_NOINLINE static void _os_nospin_lock_lock_slow(_os_nospin_lock_t l) @@ -886,26 +762,15 @@ _os_nospin_lock_unlock(_os_nospin_lock_t l) return _os_nospin_lock_unlock_slow(l, current); } + #pragma mark - #pragma mark _os_lock_nospin_t OS_LOCK_STRUCT_DECL_INTERNAL(nospin, _os_nospin_lock osl_nospin_lock; ); -#if !OS_VARIANT_ONLY OS_LOCK_METHODS_DECL(nospin); OS_LOCK_TYPE_INSTANCE(nospin); -#endif // !OS_VARIANT_ONLY - -#ifdef OS_VARIANT_SELECTOR -#define _os_lock_nospin_lock \ - OS_VARIANT(_os_lock_nospin_lock, OS_VARIANT_SELECTOR) -#define _os_lock_nospin_trylock \ - OS_VARIANT(_os_lock_nospin_trylock, OS_VARIANT_SELECTOR) -#define _os_lock_nospin_unlock \ - OS_VARIANT(_os_lock_nospin_unlock, OS_VARIANT_SELECTOR) -OS_LOCK_METHODS_DECL(nospin); -#endif // OS_VARIANT_SELECTOR void _os_lock_nospin_lock(_os_lock_nospin_t l) @@ -925,6 +790,7 @@ _os_lock_nospin_unlock(_os_lock_nospin_t l) return _os_nospin_lock_unlock(&l->osl_nospin_lock); } + #pragma mark - #pragma mark os_once_t @@ -942,7 +808,15 @@ OS_ATOMIC_EXPORT void _os_once(os_once_t *val, void *ctxt, os_function_t func); OS_ATOMIC_EXPORT void __os_once_reset(os_once_t *val); OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_once_gate_recursive_abort(os_lock_owner_t owner); +OS_NOINLINE OS_NORETURN OS_COLD +void _os_once_gate_unowned_abort(os_lock_owner_t owner); +OS_NOINLINE OS_NORETURN OS_COLD +void _os_once_gate_corruption_abort(os_ulock_value_t current); + + +OS_NOINLINE OS_NORETURN OS_COLD +void _os_once_gate_recursive_abort(os_lock_owner_t owner) { __LIBPLATFORM_CLIENT_CRASH__(owner, "Trying to recursively lock an " @@ -950,7 +824,7 @@ _os_once_gate_recursive_abort(os_lock_owner_t owner) } OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_once_gate_unowned_abort(os_lock_owner_t owner) { __LIBPLATFORM_CLIENT_CRASH__(owner, "Unlock of an os_once_t not " @@ -958,12 +832,13 @@ _os_once_gate_unowned_abort(os_lock_owner_t owner) } OS_NOINLINE OS_NORETURN OS_COLD -static void +void _os_once_gate_corruption_abort(os_ulock_value_t current) { __LIBPLATFORM_CLIENT_CRASH__(current, "os_once_t is corrupt"); } + OS_NOINLINE static void _os_once_gate_wait_slow(os_ulock_value_t *gate, os_lock_owner_t self) @@ -980,7 +855,7 @@ _os_once_gate_wait_slow(os_ulock_value_t *gate, os_lock_owner_t self) tid_new = tid_old & ~OS_ULOCK_NOWAITERS_BIT; if (tid_new == tid_old) os_atomic_rmw_loop_give_up(break); }); - if (unlikely(OS_ULOCK_IS_OWNER(tid_old, self))) { + if (unlikely(OS_ULOCK_IS_OWNER(tid_old, self, 0))) { return _os_once_gate_recursive_abort(self); } int ret = __ulock_wait(UL_UNFAIR_LOCK | ULF_NO_ERRNO, @@ -1005,7 +880,7 @@ static void _os_once_gate_broadcast_slow(os_ulock_value_t *gate, os_ulock_value_t current, os_lock_owner_t self) { - if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self))) { + if (unlikely(OS_ULOCK_IS_NOT_OWNER(current, self, 0))) { return _os_once_gate_unowned_abort(OS_ULOCK_OWNER(current)); } if (current & OS_ULOCK_NOWAITERS_BIT) { @@ -1033,60 +908,13 @@ static void _os_once_gate_set_value_and_broadcast(os_once_gate_t og, os_lock_owner_t self, os_once_t value) { - // The next barrier must be long and strong. - // - // The scenario: SMP systems with weakly ordered memory models - // and aggressive out-of-order instruction execution. - // - // The problem: - // - // The os_once*() wrapper macro causes the callee's - // instruction stream to look like this (pseudo-RISC): - // - // load r5, pred-addr - // cmpi r5, -1 - // beq 1f - // call os_once*() - // 1f: - // load r6, data-addr - // - // May be re-ordered like so: - // - // load r6, data-addr - // load r5, pred-addr - // cmpi r5, -1 - // beq 1f - // call os_once*() - // 1f: - // - // Normally, a barrier on the read side is used to workaround - // the weakly ordered memory model. But barriers are expensive - // and we only need to synchronize once! After func(ctxt) - // completes, the predicate will be marked as "done" and the - // branch predictor will correctly skip the call to - // os_once*(). - // - // A far faster alternative solution: Defeat the speculative - // read-ahead of peer CPUs. - // - // Modern architectures will throw away speculative results - // once a branch mis-prediction occurs. Therefore, if we can - // ensure that the predicate is not marked as being complete - // until long after the last store by func(ctxt), then we have - // defeated the read-ahead of peer CPUs. - // - // In other words, the last "store" by func(ctxt) must complete - // and then N cycles must elapse before ~0l is stored to *val. - // The value of N is whatever is sufficient to defeat the - // read-ahead mechanism of peer CPUs. - // - // On some CPUs, the most fully synchronizing instruction might - // need to be issued. - os_atomic_maximally_synchronizing_barrier(); - - // above assumed to contain release barrier - os_ulock_value_t current = - (os_ulock_value_t)os_atomic_xchg(&og->ogo_once, value, relaxed); + os_ulock_value_t current; +#if defined(__i386__) || defined(__x86_64__) + // On Intel, any load is a load-acquire, so we don't need to be fancy + current = (os_ulock_value_t)os_atomic_xchg(&og->ogo_once, value, release); +#else +# error os_once algorithm not available for this architecture +#endif if (likely(current == self)) return; _os_once_gate_broadcast_slow(&og->ogo_lock, current, self); } @@ -1116,7 +944,6 @@ _os_once(os_once_t *val, void *ctxt, os_function_t func) } } -#if !OS_VARIANT_ONLY #pragma mark - #pragma mark os_lock_eliding_t @@ -1145,5 +972,3 @@ OS_LOCK_METHODS_DECL(transactional); OS_LOCK_TYPE_INSTANCE(transactional); #endif // !TARGET_OS_IPHONE -#endif // !OS_VARIANT_ONLY -#endif // !OS_LOCK_VARIANT_ONLY diff --git a/src/os/lock_internal.h b/src/os/lock_internal.h index af6a862..01c2d80 100644 --- a/src/os/lock_internal.h +++ b/src/os/lock_internal.h @@ -56,6 +56,4 @@ .osl_unlock = _os_lock_##type##_unlock, \ } -#include "os/internal.h" - #endif // __OS_LOCK_INTERNAL__ diff --git a/src/os/lock_up.c b/src/os/lock_up.c deleted file mode 100644 index e03a22d..0000000 --- a/src/os/lock_up.c +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2013 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -#define OS_ATOMIC_UP 1 -#include "lock_internal.h" - -#ifdef OS_VARIANT_SELECTOR -#define OS_VARIANT_ONLY 1 -#include "lock.c" -#endif - -struct _os_empty_files_are_not_c_files; diff --git a/src/os/lock_wfe.c b/src/os/lock_wfe.c deleted file mode 100644 index 36dd190..0000000 --- a/src/os/lock_wfe.c +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2013 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -#include - -#if TARGET_OS_EMBEDDED - -#define OS_ATOMIC_WFE 1 -#include "lock_internal.h" - -#ifdef OS_LOCK_VARIANT_SELECTOR -#define OS_VARIANT_ONLY 1 -#define OS_LOCK_VARIANT_ONLY 1 -#include "lock.c" -#endif - -#endif // TARGET_OS_EMBEDDED - -struct _os_empty_files_are_not_c_files; diff --git a/src/os/resolver.c b/src/os/resolver.c deleted file mode 100644 index 2fa6b0a..0000000 --- a/src/os/resolver.c +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) 2013 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -#include "lock_internal.h" - -#ifdef OS_VARIANT_SELECTOR - -#if TARGET_OS_EMBEDDED -OS_VARIANT_UPMP_RESOLVER(OSAtomicAdd32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicIncrement32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicDecrement32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicAdd64Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicIncrement64Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicDecrement64Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicAnd32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicAnd32OrigBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicOr32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicOr32OrigBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicXor32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicXor32OrigBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicCompareAndSwap32Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicCompareAndSwap64Barrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicCompareAndSwapIntBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicCompareAndSwapLongBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicCompareAndSwapPtrBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicTestAndSetBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicTestAndClearBarrier) -OS_VARIANT_UPMP_RESOLVER(OSAtomicEnqueue) -OS_VARIANT_UPMP_RESOLVER(OSAtomicDequeue) -OS_VARIANT_UPMP_RESOLVER(OSMemoryBarrier) - -OS_VARIANT_UPMP_RESOLVER(OSSpinLockLock) -OS_VARIANT_UPMP_RESOLVER(OSSpinLockTry) -OS_VARIANT_UPMP_RESOLVER(OSSpinLockUnlock) -OS_VARIANT_UPMP_RESOLVER(spin_lock) -OS_VARIANT_UPMP_RESOLVER(spin_lock_try) -OS_VARIANT_UPMP_RESOLVER(spin_unlock) -OS_VARIANT_UPMP_RESOLVER(_spin_lock) -OS_VARIANT_UPMP_RESOLVER(_spin_lock_try) -OS_VARIANT_UPMP_RESOLVER(_spin_unlock) - -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_lock) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_lock_with_options) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_trylock) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_unlock) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_lock_no_tsd_4libpthread) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_unlock_no_tsd_4libpthread) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_lock_with_options_4Libc) -OS_VARIANT_UPMP_RESOLVER(os_unfair_lock_unlock_4Libc) -OS_VARIANT_UPMP_RESOLVER(_os_nospin_lock_lock) -OS_VARIANT_UPMP_RESOLVER(_os_nospin_lock_trylock) -OS_VARIANT_UPMP_RESOLVER(_os_nospin_lock_unlock) - -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_spin_lock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_spin_trylock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_spin_unlock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_handoff_lock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_handoff_trylock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_handoff_unlock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_unfair_lock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_unfair_trylock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_unfair_unlock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_nospin_lock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_nospin_trylock) -OS_VARIANT_UPMP_RESOLVER_INTERNAL(_os_lock_nospin_unlock) - -OS_VARIANT_UPMP_RESOLVER(_os_once) -OS_VARIANT_UPMP_RESOLVER(__os_once_reset) -#endif // TARGET_OS_EMBEDDED - -#endif // OS_VARIANT_SELECTOR - -#ifdef OS_LOCK_VARIANT_SELECTOR - -#define WFE_RESOLVER(s) \ - _OS_VARIANT_RESOLVER(s, hidden, \ - uint32_t *_c = (void*)(uintptr_t)_COMM_PAGE_CPU_CAPABILITIES; \ - if (*_c & kHasEvent) { \ - extern void OS_VARIANT(s, wfe)(void); \ - return &OS_VARIANT(s, wfe); \ - } else { \ - extern void OS_VARIANT(s, mp)(void); \ - return &OS_VARIANT(s, mp); \ - }) - -WFE_RESOLVER(_OSSpinLockLockSlow) - -#endif // OS_LOCK_VARIANT_SELECTOR - diff --git a/src/os/resolver.h b/src/os/resolver.h deleted file mode 100644 index 345a6b6..0000000 --- a/src/os/resolver.h +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2013 Apple Inc. All rights reserved. - * - * @APPLE_APACHE_LICENSE_HEADER_START@ - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * @APPLE_APACHE_LICENSE_HEADER_END@ - */ - -#ifndef __OS_RESOLVER_H__ -#define __OS_RESOLVER_H__ - -#include - -#ifdef OS_VARIANT_SELECTOR - -#if TARGET_OS_EMBEDDED -#define OSAtomicAdd32Barrier \ - OS_VARIANT(OSAtomicAdd32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicIncrement32Barrier \ - OS_VARIANT(OSAtomicIncrement32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicDecrement32Barrier \ - OS_VARIANT(OSAtomicDecrement32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicAdd64Barrier \ - OS_VARIANT(OSAtomicAdd64Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicIncrement64Barrier \ - OS_VARIANT(OSAtomicIncrement64Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicDecrement64Barrier \ - OS_VARIANT(OSAtomicDecrement64Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicAnd32Barrier \ - OS_VARIANT(OSAtomicAnd32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicAnd32OrigBarrier \ - OS_VARIANT(OSAtomicAnd32OrigBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicOr32Barrier \ - OS_VARIANT(OSAtomicOr32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicOr32OrigBarrier \ - OS_VARIANT(OSAtomicOr32OrigBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicXor32Barrier \ - OS_VARIANT(OSAtomicXor32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicXor32OrigBarrier \ - OS_VARIANT(OSAtomicXor32OrigBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicCompareAndSwap32Barrier \ - OS_VARIANT(OSAtomicCompareAndSwap32Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicCompareAndSwap64Barrier \ - OS_VARIANT(OSAtomicCompareAndSwap64Barrier, OS_VARIANT_SELECTOR) -#define OSAtomicCompareAndSwapIntBarrier \ - OS_VARIANT(OSAtomicCompareAndSwapIntBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicCompareAndSwapLongBarrier \ - OS_VARIANT(OSAtomicCompareAndSwapLongBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicCompareAndSwapPtrBarrier \ - OS_VARIANT(OSAtomicCompareAndSwapPtrBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicTestAndSetBarrier \ - OS_VARIANT(OSAtomicTestAndSetBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicTestAndClearBarrier \ - OS_VARIANT(OSAtomicTestAndClearBarrier, OS_VARIANT_SELECTOR) -#define OSAtomicEnqueue \ - OS_VARIANT(OSAtomicEnqueue, OS_VARIANT_SELECTOR) -#define OSAtomicDequeue \ - OS_VARIANT(OSAtomicDequeue, OS_VARIANT_SELECTOR) -#define OSMemoryBarrier \ - OS_VARIANT(OSMemoryBarrier, OS_VARIANT_SELECTOR) - -#define OSSpinLockLock \ - OS_VARIANT(OSSpinLockLock, OS_VARIANT_SELECTOR) -#define OSSpinLockTry \ - OS_VARIANT(OSSpinLockTry, OS_VARIANT_SELECTOR) -#define OSSpinLockUnlock \ - OS_VARIANT(OSSpinLockUnlock, OS_VARIANT_SELECTOR) -#define spin_lock \ - OS_VARIANT(spin_lock, OS_VARIANT_SELECTOR) -#define spin_lock_try \ - OS_VARIANT(spin_lock_try, OS_VARIANT_SELECTOR) -#define spin_unlock \ - OS_VARIANT(spin_unlock, OS_VARIANT_SELECTOR) -#define _spin_lock \ - OS_VARIANT(_spin_lock, OS_VARIANT_SELECTOR) -#define _spin_lock_try \ - OS_VARIANT(_spin_lock_try, OS_VARIANT_SELECTOR) -#define _spin_unlock \ - OS_VARIANT(_spin_unlock, OS_VARIANT_SELECTOR) -#define os_unfair_lock_lock \ - OS_VARIANT(os_unfair_lock_lock, OS_VARIANT_SELECTOR) -#define os_unfair_lock_lock_with_options \ - OS_VARIANT(os_unfair_lock_lock_with_options, OS_VARIANT_SELECTOR) -#define os_unfair_lock_trylock \ - OS_VARIANT(os_unfair_lock_trylock, OS_VARIANT_SELECTOR) -#define os_unfair_lock_unlock \ - OS_VARIANT(os_unfair_lock_unlock, OS_VARIANT_SELECTOR) -#define os_unfair_lock_lock_no_tsd_4libpthread \ - OS_VARIANT(os_unfair_lock_lock_no_tsd_4libpthread, OS_VARIANT_SELECTOR) -#define os_unfair_lock_unlock_no_tsd_4libpthread \ - OS_VARIANT(os_unfair_lock_unlock_no_tsd_4libpthread, OS_VARIANT_SELECTOR) -#define os_unfair_lock_lock_with_options_4Libc \ - OS_VARIANT(os_unfair_lock_lock_with_options_4Libc, OS_VARIANT_SELECTOR) -#define os_unfair_lock_unlock_4Libc \ - OS_VARIANT(os_unfair_lock_unlock_4Libc, OS_VARIANT_SELECTOR) -#define _os_nospin_lock_lock \ - OS_VARIANT(_os_nospin_lock_lock, OS_VARIANT_SELECTOR) -#define _os_nospin_lock_trylock \ - OS_VARIANT(_os_nospin_lock_trylock, OS_VARIANT_SELECTOR) -#define _os_nospin_lock_unlock \ - OS_VARIANT(_os_nospin_lock_unlock, OS_VARIANT_SELECTOR) -#define _os_once \ - OS_VARIANT(_os_once, OS_VARIANT_SELECTOR) -#define __os_once_reset \ - OS_VARIANT(__os_once_reset, OS_VARIANT_SELECTOR) -#endif // TARGET_OS_EMBEDDED -#endif // OS_VARIANT_SELECTOR - -#ifdef OS_LOCK_VARIANT_SELECTOR -#define _OSSpinLockLockSlow \ - OS_VARIANT(_OSSpinLockLockSlow, OS_LOCK_VARIANT_SELECTOR) -#endif // OS_LOCK_VARIANT_SELECTOR - -#endif // __OS_RESOLVER_H__ diff --git a/src/os/resolver/resolver.h b/src/os/resolver/resolver.h new file mode 100644 index 0000000..284575b --- /dev/null +++ b/src/os/resolver/resolver.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2013 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __OS_RESOLVER_H__ +#define __OS_RESOLVER_H__ + +#include "resolver_internal.h" + + +#endif // __OS_RESOLVER_H__ diff --git a/src/os/resolver/resolver_internal.h b/src/os/resolver/resolver_internal.h new file mode 100644 index 0000000..e87c918 --- /dev/null +++ b/src/os/resolver/resolver_internal.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +#ifndef __OS_RESOLVER_INTERNAL_H__ +#define __OS_RESOLVER_INTERNAL_H__ + +#include "os/internal.h" + + +#endif // __OS_RESOLVER_INTERNAL_H__ diff --git a/src/os/semaphore.c b/src/os/semaphore.c index d805afc..513de98 100644 --- a/src/os/semaphore.c +++ b/src/os/semaphore.c @@ -41,11 +41,9 @@ _os_semaphore_create(void) { semaphore_t s4; kern_return_t kr; - while (unlikely(kr = semaphore_create(mach_task_self(), &s4, - SYNC_POLICY_FIFO, 0))) { - OS_VERIFY_MIG(kr, "Allocating semaphore failed with MIG_REPLY_MISMATCH"); - thread_switch(MACH_PORT_NULL, SWITCH_OPTION_WAIT, 100); - } + kr = semaphore_create(mach_task_self(), &s4, SYNC_POLICY_FIFO, 0); + OS_VERIFY_MIG(kr, "Allocating semaphore failed with MIG_REPLY_MISMATCH"); + OS_SEMAPHORE_VERIFY_KR(kr, "Creating semaphore failed, possible port leak"); return (os_semaphore_t)s4; } diff --git a/src/simple/asl.c b/src/simple/asl.c index d970ea0..5511e0f 100644 --- a/src/simple/asl.c +++ b/src/simple/asl.c @@ -207,7 +207,7 @@ _simple_asl_msg_new(void) if (b == NULL) return NULL; - if (_simple_sprintf(b, " 0", 0)) + if (_simple_sprintf(b, " 0")) { _simple_sfree(b); return NULL; @@ -224,7 +224,7 @@ _simple_asl_msg_set(_SIMPLE_STRING __b, const char *__key, const char *__val) do { - if (_simple_sprintf(__b, " [", 0)) break; + if (_simple_sprintf(__b, " [")) break; if (_simple_esprintf(__b, _simple_asl_escape_key, "%s", __key)) break; if (__val != NULL) { @@ -265,13 +265,13 @@ _simple_asl_send(_SIMPLE_STRING __b) { char *cp; - if (_simple_sprintf(__b, " [PID ", 0)) break; + if (_simple_sprintf(__b, " [PID ")) break; if (_simple_esprintf(__b, _simple_asl_escape_val, "%u", getpid())) break; - if (_simple_sprintf(__b, "] [UID ", 0)) break; + if (_simple_sprintf(__b, "] [UID ")) break; if (_simple_esprintf(__b, _simple_asl_escape_val, "%u", getuid())) break; - if (_simple_sprintf(__b, "] [GID ", 0)) break; + if (_simple_sprintf(__b, "] [GID ")) break; if (_simple_esprintf(__b, _simple_asl_escape_val, "%u", getgid())) break; - if (_simple_sprintf(__b, "] [Time ", 0)) break; + if (_simple_sprintf(__b, "] [Time ")) break; if (_simple_esprintf(__b, _simple_asl_escape_val, "%lu", tv.tv_sec)) break; if (_simple_sappend(__b, "] [TimeNanoSec ")) break; if (_simple_esprintf(__b, _simple_asl_escape_val, "%d", tv.tv_usec * 1000)) break; diff --git a/src/simple/string_io.c b/src/simple/string_io.c index 2e581e8..4bffac6 100644 --- a/src/simple/string_io.c +++ b/src/simple/string_io.c @@ -37,8 +37,12 @@ #endif #define BUF_SIZE(s) (((BUF *)(s))->end - ((BUF *)(s))->buf + 1) +#if DEBUG +#define MYBUFSIZE 256 +#else /* we use a small buffer to minimize stack usage constraints */ #define MYBUFSIZE 32 +#endif typedef struct _BUF { char *buf; @@ -439,13 +443,10 @@ _simple_dprintf(int fd, const char *fmt, ...) _SIMPLE_STRING _simple_salloc(void) { - kern_return_t kr; BUF *b; - kr = vm_allocate(mach_task_self(), (vm_address_t *)&b, VM_PAGE_SIZE, 1); - if (kr) { - __LIBPLATFORM_CLIENT_CRASH__(kr, "Failed to allocate memory for string"); - } + if(vm_allocate(mach_task_self(), (vm_address_t *)&b, VM_PAGE_SIZE, 1)) + return NULL; b->ptr = b->buf = (char *)b + sizeof(BUF); b->end = (char *)b + VM_PAGE_SIZE - 1; b->full = _enlarge; @@ -455,7 +456,8 @@ _simple_salloc(void) /* * The format string is interpreted with arguments from the va_list, and the * results are appended to the string maintained by the opaque structure, as - * returned by a previous call to _simple_salloc(). Always returns 0. + * returned by a previous call to _simple_salloc(). Non-zero is returned on + * out-of-memory error. */ int _simple_vsprintf(_SIMPLE_STRING b, const char *fmt, va_list ap) @@ -466,8 +468,8 @@ _simple_vsprintf(_SIMPLE_STRING b, const char *fmt, va_list ap) /* * The format string is interpreted with arguments from the variable argument * list, and the results are appended to the string maintained by the opaque - * structure, as returned by a previous call to _simple_salloc(). - * Always returns 0. + * structure, as returned by a previous call to _simple_salloc(). Non-zero is + * returned on out-of-memory error. */ int _simple_sprintf(_SIMPLE_STRING b, const char *fmt, ...) @@ -532,7 +534,7 @@ _simple_sresize(_SIMPLE_STRING b) /* * Append the null-terminated string to the string associated with the opaque - * structure. Always returns 0. + * structure. Non-zero is returned on out-of-memory error. */ int _simple_sappend(_SIMPLE_STRING b, const char *str) diff --git a/xcodeconfig/libplatform.aliases b/xcodeconfig/libplatform.aliases index 842ba78..4e8ce39 100644 --- a/xcodeconfig/libplatform.aliases +++ b/xcodeconfig/libplatform.aliases @@ -1 +1,3 @@ __platform_bzero ___bzero +_os_unfair_lock_lock_with_options _os_unfair_lock_lock_with_options_4Libc +_os_unfair_lock_unlock _os_unfair_lock_unlock_4Libc diff --git a/xcodeconfig/libplatform.xcconfig b/xcodeconfig/libplatform.xcconfig index e607f4b..ce479ba 100644 --- a/xcodeconfig/libplatform.xcconfig +++ b/xcodeconfig/libplatform.xcconfig @@ -19,16 +19,16 @@ INSTALLHDRS_SCRIPT_PHASE = YES GCC_OPTIMIZATION_LEVEL = s // TODO: Remove -fno-stack-protector once it has been moved down (after libproc is moved down) -OTHER_CFLAGS = -fno-stack-protector -fdollars-in-identifiers -fno-common -fverbose-asm $(COMPILER_CFLAGS) -isystem $(SYSTEM_FRAMEWORK_HEADERS) +OTHER_CFLAGS = -fno-stack-protector -fdollars-in-identifiers -fno-common -fverbose-asm $(COMPILER_CFLAGS) $(PLATFORM_CFLAGS) -isystem $(SYSTEM_FRAMEWORK_HEADERS) OTHER_CFLAGS_normal = -momit-leaf-frame-pointer OTHER_CFLAGS_debug = -fno-inline -O0 -GCC_PREPROCESSOR_DEFINITIONS = $(GCC_PREPROCESSOR_DEFINITIONS_$(CURRENT_VARIANT)) $(OSATOMIC_PREPROCESSOR_DEFINITIONS) +GCC_PREPROCESSOR_DEFINITIONS = _FORTIFY_SOURCE=0 $(OSATOMIC_PREPROCESSOR_DEFINITIONS) $(PLATFORM_PREPROCESSOR_DEFINITIONS) $(GCC_PREPROCESSOR_DEFINITIONS_$(CURRENT_VARIANT)) GCC_PREPROCESSOR_DEFINITIONS_dyld = VARIANT_DYLD=1 VARIANT_NO_RESOLVERS=1 VARIANT_STATIC=1 GCC_PREPROCESSOR_DEFINITIONS_static = VARIANT_NO_RESOLVERS=1 VARIANT_STATIC=1 GCC_PREPROCESSOR_DEFINITIONS_debug = DEBUG=1 -OSATOMIC_PREPROCESSOR_DEFINITIONS = OSATOMIC_USE_INLINED=1 OSSPINLOCK_USE_INLINED=1 OS_UNFAIR_LOCK_INLINE=0 +OSATOMIC_PREPROCESSOR_DEFINITIONS = OSATOMIC_USE_INLINED=0 OSATOMIC_DEPRECATED=0 OSSPINLOCK_USE_INLINED=1 OS_UNFAIR_LOCK_INLINE=0 STRIP_INSTALLED_PRODUCT = $(STRIP_INSTALLED_PRODUCT_$(CURRENT_VARIANT)) STRIP_INSTALLED_PRODUCT_normal = YES @@ -63,6 +63,7 @@ OTHER_LIBTOOLFLAGS_libplatform_dyld = $(PLATFORM_LIBRARIES) OTHER_LIBTOOLFLAGS_libplatform_static = $(PLATFORM_LIBRARIES) OTHER_LIBTOOLFLAGS_libatomics = $(ATOMICS_LIBRARIES) OTHER_LIBTOOLFLAGS_libcachecontrol = $(CACHECONTROL_LIBRARIES) +OTHER_LIBTOOLFLAGS_libos = $(OS_LIBRARIES_$(CURRENT_VARIANT)) OTHER_LIBTOOLFLAGS_libsetjmp = $(SETJMP_LIBRARIES) OTHER_LIBTOOLFLAGS_libstring = $(STRING_LIBRARIES) OTHER_LIBTOOLFLAGS_libucontext = $(UCONTEXT_LIBRARIES) diff --git a/xcodeconfig/os.xcconfig b/xcodeconfig/os.xcconfig index b163c0f..3a94018 100644 --- a/xcodeconfig/os.xcconfig +++ b/xcodeconfig/os.xcconfig @@ -1,5 +1,3 @@ -#include "libplatform.xcconfig" - GCC_STRICT_ALIASING = YES GCC_SYMBOLS_PRIVATE_EXTERN = YES GCC_WARN_SHADOW = YES @@ -21,8 +19,10 @@ WARNING_CFLAGS = -Wall -Wextra -Waggregate-return -Wfloat-equal -Wpacked -Wmissi COMPILER_CFLAGS = -momit-leaf-frame-pointer OTHER_CFLAGS_debug = +SRCROOT_SEARCH_PATHS = $(inherited) $(SRCROOT)/src/os/resolver OSATOMIC_PREPROCESSOR_DEFINITIONS = OSATOMIC_USE_INLINED=0 OSATOMIC_DEPRECATED=0 OSSPINLOCK_USE_INLINED=0 OSSPINLOCK_DEPRECATED=0 PUBLIC_HEADERS_FOLDER_PATH = /usr/include/os PRIVATE_HEADERS_FOLDER_PATH = /usr/local/include/os OS_INTERNAL_HEADERS_FOLDER_PATH = /usr/local/include/os/internal +