X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/2d21ac55c334faf3a56e5634905ed6987fc787d4..813fb2f63a553c957e917ede5f119b021d6ce391:/bsd/sys/mcache.h diff --git a/bsd/sys/mcache.h b/bsd/sys/mcache.h index caa625031..906363384 100644 --- a/bsd/sys/mcache.h +++ b/bsd/sys/mcache.h @@ -1,8 +1,8 @@ /* - * Copyright (c) 2006-2007 Apple Inc. All rights reserved. + * Copyright (c) 2006-2015 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _SYS_MCACHE_H @@ -37,7 +37,9 @@ extern "C" { #include #include #include +#include #include +#include #ifdef ASSERT #undef ASSERT @@ -50,20 +52,136 @@ extern "C" { /* * Unlike VERIFY(), ASSERT() is evaluated only in DEBUG build. */ -#define VERIFY(EX) ((void)((EX) || assfail(#EX, __FILE__, __LINE__))) +#define VERIFY(EX) \ + ((void)(__probable((EX)) || assfail(#EX, __FILE__, __LINE__))) #if DEBUG #define ASSERT(EX) VERIFY(EX) #else #define ASSERT(EX) ((void)0) #endif -#if defined(__ppc__) -#define CPU_CACHE_SIZE 128 -#elif defined(__arm__) -#define CPU_CACHE_SIZE 32 +/* + * Compile time assert; this should be on its own someday. + */ +#define _CASSERT(x) _Static_assert(x, "compile-time assertion failed") + +/* + * Atomic macros; these should be on their own someday. + */ +#define atomic_add_16_ov(a, n) \ + ((u_int16_t) OSAddAtomic16(n, (volatile SInt16 *)a)) + +#define atomic_add_16(a, n) \ + ((void) atomic_add_16_ov(a, n)) + +#define atomic_add_32_ov(a, n) \ + ((u_int32_t) OSAddAtomic(n, (volatile SInt32 *)a)) + +#define atomic_add_32(a, n) \ + ((void) atomic_add_32_ov(a, n)) + +#define atomic_add_64_ov(a, n) \ + ((u_int64_t) OSAddAtomic64(n, (volatile SInt64 *)a)) + +#define atomic_add_64(a, n) \ + ((void) atomic_add_64_ov(a, n)) + +#define atomic_test_set_32(a, o, n) \ + OSCompareAndSwap(o, n, (volatile UInt32 *)a) + +#define atomic_set_32(a, n) do { \ + while (!atomic_test_set_32(a, *a, n)) \ + ; \ +} while (0) + +#define atomic_test_set_64(a, o, n) \ + OSCompareAndSwap64(o, n, (volatile UInt64 *)a) + +#define atomic_set_64(a, n) do { \ + while (!atomic_test_set_64(a, *a, n)) \ + ; \ +} while (0) + +#if defined(__LP64__) +#define atomic_get_64(n, a) do { \ + (n) = *(a); \ +} while (0) #else -#define CPU_CACHE_SIZE 64 -#endif +#define atomic_get_64(n, a) do { \ + (n) = atomic_add_64_ov(a, 0); \ +} while (0) +#endif /* __LP64__ */ + +#define atomic_test_set_ptr(a, o, n) \ + OSCompareAndSwapPtr(o, n, (void * volatile *)a) + +#define atomic_set_ptr(a, n) do { \ + while (!atomic_test_set_ptr(a, *a, n)) \ + ; \ +} while (0) + +#define atomic_or_8_ov(a, n) \ + ((u_int8_t) OSBitOrAtomic8(n, (volatile UInt8 *)a)) + +#define atomic_or_8(a, n) \ + ((void) atomic_or_8_ov(a, n)) + +#define atomic_bitset_8(a, n) \ + atomic_or_8(a, n) + +#define atomic_or_16_ov(a, n) \ + ((u_int16_t) OSBitOrAtomic16(n, (volatile UInt16 *)a)) + +#define atomic_or_16(a, n) \ + ((void) atomic_or_16_ov(a, n)) + +#define atomic_bitset_16(a, n) \ + atomic_or_16(a, n) + +#define atomic_or_32_ov(a, n) \ + ((u_int32_t) OSBitOrAtomic(n, (volatile UInt32 *)a)) + +#define atomic_or_32(a, n) \ + ((void) atomic_or_32_ov(a, n)) + +#define atomic_bitset_32(a, n) \ + atomic_or_32(a, n) + +#define atomic_and_8_ov(a, n) \ + ((u_int8_t) OSBitAndAtomic8(n, (volatile UInt8 *)a)) + +#define atomic_and_8(a, n) \ + ((void) atomic_and_8_ov(a, n)) + +#define atomic_bitclear_8(a, n) \ + atomic_and_8(a, ~(n)) + +#define atomic_and_16_ov(a, n) \ + ((u_int16_t) OSBitAndAtomic16(n, (volatile UInt16 *)a)) + +#define atomic_and_16(a, n) \ + ((void) atomic_and_16_ov(a, n)) + +#define atomic_bitclear_16(a, n) \ + atomic_and_16(a, ~(n)) + +#define atomic_and_32_ov(a, n) \ + ((u_int32_t) OSBitAndAtomic(n, (volatile UInt32 *)a)) + +#define atomic_and_32(a, n) \ + ((void) atomic_and_32_ov(a, n)) + +#define atomic_bitclear_32(a, n) \ + atomic_and_32(a, ~(n)) + +#define membar_sync OSMemoryBarrier + +/* + * Use CPU_CACHE_LINE_SIZE instead of MAX_CPU_CACHE_LINE_SIZE, unless + * wasting space is of no concern. + */ +#define MAX_CPU_CACHE_LINE_SIZE 128 +#define CPU_CACHE_LINE_SIZE mcache_cache_line_size() #ifndef IS_P2ALIGNED #define IS_P2ALIGNED(v, a) \ @@ -72,7 +190,7 @@ extern "C" { #ifndef P2ROUNDUP #define P2ROUNDUP(x, align) \ - (-(-((uintptr_t)(x)) & -(align))) + (-(-((uintptr_t)(x)) & -((uintptr_t)align))) #endif /* P2ROUNDUP */ #ifndef P2ROUNDDOWN @@ -148,12 +266,13 @@ typedef struct mcache_cpu { int cc_objs; /* number of objects in filled bkt */ int cc_pobjs; /* number of objects in previous bkt */ int cc_bktsize; /* number of elements in a full bkt */ -} __attribute__((aligned(CPU_CACHE_SIZE), packed)) mcache_cpu_t; +} __attribute__((aligned(MAX_CPU_CACHE_LINE_SIZE))) mcache_cpu_t; typedef unsigned int (*mcache_allocfn_t)(void *, mcache_obj_t ***, unsigned int, int); typedef void (*mcache_freefn_t)(void *, mcache_obj_t *, boolean_t); typedef void (*mcache_auditfn_t)(void *, mcache_obj_t *, boolean_t); +typedef void (*mcache_logfn_t)(u_int32_t, mcache_obj_t *, boolean_t); typedef void (*mcache_notifyfn_t)(void *, u_int32_t); typedef struct mcache { @@ -166,6 +285,7 @@ typedef struct mcache { mcache_allocfn_t mc_slab_alloc; /* slab layer allocate callback */ mcache_freefn_t mc_slab_free; /* slab layer free callback */ mcache_auditfn_t mc_slab_audit; /* slab layer audit callback */ + mcache_logfn_t mc_slab_log; /* slab layer log callback */ mcache_notifyfn_t mc_slab_notify; /* slab layer notify callback */ void *mc_private; /* opaque arg to callbacks */ size_t mc_bufsize; /* object size */ @@ -205,61 +325,70 @@ typedef struct mcache { /* * Per-CPU layer, aligned at cache line boundary */ - mcache_cpu_t mc_cpu[1]; + mcache_cpu_t mc_cpu[1] + __attribute__((aligned(MAX_CPU_CACHE_LINE_SIZE))); } mcache_t; #define MCACHE_ALIGN 8 /* default guaranteed alignment */ /* Valid values for mc_flags */ #define MCF_VERIFY 0x00000001 /* enable verification */ -#define MCF_AUDIT 0x00000002 /* enable transaction auditing */ +#define MCF_TRACE 0x00000002 /* enable transaction auditing */ #define MCF_NOCPUCACHE 0x00000010 /* disable CPU layer caching */ +#define MCF_NOLEAKLOG 0x00000100 /* disable leak logging */ +#define MCF_EXPLEAKLOG 0x00000200 /* expose leak info to user space */ -#define MCF_DEBUG (MCF_VERIFY | MCF_AUDIT) -#define MCF_FLAGS_MASK (MCF_DEBUG | MCF_NOCPUCACHE) +#define MCF_DEBUG (MCF_VERIFY | MCF_TRACE) +#define MCF_FLAGS_MASK \ + (MCF_DEBUG | MCF_NOCPUCACHE | MCF_NOLEAKLOG | MCF_EXPLEAKLOG) /* Valid values for notify callback */ #define MCN_RETRYALLOC 0x00000001 /* Allocation should be retried */ #define MCACHE_STACK_DEPTH 16 +#define MCA_TRN_MAX 2 /* Number of transactions to record */ + typedef struct mcache_audit { struct mcache_audit *mca_next; /* next audit struct */ void *mca_addr; /* address of buffer */ mcache_t *mca_cache; /* parent cache of the buffer */ - struct thread *mca_thread; /* thread doing transaction */ - struct thread *mca_pthread; /* previous transaction thread */ - size_t mca_contents_size; /* size of contents */ - void *mca_contents; /* contents at last free */ - uint16_t mca_depth; /* pc stack depth */ - uint16_t mca_pdepth; /* previous transaction pc stack */ - void *mca_stack[MCACHE_STACK_DEPTH]; - void *mca_pstack[MCACHE_STACK_DEPTH]; + size_t mca_contents_size; /* size of saved contents */ + void *mca_contents; /* user-specific saved contents */ void *mca_uptr; /* user-specific pointer */ uint32_t mca_uflags; /* user-specific flags */ + uint32_t mca_next_trn; + struct mca_trn { + struct thread *mca_thread; /* thread doing transaction */ + uint32_t mca_tstamp; + uint16_t mca_depth; + void *mca_stack[MCACHE_STACK_DEPTH]; + } mca_trns[MCA_TRN_MAX]; } mcache_audit_t; __private_extern__ int assfail(const char *, const char *, int); __private_extern__ void mcache_init(void); __private_extern__ unsigned int mcache_getflags(void); +__private_extern__ unsigned int mcache_cache_line_size(void); __private_extern__ mcache_t *mcache_create(const char *, size_t, size_t, u_int32_t, int); __private_extern__ void *mcache_alloc(mcache_t *, int); __private_extern__ void mcache_free(mcache_t *, void *); __private_extern__ mcache_t *mcache_create_ext(const char *, size_t, - mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_notifyfn_t, - void *, u_int32_t, int); + mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_logfn_t, + mcache_notifyfn_t, void *, u_int32_t, int); __private_extern__ void mcache_destroy(mcache_t *); __private_extern__ unsigned int mcache_alloc_ext(mcache_t *, mcache_obj_t **, unsigned int, int); __private_extern__ void mcache_free_ext(mcache_t *, mcache_obj_t *); __private_extern__ void mcache_reap(void); -__private_extern__ boolean_t mcache_purge_cache(mcache_t *); +__private_extern__ boolean_t mcache_purge_cache(mcache_t *, boolean_t); __private_extern__ void mcache_waiter_inc(mcache_t *); __private_extern__ void mcache_waiter_dec(mcache_t *); __private_extern__ boolean_t mcache_bkt_isempty(mcache_t *); -__private_extern__ void mcache_buffer_log(mcache_audit_t *, void *, mcache_t *); +__private_extern__ void mcache_buffer_log(mcache_audit_t *, void *, mcache_t *, + struct timeval *); __private_extern__ void mcache_set_pattern(u_int64_t, void *, size_t); __private_extern__ void *mcache_verify_pattern(u_int64_t, void *, size_t); __private_extern__ void *mcache_verify_set_pattern(u_int64_t, u_int64_t, @@ -272,7 +401,11 @@ __private_extern__ char *mcache_dump_mca(mcache_audit_t *); __private_extern__ void mcache_audit_panic(mcache_audit_t *, void *, size_t, int64_t, int64_t); -__private_extern__ mcache_t *mcache_audit_cache; +extern int32_t total_sbmb_cnt; +extern int32_t total_sbmb_cnt_floor; +extern int32_t total_sbmb_cnt_peak; +extern int64_t sbmb_limreached; +extern mcache_t *mcache_audit_cache; #ifdef __cplusplus }