osfmk/i386/locks.h

   1 /*
   2  * Copyright (c) 2004-2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #ifndef _I386_LOCKS_H_
  30 #define _I386_LOCKS_H_
  31
  32 #include <sys/appleapiopts.h>
  33 #include <kern/kern_types.h>
  34 #include <kern/assert.h>
  35
  36 #ifdef  MACH_KERNEL_PRIVATE
  37
  38 #include <i386/hw_lock_types.h>
  39
  40 extern  unsigned int    LcksOpts;
  41 #if DEVELOPMENT || DEBUG
  42 extern  unsigned int    LckDisablePreemptCheck;
  43 #endif
  44
  45 #define enaLkDeb                0x00000001      /* Request debug in default attribute */
  46 #define enaLkStat               0x00000002      /* Request statistic in default attribute */
  47 #define disLkRWPrio             0x00000004      /* Disable RW lock priority promotion */
  48 #define enaLkTimeStat           0x00000008      /* Request time statistics in default attribute */
  49
  50 #endif /* MACH_KERNEL_PRIVATE */
  51
  52 #if     defined(MACH_KERNEL_PRIVATE)
  53 typedef struct {
  54         volatile uintptr_t      interlock;
  55 #if     MACH_LDEBUG
  56         unsigned long   lck_spin_pad[9];        /* XXX - usimple_lock_data_t */
  57 #endif
  58 } lck_spin_t;
  59
  60 #define LCK_SPIN_TAG_DESTROYED          0x00002007      /* lock marked as Destroyed */
  61
  62 #else /* MACH_KERNEL_PRIVATE */
  63 #ifdef  KERNEL_PRIVATE
  64 typedef struct {
  65         unsigned long    opaque[10];
  66 } lck_spin_t;
  67 #else /* KERNEL_PRIVATE */
  68 typedef struct __lck_spin_t__   lck_spin_t;
  69 #endif
  70 #endif
  71
  72 #ifdef  MACH_KERNEL_PRIVATE
  73 /* The definition of this structure, including the layout of the
  74  * state bitfield, is tailored to the asm implementation in i386_lock.s
  75  */
  76 typedef struct _lck_mtx_ {
  77         union {
  78                 struct {
  79                         volatile uintptr_t              lck_mtx_owner;
  80                         union {
  81                                 struct {
  82                                         volatile uint32_t
  83                                             lck_mtx_waiters:16,
  84                                             lck_mtx_pri:8, // unused
  85                                             lck_mtx_ilocked:1,
  86                                             lck_mtx_mlocked:1,
  87                                             lck_mtx_promoted:1, // unused
  88                                             lck_mtx_spin:1,
  89                                             lck_mtx_is_ext:1,
  90                                             lck_mtx_pad3:3;
  91                                 };
  92                                 uint32_t        lck_mtx_state;
  93                         };
  94                         /* Pad field used as a canary, initialized to ~0 */
  95                         uint32_t                        lck_mtx_pad32;
  96                 };
  97                 struct {
  98                         struct _lck_mtx_ext_            *lck_mtx_ptr;
  99                         uint32_t                        lck_mtx_tag;
 100                         uint32_t                        lck_mtx_pad32_2;
 101                 };
 102         };
 103 } lck_mtx_t;
 104
 105 #define LCK_MTX_WAITERS_MSK             0x0000ffff
 106 #define LCK_MTX_WAITER                  0x00000001
 107 #define LCK_MTX_PRIORITY_MSK            0x00ff0000
 108 #define LCK_MTX_ILOCKED_MSK             0x01000000
 109 #define LCK_MTX_MLOCKED_MSK             0x02000000
 110 #define LCK_MTX_SPIN_MSK                0x08000000
 111
 112 /* This pattern must subsume the interlocked, mlocked and spin bits */
 113 #define LCK_MTX_TAG_INDIRECT                    0x07ff1007      /* lock marked as Indirect  */
 114 #define LCK_MTX_TAG_DESTROYED                   0x07fe2007      /* lock marked as Destroyed */
 115
 116 /* Adaptive spin before blocking */
 117 extern uint64_t         MutexSpin;
 118 extern uint64_t         low_MutexSpin;
 119 extern int64_t         high_MutexSpin;
 120
 121 typedef enum lck_mtx_spinwait_ret_type {
 122         LCK_MTX_SPINWAIT_ACQUIRED = 0,
 123
 124         LCK_MTX_SPINWAIT_SPUN_HIGH_THR = 1,
 125         LCK_MTX_SPINWAIT_SPUN_OWNER_NOT_CORE = 2,
 126         LCK_MTX_SPINWAIT_SPUN_NO_WINDOW_CONTENTION = 3,
 127         LCK_MTX_SPINWAIT_SPUN_SLIDING_THR = 4,
 128
 129         LCK_MTX_SPINWAIT_NO_SPIN = 5,
 130 } lck_mtx_spinwait_ret_type_t;
 131
 132 extern lck_mtx_spinwait_ret_type_t              lck_mtx_lock_spinwait_x86(lck_mtx_t *mutex);
 133 struct turnstile;
 134 extern void                                     lck_mtx_lock_wait_x86(lck_mtx_t *mutex, struct turnstile **ts);
 135 extern void                                     lck_mtx_lock_acquire_x86(lck_mtx_t *mutex);
 136
 137 extern void                                     lck_mtx_lock_slow(lck_mtx_t *lock);
 138 extern boolean_t                                lck_mtx_try_lock_slow(lck_mtx_t *lock);
 139 extern void                                     lck_mtx_unlock_slow(lck_mtx_t *lock);
 140 extern void                                     lck_mtx_lock_spin_slow(lck_mtx_t *lock);
 141 extern boolean_t                                lck_mtx_try_lock_spin_slow(lck_mtx_t *lock);
 142 extern void                                     hw_lock_byte_init(volatile uint8_t *lock_byte);
 143 extern void                                     hw_lock_byte_lock(volatile uint8_t *lock_byte);
 144 extern void                                     hw_lock_byte_unlock(volatile uint8_t *lock_byte);
 145
 146 typedef struct {
 147         unsigned int            type;
 148         unsigned int            pad4;
 149         vm_offset_t             pc;
 150         vm_offset_t             thread;
 151 } lck_mtx_deb_t;
 152
 153 #define MUTEX_TAG       0x4d4d
 154
 155 typedef struct {
 156         unsigned int            lck_mtx_stat_data;
 157 } lck_mtx_stat_t;
 158
 159 typedef struct _lck_mtx_ext_ {
 160         lck_mtx_t               lck_mtx;
 161         struct _lck_grp_        *lck_mtx_grp;
 162         unsigned int            lck_mtx_attr;
 163         unsigned int            lck_mtx_pad1;
 164         lck_mtx_deb_t           lck_mtx_deb;
 165         uint64_t                lck_mtx_stat;
 166         unsigned int            lck_mtx_pad2[2];
 167 } lck_mtx_ext_t;
 168
 169 #define LCK_MTX_ATTR_DEBUG      0x1
 170 #define LCK_MTX_ATTR_DEBUGb     0
 171 #define LCK_MTX_ATTR_STAT       0x2
 172 #define LCK_MTX_ATTR_STATb      1
 173
 174 #define LCK_MTX_EVENT(lck)        ((event_t)(((unsigned int*)(lck))+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int)))
 175 #define LCK_EVENT_TO_MUTEX(event) ((lck_mtx_t *)(uintptr_t)(((unsigned int *)(event)) - ((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))))
 176
 177 #else /* MACH_KERNEL_PRIVATE */
 178 #ifdef  XNU_KERNEL_PRIVATE
 179 typedef struct {
 180         unsigned long           opaque[2];
 181 } lck_mtx_t;
 182
 183 typedef struct {
 184         unsigned long           opaque[10];
 185 } lck_mtx_ext_t;
 186 #else
 187 #ifdef  KERNEL_PRIVATE
 188 typedef struct {
 189         unsigned long           opaque[2];
 190 } lck_mtx_t;
 191
 192 typedef struct {
 193         unsigned long           opaque[10];
 194 } lck_mtx_ext_t;
 195
 196 #else
 197 typedef struct __lck_mtx_t__            lck_mtx_t;
 198 typedef struct __lck_mtx_ext_t__        lck_mtx_ext_t;
 199 #endif
 200 #endif
 201 #endif
 202
 203 #ifdef  MACH_KERNEL_PRIVATE
 204 typedef union _lck_rw_t_internal_ {
 205         struct {
 206                 volatile uint16_t       lck_rw_shared_count;    /* No. of accepted readers */
 207                 volatile uint8_t        lck_rw_interlock;       /* Interlock byte */
 208                 volatile uint8_t
 209                     lck_rw_priv_excl:1,                         /* Writers prioritized if set */
 210                     lck_rw_want_upgrade:1,                      /* Read-to-write upgrade waiting */
 211                     lck_rw_want_write:1,                        /* Writer waiting or locked for write */
 212                     lck_r_waiting:1,                            /* Reader is sleeping on lock */
 213                     lck_w_waiting:1,                            /* Writer is sleeping on lock */
 214                     lck_rw_can_sleep:1,                         /* Can attempts to lock go to sleep? */
 215                     lck_rw_padb6:2;                             /* padding */
 216                 uint32_t                lck_rw_tag;             /* This can be obsoleted when stats are in */
 217                 thread_t                lck_rw_owner;           /* Unused */
 218         };
 219         struct {
 220                 uint32_t                data;                   /* Single word for count, ilk, and bitfields */
 221                 uint32_t                lck_rw_pad4;
 222                 uint32_t                lck_rw_pad8;
 223                 uint32_t                lck_rw_pad12;
 224         };
 225 } lck_rw_t;
 226 #define LCK_RW_T_SIZE           16
 227
 228 static_assert(sizeof(lck_rw_t) == LCK_RW_T_SIZE);
 229
 230 #define LCK_RW_SHARED_SHIFT      0
 231 #define LCK_RW_INTERLOCK_BIT    16
 232 #define LCK_RW_PRIV_EXCL_BIT    24
 233 #define LCK_RW_WANT_UPGRADE_BIT 25
 234 #define LCK_RW_WANT_EXCL_BIT    26
 235 #define LCK_RW_R_WAITING_BIT    27
 236 #define LCK_RW_W_WAITING_BIT    28
 237 #define LCK_RW_CAN_SLEEP_BIT    29
 238
 239 #define LCK_RW_INTERLOCK        (1 << LCK_RW_INTERLOCK_BIT)
 240 #define LCK_RW_WANT_UPGRADE     (1 << LCK_RW_WANT_UPGRADE_BIT)
 241 #define LCK_RW_WANT_EXCL        (1 << LCK_RW_WANT_EXCL_BIT)
 242 #define LCK_RW_R_WAITING        (1 << LCK_RW_R_WAITING_BIT)
 243 #define LCK_RW_W_WAITING        (1 << LCK_RW_W_WAITING_BIT)
 244 #define LCK_RW_PRIV_EXCL        (1 << LCK_RW_PRIV_EXCL_BIT)
 245 #define LCK_RW_TAG_VALID        (1 << LCK_RW_TAG_VALID_BIT)
 246 #define LCK_RW_SHARED_MASK      (0xffff << LCK_RW_SHARED_SHIFT)
 247 #define LCK_RW_SHARED_READER    (1 << LCK_RW_SHARED_SHIFT)
 248
 249 #define LCK_RW_WANT_WRITE       LCK_RW_WANT_EXCL
 250
 251
 252 #define LCK_RW_ATTR_DEBUG       0x1
 253 #define LCK_RW_ATTR_DEBUGb      0
 254 #define LCK_RW_ATTR_STAT        0x2
 255 #define LCK_RW_ATTR_STATb       1
 256 #define LCK_RW_ATTR_READ_PRI    0x3
 257 #define LCK_RW_ATTR_READ_PRIb   2
 258 #define LCK_RW_ATTR_DIS_THREAD  0x40000000
 259 #define LCK_RW_ATTR_DIS_THREADb 30
 260 #define LCK_RW_ATTR_DIS_MYLOCK  0x10000000
 261 #define LCK_RW_ATTR_DIS_MYLOCKb 28
 262
 263 #define LCK_RW_TAG_DESTROYED            0x00002007      /* lock marked as Destroyed */
 264
 265 #define RW_LOCK_READER_EVENT(x)   ((event_t) (((unsigned char*) (x)) + (offsetof(lck_rw_t, lck_rw_tag))))
 266 #define RW_LOCK_WRITER_EVENT(x)   ((event_t) (((unsigned char*) (x)) + (offsetof(lck_rw_t, lck_rw_pad8))))
 267 #define READ_EVENT_TO_RWLOCK(x)   ((lck_rw_t *)(((unsigned char*)(x) - (offsetof(lck_rw_t, lck_rw_tag)))))
 268 #define WRITE_EVENT_TO_RWLOCK(x)  ((lck_rw_t *)(((unsigned char*)(x) - (offsetof(lck_rw_t, lck_rw_pad8)))))
 269
 270 #if LOCK_PRIVATE
 271
 272 #define disable_preemption_for_thread(t) ((cpu_data_t GS_RELATIVE *)0UL)->cpu_preemption_level++
 273 #define preemption_disabled_for_thread(t) (((cpu_data_t GS_RELATIVE *)0UL)->cpu_preemption_level > 0)
 274
 275 #define LCK_MTX_THREAD_TO_STATE(t)      ((uintptr_t)t)
 276 #define PLATFORM_LCK_ILOCK              0
 277
 278 #define LOCK_SNOOP_SPINS        1000
 279 #define LOCK_PRETEST            1
 280
 281 /* hw_lock_lock static panic deadline, in timebase units. hw_lock_to() uses
 282  * LockTimeoutTSC computed at startup
 283  */
 284 #define LOCK_PANIC_TIMEOUT      0xf000000  /* 251e6 TSC ticks */
 285
 286 #endif  // LOCK_PRIVATE
 287
 288 #else
 289 #ifdef  KERNEL_PRIVATE
 290 #pragma pack(1)
 291 typedef struct {
 292         uint32_t                opaque[3];
 293         uint32_t                opaque4;
 294 } lck_rw_t;
 295 #pragma pack()
 296 #else
 297 typedef struct __lck_rw_t__     lck_rw_t;
 298 #endif
 299 #endif
 300
 301 #ifdef MACH_KERNEL_PRIVATE
 302
 303 extern void             kernel_preempt_check(void);
 304
 305 #endif /* MACH_KERNEL_PRIVATE */
 306 #endif  /* _I386_LOCKS_H_ */