osfmk/arm/locks.h

   1 /*
   2  * Copyright (c) 2007-2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #ifndef _ARM_LOCKS_H_
  30 #define _ARM_LOCKS_H_
  31
  32 #include <kern/kern_types.h>
  33 #ifdef  MACH_KERNEL_PRIVATE
  34 #include <arm/hw_lock_types.h>
  35 #endif
  36
  37
  38 #ifdef  MACH_KERNEL_PRIVATE
  39
  40 extern  unsigned int    LcksOpts;
  41
  42 #define enaLkDeb                0x00000001      /* Request debug in default attribute */
  43 #define enaLkStat               0x00000002      /* Request statistic in default attribute */
  44 #define disLkRWPrio             0x00000004      /* Disable RW lock priority promotion */
  45 #define enaLkTimeStat           0x00000008      /* Request time statistics in default attribute */
  46
  47 #define disLkType               0x80000000      /* Disable type checking */
  48 #define disLktypeb              0
  49 #define disLkThread             0x40000000      /* Disable ownership checking */
  50 #define disLkThreadb    1
  51 #define enaLkExtStck    0x20000000      /* Enable extended backtrace */
  52 #define enaLkExtStckb   2
  53 #define disLkMyLck              0x10000000      /* Disable recursive lock dectection */
  54 #define disLkMyLckb             3
  55
  56 #endif
  57
  58 #ifdef  MACH_KERNEL_PRIVATE
  59 typedef struct {
  60         struct hslock   hwlock;
  61         uintptr_t               type;
  62 } lck_spin_t;
  63
  64 #define lck_spin_data hwlock.lock_data
  65
  66 #define LCK_SPIN_TAG_DESTROYED  0xdead  /* lock marked as Destroyed */
  67
  68 #define LCK_SPIN_TYPE                   0x00000011
  69
  70 #else
  71 #ifdef  KERNEL_PRIVATE
  72
  73 typedef struct {
  74         uintptr_t               opaque[2];
  75 } lck_spin_t;
  76
  77 #else
  78 typedef struct __lck_spin_t__   lck_spin_t;
  79 #endif  // KERNEL_PRIVATE
  80 #endif  // MACH_KERNEL_PRIVATE
  81
  82 #ifdef  MACH_KERNEL_PRIVATE
  83 typedef struct _lck_mtx_ {
  84         union {
  85                 uintptr_t                                       lck_mtx_data;   /* Thread pointer plus lock bits */
  86                 uintptr_t                                       lck_mtx_tag;    /* Tag for type */
  87         };                                                                                              /* arm: 4   arm64: 8 */
  88         union {
  89                 struct {
  90                         uint16_t                                lck_mtx_waiters;/* Number of waiters */
  91                         uint8_t                                 lck_mtx_pri;    /* unused */
  92                         uint8_t                                 lck_mtx_type;   /* Type */
  93                 };
  94                 struct {
  95                         struct _lck_mtx_ext_    *lck_mtx_ptr;   /* Indirect pointer */
  96                 };
  97         };                                                                                              /* arm: 4   arm64: 8 */
  98 } lck_mtx_t;                                                                            /* arm: 8  arm64: 16 */
  99
 100 /* Shared between mutex and read-write locks */
 101 #define LCK_ILOCK_BIT           0
 102 #define ARM_LCK_WAITERS_BIT     1
 103 #define LCK_ILOCK                       (1 << LCK_ILOCK_BIT)
 104 #define ARM_LCK_WAITERS         (1 << ARM_LCK_WAITERS_BIT)
 105
 106 #define LCK_MTX_TYPE                                    0x22            /* lock type */
 107
 108 #define LCK_MTX_TAG_INDIRECT                    0x00001007      /* lock marked as Indirect  */
 109 #define LCK_MTX_TAG_DESTROYED                   0x00002007      /* lock marked as Destroyed */
 110
 111 #define LCK_FRAMES_MAX  8
 112
 113 extern uint64_t         MutexSpin;
 114 extern uint64_t         low_MutexSpin;
 115 extern int64_t          high_MutexSpin;
 116
 117 typedef struct {
 118         unsigned int            type;
 119         vm_offset_t                     stack[LCK_FRAMES_MAX];
 120         vm_offset_t                     thread;
 121 } lck_mtx_deb_t;
 122
 123 #define MUTEX_TAG       0x4d4d
 124
 125 typedef struct {
 126         unsigned int            lck_mtx_stat_data;
 127 } lck_mtx_stat_t;
 128
 129 typedef struct _lck_mtx_ext_ {
 130         lck_mtx_t                       lck_mtx;        /* arm: 12  arm64: 24 */
 131         struct _lck_grp_        *lck_mtx_grp;   /* arm: 4   arm64: 8 */
 132         unsigned int            lck_mtx_attr;   /* arm: 4   arm64: 4 */
 133         lck_mtx_stat_t          lck_mtx_stat;   /* arm: 4   arm64: 4 */
 134         lck_mtx_deb_t           lck_mtx_deb;    /* arm: 40  arm64: 80 */
 135 } lck_mtx_ext_t;                        /* arm: 64  arm64: 120 */
 136
 137 #define LCK_MTX_ATTR_DEBUG      0x1
 138 #define LCK_MTX_ATTR_DEBUGb     31
 139 #define LCK_MTX_ATTR_STAT       0x2
 140 #define LCK_MTX_ATTR_STATb      30
 141
 142 #define LCK_MTX_EVENT(lck)        ((event_t)(((unsigned int*)(lck))+((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))))
 143 #define LCK_EVENT_TO_MUTEX(event) ((lck_mtx_t *)(uintptr_t)(((unsigned int *)(event)) - ((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))))
 144
 145 #else
 146 #ifdef  KERNEL_PRIVATE
 147 typedef struct {
 148         uintptr_t        opaque[2];
 149 } lck_mtx_t;
 150
 151 typedef struct {
 152 #if defined(__arm64__)
 153         unsigned long       opaque[16];
 154 #else /* __arm__ */
 155         unsigned int            opaque[16];
 156 #endif
 157 } lck_mtx_ext_t;
 158
 159 #else
 160 typedef struct __lck_mtx_t__    lck_mtx_t;
 161 #endif
 162 #endif
 163
 164 #ifdef  MACH_KERNEL_PRIVATE
 165
 166 typedef union {
 167         struct {
 168                 uint16_t        shared_count;           /* No. of shared granted request */
 169                 uint16_t        interlock:              1,      /* Interlock */
 170                     priv_excl:              1,                          /* priority for Writer */
 171                     want_upgrade:   1,                          /* Read-to-write upgrade waiting */
 172                     want_excl:              1,                          /* Writer is waiting, or locked for write */
 173                     r_waiting:              1,                          /* Someone is sleeping on lock */
 174                     w_waiting:              1,                          /* Writer is sleeping on lock */
 175                     can_sleep:              1,                          /* Can attempts to lock go to sleep? */
 176                     _pad2:                  8,                          /* padding */
 177                     tag_valid:              1;                          /* Field is actually a tag, not a bitfield */
 178 #if __arm64__
 179                 uint32_t        _pad4;
 180 #endif
 181         };
 182         struct {
 183                 uint32_t        data;                                           /* Single word version of bitfields and shared count */
 184 #if __arm64__
 185                 uint32_t        lck_rw_pad4;
 186 #endif
 187         };
 188 } lck_rw_word_t;
 189
 190 typedef struct {
 191         lck_rw_word_t   word;
 192         thread_t                lck_rw_owner;
 193 } lck_rw_t;     /* arm: 8  arm64: 16 */
 194
 195 #define lck_rw_shared_count     word.shared_count
 196 #define lck_rw_interlock        word.interlock
 197 #define lck_rw_priv_excl        word.priv_excl
 198 #define lck_rw_want_upgrade     word.want_upgrade
 199 #define lck_rw_want_excl        word.want_excl
 200 #define lck_r_waiting           word.r_waiting
 201 #define lck_w_waiting           word.w_waiting
 202 #define lck_rw_can_sleep        word.can_sleep
 203 #define lck_rw_data                     word.data
 204 // tag and data reference the same memory. When the tag_valid bit is set,
 205 // the data word should be treated as a tag instead of a bitfield.
 206 #define lck_rw_tag_valid        word.tag_valid
 207 #define lck_rw_tag                      word.data
 208
 209 #define LCK_RW_SHARED_READER_OFFSET      0
 210 #define LCK_RW_INTERLOCK_BIT            16
 211 #define LCK_RW_PRIV_EXCL_BIT            17
 212 #define LCK_RW_WANT_UPGRADE_BIT         18
 213 #define LCK_RW_WANT_EXCL_BIT            19
 214 #define LCK_RW_R_WAITING_BIT            20
 215 #define LCK_RW_W_WAITING_BIT            21
 216 #define LCK_RW_CAN_SLEEP_BIT            22
 217 //                                                                      23-30
 218 #define LCK_RW_TAG_VALID_BIT            31
 219
 220 #define LCK_RW_INTERLOCK                (1U << LCK_RW_INTERLOCK_BIT)
 221 #define LCK_RW_R_WAITING                (1U << LCK_RW_R_WAITING_BIT)
 222 #define LCK_RW_W_WAITING                (1U << LCK_RW_W_WAITING_BIT)
 223 #define LCK_RW_WANT_UPGRADE             (1U << LCK_RW_WANT_UPGRADE_BIT)
 224 #define LCK_RW_WANT_EXCL                (1U << LCK_RW_WANT_EXCL_BIT)
 225 #define LCK_RW_TAG_VALID                (1U << LCK_RW_TAG_VALID_BIT)
 226 #define LCK_RW_PRIV_EXCL                (1U << LCK_RW_PRIV_EXCL_BIT)
 227 #define LCK_RW_SHARED_MASK              (0xffff << LCK_RW_SHARED_READER_OFFSET)
 228 #define LCK_RW_SHARED_READER    (0x1 << LCK_RW_SHARED_READER_OFFSET)
 229
 230 #define LCK_RW_TAG_DESTROYED            ((LCK_RW_TAG_VALID | 0xdddddeadu))      /* lock marked as Destroyed */
 231
 232 #define LCK_RW_WRITER_EVENT(lck)                (event_t)((uintptr_t)(lck)+1)
 233 #define LCK_RW_READER_EVENT(lck)                (event_t)((uintptr_t)(lck)+2)
 234 #define WRITE_EVENT_TO_RWLOCK(event)    ((lck_rw_t *)((uintptr_t)(event)-1))
 235 #define READ_EVENT_TO_RWLOCK(event)             ((lck_rw_t *)((uintptr_t)(event)-2))
 236
 237 #if __ARM_ENABLE_WFE_
 238
 239 #define wait_for_event()        __builtin_arm_wfe()
 240 #if __arm__
 241 #define set_event()                     do{__builtin_arm_dsb(DSB_ISHST);__builtin_arm_sev();}while(0)
 242 #define LOCK_SNOOP_SPINS        4
 243 #else
 244 #define set_event()                     do{}while(0)    // arm64 sev is implicit in stlxr
 245 #define LOCK_SNOOP_SPINS        0x300
 246 #endif
 247
 248 #else
 249
 250 #define wait_for_event()        __builtin_arm_clrex()
 251 #define set_event()             do{}while(0)
 252 #define LOCK_SNOOP_SPINS        0x300
 253
 254 #endif // __ARM_ENABLE_WFE_
 255
 256 #if LOCK_PRIVATE
 257
 258 #define LOCK_PANIC_TIMEOUT      0xc00000        // 12.5 m ticks ~= 524ms with 24MHz OSC
 259
 260 #define PLATFORM_LCK_ILOCK LCK_ILOCK
 261
 262 #if defined(__ARM_ARCH_8_2__)
 263 #define __ARM_ATOMICS_8_1       1       // ARMv8.1 atomic instructions are available
 264 #endif
 265
 266 /*
 267  * Lock state to thread pointer
 268  * Clear the bottom bits
 269  */
 270 #define LCK_MTX_STATE_TO_THREAD(s)      (thread_t)(s & ~(LCK_ILOCK | ARM_LCK_WAITERS))
 271 /*
 272  * Thread pointer to lock state
 273  * arm thread pointers are aligned such that the bottom two bits are clear
 274  */
 275 #define LCK_MTX_THREAD_TO_STATE(t)      ((uintptr_t)t)
 276 /*
 277  * Thread pointer mask
 278  */
 279 #define LCK_MTX_THREAD_MASK (~(uintptr_t)(LCK_ILOCK | ARM_LCK_WAITERS))
 280
 281 #define disable_preemption_for_thread(t) os_atomic_store(&(t->machine.preemption_count), t->machine.preemption_count + 1, compiler_acq_rel)
 282 #define preemption_disabled_for_thread(t) (t->machine.preemption_count > 0)
 283
 284
 285 __unused static void
 286 disable_interrupts_noread(void)
 287 {
 288 #if __arm__
 289         __asm__ volatile ("cpsid if" ::: "memory"); // Mask IRQ FIQ
 290 #else
 291         __builtin_arm_wsr64("DAIFSet", (DAIFSC_IRQF | DAIFSC_FIQF));    // Mask IRQ FIQ
 292 #endif
 293 }
 294
 295 __unused static inline long
 296 get_interrupts(void)
 297 {
 298         long    state;
 299
 300 #if __arm__
 301         __asm__ volatile ("mrs %[state], cpsr" :[state] "=r" (state));  // Read cpsr
 302 #else
 303         state = __builtin_arm_rsr64("DAIF");    // Read interrupt state
 304 #endif
 305         return state;
 306 }
 307
 308 __unused static inline long
 309 disable_interrupts(void)
 310 {
 311         long    state;
 312
 313         state = get_interrupts();               // Get previous state
 314         disable_interrupts_noread();    // Disable
 315         return state;
 316 }
 317
 318 __unused static inline void
 319 restore_interrupts(long state)
 320 {
 321 #if __arm__
 322         __asm__ volatile ("msr  cpsr, %[state]" :: [state] "r" (state) : "cc", "memory"); // Restore CPSR
 323 #elif __arm64__
 324         __builtin_arm_wsr64("DAIF", state);     // Restore masks
 325 #endif
 326 }
 327
 328 #endif // LOCK_PRIVATE
 329
 330 #else
 331 #ifdef  KERNEL_PRIVATE
 332 typedef struct {
 333         uintptr_t                    opaque[2];
 334 } lck_rw_t;
 335 #else
 336 typedef struct __lck_rw_t__     lck_rw_t;
 337 #endif
 338 #endif
 339
 340 #endif  /* _ARM_LOCKS_H_ */