osfmk/arm/locks.h

   1 /*
   2  * Copyright (c) 2007-2017 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #ifndef _ARM_LOCKS_H_
  30 #define _ARM_LOCKS_H_
  31
  32 #include <kern/kern_types.h>
  33 #ifdef  MACH_KERNEL_PRIVATE
  34 #include <arm/hw_lock_types.h>
  35 #endif
  36
  37
  38 #ifdef  MACH_KERNEL_PRIVATE
  39
  40 extern  unsigned int    LcksOpts;
  41
  42 #define enaLkDeb                0x00000001      /* Request debug in default attribute */
  43 #define enaLkStat               0x00000002      /* Request statistic in default attribute */
  44 #define disLkRWPrio             0x00000004      /* Disable RW lock priority promotion */
  45
  46 #define disLkType               0x80000000      /* Disable type checking */
  47 #define disLktypeb              0
  48 #define disLkThread             0x40000000      /* Disable ownership checking */
  49 #define disLkThreadb    1
  50 #define enaLkExtStck    0x20000000      /* Enable extended backtrace */
  51 #define enaLkExtStckb   2
  52 #define disLkMyLck              0x10000000      /* Disable recursive lock dectection */
  53 #define disLkMyLckb             3
  54
  55 #endif
  56
  57 #ifdef  MACH_KERNEL_PRIVATE
  58 typedef struct {
  59         struct hslock   hwlock;
  60         uintptr_t               type;
  61 } lck_spin_t;
  62
  63 #define lck_spin_data hwlock.lock_data
  64
  65 #define LCK_SPIN_TAG_DESTROYED  0xdead  /* lock marked as Destroyed */
  66
  67 #define LCK_SPIN_TYPE                   0x00000011
  68
  69 #else
  70 #ifdef  KERNEL_PRIVATE
  71
  72 typedef struct {
  73         uintptr_t               opaque[2];
  74 } lck_spin_t;
  75
  76 #else
  77 typedef struct __lck_spin_t__   lck_spin_t;
  78 #endif  // KERNEL_PRIVATE
  79 #endif  // MACH_KERNEL_PRIVATE
  80
  81 #ifdef  MACH_KERNEL_PRIVATE
  82 typedef struct _lck_mtx_ {
  83         union {
  84
  85                 uintptr_t                                       lck_mtx_data;   /* Thread pointer plus lock bits */
  86                 uintptr_t                                       lck_mtx_tag;    /* Tag for type */
  87         };                                                                                              /* arm: 4   arm64: 8 */
  88         union {
  89                 struct {
  90                         uint16_t                                lck_mtx_waiters;/* Number of waiters */
  91                         uint8_t                                 lck_mtx_pri;    /* Priority to inherit */
  92                         uint8_t                                 lck_mtx_type;   /* Type */
  93                 };
  94                 struct {
  95                         struct _lck_mtx_ext_    *lck_mtx_ptr;   /* Indirect pointer */
  96                 };
  97         };                                                                                              /* arm: 4   arm64: 8 */
  98 } lck_mtx_t;                                                                            /* arm: 8  arm64: 16 */
  99
 100 /* Shared between mutex and read-write locks */
 101 #define LCK_ILOCK_BIT           0
 102 #define ARM_LCK_WAITERS_BIT     1
 103 #define LCK_ILOCK                       (1 << LCK_ILOCK_BIT)
 104 #define ARM_LCK_WAITERS         (1 << ARM_LCK_WAITERS_BIT)
 105
 106 #define LCK_MTX_TYPE                                    0x22            /* lock type */
 107
 108 #define LCK_MTX_TAG_INDIRECT                    0x00001007      /* lock marked as Indirect  */
 109 #define LCK_MTX_TAG_DESTROYED                   0x00002007      /* lock marked as Destroyed */
 110
 111 #define LCK_FRAMES_MAX  8
 112
 113 extern uint64_t         MutexSpin;
 114
 115 typedef struct {
 116         unsigned int            type;
 117         vm_offset_t                     stack[LCK_FRAMES_MAX];
 118         vm_offset_t                     thread;
 119 } lck_mtx_deb_t;
 120
 121 #define MUTEX_TAG       0x4d4d
 122
 123 typedef struct {
 124         unsigned int            lck_mtx_stat_data;
 125 } lck_mtx_stat_t;
 126
 127 typedef struct _lck_mtx_ext_ {
 128         lck_mtx_t                       lck_mtx;        /* arm: 12  arm64: 24 */
 129         struct _lck_grp_        *lck_mtx_grp;   /* arm: 4   arm64: 8 */
 130         unsigned int            lck_mtx_attr;   /* arm: 4   arm64: 4 */
 131         lck_mtx_stat_t          lck_mtx_stat;   /* arm: 4   arm64: 4 */
 132         lck_mtx_deb_t           lck_mtx_deb;    /* arm: 40  arm64: 80 */
 133 } lck_mtx_ext_t;                        /* arm: 64  arm64: 120 */
 134
 135 #define LCK_MTX_ATTR_DEBUG      0x1
 136 #define LCK_MTX_ATTR_DEBUGb     31
 137 #define LCK_MTX_ATTR_STAT       0x2
 138 #define LCK_MTX_ATTR_STATb      30
 139
 140 #define LCK_MTX_EVENT(lck)        ((event_t)(((unsigned int*)(lck))+((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))))
 141 #define LCK_EVENT_TO_MUTEX(event) ((lck_mtx_t *)(uintptr_t)(((unsigned int *)(event)) - ((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))))
 142
 143 #else
 144 #ifdef  KERNEL_PRIVATE
 145 typedef struct {
 146     uintptr_t        opaque[2];
 147 } lck_mtx_t;
 148
 149 typedef struct {
 150 #if defined(__arm64__)
 151     unsigned long       opaque[16];
 152 #else /* __arm__ */
 153         unsigned int            opaque[16];
 154 #endif
 155 } lck_mtx_ext_t;
 156
 157 #else
 158 typedef struct __lck_mtx_t__    lck_mtx_t;
 159 #endif
 160 #endif
 161
 162 #ifdef  MACH_KERNEL_PRIVATE
 163
 164 typedef union {
 165         struct {
 166                 uint16_t        shared_count;           /* No. of shared granted request */
 167                 uint16_t        interlock:              1,      /* Interlock */
 168                                         priv_excl:              1,      /* priority for Writer */
 169                                         want_upgrade:   1,      /* Read-to-write upgrade waiting */
 170                                         want_excl:              1,      /* Writer is waiting, or locked for write */
 171                                         r_waiting:              1,      /* Someone is sleeping on lock */
 172                                         w_waiting:              1,      /* Writer is sleeping on lock */
 173                                         can_sleep:              1,      /* Can attempts to lock go to sleep? */
 174                                         _pad2:                  8,      /* padding */
 175                                         tag_valid:              1;      /* Field is actually a tag, not a bitfield */
 176 #if __arm64__
 177                 uint32_t        _pad4;
 178 #endif
 179         };
 180         struct {
 181                 uint32_t        data;                                           /* Single word version of bitfields and shared count */
 182 #if __arm64__
 183                 uint32_t        lck_rw_pad4;
 184 #endif
 185         };
 186 } lck_rw_word_t;
 187
 188 typedef struct {
 189         lck_rw_word_t   word;
 190         thread_t                lck_rw_owner;
 191 } lck_rw_t;     /* arm: 8  arm64: 16 */
 192
 193 #define lck_rw_shared_count     word.shared_count
 194 #define lck_rw_interlock        word.interlock
 195 #define lck_rw_priv_excl        word.priv_excl
 196 #define lck_rw_want_upgrade     word.want_upgrade
 197 #define lck_rw_want_excl        word.want_excl
 198 #define lck_r_waiting           word.r_waiting
 199 #define lck_w_waiting           word.w_waiting
 200 #define lck_rw_can_sleep        word.can_sleep
 201 #define lck_rw_data                     word.data
 202 // tag and data reference the same memory. When the tag_valid bit is set,
 203 // the data word should be treated as a tag instead of a bitfield.
 204 #define lck_rw_tag_valid        word.tag_valid
 205 #define lck_rw_tag                      word.data
 206
 207 #define LCK_RW_SHARED_READER_OFFSET      0
 208 #define LCK_RW_INTERLOCK_BIT            16
 209 #define LCK_RW_PRIV_EXCL_BIT            17
 210 #define LCK_RW_WANT_UPGRADE_BIT         18
 211 #define LCK_RW_WANT_EXCL_BIT            19
 212 #define LCK_RW_R_WAITING_BIT            20
 213 #define LCK_RW_W_WAITING_BIT            21
 214 #define LCK_RW_CAN_SLEEP_BIT            22
 215 //                                                                      23-30
 216 #define LCK_RW_TAG_VALID_BIT            31
 217
 218 #define LCK_RW_INTERLOCK                (1 << LCK_RW_INTERLOCK_BIT)
 219 #define LCK_RW_R_WAITING                (1 << LCK_RW_R_WAITING_BIT)
 220 #define LCK_RW_W_WAITING                (1 << LCK_RW_W_WAITING_BIT)
 221 #define LCK_RW_WANT_UPGRADE             (1 << LCK_RW_WANT_UPGRADE_BIT)
 222 #define LCK_RW_WANT_EXCL                (1 << LCK_RW_WANT_EXCL_BIT)
 223 #define LCK_RW_TAG_VALID                (1 << LCK_RW_TAG_VALID_BIT)
 224 #define LCK_RW_PRIV_EXCL                (1 << LCK_RW_PRIV_EXCL_BIT)
 225 #define LCK_RW_SHARED_MASK              (0xffff << LCK_RW_SHARED_READER_OFFSET)
 226 #define LCK_RW_SHARED_READER    (0x1 << LCK_RW_SHARED_READER_OFFSET)
 227
 228 #define LCK_RW_TAG_DESTROYED            ((LCK_RW_TAG_VALID | 0xdddddeadu))      /* lock marked as Destroyed */
 229
 230 #define LCK_RW_WRITER_EVENT(lck)                (event_t)((uintptr_t)(lck)+1)
 231 #define LCK_RW_READER_EVENT(lck)                (event_t)((uintptr_t)(lck)+2)
 232 #define WRITE_EVENT_TO_RWLOCK(event)    ((lck_rw_t *)((uintptr_t)(event)-1))
 233 #define READ_EVENT_TO_RWLOCK(event)             ((lck_rw_t *)((uintptr_t)(event)-2))
 234
 235 #if __ARM_ENABLE_WFE_
 236
 237 #define wait_for_event()        __builtin_arm_wfe()
 238 #if __arm__
 239 #define set_event()                     do{__builtin_arm_dsb(DSB_ISHST);__builtin_arm_sev();}while(0)
 240 #define LOCK_SNOOP_SPINS        4
 241 #else
 242 #define set_event()                     do{}while(0)    // arm64 sev is implicit in stlxr
 243 #define LOCK_SNOOP_SPINS        0x300
 244 #endif
 245
 246 #else
 247
 248 #define wait_for_event()        __builtin_arm_clrex()
 249 #define set_event()             do{}while(0)
 250 #define LOCK_SNOOP_SPINS        0x300
 251
 252 #endif // __ARM_ENABLE_WFE_
 253
 254 #if LOCK_PRIVATE
 255
 256 #define LOCK_PANIC_TIMEOUT      0xc00000        // 12.5 m ticks = 250ms with 24MHz OSC
 257
 258 #define PLATFORM_LCK_ILOCK LCK_ILOCK
 259
 260
 261 /*
 262  * Lock state to thread pointer
 263  * Clear the bottom bits
 264  */
 265 #define LCK_MTX_STATE_TO_THREAD(s)      (thread_t)(s & ~(LCK_ILOCK | ARM_LCK_WAITERS))
 266 /*
 267  * Thread pointer to lock state
 268  * arm thread pointers are aligned such that the bottom two bits are clear
 269  */
 270 #define LCK_MTX_THREAD_TO_STATE(t)      ((uintptr_t)t)
 271 /*
 272  * Thread pointer mask
 273  */
 274 #define LCK_MTX_THREAD_MASK (~(uintptr_t)(LCK_ILOCK | ARM_LCK_WAITERS))
 275
 276 #define disable_preemption_for_thread(t) ((volatile thread_t)t)->machine.preemption_count++
 277 #define preemption_disabled_for_thread(t) (((volatile thread_t)t)->machine.preemption_count > 0)
 278
 279
 280 __unused static void disable_interrupts_noread(void)
 281 {
 282 #if __arm__
 283         __asm__ volatile ("cpsid if" ::: "memory"); // Mask IRQ FIQ
 284 #else
 285         __builtin_arm_wsr64("DAIFSet", (DAIFSC_IRQF | DAIFSC_FIQF));    // Mask IRQ FIQ
 286 #endif
 287 }
 288
 289 __unused static inline long get_interrupts(void)
 290 {
 291         long    state;
 292
 293 #if __arm__
 294         __asm__ volatile ("mrs %[state], cpsr" :[state] "=r" (state));  // Read cpsr
 295 #else
 296         state = __builtin_arm_rsr64("DAIF");    // Read interrupt state
 297 #endif
 298         return state;
 299 }
 300
 301 __unused static inline long disable_interrupts(void)
 302 {
 303         long    state;
 304
 305         state = get_interrupts();               // Get previous state
 306         disable_interrupts_noread();    // Disable
 307         return state;
 308 }
 309
 310 __unused static inline void restore_interrupts(long state)
 311 {
 312 #if __arm__
 313         __asm__ volatile ("msr  cpsr, %[state]" :: [state] "r" (state) : "cc", "memory"); // Restore CPSR
 314 #elif __arm64__
 315         __builtin_arm_wsr64("DAIF", state);     // Restore masks
 316 #endif
 317 }
 318
 319 #endif // LOCK_PRIVATE
 320
 321 #else
 322 #ifdef  KERNEL_PRIVATE
 323 typedef struct {
 324     uintptr_t                    opaque[2];
 325 } lck_rw_t;
 326 #else
 327 typedef struct __lck_rw_t__     lck_rw_t;
 328 #endif
 329 #endif
 330
 331 #endif  /* _ARM_LOCKS_H_ */