libsyscall/os/tsd.h

   1 /*
   2  * Copyright (c) 2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #ifndef OS_TSD_H
  30 #define OS_TSD_H
  31
  32 /* The low nine slots of the TSD are reserved for libsyscall usage. */
  33 #define __TSD_RESERVED_BASE 0
  34 #define __TSD_RESERVED_MAX 9
  35
  36 #define __TSD_THREAD_SELF 0
  37 #define __TSD_ERRNO 1
  38 #define __TSD_MIG_REPLY 2
  39 #define __TSD_MACH_THREAD_SELF 3
  40 #define __TSD_THREAD_QOS_CLASS 4
  41 #define __TSD_RETURN_TO_KERNEL 5
  42 /* slot 6 is reserved for Windows/WINE compatibility reasons */
  43 #define __TSD_PTR_MUNGE 7
  44 #define __TSD_MACH_SPECIAL_REPLY 8
  45 #define __TSD_SEMAPHORE_CACHE 9
  46
  47 #ifndef __ASSEMBLER__
  48
  49 #include <stdint.h>
  50
  51 #ifdef __arm__
  52 #include <arm/arch.h>
  53 #endif
  54
  55 extern void _thread_set_tsd_base(void *tsd_base);
  56
  57 __attribute__((always_inline))
  58 static __inline__ unsigned int
  59 _os_cpu_number(void)
  60 {
  61 #if defined(__arm__) && defined(_ARM_ARCH_6)
  62         uintptr_t p;
  63         __asm__("mrc    p15, 0, %[p], c13, c0, 3" : [p] "=&r" (p));
  64         return (unsigned int)(p & 0x3ul);
  65 #elif defined(__arm64__)
  66         uint64_t p;
  67         __asm__("mrs    %[p], TPIDRRO_EL0" : [p] "=&r" (p));
  68         return (unsigned int)p & 0x7;
  69 #elif defined(__x86_64__) || defined(__i386__)
  70         struct { uintptr_t p1, p2; } p;
  71         __asm__("sidt %[p]" : [p] "=&m" (p));
  72         return (unsigned int)(p.p1 & 0xfff);
  73 #else
  74 #error _os_cpu_number not implemented on this architecture
  75 #endif
  76 }
  77
  78 #if defined(__i386__) || defined(__x86_64__)
  79
  80 #if defined(__has_attribute)
  81 #if __has_attribute(address_space)
  82 #define OS_GS_RELATIVE  __attribute__((address_space(256)))
  83 #endif
  84 #endif
  85
  86 #ifdef OS_GS_RELATIVE
  87 #define _os_tsd_get_base() ((void * OS_GS_RELATIVE *)0)
  88 #else
  89 __attribute__((always_inline))
  90 static __inline__ void*
  91 _os_tsd_get_direct(unsigned long slot)
  92 {
  93         void *ret;
  94         __asm__("mov %%gs:%1, %0" : "=r" (ret) : "m" (*(void **)(slot * sizeof(void *))));
  95         return ret;
  96 }
  97
  98 __attribute__((always_inline))
  99 static __inline__ int
 100 _os_tsd_set_direct(unsigned long slot, void *val)
 101 {
 102 #if defined(__i386__) && defined(__PIC__)
 103         __asm__("movl %1, %%gs:%0" : "=m" (*(void **)(slot * sizeof(void *))) : "rn" (val));
 104 #elif defined(__i386__) && !defined(__PIC__)
 105         __asm__("movl %1, %%gs:%0" : "=m" (*(void **)(slot * sizeof(void *))) : "ri" (val));
 106 #else
 107         __asm__("movq %1, %%gs:%0" : "=m" (*(void **)(slot * sizeof(void *))) : "rn" (val));
 108 #endif
 109         return 0;
 110 }
 111 #endif
 112
 113 #elif defined(__arm__) || defined(__arm64__)
 114
 115 __attribute__((always_inline, pure))
 116 static __inline__ void**
 117 _os_tsd_get_base(void)
 118 {
 119 #if defined(__arm__) && defined(_ARM_ARCH_6)
 120         uintptr_t tsd;
 121         __asm__("mrc p15, 0, %0, c13, c0, 3" : "=r" (tsd));
 122         tsd &= ~0x3ul; /* lower 2-bits contain CPU number */
 123 #elif defined(__arm__) && defined(_ARM_ARCH_5)
 124         register uintptr_t tsd asm ("r9");
 125 #elif defined(__arm64__)
 126         uint64_t tsd;
 127         __asm__("mrs %0, TPIDRRO_EL0" : "=r" (tsd));
 128         tsd &= ~0x7ull;
 129 #endif
 130
 131         return (void**)(uintptr_t)tsd;
 132 }
 133 #define _os_tsd_get_base()  _os_tsd_get_base()
 134
 135 #else
 136 #error _os_tsd_get_base not implemented on this architecture
 137 #endif
 138
 139 #ifdef _os_tsd_get_base
 140 __attribute__((always_inline))
 141 static __inline__ void*
 142 _os_tsd_get_direct(unsigned long slot)
 143 {
 144         return _os_tsd_get_base()[slot];
 145 }
 146
 147 __attribute__((always_inline))
 148 static __inline__ int
 149 _os_tsd_set_direct(unsigned long slot, void *val)
 150 {
 151         _os_tsd_get_base()[slot] = val;
 152         return 0;
 153 }
 154 #endif
 155
 156 __attribute__((always_inline, pure))
 157 static __inline__ uintptr_t
 158 _os_ptr_munge_token(void)
 159 {
 160         return (uintptr_t)_os_tsd_get_direct(__TSD_PTR_MUNGE);
 161 }
 162
 163 __attribute__((always_inline, pure))
 164 static __inline__ uintptr_t
 165 _os_ptr_munge(uintptr_t ptr)
 166 {
 167         return ptr ^ _os_ptr_munge_token();
 168 }
 169 #define _OS_PTR_MUNGE(_ptr) _os_ptr_munge((uintptr_t)(_ptr))
 170 #define _OS_PTR_UNMUNGE(_ptr) _os_ptr_munge((uintptr_t)(_ptr))
 171
 172 #else // __ASSEMBLER__
 173
 174 #define _OS_TSD_OFFSET(_key) \
 175         ((__POINTER_WIDTH__/__CHAR_BIT__)*_key)
 176
 177 #if defined(__i386__) || defined(__x86_64__)
 178
 179 #define _OS_PTR_MUNGE(_reg) \
 180         xor %gs:_OS_TSD_OFFSET(__TSD_PTR_MUNGE), _reg
 181
 182 #define _OS_PTR_UNMUNGE(_reg) \
 183         _OS_PTR_MUNGE(_reg)
 184
 185 #elif defined(__arm__) || defined(__arm64__)
 186
 187 #if defined(__arm__)
 188
 189 #define _OS_PTR_MUNGE_TOKEN(_reg, _token) \
 190         mrc p15, 0, _reg, c13, c0, 3; \
 191         bic     _reg, _reg, #3; \
 192         ldr     _token, [ _reg,  #_OS_TSD_OFFSET(__TSD_PTR_MUNGE) ]
 193
 194 #elif defined(__arm64__)
 195
 196 #define _OS_PTR_MUNGE_TOKEN(_reg, _token) \
 197         mrs _reg, TPIDRRO_EL0 %% \
 198         and     _reg, _reg, #~0x7 %% \
 199         ldr     _token, [ _reg,  #_OS_TSD_OFFSET(__TSD_PTR_MUNGE) ]
 200
 201 #endif // defined(__arm64__)
 202
 203 #define _OS_PTR_MUNGE(_regdest, _regsrc, _token) \
 204         eor _regdest, _regsrc, _token
 205
 206 #define _OS_PTR_UNMUNGE(_regdest, _regsrc, _token) \
 207         _OS_PTR_MUNGE(_regdest, _regsrc, _token)
 208
 209 #endif // defined(__arm__) || defined(__arm64__)
 210
 211 #endif // __ASSEMBLER__
 212
 213 #endif // OS_TSD_H