osfmk/i386/commpage/commpage.c

   1 /*
   2  * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 /*
  30  *      Here's what to do if you want to add a new routine to the comm page:
  31  *
  32  *              1. Add a definition for it's address in osfmk/ppc/cpu_capabilities.h,
  33  *                 being careful to reserve room for future expansion.
  34  *
  35  *              2. Write one or more versions of the routine, each with it's own
  36  *                 commpage_descriptor.  The tricky part is getting the "special",
  37  *                 "musthave", and "canthave" fields right, so that exactly one
  38  *                 version of the routine is selected for every machine.
  39  *                 The source files should be in osfmk/ppc/commpage/.
  40  *
  41  *              3. Add a ptr to your new commpage_descriptor(s) in the "routines"
  42  *                 array in commpage_populate().  Of course, you'll also have to
  43  *                 declare them "extern" in commpage_populate().
  44  *
  45  *              4. Write the code in Libc to use the new routine.
  46  */
  47
  48 #include <mach/mach_types.h>
  49 #include <mach/machine.h>
  50 #include <mach/vm_map.h>
  51 #include <i386/machine_routines.h>
  52 #include <machine/cpu_capabilities.h>
  53 #include <machine/commpage.h>
  54 #include <machine/pmap.h>
  55 #include <vm/vm_kern.h>
  56 #include <vm/vm_map.h>
  57 #include <ipc/ipc_port.h>
  58
  59
  60 extern vm_map_t com_region_map32;       // the shared submap, set up in vm init
  61
  62 static uintptr_t next = 0;              // next available byte in comm page
  63 static int      cur_routine = 0;        // comm page address of "current" routine
  64 static int      matched;                // true if we've found a match for "current" routine
  65
  66 int     _cpu_capabilities = 0;          // define the capability vector
  67
  68 char    *commPagePtr = NULL;            // virtual address of comm page in kernel map
  69
  70 /* Allocate the commpage and add to the shared submap created by vm:
  71  *      1. allocate a page in the kernel map (RW)
  72  *      2. wire it down
  73  *      3. make a memory entry out of it
  74  *      4. map that entry into the shared comm region map (R-only)
  75  */
  76
  77 static  void*
  78 commpage_allocate( void )
  79 {
  80     vm_offset_t         kernel_addr;                // address of commpage in kernel map
  81     vm_offset_t         zero = 0;
  82     vm_size_t           size = _COMM_PAGE_AREA_LENGTH;
  83     vm_map_entry_t      entry;
  84     ipc_port_t          handle;
  85
  86     if (com_region_map32 == NULL)
  87         panic("commpage map is null");
  88
  89     if (vm_allocate(kernel_map,&kernel_addr,_COMM_PAGE_AREA_LENGTH,VM_FLAGS_ANYWHERE))
  90         panic("cannot allocate commpage");
  91
  92     if (vm_map_wire(kernel_map,kernel_addr,kernel_addr+_COMM_PAGE_AREA_LENGTH,VM_PROT_DEFAULT,FALSE))
  93         panic("cannot wire commpage");
  94
  95     /*
  96      * Now that the object is created and wired into the kernel map, mark it so that no delay
  97      * copy-on-write will ever be performed on it as a result of mapping it into user-space.
  98      * If such a delayed copy ever occurred, we could remove the kernel's wired mapping - and
  99      * that would be a real disaster.
 100      *
 101      * JMM - What we really need is a way to create it like this in the first place.
 102      */
 103     if (!vm_map_lookup_entry( kernel_map, vm_map_trunc_page(kernel_addr), &entry) || entry->is_sub_map)
 104         panic("cannot find commpage entry");
 105     entry->object.vm_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
 106
 107     if (mach_make_memory_entry( kernel_map,         // target map
 108                                 &size,              // size
 109                                 kernel_addr,        // offset (address in kernel map)
 110                                 VM_PROT_DEFAULT,    // map it RW
 111                                 &handle,            // this is the object handle we get
 112                                 NULL ))             // parent_entry (what is this?)
 113         panic("cannot make entry for commpage");
 114
 115     if (vm_map_64(  com_region_map32,               // target map (shared submap)
 116                     &zero,                          // address (map into 1st page in submap)
 117                     _COMM_PAGE_AREA_LENGTH,         // size
 118                     0,                              // mask
 119                     VM_FLAGS_FIXED,                 // flags (it must be 1st page in submap)
 120                     handle,                         // port is the memory entry we just made
 121                     0,                              // offset (map 1st page in memory entry)
 122                     FALSE,                          // copy
 123                     VM_PROT_READ,                   // cur_protection (R-only in user map)
 124                     VM_PROT_READ,                   // max_protection
 125                     VM_INHERIT_SHARE ))             // inheritance
 126         panic("cannot map commpage");
 127
 128     ipc_port_release(handle);
 129
 130     return (void*) kernel_addr;                     // return address in kernel map
 131 }
 132
 133 /* Get address (in kernel map) of a commpage field. */
 134
 135 static void*
 136 commpage_addr_of(
 137     int     addr_at_runtime )
 138 {
 139     return  (void*) ((uintptr_t)commPagePtr + addr_at_runtime - _COMM_PAGE_BASE_ADDRESS);
 140 }
 141
 142 /* Determine number of CPUs on this system.  We cannot rely on
 143  * machine_info.max_cpus this early in the boot.
 144  */
 145 static int
 146 commpage_cpus( void )
 147 {
 148         int cpus;
 149
 150         cpus = ml_get_max_cpus();                   // NB: this call can block
 151
 152         if (cpus == 0)
 153                 panic("commpage cpus==0");
 154         if (cpus > 0xFF)
 155                 cpus = 0xFF;
 156
 157         return cpus;
 158 }
 159
 160 /* Initialize kernel version of _cpu_capabilities vector (used by KEXTs.) */
 161
 162 static void
 163 commpage_init_cpu_capabilities( void )
 164 {
 165         int bits;
 166         int cpus;
 167         ml_cpu_info_t cpu_info;
 168
 169         bits = 0;
 170         ml_cpu_get_info(&cpu_info);
 171
 172         switch (cpu_info.vector_unit) {
 173                 case 5:
 174                         bits |= kHasSSE3;
 175                         /* fall thru */
 176                 case 4:
 177                         bits |= kHasSSE2;
 178                         /* fall thru */
 179                 case 3:
 180                         bits |= kHasSSE;
 181                         /* fall thru */
 182                 case 2:
 183                         bits |= kHasMMX;
 184                 default:
 185                         break;
 186         }
 187         switch (cpu_info.cache_line_size) {
 188                 case 128:
 189                         bits |= kCache128;
 190                         break;
 191                 case 64:
 192                         bits |= kCache64;
 193                         break;
 194                 case 32:
 195                         bits |= kCache32;
 196                         break;
 197                 default:
 198                         break;
 199         }
 200         cpus = commpage_cpus();                 // how many CPUs do we have
 201
 202         if (cpus == 1)
 203                 bits |= kUP;
 204
 205         bits |= (cpus << kNumCPUsShift);
 206
 207         bits |= kFastThreadLocalStorage;        // we use %gs for TLS
 208
 209         _cpu_capabilities = bits;               // set kernel version for use by drivers etc
 210 }
 211
 212 /* Copy data into commpage. */
 213
 214 static void
 215 commpage_stuff(
 216     int         address,
 217     void        *source,
 218     int         length  )
 219 {
 220     void        *dest = commpage_addr_of(address);
 221
 222     if ((uintptr_t)dest < next)
 223         panic("commpage overlap at address 0x%x, 0x%x < 0x%x", address, dest, next);
 224
 225     bcopy(source,dest,length);
 226
 227     next = ((uintptr_t)dest + length);
 228 }
 229
 230
 231 static void
 232 commpage_stuff2(
 233         int address,
 234         void *source,
 235         int length )
 236 {
 237         commpage_stuff(address, source, length);
 238 }
 239
 240 /* Copy a routine into comm page if it matches running machine.
 241  */
 242 static void
 243 commpage_stuff_routine(
 244     commpage_descriptor *rd     )
 245 {
 246     int         must,cant;
 247
 248     if (rd->commpage_address != cur_routine) {
 249         if ((cur_routine!=0) && (matched==0))
 250             panic("commpage no match");
 251         cur_routine = rd->commpage_address;
 252         matched = 0;
 253     }
 254
 255     must = _cpu_capabilities & rd->musthave;
 256     cant = _cpu_capabilities & rd->canthave;
 257
 258     if ((must == rd->musthave) && (cant == 0)) {
 259         if (matched)
 260             panic("commpage duplicate matches");
 261         matched = 1;
 262
 263         commpage_stuff(rd->commpage_address,rd->code_address,rd->code_length);
 264         }
 265 }
 266
 267
 268 #define COMMPAGE_DESC(name)     commpage_ ## name
 269 #define EXTERN_COMMPAGE_DESC(name)                              \
 270         extern commpage_descriptor COMMPAGE_DESC(name)
 271
 272 EXTERN_COMMPAGE_DESC(compare_and_swap32_mp);
 273 EXTERN_COMMPAGE_DESC(compare_and_swap32_up);
 274 EXTERN_COMMPAGE_DESC(compare_and_swap64_mp);
 275 EXTERN_COMMPAGE_DESC(compare_and_swap64_up);
 276 EXTERN_COMMPAGE_DESC(atomic_add32_mp);
 277 EXTERN_COMMPAGE_DESC(atomic_add32_up);
 278 EXTERN_COMMPAGE_DESC(mach_absolute_time);
 279 EXTERN_COMMPAGE_DESC(spin_lock_try_mp);
 280 EXTERN_COMMPAGE_DESC(spin_lock_try_up);
 281 EXTERN_COMMPAGE_DESC(spin_lock_mp);
 282 EXTERN_COMMPAGE_DESC(spin_lock_up);
 283 EXTERN_COMMPAGE_DESC(spin_unlock);
 284 EXTERN_COMMPAGE_DESC(pthread_getspecific);
 285 EXTERN_COMMPAGE_DESC(gettimeofday);
 286 EXTERN_COMMPAGE_DESC(sys_flush_dcache);
 287 EXTERN_COMMPAGE_DESC(sys_icache_invalidate);
 288 EXTERN_COMMPAGE_DESC(pthread_self);
 289 EXTERN_COMMPAGE_DESC(relinquish);
 290 EXTERN_COMMPAGE_DESC(bit_test_and_set_mp);
 291 EXTERN_COMMPAGE_DESC(bit_test_and_set_up);
 292 EXTERN_COMMPAGE_DESC(bit_test_and_clear_mp);
 293 EXTERN_COMMPAGE_DESC(bit_test_and_clear_up);
 294 EXTERN_COMMPAGE_DESC(bzero_scalar);
 295 EXTERN_COMMPAGE_DESC(bcopy_scalar);
 296 EXTERN_COMMPAGE_DESC(nanotime);
 297
 298 static  commpage_descriptor *routines[] = {
 299         &COMMPAGE_DESC(compare_and_swap32_mp),
 300         &COMMPAGE_DESC(compare_and_swap32_up),
 301         &COMMPAGE_DESC(compare_and_swap64_mp),
 302         &COMMPAGE_DESC(compare_and_swap64_up),
 303         &COMMPAGE_DESC(atomic_add32_mp),
 304         &COMMPAGE_DESC(atomic_add32_up),
 305         &COMMPAGE_DESC(mach_absolute_time),
 306         &COMMPAGE_DESC(spin_lock_try_mp),
 307         &COMMPAGE_DESC(spin_lock_try_up),
 308         &COMMPAGE_DESC(spin_lock_mp),
 309         &COMMPAGE_DESC(spin_lock_up),
 310         &COMMPAGE_DESC(spin_unlock),
 311         &COMMPAGE_DESC(pthread_getspecific),
 312         &COMMPAGE_DESC(gettimeofday),
 313         &COMMPAGE_DESC(sys_flush_dcache),
 314         &COMMPAGE_DESC(sys_icache_invalidate),
 315         &COMMPAGE_DESC(pthread_self),
 316         &COMMPAGE_DESC(relinquish),
 317         &COMMPAGE_DESC(bit_test_and_set_mp),
 318         &COMMPAGE_DESC(bit_test_and_set_up),
 319         &COMMPAGE_DESC(bit_test_and_clear_mp),
 320         &COMMPAGE_DESC(bit_test_and_clear_up),
 321         &COMMPAGE_DESC(bzero_scalar),
 322         &COMMPAGE_DESC(bcopy_scalar),
 323         &COMMPAGE_DESC(nanotime),
 324         NULL
 325 };
 326
 327
 328 /* Fill in commpage: called once, during kernel initialization, from the
 329  * startup thread before user-mode code is running.
 330  * See the top of this file for a list of what you have to do to add
 331  * a new routine to the commpage.
 332  */
 333
 334 void
 335 commpage_populate( void )
 336 {
 337         short   c2;
 338         static double   two52 = 1048576.0 * 1048576.0 * 4096.0; // 2**52
 339         static double   ten6 = 1000000.0;                       // 10**6
 340         commpage_descriptor **rd;
 341         short   version = _COMM_PAGE_THIS_VERSION;
 342
 343         commPagePtr = (char *)commpage_allocate();
 344
 345         commpage_init_cpu_capabilities();
 346
 347         /* Stuff in the constants.  We move things into the comm page in strictly
 348         * ascending order, so we can check for overlap and panic if so.
 349         */
 350
 351         commpage_stuff2(_COMM_PAGE_VERSION,&version,sizeof(short));
 352         commpage_stuff(_COMM_PAGE_CPU_CAPABILITIES,&_cpu_capabilities,
 353                 sizeof(int));
 354
 355         if (_cpu_capabilities & kCache32)
 356                 c2 = 32;
 357         else if (_cpu_capabilities & kCache64)
 358                 c2 = 64;
 359         else if (_cpu_capabilities & kCache128)
 360                 c2 = 128;
 361         commpage_stuff(_COMM_PAGE_CACHE_LINESIZE,&c2,2);
 362
 363         c2 = 32;
 364
 365         commpage_stuff2(_COMM_PAGE_2_TO_52,&two52,8);
 366
 367         commpage_stuff2(_COMM_PAGE_10_TO_6,&ten6,8);
 368
 369         for( rd = routines; *rd != NULL ; rd++ )
 370                 commpage_stuff_routine(*rd);
 371
 372         if (!matched)
 373                 panic("commpage no match on last routine");
 374
 375         if (next > (uintptr_t)_COMM_PAGE_END)
 376                 panic("commpage overflow: next = 0x%08x, commPagePtr = 0x%08x", next, (uintptr_t)commPagePtr);
 377
 378
 379         pmap_commpage_init((vm_offset_t) commPagePtr, _COMM_PAGE_BASE_ADDRESS,
 380                            _COMM_PAGE_AREA_LENGTH/INTEL_PGBYTES);
 381 }
 382
 383 /*
 384  * This macro prevents compiler instruction scheduling:
 385  */
 386 #define NO_REORDERING   asm volatile("" : : : "memory")
 387
 388 void
 389 commpage_set_nanotime(commpage_nanotime_t *newp)
 390 {
 391         commpage_nanotime_t     *cnp;
 392
 393         /* Nop if commpage not set up yet */
 394         if (commPagePtr == NULL)
 395                 return;
 396
 397         cnp = (commpage_nanotime_t *)commpage_addr_of(_COMM_PAGE_NANOTIME_INFO);
 398
 399         /*
 400          * Update in reverse order:
 401          * check_tsc first - it's read and compared with base_tsc last.
 402          */
 403         cnp->nt_check_tsc = newp->nt_base_tsc;  NO_REORDERING;
 404         cnp->nt_shift     = newp->nt_shift;     NO_REORDERING;
 405         cnp->nt_scale     = newp->nt_scale;     NO_REORDERING;
 406         cnp->nt_base_ns   = newp->nt_base_ns;   NO_REORDERING;
 407         cnp->nt_base_tsc  = newp->nt_base_tsc;
 408 }
 409