osfmk/i386/commpage/commpage.c

   1 /*
   2  * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License.  The rights granted to you under the
  10  * License may not be used to create, or enable the creation or
  11  * redistribution of, unlawful or unlicensed copies of an Apple operating
  12  * system, or to circumvent, violate, or enable the circumvention or
  13  * violation of, any terms of an Apple operating system software license
  14  * agreement.
  15  *
  16  * Please obtain a copy of the License at
  17  * http://www.opensource.apple.com/apsl/ and read it before using this
  18  * file.
  19  *
  20  * The Original Code and all software distributed under the License are
  21  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  22  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  23  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  24  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  25  * Please see the License for the specific language governing rights and
  26  * limitations under the License.
  27  *
  28  * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
  29  */
  30
  31 /*
  32  *      Here's what to do if you want to add a new routine to the comm page:
  33  *
  34  *              1. Add a definition for it's address in osfmk/i386/cpu_capabilities.h,
  35  *                 being careful to reserve room for future expansion.
  36  *
  37  *              2. Write one or more versions of the routine, each with it's own
  38  *                 commpage_descriptor.  The tricky part is getting the "special",
  39  *                 "musthave", and "canthave" fields right, so that exactly one
  40  *                 version of the routine is selected for every machine.
  41  *                 The source files should be in osfmk/i386/commpage/.
  42  *
  43  *              3. Add a ptr to your new commpage_descriptor(s) in the "routines"
  44  *                 array in osfmk/i386/commpage/commpage_asm.s.  There are two
  45  *                 arrays, one for the 32-bit and one for the 64-bit commpage.
  46  *
  47  *              4. Write the code in Libc to use the new routine.
  48  */
  49
  50 #include <mach/mach_types.h>
  51 #include <mach/machine.h>
  52 #include <mach/vm_map.h>
  53 #include <i386/machine_routines.h>
  54 #include <i386/misc_protos.h>
  55 #include <machine/cpu_capabilities.h>
  56 #include <machine/commpage.h>
  57 #include <machine/pmap.h>
  58 #include <vm/vm_kern.h>
  59 #include <vm/vm_map.h>
  60 #include <ipc/ipc_port.h>
  61
  62 #include <kern/page_decrypt.h>
  63
  64 /* the lists of commpage routines are in commpage_asm.s  */
  65 extern  commpage_descriptor*    commpage_32_routines[];
  66 extern  commpage_descriptor*    commpage_64_routines[];
  67
  68 /* translated commpage descriptors from commpage_sigs.c  */
  69 extern  commpage_descriptor sigdata_descriptor;
  70 extern  commpage_descriptor *ba_descriptors[];
  71
  72 extern vm_map_t com_region_map32;       // the shared submap, set up in vm init
  73 extern vm_map_t com_region_map64;       // the shared submap, set up in vm init
  74
  75 char    *commPagePtr32 = NULL;          // virtual addr in kernel map of 32-bit commpage
  76 char    *commPagePtr64 = NULL;          // ...and of 64-bit commpage
  77 int     _cpu_capabilities = 0;          // define the capability vector
  78
  79 int     noVMX = 0;              /* if true, do not set kHasAltivec in ppc _cpu_capabilities */
  80
  81 void*   dsmos_blobs[3];         /* ptrs to the system integrity data in each commpage */
  82 int     dsmos_blob_count = 0;
  83
  84 static uintptr_t next;                  // next available byte in comm page
  85 static int      cur_routine;            // comm page address of "current" routine
  86 static int      matched;                // true if we've found a match for "current" routine
  87
  88 static char    *commPagePtr;            // virtual addr in kernel map of commpage we are working on
  89 static size_t   commPageBaseOffset;     // add to 32-bit runtime address to get offset in commpage
  90
  91 /* Allocate the commpage and add to the shared submap created by vm:
  92  *      1. allocate a page in the kernel map (RW)
  93  *      2. wire it down
  94  *      3. make a memory entry out of it
  95  *      4. map that entry into the shared comm region map (R-only)
  96  */
  97
  98 static  void*
  99 commpage_allocate(
 100         vm_map_t        submap,                 // com_region_map32 or com_region_map64
 101         size_t          area_used )             // _COMM_PAGE32_AREA_USED or _COMM_PAGE64_AREA_USED
 102 {
 103         vm_offset_t     kernel_addr;            // address of commpage in kernel map
 104         vm_offset_t     zero = 0;
 105         vm_size_t       size = area_used;       // size actually populated
 106         vm_map_entry_t  entry;
 107         ipc_port_t      handle;
 108
 109         if (submap == NULL)
 110                 panic("commpage submap is null");
 111
 112         if (vm_allocate(kernel_map,&kernel_addr,area_used,VM_FLAGS_ANYWHERE))
 113                 panic("cannot allocate commpage");
 114
 115         if (vm_map_wire(kernel_map,kernel_addr,kernel_addr+area_used,VM_PROT_DEFAULT,FALSE))
 116                 panic("cannot wire commpage");
 117
 118         /*
 119          * Now that the object is created and wired into the kernel map, mark it so that no delay
 120          * copy-on-write will ever be performed on it as a result of mapping it into user-space.
 121          * If such a delayed copy ever occurred, we could remove the kernel's wired mapping - and
 122          * that would be a real disaster.
 123          *
 124          * JMM - What we really need is a way to create it like this in the first place.
 125          */
 126         if (!vm_map_lookup_entry( kernel_map, vm_map_trunc_page(kernel_addr), &entry) || entry->is_sub_map)
 127                 panic("cannot find commpage entry");
 128         entry->object.vm_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
 129
 130         if (mach_make_memory_entry( kernel_map,         // target map
 131                                     &size,              // size
 132                                     kernel_addr,        // offset (address in kernel map)
 133                                     VM_PROT_DEFAULT,    // map it RW
 134                                     &handle,            // this is the object handle we get
 135                                     NULL ))             // parent_entry (what is this?)
 136                 panic("cannot make entry for commpage");
 137
 138         if (vm_map_64(  submap,                         // target map (shared submap)
 139                         &zero,                          // address (map into 1st page in submap)
 140                         area_used,                      // size
 141                         0,                              // mask
 142                         VM_FLAGS_FIXED,                 // flags (it must be 1st page in submap)
 143                         handle,                         // port is the memory entry we just made
 144                         0,                              // offset (map 1st page in memory entry)
 145                         FALSE,                          // copy
 146                         VM_PROT_READ,                   // cur_protection (R-only in user map)
 147                         VM_PROT_READ,                   // max_protection
 148                         VM_INHERIT_SHARE ))             // inheritance
 149                 panic("cannot map commpage");
 150
 151         ipc_port_release(handle);
 152
 153         return (void*) kernel_addr;                     // return address in kernel map
 154 }
 155
 156 /* Get address (in kernel map) of a commpage field. */
 157
 158 static void*
 159 commpage_addr_of(
 160     int     addr_at_runtime )
 161 {
 162     return  (void*) ((uintptr_t)commPagePtr + addr_at_runtime - commPageBaseOffset);
 163 }
 164
 165 /* Determine number of CPUs on this system.  We cannot rely on
 166  * machine_info.max_cpus this early in the boot.
 167  */
 168 static int
 169 commpage_cpus( void )
 170 {
 171         int cpus;
 172
 173         cpus = ml_get_max_cpus();                   // NB: this call can block
 174
 175         if (cpus == 0)
 176                 panic("commpage cpus==0");
 177         if (cpus > 0xFF)
 178                 cpus = 0xFF;
 179
 180         return cpus;
 181 }
 182
 183 /* Initialize kernel version of _cpu_capabilities vector (used by KEXTs.) */
 184
 185 static void
 186 commpage_init_cpu_capabilities( void )
 187 {
 188         int bits;
 189         int cpus;
 190         ml_cpu_info_t cpu_info;
 191
 192         bits = 0;
 193         ml_cpu_get_info(&cpu_info);
 194
 195         switch (cpu_info.vector_unit) {
 196                 case 6:
 197                         bits |= kHasSupplementalSSE3;
 198                         /* fall thru */
 199                 case 5:
 200                         bits |= kHasSSE3;
 201                         /* fall thru */
 202                 case 4:
 203                         bits |= kHasSSE2;
 204                         /* fall thru */
 205                 case 3:
 206                         bits |= kHasSSE;
 207                         /* fall thru */
 208                 case 2:
 209                         bits |= kHasMMX;
 210                 default:
 211                         break;
 212         }
 213         switch (cpu_info.cache_line_size) {
 214                 case 128:
 215                         bits |= kCache128;
 216                         break;
 217                 case 64:
 218                         bits |= kCache64;
 219                         break;
 220                 case 32:
 221                         bits |= kCache32;
 222                         break;
 223                 default:
 224                         break;
 225         }
 226         cpus = commpage_cpus();                 // how many CPUs do we have
 227
 228         if (cpus == 1)
 229                 bits |= kUP;
 230
 231         bits |= (cpus << kNumCPUsShift);
 232
 233         bits |= kFastThreadLocalStorage;        // we use %gs for TLS
 234
 235         if (cpu_mode_is64bit())                 // k64Bit means processor is 64-bit capable
 236                 bits |= k64Bit;
 237
 238         _cpu_capabilities = bits;               // set kernel version for use by drivers etc
 239 }
 240
 241 int
 242 _get_cpu_capabilities()
 243 {
 244         return _cpu_capabilities;
 245 }
 246
 247 /* Copy data into commpage. */
 248
 249 static void
 250 commpage_stuff(
 251     int         address,
 252     const void  *source,
 253     int         length  )
 254 {
 255     void        *dest = commpage_addr_of(address);
 256
 257     if ((uintptr_t)dest < next)
 258         panic("commpage overlap at address 0x%x, 0x%x < 0x%x", address, dest, next);
 259
 260     bcopy(source,dest,length);
 261
 262     next = ((uintptr_t)dest + length);
 263 }
 264
 265 static void
 266 commpage_stuff_swap(
 267         int     address,
 268         void    *source,
 269         int     length,
 270         int     legacy )
 271 {
 272         if ( legacy ) {
 273                 void *dest = commpage_addr_of(address);
 274                 dest = (void *)((uintptr_t) dest + _COMM_PAGE_SIGS_OFFSET);
 275                 switch (length) {
 276                         case 2:
 277                                 OSWriteSwapInt16(dest, 0, *(uint16_t *)source);
 278                                 break;
 279                         case 4:
 280                                 OSWriteSwapInt32(dest, 0, *(uint32_t *)source);
 281                                 break;
 282                         case 8:
 283                                 OSWriteSwapInt64(dest, 0, *(uint64_t *)source);
 284                                 break;
 285                 }
 286         }
 287 }
 288
 289 static void
 290 commpage_stuff2(
 291         int     address,
 292         void    *source,
 293         int     length,
 294         int     legacy )
 295 {
 296         commpage_stuff_swap(address, source, length, legacy);
 297         commpage_stuff(address, source, length);
 298 }
 299
 300 /* Copy a routine into comm page if it matches running machine.
 301  */
 302 static void
 303 commpage_stuff_routine(
 304     commpage_descriptor *rd     )
 305 {
 306     int         must,cant;
 307
 308     if (rd->commpage_address != cur_routine) {
 309         if ((cur_routine!=0) && (matched==0))
 310             panic("commpage no match for last, next address %08x", rd->commpage_address);
 311         cur_routine = rd->commpage_address;
 312         matched = 0;
 313     }
 314
 315     must = _cpu_capabilities & rd->musthave;
 316     cant = _cpu_capabilities & rd->canthave;
 317
 318     if ((must == rd->musthave) && (cant == 0)) {
 319         if (matched)
 320             panic("commpage multiple matches for address %08x", rd->commpage_address);
 321         matched = 1;
 322
 323         commpage_stuff(rd->commpage_address,rd->code_address,rd->code_length);
 324         }
 325 }
 326
 327 /* Fill in the 32- or 64-bit commpage.  Called once for each.
 328  * The 32-bit ("legacy") commpage has a bunch of stuff added to it
 329  * for translated processes, some of which is byte-swapped.
 330  */
 331
 332 static void
 333 commpage_populate_one(
 334         vm_map_t        submap,         // com_region_map32 or com_region_map64
 335         char **         kernAddressPtr, // &commPagePtr32 or &commPagePtr64
 336         size_t          area_used,      // _COMM_PAGE32_AREA_USED or _COMM_PAGE64_AREA_USED
 337         size_t          base_offset,    // will become commPageBaseOffset
 338         commpage_descriptor** commpage_routines, // list of routine ptrs for this commpage
 339         boolean_t       legacy,         // true if 32-bit commpage
 340         const char*     signature )     // "commpage 32-bit" or "commpage 64-bit"
 341 {
 342         short   c2;
 343         static double   two52 = 1048576.0 * 1048576.0 * 4096.0; // 2**52
 344         static double   ten6 = 1000000.0;                       // 10**6
 345         commpage_descriptor **rd;
 346         short   version = _COMM_PAGE_THIS_VERSION;
 347         int             swapcaps;
 348
 349         next = (uintptr_t) NULL;
 350         cur_routine = 0;
 351         commPagePtr = (char *)commpage_allocate( submap, (vm_size_t) area_used );
 352         *kernAddressPtr = commPagePtr;                          // save address either in commPagePtr32 or 64
 353         commPageBaseOffset = base_offset;
 354
 355         /* Stuff in the constants.  We move things into the comm page in strictly
 356         * ascending order, so we can check for overlap and panic if so.
 357         */
 358         commpage_stuff(_COMM_PAGE_SIGNATURE,signature,strlen(signature));
 359         commpage_stuff2(_COMM_PAGE_VERSION,&version,sizeof(short),legacy);
 360         commpage_stuff(_COMM_PAGE_CPU_CAPABILITIES,&_cpu_capabilities,sizeof(int));
 361
 362         /* excuse our magic constants, we cannot include ppc/cpu_capabilities.h */
 363         /* always set kCache32 and kDcbaAvailable */
 364         swapcaps =  0x44;
 365         if ( _cpu_capabilities & kUP )
 366                 swapcaps |= (kUP + (1 << kNumCPUsShift));
 367         else
 368                 swapcaps |= 2 << kNumCPUsShift; /* limit #cpus to 2 */
 369         if ( ! noVMX )          /* if rosetta will be emulating altivec... */
 370                 swapcaps |= 0x101;      /* ...then set kHasAltivec and kDataStreamsAvailable too */
 371         commpage_stuff_swap(_COMM_PAGE_CPU_CAPABILITIES, &swapcaps, sizeof(int), legacy);
 372         c2 = 32;
 373         commpage_stuff_swap(_COMM_PAGE_CACHE_LINESIZE,&c2,2,legacy);
 374
 375         if (_cpu_capabilities & kCache32)
 376                 c2 = 32;
 377         else if (_cpu_capabilities & kCache64)
 378                 c2 = 64;
 379         else if (_cpu_capabilities & kCache128)
 380                 c2 = 128;
 381         commpage_stuff(_COMM_PAGE_CACHE_LINESIZE,&c2,2);
 382
 383         if ( legacy ) {
 384                 commpage_stuff2(_COMM_PAGE_2_TO_52,&two52,8,legacy);
 385                 commpage_stuff2(_COMM_PAGE_10_TO_6,&ten6,8,legacy);
 386         }
 387
 388         for( rd = commpage_routines; *rd != NULL ; rd++ )
 389                 commpage_stuff_routine(*rd);
 390
 391         if (!matched)
 392                 panic("commpage no match on last routine");
 393
 394         if (next > (uintptr_t)_COMM_PAGE_END)
 395                 panic("commpage overflow: next = 0x%08x, commPagePtr = 0x%08x", next, (uintptr_t)commPagePtr);
 396
 397         if ( legacy ) {
 398                 next = (uintptr_t) NULL;
 399                 for( rd = ba_descriptors; *rd != NULL ; rd++ )
 400                         commpage_stuff_routine(*rd);
 401
 402                 next = (uintptr_t) NULL;
 403                 commpage_stuff_routine(&sigdata_descriptor);
 404         }
 405
 406         /* salt away a ptr to the system integrity data in this commpage */
 407         dsmos_blobs[dsmos_blob_count++] =
 408                 commpage_addr_of( _COMM_PAGE_SYSTEM_INTEGRITY );
 409 }
 410
 411
 412 /* Fill in commpages: called once, during kernel initialization, from the
 413  * startup thread before user-mode code is running.
 414  *
 415  * See the top of this file for a list of what you have to do to add
 416  * a new routine to the commpage.
 417  */
 418
 419 void
 420 commpage_populate( void )
 421 {
 422         commpage_init_cpu_capabilities();
 423
 424         commpage_populate_one(  com_region_map32,
 425                                 &commPagePtr32,
 426                                 _COMM_PAGE32_AREA_USED,
 427                                 _COMM_PAGE32_BASE_ADDRESS,
 428                                 commpage_32_routines,
 429                                 TRUE,                   /* legacy (32-bit) commpage */
 430                                 "commpage 32-bit");
 431         pmap_commpage32_init((vm_offset_t) commPagePtr32, _COMM_PAGE32_BASE_ADDRESS,
 432                            _COMM_PAGE32_AREA_USED/INTEL_PGBYTES);
 433
 434         if (_cpu_capabilities & k64Bit) {
 435                 commpage_populate_one(  com_region_map64,
 436                                         &commPagePtr64,
 437                                         _COMM_PAGE64_AREA_USED,
 438                                         _COMM_PAGE32_START_ADDRESS, /* because kernel is built 32-bit */
 439                                         commpage_64_routines,
 440                                         FALSE,          /* not a legacy commpage */
 441                                         "commpage 64-bit");
 442                 pmap_commpage64_init((vm_offset_t) commPagePtr64, _COMM_PAGE64_BASE_ADDRESS,
 443                                    _COMM_PAGE64_AREA_USED/INTEL_PGBYTES);
 444         }
 445
 446         rtc_nanotime_init_commpage();
 447 }