osfmk/i386/startup64.c

   1 /*
   2  * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <string.h>
  30
  31 #include <mach/machine/vm_types.h>
  32
  33 #include <mach/boolean.h>
  34 #include <kern/thread.h>
  35 #include <kern/zalloc.h>
  36
  37 #include <kern/lock.h>
  38 #include <kern/kalloc.h>
  39 #include <kern/spl.h>
  40
  41 #include <vm/pmap.h>
  42 #include <vm/vm_map.h>
  43 #include <vm/vm_kern.h>
  44 #include <mach/vm_param.h>
  45 #include <mach/vm_prot.h>
  46 #include <vm/vm_object.h>
  47 #include <vm/vm_page.h>
  48
  49 #include <mach/machine/vm_param.h>
  50 #include <machine/thread.h>
  51
  52 #include <kern/misc_protos.h>                   /* prototyping */
  53 #include <i386/misc_protos.h>
  54
  55 #include <i386/cpuid.h>
  56 #include <i386/cpu_data.h>
  57 #include <i386/mp.h>
  58 #include <i386/cpu_number.h>
  59 #include <i386/machine_cpu.h>
  60 #include <i386/mp_slave_boot.h>
  61 #include <i386/seg.h>
  62
  63 #include <vm/vm_protos.h>
  64
  65 #include <sys/kdebug.h>
  66
  67 #include <i386/postcode.h>
  68
  69 void
  70 cpu_IA32e_enable(cpu_data_t *cdp)
  71 {
  72         assert(!ml_get_interrupts_enabled());
  73
  74         if (!cdp->cpu_is64bit ||
  75             (rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) != 0)
  76                 return;
  77
  78         postcode(CPU_IA32_ENABLE_ENTRY);
  79
  80         /*
  81          * The following steps are performed by inlines so that
  82          * we can be assured we don't use the stack or any other
  83          * non-identity mapped data while paging is turned off...
  84          */
  85         /* Turn paging off */
  86         asm volatile(
  87                 "mov    %%cr0, %%eax    \n\t"
  88                 "andl   %0, %%eax       \n\t"
  89                 "mov    %%eax, %%cr0    \n\t"
  90                 :
  91                 : "i" (~CR0_PG)
  92                 : "eax" );
  93
  94         /* Pop new top level phys pg addr into CR3 */
  95         asm volatile(
  96                 "mov    %%eax, %%cr3    \n\t"
  97                 :
  98                 : "a" ((uint32_t) kernel64_cr3));
  99
 100         /* Turn on the 64-bit mode bit */
 101         asm volatile(
 102                 "rdmsr                  \n\t"
 103                 "orl    %1, %%eax       \n\t"
 104                 "wrmsr                  \n\t"
 105                 :
 106                 : "c" (MSR_IA32_EFER), "i" (MSR_IA32_EFER_LME)
 107                 : "eax", "edx");
 108
 109         /* Turn paging on again */
 110         asm volatile(
 111                 "mov    %%cr0, %%eax    \n\t"
 112                 "orl    %0, %%eax       \n\t"
 113                 "mov    %%eax, %%cr0    \n\t"
 114                 :
 115                 : "i" (CR0_PG)
 116                 : "eax" );
 117
 118         kprintf("cpu_IA32e_enable(%p)\n", cdp);
 119
 120         if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0)
 121                 panic("cpu_IA32e_enable() MSR_IA32_EFER_LMA not asserted");
 122
 123         cdp->cpu_kernel_cr3 = kernel64_cr3;
 124
 125         postcode(CPU_IA32_ENABLE_EXIT);
 126 }
 127
 128 void
 129 cpu_IA32e_disable(cpu_data_t *cdp)
 130 {
 131         assert(!ml_get_interrupts_enabled());
 132
 133         postcode(CPU_IA32_DISABLE_ENTRY);
 134
 135         if (!cdp->cpu_is64bit ||
 136             (rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) == 0)
 137                 return;
 138
 139         /*
 140          * The following steps are performed by inlines so that
 141          * we can be assured we don't use the stack or any other
 142          * non-identity mapped data while paging is turned off...
 143          */
 144         /* Turn paging off */
 145         asm volatile(
 146                 "mov    %%cr0, %%eax    \n\t"
 147                 "andl   %0, %%eax       \n\t"
 148                 "mov    %%eax, %%cr0    \n\t"
 149                 :
 150                 : "i" (~CR0_PG)
 151                 : "eax" );
 152
 153         /* Pop legacy top level phys pg addr into CR3 */
 154         asm volatile(
 155                 "mov    %%eax, %%cr3    \n\t"
 156                 :
 157                 : "a" ((uint32_t) lo_kernel_cr3));
 158
 159         /* Turn off the 64-bit mode bit */
 160         asm volatile(
 161                 "rdmsr                  \n\t"
 162                 "andl   %1, %%eax       \n\t"
 163                 "wrmsr                  \n\t"
 164                 :
 165                 : "c" (MSR_IA32_EFER), "i" (~MSR_IA32_EFER_LME)
 166                 : "eax", "edx");
 167
 168         /* Turn paging on again */
 169         asm volatile(
 170                 "mov    %%cr0, %%eax    \n\t"
 171                 "orl    %0, %%eax       \n\t"
 172                 "mov    %%eax, %%cr0    \n\t"
 173                 :
 174                 : "i" (CR0_PG)
 175                 : "eax" );
 176
 177         kprintf("cpu_IA32e_disable(%p)\n", cdp);
 178
 179         if ((rdmsr64(MSR_IA32_EFER) & MSR_IA32_EFER_LMA) != 0)
 180                 panic("cpu_IA32e_disable() MSR_IA32_EFER_LMA not cleared");
 181
 182         cdp->cpu_kernel_cr3 = 0ULL;
 183
 184         postcode(CPU_IA32_DISABLE_EXIT);
 185 }
 186
 187 void
 188 fix_desc64(void *descp, int count)
 189 {
 190         struct fake_descriptor64        *fakep;
 191         union {
 192                 struct real_gate64              gate;
 193                 struct real_descriptor64        desc;
 194         }                               real;
 195         int                             i;
 196
 197         fakep = (struct fake_descriptor64 *) descp;
 198
 199         for (i = 0; i < count; i++, fakep++) {
 200                 /*
 201                  * Construct the real decriptor locally.
 202                  */
 203
 204                 bzero((void *) &real, sizeof(real));
 205
 206                 switch (fakep->access & ACC_TYPE) {
 207                 case 0:
 208                         break;
 209                 case ACC_CALL_GATE:
 210                 case ACC_INTR_GATE:
 211                 case ACC_TRAP_GATE:
 212                         real.gate.offset_low16 = fakep->offset[0] & 0xFFFF;
 213                         real.gate.selector16 = fakep->lim_or_seg & 0xFFFF;
 214                         real.gate.IST = fakep->size_or_IST & 0x7;
 215                         real.gate.access8 = fakep->access;
 216                         real.gate.offset_high16 = (fakep->offset[0]>>16)&0xFFFF;
 217                         real.gate.offset_top32 = (uint32_t)fakep->offset[1];
 218                         break;
 219                 default:        /* Otherwise */
 220                         real.desc.limit_low16 = fakep->lim_or_seg & 0xFFFF;
 221                         real.desc.base_low16 = fakep->offset[0] & 0xFFFF;
 222                         real.desc.base_med8 = (fakep->offset[0] >> 16) & 0xFF;
 223                         real.desc.access8 = fakep->access;
 224                         real.desc.limit_high4 = (fakep->lim_or_seg >> 16) & 0xFF;
 225                         real.desc.granularity4 = fakep->size_or_IST;
 226                         real.desc.base_high8 = (fakep->offset[0] >> 24) & 0xFF;
 227                         real.desc.base_top32 = (uint32_t) fakep->offset[1];
 228                 }
 229
 230                 /*
 231                  * Now copy back over the fake structure.
 232                  */
 233                 bcopy((void *) &real, (void *) fakep, sizeof(real));
 234         }
 235 }
 236
 237 #if DEBUG
 238 extern void dump_gdt(void *);
 239 extern void dump_ldt(void *);
 240 extern void dump_idt(void *);
 241 extern void dump_tss(void *);
 242 extern void dump_frame32(x86_saved_state_compat32_t *scp);
 243 extern void dump_frame64(x86_saved_state64_t *sp);
 244 extern void dump_frame(x86_saved_state_t *sp);
 245
 246 void
 247 dump_frame(x86_saved_state_t *sp)
 248 {
 249         if (is_saved_state32(sp))
 250                 dump_frame32((x86_saved_state_compat32_t *) sp);
 251         else if (is_saved_state64(sp))
 252                 dump_frame64(&sp->ss_64);
 253         else
 254                 kprintf("dump_frame(%p) unknown type %d\n", sp, sp->flavor);
 255 }
 256
 257 void
 258 dump_frame32(x86_saved_state_compat32_t *scp)
 259 {
 260         unsigned int    i;
 261         uint32_t        *ip = (uint32_t *) scp;
 262
 263         kprintf("dump_frame32(%p):\n", scp);
 264
 265         for (i = 0;
 266              i < sizeof(x86_saved_state_compat32_t)/sizeof(uint32_t);
 267              i++, ip++)
 268                 kprintf("%p: 0x%08x\n", ip, *ip);
 269
 270         kprintf("scp->isf64.err:    0x%016llx\n", scp->isf64.err);
 271         kprintf("scp->isf64.rip:    0x%016llx\n", scp->isf64.rip);
 272         kprintf("scp->isf64.cs:     0x%016llx\n", scp->isf64.cs);
 273         kprintf("scp->isf64.rflags: 0x%016llx\n", scp->isf64.rflags);
 274         kprintf("scp->isf64.rsp:    0x%016llx\n", scp->isf64.rsp);
 275         kprintf("scp->isf64.ss:     0x%016llx\n", scp->isf64.ss);
 276
 277         kprintf("scp->iss32.tag:    0x%08x\n", scp->iss32.tag);
 278         kprintf("scp->iss32.state.gs:     0x%08x\n", scp->iss32.state.gs);
 279         kprintf("scp->iss32.state.fs:     0x%08x\n", scp->iss32.state.fs);
 280         kprintf("scp->iss32.state.es:     0x%08x\n", scp->iss32.state.es);
 281         kprintf("scp->iss32.state.ds:     0x%08x\n", scp->iss32.state.ds);
 282         kprintf("scp->iss32.state.edi:    0x%08x\n", scp->iss32.state.edi);
 283         kprintf("scp->iss32.state.esi:    0x%08x\n", scp->iss32.state.esi);
 284         kprintf("scp->iss32.state.ebp:    0x%08x\n", scp->iss32.state.ebp);
 285         kprintf("scp->iss32.state.cr2:    0x%08x\n", scp->iss32.state.cr2);
 286         kprintf("scp->iss32.state.ebx:    0x%08x\n", scp->iss32.state.ebx);
 287         kprintf("scp->iss32.state.edx:    0x%08x\n", scp->iss32.state.edx);
 288         kprintf("scp->iss32.state.ecx:    0x%08x\n", scp->iss32.state.ecx);
 289         kprintf("scp->iss32.state.eax:    0x%08x\n", scp->iss32.state.eax);
 290         kprintf("scp->iss32.state.trapno: 0x%08x\n", scp->iss32.state.eax);
 291         kprintf("scp->iss32.state.eip:    0x%08x\n", scp->iss32.state.eip);
 292         kprintf("scp->iss32.state.cs:     0x%08x\n", scp->iss32.state.cs);
 293         kprintf("scp->iss32.state.efl:    0x%08x\n", scp->iss32.state.efl);
 294         kprintf("scp->iss32.state.uesp:   0x%08x\n", scp->iss32.state.uesp);
 295         kprintf("scp->iss32.state.ss:     0x%08x\n", scp->iss32.state.ss);
 296
 297         postcode(0x99);
 298 }
 299
 300 void
 301 dump_frame64(x86_saved_state64_t *sp)
 302 {
 303         unsigned int    i;
 304         uint64_t        *ip = (uint64_t *) sp;
 305
 306         kprintf("dump_frame64(%p):\n", sp);
 307
 308         for (i = 0;
 309              i < sizeof(x86_saved_state64_t)/sizeof(uint64_t);
 310              i++, ip++)
 311                 kprintf("%p: 0x%016llx\n", ip, *ip);
 312
 313         kprintf("sp->isf.trapno: 0x%08x\n", sp->isf.trapno);
 314         kprintf("sp->isf.trapfn: 0x%08x\n", sp->isf.trapfn);
 315         kprintf("sp->isf.err:    0x%016llx\n", sp->isf.err);
 316         kprintf("sp->isf.rip:    0x%016llx\n", sp->isf.rip);
 317         kprintf("sp->isf.cs:     0x%016llx\n", sp->isf.cs);
 318         kprintf("sp->isf.rflags: 0x%016llx\n", sp->isf.rflags);
 319         kprintf("sp->isf.rsp:    0x%016llx\n", sp->isf.rsp);
 320         kprintf("sp->isf.ss:     0x%016llx\n", sp->isf.ss);
 321
 322         kprintf("sp->fs:         0x%016x\n", sp->fs);
 323         kprintf("sp->gs:         0x%016x\n", sp->gs);
 324         kprintf("sp->rax:        0x%016llx\n", sp->rax);
 325         kprintf("sp->rcx:        0x%016llx\n", sp->rcx);
 326         kprintf("sp->rbx:        0x%016llx\n", sp->rbx);
 327         kprintf("sp->rbp:        0x%016llx\n", sp->rbp);
 328         kprintf("sp->r11:        0x%016llx\n", sp->r11);
 329         kprintf("sp->r12:        0x%016llx\n", sp->r12);
 330         kprintf("sp->r13:        0x%016llx\n", sp->r13);
 331         kprintf("sp->r14:        0x%016llx\n", sp->r14);
 332         kprintf("sp->r15:        0x%016llx\n", sp->r15);
 333         kprintf("sp->cr2:        0x%016llx\n", sp->cr2);
 334         kprintf("sp->v_arg8:     0x%016llx\n", sp->v_arg8);
 335         kprintf("sp->v_arg7:     0x%016llx\n", sp->v_arg7);
 336         kprintf("sp->v_arg6:     0x%016llx\n", sp->v_arg6);
 337         kprintf("sp->r9:         0x%016llx\n", sp->r9);
 338         kprintf("sp->r8:         0x%016llx\n", sp->r8);
 339         kprintf("sp->r10:        0x%016llx\n", sp->r10);
 340         kprintf("sp->rdx:        0x%016llx\n", sp->rdx);
 341         kprintf("sp->rsi:        0x%016llx\n", sp->rsi);
 342         kprintf("sp->rdi:        0x%016llx\n", sp->rdi);
 343
 344         postcode(0x98);
 345 }
 346
 347 void
 348 dump_gdt(void *gdtp)
 349 {
 350         unsigned int    i;
 351         uint32_t        *ip = (uint32_t *) gdtp;
 352
 353         kprintf("GDT:\n");
 354         for (i = 0; i < GDTSZ; i++, ip += 2) {
 355                 kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
 356                 kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
 357         }
 358 }
 359
 360 void
 361 dump_ldt(void *ldtp)
 362 {
 363         unsigned int    i;
 364         uint32_t        *ip = (uint32_t *) ldtp;
 365
 366         kprintf("LDT:\n");
 367         for (i = 0; i < LDTSZ_MIN; i++, ip += 2) {
 368                 kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
 369                 kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
 370         }
 371 }
 372
 373 void
 374 dump_idt(void *idtp)
 375 {
 376         unsigned int    i;
 377         uint32_t        *ip = (uint32_t *) idtp;
 378
 379         kprintf("IDT64:\n");
 380         for (i = 0; i < 16; i++, ip += 4) {
 381                 kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
 382                 kprintf("%p: 0x%08x\n", ip+1, *(ip+1));
 383                 kprintf("%p: 0x%08x\n", ip+2, *(ip+2));
 384                 kprintf("%p: 0x%08x\n", ip+3, *(ip+3));
 385         }
 386 }
 387
 388 void
 389 dump_tss(void *tssp)
 390 {
 391         unsigned int    i;
 392         uint32_t        *ip = (uint32_t *) tssp;
 393
 394         kprintf("TSS64:\n");
 395         for (i = 0; i < sizeof(master_ktss64)/sizeof(uint32_t); i++, ip++) {
 396                 kprintf("%p: 0x%08x\n", ip+0, *(ip+0));
 397         }
 398 }
 399 #endif /* DEBUG */