osfmk/x86_64/start.s

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58
  59 #include <platforms.h>
  60 #include <debug.h>
  61
  62 #include <i386/asm.h>
  63 #include <i386/proc_reg.h>
  64 #include <i386/postcode.h>
  65 #include <assym.s>
  66
  67 #include <i386/mp.h>
  68 #include <i386/cpuid.h>
  69 #include <i386/acpi.h>
  70
  71 .code32
  72
  73
  74 /*
  75  * Interrupt and bootup stack for initial processor.
  76  * Note: we switch to a dynamically allocated interrupt stack once VM is up.
  77  */
  78
  79 /* in the __HIB section since the hibernate restore code uses this stack. */
  80         .section __HIB, __data
  81         .align  12
  82
  83         .globl  EXT(low_intstack)
  84 EXT(low_intstack):
  85         .globl  EXT(gIOHibernateRestoreStack)
  86 EXT(gIOHibernateRestoreStack):
  87
  88         .space  INTSTACK_SIZE
  89
  90         .globl  EXT(low_eintstack)
  91 EXT(low_eintstack:)
  92         .globl  EXT(gIOHibernateRestoreStackEnd)
  93 EXT(gIOHibernateRestoreStackEnd):
  94
  95         /* back to the regular __DATA section. */
  96
  97         .section __DATA, __data
  98
  99 /*
 100  * Stack for machine-check handler.
 101  */
 102         .align  12
 103         .globl  EXT(mc_task_stack)
 104 EXT(mc_task_stack):
 105         .space  INTSTACK_SIZE
 106         .globl  EXT(mc_task_stack_end)
 107 EXT(mc_task_stack_end):
 108
 109         /* Must not clobber EDI */
 110 #define SWITCH_TO_64BIT_MODE                                     \
 111         movl    $(CR4_PAE),%eax         /* enable PAE */        ;\
 112         movl    %eax,%cr4                                       ;\
 113         movl    $MSR_IA32_EFER,%ecx                             ;\
 114         rdmsr                                                   ;\
 115         /* enable long mode, NX */                              ;\
 116         orl     $(MSR_IA32_EFER_LME | MSR_IA32_EFER_NXE),%eax   ;\
 117         wrmsr                                                   ;\
 118         movl    $EXT(BootPML4),%eax                             ;\
 119         movl    %eax,%cr3                                       ;\
 120         movl    %cr0,%eax                                       ;\
 121         orl     $(CR0_PG|CR0_WP),%eax   /* enable paging */     ;\
 122         movl    %eax,%cr0                                       ;\
 123         /* "The Aussie Maneuver" ("Myria" variant) */           ;\
 124         pushl $(0xcb<<24)|KERNEL64_CS /* reload CS with 0x08 */ ;\
 125         call .-1                                                ;\
 126         .code64
 127
 128 /*
 129  * BSP CPU start here.
 130  *      eax points to kernbootstruct
 131  *
 132  * Environment:
 133  *      protected mode, no paging, flat 32-bit address space.
 134  *      (Code/data/stack segments have base == 0, limit == 4G)
 135  */
 136
 137 .code32
 138         .text
 139         .section __HIB, __text
 140         .align  ALIGN
 141         .globl  EXT(_start)
 142         .globl  EXT(pstart)
 143 LEXT(_start)
 144 LEXT(pstart)
 145
 146 /*
 147  * Here we do the minimal setup to switch from 32 bit mode to 64 bit long mode.
 148  *
 149  * Initial memory layout:
 150  *
 151  *      -------------------------
 152  *      |                       |
 153  *      | Kernel text/data      |
 154  *      |                       |
 155  *      |-----------------------| Kernel text base addr - 2MB-aligned
 156  *      | padding               |
 157  *      |-----------------------|
 158  *      | __HIB section         |
 159  *      |-----------------------| Page-aligned
 160  *      |                       |
 161  *      | padding               |
 162  *      |                       |
 163  *      ------------------------- 0
 164  *
 165  */
 166         mov     %eax, %edi      /* save kernbootstruct */
 167
 168         /* Use low 32-bits of address as 32-bit stack */
 169         movl    $EXT(low_eintstack), %esp
 170
 171         POSTCODE(PSTART_ENTRY)
 172
 173         /*
 174          * Set up segmentation
 175          */
 176         movl    $EXT(protected_mode_gdtr), %eax
 177         lgdtl   (%eax)
 178
 179         /*
 180          * Rebase Boot page tables to kernel base address.
 181          */
 182         movl    $EXT(BootPML4), %eax                    // Level 4:
 183         add     %eax, 0*8+0(%eax)                       //  - 1:1
 184         add     %eax, KERNEL_PML4_INDEX*8+0(%eax)       //  - kernel space
 185
 186         movl    $EXT(BootPDPT), %edx                    // Level 3:
 187         add     %eax, 0*8+0(%edx)
 188         add     %eax, 1*8+0(%edx)
 189         add     %eax, 2*8+0(%edx)
 190         add     %eax, 3*8+0(%edx)
 191
 192         POSTCODE(PSTART_REBASE)
 193
 194 /* the following code is shared by the master CPU and all slave CPUs */
 195 L_pstart_common:
 196         /*
 197          * switch to 64 bit mode
 198          */
 199         SWITCH_TO_64BIT_MODE
 200
 201         /* Flush data segment selectors */
 202         xor     %eax, %eax
 203         mov     %ax, %ss
 204         mov     %ax, %ds
 205         mov     %ax, %es
 206         mov     %ax, %fs
 207         mov     %ax, %gs
 208
 209         test    %edi, %edi /* Populate stack canary on BSP */
 210         jz      Lvstartshim
 211
 212         mov     $1, %eax
 213         cpuid
 214         test    $(1 << 30), %ecx
 215         jz      Lnon_rdrand
 216         RDRAND_RAX              /* RAX := 64 bits of DRBG entropy */
 217         jnc     Lnon_rdrand     /* TODO: complain if DRBG fails at this stage */
 218
 219 Lstore_random_guard:
 220         xor     %ah, %ah        /* Security: zero second byte of stack canary */
 221         movq    %rax, ___stack_chk_guard(%rip)
 222         /* %edi = boot_args_start if BSP */
 223 Lvstartshim:
 224
 225         POSTCODE(PSTART_VSTART)
 226
 227         /* %edi = boot_args_start */
 228
 229         leaq    _vstart(%rip), %rcx
 230         movq    $0xffffff8000000000, %rax       /* adjust pointer up high */
 231         or      %rax, %rsp                      /* and stack pointer up there */
 232         or      %rcx, %rax
 233         andq    $0xfffffffffffffff0, %rsp       /* align stack */
 234         xorq    %rbp, %rbp                      /* zero frame pointer */
 235         callq   *%rax
 236
 237 Lnon_rdrand:
 238         rdtsc /* EDX:EAX := TSC */
 239         /* Distribute low order bits */
 240         mov     %eax, %ecx
 241         xor     %al, %ah
 242         shl     $16, %rcx
 243         xor     %rcx, %rax
 244         xor     %eax, %edx
 245
 246         /* Incorporate ASLR entropy, if any */
 247         lea     (%rip), %rcx
 248         shr     $21, %rcx
 249         movzbl  %cl, %ecx
 250         shl     $16, %ecx
 251         xor     %ecx, %edx
 252
 253         mov     %ah, %cl
 254         ror     %cl, %edx /* Right rotate EDX (TSC&0xFF ^ (TSC>>8 & 0xFF))&1F */
 255         shl     $32, %rdx
 256         xor     %rdx, %rax
 257         mov     %cl, %al
 258         jmp     Lstore_random_guard
 259 /*
 260  * AP (slave) CPUs enter here.
 261  *
 262  * Environment:
 263  *      protected mode, no paging, flat 32-bit address space.
 264  *      (Code/data/stack segments have base == 0, limit == 4G)
 265  */
 266         .align  ALIGN
 267         .globl  EXT(slave_pstart)
 268 LEXT(slave_pstart)
 269         .code32
 270         cli                             /* disable interrupts, so we don`t */
 271                                         /* need IDT for a while */
 272         POSTCODE(SLAVE_PSTART)
 273
 274         movl    $EXT(mp_slave_stack) + PAGE_SIZE, %esp
 275
 276         xor     %edi, %edi              /* AP, no "kernbootstruct" */
 277
 278         jmp     L_pstart_common         /* hop a ride to vstart() */
 279
 280
 281 /* BEGIN HIBERNATE CODE */
 282
 283 .section __HIB, __text
 284 /*
 285  * This code is linked into the kernel but part of the "__HIB" section,
 286  * which means it's used by code running in the special context of restoring
 287  * the kernel text and data from the hibernation image read by the booter.
 288  * hibernate_kernel_entrypoint() and everything it calls or references
 289  * (ie. hibernate_restore_phys_page()) needs to be careful to only touch
 290  * memory also in the "__HIB" section.
 291  */
 292
 293         .align  ALIGN
 294         .globl  EXT(hibernate_machine_entrypoint)
 295 .code32
 296 LEXT(hibernate_machine_entrypoint)
 297         movl    %eax, %edi /* regparm(1) calling convention */
 298
 299         /* Use low 32-bits of address as 32-bit stack */
 300         movl $EXT(low_eintstack), %esp
 301
 302         /*
 303          * Set up GDT
 304          */
 305         movl    $EXT(master_gdtr), %eax
 306         lgdtl   (%eax)
 307
 308         /* Switch to 64-bit on the Boot PTs */
 309         SWITCH_TO_64BIT_MODE
 310
 311         leaq    EXT(hibernate_kernel_entrypoint)(%rip),%rcx
 312
 313         /* adjust the pointers to be up high */
 314         movq    $0xffffff8000000000, %rax
 315         orq     %rax, %rsp
 316         orq     %rcx, %rax
 317
 318         /* %edi is already filled with header pointer */
 319         xorl    %esi, %esi                      /* zero 2nd arg */
 320         xorl    %edx, %edx                      /* zero 3rd arg */
 321         xorl    %ecx, %ecx                      /* zero 4th arg */
 322         andq    $0xfffffffffffffff0, %rsp       /* align stack */
 323
 324         /* call instead of jmp to keep the required stack alignment */
 325         xorq    %rbp, %rbp                      /* zero frame pointer */
 326         call    *%rax
 327
 328         /* NOTREACHED */
 329         hlt
 330
 331 /* END HIBERNATE CODE */
 332
 333 #if CONFIG_SLEEP
 334 /* BEGIN ACPI WAKEUP CODE */
 335
 336 #include <i386/acpi.h>
 337
 338
 339 /*
 340  * acpi_wake_start
 341  */
 342
 343 .section __TEXT,__text
 344 .code64
 345
 346 /*
 347  * acpi_sleep_cpu(acpi_sleep_callback func, void * refcon)
 348  *
 349  * Save CPU state before platform sleep. Restore CPU state
 350  * following wake up.
 351  */
 352
 353 ENTRY(acpi_sleep_cpu)
 354         push    %rbp
 355         mov     %rsp, %rbp
 356
 357         /* save flags */
 358         pushf
 359
 360         /* save general purpose registers */
 361         push %rax
 362         push %rbx
 363         push %rcx
 364         push %rdx
 365         push %rbp
 366         push %rsi
 367         push %rdi
 368         push %r8
 369         push %r9
 370         push %r10
 371         push %r11
 372         push %r12
 373         push %r13
 374         push %r14
 375         push %r15
 376
 377         mov     %rsp, saved_rsp(%rip)
 378
 379         /* make sure tlb is flushed */
 380         mov     %cr3,%rax
 381         mov     %rax,%cr3
 382
 383         /* save control registers */
 384         mov     %cr0, %rax
 385         mov     %rax, saved_cr0(%rip)
 386         mov     %cr2, %rax
 387         mov     %rax, saved_cr2(%rip)
 388         mov     %cr3, %rax
 389         mov     %rax, saved_cr3(%rip)
 390         mov     %cr4, %rax
 391         mov     %rax, saved_cr4(%rip)
 392
 393         /* save segment registers */
 394         movw    %es, saved_es(%rip)
 395         movw    %fs, saved_fs(%rip)
 396         movw    %gs, saved_gs(%rip)
 397         movw    %ss, saved_ss(%rip)
 398
 399         /* save the 64bit user and kernel gs base */
 400         /* note: user's curently swapped into kernel base MSR */
 401         mov     $MSR_IA32_KERNEL_GS_BASE, %rcx
 402         rdmsr
 403         movl    %eax, saved_ugs_base(%rip)
 404         movl    %edx, saved_ugs_base+4(%rip)
 405         swapgs
 406         rdmsr
 407         movl    %eax, saved_kgs_base(%rip)
 408         movl    %edx, saved_kgs_base+4(%rip)
 409         swapgs
 410
 411         /* save descriptor table registers */
 412         sgdt    saved_gdt(%rip)
 413         sldt    saved_ldt(%rip)
 414         sidt    saved_idt(%rip)
 415         str     saved_tr(%rip)
 416
 417         /*
 418          * Call ACPI function provided by the caller to sleep the platform.
 419          * This call will not return on success.
 420          */
 421
 422         xchgq %rdi, %rsi
 423         call    *%rsi
 424
 425         /* sleep failed, no cpu context lost */
 426         jmp     wake_restore
 427
 428 .section __HIB, __text
 429 .code32
 430 .globl EXT(acpi_wake_prot)
 431 EXT(acpi_wake_prot):
 432         /* protected mode, paging disabled */
 433         movl    $EXT(low_eintstack), %esp
 434
 435         SWITCH_TO_64BIT_MODE
 436
 437         jmp     Lwake_64
 438
 439 .section __TEXT,__text
 440 .code64
 441
 442 .globl EXT(acpi_wake_prot_entry)
 443 EXT(acpi_wake_prot_entry):
 444         POSTCODE(ACPI_WAKE_PROT_ENTRY)
 445         /* Return from hibernate code in iokit/Kernel/IOHibernateRestoreKernel.c
 446          */
 447 Lwake_64:
 448         /*
 449          * restore cr4, PAE and NXE states in an orderly fashion
 450          */
 451         mov     saved_cr4(%rip), %rcx
 452         mov     %rcx, %cr4
 453
 454         mov     $(MSR_IA32_EFER), %ecx          /* MSR number in ecx */
 455         rdmsr                                   /* MSR value in edx:eax */
 456         or      $(MSR_IA32_EFER_NXE), %eax      /* Set NXE bit in low 32-bits */
 457         wrmsr                                   /* Update */
 458
 459         movq    saved_cr2(%rip), %rax
 460         mov     %rax, %cr2
 461
 462         /* restore CR0, paging enabled */
 463         mov     saved_cr0(%rip), %rax
 464         mov     %rax, %cr0
 465
 466         /* restore the page tables */
 467         mov     saved_cr3(%rip), %rax
 468         mov     %rax, %cr3
 469
 470         /* protected mode, paging enabled */
 471         POSTCODE(ACPI_WAKE_PAGED_ENTRY)
 472
 473         /* load null segment selectors */
 474         xor     %eax, %eax
 475         movw    %ax, %ss
 476         movw    %ax, %ds
 477
 478         /* restore descriptor tables */
 479         lgdt    saved_gdt(%rip)
 480         lldt    saved_ldt(%rip)
 481         lidt    saved_idt(%rip)
 482
 483         /* restore segment registers */
 484         movw    saved_es(%rip), %es
 485         movw    saved_fs(%rip), %fs
 486         movw    saved_gs(%rip), %gs
 487         movw    saved_ss(%rip), %ss
 488
 489         /* restore the 64bit kernel and user gs base */
 490         mov     $MSR_IA32_KERNEL_GS_BASE, %rcx
 491         movl    saved_kgs_base(%rip),   %eax
 492         movl    saved_kgs_base+4(%rip), %edx
 493         wrmsr
 494         swapgs
 495         movl    saved_ugs_base(%rip),   %eax
 496         movl    saved_ugs_base+4(%rip), %edx
 497         wrmsr
 498
 499         /*
 500          * Restore task register. Before doing this, clear the busy flag
 501          * in the TSS descriptor set by the CPU.
 502          */
 503         lea     saved_gdt(%rip), %rax
 504         movq    2(%rax), %rdx                   /* GDT base, skip limit word */
 505         movl    $(KERNEL_TSS), %eax             /* TSS segment selector */
 506         movb    $(K_TSS), 5(%rdx, %rax)         /* clear busy flag */
 507
 508         ltr     saved_tr(%rip)                  /* restore TR */
 509
 510 wake_restore:
 511         mov     saved_rsp(%rip), %rsp
 512
 513         /* restore general purpose registers */
 514         pop %r15
 515         pop %r14
 516         pop %r13
 517         pop %r12
 518         pop %r11
 519         pop %r10
 520         pop %r9
 521         pop %r8
 522         pop %rdi
 523         pop %rsi
 524         pop %rbp
 525         pop %rdx
 526         pop %rcx
 527         pop %rbx
 528         pop %rax
 529
 530         /* restore flags */
 531         popf
 532
 533         leave
 534         ret
 535
 536 /* END ACPI WAKEUP CODE */
 537 #endif /* CONFIG_SLEEP */
 538
 539 /* Code to get from real mode to protected mode */
 540
 541 #define operand_size_prefix     .byte 0x66
 542 #define address_size_prefix     .byte 0x67
 543 #define cs_base_prefix          .byte 0x2e
 544
 545 #define LJMP(segment,address)                   \
 546         operand_size_prefix                     ;\
 547         .byte   0xea                            ;\
 548         .long   address-EXT(real_mode_bootstrap_base)   ;\
 549         .word   segment
 550
 551 #define LGDT(address)                           \
 552         cs_base_prefix                          ;\
 553         address_size_prefix                     ;\
 554         operand_size_prefix                     ;\
 555         .word   0x010f                          ;\
 556         .byte   0x15                            ;\
 557         .long   address-EXT(real_mode_bootstrap_base)
 558
 559 .section __HIB, __text
 560 .align  12      /* Page align for single bcopy_phys() */
 561 .code32
 562 Entry(real_mode_bootstrap_base)
 563         cli
 564
 565         LGDT(EXT(protected_mode_gdtr))
 566
 567         /* set the PE bit of CR0 */
 568         mov     %cr0, %eax
 569         inc %eax
 570         mov     %eax, %cr0
 571
 572         /* reload CS register */
 573         LJMP(KERNEL32_CS, 1f + REAL_MODE_BOOTSTRAP_OFFSET)
 574 1:
 575
 576         /* we are in protected mode now */
 577         /* set up the segment registers */
 578         mov     $KERNEL_DS, %eax
 579         movw    %ax, %ds
 580         movw    %ax, %es
 581         movw    %ax, %ss
 582         xor     %eax,%eax
 583         movw    %ax, %fs
 584         movw    %ax, %gs
 585
 586         POSTCODE(SLAVE_STARTPROG_ENTRY);
 587
 588         mov     PROT_MODE_START+REAL_MODE_BOOTSTRAP_OFFSET, %ecx
 589         jmp     *%ecx
 590
 591 Entry(protected_mode_gdtr)
 592         .short  160             /* limit (8*20 segs) */
 593         .quad   EXT(master_gdt)
 594
 595 Entry(real_mode_bootstrap_end)
 596
 597 /* Save area used across sleep/wake */
 598 .section __HIB, __data
 599 .align  2
 600
 601 /* gdtr for real address of master_gdt in HIB (not the aliased address) */
 602 Entry(master_gdtr)
 603                 .word 160               /* limit (8*20 segs) */
 604                 .quad EXT(master_gdt)
 605
 606 saved_gdt:      .word 0
 607                 .quad 0
 608 saved_rsp:      .quad 0
 609 saved_es:       .word 0
 610 saved_fs:       .word 0
 611 saved_gs:       .word 0
 612 saved_ss:       .word 0
 613 saved_cr0:      .quad 0
 614 saved_cr2:      .quad 0
 615 saved_cr3:      .quad 0
 616 saved_cr4:      .quad 0
 617 saved_idt:      .word 0
 618                 .quad 0
 619 saved_ldt:      .word 0
 620 saved_tr:       .word 0
 621 saved_kgs_base: .quad 0
 622 saved_ugs_base: .quad 0
 623