osfmk/x86_64/start.s

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*
  29  * @OSF_COPYRIGHT@
  30  */
  31 /*
  32  * Mach Operating System
  33  * Copyright (c) 1991,1990 Carnegie Mellon University
  34  * All Rights Reserved.
  35  *
  36  * Permission to use, copy, modify and distribute this software and its
  37  * documentation is hereby granted, provided that both the copyright
  38  * notice and this permission notice appear in all copies of the
  39  * software, derivative works or modified versions, and any portions
  40  * thereof, and that both notices appear in supporting documentation.
  41  *
  42  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  43  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  44  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  45  *
  46  * Carnegie Mellon requests users of this software to return to
  47  *
  48  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  49  *  School of Computer Science
  50  *  Carnegie Mellon University
  51  *  Pittsburgh PA 15213-3890
  52  *
  53  * any improvements or extensions that they make and grant Carnegie Mellon
  54  * the rights to redistribute these changes.
  55  */
  56 /*
  57  */
  58
  59 #include <platforms.h>
  60 #include <debug.h>
  61
  62 #include <i386/asm.h>
  63 #include <i386/proc_reg.h>
  64 #include <i386/postcode.h>
  65 #include <assym.s>
  66
  67 #include <i386/mp.h>
  68 #include <i386/cpuid.h>
  69 #include <i386/acpi.h>
  70
  71 .code32
  72
  73
  74 /*
  75  * Interrupt and bootup stack for initial processor.
  76  * Note: we switch to a dynamically allocated interrupt stack once VM is up.
  77  */
  78
  79 /* in the __HIB section since the hibernate restore code uses this stack. */
  80         .section __HIB, __data
  81         .align  12
  82
  83         .globl  EXT(low_intstack)
  84 EXT(low_intstack):
  85         .globl  EXT(gIOHibernateRestoreStack)
  86 EXT(gIOHibernateRestoreStack):
  87
  88         .space  INTSTACK_SIZE
  89
  90         .globl  EXT(low_eintstack)
  91 EXT(low_eintstack:)
  92         .globl  EXT(gIOHibernateRestoreStackEnd)
  93 EXT(gIOHibernateRestoreStackEnd):
  94
  95         /* back to the regular __DATA section. */
  96
  97         .section __DATA, __data
  98
  99 /*
 100  * Stack for machine-check handler.
 101  */
 102         .align  12
 103         .globl  EXT(mc_task_stack)
 104 EXT(mc_task_stack):
 105         .space  INTSTACK_SIZE
 106         .globl  EXT(mc_task_stack_end)
 107 EXT(mc_task_stack_end):
 108
 109         /* Must not clobber EDI */
 110 #define SWITCH_TO_64BIT_MODE                                     \
 111         movl    $(CR4_PAE),%eax         /* enable PAE */        ;\
 112         movl    %eax,%cr4                                       ;\
 113         movl    $MSR_IA32_EFER,%ecx                             ;\
 114         rdmsr                                                   ;\
 115         /* enable long mode, NX */                              ;\
 116         orl     $(MSR_IA32_EFER_LME | MSR_IA32_EFER_NXE),%eax   ;\
 117         wrmsr                                                   ;\
 118         movl    $EXT(BootPML4),%eax                             ;\
 119         movl    %eax,%cr3                                       ;\
 120         movl    %cr0,%eax                                       ;\
 121         orl     $(CR0_PG|CR0_WP),%eax   /* enable paging */     ;\
 122         movl    %eax,%cr0                                       ;\
 123         ljmpl   $KERNEL64_CS,$64f                               ;\
 124 64:                                                             ;\
 125         .code64
 126
 127 /*
 128  * BSP CPU start here.
 129  *      eax points to kernbootstruct
 130  *
 131  * Environment:
 132  *      protected mode, no paging, flat 32-bit address space.
 133  *      (Code/data/stack segments have base == 0, limit == 4G)
 134  */
 135
 136 .code32
 137         .text
 138         .section __HIB, __text
 139         .align  ALIGN
 140         .globl  EXT(_start)
 141         .globl  EXT(pstart)
 142 LEXT(_start)
 143 LEXT(pstart)
 144
 145 /*
 146  * Here we do the minimal setup to switch from 32 bit mode to 64 bit long mode.
 147  *
 148  * Initial memory layout:
 149  *
 150  *      -------------------------
 151  *      |                       |
 152  *      | Kernel text/data      |
 153  *      |                       |
 154  *      |-----------------------| Kernel text base addr - 2MB-aligned
 155  *      | padding               |
 156  *      |-----------------------|
 157  *      | __HIB section         |
 158  *      |-----------------------| Page-aligned
 159  *      |                       |
 160  *      | padding               |
 161  *      |                       |
 162  *      ------------------------- 0
 163  *
 164  */
 165         mov     %eax, %edi      /* save kernbootstruct */
 166
 167         /* Use low 32-bits of address as 32-bit stack */
 168         movl    $EXT(low_eintstack), %esp
 169
 170         POSTCODE(PSTART_ENTRY)
 171
 172         /*
 173          * Set up segmentation
 174          */
 175         movl    $EXT(protected_mode_gdtr), %eax
 176         lgdtl   (%eax)
 177
 178         /*
 179          * Rebase Boot page tables to kernel base address.
 180          */
 181         movl    $EXT(BootPML4), %eax                    // Level 4:
 182         add     %eax, 0*8+0(%eax)                       //  - 1:1
 183         add     %eax, KERNEL_PML4_INDEX*8+0(%eax)       //  - kernel space
 184
 185         movl    $EXT(BootPDPT), %edx                    // Level 3:
 186         add     %eax, 0*8+0(%edx)
 187         add     %eax, 1*8+0(%edx)
 188         add     %eax, 2*8+0(%edx)
 189         add     %eax, 3*8+0(%edx)
 190
 191         POSTCODE(PSTART_REBASE)
 192
 193 /* the following code is shared by the master CPU and all slave CPUs */
 194 L_pstart_common:
 195         /*
 196          * switch to 64 bit mode
 197          */
 198         SWITCH_TO_64BIT_MODE
 199
 200         /* Flush data segment selectors */
 201         xor     %eax, %eax
 202         mov     %ax, %ss
 203         mov     %ax, %ds
 204         mov     %ax, %es
 205         mov     %ax, %fs
 206         mov     %ax, %gs
 207
 208         test    %edi, %edi /* Populate stack canary on BSP */
 209         jz      Lvstartshim
 210
 211         mov     $1, %eax
 212         cpuid
 213         test    $(1 << 30), %ecx
 214         jz      Lnon_rdrand
 215         RDRAND_RAX              /* RAX := 64 bits of DRBG entropy */
 216         jnc     Lnon_rdrand     /* TODO: complain if DRBG fails at this stage */
 217
 218 Lstore_random_guard:
 219         xor     %ah, %ah        /* Security: zero second byte of stack canary */
 220         movq    %rax, ___stack_chk_guard(%rip)
 221         /* %edi = boot_args_start if BSP */
 222 Lvstartshim:
 223
 224         POSTCODE(PSTART_VSTART)
 225
 226         /* %edi = boot_args_start */
 227
 228         leaq    _vstart(%rip), %rcx
 229         movq    $0xffffff8000000000, %rax       /* adjust pointer up high */
 230         or      %rax, %rsp                      /* and stack pointer up there */
 231         or      %rcx, %rax
 232         andq    $0xfffffffffffffff0, %rsp       /* align stack */
 233         xorq    %rbp, %rbp                      /* zero frame pointer */
 234         callq   *%rax
 235
 236 Lnon_rdrand:
 237         rdtsc /* EDX:EAX := TSC */
 238         /* Distribute low order bits */
 239         mov     %eax, %ecx
 240         xor     %al, %ah
 241         shl     $16, %rcx
 242         xor     %rcx, %rax
 243         xor     %eax, %edx
 244
 245         /* Incorporate ASLR entropy, if any */
 246         lea     (%rip), %rcx
 247         shr     $21, %rcx
 248         movzbl  %cl, %ecx
 249         shl     $16, %ecx
 250         xor     %ecx, %edx
 251
 252         mov     %ah, %cl
 253         ror     %cl, %edx /* Right rotate EDX (TSC&0xFF ^ (TSC>>8 & 0xFF))&1F */
 254         shl     $32, %rdx
 255         xor     %rdx, %rax
 256         mov     %cl, %al
 257         jmp     Lstore_random_guard
 258 /*
 259  * AP (slave) CPUs enter here.
 260  *
 261  * Environment:
 262  *      protected mode, no paging, flat 32-bit address space.
 263  *      (Code/data/stack segments have base == 0, limit == 4G)
 264  */
 265         .align  ALIGN
 266         .globl  EXT(slave_pstart)
 267 LEXT(slave_pstart)
 268         .code32
 269         cli                             /* disable interrupts, so we don`t */
 270                                         /* need IDT for a while */
 271         POSTCODE(SLAVE_PSTART)
 272
 273         movl    $EXT(mp_slave_stack) + PAGE_SIZE, %esp
 274
 275         xor     %edi, %edi              /* AP, no "kernbootstruct" */
 276
 277         jmp     L_pstart_common         /* hop a ride to vstart() */
 278
 279
 280 /* BEGIN HIBERNATE CODE */
 281
 282 .section __HIB, __text
 283 /*
 284  * This code is linked into the kernel but part of the "__HIB" section,
 285  * which means it's used by code running in the special context of restoring
 286  * the kernel text and data from the hibernation image read by the booter.
 287  * hibernate_kernel_entrypoint() and everything it calls or references
 288  * (ie. hibernate_restore_phys_page()) needs to be careful to only touch
 289  * memory also in the "__HIB" section.
 290  */
 291
 292         .align  ALIGN
 293         .globl  EXT(hibernate_machine_entrypoint)
 294 .code32
 295 LEXT(hibernate_machine_entrypoint)
 296         movl    %eax, %edi /* regparm(1) calling convention */
 297
 298         /* Use low 32-bits of address as 32-bit stack */
 299         movl $EXT(low_eintstack), %esp
 300
 301         /*
 302          * Set up GDT
 303          */
 304         movl    $EXT(master_gdtr), %eax
 305         lgdtl   (%eax)
 306
 307         /* Switch to 64-bit on the Boot PTs */
 308         SWITCH_TO_64BIT_MODE
 309
 310         leaq    EXT(hibernate_kernel_entrypoint)(%rip),%rcx
 311
 312         /* adjust the pointers to be up high */
 313         movq    $0xffffff8000000000, %rax
 314         orq     %rax, %rsp
 315         orq     %rcx, %rax
 316
 317         /* %edi is already filled with header pointer */
 318         xorl    %esi, %esi                      /* zero 2nd arg */
 319         xorl    %edx, %edx                      /* zero 3rd arg */
 320         xorl    %ecx, %ecx                      /* zero 4th arg */
 321         andq    $0xfffffffffffffff0, %rsp       /* align stack */
 322
 323         /* call instead of jmp to keep the required stack alignment */
 324         xorq    %rbp, %rbp                      /* zero frame pointer */
 325         call    *%rax
 326
 327         /* NOTREACHED */
 328         hlt
 329
 330 /* END HIBERNATE CODE */
 331
 332 #if CONFIG_SLEEP
 333 /* BEGIN ACPI WAKEUP CODE */
 334
 335 #include <i386/acpi.h>
 336
 337
 338 /*
 339  * acpi_wake_start
 340  */
 341
 342 .section __TEXT,__text
 343 .code64
 344
 345 /*
 346  * acpi_sleep_cpu(acpi_sleep_callback func, void * refcon)
 347  *
 348  * Save CPU state before platform sleep. Restore CPU state
 349  * following wake up.
 350  */
 351
 352 ENTRY(acpi_sleep_cpu)
 353         push    %rbp
 354         mov     %rsp, %rbp
 355
 356         /* save flags */
 357         pushf
 358
 359         /* save general purpose registers */
 360         push %rax
 361         push %rbx
 362         push %rcx
 363         push %rdx
 364         push %rbp
 365         push %rsi
 366         push %rdi
 367         push %r8
 368         push %r9
 369         push %r10
 370         push %r11
 371         push %r12
 372         push %r13
 373         push %r14
 374         push %r15
 375
 376         mov     %rsp, saved_rsp(%rip)
 377
 378         /* make sure tlb is flushed */
 379         mov     %cr3,%rax
 380         mov     %rax,%cr3
 381
 382         /* save control registers */
 383         mov     %cr0, %rax
 384         mov     %rax, saved_cr0(%rip)
 385         mov     %cr2, %rax
 386         mov     %rax, saved_cr2(%rip)
 387         mov     %cr3, %rax
 388         mov     %rax, saved_cr3(%rip)
 389         mov     %cr4, %rax
 390         mov     %rax, saved_cr4(%rip)
 391
 392         /* save segment registers */
 393         movw    %es, saved_es(%rip)
 394         movw    %fs, saved_fs(%rip)
 395         movw    %gs, saved_gs(%rip)
 396         movw    %ss, saved_ss(%rip)
 397
 398         /* save the 64bit user and kernel gs base */
 399         /* note: user's curently swapped into kernel base MSR */
 400         mov     $MSR_IA32_KERNEL_GS_BASE, %rcx
 401         rdmsr
 402         movl    %eax, saved_ugs_base(%rip)
 403         movl    %edx, saved_ugs_base+4(%rip)
 404         swapgs
 405         rdmsr
 406         movl    %eax, saved_kgs_base(%rip)
 407         movl    %edx, saved_kgs_base+4(%rip)
 408         swapgs
 409
 410         /* save descriptor table registers */
 411         sgdt    saved_gdt(%rip)
 412         sldt    saved_ldt(%rip)
 413         sidt    saved_idt(%rip)
 414         str     saved_tr(%rip)
 415
 416         /*
 417          * Call ACPI function provided by the caller to sleep the platform.
 418          * This call will not return on success.
 419          */
 420
 421         xchgq %rdi, %rsi
 422         call    *%rsi
 423
 424         /* sleep failed, no cpu context lost */
 425         jmp     wake_restore
 426
 427 .section __HIB, __text
 428 .code32
 429 .globl EXT(acpi_wake_prot)
 430 EXT(acpi_wake_prot):
 431         /* protected mode, paging disabled */
 432         movl    $EXT(low_eintstack), %esp
 433
 434         SWITCH_TO_64BIT_MODE
 435
 436         jmp     Lwake_64
 437
 438 .section __TEXT,__text
 439 .code64
 440
 441 .globl EXT(acpi_wake_prot_entry)
 442 EXT(acpi_wake_prot_entry):
 443         POSTCODE(ACPI_WAKE_PROT_ENTRY)
 444         /* Return from hibernate code in iokit/Kernel/IOHibernateRestoreKernel.c
 445          */
 446 Lwake_64:
 447         /*
 448          * restore cr4, PAE and NXE states in an orderly fashion
 449          */
 450         mov     saved_cr4(%rip), %rcx
 451         mov     %rcx, %cr4
 452
 453         mov     $(MSR_IA32_EFER), %ecx          /* MSR number in ecx */
 454         rdmsr                                   /* MSR value in edx:eax */
 455         or      $(MSR_IA32_EFER_NXE), %eax      /* Set NXE bit in low 32-bits */
 456         wrmsr                                   /* Update */
 457
 458         movq    saved_cr2(%rip), %rax
 459         mov     %rax, %cr2
 460
 461         /* restore CR0, paging enabled */
 462         mov     saved_cr0(%rip), %rax
 463         mov     %rax, %cr0
 464
 465         /* restore the page tables */
 466         mov     saved_cr3(%rip), %rax
 467         mov     %rax, %cr3
 468
 469         /* protected mode, paging enabled */
 470         POSTCODE(ACPI_WAKE_PAGED_ENTRY)
 471
 472         /* load null segment selectors */
 473         xor     %eax, %eax
 474         movw    %ax, %ss
 475         movw    %ax, %ds
 476
 477         /* restore descriptor tables */
 478         lgdt    saved_gdt(%rip)
 479         lldt    saved_ldt(%rip)
 480         lidt    saved_idt(%rip)
 481
 482         /* restore segment registers */
 483         movw    saved_es(%rip), %es
 484         movw    saved_fs(%rip), %fs
 485         movw    saved_gs(%rip), %gs
 486         movw    saved_ss(%rip), %ss
 487
 488         /* restore the 64bit kernel and user gs base */
 489         mov     $MSR_IA32_KERNEL_GS_BASE, %rcx
 490         movl    saved_kgs_base(%rip),   %eax
 491         movl    saved_kgs_base+4(%rip), %edx
 492         wrmsr
 493         swapgs
 494         movl    saved_ugs_base(%rip),   %eax
 495         movl    saved_ugs_base+4(%rip), %edx
 496         wrmsr
 497
 498         /*
 499          * Restore task register. Before doing this, clear the busy flag
 500          * in the TSS descriptor set by the CPU.
 501          */
 502         lea     saved_gdt(%rip), %rax
 503         movq    2(%rax), %rdx                   /* GDT base, skip limit word */
 504         movl    $(KERNEL_TSS), %eax             /* TSS segment selector */
 505         movb    $(K_TSS), 5(%rdx, %rax)         /* clear busy flag */
 506
 507         ltr     saved_tr(%rip)                  /* restore TR */
 508
 509 wake_restore:
 510         mov     saved_rsp(%rip), %rsp
 511
 512         /* restore general purpose registers */
 513         pop %r15
 514         pop %r14
 515         pop %r13
 516         pop %r12
 517         pop %r11
 518         pop %r10
 519         pop %r9
 520         pop %r8
 521         pop %rdi
 522         pop %rsi
 523         pop %rbp
 524         pop %rdx
 525         pop %rcx
 526         pop %rbx
 527         pop %rax
 528
 529         /* restore flags */
 530         popf
 531
 532         leave
 533         ret
 534
 535 /* END ACPI WAKEUP CODE */
 536 #endif /* CONFIG_SLEEP */
 537
 538 /* Code to get from real mode to protected mode */
 539
 540 #define operand_size_prefix     .byte 0x66
 541 #define address_size_prefix     .byte 0x67
 542 #define cs_base_prefix          .byte 0x2e
 543
 544 #define LJMP(segment,address)                   \
 545         operand_size_prefix                     ;\
 546         .byte   0xea                            ;\
 547         .long   address-EXT(real_mode_bootstrap_base)   ;\
 548         .word   segment
 549
 550 #define LGDT(address)                           \
 551         cs_base_prefix                          ;\
 552         address_size_prefix                     ;\
 553         operand_size_prefix                     ;\
 554         .word   0x010f                          ;\
 555         .byte   0x15                            ;\
 556         .long   address-EXT(real_mode_bootstrap_base)
 557
 558 .section __HIB, __text
 559 .align  12      /* Page align for single bcopy_phys() */
 560 .code32
 561 Entry(real_mode_bootstrap_base)
 562         cli
 563
 564         LGDT(EXT(protected_mode_gdtr))
 565
 566         /* set the PE bit of CR0 */
 567         mov     %cr0, %eax
 568         inc %eax
 569         mov     %eax, %cr0
 570
 571         /* reload CS register */
 572         LJMP(KERNEL32_CS, 1f + REAL_MODE_BOOTSTRAP_OFFSET)
 573 1:
 574
 575         /* we are in protected mode now */
 576         /* set up the segment registers */
 577         mov     $KERNEL_DS, %eax
 578         movw    %ax, %ds
 579         movw    %ax, %es
 580         movw    %ax, %ss
 581         xor     %eax,%eax
 582         movw    %ax, %fs
 583         movw    %ax, %gs
 584
 585         POSTCODE(SLAVE_STARTPROG_ENTRY);
 586
 587         mov     PROT_MODE_START+REAL_MODE_BOOTSTRAP_OFFSET, %ecx
 588         jmp     *%ecx
 589
 590 Entry(protected_mode_gdtr)
 591         .short  160             /* limit (8*20 segs) */
 592         .quad   EXT(master_gdt)
 593
 594 Entry(real_mode_bootstrap_end)
 595
 596 /* Save area used across sleep/wake */
 597 .section __HIB, __data
 598 .align  2
 599
 600 /* gdtr for real address of master_gdt in HIB (not the aliased address) */
 601 Entry(master_gdtr)
 602                 .word 160               /* limit (8*20 segs) */
 603                 .quad EXT(master_gdt)
 604
 605 saved_gdt:      .word 0
 606                 .quad 0
 607 saved_rsp:      .quad 0
 608 saved_es:       .word 0
 609 saved_fs:       .word 0
 610 saved_gs:       .word 0
 611 saved_ss:       .word 0
 612 saved_cr0:      .quad 0
 613 saved_cr2:      .quad 0
 614 saved_cr3:      .quad 0
 615 saved_cr4:      .quad 0
 616 saved_idt:      .word 0
 617                 .quad 0
 618 saved_ldt:      .word 0
 619 saved_tr:       .word 0
 620 saved_kgs_base: .quad 0
 621 saved_ugs_base: .quad 0
 622