2 * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
32 * Mach Operating System
33 * Copyright (c) 1991,1990 Carnegie Mellon University
34 * All Rights Reserved.
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
46 * Carnegie Mellon requests users of this software to return to
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
60 #include <kern/cpu_number.h>
61 #include <kern/kalloc.h>
62 #include <kern/cpu_data.h>
63 #include <mach/mach_types.h>
64 #include <mach/machine.h>
65 #include <kern/etimer.h>
66 #include <mach/vm_map.h>
67 #include <mach/machine/vm_param.h>
68 #include <vm/vm_kern.h>
69 #include <vm/vm_map.h>
71 #include <i386/lock.h>
72 #include <i386/mp_desc.h>
73 #include <i386/misc_protos.h>
75 #include <i386/pmap.h>
77 #include <i386/machine_check.h>
80 #include <kern/misc_protos.h>
86 #define K_INTR_GATE (ACC_P|ACC_PL_K|ACC_INTR_GATE)
87 #define U_INTR_GATE (ACC_P|ACC_PL_U|ACC_INTR_GATE)
89 // Declare macros that will declare the externs
90 #define TRAP(n, name) extern void *name ;
91 #define TRAP_ERR(n, name) extern void *name ;
92 #define TRAP_SPC(n, name) extern void *name ;
93 #define TRAP_IST(n, name) extern void *name ;
94 #define INTERRUPT(n) extern void *_intr_ ## n ;
95 #define USER_TRAP(n, name) extern void *name ;
96 #define USER_TRAP_SPC(n, name) extern void *name ;
98 // Include the table to declare the externs
99 #include "../x86_64/idt_table.h"
101 // Undef the macros, then redefine them so we can declare the table
110 #define TRAP(n, name) \
119 #define TRAP_ERR TRAP
120 #define TRAP_SPC TRAP
122 #define TRAP_IST(n, name) \
131 #define INTERRUPT(n) \
133 (uintptr_t)&_intr_ ## n,\
140 #define USER_TRAP(n, name) \
149 #define USER_TRAP_SPC USER_TRAP
152 // Declare the table using the macros we just set up
153 struct fake_descriptor64 master_idt64
[IDTSZ
] __attribute__ ((aligned (4096))) = {
154 #include "../x86_64/idt_table.h"
159 * The i386 needs an interrupt stack to keep the PCB stack from being
160 * overrun by interrupts. All interrupt stacks MUST lie at lower addresses
161 * than any thread`s kernel stack.
165 * First cpu`s interrupt stack.
167 extern uint32_t low_intstack
[]; /* bottom */
168 extern uint32_t low_eintstack
[]; /* top */
171 * Per-cpu data area pointers.
172 * The master cpu (cpu 0) has its data area statically allocated;
173 * others are allocated dynamically and this array is updated at runtime.
175 cpu_data_t cpu_data_master
= {
176 .cpu_this
= &cpu_data_master
,
177 .cpu_nanotime
= &rtc_nanotime_info
,
178 .cpu_int_stack_top
= (vm_offset_t
) low_eintstack
,
180 .cpu_is64bit
= FALSE
,
185 cpu_data_t
*cpu_data_ptr
[MAX_CPUS
] = { [0] &cpu_data_master
};
187 decl_simple_lock_data(,ncpus_lock
); /* protects real_ncpus */
188 unsigned int real_ncpus
= 1;
189 unsigned int max_ncpus
= MAX_CPUS
;
192 extern void *hi_remap_text
;
193 #define HI_TEXT(lo_text) \
194 (((uint32_t)&lo_text - (uint32_t)&hi_remap_text) + HIGH_MEM_BASE)
196 extern void hi_sysenter(void);
201 } __attribute__((__packed__
)) table_descriptor64_t
;
203 extern table_descriptor64_t gdtptr64
;
204 extern table_descriptor64_t idtptr64
;
206 extern void hi64_sysenter(void);
207 extern void hi64_syscall(void);
209 #if defined(__x86_64__) && !defined(UBER64)
210 #define UBER64(x) ((uintptr_t)x)
214 * Multiprocessor i386/i486 systems use a separate copy of the
215 * GDT, IDT, LDT, and kernel TSS per processor. The first three
216 * are separate to avoid lock contention: the i386 uses locked
217 * memory cycles to access the descriptor tables. The TSS is
218 * separate since each processor needs its own kernel stack,
219 * and since using a TSS marks it busy.
223 * Allocate and initialize the per-processor descriptor tables.
226 struct fake_descriptor ldt_desc_pattern
= {
228 LDTSZ_MIN
* sizeof(struct fake_descriptor
) - 1,
230 ACC_P
|ACC_PL_K
|ACC_LDT
233 struct fake_descriptor tss_desc_pattern
= {
235 sizeof(struct i386_tss
) - 1,
237 ACC_P
|ACC_PL_K
|ACC_TSS
240 struct fake_descriptor cpudata_desc_pattern
= {
242 sizeof(cpu_data_t
)-1,
244 ACC_P
|ACC_PL_K
|ACC_DATA_W
247 struct fake_descriptor userwindow_desc_pattern
= {
249 ((NBPDE
* NCOPY_WINDOWS
) / PAGE_SIZE
) - 1,
251 ACC_P
|ACC_PL_U
|ACC_DATA_W
254 struct fake_descriptor physwindow_desc_pattern
= {
258 ACC_P
|ACC_PL_K
|ACC_DATA_W
262 * This is the expanded, 64-bit variant of the kernel LDT descriptor.
263 * When switching to 64-bit mode this replaces KERNEL_LDT entry
264 * and the following empty slot. This enables the LDT to be referenced
265 * in the uber-space remapping window on the kernel.
267 struct fake_descriptor64 kernel_ldt_desc64
= {
269 LDTSZ_MIN
*sizeof(struct fake_descriptor
)-1,
271 ACC_P
|ACC_PL_K
|ACC_LDT
,
276 * This is the expanded, 64-bit variant of the kernel TSS descriptor.
277 * It is follows pattern of the KERNEL_LDT.
279 struct fake_descriptor64 kernel_tss_desc64
= {
281 sizeof(struct x86_64_tss
)-1,
283 ACC_P
|ACC_PL_K
|ACC_TSS
,
288 * Convert a descriptor from fake to real format.
290 * Fake descriptor format:
291 * bytes 0..3 base 31..0
292 * bytes 4..5 limit 15..0
293 * byte 6 access byte 2 | limit 19..16
294 * byte 7 access byte 1
296 * Real descriptor format:
297 * bytes 0..1 limit 15..0
298 * bytes 2..3 base 15..0
300 * byte 5 access byte 1
301 * byte 6 access byte 2 | limit 19..16
306 * bytes 4..5 selector
307 * byte 6 word count << 4 (to match fake descriptor)
308 * byte 7 access byte 1
311 * bytes 0..1 offset 15..0
312 * bytes 2..3 selector
314 * byte 5 access byte 1
315 * bytes 6..7 offset 31..16
318 fix_desc(void *d
, int num_desc
) {
319 //early_kprintf("fix_desc(%x, %x)\n", d, num_desc);
320 uint8_t *desc
= (uint8_t*) d
;
323 if ((desc
[7] & 0x14) == 0x04) { /* gate */
329 offset
= *((uint32_t*)(desc
));
330 selector
= *((uint32_t*)(desc
+4));
331 wordcount
= desc
[6] >> 4;
334 *((uint16_t*)desc
) = offset
& 0xFFFF;
335 *((uint16_t*)(desc
+2)) = selector
;
338 *((uint16_t*)(desc
+6)) = offset
>> 16;
340 } else { /* descriptor */
345 base
= *((uint32_t*)(desc
));
346 limit
= *((uint16_t*)(desc
+4));
350 *((uint16_t*)(desc
)) = limit
;
351 *((uint16_t*)(desc
+2)) = base
& 0xFFFF;
352 desc
[4] = (base
>> 16) & 0xFF;
355 desc
[7] = base
>> 24;
358 } while (--num_desc
);
362 fix_desc64(void *descp
, int count
)
364 struct fake_descriptor64
*fakep
;
366 struct real_gate64 gate
;
367 struct real_descriptor64 desc
;
371 fakep
= (struct fake_descriptor64
*) descp
;
373 for (i
= 0; i
< count
; i
++, fakep
++) {
375 * Construct the real decriptor locally.
378 bzero((void *) &real
, sizeof(real
));
380 switch (fakep
->access
& ACC_TYPE
) {
386 real
.gate
.offset_low16
= fakep
->offset64
& 0xFFFF;
387 real
.gate
.selector16
= fakep
->lim_or_seg
& 0xFFFF;
388 real
.gate
.IST
= fakep
->size_or_IST
& 0x7;
389 real
.gate
.access8
= fakep
->access
;
390 real
.gate
.offset_high16
= (fakep
->offset64
>>16)&0xFFFF;
391 real
.gate
.offset_top32
= (uint32_t)(fakep
->offset64
>>32);
393 default: /* Otherwise */
394 real
.desc
.limit_low16
= fakep
->lim_or_seg
& 0xFFFF;
395 real
.desc
.base_low16
= fakep
->offset64
& 0xFFFF;
396 real
.desc
.base_med8
= (fakep
->offset64
>> 16) & 0xFF;
397 real
.desc
.access8
= fakep
->access
;
398 real
.desc
.limit_high4
= (fakep
->lim_or_seg
>> 16) & 0xFF;
399 real
.desc
.granularity4
= fakep
->size_or_IST
;
400 real
.desc
.base_high8
= (fakep
->offset64
>> 24) & 0xFF;
401 real
.desc
.base_top32
= (uint32_t)(fakep
->offset64
>>32);
405 * Now copy back over the fake structure.
407 bcopy((void *) &real
, (void *) fakep
, sizeof(real
));
413 cpu_desc_init(cpu_data_t
*cdp
)
415 cpu_desc_index_t
*cdi
= &cdp
->cpu_desc_index
;
417 if (cdp
== &cpu_data_master
) {
419 * Fix up the entries in the GDT to point to
420 * this LDT and this TSS.
422 struct fake_descriptor temp_fake_desc
;
423 temp_fake_desc
= ldt_desc_pattern
;
424 temp_fake_desc
.offset
= (vm_offset_t
) &master_ldt
;
425 fix_desc(&temp_fake_desc
, 1);
426 *(struct fake_descriptor
*) &master_gdt
[sel_idx(KERNEL_LDT
)] =
428 *(struct fake_descriptor
*) &master_gdt
[sel_idx(USER_LDT
)] =
431 temp_fake_desc
= tss_desc_pattern
;
432 temp_fake_desc
.offset
= (vm_offset_t
) &master_ktss
;
433 fix_desc(&temp_fake_desc
, 1);
434 *(struct fake_descriptor
*) &master_gdt
[sel_idx(KERNEL_TSS
)] =
438 temp_fake_desc
= tss_desc_pattern
;
439 temp_fake_desc
.offset
= (vm_offset_t
) &master_dbtss
;
440 fix_desc(&temp_fake_desc
, 1);
441 *(struct fake_descriptor
*) &master_gdt
[sel_idx(DEBUG_TSS
)] =
445 temp_fake_desc
= cpudata_desc_pattern
;
446 temp_fake_desc
.offset
= (vm_offset_t
) &cpu_data_master
;
447 fix_desc(&temp_fake_desc
, 1);
448 *(struct fake_descriptor
*) &master_gdt
[sel_idx(CPU_DATA_GS
)] =
451 fix_desc((void *)&master_idt
, IDTSZ
);
453 cdi
->cdi_idt
.ptr
= master_idt
;
454 cdi
->cdi_gdt
.ptr
= (void *)master_gdt
;
458 * Master CPU uses the tables built at boot time.
459 * Just set the index pointers to the high shared-mapping space.
460 * Note that the sysenter stack uses empty space above the ktss
461 * in the HIGH_FIXED_KTSS page. In this case we don't map the
462 * the real master_sstk in low memory.
464 cdi
->cdi_ktss
= (struct i386_tss
*)
465 pmap_index_to_virt(HIGH_FIXED_KTSS
) ;
466 cdi
->cdi_sstk
= (vm_offset_t
) (cdi
->cdi_ktss
+ 1) +
467 (vm_offset_t
) &master_sstk
.top
-
468 (vm_offset_t
) &master_sstk
;
470 cpu_desc_table_t
*cdt
= (cpu_desc_table_t
*) cdp
->cpu_desc_tablep
;
472 vm_offset_t cpu_hi_desc
;
474 cpu_hi_desc
= pmap_cpu_high_shared_remap(
477 (vm_offset_t
) cdt
, 1);
480 * Per-cpu GDT, IDT, LDT, KTSS descriptors are allocated in one
481 * block (cpu_desc_table) and double-mapped into high shared space
482 * in one page window.
483 * Also, a transient stack for the fast sysenter path. The top of
484 * which is set at context switch time to point to the PCB using
487 cdi
->cdi_gdt
.ptr
= (struct fake_descriptor
*) (cpu_hi_desc
+
488 offsetof(cpu_desc_table_t
, gdt
[0]));
489 cdi
->cdi_idt
.ptr
= (struct fake_descriptor
*) (cpu_hi_desc
+
490 offsetof(cpu_desc_table_t
, idt
[0]));
491 cdi
->cdi_ktss
= (struct i386_tss
*) (cpu_hi_desc
+
492 offsetof(cpu_desc_table_t
, ktss
));
493 cdi
->cdi_sstk
= cpu_hi_desc
+ offsetof(cpu_desc_table_t
, sstk
.top
);
496 * LDT descriptors are mapped into a seperate area.
498 cdi
->cdi_ldt
= (struct fake_descriptor
*)
499 pmap_cpu_high_shared_remap(
502 (vm_offset_t
) cdp
->cpu_ldtp
,
503 HIGH_CPU_LDT_END
- HIGH_CPU_LDT_BEGIN
+ 1);
508 bcopy((char *)master_idt
, (char *)cdt
->idt
, sizeof(master_idt
));
509 bcopy((char *)master_gdt
, (char *)cdt
->gdt
, sizeof(master_gdt
));
510 bcopy((char *)master_ldt
, (char *)cdp
->cpu_ldtp
, sizeof(master_ldt
));
511 bzero((char *)&cdt
->ktss
, sizeof(struct i386_tss
));
513 cdi
->cdi_dbtss
= (struct i386_tss
*) (cpu_hi_desc
+
514 offsetof(cpu_desc_table_t
, dbtss
));
515 bcopy((char *)&master_dbtss
,
517 sizeof(struct i386_tss
));
518 #endif /* MACH_KDB */
521 * Fix up the entries in the GDT to point to
522 * this LDT and this TSS.
524 struct fake_descriptor temp_ldt
= ldt_desc_pattern
;
525 temp_ldt
.offset
= (vm_offset_t
)cdi
->cdi_ldt
;
526 fix_desc(&temp_ldt
, 1);
528 cdt
->gdt
[sel_idx(KERNEL_LDT
)] = temp_ldt
;
529 cdt
->gdt
[sel_idx(USER_LDT
)] = temp_ldt
;
531 cdt
->gdt
[sel_idx(KERNEL_TSS
)] = tss_desc_pattern
;
532 cdt
->gdt
[sel_idx(KERNEL_TSS
)].offset
= (vm_offset_t
) cdi
->cdi_ktss
;
533 fix_desc(&cdt
->gdt
[sel_idx(KERNEL_TSS
)], 1);
535 cdt
->gdt
[sel_idx(CPU_DATA_GS
)] = cpudata_desc_pattern
;
536 cdt
->gdt
[sel_idx(CPU_DATA_GS
)].offset
= (vm_offset_t
) cdp
;
537 fix_desc(&cdt
->gdt
[sel_idx(CPU_DATA_GS
)], 1);
540 cdt
->gdt
[sel_idx(DEBUG_TSS
)] = tss_desc_pattern
;
541 cdt
->gdt
[sel_idx(DEBUG_TSS
)].offset
= (vm_offset_t
) cdi
->cdi_dbtss
;
542 fix_desc(&cdt
->gdt
[sel_idx(DEBUG_TSS
)], 1);
544 cdt
->dbtss
.esp0
= (int)(db_task_stack_store
+
545 (INTSTACK_SIZE
* (cdp
->cpu_number
)) - sizeof (natural_t
));
546 cdt
->dbtss
.esp
= cdt
->dbtss
.esp0
;
547 cdt
->dbtss
.eip
= (int)&db_task_start
;
548 #endif /* MACH_KDB */
550 cdt
->ktss
.ss0
= KERNEL_DS
;
551 cdt
->ktss
.io_bit_map_offset
= 0x0FFF; /* no IO bitmap */
553 cpu_userwindow_init(cdp
->cpu_number
);
554 cpu_physwindow_init(cdp
->cpu_number
);
558 #endif /* __i386__ */
561 cpu_desc_init64(cpu_data_t
*cdp
)
563 cpu_desc_index_t
*cdi
= &cdp
->cpu_desc_index
;
565 if (cdp
== &cpu_data_master
) {
567 * Master CPU uses the tables built at boot time.
568 * Just set the index pointers to the low memory space.
570 cdi
->cdi_ktss
= (void *)&master_ktss64
;
571 cdi
->cdi_sstk
= (vm_offset_t
) &master_sstk
.top
;
572 cdi
->cdi_gdt
.ptr
= (void *)master_gdt
;
573 cdi
->cdi_idt
.ptr
= (void *)master_idt64
;
574 cdi
->cdi_ldt
= (struct fake_descriptor
*) master_ldt
;
577 /* Replace the expanded LDTs and TSS slots in the GDT */
578 kernel_ldt_desc64
.offset64
= UBER64(&master_ldt
);
579 *(struct fake_descriptor64
*) &master_gdt
[sel_idx(KERNEL_LDT
)] =
581 *(struct fake_descriptor64
*) &master_gdt
[sel_idx(USER_LDT
)] =
583 kernel_tss_desc64
.offset64
= UBER64(&master_ktss64
);
584 *(struct fake_descriptor64
*) &master_gdt
[sel_idx(KERNEL_TSS
)] =
587 /* Fix up the expanded descriptors for 64-bit. */
588 fix_desc64((void *) &master_idt64
, IDTSZ
);
589 fix_desc64((void *) &master_gdt
[sel_idx(KERNEL_LDT
)], 1);
590 fix_desc64((void *) &master_gdt
[sel_idx(USER_LDT
)], 1);
591 fix_desc64((void *) &master_gdt
[sel_idx(KERNEL_TSS
)], 1);
594 * Set the double-fault stack as IST1 in the 64-bit TSS
596 master_ktss64
.ist1
= UBER64((uintptr_t) df_task_stack_end
);
599 cpu_desc_table64_t
*cdt
= (cpu_desc_table64_t
*) cdp
->cpu_desc_tablep
;
601 * Per-cpu GDT, IDT, KTSS descriptors are allocated in kernel
602 * heap (cpu_desc_table).
603 * LDT descriptors are mapped into a separate area.
605 cdi
->cdi_gdt
.ptr
= (struct fake_descriptor
*)cdt
->gdt
;
606 cdi
->cdi_idt
.ptr
= (void *)cdt
->idt
;
607 cdi
->cdi_ktss
= (void *)&cdt
->ktss
;
608 cdi
->cdi_sstk
= (vm_offset_t
)&cdt
->sstk
.top
;
609 cdi
->cdi_ldt
= cdp
->cpu_ldtp
;
614 bcopy((char *)master_idt64
, (char *)cdt
->idt
, sizeof(master_idt64
));
615 bcopy((char *)master_gdt
, (char *)cdt
->gdt
, sizeof(master_gdt
));
616 bcopy((char *)master_ldt
, (char *)cdp
->cpu_ldtp
, sizeof(master_ldt
));
617 bcopy((char *)&master_ktss64
, (char *)&cdt
->ktss
, sizeof(struct x86_64_tss
));
620 * Fix up the entries in the GDT to point to
621 * this LDT and this TSS.
623 kernel_ldt_desc64
.offset64
= UBER64(cdi
->cdi_ldt
);
624 *(struct fake_descriptor64
*) &cdt
->gdt
[sel_idx(KERNEL_LDT
)] =
626 fix_desc64(&cdt
->gdt
[sel_idx(KERNEL_LDT
)], 1);
628 kernel_ldt_desc64
.offset64
= UBER64(cdi
->cdi_ldt
);
629 *(struct fake_descriptor64
*) &cdt
->gdt
[sel_idx(USER_LDT
)] =
631 fix_desc64(&cdt
->gdt
[sel_idx(USER_LDT
)], 1);
633 kernel_tss_desc64
.offset64
= UBER64(cdi
->cdi_ktss
);
634 *(struct fake_descriptor64
*) &cdt
->gdt
[sel_idx(KERNEL_TSS
)] =
636 fix_desc64(&cdt
->gdt
[sel_idx(KERNEL_TSS
)], 1);
638 /* Set double-fault stack as IST1 */
639 cdt
->ktss
.ist1
= UBER64((unsigned long)cdt
->dfstk
+ sizeof(cdt
->dfstk
));
641 cdt
->gdt
[sel_idx(CPU_DATA_GS
)] = cpudata_desc_pattern
;
642 cdt
->gdt
[sel_idx(CPU_DATA_GS
)].offset
= (vm_offset_t
) cdp
;
643 fix_desc(&cdt
->gdt
[sel_idx(CPU_DATA_GS
)], 1);
645 /* Allocate copyio windows */
646 cpu_userwindow_init(cdp
->cpu_number
);
647 cpu_physwindow_init(cdp
->cpu_number
);
651 /* Require that the top of the sysenter stack is 16-byte aligned */
652 if ((cdi
->cdi_sstk
% 16) != 0)
653 panic("cpu_desc_init64() sysenter stack not 16-byte aligned");
658 cpu_desc_load(cpu_data_t
*cdp
)
660 cpu_desc_index_t
*cdi
= &cdp
->cpu_desc_index
;
662 cdi
->cdi_idt
.size
= 0x1000 + cdp
->cpu_number
;
663 cdi
->cdi_gdt
.size
= sizeof(struct real_descriptor
)*GDTSZ
- 1;
665 lgdt((unsigned long *) &cdi
->cdi_gdt
);
666 lidt((unsigned long *) &cdi
->cdi_idt
);
671 __asm__
volatile("mov %0, %%gs" : : "rm" ((unsigned short)(CPU_DATA_GS
)));
673 #endif /* __i386__ */
676 cpu_desc_load64(cpu_data_t
*cdp
)
678 cpu_desc_index_t
*cdi
= &cdp
->cpu_desc_index
;
682 * Load up the new descriptors etc
683 * ml_load_desc64() expects these global pseudo-descriptors:
684 * gdtptr64 -> per-cpu gdt
685 * idtptr64 -> per-cpu idt
686 * These are 10-byte descriptors with 64-bit addresses into
689 * Refer to commpage/cpu_number.s for the IDT limit trick.
691 gdtptr64
.length
= GDTSZ
* sizeof(struct real_descriptor
) - 1;
692 gdtptr64
.offset
[0] = (uint32_t) cdi
->cdi_gdt
.ptr
;
693 gdtptr64
.offset
[1] = KERNEL_UBER_BASE_HI32
;
694 idtptr64
.length
= 0x1000 + cdp
->cpu_number
;
695 idtptr64
.offset
[0] = (uint32_t) cdi
->cdi_idt
.ptr
;
696 idtptr64
.offset
[1] = KERNEL_UBER_BASE_HI32
;
698 /* Make sure busy bit is cleared in the TSS */
699 gdt_desc_p(KERNEL_TSS
)->access
&= ~ACC_TSS_BUSY
;
703 /* Load the GDT, LDT, IDT and TSS */
704 cdi
->cdi_gdt
.size
= sizeof(struct real_descriptor
)*GDTSZ
- 1;
705 cdi
->cdi_idt
.size
= 0x1000 + cdp
->cpu_number
;
706 lgdt((unsigned long *) &cdi
->cdi_gdt
);
707 lidt((unsigned long *) &cdi
->cdi_idt
);
711 /* Stuff the pre-cpu data area into the MSR and swapgs to activate */
712 wrmsr64(MSR_IA32_KERNEL_GS_BASE
, (unsigned long)cdp
);
713 #if GPROF // Hack to enable mcount to work on K64
714 __asm__
volatile("mov %0, %%gs" : : "rm" ((unsigned short)(KERNEL_DS
)));
724 * Set MSRs for sysenter/sysexit for 32-bit.
727 fast_syscall_init(__unused cpu_data_t
*cdp
)
729 wrmsr(MSR_IA32_SYSENTER_CS
, SYSENTER_CS
, 0);
730 wrmsr(MSR_IA32_SYSENTER_EIP
, HI_TEXT(hi_sysenter
), 0);
731 wrmsr(MSR_IA32_SYSENTER_ESP
, current_sstk(), 0);
736 * Set MSRs for sysenter/sysexit and syscall/sysret for 64-bit.
739 fast_syscall_init64(__unused cpu_data_t
*cdp
)
741 wrmsr64(MSR_IA32_SYSENTER_CS
, SYSENTER_CS
);
742 wrmsr64(MSR_IA32_SYSENTER_EIP
, UBER64((uintptr_t) hi64_sysenter
));
743 wrmsr64(MSR_IA32_SYSENTER_ESP
, UBER64(current_sstk()));
744 /* Enable syscall/sysret */
745 wrmsr64(MSR_IA32_EFER
, rdmsr64(MSR_IA32_EFER
) | MSR_IA32_EFER_SCE
);
748 * MSRs for 64-bit syscall/sysret
749 * Note USER_CS because sysret uses this + 16 when returning to
752 wrmsr64(MSR_IA32_LSTAR
, UBER64((uintptr_t) hi64_syscall
));
753 wrmsr64(MSR_IA32_STAR
, (((uint64_t)USER_CS
) << 48) |
754 (((uint64_t)KERNEL64_CS
) << 32));
756 * Emulate eflags cleared by sysenter but note that
757 * we also clear the trace trap to avoid the complications
758 * of single-stepping into a syscall. The nested task bit
759 * is also cleared to avoid a spurious "task switch"
760 * should we choose to return via an IRET.
762 wrmsr64(MSR_IA32_FMASK
, EFL_DF
|EFL_IF
|EFL_TF
|EFL_NT
);
766 * Set the Kernel GS base MSR to point to per-cpu data in uber-space.
767 * The uber-space handler (hi64_syscall) uses the swapgs instruction.
769 wrmsr64(MSR_IA32_KERNEL_GS_BASE
, UBER64(cdp
));
771 #if ONLY_SAFE_FOR_LINDA_SERIAL
772 kprintf("fast_syscall_init64() KERNEL_GS_BASE=0x%016llx\n",
773 rdmsr64(MSR_IA32_KERNEL_GS_BASE
));
779 cpu_data_alloc(boolean_t is_boot_cpu
)
785 assert(real_ncpus
== 1);
786 cdp
= &cpu_data_master
;
787 if (cdp
->cpu_processor
== NULL
) {
788 simple_lock_init(&ncpus_lock
, 0);
789 cdp
->cpu_processor
= cpu_processor_alloc(TRUE
);
790 #if NCOPY_WINDOWS > 0
791 cdp
->cpu_pmap
= pmap_cpu_alloc(TRUE
);
793 queue_init(&cdp
->rtclock_timer
.queue
);
794 cdp
->rtclock_timer
.deadline
= EndOfAllTime
;
800 * Allocate per-cpu data:
802 ret
= kmem_alloc(kernel_map
, (vm_offset_t
*) &cdp
, sizeof(cpu_data_t
));
803 if (ret
!= KERN_SUCCESS
) {
804 printf("cpu_data_alloc() failed, ret=%d\n", ret
);
807 bzero((void*) cdp
, sizeof(cpu_data_t
));
811 cdp
->cpu_is64bit
= cpu_mode_is64bit();
814 * Allocate interrupt stack:
816 ret
= kmem_alloc(kernel_map
,
817 (vm_offset_t
*) &cdp
->cpu_int_stack_top
,
819 if (ret
!= KERN_SUCCESS
) {
820 printf("cpu_data_alloc() int stack failed, ret=%d\n", ret
);
823 bzero((void*) cdp
->cpu_int_stack_top
, INTSTACK_SIZE
);
824 cdp
->cpu_int_stack_top
+= INTSTACK_SIZE
;
828 * Allocate descriptor table:
829 * Size depends on cpu mode.
831 ret
= kmem_alloc(kernel_map
,
832 (vm_offset_t
*) &cdp
->cpu_desc_tablep
,
833 cdp
->cpu_is64bit
? sizeof(cpu_desc_table64_t
)
834 : sizeof(cpu_desc_table_t
));
835 if (ret
!= KERN_SUCCESS
) {
836 printf("cpu_data_alloc() desc_table failed, ret=%d\n", ret
);
843 ret
= kmem_alloc(kernel_map
,
844 (vm_offset_t
*) &cdp
->cpu_ldtp
,
845 sizeof(struct real_descriptor
) * LDTSZ
);
846 if (ret
!= KERN_SUCCESS
) {
847 printf("cpu_data_alloc() ldt failed, ret=%d\n", ret
);
852 /* Machine-check shadow register allocation. */
856 simple_lock(&ncpus_lock
);
858 cpu_data_ptr
[real_ncpus
] = cdp
;
859 cdp
->cpu_number
= real_ncpus
;
861 simple_unlock(&ncpus_lock
);
863 cdp
->cpu_nanotime
= &rtc_nanotime_info
;
864 queue_init(&cdp
->rtclock_timer
.queue
);
865 cdp
->rtclock_timer
.deadline
= EndOfAllTime
;
867 kprintf("cpu_data_alloc(%d) %p desc_table: %p "
869 "int_stack: 0x%lx-0x%lx\n",
870 cdp
->cpu_number
, cdp
, cdp
->cpu_desc_tablep
, cdp
->cpu_ldtp
,
871 (long)(cdp
->cpu_int_stack_top
- INTSTACK_SIZE
), (long)(cdp
->cpu_int_stack_top
));
877 if (cdp
->cpu_desc_tablep
)
878 kfree((void *) cdp
->cpu_desc_tablep
,
879 sizeof(*cdp
->cpu_desc_tablep
));
880 if (cdp
->cpu_int_stack_top
)
881 kfree((void *) (cdp
->cpu_int_stack_top
- INTSTACK_SIZE
),
883 kfree((void *) cdp
, sizeof(*cdp
));
889 valid_user_segment_selectors(uint16_t cs
,
896 return valid_user_code_selector(cs
) &&
897 valid_user_stack_selector(ss
) &&
898 valid_user_data_selector(ds
) &&
899 valid_user_data_selector(es
) &&
900 valid_user_data_selector(fs
) &&
901 valid_user_data_selector(gs
);
905 #if NCOPY_WINDOWS > 0
907 static vm_offset_t user_window_base
= 0;
910 cpu_userwindow_init(int cpu
)
912 cpu_data_t
*cdp
= cpu_data_ptr
[cpu
];
913 vm_offset_t user_window
;
917 num_cpus
= ml_get_max_cpus();
920 panic("cpu_userwindow_init: cpu > num_cpus");
922 if (user_window_base
== 0) {
924 if (vm_allocate(kernel_map
, &vaddr
,
925 (NBPDE
* NCOPY_WINDOWS
* num_cpus
) + NBPDE
,
926 VM_FLAGS_ANYWHERE
) != KERN_SUCCESS
)
927 panic("cpu_userwindow_init: "
928 "couldn't allocate user map window");
931 * window must start on a page table boundary
932 * in the virtual address space
934 user_window_base
= (vaddr
+ (NBPDE
- 1)) & ~(NBPDE
- 1);
937 * get rid of any allocation leading up to our
940 vm_deallocate(kernel_map
, vaddr
, user_window_base
- vaddr
);
943 * get rid of tail that we don't need
945 user_window
= user_window_base
+
946 (NBPDE
* NCOPY_WINDOWS
* num_cpus
);
948 vm_deallocate(kernel_map
, user_window
,
950 ((NBPDE
* NCOPY_WINDOWS
* num_cpus
) + NBPDE
)) -
954 user_window
= user_window_base
+ (cpu
* NCOPY_WINDOWS
* NBPDE
);
956 cdp
->cpu_copywindow_base
= user_window
;
957 cdp
->cpu_copywindow_pdp
= pmap_pde(kernel_pmap
, user_window
);
960 cpu_desc_index_t
*cdi
= &cdp
->cpu_desc_index
;
961 cdi
->cdi_gdt
.ptr
[sel_idx(USER_WINDOW_SEL
)] = userwindow_desc_pattern
;
962 cdi
->cdi_gdt
.ptr
[sel_idx(USER_WINDOW_SEL
)].offset
= user_window
;
964 fix_desc(&cdi
->cdi_gdt
.ptr
[sel_idx(USER_WINDOW_SEL
)], 1);
965 #endif /* __i386__ */
969 cpu_physwindow_init(int cpu
)
971 cpu_data_t
*cdp
= cpu_data_ptr
[cpu
];
972 vm_offset_t phys_window
= cdp
->cpu_physwindow_base
;
974 if (phys_window
== 0) {
975 if (vm_allocate(kernel_map
, &phys_window
,
976 PAGE_SIZE
, VM_FLAGS_ANYWHERE
)
978 panic("cpu_physwindow_init: "
979 "couldn't allocate phys map window");
982 * make sure the page that encompasses the
983 * pte pointer we're interested in actually
984 * exists in the page table
986 pmap_expand(kernel_pmap
, phys_window
);
988 cdp
->cpu_physwindow_base
= phys_window
;
989 cdp
->cpu_physwindow_ptep
= vtopte(phys_window
);
992 cpu_desc_index_t
*cdi
= &cdp
->cpu_desc_index
;
993 cdi
->cdi_gdt
.ptr
[sel_idx(PHYS_WINDOW_SEL
)] = physwindow_desc_pattern
;
994 cdi
->cdi_gdt
.ptr
[sel_idx(PHYS_WINDOW_SEL
)].offset
= phys_window
;
996 fix_desc(&cdi
->cdi_gdt
.ptr
[sel_idx(PHYS_WINDOW_SEL
)], 1);
997 #endif /* __i386__ */
999 #endif /* NCOPY_WINDOWS > 0 */
1002 * Load the segment descriptor tables for the current processor.
1005 cpu_mode_init(cpu_data_t
*cdp
)
1008 if (cpu_mode_is64bit()) {
1009 cpu_IA32e_enable(cdp
);
1010 cpu_desc_load64(cdp
);
1011 fast_syscall_init64(cdp
);
1013 fast_syscall_init(cdp
);
1016 fast_syscall_init64(cdp
);
1019 /* Call for per-cpu pmap mode initialization */