]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/mp_desc.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-4570.1.46.tar.gz
[apple/xnu.git] / osfmk / i386 / mp_desc.c
1 /*
2 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57 /*
58 */
59
60 #include <kern/cpu_number.h>
61 #include <kern/kalloc.h>
62 #include <kern/cpu_data.h>
63 #include <mach/mach_types.h>
64 #include <mach/machine.h>
65 #include <mach/vm_map.h>
66 #include <mach/machine/vm_param.h>
67 #include <vm/vm_kern.h>
68 #include <vm/vm_map.h>
69
70 #include <i386/bit_routines.h>
71 #include <i386/mp_desc.h>
72 #include <i386/misc_protos.h>
73 #include <i386/mp.h>
74 #include <i386/pmap.h>
75 #include <i386/postcode.h>
76 #include <i386/pmap_internal.h>
77 #if CONFIG_MCA
78 #include <i386/machine_check.h>
79 #endif
80
81 #include <kern/misc_protos.h>
82
83 #if MONOTONIC
84 #include <kern/monotonic.h>
85 #endif /* MONOTONIC */
86 #include <san/kasan.h>
87
88 #define K_INTR_GATE (ACC_P|ACC_PL_K|ACC_INTR_GATE)
89 #define U_INTR_GATE (ACC_P|ACC_PL_U|ACC_INTR_GATE)
90
91 // Declare macros that will declare the externs
92 #define TRAP(n, name) extern void *name ;
93 #define TRAP_ERR(n, name) extern void *name ;
94 #define TRAP_SPC(n, name) extern void *name ;
95 #define TRAP_IST1(n, name) extern void *name ;
96 #define TRAP_IST2(n, name) extern void *name ;
97 #define INTERRUPT(n) extern void *_intr_ ## n ;
98 #define USER_TRAP(n, name) extern void *name ;
99 #define USER_TRAP_SPC(n, name) extern void *name ;
100
101 // Include the table to declare the externs
102 #include "../x86_64/idt_table.h"
103
104 // Undef the macros, then redefine them so we can declare the table
105 #undef TRAP
106 #undef TRAP_ERR
107 #undef TRAP_SPC
108 #undef TRAP_IST1
109 #undef TRAP_IST2
110 #undef INTERRUPT
111 #undef USER_TRAP
112 #undef USER_TRAP_SPC
113
114 #define TRAP(n, name) \
115 [n] = { \
116 (uintptr_t)&name, \
117 KERNEL64_CS, \
118 0, \
119 K_INTR_GATE, \
120 0 \
121 },
122
123 #define TRAP_ERR TRAP
124 #define TRAP_SPC TRAP
125
126 #define TRAP_IST1(n, name) \
127 [n] = { \
128 (uintptr_t)&name, \
129 KERNEL64_CS, \
130 1, \
131 K_INTR_GATE, \
132 0 \
133 },
134
135 #define TRAP_IST2(n, name) \
136 [n] = { \
137 (uintptr_t)&name, \
138 KERNEL64_CS, \
139 2, \
140 K_INTR_GATE, \
141 0 \
142 },
143
144 #define INTERRUPT(n) \
145 [n] = { \
146 (uintptr_t)&_intr_ ## n,\
147 KERNEL64_CS, \
148 0, \
149 K_INTR_GATE, \
150 0 \
151 },
152
153 #define USER_TRAP(n, name) \
154 [n] = { \
155 (uintptr_t)&name, \
156 KERNEL64_CS, \
157 0, \
158 U_INTR_GATE, \
159 0 \
160 },
161
162 #define USER_TRAP_SPC USER_TRAP
163
164 // Declare the table using the macros we just set up
165 struct fake_descriptor64 master_idt64[IDTSZ]
166 __attribute__ ((section("__HIB,__desc")))
167 __attribute__ ((aligned(PAGE_SIZE))) = {
168 #include "../x86_64/idt_table.h"
169 };
170
171 /*
172 * First cpu`s interrupt stack.
173 */
174 extern uint32_t low_intstack[]; /* bottom */
175 extern uint32_t low_eintstack[]; /* top */
176
177 /*
178 * Per-cpu data area pointers.
179 * The master cpu (cpu 0) has its data area statically allocated;
180 * others are allocated dynamically and this array is updated at runtime.
181 */
182 static cpu_data_t cpu_data_master = {
183 .cpu_this = &cpu_data_master,
184 .cpu_nanotime = &pal_rtc_nanotime_info,
185 .cpu_int_stack_top = (vm_offset_t) low_eintstack,
186 };
187 cpu_data_t *cpu_data_ptr[MAX_CPUS] = { [0] = &cpu_data_master };
188
189 decl_simple_lock_data(,ncpus_lock); /* protects real_ncpus */
190 unsigned int real_ncpus = 1;
191 unsigned int max_ncpus = MAX_CPUS;
192
193 extern void hi64_sysenter(void);
194 extern void hi64_syscall(void);
195
196 /*
197 * Multiprocessor i386/i486 systems use a separate copy of the
198 * GDT, IDT, LDT, and kernel TSS per processor. The first three
199 * are separate to avoid lock contention: the i386 uses locked
200 * memory cycles to access the descriptor tables. The TSS is
201 * separate since each processor needs its own kernel stack,
202 * and since using a TSS marks it busy.
203 */
204
205 /*
206 * Allocate and initialize the per-processor descriptor tables.
207 */
208
209 /*
210 * This is the expanded, 64-bit variant of the kernel LDT descriptor.
211 * When switching to 64-bit mode this replaces KERNEL_LDT entry
212 * and the following empty slot. This enables the LDT to be referenced
213 * in the uber-space remapping window on the kernel.
214 */
215 struct fake_descriptor64 kernel_ldt_desc64 = {
216 0,
217 LDTSZ_MIN*sizeof(struct fake_descriptor)-1,
218 0,
219 ACC_P|ACC_PL_K|ACC_LDT,
220 0
221 };
222
223 /*
224 * This is the expanded, 64-bit variant of the kernel TSS descriptor.
225 * It is follows pattern of the KERNEL_LDT.
226 */
227 struct fake_descriptor64 kernel_tss_desc64 = {
228 0,
229 sizeof(struct x86_64_tss)-1,
230 0,
231 ACC_P|ACC_PL_K|ACC_TSS,
232 0
233 };
234
235 /*
236 * Convert a descriptor from fake to real format.
237 *
238 * Fake descriptor format:
239 * bytes 0..3 base 31..0
240 * bytes 4..5 limit 15..0
241 * byte 6 access byte 2 | limit 19..16
242 * byte 7 access byte 1
243 *
244 * Real descriptor format:
245 * bytes 0..1 limit 15..0
246 * bytes 2..3 base 15..0
247 * byte 4 base 23..16
248 * byte 5 access byte 1
249 * byte 6 access byte 2 | limit 19..16
250 * byte 7 base 31..24
251 *
252 * Fake gate format:
253 * bytes 0..3 offset
254 * bytes 4..5 selector
255 * byte 6 word count << 4 (to match fake descriptor)
256 * byte 7 access byte 1
257 *
258 * Real gate format:
259 * bytes 0..1 offset 15..0
260 * bytes 2..3 selector
261 * byte 4 word count
262 * byte 5 access byte 1
263 * bytes 6..7 offset 31..16
264 */
265 void
266 fix_desc(void *d, int num_desc) {
267 //early_kprintf("fix_desc(%x, %x)\n", d, num_desc);
268 uint8_t *desc = (uint8_t*) d;
269
270 do {
271 if ((desc[7] & 0x14) == 0x04) { /* gate */
272 uint32_t offset;
273 uint16_t selector;
274 uint8_t wordcount;
275 uint8_t acc;
276
277 offset = *((uint32_t*)(desc));
278 selector = *((uint32_t*)(desc+4));
279 wordcount = desc[6] >> 4;
280 acc = desc[7];
281
282 *((uint16_t*)desc) = offset & 0xFFFF;
283 *((uint16_t*)(desc+2)) = selector;
284 desc[4] = wordcount;
285 desc[5] = acc;
286 *((uint16_t*)(desc+6)) = offset >> 16;
287
288 } else { /* descriptor */
289 uint32_t base;
290 uint16_t limit;
291 uint8_t acc1, acc2;
292
293 base = *((uint32_t*)(desc));
294 limit = *((uint16_t*)(desc+4));
295 acc2 = desc[6];
296 acc1 = desc[7];
297
298 *((uint16_t*)(desc)) = limit;
299 *((uint16_t*)(desc+2)) = base & 0xFFFF;
300 desc[4] = (base >> 16) & 0xFF;
301 desc[5] = acc1;
302 desc[6] = acc2;
303 desc[7] = base >> 24;
304 }
305 desc += 8;
306 } while (--num_desc);
307 }
308
309 void
310 fix_desc64(void *descp, int count)
311 {
312 struct fake_descriptor64 *fakep;
313 union {
314 struct real_gate64 gate;
315 struct real_descriptor64 desc;
316 } real;
317 int i;
318
319 fakep = (struct fake_descriptor64 *) descp;
320
321 for (i = 0; i < count; i++, fakep++) {
322 /*
323 * Construct the real decriptor locally.
324 */
325
326 bzero((void *) &real, sizeof(real));
327
328 switch (fakep->access & ACC_TYPE) {
329 case 0:
330 break;
331 case ACC_CALL_GATE:
332 case ACC_INTR_GATE:
333 case ACC_TRAP_GATE:
334 real.gate.offset_low16 = (uint16_t)(fakep->offset64 & 0xFFFF);
335 real.gate.selector16 = fakep->lim_or_seg & 0xFFFF;
336 real.gate.IST = fakep->size_or_IST & 0x7;
337 real.gate.access8 = fakep->access;
338 real.gate.offset_high16 = (uint16_t)((fakep->offset64>>16) & 0xFFFF);
339 real.gate.offset_top32 = (uint32_t)(fakep->offset64>>32);
340 break;
341 default: /* Otherwise */
342 real.desc.limit_low16 = fakep->lim_or_seg & 0xFFFF;
343 real.desc.base_low16 = (uint16_t)(fakep->offset64 & 0xFFFF);
344 real.desc.base_med8 = (uint8_t)((fakep->offset64 >> 16) & 0xFF);
345 real.desc.access8 = fakep->access;
346 real.desc.limit_high4 = (fakep->lim_or_seg >> 16) & 0xFF;
347 real.desc.granularity4 = fakep->size_or_IST;
348 real.desc.base_high8 = (uint8_t)((fakep->offset64 >> 24) & 0xFF);
349 real.desc.base_top32 = (uint32_t)(fakep->offset64>>32);
350 }
351
352 /*
353 * Now copy back over the fake structure.
354 */
355 bcopy((void *) &real, (void *) fakep, sizeof(real));
356 }
357 }
358
359 static void
360 cpu_gdt_alias(vm_map_offset_t gdt, vm_map_offset_t alias)
361 {
362 pt_entry_t *pte = NULL;
363
364 /* Require page alignment */
365 assert(page_aligned(gdt));
366 assert(page_aligned(alias));
367
368 pte = pmap_pte(kernel_pmap, alias);
369 pmap_store_pte(pte, kvtophys(gdt) | INTEL_PTE_REF
370 | INTEL_PTE_MOD
371 | INTEL_PTE_WIRED
372 | INTEL_PTE_VALID
373 | INTEL_PTE_WRITE
374 | INTEL_PTE_NX);
375 #if KASAN
376 kasan_notify_address(alias, PAGE_SIZE);
377 #endif
378 }
379
380
381 void
382 cpu_desc_init(cpu_data_t *cdp)
383 {
384 cpu_desc_index_t *cdi = &cdp->cpu_desc_index;
385
386 if (cdp == &cpu_data_master) {
387 /*
388 * Master CPU uses the tables built at boot time.
389 * Just set the index pointers to the low memory space.
390 */
391 cdi->cdi_ktss = (void *)&master_ktss64;
392 cdi->cdi_sstk = (vm_offset_t) &master_sstk.top;
393 cdi->cdi_gdt.ptr = (void *)MASTER_GDT_ALIAS;
394 cdi->cdi_idt.ptr = (void *)MASTER_IDT_ALIAS;
395 cdi->cdi_ldt = (struct fake_descriptor *) master_ldt;
396
397 /* Replace the expanded LDTs and TSS slots in the GDT */
398 kernel_ldt_desc64.offset64 = (uintptr_t) &master_ldt;
399 *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_LDT)] =
400 kernel_ldt_desc64;
401 *(struct fake_descriptor64 *) &master_gdt[sel_idx(USER_LDT)] =
402 kernel_ldt_desc64;
403 kernel_tss_desc64.offset64 = (uintptr_t) &master_ktss64;
404 *(struct fake_descriptor64 *) &master_gdt[sel_idx(KERNEL_TSS)] =
405 kernel_tss_desc64;
406
407 /* Fix up the expanded descriptors for 64-bit. */
408 fix_desc64((void *) &master_idt64, IDTSZ);
409 fix_desc64((void *) &master_gdt[sel_idx(KERNEL_LDT)], 1);
410 fix_desc64((void *) &master_gdt[sel_idx(USER_LDT)], 1);
411 fix_desc64((void *) &master_gdt[sel_idx(KERNEL_TSS)], 1);
412
413 /*
414 * Set the NMI/fault stacks as IST2/IST1 in the 64-bit TSS
415 * Note: this will be dynamically re-allocated in VM later.
416 */
417 master_ktss64.ist2 = (uintptr_t) low_eintstack;
418 master_ktss64.ist1 = (uintptr_t) low_eintstack
419 - sizeof(x86_64_intr_stack_frame_t);
420 } else if (cdi->cdi_ktss == NULL) { /* Skipping re-init on wake */
421 cpu_desc_table64_t *cdt = (cpu_desc_table64_t *) cdp->cpu_desc_tablep;
422
423 /*
424 * Per-cpu GDT, IDT, KTSS descriptors are allocated in kernel
425 * heap (cpu_desc_table).
426 * LDT descriptors are mapped into a separate area.
427 * GDT descriptors are addressed by alias to avoid sgdt leaks to user-space.
428 */
429 cdi->cdi_idt.ptr = (void *)MASTER_IDT_ALIAS;
430 cdi->cdi_gdt.ptr = (void *)CPU_GDT_ALIAS(cdp->cpu_number);
431 cdi->cdi_ktss = (void *)&cdt->ktss;
432 cdi->cdi_sstk = (vm_offset_t)&cdt->sstk.top;
433 cdi->cdi_ldt = cdp->cpu_ldtp;
434
435 /* Make the virtual alias address for the GDT */
436 cpu_gdt_alias((vm_map_offset_t) &cdt->gdt,
437 (vm_map_offset_t) cdi->cdi_gdt.ptr);
438
439 /*
440 * Copy the tables
441 */
442 bcopy((char *)master_gdt, (char *)cdt->gdt, sizeof(master_gdt));
443 bcopy((char *)master_ldt, (char *)cdp->cpu_ldtp, sizeof(master_ldt));
444 bcopy((char *)&master_ktss64, (char *)&cdt->ktss, sizeof(struct x86_64_tss));
445
446 /*
447 * Fix up the entries in the GDT to point to
448 * this LDT and this TSS.
449 */
450 kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldt;
451 *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_LDT)] =
452 kernel_ldt_desc64;
453 fix_desc64(&cdt->gdt[sel_idx(KERNEL_LDT)], 1);
454
455 kernel_ldt_desc64.offset64 = (uintptr_t) cdi->cdi_ldt;
456 *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(USER_LDT)] =
457 kernel_ldt_desc64;
458 fix_desc64(&cdt->gdt[sel_idx(USER_LDT)], 1);
459
460 kernel_tss_desc64.offset64 = (uintptr_t) cdi->cdi_ktss;
461 *(struct fake_descriptor64 *) &cdt->gdt[sel_idx(KERNEL_TSS)] =
462 kernel_tss_desc64;
463 fix_desc64(&cdt->gdt[sel_idx(KERNEL_TSS)], 1);
464
465 /* Set (zeroed) fault stack as IST1, NMI intr stack IST2 */
466 bzero((void *) cdt->fstk, sizeof(cdt->fstk));
467 cdt->ktss.ist2 = (unsigned long)cdt->fstk + sizeof(cdt->fstk);
468 cdt->ktss.ist1 = cdt->ktss.ist2
469 - sizeof(x86_64_intr_stack_frame_t);
470 }
471
472 /* Require that the top of the sysenter stack is 16-byte aligned */
473 if ((cdi->cdi_sstk % 16) != 0)
474 panic("cpu_desc_init() sysenter stack not 16-byte aligned");
475 }
476
477
478 void
479 cpu_desc_load(cpu_data_t *cdp)
480 {
481 cpu_desc_index_t *cdi = &cdp->cpu_desc_index;
482
483 postcode(CPU_DESC_LOAD_ENTRY);
484
485 /* Stuff the kernel per-cpu data area address into the MSRs */
486 postcode(CPU_DESC_LOAD_GS_BASE);
487 wrmsr64(MSR_IA32_GS_BASE, (uintptr_t) cdp);
488 postcode(CPU_DESC_LOAD_KERNEL_GS_BASE);
489 wrmsr64(MSR_IA32_KERNEL_GS_BASE, (uintptr_t) cdp);
490
491 /*
492 * Ensure the TSS segment's busy bit is clear. This is required
493 * for the case of reloading descriptors at wake to avoid
494 * their complete re-initialization.
495 */
496 gdt_desc_p(KERNEL_TSS)->access &= ~ACC_TSS_BUSY;
497
498 /* Load the GDT, LDT, IDT and TSS */
499 cdi->cdi_gdt.size = sizeof(struct real_descriptor)*GDTSZ - 1;
500 cdi->cdi_idt.size = 0x1000 + cdp->cpu_number;
501
502 postcode(CPU_DESC_LOAD_GDT);
503 lgdt((uintptr_t *) &cdi->cdi_gdt);
504 postcode(CPU_DESC_LOAD_IDT);
505 lidt((uintptr_t *) &cdi->cdi_idt);
506 postcode(CPU_DESC_LOAD_LDT);
507 lldt(KERNEL_LDT);
508 postcode(CPU_DESC_LOAD_TSS);
509 set_tr(KERNEL_TSS);
510
511 #if GPROF // Hack to enable mcount to work on K64
512 __asm__ volatile("mov %0, %%gs" : : "rm" ((unsigned short)(KERNEL_DS)));
513 #endif
514 postcode(CPU_DESC_LOAD_EXIT);
515 }
516
517
518 /*
519 * Set MSRs for sysenter/sysexit and syscall/sysret for 64-bit.
520 */
521 void
522 cpu_syscall_init(cpu_data_t *cdp)
523 {
524 #if MONOTONIC
525 mt_cpu_up(cdp);
526 #else /* MONOTONIC */
527 #pragma unused(cdp)
528 #endif /* !MONOTONIC */
529 wrmsr64(MSR_IA32_SYSENTER_CS, SYSENTER_CS);
530 wrmsr64(MSR_IA32_SYSENTER_EIP, (uintptr_t) hi64_sysenter);
531 wrmsr64(MSR_IA32_SYSENTER_ESP, current_sstk());
532 /* Enable syscall/sysret */
533 wrmsr64(MSR_IA32_EFER, rdmsr64(MSR_IA32_EFER) | MSR_IA32_EFER_SCE);
534
535 /*
536 * MSRs for 64-bit syscall/sysret
537 * Note USER_CS because sysret uses this + 16 when returning to
538 * 64-bit code.
539 */
540 wrmsr64(MSR_IA32_LSTAR, (uintptr_t) hi64_syscall);
541 wrmsr64(MSR_IA32_STAR, (((uint64_t)USER_CS) << 48) |
542 (((uint64_t)KERNEL64_CS) << 32));
543 /*
544 * Emulate eflags cleared by sysenter but note that
545 * we also clear the trace trap to avoid the complications
546 * of single-stepping into a syscall. The nested task bit
547 * is also cleared to avoid a spurious "task switch"
548 * should we choose to return via an IRET.
549 */
550 wrmsr64(MSR_IA32_FMASK, EFL_DF|EFL_IF|EFL_TF|EFL_NT);
551
552 }
553
554
555 cpu_data_t *
556 cpu_data_alloc(boolean_t is_boot_cpu)
557 {
558 int ret;
559 cpu_data_t *cdp;
560
561 if (is_boot_cpu) {
562 assert(real_ncpus == 1);
563 cdp = cpu_datap(0);
564 if (cdp->cpu_processor == NULL) {
565 simple_lock_init(&ncpus_lock, 0);
566 cdp->cpu_processor = cpu_processor_alloc(TRUE);
567 #if NCOPY_WINDOWS > 0
568 cdp->cpu_pmap = pmap_cpu_alloc(TRUE);
569 #endif
570 }
571 return cdp;
572 }
573
574 /*
575 * Allocate per-cpu data:
576 */
577 ret = kmem_alloc(kernel_map, (vm_offset_t *) &cdp, sizeof(cpu_data_t), VM_KERN_MEMORY_CPU);
578 if (ret != KERN_SUCCESS) {
579 printf("cpu_data_alloc() failed, ret=%d\n", ret);
580 goto abort;
581 }
582 bzero((void*) cdp, sizeof(cpu_data_t));
583 cdp->cpu_this = cdp;
584
585 /*
586 * Allocate interrupt stack:
587 */
588 ret = kmem_alloc(kernel_map,
589 (vm_offset_t *) &cdp->cpu_int_stack_top,
590 INTSTACK_SIZE, VM_KERN_MEMORY_CPU);
591 if (ret != KERN_SUCCESS) {
592 printf("cpu_data_alloc() int stack failed, ret=%d\n", ret);
593 goto abort;
594 }
595 bzero((void*) cdp->cpu_int_stack_top, INTSTACK_SIZE);
596 cdp->cpu_int_stack_top += INTSTACK_SIZE;
597
598 /*
599 * Allocate descriptor table:
600 */
601 ret = kmem_alloc(kernel_map,
602 (vm_offset_t *) &cdp->cpu_desc_tablep,
603 sizeof(cpu_desc_table64_t),
604 VM_KERN_MEMORY_CPU);
605 if (ret != KERN_SUCCESS) {
606 printf("cpu_data_alloc() desc_table failed, ret=%d\n", ret);
607 goto abort;
608 }
609
610 /*
611 * Allocate LDT
612 */
613 ret = kmem_alloc(kernel_map,
614 (vm_offset_t *) &cdp->cpu_ldtp,
615 sizeof(struct real_descriptor) * LDTSZ,
616 VM_KERN_MEMORY_CPU);
617 if (ret != KERN_SUCCESS) {
618 printf("cpu_data_alloc() ldt failed, ret=%d\n", ret);
619 goto abort;
620 }
621
622 #if CONFIG_MCA
623 /* Machine-check shadow register allocation. */
624 mca_cpu_alloc(cdp);
625 #endif
626
627 simple_lock(&ncpus_lock);
628
629 cpu_data_ptr[real_ncpus] = cdp;
630 cdp->cpu_number = real_ncpus;
631 real_ncpus++;
632 simple_unlock(&ncpus_lock);
633
634 /*
635 * Before this cpu has been assigned a real thread context,
636 * we give it a fake, unique, non-zero thread id which the locking
637 * primitives use as their lock value.
638 * Note that this does not apply to the boot processor, cpu 0, which
639 * transitions to a thread context well before other processors are
640 * started.
641 */
642 cdp->cpu_active_thread = (thread_t) (uintptr_t) cdp->cpu_number;
643
644 cdp->cpu_nanotime = &pal_rtc_nanotime_info;
645
646 kprintf("cpu_data_alloc(%d) %p desc_table: %p "
647 "ldt: %p "
648 "int_stack: 0x%lx-0x%lx\n",
649 cdp->cpu_number, cdp, cdp->cpu_desc_tablep, cdp->cpu_ldtp,
650 (long)(cdp->cpu_int_stack_top - INTSTACK_SIZE), (long)(cdp->cpu_int_stack_top));
651
652 return cdp;
653
654 abort:
655 if (cdp) {
656 if (cdp->cpu_desc_tablep)
657 kfree((void *) cdp->cpu_desc_tablep,
658 sizeof(cpu_desc_table64_t));
659 if (cdp->cpu_int_stack_top)
660 kfree((void *) (cdp->cpu_int_stack_top - INTSTACK_SIZE),
661 INTSTACK_SIZE);
662 kfree((void *) cdp, sizeof(*cdp));
663 }
664 return NULL;
665 }
666
667 boolean_t
668 valid_user_data_selector(uint16_t selector)
669 {
670 sel_t sel = selector_to_sel(selector);
671
672 if (selector == 0)
673 return (TRUE);
674
675 if (sel.ti == SEL_LDT)
676 return (TRUE);
677 else if (sel.index < GDTSZ) {
678 if ((gdt_desc_p(selector)->access & ACC_PL_U) == ACC_PL_U)
679 return (TRUE);
680 }
681
682 return (FALSE);
683 }
684
685 boolean_t
686 valid_user_code_selector(uint16_t selector)
687 {
688 sel_t sel = selector_to_sel(selector);
689
690 if (selector == 0)
691 return (FALSE);
692
693 if (sel.ti == SEL_LDT) {
694 if (sel.rpl == USER_PRIV)
695 return (TRUE);
696 }
697 else if (sel.index < GDTSZ && sel.rpl == USER_PRIV) {
698 if ((gdt_desc_p(selector)->access & ACC_PL_U) == ACC_PL_U)
699 return (TRUE);
700 /* Explicitly validate the system code selectors
701 * even if not instantaneously privileged,
702 * since they are dynamically re-privileged
703 * at context switch
704 */
705 if ((selector == USER_CS) || (selector == USER64_CS))
706 return (TRUE);
707 }
708
709 return (FALSE);
710 }
711
712 boolean_t
713 valid_user_stack_selector(uint16_t selector)
714 {
715 sel_t sel = selector_to_sel(selector);
716
717 if (selector == 0)
718 return (FALSE);
719
720 if (sel.ti == SEL_LDT) {
721 if (sel.rpl == USER_PRIV)
722 return (TRUE);
723 }
724 else if (sel.index < GDTSZ && sel.rpl == USER_PRIV) {
725 if ((gdt_desc_p(selector)->access & ACC_PL_U) == ACC_PL_U)
726 return (TRUE);
727 }
728
729 return (FALSE);
730 }
731
732 boolean_t
733 valid_user_segment_selectors(uint16_t cs,
734 uint16_t ss,
735 uint16_t ds,
736 uint16_t es,
737 uint16_t fs,
738 uint16_t gs)
739 {
740 return valid_user_code_selector(cs) &&
741 valid_user_stack_selector(ss) &&
742 valid_user_data_selector(ds) &&
743 valid_user_data_selector(es) &&
744 valid_user_data_selector(fs) &&
745 valid_user_data_selector(gs);
746 }
747
748 #if NCOPY_WINDOWS > 0
749
750 static vm_offset_t user_window_base = 0;
751
752 void
753 cpu_userwindow_init(int cpu)
754 {
755 cpu_data_t *cdp = cpu_data_ptr[cpu];
756 vm_offset_t user_window;
757 vm_offset_t vaddr;
758 int num_cpus;
759
760 num_cpus = ml_get_max_cpus();
761
762 if (cpu >= num_cpus)
763 panic("cpu_userwindow_init: cpu > num_cpus");
764
765 if (user_window_base == 0) {
766
767 if (vm_allocate(kernel_map, &vaddr,
768 (NBPDE * NCOPY_WINDOWS * num_cpus) + NBPDE,
769 VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_CPU)) != KERN_SUCCESS)
770 panic("cpu_userwindow_init: "
771 "couldn't allocate user map window");
772
773 /*
774 * window must start on a page table boundary
775 * in the virtual address space
776 */
777 user_window_base = (vaddr + (NBPDE - 1)) & ~(NBPDE - 1);
778
779 /*
780 * get rid of any allocation leading up to our
781 * starting boundary
782 */
783 vm_deallocate(kernel_map, vaddr, user_window_base - vaddr);
784
785 /*
786 * get rid of tail that we don't need
787 */
788 user_window = user_window_base +
789 (NBPDE * NCOPY_WINDOWS * num_cpus);
790
791 vm_deallocate(kernel_map, user_window,
792 (vaddr +
793 ((NBPDE * NCOPY_WINDOWS * num_cpus) + NBPDE)) -
794 user_window);
795 }
796
797 user_window = user_window_base + (cpu * NCOPY_WINDOWS * NBPDE);
798
799 cdp->cpu_copywindow_base = user_window;
800 /*
801 * Abuse this pdp entry, the pdp now actually points to
802 * an array of copy windows addresses.
803 */
804 cdp->cpu_copywindow_pdp = pmap_pde(kernel_pmap, user_window);
805
806 }
807
808 void
809 cpu_physwindow_init(int cpu)
810 {
811 cpu_data_t *cdp = cpu_data_ptr[cpu];
812 vm_offset_t phys_window = cdp->cpu_physwindow_base;
813
814 if (phys_window == 0) {
815 if (vm_allocate(kernel_map, &phys_window,
816 PAGE_SIZE, VM_FLAGS_ANYWHERE | VM_MAKE_TAG(VM_KERN_MEMORY_CPU))
817 != KERN_SUCCESS)
818 panic("cpu_physwindow_init: "
819 "couldn't allocate phys map window");
820
821 /*
822 * make sure the page that encompasses the
823 * pte pointer we're interested in actually
824 * exists in the page table
825 */
826 pmap_expand(kernel_pmap, phys_window, PMAP_EXPAND_OPTIONS_NONE);
827
828 cdp->cpu_physwindow_base = phys_window;
829 cdp->cpu_physwindow_ptep = vtopte(phys_window);
830 }
831 }
832 #endif /* NCOPY_WINDOWS > 0 */
833
834 /*
835 * Allocate a new interrupt stack for the boot processor from the
836 * heap rather than continue to use the statically allocated space.
837 * Also switch to a dynamically allocated cpu data area.
838 */
839 void
840 cpu_data_realloc(void)
841 {
842 int ret;
843 vm_offset_t istk;
844 vm_offset_t fstk;
845 cpu_data_t *cdp;
846 boolean_t istate;
847
848 ret = kmem_alloc(kernel_map, &istk, INTSTACK_SIZE, VM_KERN_MEMORY_CPU);
849 if (ret != KERN_SUCCESS) {
850 panic("cpu_data_realloc() stack alloc, ret=%d\n", ret);
851 }
852 bzero((void*) istk, INTSTACK_SIZE);
853 istk += INTSTACK_SIZE;
854
855 ret = kmem_alloc(kernel_map, (vm_offset_t *) &cdp, sizeof(cpu_data_t), VM_KERN_MEMORY_CPU);
856 if (ret != KERN_SUCCESS) {
857 panic("cpu_data_realloc() cpu data alloc, ret=%d\n", ret);
858 }
859
860 /* Copy old contents into new area and make fix-ups */
861 assert(cpu_number() == 0);
862 bcopy((void *) cpu_data_ptr[0], (void*) cdp, sizeof(cpu_data_t));
863 cdp->cpu_this = cdp;
864 cdp->cpu_int_stack_top = istk;
865 timer_call_queue_init(&cdp->rtclock_timer.queue);
866
867 /* Allocate the separate fault stack */
868 ret = kmem_alloc(kernel_map, &fstk, PAGE_SIZE, VM_KERN_MEMORY_CPU);
869 if (ret != KERN_SUCCESS) {
870 panic("cpu_data_realloc() fault stack alloc, ret=%d\n", ret);
871 }
872 bzero((void*) fstk, PAGE_SIZE);
873 fstk += PAGE_SIZE;
874
875 /*
876 * With interrupts disabled commmit the new areas.
877 */
878 istate = ml_set_interrupts_enabled(FALSE);
879 cpu_data_ptr[0] = cdp;
880 master_ktss64.ist2 = (uintptr_t) fstk;
881 master_ktss64.ist1 = (uintptr_t) fstk
882 - sizeof(x86_64_intr_stack_frame_t);
883 wrmsr64(MSR_IA32_GS_BASE, (uintptr_t) cdp);
884 wrmsr64(MSR_IA32_KERNEL_GS_BASE, (uintptr_t) cdp);
885 (void) ml_set_interrupts_enabled(istate);
886
887 kprintf("Reallocated master cpu data: %p,"
888 " interrupt stack: %p, fault stack: %p\n",
889 (void *) cdp, (void *) istk, (void *) fstk);
890 }
mirror of XNU