]> git.saurik.com Git - apple/xnu.git/blob - osfmk/i386/pcb.c
projects / apple / xnu.git / blob
? search:


36d4fec6e907598d310bf4021dbaca8fbdd9a4a3
[apple/xnu.git] / osfmk / i386 / pcb.c
1 /*
2 * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57 #include <mach_rt.h>
58 #include <mach_debug.h>
59 #include <mach_ldebug.h>
60
61 #include <sys/kdebug.h>
62
63 #include <mach/kern_return.h>
64 #include <mach/thread_status.h>
65 #include <mach/vm_param.h>
66
67 #include <kern/counters.h>
68 #include <kern/kalloc.h>
69 #include <kern/mach_param.h>
70 #include <kern/processor.h>
71 #include <kern/cpu_data.h>
72 #include <kern/cpu_number.h>
73 #include <kern/task.h>
74 #include <kern/thread.h>
75 #include <kern/sched_prim.h>
76 #include <kern/misc_protos.h>
77 #include <kern/assert.h>
78 #include <kern/spl.h>
79 #include <kern/machine.h>
80 #include <ipc/ipc_port.h>
81 #include <vm/vm_kern.h>
82 #include <vm/vm_map.h>
83 #include <vm/pmap.h>
84 #include <vm/vm_protos.h>
85
86 #include <i386/cpu_data.h>
87 #include <i386/cpu_number.h>
88 #include <i386/eflags.h>
89 #include <i386/proc_reg.h>
90 #include <i386/tss.h>
91 #include <i386/user_ldt.h>
92 #include <i386/fpu.h>
93 #include <i386/mp_desc.h>
94 #include <i386/misc_protos.h>
95 #include <i386/thread.h>
96 #if defined(__i386__)
97 #include <i386/fpu.h>
98 #endif
99 #include <i386/seg.h>
100 #include <i386/machine_routines.h>
101 #include <i386/lapic.h> /* LAPIC_PMC_SWI_VECTOR */
102
103 #include <machine/commpage.h>
104
105 #if CONFIG_COUNTERS
106 #include <pmc/pmc.h>
107 #endif /* CONFIG_COUNTERS */
108
109 /*
110 * Maps state flavor to number of words in the state:
111 */
112 unsigned int _MachineStateCount[] = {
113 /* FLAVOR_LIST */
114 0,
115 x86_THREAD_STATE32_COUNT,
116 x86_FLOAT_STATE32_COUNT,
117 x86_EXCEPTION_STATE32_COUNT,
118 x86_THREAD_STATE64_COUNT,
119 x86_FLOAT_STATE64_COUNT,
120 x86_EXCEPTION_STATE64_COUNT,
121 x86_THREAD_STATE_COUNT,
122 x86_FLOAT_STATE_COUNT,
123 x86_EXCEPTION_STATE_COUNT,
124 0,
125 x86_SAVED_STATE32_COUNT,
126 x86_SAVED_STATE64_COUNT,
127 x86_DEBUG_STATE32_COUNT,
128 x86_DEBUG_STATE64_COUNT,
129 x86_DEBUG_STATE_COUNT
130 };
131
132 zone_t iss_zone; /* zone for saved_state area */
133 zone_t ids_zone; /* zone for debug_state area */
134
135 /* Forward */
136
137 void act_machine_throughcall(thread_t thr_act);
138 void act_machine_return(int);
139
140 extern void Thread_continue(void);
141 extern void Load_context(
142 thread_t thread);
143
144 static void
145 get_exception_state32(thread_t thread, x86_exception_state32_t *es);
146
147 static void
148 get_exception_state64(thread_t thread, x86_exception_state64_t *es);
149
150 static void
151 get_thread_state32(thread_t thread, x86_thread_state32_t *ts);
152
153 static void
154 get_thread_state64(thread_t thread, x86_thread_state64_t *ts);
155
156 static int
157 set_thread_state32(thread_t thread, x86_thread_state32_t *ts);
158
159 static int
160 set_thread_state64(thread_t thread, x86_thread_state64_t *ts);
161
162 #if CONFIG_COUNTERS
163 static inline void
164 machine_pmc_cswitch(thread_t /* old */, thread_t /* new */);
165
166 static inline boolean_t
167 machine_thread_pmc_eligible(thread_t);
168
169 static inline void
170 pmc_swi(thread_t /* old */, thread_t /*new */);
171
172 static inline boolean_t
173 machine_thread_pmc_eligible(thread_t t) {
174 /*
175 * NOTE: Task-level reservations are propagated to child threads via
176 * thread_create_internal. Any mutation of task reservations forces a
177 * recalculate of t_chud (for the pmc flag) for all threads in that task.
178 * Consequently, we can simply check the current thread's flag against
179 * THREAD_PMC_FLAG. If the result is non-zero, we SWI for a PMC switch.
180 */
181 return (t != NULL) ? ((t->t_chud & THREAD_PMC_FLAG) ? TRUE : FALSE) : FALSE;
182 }
183
184 static inline void
185 pmc_swi(thread_t old, thread_t new) {
186 current_cpu_datap()->csw_old_thread = old;
187 current_cpu_datap()->csw_new_thread = new;
188 __asm__ __volatile__("int %0"::"i"(LAPIC_PMC_SWI_VECTOR):"memory");
189 }
190
191 static inline void
192 machine_pmc_cswitch(thread_t old, thread_t new) {
193 if (machine_thread_pmc_eligible(old) || machine_thread_pmc_eligible(new)) {
194 pmc_swi(old, new);
195 }
196 }
197
198 void ml_get_csw_threads(thread_t *old, thread_t *new) {
199 *old = current_cpu_datap()->csw_old_thread;
200 *new = current_cpu_datap()->csw_new_thread;
201 }
202
203 #endif /* CONFIG_COUNTERS */
204
205 /*
206 * Don't let an illegal value for dr7 get set. Specifically,
207 * check for undefined settings. Setting these bit patterns
208 * result in undefined behaviour and can lead to an unexpected
209 * TRCTRAP.
210 */
211 static boolean_t
212 dr7_is_valid(uint32_t *dr7)
213 {
214 int i;
215 uint32_t mask1, mask2;
216
217 /*
218 * If the DE bit is set in CR4, R/W0-3 can be pattern
219 * "10B" to indicate i/o reads and write
220 */
221 if (!(get_cr4() & CR4_DE))
222 for (i = 0, mask1 = 0x3<<16, mask2 = 0x2<<16; i < 4;
223 i++, mask1 <<= 4, mask2 <<= 4)
224 if ((*dr7 & mask1) == mask2)
225 return (FALSE);
226
227 /*
228 * len0-3 pattern "10B" is ok for len on Merom and newer processors
229 * (it signifies an 8-byte wide region). We use the 64bit capability
230 * of the processor in lieu of the more laborious model/family checks
231 * as all 64-bit capable processors so far support this.
232 * Reject an attempt to use this on 64-bit incapable processors.
233 */
234 if (current_cpu_datap()->cpu_is64bit == FALSE)
235 for (i = 0, mask1 = 0x3<<18, mask2 = 0x2<<18; i < 4;
236 i++, mask1 <<= 4, mask2 <<= 4)
237 if ((*dr7 & mask1) == mask2)
238 return (FALSE);
239
240 /*
241 * if we are doing an instruction execution break (indicated
242 * by r/w[x] being "00B"), then the len[x] must also be set
243 * to "00B"
244 */
245 for (i = 0; i < 4; i++)
246 if (((((*dr7 >> (16 + i*4))) & 0x3) == 0) &&
247 ((((*dr7 >> (18 + i*4))) & 0x3) != 0))
248 return (FALSE);
249
250 /*
251 * Intel docs have these bits fixed.
252 */
253 *dr7 |= 0x1 << 10; /* set bit 10 to 1 */
254 *dr7 &= ~(0x1 << 11); /* set bit 11 to 0 */
255 *dr7 &= ~(0x1 << 12); /* set bit 12 to 0 */
256 *dr7 &= ~(0x1 << 14); /* set bit 14 to 0 */
257 *dr7 &= ~(0x1 << 15); /* set bit 15 to 0 */
258
259 /*
260 * We don't allow anything to set the global breakpoints.
261 */
262
263 if (*dr7 & 0x2)
264 return (FALSE);
265
266 if (*dr7 & (0x2<<2))
267 return (FALSE);
268
269 if (*dr7 & (0x2<<4))
270 return (FALSE);
271
272 if (*dr7 & (0x2<<6))
273 return (FALSE);
274
275 return (TRUE);
276 }
277
278 static inline void
279 set_live_debug_state32(cpu_data_t *cdp, x86_debug_state32_t *ds)
280 {
281 __asm__ volatile ("movl %0,%%db0" : :"r" (ds->dr0));
282 __asm__ volatile ("movl %0,%%db1" : :"r" (ds->dr1));
283 __asm__ volatile ("movl %0,%%db2" : :"r" (ds->dr2));
284 __asm__ volatile ("movl %0,%%db3" : :"r" (ds->dr3));
285 if (cpu_mode_is64bit())
286 cdp->cpu_dr7 = ds->dr7;
287 }
288
289 extern void set_64bit_debug_regs(x86_debug_state64_t *ds);
290
291 static inline void
292 set_live_debug_state64(cpu_data_t *cdp, x86_debug_state64_t *ds)
293 {
294 /*
295 * We need to enter 64-bit mode in order to set the full
296 * width of these registers
297 */
298 set_64bit_debug_regs(ds);
299 cdp->cpu_dr7 = ds->dr7;
300 }
301
302 boolean_t
303 debug_state_is_valid32(x86_debug_state32_t *ds)
304 {
305 if (!dr7_is_valid(&ds->dr7))
306 return FALSE;
307
308 #if defined(__i386__)
309 /*
310 * Only allow local breakpoints and make sure they are not
311 * in the trampoline code.
312 */
313 if (ds->dr7 & 0x1)
314 if (ds->dr0 >= (unsigned long)HIGH_MEM_BASE)
315 return FALSE;
316
317 if (ds->dr7 & (0x1<<2))
318 if (ds->dr1 >= (unsigned long)HIGH_MEM_BASE)
319 return FALSE;
320
321 if (ds->dr7 & (0x1<<4))
322 if (ds->dr2 >= (unsigned long)HIGH_MEM_BASE)
323 return FALSE;
324
325 if (ds->dr7 & (0x1<<6))
326 if (ds->dr3 >= (unsigned long)HIGH_MEM_BASE)
327 return FALSE;
328 #endif
329
330 return TRUE;
331 }
332
333 boolean_t
334 debug_state_is_valid64(x86_debug_state64_t *ds)
335 {
336 if (!dr7_is_valid((uint32_t *)&ds->dr7))
337 return FALSE;
338
339 /*
340 * Don't allow the user to set debug addresses above their max
341 * value
342 */
343 if (ds->dr7 & 0x1)
344 if (ds->dr0 >= VM_MAX_PAGE_ADDRESS)
345 return FALSE;
346
347 if (ds->dr7 & (0x1<<2))
348 if (ds->dr1 >= VM_MAX_PAGE_ADDRESS)
349 return FALSE;
350
351 if (ds->dr7 & (0x1<<4))
352 if (ds->dr2 >= VM_MAX_PAGE_ADDRESS)
353 return FALSE;
354
355 if (ds->dr7 & (0x1<<6))
356 if (ds->dr3 >= VM_MAX_PAGE_ADDRESS)
357 return FALSE;
358
359 return TRUE;
360 }
361
362
363 static kern_return_t
364 set_debug_state32(thread_t thread, x86_debug_state32_t *ds)
365 {
366 x86_debug_state32_t *ids;
367 pcb_t pcb;
368
369 pcb = thread->machine.pcb;
370 ids = pcb->ids;
371
372 if (debug_state_is_valid32(ds) != TRUE) {
373 return KERN_INVALID_ARGUMENT;
374 }
375
376 if (ids == NULL) {
377 ids = zalloc(ids_zone);
378 bzero(ids, sizeof *ids);
379
380 simple_lock(&pcb->lock);
381 /* make sure it wasn't already alloc()'d elsewhere */
382 if (pcb->ids == NULL) {
383 pcb->ids = ids;
384 simple_unlock(&pcb->lock);
385 } else {
386 simple_unlock(&pcb->lock);
387 zfree(ids_zone, ids);
388 }
389 }
390
391
392 copy_debug_state32(ds, ids, FALSE);
393
394 return (KERN_SUCCESS);
395 }
396
397 static kern_return_t
398 set_debug_state64(thread_t thread, x86_debug_state64_t *ds)
399 {
400 x86_debug_state64_t *ids;
401 pcb_t pcb;
402
403 pcb = thread->machine.pcb;
404 ids = pcb->ids;
405
406 if (debug_state_is_valid64(ds) != TRUE) {
407 return KERN_INVALID_ARGUMENT;
408 }
409
410 if (ids == NULL) {
411 ids = zalloc(ids_zone);
412 bzero(ids, sizeof *ids);
413
414 simple_lock(&pcb->lock);
415 /* make sure it wasn't already alloc()'d elsewhere */
416 if (pcb->ids == NULL) {
417 pcb->ids = ids;
418 simple_unlock(&pcb->lock);
419 } else {
420 simple_unlock(&pcb->lock);
421 zfree(ids_zone, ids);
422 }
423 }
424
425 copy_debug_state64(ds, ids, FALSE);
426
427 return (KERN_SUCCESS);
428 }
429
430 static void
431 get_debug_state32(thread_t thread, x86_debug_state32_t *ds)
432 {
433 x86_debug_state32_t *saved_state;
434
435 saved_state = thread->machine.pcb->ids;
436
437 if (saved_state) {
438 copy_debug_state32(saved_state, ds, TRUE);
439 } else
440 bzero(ds, sizeof *ds);
441 }
442
443 static void
444 get_debug_state64(thread_t thread, x86_debug_state64_t *ds)
445 {
446 x86_debug_state64_t *saved_state;
447
448 saved_state = (x86_debug_state64_t *)thread->machine.pcb->ids;
449
450 if (saved_state) {
451 copy_debug_state64(saved_state, ds, TRUE);
452 } else
453 bzero(ds, sizeof *ds);
454 }
455
456 /*
457 * consider_machine_collect:
458 *
459 * Try to collect machine-dependent pages
460 */
461 void
462 consider_machine_collect(void)
463 {
464 }
465
466 void
467 consider_machine_adjust(void)
468 {
469 }
470 extern void *get_bsduthreadarg(thread_t th);
471
472 #if defined(__x86_64__)
473 static void
474 act_machine_switch_pcb( thread_t new )
475 {
476 pcb_t pcb = new->machine.pcb;
477 struct real_descriptor *ldtp;
478 mach_vm_offset_t pcb_stack_top;
479 cpu_data_t *cdp = current_cpu_datap();
480
481 assert(new->kernel_stack != 0);
482
483 if (!cpu_mode_is64bit()) {
484 panic("K64 is 64bit!");
485 } else if (is_saved_state64(pcb->iss)) {
486 /*
487 * The test above is performed against the thread save state
488 * flavor and not task's 64-bit feature flag because of the
489 * thread/task 64-bit state divergence that can arise in
490 * task_set_64bit() x86: the task state is changed before
491 * the individual thread(s).
492 */
493 x86_saved_state64_tagged_t *iss64;
494 vm_offset_t isf;
495
496 assert(is_saved_state64(pcb->iss));
497
498 iss64 = (x86_saved_state64_tagged_t *) pcb->iss;
499
500 /*
501 * Set pointer to PCB's interrupt stack frame in cpu data.
502 * Used by syscall and double-fault trap handlers.
503 */
504 isf = (vm_offset_t) &iss64->state.isf;
505 cdp->cpu_uber.cu_isf = isf;
506 pcb_stack_top = (vm_offset_t) (iss64 + 1);
507 /* require 16-byte alignment */
508 assert((pcb_stack_top & 0xF) == 0);
509
510 /* Interrupt stack is pcb */
511 current_ktss64()->rsp0 = pcb_stack_top;
512
513 /*
514 * Top of temporary sysenter stack points to pcb stack.
515 * Although this is not normally used by 64-bit users,
516 * it needs to be set in case a sysenter is attempted.
517 */
518 *current_sstk64() = pcb_stack_top;
519
520 cdp->cpu_task_map = new->map->pmap->pm_task_map;
521
522 /*
523 * Enable the 64-bit user code segment, USER64_CS.
524 * Disable the 32-bit user code segment, USER_CS.
525 */
526 ldt_desc_p(USER64_CS)->access |= ACC_PL_U;
527 ldt_desc_p(USER_CS)->access &= ~ACC_PL_U;
528
529 /*
530 * Switch user's GS base if necessary
531 * by setting the Kernel's GS base MSR
532 * - this will become the user's on the swapgs when
533 * returning to user-space.
534 */
535 if (cdp->cpu_uber.cu_user_gs_base != pcb->cthread_self) {
536 cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self;
537 wrmsr64(MSR_IA32_KERNEL_GS_BASE, pcb->cthread_self);
538 }
539 } else {
540 x86_saved_state_compat32_t *iss32compat;
541 vm_offset_t isf;
542
543 assert(is_saved_state32(pcb->iss));
544 iss32compat = (x86_saved_state_compat32_t *) pcb->iss;
545
546 pcb_stack_top = (uintptr_t) (iss32compat + 1);
547 /* require 16-byte alignment */
548 assert((pcb_stack_top & 0xF) == 0);
549
550 /*
551 * Set pointer to PCB's interrupt stack frame in cpu data.
552 * Used by debug trap handler.
553 */
554 isf = (vm_offset_t) &iss32compat->isf64;
555 cdp->cpu_uber.cu_isf = isf;
556
557 /* Top of temporary sysenter stack points to pcb stack */
558 *current_sstk64() = pcb_stack_top;
559
560 /* Interrupt stack is pcb */
561 current_ktss64()->rsp0 = pcb_stack_top;
562
563 cdp->cpu_task_map = TASK_MAP_32BIT;
564 /* Precalculate pointers to syscall argument store, for use
565 * in the trampolines.
566 */
567 cdp->cpu_uber_arg_store = (vm_offset_t)get_bsduthreadarg(new);
568 cdp->cpu_uber_arg_store_valid = (vm_offset_t)&pcb->arg_store_valid;
569 pcb->arg_store_valid = 0;
570
571 /*
572 * Disable USER64_CS
573 * Enable USER_CS
574 */
575 ldt_desc_p(USER64_CS)->access &= ~ACC_PL_U;
576 ldt_desc_p(USER_CS)->access |= ACC_PL_U;
577
578 /*
579 * Set the thread`s cthread (a.k.a pthread)
580 * For 32-bit user this involves setting the USER_CTHREAD
581 * descriptor in the LDT to point to the cthread data.
582 * The involves copying in the pre-initialized descriptor.
583 */
584 ldtp = (struct real_descriptor *)current_ldt();
585 ldtp[sel_idx(USER_CTHREAD)] = pcb->cthread_desc;
586 if (pcb->uldt_selector != 0)
587 ldtp[sel_idx(pcb->uldt_selector)] = pcb->uldt_desc;
588 cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self;
589
590 /*
591 * Set the thread`s LDT or LDT entry.
592 */
593 if (new->task == TASK_NULL || new->task->i386_ldt == 0) {
594 /*
595 * Use system LDT.
596 */
597 ml_cpu_set_ldt(KERNEL_LDT);
598 } else {
599 /*
600 * Task has its own LDT.
601 */
602 user_ldt_set(new);
603 }
604 }
605
606 /*
607 * Bump the scheduler generation count in the commpage.
608 * This can be read by user code to detect its preemption.
609 */
610 commpage_sched_gen_inc();
611 }
612 #else
613 static void
614 act_machine_switch_pcb( thread_t new )
615 {
616 pcb_t pcb = new->machine.pcb;
617 struct real_descriptor *ldtp;
618 vm_offset_t pcb_stack_top;
619 vm_offset_t hi_pcb_stack_top;
620 vm_offset_t hi_iss;
621 cpu_data_t *cdp = current_cpu_datap();
622
623 assert(new->kernel_stack != 0);
624 STACK_IEL(new->kernel_stack)->saved_state = pcb->iss;
625
626 if (!cpu_mode_is64bit()) {
627 x86_saved_state32_tagged_t *hi_iss32;
628 /*
629 * Save a pointer to the top of the "kernel" stack -
630 * actually the place in the PCB where a trap into
631 * kernel mode will push the registers.
632 */
633 hi_iss = (vm_offset_t)((unsigned long)
634 pmap_cpu_high_map_vaddr(cpu_number(), HIGH_CPU_ISS0) |
635 ((unsigned long)pcb->iss & PAGE_MASK));
636
637 cdp->cpu_hi_iss = (void *)hi_iss;
638
639 pmap_high_map(pcb->iss_pte0, HIGH_CPU_ISS0);
640 pmap_high_map(pcb->iss_pte1, HIGH_CPU_ISS1);
641
642 hi_iss32 = (x86_saved_state32_tagged_t *) hi_iss;
643 assert(hi_iss32->tag == x86_SAVED_STATE32);
644
645 hi_pcb_stack_top = (int) (hi_iss32 + 1);
646
647 /*
648 * For fast syscall, top of interrupt stack points to pcb stack
649 */
650 *(vm_offset_t *) current_sstk() = hi_pcb_stack_top;
651
652 current_ktss()->esp0 = hi_pcb_stack_top;
653
654 } else if (is_saved_state64(pcb->iss)) {
655 /*
656 * The test above is performed against the thread save state
657 * flavor and not task's 64-bit feature flag because of the
658 * thread/task 64-bit state divergence that can arise in
659 * task_set_64bit() x86: the task state is changed before
660 * the individual thread(s).
661 */
662 x86_saved_state64_tagged_t *iss64;
663 vm_offset_t isf;
664
665 assert(is_saved_state64(pcb->iss));
666
667 iss64 = (x86_saved_state64_tagged_t *) pcb->iss;
668
669 /*
670 * Set pointer to PCB's interrupt stack frame in cpu data.
671 * Used by syscall and double-fault trap handlers.
672 */
673 isf = (vm_offset_t) &iss64->state.isf;
674 cdp->cpu_uber.cu_isf = UBER64(isf);
675 pcb_stack_top = (vm_offset_t) (iss64 + 1);
676 /* require 16-byte alignment */
677 assert((pcb_stack_top & 0xF) == 0);
678 /* Interrupt stack is pcb */
679 current_ktss64()->rsp0 = UBER64(pcb_stack_top);
680
681 /*
682 * Top of temporary sysenter stack points to pcb stack.
683 * Although this is not normally used by 64-bit users,
684 * it needs to be set in case a sysenter is attempted.
685 */
686 *current_sstk64() = UBER64(pcb_stack_top);
687
688 cdp->cpu_task_map = new->map->pmap->pm_task_map;
689
690 /*
691 * Enable the 64-bit user code segment, USER64_CS.
692 * Disable the 32-bit user code segment, USER_CS.
693 */
694 ldt_desc_p(USER64_CS)->access |= ACC_PL_U;
695 ldt_desc_p(USER_CS)->access &= ~ACC_PL_U;
696
697 } else {
698 x86_saved_state_compat32_t *iss32compat;
699 vm_offset_t isf;
700
701 assert(is_saved_state32(pcb->iss));
702 iss32compat = (x86_saved_state_compat32_t *) pcb->iss;
703
704 pcb_stack_top = (int) (iss32compat + 1);
705 /* require 16-byte alignment */
706 assert((pcb_stack_top & 0xF) == 0);
707
708 /*
709 * Set pointer to PCB's interrupt stack frame in cpu data.
710 * Used by debug trap handler.
711 */
712 isf = (vm_offset_t) &iss32compat->isf64;
713 cdp->cpu_uber.cu_isf = UBER64(isf);
714
715 /* Top of temporary sysenter stack points to pcb stack */
716 *current_sstk64() = UBER64(pcb_stack_top);
717
718 /* Interrupt stack is pcb */
719 current_ktss64()->rsp0 = UBER64(pcb_stack_top);
720
721 cdp->cpu_task_map = TASK_MAP_32BIT;
722 /* Precalculate pointers to syscall argument store, for use
723 * in the trampolines.
724 */
725 cdp->cpu_uber_arg_store = UBER64((vm_offset_t)get_bsduthreadarg(new));
726 cdp->cpu_uber_arg_store_valid = UBER64((vm_offset_t)&pcb->arg_store_valid);
727 pcb->arg_store_valid = 0;
728
729 /*
730 * Disable USER64_CS
731 * Enable USER_CS
732 */
733 ldt_desc_p(USER64_CS)->access &= ~ACC_PL_U;
734 ldt_desc_p(USER_CS)->access |= ACC_PL_U;
735 }
736
737 /*
738 * Set the thread`s cthread (a.k.a pthread)
739 * For 32-bit user this involves setting the USER_CTHREAD
740 * descriptor in the LDT to point to the cthread data.
741 * The involves copying in the pre-initialized descriptor.
742 */
743 ldtp = (struct real_descriptor *)current_ldt();
744 ldtp[sel_idx(USER_CTHREAD)] = pcb->cthread_desc;
745 if (pcb->uldt_selector != 0)
746 ldtp[sel_idx(pcb->uldt_selector)] = pcb->uldt_desc;
747
748
749 /*
750 * For 64-bit, we additionally set the 64-bit User GS base
751 * address. On return to 64-bit user, the GS.Base MSR will be written.
752 */
753 cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self;
754
755 /*
756 * Set the thread`s LDT or LDT entry.
757 */
758 if (new->task == TASK_NULL || new->task->i386_ldt == 0) {
759 /*
760 * Use system LDT.
761 */
762 ml_cpu_set_ldt(KERNEL_LDT);
763 } else {
764 /*
765 * Task has its own LDT.
766 */
767 user_ldt_set(new);
768 }
769
770 /*
771 * Bump the scheduler generation count in the commpage.
772 * This can be read by user code to detect its preemption.
773 */
774 commpage_sched_gen_inc();
775 }
776 #endif
777
778 /*
779 * Switch to the first thread on a CPU.
780 */
781 void
782 machine_load_context(
783 thread_t new)
784 {
785 #if CONFIG_COUNTERS
786 machine_pmc_cswitch(NULL, new);
787 #endif
788 new->machine.specFlags |= OnProc;
789 act_machine_switch_pcb(new);
790 Load_context(new);
791 }
792
793 /*
794 * Switch to a new thread.
795 * Save the old thread`s kernel state or continuation,
796 * and return it.
797 */
798 thread_t
799 machine_switch_context(
800 thread_t old,
801 thread_continue_t continuation,
802 thread_t new)
803 {
804 #if MACH_RT
805 assert(current_cpu_datap()->cpu_active_stack == old->kernel_stack);
806 #endif
807 #if CONFIG_COUNTERS
808 machine_pmc_cswitch(old, new);
809 #endif
810 /*
811 * Save FP registers if in use.
812 */
813 fpu_save_context(old);
814
815
816 old->machine.specFlags &= ~OnProc;
817 new->machine.specFlags |= OnProc;
818
819 /*
820 * Monitor the stack depth and report new max,
821 * not worrying about races.
822 */
823 vm_offset_t depth = current_stack_depth();
824 if (depth > kernel_stack_depth_max) {
825 kernel_stack_depth_max = depth;
826 KERNEL_DEBUG_CONSTANT(
827 MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DEPTH),
828 (long) depth, 0, 0, 0, 0);
829 }
830
831 /*
832 * Switch address maps if need be, even if not switching tasks.
833 * (A server activation may be "borrowing" a client map.)
834 */
835 PMAP_SWITCH_CONTEXT(old, new, cpu_number())
836
837 /*
838 * Load the rest of the user state for the new thread
839 */
840 act_machine_switch_pcb(new);
841
842 return(Switch_context(old, continuation, new));
843 }
844
845 thread_t
846 machine_processor_shutdown(
847 thread_t thread,
848 void (*doshutdown)(processor_t),
849 processor_t processor)
850 {
851 #if CONFIG_VMX
852 vmx_suspend();
853 #endif
854 fpu_save_context(thread);
855 PMAP_SWITCH_CONTEXT(thread, processor->idle_thread, cpu_number());
856 return(Shutdown_context(thread, doshutdown, processor));
857 }
858
859 /*
860 * act_machine_sv_free
861 * release saveareas associated with an act. if flag is true, release
862 * user level savearea(s) too, else don't
863 */
864 void
865 act_machine_sv_free(__unused thread_t act, __unused int flag)
866 {
867 }
868
869
870 /*
871 * This is where registers that are not normally specified by the mach-o
872 * file on an execve would be nullified, perhaps to avoid a covert channel.
873 */
874 kern_return_t
875 machine_thread_state_initialize(
876 thread_t thread)
877 {
878 /*
879 * If there's an fpu save area, free it.
880 * The initialized state will then be lazily faulted-in, if required.
881 * And if we're target, re-arm the no-fpu trap.
882 */
883 if (thread->machine.pcb->ifps) {
884 (void) fpu_set_fxstate(thread, NULL);
885
886 if (thread == current_thread())
887 clear_fpu();
888 }
889
890 if (thread->machine.pcb->ids) {
891 zfree(ids_zone, thread->machine.pcb->ids);
892 thread->machine.pcb->ids = NULL;
893 }
894
895 return KERN_SUCCESS;
896 }
897
898 uint32_t
899 get_eflags_exportmask(void)
900 {
901 return EFL_USER_SET;
902 }
903
904 /*
905 * x86_SAVED_STATE32 - internal save/restore general register state on 32/64 bit processors
906 * for 32bit tasks only
907 * x86_SAVED_STATE64 - internal save/restore general register state on 64 bit processors
908 * for 64bit tasks only
909 * x86_THREAD_STATE32 - external set/get general register state on 32/64 bit processors
910 * for 32bit tasks only
911 * x86_THREAD_STATE64 - external set/get general register state on 64 bit processors
912 * for 64bit tasks only
913 * x86_SAVED_STATE - external set/get general register state on 32/64 bit processors
914 * for either 32bit or 64bit tasks
915 * x86_FLOAT_STATE32 - internal/external save/restore float and xmm state on 32/64 bit processors
916 * for 32bit tasks only
917 * x86_FLOAT_STATE64 - internal/external save/restore float and xmm state on 64 bit processors
918 * for 64bit tasks only
919 * x86_FLOAT_STATE - external save/restore float and xmm state on 32/64 bit processors
920 * for either 32bit or 64bit tasks
921 * x86_EXCEPTION_STATE32 - external get exception state on 32/64 bit processors
922 * for 32bit tasks only
923 * x86_EXCEPTION_STATE64 - external get exception state on 64 bit processors
924 * for 64bit tasks only
925 * x86_EXCEPTION_STATE - external get exception state on 323/64 bit processors
926 * for either 32bit or 64bit tasks
927 */
928
929
930 static void
931 get_exception_state64(thread_t thread, x86_exception_state64_t *es)
932 {
933 x86_saved_state64_t *saved_state;
934
935 saved_state = USER_REGS64(thread);
936
937 es->trapno = saved_state->isf.trapno;
938 es->err = (typeof(es->err))saved_state->isf.err;
939 es->faultvaddr = saved_state->cr2;
940 }
941
942 static void
943 get_exception_state32(thread_t thread, x86_exception_state32_t *es)
944 {
945 x86_saved_state32_t *saved_state;
946
947 saved_state = USER_REGS32(thread);
948
949 es->trapno = saved_state->trapno;
950 es->err = saved_state->err;
951 es->faultvaddr = saved_state->cr2;
952 }
953
954
955 static int
956 set_thread_state32(thread_t thread, x86_thread_state32_t *ts)
957 {
958 x86_saved_state32_t *saved_state;
959
960
961 saved_state = USER_REGS32(thread);
962
963 /*
964 * Scrub segment selector values:
965 */
966 ts->cs = USER_CS;
967 #ifdef __i386__
968 if (ts->ss == 0) ts->ss = USER_DS;
969 if (ts->ds == 0) ts->ds = USER_DS;
970 if (ts->es == 0) ts->es = USER_DS;
971 #else /* __x86_64__ */
972 /*
973 * On a 64 bit kernel, we always override the data segments,
974 * as the actual selector numbers have changed. This also
975 * means that we don't support setting the data segments
976 * manually any more.
977 */
978 ts->ss = USER_DS;
979 ts->ds = USER_DS;
980 ts->es = USER_DS;
981 #endif
982
983 /* Check segment selectors are safe */
984 if (!valid_user_segment_selectors(ts->cs,
985 ts->ss,
986 ts->ds,
987 ts->es,
988 ts->fs,
989 ts->gs))
990 return(KERN_INVALID_ARGUMENT);
991
992 saved_state->eax = ts->eax;
993 saved_state->ebx = ts->ebx;
994 saved_state->ecx = ts->ecx;
995 saved_state->edx = ts->edx;
996 saved_state->edi = ts->edi;
997 saved_state->esi = ts->esi;
998 saved_state->ebp = ts->ebp;
999 saved_state->uesp = ts->esp;
1000 saved_state->efl = (ts->eflags & ~EFL_USER_CLEAR) | EFL_USER_SET;
1001 saved_state->eip = ts->eip;
1002 saved_state->cs = ts->cs;
1003 saved_state->ss = ts->ss;
1004 saved_state->ds = ts->ds;
1005 saved_state->es = ts->es;
1006 saved_state->fs = ts->fs;
1007 saved_state->gs = ts->gs;
1008
1009 /*
1010 * If the trace trap bit is being set,
1011 * ensure that the user returns via iret
1012 * - which is signaled thusly:
1013 */
1014 if ((saved_state->efl & EFL_TF) && saved_state->cs == SYSENTER_CS)
1015 saved_state->cs = SYSENTER_TF_CS;
1016
1017 return(KERN_SUCCESS);
1018 }
1019
1020 static int
1021 set_thread_state64(thread_t thread, x86_thread_state64_t *ts)
1022 {
1023 x86_saved_state64_t *saved_state;
1024
1025
1026 saved_state = USER_REGS64(thread);
1027
1028 if (!IS_USERADDR64_CANONICAL(ts->rsp) ||
1029 !IS_USERADDR64_CANONICAL(ts->rip))
1030 return(KERN_INVALID_ARGUMENT);
1031
1032 saved_state->r8 = ts->r8;
1033 saved_state->r9 = ts->r9;
1034 saved_state->r10 = ts->r10;
1035 saved_state->r11 = ts->r11;
1036 saved_state->r12 = ts->r12;
1037 saved_state->r13 = ts->r13;
1038 saved_state->r14 = ts->r14;
1039 saved_state->r15 = ts->r15;
1040 saved_state->rax = ts->rax;
1041 saved_state->rbx = ts->rbx;
1042 saved_state->rcx = ts->rcx;
1043 saved_state->rdx = ts->rdx;
1044 saved_state->rdi = ts->rdi;
1045 saved_state->rsi = ts->rsi;
1046 saved_state->rbp = ts->rbp;
1047 saved_state->isf.rsp = ts->rsp;
1048 saved_state->isf.rflags = (ts->rflags & ~EFL_USER_CLEAR) | EFL_USER_SET;
1049 saved_state->isf.rip = ts->rip;
1050 saved_state->isf.cs = USER64_CS;
1051 saved_state->fs = (uint32_t)ts->fs;
1052 saved_state->gs = (uint32_t)ts->gs;
1053
1054 return(KERN_SUCCESS);
1055 }
1056
1057
1058
1059 static void
1060 get_thread_state32(thread_t thread, x86_thread_state32_t *ts)
1061 {
1062 x86_saved_state32_t *saved_state;
1063
1064
1065 saved_state = USER_REGS32(thread);
1066
1067 ts->eax = saved_state->eax;
1068 ts->ebx = saved_state->ebx;
1069 ts->ecx = saved_state->ecx;
1070 ts->edx = saved_state->edx;
1071 ts->edi = saved_state->edi;
1072 ts->esi = saved_state->esi;
1073 ts->ebp = saved_state->ebp;
1074 ts->esp = saved_state->uesp;
1075 ts->eflags = saved_state->efl;
1076 ts->eip = saved_state->eip;
1077 ts->cs = saved_state->cs;
1078 ts->ss = saved_state->ss;
1079 ts->ds = saved_state->ds;
1080 ts->es = saved_state->es;
1081 ts->fs = saved_state->fs;
1082 ts->gs = saved_state->gs;
1083 }
1084
1085
1086 static void
1087 get_thread_state64(thread_t thread, x86_thread_state64_t *ts)
1088 {
1089 x86_saved_state64_t *saved_state;
1090
1091
1092 saved_state = USER_REGS64(thread);
1093
1094 ts->r8 = saved_state->r8;
1095 ts->r9 = saved_state->r9;
1096 ts->r10 = saved_state->r10;
1097 ts->r11 = saved_state->r11;
1098 ts->r12 = saved_state->r12;
1099 ts->r13 = saved_state->r13;
1100 ts->r14 = saved_state->r14;
1101 ts->r15 = saved_state->r15;
1102 ts->rax = saved_state->rax;
1103 ts->rbx = saved_state->rbx;
1104 ts->rcx = saved_state->rcx;
1105 ts->rdx = saved_state->rdx;
1106 ts->rdi = saved_state->rdi;
1107 ts->rsi = saved_state->rsi;
1108 ts->rbp = saved_state->rbp;
1109 ts->rsp = saved_state->isf.rsp;
1110 ts->rflags = saved_state->isf.rflags;
1111 ts->rip = saved_state->isf.rip;
1112 ts->cs = saved_state->isf.cs;
1113 ts->fs = saved_state->fs;
1114 ts->gs = saved_state->gs;
1115 }
1116
1117
1118 void
1119 thread_set_wq_state32(thread_t thread, thread_state_t tstate)
1120 {
1121 x86_thread_state32_t *state;
1122 x86_saved_state32_t *saved_state;
1123 thread_t curth = current_thread();
1124 spl_t s=0;
1125
1126
1127 saved_state = USER_REGS32(thread);
1128
1129 state = (x86_thread_state32_t *)tstate;
1130
1131 if (curth != thread) {
1132 s = splsched();
1133 thread_lock(thread);
1134 }
1135
1136 saved_state->ebp = 0;
1137 saved_state->eip = state->eip;
1138 saved_state->eax = state->eax;
1139 saved_state->ebx = state->ebx;
1140 saved_state->ecx = state->ecx;
1141 saved_state->edx = state->edx;
1142 saved_state->edi = state->edi;
1143 saved_state->esi = state->esi;
1144 saved_state->uesp = state->esp;
1145 saved_state->efl = EFL_USER_SET;
1146
1147 saved_state->cs = USER_CS;
1148 saved_state->ss = USER_DS;
1149 saved_state->ds = USER_DS;
1150 saved_state->es = USER_DS;
1151
1152
1153 if (curth != thread) {
1154 thread_unlock(thread);
1155 splx(s);
1156 }
1157 }
1158
1159
1160 void
1161 thread_set_wq_state64(thread_t thread, thread_state_t tstate)
1162 {
1163 x86_thread_state64_t *state;
1164 x86_saved_state64_t *saved_state;
1165 thread_t curth = current_thread();
1166 spl_t s=0;
1167
1168
1169 saved_state = USER_REGS64(thread);
1170 state = (x86_thread_state64_t *)tstate;
1171
1172 if (curth != thread) {
1173 s = splsched();
1174 thread_lock(thread);
1175 }
1176
1177 saved_state->rbp = 0;
1178 saved_state->rdi = state->rdi;
1179 saved_state->rsi = state->rsi;
1180 saved_state->rdx = state->rdx;
1181 saved_state->rcx = state->rcx;
1182 saved_state->r8 = state->r8;
1183 saved_state->r9 = state->r9;
1184
1185 saved_state->isf.rip = state->rip;
1186 saved_state->isf.rsp = state->rsp;
1187 saved_state->isf.cs = USER64_CS;
1188 saved_state->isf.rflags = EFL_USER_SET;
1189
1190
1191 if (curth != thread) {
1192 thread_unlock(thread);
1193 splx(s);
1194 }
1195 }
1196
1197
1198
1199 /*
1200 * act_machine_set_state:
1201 *
1202 * Set the status of the specified thread.
1203 */
1204
1205 kern_return_t
1206 machine_thread_set_state(
1207 thread_t thr_act,
1208 thread_flavor_t flavor,
1209 thread_state_t tstate,
1210 mach_msg_type_number_t count)
1211 {
1212 switch (flavor) {
1213 case x86_SAVED_STATE32:
1214 {
1215 x86_saved_state32_t *state;
1216 x86_saved_state32_t *saved_state;
1217
1218 if (count < x86_SAVED_STATE32_COUNT)
1219 return(KERN_INVALID_ARGUMENT);
1220
1221 if (thread_is_64bit(thr_act))
1222 return(KERN_INVALID_ARGUMENT);
1223
1224 state = (x86_saved_state32_t *) tstate;
1225
1226 /* Check segment selectors are safe */
1227 if (!valid_user_segment_selectors(state->cs,
1228 state->ss,
1229 state->ds,
1230 state->es,
1231 state->fs,
1232 state->gs))
1233 return KERN_INVALID_ARGUMENT;
1234
1235
1236 saved_state = USER_REGS32(thr_act);
1237
1238 /*
1239 * General registers
1240 */
1241 saved_state->edi = state->edi;
1242 saved_state->esi = state->esi;
1243 saved_state->ebp = state->ebp;
1244 saved_state->uesp = state->uesp;
1245 saved_state->ebx = state->ebx;
1246 saved_state->edx = state->edx;
1247 saved_state->ecx = state->ecx;
1248 saved_state->eax = state->eax;
1249 saved_state->eip = state->eip;
1250
1251 saved_state->efl = (state->efl & ~EFL_USER_CLEAR) | EFL_USER_SET;
1252
1253 /*
1254 * If the trace trap bit is being set,
1255 * ensure that the user returns via iret
1256 * - which is signaled thusly:
1257 */
1258 if ((saved_state->efl & EFL_TF) && state->cs == SYSENTER_CS)
1259 state->cs = SYSENTER_TF_CS;
1260
1261 /*
1262 * User setting segment registers.
1263 * Code and stack selectors have already been
1264 * checked. Others will be reset by 'iret'
1265 * if they are not valid.
1266 */
1267 saved_state->cs = state->cs;
1268 saved_state->ss = state->ss;
1269 saved_state->ds = state->ds;
1270 saved_state->es = state->es;
1271 saved_state->fs = state->fs;
1272 saved_state->gs = state->gs;
1273
1274 break;
1275 }
1276
1277 case x86_SAVED_STATE64:
1278 {
1279 x86_saved_state64_t *state;
1280 x86_saved_state64_t *saved_state;
1281
1282 if (count < x86_SAVED_STATE64_COUNT)
1283 return(KERN_INVALID_ARGUMENT);
1284
1285 if (!thread_is_64bit(thr_act))
1286 return(KERN_INVALID_ARGUMENT);
1287
1288 state = (x86_saved_state64_t *) tstate;
1289
1290 /* Verify that the supplied code segment selector is
1291 * valid. In 64-bit mode, the FS and GS segment overrides
1292 * use the FS.base and GS.base MSRs to calculate
1293 * base addresses, and the trampolines don't directly
1294 * restore the segment registers--hence they are no
1295 * longer relevant for validation.
1296 */
1297 if (!valid_user_code_selector(state->isf.cs))
1298 return KERN_INVALID_ARGUMENT;
1299
1300 /* Check pc and stack are canonical addresses */
1301 if (!IS_USERADDR64_CANONICAL(state->isf.rsp) ||
1302 !IS_USERADDR64_CANONICAL(state->isf.rip))
1303 return KERN_INVALID_ARGUMENT;
1304
1305
1306 saved_state = USER_REGS64(thr_act);
1307
1308 /*
1309 * General registers
1310 */
1311 saved_state->r8 = state->r8;
1312 saved_state->r9 = state->r9;
1313 saved_state->r10 = state->r10;
1314 saved_state->r11 = state->r11;
1315 saved_state->r12 = state->r12;
1316 saved_state->r13 = state->r13;
1317 saved_state->r14 = state->r14;
1318 saved_state->r15 = state->r15;
1319 saved_state->rdi = state->rdi;
1320 saved_state->rsi = state->rsi;
1321 saved_state->rbp = state->rbp;
1322 saved_state->rbx = state->rbx;
1323 saved_state->rdx = state->rdx;
1324 saved_state->rcx = state->rcx;
1325 saved_state->rax = state->rax;
1326 saved_state->isf.rsp = state->isf.rsp;
1327 saved_state->isf.rip = state->isf.rip;
1328
1329 saved_state->isf.rflags = (state->isf.rflags & ~EFL_USER_CLEAR) | EFL_USER_SET;
1330
1331 /*
1332 * User setting segment registers.
1333 * Code and stack selectors have already been
1334 * checked. Others will be reset by 'sys'
1335 * if they are not valid.
1336 */
1337 saved_state->isf.cs = state->isf.cs;
1338 saved_state->isf.ss = state->isf.ss;
1339 saved_state->fs = state->fs;
1340 saved_state->gs = state->gs;
1341
1342 break;
1343 }
1344
1345 case x86_FLOAT_STATE32:
1346 {
1347 if (count != x86_FLOAT_STATE32_COUNT)
1348 return(KERN_INVALID_ARGUMENT);
1349
1350 if (thread_is_64bit(thr_act))
1351 return(KERN_INVALID_ARGUMENT);
1352
1353 return fpu_set_fxstate(thr_act, tstate);
1354 }
1355
1356 case x86_FLOAT_STATE64:
1357 {
1358 if (count != x86_FLOAT_STATE64_COUNT)
1359 return(KERN_INVALID_ARGUMENT);
1360
1361 if ( !thread_is_64bit(thr_act))
1362 return(KERN_INVALID_ARGUMENT);
1363
1364 return fpu_set_fxstate(thr_act, tstate);
1365 }
1366
1367 case x86_FLOAT_STATE:
1368 {
1369 x86_float_state_t *state;
1370
1371 if (count != x86_FLOAT_STATE_COUNT)
1372 return(KERN_INVALID_ARGUMENT);
1373
1374 state = (x86_float_state_t *)tstate;
1375 if (state->fsh.flavor == x86_FLOAT_STATE64 && state->fsh.count == x86_FLOAT_STATE64_COUNT &&
1376 thread_is_64bit(thr_act)) {
1377 return fpu_set_fxstate(thr_act, (thread_state_t)&state->ufs.fs64);
1378 }
1379 if (state->fsh.flavor == x86_FLOAT_STATE32 && state->fsh.count == x86_FLOAT_STATE32_COUNT &&
1380 !thread_is_64bit(thr_act)) {
1381 return fpu_set_fxstate(thr_act, (thread_state_t)&state->ufs.fs32);
1382 }
1383 return(KERN_INVALID_ARGUMENT);
1384 }
1385
1386 case x86_THREAD_STATE32:
1387 {
1388 if (count != x86_THREAD_STATE32_COUNT)
1389 return(KERN_INVALID_ARGUMENT);
1390
1391 if (thread_is_64bit(thr_act))
1392 return(KERN_INVALID_ARGUMENT);
1393
1394 return set_thread_state32(thr_act, (x86_thread_state32_t *)tstate);
1395 }
1396
1397 case x86_THREAD_STATE64:
1398 {
1399 if (count != x86_THREAD_STATE64_COUNT)
1400 return(KERN_INVALID_ARGUMENT);
1401
1402 if (!thread_is_64bit(thr_act))
1403 return(KERN_INVALID_ARGUMENT);
1404
1405 return set_thread_state64(thr_act, (x86_thread_state64_t *)tstate);
1406
1407 }
1408 case x86_THREAD_STATE:
1409 {
1410 x86_thread_state_t *state;
1411
1412 if (count != x86_THREAD_STATE_COUNT)
1413 return(KERN_INVALID_ARGUMENT);
1414
1415 state = (x86_thread_state_t *)tstate;
1416
1417 if (state->tsh.flavor == x86_THREAD_STATE64 &&
1418 state->tsh.count == x86_THREAD_STATE64_COUNT &&
1419 thread_is_64bit(thr_act)) {
1420 return set_thread_state64(thr_act, &state->uts.ts64);
1421 } else if (state->tsh.flavor == x86_THREAD_STATE32 &&
1422 state->tsh.count == x86_THREAD_STATE32_COUNT &&
1423 !thread_is_64bit(thr_act)) {
1424 return set_thread_state32(thr_act, &state->uts.ts32);
1425 } else
1426 return(KERN_INVALID_ARGUMENT);
1427
1428 break;
1429 }
1430 case x86_DEBUG_STATE32:
1431 {
1432 x86_debug_state32_t *state;
1433 kern_return_t ret;
1434
1435 if (thread_is_64bit(thr_act))
1436 return(KERN_INVALID_ARGUMENT);
1437
1438 state = (x86_debug_state32_t *)tstate;
1439
1440 ret = set_debug_state32(thr_act, state);
1441
1442 return ret;
1443 }
1444 case x86_DEBUG_STATE64:
1445 {
1446 x86_debug_state64_t *state;
1447 kern_return_t ret;
1448
1449 if (!thread_is_64bit(thr_act))
1450 return(KERN_INVALID_ARGUMENT);
1451
1452 state = (x86_debug_state64_t *)tstate;
1453
1454 ret = set_debug_state64(thr_act, state);
1455
1456 return ret;
1457 }
1458 case x86_DEBUG_STATE:
1459 {
1460 x86_debug_state_t *state;
1461 kern_return_t ret = KERN_INVALID_ARGUMENT;
1462
1463 if (count != x86_DEBUG_STATE_COUNT)
1464 return (KERN_INVALID_ARGUMENT);
1465
1466 state = (x86_debug_state_t *)tstate;
1467 if (state->dsh.flavor == x86_DEBUG_STATE64 &&
1468 state->dsh.count == x86_DEBUG_STATE64_COUNT &&
1469 thread_is_64bit(thr_act)) {
1470 ret = set_debug_state64(thr_act, &state->uds.ds64);
1471 }
1472 else
1473 if (state->dsh.flavor == x86_DEBUG_STATE32 &&
1474 state->dsh.count == x86_DEBUG_STATE32_COUNT &&
1475 !thread_is_64bit(thr_act)) {
1476 ret = set_debug_state32(thr_act, &state->uds.ds32);
1477 }
1478 return ret;
1479 }
1480 default:
1481 return(KERN_INVALID_ARGUMENT);
1482 }
1483
1484 return(KERN_SUCCESS);
1485 }
1486
1487
1488
1489 /*
1490 * thread_getstatus:
1491 *
1492 * Get the status of the specified thread.
1493 */
1494
1495 kern_return_t
1496 machine_thread_get_state(
1497 thread_t thr_act,
1498 thread_flavor_t flavor,
1499 thread_state_t tstate,
1500 mach_msg_type_number_t *count)
1501 {
1502
1503 switch (flavor) {
1504
1505 case THREAD_STATE_FLAVOR_LIST:
1506 {
1507 if (*count < 3)
1508 return (KERN_INVALID_ARGUMENT);
1509
1510 tstate[0] = i386_THREAD_STATE;
1511 tstate[1] = i386_FLOAT_STATE;
1512 tstate[2] = i386_EXCEPTION_STATE;
1513
1514 *count = 3;
1515 break;
1516 }
1517
1518 case THREAD_STATE_FLAVOR_LIST_NEW:
1519 {
1520 if (*count < 4)
1521 return (KERN_INVALID_ARGUMENT);
1522
1523 tstate[0] = x86_THREAD_STATE;
1524 tstate[1] = x86_FLOAT_STATE;
1525 tstate[2] = x86_EXCEPTION_STATE;
1526 tstate[3] = x86_DEBUG_STATE;
1527
1528 *count = 4;
1529 break;
1530 }
1531
1532 case x86_SAVED_STATE32:
1533 {
1534 x86_saved_state32_t *state;
1535 x86_saved_state32_t *saved_state;
1536
1537 if (*count < x86_SAVED_STATE32_COUNT)
1538 return(KERN_INVALID_ARGUMENT);
1539
1540 if (thread_is_64bit(thr_act))
1541 return(KERN_INVALID_ARGUMENT);
1542
1543 state = (x86_saved_state32_t *) tstate;
1544 saved_state = USER_REGS32(thr_act);
1545
1546 /*
1547 * First, copy everything:
1548 */
1549 *state = *saved_state;
1550 state->ds = saved_state->ds & 0xffff;
1551 state->es = saved_state->es & 0xffff;
1552 state->fs = saved_state->fs & 0xffff;
1553 state->gs = saved_state->gs & 0xffff;
1554
1555 *count = x86_SAVED_STATE32_COUNT;
1556 break;
1557 }
1558
1559 case x86_SAVED_STATE64:
1560 {
1561 x86_saved_state64_t *state;
1562 x86_saved_state64_t *saved_state;
1563
1564 if (*count < x86_SAVED_STATE64_COUNT)
1565 return(KERN_INVALID_ARGUMENT);
1566
1567 if (!thread_is_64bit(thr_act))
1568 return(KERN_INVALID_ARGUMENT);
1569
1570 state = (x86_saved_state64_t *)tstate;
1571 saved_state = USER_REGS64(thr_act);
1572
1573 /*
1574 * First, copy everything:
1575 */
1576 *state = *saved_state;
1577 state->fs = saved_state->fs & 0xffff;
1578 state->gs = saved_state->gs & 0xffff;
1579
1580 *count = x86_SAVED_STATE64_COUNT;
1581 break;
1582 }
1583
1584 case x86_FLOAT_STATE32:
1585 {
1586 if (*count < x86_FLOAT_STATE32_COUNT)
1587 return(KERN_INVALID_ARGUMENT);
1588
1589 if (thread_is_64bit(thr_act))
1590 return(KERN_INVALID_ARGUMENT);
1591
1592 *count = x86_FLOAT_STATE32_COUNT;
1593
1594 return fpu_get_fxstate(thr_act, tstate);
1595 }
1596
1597 case x86_FLOAT_STATE64:
1598 {
1599 if (*count < x86_FLOAT_STATE64_COUNT)
1600 return(KERN_INVALID_ARGUMENT);
1601
1602 if ( !thread_is_64bit(thr_act))
1603 return(KERN_INVALID_ARGUMENT);
1604
1605 *count = x86_FLOAT_STATE64_COUNT;
1606
1607 return fpu_get_fxstate(thr_act, tstate);
1608 }
1609
1610 case x86_FLOAT_STATE:
1611 {
1612 x86_float_state_t *state;
1613 kern_return_t kret;
1614
1615 if (*count < x86_FLOAT_STATE_COUNT)
1616 return(KERN_INVALID_ARGUMENT);
1617
1618 state = (x86_float_state_t *)tstate;
1619
1620 /*
1621 * no need to bzero... currently
1622 * x86_FLOAT_STATE64_COUNT == x86_FLOAT_STATE32_COUNT
1623 */
1624 if (thread_is_64bit(thr_act)) {
1625 state->fsh.flavor = x86_FLOAT_STATE64;
1626 state->fsh.count = x86_FLOAT_STATE64_COUNT;
1627
1628 kret = fpu_get_fxstate(thr_act, (thread_state_t)&state->ufs.fs64);
1629 } else {
1630 state->fsh.flavor = x86_FLOAT_STATE32;
1631 state->fsh.count = x86_FLOAT_STATE32_COUNT;
1632
1633 kret = fpu_get_fxstate(thr_act, (thread_state_t)&state->ufs.fs32);
1634 }
1635 *count = x86_FLOAT_STATE_COUNT;
1636
1637 return(kret);
1638 }
1639
1640 case x86_THREAD_STATE32:
1641 {
1642 if (*count < x86_THREAD_STATE32_COUNT)
1643 return(KERN_INVALID_ARGUMENT);
1644
1645 if (thread_is_64bit(thr_act))
1646 return(KERN_INVALID_ARGUMENT);
1647
1648 *count = x86_THREAD_STATE32_COUNT;
1649
1650 get_thread_state32(thr_act, (x86_thread_state32_t *)tstate);
1651 break;
1652 }
1653
1654 case x86_THREAD_STATE64:
1655 {
1656 if (*count < x86_THREAD_STATE64_COUNT)
1657 return(KERN_INVALID_ARGUMENT);
1658
1659 if ( !thread_is_64bit(thr_act))
1660 return(KERN_INVALID_ARGUMENT);
1661
1662 *count = x86_THREAD_STATE64_COUNT;
1663
1664 get_thread_state64(thr_act, (x86_thread_state64_t *)tstate);
1665 break;
1666 }
1667
1668 case x86_THREAD_STATE:
1669 {
1670 x86_thread_state_t *state;
1671
1672 if (*count < x86_THREAD_STATE_COUNT)
1673 return(KERN_INVALID_ARGUMENT);
1674
1675 state = (x86_thread_state_t *)tstate;
1676
1677 bzero((char *)state, sizeof(x86_thread_state_t));
1678
1679 if (thread_is_64bit(thr_act)) {
1680 state->tsh.flavor = x86_THREAD_STATE64;
1681 state->tsh.count = x86_THREAD_STATE64_COUNT;
1682
1683 get_thread_state64(thr_act, &state->uts.ts64);
1684 } else {
1685 state->tsh.flavor = x86_THREAD_STATE32;
1686 state->tsh.count = x86_THREAD_STATE32_COUNT;
1687
1688 get_thread_state32(thr_act, &state->uts.ts32);
1689 }
1690 *count = x86_THREAD_STATE_COUNT;
1691
1692 break;
1693 }
1694
1695
1696 case x86_EXCEPTION_STATE32:
1697 {
1698 if (*count < x86_EXCEPTION_STATE32_COUNT)
1699 return(KERN_INVALID_ARGUMENT);
1700
1701 if (thread_is_64bit(thr_act))
1702 return(KERN_INVALID_ARGUMENT);
1703
1704 *count = x86_EXCEPTION_STATE32_COUNT;
1705
1706 get_exception_state32(thr_act, (x86_exception_state32_t *)tstate);
1707 break;
1708 }
1709
1710 case x86_EXCEPTION_STATE64:
1711 {
1712 if (*count < x86_EXCEPTION_STATE64_COUNT)
1713 return(KERN_INVALID_ARGUMENT);
1714
1715 if ( !thread_is_64bit(thr_act))
1716 return(KERN_INVALID_ARGUMENT);
1717
1718 *count = x86_EXCEPTION_STATE64_COUNT;
1719
1720 get_exception_state64(thr_act, (x86_exception_state64_t *)tstate);
1721 break;
1722 }
1723
1724 case x86_EXCEPTION_STATE:
1725 {
1726 x86_exception_state_t *state;
1727
1728 if (*count < x86_EXCEPTION_STATE_COUNT)
1729 return(KERN_INVALID_ARGUMENT);
1730
1731 state = (x86_exception_state_t *)tstate;
1732
1733 bzero((char *)state, sizeof(x86_exception_state_t));
1734
1735 if (thread_is_64bit(thr_act)) {
1736 state->esh.flavor = x86_EXCEPTION_STATE64;
1737 state->esh.count = x86_EXCEPTION_STATE64_COUNT;
1738
1739 get_exception_state64(thr_act, &state->ues.es64);
1740 } else {
1741 state->esh.flavor = x86_EXCEPTION_STATE32;
1742 state->esh.count = x86_EXCEPTION_STATE32_COUNT;
1743
1744 get_exception_state32(thr_act, &state->ues.es32);
1745 }
1746 *count = x86_EXCEPTION_STATE_COUNT;
1747
1748 break;
1749 }
1750 case x86_DEBUG_STATE32:
1751 {
1752 if (*count < x86_DEBUG_STATE32_COUNT)
1753 return(KERN_INVALID_ARGUMENT);
1754
1755 if (thread_is_64bit(thr_act))
1756 return(KERN_INVALID_ARGUMENT);
1757
1758 get_debug_state32(thr_act, (x86_debug_state32_t *)tstate);
1759
1760 *count = x86_DEBUG_STATE32_COUNT;
1761
1762 break;
1763 }
1764 case x86_DEBUG_STATE64:
1765 {
1766 if (*count < x86_DEBUG_STATE64_COUNT)
1767 return(KERN_INVALID_ARGUMENT);
1768
1769 if (!thread_is_64bit(thr_act))
1770 return(KERN_INVALID_ARGUMENT);
1771
1772 get_debug_state64(thr_act, (x86_debug_state64_t *)tstate);
1773
1774 *count = x86_DEBUG_STATE64_COUNT;
1775
1776 break;
1777 }
1778 case x86_DEBUG_STATE:
1779 {
1780 x86_debug_state_t *state;
1781
1782 if (*count < x86_DEBUG_STATE_COUNT)
1783 return(KERN_INVALID_ARGUMENT);
1784
1785 state = (x86_debug_state_t *)tstate;
1786
1787 bzero(state, sizeof *state);
1788
1789 if (thread_is_64bit(thr_act)) {
1790 state->dsh.flavor = x86_DEBUG_STATE64;
1791 state->dsh.count = x86_DEBUG_STATE64_COUNT;
1792
1793 get_debug_state64(thr_act, &state->uds.ds64);
1794 } else {
1795 state->dsh.flavor = x86_DEBUG_STATE32;
1796 state->dsh.count = x86_DEBUG_STATE32_COUNT;
1797
1798 get_debug_state32(thr_act, &state->uds.ds32);
1799 }
1800 *count = x86_DEBUG_STATE_COUNT;
1801 break;
1802 }
1803 default:
1804 return(KERN_INVALID_ARGUMENT);
1805 }
1806
1807 return(KERN_SUCCESS);
1808 }
1809
1810 kern_return_t
1811 machine_thread_get_kern_state(
1812 thread_t thread,
1813 thread_flavor_t flavor,
1814 thread_state_t tstate,
1815 mach_msg_type_number_t *count)
1816 {
1817 x86_saved_state_t *int_state = current_cpu_datap()->cpu_int_state;
1818
1819 /*
1820 * This works only for an interrupted kernel thread
1821 */
1822 if (thread != current_thread() || int_state == NULL)
1823 return KERN_FAILURE;
1824
1825 switch (flavor) {
1826 case x86_THREAD_STATE32: {
1827 x86_thread_state32_t *state;
1828 x86_saved_state32_t *saved_state;
1829
1830 if (!is_saved_state32(int_state) ||
1831 *count < x86_THREAD_STATE32_COUNT)
1832 return (KERN_INVALID_ARGUMENT);
1833
1834 state = (x86_thread_state32_t *) tstate;
1835
1836 saved_state = saved_state32(int_state);
1837 /*
1838 * General registers.
1839 */
1840 state->eax = saved_state->eax;
1841 state->ebx = saved_state->ebx;
1842 state->ecx = saved_state->ecx;
1843 state->edx = saved_state->edx;
1844 state->edi = saved_state->edi;
1845 state->esi = saved_state->esi;
1846 state->ebp = saved_state->ebp;
1847 state->esp = saved_state->uesp;
1848 state->eflags = saved_state->efl;
1849 state->eip = saved_state->eip;
1850 state->cs = saved_state->cs;
1851 state->ss = saved_state->ss;
1852 state->ds = saved_state->ds & 0xffff;
1853 state->es = saved_state->es & 0xffff;
1854 state->fs = saved_state->fs & 0xffff;
1855 state->gs = saved_state->gs & 0xffff;
1856
1857 *count = x86_THREAD_STATE32_COUNT;
1858
1859 return KERN_SUCCESS;
1860 }
1861
1862 case x86_THREAD_STATE64: {
1863 x86_thread_state64_t *state;
1864 x86_saved_state64_t *saved_state;
1865
1866 if (!is_saved_state64(int_state) ||
1867 *count < x86_THREAD_STATE64_COUNT)
1868 return (KERN_INVALID_ARGUMENT);
1869
1870 state = (x86_thread_state64_t *) tstate;
1871
1872 saved_state = saved_state64(int_state);
1873 /*
1874 * General registers.
1875 */
1876 state->rax = saved_state->rax;
1877 state->rbx = saved_state->rbx;
1878 state->rcx = saved_state->rcx;
1879 state->rdx = saved_state->rdx;
1880 state->rdi = saved_state->rdi;
1881 state->rsi = saved_state->rsi;
1882 state->rbp = saved_state->rbp;
1883 state->rsp = saved_state->isf.rsp;
1884 state->r8 = saved_state->r8;
1885 state->r9 = saved_state->r9;
1886 state->r10 = saved_state->r10;
1887 state->r11 = saved_state->r11;
1888 state->r12 = saved_state->r12;
1889 state->r13 = saved_state->r13;
1890 state->r14 = saved_state->r14;
1891 state->r15 = saved_state->r15;
1892
1893 state->rip = saved_state->isf.rip;
1894 state->rflags = saved_state->isf.rflags;
1895 state->cs = saved_state->isf.cs;
1896 state->fs = saved_state->fs & 0xffff;
1897 state->gs = saved_state->gs & 0xffff;
1898 *count = x86_THREAD_STATE64_COUNT;
1899
1900 return KERN_SUCCESS;
1901 }
1902
1903 case x86_THREAD_STATE: {
1904 x86_thread_state_t *state = NULL;
1905
1906 if (*count < x86_THREAD_STATE_COUNT)
1907 return (KERN_INVALID_ARGUMENT);
1908
1909 state = (x86_thread_state_t *) tstate;
1910
1911 if (is_saved_state32(int_state)) {
1912 x86_saved_state32_t *saved_state = saved_state32(int_state);
1913
1914 state->tsh.flavor = x86_THREAD_STATE32;
1915 state->tsh.count = x86_THREAD_STATE32_COUNT;
1916
1917 /*
1918 * General registers.
1919 */
1920 state->uts.ts32.eax = saved_state->eax;
1921 state->uts.ts32.ebx = saved_state->ebx;
1922 state->uts.ts32.ecx = saved_state->ecx;
1923 state->uts.ts32.edx = saved_state->edx;
1924 state->uts.ts32.edi = saved_state->edi;
1925 state->uts.ts32.esi = saved_state->esi;
1926 state->uts.ts32.ebp = saved_state->ebp;
1927 state->uts.ts32.esp = saved_state->uesp;
1928 state->uts.ts32.eflags = saved_state->efl;
1929 state->uts.ts32.eip = saved_state->eip;
1930 state->uts.ts32.cs = saved_state->cs;
1931 state->uts.ts32.ss = saved_state->ss;
1932 state->uts.ts32.ds = saved_state->ds & 0xffff;
1933 state->uts.ts32.es = saved_state->es & 0xffff;
1934 state->uts.ts32.fs = saved_state->fs & 0xffff;
1935 state->uts.ts32.gs = saved_state->gs & 0xffff;
1936 } else if (is_saved_state64(int_state)) {
1937 x86_saved_state64_t *saved_state = saved_state64(int_state);
1938
1939 state->tsh.flavor = x86_THREAD_STATE64;
1940 state->tsh.count = x86_THREAD_STATE64_COUNT;
1941
1942 /*
1943 * General registers.
1944 */
1945 state->uts.ts64.rax = saved_state->rax;
1946 state->uts.ts64.rbx = saved_state->rbx;
1947 state->uts.ts64.rcx = saved_state->rcx;
1948 state->uts.ts64.rdx = saved_state->rdx;
1949 state->uts.ts64.rdi = saved_state->rdi;
1950 state->uts.ts64.rsi = saved_state->rsi;
1951 state->uts.ts64.rbp = saved_state->rbp;
1952 state->uts.ts64.rsp = saved_state->isf.rsp;
1953 state->uts.ts64.r8 = saved_state->r8;
1954 state->uts.ts64.r9 = saved_state->r9;
1955 state->uts.ts64.r10 = saved_state->r10;
1956 state->uts.ts64.r11 = saved_state->r11;
1957 state->uts.ts64.r12 = saved_state->r12;
1958 state->uts.ts64.r13 = saved_state->r13;
1959 state->uts.ts64.r14 = saved_state->r14;
1960 state->uts.ts64.r15 = saved_state->r15;
1961
1962 state->uts.ts64.rip = saved_state->isf.rip;
1963 state->uts.ts64.rflags = saved_state->isf.rflags;
1964 state->uts.ts64.cs = saved_state->isf.cs;
1965 state->uts.ts64.fs = saved_state->fs & 0xffff;
1966 state->uts.ts64.gs = saved_state->gs & 0xffff;
1967 } else {
1968 panic("unknown thread state");
1969 }
1970
1971 *count = x86_THREAD_STATE_COUNT;
1972 return KERN_SUCCESS;
1973 }
1974 }
1975 return KERN_FAILURE;
1976 }
1977
1978
1979 /*
1980 * Initialize the machine-dependent state for a new thread.
1981 */
1982 kern_return_t
1983 machine_thread_create(
1984 thread_t thread,
1985 task_t task)
1986 {
1987 pcb_t pcb = &thread->machine.xxx_pcb;
1988 x86_saved_state_t *iss;
1989
1990 #if NCOPY_WINDOWS > 0
1991 inval_copy_windows(thread);
1992
1993 thread->machine.physwindow_pte = 0;
1994 thread->machine.physwindow_busy = 0;
1995 #endif
1996
1997 /*
1998 * Allocate pcb only if required.
1999 */
2000 if (pcb->sf == NULL) {
2001 pcb->sf = zalloc(iss_zone);
2002 if (pcb->sf == NULL)
2003 panic("iss_zone");
2004 }
2005
2006 if (task_has_64BitAddr(task)) {
2007 x86_sframe64_t *sf64;
2008
2009 sf64 = (x86_sframe64_t *) pcb->sf;
2010
2011 bzero((char *)sf64, sizeof(x86_sframe64_t));
2012
2013 iss = (x86_saved_state_t *) &sf64->ssf;
2014 iss->flavor = x86_SAVED_STATE64;
2015 /*
2016 * Guarantee that the bootstrapped thread will be in user
2017 * mode.
2018 */
2019 iss->ss_64.isf.rflags = EFL_USER_SET;
2020 iss->ss_64.isf.cs = USER64_CS;
2021 iss->ss_64.isf.ss = USER_DS;
2022 iss->ss_64.fs = USER_DS;
2023 iss->ss_64.gs = USER_DS;
2024 } else {
2025 if (cpu_mode_is64bit()) {
2026 x86_sframe_compat32_t *sfc32;
2027
2028 sfc32 = (x86_sframe_compat32_t *)pcb->sf;
2029
2030 bzero((char *)sfc32, sizeof(x86_sframe_compat32_t));
2031
2032 iss = (x86_saved_state_t *) &sfc32->ssf.iss32;
2033 iss->flavor = x86_SAVED_STATE32;
2034 #if defined(__i386__)
2035 #if DEBUG
2036 {
2037 x86_saved_state_compat32_t *xssc;
2038
2039 xssc = (x86_saved_state_compat32_t *) iss;
2040
2041 xssc->pad_for_16byte_alignment[0] = 0x64326432;
2042 xssc->pad_for_16byte_alignment[1] = 0x64326432;
2043 }
2044 #endif /* DEBUG */
2045 } else {
2046 x86_sframe32_t *sf32;
2047 struct real_descriptor *ldtp;
2048 pmap_paddr_t paddr;
2049
2050 sf32 = (x86_sframe32_t *) pcb->sf;
2051
2052 bzero((char *)sf32, sizeof(x86_sframe32_t));
2053
2054 iss = (x86_saved_state_t *) &sf32->ssf;
2055 iss->flavor = x86_SAVED_STATE32;
2056 pcb->iss_pte0 = pte_kernel_rw(kvtophys((vm_offset_t)iss));
2057 if (0 == (paddr = pa_to_pte(kvtophys((vm_offset_t)iss + PAGE_SIZE))))
2058 pcb->iss_pte1 = INTEL_PTE_INVALID;
2059 else
2060 pcb->iss_pte1 = pte_kernel_rw(paddr);
2061
2062
2063 ldtp = (struct real_descriptor *)
2064 pmap_index_to_virt(HIGH_FIXED_LDT_BEGIN);
2065 pcb->cthread_desc = ldtp[sel_idx(USER_DS)];
2066 pcb->uldt_desc = ldtp[sel_idx(USER_DS)];
2067 #endif /* __i386__ */
2068 }
2069 /*
2070 * Guarantee that the bootstrapped thread will be in user
2071 * mode.
2072 */
2073 iss->ss_32.cs = USER_CS;
2074 iss->ss_32.ss = USER_DS;
2075 iss->ss_32.ds = USER_DS;
2076 iss->ss_32.es = USER_DS;
2077 iss->ss_32.fs = USER_DS;
2078 iss->ss_32.gs = USER_DS;
2079 iss->ss_32.efl = EFL_USER_SET;
2080
2081 }
2082 pcb->iss = iss;
2083
2084 thread->machine.pcb = pcb;
2085 simple_lock_init(&pcb->lock, 0);
2086
2087 pcb->arg_store_valid = 0;
2088 pcb->cthread_self = 0;
2089 pcb->uldt_selector = 0;
2090
2091
2092 return(KERN_SUCCESS);
2093 }
2094
2095 /*
2096 * Machine-dependent cleanup prior to destroying a thread
2097 */
2098 void
2099 machine_thread_destroy(
2100 thread_t thread)
2101 {
2102 register pcb_t pcb = thread->machine.pcb;
2103
2104 assert(pcb);
2105
2106 if (pcb->ifps != 0)
2107 fpu_free(pcb->ifps);
2108 if (pcb->sf != 0) {
2109 zfree(iss_zone, pcb->sf);
2110 pcb->sf = 0;
2111 }
2112 if (pcb->ids) {
2113 zfree(ids_zone, pcb->ids);
2114 pcb->ids = NULL;
2115 }
2116 thread->machine.pcb = (pcb_t)0;
2117
2118 }
2119
2120 void
2121 machine_thread_switch_addrmode(thread_t thread)
2122 {
2123 /*
2124 * We don't want to be preempted until we're done
2125 * - particularly if we're switching the current thread
2126 */
2127 disable_preemption();
2128
2129 /*
2130 * Reset the state saveareas.
2131 */
2132 machine_thread_create(thread, thread->task);
2133
2134 /* If we're switching ourselves, reset the pcb addresses etc. */
2135 if (thread == current_thread()) {
2136 #if defined(__i386__)
2137 if (current_cpu_datap()->cpu_active_cr3 != kernel_pmap->pm_cr3)
2138 pmap_load_kernel_cr3();
2139 #endif /* defined(__i386) */
2140 act_machine_switch_pcb(thread);
2141 }
2142 enable_preemption();
2143 }
2144
2145
2146
2147 /*
2148 * This is used to set the current thr_act/thread
2149 * when starting up a new processor
2150 */
2151 void
2152 machine_set_current_thread(thread_t thread)
2153 {
2154 current_cpu_datap()->cpu_active_thread = thread;
2155 }
2156
2157 /*
2158 * This is called when a task is terminated, and also on exec().
2159 * Clear machine-dependent state that is stored on the task.
2160 */
2161 void
2162 machine_thread_terminate_self(void)
2163 {
2164 task_t self_task = current_task();
2165 if (self_task) {
2166 user_ldt_t user_ldt = self_task->i386_ldt;
2167 if (user_ldt != 0) {
2168 self_task->i386_ldt = 0;
2169 user_ldt_free(user_ldt);
2170 }
2171
2172 if (self_task->task_debug != NULL) {
2173 zfree(ids_zone, self_task->task_debug);
2174 self_task->task_debug = NULL;
2175 }
2176 }
2177 }
2178
2179 void
2180 act_machine_return(
2181 int code
2182 )
2183 {
2184 /*
2185 * This code is called with nothing locked.
2186 * It also returns with nothing locked, if it returns.
2187 *
2188 * This routine terminates the current thread activation.
2189 * If this is the only activation associated with its
2190 * thread shuttle, then the entire thread (shuttle plus
2191 * activation) is terminated.
2192 */
2193 assert( code == KERN_TERMINATED );
2194
2195 thread_terminate_self();
2196
2197 /*NOTREACHED*/
2198
2199 panic("act_machine_return(%d): TALKING ZOMBIE! (1)", code);
2200 }
2201
2202
2203 /*
2204 * Perform machine-dependent per-thread initializations
2205 */
2206 void
2207 machine_thread_init(void)
2208 {
2209 if (cpu_mode_is64bit()) {
2210 assert(sizeof(x86_sframe_compat32_t) % 16 == 0);
2211 iss_zone = zinit(sizeof(x86_sframe64_t),
2212 thread_max * sizeof(x86_sframe64_t),
2213 THREAD_CHUNK * sizeof(x86_sframe64_t),
2214 "x86_64 saved state");
2215
2216 ids_zone = zinit(sizeof(x86_debug_state64_t),
2217 thread_max * sizeof(x86_debug_state64_t),
2218 THREAD_CHUNK * sizeof(x86_debug_state64_t),
2219 "x86_64 debug state");
2220
2221 } else {
2222 iss_zone = zinit(sizeof(x86_sframe32_t),
2223 thread_max * sizeof(x86_sframe32_t),
2224 THREAD_CHUNK * sizeof(x86_sframe32_t),
2225 "x86 saved state");
2226 ids_zone = zinit(sizeof(x86_debug_state32_t),
2227 thread_max * (sizeof(x86_debug_state32_t)),
2228 THREAD_CHUNK * (sizeof(x86_debug_state32_t)),
2229 "x86 debug state");
2230 }
2231 fpu_module_init();
2232 }
2233
2234
2235 #if defined(__i386__)
2236 /*
2237 * Some routines for debugging activation code
2238 */
2239 static void dump_handlers(thread_t);
2240 void dump_regs(thread_t);
2241 int dump_act(thread_t thr_act);
2242
2243 static void
2244 dump_handlers(thread_t thr_act)
2245 {
2246 ReturnHandler *rhp = thr_act->handlers;
2247 int counter = 0;
2248
2249 printf("\t");
2250 while (rhp) {
2251 if (rhp == &thr_act->special_handler){
2252 if (rhp->next)
2253 printf("[NON-Zero next ptr(%p)]", rhp->next);
2254 printf("special_handler()->");
2255 break;
2256 }
2257 printf("hdlr_%d(%p)->", counter, rhp->handler);
2258 rhp = rhp->next;
2259 if (++counter > 32) {
2260 printf("Aborting: HUGE handler chain\n");
2261 break;
2262 }
2263 }
2264 printf("HLDR_NULL\n");
2265 }
2266
2267 void
2268 dump_regs(thread_t thr_act)
2269 {
2270 if (thr_act->machine.pcb == NULL)
2271 return;
2272
2273 if (thread_is_64bit(thr_act)) {
2274 x86_saved_state64_t *ssp;
2275
2276 ssp = USER_REGS64(thr_act);
2277
2278 panic("dump_regs: 64bit tasks not yet supported");
2279
2280 } else {
2281 x86_saved_state32_t *ssp;
2282
2283 ssp = USER_REGS32(thr_act);
2284
2285 /*
2286 * Print out user register state
2287 */
2288 printf("\tRegs:\tedi=%x esi=%x ebp=%x ebx=%x edx=%x\n",
2289 ssp->edi, ssp->esi, ssp->ebp, ssp->ebx, ssp->edx);
2290
2291 printf("\t\tecx=%x eax=%x eip=%x efl=%x uesp=%x\n",
2292 ssp->ecx, ssp->eax, ssp->eip, ssp->efl, ssp->uesp);
2293
2294 printf("\t\tcs=%x ss=%x\n", ssp->cs, ssp->ss);
2295 }
2296 }
2297
2298 int
2299 dump_act(thread_t thr_act)
2300 {
2301 if (!thr_act)
2302 return(0);
2303
2304 printf("thread(%p)(%d): task=%p(%d)\n",
2305 thr_act, thr_act->ref_count,
2306 thr_act->task,
2307 thr_act->task ? thr_act->task->ref_count : 0);
2308
2309 printf("\tsusp=%d user_stop=%d active=%x ast=%x\n",
2310 thr_act->suspend_count, thr_act->user_stop_count,
2311 thr_act->active, thr_act->ast);
2312 printf("\tpcb=%p\n", thr_act->machine.pcb);
2313
2314 if (thr_act->kernel_stack) {
2315 vm_offset_t stack = thr_act->kernel_stack;
2316
2317 printf("\tk_stk %lx eip %x ebx %x esp %x iss %p\n",
2318 (long)stack, STACK_IKS(stack)->k_eip, STACK_IKS(stack)->k_ebx,
2319 STACK_IKS(stack)->k_esp, STACK_IEL(stack)->saved_state);
2320 }
2321
2322 dump_handlers(thr_act);
2323 dump_regs(thr_act);
2324 return((int)thr_act);
2325 }
2326 #endif
2327
2328 user_addr_t
2329 get_useraddr(void)
2330 {
2331 thread_t thr_act = current_thread();
2332
2333 if (thr_act->machine.pcb == NULL)
2334 return(0);
2335
2336 if (thread_is_64bit(thr_act)) {
2337 x86_saved_state64_t *iss64;
2338
2339 iss64 = USER_REGS64(thr_act);
2340
2341 return(iss64->isf.rip);
2342 } else {
2343 x86_saved_state32_t *iss32;
2344
2345 iss32 = USER_REGS32(thr_act);
2346
2347 return(iss32->eip);
2348 }
2349 }
2350
2351 /*
2352 * detach and return a kernel stack from a thread
2353 */
2354
2355 vm_offset_t
2356 machine_stack_detach(thread_t thread)
2357 {
2358 vm_offset_t stack;
2359
2360 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DETACH),
2361 (uintptr_t)thread_tid(thread), thread->priority,
2362 thread->sched_pri, 0,
2363 0);
2364
2365 stack = thread->kernel_stack;
2366 thread->kernel_stack = 0;
2367
2368 return (stack);
2369 }
2370
2371 /*
2372 * attach a kernel stack to a thread and initialize it
2373 */
2374
2375 void
2376 machine_stack_attach(
2377 thread_t thread,
2378 vm_offset_t stack)
2379 {
2380 struct x86_kernel_state *statep;
2381
2382 KERNEL_DEBUG(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_ATTACH),
2383 (uintptr_t)thread_tid(thread), thread->priority,
2384 thread->sched_pri, 0, 0);
2385
2386 assert(stack);
2387 thread->kernel_stack = stack;
2388
2389 statep = STACK_IKS(stack);
2390 #if defined(__x86_64__)
2391 statep->k_rip = (unsigned long) Thread_continue;
2392 statep->k_rbx = (unsigned long) thread_continue;
2393 statep->k_rsp = (unsigned long) STACK_IEL(stack);
2394 #else
2395 statep->k_eip = (unsigned long) Thread_continue;
2396 statep->k_ebx = (unsigned long) thread_continue;
2397 statep->k_esp = (unsigned long) STACK_IEL(stack);
2398 #endif
2399
2400 return;
2401 }
2402
2403 /*
2404 * move a stack from old to new thread
2405 */
2406
2407 void
2408 machine_stack_handoff(thread_t old,
2409 thread_t new)
2410 {
2411 vm_offset_t stack;
2412
2413 assert(new);
2414 assert(old);
2415
2416 #if CONFIG_COUNTERS
2417 machine_pmc_cswitch(old, new);
2418 #endif
2419
2420 stack = old->kernel_stack;
2421 if (stack == old->reserved_stack) {
2422 assert(new->reserved_stack);
2423 old->reserved_stack = new->reserved_stack;
2424 new->reserved_stack = stack;
2425 }
2426 old->kernel_stack = 0;
2427 /*
2428 * A full call to machine_stack_attach() is unnecessry
2429 * because old stack is already initialized.
2430 */
2431 new->kernel_stack = stack;
2432
2433 fpu_save_context(old);
2434
2435
2436 old->machine.specFlags &= ~OnProc;
2437 new->machine.specFlags |= OnProc;
2438
2439 PMAP_SWITCH_CONTEXT(old, new, cpu_number());
2440 act_machine_switch_pcb(new);
2441
2442 machine_set_current_thread(new);
2443
2444 return;
2445 }
2446
2447
2448
2449
2450 struct x86_act_context32 {
2451 x86_saved_state32_t ss;
2452 x86_float_state32_t fs;
2453 x86_debug_state32_t ds;
2454 };
2455
2456 struct x86_act_context64 {
2457 x86_saved_state64_t ss;
2458 x86_float_state64_t fs;
2459 x86_debug_state64_t ds;
2460 };
2461
2462
2463
2464 void *
2465 act_thread_csave(void)
2466 {
2467 kern_return_t kret;
2468 mach_msg_type_number_t val;
2469 thread_t thr_act = current_thread();
2470
2471 if (thread_is_64bit(thr_act)) {
2472 struct x86_act_context64 *ic64;
2473
2474 ic64 = (struct x86_act_context64 *)kalloc(sizeof(struct x86_act_context64));
2475
2476 if (ic64 == (struct x86_act_context64 *)NULL)
2477 return((void *)0);
2478
2479 val = x86_SAVED_STATE64_COUNT;
2480 kret = machine_thread_get_state(thr_act, x86_SAVED_STATE64,
2481 (thread_state_t) &ic64->ss, &val);
2482 if (kret != KERN_SUCCESS) {
2483 kfree(ic64, sizeof(struct x86_act_context64));
2484 return((void *)0);
2485 }
2486 val = x86_FLOAT_STATE64_COUNT;
2487 kret = machine_thread_get_state(thr_act, x86_FLOAT_STATE64,
2488 (thread_state_t) &ic64->fs, &val);
2489
2490 if (kret != KERN_SUCCESS) {
2491 kfree(ic64, sizeof(struct x86_act_context64));
2492 return((void *)0);
2493 }
2494
2495 val = x86_DEBUG_STATE64_COUNT;
2496 kret = machine_thread_get_state(thr_act,
2497 x86_DEBUG_STATE64,
2498 (thread_state_t)&ic64->ds,
2499 &val);
2500 if (kret != KERN_SUCCESS) {
2501 kfree(ic64, sizeof(struct x86_act_context64));
2502 return((void *)0);
2503 }
2504 return(ic64);
2505
2506 } else {
2507 struct x86_act_context32 *ic32;
2508
2509 ic32 = (struct x86_act_context32 *)kalloc(sizeof(struct x86_act_context32));
2510
2511 if (ic32 == (struct x86_act_context32 *)NULL)
2512 return((void *)0);
2513
2514 val = x86_SAVED_STATE32_COUNT;
2515 kret = machine_thread_get_state(thr_act, x86_SAVED_STATE32,
2516 (thread_state_t) &ic32->ss, &val);
2517 if (kret != KERN_SUCCESS) {
2518 kfree(ic32, sizeof(struct x86_act_context32));
2519 return((void *)0);
2520 }
2521 val = x86_FLOAT_STATE32_COUNT;
2522 kret = machine_thread_get_state(thr_act, x86_FLOAT_STATE32,
2523 (thread_state_t) &ic32->fs, &val);
2524 if (kret != KERN_SUCCESS) {
2525 kfree(ic32, sizeof(struct x86_act_context32));
2526 return((void *)0);
2527 }
2528
2529 val = x86_DEBUG_STATE32_COUNT;
2530 kret = machine_thread_get_state(thr_act,
2531 x86_DEBUG_STATE32,
2532 (thread_state_t)&ic32->ds,
2533 &val);
2534 if (kret != KERN_SUCCESS) {
2535 kfree(ic32, sizeof(struct x86_act_context32));
2536 return((void *)0);
2537 }
2538 return(ic32);
2539 }
2540 }
2541
2542
2543 void
2544 act_thread_catt(void *ctx)
2545 {
2546 thread_t thr_act = current_thread();
2547 kern_return_t kret;
2548
2549 if (ctx == (void *)NULL)
2550 return;
2551
2552 if (thread_is_64bit(thr_act)) {
2553 struct x86_act_context64 *ic64;
2554
2555 ic64 = (struct x86_act_context64 *)ctx;
2556
2557 kret = machine_thread_set_state(thr_act, x86_SAVED_STATE64,
2558 (thread_state_t) &ic64->ss, x86_SAVED_STATE64_COUNT);
2559 if (kret == KERN_SUCCESS) {
2560 machine_thread_set_state(thr_act, x86_FLOAT_STATE64,
2561 (thread_state_t) &ic64->fs, x86_FLOAT_STATE64_COUNT);
2562 }
2563 kfree(ic64, sizeof(struct x86_act_context64));
2564 } else {
2565 struct x86_act_context32 *ic32;
2566
2567 ic32 = (struct x86_act_context32 *)ctx;
2568
2569 kret = machine_thread_set_state(thr_act, x86_SAVED_STATE32,
2570 (thread_state_t) &ic32->ss, x86_SAVED_STATE32_COUNT);
2571 if (kret == KERN_SUCCESS) {
2572 kret = machine_thread_set_state(thr_act, x86_FLOAT_STATE32,
2573 (thread_state_t) &ic32->fs, x86_FLOAT_STATE32_COUNT);
2574 if (kret == KERN_SUCCESS && thr_act->machine.pcb->ids)
2575 machine_thread_set_state(thr_act,
2576 x86_DEBUG_STATE32,
2577 (thread_state_t)&ic32->ds,
2578 x86_DEBUG_STATE32_COUNT);
2579 }
2580 kfree(ic32, sizeof(struct x86_act_context32));
2581 }
2582 }
2583
2584
2585 void act_thread_cfree(__unused void *ctx)
2586 {
2587 /* XXX - Unused */
2588 }
2589 void x86_toggle_sysenter_arg_store(thread_t thread, boolean_t valid);
2590 void x86_toggle_sysenter_arg_store(thread_t thread, boolean_t valid) {
2591 thread->machine.pcb->arg_store_valid = valid;
2592 }
2593
2594 boolean_t x86_sysenter_arg_store_isvalid(thread_t thread);
2595
2596 boolean_t x86_sysenter_arg_store_isvalid(thread_t thread) {
2597 return (thread->machine.pcb->arg_store_valid);
2598 }
2599
2600 /*
2601 * Duplicate one x86_debug_state32_t to another. "all" parameter
2602 * chooses whether dr4 and dr5 are copied (they are never meant
2603 * to be installed when we do machine_task_set_state() or
2604 * machine_thread_set_state()).
2605 */
2606 void
2607 copy_debug_state32(
2608 x86_debug_state32_t *src,
2609 x86_debug_state32_t *target,
2610 boolean_t all)
2611 {
2612 if (all) {
2613 target->dr4 = src->dr4;
2614 target->dr5 = src->dr5;
2615 }
2616
2617 target->dr0 = src->dr0;
2618 target->dr1 = src->dr1;
2619 target->dr2 = src->dr2;
2620 target->dr3 = src->dr3;
2621 target->dr6 = src->dr6;
2622 target->dr7 = src->dr7;
2623 }
2624
2625 /*
2626 * Duplicate one x86_debug_state64_t to another. "all" parameter
2627 * chooses whether dr4 and dr5 are copied (they are never meant
2628 * to be installed when we do machine_task_set_state() or
2629 * machine_thread_set_state()).
2630 */
2631 void
2632 copy_debug_state64(
2633 x86_debug_state64_t *src,
2634 x86_debug_state64_t *target,
2635 boolean_t all)
2636 {
2637 if (all) {
2638 target->dr4 = src->dr4;
2639 target->dr5 = src->dr5;
2640 }
2641
2642 target->dr0 = src->dr0;
2643 target->dr1 = src->dr1;
2644 target->dr2 = src->dr2;
2645 target->dr3 = src->dr3;
2646 target->dr6 = src->dr6;
2647 target->dr7 = src->dr7;
2648 }
2649
mirror of XNU