]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Copyright (c) 2000-2012 Apple Inc. All rights reserved. | |
3 | * | |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ | |
5 | * | |
6 | * This file contains Original Code and/or Modifications of Original Code | |
7 | * as defined in and that are subject to the Apple Public Source License | |
8 | * Version 2.0 (the 'License'). You may not use this file except in | |
9 | * compliance with the License. The rights granted to you under the License | |
10 | * may not be used to create, or enable the creation or redistribution of, | |
11 | * unlawful or unlicensed copies of an Apple operating system, or to | |
12 | * circumvent, violate, or enable the circumvention or violation of, any | |
13 | * terms of an Apple operating system software license agreement. | |
14 | * | |
15 | * Please obtain a copy of the License at | |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. | |
17 | * | |
18 | * The Original Code and all software distributed under the License are | |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, | |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. | |
23 | * Please see the License for the specific language governing rights and | |
24 | * limitations under the License. | |
25 | * | |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ | |
27 | */ | |
28 | /* | |
29 | * @OSF_COPYRIGHT@ | |
30 | */ | |
31 | /* | |
32 | * Mach Operating System | |
33 | * Copyright (c) 1991,1990 Carnegie Mellon University | |
34 | * All Rights Reserved. | |
35 | * | |
36 | * Permission to use, copy, modify and distribute this software and its | |
37 | * documentation is hereby granted, provided that both the copyright | |
38 | * notice and this permission notice appear in all copies of the | |
39 | * software, derivative works or modified versions, and any portions | |
40 | * thereof, and that both notices appear in supporting documentation. | |
41 | * | |
42 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" | |
43 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR | |
44 | * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. | |
45 | * | |
46 | * Carnegie Mellon requests users of this software to return to | |
47 | * | |
48 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU | |
49 | * School of Computer Science | |
50 | * Carnegie Mellon University | |
51 | * Pittsburgh PA 15213-3890 | |
52 | * | |
53 | * any improvements or extensions that they make and grant Carnegie Mellon | |
54 | * the rights to redistribute these changes. | |
55 | */ | |
56 | ||
57 | #include <mach_debug.h> | |
58 | #include <mach_ldebug.h> | |
59 | ||
60 | #include <sys/kdebug.h> | |
61 | ||
62 | #include <mach/kern_return.h> | |
63 | #include <mach/thread_status.h> | |
64 | #include <mach/vm_param.h> | |
65 | ||
66 | #include <kern/counters.h> | |
67 | #include <kern/kalloc.h> | |
68 | #include <kern/mach_param.h> | |
69 | #include <kern/processor.h> | |
70 | #include <kern/cpu_data.h> | |
71 | #include <kern/cpu_number.h> | |
72 | #include <kern/task.h> | |
73 | #include <kern/thread.h> | |
74 | #include <kern/sched_prim.h> | |
75 | #include <kern/misc_protos.h> | |
76 | #include <kern/assert.h> | |
77 | #include <kern/spl.h> | |
78 | #include <kern/machine.h> | |
79 | #include <ipc/ipc_port.h> | |
80 | #include <vm/vm_kern.h> | |
81 | #include <vm/vm_map.h> | |
82 | #include <vm/pmap.h> | |
83 | #include <vm/vm_protos.h> | |
84 | ||
85 | #include <i386/commpage/commpage.h> | |
86 | #include <i386/cpu_data.h> | |
87 | #include <i386/cpu_number.h> | |
88 | #include <i386/eflags.h> | |
89 | #include <i386/proc_reg.h> | |
90 | #include <i386/tss.h> | |
91 | #include <i386/user_ldt.h> | |
92 | #include <i386/fpu.h> | |
93 | #include <i386/mp_desc.h> | |
94 | #include <i386/misc_protos.h> | |
95 | #include <i386/thread.h> | |
96 | #include <i386/seg.h> | |
97 | #include <i386/machine_routines.h> | |
98 | ||
99 | #if HYPERVISOR | |
100 | #include <kern/hv_support.h> | |
101 | #endif | |
102 | ||
103 | #define ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(_type_) \ | |
104 | extern char assert_is_16byte_multiple_sizeof_ ## _type_ \ | |
105 | [(sizeof(_type_) % 16) == 0 ? 1 : -1] | |
106 | ||
107 | /* Compile-time checks for vital save area sizing: */ | |
108 | ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(x86_64_intr_stack_frame_t); | |
109 | ASSERT_IS_16BYTE_MULTIPLE_SIZEOF(x86_saved_state_t); | |
110 | ||
111 | #define DIRECTION_FLAG_DEBUG (DEBUG | DEVELOPMENT) | |
112 | ||
113 | extern zone_t iss_zone; /* zone for saved_state area */ | |
114 | extern zone_t ids_zone; /* zone for debug_state area */ | |
115 | ||
116 | void | |
117 | act_machine_switch_pcb(__unused thread_t old, thread_t new) | |
118 | { | |
119 | pcb_t pcb = THREAD_TO_PCB(new); | |
120 | cpu_data_t *cdp = current_cpu_datap(); | |
121 | struct real_descriptor *ldtp; | |
122 | mach_vm_offset_t pcb_stack_top; | |
123 | ||
124 | assert(new->kernel_stack != 0); | |
125 | assert(ml_get_interrupts_enabled() == FALSE); | |
126 | #ifdef DIRECTION_FLAG_DEBUG | |
127 | if (x86_get_flags() & EFL_DF) { | |
128 | panic("Direction flag detected: 0x%lx", x86_get_flags()); | |
129 | } | |
130 | #endif | |
131 | ||
132 | /* | |
133 | * Clear segment state | |
134 | * unconditionally for DS/ES/FS but more carefully for GS whose | |
135 | * cached state we track. | |
136 | */ | |
137 | set_ds(NULL_SEG); | |
138 | set_es(NULL_SEG); | |
139 | set_fs(NULL_SEG); | |
140 | ||
141 | if (get_gs() != NULL_SEG) { | |
142 | swapgs(); /* switch to user's GS context */ | |
143 | set_gs(NULL_SEG); | |
144 | swapgs(); /* and back to kernel */ | |
145 | ||
146 | /* record the active machine state lost */ | |
147 | cdp->cpu_uber.cu_user_gs_base = 0; | |
148 | } | |
149 | ||
150 | vm_offset_t isf; | |
151 | ||
152 | /* | |
153 | * Set pointer to PCB's interrupt stack frame in cpu data. | |
154 | * Used by syscall and double-fault trap handlers. | |
155 | */ | |
156 | isf = (vm_offset_t) &pcb->iss->ss_64.isf; | |
157 | cdp->cpu_uber.cu_isf = isf; | |
158 | pcb_stack_top = (vm_offset_t) (pcb->iss + 1); | |
159 | /* require 16-byte alignment */ | |
160 | assert((pcb_stack_top & 0xF) == 0); | |
161 | ||
162 | current_ktss64()->rsp0 = cdp->cpu_desc_index.cdi_sstku; | |
163 | /* | |
164 | * Top of temporary sysenter stack points to pcb stack. | |
165 | * Although this is not normally used by 64-bit users, | |
166 | * it needs to be set in case a sysenter is attempted. | |
167 | */ | |
168 | *current_sstk64() = pcb_stack_top; | |
169 | ||
170 | cdp->cd_estack = cpu_shadowp(cdp->cpu_number)->cd_estack = cdp->cpu_desc_index.cdi_sstku; | |
171 | ||
172 | if (is_saved_state64(pcb->iss)) { | |
173 | ||
174 | cdp->cpu_task_map = new->map->pmap->pm_task_map; | |
175 | ||
176 | /* | |
177 | * Enable the 64-bit user code segment, USER64_CS. | |
178 | * Disable the 32-bit user code segment, USER_CS. | |
179 | */ | |
180 | gdt_desc_p(USER64_CS)->access |= ACC_PL_U; | |
181 | gdt_desc_p(USER_CS)->access &= ~ACC_PL_U; | |
182 | ||
183 | /* | |
184 | * Switch user's GS base if necessary | |
185 | * by setting the Kernel's GS base MSR | |
186 | * - this will become the user's on the swapgs when | |
187 | * returning to user-space. Avoid this for | |
188 | * kernel threads (no user TLS support required) | |
189 | * and verify the memory shadow of the segment base | |
190 | * in the event it was altered in user space. | |
191 | */ | |
192 | if ((pcb->cthread_self != 0) || (new->task != kernel_task)) { | |
193 | if ((cdp->cpu_uber.cu_user_gs_base != pcb->cthread_self) || | |
194 | (pcb->cthread_self != rdmsr64(MSR_IA32_KERNEL_GS_BASE))) { | |
195 | cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self; | |
196 | wrmsr64(MSR_IA32_KERNEL_GS_BASE, pcb->cthread_self); | |
197 | } | |
198 | } | |
199 | } else { | |
200 | ||
201 | cdp->cpu_task_map = TASK_MAP_32BIT; | |
202 | ||
203 | /* | |
204 | * Disable USER64_CS | |
205 | * Enable USER_CS | |
206 | */ | |
207 | ||
208 | /* It's possible that writing to the GDT areas | |
209 | * is expensive, if the processor intercepts those | |
210 | * writes to invalidate its internal segment caches | |
211 | * TODO: perhaps only do this if switching bitness | |
212 | */ | |
213 | gdt_desc_p(USER64_CS)->access &= ~ACC_PL_U; | |
214 | gdt_desc_p(USER_CS)->access |= ACC_PL_U; | |
215 | ||
216 | /* | |
217 | * Set the thread`s cthread (a.k.a pthread) | |
218 | * For 32-bit user this involves setting the USER_CTHREAD | |
219 | * descriptor in the LDT to point to the cthread data. | |
220 | * The involves copying in the pre-initialized descriptor. | |
221 | */ | |
222 | ldtp = (struct real_descriptor *)current_ldt(); | |
223 | ldtp[sel_idx(USER_CTHREAD)] = pcb->cthread_desc; | |
224 | if (pcb->uldt_selector != 0) | |
225 | ldtp[sel_idx(pcb->uldt_selector)] = pcb->uldt_desc; | |
226 | cdp->cpu_uber.cu_user_gs_base = pcb->cthread_self; | |
227 | ||
228 | /* | |
229 | * Set the thread`s LDT or LDT entry. | |
230 | */ | |
231 | if (new->task == TASK_NULL || new->task->i386_ldt == 0) { | |
232 | /* | |
233 | * Use system LDT. | |
234 | */ | |
235 | ml_cpu_set_ldt(KERNEL_LDT); | |
236 | } else { | |
237 | /* | |
238 | * Task has its own LDT. | |
239 | */ | |
240 | user_ldt_set(new); | |
241 | } | |
242 | } | |
243 | ||
244 | /* | |
245 | * Bump the scheduler generation count in the commpage. | |
246 | * This can be read by user code to detect its preemption. | |
247 | */ | |
248 | commpage_sched_gen_inc(); | |
249 | } | |
250 | ||
251 | kern_return_t | |
252 | thread_set_wq_state32(thread_t thread, thread_state_t tstate) | |
253 | { | |
254 | x86_thread_state32_t *state; | |
255 | x86_saved_state32_t *saved_state; | |
256 | thread_t curth = current_thread(); | |
257 | spl_t s=0; | |
258 | ||
259 | pal_register_cache_state(thread, DIRTY); | |
260 | ||
261 | saved_state = USER_REGS32(thread); | |
262 | ||
263 | state = (x86_thread_state32_t *)tstate; | |
264 | ||
265 | if (curth != thread) { | |
266 | s = splsched(); | |
267 | thread_lock(thread); | |
268 | } | |
269 | ||
270 | saved_state->ebp = 0; | |
271 | saved_state->eip = state->eip; | |
272 | saved_state->eax = state->eax; | |
273 | saved_state->ebx = state->ebx; | |
274 | saved_state->ecx = state->ecx; | |
275 | saved_state->edx = state->edx; | |
276 | saved_state->edi = state->edi; | |
277 | saved_state->esi = state->esi; | |
278 | saved_state->uesp = state->esp; | |
279 | saved_state->efl = EFL_USER_SET; | |
280 | ||
281 | saved_state->cs = USER_CS; | |
282 | saved_state->ss = USER_DS; | |
283 | saved_state->ds = USER_DS; | |
284 | saved_state->es = USER_DS; | |
285 | ||
286 | if (curth != thread) { | |
287 | thread_unlock(thread); | |
288 | splx(s); | |
289 | } | |
290 | ||
291 | return KERN_SUCCESS; | |
292 | } | |
293 | ||
294 | ||
295 | kern_return_t | |
296 | thread_set_wq_state64(thread_t thread, thread_state_t tstate) | |
297 | { | |
298 | x86_thread_state64_t *state; | |
299 | x86_saved_state64_t *saved_state; | |
300 | thread_t curth = current_thread(); | |
301 | spl_t s=0; | |
302 | ||
303 | saved_state = USER_REGS64(thread); | |
304 | state = (x86_thread_state64_t *)tstate; | |
305 | ||
306 | /* Disallow setting non-canonical PC or stack */ | |
307 | if (!IS_USERADDR64_CANONICAL(state->rsp) || | |
308 | !IS_USERADDR64_CANONICAL(state->rip)) { | |
309 | return KERN_FAILURE; | |
310 | } | |
311 | ||
312 | pal_register_cache_state(thread, DIRTY); | |
313 | ||
314 | if (curth != thread) { | |
315 | s = splsched(); | |
316 | thread_lock(thread); | |
317 | } | |
318 | ||
319 | saved_state->rbp = 0; | |
320 | saved_state->rdi = state->rdi; | |
321 | saved_state->rsi = state->rsi; | |
322 | saved_state->rdx = state->rdx; | |
323 | saved_state->rcx = state->rcx; | |
324 | saved_state->r8 = state->r8; | |
325 | saved_state->r9 = state->r9; | |
326 | ||
327 | saved_state->isf.rip = state->rip; | |
328 | saved_state->isf.rsp = state->rsp; | |
329 | saved_state->isf.cs = USER64_CS; | |
330 | saved_state->isf.rflags = EFL_USER_SET; | |
331 | ||
332 | if (curth != thread) { | |
333 | thread_unlock(thread); | |
334 | splx(s); | |
335 | } | |
336 | ||
337 | return KERN_SUCCESS; | |
338 | } | |
339 | ||
340 | /* | |
341 | * Initialize the machine-dependent state for a new thread. | |
342 | */ | |
343 | kern_return_t | |
344 | machine_thread_create( | |
345 | thread_t thread, | |
346 | task_t task) | |
347 | { | |
348 | pcb_t pcb = THREAD_TO_PCB(thread); | |
349 | ||
350 | #if NCOPY_WINDOWS > 0 | |
351 | inval_copy_windows(thread); | |
352 | ||
353 | thread->machine.physwindow_pte = 0; | |
354 | thread->machine.physwindow_busy = 0; | |
355 | #endif | |
356 | ||
357 | /* | |
358 | * Allocate save frame only if required. | |
359 | */ | |
360 | if (pcb->iss == NULL) { | |
361 | assert((get_preemption_level() == 0)); | |
362 | pcb->iss = (x86_saved_state_t *) zalloc(iss_zone); | |
363 | if (pcb->iss == NULL) | |
364 | panic("iss_zone"); | |
365 | } | |
366 | ||
367 | /* | |
368 | * Ensure that the synthesized 32-bit state including | |
369 | * the 64-bit interrupt state can be acommodated in the | |
370 | * 64-bit state we allocate for both 32-bit and 64-bit threads. | |
371 | */ | |
372 | assert(sizeof(pcb->iss->ss_32) + sizeof(pcb->iss->ss_64.isf) <= | |
373 | sizeof(pcb->iss->ss_64)); | |
374 | ||
375 | bzero((char *)pcb->iss, sizeof(x86_saved_state_t)); | |
376 | ||
377 | if (task_has_64BitAddr(task)) { | |
378 | pcb->iss->flavor = x86_SAVED_STATE64; | |
379 | ||
380 | pcb->iss->ss_64.isf.cs = USER64_CS; | |
381 | pcb->iss->ss_64.isf.ss = USER_DS; | |
382 | pcb->iss->ss_64.fs = USER_DS; | |
383 | pcb->iss->ss_64.gs = USER_DS; | |
384 | pcb->iss->ss_64.isf.rflags = EFL_USER_SET; | |
385 | } else { | |
386 | pcb->iss->flavor = x86_SAVED_STATE32; | |
387 | ||
388 | pcb->iss->ss_32.cs = USER_CS; | |
389 | pcb->iss->ss_32.ss = USER_DS; | |
390 | pcb->iss->ss_32.ds = USER_DS; | |
391 | pcb->iss->ss_32.es = USER_DS; | |
392 | pcb->iss->ss_32.fs = USER_DS; | |
393 | pcb->iss->ss_32.gs = USER_DS; | |
394 | pcb->iss->ss_32.efl = EFL_USER_SET; | |
395 | } | |
396 | ||
397 | simple_lock_init(&pcb->lock, 0); | |
398 | ||
399 | pcb->cthread_self = 0; | |
400 | pcb->uldt_selector = 0; | |
401 | pcb->thread_gpu_ns = 0; | |
402 | /* Ensure that the "cthread" descriptor describes a valid | |
403 | * segment. | |
404 | */ | |
405 | if ((pcb->cthread_desc.access & ACC_P) == 0) { | |
406 | pcb->cthread_desc = *gdt_desc_p(USER_DS); | |
407 | } | |
408 | ||
409 | return(KERN_SUCCESS); | |
410 | } | |
411 | ||
412 | /* | |
413 | * Machine-dependent cleanup prior to destroying a thread | |
414 | */ | |
415 | void | |
416 | machine_thread_destroy( | |
417 | thread_t thread) | |
418 | { | |
419 | pcb_t pcb = THREAD_TO_PCB(thread); | |
420 | ||
421 | #if HYPERVISOR | |
422 | if (thread->hv_thread_target) { | |
423 | hv_callbacks.thread_destroy(thread->hv_thread_target); | |
424 | thread->hv_thread_target = NULL; | |
425 | } | |
426 | #endif | |
427 | ||
428 | if (pcb->ifps != 0) | |
429 | fpu_free(thread, pcb->ifps); | |
430 | if (pcb->iss != 0) { | |
431 | zfree(iss_zone, pcb->iss); | |
432 | pcb->iss = 0; | |
433 | } | |
434 | if (pcb->ids) { | |
435 | zfree(ids_zone, pcb->ids); | |
436 | pcb->ids = NULL; | |
437 | } | |
438 | } | |
439 | ||
440 | kern_return_t | |
441 | machine_thread_set_tsd_base( | |
442 | thread_t thread, | |
443 | mach_vm_offset_t tsd_base) | |
444 | { | |
445 | ||
446 | if (thread->task == kernel_task) { | |
447 | return KERN_INVALID_ARGUMENT; | |
448 | } | |
449 | ||
450 | if (thread_is_64bit(thread)) { | |
451 | /* check for canonical address, set 0 otherwise */ | |
452 | if (!IS_USERADDR64_CANONICAL(tsd_base)) | |
453 | tsd_base = 0ULL; | |
454 | } else { | |
455 | if (tsd_base > UINT32_MAX) | |
456 | tsd_base = 0ULL; | |
457 | } | |
458 | ||
459 | pcb_t pcb = THREAD_TO_PCB(thread); | |
460 | pcb->cthread_self = tsd_base; | |
461 | ||
462 | if (!thread_is_64bit(thread)) { | |
463 | /* Set up descriptor for later use */ | |
464 | struct real_descriptor desc = { | |
465 | .limit_low = 1, | |
466 | .limit_high = 0, | |
467 | .base_low = tsd_base & 0xffff, | |
468 | .base_med = (tsd_base >> 16) & 0xff, | |
469 | .base_high = (tsd_base >> 24) & 0xff, | |
470 | .access = ACC_P|ACC_PL_U|ACC_DATA_W, | |
471 | .granularity = SZ_32|SZ_G, | |
472 | }; | |
473 | ||
474 | pcb->cthread_desc = desc; | |
475 | saved_state32(pcb->iss)->gs = USER_CTHREAD; | |
476 | } | |
477 | ||
478 | /* For current thread, make the TSD base active immediately */ | |
479 | if (thread == current_thread()) { | |
480 | ||
481 | if (thread_is_64bit(thread)) { | |
482 | cpu_data_t *cdp; | |
483 | ||
484 | mp_disable_preemption(); | |
485 | cdp = current_cpu_datap(); | |
486 | if ((cdp->cpu_uber.cu_user_gs_base != pcb->cthread_self) || | |
487 | (pcb->cthread_self != rdmsr64(MSR_IA32_KERNEL_GS_BASE))) | |
488 | wrmsr64(MSR_IA32_KERNEL_GS_BASE, tsd_base); | |
489 | cdp->cpu_uber.cu_user_gs_base = tsd_base; | |
490 | mp_enable_preemption(); | |
491 | } else { | |
492 | ||
493 | /* assign descriptor */ | |
494 | mp_disable_preemption(); | |
495 | *ldt_desc_p(USER_CTHREAD) = pcb->cthread_desc; | |
496 | mp_enable_preemption(); | |
497 | } | |
498 | } | |
499 | ||
500 | return KERN_SUCCESS; | |
501 | } |