]>
Commit | Line | Data |
---|---|---|
43866e37 | 1 | /* |
6d2010ae | 2 | * Copyright (c) 2003-2010 Apple Inc. All rights reserved. |
43866e37 | 3 | * |
2d21ac55 | 4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
43866e37 | 5 | * |
2d21ac55 A |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License | |
8 | * Version 2.0 (the 'License'). You may not use this file except in | |
9 | * compliance with the License. The rights granted to you under the License | |
10 | * may not be used to create, or enable the creation or redistribution of, | |
11 | * unlawful or unlicensed copies of an Apple operating system, or to | |
12 | * circumvent, violate, or enable the circumvention or violation of, any | |
13 | * terms of an Apple operating system software license agreement. | |
8f6c56a5 | 14 | * |
2d21ac55 A |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. | |
17 | * | |
18 | * The Original Code and all software distributed under the License are | |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
8f6c56a5 A |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
2d21ac55 A |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and | |
24 | * limitations under the License. | |
8f6c56a5 | 25 | * |
2d21ac55 | 26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
43866e37 A |
27 | */ |
28 | ||
55e303ae A |
29 | /* |
30 | * Here's what to do if you want to add a new routine to the comm page: | |
31 | * | |
0c530ab8 | 32 | * 1. Add a definition for it's address in osfmk/i386/cpu_capabilities.h, |
55e303ae A |
33 | * being careful to reserve room for future expansion. |
34 | * | |
35 | * 2. Write one or more versions of the routine, each with it's own | |
36 | * commpage_descriptor. The tricky part is getting the "special", | |
37 | * "musthave", and "canthave" fields right, so that exactly one | |
38 | * version of the routine is selected for every machine. | |
0c530ab8 | 39 | * The source files should be in osfmk/i386/commpage/. |
55e303ae A |
40 | * |
41 | * 3. Add a ptr to your new commpage_descriptor(s) in the "routines" | |
0c530ab8 A |
42 | * array in osfmk/i386/commpage/commpage_asm.s. There are two |
43 | * arrays, one for the 32-bit and one for the 64-bit commpage. | |
55e303ae A |
44 | * |
45 | * 4. Write the code in Libc to use the new routine. | |
46 | */ | |
47 | ||
48 | #include <mach/mach_types.h> | |
49 | #include <mach/machine.h> | |
91447636 | 50 | #include <mach/vm_map.h> |
b0d623f7 | 51 | #include <mach/mach_vm.h> |
7e4a7d39 A |
52 | #include <mach/machine.h> |
53 | #include <i386/cpuid.h> | |
2d21ac55 | 54 | #include <i386/tsc.h> |
6d2010ae | 55 | #include <i386/rtclock_protos.h> |
2d21ac55 | 56 | #include <i386/cpu_data.h> |
b0d623f7 A |
57 | #include <i386/machine_routines.h> |
58 | #include <i386/misc_protos.h> | |
7e4a7d39 | 59 | #include <i386/cpuid.h> |
43866e37 A |
60 | #include <machine/cpu_capabilities.h> |
61 | #include <machine/commpage.h> | |
55e303ae A |
62 | #include <machine/pmap.h> |
63 | #include <vm/vm_kern.h> | |
91447636 | 64 | #include <vm/vm_map.h> |
b0d623f7 | 65 | |
91447636 A |
66 | #include <ipc/ipc_port.h> |
67 | ||
0c530ab8 | 68 | #include <kern/page_decrypt.h> |
6d2010ae | 69 | #include <kern/processor.h> |
4452a7af | 70 | |
0c530ab8 A |
71 | /* the lists of commpage routines are in commpage_asm.s */ |
72 | extern commpage_descriptor* commpage_32_routines[]; | |
73 | extern commpage_descriptor* commpage_64_routines[]; | |
4452a7af | 74 | |
2d21ac55 A |
75 | extern vm_map_t commpage32_map; // the shared submap, set up in vm init |
76 | extern vm_map_t commpage64_map; // the shared submap, set up in vm init | |
316670eb A |
77 | extern vm_map_t commpage_text32_map; // the shared submap, set up in vm init |
78 | extern vm_map_t commpage_text64_map; // the shared submap, set up in vm init | |
79 | ||
4452a7af | 80 | |
0c530ab8 A |
81 | char *commPagePtr32 = NULL; // virtual addr in kernel map of 32-bit commpage |
82 | char *commPagePtr64 = NULL; // ...and of 64-bit commpage | |
316670eb A |
83 | char *commPageTextPtr32 = NULL; // virtual addr in kernel map of 32-bit commpage |
84 | char *commPageTextPtr64 = NULL; // ...and of 64-bit commpage | |
6d2010ae | 85 | uint32_t _cpu_capabilities = 0; // define the capability vector |
6601e61a | 86 | |
0c530ab8 A |
87 | int noVMX = 0; /* if true, do not set kHasAltivec in ppc _cpu_capabilities */ |
88 | ||
b0d623f7 A |
89 | typedef uint32_t commpage_address_t; |
90 | ||
91 | static commpage_address_t next; // next available address in comm page | |
92 | static commpage_address_t cur_routine; // comm page address of "current" routine | |
93 | static boolean_t matched; // true if we've found a match for "current" routine | |
0c530ab8 A |
94 | |
95 | static char *commPagePtr; // virtual addr in kernel map of commpage we are working on | |
b0d623f7 | 96 | static commpage_address_t commPageBaseOffset; // subtract from 32-bit runtime address to get offset in virtual commpage in kernel map |
55e303ae | 97 | |
2d21ac55 A |
98 | static commpage_time_data *time_data32 = NULL; |
99 | static commpage_time_data *time_data64 = NULL; | |
100 | ||
6d2010ae A |
101 | decl_simple_lock_data(static,commpage_active_cpus_lock); |
102 | ||
55e303ae A |
103 | /* Allocate the commpage and add to the shared submap created by vm: |
104 | * 1. allocate a page in the kernel map (RW) | |
105 | * 2. wire it down | |
106 | * 3. make a memory entry out of it | |
107 | * 4. map that entry into the shared comm region map (R-only) | |
108 | */ | |
109 | ||
110 | static void* | |
0c530ab8 | 111 | commpage_allocate( |
2d21ac55 | 112 | vm_map_t submap, // commpage32_map or commpage_map64 |
316670eb A |
113 | size_t area_used, // _COMM_PAGE32_AREA_USED or _COMM_PAGE64_AREA_USED |
114 | vm_prot_t uperm) | |
55e303ae | 115 | { |
2d21ac55 | 116 | vm_offset_t kernel_addr = 0; // address of commpage in kernel map |
0c530ab8 A |
117 | vm_offset_t zero = 0; |
118 | vm_size_t size = area_used; // size actually populated | |
119 | vm_map_entry_t entry; | |
120 | ipc_port_t handle; | |
316670eb | 121 | kern_return_t kr; |
0c530ab8 A |
122 | |
123 | if (submap == NULL) | |
124 | panic("commpage submap is null"); | |
125 | ||
316670eb A |
126 | if ((kr = vm_map(kernel_map,&kernel_addr,area_used,0,VM_FLAGS_ANYWHERE,NULL,0,FALSE,VM_PROT_ALL,VM_PROT_ALL,VM_INHERIT_NONE))) |
127 | panic("cannot allocate commpage %d", kr); | |
0c530ab8 | 128 | |
316670eb A |
129 | if ((kr = vm_map_wire(kernel_map,kernel_addr,kernel_addr+area_used,VM_PROT_DEFAULT,FALSE))) |
130 | panic("cannot wire commpage: %d", kr); | |
0c530ab8 A |
131 | |
132 | /* | |
133 | * Now that the object is created and wired into the kernel map, mark it so that no delay | |
134 | * copy-on-write will ever be performed on it as a result of mapping it into user-space. | |
135 | * If such a delayed copy ever occurred, we could remove the kernel's wired mapping - and | |
136 | * that would be a real disaster. | |
137 | * | |
138 | * JMM - What we really need is a way to create it like this in the first place. | |
139 | */ | |
316670eb A |
140 | if (!(kr = vm_map_lookup_entry( kernel_map, vm_map_trunc_page(kernel_addr), &entry) || entry->is_sub_map)) |
141 | panic("cannot find commpage entry %d", kr); | |
0c530ab8 A |
142 | entry->object.vm_object->copy_strategy = MEMORY_OBJECT_COPY_NONE; |
143 | ||
316670eb | 144 | if ((kr = mach_make_memory_entry( kernel_map, // target map |
0c530ab8 A |
145 | &size, // size |
146 | kernel_addr, // offset (address in kernel map) | |
316670eb | 147 | uperm, // protections as specified |
0c530ab8 | 148 | &handle, // this is the object handle we get |
316670eb A |
149 | NULL ))) // parent_entry (what is this?) |
150 | panic("cannot make entry for commpage %d", kr); | |
0c530ab8 | 151 | |
316670eb | 152 | if ((kr = vm_map_64( submap, // target map (shared submap) |
0c530ab8 A |
153 | &zero, // address (map into 1st page in submap) |
154 | area_used, // size | |
155 | 0, // mask | |
156 | VM_FLAGS_FIXED, // flags (it must be 1st page in submap) | |
157 | handle, // port is the memory entry we just made | |
158 | 0, // offset (map 1st page in memory entry) | |
159 | FALSE, // copy | |
316670eb A |
160 | uperm, // cur_protection (R-only in user map) |
161 | uperm, // max_protection | |
162 | VM_INHERIT_SHARE ))) // inheritance | |
163 | panic("cannot map commpage %d", kr); | |
0c530ab8 A |
164 | |
165 | ipc_port_release(handle); | |
316670eb A |
166 | /* Make the kernel mapping non-executable. This cannot be done |
167 | * at the time of map entry creation as mach_make_memory_entry | |
168 | * cannot handle disjoint permissions at this time. | |
169 | */ | |
170 | kr = vm_protect(kernel_map, kernel_addr, area_used, FALSE, VM_PROT_READ | VM_PROT_WRITE); | |
171 | assert (kr == KERN_SUCCESS); | |
0c530ab8 | 172 | |
b0d623f7 | 173 | return (void*)(intptr_t)kernel_addr; // return address in kernel map |
55e303ae A |
174 | } |
175 | ||
176 | /* Get address (in kernel map) of a commpage field. */ | |
177 | ||
91447636 | 178 | static void* |
55e303ae | 179 | commpage_addr_of( |
b0d623f7 | 180 | commpage_address_t addr_at_runtime ) |
55e303ae | 181 | { |
b0d623f7 | 182 | return (void*) ((uintptr_t)commPagePtr + (addr_at_runtime - commPageBaseOffset)); |
55e303ae A |
183 | } |
184 | ||
185 | /* Determine number of CPUs on this system. We cannot rely on | |
186 | * machine_info.max_cpus this early in the boot. | |
187 | */ | |
188 | static int | |
189 | commpage_cpus( void ) | |
190 | { | |
191 | int cpus; | |
192 | ||
193 | cpus = ml_get_max_cpus(); // NB: this call can block | |
194 | ||
195 | if (cpus == 0) | |
196 | panic("commpage cpus==0"); | |
197 | if (cpus > 0xFF) | |
198 | cpus = 0xFF; | |
199 | ||
200 | return cpus; | |
201 | } | |
43866e37 | 202 | |
55e303ae | 203 | /* Initialize kernel version of _cpu_capabilities vector (used by KEXTs.) */ |
43866e37 | 204 | |
55e303ae A |
205 | static void |
206 | commpage_init_cpu_capabilities( void ) | |
207 | { | |
6d2010ae | 208 | uint32_t bits; |
55e303ae A |
209 | int cpus; |
210 | ml_cpu_info_t cpu_info; | |
43866e37 | 211 | |
55e303ae A |
212 | bits = 0; |
213 | ml_cpu_get_info(&cpu_info); | |
214 | ||
215 | switch (cpu_info.vector_unit) { | |
6d2010ae A |
216 | case 9: |
217 | bits |= kHasAVX1_0; | |
218 | /* fall thru */ | |
2d21ac55 A |
219 | case 8: |
220 | bits |= kHasSSE4_2; | |
221 | /* fall thru */ | |
222 | case 7: | |
223 | bits |= kHasSSE4_1; | |
224 | /* fall thru */ | |
0c530ab8 A |
225 | case 6: |
226 | bits |= kHasSupplementalSSE3; | |
227 | /* fall thru */ | |
55e303ae | 228 | case 5: |
91447636 | 229 | bits |= kHasSSE3; |
55e303ae A |
230 | /* fall thru */ |
231 | case 4: | |
232 | bits |= kHasSSE2; | |
233 | /* fall thru */ | |
234 | case 3: | |
235 | bits |= kHasSSE; | |
236 | /* fall thru */ | |
237 | case 2: | |
238 | bits |= kHasMMX; | |
239 | default: | |
240 | break; | |
241 | } | |
242 | switch (cpu_info.cache_line_size) { | |
243 | case 128: | |
244 | bits |= kCache128; | |
245 | break; | |
246 | case 64: | |
247 | bits |= kCache64; | |
248 | break; | |
249 | case 32: | |
250 | bits |= kCache32; | |
251 | break; | |
252 | default: | |
253 | break; | |
254 | } | |
255 | cpus = commpage_cpus(); // how many CPUs do we have | |
256 | ||
257 | if (cpus == 1) | |
258 | bits |= kUP; | |
259 | ||
260 | bits |= (cpus << kNumCPUsShift); | |
261 | ||
91447636 A |
262 | bits |= kFastThreadLocalStorage; // we use %gs for TLS |
263 | ||
0c530ab8 A |
264 | if (cpu_mode_is64bit()) // k64Bit means processor is 64-bit capable |
265 | bits |= k64Bit; | |
266 | ||
2d21ac55 A |
267 | if (tscFreq <= SLOW_TSC_THRESHOLD) /* is TSC too slow for _commpage_nanotime? */ |
268 | bits |= kSlow; | |
269 | ||
7ddcb079 | 270 | bits |= (cpuid_features() & CPUID_FEATURE_AES) ? kHasAES : 0; |
d1ecb069 | 271 | |
13f56ec4 A |
272 | bits |= (cpuid_features() & CPUID_FEATURE_F16C) ? kHasF16C : 0; |
273 | bits |= (cpuid_features() & CPUID_FEATURE_RDRAND) ? kHasRDRAND : 0; | |
274 | bits |= ((cpuid_leaf7_features() & CPUID_LEAF7_FEATURE_ENFSTRG) && | |
275 | (rdmsr64(MSR_IA32_MISC_ENABLE) & 1ULL )) ? kHasENFSTRG : 0; | |
276 | ||
55e303ae A |
277 | _cpu_capabilities = bits; // set kernel version for use by drivers etc |
278 | } | |
279 | ||
0c530ab8 | 280 | int |
2d21ac55 | 281 | _get_cpu_capabilities(void) |
0c530ab8 A |
282 | { |
283 | return _cpu_capabilities; | |
284 | } | |
285 | ||
55e303ae A |
286 | /* Copy data into commpage. */ |
287 | ||
288 | static void | |
289 | commpage_stuff( | |
b0d623f7 | 290 | commpage_address_t address, |
0c530ab8 | 291 | const void *source, |
55e303ae A |
292 | int length ) |
293 | { | |
294 | void *dest = commpage_addr_of(address); | |
295 | ||
b0d623f7 | 296 | if (address < next) |
6d2010ae | 297 | panic("commpage overlap at address 0x%p, 0x%x < 0x%x", dest, address, next); |
55e303ae A |
298 | |
299 | bcopy(source,dest,length); | |
43866e37 | 300 | |
b0d623f7 | 301 | next = address + length; |
55e303ae A |
302 | } |
303 | ||
304 | /* Copy a routine into comm page if it matches running machine. | |
305 | */ | |
306 | static void | |
307 | commpage_stuff_routine( | |
308 | commpage_descriptor *rd ) | |
309 | { | |
b0d623f7 | 310 | uint32_t must,cant; |
55e303ae A |
311 | |
312 | if (rd->commpage_address != cur_routine) { | |
313 | if ((cur_routine!=0) && (matched==0)) | |
b0d623f7 | 314 | panic("commpage no match for last, next address %08x", rd->commpage_address); |
55e303ae A |
315 | cur_routine = rd->commpage_address; |
316 | matched = 0; | |
317 | } | |
318 | ||
319 | must = _cpu_capabilities & rd->musthave; | |
320 | cant = _cpu_capabilities & rd->canthave; | |
321 | ||
322 | if ((must == rd->musthave) && (cant == 0)) { | |
323 | if (matched) | |
b0d623f7 | 324 | panic("commpage multiple matches for address %08x", rd->commpage_address); |
55e303ae A |
325 | matched = 1; |
326 | ||
327 | commpage_stuff(rd->commpage_address,rd->code_address,rd->code_length); | |
328 | } | |
329 | } | |
330 | ||
0c530ab8 | 331 | /* Fill in the 32- or 64-bit commpage. Called once for each. |
55e303ae A |
332 | */ |
333 | ||
0c530ab8 A |
334 | static void |
335 | commpage_populate_one( | |
2d21ac55 | 336 | vm_map_t submap, // commpage32_map or compage64_map |
0c530ab8 A |
337 | char ** kernAddressPtr, // &commPagePtr32 or &commPagePtr64 |
338 | size_t area_used, // _COMM_PAGE32_AREA_USED or _COMM_PAGE64_AREA_USED | |
b0d623f7 | 339 | commpage_address_t base_offset, // will become commPageBaseOffset |
2d21ac55 | 340 | commpage_time_data** time_data, // &time_data32 or &time_data64 |
316670eb A |
341 | const char* signature, // "commpage 32-bit" or "commpage 64-bit" |
342 | vm_prot_t uperm) | |
55e303ae | 343 | { |
6d2010ae | 344 | uint8_t c1; |
91447636 | 345 | short c2; |
6d2010ae A |
346 | int c4; |
347 | uint64_t c8; | |
348 | uint32_t cfamily; | |
55e303ae | 349 | short version = _COMM_PAGE_THIS_VERSION; |
55e303ae | 350 | |
b0d623f7 | 351 | next = 0; |
0c530ab8 | 352 | cur_routine = 0; |
316670eb | 353 | commPagePtr = (char *)commpage_allocate( submap, (vm_size_t) area_used, uperm ); |
0c530ab8 A |
354 | *kernAddressPtr = commPagePtr; // save address either in commPagePtr32 or 64 |
355 | commPageBaseOffset = base_offset; | |
b0d623f7 | 356 | |
2d21ac55 | 357 | *time_data = commpage_addr_of( _COMM_PAGE_TIME_DATA_START ); |
55e303ae A |
358 | |
359 | /* Stuff in the constants. We move things into the comm page in strictly | |
360 | * ascending order, so we can check for overlap and panic if so. | |
361 | */ | |
b0d623f7 | 362 | commpage_stuff(_COMM_PAGE_SIGNATURE,signature,(int)strlen(signature)); |
6d2010ae | 363 | commpage_stuff(_COMM_PAGE_VERSION,&version,sizeof(short)); |
0c530ab8 A |
364 | commpage_stuff(_COMM_PAGE_CPU_CAPABILITIES,&_cpu_capabilities,sizeof(int)); |
365 | ||
6d2010ae A |
366 | c2 = 32; // default |
367 | if (_cpu_capabilities & kCache64) | |
91447636 A |
368 | c2 = 64; |
369 | else if (_cpu_capabilities & kCache128) | |
370 | c2 = 128; | |
371 | commpage_stuff(_COMM_PAGE_CACHE_LINESIZE,&c2,2); | |
b0d623f7 A |
372 | |
373 | c4 = MP_SPIN_TRIES; | |
374 | commpage_stuff(_COMM_PAGE_SPIN_COUNT,&c4,4); | |
91447636 | 375 | |
6d2010ae A |
376 | /* machine_info valid after ml_get_max_cpus() */ |
377 | c1 = machine_info.physical_cpu_max; | |
378 | commpage_stuff(_COMM_PAGE_PHYSICAL_CPUS,&c1,1); | |
379 | c1 = machine_info.logical_cpu_max; | |
380 | commpage_stuff(_COMM_PAGE_LOGICAL_CPUS,&c1,1); | |
381 | ||
382 | c8 = ml_cpu_cache_size(0); | |
383 | commpage_stuff(_COMM_PAGE_MEMORY_SIZE, &c8, 8); | |
384 | ||
385 | cfamily = cpuid_info()->cpuid_cpufamily; | |
386 | commpage_stuff(_COMM_PAGE_CPUFAMILY, &cfamily, 4); | |
6601e61a | 387 | |
b0d623f7 A |
388 | if (next > _COMM_PAGE_END) |
389 | panic("commpage overflow: next = 0x%08x, commPagePtr = 0x%p", next, commPagePtr); | |
91447636 | 390 | |
43866e37 | 391 | } |
91447636 | 392 | |
0c530ab8 A |
393 | |
394 | /* Fill in commpages: called once, during kernel initialization, from the | |
395 | * startup thread before user-mode code is running. | |
396 | * | |
397 | * See the top of this file for a list of what you have to do to add | |
398 | * a new routine to the commpage. | |
399 | */ | |
91447636 A |
400 | |
401 | void | |
0c530ab8 | 402 | commpage_populate( void ) |
91447636 | 403 | { |
0c530ab8 A |
404 | commpage_init_cpu_capabilities(); |
405 | ||
2d21ac55 | 406 | commpage_populate_one( commpage32_map, |
0c530ab8 A |
407 | &commPagePtr32, |
408 | _COMM_PAGE32_AREA_USED, | |
409 | _COMM_PAGE32_BASE_ADDRESS, | |
2d21ac55 | 410 | &time_data32, |
316670eb A |
411 | "commpage 32-bit", |
412 | VM_PROT_READ); | |
b0d623f7 | 413 | #ifndef __LP64__ |
0c530ab8 A |
414 | pmap_commpage32_init((vm_offset_t) commPagePtr32, _COMM_PAGE32_BASE_ADDRESS, |
415 | _COMM_PAGE32_AREA_USED/INTEL_PGBYTES); | |
b0d623f7 | 416 | #endif |
2d21ac55 | 417 | time_data64 = time_data32; /* if no 64-bit commpage, point to 32-bit */ |
0c530ab8 A |
418 | |
419 | if (_cpu_capabilities & k64Bit) { | |
2d21ac55 | 420 | commpage_populate_one( commpage64_map, |
0c530ab8 A |
421 | &commPagePtr64, |
422 | _COMM_PAGE64_AREA_USED, | |
b0d623f7 | 423 | _COMM_PAGE32_START_ADDRESS, /* commpage address are relative to 32-bit commpage placement */ |
2d21ac55 | 424 | &time_data64, |
316670eb A |
425 | "commpage 64-bit", |
426 | VM_PROT_READ); | |
b0d623f7 | 427 | #ifndef __LP64__ |
0c530ab8 A |
428 | pmap_commpage64_init((vm_offset_t) commPagePtr64, _COMM_PAGE64_BASE_ADDRESS, |
429 | _COMM_PAGE64_AREA_USED/INTEL_PGBYTES); | |
b0d623f7 | 430 | #endif |
0c530ab8 | 431 | } |
6601e61a | 432 | |
6d2010ae A |
433 | simple_lock_init(&commpage_active_cpus_lock, 0); |
434 | ||
435 | commpage_update_active_cpus(); | |
0c530ab8 | 436 | rtc_nanotime_init_commpage(); |
91447636 | 437 | } |
2d21ac55 | 438 | |
316670eb A |
439 | /* Fill in the common routines during kernel initialization. |
440 | * This is called before user-mode code is running. | |
441 | */ | |
442 | void commpage_text_populate( void ){ | |
443 | commpage_descriptor **rd; | |
444 | ||
445 | next =0; | |
446 | cur_routine=0; | |
447 | commPagePtr = (char *) commpage_allocate(commpage_text32_map, (vm_size_t) _COMM_PAGE_TEXT_AREA_USED, VM_PROT_READ | VM_PROT_EXECUTE); | |
448 | commPageTextPtr32 = commPagePtr; | |
449 | ||
450 | char *cptr = commPagePtr; | |
451 | int i=0; | |
452 | for(; i< _COMM_PAGE_TEXT_AREA_USED; i++){ | |
453 | cptr[i]=0xCC; | |
454 | } | |
455 | ||
456 | commPageBaseOffset = _COMM_PAGE_TEXT_START; | |
457 | for (rd = commpage_32_routines; *rd != NULL; rd++) { | |
458 | commpage_stuff_routine(*rd); | |
459 | } | |
460 | if (!matched) | |
461 | panic(" commpage_text no match for last routine "); | |
462 | ||
463 | #ifndef __LP64__ | |
464 | pmap_commpage32_init((vm_offset_t) commPageTextPtr32, _COMM_PAGE_TEXT_START, | |
465 | _COMM_PAGE_TEXT_AREA_USED/INTEL_PGBYTES); | |
466 | #endif | |
467 | ||
468 | if (_cpu_capabilities & k64Bit) { | |
469 | next =0; | |
470 | cur_routine=0; | |
471 | commPagePtr = (char *) commpage_allocate(commpage_text64_map, (vm_size_t) _COMM_PAGE_TEXT_AREA_USED, VM_PROT_READ | VM_PROT_EXECUTE); | |
472 | commPageTextPtr64 = commPagePtr; | |
473 | ||
474 | cptr=commPagePtr; | |
475 | for(i=0; i<_COMM_PAGE_TEXT_AREA_USED; i++){ | |
476 | cptr[i]=0xCC; | |
477 | } | |
478 | ||
479 | for (rd = commpage_64_routines; *rd !=NULL; rd++) { | |
480 | commpage_stuff_routine(*rd); | |
481 | } | |
482 | ||
483 | #ifndef __LP64__ | |
484 | pmap_commpage64_init((vm_offset_t) commPageTextPtr64, _COMM_PAGE_TEXT_START, | |
485 | _COMM_PAGE_TEXT_AREA_USED/INTEL_PGBYTES); | |
486 | #endif | |
487 | } | |
488 | ||
489 | if (!matched) | |
490 | panic(" commpage_text no match for last routine "); | |
491 | ||
492 | if (next > _COMM_PAGE_TEXT_END) | |
493 | panic("commpage text overflow: next=0x%08x, commPagePtr=%p", next, commPagePtr); | |
494 | ||
495 | } | |
2d21ac55 A |
496 | |
497 | /* Update commpage nanotime information. Note that we interleave | |
498 | * setting the 32- and 64-bit commpages, in order to keep nanotime more | |
499 | * nearly in sync between the two environments. | |
500 | * | |
501 | * This routine must be serialized by some external means, ie a lock. | |
502 | */ | |
503 | ||
504 | void | |
505 | commpage_set_nanotime( | |
506 | uint64_t tsc_base, | |
507 | uint64_t ns_base, | |
508 | uint32_t scale, | |
509 | uint32_t shift ) | |
510 | { | |
511 | commpage_time_data *p32 = time_data32; | |
512 | commpage_time_data *p64 = time_data64; | |
513 | static uint32_t generation = 0; | |
514 | uint32_t next_gen; | |
515 | ||
516 | if (p32 == NULL) /* have commpages been allocated yet? */ | |
517 | return; | |
518 | ||
519 | if ( generation != p32->nt_generation ) | |
520 | panic("nanotime trouble 1"); /* possibly not serialized */ | |
521 | if ( ns_base < p32->nt_ns_base ) | |
522 | panic("nanotime trouble 2"); | |
523 | if ((shift != 32) && ((_cpu_capabilities & kSlow)==0) ) | |
524 | panic("nanotime trouble 3"); | |
525 | ||
526 | next_gen = ++generation; | |
527 | if (next_gen == 0) | |
528 | next_gen = ++generation; | |
529 | ||
530 | p32->nt_generation = 0; /* mark invalid, so commpage won't try to use it */ | |
531 | p64->nt_generation = 0; | |
532 | ||
533 | p32->nt_tsc_base = tsc_base; | |
534 | p64->nt_tsc_base = tsc_base; | |
535 | ||
536 | p32->nt_ns_base = ns_base; | |
537 | p64->nt_ns_base = ns_base; | |
538 | ||
539 | p32->nt_scale = scale; | |
540 | p64->nt_scale = scale; | |
541 | ||
542 | p32->nt_shift = shift; | |
543 | p64->nt_shift = shift; | |
544 | ||
545 | p32->nt_generation = next_gen; /* mark data as valid */ | |
546 | p64->nt_generation = next_gen; | |
547 | } | |
548 | ||
549 | ||
550 | /* Disable commpage gettimeofday(), forcing commpage to call through to the kernel. */ | |
551 | ||
552 | void | |
553 | commpage_disable_timestamp( void ) | |
554 | { | |
555 | time_data32->gtod_generation = 0; | |
556 | time_data64->gtod_generation = 0; | |
557 | } | |
558 | ||
559 | ||
560 | /* Update commpage gettimeofday() information. As with nanotime(), we interleave | |
561 | * updates to the 32- and 64-bit commpage, in order to keep time more nearly in sync | |
562 | * between the two environments. | |
563 | * | |
564 | * This routine must be serializeed by some external means, ie a lock. | |
565 | */ | |
566 | ||
567 | void | |
568 | commpage_set_timestamp( | |
569 | uint64_t abstime, | |
570 | uint64_t secs ) | |
571 | { | |
572 | commpage_time_data *p32 = time_data32; | |
573 | commpage_time_data *p64 = time_data64; | |
574 | static uint32_t generation = 0; | |
575 | uint32_t next_gen; | |
576 | ||
577 | next_gen = ++generation; | |
578 | if (next_gen == 0) | |
579 | next_gen = ++generation; | |
580 | ||
581 | p32->gtod_generation = 0; /* mark invalid, so commpage won't try to use it */ | |
582 | p64->gtod_generation = 0; | |
583 | ||
584 | p32->gtod_ns_base = abstime; | |
585 | p64->gtod_ns_base = abstime; | |
586 | ||
587 | p32->gtod_sec_base = secs; | |
588 | p64->gtod_sec_base = secs; | |
589 | ||
590 | p32->gtod_generation = next_gen; /* mark data as valid */ | |
591 | p64->gtod_generation = next_gen; | |
592 | } | |
b0d623f7 A |
593 | |
594 | ||
595 | /* Update _COMM_PAGE_MEMORY_PRESSURE. Called periodically from vm's compute_memory_pressure() */ | |
596 | ||
597 | void | |
598 | commpage_set_memory_pressure( | |
599 | unsigned int pressure ) | |
600 | { | |
601 | char *cp; | |
602 | uint32_t *ip; | |
603 | ||
604 | cp = commPagePtr32; | |
605 | if ( cp ) { | |
606 | cp += (_COMM_PAGE_MEMORY_PRESSURE - _COMM_PAGE32_BASE_ADDRESS); | |
607 | ip = (uint32_t*) cp; | |
608 | *ip = (uint32_t) pressure; | |
609 | } | |
610 | ||
611 | cp = commPagePtr64; | |
612 | if ( cp ) { | |
613 | cp += (_COMM_PAGE_MEMORY_PRESSURE - _COMM_PAGE32_START_ADDRESS); | |
614 | ip = (uint32_t*) cp; | |
615 | *ip = (uint32_t) pressure; | |
616 | } | |
617 | ||
618 | } | |
619 | ||
620 | ||
621 | /* Update _COMM_PAGE_SPIN_COUNT. We might want to reduce when running on a battery, etc. */ | |
622 | ||
623 | void | |
624 | commpage_set_spin_count( | |
625 | unsigned int count ) | |
626 | { | |
627 | char *cp; | |
628 | uint32_t *ip; | |
629 | ||
630 | if (count == 0) /* we test for 0 after decrement, not before */ | |
631 | count = 1; | |
632 | ||
633 | cp = commPagePtr32; | |
634 | if ( cp ) { | |
635 | cp += (_COMM_PAGE_SPIN_COUNT - _COMM_PAGE32_BASE_ADDRESS); | |
636 | ip = (uint32_t*) cp; | |
637 | *ip = (uint32_t) count; | |
638 | } | |
639 | ||
640 | cp = commPagePtr64; | |
641 | if ( cp ) { | |
642 | cp += (_COMM_PAGE_SPIN_COUNT - _COMM_PAGE32_START_ADDRESS); | |
643 | ip = (uint32_t*) cp; | |
644 | *ip = (uint32_t) count; | |
645 | } | |
646 | ||
647 | } | |
648 | ||
6d2010ae A |
649 | /* Updated every time a logical CPU goes offline/online */ |
650 | void | |
651 | commpage_update_active_cpus(void) | |
652 | { | |
653 | char *cp; | |
654 | volatile uint8_t *ip; | |
655 | ||
656 | /* At least 32-bit commpage must be initialized */ | |
657 | if (!commPagePtr32) | |
658 | return; | |
659 | ||
660 | simple_lock(&commpage_active_cpus_lock); | |
661 | ||
662 | cp = commPagePtr32; | |
663 | cp += (_COMM_PAGE_ACTIVE_CPUS - _COMM_PAGE32_BASE_ADDRESS); | |
664 | ip = (volatile uint8_t*) cp; | |
665 | *ip = (uint8_t) processor_avail_count; | |
666 | ||
667 | cp = commPagePtr64; | |
668 | if ( cp ) { | |
669 | cp += (_COMM_PAGE_ACTIVE_CPUS - _COMM_PAGE32_START_ADDRESS); | |
670 | ip = (volatile uint8_t*) cp; | |
671 | *ip = (uint8_t) processor_avail_count; | |
672 | } | |
673 | ||
674 | simple_unlock(&commpage_active_cpus_lock); | |
675 | } | |
676 | ||
316670eb A |
677 | extern user32_addr_t commpage_text32_location; |
678 | extern user64_addr_t commpage_text64_location; | |
b0d623f7 A |
679 | |
680 | /* Check to see if a given address is in the Preemption Free Zone (PFZ) */ | |
681 | ||
682 | uint32_t | |
683 | commpage_is_in_pfz32(uint32_t addr32) | |
684 | { | |
316670eb A |
685 | if ( (addr32 >= (commpage_text32_location + _COMM_TEXT_PFZ_START_OFFSET)) |
686 | && (addr32 < (commpage_text32_location+_COMM_TEXT_PFZ_END_OFFSET))) { | |
b0d623f7 A |
687 | return 1; |
688 | } | |
689 | else | |
690 | return 0; | |
691 | } | |
692 | ||
693 | uint32_t | |
694 | commpage_is_in_pfz64(addr64_t addr64) | |
695 | { | |
316670eb A |
696 | if ( (addr64 >= (commpage_text64_location + _COMM_TEXT_PFZ_START_OFFSET)) |
697 | && (addr64 < (commpage_text64_location + _COMM_TEXT_PFZ_END_OFFSET))) { | |
b0d623f7 A |
698 | return 1; |
699 | } | |
700 | else | |
701 | return 0; | |
702 | } | |
703 |