]>
Commit | Line | Data |
---|---|---|
1c79356b | 1 | /* |
2d21ac55 | 2 | * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved. |
1c79356b | 3 | * |
2d21ac55 | 4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
1c79356b | 5 | * |
2d21ac55 A |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License | |
8 | * Version 2.0 (the 'License'). You may not use this file except in | |
9 | * compliance with the License. The rights granted to you under the License | |
10 | * may not be used to create, or enable the creation or redistribution of, | |
11 | * unlawful or unlicensed copies of an Apple operating system, or to | |
12 | * circumvent, violate, or enable the circumvention or violation of, any | |
13 | * terms of an Apple operating system software license agreement. | |
8f6c56a5 | 14 | * |
2d21ac55 A |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. | |
17 | * | |
18 | * The Original Code and all software distributed under the License are | |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
8f6c56a5 A |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
2d21ac55 A |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and | |
24 | * limitations under the License. | |
8f6c56a5 | 25 | * |
2d21ac55 | 26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
1c79356b A |
27 | */ |
28 | /* | |
1c79356b A |
29 | * HISTORY |
30 | * | |
31 | * 17-Apr-91 Portions from libIO.m, Doug Mitchell at NeXT. | |
32 | * 17-Nov-98 cpp | |
33 | * | |
34 | */ | |
35 | ||
36 | #include <IOKit/system.h> | |
37 | #include <mach/sync_policy.h> | |
38 | #include <machine/machine_routines.h> | |
b0d623f7 | 39 | #include <vm/vm_kern.h> |
1c79356b A |
40 | #include <libkern/c++/OSCPPDebug.h> |
41 | ||
42 | #include <IOKit/assert.h> | |
43 | ||
44 | #include <IOKit/IOReturn.h> | |
45 | #include <IOKit/IOLib.h> | |
91447636 | 46 | #include <IOKit/IOLocks.h> |
55e303ae | 47 | #include <IOKit/IOMapper.h> |
0c530ab8 | 48 | #include <IOKit/IOBufferMemoryDescriptor.h> |
1c79356b A |
49 | #include <IOKit/IOKitDebug.h> |
50 | ||
91447636 A |
51 | #include "IOKitKernelInternal.h" |
52 | ||
2d21ac55 A |
53 | #ifdef IOALLOCDEBUG |
54 | #include <libkern/OSDebug.h> | |
55 | #include <sys/sysctl.h> | |
56 | #endif | |
57 | ||
6d2010ae A |
58 | #include "libkern/OSAtomic.h" |
59 | #include <libkern/c++/OSKext.h> | |
60 | #include <IOKit/IOStatisticsPrivate.h> | |
61 | #include <sys/msgbuf.h> | |
62 | ||
63 | #if IOKITSTATS | |
64 | ||
65 | #define IOStatisticsAlloc(type, size) \ | |
66 | do { \ | |
67 | IOStatistics::countAlloc(type, size); \ | |
68 | } while (0) | |
69 | ||
70 | #else | |
71 | ||
72 | #define IOStatisticsAlloc(type, size) | |
73 | ||
74 | #endif /* IOKITSTATS */ | |
75 | ||
0c530ab8 A |
76 | extern "C" |
77 | { | |
78 | ||
79 | ||
1c79356b A |
80 | mach_timespec_t IOZeroTvalspec = { 0, 0 }; |
81 | ||
55e303ae A |
82 | extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); |
83 | ||
6d2010ae | 84 | extern int |
b0d623f7 A |
85 | __doprnt( |
86 | const char *fmt, | |
87 | va_list argp, | |
88 | void (*putc)(int, void *), | |
89 | void *arg, | |
90 | int radix); | |
91 | ||
6d2010ae A |
92 | extern void cons_putc_locked(char); |
93 | extern void bsd_log_lock(void); | |
94 | extern void bsd_log_unlock(void); | |
b0d623f7 | 95 | |
0c530ab8 | 96 | |
55e303ae | 97 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
9bccf70c | 98 | |
91447636 A |
99 | lck_grp_t *IOLockGroup; |
100 | ||
9bccf70c A |
101 | /* |
102 | * Global variables for use by iLogger | |
103 | * These symbols are for use only by Apple diagnostic code. | |
104 | * Binary compatibility is not guaranteed for kexts that reference these symbols. | |
105 | */ | |
106 | ||
107 | void *_giDebugLogInternal = NULL; | |
108 | void *_giDebugLogDataInternal = NULL; | |
109 | void *_giDebugReserved1 = NULL; | |
110 | void *_giDebugReserved2 = NULL; | |
111 | ||
112 | ||
1c79356b A |
113 | /* |
114 | * Static variables for this module. | |
115 | */ | |
116 | ||
55e303ae | 117 | static queue_head_t gIOMallocContiguousEntries; |
91447636 | 118 | static lck_mtx_t * gIOMallocContiguousEntriesLock; |
1c79356b A |
119 | |
120 | enum { kIOMaxPageableMaps = 16 }; | |
483a1d10 | 121 | enum { kIOPageableMapSize = 96 * 1024 * 1024 }; |
55e303ae | 122 | enum { kIOPageableMaxMapSize = 96 * 1024 * 1024 }; |
1c79356b A |
123 | |
124 | typedef struct { | |
b0d623f7 | 125 | vm_map_t map; |
1c79356b A |
126 | vm_offset_t address; |
127 | vm_offset_t end; | |
128 | } IOMapData; | |
129 | ||
130 | static struct { | |
131 | UInt32 count; | |
132 | UInt32 hint; | |
133 | IOMapData maps[ kIOMaxPageableMaps ]; | |
91447636 | 134 | lck_mtx_t * lock; |
1c79356b A |
135 | } gIOKitPageableSpace; |
136 | ||
55e303ae | 137 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
1c79356b A |
138 | |
139 | void IOLibInit(void) | |
140 | { | |
141 | kern_return_t ret; | |
142 | ||
143 | static bool libInitialized; | |
144 | ||
145 | if(libInitialized) | |
146 | return; | |
147 | ||
1c79356b A |
148 | gIOKitPageableSpace.maps[0].address = 0; |
149 | ret = kmem_suballoc(kernel_map, | |
150 | &gIOKitPageableSpace.maps[0].address, | |
151 | kIOPageableMapSize, | |
152 | TRUE, | |
91447636 | 153 | VM_FLAGS_ANYWHERE, |
1c79356b A |
154 | &gIOKitPageableSpace.maps[0].map); |
155 | if (ret != KERN_SUCCESS) | |
156 | panic("failed to allocate iokit pageable map\n"); | |
157 | ||
91447636 A |
158 | IOLockGroup = lck_grp_alloc_init("IOKit", LCK_GRP_ATTR_NULL); |
159 | ||
160 | gIOKitPageableSpace.lock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); | |
1c79356b A |
161 | gIOKitPageableSpace.maps[0].end = gIOKitPageableSpace.maps[0].address + kIOPageableMapSize; |
162 | gIOKitPageableSpace.hint = 0; | |
163 | gIOKitPageableSpace.count = 1; | |
164 | ||
91447636 | 165 | gIOMallocContiguousEntriesLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); |
55e303ae A |
166 | queue_init( &gIOMallocContiguousEntries ); |
167 | ||
1c79356b A |
168 | libInitialized = true; |
169 | } | |
170 | ||
171 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ | |
172 | ||
1c79356b A |
173 | IOThread IOCreateThread(IOThreadFunc fcn, void *arg) |
174 | { | |
91447636 A |
175 | kern_return_t result; |
176 | thread_t thread; | |
1c79356b | 177 | |
91447636 A |
178 | result = kernel_thread_start((thread_continue_t)fcn, arg, &thread); |
179 | if (result != KERN_SUCCESS) | |
180 | return (NULL); | |
1c79356b | 181 | |
91447636 | 182 | thread_deallocate(thread); |
1c79356b | 183 | |
91447636 | 184 | return (thread); |
1c79356b A |
185 | } |
186 | ||
187 | ||
0c530ab8 | 188 | void IOExitThread(void) |
1c79356b | 189 | { |
0c530ab8 | 190 | (void) thread_terminate(current_thread()); |
1c79356b A |
191 | } |
192 | ||
193 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ | |
194 | ||
195 | ||
196 | void * IOMalloc(vm_size_t size) | |
197 | { | |
198 | void * address; | |
199 | ||
200 | address = (void *)kalloc(size); | |
6d2010ae | 201 | if ( address ) { |
1c79356b | 202 | #if IOALLOCDEBUG |
2d21ac55 | 203 | debug_iomalloc_size += size; |
1c79356b | 204 | #endif |
6d2010ae A |
205 | IOStatisticsAlloc(kIOStatisticsMalloc, size); |
206 | } | |
207 | ||
1c79356b A |
208 | return address; |
209 | } | |
210 | ||
211 | void IOFree(void * address, vm_size_t size) | |
212 | { | |
213 | if (address) { | |
2d21ac55 | 214 | kfree(address, size); |
1c79356b | 215 | #if IOALLOCDEBUG |
2d21ac55 | 216 | debug_iomalloc_size -= size; |
1c79356b | 217 | #endif |
6d2010ae | 218 | IOStatisticsAlloc(kIOStatisticsFree, size); |
1c79356b A |
219 | } |
220 | } | |
221 | ||
222 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ | |
223 | ||
224 | void * IOMallocAligned(vm_size_t size, vm_size_t alignment) | |
225 | { | |
226 | kern_return_t kr; | |
b0d623f7 A |
227 | vm_offset_t address; |
228 | vm_offset_t allocationAddress; | |
1c79356b | 229 | vm_size_t adjustedSize; |
b0d623f7 | 230 | uintptr_t alignMask; |
1c79356b A |
231 | |
232 | if (size == 0) | |
233 | return 0; | |
234 | if (alignment == 0) | |
235 | alignment = 1; | |
236 | ||
237 | alignMask = alignment - 1; | |
238 | adjustedSize = size + sizeof(vm_size_t) + sizeof(vm_address_t); | |
239 | ||
316670eb A |
240 | if (size > adjustedSize) { |
241 | address = 0; /* overflow detected */ | |
242 | } | |
243 | else if (adjustedSize >= page_size) { | |
1c79356b A |
244 | |
245 | kr = kernel_memory_allocate(kernel_map, &address, | |
9bccf70c A |
246 | size, alignMask, 0); |
247 | if (KERN_SUCCESS != kr) | |
1c79356b | 248 | address = 0; |
1c79356b A |
249 | |
250 | } else { | |
251 | ||
252 | adjustedSize += alignMask; | |
9bccf70c A |
253 | |
254 | if (adjustedSize >= page_size) { | |
255 | ||
256 | kr = kernel_memory_allocate(kernel_map, &allocationAddress, | |
257 | adjustedSize, 0, 0); | |
258 | if (KERN_SUCCESS != kr) | |
259 | allocationAddress = 0; | |
260 | ||
261 | } else | |
262 | allocationAddress = (vm_address_t) kalloc(adjustedSize); | |
1c79356b A |
263 | |
264 | if (allocationAddress) { | |
265 | address = (allocationAddress + alignMask | |
266 | + (sizeof(vm_size_t) + sizeof(vm_address_t))) | |
267 | & (~alignMask); | |
268 | ||
b0d623f7 A |
269 | *((vm_size_t *)(address - sizeof(vm_size_t) - sizeof(vm_address_t))) |
270 | = adjustedSize; | |
1c79356b A |
271 | *((vm_address_t *)(address - sizeof(vm_address_t))) |
272 | = allocationAddress; | |
273 | } else | |
274 | address = 0; | |
275 | } | |
276 | ||
277 | assert(0 == (address & alignMask)); | |
278 | ||
2d21ac55 | 279 | if( address) { |
6d2010ae | 280 | #if IOALLOCDEBUG |
2d21ac55 | 281 | debug_iomalloc_size += size; |
1c79356b | 282 | #endif |
6d2010ae A |
283 | IOStatisticsAlloc(kIOStatisticsMallocAligned, size); |
284 | } | |
1c79356b A |
285 | |
286 | return (void *) address; | |
287 | } | |
288 | ||
289 | void IOFreeAligned(void * address, vm_size_t size) | |
290 | { | |
291 | vm_address_t allocationAddress; | |
b0d623f7 | 292 | vm_size_t adjustedSize; |
1c79356b A |
293 | |
294 | if( !address) | |
295 | return; | |
296 | ||
297 | assert(size); | |
298 | ||
299 | adjustedSize = size + sizeof(vm_size_t) + sizeof(vm_address_t); | |
300 | if (adjustedSize >= page_size) { | |
301 | ||
b0d623f7 | 302 | kmem_free( kernel_map, (vm_offset_t) address, size); |
1c79356b A |
303 | |
304 | } else { | |
b0d623f7 | 305 | adjustedSize = *((vm_size_t *)( (vm_address_t) address |
1c79356b A |
306 | - sizeof(vm_address_t) - sizeof(vm_size_t))); |
307 | allocationAddress = *((vm_address_t *)( (vm_address_t) address | |
308 | - sizeof(vm_address_t) )); | |
309 | ||
9bccf70c | 310 | if (adjustedSize >= page_size) |
91447636 | 311 | kmem_free( kernel_map, allocationAddress, adjustedSize); |
9bccf70c | 312 | else |
91447636 | 313 | kfree((void *)allocationAddress, adjustedSize); |
1c79356b A |
314 | } |
315 | ||
316 | #if IOALLOCDEBUG | |
317 | debug_iomalloc_size -= size; | |
318 | #endif | |
6d2010ae A |
319 | |
320 | IOStatisticsAlloc(kIOStatisticsFreeAligned, size); | |
1c79356b A |
321 | } |
322 | ||
323 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ | |
324 | ||
0c530ab8 | 325 | void |
0b4c1975 | 326 | IOKernelFreePhysical(mach_vm_address_t address, mach_vm_size_t size) |
55e303ae | 327 | { |
0c530ab8 A |
328 | mach_vm_address_t allocationAddress; |
329 | mach_vm_size_t adjustedSize; | |
4452a7af | 330 | |
0c530ab8 A |
331 | if (!address) |
332 | return; | |
333 | ||
334 | assert(size); | |
335 | ||
336 | adjustedSize = (2 * size) + sizeof(mach_vm_size_t) + sizeof(mach_vm_address_t); | |
337 | if (adjustedSize >= page_size) { | |
338 | ||
b0d623f7 | 339 | kmem_free( kernel_map, (vm_offset_t) address, size); |
0c530ab8 A |
340 | |
341 | } else { | |
342 | ||
343 | adjustedSize = *((mach_vm_size_t *) | |
344 | (address - sizeof(mach_vm_address_t) - sizeof(mach_vm_size_t))); | |
345 | allocationAddress = *((mach_vm_address_t *) | |
346 | (address - sizeof(mach_vm_address_t) )); | |
347 | kfree((void *)allocationAddress, adjustedSize); | |
348 | } | |
349 | ||
7ddcb079 | 350 | IOStatisticsAlloc(kIOStatisticsFreeContiguous, size); |
0c530ab8 A |
351 | #if IOALLOCDEBUG |
352 | debug_iomalloc_size -= size; | |
353 | #endif | |
354 | } | |
355 | ||
356 | mach_vm_address_t | |
0b4c1975 | 357 | IOKernelAllocateWithPhysicalRestrict(mach_vm_size_t size, mach_vm_address_t maxPhys, |
6d2010ae | 358 | mach_vm_size_t alignment, bool contiguous) |
1c79356b A |
359 | { |
360 | kern_return_t kr; | |
0c530ab8 A |
361 | mach_vm_address_t address; |
362 | mach_vm_address_t allocationAddress; | |
363 | mach_vm_size_t adjustedSize; | |
364 | mach_vm_address_t alignMask; | |
1c79356b A |
365 | |
366 | if (size == 0) | |
0c530ab8 | 367 | return (0); |
1c79356b A |
368 | if (alignment == 0) |
369 | alignment = 1; | |
370 | ||
371 | alignMask = alignment - 1; | |
0c530ab8 | 372 | adjustedSize = (2 * size) + sizeof(mach_vm_size_t) + sizeof(mach_vm_address_t); |
1c79356b | 373 | |
0b4c1975 A |
374 | contiguous = (contiguous && (adjustedSize > page_size)) |
375 | || (alignment > page_size); | |
376 | ||
377 | if (contiguous || maxPhys) | |
55e303ae | 378 | { |
0b4c1975 | 379 | int options = 0; |
0c530ab8 | 380 | vm_offset_t virt; |
0b4c1975 | 381 | |
55e303ae | 382 | adjustedSize = size; |
0b4c1975 A |
383 | contiguous = (contiguous && (adjustedSize > page_size)) |
384 | || (alignment > page_size); | |
385 | ||
7ddcb079 A |
386 | if (!contiguous) |
387 | { | |
388 | if (maxPhys <= 0xFFFFFFFF) | |
389 | { | |
390 | maxPhys = 0; | |
391 | options |= KMA_LOMEM; | |
392 | } | |
393 | else if (gIOLastPage && (atop_64(maxPhys) > gIOLastPage)) | |
394 | { | |
395 | maxPhys = 0; | |
396 | } | |
397 | } | |
0b4c1975 | 398 | if (contiguous || maxPhys) |
55e303ae | 399 | { |
0c530ab8 | 400 | kr = kmem_alloc_contig(kernel_map, &virt, size, |
b0d623f7 | 401 | alignMask, atop(maxPhys), atop(alignMask), 0); |
55e303ae A |
402 | } |
403 | else | |
404 | { | |
0c530ab8 | 405 | kr = kernel_memory_allocate(kernel_map, &virt, |
0b4c1975 | 406 | size, alignMask, options); |
55e303ae | 407 | } |
0c530ab8 A |
408 | if (KERN_SUCCESS == kr) |
409 | address = virt; | |
410 | else | |
1c79356b | 411 | address = 0; |
55e303ae A |
412 | } |
413 | else | |
414 | { | |
1c79356b | 415 | adjustedSize += alignMask; |
0c530ab8 | 416 | allocationAddress = (mach_vm_address_t) kalloc(adjustedSize); |
9bccf70c | 417 | |
1c79356b A |
418 | if (allocationAddress) { |
419 | ||
420 | address = (allocationAddress + alignMask | |
0c530ab8 | 421 | + (sizeof(mach_vm_size_t) + sizeof(mach_vm_address_t))) |
1c79356b A |
422 | & (~alignMask); |
423 | ||
55e303ae | 424 | if (atop_32(address) != atop_32(address + size - 1)) |
b0d623f7 | 425 | address = round_page(address); |
1c79356b | 426 | |
0c530ab8 A |
427 | *((mach_vm_size_t *)(address - sizeof(mach_vm_size_t) |
428 | - sizeof(mach_vm_address_t))) = adjustedSize; | |
429 | *((mach_vm_address_t *)(address - sizeof(mach_vm_address_t))) | |
1c79356b A |
430 | = allocationAddress; |
431 | } else | |
432 | address = 0; | |
433 | } | |
434 | ||
2d21ac55 | 435 | if (address) { |
7ddcb079 A |
436 | IOStatisticsAlloc(kIOStatisticsMallocContiguous, size); |
437 | #if IOALLOCDEBUG | |
b0d623f7 | 438 | debug_iomalloc_size += size; |
0c530ab8 | 439 | #endif |
7ddcb079 | 440 | } |
0c530ab8 A |
441 | |
442 | return (address); | |
443 | } | |
444 | ||
6d2010ae | 445 | |
0c530ab8 A |
446 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
447 | ||
448 | struct _IOMallocContiguousEntry | |
449 | { | |
450 | mach_vm_address_t virtualAddr; | |
451 | IOBufferMemoryDescriptor * md; | |
452 | queue_chain_t link; | |
453 | }; | |
454 | typedef struct _IOMallocContiguousEntry _IOMallocContiguousEntry; | |
455 | ||
456 | void * IOMallocContiguous(vm_size_t size, vm_size_t alignment, | |
457 | IOPhysicalAddress * physicalAddress) | |
458 | { | |
459 | mach_vm_address_t address = 0; | |
460 | ||
461 | if (size == 0) | |
462 | return 0; | |
463 | if (alignment == 0) | |
464 | alignment = 1; | |
465 | ||
55e303ae | 466 | /* Do we want a physical address? */ |
0c530ab8 | 467 | if (!physicalAddress) |
c0fea474 | 468 | { |
0b4c1975 | 469 | address = IOKernelAllocateWithPhysicalRestrict(size, 0 /*maxPhys*/, alignment, true); |
0c530ab8 A |
470 | } |
471 | else do | |
472 | { | |
473 | IOBufferMemoryDescriptor * bmd; | |
474 | mach_vm_address_t physicalMask; | |
b0d623f7 | 475 | vm_offset_t alignMask; |
0c530ab8 A |
476 | |
477 | alignMask = alignment - 1; | |
b0d623f7 A |
478 | physicalMask = (0xFFFFFFFF ^ alignMask); |
479 | ||
0c530ab8 A |
480 | bmd = IOBufferMemoryDescriptor::inTaskWithPhysicalMask( |
481 | kernel_task, kIOMemoryPhysicallyContiguous, size, physicalMask); | |
482 | if (!bmd) | |
483 | break; | |
484 | ||
485 | _IOMallocContiguousEntry * | |
486 | entry = IONew(_IOMallocContiguousEntry, 1); | |
487 | if (!entry) | |
55e303ae | 488 | { |
0c530ab8 A |
489 | bmd->release(); |
490 | break; | |
55e303ae | 491 | } |
0c530ab8 A |
492 | entry->virtualAddr = (mach_vm_address_t) bmd->getBytesNoCopy(); |
493 | entry->md = bmd; | |
494 | lck_mtx_lock(gIOMallocContiguousEntriesLock); | |
495 | queue_enter( &gIOMallocContiguousEntries, entry, | |
496 | _IOMallocContiguousEntry *, link ); | |
497 | lck_mtx_unlock(gIOMallocContiguousEntriesLock); | |
498 | ||
499 | address = (mach_vm_address_t) entry->virtualAddr; | |
500 | *physicalAddress = bmd->getPhysicalAddress(); | |
55e303ae | 501 | } |
0c530ab8 | 502 | while (false); |
1c79356b A |
503 | |
504 | return (void *) address; | |
505 | } | |
506 | ||
0c530ab8 | 507 | void IOFreeContiguous(void * _address, vm_size_t size) |
1c79356b | 508 | { |
55e303ae | 509 | _IOMallocContiguousEntry * entry; |
0c530ab8 A |
510 | IOMemoryDescriptor * md = NULL; |
511 | ||
512 | mach_vm_address_t address = (mach_vm_address_t) _address; | |
1c79356b A |
513 | |
514 | if( !address) | |
515 | return; | |
516 | ||
517 | assert(size); | |
518 | ||
91447636 | 519 | lck_mtx_lock(gIOMallocContiguousEntriesLock); |
55e303ae A |
520 | queue_iterate( &gIOMallocContiguousEntries, entry, |
521 | _IOMallocContiguousEntry *, link ) | |
522 | { | |
0c530ab8 A |
523 | if( entry->virtualAddr == address ) { |
524 | md = entry->md; | |
55e303ae A |
525 | queue_remove( &gIOMallocContiguousEntries, entry, |
526 | _IOMallocContiguousEntry *, link ); | |
527 | break; | |
528 | } | |
529 | } | |
91447636 | 530 | lck_mtx_unlock(gIOMallocContiguousEntriesLock); |
55e303ae | 531 | |
0c530ab8 | 532 | if (md) |
55e303ae | 533 | { |
0c530ab8 | 534 | md->release(); |
55e303ae A |
535 | IODelete(entry, _IOMallocContiguousEntry, 1); |
536 | } | |
0c530ab8 A |
537 | else |
538 | { | |
0b4c1975 | 539 | IOKernelFreePhysical((mach_vm_address_t) address, size); |
1c79356b | 540 | } |
1c79356b A |
541 | } |
542 | ||
543 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ | |
544 | ||
0b4e3aa0 A |
545 | kern_return_t IOIteratePageableMaps(vm_size_t size, |
546 | IOIteratePageableMapsCallback callback, void * ref) | |
1c79356b A |
547 | { |
548 | kern_return_t kr = kIOReturnNotReady; | |
1c79356b A |
549 | vm_size_t segSize; |
550 | UInt32 attempts; | |
551 | UInt32 index; | |
552 | vm_offset_t min; | |
553 | vm_map_t map; | |
554 | ||
1c79356b | 555 | if (size > kIOPageableMaxMapSize) |
0b4e3aa0 | 556 | return( kIOReturnBadArgument ); |
1c79356b A |
557 | |
558 | do { | |
559 | index = gIOKitPageableSpace.hint; | |
560 | attempts = gIOKitPageableSpace.count; | |
561 | while( attempts--) { | |
0b4e3aa0 | 562 | kr = (*callback)(gIOKitPageableSpace.maps[index].map, ref); |
1c79356b A |
563 | if( KERN_SUCCESS == kr) { |
564 | gIOKitPageableSpace.hint = index; | |
565 | break; | |
566 | } | |
567 | if( index) | |
568 | index--; | |
569 | else | |
570 | index = gIOKitPageableSpace.count - 1; | |
571 | } | |
572 | if( KERN_SUCCESS == kr) | |
573 | break; | |
574 | ||
91447636 | 575 | lck_mtx_lock( gIOKitPageableSpace.lock ); |
1c79356b A |
576 | |
577 | index = gIOKitPageableSpace.count; | |
578 | if( index >= (kIOMaxPageableMaps - 1)) { | |
91447636 | 579 | lck_mtx_unlock( gIOKitPageableSpace.lock ); |
1c79356b A |
580 | break; |
581 | } | |
582 | ||
583 | if( size < kIOPageableMapSize) | |
584 | segSize = kIOPageableMapSize; | |
585 | else | |
586 | segSize = size; | |
587 | ||
588 | min = 0; | |
589 | kr = kmem_suballoc(kernel_map, | |
590 | &min, | |
591 | segSize, | |
592 | TRUE, | |
91447636 | 593 | VM_FLAGS_ANYWHERE, |
1c79356b A |
594 | &map); |
595 | if( KERN_SUCCESS != kr) { | |
91447636 | 596 | lck_mtx_unlock( gIOKitPageableSpace.lock ); |
1c79356b A |
597 | break; |
598 | } | |
599 | ||
600 | gIOKitPageableSpace.maps[index].map = map; | |
601 | gIOKitPageableSpace.maps[index].address = min; | |
602 | gIOKitPageableSpace.maps[index].end = min + segSize; | |
603 | gIOKitPageableSpace.hint = index; | |
604 | gIOKitPageableSpace.count = index + 1; | |
605 | ||
91447636 | 606 | lck_mtx_unlock( gIOKitPageableSpace.lock ); |
1c79356b A |
607 | |
608 | } while( true ); | |
609 | ||
0b4e3aa0 A |
610 | return kr; |
611 | } | |
612 | ||
613 | struct IOMallocPageableRef | |
614 | { | |
b0d623f7 | 615 | vm_offset_t address; |
0b4e3aa0 A |
616 | vm_size_t size; |
617 | }; | |
618 | ||
619 | static kern_return_t IOMallocPageableCallback(vm_map_t map, void * _ref) | |
620 | { | |
621 | struct IOMallocPageableRef * ref = (struct IOMallocPageableRef *) _ref; | |
622 | kern_return_t kr; | |
623 | ||
624 | kr = kmem_alloc_pageable( map, &ref->address, ref->size ); | |
625 | ||
626 | return( kr ); | |
627 | } | |
628 | ||
629 | void * IOMallocPageable(vm_size_t size, vm_size_t alignment) | |
630 | { | |
631 | kern_return_t kr = kIOReturnNotReady; | |
632 | struct IOMallocPageableRef ref; | |
633 | ||
634 | if (alignment > page_size) | |
635 | return( 0 ); | |
636 | if (size > kIOPageableMaxMapSize) | |
637 | return( 0 ); | |
638 | ||
639 | ref.size = size; | |
640 | kr = IOIteratePageableMaps( size, &IOMallocPageableCallback, &ref ); | |
641 | if( kIOReturnSuccess != kr) | |
642 | ref.address = 0; | |
1c79356b | 643 | |
6d2010ae | 644 | if( ref.address) { |
1c79356b | 645 | #if IOALLOCDEBUG |
b0d623f7 | 646 | debug_iomallocpageable_size += round_page(size); |
1c79356b | 647 | #endif |
6d2010ae A |
648 | IOStatisticsAlloc(kIOStatisticsMallocPageable, size); |
649 | } | |
1c79356b | 650 | |
0b4e3aa0 | 651 | return( (void *) ref.address ); |
1c79356b A |
652 | } |
653 | ||
b0d623f7 | 654 | vm_map_t IOPageableMapForAddress( uintptr_t address ) |
1c79356b A |
655 | { |
656 | vm_map_t map = 0; | |
657 | UInt32 index; | |
658 | ||
659 | for( index = 0; index < gIOKitPageableSpace.count; index++) { | |
660 | if( (address >= gIOKitPageableSpace.maps[index].address) | |
661 | && (address < gIOKitPageableSpace.maps[index].end) ) { | |
662 | map = gIOKitPageableSpace.maps[index].map; | |
663 | break; | |
664 | } | |
665 | } | |
666 | if( !map) | |
b0d623f7 | 667 | panic("IOPageableMapForAddress: null"); |
1c79356b A |
668 | |
669 | return( map ); | |
670 | } | |
671 | ||
672 | void IOFreePageable(void * address, vm_size_t size) | |
673 | { | |
674 | vm_map_t map; | |
675 | ||
676 | map = IOPageableMapForAddress( (vm_address_t) address); | |
677 | if( map) | |
678 | kmem_free( map, (vm_offset_t) address, size); | |
679 | ||
680 | #if IOALLOCDEBUG | |
b0d623f7 | 681 | debug_iomallocpageable_size -= round_page(size); |
1c79356b | 682 | #endif |
6d2010ae A |
683 | |
684 | IOStatisticsAlloc(kIOStatisticsFreePageable, size); | |
1c79356b | 685 | } |
b0d623f7 | 686 | |
1c79356b A |
687 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
688 | ||
1c79356b A |
689 | IOReturn IOSetProcessorCacheMode( task_t task, IOVirtualAddress address, |
690 | IOByteCount length, IOOptionBits cacheMode ) | |
691 | { | |
692 | IOReturn ret = kIOReturnSuccess; | |
55e303ae | 693 | ppnum_t pagenum; |
1c79356b A |
694 | |
695 | if( task != kernel_task) | |
696 | return( kIOReturnUnsupported ); | |
b0d623f7 A |
697 | if ((address | length) & PAGE_MASK) |
698 | { | |
699 | // OSReportWithBacktrace("IOSetProcessorCacheMode(0x%x, 0x%x, 0x%x) fails\n", address, length, cacheMode); | |
700 | return( kIOReturnUnsupported ); | |
701 | } | |
702 | length = round_page(address + length) - trunc_page( address ); | |
703 | address = trunc_page( address ); | |
1c79356b A |
704 | |
705 | // make map mode | |
706 | cacheMode = (cacheMode << kIOMapCacheShift) & kIOMapCacheMask; | |
707 | ||
708 | while( (kIOReturnSuccess == ret) && (length > 0) ) { | |
709 | ||
55e303ae A |
710 | // Get the physical page number |
711 | pagenum = pmap_find_phys(kernel_pmap, (addr64_t)address); | |
712 | if( pagenum) { | |
713 | ret = IOUnmapPages( get_task_map(task), address, page_size ); | |
0c530ab8 | 714 | ret = IOMapPages( get_task_map(task), address, ptoa_64(pagenum), page_size, cacheMode ); |
55e303ae | 715 | } else |
1c79356b A |
716 | ret = kIOReturnVMError; |
717 | ||
55e303ae | 718 | address += page_size; |
1c79356b A |
719 | length -= page_size; |
720 | } | |
721 | ||
722 | return( ret ); | |
723 | } | |
724 | ||
725 | ||
726 | IOReturn IOFlushProcessorCache( task_t task, IOVirtualAddress address, | |
727 | IOByteCount length ) | |
728 | { | |
729 | if( task != kernel_task) | |
730 | return( kIOReturnUnsupported ); | |
731 | ||
55e303ae | 732 | flush_dcache64( (addr64_t) address, (unsigned) length, false ); |
1c79356b A |
733 | |
734 | return( kIOReturnSuccess ); | |
735 | } | |
736 | ||
737 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ | |
738 | ||
b0d623f7 | 739 | vm_offset_t OSKernelStackRemaining( void ) |
1c79356b | 740 | { |
b0d623f7 | 741 | return (ml_stack_remaining()); |
1c79356b A |
742 | } |
743 | ||
744 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ | |
745 | ||
2d21ac55 A |
746 | /* |
747 | * Spin for indicated number of milliseconds. | |
748 | */ | |
1c79356b A |
749 | void IOSleep(unsigned milliseconds) |
750 | { | |
91447636 | 751 | delay_for_interval(milliseconds, kMillisecondScale); |
1c79356b A |
752 | } |
753 | ||
754 | /* | |
755 | * Spin for indicated number of microseconds. | |
756 | */ | |
757 | void IODelay(unsigned microseconds) | |
758 | { | |
91447636 | 759 | delay_for_interval(microseconds, kMicrosecondScale); |
1c79356b A |
760 | } |
761 | ||
2d21ac55 A |
762 | /* |
763 | * Spin for indicated number of nanoseconds. | |
764 | */ | |
765 | void IOPause(unsigned nanoseconds) | |
766 | { | |
767 | delay_for_interval(nanoseconds, kNanosecondScale); | |
768 | } | |
769 | ||
1c79356b A |
770 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
771 | ||
6d2010ae | 772 | static void _iolog_consputc(int ch, void *arg __unused) |
b0d623f7 | 773 | { |
6d2010ae A |
774 | cons_putc_locked(ch); |
775 | } | |
776 | ||
777 | static void _iolog_logputc(int ch, void *arg __unused) | |
778 | { | |
779 | log_putc_locked(ch); | |
b0d623f7 A |
780 | } |
781 | ||
1c79356b A |
782 | void IOLog(const char *format, ...) |
783 | { | |
6d2010ae | 784 | va_list ap; |
1c79356b | 785 | |
6d2010ae A |
786 | va_start(ap, format); |
787 | IOLogv(format, ap); | |
788 | va_end(ap); | |
1c79356b A |
789 | } |
790 | ||
b0d623f7 A |
791 | void IOLogv(const char *format, va_list ap) |
792 | { | |
6d2010ae A |
793 | va_list ap2; |
794 | ||
795 | va_copy(ap2, ap); | |
796 | ||
797 | bsd_log_lock(); | |
798 | __doprnt(format, ap, _iolog_logputc, NULL, 16); | |
799 | bsd_log_unlock(); | |
800 | ||
801 | __doprnt(format, ap2, _iolog_consputc, NULL, 16); | |
b0d623f7 A |
802 | } |
803 | ||
804 | #if !__LP64__ | |
1c79356b A |
805 | void IOPanic(const char *reason) |
806 | { | |
2d21ac55 | 807 | panic("%s", reason); |
1c79356b | 808 | } |
b0d623f7 | 809 | #endif |
1c79356b A |
810 | |
811 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ | |
812 | ||
813 | /* | |
814 | * Convert a integer constant (typically a #define or enum) to a string. | |
815 | */ | |
816 | static char noValue[80]; // that's pretty | |
817 | ||
818 | const char *IOFindNameForValue(int value, const IONamedValue *regValueArray) | |
819 | { | |
820 | for( ; regValueArray->name; regValueArray++) { | |
821 | if(regValueArray->value == value) | |
822 | return(regValueArray->name); | |
823 | } | |
2d21ac55 | 824 | snprintf(noValue, sizeof(noValue), "0x%x (UNDEFINED)", value); |
1c79356b A |
825 | return((const char *)noValue); |
826 | } | |
827 | ||
828 | IOReturn IOFindValueForName(const char *string, | |
829 | const IONamedValue *regValueArray, | |
830 | int *value) | |
831 | { | |
832 | for( ; regValueArray->name; regValueArray++) { | |
833 | if(!strcmp(regValueArray->name, string)) { | |
834 | *value = regValueArray->value; | |
835 | return kIOReturnSuccess; | |
836 | } | |
837 | } | |
838 | return kIOReturnBadArgument; | |
839 | } | |
840 | ||
2d21ac55 A |
841 | OSString * IOCopyLogNameForPID(int pid) |
842 | { | |
843 | char buf[128]; | |
844 | size_t len; | |
845 | snprintf(buf, sizeof(buf), "pid %d, ", pid); | |
846 | len = strlen(buf); | |
847 | proc_name(pid, buf + len, sizeof(buf) - len); | |
848 | return (OSString::withCString(buf)); | |
849 | } | |
850 | ||
1c79356b A |
851 | /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ |
852 | ||
853 | IOAlignment IOSizeToAlignment(unsigned int size) | |
854 | { | |
855 | register int shift; | |
856 | const int intsize = sizeof(unsigned int) * 8; | |
857 | ||
858 | for (shift = 1; shift < intsize; shift++) { | |
859 | if (size & 0x80000000) | |
860 | return (IOAlignment)(intsize - shift); | |
861 | size <<= 1; | |
862 | } | |
863 | return 0; | |
864 | } | |
865 | ||
866 | unsigned int IOAlignmentToSize(IOAlignment align) | |
867 | { | |
868 | unsigned int size; | |
869 | ||
870 | for (size = 1; align; align--) { | |
871 | size <<= 1; | |
872 | } | |
873 | return size; | |
874 | } | |
0c530ab8 A |
875 | |
876 | } /* extern "C" */ | |
2d21ac55 A |
877 | |
878 | ||
879 |