]> git.saurik.com Git - apple/xnu.git/blame - iokit/Kernel/IOLib.cpp
xnu-2050.48.11.tar.gz
[apple/xnu.git] / iokit / Kernel / IOLib.cpp
CommitLineData
1c79356b 1/*
2d21ac55 2 * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved.
1c79356b 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b 5 *
2d21ac55
A
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A
27 */
28/*
1c79356b
A
29 * HISTORY
30 *
31 * 17-Apr-91 Portions from libIO.m, Doug Mitchell at NeXT.
32 * 17-Nov-98 cpp
33 *
34 */
35
36#include <IOKit/system.h>
37#include <mach/sync_policy.h>
38#include <machine/machine_routines.h>
b0d623f7 39#include <vm/vm_kern.h>
1c79356b
A
40#include <libkern/c++/OSCPPDebug.h>
41
42#include <IOKit/assert.h>
43
44#include <IOKit/IOReturn.h>
45#include <IOKit/IOLib.h>
91447636 46#include <IOKit/IOLocks.h>
55e303ae 47#include <IOKit/IOMapper.h>
0c530ab8 48#include <IOKit/IOBufferMemoryDescriptor.h>
1c79356b
A
49#include <IOKit/IOKitDebug.h>
50
91447636
A
51#include "IOKitKernelInternal.h"
52
2d21ac55
A
53#ifdef IOALLOCDEBUG
54#include <libkern/OSDebug.h>
55#include <sys/sysctl.h>
56#endif
57
6d2010ae
A
58#include "libkern/OSAtomic.h"
59#include <libkern/c++/OSKext.h>
60#include <IOKit/IOStatisticsPrivate.h>
61#include <sys/msgbuf.h>
62
63#if IOKITSTATS
64
65#define IOStatisticsAlloc(type, size) \
66do { \
67 IOStatistics::countAlloc(type, size); \
68} while (0)
69
70#else
71
72#define IOStatisticsAlloc(type, size)
73
74#endif /* IOKITSTATS */
75
0c530ab8
A
76extern "C"
77{
78
79
1c79356b
A
80mach_timespec_t IOZeroTvalspec = { 0, 0 };
81
55e303ae
A
82extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
83
6d2010ae 84extern int
b0d623f7
A
85__doprnt(
86 const char *fmt,
87 va_list argp,
88 void (*putc)(int, void *),
89 void *arg,
90 int radix);
91
6d2010ae
A
92extern void cons_putc_locked(char);
93extern void bsd_log_lock(void);
94extern void bsd_log_unlock(void);
b0d623f7 95
0c530ab8 96
55e303ae 97/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
9bccf70c 98
91447636
A
99lck_grp_t *IOLockGroup;
100
9bccf70c
A
101/*
102 * Global variables for use by iLogger
103 * These symbols are for use only by Apple diagnostic code.
104 * Binary compatibility is not guaranteed for kexts that reference these symbols.
105 */
106
107void *_giDebugLogInternal = NULL;
108void *_giDebugLogDataInternal = NULL;
109void *_giDebugReserved1 = NULL;
110void *_giDebugReserved2 = NULL;
111
112
1c79356b
A
113/*
114 * Static variables for this module.
115 */
116
55e303ae 117static queue_head_t gIOMallocContiguousEntries;
91447636 118static lck_mtx_t * gIOMallocContiguousEntriesLock;
1c79356b
A
119
120enum { kIOMaxPageableMaps = 16 };
483a1d10 121enum { kIOPageableMapSize = 96 * 1024 * 1024 };
55e303ae 122enum { kIOPageableMaxMapSize = 96 * 1024 * 1024 };
1c79356b
A
123
124typedef struct {
b0d623f7 125 vm_map_t map;
1c79356b
A
126 vm_offset_t address;
127 vm_offset_t end;
128} IOMapData;
129
130static struct {
131 UInt32 count;
132 UInt32 hint;
133 IOMapData maps[ kIOMaxPageableMaps ];
91447636 134 lck_mtx_t * lock;
1c79356b
A
135} gIOKitPageableSpace;
136
55e303ae 137/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1c79356b
A
138
139void IOLibInit(void)
140{
141 kern_return_t ret;
142
143 static bool libInitialized;
144
145 if(libInitialized)
146 return;
147
1c79356b
A
148 gIOKitPageableSpace.maps[0].address = 0;
149 ret = kmem_suballoc(kernel_map,
150 &gIOKitPageableSpace.maps[0].address,
151 kIOPageableMapSize,
152 TRUE,
91447636 153 VM_FLAGS_ANYWHERE,
1c79356b
A
154 &gIOKitPageableSpace.maps[0].map);
155 if (ret != KERN_SUCCESS)
156 panic("failed to allocate iokit pageable map\n");
157
91447636
A
158 IOLockGroup = lck_grp_alloc_init("IOKit", LCK_GRP_ATTR_NULL);
159
160 gIOKitPageableSpace.lock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL);
1c79356b
A
161 gIOKitPageableSpace.maps[0].end = gIOKitPageableSpace.maps[0].address + kIOPageableMapSize;
162 gIOKitPageableSpace.hint = 0;
163 gIOKitPageableSpace.count = 1;
164
91447636 165 gIOMallocContiguousEntriesLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL);
55e303ae
A
166 queue_init( &gIOMallocContiguousEntries );
167
1c79356b
A
168 libInitialized = true;
169}
170
171/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
172
1c79356b
A
173IOThread IOCreateThread(IOThreadFunc fcn, void *arg)
174{
91447636
A
175 kern_return_t result;
176 thread_t thread;
1c79356b 177
91447636
A
178 result = kernel_thread_start((thread_continue_t)fcn, arg, &thread);
179 if (result != KERN_SUCCESS)
180 return (NULL);
1c79356b 181
91447636 182 thread_deallocate(thread);
1c79356b 183
91447636 184 return (thread);
1c79356b
A
185}
186
187
0c530ab8 188void IOExitThread(void)
1c79356b 189{
0c530ab8 190 (void) thread_terminate(current_thread());
1c79356b
A
191}
192
193/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
194
195
196void * IOMalloc(vm_size_t size)
197{
198 void * address;
199
200 address = (void *)kalloc(size);
6d2010ae 201 if ( address ) {
1c79356b 202#if IOALLOCDEBUG
2d21ac55 203 debug_iomalloc_size += size;
1c79356b 204#endif
6d2010ae
A
205 IOStatisticsAlloc(kIOStatisticsMalloc, size);
206 }
207
1c79356b
A
208 return address;
209}
210
211void IOFree(void * address, vm_size_t size)
212{
213 if (address) {
2d21ac55 214 kfree(address, size);
1c79356b 215#if IOALLOCDEBUG
2d21ac55 216 debug_iomalloc_size -= size;
1c79356b 217#endif
6d2010ae 218 IOStatisticsAlloc(kIOStatisticsFree, size);
1c79356b
A
219 }
220}
221
222/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
223
224void * IOMallocAligned(vm_size_t size, vm_size_t alignment)
225{
226 kern_return_t kr;
b0d623f7
A
227 vm_offset_t address;
228 vm_offset_t allocationAddress;
1c79356b 229 vm_size_t adjustedSize;
b0d623f7 230 uintptr_t alignMask;
1c79356b
A
231
232 if (size == 0)
233 return 0;
234 if (alignment == 0)
235 alignment = 1;
236
237 alignMask = alignment - 1;
238 adjustedSize = size + sizeof(vm_size_t) + sizeof(vm_address_t);
239
316670eb
A
240 if (size > adjustedSize) {
241 address = 0; /* overflow detected */
242 }
243 else if (adjustedSize >= page_size) {
1c79356b
A
244
245 kr = kernel_memory_allocate(kernel_map, &address,
9bccf70c
A
246 size, alignMask, 0);
247 if (KERN_SUCCESS != kr)
1c79356b 248 address = 0;
1c79356b
A
249
250 } else {
251
252 adjustedSize += alignMask;
9bccf70c
A
253
254 if (adjustedSize >= page_size) {
255
256 kr = kernel_memory_allocate(kernel_map, &allocationAddress,
257 adjustedSize, 0, 0);
258 if (KERN_SUCCESS != kr)
259 allocationAddress = 0;
260
261 } else
262 allocationAddress = (vm_address_t) kalloc(adjustedSize);
1c79356b
A
263
264 if (allocationAddress) {
265 address = (allocationAddress + alignMask
266 + (sizeof(vm_size_t) + sizeof(vm_address_t)))
267 & (~alignMask);
268
b0d623f7
A
269 *((vm_size_t *)(address - sizeof(vm_size_t) - sizeof(vm_address_t)))
270 = adjustedSize;
1c79356b
A
271 *((vm_address_t *)(address - sizeof(vm_address_t)))
272 = allocationAddress;
273 } else
274 address = 0;
275 }
276
277 assert(0 == (address & alignMask));
278
2d21ac55 279 if( address) {
6d2010ae 280#if IOALLOCDEBUG
2d21ac55 281 debug_iomalloc_size += size;
1c79356b 282#endif
6d2010ae
A
283 IOStatisticsAlloc(kIOStatisticsMallocAligned, size);
284 }
1c79356b
A
285
286 return (void *) address;
287}
288
289void IOFreeAligned(void * address, vm_size_t size)
290{
291 vm_address_t allocationAddress;
b0d623f7 292 vm_size_t adjustedSize;
1c79356b
A
293
294 if( !address)
295 return;
296
297 assert(size);
298
299 adjustedSize = size + sizeof(vm_size_t) + sizeof(vm_address_t);
300 if (adjustedSize >= page_size) {
301
b0d623f7 302 kmem_free( kernel_map, (vm_offset_t) address, size);
1c79356b
A
303
304 } else {
b0d623f7 305 adjustedSize = *((vm_size_t *)( (vm_address_t) address
1c79356b
A
306 - sizeof(vm_address_t) - sizeof(vm_size_t)));
307 allocationAddress = *((vm_address_t *)( (vm_address_t) address
308 - sizeof(vm_address_t) ));
309
9bccf70c 310 if (adjustedSize >= page_size)
91447636 311 kmem_free( kernel_map, allocationAddress, adjustedSize);
9bccf70c 312 else
91447636 313 kfree((void *)allocationAddress, adjustedSize);
1c79356b
A
314 }
315
316#if IOALLOCDEBUG
317 debug_iomalloc_size -= size;
318#endif
6d2010ae
A
319
320 IOStatisticsAlloc(kIOStatisticsFreeAligned, size);
1c79356b
A
321}
322
323/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
324
0c530ab8 325void
0b4c1975 326IOKernelFreePhysical(mach_vm_address_t address, mach_vm_size_t size)
55e303ae 327{
0c530ab8
A
328 mach_vm_address_t allocationAddress;
329 mach_vm_size_t adjustedSize;
4452a7af 330
0c530ab8
A
331 if (!address)
332 return;
333
334 assert(size);
335
336 adjustedSize = (2 * size) + sizeof(mach_vm_size_t) + sizeof(mach_vm_address_t);
337 if (adjustedSize >= page_size) {
338
b0d623f7 339 kmem_free( kernel_map, (vm_offset_t) address, size);
0c530ab8
A
340
341 } else {
342
343 adjustedSize = *((mach_vm_size_t *)
344 (address - sizeof(mach_vm_address_t) - sizeof(mach_vm_size_t)));
345 allocationAddress = *((mach_vm_address_t *)
346 (address - sizeof(mach_vm_address_t) ));
347 kfree((void *)allocationAddress, adjustedSize);
348 }
349
7ddcb079 350 IOStatisticsAlloc(kIOStatisticsFreeContiguous, size);
0c530ab8
A
351#if IOALLOCDEBUG
352 debug_iomalloc_size -= size;
353#endif
354}
355
356mach_vm_address_t
0b4c1975 357IOKernelAllocateWithPhysicalRestrict(mach_vm_size_t size, mach_vm_address_t maxPhys,
6d2010ae 358 mach_vm_size_t alignment, bool contiguous)
1c79356b
A
359{
360 kern_return_t kr;
0c530ab8
A
361 mach_vm_address_t address;
362 mach_vm_address_t allocationAddress;
363 mach_vm_size_t adjustedSize;
364 mach_vm_address_t alignMask;
1c79356b
A
365
366 if (size == 0)
0c530ab8 367 return (0);
1c79356b
A
368 if (alignment == 0)
369 alignment = 1;
370
371 alignMask = alignment - 1;
0c530ab8 372 adjustedSize = (2 * size) + sizeof(mach_vm_size_t) + sizeof(mach_vm_address_t);
1c79356b 373
0b4c1975
A
374 contiguous = (contiguous && (adjustedSize > page_size))
375 || (alignment > page_size);
376
377 if (contiguous || maxPhys)
55e303ae 378 {
0b4c1975 379 int options = 0;
0c530ab8 380 vm_offset_t virt;
0b4c1975 381
55e303ae 382 adjustedSize = size;
0b4c1975
A
383 contiguous = (contiguous && (adjustedSize > page_size))
384 || (alignment > page_size);
385
7ddcb079
A
386 if (!contiguous)
387 {
388 if (maxPhys <= 0xFFFFFFFF)
389 {
390 maxPhys = 0;
391 options |= KMA_LOMEM;
392 }
393 else if (gIOLastPage && (atop_64(maxPhys) > gIOLastPage))
394 {
395 maxPhys = 0;
396 }
397 }
0b4c1975 398 if (contiguous || maxPhys)
55e303ae 399 {
0c530ab8 400 kr = kmem_alloc_contig(kernel_map, &virt, size,
b0d623f7 401 alignMask, atop(maxPhys), atop(alignMask), 0);
55e303ae
A
402 }
403 else
404 {
0c530ab8 405 kr = kernel_memory_allocate(kernel_map, &virt,
0b4c1975 406 size, alignMask, options);
55e303ae 407 }
0c530ab8
A
408 if (KERN_SUCCESS == kr)
409 address = virt;
410 else
1c79356b 411 address = 0;
55e303ae
A
412 }
413 else
414 {
1c79356b 415 adjustedSize += alignMask;
0c530ab8 416 allocationAddress = (mach_vm_address_t) kalloc(adjustedSize);
9bccf70c 417
1c79356b
A
418 if (allocationAddress) {
419
420 address = (allocationAddress + alignMask
0c530ab8 421 + (sizeof(mach_vm_size_t) + sizeof(mach_vm_address_t)))
1c79356b
A
422 & (~alignMask);
423
55e303ae 424 if (atop_32(address) != atop_32(address + size - 1))
b0d623f7 425 address = round_page(address);
1c79356b 426
0c530ab8
A
427 *((mach_vm_size_t *)(address - sizeof(mach_vm_size_t)
428 - sizeof(mach_vm_address_t))) = adjustedSize;
429 *((mach_vm_address_t *)(address - sizeof(mach_vm_address_t)))
1c79356b
A
430 = allocationAddress;
431 } else
432 address = 0;
433 }
434
2d21ac55 435 if (address) {
7ddcb079
A
436 IOStatisticsAlloc(kIOStatisticsMallocContiguous, size);
437#if IOALLOCDEBUG
b0d623f7 438 debug_iomalloc_size += size;
0c530ab8 439#endif
7ddcb079 440 }
0c530ab8
A
441
442 return (address);
443}
444
6d2010ae 445
0c530ab8
A
446/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
447
448struct _IOMallocContiguousEntry
449{
450 mach_vm_address_t virtualAddr;
451 IOBufferMemoryDescriptor * md;
452 queue_chain_t link;
453};
454typedef struct _IOMallocContiguousEntry _IOMallocContiguousEntry;
455
456void * IOMallocContiguous(vm_size_t size, vm_size_t alignment,
457 IOPhysicalAddress * physicalAddress)
458{
459 mach_vm_address_t address = 0;
460
461 if (size == 0)
462 return 0;
463 if (alignment == 0)
464 alignment = 1;
465
55e303ae 466 /* Do we want a physical address? */
0c530ab8 467 if (!physicalAddress)
c0fea474 468 {
0b4c1975 469 address = IOKernelAllocateWithPhysicalRestrict(size, 0 /*maxPhys*/, alignment, true);
0c530ab8
A
470 }
471 else do
472 {
473 IOBufferMemoryDescriptor * bmd;
474 mach_vm_address_t physicalMask;
b0d623f7 475 vm_offset_t alignMask;
0c530ab8
A
476
477 alignMask = alignment - 1;
b0d623f7
A
478 physicalMask = (0xFFFFFFFF ^ alignMask);
479
0c530ab8
A
480 bmd = IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
481 kernel_task, kIOMemoryPhysicallyContiguous, size, physicalMask);
482 if (!bmd)
483 break;
484
485 _IOMallocContiguousEntry *
486 entry = IONew(_IOMallocContiguousEntry, 1);
487 if (!entry)
55e303ae 488 {
0c530ab8
A
489 bmd->release();
490 break;
55e303ae 491 }
0c530ab8
A
492 entry->virtualAddr = (mach_vm_address_t) bmd->getBytesNoCopy();
493 entry->md = bmd;
494 lck_mtx_lock(gIOMallocContiguousEntriesLock);
495 queue_enter( &gIOMallocContiguousEntries, entry,
496 _IOMallocContiguousEntry *, link );
497 lck_mtx_unlock(gIOMallocContiguousEntriesLock);
498
499 address = (mach_vm_address_t) entry->virtualAddr;
500 *physicalAddress = bmd->getPhysicalAddress();
55e303ae 501 }
0c530ab8 502 while (false);
1c79356b
A
503
504 return (void *) address;
505}
506
0c530ab8 507void IOFreeContiguous(void * _address, vm_size_t size)
1c79356b 508{
55e303ae 509 _IOMallocContiguousEntry * entry;
0c530ab8
A
510 IOMemoryDescriptor * md = NULL;
511
512 mach_vm_address_t address = (mach_vm_address_t) _address;
1c79356b
A
513
514 if( !address)
515 return;
516
517 assert(size);
518
91447636 519 lck_mtx_lock(gIOMallocContiguousEntriesLock);
55e303ae
A
520 queue_iterate( &gIOMallocContiguousEntries, entry,
521 _IOMallocContiguousEntry *, link )
522 {
0c530ab8
A
523 if( entry->virtualAddr == address ) {
524 md = entry->md;
55e303ae
A
525 queue_remove( &gIOMallocContiguousEntries, entry,
526 _IOMallocContiguousEntry *, link );
527 break;
528 }
529 }
91447636 530 lck_mtx_unlock(gIOMallocContiguousEntriesLock);
55e303ae 531
0c530ab8 532 if (md)
55e303ae 533 {
0c530ab8 534 md->release();
55e303ae
A
535 IODelete(entry, _IOMallocContiguousEntry, 1);
536 }
0c530ab8
A
537 else
538 {
0b4c1975 539 IOKernelFreePhysical((mach_vm_address_t) address, size);
1c79356b 540 }
1c79356b
A
541}
542
543/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
544
0b4e3aa0
A
545kern_return_t IOIteratePageableMaps(vm_size_t size,
546 IOIteratePageableMapsCallback callback, void * ref)
1c79356b
A
547{
548 kern_return_t kr = kIOReturnNotReady;
1c79356b
A
549 vm_size_t segSize;
550 UInt32 attempts;
551 UInt32 index;
552 vm_offset_t min;
553 vm_map_t map;
554
1c79356b 555 if (size > kIOPageableMaxMapSize)
0b4e3aa0 556 return( kIOReturnBadArgument );
1c79356b
A
557
558 do {
559 index = gIOKitPageableSpace.hint;
560 attempts = gIOKitPageableSpace.count;
561 while( attempts--) {
0b4e3aa0 562 kr = (*callback)(gIOKitPageableSpace.maps[index].map, ref);
1c79356b
A
563 if( KERN_SUCCESS == kr) {
564 gIOKitPageableSpace.hint = index;
565 break;
566 }
567 if( index)
568 index--;
569 else
570 index = gIOKitPageableSpace.count - 1;
571 }
572 if( KERN_SUCCESS == kr)
573 break;
574
91447636 575 lck_mtx_lock( gIOKitPageableSpace.lock );
1c79356b
A
576
577 index = gIOKitPageableSpace.count;
578 if( index >= (kIOMaxPageableMaps - 1)) {
91447636 579 lck_mtx_unlock( gIOKitPageableSpace.lock );
1c79356b
A
580 break;
581 }
582
583 if( size < kIOPageableMapSize)
584 segSize = kIOPageableMapSize;
585 else
586 segSize = size;
587
588 min = 0;
589 kr = kmem_suballoc(kernel_map,
590 &min,
591 segSize,
592 TRUE,
91447636 593 VM_FLAGS_ANYWHERE,
1c79356b
A
594 &map);
595 if( KERN_SUCCESS != kr) {
91447636 596 lck_mtx_unlock( gIOKitPageableSpace.lock );
1c79356b
A
597 break;
598 }
599
600 gIOKitPageableSpace.maps[index].map = map;
601 gIOKitPageableSpace.maps[index].address = min;
602 gIOKitPageableSpace.maps[index].end = min + segSize;
603 gIOKitPageableSpace.hint = index;
604 gIOKitPageableSpace.count = index + 1;
605
91447636 606 lck_mtx_unlock( gIOKitPageableSpace.lock );
1c79356b
A
607
608 } while( true );
609
0b4e3aa0
A
610 return kr;
611}
612
613struct IOMallocPageableRef
614{
b0d623f7 615 vm_offset_t address;
0b4e3aa0
A
616 vm_size_t size;
617};
618
619static kern_return_t IOMallocPageableCallback(vm_map_t map, void * _ref)
620{
621 struct IOMallocPageableRef * ref = (struct IOMallocPageableRef *) _ref;
622 kern_return_t kr;
623
624 kr = kmem_alloc_pageable( map, &ref->address, ref->size );
625
626 return( kr );
627}
628
629void * IOMallocPageable(vm_size_t size, vm_size_t alignment)
630{
631 kern_return_t kr = kIOReturnNotReady;
632 struct IOMallocPageableRef ref;
633
634 if (alignment > page_size)
635 return( 0 );
636 if (size > kIOPageableMaxMapSize)
637 return( 0 );
638
639 ref.size = size;
640 kr = IOIteratePageableMaps( size, &IOMallocPageableCallback, &ref );
641 if( kIOReturnSuccess != kr)
642 ref.address = 0;
1c79356b 643
6d2010ae 644 if( ref.address) {
1c79356b 645#if IOALLOCDEBUG
b0d623f7 646 debug_iomallocpageable_size += round_page(size);
1c79356b 647#endif
6d2010ae
A
648 IOStatisticsAlloc(kIOStatisticsMallocPageable, size);
649 }
1c79356b 650
0b4e3aa0 651 return( (void *) ref.address );
1c79356b
A
652}
653
b0d623f7 654vm_map_t IOPageableMapForAddress( uintptr_t address )
1c79356b
A
655{
656 vm_map_t map = 0;
657 UInt32 index;
658
659 for( index = 0; index < gIOKitPageableSpace.count; index++) {
660 if( (address >= gIOKitPageableSpace.maps[index].address)
661 && (address < gIOKitPageableSpace.maps[index].end) ) {
662 map = gIOKitPageableSpace.maps[index].map;
663 break;
664 }
665 }
666 if( !map)
b0d623f7 667 panic("IOPageableMapForAddress: null");
1c79356b
A
668
669 return( map );
670}
671
672void IOFreePageable(void * address, vm_size_t size)
673{
674 vm_map_t map;
675
676 map = IOPageableMapForAddress( (vm_address_t) address);
677 if( map)
678 kmem_free( map, (vm_offset_t) address, size);
679
680#if IOALLOCDEBUG
b0d623f7 681 debug_iomallocpageable_size -= round_page(size);
1c79356b 682#endif
6d2010ae
A
683
684 IOStatisticsAlloc(kIOStatisticsFreePageable, size);
1c79356b 685}
b0d623f7 686
1c79356b
A
687/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
688
1c79356b
A
689IOReturn IOSetProcessorCacheMode( task_t task, IOVirtualAddress address,
690 IOByteCount length, IOOptionBits cacheMode )
691{
692 IOReturn ret = kIOReturnSuccess;
55e303ae 693 ppnum_t pagenum;
1c79356b
A
694
695 if( task != kernel_task)
696 return( kIOReturnUnsupported );
b0d623f7
A
697 if ((address | length) & PAGE_MASK)
698 {
699// OSReportWithBacktrace("IOSetProcessorCacheMode(0x%x, 0x%x, 0x%x) fails\n", address, length, cacheMode);
700 return( kIOReturnUnsupported );
701 }
702 length = round_page(address + length) - trunc_page( address );
703 address = trunc_page( address );
1c79356b
A
704
705 // make map mode
706 cacheMode = (cacheMode << kIOMapCacheShift) & kIOMapCacheMask;
707
708 while( (kIOReturnSuccess == ret) && (length > 0) ) {
709
55e303ae
A
710 // Get the physical page number
711 pagenum = pmap_find_phys(kernel_pmap, (addr64_t)address);
712 if( pagenum) {
713 ret = IOUnmapPages( get_task_map(task), address, page_size );
0c530ab8 714 ret = IOMapPages( get_task_map(task), address, ptoa_64(pagenum), page_size, cacheMode );
55e303ae 715 } else
1c79356b
A
716 ret = kIOReturnVMError;
717
55e303ae 718 address += page_size;
1c79356b
A
719 length -= page_size;
720 }
721
722 return( ret );
723}
724
725
726IOReturn IOFlushProcessorCache( task_t task, IOVirtualAddress address,
727 IOByteCount length )
728{
729 if( task != kernel_task)
730 return( kIOReturnUnsupported );
731
55e303ae 732 flush_dcache64( (addr64_t) address, (unsigned) length, false );
1c79356b
A
733
734 return( kIOReturnSuccess );
735}
736
737/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
738
b0d623f7 739vm_offset_t OSKernelStackRemaining( void )
1c79356b 740{
b0d623f7 741 return (ml_stack_remaining());
1c79356b
A
742}
743
744/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
745
2d21ac55
A
746/*
747 * Spin for indicated number of milliseconds.
748 */
1c79356b
A
749void IOSleep(unsigned milliseconds)
750{
91447636 751 delay_for_interval(milliseconds, kMillisecondScale);
1c79356b
A
752}
753
754/*
755 * Spin for indicated number of microseconds.
756 */
757void IODelay(unsigned microseconds)
758{
91447636 759 delay_for_interval(microseconds, kMicrosecondScale);
1c79356b
A
760}
761
2d21ac55
A
762/*
763 * Spin for indicated number of nanoseconds.
764 */
765void IOPause(unsigned nanoseconds)
766{
767 delay_for_interval(nanoseconds, kNanosecondScale);
768}
769
1c79356b
A
770/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
771
6d2010ae 772static void _iolog_consputc(int ch, void *arg __unused)
b0d623f7 773{
6d2010ae
A
774 cons_putc_locked(ch);
775}
776
777static void _iolog_logputc(int ch, void *arg __unused)
778{
779 log_putc_locked(ch);
b0d623f7
A
780}
781
1c79356b
A
782void IOLog(const char *format, ...)
783{
6d2010ae 784 va_list ap;
1c79356b 785
6d2010ae
A
786 va_start(ap, format);
787 IOLogv(format, ap);
788 va_end(ap);
1c79356b
A
789}
790
b0d623f7
A
791void IOLogv(const char *format, va_list ap)
792{
6d2010ae
A
793 va_list ap2;
794
795 va_copy(ap2, ap);
796
797 bsd_log_lock();
798 __doprnt(format, ap, _iolog_logputc, NULL, 16);
799 bsd_log_unlock();
800
801 __doprnt(format, ap2, _iolog_consputc, NULL, 16);
b0d623f7
A
802}
803
804#if !__LP64__
1c79356b
A
805void IOPanic(const char *reason)
806{
2d21ac55 807 panic("%s", reason);
1c79356b 808}
b0d623f7 809#endif
1c79356b
A
810
811/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
812
813/*
814 * Convert a integer constant (typically a #define or enum) to a string.
815 */
816static char noValue[80]; // that's pretty
817
818const char *IOFindNameForValue(int value, const IONamedValue *regValueArray)
819{
820 for( ; regValueArray->name; regValueArray++) {
821 if(regValueArray->value == value)
822 return(regValueArray->name);
823 }
2d21ac55 824 snprintf(noValue, sizeof(noValue), "0x%x (UNDEFINED)", value);
1c79356b
A
825 return((const char *)noValue);
826}
827
828IOReturn IOFindValueForName(const char *string,
829 const IONamedValue *regValueArray,
830 int *value)
831{
832 for( ; regValueArray->name; regValueArray++) {
833 if(!strcmp(regValueArray->name, string)) {
834 *value = regValueArray->value;
835 return kIOReturnSuccess;
836 }
837 }
838 return kIOReturnBadArgument;
839}
840
2d21ac55
A
841OSString * IOCopyLogNameForPID(int pid)
842{
843 char buf[128];
844 size_t len;
845 snprintf(buf, sizeof(buf), "pid %d, ", pid);
846 len = strlen(buf);
847 proc_name(pid, buf + len, sizeof(buf) - len);
848 return (OSString::withCString(buf));
849}
850
1c79356b
A
851/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
852
853IOAlignment IOSizeToAlignment(unsigned int size)
854{
855 register int shift;
856 const int intsize = sizeof(unsigned int) * 8;
857
858 for (shift = 1; shift < intsize; shift++) {
859 if (size & 0x80000000)
860 return (IOAlignment)(intsize - shift);
861 size <<= 1;
862 }
863 return 0;
864}
865
866unsigned int IOAlignmentToSize(IOAlignment align)
867{
868 unsigned int size;
869
870 for (size = 1; align; align--) {
871 size <<= 1;
872 }
873 return size;
874}
0c530ab8
A
875
876} /* extern "C" */
2d21ac55
A
877
878
879