iokit/Kernel/IOMapper.cpp

   1 /*
   2  * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22 #include <IOKit/IOLib.h>
  23 #include <IOKit/IOMapper.h>
  24 #include <libkern/c++/OSData.h>
  25
  26 #define super IOService
  27 OSDefineMetaClassAndAbstractStructors(IOMapper, IOService);
  28
  29 OSMetaClassDefineReservedUnused(IOMapper, 0);
  30 OSMetaClassDefineReservedUnused(IOMapper, 1);
  31 OSMetaClassDefineReservedUnused(IOMapper, 2);
  32 OSMetaClassDefineReservedUnused(IOMapper, 3);
  33 OSMetaClassDefineReservedUnused(IOMapper, 4);
  34 OSMetaClassDefineReservedUnused(IOMapper, 5);
  35 OSMetaClassDefineReservedUnused(IOMapper, 6);
  36 OSMetaClassDefineReservedUnused(IOMapper, 7);
  37 OSMetaClassDefineReservedUnused(IOMapper, 8);
  38 OSMetaClassDefineReservedUnused(IOMapper, 9);
  39 OSMetaClassDefineReservedUnused(IOMapper, 10);
  40 OSMetaClassDefineReservedUnused(IOMapper, 11);
  41 OSMetaClassDefineReservedUnused(IOMapper, 12);
  42 OSMetaClassDefineReservedUnused(IOMapper, 13);
  43 OSMetaClassDefineReservedUnused(IOMapper, 14);
  44 OSMetaClassDefineReservedUnused(IOMapper, 15);
  45
  46 IOMapper * IOMapper::gSystem = (IOMapper *) IOMapper::kUnknown;
  47
  48 class IOMapperLock {
  49     IOLock *fWaitLock;
  50 public:
  51     IOMapperLock() { fWaitLock = IOLockAlloc(); };
  52     ~IOMapperLock() { IOLockFree(fWaitLock); };
  53
  54     void lock()   { IOLockLock(fWaitLock); };
  55     void unlock() { IOLockUnlock(fWaitLock); };
  56     void sleep(void *event)  { IOLockSleep(fWaitLock, event, THREAD_UNINT); };
  57     void wakeup(void *event) { IOLockWakeup(fWaitLock, event, false); };
  58 };
  59
  60 static IOMapperLock sMapperLock;
  61
  62 bool IOMapper::start(IOService *provider)
  63 {
  64     if (!super::start(provider))
  65         return false;
  66
  67     if (!initHardware(provider))
  68         return false;
  69
  70     if (fIsSystem) {
  71         sMapperLock.lock();
  72         IOMapper::gSystem = this;
  73         sMapperLock.wakeup(&IOMapper::gSystem);
  74         sMapperLock.unlock();
  75     }
  76
  77     return true;
  78 }
  79
  80 bool IOMapper::allocTable(IOByteCount size)
  81 {
  82     assert(!fTable);
  83
  84     fTableSize = size;
  85     fTableHandle = NewARTTable(size, &fTable, &fTablePhys);
  86     return fTableHandle != 0;
  87 }
  88
  89 void IOMapper::free()
  90 {
  91     if (fTableHandle) {
  92         FreeARTTable(fTableHandle, fTableSize);
  93         fTableHandle = 0;
  94     }
  95
  96     super::free();
  97 }
  98
  99 void IOMapper::setMapperRequired(bool hasMapper)
 100 {
 101     if (hasMapper)
 102         IOMapper::gSystem = (IOMapper *) kHasMapper;
 103     else {
 104         sMapperLock.lock();
 105         IOMapper::gSystem = (IOMapper *) kNoMapper;
 106         sMapperLock.unlock();
 107         sMapperLock.wakeup(&IOMapper::gSystem);
 108     }
 109 }
 110
 111 void IOMapper::waitForSystemMapper()
 112 {
 113     sMapperLock.lock();
 114     while ((vm_address_t) IOMapper::gSystem & kWaitMask)
 115         sMapperLock.sleep(&IOMapper::gSystem);
 116     sMapperLock.unlock();
 117 }
 118
 119 void IOMapper::iovmInsert(ppnum_t addr, IOItemCount offset,
 120                             ppnum_t *pageList, IOItemCount pageCount)
 121 {
 122     while (pageCount--)
 123         iovmInsert(addr, offset++, *pageList++);
 124 }
 125
 126 void IOMapper::iovmInsert(ppnum_t addr, IOItemCount offset,
 127                             upl_page_info_t *pageList, IOItemCount pageCount)
 128 {
 129     for (IOItemCount i = 0; i < pageCount; i++)
 130         iovmInsert(addr, offset + i, pageList[i].phys_addr);
 131 }
 132
 133 struct ARTTableData {
 134     void *v;
 135     upl_t u[0];
 136 };
 137 #define getARTDataP(data) ((ARTTableData *) (data)->getBytesNoCopy())
 138
 139 OSData *
 140 IOMapper::NewARTTable(IOByteCount size,
 141                       void ** virtAddrP, ppnum_t *physAddrP)
 142 {
 143     OSData *ret;
 144     kern_return_t kr;
 145     vm_address_t startUpl;
 146     ARTTableData *dataP;
 147     unsigned int dataSize;
 148     upl_page_info_t *pl = 0;
 149
 150     // Each UPL can deal with about one meg at the moment
 151     size = round_page_32(size);
 152     dataSize = sizeof(ARTTableData) + sizeof(upl_t) * size / (1024 * 1024);
 153     ret = OSData::withCapacity(dataSize);
 154     if (!ret)
 155         return 0;
 156
 157     // Append 0's to the buffer, in-other-words reset to nulls.
 158     ret->appendBytes(NULL, sizeof(ARTTableData));
 159     dataP = getARTDataP(ret);
 160
 161     kr = kmem_alloc_contig(kernel_map, &startUpl, size, PAGE_MASK, 0);
 162     if (kr)
 163         return 0;
 164
 165     dataP->v = (void *) startUpl;
 166
 167     do {
 168         upl_t iopl;
 169         int upl_flags = UPL_SET_INTERNAL | UPL_SET_LITE
 170                       | UPL_SET_IO_WIRE | UPL_COPYOUT_FROM;
 171         vm_size_t iopl_size = size;
 172
 173         kr = vm_map_get_upl(kernel_map,
 174                             (vm_map_offset_t)startUpl,
 175                             &iopl_size,
 176                             &iopl,
 177                             0,
 178                             0,
 179                             &upl_flags,
 180                             0);
 181         if (kr) {
 182             panic("IOMapper:vm_map_get_upl returned 0x%x\n");
 183             goto bail;
 184         }
 185
 186         if (!ret->appendBytes(&iopl, sizeof(upl_t)))
 187             goto bail;
 188
 189         startUpl += iopl_size;
 190         size -= iopl_size;
 191     } while(size);
 192
 193     // Need to re-establish the dataP as the OSData may have grown.
 194     dataP = getARTDataP(ret);
 195
 196     // Now grab the page entry of the first page and get its phys addr
 197     pl = UPL_GET_INTERNAL_PAGE_LIST(dataP->u[0]);
 198     *physAddrP = pl->phys_addr;
 199     *virtAddrP = dataP->v;
 200
 201     return ret;
 202
 203 bail:
 204     FreeARTTable(ret, size);
 205     return 0;
 206 }
 207
 208 void IOMapper::FreeARTTable(OSData *artHandle, IOByteCount size)
 209 {
 210     assert(artHandle);
 211
 212     ARTTableData *dataP = getARTDataP(artHandle);
 213
 214     int numupls = ((artHandle->getLength() - sizeof(*dataP)) / sizeof(upl_t));
 215     for (int i = 0; i < numupls; i++) {
 216         upl_abort(dataP->u[i], 0);
 217         upl_deallocate(dataP->u[i]);
 218     }
 219
 220     if (dataP->v) {
 221         size = round_page_32(size);
 222         kmem_free(kernel_map, (vm_address_t) dataP->v, size);
 223     }
 224     artHandle->release();
 225 }
 226
 227 __BEGIN_DECLS
 228
 229 // These are C accessors to the system mapper for non-IOKit clients
 230 ppnum_t IOMapperIOVMAlloc(unsigned pages)
 231 {
 232     IOMapper::checkForSystemMapper();
 233
 234     if (IOMapper::gSystem)
 235         return IOMapper::gSystem->iovmAlloc((IOItemCount) pages);
 236     else
 237         return 0;
 238 }
 239
 240 void IOMapperIOVMFree(ppnum_t addr, unsigned pages)
 241 {
 242     if (IOMapper::gSystem)
 243         IOMapper::gSystem->iovmFree(addr, (IOItemCount) pages);
 244 }
 245
 246 ppnum_t IOMapperInsertPage(ppnum_t addr, unsigned offset, ppnum_t page)
 247 {
 248     if (IOMapper::gSystem) {
 249         IOMapper::gSystem->iovmInsert(addr, (IOItemCount) offset, page);
 250         return addr + offset;
 251     }
 252     else
 253         return page;
 254 }
 255
 256 void IOMapperInsertPPNPages(ppnum_t addr, unsigned offset,
 257                             ppnum_t *pageList, unsigned pageCount)
 258 {
 259     if (!IOMapper::gSystem)
 260         panic("IOMapperInsertPPNPages no system mapper");
 261     else
 262         assert(!((vm_address_t) IOMapper::gSystem & 3));
 263
 264     IOMapper::gSystem->
 265         iovmInsert(addr, (IOItemCount) offset, pageList, pageCount);
 266 }
 267
 268 void IOMapperInsertUPLPages(ppnum_t addr, unsigned offset,
 269                             upl_page_info_t *pageList, unsigned pageCount)
 270 {
 271     if (!IOMapper::gSystem)
 272         panic("IOMapperInsertUPLPages no system mapper");
 273     else
 274         assert(!((vm_address_t) IOMapper::gSystem & 3));
 275
 276     IOMapper::gSystem->iovmInsert(addr,
 277                                  (IOItemCount) offset,
 278                                   pageList,
 279                                   (IOItemCount) pageCount);
 280 }
 281
 282 /////////////////////////////////////////////////////////////////////////////
 283 //
 284 //
 285 //      IOLib.h APIs
 286 //
 287 //
 288 /////////////////////////////////////////////////////////////////////////////
 289
 290 #include <machine/machine_routines.h>
 291
 292 UInt8 IOMappedRead8(IOPhysicalAddress address)
 293 {
 294     IOMapper::checkForSystemMapper();
 295
 296     if (IOMapper::gSystem) {
 297         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 298         return (UInt8) ml_phys_read_byte_64(addr);
 299     }
 300     else
 301         return (UInt8) ml_phys_read_byte((vm_offset_t) address);
 302 }
 303
 304 UInt16 IOMappedRead16(IOPhysicalAddress address)
 305 {
 306     IOMapper::checkForSystemMapper();
 307
 308     if (IOMapper::gSystem) {
 309         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 310         return (UInt16) ml_phys_read_half_64(addr);
 311     }
 312     else
 313         return (UInt16) ml_phys_read_half((vm_offset_t) address);
 314 }
 315
 316 UInt32 IOMappedRead32(IOPhysicalAddress address)
 317 {
 318     IOMapper::checkForSystemMapper();
 319
 320     if (IOMapper::gSystem) {
 321         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 322         return (UInt32) ml_phys_read_word_64(addr);
 323     }
 324     else
 325         return (UInt32) ml_phys_read_word((vm_offset_t) address);
 326 }
 327
 328 UInt64 IOMappedRead64(IOPhysicalAddress address)
 329 {
 330     IOMapper::checkForSystemMapper();
 331
 332     if (IOMapper::gSystem) {
 333         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 334         return (UInt64) ml_phys_read_double_64(addr);
 335     }
 336     else
 337         return (UInt64) ml_phys_read_double((vm_offset_t) address);
 338 }
 339
 340 void IOMappedWrite8(IOPhysicalAddress address, UInt8 value)
 341 {
 342     IOMapper::checkForSystemMapper();
 343
 344     if (IOMapper::gSystem) {
 345         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 346         ml_phys_write_byte_64(addr, value);
 347     }
 348     else
 349         ml_phys_write_byte((vm_offset_t) address, value);
 350 }
 351
 352 void IOMappedWrite16(IOPhysicalAddress address, UInt16 value)
 353 {
 354     IOMapper::checkForSystemMapper();
 355
 356     if (IOMapper::gSystem) {
 357         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 358         ml_phys_write_half_64(addr, value);
 359     }
 360     else
 361         ml_phys_write_half((vm_offset_t) address, value);
 362 }
 363
 364 void IOMappedWrite32(IOPhysicalAddress address, UInt32 value)
 365 {
 366     IOMapper::checkForSystemMapper();
 367
 368     if (IOMapper::gSystem) {
 369         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 370         ml_phys_write_word_64(addr, value);
 371     }
 372     else
 373         ml_phys_write_word((vm_offset_t) address, value);
 374 }
 375
 376 void IOMappedWrite64(IOPhysicalAddress address, UInt64 value)
 377 {
 378     IOMapper::checkForSystemMapper();
 379
 380     if (IOMapper::gSystem) {
 381         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 382         ml_phys_write_double_64(addr, value);
 383     }
 384     else
 385         ml_phys_write_double((vm_offset_t) address, value);
 386 }
 387
 388 __END_DECLS