iokit/Kernel/IOMapper.cpp

   1 /*
   2  * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 #include <IOKit/IOLib.h>
  29 #include <IOKit/IOMapper.h>
  30 #include <libkern/c++/OSData.h>
  31
  32 #define super IOService
  33 OSDefineMetaClassAndAbstractStructors(IOMapper, IOService);
  34
  35 OSMetaClassDefineReservedUnused(IOMapper, 0);
  36 OSMetaClassDefineReservedUnused(IOMapper, 1);
  37 OSMetaClassDefineReservedUnused(IOMapper, 2);
  38 OSMetaClassDefineReservedUnused(IOMapper, 3);
  39 OSMetaClassDefineReservedUnused(IOMapper, 4);
  40 OSMetaClassDefineReservedUnused(IOMapper, 5);
  41 OSMetaClassDefineReservedUnused(IOMapper, 6);
  42 OSMetaClassDefineReservedUnused(IOMapper, 7);
  43 OSMetaClassDefineReservedUnused(IOMapper, 8);
  44 OSMetaClassDefineReservedUnused(IOMapper, 9);
  45 OSMetaClassDefineReservedUnused(IOMapper, 10);
  46 OSMetaClassDefineReservedUnused(IOMapper, 11);
  47 OSMetaClassDefineReservedUnused(IOMapper, 12);
  48 OSMetaClassDefineReservedUnused(IOMapper, 13);
  49 OSMetaClassDefineReservedUnused(IOMapper, 14);
  50 OSMetaClassDefineReservedUnused(IOMapper, 15);
  51
  52 IOMapper * IOMapper::gSystem = (IOMapper *) IOMapper::kUnknown;
  53
  54 class IOMapperLock {
  55     IOLock *fWaitLock;
  56 public:
  57     IOMapperLock() { fWaitLock = IOLockAlloc(); };
  58     ~IOMapperLock() { IOLockFree(fWaitLock); };
  59
  60     void lock()   { IOLockLock(fWaitLock); };
  61     void unlock() { IOLockUnlock(fWaitLock); };
  62     void sleep(void *event)  { IOLockSleep(fWaitLock, event, THREAD_UNINT); };
  63     void wakeup(void *event) { IOLockWakeup(fWaitLock, event, false); };
  64 };
  65
  66 static IOMapperLock sMapperLock;
  67
  68 bool IOMapper::start(IOService *provider)
  69 {
  70     if (!super::start(provider))
  71         return false;
  72
  73     if (!initHardware(provider))
  74         return false;
  75
  76     if (fIsSystem) {
  77         sMapperLock.lock();
  78         IOMapper::gSystem = this;
  79         sMapperLock.wakeup(&IOMapper::gSystem);
  80         sMapperLock.unlock();
  81     }
  82
  83     return true;
  84 }
  85
  86 bool IOMapper::allocTable(IOByteCount size)
  87 {
  88     assert(!fTable);
  89
  90     fTableSize = size;
  91     fTableHandle = NewARTTable(size, &fTable, &fTablePhys);
  92     return fTableHandle != 0;
  93 }
  94
  95 void IOMapper::free()
  96 {
  97     if (fTableHandle) {
  98         FreeARTTable(fTableHandle, fTableSize);
  99         fTableHandle = 0;
 100     }
 101
 102     super::free();
 103 }
 104
 105 void IOMapper::setMapperRequired(bool hasMapper)
 106 {
 107     if (hasMapper)
 108         IOMapper::gSystem = (IOMapper *) kHasMapper;
 109     else {
 110         sMapperLock.lock();
 111         IOMapper::gSystem = (IOMapper *) kNoMapper;
 112         sMapperLock.unlock();
 113         sMapperLock.wakeup(&IOMapper::gSystem);
 114     }
 115 }
 116
 117 void IOMapper::waitForSystemMapper()
 118 {
 119     sMapperLock.lock();
 120     while ((vm_address_t) IOMapper::gSystem & kWaitMask)
 121         sMapperLock.sleep(&IOMapper::gSystem);
 122     sMapperLock.unlock();
 123 }
 124
 125 void IOMapper::iovmInsert(ppnum_t addr, IOItemCount offset,
 126                             ppnum_t *pageList, IOItemCount pageCount)
 127 {
 128     while (pageCount--)
 129         iovmInsert(addr, offset++, *pageList++);
 130 }
 131
 132 void IOMapper::iovmInsert(ppnum_t addr, IOItemCount offset,
 133                             upl_page_info_t *pageList, IOItemCount pageCount)
 134 {
 135     for (IOItemCount i = 0; i < pageCount; i++)
 136         iovmInsert(addr, offset + i, pageList[i].phys_addr);
 137 }
 138
 139 struct ARTTableData {
 140     void *v;
 141     upl_t u[0];
 142 };
 143 #define getARTDataP(data) ((ARTTableData *) (data)->getBytesNoCopy())
 144
 145 OSData *
 146 IOMapper::NewARTTable(IOByteCount size,
 147                       void ** virtAddrP, ppnum_t *physAddrP)
 148 {
 149     OSData *ret;
 150     kern_return_t kr;
 151     vm_address_t startUpl;
 152     ARTTableData *dataP;
 153     unsigned int dataSize;
 154     upl_page_info_t *pl = 0;
 155
 156     // Each UPL can deal with about one meg at the moment
 157     size = round_page_32(size);
 158     dataSize = sizeof(ARTTableData) + sizeof(upl_t) * size / (1024 * 1024);
 159     ret = OSData::withCapacity(dataSize);
 160     if (!ret)
 161         return 0;
 162
 163     // Append 0's to the buffer, in-other-words reset to nulls.
 164     ret->appendBytes(NULL, sizeof(ARTTableData));
 165     dataP = getARTDataP(ret);
 166
 167     kr = kmem_alloc_contig(kernel_map, &startUpl, size, PAGE_MASK, 0);
 168     if (kr)
 169         return 0;
 170
 171     dataP->v = (void *) startUpl;
 172
 173     do {
 174         upl_t iopl;
 175         int upl_flags = UPL_SET_INTERNAL | UPL_SET_LITE
 176                       | UPL_SET_IO_WIRE | UPL_COPYOUT_FROM;
 177         vm_size_t iopl_size = size;
 178
 179         kr = vm_map_get_upl(kernel_map,
 180                             (vm_map_offset_t)startUpl,
 181                             &iopl_size,
 182                             &iopl,
 183                             0,
 184                             0,
 185                             &upl_flags,
 186                             0);
 187         if (kr) {
 188             panic("IOMapper:vm_map_get_upl returned 0x%x\n");
 189             goto bail;
 190         }
 191
 192         if (!ret->appendBytes(&iopl, sizeof(upl_t)))
 193             goto bail;
 194
 195         startUpl += iopl_size;
 196         size -= iopl_size;
 197     } while(size);
 198
 199     // Need to re-establish the dataP as the OSData may have grown.
 200     dataP = getARTDataP(ret);
 201
 202     // Now grab the page entry of the first page and get its phys addr
 203     pl = UPL_GET_INTERNAL_PAGE_LIST(dataP->u[0]);
 204     *physAddrP = pl->phys_addr;
 205     *virtAddrP = dataP->v;
 206
 207     return ret;
 208
 209 bail:
 210     FreeARTTable(ret, size);
 211     return 0;
 212 }
 213
 214 void IOMapper::FreeARTTable(OSData *artHandle, IOByteCount size)
 215 {
 216     assert(artHandle);
 217
 218     ARTTableData *dataP = getARTDataP(artHandle);
 219
 220     int numupls = ((artHandle->getLength() - sizeof(*dataP)) / sizeof(upl_t));
 221     for (int i = 0; i < numupls; i++) {
 222         upl_abort(dataP->u[i], 0);
 223         upl_deallocate(dataP->u[i]);
 224     }
 225
 226     if (dataP->v) {
 227         size = round_page_32(size);
 228         kmem_free(kernel_map, (vm_address_t) dataP->v, size);
 229     }
 230     artHandle->release();
 231 }
 232
 233 __BEGIN_DECLS
 234
 235 // These are C accessors to the system mapper for non-IOKit clients
 236 ppnum_t IOMapperIOVMAlloc(unsigned pages)
 237 {
 238     IOMapper::checkForSystemMapper();
 239
 240     if (IOMapper::gSystem)
 241         return IOMapper::gSystem->iovmAlloc((IOItemCount) pages);
 242     else
 243         return 0;
 244 }
 245
 246 void IOMapperIOVMFree(ppnum_t addr, unsigned pages)
 247 {
 248     if (IOMapper::gSystem)
 249         IOMapper::gSystem->iovmFree(addr, (IOItemCount) pages);
 250 }
 251
 252 ppnum_t IOMapperInsertPage(ppnum_t addr, unsigned offset, ppnum_t page)
 253 {
 254     if (IOMapper::gSystem) {
 255         IOMapper::gSystem->iovmInsert(addr, (IOItemCount) offset, page);
 256         return addr + offset;
 257     }
 258     else
 259         return page;
 260 }
 261
 262 void IOMapperInsertPPNPages(ppnum_t addr, unsigned offset,
 263                             ppnum_t *pageList, unsigned pageCount)
 264 {
 265     if (!IOMapper::gSystem)
 266         panic("IOMapperInsertPPNPages no system mapper");
 267     else
 268         assert(!((vm_address_t) IOMapper::gSystem & 3));
 269
 270     IOMapper::gSystem->
 271         iovmInsert(addr, (IOItemCount) offset, pageList, pageCount);
 272 }
 273
 274 void IOMapperInsertUPLPages(ppnum_t addr, unsigned offset,
 275                             upl_page_info_t *pageList, unsigned pageCount)
 276 {
 277     if (!IOMapper::gSystem)
 278         panic("IOMapperInsertUPLPages no system mapper");
 279     else
 280         assert(!((vm_address_t) IOMapper::gSystem & 3));
 281
 282     IOMapper::gSystem->iovmInsert(addr,
 283                                  (IOItemCount) offset,
 284                                   pageList,
 285                                   (IOItemCount) pageCount);
 286 }
 287
 288 /////////////////////////////////////////////////////////////////////////////
 289 //
 290 //
 291 //      IOLib.h APIs
 292 //
 293 //
 294 /////////////////////////////////////////////////////////////////////////////
 295
 296 #include <machine/machine_routines.h>
 297
 298 UInt8 IOMappedRead8(IOPhysicalAddress address)
 299 {
 300     IOMapper::checkForSystemMapper();
 301
 302     if (IOMapper::gSystem) {
 303         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 304         return (UInt8) ml_phys_read_byte_64(addr);
 305     }
 306     else
 307         return (UInt8) ml_phys_read_byte((vm_offset_t) address);
 308 }
 309
 310 UInt16 IOMappedRead16(IOPhysicalAddress address)
 311 {
 312     IOMapper::checkForSystemMapper();
 313
 314     if (IOMapper::gSystem) {
 315         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 316         return (UInt16) ml_phys_read_half_64(addr);
 317     }
 318     else
 319         return (UInt16) ml_phys_read_half((vm_offset_t) address);
 320 }
 321
 322 UInt32 IOMappedRead32(IOPhysicalAddress address)
 323 {
 324     IOMapper::checkForSystemMapper();
 325
 326     if (IOMapper::gSystem) {
 327         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 328         return (UInt32) ml_phys_read_word_64(addr);
 329     }
 330     else
 331         return (UInt32) ml_phys_read_word((vm_offset_t) address);
 332 }
 333
 334 UInt64 IOMappedRead64(IOPhysicalAddress address)
 335 {
 336     IOMapper::checkForSystemMapper();
 337
 338     if (IOMapper::gSystem) {
 339         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 340         return (UInt64) ml_phys_read_double_64(addr);
 341     }
 342     else
 343         return (UInt64) ml_phys_read_double((vm_offset_t) address);
 344 }
 345
 346 void IOMappedWrite8(IOPhysicalAddress address, UInt8 value)
 347 {
 348     IOMapper::checkForSystemMapper();
 349
 350     if (IOMapper::gSystem) {
 351         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 352         ml_phys_write_byte_64(addr, value);
 353     }
 354     else
 355         ml_phys_write_byte((vm_offset_t) address, value);
 356 }
 357
 358 void IOMappedWrite16(IOPhysicalAddress address, UInt16 value)
 359 {
 360     IOMapper::checkForSystemMapper();
 361
 362     if (IOMapper::gSystem) {
 363         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 364         ml_phys_write_half_64(addr, value);
 365     }
 366     else
 367         ml_phys_write_half((vm_offset_t) address, value);
 368 }
 369
 370 void IOMappedWrite32(IOPhysicalAddress address, UInt32 value)
 371 {
 372     IOMapper::checkForSystemMapper();
 373
 374     if (IOMapper::gSystem) {
 375         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 376         ml_phys_write_word_64(addr, value);
 377     }
 378     else
 379         ml_phys_write_word((vm_offset_t) address, value);
 380 }
 381
 382 void IOMappedWrite64(IOPhysicalAddress address, UInt64 value)
 383 {
 384     IOMapper::checkForSystemMapper();
 385
 386     if (IOMapper::gSystem) {
 387         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 388         ml_phys_write_double_64(addr, value);
 389     }
 390     else
 391         ml_phys_write_double((vm_offset_t) address, value);
 392 }
 393
 394 __END_DECLS