iokit/Kernel/IOMapper.cpp

   1 /*
   2  * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23 #include <IOKit/IOLib.h>
  24 #include <IOKit/IOMapper.h>
  25 #include <libkern/c++/OSData.h>
  26
  27 #define super IOService
  28 OSDefineMetaClassAndAbstractStructors(IOMapper, IOService);
  29
  30 OSMetaClassDefineReservedUnused(IOMapper, 0);
  31 OSMetaClassDefineReservedUnused(IOMapper, 1);
  32 OSMetaClassDefineReservedUnused(IOMapper, 2);
  33 OSMetaClassDefineReservedUnused(IOMapper, 3);
  34 OSMetaClassDefineReservedUnused(IOMapper, 4);
  35 OSMetaClassDefineReservedUnused(IOMapper, 5);
  36 OSMetaClassDefineReservedUnused(IOMapper, 6);
  37 OSMetaClassDefineReservedUnused(IOMapper, 7);
  38 OSMetaClassDefineReservedUnused(IOMapper, 8);
  39 OSMetaClassDefineReservedUnused(IOMapper, 9);
  40 OSMetaClassDefineReservedUnused(IOMapper, 10);
  41 OSMetaClassDefineReservedUnused(IOMapper, 11);
  42 OSMetaClassDefineReservedUnused(IOMapper, 12);
  43 OSMetaClassDefineReservedUnused(IOMapper, 13);
  44 OSMetaClassDefineReservedUnused(IOMapper, 14);
  45 OSMetaClassDefineReservedUnused(IOMapper, 15);
  46
  47 IOMapper * IOMapper::gSystem = (IOMapper *) IOMapper::kUnknown;
  48
  49 class IOMapperLock {
  50     IOLock *fWaitLock;
  51 public:
  52     IOMapperLock() { fWaitLock = IOLockAlloc(); };
  53     ~IOMapperLock() { IOLockFree(fWaitLock); };
  54
  55     void lock()   { IOLockLock(fWaitLock); };
  56     void unlock() { IOLockUnlock(fWaitLock); };
  57     void sleep(void *event)  { IOLockSleep(fWaitLock, event, THREAD_UNINT); };
  58     void wakeup(void *event) { IOLockWakeup(fWaitLock, event, false); };
  59 };
  60
  61 static IOMapperLock sMapperLock;
  62
  63 bool IOMapper::start(IOService *provider)
  64 {
  65     if (!super::start(provider))
  66         return false;
  67
  68     if (!initHardware(provider))
  69         return false;
  70
  71     if (fIsSystem) {
  72         sMapperLock.lock();
  73         IOMapper::gSystem = this;
  74         sMapperLock.wakeup(&IOMapper::gSystem);
  75         sMapperLock.unlock();
  76     }
  77
  78     return true;
  79 }
  80
  81 bool IOMapper::allocTable(IOByteCount size)
  82 {
  83     assert(!fTable);
  84
  85     fTableSize = size;
  86     fTableHandle = NewARTTable(size, &fTable, &fTablePhys);
  87     return fTableHandle != 0;
  88 }
  89
  90 void IOMapper::free()
  91 {
  92     if (fTableHandle) {
  93         FreeARTTable(fTableHandle, fTableSize);
  94         fTableHandle = 0;
  95     }
  96
  97     super::free();
  98 }
  99
 100 void IOMapper::setMapperRequired(bool hasMapper)
 101 {
 102     if (hasMapper)
 103         IOMapper::gSystem = (IOMapper *) kHasMapper;
 104     else {
 105         sMapperLock.lock();
 106         IOMapper::gSystem = (IOMapper *) kNoMapper;
 107         sMapperLock.unlock();
 108         sMapperLock.wakeup(&IOMapper::gSystem);
 109     }
 110 }
 111
 112 void IOMapper::waitForSystemMapper()
 113 {
 114     sMapperLock.lock();
 115     while ((vm_address_t) IOMapper::gSystem & kWaitMask)
 116         sMapperLock.sleep(&IOMapper::gSystem);
 117     sMapperLock.unlock();
 118 }
 119
 120 void IOMapper::iovmInsert(ppnum_t addr, IOItemCount offset,
 121                             ppnum_t *pageList, IOItemCount pageCount)
 122 {
 123     while (pageCount--)
 124         iovmInsert(addr, offset++, *pageList++);
 125 }
 126
 127 void IOMapper::iovmInsert(ppnum_t addr, IOItemCount offset,
 128                             upl_page_info_t *pageList, IOItemCount pageCount)
 129 {
 130     for (IOItemCount i = 0; i < pageCount; i++)
 131         iovmInsert(addr, offset + i, pageList[i].phys_addr);
 132 }
 133
 134 struct ARTTableData {
 135     void *v;
 136     upl_t u[0];
 137 };
 138 #define getARTDataP(data) ((ARTTableData *) (data)->getBytesNoCopy())
 139
 140 OSData *
 141 IOMapper::NewARTTable(IOByteCount size,
 142                       void ** virtAddrP, ppnum_t *physAddrP)
 143 {
 144     OSData *ret;
 145     kern_return_t kr;
 146     vm_address_t startUpl;
 147     ARTTableData *dataP;
 148     unsigned int dataSize;
 149     upl_page_info_t *pl = 0;
 150
 151     // Each UPL can deal with about one meg at the moment
 152     size = round_page_32(size);
 153     dataSize = sizeof(ARTTableData) + sizeof(upl_t) * size / (1024 * 1024);
 154     ret = OSData::withCapacity(dataSize);
 155     if (!ret)
 156         return 0;
 157
 158     // Append 0's to the buffer, in-other-words reset to nulls.
 159     ret->appendBytes(NULL, sizeof(ARTTableData));
 160     dataP = getARTDataP(ret);
 161
 162     kr = kmem_alloc_contig(kernel_map, &startUpl, size, PAGE_MASK, 0);
 163     if (kr)
 164         return 0;
 165
 166     dataP->v = (void *) startUpl;
 167
 168     do {
 169         upl_t iopl;
 170         int upl_flags = UPL_SET_INTERNAL | UPL_SET_LITE
 171                       | UPL_SET_IO_WIRE | UPL_COPYOUT_FROM;
 172         vm_size_t iopl_size = size;
 173
 174         kr = vm_map_get_upl(kernel_map,
 175                             (vm_map_offset_t)startUpl,
 176                             &iopl_size,
 177                             &iopl,
 178                             0,
 179                             0,
 180                             &upl_flags,
 181                             0);
 182         if (kr) {
 183             panic("IOMapper:vm_map_get_upl returned 0x%x\n");
 184             goto bail;
 185         }
 186
 187         if (!ret->appendBytes(&iopl, sizeof(upl_t)))
 188             goto bail;
 189
 190         startUpl += iopl_size;
 191         size -= iopl_size;
 192     } while(size);
 193
 194     // Need to re-establish the dataP as the OSData may have grown.
 195     dataP = getARTDataP(ret);
 196
 197     // Now grab the page entry of the first page and get its phys addr
 198     pl = UPL_GET_INTERNAL_PAGE_LIST(dataP->u[0]);
 199     *physAddrP = pl->phys_addr;
 200     *virtAddrP = dataP->v;
 201
 202     return ret;
 203
 204 bail:
 205     FreeARTTable(ret, size);
 206     return 0;
 207 }
 208
 209 void IOMapper::FreeARTTable(OSData *artHandle, IOByteCount size)
 210 {
 211     assert(artHandle);
 212
 213     ARTTableData *dataP = getARTDataP(artHandle);
 214
 215     int numupls = ((artHandle->getLength() - sizeof(*dataP)) / sizeof(upl_t));
 216     for (int i = 0; i < numupls; i++) {
 217         upl_abort(dataP->u[i], 0);
 218         upl_deallocate(dataP->u[i]);
 219     }
 220
 221     if (dataP->v) {
 222         size = round_page_32(size);
 223         kmem_free(kernel_map, (vm_address_t) dataP->v, size);
 224     }
 225     artHandle->release();
 226 }
 227
 228 __BEGIN_DECLS
 229
 230 // These are C accessors to the system mapper for non-IOKit clients
 231 ppnum_t IOMapperIOVMAlloc(unsigned pages)
 232 {
 233     IOMapper::checkForSystemMapper();
 234
 235     if (IOMapper::gSystem)
 236         return IOMapper::gSystem->iovmAlloc((IOItemCount) pages);
 237     else
 238         return 0;
 239 }
 240
 241 void IOMapperIOVMFree(ppnum_t addr, unsigned pages)
 242 {
 243     if (IOMapper::gSystem)
 244         IOMapper::gSystem->iovmFree(addr, (IOItemCount) pages);
 245 }
 246
 247 ppnum_t IOMapperInsertPage(ppnum_t addr, unsigned offset, ppnum_t page)
 248 {
 249     if (IOMapper::gSystem) {
 250         IOMapper::gSystem->iovmInsert(addr, (IOItemCount) offset, page);
 251         return addr + offset;
 252     }
 253     else
 254         return page;
 255 }
 256
 257 void IOMapperInsertPPNPages(ppnum_t addr, unsigned offset,
 258                             ppnum_t *pageList, unsigned pageCount)
 259 {
 260     if (!IOMapper::gSystem)
 261         panic("IOMapperInsertPPNPages no system mapper");
 262     else
 263         assert(!((vm_address_t) IOMapper::gSystem & 3));
 264
 265     IOMapper::gSystem->
 266         iovmInsert(addr, (IOItemCount) offset, pageList, pageCount);
 267 }
 268
 269 void IOMapperInsertUPLPages(ppnum_t addr, unsigned offset,
 270                             upl_page_info_t *pageList, unsigned pageCount)
 271 {
 272     if (!IOMapper::gSystem)
 273         panic("IOMapperInsertUPLPages no system mapper");
 274     else
 275         assert(!((vm_address_t) IOMapper::gSystem & 3));
 276
 277     IOMapper::gSystem->iovmInsert(addr,
 278                                  (IOItemCount) offset,
 279                                   pageList,
 280                                   (IOItemCount) pageCount);
 281 }
 282
 283 /////////////////////////////////////////////////////////////////////////////
 284 //
 285 //
 286 //      IOLib.h APIs
 287 //
 288 //
 289 /////////////////////////////////////////////////////////////////////////////
 290
 291 #include <machine/machine_routines.h>
 292
 293 UInt8 IOMappedRead8(IOPhysicalAddress address)
 294 {
 295     IOMapper::checkForSystemMapper();
 296
 297     if (IOMapper::gSystem) {
 298         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 299         return (UInt8) ml_phys_read_byte_64(addr);
 300     }
 301     else
 302         return (UInt8) ml_phys_read_byte((vm_offset_t) address);
 303 }
 304
 305 UInt16 IOMappedRead16(IOPhysicalAddress address)
 306 {
 307     IOMapper::checkForSystemMapper();
 308
 309     if (IOMapper::gSystem) {
 310         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 311         return (UInt16) ml_phys_read_half_64(addr);
 312     }
 313     else
 314         return (UInt16) ml_phys_read_half((vm_offset_t) address);
 315 }
 316
 317 UInt32 IOMappedRead32(IOPhysicalAddress address)
 318 {
 319     IOMapper::checkForSystemMapper();
 320
 321     if (IOMapper::gSystem) {
 322         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 323         return (UInt32) ml_phys_read_word_64(addr);
 324     }
 325     else
 326         return (UInt32) ml_phys_read_word((vm_offset_t) address);
 327 }
 328
 329 UInt64 IOMappedRead64(IOPhysicalAddress address)
 330 {
 331     IOMapper::checkForSystemMapper();
 332
 333     if (IOMapper::gSystem) {
 334         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 335         return (UInt64) ml_phys_read_double_64(addr);
 336     }
 337     else
 338         return (UInt64) ml_phys_read_double((vm_offset_t) address);
 339 }
 340
 341 void IOMappedWrite8(IOPhysicalAddress address, UInt8 value)
 342 {
 343     IOMapper::checkForSystemMapper();
 344
 345     if (IOMapper::gSystem) {
 346         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 347         ml_phys_write_byte_64(addr, value);
 348     }
 349     else
 350         ml_phys_write_byte((vm_offset_t) address, value);
 351 }
 352
 353 void IOMappedWrite16(IOPhysicalAddress address, UInt16 value)
 354 {
 355     IOMapper::checkForSystemMapper();
 356
 357     if (IOMapper::gSystem) {
 358         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 359         ml_phys_write_half_64(addr, value);
 360     }
 361     else
 362         ml_phys_write_half((vm_offset_t) address, value);
 363 }
 364
 365 void IOMappedWrite32(IOPhysicalAddress address, UInt32 value)
 366 {
 367     IOMapper::checkForSystemMapper();
 368
 369     if (IOMapper::gSystem) {
 370         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 371         ml_phys_write_word_64(addr, value);
 372     }
 373     else
 374         ml_phys_write_word((vm_offset_t) address, value);
 375 }
 376
 377 void IOMappedWrite64(IOPhysicalAddress address, UInt64 value)
 378 {
 379     IOMapper::checkForSystemMapper();
 380
 381     if (IOMapper::gSystem) {
 382         addr64_t addr = IOMapper::gSystem->mapAddr(address);
 383         ml_phys_write_double_64(addr, value);
 384     }
 385     else
 386         ml_phys_write_double((vm_offset_t) address, value);
 387 }
 388
 389 __END_DECLS